//===-- RISCVTargetTransformInfo.cpp - RISC-V specific TTI ----------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "RISCVTargetTransformInfo.h" #include "MCTargetDesc/RISCVMatInt.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/BasicTTIImpl.h" #include "llvm/CodeGen/CostTable.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/IR/Instructions.h" #include #include using namespace llvm; #define DEBUG_TYPE "riscvtti" static cl::opt RVVRegisterWidthLMUL( "riscv-v-register-bit-width-lmul", cl::desc( "The LMUL to use for getRegisterBitWidth queries. Affects LMUL used " "by autovectorized code. Fractional LMULs are not supported."), cl::init(2), cl::Hidden); static cl::opt SLPMaxVF( "riscv-v-slp-max-vf", cl::desc( "Overrides result used for getMaximumVF query which is used " "exclusively by SLP vectorizer."), cl::Hidden); InstructionCost RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind) { assert(Ty->isIntegerTy() && "getIntImmCost can only estimate cost of materialising integers"); // We have a Zero register, so 0 is always free. if (Imm == 0) return TTI::TCC_Free; // Otherwise, we check how many instructions it will take to materialise. const DataLayout &DL = getDataLayout(); return RISCVMatInt::getIntMatCost(Imm, DL.getTypeSizeInBits(Ty), *getST()); } // Look for patterns of shift followed by AND that can be turned into a pair of // shifts. We won't need to materialize an immediate for the AND so these can // be considered free. static bool canUseShiftPair(Instruction *Inst, const APInt &Imm) { uint64_t Mask = Imm.getZExtValue(); auto *BO = dyn_cast(Inst->getOperand(0)); if (!BO || !BO->hasOneUse()) return false; if (BO->getOpcode() != Instruction::Shl) return false; if (!isa(BO->getOperand(1))) return false; unsigned ShAmt = cast(BO->getOperand(1))->getZExtValue(); // (and (shl x, c2), c1) will be matched to (srli (slli x, c2+c3), c3) if c1 // is a mask shifted by c2 bits with c3 leading zeros. if (isShiftedMask_64(Mask)) { unsigned Trailing = llvm::countr_zero(Mask); if (ShAmt == Trailing) return true; } return false; } InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind, Instruction *Inst) { assert(Ty->isIntegerTy() && "getIntImmCost can only estimate cost of materialising integers"); // We have a Zero register, so 0 is always free. if (Imm == 0) return TTI::TCC_Free; // Some instructions in RISC-V can take a 12-bit immediate. Some of these are // commutative, in others the immediate comes from a specific argument index. bool Takes12BitImm = false; unsigned ImmArgIdx = ~0U; switch (Opcode) { case Instruction::GetElementPtr: // Never hoist any arguments to a GetElementPtr. CodeGenPrepare will // split up large offsets in GEP into better parts than ConstantHoisting // can. return TTI::TCC_Free; case Instruction::And: // zext.h if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb()) return TTI::TCC_Free; // zext.w if (Imm == UINT64_C(0xffffffff) && ST->hasStdExtZba()) return TTI::TCC_Free; // bclri if (ST->hasStdExtZbs() && (~Imm).isPowerOf2()) return TTI::TCC_Free; if (Inst && Idx == 1 && Imm.getBitWidth() <= ST->getXLen() && canUseShiftPair(Inst, Imm)) return TTI::TCC_Free; Takes12BitImm = true; break; case Instruction::Add: Takes12BitImm = true; break; case Instruction::Or: case Instruction::Xor: // bseti/binvi if (ST->hasStdExtZbs() && Imm.isPowerOf2()) return TTI::TCC_Free; Takes12BitImm = true; break; case Instruction::Mul: // Power of 2 is a shift. Negated power of 2 is a shift and a negate. if (Imm.isPowerOf2() || Imm.isNegatedPowerOf2()) return TTI::TCC_Free; // One more or less than a power of 2 can use SLLI+ADD/SUB. if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2()) return TTI::TCC_Free; // FIXME: There is no MULI instruction. Takes12BitImm = true; break; case Instruction::Sub: case Instruction::Shl: case Instruction::LShr: case Instruction::AShr: Takes12BitImm = true; ImmArgIdx = 1; break; default: break; } if (Takes12BitImm) { // Check immediate is the correct argument... if (Instruction::isCommutative(Opcode) || Idx == ImmArgIdx) { // ... and fits into the 12-bit immediate. if (Imm.getSignificantBits() <= 64 && getTLI()->isLegalAddImmediate(Imm.getSExtValue())) { return TTI::TCC_Free; } } // Otherwise, use the full materialisation cost. return getIntImmCost(Imm, Ty, CostKind); } // By default, prevent hoisting. return TTI::TCC_Free; } InstructionCost RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind) { // Prevent hoisting in unknown cases. return TTI::TCC_Free; } TargetTransformInfo::PopcntSupportKind RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) { assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2"); return ST->hasStdExtZbb() || ST->hasVendorXCVbitmanip() ? TTI::PSK_FastHardware : TTI::PSK_Software; } bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const { // Currently, the ExpandReductions pass can't expand scalable-vector // reductions, but we still request expansion as RVV doesn't support certain // reductions and the SelectionDAG can't legalize them either. switch (II->getIntrinsicID()) { default: return false; // These reductions have no equivalent in RVV case Intrinsic::vector_reduce_mul: case Intrinsic::vector_reduce_fmul: return true; } } std::optional RISCVTTIImpl::getMaxVScale() const { if (ST->hasVInstructions()) return ST->getRealMaxVLen() / RISCV::RVVBitsPerBlock; return BaseT::getMaxVScale(); } std::optional RISCVTTIImpl::getVScaleForTuning() const { if (ST->hasVInstructions()) if (unsigned MinVLen = ST->getRealMinVLen(); MinVLen >= RISCV::RVVBitsPerBlock) return MinVLen / RISCV::RVVBitsPerBlock; return BaseT::getVScaleForTuning(); } TypeSize RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const { unsigned LMUL = llvm::bit_floor(std::clamp(RVVRegisterWidthLMUL, 1, 8)); switch (K) { case TargetTransformInfo::RGK_Scalar: return TypeSize::getFixed(ST->getXLen()); case TargetTransformInfo::RGK_FixedWidthVector: return TypeSize::getFixed( ST->useRVVForFixedLengthVectors() ? LMUL * ST->getRealMinVLen() : 0); case TargetTransformInfo::RGK_ScalableVector: return TypeSize::getScalable( (ST->hasVInstructions() && ST->getRealMinVLen() >= RISCV::RVVBitsPerBlock) ? LMUL * RISCV::RVVBitsPerBlock : 0); } llvm_unreachable("Unsupported register kind"); } InstructionCost RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty, TTI::TargetCostKind CostKind) { // Add a cost of address generation + the cost of the load. The address // is expected to be a PC relative offset to a constant pool entry // using auipc/addi. return 2 + getMemoryOpCost(Instruction::Load, Ty, DL.getABITypeAlign(Ty), /*AddressSpace=*/0, CostKind); } static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST, LLVMContext &C) { assert((DataVT.getScalarSizeInBits() != 8 || DataVT.getVectorNumElements() <= 256) && "unhandled case in lowering"); MVT IndexVT = DataVT.changeTypeToInteger(); if (IndexVT.getScalarType().bitsGT(ST.getXLenVT())) IndexVT = IndexVT.changeVectorElementType(MVT::i16); return cast(EVT(IndexVT).getTypeForEVT(C)); } InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef Mask, TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, ArrayRef Args) { Kind = improveShuffleKindFromMask(Kind, Mask, Tp, Index, SubTp); std::pair LT = getTypeLegalizationCost(Tp); // First, handle cases where having a fixed length vector enables us to // give a more accurate cost than falling back to generic scalable codegen. // TODO: Each of these cases hints at a modeling gap around scalable vectors. if (isa(Tp)) { switch (Kind) { default: break; case TTI::SK_PermuteSingleSrc: { if (Mask.size() >= 2 && LT.second.isFixedLengthVector()) { MVT EltTp = LT.second.getVectorElementType(); // If the size of the element is < ELEN then shuffles of interleaves and // deinterleaves of 2 vectors can be lowered into the following // sequences if (EltTp.getScalarSizeInBits() < ST->getELen()) { // Example sequence: // vsetivli zero, 4, e8, mf4, ta, ma (ignored) // vwaddu.vv v10, v8, v9 // li a0, -1 (ignored) // vwmaccu.vx v10, a0, v9 if (ShuffleVectorInst::isInterleaveMask(Mask, 2, Mask.size())) return 2 * LT.first * TLI->getLMULCost(LT.second); if (Mask[0] == 0 || Mask[0] == 1) { auto DeinterleaveMask = createStrideMask(Mask[0], 2, Mask.size()); // Example sequence: // vnsrl.wi v10, v8, 0 if (equal(DeinterleaveMask, Mask)) return LT.first * TLI->getLMULCost(LT.second); } } } // vrgather + cost of generating the mask constant. // We model this for an unknown mask with a single vrgather. if (LT.second.isFixedLengthVector() && LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 || LT.second.getVectorNumElements() <= 256)) { VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, Tp->getContext()); InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind); return IndexCost + TLI->getVRGatherVVCost(LT.second); } [[fallthrough]]; } case TTI::SK_Transpose: case TTI::SK_PermuteTwoSrc: { // 2 x (vrgather + cost of generating the mask constant) + cost of mask // register for the second vrgather. We model this for an unknown // (shuffle) mask. if (LT.second.isFixedLengthVector() && LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 || LT.second.getVectorNumElements() <= 256)) { auto &C = Tp->getContext(); auto EC = Tp->getElementCount(); VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, C); VectorType *MaskTy = VectorType::get(IntegerType::getInt1Ty(C), EC); InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind); InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind); return 2 * IndexCost + 2 * TLI->getVRGatherVVCost(LT.second) + MaskCost; } [[fallthrough]]; } case TTI::SK_Select: { // We are going to permute multiple sources and the result will be in // multiple destinations. Providing an accurate cost only for splits where // the element type remains the same. if (!Mask.empty() && LT.first.isValid() && LT.first != 1 && LT.second.isFixedLengthVector() && LT.second.getVectorElementType().getSizeInBits() == Tp->getElementType()->getPrimitiveSizeInBits() && LT.second.getVectorNumElements() < cast(Tp)->getNumElements() && divideCeil(Mask.size(), cast(Tp)->getNumElements()) == static_cast(*LT.first.getValue())) { unsigned NumRegs = *LT.first.getValue(); unsigned VF = cast(Tp)->getNumElements(); unsigned SubVF = PowerOf2Ceil(VF / NumRegs); auto *SubVecTy = FixedVectorType::get(Tp->getElementType(), SubVF); InstructionCost Cost = 0; for (unsigned I = 0; I < NumRegs; ++I) { bool IsSingleVector = true; SmallVector SubMask(SubVF, PoisonMaskElem); transform(Mask.slice(I * SubVF, I == NumRegs - 1 ? Mask.size() % SubVF : SubVF), SubMask.begin(), [&](int I) { bool SingleSubVector = I / VF == 0; IsSingleVector &= SingleSubVector; return (SingleSubVector ? 0 : 1) * SubVF + I % VF; }); Cost += getShuffleCost(IsSingleVector ? TTI::SK_PermuteSingleSrc : TTI::SK_PermuteTwoSrc, SubVecTy, SubMask, CostKind, 0, nullptr); return Cost; } } break; } } }; // Handle scalable vectors (and fixed vectors legalized to scalable vectors). switch (Kind) { default: // Fallthrough to generic handling. // TODO: Most of these cases will return getInvalid in generic code, and // must be implemented here. break; case TTI::SK_ExtractSubvector: // Example sequence: // vsetivli zero, 4, e8, mf2, tu, ma (ignored) // vslidedown.vi v8, v9, 2 return LT.first * TLI->getVSlideCost(LT.second); case TTI::SK_InsertSubvector: // Example sequence: // vsetivli zero, 4, e8, mf2, tu, ma (ignored) // vslideup.vi v8, v9, 2 return LT.first * TLI->getVSlideCost(LT.second); case TTI::SK_Select: { // Example sequence: // li a0, 90 // vsetivli zero, 8, e8, mf2, ta, ma (ignored) // vmv.s.x v0, a0 // vmerge.vvm v8, v9, v8, v0 return LT.first * 3 * TLI->getLMULCost(LT.second); } case TTI::SK_Broadcast: { bool HasScalar = (Args.size() > 0) && (Operator::getOpcode(Args[0]) == Instruction::InsertElement); if (LT.second.getScalarSizeInBits() == 1) { if (HasScalar) { // Example sequence: // andi a0, a0, 1 // vsetivli zero, 2, e8, mf8, ta, ma (ignored) // vmv.v.x v8, a0 // vmsne.vi v0, v8, 0 return LT.first * TLI->getLMULCost(LT.second) * 3; } // Example sequence: // vsetivli zero, 2, e8, mf8, ta, mu (ignored) // vmv.v.i v8, 0 // vmerge.vim v8, v8, 1, v0 // vmv.x.s a0, v8 // andi a0, a0, 1 // vmv.v.x v8, a0 // vmsne.vi v0, v8, 0 return LT.first * TLI->getLMULCost(LT.second) * 6; } if (HasScalar) { // Example sequence: // vmv.v.x v8, a0 return LT.first * TLI->getLMULCost(LT.second); } // Example sequence: // vrgather.vi v9, v8, 0 return LT.first * TLI->getVRGatherVICost(LT.second); } case TTI::SK_Splice: // vslidedown+vslideup. // TODO: Multiplying by LT.first implies this legalizes into multiple copies // of similar code, but I think we expand through memory. return 2 * LT.first * TLI->getVSlideCost(LT.second); case TTI::SK_Reverse: { // TODO: Cases to improve here: // * Illegal vector types // * i64 on RV32 // * i1 vector // At low LMUL, most of the cost is producing the vrgather index register. // At high LMUL, the cost of the vrgather itself will dominate. // Example sequence: // csrr a0, vlenb // srli a0, a0, 3 // addi a0, a0, -1 // vsetvli a1, zero, e8, mf8, ta, mu (ignored) // vid.v v9 // vrsub.vx v10, v9, a0 // vrgather.vv v9, v8, v10 InstructionCost LenCost = 3; if (LT.second.isFixedLengthVector()) // vrsub.vi has a 5 bit immediate field, otherwise an li suffices LenCost = isInt<5>(LT.second.getVectorNumElements() - 1) ? 0 : 1; InstructionCost GatherCost = 2 + TLI->getVRGatherVVCost(LT.second); // Mask operation additionally required extend and truncate InstructionCost ExtendCost = Tp->getElementType()->isIntegerTy(1) ? 3 : 0; return LT.first * (LenCost + GatherCost + ExtendCost); } } return BaseT::getShuffleCost(Kind, Tp, Mask, CostKind, Index, SubTp); } InstructionCost RISCVTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind) { if (!isLegalMaskedLoadStore(Src, Alignment) || CostKind != TTI::TCK_RecipThroughput) return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); } InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond, bool UseMaskForGaps) { if (isa(VecTy)) return InstructionCost::getInvalid(); auto *FVTy = cast(VecTy); InstructionCost MemCost = getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind); unsigned VF = FVTy->getNumElements() / Factor; // The interleaved memory access pass will lower interleaved memory ops (i.e // a load and store followed by a specific shuffle) to vlseg/vsseg // intrinsics. In those cases then we can treat it as if it's just one (legal) // memory op if (!UseMaskForCond && !UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) { std::pair LT = getTypeLegalizationCost(FVTy); // Need to make sure type has't been scalarized if (LT.second.isFixedLengthVector()) { auto *LegalFVTy = FixedVectorType::get(FVTy->getElementType(), LT.second.getVectorNumElements()); // FIXME: We use the memory op cost of the *legalized* type here, becuase // it's getMemoryOpCost returns a really expensive cost for types like // <6 x i8>, which show up when doing interleaves of Factor=3 etc. // Should the memory op cost of these be cheaper? if (TLI->isLegalInterleavedAccessType(LegalFVTy, Factor, Alignment, AddressSpace, DL)) { InstructionCost LegalMemCost = getMemoryOpCost( Opcode, LegalFVTy, Alignment, AddressSpace, CostKind); return LT.first + LegalMemCost; } } } // An interleaved load will look like this for Factor=3: // %wide.vec = load <12 x i32>, ptr %3, align 4 // %strided.vec = shufflevector %wide.vec, poison, <4 x i32> // %strided.vec1 = shufflevector %wide.vec, poison, <4 x i32> // %strided.vec2 = shufflevector %wide.vec, poison, <4 x i32> if (Opcode == Instruction::Load) { InstructionCost Cost = MemCost; for (unsigned Index : Indices) { FixedVectorType *SubVecTy = FixedVectorType::get(FVTy->getElementType(), VF * Factor); auto Mask = createStrideMask(Index, Factor, VF); InstructionCost ShuffleCost = getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, SubVecTy, Mask, CostKind, 0, nullptr, {}); Cost += ShuffleCost; } return Cost; } // TODO: Model for NF > 2 // We'll need to enhance getShuffleCost to model shuffles that are just // inserts and extracts into subvectors, since they won't have the full cost // of a vrgather. // An interleaved store for 3 vectors of 4 lanes will look like // %11 = shufflevector <4 x i32> %4, <4 x i32> %6, <8 x i32> <0...7> // %12 = shufflevector <4 x i32> %9, <4 x i32> poison, <8 x i32> <0...3> // %13 = shufflevector <8 x i32> %11, <8 x i32> %12, <12 x i32> <0...11> // %interleaved.vec = shufflevector %13, poison, <12 x i32> // store <12 x i32> %interleaved.vec, ptr %10, align 4 if (Factor != 2) return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind, UseMaskForCond, UseMaskForGaps); assert(Opcode == Instruction::Store && "Opcode must be a store"); // For an interleaving store of 2 vectors, we perform one large interleaving // shuffle that goes into the wide store auto Mask = createInterleaveMask(VF, Factor); InstructionCost ShuffleCost = getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, FVTy, Mask, CostKind, 0, nullptr, {}); return MemCost + ShuffleCost; } InstructionCost RISCVTTIImpl::getGatherScatterOpCost( unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) { if (CostKind != TTI::TCK_RecipThroughput) return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I); if ((Opcode == Instruction::Load && !isLegalMaskedGather(DataTy, Align(Alignment))) || (Opcode == Instruction::Store && !isLegalMaskedScatter(DataTy, Align(Alignment)))) return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I); // Cost is proportional to the number of memory operations implied. For // scalable vectors, we use an estimate on that number since we don't // know exactly what VL will be. auto &VTy = *cast(DataTy); InstructionCost MemOpCost = getMemoryOpCost(Opcode, VTy.getElementType(), Alignment, 0, CostKind, {TTI::OK_AnyValue, TTI::OP_None}, I); unsigned NumLoads = getEstimatedVLFor(&VTy); return NumLoads * MemOpCost; } // Currently, these represent both throughput and codesize costs // for the respective intrinsics. The costs in this table are simply // instruction counts with the following adjustments made: // * One vsetvli is considered free. static const CostTblEntry VectorIntrinsicCostTable[]{ {Intrinsic::floor, MVT::v2f32, 9}, {Intrinsic::floor, MVT::v4f32, 9}, {Intrinsic::floor, MVT::v8f32, 9}, {Intrinsic::floor, MVT::v16f32, 9}, {Intrinsic::floor, MVT::nxv1f32, 9}, {Intrinsic::floor, MVT::nxv2f32, 9}, {Intrinsic::floor, MVT::nxv4f32, 9}, {Intrinsic::floor, MVT::nxv8f32, 9}, {Intrinsic::floor, MVT::nxv16f32, 9}, {Intrinsic::floor, MVT::v2f64, 9}, {Intrinsic::floor, MVT::v4f64, 9}, {Intrinsic::floor, MVT::v8f64, 9}, {Intrinsic::floor, MVT::v16f64, 9}, {Intrinsic::floor, MVT::nxv1f64, 9}, {Intrinsic::floor, MVT::nxv2f64, 9}, {Intrinsic::floor, MVT::nxv4f64, 9}, {Intrinsic::floor, MVT::nxv8f64, 9}, {Intrinsic::ceil, MVT::v2f32, 9}, {Intrinsic::ceil, MVT::v4f32, 9}, {Intrinsic::ceil, MVT::v8f32, 9}, {Intrinsic::ceil, MVT::v16f32, 9}, {Intrinsic::ceil, MVT::nxv1f32, 9}, {Intrinsic::ceil, MVT::nxv2f32, 9}, {Intrinsic::ceil, MVT::nxv4f32, 9}, {Intrinsic::ceil, MVT::nxv8f32, 9}, {Intrinsic::ceil, MVT::nxv16f32, 9}, {Intrinsic::ceil, MVT::v2f64, 9}, {Intrinsic::ceil, MVT::v4f64, 9}, {Intrinsic::ceil, MVT::v8f64, 9}, {Intrinsic::ceil, MVT::v16f64, 9}, {Intrinsic::ceil, MVT::nxv1f64, 9}, {Intrinsic::ceil, MVT::nxv2f64, 9}, {Intrinsic::ceil, MVT::nxv4f64, 9}, {Intrinsic::ceil, MVT::nxv8f64, 9}, {Intrinsic::trunc, MVT::v2f32, 7}, {Intrinsic::trunc, MVT::v4f32, 7}, {Intrinsic::trunc, MVT::v8f32, 7}, {Intrinsic::trunc, MVT::v16f32, 7}, {Intrinsic::trunc, MVT::nxv1f32, 7}, {Intrinsic::trunc, MVT::nxv2f32, 7}, {Intrinsic::trunc, MVT::nxv4f32, 7}, {Intrinsic::trunc, MVT::nxv8f32, 7}, {Intrinsic::trunc, MVT::nxv16f32, 7}, {Intrinsic::trunc, MVT::v2f64, 7}, {Intrinsic::trunc, MVT::v4f64, 7}, {Intrinsic::trunc, MVT::v8f64, 7}, {Intrinsic::trunc, MVT::v16f64, 7}, {Intrinsic::trunc, MVT::nxv1f64, 7}, {Intrinsic::trunc, MVT::nxv2f64, 7}, {Intrinsic::trunc, MVT::nxv4f64, 7}, {Intrinsic::trunc, MVT::nxv8f64, 7}, {Intrinsic::round, MVT::v2f32, 9}, {Intrinsic::round, MVT::v4f32, 9}, {Intrinsic::round, MVT::v8f32, 9}, {Intrinsic::round, MVT::v16f32, 9}, {Intrinsic::round, MVT::nxv1f32, 9}, {Intrinsic::round, MVT::nxv2f32, 9}, {Intrinsic::round, MVT::nxv4f32, 9}, {Intrinsic::round, MVT::nxv8f32, 9}, {Intrinsic::round, MVT::nxv16f32, 9}, {Intrinsic::round, MVT::v2f64, 9}, {Intrinsic::round, MVT::v4f64, 9}, {Intrinsic::round, MVT::v8f64, 9}, {Intrinsic::round, MVT::v16f64, 9}, {Intrinsic::round, MVT::nxv1f64, 9}, {Intrinsic::round, MVT::nxv2f64, 9}, {Intrinsic::round, MVT::nxv4f64, 9}, {Intrinsic::round, MVT::nxv8f64, 9}, {Intrinsic::roundeven, MVT::v2f32, 9}, {Intrinsic::roundeven, MVT::v4f32, 9}, {Intrinsic::roundeven, MVT::v8f32, 9}, {Intrinsic::roundeven, MVT::v16f32, 9}, {Intrinsic::roundeven, MVT::nxv1f32, 9}, {Intrinsic::roundeven, MVT::nxv2f32, 9}, {Intrinsic::roundeven, MVT::nxv4f32, 9}, {Intrinsic::roundeven, MVT::nxv8f32, 9}, {Intrinsic::roundeven, MVT::nxv16f32, 9}, {Intrinsic::roundeven, MVT::v2f64, 9}, {Intrinsic::roundeven, MVT::v4f64, 9}, {Intrinsic::roundeven, MVT::v8f64, 9}, {Intrinsic::roundeven, MVT::v16f64, 9}, {Intrinsic::roundeven, MVT::nxv1f64, 9}, {Intrinsic::roundeven, MVT::nxv2f64, 9}, {Intrinsic::roundeven, MVT::nxv4f64, 9}, {Intrinsic::roundeven, MVT::nxv8f64, 9}, {Intrinsic::rint, MVT::v2f32, 7}, {Intrinsic::rint, MVT::v4f32, 7}, {Intrinsic::rint, MVT::v8f32, 7}, {Intrinsic::rint, MVT::v16f32, 7}, {Intrinsic::rint, MVT::nxv1f32, 7}, {Intrinsic::rint, MVT::nxv2f32, 7}, {Intrinsic::rint, MVT::nxv4f32, 7}, {Intrinsic::rint, MVT::nxv8f32, 7}, {Intrinsic::rint, MVT::nxv16f32, 7}, {Intrinsic::rint, MVT::v2f64, 7}, {Intrinsic::rint, MVT::v4f64, 7}, {Intrinsic::rint, MVT::v8f64, 7}, {Intrinsic::rint, MVT::v16f64, 7}, {Intrinsic::rint, MVT::nxv1f64, 7}, {Intrinsic::rint, MVT::nxv2f64, 7}, {Intrinsic::rint, MVT::nxv4f64, 7}, {Intrinsic::rint, MVT::nxv8f64, 7}, {Intrinsic::lrint, MVT::v2i32, 1}, {Intrinsic::lrint, MVT::v4i32, 1}, {Intrinsic::lrint, MVT::v8i32, 1}, {Intrinsic::lrint, MVT::v16i32, 1}, {Intrinsic::lrint, MVT::nxv1i32, 1}, {Intrinsic::lrint, MVT::nxv2i32, 1}, {Intrinsic::lrint, MVT::nxv4i32, 1}, {Intrinsic::lrint, MVT::nxv8i32, 1}, {Intrinsic::lrint, MVT::nxv16i32, 1}, {Intrinsic::lrint, MVT::v2i64, 1}, {Intrinsic::lrint, MVT::v4i64, 1}, {Intrinsic::lrint, MVT::v8i64, 1}, {Intrinsic::lrint, MVT::v16i64, 1}, {Intrinsic::lrint, MVT::nxv1i64, 1}, {Intrinsic::lrint, MVT::nxv2i64, 1}, {Intrinsic::lrint, MVT::nxv4i64, 1}, {Intrinsic::lrint, MVT::nxv8i64, 1}, {Intrinsic::llrint, MVT::v2i64, 1}, {Intrinsic::llrint, MVT::v4i64, 1}, {Intrinsic::llrint, MVT::v8i64, 1}, {Intrinsic::llrint, MVT::v16i64, 1}, {Intrinsic::llrint, MVT::nxv1i64, 1}, {Intrinsic::llrint, MVT::nxv2i64, 1}, {Intrinsic::llrint, MVT::nxv4i64, 1}, {Intrinsic::llrint, MVT::nxv8i64, 1}, {Intrinsic::nearbyint, MVT::v2f32, 9}, {Intrinsic::nearbyint, MVT::v4f32, 9}, {Intrinsic::nearbyint, MVT::v8f32, 9}, {Intrinsic::nearbyint, MVT::v16f32, 9}, {Intrinsic::nearbyint, MVT::nxv1f32, 9}, {Intrinsic::nearbyint, MVT::nxv2f32, 9}, {Intrinsic::nearbyint, MVT::nxv4f32, 9}, {Intrinsic::nearbyint, MVT::nxv8f32, 9}, {Intrinsic::nearbyint, MVT::nxv16f32, 9}, {Intrinsic::nearbyint, MVT::v2f64, 9}, {Intrinsic::nearbyint, MVT::v4f64, 9}, {Intrinsic::nearbyint, MVT::v8f64, 9}, {Intrinsic::nearbyint, MVT::v16f64, 9}, {Intrinsic::nearbyint, MVT::nxv1f64, 9}, {Intrinsic::nearbyint, MVT::nxv2f64, 9}, {Intrinsic::nearbyint, MVT::nxv4f64, 9}, {Intrinsic::nearbyint, MVT::nxv8f64, 9}, {Intrinsic::bswap, MVT::v2i16, 3}, {Intrinsic::bswap, MVT::v4i16, 3}, {Intrinsic::bswap, MVT::v8i16, 3}, {Intrinsic::bswap, MVT::v16i16, 3}, {Intrinsic::bswap, MVT::nxv1i16, 3}, {Intrinsic::bswap, MVT::nxv2i16, 3}, {Intrinsic::bswap, MVT::nxv4i16, 3}, {Intrinsic::bswap, MVT::nxv8i16, 3}, {Intrinsic::bswap, MVT::nxv16i16, 3}, {Intrinsic::bswap, MVT::v2i32, 12}, {Intrinsic::bswap, MVT::v4i32, 12}, {Intrinsic::bswap, MVT::v8i32, 12}, {Intrinsic::bswap, MVT::v16i32, 12}, {Intrinsic::bswap, MVT::nxv1i32, 12}, {Intrinsic::bswap, MVT::nxv2i32, 12}, {Intrinsic::bswap, MVT::nxv4i32, 12}, {Intrinsic::bswap, MVT::nxv8i32, 12}, {Intrinsic::bswap, MVT::nxv16i32, 12}, {Intrinsic::bswap, MVT::v2i64, 31}, {Intrinsic::bswap, MVT::v4i64, 31}, {Intrinsic::bswap, MVT::v8i64, 31}, {Intrinsic::bswap, MVT::v16i64, 31}, {Intrinsic::bswap, MVT::nxv1i64, 31}, {Intrinsic::bswap, MVT::nxv2i64, 31}, {Intrinsic::bswap, MVT::nxv4i64, 31}, {Intrinsic::bswap, MVT::nxv8i64, 31}, {Intrinsic::vp_bswap, MVT::v2i16, 3}, {Intrinsic::vp_bswap, MVT::v4i16, 3}, {Intrinsic::vp_bswap, MVT::v8i16, 3}, {Intrinsic::vp_bswap, MVT::v16i16, 3}, {Intrinsic::vp_bswap, MVT::nxv1i16, 3}, {Intrinsic::vp_bswap, MVT::nxv2i16, 3}, {Intrinsic::vp_bswap, MVT::nxv4i16, 3}, {Intrinsic::vp_bswap, MVT::nxv8i16, 3}, {Intrinsic::vp_bswap, MVT::nxv16i16, 3}, {Intrinsic::vp_bswap, MVT::v2i32, 12}, {Intrinsic::vp_bswap, MVT::v4i32, 12}, {Intrinsic::vp_bswap, MVT::v8i32, 12}, {Intrinsic::vp_bswap, MVT::v16i32, 12}, {Intrinsic::vp_bswap, MVT::nxv1i32, 12}, {Intrinsic::vp_bswap, MVT::nxv2i32, 12}, {Intrinsic::vp_bswap, MVT::nxv4i32, 12}, {Intrinsic::vp_bswap, MVT::nxv8i32, 12}, {Intrinsic::vp_bswap, MVT::nxv16i32, 12}, {Intrinsic::vp_bswap, MVT::v2i64, 31}, {Intrinsic::vp_bswap, MVT::v4i64, 31}, {Intrinsic::vp_bswap, MVT::v8i64, 31}, {Intrinsic::vp_bswap, MVT::v16i64, 31}, {Intrinsic::vp_bswap, MVT::nxv1i64, 31}, {Intrinsic::vp_bswap, MVT::nxv2i64, 31}, {Intrinsic::vp_bswap, MVT::nxv4i64, 31}, {Intrinsic::vp_bswap, MVT::nxv8i64, 31}, {Intrinsic::vp_fshl, MVT::v2i8, 7}, {Intrinsic::vp_fshl, MVT::v4i8, 7}, {Intrinsic::vp_fshl, MVT::v8i8, 7}, {Intrinsic::vp_fshl, MVT::v16i8, 7}, {Intrinsic::vp_fshl, MVT::nxv1i8, 7}, {Intrinsic::vp_fshl, MVT::nxv2i8, 7}, {Intrinsic::vp_fshl, MVT::nxv4i8, 7}, {Intrinsic::vp_fshl, MVT::nxv8i8, 7}, {Intrinsic::vp_fshl, MVT::nxv16i8, 7}, {Intrinsic::vp_fshl, MVT::nxv32i8, 7}, {Intrinsic::vp_fshl, MVT::nxv64i8, 7}, {Intrinsic::vp_fshl, MVT::v2i16, 7}, {Intrinsic::vp_fshl, MVT::v4i16, 7}, {Intrinsic::vp_fshl, MVT::v8i16, 7}, {Intrinsic::vp_fshl, MVT::v16i16, 7}, {Intrinsic::vp_fshl, MVT::nxv1i16, 7}, {Intrinsic::vp_fshl, MVT::nxv2i16, 7}, {Intrinsic::vp_fshl, MVT::nxv4i16, 7}, {Intrinsic::vp_fshl, MVT::nxv8i16, 7}, {Intrinsic::vp_fshl, MVT::nxv16i16, 7}, {Intrinsic::vp_fshl, MVT::nxv32i16, 7}, {Intrinsic::vp_fshl, MVT::v2i32, 7}, {Intrinsic::vp_fshl, MVT::v4i32, 7}, {Intrinsic::vp_fshl, MVT::v8i32, 7}, {Intrinsic::vp_fshl, MVT::v16i32, 7}, {Intrinsic::vp_fshl, MVT::nxv1i32, 7}, {Intrinsic::vp_fshl, MVT::nxv2i32, 7}, {Intrinsic::vp_fshl, MVT::nxv4i32, 7}, {Intrinsic::vp_fshl, MVT::nxv8i32, 7}, {Intrinsic::vp_fshl, MVT::nxv16i32, 7}, {Intrinsic::vp_fshl, MVT::v2i64, 7}, {Intrinsic::vp_fshl, MVT::v4i64, 7}, {Intrinsic::vp_fshl, MVT::v8i64, 7}, {Intrinsic::vp_fshl, MVT::v16i64, 7}, {Intrinsic::vp_fshl, MVT::nxv1i64, 7}, {Intrinsic::vp_fshl, MVT::nxv2i64, 7}, {Intrinsic::vp_fshl, MVT::nxv4i64, 7}, {Intrinsic::vp_fshl, MVT::nxv8i64, 7}, {Intrinsic::vp_fshr, MVT::v2i8, 7}, {Intrinsic::vp_fshr, MVT::v4i8, 7}, {Intrinsic::vp_fshr, MVT::v8i8, 7}, {Intrinsic::vp_fshr, MVT::v16i8, 7}, {Intrinsic::vp_fshr, MVT::nxv1i8, 7}, {Intrinsic::vp_fshr, MVT::nxv2i8, 7}, {Intrinsic::vp_fshr, MVT::nxv4i8, 7}, {Intrinsic::vp_fshr, MVT::nxv8i8, 7}, {Intrinsic::vp_fshr, MVT::nxv16i8, 7}, {Intrinsic::vp_fshr, MVT::nxv32i8, 7}, {Intrinsic::vp_fshr, MVT::nxv64i8, 7}, {Intrinsic::vp_fshr, MVT::v2i16, 7}, {Intrinsic::vp_fshr, MVT::v4i16, 7}, {Intrinsic::vp_fshr, MVT::v8i16, 7}, {Intrinsic::vp_fshr, MVT::v16i16, 7}, {Intrinsic::vp_fshr, MVT::nxv1i16, 7}, {Intrinsic::vp_fshr, MVT::nxv2i16, 7}, {Intrinsic::vp_fshr, MVT::nxv4i16, 7}, {Intrinsic::vp_fshr, MVT::nxv8i16, 7}, {Intrinsic::vp_fshr, MVT::nxv16i16, 7}, {Intrinsic::vp_fshr, MVT::nxv32i16, 7}, {Intrinsic::vp_fshr, MVT::v2i32, 7}, {Intrinsic::vp_fshr, MVT::v4i32, 7}, {Intrinsic::vp_fshr, MVT::v8i32, 7}, {Intrinsic::vp_fshr, MVT::v16i32, 7}, {Intrinsic::vp_fshr, MVT::nxv1i32, 7}, {Intrinsic::vp_fshr, MVT::nxv2i32, 7}, {Intrinsic::vp_fshr, MVT::nxv4i32, 7}, {Intrinsic::vp_fshr, MVT::nxv8i32, 7}, {Intrinsic::vp_fshr, MVT::nxv16i32, 7}, {Intrinsic::vp_fshr, MVT::v2i64, 7}, {Intrinsic::vp_fshr, MVT::v4i64, 7}, {Intrinsic::vp_fshr, MVT::v8i64, 7}, {Intrinsic::vp_fshr, MVT::v16i64, 7}, {Intrinsic::vp_fshr, MVT::nxv1i64, 7}, {Intrinsic::vp_fshr, MVT::nxv2i64, 7}, {Intrinsic::vp_fshr, MVT::nxv4i64, 7}, {Intrinsic::vp_fshr, MVT::nxv8i64, 7}, {Intrinsic::bitreverse, MVT::v2i8, 17}, {Intrinsic::bitreverse, MVT::v4i8, 17}, {Intrinsic::bitreverse, MVT::v8i8, 17}, {Intrinsic::bitreverse, MVT::v16i8, 17}, {Intrinsic::bitreverse, MVT::nxv1i8, 17}, {Intrinsic::bitreverse, MVT::nxv2i8, 17}, {Intrinsic::bitreverse, MVT::nxv4i8, 17}, {Intrinsic::bitreverse, MVT::nxv8i8, 17}, {Intrinsic::bitreverse, MVT::nxv16i8, 17}, {Intrinsic::bitreverse, MVT::v2i16, 24}, {Intrinsic::bitreverse, MVT::v4i16, 24}, {Intrinsic::bitreverse, MVT::v8i16, 24}, {Intrinsic::bitreverse, MVT::v16i16, 24}, {Intrinsic::bitreverse, MVT::nxv1i16, 24}, {Intrinsic::bitreverse, MVT::nxv2i16, 24}, {Intrinsic::bitreverse, MVT::nxv4i16, 24}, {Intrinsic::bitreverse, MVT::nxv8i16, 24}, {Intrinsic::bitreverse, MVT::nxv16i16, 24}, {Intrinsic::bitreverse, MVT::v2i32, 33}, {Intrinsic::bitreverse, MVT::v4i32, 33}, {Intrinsic::bitreverse, MVT::v8i32, 33}, {Intrinsic::bitreverse, MVT::v16i32, 33}, {Intrinsic::bitreverse, MVT::nxv1i32, 33}, {Intrinsic::bitreverse, MVT::nxv2i32, 33}, {Intrinsic::bitreverse, MVT::nxv4i32, 33}, {Intrinsic::bitreverse, MVT::nxv8i32, 33}, {Intrinsic::bitreverse, MVT::nxv16i32, 33}, {Intrinsic::bitreverse, MVT::v2i64, 52}, {Intrinsic::bitreverse, MVT::v4i64, 52}, {Intrinsic::bitreverse, MVT::v8i64, 52}, {Intrinsic::bitreverse, MVT::v16i64, 52}, {Intrinsic::bitreverse, MVT::nxv1i64, 52}, {Intrinsic::bitreverse, MVT::nxv2i64, 52}, {Intrinsic::bitreverse, MVT::nxv4i64, 52}, {Intrinsic::bitreverse, MVT::nxv8i64, 52}, {Intrinsic::vp_bitreverse, MVT::v2i8, 17}, {Intrinsic::vp_bitreverse, MVT::v4i8, 17}, {Intrinsic::vp_bitreverse, MVT::v8i8, 17}, {Intrinsic::vp_bitreverse, MVT::v16i8, 17}, {Intrinsic::vp_bitreverse, MVT::nxv1i8, 17}, {Intrinsic::vp_bitreverse, MVT::nxv2i8, 17}, {Intrinsic::vp_bitreverse, MVT::nxv4i8, 17}, {Intrinsic::vp_bitreverse, MVT::nxv8i8, 17}, {Intrinsic::vp_bitreverse, MVT::nxv16i8, 17}, {Intrinsic::vp_bitreverse, MVT::v2i16, 24}, {Intrinsic::vp_bitreverse, MVT::v4i16, 24}, {Intrinsic::vp_bitreverse, MVT::v8i16, 24}, {Intrinsic::vp_bitreverse, MVT::v16i16, 24}, {Intrinsic::vp_bitreverse, MVT::nxv1i16, 24}, {Intrinsic::vp_bitreverse, MVT::nxv2i16, 24}, {Intrinsic::vp_bitreverse, MVT::nxv4i16, 24}, {Intrinsic::vp_bitreverse, MVT::nxv8i16, 24}, {Intrinsic::vp_bitreverse, MVT::nxv16i16, 24}, {Intrinsic::vp_bitreverse, MVT::v2i32, 33}, {Intrinsic::vp_bitreverse, MVT::v4i32, 33}, {Intrinsic::vp_bitreverse, MVT::v8i32, 33}, {Intrinsic::vp_bitreverse, MVT::v16i32, 33}, {Intrinsic::vp_bitreverse, MVT::nxv1i32, 33}, {Intrinsic::vp_bitreverse, MVT::nxv2i32, 33}, {Intrinsic::vp_bitreverse, MVT::nxv4i32, 33}, {Intrinsic::vp_bitreverse, MVT::nxv8i32, 33}, {Intrinsic::vp_bitreverse, MVT::nxv16i32, 33}, {Intrinsic::vp_bitreverse, MVT::v2i64, 52}, {Intrinsic::vp_bitreverse, MVT::v4i64, 52}, {Intrinsic::vp_bitreverse, MVT::v8i64, 52}, {Intrinsic::vp_bitreverse, MVT::v16i64, 52}, {Intrinsic::vp_bitreverse, MVT::nxv1i64, 52}, {Intrinsic::vp_bitreverse, MVT::nxv2i64, 52}, {Intrinsic::vp_bitreverse, MVT::nxv4i64, 52}, {Intrinsic::vp_bitreverse, MVT::nxv8i64, 52}, {Intrinsic::ctpop, MVT::v2i8, 12}, {Intrinsic::ctpop, MVT::v4i8, 12}, {Intrinsic::ctpop, MVT::v8i8, 12}, {Intrinsic::ctpop, MVT::v16i8, 12}, {Intrinsic::ctpop, MVT::nxv1i8, 12}, {Intrinsic::ctpop, MVT::nxv2i8, 12}, {Intrinsic::ctpop, MVT::nxv4i8, 12}, {Intrinsic::ctpop, MVT::nxv8i8, 12}, {Intrinsic::ctpop, MVT::nxv16i8, 12}, {Intrinsic::ctpop, MVT::v2i16, 19}, {Intrinsic::ctpop, MVT::v4i16, 19}, {Intrinsic::ctpop, MVT::v8i16, 19}, {Intrinsic::ctpop, MVT::v16i16, 19}, {Intrinsic::ctpop, MVT::nxv1i16, 19}, {Intrinsic::ctpop, MVT::nxv2i16, 19}, {Intrinsic::ctpop, MVT::nxv4i16, 19}, {Intrinsic::ctpop, MVT::nxv8i16, 19}, {Intrinsic::ctpop, MVT::nxv16i16, 19}, {Intrinsic::ctpop, MVT::v2i32, 20}, {Intrinsic::ctpop, MVT::v4i32, 20}, {Intrinsic::ctpop, MVT::v8i32, 20}, {Intrinsic::ctpop, MVT::v16i32, 20}, {Intrinsic::ctpop, MVT::nxv1i32, 20}, {Intrinsic::ctpop, MVT::nxv2i32, 20}, {Intrinsic::ctpop, MVT::nxv4i32, 20}, {Intrinsic::ctpop, MVT::nxv8i32, 20}, {Intrinsic::ctpop, MVT::nxv16i32, 20}, {Intrinsic::ctpop, MVT::v2i64, 21}, {Intrinsic::ctpop, MVT::v4i64, 21}, {Intrinsic::ctpop, MVT::v8i64, 21}, {Intrinsic::ctpop, MVT::v16i64, 21}, {Intrinsic::ctpop, MVT::nxv1i64, 21}, {Intrinsic::ctpop, MVT::nxv2i64, 21}, {Intrinsic::ctpop, MVT::nxv4i64, 21}, {Intrinsic::ctpop, MVT::nxv8i64, 21}, {Intrinsic::vp_ctpop, MVT::v2i8, 12}, {Intrinsic::vp_ctpop, MVT::v4i8, 12}, {Intrinsic::vp_ctpop, MVT::v8i8, 12}, {Intrinsic::vp_ctpop, MVT::v16i8, 12}, {Intrinsic::vp_ctpop, MVT::nxv1i8, 12}, {Intrinsic::vp_ctpop, MVT::nxv2i8, 12}, {Intrinsic::vp_ctpop, MVT::nxv4i8, 12}, {Intrinsic::vp_ctpop, MVT::nxv8i8, 12}, {Intrinsic::vp_ctpop, MVT::nxv16i8, 12}, {Intrinsic::vp_ctpop, MVT::v2i16, 19}, {Intrinsic::vp_ctpop, MVT::v4i16, 19}, {Intrinsic::vp_ctpop, MVT::v8i16, 19}, {Intrinsic::vp_ctpop, MVT::v16i16, 19}, {Intrinsic::vp_ctpop, MVT::nxv1i16, 19}, {Intrinsic::vp_ctpop, MVT::nxv2i16, 19}, {Intrinsic::vp_ctpop, MVT::nxv4i16, 19}, {Intrinsic::vp_ctpop, MVT::nxv8i16, 19}, {Intrinsic::vp_ctpop, MVT::nxv16i16, 19}, {Intrinsic::vp_ctpop, MVT::v2i32, 20}, {Intrinsic::vp_ctpop, MVT::v4i32, 20}, {Intrinsic::vp_ctpop, MVT::v8i32, 20}, {Intrinsic::vp_ctpop, MVT::v16i32, 20}, {Intrinsic::vp_ctpop, MVT::nxv1i32, 20}, {Intrinsic::vp_ctpop, MVT::nxv2i32, 20}, {Intrinsic::vp_ctpop, MVT::nxv4i32, 20}, {Intrinsic::vp_ctpop, MVT::nxv8i32, 20}, {Intrinsic::vp_ctpop, MVT::nxv16i32, 20}, {Intrinsic::vp_ctpop, MVT::v2i64, 21}, {Intrinsic::vp_ctpop, MVT::v4i64, 21}, {Intrinsic::vp_ctpop, MVT::v8i64, 21}, {Intrinsic::vp_ctpop, MVT::v16i64, 21}, {Intrinsic::vp_ctpop, MVT::nxv1i64, 21}, {Intrinsic::vp_ctpop, MVT::nxv2i64, 21}, {Intrinsic::vp_ctpop, MVT::nxv4i64, 21}, {Intrinsic::vp_ctpop, MVT::nxv8i64, 21}, {Intrinsic::vp_ctlz, MVT::v2i8, 19}, {Intrinsic::vp_ctlz, MVT::v4i8, 19}, {Intrinsic::vp_ctlz, MVT::v8i8, 19}, {Intrinsic::vp_ctlz, MVT::v16i8, 19}, {Intrinsic::vp_ctlz, MVT::nxv1i8, 19}, {Intrinsic::vp_ctlz, MVT::nxv2i8, 19}, {Intrinsic::vp_ctlz, MVT::nxv4i8, 19}, {Intrinsic::vp_ctlz, MVT::nxv8i8, 19}, {Intrinsic::vp_ctlz, MVT::nxv16i8, 19}, {Intrinsic::vp_ctlz, MVT::nxv32i8, 19}, {Intrinsic::vp_ctlz, MVT::nxv64i8, 19}, {Intrinsic::vp_ctlz, MVT::v2i16, 28}, {Intrinsic::vp_ctlz, MVT::v4i16, 28}, {Intrinsic::vp_ctlz, MVT::v8i16, 28}, {Intrinsic::vp_ctlz, MVT::v16i16, 28}, {Intrinsic::vp_ctlz, MVT::nxv1i16, 28}, {Intrinsic::vp_ctlz, MVT::nxv2i16, 28}, {Intrinsic::vp_ctlz, MVT::nxv4i16, 28}, {Intrinsic::vp_ctlz, MVT::nxv8i16, 28}, {Intrinsic::vp_ctlz, MVT::nxv16i16, 28}, {Intrinsic::vp_ctlz, MVT::nxv32i16, 28}, {Intrinsic::vp_ctlz, MVT::v2i32, 31}, {Intrinsic::vp_ctlz, MVT::v4i32, 31}, {Intrinsic::vp_ctlz, MVT::v8i32, 31}, {Intrinsic::vp_ctlz, MVT::v16i32, 31}, {Intrinsic::vp_ctlz, MVT::nxv1i32, 31}, {Intrinsic::vp_ctlz, MVT::nxv2i32, 31}, {Intrinsic::vp_ctlz, MVT::nxv4i32, 31}, {Intrinsic::vp_ctlz, MVT::nxv8i32, 31}, {Intrinsic::vp_ctlz, MVT::nxv16i32, 31}, {Intrinsic::vp_ctlz, MVT::v2i64, 35}, {Intrinsic::vp_ctlz, MVT::v4i64, 35}, {Intrinsic::vp_ctlz, MVT::v8i64, 35}, {Intrinsic::vp_ctlz, MVT::v16i64, 35}, {Intrinsic::vp_ctlz, MVT::nxv1i64, 35}, {Intrinsic::vp_ctlz, MVT::nxv2i64, 35}, {Intrinsic::vp_ctlz, MVT::nxv4i64, 35}, {Intrinsic::vp_ctlz, MVT::nxv8i64, 35}, {Intrinsic::vp_cttz, MVT::v2i8, 16}, {Intrinsic::vp_cttz, MVT::v4i8, 16}, {Intrinsic::vp_cttz, MVT::v8i8, 16}, {Intrinsic::vp_cttz, MVT::v16i8, 16}, {Intrinsic::vp_cttz, MVT::nxv1i8, 16}, {Intrinsic::vp_cttz, MVT::nxv2i8, 16}, {Intrinsic::vp_cttz, MVT::nxv4i8, 16}, {Intrinsic::vp_cttz, MVT::nxv8i8, 16}, {Intrinsic::vp_cttz, MVT::nxv16i8, 16}, {Intrinsic::vp_cttz, MVT::nxv32i8, 16}, {Intrinsic::vp_cttz, MVT::nxv64i8, 16}, {Intrinsic::vp_cttz, MVT::v2i16, 23}, {Intrinsic::vp_cttz, MVT::v4i16, 23}, {Intrinsic::vp_cttz, MVT::v8i16, 23}, {Intrinsic::vp_cttz, MVT::v16i16, 23}, {Intrinsic::vp_cttz, MVT::nxv1i16, 23}, {Intrinsic::vp_cttz, MVT::nxv2i16, 23}, {Intrinsic::vp_cttz, MVT::nxv4i16, 23}, {Intrinsic::vp_cttz, MVT::nxv8i16, 23}, {Intrinsic::vp_cttz, MVT::nxv16i16, 23}, {Intrinsic::vp_cttz, MVT::nxv32i16, 23}, {Intrinsic::vp_cttz, MVT::v2i32, 24}, {Intrinsic::vp_cttz, MVT::v4i32, 24}, {Intrinsic::vp_cttz, MVT::v8i32, 24}, {Intrinsic::vp_cttz, MVT::v16i32, 24}, {Intrinsic::vp_cttz, MVT::nxv1i32, 24}, {Intrinsic::vp_cttz, MVT::nxv2i32, 24}, {Intrinsic::vp_cttz, MVT::nxv4i32, 24}, {Intrinsic::vp_cttz, MVT::nxv8i32, 24}, {Intrinsic::vp_cttz, MVT::nxv16i32, 24}, {Intrinsic::vp_cttz, MVT::v2i64, 25}, {Intrinsic::vp_cttz, MVT::v4i64, 25}, {Intrinsic::vp_cttz, MVT::v8i64, 25}, {Intrinsic::vp_cttz, MVT::v16i64, 25}, {Intrinsic::vp_cttz, MVT::nxv1i64, 25}, {Intrinsic::vp_cttz, MVT::nxv2i64, 25}, {Intrinsic::vp_cttz, MVT::nxv4i64, 25}, {Intrinsic::vp_cttz, MVT::nxv8i64, 25}, }; static unsigned getISDForVPIntrinsicID(Intrinsic::ID ID) { switch (ID) { #define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \ case Intrinsic::VPID: \ return ISD::VPSD; #include "llvm/IR/VPIntrinsics.def" #undef HELPER_MAP_VPID_TO_VPSD } return ISD::DELETED_NODE; } InstructionCost RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) { auto *RetTy = ICA.getReturnType(); switch (ICA.getID()) { case Intrinsic::ceil: case Intrinsic::floor: case Intrinsic::trunc: case Intrinsic::rint: case Intrinsic::lrint: case Intrinsic::llrint: case Intrinsic::round: case Intrinsic::roundeven: { // These all use the same code. auto LT = getTypeLegalizationCost(RetTy); if (!LT.second.isVector() && TLI->isOperationCustom(ISD::FCEIL, LT.second)) return LT.first * 8; break; } case Intrinsic::umin: case Intrinsic::umax: case Intrinsic::smin: case Intrinsic::smax: { auto LT = getTypeLegalizationCost(RetTy); if ((ST->hasVInstructions() && LT.second.isVector()) || (LT.second.isScalarInteger() && ST->hasStdExtZbb())) return LT.first; break; } case Intrinsic::sadd_sat: case Intrinsic::ssub_sat: case Intrinsic::uadd_sat: case Intrinsic::usub_sat: case Intrinsic::fabs: case Intrinsic::sqrt: { auto LT = getTypeLegalizationCost(RetTy); if (ST->hasVInstructions() && LT.second.isVector()) return LT.first; break; } case Intrinsic::ctpop: { auto LT = getTypeLegalizationCost(RetTy); if (ST->hasVInstructions() && ST->hasStdExtZvbb() && LT.second.isVector()) return LT.first; break; } case Intrinsic::abs: { auto LT = getTypeLegalizationCost(RetTy); if (ST->hasVInstructions() && LT.second.isVector()) { // vrsub.vi v10, v8, 0 // vmax.vv v8, v8, v10 return LT.first * 2; } break; } // TODO: add more intrinsic case Intrinsic::experimental_stepvector: { unsigned Cost = 1; // vid auto LT = getTypeLegalizationCost(RetTy); return Cost + (LT.first - 1); } case Intrinsic::vp_rint: { // RISC-V target uses at least 5 instructions to lower rounding intrinsics. unsigned Cost = 5; auto LT = getTypeLegalizationCost(RetTy); if (TLI->isOperationCustom(ISD::VP_FRINT, LT.second)) return Cost * LT.first; break; } case Intrinsic::vp_nearbyint: { // More one read and one write for fflags than vp_rint. unsigned Cost = 7; auto LT = getTypeLegalizationCost(RetTy); if (TLI->isOperationCustom(ISD::VP_FRINT, LT.second)) return Cost * LT.first; break; } case Intrinsic::vp_ceil: case Intrinsic::vp_floor: case Intrinsic::vp_round: case Intrinsic::vp_roundeven: case Intrinsic::vp_roundtozero: { // Rounding with static rounding mode needs two more instructions to // swap/write FRM than vp_rint. unsigned Cost = 7; auto LT = getTypeLegalizationCost(RetTy); unsigned VPISD = getISDForVPIntrinsicID(ICA.getID()); if (TLI->isOperationCustom(VPISD, LT.second)) return Cost * LT.first; break; } } if (ST->hasVInstructions() && RetTy->isVectorTy()) { auto LT = getTypeLegalizationCost(RetTy); if (const auto *Entry = CostTableLookup(VectorIntrinsicCostTable, ICA.getID(), LT.second)) return LT.first * Entry->Cost; } return BaseT::getIntrinsicInstrCost(ICA, CostKind); } InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, const Instruction *I) { if (isa(Dst) && isa(Src)) { // FIXME: Need to compute legalizing cost for illegal types. if (!isTypeLegal(Src) || !isTypeLegal(Dst)) return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); // Skip if element size of Dst or Src is bigger than ELEN. if (Src->getScalarSizeInBits() > ST->getELen() || Dst->getScalarSizeInBits() > ST->getELen()) return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); int ISD = TLI->InstructionOpcodeToISD(Opcode); assert(ISD && "Invalid opcode"); // FIXME: Need to consider vsetvli and lmul. int PowDiff = (int)Log2_32(Dst->getScalarSizeInBits()) - (int)Log2_32(Src->getScalarSizeInBits()); switch (ISD) { case ISD::SIGN_EXTEND: case ISD::ZERO_EXTEND: if (Src->getScalarSizeInBits() == 1) { // We do not use vsext/vzext to extend from mask vector. // Instead we use the following instructions to extend from mask vector: // vmv.v.i v8, 0 // vmerge.vim v8, v8, -1, v0 return 2; } return 1; case ISD::TRUNCATE: if (Dst->getScalarSizeInBits() == 1) { // We do not use several vncvt to truncate to mask vector. So we could // not use PowDiff to calculate it. // Instead we use the following instructions to truncate to mask vector: // vand.vi v8, v8, 1 // vmsne.vi v0, v8, 0 return 2; } [[fallthrough]]; case ISD::FP_EXTEND: case ISD::FP_ROUND: // Counts of narrow/widen instructions. return std::abs(PowDiff); case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: if (Src->getScalarSizeInBits() == 1 || Dst->getScalarSizeInBits() == 1) { // The cost of convert from or to mask vector is different from other // cases. We could not use PowDiff to calculate it. // For mask vector to fp, we should use the following instructions: // vmv.v.i v8, 0 // vmerge.vim v8, v8, -1, v0 // vfcvt.f.x.v v8, v8 // And for fp vector to mask, we use: // vfncvt.rtz.x.f.w v9, v8 // vand.vi v8, v9, 1 // vmsne.vi v0, v8, 0 return 3; } if (std::abs(PowDiff) <= 1) return 1; // Backend could lower (v[sz]ext i8 to double) to vfcvt(v[sz]ext.f8 i8), // so it only need two conversion. if (Src->isIntOrIntVectorTy()) return 2; // Counts of narrow/widen instructions. return std::abs(PowDiff); } } return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); } unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) { if (isa(Ty)) { const unsigned EltSize = DL.getTypeSizeInBits(Ty->getElementType()); const unsigned MinSize = DL.getTypeSizeInBits(Ty).getKnownMinValue(); const unsigned VectorBits = *getVScaleForTuning() * RISCV::RVVBitsPerBlock; return RISCVTargetLowering::computeVLMAX(VectorBits, EltSize, MinSize); } return cast(Ty)->getNumElements(); } InstructionCost RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, TTI::TargetCostKind CostKind) { if (isa(Ty) && !ST->useRVVForFixedLengthVectors()) return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind); // Skip if scalar size of Ty is bigger than ELEN. if (Ty->getScalarSizeInBits() > ST->getELen()) return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind); std::pair LT = getTypeLegalizationCost(Ty); if (Ty->getElementType()->isIntegerTy(1)) // vcpop sequences, see vreduction-mask.ll. umax, smin actually only // cost 2, but we don't have enough info here so we slightly over cost. return (LT.first - 1) + 3; // IR Reduction is composed by two vmv and one rvv reduction instruction. InstructionCost BaseCost = 2; if (CostKind == TTI::TCK_CodeSize) return (LT.first - 1) + BaseCost; unsigned VL = getEstimatedVLFor(Ty); return (LT.first - 1) + BaseCost + Log2_32_Ceil(VL); } InstructionCost RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional FMF, TTI::TargetCostKind CostKind) { if (isa(Ty) && !ST->useRVVForFixedLengthVectors()) return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind); // Skip if scalar size of Ty is bigger than ELEN. if (Ty->getScalarSizeInBits() > ST->getELen()) return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind); int ISD = TLI->InstructionOpcodeToISD(Opcode); assert(ISD && "Invalid opcode"); if (ISD != ISD::ADD && ISD != ISD::OR && ISD != ISD::XOR && ISD != ISD::AND && ISD != ISD::FADD) return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind); std::pair LT = getTypeLegalizationCost(Ty); if (Ty->getElementType()->isIntegerTy(1)) // vcpop sequences, see vreduction-mask.ll return (LT.first - 1) + (ISD == ISD::AND ? 3 : 2); // IR Reduction is composed by two vmv and one rvv reduction instruction. InstructionCost BaseCost = 2; if (CostKind == TTI::TCK_CodeSize) return (LT.first - 1) + BaseCost; unsigned VL = getEstimatedVLFor(Ty); if (TTI::requiresOrderedReduction(FMF)) return (LT.first - 1) + BaseCost + VL; return (LT.first - 1) + BaseCost + Log2_32_Ceil(VL); } InstructionCost RISCVTTIImpl::getExtendedReductionCost( unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy, FastMathFlags FMF, TTI::TargetCostKind CostKind) { if (isa(ValTy) && !ST->useRVVForFixedLengthVectors()) return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy, FMF, CostKind); // Skip if scalar size of ResTy is bigger than ELEN. if (ResTy->getScalarSizeInBits() > ST->getELen()) return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy, FMF, CostKind); if (Opcode != Instruction::Add && Opcode != Instruction::FAdd) return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy, FMF, CostKind); std::pair LT = getTypeLegalizationCost(ValTy); if (ResTy->getScalarSizeInBits() != 2 * LT.second.getScalarSizeInBits()) return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy, FMF, CostKind); return (LT.first - 1) + getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind); } InstructionCost RISCVTTIImpl::getStoreImmCost(Type *Ty, TTI::OperandValueInfo OpInfo, TTI::TargetCostKind CostKind) { assert(OpInfo.isConstant() && "non constant operand?"); if (!isa(Ty)) // FIXME: We need to account for immediate materialization here, but doing // a decent job requires more knowledge about the immediate than we // currently have here. return 0; if (OpInfo.isUniform()) // vmv.x.i, vmv.v.x, or vfmv.v.f // We ignore the cost of the scalar constant materialization to be consistent // with how we treat scalar constants themselves just above. return 1; return getConstantPoolLoadCost(Ty, CostKind); } InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo, const Instruction *I) { EVT VT = TLI->getValueType(DL, Src, true); // Type legalization can't handle structs if (VT == MVT::Other) return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I); InstructionCost Cost = 0; if (Opcode == Instruction::Store && OpInfo.isConstant()) Cost += getStoreImmCost(Src, OpInfo, CostKind); InstructionCost BaseCost = BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I); // Assume memory ops cost scale with the number of vector registers // possible accessed by the instruction. Note that BasicTTI already // handles the LT.first term for us. if (std::pair LT = getTypeLegalizationCost(Src); LT.second.isVector()) BaseCost *= TLI->getLMULCost(LT.second); return Cost + BaseCost; } InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, const Instruction *I) { if (CostKind != TTI::TCK_RecipThroughput) return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); if (isa(ValTy) && !ST->useRVVForFixedLengthVectors()) return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); // Skip if scalar size of ValTy is bigger than ELEN. if (ValTy->isVectorTy() && ValTy->getScalarSizeInBits() > ST->getELen()) return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); if (Opcode == Instruction::Select && ValTy->isVectorTy()) { std::pair LT = getTypeLegalizationCost(ValTy); if (CondTy->isVectorTy()) { if (ValTy->getScalarSizeInBits() == 1) { // vmandn.mm v8, v8, v9 // vmand.mm v9, v0, v9 // vmor.mm v0, v9, v8 return LT.first * 3; } // vselect and max/min are supported natively. return LT.first * 1; } if (ValTy->getScalarSizeInBits() == 1) { // vmv.v.x v9, a0 // vmsne.vi v9, v9, 0 // vmandn.mm v8, v8, v9 // vmand.mm v9, v0, v9 // vmor.mm v0, v9, v8 return LT.first * 5; } // vmv.v.x v10, a0 // vmsne.vi v0, v10, 0 // vmerge.vvm v8, v9, v8, v0 return LT.first * 3; } if ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) && ValTy->isVectorTy()) { std::pair LT = getTypeLegalizationCost(ValTy); // Support natively. if (CmpInst::isIntPredicate(VecPred)) return LT.first * 1; // If we do not support the input floating point vector type, use the base // one which will calculate as: // ScalarizeCost + Num * Cost for fixed vector, // InvalidCost for scalable vector. if ((ValTy->getScalarSizeInBits() == 16 && !ST->hasVInstructionsF16()) || (ValTy->getScalarSizeInBits() == 32 && !ST->hasVInstructionsF32()) || (ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64())) return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); switch (VecPred) { // Support natively. case CmpInst::FCMP_OEQ: case CmpInst::FCMP_OGT: case CmpInst::FCMP_OGE: case CmpInst::FCMP_OLT: case CmpInst::FCMP_OLE: case CmpInst::FCMP_UNE: return LT.first * 1; // TODO: Other comparisons? default: break; } } // TODO: Add cost for scalar type. return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); } InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) { if (CostKind != TTI::TCK_RecipThroughput) return Opcode == Instruction::PHI ? 0 : 1; // Branches are assumed to be predicted. return 0; } InstructionCost RISCVTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, Value *Op0, Value *Op1) { assert(Val->isVectorTy() && "This must be a vector type"); if (Opcode != Instruction::ExtractElement && Opcode != Instruction::InsertElement) return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1); // Legalize the type. std::pair LT = getTypeLegalizationCost(Val); // This type is legalized to a scalar type. if (!LT.second.isVector()) { auto *FixedVecTy = cast(Val); // If Index is a known constant, cost is zero. if (Index != -1U) return 0; // Extract/InsertElement with non-constant index is very costly when // scalarized; estimate cost of loads/stores sequence via the stack: // ExtractElement cost: store vector to stack, load scalar; // InsertElement cost: store vector to stack, store scalar, load vector. Type *ElemTy = FixedVecTy->getElementType(); auto NumElems = FixedVecTy->getNumElements(); auto Align = DL.getPrefTypeAlign(ElemTy); InstructionCost LoadCost = getMemoryOpCost(Instruction::Load, ElemTy, Align, 0, CostKind); InstructionCost StoreCost = getMemoryOpCost(Instruction::Store, ElemTy, Align, 0, CostKind); return Opcode == Instruction::ExtractElement ? StoreCost * NumElems + LoadCost : (StoreCost + LoadCost) * NumElems + StoreCost; } // For unsupported scalable vector. if (LT.second.isScalableVector() && !LT.first.isValid()) return LT.first; if (!isTypeLegal(Val)) return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1); // Mask vector extract/insert is expanded via e8. if (Val->getScalarSizeInBits() == 1) { VectorType *WideTy = VectorType::get(IntegerType::get(Val->getContext(), 8), cast(Val)->getElementCount()); if (Opcode == Instruction::ExtractElement) { InstructionCost ExtendCost = getCastInstrCost(Instruction::ZExt, WideTy, Val, TTI::CastContextHint::None, CostKind); InstructionCost ExtractCost = getVectorInstrCost(Opcode, WideTy, CostKind, Index, nullptr, nullptr); return ExtendCost + ExtractCost; } InstructionCost ExtendCost = getCastInstrCost(Instruction::ZExt, WideTy, Val, TTI::CastContextHint::None, CostKind); InstructionCost InsertCost = getVectorInstrCost(Opcode, WideTy, CostKind, Index, nullptr, nullptr); InstructionCost TruncCost = getCastInstrCost(Instruction::Trunc, Val, WideTy, TTI::CastContextHint::None, CostKind); return ExtendCost + InsertCost + TruncCost; } // In RVV, we could use vslidedown + vmv.x.s to extract element from vector // and vslideup + vmv.s.x to insert element to vector. unsigned BaseCost = 1; // When insertelement we should add the index with 1 as the input of vslideup. unsigned SlideCost = Opcode == Instruction::InsertElement ? 2 : 1; if (Index != -1U) { // The type may be split. For fixed-width vectors we can normalize the // index to the new type. if (LT.second.isFixedLengthVector()) { unsigned Width = LT.second.getVectorNumElements(); Index = Index % Width; } // We could extract/insert the first element without vslidedown/vslideup. if (Index == 0) SlideCost = 0; else if (Opcode == Instruction::InsertElement) SlideCost = 1; // With a constant index, we do not need to use addi. } // Extract i64 in the target that has XLEN=32 need more instruction. if (Val->getScalarType()->isIntegerTy() && ST->getXLen() < Val->getScalarSizeInBits()) { // For extractelement, we need the following instructions: // vsetivli zero, 1, e64, m1, ta, mu (not count) // vslidedown.vx v8, v8, a0 // vmv.x.s a0, v8 // li a1, 32 // vsrl.vx v8, v8, a1 // vmv.x.s a1, v8 // For insertelement, we need the following instructions: // vsetivli zero, 2, e32, m4, ta, mu (not count) // vmv.v.i v12, 0 // vslide1up.vx v16, v12, a1 // vslide1up.vx v12, v16, a0 // addi a0, a2, 1 // vsetvli zero, a0, e64, m4, tu, mu (not count) // vslideup.vx v8, v12, a2 // TODO: should we count these special vsetvlis? BaseCost = Opcode == Instruction::InsertElement ? 3 : 4; } return BaseCost + SlideCost; } InstructionCost RISCVTTIImpl::getArithmeticInstrCost( unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info, ArrayRef Args, const Instruction *CxtI) { // TODO: Handle more cost kinds. if (CostKind != TTI::TCK_RecipThroughput) return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CxtI); if (isa(Ty) && !ST->useRVVForFixedLengthVectors()) return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CxtI); // Skip if scalar size of Ty is bigger than ELEN. if (isa(Ty) && Ty->getScalarSizeInBits() > ST->getELen()) return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CxtI); // Legalize the type. std::pair LT = getTypeLegalizationCost(Ty); // TODO: Handle scalar type. if (!LT.second.isVector()) return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CxtI); auto getConstantMatCost = [&](unsigned Operand, TTI::OperandValueInfo OpInfo) -> InstructionCost { if (OpInfo.isUniform() && TLI->canSplatOperand(Opcode, Operand)) // Two sub-cases: // * Has a 5 bit immediate operand which can be splatted. // * Has a larger immediate which must be materialized in scalar register // We return 0 for both as we currently ignore the cost of materializing // scalar constants in GPRs. return 0; return getConstantPoolLoadCost(Ty, CostKind); }; // Add the cost of materializing any constant vectors required. InstructionCost ConstantMatCost = 0; if (Op1Info.isConstant()) ConstantMatCost += getConstantMatCost(0, Op1Info); if (Op2Info.isConstant()) ConstantMatCost += getConstantMatCost(1, Op2Info); switch (TLI->InstructionOpcodeToISD(Opcode)) { case ISD::ADD: case ISD::SUB: case ISD::AND: case ISD::OR: case ISD::XOR: case ISD::SHL: case ISD::SRL: case ISD::SRA: case ISD::MUL: case ISD::MULHS: case ISD::MULHU: case ISD::FADD: case ISD::FSUB: case ISD::FMUL: case ISD::FNEG: { return ConstantMatCost + TLI->getLMULCost(LT.second) * LT.first * 1; } default: return ConstantMatCost + BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info, Args, CxtI); } } // TODO: Deduplicate from TargetTransformInfoImplCRTPBase. InstructionCost RISCVTTIImpl::getPointersChainCost( ArrayRef Ptrs, const Value *Base, const TTI::PointersChainInfo &Info, Type *AccessTy, TTI::TargetCostKind CostKind) { InstructionCost Cost = TTI::TCC_Free; // In the basic model we take into account GEP instructions only // (although here can come alloca instruction, a value, constants and/or // constant expressions, PHIs, bitcasts ... whatever allowed to be used as a // pointer). Typically, if Base is a not a GEP-instruction and all the // pointers are relative to the same base address, all the rest are // either GEP instructions, PHIs, bitcasts or constants. When we have same // base, we just calculate cost of each non-Base GEP as an ADD operation if // any their index is a non-const. // If no known dependecies between the pointers cost is calculated as a sum // of costs of GEP instructions. for (auto [I, V] : enumerate(Ptrs)) { const auto *GEP = dyn_cast(V); if (!GEP) continue; if (Info.isSameBase() && V != Base) { if (GEP->hasAllConstantIndices()) continue; // If the chain is unit-stride and BaseReg + stride*i is a legal // addressing mode, then presume the base GEP is sitting around in a // register somewhere and check if we can fold the offset relative to // it. unsigned Stride = DL.getTypeStoreSize(AccessTy); if (Info.isUnitStride() && isLegalAddressingMode(AccessTy, /* BaseGV */ nullptr, /* BaseOffset */ Stride * I, /* HasBaseReg */ true, /* Scale */ 0, GEP->getType()->getPointerAddressSpace())) continue; Cost += getArithmeticInstrCost(Instruction::Add, GEP->getType(), CostKind, {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, std::nullopt); } else { SmallVector Indices(GEP->indices()); Cost += getGEPCost(GEP->getSourceElementType(), GEP->getPointerOperand(), Indices, AccessTy, CostKind); } } return Cost; } void RISCVTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP, OptimizationRemarkEmitter *ORE) { // TODO: More tuning on benchmarks and metrics with changes as needed // would apply to all settings below to enable performance. if (ST->enableDefaultUnroll()) return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE); // Enable Upper bound unrolling universally, not dependant upon the conditions // below. UP.UpperBound = true; // Disable loop unrolling for Oz and Os. UP.OptSizeThreshold = 0; UP.PartialOptSizeThreshold = 0; if (L->getHeader()->getParent()->hasOptSize()) return; SmallVector ExitingBlocks; L->getExitingBlocks(ExitingBlocks); LLVM_DEBUG(dbgs() << "Loop has:\n" << "Blocks: " << L->getNumBlocks() << "\n" << "Exit blocks: " << ExitingBlocks.size() << "\n"); // Only allow another exit other than the latch. This acts as an early exit // as it mirrors the profitability calculation of the runtime unroller. if (ExitingBlocks.size() > 2) return; // Limit the CFG of the loop body for targets with a branch predictor. // Allowing 4 blocks permits if-then-else diamonds in the body. if (L->getNumBlocks() > 4) return; // Don't unroll vectorized loops, including the remainder loop if (getBooleanLoopAttribute(L, "llvm.loop.isvectorized")) return; // Scan the loop: don't unroll loops with calls as this could prevent // inlining. InstructionCost Cost = 0; for (auto *BB : L->getBlocks()) { for (auto &I : *BB) { // Initial setting - Don't unroll loops containing vectorized // instructions. if (I.getType()->isVectorTy()) return; if (isa(I) || isa(I)) { if (const Function *F = cast(I).getCalledFunction()) { if (!isLoweredToCall(F)) continue; } return; } SmallVector Operands(I.operand_values()); Cost += getInstructionCost(&I, Operands, TargetTransformInfo::TCK_SizeAndLatency); } } LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n"); UP.Partial = true; UP.Runtime = true; UP.UnrollRemainder = true; UP.UnrollAndJam = true; UP.UnrollAndJamInnerLoopThreshold = 60; // Force unrolling small loops can be very useful because of the branch // taken cost of the backedge. if (Cost < 12) UP.Force = true; } void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, TTI::PeelingPreferences &PP) { BaseT::getPeelingPreferences(L, SE, PP); } unsigned RISCVTTIImpl::getRegUsageForType(Type *Ty) { TypeSize Size = DL.getTypeSizeInBits(Ty); if (Ty->isVectorTy()) { if (Size.isScalable() && ST->hasVInstructions()) return divideCeil(Size.getKnownMinValue(), RISCV::RVVBitsPerBlock); if (ST->useRVVForFixedLengthVectors()) return divideCeil(Size, ST->getRealMinVLen()); } return BaseT::getRegUsageForType(Ty); } unsigned RISCVTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const { if (SLPMaxVF.getNumOccurrences()) return SLPMaxVF; // Return how many elements can fit in getRegisterBitwidth. This is the // same routine as used in LoopVectorizer. We should probably be // accounting for whether we actually have instructions with the right // lane type, but we don't have enough information to do that without // some additional plumbing which hasn't been justified yet. TypeSize RegWidth = getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector); // If no vector registers, or absurd element widths, disable // vectorization by returning 1. return std::max(1U, RegWidth.getFixedValue() / ElemWidth); } bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1, const TargetTransformInfo::LSRCost &C2) { // RISC-V specific here are "instruction number 1st priority". return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost, C1.NumIVMuls, C1.NumBaseAdds, C1.ScaleCost, C1.ImmCost, C1.SetupCost) < std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost, C2.NumIVMuls, C2.NumBaseAdds, C2.ScaleCost, C2.ImmCost, C2.SetupCost); }