diff options
Diffstat (limited to 'llvm/lib/Target/RISCV/RISCVISelLowering.cpp')
| -rw-r--r-- | llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 196 |
1 files changed, 86 insertions, 110 deletions
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index adbfbeb..0077ecf 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -927,6 +927,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT, Custom); + setOperationAction(ISD::VP_LOAD_FF, VT, Custom); setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR}, @@ -1105,6 +1106,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT, Custom); + setOperationAction(ISD::VP_LOAD_FF, VT, Custom); setOperationAction(ISD::SELECT, VT, Custom); setOperationAction(ISD::SELECT_CC, VT, Expand); @@ -1181,6 +1183,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT, Custom); + setOperationAction(ISD::VP_LOAD_FF, VT, Custom); setOperationAction(ISD::FNEG, VT, Expand); setOperationAction(ISD::FABS, VT, Expand); @@ -1352,6 +1355,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT, Custom); + setOperationAction(ISD::VP_LOAD_FF, VT, Custom); setOperationAction({ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR, ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV, @@ -1442,6 +1446,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, ISD::EXPERIMENTAL_VP_STRIDED_STORE}, VT, Custom); + setOperationAction(ISD::VP_LOAD_FF, VT, Custom); setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT, @@ -7012,6 +7017,7 @@ static unsigned getRISCVVLOp(SDValue Op) { OP_CASE(FDIV) OP_CASE(FNEG) OP_CASE(FABS) + OP_CASE(FCOPYSIGN) OP_CASE(FSQRT) OP_CASE(SMIN) OP_CASE(SMAX) @@ -7079,6 +7085,15 @@ static unsigned getRISCVVLOp(SDValue Op) { if (Op.getSimpleValueType().getVectorElementType() == MVT::i1) return RISCVISD::VMXOR_VL; return RISCVISD::XOR_VL; + case ISD::ANY_EXTEND: + case ISD::ZERO_EXTEND: + return RISCVISD::VZEXT_VL; + case ISD::SIGN_EXTEND: + return RISCVISD::VSEXT_VL; + case ISD::SETCC: + return RISCVISD::SETCC_VL; + case ISD::VSELECT: + return RISCVISD::VMERGE_VL; case ISD::VP_SELECT: case ISD::VP_MERGE: return RISCVISD::VMERGE_VL; @@ -7419,12 +7434,16 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, if (Op.getOperand(0).getValueType().isVector() && Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1) return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1); - return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL); + if (Op.getValueType().isScalableVector()) + return Op; + return lowerToScalableOp(Op, DAG); case ISD::SIGN_EXTEND: if (Op.getOperand(0).getValueType().isVector() && Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1) return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1); - return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL); + if (Op.getValueType().isScalableVector()) + return Op; + return lowerToScalableOp(Op, DAG); case ISD::SPLAT_VECTOR_PARTS: return lowerSPLAT_VECTOR_PARTS(Op, DAG); case ISD::INSERT_VECTOR_ELT: @@ -8103,6 +8122,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, case ISD::MLOAD: case ISD::VP_LOAD: return lowerMaskedLoad(Op, DAG); + case ISD::VP_LOAD_FF: + return lowerLoadFF(Op, DAG); case ISD::MSTORE: case ISD::VP_STORE: return lowerMaskedStore(Op, DAG); @@ -8166,7 +8187,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, if (isPromotedOpNeedingSplit(Op.getOperand(0), Subtarget)) return SplitVectorOp(Op, DAG); - return lowerFixedLengthVectorSetccToRVV(Op, DAG); + return lowerToScalableOp(Op, DAG); } case ISD::ADD: case ISD::SUB: @@ -8182,6 +8203,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, case ISD::UREM: case ISD::BSWAP: case ISD::CTPOP: + case ISD::VSELECT: return lowerToScalableOp(Op, DAG); case ISD::SHL: case ISD::SRA: @@ -8250,14 +8272,12 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, return lowerToScalableOp(Op, DAG); assert(Op.getOpcode() != ISD::CTTZ); return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG); - case ISD::VSELECT: - return lowerFixedLengthVectorSelectToRVV(Op, DAG); case ISD::FCOPYSIGN: if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) return lowerFCOPYSIGN(Op, DAG, Subtarget); if (isPromotedOpNeedingSplit(Op, Subtarget)) return SplitVectorOp(Op, DAG); - return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG); + return lowerToScalableOp(Op, DAG); case ISD::STRICT_FADD: case ISD::STRICT_FSUB: case ISD::STRICT_FMUL: @@ -9694,33 +9714,6 @@ SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG, return convertFromScalableVector(VecVT, Select, DAG, Subtarget); } -SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV( - SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const { - MVT ExtVT = Op.getSimpleValueType(); - // Only custom-lower extensions from fixed-length vector types. - if (!ExtVT.isFixedLengthVector()) - return Op; - MVT VT = Op.getOperand(0).getSimpleValueType(); - // Grab the canonical container type for the extended type. Infer the smaller - // type from that to ensure the same number of vector elements, as we know - // the LMUL will be sufficient to hold the smaller type. - MVT ContainerExtVT = getContainerForFixedLengthVector(ExtVT); - // Get the extended container type manually to ensure the same number of - // vector elements between source and dest. - MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(), - ContainerExtVT.getVectorElementCount()); - - SDValue Op1 = - convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget); - - SDLoc DL(Op); - auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); - - SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL); - - return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget); -} - // Custom-lower truncations from vectors to mask vectors by using a mask and a // setcc operation: // (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne) @@ -12739,6 +12732,51 @@ SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op, return DAG.getMergeValues({Result, Chain}, DL); } +SDValue RISCVTargetLowering::lowerLoadFF(SDValue Op, SelectionDAG &DAG) const { + SDLoc DL(Op); + MVT VT = Op->getSimpleValueType(0); + + const auto *VPLoadFF = cast<VPLoadFFSDNode>(Op); + EVT MemVT = VPLoadFF->getMemoryVT(); + MachineMemOperand *MMO = VPLoadFF->getMemOperand(); + SDValue Chain = VPLoadFF->getChain(); + SDValue BasePtr = VPLoadFF->getBasePtr(); + + SDValue Mask = VPLoadFF->getMask(); + SDValue VL = VPLoadFF->getVectorLength(); + + MVT XLenVT = Subtarget.getXLenVT(); + + MVT ContainerVT = VT; + if (VT.isFixedLengthVector()) { + ContainerVT = getContainerForFixedLengthVector(VT); + MVT MaskVT = getMaskTypeFor(ContainerVT); + Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget); + } + + unsigned IntID = Intrinsic::riscv_vleff_mask; + SDValue Ops[] = { + Chain, + DAG.getTargetConstant(IntID, DL, XLenVT), + DAG.getUNDEF(ContainerVT), + BasePtr, + Mask, + VL, + DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT)}; + + SDVTList VTs = DAG.getVTList({ContainerVT, Op->getValueType(1), MVT::Other}); + + SDValue Result = + DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO); + SDValue OutVL = Result.getValue(1); + Chain = Result.getValue(2); + + if (VT.isFixedLengthVector()) + Result = convertFromScalableVector(VT, Result, DAG, Subtarget); + + return DAG.getMergeValues({Result, OutVL, Chain}, DL); +} + SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op, SelectionDAG &DAG) const { SDLoc DL(Op); @@ -12834,31 +12872,6 @@ SDValue RISCVTargetLowering::lowerVectorCompress(SDValue Op, return Res; } -SDValue -RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(SDValue Op, - SelectionDAG &DAG) const { - MVT InVT = Op.getOperand(0).getSimpleValueType(); - MVT ContainerVT = getContainerForFixedLengthVector(InVT); - - MVT VT = Op.getSimpleValueType(); - - SDValue Op1 = - convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget); - SDValue Op2 = - convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget); - - SDLoc DL(Op); - auto [Mask, VL] = getDefaultVLOps(VT.getVectorNumElements(), ContainerVT, DL, - DAG, Subtarget); - MVT MaskVT = getMaskTypeFor(ContainerVT); - - SDValue Cmp = - DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, - {Op1, Op2, Op.getOperand(2), DAG.getUNDEF(MaskVT), Mask, VL}); - - return convertFromScalableVector(VT, Cmp, DAG, Subtarget); -} - SDValue RISCVTargetLowering::lowerVectorStrictFSetcc(SDValue Op, SelectionDAG &DAG) const { unsigned Opc = Op.getOpcode(); @@ -12985,51 +12998,6 @@ SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const { return Max; } -SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV( - SDValue Op, SelectionDAG &DAG) const { - SDLoc DL(Op); - MVT VT = Op.getSimpleValueType(); - SDValue Mag = Op.getOperand(0); - SDValue Sign = Op.getOperand(1); - assert(Mag.getValueType() == Sign.getValueType() && - "Can only handle COPYSIGN with matching types."); - - MVT ContainerVT = getContainerForFixedLengthVector(VT); - Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget); - Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget); - - auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget); - - SDValue CopySign = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag, - Sign, DAG.getUNDEF(ContainerVT), Mask, VL); - - return convertFromScalableVector(VT, CopySign, DAG, Subtarget); -} - -SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV( - SDValue Op, SelectionDAG &DAG) const { - MVT VT = Op.getSimpleValueType(); - MVT ContainerVT = getContainerForFixedLengthVector(VT); - - MVT I1ContainerVT = - MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount()); - - SDValue CC = - convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget); - SDValue Op1 = - convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget); - SDValue Op2 = - convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget); - - SDLoc DL(Op); - SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; - - SDValue Select = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, Op1, - Op2, DAG.getUNDEF(ContainerVT), VL); - - return convertFromScalableVector(VT, Select, DAG, Subtarget); -} - SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG) const { const auto &TSInfo = @@ -13056,7 +13024,9 @@ SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, // "cast" fixed length vector to a scalable vector. assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) && "Only fixed length vectors are supported!"); - Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget)); + MVT VContainerVT = ContainerVT.changeVectorElementType( + V.getSimpleValueType().getVectorElementType()); + Ops.push_back(convertToScalableVector(VContainerVT, V, DAG, Subtarget)); } SDLoc DL(Op); @@ -21478,11 +21448,10 @@ bool RISCVTargetLowering::canCreateUndefOrPoisonForTargetNode( // TODO: Add more target nodes. switch (Op.getOpcode()) { case RISCVISD::SELECT_CC: - // Integer select_cc cannot create poison. - // TODO: What are the FP poison semantics? - // TODO: This instruction blocks poison from the unselected operand, can - // we do anything with that? - return !Op.getValueType().isInteger(); + // Integer comparisons cannot create poison. + assert(Op.getOperand(0).getValueType().isInteger() && + "RISCVISD::SELECT_CC only compares integers"); + return false; } return TargetLowering::canCreateUndefOrPoisonForTargetNode( Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth); @@ -22550,6 +22519,7 @@ SDValue RISCVTargetLowering::LowerFormalArguments( constexpr StringLiteral SupportedInterruptKinds[] = { "machine", "supervisor", + "rnmi", "qci-nest", "qci-nonest", "SiFive-CLIC-preemptible", @@ -22567,6 +22537,8 @@ SDValue RISCVTargetLowering::LowerFormalArguments( reportFatalUsageError( "'SiFive-CLIC-*' interrupt kinds require XSfmclic extension"); + if (Kind == "rnmi" && !Subtarget.hasStdExtSmrnmi()) + reportFatalUsageError("'rnmi' interrupt kind requires Srnmi extension"); const TargetFrameLowering *TFI = Subtarget.getFrameLowering(); if (Kind.starts_with("SiFive-CLIC-preemptible") && TFI->hasFP(MF)) reportFatalUsageError("'SiFive-CLIC-preemptible' interrupt kinds cannot " @@ -23212,7 +23184,11 @@ RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, if (Kind == "supervisor") RetOpc = RISCVISD::SRET_GLUE; - else if (Kind == "qci-nest" || Kind == "qci-nonest") { + else if (Kind == "rnmi") { + assert(STI.hasFeature(RISCV::FeatureStdExtSmrnmi) && + "Need Smrnmi extension for rnmi"); + RetOpc = RISCVISD::MNRET_GLUE; + } else if (Kind == "qci-nest" || Kind == "qci-nonest") { assert(STI.hasFeature(RISCV::FeatureVendorXqciint) && "Need Xqciint for qci-(no)nest"); RetOpc = RISCVISD::QC_C_MILEAVERET_GLUE; |