diff options
Diffstat (limited to 'llvm/lib/Target')
19 files changed, 209 insertions, 139 deletions
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 7294f3e..fbce3b0 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -18640,7 +18640,7 @@ bool AArch64TargetLowering::isDesirableToCommuteXorWithShift( } bool AArch64TargetLowering::shouldFoldConstantShiftPairToMask( - const SDNode *N, CombineLevel Level) const { + const SDNode *N) const { assert(((N->getOpcode() == ISD::SHL && N->getOperand(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL && diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index e472e7d..00956fd 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -300,8 +300,7 @@ public: bool isDesirableToCommuteXorWithShift(const SDNode *N) const override; /// Return true if it is profitable to fold a pair of shifts into a mask. - bool shouldFoldConstantShiftPairToMask(const SDNode *N, - CombineLevel Level) const override; + bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override; /// Return true if it is profitable to fold a pair of shifts into a mask. bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override { diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 83c7def..67ea2dd 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -13816,7 +13816,7 @@ bool ARMTargetLowering::isDesirableToCommuteXorWithShift( } bool ARMTargetLowering::shouldFoldConstantShiftPairToMask( - const SDNode *N, CombineLevel Level) const { + const SDNode *N) const { assert(((N->getOpcode() == ISD::SHL && N->getOperand(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL && @@ -13826,7 +13826,8 @@ bool ARMTargetLowering::shouldFoldConstantShiftPairToMask( if (!Subtarget->isThumb1Only()) return true; - if (Level == BeforeLegalizeTypes) + EVT VT = N->getValueType(0); + if (VT.getScalarSizeInBits() > 32) return true; return false; diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h index 26ff54c..70aa001 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.h +++ b/llvm/lib/Target/ARM/ARMISelLowering.h @@ -772,8 +772,7 @@ class VectorType; bool isDesirableToCommuteXorWithShift(const SDNode *N) const override; - bool shouldFoldConstantShiftPairToMask(const SDNode *N, - CombineLevel Level) const override; + bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override; /// Return true if it is profitable to fold a pair of shifts into a mask. bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override { diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp index b05de49..7f1ff45 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.cpp +++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp @@ -1306,7 +1306,7 @@ bool MipsTargetLowering::hasBitTest(SDValue X, SDValue Y) const { } bool MipsTargetLowering::shouldFoldConstantShiftPairToMask( - const SDNode *N, CombineLevel Level) const { + const SDNode *N) const { assert(((N->getOpcode() == ISD::SHL && N->getOperand(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL && diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h index c65c76c..25a0bf9 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.h +++ b/llvm/lib/Target/Mips/MipsISelLowering.h @@ -290,8 +290,7 @@ class TargetRegisterClass; bool isCheapToSpeculateCttz(Type *Ty) const override; bool isCheapToSpeculateCtlz(Type *Ty) const override; bool hasBitTest(SDValue X, SDValue Y) const override; - bool shouldFoldConstantShiftPairToMask(const SDNode *N, - CombineLevel Level) const override; + bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override; /// Return the register type for a given MVT, ensuring vectors are treated /// as a series of gpr sized integers. diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index f692180..944a1e2 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -585,6 +585,10 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, // We cannot sextinreg(i1). Expand to shifts. setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); + // Custom handling for PowerPC ucmp instruction + setOperationAction(ISD::UCMP, MVT::i32, Custom); + setOperationAction(ISD::UCMP, MVT::i64, isPPC64 ? Custom : Expand); + // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support // SjLj exception handling but a light-weight setjmp/longjmp replacement to // support continuation, user-level threading, and etc.. As a result, no @@ -12618,6 +12622,33 @@ SDValue PPCTargetLowering::LowerSSUBO(SDValue Op, SelectionDAG &DAG) const { return DAG.getMergeValues({Sub, OverflowTrunc}, dl); } +// Lower unsigned 3-way compare producing -1/0/1. +SDValue PPCTargetLowering::LowerUCMP(SDValue Op, SelectionDAG &DAG) const { + SDLoc DL(Op); + SDValue A = DAG.getFreeze(Op.getOperand(0)); + SDValue B = DAG.getFreeze(Op.getOperand(1)); + EVT OpVT = A.getValueType(); // operand type + EVT ResVT = Op.getValueType(); // result type + + // First compute diff = A - B (will become subf). + SDValue Diff = DAG.getNode(ISD::SUB, DL, OpVT, A, B); + + // Generate B - A using SUBC to capture carry. + SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); + SDValue SubC = DAG.getNode(PPCISD::SUBC, DL, VTs, B, A); + SDValue CA0 = SubC.getValue(1); + + // t2 = A - B + CA0 using SUBE. + SDValue SubE1 = DAG.getNode(PPCISD::SUBE, DL, VTs, A, B, CA0); + SDValue CA1 = SubE1.getValue(1); + + // res = diff - t2 + CA1 using SUBE (produces desired -1/0/1). + SDValue ResPair = DAG.getNode(PPCISD::SUBE, DL, VTs, Diff, SubE1, CA1); + + // Extract the first result and truncate to result type if needed + return DAG.getSExtOrTrunc(ResPair.getValue(0), DL, ResVT); +} + /// LowerOperation - Provide custom lowering hooks for some operations. /// SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { @@ -12722,6 +12753,8 @@ SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::UADDO_CARRY: case ISD::USUBO_CARRY: return LowerADDSUBO_CARRY(Op, DAG); + case ISD::UCMP: + return LowerUCMP(Op, DAG); } } diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h index 6694305..59f3387 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.h +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -1318,6 +1318,7 @@ namespace llvm { SDValue LowerIS_FPCLASS(SDValue Op, SelectionDAG &DAG) const; SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) const; SDValue LowerADDSUBO(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerUCMP(SDValue Op, SelectionDAG &DAG) const; SDValue lowerToLibCall(const char *LibCallName, SDValue Op, SelectionDAG &DAG) const; SDValue lowerLibCallBasedOnType(const char *LibCallFloatName, diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td index 7a14929..b9e01c3 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td @@ -1653,17 +1653,18 @@ def riscv_selectcc_frag : PatFrag<(ops node:$lhs, node:$rhs, node:$cc, node:$falsev), [{}], IntCCtoRISCVCC>; -multiclass SelectCC_GPR_rrirr<DAGOperand valty, ValueType vt> { +multiclass SelectCC_GPR_rrirr<DAGOperand valty, ValueType vt, + ValueType cmpvt = XLenVT> { let usesCustomInserter = 1 in def _Using_CC_GPR : Pseudo<(outs valty:$dst), (ins GPR:$lhs, GPR:$rhs, cond_code:$cc, valty:$truev, valty:$falsev), [(set valty:$dst, - (riscv_selectcc_frag:$cc (XLenVT GPR:$lhs), GPR:$rhs, cond, + (riscv_selectcc_frag:$cc (cmpvt GPR:$lhs), GPR:$rhs, cond, (vt valty:$truev), valty:$falsev))]>; // Explicitly select 0 in the condition to X0. The register coalescer doesn't // always do it. - def : Pat<(riscv_selectcc_frag:$cc (XLenVT GPR:$lhs), 0, cond, (vt valty:$truev), + def : Pat<(riscv_selectcc_frag:$cc (cmpvt GPR:$lhs), 0, cond, (vt valty:$truev), valty:$falsev), (!cast<Instruction>(NAME#"_Using_CC_GPR") GPR:$lhs, (XLenVT X0), (IntCCtoRISCVCC $cc), valty:$truev, valty:$falsev)>; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td index b9510ef..65e7e3b 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td @@ -59,9 +59,9 @@ def FPR64IN32X : RegisterOperand<GPRPair> { def DExt : ExtInfo<"", "", [HasStdExtD], f64, FPR64, FPR32, FPR64, ?>; def ZdinxExt : ExtInfo<"_INX", "Zfinx", [HasStdExtZdinx, IsRV64], - f64, FPR64INX, FPR32INX, FPR64INX, ?>; + f64, FPR64INX, FPR32INX, FPR64INX, ?, i64>; def Zdinx32Ext : ExtInfo<"_IN32X", "ZdinxRV32Only", [HasStdExtZdinx, IsRV32], - f64, FPR64IN32X, FPR32INX, FPR64IN32X, ?>; + f64, FPR64IN32X, FPR32INX, FPR64IN32X, ?, i32>; defvar DExts = [DExt, ZdinxExt, Zdinx32Ext]; defvar DExtsRV64 = [DExt, ZdinxExt]; @@ -261,8 +261,10 @@ let Predicates = [HasStdExtZdinx, IsRV32] in { /// Float conversion operations // f64 -> f32, f32 -> f64 -def : Pat<(any_fpround FPR64IN32X:$rs1), (FCVT_S_D_IN32X FPR64IN32X:$rs1, FRM_DYN)>; -def : Pat<(any_fpextend FPR32INX:$rs1), (FCVT_D_S_IN32X FPR32INX:$rs1, FRM_RNE)>; +def : Pat<(any_fpround FPR64IN32X:$rs1), + (FCVT_S_D_IN32X FPR64IN32X:$rs1, (i32 FRM_DYN))>; +def : Pat<(any_fpextend FPR32INX:$rs1), + (FCVT_D_S_IN32X FPR32INX:$rs1, (i32 FRM_RNE))>; } // Predicates = [HasStdExtZdinx, IsRV32] // [u]int<->double conversion patterns must be gated on IsRV32 or IsRV64, so @@ -321,7 +323,7 @@ def : Pat<(any_fsqrt FPR64INX:$rs1), (FSQRT_D_INX FPR64INX:$rs1, FRM_DYN)>; def : Pat<(fneg FPR64INX:$rs1), (FSGNJN_D_INX $rs1, $rs1)>; def : Pat<(fabs FPR64INX:$rs1), (FSGNJX_D_INX $rs1, $rs1)>; -def : Pat<(riscv_fclass FPR64INX:$rs1), (FCLASS_D_INX $rs1)>; +def : Pat<(i64 (riscv_fclass FPR64INX:$rs1)), (FCLASS_D_INX $rs1)>; def : PatFprFpr<fcopysign, FSGNJ_D_INX, FPR64INX, f64>; def : PatFprFpr<riscv_fsgnjx, FSGNJX_D_INX, FPR64INX, f64>; @@ -354,41 +356,46 @@ def : Pat<(fneg (any_fma_nsz FPR64INX:$rs1, FPR64INX:$rs2, FPR64INX:$rs3)), } // Predicates = [HasStdExtZdinx, IsRV64] let Predicates = [HasStdExtZdinx, IsRV32] in { -def : Pat<(any_fsqrt FPR64IN32X:$rs1), (FSQRT_D_IN32X FPR64IN32X:$rs1, FRM_DYN)>; +def : Pat<(any_fsqrt FPR64IN32X:$rs1), + (FSQRT_D_IN32X FPR64IN32X:$rs1, (i32 FRM_DYN))>; def : Pat<(fneg FPR64IN32X:$rs1), (FSGNJN_D_IN32X $rs1, $rs1)>; def : Pat<(fabs FPR64IN32X:$rs1), (FSGNJX_D_IN32X $rs1, $rs1)>; -def : Pat<(riscv_fclass FPR64IN32X:$rs1), (FCLASS_D_IN32X $rs1)>; +def : Pat<(i32 (riscv_fclass FPR64IN32X:$rs1)), (FCLASS_D_IN32X $rs1)>; def : PatFprFpr<fcopysign, FSGNJ_D_IN32X, FPR64IN32X, f64>; def : PatFprFpr<riscv_fsgnjx, FSGNJX_D_IN32X, FPR64IN32X, f64>; def : Pat<(fcopysign FPR64IN32X:$rs1, (fneg FPR64IN32X:$rs2)), (FSGNJN_D_IN32X FPR64IN32X:$rs1, FPR64IN32X:$rs2)>; def : Pat<(fcopysign FPR64IN32X:$rs1, FPR32INX:$rs2), - (FSGNJ_D_IN32X $rs1, (FCVT_D_S_IN32X $rs2, FRM_RNE))>; + (FSGNJ_D_IN32X $rs1, (FCVT_D_S_IN32X $rs2, (i32 FRM_RNE)))>; def : Pat<(fcopysign FPR32INX:$rs1, FPR64IN32X:$rs2), - (FSGNJ_S_INX $rs1, (FCVT_S_D_IN32X $rs2, FRM_DYN))>; + (FSGNJ_S_INX $rs1, (FCVT_S_D_IN32X $rs2, (i32 FRM_DYN)))>; // fmadd: rs1 * rs2 + rs3 def : Pat<(any_fma FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3), - (FMADD_D_IN32X $rs1, $rs2, $rs3, FRM_DYN)>; + (FMADD_D_IN32X $rs1, $rs2, $rs3, (i32 FRM_DYN))>; // fmsub: rs1 * rs2 - rs3 def : Pat<(any_fma FPR64IN32X:$rs1, FPR64IN32X:$rs2, (fneg FPR64IN32X:$rs3)), - (FMSUB_D_IN32X FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3, FRM_DYN)>; + (FMSUB_D_IN32X FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3, + (i32 FRM_DYN))>; // fnmsub: -rs1 * rs2 + rs3 def : Pat<(any_fma (fneg FPR64IN32X:$rs1), FPR64IN32X:$rs2, FPR64IN32X:$rs3), - (FNMSUB_D_IN32X FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3, FRM_DYN)>; + (FNMSUB_D_IN32X FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3, + (i32 FRM_DYN))>; // fnmadd: -rs1 * rs2 - rs3 def : Pat<(any_fma (fneg FPR64IN32X:$rs1), FPR64IN32X:$rs2, (fneg FPR64IN32X:$rs3)), - (FNMADD_D_IN32X FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3, FRM_DYN)>; + (FNMADD_D_IN32X FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3, + (i32 FRM_DYN))>; // fnmadd: -(rs1 * rs2 + rs3) (the nsz flag on the FMA) def : Pat<(fneg (any_fma_nsz FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3)), - (FNMADD_D_IN32X FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3, FRM_DYN)>; + (FNMADD_D_IN32X FPR64IN32X:$rs1, FPR64IN32X:$rs2, FPR64IN32X:$rs3, + (i32 FRM_DYN))>; } // Predicates = [HasStdExtZdinx, IsRV32] // The ratified 20191213 ISA spec defines fmin and fmax in a way that matches @@ -441,42 +448,42 @@ def : PatSetCC<FPR64, any_fsetccs, SETOLE, FLE_D, f64>; let Predicates = [HasStdExtZdinx, IsRV64] in { // Match signaling FEQ_D -def : Pat<(XLenVT (strict_fsetccs (f64 FPR64INX:$rs1), FPR64INX:$rs2, SETEQ)), +def : Pat<(XLenVT (strict_fsetccs FPR64INX:$rs1, FPR64INX:$rs2, SETEQ)), (AND (XLenVT (FLE_D_INX $rs1, $rs2)), (XLenVT (FLE_D_INX $rs2, $rs1)))>; -def : Pat<(XLenVT (strict_fsetccs (f64 FPR64INX:$rs1), FPR64INX:$rs2, SETOEQ)), +def : Pat<(XLenVT (strict_fsetccs FPR64INX:$rs1, FPR64INX:$rs2, SETOEQ)), (AND (XLenVT (FLE_D_INX $rs1, $rs2)), (XLenVT (FLE_D_INX $rs2, $rs1)))>; // If both operands are the same, use a single FLE. -def : Pat<(XLenVT (strict_fsetccs (f64 FPR64INX:$rs1), FPR64INX:$rs1, SETEQ)), +def : Pat<(XLenVT (strict_fsetccs FPR64INX:$rs1, FPR64INX:$rs1, SETEQ)), (FLE_D_INX $rs1, $rs1)>; -def : Pat<(XLenVT (strict_fsetccs (f64 FPR64INX:$rs1), FPR64INX:$rs1, SETOEQ)), +def : Pat<(XLenVT (strict_fsetccs FPR64INX:$rs1, FPR64INX:$rs1, SETOEQ)), (FLE_D_INX $rs1, $rs1)>; -def : PatSetCC<FPR64INX, any_fsetccs, SETLT, FLT_D_INX, f64>; -def : PatSetCC<FPR64INX, any_fsetccs, SETOLT, FLT_D_INX, f64>; -def : PatSetCC<FPR64INX, any_fsetccs, SETLE, FLE_D_INX, f64>; -def : PatSetCC<FPR64INX, any_fsetccs, SETOLE, FLE_D_INX, f64>; +def : PatSetCC<FPR64INX, any_fsetccs, SETLT, FLT_D_INX, f64, i64>; +def : PatSetCC<FPR64INX, any_fsetccs, SETOLT, FLT_D_INX, f64, i64>; +def : PatSetCC<FPR64INX, any_fsetccs, SETLE, FLE_D_INX, f64, i64>; +def : PatSetCC<FPR64INX, any_fsetccs, SETOLE, FLE_D_INX, f64, i64>; } // Predicates = [HasStdExtZdinx, IsRV64] let Predicates = [HasStdExtZdinx, IsRV32] in { // Match signaling FEQ_D -def : Pat<(XLenVT (strict_fsetccs (f64 FPR64IN32X:$rs1), FPR64IN32X:$rs2, SETEQ)), +def : Pat<(i32 (strict_fsetccs FPR64IN32X:$rs1, FPR64IN32X:$rs2, SETEQ)), (AND (XLenVT (FLE_D_IN32X $rs1, $rs2)), (XLenVT (FLE_D_IN32X $rs2, $rs1)))>; -def : Pat<(XLenVT (strict_fsetccs (f64 FPR64IN32X:$rs1), FPR64IN32X:$rs2, SETOEQ)), +def : Pat<(i32 (strict_fsetccs FPR64IN32X:$rs1, FPR64IN32X:$rs2, SETOEQ)), (AND (XLenVT (FLE_D_IN32X $rs1, $rs2)), (XLenVT (FLE_D_IN32X $rs2, $rs1)))>; // If both operands are the same, use a single FLE. -def : Pat<(XLenVT (strict_fsetccs (f64 FPR64IN32X:$rs1), FPR64IN32X:$rs1, SETEQ)), +def : Pat<(i32 (strict_fsetccs FPR64IN32X:$rs1, FPR64IN32X:$rs1, SETEQ)), (FLE_D_IN32X $rs1, $rs1)>; -def : Pat<(XLenVT (strict_fsetccs (f64 FPR64IN32X:$rs1), FPR64IN32X:$rs1, SETOEQ)), +def : Pat<(i32 (strict_fsetccs FPR64IN32X:$rs1, FPR64IN32X:$rs1, SETOEQ)), (FLE_D_IN32X $rs1, $rs1)>; -def : PatSetCC<FPR64IN32X, any_fsetccs, SETLT, FLT_D_IN32X, f64>; -def : PatSetCC<FPR64IN32X, any_fsetccs, SETOLT, FLT_D_IN32X, f64>; -def : PatSetCC<FPR64IN32X, any_fsetccs, SETLE, FLE_D_IN32X, f64>; -def : PatSetCC<FPR64IN32X, any_fsetccs, SETOLE, FLE_D_IN32X, f64>; +def : PatSetCC<FPR64IN32X, any_fsetccs, SETLT, FLT_D_IN32X, f64, i32>; +def : PatSetCC<FPR64IN32X, any_fsetccs, SETOLT, FLT_D_IN32X, f64, i32>; +def : PatSetCC<FPR64IN32X, any_fsetccs, SETLE, FLE_D_IN32X, f64, i32>; +def : PatSetCC<FPR64IN32X, any_fsetccs, SETOLE, FLE_D_IN32X, f64, i32>; } // Predicates = [HasStdExtZdinx, IsRV32] let Predicates = [HasStdExtD] in { @@ -511,7 +518,7 @@ def SplitF64Pseudo } // Predicates = [HasStdExtD, NoStdExtZfa, IsRV32] let Predicates = [HasStdExtZdinx, IsRV64] in { -defm Select_FPR64INX : SelectCC_GPR_rrirr<FPR64INX, f64>; +defm Select_FPR64INX : SelectCC_GPR_rrirr<FPR64INX, f64, i64>; def PseudoFROUND_D_INX : PseudoFROUND<FPR64INX, f64>; @@ -523,9 +530,9 @@ def : StPat<store, SD, GPR, f64>; } // Predicates = [HasStdExtZdinx, IsRV64] let Predicates = [HasStdExtZdinx, IsRV32] in { -defm Select_FPR64IN32X : SelectCC_GPR_rrirr<FPR64IN32X, f64>; +defm Select_FPR64IN32X : SelectCC_GPR_rrirr<FPR64IN32X, f64, i32>; -def PseudoFROUND_D_IN32X : PseudoFROUND<FPR64IN32X, f64>; +def PseudoFROUND_D_IN32X : PseudoFROUND<FPR64IN32X, f64, i32>; /// Loads let hasSideEffects = 0, mayLoad = 1, mayStore = 0, Size = 8, isCodeGenOnly = 1 in diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td index fde030e..6571d99 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td @@ -131,7 +131,7 @@ def FPR32INX : RegisterOperand<GPRF32> { // The DAGOperand can be unset if the predicates are not enough to define it. class ExtInfo<string suffix, string space, list<Predicate> predicates, ValueType primaryvt, DAGOperand primaryty, DAGOperand f32ty, - DAGOperand f64ty, DAGOperand f16ty> { + DAGOperand f64ty, DAGOperand f16ty, ValueType intvt = XLenVT> { list<Predicate> Predicates = predicates; string Suffix = suffix; string Space = space; @@ -140,6 +140,7 @@ class ExtInfo<string suffix, string space, list<Predicate> predicates, DAGOperand F32Ty = f32ty; DAGOperand F64Ty = f64ty; ValueType PrimaryVT = primaryvt; + ValueType IntVT = intvt; } def FExt : ExtInfo<"", "", [HasStdExtF], f32, FPR32, FPR32, ?, ?>; @@ -314,9 +315,9 @@ multiclass FPCmp_rr_m<bits<7> funct7, bits<3> funct3, string opcodestr, def Ext.Suffix : FPCmp_rr<funct7, funct3, opcodestr, Ext.PrimaryTy, Commutable>; } -class PseudoFROUND<DAGOperand Ty, ValueType vt> +class PseudoFROUND<DAGOperand Ty, ValueType vt, ValueType intvt = XLenVT> : Pseudo<(outs Ty:$rd), (ins Ty:$rs1, Ty:$rs2, ixlenimm:$rm), - [(set Ty:$rd, (vt (riscv_fround Ty:$rs1, Ty:$rs2, timm:$rm)))]> { + [(set Ty:$rd, (vt (riscv_fround Ty:$rs1, Ty:$rs2, (intvt timm:$rm))))]> { let hasSideEffects = 0; let mayLoad = 0; let mayStore = 0; @@ -529,13 +530,14 @@ def fpimm0 : PatLeaf<(fpimm), [{ return N->isExactlyValue(+0.0); }]>; /// Generic pattern classes class PatSetCC<DAGOperand Ty, SDPatternOperator OpNode, CondCode Cond, - RVInstCommon Inst, ValueType vt> - : Pat<(XLenVT (OpNode (vt Ty:$rs1), Ty:$rs2, Cond)), (Inst $rs1, $rs2)>; + RVInstCommon Inst, ValueType vt, ValueType intvt = XLenVT> + : Pat<(intvt (OpNode (vt Ty:$rs1), Ty:$rs2, Cond)), (Inst $rs1, $rs2)>; multiclass PatSetCC_m<SDPatternOperator OpNode, CondCode Cond, RVInstCommon Inst, ExtInfo Ext> { let Predicates = Ext.Predicates in def Ext.Suffix : PatSetCC<Ext.PrimaryTy, OpNode, Cond, - !cast<RVInstCommon>(Inst#Ext.Suffix), Ext.PrimaryVT>; + !cast<RVInstCommon>(Inst#Ext.Suffix), + Ext.PrimaryVT, Ext.IntVT>; } class PatFprFpr<SDPatternOperator OpNode, RVInstR Inst, @@ -549,14 +551,15 @@ multiclass PatFprFpr_m<SDPatternOperator OpNode, RVInstR Inst, } class PatFprFprDynFrm<SDPatternOperator OpNode, RVInstRFrm Inst, - DAGOperand RegTy, ValueType vt> - : Pat<(OpNode (vt RegTy:$rs1), (vt RegTy:$rs2)), (Inst $rs1, $rs2, FRM_DYN)>; + DAGOperand RegTy, ValueType vt, ValueType intvt> + : Pat<(OpNode (vt RegTy:$rs1), (vt RegTy:$rs2)), + (Inst $rs1, $rs2,(intvt FRM_DYN))>; multiclass PatFprFprDynFrm_m<SDPatternOperator OpNode, RVInstRFrm Inst, ExtInfo Ext> { let Predicates = Ext.Predicates in def Ext.Suffix : PatFprFprDynFrm<OpNode, !cast<RVInstRFrm>(Inst#Ext.Suffix), - Ext.PrimaryTy, Ext.PrimaryVT>; + Ext.PrimaryTy, Ext.PrimaryVT, Ext.IntVT>; } /// Float conversion operations diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td index 014da99..52a2b29 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td @@ -69,16 +69,16 @@ def ZhinxminExt : ExtInfo<"_INX", "Zfinx", f16, FPR16INX, FPR32INX, ?, FPR16INX>; def ZhinxZdinxExt : ExtInfo<"_INX", "Zfinx", [HasStdExtZhinx, HasStdExtZdinx, IsRV64], - ?, ?, FPR32INX, FPR64INX, FPR16INX>; + ?, ?, FPR32INX, FPR64INX, FPR16INX, i64>; def ZhinxminZdinxExt : ExtInfo<"_INX", "Zfinx", [HasStdExtZhinxmin, HasStdExtZdinx, IsRV64], - ?, ?, FPR32INX, FPR64INX, FPR16INX>; + ?, ?, FPR32INX, FPR64INX, FPR16INX, i64>; def ZhinxZdinx32Ext : ExtInfo<"_IN32X", "ZdinxGPRPairRV32", [HasStdExtZhinx, HasStdExtZdinx, IsRV32], - ?, ?, FPR32INX, FPR64IN32X, FPR16INX>; + ?, ?, FPR32INX, FPR64IN32X, FPR16INX, i32>; def ZhinxminZdinx32Ext : ExtInfo<"_IN32X", "ZdinxGPRPairRV32", [HasStdExtZhinxmin, HasStdExtZdinx, IsRV32], - ?, ?, FPR32INX, FPR64IN32X, FPR16INX>; + ?, ?, FPR32INX, FPR64IN32X, FPR16INX, i32>; defvar ZfhExts = [ZfhExt, ZhinxExt]; defvar ZfhminExts = [ZfhminExt, ZhinxminExt]; diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt index 46afe03..eab7b21 100644 --- a/llvm/lib/Target/SPIRV/CMakeLists.txt +++ b/llvm/lib/Target/SPIRV/CMakeLists.txt @@ -36,6 +36,7 @@ add_llvm_target(SPIRVCodeGen SPIRVMetadata.cpp SPIRVModuleAnalysis.cpp SPIRVStructurizer.cpp + SPIRVCombinerHelper.cpp SPIRVPreLegalizer.cpp SPIRVPreLegalizerCombiner.cpp SPIRVPostLegalizer.cpp diff --git a/llvm/lib/Target/SPIRV/SPIRVCombine.td b/llvm/lib/Target/SPIRV/SPIRVCombine.td index 6f726e0..fde56c4 100644 --- a/llvm/lib/Target/SPIRV/SPIRVCombine.td +++ b/llvm/lib/Target/SPIRV/SPIRVCombine.td @@ -11,8 +11,8 @@ include "llvm/Target/GlobalISel/Combine.td" def vector_length_sub_to_distance_lowering : GICombineRule < (defs root:$root), (match (wip_match_opcode G_INTRINSIC):$root, - [{ return matchLengthToDistance(*${root}, MRI); }]), - (apply [{ applySPIRVDistance(*${root}, MRI, B); }]) + [{ return Helper.matchLengthToDistance(*${root}); }]), + (apply [{ Helper.applySPIRVDistance(*${root}); }]) >; def SPIRVPreLegalizerCombiner diff --git a/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp new file mode 100644 index 0000000..267794c --- /dev/null +++ b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp @@ -0,0 +1,60 @@ +//===-- SPIRVCombinerHelper.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "SPIRVCombinerHelper.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" +#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" +#include "llvm/IR/IntrinsicsSPIRV.h" +#include "llvm/Target/TargetMachine.h" + +using namespace llvm; +using namespace MIPatternMatch; + +SPIRVCombinerHelper::SPIRVCombinerHelper( + GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize, + GISelValueTracking *VT, MachineDominatorTree *MDT, const LegalizerInfo *LI, + const SPIRVSubtarget &STI) + : CombinerHelper(Observer, B, IsPreLegalize, VT, MDT, LI), STI(STI) {} + +/// This match is part of a combine that +/// rewrites length(X - Y) to distance(X, Y) +/// (f32 (g_intrinsic length +/// (g_fsub (vXf32 X) (vXf32 Y)))) +/// -> +/// (f32 (g_intrinsic distance +/// (vXf32 X) (vXf32 Y))) +/// +bool SPIRVCombinerHelper::matchLengthToDistance(MachineInstr &MI) const { + if (MI.getOpcode() != TargetOpcode::G_INTRINSIC || + cast<GIntrinsic>(MI).getIntrinsicID() != Intrinsic::spv_length) + return false; + + // First operand of MI is `G_INTRINSIC` so start at operand 2. + Register SubReg = MI.getOperand(2).getReg(); + MachineInstr *SubInstr = MRI.getVRegDef(SubReg); + if (SubInstr->getOpcode() != TargetOpcode::G_FSUB) + return false; + + return true; +} + +void SPIRVCombinerHelper::applySPIRVDistance(MachineInstr &MI) const { + // Extract the operands for X and Y from the match criteria. + Register SubDestReg = MI.getOperand(2).getReg(); + MachineInstr *SubInstr = MRI.getVRegDef(SubDestReg); + Register SubOperand1 = SubInstr->getOperand(1).getReg(); + Register SubOperand2 = SubInstr->getOperand(2).getReg(); + Register ResultReg = MI.getOperand(0).getReg(); + + Builder.setInstrAndDebugLoc(MI); + Builder.buildIntrinsic(Intrinsic::spv_distance, ResultReg) + .addUse(SubOperand1) + .addUse(SubOperand2); + + MI.eraseFromParent(); +} diff --git a/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h new file mode 100644 index 0000000..0b39d34 --- /dev/null +++ b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h @@ -0,0 +1,38 @@ +//===-- SPIRVCombinerHelper.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// This contains common combine transformations that may be used in a combine +/// pass. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_SPIRV_SPIRVCOMBINERHELPER_H +#define LLVM_LIB_TARGET_SPIRV_SPIRVCOMBINERHELPER_H + +#include "SPIRVSubtarget.h" +#include "llvm/CodeGen/GlobalISel/CombinerHelper.h" + +namespace llvm { +class SPIRVCombinerHelper : public CombinerHelper { +protected: + const SPIRVSubtarget &STI; + +public: + using CombinerHelper::CombinerHelper; + SPIRVCombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B, + bool IsPreLegalize, GISelValueTracking *VT, + MachineDominatorTree *MDT, const LegalizerInfo *LI, + const SPIRVSubtarget &STI); + + bool matchLengthToDistance(MachineInstr &MI) const; + void applySPIRVDistance(MachineInstr &MI) const; +}; + +} // end namespace llvm + +#endif // LLVM_LIB_TARGET_SPIRV_SPIRVCOMBINERHELPER_H diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp index 8356751..48f4047 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizerCombiner.cpp @@ -1,4 +1,3 @@ - //===-- SPIRVPreLegalizerCombiner.cpp - combine legalization ----*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. @@ -13,24 +12,17 @@ //===----------------------------------------------------------------------===// #include "SPIRV.h" -#include "SPIRVTargetMachine.h" +#include "SPIRVCombinerHelper.h" #include "llvm/CodeGen/GlobalISel/CSEInfo.h" #include "llvm/CodeGen/GlobalISel/Combiner.h" -#include "llvm/CodeGen/GlobalISel/CombinerHelper.h" #include "llvm/CodeGen/GlobalISel/CombinerInfo.h" #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/GISelValueTracking.h" -#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" -#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetPassConfig.h" -#include "llvm/IR/IntrinsicsSPIRV.h" #define GET_GICOMBINER_DEPS #include "SPIRVGenPreLegalizeGICombiner.inc" @@ -47,72 +39,9 @@ namespace { #include "SPIRVGenPreLegalizeGICombiner.inc" #undef GET_GICOMBINER_TYPES -/// This match is part of a combine that -/// rewrites length(X - Y) to distance(X, Y) -/// (f32 (g_intrinsic length -/// (g_fsub (vXf32 X) (vXf32 Y)))) -/// -> -/// (f32 (g_intrinsic distance -/// (vXf32 X) (vXf32 Y))) -/// -bool matchLengthToDistance(MachineInstr &MI, MachineRegisterInfo &MRI) { - if (MI.getOpcode() != TargetOpcode::G_INTRINSIC || - cast<GIntrinsic>(MI).getIntrinsicID() != Intrinsic::spv_length) - return false; - - // First operand of MI is `G_INTRINSIC` so start at operand 2. - Register SubReg = MI.getOperand(2).getReg(); - MachineInstr *SubInstr = MRI.getVRegDef(SubReg); - if (!SubInstr || SubInstr->getOpcode() != TargetOpcode::G_FSUB) - return false; - - return true; -} -void applySPIRVDistance(MachineInstr &MI, MachineRegisterInfo &MRI, - MachineIRBuilder &B) { - - // Extract the operands for X and Y from the match criteria. - Register SubDestReg = MI.getOperand(2).getReg(); - MachineInstr *SubInstr = MRI.getVRegDef(SubDestReg); - Register SubOperand1 = SubInstr->getOperand(1).getReg(); - Register SubOperand2 = SubInstr->getOperand(2).getReg(); - - // Remove the original `spv_length` instruction. - - Register ResultReg = MI.getOperand(0).getReg(); - DebugLoc DL = MI.getDebugLoc(); - MachineBasicBlock &MBB = *MI.getParent(); - MachineBasicBlock::iterator InsertPt = MI.getIterator(); - - // Build the `spv_distance` intrinsic. - MachineInstrBuilder NewInstr = - BuildMI(MBB, InsertPt, DL, B.getTII().get(TargetOpcode::G_INTRINSIC)); - NewInstr - .addDef(ResultReg) // Result register - .addIntrinsicID(Intrinsic::spv_distance) // Intrinsic ID - .addUse(SubOperand1) // Operand X - .addUse(SubOperand2); // Operand Y - - SPIRVGlobalRegistry *GR = - MI.getMF()->getSubtarget<SPIRVSubtarget>().getSPIRVGlobalRegistry(); - auto RemoveAllUses = [&](Register Reg) { - SmallVector<MachineInstr *, 4> UsesToErase( - llvm::make_pointer_range(MRI.use_instructions(Reg))); - - // calling eraseFromParent to early invalidates the iterator. - for (auto *MIToErase : UsesToErase) { - GR->invalidateMachineInstr(MIToErase); - MIToErase->eraseFromParent(); - } - }; - RemoveAllUses(SubDestReg); // remove all uses of FSUB Result - GR->invalidateMachineInstr(SubInstr); - SubInstr->eraseFromParent(); // remove FSUB instruction -} - class SPIRVPreLegalizerCombinerImpl : public Combiner { protected: - const CombinerHelper Helper; + const SPIRVCombinerHelper Helper; const SPIRVPreLegalizerCombinerImplRuleConfig &RuleConfig; const SPIRVSubtarget &STI; @@ -147,7 +76,7 @@ SPIRVPreLegalizerCombinerImpl::SPIRVPreLegalizerCombinerImpl( const SPIRVSubtarget &STI, MachineDominatorTree *MDT, const LegalizerInfo *LI) : Combiner(MF, CInfo, TPC, &VT, CSEInfo), - Helper(Observer, B, /*IsPreLegalize*/ true, &VT, MDT, LI), + Helper(Observer, B, /*IsPreLegalize*/ true, &VT, MDT, LI, STI), RuleConfig(RuleConfig), STI(STI), #define GET_GICOMBINER_CONSTRUCTOR_INITS #include "SPIRVGenPreLegalizeGICombiner.inc" diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 1cfcb1f..eea84a2 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -3633,7 +3633,7 @@ bool X86TargetLowering::preferScalarizeSplat(SDNode *N) const { } bool X86TargetLowering::shouldFoldConstantShiftPairToMask( - const SDNode *N, CombineLevel Level) const { + const SDNode *N) const { assert(((N->getOpcode() == ISD::SHL && N->getOperand(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL && @@ -3648,7 +3648,7 @@ bool X86TargetLowering::shouldFoldConstantShiftPairToMask( // the fold for non-splats yet. return N->getOperand(1) == N->getOperand(0).getOperand(1); } - return TargetLoweringBase::shouldFoldConstantShiftPairToMask(N, Level); + return TargetLoweringBase::shouldFoldConstantShiftPairToMask(N); } bool X86TargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const { diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index b55556a..e28b9c1 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -1244,8 +1244,7 @@ namespace llvm { getJumpConditionMergingParams(Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs) const override; - bool shouldFoldConstantShiftPairToMask(const SDNode *N, - CombineLevel Level) const override; + bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override; bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override; |