[RISCV] Rework fault first only load isel.

-Remove the ISD opcode for READ_VL. Just emit the MachineSDNode directly.
-Move segmented fault first only load intrinsic handling completely to
 RISCVISelDAGToDAG.cpp and emit the ReadVL MachineSDNode there
 instead of lowering to ISD opcodes first.

4 files changed, 41 insertions, 114 deletions

diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 7b0f386..7292cfe 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -225,7 +225,7 @@ void RISCVDAGToDAGISel::selectVLSEGMask(SDNode *Node, unsigned IntNo,
 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node) {
   SDLoc DL(Node);
   unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
-  unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue.
+  unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
   EVT VT = Node->getValueType(0);
   unsigned ScalarSize = VT.getScalarSizeInBits();
   MVT XLenVT = Subtarget->getXLenVT();
@@ -241,21 +241,24 @@ void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node) {
       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
   SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
                                         MVT::Glue, Operands);
+  SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
+                                          /*Glue*/ SDValue(Load, 2));
+
   SDValue SuperReg = SDValue(Load, 0);
   for (unsigned I = 0; I < NF; ++I)
     ReplaceUses(SDValue(Node, I),
                 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
                                                VT, SuperReg));
 
-  ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));     // Chain.
-  ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue.
+  ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0));   // VL
+  ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
   CurDAG->RemoveDeadNode(Node);
 }
 
 void RISCVDAGToDAGISel::selectVLSEGFFMask(SDNode *Node) {
   SDLoc DL(Node);
   unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
-  unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue.
+  unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
   EVT VT = Node->getValueType(0);
   unsigned ScalarSize = VT.getScalarSizeInBits();
   MVT XLenVT = Subtarget->getXLenVT();
@@ -275,14 +278,17 @@ void RISCVDAGToDAGISel::selectVLSEGFFMask(SDNode *Node) {
       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
   SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
                                         MVT::Glue, Operands);
+  SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
+                                          /*Glue*/ SDValue(Load, 2));
+
   SDValue SuperReg = SDValue(Load, 0);
   for (unsigned I = 0; I < NF; ++I)
     ReplaceUses(SDValue(Node, I),
                 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
                                                VT, SuperReg));
 
-  ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));     // Chain.
-  ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue.
+  ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0));   // VL
+  ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
   CurDAG->RemoveDeadNode(Node);
 }
 
@@ -680,6 +686,26 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       selectVLXSEGMask(Node, IntNo);
       return;
     }
+    case Intrinsic::riscv_vlseg8ff:
+    case Intrinsic::riscv_vlseg7ff:
+    case Intrinsic::riscv_vlseg6ff:
+    case Intrinsic::riscv_vlseg5ff:
+    case Intrinsic::riscv_vlseg4ff:
+    case Intrinsic::riscv_vlseg3ff:
+    case Intrinsic::riscv_vlseg2ff: {
+      selectVLSEGFF(Node);
+      return;
+    }
+    case Intrinsic::riscv_vlseg8ff_mask:
+    case Intrinsic::riscv_vlseg7ff_mask:
+    case Intrinsic::riscv_vlseg6ff_mask:
+    case Intrinsic::riscv_vlseg5ff_mask:
+    case Intrinsic::riscv_vlseg4ff_mask:
+    case Intrinsic::riscv_vlseg3ff_mask:
+    case Intrinsic::riscv_vlseg2ff_mask: {
+      selectVLSEGFFMask(Node);
+      return;
+    }
     }
     break;
   }
@@ -763,14 +789,6 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     }
     break;
   }
-  case RISCVISD::VLSEGFF: {
-    selectVLSEGFF(Node);
-    return;
-  }
-  case RISCVISD::VLSEGFF_MASK: {
-    selectVLSEGFFMask(Node);
-    return;
-  }
   }
 
   // Select the default instruction.
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 97862f9..23a677d 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1455,7 +1455,6 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
     }
   }
 
-  unsigned NF = 1;
   switch (IntNo) {
   default:
     return SDValue(); // Don't custom lower most intrinsics.
@@ -1464,8 +1463,10 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other, MVT::Glue);
     SDValue Load = DAG.getNode(RISCVISD::VLEFF, DL, VTs, Op.getOperand(0),
                                Op.getOperand(2), Op.getOperand(3));
-    VTs = DAG.getVTList(Op->getValueType(1), MVT::Other);
-    SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, Load.getValue(2));
+    SDValue ReadVL =
+        SDValue(DAG.getMachineNode(RISCV::PseudoReadVL, DL, Op->getValueType(1),
+                                   Load.getValue(2)),
+                0);
     return DAG.getMergeValues({Load, ReadVL, Load.getValue(1)}, DL);
   }
   case Intrinsic::riscv_vleff_mask: {
@@ -1474,92 +1475,12 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
     SDValue Load = DAG.getNode(RISCVISD::VLEFF_MASK, DL, VTs, Op.getOperand(0),
                                Op.getOperand(2), Op.getOperand(3),
                                Op.getOperand(4), Op.getOperand(5));
-    VTs = DAG.getVTList(Op->getValueType(1), MVT::Other);
-    SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, Load.getValue(2));
+    SDValue ReadVL =
+        SDValue(DAG.getMachineNode(RISCV::PseudoReadVL, DL, Op->getValueType(1),
+                                   Load.getValue(2)),
+                0);
     return DAG.getMergeValues({Load, ReadVL, Load.getValue(1)}, DL);
   }
-  case Intrinsic::riscv_vlseg8ff:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg7ff:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg6ff:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg5ff:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg4ff:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg3ff:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg2ff: {
-    NF++;
-    SDLoc DL(Op);
-    SmallVector<EVT, 8> EVTs(NF, Op.getValueType());
-    EVTs.push_back(MVT::Other);
-    EVTs.push_back(MVT::Glue);
-    SDVTList VTs = DAG.getVTList(EVTs);
-    SDValue Load =
-        DAG.getNode(RISCVISD::VLSEGFF, DL, VTs, Op.getOperand(0),
-                    Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
-    VTs = DAG.getVTList(Op->getValueType(NF), MVT::Other);
-    SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs,
-                                 /*Glue*/ Load.getValue(NF + 1));
-    SmallVector<SDValue, 8> Results;
-    for (unsigned i = 0; i < NF; ++i)
-      Results.push_back(Load.getValue(i));
-    Results.push_back(ReadVL);
-    Results.push_back(Load.getValue(NF)); // Chain.
-    return DAG.getMergeValues(Results, DL);
-  }
-  case Intrinsic::riscv_vlseg8ff_mask:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg7ff_mask:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg6ff_mask:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg5ff_mask:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg4ff_mask:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg3ff_mask:
-    NF++;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::riscv_vlseg2ff_mask: {
-    NF++;
-    SDLoc DL(Op);
-    SmallVector<EVT, 8> EVTs(NF, Op.getValueType());
-    EVTs.push_back(MVT::Other);
-    EVTs.push_back(MVT::Glue);
-    SDVTList VTs = DAG.getVTList(EVTs);
-    SmallVector<SDValue, 13> LoadOps;
-    LoadOps.push_back(Op.getOperand(0)); // Chain.
-    LoadOps.push_back(Op.getOperand(1)); // Intrinsic ID.
-    for (unsigned i = 0; i < NF; ++i)
-      LoadOps.push_back(Op.getOperand(2 + i)); // MaskedOff.
-    LoadOps.push_back(Op.getOperand(2 + NF));  // Base.
-    LoadOps.push_back(Op.getOperand(3 + NF));  // Mask.
-    LoadOps.push_back(Op.getOperand(4 + NF));  // VL.
-    SDValue Load = DAG.getNode(RISCVISD::VLSEGFF_MASK, DL, VTs, LoadOps);
-    VTs = DAG.getVTList(Op->getValueType(NF), MVT::Other);
-    SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs,
-                                 /*Glue*/ Load.getValue(NF + 1));
-    SmallVector<SDValue, 8> Results;
-    for (unsigned i = 0; i < NF; ++i)
-      Results.push_back(Load.getValue(i));
-    Results.push_back(ReadVL);
-    Results.push_back(Load.getValue(NF)); // Chain.
-    return DAG.getMergeValues(Results, DL);
-  }
   }
 }
 
@@ -4088,9 +4009,6 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(TRUNCATE_VECTOR)
   NODE_NAME_CASE(VLEFF)
   NODE_NAME_CASE(VLEFF_MASK)
-  NODE_NAME_CASE(VLSEGFF)
-  NODE_NAME_CASE(VLSEGFF_MASK)
-  NODE_NAME_CASE(READ_VL)
   NODE_NAME_CASE(VSLIDEUP)
   NODE_NAME_CASE(VSLIDEDOWN)
   NODE_NAME_CASE(VID)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 40b1a45..bff3f85 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -99,11 +99,6 @@ enum NodeType : unsigned {
   // Unit-stride fault-only-first load
   VLEFF,
   VLEFF_MASK,
-  // Unit-stride fault-only-first segment load
-  VLSEGFF,
-  VLSEGFF_MASK,
-  // read vl CSR
-  READ_VL,
   // Matches the semantics of vslideup/vslidedown. The first operand is the
   // pass-thru operand, the second is the source vector, and the third is the
   // XLenVT index (either constant or non-constant).
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 06e4d05..9ff6b4e 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -33,9 +33,6 @@ def riscv_vleff_mask : SDNode<"RISCVISD::VLEFF_MASK",
                                                    SDTCisVT<4, XLenVT>]>,
                               [SDNPHasChain, SDNPOutGlue, SDNPMayLoad,
                                SDNPSideEffect]>;
-def riscv_read_vl : SDNode<"RISCVISD::READ_VL",
-                           SDTypeProfile<1, 0, [SDTCisVT<0, XLenVT>]>,
-                           [SDNPInGlue]>;
 
 // X0 has special meaning for vsetvl/vsetvli.
 //  rd | rs1 |   AVL value | Effect on vl
@@ -3115,8 +3112,7 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1 in {
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1,
     Uses = [VL] in
-def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins),
-                          [(set GPR:$rd, (riscv_read_vl))]>;
+def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins), []>;
 
 //===----------------------------------------------------------------------===//
 // 6. Configuration-Setting Instructions