phb3/capi: Add capp recovery to phb3

Add a flag indicating the CAPP unit is in recovery.  When a capp recoverable
malfunction HMI comes in, the HMI handler will call into
phb3_set_capp_recovery, which will put set the flag and send the event to
Linux.

EEH will call phb3_next_error which will tell it the phb is fenced.

EEH will then call into sapphire to reinitialize the phb which contains steps
3-5 of capp recovery procedure.  The code increases wait time of PERST to 1s to
ensure fpga download is complete before polling linkup.

EEH will then rebind the cxl driver and it will complete recovery once it
initializes and turns snoops on, steps 7-8, completing capp recovery procedure.

Signed-off-by: Ryan Grimm <grimm@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

4 files changed, 65 insertions, 3 deletions

diff --git a/hw/phb3.c b/hw/phb3.c
index 3c03c20..10e3e59 100644
--- a/hw/phb3.c
+++ b/hw/phb3.c
@@ -2125,8 +2125,11 @@ static int64_t phb3_sm_fundamental_reset(struct phb3 *p)
 		out_be64(p->regs + PHB_RESET, reg);
 		PHBDBG(p, "Slot freset: Deasserting PERST\n");
 
-		/* Wait 200ms before polling link */
 		p->state = PHB3_STATE_FRESET_DEASSERT_DELAY;
+		/* CAPP fpga requires 1s to flash before polling link */
+		if (p->flags & PHB3_CAPP_RECOVERY)
+			return phb3_set_sm_timeout(p, secs_to_tb(1));
+		/* Wait 200ms before polling link dd*/
 		return phb3_set_sm_timeout(p, msecs_to_tb(200));
 
 	case PHB3_STATE_FRESET_DEASSERT_DELAY:
@@ -2156,6 +2159,7 @@ static int64_t phb3_fundamental_reset(struct phb *phb)
 	return phb3_sm_fundamental_reset(p);
 }
 
+static void do_capp_recovery_scoms(struct phb3 *phb);
 /*
  * The OS is expected to do fundamental reset after complete
  * reset to make sure the PHB could be recovered from the
@@ -2176,6 +2180,11 @@ static int64_t phb3_sm_complete_reset(struct phb3 *p)
 	switch (p->state) {
 	case PHB3_STATE_FENCED:
 	case PHB3_STATE_FUNCTIONAL:
+
+		/* do steps 3-5 of capp recovery procedure */
+		if (p->flags & PHB3_CAPP_RECOVERY)
+			do_capp_recovery_scoms(p);
+
 		/*
 		 * The users might be doing error injection through PBCQ
 		 * Error Inject Control Register. Without clearing that,
@@ -2221,6 +2230,7 @@ static int64_t phb3_sm_complete_reset(struct phb3 *p)
 		return phb3_set_sm_timeout(p, msecs_to_tb(10));
 	case PHB3_STATE_CRESET_REINIT:
 		p->flags &= ~PHB3_AIB_FENCED;
+		p->flags &= ~PHB3_CAPP_RECOVERY;
 		phb3_init_hw(p);
 
 		p->state = PHB3_STATE_CRESET_FRESET;
@@ -2330,8 +2340,8 @@ static int64_t phb3_eeh_freeze_status(struct phb *phb, uint64_t pe_number,
 		return OPAL_HARDWARE;
 	}
 
-	/* Check fence */
-	if (phb3_fenced(p)) {
+	/* Check fence and CAPP recovery */
+	if (phb3_fenced(p) || (p->flags & PHB3_CAPP_RECOVERY)) {
 		*freeze_state = OPAL_EEH_STOPPED_MMIO_DMA_FREEZE;
 		*pci_error_type = OPAL_EEH_PHB_ERROR;
 		if (severity)
@@ -2480,6 +2490,12 @@ static int64_t phb3_eeh_next_error(struct phb *phb,
 		return OPAL_SUCCESS;
 	}
 
+	if ((p->flags & PHB3_CAPP_RECOVERY)) {
+		*pci_error_type = OPAL_EEH_PHB_ERROR;
+		*severity = OPAL_EEH_SEV_PHB_FENCED;
+		return OPAL_SUCCESS;
+	}
+
 	/*
 	 * Check if we already have pending errors. If that's
 	 * the case, then to get more information about the
@@ -2925,6 +2941,15 @@ static int64_t phb3_set_capi_mode(struct phb *phb, uint64_t mode,
 		return OPAL_BUSY;
 	}
 
+	xscom_read(p->chip_id, CAPP_ERR_STATUS_CTRL, &reg);
+	if ((reg & PPC_BIT(5))) {
+		PHBERR(p, "CAPP recovery failed (%016llx)\n", reg);
+		return OPAL_HARDWARE;
+	} else if ((reg & PPC_BIT(0)) && (!(reg & PPC_BIT(1)))) {
+		PHBDBG(p, "CAPP recovery in progress\n");
+		return OPAL_BUSY;
+	}
+
 	if (mode == OPAL_PHB_MODE_PCIE)
 		return OPAL_UNSUPPORTED;
 
@@ -3036,6 +3061,37 @@ static int64_t phb3_set_capi_mode(struct phb *phb, uint64_t mode,
 	return OPAL_SUCCESS;
 }
 
+static int64_t phb3_set_capp_recovery(struct phb *phb)
+{
+	struct phb3 *p = phb_to_phb3(phb);
+
+	if (p->flags & PHB3_CAPP_RECOVERY)
+		return 0;
+
+	/* set opal event flag to indicate eeh condition */
+	opal_update_pending_evt(OPAL_EVENT_PCI_ERROR,
+				OPAL_EVENT_PCI_ERROR);
+
+	p->flags |= PHB3_CAPP_RECOVERY;
+
+	return 0;
+}
+
+static void do_capp_recovery_scoms(struct phb3 *p)
+{
+	uint64_t reg;
+	PHBDBG(p, "Doing CAPP recovery scoms\n");
+
+	xscom_write(p->chip_id, 0x201301a, 0); /* disable snoops */
+	capp_load_ucode(p);
+	xscom_write(p->chip_id, 0x2013013, 0); /* clear err rpt reg*/
+	xscom_write(p->chip_id, 0x2013000, 0); /* clear capp fir */
+
+	xscom_read(p->chip_id, CAPP_ERR_STATUS_CTRL, &reg);
+	reg &= ~(PPC_BIT(0) | PPC_BIT(1));
+	xscom_write(p->chip_id, CAPP_ERR_STATUS_CTRL, reg);
+}
+
 static const struct phb_ops phb3_ops = {
 	.lock			= phb3_lock,
 	.unlock			= phb3_unlock,
@@ -3078,6 +3134,7 @@ static const struct phb_ops phb3_ops = {
 	.get_diag_data		= NULL,
 	.get_diag_data2		= phb3_get_diag_data,
 	.set_capi_mode		= phb3_set_capi_mode,
+	.set_capp_recovery	= phb3_set_capp_recovery,
 };
 
 /*
diff --git a/include/capp.h b/include/capp.h
index c0b67e1..ef052b8 100644
--- a/include/capp.h
+++ b/include/capp.h
@@ -57,6 +57,8 @@ enum capp_reg {
 #define CAPP_SNP_ARRAY_WRITE_REG		0x2013801
 #define CAPP_APC_MASTER_ARRAY_WRITE_REG		0x2013802
 
+#define CAPP_FIR                                0x2013000
+#define CAPP_ERR_RPT_CLR                        0x2013013
 #define APC_MASTER_PB_CTRL			0x2013018
 #define APC_MASTER_CAPI_CTRL			0x2013019
 #define TRANSPORT_CONTROL			0x201301C
diff --git a/include/pci.h b/include/pci.h
index 8b0cca1..c15935c 100644
--- a/include/pci.h
+++ b/include/pci.h
@@ -412,6 +412,8 @@ struct phb_ops {
 
 	/* Put phb in capi mode or pcie mode */
 	int64_t (*set_capi_mode)(struct phb *phb, uint64_t mode, uint64_t pe_number);
+
+	int64_t (*set_capp_recovery)(struct phb *phb);
 };
 
 enum phb_type {
diff --git a/include/phb3.h b/include/phb3.h
index 6305a1c..c004056 100644
--- a/include/phb3.h
+++ b/include/phb3.h
@@ -258,6 +258,7 @@ struct phb3_err {
 #define PHB3_CFG_USE_ASB	0x00000002
 #define PHB3_CFG_BLOCKED	0x00000004
 #define PHB3_RESTORE_BUS_NUM	0x00000008
+#define PHB3_CAPP_RECOVERY	0x00000010
 
 struct phb3 {
 	unsigned int		index;	    /* 0..2 index inside P8 */