PHB3: Retry fundamental reset

When issuing fundamental reset on below IPR adapter that seats
behind root complex, there is 50% possibility that the link
fails to come up after the reset. In that case, the adapter's
config space is blocked and it's not usable.

   host# lspci -ns 0004:01:00.0
   0004:01:00.0 0104: 1014:034a (rev 01)
   host# lspci -s 0004:01:00.0
   0004:01:00.0 RAID bus controller: IBM PCI-E IPR SAS Adapter
                (ASIC) (rev 01)

This introduces another PHB3 state (PHB3_STATE_FRESET_START)
allowing to redo fundamental reset if the link doesn't come up
in time at the first attempt, to improve the robustness of PHB's
fundamental reset. If the link comes up after the first reset,
the 2nd reset won't be issued at all.

Reported-by: Paul Nguyen <nguyenp@us.ibm.com>
Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>

2 files changed, 31 insertions, 2 deletions

diff --git a/hw/phb3.c b/hw/phb3.c
index fd89d16..ebac9d6 100644
--- a/hw/phb3.c
+++ b/hw/phb3.c
@@ -1964,8 +1964,19 @@ static void phb3_setup_for_link_up(struct phb3 *p)
 	}
 }
 
+static int64_t phb3_retry_state(struct phb3 *p)
+{
+	if (p->retry_state <= PHB3_STATE_FUNCTIONAL)
+		return OPAL_WRONG_STATE;
+
+	p->delay_tgt_tb = 0;
+	p->state = p->retry_state;
+	return p->phb.ops->poll(&p->phb);
+}
+
 static int64_t phb3_sm_link_poll(struct phb3 *p)
 {
+	int64_t rc;
 	uint64_t reg;
 
 	/* This is the state machine to wait for the link to come
@@ -1992,6 +2003,10 @@ static int64_t phb3_sm_link_poll(struct phb3 *p)
 		} else if (p->retries-- == 0) {
 			PHBDBG(p, "Timeout waiting for electrical link\n");
 			PHBDBG(p, "DLP train control: 0x%016llx\n", reg);
+			rc = phb3_retry_state(p);
+			if (rc >= OPAL_SUCCESS)
+				return rc;
+
 			/* No link, we still mark the PHB as functional */
 			p->state = PHB3_STATE_FUNCTIONAL;
 			return OPAL_SUCCESS;
@@ -2013,6 +2028,10 @@ static int64_t phb3_sm_link_poll(struct phb3 *p)
 		if (p->retries-- == 0) {
 			PHBDBG(p, "Timeout waiting for link up\n");
 			PHBDBG(p, "DLP train control: 0x%016llx\n", reg);
+			rc = phb3_retry_state(p);
+			if (rc >= OPAL_SUCCESS)
+				return rc;
+
 			/* No link, we still mark the PHB as functional */
 			p->state = PHB3_STATE_FUNCTIONAL;
 			return OPAL_SUCCESS;
@@ -2127,12 +2146,17 @@ static int64_t phb3_sm_fundamental_reset(struct phb3 *p)
 
 	switch(p->state) {
 	case PHB3_STATE_FUNCTIONAL:
-		PHBINF(p, "Performing PERST...\n");
-
 		/* Prepare for link going down */
 		phb3_setup_for_link_down(p);
+		/* Fall-through */
+	case PHB3_STATE_FRESET_START:
+		if (p->state == PHB3_STATE_FRESET_START) {
+			PHBDBG(p, "Slot freset: Retrying\n");
+			p->retry_state = 0;
+		}
 
 		/* Assert PERST */
+		PHBINF(p, "Performing PERST...\n");
 		reg = in_be64(p->regs + PHB_RESET);
 		reg &= ~0x2000000000000000ul;
 		out_be64(p->regs + PHB_RESET, reg);
@@ -2176,6 +2200,8 @@ static int64_t phb3_fundamental_reset(struct phb *phb)
 		return OPAL_HARDWARE;
 	}
 
+	/* Allow to retry fundamental reset */
+	p->retry_state = PHB3_STATE_FRESET_START;
 	p->flags |= PHB3_CFG_BLOCKED;
 	return phb3_sm_fundamental_reset(p);
 }
@@ -2456,6 +2482,7 @@ static int64_t phb3_poll(struct phb *phb)
 	case PHB3_STATE_HRESET_DELAY:
 	case PHB3_STATE_HRESET_DELAY2:
 		return phb3_sm_hot_reset(p);
+	case PHB3_STATE_FRESET_START:
 	case PHB3_STATE_FRESET_ASSERT_DELAY:
 	case PHB3_STATE_FRESET_DEASSERT_DELAY:
 		return phb3_sm_fundamental_reset(p);
diff --git a/include/phb3.h b/include/phb3.h
index 3accd9e..b2aae15 100644
--- a/include/phb3.h
+++ b/include/phb3.h
@@ -212,6 +212,7 @@ enum phb3_state {
 	PHB3_STATE_HRESET_DELAY2,
 
 	/* Fundamental reset */
+	PHB3_STATE_FRESET_START,
 	PHB3_STATE_FRESET_ASSERT_DELAY,
 	PHB3_STATE_FRESET_DEASSERT_DELAY,
 
@@ -291,6 +292,7 @@ struct phb3 {
 	bool			skip_perst; /* Skip first perst */
 	bool			has_link;
 	enum phb3_state		state;
+	enum phb3_state		retry_state;
 	uint64_t		delay_tgt_tb;
 	uint64_t		retries;
 	int64_t			ecap;	    /* cached PCI-E cap offset */