phb3: Add support for CAPP DMA mode

The XSL used in the Mellanox CX4 card uses a DMA mode of CAPI, which
requires a few registers configured specially. In addition to enabling
the mode,

- The CAPP only owns some of the PHB read buffers, and must be
  configured to use the correct ones, and the self-snoop configured for
  the same ones.

- The tve needs to be configured to allow the card to access all kernel
  memory as it uses DMA accesses to read the scheduled process area from
  the kernel, among other things.

These cannot be configured unconditionally, as doing so will break
existing CAPI devices that do not use DMA mode. This adds a new mode to
the OPAL_PCI_SET_PHB_CAPI_MODE API to enable CAPI in DMA mode.

Since the snoop on/off modes write to the capi snoop configuration
register, which is configured differently in DMA mode, it uses the
redundant bits from the apc master powerbus control register to
determine if it should configure the register for DMA mode rather than
requiring any more permutations of the mode parameter.

Signed-off-by: Ian Munsie <imunsie@au1.ibm.com>
Reviewed-by: Frederic Barrat <fbarrat@linux.vnet.ibm.com>
Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>

2 files changed, 47 insertions, 10 deletions

diff --git a/hw/phb3.c b/hw/phb3.c
index b2be3e4..3bfa000 100644
--- a/hw/phb3.c
+++ b/hw/phb3.c
@@ -3324,14 +3324,22 @@ static int64_t phb3_get_diag_data(struct phb *phb,
 	return OPAL_SUCCESS;
 }
 
-static void phb3_init_capp_regs(struct phb3 *p)
+static void phb3_init_capp_regs(struct phb3 *p, bool dma_mode)
 {
 	uint64_t reg;
 	uint32_t offset;
+	uint64_t read_buffers = 0;
+
+	if (dma_mode) {
+		/* In DMA mode, the CAPP only owns some of the PHB read buffers */
+		read_buffers = 0x1;
+	}
 
 	offset = PHB3_CAPP_REG_OFFSET(p);
 	xscom_read(p->chip_id, APC_MASTER_PB_CTRL + offset, &reg);
+	reg &= ~PPC_BITMASK(10, 11);
 	reg |= PPC_BIT(3);
+	reg |= read_buffers << PPC_BITLSHIFT(11);
 	xscom_write(p->chip_id, APC_MASTER_PB_CTRL + offset, reg);
 
 	/* Dynamically workout which PHB to connect to port 0 of the CAPP.
@@ -3378,7 +3386,10 @@ static void phb3_init_capp_regs(struct phb3 *p)
 	xscom_write(p->chip_id,  FLUSH_UOP_CONFIG1 + offset,
 		    0xB188280728000000);
 	xscom_write(p->chip_id, FLUSH_UOP_CONFIG2 + offset, 0xB188400F00000000);
-	xscom_write(p->chip_id, SNOOP_CAPI_CONFIG + offset, 0xA1F0000000000000);
+
+	reg = 0xA1F0000000000000;
+	reg |= read_buffers << PPC_BITLSHIFT(39);
+	xscom_write(p->chip_id, SNOOP_CAPI_CONFIG + offset, reg);
 }
 
 /* override some inits with CAPI defaults */
@@ -3395,7 +3406,7 @@ static void phb3_init_capp_errors(struct phb3 *p)
 #define PE_REG_OFFSET(p) \
 	((PHB3_IS_NAPLES(p) && (p)->index) ? 0x40 : 0x0)
 
-static int64_t enable_capi_mode(struct phb3 *p, uint64_t pe_number)
+static int64_t enable_capi_mode(struct phb3 *p, uint64_t pe_number, bool dma_mode)
 {
 	uint64_t reg;
 	int i;
@@ -3417,7 +3428,12 @@ static int64_t enable_capi_mode(struct phb3 *p, uint64_t pe_number)
 		return OPAL_HARDWARE;
 	}
 
-	xscom_write(p->chip_id, p->spci_xscom + 0x3, 0x8000000000000000ull);
+	/* pb aib capp enable */
+	reg = PPC_BIT(0); /* capp enable */
+	if (dma_mode)
+		reg |= PPC_BIT(1); /* capp dma mode */
+	xscom_write(p->chip_id, p->spci_xscom + 0x3, reg);
+
 	/* FIXME security timer bar
 	xscom_write(p->chip_id, p->spci_xscom + 0x4, 0x8000000000000000ull);
 	*/
@@ -3456,8 +3472,16 @@ static int64_t enable_capi_mode(struct phb3 *p, uint64_t pe_number)
 
 	/* set tve no translate mode allow mmio window */
 	memset(p->tve_cache, 0x0, sizeof(p->tve_cache));
-	/* Allow address range 0x0002000000000000: 0x0002FFFFFFFFFFF */
-	p->tve_cache[pe_number * 2] = 0x000000FFFFFF0a00ULL;
+	if (dma_mode) {
+		/*
+		 * CAPP DMA mode needs access to all of memory, set address
+		 * range to 0x0000000000000000: 0x0002FFFFFFFFFFF
+		 */
+		p->tve_cache[pe_number * 2] = 0x000000FFFFFF0200ULL;
+	} else {
+		/* Allow address range 0x0002000000000000: 0x0002FFFFFFFFFFF */
+		p->tve_cache[pe_number * 2] = 0x000000FFFFFF0a00ULL;
+	}
 
 	phb3_ioda_sel(p, IODA2_TBL_TVT, 0, true);
 	for (i = 0; i < ARRAY_SIZE(p->tve_cache); i++)
@@ -3484,7 +3508,7 @@ static int64_t enable_capi_mode(struct phb3 *p, uint64_t pe_number)
 
 	phb3_init_capp_errors(p);
 
-	phb3_init_capp_regs(p);
+	phb3_init_capp_regs(p, dma_mode);
 
 	if (!chiptod_capp_timebase_sync(p)) {
 		PHBERR(p, "CAPP: Failed to sync timebase\n");
@@ -3500,6 +3524,7 @@ static int64_t phb3_set_capi_mode(struct phb *phb, uint64_t mode,
 	struct phb3 *p = phb_to_phb3(phb);
 	struct proc_chip *chip = get_chip(p->chip_id);
 	uint64_t reg;
+	uint64_t read_buffers;
 	uint32_t offset;
 	u8 mask;
 
@@ -3545,7 +3570,10 @@ static int64_t phb3_set_capi_mode(struct phb *phb, uint64_t mode,
 		return OPAL_UNSUPPORTED;
 
 	case OPAL_PHB_CAPI_MODE_CAPI:
-		return enable_capi_mode(p, pe_number);
+		return enable_capi_mode(p, pe_number, false);
+
+	case OPAL_PHB_CAPI_MODE_DMA:
+		return enable_capi_mode(p, pe_number, true);
 
 	case OPAL_PHB_CAPI_MODE_SNOOP_OFF:
 		xscom_write(p->chip_id, SNOOP_CAPI_CONFIG + offset,
@@ -3555,8 +3583,16 @@ static int64_t phb3_set_capi_mode(struct phb *phb, uint64_t mode,
 	case OPAL_PHB_CAPI_MODE_SNOOP_ON:
 		xscom_write(p->chip_id, CAPP_ERR_STATUS_CTRL + offset,
 			    0x0000000000000000);
-		xscom_write(p->chip_id, SNOOP_CAPI_CONFIG + offset,
-			    0xA1F0000000000000);
+		/*
+		 * Make sure the PHB read buffers being snooped match those
+		 * being used so we don't need another mode to set SNOOP+DMA
+		 */
+		xscom_read(p->chip_id, APC_MASTER_PB_CTRL + offset, &reg);
+		read_buffers = (reg >> PPC_BITLSHIFT(11)) & 0x3;
+		reg = 0xA1F0000000000000;
+		reg |= read_buffers << PPC_BITLSHIFT(39);
+		xscom_write(p->chip_id, SNOOP_CAPI_CONFIG + offset, reg);
+
 		return OPAL_SUCCESS;
 	}
 
diff --git a/include/opal-api.h b/include/opal-api.h
index f720f0e..f88e97f 100644
--- a/include/opal-api.h
+++ b/include/opal-api.h
@@ -951,6 +951,7 @@ enum {
 	OPAL_PHB_CAPI_MODE_CAPI		= 1,
 	OPAL_PHB_CAPI_MODE_SNOOP_OFF    = 2,
 	OPAL_PHB_CAPI_MODE_SNOOP_ON	= 3,
+	OPAL_PHB_CAPI_MODE_DMA		= 4,
 };
 
 /* CAPI feature flags (in device-tree) */