pau: enabling opencapi

Enable OpenCAPI mode for each brick which are connected to be used in
this mode. This is be done through 7 steps as described in the
P10 OCAPI 5.0 Processing Unit Workbook document, section:
17.1.3.1 Enabling OpenCAPI.
The following sequences must be performed:
1. Set Transport MUX controls to select OpenCAPI
2. Enable Clocks in XSL
3. Enable Clocks in MISC
4. Set NPCQ configuration
5. Enable XSL-XTS Interfaces
6. Enable State-machine allocation

Enabling the NTL/GENID BARS allows to access to the MMIO registers.

Signed-off-by: Christophe Lombard <clombard@linux.vnet.ibm.com>

Reviewed-by: Frederic Barrat <fbarrat@linux.ibm.com>
Signed-off-by: Vasant Hegde <hegdevasant@linux.vnet.ibm.com>

3 files changed, 266 insertions, 0 deletions

diff --git a/hw/pau.c b/hw/pau.c
index 1a370a9..5f18101 100644
--- a/hw/pau.c
+++ b/hw/pau.c
@@ -199,6 +199,42 @@ static void pau_device_detect_fixup(struct pau_dev *dev)
 	dt_add_property_strings(dn, "ibm,pau-link-type", "unknown");
 }
 
+#define CQ_CTL_STATUS_TIMEOUT  10 /* milliseconds */
+
+static int pau_opencapi_set_fence_control(struct pau_dev *dev,
+					  uint8_t state_requested)
+{
+	uint64_t timeout = mftb() + msecs_to_tb(CQ_CTL_STATUS_TIMEOUT);
+	uint8_t status;
+	struct pau *pau = dev->pau;
+	uint64_t reg, val;
+
+	reg = PAU_CTL_MISC_FENCE_CTRL(dev->index);
+	val = pau_read(pau, reg);
+	val = SETFIELD(PAU_CTL_MISC_FENCE_REQUEST, val, state_requested);
+	pau_write(pau, reg, val);
+
+	/* Wait for fence status to update */
+	do {
+		reg = PAU_CTL_MISC_STATUS(dev->index);
+		val = pau_read(pau, reg);
+		status = GETFIELD(PAU_CTL_MISC_STATUS_AM_FENCED(dev->index), val);
+		if (status == state_requested)
+			return OPAL_SUCCESS;
+		time_wait_ms(1);
+	} while (tb_compare(mftb(), timeout) == TB_ABEFOREB);
+
+	/*
+	 * @fwts-label OCAPIFenceStatusTimeout
+	 * @fwts-advice The PAU fence status did not update as expected. This
+	 * could be the result of a firmware or hardware bug. OpenCAPI
+	 * functionality could be broken.
+	 */
+	PAUDEVERR(dev, "Bad fence status: expected 0x%x, got 0x%x\n",
+		       state_requested, status);
+	return OPAL_HARDWARE;
+}
+
 static void pau_opencapi_assign_bars(struct pau *pau)
 {
 	struct pau_dev *dev;
@@ -256,6 +292,37 @@ static void pau_opencapi_assign_bars(struct pau *pau)
 	}
 }
 
+static void pau_opencapi_enable_bars(struct pau_dev *dev, bool enable)
+{
+	struct pau *pau = dev->pau;
+	uint64_t reg, val;
+
+	if (dev->ntl_bar.enable == enable) /* No state change */
+		return;
+
+	dev->ntl_bar.enable = enable;
+	dev->genid_bar.enable = enable;
+
+	reg = PAU_NTL_BAR(dev->index);
+	val = pau_read(pau, reg);
+	val = SETFIELD(PAU_NTL_BAR_ENABLE, val, enable);
+	pau_write(pau, reg, val);
+
+	/*
+	 * Generation IDs are a single space in the hardware but we split them
+	 * per device. Only disable in hardware if every device has disabled.
+	 */
+	if (!enable)
+		pau_for_each_dev(dev, pau)
+			if (dev->genid_bar.enable)
+				return;
+
+	reg = PAU_GENID_BAR;
+	val = pau_read(pau, reg);
+	val = SETFIELD(PAU_GENID_BAR_ENABLE, val, enable);
+	pau_write(pau, reg, val);
+}
+
 static void pau_opencapi_create_phb_slot(struct pau_dev *dev)
 {
 	struct pci_slot *slot;
@@ -474,6 +541,135 @@ static void pau_opencapi_dt_add_props(struct pau_dev *dev)
 	pau_opencapi_dt_add_mmio_window(dev);
 }
 
+static void pau_opencapi_set_transport_mux_controls(struct pau_dev *dev)
+{
+	struct pau *pau = dev->pau;
+	uint32_t typemap = 0;
+	uint64_t reg, val = 0;
+
+	PAUDEVDBG(dev, "Setting transport mux controls\n");
+	typemap = 0x2 >> dev->index;
+
+	reg = PAU_MISC_OPTICAL_IO_CONFIG;
+	val = pau_read(pau, reg);
+	typemap |= GETFIELD(PAU_MISC_OPTICAL_IO_CONFIG_OTL, val);
+	val = SETFIELD(PAU_MISC_OPTICAL_IO_CONFIG_OTL, val, typemap);
+	pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_enable_xsl_clocks(struct pau *pau)
+{
+	uint64_t reg, val;
+
+	PAUDBG(pau, "Enable clocks in XSL\n");
+
+	reg = PAU_XSL_WRAP_CFG;
+	val = pau_read(pau, reg);
+	val |= PAU_XSL_WRAP_CFG_CLOCK_ENABLE;
+	pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_enable_misc_clocks(struct pau *pau)
+{
+	uint64_t reg, val;
+
+	PAUDBG(pau, "Enable clocks in MISC\n");
+
+	/* clear any spurious NDL stall or no_stall_c_err_rpts */
+	reg = PAU_MISC_HOLD;
+	val = pau_read(pau, reg);
+	val = SETFIELD(PAU_MISC_HOLD_NDL_STALL, val, 0b0000);
+	pau_write(pau, reg, val);
+
+	reg = PAU_MISC_CONFIG;
+	val = pau_read(pau, reg);
+	val |= PAU_MISC_CONFIG_OC_MODE;
+	pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_set_npcq_config(struct pau *pau)
+{
+	struct pau_dev *dev;
+	uint8_t oc_typemap = 0;
+	uint64_t reg, val;
+
+	/* MCP_MISC_CFG0
+	 * SNP_MISC_CFG0 done in pau_opencapi_enable_pb
+	 */
+	pau_for_each_opencapi_dev(dev, pau)
+		oc_typemap |= 0x10 >> dev->index;
+
+	PAUDBG(pau, "Set NPCQ Config\n");
+	reg = PAU_CTL_MISC_CFG2;
+	val = pau_read(pau, reg);
+	val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_MODE, val, oc_typemap);
+	val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_4, val, oc_typemap);
+	val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_C2, val, oc_typemap);
+	val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_AMO, val, oc_typemap);
+	val = SETFIELD(PAU_CTL_MISC_CFG2_OCAPI_MEM_OS_BIT, val, oc_typemap);
+	pau_write(pau, reg, val);
+
+	reg = PAU_DAT_MISC_CFG1;
+	val = pau_read(pau, reg);
+	val = SETFIELD(PAU_DAT_MISC_CFG1_OCAPI_MODE, val, oc_typemap);
+	pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_enable_xsl_xts_interfaces(struct pau *pau)
+{
+	uint64_t reg, val;
+
+	PAUDBG(pau, "Enable XSL-XTS Interfaces\n");
+	reg = PAU_XTS_CFG;
+	val = pau_read(pau, reg);
+	val |= PAU_XTS_CFG_OPENCAPI;
+	pau_write(pau, reg, val);
+
+	reg = PAU_XTS_CFG2;
+	val = pau_read(pau, reg);
+	val |= PAU_XTS_CFG2_XSL2_ENA;
+	pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_enable_sm_allocation(struct pau *pau)
+{
+	uint64_t reg, val;
+
+	PAUDBG(pau, "Enable State Machine Allocation\n");
+
+	reg = PAU_MISC_MACHINE_ALLOC;
+	val = pau_read(pau, reg);
+	val |= PAU_MISC_MACHINE_ALLOC_ENABLE;
+	pau_write(pau, reg, val);
+}
+
+static void pau_opencapi_enable_powerbus(struct pau *pau)
+{
+	struct pau_dev *dev;
+	uint8_t oc_typemap = 0;
+	uint64_t reg, val;
+
+	PAUDBG(pau, "Enable PowerBus\n");
+
+	pau_for_each_opencapi_dev(dev, pau)
+		oc_typemap |= 0x10 >> dev->index;
+
+	/* PowerBus interfaces must be enabled prior to MMIO */
+	reg = PAU_MCP_MISC_CFG0;
+	val = pau_read(pau, reg);
+	val |= PAU_MCP_MISC_CFG0_ENABLE_PBUS;
+	val |= PAU_MCP_MISC_CFG0_MA_MCRESP_OPT_WRP;
+	val = SETFIELD(PAU_MCP_MISC_CFG0_OCAPI_MODE, val, oc_typemap);
+	pau_write(pau, reg, val);
+
+	reg = PAU_SNP_MISC_CFG0;
+	val = pau_read(pau, reg);
+	val |= PAU_SNP_MISC_CFG0_ENABLE_PBUS;
+	val = SETFIELD(PAU_SNP_MISC_CFG0_OCAPI_MODE, val, oc_typemap);
+	val = SETFIELD(PAU_SNP_MISC_CFG0_OCAPI_C2, val, oc_typemap);
+	pau_write(pau, reg, val);
+}
+
 static void pau_opencapi_init_hw(struct pau *pau)
 {
 	struct pau_dev *dev = NULL;
@@ -482,9 +678,35 @@ static void pau_opencapi_init_hw(struct pau *pau)
 
 	/* Create phb */
 	pau_for_each_opencapi_dev(dev, pau) {
+		PAUDEVINF(dev, "Create phb\n");
 		pau_opencapi_create_phb(dev);
+		pau_opencapi_enable_bars(dev, true);
 		pau_opencapi_dt_add_props(dev);
 	}
+
+	/* Procedure 17.1.3.1 - Enabling OpenCAPI */
+	pau_for_each_opencapi_dev(dev, pau) {
+		PAUDEVINF(dev, "Configuring link ...\n");
+		pau_opencapi_set_transport_mux_controls(dev);	/* step 1 */
+	}
+	pau_opencapi_enable_xsl_clocks(pau);		/* step 2 */
+	pau_opencapi_enable_misc_clocks(pau);		/* step 3 */
+
+	/* OTL disabled */
+	pau_for_each_opencapi_dev(dev, pau)
+		pau_opencapi_set_fence_control(dev, 0b01);
+
+	pau_opencapi_set_npcq_config(pau);		/* step 4 */
+	pau_opencapi_enable_xsl_xts_interfaces(pau);	/* step 5 */
+	pau_opencapi_enable_sm_allocation(pau);		/* step 6 */
+	pau_opencapi_enable_powerbus(pau);		/* step 7 */
+
+	/*
+	 * access to the PAU registers through mmio requires setting
+	 * up the PAU mmio BAR (in pau_opencapi_assign_bars() above)
+	 * and machine state allocation
+	 */
+	pau->mmio_access = true;
 }
 
 static void pau_opencapi_init(struct pau *pau)
diff --git a/include/pau-regs.h b/include/pau-regs.h
index 5779692..6aeb758 100644
--- a/include/pau-regs.h
+++ b/include/pau-regs.h
@@ -27,6 +27,10 @@
 
 #define PAU_BLOCK_CQ_SM(n)			PAU_BLOCK(4, (n))
 #define PAU_BLOCK_CQ_CTL			PAU_BLOCK(4, 4)
+#define PAU_BLOCK_CQ_DAT			PAU_BLOCK(4, 5)
+#define PAU_BLOCK_XSL				PAU_BLOCK(4, 0xE)
+#define PAU_BLOCK_PAU_XTS			PAU_BLOCK(7, 1)
+#define PAU_BLOCK_PAU_MISC			PAU_BLOCK(7, 2)
 
 /*
  * CQ_SM block registers
@@ -37,21 +41,38 @@
 #define PAU_MCP_MISC_CFG0			(PAU_BLOCK_CQ_SM(0) + 0x000)
 #define   PAU_MCP_MISC_CFG0_MA_MCRESP_OPT_WRP	PPC_BIT(9)
 #define   PAU_MCP_MISC_CFG0_ENABLE_PBUS		PPC_BIT(26)
+#define   PAU_MCP_MISC_CFG0_OCAPI_MODE		PPC_BITMASK(44, 48)
 #define PAU_SNP_MISC_CFG0			(PAU_BLOCK_CQ_SM(0) + 0x180)
 #define   PAU_SNP_MISC_CFG0_ENABLE_PBUS		PPC_BIT(2)
+#define   PAU_SNP_MISC_CFG0_OCAPI_MODE		PPC_BITMASK(32, 36)
+#define   PAU_SNP_MISC_CFG0_OCAPI_C2		PPC_BITMASK(45, 49)
 #define PAU_NTL_BAR(brk)			(PAU_BLOCK_CQ_SM(0) + 0x1b8 + (brk) * 8)
+#define   PAU_NTL_BAR_ENABLE			PPC_BIT(0)
 #define   PAU_NTL_BAR_ADDR			PPC_BITMASK(3, 35)
 #define   PAU_NTL_BAR_SIZE			PPC_BITMASK(39, 43)
 #define PAU_MMIO_BAR				(PAU_BLOCK_CQ_SM(0) + 0x1e0)
 #define   PAU_MMIO_BAR_ENABLE			PPC_BIT(0)
 #define   PAU_MMIO_BAR_ADDR			PPC_BITMASK(3, 27)
 #define PAU_GENID_BAR				(PAU_BLOCK_CQ_SM(0) + 0x1e8)
+#define   PAU_GENID_BAR_ENABLE			PPC_BIT(0)
 #define   PAU_GENID_BAR_ADDR			PPC_BITMASK(3, 32)
+#define PAU_MISC_MACHINE_ALLOC			(PAU_BLOCK_CQ_SM(0) + 0x268)
+#define   PAU_MISC_MACHINE_ALLOC_ENABLE		PPC_BIT(0)
 
 /* CQ_CTL block registers */
+#define PAU_CTL_MISC_CFG2			(PAU_BLOCK_CQ_CTL + 0x010)
+#define   PAU_CTL_MISC_CFG2_OCAPI_MODE		PPC_BITMASK(0, 4)
+#define   PAU_CTL_MISC_CFG2_OCAPI_4		PPC_BITMASK(10, 14)
+#define   PAU_CTL_MISC_CFG2_OCAPI_C2		PPC_BITMASK(15, 19)
+#define   PAU_CTL_MISC_CFG2_OCAPI_AMO		PPC_BITMASK(20, 24)
+#define   PAU_CTL_MISC_CFG2_OCAPI_MEM_OS_BIT	PPC_BITMASK(25, 29)
+#define PAU_CTL_MISC_STATUS(brk)		(PAU_BLOCK_CQ_CTL + 0x060 + (brk) * 8)
+#define   PAU_CTL_MISC_STATUS_AM_FENCED(brk)	(PPC_BITMASK(41, 42) << ((brk)*32))
 #define PAU_CTL_MISC_MMIOPA_CONFIG(brk)		(PAU_BLOCK_CQ_CTL + 0x098 + (brk) * 8)
 #define   PAU_CTL_MISC_MMIOPA_CONFIG_BAR_ADDR	PPC_BITMASK(1, 35)
 #define   PAU_CTL_MISC_MMIOPA_CONFIG_BAR_SIZE	PPC_BITMASK(39, 43)
+#define PAU_CTL_MISC_FENCE_CTRL(brk)		(PAU_BLOCK_CQ_CTL + 0x108 + (brk) * 8)
+#define   PAU_CTL_MISC_FENCE_REQUEST		PPC_BITMASK(0, 1)
 #define PAU_CTL_MISC_CFG_ADDR(brk)		(PAU_BLOCK_CQ_CTL + 0x250 + (brk) * 8)
 #define   PAU_CTL_MISC_CFG_ADDR_ENABLE		PPC_BIT(0)
 #define   PAU_CTL_MISC_CFG_ADDR_STATUS		PPC_BITMASK(1, 3)
@@ -61,4 +82,26 @@
 #define   PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR	PPC_BITMASK(20, 31)
 #define   PAU_CTL_MISC_CFG_ADDR_TYPE		PPC_BIT(32)
 
+/* CQ_DAT block registers */
+#define PAU_DAT_MISC_CFG1			(PAU_BLOCK_CQ_DAT + 0x008)
+#define   PAU_DAT_MISC_CFG1_OCAPI_MODE		PPC_BITMASK(40, 44)
+
+/* XSL block registers */
+#define PAU_XSL_WRAP_CFG			(PAU_BLOCK_XSL + 0x100)
+#define   PAU_XSL_WRAP_CFG_CLOCK_ENABLE		PPC_BIT(0)
+
+/* XTS block registers */
+#define PAU_XTS_CFG				(PAU_BLOCK_PAU_XTS + 0x020)
+#define   PAU_XTS_CFG_OPENCAPI			PPC_BIT(15)
+#define PAU_XTS_CFG2				(PAU_BLOCK_PAU_XTS + 0x028)
+#define   PAU_XTS_CFG2_XSL2_ENA			PPC_BIT(55)
+
+/* MISC block registers */
+#define PAU_MISC_OPTICAL_IO_CONFIG		(PAU_BLOCK_PAU_MISC + 0x018)
+#define   PAU_MISC_OPTICAL_IO_CONFIG_OTL	PPC_BITMASK(2, 3)
+#define PAU_MISC_HOLD				(PAU_BLOCK_PAU_MISC + 0x020)
+#define   PAU_MISC_HOLD_NDL_STALL		PPC_BITMASK(0, 3)
+#define PAU_MISC_CONFIG				(PAU_BLOCK_PAU_MISC + 0x030)
+#define   PAU_MISC_CONFIG_OC_MODE		PPC_BIT(16)
+
 #endif /* __PAU_REGS_H */
diff --git a/include/pau.h b/include/pau.h
index fdf85f8..0246a63 100644
--- a/include/pau.h
+++ b/include/pau.h
@@ -22,6 +22,7 @@ enum pau_dev_type {
 
 /* Used to expose a hardware BAR (or logical slice of it) outside skiboot */
 struct pau_bar {
+	bool			enable;
 	uint64_t		addr;
 	uint64_t		size;
 	uint64_t		cfg;