pau: create phb

Implement the necessary operations for the OpenCAPI PHB type and
inform the device-tree properties associated.

The OpenCapi PCI config Addr/Data registers are reachable through
the Generation-ID Registers MMIO BARS.
The Config Address and Data registers are located at the following offsets
from the AFU Config BAR plus 320 KB.
• Config Address for Brick 0 – Offset 0
• Config Data for Brick 0 – Offsets:
◦ 128 – 4-byte config register

• Config Address for Brick 1 – Offset 256
• Config Data for Brick 1 – Offsets:
◦ 384 – 4-byte config register

Signed-off-by: Christophe Lombard <clombard@linux.vnet.ibm.com>

Reviewed-by: Frederic Barrat <fbarrat@linux.ibm.com>
Signed-off-by: Vasant Hegde <hegdevasant@linux.vnet.ibm.com>

6 files changed, 264 insertions, 5 deletions

diff --git a/core/pci-opal.c b/core/pci-opal.c
index aa375c6..acbcd2a 100644
--- a/core/pci-opal.c
+++ b/core/pci-opal.c
@@ -748,7 +748,8 @@ static void rescan_slot_devices(struct pci_slot *slot)
 	 * prepare_link_change() is called (if needed) by the state
 	 * machine during the slot reset or link polling
 	 */
-	if (phb->phb_type != phb_type_npu_v2_opencapi) {
+	if ((phb->phb_type != phb_type_npu_v2_opencapi) &&
+	    (phb->phb_type != phb_type_pau_opencapi)) {
 		pci_scan_bus(phb, pd->secondary_bus,
 			     pd->subordinate_bus, &pd->children, pd, true);
 		pci_add_device_nodes(phb, &pd->children, pd->dn,
@@ -766,7 +767,8 @@ static void remove_slot_devices(struct pci_slot *slot)
 	struct phb *phb = slot->phb;
 	struct pci_device *pd = slot->pd;
 
-	if (phb->phb_type != phb_type_npu_v2_opencapi)
+	if ((phb->phb_type != phb_type_npu_v2_opencapi) &&
+	    (phb->phb_type != phb_type_pau_opencapi))
 		pci_remove_bus(phb, &pd->children);
 	else
 		pci_remove_bus(phb, &phb->devices);
@@ -817,7 +819,8 @@ static bool training_needed(struct pci_slot *slot)
 	struct pci_device *pd = slot->pd;
 
 	/* only for opencapi slots for now */
-	if (!pd && phb->phb_type == phb_type_npu_v2_opencapi)
+	if (!pd && ((phb->phb_type == phb_type_npu_v2_opencapi) ||
+		    (phb->phb_type == phb_type_pau_opencapi)))
 		return true;
 	return false;
 }
diff --git a/core/pci.c b/core/pci.c
index e195ecb..0a146c8 100644
--- a/core/pci.c
+++ b/core/pci.c
@@ -1517,7 +1517,9 @@ static void __noinline pci_add_one_device_node(struct phb *phb,
 	 * device has a 4KB config space. It's got nothing to do with the
 	 * standard Type 0/1 config spaces defined by PCI.
 	 */
-	if (is_pcie || phb->phb_type == phb_type_npu_v2_opencapi) {
+	if (is_pcie ||
+		(phb->phb_type == phb_type_npu_v2_opencapi) ||
+		(phb->phb_type == phb_type_pau_opencapi)) {
 		snprintf(compat, MAX_NAME, "pciex%x,%x",
 			 PCI_VENDOR_ID(pd->vdid), PCI_DEVICE_ID(pd->vdid));
 		dt_add_property_cells(np, "ibm,pci-config-space-type", 1);
diff --git a/hw/pau.c b/hw/pau.c
index 5d4b157..1a370a9 100644
--- a/hw/pau.c
+++ b/hw/pau.c
@@ -2,12 +2,18 @@
  * Copyright 2021 IBM Corp.
  */
 
+#include <interrupts.h>
+#include <pci-slot.h>
 #include <phys-map.h>
 #include <pau.h>
 #include <pau-regs.h>
 
+/* Number of PEs supported */
+#define PAU_MAX_PE_NUM		16
+#define PAU_RESERVED_PE_NUM	15
+
 struct pau_dev *pau_next_dev(struct pau *pau, struct pau_dev *dev,
-			       enum pau_dev_type type)
+			     enum pau_dev_type type)
 {
 	uint32_t i = 0;
 
@@ -250,9 +256,235 @@ static void pau_opencapi_assign_bars(struct pau *pau)
 	}
 }
 
+static void pau_opencapi_create_phb_slot(struct pau_dev *dev)
+{
+	struct pci_slot *slot;
+
+	slot = pci_slot_alloc(&dev->phb, NULL);
+	if (!slot) {
+		/**
+		 * @fwts-label OCAPICannotCreatePHBSlot
+		 * @fwts-advice Firmware probably ran out of memory creating
+		 * PAU slot. OpenCAPI functionality could be broken.
+		 */
+		PAUDEVERR(dev, "Cannot create PHB slot\n");
+	}
+}
+
+static int64_t pau_opencapi_pcicfg_check(struct pau_dev *dev,
+					 uint32_t offset,
+					 uint32_t size)
+{
+	if (!dev || offset > 0xfff || (offset & (size - 1)))
+		return OPAL_PARAMETER;
+
+	return OPAL_SUCCESS;
+}
+
+static int64_t pau_opencapi_pcicfg_read(struct phb *phb, uint32_t bdfn,
+					uint32_t offset, uint32_t size,
+					void *data)
+{
+	struct pau_dev *dev = pau_phb_to_opencapi_dev(phb);
+	uint64_t cfg_addr, genid_base;
+	int64_t rc;
+
+	rc = pau_opencapi_pcicfg_check(dev, offset, size);
+	if (rc)
+		return rc;
+
+	/* Config Address for Brick 0 – Offset 0
+	 * Config Address for Brick 1 – Offset 256
+	 */
+	genid_base = dev->genid_bar.cfg + (dev->index << 8);
+
+	cfg_addr = PAU_CTL_MISC_CFG_ADDR_ENABLE;
+	cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_BUS_NBR |
+			    PAU_CTL_MISC_CFG_ADDR_DEVICE_NBR |
+			    PAU_CTL_MISC_CFG_ADDR_FUNCTION_NBR,
+			    cfg_addr, bdfn);
+	cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR,
+			    cfg_addr, offset & ~3u);
+
+	out_be64((uint64_t *)genid_base, cfg_addr);
+	sync();
+
+	switch (size) {
+	case 1:
+		*((uint8_t *)data) =
+			in_8((uint8_t *)(genid_base + 128 + (offset & 3)));
+		break;
+	case 2:
+		*((uint16_t *)data) =
+			in_le16((uint16_t *)(genid_base + 128 + (offset & 2)));
+		break;
+	case 4:
+		*((uint32_t *)data) = in_le32((uint32_t *)(genid_base + 128));
+		break;
+	default:
+		return OPAL_PARAMETER;
+	}
+
+	return OPAL_SUCCESS;
+}
+
+#define PAU_OPENCAPI_PCI_CFG_READ(size, type)					\
+static int64_t pau_opencapi_pcicfg_read##size(struct phb *phb, uint32_t bdfn,	\
+					      uint32_t offset, type * data)	\
+{										\
+	/* Initialize data in case of error */					\
+	*data = (type)0xffffffff;						\
+	return pau_opencapi_pcicfg_read(phb, bdfn, offset, sizeof(type), data);	\
+}
+
+static int64_t pau_opencapi_pcicfg_write(struct phb *phb, uint32_t bdfn,
+					 uint32_t offset, uint32_t size,
+					 uint32_t data)
+{
+	struct pau_dev *dev = pau_phb_to_opencapi_dev(phb);
+	uint64_t genid_base, cfg_addr;
+	int64_t rc;
+
+	rc = pau_opencapi_pcicfg_check(dev, offset, size);
+	if (rc)
+		return rc;
+
+	/* Config Address for Brick 0 – Offset 0
+	 * Config Address for Brick 1 – Offset 256
+	 */
+	genid_base = dev->genid_bar.cfg + (dev->index << 8);
+
+	cfg_addr = PAU_CTL_MISC_CFG_ADDR_ENABLE;
+	cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_BUS_NBR |
+			    PAU_CTL_MISC_CFG_ADDR_DEVICE_NBR |
+			    PAU_CTL_MISC_CFG_ADDR_FUNCTION_NBR,
+			    cfg_addr, bdfn);
+	cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR,
+			    cfg_addr, offset & ~3u);
+
+	out_be64((uint64_t *)genid_base, cfg_addr);
+	sync();
+
+	switch (size) {
+	case 1:
+		out_8((uint8_t *)(genid_base + 128 + (offset & 3)), data);
+		break;
+	case 2:
+		out_le16((uint16_t *)(genid_base + 128 + (offset & 2)), data);
+		break;
+	case 4:
+		out_le32((uint32_t *)(genid_base + 128), data);
+		break;
+	default:
+		return OPAL_PARAMETER;
+	}
+
+	return OPAL_SUCCESS;
+}
+
+#define PAU_OPENCAPI_PCI_CFG_WRITE(size, type)					\
+static int64_t pau_opencapi_pcicfg_write##size(struct phb *phb, uint32_t bdfn,	\
+						uint32_t offset, type data)	\
+{										\
+	return pau_opencapi_pcicfg_write(phb, bdfn, offset, sizeof(type), data);\
+}
+
+PAU_OPENCAPI_PCI_CFG_READ(8, u8)
+PAU_OPENCAPI_PCI_CFG_READ(16, u16)
+PAU_OPENCAPI_PCI_CFG_READ(32, u32)
+PAU_OPENCAPI_PCI_CFG_WRITE(8, u8)
+PAU_OPENCAPI_PCI_CFG_WRITE(16, u16)
+PAU_OPENCAPI_PCI_CFG_WRITE(32, u32)
+
+static const struct phb_ops pau_opencapi_ops = {
+	.cfg_read8		= pau_opencapi_pcicfg_read8,
+	.cfg_read16		= pau_opencapi_pcicfg_read16,
+	.cfg_read32		= pau_opencapi_pcicfg_read32,
+	.cfg_write8		= pau_opencapi_pcicfg_write8,
+	.cfg_write16		= pau_opencapi_pcicfg_write16,
+	.cfg_write32		= pau_opencapi_pcicfg_write32,
+};
+
+static void pau_opencapi_create_phb(struct pau_dev *dev)
+{
+	struct phb *phb = &dev->phb;
+	uint64_t mm_win[2];
+
+	mm_win[0] = dev->ntl_bar.addr;
+	mm_win[1] = dev->ntl_bar.size;
+
+	phb->phb_type = phb_type_pau_opencapi;
+	phb->scan_map = 0;
+
+	phb->ops = &pau_opencapi_ops;
+	phb->dt_node = dt_new_addr(dt_root, "pciex", mm_win[0]);
+	assert(phb->dt_node);
+
+	pci_register_phb(phb, pau_get_opal_id(dev->pau->chip_id,
+					      pau_get_phb_index(dev->pau->index, dev->index)));
+	pau_opencapi_create_phb_slot(dev);
+}
+
+static void pau_opencapi_dt_add_mmio_window(struct pau_dev *dev)
+{
+	struct dt_node *dn = dev->phb.dt_node;
+	uint64_t mm_win[2];
+
+	mm_win[0] = dev->ntl_bar.addr;
+	mm_win[1] = dev->ntl_bar.size;
+	PAUDEVDBG(dev, "Setting AFU MMIO window to %016llx  %016llx\n",
+			mm_win[0], mm_win[1]);
+
+	dt_add_property(dn, "reg", mm_win, sizeof(mm_win));
+	dt_add_property(dn, "ibm,mmio-window", mm_win, sizeof(mm_win));
+	dt_add_property_cells(dn, "ranges", 0x02000000,
+			      hi32(mm_win[0]), lo32(mm_win[0]),
+			      hi32(mm_win[0]), lo32(mm_win[0]),
+			      hi32(mm_win[1]), lo32(mm_win[1]));
+}
+
+static void pau_opencapi_dt_add_props(struct pau_dev *dev)
+{
+	struct dt_node *dn = dev->phb.dt_node;
+	struct pau *pau = dev->pau;
+
+	dt_add_property_strings(dn,
+				"compatible",
+				"ibm,power10-pau-opencapi-pciex",
+				"ibm,ioda3-pau-opencapi-phb",
+				"ibm,ioda2-npu2-opencapi-phb");
+
+	dt_add_property_cells(dn, "#address-cells", 3);
+	dt_add_property_cells(dn, "#size-cells", 2);
+	dt_add_property_cells(dn, "#interrupt-cells", 1);
+	dt_add_property_cells(dn, "bus-range", 0, 0xff);
+	dt_add_property_cells(dn, "clock-frequency", 0x200, 0);
+	dt_add_property_cells(dn, "interrupt-parent", get_ics_phandle());
+
+	dt_add_property_strings(dn, "device_type", "pciex");
+	dt_add_property_cells(dn, "ibm,pau-index", pau->index);
+	dt_add_property_cells(dn, "ibm,chip-id", pau->chip_id);
+	dt_add_property_cells(dn, "ibm,xscom-base", pau->xscom_base);
+	dt_add_property_cells(dn, "ibm,npcq", pau->dt_node->phandle);
+	dt_add_property_cells(dn, "ibm,links", 1);
+	dt_add_property_cells(dn, "ibm,phb-diag-data-size", 0);
+	dt_add_property_cells(dn, "ibm,opal-num-pes", PAU_MAX_PE_NUM);
+	dt_add_property_cells(dn, "ibm,opal-reserved-pe", PAU_RESERVED_PE_NUM);
+
+	pau_opencapi_dt_add_mmio_window(dev);
+}
+
 static void pau_opencapi_init_hw(struct pau *pau)
 {
+	struct pau_dev *dev = NULL;
+
 	pau_opencapi_assign_bars(pau);
+
+	/* Create phb */
+	pau_for_each_opencapi_dev(dev, pau) {
+		pau_opencapi_create_phb(dev);
+		pau_opencapi_dt_add_props(dev);
+	}
 }
 
 static void pau_opencapi_init(struct pau *pau)
diff --git a/include/pau-regs.h b/include/pau-regs.h
index afe6f95..5779692 100644
--- a/include/pau-regs.h
+++ b/include/pau-regs.h
@@ -52,5 +52,13 @@
 #define PAU_CTL_MISC_MMIOPA_CONFIG(brk)		(PAU_BLOCK_CQ_CTL + 0x098 + (brk) * 8)
 #define   PAU_CTL_MISC_MMIOPA_CONFIG_BAR_ADDR	PPC_BITMASK(1, 35)
 #define   PAU_CTL_MISC_MMIOPA_CONFIG_BAR_SIZE	PPC_BITMASK(39, 43)
+#define PAU_CTL_MISC_CFG_ADDR(brk)		(PAU_BLOCK_CQ_CTL + 0x250 + (brk) * 8)
+#define   PAU_CTL_MISC_CFG_ADDR_ENABLE		PPC_BIT(0)
+#define   PAU_CTL_MISC_CFG_ADDR_STATUS		PPC_BITMASK(1, 3)
+#define   PAU_CTL_MISC_CFG_ADDR_BUS_NBR		PPC_BITMASK(4, 11)
+#define   PAU_CTL_MISC_CFG_ADDR_DEVICE_NBR	PPC_BITMASK(12, 16)
+#define   PAU_CTL_MISC_CFG_ADDR_FUNCTION_NBR	PPC_BITMASK(17, 19)
+#define   PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR	PPC_BITMASK(20, 31)
+#define   PAU_CTL_MISC_CFG_ADDR_TYPE		PPC_BIT(32)
 
 #endif /* __PAU_REGS_H */
diff --git a/include/pau.h b/include/pau.h
index be8ed26..fdf85f8 100644
--- a/include/pau.h
+++ b/include/pau.h
@@ -8,6 +8,7 @@
 #include <io.h>
 #include <pci.h>
 #include <xscom.h>
+#include <phb4.h>
 #include <pau-regs.h>
 
 #define PAU_NBR 6
@@ -30,6 +31,7 @@ struct pau_dev {
 	enum pau_dev_type	type;
 	uint32_t		index;
 	struct dt_node		*dn;
+	struct phb		phb;
 
 	struct pau_bar		ntl_bar;
 	struct pau_bar		genid_bar;
@@ -86,6 +88,12 @@ static inline uint32_t pau_dev_index(struct pau_dev *dev, int links)
 	return dev->pau->index * links + dev->index;
 }
 
+static inline struct pau_dev *pau_phb_to_opencapi_dev(struct phb *phb)
+{
+	assert(phb->phb_type == phb_type_pau_opencapi);
+	return container_of(phb, struct pau_dev, phb);
+}
+
 struct pau_dev *pau_next_dev(struct pau *pau, struct pau_dev *dev,
 			       enum pau_dev_type type);
 
@@ -105,6 +113,11 @@ static inline int pau_get_phb_index(unsigned int pau_index,
 	return PAU_PHB_INDEX_BASE + pau_index * 2 + link_index;
 }
 
+static inline int pau_get_opal_id(unsigned int chip_id, unsigned int index)
+{
+	return phb4_get_opal_id(chip_id, index);
+}
+
 /*
  * We use the indirect method because it uses the same addresses as
  * the MMIO offsets (PAU RING)
diff --git a/include/pci.h b/include/pci.h
index 8d46721..caae744 100644
--- a/include/pci.h
+++ b/include/pci.h
@@ -352,6 +352,7 @@ enum phb_type {
 	phb_type_pcie_v4,
 	phb_type_npu_v2,
 	phb_type_npu_v2_opencapi,
+	phb_type_pau_opencapi,
 };
 
 /* Generic PCI NVRAM flags */