npu2: Rework NPU data structures for OpenCAPI

Unlike NVLink, OpenCAPI registers a separate PHB for each device, in order
to allow us to force Linux to use the correct MMIO windows for each NPU
link. This requires some reworking of NPU data structures to account for
the fact that a PHB could correspond to either an NPU (NVLink) or a single
link (OpenCAPI).

At some later point, we may want to rework the NVLink code to present a
separate PHB per device in order to simplify this. For now, we split
NVLink-specific device data into a separate struct in order to make it
clear which fields are NVLink-only.

Additionally, add helper functions to correctly translate between
OpenCAPI/NVLink PHBs and the underlying structures, and various fields
for OpenCAPI data that we're going to need later on.

Signed-off-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
Acked-by: Reza Arbab <arbab@linux.vnet.ibm.com>
Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>

2 files changed, 59 insertions, 19 deletions

diff --git a/include/npu2.h b/include/npu2.h
index a1d57e4..a8db9ed 100644
--- a/include/npu2.h
+++ b/include/npu2.h
@@ -21,14 +21,14 @@
 
 /* Debugging options */
 #define NPU2DBG(p, fmt, a...)	prlog(PR_DEBUG, "NPU%d: " fmt, \
-				      (p)->phb.opal_id, ##a)
+				      (p)->phb_nvlink.opal_id, ##a)
 #define NPU2INF(p, fmt, a...)	prlog(PR_INFO,  "NPU%d: " fmt, \
-				      (p)->phb.opal_id, ##a)
+				      (p)->phb_nvlink.opal_id, ##a)
 #define NPU2ERR(p, fmt, a...)	prlog(PR_ERR,   "NPU%d: " fmt, \
-				      (p)->phb.opal_id, ##a)
+				      (p)->phb_nvlink.opal_id, ##a)
 
 #define NPU2DEVLOG(l, p, fmt, a...)	prlog(l, "NPU%d:%d:%d.%d " fmt, \
-					      (p)->npu->phb.opal_id, \
+					      (p)->npu->phb_nvlink.opal_id, \
 					      ((p)->bdfn >> 8) & 0xff, \
 					      ((p)->bdfn >> 3) & 0x1f, \
 					      (p)->bdfn & 0x7, ##a)
@@ -80,18 +80,18 @@ struct npu2_pcie_bar {
 	struct npu2_bar		npu2_bar;
 };
 
+enum npu2_dev_type {
+	NPU2_DEV_TYPE_NVLINK,
+	NPU2_DEV_TYPE_OPENCAPI,
+};
+
 struct npu2;
-struct npu2_dev {
-	uint32_t		index;
-	uint64_t		pl_xscom_base;
-	struct dt_node		*dt_node;
-	struct npu2_pcie_bar	bars[2];
-	struct npu2		*npu;
 
-	/* Device and function numbers are allocated based on GPU
-	 * association. Links to connected to the same GPU will be
-	 * exposed as different functions of the same bus/device. */
-	uint32_t		bdfn;
+struct npu2_dev_nvlink {
+	/* For NVLink, device and function numbers are allocated based
+	 * on GPU association. Links to connected to the same GPU will
+	 * be exposed as different functions of the same
+	 * bus/device. */
 	uint32_t		gpu_bdfn;
 
 	/* PCI virtual device and the associated GPU device */
@@ -104,8 +104,23 @@ struct npu2_dev {
 	/* Vendor specific capability */
 	uint32_t		vendor_cap;
 
+	/* Used to associate the NPU device with GPU PCI devices */
+	const char		*slot_label;
+};
+
+struct npu2_dev {
+	enum npu2_dev_type	type;
+	uint32_t		index;
+	uint64_t		pl_xscom_base;
+	struct dt_node		*dt_node;
+	struct npu2_pcie_bar	bars[2];
+	struct npu2		*npu;
+
+	uint32_t		bdfn;
+
 	/* Which PHY lanes this device is associated with */
 	uint32_t		lane_mask;
+	uint64_t		link_speed; /* not used for NVLink */
 
 	/* Track currently running procedure and step number */
 	uint16_t		procedure_number;
@@ -114,8 +129,12 @@ struct npu2_dev {
 	unsigned long		procedure_tb;
 	uint32_t		procedure_status;
 
-	/* Used to associate the NPU device with GPU PCI devices */
-	const char		*slot_label;
+	/* NVLink */
+	struct npu2_dev_nvlink	nvlink;
+
+	/* OpenCAPI */
+	struct phb		phb_ocapi;
+	uint64_t		i2c_port_id_ocapi;
 };
 
 struct npu2 {
@@ -142,12 +161,32 @@ struct npu2 {
 	 * tables. */
 	struct lock	lock;
 
-	struct phb	phb;
+	/* NVLink */
+	struct phb	phb_nvlink;
 };
 
-static inline struct npu2 *phb_to_npu2(struct phb *phb)
+static inline struct npu2 *phb_to_npu2_nvlink(struct phb *phb)
+{
+	assert(phb->phb_type == phb_type_npu_v2);
+	return container_of(phb, struct npu2, phb_nvlink);
+}
+
+static inline struct npu2_dev *phb_to_npu2_dev_ocapi(struct phb *phb)
+{
+	assert(phb->phb_type == phb_type_npu_v2_opencapi);
+	return container_of(phb, struct npu2_dev, phb_ocapi);
+}
+
+static inline struct phb *npu2_dev_to_phb(struct npu2_dev *ndev)
 {
-	return container_of(phb, struct npu2, phb);
+	switch (ndev->type) {
+	case NPU2_DEV_TYPE_NVLINK:
+		return &ndev->npu->phb_nvlink;
+	case NPU2_DEV_TYPE_OPENCAPI:
+		return &ndev->phb_ocapi;
+	default:
+		assert(false);
+	}
 }
 
 void npu2_write_4b(struct npu2 *p, uint64_t reg, uint32_t val);
diff --git a/include/pci.h b/include/pci.h
index 508ebf4..17b5c96 100644
--- a/include/pci.h
+++ b/include/pci.h
@@ -344,6 +344,7 @@ enum phb_type {
 	phb_type_pcie_v3,
 	phb_type_pcie_v4,
 	phb_type_npu_v2,
+	phb_type_npu_v2_opencapi,
 };
 
 struct phb {