npu2: Refactor NPU OPAL calls

Abstract the OPAL entry points for npu2, moving them to a new file. This
prepares us to add parallel npu3 versions of the same APIs.

No functional change.

Signed-off-by: Reza Arbab <arbab@linux.ibm.com>
Reviewed-by: Andrew Donnellan <ajd@linux.ibm.com>
Signed-off-by: Oliver O'Halloran <oohall@gmail.com>

4 files changed, 191 insertions, 132 deletions

diff --git a/hw/Makefile.inc b/hw/Makefile.inc
index 1ab1f40..2c8963c 100644
--- a/hw/Makefile.inc
+++ b/hw/Makefile.inc
@@ -8,6 +8,7 @@ HW_OBJS += dts.o lpc-rtc.o npu.o npu-hw-procedures.o xive.o phb4.o
 HW_OBJS += fake-nvram.o lpc-mbox.o npu2.o npu2-hw-procedures.o
 HW_OBJS += npu2-common.o npu2-opencapi.o phys-map.o sbe-p9.o capp.o
 HW_OBJS += occ-sensor.o vas.o sbe-p8.o dio-p9.o lpc-port80h.o cache-p9.o
+HW_OBJS += npu-opal.o
 HW=hw/built-in.a
 
 include $(SRC)/hw/fsp/Makefile.inc
diff --git a/hw/npu-opal.c b/hw/npu-opal.c
new file mode 100644
index 0000000..f106c73
--- /dev/null
+++ b/hw/npu-opal.c
@@ -0,0 +1,165 @@
+/* Copyright 2019 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *	http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <skiboot.h>
+#include <pci.h>
+#include <phb4.h>
+#include <npu2.h>
+
+static int64_t opal_npu_init_context(uint64_t phb_id, int pid __unused,
+				     uint64_t msr, uint64_t bdf)
+{
+	struct phb *phb = pci_get_phb(phb_id);
+
+	if (!phb || phb->phb_type != phb_type_npu_v2)
+		return OPAL_PARAMETER;
+
+	return npu2_init_context(phb, msr, bdf);
+}
+opal_call(OPAL_NPU_INIT_CONTEXT, opal_npu_init_context, 4);
+
+static int64_t opal_npu_destroy_context(uint64_t phb_id, uint64_t pid __unused,
+					uint64_t bdf)
+{
+	struct phb *phb = pci_get_phb(phb_id);
+
+	if (!phb || phb->phb_type != phb_type_npu_v2)
+		return OPAL_PARAMETER;
+
+	return npu2_destroy_context(phb, bdf);
+}
+opal_call(OPAL_NPU_DESTROY_CONTEXT, opal_npu_destroy_context, 3);
+
+static int64_t opal_npu_map_lpar(uint64_t phb_id, uint64_t bdf, uint64_t lparid,
+				 uint64_t lpcr)
+{
+	struct phb *phb = pci_get_phb(phb_id);
+
+	if (!phb || phb->phb_type != phb_type_npu_v2)
+		return OPAL_PARAMETER;
+
+	return npu2_map_lpar(phb, bdf, lparid, lpcr);
+}
+opal_call(OPAL_NPU_MAP_LPAR, opal_npu_map_lpar, 4);
+
+static int npu_check_relaxed_ordering(struct phb *phb, struct pci_device *pd,
+				      void *enable)
+{
+	/*
+	 * IBM PCIe bridge devices (ie. the root ports) can always allow relaxed
+	 * ordering
+	 */
+	if (pd->vdid == 0x04c11014)
+		pd->allow_relaxed_ordering = true;
+
+	PCIDBG(phb, pd->bdfn, "Checking relaxed ordering config\n");
+	if (pd->allow_relaxed_ordering)
+		return 0;
+
+	PCIDBG(phb, pd->bdfn, "Relaxed ordering not allowed\n");
+	*(bool *)enable = false;
+
+	return 1;
+}
+
+static int64_t npu_set_relaxed_order(uint32_t gcid, int pec, bool enable)
+{
+	struct phb *phb;
+	int64_t rc;
+
+	for_each_phb(phb) {
+		if (phb->phb_type != phb_type_npu_v2)
+			continue;
+
+		rc = npu2_set_relaxed_order(phb, gcid, pec, enable);
+		if (rc)
+			return rc;
+	}
+
+	return OPAL_SUCCESS;
+}
+
+static int64_t opal_npu_set_relaxed_order(uint64_t phb_id, uint16_t bdfn,
+					  bool request_enabled)
+{
+	struct phb *phb = pci_get_phb(phb_id);
+	struct phb4 *phb4;
+	uint32_t chip_id, pec;
+	struct pci_device *pd;
+	bool enable = true;
+
+	if (!phb || phb->phb_type != phb_type_pcie_v4)
+		return OPAL_PARAMETER;
+
+	phb4 = phb_to_phb4(phb);
+	pec = phb4->pec;
+	chip_id = phb4->chip_id;
+
+	if (chip_id & ~0x1b)
+		return OPAL_PARAMETER;
+
+	pd = pci_find_dev(phb, bdfn);
+	if (!pd)
+		return OPAL_PARAMETER;
+
+	/*
+	 * Not changing state, so no need to rescan PHB devices to determine if
+	 * we need to enable/disable it
+	 */
+	if (pd->allow_relaxed_ordering == request_enabled)
+		return OPAL_SUCCESS;
+
+	pd->allow_relaxed_ordering = request_enabled;
+
+	/*
+	 * Walk all devices on this PHB to ensure they all support relaxed
+	 * ordering
+	 */
+	pci_walk_dev(phb, NULL, npu_check_relaxed_ordering, &enable);
+
+	if (request_enabled && !enable) {
+		/*
+		 * Not all devices on this PHB support relaxed-ordering
+		 * mode so we can't enable it as requested
+		 */
+		prlog(PR_INFO, "Cannot set relaxed ordering for PEC %d on chip %d\n",
+		      pec, chip_id);
+		return OPAL_CONSTRAINED;
+	}
+
+	if (npu_set_relaxed_order(chip_id, pec, request_enabled)) {
+		npu_set_relaxed_order(chip_id, pec, false);
+		return OPAL_RESOURCE;
+	}
+
+	phb4->ro_state = request_enabled;
+	return OPAL_SUCCESS;
+}
+opal_call(OPAL_NPU_SET_RELAXED_ORDER, opal_npu_set_relaxed_order, 3);
+
+static int64_t opal_npu_get_relaxed_order(uint64_t phb_id,
+					  uint16_t bdfn __unused)
+{
+	struct phb *phb = pci_get_phb(phb_id);
+	struct phb4 *phb4;
+
+	if (!phb || phb->phb_type != phb_type_pcie_v4)
+		return OPAL_PARAMETER;
+
+	phb4 = phb_to_phb4(phb);
+	return phb4->ro_state;
+}
+opal_call(OPAL_NPU_GET_RELAXED_ORDER, opal_npu_get_relaxed_order, 2);
diff --git a/hw/npu2.c b/hw/npu2.c
index a904ccb..29c998f 100644
--- a/hw/npu2.c
+++ b/hw/npu2.c
@@ -1930,17 +1930,12 @@ static int npu_table_search(struct npu2 *p, uint64_t table_addr, int stride,
  * allocated.
  */
 #define NPU2_VALID_ATS_MSR_BITS (MSR_DR | MSR_HV | MSR_PR | MSR_SF)
-static int64_t opal_npu_init_context(uint64_t phb_id, int pasid __unused,
-				     uint64_t msr, uint64_t bdf)
+int64_t npu2_init_context(struct phb *phb, uint64_t msr, uint64_t bdf)
 {
-	struct phb *phb = pci_get_phb(phb_id);
 	struct npu2 *p;
 	uint64_t xts_bdf, old_xts_bdf_pid, xts_bdf_pid;
 	int id;
 
-	if (!phb || phb->phb_type != phb_type_npu_v2)
-		return OPAL_PARAMETER;
-
 	/*
 	 * MSR bits should be masked by the caller to allow for future
 	 * expansion if required.
@@ -2018,19 +2013,13 @@ out:
 	unlock(&p->lock);
 	return id;
 }
-opal_call(OPAL_NPU_INIT_CONTEXT, opal_npu_init_context, 4);
 
-static int opal_npu_destroy_context(uint64_t phb_id, uint64_t pid __unused,
-				    uint64_t bdf)
+int64_t npu2_destroy_context(struct phb *phb, uint64_t bdf)
 {
-	struct phb *phb = pci_get_phb(phb_id);
 	struct npu2 *p;
 	uint64_t xts_bdf;
 	int rc = OPAL_PARAMETER, id;
 
-	if (!phb || phb->phb_type != phb_type_npu_v2)
-		return OPAL_PARAMETER;
-
 	p = phb_to_npu2_nvlink(phb);
 	lock(&p->lock);
 
@@ -2059,15 +2048,13 @@ static int opal_npu_destroy_context(uint64_t phb_id, uint64_t pid __unused,
 	unlock(&p->lock);
 	return rc;
 }
-opal_call(OPAL_NPU_DESTROY_CONTEXT, opal_npu_destroy_context, 3);
 
 /*
  * Map the given virtual bdf to lparid with given lpcr.
  */
-static int opal_npu_map_lpar(uint64_t phb_id, uint64_t bdf, uint64_t lparid,
-			     uint64_t lpcr)
+int64_t npu2_map_lpar(struct phb *phb, uint64_t bdf, uint64_t lparid,
+		      uint64_t lpcr)
 {
-	struct phb *phb = pci_get_phb(phb_id);
 	struct npu2 *p;
 	struct npu2_dev *ndev = NULL;
 	uint64_t xts_bdf_lpar, atsd_lpar, rc = OPAL_SUCCESS;
@@ -2080,9 +2067,6 @@ static int opal_npu_map_lpar(uint64_t phb_id, uint64_t bdf, uint64_t lparid,
 		NPU2_XTS_MMIO_ATSD6_LPARID, NPU2_XTS_MMIO_ATSD7_LPARID
 	};
 
-	if (!phb || phb->phb_type != phb_type_npu_v2)
-		return OPAL_PARAMETER;
-
 	if (lpcr)
 		/* The LPCR bits are only required for hash based ATS,
 		 * which we don't currently support but may need to in
@@ -2165,7 +2149,6 @@ out:
 	unlock(&p->lock);
 	return rc;
 }
-opal_call(OPAL_NPU_MAP_LPAR, opal_npu_map_lpar, 4);
 
 static inline uint32_t npu2_relaxed_ordering_source_grpchp(uint32_t gcid)
 {
@@ -2311,123 +2294,25 @@ static void npu2_disable_relaxed_ordering(struct npu2_dev *ndev, uint32_t gcid,
  * relaxed ordering partially enabled if there are insufficient HW resources to
  * enable it on all links.
  */
-static int npu2_set_relaxed_ordering(uint32_t gcid, int pec, bool enable)
+int64_t npu2_set_relaxed_order(struct phb *phb, uint32_t gcid, int pec,
+			       bool enable)
 {
-	int rc = OPAL_SUCCESS;
-	struct phb *phb;
-	struct npu2 *npu;
+	struct npu2 *npu = phb_to_npu2_nvlink(phb);
 	struct npu2_dev *ndev;
+	int64_t rc = OPAL_SUCCESS;
 
-	for_each_phb(phb) {
-		if (phb->phb_type != phb_type_npu_v2)
-			continue;
+	for (int i = 0; i < npu->total_devices; i++) {
+		ndev = &npu->devices[i];
+		if (enable)
+			rc = npu2_enable_relaxed_ordering(ndev, gcid, pec);
+		else
+			npu2_disable_relaxed_ordering(ndev, gcid, pec);
 
-		npu = phb_to_npu2_nvlink(phb);
-		for (int i = 0; i < npu->total_devices; i++) {
-			ndev = &npu->devices[i];
-			if (enable)
-				rc = npu2_enable_relaxed_ordering(ndev, gcid, pec);
-			else
-				npu2_disable_relaxed_ordering(ndev, gcid, pec);
-
-			if (rc != OPAL_SUCCESS) {
-				NPU2DEVINF(ndev, "Insufficient resources to activate relaxed ordering mode\n");
-				return OPAL_RESOURCE;
-			}
+		if (rc != OPAL_SUCCESS) {
+			NPU2DEVINF(ndev, "Insufficient resources to activate relaxed ordering mode\n");
+			return OPAL_RESOURCE;
 		}
 	}
 
 	return OPAL_SUCCESS;
 }
-
-static int npu2_check_relaxed_ordering(struct phb *phb __unused,
-				       struct pci_device *pd, void *enable)
-{
-	/*
-	 * IBM PCIe bridge devices (ie. the root ports) can always allow relaxed
-	 * ordering
-	 */
-	if (pd->vdid == 0x04c11014)
-		pd->allow_relaxed_ordering = true;
-
-	PCIDBG(phb, pd->bdfn, "Checking relaxed ordering config\n");
-	if (pd->allow_relaxed_ordering)
-		return 0;
-
-	PCIDBG(phb, pd->bdfn, "Relaxed ordering not allowed\n");
-	*(bool *) enable = false;
-
-	return 1;
-}
-
-static int64_t opal_npu_set_relaxed_order(uint64_t phb_id, uint16_t bdfn,
-					  bool request_enabled)
-{
-	struct phb *phb = pci_get_phb(phb_id);
-	struct phb4 *phb4;
-	uint32_t chip_id, pec;
-	struct pci_device *pd;
-	bool enable = true;
-
-	if (!phb || phb->phb_type != phb_type_pcie_v4)
-		return OPAL_PARAMETER;
-
-	phb4 = phb_to_phb4(phb);
-	pec = phb4->pec;
-	chip_id = phb4->chip_id;
-
-	if (npu2_relaxed_ordering_source_grpchp(chip_id) == OPAL_PARAMETER)
-		return OPAL_PARAMETER;
-
-	pd = pci_find_dev(phb, bdfn);
-	if (!pd)
-		return OPAL_PARAMETER;
-
-	/*
-	 * Not changing state, so no need to rescan PHB devices to determine if
-	 * we need to enable/disable it
-	 */
-	if (pd->allow_relaxed_ordering == request_enabled)
-		return OPAL_SUCCESS;
-
-	pd->allow_relaxed_ordering = request_enabled;
-
-	/*
-	 * Walk all devices on this PHB to ensure they all support relaxed
-	 * ordering
-	 */
-	pci_walk_dev(phb, NULL, npu2_check_relaxed_ordering, &enable);
-
-	if (request_enabled && !enable) {
-		/*
-		 * Not all devices on this PHB support relaxed-ordering
-		 * mode so we can't enable it as requested
-		 */
-		prlog(PR_INFO, "Cannot set relaxed ordering for PEC %d on chip %d\n",
-		      pec, chip_id);
-		return OPAL_CONSTRAINED;
-	}
-
-	if (npu2_set_relaxed_ordering(chip_id, pec, request_enabled) != OPAL_SUCCESS) {
-		npu2_set_relaxed_ordering(chip_id, pec, false);
-		return OPAL_RESOURCE;
-	}
-
-	phb4->ro_state = request_enabled;
-	return OPAL_SUCCESS;
-}
-opal_call(OPAL_NPU_SET_RELAXED_ORDER, opal_npu_set_relaxed_order, 3);
-
-static int64_t opal_npu_get_relaxed_order(uint64_t phb_id,
-					  uint16_t bdfn __unused)
-{
-	struct phb *phb = pci_get_phb(phb_id);
-	struct phb4 *phb4;
-
-	if (!phb || phb->phb_type != phb_type_pcie_v4)
-		return OPAL_PARAMETER;
-
-	phb4 = phb_to_phb4(phb);
-	return phb4->ro_state;
-}
-opal_call(OPAL_NPU_GET_RELAXED_ORDER, opal_npu_get_relaxed_order, 2);
diff --git a/include/npu2.h b/include/npu2.h
index b63eba8..92b5898 100644
--- a/include/npu2.h
+++ b/include/npu2.h
@@ -243,4 +243,12 @@ int64_t npu2_freeze_status(struct phb *phb __unused,
 			   uint16_t *pci_error_type __unused,
 			   uint16_t *severity __unused);
 void npu2_dump_scoms(int chip_id);
+
+int64_t npu2_init_context(struct phb *phb, uint64_t msr, uint64_t bdf);
+int64_t npu2_destroy_context(struct phb *phb, uint64_t bdf);
+int64_t npu2_map_lpar(struct phb *phb, uint64_t bdf, uint64_t lparid,
+		      uint64_t lpcr);
+int64_t npu2_set_relaxed_order(struct phb *phb, uint32_t gcid, int pec,
+			       bool enable);
+
 #endif /* __NPU2_H */