From 882e867012a8c4af8a7bbd42b93816b2a2f3b827 Mon Sep 17 00:00:00 2001 From: Christophe Lombard Date: Thu, 14 Oct 2021 17:56:54 +0200 Subject: pau: create phb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the necessary operations for the OpenCAPI PHB type and inform the device-tree properties associated. The OpenCapi PCI config Addr/Data registers are reachable through the Generation-ID Registers MMIO BARS. The Config Address and Data registers are located at the following offsets from the AFU Config BAR plus 320 KB. • Config Address for Brick 0 – Offset 0 • Config Data for Brick 0 – Offsets: ◦ 128 – 4-byte config register • Config Address for Brick 1 – Offset 256 • Config Data for Brick 1 – Offsets: ◦ 384 – 4-byte config register Signed-off-by: Christophe Lombard Reviewed-by: Frederic Barrat Signed-off-by: Vasant Hegde --- core/pci-opal.c | 9 ++- core/pci.c | 4 +- hw/pau.c | 234 ++++++++++++++++++++++++++++++++++++++++++++++++++++- include/pau-regs.h | 8 ++ include/pau.h | 13 +++ include/pci.h | 1 + 6 files changed, 264 insertions(+), 5 deletions(-) diff --git a/core/pci-opal.c b/core/pci-opal.c index aa375c6..acbcd2a 100644 --- a/core/pci-opal.c +++ b/core/pci-opal.c @@ -748,7 +748,8 @@ static void rescan_slot_devices(struct pci_slot *slot) * prepare_link_change() is called (if needed) by the state * machine during the slot reset or link polling */ - if (phb->phb_type != phb_type_npu_v2_opencapi) { + if ((phb->phb_type != phb_type_npu_v2_opencapi) && + (phb->phb_type != phb_type_pau_opencapi)) { pci_scan_bus(phb, pd->secondary_bus, pd->subordinate_bus, &pd->children, pd, true); pci_add_device_nodes(phb, &pd->children, pd->dn, @@ -766,7 +767,8 @@ static void remove_slot_devices(struct pci_slot *slot) struct phb *phb = slot->phb; struct pci_device *pd = slot->pd; - if (phb->phb_type != phb_type_npu_v2_opencapi) + if ((phb->phb_type != phb_type_npu_v2_opencapi) && + (phb->phb_type != phb_type_pau_opencapi)) pci_remove_bus(phb, &pd->children); else pci_remove_bus(phb, &phb->devices); @@ -817,7 +819,8 @@ static bool training_needed(struct pci_slot *slot) struct pci_device *pd = slot->pd; /* only for opencapi slots for now */ - if (!pd && phb->phb_type == phb_type_npu_v2_opencapi) + if (!pd && ((phb->phb_type == phb_type_npu_v2_opencapi) || + (phb->phb_type == phb_type_pau_opencapi))) return true; return false; } diff --git a/core/pci.c b/core/pci.c index e195ecb..0a146c8 100644 --- a/core/pci.c +++ b/core/pci.c @@ -1517,7 +1517,9 @@ static void __noinline pci_add_one_device_node(struct phb *phb, * device has a 4KB config space. It's got nothing to do with the * standard Type 0/1 config spaces defined by PCI. */ - if (is_pcie || phb->phb_type == phb_type_npu_v2_opencapi) { + if (is_pcie || + (phb->phb_type == phb_type_npu_v2_opencapi) || + (phb->phb_type == phb_type_pau_opencapi)) { snprintf(compat, MAX_NAME, "pciex%x,%x", PCI_VENDOR_ID(pd->vdid), PCI_DEVICE_ID(pd->vdid)); dt_add_property_cells(np, "ibm,pci-config-space-type", 1); diff --git a/hw/pau.c b/hw/pau.c index 5d4b157..1a370a9 100644 --- a/hw/pau.c +++ b/hw/pau.c @@ -2,12 +2,18 @@ * Copyright 2021 IBM Corp. */ +#include +#include #include #include #include +/* Number of PEs supported */ +#define PAU_MAX_PE_NUM 16 +#define PAU_RESERVED_PE_NUM 15 + struct pau_dev *pau_next_dev(struct pau *pau, struct pau_dev *dev, - enum pau_dev_type type) + enum pau_dev_type type) { uint32_t i = 0; @@ -250,9 +256,235 @@ static void pau_opencapi_assign_bars(struct pau *pau) } } +static void pau_opencapi_create_phb_slot(struct pau_dev *dev) +{ + struct pci_slot *slot; + + slot = pci_slot_alloc(&dev->phb, NULL); + if (!slot) { + /** + * @fwts-label OCAPICannotCreatePHBSlot + * @fwts-advice Firmware probably ran out of memory creating + * PAU slot. OpenCAPI functionality could be broken. + */ + PAUDEVERR(dev, "Cannot create PHB slot\n"); + } +} + +static int64_t pau_opencapi_pcicfg_check(struct pau_dev *dev, + uint32_t offset, + uint32_t size) +{ + if (!dev || offset > 0xfff || (offset & (size - 1))) + return OPAL_PARAMETER; + + return OPAL_SUCCESS; +} + +static int64_t pau_opencapi_pcicfg_read(struct phb *phb, uint32_t bdfn, + uint32_t offset, uint32_t size, + void *data) +{ + struct pau_dev *dev = pau_phb_to_opencapi_dev(phb); + uint64_t cfg_addr, genid_base; + int64_t rc; + + rc = pau_opencapi_pcicfg_check(dev, offset, size); + if (rc) + return rc; + + /* Config Address for Brick 0 – Offset 0 + * Config Address for Brick 1 – Offset 256 + */ + genid_base = dev->genid_bar.cfg + (dev->index << 8); + + cfg_addr = PAU_CTL_MISC_CFG_ADDR_ENABLE; + cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_BUS_NBR | + PAU_CTL_MISC_CFG_ADDR_DEVICE_NBR | + PAU_CTL_MISC_CFG_ADDR_FUNCTION_NBR, + cfg_addr, bdfn); + cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR, + cfg_addr, offset & ~3u); + + out_be64((uint64_t *)genid_base, cfg_addr); + sync(); + + switch (size) { + case 1: + *((uint8_t *)data) = + in_8((uint8_t *)(genid_base + 128 + (offset & 3))); + break; + case 2: + *((uint16_t *)data) = + in_le16((uint16_t *)(genid_base + 128 + (offset & 2))); + break; + case 4: + *((uint32_t *)data) = in_le32((uint32_t *)(genid_base + 128)); + break; + default: + return OPAL_PARAMETER; + } + + return OPAL_SUCCESS; +} + +#define PAU_OPENCAPI_PCI_CFG_READ(size, type) \ +static int64_t pau_opencapi_pcicfg_read##size(struct phb *phb, uint32_t bdfn, \ + uint32_t offset, type * data) \ +{ \ + /* Initialize data in case of error */ \ + *data = (type)0xffffffff; \ + return pau_opencapi_pcicfg_read(phb, bdfn, offset, sizeof(type), data); \ +} + +static int64_t pau_opencapi_pcicfg_write(struct phb *phb, uint32_t bdfn, + uint32_t offset, uint32_t size, + uint32_t data) +{ + struct pau_dev *dev = pau_phb_to_opencapi_dev(phb); + uint64_t genid_base, cfg_addr; + int64_t rc; + + rc = pau_opencapi_pcicfg_check(dev, offset, size); + if (rc) + return rc; + + /* Config Address for Brick 0 – Offset 0 + * Config Address for Brick 1 – Offset 256 + */ + genid_base = dev->genid_bar.cfg + (dev->index << 8); + + cfg_addr = PAU_CTL_MISC_CFG_ADDR_ENABLE; + cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_BUS_NBR | + PAU_CTL_MISC_CFG_ADDR_DEVICE_NBR | + PAU_CTL_MISC_CFG_ADDR_FUNCTION_NBR, + cfg_addr, bdfn); + cfg_addr = SETFIELD(PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR, + cfg_addr, offset & ~3u); + + out_be64((uint64_t *)genid_base, cfg_addr); + sync(); + + switch (size) { + case 1: + out_8((uint8_t *)(genid_base + 128 + (offset & 3)), data); + break; + case 2: + out_le16((uint16_t *)(genid_base + 128 + (offset & 2)), data); + break; + case 4: + out_le32((uint32_t *)(genid_base + 128), data); + break; + default: + return OPAL_PARAMETER; + } + + return OPAL_SUCCESS; +} + +#define PAU_OPENCAPI_PCI_CFG_WRITE(size, type) \ +static int64_t pau_opencapi_pcicfg_write##size(struct phb *phb, uint32_t bdfn, \ + uint32_t offset, type data) \ +{ \ + return pau_opencapi_pcicfg_write(phb, bdfn, offset, sizeof(type), data);\ +} + +PAU_OPENCAPI_PCI_CFG_READ(8, u8) +PAU_OPENCAPI_PCI_CFG_READ(16, u16) +PAU_OPENCAPI_PCI_CFG_READ(32, u32) +PAU_OPENCAPI_PCI_CFG_WRITE(8, u8) +PAU_OPENCAPI_PCI_CFG_WRITE(16, u16) +PAU_OPENCAPI_PCI_CFG_WRITE(32, u32) + +static const struct phb_ops pau_opencapi_ops = { + .cfg_read8 = pau_opencapi_pcicfg_read8, + .cfg_read16 = pau_opencapi_pcicfg_read16, + .cfg_read32 = pau_opencapi_pcicfg_read32, + .cfg_write8 = pau_opencapi_pcicfg_write8, + .cfg_write16 = pau_opencapi_pcicfg_write16, + .cfg_write32 = pau_opencapi_pcicfg_write32, +}; + +static void pau_opencapi_create_phb(struct pau_dev *dev) +{ + struct phb *phb = &dev->phb; + uint64_t mm_win[2]; + + mm_win[0] = dev->ntl_bar.addr; + mm_win[1] = dev->ntl_bar.size; + + phb->phb_type = phb_type_pau_opencapi; + phb->scan_map = 0; + + phb->ops = &pau_opencapi_ops; + phb->dt_node = dt_new_addr(dt_root, "pciex", mm_win[0]); + assert(phb->dt_node); + + pci_register_phb(phb, pau_get_opal_id(dev->pau->chip_id, + pau_get_phb_index(dev->pau->index, dev->index))); + pau_opencapi_create_phb_slot(dev); +} + +static void pau_opencapi_dt_add_mmio_window(struct pau_dev *dev) +{ + struct dt_node *dn = dev->phb.dt_node; + uint64_t mm_win[2]; + + mm_win[0] = dev->ntl_bar.addr; + mm_win[1] = dev->ntl_bar.size; + PAUDEVDBG(dev, "Setting AFU MMIO window to %016llx %016llx\n", + mm_win[0], mm_win[1]); + + dt_add_property(dn, "reg", mm_win, sizeof(mm_win)); + dt_add_property(dn, "ibm,mmio-window", mm_win, sizeof(mm_win)); + dt_add_property_cells(dn, "ranges", 0x02000000, + hi32(mm_win[0]), lo32(mm_win[0]), + hi32(mm_win[0]), lo32(mm_win[0]), + hi32(mm_win[1]), lo32(mm_win[1])); +} + +static void pau_opencapi_dt_add_props(struct pau_dev *dev) +{ + struct dt_node *dn = dev->phb.dt_node; + struct pau *pau = dev->pau; + + dt_add_property_strings(dn, + "compatible", + "ibm,power10-pau-opencapi-pciex", + "ibm,ioda3-pau-opencapi-phb", + "ibm,ioda2-npu2-opencapi-phb"); + + dt_add_property_cells(dn, "#address-cells", 3); + dt_add_property_cells(dn, "#size-cells", 2); + dt_add_property_cells(dn, "#interrupt-cells", 1); + dt_add_property_cells(dn, "bus-range", 0, 0xff); + dt_add_property_cells(dn, "clock-frequency", 0x200, 0); + dt_add_property_cells(dn, "interrupt-parent", get_ics_phandle()); + + dt_add_property_strings(dn, "device_type", "pciex"); + dt_add_property_cells(dn, "ibm,pau-index", pau->index); + dt_add_property_cells(dn, "ibm,chip-id", pau->chip_id); + dt_add_property_cells(dn, "ibm,xscom-base", pau->xscom_base); + dt_add_property_cells(dn, "ibm,npcq", pau->dt_node->phandle); + dt_add_property_cells(dn, "ibm,links", 1); + dt_add_property_cells(dn, "ibm,phb-diag-data-size", 0); + dt_add_property_cells(dn, "ibm,opal-num-pes", PAU_MAX_PE_NUM); + dt_add_property_cells(dn, "ibm,opal-reserved-pe", PAU_RESERVED_PE_NUM); + + pau_opencapi_dt_add_mmio_window(dev); +} + static void pau_opencapi_init_hw(struct pau *pau) { + struct pau_dev *dev = NULL; + pau_opencapi_assign_bars(pau); + + /* Create phb */ + pau_for_each_opencapi_dev(dev, pau) { + pau_opencapi_create_phb(dev); + pau_opencapi_dt_add_props(dev); + } } static void pau_opencapi_init(struct pau *pau) diff --git a/include/pau-regs.h b/include/pau-regs.h index afe6f95..5779692 100644 --- a/include/pau-regs.h +++ b/include/pau-regs.h @@ -52,5 +52,13 @@ #define PAU_CTL_MISC_MMIOPA_CONFIG(brk) (PAU_BLOCK_CQ_CTL + 0x098 + (brk) * 8) #define PAU_CTL_MISC_MMIOPA_CONFIG_BAR_ADDR PPC_BITMASK(1, 35) #define PAU_CTL_MISC_MMIOPA_CONFIG_BAR_SIZE PPC_BITMASK(39, 43) +#define PAU_CTL_MISC_CFG_ADDR(brk) (PAU_BLOCK_CQ_CTL + 0x250 + (brk) * 8) +#define PAU_CTL_MISC_CFG_ADDR_ENABLE PPC_BIT(0) +#define PAU_CTL_MISC_CFG_ADDR_STATUS PPC_BITMASK(1, 3) +#define PAU_CTL_MISC_CFG_ADDR_BUS_NBR PPC_BITMASK(4, 11) +#define PAU_CTL_MISC_CFG_ADDR_DEVICE_NBR PPC_BITMASK(12, 16) +#define PAU_CTL_MISC_CFG_ADDR_FUNCTION_NBR PPC_BITMASK(17, 19) +#define PAU_CTL_MISC_CFG_ADDR_REGISTER_NBR PPC_BITMASK(20, 31) +#define PAU_CTL_MISC_CFG_ADDR_TYPE PPC_BIT(32) #endif /* __PAU_REGS_H */ diff --git a/include/pau.h b/include/pau.h index be8ed26..fdf85f8 100644 --- a/include/pau.h +++ b/include/pau.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #define PAU_NBR 6 @@ -30,6 +31,7 @@ struct pau_dev { enum pau_dev_type type; uint32_t index; struct dt_node *dn; + struct phb phb; struct pau_bar ntl_bar; struct pau_bar genid_bar; @@ -86,6 +88,12 @@ static inline uint32_t pau_dev_index(struct pau_dev *dev, int links) return dev->pau->index * links + dev->index; } +static inline struct pau_dev *pau_phb_to_opencapi_dev(struct phb *phb) +{ + assert(phb->phb_type == phb_type_pau_opencapi); + return container_of(phb, struct pau_dev, phb); +} + struct pau_dev *pau_next_dev(struct pau *pau, struct pau_dev *dev, enum pau_dev_type type); @@ -105,6 +113,11 @@ static inline int pau_get_phb_index(unsigned int pau_index, return PAU_PHB_INDEX_BASE + pau_index * 2 + link_index; } +static inline int pau_get_opal_id(unsigned int chip_id, unsigned int index) +{ + return phb4_get_opal_id(chip_id, index); +} + /* * We use the indirect method because it uses the same addresses as * the MMIO offsets (PAU RING) diff --git a/include/pci.h b/include/pci.h index 8d46721..caae744 100644 --- a/include/pci.h +++ b/include/pci.h @@ -352,6 +352,7 @@ enum phb_type { phb_type_pcie_v4, phb_type_npu_v2, phb_type_npu_v2_opencapi, + phb_type_pau_opencapi, }; /* Generic PCI NVRAM flags */ -- cgit v1.1