// SPDX-License-Identifier: Apache-2.0 /* * Copyright 2019 IBM Corp. */ #include #include #include #include #include #include #include #include #include #include #include #include #define NPU3LOG(l, npu, fmt, a...) \ prlog(l, "NPU#%04x[%d:%d]: " fmt, \ (npu)->nvlink.phb.opal_id, \ (npu)->chip_id, \ (npu)->index, ##a) #define NPU3DBG(npu, fmt, a...) NPU3LOG(PR_DEBUG, npu, fmt, ##a) #define NPU3INF(npu, fmt, a...) NPU3LOG(PR_INFO, npu, fmt, ##a) #define NPU3ERR(npu, fmt, a...) NPU3LOG(PR_ERR, npu, fmt, ##a) #define NPU3DEVLOG(l, dev, fmt, a...) \ prlog(l, "NPU#%04x:%02x:%02x.%x " fmt, \ (dev)->npu->nvlink.phb.opal_id, \ PCI_BUS_NUM((dev)->nvlink.pvd->bdfn), \ PCI_DEV((dev)->nvlink.pvd->bdfn), \ PCI_FUNC((dev)->nvlink.pvd->bdfn), ##a) #define NPU3DEVDBG(dev, fmt, a...) NPU3DEVLOG(PR_DEBUG, dev, fmt, ##a) #define NPU3DEVINF(dev, fmt, a...) NPU3DEVLOG(PR_INFO, dev, fmt, ##a) #define NPU3DEVERR(dev, fmt, a...) NPU3DEVLOG(PR_ERR, dev, fmt, ##a) #define NPU3_CFG_READ(size, type) \ static int64_t npu3_cfg_read##size(struct phb *phb, uint32_t bdfn, \ uint32_t offset, type *data) \ { \ uint32_t val; \ int64_t ret; \ \ ret = pci_virt_cfg_read(phb, bdfn, offset, \ sizeof(*data), &val); \ *data = (type)val; \ return ret; \ } #define NPU3_CFG_WRITE(size, type) \ static int64_t npu3_cfg_write##size(struct phb *phb, uint32_t bdfn, \ uint32_t offset, type data) \ { \ uint32_t val = data; \ int64_t ret; \ \ ret = pci_virt_cfg_write(phb, bdfn, offset, \ sizeof(data), val); \ return ret; \ } NPU3_CFG_READ(8, u8); NPU3_CFG_READ(16, u16); NPU3_CFG_READ(32, u32); NPU3_CFG_WRITE(8, u8); NPU3_CFG_WRITE(16, u16); NPU3_CFG_WRITE(32, u32); static int64_t npu3_eeh_freeze_status(struct phb *phb __unused, uint64_t pe_num __unused, uint8_t *freeze_state, uint16_t *pci_error_type, uint16_t *severity) { /* * FIXME: When it's called by skiboot PCI config accessor, * the PE number is fixed to 0, which is incorrect. We need * introduce another PHB callback to translate it. For now, * it keeps the skiboot PCI enumeration going. */ *freeze_state = OPAL_EEH_STOPPED_NOT_FROZEN; *pci_error_type = OPAL_EEH_NO_ERROR; if (severity) *severity = OPAL_EEH_SEV_NO_ERROR; return OPAL_SUCCESS; } /* Number of PEs supported */ #define NPU3_MAX_PE_NUM 16 #define NPU3_RESERVED_PE_NUM 15 static int64_t npu3_ioda_reset(struct phb *phb, bool purge __unused) { struct npu3 *npu = npu3_phb_to_npu(phb); uint64_t val; val = NPU3_ATS_IODA_ADDR_AUTO_INC; val = SETFIELD(NPU3_ATS_IODA_ADDR_TBL_SEL, val, NPU3_ATS_IODA_ADDR_TBL_TVT); npu3_write(npu, NPU3_ATS_IODA_ADDR, val); for (uint32_t i = 0; i < NPU3_MAX_PE_NUM; i++) npu3_write(npu, NPU3_ATS_IODA_DATA, 0ull); return OPAL_SUCCESS; } static inline void npu3_ioda_sel(struct npu3 *npu, uint32_t table, uint32_t index) { uint64_t val; val = SETFIELD(NPU3_ATS_IODA_ADDR_TBL_SEL, 0ull, table); val = SETFIELD(NPU3_ATS_IODA_ADDR_TBL_ADDR, val, index); npu3_write(npu, NPU3_ATS_IODA_ADDR, val); } static int64_t npu3_map_pe_dma_window(struct phb *phb, uint64_t pe_num, uint16_t window_id, uint16_t tce_levels, uint64_t tce_table_addr, uint64_t tce_table_size, uint64_t tce_page_size) { struct npu3 *npu = npu3_phb_to_npu(phb); uint64_t tts_encoded, val; uint32_t page_size; /* Each PE has one corresponding TVE */ if (window_id != pe_num || pe_num >= NPU3_MAX_PE_NUM) return OPAL_PARAMETER; npu3_ioda_sel(npu, NPU3_ATS_IODA_ADDR_TBL_TVT, pe_num); /* TCE table size zero is used to disable the TVE */ if (!tce_table_size) { npu3_write(npu, NPU3_ATS_IODA_DATA, 0ull); return OPAL_SUCCESS; } /* TCE table size */ if (!is_pow2(tce_table_size) || tce_table_size < 0x1000) return OPAL_PARAMETER; tts_encoded = ilog2(tce_table_size) - 11; if (tts_encoded > 39) return OPAL_PARAMETER; val = SETFIELD(NPU3_ATS_IODA_TVT_TABLE_SIZE, 0ull, tts_encoded); /* Number of levels */ if (tce_levels < 1 || tce_levels > 4) return OPAL_PARAMETER; val = SETFIELD(NPU3_ATS_IODA_TVT_TABLE_LEVEL, val, tce_levels - 1); /* TCE page size */ switch (tce_page_size) { case 256 << 20: page_size = 17; break; case 16 << 20: page_size = 13; break; case 64 << 10: page_size = 5; break; default: page_size = 1; } val = SETFIELD(NPU3_ATS_IODA_TVT_PAGE_SIZE, val, page_size); val = SETFIELD(NPU3_ATS_IODA_TVT_XLAT_ADDR, val, tce_table_addr >> 12); npu3_write(npu, NPU3_ATS_IODA_DATA, val); return OPAL_SUCCESS; } static int64_t npu3_map_pe_dma_window_real(struct phb *phb, uint64_t pe_num, uint16_t window_id, uint64_t pci_start_addr __unused, uint64_t pci_mem_size __unused) { struct npu3 *npu = npu3_phb_to_npu(phb); uint64_t val; /* Each PE has one corresponding TVE */ if (window_id != pe_num || pe_num >= NPU3_MAX_PE_NUM) return OPAL_PARAMETER; if (pci_mem_size) { /* * GPUs need to be able to access the MMIO memory space as well. * On POWER9 this is above the top of RAM, so disable the TVT * range check, allowing access to all memory addresses. */ val = 0; } else { /* Disable */ val = PPC_BIT(51); } npu3_ioda_sel(npu, NPU3_ATS_IODA_ADDR_TBL_TVT, pe_num); npu3_write(npu, NPU3_ATS_IODA_DATA, val); return OPAL_SUCCESS; } static int64_t npu3_next_error(struct phb *phb, uint64_t *first_frozen_pe, uint16_t *pci_error_type, uint16_t *severity) { struct npu3 *npu = npu3_phb_to_npu(phb); uint64_t val; uint32_t pe_num; if (!first_frozen_pe || !pci_error_type || !severity) return OPAL_PARAMETER; *first_frozen_pe = -1; *pci_error_type = OPAL_EEH_NO_ERROR; *severity = OPAL_EEH_SEV_NO_ERROR; for (pe_num = 0; pe_num < NPU3_MAX_PE_NUM; pe_num++) { val = npu3_read(npu, NPU3_MISC_PESTB_DATA(pe_num)); if (!GETFIELD(NPU3_MISC_PESTB_DATA_DMA_STOPPED_STATE, val)) continue; *first_frozen_pe = pe_num; *pci_error_type = OPAL_EEH_PE_ERROR; *severity = OPAL_EEH_SEV_PE_ER; break; } return OPAL_SUCCESS; } static struct npu3_dev *npu3_bdfn_to_dev(struct npu3 *npu, uint32_t bdfn) { struct pci_virt_device *pvd; /* All emulated devices are attached to root bus */ if (bdfn & ~0xff) return NULL; pvd = pci_virt_find_device(&npu->nvlink.phb, bdfn); if (pvd) return pvd->data; return NULL; } static int npu3_match_gpu(struct phb *phb __unused, struct pci_device *pd, void *data) { const char *slot = data; struct dt_node *dn; char *loc_code; /* Ignore non-NVIDIA devices */ if (PCI_VENDOR_ID(pd->vdid) != 0x10de) return 0; /* Find the PCI device's slot location */ for (dn = pd->dn; dn && !dt_find_property(dn, "ibm,loc-code"); dn = dn->parent); if (!dn) return 0; loc_code = (char *)dt_prop_get(dn, "ibm,loc-code"); if (streq(loc_code, slot)) return 1; return 0; } static void npu3_dev_find_gpu(struct npu3_dev *dev) { const char *slot = dev->nvlink.loc_code; struct phb *phb; struct pci_device *gpu; if (!slot) return; for_each_phb(phb) { gpu = pci_walk_dev(phb, NULL, npu3_match_gpu, (void *)slot); if (!gpu) continue; dev->nvlink.gpu = gpu; return; } NPU3DEVINF(dev, "No PCI device found for slot '%s'\n", slot); } #define VENDOR_CAP_START 0x80 #define VENDOR_CAP_LINK_FLAG_OFFSET 0x0d void npu3_pvd_flag_set(struct npu3_dev *dev, uint8_t flag) { uint32_t offset = VENDOR_CAP_START + VENDOR_CAP_LINK_FLAG_OFFSET; uint32_t flags; PCI_VIRT_CFG_RDONLY_RD(dev->nvlink.pvd, offset, 1, &flags); flags |= flag; PCI_VIRT_CFG_INIT_RO(dev->nvlink.pvd, offset, 1, flags); } void npu3_pvd_flag_clear(struct npu3_dev *dev, uint8_t flag) { uint32_t offset = VENDOR_CAP_START + VENDOR_CAP_LINK_FLAG_OFFSET; uint32_t flags; PCI_VIRT_CFG_RDONLY_RD(dev->nvlink.pvd, offset, 1, &flags); flags &= ~flag; PCI_VIRT_CFG_INIT_RO(dev->nvlink.pvd, offset, 1, flags); } static struct lock npu3_phandle_lock = LOCK_UNLOCKED; static void npu3_append_phandle(struct dt_node *dn, const char *name, uint32_t phandle) { struct dt_property *prop; uint32_t *phandles; size_t len; prop = __dt_find_property(dn, name); if (!prop) { dt_add_property_cells(dn, name, phandle); return; } /* * Make sure no one else has a reference to the property. Assume * this is the only function that holds a reference to it. */ lock(&npu3_phandle_lock); /* Need to append to the property */ len = prop->len + sizeof(*phandles); dt_resize_property(&prop, len); prop->len = len; phandles = (uint32_t *)prop->prop; phandles[len / sizeof(*phandles) - 1] = phandle; unlock(&npu3_phandle_lock); } static void npu3_dev_fixup_dt(struct npu3_dev *dev) { struct pci_device *pd = dev->nvlink.pd; struct pci_device *gpu = dev->nvlink.gpu; dt_add_property_cells(pd->dn, "ibm,nvlink", dev->dn->phandle); dt_add_property_string(pd->dn, "ibm,loc-code", dev->nvlink.loc_code); if (dev->link_speed != 0xff) dt_add_property_cells(pd->dn, "ibm,nvlink-speed", lo32(dev->link_speed)); if (!gpu) return; npu3_append_phandle(gpu->dn, "ibm,npu", pd->dn->phandle); dt_add_property_cells(pd->dn, "ibm,gpu", gpu->dn->phandle); } static int64_t npu3_gpu_bridge_sec_bus_reset(void *pdev, struct pci_cfg_reg_filter *pcrf __unused, uint32_t offset, uint32_t len, uint32_t *data, bool write) { struct pci_device *pd = pdev; struct pci_device *gpu; struct npu3 *npu; struct npu3_dev *dev; bool purge = false; if (!write) return OPAL_PARAMETER; if (len != 2 || offset & 1) { PCIERR(pd->phb, pd->bdfn, "Unsupported write to bridge control register\n"); return OPAL_PARAMETER; } if (!(*data & PCI_CFG_BRCTL_SECONDARY_RESET)) return OPAL_PARTIAL; gpu = list_top(&pd->children, struct pci_device, link); if (!gpu) return OPAL_PARTIAL; npu3_for_each_nvlink_npu(npu) npu3_for_each_nvlink_dev(dev, npu) if (dev->nvlink.gpu == gpu) if (!npu3_dev_reset(dev)) purge = true; if (purge) purge_l2_l3_caches(); return OPAL_PARTIAL; } static int npu3_dev_bind(struct phb *phb, struct pci_device *pd, void *data __unused) { struct npu3 *npu = npu3_phb_to_npu(phb); struct npu3_dev *dev = npu3_bdfn_to_dev(npu, pd->bdfn); struct pci_device *gpu; dev->nvlink.pd = pd; /* The slot label indicates which GPU this link is connected to */ dev->nvlink.loc_code = dt_prop_get_def(dev->dn, "ibm,slot-label", NULL); if (!dev->nvlink.loc_code) { /** * @fwts-label NPUNoPHBSlotLabel * @fwts-advice No GPU/NPU slot information was found. * NVLink3 functionality will not work. */ NPU3DEVERR(dev, "Cannot find GPU slot information\n"); } npu3_dev_find_gpu(dev); npu3_dev_fixup_dt(dev); gpu = dev->nvlink.gpu; if (!gpu) return 0; /* When a GPU is reset, ensure all of its links are reset too */ if (gpu->parent && gpu->parent->slot) pci_add_cfg_reg_filter(gpu->parent, PCI_CFG_BRCTL, 2, PCI_REG_FLAG_WRITE, npu3_gpu_bridge_sec_bus_reset); npu3_pvd_flag_set(dev, NPU3_DEV_PCI_LINKED); return 0; } struct npu3 *npu3_next_nvlink_npu(struct npu3 *npu, uint32_t chip_id) { uint64_t phb_id = 0; struct phb *phb; if (npu) phb_id = npu->nvlink.phb.opal_id + 1; for (; (phb = __pci_next_phb_idx(&phb_id));) { if (phb->phb_type != phb_type_npu_v3) continue; npu = npu3_phb_to_npu(phb); if (npu->chip_id == chip_id || chip_id == NPU3_ANY_CHIP) return npu; } return NULL; } static struct npu3 *npu3_last_npu(void) { static struct npu3 *last = NULL; struct npu3 *npu; if (last) return last; npu3_for_each_nvlink_npu(npu) last = npu; return last; } static uint32_t npu3_gpu_links(struct pci_device *gpu) { const struct dt_property *prop; if (!gpu) return 0; /* The link count is the number of phandles in "ibm,npu" */ prop = dt_find_property(gpu->dn, "ibm,npu"); if (!prop) return 0; return prop->len / sizeof(uint32_t); } static uint32_t npu3_links_per_gpu(void) { struct npu3 *npu; struct npu3_dev *dev; uint32_t links = 0; /* Use the first GPU we find to figure this out */ npu3_for_each_nvlink_npu(npu) { npu3_for_each_nvlink_dev(dev, npu) { links = npu3_gpu_links(dev->nvlink.gpu); if (links) goto out; } } out: prlog(PR_DEBUG, "NPU: %s: %d\n", __func__, links); return links; } int32_t npu3_dev_gpu_index(struct npu3_dev *dev) { const char *slot; char *p = NULL; int ret; slot = dev->nvlink.loc_code; if (!slot) return -1; if (memcmp(slot, "GPU", 3)) return -1; ret = strtol(slot + 3, &p, 10); if (*p || p == slot + 3) return -1; return ret; } static uint32_t npu3_chip_possible_gpu_links(void) { struct proc_chip *chip; struct npu3 *npu; struct npu3_dev *dev; uint32_t possible = 0; for_each_chip(chip) { npu3_for_each_chip_nvlink_npu(npu, chip->id) npu3_for_each_nvlink_dev(dev, npu) if (npu3_dev_gpu_index(dev) != -1) possible++; if (possible) break; } prlog(PR_DEBUG, "NPU: %s: %d\n", __func__, possible); return possible; } uint32_t npu3_chip_possible_gpus(void) { static uint32_t possible = -1; uint32_t links_per_gpu; /* Static value, same for all chips; only do this once */ if (possible != -1) return possible; possible = 0; links_per_gpu = npu3_links_per_gpu(); if (links_per_gpu) possible = npu3_chip_possible_gpu_links() / links_per_gpu; prlog(PR_DEBUG, "NPU: %s: %d\n", __func__, possible); return possible; } static void npu3_dev_assign_gmb(struct npu3_dev *dev, uint64_t addr, uint64_t size) { uint32_t mode; uint64_t val; switch (npu3_gpu_links(dev->nvlink.gpu)) { case 0: return; case 1: mode = 0; break; case 2: mode = 1; break; case 3: mode = 3; break; case 4: mode = 6; break; case 6: mode = 10; break; default: /* Hardware does not support this configuration */ assert(0); } mode += PCI_FUNC(dev->nvlink.pvd->bdfn); val = NPU3_GPU_MEM_BAR_ENABLE | NPU3_GPU_MEM_BAR_POISON; val = SETFIELD(NPU3_GPU_MEM_BAR_ADDR, val, addr >> 30); val = SETFIELD(NPU3_GPU_MEM_BAR_SIZE, val, size >> 30); val = SETFIELD(NPU3_GPU_MEM_BAR_MODE, val, mode); npu3_write(dev->npu, NPU3_GPU_MEM_BAR(dev->index), val); } static struct dt_node *npu3_create_memory_dn(struct npu3_dev *dev, uint32_t gpu_index, uint64_t addr, uint64_t size) { uint32_t nid = 255 - gpu_index; struct dt_node *mem; mem = dt_find_by_name_addr(dt_root, "memory", addr); if (mem) return mem; mem = dt_new_addr(dt_root, "memory", addr); assert(mem); dt_add_property_string(mem, "device_type", "memory"); dt_add_property_string(mem, "compatible", "ibm,coherent-device-memory"); dt_add_property_u64s(mem, "reg", addr, size); dt_add_property_u64s(mem, "linux,usable-memory", addr, 0); dt_add_property_cells(mem, "ibm,chip-id", nid); dt_add_property_cells(mem, "ibm,associativity", 4, nid, nid, nid, nid); NPU3INF(dev->npu, "%s mem: 0x%016llx (nid %d)\n", dev->nvlink.loc_code, addr, nid); return mem; } static void npu3_dev_init_gpu_mem(struct npu3_dev *dev) { struct pci_device *pd = dev->nvlink.pd; struct npu3 *npu = dev->npu; struct dt_node *mem; uint64_t addr, size, gta; uint32_t gpu_index; if (!dev->nvlink.gpu) return; gpu_index = npu3_dev_gpu_index(dev) % npu3_chip_possible_gpus(); phys_map_get(npu->chip_id, GPU_MEM_4T_DOWN, gpu_index, &addr, &size); npu3_dev_assign_gmb(dev, addr, size); mem = npu3_create_memory_dn(dev, gpu_index, addr, size); /* * Coral mode address compression. This is documented in Figure 3.5 of * the NPU workbook; "P9->GPU RA Compression (Coral)". */ gta = (addr >> 42 & 0x1) << 42; gta |= (addr >> 45 & 0x3) << 43; gta |= (addr >> 49 & 0x3) << 45; gta |= addr & ((1ul << 43) - 1); dt_add_property_cells(pd->dn, "memory-region", mem->phandle); dt_add_property_u64s(pd->dn, "ibm,device-tgt-addr", gta); } static void npu3_final_fixup(void) { struct npu3 *npu; struct npu3_dev *dev; npu3_for_each_nvlink_npu(npu) npu3_for_each_nvlink_dev(dev, npu) npu3_dev_init_gpu_mem(dev); } static void npu3_phb_final_fixup(struct phb *phb) { struct npu3 *npu = npu3_phb_to_npu(phb); pci_walk_dev(phb, NULL, npu3_dev_bind, NULL); /* * After every npu's devices are bound, do gpu-related fixup. This * counts on npu3_last_npu() walking the phbs in the same order as * the PHB final fixup loop in __pci_init_slots(). */ if (npu == npu3_last_npu()) npu3_final_fixup(); } static int64_t npu3_set_pe(struct phb *phb, uint64_t pe_num, uint64_t bdfn, uint8_t bcompare, uint8_t dcompare, uint8_t fcompare, uint8_t action) { struct npu3 *npu = npu3_phb_to_npu(phb); struct npu3_dev *dev; uint64_t val; dev = npu3_bdfn_to_dev(npu, bdfn); if (!dev) return OPAL_PARAMETER; if (action != OPAL_MAP_PE && action != OPAL_UNMAP_PE) return OPAL_PARAMETER; if (pe_num >= NPU3_MAX_PE_NUM) return OPAL_PARAMETER; if (bcompare != OpalPciBusAll || dcompare != OPAL_COMPARE_RID_DEVICE_NUMBER || fcompare != OPAL_COMPARE_RID_FUNCTION_NUMBER) return OPAL_UNSUPPORTED; if (!dev->nvlink.gpu) return OPAL_SUCCESS; val = NPU3_CTL_BDF2PE_CFG_ENABLE; val = SETFIELD(NPU3_CTL_BDF2PE_CFG_PE, val, pe_num); val = SETFIELD(NPU3_CTL_BDF2PE_CFG_BDF, val, dev->nvlink.gpu->bdfn); npu3_write(npu, NPU3_CTL_BDF2PE_CFG(pe_num), val); val = NPU3_MISC_BDF2PE_CFG_ENABLE; val = SETFIELD(NPU3_MISC_BDF2PE_CFG_PE, val, pe_num); val = SETFIELD(NPU3_MISC_BDF2PE_CFG_BDF, val, dev->nvlink.gpu->bdfn); npu3_write(npu, NPU3_MISC_BDF2PE_CFG(pe_num), val); return OPAL_SUCCESS; } static int64_t npu3_tce_kill_pages(struct npu3 *npu, uint64_t pe_num, uint32_t tce_size, uint64_t dma_addr, uint32_t npages) { uint32_t check_tce_size; uint64_t val; if (pe_num >= NPU3_MAX_PE_NUM) return OPAL_PARAMETER; npu3_ioda_sel(npu, NPU3_ATS_IODA_ADDR_TBL_TVT, pe_num); val = npu3_read(npu, NPU3_ATS_IODA_DATA); check_tce_size = 0x800 << GETFIELD(NPU3_ATS_IODA_TVT_PAGE_SIZE, val); if (check_tce_size != tce_size) { NPU3ERR(npu, "%s: Unexpected TCE size (got 0x%x, expected 0x%x)\n", __func__, tce_size, check_tce_size); return OPAL_PARAMETER; } val = NPU3_ATS_TCE_KILL_ONE; val = SETFIELD(NPU3_ATS_TCE_KILL_PE_NUMBER, val, pe_num); while (npages--) { val = SETFIELD(NPU3_ATS_TCE_KILL_ADDRESS, val, dma_addr >> 12); npu3_write(npu, NPU3_ATS_TCE_KILL, val); dma_addr += tce_size; } return OPAL_SUCCESS; } static int64_t npu3_tce_kill(struct phb *phb, uint32_t kill_type, uint64_t pe_num, uint32_t tce_size, uint64_t dma_addr, uint32_t npages) { struct npu3 *npu = npu3_phb_to_npu(phb); sync(); switch(kill_type) { case OPAL_PCI_TCE_KILL_PAGES: return npu3_tce_kill_pages(npu, pe_num, tce_size, dma_addr, npages); case OPAL_PCI_TCE_KILL_PE: /* * NPU doesn't support killing a PE so fall through * and do a kill all instead. */ case OPAL_PCI_TCE_KILL_ALL: npu3_write(npu, NPU3_ATS_TCE_KILL, NPU3_ATS_TCE_KILL_ALL); return OPAL_SUCCESS; } return OPAL_PARAMETER; } static const struct phb_ops npu_ops = { .cfg_read8 = npu3_cfg_read8, .cfg_read16 = npu3_cfg_read16, .cfg_read32 = npu3_cfg_read32, .cfg_write8 = npu3_cfg_write8, .cfg_write16 = npu3_cfg_write16, .cfg_write32 = npu3_cfg_write32, .eeh_freeze_status = npu3_eeh_freeze_status, .ioda_reset = npu3_ioda_reset, .map_pe_dma_window = npu3_map_pe_dma_window, .map_pe_dma_window_real = npu3_map_pe_dma_window_real, .next_error = npu3_next_error, .phb_final_fixup = npu3_phb_final_fixup, .set_pe = npu3_set_pe, .tce_kill = npu3_tce_kill, }; static int64_t npu3_reset(struct pci_slot *slot) { struct npu3 *npu = npu3_phb_to_npu(slot->phb); struct npu3_dev *dev; int64_t rc = OPAL_SUCCESS; bool purge = false; npu3_for_each_nvlink_dev(dev, npu) { rc = npu3_dev_reset(dev); if (rc) break; purge = true; } /* No devices reset; don't purge, just return */ if (!purge) return rc; /* All devices reset */ if (!rc) return purge_l2_l3_caches(); /* Some devices successfully reset; purge, but still return error */ purge_l2_l3_caches(); return rc; } static int64_t npu3_freset(struct pci_slot *slot __unused) { return OPAL_SUCCESS; } static int64_t npu3_get_link_state(struct pci_slot *slot __unused, uint8_t *val) { *val = OPAL_SHPC_LINK_UP_x1; return OPAL_SUCCESS; } static int64_t npu3_get_power_state(struct pci_slot *slot __unused, uint8_t *val) { *val = PCI_SLOT_POWER_ON; return OPAL_SUCCESS; } static void npu3_create_phb_slot(struct npu3 *npu) { struct pci_slot *slot; slot = pci_slot_alloc(&npu->nvlink.phb, NULL); if (!slot) return; /* Elementary functions */ slot->ops.creset = npu3_reset; slot->ops.freset = npu3_freset; slot->ops.hreset = npu3_reset; slot->ops.get_link_state = npu3_get_link_state; slot->ops.get_power_state = npu3_get_power_state; } static void npu3_create_phb(struct npu3 *npu) { struct phb *phb = &npu->nvlink.phb; phb->phb_type = phb_type_npu_v3; phb->ops = &npu_ops; phb->dt_node = dt_new_addr(dt_root, "pciex", npu->regs[0]); assert(phb->dt_node); list_head_init(&phb->virt_devices); pci_register_phb(phb, OPAL_DYNAMIC_PHB_ID); npu3_create_phb_slot(npu); npu3_ioda_reset(phb, true); } static void npu3_dev_init_hw(struct npu3_dev *dev) { struct npu3 *npu = dev->npu; uint64_t reg, val; reg = NPU3_RELAXED_CFG2(dev->index); val = npu3_read(npu, reg); val |= NPU3_RELAXED_CFG2_CMD_CL_DMA_W | NPU3_RELAXED_CFG2_CMD_CL_DMA_W_HP | NPU3_RELAXED_CFG2_CMD_CL_DMA_INJ | NPU3_RELAXED_CFG2_CMD_PR_DMA_INJ | NPU3_RELAXED_CFG2_CMD_DMA_PR_W | NPU3_RELAXED_CFG2_CMD_CL_RD_NC_F0 | NPU3_RELAXED_CFG2_SRC_RDENA(0); npu3_write(npu, reg, val); reg = NPU3_NTL_MISC_CFG2(dev->index); val = npu3_read(npu, reg); val |= NPU3_NTL_MISC_CFG2_BRICK_ENABLE | NPU3_NTL_MISC_CFG2_RCV_CREDIT_OVERFLOW_ENA; npu3_write(npu, reg, val); } static void npu3_init_hw(struct npu3 *npu) { struct npu3_dev *dev; uint64_t reg, val; reg = NPU3_XTS_CFG; val = npu3_read(npu, reg); val |= NPU3_XTS_CFG_MMIOSD | NPU3_XTS_CFG_TRY_ATR_RO; npu3_write(npu, reg, val); reg = NPU3_XTS_CFG2; val = npu3_read(npu, reg); val |= NPU3_XTS_CFG2_NO_FLUSH_ENA; npu3_write(npu, reg, val); reg = NPU3_RELAXED_SRC(0); val = NPU3_RELAXED_SRC_MASK_NPU; npu3_write(npu, reg, val); npu3_for_each_nvlink_dev(dev, npu) npu3_dev_init_hw(dev); } /* PCI command register (BAR enable/disable) */ static int64_t npu3_cfg_cmd(void *pvd, struct pci_cfg_reg_filter *pcrf __unused, uint32_t offset, uint32_t size, uint32_t *data, bool write) { struct npu3_dev *dev = ((struct pci_virt_device *)pvd)->data; if (!write) return OPAL_PARTIAL; if (offset != PCI_CFG_CMD) return OPAL_PARAMETER; if (size != 1 && size != 2 && size != 4) return OPAL_PARAMETER; npu3_dev_enable_bars(dev, !!(*data & PCI_CFG_CMD_MEM_EN)); return OPAL_PARTIAL; } static int64_t npu3_cfg_bar_write(struct npu3_bar *bar, uint64_t mask, uint32_t data) { if (data != 0xffffffff) return OPAL_HARDWARE; /* Return BAR size on next read */ bar->trap |= mask; return OPAL_SUCCESS; } static int64_t npu3_cfg_bar_read(struct npu3_bar *bar, uint64_t mask, uint32_t *data) { if (!(bar->trap & mask)) return OPAL_PARTIAL; *data = GETFIELD(mask, bar->size); bar->trap &= ~mask; return OPAL_SUCCESS; } /* PCI BAR registers (NTL/GENID) */ static int64_t npu3_cfg_bar(void *pvd __unused, struct pci_cfg_reg_filter *pcrf, uint32_t offset, uint32_t size, uint32_t *data, bool write) { struct npu3_bar *bar = (struct npu3_bar *)pcrf->data; uint64_t mask; if (size != 4) return OPAL_PARAMETER; if (offset == pcrf->start) mask = 0xffffffff; else if (offset == pcrf->start + 4) mask = 0xffffffffull << 32; else return OPAL_PARAMETER; if (write) return npu3_cfg_bar_write(bar, mask, *data); return npu3_cfg_bar_read(bar, mask, data); } /* PCI control register */ static int64_t npu3_cfg_devctl(void *pvd, struct pci_cfg_reg_filter *pcrf __unused, uint32_t offset, uint32_t size, uint32_t *data, bool write) { struct npu3_dev *dev = ((struct pci_virt_device *)pvd)->data; if (!write) return OPAL_HARDWARE; if (size != 2 || offset & 1) { NPU3DEVERR(dev, "Unsupported write to pcie control register\n"); return OPAL_PARAMETER; } if (*data & PCICAP_EXP_DEVCTL_FUNC_RESET) if (!npu3_dev_reset(dev)) purge_l2_l3_caches(); return OPAL_PARTIAL; } static uint32_t npu3_cfg_populate_pcie_cap(struct npu3_dev *dev, uint32_t start, uint32_t prev_cap) { struct pci_virt_device *pvd = dev->nvlink.pvd; uint32_t val; /* Add capability list */ PCI_VIRT_CFG_INIT_RO(pvd, prev_cap, 1, start); PCI_VIRT_CFG_INIT_RO(pvd, start, 1, PCI_CFG_CAP_ID_EXP); /* 0x00 - ID/PCIE capability */ val = PCI_CFG_CAP_ID_EXP; val |= 0x2 << 16 | PCIE_TYPE_ENDPOINT << 20; PCI_VIRT_CFG_INIT_RO(pvd, start, 4, val); /* 0x04 - Device capability */ val = PCIE_MPSS_128 | PCIE_PHANTOM_NONE << 3 | PCIE_L0SL_MAX_NO_LIMIT << 6 | PCIE_L1L_MAX_NO_LIMIT << 9 | PCICAP_EXP_DEVCAP_FUNC_RESET; PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_DEVCAP, 4, val); pci_virt_add_filter(pvd, start + PCICAP_EXP_DEVCTL, 2, PCI_REG_FLAG_WRITE, npu3_cfg_devctl, NULL); /* 0x08 - Device control and status */ PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_DEVCTL, 4, 0x00002810, 0xffff0000, 0x000f0000); /* 0x0c - Link capability */ val = PCIE_LSPEED_VECBIT_2 | PCIE_LWIDTH_1X << 4; PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_LCAP, 4, val); /* 0x10 - Link control and status */ PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_LCTL, 4, 0x00130000, 0xfffff000, 0xc0000000); /* 0x14 - Slot capability */ PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_SLOTCAP, 4, 0x00000000); /* 0x18 - Slot control and status */ PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_SLOTCTL, 4, 0x00000000); /* 0x1c - Root control and capability */ PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_RC, 4, 0x00000000, 0xffffffe0, 0x00000000); /* 0x20 - Root status */ PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_RSTAT, 4, 0x00000000, 0xffffffff, 0x00010000); /* 0x24 - Device capability 2 */ PCI_VIRT_CFG_INIT_RO(pvd, start + PCIECAP_EXP_DCAP2, 4, 0x00000000); /* 0x28 - Device Control and status 2 */ PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_DCTL2, 4, 0x00070000, 0xffff0000, 0x00000000); /* 0x2c - Link capability 2 */ PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_LCAP2, 4, 0x00000007); /* 0x30 - Link control and status 2 */ PCI_VIRT_CFG_INIT(pvd, start + PCICAP_EXP_LCTL2, 4, 0x00000003, 0xffff0000, 0x00200000); /* 0x34 - Slot capability 2 */ PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_SCAP2, 4, 0x00000000); /* 0x38 - Slot control and status 2 */ PCI_VIRT_CFG_INIT_RO(pvd, start + PCICAP_EXP_SCTL2, 4, 0x00000000); return start + PCICAP_EXP_SCTL2 + 8; } static int64_t npu3_dev_procedure_write(struct npu3_dev *dev, uint32_t offset, uint32_t data) { switch (offset) { case 0: NPU3DEVINF(dev, "Ignoring write to status register\n"); break; case 4: npu3_dev_procedure_init(dev, data); break; default: return OPAL_PARAMETER; } return OPAL_SUCCESS; } static int64_t npu3_dev_procedure_read(struct npu3_dev *dev, uint32_t offset, uint32_t *data) { switch (offset) { case 0: *data = npu3_dev_procedure_status(dev); break; case 4: *data = dev->proc.number; break; default: *data = 0; return OPAL_PARAMETER; } return OPAL_SUCCESS; } /* Hardware procedure control/status registers */ static int64_t npu3_dev_procedure(void *pvd, struct pci_cfg_reg_filter *pcrf, uint32_t offset, uint32_t size, uint32_t *data, bool write) { struct npu3_dev *dev = ((struct pci_virt_device *)pvd)->data; if (size != 4) return OPAL_PARAMETER; offset -= pcrf->start; if (write) return npu3_dev_procedure_write(dev, offset, *data); return npu3_dev_procedure_read(dev, offset, data); } /* PPE SRAM access is indirect via CSAR/CSDR */ static void npu3_dev_ppe_sram_sel(struct npu3_dev *dev, uint32_t reg) { uint64_t val; val = SETFIELD(OB_PPE_CSAR_SRAM_ADDR, 0ull, reg); xscom_write(dev->npu->chip_id, OB_PPE_CSAR(dev->ob_chiplet), val); } static void npu3_dev_ppe_sram_write(struct npu3_dev *dev, uint32_t reg, uint64_t val) { npu3_dev_ppe_sram_sel(dev, reg); xscom_write(dev->npu->chip_id, OB_PPE_CSDR(dev->ob_chiplet), val); } static uint64_t npu3_dev_ppe_sram_read(struct npu3_dev *dev, uint32_t reg) { uint64_t val; npu3_dev_ppe_sram_sel(dev, reg); xscom_read(dev->npu->chip_id, OB_PPE_CSDR(dev->ob_chiplet), &val); return val; } /* Software-implemented autonomous link training (SALT) */ static int64_t npu3_dev_salt(void *pvd, struct pci_cfg_reg_filter *pcrf, uint32_t offset, uint32_t size, uint32_t *data, bool write) { struct npu3_dev *dev = ((struct pci_virt_device *)pvd)->data; unsigned long timeout; uint32_t cmd_reg; uint64_t val; if (size != 4 || offset != pcrf->start) return OPAL_PARAMETER; /* The config register before this one holds CMD_REG */ PCI_VIRT_CFG_NORMAL_RD(pvd, pcrf->start - 4, 4, &cmd_reg); if (cmd_reg == 0xffffffff) return OPAL_PARAMETER; /* Check for another command in progress */ val = npu3_dev_ppe_sram_read(dev, OB_PPE_SALT_CMD); if (GETFIELD(OB_PPE_SALT_CMD_READY, val)) { NPU3DEVINF(dev, "SALT_CMD 0x%x: Not ready\n", cmd_reg); return OPAL_BUSY; } val = OB_PPE_SALT_CMD_READY; val = SETFIELD(OB_PPE_SALT_CMD_RW, val, write); val = SETFIELD(OB_PPE_SALT_CMD_LINKNUM, val, npu3_chip_dev_index(dev)); val = SETFIELD(OB_PPE_SALT_CMD_REG, val, cmd_reg); if (write) val = SETFIELD(OB_PPE_SALT_CMD_DATA, val, *data); npu3_dev_ppe_sram_write(dev, OB_PPE_SALT_CMD, val); /* Wait for the go bit to clear */ timeout = mftb() + msecs_to_tb(1000); while (GETFIELD(OB_PPE_SALT_CMD_READY, val)) { if (tb_compare(mftb(), timeout) == TB_AAFTERB) { NPU3DEVINF(dev, "SALT_CMD 0x%x: Timeout\n", cmd_reg); return OPAL_BUSY; } val = npu3_dev_ppe_sram_read(dev, OB_PPE_SALT_CMD); } if (GETFIELD(OB_PPE_SALT_CMD_ERR, val)) NPU3DEVINF(dev, "SALT_CMD 0x%x: Error\n", cmd_reg); if (!write) *data = GETFIELD(OB_PPE_SALT_CMD_DATA, val); return OPAL_SUCCESS; } #define VENDOR_CAP_LEN 0x1c #define VENDOR_CAP_VERSION 0x02 static uint32_t npu3_cfg_populate_vendor_cap(struct npu3_dev *dev, uint32_t start, uint32_t prev_cap) { struct pci_virt_device *pvd = dev->nvlink.pvd; /* Capabilities list */ PCI_VIRT_CFG_INIT_RO(pvd, prev_cap, 1, start); PCI_VIRT_CFG_INIT_RO(pvd, start, 1, PCI_CFG_CAP_ID_VENDOR); /* Length and version */ PCI_VIRT_CFG_INIT_RO(pvd, start + 2, 1, VENDOR_CAP_LEN); PCI_VIRT_CFG_INIT_RO(pvd, start + 3, 1, VENDOR_CAP_VERSION); /* * Defaults when the trap can't handle the read/write (eg. due to * reading/writing less than 4 bytes). */ PCI_VIRT_CFG_INIT_RO(pvd, start + 4, 4, 0); PCI_VIRT_CFG_INIT_RO(pvd, start + 8, 4, 0); /* PHY procedure trap */ pci_virt_add_filter(pvd, start + 4, 8, PCI_REG_FLAG_READ | PCI_REG_FLAG_WRITE, npu3_dev_procedure, NULL); /* Link index */ PCI_VIRT_CFG_INIT_RO(pvd, start + 0xc, 1, npu3_chip_dev_index(dev)); /* SALT registers */ PCI_VIRT_CFG_INIT(pvd, start + 0x10, 4, 0xffffffff, 0, 0); PCI_VIRT_CFG_INIT_RO(pvd, start + 0x14, 4, 0); pci_virt_add_filter(pvd, start + 0x14, 4, PCI_REG_FLAG_READ | PCI_REG_FLAG_WRITE, npu3_dev_salt, NULL); return start + VENDOR_CAP_LEN; } static void npu3_cfg_populate(struct npu3_dev *dev) { struct pci_virt_device *pvd = dev->nvlink.pvd; uint64_t addr; uint32_t pos; /* 0x00 - Vendor/Device ID */ PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_VENDOR_ID, 4, 0x04ea1014); /* 0x04 - Command/Status */ PCI_VIRT_CFG_INIT(pvd, PCI_CFG_CMD, 4, 0x00100000, 0xffb802b8, 0xf9000000); pci_virt_add_filter(pvd, PCI_CFG_CMD, 1, PCI_REG_FLAG_WRITE, npu3_cfg_cmd, NULL); /* 0x08 - Rev/Class/Cache */ PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_REV_ID, 4, 0x06800102); /* 0x0c - CLS/Latency Timer/Header/BIST */ PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_CACHE_LINE_SIZE, 4, 0x00800000); /* 0x10/14 - NTL BAR */ addr = SETFIELD(0xf, dev->ntl_bar.addr, PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64); PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR0, 4, lo32(addr), 0xf, 0); PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR1, 4, hi32(addr), 0, 0); pci_virt_add_filter(pvd, PCI_CFG_BAR0, 8, PCI_REG_FLAG_READ | PCI_REG_FLAG_WRITE, npu3_cfg_bar, &dev->ntl_bar); /* 0x18/1c - GENID BAR */ addr = SETFIELD(0xf, dev->genid_bar.addr, PCI_CFG_BAR_TYPE_MEM | PCI_CFG_BAR_MEM64); PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR2, 4, lo32(addr), 0xf, 0); PCI_VIRT_CFG_INIT(pvd, PCI_CFG_BAR3, 4, hi32(addr), 0, 0); pci_virt_add_filter(pvd, PCI_CFG_BAR2, 8, PCI_REG_FLAG_READ | PCI_REG_FLAG_WRITE, npu3_cfg_bar, &dev->genid_bar); /* 0x20/0x24 - BARs, disabled */ PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_BAR4, 4, 0x00000000); PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_BAR5, 4, 0x00000000); /* 0x28 - Cardbus CIS pointer */ PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_CARDBUS_CIS, 4, 0x00000000); /* 0x2c - Subsystem ID */ PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_SUBSYS_VENDOR_ID, 4, 0x00000000); /* 0x30 - ROM BAR, zero sized */ PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_ROMBAR, 4, 0xffffffff); /* 0x34 - PCI Capability */ PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_CAP, 4, 0x00000000); /* 0x38 - Reserved */ PCI_VIRT_CFG_INIT_RO(pvd, 0x38, 4, 0x00000000); /* 0x3c - INT line/pin/Minimal grant/Maximal latency */ PCI_VIRT_CFG_INIT_RO(pvd, PCI_CFG_INT_LINE, 4, 0x00000100); /* INT A */ /* PCIE and vendor specific capability */ pos = npu3_cfg_populate_pcie_cap(dev, 0x40, PCI_CFG_CAP); pos = npu3_cfg_populate_vendor_cap(dev, pos, 0x41); PCI_VIRT_CFG_INIT_RO(pvd, pos + 1, 1, 0); } static void npu3_dev_create_pvd(struct npu3_dev *dev) { struct npu3 *npu = dev->npu; struct phb *phb = &npu->nvlink.phb; dev->nvlink.pvd = pci_virt_add_device(phb, dev->index, 0x100, dev); if (!dev->nvlink.pvd) return; phb->scan_map |= 0x1 << GETFIELD(0xf8, dev->nvlink.pvd->bdfn); npu3_cfg_populate(dev); } static void npu3_dt_add_mmio_atsd(struct npu3 *npu) { struct dt_node *dn = npu->nvlink.phb.dt_node; uint64_t mmio_atsd[NPU3_XTS_ATSD_MAX]; for (uint32_t i = 0; i < NPU3_XTS_ATSD_MAX; i++) mmio_atsd[i] = npu->regs[0] + NPU3_XTS_ATSD_LAUNCH(i); dt_add_property(dn, "ibm,mmio-atsd", mmio_atsd, sizeof(mmio_atsd)); } static void npu3_dt_add_mmio_window(struct npu3 *npu) { struct dt_node *dn = npu->nvlink.phb.dt_node; uint32_t ntl0_index = npu->index * NPU3_LINKS_PER_NPU; uint64_t addr, size, win[2]; /* Device MMIO window (NTL/GENID regs only) */ phys_map_get(npu->chip_id, NPU_NTL, ntl0_index, &win[0], NULL); phys_map_get(npu->chip_id, NPU_GENID, npu->index, &addr, &size); win[1] = addr + size - win[0]; dt_add_property(dn, "ibm,mmio-window", win, sizeof(win)); dt_add_property_cells(dn, "ranges", 0x02000000, hi32(win[0]), lo32(win[0]), hi32(win[0]), lo32(win[0]), hi32(win[1]), lo32(win[1])); } /* NDL No-Stall Event level */ static uint32_t npu3_dev_interrupt_level(struct npu3_dev *dev) { const uint32_t level[12] = { 1, 3, 5, 7, 9, 11, 43, 45, 47, 49, 51, 53 }; return level[npu3_chip_dev_index(dev)]; } static void npu3_dt_add_interrupts(struct npu3 *npu) { struct dt_node *dn = npu->nvlink.phb.dt_node; uint32_t *map, icsp, i = 0; struct npu3_dev *dev; size_t map_size = 0; npu3_for_each_nvlink_dev(dev, npu) map_size += sizeof(*map) * 7; if (!map_size) return; icsp = get_ics_phandle(); map = zalloc(map_size); assert(map); npu3_for_each_nvlink_dev(dev, npu) { map[i] = dev->nvlink.pvd->bdfn << 8; map[i + 3] = 1; /* INT A */ map[i + 4] = icsp; /* interrupt-parent */ map[i + 5] = npu->irq_base + npu3_dev_interrupt_level(dev); map[i + 6] = 0; /* 0 = EDGE, 1 = LEVEL */ i += 7; } dt_add_property_cells(dn, "interrupt-parent", icsp); dt_add_property(dn, "interrupt-map", map, map_size); dt_add_property_cells(dn, "interrupt-map-mask", 0xff00, 0x0, 0x0, 0x7); free(map); } /* Populate PCI root device node */ static void npu3_dt_add_props(struct npu3 *npu) { struct dt_node *dn = npu->nvlink.phb.dt_node; dt_add_property_cells(dn, "#address-cells", 3); dt_add_property_cells(dn, "#size-cells", 2); dt_add_property_cells(dn, "#interrupt-cells", 1); dt_add_property_cells(dn, "bus-range", 0, 0xff); dt_add_property_cells(dn, "clock-frequency", 0x200, 0); dt_add_property_strings(dn, "device_type", "pciex"); /* * To the OS, npu2 and npu3 are both ibm,ioda2-npu2-phb. The added * ibm,ioda3-npu3-phb allows for possible quirks. */ dt_add_property_strings(dn, "compatible", "ibm,power9-npu-pciex", "ibm,ioda2-npu2-phb", "ibm,ioda2-npu3-phb"); dt_add_property_cells(dn, "ibm,phb-index", dt_prop_get_u32(npu->dt_node, "ibm,phb-index")); dt_add_property_cells(dn, "ibm,phb-diag-data-size", 0); dt_add_property_cells(dn, "ibm,opal-num-pes", NPU3_MAX_PE_NUM); dt_add_property_cells(dn, "ibm,opal-reserved-pe", NPU3_RESERVED_PE_NUM); dt_add_property_cells(dn, "ibm,supported-tce-sizes", 12, /* 4K */ 16, /* 64K */ 24, /* 16M */ 28); /* 256M */ dt_add_property_cells(dn, "ibm,chip-id", npu->chip_id); dt_add_property_cells(dn, "ibm,npu-index", npu->index); dt_add_property_cells(dn, "ibm,npcq", npu->dt_node->phandle); dt_add_property_cells(dn, "ibm,xscom-base", npu->xscom_base); dt_add_property_cells(dn, "ibm,links", NPU3_LINKS_PER_NPU); dt_add_property(dn, "reg", npu->regs, sizeof(npu->regs)); npu3_dt_add_mmio_atsd(npu); npu3_dt_add_mmio_window(npu); npu3_dt_add_interrupts(npu); } void npu3_init_nvlink(struct npu3 *npu) { struct npu3_dev *dev; if (!npu3_next_dev(npu, NULL, NPU3_DEV_TYPE_NVLINK)) return; npu3_init_hw(npu); npu3_create_phb(npu); npu3_for_each_nvlink_dev(dev, npu) npu3_dev_create_pvd(dev); npu3_dt_add_props(npu); /* TODO: Sort out if/why we still can't enable this */ disable_fast_reboot("NVLink device enabled"); } static int64_t npu3_init_context_pid(struct npu3 *npu, uint32_t index, uint64_t msr) { uint64_t map, old_map; /* Unfiltered XTS mode; index is lparshort */ map = SETFIELD(NPU3_XTS_PID_MAP_LPARSHORT, 0ull, index); /* Enable this mapping for both real and virtual addresses */ map |= NPU3_XTS_PID_MAP_VALID_ATRGPA0 | NPU3_XTS_PID_MAP_VALID_ATRGPA1; /* Enable TLBIE/MMIOSD forwarding for this entry */ map |= NPU3_XTS_PID_MAP_VALID_ATSD; /* Set the relevant MSR bits */ if (msr & MSR_DR) map |= NPU3_XTS_PID_MAP_MSR_DR; if (msr & MSR_HV) map |= NPU3_XTS_PID_MAP_MSR_HV; if (msr & MSR_PR) map |= NPU3_XTS_PID_MAP_MSR_PR; /* We don't support anything other than 64-bit so hardcode it here */ map |= NPU3_XTS_PID_MAP_MSR_SF; old_map = npu3_read(npu, NPU3_XTS_PID_MAP(index)); /* Error out if this entry is already set with different msr bits */ if (old_map && GETFIELD(NPU3_XTS_PID_MAP_MSR, old_map) != GETFIELD(NPU3_XTS_PID_MAP_MSR, map)) { NPU3ERR(npu, "%s: Unexpected MSR value\n", __func__); return OPAL_PARAMETER; } if (!old_map) { NPU3DBG(npu, "XTS_PID_MAP[%03d] = 0x%08llx\n", index, map); npu3_write(npu, NPU3_XTS_PID_MAP(index), map); } npu->nvlink.ctx_ref[index]++; return OPAL_SUCCESS; } #define NPU3_VALID_ATS_MSR_BITS (MSR_DR | MSR_HV | MSR_PR | MSR_SF) /* * Allocate a context ID and initialize the tables with the relevant * information. Returns the ID or error if one couldn't be allocated. */ int64_t npu3_init_context(struct phb *phb, uint64_t msr, uint64_t bdf) { struct npu3 *npu = npu3_phb_to_npu(phb); uint32_t lparshort, i; uint64_t map; int64_t rc; /* * MSR bits should be masked by the caller to allow for future * expansion if required. */ if (msr & ~NPU3_VALID_ATS_MSR_BITS) return OPAL_UNSUPPORTED; lock(&npu->lock); for (i = 0; i < NPU3_XTS_BDF_MAP_MAX; i++) { map = npu3_read(npu, NPU3_XTS_BDF_MAP(i)); if (map && GETFIELD(NPU3_XTS_BDF_MAP_BDF, map) == bdf) break; } if (i == NPU3_XTS_BDF_MAP_MAX) { NPU3ERR(npu, "LPARID not associated with any GPU\n"); rc = OPAL_PARAMETER; goto out; } lparshort = GETFIELD(NPU3_XTS_BDF_MAP_LPARSHORT, map); NPU3DBG(npu, "Found LPARSHORT 0x%x for bdf %02llx:%02llx.%llx\n", lparshort, PCI_BUS_NUM(bdf), PCI_DEV(bdf), PCI_FUNC(bdf)); rc = npu3_init_context_pid(npu, lparshort, msr); if (rc) goto out; if (!(map & NPU3_XTS_BDF_MAP_VALID)) { map |= NPU3_XTS_BDF_MAP_VALID; npu3_write(npu, NPU3_XTS_BDF_MAP(i), map); } rc = lparshort; out: unlock(&npu->lock); return rc; } static int64_t npu3_destroy_context_pid(struct npu3 *npu, uint32_t index) { if (!npu->nvlink.ctx_ref[index]) return OPAL_PARAMETER; /* Only destroy when refcount hits 0 */ if (--npu->nvlink.ctx_ref[index]) return OPAL_PARTIAL; NPU3DBG(npu, "XTS_PID_MAP[%03d] = 0 (destroy)\n", index); npu3_write(npu, NPU3_XTS_PID_MAP(index), 0ull); return OPAL_SUCCESS; } int64_t npu3_destroy_context(struct phb *phb, uint64_t bdf) { struct npu3 *npu = npu3_phb_to_npu(phb); uint32_t lparshort, i; int64_t map, rc; lock(&npu->lock); for (i = 0; i < NPU3_XTS_BDF_MAP_MAX; i++) { map = npu3_read(npu, NPU3_XTS_BDF_MAP(i)); if (map && GETFIELD(NPU3_XTS_BDF_MAP_BDF, map) == bdf) break; } if (i == NPU3_XTS_BDF_MAP_MAX) { NPU3ERR(npu, "LPARID not associated with any GPU\n"); rc = OPAL_PARAMETER; goto out; } lparshort = GETFIELD(NPU3_XTS_BDF_MAP_LPARSHORT, map); rc = npu3_destroy_context_pid(npu, lparshort); out: unlock(&npu->lock); return rc; } /* Map the given virtual bdf to lparid with given lpcr */ int64_t npu3_map_lpar(struct phb *phb, uint64_t bdf, uint64_t lparid, uint64_t lpcr) { struct npu3 *npu = npu3_phb_to_npu(phb); struct npu3_dev *dev; int64_t rc = OPAL_SUCCESS; uint64_t map, val; uint32_t i; /* * The LPCR bits are only required for hash based ATS, which we don't * currently support, but may need to in the future. */ if (lpcr) return OPAL_UNSUPPORTED; lock(&npu->lock); /* Update the entry if it already exists */ for (i = 0; i < NPU3_XTS_BDF_MAP_MAX; i++) { map = npu3_read(npu, NPU3_XTS_BDF_MAP(i)); if (map && GETFIELD(NPU3_XTS_BDF_MAP_BDF, map) == bdf) break; } if (i == NPU3_XTS_BDF_MAP_MAX) { /* No existing mapping found, find space for a new one */ for (i = 0; i < NPU3_XTS_BDF_MAP_MAX; i++) if (!npu3_read(npu, NPU3_XTS_BDF_MAP(i))) break; } if (i == NPU3_XTS_BDF_MAP_MAX) { NPU3ERR(npu, "No free XTS_BDF[] entry\n"); rc = OPAL_RESOURCE; goto out; } map = NPU3_XTS_BDF_MAP_UNFILT; map = SETFIELD(NPU3_XTS_BDF_MAP_BDF, map, bdf); map = SETFIELD(NPU3_XTS_BDF_MAP_LPARID, map, lparid); map = SETFIELD(NPU3_XTS_BDF_MAP_LPARSHORT, map, i); /* We only support radix at the moment */ map = SETFIELD(NPU3_XTS_BDF_MAP_XLAT, map, 0x3); /* Find a link on which to send ATSDs for this device */ npu3_for_each_nvlink_dev(dev, npu) if (dev->nvlink.gpu->bdfn == bdf) break; if (!dev || dev->nvlink.gpu->bdfn != bdf) { NPU3ERR(npu, "Can't find a link for bdf %02llx:%02llx.%llx\n", PCI_BUS_NUM(bdf), PCI_DEV(bdf), PCI_FUNC(bdf)); rc = OPAL_PARAMETER; goto out; } map = SETFIELD(NPU3_XTS_BDF_MAP_BRICK, map, dev->index); NPU3DBG(npu, "XTS_BDF_MAP[%03d] = 0x%08llx\n", i, map); npu3_write(npu, NPU3_XTS_BDF_MAP(i), map); /* We need to allocate an ATSD per link */ val = SETFIELD(NPU3_XTS_ATSD_HYP_LPARID, 0ull, lparid); if (!lparid) val |= NPU3_XTS_ATSD_HYP_MSR_HV; npu3_write(npu, NPU3_XTS_ATSD_HYP(dev->index), val); out: unlock(&npu->lock); return rc; } static int64_t npu3_relaxed_order_enable(struct npu3 *npu, uint64_t src) { struct npu3_dev *dev; uint32_t i; for (i = 0; i < NPU3_RELAXED_SRC_MAX; i++) if (npu3_read(npu, NPU3_RELAXED_SRC(i)) == src) return OPAL_SUCCESS; /* Already enabled */ /* Find somewhere to write this source */ for (i = 0; i < NPU3_RELAXED_SRC_MAX; i++) if (!npu3_read(npu, NPU3_RELAXED_SRC(i))) break; if (i == NPU3_RELAXED_SRC_MAX) { NPU3ERR(npu, "Insufficient resources to activate relaxed ordering mode\n"); return OPAL_RESOURCE; } npu3_write(npu, NPU3_RELAXED_SRC(i), src); npu3_for_each_nvlink_dev(dev, npu) { uint64_t val = npu3_read(npu, NPU3_RELAXED_CFG2(dev->index)); val |= NPU3_RELAXED_CFG2_SRC_WRENA(i) | NPU3_RELAXED_CFG2_SRC_RDENA(i); npu3_write(npu, NPU3_RELAXED_CFG2(dev->index), val); } return OPAL_SUCCESS; } static void npu3_relaxed_order_disable(struct npu3 *npu, uint64_t src) { struct npu3_dev *dev; uint32_t i; for (i = 0; i < NPU3_RELAXED_SRC_MAX; i++) if (npu3_read(npu, NPU3_RELAXED_SRC(i)) == src) break; if (i == NPU3_RELAXED_SRC_MAX) return; /* Already disabled */ npu3_for_each_nvlink_dev(dev, npu) { uint64_t val = npu3_read(npu, NPU3_RELAXED_CFG2(dev->index)); val &= ~NPU3_RELAXED_CFG2_SRC_WRENA(i); val &= ~NPU3_RELAXED_CFG2_SRC_RDENA(i); npu3_write(npu, NPU3_RELAXED_CFG2(dev->index), val); } npu3_write(npu, NPU3_RELAXED_SRC(i), 0ull); } /* Enable or disable relaxed ordering on all nvlinks for a given PEC. */ int64_t npu3_set_relaxed_order(struct phb *phb, uint32_t gcid, int pec, bool enable) { struct npu3 *npu = npu3_phb_to_npu(phb); int64_t rc = OPAL_SUCCESS; uint64_t src; NPU3INF(npu, "%s relaxed ordering for PEC %d on chip %d\n", enable ? "Enabling" : "Disabling", pec, gcid); lock(&npu->lock); src = SETFIELD(NPU3_RELAXED_SRC_GRPCHP, 0ull, gcid); src = SETFIELD(NPU3_RELAXED_SRC_PEC, src, pec); src = SETFIELD(NPU3_RELAXED_SRC_RDSTART, src, 0); src = SETFIELD(NPU3_RELAXED_SRC_RDEND, src, 47); src = SETFIELD(NPU3_RELAXED_SRC_WRSTART, src, 0); src = SETFIELD(NPU3_RELAXED_SRC_WREND, src, 23); if (enable) rc = npu3_relaxed_order_enable(npu, src); else npu3_relaxed_order_disable(npu, src); unlock(&npu->lock); return rc; }