aboutsummaryrefslogtreecommitdiff
path: root/hw/npu2.c
diff options
context:
space:
mode:
authorAlexey Kardashevskiy <aik@ozlabs.ru>2019-04-29 19:12:27 +1000
committerStewart Smith <stewart@linux.ibm.com>2019-05-02 09:57:15 +1000
commit0f492a92590850af6360bdcc93e2047b285d41c7 (patch)
treef9f79e96583903bc23546280c6457f62560c6b5c /hw/npu2.c
parent44afdc1afb1fe17ad7ae6758279f856a464cb922 (diff)
downloadskiboot-0f492a92590850af6360bdcc93e2047b285d41c7.zip
skiboot-0f492a92590850af6360bdcc93e2047b285d41c7.tar.gz
skiboot-0f492a92590850af6360bdcc93e2047b285d41c7.tar.bz2
npu2: Disable Probe-to-Invalid-Return-Modified-or-Owned snarfing by default
V100 GPUs are known to violate NVLink2 protocol in some cases (one is when memory was accessed by the CPU and they by GPU using so called block linear mapping) and issue double probes to NPU which can cope with this problem only if CONFIG_ENABLE_SNARF_CPM ("disable/enable Probe.I.MO snarfing a cp_m") is not set in the CQ_SM Misc Config register #0. If the bit is set (which is the case today), NPU issues the machine check stop. The snarfing feature is designed to detect 2 probes in flight and combine them into one. This adds a new "opal-npu2-snarf-cpm" nvram variable which controls CONFIG_ENABLE_SNARF_CPM for all NVLinks to prevent the machine check stop from happening. This disables snarfing by default as otherwise a broken GPU driver can crash the entire box even when a GPU is passed through to a guest. This provides a dial to allow regression tests (might be useful for a bare metal). To enable snarfing, the user needs to run: sudo nvram -p ibm,skiboot --update-config opal-npu2-snarf-cpm=enable and reboot the host system. While at this, define macros for register names as well to avoid touching same lines over and over again. Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru> Signed-off-by: Stewart Smith <stewart@linux.ibm.com>
Diffstat (limited to 'hw/npu2.c')
-rw-r--r--hw/npu2.c56
1 files changed, 43 insertions, 13 deletions
diff --git a/hw/npu2.c b/hw/npu2.c
index d532c4d..0d79d8a 100644
--- a/hw/npu2.c
+++ b/hw/npu2.c
@@ -1452,7 +1452,7 @@ static void assign_mmio_bars(uint64_t gcid, uint32_t scom, uint64_t reg[2], uint
int npu2_nvlink_init_npu(struct npu2 *npu)
{
struct dt_node *np;
- uint64_t reg[2], mm_win[2], val;
+ uint64_t reg[2], mm_win[2], val, mask;
/* TODO: Clean this up with register names, etc. when we get
* time. This just turns NVLink mode on in each brick and should
@@ -1461,18 +1461,48 @@ int npu2_nvlink_init_npu(struct npu2 *npu)
*
* Obviously if the year is now 2020 that didn't happen and you
* should fix this :-) */
- xscom_write_mask(npu->chip_id, 0x5011000, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011030, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011060, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011090, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011200, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011230, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011260, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011290, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011400, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011430, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011460, PPC_BIT(58), PPC_BIT(58));
- xscom_write_mask(npu->chip_id, 0x5011490, PPC_BIT(58), PPC_BIT(58));
+
+ val = PPC_BIT(58);
+ mask = PPC_BIT(58) | /* CONFIG_NVLINK_MODE */
+ PPC_BIT(40); /* CONFIG_ENABLE_SNARF_CPM */
+
+ /*
+ * V100 GPUs are known to violate NVLink2 protocol if some GPU memory
+ * mapped by a CPU was also "linear-block" mapped by a GPU. When this
+ * happens, it breaks the NPU2 cache coherency state machine and
+ * it throws machine checkstop. Disabling snarfing fixes this so let's
+ * disable it by default.
+ */
+ if (nvram_query_eq("opal-npu2-snarf-cpm", "enable")) {
+ prlog(PR_WARNING, "NPU2#%d: enabling Probe.I.MO snarfing, a bad GPU driver may crash the system!\n",
+ npu->index);
+ val |= PPC_BIT(40); /* CONFIG_ENABLE_SNARF_CPM */
+ }
+
+ xscom_write_mask(npu->chip_id, NPU_STCK0_CS_SM0_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK0_CS_SM1_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK0_CS_SM2_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK0_CS_SM3_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK1_CS_SM0_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK1_CS_SM1_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK1_CS_SM2_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK1_CS_SM3_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK2_CS_SM0_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK2_CS_SM1_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK2_CS_SM2_MISC_CONFIG0,
+ val, mask);
+ xscom_write_mask(npu->chip_id, NPU_STCK2_CS_SM3_MISC_CONFIG0,
+ val, mask);
xscom_write_mask(npu->chip_id, 0x50110c0, PPC_BIT(53), PPC_BIT(53));
xscom_write_mask(npu->chip_id, 0x50112c0, PPC_BIT(53), PPC_BIT(53));