From 931150e56b056b120c868f94751722710df0b6a7 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Wed, 16 Aug 2023 15:55:49 -0600 Subject: vfio/display: Fix missing update to set backing fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The below referenced commit renames scanout_width/height to backing_width/height, but also promotes these fields in various portions of the egl interface. Meanwhile vfio dmabuf support has never used the previous scanout fields and is therefore missed in the update. This results in a black screen when transitioning from ramfb to dmabuf display when using Intel vGPU with these features. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1891 Link: https://lists.gnu.org/archive/html/qemu-devel/2023-08/msg02726.html Fixes: 9ac06df8b684 ("virtio-gpu-udmabuf: correct naming of QemuDmaBuf size properties") Signed-off-by: Alex Williamson Tested-by: Cédric Le Goater Signed-off-by: Cédric Le Goater --- hw/vfio/display.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hw/vfio/display.c b/hw/vfio/display.c index bec864f..837d9e6 100644 --- a/hw/vfio/display.c +++ b/hw/vfio/display.c @@ -243,6 +243,8 @@ static VFIODMABuf *vfio_display_get_dmabuf(VFIOPCIDevice *vdev, dmabuf->dmabuf_id = plane.dmabuf_id; dmabuf->buf.width = plane.width; dmabuf->buf.height = plane.height; + dmabuf->buf.backing_width = plane.width; + dmabuf->buf.backing_height = plane.height; dmabuf->buf.stride = plane.stride; dmabuf->buf.fourcc = plane.drm_format; dmabuf->buf.modifier = plane.drm_format_mod; -- cgit v1.1 From c06327c9db960158edc8756dbc0eb25fab920656 Mon Sep 17 00:00:00 2001 From: Zhenzhong Duan Date: Fri, 22 Sep 2023 10:52:23 +0800 Subject: vfio/pci: rename vfio_put_device to vfio_pci_put_device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vfio_put_device() is a VFIO PCI specific function, rename it with 'vfio_pci' prefix to avoid confusing. No functional change. Suggested-by: Cédric Le Goater Signed-off-by: Zhenzhong Duan Signed-off-by: Cédric Le Goater --- hw/vfio/pci.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 3b2ca3c..b2d5010 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -2826,7 +2826,7 @@ static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp) } } -static void vfio_put_device(VFIOPCIDevice *vdev) +static void vfio_pci_put_device(VFIOPCIDevice *vdev) { g_free(vdev->vbasedev.name); g_free(vdev->msix); @@ -3317,7 +3317,7 @@ static void vfio_instance_finalize(Object *obj) * * g_free(vdev->igd_opregion); */ - vfio_put_device(vdev); + vfio_pci_put_device(vdev); vfio_put_group(group); } -- cgit v1.1 From 45d85f6228117dbe01d3680f1ad834389f69012f Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Mon, 25 Sep 2023 22:14:04 -0400 Subject: vfio/pci: detect the support of dynamic MSI-X allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kernel provides the guidance of dynamic MSI-X allocation support of passthrough device, by clearing the VFIO_IRQ_INFO_NORESIZE flag to guide user space. Fetch the flags from host to determine if dynamic MSI-X allocation is supported. Originally-by: Reinette Chatre Signed-off-by: Jing Liu Reviewed-by: Cédric Le Goater Reviewed-by: Alex Williamson Signed-off-by: Cédric Le Goater --- hw/vfio/pci.c | 16 ++++++++++++++-- hw/vfio/pci.h | 1 + hw/vfio/trace-events | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index b2d5010..86c92c5 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -1493,7 +1493,9 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp) uint8_t pos; uint16_t ctrl; uint32_t table, pba; - int fd = vdev->vbasedev.fd; + int ret, fd = vdev->vbasedev.fd; + struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info), + .index = VFIO_PCI_MSIX_IRQ_INDEX }; VFIOMSIXInfo *msix; pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX); @@ -1530,6 +1532,15 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp) msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK; msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; + ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info); + if (ret < 0) { + error_setg_errno(errp, -ret, "failed to get MSI-X irq info"); + g_free(msix); + return; + } + + msix->noresize = !!(irq_info.flags & VFIO_IRQ_INFO_NORESIZE); + /* * Test the size of the pba_offset variable and catch if it extends outside * of the specified BAR. If it is the case, we need to apply a hardware @@ -1562,7 +1573,8 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp) } trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar, - msix->table_offset, msix->entries); + msix->table_offset, msix->entries, + msix->noresize); vdev->msix = msix; vfio_pci_fixup_msix_region(vdev); diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h index 2d83609..0d89eb7 100644 --- a/hw/vfio/pci.h +++ b/hw/vfio/pci.h @@ -113,6 +113,7 @@ typedef struct VFIOMSIXInfo { uint32_t table_offset; uint32_t pba_offset; unsigned long *pending; + bool noresize; } VFIOMSIXInfo; #define TYPE_VFIO_PCI "vfio-pci" diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events index e64ca4a..0ba3c5a 100644 --- a/hw/vfio/trace-events +++ b/hw/vfio/trace-events @@ -27,7 +27,7 @@ vfio_vga_read(uint64_t addr, int size, uint64_t data) " (0x%"PRIx64", %d) = 0x%" vfio_pci_read_config(const char *name, int addr, int len, int val) " (%s, @0x%x, len=0x%x) 0x%x" vfio_pci_write_config(const char *name, int addr, int val, int len) " (%s, @0x%x, 0x%x, len=0x%x)" vfio_msi_setup(const char *name, int pos) "%s PCI MSI CAP @0x%x" -vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d" +vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries, bool noresize) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d, noresize %d" vfio_check_pcie_flr(const char *name) "%s Supports FLR via PCIe cap" vfio_check_pm_reset(const char *name) "%s Supports PM reset" vfio_check_af_flr(const char *name) "%s Supports FLR via AF cap" -- cgit v1.1 From d9e6710d7d59c1f774bfb49f3b1c2b5ee06ed33e Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Mon, 25 Sep 2023 22:14:05 -0400 Subject: vfio/pci: enable vector on dynamic MSI-X allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vector_use callback is used to enable vector that is unmasked in guest. The kernel used to only support static MSI-X allocation. When allocating a new interrupt using "static MSI-X allocation" kernels, QEMU first disables all previously allocated vectors and then re-allocates all including the new one. The nr_vectors of VFIOPCIDevice indicates that all vectors from 0 to nr_vectors are allocated (and may be enabled), which is used to loop all the possibly used vectors when e.g., disabling MSI-X interrupts. Extend the vector_use function to support dynamic MSI-X allocation when host supports the capability. QEMU therefore can individually allocate and enable a new interrupt without affecting others or causing interrupts lost during runtime. Utilize nr_vectors to calculate the upper bound of enabled vectors in dynamic MSI-X allocation mode since looping all msix_entries_nr is not efficient and unnecessary. Signed-off-by: Jing Liu Tested-by: Reinette Chatre Reviewed-by: Alex Williamson Signed-off-by: Cédric Le Goater --- hw/vfio/pci.c | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 86c92c5..eb7627c 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -470,6 +470,7 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr, VFIOPCIDevice *vdev = VFIO_PCI(pdev); VFIOMSIVector *vector; int ret; + bool resizing = !!(vdev->nr_vectors < nr + 1); trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr); @@ -512,33 +513,42 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr, } /* - * We don't want to have the host allocate all possible MSI vectors - * for a device if they're not in use, so we shutdown and incrementally - * increase them as needed. + * When dynamic allocation is not supported, we don't want to have the + * host allocate all possible MSI vectors for a device if they're not + * in use, so we shutdown and incrementally increase them as needed. + * nr_vectors represents the total number of vectors allocated. + * + * When dynamic allocation is supported, let the host only allocate + * and enable a vector when it is in use in guest. nr_vectors represents + * the upper bound of vectors being enabled (but not all of the ranges + * is allocated or enabled). */ - if (vdev->nr_vectors < nr + 1) { + if (resizing) { vdev->nr_vectors = nr + 1; - if (!vdev->defer_kvm_irq_routing) { + } + + if (!vdev->defer_kvm_irq_routing) { + if (vdev->msix->noresize && resizing) { vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); ret = vfio_enable_vectors(vdev, true); if (ret) { error_report("vfio: failed to enable vectors, %d", ret); } - } - } else { - Error *err = NULL; - int32_t fd; - - if (vector->virq >= 0) { - fd = event_notifier_get_fd(&vector->kvm_interrupt); } else { - fd = event_notifier_get_fd(&vector->interrupt); - } + Error *err = NULL; + int32_t fd; - if (vfio_set_irq_signaling(&vdev->vbasedev, - VFIO_PCI_MSIX_IRQ_INDEX, nr, - VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) { - error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); + if (vector->virq >= 0) { + fd = event_notifier_get_fd(&vector->kvm_interrupt); + } else { + fd = event_notifier_get_fd(&vector->interrupt); + } + + if (vfio_set_irq_signaling(&vdev->vbasedev, + VFIO_PCI_MSIX_IRQ_INDEX, nr, + VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) { + error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); + } } } -- cgit v1.1 From 5ebffa4e87e7a3127650ef2c0fc12e8624337ee4 Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Mon, 25 Sep 2023 22:14:06 -0400 Subject: vfio/pci: use an invalid fd to enable MSI-X MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Guests typically enable MSI-X with all of the vectors masked in the MSI-X vector table. To match the guest state of device, QEMU enables MSI-X by enabling vector 0 with userspace triggering and immediately release. However the release function actually does not release it due to already using userspace mode. It is no need to enable triggering on host and rely on the mask bit to avoid spurious interrupts. Use an invalid fd (i.e. fd = -1) is enough to get MSI-X enabled. After dynamic MSI-X allocation is supported, the interrupt restoring also need use such way to enable MSI-X, therefore, create a function for that. Suggested-by: Alex Williamson Signed-off-by: Jing Liu Reviewed-by: Cédric Le Goater Reviewed-by: Alex Williamson Signed-off-by: Cédric Le Goater --- hw/vfio/pci.c | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index eb7627c..ad508ab 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -369,6 +369,33 @@ static void vfio_msi_interrupt(void *opaque) notify(&vdev->pdev, nr); } +/* + * Get MSI-X enabled, but no vector enabled, by setting vector 0 with an invalid + * fd to kernel. + */ +static int vfio_enable_msix_no_vec(VFIOPCIDevice *vdev) +{ + g_autofree struct vfio_irq_set *irq_set = NULL; + int ret = 0, argsz; + int32_t *fd; + + argsz = sizeof(*irq_set) + sizeof(*fd); + + irq_set = g_malloc0(argsz); + irq_set->argsz = argsz; + irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | + VFIO_IRQ_SET_ACTION_TRIGGER; + irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX; + irq_set->start = 0; + irq_set->count = 1; + fd = (int32_t *)&irq_set->data; + *fd = -1; + + ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set); + + return ret; +} + static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix) { struct vfio_irq_set *irq_set; @@ -618,6 +645,8 @@ static void vfio_commit_kvm_msi_virq_batch(VFIOPCIDevice *vdev) static void vfio_msix_enable(VFIOPCIDevice *vdev) { + int ret; + vfio_disable_interrupts(vdev); vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries); @@ -640,8 +669,6 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) vfio_commit_kvm_msi_virq_batch(vdev); if (vdev->nr_vectors) { - int ret; - ret = vfio_enable_vectors(vdev, true); if (ret) { error_report("vfio: failed to enable vectors, %d", ret); @@ -655,13 +682,14 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) * MSI-X capability, but leaves the vector table masked. We therefore * can't rely on a vector_use callback (from request_irq() in the guest) * to switch the physical device into MSI-X mode because that may come a - * long time after pci_enable_msix(). This code enables vector 0 with - * triggering to userspace, then immediately release the vector, leaving - * the physical device with no vectors enabled, but MSI-X enabled, just - * like the guest view. + * long time after pci_enable_msix(). This code sets vector 0 with an + * invalid fd to make the physical device MSI-X enabled, but with no + * vectors enabled, just like the guest view. */ - vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL); - vfio_msix_vector_release(&vdev->pdev, 0); + ret = vfio_enable_msix_no_vec(vdev); + if (ret) { + error_report("vfio: failed to enable MSI-X, %d", ret); + } } trace_vfio_msix_enable(vdev->vbasedev.name); -- cgit v1.1 From eaadba6f9b14823e52ee154d0052d69907deee8a Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Mon, 25 Sep 2023 22:14:07 -0400 Subject: vfio/pci: enable MSI-X in interrupt restoring on dynamic allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During migration restoring, vfio_enable_vectors() is called to restore enabling MSI-X interrupts for assigned devices. It sets the range from 0 to nr_vectors to kernel to enable MSI-X and the vectors unmasked in guest. During the MSI-X enabling, all the vectors within the range are allocated according to the VFIO_DEVICE_SET_IRQS ioctl. When dynamic MSI-X allocation is supported, we only want the guest unmasked vectors being allocated and enabled. Use vector 0 with an invalid fd to get MSI-X enabled, after that, all the vectors can be allocated in need. Signed-off-by: Jing Liu Reviewed-by: Cédric Le Goater Reviewed-by: Alex Williamson Signed-off-by: Cédric Le Goater --- hw/vfio/pci.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index ad508ab..898296f 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -402,6 +402,23 @@ static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix) int ret = 0, i, argsz; int32_t *fds; + /* + * If dynamic MSI-X allocation is supported, the vectors to be allocated + * and enabled can be scattered. Before kernel enabling MSI-X, setting + * nr_vectors causes all these vectors to be allocated on host. + * + * To keep allocation as needed, use vector 0 with an invalid fd to get + * MSI-X enabled first, then set vectors with a potentially sparse set of + * eventfds to enable interrupts only when enabled in guest. + */ + if (msix && !vdev->msix->noresize) { + ret = vfio_enable_msix_no_vec(vdev); + + if (ret) { + return ret; + } + } + argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds)); irq_set = g_malloc0(argsz); -- cgit v1.1