aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorStefan Hajnoczi <stefanha@redhat.com>2023-10-09 10:11:35 -0400
committerStefan Hajnoczi <stefanha@redhat.com>2023-10-09 10:11:35 -0400
commitcea3ea670fe265421131aad90c36fbb87bc4d206 (patch)
tree3ddc7dc2f74ebb103b157601269211c3a3592836
parent1527c6b6fa6c6775523287e33f78b41afc7ba46c (diff)
parenteaadba6f9b14823e52ee154d0052d69907deee8a (diff)
downloadqemu-cea3ea670fe265421131aad90c36fbb87bc4d206.zip
qemu-cea3ea670fe265421131aad90c36fbb87bc4d206.tar.gz
qemu-cea3ea670fe265421131aad90c36fbb87bc4d206.tar.bz2
Merge tag 'pull-vfio-20231009' of https://github.com/legoater/qemu into staging
vfio queue: * Fix for VFIO display when using Intel vGPUs * Support for dynamic MSI-X # -----BEGIN PGP SIGNATURE----- # # iQIzBAABCAAdFiEEoPZlSPBIlev+awtgUaNDx8/77KEFAmUjoLIACgkQUaNDx8/7 # 7KE+gw/9FTQFRkmlkSMlqRGjINF/VmfX6TsX+dy3ZB+aJia6qahco+u9hd3yQxiA # /KI4FZnQCH/ZFizjR7hJdsxLnd+l989RFmoy+NTEXfgBMSLu4aU1UlVC1pyuhJ5L # xadGQ2UIclD1Gz70laa9ketebLHdyc/Pku2xt9oreR6kRRFHZ3V4QhMNhcwGapO1 # 0wytLFXPVyGa7YYTB5qQPHPWyY9sM0n6E4E7jVnhfOw75cUVNvSr+9HlJbR1FN3Z # 4klNMXayKGAZmh9oKpQWBsf4aUwLDu//eCk64TkQHp0pNrvRAJJBwgkhsI1FigeW # SJ2JjQsIg/vLu2oyUhp2PJ59cQSMFZPgEqRhhRQ2RKhIfwOZY4kgfvKFtSHvWijV # u0r8/HMIJE0fNffigyDlfLCsUEYu3OuJXMlU+5xrwi77hWlPrGb8D1J7LhwUnldk # kZaw9VEranlbMQT773cMA7f/pgS1Sc6CkdqfJLGIHA4PsEk44Lzen2BzRroz8+Km # tn8hHt+GQK/ZGKmOPXWm44Bd48Be08cMz/pOI2cqoScEKKEQ8HUul3H1/k8sqauh # 1gPo1hIPXo/GaGRvUvPsj4cK8oQm77EHksEQ4Nxvn+ZWTW2FnMQkb9QFbF8bTmEo # KiJJ6s8qbd1CWGYbO0GSE8ss3NUZq1YbWsMXmUP0JccEgvjeL2M= # =QRhQ # -----END PGP SIGNATURE----- # gpg: Signature made Mon 09 Oct 2023 02:41:54 EDT # gpg: using RSA key A0F66548F04895EBFE6B0B6051A343C7CFFBECA1 # gpg: Good signature from "Cédric Le Goater <clg@redhat.com>" [unknown] # gpg: aka "Cédric Le Goater <clg@kaod.org>" [unknown] # gpg: WARNING: This key is not certified with a trusted signature! # gpg: There is no indication that the signature belongs to the owner. # Primary key fingerprint: A0F6 6548 F048 95EB FE6B 0B60 51A3 43C7 CFFB ECA1 * tag 'pull-vfio-20231009' of https://github.com/legoater/qemu: vfio/pci: enable MSI-X in interrupt restoring on dynamic allocation vfio/pci: use an invalid fd to enable MSI-X vfio/pci: enable vector on dynamic MSI-X allocation vfio/pci: detect the support of dynamic MSI-X allocation vfio/pci: rename vfio_put_device to vfio_pci_put_device vfio/display: Fix missing update to set backing fields Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
-rw-r--r--hw/vfio/display.c2
-rw-r--r--hw/vfio/pci.c127
-rw-r--r--hw/vfio/pci.h1
-rw-r--r--hw/vfio/trace-events2
4 files changed, 101 insertions, 31 deletions
diff --git a/hw/vfio/display.c b/hw/vfio/display.c
index bec864f..837d9e6 100644
--- a/hw/vfio/display.c
+++ b/hw/vfio/display.c
@@ -243,6 +243,8 @@ static VFIODMABuf *vfio_display_get_dmabuf(VFIOPCIDevice *vdev,
dmabuf->dmabuf_id = plane.dmabuf_id;
dmabuf->buf.width = plane.width;
dmabuf->buf.height = plane.height;
+ dmabuf->buf.backing_width = plane.width;
+ dmabuf->buf.backing_height = plane.height;
dmabuf->buf.stride = plane.stride;
dmabuf->buf.fourcc = plane.drm_format;
dmabuf->buf.modifier = plane.drm_format_mod;
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 3b2ca3c..898296f 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -369,12 +369,56 @@ static void vfio_msi_interrupt(void *opaque)
notify(&vdev->pdev, nr);
}
+/*
+ * Get MSI-X enabled, but no vector enabled, by setting vector 0 with an invalid
+ * fd to kernel.
+ */
+static int vfio_enable_msix_no_vec(VFIOPCIDevice *vdev)
+{
+ g_autofree struct vfio_irq_set *irq_set = NULL;
+ int ret = 0, argsz;
+ int32_t *fd;
+
+ argsz = sizeof(*irq_set) + sizeof(*fd);
+
+ irq_set = g_malloc0(argsz);
+ irq_set->argsz = argsz;
+ irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+ VFIO_IRQ_SET_ACTION_TRIGGER;
+ irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+ irq_set->start = 0;
+ irq_set->count = 1;
+ fd = (int32_t *)&irq_set->data;
+ *fd = -1;
+
+ ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
+
+ return ret;
+}
+
static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
{
struct vfio_irq_set *irq_set;
int ret = 0, i, argsz;
int32_t *fds;
+ /*
+ * If dynamic MSI-X allocation is supported, the vectors to be allocated
+ * and enabled can be scattered. Before kernel enabling MSI-X, setting
+ * nr_vectors causes all these vectors to be allocated on host.
+ *
+ * To keep allocation as needed, use vector 0 with an invalid fd to get
+ * MSI-X enabled first, then set vectors with a potentially sparse set of
+ * eventfds to enable interrupts only when enabled in guest.
+ */
+ if (msix && !vdev->msix->noresize) {
+ ret = vfio_enable_msix_no_vec(vdev);
+
+ if (ret) {
+ return ret;
+ }
+ }
+
argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
irq_set = g_malloc0(argsz);
@@ -470,6 +514,7 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
VFIOPCIDevice *vdev = VFIO_PCI(pdev);
VFIOMSIVector *vector;
int ret;
+ bool resizing = !!(vdev->nr_vectors < nr + 1);
trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
@@ -512,33 +557,42 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
}
/*
- * We don't want to have the host allocate all possible MSI vectors
- * for a device if they're not in use, so we shutdown and incrementally
- * increase them as needed.
+ * When dynamic allocation is not supported, we don't want to have the
+ * host allocate all possible MSI vectors for a device if they're not
+ * in use, so we shutdown and incrementally increase them as needed.
+ * nr_vectors represents the total number of vectors allocated.
+ *
+ * When dynamic allocation is supported, let the host only allocate
+ * and enable a vector when it is in use in guest. nr_vectors represents
+ * the upper bound of vectors being enabled (but not all of the ranges
+ * is allocated or enabled).
*/
- if (vdev->nr_vectors < nr + 1) {
+ if (resizing) {
vdev->nr_vectors = nr + 1;
- if (!vdev->defer_kvm_irq_routing) {
+ }
+
+ if (!vdev->defer_kvm_irq_routing) {
+ if (vdev->msix->noresize && resizing) {
vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
ret = vfio_enable_vectors(vdev, true);
if (ret) {
error_report("vfio: failed to enable vectors, %d", ret);
}
- }
- } else {
- Error *err = NULL;
- int32_t fd;
-
- if (vector->virq >= 0) {
- fd = event_notifier_get_fd(&vector->kvm_interrupt);
} else {
- fd = event_notifier_get_fd(&vector->interrupt);
- }
+ Error *err = NULL;
+ int32_t fd;
- if (vfio_set_irq_signaling(&vdev->vbasedev,
- VFIO_PCI_MSIX_IRQ_INDEX, nr,
- VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
- error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
+ if (vector->virq >= 0) {
+ fd = event_notifier_get_fd(&vector->kvm_interrupt);
+ } else {
+ fd = event_notifier_get_fd(&vector->interrupt);
+ }
+
+ if (vfio_set_irq_signaling(&vdev->vbasedev,
+ VFIO_PCI_MSIX_IRQ_INDEX, nr,
+ VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
+ error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
+ }
}
}
@@ -608,6 +662,8 @@ static void vfio_commit_kvm_msi_virq_batch(VFIOPCIDevice *vdev)
static void vfio_msix_enable(VFIOPCIDevice *vdev)
{
+ int ret;
+
vfio_disable_interrupts(vdev);
vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
@@ -630,8 +686,6 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
vfio_commit_kvm_msi_virq_batch(vdev);
if (vdev->nr_vectors) {
- int ret;
-
ret = vfio_enable_vectors(vdev, true);
if (ret) {
error_report("vfio: failed to enable vectors, %d", ret);
@@ -645,13 +699,14 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
* MSI-X capability, but leaves the vector table masked. We therefore
* can't rely on a vector_use callback (from request_irq() in the guest)
* to switch the physical device into MSI-X mode because that may come a
- * long time after pci_enable_msix(). This code enables vector 0 with
- * triggering to userspace, then immediately release the vector, leaving
- * the physical device with no vectors enabled, but MSI-X enabled, just
- * like the guest view.
+ * long time after pci_enable_msix(). This code sets vector 0 with an
+ * invalid fd to make the physical device MSI-X enabled, but with no
+ * vectors enabled, just like the guest view.
*/
- vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
- vfio_msix_vector_release(&vdev->pdev, 0);
+ ret = vfio_enable_msix_no_vec(vdev);
+ if (ret) {
+ error_report("vfio: failed to enable MSI-X, %d", ret);
+ }
}
trace_vfio_msix_enable(vdev->vbasedev.name);
@@ -1493,7 +1548,9 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
uint8_t pos;
uint16_t ctrl;
uint32_t table, pba;
- int fd = vdev->vbasedev.fd;
+ int ret, fd = vdev->vbasedev.fd;
+ struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
+ .index = VFIO_PCI_MSIX_IRQ_INDEX };
VFIOMSIXInfo *msix;
pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
@@ -1530,6 +1587,15 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
+ ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
+ if (ret < 0) {
+ error_setg_errno(errp, -ret, "failed to get MSI-X irq info");
+ g_free(msix);
+ return;
+ }
+
+ msix->noresize = !!(irq_info.flags & VFIO_IRQ_INFO_NORESIZE);
+
/*
* Test the size of the pba_offset variable and catch if it extends outside
* of the specified BAR. If it is the case, we need to apply a hardware
@@ -1562,7 +1628,8 @@ static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
}
trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
- msix->table_offset, msix->entries);
+ msix->table_offset, msix->entries,
+ msix->noresize);
vdev->msix = msix;
vfio_pci_fixup_msix_region(vdev);
@@ -2826,7 +2893,7 @@ static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
}
}
-static void vfio_put_device(VFIOPCIDevice *vdev)
+static void vfio_pci_put_device(VFIOPCIDevice *vdev)
{
g_free(vdev->vbasedev.name);
g_free(vdev->msix);
@@ -3317,7 +3384,7 @@ static void vfio_instance_finalize(Object *obj)
*
* g_free(vdev->igd_opregion);
*/
- vfio_put_device(vdev);
+ vfio_pci_put_device(vdev);
vfio_put_group(group);
}
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
index 2d83609..0d89eb7 100644
--- a/hw/vfio/pci.h
+++ b/hw/vfio/pci.h
@@ -113,6 +113,7 @@ typedef struct VFIOMSIXInfo {
uint32_t table_offset;
uint32_t pba_offset;
unsigned long *pending;
+ bool noresize;
} VFIOMSIXInfo;
#define TYPE_VFIO_PCI "vfio-pci"
diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
index e64ca4a..0ba3c5a 100644
--- a/hw/vfio/trace-events
+++ b/hw/vfio/trace-events
@@ -27,7 +27,7 @@ vfio_vga_read(uint64_t addr, int size, uint64_t data) " (0x%"PRIx64", %d) = 0x%"
vfio_pci_read_config(const char *name, int addr, int len, int val) " (%s, @0x%x, len=0x%x) 0x%x"
vfio_pci_write_config(const char *name, int addr, int val, int len) " (%s, @0x%x, 0x%x, len=0x%x)"
vfio_msi_setup(const char *name, int pos) "%s PCI MSI CAP @0x%x"
-vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d"
+vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries, bool noresize) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d, noresize %d"
vfio_check_pcie_flr(const char *name) "%s Supports FLR via PCIe cap"
vfio_check_pm_reset(const char *name) "%s Supports PM reset"
vfio_check_af_flr(const char *name) "%s Supports FLR via AF cap"