From 6cd1fe11598a304b8d22203365f4fc39172ec9b2 Mon Sep 17 00:00:00 2001 From: Avihai Horon Date: Wed, 21 Jun 2023 14:11:59 +0300 Subject: vfio/migration: Store VFIO migration flags in VFIOMigration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VFIO migration flags are queried once in vfio_migration_init(). Store them in VFIOMigration so they can be used later to check the device's migration capabilities without re-querying them. This will be used in the next patch to check if the device supports precopy migration. Signed-off-by: Avihai Horon Reviewed-by: Cédric Le Goater Tested-by: YangHang Liu Acked-by: Alex Williamson Signed-off-by: Cédric Le Goater --- include/hw/vfio/vfio-common.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/hw') diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index eed244f..5f29dab 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -66,6 +66,7 @@ typedef struct VFIOMigration { int data_fd; void *data_buffer; size_t data_buffer_size; + uint64_t mig_flags; } VFIOMigration; typedef struct VFIOAddressSpace { -- cgit v1.1 From eda7362af9595a9c6b1f1fefdd94b5ef711c250c Mon Sep 17 00:00:00 2001 From: Avihai Horon Date: Wed, 21 Jun 2023 14:12:00 +0300 Subject: vfio/migration: Add VFIO migration pre-copy support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-copy support allows the VFIO device data to be transferred while the VM is running. This helps to accommodate VFIO devices that have a large amount of data that needs to be transferred, and it can reduce migration downtime. Pre-copy support is optional in VFIO migration protocol v2. Implement pre-copy of VFIO migration protocol v2 and use it for devices that support it. Full description of it can be found in the following Linux commit: 4db52602a607 ("vfio: Extend the device migration protocol with PRE_COPY"). Signed-off-by: Avihai Horon Reviewed-by: Cédric Le Goater Tested-by: YangHang Liu Acked-by: Alex Williamson Signed-off-by: Cédric Le Goater --- include/hw/vfio/vfio-common.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/hw') diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index 5f29dab..1db901c 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -67,6 +67,8 @@ typedef struct VFIOMigration { void *data_buffer; size_t data_buffer_size; uint64_t mig_flags; + uint64_t precopy_init_size; + uint64_t precopy_dirty_size; } VFIOMigration; typedef struct VFIOAddressSpace { -- cgit v1.1 From 745c42912a04306e8c7765dd4cc1574a92de4446 Mon Sep 17 00:00:00 2001 From: Avihai Horon Date: Wed, 21 Jun 2023 14:12:01 +0300 Subject: vfio/migration: Add support for switchover ack capability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loading of a VFIO device's data can take a substantial amount of time as the device may need to allocate resources, prepare internal data structures, etc. This can increase migration downtime, especially for VFIO devices with a lot of resources. To solve this, VFIO migration uAPI defines "initial bytes" as part of its precopy data stream. Initial bytes can be used in various ways to improve VFIO migration performance. For example, it can be used to transfer device metadata to pre-allocate resources in the destination. However, for this to work we need to make sure that all initial bytes are sent and loaded in the destination before the source VM is stopped. Use migration switchover ack capability to make sure a VFIO device's initial bytes are sent and loaded in the destination before the source stops the VM and attempts to complete the migration. This can significantly reduce migration downtime for some devices. Signed-off-by: Avihai Horon Reviewed-by: Cédric Le Goater Tested-by: YangHang Liu Acked-by: Alex Williamson Signed-off-by: Cédric Le Goater --- include/hw/vfio/vfio-common.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/hw') diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index 1db901c..3dc5f21 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -69,6 +69,7 @@ typedef struct VFIOMigration { uint64_t mig_flags; uint64_t precopy_init_size; uint64_t precopy_dirty_size; + bool initial_data_sent; } VFIOMigration; typedef struct VFIOAddressSpace { -- cgit v1.1 From 634f38f0f73f12b86a53195782b26506c107a418 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Thu, 1 Jun 2023 08:45:06 -0600 Subject: vfio: Implement a common device info helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A common helper implementing the realloc algorithm for handling capabilities. Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Cédric Le Goater Signed-off-by: Alex Williamson Reviewed-by: Robin Voetter Signed-off-by: Cédric Le Goater --- include/hw/vfio/vfio-common.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/hw') diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index 3dc5f21..6d1b848 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -216,6 +216,7 @@ void vfio_region_finalize(VFIORegion *region); void vfio_reset_handler(void *opaque); VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp); void vfio_put_group(VFIOGroup *group); +struct vfio_device_info *vfio_get_device_info(int fd); int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vbasedev, Error **errp); -- cgit v1.1 From 808642a2f6407157f8cfb48ba6c2d28efd038d62 Mon Sep 17 00:00:00 2001 From: Avihai Horon Date: Wed, 28 Jun 2023 10:31:11 +0300 Subject: vfio/migration: Reset bytes_transferred properly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, VFIO bytes_transferred is not reset properly: 1. bytes_transferred is not reset after a VM snapshot (so a migration following a snapshot will report incorrect value). 2. bytes_transferred is a single counter for all VFIO devices, however upon migration failure it is reset multiple times, by each VFIO device. Fix it by introducing a new function vfio_reset_bytes_transferred() and calling it during migration and snapshot start. Remove existing bytes_transferred reset in VFIO migration state notifier, which is not needed anymore. Fixes: 3710586caa5d ("qapi: Add VFIO devices migration stats in Migration stats") Signed-off-by: Avihai Horon Reviewed-by: Cédric Le Goater Reviewed-by: Alex Williamson Signed-off-by: Cédric Le Goater --- include/hw/vfio/vfio-common.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/hw') diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index 6d1b848..1d19c6f 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -229,6 +229,7 @@ int vfio_block_multiple_devices_migration(Error **errp); void vfio_unblock_multiple_devices_migration(void); int vfio_block_giommu_migration(Error **errp); int64_t vfio_mig_bytes_transferred(void); +void vfio_reset_bytes_transferred(void); #ifdef CONFIG_LINUX int vfio_get_region_info(VFIODevice *vbasedev, int index, -- cgit v1.1 From 8bbcb64a71d84627b0171a205a5f3586eaa1e081 Mon Sep 17 00:00:00 2001 From: Avihai Horon Date: Wed, 28 Jun 2023 10:31:12 +0300 Subject: vfio/migration: Make VFIO migration non-experimental MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The major parts of VFIO migration are supported today in QEMU. This includes basic VFIO migration, device dirty page tracking and precopy support. Thus, at this point in time, it seems appropriate to make VFIO migration non-experimental: remove the x prefix from enable_migration property, change it to ON_OFF_AUTO and let the default value be AUTO. In addition, make the following adjustments: 1. When enable_migration is ON and migration is not supported, fail VFIO device realization. 2. When enable_migration is AUTO (i.e., not explicitly enabled), require device dirty tracking support. This is because device dirty tracking is currently the only method to do dirty page tracking, which is essential for migrating in a reasonable downtime. Setting enable_migration to ON will not require device dirty tracking. 3. Make migration error and blocker messages more elaborate. 4. Remove error prints in vfio_migration_query_flags(). 5. Rename trace_vfio_migration_probe() to trace_vfio_migration_realize(). Signed-off-by: Avihai Horon Reviewed-by: Joao Martins Reviewed-by: Cédric Le Goater Reviewed-by: Alex Williamson Signed-off-by: Cédric Le Goater --- include/hw/vfio/vfio-common.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/hw') diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index 1d19c6f..93429b9 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -139,7 +139,7 @@ typedef struct VFIODevice { bool needs_reset; bool no_mmap; bool ram_block_discard_allowed; - bool enable_migration; + OnOffAuto enable_migration; VFIODeviceOps *ops; unsigned int num_irqs; unsigned int num_regions; @@ -225,9 +225,9 @@ typedef QLIST_HEAD(VFIOGroupList, VFIOGroup) VFIOGroupList; extern VFIOGroupList vfio_group_list; bool vfio_mig_active(void); -int vfio_block_multiple_devices_migration(Error **errp); +int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp); void vfio_unblock_multiple_devices_migration(void); -int vfio_block_giommu_migration(Error **errp); +int vfio_block_giommu_migration(VFIODevice *vbasedev, Error **errp); int64_t vfio_mig_bytes_transferred(void); void vfio_reset_bytes_transferred(void); -- cgit v1.1