From 3f6ed59ec400a58646e766a323f7f8047f044b98 Mon Sep 17 00:00:00 2001 From: Avihai Horon Date: Mon, 4 Mar 2024 12:53:38 +0200 Subject: vfio/migration: Refactor vfio_save_state() return value Currently, vfio_save_state() returns 1 regardless of whether there is more data to send or not. This was done to prevent a fast changing VFIO device from potentially blocking other devices from sending their data, as qemu_savevm_state_iterate() serialized devices. Now that qemu_savevm_state_iterate() no longer serializes devices, there is no need for that. Refactor vfio_save_state() to return 0 if more data is available and 1 if no more data is available. Signed-off-by: Avihai Horon Reviewed-by: Fabiano Rosas Link: https://lore.kernel.org/r/20240304105339.20713-3-avihaih@nvidia.com Signed-off-by: Peter Xu --- hw/vfio/migration.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'hw') diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c index 50140ed..0af783a 100644 --- a/hw/vfio/migration.c +++ b/hw/vfio/migration.c @@ -529,11 +529,7 @@ static int vfio_save_iterate(QEMUFile *f, void *opaque) trace_vfio_save_iterate(vbasedev->name, migration->precopy_init_size, migration->precopy_dirty_size); - /* - * A VFIO device's pre-copy dirty_bytes is not guaranteed to reach zero. - * Return 1 so following handlers will not be potentially blocked. - */ - return 1; + return !migration->precopy_init_size && !migration->precopy_dirty_size; } static int vfio_save_complete_precopy(QEMUFile *f, void *opaque) -- cgit v1.1 From ff64e0ba817d79968bffa88d205397abf05cc6e8 Mon Sep 17 00:00:00 2001 From: Avihai Horon Date: Mon, 4 Mar 2024 12:53:39 +0200 Subject: vfio/migration: Add a note about migration rate limiting VFIO migration buffer size is currently limited to 1MB. Therefore, there is no need to check if migration rate exceeded, as in the worst case it will exceed by only 1MB. However, if the buffer size is later changed to a bigger value, vfio_save_iterate() should enforce migration rate (similar to migration RAM code). Add a note about this in vfio_save_iterate() to serve as a reminder. Suggested-by: Peter Xu Signed-off-by: Avihai Horon Reviewed-by: Fabiano Rosas Link: https://lore.kernel.org/r/20240304105339.20713-4-avihaih@nvidia.com Signed-off-by: Peter Xu --- hw/vfio/migration.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'hw') diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c index 0af783a..f82dcab 100644 --- a/hw/vfio/migration.c +++ b/hw/vfio/migration.c @@ -505,6 +505,12 @@ static bool vfio_is_active_iterate(void *opaque) return vfio_device_state_is_precopy(vbasedev); } +/* + * Note about migration rate limiting: VFIO migration buffer size is currently + * limited to 1MB, so there is no need to check if migration rate exceeded (as + * in the worst case it will exceed by 1MB). However, if the buffer size is + * later changed to a bigger value, migration rate should be enforced here. + */ static int vfio_save_iterate(QEMUFile *f, void *opaque) { VFIODevice *vbasedev = opaque; -- cgit v1.1 From f3bff6c44304a21ea99eeed336672bd46ca102d5 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Mon, 11 Mar 2024 10:48:39 -0700 Subject: migration: export fewer options A small number of migration options are accessed by migration clients, but to see them clients must include all of options.h, which is mostly for migration core code. migrate_mode() in particular will be needed by multiple clients. Refactor the option declarations so clients can see the necessary few via misc.h, which already exports a portion of the client API. Signed-off-by: Steve Sistare Link: https://lore.kernel.org/r/1710179319-294320-1-git-send-email-steven.sistare@oracle.com Signed-off-by: Peter Xu --- hw/vfio/migration.c | 1 - hw/virtio/virtio-balloon.c | 1 - 2 files changed, 2 deletions(-) (limited to 'hw') diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c index f82dcab..49c0016 100644 --- a/hw/vfio/migration.c +++ b/hw/vfio/migration.c @@ -18,7 +18,6 @@ #include "sysemu/runstate.h" #include "hw/vfio/vfio-common.h" #include "migration/migration.h" -#include "migration/options.h" #include "migration/savevm.h" #include "migration/vmstate.h" #include "migration/qemu-file.h" diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c index 89f853f..a59ff17 100644 --- a/hw/virtio/virtio-balloon.c +++ b/hw/virtio/virtio-balloon.c @@ -32,7 +32,6 @@ #include "qemu/error-report.h" #include "migration/misc.h" #include "migration/migration.h" -#include "migration/options.h" #include "hw/virtio/virtio-bus.h" #include "hw/virtio/virtio-access.h" -- cgit v1.1 From f853fa0714061989b14d7f85667a353b69e2d9b6 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Mon, 11 Mar 2024 10:48:48 -0700 Subject: migration: remove migration.h references Remove migration.h from files that no longer need it due to previous commits. Signed-off-by: Steve Sistare Link: https://lore.kernel.org/r/1710179338-294359-2-git-send-email-steven.sistare@oracle.com Signed-off-by: Peter Xu --- hw/vfio/container.c | 1 - hw/virtio/vhost-user.c | 1 - hw/virtio/virtio-balloon.c | 1 - 3 files changed, 3 deletions(-) (limited to 'hw') diff --git a/hw/vfio/container.c b/hw/vfio/container.c index bd25b9f..ff081a1 100644 --- a/hw/vfio/container.c +++ b/hw/vfio/container.c @@ -32,7 +32,6 @@ #include "sysemu/reset.h" #include "trace.h" #include "qapi/error.h" -#include "migration/migration.h" #include "pci.h" VFIOGroupList vfio_group_list = diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index a1eea85..1af8621 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -26,7 +26,6 @@ #include "qemu/sockets.h" #include "sysemu/runstate.h" #include "sysemu/cryptodev.h" -#include "migration/migration.h" #include "migration/postcopy-ram.h" #include "trace.h" #include "exec/ramblock.h" diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c index a59ff17..609e39a 100644 --- a/hw/virtio/virtio-balloon.c +++ b/hw/virtio/virtio-balloon.c @@ -31,7 +31,6 @@ #include "trace.h" #include "qemu/error-report.h" #include "migration/misc.h" -#include "migration/migration.h" #include "hw/virtio/virtio-bus.h" #include "hw/virtio/virtio-access.h" -- cgit v1.1 From 7dcb3c87d87cbf3359c9bbef8066e5195ee92f4d Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Mon, 11 Mar 2024 10:48:49 -0700 Subject: migration: export migration_is_setup_or_active MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete the MigrationState parameter from migration_is_setup_or_active and move it to the public API in misc.h. Signed-off-by: Steve Sistare Reviewed-by: Philippe Mathieu-Daudé Link: https://lore.kernel.org/r/1710179338-294359-3-git-send-email-steven.sistare@oracle.com Signed-off-by: Peter Xu --- hw/vfio/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'hw') diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 059bfdc..896eab8 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -152,7 +152,7 @@ static void vfio_set_migration_error(int err) { MigrationState *ms = migrate_get_current(); - if (migration_is_setup_or_active(ms->state)) { + if (migration_is_setup_or_active()) { WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) { if (ms->to_dst_file) { qemu_file_set_error(ms->to_dst_file, err); -- cgit v1.1 From 3a6813b68cf21a366522564f83f885bf2657dcc8 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Mon, 11 Mar 2024 10:48:50 -0700 Subject: migration: export migration_is_active Delete the MigrationState parameter from migration_is_active so it can be exported and used without including migration.h. Signed-off-by: Steve Sistare Link: https://lore.kernel.org/r/1710179338-294359-4-git-send-email-steven.sistare@oracle.com Signed-off-by: Peter Xu --- hw/vfio/common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'hw') diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 896eab8..2dbbf62 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -182,7 +182,7 @@ static bool vfio_devices_all_dirty_tracking(VFIOContainerBase *bcontainer) VFIODevice *vbasedev; MigrationState *ms = migrate_get_current(); - if (ms->state != MIGRATION_STATUS_ACTIVE && + if (!migration_is_active() && ms->state != MIGRATION_STATUS_DEVICE) { return false; } @@ -225,7 +225,7 @@ vfio_devices_all_running_and_mig_active(const VFIOContainerBase *bcontainer) { VFIODevice *vbasedev; - if (!migration_is_active(migrate_get_current())) { + if (!migration_is_active()) { return false; } -- cgit v1.1 From 9bb630c6ee6e930e1c521829681365e5672bbd99 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Mon, 11 Mar 2024 10:48:54 -0700 Subject: migration: migration_is_device Define and export migration_is_device to eliminate a dependency on MigrationState. Signed-off-by: Steve Sistare Link: https://lore.kernel.org/r/1710179338-294359-8-git-send-email-steven.sistare@oracle.com Signed-off-by: Peter Xu --- hw/vfio/common.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'hw') diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 2dbbf62..de01068 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -180,10 +180,8 @@ bool vfio_device_state_is_precopy(VFIODevice *vbasedev) static bool vfio_devices_all_dirty_tracking(VFIOContainerBase *bcontainer) { VFIODevice *vbasedev; - MigrationState *ms = migrate_get_current(); - if (!migration_is_active() && - ms->state != MIGRATION_STATUS_DEVICE) { + if (!migration_is_active() && !migration_is_device()) { return false; } -- cgit v1.1 From 20c64c8a51a477115589b3a9f7304e423dcbe7f2 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Mon, 11 Mar 2024 10:48:55 -0700 Subject: migration: migration_file_set_error Define and export migration_file_set_error to eliminate a dependency on MigrationState. Signed-off-by: Steve Sistare Link: https://lore.kernel.org/r/1710179338-294359-9-git-send-email-steven.sistare@oracle.com Signed-off-by: Peter Xu --- hw/vfio/common.c | 9 +-------- hw/vfio/migration.c | 11 +++-------- 2 files changed, 4 insertions(+), 16 deletions(-) (limited to 'hw') diff --git a/hw/vfio/common.c b/hw/vfio/common.c index de01068..b44204e 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -39,7 +39,6 @@ #include "sysemu/runstate.h" #include "trace.h" #include "qapi/error.h" -#include "migration/migration.h" #include "migration/misc.h" #include "migration/blocker.h" #include "migration/qemu-file.h" @@ -150,14 +149,8 @@ bool vfio_viommu_preset(VFIODevice *vbasedev) static void vfio_set_migration_error(int err) { - MigrationState *ms = migrate_get_current(); - if (migration_is_setup_or_active()) { - WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) { - if (ms->to_dst_file) { - qemu_file_set_error(ms->to_dst_file, err); - } - } + migration_file_set_error(err); } } diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c index 49c0016..a15fd48 100644 --- a/hw/vfio/migration.c +++ b/hw/vfio/migration.c @@ -17,13 +17,12 @@ #include "sysemu/runstate.h" #include "hw/vfio/vfio-common.h" -#include "migration/migration.h" +#include "migration/misc.h" #include "migration/savevm.h" #include "migration/vmstate.h" #include "migration/qemu-file.h" #include "migration/register.h" #include "migration/blocker.h" -#include "migration/misc.h" #include "qapi/error.h" #include "exec/ramlist.h" #include "exec/ram_addr.h" @@ -714,9 +713,7 @@ static void vfio_vmstate_change_prepare(void *opaque, bool running, * Migration should be aborted in this case, but vm_state_notify() * currently does not support reporting failures. */ - if (migrate_get_current()->to_dst_file) { - qemu_file_set_error(migrate_get_current()->to_dst_file, ret); - } + migration_file_set_error(ret); } trace_vfio_vmstate_change_prepare(vbasedev->name, running, @@ -746,9 +743,7 @@ static void vfio_vmstate_change(void *opaque, bool running, RunState state) * Migration should be aborted in this case, but vm_state_notify() * currently does not support reporting failures. */ - if (migrate_get_current()->to_dst_file) { - qemu_file_set_error(migrate_get_current()->to_dst_file, ret); - } + migration_file_set_error(ret); } trace_vfio_vmstate_change(vbasedev->name, running, RunState_str(state), -- cgit v1.1 From 5fdbb1dfccfd59661c95cae760b8e276c5b8e65c Mon Sep 17 00:00:00 2001 From: Hao Xiang Date: Mon, 11 Mar 2024 18:00:11 +0000 Subject: migration/multifd: Add new migration option zero-page-detection. This new parameter controls where the zero page checking is running. 1. If this parameter is set to 'legacy', zero page checking is done in the migration main thread. 2. If this parameter is set to 'none', zero page checking is disabled. Signed-off-by: Hao Xiang Reviewed-by: Peter Xu Acked-by: Markus Armbruster Link: https://lore.kernel.org/r/20240311180015.3359271-4-hao.xiang@linux.dev Signed-off-by: Peter Xu --- hw/core/qdev-properties-system.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'hw') diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c index b45e90e..71a21bf 100644 --- a/hw/core/qdev-properties-system.c +++ b/hw/core/qdev-properties-system.c @@ -693,6 +693,16 @@ const PropertyInfo qdev_prop_granule_mode = { .set_default_value = qdev_propinfo_set_default_value_enum, }; +const PropertyInfo qdev_prop_zero_page_detection = { + .name = "ZeroPageDetection", + .description = "zero_page_detection values, " + "none,legacy", + .enum_table = &ZeroPageDetection_lookup, + .get = qdev_propinfo_get_enum, + .set = qdev_propinfo_set_enum, + .set_default_value = qdev_propinfo_set_default_value_enum, +}; + /* --- Reserved Region --- */ /* -- cgit v1.1 From 303e6f54f9657be76ee060006ee2d4cacff263a0 Mon Sep 17 00:00:00 2001 From: Hao Xiang Date: Mon, 11 Mar 2024 18:00:12 +0000 Subject: migration/multifd: Implement zero page transmission on the multifd thread. 1. Add zero_pages field in MultiFDPacket_t. 2. Implements the zero page detection and handling on the multifd threads for non-compression, zlib and zstd compression backends. 3. Added a new value 'multifd' in ZeroPageDetection enumeration. 4. Adds zero page counters and updates multifd send/receive tracing format to track the newly added counters. Signed-off-by: Hao Xiang Acked-by: Markus Armbruster Reviewed-by: Fabiano Rosas Link: https://lore.kernel.org/r/20240311180015.3359271-5-hao.xiang@linux.dev Signed-off-by: Peter Xu --- hw/core/qdev-properties-system.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'hw') diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c index 71a21bf..7eca2f2 100644 --- a/hw/core/qdev-properties-system.c +++ b/hw/core/qdev-properties-system.c @@ -696,7 +696,7 @@ const PropertyInfo qdev_prop_granule_mode = { const PropertyInfo qdev_prop_zero_page_detection = { .name = "ZeroPageDetection", .description = "zero_page_detection values, " - "none,legacy", + "none,legacy,multifd", .enum_table = &ZeroPageDetection_lookup, .get = qdev_propinfo_get_enum, .set = qdev_propinfo_set_enum, -- cgit v1.1 From 70c25c92e602f393d3c33596530c5f2b18491e55 Mon Sep 17 00:00:00 2001 From: Hao Xiang Date: Mon, 11 Mar 2024 18:00:14 +0000 Subject: migration/multifd: Enable multifd zero page checking by default. 1. Set default "zero-page-detection" option to "multifd". Now zero page checking can be done in the multifd threads and this becomes the default configuration. 2. Handle migration QEMU9.0 -> QEMU8.2 compatibility. We provide backward compatibility where zero page checking is done from the migration main thread. Signed-off-by: Hao Xiang Reviewed-by: Fabiano Rosas Reviewed-by: Peter Xu Link: https://lore.kernel.org/r/20240311180015.3359271-7-hao.xiang@linux.dev Signed-off-by: Peter Xu --- hw/core/machine.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'hw') diff --git a/hw/core/machine.c b/hw/core/machine.c index 9ac5d53..0e9d646 100644 --- a/hw/core/machine.c +++ b/hw/core/machine.c @@ -32,7 +32,9 @@ #include "hw/virtio/virtio-net.h" #include "audio/audio.h" -GlobalProperty hw_compat_8_2[] = {}; +GlobalProperty hw_compat_8_2[] = { + { "migration", "zero-page-detection", "legacy"}, +}; const size_t hw_compat_8_2_len = G_N_ELEMENTS(hw_compat_8_2); GlobalProperty hw_compat_8_1[] = { -- cgit v1.1