diff options
author | Stefan Hajnoczi <stefanha@redhat.com> | 2025-05-21 10:25:55 -0400 |
---|---|---|
committer | Stefan Hajnoczi <stefanha@redhat.com> | 2025-05-21 10:25:55 -0400 |
commit | c6e1f60cc73c787317316bb2956f9a95a5daee15 (patch) | |
tree | abcbdffb708bb2c96def5138a64f1dde0bd460ce | |
parent | f0737158b483e7ec2b2512145aeab888b85cc1f7 (diff) | |
parent | e09c6d837593aa1e12d92d7031c65a881eb2eb27 (diff) | |
download | qemu-c6e1f60cc73c787317316bb2956f9a95a5daee15.zip qemu-c6e1f60cc73c787317316bb2956f9a95a5daee15.tar.gz qemu-c6e1f60cc73c787317316bb2956f9a95a5daee15.tar.bz2 |
Merge tag 'migration-20250520-pull-request' of https://gitlab.com/peterx/qemu into staging
Migration pull
- Peter's vmstate static checker entry for recent hpet change
- Zhijian's rdma test patch to properly skip tests when locked_vm too low,
and ipv6 test
- Fabiano's few CI changes
- Prasad's last patches to enable postcopy on precopy-multifd
- Maciej's fix to disable multifd zerocopy for device states
- Peter's HMP change to "info migrate", plus a small fix on cap set
# -----BEGIN PGP SIGNATURE-----
#
# iIgEABYKADAWIQS5GE3CDMRX2s990ak7X8zN86vXBgUCaCy2xxIccGV0ZXJ4QHJl
# ZGhhdC5jb20ACgkQO1/MzfOr1waNZgD9HRflpY6sqcXt9wPKYgJFAyPN9Jg05kGO
# jeRIhD9cvyIBAN5AjbE0gaN8f+BLR5vcNk0Q5uOaHHIFZxL7tJl8ob4L
# =WTpL
# -----END PGP SIGNATURE-----
# gpg: Signature made Tue 20 May 2025 13:07:19 EDT
# gpg: using EDDSA key B9184DC20CC457DACF7DD1A93B5FCCCDF3ABD706
# gpg: issuer "peterx@redhat.com"
# gpg: Good signature from "Peter Xu <xzpeter@gmail.com>" [full]
# gpg: aka "Peter Xu <peterx@redhat.com>" [full]
# Primary key fingerprint: B918 4DC2 0CC4 57DA CF7D D1A9 3B5F CCCD F3AB D706
* tag 'migration-20250520-pull-request' of https://gitlab.com/peterx/qemu:
migration/hmp: Add "info migrate -a", reorg the dump
migration: Allow caps to be set when preempt or multifd cap enabled
migration/multifd: Don't send device state packets with zerocopy flag
tests/qtest/migration: add postcopy tests with multifd
migration: enable multifd and postcopy together
migration: write zero pages when postcopy enabled
ci: Reduce the size of artifacts for build-previous-qemu
ci: Fix build-previous-qemu when the version tag is absent
ci: Re-enable python subtests in qtest migration suite
qtest/migration/rdma: Add test for rdma migration with ipv6
qtest/migration/rdma: Enforce RLIMIT_MEMLOCK >= 128MB requirement
scripts/vmstate-static-checker.py: Add new hpet entry for num_timers
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
-rw-r--r-- | .gitlab-ci.d/buildtest.yml | 27 | ||||
-rw-r--r-- | hmp-commands-info.hx | 6 | ||||
-rw-r--r-- | migration/migration-hmp-cmds.c | 186 | ||||
-rw-r--r-- | migration/multifd-nocomp.c | 3 | ||||
-rw-r--r-- | migration/multifd-zero-page.c | 22 | ||||
-rw-r--r-- | migration/multifd.c | 14 | ||||
-rw-r--r-- | migration/options.c | 9 | ||||
-rw-r--r-- | migration/ram.c | 5 | ||||
-rwxr-xr-x | scripts/rdma-migration-helper.sh | 57 | ||||
-rwxr-xr-x | scripts/vmstate-static-checker.py | 1 | ||||
-rw-r--r-- | tests/qtest/migration/compression-tests.c | 18 | ||||
-rw-r--r-- | tests/qtest/migration/postcopy-tests.c | 27 | ||||
-rw-r--r-- | tests/qtest/migration/precopy-tests.c | 83 | ||||
-rw-r--r-- | tests/qtest/migration/tls-tests.c | 51 |
14 files changed, 373 insertions, 136 deletions
diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index 248aaed..ca1a9c6 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -188,12 +188,11 @@ build-previous-qemu: when: on_success expire_in: 2 days paths: - - build-previous - exclude: - - build-previous/**/*.p - - build-previous/**/*.a.p - - build-previous/**/*.c.o - - build-previous/**/*.c.o.d + - build-previous/qemu-bundle + - build-previous/qemu-system-aarch64 + - build-previous/qemu-system-x86_64 + - build-previous/tests/qtest/migration-test + - build-previous/scripts needs: job: amd64-opensuse-leap-container variables: @@ -203,6 +202,11 @@ build-previous-qemu: GIT_FETCH_EXTRA_FLAGS: --prune --quiet before_script: - source scripts/ci/gitlab-ci-section + # Skip if this series contains the release bump commit. During the + # release process there might be a window of commits when the + # version tag is not yet present in the remote and git fetch would + # fail. + - if grep -q "\.0$" VERSION; then exit 0; fi - export QEMU_PREV_VERSION="$(sed 's/\([0-9.]*\)\.[0-9]*/v\1.0/' VERSION)" - git remote add upstream https://gitlab.com/qemu-project/qemu - git fetch upstream refs/tags/$QEMU_PREV_VERSION:refs/tags/$QEMU_PREV_VERSION @@ -223,18 +227,13 @@ build-previous-qemu: IMAGE: opensuse-leap MAKE_CHECK_ARGS: check-build script: + # Skip for round release numbers, this job is only relevant for + # testing a development tree. + - if grep -q "\.0$" VERSION; then exit 0; fi # Use the migration-tests from the older QEMU tree. This avoids # testing an old QEMU against new features/tests that it is not # compatible with. - cd build-previous - # Don't allow python-based tests to run. The - # vmstate-checker-script test has a race that causes it to fail - # sometimes. It cannot be fixed it because this job runs the test - # from the old QEMU version. The test will be removed on master, - # but this job will only see the change in the next release. - # - # TODO: remove this line after 9.2 release - - unset PYTHON # old to new - QTEST_QEMU_BINARY_SRC=./qemu-system-${TARGET} QTEST_QEMU_BINARY=../build/qemu-system-${TARGET} ./tests/qtest/migration-test diff --git a/hmp-commands-info.hx b/hmp-commands-info.hx index c59cd66..639a450 100644 --- a/hmp-commands-info.hx +++ b/hmp-commands-info.hx @@ -475,9 +475,9 @@ ERST { .name = "migrate", - .args_type = "", - .params = "", - .help = "show migration status", + .args_type = "all:-a", + .params = "[-a]", + .help = "show migration status (-a: all, dump all status)", .cmd = hmp_info_migrate, }, diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c index 49c26da..e8a563c 100644 --- a/migration/migration-hmp-cmds.c +++ b/migration/migration-hmp-cmds.c @@ -37,29 +37,28 @@ static void migration_global_dump(Monitor *mon) { MigrationState *ms = migrate_get_current(); - monitor_printf(mon, "globals:\n"); - monitor_printf(mon, "store-global-state: %s\n", + monitor_printf(mon, "Globals:\n"); + monitor_printf(mon, " store-global-state: %s\n", ms->store_global_state ? "on" : "off"); - monitor_printf(mon, "only-migratable: %s\n", + monitor_printf(mon, " only-migratable: %s\n", only_migratable ? "on" : "off"); - monitor_printf(mon, "send-configuration: %s\n", + monitor_printf(mon, " send-configuration: %s\n", ms->send_configuration ? "on" : "off"); - monitor_printf(mon, "send-section-footer: %s\n", + monitor_printf(mon, " send-section-footer: %s\n", ms->send_section_footer ? "on" : "off"); - monitor_printf(mon, "send-switchover-start: %s\n", + monitor_printf(mon, " send-switchover-start: %s\n", ms->send_switchover_start ? "on" : "off"); - monitor_printf(mon, "clear-bitmap-shift: %u\n", + monitor_printf(mon, " clear-bitmap-shift: %u\n", ms->clear_bitmap_shift); } void hmp_info_migrate(Monitor *mon, const QDict *qdict) { + bool show_all = qdict_get_try_bool(qdict, "all", false); MigrationInfo *info; info = qmp_query_migrate(NULL); - migration_global_dump(mon); - if (info->blocked_reasons) { strList *reasons = info->blocked_reasons; monitor_printf(mon, "Outgoing migration blocked:\n"); @@ -70,7 +69,7 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict) } if (info->has_status) { - monitor_printf(mon, "Migration status: %s", + monitor_printf(mon, "Status: %s", MigrationStatus_str(info->status)); if (info->status == MIGRATION_STATUS_FAILED && info->error_desc) { monitor_printf(mon, " (%s)\n", info->error_desc); @@ -78,107 +77,130 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict) monitor_printf(mon, "\n"); } - monitor_printf(mon, "total time: %" PRIu64 " ms\n", - info->total_time); - if (info->has_expected_downtime) { - monitor_printf(mon, "expected downtime: %" PRIu64 " ms\n", - info->expected_downtime); - } - if (info->has_downtime) { - monitor_printf(mon, "downtime: %" PRIu64 " ms\n", - info->downtime); + if (info->total_time) { + monitor_printf(mon, "Time (ms): total=%" PRIu64, + info->total_time); + if (info->has_setup_time) { + monitor_printf(mon, ", setup=%" PRIu64, + info->setup_time); + } + if (info->has_expected_downtime) { + monitor_printf(mon, ", exp_down=%" PRIu64, + info->expected_downtime); + } + if (info->has_downtime) { + monitor_printf(mon, ", down=%" PRIu64, + info->downtime); + } + monitor_printf(mon, "\n"); } - if (info->has_setup_time) { - monitor_printf(mon, "setup: %" PRIu64 " ms\n", - info->setup_time); + } + + if (info->has_socket_address) { + SocketAddressList *addr; + + monitor_printf(mon, "Sockets: [\n"); + + for (addr = info->socket_address; addr; addr = addr->next) { + char *s = socket_uri(addr->value); + monitor_printf(mon, "\t%s\n", s); + g_free(s); } + monitor_printf(mon, "]\n"); } if (info->ram) { - monitor_printf(mon, "transferred ram: %" PRIu64 " kbytes\n", - info->ram->transferred >> 10); - monitor_printf(mon, "throughput: %0.2f mbps\n", + monitor_printf(mon, "RAM info:\n"); + monitor_printf(mon, " Throughput (Mbps): %0.2f\n", info->ram->mbps); - monitor_printf(mon, "remaining ram: %" PRIu64 " kbytes\n", - info->ram->remaining >> 10); - monitor_printf(mon, "total ram: %" PRIu64 " kbytes\n", + monitor_printf(mon, " Sizes (KiB): pagesize=%" PRIu64 + ", total=%" PRIu64 ",\n", + info->ram->page_size >> 10, info->ram->total >> 10); - monitor_printf(mon, "duplicate: %" PRIu64 " pages\n", - info->ram->duplicate); - monitor_printf(mon, "normal: %" PRIu64 " pages\n", - info->ram->normal); - monitor_printf(mon, "normal bytes: %" PRIu64 " kbytes\n", - info->ram->normal_bytes >> 10); - monitor_printf(mon, "dirty sync count: %" PRIu64 "\n", - info->ram->dirty_sync_count); - monitor_printf(mon, "page size: %" PRIu64 " kbytes\n", - info->ram->page_size >> 10); - monitor_printf(mon, "multifd bytes: %" PRIu64 " kbytes\n", - info->ram->multifd_bytes >> 10); - monitor_printf(mon, "pages-per-second: %" PRIu64 "\n", + monitor_printf(mon, " transferred=%" PRIu64 + ", remain=%" PRIu64 ",\n", + info->ram->transferred >> 10, + info->ram->remaining >> 10); + monitor_printf(mon, " precopy=%" PRIu64 + ", multifd=%" PRIu64 + ", postcopy=%" PRIu64, + info->ram->precopy_bytes >> 10, + info->ram->multifd_bytes >> 10, + info->ram->postcopy_bytes >> 10); + + if (info->vfio) { + monitor_printf(mon, ", vfio=%" PRIu64, + info->vfio->transferred >> 10); + } + monitor_printf(mon, "\n"); + + monitor_printf(mon, " Pages: normal=%" PRIu64 ", zero=%" PRIu64 + ", rate_per_sec=%" PRIu64 "\n", + info->ram->normal, + info->ram->duplicate, info->ram->pages_per_second); + monitor_printf(mon, " Others: dirty_syncs=%" PRIu64, + info->ram->dirty_sync_count); if (info->ram->dirty_pages_rate) { - monitor_printf(mon, "dirty pages rate: %" PRIu64 " pages\n", + monitor_printf(mon, ", dirty_pages_rate=%" PRIu64, info->ram->dirty_pages_rate); } if (info->ram->postcopy_requests) { - monitor_printf(mon, "postcopy request count: %" PRIu64 "\n", + monitor_printf(mon, ", postcopy_req=%" PRIu64, info->ram->postcopy_requests); } - if (info->ram->precopy_bytes) { - monitor_printf(mon, "precopy ram: %" PRIu64 " kbytes\n", - info->ram->precopy_bytes >> 10); - } if (info->ram->downtime_bytes) { - monitor_printf(mon, "downtime ram: %" PRIu64 " kbytes\n", - info->ram->downtime_bytes >> 10); - } - if (info->ram->postcopy_bytes) { - monitor_printf(mon, "postcopy ram: %" PRIu64 " kbytes\n", - info->ram->postcopy_bytes >> 10); + monitor_printf(mon, ", downtime_ram=%" PRIu64, + info->ram->downtime_bytes); } if (info->ram->dirty_sync_missed_zero_copy) { - monitor_printf(mon, - "Zero-copy-send fallbacks happened: %" PRIu64 " times\n", + monitor_printf(mon, ", zerocopy_fallbacks=%" PRIu64, info->ram->dirty_sync_missed_zero_copy); } + monitor_printf(mon, "\n"); + } + + if (!show_all) { + goto out; } + migration_global_dump(mon); + if (info->xbzrle_cache) { - monitor_printf(mon, "cache size: %" PRIu64 " bytes\n", - info->xbzrle_cache->cache_size); - monitor_printf(mon, "xbzrle transferred: %" PRIu64 " kbytes\n", - info->xbzrle_cache->bytes >> 10); - monitor_printf(mon, "xbzrle pages: %" PRIu64 " pages\n", - info->xbzrle_cache->pages); - monitor_printf(mon, "xbzrle cache miss: %" PRIu64 " pages\n", - info->xbzrle_cache->cache_miss); - monitor_printf(mon, "xbzrle cache miss rate: %0.2f\n", - info->xbzrle_cache->cache_miss_rate); - monitor_printf(mon, "xbzrle encoding rate: %0.2f\n", - info->xbzrle_cache->encoding_rate); - monitor_printf(mon, "xbzrle overflow: %" PRIu64 "\n", + monitor_printf(mon, "XBZRLE: size=%" PRIu64 + ", transferred=%" PRIu64 + ", pages=%" PRIu64 + ", miss=%" PRIu64 "\n" + " miss_rate=%0.2f" + ", encode_rate=%0.2f" + ", overflow=%" PRIu64 "\n", + info->xbzrle_cache->cache_size, + info->xbzrle_cache->bytes, + info->xbzrle_cache->pages, + info->xbzrle_cache->cache_miss, + info->xbzrle_cache->cache_miss_rate, + info->xbzrle_cache->encoding_rate, info->xbzrle_cache->overflow); } if (info->has_cpu_throttle_percentage) { - monitor_printf(mon, "cpu throttle percentage: %" PRIu64 "\n", + monitor_printf(mon, "CPU Throttle (%%): %" PRIu64 "\n", info->cpu_throttle_percentage); } if (info->has_dirty_limit_throttle_time_per_round) { - monitor_printf(mon, "dirty-limit throttle time: %" PRIu64 " us\n", + monitor_printf(mon, "Dirty-limit Throttle (us): %" PRIu64 "\n", info->dirty_limit_throttle_time_per_round); } if (info->has_dirty_limit_ring_full_time) { - monitor_printf(mon, "dirty-limit ring full time: %" PRIu64 " us\n", + monitor_printf(mon, "Dirty-limit Ring Full (us): %" PRIu64 "\n", info->dirty_limit_ring_full_time); } if (info->has_postcopy_blocktime) { - monitor_printf(mon, "postcopy blocktime: %u\n", + monitor_printf(mon, "Postcopy Blocktime (ms): %" PRIu32 "\n", info->postcopy_blocktime); } @@ -189,28 +211,12 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict) visit_type_uint32List(v, NULL, &info->postcopy_vcpu_blocktime, &error_abort); visit_complete(v, &str); - monitor_printf(mon, "postcopy vcpu blocktime: %s\n", str); + monitor_printf(mon, "Postcopy vCPU Blocktime: %s\n", str); g_free(str); visit_free(v); } - if (info->has_socket_address) { - SocketAddressList *addr; - - monitor_printf(mon, "socket address: [\n"); - - for (addr = info->socket_address; addr; addr = addr->next) { - char *s = socket_uri(addr->value); - monitor_printf(mon, "\t%s\n", s); - g_free(s); - } - monitor_printf(mon, "]\n"); - } - - if (info->vfio) { - monitor_printf(mon, "vfio device transferred: %" PRIu64 " kbytes\n", - info->vfio->transferred >> 10); - } +out: qapi_free_MigrationInfo(info); } diff --git a/migration/multifd-nocomp.c b/migration/multifd-nocomp.c index 88fe0f9..b48eae3 100644 --- a/migration/multifd-nocomp.c +++ b/migration/multifd-nocomp.c @@ -17,6 +17,7 @@ #include "migration-stats.h" #include "multifd.h" #include "options.h" +#include "migration.h" #include "qapi/error.h" #include "qemu/cutils.h" #include "qemu/error-report.h" @@ -398,7 +399,7 @@ int multifd_ram_flush_and_sync(QEMUFile *f) MultiFDSyncReq req; int ret; - if (!migrate_multifd()) { + if (!migrate_multifd() || migration_in_postcopy()) { return 0; } diff --git a/migration/multifd-zero-page.c b/migration/multifd-zero-page.c index dbc1184..4cde868 100644 --- a/migration/multifd-zero-page.c +++ b/migration/multifd-zero-page.c @@ -85,9 +85,27 @@ void multifd_recv_zero_page_process(MultiFDRecvParams *p) { for (int i = 0; i < p->zero_num; i++) { void *page = p->host + p->zero[i]; - if (ramblock_recv_bitmap_test_byte_offset(p->block, p->zero[i])) { + bool received = + ramblock_recv_bitmap_test_byte_offset(p->block, p->zero[i]); + + /* + * During multifd migration zero page is written to the memory + * only if it is migrated more than once. + * + * It becomes a problem when both multifd & postcopy options are + * enabled. If the zero page which was skipped during multifd phase, + * is accessed during the postcopy phase of the migration, a page + * fault occurs. But this page fault is not served because the + * 'receivedmap' says the zero page is already received. Thus the + * thread accessing that page may hang. + * + * When postcopy is enabled, always write the zero page as and when + * it is migrated. + */ + if (migrate_postcopy_ram() || received) { memset(page, 0, multifd_ram_page_size()); - } else { + } + if (!received) { ramblock_recv_bitmap_set_offset(p->block, p->zero[i]); } } diff --git a/migration/multifd.c b/migration/multifd.c index ec108af..b255778 100644 --- a/migration/multifd.c +++ b/migration/multifd.c @@ -690,6 +690,7 @@ static void *multifd_send_thread(void *opaque) if (qatomic_load_acquire(&p->pending_job)) { bool is_device_state = multifd_payload_device_state(p->data); size_t total_size; + int write_flags_masked = 0; p->flags = 0; p->iovs_num = 0; @@ -697,6 +698,9 @@ static void *multifd_send_thread(void *opaque) if (is_device_state) { multifd_device_state_send_prepare(p); + + /* Device state packets cannot be sent via zerocopy */ + write_flags_masked |= QIO_CHANNEL_WRITE_FLAG_ZERO_COPY; } else { ret = multifd_send_state->ops->send_prepare(p, &local_err); if (ret != 0) { @@ -718,7 +722,8 @@ static void *multifd_send_thread(void *opaque) &p->data->u.ram, &local_err); } else { ret = qio_channel_writev_full_all(p->c, p->iov, p->iovs_num, - NULL, 0, p->write_flags, + NULL, 0, + p->write_flags & ~write_flags_masked, &local_err); } @@ -1379,6 +1384,13 @@ static void *multifd_recv_thread(void *opaque) } if (has_data) { + /* + * multifd thread should not be active and receive data + * when migration is in the Postcopy phase. Two threads + * writing the same memory area could easily corrupt + * the guest state. + */ + assert(!migration_in_postcopy()); if (is_device_state) { assert(use_packets); ret = multifd_device_state_recv(p, &local_err); diff --git a/migration/options.c b/migration/options.c index b6ae953..162c72c 100644 --- a/migration/options.c +++ b/migration/options.c @@ -509,11 +509,6 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp) error_setg(errp, "Postcopy is not compatible with ignore-shared"); return false; } - - if (new_caps[MIGRATION_CAPABILITY_MULTIFD]) { - error_setg(errp, "Postcopy is not yet compatible with multifd"); - return false; - } } if (new_caps[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) { @@ -573,7 +568,7 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp) return false; } - if (migrate_incoming_started()) { + if (!migrate_postcopy_preempt() && migrate_incoming_started()) { error_setg(errp, "Postcopy preempt must be set before incoming starts"); return false; @@ -581,7 +576,7 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp) } if (new_caps[MIGRATION_CAPABILITY_MULTIFD]) { - if (migrate_incoming_started()) { + if (!migrate_multifd() && migrate_incoming_started()) { error_setg(errp, "Multifd must be set before incoming starts"); return false; } diff --git a/migration/ram.c b/migration/ram.c index e12913b..d26dbd3 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -1993,9 +1993,8 @@ static int ram_save_target_page(RAMState *rs, PageSearchStatus *pss) } } - if (migrate_multifd()) { - RAMBlock *block = pss->block; - return ram_save_multifd_page(block, offset); + if (migrate_multifd() && !migration_in_postcopy()) { + return ram_save_multifd_page(pss->block, offset); } return ram_save_page(rs, pss); diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh index a39f2fb..d784d15 100755 --- a/scripts/rdma-migration-helper.sh +++ b/scripts/rdma-migration-helper.sh @@ -8,23 +8,44 @@ get_ipv4_addr() head -1 | tr -d '\n' } +get_ipv6_addr() { + ipv6=$(ip -6 -o addr show dev "$1" | + sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' | + head -1 | tr -d '\n') + + [ $? -eq 0 ] || return + + if [[ "$ipv6" =~ ^fe80: ]]; then + echo -n "[$ipv6%$1]" + else + echo -n "[$ipv6]" + fi +} + # existing rdma interfaces rdma_interfaces() { - rdma link show | sed -nE 's/^link .* netdev ([^ ]+).*$/\1 /p' + rdma link show | sed -nE 's/^link .* netdev ([^ ]+).*$/\1 /p' | + grep -Ev '^(lo|tun|tap)' } # existing valid ipv4 interfaces ipv4_interfaces() { - ip -o addr show | awk '/inet / {print $2}' | grep -v -w lo + ip -o addr show | awk '/inet / {print $2}' | grep -Ev '^(lo|tun|tap)' +} + +ipv6_interfaces() +{ + ip -o addr show | awk '/inet6 / {print $2}' | grep -Ev '^(lo|tun|tap)' } rdma_rxe_detect() { + family=$1 for r in $(rdma_interfaces) do - ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return + "$family"_interfaces | grep -qw $r && get_"$family"_addr $r && return done return 1 @@ -32,16 +53,23 @@ rdma_rxe_detect() rdma_rxe_setup() { - for i in $(ipv4_interfaces) + family=$1 + for i in $("$family"_interfaces) do - rdma_interfaces | grep -qw $i && continue + if rdma_interfaces | grep -qw $i; then + echo "$family: Reuse the existing rdma/rxe ${i}_rxe" \ + "for $i with $(get_"$family"_addr $i)" + return + fi + rdma link add "${i}_rxe" type rxe netdev "$i" && { - echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)" + echo "$family: Setup new rdma/rxe ${i}_rxe" \ + "for $i with $(get_"$family"_addr $i)" return } done - echo "Failed to setup any new rdma/rxe link" >&2 + echo "$family: Failed to setup any new rdma/rxe link" >&2 return 1 } @@ -50,6 +78,12 @@ rdma_rxe_clean() modprobe -r rdma_rxe } +IP_FAMILY=${IP_FAMILY:-ipv4} +if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then + echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported." >&2 + exit 1 +fi + operation=${1:-detect} command -v rdma >/dev/null || { @@ -62,9 +96,14 @@ if [ "$operation" == "setup" ] || [ "$operation" == "clean" ]; then echo "Root privilege is required to setup/clean a rdma/rxe link" >&2 exit 1 } - rdma_rxe_"$operation" + if [ "$operation" == "setup" ]; then + rdma_rxe_setup ipv4 + rdma_rxe_setup ipv6 + else + rdma_rxe_clean + fi elif [ "$operation" == "detect" ]; then - rdma_rxe_detect + rdma_rxe_detect "$IP_FAMILY" else echo "Usage: $0 [setup | detect | clean]" fi diff --git a/scripts/vmstate-static-checker.py b/scripts/vmstate-static-checker.py index 25aca83..2335e25 100755 --- a/scripts/vmstate-static-checker.py +++ b/scripts/vmstate-static-checker.py @@ -91,6 +91,7 @@ def check_fields_match(name, s_field, d_field): 'mem_win_size', 'mig_mem_win_size', 'io_win_addr', 'mig_io_win_addr', 'io_win_size', 'mig_io_win_size'], + 'hpet': ['num_timers', 'num_timers_save'], } if not name in changed_names: diff --git a/tests/qtest/migration/compression-tests.c b/tests/qtest/migration/compression-tests.c index 41e79f0..b827665 100644 --- a/tests/qtest/migration/compression-tests.c +++ b/tests/qtest/migration/compression-tests.c @@ -42,6 +42,20 @@ static void test_multifd_tcp_zstd(void) }; test_precopy_common(&args); } + +static void test_multifd_postcopy_tcp_zstd(void) +{ + MigrateCommon args = { + .listen_uri = "defer", + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + .caps[MIGRATION_CAPABILITY_POSTCOPY_RAM] = true, + }, + .start_hook = migrate_hook_start_precopy_tcp_multifd_zstd, + }; + + test_precopy_common(&args); +} #endif /* CONFIG_ZSTD */ #ifdef CONFIG_QATZIP @@ -184,6 +198,10 @@ void migration_test_add_compression(MigrationTestEnv *env) #ifdef CONFIG_ZSTD migration_test_add("/migration/multifd/tcp/plain/zstd", test_multifd_tcp_zstd); + if (env->has_uffd) { + migration_test_add("/migration/multifd+postcopy/tcp/plain/zstd", + test_multifd_postcopy_tcp_zstd); + } #endif #ifdef CONFIG_QATZIP diff --git a/tests/qtest/migration/postcopy-tests.c b/tests/qtest/migration/postcopy-tests.c index 483e3ff..3773525 100644 --- a/tests/qtest/migration/postcopy-tests.c +++ b/tests/qtest/migration/postcopy-tests.c @@ -94,6 +94,29 @@ static void migration_test_add_postcopy_smoke(MigrationTestEnv *env) } } +static void test_multifd_postcopy(void) +{ + MigrateCommon args = { + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, + }; + + test_postcopy_common(&args); +} + +static void test_multifd_postcopy_preempt(void) +{ + MigrateCommon args = { + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + .caps[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT] = true, + }, + }; + + test_postcopy_common(&args); +} + void migration_test_add_postcopy(MigrationTestEnv *env) { migration_test_add_postcopy_smoke(env); @@ -114,6 +137,10 @@ void migration_test_add_postcopy(MigrationTestEnv *env) "/migration/postcopy/recovery/double-failures/reconnect", test_postcopy_recovery_fail_reconnect); + migration_test_add("/migration/multifd+postcopy/plain", + test_multifd_postcopy); + migration_test_add("/migration/multifd+postcopy/preempt/plain", + test_multifd_postcopy_preempt); if (env->is_x86) { migration_test_add("/migration/postcopy/suspend", test_postcopy_suspend); diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c index 87b0a7e..bb38292 100644 --- a/tests/qtest/migration/precopy-tests.c +++ b/tests/qtest/migration/precopy-tests.c @@ -101,13 +101,43 @@ static void test_precopy_unix_dirty_ring(void) #ifdef CONFIG_RDMA +#include <sys/resource.h> + +/* + * During migration over RDMA, it will try to pin portions of guest memory, + * typically exceeding 100MB in this test, while the remainder will be + * transmitted as compressed zero pages. + * + * REQUIRED_MEMLOCK_SZ indicates the minimal mlock size in the current context. + */ +#define REQUIRED_MEMLOCK_SZ (128 << 20) /* 128MB */ + +/* check 'ulimit -l' */ +static bool mlock_check(void) +{ + uid_t uid; + struct rlimit rlim; + + uid = getuid(); + if (uid == 0) { + return true; + } + + if (getrlimit(RLIMIT_MEMLOCK, &rlim) != 0) { + return false; + } + + return rlim.rlim_cur >= REQUIRED_MEMLOCK_SZ; +} + #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh" -static int new_rdma_link(char *buffer) +static int new_rdma_link(char *buffer, bool ipv6) { char cmd[256]; bool verbose = g_getenv("QTEST_LOG"); - snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER, + snprintf(cmd, sizeof(cmd), "IP_FAMILY=%s %s detect %s", + ipv6 ? "ipv6" : "ipv4", RDMA_MIGRATION_HELPER, verbose ? "" : "2>/dev/null"); FILE *pipe = popen(cmd, "r"); @@ -132,11 +162,16 @@ static int new_rdma_link(char *buffer) return -1; } -static void test_precopy_rdma_plain(void) +static void __test_precopy_rdma_plain(bool ipv6) { char buffer[128] = {}; - if (new_rdma_link(buffer)) { + if (!mlock_check()) { + g_test_skip("'ulimit -l' is too small, require >=128M"); + return; + } + + if (new_rdma_link(buffer, ipv6)) { g_test_skip("No rdma link available\n" "# To enable the test:\n" "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to " @@ -159,6 +194,16 @@ static void test_precopy_rdma_plain(void) test_precopy_common(&args); } + +static void test_precopy_rdma_plain(void) +{ + __test_precopy_rdma_plain(false); +} + +static void test_precopy_rdma_plain_ipv6(void) +{ + __test_precopy_rdma_plain(true); +} #endif static void test_precopy_tcp_plain(void) @@ -524,7 +569,7 @@ static void test_multifd_tcp_channels_none(void) * * And see that it works */ -static void test_multifd_tcp_cancel(void) +static void test_multifd_tcp_cancel(bool postcopy_ram) { MigrateStart args = { .hide_stderr = true, @@ -538,6 +583,11 @@ static void test_multifd_tcp_cancel(void) migrate_ensure_non_converge(from); migrate_prepare_for_dirty_mem(from); + if (postcopy_ram) { + migrate_set_capability(from, "postcopy-ram", true); + migrate_set_capability(to, "postcopy-ram", true); + } + migrate_set_parameter_int(from, "multifd-channels", 16); migrate_set_parameter_int(to, "multifd-channels", 16); @@ -579,6 +629,10 @@ static void test_multifd_tcp_cancel(void) return; } + if (postcopy_ram) { + migrate_set_capability(to2, "postcopy-ram", true); + } + migrate_set_parameter_int(to2, "multifd-channels", 16); migrate_set_capability(to2, "multifd", true); @@ -602,6 +656,16 @@ static void test_multifd_tcp_cancel(void) migrate_end(from, to2, true); } +static void test_multifd_precopy_tcp_cancel(void) +{ + test_multifd_tcp_cancel(false); +} + +static void test_multifd_postcopy_tcp_cancel(void) +{ + test_multifd_tcp_cancel(true); +} + static void test_cancel_src_after_failed(QTestState *from, QTestState *to, const char *uri, const char *phase) { @@ -1188,10 +1252,17 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env) migration_test_add("/migration/multifd/tcp/uri/plain/none", test_multifd_tcp_uri_none); migration_test_add("/migration/multifd/tcp/plain/cancel", - test_multifd_tcp_cancel); + test_multifd_precopy_tcp_cancel); + if (env->has_uffd) { + migration_test_add("/migration/multifd+postcopy/tcp/plain/cancel", + test_multifd_postcopy_tcp_cancel); + } + #ifdef CONFIG_RDMA migration_test_add("/migration/precopy/rdma/plain", test_precopy_rdma_plain); + migration_test_add("/migration/precopy/rdma/plain/ipv6", + test_precopy_rdma_plain_ipv6); #endif } diff --git a/tests/qtest/migration/tls-tests.c b/tests/qtest/migration/tls-tests.c index 72f44de..21e9fec 100644 --- a/tests/qtest/migration/tls-tests.c +++ b/tests/qtest/migration/tls-tests.c @@ -395,6 +395,19 @@ static void test_postcopy_recovery_tls_psk(void) test_postcopy_recovery_common(&args); } +static void test_multifd_postcopy_recovery_tls_psk(void) +{ + MigrateCommon args = { + .start_hook = migrate_hook_start_tls_psk_match, + .end_hook = migrate_hook_end_tls_psk, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + }, + }; + + test_postcopy_recovery_common(&args); +} + /* This contains preempt+recovery+tls test altogether */ static void test_postcopy_preempt_all(void) { @@ -409,6 +422,20 @@ static void test_postcopy_preempt_all(void) test_postcopy_recovery_common(&args); } +static void test_multifd_postcopy_preempt_recovery_tls_psk(void) +{ + MigrateCommon args = { + .start_hook = migrate_hook_start_tls_psk_match, + .end_hook = migrate_hook_end_tls_psk, + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + .caps[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT] = true, + }, + }; + + test_postcopy_recovery_common(&args); +} + static void test_precopy_unix_tls_psk(void) { g_autofree char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs); @@ -657,6 +684,21 @@ static void test_multifd_tcp_tls_psk_mismatch(void) test_precopy_common(&args); } +static void test_multifd_postcopy_tcp_tls_psk_match(void) +{ + MigrateCommon args = { + .start = { + .caps[MIGRATION_CAPABILITY_MULTIFD] = true, + .caps[MIGRATION_CAPABILITY_POSTCOPY_RAM] = true, + }, + .listen_uri = "defer", + .start_hook = migrate_hook_start_multifd_tcp_tls_psk_match, + .end_hook = migrate_hook_end_tls_psk, + }; + + test_precopy_common(&args); +} + #ifdef CONFIG_TASN1 static void test_multifd_tcp_tls_x509_default_host(void) { @@ -774,6 +816,11 @@ void migration_test_add_tls(MigrationTestEnv *env) test_postcopy_preempt_tls_psk); migration_test_add("/migration/postcopy/preempt/recovery/tls/psk", test_postcopy_preempt_all); + migration_test_add("/migration/multifd+postcopy/recovery/tls/psk", + test_multifd_postcopy_recovery_tls_psk); + migration_test_add( + "/migration/multifd+postcopy/preempt/recovery/tls/psk", + test_multifd_postcopy_preempt_recovery_tls_psk); } #ifdef CONFIG_TASN1 migration_test_add("/migration/precopy/unix/tls/x509/default-host", @@ -805,6 +852,10 @@ void migration_test_add_tls(MigrationTestEnv *env) test_multifd_tcp_tls_psk_match); migration_test_add("/migration/multifd/tcp/tls/psk/mismatch", test_multifd_tcp_tls_psk_mismatch); + if (env->has_uffd) { + migration_test_add("/migration/multifd+postcopy/tcp/tls/psk/match", + test_multifd_postcopy_tcp_tls_psk_match); + } #ifdef CONFIG_TASN1 migration_test_add("/migration/multifd/tcp/tls/x509/default-host", test_multifd_tcp_tls_x509_default_host); |