hw/intc/s390_flic: Fix interrupt controller migration on s390x with TCG

Migration of a s390x guest with TCG was long known to be very unstable,
so the tests in tests/qtest/migration-test.c are disabled if running
with TCG instead of KVM.

Nicholas Piggin did a great analysis of the problem:

"The flic pending state is not migrated, so if the machine is migrated
 while an interrupt is pending, it can be lost. This shows up in
 qtest migration test, an extint is pending (due to console writes?)
 and the CPU waits via s390_cpu_set_psw and expects the interrupt to
 wake it. However when the flic pending state is lost, s390_cpu_has_int
 returns false, so s390_cpu_exec_interrupt falls through to halting
 again."

Thus let's finally migrate the pending state, and to be on the safe
side, also the other state variables of the QEMUS390FLICState structure.

Message-ID: <20240619144421.261342-1-thuth@redhat.com>
Signed-off-by: Thomas Huth <thuth@redhat.com>

3 files changed, 78 insertions, 3 deletions

diff --git a/hw/intc/s390_flic.c b/hw/intc/s390_flic.c
index 6771645..a91a4a4 100644
--- a/hw/intc/s390_flic.c
+++ b/hw/intc/s390_flic.c
@@ -361,15 +361,77 @@ bool ais_needed(void *opaque)
     return s->ais_supported;
 }
 
+static bool ais_needed_v(void *opaque, int version_id)
+{
+    return ais_needed(opaque);
+}
+
+static bool qemu_s390_flic_full_state_needed(void *opaque)
+{
+    QEMUS390FLICState *s = opaque;
+
+    return s->migrate_all_state;
+}
+
+static bool qemu_s390_flic_state_needed(void *opaque)
+{
+    return ais_needed(opaque) || qemu_s390_flic_full_state_needed(opaque);
+}
+
+static const VMStateDescription vmstate_qemu_s390_flic_io = {
+     .name = "qemu-s390-flic-io",
+     .version_id = 1,
+     .minimum_version_id = 1,
+     .fields = (const VMStateField[]) {
+         VMSTATE_UINT16(id, QEMUS390FlicIO),
+         VMSTATE_UINT16(nr, QEMUS390FlicIO),
+         VMSTATE_UINT32(parm, QEMUS390FlicIO),
+         VMSTATE_UINT32(word, QEMUS390FlicIO),
+         VMSTATE_END_OF_LIST()
+     },
+};
+
+static const VMStateDescription vmstate_qemu_s390_flic_full = {
+    .name = "qemu-s390-flic-full",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = qemu_s390_flic_full_state_needed,
+    .fields = (const VMStateField[]) {
+        VMSTATE_UINT32(pending, QEMUS390FLICState),
+        VMSTATE_UINT32(service_param, QEMUS390FLICState),
+        VMSTATE_QLIST_V(io[0], QEMUS390FLICState, 1,
+                        vmstate_qemu_s390_flic_io, QEMUS390FlicIO, next),
+        VMSTATE_QLIST_V(io[1], QEMUS390FLICState, 1,
+                        vmstate_qemu_s390_flic_io, QEMUS390FlicIO, next),
+        VMSTATE_QLIST_V(io[2], QEMUS390FLICState, 1,
+                        vmstate_qemu_s390_flic_io, QEMUS390FlicIO, next),
+        VMSTATE_QLIST_V(io[3], QEMUS390FLICState, 1,
+                        vmstate_qemu_s390_flic_io, QEMUS390FlicIO, next),
+        VMSTATE_QLIST_V(io[4], QEMUS390FLICState, 1,
+                        vmstate_qemu_s390_flic_io, QEMUS390FlicIO, next),
+        VMSTATE_QLIST_V(io[5], QEMUS390FLICState, 1,
+                        vmstate_qemu_s390_flic_io, QEMUS390FlicIO, next),
+        VMSTATE_QLIST_V(io[6], QEMUS390FLICState, 1,
+                        vmstate_qemu_s390_flic_io, QEMUS390FlicIO, next),
+        VMSTATE_QLIST_V(io[7], QEMUS390FLICState, 1,
+                        vmstate_qemu_s390_flic_io, QEMUS390FlicIO, next),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription qemu_s390_flic_vmstate = {
     .name = "qemu-s390-flic",
     .version_id = 1,
     .minimum_version_id = 1,
-    .needed = ais_needed,
+    .needed = qemu_s390_flic_state_needed,
     .fields = (const VMStateField[]) {
-        VMSTATE_UINT8(simm, QEMUS390FLICState),
-        VMSTATE_UINT8(nimm, QEMUS390FLICState),
+        VMSTATE_UINT8_TEST(simm, QEMUS390FLICState, ais_needed_v),
+        VMSTATE_UINT8_TEST(nimm, QEMUS390FLICState, ais_needed_v),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription * const []) {
+        &vmstate_qemu_s390_flic_full,
+        NULL
     }
 };
 
@@ -383,11 +445,18 @@ static void qemu_s390_flic_instance_init(Object *obj)
     }
 }
 
+static Property qemu_s390_flic_properties[] = {
+    DEFINE_PROP_BOOL("migrate-all-state", QEMUS390FLICState,
+                     migrate_all_state, true),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void qemu_s390_flic_class_init(ObjectClass *oc, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(oc);
     S390FLICStateClass *fsc = S390_FLIC_COMMON_CLASS(oc);
 
+    device_class_set_props(dc, qemu_s390_flic_properties);
     dc->reset = qemu_s390_flic_reset;
     dc->vmsd = &qemu_s390_flic_vmstate;
     fsc->register_io_adapter = qemu_s390_register_io_adapter;
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index cd063f8..f87ca36 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -863,8 +863,13 @@ static void ccw_machine_9_0_instance_options(MachineState *machine)
 
 static void ccw_machine_9_0_class_options(MachineClass *mc)
 {
+    static GlobalProperty compat[] = {
+        { TYPE_QEMU_S390_FLIC, "migrate-all-state", "off", },
+    };
+
     ccw_machine_9_1_class_options(mc);
     compat_props_add(mc->compat_props, hw_compat_9_0, hw_compat_9_0_len);
+    compat_props_add(mc->compat_props, compat, G_N_ELEMENTS(compat));
 }
 DEFINE_CCW_MACHINE(9_0, "9.0", false);
 
diff --git a/include/hw/s390x/s390_flic.h b/include/hw/s390x/s390_flic.h
index 382d983..4d66c5e 100644
--- a/include/hw/s390x/s390_flic.h
+++ b/include/hw/s390x/s390_flic.h
@@ -116,6 +116,7 @@ struct QEMUS390FLICState {
     uint8_t simm;
     uint8_t nimm;
     QLIST_HEAD(, QEMUS390FlicIO) io[8];
+    bool migrate_all_state;
 };
 
 uint32_t qemu_s390_flic_dequeue_service(QEMUS390FLICState *flic);