migration: various dirty page tracking fixes (#457)

- document how to use a vfio-user device with libvirt
- document how to use SPDK's nvmf/vfio-user target with libvirt
- replace vfio_bitmap with vfio_user_bitmap and vfio_iommu_type1_dirty_bitmap_get with vfio_user_bitmap_range
- fix bug for calculating number of pages needed for dirty page bitmap
- align number of bytes for dirty page bitmap to QWORD
- add debug messages around dirty page tracking
- only support flags=0 when doing DMA unmap
- set device state to running after reset
- allow region read/write even if device is in stopped state
- allow transitioning from stopped/stop-and-copy state to running state
- fix unit tests

Signed-off-by: Thanos Makatos <thanos.makatos@nutanix.com>
Reviewed-by: John Levon <john.levon@nutanix.com>

1 files changed, 16 insertions, 10 deletions

diff --git a/samples/client.c b/samples/client.c
index 6c2edd4..420269c 100644
--- a/samples/client.c
+++ b/samples/client.c
@@ -642,7 +642,7 @@ get_dirty_bitmaps(int sock, struct vfio_user_dma_region *dma_regions,
                   UNUSED int nr_dma_regions)
 {
     struct vfio_iommu_type1_dirty_bitmap dirty_bitmap = { 0 };
-    struct vfio_iommu_type1_dirty_bitmap_get bitmaps[2] = { { 0 }, };
+    struct vfio_user_bitmap_range bitmaps[2] = { { 0 }, };
     int ret;
     size_t i;
     struct iovec iovecs[4] = {
@@ -652,7 +652,7 @@ get_dirty_bitmaps(int sock, struct vfio_user_dma_region *dma_regions,
         }
     };
     struct vfio_user_header hdr = {0};
-    char data[ARRAY_SIZE(bitmaps)];
+    uint64_t data[ARRAY_SIZE(bitmaps)];
 
     assert(dma_regions != NULL);
     //FIXME: Is below assert correct?
@@ -661,28 +661,28 @@ get_dirty_bitmaps(int sock, struct vfio_user_dma_region *dma_regions,
     for (i = 0; i < ARRAY_SIZE(bitmaps); i++) {
         bitmaps[i].iova = dma_regions[i].addr;
         bitmaps[i].size = dma_regions[i].size;
-        bitmaps[i].bitmap.size = 1; /* FIXME calculate based on page and IOVA size, don't hardcode */
+        bitmaps[i].bitmap.size = sizeof(uint64_t); /* FIXME calculate based on page and IOVA size, don't hardcode */
         bitmaps[i].bitmap.pgsize = sysconf(_SC_PAGESIZE);
         iovecs[(i + 2)].iov_base = &bitmaps[i]; /* FIXME the +2 is because iovecs[0] is the vfio_user_header and iovecs[1] is vfio_iommu_type1_dirty_bitmap */
-        iovecs[(i + 2)].iov_len = sizeof(struct vfio_iommu_type1_dirty_bitmap_get);
+        iovecs[(i + 2)].iov_len = sizeof(struct vfio_user_bitmap_range);
     }
 
     /*
      * FIXME there should be at least two IOVAs. Send single message for two
      * IOVAs and ensure only one bit is set in first IOVA.
      */
-    dirty_bitmap.argsz = sizeof(dirty_bitmap) + ARRAY_SIZE(bitmaps) * sizeof(struct vfio_iommu_type1_dirty_bitmap_get);
+    dirty_bitmap.argsz = sizeof(dirty_bitmap) + ARRAY_SIZE(bitmaps) * sizeof(struct vfio_user_bitmap_range);
     dirty_bitmap.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP;
     ret = tran_sock_msg_iovec(sock, 0, VFIO_USER_DIRTY_PAGES,
                               iovecs, ARRAY_SIZE(iovecs),
                               NULL, 0,
-                              &hdr, data, ARRAY_SIZE(data), NULL, 0);
+                              &hdr, data, ARRAY_SIZE(data) * sizeof(uint64_t), NULL, 0);
     if (ret != 0) {
         err(EXIT_FAILURE, "failed to start dirty page logging");
     }
 
     for (i = 0; i < ARRAY_SIZE(bitmaps); i++) {
-        printf("client: %s: %#llx-%#llx\t%hhu\n", __func__, bitmaps[i].iova,
+        printf("client: %s: %#lx-%#lx\t%#lx\n", __func__, bitmaps[i].iova,
                bitmaps[i].iova + bitmaps[i].size - 1, data[i]);
     }
 }
@@ -1212,9 +1212,15 @@ int main(int argc, char *argv[])
      *
      * unmap the first group of the DMA regions
      */
-    ret = tran_sock_msg(sock, 7, VFIO_USER_DMA_UNMAP,
-                        dma_regions, sizeof(*dma_regions) * server_max_fds,
-                        NULL, NULL, 0);
+    {
+        struct vfio_user_dma_region r[server_max_fds];
+        memcpy(r, dma_regions, sizeof(r));
+        for (i = 0; i < (int)ARRAY_SIZE(r); i++) {
+            r[i].flags = 0;
+        }
+        ret = tran_sock_msg(sock, 7, VFIO_USER_DMA_UNMAP, r, sizeof(r),
+                            NULL, NULL, 0);
+    }
     if (ret < 0) {
         err(EXIT_FAILURE, "failed to unmap DMA regions");
     }