Add test for VFIO_USER_DMA_READ and VFIO_USER_DMA_WRITE

Signed-off-by: Swapnil Ingle <swapnil.ingle@nutanix.com>

3 files changed, 236 insertions, 20 deletions

diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt
index 08cfd6b..64eb6df 100644
--- a/samples/CMakeLists.txt
+++ b/samples/CMakeLists.txt
@@ -33,7 +33,7 @@ add_executable(test_mmap test_mmap.c)
 add_executable(test_dma_map test_dma_map.c)
 
 add_executable(server server.c)
-target_link_libraries(server muser)
+target_link_libraries(server muser ssl crypto)
 add_executable(client client.c ../lib/libmuser.c ../lib/libmuser_pci.c ../lib/dma.c ../lib/cap.c)
 
 add_executable(null null.c)
diff --git a/samples/client.c b/samples/client.c
index f916e4c..aad15f6 100644
--- a/samples/client.c
+++ b/samples/client.c
@@ -288,6 +288,126 @@ access_bar0(int sock)
     return 0;
 }
 
+static int handle_dma_write(int sock, struct vfio_user_dma_region *dma_regions,
+                            int nr_dma_regions, int *dma_region_fds)
+{
+    struct vfio_user_dma_region_access dma_access;
+    struct vfio_user_header hdr;
+    int ret, size = sizeof(dma_access), i;
+    uint16_t msg_id;
+    void *data;
+
+    msg_id = 1;
+    ret = recv_vfio_user_msg(sock, &hdr, false, &msg_id, &dma_access, &size);
+    if (ret < 0) {
+        fprintf(stderr, "failed to recieve DMA read: %m\n");
+        return ret;
+    }
+
+    data = calloc(dma_access.count, 1);
+    if (data == NULL) {
+        return -ENOMEM;
+    }
+
+    ret = recv(sock, data, dma_access.count, 0);
+    if (ret < 0) {
+        fprintf(stderr, "failed to recieve DMA read data: %m\n");
+        goto out;
+    }
+
+    for (i = 0; i < nr_dma_regions; i++) {
+        if (dma_regions[i].addr == dma_access.addr) {
+            ret = pwrite(dma_region_fds[i], data, dma_access.count,
+                         dma_regions[i].offset);
+            if (ret < 0) {
+                fprintf(stderr, "failed to write data at %#lu: %m\n",
+                        dma_regions[i].offset);
+                goto out;
+            }
+            break;
+	    }
+    }
+
+    dma_access.count = 0;
+    ret = send_vfio_user_msg(sock, msg_id, true, VFIO_USER_DMA_WRITE,
+                             &dma_access, sizeof(dma_access), NULL, 0);
+    if (ret < 0) {
+        fprintf(stderr, "failed to send reply of DMA write: %m\n");
+    }
+
+out:
+    free(data);
+    return ret;
+}
+
+static int handle_dma_read(int sock, struct vfio_user_dma_region *dma_regions,
+                            int nr_dma_regions, int *dma_region_fds)
+{
+    struct vfio_user_dma_region_access dma_access, *response;
+    struct vfio_user_header hdr;
+    int ret, size = sizeof(dma_access), i, response_sz;
+    uint16_t msg_id;
+    void *data;
+
+    msg_id = 1;
+    ret = recv_vfio_user_msg(sock, &hdr, false, &msg_id, &dma_access, &size);
+    if (ret < 0) {
+        fprintf(stderr, "failed to recieve DMA read: %m\n");
+        return ret;
+    }
+
+    response_sz = sizeof(dma_access) + dma_access.count;
+    response = calloc(response_sz, 1);
+    if (response == NULL) {
+        return -ENOMEM;
+    }
+    response->count = dma_access.count;
+    data = (char *)response->data;
+
+    for (i = 0; i < nr_dma_regions; i++) {
+        if (dma_regions[i].addr == dma_access.addr) {
+            ret = pread(dma_region_fds[i], data, dma_access.count,
+                         dma_regions[i].offset);
+            if (ret < 0) {
+                fprintf(stderr, "failed to write data at %#lu: %m\n",
+                        dma_regions[i].offset);
+                goto out;
+            }
+            break;
+	    }
+    }
+
+    ret = send_vfio_user_msg(sock, msg_id, true, VFIO_USER_DMA_READ,
+                             response, response_sz, NULL, 0);
+    if (ret < 0) {
+        fprintf(stderr, "failed to send reply of DMA write: %m\n");
+    }
+
+out:
+    free(response);
+    return ret;
+}
+
+static int handle_dma_io(int sock, struct vfio_user_dma_region *dma_regions,
+                     int nr_dma_regions, int *dma_region_fds)
+{
+    int ret;
+
+    ret = handle_dma_write(sock, dma_regions, nr_dma_regions, dma_region_fds);
+    if (ret < 0) {
+        fprintf(stderr, "failed to handle DMA write data: %m\n");
+        return ret;
+    }
+
+    ret = handle_dma_read(sock, dma_regions, nr_dma_regions, dma_region_fds);
+    if (ret < 0) {
+        fprintf(stderr, "failed to handle DMA read data: %m\n");
+        return ret;
+    }
+
+    return 0;
+}
+
 int main(int argc, char *argv[])
 {
 	int ret, sock;
@@ -375,20 +495,6 @@ int main(int argc, char *argv[])
     }
 
     /*
-     * XXX VFIO_USER_DMA_UNMAP
-     *
-     * unmap the first group of the DMA regions
-     */
-    ret = send_recv_vfio_user_msg(sock, msg_id, VFIO_USER_DMA_UNMAP,
-                                  dma_regions,
-                                  sizeof *dma_regions * server_max_fds,
-                                  NULL, 0, NULL, NULL, 0);
-    if (ret < 0) {
-        fprintf(stderr, "failed to unmap DMA regions: %s\n", strerror(-ret));
-        return ret;
-    }
-
-    /*
      * XXX VFIO_USER_DEVICE_GET_IRQ_INFO and VFIO_IRQ_SET_ACTION_TRIGGER
      * Query interrupts, configure an eventfd to be associated with INTx, and
      * finally wait for the server to fire the interrupt.     
@@ -399,6 +505,12 @@ int main(int argc, char *argv[])
         exit(EXIT_FAILURE);
     }
 
+    ret = handle_dma_io(sock, dma_regions, nr_dma_regions, dma_region_fds);
+    if (ret < 0) {
+        fprintf(stderr, "DMA IO failed: %s\n", strerror(-ret));
+        exit(EXIT_FAILURE);
+    }
+
     /*
      * XXX VFIO_USER_REGION_READ and VFIO_USER_REGION_WRITE
      *
@@ -420,6 +532,20 @@ int main(int argc, char *argv[])
     /* BAR1 can be memory mapped and read directly */
     /* TODO implement the following: write a value in BAR1, a server timer will increase it every second (SIGALARM) */
 
+    /*
+     * XXX VFIO_USER_DMA_UNMAP
+     *
+     * unmap the first group of the DMA regions
+     */
+    ret = send_recv_vfio_user_msg(sock, msg_id, VFIO_USER_DMA_UNMAP,
+                                  dma_regions,
+                                  sizeof *dma_regions * server_max_fds,
+                                  NULL, 0, NULL, NULL, 0);
+    if (ret < 0) {
+        fprintf(stderr, "failed to unmap DMA regions: %s\n", strerror(-ret));
+        return ret;
+    }
+
     return 0;
 }
 
diff --git a/samples/server.c b/samples/server.c
index 4af0edc..11f7064 100644
--- a/samples/server.c
+++ b/samples/server.c
@@ -38,12 +38,21 @@
 #include <errno.h>
 #include <time.h>
 #include <assert.h>
+#include <openssl/md5.h>
 
 #include "../lib/muser.h"
 
+struct dma_regions {
+    uint64_t addr;
+    uint64_t len;
+};
+
+#define NR_DMA_REGIONS  96
+
 struct server_data {
     time_t bar0;
     uint8_t *bar1;
+    struct dma_regions regions[NR_DMA_REGIONS];
 };
 
 static void
@@ -91,10 +100,85 @@ static void _sa_handler(int signum)
     errno = _errno;
 }
 
-static int
-unmap_dma(void *pvt __attribute__((unused)),
-          uint64_t iova __attribute__((unused)))
+static void map_dma(void *pvt, uint64_t iova, uint64_t len)
+{
+    struct server_data *server_data = pvt;
+    int idx;
+
+    for (idx = 0; idx < NR_DMA_REGIONS; idx++) {
+        if (server_data->regions[idx].addr == 0 &&
+            server_data->regions[idx].len == 0)
+            break;
+    }
+    if (idx >= NR_DMA_REGIONS) {
+        fprintf(stderr, "Failed to add dma region, slots full\n");
+        return;
+    }
+
+    server_data->regions[idx].addr = iova;
+    server_data->regions[idx].len = len;
+}
+
+static int unmap_dma(void *pvt, uint64_t iova)
+{
+    struct server_data *server_data = pvt;
+    int idx;
+
+    for (idx = 0; idx < NR_DMA_REGIONS; idx++) {
+        if (server_data->regions[idx].addr == iova) {
+            server_data->regions[idx].addr = 0;
+            server_data->regions[idx].len = 0;
+            return 0;
+        }
+    }
+
+    return -EINVAL;
+}
+
+void get_md5sum(char *buf, int len, char *md5sum)
+{
+	MD5_CTX ctx;
+
+    MD5_Init(&ctx);
+    MD5_Update(&ctx, buf, len);
+    MD5_Final(md5sum, &ctx);
+
+    return;
+}
+
+static int do_dma_io(lm_ctx_t *lm_ctx, struct server_data *server_data)
 {
+    int count = 4096;
+    char buf[count], md5sum1[MD5_DIGEST_LENGTH], md5sum2[MD5_DIGEST_LENGTH];
+    int i, ret;
+
+    memset(buf, 'A', count);
+    get_md5sum(buf, count, md5sum1);
+    printf("%s: WRITE addr %#lx count %llu\n", __func__,
+           server_data->regions[0].addr, count);
+    ret = lm_dma_write(lm_ctx, server_data->regions[0].addr, count, buf);
+    if (ret < 0) {
+        fprintf(stderr, "lm_dma_write failed: %s\n", strerror(-ret));
+        return ret;
+    }
+
+    memset(buf, 0, count);
+    printf("%s: READ  addr %#lx count %llu\n", __func__,
+	   server_data->regions[0].addr, count);
+    ret = lm_dma_read(lm_ctx, server_data->regions[0].addr, count, buf);
+    if (ret < 0) {
+        fprintf(stderr, "lm_dma_read failed: %s\n", strerror(-ret));
+        return ret;
+    }
+    get_md5sum(buf, count, md5sum2);
+    for(i = 0; i < MD5_DIGEST_LENGTH; i++) {
+        if (md5sum2[i] != md5sum1[i]) {
+            fprintf(stderr, "DMA write and DMA read mismatch\n");
+            return -EIO;
+        }
+    }
+
+    return 0;
 }
 
 unsigned long map_area(void *pvt, unsigned long off, unsigned long len)
@@ -108,7 +192,7 @@ int main(int argc, char *argv[])
     bool trans_sock = false, verbose = false;
     char opt;
     struct sigaction act = {.sa_handler = _sa_handler};
-    struct server_data server_data;
+    struct server_data server_data = {0};
     int nr_sparse_areas = 2, size = 1024, i;
     struct lm_sparse_mmap_areas *sparse_areas;
 
@@ -165,6 +249,7 @@ int main(int argc, char *argv[])
             .irq_count[LM_DEV_INTX_IRQ_IDX] = 1,
         },
         .uuid = argv[optind],
+        .map_dma = map_dma,
         .unmap_dma = unmap_dma,
         .pvt = &server_data
     };
@@ -188,8 +273,13 @@ int main(int argc, char *argv[])
             if (irq_triggered) {
                 irq_triggered = false;
                 lm_irq_trigger(lm_ctx, 0);
+
+                ret = do_dma_io(lm_ctx, &server_data);
+                if (ret < 0) {
+                    fprintf(stderr, "DMA read/write failed: %m\n");
+                }
                 ret = 0;
-            }            
+            }
         }
     } while (ret == 0);
     if (ret != -ENOTCONN && ret != -EINTR) {