refactor sock send/recv functions (#114)

Use shorter, more readable, function names, add re-jig the wrappers such that
the most common operations are shortest.

6 files changed, 217 insertions, 141 deletions

diff --git a/lib/irq.c b/lib/irq.c
index 4c67915..4fa7513 100644
--- a/lib/irq.c
+++ b/lib/irq.c
@@ -413,10 +413,10 @@ lm_irq_message(lm_ctx_t *lm_ctx, uint32_t subindex)
     }
 
     irq_info.subindex = subindex;
-    ret = send_recv_vfio_user_msg(lm_ctx->conn_fd, msg_id,
-                                  VFIO_USER_VM_INTERRUPT,
-                                  &irq_info, sizeof irq_info,
-                                  NULL, 0, NULL, NULL, 0);
+    ret = vfio_user_msg(lm_ctx->conn_fd, msg_id,
+                        VFIO_USER_VM_INTERRUPT,
+                        &irq_info, sizeof irq_info,
+                        NULL, NULL, 0);
     if (ret < 0) {
         /* FIXME should return -errno */
 	    errno = -ret;
diff --git a/lib/migration.h b/lib/migration.h
index ccbbe1e..535f97b 100644
--- a/lib/migration.h
+++ b/lib/migration.h
@@ -28,10 +28,18 @@
  *
  */
 
+
+// FIXME: license header (and SPDX ?) everywhere
+
 /* FIXME: regularize across private headers */
 #ifndef MUSER_MIGRATION_H
 #define MUSER_MIGRATION_H
 
+/*
+ * These are not public routines, but for convenience, they are used by the
+ * sample/test code as well as privately within libmuser.
+ */
+
 #include <stddef.h>
 
 #include "muser.h"
diff --git a/lib/muser_ctx.c b/lib/muser_ctx.c
index 1b606d5..617a218 100644
--- a/lib/muser_ctx.c
+++ b/lib/muser_ctx.c
@@ -978,8 +978,10 @@ reply:
     } else {
         ret = 0;
     }
-    ret = _send_vfio_user_msg(lm_ctx->conn_fd, hdr.msg_id, true,
-                             0, iovecs, nr_iovecs, NULL, 0, -ret);
+
+    // FIXME: SPEC: should the reply include the command? I'd say yes?
+    ret = vfio_user_send_iovec(lm_ctx->conn_fd, hdr.msg_id, true,
+                               0, iovecs, nr_iovecs, NULL, 0, -ret);
     if (unlikely(ret < 0)) {
         lm_log(lm_ctx, LM_ERR, "failed to complete command: %s",
                 strerror(-ret));
@@ -1478,9 +1480,9 @@ lm_dma_read(lm_ctx_t *lm_ctx, dma_sg_t *sg, void *data)
 
     dma_send.addr = sg->dma_addr;
     dma_send.count = sg->length;
-    ret = send_recv_vfio_user_msg(lm_ctx->conn_fd, msg_id, VFIO_USER_DMA_READ,
-                                  &dma_send, sizeof dma_send, NULL, 0, NULL,
-                                  dma_recv, recv_size);
+    ret = vfio_user_msg(lm_ctx->conn_fd, msg_id, VFIO_USER_DMA_READ,
+                        &dma_send, sizeof dma_send, NULL,
+                        dma_recv, recv_size);
     memcpy(data, dma_recv->data, sg->length); /* FIXME no need for memcpy */
     free(dma_recv);
 
@@ -1504,9 +1506,9 @@ lm_dma_write(lm_ctx_t *lm_ctx, dma_sg_t *sg, void *data)
     dma_send->addr = sg->dma_addr;
     dma_send->count = sg->length;
     memcpy(dma_send->data, data, sg->length); /* FIXME no need to copy! */
-    ret = send_recv_vfio_user_msg(lm_ctx->conn_fd, msg_id, VFIO_USER_DMA_WRITE,
-                                  dma_send, send_size,
-                                  NULL, 0, NULL, &dma_recv, sizeof(dma_recv));
+    ret = vfio_user_msg(lm_ctx->conn_fd, msg_id, VFIO_USER_DMA_WRITE,
+                        dma_send, send_size, NULL,
+                        &dma_recv, sizeof(dma_recv));
     free(dma_send);
 
     return ret;
diff --git a/lib/tran_sock.c b/lib/tran_sock.c
index 2e8e2bc..7834a77 100644
--- a/lib/tran_sock.c
+++ b/lib/tran_sock.c
@@ -131,7 +131,7 @@ out:
 }
 
 int
-_send_vfio_user_msg(int sock, uint16_t msg_id, bool is_reply,
+vfio_user_send_iovec(int sock, uint16_t msg_id, bool is_reply,
                     enum vfio_user_command cmd,
                     struct iovec *iovecs, size_t nr_iovecs,
                     int *fds, int count, int err)
@@ -194,10 +194,9 @@ _send_vfio_user_msg(int sock, uint16_t msg_id, bool is_reply,
 }
 
 int
-send_vfio_user_msg(int sock, uint16_t msg_id, bool is_reply,
-                   enum vfio_user_command cmd,
-                   void *data, size_t data_len,
-                   int *fds, size_t count)
+vfio_user_send(int sock, uint16_t msg_id, bool is_reply,
+               enum vfio_user_command cmd,
+               void *data, size_t data_len)
 {
     /* [0] is for the header. */
     struct iovec iovecs[2] = {
@@ -206,8 +205,17 @@ send_vfio_user_msg(int sock, uint16_t msg_id, bool is_reply,
             .iov_len = data_len
         }
     };
-    return _send_vfio_user_msg(sock, msg_id, is_reply, cmd, iovecs,
-                               ARRAY_SIZE(iovecs), fds, count, 0);
+    return vfio_user_send_iovec(sock, msg_id, is_reply, cmd, iovecs,
+                                ARRAY_SIZE(iovecs), NULL, 0, 0);
+}
+
+int
+send_vfio_user_error(int sock, uint16_t msg_id,
+                     enum vfio_user_command cmd,
+                     int error)
+{
+    return vfio_user_send_iovec(sock, msg_id, true, cmd,
+                                NULL, 0, NULL, 0, error);
 }
 
 /*
@@ -220,7 +228,7 @@ send_vfio_user_msg(int sock, uint16_t msg_id, bool is_reply,
  * better.
  */
 int
-recv_vfio_user_msg(int sock, struct vfio_user_header *hdr, bool is_reply,
+vfio_user_recv(int sock, struct vfio_user_header *hdr, bool is_reply,
                    uint16_t *msg_id, void *data, size_t *len)
 {
     int ret;
@@ -272,19 +280,19 @@ recv_vfio_user_msg(int sock, struct vfio_user_header *hdr, bool is_reply,
 }
 
 /*
- * Like recv_vfio_user_msg(), but will automatically allocate reply data.
+ * Like vfio_user_recv(), but will automatically allocate reply data.
  *
  * FIXME: this does an unconstrained alloc of client-supplied data.
  */
 int
-recv_vfio_user_msg_alloc(int sock, struct vfio_user_header *hdr, bool is_reply,
+vfio_user_recv_alloc(int sock, struct vfio_user_header *hdr, bool is_reply,
                          uint16_t *msg_id, void **datap, size_t *lenp)
 {
     void *data;
     size_t len;
     int ret;
 
-    ret = recv_vfio_user_msg(sock, hdr, is_reply, msg_id, NULL, NULL);
+    ret = vfio_user_recv(sock, hdr, is_reply, msg_id, NULL, NULL);
 
     if (ret != 0) {
         return ret;
@@ -322,30 +330,33 @@ recv_vfio_user_msg_alloc(int sock, struct vfio_user_header *hdr, bool is_reply,
     return 0;
 }
 
+/*
+ * FIXME: all these send/recv handlers need to be made robust against async
+ * messages.
+ */
 int
-_send_recv_vfio_user_msg(int sock, uint16_t msg_id, enum vfio_user_command cmd,
-                         struct iovec *iovecs, size_t nr_iovecs,
-                         int *send_fds, size_t fd_count,
-                         struct vfio_user_header *hdr,
-                         void *recv_data, size_t recv_len)
+vfio_user_msg_iovec(int sock, uint16_t msg_id, enum vfio_user_command cmd,
+                    struct iovec *iovecs, size_t nr_iovecs,
+                    int *send_fds, size_t fd_count,
+                    struct vfio_user_header *hdr,
+                    void *recv_data, size_t recv_len)
 {
-    int ret = _send_vfio_user_msg(sock, msg_id, false, cmd, iovecs, nr_iovecs,
-                                  send_fds, fd_count, 0);
+    int ret = vfio_user_send_iovec(sock, msg_id, false, cmd, iovecs, nr_iovecs,
+                                   send_fds, fd_count, 0);
     if (ret < 0) {
         return ret;
     }
     if (hdr == NULL) {
         hdr = alloca(sizeof *hdr);
     }
-    return recv_vfio_user_msg(sock, hdr, true, &msg_id, recv_data, &recv_len);
+    return vfio_user_recv(sock, hdr, true, &msg_id, recv_data, &recv_len);
 }
 
 int
-send_recv_vfio_user_msg(int sock, uint16_t msg_id, enum vfio_user_command cmd,
-                        void *send_data, size_t send_len,
-                        int *send_fds, size_t fd_count,
-                        struct vfio_user_header *hdr,
-                        void *recv_data, size_t recv_len)
+vfio_user_msg(int sock, uint16_t msg_id, enum vfio_user_command cmd,
+              void *send_data, size_t send_len,
+              struct vfio_user_header *hdr,
+              void *recv_data, size_t recv_len)
 {
     /* [0] is for the header. */
     struct iovec iovecs[2] = {
@@ -354,9 +365,8 @@ send_recv_vfio_user_msg(int sock, uint16_t msg_id, enum vfio_user_command cmd,
             .iov_len = send_len
         }
     };
-    return _send_recv_vfio_user_msg(sock, msg_id, cmd, iovecs,
-                                    ARRAY_SIZE(iovecs), send_fds, fd_count,
-                                    hdr, recv_data, recv_len);
+    return vfio_user_msg_iovec(sock, msg_id, cmd, iovecs, ARRAY_SIZE(iovecs),
+                               NULL, 0, hdr, recv_data, recv_len);
 }
 
 /*
@@ -375,7 +385,8 @@ send_recv_vfio_user_msg(int sock, uint16_t msg_id, enum vfio_user_command cmd,
  * available in newer library versions, so we don't use it.
  */
 int
-parse_version_json(const char *json_str, int *client_max_fdsp, size_t *pgsizep)
+vfio_user_parse_version_json(const char *json_str,
+                             int *client_max_fdsp, size_t *pgsizep)
 {
     struct json_object *jo_caps = NULL;
     struct json_object *jo_top = NULL;
@@ -448,12 +459,12 @@ recv_version(lm_ctx_t *lm_ctx, int sock, uint16_t *msg_idp,
 
     *versionp = NULL;
 
-    ret = recv_vfio_user_msg_alloc(sock, &hdr, false, msg_idp,
+    ret = vfio_user_recv_alloc(sock, &hdr, false, msg_idp,
                                    (void **)&cversion, &vlen);
 
     if (ret < 0) {
         lm_log(lm_ctx, LM_ERR, "failed to receive version: %s", strerror(-ret));
-        goto out;
+        return ret;
     }
 
     if (hdr.cmd != VFIO_USER_VERSION) {
@@ -490,7 +501,8 @@ recv_version(lm_ctx_t *lm_ctx, int sock, uint16_t *msg_idp,
             goto out;
         }
 
-        ret = parse_version_json(json_str, &lm_ctx->client_max_fds, &pgsize);
+        ret = vfio_user_parse_version_json(json_str, &lm_ctx->client_max_fds,
+                                           &pgsize);
 
         if (ret < 0) {
             /* No client-supplied strings in the log for release build. */
@@ -525,6 +537,8 @@ recv_version(lm_ctx_t *lm_ctx, int sock, uint16_t *msg_idp,
 
 out:
     if (ret != 0) {
+        // FIXME: spec, is it OK to just have the header?
+        (void) send_vfio_user_error(sock, *msg_idp, hdr.cmd, ret);
         free(cversion);
         cversion = NULL;
     }
@@ -573,8 +587,8 @@ send_version(lm_ctx_t *lm_ctx, int sock, uint16_t msg_id,
     /* Include the NUL. */
     iovecs[2].iov_len = slen + 1;
 
-    return _send_vfio_user_msg(sock, msg_id, true, VFIO_USER_VERSION,
-                               iovecs, ARRAY_SIZE(iovecs), NULL, 0, 0);
+    return vfio_user_send_iovec(sock, msg_id, true, VFIO_USER_VERSION,
+                                iovecs, ARRAY_SIZE(iovecs), NULL, 0, 0);
 }
 
 static int
diff --git a/lib/tran_sock.h b/lib/tran_sock.h
index cbd6603..9add12c 100644
--- a/lib/tran_sock.h
+++ b/lib/tran_sock.h
@@ -48,44 +48,80 @@
 
 extern struct transport_ops sock_transport_ops;
 
-int
-parse_version_json(const char *json_str, int *client_max_fdsp, size_t *pgsizep);
+/*
+ * Parse JSON supplied from the other side into the known parameters. Note: they
+ * will not be set if not found in the JSON.
+ */
+int vfio_user_parse_version_json(const char *json_str, int *client_max_fdsp,
+                                 size_t *pgsizep);
 
-int
-_send_vfio_user_msg(int sock, uint16_t msg_id, bool is_reply,
-                   enum vfio_user_command cmd,
-                   struct iovec *iovecs, size_t nr_iovecs,
-                   int *fds, int count, int err);
+/*
+ * Send a message to the other end.  The iovecs array should leave the first
+ * entry empty, as it will be used for the header.
+ */
+int vfio_user_send_iovec(int sock, uint16_t msg_id, bool is_reply,
+                         enum vfio_user_command cmd,
+                         struct iovec *iovecs, size_t nr_iovecs,
+                         int *fds, int count,
+                         int err);
 
-int
-send_vfio_user_msg(int sock, uint16_t msg_id, bool is_reply,
+/*
+ * Send a message to the other end with the given data.
+ */
+int vfio_user_send(int sock, uint16_t msg_id, bool is_reply,
                    enum vfio_user_command cmd,
-                   void *data, size_t data_len,
-                   int *fds, size_t count);
-
+                   void *data, size_t data_len);
 
-int
-recv_vfio_user_msg(int sock, struct vfio_user_header *hdr, bool is_reply,
-                   uint16_t *msg_id, void *data, size_t *len);
+/*
+ * Send an empty reply back to the other end with the given errno.
+ */
+int vfio_user_send_error(int sock, uint16_t msg_id,
+                         enum vfio_user_command cmd,
+                         int error);
 
-int
-recv_vfio_user_msg_alloc(int sock, struct vfio_user_header *hdr, bool is_reply,
-                   uint16_t *msg_id, void **datap, size_t *lenp);
+/*
+ * Receive a message from the other end, and place the data into the given
+ * buffer. If data is supplied by the other end, it must be exactly *len in
+ * size.
+ */
+int vfio_user_recv(int sock, struct vfio_user_header *hdr,
+                   bool is_reply, uint16_t *msg_id,
+                   void *data, size_t *len);
 
-int
-_send_recv_vfio_user_msg(int sock, uint16_t msg_id, enum vfio_user_command cmd,
-                         struct iovec *iovecs, size_t nr_iovecs,
-                         int *send_fds, size_t fd_count,
-                         struct vfio_user_header *hdr,
-                         void *recv_data, size_t recv_len);
+/*
+ * Receive a message from the other end, but automatically allocate a buffer for
+ * it, which must be freed by the caller.  If there is no data, *datap is set to
+ * NULL.
+ */
+int vfio_user_recv_alloc(int sock, struct vfio_user_header *hdr,
+                         bool is_reply, uint16_t *msg_id,
+                         void **datap, size_t *lenp);
 
-int
-send_recv_vfio_user_msg(int sock, uint16_t msg_id, enum vfio_user_command cmd,
-                        void *send_data, size_t send_len,
+/*
+ * Send and receive a message to the other end, using iovecs for the send. The
+ * iovecs array should leave the first entry empty, as it will be used for the
+ * header.
+ *
+ * If specified, the given fds are sent to the other side. @hdr is filled with
+ * the reply header if non-NULL.
+ */
+int vfio_user_msg_iovec(int sock, uint16_t msg_id,
+                        enum vfio_user_command cmd,
+                        struct iovec *iovecs, size_t nr_iovecs,
                         int *send_fds, size_t fd_count,
                         struct vfio_user_header *hdr,
                         void *recv_data, size_t recv_len);
 
+/*
+ * Send and receive a message to the other end.  @hdr is filled with the reply
+ * header if non-NULL.
+ */
+int vfio_user_msg(int sock, uint16_t msg_id,
+                  enum vfio_user_command cmd,
+                  void *send_data, size_t send_len,
+                  struct vfio_user_header *hdr,
+                  void *recv_data, size_t recv_len);
+
 #endif /* TRAN_SOCK_H */
 
 /* ex: set tabstop=4 shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/samples/client.c b/samples/client.c
index 9daab81..3231e60 100644
--- a/samples/client.c
+++ b/samples/client.c
@@ -116,7 +116,7 @@ send_version(int sock)
     /* Include the NUL. */
     iovecs[2].iov_len = slen + 1;
 
-    ret = _send_vfio_user_msg(sock, msg_id, false, VFIO_USER_VERSION,
+    ret = vfio_user_send_iovec(sock, msg_id, false, VFIO_USER_VERSION,
                               iovecs, ARRAY_SIZE(iovecs), NULL, 0, 0);
 
     if (ret < 0) {
@@ -133,8 +133,8 @@ recv_version(int sock, int *server_max_fds, size_t *pgsize)
     size_t vlen;
     int ret;
 
-    ret = recv_vfio_user_msg_alloc(sock, &hdr, true, &msg_id,
-                                   (void **)&sversion, &vlen);
+    ret = vfio_user_recv_alloc(sock, &hdr, true, &msg_id,
+                               (void **)&sversion, &vlen);
 
     if (ret < 0) {
         errx(EXIT_FAILURE, "failed to receive version: %s", strerror(-ret));
@@ -177,7 +177,7 @@ recv_version(int sock, int *server_max_fds, size_t *pgsize)
             errx(EXIT_FAILURE, "ignoring invalid JSON from server");
         }
 
-        ret = parse_version_json(json_str, server_max_fds, pgsize);
+        ret = vfio_user_parse_version_json(json_str, server_max_fds, pgsize);
 
         if (ret < 0) {
             errx(EXIT_FAILURE, "failed to parse server JSON \"%s\"", json_str);
@@ -197,8 +197,8 @@ negotiate(int sock, int *server_max_fds, size_t *pgsize)
 static void
 send_device_reset(int sock)
 {
-    int ret = send_recv_vfio_user_msg(sock, 1, VFIO_USER_DEVICE_RESET,
-                                      NULL, 0, NULL, 0, NULL, NULL, 0);
+    int ret = vfio_user_msg(sock, 1, VFIO_USER_DEVICE_RESET,
+                            NULL, 0, NULL, NULL, 0);
     if (ret < 0) {
         errx(EXIT_FAILURE, "failed to reset device: %s\n", strerror(-ret));
     }
@@ -266,22 +266,21 @@ static int
 get_device_region_info(int sock, struct vfio_device_info *client_dev_info)
 {
     struct vfio_region_info region_info;
-    uint16_t msg_id = 0;
+    uint16_t msg_id = 1;
     size_t cap_sz;
     int ret, migr_reg_index = -1;
     unsigned int i;
 
-    msg_id = 1;
     for (i = 0; i < client_dev_info->num_regions; i++) {
         memset(&region_info, 0, sizeof(region_info));
         region_info.argsz = sizeof(region_info);
         region_info.index = i;
         msg_id++;
-        ret = send_recv_vfio_user_msg(sock, msg_id,
-                                      VFIO_USER_DEVICE_GET_REGION_INFO,
-                                      &region_info, sizeof region_info,
-                                      NULL, 0, NULL,
-                                      &region_info, sizeof(region_info));
+        ret = vfio_user_msg(sock, msg_id,
+                            VFIO_USER_DEVICE_GET_REGION_INFO,
+                            &region_info, sizeof region_info,
+                            NULL,
+                            &region_info, sizeof(region_info));
         if (ret < 0) {
             errx(EXIT_FAILURE, "failed to get device region info: %s",
                     strerror(-ret));
@@ -304,14 +303,17 @@ get_device_region_info(int sock, struct vfio_device_info *client_dev_info)
 static void
 get_device_info(int sock, struct vfio_device_info *dev_info)
 {
-    uint16_t msg_id;
+    uint16_t msg_id = 2;
     int ret;
 
     dev_info->argsz = sizeof(*dev_info);
-    msg_id = 1;
-    ret = send_recv_vfio_user_msg(sock, msg_id, VFIO_USER_DEVICE_GET_INFO,
-                                  dev_info, sizeof(*dev_info), NULL, 0, NULL,
-                                  dev_info, sizeof(*dev_info));
+
+    ret = vfio_user_msg(sock, msg_id,
+                        VFIO_USER_DEVICE_GET_INFO,
+                        dev_info, sizeof(*dev_info),
+                        NULL,
+                        dev_info, sizeof(*dev_info));
+
     if (ret < 0) {
         errx(EXIT_FAILURE, "failed to get device info: %s", strerror(-ret));
     }
@@ -323,21 +325,22 @@ get_device_info(int sock, struct vfio_device_info *dev_info)
 static int
 configure_irqs(int sock)
 {
-    int i, ret;
+    struct iovec iovecs[2] = { { 0, } };
     struct vfio_irq_set irq_set;
-    uint16_t msg_id = 1;
+    uint16_t msg_id = 3;
     int irq_fd;
+    int i, ret;
 
     for (i = 0; i < LM_DEV_NUM_IRQS; i++) { /* TODO move body of loop into function */
         struct vfio_irq_info vfio_irq_info = {
             .argsz = sizeof vfio_irq_info,
             .index = i
         };
-        ret = send_recv_vfio_user_msg(sock, msg_id,
-                                      VFIO_USER_DEVICE_GET_IRQ_INFO,
-                                      &vfio_irq_info, sizeof vfio_irq_info,
-                                      NULL, 0, NULL,
-                                      &vfio_irq_info, sizeof vfio_irq_info);
+        ret = vfio_user_msg(sock, msg_id,
+                            VFIO_USER_DEVICE_GET_IRQ_INFO,
+                            &vfio_irq_info, sizeof vfio_irq_info,
+                            NULL,
+                            &vfio_irq_info, sizeof vfio_irq_info);
         if (ret < 0) {
             errx(EXIT_FAILURE, "failed to get  %s info: %s", irq_to_str[i],
                  strerror(-ret));
@@ -359,9 +362,16 @@ configure_irqs(int sock)
     if (irq_fd == -1) {
         err(EXIT_FAILURE, "failed to create eventfd");
     }
-    ret = send_recv_vfio_user_msg(sock, msg_id, VFIO_USER_DEVICE_SET_IRQS,
-                                  &irq_set, sizeof irq_set, &irq_fd, 1, NULL,
-                                  NULL, 0);
+
+    /* [0] is for the header. */
+    iovecs[1].iov_base = &irq_set;
+    iovecs[1].iov_len = sizeof (irq_set);
+
+    ret = vfio_user_msg_iovec(sock, msg_id, VFIO_USER_DEVICE_SET_IRQS,
+                              iovecs, ARRAY_SIZE(iovecs),
+                              &irq_fd, 1,
+                              NULL, NULL, 0);
+
     if (ret < 0) {
         errx(EXIT_FAILURE, "failed to send configure IRQs message: %s",
              strerror(-ret));
@@ -406,10 +416,10 @@ access_region(int sock, int region, bool is_write, uint64_t offset,
         recv_data_len = sizeof(recv_data);
     }
 
-    ret = _send_recv_vfio_user_msg(sock, 0, op,
-                                   send_iovecs, nr_send_iovecs,
-                                   NULL, 0, NULL,
-                                   &recv_data, recv_data_len);
+    ret = vfio_user_msg_iovec(sock, 0, op,
+                              send_iovecs, nr_send_iovecs,
+                              NULL, 0, NULL,
+                              &recv_data, recv_data_len);
     if (ret != 0) {
         warnx("failed to %s region %d %#lx-%#lx: %s",
              is_write ? "write to" : "read from", region, offset,
@@ -424,7 +434,7 @@ access_region(int sock, int region, bool is_write, uint64_t offset,
     }
 
     /*
-     * TODO we could avoid the memcpy if _sed_recv_vfio_user_msg received the
+     * TODO we could avoid the memcpy if _sed_vfio_user_recv received the
      * response into an iovec, but it's some work to implement it.
      */
     if (!is_write) {
@@ -441,7 +451,7 @@ wait_for_irqs(int sock, int irq_fd)
     size_t size;
     struct vfio_user_irq_info vfio_user_irq_info;
     struct vfio_user_header hdr;
-    uint16_t msg_id = 1;
+    uint16_t msg_id = 4;
 
     if (read(irq_fd, &val, sizeof val) == -1) {
         err(EXIT_FAILURE, "failed to read from irqfd");
@@ -449,24 +459,28 @@ wait_for_irqs(int sock, int irq_fd)
     printf("INTx triggered!\n");
 
     size = sizeof(vfio_user_irq_info);
-    ret = recv_vfio_user_msg(sock, &hdr, false, &msg_id, &vfio_user_irq_info,
-                             &size);
+    ret = vfio_user_recv(sock, &hdr, false, &msg_id,
+                         &vfio_user_irq_info, &size);
     if (ret < 0) {
         errx(EXIT_FAILURE, "failed to receive IRQ message: %s",
              strerror(-ret));
     }
+
+    if (hdr.cmd != VFIO_USER_VM_INTERRUPT) {
+        errx(EXIT_FAILURE, "unexpected cmd %d\n", hdr.cmd);
+    }
+
     if (vfio_user_irq_info.subindex >= 1) {
         errx(EXIT_FAILURE, "bad IRQ %d, max=1\n", vfio_user_irq_info.subindex);
     }
 
-    ret = send_vfio_user_msg(sock, msg_id, true, VFIO_USER_VM_INTERRUPT,
-                             NULL, 0, NULL, 0);
+    // Is a NULL iovec like this OK?
+    ret = vfio_user_send(sock, msg_id, true, hdr.cmd, NULL, 0);
     if (ret < 0) {
         errx(EXIT_FAILURE,
              "failed to send reply for VFIO_USER_VM_INTERRUPT: %s",
              strerror(-ret));
     }
-    printf("INTx messaged triggered!\n");
 }
 
 static void
@@ -501,11 +515,10 @@ handle_dma_write(int sock, struct vfio_user_dma_region *dma_regions,
     struct vfio_user_header hdr;
     int ret, i;
     size_t size = sizeof(dma_access);
-    uint16_t msg_id;
+    uint16_t msg_id = 5;
     void *data;
 
-    msg_id = 1;
-    ret = recv_vfio_user_msg(sock, &hdr, false, &msg_id, &dma_access, &size);
+    ret = vfio_user_recv(sock, &hdr, false, &msg_id, &dma_access, &size);
     if (ret < 0) {
         errx(EXIT_FAILURE, "failed to receive DMA read: %s", strerror(-ret));
     }
@@ -535,8 +548,8 @@ handle_dma_write(int sock, struct vfio_user_dma_region *dma_regions,
     }
 
     dma_access.count = 0;
-    ret = send_vfio_user_msg(sock, msg_id, true, VFIO_USER_DMA_WRITE,
-                             &dma_access, sizeof dma_access, NULL, 0);
+    ret = vfio_user_send(sock, msg_id, true, VFIO_USER_DMA_WRITE,
+                         &dma_access, sizeof dma_access);
     if (ret < 0) {
         errx(EXIT_FAILURE, "failed to send reply of DMA write: %s",
              strerror(-ret));
@@ -552,11 +565,10 @@ handle_dma_read(int sock, struct vfio_user_dma_region *dma_regions,
     struct vfio_user_header hdr;
     int ret, i, response_sz;
     size_t size = sizeof(dma_access);
-    uint16_t msg_id;
+    uint16_t msg_id = 6;
     void *data;
 
-    msg_id = 1;
-    ret = recv_vfio_user_msg(sock, &hdr, false, &msg_id, &dma_access, &size);
+    ret = vfio_user_recv(sock, &hdr, false, &msg_id, &dma_access, &size);
     if (ret < 0) {
         errx(EXIT_FAILURE, "failed to recieve DMA read");
     }
@@ -580,8 +592,8 @@ handle_dma_read(int sock, struct vfio_user_dma_region *dma_regions,
 	    }
     }
 
-    ret = send_vfio_user_msg(sock, msg_id, true, VFIO_USER_DMA_READ,
-                             response, response_sz, NULL, 0);
+    ret = vfio_user_send(sock, msg_id, true, VFIO_USER_DMA_READ,
+                         response, response_sz);
     if (ret < 0) {
         errx(EXIT_FAILURE, "failed to send reply of DMA write: %s",
              strerror(-ret));
@@ -633,10 +645,10 @@ get_dirty_bitmaps(int sock, struct vfio_user_dma_region *dma_regions,
      */
     dirty_bitmap.argsz = sizeof(dirty_bitmap) + ARRAY_SIZE(bitmaps) * sizeof(struct vfio_iommu_type1_dirty_bitmap_get);
     dirty_bitmap.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP;
-    ret = _send_recv_vfio_user_msg(sock, 0, VFIO_USER_DIRTY_PAGES,
-                                  iovecs, ARRAY_SIZE(iovecs),
-                                  NULL, 0,
-                                  &hdr, data, ARRAY_SIZE(data));
+    ret = vfio_user_msg_iovec(sock, 0, VFIO_USER_DIRTY_PAGES,
+                              iovecs, ARRAY_SIZE(iovecs),
+                              NULL, 0,
+                              &hdr, data, ARRAY_SIZE(data));
     if (ret != 0) {
         errx(EXIT_FAILURE, "failed to start dirty page logging: %s",
              strerror(-ret));
@@ -861,16 +873,21 @@ migrate_to(char *old_sock_path, int *server_max_fds,
 
 static void
 map_dma_regions(int sock, int max_fds, struct vfio_user_dma_region *dma_regions,
-                int *dma_region_fds, int nr_dma_regions) 
+                int *dma_region_fds, int nr_dma_regions)
 {
     int i, ret;
 
     for (i = 0; i < nr_dma_regions / max_fds; i++) {
-        ret = send_recv_vfio_user_msg(sock, i, VFIO_USER_DMA_MAP,
-                                      dma_regions + (i * max_fds),
-                                      sizeof(*dma_regions) * max_fds,
-                                      dma_region_fds + (i * max_fds),
-                                      max_fds, NULL, NULL, 0);
+        struct iovec iovecs[2] = { { 0, } };
+
+        /* [0] is for the header. */
+        iovecs[1].iov_base = dma_regions + (i * max_fds);
+        iovecs[1].iov_len = sizeof (*dma_regions) * max_fds;
+
+        ret = vfio_user_msg_iovec(sock, i, VFIO_USER_DMA_MAP,
+                                  iovecs, ARRAY_SIZE(iovecs),
+                                  dma_region_fds + (i * max_fds), max_fds,
+                                  NULL, NULL, 0);
         if (ret < 0) {
             errx(EXIT_FAILURE, "failed to map DMA regions: %s", strerror(-ret));
         }
@@ -883,7 +900,6 @@ int main(int argc, char *argv[])
     struct vfio_user_dma_region *dma_regions;
     struct vfio_device_info client_dev_info = {0};
     int *dma_region_fds;
-    uint16_t msg_id = 1;
     int i;
     FILE *fp;
     int server_max_fds;
@@ -994,9 +1010,9 @@ int main(int argc, char *argv[])
 
     dirty_bitmap.argsz = sizeof dirty_bitmap;
     dirty_bitmap.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START;
-    ret = send_recv_vfio_user_msg(sock, 0, VFIO_USER_DIRTY_PAGES,
-                                  &dirty_bitmap, sizeof dirty_bitmap,
-                                  NULL, 0, NULL, NULL, 0);
+    ret = vfio_user_msg(sock, 0, VFIO_USER_DIRTY_PAGES,
+                        &dirty_bitmap, sizeof dirty_bitmap,
+                        NULL, NULL, 0);
     if (ret != 0) {
         errx(EXIT_FAILURE, "failed to start dirty page logging: %s",
              strerror(-ret));
@@ -1019,9 +1035,9 @@ int main(int argc, char *argv[])
 
     dirty_bitmap.argsz = sizeof dirty_bitmap;
     dirty_bitmap.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP;
-    ret = send_recv_vfio_user_msg(sock, 0, VFIO_USER_DIRTY_PAGES,
-                                  &dirty_bitmap, sizeof dirty_bitmap,
-                                  NULL, 0, NULL, NULL, 0);
+    ret = vfio_user_msg(sock, 0, VFIO_USER_DIRTY_PAGES,
+                        &dirty_bitmap, sizeof dirty_bitmap,
+                        NULL, NULL, 0);
     if (ret != 0) {
         errx(EXIT_FAILURE, "failed to stop dirty page logging: %s",
              strerror(-ret));
@@ -1034,9 +1050,9 @@ int main(int argc, char *argv[])
      *
      * unmap the first group of the DMA regions
      */
-    ret = send_recv_vfio_user_msg(sock, msg_id, VFIO_USER_DMA_UNMAP,
-                                  dma_regions, sizeof *dma_regions * server_max_fds,
-                                  NULL, 0, NULL, NULL, 0);
+    ret = vfio_user_msg(sock, 7, VFIO_USER_DMA_UNMAP,
+                        dma_regions, sizeof *dma_regions * server_max_fds,
+                        NULL, NULL, 0);
     if (ret < 0) {
         errx(EXIT_FAILURE, "failed to unmap DMA regions: %s", strerror(-ret));
     }