fix dma unmap validation (#626)

There were two issues with unmap request validation when the dirty bitmap flag was set:

- we weren't checking ->argsz against the maximum transfer size, allowing a client
to trigger unbounded allocations

- we needed to check for overflow when calculating the requested message out size

Found via AFL++.

Signed-off-by: John Levon <john.levon@nutanix.com>
Reviewed-by: Thanos Makatos <thanos.makatos@nutanix.com>

1 files changed, 21 insertions, 2 deletions

diff --git a/test/py/test_dma_unmap.py b/test/py/test_dma_unmap.py
index 1c7cea0..f1da4b0 100644
--- a/test/py/test_dma_unmap.py
+++ b/test/py/test_dma_unmap.py
@@ -61,14 +61,33 @@ def test_dma_unmap_short_write():
 
 def test_dma_unmap_bad_argsz():
 
-    vfio_user_dma_unmap(argsz=8, flags=0x2323, addr=0x1000, size=4096)
+    payload = vfio_user_dma_unmap(argsz=8, flags=0, addr=0x1000, size=4096)
+    msg(ctx, sock, VFIO_USER_DMA_UNMAP, payload, expect=errno.EINVAL)
+
+
+def test_dma_unmap_bad_argsz2():
+
+    payload = vfio_user_dma_unmap(argsz=SERVER_MAX_DATA_XFER_SIZE + 8, flags=0,
+                                  addr=0x1000, size=4096)
+    msg(ctx, sock, VFIO_USER_DMA_UNMAP, payload, expect=errno.EINVAL)
+
+
+def test_dma_unmap_dirty_bad_argsz():
+
+    argsz = len(vfio_user_dma_unmap()) + len(vfio_user_bitmap())
+    unmap = vfio_user_dma_unmap(argsz=argsz,
+        flags=VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP, addr=0x10000, size=4096)
+    bitmap = vfio_user_bitmap(pgsize=4096, size=(UINT64_MAX - argsz) + 8)
+    payload = bytes(unmap) + bytes(bitmap)
+
+    msg(ctx, sock, VFIO_USER_DMA_UNMAP, payload, expect=errno.EINVAL)
 
 
 def test_dma_unmap_invalid_flags():
 
     payload = vfio_user_dma_unmap(argsz=len(vfio_user_dma_unmap()),
                                   flags=0x4, addr=0x1000, size=4096)
-    msg(ctx, sock, VFIO_USER_DMA_UNMAP, payload, expect=errno.ENOTSUP)
+    msg(ctx, sock, VFIO_USER_DMA_UNMAP, payload, expect=errno.EINVAL)
 
 
 def test_dma_unmap():