start python-based testing framework (#447)

Trying to do our unit/functional testing with C is very tedious, and cmocka
especially is a continual pain point. This commit introduces a Python-based
testing infrastructure, and adds an initial set of functional tests for client
negotiation.

The tests work under Valgrind for leak/bad access detection of the C code, but
not under ASAN, which lacks any meaningful shared-library support.

We should be able to replace all of current C-based unit tests with this,
reverting samples/ back to demo code only.

Signed-off-by: John Levon <john.levon@nutanix.com>
Reviewed-by: Thanos Makatos <thanos.makatos@nutanix.com>

7 files changed, 421 insertions, 8 deletions

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 7f8dc3d..c066619 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -9,7 +9,8 @@ jobs:
       - name: pre-push
         run: |
           sudo apt-get update
-          sudo apt-get -y install libjson-c-dev libcmocka-dev clang valgrind
+          sudo apt-get -y install libjson-c-dev libcmocka-dev clang valgrind \
+                                  python3-pytest debianutils
           make pre-push VERBOSE=1
   ubuntu-18:
     timeout-minutes: 5
@@ -19,7 +20,8 @@ jobs:
       - name: pre-push
         run: |
           sudo apt update
-          sudo apt-get -y install libjson-c-dev libcmocka-dev clang valgrind
+          sudo apt-get -y install libjson-c-dev libcmocka-dev clang valgrind \
+                                  python3-pytest debianutils
           make pre-push VERBOSE=1
   centos-7:
     timeout-minutes: 5
@@ -31,7 +33,7 @@ jobs:
         run: |
           yum -y install make gcc-4.8.5 epel-release pciutils
           yum -y install clang cmake json-c-devel libcmocka-devel \
-              openssl-devel valgrind
+              openssl-devel valgrind python36-pytest which
           make pre-push VERBOSE=1
   fedora-34:
     timeout-minutes: 5
@@ -41,6 +43,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: pre-push
         run: |
-          dnf -y install --releasever=34 gcc make clang cmake json-c-devel libcmocka-devel \
-              openssl-devel pciutils diffutils valgrind
+          dnf -y install --releasever=34 \
+              gcc make clang cmake json-c-devel libcmocka-devel openssl-devel \
+              pciutils diffutils valgrind python3-pytest which
           make pre-push VERBOSE=1
diff --git a/.gitignore b/.gitignore
index 55d6ffe..c4a5874 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
-build/
+.cache
+build
 cscope.out
 tags
diff --git a/Makefile b/Makefile
index 6607fe6..6e801cf 100644
--- a/Makefile
+++ b/Makefile
@@ -54,13 +54,61 @@ BUILD_DIR = $(BUILD_DIR_BASE)/$(BUILD_TYPE)
 
 INSTALL_PREFIX ?= /usr/local
 
-PHONY_TARGETS := all test pre-push realclean buildclean force_cmake tags
+PHONY_TARGETS := all pytest pytest-valgrind test pre-push realclean buildclean force_cmake tags
 .PHONY: $(PHONY_TARGETS)
 
 all $(filter-out $(PHONY_TARGETS), $(MAKECMDGOALS)): $(BUILD_DIR)/Makefile
 	+$(MAKE) -C $(BUILD_DIR) $@
 
-test: all
+#
+# NB: add --capture=no to get a C-level assert failure output.
+#
+PYTESTCMD = \
+	$(shell which -a pytest-3 /bin/true 2>/dev/null | head -1) \
+	-rP \
+	--quiet
+
+PYTEST = \
+    BUILD_TYPE=$(BUILD_TYPE) \
+	$(PYTESTCMD)
+
+#
+# In our tests, we make sure to destroy the ctx at the end of each test; this is
+# enough for these settings to detect (most?) library leaks as "definite",
+# without all the noise from the rest of the Python runtime.
+#
+# As running under valgrind is very slow, we don't run this unless requested.
+#
+PYTESTVALGRIND = \
+	BUILD_TYPE=$(BUILD_TYPE) \
+	PYTHONMALLOC=malloc \
+	valgrind \
+	--suppressions=$(CURDIR)/test/py/valgrind.supp \
+	--quiet \
+	--track-origins=yes \
+	--errors-for-leak-kinds=definite \
+	--show-leak-kinds=definite \
+	--leak-check=full \
+	--error-exitcode=1 \
+	$(PYTESTCMD)
+
+ifdef WITH_ASAN
+
+pytest pytest-valgrind:
+
+else
+
+pytest: all
+	@echo "=== Running python tests ==="
+	$(PYTEST)
+
+pytest-valgrind: all
+	@echo "=== Running python tests with valgrind ==="
+	$(PYTESTVALGRIND)
+
+endif
+
+test: all pytest
 	cd $(BUILD_DIR)/test; ctest --verbose
 
 pre-push: realclean
@@ -71,6 +119,7 @@ pre-push: realclean
 	make realclean
 	make test CC=gcc BUILD_TYPE=rel
 	make test CC=gcc
+	make pytest-valgrind
 
 realclean:
 	rm -rf $(BUILD_DIR_BASE)
diff --git a/test/py/.gitignore b/test/py/.gitignore
new file mode 100644
index 0000000..bee8a64
--- /dev/null
+++ b/test/py/.gitignore
@@ -0,0 +1 @@
+__pycache__
diff --git a/test/py/libvfio_user.py b/test/py/libvfio_user.py
new file mode 100644
index 0000000..8dee414
--- /dev/null
+++ b/test/py/libvfio_user.py
@@ -0,0 +1,166 @@
+#
+# Copyright (c) 2021 Nutanix Inc. All rights reserved.
+#
+# Authors: John Levon <john.levon@nutanix.com>
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#      * Redistributions of source code must retain the above copyright
+#        notice, this list of conditions and the following disclaimer.
+#      * Redistributions in binary form must reproduce the above copyright
+#        notice, this list of conditions and the following disclaimer in the
+#        documentation and/or other materials provided with the distribution.
+#      * Neither the name of Nutanix nor the names of its contributors may be
+#        used to endorse or promote products derived from this software without
+#        specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+#  ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+#  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+#  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+#  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+#  OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+#  DAMAGE.
+#
+
+#
+# Note that we don't use enum here, as class.value is a little verbose
+#
+
+from types import SimpleNamespace
+import ctypes as c
+import json
+import os
+import pathlib
+import socket
+import struct
+import syslog
+
+SOCK_PATH = b"/tmp/vfio-user.sock.%d" % os.getpid()
+
+VFU_TRANS_SOCK = 0
+LIBVFIO_USER_FLAG_ATTACH_NB = (1 << 0)
+VFU_DEV_TYPE_PCI = 0
+
+LIBVFIO_USER_MAJOR = 0
+LIBVFIO_USER_MINOR = 1
+
+VFIO_USER_CLIENT_MAX_FDS_LIMIT = 1024
+
+SERVER_MAX_FDS = 8
+
+SERVER_MAX_MSG_SIZE = 65536
+
+# enum vfio_user_command
+VFIO_USER_VERSION                   = 1
+VFIO_USER_DMA_MAP                   = 2
+VFIO_USER_DMA_UNMAP                 = 3
+VFIO_USER_DEVICE_GET_INFO           = 4
+VFIO_USER_DEVICE_GET_REGION_INFO    = 5
+VFIO_USER_DEVICE_GET_IRQ_INFO       = 6
+VFIO_USER_DEVICE_SET_IRQS           = 7
+VFIO_USER_REGION_READ               = 8
+VFIO_USER_REGION_WRITE              = 9
+VFIO_USER_DMA_READ                  = 10
+VFIO_USER_DMA_WRITE                 = 11
+VFIO_USER_VM_INTERRUPT              = 12
+VFIO_USER_DEVICE_RESET              = 13
+VFIO_USER_DIRTY_PAGES               = 14
+VFIO_USER_MAX                       = 15
+
+VFIO_USER_F_TYPE_COMMAND = 0
+VFIO_USER_F_TYPE_REPLY = 1
+
+SIZEOF_VFIO_USER_HEADER = 16
+
+topdir = os.path.realpath(os.path.dirname(__file__) + "/../..")
+build_type = os.getenv("BUILD_TYPE", default="dbg")
+libname = "%s/build/%s/lib/libvfio-user.so" % (topdir, build_type)
+lib = c.CDLL(libname, use_errno=True)
+lib.vfu_create_ctx.argtypes = (c.c_int, c.c_char_p, c.c_int,
+                               c.c_void_p, c.c_int)
+lib.vfu_create_ctx.restype = (c.c_void_p)
+lib.vfu_setup_log.argtypes = (c.c_void_p, c.c_void_p, c.c_int)
+lib.vfu_realize_ctx.argtypes = (c.c_void_p,)
+lib.vfu_attach_ctx.argtypes = (c.c_void_p,)
+lib.vfu_run_ctx.argtypes = (c.c_void_p,)
+lib.vfu_destroy_ctx.argtypes = (c.c_void_p,)
+
+msg_id = 1
+
+#
+# Util functions
+#
+
+def connect_sock():
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.connect(SOCK_PATH)
+    return sock
+
+def get_reply(sock, expect=0):
+    buf = sock.recv(4096)
+    (msg_id, cmd, msg_size, flags, errno) = struct.unpack("HHIII", buf[0:16])
+    assert errno == expect
+    return buf[16:]
+
+#
+# Parse JSON into an object with attributes (instead of using a dict)
+#
+def parse_json(json_str):
+    return json.loads(json_str, object_hook=lambda d: SimpleNamespace(**d))
+
+
+#
+# Library wrappers
+#
+
+@c.CFUNCTYPE(None, c.c_void_p, c.c_int, c.c_char_p)
+def log(ctx, level, msg):
+    print(msg.decode("utf-8"))
+
+def vfio_user_header(cmd, size, no_reply=False, error=False, error_no=0):
+    global msg_id
+
+    buf = struct.pack("HHIII", msg_id, cmd, SIZEOF_VFIO_USER_HEADER + size,
+                      0, error_no)
+
+    msg_id += 1
+
+    return buf
+
+def vfu_create_ctx(trans=VFU_TRANS_SOCK, sock_path=SOCK_PATH, flags=0,
+                   private=None, dev_type=VFU_DEV_TYPE_PCI):
+    if os.path.exists(sock_path):
+        os.remove(sock_path)
+
+    ctx = lib.vfu_create_ctx(trans, sock_path, flags, private, dev_type)
+
+    if ctx:
+        lib.vfu_setup_log(ctx, log, syslog.LOG_DEBUG)
+
+    return ctx
+
+def vfu_realize_ctx(ctx):
+    return lib.vfu_realize_ctx(ctx)
+
+def vfu_attach_ctx(ctx, expect=0):
+    ret = lib.vfu_attach_ctx(ctx)
+    if expect == 0:
+        assert ret == 0
+    else:
+        assert ret == -1
+        assert c.get_errno() == expect
+    return ret
+
+def vfu_run_ctx(ctx):
+    lib.vfu_run_ctx(ctx)
+
+def vfu_destroy_ctx(ctx):
+    lib.vfu_destroy_ctx(ctx)
+    ctx = None
+    if os.path.exists(SOCK_PATH):
+        os.remove(SOCK_PATH)
diff --git a/test/py/test_negotiate.py b/test/py/test_negotiate.py
new file mode 100644
index 0000000..9a6afd9
--- /dev/null
+++ b/test/py/test_negotiate.py
@@ -0,0 +1,177 @@
+#
+# Copyright (c) 2021 Nutanix Inc. All rights reserved.
+#
+# Authors: John Levon <john.levon@nutanix.com>
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#      * Redistributions of source code must retain the above copyright
+#        notice, this list of conditions and the following disclaimer.
+#      * Redistributions in binary form must reproduce the above copyright
+#        notice, this list of conditions and the following disclaimer in the
+#        documentation and/or other materials provided with the distribution.
+#      * Neither the name of Nutanix nor the names of its contributors may be
+#        used to endorse or promote products derived from this software without
+#        specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+#  ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+#  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+#  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+#  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+#  OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+#  DAMAGE.
+#
+
+from libvfio_user import *
+import errno
+import json
+import os
+import socket
+import struct
+
+ctx = None
+
+def client_version_json(expect=0, json=''):
+    sock = connect_sock()
+
+    payload = struct.pack("HH%dsc" % len(json),
+                          LIBVFIO_USER_MAJOR, LIBVFIO_USER_MINOR, json, b'\0')
+    hdr = vfio_user_header(VFIO_USER_VERSION, size=len(payload))
+    sock.send(hdr + payload)
+    vfu_attach_ctx(ctx, expect=expect)
+    payload = get_reply(sock, expect=expect)
+    sock.close()
+
+    return payload
+
+def test_server_setup():
+    global ctx
+
+    ctx = vfu_create_ctx()
+    assert ctx != None
+
+    ret = vfu_realize_ctx(ctx)
+    assert ret == 0
+
+def test_short_write():
+    sock = connect_sock()
+    hdr = vfio_user_header(VFIO_USER_VERSION, size=0)
+    sock.send(hdr)
+
+    vfu_attach_ctx(ctx, expect=errno.EINVAL)
+    get_reply(sock, expect=errno.EINVAL)
+
+def test_bad_command():
+    sock = connect_sock()
+
+    payload = struct.pack("HHs", LIBVFIO_USER_MAJOR, LIBVFIO_USER_MINOR, b"")
+    hdr = vfio_user_header(999, size=len(payload))
+    sock.send(hdr + payload)
+
+    vfu_attach_ctx(ctx, expect=errno.EINVAL)
+    get_reply(sock, expect=errno.EINVAL)
+
+def test_invalid_major():
+    sock = connect_sock()
+
+    payload = struct.pack("HHs", 999, LIBVFIO_USER_MINOR, b"")
+    hdr = vfio_user_header(VFIO_USER_VERSION, size=len(payload))
+    sock.send(hdr + payload)
+
+    vfu_attach_ctx(ctx, expect=errno.EINVAL)
+    get_reply(sock, expect=errno.EINVAL)
+
+def test_invalid_json_missing_NUL():
+    sock = connect_sock()
+
+    payload = struct.pack("HHcc", LIBVFIO_USER_MAJOR, LIBVFIO_USER_MINOR,
+                          b"{", b"}")
+    hdr = vfio_user_header(VFIO_USER_VERSION, size=len(payload))
+    sock.send(hdr + payload)
+
+    vfu_attach_ctx(ctx, expect=errno.EINVAL)
+    get_reply(sock, expect=errno.EINVAL)
+
+def test_invalid_json_missing_closing_brace():
+    client_version_json(errno.EINVAL,  b"{")
+
+def test_invalid_json_missing_closing_quote():
+    client_version_json(errno.EINVAL, b'"')
+
+def test_invalid_json_bad_capabilities_object():
+    client_version_json(errno.EINVAL, b'{ "capabilities": "23" }')
+
+def test_invalid_json_bad_max_fds():
+    client_version_json(errno.EINVAL,
+                        b'{ "capabilities": { "max_fds": "foo" } }')
+
+def test_invalid_json_bad_max_fds():
+    client_version_json(errno.EINVAL,
+                        b'{ "capabilities": { "max_fds": -1 } }')
+
+def test_invalid_json_bad_max_fds2():
+    client_version_json(errno.EINVAL,
+                        b'{ "capabilities": { "max_fds": %d } }' %
+                        (VFIO_USER_CLIENT_MAX_FDS_LIMIT + 1))
+
+def test_invalid_json_bad_migration_object():
+    client_version_json(errno.EINVAL,
+                        b'{ "capabilities": { "migration": "23" } }')
+
+def test_invalid_json_bad_pgsize():
+    client_version_json(errno.EINVAL, b'{ "capabilities": ' +
+                        b'{ "migration": { "pgsize": "foo" } } }')
+
+#
+# FIXME: need vfu_setup_device_migration_callbacks() to be able to test this
+# failure mode.
+#
+def test_invalid_json_bad_pgsize():
+    if False:
+        client_version_json(errno.EINVAL,
+            b'{ "capabilities": { "migration": { "pgsize": 4095 } } }')
+
+def test_valid_negotiate_no_json():
+    sock = connect_sock()
+
+    payload = struct.pack("HH", LIBVFIO_USER_MAJOR, LIBVFIO_USER_MINOR)
+    hdr = vfio_user_header(VFIO_USER_VERSION, size=len(payload))
+    sock.send(hdr + payload)
+
+    vfu_attach_ctx(ctx)
+
+    payload = get_reply(sock)
+    (major, minor, json_str, _) = struct.unpack("HH%dsc" % (len(payload) - 5),
+                                                payload)
+    assert major == LIBVFIO_USER_MAJOR
+    assert minor == LIBVFIO_USER_MINOR
+    json = parse_json(json_str)
+    assert json.capabilities.max_fds == SERVER_MAX_FDS
+    assert json.capabilities.max_msg_size == SERVER_MAX_MSG_SIZE
+    # FIXME: migration object checks
+
+    sock.close()
+
+    # notice client closed connection
+    vfu_run_ctx(ctx)
+
+def test_valid_negotiate_empty_json():
+    client_version_json(json=b'{}')
+
+    # notice client closed connection
+    vfu_run_ctx(ctx)
+
+def test_valid_negotiate_json():
+    client_version_json(json=bytes('{ "capabilities": { "max_fds": %s } }' %
+                        VFIO_USER_CLIENT_MAX_FDS_LIMIT, "utf-8"))
+
+    # notice client closed connection
+    vfu_run_ctx(ctx)
+
+def test_destroying():
+    vfu_destroy_ctx(ctx)
diff --git a/test/py/valgrind.supp b/test/py/valgrind.supp
new file mode 100644
index 0000000..3c3ed44
--- /dev/null
+++ b/test/py/valgrind.supp
@@ -0,0 +1,16 @@
+#
+# Various versions of Python in our CI do not behave well under valgrind.
+#
+{
+    python-is-b0rked
+    Memcheck:Cond
+    ...
+    fun:PyUnicode*
+}
+
+{
+    python-is-b0rked
+    Memcheck:Cond
+    ...
+    fun:wcstombs
+}