From 2461d80e6c36dfefdcde1ec8735c317c31895c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Fri, 12 Jul 2019 14:50:52 +0400 Subject: docker.py: add --run-as-current-user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (podman will need further tweaks) Suggested-by: Paolo Bonzini Signed-off-by: Marc-André Lureau Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Alex Bennée Reviewed-by: Daniel P. Berrangé --- tests/docker/Makefile.include | 2 +- tests/docker/docker.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index dbd58e5..582846a 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -212,7 +212,7 @@ docker-run: docker-qemu-src " COPYING $(EXECUTABLE) to $(IMAGE)")) $(call quiet-command, \ $(DOCKER_SCRIPT) run \ - $(if $(NOUSER),,-u $(shell id -u)) \ + $(if $(NOUSER),,--run-as-current-user) \ --security-opt seccomp=unconfined \ $(if $V,,--rm) \ $(if $(DEBUG),-ti,) \ diff --git a/tests/docker/docker.py b/tests/docker/docker.py index 53a8c9c..f15545a 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -333,8 +333,13 @@ class RunCommand(SubCommand): def args(self, parser): parser.add_argument("--keep", action="store_true", help="Don't remove image when command completes") + parser.add_argument("--run-as-current-user", action="store_true", + help="Run container using the current user's uid") def run(self, args, argv): + if args.run_as_current_user: + uid = os.getuid() + argv = [ "-u", str(uid) ] + argv return Docker().run(argv, args.keep, quiet=args.quiet) -- cgit v1.1 From 9459f754134bb786edf85ca9fc00f1805e67bd74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Fri, 12 Jul 2019 16:46:13 +0400 Subject: docker.py: add podman support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a --engine option to select either docker, podman or auto. Among other advantages, podman allows to run rootless & daemonless containers, fortunately sharing compatible CLI with docker. With current podman, we have to use a uidmap trick in order to be able to rw-share the ccache directory with the container user. With a user 1000, the default mapping is: 1000 (host) -> 0 (container). So write access to /var/tmp/ccache ends will end with permission denied error. With "--uidmap 1000:0:1 --uidmap 0:1:1000", the mapping is: 1000 (host) -> 0 (container, 1st namespace) -> 1000 (container, 2nd namespace). (the rest is mumbo jumbo to avoid holes in the range of UIDs) A future podman version may have an option such as --userns-keep-uid. Thanks to Debarshi Ray for the help! Signed-off-by: Marc-André Lureau Acked-by: Alex Bennée Reviewed-by: Daniel P. Berrangé --- tests/docker/docker.py | 48 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/tests/docker/docker.py b/tests/docker/docker.py index f15545a..ac5baab 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -20,6 +20,7 @@ import hashlib import atexit import uuid import argparse +import enum import tempfile import re import signal @@ -38,6 +39,26 @@ FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy'] DEVNULL = open(os.devnull, 'wb') +class EngineEnum(enum.IntEnum): + AUTO = 1 + DOCKER = 2 + PODMAN = 3 + + def __str__(self): + return self.name.lower() + + def __repr__(self): + return str(self) + + @staticmethod + def argparse(s): + try: + return EngineEnum[s.upper()] + except KeyError: + return s + + +USE_ENGINE = EngineEnum.AUTO def _text_checksum(text): """Calculate a digest string unique to the text content""" @@ -48,9 +69,14 @@ def _file_checksum(filename): return _text_checksum(open(filename, 'rb').read()) -def _guess_docker_command(): - """ Guess a working docker command or raise exception if not found""" - commands = [["docker"], ["sudo", "-n", "docker"]] +def _guess_engine_command(): + """ Guess a working engine command or raise exception if not found""" + commands = [] + + if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.PODMAN]: + commands += [["podman"]] + if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.DOCKER]: + commands += [["docker"], ["sudo", "-n", "docker"]] for cmd in commands: try: # docker version will return the client details in stdout @@ -61,7 +87,7 @@ def _guess_docker_command(): except OSError: pass commands_txt = "\n".join([" " + " ".join(x) for x in commands]) - raise Exception("Cannot find working docker command. Tried:\n%s" % + raise Exception("Cannot find working engine command. Tried:\n%s" % commands_txt) @@ -190,7 +216,7 @@ def _dockerfile_preprocess(df): class Docker(object): """ Running Docker commands """ def __init__(self): - self._command = _guess_docker_command() + self._command = _guess_engine_command() self._instances = [] atexit.register(self._kill_instances) signal.signal(signal.SIGTERM, self._kill_instances) @@ -340,6 +366,11 @@ class RunCommand(SubCommand): if args.run_as_current_user: uid = os.getuid() argv = [ "-u", str(uid) ] + argv + docker = Docker() + if docker._command[0] == "podman": + argv = [ "--uidmap", "%d:0:1" % uid, + "--uidmap", "0:1:%d" % uid, + "--uidmap", "%d:%d:64536" % (uid + 1, uid + 1)] + argv return Docker().run(argv, args.keep, quiet=args.quiet) @@ -507,6 +538,8 @@ class ProbeCommand(SubCommand): print("yes") elif docker._command[0] == "sudo": print("sudo") + elif docker._command[0] == "podman": + print("podman") except Exception: print("no") @@ -602,9 +635,13 @@ class CheckCommand(SubCommand): def main(): + global USE_ENGINE + parser = argparse.ArgumentParser(description="A Docker helper", usage="%s ..." % os.path.basename(sys.argv[0])) + parser.add_argument("--engine", type=EngineEnum.argparse, choices=list(EngineEnum), + help="specify which container engine to use") subparsers = parser.add_subparsers(title="subcommands", help=None) for cls in SubCommand.__subclasses__(): cmd = cls() @@ -613,6 +650,7 @@ def main(): cmd.args(subp) subp.set_defaults(cmdobj=cmd) args, argv = parser.parse_known_args() + USE_ENGINE = args.engine return args.cmdobj.run(args, argv) -- cgit v1.1 From 05af039d1e0baec1e1577701ec8309313f832f06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Thu, 23 May 2019 17:37:51 +0200 Subject: tests/docker: add podman support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow to specify the container engine to run with ENGINE variable. By default, ENGINE=auto and will select either podman or docker. Signed-off-by: Marc-André Lureau Reviewed-by: Alex Bennée Reviewed-by: Daniel P. Berrangé --- Makefile | 2 +- tests/docker/Makefile.include | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 574fede..ae17a83 100644 --- a/Makefile +++ b/Makefile @@ -1157,7 +1157,7 @@ endif @echo '' @echo 'Test targets:' @echo ' check - Run all tests (check-help for details)' - @echo ' docker - Help about targets running tests inside Docker containers' + @echo ' docker - Help about targets running tests inside containers' @echo ' vm-help - Help about targets running tests inside VM' @echo '' @echo 'Documentation targets:' diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 582846a..cf535cb 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -17,7 +17,9 @@ DOCKER_TESTS := $(notdir $(shell \ DOCKER_TOOLS := travis -DOCKER_SCRIPT=$(SRC_PATH)/tests/docker/docker.py +ENGINE := auto + +DOCKER_SCRIPT=$(SRC_PATH)/tests/docker/docker.py --engine $(ENGINE) TESTS ?= % IMAGES ?= % @@ -146,7 +148,7 @@ $(foreach i,$(filter-out $(DOCKER_PARTIAL_IMAGES),$(DOCKER_IMAGES) $(DOCKER_DEPR ) docker: - @echo 'Build QEMU and run tests inside Docker containers' + @echo 'Build QEMU and run tests inside Docker or Podman containers' @echo @echo 'Available targets:' @echo @@ -193,6 +195,8 @@ endif @echo ' EXECUTABLE= Include executable in image.' @echo ' EXTRA_FILES=" [... ]"' @echo ' Include extra files in image.' + @echo ' ENGINE=auto/docker/podman' + @echo ' Specify which container engine to run.' # This rule if for directly running against an arbitrary docker target. # It is called by the expanded docker targets (e.g. make -- cgit v1.1 From 71714178fa35dff1b9afc0762501c81360f7db82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 9 Jul 2019 18:12:28 +0400 Subject: tests: specify the address family when checking bind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getaddrinfo() may succeed with PF_UNSPEC, but fail when more specific. (this allows to skip some tests that would fail under podman) Signed-off-by: Marc-André Lureau Reviewed-by: Daniel P. Berrangé --- tests/socket-helpers.c | 17 +++++++++++++---- tests/socket-helpers.h | 11 ----------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/tests/socket-helpers.c b/tests/socket-helpers.c index 8112763..19a51e8 100644 --- a/tests/socket-helpers.c +++ b/tests/socket-helpers.c @@ -30,7 +30,16 @@ # define EAI_ADDRFAMILY 0 #endif -int socket_can_bind_connect(const char *hostname) +/* + * @hostname: a DNS name or numeric IP address + * + * Check whether it is possible to bind & connect to ports + * on the DNS name or IP address @hostname. If an IP address + * is used, it must not be a wildcard address. + * + * Returns 0 on success, -1 on error with errno set + */ +static int socket_can_bind_connect(const char *hostname, int family) { int lfd = -1, cfd = -1, afd = -1; struct addrinfo ai, *res = NULL; @@ -44,7 +53,7 @@ int socket_can_bind_connect(const char *hostname) memset(&ai, 0, sizeof(ai)); ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG; - ai.ai_family = AF_UNSPEC; + ai.ai_family = family; ai.ai_socktype = SOCK_STREAM; /* lookup */ @@ -129,7 +138,7 @@ int socket_check_protocol_support(bool *has_ipv4, bool *has_ipv6) { *has_ipv4 = *has_ipv6 = false; - if (socket_can_bind_connect("127.0.0.1") < 0) { + if (socket_can_bind_connect("127.0.0.1", PF_INET) < 0) { if (errno != EADDRNOTAVAIL) { return -1; } @@ -137,7 +146,7 @@ int socket_check_protocol_support(bool *has_ipv4, bool *has_ipv6) *has_ipv4 = true; } - if (socket_can_bind_connect("::1") < 0) { + if (socket_can_bind_connect("::1", PF_INET6) < 0) { if (errno != EADDRNOTAVAIL) { return -1; } diff --git a/tests/socket-helpers.h b/tests/socket-helpers.h index 9de0e6b..512a004 100644 --- a/tests/socket-helpers.h +++ b/tests/socket-helpers.h @@ -21,17 +21,6 @@ #define TESTS_SOCKET_HELPERS_H /* - * @hostname: a DNS name or numeric IP address - * - * Check whether it is possible to bind & connect to ports - * on the DNS name or IP address @hostname. If an IP address - * is used, it must not be a wildcard address. - * - * Returns 0 on success, -1 on error with errno set - */ -int socket_can_bind_connect(const char *hostname); - -/* * @has_ipv4: set to true on return if IPv4 is available * @has_ipv6: set to true on return if IPv6 is available * -- cgit v1.1 From e7b6ba4186f243f149b0d8cddc129fe681ba3912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 9 Jul 2019 23:24:11 +0400 Subject: test-char: skip tcp tests if ipv4 check failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marc-André Lureau Reviewed-by: Daniel P. Berrangé --- tests/Makefile.include | 2 +- tests/test-char.c | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/Makefile.include b/tests/Makefile.include index 39bed75..49684fd 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -525,7 +525,7 @@ tests/check-qlit$(EXESUF): tests/check-qlit.o $(test-util-obj-y) tests/check-qom-interface$(EXESUF): tests/check-qom-interface.o $(test-qom-obj-y) tests/check-qom-proplist$(EXESUF): tests/check-qom-proplist.o $(test-qom-obj-y) -tests/test-char$(EXESUF): tests/test-char.o $(test-util-obj-y) $(qtest-obj-y) $(test-io-obj-y) $(chardev-obj-y) +tests/test-char$(EXESUF): tests/test-char.o $(test-util-obj-y) $(qtest-obj-y) $(test-io-obj-y) $(chardev-obj-y) tests/socket-helpers.o tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(test-block-obj-y) tests/test-aio$(EXESUF): tests/test-aio.o $(test-block-obj-y) tests/test-aio-multithread$(EXESUF): tests/test-aio-multithread.o $(test-block-obj-y) diff --git a/tests/test-char.c b/tests/test-char.c index f9440cd..2dde620 100644 --- a/tests/test-char.c +++ b/tests/test-char.c @@ -15,6 +15,7 @@ #include "io/channel-socket.h" #include "qapi/qobject-input-visitor.h" #include "qapi/qapi-visit-sockets.h" +#include "socket-helpers.h" static bool quit; @@ -1356,11 +1357,17 @@ static void char_hotswap_test(void) int main(int argc, char **argv) { + bool has_ipv4, has_ipv6; + qemu_init_main_loop(&error_abort); socket_init(); g_test_init(&argc, &argv, NULL); + if (socket_check_protocol_support(&has_ipv4, &has_ipv6) < 0) { + return -1; + } + module_call_init(MODULE_INIT_QOM); qemu_add_opts(&qemu_chardev_opts); @@ -1438,10 +1445,12 @@ int main(int argc, char **argv) g_test_add_data_func("/char/socket/client/wait-conn-fdpass/" # name, \ &client6 ##name, char_socket_client_test) - SOCKET_SERVER_TEST(tcp, &tcpaddr); - SOCKET_CLIENT_TEST(tcp, &tcpaddr); - g_test_add_data_func("/char/socket/server/two-clients/tcp", &tcpaddr, - char_socket_server_two_clients_test); + if (has_ipv4) { + SOCKET_SERVER_TEST(tcp, &tcpaddr); + SOCKET_CLIENT_TEST(tcp, &tcpaddr); + g_test_add_data_func("/char/socket/server/two-clients/tcp", &tcpaddr, + char_socket_server_two_clients_test); + } #ifndef WIN32 SOCKET_SERVER_TEST(unix, &unixaddr); SOCKET_CLIENT_TEST(unix, &unixaddr); -- cgit v1.1 From a4eb74a66a0f34d53127e240c1eeae42073bc558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 9 Jul 2019 23:24:46 +0400 Subject: test: skip tests if socket_check_protocol_support() failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skip the tests if socket_check_protocol_support() failed, but do run g_test_run() to keep TAP harness happy. Signed-off-by: Marc-André Lureau Reviewed-by: Daniel P. Berrangé --- tests/test-char.c | 4 +++- tests/test-io-channel-socket.c | 4 +++- tests/test-util-sockets.c | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test-char.c b/tests/test-char.c index 2dde620..b56e43c 100644 --- a/tests/test-char.c +++ b/tests/test-char.c @@ -1365,7 +1365,8 @@ int main(int argc, char **argv) g_test_init(&argc, &argv, NULL); if (socket_check_protocol_support(&has_ipv4, &has_ipv6) < 0) { - return -1; + g_printerr("socket_check_protocol_support() failed\n"); + goto end; } module_call_init(MODULE_INIT_QOM); @@ -1465,5 +1466,6 @@ int main(int argc, char **argv) g_test_add_func("/char/hotswap", char_hotswap_test); g_test_add_func("/char/websocket", char_websock_test); +end: return g_test_run(); } diff --git a/tests/test-io-channel-socket.c b/tests/test-io-channel-socket.c index d2053c4..d172f30 100644 --- a/tests/test-io-channel-socket.c +++ b/tests/test-io-channel-socket.c @@ -566,7 +566,8 @@ int main(int argc, char **argv) * with either IPv4 or IPv6 disabled. */ if (socket_check_protocol_support(&has_ipv4, &has_ipv6) < 0) { - return 1; + g_printerr("socket_check_protocol_support() failed\n"); + goto end; } if (has_ipv4) { @@ -595,5 +596,6 @@ int main(int argc, char **argv) test_io_channel_unix_listen_cleanup); #endif /* _WIN32 */ +end: return g_test_run(); } diff --git a/tests/test-util-sockets.c b/tests/test-util-sockets.c index f1ebffe..e2a3a8a 100644 --- a/tests/test-util-sockets.c +++ b/tests/test-util-sockets.c @@ -242,7 +242,8 @@ int main(int argc, char **argv) * with either IPv4 or IPv6 disabled. */ if (socket_check_protocol_support(&has_ipv4, &has_ipv6) < 0) { - return 1; + g_printerr("socket_check_protocol_support() failed\n"); + goto end; } if (has_ipv4) { @@ -264,5 +265,6 @@ int main(int argc, char **argv) test_socket_fd_pass_num_nocli); } +end: return g_test_run(); } -- cgit v1.1