From d6db2a1cdf6c173304c32be3dfc18e0c8985d9af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 7 Feb 2019 15:34:23 +0000 Subject: docker: add debian-buster-arm64-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can't build QEMU with this but we can use this image to build newer arm64 testcases which need more up to date tools. Signed-off-by: Alex Bennée --- tests/docker/Makefile.include | 1 + .../dockerfiles/debian-buster-arm64-cross.docker | 15 +++++++++++ tests/docker/dockerfiles/debian10.docker | 31 ++++++++++++++++++++++ tests/tcg/aarch64/Makefile.include | 2 +- 4 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 tests/docker/dockerfiles/debian-buster-arm64-cross.docker create mode 100644 tests/docker/dockerfiles/debian10.docker (limited to 'tests') diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 7032c68..83d43c5 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -90,6 +90,7 @@ docker-image-debian-amd64: docker-image-debian9 docker-image-debian-armel-cross: docker-image-debian9 docker-image-debian-armhf-cross: docker-image-debian9 docker-image-debian-arm64-cross: docker-image-debian9 +docker-image-debian-buster-arm64-cross: docker-image-debian10 docker-image-debian-mips-cross: docker-image-debian9 docker-image-debian-mipsel-cross: docker-image-debian9 docker-image-debian-mips64el-cross: docker-image-debian9 diff --git a/tests/docker/dockerfiles/debian-buster-arm64-cross.docker b/tests/docker/dockerfiles/debian-buster-arm64-cross.docker new file mode 100644 index 0000000..33ada13 --- /dev/null +++ b/tests/docker/dockerfiles/debian-buster-arm64-cross.docker @@ -0,0 +1,15 @@ +# +# Docker arm64 cross-compiler target +# +# This docker target builds on the Debian's Buster base image. There +# are no QEMU pre-requistes so this image can only be used to build +# test cases. +# +FROM qemu:debian10 + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture arm64 +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt-get install -y --no-install-recommends \ + crossbuild-essential-arm64 diff --git a/tests/docker/dockerfiles/debian10.docker b/tests/docker/dockerfiles/debian10.docker new file mode 100644 index 0000000..aeeb151 --- /dev/null +++ b/tests/docker/dockerfiles/debian10.docker @@ -0,0 +1,31 @@ +# +# Docker multiarch cross-compiler target +# +# This docker target is builds on Debian cross compiler targets to build distro +# with a selection of cross compilers for building test binaries. +# +# On its own you can't build much but the docker-foo-cross targets +# build on top of the base debian image. +# +FROM debian:buster-slim + +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list + +# Install common build utilities +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + bison \ + build-essential \ + ca-certificates \ + clang \ + flex \ + gettext \ + git \ + pkg-config \ + psmisc \ + python \ + texinfo \ + $(apt-get -s build-dep qemu | egrep ^Inst | fgrep '[all]' | cut -d\ -f2) diff --git a/tests/tcg/aarch64/Makefile.include b/tests/tcg/aarch64/Makefile.include index de32c91..5d4e4c6 100644 --- a/tests/tcg/aarch64/Makefile.include +++ b/tests/tcg/aarch64/Makefile.include @@ -3,6 +3,6 @@ # We don't have any bigendian build tools so we only use this for AArch64 ifeq ($(TARGET_NAME),aarch64) -DOCKER_IMAGE=debian-arm64-cross +DOCKER_IMAGE=debian-buster-arm64-cross DOCKER_CROSS_COMPILER=aarch64-linux-gnu-gcc endif -- cgit v1.1 From 7e81d198407580992978efb911c74107fad0b05f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 15 Jan 2019 14:28:39 +0000 Subject: tests: make docker.py update use configured binfmt path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When copying a QEMU binary into a linux-user docker image we should check what the current configured binfmt_misc path is rather than just assuming "/usr/bin/qemu-bin". Obviously if the user changes the configuration afterwards they will break their images again. Signed-off-by: Alex Bennée --- tests/docker/docker.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'tests') diff --git a/tests/docker/docker.py b/tests/docker/docker.py index 02d8a83..30f463a 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -123,17 +123,17 @@ def _check_binfmt_misc(executable): if not os.path.exists(binfmt_entry): print ("No binfmt_misc entry for %s" % (binary)) - return False + return None with open(binfmt_entry) as x: entry = x.read() - qpath = "/usr/bin/%s" % (binary) - if not re.search("interpreter %s\n" % (qpath), entry): - print ("binfmt_misc for %s does not point to %s" % (binary, qpath)) - return False - - return True + m = re.search("interpreter (\S+)\n", entry) + interp = m.group(1) + if interp and interp != executable: + print("binfmt_misc for %s does not point to %s, using %s" % + (binary, executable, interp)) + return interp def _read_qemu_dockerfile(img_name): # special case for Debian linux-user images @@ -394,9 +394,14 @@ class UpdateCommand(SubCommand): tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz") tmp_tar = TarFile(fileobj=tmp, mode='w') - # Add the executable to the tarball - bn = os.path.basename(args.executable) - ff = "/usr/bin/%s" % bn + # Add the executable to the tarball, using the current + # configured binfmt_misc path. + ff = _check_binfmt_misc(args.executable) + if not ff: + bn = os.path.basename(args.executable) + ff = "/usr/bin/%s" % bn + print ("No binfmt_misc configured: copied to %s" % (ff)) + tmp_tar.add(args.executable, arcname=ff) # Add any associated libraries -- cgit v1.1 From 43c898b75a3bb92b113f9a366a06cac013d055a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 15 Jan 2019 14:37:51 +0000 Subject: tests: make docker.py check for persistent configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit binfmt_misc configured with the "F" flag opens the interpreter at config time. This means it can use an already open file-descriptor to run QEMU so there is no point trying to copy the binary into a container. Signed-off-by: Alex Bennée --- tests/docker/docker.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'tests') diff --git a/tests/docker/docker.py b/tests/docker/docker.py index 30f463a..7687287 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -127,6 +127,11 @@ def _check_binfmt_misc(executable): with open(binfmt_entry) as x: entry = x.read() + if re.search("flags:.*F.*\n", entry): + print("binfmt_misc for %s uses persistent(F) mapping to host binary\n" % + (binary)) + return None + m = re.search("interpreter (\S+)\n", entry) interp = m.group(1) if interp and interp != executable: -- cgit v1.1 From d10404b19393808394a129dd77974810ea87235d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 23 Jan 2019 17:07:08 +0000 Subject: tests: docker.py be even smarter with persistent binfmt_misc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we have a persistent mapping we don't need the QEMU binary copied into the container as the kernel has already opened the file and will pass the fd in. However the support libraries will still need to be there. Signed-off-by: Alex Bennée --- tests/docker/docker.py | 56 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 18 deletions(-) (limited to 'tests') diff --git a/tests/docker/docker.py b/tests/docker/docker.py index 7687287..a74338c 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -96,13 +96,22 @@ def _get_so_libs(executable): return libs -def _copy_binary_with_libs(src, dest_dir): - """Copy a binary executable and all its dependent libraries. +def _copy_binary_with_libs(src, bin_dest, dest_dir): + """Maybe copy a binary and all its dependent libraries. + + If bin_dest isn't set we only copy the support libraries because + we don't need qemu in the docker path to run (due to persistent + mapping). Indeed users may get confused if we aren't running what + is in the image. This does rely on the host file-system being fairly multi-arch - aware so the file don't clash with the guests layout.""" + aware so the file don't clash with the guests layout. + """ - _copy_with_mkdir(src, dest_dir, "/usr/bin") + if bin_dest: + _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest)) + else: + print("only copying support libraries for %s" % (src)) libs = _get_so_libs(src) if libs: @@ -116,21 +125,26 @@ def _check_binfmt_misc(executable): The details of setting up binfmt_misc are outside the scope of this script but we should at least fail early with a useful - message if it won't work.""" + message if it won't work. + + Returns the configured binfmt path and a valid flag. For + persistent configurations we will still want to copy and dependent + libraries. + """ binary = os.path.basename(executable) binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary) if not os.path.exists(binfmt_entry): print ("No binfmt_misc entry for %s" % (binary)) - return None + return None, False with open(binfmt_entry) as x: entry = x.read() if re.search("flags:.*F.*\n", entry): print("binfmt_misc for %s uses persistent(F) mapping to host binary\n" % (binary)) - return None + return None, True m = re.search("interpreter (\S+)\n", entry) interp = m.group(1) @@ -138,7 +152,8 @@ def _check_binfmt_misc(executable): print("binfmt_misc for %s does not point to %s, using %s" % (binary, executable, interp)) - return interp + return interp, True + def _read_qemu_dockerfile(img_name): # special case for Debian linux-user images @@ -345,7 +360,8 @@ class BuildCommand(SubCommand): # Validate binfmt_misc will work if args.include_executable: - if not _check_binfmt_misc(args.include_executable): + qpath, enabled = _check_binfmt_misc(args.include_executable) + if not enabled: return 1 # Is there a .pre file to run in the build context? @@ -368,7 +384,9 @@ class BuildCommand(SubCommand): # FIXME: there is no checksum of this executable and the linked # libraries, once the image built any change of this executable # or any library won't trigger another build. - _copy_binary_with_libs(args.include_executable, docker_dir) + _copy_binary_with_libs(args.include_executable, + qpath, docker_dir) + for filename in args.extra_files or []: _copy_with_mkdir(filename, docker_dir) cksum += [(filename, _file_checksum(filename))] @@ -400,14 +418,16 @@ class UpdateCommand(SubCommand): tmp_tar = TarFile(fileobj=tmp, mode='w') # Add the executable to the tarball, using the current - # configured binfmt_misc path. - ff = _check_binfmt_misc(args.executable) - if not ff: - bn = os.path.basename(args.executable) - ff = "/usr/bin/%s" % bn - print ("No binfmt_misc configured: copied to %s" % (ff)) - - tmp_tar.add(args.executable, arcname=ff) + # configured binfmt_misc path. If we don't get a path then we + # only need the support libraries copied + ff, enabled = _check_binfmt_misc(args.executable) + + if not enabled: + print("binfmt_misc not enabled, update disabled") + return 1 + + if ff: + tmp_tar.add(args.executable, arcname=ff) # Add any associated libraries libs = _get_so_libs(args.executable) -- cgit v1.1 From 432d8ad5f6a47c09648320412c9552637261bdc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 23 Jan 2019 17:13:55 +0000 Subject: tests: PEP8 cleanup of docker.py, mostly white space MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit My editor keeps putting squiggly lines under a bunch of the python lines to remind me how non-PEP8 compliant it is. Clean that up so it's easier to spot new errors. Signed-off-by: Alex Bennée --- tests/docker/docker.py | 48 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) (limited to 'tests') diff --git a/tests/docker/docker.py b/tests/docker/docker.py index a74338c..53a8c9c 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -30,7 +30,7 @@ except ImportError: from io import StringIO from shutil import copy, rmtree from pwd import getpwuid -from datetime import datetime,timedelta +from datetime import datetime, timedelta FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy'] @@ -43,9 +43,11 @@ def _text_checksum(text): """Calculate a digest string unique to the text content""" return hashlib.sha1(text).hexdigest() + def _file_checksum(filename): return _text_checksum(open(filename, 'rb').read()) + def _guess_docker_command(): """ Guess a working docker command or raise exception if not found""" commands = [["docker"], ["sudo", "-n", "docker"]] @@ -59,9 +61,10 @@ def _guess_docker_command(): except OSError: pass commands_txt = "\n".join([" " + " ".join(x) for x in commands]) - raise Exception("Cannot find working docker command. Tried:\n%s" % \ + raise Exception("Cannot find working docker command. Tried:\n%s" % commands_txt) + def _copy_with_mkdir(src, root_dir, sub_path='.'): """Copy src into root_dir, creating sub_path as needed.""" dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path)) @@ -96,6 +99,7 @@ def _get_so_libs(executable): return libs + def _copy_binary_with_libs(src, bin_dest, dest_dir): """Maybe copy a binary and all its dependent libraries. @@ -117,7 +121,7 @@ def _copy_binary_with_libs(src, bin_dest, dest_dir): if libs: for l in libs: so_path = os.path.dirname(l) - _copy_with_mkdir(l , dest_dir, so_path) + _copy_with_mkdir(l, dest_dir, so_path) def _check_binfmt_misc(executable): @@ -142,7 +146,7 @@ def _check_binfmt_misc(executable): with open(binfmt_entry) as x: entry = x.read() if re.search("flags:.*F.*\n", entry): - print("binfmt_misc for %s uses persistent(F) mapping to host binary\n" % + print("binfmt_misc for %s uses persistent(F) mapping to host binary" % (binary)) return None, True @@ -164,6 +168,7 @@ def _read_qemu_dockerfile(img_name): img_name + ".docker") return open(df, "r").read() + def _dockerfile_preprocess(df): out = "" for l in df.splitlines(): @@ -181,6 +186,7 @@ def _dockerfile_preprocess(df): out += l + "\n" return out + class Docker(object): """ Running Docker commands """ def __init__(self): @@ -248,7 +254,7 @@ class Docker(object): def build_image(self, tag, docker_dir, dockerfile, quiet=True, user=False, argv=None, extra_files_cksum=[]): - if argv == None: + if argv is None: argv = [] tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker") @@ -269,7 +275,7 @@ class Docker(object): tmp_df.flush() - self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \ + self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + [docker_dir], quiet=quiet) @@ -299,9 +305,11 @@ class Docker(object): def command(self, cmd, argv, quiet): return self._do([cmd] + argv, quiet=quiet) + class SubCommand(object): """A SubCommand template base class""" - name = None # Subcommand name + name = None # Subcommand name + def shared_args(self, parser): parser.add_argument("--quiet", action="store_true", help="Run quietly unless an error occurred") @@ -309,6 +317,7 @@ class SubCommand(object): def args(self, parser): """Setup argument parser""" pass + def run(self, args, argv): """Run command. args: parsed argument by argument parser. @@ -316,18 +325,23 @@ class SubCommand(object): """ pass + class RunCommand(SubCommand): """Invoke docker run and take care of cleaning up""" name = "run" + def args(self, parser): parser.add_argument("--keep", action="store_true", help="Don't remove image when command completes") + def run(self, args, argv): return Docker().run(argv, args.keep, quiet=args.quiet) + class BuildCommand(SubCommand): - """ Build docker image out of a dockerfile. Arguments: """ + """ Build docker image out of a dockerfile. Arg: """ name = "build" + def args(self, parser): parser.add_argument("--include-executable", "-e", help="""Specify a binary that will be copied to the @@ -392,8 +406,8 @@ class BuildCommand(SubCommand): cksum += [(filename, _file_checksum(filename))] argv += ["--build-arg=" + k.lower() + "=" + v - for k, v in os.environ.iteritems() - if k.lower() in FILTERED_ENV_NAMES] + for k, v in os.environ.iteritems() + if k.lower() in FILTERED_ENV_NAMES] dkr.build_image(tag, docker_dir, dockerfile, quiet=args.quiet, user=args.user, argv=argv, extra_files_cksum=cksum) @@ -402,9 +416,11 @@ class BuildCommand(SubCommand): return 0 + class UpdateCommand(SubCommand): - """ Update a docker image with new executables. Arguments: """ + """ Update a docker image with new executables. Args: """ name = "update" + def args(self, parser): parser.add_argument("tag", help="Image Tag") @@ -457,16 +473,20 @@ class UpdateCommand(SubCommand): return 0 + class CleanCommand(SubCommand): """Clean up docker instances""" name = "clean" + def run(self, args, argv): Docker().clean() return 0 + class ImagesCommand(SubCommand): """Run "docker images" command""" name = "images" + def run(self, args, argv): return Docker().command("images", argv, args.quiet) @@ -539,7 +559,7 @@ class CheckCommand(SubCommand): try: dkr = Docker() - except: + except subprocess.CalledProcessError: print("Docker not set up") return 1 @@ -578,7 +598,8 @@ class CheckCommand(SubCommand): def main(): parser = argparse.ArgumentParser(description="A Docker helper", - usage="%s ..." % os.path.basename(sys.argv[0])) + usage="%s ..." % + os.path.basename(sys.argv[0])) subparsers = parser.add_subparsers(title="subcommands", help=None) for cls in SubCommand.__subclasses__(): cmd = cls() @@ -589,5 +610,6 @@ def main(): args, argv = parser.parse_known_args() return args.cmdobj.run(args, argv) + if __name__ == "__main__": sys.exit(main()) -- cgit v1.1 From bd1497dc87364cff481d4c89ef1f248cfc47e4cf Mon Sep 17 00:00:00 2001 From: Gerd Hoffmann Date: Thu, 24 Jan 2019 12:54:48 +0100 Subject: tests/vm: move images to $HOME/.cache/qemu-vm/images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's easier to move around the images then, by replacing the subdirectory with a symlink. Allows to share the images between multiple qemu checkouts for example. Signed-off-by: Gerd Hoffmann Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée --- tests/vm/Makefile.include | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index a98fb30..a58383d 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -3,7 +3,8 @@ .PHONY: vm-build-all vm-clean-all IMAGES := ubuntu.i386 freebsd netbsd openbsd centos -IMAGE_FILES := $(patsubst %, tests/vm/%.img, $(IMAGES)) +IMAGES_DIR := $(HOME)/.cache/qemu-vm/images +IMAGE_FILES := $(patsubst %, $(IMAGES_DIR)/%.img, $(IMAGES)) .PRECIOUS: $(IMAGE_FILES) @@ -24,9 +25,10 @@ vm-build-all: $(addprefix vm-build-, $(IMAGES)) vm-clean-all: rm -f $(IMAGE_FILES) -tests/vm/%.img: $(SRC_PATH)/tests/vm/% \ - $(SRC_PATH)/tests/vm/basevm.py \ - $(SRC_PATH)/tests/vm/Makefile.include +$(IMAGES_DIR)/%.img: $(SRC_PATH)/tests/vm/% \ + $(SRC_PATH)/tests/vm/basevm.py \ + $(SRC_PATH)/tests/vm/Makefile.include + @mkdir -p $(IMAGES_DIR) $(call quiet-command, \ $< \ $(if $(V)$(DEBUG), --debug) \ @@ -37,7 +39,7 @@ tests/vm/%.img: $(SRC_PATH)/tests/vm/% \ # Build in VM $(IMAGE) -vm-build-%: tests/vm/%.img +vm-build-%: $(IMAGES_DIR)/%.img $(call quiet-command, \ $(SRC_PATH)/tests/vm/$* \ $(if $(V)$(DEBUG), --debug) \ -- cgit v1.1 From a4cefbcd9bc8403c4de8847d422451c2e85aa3f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 21 Jan 2019 16:34:35 +0000 Subject: tests/vm: call make check directly for netbsd/freebsd/ubuntu.i386 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "make check" target calls check-qtest which has the appropriate system binaries as dependencies so we shouldn't need to do two steps of make invocation. Doing it in two steps was a hangover from when our make check couldn't run tests in parallel. Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé --- tests/vm/freebsd | 1 - tests/vm/netbsd | 1 - tests/vm/ubuntu.i386 | 1 - 3 files changed, 3 deletions(-) (limited to 'tests') diff --git a/tests/vm/freebsd b/tests/vm/freebsd index 19a3729..a85c866 100755 --- a/tests/vm/freebsd +++ b/tests/vm/freebsd @@ -25,7 +25,6 @@ class FreeBSDVM(basevm.BaseVM): cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); tar -xf /dev/vtbd1; ./configure {configure_opts}; - gmake --output-sync -j{jobs} {verbose}; gmake --output-sync -j{jobs} check {verbose}; """ diff --git a/tests/vm/netbsd b/tests/vm/netbsd index fac6a7c..edea113 100755 --- a/tests/vm/netbsd +++ b/tests/vm/netbsd @@ -25,7 +25,6 @@ class NetBSDVM(basevm.BaseVM): cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); tar -xf /dev/rld1a; ./configure --python=python2.7 {configure_opts}; - gmake --output-sync -j{jobs} {verbose}; gmake --output-sync -j{jobs} check {verbose}; """ diff --git a/tests/vm/ubuntu.i386 b/tests/vm/ubuntu.i386 index 1b7e1ab..252e514 100755 --- a/tests/vm/ubuntu.i386 +++ b/tests/vm/ubuntu.i386 @@ -26,7 +26,6 @@ class UbuntuX86VM(basevm.BaseVM): sudo chmod a+r /dev/vdb; tar -xf /dev/vdb; ./configure {configure_opts}; - make --output-sync -j{jobs}; make --output-sync check -j{jobs} {verbose}; """ -- cgit v1.1 From 5c2ec9b61f33c18ef734e98a42d28ba99775b20c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 21 Jan 2019 16:48:15 +0000 Subject: tests/vm: add --build-target option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows us to invoke the build with a custom target (for the VMs that use the {target} format string specifier). Currently OpenBSD is still hardwired due to problems running check. Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé --- tests/vm/basevm.py | 3 +++ tests/vm/freebsd | 2 +- tests/vm/netbsd | 2 +- tests/vm/ubuntu.i386 | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py index 5caf77d..bdca6cb 100755 --- a/tests/vm/basevm.py +++ b/tests/vm/basevm.py @@ -228,6 +228,8 @@ def parse_args(vmcls): help="build image") parser.add_option("--build-qemu", help="build QEMU from source in guest") + parser.add_option("--build-target", + help="QEMU build target", default="check") parser.add_option("--interactive", "-I", action="store_true", help="Interactively run command") parser.add_option("--snapshot", "-s", action="store_true", @@ -255,6 +257,7 @@ def main(vmcls): cmd = [vm.BUILD_SCRIPT.format( configure_opts = " ".join(argv), jobs=args.jobs, + target=args.build_target, verbose = "V=1" if args.verbose else "")] else: cmd = argv diff --git a/tests/vm/freebsd b/tests/vm/freebsd index a85c866..1d64c31 100755 --- a/tests/vm/freebsd +++ b/tests/vm/freebsd @@ -25,7 +25,7 @@ class FreeBSDVM(basevm.BaseVM): cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); tar -xf /dev/vtbd1; ./configure {configure_opts}; - gmake --output-sync -j{jobs} check {verbose}; + gmake --output-sync -j{jobs} {target} {verbose}; """ def build_image(self, img): diff --git a/tests/vm/netbsd b/tests/vm/netbsd index edea113..a2c3d13 100755 --- a/tests/vm/netbsd +++ b/tests/vm/netbsd @@ -25,7 +25,7 @@ class NetBSDVM(basevm.BaseVM): cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); tar -xf /dev/rld1a; ./configure --python=python2.7 {configure_opts}; - gmake --output-sync -j{jobs} check {verbose}; + gmake --output-sync -j{jobs} {target} {verbose}; """ def build_image(self, img): diff --git a/tests/vm/ubuntu.i386 b/tests/vm/ubuntu.i386 index 252e514..a22d137 100755 --- a/tests/vm/ubuntu.i386 +++ b/tests/vm/ubuntu.i386 @@ -26,7 +26,7 @@ class UbuntuX86VM(basevm.BaseVM): sudo chmod a+r /dev/vdb; tar -xf /dev/vdb; ./configure {configure_opts}; - make --output-sync check -j{jobs} {verbose}; + make --output-sync {target} -j{jobs} {verbose}; """ def _gen_cloud_init_iso(self): -- cgit v1.1 From ddafa31fae2a6c8f884254373e550d3ca33a4945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 21 Jan 2019 16:54:42 +0000 Subject: tests/vm: expose BUILD_TARGET, TARGET_LIST and EXTRA_CONFIGURE_OPTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now the underlying basevm support passes these along we can expose some additional variables to our Makefile to allow more customised tweaking of the build. For example: make vm-build-freebsd TARGET_LIST=aarch64-softmmu \ EXTRA_CONFIGURE_OPTS="--disable-tools --disable-docs" \ BUILD_TARGET=check-softfloat Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé --- tests/vm/Makefile.include | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index a58383d..992d823 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -19,6 +19,11 @@ vm-test: @echo "" @echo " vm-build-all - Build QEMU in all VMs" @echo " vm-clean-all - Clean up VM images" + @echo + @echo "Special variables:" + @echo " BUILD_TARGET=foo - override the build target" + @echo " TARGET_LIST=a,b,c - Override target list in builds." + @echo ' EXTRA_CONFIGURE_OPTS="..."' vm-build-all: $(addprefix vm-build-, $(IMAGES)) @@ -47,6 +52,9 @@ vm-build-%: $(IMAGES_DIR)/%.img $(if $(J),--jobs $(J)) \ $(if $(V),--verbose) \ --image "$<" \ - --build-qemu $(SRC_PATH), \ + $(if $(BUILD_TARGET),--build-target $(BUILD_TARGET)) \ + --build-qemu $(SRC_PATH) -- \ + $(if $(TARGET_LIST),--target-list=$(TARGET_LIST)) \ + $(if $(EXTRA_CONFIGURE_OPTS),$(EXTRA_CONFIGURE_OPTS)), \ " VM-BUILD $*") -- cgit v1.1 From 920fff9093db26f3e74858e3d4182b74a6da7496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 29 Jan 2019 18:53:46 +0100 Subject: tests/vm: Be verbose while extracting compressed images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Depending of the host hardware, copying and extracting VM images can take up to few minutes. Add verbosity to avoid the user to worry about VMs hanging. Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20190129175403.18017-2-philmd@redhat.com> Signed-off-by: Alex Bennée --- tests/vm/centos | 3 ++- tests/vm/freebsd | 3 ++- tests/vm/netbsd | 3 ++- tests/vm/openbsd | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/vm/centos b/tests/vm/centos index daa2dbc..ba133ea 100755 --- a/tests/vm/centos +++ b/tests/vm/centos @@ -65,8 +65,9 @@ class CentosVM(basevm.BaseVM): def build_image(self, img): cimg = self._download_with_cache("https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1802.qcow2.xz") img_tmp = img + ".tmp" + sys.stderr.write("Extracting the image...\n") subprocess.check_call(["cp", "-f", cimg, img_tmp + ".xz"]) - subprocess.check_call(["xz", "-df", img_tmp + ".xz"]) + subprocess.check_call(["xz", "-dvf", img_tmp + ".xz"]) subprocess.check_call(["qemu-img", "resize", img_tmp, "50G"]) self.boot(img_tmp, extra_args = ["-cdrom", self._gen_cloud_init_iso()]) self.wait_ssh() diff --git a/tests/vm/freebsd b/tests/vm/freebsd index 1d64c31..b006601 100755 --- a/tests/vm/freebsd +++ b/tests/vm/freebsd @@ -33,8 +33,9 @@ class FreeBSDVM(basevm.BaseVM): sha256sum='adcb771549b37bc63826c501f05121a206ed3d9f55f49145908f7e1432d65891') img_tmp_xz = img + ".tmp.xz" img_tmp = img + ".tmp" + sys.stderr.write("Extracting the image...\n") subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) - subprocess.check_call(["xz", "-df", img_tmp_xz]) + subprocess.check_call(["xz", "-dvf", img_tmp_xz]) if os.path.exists(img): os.remove(img) os.rename(img_tmp, img) diff --git a/tests/vm/netbsd b/tests/vm/netbsd index a2c3d13..4c6624e 100755 --- a/tests/vm/netbsd +++ b/tests/vm/netbsd @@ -33,8 +33,9 @@ class NetBSDVM(basevm.BaseVM): sha256sum='b633d565b0eac3d02015cd0c81440bd8a7a8df8512615ac1ee05d318be015732') img_tmp_xz = img + ".tmp.xz" img_tmp = img + ".tmp" + sys.stderr.write("Extracting the image...\n") subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) - subprocess.check_call(["xz", "-df", img_tmp_xz]) + subprocess.check_call(["xz", "-dvf", img_tmp_xz]) if os.path.exists(img): os.remove(img) os.rename(img_tmp, img) diff --git a/tests/vm/openbsd b/tests/vm/openbsd index cfe0572..2105c01 100755 --- a/tests/vm/openbsd +++ b/tests/vm/openbsd @@ -35,8 +35,9 @@ class OpenBSDVM(basevm.BaseVM): sha256sum='8c6cedc483e602cfee5e04f0406c64eb99138495e8ca580bc0293bcf0640c1bf') img_tmp_xz = img + ".tmp.xz" img_tmp = img + ".tmp" + sys.stderr.write("Extracting the image...\n") subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) - subprocess.check_call(["xz", "-df", img_tmp_xz]) + subprocess.check_call(["xz", "-dvf", img_tmp_xz]) if os.path.exists(img): os.remove(img) os.rename(img_tmp, img) -- cgit v1.1