From 2d110c11497ac52d5ce9f4b116463cdb8c3f4ad5 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 13 May 2020 23:52:30 -0400 Subject: python: remove more instances of sys.version_info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We guarantee 3.5+ everywhere; remove more dead checks. In general, try to avoid using version checks and instead prefer to attempt behavior when possible. Signed-off-by: John Snow Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20200514035230.25756-1-jsnow@redhat.com> Signed-off-by: Philippe Mathieu-Daudé --- tests/docker/docker.py | 5 +++-- tests/qemu-iotests/nbd-fault-injector.py | 5 +---- 2 files changed, 4 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/tests/docker/docker.py b/tests/docker/docker.py index d8268c1..5a9735d 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -258,12 +258,13 @@ class Docker(object): return self._do_kill_instances(True) def _output(self, cmd, **kwargs): - if sys.version_info[1] >= 6: + try: return subprocess.check_output(self._command + cmd, stderr=subprocess.STDOUT, encoding='utf-8', **kwargs) - else: + except TypeError: + # 'encoding' argument was added in 3.6+ return subprocess.check_output(self._command + cmd, stderr=subprocess.STDOUT, **kwargs).decode('utf-8') diff --git a/tests/qemu-iotests/nbd-fault-injector.py b/tests/qemu-iotests/nbd-fault-injector.py index 588d62a..78f42c4 100755 --- a/tests/qemu-iotests/nbd-fault-injector.py +++ b/tests/qemu-iotests/nbd-fault-injector.py @@ -47,10 +47,7 @@ import sys import socket import struct import collections -if sys.version_info.major >= 3: - import configparser -else: - import ConfigParser as configparser +import configparser FAKE_DISK_SIZE = 8 * 1024 * 1024 * 1024 # 8 GB -- cgit v1.1 From d5326a24378dbf228b5ea842945eff34ed9543a0 Mon Sep 17 00:00:00 2001 From: Robert Foley Date: Fri, 29 May 2020 16:34:50 -0400 Subject: tests/vm: Pass --debug through for vm-boot-ssh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This helps debug issues that occur during the boot sequence. Signed-off-by: Robert Foley Reviewed-by: Peter Puhov Reviewed-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Message-Id: <20200529203458.1038-5-robert.foley@linaro.org> Signed-off-by: Philippe Mathieu-Daudé --- tests/vm/Makefile.include | 1 + 1 file changed, 1 insertion(+) (limited to 'tests') diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index 74ab522..80f7f6b 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -91,6 +91,7 @@ vm-boot-ssh-%: $(IMAGES_DIR)/%.img $(call quiet-command, \ $(PYTHON) $(SRC_PATH)/tests/vm/$* \ $(if $(J),--jobs $(J)) \ + $(if $(V)$(DEBUG), --debug) \ --image "$<" \ --interactive \ false, \ -- cgit v1.1 From e56c45047bd5bbcfdc36e3f4ed8b439c5d5c989a Mon Sep 17 00:00:00 2001 From: Robert Foley Date: Fri, 29 May 2020 16:34:51 -0400 Subject: tests/vm: Add ability to select QEMU from current build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added a new special variable QEMU_LOCAL=1, which will indicate to take the QEMU binary from the current build. Signed-off-by: Robert Foley Reviewed-by: Peter Puhov Reviewed-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Message-Id: <20200529203458.1038-6-robert.foley@linaro.org> Signed-off-by: Philippe Mathieu-Daudé --- tests/vm/Makefile.include | 4 ++++ tests/vm/basevm.py | 28 +++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index 80f7f6b..a253aba 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -41,6 +41,7 @@ endif @echo " J=[0..9]* - Override the -jN parameter for make commands" @echo " DEBUG=1 - Enable verbose output on host and interactive debugging" @echo " V=1 - Enable verbose ouput on host and guest commands" + @echo " QEMU_LOCAL=1 - Use QEMU binary local to this build." @echo " QEMU=/path/to/qemu - Change path to QEMU binary" @echo " QEMU_IMG=/path/to/qemu-img - Change path to qemu-img tool" @@ -57,6 +58,7 @@ $(IMAGES_DIR)/%.img: $(SRC_PATH)/tests/vm/% \ $(PYTHON) $< \ $(if $(V)$(DEBUG), --debug) \ $(if $(GENISOIMAGE),--genisoimage $(GENISOIMAGE)) \ + $(if $(QEMU_LOCAL),--build-path $(BUILD_DIR)) \ --image "$@" \ --force \ --build-image $@, \ @@ -71,6 +73,7 @@ vm-build-%: $(IMAGES_DIR)/%.img $(if $(DEBUG), --interactive) \ $(if $(J),--jobs $(J)) \ $(if $(V),--verbose) \ + $(if $(QEMU_LOCAL),--build-path $(BUILD_DIR)) \ --image "$<" \ $(if $(BUILD_TARGET),--build-target $(BUILD_TARGET)) \ --snapshot \ @@ -92,6 +95,7 @@ vm-boot-ssh-%: $(IMAGES_DIR)/%.img $(PYTHON) $(SRC_PATH)/tests/vm/$* \ $(if $(J),--jobs $(J)) \ $(if $(V)$(DEBUG), --debug) \ + $(if $(QEMU_LOCAL),--build-path $(BUILD_DIR)) \ --image "$<" \ --interactive \ false, \ diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py index a2d4054..5a3ce42 100644 --- a/tests/vm/basevm.py +++ b/tests/vm/basevm.py @@ -61,9 +61,11 @@ class BaseVM(object): # 4 is arbitrary, but greater than 2, # since we found we need to wait more than twice as long. tcg_ssh_timeout_multiplier = 4 - def __init__(self, debug=False, vcpus=None, genisoimage=None): + def __init__(self, debug=False, vcpus=None, genisoimage=None, + build_path=None): self._guest = None self._genisoimage = genisoimage + self._build_path = build_path self._tmpdir = os.path.realpath(tempfile.mkdtemp(prefix="vm-test-", suffix=".tmp", dir=".")) @@ -184,15 +186,15 @@ class BaseVM(object): "-device", "virtio-blk,drive=drive0,bootindex=0"] args += self._data_args + extra_args logging.debug("QEMU args: %s", " ".join(args)) - qemu_bin = os.environ.get("QEMU", "qemu-system-" + self.arch) - guest = QEMUMachine(binary=qemu_bin, args=args) + qemu_path = get_qemu_path(self.arch, self._build_path) + guest = QEMUMachine(binary=qemu_path, args=args) guest.set_machine('pc') guest.set_console() try: guest.launch() except: logging.error("Failed to launch QEMU, command line:") - logging.error(" ".join([qemu_bin] + args)) + logging.error(" ".join([qemu_path] + args)) logging.error("Log:") logging.error(guest.get_log()) logging.error("QEMU version >= 2.10 is required") @@ -391,6 +393,19 @@ class BaseVM(object): return os.path.join(cidir, "cloud-init.iso") +def get_qemu_path(arch, build_path=None): + """Fetch the path to the qemu binary.""" + # If QEMU environment variable set, it takes precedence + if "QEMU" in os.environ: + qemu_path = os.environ["QEMU"] + elif build_path: + qemu_path = os.path.join(build_path, arch + "-softmmu") + qemu_path = os.path.join(qemu_path, "qemu-system-" + arch) + else: + # Default is to use system path for qemu. + qemu_path = "qemu-system-" + arch + return qemu_path + def parse_args(vmcls): def get_default_jobs(): @@ -421,6 +436,9 @@ def parse_args(vmcls): help="build QEMU from source in guest") parser.add_option("--build-target", help="QEMU build target", default="check") + parser.add_option("--build-path", default=None, + help="Path of build directory, "\ + "for using build tree QEMU binary. ") parser.add_option("--interactive", "-I", action="store_true", help="Interactively run command") parser.add_option("--snapshot", "-s", action="store_true", @@ -439,7 +457,7 @@ def main(vmcls): logging.basicConfig(level=(logging.DEBUG if args.debug else logging.WARN)) vm = vmcls(debug=args.debug, vcpus=args.jobs, - genisoimage=args.genisoimage) + genisoimage=args.genisoimage, build_path=args.build_path) if args.build_image: if os.path.exists(args.image) and not args.force: sys.stderr.writelines(["Image file exists: %s\n" % args.image, -- cgit v1.1 From 6ee982c9abc42e726f9e783ba67bbb7676a9f9b4 Mon Sep 17 00:00:00 2001 From: Robert Foley Date: Fri, 29 May 2020 16:34:52 -0400 Subject: tests/vm: allow wait_ssh() to specify command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows for waiting for completion of arbitrary commands. Signed-off-by: Robert Foley Reviewed-by: Peter Puhov Reviewed-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Message-Id: <20200529203458.1038-7-robert.foley@linaro.org> Signed-off-by: Philippe Mathieu-Daudé --- tests/vm/basevm.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py index 5a3ce42..a80b616 100644 --- a/tests/vm/basevm.py +++ b/tests/vm/basevm.py @@ -320,24 +320,24 @@ class BaseVM(object): def print_step(self, text): sys.stderr.write("### %s ...\n" % text) - def wait_ssh(self, wait_root=False, seconds=300): + def wait_ssh(self, wait_root=False, seconds=300, cmd="exit 0"): # Allow more time for VM to boot under TCG. if not kvm_available(self.arch): seconds *= self.tcg_ssh_timeout_multiplier starttime = datetime.datetime.now() endtime = starttime + datetime.timedelta(seconds=seconds) - guest_up = False + cmd_success = False while datetime.datetime.now() < endtime: - if wait_root and self.ssh_root("exit 0") == 0: - guest_up = True + if wait_root and self.ssh_root(cmd) == 0: + cmd_success = True break - elif self.ssh("exit 0") == 0: - guest_up = True + elif self.ssh(cmd) == 0: + cmd_success = True break seconds = (endtime - datetime.datetime.now()).total_seconds() logging.debug("%ds before timeout", seconds) time.sleep(1) - if not guest_up: + if not cmd_success: raise Exception("Timeout while waiting for guest ssh") def shutdown(self): -- cgit v1.1 From 83389e22c5aa48877b9e4f903ae4ec49a442df2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 12 May 2020 12:32:38 +0200 Subject: tests/migration/guestperf: Use Python 3 interpreter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: John Snow Reviewed-by: Kevin Wolf Message-Id: <20200512103238.7078-7-philmd@redhat.com> --- tests/migration/guestperf-batch.py | 2 +- tests/migration/guestperf-plot.py | 2 +- tests/migration/guestperf.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/migration/guestperf-batch.py b/tests/migration/guestperf-batch.py index cb150ce..f1e9009 100755 --- a/tests/migration/guestperf-batch.py +++ b/tests/migration/guestperf-batch.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # # Migration test batch comparison invokation # diff --git a/tests/migration/guestperf-plot.py b/tests/migration/guestperf-plot.py index d70bb7a..9071510 100755 --- a/tests/migration/guestperf-plot.py +++ b/tests/migration/guestperf-plot.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # # Migration test graph plotting command # diff --git a/tests/migration/guestperf.py b/tests/migration/guestperf.py index 99b027e..ba1c4bc 100755 --- a/tests/migration/guestperf.py +++ b/tests/migration/guestperf.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # # Migration test direct invokation command # -- cgit v1.1 From 2c9120a223e666d7171e965b8f8bbcd72620f566 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Thu, 28 May 2020 12:24:04 +0100 Subject: tests/acceptance/migration.py: Wait for both sides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the source finishes migration the destination will still be receiving the data sent by the source, so it might not have quite finished yet, so won't quite have reached 'completed'. This lead to occasional asserts in the next few checks. After the source has finished, check the destination as well. (We can't just switch to checking the destination, because it doesn't give a status until it has started receiving the migration). Reported-by: Alex Bennée Signed-off-by: Dr. David Alan Gilbert Tested-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20200528112404.121972-1-dgilbert@redhat.com> Signed-off-by: Philippe Mathieu-Daudé --- tests/acceptance/migration.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'tests') diff --git a/tests/acceptance/migration.py b/tests/acceptance/migration.py index 0365289..792639c 100644 --- a/tests/acceptance/migration.py +++ b/tests/acceptance/migration.py @@ -35,6 +35,10 @@ class Migration(Test): timeout=self.timeout, step=0.1, args=(src_vm,)) + wait.wait_for(self.migration_finished, + timeout=self.timeout, + step=0.1, + args=(dst_vm,)) self.assertEqual(src_vm.command('query-migrate')['status'], 'completed') self.assertEqual(dst_vm.command('query-migrate')['status'], 'completed') self.assertEqual(dst_vm.command('query-status')['status'], 'running') -- cgit v1.1 From a5ba86d423c2b071894d86c60487f2317c7ffb60 Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 29 May 2020 10:04:39 +0300 Subject: tests/acceptance: allow console interaction with specific VMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Console interaction in avocado scripts was possible only with single default VM. This patch modifies the function parameters to allow passing a specific VM as a parameter to interact with it. Signed-off-by: Pavel Dovgalyuk Reviewed-by: Willian Rampazzo Reviewed-by: Alex Bennée Tested-by: Philippe Mathieu-Daudé Message-Id: <159073587933.20809.5122618715976660635.stgit@pasha-ThinkPad-X280> Signed-off-by: Philippe Mathieu-Daudé --- tests/acceptance/avocado_qemu/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/tests/acceptance/avocado_qemu/__init__.py b/tests/acceptance/avocado_qemu/__init__.py index 59e7b4f..77d1c1d 100644 --- a/tests/acceptance/avocado_qemu/__init__.py +++ b/tests/acceptance/avocado_qemu/__init__.py @@ -69,13 +69,15 @@ def pick_default_qemu_bin(arch=None): def _console_interaction(test, success_message, failure_message, - send_string, keep_sending=False): + send_string, keep_sending=False, vm=None): assert not keep_sending or send_string - console = test.vm.console_socket.makefile() + if vm is None: + vm = test.vm + console = vm.console_socket.makefile() console_logger = logging.getLogger('console') while True: if send_string: - test.vm.console_socket.sendall(send_string.encode()) + vm.console_socket.sendall(send_string.encode()) if not keep_sending: send_string = None # send only once msg = console.readline().strip() @@ -115,7 +117,8 @@ def interrupt_interactive_console_until_pattern(test, success_message, _console_interaction(test, success_message, failure_message, interrupt_string, True) -def wait_for_console_pattern(test, success_message, failure_message=None): +def wait_for_console_pattern(test, success_message, failure_message=None, + vm=None): """ Waits for messages to appear on the console, while logging the content @@ -125,7 +128,7 @@ def wait_for_console_pattern(test, success_message, failure_message=None): :param success_message: if this message appears, test succeeds :param failure_message: if this message appears, test fails """ - _console_interaction(test, success_message, failure_message, None) + _console_interaction(test, success_message, failure_message, None, vm=vm) def exec_command_and_wait_for_pattern(test, command, success_message, failure_message=None): -- cgit v1.1 From 12121c496fcc609e23033c4a36399b54f98bcd56 Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 29 May 2020 10:04:44 +0300 Subject: tests/acceptance: refactor boot_linux_console test to allow code reuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch splits code in BootLinuxConsole class into two different classes to allow reusing it by record/replay tests. Signed-off-by: Pavel Dovgalyuk Reviewed-by: Alex Bennée Tested-by: Philippe Mathieu-Daudé Message-Id: <159073588490.20809.13942096070255577558.stgit@pasha-ThinkPad-X280> Signed-off-by: Philippe Mathieu-Daudé --- tests/acceptance/boot_linux_console.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'tests') diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py index c6b06a1..12725d4 100644 --- a/tests/acceptance/boot_linux_console.py +++ b/tests/acceptance/boot_linux_console.py @@ -28,19 +28,13 @@ try: except CmdNotFoundError: P7ZIP_AVAILABLE = False -class BootLinuxConsole(Test): - """ - Boots a Linux kernel and checks that the console is operational and the - kernel command line is properly passed from QEMU to the kernel - """ - - timeout = 90 - +class LinuxKernelTest(Test): KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' - def wait_for_console_pattern(self, success_message): + def wait_for_console_pattern(self, success_message, vm=None): wait_for_console_pattern(self, success_message, - failure_message='Kernel panic - not syncing') + failure_message='Kernel panic - not syncing', + vm=vm) def extract_from_deb(self, deb, path): """ @@ -79,6 +73,13 @@ class BootLinuxConsole(Test): os.chdir(cwd) return os.path.normpath(os.path.join(self.workdir, path)) +class BootLinuxConsole(LinuxKernelTest): + """ + Boots a Linux kernel and checks that the console is operational and the + kernel command line is properly passed from QEMU to the kernel + """ + timeout = 90 + def test_x86_64_pc(self): """ :avocado: tags=arch:x86_64 -- cgit v1.1 From 1c80c87c8c2489e4318c93c844aa29bc1d014146 Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 29 May 2020 10:05:31 +0300 Subject: tests/acceptance: refactor boot_linux to allow code reuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch moves image downloading functions to the separate class to allow reusing them from record/replay tests. Signed-off-by: Pavel Dovgalyuk Tested-by: Philippe Mathieu-Daudé Message-Id: <159073593167.20809.17582679291556188984.stgit@pasha-ThinkPad-X280> Signed-off-by: Philippe Mathieu-Daudé --- tests/acceptance/boot_linux.py | 49 +++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 20 deletions(-) (limited to 'tests') diff --git a/tests/acceptance/boot_linux.py b/tests/acceptance/boot_linux.py index 075a386..3aa57e8 100644 --- a/tests/acceptance/boot_linux.py +++ b/tests/acceptance/boot_linux.py @@ -26,22 +26,8 @@ KVM_NOT_AVAILABLE = ACCEL_NOT_AVAILABLE_FMT % "KVM" TCG_NOT_AVAILABLE = ACCEL_NOT_AVAILABLE_FMT % "TCG" -class BootLinux(Test): - """ - Boots a Linux system, checking for a successful initialization - """ - - timeout = 900 - chksum = None - - def setUp(self): - super(BootLinux, self).setUp() - self.vm.add_args('-smp', '2') - self.vm.add_args('-m', '1024') - self.prepare_boot() - self.prepare_cloudinit() - - def prepare_boot(self): +class BootLinuxBase(Test): + def download_boot(self): self.log.debug('Looking for and selecting a qemu-img binary to be ' 'used to create the bootable snapshot image') # If qemu-img has been built, use it, otherwise the system wide one @@ -60,17 +46,17 @@ class BootLinux(Test): if image_arch == 'ppc64': image_arch = 'ppc64le' try: - self.boot = vmimage.get( + boot = vmimage.get( 'fedora', arch=image_arch, version='31', checksum=self.chksum, algorithm='sha256', cache_dir=self.cache_dirs[0], snapshot_dir=self.workdir) - self.vm.add_args('-drive', 'file=%s' % self.boot.path) except: self.cancel('Failed to download/prepare boot image') + return boot.path - def prepare_cloudinit(self): + def download_cloudinit(self): self.log.info('Preparing cloudinit image') try: cloudinit_iso = os.path.join(self.workdir, 'cloudinit.iso') @@ -81,9 +67,32 @@ class BootLinux(Test): # QEMU's hard coded usermode router address phone_home_host='10.0.2.2', phone_home_port=self.phone_home_port) - self.vm.add_args('-drive', 'file=%s,format=raw' % cloudinit_iso) except Exception: self.cancel('Failed to prepared cloudinit image') + return cloudinit_iso + +class BootLinux(BootLinuxBase): + """ + Boots a Linux system, checking for a successful initialization + """ + + timeout = 900 + chksum = None + + def setUp(self): + super(BootLinux, self).setUp() + self.vm.add_args('-smp', '2') + self.vm.add_args('-m', '1024') + self.prepare_boot() + self.prepare_cloudinit() + + def prepare_boot(self): + path = self.download_boot() + self.vm.add_args('-drive', 'file=%s' % path) + + def prepare_cloudinit(self): + cloudinit_iso = self.download_cloudinit() + self.vm.add_args('-drive', 'file=%s,format=raw' % cloudinit_iso) def launch_and_wait(self): self.vm.set_console() -- cgit v1.1