From 6a96d87cf40ddcd7b0e7ff398e8d2cd03c9be58d Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:01 -0400 Subject: iotests: do a light delinting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This doesn't fix everything in here, but it does help clean up the pylint report considerably. This should be 100% style changes only; the intent is to make pylint more useful by working on establishing a baseline for iotests that we can gate against in the future. Signed-off-by: John Snow Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-2-jsnow@redhat.com> Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/iotests.py | 83 ++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 40 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 5f8c263..6f6363f 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -16,11 +16,9 @@ # along with this program. If not, see . # -import errno import os import re import subprocess -import string import unittest import sys import struct @@ -35,7 +33,7 @@ import faulthandler sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu import qtest -assert sys.version_info >= (3,6) +assert sys.version_info >= (3, 6) faulthandler.enable() @@ -141,11 +139,11 @@ def qemu_img_log(*args): return result def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]): - args = [ 'info' ] + args = ['info'] if imgopts: args.append('--image-opts') else: - args += [ '-f', imgfmt ] + args += ['-f', imgfmt] args += extra_args args.append(filename) @@ -224,7 +222,7 @@ class QemuIoInteractive: # quit command is in close(), '\n' is added automatically assert '\n' not in cmd cmd = cmd.strip() - assert cmd != 'q' and cmd != 'quit' + assert cmd not in ('q', 'quit') self._p.stdin.write(cmd + '\n') self._p.stdin.flush() return self._read_output() @@ -246,10 +244,8 @@ def qemu_nbd_early_pipe(*args): sys.stderr.write('qemu-nbd received signal %i: %s\n' % (-exitcode, ' '.join(qemu_nbd_args + ['--fork'] + list(args)))) - if exitcode == 0: - return exitcode, '' - else: - return exitcode, subp.communicate()[0] + + return exitcode, subp.communicate()[0] if exitcode else '' def qemu_nbd_popen(*args): '''Run qemu-nbd in daemon mode and return the parent's exit code''' @@ -313,7 +309,7 @@ def filter_qmp(qmsg, filter_fn): items = qmsg.items() for k, v in items: - if isinstance(v, list) or isinstance(v, dict): + if isinstance(v, (dict, list)): qmsg[k] = filter_qmp(v, filter_fn) else: qmsg[k] = filter_fn(k, v) @@ -324,7 +320,7 @@ def filter_testfiles(msg): return msg.replace(prefix, 'TEST_DIR/PID-') def filter_qmp_testfiles(qmsg): - def _filter(key, value): + def _filter(_key, value): if is_str(value): return filter_testfiles(value) return value @@ -351,7 +347,7 @@ def filter_imgfmt(msg): return msg.replace(imgfmt, 'IMGFMT') def filter_qmp_imgfmt(qmsg): - def _filter(key, value): + def _filter(_key, value): if is_str(value): return filter_imgfmt(value) return value @@ -362,7 +358,7 @@ def log(msg, filters=[], indent=None): If indent is provided, JSON serializable messages are pretty-printed.''' for flt in filters: msg = flt(msg) - if isinstance(msg, dict) or isinstance(msg, list): + if isinstance(msg, (dict, list)): # Python < 3.4 needs to know not to add whitespace when pretty-printing: separators = (', ', ': ') if indent is None else (',', ': ') # Don't sort if it's already sorted @@ -373,14 +369,14 @@ def log(msg, filters=[], indent=None): print(msg) class Timeout: - def __init__(self, seconds, errmsg = "Timeout"): + def __init__(self, seconds, errmsg="Timeout"): self.seconds = seconds self.errmsg = errmsg def __enter__(self): signal.signal(signal.SIGALRM, self.timeout) signal.setitimer(signal.ITIMER_REAL, self.seconds) return self - def __exit__(self, type, value, traceback): + def __exit__(self, exc_type, value, traceback): signal.setitimer(signal.ITIMER_REAL, 0) return False def timeout(self, signum, frame): @@ -389,7 +385,7 @@ class Timeout: def file_pattern(name): return "{0}-{1}".format(os.getpid(), name) -class FilePaths(object): +class FilePaths: """ FilePaths is an auto-generated filename that cleans itself up. @@ -536,11 +532,11 @@ class VM(qtest.QEMUQtestMachine): self.pause_drive(drive, "write_aio") return self.qmp('human-monitor-command', - command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive)) + command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive)) def resume_drive(self, drive): self.qmp('human-monitor-command', - command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive)) + command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive)) def hmp_qemu_io(self, drive, cmd): '''Write to a given drive using an HMP command''' @@ -551,8 +547,8 @@ class VM(qtest.QEMUQtestMachine): if output is None: output = dict() if isinstance(obj, list): - for i in range(len(obj)): - self.flatten_qmp_object(obj[i], output, basestr + str(i) + '.') + for i, item in enumerate(obj): + self.flatten_qmp_object(item, output, basestr + str(i) + '.') elif isinstance(obj, dict): for key in obj: self.flatten_qmp_object(obj[key], output, basestr + key + '.') @@ -710,9 +706,7 @@ class VM(qtest.QEMUQtestMachine): for bitmap in bitmaps[node_name]: if bitmap.get('name', '') == bitmap_name: - if recording is None: - return bitmap - elif bitmap.get('recording') == recording: + if recording is None or bitmap.get('recording') == recording: return bitmap return None @@ -763,12 +757,13 @@ class VM(qtest.QEMUQtestMachine): assert node is not None, 'Cannot follow path %s%s' % (root, path) try: - node_id = next(edge['child'] for edge in graph['edges'] \ - if edge['parent'] == node['id'] and - edge['name'] == child_name) + node_id = next(edge['child'] for edge in graph['edges'] + if (edge['parent'] == node['id'] and + edge['name'] == child_name)) + + node = next(node for node in graph['nodes'] + if node['id'] == node_id) - node = next(node for node in graph['nodes'] \ - if node['id'] == node_id) except StopIteration: node = None @@ -786,6 +781,12 @@ index_re = re.compile(r'([^\[]+)\[([^\]]+)\]') class QMPTestCase(unittest.TestCase): '''Abstract base class for QMP test cases''' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Many users of this class set a VM property we rely on heavily + # in the methods below. + self.vm = None + def dictpath(self, d, path): '''Traverse a path in a nested dict''' for component in path.split('/'): @@ -831,7 +832,7 @@ class QMPTestCase(unittest.TestCase): else: self.assertEqual(result, value, '"%s" is "%s", expected "%s"' - % (path, str(result), str(value))) + % (path, str(result), str(value))) def assert_no_active_block_jobs(self): result = self.vm.qmp('query-block-jobs') @@ -841,15 +842,15 @@ class QMPTestCase(unittest.TestCase): """Issue a query-named-block-nodes and assert node_name and/or file_name is present in the result""" def check_equal_or_none(a, b): - return a == None or b == None or a == b + return a is None or b is None or a == b assert node_name or file_name result = self.vm.qmp('query-named-block-nodes') for x in result["return"]: if check_equal_or_none(x.get("node-name"), node_name) and \ check_equal_or_none(x.get("file"), file_name): return - self.assertTrue(False, "Cannot find %s %s in result:\n%s" % \ - (node_name, file_name, result)) + self.fail("Cannot find %s %s in result:\n%s" % + (node_name, file_name, result)) def assert_json_filename_equal(self, json_filename, reference): '''Asserts that the given filename is a json: filename and that its @@ -898,13 +899,13 @@ class QMPTestCase(unittest.TestCase): self.assert_qmp(event, 'data/error', error) self.assert_no_active_block_jobs() return event - elif event['event'] == 'JOB_STATUS_CHANGE': + if event['event'] == 'JOB_STATUS_CHANGE': self.assert_qmp(event, 'data/id', drive) def wait_ready(self, drive='drive0'): - '''Wait until a block job BLOCK_JOB_READY event''' - f = {'data': {'type': 'mirror', 'device': drive } } - event = self.vm.event_wait(name='BLOCK_JOB_READY', match=f) + """Wait until a BLOCK_JOB_READY event, and return the event.""" + f = {'data': {'type': 'mirror', 'device': drive}} + return self.vm.event_wait(name='BLOCK_JOB_READY', match=f) def wait_ready_and_cancel(self, drive='drive0'): self.wait_ready(drive=drive) @@ -933,7 +934,7 @@ class QMPTestCase(unittest.TestCase): for job in result['return']: if job['device'] == job_id: found = True - if job['paused'] == True and job['busy'] == False: + if job['paused'] and not job['busy']: return job break assert found @@ -1030,8 +1031,8 @@ def qemu_pipe(*args): universal_newlines=True) exitcode = subp.wait() if exitcode < 0: - sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode, - ' '.join(args))) + sys.stderr.write('qemu received signal %i: %s\n' % + (-exitcode, ' '.join(args))) return subp.communicate()[0] def supported_formats(read_only=False): @@ -1063,6 +1064,7 @@ def skip_if_unsupported(required_formats=[], read_only=False): if usf_list: test_case.case_skip('{}: formats {} are not whitelisted'.format( test_case, usf_list)) + return None else: return func(test_case, *args, **kwargs) return func_wrapper @@ -1074,6 +1076,7 @@ def skip_if_user_is_root(func): def func_wrapper(*args, **kwargs): if os.getuid() == 0: case_notrun('{}: cannot be run as root'.format(args[0])) + return None else: return func(*args, **kwargs) return func_wrapper -- cgit v1.1 From 1d3d4b630c6ea8b19420c097f0c448b6ded95072 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:02 -0400 Subject: iotests: don't use 'format' for drive_add MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It shadows (with a different type) the built-in format. Use something else. Signed-off-by: John Snow Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-3-jsnow@redhat.com> Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/055 | 3 ++- tests/qemu-iotests/iotests.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/055 b/tests/qemu-iotests/055 index 82b9f5f..4175fff 100755 --- a/tests/qemu-iotests/055 +++ b/tests/qemu-iotests/055 @@ -469,7 +469,8 @@ class TestDriveCompression(iotests.QMPTestCase): qemu_img('create', '-f', fmt, blockdev_target_img, str(TestDriveCompression.image_len), *args) if attach_target: - self.vm.add_drive(blockdev_target_img, format=fmt, interface="none") + self.vm.add_drive(blockdev_target_img, + img_format=fmt, interface="none") self.vm.launch() diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 6f6363f..aca7d50 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -486,21 +486,21 @@ class VM(qtest.QEMUQtestMachine): self._args.append(opts) return self - def add_drive(self, path, opts='', interface='virtio', format=imgfmt): + def add_drive(self, path, opts='', interface='virtio', img_format=imgfmt): '''Add a virtio-blk drive to the VM''' options = ['if=%s' % interface, 'id=drive%d' % self._num_drives] if path is not None: options.append('file=%s' % path) - options.append('format=%s' % format) + options.append('format=%s' % img_format) options.append('cache=%s' % cachemode) options.append('aio=%s' % aiomode) if opts: options.append(opts) - if format == 'luks' and 'key-secret' not in opts: + if img_format == 'luks' and 'key-secret' not in opts: # default luks support if luks_default_secret_object not in self._args: self.add_object(luks_default_secret_object) -- cgit v1.1 From 368e06200320fd1c046788832531c0148ccbdf11 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:03 -0400 Subject: iotests: ignore import warnings from pylint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The right way to solve this is to come up with a virtual environment infrastructure that sets all the paths correctly, and/or to create installable python modules that can be imported normally. That's hard, so just silence this error for now. Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Max Reitz Signed-off-by: John Snow Message-Id: <20200331000014.11581-4-jsnow@redhat.com> Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/iotests.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tests') diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index aca7d50..37a32c7 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -30,6 +30,7 @@ import io from collections import OrderedDict import faulthandler +# pylint: disable=import-error, wrong-import-position sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu import qtest -- cgit v1.1 From 4eabe0515ad7947da566fa52f9bd29e6fc1e6dcc Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:04 -0400 Subject: iotests: replace mutable list default args MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's bad hygiene: if we modify this list, it will be modified across all invocations. (Remaining bad usages are fixed in a subsequent patch which changes the function signature anyway.) Signed-off-by: John Snow Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-5-jsnow@redhat.com> Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/iotests.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 37a32c7..d8dc602 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -139,7 +139,7 @@ def qemu_img_log(*args): log(result, filters=[filter_testfiles]) return result -def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]): +def img_info_log(filename, filter_path=None, imgopts=False, extra_args=()): args = ['info'] if imgopts: args.append('--image-opts') @@ -354,7 +354,7 @@ def filter_qmp_imgfmt(qmsg): return value return filter_qmp(qmsg, _filter) -def log(msg, filters=[], indent=None): +def log(msg, filters=(), indent=None): '''Logs either a string message or a JSON serializable message (like QMP). If indent is provided, JSON serializable messages are pretty-printed.''' for flt in filters: @@ -570,7 +570,7 @@ class VM(qtest.QEMUQtestMachine): result.append(filter_qmp_event(ev)) return result - def qmp_log(self, cmd, filters=[], indent=None, **kwargs): + def qmp_log(self, cmd, filters=(), indent=None, **kwargs): full_cmd = OrderedDict(( ("execute", cmd), ("arguments", ordered_qmp(kwargs)) @@ -974,7 +974,7 @@ def case_notrun(reason): open('%s/%s.casenotrun' % (output_dir, seq), 'a').write( ' [case not run] ' + reason + '\n') -def verify_image_format(supported_fmts=[], unsupported_fmts=[]): +def verify_image_format(supported_fmts=(), unsupported_fmts=()): assert not (supported_fmts and unsupported_fmts) if 'generic' in supported_fmts and \ @@ -988,7 +988,7 @@ def verify_image_format(supported_fmts=[], unsupported_fmts=[]): if not_sup or (imgfmt in unsupported_fmts): notrun('not suitable for this image format: %s' % imgfmt) -def verify_protocol(supported=[], unsupported=[]): +def verify_protocol(supported=(), unsupported=()): assert not (supported and unsupported) if 'generic' in supported: @@ -1007,11 +1007,11 @@ def verify_platform(supported=None, unsupported=None): if not any((sys.platform.startswith(x) for x in supported)): notrun('not suitable for this OS: %s' % sys.platform) -def verify_cache_mode(supported_cache_modes=[]): +def verify_cache_mode(supported_cache_modes=()): if supported_cache_modes and (cachemode not in supported_cache_modes): notrun('not suitable for this cache mode: %s' % cachemode) -def verify_aio_mode(supported_aio_modes=[]): +def verify_aio_mode(supported_aio_modes=()): if supported_aio_modes and (aiomode not in supported_aio_modes): notrun('not suitable for this aio mode: %s' % aiomode) @@ -1051,7 +1051,7 @@ def supported_formats(read_only=False): return supported_formats.formats[read_only] -def skip_if_unsupported(required_formats=[], read_only=False): +def skip_if_unsupported(required_formats=(), read_only=False): '''Skip Test Decorator Runs the test if all the required formats are whitelisted''' def skip_test_decorator(func): @@ -1102,11 +1102,11 @@ def execute_unittest(output, verbosity, debug): sys.stderr.write(out) def execute_test(test_function=None, - supported_fmts=[], + supported_fmts=(), supported_platforms=None, - supported_cache_modes=[], supported_aio_modes={}, - unsupported_fmts=[], supported_protocols=[], - unsupported_protocols=[]): + supported_cache_modes=(), supported_aio_modes=(), + unsupported_fmts=(), supported_protocols=(), + unsupported_protocols=()): """Run either unittest or script-style tests.""" # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to -- cgit v1.1 From 576dc22be11d3cd95eec0e75ec5603cde671a370 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:05 -0400 Subject: iotests: add pylintrc file This allows others to get repeatable results with pylint. If you run `pylint iotests.py`, you should see a 100% pass. Signed-off-by: John Snow Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-6-jsnow@redhat.com> Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/pylintrc | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tests/qemu-iotests/pylintrc (limited to 'tests') diff --git a/tests/qemu-iotests/pylintrc b/tests/qemu-iotests/pylintrc new file mode 100644 index 0000000..daec2c4 --- /dev/null +++ b/tests/qemu-iotests/pylintrc @@ -0,0 +1,22 @@ +[MESSAGES CONTROL] + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=invalid-name, + no-else-return, + too-few-public-methods, + too-many-arguments, + too-many-branches, + too-many-lines, + too-many-locals, + too-many-public-methods, + # These are temporary, and should be removed: + line-too-long, + missing-docstring, -- cgit v1.1 From b404b13bd519301c99e603ac660122158ea5b93b Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:06 -0400 Subject: iotests: alphabetize standard imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I had to fix a merge conflict, so do this tiny harmless thing while I'm here. Signed-off-by: John Snow Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-7-jsnow@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/iotests.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index d8dc602..dae1248 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -16,19 +16,19 @@ # along with this program. If not, see . # +import atexit +from collections import OrderedDict +import faulthandler +import io +import json +import logging import os import re +import signal +import struct import subprocess -import unittest import sys -import struct -import json -import signal -import logging -import atexit -import io -from collections import OrderedDict -import faulthandler +import unittest # pylint: disable=import-error, wrong-import-position sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -- cgit v1.1 From 229fc0742a727c1733dd4d0c405c4c590f8dc627 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:07 -0400 Subject: iotests: drop pre-Python 3.4 compatibility code We no longer need to accommodate <3.4, drop this code. (The lines were > 79 chars and it stood out.) Signed-off-by: John Snow Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-8-jsnow@redhat.com> Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/iotests.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index dae1248..374a8f6 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -360,12 +360,9 @@ def log(msg, filters=(), indent=None): for flt in filters: msg = flt(msg) if isinstance(msg, (dict, list)): - # Python < 3.4 needs to know not to add whitespace when pretty-printing: - separators = (', ', ': ') if indent is None else (',', ': ') # Don't sort if it's already sorted do_sort = not isinstance(msg, OrderedDict) - print(json.dumps(msg, sort_keys=do_sort, - indent=indent, separators=separators)) + print(json.dumps(msg, sort_keys=do_sort, indent=indent)) else: print(msg) -- cgit v1.1 From 1cd0dbfc12019fcd5f0529094a939a1557dfc2df Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:08 -0400 Subject: iotests: touch up log function signature Representing nested, recursive data structures in mypy is notoriously difficult; the best we can reliably do right now is denote the leaf types as "Any" while describing the general shape of the data. Regardless, this fully annotates the log() function. Typing notes: TypeVar is a Type variable that can optionally be constrained by a sequence of possible types. This variable is bound to a specific type per-invocation, like a Generic. log() behaves as log() now, where the incoming type informs the signature it expects for any filter arguments passed in. If Msg is a str, then filter should take and return a str. Signed-off-by: John Snow Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-9-jsnow@redhat.com> Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/iotests.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 374a8f6..69f2422 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -28,6 +28,7 @@ import signal import struct import subprocess import sys +from typing import (Any, Callable, Dict, Iterable, List, Optional, TypeVar) import unittest # pylint: disable=import-error, wrong-import-position @@ -354,9 +355,16 @@ def filter_qmp_imgfmt(qmsg): return value return filter_qmp(qmsg, _filter) -def log(msg, filters=(), indent=None): - '''Logs either a string message or a JSON serializable message (like QMP). - If indent is provided, JSON serializable messages are pretty-printed.''' + +Msg = TypeVar('Msg', Dict[str, Any], List[Any], str) + +def log(msg: Msg, + filters: Iterable[Callable[[Msg], Msg]] = (), + indent: Optional[int] = None) -> None: + """ + Logs either a string message or a JSON serializable message (like QMP). + If indent is provided, JSON serializable messages are pretty-printed. + """ for flt in filters: msg = flt(msg) if isinstance(msg, (dict, list)): -- cgit v1.1 From b031e9a5a65ceef722ba506bc5ee69ec785dc5c5 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:09 -0400 Subject: iotests: limit line length to 79 chars 79 is the PEP8 recommendation. This recommendation works well for reading patch diffs in TUI email clients. Signed-off-by: John Snow Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-10-jsnow@redhat.com> Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/iotests.py | 64 ++++++++++++++++++++++++++++--------------- tests/qemu-iotests/pylintrc | 6 +++- 2 files changed, 47 insertions(+), 23 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 69f2422..9f5da32 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -80,9 +80,11 @@ luks_default_key_secret_opt = 'key-secret=keysec0' def qemu_img(*args): '''Run qemu-img and return the exit code''' devnull = open('/dev/null', 'r+') - exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull) + exitcode = subprocess.call(qemu_img_args + list(args), + stdin=devnull, stdout=devnull) if exitcode < 0: - sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) + sys.stderr.write('qemu-img received signal %i: %s\n' + % (-exitcode, ' '.join(qemu_img_args + list(args)))) return exitcode def ordered_qmp(qmsg, conv_keys=True): @@ -121,7 +123,8 @@ def qemu_img_verbose(*args): '''Run qemu-img without suppressing its output and return the exit code''' exitcode = subprocess.call(qemu_img_args + list(args)) if exitcode < 0: - sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) + sys.stderr.write('qemu-img received signal %i: %s\n' + % (-exitcode, ' '.join(qemu_img_args + list(args)))) return exitcode def qemu_img_pipe(*args): @@ -132,7 +135,8 @@ def qemu_img_pipe(*args): universal_newlines=True) exitcode = subp.wait() if exitcode < 0: - sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) + sys.stderr.write('qemu-img received signal %i: %s\n' + % (-exitcode, ' '.join(qemu_img_args + list(args)))) return subp.communicate()[0] def qemu_img_log(*args): @@ -162,7 +166,8 @@ def qemu_io(*args): universal_newlines=True) exitcode = subp.wait() if exitcode < 0: - sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args))) + sys.stderr.write('qemu-io received signal %i: %s\n' + % (-exitcode, ' '.join(args))) return subp.communicate()[0] def qemu_io_log(*args): @@ -284,10 +289,13 @@ win32_re = re.compile(r"\r") def filter_win32(msg): return win32_re.sub("", msg) -qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* \([0-9\/.inf]* [EPTGMKiBbytes]*\/sec and [0-9\/.inf]* ops\/sec\)") +qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* " + r"\([0-9\/.inf]* [EPTGMKiBbytes]*\/sec " + r"and [0-9\/.inf]* ops\/sec\)") def filter_qemu_io(msg): msg = filter_win32(msg) - return qemu_io_re.sub("X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)", msg) + return qemu_io_re.sub("X ops; XX:XX:XX.X " + "(XXX YYY/sec and XXX ops/sec)", msg) chown_re = re.compile(r"chown [0-9]+:[0-9]+") def filter_chown(msg): @@ -340,7 +348,9 @@ def filter_img_info(output, filename): line = filter_testfiles(line) line = line.replace(imgfmt, 'IMGFMT') line = re.sub('iters: [0-9]+', 'iters: XXX', line) - line = re.sub('uuid: [-a-f0-9]+', 'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', line) + line = re.sub('uuid: [-a-f0-9]+', + 'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', + line) line = re.sub('cid: [0-9]+', 'cid: XXXXXXXXXX', line) lines.append(line) return '\n'.join(lines) @@ -538,11 +548,13 @@ class VM(qtest.QEMUQtestMachine): self.pause_drive(drive, "write_aio") return self.qmp('human-monitor-command', - command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive)) + command_line='qemu-io %s "break %s bp_%s"' + % (drive, event, drive)) def resume_drive(self, drive): self.qmp('human-monitor-command', - command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive)) + command_line='qemu-io %s "remove_break bp_%s"' + % (drive, drive)) def hmp_qemu_io(self, drive, cmd): '''Write to a given drive using an HMP command''' @@ -802,16 +814,18 @@ class QMPTestCase(unittest.TestCase): idx = int(idx) if not isinstance(d, dict) or component not in d: - self.fail('failed path traversal for "%s" in "%s"' % (path, str(d))) + self.fail(f'failed path traversal for "{path}" in "{d}"') d = d[component] if m: if not isinstance(d, list): - self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d))) + self.fail(f'path component "{component}" in "{path}" ' + f'is not a list in "{d}"') try: d = d[idx] except IndexError: - self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d))) + self.fail(f'invalid index "{idx}" in path "{path}" ' + f'in "{d}"') return d def assert_qmp_absent(self, d, path): @@ -862,10 +876,13 @@ class QMPTestCase(unittest.TestCase): '''Asserts that the given filename is a json: filename and that its content is equal to the given reference object''' self.assertEqual(json_filename[:5], 'json:') - self.assertEqual(self.vm.flatten_qmp_object(json.loads(json_filename[5:])), - self.vm.flatten_qmp_object(reference)) + self.assertEqual( + self.vm.flatten_qmp_object(json.loads(json_filename[5:])), + self.vm.flatten_qmp_object(reference) + ) - def cancel_and_wait(self, drive='drive0', force=False, resume=False, wait=60.0): + def cancel_and_wait(self, drive='drive0', force=False, + resume=False, wait=60.0): '''Cancel a block job and wait for it to finish, returning the event''' result = self.vm.qmp('block-job-cancel', device=drive, force=force) self.assert_qmp(result, 'return', {}) @@ -889,8 +906,8 @@ class QMPTestCase(unittest.TestCase): self.assert_no_active_block_jobs() return result - def wait_until_completed(self, drive='drive0', check_offset=True, wait=60.0, - error=None): + def wait_until_completed(self, drive='drive0', check_offset=True, + wait=60.0, error=None): '''Wait for a block job to finish, returning the event''' while True: for event in self.vm.get_qmp_events(wait=wait): @@ -1029,8 +1046,11 @@ def verify_quorum(): notrun('quorum support missing') def qemu_pipe(*args): - '''Run qemu with an option to print something and exit (e.g. a help option), - and return its output''' + """ + Run qemu with an option to print something and exit (e.g. a help option). + + :return: QEMU's stdout output. + """ args = [qemu_prog] + qemu_opts + list(args) subp = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -1068,8 +1088,8 @@ def skip_if_unsupported(required_formats=(), read_only=False): usf_list = list(set(fmts) - set(supported_formats(read_only))) if usf_list: - test_case.case_skip('{}: formats {} are not whitelisted'.format( - test_case, usf_list)) + msg = f'{test_case}: formats {usf_list} are not whitelisted' + test_case.case_skip(msg) return None else: return func(test_case, *args, **kwargs) diff --git a/tests/qemu-iotests/pylintrc b/tests/qemu-iotests/pylintrc index daec2c4..5481afe 100644 --- a/tests/qemu-iotests/pylintrc +++ b/tests/qemu-iotests/pylintrc @@ -18,5 +18,9 @@ disable=invalid-name, too-many-locals, too-many-public-methods, # These are temporary, and should be removed: - line-too-long, missing-docstring, + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=79 -- cgit v1.1 From 239bbcc0ae6ba24733839150955bdf8402e9ec55 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:10 -0400 Subject: iotests: add hmp helper with logging Minor cleanup for HMP functions; helps with line length and consolidates HMP helpers through one implementation function. Although we are adding a universal toggle to turn QMP logging on or off, many existing callers to hmp functions don't expect that output to be logged, which causes quite a few changes in the test output. For now, offer a use_log parameter. Typing notes: QMPResponse is just an alias for Dict[str, Any]. It holds no special meanings and it is not a formal subtype of Dict[str, Any]. It is best thought of as a lexical synonym. We may well wish to add stricter subtypes in the future for certain shapes of data that are not formalized as Python objects, at which point we can simply retire the alias and allow mypy to more strictly check usages of the name. Signed-off-by: John Snow Message-Id: <20200331000014.11581-11-jsnow@redhat.com> Reviewed-by: Max Reitz Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/iotests.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 9f5da32..cf10c42 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -37,6 +37,10 @@ from qemu import qtest assert sys.version_info >= (3, 6) +# Type Aliases +QMPResponse = Dict[str, Any] + + faulthandler.enable() # This will not work if arguments contain spaces but is necessary if we @@ -541,25 +545,30 @@ class VM(qtest.QEMUQtestMachine): self._args.append(addr) return self - def pause_drive(self, drive, event=None): - '''Pause drive r/w operations''' + def hmp(self, command_line: str, use_log: bool = False) -> QMPResponse: + cmd = 'human-monitor-command' + kwargs = {'command-line': command_line} + if use_log: + return self.qmp_log(cmd, **kwargs) + else: + return self.qmp(cmd, **kwargs) + + def pause_drive(self, drive: str, event: Optional[str] = None) -> None: + """Pause drive r/w operations""" if not event: self.pause_drive(drive, "read_aio") self.pause_drive(drive, "write_aio") return - self.qmp('human-monitor-command', - command_line='qemu-io %s "break %s bp_%s"' - % (drive, event, drive)) - - def resume_drive(self, drive): - self.qmp('human-monitor-command', - command_line='qemu-io %s "remove_break bp_%s"' - % (drive, drive)) - - def hmp_qemu_io(self, drive, cmd): - '''Write to a given drive using an HMP command''' - return self.qmp('human-monitor-command', - command_line='qemu-io %s "%s"' % (drive, cmd)) + self.hmp(f'qemu-io {drive} "break {event} bp_{drive}"') + + def resume_drive(self, drive: str) -> None: + """Resume drive r/w operations""" + self.hmp(f'qemu-io {drive} "remove_break bp_{drive}"') + + def hmp_qemu_io(self, drive: str, cmd: str, + use_log: bool = False) -> QMPResponse: + """Write to a given drive using an HMP command""" + return self.hmp(f'qemu-io {drive} "{cmd}"', use_log=use_log) def flatten_qmp_object(self, obj, output=None, basestr=''): if output is None: -- cgit v1.1 From 7d8140595f1e131935ba1c98a55af7d066660707 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:11 -0400 Subject: iotests: add script_initialize Like script_main, but doesn't require a single point of entry. Replace all existing initialization sections with this drop-in replacement. This brings debug support to all existing script-style iotests. Signed-off-by: John Snow Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-12-jsnow@redhat.com> Reviewed-by: Kevin Wolf [mreitz: Give 274 the same treatment] Signed-off-by: Max Reitz --- tests/qemu-iotests/149 | 3 +- tests/qemu-iotests/194 | 4 +-- tests/qemu-iotests/202 | 4 +-- tests/qemu-iotests/203 | 4 +-- tests/qemu-iotests/206 | 2 +- tests/qemu-iotests/207 | 6 ++-- tests/qemu-iotests/208 | 2 +- tests/qemu-iotests/209 | 2 +- tests/qemu-iotests/210 | 6 ++-- tests/qemu-iotests/211 | 6 ++-- tests/qemu-iotests/212 | 6 ++-- tests/qemu-iotests/213 | 6 ++-- tests/qemu-iotests/216 | 4 +-- tests/qemu-iotests/218 | 2 +- tests/qemu-iotests/219 | 2 +- tests/qemu-iotests/222 | 7 ++-- tests/qemu-iotests/224 | 4 +-- tests/qemu-iotests/228 | 6 ++-- tests/qemu-iotests/234 | 4 +-- tests/qemu-iotests/235 | 4 +-- tests/qemu-iotests/236 | 2 +- tests/qemu-iotests/237 | 2 +- tests/qemu-iotests/238 | 2 ++ tests/qemu-iotests/242 | 2 +- tests/qemu-iotests/246 | 2 +- tests/qemu-iotests/248 | 2 +- tests/qemu-iotests/254 | 2 +- tests/qemu-iotests/255 | 2 +- tests/qemu-iotests/256 | 2 +- tests/qemu-iotests/258 | 7 ++-- tests/qemu-iotests/260 | 4 ++- tests/qemu-iotests/262 | 4 +-- tests/qemu-iotests/264 | 4 ++- tests/qemu-iotests/274 | 4 +-- tests/qemu-iotests/277 | 2 ++ tests/qemu-iotests/280 | 8 +++-- tests/qemu-iotests/283 | 4 ++- tests/qemu-iotests/iotests.py | 76 ++++++++++++++++++++++++++++--------------- 38 files changed, 132 insertions(+), 83 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/149 b/tests/qemu-iotests/149 index b4a21bf..852768f 100755 --- a/tests/qemu-iotests/149 +++ b/tests/qemu-iotests/149 @@ -382,8 +382,7 @@ def test_once(config, qemu_img=False): # Obviously we only work with the luks image format -iotests.verify_image_format(supported_fmts=['luks']) -iotests.verify_platform() +iotests.script_initialize(supported_fmts=['luks']) # We need sudo in order to run cryptsetup to create # dm-crypt devices. This is safe to use on any diff --git a/tests/qemu-iotests/194 b/tests/qemu-iotests/194 index 9dc1bd3..8b1f720 100755 --- a/tests/qemu-iotests/194 +++ b/tests/qemu-iotests/194 @@ -21,8 +21,8 @@ import iotests -iotests.verify_image_format(supported_fmts=['qcow2', 'qed', 'raw']) -iotests.verify_platform(['linux']) +iotests.script_initialize(supported_fmts=['qcow2', 'qed', 'raw'], + supported_platforms=['linux']) with iotests.FilePath('source.img') as source_img_path, \ iotests.FilePath('dest.img') as dest_img_path, \ diff --git a/tests/qemu-iotests/202 b/tests/qemu-iotests/202 index 920a868..e3900a4 100755 --- a/tests/qemu-iotests/202 +++ b/tests/qemu-iotests/202 @@ -24,8 +24,8 @@ import iotests -iotests.verify_image_format(supported_fmts=['qcow2']) -iotests.verify_platform(['linux']) +iotests.script_initialize(supported_fmts=['qcow2'], + supported_platforms=['linux']) with iotests.FilePath('disk0.img') as disk0_img_path, \ iotests.FilePath('disk1.img') as disk1_img_path, \ diff --git a/tests/qemu-iotests/203 b/tests/qemu-iotests/203 index 49eff5d..4b4bd33 100755 --- a/tests/qemu-iotests/203 +++ b/tests/qemu-iotests/203 @@ -24,8 +24,8 @@ import iotests -iotests.verify_image_format(supported_fmts=['qcow2']) -iotests.verify_platform(['linux']) +iotests.script_initialize(supported_fmts=['qcow2'], + supported_platforms=['linux']) with iotests.FilePath('disk0.img') as disk0_img_path, \ iotests.FilePath('disk1.img') as disk1_img_path, \ diff --git a/tests/qemu-iotests/206 b/tests/qemu-iotests/206 index e2b50ae2..f42432a 100755 --- a/tests/qemu-iotests/206 +++ b/tests/qemu-iotests/206 @@ -23,7 +23,7 @@ import iotests from iotests import imgfmt -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize(supported_fmts=['qcow2']) with iotests.FilePath('t.qcow2') as disk_path, \ iotests.FilePath('t.qcow2.base') as backing_path, \ diff --git a/tests/qemu-iotests/207 b/tests/qemu-iotests/207 index 3d9c120..a662141 100755 --- a/tests/qemu-iotests/207 +++ b/tests/qemu-iotests/207 @@ -24,8 +24,10 @@ import iotests import subprocess import re -iotests.verify_image_format(supported_fmts=['raw']) -iotests.verify_protocol(supported=['ssh']) +iotests.script_initialize( + supported_fmts=['raw'], + supported_protocols=['ssh'], +) def filter_hash(qmsg): def _filter(key, value): diff --git a/tests/qemu-iotests/208 b/tests/qemu-iotests/208 index 1c3fc8c..6cb642f 100755 --- a/tests/qemu-iotests/208 +++ b/tests/qemu-iotests/208 @@ -22,7 +22,7 @@ import iotests -iotests.verify_image_format(supported_fmts=['generic']) +iotests.script_initialize(supported_fmts=['generic']) with iotests.FilePath('disk.img') as disk_img_path, \ iotests.FilePath('disk-snapshot.img') as disk_snapshot_img_path, \ diff --git a/tests/qemu-iotests/209 b/tests/qemu-iotests/209 index 65c1a1e..8c804f4 100755 --- a/tests/qemu-iotests/209 +++ b/tests/qemu-iotests/209 @@ -22,7 +22,7 @@ import iotests from iotests import qemu_img_create, qemu_io, qemu_img_verbose, qemu_nbd, \ file_path -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize(supported_fmts=['qcow2']) disk = file_path('disk') nbd_sock = file_path('nbd-sock', base_dir=iotests.sock_dir) diff --git a/tests/qemu-iotests/210 b/tests/qemu-iotests/210 index e49896e..7bf591f 100755 --- a/tests/qemu-iotests/210 +++ b/tests/qemu-iotests/210 @@ -23,8 +23,10 @@ import iotests from iotests import imgfmt -iotests.verify_image_format(supported_fmts=['luks']) -iotests.verify_protocol(supported=['file']) +iotests.script_initialize( + supported_fmts=['luks'], + supported_protocols=['file'], +) with iotests.FilePath('t.luks') as disk_path, \ iotests.VM() as vm: diff --git a/tests/qemu-iotests/211 b/tests/qemu-iotests/211 index 163994d..4969edb 100755 --- a/tests/qemu-iotests/211 +++ b/tests/qemu-iotests/211 @@ -23,8 +23,10 @@ import iotests from iotests import imgfmt -iotests.verify_image_format(supported_fmts=['vdi']) -iotests.verify_protocol(supported=['file']) +iotests.script_initialize( + supported_fmts=['vdi'], + supported_protocols=['file'], +) def blockdev_create(vm, options): error = vm.blockdev_create(options) diff --git a/tests/qemu-iotests/212 b/tests/qemu-iotests/212 index 800f92d..45d0884 100755 --- a/tests/qemu-iotests/212 +++ b/tests/qemu-iotests/212 @@ -23,8 +23,10 @@ import iotests from iotests import imgfmt -iotests.verify_image_format(supported_fmts=['parallels']) -iotests.verify_protocol(supported=['file']) +iotests.script_initialize( + supported_fmts=['parallels'], + supported_protocols=['file'], +) with iotests.FilePath('t.parallels') as disk_path, \ iotests.VM() as vm: diff --git a/tests/qemu-iotests/213 b/tests/qemu-iotests/213 index 1eee452..cf638eb 100755 --- a/tests/qemu-iotests/213 +++ b/tests/qemu-iotests/213 @@ -23,8 +23,10 @@ import iotests from iotests import imgfmt -iotests.verify_image_format(supported_fmts=['vhdx']) -iotests.verify_protocol(supported=['file']) +iotests.script_initialize( + supported_fmts=['vhdx'], + supported_protocols=['file'], +) with iotests.FilePath('t.vhdx') as disk_path, \ iotests.VM() as vm: diff --git a/tests/qemu-iotests/216 b/tests/qemu-iotests/216 index 372f042..de11d85 100755 --- a/tests/qemu-iotests/216 +++ b/tests/qemu-iotests/216 @@ -23,8 +23,8 @@ import iotests from iotests import log, qemu_img, qemu_io_silent # Need backing file support -iotests.verify_image_format(supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk']) -iotests.verify_platform(['linux']) +iotests.script_initialize(supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk'], + supported_platforms=['linux']) log('') log('=== Copy-on-read across nodes ===') diff --git a/tests/qemu-iotests/218 b/tests/qemu-iotests/218 index 1325ba9..5586870 100755 --- a/tests/qemu-iotests/218 +++ b/tests/qemu-iotests/218 @@ -29,7 +29,7 @@ import iotests from iotests import log, qemu_img, qemu_io_silent -iotests.verify_image_format(supported_fmts=['qcow2', 'raw']) +iotests.script_initialize(supported_fmts=['qcow2', 'raw']) # Launches the VM, adds two null-co nodes (source and target), and diff --git a/tests/qemu-iotests/219 b/tests/qemu-iotests/219 index b877477..db272c5 100755 --- a/tests/qemu-iotests/219 +++ b/tests/qemu-iotests/219 @@ -21,7 +21,7 @@ import iotests -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize(supported_fmts=['qcow2']) img_size = 4 * 1024 * 1024 diff --git a/tests/qemu-iotests/222 b/tests/qemu-iotests/222 index bf1718e..6602f6b 100755 --- a/tests/qemu-iotests/222 +++ b/tests/qemu-iotests/222 @@ -24,9 +24,10 @@ import iotests from iotests import log, qemu_img, qemu_io, qemu_io_silent -iotests.verify_platform(['linux']) -iotests.verify_image_format(supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk', - 'vhdx', 'raw']) +iotests.script_initialize( + supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk', 'vhdx', 'raw'], + supported_platforms=['linux'], +) patterns = [("0x5d", "0", "64k"), ("0xd5", "1M", "64k"), diff --git a/tests/qemu-iotests/224 b/tests/qemu-iotests/224 index e91fb26..81ca1e4 100755 --- a/tests/qemu-iotests/224 +++ b/tests/qemu-iotests/224 @@ -26,8 +26,8 @@ from iotests import log, qemu_img, qemu_io_silent, filter_qmp_testfiles, \ import json # Need backing file support (for arbitrary backing formats) -iotests.verify_image_format(supported_fmts=['qcow2', 'qcow', 'qed']) -iotests.verify_platform(['linux']) +iotests.script_initialize(supported_fmts=['qcow2', 'qcow', 'qed'], + supported_platforms=['linux']) # There are two variations of this test: diff --git a/tests/qemu-iotests/228 b/tests/qemu-iotests/228 index 64bc82e..da0900f 100755 --- a/tests/qemu-iotests/228 +++ b/tests/qemu-iotests/228 @@ -25,8 +25,10 @@ from iotests import log, qemu_img, filter_testfiles, filter_imgfmt, \ filter_qmp_testfiles, filter_qmp_imgfmt # Need backing file and change-backing-file support -iotests.verify_image_format(supported_fmts=['qcow2', 'qed']) -iotests.verify_platform(['linux']) +iotests.script_initialize( + supported_fmts=['qcow2', 'qed'], + supported_platforms=['linux'], +) def log_node_info(node): diff --git a/tests/qemu-iotests/234 b/tests/qemu-iotests/234 index 324c154..73c899d 100755 --- a/tests/qemu-iotests/234 +++ b/tests/qemu-iotests/234 @@ -23,8 +23,8 @@ import iotests import os -iotests.verify_image_format(supported_fmts=['qcow2']) -iotests.verify_platform(['linux']) +iotests.script_initialize(supported_fmts=['qcow2'], + supported_platforms=['linux']) with iotests.FilePath('img') as img_path, \ iotests.FilePath('backing') as backing_path, \ diff --git a/tests/qemu-iotests/235 b/tests/qemu-iotests/235 index 7608261..d1b10ac 100755 --- a/tests/qemu-iotests/235 +++ b/tests/qemu-iotests/235 @@ -27,6 +27,8 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu.machine import QEMUMachine +iotests.script_initialize(supported_fmts=['qcow2']) + # Note: # This test was added to check that mirror dead-lock was fixed (see previous # commit before this test addition). @@ -40,8 +42,6 @@ from qemu.machine import QEMUMachine size = 1 * 1024 * 1024 * 1024 -iotests.verify_image_format(supported_fmts=['qcow2']) - disk = file_path('disk') # prepare source image diff --git a/tests/qemu-iotests/236 b/tests/qemu-iotests/236 index 8ce927a..6f5cee2 100755 --- a/tests/qemu-iotests/236 +++ b/tests/qemu-iotests/236 @@ -22,7 +22,7 @@ import iotests from iotests import log -iotests.verify_image_format(supported_fmts=['generic']) +iotests.script_initialize(supported_fmts=['generic']) size = 64 * 1024 * 1024 granularity = 64 * 1024 diff --git a/tests/qemu-iotests/237 b/tests/qemu-iotests/237 index 50ba364..5b21ad3 100755 --- a/tests/qemu-iotests/237 +++ b/tests/qemu-iotests/237 @@ -24,7 +24,7 @@ import math import iotests from iotests import imgfmt -iotests.verify_image_format(supported_fmts=['vmdk']) +iotests.script_initialize(supported_fmts=['vmdk']) with iotests.FilePath('t.vmdk') as disk_path, \ iotests.FilePath('t.vmdk.1') as extent1_path, \ diff --git a/tests/qemu-iotests/238 b/tests/qemu-iotests/238 index d4e0602..b8fcf15 100755 --- a/tests/qemu-iotests/238 +++ b/tests/qemu-iotests/238 @@ -23,6 +23,8 @@ import os import iotests from iotests import log +iotests.script_initialize() + virtio_scsi_device = iotests.get_virtio_scsi_device() vm = iotests.VM() diff --git a/tests/qemu-iotests/242 b/tests/qemu-iotests/242 index 9761787..64f1bd9 100755 --- a/tests/qemu-iotests/242 +++ b/tests/qemu-iotests/242 @@ -24,7 +24,7 @@ import struct from iotests import qemu_img_create, qemu_io, qemu_img_pipe, \ file_path, img_info_log, log, filter_qemu_io -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize(supported_fmts=['qcow2']) disk = file_path('disk') chunk = 256 * 1024 diff --git a/tests/qemu-iotests/246 b/tests/qemu-iotests/246 index 59a216a..58a479c 100755 --- a/tests/qemu-iotests/246 +++ b/tests/qemu-iotests/246 @@ -22,7 +22,7 @@ import iotests from iotests import log -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize(supported_fmts=['qcow2']) size = 64 * 1024 * 1024 * 1024 gran_small = 32 * 1024 gran_large = 128 * 1024 diff --git a/tests/qemu-iotests/248 b/tests/qemu-iotests/248 index 68c3746..18ba034 100755 --- a/tests/qemu-iotests/248 +++ b/tests/qemu-iotests/248 @@ -21,7 +21,7 @@ import iotests from iotests import qemu_img_create, qemu_io, file_path, filter_qmp_testfiles -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize(supported_fmts=['qcow2']) source, target = file_path('source', 'target') size = 5 * 1024 * 1024 diff --git a/tests/qemu-iotests/254 b/tests/qemu-iotests/254 index ee66c98..150e58b 100755 --- a/tests/qemu-iotests/254 +++ b/tests/qemu-iotests/254 @@ -21,7 +21,7 @@ import iotests from iotests import qemu_img_create, file_path, log -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize(supported_fmts=['qcow2']) disk, top = file_path('disk', 'top') size = 1024 * 1024 diff --git a/tests/qemu-iotests/255 b/tests/qemu-iotests/255 index 4a4818b..8f08f74 100755 --- a/tests/qemu-iotests/255 +++ b/tests/qemu-iotests/255 @@ -23,7 +23,7 @@ import iotests from iotests import imgfmt -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize(supported_fmts=['qcow2']) iotests.log('Finishing a commit job with background reads') iotests.log('============================================') diff --git a/tests/qemu-iotests/256 b/tests/qemu-iotests/256 index e34074c..db8d6f3 100755 --- a/tests/qemu-iotests/256 +++ b/tests/qemu-iotests/256 @@ -23,7 +23,7 @@ import os import iotests from iotests import log -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize(supported_fmts=['qcow2']) size = 64 * 1024 * 1024 with iotests.FilePath('img0') as img0_path, \ diff --git a/tests/qemu-iotests/258 b/tests/qemu-iotests/258 index 091755a..a65151d 100755 --- a/tests/qemu-iotests/258 +++ b/tests/qemu-iotests/258 @@ -24,9 +24,10 @@ from iotests import log, qemu_img, qemu_io_silent, \ filter_qmp_testfiles, filter_qmp_imgfmt # Need backing file and change-backing-file support -iotests.verify_image_format(supported_fmts=['qcow2', 'qed']) -iotests.verify_platform(['linux']) - +iotests.script_initialize( + supported_fmts=['qcow2', 'qed'], + supported_platforms=['linux'], +) # Returns a node for blockdev-add def node(node_name, path, backing=None, fmt=None, throttle=None): diff --git a/tests/qemu-iotests/260 b/tests/qemu-iotests/260 index 30c0de3..804a7ad 100755 --- a/tests/qemu-iotests/260 +++ b/tests/qemu-iotests/260 @@ -21,7 +21,9 @@ import iotests from iotests import qemu_img_create, file_path, log, filter_qmp_event -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize( + supported_fmts=['qcow2'] +) base, top = file_path('base', 'top') size = 64 * 1024 * 3 diff --git a/tests/qemu-iotests/262 b/tests/qemu-iotests/262 index 8835dce..03af061 100755 --- a/tests/qemu-iotests/262 +++ b/tests/qemu-iotests/262 @@ -23,8 +23,8 @@ import iotests import os -iotests.verify_image_format(supported_fmts=['qcow2']) -iotests.verify_platform(['linux']) +iotests.script_initialize(supported_fmts=['qcow2'], + supported_platforms=['linux']) with iotests.FilePath('img') as img_path, \ iotests.FilePath('mig_fifo') as fifo, \ diff --git a/tests/qemu-iotests/264 b/tests/qemu-iotests/264 index 879123a..304a744 100755 --- a/tests/qemu-iotests/264 +++ b/tests/qemu-iotests/264 @@ -24,7 +24,9 @@ import iotests from iotests import qemu_img_create, qemu_io_silent_check, file_path, \ qemu_nbd_popen, log -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize( + supported_fmts=['qcow2'], +) disk_a, disk_b, nbd_sock = file_path('disk_a', 'disk_b', 'nbd-sock') nbd_uri = 'nbd+unix:///?socket=' + nbd_sock diff --git a/tests/qemu-iotests/274 b/tests/qemu-iotests/274 index e951f72..5d1bf34 100755 --- a/tests/qemu-iotests/274 +++ b/tests/qemu-iotests/274 @@ -21,8 +21,8 @@ import iotests -iotests.verify_image_format(supported_fmts=['qcow2']) -iotests.verify_platform(['linux']) +iotests.script_initialize(supported_fmts=['qcow2'], + supported_platforms=['linux']) size_short = 1 * 1024 * 1024 size_long = 2 * 1024 * 1024 diff --git a/tests/qemu-iotests/277 b/tests/qemu-iotests/277 index 04aa15a..d34f870 100755 --- a/tests/qemu-iotests/277 +++ b/tests/qemu-iotests/277 @@ -23,6 +23,8 @@ import subprocess import iotests from iotests import file_path, log +iotests.script_initialize() + nbd_sock, conf_file = file_path('nbd-sock', 'nbd-fault-injector.conf') diff --git a/tests/qemu-iotests/280 b/tests/qemu-iotests/280 index 69288fd..f594bb9 100755 --- a/tests/qemu-iotests/280 +++ b/tests/qemu-iotests/280 @@ -22,9 +22,11 @@ import iotests import os -iotests.verify_image_format(supported_fmts=['qcow2']) -iotests.verify_protocol(supported=['file']) -iotests.verify_platform(['linux']) +iotests.script_initialize( + supported_fmts=['qcow2'], + supported_protocols=['file'], + supported_platforms=['linux'], +) with iotests.FilePath('base') as base_path , \ iotests.FilePath('top') as top_path, \ diff --git a/tests/qemu-iotests/283 b/tests/qemu-iotests/283 index 55b7cff..e17b953 100644 --- a/tests/qemu-iotests/283 +++ b/tests/qemu-iotests/283 @@ -21,7 +21,9 @@ import iotests # The test is unrelated to formats, restrict it to qcow2 to avoid extra runs -iotests.verify_image_format(supported_fmts=['qcow2']) +iotests.script_initialize( + supported_fmts=['qcow2'], +) size = 1024 * 1024 diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index cf10c42..9f85e1f 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -28,7 +28,8 @@ import signal import struct import subprocess import sys -from typing import (Any, Callable, Dict, Iterable, List, Optional, TypeVar) +from typing import (Any, Callable, Dict, Iterable, + List, Optional, Sequence, TypeVar) import unittest # pylint: disable=import-error, wrong-import-position @@ -1029,12 +1030,11 @@ def verify_protocol(supported=(), unsupported=()): if not_sup or (imgproto in unsupported): notrun('not suitable for this protocol: %s' % imgproto) -def verify_platform(supported=None, unsupported=None): - if unsupported is not None: - if any((sys.platform.startswith(x) for x in unsupported)): - notrun('not suitable for this OS: %s' % sys.platform) +def verify_platform(supported=(), unsupported=()): + if any((sys.platform.startswith(x) for x in unsupported)): + notrun('not suitable for this OS: %s' % sys.platform) - if supported is not None: + if supported: if not any((sys.platform.startswith(x) for x in supported)): notrun('not suitable for this OS: %s' % sys.platform) @@ -1116,7 +1116,18 @@ def skip_if_user_is_root(func): return func(*args, **kwargs) return func_wrapper -def execute_unittest(output, verbosity, debug): +def execute_unittest(debug=False): + """Executes unittests within the calling module.""" + + verbosity = 2 if debug else 1 + + if debug: + output = sys.stdout + else: + # We need to filter out the time taken from the output so that + # qemu-iotest can reliably diff the results against master output. + output = io.StringIO() + runner = unittest.TextTestRunner(stream=output, descriptions=True, verbosity=verbosity) try: @@ -1124,6 +1135,8 @@ def execute_unittest(output, verbosity, debug): # exception unittest.main(testRunner=runner) finally: + # We need to filter out the time taken from the output so that + # qemu-iotest can reliably diff the results against master output. if not debug: out = output.getvalue() out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out) @@ -1135,13 +1148,19 @@ def execute_unittest(output, verbosity, debug): sys.stderr.write(out) -def execute_test(test_function=None, - supported_fmts=(), - supported_platforms=None, - supported_cache_modes=(), supported_aio_modes=(), - unsupported_fmts=(), supported_protocols=(), - unsupported_protocols=()): - """Run either unittest or script-style tests.""" +def execute_setup_common(supported_fmts: Sequence[str] = (), + supported_platforms: Sequence[str] = (), + supported_cache_modes: Sequence[str] = (), + supported_aio_modes: Sequence[str] = (), + unsupported_fmts: Sequence[str] = (), + supported_protocols: Sequence[str] = (), + unsupported_protocols: Sequence[str] = ()) -> bool: + """ + Perform necessary setup for either script-style or unittest-style tests. + + :return: Bool; Whether or not debug mode has been requested via the CLI. + """ + # Note: Python 3.6 and pylint do not like 'Collection' so use 'Sequence'. # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to # indicate that we're not being run via "check". There may be @@ -1151,34 +1170,39 @@ def execute_test(test_function=None, sys.stderr.write('Please run this test via the "check" script\n') sys.exit(os.EX_USAGE) - debug = '-d' in sys.argv - verbosity = 1 verify_image_format(supported_fmts, unsupported_fmts) verify_protocol(supported_protocols, unsupported_protocols) verify_platform(supported=supported_platforms) verify_cache_mode(supported_cache_modes) verify_aio_mode(supported_aio_modes) + debug = '-d' in sys.argv if debug: - output = sys.stdout - verbosity = 2 sys.argv.remove('-d') - else: - # We need to filter out the time taken from the output so that - # qemu-iotest can reliably diff the results against master output. - output = io.StringIO() - logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN)) + return debug + +def execute_test(*args, test_function=None, **kwargs): + """Run either unittest or script-style tests.""" + + debug = execute_setup_common(*args, **kwargs) if not test_function: - execute_unittest(output, verbosity, debug) + execute_unittest(debug) else: test_function() +# This is called from script-style iotests without a single point of entry +def script_initialize(*args, **kwargs): + """Initialize script-style tests without running any tests.""" + execute_setup_common(*args, **kwargs) + +# This is called from script-style iotests with a single point of entry def script_main(test_function, *args, **kwargs): """Run script-style tests outside of the unittest framework""" - execute_test(test_function, *args, **kwargs) + execute_test(*args, test_function=test_function, **kwargs) +# This is called from unittest style iotests def main(*args, **kwargs): """Run tests using the unittest framework""" - execute_test(None, *args, **kwargs) + execute_test(*args, **kwargs) -- cgit v1.1 From 5e089feb93e1c0ce55a5900b00b03d772587c396 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:12 -0400 Subject: iotest 258: use script_main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since this one is nicely factored to use a single entry point, use script_main to run the tests. Signed-off-by: John Snow Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-13-jsnow@redhat.com> Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/258 | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/258 b/tests/qemu-iotests/258 index a65151d..e305a15 100755 --- a/tests/qemu-iotests/258 +++ b/tests/qemu-iotests/258 @@ -23,12 +23,6 @@ import iotests from iotests import log, qemu_img, qemu_io_silent, \ filter_qmp_testfiles, filter_qmp_imgfmt -# Need backing file and change-backing-file support -iotests.script_initialize( - supported_fmts=['qcow2', 'qed'], - supported_platforms=['linux'], -) - # Returns a node for blockdev-add def node(node_name, path, backing=None, fmt=None, throttle=None): if fmt is None: @@ -161,4 +155,7 @@ def main(): test_concurrent_finish(False) if __name__ == '__main__': - main() + # Need backing file and change-backing-file support + iotests.script_main(main, + supported_fmts=['qcow2', 'qed'], + supported_platforms=['linux']) -- cgit v1.1 From 59c29869e0c5e8875edc50a0e3073eeaa8fe6143 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:13 -0400 Subject: iotests: Mark verify functions as private Mark the verify functions as "private" with a leading underscore, to discourage their use. Update type signatures while we're here. (Also, make pending patches not yet using the new entry points fail in a very obvious way.) Signed-off-by: John Snow Message-Id: <20200331000014.11581-14-jsnow@redhat.com> Reviewed-by: Max Reitz Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/iotests.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 9f85e1f..35d8cae 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -1006,7 +1006,8 @@ def case_notrun(reason): open('%s/%s.casenotrun' % (output_dir, seq), 'a').write( ' [case not run] ' + reason + '\n') -def verify_image_format(supported_fmts=(), unsupported_fmts=()): +def _verify_image_format(supported_fmts: Sequence[str] = (), + unsupported_fmts: Sequence[str] = ()) -> None: assert not (supported_fmts and unsupported_fmts) if 'generic' in supported_fmts and \ @@ -1020,7 +1021,8 @@ def verify_image_format(supported_fmts=(), unsupported_fmts=()): if not_sup or (imgfmt in unsupported_fmts): notrun('not suitable for this image format: %s' % imgfmt) -def verify_protocol(supported=(), unsupported=()): +def _verify_protocol(supported: Sequence[str] = (), + unsupported: Sequence[str] = ()) -> None: assert not (supported and unsupported) if 'generic' in supported: @@ -1030,7 +1032,8 @@ def verify_protocol(supported=(), unsupported=()): if not_sup or (imgproto in unsupported): notrun('not suitable for this protocol: %s' % imgproto) -def verify_platform(supported=(), unsupported=()): +def _verify_platform(supported: Sequence[str] = (), + unsupported: Sequence[str] = ()) -> None: if any((sys.platform.startswith(x) for x in unsupported)): notrun('not suitable for this OS: %s' % sys.platform) @@ -1038,11 +1041,11 @@ def verify_platform(supported=(), unsupported=()): if not any((sys.platform.startswith(x) for x in supported)): notrun('not suitable for this OS: %s' % sys.platform) -def verify_cache_mode(supported_cache_modes=()): +def _verify_cache_mode(supported_cache_modes: Sequence[str] = ()) -> None: if supported_cache_modes and (cachemode not in supported_cache_modes): notrun('not suitable for this cache mode: %s' % cachemode) -def verify_aio_mode(supported_aio_modes=()): +def _verify_aio_mode(supported_aio_modes: Sequence[str] = ()): if supported_aio_modes and (aiomode not in supported_aio_modes): notrun('not suitable for this aio mode: %s' % aiomode) @@ -1170,11 +1173,11 @@ def execute_setup_common(supported_fmts: Sequence[str] = (), sys.stderr.write('Please run this test via the "check" script\n') sys.exit(os.EX_USAGE) - verify_image_format(supported_fmts, unsupported_fmts) - verify_protocol(supported_protocols, unsupported_protocols) - verify_platform(supported=supported_platforms) - verify_cache_mode(supported_cache_modes) - verify_aio_mode(supported_aio_modes) + _verify_image_format(supported_fmts, unsupported_fmts) + _verify_protocol(supported_protocols, unsupported_protocols) + _verify_platform(supported=supported_platforms) + _verify_cache_mode(supported_cache_modes) + _verify_aio_mode(supported_aio_modes) debug = '-d' in sys.argv if debug: -- cgit v1.1 From 52ea799e9631ff00c230f1fed50986246149d8f2 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 30 Mar 2020 20:00:14 -0400 Subject: iotests: use python logging for iotests.log() We can turn logging on/off globally instead of per-function. Remove use_log from run_job, and use python logging to turn on diffable output when we run through a script entry point. iotest 245 changes output order due to buffering reasons. An extended note on python logging: A NullHandler is added to `qemu.iotests` to stop output from being generated if this code is used as a library without configuring logging. A NullHandler is only needed at the root, so a duplicate handler is not needed for `qemu.iotests.diff_io`. When logging is not configured, messages at the 'WARNING' levels or above are printed with default settings. The NullHandler stops this from occurring, which is considered good hygiene for code used as a library. See https://docs.python.org/3/howto/logging.html#library-config When logging is actually enabled (always at the behest of an explicit call by a client script), a root logger is implicitly created at the root, which allows messages to propagate upwards and be handled/emitted from the root logger with default settings. When we want iotest logging, we attach a handler to the qemu.iotests.diff_io logger and disable propagation to avoid possible double-printing. For more information on python logging infrastructure, I highly recommend downloading the pip package `logging_tree`, which provides convenient visualizations of the hierarchical logging configuration under different circumstances. See https://pypi.org/project/logging_tree/ for more information. Signed-off-by: John Snow Reviewed-by: Max Reitz Message-Id: <20200331000014.11581-15-jsnow@redhat.com> Reviewed-by: Kevin Wolf Signed-off-by: Max Reitz --- tests/qemu-iotests/030 | 4 ++-- tests/qemu-iotests/155 | 2 +- tests/qemu-iotests/245 | 1 + tests/qemu-iotests/245.out | 10 ++++---- tests/qemu-iotests/iotests.py | 53 ++++++++++++++++++++++++------------------- 5 files changed, 39 insertions(+), 31 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030 index aa911d2..104e3ce 100755 --- a/tests/qemu-iotests/030 +++ b/tests/qemu-iotests/030 @@ -411,8 +411,8 @@ class TestParallelOps(iotests.QMPTestCase): result = self.vm.qmp('block-job-set-speed', device='drive0', speed=0) self.assert_qmp(result, 'return', {}) - self.vm.run_job(job='drive0', auto_dismiss=True, use_log=False) - self.vm.run_job(job='node4', auto_dismiss=True, use_log=False) + self.vm.run_job(job='drive0', auto_dismiss=True) + self.vm.run_job(job='node4', auto_dismiss=True) self.assert_no_active_block_jobs() # Test a block-stream and a block-commit job in parallel diff --git a/tests/qemu-iotests/155 b/tests/qemu-iotests/155 index 571bce9..cb371d4 100755 --- a/tests/qemu-iotests/155 +++ b/tests/qemu-iotests/155 @@ -188,7 +188,7 @@ class MirrorBaseClass(BaseClass): self.assert_qmp(result, 'return', {}) - self.vm.run_job('mirror-job', use_log=False, auto_finalize=False, + self.vm.run_job('mirror-job', auto_finalize=False, pre_finalize=self.openBacking, auto_dismiss=True) def testFull(self): diff --git a/tests/qemu-iotests/245 b/tests/qemu-iotests/245 index 1001275..4f5f0bb 100755 --- a/tests/qemu-iotests/245 +++ b/tests/qemu-iotests/245 @@ -1027,5 +1027,6 @@ class TestBlockdevReopen(iotests.QMPTestCase): self.run_test_iothreads(None, 'iothread0') if __name__ == '__main__': + iotests.activate_logging() iotests.main(supported_fmts=["qcow2"], supported_protocols=["file"]) diff --git a/tests/qemu-iotests/245.out b/tests/qemu-iotests/245.out index 682b933..4b33dca 100644 --- a/tests/qemu-iotests/245.out +++ b/tests/qemu-iotests/245.out @@ -1,8 +1,3 @@ -..................... ----------------------------------------------------------------------- -Ran 21 tests - -OK {"execute": "job-finalize", "arguments": {"id": "commit0"}} {"return": {}} {"data": {"id": "commit0", "type": "commit"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -15,3 +10,8 @@ OK {"return": {}} {"data": {"id": "stream0", "type": "stream"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"device": "stream0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "stream"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} +..................... +---------------------------------------------------------------------- +Ran 21 tests + +OK diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 35d8cae..6c0e781 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -42,6 +42,14 @@ assert sys.version_info >= (3, 6) QMPResponse = Dict[str, Any] +# Use this logger for logging messages directly from the iotests module +logger = logging.getLogger('qemu.iotests') +logger.addHandler(logging.NullHandler()) + +# Use this logger for messages that ought to be used for diff output. +test_logger = logging.getLogger('qemu.iotests.diff_io') + + faulthandler.enable() # This will not work if arguments contain spaces but is necessary if we @@ -385,9 +393,9 @@ def log(msg: Msg, if isinstance(msg, (dict, list)): # Don't sort if it's already sorted do_sort = not isinstance(msg, OrderedDict) - print(json.dumps(msg, sort_keys=do_sort, indent=indent)) + test_logger.info(json.dumps(msg, sort_keys=do_sort, indent=indent)) else: - print(msg) + test_logger.info(msg) class Timeout: def __init__(self, seconds, errmsg="Timeout"): @@ -609,7 +617,7 @@ class VM(qtest.QEMUQtestMachine): # Returns None on success, and an error string on failure def run_job(self, job, auto_finalize=True, auto_dismiss=False, - pre_finalize=None, cancel=False, use_log=True, wait=60.0): + pre_finalize=None, cancel=False, wait=60.0): """ run_job moves a job from creation through to dismissal. @@ -622,7 +630,6 @@ class VM(qtest.QEMUQtestMachine): invoked prior to issuing job-finalize, if any. :param cancel: Bool. When true, cancels the job after the pre_finalize callback. - :param use_log: Bool. When false, does not log QMP messages. :param wait: Float. Timeout value specifying how long to wait for any event, in seconds. Defaults to 60.0. """ @@ -640,8 +647,7 @@ class VM(qtest.QEMUQtestMachine): while True: ev = filter_qmp_event(self.events_wait(events, timeout=wait)) if ev['event'] != 'JOB_STATUS_CHANGE': - if use_log: - log(ev) + log(ev) continue status = ev['data']['status'] if status == 'aborting': @@ -649,29 +655,18 @@ class VM(qtest.QEMUQtestMachine): for j in result['return']: if j['id'] == job: error = j['error'] - if use_log: - log('Job failed: %s' % (j['error'])) + log('Job failed: %s' % (j['error'])) elif status == 'ready': - if use_log: - self.qmp_log('job-complete', id=job) - else: - self.qmp('job-complete', id=job) + self.qmp_log('job-complete', id=job) elif status == 'pending' and not auto_finalize: if pre_finalize: pre_finalize() - if cancel and use_log: + if cancel: self.qmp_log('job-cancel', id=job) - elif cancel: - self.qmp('job-cancel', id=job) - elif use_log: - self.qmp_log('job-finalize', id=job) else: - self.qmp('job-finalize', id=job) + self.qmp_log('job-finalize', id=job) elif status == 'concluded' and not auto_dismiss: - if use_log: - self.qmp_log('job-dismiss', id=job) - else: - self.qmp('job-dismiss', id=job) + self.qmp_log('job-dismiss', id=job) elif status == 'null': return error @@ -991,7 +986,7 @@ def notrun(reason): seq = os.path.basename(sys.argv[0]) open('%s/%s.notrun' % (output_dir, seq), 'w').write(reason + '\n') - print('%s not run: %s' % (seq, reason)) + logger.warning("%s not run: %s", seq, reason) sys.exit(0) def case_notrun(reason): @@ -1183,6 +1178,7 @@ def execute_setup_common(supported_fmts: Sequence[str] = (), if debug: sys.argv.remove('-d') logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN)) + logger.debug("iotests debugging messages active") return debug @@ -1195,14 +1191,25 @@ def execute_test(*args, test_function=None, **kwargs): else: test_function() +def activate_logging(): + """Activate iotests.log() output to stdout for script-style tests.""" + handler = logging.StreamHandler(stream=sys.stdout) + formatter = logging.Formatter('%(message)s') + handler.setFormatter(formatter) + test_logger.addHandler(handler) + test_logger.setLevel(logging.INFO) + test_logger.propagate = False + # This is called from script-style iotests without a single point of entry def script_initialize(*args, **kwargs): """Initialize script-style tests without running any tests.""" + activate_logging() execute_setup_common(*args, **kwargs) # This is called from script-style iotests with a single point of entry def script_main(test_function, *args, **kwargs): """Run script-style tests outside of the unittest framework""" + activate_logging() execute_test(*args, test_function=test_function, **kwargs) # This is called from unittest style iotests -- cgit v1.1 From 7fa140abf69675b7b83af32de07fa8075c1da298 Mon Sep 17 00:00:00 2001 From: Eric Blake Date: Tue, 28 Apr 2020 14:26:47 -0500 Subject: qcow2: Allow resize of images with internal snapshots We originally refused to allow resize of images with internal snapshots because the v2 image format did not require the tracking of snapshot size, making it impossible to safely revert to a snapshot with a different size than the current view of the image. But the snapshot size tracking was rectified in v3, and our recent fixes to qemu-img amend (see 0a85af35) guarantee that we always have a valid snapshot size. Thus, we no longer need to artificially limit image resizes, but it does become one more thing that would prevent a downgrade back to v2. And now that we support different-sized snapshots, it's also easy to fix reverting to a snapshot to apply the new size. Upgrade iotest 61 to cover this (we previously had NO coverage of refusal to resize while snapshots exist). Note that the amend process can fail but still have effects: in particular, since we break things into upgrade, resize, downgrade, a failure during resize does not roll back changes made during upgrade, nor does failure in downgrade roll back a resize. But this situation is pre-existing even without this patch; and without journaling, the best we could do is minimize the chance of partial failure by collecting all changes prior to doing any writes - which adds a lot of complexity but could still fail with EIO. On the other hand, we are careful that even if we have partial modification but then fail, the image is left viable (that is, we are careful to sequence things so that after each successful cluster write, there may be transient leaked clusters but no corrupt metadata). And complicating the code to make it more transaction-like is not worth the effort: a user can always request multiple 'qemu-img amend' changing one thing each, if they need finer-grained control over detecting the first failure than what they get by letting qemu decide how to sequence multiple changes. Signed-off-by: Eric Blake Reviewed-by: Max Reitz Message-Id: <20200428192648.749066-3-eblake@redhat.com> Signed-off-by: Max Reitz --- tests/qemu-iotests/061 | 35 +++++++++++++++++++++++++++++++++++ tests/qemu-iotests/061.out | 28 ++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) (limited to 'tests') diff --git a/tests/qemu-iotests/061 b/tests/qemu-iotests/061 index ce285d3..10eb243 100755 --- a/tests/qemu-iotests/061 +++ b/tests/qemu-iotests/061 @@ -112,6 +112,41 @@ $QEMU_IO -c "read -P 0x2a 42M 64k" "$TEST_IMG" | _filter_qemu_io _check_test_img echo +echo "=== Testing resize with snapshots ===" +echo +_make_test_img -o "compat=0.10" 32M +$QEMU_IO -c "write -P 0x2a 24M 64k" "$TEST_IMG" | _filter_qemu_io +$QEMU_IMG snapshot -c foo "$TEST_IMG" +$QEMU_IMG resize "$TEST_IMG" 64M && + echo "unexpected pass" +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)' + +$QEMU_IMG amend -o "compat=1.1,size=128M" "$TEST_IMG" || + echo "unexpected fail" +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)' + +$QEMU_IMG snapshot -c bar "$TEST_IMG" +$QEMU_IMG resize --shrink "$TEST_IMG" 64M || + echo "unexpected fail" +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)' + +$QEMU_IMG amend -o "compat=0.10,size=32M" "$TEST_IMG" && + echo "unexpected pass" +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)' + +$QEMU_IMG snapshot -a bar "$TEST_IMG" || + echo "unexpected fail" +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)' + +$QEMU_IMG snapshot -d bar "$TEST_IMG" +$QEMU_IMG amend -o "compat=0.10,size=32M" "$TEST_IMG" || + echo "unexpected fail" +$PYTHON qcow2.py "$TEST_IMG" dump-header | grep '^\(version\|size\|nb_snap\)' + +_check_test_img + + +echo echo "=== Testing dirty lazy_refcounts=off ===" echo _make_test_img -o "compat=1.1,lazy_refcounts=on" 64M diff --git a/tests/qemu-iotests/061.out b/tests/qemu-iotests/061.out index 413cc4e..5a8d36d 100644 --- a/tests/qemu-iotests/061.out +++ b/tests/qemu-iotests/061.out @@ -271,6 +271,34 @@ read 65536/65536 bytes at offset 44040192 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) No errors were found on the image. +=== Testing resize with snapshots === + +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33554432 +wrote 65536/65536 bytes at offset 25165824 +64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +qemu-img: Can't resize a v2 image which has snapshots +version 2 +size 33554432 +nb_snapshots 1 +version 3 +size 134217728 +nb_snapshots 1 +Image resized. +version 3 +size 67108864 +nb_snapshots 2 +qemu-img: Internal snapshots prevent downgrade of image +version 3 +size 33554432 +nb_snapshots 2 +version 3 +size 134217728 +nb_snapshots 2 +version 2 +size 33554432 +nb_snapshots 1 +No errors were found on the image. + === Testing dirty lazy_refcounts=off === Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 -- cgit v1.1 From f4649069517b71c4038add8d977a41644fb117dc Mon Sep 17 00:00:00 2001 From: Eric Blake Date: Tue, 28 Apr 2020 16:38:07 -0500 Subject: block: Comment cleanups It's been a while since we got rid of the sector-based bdrv_read and bdrv_write (commit 2e11d756); let's finish the job on a few remaining comments. Signed-off-by: Eric Blake Message-Id: <20200428213807.776655-1-eblake@redhat.com> Reviewed-by: Alberto Garcia Signed-off-by: Max Reitz --- tests/qemu-iotests/001 | 2 +- tests/qemu-iotests/052 | 2 +- tests/qemu-iotests/134 | 2 +- tests/qemu-iotests/188 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/001 b/tests/qemu-iotests/001 index d87a535..696726e 100755 --- a/tests/qemu-iotests/001 +++ b/tests/qemu-iotests/001 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Test simple read/write using plain bdrv_read/bdrv_write +# Test simple read/write using plain bdrv_pread/bdrv_pwrite # # Copyright (C) 2009 Red Hat, Inc. # diff --git a/tests/qemu-iotests/052 b/tests/qemu-iotests/052 index 45a1409..8d5c106 100755 --- a/tests/qemu-iotests/052 +++ b/tests/qemu-iotests/052 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Test bdrv_read/bdrv_write using BDRV_O_SNAPSHOT +# Test bdrv_pread/bdrv_pwrite using BDRV_O_SNAPSHOT # # Copyright (C) 2013 Red Hat, Inc. # diff --git a/tests/qemu-iotests/134 b/tests/qemu-iotests/134 index 5f0fb86..5162d21 100755 --- a/tests/qemu-iotests/134 +++ b/tests/qemu-iotests/134 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Test encrypted read/write using plain bdrv_read/bdrv_write +# Test encrypted read/write using plain bdrv_pread/bdrv_pwrite # # Copyright (C) 2015 Red Hat, Inc. # diff --git a/tests/qemu-iotests/188 b/tests/qemu-iotests/188 index afca44d..09b9b60 100755 --- a/tests/qemu-iotests/188 +++ b/tests/qemu-iotests/188 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Test encrypted read/write using plain bdrv_read/bdrv_write +# Test encrypted read/write using plain bdrv_pread/bdrv_pwrite # # Copyright (C) 2017 Red Hat, Inc. # -- cgit v1.1 From 44e808c13045b1032143ab8e144b4d48264ad609 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Mon, 4 May 2020 16:19:59 +0300 Subject: Fix iotest 153 Commit f62514b3def5fb2acbef64d0e053c0c31fa45aff made qemu-img reject -o "" but this test uses it. Since this test only tries to do a dry-run run of qemu-img amend, replace the -o "" with dummy -o "size=$size". Fixes: f62514b3def5fb2acbef64d0e053c0c31fa45aff Signed-off-by: Maxim Levitsky Message-Id: <20200504131959.9533-1-mlevitsk@redhat.com> Signed-off-by: Max Reitz --- tests/qemu-iotests/153 | 2 +- tests/qemu-iotests/153.out | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/153 b/tests/qemu-iotests/153 index 2b13111..cf961d3 100755 --- a/tests/qemu-iotests/153 +++ b/tests/qemu-iotests/153 @@ -122,7 +122,7 @@ for opts1 in "" "read-only=on" "read-only=on,force-share=on"; do _run_cmd $QEMU_IMG check $L "${TEST_IMG}" _run_cmd $QEMU_IMG compare $L "${TEST_IMG}" "${TEST_IMG}" _run_cmd $QEMU_IMG map $L "${TEST_IMG}" - _run_cmd $QEMU_IMG amend -o "" $L "${TEST_IMG}" + _run_cmd $QEMU_IMG amend -o "size=$size" $L "${TEST_IMG}" _run_cmd $QEMU_IMG commit $L "${TEST_IMG}" _run_cmd $QEMU_IMG resize $L "${TEST_IMG}" $size _run_cmd $QEMU_IMG rebase $L "${TEST_IMG}" -b "${TEST_IMG}.base" diff --git a/tests/qemu-iotests/153.out b/tests/qemu-iotests/153.out index f7464dd..b2a90ca 100644 --- a/tests/qemu-iotests/153.out +++ b/tests/qemu-iotests/153.out @@ -56,7 +56,7 @@ _qemu_img_wrapper map TEST_DIR/t.qcow2 qemu-img: Could not open 'TEST_DIR/t.qcow2': Failed to get shared "write" lock Is another process using the image [TEST_DIR/t.qcow2]? -_qemu_img_wrapper amend -o TEST_DIR/t.qcow2 +_qemu_img_wrapper amend -o size=32M TEST_DIR/t.qcow2 qemu-img: Could not open 'TEST_DIR/t.qcow2': Failed to get "write" lock Is another process using the image [TEST_DIR/t.qcow2]? @@ -118,7 +118,7 @@ _qemu_img_wrapper compare -U TEST_DIR/t.qcow2 TEST_DIR/t.qcow2 _qemu_img_wrapper map -U TEST_DIR/t.qcow2 -_qemu_img_wrapper amend -o -U TEST_DIR/t.qcow2 +_qemu_img_wrapper amend -o size=32M -U TEST_DIR/t.qcow2 qemu-img: unrecognized option '-U' Try 'qemu-img --help' for more information @@ -187,7 +187,7 @@ _qemu_img_wrapper compare TEST_DIR/t.qcow2 TEST_DIR/t.qcow2 _qemu_img_wrapper map TEST_DIR/t.qcow2 -_qemu_img_wrapper amend -o TEST_DIR/t.qcow2 +_qemu_img_wrapper amend -o size=32M TEST_DIR/t.qcow2 qemu-img: Could not open 'TEST_DIR/t.qcow2': Failed to get "write" lock Is another process using the image [TEST_DIR/t.qcow2]? @@ -241,7 +241,7 @@ _qemu_img_wrapper compare -U TEST_DIR/t.qcow2 TEST_DIR/t.qcow2 _qemu_img_wrapper map -U TEST_DIR/t.qcow2 -_qemu_img_wrapper amend -o -U TEST_DIR/t.qcow2 +_qemu_img_wrapper amend -o size=32M -U TEST_DIR/t.qcow2 qemu-img: unrecognized option '-U' Try 'qemu-img --help' for more information @@ -303,7 +303,7 @@ _qemu_img_wrapper compare TEST_DIR/t.qcow2 TEST_DIR/t.qcow2 _qemu_img_wrapper map TEST_DIR/t.qcow2 -_qemu_img_wrapper amend -o TEST_DIR/t.qcow2 +_qemu_img_wrapper amend -o size=32M TEST_DIR/t.qcow2 _qemu_img_wrapper commit TEST_DIR/t.qcow2 @@ -345,7 +345,7 @@ _qemu_img_wrapper compare -U TEST_DIR/t.qcow2 TEST_DIR/t.qcow2 _qemu_img_wrapper map -U TEST_DIR/t.qcow2 -_qemu_img_wrapper amend -o -U TEST_DIR/t.qcow2 +_qemu_img_wrapper amend -o size=32M -U TEST_DIR/t.qcow2 qemu-img: unrecognized option '-U' Try 'qemu-img --help' for more information -- cgit v1.1