From e77964f79b905760ce7f0406fc6b988daeabbb65 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:01:27 +0100 Subject: qemu-iotests: add qed.py image manipulation utility The qed.py utility can inspect and manipulate QED image files. It can be used for testing to see the state of image metadata and also to inject corruptions into the image file. It also has a scrubbing feature to copy just the metadata out of an image file, allowing users to share broken image files without revealing data in bug reports. This has lived in my local repo for a long time but could be useful to others. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- tests/qemu-iotests/qed.py | 235 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100755 tests/qemu-iotests/qed.py (limited to 'tests') diff --git a/tests/qemu-iotests/qed.py b/tests/qemu-iotests/qed.py new file mode 100755 index 0000000..52ff845 --- /dev/null +++ b/tests/qemu-iotests/qed.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python +# +# Tool to manipulate QED image files +# +# Copyright (C) 2010 IBM, Corp. +# +# Authors: +# Stefan Hajnoczi +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. + +import sys +import struct +import random +import optparse + +# This can be used as a module +__all__ = ['QED_F_NEED_CHECK', 'QED'] + +QED_F_NEED_CHECK = 0x02 + +header_fmt = ']- Show header or l1/l2 tables''' + if not args or args[0] == 'header': + print qed.header + elif args[0] == 'l1': + print qed.l1_table + elif len(args) == 2 and args[0] == 'l2': + offset = int(args[1]) + print qed.read_table(offset) + else: + err('unrecognized sub-command') + +def cmd_duplicate(qed, table_level): + '''duplicate l1|l2 - Duplicate a random table element''' + if table_level == 'l1': + offset = qed.header['l1_table_offset'] + table = qed.l1_table + elif table_level == 'l2': + _, offset = random_table_item(qed.l1_table) + table = qed.read_table(offset) + else: + err('unrecognized sub-command') + corrupt_table_duplicate(table) + qed.write_table(offset, table) + +def cmd_invalidate(qed, table_level): + '''invalidate l1|l2 - Plant an invalid table element at random''' + if table_level == 'l1': + offset = qed.header['l1_table_offset'] + table = qed.l1_table + elif table_level == 'l2': + _, offset = random_table_item(qed.l1_table) + table = qed.read_table(offset) + else: + err('unrecognized sub-command') + corrupt_table_invalidate(qed, table) + qed.write_table(offset, table) + +def cmd_need_check(qed, *args): + '''need-check [on|off] - Test, set, or clear the QED_F_NEED_CHECK header bit''' + if not args: + print bool(qed.header['features'] & QED_F_NEED_CHECK) + return + + if args[0] == 'on': + qed.header['features'] |= QED_F_NEED_CHECK + elif args[0] == 'off': + qed.header['features'] &= ~QED_F_NEED_CHECK + else: + err('unrecognized sub-command') + qed.store_header() + +def cmd_zero_cluster(qed, pos, *args): + '''zero-cluster [] - Zero data clusters''' + pos, n = int(pos), 1 + if args: + if len(args) != 1: + err('expected one argument') + n = int(args[0]) + + for i in xrange(n): + l1_index = pos / qed.header['cluster_size'] / len(qed.l1_table) + if qed.l1_table[l1_index] == 0: + err('no l2 table allocated') + + l2_offset = qed.l1_table[l1_index] + l2_table = qed.read_table(l2_offset) + + l2_index = (pos / qed.header['cluster_size']) % len(qed.l1_table) + l2_table[l2_index] = 1 # zero the data cluster + qed.write_table(l2_offset, l2_table) + pos += qed.header['cluster_size'] + +def cmd_copy_metadata(qed, outfile): + '''copy-metadata - Copy metadata only (for scrubbing corrupted images)''' + out = open(outfile, 'wb') + + # Match file size + out.seek(qed.filesize - 1) + out.write('\0') + + # Copy header clusters + out.seek(0) + header_size_bytes = qed.header['header_size'] * qed.header['cluster_size'] + out.write(qed.raw_pread(0, header_size_bytes)) + + # Copy L1 table + out.seek(qed.header['l1_table_offset']) + s = ''.join(pack_table_elem(x) for x in qed.l1_table) + out.write(s) + + # Copy L2 tables + for l2_offset in qed.l1_table: + if l2_offset == 0: + continue + l2_table = qed.read_table(l2_offset) + out.seek(l2_offset) + s = ''.join(pack_table_elem(x) for x in l2_table) + out.write(s) + + out.close() + +def usage(): + print 'Usage: %s [, ...]' % sys.argv[0] + print + print 'Supported commands:' + for cmd in sorted(x for x in globals() if x.startswith('cmd_')): + print globals()[cmd].__doc__ + sys.exit(1) + +def main(): + if len(sys.argv) < 3: + usage() + filename, cmd = sys.argv[1:3] + + cmd = 'cmd_' + cmd.replace('-', '_') + if cmd not in globals(): + usage() + + qed = QED(open(filename, 'r+b')) + try: + globals()[cmd](qed, *sys.argv[3:]) + except TypeError, e: + sys.stderr.write(globals()[cmd].__doc__ + '\n') + sys.exit(1) + +if __name__ == '__main__': + main() -- cgit v1.1 From c61d0004bc37182cae0f92a865ad2f0625762610 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:19 +0100 Subject: qcow2: introduce dirty bit This patch adds an incompatible feature bit to mark images that have not been closed cleanly. When a dirty image file is opened a consistency check and repair is performed. Update qemu-iotests 031 and 036 since the extension header size changes when we add feature bit table entries. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- tests/qemu-iotests/031.out | 20 ++++++++++---------- tests/qemu-iotests/036.out | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/031.out b/tests/qemu-iotests/031.out index d3cab30..297b458 100644 --- a/tests/qemu-iotests/031.out +++ b/tests/qemu-iotests/031.out @@ -54,8 +54,8 @@ header_length 72 Header extension: magic 0x6803f857 -length 0 -data '' +length 48 +data Header extension: magic 0x12345678 @@ -68,7 +68,7 @@ No errors were found on the image. magic 0x514649fb version 2 -backing_file_offset 0x98 +backing_file_offset 0xc8 backing_file_size 0x17 cluster_bits 16 size 67108864 @@ -92,8 +92,8 @@ data 'host_device' Header extension: magic 0x6803f857 -length 0 -data '' +length 48 +data Header extension: magic 0x12345678 @@ -155,8 +155,8 @@ header_length 104 Header extension: magic 0x6803f857 -length 0 -data '' +length 48 +data Header extension: magic 0x12345678 @@ -169,7 +169,7 @@ No errors were found on the image. magic 0x514649fb version 3 -backing_file_offset 0xb8 +backing_file_offset 0xe8 backing_file_size 0x17 cluster_bits 16 size 67108864 @@ -193,8 +193,8 @@ data 'host_device' Header extension: magic 0x6803f857 -length 0 -data '' +length 48 +data Header extension: magic 0x12345678 diff --git a/tests/qemu-iotests/036.out b/tests/qemu-iotests/036.out index 6953e37..ca0fda1 100644 --- a/tests/qemu-iotests/036.out +++ b/tests/qemu-iotests/036.out @@ -46,7 +46,7 @@ header_length 104 Header extension: magic 0x6803f857 -length 0 -data '' +length 48 +data *** done -- cgit v1.1 From 91cf8a35e777cdbdc3bf0d39c319ac9f76b7a296 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:21 +0100 Subject: qemu-iotests: ignore qemu-img create lazy_refcounts output Hide the default lazy_refcounts=off output from qemu-img like we do with other image creation options. This ensures that existing golden outputs continue to pass despite the new option that has been added. Note that this patch applies before the one that actually introduces the lazy_refcounts=on|off option. This ensures git-bisect(1) continues to work. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- tests/qemu-iotests/common.rc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc index 5e3a524..cc4e39b 100644 --- a/tests/qemu-iotests/common.rc +++ b/tests/qemu-iotests/common.rc @@ -113,7 +113,8 @@ _make_test_img() sed -e "s# table_size=0##g" | \ sed -e "s# compat='[^']*'##g" | \ sed -e "s# compat6=off##g" | \ - sed -e "s# static=off##g" + sed -e "s# static=off##g" | \ + sed -e "s# lazy_refcounts=off##g" } _cleanup_test_img() -- cgit v1.1 From bfe8043e9214d2fc6572cc72b5f2218308747acd Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:22 +0100 Subject: qcow2: implement lazy refcounts Lazy refcounts is a performance optimization for qcow2 that postpones refcount metadata updates and instead marks the image dirty. In the case of crash or power failure the image will be left in a dirty state and repaired next time it is opened. Reducing metadata I/O is important for cache=writethrough and cache=directsync because these modes guarantee that data is on disk after each write (hence we cannot take advantage of caching updates in RAM). Refcount metadata is not needed for guest->file block address translation and therefore does not need to be on-disk at the time of write completion - this is the motivation behind the lazy refcount optimization. The lazy refcount optimization must be enabled at image creation time: qemu-img create -f qcow2 -o compat=1.1,lazy_refcounts=on a.qcow2 10G qemu-system-x86_64 -drive if=virtio,file=a.qcow2,cache=writethrough Update qemu-iotests 031 and 036 since the extension header size changes when we add feature bit table entries. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- tests/qemu-iotests/031.out | 12 ++++++------ tests/qemu-iotests/036.out | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/031.out b/tests/qemu-iotests/031.out index 297b458..796c993 100644 --- a/tests/qemu-iotests/031.out +++ b/tests/qemu-iotests/031.out @@ -54,7 +54,7 @@ header_length 72 Header extension: magic 0x6803f857 -length 48 +length 96 data Header extension: @@ -68,7 +68,7 @@ No errors were found on the image. magic 0x514649fb version 2 -backing_file_offset 0xc8 +backing_file_offset 0xf8 backing_file_size 0x17 cluster_bits 16 size 67108864 @@ -92,7 +92,7 @@ data 'host_device' Header extension: magic 0x6803f857 -length 48 +length 96 data Header extension: @@ -155,7 +155,7 @@ header_length 104 Header extension: magic 0x6803f857 -length 48 +length 96 data Header extension: @@ -169,7 +169,7 @@ No errors were found on the image. magic 0x514649fb version 3 -backing_file_offset 0xe8 +backing_file_offset 0x118 backing_file_size 0x17 cluster_bits 16 size 67108864 @@ -193,7 +193,7 @@ data 'host_device' Header extension: magic 0x6803f857 -length 48 +length 96 data Header extension: diff --git a/tests/qemu-iotests/036.out b/tests/qemu-iotests/036.out index ca0fda1..063ca22 100644 --- a/tests/qemu-iotests/036.out +++ b/tests/qemu-iotests/036.out @@ -46,7 +46,7 @@ header_length 104 Header extension: magic 0x6803f857 -length 48 +length 96 data *** done -- cgit v1.1 From dc68afe0f3e8be913a0eb639afc9196e0abab261 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 27 Jul 2012 09:05:24 +0100 Subject: qemu-iotests: add 039 qcow2 lazy refcounts test This tests establishes the basic post-conditions of the qcow2 lazy refcounts features: 1. If the image was closed normally, it is marked clean. 2. If an allocating write was performed and the image was not closed normally, then it is marked dirty. a. Written data can be read back successfully. b. The image file can be repaired and will be marked clean again. c. The image file is automatically repaired when opened read/write. Signed-off-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- tests/qemu-iotests/039 | 136 +++++++++++++++++++++++++++++++++++++++++++++ tests/qemu-iotests/039.out | 53 ++++++++++++++++++ tests/qemu-iotests/group | 1 + 3 files changed, 190 insertions(+) create mode 100755 tests/qemu-iotests/039 create mode 100644 tests/qemu-iotests/039.out (limited to 'tests') diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039 new file mode 100755 index 0000000..a749fcf --- /dev/null +++ b/tests/qemu-iotests/039 @@ -0,0 +1,136 @@ +#!/bin/bash +# +# Test qcow2 lazy refcounts +# +# Copyright (C) 2012 Red Hat, Inc. +# Copyright IBM, Corp. 2010 +# +# Based on test 038. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# creator +owner=stefanha@linux.vnet.ibm.com + +seq=`basename $0` +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! + +_cleanup() +{ + _cleanup_test_img +} +trap "_cleanup; exit \$status" 0 1 2 3 15 + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +_supported_fmt qcow2 +_supported_proto generic +_supported_os Linux + +size=128M + +echo +echo "== Checking that image is clean on shutdown ==" + +IMGOPTS="compat=1.1,lazy_refcounts=on" +_make_test_img $size + +$QEMU_IO -c "write -P 0x5a 0 512" $TEST_IMG | _filter_qemu_io + +# The dirty bit must not be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features +_check_test_img + +echo +echo "== Creating a dirty image file ==" + +IMGOPTS="compat=1.1,lazy_refcounts=on" +_make_test_img $size + +old_ulimit=$(ulimit -c) +ulimit -c 0 # do not produce a core dump on abort(3) +$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" $TEST_IMG | _filter_qemu_io +ulimit -c "$old_ulimit" + +# The dirty bit must be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features +_check_test_img + +echo +echo "== Read-only access must still work ==" + +$QEMU_IO -r -c "read -P 0x5a 0 512" $TEST_IMG | _filter_qemu_io + +# The dirty bit must be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features + +echo +echo "== Repairing the image file must succeed ==" + +$QEMU_IMG check -r all $TEST_IMG + +# The dirty bit must not be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features + +echo +echo "== Data should still be accessible after repair ==" + +$QEMU_IO -c "read -P 0x5a 0 512" $TEST_IMG | _filter_qemu_io + +echo +echo "== Opening a dirty image read/write should repair it ==" + +IMGOPTS="compat=1.1,lazy_refcounts=on" +_make_test_img $size + +old_ulimit=$(ulimit -c) +ulimit -c 0 # do not produce a core dump on abort(3) +$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" $TEST_IMG | _filter_qemu_io +ulimit -c "$old_ulimit" + +# The dirty bit must be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features + +$QEMU_IO -c "write 0 512" $TEST_IMG | _filter_qemu_io + +# The dirty bit must not be set +./qcow2.py $TEST_IMG dump-header | grep incompatible_features + +echo +echo "== Creating an image file with lazy_refcounts=off ==" + +IMGOPTS="compat=1.1,lazy_refcounts=off" +_make_test_img $size + +old_ulimit=$(ulimit -c) +ulimit -c 0 # do not produce a core dump on abort(3) +$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" $TEST_IMG | _filter_qemu_io +ulimit -c "$old_ulimit" + +# The dirty bit must not be set since lazy_refcounts=off +./qcow2.py $TEST_IMG dump-header | grep incompatible_features +_check_test_img + +# success, all done +echo "*** done" +rm -f $seq.full +status=0 + diff --git a/tests/qemu-iotests/039.out b/tests/qemu-iotests/039.out new file mode 100644 index 0000000..8ad570d --- /dev/null +++ b/tests/qemu-iotests/039.out @@ -0,0 +1,53 @@ +QA output created by 039 + +== Checking that image is clean on shutdown == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x0 +No errors were found on the image. + +== Creating a dirty image file == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x1 +ERROR OFLAG_COPIED: offset=8000000000050000 refcount=0 +ERROR cluster 5 refcount=0 reference=1 + +2 errors were found on the image. +Data may be corrupted, or further writes to the image may corrupt it. + +== Read-only access must still work == +read 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x1 + +== Repairing the image file must succeed == +ERROR OFLAG_COPIED: offset=8000000000050000 refcount=0 +Repairing cluster 5 refcount=0 reference=1 +No errors were found on the image. +incompatible_features 0x0 + +== Data should still be accessible after repair == +read 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +== Opening a dirty image read/write should repair it == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x1 +ERROR OFLAG_COPIED: offset=8000000000050000 refcount=0 +Repairing cluster 5 refcount=0 reference=1 +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x0 + +== Creating an image file with lazy_refcounts=off == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 +wrote 512/512 bytes at offset 0 +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +incompatible_features 0x0 +No errors were found on the image. +*** done diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index 7a2c92b..ebb5ca4 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -45,3 +45,4 @@ 036 rw auto quick 037 rw auto backing 038 rw auto backing +039 rw auto -- cgit v1.1 From b0869a46b2dc2e6e4d6d6cca8bb5c5bf611f33d0 Mon Sep 17 00:00:00 2001 From: Kevin Wolf Date: Fri, 27 Jul 2012 15:14:28 +0200 Subject: qemu-iotests: Be more flexible with image creation options qemu-iotests already filters out image creation options that may be present or not in order to get the same output in both cases. However, often it only considers the default value of the option. Cover all valid values instead so that ./check -o name=value can be used successfull for all of them. Signed-off-by: Kevin Wolf --- tests/qemu-iotests/039.out | 6 +++--- tests/qemu-iotests/common.rc | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/039.out b/tests/qemu-iotests/039.out index 8ad570d..155a05e 100644 --- a/tests/qemu-iotests/039.out +++ b/tests/qemu-iotests/039.out @@ -1,14 +1,14 @@ QA output created by 039 == Checking that image is clean on shutdown == -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 wrote 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) incompatible_features 0x0 No errors were found on the image. == Creating a dirty image file == -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 wrote 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) incompatible_features 0x1 @@ -34,7 +34,7 @@ read 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) == Opening a dirty image read/write should repair it == -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 lazy_refcounts=on +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 wrote 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) incompatible_features 0x1 diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc index cc4e39b..7782808 100644 --- a/tests/qemu-iotests/common.rc +++ b/tests/qemu-iotests/common.rc @@ -110,11 +110,11 @@ _make_test_img() sed -e "s#$IMGFMT#IMGFMT#g" | \ sed -e "s# encryption=off##g" | \ sed -e "s# cluster_size=[0-9]\\+##g" | \ - sed -e "s# table_size=0##g" | \ + sed -e "s# table_size=[0-9]\\+##g" | \ sed -e "s# compat='[^']*'##g" | \ - sed -e "s# compat6=off##g" | \ - sed -e "s# static=off##g" | \ - sed -e "s# lazy_refcounts=off##g" + sed -e "s# compat6=\\(on\\|off\\)##g" | \ + sed -e "s# static=\\(on\\|off\\)##g" | \ + sed -e "s# lazy_refcounts=\\(on\\|off\\)##g" } _cleanup_test_img() -- cgit v1.1