From 88da4b043b4f91a265947149b1cd6758c046a4bd Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 13 Oct 2020 21:21:21 +0200 Subject: tests/Makefile.include: unbreak non-tcg builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove from check-block the requirement that all TARGET_DIRS are built. Reviewed-by: Daniel P. Berrangé Signed-off-by: Paolo Bonzini --- tests/Makefile.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/Makefile.include b/tests/Makefile.include index 5aca98e..4037490 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -140,7 +140,7 @@ QEMU_IOTESTS_HELPERS-$(CONFIG_LINUX) = tests/qemu-iotests/socket_scm_helper$(EXE check: check-block check-block: $(SRC_PATH)/tests/check-block.sh qemu-img$(EXESUF) \ qemu-io$(EXESUF) qemu-nbd$(EXESUF) $(QEMU_IOTESTS_HELPERS-y) \ - $(patsubst %-softmmu,qemu-system-%,$(filter %-softmmu,$(TARGET_DIRS))) + $(filter qemu-system-%, $(ninja-targets-c_LINKER) $(ninja-targets-cpp_LINKER)) @$< endif -- cgit v1.1 From 97d6efd0a3f3a08942de6c2aee5d2983c54ca84c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 14 Oct 2020 07:20:17 -0400 Subject: tests: add missing generated sources to testqapi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ninja notices them due to a different order in visiting the graph. Reviewed-by: Daniel P. Berrangé Signed-off-by: Paolo Bonzini --- tests/include/meson.build | 8 ++++---- tests/meson.build | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/tests/include/meson.build b/tests/include/meson.build index fea3a63..9abba30 100644 --- a/tests/include/meson.build +++ b/tests/include/meson.build @@ -10,7 +10,7 @@ test_qapi_outputs_extra = [ 'test-qapi-visit-sub-module.h', ] -test_qapi_outputs_extra = custom_target('QAPI test (include)', - output: test_qapi_outputs_extra, - input: test_qapi_files, - command: 'true') +test_qapi_files_extra = custom_target('QAPI test (include)', + output: test_qapi_outputs_extra, + input: test_qapi_files, + command: 'true') diff --git a/tests/meson.build b/tests/meson.build index bf47a38..afeb6be 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -56,8 +56,18 @@ test_qapi_files = custom_target('Test QAPI files', # perhaps change qapi_gen to replace / with _, like Meson itself does? subdir('include') -libtestqapi = static_library('testqapi', sources: [test_qapi_files, genh, test_qapi_outputs_extra]) -testqapi = declare_dependency(link_with: libtestqapi) +test_qapi_sources = [] +test_qapi_headers = [] +i = 0 +foreach o: test_qapi_files.to_list() + test_qapi_files_extra.to_list() + if o.full_path().endswith('.h') + test_qapi_headers += o + endif + test_qapi_sources += o +endforeach + +libtestqapi = static_library('testqapi', sources: [genh, test_qapi_sources]) +testqapi = declare_dependency(link_with: libtestqapi, sources: [genh, test_qapi_headers]) testblock = declare_dependency(dependencies: [block], sources: 'iothread.c') -- cgit v1.1 From f2f984a3b3bc8322df2efa3937bf11e8ea2bcaa5 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 14 Oct 2020 07:12:37 -0400 Subject: dockerfiles: enable Centos 8 PowerTools ninja is included in the CentOS PowerTools repository. Signed-off-by: Paolo Bonzini --- tests/docker/dockerfiles/centos8.docker | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker index 0fc2697..e29e965 100644 --- a/tests/docker/dockerfiles/centos8.docker +++ b/tests/docker/dockerfiles/centos8.docker @@ -28,5 +28,7 @@ ENV PACKAGES \ tar \ zlib-devel -RUN dnf install -y $PACKAGES +RUN dnf install -y dnf-plugins-core && \ + dnf config-manager --set-enabled PowerTools && \ + dnf install -y $PACKAGES RUN rpm -q $PACKAGES | sort > /packages.txt -- cgit v1.1 From 345d7053ca4a39b0496366f3c953ae2681570ce3 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 13 Aug 2020 09:58:50 -0400 Subject: add ninja to dockerfiles, CI configurations and test VMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Daniel P. Berrangé Acked-by: Alex Bennée Signed-off-by: Paolo Bonzini --- tests/docker/dockerfiles/centos7.docker | 1 + tests/docker/dockerfiles/centos8.docker | 1 + tests/docker/dockerfiles/debian10.docker | 1 + tests/docker/dockerfiles/fedora.docker | 1 + tests/docker/dockerfiles/travis.docker | 2 +- tests/docker/dockerfiles/ubuntu.docker | 1 + tests/docker/dockerfiles/ubuntu1804.docker | 1 + tests/docker/dockerfiles/ubuntu2004.docker | 1 + tests/vm/centos | 2 +- tests/vm/centos.aarch64 | 2 +- tests/vm/fedora | 2 +- tests/vm/freebsd | 1 + tests/vm/netbsd | 1 + tests/vm/openbsd | 1 + tests/vm/ubuntu.aarch64 | 2 +- tests/vm/ubuntu.i386 | 2 +- 16 files changed, 16 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/tests/docker/dockerfiles/centos7.docker b/tests/docker/dockerfiles/centos7.docker index 4627777..8b27372 100644 --- a/tests/docker/dockerfiles/centos7.docker +++ b/tests/docker/dockerfiles/centos7.docker @@ -27,6 +27,7 @@ ENV PACKAGES \ mesa-libEGL-devel \ mesa-libgbm-devel \ nettle-devel \ + ninja-build \ perl-Test-Harness \ pixman-devel \ python3 \ diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker index e29e965..585dfad 100644 --- a/tests/docker/dockerfiles/centos8.docker +++ b/tests/docker/dockerfiles/centos8.docker @@ -19,6 +19,7 @@ ENV PACKAGES \ make \ mesa-libEGL-devel \ nettle-devel \ + ninja-build \ perl-Test-Harness \ pixman-devel \ python36 \ diff --git a/tests/docker/dockerfiles/debian10.docker b/tests/docker/dockerfiles/debian10.docker index 1e4188b..21cc671 100644 --- a/tests/docker/dockerfiles/debian10.docker +++ b/tests/docker/dockerfiles/debian10.docker @@ -26,6 +26,7 @@ RUN apt update && \ gettext \ git \ libncurses5-dev \ + ninja-build \ pkg-config \ psmisc \ python3 \ diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index 85c9755..ac79d95 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -75,6 +75,7 @@ ENV PACKAGES \ mingw64-SDL2 \ ncurses-devel \ nettle-devel \ + ninja-build \ nss-devel \ numactl-devel \ perl \ diff --git a/tests/docker/dockerfiles/travis.docker b/tests/docker/dockerfiles/travis.docker index 5912825..cd1435a 100644 --- a/tests/docker/dockerfiles/travis.docker +++ b/tests/docker/dockerfiles/travis.docker @@ -9,7 +9,7 @@ ENV LC_ALL en_US.UTF-8 RUN sed -i "s/# deb-src/deb-src/" /etc/apt/sources.list RUN apt-get update RUN apt-get -y build-dep qemu -RUN apt-get -y install device-tree-compiler python3 python3-yaml dh-autoreconf gdb strace lsof net-tools gcovr +RUN apt-get -y install device-tree-compiler python3 python3-yaml dh-autoreconf gdb strace lsof net-tools gcovr ninja-build # Travis tools require PhantomJS / Neo4j / Maven accessible # in their PATH (QEMU build won't access them). ENV PATH /usr/local/phantomjs/bin:/usr/local/phantomjs:/usr/local/neo4j-3.2.7/bin:/usr/local/maven-3.5.2/bin:/usr/local/cmake-3.9.2/bin:/usr/local/clang-5.0.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/tests/docker/dockerfiles/ubuntu.docker b/tests/docker/dockerfiles/ubuntu.docker index b556ed1..b5ef7a8 100644 --- a/tests/docker/dockerfiles/ubuntu.docker +++ b/tests/docker/dockerfiles/ubuntu.docker @@ -60,6 +60,7 @@ ENV PACKAGES \ libxen-dev \ libzstd-dev \ make \ + ninja-build \ python3-yaml \ python3-sphinx \ sparse \ diff --git a/tests/docker/dockerfiles/ubuntu1804.docker b/tests/docker/dockerfiles/ubuntu1804.docker index a6a7617..9b0a19b 100644 --- a/tests/docker/dockerfiles/ubuntu1804.docker +++ b/tests/docker/dockerfiles/ubuntu1804.docker @@ -48,6 +48,7 @@ ENV PACKAGES \ make \ python3-yaml \ python3-sphinx \ + ninja-build \ sparse \ xfslibs-dev RUN apt-get update && \ diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker index f4b9556..17b37cd 100644 --- a/tests/docker/dockerfiles/ubuntu2004.docker +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -47,6 +47,7 @@ ENV PACKAGES flex bison \ libxen-dev \ libzstd-dev \ make \ + ninja-build \ python3-numpy \ python3-opencv \ python3-pil \ diff --git a/tests/vm/centos b/tests/vm/centos index 0ad4ecf..efe3dbb 100755 --- a/tests/vm/centos +++ b/tests/vm/centos @@ -42,7 +42,7 @@ class CentosVM(basevm.BaseVM): self.wait_ssh() self.ssh_root_check("touch /etc/cloud/cloud-init.disabled") self.ssh_root_check("yum update -y") - self.ssh_root_check("yum install -y docker make git python3") + self.ssh_root_check("yum install -y docker make ninja-build git python3") self.ssh_root_check("systemctl enable docker") self.ssh_root("poweroff") self.wait() diff --git a/tests/vm/centos.aarch64 b/tests/vm/centos.aarch64 index d5232ec..e687b93 100755 --- a/tests/vm/centos.aarch64 +++ b/tests/vm/centos.aarch64 @@ -23,7 +23,7 @@ import aarch64vm DEFAULT_CONFIG = { 'cpu' : "max", 'machine' : "virt,gic-version=max", - 'install_cmds' : "yum install -y make git python3 gcc gcc-c++ flex bison, "\ + 'install_cmds' : "yum install -y make ninja-build git python3 gcc gcc-c++ flex bison, "\ "yum install -y glib2-devel pixman-devel zlib-devel, "\ "yum install -y perl-Test-Harness, "\ "alternatives --set python /usr/bin/python3, "\ diff --git a/tests/vm/fedora b/tests/vm/fedora index b2b478f..b977efe 100755 --- a/tests/vm/fedora +++ b/tests/vm/fedora @@ -32,7 +32,7 @@ class FedoraVM(basevm.BaseVM): pkgs = [ # tools 'git-core', - 'gcc', 'binutils', 'make', + 'gcc', 'binutils', 'make', 'ninja-build', # perl 'perl-Test-Harness', diff --git a/tests/vm/freebsd b/tests/vm/freebsd index 5f866e0..04ee793 100755 --- a/tests/vm/freebsd +++ b/tests/vm/freebsd @@ -34,6 +34,7 @@ class FreeBSDVM(basevm.BaseVM): "bzip2", "python37", "py37-setuptools", + "ninja", # gnu tools "bash", diff --git a/tests/vm/netbsd b/tests/vm/netbsd index ffb65a8..447de97 100755 --- a/tests/vm/netbsd +++ b/tests/vm/netbsd @@ -32,6 +32,7 @@ class NetBSDVM(basevm.BaseVM): "xz", "python37", "py37-setuptools", + "ninja-build", # gnu tools "bash", diff --git a/tests/vm/openbsd b/tests/vm/openbsd index 8356646..ad882a7 100755 --- a/tests/vm/openbsd +++ b/tests/vm/openbsd @@ -31,6 +31,7 @@ class OpenBSDVM(basevm.BaseVM): "pkgconf", "bzip2", "xz", "py3-setuptools", + "ninja", # gnu tools "bash", diff --git a/tests/vm/ubuntu.aarch64 b/tests/vm/ubuntu.aarch64 index 21d454c..b291945 100755 --- a/tests/vm/ubuntu.aarch64 +++ b/tests/vm/ubuntu.aarch64 @@ -22,7 +22,7 @@ DEFAULT_CONFIG = { 'machine' : "virt,gic-version=3", 'install_cmds' : "apt-get update,"\ "apt-get build-dep -y --arch-only qemu,"\ - "apt-get install -y libfdt-dev pkg-config language-pack-en", + "apt-get install -y libfdt-dev pkg-config language-pack-en ninja-build", # We increase beyond the default time since during boot # it can take some time (many seconds) to log into the VM # especially using softmmu. diff --git a/tests/vm/ubuntu.i386 b/tests/vm/ubuntu.i386 index 5ce7261..47681b6 100755 --- a/tests/vm/ubuntu.i386 +++ b/tests/vm/ubuntu.i386 @@ -18,7 +18,7 @@ import ubuntuvm DEFAULT_CONFIG = { 'install_cmds' : "apt-get update,"\ "apt-get build-dep -y qemu,"\ - "apt-get install -y libfdt-dev language-pack-en", + "apt-get install -y libfdt-dev language-pack-en ninja-build", } class UbuntuX86VM(ubuntuvm.UbuntuVM): -- cgit v1.1 From 09e93326e448ab43fa26a9e2d9cc20ecf951f32b Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 13 Aug 2020 09:28:11 -0400 Subject: build: replace ninjatool with ninja Now that the build is done entirely by Meson, there is no need to keep the Makefile conversion. Instead, we can ask Ninja about the targets it exposes and forward them. The main advantages are, from smallest to largest: - reducing the possible namespace pollution within the Makefile - removal of a relatively large Python program - faster build because parsing Makefile.ninja is slower than parsing build.ninja; and faster build after Meson runs because we do not have to generate Makefile.ninja. - tracking of command lines, which provides more accurate rebuilds In addition the change removes the requirement for GNU make 3.82, which was annoying on Mac, and avoids bugs on Windows due to ninjatool not knowing how to convert Windows escapes to POSIX escapes. Signed-off-by: Paolo Bonzini --- tests/Makefile.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/Makefile.include b/tests/Makefile.include index 4037490..3a0524c 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -140,7 +140,7 @@ QEMU_IOTESTS_HELPERS-$(CONFIG_LINUX) = tests/qemu-iotests/socket_scm_helper$(EXE check: check-block check-block: $(SRC_PATH)/tests/check-block.sh qemu-img$(EXESUF) \ qemu-io$(EXESUF) qemu-nbd$(EXESUF) $(QEMU_IOTESTS_HELPERS-y) \ - $(filter qemu-system-%, $(ninja-targets-c_LINKER) $(ninja-targets-cpp_LINKER)) + $(filter qemu-system-%, $(ninja-targets)) @$< endif -- cgit v1.1 From 9dc6ee3fd78a478935eecf936cddd575c6dfb20a Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 16 Oct 2020 04:05:26 -0400 Subject: meson: move SPHINX_ARGS references within "if build_docs" Signed-off-by: Paolo Bonzini --- tests/qapi-schema/meson.build | 88 +++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 44 deletions(-) (limited to 'tests') diff --git a/tests/qapi-schema/meson.build b/tests/qapi-schema/meson.build index 1f222a7..304ef93 100644 --- a/tests/qapi-schema/meson.build +++ b/tests/qapi-schema/meson.build @@ -219,53 +219,53 @@ qapi_doc = custom_target('QAPI doc', '-p', 'doc-good-', '@INPUT0@' ], depend_files: qapi_gen_depends) -# Test the document-comment document generation code by running a test schema -# file through Sphinx's plain-text builder and comparing the result against -# a golden reference. This is in theory susceptible to failures if Sphinx -# changes its output, but the text output has historically been very stable -# (no changes between Sphinx 1.6 and 3.0), so it is a better bet than -# texinfo or HTML generation, both of which have had changes. We might -# need to add more sophisticated logic here in future for some sort of -# fuzzy comparison if future Sphinx versions produce different text, -# but for now the simple comparison suffices. -qapi_doc_out = custom_target('QAPI rST doc', - output: ['doc-good.txt'], - input: files('doc-good.json', 'doc-good.rst'), - build_by_default: build_docs, - depend_files: sphinx_extn_depends, - # We use -E to suppress Sphinx's caching, because - # we want it to always really run the QAPI doc - # generation code. It also means we don't - # clutter up the build dir with the cache. - command: [SPHINX_ARGS, - '-b', 'text', '-E', - '-c', meson.source_root() / 'docs', - '-D', 'master_doc=doc-good', - meson.current_source_dir(), - meson.current_build_dir()]) +if build_docs + # Test the document-comment document generation code by running a test schema + # file through Sphinx's plain-text builder and comparing the result against + # a golden reference. This is in theory susceptible to failures if Sphinx + # changes its output, but the text output has historically been very stable + # (no changes between Sphinx 1.6 and 3.0), so it is a better bet than + # texinfo or HTML generation, both of which have had changes. We might + # need to add more sophisticated logic here in future for some sort of + # fuzzy comparison if future Sphinx versions produce different text, + # but for now the simple comparison suffices. + qapi_doc_out = custom_target('QAPI rST doc', + output: ['doc-good.txt'], + input: files('doc-good.json', 'doc-good.rst'), + build_by_default: true, + depend_files: sphinx_extn_depends, + # We use -E to suppress Sphinx's caching, because + # we want it to always really run the QAPI doc + # generation code. It also means we don't + # clutter up the build dir with the cache. + command: [SPHINX_ARGS, + '-b', 'text', '-E', + '-c', meson.source_root() / 'docs', + '-D', 'master_doc=doc-good', + meson.current_source_dir(), + meson.current_build_dir()]) -# Fix possible inconsistency in line endings in generated output and -# in the golden reference (which could otherwise cause test failures -# on Windows hosts). Unfortunately diff --strip-trailing-cr -# is GNU-diff only. The odd-looking perl is because we must avoid -# using an explicit '\' character in the command arguments to -# a custom_target(), as Meson will unhelpfully replace it with a '/' -# (https://github.com/mesonbuild/meson/issues/1564) -qapi_doc_out_nocr = custom_target('QAPI rST doc newline-sanitized', - output: ['doc-good.txt.nocr'], - input: qapi_doc_out[0], - build_by_default: build_docs, - command: ['perl', '-pe', '$x = chr 13; s/$x$//', '@INPUT@'], - capture: true) + # Fix possible inconsistency in line endings in generated output and + # in the golden reference (which could otherwise cause test failures + # on Windows hosts). Unfortunately diff --strip-trailing-cr + # is GNU-diff only. The odd-looking perl is because we must avoid + # using an explicit '\' character in the command arguments to + # a custom_target(), as Meson will unhelpfully replace it with a '/' + # (https://github.com/mesonbuild/meson/issues/1564) + qapi_doc_out_nocr = custom_target('QAPI rST doc newline-sanitized', + output: ['doc-good.txt.nocr'], + input: qapi_doc_out[0], + build_by_default: true, + command: ['perl', '-pe', '$x = chr 13; s/$x$//', '@INPUT@'], + capture: true) -qapi_doc_ref_nocr = custom_target('QAPI rST doc reference newline-sanitized', - output: ['doc-good.ref.nocr'], - input: files('doc-good.txt'), - build_by_default: build_docs, - command: ['perl', '-pe', '$x = chr 13; s/$x$//', '@INPUT@'], - capture: true) + qapi_doc_ref_nocr = custom_target('QAPI rST doc reference newline-sanitized', + output: ['doc-good.ref.nocr'], + input: files('doc-good.txt'), + build_by_default: true, + command: ['perl', '-pe', '$x = chr 13; s/$x$//', '@INPUT@'], + capture: true) -if build_docs # "full_path()" needed here to work around # https://github.com/mesonbuild/meson/issues/7585 test('QAPI rST doc', diff, args: ['-u', qapi_doc_ref_nocr[0].full_path(), -- cgit v1.1 From fc69fa216cf52709b1279a592364e50c674db6ff Mon Sep 17 00:00:00 2001 From: Alexander Bulekov Date: Wed, 14 Oct 2020 10:21:57 -0400 Subject: fuzz: Disable QEMU's SIG{INT,HUP,TERM} handlers Prior to this patch, the only way I found to terminate the fuzzer was either to: 1. Explicitly specify the number of fuzzer runs with the -runs= flag 2. SIGKILL the process with "pkill -9 qemu-fuzz-*" or similar In addition to being annoying to deal with, SIGKILLing the process skips over any exit handlers(e.g. registered with atexit()). This is bad, since some fuzzers might create temporary files that should ideally be removed on exit using an exit handler. The only way to achieve a clean exit now is to specify -runs=N , but the desired "N" is tricky to identify prior to fuzzing. Why doesn't the process exit with standard SIGINT,SIGHUP,SIGTERM signals? QEMU installs its own handlers for these signals in os-posix.c:os_setup_signal_handling, which notify the main loop that an exit was requested. The fuzzer, however, does not run qemu_main_loop, which performs the main_loop_should_exit() check. This means that the fuzzer effectively ignores these signals. As we don't really care about cleanly stopping the disposable fuzzer "VM", this patch uninstalls QEMU's signal handlers. Thus, we can stop the fuzzer with SIG{INT,HUP,TERM} and the fuzzing code can optionally use atexit() to clean up temporary files/resources. Reviewed-by: Darren Kenny Signed-off-by: Alexander Bulekov Message-Id: <20201014142157.46028-1-alxndr@bu.edu> Signed-off-by: Paolo Bonzini --- tests/qtest/fuzz/fuzz.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'tests') diff --git a/tests/qtest/fuzz/fuzz.c b/tests/qtest/fuzz/fuzz.c index d926c49..eb00704 100644 --- a/tests/qtest/fuzz/fuzz.c +++ b/tests/qtest/fuzz/fuzz.c @@ -217,5 +217,13 @@ int LLVMFuzzerInitialize(int *argc, char ***argv, char ***envp) /* re-enable the rcu atfork, which was previously disabled in qemu_init */ rcu_enable_atfork(); + /* + * Disable QEMU's signal handlers, since we manually control the main_loop, + * and don't check for main_loop_should_exit + */ + signal(SIGINT, SIG_DFL); + signal(SIGHUP, SIG_DFL); + signal(SIGTERM, SIG_DFL); + return 0; } -- cgit v1.1