diff options
author | Richard Henderson <richard.henderson@linaro.org> | 2025-10-16 12:27:12 -0700 |
---|---|---|
committer | Richard Henderson <richard.henderson@linaro.org> | 2025-10-16 12:27:12 -0700 |
commit | 18f6f30b0089b470f3e737637a86dfb81ebd6eae (patch) | |
tree | 793e6ea727106ac2f3d24c69820f90f3eec2f156 | |
parent | 3ad3326bd658dd3bb210d5f2997e6b9a913ccb95 (diff) | |
parent | d6f7f9254e333c56226bb7051e74ea57daea2fff (diff) | |
download | qemu-master.zip qemu-master.tar.gz qemu-master.tar.bz2 |
* Improve cache handling for the msys2 CI and the functional asset cache
* Clean ups for some minor issues in functional tests
* Don't ignore errors of address_space_rw in s390x MMU code
# -----BEGIN PGP SIGNATURE-----
#
# iQJFBAABCgAvFiEEJ7iIR+7gJQEY8+q5LtnXdP5wLbUFAmjxHGwRHHRodXRoQHJl
# ZGhhdC5jb20ACgkQLtnXdP5wLbUSDw//dSoNzE+qqJ2EWIfzHN/N7iDRf4OWQY6X
# 12S7zfcBt0W7RESf6JGy0h3cwDcyUxhB7UtC9rvT8cYPFhun+7T07H72/Se+95+a
# d3Yih1HHfYKYN6Zl3DAzUfSUvLBorYI76Ab5yeTs5nCg3ewK6IWPvMA9pS+4P2Pj
# iLY+ycflsGd36tHrJbR/G0cda5p6jaxDzpLh/d+D7hh3XoeS+PZv2xeknsDvJlOs
# ykIrrB+XJ8x81NxAFdA3PSeEh7VU5IzVv+t9hSJiBdi98/dAzwfQdH6bTekA8JL0
# pzfT4A5cwoIEHg5yCqsAllG6+sa6RaVzt6tDeRw4UKFeUP33zFXCoVurUygR1TP7
# Bs7E4E4HbXDgSWvd/UFNtE+4MdYicwP/1qBootnJXc4/v8cFEZBVdOlnsLXDmg1S
# qPr4ITAQDE3LEq4vXITKLmUIZehLG2K6N32XalnK9XrQD6Flcvr1BwDepFAP9Gku
# zmQMkeES1F2MhGMtxPwAxWIMfqsRtaPNWGqH4tSlnBYslwqPWwz+XLDGKlUsPpV/
# kXV/TKkeE/bb0DO7WoLFaw3Q5F+Mdj6C0B2cevKVGCurH4wGLRLaIhzGl8J6ZSDH
# Lve4Fc3BvMMO3CFYezaRQcEls+w4cXkOtSTZNwame1Wk7cUiqq+ElmFttYSl4FSC
# WKyZHtapq1U=
# =Ml5x
# -----END PGP SIGNATURE-----
# gpg: Signature made Thu 16 Oct 2025 09:25:16 AM PDT
# gpg: using RSA key 27B88847EEE0250118F3EAB92ED9D774FE702DB5
# gpg: issuer "thuth@redhat.com"
# gpg: Good signature from "Thomas Huth <th.huth@gmx.de>" [unknown]
# gpg: aka "Thomas Huth <thuth@redhat.com>" [unknown]
# gpg: aka "Thomas Huth <th.huth@posteo.de>" [unknown]
# gpg: aka "Thomas Huth <huth@tuxfamily.org>" [unknown]
# gpg: WARNING: The key's User ID is not certified with a trusted signature!
# gpg: There is no indication that the signature belongs to the owner.
# Primary key fingerprint: 27B8 8847 EEE0 2501 18F3 EAB9 2ED9 D774 FE70 2DB5
* tag 'pull-request-2025-10-16' of https://gitlab.com/thuth/qemu:
target/s390x/mmu_helper: Do not ignore address_space_rw() errors
target/s390x/mmu_helper: Simplify s390_cpu_virt_mem_rw() logic
tests/functional: ensure GDB client is stopped on error
tests/functional: remove use of getLogger in reverse debuging
tests/functional/alpha: Remove superfluous fetch() line from the clipper test
tests: Evict stale files in the functional download cache after a while
tests/functional: Set current time stamp of assets when using them
gitlab: purge msys pacman cache
tests/functional/aarch64: Drop some sbsaref_alpine tests
python/qemu: Replace some remaining "avocados" with "functional tests"
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
-rw-r--r-- | .gitlab-ci.d/windows.yml | 1 | ||||
-rw-r--r-- | MAINTAINERS | 1 | ||||
-rw-r--r-- | python/qemu/machine/README.rst | 2 | ||||
-rw-r--r-- | python/qemu/utils/README.rst | 2 | ||||
-rwxr-xr-x | scripts/clean_functional_cache.py | 45 | ||||
-rw-r--r-- | target/s390x/mmu_helper.c | 17 | ||||
-rw-r--r-- | tests/Makefile.include | 1 | ||||
-rwxr-xr-x | tests/functional/aarch64/test_sbsaref_alpine.py | 6 | ||||
-rwxr-xr-x | tests/functional/alpha/test_clipper.py | 1 | ||||
-rw-r--r-- | tests/functional/qemu_test/asset.py | 13 | ||||
-rw-r--r-- | tests/functional/reverse_debugging.py | 65 |
11 files changed, 105 insertions, 49 deletions
diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index 1e6a01b..6e1135d 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -87,6 +87,7 @@ msys2-64bit: mingw-w64-x86_64-pkgconf mingw-w64-x86_64-python mingw-w64-x86_64-zstd" + - .\msys64\usr\bin\bash -lc "pacman -Sc --noconfirm" - Write-Output "Running build at $(Get-Date -Format u)" - $env:JOBS = $(.\msys64\usr\bin\bash -lc nproc) - $env:CHERE_INVOKING = 'yes' # Preserve the current working directory diff --git a/MAINTAINERS b/MAINTAINERS index 0c76696..667acd9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4397,6 +4397,7 @@ M: Thomas Huth <thuth@redhat.com> R: Philippe Mathieu-Daudé <philmd@linaro.org> R: Daniel P. Berrange <berrange@redhat.com> F: docs/devel/testing/functional.rst +F: scripts/clean_functional_cache.py F: tests/functional/qemu_test/ Windows Hosted Continuous Integration diff --git a/python/qemu/machine/README.rst b/python/qemu/machine/README.rst index 8de2c3d..6554c69 100644 --- a/python/qemu/machine/README.rst +++ b/python/qemu/machine/README.rst @@ -2,7 +2,7 @@ qemu.machine package ==================== This package provides core utilities used for testing and debugging -QEMU. It is used by the iotests, vm tests, avocado tests, and several +QEMU. It is used by the iotests, vm tests, functional tests, and several other utilities in the ./scripts directory. It is not a fully-fledged SDK and it is subject to change at any time. diff --git a/python/qemu/utils/README.rst b/python/qemu/utils/README.rst index d5f2da1..5027f0b 100644 --- a/python/qemu/utils/README.rst +++ b/python/qemu/utils/README.rst @@ -2,6 +2,6 @@ qemu.utils package ================== This package provides miscellaneous utilities used for testing and -debugging QEMU. It is used primarily by the vm and avocado tests. +debugging QEMU. It is used primarily by the vm and functional tests. See the documentation in ``__init__.py`` for more information. diff --git a/scripts/clean_functional_cache.py b/scripts/clean_functional_cache.py new file mode 100755 index 0000000..c3370ff --- /dev/null +++ b/scripts/clean_functional_cache.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +"""Delete stale assets from the download cache of the functional tests""" + +import os +import stat +import sys +import time +from pathlib import Path + + +cache_dir_env = os.getenv('QEMU_TEST_CACHE_DIR') +if cache_dir_env: + cache_dir = Path(cache_dir_env, "download") +else: + cache_dir = Path(Path("~").expanduser(), ".cache", "qemu", "download") + +if not cache_dir.exists(): + print(f"Cache dir {cache_dir} does not exist!", file=sys.stderr) + sys.exit(1) + +os.chdir(cache_dir) + +for file in cache_dir.iterdir(): + # Only consider the files that use a sha256 as filename: + if len(file.name) != 64: + continue + + try: + timestamp = int(file.with_suffix(".stamp").read_text()) + except FileNotFoundError: + # Assume it's an old file that was already in the cache before we + # added the code for evicting stale assets. Use the release date + # of QEMU v10.1 as a default timestamp. + timestamp = time.mktime((2025, 8, 26, 0, 0, 0, 0, 0, 0)) + + age = time.time() - timestamp + + # Delete files older than half of a year (183 days * 24h * 60m * 60s) + if age > 15811200: + print(f"Removing {cache_dir}/{file.name}.") + file.chmod(stat.S_IWRITE) + file.unlink() diff --git a/target/s390x/mmu_helper.c b/target/s390x/mmu_helper.c index 487c41b..3b1e75f 100644 --- a/target/s390x/mmu_helper.c +++ b/target/s390x/mmu_helper.c @@ -541,21 +541,28 @@ int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf, pages = g_malloc(nr_pages * sizeof(*pages)); ret = translate_pages(cpu, laddr, nr_pages, pages, is_write, &tec); - if (ret) { - trigger_access_exception(&cpu->env, ret, tec); - } else if (hostbuf != NULL) { + if (ret == 0 && hostbuf != NULL) { AddressSpace *as = CPU(cpu)->as; /* Copy data by stepping through the area page by page */ for (i = 0; i < nr_pages; i++) { + MemTxResult res; + currlen = MIN(len, TARGET_PAGE_SIZE - (laddr % TARGET_PAGE_SIZE)); - address_space_rw(as, pages[i] | (laddr & ~TARGET_PAGE_MASK), - attrs, hostbuf, currlen, is_write); + res = address_space_rw(as, pages[i] | (laddr & ~TARGET_PAGE_MASK), + attrs, hostbuf, currlen, is_write); + if (res != MEMTX_OK) { + ret = PGM_ADDRESSING; + break; + } laddr += currlen; hostbuf += currlen; len -= currlen; } } + if (ret) { + trigger_access_exception(&cpu->env, ret, tec); + } g_free(pages); return ret; diff --git a/tests/Makefile.include b/tests/Makefile.include index e47ef4d..d4dfbf3 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -111,6 +111,7 @@ $(FUNCTIONAL_TARGETS): check-venv .PHONY: check-functional check-functional: check-venv @$(NINJA) precache-functional + @$(PYTHON) $(SRC_PATH)/scripts/clean_functional_cache.py @QEMU_TEST_NO_DOWNLOAD=1 $(MAKE) SPEED=thorough check-func check-func-quick .PHONY: check-func check-func-quick diff --git a/tests/functional/aarch64/test_sbsaref_alpine.py b/tests/functional/aarch64/test_sbsaref_alpine.py index abb8f51..be84b7a 100755 --- a/tests/functional/aarch64/test_sbsaref_alpine.py +++ b/tests/functional/aarch64/test_sbsaref_alpine.py @@ -41,15 +41,9 @@ class Aarch64SbsarefAlpine(QemuSystemTest): self.vm.launch() wait_for_console_pattern(self, "Welcome to Alpine Linux 3.17") - def test_sbsaref_alpine_linux_cortex_a57(self): - self.boot_alpine_linux("cortex-a57") - def test_sbsaref_alpine_linux_default_cpu(self): self.boot_alpine_linux() - def test_sbsaref_alpine_linux_max_pauth_off(self): - self.boot_alpine_linux("max,pauth=off") - def test_sbsaref_alpine_linux_max_pauth_impdef(self): self.boot_alpine_linux("max,pauth-impdef=on") diff --git a/tests/functional/alpha/test_clipper.py b/tests/functional/alpha/test_clipper.py index c5d7181..d2a4c2a 100755 --- a/tests/functional/alpha/test_clipper.py +++ b/tests/functional/alpha/test_clipper.py @@ -17,7 +17,6 @@ class AlphaClipperTest(LinuxKernelTest): def test_alpha_clipper(self): self.set_machine('clipper') - kernel_path = self.ASSET_KERNEL.fetch() uncompressed_kernel = self.uncompress(self.ASSET_KERNEL, format="gz") diff --git a/tests/functional/qemu_test/asset.py b/tests/functional/qemu_test/asset.py index f666125..ab3a7bb 100644 --- a/tests/functional/qemu_test/asset.py +++ b/tests/functional/qemu_test/asset.py @@ -10,6 +10,7 @@ import logging import os import stat import sys +import time import unittest import urllib.request from time import sleep @@ -113,6 +114,16 @@ class Asset: self.log.debug("Time out while waiting for %s!", tmp_cache_file) raise + def _save_time_stamp(self): + ''' + Update the time stamp of the asset in the cache. Unfortunately, we + cannot use the modification or access time of the asset file itself, + since e.g. the functional jobs in the gitlab CI reload the files + from the gitlab cache and thus always have recent file time stamps, + so we have to save our asset time stamp to a separate file instead. + ''' + self.cache_file.with_suffix(".stamp").write_text(f"{int(time.time())}") + def fetch(self): if not self.cache_dir.exists(): self.cache_dir.mkdir(parents=True, exist_ok=True) @@ -120,6 +131,7 @@ class Asset: if self.valid(): self.log.debug("Using cached asset %s for %s", self.cache_file, self.url) + self._save_time_stamp() return str(self.cache_file) if not self.fetchable(): @@ -208,6 +220,7 @@ class Asset: tmp_cache_file.unlink() raise AssetError(self, "Hash does not match %s" % self.hash) tmp_cache_file.replace(self.cache_file) + self._save_time_stamp() # Remove write perms to stop tests accidentally modifying them os.chmod(self.cache_file, stat.S_IRUSR | stat.S_IRGRP) diff --git a/tests/functional/reverse_debugging.py b/tests/functional/reverse_debugging.py index 68cfcb3..86fca8d 100644 --- a/tests/functional/reverse_debugging.py +++ b/tests/functional/reverse_debugging.py @@ -36,14 +36,13 @@ class ReverseDebugging(LinuxKernelTest): STEPS = 10 def run_vm(self, record, shift, args, replay_path, image_path, port): - logger = logging.getLogger('replay') vm = self.get_vm(name='record' if record else 'replay') vm.set_console() if record: - logger.info('recording the execution...') + self.log.info('recording the execution...') mode = 'record' else: - logger.info('replaying the execution...') + self.log.info('replaying the execution...') mode = 'replay' vm.add_args('-gdb', 'tcp::%d' % port, '-S') vm.add_args('-icount', 'shift=%s,rr=%s,rrfile=%s,rrsnapshot=init' % @@ -68,10 +67,8 @@ class ReverseDebugging(LinuxKernelTest): def reverse_debugging(self, gdb_arch, shift=7, args=None): from qemu_test import GDB - logger = logging.getLogger('replay') - # create qcow2 for snapshots - logger.info('creating qcow2 image for VM snapshots') + self.log.info('creating qcow2 image for VM snapshots') image_path = os.path.join(self.workdir, 'disk.qcow2') qemu_img = get_qemu_img(self) if qemu_img is None: @@ -79,7 +76,7 @@ class ReverseDebugging(LinuxKernelTest): 'create the temporary qcow2 image') out = check_output([qemu_img, 'create', '-f', 'qcow2', image_path, '128M'], encoding='utf8') - logger.info("qemu-img: %s" % out) + self.log.info("qemu-img: %s" % out) replay_path = os.path.join(self.workdir, 'replay.bin') @@ -90,7 +87,7 @@ class ReverseDebugging(LinuxKernelTest): last_icount = self.vm_get_icount(vm) vm.shutdown() - logger.info("recorded log with %s+ steps" % last_icount) + self.log.info("recorded log with %s+ steps" % last_icount) # replay and run debug commands with Ports() as ports: @@ -98,9 +95,16 @@ class ReverseDebugging(LinuxKernelTest): vm = self.run_vm(False, shift, args, replay_path, image_path, port) try: - logger.info('Connecting to gdbstub...') - self.reverse_debugging_run(vm, port, gdb_arch, last_icount) - logger.info('Test passed.') + self.log.info('Connecting to gdbstub...') + gdb_cmd = os.getenv('QEMU_TEST_GDB') + gdb = GDB(gdb_cmd) + try: + self.reverse_debugging_run(gdb, vm, port, gdb_arch, last_icount) + finally: + self.log.info('exiting gdb and qemu') + gdb.exit() + vm.shutdown() + self.log.info('Test passed.') except GDB.TimeoutError: # Convert a GDB timeout exception into a unittest failure exception. raise self.failureException("Timeout while connecting to or " @@ -110,12 +114,7 @@ class ReverseDebugging(LinuxKernelTest): # skipTest(), etc. raise - def reverse_debugging_run(self, vm, port, gdb_arch, last_icount): - logger = logging.getLogger('replay') - - gdb_cmd = os.getenv('QEMU_TEST_GDB') - gdb = GDB(gdb_cmd) - + def reverse_debugging_run(self, gdb, vm, port, gdb_arch, last_icount): r = gdb.cli("set architecture").get_log() if gdb_arch not in r: self.skipTest(f"GDB does not support arch '{gdb_arch}'") @@ -135,43 +134,43 @@ class ReverseDebugging(LinuxKernelTest): gdb.cli("set debug remote 0") - logger.info('stepping forward') + self.log.info('stepping forward') steps = [] # record first instruction addresses for _ in range(self.STEPS): pc = self.get_pc(gdb) - logger.info('saving position %x' % pc) + self.log.info('saving position %x' % pc) steps.append(pc) gdb.cli("stepi") # visit the recorded instruction in reverse order - logger.info('stepping backward') + self.log.info('stepping backward') for addr in steps[::-1]: - logger.info('found position %x' % addr) + self.log.info('found position %x' % addr) gdb.cli("reverse-stepi") pc = self.get_pc(gdb) if pc != addr: - logger.info('Invalid PC (read %x instead of %x)' % (pc, addr)) + self.log.info('Invalid PC (read %x instead of %x)' % (pc, addr)) self.fail('Reverse stepping failed!') # visit the recorded instruction in forward order - logger.info('stepping forward') + self.log.info('stepping forward') for addr in steps: - logger.info('found position %x' % addr) + self.log.info('found position %x' % addr) pc = self.get_pc(gdb) if pc != addr: - logger.info('Invalid PC (read %x instead of %x)' % (pc, addr)) + self.log.info('Invalid PC (read %x instead of %x)' % (pc, addr)) self.fail('Forward stepping failed!') gdb.cli("stepi") # set breakpoints for the instructions just stepped over - logger.info('setting breakpoints') + self.log.info('setting breakpoints') for addr in steps: gdb.cli(f"break *{hex(addr)}") # this may hit a breakpoint if first instructions are executed # again - logger.info('continuing execution') + self.log.info('continuing execution') vm.qmp('replay-break', icount=last_icount - 1) # continue - will return after pausing # This can stop at the end of the replay-break and gdb gets a SIGINT, @@ -180,12 +179,12 @@ class ReverseDebugging(LinuxKernelTest): gdb.cli("continue") if self.vm_get_icount(vm) == last_icount - 1: - logger.info('reached the end (icount %s)' % (last_icount - 1)) + self.log.info('reached the end (icount %s)' % (last_icount - 1)) else: - logger.info('hit a breakpoint again at %x (icount %s)' % + self.log.info('hit a breakpoint again at %x (icount %s)' % (self.get_pc(gdb), self.vm_get_icount(vm))) - logger.info('running reverse continue to reach %x' % steps[-1]) + self.log.info('running reverse continue to reach %x' % steps[-1]) # reverse continue - will return after stopping at the breakpoint gdb.cli("reverse-continue") @@ -195,8 +194,4 @@ class ReverseDebugging(LinuxKernelTest): if pc != steps[-1]: self.fail("'reverse-continue' did not hit the first PC in reverse order!") - logger.info('successfully reached %x' % steps[-1]) - - logger.info('exiting gdb and qemu') - gdb.exit() - vm.shutdown() + self.log.info('successfully reached %x' % steps[-1]) |