diff options
62 files changed, 2162 insertions, 1348 deletions
diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml index a6cfe9b..7edb50b 100644 --- a/.gitlab-ci.d/buildtest-template.yml +++ b/.gitlab-ci.d/buildtest-template.yml @@ -12,12 +12,12 @@ - mkdir build - cd build - ../configure --enable-werror --disable-docs --enable-fdt=system - ${LD_JOBS:+--meson=git} ${TARGETS:+--target-list="$TARGETS"} + ${TARGETS:+--target-list="$TARGETS"} $CONFIGURE_ARGS || { cat config.log meson-logs/meson-log.txt && exit 1; } - if test -n "$LD_JOBS"; then - ../meson/meson.py configure . -Dbackend_max_links="$LD_JOBS" ; + pyvenv/bin/meson configure . -Dbackend_max_links="$LD_JOBS" ; fi || exit 1; - make -j"$JOBS" - if test -n "$MAKE_CHECK_ARGS"; diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index bb3650a..307cba1 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -103,7 +103,7 @@ crash-test-debian: script: - cd build - make NINJA=":" check-venv - - tests/venv/bin/python3 scripts/device-crash-test -q --tcg-only ./qemu-system-i386 + - pyvenv/bin/python3 scripts/device-crash-test -q --tcg-only ./qemu-system-i386 build-system-fedora: extends: @@ -146,8 +146,8 @@ crash-test-fedora: script: - cd build - make NINJA=":" check-venv - - tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-ppc - - tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-riscv32 + - pyvenv/bin/python3 scripts/device-crash-test -q ./qemu-system-ppc + - pyvenv/bin/python3 scripts/device-crash-test -q ./qemu-system-riscv32 build-system-centos: extends: diff --git a/.gitmodules b/.gitmodules index 6ce5bf4..2a3a120 100644 --- a/.gitmodules +++ b/.gitmodules @@ -49,9 +49,6 @@ [submodule "roms/qboot"] path = roms/qboot url = https://gitlab.com/qemu-project/qboot.git -[submodule "meson"] - path = meson - url = https://gitlab.com/qemu-project/meson.git [submodule "roms/vbootrom"] path = roms/vbootrom url = https://gitlab.com/qemu-project/vbootrom.git @@ -26,7 +26,7 @@ quiet-command-run = $(if $(V),,$(if $2,printf " %-7s %s\n" $2 $3 && ))$1 quiet-@ = $(if $(V),,@) quiet-command = $(quiet-@)$(call quiet-command-run,$1,$2,$3) -UNCHECKED_GOALS := %clean TAGS cscope ctags dist \ +UNCHECKED_GOALS := TAGS gtags cscope ctags dist \ help check-help print-% \ docker docker-% vm-help vm-test vm-build-% @@ -176,10 +176,8 @@ plugins: endif # $(CONFIG_PLUGIN) else # config-host.mak does not exist -config-host.mak: ifneq ($(filter-out $(UNCHECKED_GOALS),$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail)) - @echo "Please call configure before running make!" - @exit 1 +$(error Please call configure before running make) endif endif # config-host.mak does not exist diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index cf3a88d..7679f39 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -1361,6 +1361,10 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml, */ if (kvm_state->kvm_dirty_ring_size) { kvm_dirty_ring_reap_locked(kvm_state, NULL); + if (kvm_state->kvm_dirty_ring_with_bitmap) { + kvm_slot_sync_dirty_pages(mem); + kvm_slot_get_dirty_log(kvm_state, mem); + } } else { kvm_slot_get_dirty_log(kvm_state, mem); } @@ -1458,6 +1462,69 @@ static int kvm_dirty_ring_reaper_init(KVMState *s) return 0; } +static int kvm_dirty_ring_init(KVMState *s) +{ + uint32_t ring_size = s->kvm_dirty_ring_size; + uint64_t ring_bytes = ring_size * sizeof(struct kvm_dirty_gfn); + unsigned int capability = KVM_CAP_DIRTY_LOG_RING; + int ret; + + s->kvm_dirty_ring_size = 0; + s->kvm_dirty_ring_bytes = 0; + + /* Bail if the dirty ring size isn't specified */ + if (!ring_size) { + return 0; + } + + /* + * Read the max supported pages. Fall back to dirty logging mode + * if the dirty ring isn't supported. + */ + ret = kvm_vm_check_extension(s, capability); + if (ret <= 0) { + capability = KVM_CAP_DIRTY_LOG_RING_ACQ_REL; + ret = kvm_vm_check_extension(s, capability); + } + + if (ret <= 0) { + warn_report("KVM dirty ring not available, using bitmap method"); + return 0; + } + + if (ring_bytes > ret) { + error_report("KVM dirty ring size %" PRIu32 " too big " + "(maximum is %ld). Please use a smaller value.", + ring_size, (long)ret / sizeof(struct kvm_dirty_gfn)); + return -EINVAL; + } + + ret = kvm_vm_enable_cap(s, capability, 0, ring_bytes); + if (ret) { + error_report("Enabling of KVM dirty ring failed: %s. " + "Suggested minimum value is 1024.", strerror(-ret)); + return -EIO; + } + + /* Enable the backup bitmap if it is supported */ + ret = kvm_vm_check_extension(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP); + if (ret > 0) { + ret = kvm_vm_enable_cap(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP, 0); + if (ret) { + error_report("Enabling of KVM dirty ring's backup bitmap failed: " + "%s. ", strerror(-ret)); + return -EIO; + } + + s->kvm_dirty_ring_with_bitmap = true; + } + + s->kvm_dirty_ring_size = ring_size; + s->kvm_dirty_ring_bytes = ring_bytes; + + return 0; +} + static void kvm_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -1563,7 +1630,7 @@ static void kvm_log_sync(MemoryListener *listener, kvm_slots_unlock(); } -static void kvm_log_sync_global(MemoryListener *l) +static void kvm_log_sync_global(MemoryListener *l, bool last_stage) { KVMMemoryListener *kml = container_of(l, KVMMemoryListener, listener); KVMState *s = kvm_state; @@ -1582,6 +1649,12 @@ static void kvm_log_sync_global(MemoryListener *l) mem = &kml->slots[i]; if (mem->memory_size && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { kvm_slot_sync_dirty_pages(mem); + + if (s->kvm_dirty_ring_with_bitmap && last_stage && + kvm_slot_get_dirty_log(s, mem)) { + kvm_slot_sync_dirty_pages(mem); + } + /* * This is not needed by KVM_GET_DIRTY_LOG because the * ioctl will unconditionally overwrite the whole region. @@ -2521,35 +2594,9 @@ static int kvm_init(MachineState *ms) * Enable KVM dirty ring if supported, otherwise fall back to * dirty logging mode */ - if (s->kvm_dirty_ring_size > 0) { - uint64_t ring_bytes; - - ring_bytes = s->kvm_dirty_ring_size * sizeof(struct kvm_dirty_gfn); - - /* Read the max supported pages */ - ret = kvm_vm_check_extension(s, KVM_CAP_DIRTY_LOG_RING); - if (ret > 0) { - if (ring_bytes > ret) { - error_report("KVM dirty ring size %" PRIu32 " too big " - "(maximum is %ld). Please use a smaller value.", - s->kvm_dirty_ring_size, - (long)ret / sizeof(struct kvm_dirty_gfn)); - ret = -EINVAL; - goto err; - } - - ret = kvm_vm_enable_cap(s, KVM_CAP_DIRTY_LOG_RING, 0, ring_bytes); - if (ret) { - error_report("Enabling of KVM dirty ring failed: %s. " - "Suggested minimum value is 1024.", strerror(-ret)); - goto err; - } - - s->kvm_dirty_ring_bytes = ring_bytes; - } else { - warn_report("KVM dirty ring not available, using bitmap method"); - s->kvm_dirty_ring_size = 0; - } + ret = kvm_dirty_ring_init(s); + if (ret < 0) { + goto err; } /* @@ -3710,6 +3757,7 @@ static void kvm_accel_instance_init(Object *obj) s->kernel_irqchip_split = ON_OFF_AUTO_AUTO; /* KVM dirty ring is by default off */ s->kvm_dirty_ring_size = 0; + s->kvm_dirty_ring_with_bitmap = false; s->notify_vmexit = NOTIFY_VMEXIT_OPTION_RUN; s->notify_window = 0; s->xen_version = 0; diff --git a/accel/tcg/tcg-accel-ops-rr.c b/accel/tcg/tcg-accel-ops-rr.c index 5788efa..b6d10fa 100644 --- a/accel/tcg/tcg-accel-ops-rr.c +++ b/accel/tcg/tcg-accel-ops-rr.c @@ -72,11 +72,13 @@ static void rr_kick_next_cpu(void) { CPUState *cpu; do { - cpu = qatomic_mb_read(&rr_current_cpu); + cpu = qatomic_read(&rr_current_cpu); if (cpu) { cpu_exit(cpu); } - } while (cpu != qatomic_mb_read(&rr_current_cpu)); + /* Finish kicking this cpu before reading again. */ + smp_mb(); + } while (cpu != qatomic_read(&rr_current_cpu)); } static void rr_kick_thread(void *opaque) @@ -241,8 +243,9 @@ static void *rr_cpu_thread_fn(void *arg) } while (cpu && cpu_work_list_empty(cpu) && !cpu->exit_request) { - + /* Store rr_current_cpu before evaluating cpu_can_run(). */ qatomic_mb_set(&rr_current_cpu, cpu); + current_cpu = cpu; qemu_clock_enable(QEMU_CLOCK_VIRTUAL, @@ -280,7 +283,7 @@ static void *rr_cpu_thread_fn(void *arg) cpu = CPU_NEXT(cpu); } /* while (cpu && !cpu->exit_request).. */ - /* Does not need qatomic_mb_set because a spurious wakeup is okay. */ + /* Does not need a memory barrier because a spurious wakeup is okay. */ qatomic_set(&rr_current_cpu, NULL); if (cpu && cpu->exit_request) { @@ -79,7 +79,6 @@ fi TMPB="qemu-conf" TMPC="${TMPDIR1}/${TMPB}.c" TMPO="${TMPDIR1}/${TMPB}.o" -TMPM="${TMPDIR1}/${TMPB}.m" TMPE="${TMPDIR1}/${TMPB}.exe" rm -f config.log @@ -125,62 +124,20 @@ lines: ${BASH_LINENO[*]}" $compiler "$@" >> config.log 2>&1 || return $? } -do_compiler_werror() { - # Run the compiler, capturing its output to the log. First argument - # is compiler binary to execute. - compiler="$1" - shift - if test -n "$BASH_VERSION"; then eval ' - echo >>config.log " -funcs: ${FUNCNAME[*]} -lines: ${BASH_LINENO[*]}" - '; fi - echo $compiler "$@" >> config.log - $compiler "$@" >> config.log 2>&1 || return $? - # Test passed. If this is an --enable-werror build, rerun - # the test with -Werror and bail out if it fails. This - # makes warning-generating-errors in configure test code - # obvious to developers. - if test "$werror" != "yes"; then - return 0 - fi - # Don't bother rerunning the compile if we were already using -Werror - case "$*" in - *-Werror*) - return 0 - ;; - esac - echo $compiler -Werror "$@" >> config.log - $compiler -Werror "$@" >> config.log 2>&1 && return $? - error_exit "configure test passed without -Werror but failed with -Werror." \ - "This is probably a bug in the configure script. The failing command" \ - "will be at the bottom of config.log." \ - "You can run configure with --disable-werror to bypass this check." -} - do_cc() { - do_compiler_werror "$cc" $CPU_CFLAGS "$@" -} - -do_objc() { - do_compiler_werror "$objcc" $CPU_CFLAGS "$@" -} - -# Append $2 to the variable named $1, with space separation -add_to() { - eval $1=\${$1:+\"\$$1 \"}\$2 + do_compiler "$cc" $CPU_CFLAGS "$@" } compile_object() { local_cflags="$1" - do_cc $CFLAGS $EXTRA_CFLAGS $CONFIGURE_CFLAGS $QEMU_CFLAGS $local_cflags -c -o $TMPO $TMPC + do_cc $CFLAGS $EXTRA_CFLAGS $local_cflags -c -o $TMPO $TMPC } compile_prog() { local_cflags="$1" local_ldflags="$2" - do_cc $CFLAGS $EXTRA_CFLAGS $CONFIGURE_CFLAGS $QEMU_CFLAGS $local_cflags -o $TMPE $TMPC \ - $LDFLAGS $EXTRA_LDFLAGS $CONFIGURE_LDFLAGS $QEMU_LDFLAGS $local_ldflags + do_cc $CFLAGS $EXTRA_CFLAGS $local_cflags -o $TMPE $TMPC \ + $LDFLAGS $EXTRA_LDFLAGS $local_ldflags } # symbolically link $1 to $2. Portable version of "ln -sf". @@ -220,32 +177,18 @@ then error_exit "main directory cannot contain spaces nor colons" fi +# parse CC options first; some compiler tests are used to establish +# some defaults, based on the host environment + # default parameters cpu="" -static="no" cross_compile="no" cross_prefix="" host_cc="cc" -stack_protector="" -safe_stack="" -use_containers="yes" -gdb_bin=$(command -v "gdb-multiarch" || command -v "gdb") -gdb_arches="" -glib_has_gslice="no" - -if test -e "$source_path/.git" -then - git_submodules_action="update" -else - git_submodules_action="ignore" -fi - -git_submodules="ui/keycodemapdb" -git="git" - -# Don't accept a target_list environment variable. -unset target_list -unset target_list_exclude +EXTRA_CFLAGS="" +EXTRA_CXXFLAGS="" +EXTRA_OBJCFLAGS="" +EXTRA_LDFLAGS="" # Default value for a variable defining feature "foo". # * foo="no" feature will only be used if --enable-foo arg is given @@ -258,54 +201,8 @@ unset target_list_exclude # Always add --enable-foo and --disable-foo command line args. # Distributions want to ensure that several features are compiled in, and it # is impossible without a --enable-foo that exits if a feature is not found. - default_feature="" -# parse CC options second -for opt do - optarg=$(expr "x$opt" : 'x[^=]*=\(.*\)') - case "$opt" in - --without-default-features) - default_feature="no" - ;; - esac -done - -EXTRA_CFLAGS="" -EXTRA_CXXFLAGS="" -EXTRA_OBJCFLAGS="" -EXTRA_LDFLAGS="" - -debug_tcg="no" -sanitizers="no" -tsan="no" -fortify_source="yes" -EXESUF="" -modules="no" -prefix="/usr/local" -qemu_suffix="qemu" -softmmu="yes" -linux_user="" -bsd_user="" -pie="" -coroutine="" -plugins="$default_feature" -meson="" -ninja="" -bindir="bin" -skip_meson=no -vfio_user_server="disabled" - -# The following Meson options are handled manually (still they -# are included in the automatically generated help message) - -# 1. Track which submodules are needed -fdt="auto" - -# 2. Automatically enable/disable other options -tcg="auto" -cfi="false" -# parse CC options second for opt do optarg=$(expr "x$opt" : 'x[^=]*=\(.*\)') case "$opt" in @@ -344,9 +241,60 @@ for opt do --cross-prefix-*) cc_arch=${opt#--cross-prefix-}; cc_arch=${cc_arch%%=*} eval "cross_prefix_${cc_arch}=\$optarg" ;; + --without-default-features) default_feature="no" + ;; esac done + +if test -e "$source_path/.git" +then + git_submodules_action="update" +else + git_submodules_action="ignore" +fi + +git_submodules="ui/keycodemapdb" +git="git" +debug_tcg="no" +docs="auto" +EXESUF="" +prefix="/usr/local" +qemu_suffix="qemu" +softmmu="yes" +linux_user="" +bsd_user="" +plugins="$default_feature" +ninja="" +python= +pypi="enabled" +bindir="bin" +skip_meson=no +vfio_user_server="disabled" +use_containers="yes" +gdb_bin=$(command -v "gdb-multiarch" || command -v "gdb") +gdb_arches="" +werror="" + +# Don't accept a target_list environment variable. +unset target_list +unset target_list_exclude + +# The following Meson options are handled manually (still they +# are included in the automatically generated help message) + +# 1. Track which submodules are needed +fdt="auto" + +# 2. Automatically enable/disable other options +tcg="auto" +cfi="false" + +# 3. Need to check for -static-pie before Meson runs. Also, +# Meson has PIE as a boolean rather than enabled/disabled/auto. +pie="" +static="no" + # Preferred compiler: # ${CC} (if set) # ${cross_prefix}gcc (if cross-prefix specified) @@ -390,28 +338,9 @@ strip="${STRIP-${cross_prefix}strip}" widl="${WIDL-${cross_prefix}widl}" windres="${WINDRES-${cross_prefix}windres}" windmc="${WINDMC-${cross_prefix}windmc}" -pkg_config_exe="${PKG_CONFIG-${cross_prefix}pkg-config}" -query_pkg_config() { - "${pkg_config_exe}" ${QEMU_PKG_CONFIG_FLAGS} "$@" -} -pkg_config=query_pkg_config +pkg_config="${PKG_CONFIG-${cross_prefix}pkg-config}" sdl2_config="${SDL2_CONFIG-${cross_prefix}sdl2-config}" -# default flags for all hosts -# We use -fwrapv to tell the compiler that we require a C dialect where -# left shift of signed integers is well defined and has the expected -# 2s-complement style results. (Both clang and gcc agree that it -# provides these semantics.) -QEMU_CFLAGS="-fno-strict-aliasing -fno-common -fwrapv" -QEMU_CFLAGS="-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE $QEMU_CFLAGS" - -QEMU_LDFLAGS= - -# Flags that are needed during configure but later taken care of by Meson -CONFIGURE_CFLAGS="-std=gnu11 -Wall" -CONFIGURE_LDFLAGS= - - check_define() { cat > $TMPC <<EOF #if !defined($1) @@ -422,14 +351,6 @@ EOF compile_object } -check_include() { -cat > $TMPC <<EOF -#include <$1> -int main(void) { return 0; } -EOF - compile_object -} - write_c_skeleton() { cat > $TMPC <<EOF int main(void) { return 0; } @@ -498,21 +419,13 @@ openbsd) darwin) bsd="yes" darwin="yes" - # Disable attempts to use ObjectiveC features in os/object.h since they - # won't work when we're compiling with gcc as a C compiler. - QEMU_CFLAGS="-DOS_OBJECT_USE_OBJC=0 $QEMU_CFLAGS" ;; sunos) solaris="yes" make="${MAKE-gmake}" -# needed for CMSG_ macros in sys/socket.h - QEMU_CFLAGS="-D_XOPEN_SOURCE=600 $QEMU_CFLAGS" -# needed for TIOCWIN* defines in termios.h - QEMU_CFLAGS="-D__EXTENSIONS__ $QEMU_CFLAGS" ;; haiku) pie="no" - QEMU_CFLAGS="-DB_USE_POSITIVE_POSIX_ERRORS -D_BSD_SOURCE -fPIC $QEMU_CFLAGS" ;; linux) linux="yes" @@ -617,18 +530,16 @@ esac check_py_version() { - # We require python >= 3.6. + # We require python >= 3.7. # NB: a True python conditional creates a non-zero return code (Failure) - "$1" -c 'import sys; sys.exit(sys.version_info < (3,6))' + "$1" -c 'import sys; sys.exit(sys.version_info < (3,7))' } -python= first_python= if test -z "${PYTHON}"; then - explicit_python=no # A bare 'python' is traditionally python 2.x, but some distros # have it as python 3.x, so check in both places. - for binary in python3 python python3.11 python3.10 python3.9 python3.8 python3.7 python3.6; do + for binary in python3 python python3.11 python3.10 python3.9 python3.8 python3.7; do if has "$binary"; then python=$(command -v "$binary") if check_py_version "$python"; then @@ -644,7 +555,6 @@ else # Same as above, but only check the environment variable. has "${PYTHON}" || error_exit "The PYTHON environment variable does not point to an executable" python=$(command -v "$PYTHON") - explicit_python=yes if check_py_version "$python"; then # This one is good. first_python= @@ -666,15 +576,11 @@ done if test "$mingw32" = "yes" ; then EXESUF=".exe" - # MinGW needs -mthreads for TLS and macro _MT. - CONFIGURE_CFLAGS="-mthreads $CONFIGURE_CFLAGS" prefix="/qemu" bindir="" qemu_suffix="" fi -werror="" - meson_option_build_array() { printf '[' (if test "$targetos" = windows; then @@ -729,12 +635,10 @@ for opt do ;; --install=*) ;; - --python=*) python="$optarg" ; explicit_python=yes + --python=*) python="$optarg" ;; --skip-meson) skip_meson=yes ;; - --meson=*) meson="$optarg" - ;; --ninja=*) ninja="$optarg" ;; --smbd=*) smbd="$optarg" @@ -751,15 +655,9 @@ for opt do ;; --cross-prefix-*) ;; - --enable-debug-info) meson_option_add -Ddebug=true - ;; - --disable-debug-info) meson_option_add -Ddebug=false - ;; - --enable-modules) - modules="yes" + --enable-docs) docs=enabled ;; - --disable-modules) - modules="no" + --disable-docs) docs=disabled ;; --cpu=*) ;; @@ -791,9 +689,7 @@ for opt do ;; --without-default-features) # processed above ;; - --static) - static="yes" - QEMU_PKG_CONFIG_FLAGS="--static $QEMU_PKG_CONFIG_FLAGS" + --static) static="yes" ;; --bindir=*) bindir="$optarg" ;; @@ -819,15 +715,6 @@ for opt do meson_option_parse --enable-debug-graph-lock "" meson_option_parse --enable-debug-mutex "" meson_option_add -Doptimization=0 - fortify_source="no" - ;; - --enable-sanitizers) sanitizers="yes" - ;; - --disable-sanitizers) sanitizers="no" - ;; - --enable-tsan) tsan="yes" - ;; - --disable-tsan) tsan="no" ;; --disable-tcg) tcg="disabled" plugins="no" @@ -859,14 +746,6 @@ for opt do ;; --disable-werror) werror="no" ;; - --enable-stack-protector) stack_protector="yes" - ;; - --disable-stack-protector) stack_protector="no" - ;; - --enable-safe-stack) safe_stack="yes" - ;; - --disable-safe-stack) safe_stack="no" - ;; --enable-cfi) cfi="true"; meson_option_add -Db_lto=true @@ -881,13 +760,15 @@ for opt do ;; --enable-fdt=*) fdt="$optarg" ;; - --with-coroutine=*) coroutine="$optarg" - ;; --with-git=*) git="$optarg" ;; --with-git-submodules=*) git_submodules_action="$optarg" ;; + --disable-pypi) pypi="disabled" + ;; + --enable-pypi) pypi="enabled" + ;; --enable-plugins) if test "$mingw32" = "yes"; then error_exit "TCG plugins not currently supported on Windows platforms" else @@ -1019,7 +900,6 @@ Advanced options (experts only): --cross-prefix-ARCH=PREFIX cross compiler prefix when building ARCH guest test cases --make=MAKE use specified make [$make] --python=PYTHON use specified python [$python] - --meson=MESON use specified meson [$meson] --ninja=NINJA use specified ninja [$ninja] --smbd=SMBD use specified smbd [$smbd] --with-git=GIT use specified git [$git] @@ -1035,13 +915,8 @@ Advanced options (experts only): desired devices in configs/devices/) --with-devices-ARCH=NAME override default configs/devices --enable-debug enable common debug build options - --enable-sanitizers enable default sanitizers - --enable-tsan enable thread sanitizer --disable-werror disable compilation abort on warning - --disable-stack-protector disable compiler-provided stack protection --cpu=CPU Build for host CPU [$cpu] - --with-coroutine=BACKEND coroutine backend. Supported options: - ucontext, sigaltstack, windows --enable-plugins enable plugins via shared library loading --disable-containers don't use containers for cross-building @@ -1054,11 +929,7 @@ cat << EOF linux-user all linux usermode emulation targets bsd-user all BSD usermode emulation targets pie Position Independent Executables - modules modules support (non-Windows) debug-tcg TCG debugging (default is disabled) - debug-info debugging information - safe-stack SafeStack Stack Smash Protection. Depends on - clang/llvm and requires coroutine backend ucontext. NOTE: The object files are built at the place where configure is launched EOF @@ -1073,7 +944,7 @@ then # If first_python is set, there was a binary somewhere even though # it was not suitable. Use it for the error message. if test -n "$first_python"; then - error_exit "Cannot use '$first_python', Python >= 3.6 is required." \ + error_exit "Cannot use '$first_python', Python >= 3.7 is required." \ "Use --python=/path/to/python to specify a supported Python." else error_exit "Python not found. Use --python=/path/to/python" @@ -1086,65 +957,86 @@ then fi if ! check_py_version "$python"; then - error_exit "Cannot use '$python', Python >= 3.6 is required." \ - "Use --python=/path/to/python to specify a supported Python." + error_exit "Cannot use '$python', Python >= 3.7 is required." \ + "Use --python=/path/to/python to specify a supported Python." \ + "Maybe try:" \ + " openSUSE Leap 15.3+: zypper install python39" \ + " CentOS 8: dnf install python38" fi -# Resolve PATH + suppress writing compiled files -python="$(command -v "$python") -B" - -has_meson() { - local python_dir=$(dirname "$python") - # PEP405: pyvenv.cfg is either adjacent to the Python executable - # or one directory above - if test -f $python_dir/pyvenv.cfg || test -f $python_dir/../pyvenv.cfg; then - # Ensure that Meson and Python come from the same virtual environment - test -x "$python_dir/meson" && - test "$(command -v meson)" -ef "$python_dir/meson" - else - has meson - fi -} +# Resolve PATH +python="$(command -v "$python")" -if test -z "$meson"; then - if test "$explicit_python" = no && has_meson && version_ge "$(meson --version)" 0.61.5; then - meson=meson - elif test "$git_submodules_action" != 'ignore' ; then - meson=git - elif test -e "${source_path}/meson/meson.py" ; then - meson=internal - else - if test "$explicit_python" = yes; then - error_exit "--python requires using QEMU's embedded Meson distribution, but it was not found." - else - error_exit "Meson not found. Use --meson=/path/to/meson" - fi - fi -else - # Meson uses its own Python interpreter to invoke other Python scripts, - # but the user wants to use the one they specified with --python. - # - # We do not want to override the distro Python interpreter (and sometimes - # cannot: for example in Homebrew /usr/bin/meson is a bash script), so - # just require --meson=git|internal together with --python. - if test "$explicit_python" = yes; then - case "$meson" in - git | internal) ;; - *) error_exit "--python requires using QEMU's embedded Meson distribution." ;; - esac - fi +# Create a Python virtual environment using our configured python. +# The stdout of this script will be the location of a symlink that +# points to the configured Python. +# Entry point scripts for pip, meson, and sphinx are generated if those +# packages are present. + +# Defaults assumed for now: +# - venv is cleared if it exists already; +# - venv is allowed to use system packages; +# - all setup can be performed offline; +# - missing packages may be fetched from PyPI, +# unless --disable-pypi is passed. +# - pip is not installed into the venv when possible, +# but ensurepip is called as a fallback when necessary. + +echo "python determined to be '$python'" +echo "python version: $($python --version)" + +python="$($python -B "${source_path}/python/scripts/mkvenv.py" create pyvenv)" +if test "$?" -ne 0 ; then + error_exit "python venv creation failed" fi -if test "$meson" = git; then - git_submodules="${git_submodules} meson" +# Suppress writing compiled files +python="$python -B" +mkvenv="$python ${source_path}/python/scripts/mkvenv.py" + +mkvenv_flags="" +if test "$pypi" = "enabled" ; then + mkvenv_flags="--online" fi -case "$meson" in - git | internal) - meson="$python ${source_path}/meson/meson.py" - ;; - *) meson=$(command -v "$meson") ;; -esac +if ! $mkvenv ensure \ + $mkvenv_flags \ + --dir "${source_path}/python/wheels" \ + --diagnose "meson" \ + "meson>=0.63.0" ; +then + exit 1 +fi + +# At this point, we expect Meson to be installed and available. +# We expect mkvenv or pip to have created pyvenv/bin/meson for us. +# We ignore PATH completely here: we want to use the venv's Meson +# *exclusively*. + +meson="$(cd pyvenv/bin; pwd)/meson" + +# Conditionally ensure Sphinx is installed. + +mkvenv_flags="" +if test "$pypi" = "enabled" -a "$docs" = "enabled" ; then + mkvenv_flags="--online" +fi + +if test "$docs" != "disabled" ; then + if ! $mkvenv ensure \ + $mkvenv_flags \ + --diagnose "sphinx-build" \ + "sphinx>=1.6.0" "sphinx-rtd-theme>=0.5.0"; + then + if test "$docs" = "enabled" ; then + exit 1 + fi + echo "Sphinx not found/usable, disabling docs." + docs=disabled + else + docs=enabled + fi +fi # Probe for ninja @@ -1160,20 +1052,6 @@ if test -z "$ninja"; then fi fi -# Check that the C compiler works. Doing this here before testing -# the host CPU ensures that we had a valid CC to autodetect the -# $cpu var (and we should bail right here if that's not the case). -# It also allows the help message to be printed without a CC. -write_c_skeleton; -if compile_object ; then - : C compiler works ok -else - error_exit "\"$cc\" either does not exist or does not work" -fi -if ! compile_prog ; then - error_exit "\"$cc\" cannot build an executable (is your linker broken?)" -fi - # Consult white-list to determine whether to enable werror # by default. Only enable by default for git builds if test -z "$werror" ; then @@ -1193,147 +1071,7 @@ if test "$targetos" = "bogus"; then error_exit "Unrecognized host OS (uname -s reports '$(uname -s)')" fi -# Check whether the compiler matches our minimum requirements: -cat > $TMPC << EOF -#if defined(__clang_major__) && defined(__clang_minor__) -# ifdef __apple_build_version__ -# if __clang_major__ < 12 || (__clang_major__ == 12 && __clang_minor__ < 0) -# error You need at least XCode Clang v12.0 to compile QEMU -# endif -# else -# if __clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 0) -# error You need at least Clang v10.0 to compile QEMU -# endif -# endif -#elif defined(__GNUC__) && defined(__GNUC_MINOR__) -# if __GNUC__ < 7 || (__GNUC__ == 7 && __GNUC_MINOR__ < 4) -# error You need at least GCC v7.4.0 to compile QEMU -# endif -#else -# error You either need GCC or Clang to compiler QEMU -#endif -int main (void) { return 0; } -EOF -if ! compile_prog "" "" ; then - error_exit "You need at least GCC v7.4 or Clang v10.0 (or XCode Clang v12.0)" -fi - -# Accumulate -Wfoo and -Wno-bar separately. -# We will list all of the enable flags first, and the disable flags second. -# Note that we do not add -Werror, because that would enable it for all -# configure tests. If a configure test failed due to -Werror this would -# just silently disable some features, so it's too error prone. - -warn_flags= -add_to warn_flags -Wundef -add_to warn_flags -Wwrite-strings -add_to warn_flags -Wmissing-prototypes -add_to warn_flags -Wstrict-prototypes -add_to warn_flags -Wredundant-decls -add_to warn_flags -Wold-style-declaration -add_to warn_flags -Wold-style-definition -add_to warn_flags -Wtype-limits -add_to warn_flags -Wformat-security -add_to warn_flags -Wformat-y2k -add_to warn_flags -Winit-self -add_to warn_flags -Wignored-qualifiers -add_to warn_flags -Wempty-body -add_to warn_flags -Wnested-externs -add_to warn_flags -Wendif-labels -add_to warn_flags -Wexpansion-to-defined -add_to warn_flags -Wimplicit-fallthrough=2 -add_to warn_flags -Wmissing-format-attribute - -if test "$targetos" != "darwin"; then - add_to warn_flags -Wthread-safety -fi - -nowarn_flags= -add_to nowarn_flags -Wno-initializer-overrides -add_to nowarn_flags -Wno-missing-include-dirs -add_to nowarn_flags -Wno-shift-negative-value -add_to nowarn_flags -Wno-string-plus-int -add_to nowarn_flags -Wno-typedef-redefinition -add_to nowarn_flags -Wno-tautological-type-limit-compare -add_to nowarn_flags -Wno-psabi -add_to nowarn_flags -Wno-gnu-variable-sized-type-not-at-end - -gcc_flags="$warn_flags $nowarn_flags" - -cc_has_warning_flag() { - write_c_skeleton; - - # Use the positive sense of the flag when testing for -Wno-wombat - # support (gcc will happily accept the -Wno- form of unknown - # warning options). - optflag="$(echo $1 | sed -e 's/^-Wno-/-W/')" - compile_prog "-Werror $optflag" "" -} - -objcc_has_warning_flag() { - cat > $TMPM <<EOF -int main(void) { return 0; } -EOF - - # Use the positive sense of the flag when testing for -Wno-wombat - # support (gcc will happily accept the -Wno- form of unknown - # warning options). - optflag="$(echo $1 | sed -e 's/^-Wno-/-W/')" - do_objc -Werror $optflag \ - $OBJCFLAGS $EXTRA_OBJCFLAGS $CONFIGURE_OBJCFLAGS $QEMU_OBJCFLAGS \ - -o $TMPE $TMPM $QEMU_LDFLAGS -} - -for flag in $gcc_flags; do - if cc_has_warning_flag $flag ; then - QEMU_CFLAGS="$QEMU_CFLAGS $flag" - fi - if objcc_has_warning_flag $flag ; then - QEMU_OBJCFLAGS="$QEMU_OBJCFLAGS $flag" - fi -done - -if test "$stack_protector" != "no"; then - cat > $TMPC << EOF -int main(int argc, char *argv[]) -{ - char arr[64], *p = arr, *c = argv[argc - 1]; - while (*c) { - *p++ = *c++; - } - return 0; -} -EOF - gcc_flags="-fstack-protector-strong -fstack-protector-all" - sp_on=0 - for flag in $gcc_flags; do - # We need to check both a compile and a link, since some compiler - # setups fail only on a .c->.o compile and some only at link time - if compile_object "-Werror $flag" && - compile_prog "-Werror $flag" ""; then - QEMU_CFLAGS="$QEMU_CFLAGS $flag" - QEMU_LDFLAGS="$QEMU_LDFLAGS $flag" - sp_on=1 - break - fi - done - if test "$stack_protector" = yes; then - if test $sp_on = 0; then - error_exit "Stack protector not supported" - fi - fi -fi - -# Our module code doesn't support Windows -if test "$modules" = "yes" && test "$mingw32" = "yes" ; then - error_exit "Modules are not available for Windows" -fi - -# Static linking is not possible with plugins, modules or PIE if test "$static" = "yes" ; then - if test "$modules" = "yes" ; then - error_exit "static and modules are mutually incompatible" - fi if test "$plugins" = "yes"; then error_exit "static and plugins are mutually incompatible" else @@ -1353,59 +1091,26 @@ static THREAD int tls_var; int main(void) { return tls_var; } EOF -# Meson currently only handles pie as a boolean for now so if we have -# explicitly disabled PIE we need to extend our cflags because it wont. if test "$static" = "yes"; then if test "$pie" != "no" && compile_prog "-Werror -fPIE -DPIE" "-static-pie"; then - CONFIGURE_CFLAGS="-fPIE -DPIE $CONFIGURE_CFLAGS" pie="yes" elif test "$pie" = "yes"; then error_exit "-static-pie not available due to missing toolchain support" else pie="no" - QEMU_CFLAGS="-fno-pie $QEMU_CFLAGS" fi -elif test "$pie" = "no"; then - if compile_prog "-Werror -fno-pie" "-no-pie"; then - CONFIGURE_CFLAGS="-fno-pie $CONFIGURE_CFLAGS" - CONFIGURE_LDFLAGS="-no-pie $CONFIGURE_LDFLAGS" - QEMU_CFLAGS="-fno-pie -no-pie $QEMU_CFLAGS" +elif test "$pie" != "no"; then + if compile_prog "-Werror -fPIE -DPIE" "-pie"; then + pie="yes" + elif test "$pie" = "yes"; then + error_exit "PIE not available due to missing toolchain support" + else + echo "Disabling PIE due to missing toolchain support" + pie="no" fi -elif compile_prog "-Werror -fPIE -DPIE" "-pie"; then - CONFIGURE_CFLAGS="-fPIE -DPIE $CONFIGURE_CFLAGS" - CONFIGURE_LDFLAGS="-pie $CONFIGURE_LDFLAGS" - pie="yes" -elif test "$pie" = "yes"; then - error_exit "PIE not available due to missing toolchain support" -else - echo "Disabling PIE due to missing toolchain support" - pie="no" fi ########################################## -# __sync_fetch_and_and requires at least -march=i486. Many toolchains -# use i686 as default anyway, but for those that don't, an explicit -# specification is necessary - -if test "$cpu" = "i386"; then - cat > $TMPC << EOF -static int sfaa(int *ptr) -{ - return __sync_fetch_and_and(ptr, 0); -} - -int main(void) -{ - int val = 42; - val = __sync_val_compare_and_swap(&val, 0, 1); - sfaa(&val); - return val; -} -EOF - if ! compile_prog "" "" ; then - QEMU_CFLAGS="-march=i486 $QEMU_CFLAGS" - fi -fi if test -z "${target_list+xxx}" ; then default_targets=yes @@ -1478,102 +1183,6 @@ EOF fi ########################################## -# pkg-config probe - -if ! has "$pkg_config_exe"; then - error_exit "pkg-config binary '$pkg_config_exe' not found" -fi - -########################################## -# glib support probe - -# When bumping glib_req_ver, please check also whether we should increase -# the _WIN32_WINNT setting in osdep.h according to the value from glib -glib_req_ver=2.56 -glib_modules=gthread-2.0 -if test "$modules" = yes; then - glib_modules="$glib_modules gmodule-export-2.0" -elif test "$plugins" = "yes"; then - glib_modules="$glib_modules gmodule-no-export-2.0" -fi - -for i in $glib_modules; do - if $pkg_config --atleast-version=$glib_req_ver $i; then - glib_cflags=$($pkg_config --cflags $i) - glib_libs=$($pkg_config --libs $i) - else - error_exit "glib-$glib_req_ver $i is required to compile QEMU" - fi -done - -# Check whether glib has gslice, which we have to avoid for correctness. -# TODO: remove this check and the corresponding workaround (qtree) when -# the minimum supported glib is >= $glib_dropped_gslice_version. -glib_dropped_gslice_version=2.75.3 -for i in $glib_modules; do - if ! $pkg_config --atleast-version=$glib_dropped_gslice_version $i; then - glib_has_gslice="yes" - break - fi -done - -glib_bindir="$($pkg_config --variable=bindir glib-2.0)" -if test -z "$glib_bindir" ; then - glib_bindir="$($pkg_config --variable=prefix glib-2.0)"/bin -fi - -# This workaround is required due to a bug in pkg-config file for glib as it -# doesn't define GLIB_STATIC_COMPILATION for pkg-config --static - -if test "$static" = yes && test "$mingw32" = yes; then - glib_cflags="-DGLIB_STATIC_COMPILATION $glib_cflags" -fi - -# Sanity check that the current size_t matches the -# size that glib thinks it should be. This catches -# problems on multi-arch where people try to build -# 32-bit QEMU while pointing at 64-bit glib headers -cat > $TMPC <<EOF -#include <glib.h> -#include <unistd.h> - -#define QEMU_BUILD_BUG_ON(x) \ - typedef char qemu_build_bug_on[(x)?-1:1] __attribute__((unused)); - -int main(void) { - QEMU_BUILD_BUG_ON(sizeof(size_t) != GLIB_SIZEOF_SIZE_T); - return 0; -} -EOF - -if ! compile_prog "$glib_cflags" "$glib_libs" ; then - error_exit "sizeof(size_t) doesn't match GLIB_SIZEOF_SIZE_T."\ - "You probably need to set PKG_CONFIG_LIBDIR"\ - "to point to the right pkg-config files for your"\ - "build target" -fi - -# Silence clang warnings triggered by glib < 2.57.2 -cat > $TMPC << EOF -#include <glib.h> -typedef struct Foo { - int i; -} Foo; -static void foo_free(Foo *f) -{ - g_free(f); -} -G_DEFINE_AUTOPTR_CLEANUP_FUNC(Foo, foo_free) -int main(void) { return 0; } -EOF -if ! compile_prog "$glib_cflags -Werror" "$glib_libs" ; then - if cc_has_warning_flag "-Wno-unused-function"; then - glib_cflags="$glib_cflags -Wno-unused-function" - CONFIGURE_CFLAGS="$CONFIGURE_CFLAGS -Wno-unused-function" - fi -fi - -########################################## # fdt probe case "$fdt" in @@ -1583,123 +1192,6 @@ case "$fdt" in ;; esac -########################################## -# check and set a backend for coroutine - -# We prefer ucontext, but it's not always possible. The fallback -# is sigcontext. On Windows the only valid backend is the Windows -# specific one. - -ucontext_works=no -if test "$darwin" != "yes"; then - cat > $TMPC << EOF -#include <ucontext.h> -#ifdef __stub_makecontext -#error Ignoring glibc stub makecontext which will always fail -#endif -int main(void) { makecontext(0, 0, 0); return 0; } -EOF - if compile_prog "" "" ; then - ucontext_works=yes - fi -fi - -if test "$coroutine" = ""; then - if test "$mingw32" = "yes"; then - coroutine=win32 - elif test "$ucontext_works" = "yes"; then - coroutine=ucontext - else - coroutine=sigaltstack - fi -else - case $coroutine in - windows) - if test "$mingw32" != "yes"; then - error_exit "'windows' coroutine backend only valid for Windows" - fi - # Unfortunately the user visible backend name doesn't match the - # coroutine-*.c filename for this case, so we have to adjust it here. - coroutine=win32 - ;; - ucontext) - if test "$ucontext_works" != "yes"; then - error_exit "'ucontext' backend requested but makecontext not available" - fi - ;; - sigaltstack) - if test "$mingw32" = "yes"; then - error_exit "only the 'windows' coroutine backend is valid for Windows" - fi - ;; - *) - error_exit "unknown coroutine backend $coroutine" - ;; - esac -fi - -################################################## -# SafeStack - - -if test "$safe_stack" = "yes"; then -cat > $TMPC << EOF -int main(void) -{ -#if ! __has_feature(safe_stack) -#error SafeStack Disabled -#endif - return 0; -} -EOF - flag="-fsanitize=safe-stack" - # Check that safe-stack is supported and enabled. - if compile_prog "-Werror $flag" "$flag"; then - # Flag needed both at compilation and at linking - QEMU_CFLAGS="$QEMU_CFLAGS $flag" - QEMU_LDFLAGS="$QEMU_LDFLAGS $flag" - else - error_exit "SafeStack not supported by your compiler" - fi - if test "$coroutine" != "ucontext"; then - error_exit "SafeStack is only supported by the coroutine backend ucontext" - fi -else -cat > $TMPC << EOF -int main(void) -{ -#if defined(__has_feature) -#if __has_feature(safe_stack) -#error SafeStack Enabled -#endif -#endif - return 0; -} -EOF -if test "$safe_stack" = "no"; then - # Make sure that safe-stack is disabled - if ! compile_prog "-Werror" ""; then - # SafeStack was already enabled, try to explicitly remove the feature - flag="-fno-sanitize=safe-stack" - if ! compile_prog "-Werror $flag" "$flag"; then - error_exit "Configure cannot disable SafeStack" - fi - QEMU_CFLAGS="$QEMU_CFLAGS $flag" - QEMU_LDFLAGS="$QEMU_LDFLAGS $flag" - fi -else # "$safe_stack" = "" - # Set safe_stack to yes or no based on pre-existing flags - if compile_prog "-Werror" ""; then - safe_stack="no" - else - safe_stack="yes" - if test "$coroutine" != "ucontext"; then - error_exit "SafeStack is only supported by the coroutine backend ucontext" - fi - fi -fi -fi - ######################################## # check if ccache is interfering with # semantic analysis of macros @@ -1728,87 +1220,6 @@ if ! compile_object "-Werror"; then ccache_cpp2=yes fi -################################################# -# clang does not support glibc + FORTIFY_SOURCE. - -if test "$fortify_source" != "no"; then - if echo | $cc -dM -E - | grep __clang__ > /dev/null 2>&1 ; then - fortify_source="no"; - elif test -n "$cxx" && has $cxx && - echo | $cxx -dM -E - | grep __clang__ >/dev/null 2>&1 ; then - fortify_source="no"; - else - fortify_source="yes" - fi -fi - -########################################## -# checks for sanitizers - -have_asan=no -have_ubsan=no -have_asan_iface_h=no -have_asan_iface_fiber=no - -if test "$sanitizers" = "yes" ; then - write_c_skeleton - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=address" ""; then - have_asan=yes - fi - - # we could use a simple skeleton for flags checks, but this also - # detect the static linking issue of ubsan, see also: - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84285 - cat > $TMPC << EOF -int main(int argc, char **argv) -{ - return argc + 1; -} -EOF - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=undefined" ""; then - have_ubsan=yes - fi - - if check_include "sanitizer/asan_interface.h" ; then - have_asan_iface_h=yes - fi - - cat > $TMPC << EOF -#include <sanitizer/asan_interface.h> -int main(void) { - __sanitizer_start_switch_fiber(0, 0, 0); - return 0; -} -EOF - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=address" "" ; then - have_asan_iface_fiber=yes - fi -fi - -# Thread sanitizer is, for now, much noisier than the other sanitizers; -# keep it separate until that is not the case. -if test "$tsan" = "yes" && test "$sanitizers" = "yes"; then - error_exit "TSAN is not supported with other sanitiziers." -fi -have_tsan=no -have_tsan_iface_fiber=no -if test "$tsan" = "yes" ; then - write_c_skeleton - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then - have_tsan=yes - fi - cat > $TMPC << EOF -#include <sanitizer/tsan_interface.h> -int main(void) { - __tsan_create_fiber(0); - return 0; -} -EOF - if compile_prog "$CPU_CFLAGS -Werror -fsanitize=thread" "" ; then - have_tsan_iface_fiber=yes - fi -fi - ########################################## # functions to probe cross compilers @@ -1913,13 +1324,6 @@ probe_target_compiler() { container_cross_ranlib= container_cross_strip= - # We shall skip configuring the target compiler if the user didn't - # bother enabling an appropriate guest. This avoids building - # extraneous firmware images and tests. - if test "${target_list#*$1}" = "$1"; then - return 1 - fi - target_arch=${1%%-*} case $target_arch in aarch64) container_hosts="x86_64 aarch64" ;; @@ -2234,42 +1638,6 @@ case "$vfio_user_server" in ;; esac -########################################## -# End of CC checks -# After here, no more $cc or $ld runs - -write_c_skeleton - -if test "$fortify_source" = "yes" ; then - QEMU_CFLAGS="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 $QEMU_CFLAGS" -fi - -if test "$have_asan" = "yes"; then - QEMU_CFLAGS="-fsanitize=address $QEMU_CFLAGS" - QEMU_LDFLAGS="-fsanitize=address $QEMU_LDFLAGS" - if test "$have_asan_iface_h" = "no" ; then - echo "ASAN build enabled, but ASAN header missing." \ - "Without code annotation, the report may be inferior." - elif test "$have_asan_iface_fiber" = "no" ; then - echo "ASAN build enabled, but ASAN header is too old." \ - "Without code annotation, the report may be inferior." - fi -fi -if test "$have_tsan" = "yes" ; then - if test "$have_tsan_iface_fiber" = "yes" ; then - QEMU_CFLAGS="-fsanitize=thread $QEMU_CFLAGS" - QEMU_LDFLAGS="-fsanitize=thread $QEMU_LDFLAGS" - else - error_exit "Cannot enable TSAN due to missing fiber annotation interface." - fi -elif test "$tsan" = "yes" ; then - error_exit "Cannot enable TSAN due to missing sanitize thread interface." -fi -if test "$have_ubsan" = "yes"; then - QEMU_CFLAGS="-fsanitize=undefined $QEMU_CFLAGS" - QEMU_LDFLAGS="-fsanitize=undefined $QEMU_LDFLAGS" -fi - ####################################### # cross-compiled firmware targets @@ -2293,7 +1661,6 @@ LINKS="$LINKS python" LINKS="$LINKS contrib/plugins/Makefile " for f in $LINKS ; do if [ -e "$source_path/$f" ]; then - mkdir -p "$(dirname ./"$f")" symlink "$source_path/$f" "$f" fi done @@ -2384,30 +1751,14 @@ fi if test "$solaris" = "yes" ; then echo "CONFIG_SOLARIS=y" >> $config_host_mak fi -if test "$static" = "yes" ; then - echo "CONFIG_STATIC=y" >> $config_host_mak -fi echo "SRC_PATH=$source_path" >> $config_host_mak echo "TARGET_DIRS=$target_list" >> $config_host_mak -if test "$modules" = "yes"; then - echo "CONFIG_MODULES=y" >> $config_host_mak -fi # XXX: suppress that if [ "$bsd" = "yes" ] ; then echo "CONFIG_BSD=y" >> $config_host_mak fi -echo "CONFIG_COROUTINE_BACKEND=$coroutine" >> $config_host_mak - -if test "$have_asan_iface_fiber" = "yes" ; then - echo "CONFIG_ASAN_IFACE_FIBER=y" >> $config_host_mak -fi - -if test "$have_tsan" = "yes" && test "$have_tsan_iface_fiber" = "yes" ; then - echo "CONFIG_TSAN=y" >> $config_host_mak -fi - if test "$plugins" = "yes" ; then echo "CONFIG_PLUGIN=y" >> $config_host_mak fi @@ -2432,17 +1783,8 @@ echo "PYTHON=$python" >> $config_host_mak echo "GENISOIMAGE=$genisoimage" >> $config_host_mak echo "MESON=$meson" >> $config_host_mak echo "NINJA=$ninja" >> $config_host_mak +echo "PKG_CONFIG=${pkg_config}" >> $config_host_mak echo "CC=$cc" >> $config_host_mak -echo "QEMU_CFLAGS=$QEMU_CFLAGS" >> $config_host_mak -echo "QEMU_OBJCFLAGS=$QEMU_OBJCFLAGS" >> $config_host_mak -echo "GLIB_CFLAGS=$glib_cflags" >> $config_host_mak -echo "GLIB_LIBS=$glib_libs" >> $config_host_mak -echo "GLIB_BINDIR=$glib_bindir" >> $config_host_mak -echo "GLIB_VERSION=$($pkg_config --modversion glib-2.0)" >> $config_host_mak -if test "$glib_has_gslice" = "yes" ; then - echo "HAVE_GLIB_WITH_SLICE_ALLOCATOR=y" >> $config_host_mak -fi -echo "QEMU_LDFLAGS=$QEMU_LDFLAGS" >> $config_host_mak echo "EXESUF=$EXESUF" >> $config_host_mak # use included Linux headers @@ -2496,10 +1838,6 @@ if test "$ccache_cpp2" = "yes"; then echo "export CCACHE_CPP2=y" >> $config_host_mak fi -if test "$safe_stack" = "yes"; then - echo "CONFIG_SAFESTACK=y" >> $config_host_mak -fi - # tests/tcg configuration (config_host_mak=tests/tcg/config-host.mak mkdir -p tests/tcg @@ -2588,7 +1926,7 @@ if test "$skip_meson" = no; then test -n "$objcc" && echo "objc = [$(meson_quote $objcc $CPU_CFLAGS)]" >> $cross echo "ar = [$(meson_quote $ar)]" >> $cross echo "nm = [$(meson_quote $nm)]" >> $cross - echo "pkgconfig = [$(meson_quote $pkg_config_exe)]" >> $cross + echo "pkgconfig = [$(meson_quote $pkg_config)]" >> $cross echo "ranlib = [$(meson_quote $ranlib)]" >> $cross if has $sdl2_config; then echo "sdl2-config = [$(meson_quote $sdl2_config)]" >> $cross @@ -2629,11 +1967,13 @@ if test "$skip_meson" = no; then # Built-in options test "$bindir" != "bin" && meson_option_add "-Dbindir=$bindir" test "$default_feature" = no && meson_option_add -Dauto_features=disabled + test "$static" = yes && meson_option_add -Dprefer_static=true test "$pie" = no && meson_option_add -Db_pie=false test "$werror" = yes && meson_option_add -Dwerror=true # QEMU options test "$cfi" != false && meson_option_add "-Dcfi=$cfi" + test "$docs" != auto && meson_option_add "-Ddocs=$docs" test "$fdt" != auto && meson_option_add "-Dfdt=$fdt" test -n "${LIB_FUZZING_ENGINE+xxx}" && meson_option_add "-Dfuzzing_engine=$LIB_FUZZING_ENGINE" test "$qemu_suffix" != qemu && meson_option_add "-Dqemu_suffix=$qemu_suffix" diff --git a/contrib/plugins/Makefile b/contrib/plugins/Makefile index 23e0396..b2b9db9 100644 --- a/contrib/plugins/Makefile +++ b/contrib/plugins/Makefile @@ -3,7 +3,7 @@ # This Makefile example is fairly independent from the main makefile # so users can take and adapt it for their build. We only really # include config-host.mak so we don't have to repeat probing for -# cflags that the main configure has already done for us. +# programs that the main configure has already done for us. # BUILD_DIR := $(CURDIR)/../.. @@ -26,9 +26,8 @@ SONAMES := $(addsuffix .so,$(addprefix lib,$(NAMES))) # The main QEMU uses Glib extensively so it's perfectly fine to use it # in plugins (which many example do). -CFLAGS = $(GLIB_CFLAGS) -CFLAGS += -fPIC -Wall $(filter -W%, $(QEMU_CFLAGS)) -CFLAGS += $(if $(findstring no-psabi,$(QEMU_CFLAGS)),-Wpsabi) +CFLAGS := $(shell $(PKG_CONFIG) --cflags glib-2.0) +CFLAGS += -fPIC -Wall CFLAGS += $(if $(CONFIG_DEBUG_TCG), -ggdb -O0) CFLAGS += -I$(SRC_PATH)/include/qemu diff --git a/docs/about/build-platforms.rst b/docs/about/build-platforms.rst index 89cae5a..0e2cb9e 100644 --- a/docs/about/build-platforms.rst +++ b/docs/about/build-platforms.rst @@ -98,7 +98,7 @@ Python runtime option of the ``configure`` script to point QEMU to a supported version of the Python runtime. - As of QEMU |version|, the minimum supported version of Python is 3.6. + As of QEMU |version|, the minimum supported version of Python is 3.7. Python build dependencies Some of QEMU's build dependencies are written in Python. Usually these diff --git a/docs/conf.py b/docs/conf.py index 00767b0..c687ff2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,15 +32,6 @@ import sphinx from distutils.version import LooseVersion from sphinx.errors import ConfigError -# Make Sphinx fail cleanly if using an old Python, rather than obscurely -# failing because some code in one of our extensions doesn't work there. -# In newer versions of Sphinx this will display nicely; in older versions -# Sphinx will also produce a Python backtrace but at least the information -# gets printed... -if sys.version_info < (3,6): - raise ConfigError( - "QEMU requires a Sphinx that uses Python 3.6 or better\n") - # The per-manual conf.py will set qemu_docdir for a single-manual build; # otherwise set it here if this is an entire-manual-set build. # This is always the absolute path of the docs/ directory in the source tree. diff --git a/docs/devel/acpi-bits.rst b/docs/devel/acpi-bits.rst index 22e2580..9677b00 100644 --- a/docs/devel/acpi-bits.rst +++ b/docs/devel/acpi-bits.rst @@ -61,19 +61,19 @@ Under ``tests/avocado/`` as the root we have: :: $ make check-venv (needed only the first time to create the venv) - $ ./tests/venv/bin/avocado run -t acpi tests/avocado + $ ./pyvenv/bin/avocado run -t acpi tests/avocado The above will run all acpi avocado tests including this one. In order to run the individual tests, perform the following: :: - $ ./tests/venv/bin/avocado run tests/avocado/acpi-bits.py --tap - + $ ./pyvenv/bin/avocado run tests/avocado/acpi-bits.py --tap - The above will produce output in tap format. You can omit "--tap -" in the end and it will produce output like the following: :: - $ ./tests/venv/bin/avocado run tests/avocado/acpi-bits.py + $ ./pyvenv/bin/avocado run tests/avocado/acpi-bits.py Fetching asset from tests/avocado/acpi-bits.py:AcpiBitsTest.test_acpi_smbios_bits JOB ID : eab225724da7b64c012c65705dc2fa14ab1defef JOB LOG : /home/anisinha/avocado/job-results/job-2022-10-10T17.58-eab2257/job.log diff --git a/docs/devel/build-system.rst b/docs/devel/build-system.rst index 1894721..551c5a5 100644 --- a/docs/devel/build-system.rst +++ b/docs/devel/build-system.rst @@ -4,30 +4,14 @@ The QEMU build system architecture This document aims to help developers understand the architecture of the QEMU build system. As with projects using GNU autotools, the QEMU build -system has two stages, first the developer runs the "configure" script +system has two stages; first the developer runs the "configure" script to determine the local build environment characteristics, then they run -"make" to build the project. There is about where the similarities with +"make" to build the project. This is about where the similarities with GNU autotools end, so try to forget what you know about them. +The two general ways to perform a build are as follows: -Stage 1: configure -================== - -The QEMU configure script is written directly in shell, and should be -compatible with any POSIX shell, hence it uses #!/bin/sh. An important -implication of this is that it is important to avoid using bash-isms on -development platforms where bash is the primary host. - -In contrast to autoconf scripts, QEMU's configure is expected to be -silent while it is checking for features. It will only display output -when an error occurs, or to show the final feature enablement summary -on completion. - -Because QEMU uses the Meson build system under the hood, only VPATH -builds are supported. There are two general ways to invoke configure & -perform a build: - - - VPATH, build artifacts outside of QEMU source tree entirely:: + - build artifacts outside of QEMU source tree entirely:: cd ../ mkdir build @@ -35,88 +19,122 @@ perform a build: ../qemu/configure make - - VPATH, build artifacts in a subdir of QEMU source tree:: + - build artifacts in a subdir of QEMU source tree:: mkdir build cd build ../configure make -The configure script automatically recognizes -command line options for which a same-named Meson option exists; -dashes in the command line are replaced with underscores. +Most of the actual build process uses Meson under the hood, therefore +build artifacts cannot be placed in the source tree itself. + + +Stage 1: configure +================== + +The configure script has five tasks: + + - detect the host architecture -Many checks on the compilation environment are still found in configure -rather than ``meson.build``, but new checks should be added directly to -``meson.build``. + - list the targets for which to build emulators; the list of + targets also affects which firmware binaries and tests to build + + - find the compilers (native and cross) used to build executables, + firmware and tests. The results are written as either Makefile + fragments (``config-host.mak``) or a Meson machine file + (``config-meson.cross``) + + - create a virtual environment in which all Python code runs during + the build, and possibly install packages into it from PyPI + + - invoke Meson in the virtual environment, to perform the actual + configuration step for the emulator build + +The configure script automatically recognizes command line options for +which a same-named Meson option exists; dashes in the command line are +replaced with underscores. + +Almost all QEMU developers that need to modify the build system will +only be concerned with Meson, and therefore can skip the rest of this +section. + + +Modifying ``configure`` +----------------------- + +``configure`` is a shell script; it uses ``#!/bin/sh`` and therefore +should be compatible with any POSIX shell. It is important to avoid +using bash-isms to avoid breaking development platforms where bash is +the primary host. + +The configure script provides a variety of functions to help writing +portable shell code and providing consistent behavior across architectures +and operating systems: + +``error_exit $MESSAGE $MORE...`` + Print $MESSAGE to stderr, followed by $MORE... and then exit from the + configure script with non-zero status. -Patches are also welcome to move existing checks from the configure -phase to ``meson.build``. When doing so, ensure that ``meson.build`` does -not use anymore the keys that you have removed from ``config-host.mak``. -Typically these will be replaced in ``meson.build`` by boolean variables, -``get_option('optname')`` invocations, or ``dep.found()`` expressions. -In general, the remaining checks have little or no interdependencies, -so they can be moved one by one. +``has $COMMAND`` + Determine if $COMMAND exists in the current environment, either as a + shell builtin, or executable binary, returning 0 on success. The + replacement in Meson is ``find_program()``. -Helper functions ----------------- +``probe_target_compiler $TARGET`` + Detect a cross compiler and cross tools for the QEMU target $TARGET (e.g., + ``$CPU-softmmu``, ``$CPU-linux-user``, ``$CPU-bsd-user``). If a working + compiler is present, return success and set variables ``$target_cc``, + ``$target_ar``, etc. to non-empty values. -The configure script provides a variety of helper functions to assist -developers in checking for system features: +``write_target_makefile`` + Write a Makefile fragment to stdout, exposing the result of the most + ``probe_target_compiler`` call as the usual Make variables (``CC``, + ``AR``, ``LD``, etc.). -``do_cc $ARGS...`` - Attempt to run the system C compiler passing it $ARGS... -``do_cxx $ARGS...`` - Attempt to run the system C++ compiler passing it $ARGS... +Configure does not generally perform tests for compiler options beyond +basic checks to detect the host platform and ensure the compiler is +functioning. These are performed using a few more helper functions: ``compile_object $CFLAGS`` Attempt to compile a test program with the system C compiler using $CFLAGS. The test program must have been previously written to a file - called $TMPC. The replacement in Meson is the compiler object ``cc``, - which has methods such as ``cc.compiles()``, - ``cc.check_header()``, ``cc.has_function()``. + called $TMPC. ``compile_prog $CFLAGS $LDFLAGS`` Attempt to compile a test program with the system C compiler using $CFLAGS and link it with the system linker using $LDFLAGS. The test program must have been previously written to a file called $TMPC. - The replacement in Meson is ``cc.find_library()`` and ``cc.links()``. - -``has $COMMAND`` - Determine if $COMMAND exists in the current environment, either as a - shell builtin, or executable binary, returning 0 on success. The - replacement in Meson is ``find_program()``. ``check_define $NAME`` - Determine if the macro $NAME is defined by the system C compiler + Determine if the macro $NAME is defined by the system C compiler. -``check_include $NAME`` - Determine if the include $NAME file is available to the system C - compiler. The replacement in Meson is ``cc.has_header()``. +``do_compiler $CC $ARGS...`` + Attempt to run the C compiler $CC, passing it $ARGS... This function + does not use flags passed via options such as ``--extra-cflags``, and + therefore can be used to check for cross compilers. However, most + such checks are done at ``make`` time instead (see for example the + ``cc-option`` macro in ``pc-bios/option-rom/Makefile``). ``write_c_skeleton`` Write a minimal C program main() function to the temporary file - indicated by $TMPC + indicated by $TMPC. -``error_exit $MESSAGE $MORE...`` - Print $MESSAGE to stderr, followed by $MORE... and then exit from the - configure script with non-zero status -``query_pkg_config $ARGS...`` - Run pkg-config passing it $ARGS. If QEMU is doing a static build, - then --static will be automatically added to $ARGS +Python virtual environments and the QEMU build system +----------------------------------------------------- +TBD Stage 2: Meson ============== -The Meson build system is currently used to describe the build -process for: +The Meson build system describes the build and install process for: 1) executables, which include: - - Tools - ``qemu-img``, ``qemu-nbd``, ``qga`` (guest agent), etc + - Tools - ``qemu-img``, ``qemu-nbd``, ``qemu-ga`` (guest agent), etc - System emulators - ``qemu-system-$ARCH`` @@ -126,7 +144,8 @@ process for: 2) documentation -3) ROMs, which can be either installed as binary blobs or compiled +3) ROMs, whether provided as binary blobs in the QEMU distributions + or cross compiled under the direction of the configure script 4) other data files, such as icons or desktop files @@ -281,8 +300,7 @@ system/userspace emulation target Adding checks ------------- -New checks should be added to Meson. Compiler checks can be as simple as -the following:: +Compiler checks can be as simple as the following:: config_host_data.set('HAVE_BTRFS_H', cc.has_header('linux/btrfs.h')) @@ -311,8 +329,7 @@ dependency will be used:: sdl_image = not_found if not get_option('sdl_image').auto() or have_system sdl_image = dependency('SDL2_image', required: get_option('sdl_image'), - method: 'pkg-config', - static: enable_static) + method: 'pkg-config') endif This avoids warnings on static builds of user-mode emulators, for example. @@ -360,22 +377,30 @@ script, which may point to something other than the first python3 binary on the path. -Stage 3: makefiles -================== +Stage 3: Make +============= -The use of GNU make is required with the QEMU build system. +The next step in building QEMU is to invoke make. GNU Make is required +to build QEMU, and may be installed as ``gmake`` on some hosts. -The output of Meson is a build.ninja file, which is used with the Ninja -build system. QEMU uses a different approach, where Makefile rules are -synthesized from the build.ninja file. The main Makefile includes these -rules and wraps them so that e.g. submodules are built before QEMU. -The resulting build system is largely non-recursive in nature, in -contrast to common practices seen with automake. +The output of Meson is a ``build.ninja`` file, which is used with the +Ninja build tool. However, QEMU's build comprises other components than +just the emulators (namely firmware and the tests in ``tests/tcg``) which +need different cross compilers. The QEMU Makefile wraps both Ninja and +the smaller build systems for firmware and tests; it also takes care of +running ``configure`` again when the script changes. Apart from invoking +these sub-Makefiles, the resulting build is largely non-recursive. -Tests are also ran by the Makefile with the traditional ``make check`` -phony target, while benchmarks are run with ``make bench``. Meson test -suites such as ``unit`` can be ran with ``make check-unit`` too. It is also -possible to run tests defined in meson.build with ``meson test``. +Tests, whether defined in ``meson.build`` or not, are also ran by the +Makefile with the traditional ``make check`` phony target, while benchmarks +are run with ``make bench``. Meson test suites such as ``unit`` can be ran +with ``make check-unit``, and ``make check-tcg`` builds and runs "non-Meson" +tests for all targets. + +If desired, it is also possible to use ``ninja`` and ``meson test``, +respectively to build emulators and run tests defined in meson.build. +The main difference is that ``make`` needs the ``-jN`` flag in order to +enable parallel builds or tests. Useful make targets ------------------- @@ -387,6 +412,7 @@ Useful make targets Print the value of the variable VAR. Useful for debugging the build system. + Important files for the build system ==================================== @@ -400,8 +426,7 @@ number of dynamically created files listed later. ``Makefile`` The main entry point used when invoking make to build all the components of QEMU. The default 'all' target will naturally result in the build of - every component. Makefile takes care of recursively building submodules - directly via a non-recursive set of rules. + every component. ``*/meson.build`` The meson.build file in the root directory is the main entry point for the @@ -410,59 +435,92 @@ number of dynamically created files listed later. other meson.build files spread throughout the QEMU source tree. ``tests/Makefile.include`` - Rules for external test harnesses. These include the TCG tests, - ``qemu-iotests`` and the Avocado-based integration tests. + Rules for external test harnesses. These include the TCG tests + and the Avocado-based integration tests. ``tests/docker/Makefile.include`` - Rules for Docker tests. Like tests/Makefile, this file is included - directly by the top level Makefile, anything defined in this file will - influence the entire build system. + Rules for Docker tests. Like ``tests/Makefile.include``, this file is + included directly by the top level Makefile, anything defined in this + file will influence the entire build system. ``tests/vm/Makefile.include`` - Rules for VM-based tests. Like tests/Makefile, this file is included - directly by the top level Makefile, anything defined in this file will - influence the entire build system. + Rules for VM-based tests. Like ``tests/Makefile.include``, this file is + included directly by the top level Makefile, anything defined in this + file will influence the entire build system. Dynamically created files ------------------------- -The following files are generated dynamically by configure in order to -control the behaviour of the statically defined makefiles. This avoids -the need for QEMU makefiles to go through any pre-processing as seen -with autotools, where Makefile.am generates Makefile.in which generates -Makefile. +The following files are generated at run-time in order to control the +behaviour of the Makefiles. This avoids the need for QEMU makefiles to +go through any pre-processing as seen with autotools, where configure +generates ``Makefile`` from ``Makefile.in``. Built by configure: ``config-host.mak`` When configure has determined the characteristics of the build host it - will write a long list of variables to config-host.mak file. This - provides the various install directories, compiler / linker flags and a + will write them to this file for use in ``Makefile`` and to a smaller + extent ``meson.build``. These include the paths to various tools and a variety of ``CONFIG_*`` variables related to optionally enabled features. - This is imported by the top level Makefile and meson.build in order to - tailor the build output. - config-host.mak is also used as a dependency checking mechanism. If make + ``config-host.mak`` is also used as a dependency checking mechanism. If make sees that the modification timestamp on configure is newer than that on - config-host.mak, then configure will be re-run. + ``config-host.mak``, then configure will be re-run. + + The variables defined here apply to all QEMU + build outputs. + +``config-meson.cross`` + + A Meson "cross file" (or native file) used to communicate the paths to + the toolchain and other configuration options. + +``config.status`` - The variables defined here are those which are applicable to all QEMU - build outputs. Variables which are potentially different for each - emulator target are defined by the next file... + A small shell script that will invoke configure again with the same + environment variables that were set during the first run. It's used to + rerun configure after changes to the source code, but it can also be + inspected manually to check the contents of the environment. +``Makefile.prereqs`` + + A set of Makefile dependencies that order the build and execution of + firmware and tests after the container images and emulators that they + need. + +``pc-bios/*/config.mak``, ``tests/tcg/config-host.mak``, ``tests/tcg/*/config-target.mak`` + + Configuration variables used to build the firmware and TCG tests, + including paths to cross compilation toolchains. + +``pyvenv`` + + A Python virtual environment that is used for all Python code running + during the build. Using a virtual environment ensures that even code + that is run via ``sphinx-build``, ``meson`` etc. uses the same interpreter + and packages. Built by Meson: +``config-host.h`` + Used by C code to determine the properties of the build environment + and the set of enabled features for the entire build. + ``${TARGET-NAME}-config-devices.mak`` - TARGET-NAME is again the name of a system or userspace emulator. The - config-devices.mak file is automatically generated by make using the - scripts/make_device_config.sh program, feeding it the - default-configs/$TARGET-NAME file as input. - -``config-host.h``, ``$TARGET_NAME-config-target.h``, ``$TARGET_NAME-config-devices.h`` - These files are used by source code to determine what features are - enabled. They are generated from the contents of the corresponding - ``*.mak`` files using Meson's ``configure_file()`` function. + TARGET-NAME is the name of a system emulator. The file is + generated by Meson using files under ``configs/devices`` as input. + +``${TARGET-NAME}-config-target.mak`` + TARGET-NAME is the name of a system or usermode emulator. The file is + generated by Meson using files under ``configs/targets`` as input. + +``$TARGET_NAME-config-target.h``, ``$TARGET_NAME-config-devices.h`` + Used by C code to determine the properties and enabled + features for each target. enabled. They are generated from + the contents of the corresponding ``*.mak`` files using Meson's + ``configure_file()`` function; each target can include them using + the ``CONFIG_TARGET`` and ``CONFIG_DEVICES`` macro respectively. ``build.ninja`` The build rules. diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst index 203facb..2cafec4 100644 --- a/docs/devel/testing.rst +++ b/docs/devel/testing.rst @@ -888,9 +888,9 @@ You can run the avocado tests simply by executing: make check-avocado -This involves the automatic creation of Python virtual environment -within the build tree (at ``tests/venv``) which will have all the -right dependencies, and will save tests results also within the +This involves the automatic installation, from PyPI, of all the +necessary avocado-framework dependencies into the QEMU venv within the +build tree (at ``./pyvenv``). Test results are also saved within the build tree (at ``tests/results``). Note: the build environment must be using a Python 3 stack, and have @@ -947,7 +947,7 @@ may be invoked by running: .. code:: - tests/venv/bin/avocado run $OPTION1 $OPTION2 tests/avocado/ + pyvenv/bin/avocado run $OPTION1 $OPTION2 tests/avocado/ Note that if ``make check-avocado`` was not executed before, it is possible to create the Python virtual environment with the dependencies @@ -962,20 +962,20 @@ a test file. To run tests from a single file within the build tree, use: .. code:: - tests/venv/bin/avocado run tests/avocado/$TESTFILE + pyvenv/bin/avocado run tests/avocado/$TESTFILE To run a single test within a test file, use: .. code:: - tests/venv/bin/avocado run tests/avocado/$TESTFILE:$TESTCLASS.$TESTNAME + pyvenv/bin/avocado run tests/avocado/$TESTFILE:$TESTCLASS.$TESTNAME Valid test names are visible in the output from any previous execution of Avocado or ``make check-avocado``, and can also be queried using: .. code:: - tests/venv/bin/avocado list tests/avocado + pyvenv/bin/avocado list tests/avocado Manual Installation ~~~~~~~~~~~~~~~~~~~ diff --git a/docs/meson.build b/docs/meson.build index f220800..9040f86 100644 --- a/docs/meson.build +++ b/docs/meson.build @@ -1,4 +1,4 @@ -sphinx_build = find_program(get_option('sphinx_build'), +sphinx_build = find_program(fs.parent(python.full_path()) / 'sphinx-build', required: get_option('docs')) # Check if tools are available to build documentation. @@ -10,6 +10,18 @@ if sphinx_build.found() SPHINX_ARGS += [ '-W', '-Dkerneldoc_werror=1' ] endif + sphinx_version = run_command(SPHINX_ARGS + ['--version'], + check: true).stdout().split()[1] + if sphinx_version.version_compare('>=1.7.0') + SPHINX_ARGS += ['-j', 'auto'] + else + nproc = find_program('nproc') + if nproc.found() + jobs = run_command(nproc, check: true).stdout() + SPHINX_ARGS += ['-j', jobs] + endif + endif + # This is a bit awkward but works: create a trivial document and # try to run it with our configuration file (which enforces a # version requirement). This will fail if sphinx-build is too old. diff --git a/docs/sphinx/dbusdomain.py b/docs/sphinx/dbusdomain.py index 2ea95af..9872fd5 100644 --- a/docs/sphinx/dbusdomain.py +++ b/docs/sphinx/dbusdomain.py @@ -400,6 +400,10 @@ class DBusDomain(Domain): for refname, obj in self.objects.items(): yield (refname, refname, obj.objtype, obj.docname, obj.node_id, 1) + def merge_domaindata(self, docnames, otherdata): + for name, obj in otherdata['objects'].items(): + if obj.docname in docnames: + self.data['objects'][name] = obj def setup(app): app.add_domain(DBusDomain) diff --git a/docs/sphinx/fakedbusdoc.py b/docs/sphinx/fakedbusdoc.py index d2c5079..2d2e6ef 100644 --- a/docs/sphinx/fakedbusdoc.py +++ b/docs/sphinx/fakedbusdoc.py @@ -23,3 +23,8 @@ class FakeDBusDocDirective(Directive): def setup(app: Sphinx) -> Dict[str, Any]: """Register a fake dbus-doc directive with Sphinx""" app.add_directive("dbus-doc", FakeDBusDocDirective) + + return dict( + parallel_read_safe = True, + parallel_write_safe = True + ) diff --git a/docs/sphinx/qmp_lexer.py b/docs/sphinx/qmp_lexer.py index f7e4c0e..a59de8a 100644 --- a/docs/sphinx/qmp_lexer.py +++ b/docs/sphinx/qmp_lexer.py @@ -41,3 +41,8 @@ def setup(sphinx): sphinx.add_lexer('QMP', QMPExampleLexer) except errors.VersionRequirementError: sphinx.add_lexer('QMP', QMPExampleLexer()) + + return dict( + parallel_read_safe = True, + parallel_write_safe = True + ) diff --git a/hw/scsi/scsi-generic.c b/hw/scsi/scsi-generic.c index ac9fa66..2417f0a 100644 --- a/hw/scsi/scsi-generic.c +++ b/hw/scsi/scsi-generic.c @@ -191,12 +191,16 @@ static int scsi_handle_inquiry_reply(SCSIGenericReq *r, SCSIDevice *s, int len) if ((s->type == TYPE_DISK || s->type == TYPE_ZBC) && (r->req.cmd.buf[1] & 0x01)) { page = r->req.cmd.buf[2]; - if (page == 0xb0) { + if (page == 0xb0 && r->buflen >= 8) { + uint8_t buf[16] = {}; + uint8_t buf_used = MIN(r->buflen, 16); uint64_t max_transfer = calculate_max_transfer(s); - stl_be_p(&r->buf[8], max_transfer); - /* Also take care of the opt xfer len. */ - stl_be_p(&r->buf[12], - MIN_NON_ZERO(max_transfer, ldl_be_p(&r->buf[12]))); + + memcpy(buf, r->buf, buf_used); + stl_be_p(&buf[8], max_transfer); + stl_be_p(&buf[12], MIN_NON_ZERO(max_transfer, ldl_be_p(&buf[12]))); + memcpy(r->buf + 8, buf + 8, buf_used - 8); + } else if (s->needs_vpd_bl_emulation && page == 0x00 && r->buflen >= 4) { /* * Now we're capable of supplying the VPD Block Limits diff --git a/include/exec/memory.h b/include/exec/memory.h index e45ce60..6f24a3d 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -934,8 +934,11 @@ struct MemoryListener { * its @log_sync must be NULL. Vice versa. * * @listener: The #MemoryListener. + * @last_stage: The last stage to synchronize the log during migration. + * The caller should gurantee that the synchronization with true for + * @last_stage is triggered for once after all VCPUs have been stopped. */ - void (*log_sync_global)(MemoryListener *listener); + void (*log_sync_global)(MemoryListener *listener, bool last_stage); /** * @log_clear: @@ -2422,8 +2425,10 @@ MemoryRegionSection memory_region_find(MemoryRegion *mr, * memory_global_dirty_log_sync: synchronize the dirty log for all memory * * Synchronizes the dirty page log for all address spaces. + * + * @last_stage: whether this is the last stage of live migration */ -void memory_global_dirty_log_sync(void); +void memory_global_dirty_log_sync(bool last_stage); /** * memory_global_dirty_log_sync: synchronize the dirty log for all memory diff --git a/include/sysemu/kvm_int.h b/include/sysemu/kvm_int.h index a641c97..511b42b 100644 --- a/include/sysemu/kvm_int.h +++ b/include/sysemu/kvm_int.h @@ -115,6 +115,7 @@ struct KVMState } *as; uint64_t kvm_dirty_ring_bytes; /* Size of the per-vcpu dirty ring */ uint32_t kvm_dirty_ring_size; /* Number of dirty GFNs per ring */ + bool kvm_dirty_ring_with_bitmap; struct KVMDirtyRingReaper reaper; NotifyVmexitOption notify_vmexit; uint32_t notify_window; diff --git a/meson b/meson deleted file mode 160000 -Subproject 3a9b285a55b91b53b2acda987192274352ecb5b diff --git a/meson.build b/meson.build index 4dddccb..41c87c4 100644 --- a/meson.build +++ b/meson.build @@ -1,4 +1,4 @@ -project('qemu', ['c'], meson_version: '>=0.61.3', +project('qemu', ['c'], meson_version: '>=0.63.0', default_options: ['warning_level=1', 'c_std=gnu11', 'cpp_std=gnu++11', 'b_colorout=auto', 'b_staticpic=false', 'stdsplit=false', 'optimization=2', 'b_pie=true'], version: files('VERSION')) @@ -17,11 +17,6 @@ fs = import('fs') targetos = host_machine.system() sh = find_program('sh') config_host = keyval.load(meson.current_build_dir() / 'config-host.mak') -enable_modules = 'CONFIG_MODULES' in config_host -enable_static = 'CONFIG_STATIC' in config_host - -# Allow both shared and static libraries unless --enable-static -static_kwargs = enable_static ? {'static': true} : {} cc = meson.get_compiler('c') all_languages = ['c'] @@ -88,6 +83,12 @@ have_ga = get_option('guest_agent') \ .require(targetos in ['sunos', 'linux', 'windows', 'freebsd', 'netbsd', 'openbsd'], error_message: 'unsupported OS for QEMU guest agent') \ .allowed() +enable_modules = get_option('modules') \ + .require(targetos != 'windows', + error_message: 'Modules are not available for Windows') \ + .require(not get_option('prefer_static'), + error_message: 'Modules are incompatible with static linking') \ + .allowed() have_block = have_system or have_tools python = import('python').find_installation() @@ -189,14 +190,197 @@ endif # Compiler flags # ################## -qemu_cflags = config_host['QEMU_CFLAGS'].split() -qemu_objcflags = config_host['QEMU_OBJCFLAGS'].split() -qemu_ldflags = config_host['QEMU_LDFLAGS'].split() +foreach lang : all_languages + compiler = meson.get_compiler(lang) + if compiler.get_id() == 'gcc' and compiler.version().version_compare('>=7.4') + # ok + elif compiler.get_id() == 'clang' and compiler.compiles(''' + #ifdef __apple_build_version__ + # if __clang_major__ < 12 || (__clang_major__ == 12 && __clang_minor__ < 0) + # error You need at least XCode Clang v12.0 to compile QEMU + # endif + #else + # if __clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 0) + # error You need at least Clang v10.0 to compile QEMU + # endif + #endif''') + # ok + else + error('You either need GCC v7.4 or Clang v10.0 (or XCode Clang v12.0) to compile QEMU') + endif +endforeach + +# default flags for all hosts +# We use -fwrapv to tell the compiler that we require a C dialect where +# left shift of signed integers is well defined and has the expected +# 2s-complement style results. (Both clang and gcc agree that it +# provides these semantics.) + +qemu_common_flags = [ + '-D_GNU_SOURCE', '-D_FILE_OFFSET_BITS=64', '-D_LARGEFILE_SOURCE', + '-fno-strict-aliasing', '-fno-common', '-fwrapv' ] +qemu_cflags = [] +qemu_ldflags = [] -if enable_static +if targetos == 'darwin' + # Disable attempts to use ObjectiveC features in os/object.h since they + # won't work when we're compiling with gcc as a C compiler. + qemu_common_flags += '-DOS_OBJECT_USE_OBJC=0' +elif targetos == 'solaris' + # needed for CMSG_ macros in sys/socket.h + qemu_common_flags += '-D_XOPEN_SOURCE=600' + # needed for TIOCWIN* defines in termios.h + qemu_common_flags += '-D__EXTENSIONS__' +elif targetos == 'haiku' + qemu_common_flags += ['-DB_USE_POSITIVE_POSIX_ERRORS', '-D_BSD_SOURCE', '-fPIC'] +endif + +# __sync_fetch_and_and requires at least -march=i486. Many toolchains +# use i686 as default anyway, but for those that don't, an explicit +# specification is necessary +if host_arch == 'i386' and not cc.links(''' + static int sfaa(int *ptr) + { + return __sync_fetch_and_and(ptr, 0); + } + + int main(void) + { + int val = 42; + val = __sync_val_compare_and_swap(&val, 0, 1); + sfaa(&val); + return val; + }''') + qemu_common_flags = ['-march=i486'] + qemu_common_flags +endif + +if get_option('gprof') + qemu_common_flags += ['-p'] + qemu_ldflags += ['-p'] +endif + +if get_option('prefer_static') qemu_ldflags += get_option('b_pie') ? '-static-pie' : '-static' endif +# Meson currently only handles pie as a boolean for now, so if the user +# has explicitly disabled PIE we need to extend our cflags. +if not get_option('b_pie') + qemu_common_flags += cc.get_supported_arguments('-fno-pie') + if not get_option('prefer_static') + # No PIE is implied by -static which we added above. + qemu_ldflags += cc.get_supported_link_arguments('-no-pie') + endif +endif + +if not get_option('stack_protector').disabled() + stack_protector_probe = ''' + int main(int argc, char *argv[]) + { + char arr[64], *p = arr, *c = argv[argc - 1]; + while (*c) { + *p++ = *c++; + } + return 0; + }''' + have_stack_protector = false + foreach arg : ['-fstack-protector-strong', '-fstack-protector-all'] + # We need to check both a compile and a link, since some compiler + # setups fail only on a .c->.o compile and some only at link time + if cc.compiles(stack_protector_probe, args: ['-Werror', arg]) and \ + cc.links(stack_protector_probe, args: ['-Werror', arg]) + have_stack_protector = true + qemu_cflags += arg + qemu_ldflags += arg + break + endif + endforeach + get_option('stack_protector') \ + .require(have_stack_protector, error_message: 'Stack protector not supported') +endif + +coroutine_backend = get_option('coroutine_backend') +ucontext_probe = ''' + #include <ucontext.h> + #ifdef __stub_makecontext + #error Ignoring glibc stub makecontext which will always fail + #endif + int main(void) { makecontext(0, 0, 0); return 0; }''' + +# On Windows the only valid backend is the Windows specific one. +# For POSIX prefer ucontext, but it's not always possible. The fallback +# is sigcontext. +supported_backends = [] +if targetos == 'windows' + supported_backends += ['windows'] +else + if targetos != 'darwin' and cc.links(ucontext_probe) + supported_backends += ['ucontext'] + endif + supported_backends += ['sigaltstack'] +endif + +if coroutine_backend == 'auto' + coroutine_backend = supported_backends[0] +elif coroutine_backend not in supported_backends + error('"@0@" backend requested but not available. Available backends: @1@' \ + .format(coroutine_backend, ', '.join(supported_backends))) +endif + +# Compiles if SafeStack *not* enabled +safe_stack_probe = ''' + int main(void) + { + #if defined(__has_feature) + #if __has_feature(safe_stack) + #error SafeStack Enabled + #endif + #endif + return 0; + }''' +if get_option('safe_stack') != not cc.compiles(safe_stack_probe) + safe_stack_arg = get_option('safe_stack') ? '-fsanitize=safe-stack' : '-fno-sanitize=safe-stack' + if get_option('safe_stack') != not cc.compiles(safe_stack_probe, args: safe_stack_arg) + error(get_option('safe_stack') \ + ? 'SafeStack not supported by your compiler' \ + : 'Cannot disable SafeStack') + endif + qemu_cflags += safe_stack_arg + qemu_ldflags += safe_stack_arg +endif +if get_option('safe_stack') and coroutine_backend != 'ucontext' + error('SafeStack is only supported with the ucontext coroutine backend') +endif + +if get_option('sanitizers') + if cc.has_argument('-fsanitize=address') + qemu_cflags = ['-fsanitize=address'] + qemu_cflags + qemu_ldflags = ['-fsanitize=address'] + qemu_ldflags + endif + + # Detect static linking issue with ubsan - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84285 + if cc.links('int main(int argc, char **argv) { return argc + 1; }', + args: [qemu_ldflags, '-fsanitize=undefined']) + qemu_cflags = ['-fsanitize=undefined'] + qemu_cflags + qemu_ldflags = ['-fsanitize=undefined'] + qemu_ldflags + endif +endif + +# Thread sanitizer is, for now, much noisier than the other sanitizers; +# keep it separate until that is not the case. +if get_option('tsan') + if get_option('sanitizers') + error('TSAN is not supported with other sanitizers') + endif + if not cc.has_function('__tsan_create_fiber', + args: '-fsanitize=thread', + prefix: '#include <sanitizer/tsan_interface.h>') + error('Cannot enable TSAN due to missing fiber annotation interface') + endif + qemu_cflags = ['-fsanitize=thread'] + qemu_cflags + qemu_ldflags = ['-fsanitize=thread'] + qemu_ldflags +endif + # Detect support for PT_GNU_RELRO + DT_BIND_NOW. # The combination is known as "full relro", because .got.plt is read-only too. qemu_ldflags += cc.get_supported_link_arguments('-Wl,-z,relro', '-Wl,-z,now') @@ -206,10 +390,9 @@ if targetos == 'windows' qemu_ldflags += cc.get_supported_link_arguments('-Wl,--dynamicbase', '-Wl,--high-entropy-va') endif -if get_option('gprof') - qemu_cflags += ['-p'] - qemu_objcflags += ['-p'] - qemu_ldflags += ['-p'] +# Exclude --warn-common with TSan to suppress warnings from the TSan libraries. +if targetos != 'sunos' and not get_option('tsan') + qemu_ldflags += cc.get_supported_link_arguments('-Wl,--warn-common') endif # Specify linker-script with add_project_link_arguments so that it is not placed @@ -225,8 +408,7 @@ if get_option('fuzzing') name: '-fsanitize-coverage-allowlist=/dev/null', args: ['-fsanitize-coverage-allowlist=/dev/null', '-fsanitize-coverage=trace-pc'] ) - add_global_arguments('-fsanitize-coverage-allowlist=instrumentation-filter', - native: false, language: all_languages) + qemu_common_flags += ['-fsanitize-coverage-allowlist=instrumentation-filter'] endif if get_option('fuzzing_engine') == '' @@ -234,10 +416,8 @@ if get_option('fuzzing') # compiled code. To build non-fuzzer binaries with --enable-fuzzing, link # everything with fsanitize=fuzzer-no-link. Otherwise, the linker will be # unable to bind the fuzzer-related callbacks added by instrumentation. - add_global_arguments('-fsanitize=fuzzer-no-link', - native: false, language: all_languages) - add_global_link_arguments('-fsanitize=fuzzer-no-link', - native: false, language: all_languages) + qemu_common_flags += ['-fsanitize=fuzzer-no-link'] + qemu_ldflags += ['-fsanitize=fuzzer-no-link'] # For the actual fuzzer binaries, we need to link against the libfuzzer # library. They need to be configurable, to support OSS-Fuzz fuzz_exe_ldflags = ['-fsanitize=fuzzer'] @@ -248,20 +428,50 @@ if get_option('fuzzing') endif endif +add_global_arguments(qemu_common_flags, native: false, language: all_languages) +add_global_link_arguments(qemu_ldflags, native: false, language: all_languages) + +# Collect warnings that we want to enable + +warn_flags = [ + '-Wundef', + '-Wwrite-strings', + '-Wmissing-prototypes', + '-Wstrict-prototypes', + '-Wredundant-decls', + '-Wold-style-declaration', + '-Wold-style-definition', + '-Wtype-limits', + '-Wformat-security', + '-Wformat-y2k', + '-Winit-self', + '-Wignored-qualifiers', + '-Wempty-body', + '-Wnested-externs', + '-Wendif-labels', + '-Wexpansion-to-defined', + '-Wimplicit-fallthrough=2', + '-Wmissing-format-attribute', + '-Wno-initializer-overrides', + '-Wno-missing-include-dirs', + '-Wno-shift-negative-value', + '-Wno-string-plus-int', + '-Wno-typedef-redefinition', + '-Wno-tautological-type-limit-compare', + '-Wno-psabi', + '-Wno-gnu-variable-sized-type-not-at-end', +] + +if targetos != 'darwin' + warn_flags += ['-Wthread-safety'] +endif + # Check that the C++ compiler exists and works with the C compiler. link_language = 'c' linker = cc qemu_cxxflags = [] if 'cpp' in all_languages - add_global_arguments(['-D__STDC_LIMIT_MACROS', '-D__STDC_CONSTANT_MACROS', '-D__STDC_FORMAT_MACROS'], - native: false, language: 'cpp') - foreach k: qemu_cflags - if k not in ['-Wstrict-prototypes', '-Wmissing-prototypes', '-Wnested-externs', - '-Wold-style-declaration', '-Wold-style-definition', '-Wredundant-decls'] - qemu_cxxflags += [k] - endif - endforeach - + qemu_cxxflags = ['-D__STDC_LIMIT_MACROS', '-D__STDC_CONSTANT_MACROS', '-D__STDC_FORMAT_MACROS'] + qemu_cflags if cxx.links(files('scripts/main.c'), args: qemu_cflags) link_language = 'cpp' linker = cxx @@ -271,16 +481,26 @@ if 'cpp' in all_languages endif endif -# Exclude --warn-common with TSan to suppress warnings from the TSan libraries. -if targetos != 'sunos' and not config_host.has_key('CONFIG_TSAN') - qemu_ldflags += linker.get_supported_link_arguments('-Wl,--warn-common') +# clang does not support glibc + FORTIFY_SOURCE (is it still true?) +if get_option('optimization') != '0' and targetos == 'linux' + if cc.get_id() == 'gcc' + qemu_cflags += ['-U_FORTIFY_SOURCE', '-D_FORTIFY_SOURCE=2'] + endif + if 'cpp' in all_languages and cxx.get_id() == 'gcc' + qemu_cxxflags += ['-U_FORTIFY_SOURCE', '-D_FORTIFY_SOURCE=2'] + endif endif -add_global_link_arguments(qemu_ldflags, native: false, language: all_languages) - -add_global_arguments(qemu_cflags, native: false, language: 'c') -add_global_arguments(qemu_cxxflags, native: false, language: 'cpp') -add_global_arguments(qemu_objcflags, native: false, language: 'objc') +add_project_arguments(qemu_cflags, native: false, language: 'c') +add_project_arguments(cc.get_supported_arguments(warn_flags), native: false, language: 'c') +if 'cpp' in all_languages + add_project_arguments(qemu_cxxflags, native: false, language: 'cpp') + add_project_arguments(cxx.get_supported_arguments(warn_flags), native: false, language: 'cpp') +endif +if 'objc' in all_languages + # Note sanitizer flags are not applied to Objective-C sources! + add_project_arguments(objc.get_supported_arguments(warn_flags), native: false, language: 'objc') +endif if targetos == 'linux' add_project_arguments('-isystem', meson.current_source_dir() / 'linux-headers', '-isystem', 'linux-headers', @@ -496,29 +716,85 @@ endif # Dependencies # ################ -# The path to glib.h is added to all compilation commands. This was -# grandfathered in from the QEMU Makefiles. -add_project_arguments(config_host['GLIB_CFLAGS'].split(), - native: false, language: all_languages) -glib = declare_dependency(compile_args: config_host['GLIB_CFLAGS'].split(), - link_args: config_host['GLIB_LIBS'].split(), - version: config_host['GLIB_VERSION'], - variables: { - 'bindir': config_host['GLIB_BINDIR'], - }) -# override glib dep with the configure results (for subprojects) -meson.override_dependency('glib-2.0', glib) -# pass down whether Glib has the slice allocator -if config_host.has_key('HAVE_GLIB_WITH_SLICE_ALLOCATOR') - config_host_data.set('HAVE_GLIB_WITH_SLICE_ALLOCATOR', true) +# When bumping glib minimum version, please check also whether to increase +# the _WIN32_WINNT setting in osdep.h according to the value from glib +glib_req_ver = '>=2.56.0' +glib_pc = dependency('glib-2.0', version: glib_req_ver, required: true, + method: 'pkg-config') +glib_cflags = [] +if enable_modules + gmodule = dependency('gmodule-export-2.0', version: glib_req_ver, required: true, + method: 'pkg-config') +elif config_host.has_key('CONFIG_PLUGIN') + gmodule = dependency('gmodule-no-export-2.0', version: glib_req_ver, required: true, + method: 'pkg-config') +else + gmodule = not_found endif +# This workaround is required due to a bug in pkg-config file for glib as it +# doesn't define GLIB_STATIC_COMPILATION for pkg-config --static +if targetos == 'windows' and get_option('prefer_static') + glib_cflags += ['-DGLIB_STATIC_COMPILATION'] +endif + +# Sanity check that the current size_t matches the +# size that glib thinks it should be. This catches +# problems on multi-arch where people try to build +# 32-bit QEMU while pointing at 64-bit glib headers + +if not cc.compiles(''' + #include <glib.h> + #include <unistd.h> + + #define QEMU_BUILD_BUG_ON(x) \ + typedef char qemu_build_bug_on[(x)?-1:1] __attribute__((unused)); + + int main(void) { + QEMU_BUILD_BUG_ON(sizeof(size_t) != GLIB_SIZEOF_SIZE_T); + return 0; + }''', dependencies: glib_pc, args: glib_cflags) + error('''sizeof(size_t) doesn't match GLIB_SIZEOF_SIZE_T. + You probably need to set PKG_CONFIG_LIBDIR" to point + to the right pkg-config files for your build target.''') +endif + +# Silence clang warnings triggered by glib < 2.57.2 +if not cc.compiles(''' + #include <glib.h> + typedef struct Foo { + int i; + } Foo; + static void foo_free(Foo *f) + { + g_free(f); + } + G_DEFINE_AUTOPTR_CLEANUP_FUNC(Foo, foo_free) + int main(void) { return 0; }''', dependencies: glib_pc, args: ['-Werror']) + glib_cflags += cc.get_supported_arguments('-Wno-unused-function') +endif +glib = declare_dependency(dependencies: [glib_pc, gmodule], + compile_args: glib_cflags, + version: glib_pc.version()) + +# Check whether glib has gslice, which we have to avoid for correctness. +# TODO: remove this check and the corresponding workaround (qtree) when +# the minimum supported glib is >= 2.75.3 +glib_has_gslice = glib.version().version_compare('<2.75.3') + +# override glib dep to include the above refinements +meson.override_dependency('glib-2.0', glib) + +# The path to glib.h is added to all compilation commands. +add_project_dependencies(glib.partial_dependency(compile_args: true, includes: true), + native: false, language: all_languages) + gio = not_found gdbus_codegen = not_found gdbus_codegen_error = '@0@ requires gdbus-codegen, please install libgio' if not get_option('gio').auto() or have_system gio = dependency('gio-2.0', required: get_option('gio'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') if gio.found() and not cc.links(''' #include <gio/gio.h> int main(void) @@ -535,7 +811,7 @@ if not get_option('gio').auto() or have_system gdbus_codegen = find_program(gio.get_variable('gdbus_codegen'), required: get_option('gio')) gio_unix = dependency('gio-unix-2.0', required: get_option('gio'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') gio = declare_dependency(dependencies: [gio, gio_unix], version: gio.version()) endif @@ -548,20 +824,19 @@ endif lttng = not_found if 'ust' in get_option('trace_backends') lttng = dependency('lttng-ust', required: true, version: '>= 2.1', - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif pixman = not_found if have_system or have_tools pixman = dependency('pixman-1', required: have_system, version:'>=0.21.8', - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif -zlib = dependency('zlib', required: true, kwargs: static_kwargs) +zlib = dependency('zlib', required: true) libaio = not_found if not get_option('linux_aio').auto() or have_block libaio = cc.find_library('aio', has_headers: ['libaio.h'], - required: get_option('linux_aio'), - kwargs: static_kwargs) + required: get_option('linux_aio')) endif linux_io_uring_test = ''' @@ -574,7 +849,7 @@ linux_io_uring = not_found if not get_option('linux_io_uring').auto() or have_block linux_io_uring = dependency('liburing', version: '>=0.3', required: get_option('linux_io_uring'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') if not cc.links(linux_io_uring_test) linux_io_uring = not_found endif @@ -584,7 +859,7 @@ libnfs = not_found if not get_option('libnfs').auto() or have_block libnfs = dependency('libnfs', version: '>=1.9.3', required: get_option('libnfs'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif libattr_test = ''' @@ -604,8 +879,7 @@ if get_option('attr').allowed() libattr = declare_dependency() else libattr = cc.find_library('attr', has_headers: ['attr/xattr.h'], - required: get_option('attr'), - kwargs: static_kwargs) + required: get_option('attr')) if libattr.found() and not \ cc.links(libattr_test, dependencies: libattr, args: '-DCONFIG_LIBATTR') libattr = not_found @@ -640,7 +914,7 @@ seccomp_has_sysrawrc = false if not get_option('seccomp').auto() or have_system or have_tools seccomp = dependency('libseccomp', version: '>=2.3.0', required: get_option('seccomp'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') if seccomp.found() seccomp_has_sysrawrc = cc.has_header_symbol('seccomp.h', 'SCMP_FLTATR_API_SYSRAWRC', @@ -651,8 +925,7 @@ endif libcap_ng = not_found if not get_option('cap_ng').auto() or have_system or have_tools libcap_ng = cc.find_library('cap-ng', has_headers: ['cap-ng.h'], - required: get_option('cap_ng'), - kwargs: static_kwargs) + required: get_option('cap_ng')) endif if libcap_ng.found() and not cc.links(''' #include <cap-ng.h> @@ -673,13 +946,13 @@ if get_option('xkbcommon').auto() and not have_system and not have_tools xkbcommon = not_found else xkbcommon = dependency('xkbcommon', required: get_option('xkbcommon'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif slirp = not_found if not get_option('slirp').auto() or have_system slirp = dependency('slirp', required: get_option('slirp'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') # slirp < 4.7 is incompatible with CFI support in QEMU. This is because # it passes function pointers within libslirp as callbacks for timers. # When using a system-wide shared libslirp, the type information for the @@ -699,8 +972,7 @@ endif vde = not_found if not get_option('vde').auto() or have_system or have_tools vde = cc.find_library('vdeplug', has_headers: ['libvdeplug.h'], - required: get_option('vde'), - kwargs: static_kwargs) + required: get_option('vde')) endif if vde.found() and not cc.links(''' #include <libvdeplug.h> @@ -722,41 +994,41 @@ endif pulse = not_found if not get_option('pa').auto() or (targetos == 'linux' and have_system) pulse = dependency('libpulse', required: get_option('pa'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif alsa = not_found if not get_option('alsa').auto() or (targetos == 'linux' and have_system) alsa = dependency('alsa', required: get_option('alsa'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif jack = not_found if not get_option('jack').auto() or have_system jack = dependency('jack', required: get_option('jack'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif pipewire = not_found if not get_option('pipewire').auto() or (targetos == 'linux' and have_system) pipewire = dependency('libpipewire-0.3', version: '>=0.3.60', required: get_option('pipewire'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif sndio = not_found if not get_option('sndio').auto() or have_system sndio = dependency('sndio', required: get_option('sndio'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif spice_protocol = not_found if not get_option('spice_protocol').auto() or have_system spice_protocol = dependency('spice-protocol', version: '>=0.14.0', required: get_option('spice_protocol'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif spice = not_found if not get_option('spice').auto() or have_system spice = dependency('spice-server', version: '>=0.14.0', required: get_option('spice'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif spice_headers = spice.partial_dependency(compile_args: true, includes: true) @@ -766,13 +1038,13 @@ libiscsi = not_found if not get_option('libiscsi').auto() or have_block libiscsi = dependency('libiscsi', version: '>=1.9.0', required: get_option('libiscsi'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif zstd = not_found if not get_option('zstd').auto() or have_block zstd = dependency('libzstd', version: '>=1.4.0', required: get_option('zstd'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif virgl = not_found @@ -780,29 +1052,25 @@ have_vhost_user_gpu = have_tools and targetos == 'linux' and pixman.found() if not get_option('virglrenderer').auto() or have_system or have_vhost_user_gpu virgl = dependency('virglrenderer', method: 'pkg-config', - required: get_option('virglrenderer'), - kwargs: static_kwargs) + required: get_option('virglrenderer')) endif blkio = not_found if not get_option('blkio').auto() or have_block blkio = dependency('blkio', method: 'pkg-config', - required: get_option('blkio'), - kwargs: static_kwargs) + required: get_option('blkio')) endif curl = not_found if not get_option('curl').auto() or have_block curl = dependency('libcurl', version: '>=7.29.0', method: 'pkg-config', - required: get_option('curl'), - kwargs: static_kwargs) + required: get_option('curl')) endif libudev = not_found if targetos == 'linux' and (have_system or have_tools) libudev = dependency('libudev', method: 'pkg-config', - required: get_option('libudev'), - kwargs: static_kwargs) + required: get_option('libudev')) endif mpathlibs = [libudev] @@ -837,18 +1105,15 @@ if targetos == 'linux' and have_tools and get_option('mpath').allowed() return 0; }''' libmpathpersist = cc.find_library('mpathpersist', - required: get_option('mpath'), - kwargs: static_kwargs) + required: get_option('mpath')) if libmpathpersist.found() mpathlibs += libmpathpersist - if enable_static + if get_option('prefer_static') mpathlibs += cc.find_library('devmapper', - required: get_option('mpath'), - kwargs: static_kwargs) + required: get_option('mpath')) endif mpathlibs += cc.find_library('multipath', - required: get_option('mpath'), - kwargs: static_kwargs) + required: get_option('mpath')) foreach lib: mpathlibs if not lib.found() mpathlibs = [] @@ -898,13 +1163,13 @@ if have_system and get_option('curses').allowed() curses_dep_list = targetos == 'windows' ? ['ncurses', 'ncursesw'] : ['ncursesw'] curses = dependency(curses_dep_list, required: false, - method: 'pkg-config', - kwargs: static_kwargs) + method: 'pkg-config') msg = get_option('curses').enabled() ? 'curses library not found' : '' curses_compile_args = ['-DNCURSES_WIDECHAR=1'] if curses.found() if cc.links(curses_test, args: curses_compile_args, dependencies: [curses]) - curses = declare_dependency(compile_args: curses_compile_args, dependencies: [curses]) + curses = declare_dependency(compile_args: curses_compile_args, dependencies: [curses], + version: curses.version()) else msg = 'curses package not usable' curses = not_found @@ -921,8 +1186,7 @@ if have_system and get_option('curses').allowed() curses_libname_list = (targetos == 'windows' ? ['pdcurses'] : ['ncursesw', 'cursesw']) foreach curses_libname : curses_libname_list libcurses = cc.find_library(curses_libname, - required: false, - kwargs: static_kwargs) + required: false) if libcurses.found() if cc.links(curses_test, args: curses_compile_args, dependencies: libcurses) curses = declare_dependency(compile_args: curses_compile_args, @@ -947,7 +1211,7 @@ if have_system and get_option('curses').allowed() int main(void) { iconv_t conv = iconv_open("WCHAR_T", "UCS-2"); return conv != (iconv_t) -1; - }''', args: config_host['GLIB_CFLAGS'].split() + config_host['GLIB_LIBS'].split() + link_args) + }''', args: link_args, dependencies: glib) iconv = declare_dependency(link_args: link_args, dependencies: glib) break endif @@ -972,8 +1236,7 @@ endif brlapi = not_found if not get_option('brlapi').auto() or have_system brlapi = cc.find_library('brlapi', has_headers: ['brlapi.h'], - required: get_option('brlapi'), - kwargs: static_kwargs) + required: get_option('brlapi')) if brlapi.found() and not cc.links(''' #include <brlapi.h> #include <stddef.h> @@ -989,15 +1252,16 @@ endif sdl = not_found if not get_option('sdl').auto() or have_system - sdl = dependency('sdl2', required: get_option('sdl'), kwargs: static_kwargs) + sdl = dependency('sdl2', required: get_option('sdl')) sdl_image = not_found endif if sdl.found() # work around 2.0.8 bug sdl = declare_dependency(compile_args: '-Wno-undef', - dependencies: sdl) + dependencies: sdl, + version: sdl.version()) sdl_image = dependency('SDL2_image', required: get_option('sdl_image'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') else if get_option('sdl_image').enabled() error('sdl-image required, but SDL was @0@'.format( @@ -1008,11 +1272,9 @@ endif rbd = not_found if not get_option('rbd').auto() or have_block - librados = cc.find_library('rados', required: get_option('rbd'), - kwargs: static_kwargs) + librados = cc.find_library('rados', required: get_option('rbd')) librbd = cc.find_library('rbd', has_headers: ['rbd/librbd.h'], - required: get_option('rbd'), - kwargs: static_kwargs) + required: get_option('rbd')) if librados.found() and librbd.found() if cc.links(''' #include <stdio.h> @@ -1040,7 +1302,7 @@ glusterfs_iocb_has_stat = false if not get_option('glusterfs').auto() or have_block glusterfs = dependency('glusterfs-api', version: '>=3', required: get_option('glusterfs'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') if glusterfs.found() glusterfs_ftruncate_has_stat = cc.links(''' #include <glusterfs/api/glfs.h> @@ -1075,15 +1337,13 @@ libssh = not_found if not get_option('libssh').auto() or have_block libssh = dependency('libssh', version: '>=0.8.7', method: 'pkg-config', - required: get_option('libssh'), - kwargs: static_kwargs) + required: get_option('libssh')) endif libbzip2 = not_found if not get_option('bzip2').auto() or have_block libbzip2 = cc.find_library('bz2', has_headers: ['bzlib.h'], - required: get_option('bzip2'), - kwargs: static_kwargs) + required: get_option('bzip2')) if libbzip2.found() and not cc.links(''' #include <bzlib.h> int main(void) { BZ2_bzlibVersion(); return 0; }''', dependencies: libbzip2) @@ -1099,8 +1359,7 @@ endif liblzfse = not_found if not get_option('lzfse').auto() or have_block liblzfse = cc.find_library('lzfse', has_headers: ['lzfse.h'], - required: get_option('lzfse'), - kwargs: static_kwargs) + required: get_option('lzfse')) endif if liblzfse.found() and not cc.links(''' #include <lzfse.h> @@ -1118,8 +1377,7 @@ if get_option('oss').allowed() and have_system if not cc.has_header('sys/soundcard.h') # not found elif targetos == 'netbsd' - oss = cc.find_library('ossaudio', required: get_option('oss'), - kwargs: static_kwargs) + oss = cc.find_library('ossaudio', required: get_option('oss')) else oss = declare_dependency() endif @@ -1152,7 +1410,7 @@ endif opengl = not_found if not get_option('opengl').auto() or have_system or have_vhost_user_gpu epoxy = dependency('epoxy', method: 'pkg-config', - required: get_option('opengl'), kwargs: static_kwargs) + required: get_option('opengl')) if cc.has_header('epoxy/egl.h', dependencies: epoxy) opengl = epoxy elif get_option('opengl').enabled() @@ -1161,8 +1419,7 @@ if not get_option('opengl').auto() or have_system or have_vhost_user_gpu endif gbm = not_found if (have_system or have_tools) and (virgl.found() or opengl.found()) - gbm = dependency('gbm', method: 'pkg-config', required: false, - kwargs: static_kwargs) + gbm = dependency('gbm', method: 'pkg-config', required: false) endif have_vhost_user_gpu = have_vhost_user_gpu and virgl.found() and opengl.found() and gbm.found() @@ -1184,16 +1441,14 @@ if get_option('gnutls').enabled() or (get_option('gnutls').auto() and have_syste # the platform support requirements gnutls_crypto = dependency('gnutls', version: '>=3.6.14', method: 'pkg-config', - required: false, - kwargs: static_kwargs) + required: false) if gnutls_crypto.found() gnutls = gnutls_crypto else # Our min version if all we need is TLS gnutls = dependency('gnutls', version: '>=3.5.18', method: 'pkg-config', - required: get_option('gnutls'), - kwargs: static_kwargs) + required: get_option('gnutls')) endif endif @@ -1220,34 +1475,32 @@ if not gnutls_crypto.found() if (not get_option('gcrypt').auto() or have_system) and not get_option('nettle').enabled() gcrypt = dependency('libgcrypt', version: '>=1.8', method: 'config-tool', - required: get_option('gcrypt'), - kwargs: static_kwargs) + required: get_option('gcrypt')) # Debian has removed -lgpg-error from libgcrypt-config # as it "spreads unnecessary dependencies" which in # turn breaks static builds... - if gcrypt.found() and enable_static - gcrypt = declare_dependency(dependencies: [ - gcrypt, - cc.find_library('gpg-error', required: true, kwargs: static_kwargs)]) + if gcrypt.found() and get_option('prefer_static') + gcrypt = declare_dependency(dependencies: + [gcrypt, + cc.find_library('gpg-error', required: true)], + version: gcrypt.version()) endif endif if (not get_option('nettle').auto() or have_system) and not gcrypt.found() nettle = dependency('nettle', version: '>=3.4', method: 'pkg-config', - required: get_option('nettle'), - kwargs: static_kwargs) + required: get_option('nettle')) if nettle.found() and not cc.has_header('nettle/xts.h', dependencies: nettle) xts = 'private' endif endif endif -gmp = dependency('gmp', required: false, method: 'pkg-config', kwargs: static_kwargs) +gmp = dependency('gmp', required: false, method: 'pkg-config') if nettle.found() and gmp.found() hogweed = dependency('hogweed', version: '>=3.4', method: 'pkg-config', - required: get_option('nettle'), - kwargs: static_kwargs) + required: get_option('nettle')) endif @@ -1259,20 +1512,18 @@ have_gtk_clipboard = get_option('gtk_clipboard').enabled() if not get_option('gtk').auto() or have_system gtk = dependency('gtk+-3.0', version: '>=3.22.0', method: 'pkg-config', - required: get_option('gtk'), - kwargs: static_kwargs) + required: get_option('gtk')) if gtk.found() gtkx11 = dependency('gtk+-x11-3.0', version: '>=3.22.0', method: 'pkg-config', - required: false, - kwargs: static_kwargs) - gtk = declare_dependency(dependencies: [gtk, gtkx11]) + required: false) + gtk = declare_dependency(dependencies: [gtk, gtkx11], + version: gtk.version()) if not get_option('vte').auto() or have_system vte = dependency('vte-2.91', method: 'pkg-config', - required: get_option('vte'), - kwargs: static_kwargs) + required: get_option('vte')) endif elif have_gtk_clipboard error('GTK clipboard requested, but GTK not found') @@ -1281,13 +1532,12 @@ endif x11 = not_found if gtkx11.found() - x11 = dependency('x11', method: 'pkg-config', required: gtkx11.found(), - kwargs: static_kwargs) + x11 = dependency('x11', method: 'pkg-config', required: gtkx11.found()) endif png = not_found if get_option('png').allowed() and have_system png = dependency('libpng', version: '>=1.6.34', required: get_option('png'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif vnc = not_found jpeg = not_found @@ -1295,10 +1545,9 @@ sasl = not_found if get_option('vnc').allowed() and have_system vnc = declare_dependency() # dummy dependency jpeg = dependency('libjpeg', required: get_option('vnc_jpeg'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') sasl = cc.find_library('sasl2', has_headers: ['sasl/sasl.h'], - required: get_option('vnc_sasl'), - kwargs: static_kwargs) + required: get_option('vnc_sasl')) if sasl.found() sasl = declare_dependency(dependencies: sasl, compile_args: '-DSTRUCT_IOVEC_DEFINED') @@ -1308,8 +1557,7 @@ endif pam = not_found if not get_option('auth_pam').auto() or have_system pam = cc.find_library('pam', has_headers: ['security/pam_appl.h'], - required: get_option('auth_pam'), - kwargs: static_kwargs) + required: get_option('auth_pam')) endif if pam.found() and not cc.links(''' #include <stddef.h> @@ -1333,8 +1581,7 @@ endif snappy = not_found if not get_option('snappy').auto() or have_system snappy = cc.find_library('snappy', has_headers: ['snappy-c.h'], - required: get_option('snappy'), - kwargs: static_kwargs) + required: get_option('snappy')) endif if snappy.found() and not linker.links(''' #include <snappy-c.h> @@ -1350,8 +1597,7 @@ endif lzo = not_found if not get_option('lzo').auto() or have_system lzo = cc.find_library('lzo2', has_headers: ['lzo/lzo1x.h'], - required: get_option('lzo'), - kwargs: static_kwargs) + required: get_option('lzo')) endif if lzo.found() and not cc.links(''' #include <lzo/lzo1x.h> @@ -1367,8 +1613,7 @@ endif numa = not_found if not get_option('numa').auto() or have_system or have_tools numa = cc.find_library('numa', has_headers: ['numa.h'], - required: get_option('numa'), - kwargs: static_kwargs) + required: get_option('numa')) endif if numa.found() and not cc.links(''' #include <numa.h> @@ -1386,10 +1631,8 @@ rdma = not_found if not get_option('rdma').auto() or have_system libumad = cc.find_library('ibumad', required: get_option('rdma')) rdma_libs = [cc.find_library('rdmacm', has_headers: ['rdma/rdma_cma.h'], - required: get_option('rdma'), - kwargs: static_kwargs), - cc.find_library('ibverbs', required: get_option('rdma'), - kwargs: static_kwargs), + required: get_option('rdma')), + cc.find_library('ibverbs', required: get_option('rdma')), libumad] rdma = declare_dependency(dependencies: rdma_libs) foreach lib: rdma_libs @@ -1402,30 +1645,30 @@ endif xen = not_found if get_option('xen').enabled() or (get_option('xen').auto() and have_system) xencontrol = dependency('xencontrol', required: false, - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') if xencontrol.found() xen_pc = declare_dependency(version: xencontrol.version(), dependencies: [ xencontrol, # disabler: true makes xen_pc.found() return false if any is not found dependency('xenstore', required: false, - method: 'pkg-config', kwargs: static_kwargs, + method: 'pkg-config', disabler: true), dependency('xenforeignmemory', required: false, - method: 'pkg-config', kwargs: static_kwargs, + method: 'pkg-config', disabler: true), dependency('xengnttab', required: false, - method: 'pkg-config', kwargs: static_kwargs, + method: 'pkg-config', disabler: true), dependency('xenevtchn', required: false, - method: 'pkg-config', kwargs: static_kwargs, + method: 'pkg-config', disabler: true), dependency('xendevicemodel', required: false, - method: 'pkg-config', kwargs: static_kwargs, + method: 'pkg-config', disabler: true), # optional, no "disabler: true" dependency('xentoolcore', required: false, - method: 'pkg-config', kwargs: static_kwargs)]) + method: 'pkg-config')]) if xen_pc.found() xen = xen_pc endif @@ -1483,60 +1726,53 @@ have_xen_pci_passthrough = get_option('xen_pci_passthrough') \ cacard = not_found if not get_option('smartcard').auto() or have_system cacard = dependency('libcacard', required: get_option('smartcard'), - version: '>=2.5.1', method: 'pkg-config', - kwargs: static_kwargs) + version: '>=2.5.1', method: 'pkg-config') endif u2f = not_found if have_system u2f = dependency('u2f-emu', required: get_option('u2f'), - method: 'pkg-config', - kwargs: static_kwargs) + method: 'pkg-config') endif canokey = not_found if have_system canokey = dependency('canokey-qemu', required: get_option('canokey'), - method: 'pkg-config', - kwargs: static_kwargs) + method: 'pkg-config') endif usbredir = not_found if not get_option('usb_redir').auto() or have_system usbredir = dependency('libusbredirparser-0.5', required: get_option('usb_redir'), - version: '>=0.6', method: 'pkg-config', - kwargs: static_kwargs) + version: '>=0.6', method: 'pkg-config') endif libusb = not_found if not get_option('libusb').auto() or have_system libusb = dependency('libusb-1.0', required: get_option('libusb'), - version: '>=1.0.13', method: 'pkg-config', - kwargs: static_kwargs) + version: '>=1.0.13', method: 'pkg-config') endif libpmem = not_found if not get_option('libpmem').auto() or have_system libpmem = dependency('libpmem', required: get_option('libpmem'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') endif libdaxctl = not_found if not get_option('libdaxctl').auto() or have_system libdaxctl = dependency('libdaxctl', required: get_option('libdaxctl'), - version: '>=57', method: 'pkg-config', - kwargs: static_kwargs) + version: '>=57', method: 'pkg-config') endif tasn1 = not_found if gnutls.found() tasn1 = dependency('libtasn1', - method: 'pkg-config', - kwargs: static_kwargs) + method: 'pkg-config') endif keyutils = dependency('libkeyutils', required: false, - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') has_gettid = cc.has_function('gettid') # libselinux selinux = dependency('libselinux', required: get_option('selinux'), - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') # Malloc tests @@ -1600,8 +1836,7 @@ if get_option('fuse').disabled() and get_option('fuse_lseek').enabled() endif fuse = dependency('fuse3', required: get_option('fuse'), - version: '>=3.1', method: 'pkg-config', - kwargs: static_kwargs) + version: '>=3.1', method: 'pkg-config') fuse_lseek = not_found if get_option('fuse_lseek').allowed() @@ -1657,10 +1892,9 @@ endif # libdw libdw = not_found if not get_option('libdw').auto() or \ - (not enable_static and (have_system or have_user)) + (not get_option('prefer_static') and (have_system or have_user)) libdw = dependency('libdw', method: 'pkg-config', - kwargs: static_kwargs, required: get_option('libdw')) endif @@ -1715,7 +1949,7 @@ if get_option('cfi') if not get_option('b_lto') error('Selected Control-Flow Integrity but LTO is disabled') endif - if config_host.has_key('CONFIG_MODULES') + if enable_modules error('Selected Control-Flow Integrity is not compatible with modules') endif # Check for cfi flags. CFI requires LTO so we can't use @@ -1818,7 +2052,7 @@ config_host_data.set_quoted('CONFIG_QEMU_LOCALSTATEDIR', get_option('prefix') / config_host_data.set_quoted('CONFIG_QEMU_MODDIR', get_option('prefix') / qemu_moddir) config_host_data.set_quoted('CONFIG_SYSCONFDIR', get_option('prefix') / get_option('sysconfdir')) -if config_host.has_key('CONFIG_MODULES') +if enable_modules config_host_data.set('CONFIG_STAMP', run_command( meson.current_source_dir() / 'scripts/qemu-stamp.py', meson.project_version(), get_option('pkgversion'), '--', @@ -1881,6 +2115,7 @@ config_host_data.set('CONFIG_LIBSSH', libssh.found()) config_host_data.set('CONFIG_LINUX_AIO', libaio.found()) config_host_data.set('CONFIG_LINUX_IO_URING', linux_io_uring.found()) config_host_data.set('CONFIG_LIBPMEM', libpmem.found()) +config_host_data.set('CONFIG_MODULES', enable_modules) config_host_data.set('CONFIG_NUMA', numa.found()) if numa.found() config_host_data.set('HAVE_NUMA_HAS_PREFERRED_MANY', @@ -1891,6 +2126,7 @@ config_host_data.set('CONFIG_OPENGL', opengl.found()) config_host_data.set('CONFIG_PROFILER', get_option('profiler')) config_host_data.set('CONFIG_RBD', rbd.found()) config_host_data.set('CONFIG_RDMA', rdma.found()) +config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack')) config_host_data.set('CONFIG_SDL', sdl.found()) config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found()) config_host_data.set('CONFIG_SECCOMP', seccomp.found()) @@ -1899,6 +2135,7 @@ if seccomp.found() endif config_host_data.set('CONFIG_SNAPPY', snappy.found()) config_host_data.set('CONFIG_TPM', have_tpm) +config_host_data.set('CONFIG_TSAN', get_option('tsan')) config_host_data.set('CONFIG_USB_LIBUSB', libusb.found()) config_host_data.set('CONFIG_VDE', vde.found()) config_host_data.set('CONFIG_VHOST_NET', have_vhost_net) @@ -2013,6 +2250,7 @@ config_host_data.set('CONFIG_SYNC_FILE_RANGE', cc.has_function('sync_file_range' config_host_data.set('CONFIG_TIMERFD', cc.has_function('timerfd_create')) config_host_data.set('HAVE_COPY_FILE_RANGE', cc.has_function('copy_file_range')) config_host_data.set('HAVE_GETIFADDRS', cc.has_function('getifaddrs')) +config_host_data.set('HAVE_GLIB_WITH_SLICE_ALLOCATOR', glib_has_gslice) config_host_data.set('HAVE_OPENPTY', cc.has_function('openpty', dependencies: util)) config_host_data.set('HAVE_STRCHRNUL', cc.has_function('strchrnul')) config_host_data.set('HAVE_SYSTEM_FUNCTION', cc.has_function('system', prefix: '#include <stdlib.h>')) @@ -2029,6 +2267,18 @@ if rdma.found() prefix: '#include <infiniband/verbs.h>')) endif +have_asan_fiber = false +if get_option('sanitizers') and \ + not cc.has_function('__sanitizer_start_switch_fiber', + args: '-fsanitize=address', + prefix: '#include <sanitizer/asan_interface.h>') + warning('Missing ASAN due to missing fiber annotation interface') + warning('Without code annotation, the report may be inferior.') +else + have_asan_fiber = true +endif +config_host_data.set('CONFIG_ASAN_IFACE_FIBER', have_asan_fiber) + # has_header_symbol config_host_data.set('CONFIG_BLKZONED', cc.has_header_symbol('linux/blkzoned.h', 'BLKOPENZONE')) @@ -2761,7 +3011,7 @@ genh += custom_target('config-poison.h', capstone = not_found if not get_option('capstone').auto() or have_system or have_user capstone = dependency('capstone', version: '>=3.0.5', - kwargs: static_kwargs, method: 'pkg-config', + method: 'pkg-config', required: get_option('capstone')) # Some versions of capstone have broken pkg-config file @@ -2787,9 +3037,7 @@ if have_system and vfio_user_server_allowed libvfio_user_proj = subproject('libvfio-user') - libvfio_user_lib = libvfio_user_proj.get_variable('libvfio_user_dep') - - libvfio_user_dep = declare_dependency(dependencies: [libvfio_user_lib]) + libvfio_user_dep = libvfio_user_proj.get_variable('libvfio_user_dep') endif fdt = not_found @@ -2797,7 +3045,7 @@ if have_system fdt_opt = get_option('fdt') if fdt_opt in ['enabled', 'auto', 'system'] have_internal = fs.exists(meson.current_source_dir() / 'dtc/libfdt/Makefile.libfdt') - fdt = cc.find_library('fdt', kwargs: static_kwargs, + fdt = cc.find_library('fdt', required: fdt_opt == 'system' or fdt_opt == 'enabled' and not have_internal) if fdt.found() and cc.links(''' @@ -3103,7 +3351,6 @@ subdir('ui') subdir('hw') subdir('gdbstub') - if enable_modules libmodulecommon = static_library('module-common', files('module-common.c') + genh, pic: true, c_args: '-DBUILD_DSO') modulecommon = declare_dependency(link_whole: libmodulecommon, compile_args: '-DBUILD_DSO') @@ -3246,7 +3493,7 @@ foreach d, list : modules endif foreach m, module_ss : list - if enable_modules and targetos != 'windows' + if enable_modules module_ss = module_ss.apply(config_all, strict: false) sl = static_library(d + '-' + m, [genh, module_ss.sources()], dependencies: [modulecommon, module_ss.dependencies()], pic: true) @@ -3279,7 +3526,7 @@ endforeach foreach d, list : target_modules foreach m, module_ss : list - if enable_modules and targetos != 'windows' + if enable_modules foreach target : target_dirs if target.endswith('-softmmu') config_target = config_target_mak[target] @@ -3725,7 +3972,7 @@ if host_machine.system() == 'windows' '@OUTPUT@', get_option('prefix'), meson.current_source_dir(), - config_host['GLIB_BINDIR'], + glib_pc.get_variable('bindir'), host_machine.cpu(), '--', '-DDISPLAYVERSION=' + meson.project_version(), @@ -3799,8 +4046,8 @@ summary_info += {'system-mode emulation': have_system} summary_info += {'user-mode emulation': have_user} summary_info += {'block layer': have_block} summary_info += {'Install blobs': get_option('install_blobs')} -summary_info += {'module support': config_host.has_key('CONFIG_MODULES')} -if config_host.has_key('CONFIG_MODULES') +summary_info += {'module support': enable_modules} +if enable_modules summary_info += {'alternative module path': get_option('module_upgrades')} endif summary_info += {'fuzzing support': get_option('fuzzing')} @@ -3851,12 +4098,12 @@ link_args = get_option(link_language + '_link_args') if link_args.length() > 0 summary_info += {'LDFLAGS': ' '.join(link_args)} endif -summary_info += {'QEMU_CFLAGS': ' '.join(qemu_cflags)} +summary_info += {'QEMU_CFLAGS': ' '.join(qemu_common_flags + qemu_cflags)} if 'cpp' in all_languages - summary_info += {'QEMU_CXXFLAGS': ' '.join(qemu_cxxflags)} + summary_info += {'QEMU_CXXFLAGS': ' '.join(qemu_common_flags + qemu_cxxflags)} endif if 'objc' in all_languages - summary_info += {'QEMU_OBJCFLAGS': ' '.join(qemu_objcflags)} + summary_info += {'QEMU_OBJCFLAGS': ' '.join(qemu_common_flags)} endif summary_info += {'QEMU_LDFLAGS': ' '.join(qemu_ldflags)} summary_info += {'profiler': get_option('profiler')} @@ -3879,7 +4126,7 @@ else endif summary_info += {'gprof': gprof_info} summary_info += {'gcov': get_option('b_coverage')} -summary_info += {'thread sanitizer': config_host.has_key('CONFIG_TSAN')} +summary_info += {'thread sanitizer': get_option('tsan')} summary_info += {'CFI support': get_option('cfi')} if get_option('cfi') summary_info += {'CFI debug support': get_option('cfi_debug')} @@ -3940,7 +4187,7 @@ summary(summary_info, bool_yn: true, section: 'Targets and accelerators') # Block layer summary_info = {} -summary_info += {'coroutine backend': config_host['CONFIG_COROUTINE_BACKEND']} +summary_info += {'coroutine backend': coroutine_backend} summary_info += {'coroutine pool': have_coroutine_pool} if have_block summary_info += {'Block whitelist (rw)': get_option('block_drv_rw_whitelist')} diff --git a/meson_options.txt b/meson_options.txt index 11aec2a..972c458 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -12,8 +12,6 @@ option('pkgversion', type : 'string', value : '', description: 'use specified string as sub-version of the package') option('smbd', type : 'string', value : '', description: 'Path to smbd for slirp networking') -option('sphinx_build', type : 'string', value : 'sphinx-build', - description: 'Use specified sphinx-build for building document') option('iasl', type : 'string', value : '', description: 'Path to ACPI disassembler') option('tls_priority', type : 'string', value : 'NORMAL', @@ -33,6 +31,9 @@ option('fuzzing_engine', type : 'string', value : '', description: 'fuzzing engine library for OSS-Fuzz') option('trace_file', type: 'string', value: 'trace', description: 'Trace file prefix for simple backend') +option('coroutine_backend', type: 'combo', + choices: ['ucontext', 'sigaltstack', 'windows', 'auto'], + value: 'auto', description: 'coroutine backend to use') # Everything else can be set via --enable/--disable-* option # on the configure script command line. After adding an option @@ -44,6 +45,8 @@ option('fuzzing', type : 'boolean', value: false, description: 'build fuzzing targets') option('gettext', type : 'feature', value : 'auto', description: 'Localization of the GTK+ user interface') +option('modules', type : 'feature', value : 'disabled', + description: 'modules support (non Windows)') option('module_upgrades', type : 'boolean', value : false, description: 'try to load modules from alternate paths for upgrades') option('install_blobs', type : 'boolean', value : true, @@ -82,6 +85,14 @@ option('tcg', type: 'feature', value: 'enabled', description: 'TCG support') option('tcg_interpreter', type: 'boolean', value: false, description: 'TCG with bytecode interpreter (slow)') +option('safe_stack', type: 'boolean', value: false, + description: 'SafeStack Stack Smash Protection (requires clang/llvm and coroutine backend ucontext)') +option('sanitizers', type: 'boolean', value: false, + description: 'enable default sanitizers') +option('tsan', type: 'boolean', value: false, + description: 'enable thread sanitizer') +option('stack_protector', type: 'feature', value: 'auto', + description: 'compiler-provided stack protection') option('cfi', type: 'boolean', value: false, description: 'Control-Flow Integrity (CFI)') option('cfi_debug', type: 'boolean', value: false, diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 0220db8..84f1b0f 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -101,7 +101,7 @@ void global_dirty_log_change(unsigned int flag, bool start) static void global_dirty_log_sync(unsigned int flag, bool one_shot) { qemu_mutex_lock_iothread(); - memory_global_dirty_log_sync(); + memory_global_dirty_log_sync(false); if (one_shot) { memory_global_dirty_log_stop(flag); } @@ -581,7 +581,7 @@ static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config) * skip it unconditionally and start dirty tracking * from 2'round of log sync */ - memory_global_dirty_log_sync(); + memory_global_dirty_log_sync(false); /* * reset page protect manually and unconditionally. diff --git a/migration/ram.c b/migration/ram.c index f69d8d4..5900cab 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -1039,7 +1039,7 @@ static void migration_trigger_throttle(RAMState *rs) } } -static void migration_bitmap_sync(RAMState *rs) +static void migration_bitmap_sync(RAMState *rs, bool last_stage) { RAMBlock *block; int64_t end_time; @@ -1051,7 +1051,7 @@ static void migration_bitmap_sync(RAMState *rs) } trace_migration_bitmap_sync_start(); - memory_global_dirty_log_sync(); + memory_global_dirty_log_sync(last_stage); qemu_mutex_lock(&rs->bitmap_mutex); WITH_RCU_READ_LOCK_GUARD() { @@ -1086,7 +1086,7 @@ static void migration_bitmap_sync(RAMState *rs) } } -static void migration_bitmap_sync_precopy(RAMState *rs) +static void migration_bitmap_sync_precopy(RAMState *rs, bool last_stage) { Error *local_err = NULL; @@ -1099,7 +1099,7 @@ static void migration_bitmap_sync_precopy(RAMState *rs) local_err = NULL; } - migration_bitmap_sync(rs); + migration_bitmap_sync(rs, last_stage); if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) { error_report_err(local_err); @@ -2699,7 +2699,7 @@ void ram_postcopy_send_discard_bitmap(MigrationState *ms) RCU_READ_LOCK_GUARD(); /* This should be our last sync, the src is now paused */ - migration_bitmap_sync(rs); + migration_bitmap_sync(rs, false); /* Easiest way to make sure we don't resume in the middle of a host-page */ rs->pss[RAM_CHANNEL_PRECOPY].last_sent_block = NULL; @@ -2890,7 +2890,7 @@ static void ram_init_bitmaps(RAMState *rs) /* We don't use dirty log with background snapshots */ if (!migrate_background_snapshot()) { memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION); - migration_bitmap_sync_precopy(rs); + migration_bitmap_sync_precopy(rs, false); } } qemu_mutex_unlock_ramlist(); @@ -3214,7 +3214,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque) WITH_RCU_READ_LOCK_GUARD() { if (!migration_in_postcopy()) { - migration_bitmap_sync_precopy(rs); + migration_bitmap_sync_precopy(rs, true); } ram_control_before_iterate(f, RAM_CONTROL_FINISH); @@ -3288,7 +3288,7 @@ static void ram_state_pending_exact(void *opaque, uint64_t *must_precopy, if (!migration_in_postcopy() && remaining_size < s->threshold_size) { qemu_mutex_lock_iothread(); WITH_RCU_READ_LOCK_GUARD() { - migration_bitmap_sync_precopy(rs); + migration_bitmap_sync_precopy(rs, false); } qemu_mutex_unlock_iothread(); remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE; @@ -3523,7 +3523,7 @@ void colo_incoming_start_dirty_log(void) qemu_mutex_lock_iothread(); qemu_mutex_lock_ramlist(); - memory_global_dirty_log_sync(); + memory_global_dirty_log_sync(false); WITH_RCU_READ_LOCK_GUARD() { RAMBLOCK_FOREACH_NOT_IGNORED(block) { ramblock_sync_dirty_bitmap(ram_state, block); @@ -3813,7 +3813,7 @@ void colo_flush_ram_cache(void) void *src_host; unsigned long offset = 0; - memory_global_dirty_log_sync(); + memory_global_dirty_log_sync(false); qemu_mutex_lock(&ram_state->bitmap_mutex); WITH_RCU_READ_LOCK_GUARD() { RAMBLOCK_FOREACH_NOT_IGNORED(block) { diff --git a/python/Makefile b/python/Makefile index c5bd6ff..7c70dcc 100644 --- a/python/Makefile +++ b/python/Makefile @@ -9,14 +9,14 @@ help: @echo "make check-minreqs:" @echo " Run tests in the minreqs virtual environment." @echo " These tests use the oldest dependencies." - @echo " Requires: Python 3.6" - @echo " Hint (Fedora): 'sudo dnf install python3.6'" + @echo " Requires: Python 3.7" + @echo " Hint (Fedora): 'sudo dnf install python3.7'" @echo "" @echo "make check-tox:" @echo " Run tests against multiple python versions." @echo " These tests use the newest dependencies." - @echo " Requires: Python 3.6 - 3.10, and tox." - @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.10'" + @echo " Requires: Python 3.7 - 3.11, and tox." + @echo " Hint (Fedora): 'sudo dnf install python3-tox python3.11'" @echo " The variable QEMU_TOX_EXTRA_ARGS can be use to pass extra" @echo " arguments to tox". @echo "" @@ -54,18 +54,21 @@ pipenv check-pipenv: @echo "pipenv was dropped; try 'make check-minreqs' or 'make min-venv'" @exit 1 +PIP_INSTALL = pip install --disable-pip-version-check .PHONY: min-venv min-venv: $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate: setup.cfg tests/minreqs.txt @echo "VENV $(QEMU_MINVENV_DIR)" - @python3.6 -m venv $(QEMU_MINVENV_DIR) + @python3.7 -m venv $(QEMU_MINVENV_DIR) @( \ echo "ACTIVATE $(QEMU_MINVENV_DIR)"; \ . $(QEMU_MINVENV_DIR)/bin/activate; \ + echo "INSTALL wheel $(QEMU_MINVENV_DIR)"; \ + $(PIP_INSTALL) wheel 1>/dev/null; \ echo "INSTALL -r tests/minreqs.txt $(QEMU_MINVENV_DIR)";\ - pip install -r tests/minreqs.txt 1>/dev/null; \ + $(PIP_INSTALL) -r tests/minreqs.txt 1>/dev/null; \ echo "INSTALL -e qemu $(QEMU_MINVENV_DIR)"; \ - pip install -e . 1>/dev/null; \ + $(PIP_INSTALL) -e . 1>/dev/null; \ ) @touch $(QEMU_MINVENV_DIR) @@ -100,7 +103,7 @@ check-dev: dev-venv .PHONY: develop develop: - pip3 install --disable-pip-version-check -e .[devel] + $(PIP_INSTALL) -e .[devel] .PHONY: check check: diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py new file mode 100644 index 0000000..8c036c0 --- /dev/null +++ b/python/scripts/mkvenv.py @@ -0,0 +1,897 @@ +""" +mkvenv - QEMU pyvenv bootstrapping utility + +usage: mkvenv [-h] command ... + +QEMU pyvenv bootstrapping utility + +options: + -h, --help show this help message and exit + +Commands: + command Description + create create a venv + post_init + post-venv initialization + ensure Ensure that the specified package is installed. + +-------------------------------------------------- + +usage: mkvenv create [-h] target + +positional arguments: + target Target directory to install virtual environment into. + +options: + -h, --help show this help message and exit + +-------------------------------------------------- + +usage: mkvenv post_init [-h] + +options: + -h, --help show this help message and exit + +-------------------------------------------------- + +usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... + +positional arguments: + dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' + +options: + -h, --help show this help message and exit + --online Install packages from PyPI, if necessary. + --dir DIR Path to vendored packages where we may install from. + +""" + +# Copyright (C) 2022-2023 Red Hat, Inc. +# +# Authors: +# John Snow <jsnow@redhat.com> +# Paolo Bonzini <pbonzini@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + +import argparse +from importlib.util import find_spec +import logging +import os +from pathlib import Path +import re +import shutil +import site +import subprocess +import sys +import sysconfig +from types import SimpleNamespace +from typing import ( + Any, + Iterator, + Optional, + Sequence, + Tuple, + Union, +) +import venv +import warnings + + +# Try to load distlib, with a fallback to pip's vendored version. +# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail +# outside the venv or before a potential call to ensurepip in checkpip(). +HAVE_DISTLIB = True +try: + import distlib.database + import distlib.scripts + import distlib.version +except ImportError: + try: + # Reach into pip's cookie jar. pylint and flake8 don't understand + # that these imports will be used via distlib.xxx. + from pip._vendor import distlib + import pip._vendor.distlib.database # noqa, pylint: disable=unused-import + import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import + import pip._vendor.distlib.version # noqa, pylint: disable=unused-import + except ImportError: + HAVE_DISTLIB = False + +# Do not add any mandatory dependencies from outside the stdlib: +# This script *must* be usable standalone! + +DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] +logger = logging.getLogger("mkvenv") + + +def inside_a_venv() -> bool: + """Returns True if it is executed inside of a virtual environment.""" + return sys.prefix != sys.base_prefix + + +class Ouch(RuntimeError): + """An Exception class we can't confuse with a builtin.""" + + +class QemuEnvBuilder(venv.EnvBuilder): + """ + An extension of venv.EnvBuilder for building QEMU's configure-time venv. + + The primary difference is that it emulates a "nested" virtual + environment when invoked from inside of an existing virtual + environment by including packages from the parent. Also, + "ensurepip" is replaced if possible with just recreating pip's + console_scripts inside the virtual environment. + + Parameters for base class init: + - system_site_packages: bool = False + - clear: bool = False + - symlinks: bool = False + - upgrade: bool = False + - with_pip: bool = False + - prompt: Optional[str] = None + - upgrade_deps: bool = False (Since 3.9) + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + logger.debug("QemuEnvBuilder.__init__(...)") + + # For nested venv emulation: + self.use_parent_packages = False + if inside_a_venv(): + # Include parent packages only if we're in a venv and + # system_site_packages was True. + self.use_parent_packages = kwargs.pop( + "system_site_packages", False + ) + # Include system_site_packages only when the parent, + # The venv we are currently in, also does so. + kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES + + # ensurepip is slow: venv creation can be very fast for cases where + # we allow the use of system_site_packages. Therefore, ensurepip is + # replaced with our own script generation once the virtual environment + # is setup. + self.want_pip = kwargs.get("with_pip", False) + if self.want_pip: + if ( + kwargs.get("system_site_packages", False) + and not need_ensurepip() + ): + kwargs["with_pip"] = False + else: + check_ensurepip(suggest_remedy=True) + + super().__init__(*args, **kwargs) + + # Make the context available post-creation: + self._context: Optional[SimpleNamespace] = None + + def get_parent_libpath(self) -> Optional[str]: + """Return the libpath of the parent venv, if applicable.""" + if self.use_parent_packages: + return sysconfig.get_path("purelib") + return None + + @staticmethod + def compute_venv_libpath(context: SimpleNamespace) -> str: + """ + Compatibility wrapper for context.lib_path for Python < 3.12 + """ + # Python 3.12+, not strictly necessary because it's documented + # to be the same as 3.10 code below: + if sys.version_info >= (3, 12): + return context.lib_path + + # Python 3.10+ + if "venv" in sysconfig.get_scheme_names(): + lib_path = sysconfig.get_path( + "purelib", scheme="venv", vars={"base": context.env_dir} + ) + assert lib_path is not None + return lib_path + + # For Python <= 3.9 we need to hardcode this. Fortunately the + # code below was the same in Python 3.6-3.10, so there is only + # one case. + if sys.platform == "win32": + return os.path.join(context.env_dir, "Lib", "site-packages") + return os.path.join( + context.env_dir, + "lib", + "python%d.%d" % sys.version_info[:2], + "site-packages", + ) + + def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: + logger.debug("ensure_directories(env_dir=%s)", env_dir) + self._context = super().ensure_directories(env_dir) + return self._context + + def create(self, env_dir: DirType) -> None: + logger.debug("create(env_dir=%s)", env_dir) + super().create(env_dir) + assert self._context is not None + self.post_post_setup(self._context) + + def post_post_setup(self, context: SimpleNamespace) -> None: + """ + The final, final hook. Enter the venv and run commands inside of it. + """ + if self.use_parent_packages: + # We're inside of a venv and we want to include the parent + # venv's packages. + parent_libpath = self.get_parent_libpath() + assert parent_libpath is not None + logger.debug("parent_libpath: %s", parent_libpath) + + our_libpath = self.compute_venv_libpath(context) + logger.debug("our_libpath: %s", our_libpath) + + pth_file = os.path.join(our_libpath, "nested.pth") + with open(pth_file, "w", encoding="UTF-8") as file: + file.write(parent_libpath + os.linesep) + + if self.want_pip: + args = [ + context.env_exe, + __file__, + "post_init", + ] + subprocess.run(args, check=True) + + def get_value(self, field: str) -> str: + """ + Get a string value from the context namespace after a call to build. + + For valid field names, see: + https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories + """ + ret = getattr(self._context, field) + assert isinstance(ret, str) + return ret + + +def need_ensurepip() -> bool: + """ + Tests for the presence of setuptools and pip. + + :return: `True` if we do not detect both packages. + """ + # Don't try to actually import them, it's fraught with danger: + # https://github.com/pypa/setuptools/issues/2993 + if find_spec("setuptools") and find_spec("pip"): + return False + return True + + +def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: + """ + Check that we have ensurepip. + + Raise a fatal exception with a helpful hint if it isn't available. + """ + if not find_spec("ensurepip"): + msg = ( + "Python's ensurepip module is not found.\n" + "It's normally part of the Python standard library, " + "maybe your distribution packages it separately?\n" + "(Debian puts ensurepip in its python3-venv package.)\n" + ) + if suggest_remedy: + msg += ( + "Either install ensurepip, or alleviate the need for it in the" + " first place by installing pip and setuptools for " + f"'{sys.executable}'.\n" + ) + raise Ouch(prefix + msg) + + # ensurepip uses pyexpat, which can also go missing on us: + if not find_spec("pyexpat"): + msg = ( + "Python's pyexpat module is not found.\n" + "It's normally part of the Python standard library, " + "maybe your distribution packages it separately?\n" + "(NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)\n" + ) + if suggest_remedy: + msg += ( + "Either install pyexpat, or alleviate the need for it in the " + "first place by installing pip and setuptools for " + f"'{sys.executable}'.\n" + ) + raise Ouch(prefix + msg) + + +def make_venv( # pylint: disable=too-many-arguments + env_dir: Union[str, Path], + system_site_packages: bool = False, + clear: bool = True, + symlinks: Optional[bool] = None, + with_pip: bool = True, +) -> None: + """ + Create a venv using `QemuEnvBuilder`. + + This is analogous to the `venv.create` module-level convenience + function that is part of the Python stdblib, except it uses + `QemuEnvBuilder` instead. + + :param env_dir: The directory to create/install to. + :param system_site_packages: + Allow inheriting packages from the system installation. + :param clear: When True, fully remove any prior venv and files. + :param symlinks: + Whether to use symlinks to the target interpreter or not. If + left unspecified, it will use symlinks except on Windows to + match behavior with the "venv" CLI tool. + :param with_pip: + Whether to install "pip" binaries or not. + """ + logger.debug( + "%s: make_venv(env_dir=%s, system_site_packages=%s, " + "clear=%s, symlinks=%s, with_pip=%s)", + __file__, + str(env_dir), + system_site_packages, + clear, + symlinks, + with_pip, + ) + + if symlinks is None: + # Default behavior of standard venv CLI + symlinks = os.name != "nt" + + builder = QemuEnvBuilder( + system_site_packages=system_site_packages, + clear=clear, + symlinks=symlinks, + with_pip=with_pip, + ) + + style = "non-isolated" if builder.system_site_packages else "isolated" + nested = "" + if builder.use_parent_packages: + nested = f"(with packages from '{builder.get_parent_libpath()}') " + print( + f"mkvenv: Creating {style} virtual environment" + f" {nested}at '{str(env_dir)}'", + file=sys.stderr, + ) + + try: + logger.debug("Invoking builder.create()") + try: + builder.create(str(env_dir)) + except SystemExit as exc: + # Some versions of the venv module raise SystemExit; *nasty*! + # We want the exception that prompted it. It might be a subprocess + # error that has output we *really* want to see. + logger.debug("Intercepted SystemExit from EnvBuilder.create()") + raise exc.__cause__ or exc.__context__ or exc + logger.debug("builder.create() finished") + except subprocess.CalledProcessError as exc: + logger.error("mkvenv subprocess failed:") + logger.error("cmd: %s", exc.cmd) + logger.error("returncode: %d", exc.returncode) + + def _stringify(data: Union[str, bytes]) -> str: + if isinstance(data, bytes): + return data.decode() + return data + + lines = [] + if exc.stdout: + lines.append("========== stdout ==========") + lines.append(_stringify(exc.stdout)) + lines.append("============================") + if exc.stderr: + lines.append("========== stderr ==========") + lines.append(_stringify(exc.stderr)) + lines.append("============================") + if lines: + logger.error(os.linesep.join(lines)) + + raise Ouch("VENV creation subprocess failed.") from exc + + # print the python executable to stdout for configure. + print(builder.get_value("env_exe")) + + +def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: + # pylint: disable=import-outside-toplevel + # pylint: disable=no-name-in-module + # pylint: disable=import-error + try: + # First preference: Python 3.8+ stdlib + from importlib.metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + except ImportError as exc: + logger.debug("%s", str(exc)) + # Second preference: Commonly available PyPI backport + from importlib_metadata import ( # type: ignore + PackageNotFoundError, + distribution, + ) + + def _generator() -> Iterator[str]: + for package in packages: + try: + entry_points = distribution(package).entry_points + except PackageNotFoundError: + continue + + # The EntryPoints type is only available in 3.10+, + # treat this as a vanilla list and filter it ourselves. + entry_points = filter( + lambda ep: ep.group == "console_scripts", entry_points + ) + + for entry_point in entry_points: + yield f"{entry_point.name} = {entry_point.value}" + + return _generator() + + +def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]: + # pylint: disable=import-outside-toplevel + # Bundled with setuptools; has a good chance of being available. + import pkg_resources + + def _generator() -> Iterator[str]: + for package in packages: + try: + eps = pkg_resources.get_entry_map(package, "console_scripts") + except pkg_resources.DistributionNotFound: + continue + + for entry_point in eps.values(): + yield str(entry_point) + + return _generator() + + +def generate_console_scripts( + packages: Sequence[str], + python_path: Optional[str] = None, + bin_path: Optional[str] = None, +) -> None: + """ + Generate script shims for console_script entry points in @packages. + """ + if python_path is None: + python_path = sys.executable + if bin_path is None: + bin_path = sysconfig.get_path("scripts") + assert bin_path is not None + + logger.debug( + "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", + packages, + python_path, + bin_path, + ) + + if not packages: + return + + def _get_entry_points() -> Iterator[str]: + """Python 3.7 compatibility shim for iterating entry points.""" + # Python 3.8+, or Python 3.7 with importlib_metadata installed. + try: + return _gen_importlib(packages) + except ImportError as exc: + logger.debug("%s", str(exc)) + + # Python 3.7 with setuptools installed. + try: + return _gen_pkg_resources(packages) + except ImportError as exc: + logger.debug("%s", str(exc)) + raise Ouch( + "Neither importlib.metadata nor pkg_resources found, " + "can't generate console script shims.\n" + "Use Python 3.8+, or install importlib-metadata or setuptools." + ) from exc + + maker = distlib.scripts.ScriptMaker(None, bin_path) + maker.variants = {""} + maker.clobber = False + + for entry_point in _get_entry_points(): + for filename in maker.make(entry_point): + logger.debug("wrote console_script '%s'", filename) + + +def checkpip() -> bool: + """ + Debian10 has a pip that's broken when used inside of a virtual environment. + + We try to detect and correct that case here. + """ + try: + # pylint: disable=import-outside-toplevel,unused-import,import-error + # pylint: disable=redefined-outer-name + import pip._internal # type: ignore # noqa: F401 + + logger.debug("pip appears to be working correctly.") + return False + except ModuleNotFoundError as exc: + if exc.name == "pip._internal": + # Uh, fair enough. They did say "internal". + # Let's just assume it's fine. + return False + logger.warning("pip appears to be malfunctioning: %s", str(exc)) + + check_ensurepip("pip appears to be non-functional, and ") + + logger.debug("Attempting to repair pip ...") + subprocess.run( + (sys.executable, "-m", "ensurepip"), + stdout=subprocess.DEVNULL, + check=True, + ) + logger.debug("Pip is now (hopefully) repaired!") + return True + + +def pkgname_from_depspec(dep_spec: str) -> str: + """ + Parse package name out of a PEP-508 depspec. + + See https://peps.python.org/pep-0508/#names + """ + match = re.match( + r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE + ) + if not match: + raise ValueError( + f"dep_spec '{dep_spec}'" + " does not appear to contain a valid package name" + ) + return match.group(0) + + +def diagnose( + dep_spec: str, + online: bool, + wheels_dir: Optional[Union[str, Path]], + prog: Optional[str], +) -> Tuple[str, bool]: + """ + Offer a summary to the user as to why a package failed to be installed. + + :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' + :param online: Did we allow PyPI access? + :param prog: + Optionally, a shell program name that can be used as a + bellwether to detect if this program is installed elsewhere on + the system. This is used to offer advice when a program is + detected for a different python version. + :param wheels_dir: + Optionally, a directory that was searched for vendored packages. + """ + # pylint: disable=too-many-branches + + # Some errors are not particularly serious + bad = False + + pkg_name = pkgname_from_depspec(dep_spec) + pkg_version = None + + has_importlib = False + try: + # Python 3.8+ stdlib + # pylint: disable=import-outside-toplevel + # pylint: disable=no-name-in-module + # pylint: disable=import-error + from importlib.metadata import ( # type: ignore + PackageNotFoundError, + version, + ) + + has_importlib = True + try: + pkg_version = version(pkg_name) + except PackageNotFoundError: + pass + except ModuleNotFoundError: + pass + + lines = [] + + if pkg_version: + lines.append( + f"Python package '{pkg_name}' version '{pkg_version}' was found," + " but isn't suitable." + ) + elif has_importlib: + lines.append( + f"Python package '{pkg_name}' was not found nor installed." + ) + else: + lines.append( + f"Python package '{pkg_name}' is either not found or" + " not a suitable version." + ) + + if wheels_dir: + lines.append( + "No suitable version found in, or failed to install from" + f" '{wheels_dir}'." + ) + bad = True + + if online: + lines.append("A suitable version could not be obtained from PyPI.") + bad = True + else: + lines.append( + "mkvenv was configured to operate offline and did not check PyPI." + ) + + if prog and not pkg_version: + which = shutil.which(prog) + if which: + if sys.base_prefix in site.PREFIXES: + pypath = Path(sys.executable).resolve() + lines.append( + f"'{prog}' was detected on your system at '{which}', " + f"but the Python package '{pkg_name}' was not found by " + f"this Python interpreter ('{pypath}'). " + f"Typically this means that '{prog}' has been installed " + "against a different Python interpreter on your system." + ) + else: + lines.append( + f"'{prog}' was detected on your system at '{which}', " + "but the build is using an isolated virtual environment." + ) + bad = True + + lines = [f" • {line}" for line in lines] + if bad: + lines.insert(0, f"Could not provide build dependency '{dep_spec}':") + else: + lines.insert(0, f"'{dep_spec}' not found:") + return os.linesep.join(lines), bad + + +def pip_install( + args: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, +) -> None: + """ + Use pip to install a package or package(s) as specified in @args. + """ + loud = bool( + os.environ.get("DEBUG") + or os.environ.get("GITLAB_CI") + or os.environ.get("V") + ) + + full_args = [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "-v" if loud else "-q", + ] + if not online: + full_args += ["--no-index"] + if wheels_dir: + full_args += ["--find-links", f"file://{str(wheels_dir)}"] + full_args += list(args) + subprocess.run( + full_args, + check=True, + ) + + +def _do_ensure( + dep_specs: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, +) -> None: + """ + Use pip to ensure we have the package specified by @dep_specs. + + If the package is already installed, do nothing. If online and + wheels_dir are both provided, prefer packages found in wheels_dir + first before connecting to PyPI. + + :param dep_specs: + PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. + :param online: If True, fall back to PyPI. + :param wheels_dir: If specified, search this path for packages. + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=UserWarning, module="distlib" + ) + dist_path = distlib.database.DistributionPath(include_egg=True) + absent = [] + present = [] + for spec in dep_specs: + matcher = distlib.version.LegacyMatcher(spec) + dist = dist_path.get_distribution(matcher.name) + if dist is None or not matcher.match(dist.version): + absent.append(spec) + else: + logger.info("found %s", dist) + present.append(matcher.name) + + if present: + generate_console_scripts(present) + + if absent: + # Some packages are missing or aren't a suitable version, + # install a suitable (possibly vendored) package. + print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) + pip_install(args=absent, online=online, wheels_dir=wheels_dir) + + +def ensure( + dep_specs: Sequence[str], + online: bool = False, + wheels_dir: Optional[Union[str, Path]] = None, + prog: Optional[str] = None, +) -> None: + """ + Use pip to ensure we have the package specified by @dep_specs. + + If the package is already installed, do nothing. If online and + wheels_dir are both provided, prefer packages found in wheels_dir + first before connecting to PyPI. + + :param dep_specs: + PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. + :param online: If True, fall back to PyPI. + :param wheels_dir: If specified, search this path for packages. + :param prog: + If specified, use this program name for error diagnostics that will + be presented to the user. e.g., 'sphinx-build' can be used as a + bellwether for the presence of 'sphinx'. + """ + print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr) + + if not HAVE_DISTLIB: + raise Ouch("a usable distlib could not be found, please install it") + + try: + _do_ensure(dep_specs, online, wheels_dir) + except subprocess.CalledProcessError as exc: + # Well, that's not good. + msg, bad = diagnose(dep_specs[0], online, wheels_dir, prog) + if bad: + raise Ouch(msg) from exc + raise SystemExit(f"\n{msg}\n\n") from exc + + +def post_venv_setup() -> None: + """ + This is intended to be run *inside the venv* after it is created. + """ + logger.debug("post_venv_setup()") + # Test for a broken pip (Debian 10 or derivative?) and fix it if needed + if not checkpip(): + # Finally, generate a 'pip' script so the venv is usable in a normal + # way from the CLI. This only happens when we inherited pip from a + # parent/system-site and haven't run ensurepip in some way. + generate_console_scripts(["pip"]) + + +def _add_create_subcommand(subparsers: Any) -> None: + subparser = subparsers.add_parser("create", help="create a venv") + subparser.add_argument( + "target", + type=str, + action="store", + help="Target directory to install virtual environment into.", + ) + + +def _add_post_init_subcommand(subparsers: Any) -> None: + subparsers.add_parser("post_init", help="post-venv initialization") + + +def _add_ensure_subcommand(subparsers: Any) -> None: + subparser = subparsers.add_parser( + "ensure", help="Ensure that the specified package is installed." + ) + subparser.add_argument( + "--online", + action="store_true", + help="Install packages from PyPI, if necessary.", + ) + subparser.add_argument( + "--dir", + type=str, + action="store", + help="Path to vendored packages where we may install from.", + ) + subparser.add_argument( + "--diagnose", + type=str, + action="store", + help=( + "Name of a shell utility to use for " + "diagnostics if this command fails." + ), + ) + subparser.add_argument( + "dep_specs", + type=str, + action="store", + help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", + nargs="+", + ) + + +def main() -> int: + """CLI interface to make_qemu_venv. See module docstring.""" + if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): + # You're welcome. + logging.basicConfig(level=logging.DEBUG) + else: + if os.environ.get("V"): + logging.basicConfig(level=logging.INFO) + + # These are incredibly noisy even for V=1 + logging.getLogger("distlib.metadata").addFilter(lambda record: False) + logging.getLogger("distlib.database").addFilter(lambda record: False) + + parser = argparse.ArgumentParser( + prog="mkvenv", + description="QEMU pyvenv bootstrapping utility", + ) + subparsers = parser.add_subparsers( + title="Commands", + dest="command", + required=True, + metavar="command", + help="Description", + ) + + _add_create_subcommand(subparsers) + _add_post_init_subcommand(subparsers) + _add_ensure_subcommand(subparsers) + + args = parser.parse_args() + try: + if args.command == "create": + make_venv( + args.target, + system_site_packages=True, + clear=True, + ) + if args.command == "post_init": + post_venv_setup() + if args.command == "ensure": + ensure( + dep_specs=args.dep_specs, + online=args.online, + wheels_dir=args.dir, + prog=args.diagnose, + ) + logger.debug("mkvenv.py %s: exiting", args.command) + except Ouch as exc: + print("\n*** Ouch! ***\n", file=sys.stderr) + print(str(exc), "\n\n", file=sys.stderr) + return 1 + except SystemExit: + raise + except: # pylint: disable=bare-except + logger.exception("mkvenv did not complete successfully:") + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/scripts/vendor.py b/python/scripts/vendor.py new file mode 100755 index 0000000..34486a5 --- /dev/null +++ b/python/scripts/vendor.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +""" +vendor - QEMU python vendoring utility + +usage: vendor [-h] + +QEMU python vendoring utility + +options: + -h, --help show this help message and exit +""" + +# Copyright (C) 2023 Red Hat, Inc. +# +# Authors: +# John Snow <jsnow@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. + +import argparse +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +def main() -> int: + """Run the vendoring utility. See module-level docstring.""" + loud = False + if os.environ.get("DEBUG") or os.environ.get("V"): + loud = True + + # No options or anything for now, but I guess + # you'll figure that out when you run --help. + parser = argparse.ArgumentParser( + prog="vendor", + description="QEMU python vendoring utility", + ) + parser.parse_args() + + packages = { + "meson==0.63.3": + "d677b809c4895dcbaac9bf6c43703fcb3609a4b24c6057c78f828590049cf43a", + } + + vendor_dir = Path(__file__, "..", "..", "wheels").resolve() + + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as file: + for dep_spec, checksum in packages.items(): + file.write(f"{dep_spec} --hash=sha256:{checksum}") + file.flush() + + cli_args = [ + "pip", + "download", + "--dest", + str(vendor_dir), + "--require-hashes", + "-r", + file.name, + ] + if loud: + cli_args.append("-v") + + print(" ".join(cli_args)) + subprocess.run(cli_args, check=True) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/setup.cfg b/python/setup.cfg index 9e923d9..5abb7d3 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -14,7 +14,6 @@ classifiers = Natural Language :: English Operating System :: OS Independent Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -23,7 +22,7 @@ classifiers = Typing :: Typed [options] -python_requires = >= 3.6 +python_requires = >= 3.7 packages = qemu.qmp qemu.machine @@ -36,11 +35,12 @@ packages = # Remember to update tests/minreqs.txt if changing anything below: devel = avocado-framework >= 90.0 - flake8 >= 3.6.0 + distlib >= 0.3.6 + flake8 >= 5.0.4 fusepy >= 2.0.4 isort >= 5.1.2 mypy >= 0.780 - pylint >= 2.8.0 + pylint >= 2.17.3 tox >= 3.18.0 urwid >= 2.1.2 urwid-readline >= 0.13 @@ -76,7 +76,7 @@ exclude = __pycache__, [mypy] strict = True -python_version = 3.6 +python_version = 3.7 warn_unused_configs = True namespace_packages = True warn_unused_ignores = False @@ -103,6 +103,39 @@ ignore_missing_imports = True [mypy-pygments] ignore_missing_imports = True +[mypy-importlib.metadata] +ignore_missing_imports = True + +[mypy-importlib_metadata] +ignore_missing_imports = True + +[mypy-pkg_resources] +ignore_missing_imports = True + +[mypy-distlib] +ignore_missing_imports = True + +[mypy-distlib.database] +ignore_missing_imports = True + +[mypy-distlib.scripts] +ignore_missing_imports = True + +[mypy-distlib.version] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.database] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.scripts] +ignore_missing_imports = True + +[mypy-pip._vendor.distlib.version] +ignore_missing_imports = True + [pylint.messages control] # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this @@ -132,6 +165,7 @@ good-names=i, fd, # fd = os.open(...) c, # for c in string: ... T, # for TypeVars. See pylint#3401 + SocketAddrT, # Not sure why this is invalid. [pylint.similarities] # Ignore imports when computing similarities. @@ -158,7 +192,7 @@ multi_line_output=3 # of python available on your system to run this test. [tox:tox] -envlist = py36, py37, py38, py39, py310, py311 +envlist = py37, py38, py39, py310, py311 skip_missing_interpreters = true [testenv] diff --git a/python/tests/flake8.sh b/python/tests/flake8.sh index 1cd7d40..e013699 100755 --- a/python/tests/flake8.sh +++ b/python/tests/flake8.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m flake8 qemu/ +python3 -m flake8 scripts/ diff --git a/python/tests/isort.sh b/python/tests/isort.sh index 4480405..66c2f7d 100755 --- a/python/tests/isort.sh +++ b/python/tests/isort.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m isort -c qemu/ +python3 -m isort -c scripts/ diff --git a/python/tests/minreqs.txt b/python/tests/minreqs.txt index dfb8abb..1ce72cef 100644 --- a/python/tests/minreqs.txt +++ b/python/tests/minreqs.txt @@ -1,5 +1,5 @@ # This file lists the ***oldest possible dependencies*** needed to run -# "make check" successfully under ***Python 3.6***. It is used primarily +# "make check" successfully under ***Python 3.7***. It is used primarily # by GitLab CI to ensure that our stated minimum versions in setup.cfg # are truthful and regularly validated. # @@ -16,6 +16,9 @@ urwid==2.1.2 urwid-readline==0.13 Pygments==2.9.0 +# Dependencies for mkvenv +distlib==0.3.6 + # Dependencies for FUSE support for qom-fuse fusepy==2.0.4 @@ -23,23 +26,23 @@ fusepy==2.0.4 avocado-framework==90.0 # Linters -flake8==3.6.0 +flake8==5.0.4 isort==5.1.2 mypy==0.780 -pylint==2.8.0 +pylint==2.17.3 # Transitive flake8 dependencies -mccabe==0.6.0 -pycodestyle==2.4.0 -pyflakes==2.0.0 +mccabe==0.7.0 +pycodestyle==2.9.1 +pyflakes==2.5.0 # Transitive mypy dependencies mypy-extensions==0.4.3 typed-ast==1.4.0 -typing-extensions==3.7.4 +typing-extensions==4.5.0 # Transitive pylint dependencies -astroid==2.5.4 +astroid==2.15.4 lazy-object-proxy==1.4.0 toml==0.10.0 wrapt==1.12.1 diff --git a/python/tests/mypy.sh b/python/tests/mypy.sh index 5f980f5..a33a3f5 100755 --- a/python/tests/mypy.sh +++ b/python/tests/mypy.sh @@ -1,2 +1,3 @@ #!/bin/sh -e python3 -m mypy -p qemu +python3 -m mypy scripts/ diff --git a/python/tests/pylint.sh b/python/tests/pylint.sh index 03d6470..2b68da9 100755 --- a/python/tests/pylint.sh +++ b/python/tests/pylint.sh @@ -1,3 +1,4 @@ #!/bin/sh -e # See commit message for environment variable explainer. SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pylint qemu/ +SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pylint scripts/ diff --git a/python/wheels/meson-0.63.3-py3-none-any.whl b/python/wheels/meson-0.63.3-py3-none-any.whl Binary files differnew file mode 100644 index 0000000..8a191e3 --- /dev/null +++ b/python/wheels/meson-0.63.3-py3-none-any.whl diff --git a/qga/meson.build b/qga/meson.build index ad17dc7..622b5f9 100644 --- a/qga/meson.build +++ b/qga/meson.build @@ -22,7 +22,7 @@ have_qga_vss = get_option('qga_vss') \ Then run configure with: --extra-cxxflags="-isystem /path/to/vss/inc/win2003"''') \ .require(midl.found() or widl.found(), error_message: 'VSS support requires midl or widl') \ - .require(not enable_static, + .require(not get_option('prefer_static'), error_message: 'VSS support requires dynamic linking with GLib') \ .allowed() diff --git a/scripts/ci/org.centos/stream/8/x86_64/test-avocado b/scripts/ci/org.centos/stream/8/x86_64/test-avocado index d2c0e5f..7bb5b31 100755 --- a/scripts/ci/org.centos/stream/8/x86_64/test-avocado +++ b/scripts/ci/org.centos/stream/8/x86_64/test-avocado @@ -4,7 +4,7 @@ # KVM and x86_64, or tests that are generic enough to be valid for all # targets. Such a test list can be generated with: # -# ./tests/venv/bin/avocado list --filter-by-tags-include-empty \ +# ./pyvenv/bin/avocado list --filter-by-tags-include-empty \ # --filter-by-tags-include-empty-key -t accel:kvm,arch:x86_64 \ # tests/avocado/ # @@ -22,7 +22,7 @@ # - tests/avocado/virtio_check_params.py:VirtioMaxSegSettingsCheck.test_machine_types # make get-vm-images -./tests/venv/bin/avocado run \ +./pyvenv/bin/avocado run \ --job-results-dir=tests/results/ \ tests/avocado/boot_linux.py:BootLinuxX8664.test_pc_i440fx_kvm \ tests/avocado/boot_linux.py:BootLinuxX8664.test_pc_q35_kvm \ diff --git a/scripts/coverity-scan/COMPONENTS.md b/scripts/coverity-scan/COMPONENTS.md index 7c48e0f..883da95 100644 --- a/scripts/coverity-scan/COMPONENTS.md +++ b/scripts/coverity-scan/COMPONENTS.md @@ -24,6 +24,9 @@ hppa i386 ~ (/qemu)?((/include)?/hw/i386/.*|/target/i386/.*|/hw/intc/[^/]*apic[^/]*\.c) +loongarch + ~ (/qemu)?((/include)?/hw/(loongarch/.*|.*/loongarch.*)|/target/loongarch/.*) + m68k ~ (/qemu)?((/include)?/hw/m68k/.*|/target/m68k/.*|(/include)?/hw(/.*)?/mcf.*|(/include)?/hw/nubus/.*) @@ -36,11 +39,14 @@ mips nios2 ~ (/qemu)?((/include)?/hw/nios2/.*|/target/nios2/.*) +openrisc + ~ (/qemu)?((/include)?/hw/openrisc/.*|/target/openrisc/.*) + ppc ~ (/qemu)?((/include)?/hw/ppc/.*|/target/ppc/.*|/hw/pci-host/(uninorth.*|dec.*|prep.*|ppc.*)|/hw/misc/macio/.*|(/include)?/hw/.*/(xics|openpic|spapr).*) riscv - ~ (/qemu)?((/include)?/hw/riscv/.*|/target/riscv/.*) + ~ (/qemu)?((/include)?/hw/riscv/.*|/target/riscv/.*|/hw/.*/(riscv_|ibex_|sifive_).*) rx ~ (/qemu)?((/include)?/hw/rx/.*|/target/rx/.*) @@ -54,12 +60,12 @@ sh4 sparc ~ (/qemu)?((/include)?/hw/sparc(64)?.*|/target/sparc/.*|/hw/.*/grlib.*|/hw/display/cg3.c) -tilegx - ~ (/qemu)?(/target/tilegx/.*) - tricore ~ (/qemu)?((/include)?/hw/tricore/.*|/target/tricore/.*) +xtensa + ~ (/qemu)?((/include)?/hw/xtensa/.*|/target/xtensa/.*) + 9pfs ~ (/qemu)?(/hw/9pfs/.*|/fsdev/.*) @@ -73,7 +79,7 @@ char ~ (/qemu)?(/qemu-char\.c|/include/sysemu/char\.h|(/include)?/hw/char/.*) crypto - ~ (/qemu)?((/include)?/crypto/.*|/hw/.*/crypto.*) + ~ (/qemu)?((/include)?/crypto/.*|/hw/.*/.*crypto.*|(/include/sysemu|/backends)/cryptodev.*) disas ~ (/qemu)?((/include)?/disas.*) @@ -100,7 +106,7 @@ net ~ (/qemu)?((/include)?(/hw)?/(net|rdma)/.*) pci - ~ (/qemu)?(/hw/pci.*|/include/hw/pci.*) + ~ (/qemu)?(/include)?/hw/(cxl/|pci).* qemu-ga ~ (/qemu)?(/qga/.*) @@ -108,9 +114,6 @@ qemu-ga scsi ~ (/qemu)?(/scsi/.*|/hw/scsi/.*|/include/hw/scsi/.*) -tcg - ~ (/qemu)?(/accel/tcg/.*|/replay/.*|/(.*/)?softmmu.*) - trace ~ (/qemu)?(/.*trace.*\.[ch]) @@ -126,9 +129,27 @@ user util ~ (/qemu)?(/util/.*|/include/qemu/.*) +vfio + ~ (/qemu)?(/include)?/hw/vfio/.* + +virtio + ~ (/qemu)?(/include)?/hw/virtio/.* + xen ~ (/qemu)?(.*/xen.*) +hvf + ~ (/qemu)?(.*/hvf.*) + +kvm + ~ (/qemu)?(.*/kvm.*) + +tcg + ~ (/qemu)?(/accel/tcg|/replay|/tcg)/.* + +sysemu + ~ (/qemu)?(/softmmu/.*|/accel/.*) + (headers) ~ (/qemu)?(/include/.*) @@ -137,9 +158,3 @@ testlibs tests ~ (/qemu)?(/tests/.*) - -loongarch - ~ (/qemu)?((/include)?/hw/(loongarch/.*|.*/loongarch.*)|/target/loongarch/.*) - -riscv - ~ (/qemu)?((/include)?/hw/riscv/.*|/target/riscv/.*|/hw/.*/(riscv_|ibex_|sifive_).*) diff --git a/scripts/device-crash-test b/scripts/device-crash-test index b74d887..353aa57 100755 --- a/scripts/device-crash-test +++ b/scripts/device-crash-test @@ -43,7 +43,7 @@ except ModuleNotFoundError as exc: print(f"Module '{exc.name}' not found.") print(" Try 'make check-venv' from your build directory,") print(" and then one way to run this script is like so:") - print(f' > $builddir/tests/venv/bin/python3 "{path}"') + print(f' > $builddir/pyvenv/bin/python3 "{path}"') sys.exit(1) logger = logging.getLogger('device-crash-test') diff --git a/scripts/meson-buildoptions.py b/scripts/meson-buildoptions.py index a04dcc7..8d2e526 100755 --- a/scripts/meson-buildoptions.py +++ b/scripts/meson-buildoptions.py @@ -35,6 +35,8 @@ SKIP_OPTIONS = { OPTION_NAMES = { "b_coverage": "gcov", "b_lto": "lto", + "coroutine_backend": "with-coroutine", + "debug": "debug-info", "malloc": "enable-malloc", "pkgversion": "with-pkgversion", "qemu_firmwarepath": "firmwarepath", @@ -46,6 +48,7 @@ BUILTIN_OPTIONS = { "b_coverage", "b_lto", "datadir", + "debug", "includedir", "libdir", "libexecdir", diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 52fb079..53124e1 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -1,8 +1,8 @@ # This file is generated by meson-buildoptions.py, do not edit! meson_options_help() { - printf "%s\n" ' --audio-drv-list=CHOICES Set audio driver list [default] (choices: al' - printf "%s\n" ' sa/coreaudio/default/dsound/jack/oss/pa/' - printf "%s\n" ' pipewire/sdl/sndio)' + printf "%s\n" ' --audio-drv-list=CHOICES Set audio driver list [default] (choices: alsa/co' + printf "%s\n" ' reaudio/default/dsound/jack/oss/pa/pipewire/sdl/s' + printf "%s\n" ' ndio)' printf "%s\n" ' --block-drv-ro-whitelist=VALUE' printf "%s\n" ' set block driver read-only whitelist (by default' printf "%s\n" ' affects only QEMU, not tools like qemu-img)' @@ -11,6 +11,7 @@ meson_options_help() { printf "%s\n" ' affects only QEMU, not tools like qemu-img)' printf "%s\n" ' --datadir=VALUE Data file directory [share]' printf "%s\n" ' --disable-coroutine-pool coroutine freelist (better performance)' + printf "%s\n" ' --disable-debug-info Enable debug symbols and other information' printf "%s\n" ' --disable-hexagon-idef-parser' printf "%s\n" ' use idef-parser to automatically generate TCG' printf "%s\n" ' code for the Hexagon frontend' @@ -41,11 +42,15 @@ meson_options_help() { printf "%s\n" ' --enable-profiler profiler support' printf "%s\n" ' --enable-rng-none dummy RNG, avoid using /dev/(u)random and' printf "%s\n" ' getrandom()' + printf "%s\n" ' --enable-safe-stack SafeStack Stack Smash Protection (requires' + printf "%s\n" ' clang/llvm and coroutine backend ucontext)' + printf "%s\n" ' --enable-sanitizers enable default sanitizers' printf "%s\n" ' --enable-strip Strip targets on install' printf "%s\n" ' --enable-tcg-interpreter TCG with bytecode interpreter (slow)' printf "%s\n" ' --enable-trace-backends=CHOICES' printf "%s\n" ' Set available tracing backends [log] (choices:' printf "%s\n" ' dtrace/ftrace/log/nop/simple/syslog/ust)' + printf "%s\n" ' --enable-tsan enable thread sanitizer' printf "%s\n" ' --firmwarepath=VALUES search PATH for firmware files [share/qemu-' printf "%s\n" ' firmware]' printf "%s\n" ' --iasl=VALUE Path to ACPI disassembler' @@ -57,11 +62,11 @@ meson_options_help() { printf "%s\n" ' --localedir=VALUE Locale data directory [share/locale]' printf "%s\n" ' --localstatedir=VALUE Localstate data directory [/var/local]' printf "%s\n" ' --mandir=VALUE Manual page directory [share/man]' - printf "%s\n" ' --sphinx-build=VALUE Use specified sphinx-build for building document' - printf "%s\n" ' [sphinx-build]' printf "%s\n" ' --sysconfdir=VALUE Sysconf data directory [etc]' printf "%s\n" ' --tls-priority=VALUE Default TLS protocol/cipher priority string' printf "%s\n" ' [NORMAL]' + printf "%s\n" ' --with-coroutine=CHOICE coroutine backend to use (choices:' + printf "%s\n" ' auto/sigaltstack/ucontext/windows)' printf "%s\n" ' --with-pkgversion=VALUE use specified string as sub-version of the' printf "%s\n" ' package' printf "%s\n" ' --with-trace-file=VALUE Trace file prefix for simple backend [trace]' @@ -129,6 +134,7 @@ meson_options_help() { printf "%s\n" ' lzo lzo compression support' printf "%s\n" ' malloc-trim enable libc malloc_trim() for memory optimization' printf "%s\n" ' membarrier membarrier system call (for Linux 4.14+ or Windows' + printf "%s\n" ' modules modules support (non Windows)' printf "%s\n" ' mpath Multipath persistent reservation passthrough' printf "%s\n" ' multiprocess Out of process device emulation support' printf "%s\n" ' netmap netmap network backend support' @@ -160,6 +166,7 @@ meson_options_help() { printf "%s\n" ' sparse sparse checker' printf "%s\n" ' spice Spice server support' printf "%s\n" ' spice-protocol Spice protocol support' + printf "%s\n" ' stack-protector compiler-provided stack protection' printf "%s\n" ' tcg TCG support' printf "%s\n" ' tools build support utilities that come with QEMU' printf "%s\n" ' tpm TPM support' @@ -247,6 +254,7 @@ _meson_option_parse() { --disable-cocoa) printf "%s" -Dcocoa=disabled ;; --enable-coreaudio) printf "%s" -Dcoreaudio=enabled ;; --disable-coreaudio) printf "%s" -Dcoreaudio=disabled ;; + --with-coroutine=*) quote_sh "-Dcoroutine_backend=$2" ;; --enable-coroutine-pool) printf "%s" -Dcoroutine_pool=true ;; --disable-coroutine-pool) printf "%s" -Dcoroutine_pool=false ;; --enable-crypto-afalg) printf "%s" -Dcrypto_afalg=enabled ;; @@ -258,6 +266,8 @@ _meson_option_parse() { --datadir=*) quote_sh "-Ddatadir=$2" ;; --enable-dbus-display) printf "%s" -Ddbus_display=enabled ;; --disable-dbus-display) printf "%s" -Ddbus_display=disabled ;; + --enable-debug-info) printf "%s" -Ddebug=true ;; + --disable-debug-info) printf "%s" -Ddebug=false ;; --enable-debug-graph-lock) printf "%s" -Ddebug_graph_lock=true ;; --disable-debug-graph-lock) printf "%s" -Ddebug_graph_lock=false ;; --enable-debug-mutex) printf "%s" -Ddebug_mutex=true ;; @@ -361,6 +371,8 @@ _meson_option_parse() { --disable-membarrier) printf "%s" -Dmembarrier=disabled ;; --enable-module-upgrades) printf "%s" -Dmodule_upgrades=true ;; --disable-module-upgrades) printf "%s" -Dmodule_upgrades=false ;; + --enable-modules) printf "%s" -Dmodules=enabled ;; + --disable-modules) printf "%s" -Dmodules=disabled ;; --enable-mpath) printf "%s" -Dmpath=enabled ;; --disable-mpath) printf "%s" -Dmpath=disabled ;; --enable-multiprocess) printf "%s" -Dmultiprocess=enabled ;; @@ -407,6 +419,10 @@ _meson_option_parse() { --disable-replication) printf "%s" -Dreplication=disabled ;; --enable-rng-none) printf "%s" -Drng_none=true ;; --disable-rng-none) printf "%s" -Drng_none=false ;; + --enable-safe-stack) printf "%s" -Dsafe_stack=true ;; + --disable-safe-stack) printf "%s" -Dsafe_stack=false ;; + --enable-sanitizers) printf "%s" -Dsanitizers=true ;; + --disable-sanitizers) printf "%s" -Dsanitizers=false ;; --enable-sdl) printf "%s" -Dsdl=enabled ;; --disable-sdl) printf "%s" -Dsdl=disabled ;; --enable-sdl-image) printf "%s" -Dsdl_image=enabled ;; @@ -427,11 +443,12 @@ _meson_option_parse() { --disable-sndio) printf "%s" -Dsndio=disabled ;; --enable-sparse) printf "%s" -Dsparse=enabled ;; --disable-sparse) printf "%s" -Dsparse=disabled ;; - --sphinx-build=*) quote_sh "-Dsphinx_build=$2" ;; --enable-spice) printf "%s" -Dspice=enabled ;; --disable-spice) printf "%s" -Dspice=disabled ;; --enable-spice-protocol) printf "%s" -Dspice_protocol=enabled ;; --disable-spice-protocol) printf "%s" -Dspice_protocol=disabled ;; + --enable-stack-protector) printf "%s" -Dstack_protector=enabled ;; + --disable-stack-protector) printf "%s" -Dstack_protector=disabled ;; --enable-strip) printf "%s" -Dstrip=true ;; --disable-strip) printf "%s" -Dstrip=false ;; --sysconfdir=*) quote_sh "-Dsysconfdir=$2" ;; @@ -446,6 +463,8 @@ _meson_option_parse() { --disable-tpm) printf "%s" -Dtpm=disabled ;; --enable-trace-backends=*) quote_sh "-Dtrace_backends=$2" ;; --with-trace-file=*) quote_sh "-Dtrace_file=$2" ;; + --enable-tsan) printf "%s" -Dtsan=true ;; + --disable-tsan) printf "%s" -Dtsan=false ;; --enable-u2f) printf "%s" -Du2f=enabled ;; --disable-u2f) printf "%s" -Du2f=disabled ;; --enable-usb-redir) printf "%s" -Dusb_redir=enabled ;; diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini index 6625356..3463307 100644 --- a/scripts/qapi/mypy.ini +++ b/scripts/qapi/mypy.ini @@ -1,7 +1,7 @@ [mypy] strict = True disallow_untyped_calls = False -python_version = 3.6 +python_version = 3.7 [mypy-qapi.schema] disallow_untyped_defs = False diff --git a/softmmu/memory.c b/softmmu/memory.c index b7b3386..342c121 100644 --- a/softmmu/memory.c +++ b/softmmu/memory.c @@ -2253,7 +2253,7 @@ void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, * If memory region `mr' is NULL, do global sync. Otherwise, sync * dirty bitmap for the specified memory region. */ -static void memory_region_sync_dirty_bitmap(MemoryRegion *mr) +static void memory_region_sync_dirty_bitmap(MemoryRegion *mr, bool last_stage) { MemoryListener *listener; AddressSpace *as; @@ -2283,7 +2283,7 @@ static void memory_region_sync_dirty_bitmap(MemoryRegion *mr) * is to do a global sync, because we are not capable to * sync in a finer granularity. */ - listener->log_sync_global(listener); + listener->log_sync_global(listener, last_stage); trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 1); } } @@ -2347,7 +2347,7 @@ DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr, { DirtyBitmapSnapshot *snapshot; assert(mr->ram_block); - memory_region_sync_dirty_bitmap(mr); + memory_region_sync_dirty_bitmap(mr, false); snapshot = cpu_physical_memory_snapshot_and_clear_dirty(mr, addr, size, client); memory_global_after_dirty_log_sync(); return snapshot; @@ -2873,9 +2873,9 @@ bool memory_region_present(MemoryRegion *container, hwaddr addr) return mr && mr != container; } -void memory_global_dirty_log_sync(void) +void memory_global_dirty_log_sync(bool last_stage) { - memory_region_sync_dirty_bitmap(NULL); + memory_region_sync_dirty_bitmap(NULL, last_stage); } void memory_global_after_dirty_log_sync(void) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 4187759..a61cd6d 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -863,7 +863,7 @@ FeatureWordInfo feature_word_info[FEATURE_WORDS] = { "tsx-ldtrk", NULL, NULL /* pconfig */, "arch-lbr", NULL, NULL, "amx-bf16", "avx512-fp16", "amx-tile", "amx-int8", "spec-ctrl", "stibp", - NULL, "arch-capabilities", "core-capability", "ssbd", + "flush-l1d", "arch-capabilities", "core-capability", "ssbd", }, .cpuid = { .eax = 7, @@ -1050,7 +1050,7 @@ FeatureWordInfo feature_word_info[FEATURE_WORDS] = { "ssb-no", "mds-no", "pschange-mc-no", "tsx-ctrl", "taa-no", NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, + NULL, "fb-clear", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 8ade71a..7201a71 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -899,6 +899,8 @@ uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, #define CPUID_7_0_EDX_SPEC_CTRL (1U << 26) /* Single Thread Indirect Branch Predictors */ #define CPUID_7_0_EDX_STIBP (1U << 27) +/* Flush L1D cache */ +#define CPUID_7_0_EDX_FLUSH_L1D (1U << 28) /* Arch Capabilities */ #define CPUID_7_0_EDX_ARCH_CAPABILITIES (1U << 29) /* Core Capability */ @@ -1016,6 +1018,7 @@ uint64_t x86_cpu_get_supported_feature_word(FeatureWord w, #define MSR_ARCH_CAP_PSCHANGE_MC_NO (1U << 6) #define MSR_ARCH_CAP_TSX_CTRL_MSR (1U << 7) #define MSR_ARCH_CAP_TAA_NO (1U << 8) +#define MSR_ARCH_CAP_FB_CLEAR (1U << 17) #define MSR_CORE_CAP_SPLIT_LOCK_DETECT (1U << 5) diff --git a/target/i386/ops_sse.h b/target/i386/ops_sse.h index 0bd6bfa..fb63af7 100644 --- a/target/i386/ops_sse.h +++ b/target/i386/ops_sse.h @@ -2497,6 +2497,14 @@ void helper_vpermdq_ymm(Reg *d, Reg *v, Reg *s, uint32_t order) d->Q(1) = r1; d->Q(2) = r2; d->Q(3) = r3; + if (order & 0x8) { + d->Q(0) = 0; + d->Q(1) = 0; + } + if (order & 0x80) { + d->Q(2) = 0; + d->Q(3) = 0; + } } void helper_vpermq_ymm(Reg *d, Reg *s, uint32_t order) diff --git a/target/i386/tcg/decode-new.c.inc b/target/i386/tcg/decode-new.c.inc index 4fdd877..46afd99 100644 --- a/target/i386/tcg/decode-new.c.inc +++ b/target/i386/tcg/decode-new.c.inc @@ -237,7 +237,7 @@ static void decode_group14(DisasContext *s, CPUX86State *env, X86OpEntry *entry, static void decode_0F6F(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { static const X86OpEntry opcodes_0F6F[4] = { - X86_OP_ENTRY3(MOVDQ, P,q, None,None, Q,q, vex1 mmx), /* movq */ + X86_OP_ENTRY3(MOVDQ, P,q, None,None, Q,q, vex5 mmx), /* movq */ X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex1), /* movdqa */ X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex4_unal), /* movdqu */ {}, @@ -274,9 +274,9 @@ static void decode_0F78(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui { static const X86OpEntry opcodes_0F78[4] = { {}, - X86_OP_ENTRY3(EXTRQ_i, V,x, None,None, I,w, cpuid(SSE4A)), + X86_OP_ENTRY3(EXTRQ_i, V,x, None,None, I,w, cpuid(SSE4A)), /* AMD extension */ {}, - X86_OP_ENTRY3(INSERTQ_i, V,x, U,x, I,w, cpuid(SSE4A)), + X86_OP_ENTRY3(INSERTQ_i, V,x, U,x, I,w, cpuid(SSE4A)), /* AMD extension */ }; *entry = *decode_by_prefix(s, opcodes_0F78); } @@ -284,9 +284,9 @@ static void decode_0F78(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static void decode_0F79(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { if (s->prefix & PREFIX_REPNZ) { - entry->gen = gen_INSERTQ_r; + entry->gen = gen_INSERTQ_r; /* AMD extension */ } else if (s->prefix & PREFIX_DATA) { - entry->gen = gen_EXTRQ_r; + entry->gen = gen_EXTRQ_r; /* AMD extension */ } else { entry->gen = NULL; }; @@ -306,7 +306,7 @@ static void decode_0F7E(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static void decode_0F7F(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { static const X86OpEntry opcodes_0F7F[4] = { - X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex1 mmx), /* movq */ + X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex5 mmx), /* movq */ X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex1), /* movdqa */ X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4_unal), /* movdqu */ {}, @@ -639,15 +639,15 @@ static void decode_0F10(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static const X86OpEntry opcodes_0F10_reg[4] = { X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex4_unal), /* MOVUPS */ X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex4_unal), /* MOVUPD */ - X86_OP_ENTRY3(VMOVSS, V,x, H,x, W,x, vex4), - X86_OP_ENTRY3(VMOVLPx, V,x, H,x, W,x, vex4), /* MOVSD */ + X86_OP_ENTRY3(VMOVSS, V,x, H,x, W,x, vex5), + X86_OP_ENTRY3(VMOVLPx, V,x, H,x, W,x, vex5), /* MOVSD */ }; static const X86OpEntry opcodes_0F10_mem[4] = { X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex4_unal), /* MOVUPS */ X86_OP_ENTRY3(MOVDQ, V,x, None,None, W,x, vex4_unal), /* MOVUPD */ - X86_OP_ENTRY3(VMOVSS_ld, V,x, H,x, M,ss, vex4), - X86_OP_ENTRY3(VMOVSD_ld, V,x, H,x, M,sd, vex4), + X86_OP_ENTRY3(VMOVSS_ld, V,x, H,x, M,ss, vex5), + X86_OP_ENTRY3(VMOVSD_ld, V,x, H,x, M,sd, vex5), }; if ((get_modrm(s, env) >> 6) == 3) { @@ -660,17 +660,17 @@ static void decode_0F10(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static void decode_0F11(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { static const X86OpEntry opcodes_0F11_reg[4] = { - X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVPS */ - X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVPD */ - X86_OP_ENTRY3(VMOVSS, W,x, H,x, V,x, vex4), - X86_OP_ENTRY3(VMOVLPx, W,x, H,x, V,q, vex4), /* MOVSD */ + X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVUPS */ + X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVUPD */ + X86_OP_ENTRY3(VMOVSS, W,x, H,x, V,x, vex5), + X86_OP_ENTRY3(VMOVLPx, W,x, H,x, V,q, vex5), /* MOVSD */ }; static const X86OpEntry opcodes_0F11_mem[4] = { - X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVPS */ - X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVPD */ - X86_OP_ENTRY3(VMOVSS_st, M,ss, None,None, V,x, vex4), - X86_OP_ENTRY3(VMOVLPx_st, M,sd, None,None, V,x, vex4), /* MOVSD */ + X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVUPS */ + X86_OP_ENTRY3(MOVDQ, W,x, None,None, V,x, vex4), /* MOVUPD */ + X86_OP_ENTRY3(VMOVSS_st, M,ss, None,None, V,x, vex5), + X86_OP_ENTRY3(VMOVLPx_st, M,sd, None,None, V,x, vex5), /* MOVSD */ }; if ((get_modrm(s, env) >> 6) == 3) { @@ -687,16 +687,16 @@ static void decode_0F12(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui * Use dq for operand for compatibility with gen_MOVSD and * to allow VEX128 only. */ - X86_OP_ENTRY3(VMOVLPx_ld, V,dq, H,dq, M,q, vex4), /* MOVLPS */ - X86_OP_ENTRY3(VMOVLPx_ld, V,dq, H,dq, M,q, vex4), /* MOVLPD */ + X86_OP_ENTRY3(VMOVLPx_ld, V,dq, H,dq, M,q, vex5), /* MOVLPS */ + X86_OP_ENTRY3(VMOVLPx_ld, V,dq, H,dq, M,q, vex5), /* MOVLPD */ X86_OP_ENTRY3(VMOVSLDUP, V,x, None,None, W,x, vex4 cpuid(SSE3)), - X86_OP_ENTRY3(VMOVDDUP, V,x, None,None, WM,q, vex4 cpuid(SSE3)), /* qq if VEX.256 */ + X86_OP_ENTRY3(VMOVDDUP, V,x, None,None, WM,q, vex5 cpuid(SSE3)), /* qq if VEX.256 */ }; static const X86OpEntry opcodes_0F12_reg[4] = { - X86_OP_ENTRY3(VMOVHLPS, V,dq, H,dq, U,dq, vex4), - X86_OP_ENTRY3(VMOVLPx, W,x, H,x, U,q, vex4), /* MOVLPD */ + X86_OP_ENTRY3(VMOVHLPS, V,dq, H,dq, U,dq, vex7), + X86_OP_ENTRY3(VMOVLPx, W,x, H,x, U,q, vex5), /* MOVLPD */ X86_OP_ENTRY3(VMOVSLDUP, V,x, None,None, U,x, vex4 cpuid(SSE3)), - X86_OP_ENTRY3(VMOVDDUP, V,x, None,None, U,x, vex4 cpuid(SSE3)), + X86_OP_ENTRY3(VMOVDDUP, V,x, None,None, U,x, vex5 cpuid(SSE3)), }; if ((get_modrm(s, env) >> 6) == 3) { @@ -716,15 +716,15 @@ static void decode_0F16(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui * Operand 1 technically only reads the low 64 bits, but uses dq so that * it is easier to check for op0 == op1 in an endianness-neutral manner. */ - X86_OP_ENTRY3(VMOVHPx_ld, V,dq, H,dq, M,q, vex4), /* MOVHPS */ - X86_OP_ENTRY3(VMOVHPx_ld, V,dq, H,dq, M,q, vex4), /* MOVHPD */ + X86_OP_ENTRY3(VMOVHPx_ld, V,dq, H,dq, M,q, vex5), /* MOVHPS */ + X86_OP_ENTRY3(VMOVHPx_ld, V,dq, H,dq, M,q, vex5), /* MOVHPD */ X86_OP_ENTRY3(VMOVSHDUP, V,x, None,None, W,x, vex4 cpuid(SSE3)), {}, }; static const X86OpEntry opcodes_0F16_reg[4] = { /* Same as above, operand 1 could be Hq if it wasn't for big-endian. */ - X86_OP_ENTRY3(VMOVLHPS, V,dq, H,dq, U,q, vex4), - X86_OP_ENTRY3(VMOVHPx, V,x, H,x, U,x, vex4), /* MOVHPD */ + X86_OP_ENTRY3(VMOVLHPS, V,dq, H,dq, U,q, vex7), + X86_OP_ENTRY3(VMOVHPx, V,x, H,x, U,x, vex5), /* MOVHPD */ X86_OP_ENTRY3(VMOVSHDUP, V,x, None,None, U,x, vex4 cpuid(SSE3)), {}, }; @@ -750,8 +750,9 @@ static void decode_0F2A(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static void decode_0F2B(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { static const X86OpEntry opcodes_0F2B[4] = { - X86_OP_ENTRY3(MOVDQ, M,x, None,None, V,x, vex4), /* MOVNTPS */ - X86_OP_ENTRY3(MOVDQ, M,x, None,None, V,x, vex4), /* MOVNTPD */ + X86_OP_ENTRY3(MOVDQ, M,x, None,None, V,x, vex1), /* MOVNTPS */ + X86_OP_ENTRY3(MOVDQ, M,x, None,None, V,x, vex1), /* MOVNTPD */ + /* AMD extensions */ X86_OP_ENTRY3(VMOVSS_st, M,ss, None,None, V,x, vex4 cpuid(SSE4A)), /* MOVNTSS */ X86_OP_ENTRY3(VMOVLPx_st, M,sd, None,None, V,x, vex4 cpuid(SSE4A)), /* MOVNTSD */ }; @@ -783,6 +784,17 @@ static void decode_0F2D(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui *entry = *decode_by_prefix(s, opcodes_0F2D); } +static void decode_VxCOMISx(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) +{ + /* + * VUCOMISx and VCOMISx are different and use no-prefix and 0x66 for SS and SD + * respectively. Scalar values usually are associated with 0xF2 and 0xF3, for + * which X86_VEX_REPScalar exists, but here it has to be decoded by hand. + */ + entry->s1 = entry->s2 = (s->prefix & PREFIX_DATA ? X86_SIZE_sd : X86_SIZE_ss); + entry->gen = (*b == 0x2E ? gen_VUCOMI : gen_VCOMI); +} + static void decode_sse_unary(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b) { if (!(s->prefix & (PREFIX_REPZ | PREFIX_REPNZ))) { @@ -813,7 +825,7 @@ static void decode_0FE6(DisasContext *s, CPUX86State *env, X86OpEntry *entry, ui static const X86OpEntry opcodes_0FE6[4] = { {}, X86_OP_ENTRY2(VCVTTPD2DQ, V,x, W,x, vex2), - X86_OP_ENTRY2(VCVTDQ2PD, V,x, W,x, vex2), + X86_OP_ENTRY2(VCVTDQ2PD, V,x, W,x, vex5), X86_OP_ENTRY2(VCVTPD2DQ, V,x, W,x, vex2), }; *entry = *decode_by_prefix(s, opcodes_0FE6); @@ -831,17 +843,17 @@ static const X86OpEntry opcodes_0F[256] = { [0x10] = X86_OP_GROUP0(0F10), [0x11] = X86_OP_GROUP0(0F11), [0x12] = X86_OP_GROUP0(0F12), - [0x13] = X86_OP_ENTRY3(VMOVLPx_st, M,q, None,None, V,q, vex4 p_00_66), + [0x13] = X86_OP_ENTRY3(VMOVLPx_st, M,q, None,None, V,q, vex5 p_00_66), [0x14] = X86_OP_ENTRY3(VUNPCKLPx, V,x, H,x, W,x, vex4 p_00_66), [0x15] = X86_OP_ENTRY3(VUNPCKHPx, V,x, H,x, W,x, vex4 p_00_66), [0x16] = X86_OP_GROUP0(0F16), /* Incorrectly listed as Mq,Vq in the manual */ - [0x17] = X86_OP_ENTRY3(VMOVHPx_st, M,q, None,None, V,dq, vex4 p_00_66), + [0x17] = X86_OP_ENTRY3(VMOVHPx_st, M,q, None,None, V,dq, vex5 p_00_66), [0x50] = X86_OP_ENTRY3(MOVMSK, G,y, None,None, U,x, vex7 p_00_66), - [0x51] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), - [0x52] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex4_rep5 p_00_f3), - [0x53] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex4_rep5 p_00_f3), + [0x51] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), /* sqrtps */ + [0x52] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex4_rep5 p_00_f3), /* rsqrtps */ + [0x53] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex4_rep5 p_00_f3), /* rcpps */ [0x54] = X86_OP_ENTRY3(PAND, V,x, H,x, W,x, vex4 p_00_66), /* vand */ [0x55] = X86_OP_ENTRY3(PANDN, V,x, H,x, W,x, vex4 p_00_66), /* vandn */ [0x56] = X86_OP_ENTRY3(POR, V,x, H,x, W,x, vex4 p_00_66), /* vor */ @@ -871,15 +883,15 @@ static const X86OpEntry opcodes_0F[256] = { [0x2B] = X86_OP_GROUP0(0F2B), [0x2C] = X86_OP_GROUP0(0F2C), [0x2D] = X86_OP_GROUP0(0F2D), - [0x2E] = X86_OP_ENTRY3(VUCOMI, None,None, V,x, W,x, vex4 p_00_66), - [0x2F] = X86_OP_ENTRY3(VCOMI, None,None, V,x, W,x, vex4 p_00_66), + [0x2E] = X86_OP_GROUP3(VxCOMISx, None,None, V,x, W,x, vex3 p_00_66), /* VUCOMISS/SD */ + [0x2F] = X86_OP_GROUP3(VxCOMISx, None,None, V,x, W,x, vex3 p_00_66), /* VCOMISS/SD */ [0x38] = X86_OP_GROUP0(0F38), [0x3a] = X86_OP_GROUP0(0F3A), [0x58] = X86_OP_ENTRY3(VADD, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), [0x59] = X86_OP_ENTRY3(VMUL, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), - [0x5a] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), + [0x5a] = X86_OP_GROUP3(sse_unary, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), /* CVTPS2PD */ [0x5b] = X86_OP_GROUP0(0F5B), [0x5c] = X86_OP_ENTRY3(VSUB, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), [0x5d] = X86_OP_ENTRY3(VMIN, V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), diff --git a/target/i386/tcg/emit.c.inc b/target/i386/tcg/emit.c.inc index 95fb4f5..4fe8dec 100644 --- a/target/i386/tcg/emit.c.inc +++ b/target/i386/tcg/emit.c.inc @@ -2285,7 +2285,7 @@ static void gen_VZEROALL(DisasContext *s, CPUX86State *env, X86DecodedInsn *deco { TCGv_ptr ptr = tcg_temp_new_ptr(); - tcg_gen_addi_ptr(ptr, cpu_env, offsetof(CPUX86State, xmm_t0)); + tcg_gen_addi_ptr(ptr, cpu_env, offsetof(CPUX86State, xmm_regs)); gen_helper_memset(ptr, ptr, tcg_constant_i32(0), tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg))); } diff --git a/tcg/meson.build b/tcg/meson.build index f56c465..bdc185a 100644 --- a/tcg/meson.build +++ b/tcg/meson.build @@ -13,7 +13,7 @@ tcg_ss.add(files( if get_option('tcg_interpreter') libffi = dependency('libffi', version: '>=3.0', required: true, - method: 'pkg-config', kwargs: static_kwargs) + method: 'pkg-config') specific_ss.add(libffi) specific_ss.add(files('tci.c')) endif diff --git a/tests/Makefile.include b/tests/Makefile.include index a4de0ad..5b838ec 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -89,7 +89,8 @@ distclean-tcg: $(DISTCLEAN_TCG_TARGET_RULES) # Build up our target list from the filtered list of ninja targets TARGETS=$(patsubst libqemu-%.fa, %, $(filter libqemu-%.fa, $(ninja-targets))) -TESTS_VENV_DIR=$(BUILD_DIR)/tests/venv +TESTS_VENV_DIR=$(BUILD_DIR)/pyvenv +TESTS_VENV_TOKEN=$(BUILD_DIR)/pyvenv/tests.group TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results TESTS_PYTHON=$(TESTS_VENV_DIR)/bin/python3 @@ -111,8 +112,7 @@ quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \ $(TESTS_PYTHON) -m pip -q --disable-pip-version-check $1, \ "VENVPIP","$1") -$(TESTS_VENV_DIR): $(TESTS_VENV_REQ) - $(call quiet-command, $(PYTHON) -m venv $@, VENV, $@) +$(TESTS_VENV_TOKEN): $(TESTS_VENV_REQ) $(call quiet-venv-pip,install -e "$(SRC_PATH)/python/") $(call quiet-venv-pip,install -r $(TESTS_VENV_REQ)) $(call quiet-command, touch $@) @@ -121,7 +121,7 @@ $(TESTS_RESULTS_DIR): $(call quiet-command, mkdir -p $@, \ MKDIR, $@) -check-venv: $(TESTS_VENV_DIR) +check-venv: $(TESTS_VENV_TOKEN) FEDORA_31_ARCHES_TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGETS))) FEDORA_31_ARCHES_CANDIDATES=$(patsubst ppc64,ppc64le,$(FEDORA_31_ARCHES_TARGETS)) @@ -167,7 +167,7 @@ check: check-build: run-ninja check-clean: - rm -rf $(TESTS_VENV_DIR) $(TESTS_RESULTS_DIR) + rm -rf $(TESTS_RESULTS_DIR) clean: check-clean clean-tcg distclean: distclean-tcg diff --git a/tests/docker/dockerfiles/debian-all-test-cross.docker b/tests/docker/dockerfiles/debian-all-test-cross.docker index 981e9bd..f9f4015 100644 --- a/tests/docker/dockerfiles/debian-all-test-cross.docker +++ b/tests/docker/dockerfiles/debian-all-test-cross.docker @@ -57,7 +57,8 @@ RUN DEBIAN_FRONTEND=noninteractive eatmydata \ gcc-sh4-linux-gnu \ libc6-dev-sh4-cross \ gcc-sparc64-linux-gnu \ - libc6-dev-sparc64-cross + libc6-dev-sparc64-cross \ + python3-venv ENV QEMU_CONFIGURE_OPTS --disable-system --disable-docs --disable-tools ENV DEF_TARGET_LIST aarch64-linux-user,alpha-linux-user,arm-linux-user,hppa-linux-user,i386-linux-user,m68k-linux-user,mips-linux-user,mips64-linux-user,mips64el-linux-user,mipsel-linux-user,ppc-linux-user,ppc64-linux-user,ppc64le-linux-user,riscv64-linux-user,s390x-linux-user,sh4-linux-user,sparc64-linux-user diff --git a/tests/docker/dockerfiles/debian-hexagon-cross.docker b/tests/docker/dockerfiles/debian-hexagon-cross.docker index b99d99f..c2cfb6a 100644 --- a/tests/docker/dockerfiles/debian-hexagon-cross.docker +++ b/tests/docker/dockerfiles/debian-hexagon-cross.docker @@ -20,7 +20,8 @@ RUN apt-get update && \ bison \ flex \ git \ - ninja-build && \ + ninja-build \ + python3-venv && \ # Install QEMU build deps for use in CI DEBIAN_FRONTEND=noninteractive eatmydata \ apt build-dep -yy --arch-only qemu diff --git a/tests/docker/dockerfiles/debian-riscv64-cross.docker b/tests/docker/dockerfiles/debian-riscv64-cross.docker index 803afb9..081404e 100644 --- a/tests/docker/dockerfiles/debian-riscv64-cross.docker +++ b/tests/docker/dockerfiles/debian-riscv64-cross.docker @@ -28,7 +28,8 @@ RUN DEBIAN_FRONTEND=noninteractive eatmydata apt install -yy \ libglib2.0-dev \ ninja-build \ pkg-config \ - python3 + python3 \ + python3-venv # Add ports and riscv64 architecture RUN echo "deb http://ftp.ports.debian.org/debian-ports/ sid main" >> /etc/apt/sources.list diff --git a/tests/docker/dockerfiles/debian-tricore-cross.docker b/tests/docker/dockerfiles/debian-tricore-cross.docker index cfd2faf..269bfa8 100644 --- a/tests/docker/dockerfiles/debian-tricore-cross.docker +++ b/tests/docker/dockerfiles/debian-tricore-cross.docker @@ -33,7 +33,8 @@ RUN apt update && \ pkgconf \ python3-pip \ python3-setuptools \ - python3-wheel + python3-wheel \ + python3-venv RUN curl -#SL https://github.com/bkoppelmann/package_940/releases/download/tricore-toolchain-9.40/tricore-toolchain-9.4.0.tar.gz \ | tar -xzC /usr/local/ diff --git a/tests/qemu-iotests/meson.build b/tests/qemu-iotests/meson.build index 9735071..44761e1 100644 --- a/tests/qemu-iotests/meson.build +++ b/tests/qemu-iotests/meson.build @@ -2,7 +2,7 @@ if not have_tools or targetos == 'windows' or get_option('gprof') subdir_done() endif -foreach cflag: config_host['QEMU_CFLAGS'].split() +foreach cflag: qemu_ldflags if cflag.startswith('-fsanitize') and \ not cflag.contains('safe-stack') and not cflag.contains('cfi-icall') message('Sanitizers are enabled ==> Disabled the qemu-iotests.') diff --git a/tests/qtest/meson.build b/tests/qtest/meson.build index 48cd35b..ab42277 100644 --- a/tests/qtest/meson.build +++ b/tests/qtest/meson.build @@ -23,7 +23,7 @@ qtests_generic = [ 'readconfig-test', 'netdev-socket', ] -if config_host.has_key('CONFIG_MODULES') +if enable_modules qtests_generic += [ 'modules-test' ] endif diff --git a/tests/requirements.txt b/tests/requirements.txt index a6f73da..0e008b9 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,9 @@ # Add Python module requirements, one per line, to be installed -# in the tests/venv Python virtual environment. For more info, +# in the qemu build_dir/pyvenv Python virtual environment. For more info, # refer to: https://pip.pypa.io/en/stable/user_guide/#id1 -# Note that qemu.git/python/ is always implicitly installed. +# +# Note that qemu.git/python/ is implicitly installed to this venv when +# 'make check-venv' is run, and will persist until configure is run +# again. avocado-framework==101.0 pycdlib==1.11.0 diff --git a/tests/tcg/i386/test-avx.py b/tests/tcg/i386/test-avx.py index d9ca00a..641a2ef 100755 --- a/tests/tcg/i386/test-avx.py +++ b/tests/tcg/i386/test-avx.py @@ -49,7 +49,7 @@ imask = { 'VEXTRACT[FI]128': 0x01, 'VINSERT[FI]128': 0x01, 'VPBLENDD': 0xff, - 'VPERM2[FI]128': 0x33, + 'VPERM2[FI]128': 0xbb, 'VPERMPD': 0xff, 'VPERMQ': 0xff, 'VPERMILPS': 0xff, diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 3bc78d8..48ae660 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -147,7 +147,7 @@ if have_system # Some tests: test-char, test-qdev-global-props, and test-qga, # are not runnable under TSan due to a known issue. # https://github.com/google/sanitizers/issues/1116 - if 'CONFIG_TSAN' not in config_host + if not get_option('tsan') if 'CONFIG_POSIX' in config_host tests += { 'test-char': ['socket-helpers.c', qom, io, chardev] diff --git a/tests/vm/netbsd b/tests/vm/netbsd index 0b9536c..c7e3f1e 100755 --- a/tests/vm/netbsd +++ b/tests/vm/netbsd @@ -30,6 +30,8 @@ class NetBSDVM(basevm.BaseVM): "git-base", "pkgconf", "xz", + "python310", + "py310-expat", "ninja-build", # gnu tools diff --git a/util/coroutine-win32.c b/util/coroutine-windows.c index 7db2e8f..7db2e8f 100644 --- a/util/coroutine-win32.c +++ b/util/coroutine-windows.c diff --git a/util/meson.build b/util/meson.build index 3c2cfc6..e1f1c39 100644 --- a/util/meson.build +++ b/util/meson.build @@ -26,7 +26,9 @@ util_ss.add(when: 'CONFIG_WIN32', if_true: files('oslib-win32.c')) util_ss.add(when: 'CONFIG_WIN32', if_true: files('qemu-thread-win32.c')) util_ss.add(when: 'CONFIG_WIN32', if_true: winmm) util_ss.add(when: 'CONFIG_WIN32', if_true: pathcch) -util_ss.add(when: 'HAVE_GLIB_WITH_SLICE_ALLOCATOR', if_true: files('qtree.c')) +if glib_has_gslice + util_ss.add(files('qtree.c')) +endif util_ss.add(files('envlist.c', 'path.c', 'module.c')) util_ss.add(files('host-utils.c')) util_ss.add(files('bitmap.c', 'bitops.c')) @@ -76,7 +78,7 @@ if have_block or have_ga util_ss.add(files('base64.c')) util_ss.add(files('main-loop.c')) util_ss.add(files('qemu-coroutine.c', 'qemu-coroutine-lock.c', 'qemu-coroutine-io.c')) - util_ss.add(files('coroutine-@0@.c'.format(config_host['CONFIG_COROUTINE_BACKEND']))) + util_ss.add(files(f'coroutine-@coroutine_backend@.c')) util_ss.add(files('thread-pool.c', 'qemu-timer.c')) util_ss.add(files('qemu-sockets.c')) endif |