diff options
Diffstat (limited to 'accel')
36 files changed, 577 insertions, 853 deletions
diff --git a/accel/accel-common.c b/accel/accel-common.c new file mode 100644 index 0000000..4894b98 --- /dev/null +++ b/accel/accel-common.c @@ -0,0 +1,142 @@ +/* + * QEMU accel class, components common to system emulation and user mode + * + * Copyright (c) 2003-2008 Fabrice Bellard + * Copyright (c) 2014 Red Hat Inc. + * + * SPDX-License-Identifier: MIT + */ + +#include "qemu/osdep.h" +#include "qemu/accel.h" +#include "qemu/target-info.h" +#include "accel/accel-cpu.h" +#include "accel-internal.h" + +/* Lookup AccelClass from opt_name. Returns NULL if not found */ +AccelClass *accel_find(const char *opt_name) +{ + char *class_name = g_strdup_printf(ACCEL_CLASS_NAME("%s"), opt_name); + AccelClass *ac = ACCEL_CLASS(module_object_class_by_name(class_name)); + g_free(class_name); + return ac; +} + +/* Return the name of the current accelerator */ +const char *current_accel_name(void) +{ + AccelClass *ac = ACCEL_GET_CLASS(current_accel()); + + return ac->name; +} + +static void accel_init_cpu_int_aux(ObjectClass *klass, void *opaque) +{ + CPUClass *cc = CPU_CLASS(klass); + AccelCPUClass *accel_cpu = opaque; + + /* + * The first callback allows accel-cpu to run initializations + * for the CPU, customizing CPU behavior according to the accelerator. + * + * The second one allows the CPU to customize the accel-cpu + * behavior according to the CPU. + * + * The second is currently only used by TCG, to specialize the + * TCGCPUOps depending on the CPU type. + */ + cc->accel_cpu = accel_cpu; + if (accel_cpu->cpu_class_init) { + accel_cpu->cpu_class_init(cc); + } + if (cc->init_accel_cpu) { + cc->init_accel_cpu(accel_cpu, cc); + } +} + +/* initialize the arch-specific accel CpuClass interfaces */ +static void accel_init_cpu_interfaces(AccelClass *ac) +{ + const char *ac_name; /* AccelClass name */ + char *acc_name; /* AccelCPUClass name */ + ObjectClass *acc; /* AccelCPUClass */ + const char *cpu_resolving_type = target_cpu_type(); + + ac_name = object_class_get_name(OBJECT_CLASS(ac)); + g_assert(ac_name != NULL); + + acc_name = g_strdup_printf("%s-%s", ac_name, cpu_resolving_type); + acc = object_class_by_name(acc_name); + g_free(acc_name); + + if (acc) { + object_class_foreach(accel_init_cpu_int_aux, + cpu_resolving_type, false, acc); + } +} + +void accel_init_interfaces(AccelClass *ac) +{ + accel_init_ops_interfaces(ac); + accel_init_cpu_interfaces(ac); +} + +void accel_cpu_instance_init(CPUState *cpu) +{ + if (cpu->cc->accel_cpu && cpu->cc->accel_cpu->cpu_instance_init) { + cpu->cc->accel_cpu->cpu_instance_init(cpu); + } +} + +bool accel_cpu_common_realize(CPUState *cpu, Error **errp) +{ + AccelState *accel = current_accel(); + AccelClass *acc = ACCEL_GET_CLASS(accel); + + /* target specific realization */ + if (cpu->cc->accel_cpu + && cpu->cc->accel_cpu->cpu_target_realize + && !cpu->cc->accel_cpu->cpu_target_realize(cpu, errp)) { + return false; + } + + /* generic realization */ + if (acc->cpu_common_realize && !acc->cpu_common_realize(cpu, errp)) { + return false; + } + + return true; +} + +void accel_cpu_common_unrealize(CPUState *cpu) +{ + AccelState *accel = current_accel(); + AccelClass *acc = ACCEL_GET_CLASS(accel); + + /* generic unrealization */ + if (acc->cpu_common_unrealize) { + acc->cpu_common_unrealize(cpu); + } +} + +int accel_supported_gdbstub_sstep_flags(void) +{ + AccelState *accel = current_accel(); + AccelClass *acc = ACCEL_GET_CLASS(accel); + if (acc->gdbstub_supported_sstep_flags) { + return acc->gdbstub_supported_sstep_flags(); + } + return 0; +} + +static const TypeInfo accel_types[] = { + { + .name = TYPE_ACCEL, + .parent = TYPE_OBJECT, + .class_size = sizeof(AccelClass), + .instance_size = sizeof(AccelState), + .abstract = true, + }, +}; + +DEFINE_TYPES(accel_types) diff --git a/accel/accel-system.h b/accel/accel-internal.h index 2d37c73..d3a4422 100644 --- a/accel/accel-system.h +++ b/accel/accel-internal.h @@ -1,5 +1,5 @@ /* - * QEMU System Emulation accel internal functions + * QEMU accel internal functions * * Copyright 2021 SUSE LLC * @@ -7,9 +7,11 @@ * See the COPYING file in the top-level directory. */ -#ifndef ACCEL_SYSTEM_H -#define ACCEL_SYSTEM_H +#ifndef ACCEL_INTERNAL_H +#define ACCEL_INTERNAL_H -void accel_system_init_ops_interfaces(AccelClass *ac); +#include "qemu/accel.h" + +void accel_init_ops_interfaces(AccelClass *ac); #endif /* ACCEL_SYSTEM_H */ diff --git a/accel/accel-system.c b/accel/accel-system.c index 5df49fb..a0f562a 100644 --- a/accel/accel-system.c +++ b/accel/accel-system.c @@ -29,7 +29,7 @@ #include "system/accel-ops.h" #include "system/cpus.h" #include "qemu/error-report.h" -#include "accel-system.h" +#include "accel-internal.h" int accel_init_machine(AccelState *accel, MachineState *ms) { @@ -63,7 +63,7 @@ void accel_setup_post(MachineState *ms) } /* initialize the arch-independent accel operation interfaces */ -void accel_system_init_ops_interfaces(AccelClass *ac) +void accel_init_ops_interfaces(AccelClass *ac) { const char *ac_name; char *ops_name; diff --git a/accel/accel-target.c b/accel/accel-target.c index 33a539b..7fd392f 100644 --- a/accel/accel-target.c +++ b/accel/accel-target.c @@ -24,141 +24,8 @@ */ #include "qemu/osdep.h" -#include "qemu/accel.h" - -#include "cpu.h" #include "accel/accel-cpu-target.h" -#ifndef CONFIG_USER_ONLY -#include "accel-system.h" -#endif /* !CONFIG_USER_ONLY */ - -static const TypeInfo accel_type = { - .name = TYPE_ACCEL, - .parent = TYPE_OBJECT, - .class_size = sizeof(AccelClass), - .instance_size = sizeof(AccelState), - .abstract = true, -}; - -/* Lookup AccelClass from opt_name. Returns NULL if not found */ -AccelClass *accel_find(const char *opt_name) -{ - char *class_name = g_strdup_printf(ACCEL_CLASS_NAME("%s"), opt_name); - AccelClass *ac = ACCEL_CLASS(module_object_class_by_name(class_name)); - g_free(class_name); - return ac; -} - -/* Return the name of the current accelerator */ -const char *current_accel_name(void) -{ - AccelClass *ac = ACCEL_GET_CLASS(current_accel()); - - return ac->name; -} - -static void accel_init_cpu_int_aux(ObjectClass *klass, void *opaque) -{ - CPUClass *cc = CPU_CLASS(klass); - AccelCPUClass *accel_cpu = opaque; - - /* - * The first callback allows accel-cpu to run initializations - * for the CPU, customizing CPU behavior according to the accelerator. - * - * The second one allows the CPU to customize the accel-cpu - * behavior according to the CPU. - * - * The second is currently only used by TCG, to specialize the - * TCGCPUOps depending on the CPU type. - */ - cc->accel_cpu = accel_cpu; - if (accel_cpu->cpu_class_init) { - accel_cpu->cpu_class_init(cc); - } - if (cc->init_accel_cpu) { - cc->init_accel_cpu(accel_cpu, cc); - } -} - -/* initialize the arch-specific accel CpuClass interfaces */ -static void accel_init_cpu_interfaces(AccelClass *ac) -{ - const char *ac_name; /* AccelClass name */ - char *acc_name; /* AccelCPUClass name */ - ObjectClass *acc; /* AccelCPUClass */ - - ac_name = object_class_get_name(OBJECT_CLASS(ac)); - g_assert(ac_name != NULL); - - acc_name = g_strdup_printf("%s-%s", ac_name, CPU_RESOLVING_TYPE); - acc = object_class_by_name(acc_name); - g_free(acc_name); - - if (acc) { - object_class_foreach(accel_init_cpu_int_aux, - CPU_RESOLVING_TYPE, false, acc); - } -} - -void accel_init_interfaces(AccelClass *ac) -{ -#ifndef CONFIG_USER_ONLY - accel_system_init_ops_interfaces(ac); -#endif /* !CONFIG_USER_ONLY */ - - accel_init_cpu_interfaces(ac); -} - -void accel_cpu_instance_init(CPUState *cpu) -{ - if (cpu->cc->accel_cpu && cpu->cc->accel_cpu->cpu_instance_init) { - cpu->cc->accel_cpu->cpu_instance_init(cpu); - } -} - -bool accel_cpu_common_realize(CPUState *cpu, Error **errp) -{ - AccelState *accel = current_accel(); - AccelClass *acc = ACCEL_GET_CLASS(accel); - - /* target specific realization */ - if (cpu->cc->accel_cpu - && cpu->cc->accel_cpu->cpu_target_realize - && !cpu->cc->accel_cpu->cpu_target_realize(cpu, errp)) { - return false; - } - - /* generic realization */ - if (acc->cpu_common_realize && !acc->cpu_common_realize(cpu, errp)) { - return false; - } - - return true; -} - -void accel_cpu_common_unrealize(CPUState *cpu) -{ - AccelState *accel = current_accel(); - AccelClass *acc = ACCEL_GET_CLASS(accel); - - /* generic unrealization */ - if (acc->cpu_common_unrealize) { - acc->cpu_common_unrealize(cpu); - } -} - -int accel_supported_gdbstub_sstep_flags(void) -{ - AccelState *accel = current_accel(); - AccelClass *acc = ACCEL_GET_CLASS(accel); - if (acc->gdbstub_supported_sstep_flags) { - return acc->gdbstub_supported_sstep_flags(); - } - return 0; -} - static const TypeInfo accel_cpu_type = { .name = TYPE_ACCEL_CPU, .parent = TYPE_OBJECT, @@ -168,7 +35,6 @@ static const TypeInfo accel_cpu_type = { static void register_accel_types(void) { - type_register_static(&accel_type); type_register_static(&accel_cpu_type); } diff --git a/accel/accel-user.c b/accel/accel-user.c index 22b6a1a..7d19230 100644 --- a/accel/accel-user.c +++ b/accel/accel-user.c @@ -9,6 +9,12 @@ #include "qemu/osdep.h" #include "qemu/accel.h" +#include "accel-internal.h" + +void accel_init_ops_interfaces(AccelClass *ac) +{ + /* nothing */ +} AccelState *current_accel(void) { diff --git a/accel/hvf/hvf-accel-ops.c b/accel/hvf/hvf-accel-ops.c index 12fc30c..5375de7 100644 --- a/accel/hvf/hvf-accel-ops.c +++ b/accel/hvf/hvf-accel-ops.c @@ -50,7 +50,7 @@ #include "qemu/osdep.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" -#include "exec/address-spaces.h" +#include "system/address-spaces.h" #include "exec/exec-all.h" #include "gdbstub/enums.h" #include "hw/boards.h" @@ -355,7 +355,7 @@ static inline int hvf_gdbstub_sstep_flags(void) return SSTEP_ENABLE | SSTEP_NOIRQ; } -static void hvf_accel_class_init(ObjectClass *oc, void *data) +static void hvf_accel_class_init(ObjectClass *oc, const void *data) { AccelClass *ac = ACCEL_CLASS(oc); ac->name = "HVF"; @@ -578,7 +578,7 @@ static void hvf_remove_all_breakpoints(CPUState *cpu) } } -static void hvf_accel_ops_class_init(ObjectClass *oc, void *data) +static void hvf_accel_ops_class_init(ObjectClass *oc, const void *data) { AccelOpsClass *ops = ACCEL_OPS_CLASS(oc); diff --git a/accel/kvm/kvm-accel-ops.c b/accel/kvm/kvm-accel-ops.c index 54ea609..e5c1544 100644 --- a/accel/kvm/kvm-accel-ops.c +++ b/accel/kvm/kvm-accel-ops.c @@ -90,7 +90,7 @@ static int kvm_update_guest_debug_ops(CPUState *cpu) } #endif -static void kvm_accel_ops_class_init(ObjectClass *oc, void *data) +static void kvm_accel_ops_class_init(ObjectClass *oc, const void *data) { AccelOpsClass *ops = ACCEL_OPS_CLASS(oc); diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index f89568b..278a506 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -33,8 +33,9 @@ #include "system/cpus.h" #include "system/accel-blocker.h" #include "qemu/bswap.h" -#include "exec/memory.h" -#include "exec/ram_addr.h" +#include "exec/tswap.h" +#include "system/memory.h" +#include "system/ram_addr.h" #include "qemu/event_notifier.h" #include "qemu/main-loop.h" #include "trace.h" @@ -57,6 +58,11 @@ #include <sys/eventfd.h> #endif +#if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) +# define KVM_HAVE_MCE_INJECTION 1 +#endif + + /* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We * need to use the real host PAGE_SIZE, as that's what KVM will use. */ @@ -437,9 +443,8 @@ int kvm_unpark_vcpu(KVMState *s, unsigned long vcpu_id) return kvm_fd; } -static void kvm_reset_parked_vcpus(void *param) +static void kvm_reset_parked_vcpus(KVMState *s) { - KVMState *s = param; struct KVMParkedVcpu *cpu; QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) { @@ -1314,21 +1319,22 @@ bool kvm_hwpoisoned_mem(void) static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size) { -#if HOST_BIG_ENDIAN != TARGET_BIG_ENDIAN - /* The kernel expects ioeventfd values in HOST_BIG_ENDIAN - * endianness, but the memory core hands them in target endianness. - * For example, PPC is always treated as big-endian even if running - * on KVM and on PPC64LE. Correct here. - */ - switch (size) { - case 2: - val = bswap16(val); - break; - case 4: - val = bswap32(val); - break; + if (target_needs_bswap()) { + /* + * The kernel expects ioeventfd values in HOST_BIG_ENDIAN + * endianness, but the memory core hands them in target endianness. + * For example, PPC is always treated as big-endian even if running + * on KVM and on PPC64LE. Correct here, swapping back. + */ + switch (size) { + case 2: + val = bswap16(val); + break; + case 4: + val = bswap32(val); + break; + } } -#endif return val; } @@ -2738,7 +2744,6 @@ static int kvm_init(MachineState *ms) } qemu_register_reset(kvm_unpoison_all, NULL); - qemu_register_reset(kvm_reset_parked_vcpus, s); if (s->kernel_irqchip_allowed) { kvm_irqchip_create(s); @@ -2908,6 +2913,10 @@ static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg void kvm_cpu_synchronize_post_reset(CPUState *cpu) { run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL); + + if (cpu == first_cpu) { + kvm_reset_parked_vcpus(kvm_state); + } } static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg) @@ -3957,7 +3966,7 @@ static int kvm_gdbstub_sstep_flags(void) return kvm_sstep_flags; } -static void kvm_accel_class_init(ObjectClass *oc, void *data) +static void kvm_accel_class_init(ObjectClass *oc, const void *data) { AccelClass *ac = ACCEL_CLASS(oc); ac->name = "KVM"; diff --git a/accel/meson.build b/accel/meson.build index 5eaeb68..5290931 100644 --- a/accel/meson.build +++ b/accel/meson.build @@ -1,3 +1,4 @@ +common_ss.add(files('accel-common.c')) specific_ss.add(files('accel-target.c')) system_ss.add(files('accel-system.c', 'accel-blocker.c')) user_ss.add(files('accel-user.c')) diff --git a/accel/qtest/qtest.c b/accel/qtest/qtest.c index 7fae80f..92bed92 100644 --- a/accel/qtest/qtest.c +++ b/accel/qtest/qtest.c @@ -42,7 +42,7 @@ static int qtest_init_accel(MachineState *ms) return 0; } -static void qtest_accel_class_init(ObjectClass *oc, void *data) +static void qtest_accel_class_init(ObjectClass *oc, const void *data) { AccelClass *ac = ACCEL_CLASS(oc); ac->name = "QTest"; @@ -59,7 +59,7 @@ static const TypeInfo qtest_accel_type = { }; module_obj(TYPE_QTEST_ACCEL); -static void qtest_accel_ops_class_init(ObjectClass *oc, void *data) +static void qtest_accel_ops_class_init(ObjectClass *oc, const void *data) { AccelOpsClass *ops = ACCEL_OPS_CLASS(oc); diff --git a/accel/stubs/tcg-stub.c b/accel/stubs/tcg-stub.c index b2b9881..3b76b8b 100644 --- a/accel/stubs/tcg-stub.c +++ b/accel/stubs/tcg-stub.c @@ -11,8 +11,7 @@ */ #include "qemu/osdep.h" -#include "exec/tb-flush.h" -#include "exec/exec-all.h" +#include "exec/cpu-common.h" G_NORETURN void cpu_loop_exit(CPUState *cpu) { diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h index 89593b2..08a475c 100644 --- a/accel/tcg/atomic_template.h +++ b/accel/tcg/atomic_template.h @@ -77,7 +77,7 @@ # define END _le #endif -ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr, +ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, vaddr addr, ABI_TYPE cmpv, ABI_TYPE newv, MemOpIdx oi, uintptr_t retaddr) { @@ -101,7 +101,7 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr, } #if DATA_SIZE < 16 -ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val, +ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, vaddr addr, ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) { DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, @@ -120,7 +120,7 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val, } #define GEN_ATOMIC_HELPER(X) \ -ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ +ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr, \ ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ { \ DATA_TYPE *haddr, ret; \ @@ -156,7 +156,7 @@ GEN_ATOMIC_HELPER(xor_fetch) * of CF_PARALLEL's value, we'll trace just a read and a write. */ #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ -ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ +ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr, \ ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ { \ XDATA_TYPE *haddr, cmp, old, new, val = xval; \ @@ -202,7 +202,7 @@ GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new) # define END _be #endif -ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr, +ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, vaddr addr, ABI_TYPE cmpv, ABI_TYPE newv, MemOpIdx oi, uintptr_t retaddr) { @@ -226,7 +226,7 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr, } #if DATA_SIZE < 16 -ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val, +ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, vaddr addr, ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) { DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, @@ -245,7 +245,7 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val, } #define GEN_ATOMIC_HELPER(X) \ -ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ +ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr, \ ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \ { \ DATA_TYPE *haddr, ret; \ @@ -278,7 +278,7 @@ GEN_ATOMIC_HELPER(xor_fetch) * of CF_PARALLEL's value, we'll trace just a read and a write. */ #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \ -ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \ +ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr, \ ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \ { \ XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval; \ diff --git a/accel/tcg/backend-ldst.h b/accel/tcg/backend-ldst.h new file mode 100644 index 0000000..9c3a407 --- /dev/null +++ b/accel/tcg/backend-ldst.h @@ -0,0 +1,41 @@ +/* + * Internal memory barrier helpers for QEMU (target agnostic) + * + * Copyright (c) 2003 Fabrice Bellard + * + * SPDX-License-Identifier: LGPL-2.1-or-later + */ + +#ifndef ACCEL_TCG_BACKEND_LDST_H +#define ACCEL_TCG_BACKEND_LDST_H + +#include "tcg-target-mo.h" + +/** + * tcg_req_mo: + * @guest_mo: Guest default memory order + * @type: TCGBar + * + * Filter @type to the barrier that is required for the guest + * memory ordering vs the host memory ordering. A non-zero + * result indicates that some barrier is required. + */ +#define tcg_req_mo(guest_mo, type) \ + ((type) & guest_mo & ~TCG_TARGET_DEFAULT_MO) + +/** + * cpu_req_mo: + * @cpu: CPUState + * @type: TCGBar + * + * If tcg_req_mo indicates a barrier for @type is required + * for the guest memory model, issue a host memory barrier. + */ +#define cpu_req_mo(cpu, type) \ + do { \ + if (tcg_req_mo(cpu->cc->tcg_ops->guest_default_memory_order, type)) { \ + smp_mb(); \ + } \ + } while (0) + +#endif diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index ef3d967..87eba83 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -22,19 +22,22 @@ #include "qapi/error.h" #include "qapi/type-helpers.h" #include "hw/core/cpu.h" +#include "accel/tcg/cpu-ldst.h" #include "accel/tcg/cpu-ops.h" #include "trace.h" #include "disas/disas.h" #include "exec/cpu-common.h" +#include "exec/cpu-interrupt.h" #include "exec/page-protection.h" +#include "exec/mmap-lock.h" #include "exec/translation-block.h" #include "tcg/tcg.h" #include "qemu/atomic.h" #include "qemu/rcu.h" #include "exec/log.h" #include "qemu/main-loop.h" -#include "exec/cpu-all.h" -#include "system/cpu-timers.h" +#include "cpu.h" +#include "exec/icount.h" #include "exec/replay-core.h" #include "system/tcg.h" #include "exec/helper-proto-common.h" @@ -665,7 +668,6 @@ static inline void tb_add_jump(TranslationBlock *tb, int n, out_unlock_next: qemu_spin_unlock(&tb_next->jmp_lock); - return; } static inline bool cpu_handle_halt(CPUState *cpu) @@ -1074,6 +1076,7 @@ bool tcg_exec_realizefn(CPUState *cpu, Error **errp) assert(tcg_ops->cpu_exec_interrupt); #endif /* !CONFIG_USER_ONLY */ assert(tcg_ops->translate_code); + assert(tcg_ops->mmu_index); tcg_ops->initialize(); tcg_target_initialized = true; } diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index fb22048..d9fb68d 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -22,12 +22,11 @@ #include "accel/tcg/cpu-ops.h" #include "exec/exec-all.h" #include "exec/page-protection.h" -#include "exec/memory.h" -#include "exec/cpu_ldst.h" +#include "system/memory.h" +#include "accel/tcg/cpu-ldst.h" #include "exec/cputlb.h" #include "exec/tb-flush.h" -#include "exec/memory-internal.h" -#include "exec/ram_addr.h" +#include "system/ram_addr.h" #include "exec/mmu-access-type.h" #include "exec/tlb-common.h" #include "exec/vaddr.h" @@ -35,18 +34,22 @@ #include "qemu/error-report.h" #include "exec/log.h" #include "exec/helper-proto-common.h" +#include "exec/tlb-flags.h" #include "qemu/atomic.h" #include "qemu/atomic128.h" #include "tb-internal.h" #include "trace.h" #include "tb-hash.h" #include "tb-internal.h" +#include "tlb-bounds.h" #include "internal-common.h" #include "internal-target.h" #ifdef CONFIG_PLUGIN #include "qemu/plugin-memory.h" #endif #include "tcg/tcg-ldst.h" +#include "backend-ldst.h" + /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */ /* #define DEBUG_TLB */ @@ -882,18 +885,17 @@ void tlb_unprotect_code(ram_addr_t ram_addr) * * Called with tlb_c.lock held. */ -static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry, +static void tlb_reset_dirty_range_locked(CPUTLBEntryFull *full, CPUTLBEntry *ent, uintptr_t start, uintptr_t length) { - uintptr_t addr = tlb_entry->addr_write; + const uintptr_t addr = ent->addr_write; + int flags = addr | full->slow_flags[MMU_DATA_STORE]; - if ((addr & (TLB_INVALID_MASK | TLB_MMIO | - TLB_DISCARD_WRITE | TLB_NOTDIRTY)) == 0) { - addr &= TARGET_PAGE_MASK; - addr += tlb_entry->addend; - if ((addr - start) < length) { - qatomic_set(&tlb_entry->addr_write, - tlb_entry->addr_write | TLB_NOTDIRTY); + flags &= TLB_INVALID_MASK | TLB_MMIO | TLB_DISCARD_WRITE | TLB_NOTDIRTY; + if (flags == 0) { + uintptr_t host = (addr & TARGET_PAGE_MASK) + ent->addend; + if ((host - start) < length) { + qatomic_set(&ent->addr_write, addr | TLB_NOTDIRTY); } } } @@ -912,23 +914,25 @@ static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s) * We must take tlb_c.lock to avoid racing with another vCPU update. The only * thing actually updated is the target TLB entry ->addr_write flags. */ -void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length) +void tlb_reset_dirty(CPUState *cpu, uintptr_t start, uintptr_t length) { int mmu_idx; qemu_spin_lock(&cpu->neg.tlb.c.lock); for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { + CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx]; + CPUTLBDescFast *fast = &cpu->neg.tlb.f[mmu_idx]; + unsigned int n = tlb_n_entries(fast); unsigned int i; - unsigned int n = tlb_n_entries(&cpu->neg.tlb.f[mmu_idx]); for (i = 0; i < n; i++) { - tlb_reset_dirty_range_locked(&cpu->neg.tlb.f[mmu_idx].table[i], - start1, length); + tlb_reset_dirty_range_locked(&desc->fulltlb[i], &fast->table[i], + start, length); } for (i = 0; i < CPU_VTLB_SIZE; i++) { - tlb_reset_dirty_range_locked(&cpu->neg.tlb.d[mmu_idx].vtable[i], - start1, length); + tlb_reset_dirty_range_locked(&desc->vfulltlb[i], &desc->vtable[i], + start, length); } } qemu_spin_unlock(&cpu->neg.tlb.c.lock); @@ -2321,7 +2325,7 @@ static uint8_t do_ld1_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi, MMULookupLocals l; bool crosspage; - cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD); crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l); tcg_debug_assert(!crosspage); @@ -2336,7 +2340,7 @@ static uint16_t do_ld2_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi, uint16_t ret; uint8_t a, b; - cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD); crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l); if (likely(!crosspage)) { return do_ld_2(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra); @@ -2360,7 +2364,7 @@ static uint32_t do_ld4_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi, bool crosspage; uint32_t ret; - cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD); crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l); if (likely(!crosspage)) { return do_ld_4(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra); @@ -2381,7 +2385,7 @@ static uint64_t do_ld8_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi, bool crosspage; uint64_t ret; - cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD); crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l); if (likely(!crosspage)) { return do_ld_8(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra); @@ -2404,7 +2408,7 @@ static Int128 do_ld16_mmu(CPUState *cpu, vaddr addr, Int128 ret; int first; - cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD); crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_LOAD, &l); if (likely(!crosspage)) { if (unlikely(l.page[0].flags & TLB_MMIO)) { @@ -2732,7 +2736,7 @@ static void do_st1_mmu(CPUState *cpu, vaddr addr, uint8_t val, MMULookupLocals l; bool crosspage; - cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST); crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l); tcg_debug_assert(!crosspage); @@ -2746,7 +2750,7 @@ static void do_st2_mmu(CPUState *cpu, vaddr addr, uint16_t val, bool crosspage; uint8_t a, b; - cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST); crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l); if (likely(!crosspage)) { do_st_2(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra); @@ -2768,7 +2772,7 @@ static void do_st4_mmu(CPUState *cpu, vaddr addr, uint32_t val, MMULookupLocals l; bool crosspage; - cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST); crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l); if (likely(!crosspage)) { do_st_4(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra); @@ -2789,7 +2793,7 @@ static void do_st8_mmu(CPUState *cpu, vaddr addr, uint64_t val, MMULookupLocals l; bool crosspage; - cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST); crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l); if (likely(!crosspage)) { do_st_8(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra); @@ -2812,7 +2816,7 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val, uint64_t a, b; int first; - cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST); crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l); if (likely(!crosspage)) { if (unlikely(l.page[0].flags & TLB_MMIO)) { @@ -2897,53 +2901,25 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val, /* Code access functions. */ -uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr addr) -{ - CPUState *cs = env_cpu(env); - MemOpIdx oi = make_memop_idx(MO_UB, cpu_mmu_index(cs, true)); - return do_ld1_mmu(cs, addr, oi, 0, MMU_INST_FETCH); -} - -uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr addr) -{ - CPUState *cs = env_cpu(env); - MemOpIdx oi = make_memop_idx(MO_TEUW, cpu_mmu_index(cs, true)); - return do_ld2_mmu(cs, addr, oi, 0, MMU_INST_FETCH); -} - -uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr addr) -{ - CPUState *cs = env_cpu(env); - MemOpIdx oi = make_memop_idx(MO_TEUL, cpu_mmu_index(cs, true)); - return do_ld4_mmu(cs, addr, oi, 0, MMU_INST_FETCH); -} - -uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr addr) -{ - CPUState *cs = env_cpu(env); - MemOpIdx oi = make_memop_idx(MO_TEUQ, cpu_mmu_index(cs, true)); - return do_ld8_mmu(cs, addr, oi, 0, MMU_INST_FETCH); -} - -uint8_t cpu_ldb_code_mmu(CPUArchState *env, abi_ptr addr, +uint8_t cpu_ldb_code_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t retaddr) { return do_ld1_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH); } -uint16_t cpu_ldw_code_mmu(CPUArchState *env, abi_ptr addr, +uint16_t cpu_ldw_code_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t retaddr) { return do_ld2_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH); } -uint32_t cpu_ldl_code_mmu(CPUArchState *env, abi_ptr addr, +uint32_t cpu_ldl_code_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t retaddr) { return do_ld4_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH); } -uint64_t cpu_ldq_code_mmu(CPUArchState *env, abi_ptr addr, +uint64_t cpu_ldq_code_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t retaddr) { return do_ld8_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH); diff --git a/accel/tcg/icount-common.c b/accel/tcg/icount-common.c index 402d3e3..d647117 100644 --- a/accel/tcg/icount-common.c +++ b/accel/tcg/icount-common.c @@ -35,7 +35,7 @@ #include "system/replay.h" #include "system/runstate.h" #include "hw/core/cpu.h" -#include "system/cpu-timers.h" +#include "exec/icount.h" #include "system/cpu-timers-internal.h" /* diff --git a/accel/tcg/internal-common.h b/accel/tcg/internal-common.h index 9b6ab3a..2f00560 100644 --- a/accel/tcg/internal-common.h +++ b/accel/tcg/internal-common.h @@ -74,4 +74,38 @@ uint32_t curr_cflags(CPUState *cpu); void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr); +/** + * get_page_addr_code_hostp() + * @env: CPUArchState + * @addr: guest virtual address of guest code + * + * See get_page_addr_code() (full-system version) for documentation on the + * return value. + * + * Sets *@hostp (when @hostp is non-NULL) as follows. + * If the return value is -1, sets *@hostp to NULL. Otherwise, sets *@hostp + * to the host address where @addr's content is kept. + * + * Note: this function can trigger an exception. + */ +tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, vaddr addr, + void **hostp); + +/** + * get_page_addr_code() + * @env: CPUArchState + * @addr: guest virtual address of guest code + * + * If we cannot translate and execute from the entire RAM page, or if + * the region is not backed by RAM, returns -1. Otherwise, returns the + * ram_addr_t corresponding to the guest code at @addr. + * + * Note: this function can trigger an exception. + */ +static inline tb_page_addr_t get_page_addr_code(CPUArchState *env, + vaddr addr) +{ + return get_page_addr_code_hostp(env, addr, NULL); +} + #endif diff --git a/accel/tcg/internal-target.h b/accel/tcg/internal-target.h index 2cdf11c..9a9cef3 100644 --- a/accel/tcg/internal-target.h +++ b/accel/tcg/internal-target.h @@ -9,10 +9,11 @@ #ifndef ACCEL_TCG_INTERNAL_TARGET_H #define ACCEL_TCG_INTERNAL_TARGET_H +#include "cpu-param.h" #include "exec/exec-all.h" #include "exec/translation-block.h" #include "tb-internal.h" -#include "tcg-target-mo.h" +#include "exec/mmap-lock.h" /* * Access to the various translations structures need to be serialised @@ -42,38 +43,4 @@ void page_table_config_init(void); G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr); #endif /* CONFIG_USER_ONLY */ -/** - * tcg_req_mo: - * @type: TCGBar - * - * Filter @type to the barrier that is required for the guest - * memory ordering vs the host memory ordering. A non-zero - * result indicates that some barrier is required. - * - * If TCG_GUEST_DEFAULT_MO is not defined, assume that the - * guest requires strict ordering. - * - * This is a macro so that it's constant even without optimization. - */ -#ifdef TCG_GUEST_DEFAULT_MO -# define tcg_req_mo(type) \ - ((type) & TCG_GUEST_DEFAULT_MO & ~TCG_TARGET_DEFAULT_MO) -#else -# define tcg_req_mo(type) ((type) & ~TCG_TARGET_DEFAULT_MO) -#endif - -/** - * cpu_req_mo: - * @type: TCGBar - * - * If tcg_req_mo indicates a barrier for @type is required - * for the guest memory model, issue a host memory barrier. - */ -#define cpu_req_mo(type) \ - do { \ - if (tcg_req_mo(type)) { \ - smp_mb(); \ - } \ - } while (0) - #endif /* ACCEL_TCG_INTERNAL_H */ diff --git a/accel/tcg/ldst_common.c.inc b/accel/tcg/ldst_common.c.inc index ebbf380..9791a4e 100644 --- a/accel/tcg/ldst_common.c.inc +++ b/accel/tcg/ldst_common.c.inc @@ -135,7 +135,7 @@ static void plugin_load_cb(CPUArchState *env, abi_ptr addr, } } -uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) +uint8_t cpu_ldb_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra) { uint8_t ret; @@ -145,7 +145,7 @@ uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra) return ret; } -uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr addr, +uint16_t cpu_ldw_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra) { uint16_t ret; @@ -156,7 +156,7 @@ uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr addr, return ret; } -uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr addr, +uint32_t cpu_ldl_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra) { uint32_t ret; @@ -167,7 +167,7 @@ uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr addr, return ret; } -uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr addr, +uint64_t cpu_ldq_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra) { uint64_t ret; @@ -178,7 +178,7 @@ uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr addr, return ret; } -Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr, +Int128 cpu_ld16_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra) { Int128 ret; @@ -205,14 +205,14 @@ static void plugin_store_cb(CPUArchState *env, abi_ptr addr, } } -void cpu_stb_mmu(CPUArchState *env, abi_ptr addr, uint8_t val, +void cpu_stb_mmu(CPUArchState *env, vaddr addr, uint8_t val, MemOpIdx oi, uintptr_t retaddr) { helper_stb_mmu(env, addr, val, oi, retaddr); plugin_store_cb(env, addr, val, 0, oi); } -void cpu_stw_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, +void cpu_stw_mmu(CPUArchState *env, vaddr addr, uint16_t val, MemOpIdx oi, uintptr_t retaddr) { tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16); @@ -220,7 +220,7 @@ void cpu_stw_mmu(CPUArchState *env, abi_ptr addr, uint16_t val, plugin_store_cb(env, addr, val, 0, oi); } -void cpu_stl_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, +void cpu_stl_mmu(CPUArchState *env, vaddr addr, uint32_t val, MemOpIdx oi, uintptr_t retaddr) { tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32); @@ -228,7 +228,7 @@ void cpu_stl_mmu(CPUArchState *env, abi_ptr addr, uint32_t val, plugin_store_cb(env, addr, val, 0, oi); } -void cpu_stq_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, +void cpu_stq_mmu(CPUArchState *env, vaddr addr, uint64_t val, MemOpIdx oi, uintptr_t retaddr) { tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64); @@ -236,325 +236,10 @@ void cpu_stq_mmu(CPUArchState *env, abi_ptr addr, uint64_t val, plugin_store_cb(env, addr, val, 0, oi); } -void cpu_st16_mmu(CPUArchState *env, abi_ptr addr, Int128 val, +void cpu_st16_mmu(CPUArchState *env, vaddr addr, Int128 val, MemOpIdx oi, uintptr_t retaddr) { tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128); do_st16_mmu(env_cpu(env), addr, val, oi, retaddr); plugin_store_cb(env, addr, int128_getlo(val), int128_gethi(val), oi); } - -/* - * Wrappers of the above - */ - -uint32_t cpu_ldub_mmuidx_ra(CPUArchState *env, abi_ptr addr, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); - return cpu_ldb_mmu(env, addr, oi, ra); -} - -int cpu_ldsb_mmuidx_ra(CPUArchState *env, abi_ptr addr, - int mmu_idx, uintptr_t ra) -{ - return (int8_t)cpu_ldub_mmuidx_ra(env, addr, mmu_idx, ra); -} - -uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx); - return cpu_ldw_mmu(env, addr, oi, ra); -} - -int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, - int mmu_idx, uintptr_t ra) -{ - return (int16_t)cpu_lduw_be_mmuidx_ra(env, addr, mmu_idx, ra); -} - -uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx); - return cpu_ldl_mmu(env, addr, oi, ra); -} - -uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_BEUQ | MO_UNALN, mmu_idx); - return cpu_ldq_mmu(env, addr, oi, ra); -} - -uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx); - return cpu_ldw_mmu(env, addr, oi, ra); -} - -int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, - int mmu_idx, uintptr_t ra) -{ - return (int16_t)cpu_lduw_le_mmuidx_ra(env, addr, mmu_idx, ra); -} - -uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx); - return cpu_ldl_mmu(env, addr, oi, ra); -} - -uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_LEUQ | MO_UNALN, mmu_idx); - return cpu_ldq_mmu(env, addr, oi, ra); -} - -void cpu_stb_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); - cpu_stb_mmu(env, addr, val, oi, ra); -} - -void cpu_stw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx); - cpu_stw_mmu(env, addr, val, oi, ra); -} - -void cpu_stl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx); - cpu_stl_mmu(env, addr, val, oi, ra); -} - -void cpu_stq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_BEUQ | MO_UNALN, mmu_idx); - cpu_stq_mmu(env, addr, val, oi, ra); -} - -void cpu_stw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx); - cpu_stw_mmu(env, addr, val, oi, ra); -} - -void cpu_stl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx); - cpu_stl_mmu(env, addr, val, oi, ra); -} - -void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val, - int mmu_idx, uintptr_t ra) -{ - MemOpIdx oi = make_memop_idx(MO_LEUQ | MO_UNALN, mmu_idx); - cpu_stq_mmu(env, addr, val, oi, ra); -} - -/*--------------------------*/ - -uint32_t cpu_ldub_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - return cpu_ldub_mmuidx_ra(env, addr, mmu_index, ra); -} - -int cpu_ldsb_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra) -{ - return (int8_t)cpu_ldub_data_ra(env, addr, ra); -} - -uint32_t cpu_lduw_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - return cpu_lduw_be_mmuidx_ra(env, addr, mmu_index, ra); -} - -int cpu_ldsw_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra) -{ - return (int16_t)cpu_lduw_be_data_ra(env, addr, ra); -} - -uint32_t cpu_ldl_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - return cpu_ldl_be_mmuidx_ra(env, addr, mmu_index, ra); -} - -uint64_t cpu_ldq_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - return cpu_ldq_be_mmuidx_ra(env, addr, mmu_index, ra); -} - -uint32_t cpu_lduw_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - return cpu_lduw_le_mmuidx_ra(env, addr, mmu_index, ra); -} - -int cpu_ldsw_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra) -{ - return (int16_t)cpu_lduw_le_data_ra(env, addr, ra); -} - -uint32_t cpu_ldl_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - return cpu_ldl_le_mmuidx_ra(env, addr, mmu_index, ra); -} - -uint64_t cpu_ldq_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - return cpu_ldq_le_mmuidx_ra(env, addr, mmu_index, ra); -} - -void cpu_stb_data_ra(CPUArchState *env, abi_ptr addr, - uint32_t val, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - cpu_stb_mmuidx_ra(env, addr, val, mmu_index, ra); -} - -void cpu_stw_be_data_ra(CPUArchState *env, abi_ptr addr, - uint32_t val, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - cpu_stw_be_mmuidx_ra(env, addr, val, mmu_index, ra); -} - -void cpu_stl_be_data_ra(CPUArchState *env, abi_ptr addr, - uint32_t val, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - cpu_stl_be_mmuidx_ra(env, addr, val, mmu_index, ra); -} - -void cpu_stq_be_data_ra(CPUArchState *env, abi_ptr addr, - uint64_t val, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - cpu_stq_be_mmuidx_ra(env, addr, val, mmu_index, ra); -} - -void cpu_stw_le_data_ra(CPUArchState *env, abi_ptr addr, - uint32_t val, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - cpu_stw_le_mmuidx_ra(env, addr, val, mmu_index, ra); -} - -void cpu_stl_le_data_ra(CPUArchState *env, abi_ptr addr, - uint32_t val, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - cpu_stl_le_mmuidx_ra(env, addr, val, mmu_index, ra); -} - -void cpu_stq_le_data_ra(CPUArchState *env, abi_ptr addr, - uint64_t val, uintptr_t ra) -{ - int mmu_index = cpu_mmu_index(env_cpu(env), false); - cpu_stq_le_mmuidx_ra(env, addr, val, mmu_index, ra); -} - -/*--------------------------*/ - -uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr addr) -{ - return cpu_ldub_data_ra(env, addr, 0); -} - -int cpu_ldsb_data(CPUArchState *env, abi_ptr addr) -{ - return (int8_t)cpu_ldub_data(env, addr); -} - -uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr addr) -{ - return cpu_lduw_be_data_ra(env, addr, 0); -} - -int cpu_ldsw_be_data(CPUArchState *env, abi_ptr addr) -{ - return (int16_t)cpu_lduw_be_data(env, addr); -} - -uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr addr) -{ - return cpu_ldl_be_data_ra(env, addr, 0); -} - -uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr addr) -{ - return cpu_ldq_be_data_ra(env, addr, 0); -} - -uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr addr) -{ - return cpu_lduw_le_data_ra(env, addr, 0); -} - -int cpu_ldsw_le_data(CPUArchState *env, abi_ptr addr) -{ - return (int16_t)cpu_lduw_le_data(env, addr); -} - -uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr addr) -{ - return cpu_ldl_le_data_ra(env, addr, 0); -} - -uint64_t cpu_ldq_le_data(CPUArchState *env, abi_ptr addr) -{ - return cpu_ldq_le_data_ra(env, addr, 0); -} - -void cpu_stb_data(CPUArchState *env, abi_ptr addr, uint32_t val) -{ - cpu_stb_data_ra(env, addr, val, 0); -} - -void cpu_stw_be_data(CPUArchState *env, abi_ptr addr, uint32_t val) -{ - cpu_stw_be_data_ra(env, addr, val, 0); -} - -void cpu_stl_be_data(CPUArchState *env, abi_ptr addr, uint32_t val) -{ - cpu_stl_be_data_ra(env, addr, val, 0); -} - -void cpu_stq_be_data(CPUArchState *env, abi_ptr addr, uint64_t val) -{ - cpu_stq_be_data_ra(env, addr, val, 0); -} - -void cpu_stw_le_data(CPUArchState *env, abi_ptr addr, uint32_t val) -{ - cpu_stw_le_data_ra(env, addr, val, 0); -} - -void cpu_stl_le_data(CPUArchState *env, abi_ptr addr, uint32_t val) -{ - cpu_stl_le_data_ra(env, addr, val, 0); -} - -void cpu_stq_le_data(CPUArchState *env, abi_ptr addr, uint64_t val) -{ - cpu_stq_le_data_ra(env, addr, val, 0); -} diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build index 38ff227..047afa4 100644 --- a/accel/tcg/meson.build +++ b/accel/tcg/meson.build @@ -1,28 +1,41 @@ -common_ss.add(when: 'CONFIG_TCG', if_true: files( +if not get_option('tcg').allowed() + subdir_done() +endif + +tcg_ss = ss.source_set() + +tcg_ss.add(files( 'cpu-exec-common.c', 'tcg-runtime.c', 'tcg-runtime-gvec.c', + 'translator.c', )) +if get_option('plugins') + tcg_ss.add(files('plugin-gen.c')) +endif + +libuser_ss.add_all(tcg_ss) +libsystem_ss.add_all(tcg_ss) + tcg_specific_ss = ss.source_set() tcg_specific_ss.add(files( 'tcg-all.c', 'cpu-exec.c', 'tb-maint.c', 'translate-all.c', - 'translator.c', )) tcg_specific_ss.add(when: 'CONFIG_USER_ONLY', if_true: files('user-exec.c')) -tcg_specific_ss.add(when: 'CONFIG_SYSTEM_ONLY', if_false: files('user-exec-stub.c')) -if get_option('plugins') - tcg_specific_ss.add(files('plugin-gen.c')) -endif specific_ss.add_all(when: 'CONFIG_TCG', if_true: tcg_specific_ss) specific_ss.add(when: ['CONFIG_SYSTEM_ONLY', 'CONFIG_TCG'], if_true: files( 'cputlb.c', )) -system_ss.add(when: ['CONFIG_TCG'], if_true: files( +libuser_ss.add(files( + 'user-exec-stub.c', +)) + +libsystem_ss.add(files( 'icount-common.c', 'monitor.c', 'tcg-accel-ops.c', diff --git a/accel/tcg/monitor.c b/accel/tcg/monitor.c index eeb38a4..1c182b6 100644 --- a/accel/tcg/monitor.c +++ b/accel/tcg/monitor.c @@ -14,6 +14,7 @@ #include "qapi/qapi-commands-machine.h" #include "monitor/monitor.h" #include "system/cpu-timers.h" +#include "exec/icount.h" #include "system/tcg.h" #include "tcg/tcg.h" #include "internal-common.h" diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index 7e5f040..c1da753 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -22,13 +22,12 @@ #include "qemu/osdep.h" #include "qemu/plugin.h" #include "qemu/log.h" -#include "cpu.h" #include "tcg/tcg.h" #include "tcg/tcg-temp-internal.h" -#include "tcg/tcg-op.h" -#include "exec/exec-all.h" +#include "tcg/tcg-op-common.h" #include "exec/plugin-gen.h" #include "exec/translator.h" +#include "exec/translation-block.h" enum plugin_gen_from { PLUGIN_GEN_FROM_TB, @@ -89,15 +88,13 @@ static void gen_enable_mem_helper(struct qemu_plugin_tb *ptb, qemu_plugin_add_dyn_cb_arr(arr); tcg_gen_st_ptr(tcg_constant_ptr((intptr_t)arr), tcg_env, - offsetof(CPUState, neg.plugin_mem_cbs) - - offsetof(ArchCPU, env)); + offsetof(CPUState, neg.plugin_mem_cbs) - sizeof(CPUState)); } static void gen_disable_mem_helper(void) { tcg_gen_st_ptr(tcg_constant_ptr(0), tcg_env, - offsetof(CPUState, neg.plugin_mem_cbs) - - offsetof(ArchCPU, env)); + offsetof(CPUState, neg.plugin_mem_cbs) - sizeof(CPUState)); } static TCGv_i32 gen_cpu_index(void) @@ -113,7 +110,7 @@ static TCGv_i32 gen_cpu_index(void) } TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); tcg_gen_ld_i32(cpu_index, tcg_env, - -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index)); + offsetof(CPUState, cpu_index) - sizeof(CPUState)); return cpu_index; } diff --git a/accel/tcg/tb-hash.h b/accel/tcg/tb-hash.h index a5382f4..3bc5042 100644 --- a/accel/tcg/tb-hash.h +++ b/accel/tcg/tb-hash.h @@ -22,6 +22,7 @@ #include "exec/cpu-defs.h" #include "exec/exec-all.h" +#include "exec/target_page.h" #include "exec/translation-block.h" #include "qemu/xxhash.h" #include "tb-jmp-cache.h" diff --git a/accel/tcg/tb-internal.h b/accel/tcg/tb-internal.h index 68aa8d1..08538e2 100644 --- a/accel/tcg/tb-internal.h +++ b/accel/tcg/tb-internal.h @@ -9,8 +9,6 @@ #ifndef ACCEL_TCG_TB_INTERNAL_TARGET_H #define ACCEL_TCG_TB_INTERNAL_TARGET_H -#include "exec/cpu-all.h" -#include "exec/exec-all.h" #include "exec/translation-block.h" /* @@ -24,55 +22,23 @@ */ #define GETPC_ADJ 2 -#ifdef CONFIG_SOFTMMU - -#define CPU_TLB_DYN_MIN_BITS 6 -#define CPU_TLB_DYN_DEFAULT_BITS 8 - -# if HOST_LONG_BITS == 32 -/* Make sure we do not require a double-word shift for the TLB load */ -# define CPU_TLB_DYN_MAX_BITS (32 - TARGET_PAGE_BITS) -# else /* HOST_LONG_BITS == 64 */ -/* - * Assuming TARGET_PAGE_BITS==12, with 2**22 entries we can cover 2**(22+12) == - * 2**34 == 16G of address space. This is roughly what one would expect a - * TLB to cover in a modern (as of 2018) x86_64 CPU. For instance, Intel - * Skylake's Level-2 STLB has 16 1G entries. - * Also, make sure we do not size the TLB past the guest's address space. - */ -# ifdef TARGET_PAGE_BITS_VARY -# define CPU_TLB_DYN_MAX_BITS \ - MIN(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS) -# else -# define CPU_TLB_DYN_MAX_BITS \ - MIN_CONST(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS) -# endif -# endif - -#endif /* CONFIG_SOFTMMU */ +void tb_lock_page0(tb_page_addr_t); #ifdef CONFIG_USER_ONLY -#include "user/page-protection.h" /* * For user-only, page_protect sets the page read-only. * Since most execution is already on read-only pages, and we'd need to * account for other TBs on the same page, defer undoing any page protection * until we receive the write fault. */ -static inline void tb_lock_page0(tb_page_addr_t p0) -{ - page_protect(p0); -} - static inline void tb_lock_page1(tb_page_addr_t p0, tb_page_addr_t p1) { - page_protect(p1); + tb_lock_page0(p1); } static inline void tb_unlock_page1(tb_page_addr_t p0, tb_page_addr_t p1) { } static inline void tb_unlock_pages(TranslationBlock *tb) { } #else -void tb_lock_page0(tb_page_addr_t); void tb_lock_page1(tb_page_addr_t, tb_page_addr_t); void tb_unlock_page1(tb_page_addr_t, tb_page_addr_t); void tb_unlock_pages(TranslationBlock *); diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c index 3f1bebf..d479f53 100644 --- a/accel/tcg/tb-maint.c +++ b/accel/tcg/tb-maint.c @@ -20,11 +20,14 @@ #include "qemu/osdep.h" #include "qemu/interval-tree.h" #include "qemu/qtree.h" +#include "cpu.h" #include "exec/cputlb.h" #include "exec/log.h" #include "exec/exec-all.h" #include "exec/page-protection.h" +#include "exec/mmap-lock.h" #include "exec/tb-flush.h" +#include "exec/target_page.h" #include "tb-internal.h" #include "system/tcg.h" #include "tcg/tcg.h" diff --git a/accel/tcg/tcg-accel-ops-icount.c b/accel/tcg/tcg-accel-ops-icount.c index 27cf104..d0f7b41 100644 --- a/accel/tcg/tcg-accel-ops-icount.c +++ b/accel/tcg/tcg-accel-ops-icount.c @@ -25,7 +25,7 @@ #include "qemu/osdep.h" #include "system/replay.h" -#include "system/cpu-timers.h" +#include "exec/icount.h" #include "qemu/main-loop.h" #include "qemu/guest-random.h" #include "hw/core/cpu.h" diff --git a/accel/tcg/tcg-accel-ops-mttcg.c b/accel/tcg/tcg-accel-ops-mttcg.c index bdcc385..dfcee30 100644 --- a/accel/tcg/tcg-accel-ops-mttcg.c +++ b/accel/tcg/tcg-accel-ops-mttcg.c @@ -26,7 +26,7 @@ #include "qemu/osdep.h" #include "system/tcg.h" #include "system/replay.h" -#include "system/cpu-timers.h" +#include "exec/icount.h" #include "qemu/main-loop.h" #include "qemu/notify.h" #include "qemu/guest-random.h" diff --git a/accel/tcg/tcg-accel-ops-rr.c b/accel/tcg/tcg-accel-ops-rr.c index f62cf24..6eec5c9 100644 --- a/accel/tcg/tcg-accel-ops-rr.c +++ b/accel/tcg/tcg-accel-ops-rr.c @@ -27,7 +27,7 @@ #include "qemu/lockable.h" #include "system/tcg.h" #include "system/replay.h" -#include "system/cpu-timers.h" +#include "exec/icount.h" #include "qemu/main-loop.h" #include "qemu/notify.h" #include "qemu/guest-random.h" diff --git a/accel/tcg/tcg-accel-ops.c b/accel/tcg/tcg-accel-ops.c index d9b662e..b24d6a7 100644 --- a/accel/tcg/tcg-accel-ops.c +++ b/accel/tcg/tcg-accel-ops.c @@ -29,7 +29,7 @@ #include "system/accel-ops.h" #include "system/tcg.h" #include "system/replay.h" -#include "system/cpu-timers.h" +#include "exec/icount.h" #include "qemu/main-loop.h" #include "qemu/guest-random.h" #include "qemu/timer.h" @@ -37,6 +37,7 @@ #include "exec/hwaddr.h" #include "exec/tb-flush.h" #include "exec/translation-block.h" +#include "exec/watchpoint.h" #include "gdbstub/enums.h" #include "hw/core/cpu.h" @@ -223,7 +224,7 @@ static void tcg_accel_ops_init(AccelOpsClass *ops) ops->remove_all_breakpoints = tcg_remove_all_breakpoints; } -static void tcg_accel_ops_class_init(ObjectClass *oc, void *data) +static void tcg_accel_ops_class_init(ObjectClass *oc, const void *data) { AccelOpsClass *ops = ACCEL_OPS_CLASS(oc); diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c index c1a30b0..0ce34ac 100644 --- a/accel/tcg/tcg-all.c +++ b/accel/tcg/tcg-all.c @@ -26,19 +26,23 @@ #include "qemu/osdep.h" #include "system/tcg.h" #include "exec/replay-core.h" -#include "system/cpu-timers.h" +#include "exec/icount.h" #include "tcg/startup.h" #include "qapi/error.h" #include "qemu/error-report.h" #include "qemu/accel.h" #include "qemu/atomic.h" +#include "qapi/qapi-types-common.h" #include "qapi/qapi-builtin-visit.h" #include "qemu/units.h" +#include "qemu/target-info.h" #if defined(CONFIG_USER_ONLY) #include "hw/qdev-core.h" #else #include "hw/boards.h" +#include "system/tcg.h" #endif +#include "accel/tcg/cpu-ops.h" #include "internal-common.h" #include "cpu-param.h" @@ -46,7 +50,7 @@ struct TCGState { AccelState parent_obj; - bool mttcg_enabled; + OnOffAuto mttcg_enabled; bool one_insn_per_tb; int splitwx_enabled; unsigned long tb_size; @@ -58,40 +62,18 @@ typedef struct TCGState TCGState; DECLARE_INSTANCE_CHECKER(TCGState, TCG_STATE, TYPE_TCG_ACCEL) -/* - * We default to false if we know other options have been enabled - * which are currently incompatible with MTTCG. Otherwise when each - * guest (target) has been updated to support: - * - atomic instructions - * - memory ordering primitives (barriers) - * they can set the appropriate CONFIG flags in ${target}-softmmu.mak - * - * Once a guest architecture has been converted to the new primitives - * there is one remaining limitation to check: - * - The guest can't be oversized (e.g. 64 bit guest on 32 bit host) - */ - -static bool default_mttcg_enabled(void) +#ifndef CONFIG_USER_ONLY +bool qemu_tcg_mttcg_enabled(void) { - if (icount_enabled()) { - return false; - } -#ifdef TARGET_SUPPORTS_MTTCG -# ifndef TCG_GUEST_DEFAULT_MO -# error "TARGET_SUPPORTS_MTTCG without TCG_GUEST_DEFAULT_MO" -# endif - return true; -#else - return false; -#endif + TCGState *s = TCG_STATE(current_accel()); + return s->mttcg_enabled == ON_OFF_AUTO_ON; } +#endif /* !CONFIG_USER_ONLY */ static void tcg_accel_instance_init(Object *obj) { TCGState *s = TCG_STATE(obj); - s->mttcg_enabled = default_mttcg_enabled(); - /* If debugging enabled, default "auto on", otherwise off. */ #if defined(CONFIG_DEBUG_TCG) && !defined(CONFIG_USER_ONLY) s->splitwx_enabled = -1; @@ -100,24 +82,57 @@ static void tcg_accel_instance_init(Object *obj) #endif } -bool mttcg_enabled; bool one_insn_per_tb; static int tcg_init_machine(MachineState *ms) { TCGState *s = TCG_STATE(current_accel()); -#ifdef CONFIG_USER_ONLY - unsigned max_cpus = 1; -#else - unsigned max_cpus = ms->smp.max_cpus; + unsigned max_threads = 1; + +#ifndef CONFIG_USER_ONLY + CPUClass *cc = CPU_CLASS(object_class_by_name(target_cpu_type())); + bool mttcg_supported = cc->tcg_ops->mttcg_supported; + + switch (s->mttcg_enabled) { + case ON_OFF_AUTO_AUTO: + /* + * We default to false if we know other options have been enabled + * which are currently incompatible with MTTCG. Otherwise when each + * guest (target) has been updated to support: + * - atomic instructions + * - memory ordering primitives (barriers) + * they can set the appropriate CONFIG flags in ${target}-softmmu.mak + * + * Once a guest architecture has been converted to the new primitives + * there is one remaining limitation to check: + * - The guest can't be oversized (e.g. 64 bit guest on 32 bit host) + */ + if (mttcg_supported && !icount_enabled()) { + s->mttcg_enabled = ON_OFF_AUTO_ON; + max_threads = ms->smp.max_cpus; + } else { + s->mttcg_enabled = ON_OFF_AUTO_OFF; + } + break; + case ON_OFF_AUTO_ON: + if (!mttcg_supported) { + warn_report("Guest not yet converted to MTTCG - " + "you may get unexpected results"); + } + max_threads = ms->smp.max_cpus; + break; + case ON_OFF_AUTO_OFF: + break; + default: + g_assert_not_reached(); + } #endif tcg_allowed = true; - mttcg_enabled = s->mttcg_enabled; page_init(); tb_htable_init(); - tcg_init(s->tb_size * MiB, s->splitwx_enabled, max_cpus); + tcg_init(s->tb_size * MiB, s->splitwx_enabled, max_threads); #if defined(CONFIG_SOFTMMU) /* @@ -138,7 +153,7 @@ static char *tcg_get_thread(Object *obj, Error **errp) { TCGState *s = TCG_STATE(obj); - return g_strdup(s->mttcg_enabled ? "multi" : "single"); + return g_strdup(s->mttcg_enabled == ON_OFF_AUTO_ON ? "multi" : "single"); } static void tcg_set_thread(Object *obj, const char *value, Error **errp) @@ -149,14 +164,10 @@ static void tcg_set_thread(Object *obj, const char *value, Error **errp) if (icount_enabled()) { error_setg(errp, "No MTTCG when icount is enabled"); } else { -#ifndef TARGET_SUPPORTS_MTTCG - warn_report("Guest not yet converted to MTTCG - " - "you may get unexpected results"); -#endif - s->mttcg_enabled = true; + s->mttcg_enabled = ON_OFF_AUTO_ON; } } else if (strcmp(value, "single") == 0) { - s->mttcg_enabled = false; + s->mttcg_enabled = ON_OFF_AUTO_OFF; } else { error_setg(errp, "Invalid 'thread' setting %s", value); } @@ -227,7 +238,7 @@ static int tcg_gdbstub_supported_sstep_flags(void) } } -static void tcg_accel_class_init(ObjectClass *oc, void *data) +static void tcg_accel_class_init(ObjectClass *oc, const void *data) { AccelClass *ac = ACCEL_CLASS(oc); ac->name = "tcg"; diff --git a/accel/tcg/tlb-bounds.h b/accel/tcg/tlb-bounds.h new file mode 100644 index 0000000..efd34d4 --- /dev/null +++ b/accel/tcg/tlb-bounds.h @@ -0,0 +1,32 @@ +/* + * softmmu size bounds + * SPDX-License-Identifier: LGPL-2.1-or-later + */ + +#ifndef ACCEL_TCG_TLB_BOUNDS_H +#define ACCEL_TCG_TLB_BOUNDS_H + +#define CPU_TLB_DYN_MIN_BITS 6 +#define CPU_TLB_DYN_DEFAULT_BITS 8 + +# if HOST_LONG_BITS == 32 +/* Make sure we do not require a double-word shift for the TLB load */ +# define CPU_TLB_DYN_MAX_BITS (32 - TARGET_PAGE_BITS) +# else /* HOST_LONG_BITS == 64 */ +/* + * Assuming TARGET_PAGE_BITS==12, with 2**22 entries we can cover 2**(22+12) == + * 2**34 == 16G of address space. This is roughly what one would expect a + * TLB to cover in a modern (as of 2018) x86_64 CPU. For instance, Intel + * Skylake's Level-2 STLB has 16 1G entries. + * Also, make sure we do not size the TLB past the guest's address space. + */ +# ifdef TARGET_PAGE_BITS_VARY +# define CPU_TLB_DYN_MAX_BITS \ + MIN(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS) +# else +# define CPU_TLB_DYN_MAX_BITS \ + MIN_CONST(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS) +# endif +# endif + +#endif /* ACCEL_TCG_TLB_BOUNDS_H */ diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index 82bc16b..c007b9a 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -40,12 +40,15 @@ #endif #endif #else -#include "exec/ram_addr.h" +#include "system/ram_addr.h" #endif +#include "cpu-param.h" #include "exec/cputlb.h" #include "exec/page-protection.h" +#include "exec/mmap-lock.h" #include "tb-internal.h" +#include "tlb-bounds.h" #include "exec/translator.h" #include "exec/tb-flush.h" #include "qemu/bitmap.h" @@ -54,7 +57,7 @@ #include "qemu/cacheinfo.h" #include "qemu/timer.h" #include "exec/log.h" -#include "system/cpu-timers.h" +#include "exec/icount.h" #include "system/tcg.h" #include "qapi/error.h" #include "accel/tcg/cpu-ops.h" @@ -66,6 +69,7 @@ #include "internal-target.h" #include "tcg/perf.h" #include "tcg/insn-start-words.h" +#include "cpu.h" TBContext tb_ctx; @@ -349,11 +353,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, tcg_ctx->tlb_dyn_max_bits = CPU_TLB_DYN_MAX_BITS; #endif tcg_ctx->insn_start_words = TARGET_INSN_START_WORDS; -#ifdef TCG_GUEST_DEFAULT_MO - tcg_ctx->guest_mo = TCG_GUEST_DEFAULT_MO; -#else - tcg_ctx->guest_mo = TCG_MO_ALL; -#endif + tcg_ctx->guest_mo = cpu->cc->tcg_ops->guest_default_memory_order; restart_translate: trace_translate_block(tb, pc, tb->tc.ptr); diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index ef1538b..034f2f3 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -8,16 +8,16 @@ */ #include "qemu/osdep.h" +#include "qemu/bswap.h" #include "qemu/log.h" #include "qemu/error-report.h" -#include "exec/exec-all.h" +#include "accel/tcg/cpu-ldst-common.h" +#include "accel/tcg/cpu-mmu-index.h" +#include "exec/target_page.h" #include "exec/translator.h" -#include "exec/cpu_ldst.h" #include "exec/plugin-gen.h" -#include "exec/cpu_ldst.h" -#include "exec/tswap.h" #include "tcg/tcg-op-common.h" -#include "internal-target.h" +#include "internal-common.h" #include "disas/disas.h" #include "tb-internal.h" @@ -25,8 +25,7 @@ static void set_can_do_io(DisasContextBase *db, bool val) { QEMU_BUILD_BUG_ON(sizeof_field(CPUState, neg.can_do_io) != 1); tcg_gen_st8_i32(tcg_constant_i32(val), tcg_env, - offsetof(ArchCPU, parent_obj.neg.can_do_io) - - offsetof(ArchCPU, env)); + offsetof(CPUState, neg.can_do_io) - sizeof(CPUState)); } bool translator_io_start(DisasContextBase *db) @@ -49,8 +48,8 @@ static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags) if ((cflags & CF_USE_ICOUNT) || !(cflags & CF_NOIRQ)) { count = tcg_temp_new_i32(); tcg_gen_ld_i32(count, tcg_env, - offsetof(ArchCPU, parent_obj.neg.icount_decr.u32) - - offsetof(ArchCPU, env)); + offsetof(CPUState, neg.icount_decr.u32) - + sizeof(CPUState)); } if (cflags & CF_USE_ICOUNT) { @@ -79,8 +78,8 @@ static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags) if (cflags & CF_USE_ICOUNT) { tcg_gen_st16_i32(count, tcg_env, - offsetof(ArchCPU, parent_obj.neg.icount_decr.u16.low) - - offsetof(ArchCPU, env)); + offsetof(CPUState, neg.icount_decr.u16.low) - + sizeof(CPUState)); } return icount_start_insn; @@ -142,6 +141,7 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns, db->host_addr[1] = NULL; db->record_start = 0; db->record_len = 0; + db->code_mmuidx = cpu_mmu_index(cpu, true); ops->init_disas_context(db, cpu); tcg_debug_assert(db->is_jmp == DISAS_NEXT); /* no early exit */ @@ -265,12 +265,14 @@ static bool translator_ld(CPUArchState *env, DisasContextBase *db, if (likely(((base ^ last) & TARGET_PAGE_MASK) == 0)) { /* Entire read is from the first page. */ - memcpy(dest, host + (pc - base), len); - return true; + goto do_read; } if (unlikely(((base ^ pc) & TARGET_PAGE_MASK) == 0)) { - /* Read begins on the first page and extends to the second. */ + /* + * Read begins on the first page and extends to the second. + * The unaligned read is never atomic. + */ size_t len0 = -(pc | TARGET_PAGE_MASK); memcpy(dest, host + (pc - base), len0); pc += len0; @@ -329,7 +331,39 @@ static bool translator_ld(CPUArchState *env, DisasContextBase *db, host = db->host_addr[1]; } - memcpy(dest, host + (pc - base), len); + do_read: + /* + * Assume aligned reads should be atomic, if possible. + * We're not in a position to jump out with EXCP_ATOMIC. + */ + host += pc - base; + switch (len) { + case 2: + if (QEMU_IS_ALIGNED(pc, 2)) { + uint16_t t = qatomic_read((uint16_t *)host); + stw_he_p(dest, t); + return true; + } + break; + case 4: + if (QEMU_IS_ALIGNED(pc, 4)) { + uint32_t t = qatomic_read((uint32_t *)host); + stl_he_p(dest, t); + return true; + } + break; +#ifdef CONFIG_ATOMIC64 + case 8: + if (QEMU_IS_ALIGNED(pc, 8)) { + uint64_t t = qatomic_read__nocheck((uint64_t *)host); + stq_he_p(dest, t); + return true; + } + break; +#endif + } + /* Unaligned or partial read from the second page is not atomic. */ + memcpy(dest, host, len); return true; } @@ -423,55 +457,62 @@ bool translator_st(const DisasContextBase *db, void *dest, uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, vaddr pc) { - uint8_t raw; + uint8_t val; - if (!translator_ld(env, db, &raw, pc, sizeof(raw))) { - raw = cpu_ldub_code(env, pc); - record_save(db, pc, &raw, sizeof(raw)); + if (!translator_ld(env, db, &val, pc, sizeof(val))) { + MemOpIdx oi = make_memop_idx(MO_UB, db->code_mmuidx); + val = cpu_ldb_code_mmu(env, pc, oi, 0); + record_save(db, pc, &val, sizeof(val)); } - return raw; + return val; } -uint16_t translator_lduw(CPUArchState *env, DisasContextBase *db, vaddr pc) +uint16_t translator_lduw_end(CPUArchState *env, DisasContextBase *db, + vaddr pc, MemOp endian) { - uint16_t raw, tgt; + uint16_t val; - if (translator_ld(env, db, &raw, pc, sizeof(raw))) { - tgt = tswap16(raw); - } else { - tgt = cpu_lduw_code(env, pc); - raw = tswap16(tgt); - record_save(db, pc, &raw, sizeof(raw)); + if (!translator_ld(env, db, &val, pc, sizeof(val))) { + MemOpIdx oi = make_memop_idx(MO_UW, db->code_mmuidx); + val = cpu_ldw_code_mmu(env, pc, oi, 0); + record_save(db, pc, &val, sizeof(val)); + } + if (endian & MO_BSWAP) { + val = bswap16(val); } - return tgt; + return val; } -uint32_t translator_ldl(CPUArchState *env, DisasContextBase *db, vaddr pc) +uint32_t translator_ldl_end(CPUArchState *env, DisasContextBase *db, + vaddr pc, MemOp endian) { - uint32_t raw, tgt; + uint32_t val; - if (translator_ld(env, db, &raw, pc, sizeof(raw))) { - tgt = tswap32(raw); - } else { - tgt = cpu_ldl_code(env, pc); - raw = tswap32(tgt); - record_save(db, pc, &raw, sizeof(raw)); + if (!translator_ld(env, db, &val, pc, sizeof(val))) { + MemOpIdx oi = make_memop_idx(MO_UL, db->code_mmuidx); + val = cpu_ldl_code_mmu(env, pc, oi, 0); + record_save(db, pc, &val, sizeof(val)); + } + if (endian & MO_BSWAP) { + val = bswap32(val); } - return tgt; + return val; } -uint64_t translator_ldq(CPUArchState *env, DisasContextBase *db, vaddr pc) +uint64_t translator_ldq_end(CPUArchState *env, DisasContextBase *db, + vaddr pc, MemOp endian) { - uint64_t raw, tgt; + uint64_t val; - if (translator_ld(env, db, &raw, pc, sizeof(raw))) { - tgt = tswap64(raw); - } else { - tgt = cpu_ldq_code(env, pc); - raw = tswap64(tgt); - record_save(db, pc, &raw, sizeof(raw)); + if (!translator_ld(env, db, &val, pc, sizeof(val))) { + MemOpIdx oi = make_memop_idx(MO_UQ, db->code_mmuidx); + val = cpu_ldq_code_mmu(env, pc, oi, 0); + record_save(db, pc, &val, sizeof(val)); + } + if (endian & MO_BSWAP) { + val = bswap64(val); } - return tgt; + return val; } void translator_fake_ld(DisasContextBase *db, const void *data, size_t len) diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 2322181..5eef8e7 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -19,12 +19,14 @@ #include "qemu/osdep.h" #include "accel/tcg/cpu-ops.h" #include "disas/disas.h" +#include "cpu.h" #include "exec/vaddr.h" #include "exec/exec-all.h" +#include "exec/tlb-flags.h" #include "tcg/tcg.h" #include "qemu/bitops.h" #include "qemu/rcu.h" -#include "exec/cpu_ldst.h" +#include "accel/tcg/cpu-ldst.h" #include "user/cpu_loop.h" #include "qemu/main-loop.h" #include "user/page-protection.h" @@ -35,6 +37,7 @@ #include "qemu/int128.h" #include "trace.h" #include "tcg/tcg-ldst.h" +#include "backend-ldst.h" #include "internal-common.h" #include "internal-target.h" #include "tb-internal.h" @@ -656,7 +659,7 @@ target_ulong page_find_range_empty(target_ulong min, target_ulong max, } } -void page_protect(tb_page_addr_t address) +void tb_lock_page0(tb_page_addr_t address) { PageFlagsNode *p; target_ulong start, last; @@ -1059,7 +1062,7 @@ static uint8_t do_ld1_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi, void *haddr; uint8_t ret; - cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD); haddr = cpu_mmu_lookup(cpu, addr, get_memop(oi), ra, access_type); ret = ldub_p(haddr); clear_helper_retaddr(); @@ -1073,7 +1076,7 @@ static uint16_t do_ld2_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi, uint16_t ret; MemOp mop = get_memop(oi); - cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD); haddr = cpu_mmu_lookup(cpu, addr, mop, ra, access_type); ret = load_atom_2(cpu, ra, haddr, mop); clear_helper_retaddr(); @@ -1091,7 +1094,7 @@ static uint32_t do_ld4_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi, uint32_t ret; MemOp mop = get_memop(oi); - cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD); haddr = cpu_mmu_lookup(cpu, addr, mop, ra, access_type); ret = load_atom_4(cpu, ra, haddr, mop); clear_helper_retaddr(); @@ -1109,7 +1112,7 @@ static uint64_t do_ld8_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi, uint64_t ret; MemOp mop = get_memop(oi); - cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD); haddr = cpu_mmu_lookup(cpu, addr, mop, ra, access_type); ret = load_atom_8(cpu, ra, haddr, mop); clear_helper_retaddr(); @@ -1128,7 +1131,7 @@ static Int128 do_ld16_mmu(CPUState *cpu, abi_ptr addr, MemOp mop = get_memop(oi); tcg_debug_assert((mop & MO_SIZE) == MO_128); - cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); + cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD); haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_LOAD); ret = load_atom_16(cpu, ra, haddr, mop); clear_helper_retaddr(); @@ -1144,7 +1147,7 @@ static void do_st1_mmu(CPUState *cpu, vaddr addr, uint8_t val, { void *haddr; - cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST); haddr = cpu_mmu_lookup(cpu, addr, get_memop(oi), ra, MMU_DATA_STORE); stb_p(haddr, val); clear_helper_retaddr(); @@ -1156,7 +1159,7 @@ static void do_st2_mmu(CPUState *cpu, vaddr addr, uint16_t val, void *haddr; MemOp mop = get_memop(oi); - cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST); haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE); if (mop & MO_BSWAP) { @@ -1172,7 +1175,7 @@ static void do_st4_mmu(CPUState *cpu, vaddr addr, uint32_t val, void *haddr; MemOp mop = get_memop(oi); - cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST); haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE); if (mop & MO_BSWAP) { @@ -1188,7 +1191,7 @@ static void do_st8_mmu(CPUState *cpu, vaddr addr, uint64_t val, void *haddr; MemOp mop = get_memop(oi); - cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST); haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE); if (mop & MO_BSWAP) { @@ -1204,7 +1207,7 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val, void *haddr; MemOpIdx mop = get_memop(oi); - cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); + cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST); haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE); if (mop & MO_BSWAP) { @@ -1214,101 +1217,28 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val, clear_helper_retaddr(); } -uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr ptr) -{ - uint32_t ret; - - set_helper_retaddr(1); - ret = ldub_p(g2h_untagged(ptr)); - clear_helper_retaddr(); - return ret; -} - -uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr ptr) -{ - uint32_t ret; - - set_helper_retaddr(1); - ret = lduw_p(g2h_untagged(ptr)); - clear_helper_retaddr(); - return ret; -} - -uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr ptr) -{ - uint32_t ret; - - set_helper_retaddr(1); - ret = ldl_p(g2h_untagged(ptr)); - clear_helper_retaddr(); - return ret; -} - -uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr ptr) -{ - uint64_t ret; - - set_helper_retaddr(1); - ret = ldq_p(g2h_untagged(ptr)); - clear_helper_retaddr(); - return ret; -} - -uint8_t cpu_ldb_code_mmu(CPUArchState *env, abi_ptr addr, +uint8_t cpu_ldb_code_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra) { - void *haddr; - uint8_t ret; - - haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_INST_FETCH); - ret = ldub_p(haddr); - clear_helper_retaddr(); - return ret; + return do_ld1_mmu(env_cpu(env), addr, oi, ra ? ra : 1, MMU_INST_FETCH); } -uint16_t cpu_ldw_code_mmu(CPUArchState *env, abi_ptr addr, +uint16_t cpu_ldw_code_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra) { - void *haddr; - uint16_t ret; - - haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_INST_FETCH); - ret = lduw_p(haddr); - clear_helper_retaddr(); - if (get_memop(oi) & MO_BSWAP) { - ret = bswap16(ret); - } - return ret; + return do_ld2_mmu(env_cpu(env), addr, oi, ra ? ra : 1, MMU_INST_FETCH); } -uint32_t cpu_ldl_code_mmu(CPUArchState *env, abi_ptr addr, +uint32_t cpu_ldl_code_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra) { - void *haddr; - uint32_t ret; - - haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_INST_FETCH); - ret = ldl_p(haddr); - clear_helper_retaddr(); - if (get_memop(oi) & MO_BSWAP) { - ret = bswap32(ret); - } - return ret; + return do_ld4_mmu(env_cpu(env), addr, oi, ra ? ra : 1, MMU_INST_FETCH); } -uint64_t cpu_ldq_code_mmu(CPUArchState *env, abi_ptr addr, +uint64_t cpu_ldq_code_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra) { - void *haddr; - uint64_t ret; - - haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_DATA_LOAD); - ret = ldq_p(haddr); - clear_helper_retaddr(); - if (get_memop(oi) & MO_BSWAP) { - ret = bswap64(ret); - } - return ret; + return do_ld8_mmu(env_cpu(env), addr, oi, ra ? ra : 1, MMU_INST_FETCH); } #include "ldst_common.c.inc" diff --git a/accel/tcg/watchpoint.c b/accel/tcg/watchpoint.c index 65b2188..cfb37a4 100644 --- a/accel/tcg/watchpoint.c +++ b/accel/tcg/watchpoint.c @@ -124,17 +124,14 @@ void cpu_check_watchpoint(CPUState *cpu, vaddr addr, vaddr len, } cpu->watchpoint_hit = wp; - mmap_lock(); /* This call also restores vCPU state */ tb_check_watchpoint(cpu, ra); if (wp->flags & BP_STOP_BEFORE_ACCESS) { cpu->exception_index = EXCP_DEBUG; - mmap_unlock(); cpu_loop_exit(cpu); } else { /* Force execution of one insn next time. */ cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu); - mmap_unlock(); cpu_loop_exit_noexc(cpu); } } else { diff --git a/accel/xen/xen-all.c b/accel/xen/xen-all.c index 7aa28b9..de52a8f 100644 --- a/accel/xen/xen-all.c +++ b/accel/xen/xen-all.c @@ -116,7 +116,7 @@ static int xen_init(MachineState *ms) return 0; } -static void xen_accel_class_init(ObjectClass *oc, void *data) +static void xen_accel_class_init(ObjectClass *oc, const void *data) { AccelClass *ac = ACCEL_CLASS(oc); static GlobalProperty compat[] = { @@ -147,7 +147,7 @@ static const TypeInfo xen_accel_type = { .class_init = xen_accel_class_init, }; -static void xen_accel_ops_class_init(ObjectClass *oc, void *data) +static void xen_accel_ops_class_init(ObjectClass *oc, const void *data) { AccelOpsClass *ops = ACCEL_OPS_CLASS(oc); |