aboutsummaryrefslogtreecommitdiff
path: root/accel/tcg
diff options
context:
space:
mode:
Diffstat (limited to 'accel/tcg')
-rw-r--r--accel/tcg/atomic_template.h16
-rw-r--r--accel/tcg/backend-ldst.h41
-rw-r--r--accel/tcg/cpu-exec.c8
-rw-r--r--accel/tcg/cputlb.c98
-rw-r--r--accel/tcg/icount-common.c2
-rw-r--r--accel/tcg/internal-common.h34
-rw-r--r--accel/tcg/internal-target.h37
-rw-r--r--accel/tcg/ldst_common.c.inc335
-rw-r--r--accel/tcg/meson.build27
-rw-r--r--accel/tcg/monitor.c1
-rw-r--r--accel/tcg/plugin-gen.c13
-rw-r--r--accel/tcg/tb-hash.h1
-rw-r--r--accel/tcg/tb-internal.h38
-rw-r--r--accel/tcg/tb-maint.c3
-rw-r--r--accel/tcg/tcg-accel-ops-icount.c2
-rw-r--r--accel/tcg/tcg-accel-ops-mttcg.c2
-rw-r--r--accel/tcg/tcg-accel-ops-rr.c2
-rw-r--r--accel/tcg/tcg-accel-ops.c3
-rw-r--r--accel/tcg/tcg-all.c98
-rw-r--r--accel/tcg/tlb-bounds.h32
-rw-r--r--accel/tcg/translate-all.c14
-rw-r--r--accel/tcg/translator.c135
-rw-r--r--accel/tcg/user-exec.c116
-rw-r--r--accel/tcg/watchpoint.c3
24 files changed, 380 insertions, 681 deletions
diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h
index 89593b2..08a475c 100644
--- a/accel/tcg/atomic_template.h
+++ b/accel/tcg/atomic_template.h
@@ -77,7 +77,7 @@
# define END _le
#endif
-ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
+ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, vaddr addr,
ABI_TYPE cmpv, ABI_TYPE newv,
MemOpIdx oi, uintptr_t retaddr)
{
@@ -101,7 +101,7 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
}
#if DATA_SIZE < 16
-ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
+ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, vaddr addr, ABI_TYPE val,
MemOpIdx oi, uintptr_t retaddr)
{
DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
@@ -120,7 +120,7 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
}
#define GEN_ATOMIC_HELPER(X) \
-ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
+ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr, \
ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
{ \
DATA_TYPE *haddr, ret; \
@@ -156,7 +156,7 @@ GEN_ATOMIC_HELPER(xor_fetch)
* of CF_PARALLEL's value, we'll trace just a read and a write.
*/
#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
-ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
+ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr, \
ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
{ \
XDATA_TYPE *haddr, cmp, old, new, val = xval; \
@@ -202,7 +202,7 @@ GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
# define END _be
#endif
-ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
+ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, vaddr addr,
ABI_TYPE cmpv, ABI_TYPE newv,
MemOpIdx oi, uintptr_t retaddr)
{
@@ -226,7 +226,7 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
}
#if DATA_SIZE < 16
-ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
+ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, vaddr addr, ABI_TYPE val,
MemOpIdx oi, uintptr_t retaddr)
{
DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
@@ -245,7 +245,7 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
}
#define GEN_ATOMIC_HELPER(X) \
-ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
+ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr, \
ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
{ \
DATA_TYPE *haddr, ret; \
@@ -278,7 +278,7 @@ GEN_ATOMIC_HELPER(xor_fetch)
* of CF_PARALLEL's value, we'll trace just a read and a write.
*/
#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
-ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr, \
+ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr, \
ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
{ \
XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval; \
diff --git a/accel/tcg/backend-ldst.h b/accel/tcg/backend-ldst.h
new file mode 100644
index 0000000..9c3a407
--- /dev/null
+++ b/accel/tcg/backend-ldst.h
@@ -0,0 +1,41 @@
+/*
+ * Internal memory barrier helpers for QEMU (target agnostic)
+ *
+ * Copyright (c) 2003 Fabrice Bellard
+ *
+ * SPDX-License-Identifier: LGPL-2.1-or-later
+ */
+
+#ifndef ACCEL_TCG_BACKEND_LDST_H
+#define ACCEL_TCG_BACKEND_LDST_H
+
+#include "tcg-target-mo.h"
+
+/**
+ * tcg_req_mo:
+ * @guest_mo: Guest default memory order
+ * @type: TCGBar
+ *
+ * Filter @type to the barrier that is required for the guest
+ * memory ordering vs the host memory ordering. A non-zero
+ * result indicates that some barrier is required.
+ */
+#define tcg_req_mo(guest_mo, type) \
+ ((type) & guest_mo & ~TCG_TARGET_DEFAULT_MO)
+
+/**
+ * cpu_req_mo:
+ * @cpu: CPUState
+ * @type: TCGBar
+ *
+ * If tcg_req_mo indicates a barrier for @type is required
+ * for the guest memory model, issue a host memory barrier.
+ */
+#define cpu_req_mo(cpu, type) \
+ do { \
+ if (tcg_req_mo(cpu->cc->tcg_ops->guest_default_memory_order, type)) { \
+ smp_mb(); \
+ } \
+ } while (0)
+
+#endif
diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
index 8e28136..87eba83 100644
--- a/accel/tcg/cpu-exec.c
+++ b/accel/tcg/cpu-exec.c
@@ -22,19 +22,22 @@
#include "qapi/error.h"
#include "qapi/type-helpers.h"
#include "hw/core/cpu.h"
+#include "accel/tcg/cpu-ldst.h"
#include "accel/tcg/cpu-ops.h"
#include "trace.h"
#include "disas/disas.h"
#include "exec/cpu-common.h"
+#include "exec/cpu-interrupt.h"
#include "exec/page-protection.h"
+#include "exec/mmap-lock.h"
#include "exec/translation-block.h"
#include "tcg/tcg.h"
#include "qemu/atomic.h"
#include "qemu/rcu.h"
#include "exec/log.h"
#include "qemu/main-loop.h"
-#include "exec/cpu-all.h"
-#include "system/cpu-timers.h"
+#include "cpu.h"
+#include "exec/icount.h"
#include "exec/replay-core.h"
#include "system/tcg.h"
#include "exec/helper-proto-common.h"
@@ -1073,6 +1076,7 @@ bool tcg_exec_realizefn(CPUState *cpu, Error **errp)
assert(tcg_ops->cpu_exec_interrupt);
#endif /* !CONFIG_USER_ONLY */
assert(tcg_ops->translate_code);
+ assert(tcg_ops->mmu_index);
tcg_ops->initialize();
tcg_target_initialized = true;
}
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index fb22048..d9fb68d 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -22,12 +22,11 @@
#include "accel/tcg/cpu-ops.h"
#include "exec/exec-all.h"
#include "exec/page-protection.h"
-#include "exec/memory.h"
-#include "exec/cpu_ldst.h"
+#include "system/memory.h"
+#include "accel/tcg/cpu-ldst.h"
#include "exec/cputlb.h"
#include "exec/tb-flush.h"
-#include "exec/memory-internal.h"
-#include "exec/ram_addr.h"
+#include "system/ram_addr.h"
#include "exec/mmu-access-type.h"
#include "exec/tlb-common.h"
#include "exec/vaddr.h"
@@ -35,18 +34,22 @@
#include "qemu/error-report.h"
#include "exec/log.h"
#include "exec/helper-proto-common.h"
+#include "exec/tlb-flags.h"
#include "qemu/atomic.h"
#include "qemu/atomic128.h"
#include "tb-internal.h"
#include "trace.h"
#include "tb-hash.h"
#include "tb-internal.h"
+#include "tlb-bounds.h"
#include "internal-common.h"
#include "internal-target.h"
#ifdef CONFIG_PLUGIN
#include "qemu/plugin-memory.h"
#endif
#include "tcg/tcg-ldst.h"
+#include "backend-ldst.h"
+
/* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
/* #define DEBUG_TLB */
@@ -882,18 +885,17 @@ void tlb_unprotect_code(ram_addr_t ram_addr)
*
* Called with tlb_c.lock held.
*/
-static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry,
+static void tlb_reset_dirty_range_locked(CPUTLBEntryFull *full, CPUTLBEntry *ent,
uintptr_t start, uintptr_t length)
{
- uintptr_t addr = tlb_entry->addr_write;
+ const uintptr_t addr = ent->addr_write;
+ int flags = addr | full->slow_flags[MMU_DATA_STORE];
- if ((addr & (TLB_INVALID_MASK | TLB_MMIO |
- TLB_DISCARD_WRITE | TLB_NOTDIRTY)) == 0) {
- addr &= TARGET_PAGE_MASK;
- addr += tlb_entry->addend;
- if ((addr - start) < length) {
- qatomic_set(&tlb_entry->addr_write,
- tlb_entry->addr_write | TLB_NOTDIRTY);
+ flags &= TLB_INVALID_MASK | TLB_MMIO | TLB_DISCARD_WRITE | TLB_NOTDIRTY;
+ if (flags == 0) {
+ uintptr_t host = (addr & TARGET_PAGE_MASK) + ent->addend;
+ if ((host - start) < length) {
+ qatomic_set(&ent->addr_write, addr | TLB_NOTDIRTY);
}
}
}
@@ -912,23 +914,25 @@ static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s)
* We must take tlb_c.lock to avoid racing with another vCPU update. The only
* thing actually updated is the target TLB entry ->addr_write flags.
*/
-void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
+void tlb_reset_dirty(CPUState *cpu, uintptr_t start, uintptr_t length)
{
int mmu_idx;
qemu_spin_lock(&cpu->neg.tlb.c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+ CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx];
+ CPUTLBDescFast *fast = &cpu->neg.tlb.f[mmu_idx];
+ unsigned int n = tlb_n_entries(fast);
unsigned int i;
- unsigned int n = tlb_n_entries(&cpu->neg.tlb.f[mmu_idx]);
for (i = 0; i < n; i++) {
- tlb_reset_dirty_range_locked(&cpu->neg.tlb.f[mmu_idx].table[i],
- start1, length);
+ tlb_reset_dirty_range_locked(&desc->fulltlb[i], &fast->table[i],
+ start, length);
}
for (i = 0; i < CPU_VTLB_SIZE; i++) {
- tlb_reset_dirty_range_locked(&cpu->neg.tlb.d[mmu_idx].vtable[i],
- start1, length);
+ tlb_reset_dirty_range_locked(&desc->vfulltlb[i], &desc->vtable[i],
+ start, length);
}
}
qemu_spin_unlock(&cpu->neg.tlb.c.lock);
@@ -2321,7 +2325,7 @@ static uint8_t do_ld1_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
MMULookupLocals l;
bool crosspage;
- cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
tcg_debug_assert(!crosspage);
@@ -2336,7 +2340,7 @@ static uint16_t do_ld2_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
uint16_t ret;
uint8_t a, b;
- cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
if (likely(!crosspage)) {
return do_ld_2(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
@@ -2360,7 +2364,7 @@ static uint32_t do_ld4_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
bool crosspage;
uint32_t ret;
- cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
if (likely(!crosspage)) {
return do_ld_4(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
@@ -2381,7 +2385,7 @@ static uint64_t do_ld8_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
bool crosspage;
uint64_t ret;
- cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
if (likely(!crosspage)) {
return do_ld_8(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
@@ -2404,7 +2408,7 @@ static Int128 do_ld16_mmu(CPUState *cpu, vaddr addr,
Int128 ret;
int first;
- cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_LOAD, &l);
if (likely(!crosspage)) {
if (unlikely(l.page[0].flags & TLB_MMIO)) {
@@ -2732,7 +2736,7 @@ static void do_st1_mmu(CPUState *cpu, vaddr addr, uint8_t val,
MMULookupLocals l;
bool crosspage;
- cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
tcg_debug_assert(!crosspage);
@@ -2746,7 +2750,7 @@ static void do_st2_mmu(CPUState *cpu, vaddr addr, uint16_t val,
bool crosspage;
uint8_t a, b;
- cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
do_st_2(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
@@ -2768,7 +2772,7 @@ static void do_st4_mmu(CPUState *cpu, vaddr addr, uint32_t val,
MMULookupLocals l;
bool crosspage;
- cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
do_st_4(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
@@ -2789,7 +2793,7 @@ static void do_st8_mmu(CPUState *cpu, vaddr addr, uint64_t val,
MMULookupLocals l;
bool crosspage;
- cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
do_st_8(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
@@ -2812,7 +2816,7 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
uint64_t a, b;
int first;
- cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
if (unlikely(l.page[0].flags & TLB_MMIO)) {
@@ -2897,53 +2901,25 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
/* Code access functions. */
-uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr addr)
-{
- CPUState *cs = env_cpu(env);
- MemOpIdx oi = make_memop_idx(MO_UB, cpu_mmu_index(cs, true));
- return do_ld1_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
-}
-
-uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr addr)
-{
- CPUState *cs = env_cpu(env);
- MemOpIdx oi = make_memop_idx(MO_TEUW, cpu_mmu_index(cs, true));
- return do_ld2_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
-}
-
-uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr addr)
-{
- CPUState *cs = env_cpu(env);
- MemOpIdx oi = make_memop_idx(MO_TEUL, cpu_mmu_index(cs, true));
- return do_ld4_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
-}
-
-uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr addr)
-{
- CPUState *cs = env_cpu(env);
- MemOpIdx oi = make_memop_idx(MO_TEUQ, cpu_mmu_index(cs, true));
- return do_ld8_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
-}
-
-uint8_t cpu_ldb_code_mmu(CPUArchState *env, abi_ptr addr,
+uint8_t cpu_ldb_code_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t retaddr)
{
return do_ld1_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
}
-uint16_t cpu_ldw_code_mmu(CPUArchState *env, abi_ptr addr,
+uint16_t cpu_ldw_code_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t retaddr)
{
return do_ld2_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
}
-uint32_t cpu_ldl_code_mmu(CPUArchState *env, abi_ptr addr,
+uint32_t cpu_ldl_code_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t retaddr)
{
return do_ld4_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
}
-uint64_t cpu_ldq_code_mmu(CPUArchState *env, abi_ptr addr,
+uint64_t cpu_ldq_code_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t retaddr)
{
return do_ld8_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
diff --git a/accel/tcg/icount-common.c b/accel/tcg/icount-common.c
index 402d3e3..d647117 100644
--- a/accel/tcg/icount-common.c
+++ b/accel/tcg/icount-common.c
@@ -35,7 +35,7 @@
#include "system/replay.h"
#include "system/runstate.h"
#include "hw/core/cpu.h"
-#include "system/cpu-timers.h"
+#include "exec/icount.h"
#include "system/cpu-timers-internal.h"
/*
diff --git a/accel/tcg/internal-common.h b/accel/tcg/internal-common.h
index 9b6ab3a..2f00560 100644
--- a/accel/tcg/internal-common.h
+++ b/accel/tcg/internal-common.h
@@ -74,4 +74,38 @@ uint32_t curr_cflags(CPUState *cpu);
void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr);
+/**
+ * get_page_addr_code_hostp()
+ * @env: CPUArchState
+ * @addr: guest virtual address of guest code
+ *
+ * See get_page_addr_code() (full-system version) for documentation on the
+ * return value.
+ *
+ * Sets *@hostp (when @hostp is non-NULL) as follows.
+ * If the return value is -1, sets *@hostp to NULL. Otherwise, sets *@hostp
+ * to the host address where @addr's content is kept.
+ *
+ * Note: this function can trigger an exception.
+ */
+tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, vaddr addr,
+ void **hostp);
+
+/**
+ * get_page_addr_code()
+ * @env: CPUArchState
+ * @addr: guest virtual address of guest code
+ *
+ * If we cannot translate and execute from the entire RAM page, or if
+ * the region is not backed by RAM, returns -1. Otherwise, returns the
+ * ram_addr_t corresponding to the guest code at @addr.
+ *
+ * Note: this function can trigger an exception.
+ */
+static inline tb_page_addr_t get_page_addr_code(CPUArchState *env,
+ vaddr addr)
+{
+ return get_page_addr_code_hostp(env, addr, NULL);
+}
+
#endif
diff --git a/accel/tcg/internal-target.h b/accel/tcg/internal-target.h
index 2cdf11c..9a9cef3 100644
--- a/accel/tcg/internal-target.h
+++ b/accel/tcg/internal-target.h
@@ -9,10 +9,11 @@
#ifndef ACCEL_TCG_INTERNAL_TARGET_H
#define ACCEL_TCG_INTERNAL_TARGET_H
+#include "cpu-param.h"
#include "exec/exec-all.h"
#include "exec/translation-block.h"
#include "tb-internal.h"
-#include "tcg-target-mo.h"
+#include "exec/mmap-lock.h"
/*
* Access to the various translations structures need to be serialised
@@ -42,38 +43,4 @@ void page_table_config_init(void);
G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
#endif /* CONFIG_USER_ONLY */
-/**
- * tcg_req_mo:
- * @type: TCGBar
- *
- * Filter @type to the barrier that is required for the guest
- * memory ordering vs the host memory ordering. A non-zero
- * result indicates that some barrier is required.
- *
- * If TCG_GUEST_DEFAULT_MO is not defined, assume that the
- * guest requires strict ordering.
- *
- * This is a macro so that it's constant even without optimization.
- */
-#ifdef TCG_GUEST_DEFAULT_MO
-# define tcg_req_mo(type) \
- ((type) & TCG_GUEST_DEFAULT_MO & ~TCG_TARGET_DEFAULT_MO)
-#else
-# define tcg_req_mo(type) ((type) & ~TCG_TARGET_DEFAULT_MO)
-#endif
-
-/**
- * cpu_req_mo:
- * @type: TCGBar
- *
- * If tcg_req_mo indicates a barrier for @type is required
- * for the guest memory model, issue a host memory barrier.
- */
-#define cpu_req_mo(type) \
- do { \
- if (tcg_req_mo(type)) { \
- smp_mb(); \
- } \
- } while (0)
-
#endif /* ACCEL_TCG_INTERNAL_H */
diff --git a/accel/tcg/ldst_common.c.inc b/accel/tcg/ldst_common.c.inc
index ebbf380..9791a4e 100644
--- a/accel/tcg/ldst_common.c.inc
+++ b/accel/tcg/ldst_common.c.inc
@@ -135,7 +135,7 @@ static void plugin_load_cb(CPUArchState *env, abi_ptr addr,
}
}
-uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra)
+uint8_t cpu_ldb_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra)
{
uint8_t ret;
@@ -145,7 +145,7 @@ uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra)
return ret;
}
-uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr addr,
+uint16_t cpu_ldw_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t ra)
{
uint16_t ret;
@@ -156,7 +156,7 @@ uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr addr,
return ret;
}
-uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr addr,
+uint32_t cpu_ldl_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t ra)
{
uint32_t ret;
@@ -167,7 +167,7 @@ uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr addr,
return ret;
}
-uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr addr,
+uint64_t cpu_ldq_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t ra)
{
uint64_t ret;
@@ -178,7 +178,7 @@ uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr addr,
return ret;
}
-Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr,
+Int128 cpu_ld16_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t ra)
{
Int128 ret;
@@ -205,14 +205,14 @@ static void plugin_store_cb(CPUArchState *env, abi_ptr addr,
}
}
-void cpu_stb_mmu(CPUArchState *env, abi_ptr addr, uint8_t val,
+void cpu_stb_mmu(CPUArchState *env, vaddr addr, uint8_t val,
MemOpIdx oi, uintptr_t retaddr)
{
helper_stb_mmu(env, addr, val, oi, retaddr);
plugin_store_cb(env, addr, val, 0, oi);
}
-void cpu_stw_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
+void cpu_stw_mmu(CPUArchState *env, vaddr addr, uint16_t val,
MemOpIdx oi, uintptr_t retaddr)
{
tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16);
@@ -220,7 +220,7 @@ void cpu_stw_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
plugin_store_cb(env, addr, val, 0, oi);
}
-void cpu_stl_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
+void cpu_stl_mmu(CPUArchState *env, vaddr addr, uint32_t val,
MemOpIdx oi, uintptr_t retaddr)
{
tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32);
@@ -228,7 +228,7 @@ void cpu_stl_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
plugin_store_cb(env, addr, val, 0, oi);
}
-void cpu_stq_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
+void cpu_stq_mmu(CPUArchState *env, vaddr addr, uint64_t val,
MemOpIdx oi, uintptr_t retaddr)
{
tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64);
@@ -236,325 +236,10 @@ void cpu_stq_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
plugin_store_cb(env, addr, val, 0, oi);
}
-void cpu_st16_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
+void cpu_st16_mmu(CPUArchState *env, vaddr addr, Int128 val,
MemOpIdx oi, uintptr_t retaddr)
{
tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128);
do_st16_mmu(env_cpu(env), addr, val, oi, retaddr);
plugin_store_cb(env, addr, int128_getlo(val), int128_gethi(val), oi);
}
-
-/*
- * Wrappers of the above
- */
-
-uint32_t cpu_ldub_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
- return cpu_ldb_mmu(env, addr, oi, ra);
-}
-
-int cpu_ldsb_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return (int8_t)cpu_ldub_mmuidx_ra(env, addr, mmu_idx, ra);
-}
-
-uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx);
- return cpu_ldw_mmu(env, addr, oi, ra);
-}
-
-int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return (int16_t)cpu_lduw_be_mmuidx_ra(env, addr, mmu_idx, ra);
-}
-
-uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx);
- return cpu_ldl_mmu(env, addr, oi, ra);
-}
-
-uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_BEUQ | MO_UNALN, mmu_idx);
- return cpu_ldq_mmu(env, addr, oi, ra);
-}
-
-uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx);
- return cpu_ldw_mmu(env, addr, oi, ra);
-}
-
-int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return (int16_t)cpu_lduw_le_mmuidx_ra(env, addr, mmu_idx, ra);
-}
-
-uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx);
- return cpu_ldl_mmu(env, addr, oi, ra);
-}
-
-uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_LEUQ | MO_UNALN, mmu_idx);
- return cpu_ldq_mmu(env, addr, oi, ra);
-}
-
-void cpu_stb_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
- cpu_stb_mmu(env, addr, val, oi, ra);
-}
-
-void cpu_stw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx);
- cpu_stw_mmu(env, addr, val, oi, ra);
-}
-
-void cpu_stl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx);
- cpu_stl_mmu(env, addr, val, oi, ra);
-}
-
-void cpu_stq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_BEUQ | MO_UNALN, mmu_idx);
- cpu_stq_mmu(env, addr, val, oi, ra);
-}
-
-void cpu_stw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx);
- cpu_stw_mmu(env, addr, val, oi, ra);
-}
-
-void cpu_stl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx);
- cpu_stl_mmu(env, addr, val, oi, ra);
-}
-
-void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
- int mmu_idx, uintptr_t ra)
-{
- MemOpIdx oi = make_memop_idx(MO_LEUQ | MO_UNALN, mmu_idx);
- cpu_stq_mmu(env, addr, val, oi, ra);
-}
-
-/*--------------------------*/
-
-uint32_t cpu_ldub_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- return cpu_ldub_mmuidx_ra(env, addr, mmu_index, ra);
-}
-
-int cpu_ldsb_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
-{
- return (int8_t)cpu_ldub_data_ra(env, addr, ra);
-}
-
-uint32_t cpu_lduw_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- return cpu_lduw_be_mmuidx_ra(env, addr, mmu_index, ra);
-}
-
-int cpu_ldsw_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
-{
- return (int16_t)cpu_lduw_be_data_ra(env, addr, ra);
-}
-
-uint32_t cpu_ldl_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- return cpu_ldl_be_mmuidx_ra(env, addr, mmu_index, ra);
-}
-
-uint64_t cpu_ldq_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- return cpu_ldq_be_mmuidx_ra(env, addr, mmu_index, ra);
-}
-
-uint32_t cpu_lduw_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- return cpu_lduw_le_mmuidx_ra(env, addr, mmu_index, ra);
-}
-
-int cpu_ldsw_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
-{
- return (int16_t)cpu_lduw_le_data_ra(env, addr, ra);
-}
-
-uint32_t cpu_ldl_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- return cpu_ldl_le_mmuidx_ra(env, addr, mmu_index, ra);
-}
-
-uint64_t cpu_ldq_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- return cpu_ldq_le_mmuidx_ra(env, addr, mmu_index, ra);
-}
-
-void cpu_stb_data_ra(CPUArchState *env, abi_ptr addr,
- uint32_t val, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- cpu_stb_mmuidx_ra(env, addr, val, mmu_index, ra);
-}
-
-void cpu_stw_be_data_ra(CPUArchState *env, abi_ptr addr,
- uint32_t val, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- cpu_stw_be_mmuidx_ra(env, addr, val, mmu_index, ra);
-}
-
-void cpu_stl_be_data_ra(CPUArchState *env, abi_ptr addr,
- uint32_t val, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- cpu_stl_be_mmuidx_ra(env, addr, val, mmu_index, ra);
-}
-
-void cpu_stq_be_data_ra(CPUArchState *env, abi_ptr addr,
- uint64_t val, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- cpu_stq_be_mmuidx_ra(env, addr, val, mmu_index, ra);
-}
-
-void cpu_stw_le_data_ra(CPUArchState *env, abi_ptr addr,
- uint32_t val, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- cpu_stw_le_mmuidx_ra(env, addr, val, mmu_index, ra);
-}
-
-void cpu_stl_le_data_ra(CPUArchState *env, abi_ptr addr,
- uint32_t val, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- cpu_stl_le_mmuidx_ra(env, addr, val, mmu_index, ra);
-}
-
-void cpu_stq_le_data_ra(CPUArchState *env, abi_ptr addr,
- uint64_t val, uintptr_t ra)
-{
- int mmu_index = cpu_mmu_index(env_cpu(env), false);
- cpu_stq_le_mmuidx_ra(env, addr, val, mmu_index, ra);
-}
-
-/*--------------------------*/
-
-uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr addr)
-{
- return cpu_ldub_data_ra(env, addr, 0);
-}
-
-int cpu_ldsb_data(CPUArchState *env, abi_ptr addr)
-{
- return (int8_t)cpu_ldub_data(env, addr);
-}
-
-uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr addr)
-{
- return cpu_lduw_be_data_ra(env, addr, 0);
-}
-
-int cpu_ldsw_be_data(CPUArchState *env, abi_ptr addr)
-{
- return (int16_t)cpu_lduw_be_data(env, addr);
-}
-
-uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr addr)
-{
- return cpu_ldl_be_data_ra(env, addr, 0);
-}
-
-uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr addr)
-{
- return cpu_ldq_be_data_ra(env, addr, 0);
-}
-
-uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr addr)
-{
- return cpu_lduw_le_data_ra(env, addr, 0);
-}
-
-int cpu_ldsw_le_data(CPUArchState *env, abi_ptr addr)
-{
- return (int16_t)cpu_lduw_le_data(env, addr);
-}
-
-uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr addr)
-{
- return cpu_ldl_le_data_ra(env, addr, 0);
-}
-
-uint64_t cpu_ldq_le_data(CPUArchState *env, abi_ptr addr)
-{
- return cpu_ldq_le_data_ra(env, addr, 0);
-}
-
-void cpu_stb_data(CPUArchState *env, abi_ptr addr, uint32_t val)
-{
- cpu_stb_data_ra(env, addr, val, 0);
-}
-
-void cpu_stw_be_data(CPUArchState *env, abi_ptr addr, uint32_t val)
-{
- cpu_stw_be_data_ra(env, addr, val, 0);
-}
-
-void cpu_stl_be_data(CPUArchState *env, abi_ptr addr, uint32_t val)
-{
- cpu_stl_be_data_ra(env, addr, val, 0);
-}
-
-void cpu_stq_be_data(CPUArchState *env, abi_ptr addr, uint64_t val)
-{
- cpu_stq_be_data_ra(env, addr, val, 0);
-}
-
-void cpu_stw_le_data(CPUArchState *env, abi_ptr addr, uint32_t val)
-{
- cpu_stw_le_data_ra(env, addr, val, 0);
-}
-
-void cpu_stl_le_data(CPUArchState *env, abi_ptr addr, uint32_t val)
-{
- cpu_stl_le_data_ra(env, addr, val, 0);
-}
-
-void cpu_stq_le_data(CPUArchState *env, abi_ptr addr, uint64_t val)
-{
- cpu_stq_le_data_ra(env, addr, val, 0);
-}
diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build
index 38ff227..047afa4 100644
--- a/accel/tcg/meson.build
+++ b/accel/tcg/meson.build
@@ -1,28 +1,41 @@
-common_ss.add(when: 'CONFIG_TCG', if_true: files(
+if not get_option('tcg').allowed()
+ subdir_done()
+endif
+
+tcg_ss = ss.source_set()
+
+tcg_ss.add(files(
'cpu-exec-common.c',
'tcg-runtime.c',
'tcg-runtime-gvec.c',
+ 'translator.c',
))
+if get_option('plugins')
+ tcg_ss.add(files('plugin-gen.c'))
+endif
+
+libuser_ss.add_all(tcg_ss)
+libsystem_ss.add_all(tcg_ss)
+
tcg_specific_ss = ss.source_set()
tcg_specific_ss.add(files(
'tcg-all.c',
'cpu-exec.c',
'tb-maint.c',
'translate-all.c',
- 'translator.c',
))
tcg_specific_ss.add(when: 'CONFIG_USER_ONLY', if_true: files('user-exec.c'))
-tcg_specific_ss.add(when: 'CONFIG_SYSTEM_ONLY', if_false: files('user-exec-stub.c'))
-if get_option('plugins')
- tcg_specific_ss.add(files('plugin-gen.c'))
-endif
specific_ss.add_all(when: 'CONFIG_TCG', if_true: tcg_specific_ss)
specific_ss.add(when: ['CONFIG_SYSTEM_ONLY', 'CONFIG_TCG'], if_true: files(
'cputlb.c',
))
-system_ss.add(when: ['CONFIG_TCG'], if_true: files(
+libuser_ss.add(files(
+ 'user-exec-stub.c',
+))
+
+libsystem_ss.add(files(
'icount-common.c',
'monitor.c',
'tcg-accel-ops.c',
diff --git a/accel/tcg/monitor.c b/accel/tcg/monitor.c
index eeb38a4..1c182b6 100644
--- a/accel/tcg/monitor.c
+++ b/accel/tcg/monitor.c
@@ -14,6 +14,7 @@
#include "qapi/qapi-commands-machine.h"
#include "monitor/monitor.h"
#include "system/cpu-timers.h"
+#include "exec/icount.h"
#include "system/tcg.h"
#include "tcg/tcg.h"
#include "internal-common.h"
diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c
index 7e5f040..c1da753 100644
--- a/accel/tcg/plugin-gen.c
+++ b/accel/tcg/plugin-gen.c
@@ -22,13 +22,12 @@
#include "qemu/osdep.h"
#include "qemu/plugin.h"
#include "qemu/log.h"
-#include "cpu.h"
#include "tcg/tcg.h"
#include "tcg/tcg-temp-internal.h"
-#include "tcg/tcg-op.h"
-#include "exec/exec-all.h"
+#include "tcg/tcg-op-common.h"
#include "exec/plugin-gen.h"
#include "exec/translator.h"
+#include "exec/translation-block.h"
enum plugin_gen_from {
PLUGIN_GEN_FROM_TB,
@@ -89,15 +88,13 @@ static void gen_enable_mem_helper(struct qemu_plugin_tb *ptb,
qemu_plugin_add_dyn_cb_arr(arr);
tcg_gen_st_ptr(tcg_constant_ptr((intptr_t)arr), tcg_env,
- offsetof(CPUState, neg.plugin_mem_cbs) -
- offsetof(ArchCPU, env));
+ offsetof(CPUState, neg.plugin_mem_cbs) - sizeof(CPUState));
}
static void gen_disable_mem_helper(void)
{
tcg_gen_st_ptr(tcg_constant_ptr(0), tcg_env,
- offsetof(CPUState, neg.plugin_mem_cbs) -
- offsetof(ArchCPU, env));
+ offsetof(CPUState, neg.plugin_mem_cbs) - sizeof(CPUState));
}
static TCGv_i32 gen_cpu_index(void)
@@ -113,7 +110,7 @@ static TCGv_i32 gen_cpu_index(void)
}
TCGv_i32 cpu_index = tcg_temp_ebb_new_i32();
tcg_gen_ld_i32(cpu_index, tcg_env,
- -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index));
+ offsetof(CPUState, cpu_index) - sizeof(CPUState));
return cpu_index;
}
diff --git a/accel/tcg/tb-hash.h b/accel/tcg/tb-hash.h
index a5382f4..3bc5042 100644
--- a/accel/tcg/tb-hash.h
+++ b/accel/tcg/tb-hash.h
@@ -22,6 +22,7 @@
#include "exec/cpu-defs.h"
#include "exec/exec-all.h"
+#include "exec/target_page.h"
#include "exec/translation-block.h"
#include "qemu/xxhash.h"
#include "tb-jmp-cache.h"
diff --git a/accel/tcg/tb-internal.h b/accel/tcg/tb-internal.h
index 68aa8d1..08538e2 100644
--- a/accel/tcg/tb-internal.h
+++ b/accel/tcg/tb-internal.h
@@ -9,8 +9,6 @@
#ifndef ACCEL_TCG_TB_INTERNAL_TARGET_H
#define ACCEL_TCG_TB_INTERNAL_TARGET_H
-#include "exec/cpu-all.h"
-#include "exec/exec-all.h"
#include "exec/translation-block.h"
/*
@@ -24,55 +22,23 @@
*/
#define GETPC_ADJ 2
-#ifdef CONFIG_SOFTMMU
-
-#define CPU_TLB_DYN_MIN_BITS 6
-#define CPU_TLB_DYN_DEFAULT_BITS 8
-
-# if HOST_LONG_BITS == 32
-/* Make sure we do not require a double-word shift for the TLB load */
-# define CPU_TLB_DYN_MAX_BITS (32 - TARGET_PAGE_BITS)
-# else /* HOST_LONG_BITS == 64 */
-/*
- * Assuming TARGET_PAGE_BITS==12, with 2**22 entries we can cover 2**(22+12) ==
- * 2**34 == 16G of address space. This is roughly what one would expect a
- * TLB to cover in a modern (as of 2018) x86_64 CPU. For instance, Intel
- * Skylake's Level-2 STLB has 16 1G entries.
- * Also, make sure we do not size the TLB past the guest's address space.
- */
-# ifdef TARGET_PAGE_BITS_VARY
-# define CPU_TLB_DYN_MAX_BITS \
- MIN(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS)
-# else
-# define CPU_TLB_DYN_MAX_BITS \
- MIN_CONST(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS)
-# endif
-# endif
-
-#endif /* CONFIG_SOFTMMU */
+void tb_lock_page0(tb_page_addr_t);
#ifdef CONFIG_USER_ONLY
-#include "user/page-protection.h"
/*
* For user-only, page_protect sets the page read-only.
* Since most execution is already on read-only pages, and we'd need to
* account for other TBs on the same page, defer undoing any page protection
* until we receive the write fault.
*/
-static inline void tb_lock_page0(tb_page_addr_t p0)
-{
- page_protect(p0);
-}
-
static inline void tb_lock_page1(tb_page_addr_t p0, tb_page_addr_t p1)
{
- page_protect(p1);
+ tb_lock_page0(p1);
}
static inline void tb_unlock_page1(tb_page_addr_t p0, tb_page_addr_t p1) { }
static inline void tb_unlock_pages(TranslationBlock *tb) { }
#else
-void tb_lock_page0(tb_page_addr_t);
void tb_lock_page1(tb_page_addr_t, tb_page_addr_t);
void tb_unlock_page1(tb_page_addr_t, tb_page_addr_t);
void tb_unlock_pages(TranslationBlock *);
diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
index 3f1bebf..d479f53 100644
--- a/accel/tcg/tb-maint.c
+++ b/accel/tcg/tb-maint.c
@@ -20,11 +20,14 @@
#include "qemu/osdep.h"
#include "qemu/interval-tree.h"
#include "qemu/qtree.h"
+#include "cpu.h"
#include "exec/cputlb.h"
#include "exec/log.h"
#include "exec/exec-all.h"
#include "exec/page-protection.h"
+#include "exec/mmap-lock.h"
#include "exec/tb-flush.h"
+#include "exec/target_page.h"
#include "tb-internal.h"
#include "system/tcg.h"
#include "tcg/tcg.h"
diff --git a/accel/tcg/tcg-accel-ops-icount.c b/accel/tcg/tcg-accel-ops-icount.c
index 27cf104..d0f7b41 100644
--- a/accel/tcg/tcg-accel-ops-icount.c
+++ b/accel/tcg/tcg-accel-ops-icount.c
@@ -25,7 +25,7 @@
#include "qemu/osdep.h"
#include "system/replay.h"
-#include "system/cpu-timers.h"
+#include "exec/icount.h"
#include "qemu/main-loop.h"
#include "qemu/guest-random.h"
#include "hw/core/cpu.h"
diff --git a/accel/tcg/tcg-accel-ops-mttcg.c b/accel/tcg/tcg-accel-ops-mttcg.c
index bdcc385..dfcee30 100644
--- a/accel/tcg/tcg-accel-ops-mttcg.c
+++ b/accel/tcg/tcg-accel-ops-mttcg.c
@@ -26,7 +26,7 @@
#include "qemu/osdep.h"
#include "system/tcg.h"
#include "system/replay.h"
-#include "system/cpu-timers.h"
+#include "exec/icount.h"
#include "qemu/main-loop.h"
#include "qemu/notify.h"
#include "qemu/guest-random.h"
diff --git a/accel/tcg/tcg-accel-ops-rr.c b/accel/tcg/tcg-accel-ops-rr.c
index f62cf24..6eec5c9 100644
--- a/accel/tcg/tcg-accel-ops-rr.c
+++ b/accel/tcg/tcg-accel-ops-rr.c
@@ -27,7 +27,7 @@
#include "qemu/lockable.h"
#include "system/tcg.h"
#include "system/replay.h"
-#include "system/cpu-timers.h"
+#include "exec/icount.h"
#include "qemu/main-loop.h"
#include "qemu/notify.h"
#include "qemu/guest-random.h"
diff --git a/accel/tcg/tcg-accel-ops.c b/accel/tcg/tcg-accel-ops.c
index d9b662e..ccdb781 100644
--- a/accel/tcg/tcg-accel-ops.c
+++ b/accel/tcg/tcg-accel-ops.c
@@ -29,7 +29,7 @@
#include "system/accel-ops.h"
#include "system/tcg.h"
#include "system/replay.h"
-#include "system/cpu-timers.h"
+#include "exec/icount.h"
#include "qemu/main-loop.h"
#include "qemu/guest-random.h"
#include "qemu/timer.h"
@@ -37,6 +37,7 @@
#include "exec/hwaddr.h"
#include "exec/tb-flush.h"
#include "exec/translation-block.h"
+#include "exec/watchpoint.h"
#include "gdbstub/enums.h"
#include "hw/core/cpu.h"
diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c
index c1a30b0..b0d4e3e 100644
--- a/accel/tcg/tcg-all.c
+++ b/accel/tcg/tcg-all.c
@@ -26,27 +26,30 @@
#include "qemu/osdep.h"
#include "system/tcg.h"
#include "exec/replay-core.h"
-#include "system/cpu-timers.h"
+#include "exec/icount.h"
#include "tcg/startup.h"
#include "qapi/error.h"
#include "qemu/error-report.h"
#include "qemu/accel.h"
#include "qemu/atomic.h"
+#include "qapi/qapi-types-common.h"
#include "qapi/qapi-builtin-visit.h"
#include "qemu/units.h"
#if defined(CONFIG_USER_ONLY)
#include "hw/qdev-core.h"
#else
#include "hw/boards.h"
+#include "system/tcg.h"
#endif
+#include "accel/tcg/cpu-ops.h"
#include "internal-common.h"
-#include "cpu-param.h"
+#include "cpu.h"
struct TCGState {
AccelState parent_obj;
- bool mttcg_enabled;
+ OnOffAuto mttcg_enabled;
bool one_insn_per_tb;
int splitwx_enabled;
unsigned long tb_size;
@@ -58,40 +61,18 @@ typedef struct TCGState TCGState;
DECLARE_INSTANCE_CHECKER(TCGState, TCG_STATE,
TYPE_TCG_ACCEL)
-/*
- * We default to false if we know other options have been enabled
- * which are currently incompatible with MTTCG. Otherwise when each
- * guest (target) has been updated to support:
- * - atomic instructions
- * - memory ordering primitives (barriers)
- * they can set the appropriate CONFIG flags in ${target}-softmmu.mak
- *
- * Once a guest architecture has been converted to the new primitives
- * there is one remaining limitation to check:
- * - The guest can't be oversized (e.g. 64 bit guest on 32 bit host)
- */
-
-static bool default_mttcg_enabled(void)
+#ifndef CONFIG_USER_ONLY
+bool qemu_tcg_mttcg_enabled(void)
{
- if (icount_enabled()) {
- return false;
- }
-#ifdef TARGET_SUPPORTS_MTTCG
-# ifndef TCG_GUEST_DEFAULT_MO
-# error "TARGET_SUPPORTS_MTTCG without TCG_GUEST_DEFAULT_MO"
-# endif
- return true;
-#else
- return false;
-#endif
+ TCGState *s = TCG_STATE(current_accel());
+ return s->mttcg_enabled == ON_OFF_AUTO_ON;
}
+#endif /* !CONFIG_USER_ONLY */
static void tcg_accel_instance_init(Object *obj)
{
TCGState *s = TCG_STATE(obj);
- s->mttcg_enabled = default_mttcg_enabled();
-
/* If debugging enabled, default "auto on", otherwise off. */
#if defined(CONFIG_DEBUG_TCG) && !defined(CONFIG_USER_ONLY)
s->splitwx_enabled = -1;
@@ -100,24 +81,57 @@ static void tcg_accel_instance_init(Object *obj)
#endif
}
-bool mttcg_enabled;
bool one_insn_per_tb;
static int tcg_init_machine(MachineState *ms)
{
TCGState *s = TCG_STATE(current_accel());
-#ifdef CONFIG_USER_ONLY
- unsigned max_cpus = 1;
-#else
- unsigned max_cpus = ms->smp.max_cpus;
+ unsigned max_threads = 1;
+
+#ifndef CONFIG_USER_ONLY
+ CPUClass *cc = CPU_CLASS(object_class_by_name(CPU_RESOLVING_TYPE));
+ bool mttcg_supported = cc->tcg_ops->mttcg_supported;
+
+ switch (s->mttcg_enabled) {
+ case ON_OFF_AUTO_AUTO:
+ /*
+ * We default to false if we know other options have been enabled
+ * which are currently incompatible with MTTCG. Otherwise when each
+ * guest (target) has been updated to support:
+ * - atomic instructions
+ * - memory ordering primitives (barriers)
+ * they can set the appropriate CONFIG flags in ${target}-softmmu.mak
+ *
+ * Once a guest architecture has been converted to the new primitives
+ * there is one remaining limitation to check:
+ * - The guest can't be oversized (e.g. 64 bit guest on 32 bit host)
+ */
+ if (mttcg_supported && !icount_enabled()) {
+ s->mttcg_enabled = ON_OFF_AUTO_ON;
+ max_threads = ms->smp.max_cpus;
+ } else {
+ s->mttcg_enabled = ON_OFF_AUTO_OFF;
+ }
+ break;
+ case ON_OFF_AUTO_ON:
+ if (!mttcg_supported) {
+ warn_report("Guest not yet converted to MTTCG - "
+ "you may get unexpected results");
+ }
+ max_threads = ms->smp.max_cpus;
+ break;
+ case ON_OFF_AUTO_OFF:
+ break;
+ default:
+ g_assert_not_reached();
+ }
#endif
tcg_allowed = true;
- mttcg_enabled = s->mttcg_enabled;
page_init();
tb_htable_init();
- tcg_init(s->tb_size * MiB, s->splitwx_enabled, max_cpus);
+ tcg_init(s->tb_size * MiB, s->splitwx_enabled, max_threads);
#if defined(CONFIG_SOFTMMU)
/*
@@ -138,7 +152,7 @@ static char *tcg_get_thread(Object *obj, Error **errp)
{
TCGState *s = TCG_STATE(obj);
- return g_strdup(s->mttcg_enabled ? "multi" : "single");
+ return g_strdup(s->mttcg_enabled == ON_OFF_AUTO_ON ? "multi" : "single");
}
static void tcg_set_thread(Object *obj, const char *value, Error **errp)
@@ -149,14 +163,10 @@ static void tcg_set_thread(Object *obj, const char *value, Error **errp)
if (icount_enabled()) {
error_setg(errp, "No MTTCG when icount is enabled");
} else {
-#ifndef TARGET_SUPPORTS_MTTCG
- warn_report("Guest not yet converted to MTTCG - "
- "you may get unexpected results");
-#endif
- s->mttcg_enabled = true;
+ s->mttcg_enabled = ON_OFF_AUTO_ON;
}
} else if (strcmp(value, "single") == 0) {
- s->mttcg_enabled = false;
+ s->mttcg_enabled = ON_OFF_AUTO_OFF;
} else {
error_setg(errp, "Invalid 'thread' setting %s", value);
}
diff --git a/accel/tcg/tlb-bounds.h b/accel/tcg/tlb-bounds.h
new file mode 100644
index 0000000..efd34d4
--- /dev/null
+++ b/accel/tcg/tlb-bounds.h
@@ -0,0 +1,32 @@
+/*
+ * softmmu size bounds
+ * SPDX-License-Identifier: LGPL-2.1-or-later
+ */
+
+#ifndef ACCEL_TCG_TLB_BOUNDS_H
+#define ACCEL_TCG_TLB_BOUNDS_H
+
+#define CPU_TLB_DYN_MIN_BITS 6
+#define CPU_TLB_DYN_DEFAULT_BITS 8
+
+# if HOST_LONG_BITS == 32
+/* Make sure we do not require a double-word shift for the TLB load */
+# define CPU_TLB_DYN_MAX_BITS (32 - TARGET_PAGE_BITS)
+# else /* HOST_LONG_BITS == 64 */
+/*
+ * Assuming TARGET_PAGE_BITS==12, with 2**22 entries we can cover 2**(22+12) ==
+ * 2**34 == 16G of address space. This is roughly what one would expect a
+ * TLB to cover in a modern (as of 2018) x86_64 CPU. For instance, Intel
+ * Skylake's Level-2 STLB has 16 1G entries.
+ * Also, make sure we do not size the TLB past the guest's address space.
+ */
+# ifdef TARGET_PAGE_BITS_VARY
+# define CPU_TLB_DYN_MAX_BITS \
+ MIN(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS)
+# else
+# define CPU_TLB_DYN_MAX_BITS \
+ MIN_CONST(22, TARGET_VIRT_ADDR_SPACE_BITS - TARGET_PAGE_BITS)
+# endif
+# endif
+
+#endif /* ACCEL_TCG_TLB_BOUNDS_H */
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
index 82bc16b..c007b9a 100644
--- a/accel/tcg/translate-all.c
+++ b/accel/tcg/translate-all.c
@@ -40,12 +40,15 @@
#endif
#endif
#else
-#include "exec/ram_addr.h"
+#include "system/ram_addr.h"
#endif
+#include "cpu-param.h"
#include "exec/cputlb.h"
#include "exec/page-protection.h"
+#include "exec/mmap-lock.h"
#include "tb-internal.h"
+#include "tlb-bounds.h"
#include "exec/translator.h"
#include "exec/tb-flush.h"
#include "qemu/bitmap.h"
@@ -54,7 +57,7 @@
#include "qemu/cacheinfo.h"
#include "qemu/timer.h"
#include "exec/log.h"
-#include "system/cpu-timers.h"
+#include "exec/icount.h"
#include "system/tcg.h"
#include "qapi/error.h"
#include "accel/tcg/cpu-ops.h"
@@ -66,6 +69,7 @@
#include "internal-target.h"
#include "tcg/perf.h"
#include "tcg/insn-start-words.h"
+#include "cpu.h"
TBContext tb_ctx;
@@ -349,11 +353,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
tcg_ctx->tlb_dyn_max_bits = CPU_TLB_DYN_MAX_BITS;
#endif
tcg_ctx->insn_start_words = TARGET_INSN_START_WORDS;
-#ifdef TCG_GUEST_DEFAULT_MO
- tcg_ctx->guest_mo = TCG_GUEST_DEFAULT_MO;
-#else
- tcg_ctx->guest_mo = TCG_MO_ALL;
-#endif
+ tcg_ctx->guest_mo = cpu->cc->tcg_ops->guest_default_memory_order;
restart_translate:
trace_translate_block(tb, pc, tb->tc.ptr);
diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c
index ef1538b..034f2f3 100644
--- a/accel/tcg/translator.c
+++ b/accel/tcg/translator.c
@@ -8,16 +8,16 @@
*/
#include "qemu/osdep.h"
+#include "qemu/bswap.h"
#include "qemu/log.h"
#include "qemu/error-report.h"
-#include "exec/exec-all.h"
+#include "accel/tcg/cpu-ldst-common.h"
+#include "accel/tcg/cpu-mmu-index.h"
+#include "exec/target_page.h"
#include "exec/translator.h"
-#include "exec/cpu_ldst.h"
#include "exec/plugin-gen.h"
-#include "exec/cpu_ldst.h"
-#include "exec/tswap.h"
#include "tcg/tcg-op-common.h"
-#include "internal-target.h"
+#include "internal-common.h"
#include "disas/disas.h"
#include "tb-internal.h"
@@ -25,8 +25,7 @@ static void set_can_do_io(DisasContextBase *db, bool val)
{
QEMU_BUILD_BUG_ON(sizeof_field(CPUState, neg.can_do_io) != 1);
tcg_gen_st8_i32(tcg_constant_i32(val), tcg_env,
- offsetof(ArchCPU, parent_obj.neg.can_do_io) -
- offsetof(ArchCPU, env));
+ offsetof(CPUState, neg.can_do_io) - sizeof(CPUState));
}
bool translator_io_start(DisasContextBase *db)
@@ -49,8 +48,8 @@ static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags)
if ((cflags & CF_USE_ICOUNT) || !(cflags & CF_NOIRQ)) {
count = tcg_temp_new_i32();
tcg_gen_ld_i32(count, tcg_env,
- offsetof(ArchCPU, parent_obj.neg.icount_decr.u32)
- - offsetof(ArchCPU, env));
+ offsetof(CPUState, neg.icount_decr.u32) -
+ sizeof(CPUState));
}
if (cflags & CF_USE_ICOUNT) {
@@ -79,8 +78,8 @@ static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags)
if (cflags & CF_USE_ICOUNT) {
tcg_gen_st16_i32(count, tcg_env,
- offsetof(ArchCPU, parent_obj.neg.icount_decr.u16.low)
- - offsetof(ArchCPU, env));
+ offsetof(CPUState, neg.icount_decr.u16.low) -
+ sizeof(CPUState));
}
return icount_start_insn;
@@ -142,6 +141,7 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns,
db->host_addr[1] = NULL;
db->record_start = 0;
db->record_len = 0;
+ db->code_mmuidx = cpu_mmu_index(cpu, true);
ops->init_disas_context(db, cpu);
tcg_debug_assert(db->is_jmp == DISAS_NEXT); /* no early exit */
@@ -265,12 +265,14 @@ static bool translator_ld(CPUArchState *env, DisasContextBase *db,
if (likely(((base ^ last) & TARGET_PAGE_MASK) == 0)) {
/* Entire read is from the first page. */
- memcpy(dest, host + (pc - base), len);
- return true;
+ goto do_read;
}
if (unlikely(((base ^ pc) & TARGET_PAGE_MASK) == 0)) {
- /* Read begins on the first page and extends to the second. */
+ /*
+ * Read begins on the first page and extends to the second.
+ * The unaligned read is never atomic.
+ */
size_t len0 = -(pc | TARGET_PAGE_MASK);
memcpy(dest, host + (pc - base), len0);
pc += len0;
@@ -329,7 +331,39 @@ static bool translator_ld(CPUArchState *env, DisasContextBase *db,
host = db->host_addr[1];
}
- memcpy(dest, host + (pc - base), len);
+ do_read:
+ /*
+ * Assume aligned reads should be atomic, if possible.
+ * We're not in a position to jump out with EXCP_ATOMIC.
+ */
+ host += pc - base;
+ switch (len) {
+ case 2:
+ if (QEMU_IS_ALIGNED(pc, 2)) {
+ uint16_t t = qatomic_read((uint16_t *)host);
+ stw_he_p(dest, t);
+ return true;
+ }
+ break;
+ case 4:
+ if (QEMU_IS_ALIGNED(pc, 4)) {
+ uint32_t t = qatomic_read((uint32_t *)host);
+ stl_he_p(dest, t);
+ return true;
+ }
+ break;
+#ifdef CONFIG_ATOMIC64
+ case 8:
+ if (QEMU_IS_ALIGNED(pc, 8)) {
+ uint64_t t = qatomic_read__nocheck((uint64_t *)host);
+ stq_he_p(dest, t);
+ return true;
+ }
+ break;
+#endif
+ }
+ /* Unaligned or partial read from the second page is not atomic. */
+ memcpy(dest, host, len);
return true;
}
@@ -423,55 +457,62 @@ bool translator_st(const DisasContextBase *db, void *dest,
uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, vaddr pc)
{
- uint8_t raw;
+ uint8_t val;
- if (!translator_ld(env, db, &raw, pc, sizeof(raw))) {
- raw = cpu_ldub_code(env, pc);
- record_save(db, pc, &raw, sizeof(raw));
+ if (!translator_ld(env, db, &val, pc, sizeof(val))) {
+ MemOpIdx oi = make_memop_idx(MO_UB, db->code_mmuidx);
+ val = cpu_ldb_code_mmu(env, pc, oi, 0);
+ record_save(db, pc, &val, sizeof(val));
}
- return raw;
+ return val;
}
-uint16_t translator_lduw(CPUArchState *env, DisasContextBase *db, vaddr pc)
+uint16_t translator_lduw_end(CPUArchState *env, DisasContextBase *db,
+ vaddr pc, MemOp endian)
{
- uint16_t raw, tgt;
+ uint16_t val;
- if (translator_ld(env, db, &raw, pc, sizeof(raw))) {
- tgt = tswap16(raw);
- } else {
- tgt = cpu_lduw_code(env, pc);
- raw = tswap16(tgt);
- record_save(db, pc, &raw, sizeof(raw));
+ if (!translator_ld(env, db, &val, pc, sizeof(val))) {
+ MemOpIdx oi = make_memop_idx(MO_UW, db->code_mmuidx);
+ val = cpu_ldw_code_mmu(env, pc, oi, 0);
+ record_save(db, pc, &val, sizeof(val));
+ }
+ if (endian & MO_BSWAP) {
+ val = bswap16(val);
}
- return tgt;
+ return val;
}
-uint32_t translator_ldl(CPUArchState *env, DisasContextBase *db, vaddr pc)
+uint32_t translator_ldl_end(CPUArchState *env, DisasContextBase *db,
+ vaddr pc, MemOp endian)
{
- uint32_t raw, tgt;
+ uint32_t val;
- if (translator_ld(env, db, &raw, pc, sizeof(raw))) {
- tgt = tswap32(raw);
- } else {
- tgt = cpu_ldl_code(env, pc);
- raw = tswap32(tgt);
- record_save(db, pc, &raw, sizeof(raw));
+ if (!translator_ld(env, db, &val, pc, sizeof(val))) {
+ MemOpIdx oi = make_memop_idx(MO_UL, db->code_mmuidx);
+ val = cpu_ldl_code_mmu(env, pc, oi, 0);
+ record_save(db, pc, &val, sizeof(val));
+ }
+ if (endian & MO_BSWAP) {
+ val = bswap32(val);
}
- return tgt;
+ return val;
}
-uint64_t translator_ldq(CPUArchState *env, DisasContextBase *db, vaddr pc)
+uint64_t translator_ldq_end(CPUArchState *env, DisasContextBase *db,
+ vaddr pc, MemOp endian)
{
- uint64_t raw, tgt;
+ uint64_t val;
- if (translator_ld(env, db, &raw, pc, sizeof(raw))) {
- tgt = tswap64(raw);
- } else {
- tgt = cpu_ldq_code(env, pc);
- raw = tswap64(tgt);
- record_save(db, pc, &raw, sizeof(raw));
+ if (!translator_ld(env, db, &val, pc, sizeof(val))) {
+ MemOpIdx oi = make_memop_idx(MO_UQ, db->code_mmuidx);
+ val = cpu_ldq_code_mmu(env, pc, oi, 0);
+ record_save(db, pc, &val, sizeof(val));
+ }
+ if (endian & MO_BSWAP) {
+ val = bswap64(val);
}
- return tgt;
+ return val;
}
void translator_fake_ld(DisasContextBase *db, const void *data, size_t len)
diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
index 2322181..5eef8e7 100644
--- a/accel/tcg/user-exec.c
+++ b/accel/tcg/user-exec.c
@@ -19,12 +19,14 @@
#include "qemu/osdep.h"
#include "accel/tcg/cpu-ops.h"
#include "disas/disas.h"
+#include "cpu.h"
#include "exec/vaddr.h"
#include "exec/exec-all.h"
+#include "exec/tlb-flags.h"
#include "tcg/tcg.h"
#include "qemu/bitops.h"
#include "qemu/rcu.h"
-#include "exec/cpu_ldst.h"
+#include "accel/tcg/cpu-ldst.h"
#include "user/cpu_loop.h"
#include "qemu/main-loop.h"
#include "user/page-protection.h"
@@ -35,6 +37,7 @@
#include "qemu/int128.h"
#include "trace.h"
#include "tcg/tcg-ldst.h"
+#include "backend-ldst.h"
#include "internal-common.h"
#include "internal-target.h"
#include "tb-internal.h"
@@ -656,7 +659,7 @@ target_ulong page_find_range_empty(target_ulong min, target_ulong max,
}
}
-void page_protect(tb_page_addr_t address)
+void tb_lock_page0(tb_page_addr_t address)
{
PageFlagsNode *p;
target_ulong start, last;
@@ -1059,7 +1062,7 @@ static uint8_t do_ld1_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
void *haddr;
uint8_t ret;
- cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
haddr = cpu_mmu_lookup(cpu, addr, get_memop(oi), ra, access_type);
ret = ldub_p(haddr);
clear_helper_retaddr();
@@ -1073,7 +1076,7 @@ static uint16_t do_ld2_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
uint16_t ret;
MemOp mop = get_memop(oi);
- cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
haddr = cpu_mmu_lookup(cpu, addr, mop, ra, access_type);
ret = load_atom_2(cpu, ra, haddr, mop);
clear_helper_retaddr();
@@ -1091,7 +1094,7 @@ static uint32_t do_ld4_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
uint32_t ret;
MemOp mop = get_memop(oi);
- cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
haddr = cpu_mmu_lookup(cpu, addr, mop, ra, access_type);
ret = load_atom_4(cpu, ra, haddr, mop);
clear_helper_retaddr();
@@ -1109,7 +1112,7 @@ static uint64_t do_ld8_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
uint64_t ret;
MemOp mop = get_memop(oi);
- cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
haddr = cpu_mmu_lookup(cpu, addr, mop, ra, access_type);
ret = load_atom_8(cpu, ra, haddr, mop);
clear_helper_retaddr();
@@ -1128,7 +1131,7 @@ static Int128 do_ld16_mmu(CPUState *cpu, abi_ptr addr,
MemOp mop = get_memop(oi);
tcg_debug_assert((mop & MO_SIZE) == MO_128);
- cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_LOAD);
ret = load_atom_16(cpu, ra, haddr, mop);
clear_helper_retaddr();
@@ -1144,7 +1147,7 @@ static void do_st1_mmu(CPUState *cpu, vaddr addr, uint8_t val,
{
void *haddr;
- cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
haddr = cpu_mmu_lookup(cpu, addr, get_memop(oi), ra, MMU_DATA_STORE);
stb_p(haddr, val);
clear_helper_retaddr();
@@ -1156,7 +1159,7 @@ static void do_st2_mmu(CPUState *cpu, vaddr addr, uint16_t val,
void *haddr;
MemOp mop = get_memop(oi);
- cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE);
if (mop & MO_BSWAP) {
@@ -1172,7 +1175,7 @@ static void do_st4_mmu(CPUState *cpu, vaddr addr, uint32_t val,
void *haddr;
MemOp mop = get_memop(oi);
- cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE);
if (mop & MO_BSWAP) {
@@ -1188,7 +1191,7 @@ static void do_st8_mmu(CPUState *cpu, vaddr addr, uint64_t val,
void *haddr;
MemOp mop = get_memop(oi);
- cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE);
if (mop & MO_BSWAP) {
@@ -1204,7 +1207,7 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
void *haddr;
MemOpIdx mop = get_memop(oi);
- cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE);
if (mop & MO_BSWAP) {
@@ -1214,101 +1217,28 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
clear_helper_retaddr();
}
-uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr ptr)
-{
- uint32_t ret;
-
- set_helper_retaddr(1);
- ret = ldub_p(g2h_untagged(ptr));
- clear_helper_retaddr();
- return ret;
-}
-
-uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr ptr)
-{
- uint32_t ret;
-
- set_helper_retaddr(1);
- ret = lduw_p(g2h_untagged(ptr));
- clear_helper_retaddr();
- return ret;
-}
-
-uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr ptr)
-{
- uint32_t ret;
-
- set_helper_retaddr(1);
- ret = ldl_p(g2h_untagged(ptr));
- clear_helper_retaddr();
- return ret;
-}
-
-uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr ptr)
-{
- uint64_t ret;
-
- set_helper_retaddr(1);
- ret = ldq_p(g2h_untagged(ptr));
- clear_helper_retaddr();
- return ret;
-}
-
-uint8_t cpu_ldb_code_mmu(CPUArchState *env, abi_ptr addr,
+uint8_t cpu_ldb_code_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t ra)
{
- void *haddr;
- uint8_t ret;
-
- haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_INST_FETCH);
- ret = ldub_p(haddr);
- clear_helper_retaddr();
- return ret;
+ return do_ld1_mmu(env_cpu(env), addr, oi, ra ? ra : 1, MMU_INST_FETCH);
}
-uint16_t cpu_ldw_code_mmu(CPUArchState *env, abi_ptr addr,
+uint16_t cpu_ldw_code_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t ra)
{
- void *haddr;
- uint16_t ret;
-
- haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_INST_FETCH);
- ret = lduw_p(haddr);
- clear_helper_retaddr();
- if (get_memop(oi) & MO_BSWAP) {
- ret = bswap16(ret);
- }
- return ret;
+ return do_ld2_mmu(env_cpu(env), addr, oi, ra ? ra : 1, MMU_INST_FETCH);
}
-uint32_t cpu_ldl_code_mmu(CPUArchState *env, abi_ptr addr,
+uint32_t cpu_ldl_code_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t ra)
{
- void *haddr;
- uint32_t ret;
-
- haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_INST_FETCH);
- ret = ldl_p(haddr);
- clear_helper_retaddr();
- if (get_memop(oi) & MO_BSWAP) {
- ret = bswap32(ret);
- }
- return ret;
+ return do_ld4_mmu(env_cpu(env), addr, oi, ra ? ra : 1, MMU_INST_FETCH);
}
-uint64_t cpu_ldq_code_mmu(CPUArchState *env, abi_ptr addr,
+uint64_t cpu_ldq_code_mmu(CPUArchState *env, vaddr addr,
MemOpIdx oi, uintptr_t ra)
{
- void *haddr;
- uint64_t ret;
-
- haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_DATA_LOAD);
- ret = ldq_p(haddr);
- clear_helper_retaddr();
- if (get_memop(oi) & MO_BSWAP) {
- ret = bswap64(ret);
- }
- return ret;
+ return do_ld8_mmu(env_cpu(env), addr, oi, ra ? ra : 1, MMU_INST_FETCH);
}
#include "ldst_common.c.inc"
diff --git a/accel/tcg/watchpoint.c b/accel/tcg/watchpoint.c
index 65b2188..cfb37a4 100644
--- a/accel/tcg/watchpoint.c
+++ b/accel/tcg/watchpoint.c
@@ -124,17 +124,14 @@ void cpu_check_watchpoint(CPUState *cpu, vaddr addr, vaddr len,
}
cpu->watchpoint_hit = wp;
- mmap_lock();
/* This call also restores vCPU state */
tb_check_watchpoint(cpu, ra);
if (wp->flags & BP_STOP_BEFORE_ACCESS) {
cpu->exception_index = EXCP_DEBUG;
- mmap_unlock();
cpu_loop_exit(cpu);
} else {
/* Force execution of one insn next time. */
cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
- mmap_unlock();
cpu_loop_exit_noexc(cpu);
}
} else {
generated by cgit v1.1 at 2025-04-26 04:31:33 +0000