diff options
| author | Richard Henderson <richard.henderson@linaro.org> | 2023-06-05 13:16:56 -0700 |
|---|---|---|
| committer | Richard Henderson <richard.henderson@linaro.org> | 2023-06-05 13:16:56 -0700 |
| commit | 369081c4558e7e940fa36ce59bf17b2e390f55d3 (patch) | |
| tree | 5e6d2b1861f9c1125b829afb2582e94a394692e5 | |
| parent | b52daaf2c868f2bab102eb5acbf55b2917f46aea (diff) | |
| parent | a7f6911c127b1dd1b8764e03b0ebcf0a227a15e4 (diff) | |
| download | qemu-369081c4558e7e940fa36ce59bf17b2e390f55d3.zip qemu-369081c4558e7e940fa36ce59bf17b2e390f55d3.tar.gz qemu-369081c4558e7e940fa36ce59bf17b2e390f55d3.tar.bz2 | |
Merge tag 'pull-tcg-20230605' of https://gitlab.com/rth7680/qemu into staging
Build tcg/ once for system and once for user.
Unmap perf_marker.
Remove left over _link_error() definitions.
# -----BEGIN PGP SIGNATURE-----
#
# iQFRBAABCgA7FiEEekgeeIaLTbaoWgXAZN846K9+IV8FAmR+QekdHHJpY2hhcmQu
# aGVuZGVyc29uQGxpbmFyby5vcmcACgkQZN846K9+IV8NJwf+PE6ShfN+N65hwt0w
# mgIrWNadgnXezEyaUdEuzLre9Ln9uD9Are948xIGPTVoLyr2wEr+Ma9dFI+GpTHY
# 8C0b5v0SfHBBIK+wJb7VNcEdssRXVTcpikmmYv9IRpFTT8349bCdvJhuwojU5cyx
# NGgq9ydJ/dEUDtmVU6EnOyLCDSHg2mc+KPVWhjXaVbVPTyP6Xmb0BwSSUt1t5NQK
# Zw7E6G4z7QYXa7GNPTtXSWzTH3y9bSLQNj3jzHfJ6DPW23zxhFjCPhJYl1ecOHVF
# pxXEEIjieHQxdlm3kg+tcCoZSJ6OXESNzu2D8zk8Kf6xJjMItTQOOuokWM6mBX4y
# VbQoXg==
# =8vrl
# -----END PGP SIGNATURE-----
# gpg: Signature made Mon 05 Jun 2023 01:13:29 PM PDT
# gpg: using RSA key 7A481E78868B4DB6A85A05C064DF38E8AF7E215F
# gpg: issuer "richard.henderson@linaro.org"
# gpg: Good signature from "Richard Henderson <richard.henderson@linaro.org>" [ultimate]
* tag 'pull-tcg-20230605' of https://gitlab.com/rth7680/qemu: (52 commits)
tcg/tcg-op-vec: Remove left over _link_error() definitions
accel/tcg: Unmap perf_marker
tcg: Build once for system and once for user-only
exec/poison: Do not poison CONFIG_SOFTMMU
plugins: Drop unused headers from exec/plugin-gen.h
plugins: Move plugin_insn_append to translator.c
tcg: Remove target-specific headers from tcg.[ch]
tcg: Move env defines out of NEED_CPU_H in helper-head.h
tcg: Fix PAGE/PROT confusion
accel/tcg: Tidy includes for translator.[ch]
target/arm: Add missing include of exec/exec-all.h
target/*: Add missing includes of exec/translation-block.h
target/mips: Tidy helpers for translation
target/arm: Tidy helpers for translation
accel/tcg: Move translator_fake_ldb out of line
target/ppc: Inline gen_icount_io_start()
accel/tcg: Introduce translator_io_start
accel/tcg: Move most of gen-icount.h into translator.c
include/exec: Remove CODE_GEN_AVG_BLOCK_SIZE
tcg: Spit out exec/translation-block.h
...
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
135 files changed, 3084 insertions, 2778 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index 89f274f..55668d6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -154,6 +154,8 @@ F: include/exec/exec-all.h F: include/exec/tb-flush.h F: include/exec/target_long.h F: include/exec/helper*.h +F: include/exec/helper*.h.inc +F: include/exec/helper-info.c.inc F: include/sysemu/cpus.h F: include/sysemu/tcg.h F: include/hw/core/tcg-cpu-ops.h @@ -2865,7 +2867,6 @@ F: ui/cocoa.m Main loop M: Paolo Bonzini <pbonzini@redhat.com> S: Maintained -F: include/exec/gen-icount.h F: include/qemu/main-loop.h F: include/sysemu/runstate.h F: include/sysemu/runstate-action.h diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index 60ca9e2..1cf4f1f 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -297,7 +297,7 @@ static void log_cpu_exec(target_ulong pc, CPUState *cpu, { if (qemu_log_in_addr_range(pc)) { qemu_log_mask(CPU_LOG_EXEC, - "Trace %d: %p [" TARGET_FMT_lx + "Trace %d: %p [%08" PRIx64 "/" TARGET_FMT_lx "/%08x/%08x] %s\n", cpu->cpu_index, tb->tc.ptr, tb->cs_base, pc, tb->flags, tb->cflags, lookup_symbol(pc)); diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 90c72c9..5e2ca47 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -29,7 +29,7 @@ #include "tcg/tcg.h" #include "qemu/error-report.h" #include "exec/log.h" -#include "exec/helper-proto.h" +#include "exec/helper-proto-common.h" #include "qemu/atomic.h" #include "qemu/atomic128.h" #include "exec/translate-all.h" @@ -40,7 +40,7 @@ #include "qemu/plugin-memory.h" #endif #include "tcg/tcg-ldst.h" -#include "exec/helper-proto.h" +#include "tcg/oversized-guest.h" /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */ /* #define DEBUG_TLB */ @@ -1000,11 +1000,15 @@ static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry, addr &= TARGET_PAGE_MASK; addr += tlb_entry->addend; if ((addr - start) < length) { -#if TCG_OVERSIZED_GUEST +#if TARGET_LONG_BITS == 32 + uint32_t *ptr_write = (uint32_t *)&tlb_entry->addr_write; + ptr_write += HOST_BIG_ENDIAN; + qatomic_set(ptr_write, *ptr_write | TLB_NOTDIRTY); +#elif TCG_OVERSIZED_GUEST tlb_entry->addr_write |= TLB_NOTDIRTY; #else qatomic_set(&tlb_entry->addr_write, - tlb_entry->addr_write | TLB_NOTDIRTY); + tlb_entry->addr_write | TLB_NOTDIRTY); #endif } } diff --git a/accel/tcg/monitor.c b/accel/tcg/monitor.c index 92fce58..f171bc6 100644 --- a/accel/tcg/monitor.c +++ b/accel/tcg/monitor.c @@ -15,6 +15,7 @@ #include "sysemu/cpus.h" #include "sysemu/cpu-timers.h" #include "sysemu/tcg.h" +#include "tcg/tcg.h" #include "internal.h" diff --git a/accel/tcg/perf.c b/accel/tcg/perf.c index 65e35ea..cd1aa99 100644 --- a/accel/tcg/perf.c +++ b/accel/tcg/perf.c @@ -111,6 +111,8 @@ static void write_perfmap_entry(const void *start, size_t insn, } static FILE *jitdump; +static size_t perf_marker_size; +static void *perf_marker = MAP_FAILED; #define JITHEADER_MAGIC 0x4A695444 #define JITHEADER_VERSION 1 @@ -190,7 +192,6 @@ void perf_enable_jitdump(void) { struct jitheader header; char jitdump_file[32]; - void *perf_marker; if (!use_rt_clock) { warn_report("CLOCK_MONOTONIC is not available, proceeding without jitdump"); @@ -210,7 +211,8 @@ void perf_enable_jitdump(void) * PERF_RECORD_MMAP or PERF_RECORD_MMAP2 event is of the form jit-%d.dump * and will process it as a jitdump file. */ - perf_marker = mmap(NULL, qemu_real_host_page_size(), PROT_READ | PROT_EXEC, + perf_marker_size = qemu_real_host_page_size(); + perf_marker = mmap(NULL, perf_marker_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fileno(jitdump), 0); if (perf_marker == MAP_FAILED) { warn_report("Could not map %s: %s, proceeding without jitdump", @@ -311,7 +313,8 @@ void perf_report_code(uint64_t guest_pc, TranslationBlock *tb, const void *start) { struct debuginfo_query *q; - size_t insn; + size_t insn, start_words; + uint64_t *gen_insn_data; if (!perfmap && !jitdump) { return; @@ -325,9 +328,12 @@ void perf_report_code(uint64_t guest_pc, TranslationBlock *tb, debuginfo_lock(); /* Query debuginfo for each guest instruction. */ + gen_insn_data = tcg_ctx->gen_insn_data; + start_words = tcg_ctx->insn_start_words; + for (insn = 0; insn < tb->icount; insn++) { /* FIXME: This replicates the restore_state_to_opc() logic. */ - q[insn].address = tcg_ctx->gen_insn_data[insn][0]; + q[insn].address = gen_insn_data[insn * start_words + 0]; if (tb_cflags(tb) & CF_PCREL) { q[insn].address |= (guest_pc & TARGET_PAGE_MASK); } else { @@ -368,6 +374,11 @@ void perf_exit(void) perfmap = NULL; } + if (perf_marker != MAP_FAILED) { + munmap(perf_marker, perf_marker_size); + perf_marker = MAP_FAILED; + } + if (jitdump) { fclose(jitdump); jitdump = NULL; diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index 5b73a39..5c13615 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -43,12 +43,18 @@ * CPU's index into a TCG temp, since the first callback did it already. */ #include "qemu/osdep.h" +#include "cpu.h" #include "tcg/tcg.h" #include "tcg/tcg-temp-internal.h" #include "tcg/tcg-op.h" #include "exec/exec-all.h" #include "exec/plugin-gen.h" #include "exec/translator.h" +#include "exec/helper-proto-common.h" + +#define HELPER_H "accel/tcg/plugin-helpers.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H #ifdef CONFIG_SOFTMMU # define CONFIG_SOFTMMU_GATE 1 diff --git a/accel/tcg/tcg-accel-ops-mttcg.c b/accel/tcg/tcg-accel-ops-mttcg.c index d50239e..5d72c9b 100644 --- a/accel/tcg/tcg-accel-ops-mttcg.c +++ b/accel/tcg/tcg-accel-ops-mttcg.c @@ -32,7 +32,7 @@ #include "qemu/guest-random.h" #include "exec/exec-all.h" #include "hw/boards.h" - +#include "tcg/tcg.h" #include "tcg-accel-ops.h" #include "tcg-accel-ops-mttcg.h" diff --git a/accel/tcg/tcg-accel-ops-rr.c b/accel/tcg/tcg-accel-ops-rr.c index b6d10fa..70b9b89 100644 --- a/accel/tcg/tcg-accel-ops-rr.c +++ b/accel/tcg/tcg-accel-ops-rr.c @@ -32,7 +32,7 @@ #include "qemu/notify.h" #include "qemu/guest-random.h" #include "exec/exec-all.h" - +#include "tcg/tcg.h" #include "tcg-accel-ops.h" #include "tcg-accel-ops-rr.h" #include "tcg-accel-ops-icount.h" diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c index a831f8d..02af6a2 100644 --- a/accel/tcg/tcg-all.c +++ b/accel/tcg/tcg-all.c @@ -28,6 +28,7 @@ #include "exec/replay-core.h" #include "sysemu/cpu-timers.h" #include "tcg/tcg.h" +#include "tcg/oversized-guest.h" #include "qapi/error.h" #include "qemu/error-report.h" #include "qemu/accel.h" diff --git a/accel/tcg/tcg-runtime-gvec.c b/accel/tcg/tcg-runtime-gvec.c index 9739949..6c99f95 100644 --- a/accel/tcg/tcg-runtime-gvec.c +++ b/accel/tcg/tcg-runtime-gvec.c @@ -20,7 +20,7 @@ #include "qemu/osdep.h" #include "qemu/host-utils.h" #include "cpu.h" -#include "exec/helper-proto.h" +#include "exec/helper-proto-common.h" #include "tcg/tcg-gvec-desc.h" diff --git a/accel/tcg/tcg-runtime.c b/accel/tcg/tcg-runtime.c index e4e0300..9fa539a 100644 --- a/accel/tcg/tcg-runtime.c +++ b/accel/tcg/tcg-runtime.c @@ -24,13 +24,17 @@ #include "qemu/osdep.h" #include "qemu/host-utils.h" #include "cpu.h" -#include "exec/helper-proto.h" +#include "exec/helper-proto-common.h" #include "exec/cpu_ldst.h" #include "exec/exec-all.h" #include "disas/disas.h" #include "exec/log.h" #include "tcg/tcg.h" +#define HELPER_H "accel/tcg/tcg-runtime.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + /* 32-bit helpers */ int32_t HELPER(div_i32)(int32_t arg1, int32_t arg2) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index bf814b9..c4d081f 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -19,7 +19,6 @@ #include "qemu/osdep.h" -#define NO_CPU_IO_DEFS #include "trace.h" #include "disas/disas.h" #include "exec/exec-all.h" @@ -64,6 +63,7 @@ #include "tb-context.h" #include "internal.h" #include "perf.h" +#include "tcg/insn-start-words.h" TBContext tb_ctx; @@ -127,22 +127,26 @@ static int64_t decode_sleb128(const uint8_t **pp) static int encode_search(TranslationBlock *tb, uint8_t *block) { uint8_t *highwater = tcg_ctx->code_gen_highwater; + uint64_t *insn_data = tcg_ctx->gen_insn_data; + uint16_t *insn_end_off = tcg_ctx->gen_insn_end_off; uint8_t *p = block; int i, j, n; for (i = 0, n = tb->icount; i < n; ++i) { - uint64_t prev; + uint64_t prev, curr; for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { if (i == 0) { prev = (!(tb_cflags(tb) & CF_PCREL) && j == 0 ? tb->pc : 0); } else { - prev = tcg_ctx->gen_insn_data[i - 1][j]; + prev = insn_data[(i - 1) * TARGET_INSN_START_WORDS + j]; } - p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); + curr = insn_data[i * TARGET_INSN_START_WORDS + j]; + p = encode_sleb128(p, curr - prev); } - prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); - p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); + prev = (i == 0 ? 0 : insn_end_off[i - 1]); + curr = insn_end_off[i]; + p = encode_sleb128(p, curr - prev); /* Test for (pending) buffer overflow. The assumption is that any one row beginning below the high water mark cannot overrun @@ -350,11 +354,19 @@ TranslationBlock *tb_gen_code(CPUState *cpu, tb_set_page_addr0(tb, phys_pc); tb_set_page_addr1(tb, -1); tcg_ctx->gen_tb = tb; - tcg_ctx->addr_type = TCG_TYPE_TL; + tcg_ctx->addr_type = TARGET_LONG_BITS == 32 ? TCG_TYPE_I32 : TCG_TYPE_I64; #ifdef CONFIG_SOFTMMU tcg_ctx->page_bits = TARGET_PAGE_BITS; tcg_ctx->page_mask = TARGET_PAGE_MASK; tcg_ctx->tlb_dyn_max_bits = CPU_TLB_DYN_MAX_BITS; + tcg_ctx->tlb_fast_offset = + (int)offsetof(ArchCPU, neg.tlb.f) - (int)offsetof(ArchCPU, env); +#endif + tcg_ctx->insn_start_words = TARGET_INSN_START_WORDS; +#ifdef TCG_GUEST_DEFAULT_MO + tcg_ctx->guest_mo = TCG_GUEST_DEFAULT_MO; +#else + tcg_ctx->guest_mo = TCG_MO_ALL; #endif tb_overflow: @@ -449,7 +461,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, fprintf(logfile, "OUT: [size=%d]\n", gen_code_size); fprintf(logfile, " -- guest addr 0x%016" PRIx64 " + tb prologue\n", - tcg_ctx->gen_insn_data[insn][0]); + tcg_ctx->gen_insn_data[insn * TARGET_INSN_START_WORDS]); chunk_start = tcg_ctx->gen_insn_end_off[insn]; disas(logfile, tb->tc.ptr, chunk_start); @@ -462,7 +474,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, size_t chunk_end = tcg_ctx->gen_insn_end_off[insn]; if (chunk_end > chunk_start) { fprintf(logfile, " -- guest addr 0x%016" PRIx64 "\n", - tcg_ctx->gen_insn_data[insn][0]); + tcg_ctx->gen_insn_data[insn * TARGET_INSN_START_WORDS]); disas(logfile, tb->tc.ptr + chunk_start, chunk_end - chunk_start); chunk_start = chunk_end; diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index 6120ef2..918a455 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -8,15 +8,114 @@ */ #include "qemu/osdep.h" +#include "qemu/log.h" #include "qemu/error-report.h" -#include "tcg/tcg.h" -#include "tcg/tcg-op.h" #include "exec/exec-all.h" -#include "exec/gen-icount.h" -#include "exec/log.h" #include "exec/translator.h" +#include "exec/translate-all.h" #include "exec/plugin-gen.h" -#include "exec/replay-core.h" +#include "tcg/tcg-op-common.h" + +static void gen_io_start(void) +{ + tcg_gen_st_i32(tcg_constant_i32(1), cpu_env, + offsetof(ArchCPU, parent_obj.can_do_io) - + offsetof(ArchCPU, env)); +} + +bool translator_io_start(DisasContextBase *db) +{ + uint32_t cflags = tb_cflags(db->tb); + + if (!(cflags & CF_USE_ICOUNT)) { + return false; + } + if (db->num_insns == db->max_insns && (cflags & CF_LAST_IO)) { + /* Already started in translator_loop. */ + return true; + } + + gen_io_start(); + + /* + * Ensure that this instruction will be the last in the TB. + * The target may override this to something more forceful. + */ + if (db->is_jmp == DISAS_NEXT) { + db->is_jmp = DISAS_TOO_MANY; + } + return true; +} + +static TCGOp *gen_tb_start(uint32_t cflags) +{ + TCGv_i32 count = tcg_temp_new_i32(); + TCGOp *icount_start_insn = NULL; + + tcg_gen_ld_i32(count, cpu_env, + offsetof(ArchCPU, neg.icount_decr.u32) - + offsetof(ArchCPU, env)); + + if (cflags & CF_USE_ICOUNT) { + /* + * We emit a sub with a dummy immediate argument. Keep the insn index + * of the sub so that we later (when we know the actual insn count) + * can update the argument with the actual insn count. + */ + tcg_gen_sub_i32(count, count, tcg_constant_i32(0)); + icount_start_insn = tcg_last_op(); + } + + /* + * Emit the check against icount_decr.u32 to see if we should exit + * unless we suppress the check with CF_NOIRQ. If we are using + * icount and have suppressed interruption the higher level code + * should have ensured we don't run more instructions than the + * budget. + */ + if (cflags & CF_NOIRQ) { + tcg_ctx->exitreq_label = NULL; + } else { + tcg_ctx->exitreq_label = gen_new_label(); + tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, tcg_ctx->exitreq_label); + } + + if (cflags & CF_USE_ICOUNT) { + tcg_gen_st16_i32(count, cpu_env, + offsetof(ArchCPU, neg.icount_decr.u16.low) - + offsetof(ArchCPU, env)); + /* + * cpu->can_do_io is cleared automatically here at the beginning of + * each translation block. The cost is minimal and only paid for + * -icount, plus it would be very easy to forget doing it in the + * translator. Doing it here means we don't need a gen_io_end() to + * go with gen_io_start(). + */ + tcg_gen_st_i32(tcg_constant_i32(0), cpu_env, + offsetof(ArchCPU, parent_obj.can_do_io) - + offsetof(ArchCPU, env)); + } + + return icount_start_insn; +} + +static void gen_tb_end(const TranslationBlock *tb, uint32_t cflags, + TCGOp *icount_start_insn, int num_insns) +{ + if (cflags & CF_USE_ICOUNT) { + /* + * Update the num_insn immediate parameter now that we know + * the actual insn count. + */ + tcg_set_insn_param(icount_start_insn, 2, + tcgv_i32_arg(tcg_constant_i32(num_insns))); + } + + if (tcg_ctx->exitreq_label) { + gen_set_label(tcg_ctx->exitreq_label); + tcg_gen_exit_tb(tb, TB_EXIT_REQUESTED); + } +} bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest) { @@ -34,6 +133,7 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns, const TranslatorOps *ops, DisasContextBase *db) { uint32_t cflags = tb_cflags(tb); + TCGOp *icount_start_insn; bool plugin_enabled; /* Initialize DisasContext */ @@ -55,7 +155,7 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns, tcg_debug_assert(db->is_jmp == DISAS_NEXT); /* no early exit */ /* Start translating. */ - gen_tb_start(db->tb); + icount_start_insn = gen_tb_start(cflags); ops->tb_start(db, cpu); tcg_debug_assert(db->is_jmp == DISAS_NEXT); /* no early exit */ @@ -112,7 +212,7 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns, /* Emit code to exit the TB, as indicated by db->is_jmp. */ ops->tb_stop(db, cpu); - gen_tb_end(db->tb, db->num_insns); + gen_tb_end(tb, cflags, icount_start_insn, db->num_insns); if (plugin_enabled) { plugin_gen_tb_end(cpu); @@ -185,6 +285,27 @@ static void *translator_access(CPUArchState *env, DisasContextBase *db, return host + (pc - base); } +static void plugin_insn_append(abi_ptr pc, const void *from, size_t size) +{ +#ifdef CONFIG_PLUGIN + struct qemu_plugin_insn *insn = tcg_ctx->plugin_insn; + abi_ptr off; + + if (insn == NULL) { + return; + } + off = pc - insn->vaddr; + if (off < insn->data->len) { + g_byte_array_set_size(insn->data, off); + } else if (off > insn->data->len) { + /* we have an unexpected gap */ + g_assert_not_reached(); + } + + insn->data = g_byte_array_append(insn->data, from, size); +#endif +} + uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, abi_ptr pc) { uint8_t ret; @@ -243,3 +364,8 @@ uint64_t translator_ldq(CPUArchState *env, DisasContextBase *db, abi_ptr pc) plugin_insn_append(pc, &plug, sizeof(ret)); return ret; } + +void translator_fake_ldb(uint8_t insn8, abi_ptr pc) +{ + plugin_insn_append(pc, &insn8, sizeof(insn8)); +} diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index 78d258a..09bf4c0 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -314,6 +314,9 @@ CPUArchState *cpu_copy(CPUArchState *env); * * Use TARGET_PAGE_BITS_MIN so that these bits are constant * when TARGET_PAGE_BITS_VARY is in effect. + * + * The count, if not the placement of these bits is known + * to tcg/tcg-op-ldst.c, check_max_alignment(). */ /* Zero if TLB entry is valid. */ #define TLB_INVALID_MASK (1 << (TARGET_PAGE_BITS_MIN - 1)) diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index a6e0cf1..0d418a0 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -61,16 +61,11 @@ #define NB_MMU_MODES 16 #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG) +#include "exec/tlb-common.h" /* use a fully associative victim tlb of 8 entries */ #define CPU_VTLB_SIZE 8 -#if HOST_LONG_BITS == 32 && TARGET_LONG_BITS == 32 -#define CPU_TLB_ENTRY_BITS 4 -#else -#define CPU_TLB_ENTRY_BITS 5 -#endif - #define CPU_TLB_DYN_MIN_BITS 6 #define CPU_TLB_DYN_DEFAULT_BITS 8 @@ -94,34 +89,6 @@ # endif # endif -/* Minimalized TLB entry for use by TCG fast path. */ -typedef struct CPUTLBEntry { - /* bit TARGET_LONG_BITS to TARGET_PAGE_BITS : virtual address - bit TARGET_PAGE_BITS-1..4 : Nonzero for accesses that should not - go directly to ram. - bit 3 : indicates that the entry is invalid - bit 2..0 : zero - */ - union { - struct { - target_ulong addr_read; - target_ulong addr_write; - target_ulong addr_code; - /* Addend to virtual address to get host address. IO accesses - use the corresponding iotlb value. */ - uintptr_t addend; - }; - /* - * Padding to get a power of two size, as well as index - * access to addr_{read,write,code}. - */ - target_ulong addr_idx[(1 << CPU_TLB_ENTRY_BITS) / TARGET_LONG_SIZE]; - }; -} CPUTLBEntry; - -QEMU_BUILD_BUG_ON(sizeof(CPUTLBEntry) != (1 << CPU_TLB_ENTRY_BITS)); - - #endif /* !CONFIG_USER_ONLY && CONFIG_TCG */ #if !defined(CONFIG_USER_ONLY) @@ -196,17 +163,6 @@ typedef struct CPUTLBDesc { } CPUTLBDesc; /* - * Data elements that are per MMU mode, accessed by the fast path. - * The structure is aligned to aid loading the pair with one insn. - */ -typedef struct CPUTLBDescFast { - /* Contains (n_entries - 1) << CPU_TLB_ENTRY_BITS */ - uintptr_t mask; - /* The array of tlb entries itself. */ - CPUTLBEntry *table; -} CPUTLBDescFast QEMU_ALIGNED(2 * sizeof(void *)); - -/* * Data elements that are shared between all MMU modes. */ typedef struct CPUTLBCommon { @@ -241,10 +197,6 @@ typedef struct CPUTLB { CPUTLBDescFast f[NB_MMU_MODES]; } CPUTLB; -/* This will be used by TCG backends to compute offsets. */ -#define TLB_MASK_TABLE_OFS(IDX) \ - ((int)offsetof(ArchCPU, neg.tlb.f[IDX]) - (int)offsetof(ArchCPU, env)) - #else typedef struct CPUTLB { } CPUTLB; diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h index 5939688..896f305 100644 --- a/include/exec/cpu_ldst.h +++ b/include/exec/cpu_ldst.h @@ -326,26 +326,32 @@ static inline void clear_helper_retaddr(void) #else -/* Needed for TCG_OVERSIZED_GUEST */ -#include "tcg/tcg.h" +#include "tcg/oversized-guest.h" static inline target_ulong tlb_read_idx(const CPUTLBEntry *entry, MMUAccessType access_type) { /* Do not rearrange the CPUTLBEntry structure members. */ QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_read) != - MMU_DATA_LOAD * TARGET_LONG_SIZE); + MMU_DATA_LOAD * sizeof(uint64_t)); QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_write) != - MMU_DATA_STORE * TARGET_LONG_SIZE); + MMU_DATA_STORE * sizeof(uint64_t)); QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_code) != - MMU_INST_FETCH * TARGET_LONG_SIZE); + MMU_INST_FETCH * sizeof(uint64_t)); - const target_ulong *ptr = &entry->addr_idx[access_type]; -#if TCG_OVERSIZED_GUEST - return *ptr; +#if TARGET_LONG_BITS == 32 + /* Use qatomic_read, in case of addr_write; only care about low bits. */ + const uint32_t *ptr = (uint32_t *)&entry->addr_idx[access_type]; + ptr += HOST_BIG_ENDIAN; + return qatomic_read(ptr); #else + const uint64_t *ptr = &entry->addr_idx[access_type]; +# if TCG_OVERSIZED_GUEST + return *ptr; +# else /* ofs might correspond to .addr_write, so use qatomic_read */ return qatomic_read(ptr); +# endif #endif } diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 3b1b57f..698943d 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -24,20 +24,9 @@ #ifdef CONFIG_TCG #include "exec/cpu_ldst.h" #endif -#include "qemu/interval-tree.h" +#include "exec/translation-block.h" #include "qemu/clang-tsa.h" -/* Page tracking code uses ram addresses in system mode, and virtual - addresses in userspace mode. Define tb_page_addr_t to be an appropriate - type. */ -#if defined(CONFIG_USER_ONLY) -typedef abi_ulong tb_page_addr_t; -#define TB_PAGE_ADDR_FMT TARGET_ABI_FMT_lx -#else -typedef ram_addr_t tb_page_addr_t; -#define TB_PAGE_ADDR_FMT RAM_ADDR_FMT -#endif - /** * cpu_unwind_state_data: * @cpu: the cpu context @@ -478,135 +467,6 @@ int probe_access_full(CPUArchState *env, target_ulong addr, int size, CPUTLBEntryFull **pfull, uintptr_t retaddr); #endif -#define CODE_GEN_ALIGN 16 /* must be >= of the size of a icache line */ - -/* Estimated block size for TB allocation. */ -/* ??? The following is based on a 2015 survey of x86_64 host output. - Better would seem to be some sort of dynamically sized TB array, - adapting to the block sizes actually being produced. */ -#if defined(CONFIG_SOFTMMU) -#define CODE_GEN_AVG_BLOCK_SIZE 400 -#else -#define CODE_GEN_AVG_BLOCK_SIZE 150 -#endif - -/* - * Translation Cache-related fields of a TB. - * This struct exists just for convenience; we keep track of TB's in a binary - * search tree, and the only fields needed to compare TB's in the tree are - * @ptr and @size. - * Note: the address of search data can be obtained by adding @size to @ptr. - */ -struct tb_tc { - const void *ptr; /* pointer to the translated code */ - size_t size; -}; - -struct TranslationBlock { - /* - * Guest PC corresponding to this block. This must be the true - * virtual address. Therefore e.g. x86 stores EIP + CS_BASE, and - * targets like Arm, MIPS, HP-PA, which reuse low bits for ISA or - * privilege, must store those bits elsewhere. - * - * If CF_PCREL, the opcodes for the TranslationBlock are written - * such that the TB is associated only with the physical page and - * may be run in any virtual address context. In this case, PC - * must always be taken from ENV in a target-specific manner. - * Unwind information is taken as offsets from the page, to be - * deposited into the "current" PC. - */ - target_ulong pc; - - /* - * Target-specific data associated with the TranslationBlock, e.g.: - * x86: the original user, the Code Segment virtual base, - * arm: an extension of tb->flags, - * s390x: instruction data for EXECUTE, - * sparc: the next pc of the instruction queue (for delay slots). - */ - target_ulong cs_base; - - uint32_t flags; /* flags defining in which context the code was generated */ - uint32_t cflags; /* compile flags */ - -/* Note that TCG_MAX_INSNS is 512; we validate this match elsewhere. */ -#define CF_COUNT_MASK 0x000001ff -#define CF_NO_GOTO_TB 0x00000200 /* Do not chain with goto_tb */ -#define CF_NO_GOTO_PTR 0x00000400 /* Do not chain with goto_ptr */ -#define CF_SINGLE_STEP 0x00000800 /* gdbstub single-step in effect */ -#define CF_LAST_IO 0x00008000 /* Last insn may be an IO access. */ -#define CF_MEMI_ONLY 0x00010000 /* Only instrument memory ops */ -#define CF_USE_ICOUNT 0x00020000 -#define CF_INVALID 0x00040000 /* TB is stale. Set with @jmp_lock held */ -#define CF_PARALLEL 0x00080000 /* Generate code for a parallel context */ -#define CF_NOIRQ 0x00100000 /* Generate an uninterruptible TB */ -#define CF_PCREL 0x00200000 /* Opcodes in TB are PC-relative */ -#define CF_CLUSTER_MASK 0xff000000 /* Top 8 bits are cluster ID */ -#define CF_CLUSTER_SHIFT 24 - - /* - * Above fields used for comparing - */ - - /* size of target code for this block (1 <= size <= TARGET_PAGE_SIZE) */ - uint16_t size; - uint16_t icount; - - struct tb_tc tc; - - /* - * Track tb_page_addr_t intervals that intersect this TB. - * For user-only, the virtual addresses are always contiguous, - * and we use a unified interval tree. For system, we use a - * linked list headed in each PageDesc. Within the list, the lsb - * of the previous pointer tells the index of page_next[], and the - * list is protected by the PageDesc lock(s). - */ -#ifdef CONFIG_USER_ONLY - IntervalTreeNode itree; -#else - uintptr_t page_next[2]; - tb_page_addr_t page_addr[2]; -#endif - - /* jmp_lock placed here to fill a 4-byte hole. Its documentation is below */ - QemuSpin jmp_lock; - - /* The following data are used to directly call another TB from - * the code of this one. This can be done either by emitting direct or - * indirect native jump instructions. These jumps are reset so that the TB - * just continues its execution. The TB can be linked to another one by - * setting one of the jump targets (or patching the jump instruction). Only - * two of such jumps are supported. - */ -#define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */ - uint16_t jmp_reset_offset[2]; /* offset of original jump target */ - uint16_t jmp_insn_offset[2]; /* offset of direct jump insn */ - uintptr_t jmp_target_addr[2]; /* target address */ - - /* - * Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps. - * Each TB can have two outgoing jumps, and therefore can participate - * in two lists. The list entries are kept in jmp_list_next[2]. The least - * significant bit (LSB) of the pointers in these lists is used to encode - * which of the two list entries is to be used in the pointed TB. - * - * List traversals are protected by jmp_lock. The destination TB of each - * outgoing jump is kept in jmp_dest[] so that the appropriate jmp_lock - * can be acquired from any origin TB. - * - * jmp_dest[] are tagged pointers as well. The LSB is set when the TB is - * being invalidated, so that no further outgoing jumps from it can be set. - * - * jmp_lock also protects the CF_INVALID cflag; a jump must not be chained - * to a destination TB that has CF_INVALID set. - */ - uintptr_t jmp_list_head; - uintptr_t jmp_list_next[2]; - uintptr_t jmp_dest[2]; -}; - /* Hide the qatomic_read to make code a little easier on the eyes */ static inline uint32_t tb_cflags(const TranslationBlock *tb) { diff --git a/include/exec/gen-icount.h b/include/exec/gen-icount.h deleted file mode 100644 index f6de79a..0000000 --- a/include/exec/gen-icount.h +++ /dev/null @@ -1,83 +0,0 @@ -#ifndef GEN_ICOUNT_H -#define GEN_ICOUNT_H - -#include "exec/exec-all.h" - -/* Helpers for instruction counting code generation. */ - -static TCGOp *icount_start_insn; - -static inline void gen_io_start(void) -{ - tcg_gen_st_i32(tcg_constant_i32(1), cpu_env, - offsetof(ArchCPU, parent_obj.can_do_io) - - offsetof(ArchCPU, env)); -} - -static inline void gen_tb_start(const TranslationBlock *tb) -{ - TCGv_i32 count = tcg_temp_new_i32(); - - tcg_gen_ld_i32(count, cpu_env, - offsetof(ArchCPU, neg.icount_decr.u32) - - offsetof(ArchCPU, env)); - - if (tb_cflags(tb) & CF_USE_ICOUNT) { - /* - * We emit a sub with a dummy immediate argument. Keep the insn index - * of the sub so that we later (when we know the actual insn count) - * can update the argument with the actual insn count. - */ - tcg_gen_sub_i32(count, count, tcg_constant_i32(0)); - icount_start_insn = tcg_last_op(); - } - - /* - * Emit the check against icount_decr.u32 to see if we should exit - * unless we suppress the check with CF_NOIRQ. If we are using - * icount and have suppressed interruption the higher level code - * should have ensured we don't run more instructions than the - * budget. - */ - if (tb_cflags(tb) & CF_NOIRQ) { - tcg_ctx->exitreq_label = NULL; - } else { - tcg_ctx->exitreq_label = gen_new_label(); - tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, tcg_ctx->exitreq_label); - } - - if (tb_cflags(tb) & CF_USE_ICOUNT) { - tcg_gen_st16_i32(count, cpu_env, - offsetof(ArchCPU, neg.icount_decr.u16.low) - - offsetof(ArchCPU, env)); - /* - * cpu->can_do_io is cleared automatically here at the beginning of - * each translation block. The cost is minimal and only paid for - * -icount, plus it would be very easy to forget doing it in the - * translator. Doing it here means we don't need a gen_io_end() to - * go with gen_io_start(). - */ - tcg_gen_st_i32(tcg_constant_i32(0), cpu_env, - offsetof(ArchCPU, parent_obj.can_do_io) - - offsetof(ArchCPU, env)); - } -} - -static inline void gen_tb_end(const TranslationBlock *tb, int num_insns) -{ - if (tb_cflags(tb) & CF_USE_ICOUNT) { - /* - * Update the num_insn immediate parameter now that we know - * the actual insn count. - */ - tcg_set_insn_param(icount_start_insn, 2, - tcgv_i32_arg(tcg_constant_i32(num_insns))); - } - - if (tcg_ctx->exitreq_label) { - gen_set_label(tcg_ctx->exitreq_label); - tcg_gen_exit_tb(tb, TB_EXIT_REQUESTED); - } -} - -#endif diff --git a/include/exec/helper-gen-common.h b/include/exec/helper-gen-common.h new file mode 100644 index 0000000..5d6d78a --- /dev/null +++ b/include/exec/helper-gen-common.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands generation functions for tcg opcodes. + */ + +#ifndef HELPER_GEN_COMMON_H +#define HELPER_GEN_COMMON_H + +#define HELPER_H "accel/tcg/tcg-runtime.h" +#include "exec/helper-gen.h.inc" +#undef HELPER_H + +#define HELPER_H "accel/tcg/plugin-helpers.h" +#include "exec/helper-gen.h.inc" +#undef HELPER_H + +#endif /* HELPER_GEN_COMMON_H */ diff --git a/include/exec/helper-gen.h b/include/exec/helper-gen.h index 7b6ca97..f7ec155 100644 --- a/include/exec/helper-gen.h +++ b/include/exec/helper-gen.h @@ -1,95 +1,16 @@ -/* Helper file for declaring TCG helper functions. - This one expands generation functions for tcg opcodes. */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands generation functions for tcg opcodes. + */ #ifndef HELPER_GEN_H #define HELPER_GEN_H -#include "exec/helper-head.h" +#include "exec/helper-gen-common.h" -#define DEF_HELPER_FLAGS_0(name, flags, ret) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl0(ret)) \ -{ \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 0, NULL); \ -} - -#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1)) \ -{ \ - TCGTemp *args[1] = { dh_arg(t1, 1) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 1, args); \ -} - -#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2)) \ -{ \ - TCGTemp *args[2] = { dh_arg(t1, 1), dh_arg(t2, 2) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 2, args); \ -} - -#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3)) \ -{ \ - TCGTemp *args[3] = { dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 3, args); \ -} - -#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), \ - dh_arg_decl(t3, 3), dh_arg_decl(t4, 4)) \ -{ \ - TCGTemp *args[4] = { dh_arg(t1, 1), dh_arg(t2, 2), \ - dh_arg(t3, 3), dh_arg(t4, 4) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 4, args); \ -} - -#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ - dh_arg_decl(t4, 4), dh_arg_decl(t5, 5)) \ -{ \ - TCGTemp *args[5] = { dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ - dh_arg(t4, 4), dh_arg(t5, 5) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 5, args); \ -} - -#define DEF_HELPER_FLAGS_6(name, flags, ret, t1, t2, t3, t4, t5, t6) \ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ - dh_arg_decl(t4, 4), dh_arg_decl(t5, 5), dh_arg_decl(t6, 6)) \ -{ \ - TCGTemp *args[6] = { dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ - dh_arg(t4, 4), dh_arg(t5, 5), dh_arg(t6, 6) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 6, args); \ -} - -#define DEF_HELPER_FLAGS_7(name, flags, ret, t1, t2, t3, t4, t5, t6, t7)\ -static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ - dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ - dh_arg_decl(t4, 4), dh_arg_decl(t5, 5), dh_arg_decl(t6, 6), \ - dh_arg_decl(t7, 7)) \ -{ \ - TCGTemp *args[7] = { dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ - dh_arg(t4, 4), dh_arg(t5, 5), dh_arg(t6, 6), \ - dh_arg(t7, 7) }; \ - tcg_gen_callN(HELPER(name), dh_retvar(ret), 7, args); \ -} - -#include "helper.h" -#include "accel/tcg/tcg-runtime.h" -#include "accel/tcg/plugin-helpers.h" - -#undef DEF_HELPER_FLAGS_0 -#undef DEF_HELPER_FLAGS_1 -#undef DEF_HELPER_FLAGS_2 -#undef DEF_HELPER_FLAGS_3 -#undef DEF_HELPER_FLAGS_4 -#undef DEF_HELPER_FLAGS_5 -#undef DEF_HELPER_FLAGS_6 -#undef DEF_HELPER_FLAGS_7 -#undef GEN_HELPER +#define HELPER_H "helper.h" +#include "exec/helper-gen.h.inc" +#undef HELPER_H #endif /* HELPER_GEN_H */ diff --git a/include/exec/helper-gen.h.inc b/include/exec/helper-gen.h.inc new file mode 100644 index 0000000..c009641 --- /dev/null +++ b/include/exec/helper-gen.h.inc @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands generation functions for tcg opcodes. + * Define HELPER_H for the header file to be expanded, + * and static inline to change from global file scope. + */ + +#include "tcg/tcg.h" +#include "tcg/helper-info.h" +#include "exec/helper-head.h" + +#define DEF_HELPER_FLAGS_0(name, flags, ret) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl0(ret)) \ +{ \ + tcg_gen_call0(&glue(helper_info_, name), dh_retvar(ret)); \ +} + +#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1)) \ +{ \ + tcg_gen_call1(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1)); \ +} + +#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2)) \ +{ \ + tcg_gen_call2(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2)); \ +} + +#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3)) \ +{ \ + tcg_gen_call3(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3)); \ +} + +#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), \ + dh_arg_decl(t3, 3), dh_arg_decl(t4, 4)) \ +{ \ + tcg_gen_call4(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2), \ + dh_arg(t3, 3), dh_arg(t4, 4)); \ +} + +#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ + dh_arg_decl(t4, 4), dh_arg_decl(t5, 5)) \ +{ \ + tcg_gen_call5(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ + dh_arg(t4, 4), dh_arg(t5, 5)); \ +} + +#define DEF_HELPER_FLAGS_6(name, flags, ret, t1, t2, t3, t4, t5, t6) \ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ + dh_arg_decl(t4, 4), dh_arg_decl(t5, 5), dh_arg_decl(t6, 6)) \ +{ \ + tcg_gen_call6(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ + dh_arg(t4, 4), dh_arg(t5, 5), dh_arg(t6, 6)); \ +} + +#define DEF_HELPER_FLAGS_7(name, flags, ret, t1, t2, t3, t4, t5, t6, t7)\ +extern TCGHelperInfo glue(helper_info_, name); \ +static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \ + dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ + dh_arg_decl(t4, 4), dh_arg_decl(t5, 5), dh_arg_decl(t6, 6), \ + dh_arg_decl(t7, 7)) \ +{ \ + tcg_gen_call7(&glue(helper_info_, name), dh_retvar(ret), \ + dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3), \ + dh_arg(t4, 4), dh_arg(t5, 5), dh_arg(t6, 6), \ + dh_arg(t7, 7)); \ +} + +#include HELPER_H + +#undef DEF_HELPER_FLAGS_0 +#undef DEF_HELPER_FLAGS_1 +#undef DEF_HELPER_FLAGS_2 +#undef DEF_HELPER_FLAGS_3 +#undef DEF_HELPER_FLAGS_4 +#undef DEF_HELPER_FLAGS_5 +#undef DEF_HELPER_FLAGS_6 +#undef DEF_HELPER_FLAGS_7 diff --git a/include/exec/helper-head.h b/include/exec/helper-head.h index f863a6e..28ceab0 100644 --- a/include/exec/helper-head.h +++ b/include/exec/helper-head.h @@ -1,18 +1,6 @@ -/* Helper file for declaring TCG helper functions. - Used by other helper files. - - Targets should use DEF_HELPER_N and DEF_HELPER_FLAGS_N to declare helper - functions. Names should be specified without the helper_ prefix, and - the return and argument types specified. 3 basic types are understood - (i32, i64 and ptr). Additional aliases are provided for convenience and - to match the types used by the C helper implementation. - - The target helper.h should be included in all files that use/define - helper functions. THis will ensure that function prototypes are - consistent. In addition it should be included an extra two times for - helper.c, defining: - GEN_HELPER 1 to produce op generation functions (gen_helper_*) - GEN_HELPER 2 to do runtime registration helper functions. +/* + * Helper file for declaring TCG helper functions. + * Used by other helper files. */ #ifndef EXEC_HELPER_HEAD_H @@ -34,6 +22,7 @@ #define dh_alias_f64 i64 #define dh_alias_ptr ptr #define dh_alias_cptr ptr +#define dh_alias_env ptr #define dh_alias_void void #define dh_alias_noreturn noreturn #define dh_alias(t) glue(dh_alias_, t) @@ -49,6 +38,7 @@ #define dh_ctype_f64 float64 #define dh_ctype_ptr void * #define dh_ctype_cptr const void * +#define dh_ctype_env CPUArchState * #define dh_ctype_void void #define dh_ctype_noreturn G_NORETURN void #define dh_ctype(t) dh_ctype_##t @@ -64,9 +54,6 @@ # endif # endif # define dh_ctype_tl target_ulong -# define dh_alias_env ptr -# define dh_ctype_env CPUArchState * -# define dh_typecode_env dh_typecode_ptr #endif /* We can't use glue() here because it falls foul of C preprocessor @@ -108,6 +95,7 @@ #define dh_typecode_f32 dh_typecode_i32 #define dh_typecode_f64 dh_typecode_i64 #define dh_typecode_cptr dh_typecode_ptr +#define dh_typecode_env dh_typecode_ptr #define dh_typecode(t) dh_typecode_##t #define dh_callflag_i32 0 diff --git a/include/exec/helper-info.c.inc b/include/exec/helper-info.c.inc new file mode 100644 index 0000000..530d2e6 --- /dev/null +++ b/include/exec/helper-info.c.inc @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands info structures for tcg helpers. + * Define HELPER_H for the header file to be expanded. + */ + +#include "tcg/tcg.h" +#include "tcg/helper-info.h" +#include "exec/helper-head.h" + +/* + * Need one more level of indirection before stringification + * to get all the macros expanded first. + */ +#define str(s) #s + +#define DEF_HELPER_FLAGS_0(NAME, FLAGS, RET) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) \ + }; + +#define DEF_HELPER_FLAGS_1(NAME, FLAGS, RET, T1) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + }; + +#define DEF_HELPER_FLAGS_2(NAME, FLAGS, RET, T1, T2) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) \ + }; + +#define DEF_HELPER_FLAGS_3(NAME, FLAGS, RET, T1, T2, T3) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) | dh_typemask(T3, 3) \ + }; + +#define DEF_HELPER_FLAGS_4(NAME, FLAGS, RET, T1, T2, T3, T4) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) | dh_typemask(T3, 3) \ + | dh_typemask(T4, 4) \ + }; + +#define DEF_HELPER_FLAGS_5(NAME, FLAGS, RET, T1, T2, T3, T4, T5) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) | dh_typemask(T3, 3) \ + | dh_typemask(T4, 4) | dh_typemask(T5, 5) \ + }; + +#define DEF_HELPER_FLAGS_6(NAME, FLAGS, RET, T1, T2, T3, T4, T5, T6) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) | dh_typemask(T3, 3) \ + | dh_typemask(T4, 4) | dh_typemask(T5, 5) \ + | dh_typemask(T6, 6) \ + }; + +#define DEF_HELPER_FLAGS_7(NAME, FLAGS, RET, T1, T2, T3, T4, T5, T6, T7) \ + TCGHelperInfo glue(helper_info_, NAME) = { \ + .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(RET), \ + .typemask = dh_typemask(RET, 0) | dh_typemask(T1, 1) \ + | dh_typemask(T2, 2) | dh_typemask(T3, 3) \ + | dh_typemask(T4, 4) | dh_typemask(T5, 5) \ + | dh_typemask(T6, 6) | dh_typemask(T7, 7) \ + }; + +#include HELPER_H + +#undef str +#undef DEF_HELPER_FLAGS_0 +#undef DEF_HELPER_FLAGS_1 +#undef DEF_HELPER_FLAGS_2 +#undef DEF_HELPER_FLAGS_3 +#undef DEF_HELPER_FLAGS_4 +#undef DEF_HELPER_FLAGS_5 +#undef DEF_HELPER_FLAGS_6 +#undef DEF_HELPER_FLAGS_7 diff --git a/include/exec/helper-proto-common.h b/include/exec/helper-proto-common.h new file mode 100644 index 0000000..4d4b022 --- /dev/null +++ b/include/exec/helper-proto-common.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands prototypes for the helper functions. + */ + +#ifndef HELPER_PROTO_COMMON_H +#define HELPER_PROTO_COMMON_H + +#define HELPER_H "accel/tcg/tcg-runtime.h" +#include "exec/helper-proto.h.inc" +#undef HELPER_H + +#define HELPER_H "accel/tcg/plugin-helpers.h" +#include "exec/helper-proto.h.inc" +#undef HELPER_H + +#endif /* HELPER_PROTO_COMMON_H */ diff --git a/include/exec/helper-proto.h b/include/exec/helper-proto.h index 7a3f04b..6935cb4 100644 --- a/include/exec/helper-proto.h +++ b/include/exec/helper-proto.h @@ -1,71 +1,16 @@ -/* Helper file for declaring TCG helper functions. - This one expands prototypes for the helper functions. */ - -#ifndef HELPER_PROTO_H -#define HELPER_PROTO_H - -#include "exec/helper-head.h" - +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Work around an issue with --enable-lto, in which GCC's ipa-split pass - * decides to split out the noreturn code paths that raise an exception, - * taking the __builtin_return_address() along into the new function, - * where it no longer computes a value that returns to TCG generated code. - * Despite the name, the noinline attribute affects splitter, so this - * prevents the optimization in question. Given that helpers should not - * otherwise be called directly, this should have any other visible effect. - * - * See https://gitlab.com/qemu-project/qemu/-/issues/1454 + * Helper file for declaring TCG helper functions. + * This one expands prototypes for the helper functions. */ -#define DEF_HELPER_ATTR __attribute__((noinline)) - -#define DEF_HELPER_FLAGS_0(name, flags, ret) \ -dh_ctype(ret) HELPER(name) (void) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1)) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2)) DEF_HELPER_ATTR; -#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), \ - dh_ctype(t3)) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ - dh_ctype(t4)) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ - dh_ctype(t4), dh_ctype(t5)) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_6(name, flags, ret, t1, t2, t3, t4, t5, t6) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ - dh_ctype(t4), dh_ctype(t5), \ - dh_ctype(t6)) DEF_HELPER_ATTR; - -#define DEF_HELPER_FLAGS_7(name, flags, ret, t1, t2, t3, t4, t5, t6, t7) \ -dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ - dh_ctype(t4), dh_ctype(t5), dh_ctype(t6), \ - dh_ctype(t7)) DEF_HELPER_ATTR; - -#define IN_HELPER_PROTO - -#include "helper.h" -#include "accel/tcg/tcg-runtime.h" -#include "accel/tcg/plugin-helpers.h" +#ifndef HELPER_PROTO_H +#define HELPER_PROTO_H -#undef IN_HELPER_PROTO +#include "exec/helper-proto-common.h" -#undef DEF_HELPER_FLAGS_0 -#undef DEF_HELPER_FLAGS_1 -#undef DEF_HELPER_FLAGS_2 -#undef DEF_HELPER_FLAGS_3 -#undef DEF_HELPER_FLAGS_4 -#undef DEF_HELPER_FLAGS_5 -#undef DEF_HELPER_FLAGS_6 -#undef DEF_HELPER_FLAGS_7 -#undef DEF_HELPER_ATTR +#define HELPER_H "helper.h" +#include "exec/helper-proto.h.inc" +#undef HELPER_H #endif /* HELPER_PROTO_H */ diff --git a/include/exec/helper-proto.h.inc b/include/exec/helper-proto.h.inc new file mode 100644 index 0000000..c3aa666 --- /dev/null +++ b/include/exec/helper-proto.h.inc @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper file for declaring TCG helper functions. + * This one expands prototypes for the helper functions. + * Define HELPER_H for the header file to be expanded. + */ + +#include "exec/helper-head.h" + +/* + * Work around an issue with --enable-lto, in which GCC's ipa-split pass + * decides to split out the noreturn code paths that raise an exception, + * taking the __builtin_return_address() along into the new function, + * where it no longer computes a value that returns to TCG generated code. + * Despite the name, the noinline attribute affects splitter, so this + * prevents the optimization in question. Given that helpers should not + * otherwise be called directly, this should not have any other visible effect. + * + * See https://gitlab.com/qemu-project/qemu/-/issues/1454 + */ +#define DEF_HELPER_ATTR __attribute__((noinline)) + +#define DEF_HELPER_FLAGS_0(name, flags, ret) \ +dh_ctype(ret) HELPER(name) (void) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), \ + dh_ctype(t3)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ + dh_ctype(t4)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ + dh_ctype(t4), dh_ctype(t5)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_6(name, flags, ret, t1, t2, t3, t4, t5, t6) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ + dh_ctype(t4), dh_ctype(t5), \ + dh_ctype(t6)) DEF_HELPER_ATTR; + +#define DEF_HELPER_FLAGS_7(name, flags, ret, t1, t2, t3, t4, t5, t6, t7) \ +dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \ + dh_ctype(t4), dh_ctype(t5), dh_ctype(t6), \ + dh_ctype(t7)) DEF_HELPER_ATTR; + +#define IN_HELPER_PROTO + +#include HELPER_H + +#undef IN_HELPER_PROTO + +#undef DEF_HELPER_FLAGS_0 +#undef DEF_HELPER_FLAGS_1 +#undef DEF_HELPER_FLAGS_2 +#undef DEF_HELPER_FLAGS_3 +#undef DEF_HELPER_FLAGS_4 +#undef DEF_HELPER_FLAGS_5 +#undef DEF_HELPER_FLAGS_6 +#undef DEF_HELPER_FLAGS_7 +#undef DEF_HELPER_ATTR diff --git a/include/exec/helper-tcg.h b/include/exec/helper-tcg.h deleted file mode 100644 index 3933258..0000000 --- a/include/exec/helper-tcg.h +++ /dev/null @@ -1,75 +0,0 @@ -/* Helper file for declaring TCG helper functions. - This one defines data structures private to tcg.c. */ - -#ifndef HELPER_TCG_H -#define HELPER_TCG_H - -#include "exec/helper-head.h" - -/* Need one more level of indirection before stringification - to get all the macros expanded first. */ -#define str(s) #s - -#define DEF_HELPER_FLAGS_0(NAME, FLAGS, ret) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) }, - -#define DEF_HELPER_FLAGS_1(NAME, FLAGS, ret, t1) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) }, - -#define DEF_HELPER_FLAGS_2(NAME, FLAGS, ret, t1, t2) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) }, - -#define DEF_HELPER_FLAGS_3(NAME, FLAGS, ret, t1, t2, t3) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) | dh_typemask(t3, 3) }, - -#define DEF_HELPER_FLAGS_4(NAME, FLAGS, ret, t1, t2, t3, t4) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) }, - -#define DEF_HELPER_FLAGS_5(NAME, FLAGS, ret, t1, t2, t3, t4, t5) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \ - | dh_typemask(t5, 5) }, - -#define DEF_HELPER_FLAGS_6(NAME, FLAGS, ret, t1, t2, t3, t4, t5, t6) \ - { .func = HELPER(NAME), .name = str(NAME), \ - .flags = FLAGS | dh_callflag(ret), \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \ - | dh_typemask(t5, 5) | dh_typemask(t6, 6) }, - -#define DEF_HELPER_FLAGS_7(NAME, FLAGS, ret, t1, t2, t3, t4, t5, t6, t7) \ - { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \ - .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \ - | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \ - | dh_typemask(t5, 5) | dh_typemask(t6, 6) | dh_typemask(t7, 7) }, - -#include "helper.h" -#include "accel/tcg/tcg-runtime.h" -#include "accel/tcg/plugin-helpers.h" - -#undef str -#undef DEF_HELPER_FLAGS_0 -#undef DEF_HELPER_FLAGS_1 -#undef DEF_HELPER_FLAGS_2 -#undef DEF_HELPER_FLAGS_3 -#undef DEF_HELPER_FLAGS_4 -#undef DEF_HELPER_FLAGS_5 -#undef DEF_HELPER_FLAGS_6 -#undef DEF_HELPER_FLAGS_7 - -#endif /* HELPER_TCG_H */ diff --git a/include/exec/plugin-gen.h b/include/exec/plugin-gen.h index 3af0168..5282878 100644 --- a/include/exec/plugin-gen.h +++ b/include/exec/plugin-gen.h @@ -12,8 +12,6 @@ #ifndef QEMU_PLUGIN_GEN_H #define QEMU_PLUGIN_GEN_H -#include "exec/cpu_ldst.h" -#include "qemu/plugin.h" #include "tcg/tcg.h" struct DisasContextBase; @@ -29,25 +27,6 @@ void plugin_gen_insn_end(void); void plugin_gen_disable_mem_helpers(void); void plugin_gen_empty_mem_callback(TCGv_i64 addr, uint32_t info); -static inline void plugin_insn_append(abi_ptr pc, const void *from, size_t size) -{ - struct qemu_plugin_insn *insn = tcg_ctx->plugin_insn; - abi_ptr off; - - if (insn == NULL) { - return; - } - off = pc - insn->vaddr; - if (off < insn->data->len) { - g_byte_array_set_size(insn->data, off); - } else if (off > insn->data->len) { - /* we have an unexpected gap */ - g_assert_not_reached(); - } - - insn->data = g_byte_array_append(insn->data, from, size); -} - #else /* !CONFIG_PLUGIN */ static inline bool @@ -72,9 +51,6 @@ static inline void plugin_gen_disable_mem_helpers(void) static inline void plugin_gen_empty_mem_callback(TCGv_i64 addr, uint32_t info) { } -static inline void plugin_insn_append(abi_ptr pc, const void *from, size_t size) -{ } - #endif /* CONFIG_PLUGIN */ #endif /* QEMU_PLUGIN_GEN_H */ diff --git a/include/exec/poison.h b/include/exec/poison.h index 256736e..e94ee8d 100644 --- a/include/exec/poison.h +++ b/include/exec/poison.h @@ -85,7 +85,6 @@ #pragma GCC poison CONFIG_HVF #pragma GCC poison CONFIG_LINUX_USER #pragma GCC poison CONFIG_KVM -#pragma GCC poison CONFIG_SOFTMMU #pragma GCC poison CONFIG_WHPX #pragma GCC poison CONFIG_XEN diff --git a/include/exec/tlb-common.h b/include/exec/tlb-common.h new file mode 100644 index 0000000..dc5a5fa --- /dev/null +++ b/include/exec/tlb-common.h @@ -0,0 +1,56 @@ +/* + * Common definitions for the softmmu tlb + * + * Copyright (c) 2003 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ +#ifndef EXEC_TLB_COMMON_H +#define EXEC_TLB_COMMON_H 1 + +#define CPU_TLB_ENTRY_BITS 5 + +/* Minimalized TLB entry for use by TCG fast path. */ +typedef union CPUTLBEntry { + struct { + uint64_t addr_read; + uint64_t addr_write; + uint64_t addr_code; + /* + * Addend to virtual address to get host address. IO accesses + * use the corresponding iotlb value. + */ + uintptr_t addend; + }; + /* + * Padding to get a power of two size, as well as index + * access to addr_{read,write,code}. + */ + uint64_t addr_idx[(1 << CPU_TLB_ENTRY_BITS) / sizeof(uint64_t)]; +} CPUTLBEntry; + +QEMU_BUILD_BUG_ON(sizeof(CPUTLBEntry) != (1 << CPU_TLB_ENTRY_BITS)); + +/* + * Data elements that are per MMU mode, accessed by the fast path. + * The structure is aligned to aid loading the pair with one insn. + */ +typedef struct CPUTLBDescFast { + /* Contains (n_entries - 1) << CPU_TLB_ENTRY_BITS */ + uintptr_t mask; + /* The array of tlb entries itself. */ + CPUTLBEntry *table; +} CPUTLBDescFast QEMU_ALIGNED(2 * sizeof(void *)); + +#endif /* EXEC_TLB_COMMON_H */ diff --git a/include/exec/translation-block.h b/include/exec/translation-block.h new file mode 100644 index 0000000..5119924 --- /dev/null +++ b/include/exec/translation-block.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Definition of TranslationBlock. + * Copyright (c) 2003 Fabrice Bellard + */ + +#ifndef EXEC_TRANSLATION_BLOCK_H +#define EXEC_TRANSLATION_BLOCK_H + +#include "qemu/atomic.h" +#include "qemu/thread.h" +#include "qemu/interval-tree.h" +#include "exec/cpu-common.h" +#include "exec/target_page.h" + +/* + * Page tracking code uses ram addresses in system mode, and virtual + * addresses in userspace mode. Define tb_page_addr_t to be an + * appropriate type. + */ +#if defined(CONFIG_USER_ONLY) +typedef vaddr tb_page_addr_t; +#define TB_PAGE_ADDR_FMT "%" VADDR_PRIx +#else +typedef ram_addr_t tb_page_addr_t; +#define TB_PAGE_ADDR_FMT RAM_ADDR_FMT +#endif + +/* + * Translation Cache-related fields of a TB. + * This struct exists just for convenience; we keep track of TB's in a binary + * search tree, and the only fields needed to compare TB's in the tree are + * @ptr and @size. + * Note: the address of search data can be obtained by adding @size to @ptr. + */ +struct tb_tc { + const void *ptr; /* pointer to the translated code */ + size_t size; +}; + +struct TranslationBlock { + /* + * Guest PC corresponding to this block. This must be the true + * virtual address. Therefore e.g. x86 stores EIP + CS_BASE, and + * targets like Arm, MIPS, HP-PA, which reuse low bits for ISA or + * privilege, must store those bits elsewhere. + * + * If CF_PCREL, the opcodes for the TranslationBlock are written + * such that the TB is associated only with the physical page and + * may be run in any virtual address context. In this case, PC + * must always be taken from ENV in a target-specific manner. + * Unwind information is taken as offsets from the page, to be + * deposited into the "current" PC. + */ + vaddr pc; + + /* + * Target-specific data associated with the TranslationBlock, e.g.: + * x86: the original user, the Code Segment virtual base, + * arm: an extension of tb->flags, + * s390x: instruction data for EXECUTE, + * sparc: the next pc of the instruction queue (for delay slots). + */ + uint64_t cs_base; + + uint32_t flags; /* flags defining in which context the code was generated */ + uint32_t cflags; /* compile flags */ + +/* Note that TCG_MAX_INSNS is 512; we validate this match elsewhere. */ +#define CF_COUNT_MASK 0x000001ff +#define CF_NO_GOTO_TB 0x00000200 /* Do not chain with goto_tb */ +#define CF_NO_GOTO_PTR 0x00000400 /* Do not chain with goto_ptr */ +#define CF_SINGLE_STEP 0x00000800 /* gdbstub single-step in effect */ +#define CF_LAST_IO 0x00008000 /* Last insn may be an IO access. */ +#define CF_MEMI_ONLY 0x00010000 /* Only instrument memory ops */ +#define CF_USE_ICOUNT 0x00020000 +#define CF_INVALID 0x00040000 /* TB is stale. Set with @jmp_lock held */ +#define CF_PARALLEL 0x00080000 /* Generate code for a parallel context */ +#define CF_NOIRQ 0x00100000 /* Generate an uninterruptible TB */ +#define CF_PCREL 0x00200000 /* Opcodes in TB are PC-relative */ +#define CF_CLUSTER_MASK 0xff000000 /* Top 8 bits are cluster ID */ +#define CF_CLUSTER_SHIFT 24 + + /* + * Above fields used for comparing + */ + + /* size of target code for this block (1 <= size <= TARGET_PAGE_SIZE) */ + uint16_t size; + uint16_t icount; + + struct tb_tc tc; + + /* + * Track tb_page_addr_t intervals that intersect this TB. + * For user-only, the virtual addresses are always contiguous, + * and we use a unified interval tree. For system, we use a + * linked list headed in each PageDesc. Within the list, the lsb + * of the previous pointer tells the index of page_next[], and the + * list is protected by the PageDesc lock(s). + */ +#ifdef CONFIG_USER_ONLY + IntervalTreeNode itree; +#else + uintptr_t page_next[2]; + tb_page_addr_t page_addr[2]; +#endif + + /* jmp_lock placed here to fill a 4-byte hole. Its documentation is below */ + QemuSpin jmp_lock; + + /* The following data are used to directly call another TB from + * the code of this one. This can be done either by emitting direct or + * indirect native jump instructions. These jumps are reset so that the TB + * just continues its execution. The TB can be linked to another one by + * setting one of the jump targets (or patching the jump instruction). Only + * two of such jumps are supported. + */ +#define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */ + uint16_t jmp_reset_offset[2]; /* offset of original jump target */ + uint16_t jmp_insn_offset[2]; /* offset of direct jump insn */ + uintptr_t jmp_target_addr[2]; /* target address */ + + /* + * Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps. + * Each TB can have two outgoing jumps, and therefore can participate + * in two lists. The list entries are kept in jmp_list_next[2]. The least + * significant bit (LSB) of the pointers in these lists is used to encode + * which of the two list entries is to be used in the pointed TB. + * + * List traversals are protected by jmp_lock. The destination TB of each + * outgoing jump is kept in jmp_dest[] so that the appropriate jmp_lock + * can be acquired from any origin TB. + * + * jmp_dest[] are tagged pointers as well. The LSB is set when the TB is + * being invalidated, so that no further outgoing jumps from it can be set. + * + * jmp_lock also protects the CF_INVALID cflag; a jump must not be chained + * to a destination TB that has CF_INVALID set. + */ + uintptr_t jmp_list_head; + uintptr_t jmp_list_next[2]; + uintptr_t jmp_dest[2]; +}; + +/* The alignment given to TranslationBlock during allocation. */ +#define CODE_GEN_ALIGN 16 + +#endif /* EXEC_TRANSLATION_BLOCK_H */ diff --git a/include/exec/translator.h b/include/exec/translator.h index 797fef7..224ae14 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -18,13 +18,8 @@ * member in your target-specific DisasContext. */ - #include "qemu/bswap.h" -#include "exec/exec-all.h" -#include "exec/cpu_ldst.h" -#include "exec/plugin-gen.h" -#include "exec/translate-all.h" -#include "tcg/tcg.h" +#include "exec/cpu_ldst.h" /* for abi_ptr */ /** * gen_intermediate_code @@ -160,6 +155,16 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns, */ bool translator_use_goto_tb(DisasContextBase *db, target_ulong dest); +/** + * translator_io_start + * @db: Disassembly context + * + * If icount is enabled, set cpu->can_to_io, adjust db->is_jmp to + * DISAS_TOO_MANY if it is still DISAS_NEXT, and return true. + * Otherwise return false. + */ +bool translator_io_start(DisasContextBase *db); + /* * Translator Load Functions * @@ -219,12 +224,7 @@ translator_ldq_swap(CPUArchState *env, DisasContextBase *db, * re-synthesised for s390x "ex"). It ensures we update other areas of * the translator with details of the executed instruction. */ - -static inline void translator_fake_ldb(uint8_t insn8, abi_ptr pc) -{ - plugin_insn_append(pc, &insn8, sizeof(insn8)); -} - +void translator_fake_ldb(uint8_t insn8, abi_ptr pc); /* * Return whether addr is on the same page as where disassembly started. diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h index 8e9ef25..8c1840b 100644 --- a/include/qemu/typedefs.h +++ b/include/qemu/typedefs.h @@ -131,6 +131,7 @@ typedef struct ReservedRegion ReservedRegion; typedef struct SavedIOTLB SavedIOTLB; typedef struct SHPCDevice SHPCDevice; typedef struct SSIBus SSIBus; +typedef struct TCGHelperInfo TCGHelperInfo; typedef struct TranslationBlock TranslationBlock; typedef struct VirtIODevice VirtIODevice; typedef struct Visitor Visitor; diff --git a/include/tcg/helper-info.h b/include/tcg/helper-info.h new file mode 100644 index 0000000..4b6c9b4 --- /dev/null +++ b/include/tcg/helper-info.h @@ -0,0 +1,64 @@ +/* + * TCG Helper Infomation Structure + * + * Copyright (c) 2023 Linaro Ltd + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#ifndef TCG_HELPER_INFO_H +#define TCG_HELPER_INFO_H + +#ifdef CONFIG_TCG_INTERPRETER +#include <ffi.h> +#endif + +/* + * Describe the calling convention of a given argument type. + */ +typedef enum { + TCG_CALL_RET_NORMAL, /* by registers */ + TCG_CALL_RET_BY_REF, /* for i128, by reference */ + TCG_CALL_RET_BY_VEC, /* for i128, by vector register */ +} TCGCallReturnKind; + +typedef enum { + TCG_CALL_ARG_NORMAL, /* by registers (continuing onto stack) */ + TCG_CALL_ARG_EVEN, /* like normal, but skipping odd slots */ + TCG_CALL_ARG_EXTEND, /* for i32, as a sign/zero-extended i64 */ + TCG_CALL_ARG_EXTEND_U, /* ... as a zero-extended i64 */ + TCG_CALL_ARG_EXTEND_S, /* ... as a sign-extended i64 */ + TCG_CALL_ARG_BY_REF, /* for i128, by reference, first */ + TCG_CALL_ARG_BY_REF_N, /* ... by reference, subsequent */ +} TCGCallArgumentKind; + +typedef struct TCGCallArgumentLoc { + TCGCallArgumentKind kind : 8; + unsigned arg_slot : 8; + unsigned ref_slot : 8; + unsigned arg_idx : 4; + unsigned tmp_subindex : 2; +} TCGCallArgumentLoc; + +struct TCGHelperInfo { + void *func; + const char *name; + + /* Used with g_once_init_enter. */ +#ifdef CONFIG_TCG_INTERPRETER + ffi_cif *cif; +#else + uintptr_t init; +#endif + + unsigned typemask : 32; + unsigned flags : 8; + unsigned nr_in : 8; + unsigned nr_out : 8; + TCGCallReturnKind out_kind : 8; + + /* Maximum physical arguments are constrained by TCG_TYPE_I128. */ + TCGCallArgumentLoc in[MAX_CALL_IARGS * (128 / TCG_TARGET_REG_BITS)]; +}; + +#endif /* TCG_HELPER_INFO_H */ diff --git a/include/tcg/insn-start-words.h b/include/tcg/insn-start-words.h new file mode 100644 index 0000000..50c18bd --- /dev/null +++ b/include/tcg/insn-start-words.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define TARGET_INSN_START_WORDS + * Copyright (c) 2008 Fabrice Bellard + */ + +#ifndef TARGET_INSN_START_WORDS + +#include "cpu.h" + +#ifndef TARGET_INSN_START_EXTRA_WORDS +# define TARGET_INSN_START_WORDS 1 +#else +# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS) +#endif + +#endif /* TARGET_INSN_START_WORDS */ diff --git a/include/tcg/oversized-guest.h b/include/tcg/oversized-guest.h new file mode 100644 index 0000000..641b974 --- /dev/null +++ b/include/tcg/oversized-guest.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define TCG_OVERSIZED_GUEST + * Copyright (c) 2008 Fabrice Bellard + */ + +#ifndef EXEC_TCG_OVERSIZED_GUEST_H +#define EXEC_TCG_OVERSIZED_GUEST_H + +#include "tcg-target-reg-bits.h" +#include "cpu-param.h" + +/* + * Oversized TCG guests make things like MTTCG hard + * as we can't use atomics for cputlb updates. + */ +#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS +#define TCG_OVERSIZED_GUEST 1 +#else +#define TCG_OVERSIZED_GUEST 0 +#endif + +#endif diff --git a/include/tcg/tcg-op-common.h b/include/tcg/tcg-op-common.h new file mode 100644 index 0000000..be382bb --- /dev/null +++ b/include/tcg/tcg-op-common.h @@ -0,0 +1,996 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Target independent opcode generation functions. + * + * Copyright (c) 2008 Fabrice Bellard + */ + +#ifndef TCG_TCG_OP_COMMON_H +#define TCG_TCG_OP_COMMON_H + +#include "tcg/tcg.h" +#include "exec/helper-proto-common.h" +#include "exec/helper-gen-common.h" + +/* Basic output routines. Not for general consumption. */ + +void tcg_gen_op1(TCGOpcode, TCGArg); +void tcg_gen_op2(TCGOpcode, TCGArg, TCGArg); +void tcg_gen_op3(TCGOpcode, TCGArg, TCGArg, TCGArg); +void tcg_gen_op4(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg); +void tcg_gen_op5(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg); +void tcg_gen_op6(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg); + +void vec_gen_2(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg); +void vec_gen_3(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg, TCGArg); +void vec_gen_4(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg, TCGArg, TCGArg); + +static inline void tcg_gen_op1_i32(TCGOpcode opc, TCGv_i32 a1) +{ + tcg_gen_op1(opc, tcgv_i32_arg(a1)); +} + +static inline void tcg_gen_op1_i64(TCGOpcode opc, TCGv_i64 a1) +{ + tcg_gen_op1(opc, tcgv_i64_arg(a1)); +} + +static inline void tcg_gen_op1i(TCGOpcode opc, TCGArg a1) +{ + tcg_gen_op1(opc, a1); +} + +static inline void tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2) +{ + tcg_gen_op2(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2)); +} + +static inline void tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2) +{ + tcg_gen_op2(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2)); +} + +static inline void tcg_gen_op2i_i32(TCGOpcode opc, TCGv_i32 a1, TCGArg a2) +{ + tcg_gen_op2(opc, tcgv_i32_arg(a1), a2); +} + +static inline void tcg_gen_op2i_i64(TCGOpcode opc, TCGv_i64 a1, TCGArg a2) +{ + tcg_gen_op2(opc, tcgv_i64_arg(a1), a2); +} + +static inline void tcg_gen_op2ii(TCGOpcode opc, TCGArg a1, TCGArg a2) +{ + tcg_gen_op2(opc, a1, a2); +} + +static inline void tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 a1, + TCGv_i32 a2, TCGv_i32 a3) +{ + tcg_gen_op3(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), tcgv_i32_arg(a3)); +} + +static inline void tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 a1, + TCGv_i64 a2, TCGv_i64 a3) +{ + tcg_gen_op3(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), tcgv_i64_arg(a3)); +} + +static inline void tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 a1, + TCGv_i32 a2, TCGArg a3) +{ + tcg_gen_op3(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3); +} + +static inline void tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 a1, + TCGv_i64 a2, TCGArg a3) +{ + tcg_gen_op3(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3); +} + +static inline void tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val, + TCGv_ptr base, TCGArg offset) +{ + tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_ptr_arg(base), offset); +} + +static inline void tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val, + TCGv_ptr base, TCGArg offset) +{ + tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_ptr_arg(base), offset); +} + +static inline void tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGv_i32 a3, TCGv_i32 a4) +{ + tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), + tcgv_i32_arg(a3), tcgv_i32_arg(a4)); +} + +static inline void tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, + TCGv_i64 a3, TCGv_i64 a4) +{ + tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), + tcgv_i64_arg(a3), tcgv_i64_arg(a4)); +} + +static inline void tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGv_i32 a3, TCGArg a4) +{ + tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), + tcgv_i32_arg(a3), a4); +} + +static inline void tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, + TCGv_i64 a3, TCGArg a4) +{ + tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), + tcgv_i64_arg(a3), a4); +} + +static inline void tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGArg a3, TCGArg a4) +{ + tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4); +} + +static inline void tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, + TCGArg a3, TCGArg a4) +{ + tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4); +} + +static inline void tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5) +{ + tcg_gen_op5(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), + tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5)); +} + +static inline void tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, + TCGv_i64 a3, TCGv_i64 a4, TCGv_i64 a5) +{ + tcg_gen_op5(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), + tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5)); +} + +static inline void tcg_gen_op5i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGv_i32 a3, TCGv_i32 a4, TCGArg a5) +{ + tcg_gen_op5(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), + tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5); +} + +static inline void tcg_gen_op5i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, + TCGv_i64 a3, TCGv_i64 a4, TCGArg a5) +{ + tcg_gen_op5(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), + tcgv_i64_arg(a3), tcgv_i64_arg(a4), a5); +} + +static inline void tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGv_i32 a3, TCGArg a4, TCGArg a5) +{ + tcg_gen_op5(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), + tcgv_i32_arg(a3), a4, a5); +} + +static inline void tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, + TCGv_i64 a3, TCGArg a4, TCGArg a5) +{ + tcg_gen_op5(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), + tcgv_i64_arg(a3), a4, a5); +} + +static inline void tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGv_i32 a3, TCGv_i32 a4, + TCGv_i32 a5, TCGv_i32 a6) +{ + tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), + tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), + tcgv_i32_arg(a6)); +} + +static inline void tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, + TCGv_i64 a3, TCGv_i64 a4, + TCGv_i64 a5, TCGv_i64 a6) +{ + tcg_gen_op6(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), + tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), + tcgv_i64_arg(a6)); +} + +static inline void tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGv_i32 a3, TCGv_i32 a4, + TCGv_i32 a5, TCGArg a6) +{ + tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), + tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), a6); +} + +static inline void tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, + TCGv_i64 a3, TCGv_i64 a4, + TCGv_i64 a5, TCGArg a6) +{ + tcg_gen_op6(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), + tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6); +} + +static inline void tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGv_i32 a3, TCGv_i32 a4, + TCGArg a5, TCGArg a6) +{ + tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), + tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6); +} + +static inline void tcg_gen_op6ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, + TCGv_i64 a3, TCGv_i64 a4, + TCGArg a5, TCGArg a6) +{ + tcg_gen_op6(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), + tcgv_i64_arg(a3), tcgv_i64_arg(a4), a5, a6); +} + + +/* Generic ops. */ + +static inline void gen_set_label(TCGLabel *l) +{ + l->present = 1; + tcg_gen_op1(INDEX_op_set_label, label_arg(l)); +} + +void tcg_gen_br(TCGLabel *l); +void tcg_gen_mb(TCGBar); + +/** + * tcg_gen_exit_tb() - output exit_tb TCG operation + * @tb: The TranslationBlock from which we are exiting + * @idx: Direct jump slot index, or exit request + * + * See tcg/README for more info about this TCG operation. + * See also tcg.h and the block comment above TB_EXIT_MASK. + * + * For a normal exit from the TB, back to the main loop, @tb should + * be NULL and @idx should be 0. Otherwise, @tb should be valid and + * @idx should be one of the TB_EXIT_ values. + */ +void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx); + +/** + * tcg_gen_goto_tb() - output goto_tb TCG operation + * @idx: Direct jump slot index (0 or 1) + * + * See tcg/README for more info about this TCG operation. + * + * NOTE: In softmmu emulation, direct jumps with goto_tb are only safe within + * the pages this TB resides in because we don't take care of direct jumps when + * address mapping changes, e.g. in tlb_flush(). In user mode, there's only a + * static address translation, so the destination address is always valid, TBs + * are always invalidated properly, and direct jumps are reset when mapping + * changes. + */ +void tcg_gen_goto_tb(unsigned idx); + +/** + * tcg_gen_lookup_and_goto_ptr() - look up the current TB, jump to it if valid + * @addr: Guest address of the target TB + * + * If the TB is not valid, jump to the epilogue. + * + * This operation is optional. If the TCG backend does not implement goto_ptr, + * this op is equivalent to calling tcg_gen_exit_tb() with 0 as the argument. + */ +void tcg_gen_lookup_and_goto_ptr(void); + +static inline void tcg_gen_plugin_cb_start(unsigned from, unsigned type, + unsigned wr) +{ + tcg_gen_op3(INDEX_op_plugin_cb_start, from, type, wr); +} + +static inline void tcg_gen_plugin_cb_end(void) +{ + tcg_emit_op(INDEX_op_plugin_cb_end, 0); +} + +/* 32 bit ops */ + +void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg); +void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2); +void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2); +void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2); +void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg); +void tcg_gen_ctpop_i32(TCGv_i32 a1, TCGv_i32 a2); +void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); +void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, + unsigned int ofs, unsigned int len); +void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, + unsigned int ofs, unsigned int len); +void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, + unsigned int ofs, unsigned int len); +void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, + unsigned int ofs, unsigned int len); +void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, + unsigned int ofs); +void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *); +void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *); +void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, + TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, + TCGv_i32 arg1, int32_t arg2); +void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, + TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2); +void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, + TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh); +void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, + TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh); +void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg); +void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg); +void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg); +void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg); +void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags); +void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg); +void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg); +void tcg_gen_smin_i32(TCGv_i32, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_smax_i32(TCGv_i32, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_umin_i32(TCGv_i32, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_umax_i32(TCGv_i32, TCGv_i32 arg1, TCGv_i32 arg2); +void tcg_gen_abs_i32(TCGv_i32, TCGv_i32); + +/* Replicate a value of size @vece from @in to all the lanes in @out */ +void tcg_gen_dup_i32(unsigned vece, TCGv_i32 out, TCGv_i32 in); + +static inline void tcg_gen_discard_i32(TCGv_i32 arg) +{ + tcg_gen_op1_i32(INDEX_op_discard, arg); +} + +static inline void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg) +{ + if (ret != arg) { + tcg_gen_op2_i32(INDEX_op_mov_i32, ret, arg); + } +} + +static inline void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset); +} + +static inline void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset); +} + +static inline void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset); +} + +static inline void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset); +} + +static inline void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset); +} + +static inline void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset); +} + +static inline void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset); +} + +static inline void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset); +} + +static inline void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) +{ + tcg_gen_op3_i32(INDEX_op_add_i32, ret, arg1, arg2); +} + +static inline void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) +{ + tcg_gen_op3_i32(INDEX_op_sub_i32, ret, arg1, arg2); +} + +static inline void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) +{ + tcg_gen_op3_i32(INDEX_op_and_i32, ret, arg1, arg2); +} + +static inline void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) +{ + tcg_gen_op3_i32(INDEX_op_or_i32, ret, arg1, arg2); +} + +static inline void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) +{ + tcg_gen_op3_i32(INDEX_op_xor_i32, ret, arg1, arg2); +} + +static inline void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) +{ + tcg_gen_op3_i32(INDEX_op_shl_i32, ret, arg1, arg2); +} + +static inline void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) +{ + tcg_gen_op3_i32(INDEX_op_shr_i32, ret, arg1, arg2); +} + +static inline void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) +{ + tcg_gen_op3_i32(INDEX_op_sar_i32, ret, arg1, arg2); +} + +static inline void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) +{ + tcg_gen_op3_i32(INDEX_op_mul_i32, ret, arg1, arg2); +} + +static inline void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg) +{ + if (TCG_TARGET_HAS_neg_i32) { + tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg); + } else { + tcg_gen_subfi_i32(ret, 0, arg); + } +} + +static inline void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) +{ + if (TCG_TARGET_HAS_not_i32) { + tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg); + } else { + tcg_gen_xori_i32(ret, arg, -1); + } +} + +/* 64 bit ops */ + +void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg); +void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2); +void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2); +void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2); +void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_ctpop_i64(TCGv_i64 a1, TCGv_i64 a2); +void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); +void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, + unsigned int ofs, unsigned int len); +void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, + unsigned int ofs, unsigned int len); +void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, + unsigned int ofs, unsigned int len); +void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, + unsigned int ofs, unsigned int len); +void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, + unsigned int ofs); +void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *); +void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *); +void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, + TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, + TCGv_i64 arg1, int64_t arg2); +void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, + TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2); +void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, + TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh); +void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, + TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh); +void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags); +void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags); +void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_smin_i64(TCGv_i64, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_smax_i64(TCGv_i64, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_umin_i64(TCGv_i64, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_umax_i64(TCGv_i64, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_abs_i64(TCGv_i64, TCGv_i64); + +/* Replicate a value of size @vece from @in to all the lanes in @out */ +void tcg_gen_dup_i64(unsigned vece, TCGv_i64 out, TCGv_i64 in); + +#if TCG_TARGET_REG_BITS == 64 +static inline void tcg_gen_discard_i64(TCGv_i64 arg) +{ + tcg_gen_op1_i64(INDEX_op_discard, arg); +} + +static inline void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) +{ + if (ret != arg) { + tcg_gen_op2_i64(INDEX_op_mov_i64, ret, arg); + } +} + +static inline void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset); +} + +static inline void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset); +} + +static inline void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset); +} + +static inline void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset); +} + +static inline void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset); +} + +static inline void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset); +} + +static inline void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset); +} + +static inline void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset); +} + +static inline void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset); +} + +static inline void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset); +} + +static inline void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, + tcg_target_long offset) +{ + tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset); +} + +static inline void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_op3_i64(INDEX_op_add_i64, ret, arg1, arg2); +} + +static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_op3_i64(INDEX_op_sub_i64, ret, arg1, arg2); +} + +static inline void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_op3_i64(INDEX_op_and_i64, ret, arg1, arg2); +} + +static inline void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_op3_i64(INDEX_op_or_i64, ret, arg1, arg2); +} + +static inline void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_op3_i64(INDEX_op_xor_i64, ret, arg1, arg2); +} + +static inline void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_op3_i64(INDEX_op_shl_i64, ret, arg1, arg2); +} + +static inline void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_op3_i64(INDEX_op_shr_i64, ret, arg1, arg2); +} + +static inline void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_op3_i64(INDEX_op_sar_i64, ret, arg1, arg2); +} + +static inline void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_op3_i64(INDEX_op_mul_i64, ret, arg1, arg2); +} +#else /* TCG_TARGET_REG_BITS == 32 */ +void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); + +void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); + +void tcg_gen_discard_i64(TCGv_i64 arg); +void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg); +void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +#endif /* TCG_TARGET_REG_BITS */ + +static inline void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg) +{ + if (TCG_TARGET_HAS_neg_i64) { + tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg); + } else { + tcg_gen_subfi_i64(ret, 0, arg); + } +} + +/* Size changing operations. */ + +void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg); +void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg); +void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high); +void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg); +void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg); +void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg); +void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg); + +void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src); +void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg); +void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi); + +static inline void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi) +{ + tcg_gen_deposit_i64(ret, lo, hi, 32, 32); +} + +/* Local load/store bit ops */ + +void tcg_gen_qemu_ld_i32_chk(TCGv_i32, TCGTemp *, TCGArg, MemOp, TCGType); +void tcg_gen_qemu_st_i32_chk(TCGv_i32, TCGTemp *, TCGArg, MemOp, TCGType); +void tcg_gen_qemu_ld_i64_chk(TCGv_i64, TCGTemp *, TCGArg, MemOp, TCGType); +void tcg_gen_qemu_st_i64_chk(TCGv_i64, TCGTemp *, TCGArg, MemOp, TCGType); +void tcg_gen_qemu_ld_i128_chk(TCGv_i128, TCGTemp *, TCGArg, MemOp, TCGType); +void tcg_gen_qemu_st_i128_chk(TCGv_i128, TCGTemp *, TCGArg, MemOp, TCGType); + +/* Atomic ops */ + +void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128, + TCGv_i128, TCGArg, MemOp, TCGType); + +void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128, + TCGv_i128, TCGArg, MemOp, TCGType); + +void tcg_gen_atomic_xchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_xchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); + +void tcg_gen_atomic_fetch_add_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_add_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_and_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_and_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_or_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_or_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_xor_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_xor_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_smin_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_smin_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_umin_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_umin_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_smax_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_smax_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_umax_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_fetch_umax_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); + +void tcg_gen_atomic_add_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_add_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_and_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_and_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_or_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_or_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_xor_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_xor_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_smin_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_smin_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_umin_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_umin_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_smax_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_smax_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_umax_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, + TCGArg, MemOp, TCGType); +void tcg_gen_atomic_umax_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, + TCGArg, MemOp, TCGType); + +/* Vector ops */ + +void tcg_gen_mov_vec(TCGv_vec, TCGv_vec); +void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec, TCGv_i32); +void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec, TCGv_i64); +void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec, TCGv_ptr, tcg_target_long); +void tcg_gen_dupi_vec(unsigned vece, TCGv_vec, uint64_t); +void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a); +void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a); +void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a); +void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); +void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); + +void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i); +void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i); +void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i); +void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i); +void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i); + +void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s); +void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s); +void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s); +void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s); + +void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec s); +void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec s); +void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec s); +void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec s); +void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec s); + +void tcg_gen_cmp_vec(TCGCond cond, unsigned vece, TCGv_vec r, + TCGv_vec a, TCGv_vec b); + +void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a, + TCGv_vec b, TCGv_vec c); +void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r, + TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d); + +void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr base, TCGArg offset); +void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr base, TCGArg offset); +void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr base, TCGArg offset, TCGType t); + +/* Host pointer ops */ + +#if UINTPTR_MAX == UINT32_MAX +# define PTR i32 +# define NAT TCGv_i32 +#else +# define PTR i64 +# define NAT TCGv_i64 +#endif + +static inline void tcg_gen_ld_ptr(TCGv_ptr r, TCGv_ptr a, intptr_t o) +{ + glue(tcg_gen_ld_,PTR)((NAT)r, a, o); +} + +static inline void tcg_gen_st_ptr(TCGv_ptr r, TCGv_ptr a, intptr_t o) +{ + glue(tcg_gen_st_, PTR)((NAT)r, a, o); +} + +static inline void tcg_gen_discard_ptr(TCGv_ptr a) +{ + glue(tcg_gen_discard_,PTR)((NAT)a); +} + +static inline void tcg_gen_add_ptr(TCGv_ptr r, TCGv_ptr a, TCGv_ptr b) +{ + glue(tcg_gen_add_,PTR)((NAT)r, (NAT)a, (NAT)b); +} + +static inline void tcg_gen_addi_ptr(TCGv_ptr r, TCGv_ptr a, intptr_t b) +{ + glue(tcg_gen_addi_,PTR)((NAT)r, (NAT)a, b); +} + +static inline void tcg_gen_mov_ptr(TCGv_ptr d, TCGv_ptr s) +{ + glue(tcg_gen_mov_,PTR)((NAT)d, (NAT)s); +} + +static inline void tcg_gen_movi_ptr(TCGv_ptr d, intptr_t s) +{ + glue(tcg_gen_movi_,PTR)((NAT)d, s); +} + +static inline void tcg_gen_brcondi_ptr(TCGCond cond, TCGv_ptr a, + intptr_t b, TCGLabel *label) +{ + glue(tcg_gen_brcondi_,PTR)(cond, (NAT)a, b, label); +} + +static inline void tcg_gen_ext_i32_ptr(TCGv_ptr r, TCGv_i32 a) +{ +#if UINTPTR_MAX == UINT32_MAX + tcg_gen_mov_i32((NAT)r, a); +#else + tcg_gen_ext_i32_i64((NAT)r, a); +#endif +} + +static inline void tcg_gen_trunc_i64_ptr(TCGv_ptr r, TCGv_i64 a) +{ +#if UINTPTR_MAX == UINT32_MAX + tcg_gen_extrl_i64_i32((NAT)r, a); +#else + tcg_gen_mov_i64((NAT)r, a); +#endif +} + +static inline void tcg_gen_extu_ptr_i64(TCGv_i64 r, TCGv_ptr a) +{ +#if UINTPTR_MAX == UINT32_MAX + tcg_gen_extu_i32_i64(r, (NAT)a); +#else + tcg_gen_mov_i64(r, (NAT)a); +#endif +} + +static inline void tcg_gen_trunc_ptr_i32(TCGv_i32 r, TCGv_ptr a) +{ +#if UINTPTR_MAX == UINT32_MAX + tcg_gen_mov_i32(r, (NAT)a); +#else + tcg_gen_extrl_i64_i32(r, (NAT)a); +#endif +} + +#undef PTR +#undef NAT + +#endif /* TCG_TCG_OP_COMMON_H */ diff --git a/include/tcg/tcg-op-gvec-common.h b/include/tcg/tcg-op-gvec-common.h new file mode 100644 index 0000000..e2683d4 --- /dev/null +++ b/include/tcg/tcg-op-gvec-common.h @@ -0,0 +1,426 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Target independent generic vector operation expansion + * + * Copyright (c) 2018 Linaro + */ + +#ifndef TCG_TCG_OP_GVEC_COMMON_H +#define TCG_TCG_OP_GVEC_COMMON_H + +/* + * "Generic" vectors. All operands are given as offsets from ENV, + * and therefore cannot also be allocated via tcg_global_mem_new_*. + * OPRSZ is the byte size of the vector upon which the operation is performed. + * MAXSZ is the byte size of the full vector; bytes beyond OPSZ are cleared. + * + * All sizes must be 8 or any multiple of 16. + * When OPRSZ is 8, the alignment may be 8, otherwise must be 16. + * Operands may completely, but not partially, overlap. + */ + +/* Expand a call to a gvec-style helper, with pointers to two vector + operands, and a descriptor (see tcg-gvec-desc.h). */ +typedef void gen_helper_gvec_2(TCGv_ptr, TCGv_ptr, TCGv_i32); +void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs, + uint32_t oprsz, uint32_t maxsz, int32_t data, + gen_helper_gvec_2 *fn); + +/* Similarly, passing an extra data value. */ +typedef void gen_helper_gvec_2i(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32); +void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c, + uint32_t oprsz, uint32_t maxsz, int32_t data, + gen_helper_gvec_2i *fn); + +/* Similarly, passing an extra pointer (e.g. env or float_status). */ +typedef void gen_helper_gvec_2_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32); +void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs, + TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz, + int32_t data, gen_helper_gvec_2_ptr *fn); + +/* Similarly, with three vector operands. */ +typedef void gen_helper_gvec_3(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32); +void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, + uint32_t oprsz, uint32_t maxsz, int32_t data, + gen_helper_gvec_3 *fn); + +/* Similarly, with four vector operands. */ +typedef void gen_helper_gvec_4(TCGv_ptr, TCGv_ptr, TCGv_ptr, + TCGv_ptr, TCGv_i32); +void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, + uint32_t cofs, uint32_t oprsz, uint32_t maxsz, + int32_t data, gen_helper_gvec_4 *fn); + +/* Similarly, with five vector operands. */ +typedef void gen_helper_gvec_5(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_ptr, + TCGv_ptr, TCGv_i32); +void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, + uint32_t cofs, uint32_t xofs, uint32_t oprsz, + uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn); + +typedef void gen_helper_gvec_3_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, + TCGv_ptr, TCGv_i32); +void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, + TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz, + int32_t data, gen_helper_gvec_3_ptr *fn); + +typedef void gen_helper_gvec_4_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, + TCGv_ptr, TCGv_ptr, TCGv_i32); +void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, + uint32_t cofs, TCGv_ptr ptr, uint32_t oprsz, + uint32_t maxsz, int32_t data, + gen_helper_gvec_4_ptr *fn); + +typedef void gen_helper_gvec_5_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_ptr, + TCGv_ptr, TCGv_ptr, TCGv_i32); +void tcg_gen_gvec_5_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, + uint32_t cofs, uint32_t eofs, TCGv_ptr ptr, + uint32_t oprsz, uint32_t maxsz, int32_t data, + gen_helper_gvec_5_ptr *fn); + +/* Expand a gvec operation. Either inline or out-of-line depending on + the actual vector size and the operations supported by the host. */ +typedef struct { + /* Expand inline as a 64-bit or 32-bit integer. + Only one of these will be non-NULL. */ + void (*fni8)(TCGv_i64, TCGv_i64); + void (*fni4)(TCGv_i32, TCGv_i32); + /* Expand inline with a host vector type. */ + void (*fniv)(unsigned, TCGv_vec, TCGv_vec); + /* Expand out-of-line helper w/descriptor. */ + gen_helper_gvec_2 *fno; + /* The optional opcodes, if any, utilized by .fniv. */ + const TCGOpcode *opt_opc; + /* The data argument to the out-of-line helper. */ + int32_t data; + /* The vector element size, if applicable. */ + uint8_t vece; + /* Prefer i64 to v64. */ + bool prefer_i64; + /* Load dest as a 2nd source operand. */ + bool load_dest; +} GVecGen2; + +typedef struct { + /* Expand inline as a 64-bit or 32-bit integer. + Only one of these will be non-NULL. */ + void (*fni8)(TCGv_i64, TCGv_i64, int64_t); + void (*fni4)(TCGv_i32, TCGv_i32, int32_t); + /* Expand inline with a host vector type. */ + void (*fniv)(unsigned, TCGv_vec, TCGv_vec, int64_t); + /* Expand out-of-line helper w/descriptor, data in descriptor. */ + gen_helper_gvec_2 *fno; + /* Expand out-of-line helper w/descriptor, data as argument. */ + gen_helper_gvec_2i *fnoi; + /* The optional opcodes, if any, utilized by .fniv. */ + const TCGOpcode *opt_opc; + /* The vector element size, if applicable. */ + uint8_t vece; + /* Prefer i64 to v64. */ + bool prefer_i64; + /* Load dest as a 3rd source operand. */ + bool load_dest; +} GVecGen2i; + +typedef struct { + /* Expand inline as a 64-bit or 32-bit integer. + Only one of these will be non-NULL. */ + void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64); + void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32); + /* Expand inline with a host vector type. */ + void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec); + /* Expand out-of-line helper w/descriptor. */ + gen_helper_gvec_2i *fno; + /* The optional opcodes, if any, utilized by .fniv. */ + const TCGOpcode *opt_opc; + /* The data argument to the out-of-line helper. */ + uint32_t data; + /* The vector element size, if applicable. */ + uint8_t vece; + /* Prefer i64 to v64. */ + bool prefer_i64; + /* Load scalar as 1st source operand. */ + bool scalar_first; +} GVecGen2s; + +typedef struct { + /* Expand inline as a 64-bit or 32-bit integer. + Only one of these will be non-NULL. */ + void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64); + void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32); + /* Expand inline with a host vector type. */ + void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec); + /* Expand out-of-line helper w/descriptor. */ + gen_helper_gvec_3 *fno; + /* The optional opcodes, if any, utilized by .fniv. */ + const TCGOpcode *opt_opc; + /* The data argument to the out-of-line helper. */ + int32_t data; + /* The vector element size, if applicable. */ + uint8_t vece; + /* Prefer i64 to v64. */ + bool prefer_i64; + /* Load dest as a 3rd source operand. */ + bool load_dest; +} GVecGen3; + +typedef struct { + /* + * Expand inline as a 64-bit or 32-bit integer. Only one of these will be + * non-NULL. + */ + void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, int64_t); + void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, int32_t); + /* Expand inline with a host vector type. */ + void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, int64_t); + /* Expand out-of-line helper w/descriptor, data in descriptor. */ + gen_helper_gvec_3 *fno; + /* The optional opcodes, if any, utilized by .fniv. */ + const TCGOpcode *opt_opc; + /* The vector element size, if applicable. */ + uint8_t vece; + /* Prefer i64 to v64. */ + bool prefer_i64; + /* Load dest as a 3rd source operand. */ + bool load_dest; +} GVecGen3i; + +typedef struct { + /* Expand inline as a 64-bit or 32-bit integer. + Only one of these will be non-NULL. */ + void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64); + void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32); + /* Expand inline with a host vector type. */ + void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, TCGv_vec); + /* Expand out-of-line helper w/descriptor. */ + gen_helper_gvec_4 *fno; + /* The optional opcodes, if any, utilized by .fniv. */ + const TCGOpcode *opt_opc; + /* The data argument to the out-of-line helper. */ + int32_t data; + /* The vector element size, if applicable. */ + uint8_t vece; + /* Prefer i64 to v64. */ + bool prefer_i64; + /* Write aofs as a 2nd dest operand. */ + bool write_aofs; +} GVecGen4; + +typedef struct { + /* + * Expand inline as a 64-bit or 32-bit integer. Only one of these will be + * non-NULL. + */ + void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64, int64_t); + void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32, int32_t); + /* Expand inline with a host vector type. */ + void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, TCGv_vec, int64_t); + /* Expand out-of-line helper w/descriptor, data in descriptor. */ + gen_helper_gvec_4 *fno; + /* The optional opcodes, if any, utilized by .fniv. */ + const TCGOpcode *opt_opc; + /* The vector element size, if applicable. */ + uint8_t vece; + /* Prefer i64 to v64. */ + bool prefer_i64; +} GVecGen4i; + +void tcg_gen_gvec_2(uint32_t dofs, uint32_t aofs, + uint32_t oprsz, uint32_t maxsz, const GVecGen2 *); +void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz, + uint32_t maxsz, int64_t c, const GVecGen2i *); +void tcg_gen_gvec_2s(uint32_t dofs, uint32_t aofs, uint32_t oprsz, + uint32_t maxsz, TCGv_i64 c, const GVecGen2s *); +void tcg_gen_gvec_3(uint32_t dofs, uint32_t aofs, uint32_t bofs, + uint32_t oprsz, uint32_t maxsz, const GVecGen3 *); +void tcg_gen_gvec_3i(uint32_t dofs, uint32_t aofs, uint32_t bofs, + uint32_t oprsz, uint32_t maxsz, int64_t c, + const GVecGen3i *); +void tcg_gen_gvec_4(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs, + uint32_t oprsz, uint32_t maxsz, const GVecGen4 *); +void tcg_gen_gvec_4i(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs, + uint32_t oprsz, uint32_t maxsz, int64_t c, + const GVecGen4i *); + +/* Expand a specific vector operation. */ + +void tcg_gen_gvec_mov(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_not(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_neg(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_abs(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t oprsz, uint32_t maxsz); + +void tcg_gen_gvec_add(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_sub(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_mul(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); + +void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t c, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t c, uint32_t oprsz, uint32_t maxsz); + +void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_subs(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); + +/* Saturated arithmetic. */ +void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); + +/* Min/max. */ +void tcg_gen_gvec_smin(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_umin(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_smax(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_umax(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); + +void tcg_gen_gvec_and(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_or(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_xor(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_andc(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_orc(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_nand(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_nor(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_eqv(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); + +void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t c, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t c, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t c, uint32_t oprsz, uint32_t maxsz); + +void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_andcs(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); + +void tcg_gen_gvec_dup_mem(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t s, uint32_t m); +void tcg_gen_gvec_dup_imm(unsigned vece, uint32_t dofs, uint32_t s, + uint32_t m, uint64_t imm); +void tcg_gen_gvec_dup_i32(unsigned vece, uint32_t dofs, uint32_t s, + uint32_t m, TCGv_i32); +void tcg_gen_gvec_dup_i64(unsigned vece, uint32_t dofs, uint32_t s, + uint32_t m, TCGv_i64); + +void tcg_gen_gvec_shli(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t shift, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_shri(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t shift, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_sari(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t shift, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_rotli(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t shift, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_rotri(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t shift, uint32_t oprsz, uint32_t maxsz); + +void tcg_gen_gvec_shls(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_shrs(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_sars(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_rotls(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_rotrs(unsigned vece, uint32_t dofs, uint32_t aofs, + TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); + +/* + * Perform vector shift by vector element, modulo the element size. + * E.g. D[i] = A[i] << (B[i] % (8 << vece)). + */ +void tcg_gen_gvec_shlv(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_shrv(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_sarv(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_rotlv(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +void tcg_gen_gvec_rotrv(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t oprsz, uint32_t maxsz); + +void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs, + uint32_t aofs, uint32_t bofs, + uint32_t oprsz, uint32_t maxsz); + +/* + * Perform vector bit select: d = (b & a) | (c & ~a). + */ +void tcg_gen_gvec_bitsel(unsigned vece, uint32_t dofs, uint32_t aofs, + uint32_t bofs, uint32_t cofs, + uint32_t oprsz, uint32_t maxsz); + +/* + * 64-bit vector operations. Use these when the register has been allocated + * with tcg_global_mem_new_i64, and so we cannot also address it via pointer. + * OPRSZ = MAXSZ = 8. + */ + +void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 a); +void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 a); +void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 a); + +void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); +void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); +void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); + +void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); +void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); +void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); + +void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); +void tcg_gen_vec_shl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); +void tcg_gen_vec_shr8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); +void tcg_gen_vec_shr16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); +void tcg_gen_vec_sar8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); +void tcg_gen_vec_sar16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); +void tcg_gen_vec_rotl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c); +void tcg_gen_vec_rotl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c); + +/* 32-bit vector operations. */ +void tcg_gen_vec_add8_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b); +void tcg_gen_vec_add16_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b); + +void tcg_gen_vec_sub8_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b); +void tcg_gen_vec_sub16_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b); + +void tcg_gen_vec_shl8i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); +void tcg_gen_vec_shl16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); +void tcg_gen_vec_shr8i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); +void tcg_gen_vec_shr16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); +void tcg_gen_vec_sar8i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); +void tcg_gen_vec_sar16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); + +#endif diff --git a/include/tcg/tcg-op-gvec.h b/include/tcg/tcg-op-gvec.h index a8183bf..b0a81ad 100644 --- a/include/tcg/tcg-op-gvec.h +++ b/include/tcg/tcg-op-gvec.h @@ -1,447 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Generic vector operation expansion + * Target dependent generic vector operation expansion * * Copyright (c) 2018 Linaro - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, see <http://www.gnu.org/licenses/>. */ #ifndef TCG_TCG_OP_GVEC_H #define TCG_TCG_OP_GVEC_H -/* - * "Generic" vectors. All operands are given as offsets from ENV, - * and therefore cannot also be allocated via tcg_global_mem_new_*. - * OPRSZ is the byte size of the vector upon which the operation is performed. - * MAXSZ is the byte size of the full vector; bytes beyond OPSZ are cleared. - * - * All sizes must be 8 or any multiple of 16. - * When OPRSZ is 8, the alignment may be 8, otherwise must be 16. - * Operands may completely, but not partially, overlap. - */ - -/* Expand a call to a gvec-style helper, with pointers to two vector - operands, and a descriptor (see tcg-gvec-desc.h). */ -typedef void gen_helper_gvec_2(TCGv_ptr, TCGv_ptr, TCGv_i32); -void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs, - uint32_t oprsz, uint32_t maxsz, int32_t data, - gen_helper_gvec_2 *fn); - -/* Similarly, passing an extra data value. */ -typedef void gen_helper_gvec_2i(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32); -void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c, - uint32_t oprsz, uint32_t maxsz, int32_t data, - gen_helper_gvec_2i *fn); - -/* Similarly, passing an extra pointer (e.g. env or float_status). */ -typedef void gen_helper_gvec_2_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32); -void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs, - TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz, - int32_t data, gen_helper_gvec_2_ptr *fn); - -/* Similarly, with three vector operands. */ -typedef void gen_helper_gvec_3(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32); -void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, - uint32_t oprsz, uint32_t maxsz, int32_t data, - gen_helper_gvec_3 *fn); - -/* Similarly, with four vector operands. */ -typedef void gen_helper_gvec_4(TCGv_ptr, TCGv_ptr, TCGv_ptr, - TCGv_ptr, TCGv_i32); -void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, - uint32_t cofs, uint32_t oprsz, uint32_t maxsz, - int32_t data, gen_helper_gvec_4 *fn); - -/* Similarly, with five vector operands. */ -typedef void gen_helper_gvec_5(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_ptr, - TCGv_ptr, TCGv_i32); -void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, - uint32_t cofs, uint32_t xofs, uint32_t oprsz, - uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn); - -typedef void gen_helper_gvec_3_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, - TCGv_ptr, TCGv_i32); -void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, - TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz, - int32_t data, gen_helper_gvec_3_ptr *fn); - -typedef void gen_helper_gvec_4_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, - TCGv_ptr, TCGv_ptr, TCGv_i32); -void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, - uint32_t cofs, TCGv_ptr ptr, uint32_t oprsz, - uint32_t maxsz, int32_t data, - gen_helper_gvec_4_ptr *fn); - -typedef void gen_helper_gvec_5_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_ptr, - TCGv_ptr, TCGv_ptr, TCGv_i32); -void tcg_gen_gvec_5_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, - uint32_t cofs, uint32_t eofs, TCGv_ptr ptr, - uint32_t oprsz, uint32_t maxsz, int32_t data, - gen_helper_gvec_5_ptr *fn); - -/* Expand a gvec operation. Either inline or out-of-line depending on - the actual vector size and the operations supported by the host. */ -typedef struct { - /* Expand inline as a 64-bit or 32-bit integer. - Only one of these will be non-NULL. */ - void (*fni8)(TCGv_i64, TCGv_i64); - void (*fni4)(TCGv_i32, TCGv_i32); - /* Expand inline with a host vector type. */ - void (*fniv)(unsigned, TCGv_vec, TCGv_vec); - /* Expand out-of-line helper w/descriptor. */ - gen_helper_gvec_2 *fno; - /* The optional opcodes, if any, utilized by .fniv. */ - const TCGOpcode *opt_opc; - /* The data argument to the out-of-line helper. */ - int32_t data; - /* The vector element size, if applicable. */ - uint8_t vece; - /* Prefer i64 to v64. */ - bool prefer_i64; - /* Load dest as a 2nd source operand. */ - bool load_dest; -} GVecGen2; - -typedef struct { - /* Expand inline as a 64-bit or 32-bit integer. - Only one of these will be non-NULL. */ - void (*fni8)(TCGv_i64, TCGv_i64, int64_t); - void (*fni4)(TCGv_i32, TCGv_i32, int32_t); - /* Expand inline with a host vector type. */ - void (*fniv)(unsigned, TCGv_vec, TCGv_vec, int64_t); - /* Expand out-of-line helper w/descriptor, data in descriptor. */ - gen_helper_gvec_2 *fno; - /* Expand out-of-line helper w/descriptor, data as argument. */ - gen_helper_gvec_2i *fnoi; - /* The optional opcodes, if any, utilized by .fniv. */ - const TCGOpcode *opt_opc; - /* The vector element size, if applicable. */ - uint8_t vece; - /* Prefer i64 to v64. */ - bool prefer_i64; - /* Load dest as a 3rd source operand. */ - bool load_dest; -} GVecGen2i; - -typedef struct { - /* Expand inline as a 64-bit or 32-bit integer. - Only one of these will be non-NULL. */ - void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64); - void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32); - /* Expand inline with a host vector type. */ - void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec); - /* Expand out-of-line helper w/descriptor. */ - gen_helper_gvec_2i *fno; - /* The optional opcodes, if any, utilized by .fniv. */ - const TCGOpcode *opt_opc; - /* The data argument to the out-of-line helper. */ - uint32_t data; - /* The vector element size, if applicable. */ - uint8_t vece; - /* Prefer i64 to v64. */ - bool prefer_i64; - /* Load scalar as 1st source operand. */ - bool scalar_first; -} GVecGen2s; - -typedef struct { - /* Expand inline as a 64-bit or 32-bit integer. - Only one of these will be non-NULL. */ - void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64); - void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32); - /* Expand inline with a host vector type. */ - void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec); - /* Expand out-of-line helper w/descriptor. */ - gen_helper_gvec_3 *fno; - /* The optional opcodes, if any, utilized by .fniv. */ - const TCGOpcode *opt_opc; - /* The data argument to the out-of-line helper. */ - int32_t data; - /* The vector element size, if applicable. */ - uint8_t vece; - /* Prefer i64 to v64. */ - bool prefer_i64; - /* Load dest as a 3rd source operand. */ - bool load_dest; -} GVecGen3; - -typedef struct { - /* - * Expand inline as a 64-bit or 32-bit integer. Only one of these will be - * non-NULL. - */ - void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, int64_t); - void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, int32_t); - /* Expand inline with a host vector type. */ - void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, int64_t); - /* Expand out-of-line helper w/descriptor, data in descriptor. */ - gen_helper_gvec_3 *fno; - /* The optional opcodes, if any, utilized by .fniv. */ - const TCGOpcode *opt_opc; - /* The vector element size, if applicable. */ - uint8_t vece; - /* Prefer i64 to v64. */ - bool prefer_i64; - /* Load dest as a 3rd source operand. */ - bool load_dest; -} GVecGen3i; - -typedef struct { - /* Expand inline as a 64-bit or 32-bit integer. - Only one of these will be non-NULL. */ - void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64); - void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32); - /* Expand inline with a host vector type. */ - void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, TCGv_vec); - /* Expand out-of-line helper w/descriptor. */ - gen_helper_gvec_4 *fno; - /* The optional opcodes, if any, utilized by .fniv. */ - const TCGOpcode *opt_opc; - /* The data argument to the out-of-line helper. */ - int32_t data; - /* The vector element size, if applicable. */ - uint8_t vece; - /* Prefer i64 to v64. */ - bool prefer_i64; - /* Write aofs as a 2nd dest operand. */ - bool write_aofs; -} GVecGen4; - -typedef struct { - /* - * Expand inline as a 64-bit or 32-bit integer. Only one of these will be - * non-NULL. - */ - void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64, int64_t); - void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32, int32_t); - /* Expand inline with a host vector type. */ - void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, TCGv_vec, int64_t); - /* Expand out-of-line helper w/descriptor, data in descriptor. */ - gen_helper_gvec_4 *fno; - /* The optional opcodes, if any, utilized by .fniv. */ - const TCGOpcode *opt_opc; - /* The vector element size, if applicable. */ - uint8_t vece; - /* Prefer i64 to v64. */ - bool prefer_i64; -} GVecGen4i; - -void tcg_gen_gvec_2(uint32_t dofs, uint32_t aofs, - uint32_t oprsz, uint32_t maxsz, const GVecGen2 *); -void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz, - uint32_t maxsz, int64_t c, const GVecGen2i *); -void tcg_gen_gvec_2s(uint32_t dofs, uint32_t aofs, uint32_t oprsz, - uint32_t maxsz, TCGv_i64 c, const GVecGen2s *); -void tcg_gen_gvec_3(uint32_t dofs, uint32_t aofs, uint32_t bofs, - uint32_t oprsz, uint32_t maxsz, const GVecGen3 *); -void tcg_gen_gvec_3i(uint32_t dofs, uint32_t aofs, uint32_t bofs, - uint32_t oprsz, uint32_t maxsz, int64_t c, - const GVecGen3i *); -void tcg_gen_gvec_4(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs, - uint32_t oprsz, uint32_t maxsz, const GVecGen4 *); -void tcg_gen_gvec_4i(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs, - uint32_t oprsz, uint32_t maxsz, int64_t c, - const GVecGen4i *); - -/* Expand a specific vector operation. */ - -void tcg_gen_gvec_mov(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_not(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_neg(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_abs(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t oprsz, uint32_t maxsz); - -void tcg_gen_gvec_add(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_sub(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_mul(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); - -void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t c, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t c, uint32_t oprsz, uint32_t maxsz); - -void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_subs(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); - -/* Saturated arithmetic. */ -void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); +#include "tcg/tcg-op-gvec-common.h" -/* Min/max. */ -void tcg_gen_gvec_smin(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_umin(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_smax(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_umax(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); - -void tcg_gen_gvec_and(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_or(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_xor(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_andc(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_orc(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_nand(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_nor(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_eqv(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); - -void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t c, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t c, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t c, uint32_t oprsz, uint32_t maxsz); - -void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_andcs(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); - -void tcg_gen_gvec_dup_mem(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t s, uint32_t m); -void tcg_gen_gvec_dup_imm(unsigned vece, uint32_t dofs, uint32_t s, - uint32_t m, uint64_t imm); -void tcg_gen_gvec_dup_i32(unsigned vece, uint32_t dofs, uint32_t s, - uint32_t m, TCGv_i32); -void tcg_gen_gvec_dup_i64(unsigned vece, uint32_t dofs, uint32_t s, - uint32_t m, TCGv_i64); - -#if TARGET_LONG_BITS == 64 -# define tcg_gen_gvec_dup_tl tcg_gen_gvec_dup_i64 -#else -# define tcg_gen_gvec_dup_tl tcg_gen_gvec_dup_i32 +#ifndef TARGET_LONG_BITS +#error must include QEMU headers #endif -void tcg_gen_gvec_shli(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t shift, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_shri(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t shift, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_sari(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t shift, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_rotli(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t shift, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_rotri(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t shift, uint32_t oprsz, uint32_t maxsz); - -void tcg_gen_gvec_shls(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_shrs(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_sars(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_rotls(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_rotrs(unsigned vece, uint32_t dofs, uint32_t aofs, - TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); - -/* - * Perform vector shift by vector element, modulo the element size. - * E.g. D[i] = A[i] << (B[i] % (8 << vece)). - */ -void tcg_gen_gvec_shlv(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_shrv(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_sarv(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_rotlv(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); -void tcg_gen_gvec_rotrv(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t oprsz, uint32_t maxsz); - -void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs, - uint32_t aofs, uint32_t bofs, - uint32_t oprsz, uint32_t maxsz); - -/* - * Perform vector bit select: d = (b & a) | (c & ~a). - */ -void tcg_gen_gvec_bitsel(unsigned vece, uint32_t dofs, uint32_t aofs, - uint32_t bofs, uint32_t cofs, - uint32_t oprsz, uint32_t maxsz); - -/* - * 64-bit vector operations. Use these when the register has been allocated - * with tcg_global_mem_new_i64, and so we cannot also address it via pointer. - * OPRSZ = MAXSZ = 8. - */ - -void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 a); -void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 a); -void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 a); - -void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); -void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); -void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); - -void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); -void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); -void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); - -void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); -void tcg_gen_vec_shl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); -void tcg_gen_vec_shr8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); -void tcg_gen_vec_shr16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); -void tcg_gen_vec_sar8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); -void tcg_gen_vec_sar16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); -void tcg_gen_vec_rotl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c); -void tcg_gen_vec_rotl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c); - -/* 32-bit vector operations. */ -void tcg_gen_vec_add8_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b); -void tcg_gen_vec_add16_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b); - -void tcg_gen_vec_sub8_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b); -void tcg_gen_vec_sub16_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b); - -void tcg_gen_vec_shl8i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); -void tcg_gen_vec_shl16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); -void tcg_gen_vec_shr8i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); -void tcg_gen_vec_shr16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); -void tcg_gen_vec_sar8i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); -void tcg_gen_vec_sar16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); - #if TARGET_LONG_BITS == 64 +#define tcg_gen_gvec_dup_tl tcg_gen_gvec_dup_i64 #define tcg_gen_vec_add8_tl tcg_gen_vec_add8_i64 #define tcg_gen_vec_sub8_tl tcg_gen_vec_sub8_i64 #define tcg_gen_vec_add16_tl tcg_gen_vec_add16_i64 @@ -454,8 +28,8 @@ void tcg_gen_vec_sar16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); #define tcg_gen_vec_shl16i_tl tcg_gen_vec_shl16i_i64 #define tcg_gen_vec_shr16i_tl tcg_gen_vec_shr16i_i64 #define tcg_gen_vec_sar16i_tl tcg_gen_vec_sar16i_i64 - -#else +#elif TARGET_LONG_BITS == 32 +#define tcg_gen_gvec_dup_tl tcg_gen_gvec_dup_i32 #define tcg_gen_vec_add8_tl tcg_gen_vec_add8_i32 #define tcg_gen_vec_sub8_tl tcg_gen_vec_sub8_i32 #define tcg_gen_vec_add16_tl tcg_gen_vec_add16_i32 @@ -468,6 +42,8 @@ void tcg_gen_vec_sar16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t); #define tcg_gen_vec_shl16i_tl tcg_gen_vec_shl16i_i32 #define tcg_gen_vec_shr16i_tl tcg_gen_vec_shr16i_i32 #define tcg_gen_vec_sar16i_tl tcg_gen_vec_sar16i_i32 +#else +# error #endif #endif diff --git a/include/tcg/tcg-op.h b/include/tcg/tcg-op.h index 35c5700..d63683c 100644 --- a/include/tcg/tcg-op.h +++ b/include/tcg/tcg-op.h @@ -1,741 +1,41 @@ +/* SPDX-License-Identifier: MIT */ /* - * Tiny Code Generator for QEMU + * Target dependent opcode generation functions. * * Copyright (c) 2008 Fabrice Bellard - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. */ #ifndef TCG_TCG_OP_H #define TCG_TCG_OP_H -#include "tcg/tcg.h" -#include "exec/helper-proto.h" -#include "exec/helper-gen.h" - -/* Basic output routines. Not for general consumption. */ - -void tcg_gen_op1(TCGOpcode, TCGArg); -void tcg_gen_op2(TCGOpcode, TCGArg, TCGArg); -void tcg_gen_op3(TCGOpcode, TCGArg, TCGArg, TCGArg); -void tcg_gen_op4(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg); -void tcg_gen_op5(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg); -void tcg_gen_op6(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg); - -void vec_gen_2(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg); -void vec_gen_3(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg, TCGArg); -void vec_gen_4(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg, TCGArg, TCGArg); - -static inline void tcg_gen_op1_i32(TCGOpcode opc, TCGv_i32 a1) -{ - tcg_gen_op1(opc, tcgv_i32_arg(a1)); -} - -static inline void tcg_gen_op1_i64(TCGOpcode opc, TCGv_i64 a1) -{ - tcg_gen_op1(opc, tcgv_i64_arg(a1)); -} - -static inline void tcg_gen_op1i(TCGOpcode opc, TCGArg a1) -{ - tcg_gen_op1(opc, a1); -} - -static inline void tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2) -{ - tcg_gen_op2(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2)); -} - -static inline void tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2) -{ - tcg_gen_op2(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2)); -} - -static inline void tcg_gen_op2i_i32(TCGOpcode opc, TCGv_i32 a1, TCGArg a2) -{ - tcg_gen_op2(opc, tcgv_i32_arg(a1), a2); -} - -static inline void tcg_gen_op2i_i64(TCGOpcode opc, TCGv_i64 a1, TCGArg a2) -{ - tcg_gen_op2(opc, tcgv_i64_arg(a1), a2); -} - -static inline void tcg_gen_op2ii(TCGOpcode opc, TCGArg a1, TCGArg a2) -{ - tcg_gen_op2(opc, a1, a2); -} - -static inline void tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 a1, - TCGv_i32 a2, TCGv_i32 a3) -{ - tcg_gen_op3(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), tcgv_i32_arg(a3)); -} - -static inline void tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 a1, - TCGv_i64 a2, TCGv_i64 a3) -{ - tcg_gen_op3(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), tcgv_i64_arg(a3)); -} - -static inline void tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 a1, - TCGv_i32 a2, TCGArg a3) -{ - tcg_gen_op3(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3); -} - -static inline void tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 a1, - TCGv_i64 a2, TCGArg a3) -{ - tcg_gen_op3(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3); -} - -static inline void tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val, - TCGv_ptr base, TCGArg offset) -{ - tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_ptr_arg(base), offset); -} - -static inline void tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val, - TCGv_ptr base, TCGArg offset) -{ - tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_ptr_arg(base), offset); -} - -static inline void tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGv_i32 a3, TCGv_i32 a4) -{ - tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), - tcgv_i32_arg(a3), tcgv_i32_arg(a4)); -} - -static inline void tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGv_i64 a3, TCGv_i64 a4) -{ - tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), - tcgv_i64_arg(a3), tcgv_i64_arg(a4)); -} - -static inline void tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGv_i32 a3, TCGArg a4) -{ - tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), - tcgv_i32_arg(a3), a4); -} - -static inline void tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGv_i64 a3, TCGArg a4) -{ - tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), - tcgv_i64_arg(a3), a4); -} - -static inline void tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGArg a3, TCGArg a4) -{ - tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4); -} - -static inline void tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGArg a3, TCGArg a4) -{ - tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4); -} - -static inline void tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5) -{ - tcg_gen_op5(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), - tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5)); -} - -static inline void tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGv_i64 a3, TCGv_i64 a4, TCGv_i64 a5) -{ - tcg_gen_op5(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), - tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5)); -} - -static inline void tcg_gen_op5i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGv_i32 a3, TCGv_i32 a4, TCGArg a5) -{ - tcg_gen_op5(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), - tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5); -} - -static inline void tcg_gen_op5i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGv_i64 a3, TCGv_i64 a4, TCGArg a5) -{ - tcg_gen_op5(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), - tcgv_i64_arg(a3), tcgv_i64_arg(a4), a5); -} - -static inline void tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGv_i32 a3, TCGArg a4, TCGArg a5) -{ - tcg_gen_op5(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), - tcgv_i32_arg(a3), a4, a5); -} - -static inline void tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGv_i64 a3, TCGArg a4, TCGArg a5) -{ - tcg_gen_op5(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), - tcgv_i64_arg(a3), a4, a5); -} - -static inline void tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGv_i32 a3, TCGv_i32 a4, - TCGv_i32 a5, TCGv_i32 a6) -{ - tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), - tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), - tcgv_i32_arg(a6)); -} - -static inline void tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGv_i64 a3, TCGv_i64 a4, - TCGv_i64 a5, TCGv_i64 a6) -{ - tcg_gen_op6(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), - tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), - tcgv_i64_arg(a6)); -} - -static inline void tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGv_i32 a3, TCGv_i32 a4, - TCGv_i32 a5, TCGArg a6) -{ - tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), - tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), a6); -} - -static inline void tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGv_i64 a3, TCGv_i64 a4, - TCGv_i64 a5, TCGArg a6) -{ - tcg_gen_op6(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), - tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6); -} - -static inline void tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGv_i32 a3, TCGv_i32 a4, - TCGArg a5, TCGArg a6) -{ - tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), - tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6); -} - -static inline void tcg_gen_op6ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGv_i64 a3, TCGv_i64 a4, - TCGArg a5, TCGArg a6) -{ - tcg_gen_op6(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), - tcgv_i64_arg(a3), tcgv_i64_arg(a4), a5, a6); -} - - -/* Generic ops. */ - -static inline void gen_set_label(TCGLabel *l) -{ - l->present = 1; - tcg_gen_op1(INDEX_op_set_label, label_arg(l)); -} - -void tcg_gen_br(TCGLabel *l); -void tcg_gen_mb(TCGBar); - -/* Helper calls. */ - -/* 32 bit ops */ - -void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg); -void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2); -void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2); -void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2); -void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg); -void tcg_gen_ctpop_i32(TCGv_i32 a1, TCGv_i32 a2); -void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2); -void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, - unsigned int ofs, unsigned int len); -void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, - unsigned int ofs, unsigned int len); -void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, - unsigned int ofs, unsigned int len); -void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, - unsigned int ofs, unsigned int len); -void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, - unsigned int ofs); -void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *); -void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *); -void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, - TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, - TCGv_i32 arg1, int32_t arg2); -void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, - TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2); -void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, - TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh); -void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, - TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh); -void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg); -void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg); -void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg); -void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg); -void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags); -void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg); -void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg); -void tcg_gen_smin_i32(TCGv_i32, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_smax_i32(TCGv_i32, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_umin_i32(TCGv_i32, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_umax_i32(TCGv_i32, TCGv_i32 arg1, TCGv_i32 arg2); -void tcg_gen_abs_i32(TCGv_i32, TCGv_i32); - -/* Replicate a value of size @vece from @in to all the lanes in @out */ -void tcg_gen_dup_i32(unsigned vece, TCGv_i32 out, TCGv_i32 in); - -static inline void tcg_gen_discard_i32(TCGv_i32 arg) -{ - tcg_gen_op1_i32(INDEX_op_discard, arg); -} - -static inline void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg) -{ - if (ret != arg) { - tcg_gen_op2_i32(INDEX_op_mov_i32, ret, arg); - } -} - -static inline void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset); -} - -static inline void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset); -} - -static inline void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset); -} - -static inline void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset); -} - -static inline void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset); -} - -static inline void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset); -} - -static inline void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset); -} - -static inline void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset); -} - -static inline void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) -{ - tcg_gen_op3_i32(INDEX_op_add_i32, ret, arg1, arg2); -} - -static inline void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) -{ - tcg_gen_op3_i32(INDEX_op_sub_i32, ret, arg1, arg2); -} - -static inline void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) -{ - tcg_gen_op3_i32(INDEX_op_and_i32, ret, arg1, arg2); -} - -static inline void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) -{ - tcg_gen_op3_i32(INDEX_op_or_i32, ret, arg1, arg2); -} - -static inline void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) -{ - tcg_gen_op3_i32(INDEX_op_xor_i32, ret, arg1, arg2); -} - -static inline void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) -{ - tcg_gen_op3_i32(INDEX_op_shl_i32, ret, arg1, arg2); -} - -static inline void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) -{ - tcg_gen_op3_i32(INDEX_op_shr_i32, ret, arg1, arg2); -} - -static inline void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) -{ - tcg_gen_op3_i32(INDEX_op_sar_i32, ret, arg1, arg2); -} - -static inline void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) -{ - tcg_gen_op3_i32(INDEX_op_mul_i32, ret, arg1, arg2); -} - -static inline void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg) -{ - if (TCG_TARGET_HAS_neg_i32) { - tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg); - } else { - tcg_gen_subfi_i32(ret, 0, arg); - } -} - -static inline void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) -{ - if (TCG_TARGET_HAS_not_i32) { - tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg); - } else { - tcg_gen_xori_i32(ret, arg, -1); - } -} - -/* 64 bit ops */ - -void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg); -void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2); -void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2); -void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2); -void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_ctpop_i64(TCGv_i64 a1, TCGv_i64 a2); -void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2); -void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, - unsigned int ofs, unsigned int len); -void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, - unsigned int ofs, unsigned int len); -void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, - unsigned int ofs, unsigned int len); -void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, - unsigned int ofs, unsigned int len); -void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, - unsigned int ofs); -void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *); -void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *); -void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, - TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, - TCGv_i64 arg1, int64_t arg2); -void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, - TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2); -void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, - TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh); -void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, - TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh); -void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags); -void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags); -void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_smin_i64(TCGv_i64, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_smax_i64(TCGv_i64, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_umin_i64(TCGv_i64, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_umax_i64(TCGv_i64, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_abs_i64(TCGv_i64, TCGv_i64); - -/* Replicate a value of size @vece from @in to all the lanes in @out */ -void tcg_gen_dup_i64(unsigned vece, TCGv_i64 out, TCGv_i64 in); - -#if TCG_TARGET_REG_BITS == 64 -static inline void tcg_gen_discard_i64(TCGv_i64 arg) -{ - tcg_gen_op1_i64(INDEX_op_discard, arg); -} - -static inline void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) -{ - if (ret != arg) { - tcg_gen_op2_i64(INDEX_op_mov_i64, ret, arg); - } -} - -static inline void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset); -} - -static inline void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset); -} - -static inline void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset); -} - -static inline void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset); -} - -static inline void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset); -} - -static inline void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset); -} - -static inline void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset); -} - -static inline void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset); -} - -static inline void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset); -} - -static inline void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset); -} - -static inline void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset); -} - -static inline void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_op3_i64(INDEX_op_add_i64, ret, arg1, arg2); -} - -static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_op3_i64(INDEX_op_sub_i64, ret, arg1, arg2); -} - -static inline void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_op3_i64(INDEX_op_and_i64, ret, arg1, arg2); -} - -static inline void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_op3_i64(INDEX_op_or_i64, ret, arg1, arg2); -} - -static inline void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_op3_i64(INDEX_op_xor_i64, ret, arg1, arg2); -} - -static inline void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_op3_i64(INDEX_op_shl_i64, ret, arg1, arg2); -} - -static inline void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_op3_i64(INDEX_op_shr_i64, ret, arg1, arg2); -} - -static inline void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_op3_i64(INDEX_op_sar_i64, ret, arg1, arg2); -} - -static inline void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_op3_i64(INDEX_op_mul_i64, ret, arg1, arg2); -} -#else /* TCG_TARGET_REG_BITS == 32 */ -void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); -void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); -void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); - -void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); - -void tcg_gen_discard_i64(TCGv_i64 arg); -void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg); -void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); -void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); -void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); -void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); -void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); -void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); -void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset); -void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); -void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); -#endif /* TCG_TARGET_REG_BITS */ - -static inline void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg) -{ - if (TCG_TARGET_HAS_neg_i64) { - tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg); - } else { - tcg_gen_subfi_i64(ret, 0, arg); - } -} - -/* Size changing operations. */ - -void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg); -void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg); -void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high); -void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg); -void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg); -void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg); -void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg); - -void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src); -void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg); -void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi); - -static inline void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi) -{ - tcg_gen_deposit_i64(ret, lo, hi, 32, 32); -} - -/* QEMU specific operations. */ +#include "tcg/tcg-op-common.h" #ifndef TARGET_LONG_BITS #error must include QEMU headers #endif -#if TARGET_INSN_START_WORDS == 1 +#if TARGET_LONG_BITS == 32 +# define TCG_TYPE_TL TCG_TYPE_I32 +#elif TARGET_LONG_BITS == 64 +# define TCG_TYPE_TL TCG_TYPE_I64 +#else +# error +#endif + +#ifndef TARGET_INSN_START_EXTRA_WORDS static inline void tcg_gen_insn_start(target_ulong pc) { TCGOp *op = tcg_emit_op(INDEX_op_insn_start, 64 / TCG_TARGET_REG_BITS); tcg_set_insn_start_param(op, 0, pc); } -#elif TARGET_INSN_START_WORDS == 2 +#elif TARGET_INSN_START_EXTRA_WORDS == 1 static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1) { TCGOp *op = tcg_emit_op(INDEX_op_insn_start, 2 * 64 / TCG_TARGET_REG_BITS); tcg_set_insn_start_param(op, 0, pc); tcg_set_insn_start_param(op, 1, a1); } -#elif TARGET_INSN_START_WORDS == 3 +#elif TARGET_INSN_START_EXTRA_WORDS == 2 static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1, target_ulong a2) { @@ -745,83 +45,29 @@ static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1, tcg_set_insn_start_param(op, 2, a2); } #else -# error "Unhandled number of operands to insn_start" +#error Unhandled TARGET_INSN_START_EXTRA_WORDS value #endif -/** - * tcg_gen_exit_tb() - output exit_tb TCG operation - * @tb: The TranslationBlock from which we are exiting - * @idx: Direct jump slot index, or exit request - * - * See tcg/README for more info about this TCG operation. - * See also tcg.h and the block comment above TB_EXIT_MASK. - * - * For a normal exit from the TB, back to the main loop, @tb should - * be NULL and @idx should be 0. Otherwise, @tb should be valid and - * @idx should be one of the TB_EXIT_ values. - */ -void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx); - -/** - * tcg_gen_goto_tb() - output goto_tb TCG operation - * @idx: Direct jump slot index (0 or 1) - * - * See tcg/README for more info about this TCG operation. - * - * NOTE: In softmmu emulation, direct jumps with goto_tb are only safe within - * the pages this TB resides in because we don't take care of direct jumps when - * address mapping changes, e.g. in tlb_flush(). In user mode, there's only a - * static address translation, so the destination address is always valid, TBs - * are always invalidated properly, and direct jumps are reset when mapping - * changes. - */ -void tcg_gen_goto_tb(unsigned idx); - -/** - * tcg_gen_lookup_and_goto_ptr() - look up the current TB, jump to it if valid - * @addr: Guest address of the target TB - * - * If the TB is not valid, jump to the epilogue. - * - * This operation is optional. If the TCG backend does not implement goto_ptr, - * this op is equivalent to calling tcg_gen_exit_tb() with 0 as the argument. - */ -void tcg_gen_lookup_and_goto_ptr(void); - -static inline void tcg_gen_plugin_cb_start(unsigned from, unsigned type, - unsigned wr) -{ - tcg_gen_op3(INDEX_op_plugin_cb_start, from, type, wr); -} - -static inline void tcg_gen_plugin_cb_end(void) -{ - tcg_emit_op(INDEX_op_plugin_cb_end, 0); -} - #if TARGET_LONG_BITS == 32 +typedef TCGv_i32 TCGv; #define tcg_temp_new() tcg_temp_new_i32() #define tcg_global_mem_new tcg_global_mem_new_i32 #define tcg_temp_free tcg_temp_free_i32 #define tcgv_tl_temp tcgv_i32_temp #define tcg_gen_qemu_ld_tl tcg_gen_qemu_ld_i32 #define tcg_gen_qemu_st_tl tcg_gen_qemu_st_i32 -#else +#elif TARGET_LONG_BITS == 64 +typedef TCGv_i64 TCGv; #define tcg_temp_new() tcg_temp_new_i64() #define tcg_global_mem_new tcg_global_mem_new_i64 #define tcg_temp_free tcg_temp_free_i64 #define tcgv_tl_temp tcgv_i64_temp #define tcg_gen_qemu_ld_tl tcg_gen_qemu_ld_i64 #define tcg_gen_qemu_st_tl tcg_gen_qemu_st_i64 +#else +#error Unhandled TARGET_LONG_BITS value #endif -void tcg_gen_qemu_ld_i32_chk(TCGv_i32, TCGTemp *, TCGArg, MemOp, TCGType); -void tcg_gen_qemu_st_i32_chk(TCGv_i32, TCGTemp *, TCGArg, MemOp, TCGType); -void tcg_gen_qemu_ld_i64_chk(TCGv_i64, TCGTemp *, TCGArg, MemOp, TCGType); -void tcg_gen_qemu_st_i64_chk(TCGv_i64, TCGTemp *, TCGArg, MemOp, TCGType); -void tcg_gen_qemu_ld_i128_chk(TCGv_i128, TCGTemp *, TCGArg, MemOp, TCGType); -void tcg_gen_qemu_st_i128_chk(TCGv_i128, TCGTemp *, TCGArg, MemOp, TCGType); - static inline void tcg_gen_qemu_ld_i32(TCGv_i32 v, TCGv a, TCGArg i, MemOp m) { @@ -858,91 +104,6 @@ tcg_gen_qemu_st_i128(TCGv_i128 v, TCGv a, TCGArg i, MemOp m) tcg_gen_qemu_st_i128_chk(v, tcgv_tl_temp(a), i, m, TCG_TYPE_TL); } -void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128, - TCGv_i128, TCGArg, MemOp, TCGType); - -void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128, TCGTemp *, TCGv_i128, - TCGv_i128, TCGArg, MemOp, TCGType); - -void tcg_gen_atomic_xchg_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_xchg_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); - -void tcg_gen_atomic_fetch_add_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_add_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_and_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_and_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_or_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_or_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_xor_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_xor_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_smin_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_smin_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_umin_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_umin_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_smax_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_smax_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_umax_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_fetch_umax_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); - -void tcg_gen_atomic_add_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_add_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_and_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_and_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_or_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_or_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_xor_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_xor_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_smin_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_smin_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_umin_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_umin_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_smax_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_smax_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_umax_fetch_i32_chk(TCGv_i32, TCGTemp *, TCGv_i32, - TCGArg, MemOp, TCGType); -void tcg_gen_atomic_umax_fetch_i64_chk(TCGv_i64, TCGTemp *, TCGv_i64, - TCGArg, MemOp, TCGType); - #define DEF_ATOMIC2(N, S) \ static inline void N##_##S(TCGv_##S r, TCGv a, TCGv_##S v, \ TCGArg i, MemOp m) \ @@ -1001,63 +162,6 @@ DEF_ATOMIC2(tcg_gen_atomic_umax_fetch, i64) #undef DEF_ATOMIC2 #undef DEF_ATOMIC3 -void tcg_gen_mov_vec(TCGv_vec, TCGv_vec); -void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec, TCGv_i32); -void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec, TCGv_i64); -void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec, TCGv_ptr, tcg_target_long); -void tcg_gen_dupi_vec(unsigned vece, TCGv_vec, uint64_t); -void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a); -void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a); -void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a); -void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); -void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b); - -void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i); -void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i); -void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i); -void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i); -void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i); - -void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s); -void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s); -void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s); -void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s); - -void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec s); -void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec s); -void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec s); -void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec s); -void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec s); - -void tcg_gen_cmp_vec(TCGCond cond, unsigned vece, TCGv_vec r, - TCGv_vec a, TCGv_vec b); - -void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a, - TCGv_vec b, TCGv_vec c); -void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r, - TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d); - -void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr base, TCGArg offset); -void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr base, TCGArg offset); -void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr base, TCGArg offset, TCGType t); - #if TARGET_LONG_BITS == 64 #define tcg_gen_movi_tl tcg_gen_movi_i64 #define tcg_gen_mov_tl tcg_gen_mov_i64 @@ -1174,6 +278,7 @@ void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr base, TCGArg offset, TCGType t); #define tcg_gen_atomic_umax_fetch_tl tcg_gen_atomic_umax_fetch_i64 #define tcg_gen_dup_tl_vec tcg_gen_dup_i64_vec #define tcg_gen_dup_tl tcg_gen_dup_i64 +#define dup_const_tl dup_const #else #define tcg_gen_movi_tl tcg_gen_movi_i32 #define tcg_gen_mov_tl tcg_gen_mov_i32 @@ -1288,94 +393,14 @@ void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr base, TCGArg offset, TCGType t); #define tcg_gen_atomic_umax_fetch_tl tcg_gen_atomic_umax_fetch_i32 #define tcg_gen_dup_tl_vec tcg_gen_dup_i32_vec #define tcg_gen_dup_tl tcg_gen_dup_i32 -#endif - -#if UINTPTR_MAX == UINT32_MAX -# define PTR i32 -# define NAT TCGv_i32 -#else -# define PTR i64 -# define NAT TCGv_i64 -#endif - -static inline void tcg_gen_ld_ptr(TCGv_ptr r, TCGv_ptr a, intptr_t o) -{ - glue(tcg_gen_ld_,PTR)((NAT)r, a, o); -} - -static inline void tcg_gen_st_ptr(TCGv_ptr r, TCGv_ptr a, intptr_t o) -{ - glue(tcg_gen_st_, PTR)((NAT)r, a, o); -} - -static inline void tcg_gen_discard_ptr(TCGv_ptr a) -{ - glue(tcg_gen_discard_,PTR)((NAT)a); -} - -static inline void tcg_gen_add_ptr(TCGv_ptr r, TCGv_ptr a, TCGv_ptr b) -{ - glue(tcg_gen_add_,PTR)((NAT)r, (NAT)a, (NAT)b); -} - -static inline void tcg_gen_addi_ptr(TCGv_ptr r, TCGv_ptr a, intptr_t b) -{ - glue(tcg_gen_addi_,PTR)((NAT)r, (NAT)a, b); -} - -static inline void tcg_gen_mov_ptr(TCGv_ptr d, TCGv_ptr s) -{ - glue(tcg_gen_mov_,PTR)((NAT)d, (NAT)s); -} - -static inline void tcg_gen_movi_ptr(TCGv_ptr d, intptr_t s) -{ - glue(tcg_gen_movi_,PTR)((NAT)d, s); -} - -static inline void tcg_gen_brcondi_ptr(TCGCond cond, TCGv_ptr a, - intptr_t b, TCGLabel *label) -{ - glue(tcg_gen_brcondi_,PTR)(cond, (NAT)a, b, label); -} - -static inline void tcg_gen_ext_i32_ptr(TCGv_ptr r, TCGv_i32 a) -{ -#if UINTPTR_MAX == UINT32_MAX - tcg_gen_mov_i32((NAT)r, a); -#else - tcg_gen_ext_i32_i64((NAT)r, a); -#endif -} - -static inline void tcg_gen_trunc_i64_ptr(TCGv_ptr r, TCGv_i64 a) -{ -#if UINTPTR_MAX == UINT32_MAX - tcg_gen_extrl_i64_i32((NAT)r, a); -#else - tcg_gen_mov_i64((NAT)r, a); -#endif -} - -static inline void tcg_gen_extu_ptr_i64(TCGv_i64 r, TCGv_ptr a) -{ -#if UINTPTR_MAX == UINT32_MAX - tcg_gen_extu_i32_i64(r, (NAT)a); -#else - tcg_gen_mov_i64(r, (NAT)a); -#endif -} - -static inline void tcg_gen_trunc_ptr_i32(TCGv_i32 r, TCGv_ptr a) -{ -#if UINTPTR_MAX == UINT32_MAX - tcg_gen_mov_i32(r, (NAT)a); -#else - tcg_gen_extrl_i64_i32(r, (NAT)a); -#endif -} -#undef PTR -#undef NAT +#define dup_const_tl(VECE, C) \ + (__builtin_constant_p(VECE) \ + ? ( (VECE) == MO_8 ? 0x01010101ul * (uint8_t)(C) \ + : (VECE) == MO_16 ? 0x00010001ul * (uint16_t)(C) \ + : (VECE) == MO_32 ? 0x00000001ul * (uint32_t)(C) \ + : (qemu_build_not_reached_always(), 0)) \ + : (target_long)dup_const(VECE, C)) +#endif /* TARGET_LONG_BITS == 64 */ #endif /* TCG_TCG_OP_H */ diff --git a/include/tcg/tcg-opc.h b/include/tcg/tcg-opc.h index 21594c1..acfa5ba 100644 --- a/include/tcg/tcg-opc.h +++ b/include/tcg/tcg-opc.h @@ -188,9 +188,9 @@ DEF(mulsh_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_mulsh_i64)) #define DATA64_ARGS (TCG_TARGET_REG_BITS == 64 ? 1 : 2) -/* QEMU specific */ -DEF(insn_start, 0, 0, DATA64_ARGS * TARGET_INSN_START_WORDS, - TCG_OPF_NOT_PRESENT) +/* There are tcg_ctx->insn_start_words here, not just one. */ +DEF(insn_start, 0, 0, DATA64_ARGS, TCG_OPF_NOT_PRESENT) + DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_EXIT | TCG_OPF_BB_END) DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_EXIT | TCG_OPF_BB_END) DEF(goto_ptr, 0, 1, 0, TCG_OPF_BB_EXIT | TCG_OPF_BB_END) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index 072c35f..a498f31 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -25,13 +25,13 @@ #ifndef TCG_H #define TCG_H -#include "cpu.h" #include "exec/memop.h" #include "exec/memopidx.h" #include "qemu/bitops.h" #include "qemu/plugin.h" #include "qemu/queue.h" #include "tcg/tcg-mo.h" +#include "tcg-target-reg-bits.h" #include "tcg-target.h" #include "tcg/tcg-cond.h" #include "tcg/debug-assert.h" @@ -44,17 +44,6 @@ #define CPU_TEMP_BUF_NLONGS 128 #define TCG_STATIC_FRAME_SIZE (CPU_TEMP_BUF_NLONGS * sizeof(long)) -/* Default target word size to pointer size. */ -#ifndef TCG_TARGET_REG_BITS -# if UINTPTR_MAX == UINT32_MAX -# define TCG_TARGET_REG_BITS 32 -# elif UINTPTR_MAX == UINT64_MAX -# define TCG_TARGET_REG_BITS 64 -# else -# error Unknown pointer size for tcg target -# endif -#endif - #if TCG_TARGET_REG_BITS == 32 typedef int32_t tcg_target_long; typedef uint32_t tcg_target_ulong; @@ -69,15 +58,6 @@ typedef uint64_t tcg_target_ulong; #error unsupported #endif -/* Oversized TCG guests make things like MTTCG hard - * as we can't use atomics for cputlb updates. - */ -#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS -#define TCG_OVERSIZED_GUEST 1 -#else -#define TCG_OVERSIZED_GUEST 0 -#endif - #if TCG_TARGET_NB_REGS <= 32 typedef uint32_t TCGRegSet; #elif TCG_TARGET_NB_REGS <= 64 @@ -192,12 +172,6 @@ typedef uint64_t TCGRegSet; #define TCG_TARGET_HAS_v256 0 #endif -#ifndef TARGET_INSN_START_EXTRA_WORDS -# define TARGET_INSN_START_WORDS 1 -#else -# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS) -#endif - typedef enum TCGOpcode { #define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name, #include "tcg/tcg-opc.h" @@ -292,13 +266,6 @@ typedef enum TCGType { #else TCG_TYPE_PTR = TCG_TYPE_I64, #endif - - /* An alias for the size of the target "long", aka register. */ -#if TARGET_LONG_BITS == 64 - TCG_TYPE_TL = TCG_TYPE_I64, -#else - TCG_TYPE_TL = TCG_TYPE_I32, -#endif } TCGType; /** @@ -337,10 +304,6 @@ static inline unsigned get_alignment_bits(MemOp memop) /* A specific alignment requirement. */ a = a >> MO_ASHIFT; } -#if defined(CONFIG_SOFTMMU) - /* The requested alignment cannot overlap the TLB flags. */ - tcg_debug_assert((TLB_FLAGS_MASK & ((1 << a) - 1)) == 0); -#endif return a; } @@ -383,13 +346,6 @@ typedef struct TCGv_i128_d *TCGv_i128; typedef struct TCGv_ptr_d *TCGv_ptr; typedef struct TCGv_vec_d *TCGv_vec; typedef TCGv_ptr TCGv_env; -#if TARGET_LONG_BITS == 32 -#define TCGv TCGv_i32 -#elif TARGET_LONG_BITS == 64 -#define TCGv TCGv_i64 -#else -#error Unhandled TARGET_LONG_BITS value -#endif /* call flags */ /* Helper does not read globals (either directly or through an exception). It @@ -554,10 +510,13 @@ struct TCGContext { TCGType addr_type; /* TCG_TYPE_I32 or TCG_TYPE_I64 */ #ifdef CONFIG_SOFTMMU + int tlb_fast_offset; int page_mask; uint8_t page_bits; uint8_t tlb_dyn_max_bits; #endif + uint8_t insn_start_words; + TCGBar guest_mo; TCGRegSet reserved_regs; intptr_t current_frame_offset; @@ -629,7 +588,7 @@ struct TCGContext { TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS]; uint16_t gen_insn_end_off[TCG_MAX_INSNS]; - uint64_t gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS]; + uint64_t *gen_insn_data; /* Exit to translator on overflow. */ sigjmp_buf jmp_trans; @@ -662,13 +621,6 @@ static inline void *tcg_splitwx_to_rw(const void *rx) } #endif -static inline size_t temp_idx(TCGTemp *ts) -{ - ptrdiff_t n = ts - tcg_ctx->temps; - tcg_debug_assert(n >= 0 && n < tcg_ctx->nb_temps); - return n; -} - static inline TCGArg temp_arg(TCGTemp *ts) { return (uintptr_t)ts; @@ -679,16 +631,25 @@ static inline TCGTemp *arg_temp(TCGArg a) return (TCGTemp *)(uintptr_t)a; } -/* Using the offset of a temporary, relative to TCGContext, rather than - its index means that we don't use 0. That leaves offset 0 free for - a NULL representation without having to leave index 0 unused. */ +#ifdef CONFIG_DEBUG_TCG +size_t temp_idx(TCGTemp *ts); +TCGTemp *tcgv_i32_temp(TCGv_i32 v); +#else +static inline size_t temp_idx(TCGTemp *ts) +{ + return ts - tcg_ctx->temps; +} + +/* + * Using the offset of a temporary, relative to TCGContext, rather than + * its index means that we don't use 0. That leaves offset 0 free for + * a NULL representation without having to leave index 0 unused. + */ static inline TCGTemp *tcgv_i32_temp(TCGv_i32 v) { - uintptr_t o = (uintptr_t)v; - TCGTemp *t = (void *)tcg_ctx + o; - tcg_debug_assert(offsetof(TCGContext, temps[temp_idx(t)]) == o); - return t; + return (void *)tcg_ctx + (uintptr_t)v; } +#endif static inline TCGTemp *tcgv_i64_temp(TCGv_i64 v) { @@ -969,7 +930,19 @@ typedef struct TCGTargetOpDef { bool tcg_op_supported(TCGOpcode op); -void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args); +void tcg_gen_call0(TCGHelperInfo *, TCGTemp *ret); +void tcg_gen_call1(TCGHelperInfo *, TCGTemp *ret, TCGTemp *); +void tcg_gen_call2(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *); +void tcg_gen_call3(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, + TCGTemp *, TCGTemp *); +void tcg_gen_call4(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *, + TCGTemp *, TCGTemp *); +void tcg_gen_call5(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *, + TCGTemp *, TCGTemp *, TCGTemp *); +void tcg_gen_call6(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *, + TCGTemp *, TCGTemp *, TCGTemp *, TCGTemp *); +void tcg_gen_call7(TCGHelperInfo *, TCGTemp *ret, TCGTemp *, TCGTemp *, + TCGTemp *, TCGTemp *, TCGTemp *, TCGTemp *, TCGTemp *); TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs); void tcg_op_remove(TCGContext *s, TCGOp *op); @@ -1188,18 +1161,6 @@ uint64_t dup_const(unsigned vece, uint64_t c); : (qemu_build_not_reached_always(), 0)) \ : dup_const(VECE, C)) -#if TARGET_LONG_BITS == 64 -# define dup_const_tl dup_const -#else -# define dup_const_tl(VECE, C) \ - (__builtin_constant_p(VECE) \ - ? ( (VECE) == MO_8 ? 0x01010101ul * (uint8_t)(C) \ - : (VECE) == MO_16 ? 0x00010001ul * (uint16_t)(C) \ - : (VECE) == MO_32 ? 0x00000001ul * (uint32_t)(C) \ - : (qemu_build_not_reached_always(), 0)) \ - : (target_long)dup_const(VECE, C)) -#endif - #ifdef CONFIG_DEBUG_TCG void tcg_assert_listed_vecop(TCGOpcode); #else diff --git a/scripts/make-config-poison.sh b/scripts/make-config-poison.sh index 1892854..2b36907 100755 --- a/scripts/make-config-poison.sh +++ b/scripts/make-config-poison.sh @@ -4,11 +4,12 @@ if test $# = 0; then exit 0 fi -# Create list of config switches that should be poisoned in common code... -# but filter out CONFIG_TCG and CONFIG_USER_ONLY which are special. +# Create list of config switches that should be poisoned in common code, +# but filter out several which are handled manually. exec sed -n \ -e' /CONFIG_TCG/d' \ -e '/CONFIG_USER_ONLY/d' \ + -e '/CONFIG_SOFTMMU/d' \ -e '/^#define / {' \ -e 's///' \ -e 's/ .*//' \ diff --git a/target/alpha/translate.c b/target/alpha/translate.c index be8adb2..1f7dd07 100644 --- a/target/alpha/translate.c +++ b/target/alpha/translate.c @@ -30,6 +30,9 @@ #include "exec/translator.h" #include "exec/log.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H #undef ALPHA_DEBUG_DISAS #define CONFIG_SOFTFLOAT_INLINE @@ -93,8 +96,6 @@ static TCGv cpu_lock_value; static TCGv cpu_pal_ir[31]; #endif -#include "exec/gen-icount.h" - void alpha_translate_init(void) { #define DEF_VAR(V) { &cpu_##V, #V, offsetof(CPUAlphaState, V) } @@ -1233,8 +1234,7 @@ static DisasJumpType gen_mfpr(DisasContext *ctx, TCGv va, int regno) case 249: /* VMTIME */ helper = gen_helper_get_vmtime; do_helper: - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); + if (translator_io_start(&ctx->base)) { helper(va); return DISAS_PC_STALE; } else { @@ -1295,8 +1295,7 @@ static DisasJumpType gen_mtpr(DisasContext *ctx, TCGv vb, int regno) case 251: /* ALARM */ - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); + if (translator_io_start(&ctx->base)) { ret = DISAS_PC_STALE; } gen_helper_set_alarm(cpu_env, vb); @@ -2332,13 +2331,10 @@ static DisasJumpType translate_one(DisasContext *ctx, uint32_t insn) case 0xC000: /* RPCC */ va = dest_gpr(ctx, ra); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - gen_helper_load_pcc(va, cpu_env); + if (translator_io_start(&ctx->base)) { ret = DISAS_PC_STALE; - } else { - gen_helper_load_pcc(va, cpu_env); } + gen_helper_load_pcc(va, cpu_env); break; case 0xE000: /* RC */ diff --git a/target/arm/cpregs.h b/target/arm/cpregs.h index b04d344..1478568 100644 --- a/target/arm/cpregs.h +++ b/target/arm/cpregs.h @@ -67,8 +67,8 @@ enum { ARM_CP_ALIAS = 1 << 8, /* * Flag: Register does I/O and therefore its accesses need to be marked - * with gen_io_start() and also end the TB. In particular, registers which - * implement clocks or timers require this. + * with translator_io_start() and also end the TB. In particular, + * registers which implement clocks or timers require this. */ ARM_CP_IO = 1 << 9, /* diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 69c05cd..b2dc223 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -14,6 +14,7 @@ #include "cpu.h" #include "internals.h" #include "idau.h" +#include "tcg/oversized-guest.h" typedef struct S1Translate { @@ -418,6 +419,7 @@ static uint64_t arm_casq_ptw(CPUARMState *env, uint64_t old_val, uint64_t new_val, S1Translate *ptw, ARMMMUFaultInfo *fi) { +#ifdef TARGET_AARCH64 uint64_t cur_val; void *host = ptw->out_host; @@ -473,7 +475,7 @@ static uint64_t arm_casq_ptw(CPUARMState *env, uint64_t old_val, * we know that TCG_OVERSIZED_GUEST is set, which means that we are * running in round-robin mode and could only race with dma i/o. */ -#ifndef TCG_OVERSIZED_GUEST +#if !TCG_OVERSIZED_GUEST # error "Unexpected configuration" #endif bool locked = qemu_mutex_iothread_locked(); @@ -497,6 +499,10 @@ static uint64_t arm_casq_ptw(CPUARMState *env, uint64_t old_val, #endif return cur_val; +#else + /* AArch32 does not have FEAT_HADFS. */ + g_assert_not_reached(); +#endif } static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c index 741a608..d980033 100644 --- a/target/arm/tcg/translate-a64.c +++ b/target/arm/tcg/translate-a64.c @@ -18,23 +18,13 @@ */ #include "qemu/osdep.h" -#include "cpu.h" -#include "exec/exec-all.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" +#include "translate.h" +#include "translate-a64.h" #include "qemu/log.h" +#include "disas/disas.h" #include "arm_ldst.h" -#include "translate.h" -#include "internals.h" -#include "qemu/host-utils.h" #include "semihosting/semihost.h" -#include "exec/gen-icount.h" -#include "exec/helper-proto.h" -#include "exec/helper-gen.h" -#include "exec/log.h" #include "cpregs.h" -#include "translate-a64.h" -#include "qemu/atomic128.h" static TCGv_i64 cpu_X[32]; static TCGv_i64 cpu_pc; @@ -1554,9 +1544,7 @@ static bool trans_ERET(DisasContext *s, arg_ERET *a) tcg_gen_ld_i64(dst, cpu_env, offsetof(CPUARMState, elr_el[s->current_el])); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&s->base); gen_helper_exception_return(cpu_env, dst); /* Must exit loop to check un-masked IRQs */ @@ -1584,9 +1572,8 @@ static bool trans_ERETA(DisasContext *s, arg_reta *a) offsetof(CPUARMState, elr_el[s->current_el])); dst = auth_branch_target(s, dst, cpu_X[31], !a->m); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + + translator_io_start(&s->base); gen_helper_exception_return(cpu_env, dst); /* Must exit loop to check un-masked IRQs */ @@ -2046,6 +2033,7 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread, uint32_t key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP, crn, crm, op0, op1, op2); const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key); + bool need_exit_tb = false; TCGv_ptr tcg_ri = NULL; TCGv_i64 tcg_rt; @@ -2173,8 +2161,9 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread, return; } - if ((tb_cflags(s->base.tb) & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) { - gen_io_start(); + if (ri->type & ARM_CP_IO) { + /* I/O operations must end the TB here (whether read or write) */ + need_exit_tb = translator_io_start(&s->base); } tcg_rt = cpu_reg(s, rt); @@ -2204,10 +2193,6 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread, } } - if ((tb_cflags(s->base.tb) & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) { - /* I/O operations must end the TB here (whether read or write) */ - s->base.is_jmp = DISAS_UPDATE_EXIT; - } if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) { /* * A write to any coprocessor regiser that ends a TB @@ -2219,6 +2204,9 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread, * but allow this to be suppressed by the register definition * (usually only necessary to work around guest bugs). */ + need_exit_tb = true; + } + if (need_exit_tb) { s->base.is_jmp = DISAS_UPDATE_EXIT; } } diff --git a/target/arm/tcg/translate-m-nocp.c b/target/arm/tcg/translate-m-nocp.c index 9a89aab..33f6478 100644 --- a/target/arm/tcg/translate-m-nocp.c +++ b/target/arm/tcg/translate-m-nocp.c @@ -18,8 +18,6 @@ */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" #include "translate.h" #include "translate-a32.h" diff --git a/target/arm/tcg/translate-mve.c b/target/arm/tcg/translate-mve.c index 31fb211..bbc7b3f 100644 --- a/target/arm/tcg/translate-mve.c +++ b/target/arm/tcg/translate-mve.c @@ -18,10 +18,6 @@ */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" -#include "exec/exec-all.h" -#include "exec/gen-icount.h" #include "translate.h" #include "translate-a32.h" diff --git a/target/arm/tcg/translate-neon.c b/target/arm/tcg/translate-neon.c index af8685a..03913de 100644 --- a/target/arm/tcg/translate-neon.c +++ b/target/arm/tcg/translate-neon.c @@ -21,10 +21,6 @@ */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" -#include "exec/exec-all.h" -#include "exec/gen-icount.h" #include "translate.h" #include "translate-a32.h" diff --git a/target/arm/tcg/translate-sme.c b/target/arm/tcg/translate-sme.c index e3adba3..d0054e3 100644 --- a/target/arm/tcg/translate-sme.c +++ b/target/arm/tcg/translate-sme.c @@ -18,15 +18,8 @@ */ #include "qemu/osdep.h" -#include "cpu.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" -#include "tcg/tcg-gvec-desc.h" #include "translate.h" -#include "exec/helper-gen.h" #include "translate-a64.h" -#include "fpu/softfloat.h" - /* * Include the generated decoder. diff --git a/target/arm/tcg/translate-sve.c b/target/arm/tcg/translate-sve.c index 92ab290..d9d5810 100644 --- a/target/arm/tcg/translate-sve.c +++ b/target/arm/tcg/translate-sve.c @@ -18,18 +18,7 @@ */ #include "qemu/osdep.h" -#include "cpu.h" -#include "exec/exec-all.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" -#include "tcg/tcg-gvec-desc.h" -#include "qemu/log.h" -#include "arm_ldst.h" #include "translate.h" -#include "internals.h" -#include "exec/helper-proto.h" -#include "exec/helper-gen.h" -#include "exec/log.h" #include "translate-a64.h" #include "fpu/softfloat.h" diff --git a/target/arm/tcg/translate-vfp.c b/target/arm/tcg/translate-vfp.c index dd782aa..359b1e3 100644 --- a/target/arm/tcg/translate-vfp.c +++ b/target/arm/tcg/translate-vfp.c @@ -21,10 +21,6 @@ */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" -#include "exec/exec-all.h" -#include "exec/gen-icount.h" #include "translate.h" #include "translate-a32.h" @@ -117,9 +113,8 @@ static void gen_preserve_fp_state(DisasContext *s, bool skip_context_update) * so we must mark it as an IO operation for icount (and cause * this to be the last insn in the TB). */ - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { + if (translator_io_start(&s->base)) { s->base.is_jmp = DISAS_UPDATE_EXIT; - gen_io_start(); } gen_helper_v7m_preserve_fp_state(cpu_env); /* diff --git a/target/arm/tcg/translate.c b/target/arm/tcg/translate.c index 7468476..a68d3c7 100644 --- a/target/arm/tcg/translate.c +++ b/target/arm/tcg/translate.c @@ -20,21 +20,18 @@ */ #include "qemu/osdep.h" -#include "cpu.h" -#include "internals.h" -#include "disas/disas.h" -#include "exec/exec-all.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" +#include "translate.h" +#include "translate-a32.h" #include "qemu/log.h" -#include "qemu/bitops.h" +#include "disas/disas.h" #include "arm_ldst.h" #include "semihosting/semihost.h" -#include "exec/helper-proto.h" -#include "exec/helper-gen.h" -#include "exec/log.h" #include "cpregs.h" +#include "exec/helper-proto.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H #define ENABLE_ARCH_4T arm_dc_feature(s, ARM_FEATURE_V4T) #define ENABLE_ARCH_5 arm_dc_feature(s, ARM_FEATURE_V5) @@ -47,9 +44,6 @@ #define ENABLE_ARCH_7 arm_dc_feature(s, ARM_FEATURE_V7) #define ENABLE_ARCH_8 arm_dc_feature(s, ARM_FEATURE_V8) -#include "translate.h" -#include "translate-a32.h" - /* These are TCG temporaries used only by the legacy iwMMXt decoder */ static TCGv_i64 cpu_V0, cpu_V1, cpu_M0; /* These are TCG globals which alias CPUARMState fields */ @@ -58,8 +52,6 @@ TCGv_i32 cpu_CF, cpu_NF, cpu_VF, cpu_ZF; TCGv_i64 cpu_exclusive_addr; TCGv_i64 cpu_exclusive_val; -#include "exec/gen-icount.h" - static const char * const regnames[] = { "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "pc" }; @@ -2908,9 +2900,7 @@ static void gen_rfe(DisasContext *s, TCGv_i32 pc, TCGv_i32 cpsr) * appropriately depending on the new Thumb bit, so it must * be called after storing the new PC. */ - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&s->base); gen_helper_cpsr_write_eret(cpu_env, cpsr); /* Must exit loop to check un-masked IRQs */ s->base.is_jmp = DISAS_EXIT; @@ -4559,7 +4549,7 @@ static void do_coproc_insn(DisasContext *s, int cpnum, int is64, uint32_t key = ENCODE_CP_REG(cpnum, is64, s->ns, crn, crm, opc1, opc2); const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key); TCGv_ptr tcg_ri = NULL; - bool need_exit_tb; + bool need_exit_tb = false; uint32_t syndrome; /* @@ -4704,8 +4694,9 @@ static void do_coproc_insn(DisasContext *s, int cpnum, int is64, g_assert_not_reached(); } - if ((tb_cflags(s->base.tb) & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) { - gen_io_start(); + if (ri->type & ARM_CP_IO) { + /* I/O operations must end the TB here (whether read or write) */ + need_exit_tb = translator_io_start(&s->base); } if (isread) { @@ -4787,10 +4778,6 @@ static void do_coproc_insn(DisasContext *s, int cpnum, int is64, } } - /* I/O operations must end the TB here (whether read or write) */ - need_exit_tb = ((tb_cflags(s->base.tb) & CF_USE_ICOUNT) && - (ri->type & ARM_CP_IO)); - if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) { /* * A write to any coprocessor register that ends a TB @@ -8047,9 +8034,7 @@ static bool do_ldm(DisasContext *s, arg_ldst_block *a, int min_n) if (exc_return) { /* Restore CPSR from SPSR. */ tmp = load_cpu_field(spsr); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&s->base); gen_helper_cpsr_write_eret(cpu_env, tmp); /* Must exit loop to check un-masked IRQs */ s->base.is_jmp = DISAS_EXIT; diff --git a/target/arm/tcg/translate.h b/target/arm/tcg/translate.h index a9d1f4a..4d88197 100644 --- a/target/arm/tcg/translate.h +++ b/target/arm/tcg/translate.h @@ -1,7 +1,12 @@ #ifndef TARGET_ARM_TRANSLATE_H #define TARGET_ARM_TRANSLATE_H +#include "cpu.h" +#include "tcg/tcg-op.h" +#include "tcg/tcg-op-gvec.h" +#include "exec/exec-all.h" #include "exec/translator.h" +#include "exec/helper-gen.h" #include "internals.h" diff --git a/target/avr/cpu.c b/target/avr/cpu.c index a24c23c..8f741f2 100644 --- a/target/avr/cpu.c +++ b/target/avr/cpu.c @@ -24,6 +24,7 @@ #include "exec/exec-all.h" #include "cpu.h" #include "disas/dis-asm.h" +#include "tcg/debug-assert.h" static void avr_cpu_set_pc(CPUState *cs, vaddr value) { diff --git a/target/avr/helper.c b/target/avr/helper.c index 156dde4..2bad242 100644 --- a/target/avr/helper.c +++ b/target/avr/helper.c @@ -20,6 +20,7 @@ #include "qemu/osdep.h" #include "qemu/log.h" +#include "qemu/error-report.h" #include "cpu.h" #include "hw/core/tcg-cpu-ops.h" #include "exec/exec-all.h" diff --git a/target/avr/translate.c b/target/avr/translate.c index cd82f5d..ef2edd7 100644 --- a/target/avr/translate.c +++ b/target/avr/translate.c @@ -29,7 +29,11 @@ #include "exec/helper-gen.h" #include "exec/log.h" #include "exec/translator.h" -#include "exec/gen-icount.h" + +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + /* * Define if you want a BREAK instruction translated to a breakpoint diff --git a/target/cris/translate.c b/target/cris/translate.c index b2beb99..1445cd8 100644 --- a/target/cris/translate.c +++ b/target/cris/translate.c @@ -34,11 +34,13 @@ #include "exec/translator.h" #include "crisv32-decode.h" #include "qemu/qemu-print.h" - #include "exec/helper-gen.h" - #include "exec/log.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + #define DISAS_CRIS 0 #if DISAS_CRIS @@ -86,8 +88,6 @@ static TCGv env_btaken; static TCGv env_btarget; static TCGv env_pc; -#include "exec/gen-icount.h" - /* This is the state at translation time. */ typedef struct DisasContext { DisasContextBase base; diff --git a/target/hexagon/genptr.c b/target/hexagon/genptr.c index bcb287d..217bc7b 100644 --- a/target/hexagon/genptr.c +++ b/target/hexagon/genptr.c @@ -20,6 +20,7 @@ #include "internal.h" #include "tcg/tcg-op.h" #include "tcg/tcg-op-gvec.h" +#include "exec/helper-gen.h" #include "insn.h" #include "opcodes.h" #include "translate.h" diff --git a/target/hexagon/idef-parser/idef-parser.y b/target/hexagon/idef-parser/idef-parser.y index 5c98395..cd2612e 100644 --- a/target/hexagon/idef-parser/idef-parser.y +++ b/target/hexagon/idef-parser/idef-parser.y @@ -843,13 +843,14 @@ int main(int argc, char **argv) fputs("#include \"qemu/log.h\"\n", output_file); fputs("#include \"cpu.h\"\n", output_file); fputs("#include \"internal.h\"\n", output_file); + fputs("#include \"tcg/tcg.h\"\n", output_file); fputs("#include \"tcg/tcg-op.h\"\n", output_file); + fputs("#include \"exec/helper-gen.h\"\n", output_file); fputs("#include \"insn.h\"\n", output_file); fputs("#include \"opcodes.h\"\n", output_file); fputs("#include \"translate.h\"\n", output_file); fputs("#define QEMU_GENERATE\n", output_file); fputs("#include \"genptr.h\"\n", output_file); - fputs("#include \"tcg/tcg.h\"\n", output_file); fputs("#include \"macros.h\"\n", output_file); fprintf(output_file, "#include \"%s\"\n", argv[ARG_INDEX_EMITTER_H]); diff --git a/target/hexagon/translate.c b/target/hexagon/translate.c index 8838ab2..7083391 100644 --- a/target/hexagon/translate.c +++ b/target/hexagon/translate.c @@ -20,6 +20,9 @@ #include "cpu.h" #include "tcg/tcg-op.h" #include "tcg/tcg-op-gvec.h" +#include "exec/helper-gen.h" +#include "exec/helper-proto.h" +#include "exec/translation-block.h" #include "exec/cpu_ldst.h" #include "exec/log.h" #include "internal.h" @@ -30,6 +33,10 @@ #include "genptr.h" #include "printinsn.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + #include "analyze_funcs_generated.c.inc" typedef void (*AnalyzeInsn)(DisasContext *ctx); diff --git a/target/hppa/translate.c b/target/hppa/translate.c index 59e4688..d33813d 100644 --- a/target/hppa/translate.c +++ b/target/hppa/translate.c @@ -29,6 +29,11 @@ #include "exec/translator.h" #include "exec/log.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + + /* Since we have a distinction between register size and address size, we need to redefine all of these. */ @@ -359,8 +364,6 @@ static TCGv_reg cpu_psw_v; static TCGv_reg cpu_psw_cb; static TCGv_reg cpu_psw_cb_msb; -#include "exec/gen-icount.h" - void hppa_translate_init(void) { #define DEF_VAR(V) { &cpu_##V, #V, offsetof(CPUHPPAState, V) } @@ -2085,8 +2088,7 @@ static bool trans_mfctl(DisasContext *ctx, arg_mfctl *a) /* FIXME: Respect PSW_S bit. */ nullify_over(ctx); tmp = dest_gpr(ctx, rt); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); + if (translator_io_start(&ctx->base)) { gen_helper_read_interval_timer(tmp); ctx->base.is_jmp = DISAS_IAQ_N_STALE; } else { diff --git a/target/i386/helper.c b/target/i386/helper.c index 8857444..36bf210 100644 --- a/target/i386/helper.c +++ b/target/i386/helper.c @@ -28,6 +28,9 @@ #include "monitor/monitor.h" #endif #include "qemu/log.h" +#ifdef CONFIG_TCG +#include "tcg/insn-start-words.h" +#endif void cpu_sync_avx_hflag(CPUX86State *env) { diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 91c9c0c..5cf1431 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -34,6 +34,11 @@ #include "exec/log.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + + #define PREFIX_REPZ 0x01 #define PREFIX_REPNZ 0x02 #define PREFIX_LOCK 0x04 @@ -73,8 +78,6 @@ static TCGv cpu_seg_base[6]; static TCGv_i64 cpu_bndl[4]; static TCGv_i64 cpu_bndu[4]; -#include "exec/gen-icount.h" - typedef struct DisasContext { DisasContextBase base; @@ -3928,10 +3931,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) !(s->cpuid_ext_features & CPUID_EXT_RDRAND)) { goto illegal_op; } - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - s->base.is_jmp = DISAS_TOO_MANY; - } + translator_io_start(&s->base); gen_helper_rdrand(s->T0, cpu_env); rm = (modrm & 7) | REX_B(s); gen_op_mov_reg_v(s, dflag, rm, s->T0); @@ -4969,10 +4969,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) SVM_IOIO_TYPE_MASK | SVM_IOIO_STR_MASK)) { break; } - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - s->base.is_jmp = DISAS_TOO_MANY; - } + translator_io_start(&s->base); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { gen_repz_ins(s, ot); } else { @@ -4987,10 +4984,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (!gen_check_io(s, ot, s->tmp2_i32, SVM_IOIO_STR_MASK)) { break; } - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - s->base.is_jmp = DISAS_TOO_MANY; - } + translator_io_start(&s->base); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { gen_repz_outs(s, ot); } else { @@ -5009,10 +5003,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (!gen_check_io(s, ot, s->tmp2_i32, SVM_IOIO_TYPE_MASK)) { break; } - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - s->base.is_jmp = DISAS_TOO_MANY; - } + translator_io_start(&s->base); gen_helper_in_func(ot, s->T1, s->tmp2_i32); gen_op_mov_reg_v(s, ot, R_EAX, s->T1); gen_bpt_io(s, s->tmp2_i32, ot); @@ -5025,10 +5016,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (!gen_check_io(s, ot, s->tmp2_i32, 0)) { break; } - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - s->base.is_jmp = DISAS_TOO_MANY; - } + translator_io_start(&s->base); gen_op_mov_v_reg(s, ot, s->T1, R_EAX); tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); gen_helper_out_func(ot, s->tmp2_i32, s->tmp3_i32); @@ -5042,10 +5030,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (!gen_check_io(s, ot, s->tmp2_i32, SVM_IOIO_TYPE_MASK)) { break; } - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - s->base.is_jmp = DISAS_TOO_MANY; - } + translator_io_start(&s->base); gen_helper_in_func(ot, s->T1, s->tmp2_i32); gen_op_mov_reg_v(s, ot, R_EAX, s->T1); gen_bpt_io(s, s->tmp2_i32, ot); @@ -5058,10 +5043,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (!gen_check_io(s, ot, s->tmp2_i32, 0)) { break; } - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - s->base.is_jmp = DISAS_TOO_MANY; - } + translator_io_start(&s->base); gen_op_mov_v_reg(s, ot, s->T1, R_EAX); tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); gen_helper_out_func(ot, s->tmp2_i32, s->tmp3_i32); @@ -5669,10 +5651,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0x131: /* rdtsc */ gen_update_cc_op(s); gen_update_eip_cur(s); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - s->base.is_jmp = DISAS_TOO_MANY; - } + translator_io_start(&s->base); gen_helper_rdtsc(cpu_env); break; case 0x133: /* rdpmc */ @@ -6128,10 +6107,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } gen_update_cc_op(s); gen_update_eip_cur(s); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - s->base.is_jmp = DISAS_TOO_MANY; - } + translator_io_start(&s->base); gen_helper_rdtscp(cpu_env); break; @@ -6485,10 +6461,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } ot = (CODE64(s) ? MO_64 : MO_32); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - s->base.is_jmp = DISAS_TOO_MANY; - } + translator_io_start(&s->base); if (b & 2) { gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0 + reg); gen_op_mov_v_reg(s, ot, s->T0, rm); diff --git a/target/loongarch/insn_trans/trans_extra.c.inc b/target/loongarch/insn_trans/trans_extra.c.inc index ad713cd..06f4de4 100644 --- a/target/loongarch/insn_trans/trans_extra.c.inc +++ b/target/loongarch/insn_trans/trans_extra.c.inc @@ -39,9 +39,7 @@ static bool gen_rdtime(DisasContext *ctx, arg_rr *a, TCGv dst1 = gpr_dst(ctx, a->rd, EXT_NONE); TCGv dst2 = gpr_dst(ctx, a->rj, EXT_NONE); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_rdtime_d(dst1, cpu_env); if (word) { tcg_gen_sextract_tl(dst1, dst1, high ? 32 : 0, 32); diff --git a/target/loongarch/insn_trans/trans_privileged.c.inc b/target/loongarch/insn_trans/trans_privileged.c.inc index 5a04352..02bca7c 100644 --- a/target/loongarch/insn_trans/trans_privileged.c.inc +++ b/target/loongarch/insn_trans/trans_privileged.c.inc @@ -185,9 +185,7 @@ static bool check_csr_flags(DisasContext *ctx, const CSRInfo *csr, bool write) if ((csr->flags & CSRFL_READONLY) && write) { return false; } - if ((csr->flags & CSRFL_IO) && - (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT)) { - gen_io_start(); + if ((csr->flags & CSRFL_IO) && translator_io_start(&ctx->base)) { ctx->base.is_jmp = DISAS_EXIT_UPDATE; } else if ((csr->flags & CSRFL_EXITTB) && write) { ctx->base.is_jmp = DISAS_EXIT_UPDATE; diff --git a/target/loongarch/translate.c b/target/loongarch/translate.c index ae53f5e..3146a2d 100644 --- a/target/loongarch/translate.c +++ b/target/loongarch/translate.c @@ -9,11 +9,10 @@ #include "cpu.h" #include "tcg/tcg-op.h" #include "tcg/tcg-op-gvec.h" - +#include "exec/translation-block.h" #include "exec/translator.h" #include "exec/helper-proto.h" #include "exec/helper-gen.h" - #include "exec/log.h" #include "qemu/qemu-print.h" #include "fpu/softfloat.h" @@ -24,7 +23,9 @@ TCGv cpu_gpr[32], cpu_pc; static TCGv cpu_lladdr, cpu_llval; -#include "exec/gen-icount.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H #define DISAS_STOP DISAS_TARGET_0 #define DISAS_EXIT DISAS_TARGET_1 diff --git a/target/m68k/translate.c b/target/m68k/translate.c index 44d852b..551ef9e 100644 --- a/target/m68k/translate.c +++ b/target/m68k/translate.c @@ -34,6 +34,9 @@ #include "exec/log.h" #include "fpu/softfloat.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H //#define DEBUG_DISPATCH 1 @@ -62,8 +65,6 @@ static TCGv NULL_QREG; /* Used to distinguish stores from bad addressing modes. */ static TCGv store_dummy; -#include "exec/gen-icount.h" - void m68k_tcg_init(void) { char *p; diff --git a/target/microblaze/translate.c b/target/microblaze/translate.c index ee0d7b8..7e7f837 100644 --- a/target/microblaze/translate.c +++ b/target/microblaze/translate.c @@ -31,6 +31,10 @@ #include "exec/log.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + #define EXTRACT_FIELD(src, start, end) \ (((src) >> start) & ((1 << (end - start + 1)) - 1)) @@ -54,8 +58,6 @@ static TCGv_i32 cpu_iflags; static TCGv cpu_res_addr; static TCGv_i32 cpu_res_val; -#include "exec/gen-icount.h" - /* This is the state at translation time. */ typedef struct DisasContext { DisasContextBase base; diff --git a/target/mips/tcg/msa_translate.c b/target/mips/tcg/msa_translate.c index 220cd3b..b5b66fb 100644 --- a/target/mips/tcg/msa_translate.c +++ b/target/mips/tcg/msa_translate.c @@ -11,11 +11,8 @@ * SPDX-License-Identifier: LGPL-2.1-or-later */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" -#include "exec/helper-gen.h" #include "translate.h" #include "fpu_helper.h" -#include "internal.h" static int elm_n(DisasContext *ctx, int x); static int elm_df(DisasContext *ctx, int x); diff --git a/target/mips/tcg/mxu_translate.c b/target/mips/tcg/mxu_translate.c index be038b5..39348b3 100644 --- a/target/mips/tcg/mxu_translate.c +++ b/target/mips/tcg/mxu_translate.c @@ -16,8 +16,6 @@ */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" -#include "exec/helper-gen.h" #include "translate.h" /* diff --git a/target/mips/tcg/octeon_translate.c b/target/mips/tcg/octeon_translate.c index 103c304..e25c4cb 100644 --- a/target/mips/tcg/octeon_translate.c +++ b/target/mips/tcg/octeon_translate.c @@ -7,10 +7,8 @@ */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" -#include "exec/helper-gen.h" #include "translate.h" +#include "tcg/tcg-op-gvec.h" /* Include the auto-generated decoder. */ #include "decode-octeon.c.inc" diff --git a/target/mips/tcg/rel6_translate.c b/target/mips/tcg/rel6_translate.c index d631851..59f237b 100644 --- a/target/mips/tcg/rel6_translate.c +++ b/target/mips/tcg/rel6_translate.c @@ -9,8 +9,6 @@ */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" -#include "exec/helper-gen.h" #include "translate.h" /* Include the auto-generated decoders. */ diff --git a/target/mips/tcg/translate.c b/target/mips/tcg/translate.c index a6ca2e5..74af91e 100644 --- a/target/mips/tcg/translate.c +++ b/target/mips/tcg/translate.c @@ -23,19 +23,19 @@ */ #include "qemu/osdep.h" -#include "cpu.h" +#include "translate.h" #include "internal.h" -#include "tcg/tcg-op.h" -#include "exec/translator.h" #include "exec/helper-proto.h" -#include "exec/helper-gen.h" +#include "exec/translation-block.h" #include "semihosting/semihost.h" - #include "trace.h" -#include "exec/log.h" -#include "qemu/qemu-print.h" +#include "disas/disas.h" #include "fpu_helper.h" -#include "translate.h" + +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + /* * Many sysemu-only helpers are not reachable for user-only. @@ -1210,8 +1210,6 @@ static TCGv_i32 hflags; TCGv_i32 fpu_fcr0, fpu_fcr31; TCGv_i64 fpu_f64[32]; -#include "exec/gen-icount.h" - static const char regnames_HI[][4] = { "HI0", "HI1", "HI2", "HI3", }; @@ -5665,9 +5663,8 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel) switch (sel) { case CP0_REG09__COUNT: /* Mark as an IO operation because we read the time. */ - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); + gen_helper_mfc0_count(arg, cpu_env); /* * Break the TB to be able to take timer interrupts immediately @@ -6106,14 +6103,13 @@ cp0_unimplemented: static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel) { const char *register_name = "invalid"; + bool icount; if (sel != 0) { check_insn(ctx, ISA_MIPS_R1); } - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + icount = translator_io_start(&ctx->base); switch (reg) { case CP0_REGISTER_00: @@ -6851,7 +6847,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel) trace_mips_translate_c0("mtc0", register_name, reg, sel); /* For simplicity assume that all writes can cause interrupts. */ - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { + if (icount) { /* * DISAS_STOP isn't sufficient, we need to ensure we break out of * translated code to check for pending interrupts. @@ -7168,9 +7164,7 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel) switch (sel) { case CP0_REG09__COUNT: /* Mark as an IO operation because we read the time. */ - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_mfc0_count(arg, cpu_env); /* * Break the TB to be able to take timer interrupts immediately @@ -7596,14 +7590,13 @@ cp0_unimplemented: static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel) { const char *register_name = "invalid"; + bool icount; if (sel != 0) { check_insn(ctx, ISA_MIPS_R1); } - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + icount = translator_io_start(&ctx->base); switch (reg) { case CP0_REGISTER_00: @@ -8331,7 +8324,7 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel) trace_mips_translate_c0("dmtc0", register_name, reg, sel); /* For simplicity assume that all writes can cause interrupts. */ - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { + if (icount) { /* * DISAS_STOP isn't sufficient, we need to ensure we break out of * translated code to check for pending interrupts. @@ -11142,9 +11135,7 @@ void gen_rdhwr(DisasContext *ctx, int rt, int rd, int sel) gen_store_gpr(t0, rt); break; case 2: - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_rdhwr_cc(t0, cpu_env); gen_store_gpr(t0, rt); /* diff --git a/target/mips/tcg/translate.h b/target/mips/tcg/translate.h index 69f8584..3b0498a 100644 --- a/target/mips/tcg/translate.h +++ b/target/mips/tcg/translate.h @@ -8,8 +8,11 @@ #ifndef TARGET_MIPS_TRANSLATE_H #define TARGET_MIPS_TRANSLATE_H -#include "qemu/log.h" +#include "cpu.h" +#include "tcg/tcg-op.h" #include "exec/translator.h" +#include "exec/helper-gen.h" +#include "qemu/log.h" #define MIPS_DEBUG_DISAS 0 diff --git a/target/mips/tcg/translate_addr_const.c b/target/mips/tcg/translate_addr_const.c index a510da4..6f4b39f 100644 --- a/target/mips/tcg/translate_addr_const.c +++ b/target/mips/tcg/translate_addr_const.c @@ -11,7 +11,6 @@ * SPDX-License-Identifier: LGPL-2.1-or-later */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" #include "translate.h" bool gen_lsa(DisasContext *ctx, int rd, int rt, int rs, int sa) diff --git a/target/mips/tcg/tx79_translate.c b/target/mips/tcg/tx79_translate.c index 3a45a1b..dd6fb8a 100644 --- a/target/mips/tcg/tx79_translate.c +++ b/target/mips/tcg/tx79_translate.c @@ -8,10 +8,8 @@ */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" -#include "exec/helper-gen.h" #include "translate.h" +#include "tcg/tcg-op-gvec.h" /* Include the auto-generated decoder. */ #include "decode-tx79.c.inc" diff --git a/target/mips/tcg/vr54xx_translate.c b/target/mips/tcg/vr54xx_translate.c index 804672f..2c1f6cc 100644 --- a/target/mips/tcg/vr54xx_translate.c +++ b/target/mips/tcg/vr54xx_translate.c @@ -10,10 +10,7 @@ */ #include "qemu/osdep.h" -#include "tcg/tcg-op.h" -#include "exec/helper-gen.h" #include "translate.h" -#include "internal.h" /* Include the auto-generated decoder. */ #include "decode-vr54xx.c.inc" diff --git a/target/nios2/translate.c b/target/nios2/translate.c index a548e16..a365ad8 100644 --- a/target/nios2/translate.c +++ b/target/nios2/translate.c @@ -32,9 +32,13 @@ #include "exec/cpu_ldst.h" #include "exec/translator.h" #include "qemu/qemu-print.h" -#include "exec/gen-icount.h" #include "semihosting/semihost.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + + /* is_jmp field values */ #define DISAS_UPDATE DISAS_TARGET_1 /* cpu state was modified dynamically */ diff --git a/target/openrisc/sys_helper.c b/target/openrisc/sys_helper.c index ccdee3b..782a575 100644 --- a/target/openrisc/sys_helper.c +++ b/target/openrisc/sys_helper.c @@ -26,6 +26,7 @@ #ifndef CONFIG_USER_ONLY #include "hw/boards.h" #endif +#include "tcg/insn-start-words.h" #define TO_SPR(group, number) (((group) << 11) + (number)) diff --git a/target/openrisc/translate.c b/target/openrisc/translate.c index 43ba0cc..7760329 100644 --- a/target/openrisc/translate.c +++ b/target/openrisc/translate.c @@ -31,10 +31,14 @@ #include "exec/helper-proto.h" #include "exec/helper-gen.h" -#include "exec/gen-icount.h" #include "exec/log.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + + /* is_jmp field values */ #define DISAS_EXIT DISAS_TARGET_0 /* force exit to main loop */ #define DISAS_JUMP DISAS_TARGET_1 /* exit via jmp_pc/jmp_pc_imm */ @@ -823,8 +827,7 @@ static bool trans_l_mfspr(DisasContext *dc, arg_l_mfspr *a) check_r0_write(dc, a->d); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); + if (translator_io_start(&dc->base)) { if (dc->delayed_branch) { tcg_gen_mov_tl(cpu_pc, jmp_pc); tcg_gen_discard_tl(jmp_pc); @@ -843,9 +846,8 @@ static bool trans_l_mtspr(DisasContext *dc, arg_l_mtspr *a) { TCGv spr = tcg_temp_new(); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); + /* * For SR, we will need to exit the TB to recognize the new * exception state. For NPC, in theory this counts as a branch diff --git a/target/ppc/cpu.h b/target/ppc/cpu.h index 0f9f2e1..10c4ffa 100644 --- a/target/ppc/cpu.h +++ b/target/ppc/cpu.h @@ -1394,7 +1394,6 @@ void ppc_store_msr(CPUPPCState *env, target_ulong value); void ppc_cpu_list(void); /* Time-base and decrementer management */ -#ifndef NO_CPU_IO_DEFS uint64_t cpu_ppc_load_tbl(CPUPPCState *env); uint32_t cpu_ppc_load_tbu(CPUPPCState *env); void cpu_ppc_store_tbu(CPUPPCState *env, uint32_t value); @@ -1435,7 +1434,6 @@ int ppcemb_tlb_check(CPUPPCState *env, ppcemb_tlb_t *tlb, hwaddr booke206_tlb_to_page_size(CPUPPCState *env, ppcmas_tlb_t *tlb); #endif -#endif void ppc_store_fpscr(CPUPPCState *env, target_ulong val); void helper_hfscr_facility_check(CPUPPCState *env, uint32_t bit, diff --git a/target/ppc/power8-pmu-regs.c.inc b/target/ppc/power8-pmu-regs.c.inc index d900e13..c82feed 100644 --- a/target/ppc/power8-pmu-regs.c.inc +++ b/target/ppc/power8-pmu-regs.c.inc @@ -103,9 +103,9 @@ static void write_MMCR0_common(DisasContext *ctx, TCGv val) /* * helper_store_mmcr0 will make clock based operations that * will cause 'bad icount read' errors if we do not execute - * gen_icount_io_start() beforehand. + * translator_io_start() beforehand. */ - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_mmcr0(cpu_env, val); /* @@ -179,7 +179,7 @@ void spr_read_PMC(DisasContext *ctx, int gprn, int sprn) { TCGv_i32 t_sprn = tcg_constant_i32(sprn); - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_read_pmc(cpu_gpr[gprn], cpu_env, t_sprn); } @@ -212,7 +212,7 @@ void spr_write_PMC(DisasContext *ctx, int sprn, int gprn) { TCGv_i32 t_sprn = tcg_constant_i32(sprn); - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_pmc(cpu_env, t_sprn, cpu_gpr[gprn]); } @@ -248,7 +248,7 @@ void spr_write_MMCR0(DisasContext *ctx, int sprn, int gprn) void spr_write_MMCR1(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_mmcr1(cpu_env, cpu_gpr[gprn]); } #else diff --git a/target/ppc/translate.c b/target/ppc/translate.c index 9b78845..37fd431 100644 --- a/target/ppc/translate.c +++ b/target/ppc/translate.c @@ -41,6 +41,10 @@ #include "qemu/qemu-print.h" #include "qapi/error.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + #define CPU_SINGLE_STEP 0x1 #define CPU_BRANCH_STEP 0x2 @@ -76,8 +80,6 @@ static TCGv cpu_reserve_val2; static TCGv cpu_fpscr; static TCGv_i32 cpu_access_type; -#include "exec/gen-icount.h" - void ppc_translate_init(void) { int i; @@ -294,24 +296,10 @@ static void gen_exception_nip(DisasContext *ctx, uint32_t excp, ctx->base.is_jmp = DISAS_NORETURN; } -static void gen_icount_io_start(DisasContext *ctx) -{ - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - /* - * An I/O instruction must be last in the TB. - * Chain to the next TB, and let the code from gen_tb_start - * decide if we need to return to the main loop. - * Doing this first also allows this value to be overridden. - */ - ctx->base.is_jmp = DISAS_TOO_MANY; - } -} - #if !defined(CONFIG_USER_ONLY) static void gen_ppc_maybe_interrupt(DisasContext *ctx) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_ppc_maybe_interrupt(cpu_env); } #endif @@ -548,13 +536,13 @@ void spr_write_ureg(DisasContext *ctx, int sprn, int gprn) #if !defined(CONFIG_USER_ONLY) void spr_read_decr(DisasContext *ctx, int gprn, int sprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_load_decr(cpu_gpr[gprn], cpu_env); } void spr_write_decr(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_decr(cpu_env, cpu_gpr[gprn]); } #endif @@ -563,13 +551,13 @@ void spr_write_decr(DisasContext *ctx, int sprn, int gprn) /* Time base */ void spr_read_tbl(DisasContext *ctx, int gprn, int sprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_load_tbl(cpu_gpr[gprn], cpu_env); } void spr_read_tbu(DisasContext *ctx, int gprn, int sprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_load_tbu(cpu_gpr[gprn], cpu_env); } @@ -586,13 +574,13 @@ void spr_read_atbu(DisasContext *ctx, int gprn, int sprn) #if !defined(CONFIG_USER_ONLY) void spr_write_tbl(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_tbl(cpu_env, cpu_gpr[gprn]); } void spr_write_tbu(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_tbu(cpu_env, cpu_gpr[gprn]); } @@ -609,44 +597,44 @@ void spr_write_atbu(DisasContext *ctx, int sprn, int gprn) #if defined(TARGET_PPC64) void spr_read_purr(DisasContext *ctx, int gprn, int sprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_load_purr(cpu_gpr[gprn], cpu_env); } void spr_write_purr(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_purr(cpu_env, cpu_gpr[gprn]); } /* HDECR */ void spr_read_hdecr(DisasContext *ctx, int gprn, int sprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_load_hdecr(cpu_gpr[gprn], cpu_env); } void spr_write_hdecr(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_hdecr(cpu_env, cpu_gpr[gprn]); } void spr_read_vtb(DisasContext *ctx, int gprn, int sprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_load_vtb(cpu_gpr[gprn], cpu_env); } void spr_write_vtb(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_vtb(cpu_env, cpu_gpr[gprn]); } void spr_write_tbu40(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_tbu40(cpu_env, cpu_gpr[gprn]); } @@ -791,19 +779,19 @@ void spr_write_dpdes(DisasContext *ctx, int sprn, int gprn) #if !defined(CONFIG_USER_ONLY) void spr_read_40x_pit(DisasContext *ctx, int gprn, int sprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_load_40x_pit(cpu_gpr[gprn], cpu_env); } void spr_write_40x_pit(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_40x_pit(cpu_env, cpu_gpr[gprn]); } void spr_write_40x_dbcr0(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_store_spr(sprn, cpu_gpr[gprn]); gen_helper_store_40x_dbcr0(cpu_env, cpu_gpr[gprn]); /* We must stop translation as we may have rebooted */ @@ -812,19 +800,19 @@ void spr_write_40x_dbcr0(DisasContext *ctx, int sprn, int gprn) void spr_write_40x_sler(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_40x_sler(cpu_env, cpu_gpr[gprn]); } void spr_write_40x_tcr(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_40x_tcr(cpu_env, cpu_gpr[gprn]); } void spr_write_40x_tsr(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_40x_tsr(cpu_env, cpu_gpr[gprn]); } @@ -837,13 +825,13 @@ void spr_write_40x_pid(DisasContext *ctx, int sprn, int gprn) void spr_write_booke_tcr(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_booke_tcr(cpu_env, cpu_gpr[gprn]); } void spr_write_booke_tsr(DisasContext *ctx, int sprn, int gprn) { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_helper_store_booke_tsr(cpu_env, cpu_gpr[gprn]); } #endif @@ -2469,7 +2457,7 @@ static void gen_darn(DisasContext *ctx) if (l > 2) { tcg_gen_movi_i64(cpu_gpr[rD(ctx->opcode)], -1); } else { - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); if (l == 0) { gen_helper_darn32(cpu_gpr[rD(ctx->opcode)]); } else { @@ -4063,7 +4051,7 @@ static void pmu_count_insns(DisasContext *ctx) * running with icount and we do not handle it beforehand, * the helper can trigger a 'bad icount read'. */ - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); /* Avoid helper calls when only PMC5-6 are enabled. */ if (!ctx->pmc_other) { @@ -4376,7 +4364,7 @@ static void gen_rfi(DisasContext *ctx) } /* Restore CPU state */ CHK_SV(ctx); - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_update_cfar(ctx, ctx->cia); gen_helper_rfi(cpu_env); ctx->base.is_jmp = DISAS_EXIT; @@ -4391,7 +4379,7 @@ static void gen_rfid(DisasContext *ctx) #else /* Restore CPU state */ CHK_SV(ctx); - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_update_cfar(ctx, ctx->cia); gen_helper_rfid(cpu_env); ctx->base.is_jmp = DISAS_EXIT; @@ -4406,7 +4394,7 @@ static void gen_rfscv(DisasContext *ctx) #else /* Restore CPU state */ CHK_SV(ctx); - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_update_cfar(ctx, ctx->cia); gen_helper_rfscv(cpu_env); ctx->base.is_jmp = DISAS_EXIT; @@ -4731,7 +4719,7 @@ static void gen_mtmsrd(DisasContext *ctx) t0 = tcg_temp_new(); t1 = tcg_temp_new(); - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); if (ctx->opcode & 0x00010000) { /* L=1 form only updates EE and RI */ @@ -4771,7 +4759,7 @@ static void gen_mtmsr(DisasContext *ctx) t0 = tcg_temp_new(); t1 = tcg_temp_new(); - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); if (ctx->opcode & 0x00010000) { /* L=1 form only updates EE and RI */ mask &= (1ULL << MSR_RI) | (1ULL << MSR_EE); diff --git a/target/ppc/translate/branch-impl.c.inc b/target/ppc/translate/branch-impl.c.inc index 29cfa11..f9931b9 100644 --- a/target/ppc/translate/branch-impl.c.inc +++ b/target/ppc/translate/branch-impl.c.inc @@ -16,7 +16,7 @@ static bool trans_RFEBB(DisasContext *ctx, arg_XL_s *arg) { REQUIRE_INSNS_FLAGS2(ctx, ISA207S); - gen_icount_io_start(ctx); + translator_io_start(&ctx->base); gen_update_cfar(ctx, ctx->cia); gen_helper_rfebb(cpu_env, cpu_gpr[arg->s]); diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c index 57d0438..56381aa 100644 --- a/target/riscv/cpu_helper.c +++ b/target/riscv/cpu_helper.c @@ -31,6 +31,7 @@ #include "sysemu/cpu-timers.h" #include "cpu_bits.h" #include "debug.h" +#include "tcg/oversized-guest.h" int riscv_cpu_mmu_index(CPURISCVState *env, bool ifetch) { diff --git a/target/riscv/insn_trans/trans_privileged.c.inc b/target/riscv/insn_trans/trans_privileged.c.inc index 7c28371..528baa1 100644 --- a/target/riscv/insn_trans/trans_privileged.c.inc +++ b/target/riscv/insn_trans/trans_privileged.c.inc @@ -77,9 +77,7 @@ static bool trans_sret(DisasContext *ctx, arg_sret *a) #ifndef CONFIG_USER_ONLY if (has_ext(ctx, RVS)) { decode_save_opc(ctx); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_sret(cpu_pc, cpu_env); exit_tb(ctx); /* no chaining */ ctx->base.is_jmp = DISAS_NORETURN; @@ -96,9 +94,7 @@ static bool trans_mret(DisasContext *ctx, arg_mret *a) { #ifndef CONFIG_USER_ONLY decode_save_opc(ctx); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_mret(cpu_pc, cpu_env); exit_tb(ctx); /* no chaining */ ctx->base.is_jmp = DISAS_NORETURN; diff --git a/target/riscv/insn_trans/trans_rvi.c.inc b/target/riscv/insn_trans/trans_rvi.c.inc index c70c495..2031e99 100644 --- a/target/riscv/insn_trans/trans_rvi.c.inc +++ b/target/riscv/insn_trans/trans_rvi.c.inc @@ -813,9 +813,7 @@ static bool do_csrr(DisasContext *ctx, int rd, int rc) TCGv dest = dest_gpr(ctx, rd); TCGv_i32 csr = tcg_constant_i32(rc); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_csrr(dest, cpu_env, csr); gen_set_gpr(ctx, rd, dest); return do_csr_post(ctx); @@ -825,9 +823,7 @@ static bool do_csrw(DisasContext *ctx, int rc, TCGv src) { TCGv_i32 csr = tcg_constant_i32(rc); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_csrw(cpu_env, csr, src); return do_csr_post(ctx); } @@ -837,9 +833,7 @@ static bool do_csrrw(DisasContext *ctx, int rd, int rc, TCGv src, TCGv mask) TCGv dest = dest_gpr(ctx, rd); TCGv_i32 csr = tcg_constant_i32(rc); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_csrrw(dest, cpu_env, csr, src, mask); gen_set_gpr(ctx, rd, dest); return do_csr_post(ctx); @@ -851,9 +845,7 @@ static bool do_csrr_i128(DisasContext *ctx, int rd, int rc) TCGv desth = dest_gprh(ctx, rd); TCGv_i32 csr = tcg_constant_i32(rc); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_csrr_i128(destl, cpu_env, csr); tcg_gen_ld_tl(desth, cpu_env, offsetof(CPURISCVState, retxh)); gen_set_gpr128(ctx, rd, destl, desth); @@ -864,9 +856,7 @@ static bool do_csrw_i128(DisasContext *ctx, int rc, TCGv srcl, TCGv srch) { TCGv_i32 csr = tcg_constant_i32(rc); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_csrw_i128(cpu_env, csr, srcl, srch); return do_csr_post(ctx); } @@ -878,9 +868,7 @@ static bool do_csrrw_i128(DisasContext *ctx, int rd, int rc, TCGv desth = dest_gprh(ctx, rd); TCGv_i32 csr = tcg_constant_i32(rc); - if (tb_cflags(ctx->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&ctx->base); gen_helper_csrrw_i128(destl, cpu_env, csr, srcl, srch, maskl, maskh); tcg_gen_ld_tl(desth, cpu_env, offsetof(CPURISCVState, retxh)); gen_set_gpr128(ctx, rd, destl, desth); diff --git a/target/riscv/translate.c b/target/riscv/translate.c index 928da0d..933b11c 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -33,6 +33,10 @@ #include "instmap.h" #include "internals.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + /* global register indices */ static TCGv cpu_gpr[32], cpu_gprh[32], cpu_pc, cpu_vl, cpu_vstart; static TCGv_i64 cpu_fpr[32]; /* assume F and D extensions */ @@ -42,8 +46,6 @@ static TCGv load_val; static TCGv pm_mask; static TCGv pm_base; -#include "exec/gen-icount.h" - /* * If an operation is being performed on less than TARGET_LONG_BITS, * it may require the inputs to be sign- or zero-extended; which will diff --git a/target/rx/cpu.c b/target/rx/cpu.c index 67452e3..157e57d 100644 --- a/target/rx/cpu.c +++ b/target/rx/cpu.c @@ -24,6 +24,7 @@ #include "exec/exec-all.h" #include "hw/loader.h" #include "fpu/softfloat.h" +#include "tcg/debug-assert.h" static void rx_cpu_set_pc(CPUState *cs, vaddr value) { diff --git a/target/rx/op_helper.c b/target/rx/op_helper.c index acce650..dc0092c 100644 --- a/target/rx/op_helper.c +++ b/target/rx/op_helper.c @@ -23,6 +23,7 @@ #include "exec/helper-proto.h" #include "exec/cpu_ldst.h" #include "fpu/softfloat.h" +#include "tcg/debug-assert.h" static inline G_NORETURN void raise_exception(CPURXState *env, int index, diff --git a/target/rx/translate.c b/target/rx/translate.c index 70fad98..08cabbd 100644 --- a/target/rx/translate.c +++ b/target/rx/translate.c @@ -28,6 +28,11 @@ #include "exec/translator.h" #include "exec/log.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + + typedef struct DisasContext { DisasContextBase base; CPURXState *env; @@ -68,8 +73,6 @@ static TCGv_i64 cpu_acc; #define cpu_sp cpu_regs[0] -#include "exec/gen-icount.h" - /* decoder helper */ static uint32_t decode_load_bytes(DisasContext *ctx, uint32_t insn, int i, int n) diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index 3eb3708..7c549cd 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -38,7 +38,6 @@ #include "qemu/log.h" #include "qemu/host-utils.h" #include "exec/cpu_ldst.h" -#include "exec/gen-icount.h" #include "exec/helper-proto.h" #include "exec/helper-gen.h" @@ -46,6 +45,10 @@ #include "exec/log.h" #include "qemu/atomic128.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + /* Information that (most) every instruction needs to manipulate. */ typedef struct DisasContext DisasContext; @@ -6350,10 +6353,7 @@ static DisasJumpType translate_one(CPUS390XState *env, DisasContext *s) /* input/output is the special case for icount mode */ if (unlikely(insn->flags & IF_IO)) { - icount = tb_cflags(s->base.tb) & CF_USE_ICOUNT; - if (icount) { - gen_io_start(); - } + icount = translator_io_start(&s->base); } } diff --git a/target/sh4/translate.c b/target/sh4/translate.c index d9accfa..49c87d7 100644 --- a/target/sh4/translate.c +++ b/target/sh4/translate.c @@ -29,6 +29,10 @@ #include "exec/log.h" #include "qemu/qemu-print.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + typedef struct DisasContext { DisasContextBase base; @@ -71,8 +75,6 @@ static TCGv cpu_fregs[32]; /* internal register indexes */ static TCGv cpu_flags, cpu_delayed_pc, cpu_delayed_cond; -#include "exec/gen-icount.h" - void sh4_translate_init(void) { int i; @@ -2142,9 +2144,7 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) /* The entire region has been translated. */ ctx->envflags &= ~TB_FLAG_GUSA_MASK; - ctx->base.pc_next = pc_end; - ctx->base.num_insns += max_insns - 1; - return; + goto done; fail: qemu_log_mask(LOG_UNIMP, "Unrecognized gUSA sequence %08x-%08x\n", @@ -2161,8 +2161,19 @@ static void decode_gusa(DisasContext *ctx, CPUSH4State *env) purposes of accounting within the TB. We might as well report the entire region consumed via ctx->base.pc_next so that it's immediately available in the disassembly dump. */ + + done: ctx->base.pc_next = pc_end; ctx->base.num_insns += max_insns - 1; + + /* + * Emit insn_start to cover each of the insns in the region. + * This matches an assert in tcg.c making sure that we have + * tb->icount * insn_start. + */ + for (i = 1; i < max_insns; ++i) { + tcg_gen_insn_start(pc + i * 2, ctx->envflags); + } } #endif diff --git a/target/sparc/cpu.h b/target/sparc/cpu.h index fb98843..3d090e8 100644 --- a/target/sparc/cpu.h +++ b/target/sparc/cpu.h @@ -581,7 +581,6 @@ G_NORETURN void sparc_cpu_do_unaligned_access(CPUState *cpu, vaddr addr, uintptr_t retaddr); G_NORETURN void cpu_raise_exception_ra(CPUSPARCState *, int, uintptr_t); -#ifndef NO_CPU_IO_DEFS /* cpu_init.c */ void cpu_sparc_set_id(CPUSPARCState *env, unsigned int cpu); void sparc_cpu_list(void); @@ -638,7 +637,6 @@ static inline int tlb_compare_context(const SparcTLBEntry *tlb, } #endif -#endif /* cpu-exec.c */ #if !defined(CONFIG_USER_ONLY) diff --git a/target/sparc/translate.c b/target/sparc/translate.c index 93777984..bad2ec9 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -33,6 +33,9 @@ #include "exec/log.h" #include "asi.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H #define DYNAMIC_PC 1 /* dynamic pc value */ #define JUMP_PC 2 /* dynamic pc value which takes only two values @@ -63,8 +66,6 @@ static TCGv cpu_wim; /* Floating point registers */ static TCGv_i64 cpu_fpr[TARGET_DPREGS]; -#include "exec/gen-icount.h" - typedef struct DisasContext { DisasContextBase base; target_ulong pc; /* current Program Counter: integer or DYNAMIC_PC */ @@ -3214,16 +3215,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) r_const = tcg_constant_i32(dc->mem_idx); tcg_gen_ld_ptr(r_tickptr, cpu_env, offsetof(CPUSPARCState, tick)); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); + if (translator_io_start(&dc->base)) { + dc->base.is_jmp = DISAS_EXIT; } gen_helper_tick_get_count(cpu_dst, cpu_env, r_tickptr, r_const); gen_store_gpr(dc, rd, cpu_dst); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - /* I/O operations in icount mode must end the TB */ - dc->base.is_jmp = DISAS_EXIT; - } } break; case 0x5: /* V9 rdpc */ @@ -3266,16 +3263,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) r_const = tcg_constant_i32(dc->mem_idx); tcg_gen_ld_ptr(r_tickptr, cpu_env, offsetof(CPUSPARCState, stick)); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); + if (translator_io_start(&dc->base)) { + dc->base.is_jmp = DISAS_EXIT; } gen_helper_tick_get_count(cpu_dst, cpu_env, r_tickptr, r_const); gen_store_gpr(dc, rd, cpu_dst); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - /* I/O operations in icount mode must end the TB */ - dc->base.is_jmp = DISAS_EXIT; - } } break; case 0x19: /* System tick compare */ @@ -3396,15 +3389,11 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) r_const = tcg_constant_i32(dc->mem_idx); tcg_gen_ld_ptr(r_tickptr, cpu_env, offsetof(CPUSPARCState, tick)); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); + if (translator_io_start(&dc->base)) { + dc->base.is_jmp = DISAS_EXIT; } gen_helper_tick_get_count(cpu_tmp0, cpu_env, r_tickptr, r_const); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - /* I/O operations in icount mode must end the TB */ - dc->base.is_jmp = DISAS_EXIT; - } } break; case 5: // tba @@ -4209,10 +4198,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) r_tickptr = tcg_temp_new_ptr(); tcg_gen_ld_ptr(r_tickptr, cpu_env, offsetof(CPUSPARCState, tick)); - if (tb_cflags(dc->base.tb) & - CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_tick_set_limit(r_tickptr, cpu_tick_cmpr); /* End TB to handle timer interrupt */ @@ -4232,10 +4218,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) r_tickptr = tcg_temp_new_ptr(); tcg_gen_ld_ptr(r_tickptr, cpu_env, offsetof(CPUSPARCState, stick)); - if (tb_cflags(dc->base.tb) & - CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_tick_set_count(r_tickptr, cpu_tmp0); /* End TB to handle timer interrupt */ @@ -4255,10 +4238,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) r_tickptr = tcg_temp_new_ptr(); tcg_gen_ld_ptr(r_tickptr, cpu_env, offsetof(CPUSPARCState, stick)); - if (tb_cflags(dc->base.tb) & - CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_tick_set_limit(r_tickptr, cpu_stick_cmpr); /* End TB to handle timer interrupt */ @@ -4366,10 +4346,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) r_tickptr = tcg_temp_new_ptr(); tcg_gen_ld_ptr(r_tickptr, cpu_env, offsetof(CPUSPARCState, tick)); - if (tb_cflags(dc->base.tb) & - CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_tick_set_count(r_tickptr, cpu_tmp0); /* End TB to handle timer interrupt */ @@ -4381,14 +4358,10 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) break; case 6: // pstate save_state(dc); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } - gen_helper_wrpstate(cpu_env, cpu_tmp0); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - /* I/O ops in icount mode must end the TB */ + if (translator_io_start(&dc->base)) { dc->base.is_jmp = DISAS_EXIT; } + gen_helper_wrpstate(cpu_env, cpu_tmp0); dc->npc = DYNAMIC_PC; break; case 7: // tl @@ -4398,14 +4371,10 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) dc->npc = DYNAMIC_PC; break; case 8: // pil - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } - gen_helper_wrpil(cpu_env, cpu_tmp0); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - /* I/O ops in icount mode must end the TB */ + if (translator_io_start(&dc->base)) { dc->base.is_jmp = DISAS_EXIT; } + gen_helper_wrpil(cpu_env, cpu_tmp0); break; case 9: // cwp gen_helper_wrcwp(cpu_env, cpu_tmp0); @@ -4496,10 +4465,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) r_tickptr = tcg_temp_new_ptr(); tcg_gen_ld_ptr(r_tickptr, cpu_env, offsetof(CPUSPARCState, hstick)); - if (tb_cflags(dc->base.tb) & - CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_tick_set_limit(r_tickptr, cpu_hstick_cmpr); /* End TB to handle timer interrupt */ @@ -5122,9 +5088,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) goto priv_insn; dc->npc = DYNAMIC_PC; dc->pc = DYNAMIC_PC; - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_done(cpu_env); goto jmp_insn; case 1: @@ -5132,9 +5096,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn) goto priv_insn; dc->npc = DYNAMIC_PC; dc->pc = DYNAMIC_PC; - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_retry(cpu_env); goto jmp_insn; default: diff --git a/target/tricore/cpu.c b/target/tricore/cpu.c index d0a9272..7fa113f 100644 --- a/target/tricore/cpu.c +++ b/target/tricore/cpu.c @@ -22,6 +22,7 @@ #include "cpu.h" #include "exec/exec-all.h" #include "qemu/error-report.h" +#include "tcg/debug-assert.h" static inline void set_feature(CPUTriCoreState *env, int feature) { diff --git a/target/tricore/translate.c b/target/tricore/translate.c index 2646cb3..8e4f994 100644 --- a/target/tricore/translate.c +++ b/target/tricore/translate.c @@ -33,6 +33,11 @@ #include "exec/translator.h" #include "exec/log.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + + /* * TCG registers */ @@ -50,8 +55,6 @@ static TCGv cpu_PSW_SV; static TCGv cpu_PSW_AV; static TCGv cpu_PSW_SAV; -#include "exec/gen-icount.h" - static const char *regnames_a[] = { "a0" , "a1" , "a2" , "a3" , "a4" , "a5" , "a6" , "a7" , "a8" , "a9" , "sp" , "a11" , diff --git a/target/xtensa/translate.c b/target/xtensa/translate.c index 728aeeb..b7386ff 100644 --- a/target/xtensa/translate.c +++ b/target/xtensa/translate.c @@ -45,6 +45,10 @@ #include "exec/log.h" +#define HELPER_H "helper.h" +#include "exec/helper-info.c.inc" +#undef HELPER_H + struct DisasContext { DisasContextBase base; @@ -90,8 +94,6 @@ static TCGv_i32 cpu_exclusive_val; static GHashTable *xtensa_regfile_table; -#include "exec/gen-icount.h" - static char *sr_name[256]; static char *ur_name[256]; @@ -573,9 +575,7 @@ static int gen_postprocess(DisasContext *dc, int slot) #ifndef CONFIG_USER_ONLY if (op_flags & XTENSA_OP_CHECK_INTERRUPTS) { - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_check_interrupts(cpu_env); } #endif @@ -2125,9 +2125,7 @@ static void translate_rsr_ccount(DisasContext *dc, const OpcodeArg arg[], const uint32_t par[]) { #ifndef CONFIG_USER_ONLY - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_update_ccount(cpu_env); tcg_gen_mov_i32(arg[0].out, cpu_SR[par[0]]); #endif @@ -2443,9 +2441,7 @@ static void translate_waiti(DisasContext *dc, const OpcodeArg arg[], #ifndef CONFIG_USER_ONLY TCGv_i32 pc = tcg_constant_i32(dc->base.pc_next); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_waiti(cpu_env, pc, tcg_constant_i32(arg[0].imm)); #endif } @@ -2510,9 +2506,7 @@ static void translate_wsr_ccompare(DisasContext *dc, const OpcodeArg arg[], uint32_t id = par[0] - CCOMPARE; assert(id < dc->config->nccompare); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); tcg_gen_mov_i32(cpu_SR[par[0]], arg[0].in); gen_helper_update_ccompare(cpu_env, tcg_constant_i32(id)); #endif @@ -2522,9 +2516,7 @@ static void translate_wsr_ccount(DisasContext *dc, const OpcodeArg arg[], const uint32_t par[]) { #ifndef CONFIG_USER_ONLY - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } + translator_io_start(&dc->base); gen_helper_wsr_ccount(cpu_env, arg[0].in); #endif } @@ -2711,10 +2703,7 @@ static void translate_xsr_ccount(DisasContext *dc, const OpcodeArg arg[], #ifndef CONFIG_USER_ONLY TCGv_i32 tmp = tcg_temp_new_i32(); - if (tb_cflags(dc->base.tb) & CF_USE_ICOUNT) { - gen_io_start(); - } - + translator_io_start(&dc->base); gen_helper_update_ccount(cpu_env); tcg_gen_mov_i32(tmp, cpu_SR[par[0]]); gen_helper_wsr_ccount(cpu_env, arg[0].in); diff --git a/tcg/aarch64/tcg-target-reg-bits.h b/tcg/aarch64/tcg-target-reg-bits.h new file mode 100644 index 0000000..3b57a1a --- /dev/null +++ b/tcg/aarch64/tcg-target-reg-bits.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Define target-specific register size + * Copyright (c) 2023 Linaro + */ + +#ifndef TCG_TARGET_REG_BITS_H +#define TCG_TARGET_REG_BITS_H + +#define TCG_TARGET_REG_BITS 64 + +#endif diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index 261ad25..35ca80c 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -1636,6 +1636,9 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) return true; } +/* We expect to use a 7-bit scaled negative offset from ENV. */ +#define MIN_TLB_MASK_TABLE_OFS -512 + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -1674,12 +1677,10 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ? TCG_TYPE_I64 : TCG_TYPE_I32); /* Load env_tlb(env)->f[mmu_idx].{mask,table} into {tmp0,tmp1}. */ - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -512); QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, mask) != 0); QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, table) != 8); tcg_out_insn(s, 3314, LDP, TCG_REG_TMP0, TCG_REG_TMP1, TCG_AREG0, - TLB_MASK_TABLE_OFS(mem_index), 1, 0); + tlb_mask_table_ofs(s, mem_index), 1, 0); /* Extract the TLB index from the address into X0. */ tcg_out_insn(s, 3502S, AND_LSR, mask_type == TCG_TYPE_I64, @@ -1690,6 +1691,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_insn(s, 3502, ADD, 1, TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_TMP0); /* Load the tlb comparator into TMP0, and the fast path addend into TMP1. */ + QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN); tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP1, is_ld ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write)); diff --git a/tcg/arm/tcg-target-reg-bits.h b/tcg/arm/tcg-target-reg-bits.h new file mode 100644 index 0000000..23b7730 --- /dev/null +++ b/tcg/arm/tcg-target-reg-bits.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define target-specific register size + * Copyright (c) 2023 Linaro + */ + +#ifndef TCG_TARGET_REG_BITS_H +#define TCG_TARGET_REG_BITS_H + +#define TCG_TARGET_REG_BITS 32 + +#endif diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index 20cc1cc..83e2860 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -1374,6 +1374,9 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) return true; } +/* We expect to use an 9-bit sign-magnitude negative offset from ENV. */ +#define MIN_TLB_MASK_TABLE_OFS -256 + static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, TCGReg addrlo, TCGReg addrhi, MemOpIdx oi, bool is_ld) @@ -1405,7 +1408,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, int mem_index = get_mmuidx(oi); int cmp_off = is_ld ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write); - int fast_off = TLB_MASK_TABLE_OFS(mem_index); + int fast_off = tlb_mask_table_ofs(s, mem_index); unsigned s_mask = (1 << (opc & MO_SIZE)) - 1; TCGReg t_addr; @@ -1416,8 +1419,6 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->addrhi_reg = addrhi; /* Load env_tlb(env)->f[mmu_idx].{mask,table} into {r0,r1}. */ - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -256); QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, mask) != 0); QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, table) != 4); tcg_out_ldrd_8(s, COND_AL, TCG_REG_R0, TCG_AREG0, fast_off); @@ -1430,6 +1431,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, * Add the tlb_table pointer, creating the CPUTLBEntry address in R1. * Load the tlb comparator into R2/R3 and the fast path addend into R1. */ + QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN); if (cmp_off == 0) { if (s->addr_type == TCG_TYPE_I32) { tcg_out_ld32_rwb(s, COND_AL, TCG_REG_R2, TCG_REG_R1, TCG_REG_R0); diff --git a/tcg/i386/tcg-target-reg-bits.h b/tcg/i386/tcg-target-reg-bits.h new file mode 100644 index 0000000..aa38605 --- /dev/null +++ b/tcg/i386/tcg-target-reg-bits.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define target-specific register size + * Copyright (c) 2008 Fabrice Bellard + */ + +#ifndef TCG_TARGET_REG_BITS_H +#define TCG_TARGET_REG_BITS_H + +#ifdef __x86_64__ +# define TCG_TARGET_REG_BITS 64 +#else +# define TCG_TARGET_REG_BITS 32 +#endif + +#endif diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index ae54e5f..ab997b5 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -1900,6 +1900,8 @@ static inline int setup_guest_base_seg(void) #endif /* setup_guest_base_seg */ #endif /* !SOFTMMU */ +#define MIN_TLB_MASK_TABLE_OFS INT_MIN + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -1934,6 +1936,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, int trexw = 0, hrexw = 0, tlbrexw = 0; unsigned mem_index = get_mmuidx(oi); unsigned s_mask = (1 << s_bits) - 1; + int fast_ofs = tlb_mask_table_ofs(s, mem_index); int tlb_mask; ldst = new_ldst_label(s); @@ -1959,12 +1962,10 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, s->page_bits - CPU_TLB_ENTRY_BITS); tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, TCG_REG_L0, TCG_AREG0, - TLB_MASK_TABLE_OFS(mem_index) + - offsetof(CPUTLBDescFast, mask)); + fast_ofs + offsetof(CPUTLBDescFast, mask)); tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, TCG_REG_L0, TCG_AREG0, - TLB_MASK_TABLE_OFS(mem_index) + - offsetof(CPUTLBDescFast, table)); + fast_ofs + offsetof(CPUTLBDescFast, table)); /* * If the required alignment is at least as large as the access, simply diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index 1468f8e..2a2e3ff 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -30,11 +30,9 @@ #define TCG_TARGET_INSN_UNIT_SIZE 1 #ifdef __x86_64__ -# define TCG_TARGET_REG_BITS 64 # define TCG_TARGET_NB_REGS 32 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) #else -# define TCG_TARGET_REG_BITS 32 # define TCG_TARGET_NB_REGS 24 # define MAX_CODE_GEN_BUFFER_SIZE UINT32_MAX #endif diff --git a/tcg/loongarch64/tcg-target-reg-bits.h b/tcg/loongarch64/tcg-target-reg-bits.h new file mode 100644 index 0000000..51373ad --- /dev/null +++ b/tcg/loongarch64/tcg-target-reg-bits.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define target-specific register size + * Copyright (c) 2021 WANG Xuerui <git@xen0n.name> + */ + +#ifndef TCG_TARGET_REG_BITS_H +#define TCG_TARGET_REG_BITS_H + +/* + * Loongson removed the (incomplete) 32-bit support from kernel and toolchain + * for the initial upstreaming of this architecture, so don't bother and just + * support the LP64* ABI for now. + */ +#if defined(__loongarch64) +# define TCG_TARGET_REG_BITS 64 +#else +# error unsupported LoongArch register size +#endif + +#endif diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index 0bae922..baf5fc3 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -834,6 +834,9 @@ bool tcg_target_has_memory_bswap(MemOp memop) return false; } +/* We expect to use a 12-bit negative offset from ENV. */ +#define MIN_TLB_MASK_TABLE_OFS -(1 << 11) + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -855,7 +858,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, #ifdef CONFIG_SOFTMMU unsigned s_bits = opc & MO_SIZE; int mem_index = get_mmuidx(oi); - int fast_ofs = TLB_MASK_TABLE_OFS(mem_index); + int fast_ofs = tlb_mask_table_ofs(s, mem_index); int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); @@ -864,8 +867,6 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->oi = oi; ldst->addrlo_reg = addr_reg; - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_AREG0, mask_ofs); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_AREG0, table_ofs); @@ -875,6 +876,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_opc_add_d(s, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); /* Load the tlb comparator and the addend. */ + QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN); tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP2, is_ld ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write)); diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h index 482901a..26f1aab 100644 --- a/tcg/loongarch64/tcg-target.h +++ b/tcg/loongarch64/tcg-target.h @@ -29,17 +29,6 @@ #ifndef LOONGARCH_TCG_TARGET_H #define LOONGARCH_TCG_TARGET_H -/* - * Loongson removed the (incomplete) 32-bit support from kernel and toolchain - * for the initial upstreaming of this architecture, so don't bother and just - * support the LP64* ABI for now. - */ -#if defined(__loongarch64) -# define TCG_TARGET_REG_BITS 64 -#else -# error unsupported LoongArch register size -#endif - #define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_NB_REGS 32 diff --git a/tcg/meson.build b/tcg/meson.build index bdc185a..565c60b 100644 --- a/tcg/meson.build +++ b/tcg/meson.build @@ -1,3 +1,7 @@ +if not get_option('tcg').allowed() + subdir_done() +endif + tcg_ss = ss.source_set() tcg_ss.add(files( @@ -14,8 +18,28 @@ tcg_ss.add(files( if get_option('tcg_interpreter') libffi = dependency('libffi', version: '>=3.0', required: true, method: 'pkg-config') - specific_ss.add(libffi) - specific_ss.add(files('tci.c')) + tcg_ss.add(libffi) + tcg_ss.add(files('tci.c')) endif -specific_ss.add_all(when: 'CONFIG_TCG', if_true: tcg_ss) +tcg_ss = tcg_ss.apply(config_host, strict: false) + +libtcg_user = static_library('tcg_user', + tcg_ss.sources() + genh, + name_suffix: 'fa', + c_args: '-DCONFIG_USER_ONLY', + build_by_default: have_user) + +tcg_user = declare_dependency(link_with: libtcg_user, + dependencies: tcg_ss.dependencies()) +user_ss.add(tcg_user) + +libtcg_softmmu = static_library('tcg_softmmu', + tcg_ss.sources() + genh, + name_suffix: 'fa', + c_args: '-DCONFIG_SOFTMMU', + build_by_default: have_system) + +tcg_softmmu = declare_dependency(link_with: libtcg_softmmu, + dependencies: tcg_ss.dependencies()) +softmmu_ss.add(tcg_softmmu) diff --git a/tcg/mips/tcg-target-reg-bits.h b/tcg/mips/tcg-target-reg-bits.h new file mode 100644 index 0000000..56fe0a7 --- /dev/null +++ b/tcg/mips/tcg-target-reg-bits.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define target-specific register size + * Copyright (c) 2008-2009 Arnaud Patard <arnaud.patard@rtp-net.org> + */ + +#ifndef TCG_TARGET_REG_BITS_H +#define TCG_TARGET_REG_BITS_H + +#if _MIPS_SIM == _ABIO32 +# define TCG_TARGET_REG_BITS 32 +#elif _MIPS_SIM == _ABIN32 || _MIPS_SIM == _ABI64 +# define TCG_TARGET_REG_BITS 64 +#else +# error "Unknown ABI" +#endif + +#endif diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index 3274d9a..9faa8bd 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -1254,6 +1254,9 @@ bool tcg_target_has_memory_bswap(MemOp memop) return false; } +/* We expect to use a 16-bit negative offset from ENV. */ +#define MIN_TLB_MASK_TABLE_OFS -32768 + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -1279,7 +1282,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, #ifdef CONFIG_SOFTMMU unsigned s_mask = (1 << s_bits) - 1; int mem_index = get_mmuidx(oi); - int fast_off = TLB_MASK_TABLE_OFS(mem_index); + int fast_off = tlb_mask_table_ofs(s, mem_index); int mask_off = fast_off + offsetof(CPUTLBDescFast, mask); int table_off = fast_off + offsetof(CPUTLBDescFast, table); int add_off = offsetof(CPUTLBEntry, addend); @@ -1293,8 +1296,6 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->addrhi_reg = addrhi; /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */ - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -32768); tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_AREG0, mask_off); tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP1, TCG_AREG0, table_off); @@ -1311,14 +1312,17 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, /* Add the tlb_table pointer, creating the CPUTLBEntry address in TMP3. */ tcg_out_opc_reg(s, ALIAS_PADD, TCG_TMP3, TCG_TMP3, TCG_TMP1); + if (TCG_TARGET_REG_BITS == 32 || addr_type == TCG_TYPE_I32) { + /* Load the (low half) tlb comparator. */ + tcg_out_ld(s, TCG_TYPE_I32, TCG_TMP0, TCG_TMP3, + cmp_off + HOST_BIG_ENDIAN * 4); + } else { + tcg_out_ld(s, TCG_TYPE_I64, TCG_TMP0, TCG_TMP3, cmp_off); + } + if (TCG_TARGET_REG_BITS == 64 || addr_type == TCG_TYPE_I32) { - /* Load the tlb comparator. */ - tcg_out_ld(s, addr_type, TCG_TMP0, TCG_TMP3, cmp_off); /* Load the tlb addend for the fast path. */ tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP3, TCG_TMP3, add_off); - } else { - /* Load the low half of the tlb comparator. */ - tcg_out_ldst(s, OPC_LW, TCG_TMP0, TCG_TMP3, cmp_off + LO_OFF); } /* diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index e4806f6..dd2efa7 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -27,14 +27,6 @@ #ifndef MIPS_TCG_TARGET_H #define MIPS_TCG_TARGET_H -#if _MIPS_SIM == _ABIO32 -# define TCG_TARGET_REG_BITS 32 -#elif _MIPS_SIM == _ABIN32 || _MIPS_SIM == _ABI64 -# define TCG_TARGET_REG_BITS 64 -#else -# error "Unknown ABI" -#endif - #define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_NB_REGS 32 diff --git a/tcg/optimize.c b/tcg/optimize.c index bf975a3..d215636 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -25,7 +25,7 @@ #include "qemu/osdep.h" #include "qemu/int128.h" -#include "tcg/tcg-op.h" +#include "tcg/tcg-op-common.h" #include "tcg-internal.h" #define CASE_OP_32_64(x) \ diff --git a/tcg/ppc/tcg-target-reg-bits.h b/tcg/ppc/tcg-target-reg-bits.h new file mode 100644 index 0000000..0efa80e --- /dev/null +++ b/tcg/ppc/tcg-target-reg-bits.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define target-specific register size + * Copyright (c) 2008 Fabrice Bellard + */ + +#ifndef TCG_TARGET_REG_BITS_H +#define TCG_TARGET_REG_BITS_H + +#ifdef _ARCH_PPC64 +# define TCG_TARGET_REG_BITS 64 +#else +# define TCG_TARGET_REG_BITS 32 +#endif + +#endif diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index d47a9e3..507fe6c 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -2036,6 +2036,9 @@ bool tcg_target_has_memory_bswap(MemOp memop) return aa.atom <= MO_64; } +/* We expect to use a 16-bit negative offset from ENV. */ +#define MIN_TLB_MASK_TABLE_OFS -32768 + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -2046,6 +2049,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, TCGReg addrlo, TCGReg addrhi, MemOpIdx oi, bool is_ld) { + TCGType addr_type = s->addr_type; TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); MemOp a_bits, s_bits; @@ -2071,7 +2075,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, int mem_index = get_mmuidx(oi); int cmp_off = is_ld ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write); - int fast_off = TLB_MASK_TABLE_OFS(mem_index); + int fast_off = tlb_mask_table_ofs(s, mem_index); int mask_off = fast_off + offsetof(CPUTLBDescFast, mask); int table_off = fast_off + offsetof(CPUTLBDescFast, table); @@ -2082,8 +2086,6 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->addrhi_reg = addrhi; /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */ - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -32768); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_AREG0, mask_off); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_AREG0, table_off); @@ -2097,26 +2099,31 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, } tcg_out32(s, AND | SAB(TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_R0)); - /* Load the (low part) TLB comparator into TMP2. */ - if (cmp_off == 0 && TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { - uint32_t lxu = (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32 - ? LWZUX : LDUX); - tcg_out32(s, lxu | TAB(TCG_REG_TMP2, TCG_REG_TMP1, TCG_REG_TMP2)); - } else { - tcg_out32(s, ADD | TAB(TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_TMP2)); - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_TMP2, - TCG_REG_TMP1, cmp_off + 4 * HOST_BIG_ENDIAN); + /* + * Load the (low part) TLB comparator into TMP2. + * For 64-bit host, always load the entire 64-bit slot for simplicity. + * We will ignore the high bits with tcg_out_cmp(..., addr_type). + */ + if (TCG_TARGET_REG_BITS == 64) { + if (cmp_off == 0) { + tcg_out32(s, LDUX | TAB(TCG_REG_TMP2, TCG_REG_TMP1, TCG_REG_TMP2)); } else { - tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP2, TCG_REG_TMP1, cmp_off); + tcg_out32(s, ADD | TAB(TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_TMP2)); + tcg_out_ld(s, TCG_TYPE_I64, TCG_REG_TMP2, TCG_REG_TMP1, cmp_off); } + } else if (cmp_off == 0 && !HOST_BIG_ENDIAN) { + tcg_out32(s, LWZUX | TAB(TCG_REG_TMP2, TCG_REG_TMP1, TCG_REG_TMP2)); + } else { + tcg_out32(s, ADD | TAB(TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_TMP2)); + tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_TMP2, TCG_REG_TMP1, + cmp_off + 4 * HOST_BIG_ENDIAN); } /* * Load the TLB addend for use on the fast path. * Do this asap to minimize any load use delay. */ - if (TCG_TARGET_REG_BITS >= TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS == 64 || addr_type == TCG_TYPE_I32) { tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_REG_TMP1, offsetof(CPUTLBEntry, addend)); } @@ -2151,7 +2158,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, } /* Mask the address for the requested alignment. */ - if (TARGET_LONG_BITS == 32) { + if (addr_type == TCG_TYPE_I32) { tcg_out_rlw(s, RLWINM, TCG_REG_R0, t, 0, (32 - a_bits) & 31, 31 - s->page_bits); } else if (a_bits == 0) { @@ -2163,7 +2170,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, } } - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS == 32 && addr_type != TCG_TYPE_I32) { /* Low part comparison into cr7. */ tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP2, 0, 7, TCG_TYPE_I32); @@ -2183,8 +2190,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out32(s, CRAND | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ)); } else { /* Full comparison into cr7. */ - tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP2, - 0, 7, TCG_TYPE_TL); + tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP2, 0, 7, addr_type); } /* Load a pointer into the current opcode w/conditional branch-link. */ @@ -2211,7 +2217,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, h->base = guest_base ? TCG_GUEST_BASE_REG : 0; #endif - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + if (TCG_TARGET_REG_BITS == 64 && addr_type == TCG_TYPE_I32) { /* Zero-extend the guest address for use in the host address. */ tcg_out_ext32u(s, TCG_REG_R0, addrlo); h->index = TCG_REG_R0; diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index 40f20b0..c7552b6 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -25,11 +25,6 @@ #ifndef PPC_TCG_TARGET_H #define PPC_TCG_TARGET_H -#ifdef _ARCH_PPC64 -# define TCG_TARGET_REG_BITS 64 -#else -# define TCG_TARGET_REG_BITS 32 -#endif #define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) #define TCG_TARGET_NB_REGS 64 diff --git a/tcg/region.c b/tcg/region.c index bef4c47..2b28ed3 100644 --- a/tcg/region.c +++ b/tcg/region.c @@ -30,8 +30,8 @@ #include "qemu/cacheinfo.h" #include "qemu/qtree.h" #include "qapi/error.h" -#include "exec/exec-all.h" #include "tcg/tcg.h" +#include "exec/translation-block.h" #include "tcg-internal.h" @@ -505,6 +505,14 @@ static int alloc_code_gen_buffer(size_t tb_size, int splitwx, Error **errp) return PROT_READ | PROT_WRITE; } #elif defined(_WIN32) +/* + * Local source-level compatibility with Unix. + * Used by tcg_region_init below. + */ +#define PROT_READ 1 +#define PROT_WRITE 2 +#define PROT_EXEC 4 + static int alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) { void *buf; @@ -525,7 +533,7 @@ static int alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) region.start_aligned = buf; region.total_size = size; - return PAGE_READ | PAGE_WRITE | PAGE_EXEC; + return PROT_READ | PROT_WRITE | PROT_EXEC; } #else static int alloc_code_gen_buffer_anon(size_t size, int prot, @@ -794,10 +802,10 @@ void tcg_region_init(size_t tb_size, int splitwx, unsigned max_cpus) * buffer -- let that one use hugepages throughout. * Work with the page protections set up with the initial mapping. */ - need_prot = PAGE_READ | PAGE_WRITE; + need_prot = PROT_READ | PROT_WRITE; #ifndef CONFIG_TCG_INTERPRETER if (tcg_splitwx_diff == 0) { - need_prot |= PAGE_EXEC; + need_prot |= PROT_EXEC; } #endif for (size_t i = 0, n = region.n; i < n; i++) { @@ -807,9 +815,9 @@ void tcg_region_init(size_t tb_size, int splitwx, unsigned max_cpus) if (have_prot != need_prot) { int rc; - if (need_prot == (PAGE_READ | PAGE_WRITE | PAGE_EXEC)) { + if (need_prot == (PROT_READ | PROT_WRITE | PROT_EXEC)) { rc = qemu_mprotect_rwx(start, end - start); - } else if (need_prot == (PAGE_READ | PAGE_WRITE)) { + } else if (need_prot == (PROT_READ | PROT_WRITE)) { rc = qemu_mprotect_rw(start, end - start); } else { g_assert_not_reached(); diff --git a/tcg/riscv/tcg-target-reg-bits.h b/tcg/riscv/tcg-target-reg-bits.h new file mode 100644 index 0000000..761ca0d --- /dev/null +++ b/tcg/riscv/tcg-target-reg-bits.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define target-specific register size + * Copyright (c) 2018 SiFive, Inc + */ + +#ifndef TCG_TARGET_REG_BITS_H +#define TCG_TARGET_REG_BITS_H + +/* + * We don't support oversize guests. + * Since we will only build tcg once, this in turn requires a 64-bit host. + */ +#if __riscv_xlen != 64 +#error "unsupported code generation mode" +#endif +#define TCG_TARGET_REG_BITS 64 + +#endif diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index c025712..eeaeb6b 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -1185,6 +1185,9 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) return true; } +/* We expect to use a 12-bit negative offset from ENV. */ +#define MIN_TLB_MASK_TABLE_OFS -(1 << 11) + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -1195,6 +1198,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, TCGReg addr_reg, MemOpIdx oi, bool is_ld) { + TCGType addr_type = s->addr_type; TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); TCGAtomAlign aa; @@ -1207,7 +1211,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, unsigned s_bits = opc & MO_SIZE; unsigned s_mask = (1u << s_bits) - 1; int mem_index = get_mmuidx(oi); - int fast_ofs = TLB_MASK_TABLE_OFS(mem_index); + int fast_ofs = tlb_mask_table_ofs(s, mem_index); int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); int compare_mask; @@ -1218,8 +1222,6 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, ldst->oi = oi; ldst->addrlo_reg = addr_reg; - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_AREG0, mask_ofs); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_AREG0, table_ofs); @@ -1236,19 +1238,20 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, addr_adj = addr_reg; if (a_mask < s_mask) { addr_adj = TCG_REG_TMP0; - tcg_out_opc_imm(s, TARGET_LONG_BITS == 32 ? OPC_ADDIW : OPC_ADDI, + tcg_out_opc_imm(s, addr_type == TCG_TYPE_I32 ? OPC_ADDIW : OPC_ADDI, addr_adj, addr_reg, s_mask - a_mask); } compare_mask = s->page_mask | a_mask; if (compare_mask == sextreg(compare_mask, 0, 12)) { tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_adj, compare_mask); } else { - tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); + tcg_out_movi(s, addr_type, TCG_REG_TMP1, compare_mask); tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addr_adj); } /* Load the tlb comparator and the addend. */ - tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP0, TCG_REG_TMP2, + QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN); + tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP2, is_ld ? offsetof(CPUTLBEntry, addr_read) : offsetof(CPUTLBEntry, addr_write)); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2, @@ -1259,7 +1262,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); /* TLB Hit - translate address using addend. */ - if (TARGET_LONG_BITS == 64) { + if (addr_type != TCG_TYPE_I32) { tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, addr_reg, TCG_REG_TMP2); } else if (have_zba) { tcg_out_opc_reg(s, OPC_ADD_UW, TCG_REG_TMP0, addr_reg, TCG_REG_TMP2); @@ -1287,7 +1290,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, if (guest_base != 0) { base = TCG_REG_TMP0; - if (TARGET_LONG_BITS == 64) { + if (addr_type != TCG_TYPE_I32) { tcg_out_opc_reg(s, OPC_ADD, base, addr_reg, TCG_GUEST_BASE_REG); } else if (have_zba) { tcg_out_opc_reg(s, OPC_ADD_UW, base, addr_reg, TCG_GUEST_BASE_REG); @@ -1295,7 +1298,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, tcg_out_ext32u(s, base, addr_reg); tcg_out_opc_reg(s, OPC_ADD, base, base, TCG_GUEST_BASE_REG); } - } else if (TARGET_LONG_BITS == 64) { + } else if (addr_type != TCG_TYPE_I32) { base = addr_reg; } else { base = TCG_REG_TMP0; diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index 54fdff0..e1d8110 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -25,15 +25,6 @@ #ifndef RISCV_TCG_TARGET_H #define RISCV_TCG_TARGET_H -/* - * We don't support oversize guests. - * Since we will only build tcg once, this in turn requires a 64-bit host. - */ -#if __riscv_xlen != 64 -#error "unsupported code generation mode" -#endif -#define TCG_TARGET_REG_BITS 64 - #define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_NB_REGS 32 #define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) diff --git a/tcg/s390x/tcg-target-reg-bits.h b/tcg/s390x/tcg-target-reg-bits.h new file mode 100644 index 0000000..b01414e --- /dev/null +++ b/tcg/s390x/tcg-target-reg-bits.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define target-specific register size + * Copyright (c) 2009 Ulrich Hecht <uli@suse.de> + */ + +#ifndef TCG_TARGET_REG_BITS_H +#define TCG_TARGET_REG_BITS_H + +/* We only support generating code for 64-bit mode. */ +#if UINTPTR_MAX == UINT64_MAX +# define TCG_TARGET_REG_BITS 64 +#else +# error "unsupported code generation mode" +#endif + +#endif diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 503126c..a878acd 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -24,11 +24,6 @@ * THE SOFTWARE. */ -/* We only support generating code for 64-bit mode. */ -#if TCG_TARGET_REG_BITS != 64 -#error "unsupported code generation mode" -#endif - #include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" #include "elf.h" @@ -1735,6 +1730,9 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) return true; } +/* We're expecting to use a 20-bit negative offset on the tlb memory ops. */ +#define MIN_TLB_MASK_TABLE_OFS -(1 << 19) + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -1745,6 +1743,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, TCGReg addr_reg, MemOpIdx oi, bool is_ld) { + TCGType addr_type = s->addr_type; TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); MemOp s_bits = opc & MO_SIZE; @@ -1756,7 +1755,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, #ifdef CONFIG_SOFTMMU unsigned s_mask = (1 << s_bits) - 1; int mem_index = get_mmuidx(oi); - int fast_off = TLB_MASK_TABLE_OFS(mem_index); + int fast_off = tlb_mask_table_ofs(s, mem_index); int mask_off = fast_off + offsetof(CPUTLBDescFast, mask); int table_off = fast_off + offsetof(CPUTLBDescFast, table); int ofs, a_off; @@ -1770,8 +1769,6 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_sh64(s, RSY_SRLG, TCG_TMP0, addr_reg, TCG_REG_NONE, s->page_bits - CPU_TLB_ENTRY_BITS); - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 19)); tcg_out_insn(s, RXY, NG, TCG_TMP0, TCG_AREG0, TCG_REG_NONE, mask_off); tcg_out_insn(s, RXY, AG, TCG_TMP0, TCG_AREG0, TCG_REG_NONE, table_off); @@ -1786,7 +1783,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tgen_andi_risbg(s, TCG_REG_R0, addr_reg, tlb_mask); } else { tcg_out_insn(s, RX, LA, TCG_REG_R0, addr_reg, TCG_REG_NONE, a_off); - tgen_andi(s, TCG_TYPE_TL, TCG_REG_R0, tlb_mask); + tgen_andi(s, addr_type, TCG_REG_R0, tlb_mask); } if (is_ld) { @@ -1794,7 +1791,8 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, } else { ofs = offsetof(CPUTLBEntry, addr_write); } - if (TARGET_LONG_BITS == 32) { + if (addr_type == TCG_TYPE_I32) { + ofs += HOST_BIG_ENDIAN * 4; tcg_out_insn(s, RX, C, TCG_REG_R0, TCG_TMP0, TCG_REG_NONE, ofs); } else { tcg_out_insn(s, RXY, CG, TCG_REG_R0, TCG_TMP0, TCG_REG_NONE, ofs); @@ -1807,7 +1805,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, tcg_out_insn(s, RXY, LG, h->index, TCG_TMP0, TCG_REG_NONE, offsetof(CPUTLBEntry, addend)); - if (TARGET_LONG_BITS == 32) { + if (addr_type == TCG_TYPE_I32) { tcg_out_insn(s, RRE, ALGFR, h->index, addr_reg); h->base = TCG_REG_NONE; } else { @@ -1830,7 +1828,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, } h->base = addr_reg; - if (TARGET_LONG_BITS == 32) { + if (addr_type == TCG_TYPE_I32) { tcg_out_ext32u(s, TCG_TMP0, addr_reg); h->base = TCG_TMP0; } diff --git a/tcg/sparc64/tcg-target-reg-bits.h b/tcg/sparc64/tcg-target-reg-bits.h new file mode 100644 index 0000000..34a6711 --- /dev/null +++ b/tcg/sparc64/tcg-target-reg-bits.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define target-specific register size + * Copyright (c) 2023 Linaro + */ + +#ifndef TCG_TARGET_REG_BITS_H +#define TCG_TARGET_REG_BITS_H + +#define TCG_TARGET_REG_BITS 64 + +#endif diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index d2d0f60..ffcb879 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -1017,6 +1017,9 @@ bool tcg_target_has_memory_bswap(MemOp memop) return true; } +/* We expect to use a 13-bit negative offset from ENV. */ +#define MIN_TLB_MASK_TABLE_OFS -(1 << 12) + /* * For softmmu, perform the TLB load and compare. * For useronly, perform any required alignment tests. @@ -1027,6 +1030,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, TCGReg addr_reg, MemOpIdx oi, bool is_ld) { + TCGType addr_type = s->addr_type; TCGLabelQemuLdst *ldst = NULL; MemOp opc = get_memop(oi); MemOp s_bits = opc & MO_SIZE; @@ -1039,7 +1043,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, #ifdef CONFIG_SOFTMMU int mem_index = get_mmuidx(oi); - int fast_off = TLB_MASK_TABLE_OFS(mem_index); + int fast_off = tlb_mask_table_ofs(s, mem_index); int mask_off = fast_off + offsetof(CPUTLBDescFast, mask); int table_off = fast_off + offsetof(CPUTLBDescFast, table); int cmp_off = is_ld ? offsetof(CPUTLBEntry, addr_read) @@ -1049,8 +1053,6 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, int cc; /* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */ - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); - QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 12)); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_T2, TCG_AREG0, mask_off); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_T3, TCG_AREG0, table_off); @@ -1062,8 +1064,12 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, /* Add the tlb_table pointer, creating the CPUTLBEntry address into R2. */ tcg_out_arith(s, TCG_REG_T1, TCG_REG_T1, TCG_REG_T3, ARITH_ADD); - /* Load the tlb comparator and the addend. */ - tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_T2, TCG_REG_T1, cmp_off); + /* + * Load the tlb comparator and the addend. + * Always load the entire 64-bit comparator for simplicity. + * We will ignore the high bits via BPCC_ICC below. + */ + tcg_out_ld(s, TCG_TYPE_I64, TCG_REG_T2, TCG_REG_T1, cmp_off); tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_T1, TCG_REG_T1, add_off); h->base = TCG_REG_T1; @@ -1084,7 +1090,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, ldst->label_ptr[0] = s->code_ptr; /* bne,pn %[xi]cc, label0 */ - cc = TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC; + cc = addr_type == TCG_TYPE_I32 ? BPCC_ICC : BPCC_XCC; tcg_out_bpcc0(s, COND_NE, BPCC_PN | cc, 0); #else /* @@ -1110,7 +1116,7 @@ static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, #endif /* If the guest address must be zero-extended, do in the delay slot. */ - if (TARGET_LONG_BITS == 32) { + if (addr_type == TCG_TYPE_I32) { tcg_out_ext32u(s, TCG_REG_T2, addr_reg); h->index = TCG_REG_T2; } else { diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index 67b698b..fbe62b3 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -25,55 +25,10 @@ #ifndef TCG_INTERNAL_H #define TCG_INTERNAL_H -#ifdef CONFIG_TCG_INTERPRETER -#include <ffi.h> -#endif +#include "tcg/helper-info.h" #define TCG_HIGHWATER 1024 -/* - * Describe the calling convention of a given argument type. - */ -typedef enum { - TCG_CALL_RET_NORMAL, /* by registers */ - TCG_CALL_RET_BY_REF, /* for i128, by reference */ - TCG_CALL_RET_BY_VEC, /* for i128, by vector register */ -} TCGCallReturnKind; - -typedef enum { - TCG_CALL_ARG_NORMAL, /* by registers (continuing onto stack) */ - TCG_CALL_ARG_EVEN, /* like normal, but skipping odd slots */ - TCG_CALL_ARG_EXTEND, /* for i32, as a sign/zero-extended i64 */ - TCG_CALL_ARG_EXTEND_U, /* ... as a zero-extended i64 */ - TCG_CALL_ARG_EXTEND_S, /* ... as a sign-extended i64 */ - TCG_CALL_ARG_BY_REF, /* for i128, by reference, first */ - TCG_CALL_ARG_BY_REF_N, /* ... by reference, subsequent */ -} TCGCallArgumentKind; - -typedef struct TCGCallArgumentLoc { - TCGCallArgumentKind kind : 8; - unsigned arg_slot : 8; - unsigned ref_slot : 8; - unsigned arg_idx : 4; - unsigned tmp_subindex : 2; -} TCGCallArgumentLoc; - -typedef struct TCGHelperInfo { - void *func; - const char *name; -#ifdef CONFIG_TCG_INTERPRETER - ffi_cif *cif; -#endif - unsigned typemask : 32; - unsigned flags : 8; - unsigned nr_in : 8; - unsigned nr_out : 8; - TCGCallReturnKind out_kind : 8; - - /* Maximum physical arguments are constrained by TCG_TYPE_I128. */ - TCGCallArgumentLoc in[MAX_CALL_IARGS * (128 / TCG_TARGET_REG_BITS)]; -} TCGHelperInfo; - extern TCGContext tcg_init_ctx; extern TCGContext **tcg_ctxs; extern unsigned int tcg_cur_ctxs; diff --git a/tcg/tcg-op-gvec.c b/tcg/tcg-op-gvec.c index f51bcaa..95a588d 100644 --- a/tcg/tcg-op-gvec.c +++ b/tcg/tcg-op-gvec.c @@ -20,8 +20,8 @@ #include "qemu/osdep.h" #include "tcg/tcg.h" #include "tcg/tcg-temp-internal.h" -#include "tcg/tcg-op.h" -#include "tcg/tcg-op-gvec.h" +#include "tcg/tcg-op-common.h" +#include "tcg/tcg-op-gvec-common.h" #include "tcg/tcg-gvec-desc.h" #define MAX_UNROLL 4 diff --git a/tcg/tcg-op-ldst.c b/tcg/tcg-op-ldst.c index 3d27c34..a4f51bf 100644 --- a/tcg/tcg-op-ldst.c +++ b/tcg/tcg-op-ldst.c @@ -23,20 +23,32 @@ */ #include "qemu/osdep.h" -#include "exec/exec-all.h" #include "tcg/tcg.h" #include "tcg/tcg-temp-internal.h" -#include "tcg/tcg-op.h" +#include "tcg/tcg-op-common.h" #include "tcg/tcg-mo.h" +#include "exec/translation-block.h" #include "exec/plugin-gen.h" #include "tcg-internal.h" -static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) +static void check_max_alignment(unsigned a_bits) +{ +#if defined(CONFIG_SOFTMMU) + /* + * The requested alignment cannot overlap the TLB flags. + * FIXME: Must keep the count up-to-date with "exec/cpu-all.h". + */ + tcg_debug_assert(a_bits + 6 <= tcg_ctx->page_bits); +#endif +} + +static MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) { - /* Trigger the asserts within as early as possible. */ unsigned a_bits = get_alignment_bits(op); + check_max_alignment(a_bits); + /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */ if (a_bits == (op & MO_SIZE)) { op = (op & ~MO_AMASK) | MO_ALIGN; @@ -104,9 +116,7 @@ static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 v, TCGTemp *addr, MemOpIdx oi) static void tcg_gen_req_mo(TCGBar type) { -#ifdef TCG_GUEST_DEFAULT_MO - type &= TCG_GUEST_DEFAULT_MO; -#endif + type &= tcg_ctx->guest_mo; type &= ~TCG_TARGET_DEFAULT_MO; if (type) { tcg_gen_mb(type | TCG_BAR_SC); @@ -493,6 +503,7 @@ static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, TCGv_i64 ext_addr = NULL; TCGOpcode opc; + check_max_alignment(get_alignment_bits(memop)); tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); /* TODO: For now, force 32-bit hosts to use the helper. */ @@ -601,6 +612,7 @@ static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, TCGv_i64 ext_addr = NULL; TCGOpcode opc; + check_max_alignment(get_alignment_bits(memop)); tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST); /* TODO: For now, force 32-bit hosts to use the helper. */ diff --git a/tcg/tcg-op-vec.c b/tcg/tcg-op-vec.c index aeeb243..64bc8a2 100644 --- a/tcg/tcg-op-vec.c +++ b/tcg/tcg-op-vec.c @@ -20,21 +20,10 @@ #include "qemu/osdep.h" #include "tcg/tcg.h" #include "tcg/tcg-temp-internal.h" -#include "tcg/tcg-op.h" +#include "tcg/tcg-op-common.h" #include "tcg/tcg-mo.h" #include "tcg-internal.h" - -/* Reduce the number of ifdefs below. This assumes that all uses of - TCGV_HIGH and TCGV_LOW are properly protected by a conditional that - the compiler can eliminate. */ -#if TCG_TARGET_REG_BITS == 64 -extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); -extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); -#define TCGV_LOW TCGV_LOW_link_error -#define TCGV_HIGH TCGV_HIGH_link_error -#endif - /* * Vector optional opcode tracking. * Except for the basic logical operations (and, or, xor), and diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index edbd1c6..c07de5d 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -23,10 +23,10 @@ */ #include "qemu/osdep.h" -#include "exec/exec-all.h" #include "tcg/tcg.h" #include "tcg/tcg-temp-internal.h" -#include "tcg/tcg-op.h" +#include "tcg/tcg-op-common.h" +#include "exec/translation-block.h" #include "exec/plugin-gen.h" #include "tcg-internal.h" @@ -34,14 +34,9 @@ #include "qemu/cacheflush.h" #include "qemu/cacheinfo.h" #include "qemu/timer.h" - -/* Note: the long term plan is to reduce the dependencies on the QEMU - CPU definitions. Currently they are used for qemu_ld/st - instructions */ -#define NO_CPU_IO_DEFS - -#include "exec/exec-all.h" -#include "tcg/tcg-op.h" +#include "exec/translation-block.h" +#include "exec/tlb-common.h" +#include "tcg/tcg-op-common.h" #if UINTPTR_MAX == UINT32_MAX # define ELF_CLASS ELFCLASS32 @@ -407,6 +402,13 @@ static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); } +#if defined(CONFIG_SOFTMMU) && !defined(CONFIG_TCG_INTERPRETER) +static int tlb_mask_table_ofs(TCGContext *s, int which) +{ + return s->tlb_fast_offset + which * sizeof(CPUTLBDescFast); +} +#endif + /* Signal overflow, starting over with fewer guest insns. */ static G_NORETURN void tcg_raise_tb_overflow(TCGContext *s) @@ -840,13 +842,6 @@ void tcg_pool_reset(TCGContext *s) s->pool_current = NULL; } -#include "exec/helper-proto.h" - -static TCGHelperInfo all_helpers[] = { -#include "exec/helper-tcg.h" -}; -static GHashTable *helper_table; - /* * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions, * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N. @@ -956,57 +951,45 @@ static ffi_type *typecode_to_ffi(int argmask) g_assert_not_reached(); } -static void init_ffi_layouts(void) +static ffi_cif *init_ffi_layout(TCGHelperInfo *info) { - /* g_direct_hash/equal for direct comparisons on uint32_t. */ - GHashTable *ffi_table = g_hash_table_new(NULL, NULL); - - for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) { - TCGHelperInfo *info = &all_helpers[i]; - unsigned typemask = info->typemask; - gpointer hash = (gpointer)(uintptr_t)typemask; - struct { - ffi_cif cif; - ffi_type *args[]; - } *ca; - ffi_status status; - int nargs; - ffi_cif *cif; - - cif = g_hash_table_lookup(ffi_table, hash); - if (cif) { - info->cif = cif; - continue; - } - - /* Ignoring the return type, find the last non-zero field. */ - nargs = 32 - clz32(typemask >> 3); - nargs = DIV_ROUND_UP(nargs, 3); - assert(nargs <= MAX_CALL_IARGS); - - ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); - ca->cif.rtype = typecode_to_ffi(typemask & 7); - ca->cif.nargs = nargs; - - if (nargs != 0) { - ca->cif.arg_types = ca->args; - for (int j = 0; j < nargs; ++j) { - int typecode = extract32(typemask, (j + 1) * 3, 3); - ca->args[j] = typecode_to_ffi(typecode); - } + unsigned typemask = info->typemask; + struct { + ffi_cif cif; + ffi_type *args[]; + } *ca; + ffi_status status; + int nargs; + + /* Ignoring the return type, find the last non-zero field. */ + nargs = 32 - clz32(typemask >> 3); + nargs = DIV_ROUND_UP(nargs, 3); + assert(nargs <= MAX_CALL_IARGS); + + ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); + ca->cif.rtype = typecode_to_ffi(typemask & 7); + ca->cif.nargs = nargs; + + if (nargs != 0) { + ca->cif.arg_types = ca->args; + for (int j = 0; j < nargs; ++j) { + int typecode = extract32(typemask, (j + 1) * 3, 3); + ca->args[j] = typecode_to_ffi(typecode); } - - status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, - ca->cif.rtype, ca->cif.arg_types); - assert(status == FFI_OK); - - cif = &ca->cif; - info->cif = cif; - g_hash_table_insert(ffi_table, hash, (gpointer)cif); } - g_hash_table_destroy(ffi_table); + status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, + ca->cif.rtype, ca->cif.arg_types); + assert(status == FFI_OK); + + return &ca->cif; } + +#define HELPER_INFO_INIT(I) (&(I)->cif) +#define HELPER_INFO_INIT_VAL(I) init_ffi_layout(I) +#else +#define HELPER_INFO_INIT(I) (&(I)->init) +#define HELPER_INFO_INIT_VAL(I) 1 #endif /* CONFIG_TCG_INTERPRETER */ static inline bool arg_slot_reg_p(unsigned arg_slot) @@ -1319,16 +1302,6 @@ static void tcg_context_init(unsigned max_cpus) args_ct += n; } - /* Register helpers. */ - /* Use g_direct_hash/equal for direct pointer comparisons on func. */ - helper_table = g_hash_table_new(NULL, NULL); - - for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { - init_call_layout(&all_helpers[i]); - g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, - (gpointer)&all_helpers[i]); - } - init_call_layout(&info_helper_ld32_mmu); init_call_layout(&info_helper_ld64_mmu); init_call_layout(&info_helper_ld128_mmu); @@ -1336,10 +1309,6 @@ static void tcg_context_init(unsigned max_cpus) init_call_layout(&info_helper_st64_mmu); init_call_layout(&info_helper_st128_mmu); -#ifdef CONFIG_TCG_INTERPRETER - init_ffi_layouts(); -#endif - tcg_target_init(s); process_op_defs(s); @@ -1521,6 +1490,13 @@ void tcg_func_start(TCGContext *s) tcg_debug_assert(s->addr_type == TCG_TYPE_I32 || s->addr_type == TCG_TYPE_I64); + +#if defined(CONFIG_SOFTMMU) && !defined(CONFIG_TCG_INTERPRETER) + tcg_debug_assert(s->tlb_fast_offset < 0); + tcg_debug_assert(s->tlb_fast_offset >= MIN_TLB_MASK_TABLE_OFS); +#endif + + tcg_debug_assert(s->insn_start_words > 0); } static TCGTemp *tcg_temp_alloc(TCGContext *s) @@ -1820,6 +1796,25 @@ TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) return tcg_constant_vec(t->base_type, vece, val); } +#ifdef CONFIG_DEBUG_TCG +size_t temp_idx(TCGTemp *ts) +{ + ptrdiff_t n = ts - tcg_ctx->temps; + assert(n >= 0 && n < tcg_ctx->nb_temps); + return n; +} + +TCGTemp *tcgv_i32_temp(TCGv_i32 v) +{ + uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps); + + assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps); + assert(o % sizeof(TCGTemp) == 0); + + return (void *)tcg_ctx + (uintptr_t)v; +} +#endif /* CONFIG_DEBUG_TCG */ + /* Return true if OP may appear in the opcode stream. Test the runtime variable that controls each opcode. */ bool tcg_op_supported(TCGOpcode op) @@ -2128,15 +2123,18 @@ bool tcg_op_supported(TCGOpcode op) static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs); -void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) +static void tcg_gen_callN(TCGHelperInfo *info, TCGTemp *ret, TCGTemp **args) { - const TCGHelperInfo *info; TCGv_i64 extend_free[MAX_CALL_IARGS]; int n_extend = 0; TCGOp *op; int i, n, pi = 0, total_args; - info = g_hash_table_lookup(helper_table, (gpointer)func); + if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) { + init_call_layout(info); + g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info)); + } + total_args = info->nr_out + info->nr_in + 2; op = tcg_op_alloc(INDEX_op_call, total_args); @@ -2203,7 +2201,7 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) g_assert_not_reached(); } } - op->args[pi++] = (uintptr_t)func; + op->args[pi++] = (uintptr_t)info->func; op->args[pi++] = (uintptr_t)info; tcg_debug_assert(pi == total_args); @@ -2215,6 +2213,58 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) } } +void tcg_gen_call0(TCGHelperInfo *info, TCGTemp *ret) +{ + tcg_gen_callN(info, ret, NULL); +} + +void tcg_gen_call1(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1) +{ + tcg_gen_callN(info, ret, &t1); +} + +void tcg_gen_call2(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, TCGTemp *t2) +{ + TCGTemp *args[2] = { t1, t2 }; + tcg_gen_callN(info, ret, args); +} + +void tcg_gen_call3(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, + TCGTemp *t2, TCGTemp *t3) +{ + TCGTemp *args[3] = { t1, t2, t3 }; + tcg_gen_callN(info, ret, args); +} + +void tcg_gen_call4(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, + TCGTemp *t2, TCGTemp *t3, TCGTemp *t4) +{ + TCGTemp *args[4] = { t1, t2, t3, t4 }; + tcg_gen_callN(info, ret, args); +} + +void tcg_gen_call5(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, + TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5) +{ + TCGTemp *args[5] = { t1, t2, t3, t4, t5 }; + tcg_gen_callN(info, ret, args); +} + +void tcg_gen_call6(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, TCGTemp *t2, + TCGTemp *t3, TCGTemp *t4, TCGTemp *t5, TCGTemp *t6) +{ + TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 }; + tcg_gen_callN(info, ret, args); +} + +void tcg_gen_call7(TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, + TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, + TCGTemp *t5, TCGTemp *t6, TCGTemp *t7) +{ + TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 }; + tcg_gen_callN(info, ret, args); +} + static void tcg_reg_alloc_start(TCGContext *s) { int i, n; @@ -2391,7 +2441,7 @@ static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) nb_oargs = 0; col += ne_fprintf(f, "\n ----"); - for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { + for (i = 0, k = s->insn_start_words; i < k; ++i) { col += ne_fprintf(f, " %016" PRIx64, tcg_get_insn_start_param(op, i)); } @@ -5970,7 +6020,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) #ifdef CONFIG_PROFILER TCGProfile *prof = &s->prof; #endif - int i, num_insns; + int i, start_words, num_insns; TCGOp *op; #ifdef CONFIG_PROFILER @@ -6093,6 +6143,10 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) s->pool_labels = NULL; #endif + start_words = s->insn_start_words; + s->gen_insn_data = + tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * start_words); + num_insns = -1; QTAILQ_FOREACH(op, &s->ops, link) { TCGOpcode opc = op->opc; @@ -6118,8 +6172,8 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) assert(s->gen_insn_end_off[num_insns] == off); } num_insns++; - for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { - s->gen_insn_data[num_insns][i] = + for (i = 0; i < start_words; ++i) { + s->gen_insn_data[num_insns * start_words + i] = tcg_get_insn_start_param(op, i); } break; @@ -6165,7 +6219,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) return -2; } } - tcg_debug_assert(num_insns >= 0); + tcg_debug_assert(num_insns + 1 == s->gen_tb->icount); s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); /* Generate TB finalization at the end of block */ @@ -18,8 +18,7 @@ */ #include "qemu/osdep.h" -#include "exec/cpu_ldst.h" -#include "tcg/tcg-op.h" +#include "tcg/tcg.h" #include "tcg/tcg-ldst.h" #include <ffi.h> diff --git a/tcg/tci/tcg-target-reg-bits.h b/tcg/tci/tcg-target-reg-bits.h new file mode 100644 index 0000000..dcb1a20 --- /dev/null +++ b/tcg/tci/tcg-target-reg-bits.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Define target-specific register size + * Copyright (c) 2009, 2011 Stefan Weil + */ + +#ifndef TCG_TARGET_REG_BITS_H +#define TCG_TARGET_REG_BITS_H + +#if UINTPTR_MAX == UINT32_MAX +# define TCG_TARGET_REG_BITS 32 +#elif UINTPTR_MAX == UINT64_MAX +# define TCG_TARGET_REG_BITS 64 +#else +# error Unknown pointer size for tci target +#endif + +#endif diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index 60a6ed6..37ee10c 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -44,14 +44,6 @@ #define TCG_TARGET_INSN_UNIT_SIZE 4 #define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) -#if UINTPTR_MAX == UINT32_MAX -# define TCG_TARGET_REG_BITS 32 -#elif UINTPTR_MAX == UINT64_MAX -# define TCG_TARGET_REG_BITS 64 -#else -# error Unknown pointer size for tci target -#endif - /* Optional instructions. */ #define TCG_TARGET_HAS_bswap16_i32 1 |