aboutsummaryrefslogtreecommitdiff
path: root/tcg
diff options
context:
space:
mode:
Diffstat (limited to 'tcg')
-rw-r--r--tcg/README5
-rw-r--r--tcg/aarch64/tcg-target.c.inc75
-rw-r--r--tcg/aarch64/tcg-target.h3
-rw-r--r--tcg/arm/tcg-target.c.inc41
-rw-r--r--tcg/arm/tcg-target.h3
-rw-r--r--tcg/i386/tcg-target.c.inc174
-rw-r--r--tcg/i386/tcg-target.h12
-rw-r--r--tcg/mips/tcg-target.c.inc97
-rw-r--r--tcg/mips/tcg-target.h3
-rw-r--r--tcg/optimize.c1
-rw-r--r--tcg/ppc/tcg-target.c.inc88
-rw-r--r--tcg/ppc/tcg-target.h3
-rw-r--r--tcg/riscv/tcg-target.c.inc125
-rw-r--r--tcg/riscv/tcg-target.h3
-rw-r--r--tcg/s390/tcg-target.c.inc91
-rw-r--r--tcg/s390/tcg-target.h9
-rw-r--r--tcg/sparc/tcg-target.c.inc58
-rw-r--r--tcg/sparc/tcg-target.h3
-rw-r--r--tcg/tcg-ldst.c.inc2
-rw-r--r--tcg/tcg-op.c23
-rw-r--r--tcg/tcg-pool.c.inc6
-rw-r--r--tcg/tcg.c94
-rw-r--r--tcg/tci.c62
-rw-r--r--tcg/tci/tcg-target.c.inc2
-rw-r--r--tcg/tci/tcg-target.h7
25 files changed, 514 insertions, 476 deletions
diff --git a/tcg/README b/tcg/README
index 2f051e5..0cf9e27 100644
--- a/tcg/README
+++ b/tcg/README
@@ -502,6 +502,7 @@ goto_ptr opcode, emitting this op is equivalent to emitting exit_tb(0).
* qemu_ld_i32/i64 t0, t1, flags, memidx
* qemu_st_i32/i64 t0, t1, flags, memidx
+* qemu_st8_i32 t0, t1, flags, memidx
Load data at the guest address t1 into t0, or store data in t0 at guest
address t1. The _i32/_i64 size applies to the size of the input/output
@@ -518,6 +519,10 @@ of the memory access.
For a 32-bit host, qemu_ld/st_i64 is guaranteed to only be used with a
64-bit memory access specified in flags.
+For i386, qemu_st8_i32 is exactly like qemu_st_i32, except the size of
+the memory operation is known to be 8-bit. This allows the backend to
+provide a different set of register constraints.
+
********* Host vector operations
All of the vector ops have two parameters, TCGOP_VECL & TCGOP_VECE.
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index 26f71cb..ab199b1 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -78,38 +78,42 @@ static const int tcg_target_call_oarg_regs[1] = {
#define TCG_REG_GUEST_BASE TCG_REG_X28
#endif
-static inline bool reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static bool reloc_pc26(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
{
- ptrdiff_t offset = target - code_ptr;
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ ptrdiff_t offset = target - src_rx;
+
if (offset == sextract64(offset, 0, 26)) {
/* read instruction, mask away previous PC_REL26 parameter contents,
set the proper offset, then write back the instruction. */
- *code_ptr = deposit32(*code_ptr, 0, 26, offset);
+ *src_rw = deposit32(*src_rw, 0, 26, offset);
return true;
}
return false;
}
-static inline bool reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static bool reloc_pc19(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
{
- ptrdiff_t offset = target - code_ptr;
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ ptrdiff_t offset = target - src_rx;
+
if (offset == sextract64(offset, 0, 19)) {
- *code_ptr = deposit32(*code_ptr, 5, 19, offset);
+ *src_rw = deposit32(*src_rw, 5, 19, offset);
return true;
}
return false;
}
-static inline bool patch_reloc(tcg_insn_unit *code_ptr, int type,
- intptr_t value, intptr_t addend)
+static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+ intptr_t value, intptr_t addend)
{
tcg_debug_assert(addend == 0);
switch (type) {
case R_AARCH64_JUMP26:
case R_AARCH64_CALL26:
- return reloc_pc26(code_ptr, (tcg_insn_unit *)value);
+ return reloc_pc26(code_ptr, (const tcg_insn_unit *)value);
case R_AARCH64_CONDBR19:
- return reloc_pc19(code_ptr, (tcg_insn_unit *)value);
+ return reloc_pc19(code_ptr, (const tcg_insn_unit *)value);
default:
g_assert_not_reached();
}
@@ -1050,12 +1054,13 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
/* Look for host pointer values within 4G of the PC. This happens
often when loading pointers to QEMU's own data structures. */
if (type == TCG_TYPE_I64) {
- tcg_target_long disp = value - (intptr_t)s->code_ptr;
+ intptr_t src_rx = (intptr_t)tcg_splitwx_to_rx(s->code_ptr);
+ tcg_target_long disp = value - src_rx;
if (disp == sextract64(disp, 0, 21)) {
tcg_out_insn(s, 3406, ADR, rd, disp);
return;
}
- disp = (value >> 12) - ((intptr_t)s->code_ptr >> 12);
+ disp = (value >> 12) - (src_rx >> 12);
if (disp == sextract64(disp, 0, 21)) {
tcg_out_insn(s, 3406, ADRP, rd, disp);
if (value & 0xfff) {
@@ -1306,18 +1311,18 @@ static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a,
}
}
-static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target)
+static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target)
{
- ptrdiff_t offset = target - s->code_ptr;
+ ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2;
tcg_debug_assert(offset == sextract64(offset, 0, 26));
tcg_out_insn(s, 3206, B, offset);
}
-static inline void tcg_out_goto_long(TCGContext *s, tcg_insn_unit *target)
+static void tcg_out_goto_long(TCGContext *s, const tcg_insn_unit *target)
{
- ptrdiff_t offset = target - s->code_ptr;
+ ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2;
if (offset == sextract64(offset, 0, 26)) {
- tcg_out_insn(s, 3206, BL, offset);
+ tcg_out_insn(s, 3206, B, offset);
} else {
tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target);
tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
@@ -1329,9 +1334,9 @@ static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
tcg_out_insn(s, 3207, BLR, reg);
}
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target)
{
- ptrdiff_t offset = target - s->code_ptr;
+ ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2;
if (offset == sextract64(offset, 0, 26)) {
tcg_out_insn(s, 3206, BL, offset);
} else {
@@ -1340,21 +1345,21 @@ static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
}
}
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
- uintptr_t addr)
+void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+ uintptr_t jmp_rw, uintptr_t addr)
{
tcg_insn_unit i1, i2;
TCGType rt = TCG_TYPE_I64;
TCGReg rd = TCG_REG_TMP;
uint64_t pair;
- ptrdiff_t offset = addr - jmp_addr;
+ ptrdiff_t offset = addr - jmp_rx;
if (offset == sextract64(offset, 0, 26)) {
i1 = I3206_B | ((offset >> 2) & 0x3ffffff);
i2 = NOP;
} else {
- offset = (addr >> 12) - (jmp_addr >> 12);
+ offset = (addr >> 12) - (jmp_rx >> 12);
/* patch ADRP */
i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd;
@@ -1362,8 +1367,8 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd;
}
pair = (uint64_t)i2 << 32 | i1;
- qatomic_set((uint64_t *)jmp_addr, pair);
- flush_icache_range(jmp_addr, jmp_addr + 8);
+ qatomic_set((uint64_t *)jmp_rw, pair);
+ flush_idcache_range(jmp_rx, jmp_rw, 8);
}
static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l)
@@ -1393,7 +1398,7 @@ static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a,
tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0);
offset = tcg_in32(s) >> 5;
} else {
- offset = l->u.value_ptr - s->code_ptr;
+ offset = tcg_pcrel_diff(s, l->u.value_ptr) >> 2;
tcg_debug_assert(offset == sextract64(offset, 0, 19));
}
@@ -1568,7 +1573,7 @@ static void * const qemu_st_helpers[16] = {
[MO_BEQ] = helper_be_stq_mmu,
};
-static inline void tcg_out_adr(TCGContext *s, TCGReg rd, void *target)
+static inline void tcg_out_adr(TCGContext *s, TCGReg rd, const void *target)
{
ptrdiff_t offset = tcg_pcrel_diff(s, target);
tcg_debug_assert(offset == sextract64(offset, 0, 21));
@@ -1581,7 +1586,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp opc = get_memop(oi);
MemOp size = opc & MO_SIZE;
- if (!reloc_pc19(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc19(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
return false;
}
@@ -1606,7 +1611,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp opc = get_memop(oi);
MemOp size = opc & MO_SIZE;
- if (!reloc_pc19(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc19(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
return false;
}
@@ -1631,7 +1636,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
label->type = ext;
label->datalo_reg = data_reg;
label->addrlo_reg = addr_reg;
- label->raddr = raddr;
+ label->raddr = tcg_splitwx_to_rx(raddr);
label->label_ptr[0] = label_ptr;
}
@@ -1849,7 +1854,7 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
#endif /* CONFIG_SOFTMMU */
}
-static tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *tb_ret_addr;
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg args[TCG_MAX_OP_ARGS],
@@ -1873,7 +1878,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_exit_tb:
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
- tcg_out_goto_long(s, s->code_gen_epilogue);
+ tcg_out_goto_long(s, tcg_code_gen_epilogue);
} else {
tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
tcg_out_goto_long(s, tb_ret_addr);
@@ -2894,11 +2899,11 @@ static void tcg_target_qemu_prologue(TCGContext *s)
* Return path for goto_ptr. Set return value to 0, a-la exit_tb,
* and fall through to the rest of the epilogue.
*/
- s->code_gen_epilogue = s->code_ptr;
+ tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_X0, 0);
/* TB epilogue */
- tb_ret_addr = s->code_ptr;
+ tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
/* Remove TCG locals stack space. */
tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
@@ -2964,7 +2969,7 @@ static const DebugFrame debug_frame = {
}
};
-void tcg_register_jit(void *buf, size_t buf_size)
+void tcg_register_jit(const void *buf, size_t buf_size)
{
tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
}
diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index 8a6b975..5ec30db 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -88,6 +88,7 @@ typedef enum {
#define TCG_TARGET_HAS_extrl_i64_i32 0
#define TCG_TARGET_HAS_extrh_i64_i32 0
#define TCG_TARGET_HAS_goto_ptr 1
+#define TCG_TARGET_HAS_qemu_st8_i32 0
#define TCG_TARGET_HAS_div_i64 1
#define TCG_TARGET_HAS_rem_i64 1
@@ -148,7 +149,7 @@ typedef enum {
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#ifdef CONFIG_SOFTMMU
#define TCG_TARGET_NEED_LDST_LABELS
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index 62c37a9..0fd1126 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -187,29 +187,32 @@ static const uint8_t tcg_cond_to_arm_cond[] = {
[TCG_COND_GTU] = COND_HI,
};
-static inline bool reloc_pc24(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static bool reloc_pc24(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
{
- ptrdiff_t offset = (tcg_ptr_byte_diff(target, code_ptr) - 8) >> 2;
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ ptrdiff_t offset = (tcg_ptr_byte_diff(target, src_rx) - 8) >> 2;
+
if (offset == sextract32(offset, 0, 24)) {
- *code_ptr = (*code_ptr & ~0xffffff) | (offset & 0xffffff);
+ *src_rw = deposit32(*src_rw, 0, 24, offset);
return true;
}
return false;
}
-static inline bool reloc_pc13(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static bool reloc_pc13(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
{
- ptrdiff_t offset = tcg_ptr_byte_diff(target, code_ptr) - 8;
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ ptrdiff_t offset = tcg_ptr_byte_diff(target, src_rx) - 8;
if (offset >= -0xfff && offset <= 0xfff) {
- tcg_insn_unit insn = *code_ptr;
+ tcg_insn_unit insn = *src_rw;
bool u = (offset >= 0);
if (!u) {
offset = -offset;
}
insn = deposit32(insn, 23, 1, u);
insn = deposit32(insn, 0, 12, offset);
- *code_ptr = insn;
+ *src_rw = insn;
return true;
}
return false;
@@ -221,9 +224,9 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
tcg_debug_assert(addend == 0);
if (type == R_ARM_PC24) {
- return reloc_pc24(code_ptr, (tcg_insn_unit *)value);
+ return reloc_pc24(code_ptr, (const tcg_insn_unit *)value);
} else if (type == R_ARM_PC13) {
- return reloc_pc13(code_ptr, (tcg_insn_unit *)value);
+ return reloc_pc13(code_ptr, (const tcg_insn_unit *)value);
} else {
g_assert_not_reached();
}
@@ -617,7 +620,7 @@ static void tcg_out_movi32(TCGContext *s, int cond, int rd, uint32_t arg)
/* Check for a pc-relative address. This will usually be the TB,
or within the TB, which is immediately before the code block. */
- diff = arg - ((intptr_t)s->code_ptr + 8);
+ diff = tcg_pcrel_diff(s, (void *)arg) - 8;
if (diff >= 0) {
rot = encode_imm(diff);
if (rot >= 0) {
@@ -1019,7 +1022,7 @@ static inline void tcg_out_st8(TCGContext *s, int cond,
* with the code buffer limited to 16MB we wouldn't need the long case.
* But we also use it for the tail-call to the qemu_ld/st helpers, which does.
*/
-static void tcg_out_goto(TCGContext *s, int cond, tcg_insn_unit *addr)
+static void tcg_out_goto(TCGContext *s, int cond, const tcg_insn_unit *addr)
{
intptr_t addri = (intptr_t)addr;
ptrdiff_t disp = tcg_pcrel_diff(s, addr);
@@ -1033,7 +1036,7 @@ static void tcg_out_goto(TCGContext *s, int cond, tcg_insn_unit *addr)
/* The call case is mostly used for helpers - so it's not unreasonable
* for them to be beyond branch range */
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *addr)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *addr)
{
intptr_t addri = (intptr_t)addr;
ptrdiff_t disp = tcg_pcrel_diff(s, addr);
@@ -1337,7 +1340,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
label->datahi_reg = datahi;
label->addrlo_reg = addrlo;
label->addrhi_reg = addrhi;
- label->raddr = raddr;
+ label->raddr = tcg_splitwx_to_rx(raddr);
label->label_ptr[0] = label_ptr;
}
@@ -1348,7 +1351,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp opc = get_memop(oi);
void *func;
- if (!reloc_pc24(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc24(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
return false;
}
@@ -1411,7 +1414,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
TCGMemOpIdx oi = lb->oi;
MemOp opc = get_memop(oi);
- if (!reloc_pc24(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc24(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
return false;
}
@@ -1762,8 +1765,8 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
TCGReg base = TCG_REG_PC;
tcg_debug_assert(s->tb_jmp_insn_offset == 0);
- ptr = (intptr_t)(s->tb_jmp_target_addr + args[0]);
- dif = ptr - ((intptr_t)s->code_ptr + 8);
+ ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]);
+ dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
dil = sextract32(dif, 0, 12);
if (dif != dil) {
/* The TB is close, but outside the 12 bits addressable by
@@ -2297,7 +2300,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
* Return path for goto_ptr. Set return value to 0, a-la exit_tb,
* and fall through to the rest of the epilogue.
*/
- s->code_gen_epilogue = s->code_ptr;
+ tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, 0);
tcg_out_epilogue(s);
}
@@ -2353,7 +2356,7 @@ static const DebugFrame debug_frame = {
}
};
-void tcg_register_jit(void *buf, size_t buf_size)
+void tcg_register_jit(const void *buf, size_t buf_size)
{
tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
}
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index f1955ce..8d1fee6 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -126,6 +126,7 @@ extern bool use_idiv_instructions;
#define TCG_TARGET_HAS_rem_i32 0
#define TCG_TARGET_HAS_goto_ptr 1
#define TCG_TARGET_HAS_direct_jump 0
+#define TCG_TARGET_HAS_qemu_st8_i32 0
enum {
TCG_AREG0 = TCG_REG_R6,
@@ -135,7 +136,7 @@ enum {
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
/* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#ifdef CONFIG_SOFTMMU
#define TCG_TARGET_NEED_LDST_LABELS
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index d8797ed..46e856f 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -154,18 +154,17 @@ bool have_bmi1;
bool have_popcnt;
bool have_avx1;
bool have_avx2;
+bool have_movbe;
#ifdef CONFIG_CPUID_H
-static bool have_movbe;
static bool have_bmi2;
static bool have_lzcnt;
#else
-# define have_movbe 0
# define have_bmi2 0
# define have_lzcnt 0
#endif
-static tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *tb_ret_addr;
static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
@@ -173,7 +172,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
value += addend;
switch(type) {
case R_386_PC32:
- value -= (uintptr_t)code_ptr;
+ value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr);
if (value != (int32_t)value) {
return false;
}
@@ -182,7 +181,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
tcg_patch32(code_ptr, value);
break;
case R_386_PC8:
- value -= (uintptr_t)code_ptr;
+ value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr);
if (value != (int8_t)value) {
return false;
}
@@ -246,11 +245,21 @@ static const char *target_parse_constraint(TCGArgConstraint *ct,
ct->regs |= ALL_VECTOR_REGS;
break;
- /* qemu_ld/st address constraint */
case 'L':
+ /* qemu_ld/st data+address constraint */
ct->regs = TCG_TARGET_REG_BITS == 64 ? 0xffff : 0xff;
+#ifdef CONFIG_SOFTMMU
tcg_regset_reset_reg(ct->regs, TCG_REG_L0);
tcg_regset_reset_reg(ct->regs, TCG_REG_L1);
+#endif
+ break;
+ case 's':
+ /* qemu_st8_i32 data constraint */
+ ct->regs = 0xf;
+#ifdef CONFIG_SOFTMMU
+ tcg_regset_reset_reg(ct->regs, TCG_REG_L0);
+ tcg_regset_reset_reg(ct->regs, TCG_REG_L1);
+#endif
break;
case 'e':
@@ -1006,7 +1015,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type,
}
/* Try a 7 byte pc-relative lea before the 10 byte movq. */
- diff = arg - ((uintptr_t)s->code_ptr + 7);
+ diff = tcg_pcrel_diff(s, (const void *)arg) - 7;
if (diff == (int32_t)diff) {
tcg_out_opc(s, OPC_LEA | P_REXW, ret, 0, 0);
tcg_out8(s, (LOWREGMASK(ret) << 3) | 5);
@@ -1452,7 +1461,7 @@ static void tcg_out_brcond2(TCGContext *s, const TCGArg *args,
default:
tcg_abort();
}
- tcg_out_label(s, label_next, s->code_ptr);
+ tcg_out_label(s, label_next);
}
#endif
@@ -1494,10 +1503,10 @@ static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
tcg_out_jxx(s, JCC_JMP, label_over, 1);
- tcg_out_label(s, label_true, s->code_ptr);
+ tcg_out_label(s, label_true);
tcg_out_movi(s, TCG_TYPE_I32, args[0], 1);
- tcg_out_label(s, label_over, s->code_ptr);
+ tcg_out_label(s, label_over);
} else {
/* When the destination does not overlap one of the arguments,
clear the destination first, jump if cond false, and emit an
@@ -1511,7 +1520,7 @@ static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
tcg_out_brcond2(s, new_args, const_args+1, 1);
tgen_arithi(s, ARITH_ADD, args[0], 1, 0);
- tcg_out_label(s, label_over, s->code_ptr);
+ tcg_out_label(s, label_over);
}
}
#endif
@@ -1525,7 +1534,7 @@ static void tcg_out_cmov(TCGContext *s, TCGCond cond, int rexw,
TCGLabel *over = gen_new_label();
tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1);
tcg_out_mov(s, TCG_TYPE_I32, dest, v1);
- tcg_out_label(s, over, s->code_ptr);
+ tcg_out_label(s, over);
}
}
@@ -1591,7 +1600,7 @@ static void tcg_out_clz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1,
}
}
-static void tcg_out_branch(TCGContext *s, int call, tcg_insn_unit *dest)
+static void tcg_out_branch(TCGContext *s, int call, const tcg_insn_unit *dest)
{
intptr_t disp = tcg_pcrel_diff(s, dest) - 5;
@@ -1610,12 +1619,12 @@ static void tcg_out_branch(TCGContext *s, int call, tcg_insn_unit *dest)
}
}
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
+static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest)
{
tcg_out_branch(s, 1, dest);
}
-static void tcg_out_jmp(TCGContext *s, tcg_insn_unit *dest)
+static void tcg_out_jmp(TCGContext *s, const tcg_insn_unit *dest)
{
tcg_out_branch(s, 0, dest);
}
@@ -1786,7 +1795,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64,
label->datahi_reg = datahi;
label->addrlo_reg = addrlo;
label->addrhi_reg = addrhi;
- label->raddr = raddr;
+ label->raddr = tcg_splitwx_to_rx(raddr);
label->label_ptr[0] = label_ptr[0];
if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
label->label_ptr[1] = label_ptr[1];
@@ -1986,13 +1995,14 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
TCGReg base, int index, intptr_t ofs,
int seg, bool is64, MemOp memop)
{
- const MemOp real_bswap = memop & MO_BSWAP;
- MemOp bswap = real_bswap;
+ bool use_movbe = false;
int rexw = is64 * P_REXW;
int movop = OPC_MOVL_GvEv;
- if (have_movbe && real_bswap) {
- bswap = 0;
+ /* Do big-endian loads with movbe. */
+ if (memop & MO_BSWAP) {
+ tcg_debug_assert(have_movbe);
+ use_movbe = true;
movop = OPC_MOVBE_GyMy;
}
@@ -2006,23 +2016,28 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
base, index, 0, ofs);
break;
case MO_UW:
- tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo,
- base, index, 0, ofs);
- if (real_bswap) {
- tcg_out_rolw_8(s, datalo);
- }
- break;
- case MO_SW:
- if (real_bswap) {
- if (have_movbe) {
+ if (use_movbe) {
+ /* There is no extending movbe; only low 16-bits are modified. */
+ if (datalo != base && datalo != index) {
+ /* XOR breaks dependency chains. */
+ tgen_arithr(s, ARITH_XOR, datalo, datalo);
tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
datalo, base, index, 0, ofs);
} else {
- tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo,
- base, index, 0, ofs);
- tcg_out_rolw_8(s, datalo);
+ tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
+ datalo, base, index, 0, ofs);
+ tcg_out_ext16u(s, datalo, datalo);
}
- tcg_out_modrm(s, OPC_MOVSWL + rexw, datalo, datalo);
+ } else {
+ tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo,
+ base, index, 0, ofs);
+ }
+ break;
+ case MO_SW:
+ if (use_movbe) {
+ tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
+ datalo, base, index, 0, ofs);
+ tcg_out_ext16s(s, datalo, datalo, rexw);
} else {
tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + seg,
datalo, base, index, 0, ofs);
@@ -2030,18 +2045,12 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
break;
case MO_UL:
tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs);
- if (bswap) {
- tcg_out_bswap32(s, datalo);
- }
break;
#if TCG_TARGET_REG_BITS == 64
case MO_SL:
- if (real_bswap) {
- tcg_out_modrm_sib_offset(s, movop + seg, datalo,
+ if (use_movbe) {
+ tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + seg, datalo,
base, index, 0, ofs);
- if (bswap) {
- tcg_out_bswap32(s, datalo);
- }
tcg_out_ext32s(s, datalo, datalo);
} else {
tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + seg, datalo,
@@ -2053,12 +2062,9 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
if (TCG_TARGET_REG_BITS == 64) {
tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
base, index, 0, ofs);
- if (bswap) {
- tcg_out_bswap64(s, datalo);
- }
} else {
- if (real_bswap) {
- int t = datalo;
+ if (use_movbe) {
+ TCGReg t = datalo;
datalo = datahi;
datahi = t;
}
@@ -2073,14 +2079,10 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
tcg_out_modrm_sib_offset(s, movop + seg, datalo,
base, index, 0, ofs);
}
- if (bswap) {
- tcg_out_bswap32(s, datalo);
- tcg_out_bswap32(s, datahi);
- }
}
break;
default:
- tcg_abort();
+ g_assert_not_reached();
}
}
@@ -2128,69 +2130,40 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
TCGReg base, int index, intptr_t ofs,
int seg, MemOp memop)
{
- /* ??? Ideally we wouldn't need a scratch register. For user-only,
- we could perform the bswap twice to restore the original value
- instead of moving to the scratch. But as it is, the L constraint
- means that TCG_REG_L0 is definitely free here. */
- const TCGReg scratch = TCG_REG_L0;
- const MemOp real_bswap = memop & MO_BSWAP;
- MemOp bswap = real_bswap;
+ bool use_movbe = false;
int movop = OPC_MOVL_EvGv;
- if (have_movbe && real_bswap) {
- bswap = 0;
+ /*
+ * Do big-endian stores with movbe or softmmu.
+ * User-only without movbe will have its swapping done generically.
+ */
+ if (memop & MO_BSWAP) {
+ tcg_debug_assert(have_movbe);
+ use_movbe = true;
movop = OPC_MOVBE_MyGy;
}
switch (memop & MO_SIZE) {
case MO_8:
- /* In 32-bit mode, 8-bit stores can only happen from [abcd]x.
- Use the scratch register if necessary. */
- if (TCG_TARGET_REG_BITS == 32 && datalo >= 4) {
- tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
- datalo = scratch;
- }
+ /* This is handled with constraints on INDEX_op_qemu_st8_i32. */
+ tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || datalo < 4);
tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg,
datalo, base, index, 0, ofs);
break;
case MO_16:
- if (bswap) {
- tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
- tcg_out_rolw_8(s, scratch);
- datalo = scratch;
- }
tcg_out_modrm_sib_offset(s, movop + P_DATA16 + seg, datalo,
base, index, 0, ofs);
break;
case MO_32:
- if (bswap) {
- tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
- tcg_out_bswap32(s, scratch);
- datalo = scratch;
- }
tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs);
break;
case MO_64:
if (TCG_TARGET_REG_BITS == 64) {
- if (bswap) {
- tcg_out_mov(s, TCG_TYPE_I64, scratch, datalo);
- tcg_out_bswap64(s, scratch);
- datalo = scratch;
- }
tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
base, index, 0, ofs);
- } else if (bswap) {
- tcg_out_mov(s, TCG_TYPE_I32, scratch, datahi);
- tcg_out_bswap32(s, scratch);
- tcg_out_modrm_sib_offset(s, OPC_MOVL_EvGv + seg, scratch,
- base, index, 0, ofs);
- tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
- tcg_out_bswap32(s, scratch);
- tcg_out_modrm_sib_offset(s, OPC_MOVL_EvGv + seg, scratch,
- base, index, 0, ofs + 4);
} else {
- if (real_bswap) {
- int t = datalo;
+ if (use_movbe) {
+ TCGReg t = datalo;
datalo = datahi;
datahi = t;
}
@@ -2201,7 +2174,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
}
break;
default:
- tcg_abort();
+ g_assert_not_reached();
}
}
@@ -2267,7 +2240,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_exit_tb:
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
- tcg_out_jmp(s, s->code_gen_epilogue);
+ tcg_out_jmp(s, tcg_code_gen_epilogue);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
tcg_out_jmp(s, tb_ret_addr);
@@ -2280,7 +2253,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
/* jump displacement must be aligned for atomic patching;
* see if we need to add extra nops before jump
*/
- gap = tcg_pcrel_diff(s, QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4));
+ gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
if (gap != 1) {
tcg_out_nopn(s, gap - 1);
}
@@ -2520,6 +2493,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
tcg_out_qemu_ld(s, args, 1);
break;
case INDEX_op_qemu_st_i32:
+ case INDEX_op_qemu_st8_i32:
tcg_out_qemu_st(s, args, 0);
break;
case INDEX_op_qemu_st_i64:
@@ -2978,9 +2952,11 @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op)
static const TCGTargetOpDef r_0_ci = { .args_ct_str = { "r", "0", "ci" } };
static const TCGTargetOpDef r_L = { .args_ct_str = { "r", "L" } };
static const TCGTargetOpDef L_L = { .args_ct_str = { "L", "L" } };
+ static const TCGTargetOpDef s_L = { .args_ct_str = { "s", "L" } };
static const TCGTargetOpDef r_L_L = { .args_ct_str = { "r", "L", "L" } };
static const TCGTargetOpDef r_r_L = { .args_ct_str = { "r", "r", "L" } };
static const TCGTargetOpDef L_L_L = { .args_ct_str = { "L", "L", "L" } };
+ static const TCGTargetOpDef s_L_L = { .args_ct_str = { "s", "L", "L" } };
static const TCGTargetOpDef r_r_L_L
= { .args_ct_str = { "r", "r", "L", "L" } };
static const TCGTargetOpDef L_L_L_L
@@ -3174,6 +3150,8 @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op)
return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &r_L : &r_L_L;
case INDEX_op_qemu_st_i32:
return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &L_L : &L_L_L;
+ case INDEX_op_qemu_st8_i32:
+ return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &s_L : &s_L_L;
case INDEX_op_qemu_ld_i64:
return (TCG_TARGET_REG_BITS == 64 ? &r_L
: TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &r_r_L
@@ -3825,11 +3803,11 @@ static void tcg_target_qemu_prologue(TCGContext *s)
* Return path for goto_ptr. Set return value to 0, a-la exit_tb,
* and fall through to the rest of the epilogue.
*/
- s->code_gen_epilogue = s->code_ptr;
+ tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_EAX, 0);
/* TB epilogue */
- tb_ret_addr = s->code_ptr;
+ tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
tcg_out_addi(s, TCG_REG_CALL_STACK, stack_addend);
@@ -3998,7 +3976,7 @@ static const DebugFrame debug_frame = {
#endif
#if defined(ELF_HOST_MACHINE)
-void tcg_register_jit(void *buf, size_t buf_size)
+void tcg_register_jit(const void *buf, size_t buf_size)
{
tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
}
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index cd067e0..b693d36 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -101,6 +101,7 @@ extern bool have_bmi1;
extern bool have_popcnt;
extern bool have_avx1;
extern bool have_avx2;
+extern bool have_movbe;
/* optional instructions */
#define TCG_TARGET_HAS_div2_i32 1
@@ -171,6 +172,9 @@ extern bool have_avx2;
#define TCG_TARGET_HAS_muls2_i64 1
#define TCG_TARGET_HAS_muluh_i64 0
#define TCG_TARGET_HAS_mulsh_i64 0
+#define TCG_TARGET_HAS_qemu_st8_i32 0
+#else
+#define TCG_TARGET_HAS_qemu_st8_i32 1
#endif
/* We do not support older SSE systems, only beginning with AVX1. */
@@ -206,11 +210,11 @@ extern bool have_avx2;
#define TCG_TARGET_extract_i64_valid(ofs, len) \
(((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32)
-static inline void tb_target_set_jmp_target(uintptr_t tc_ptr,
- uintptr_t jmp_addr, uintptr_t addr)
+static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+ uintptr_t jmp_rw, uintptr_t addr)
{
/* patch the branch destination */
- qatomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
+ qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4));
/* no need to flush icache explicitly */
}
@@ -225,7 +229,7 @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr,
#define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD)
-#define TCG_TARGET_HAS_MEMORY_BSWAP 1
+#define TCG_TARGET_HAS_MEMORY_BSWAP have_movbe
#ifdef CONFIG_SOFTMMU
#define TCG_TARGET_NEED_LDST_LABELS
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index 41be574..add157f 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -139,33 +139,21 @@ static const TCGReg tcg_target_call_oarg_regs[2] = {
TCG_REG_V1
};
-static tcg_insn_unit *tb_ret_addr;
-static tcg_insn_unit *bswap32_addr;
-static tcg_insn_unit *bswap32u_addr;
-static tcg_insn_unit *bswap64_addr;
+static const tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *bswap32_addr;
+static const tcg_insn_unit *bswap32u_addr;
+static const tcg_insn_unit *bswap64_addr;
-static inline uint32_t reloc_pc16_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+static bool reloc_pc16(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
{
/* Let the compiler perform the right-shift as part of the arithmetic. */
- ptrdiff_t disp = target - (pc + 1);
- tcg_debug_assert(disp == (int16_t)disp);
- return disp & 0xffff;
-}
-
-static inline void reloc_pc16(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
- *pc = deposit32(*pc, 0, 16, reloc_pc16_val(pc, target));
-}
-
-static inline uint32_t reloc_26_val(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
- tcg_debug_assert((((uintptr_t)pc ^ (uintptr_t)target) & 0xf0000000) == 0);
- return ((uintptr_t)target >> 2) & 0x3ffffff;
-}
-
-static inline void reloc_26(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
- *pc = deposit32(*pc, 0, 26, reloc_26_val(pc, target));
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ ptrdiff_t disp = target - (src_rx + 1);
+ if (disp == (int16_t)disp) {
+ *src_rw = deposit32(*src_rw, 0, 16, disp);
+ return true;
+ }
+ return false;
}
static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
@@ -173,8 +161,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
{
tcg_debug_assert(type == R_MIPS_PC16);
tcg_debug_assert(addend == 0);
- reloc_pc16(code_ptr, (tcg_insn_unit *)value);
- return true;
+ return reloc_pc16(code_ptr, (const tcg_insn_unit *)value);
}
#define TCG_CT_CONST_ZERO 0x100
@@ -516,10 +503,10 @@ static void tcg_out_opc_sa64(TCGContext *s, MIPSInsn opc1, MIPSInsn opc2,
* Type jump.
* Returns true if the branch was in range and the insn was emitted.
*/
-static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, void *target)
+static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, const void *target)
{
uintptr_t dest = (uintptr_t)target;
- uintptr_t from = (uintptr_t)s->code_ptr + 4;
+ uintptr_t from = (uintptr_t)tcg_splitwx_to_rx(s->code_ptr) + 4;
int32_t inst;
/* The pc-region branch happens within the 256MB region of
@@ -631,7 +618,7 @@ static inline void tcg_out_bswap16s(TCGContext *s, TCGReg ret, TCGReg arg)
}
}
-static void tcg_out_bswap_subr(TCGContext *s, tcg_insn_unit *sub)
+static void tcg_out_bswap_subr(TCGContext *s, const tcg_insn_unit *sub)
{
bool ok = tcg_out_opc_jmp(s, OPC_JAL, sub);
tcg_debug_assert(ok);
@@ -924,11 +911,7 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1,
}
tcg_out_opc_br(s, b_opc, arg1, arg2);
- if (l->has_value) {
- reloc_pc16(s->code_ptr - 1, l->u.value_ptr);
- } else {
- tcg_out_reloc(s, s->code_ptr - 1, R_MIPS_PC16, l, 0);
- }
+ tcg_out_reloc(s, s->code_ptr - 1, R_MIPS_PC16, l, 0);
tcg_out_nop(s);
}
@@ -1079,7 +1062,7 @@ static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
}
}
-static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
+static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail)
{
/* Note that the ABI requires the called function's address to be
loaded into T9, even if a direct branch is in range. */
@@ -1097,7 +1080,7 @@ static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
}
}
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg)
{
tcg_out_call_int(s, arg, false);
tcg_out_nop(s);
@@ -1300,7 +1283,7 @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi,
label->datahi_reg = datahi;
label->addrlo_reg = addrlo;
label->addrhi_reg = addrhi;
- label->raddr = raddr;
+ label->raddr = tcg_splitwx_to_rx(raddr);
label->label_ptr[0] = label_ptr[0];
if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
label->label_ptr[1] = label_ptr[1];
@@ -1309,15 +1292,17 @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi,
static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
{
+ const tcg_insn_unit *tgt_rx = tcg_splitwx_to_rx(s->code_ptr);
TCGMemOpIdx oi = l->oi;
MemOp opc = get_memop(oi);
TCGReg v0;
int i;
/* resolve label address */
- reloc_pc16(l->label_ptr[0], s->code_ptr);
- if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
- reloc_pc16(l->label_ptr[1], s->code_ptr);
+ if (!reloc_pc16(l->label_ptr[0], tgt_rx)
+ || (TCG_TARGET_REG_BITS < TARGET_LONG_BITS
+ && !reloc_pc16(l->label_ptr[1], tgt_rx))) {
+ return false;
}
i = 1;
@@ -1345,7 +1330,9 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
}
tcg_out_opc_br(s, OPC_BEQ, TCG_REG_ZERO, TCG_REG_ZERO);
- reloc_pc16(s->code_ptr - 1, l->raddr);
+ if (!reloc_pc16(s->code_ptr - 1, l->raddr)) {
+ return false;
+ }
/* delay slot */
if (TCG_TARGET_REG_BITS == 64 && l->type == TCG_TYPE_I32) {
@@ -1359,15 +1346,17 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
{
+ const tcg_insn_unit *tgt_rx = tcg_splitwx_to_rx(s->code_ptr);
TCGMemOpIdx oi = l->oi;
MemOp opc = get_memop(oi);
MemOp s_bits = opc & MO_SIZE;
int i;
/* resolve label address */
- reloc_pc16(l->label_ptr[0], s->code_ptr);
- if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
- reloc_pc16(l->label_ptr[1], s->code_ptr);
+ if (!reloc_pc16(l->label_ptr[0], tgt_rx)
+ || (TCG_TARGET_REG_BITS < TARGET_LONG_BITS
+ && !reloc_pc16(l->label_ptr[1], tgt_rx))) {
+ return false;
}
i = 1;
@@ -2483,11 +2472,11 @@ static void tcg_target_qemu_prologue(TCGContext *s)
* Return path for goto_ptr. Set return value to 0, a-la exit_tb,
* and fall through to the rest of the epilogue.
*/
- s->code_gen_epilogue = s->code_ptr;
+ tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_V0, TCG_REG_ZERO);
/* TB epilogue */
- tb_ret_addr = s->code_ptr;
+ tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
TCG_REG_SP, SAVE_OFS + i * REG_SIZE);
@@ -2507,7 +2496,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
/*
* bswap32 -- 32-bit swap (signed result for mips64). a0 = abcd.
*/
- bswap32_addr = align_code_ptr(s);
+ bswap32_addr = tcg_splitwx_to_rx(align_code_ptr(s));
/* t3 = (ssss)d000 */
tcg_out_opc_sa(s, OPC_SLL, TCG_TMP3, TCG_TMP0, 24);
/* t1 = 000a */
@@ -2535,7 +2524,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
/*
* bswap32u -- unsigned 32-bit swap. a0 = ....abcd.
*/
- bswap32u_addr = align_code_ptr(s);
+ bswap32u_addr = tcg_splitwx_to_rx(align_code_ptr(s));
/* t1 = (0000)000d */
tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP1, TCG_TMP0, 0xff);
/* t3 = 000a */
@@ -2561,7 +2550,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
/*
* bswap64 -- 64-bit swap. a0 = abcdefgh
*/
- bswap64_addr = align_code_ptr(s);
+ bswap64_addr = tcg_splitwx_to_rx(align_code_ptr(s));
/* t3 = h0000000 */
tcg_out_dsll(s, TCG_TMP3, TCG_TMP0, 56);
/* t1 = 0000000a */
@@ -2656,11 +2645,11 @@ static void tcg_target_init(TCGContext *s)
tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP); /* global pointer */
}
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
- uintptr_t addr)
+void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+ uintptr_t jmp_rw, uintptr_t addr)
{
- qatomic_set((uint32_t *)jmp_addr, deposit32(OPC_J, 0, 26, addr >> 2));
- flush_icache_range(jmp_addr, jmp_addr + 4);
+ qatomic_set((uint32_t *)jmp_rw, deposit32(OPC_J, 0, 26, addr >> 2));
+ flush_idcache_range(jmp_rx, jmp_rw, 4);
}
typedef struct {
@@ -2702,7 +2691,7 @@ static const DebugFrame debug_frame = {
}
};
-void tcg_register_jit(void *buf, size_t buf_size)
+void tcg_register_jit(const void *buf, size_t buf_size)
{
tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
}
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index 92c1d63..c2c32fb 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -169,6 +169,7 @@ extern bool use_mips32r2_instructions;
#define TCG_TARGET_HAS_clz_i32 use_mips32r2_instructions
#define TCG_TARGET_HAS_ctz_i32 0
#define TCG_TARGET_HAS_ctpop_i32 0
+#define TCG_TARGET_HAS_qemu_st8_i32 0
#if TCG_TARGET_REG_BITS == 64
#define TCG_TARGET_HAS_movcond_i64 use_movnz_instructions
@@ -201,7 +202,7 @@ extern bool use_mips32r2_instructions;
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#ifdef CONFIG_SOFTMMU
#define TCG_TARGET_NEED_LDST_LABELS
diff --git a/tcg/optimize.c b/tcg/optimize.c
index 7ca71de..1fb42eb 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -1541,6 +1541,7 @@ void tcg_optimize(TCGContext *s)
case INDEX_op_qemu_ld_i32:
case INDEX_op_qemu_ld_i64:
case INDEX_op_qemu_st_i32:
+ case INDEX_op_qemu_st8_i32:
case INDEX_op_qemu_st_i64:
case INDEX_op_call:
/* Opcodes that touch guest memory stop the optimization. */
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index 0d068ec..19a4a12 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -62,8 +62,6 @@
#define TCG_CT_CONST_MONE 0x2000
#define TCG_CT_CONST_WSZ 0x4000
-static tcg_insn_unit *tb_ret_addr;
-
TCGPowerISA have_isa;
static bool have_isel;
bool have_altivec;
@@ -184,35 +182,41 @@ static inline bool in_range_b(tcg_target_long target)
return target == sextract64(target, 0, 26);
}
-static uint32_t reloc_pc24_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+static uint32_t reloc_pc24_val(const tcg_insn_unit *pc,
+ const tcg_insn_unit *target)
{
ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
tcg_debug_assert(in_range_b(disp));
return disp & 0x3fffffc;
}
-static bool reloc_pc24(tcg_insn_unit *pc, tcg_insn_unit *target)
+static bool reloc_pc24(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
{
- ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx);
+
if (in_range_b(disp)) {
- *pc = (*pc & ~0x3fffffc) | (disp & 0x3fffffc);
+ *src_rw = (*src_rw & ~0x3fffffc) | (disp & 0x3fffffc);
return true;
}
return false;
}
-static uint16_t reloc_pc14_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+static uint16_t reloc_pc14_val(const tcg_insn_unit *pc,
+ const tcg_insn_unit *target)
{
ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
tcg_debug_assert(disp == (int16_t) disp);
return disp & 0xfffc;
}
-static bool reloc_pc14(tcg_insn_unit *pc, tcg_insn_unit *target)
+static bool reloc_pc14(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
{
- ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx);
+
if (disp == (int16_t) disp) {
- *pc = (*pc & ~0xfffc) | (disp & 0xfffc);
+ *src_rw = (*src_rw & ~0xfffc) | (disp & 0xfffc);
return true;
}
return false;
@@ -673,12 +677,12 @@ static const uint32_t tcg_to_isel[] = {
static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
- tcg_insn_unit *target;
+ const tcg_insn_unit *target;
int16_t lo;
int32_t hi;
value += addend;
- target = (tcg_insn_unit *)value;
+ target = (const tcg_insn_unit *)value;
switch (type) {
case R_PPC_REL14:
@@ -837,7 +841,7 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
}
/* Load addresses within the TB with one insn. */
- tb_diff = arg - (intptr_t)s->code_gen_ptr;
+ tb_diff = tcg_tbrel_diff(s, (void *)arg);
if (!in_prologue && USE_REG_TB && tb_diff == (int16_t)tb_diff) {
tcg_out32(s, ADDI | TAI(ret, TCG_REG_TB, tb_diff));
return;
@@ -890,7 +894,7 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
/* Use the constant pool, if possible. */
if (!in_prologue && USE_REG_TB) {
new_pool_label(s, arg, R_PPC_ADDR16, s->code_ptr,
- -(intptr_t)s->code_gen_ptr);
+ tcg_tbrel_diff(s, NULL));
tcg_out32(s, LD | TAI(ret, TCG_REG_TB, 0));
return;
}
@@ -940,7 +944,7 @@ static void tcg_out_dupi_vec(TCGContext *s, TCGType type, TCGReg ret,
*/
if (USE_REG_TB) {
rel = R_PPC_ADDR16;
- add = -(intptr_t)s->code_gen_ptr;
+ add = tcg_tbrel_diff(s, NULL);
} else {
rel = R_PPC_ADDR32;
add = 0;
@@ -1106,7 +1110,7 @@ static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
tcg_out_zori32(s, dst, src, c, XORI, XORIS);
}
-static void tcg_out_b(TCGContext *s, int mask, tcg_insn_unit *target)
+static void tcg_out_b(TCGContext *s, int mask, const tcg_insn_unit *target)
{
ptrdiff_t disp = tcg_pcrel_diff(s, target);
if (in_range_b(disp)) {
@@ -1544,7 +1548,7 @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
static void tcg_out_bc(TCGContext *s, int bc, TCGLabel *l)
{
if (l->has_value) {
- bc |= reloc_pc14_val(s->code_ptr, l->u.value_ptr);
+ bc |= reloc_pc14_val(tcg_splitwx_to_rx(s->code_ptr), l->u.value_ptr);
} else {
tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, l, 0);
}
@@ -1722,13 +1726,13 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
tcg_out32(s, insn);
}
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
- uintptr_t addr)
+void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+ uintptr_t jmp_rw, uintptr_t addr)
{
if (TCG_TARGET_REG_BITS == 64) {
tcg_insn_unit i1, i2;
intptr_t tb_diff = addr - tc_ptr;
- intptr_t br_diff = addr - (jmp_addr + 4);
+ intptr_t br_diff = addr - (jmp_rx + 4);
uint64_t pair;
/* This does not exercise the range of the branch, but we do
@@ -1752,23 +1756,23 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
/* As per the enclosing if, this is ppc64. Avoid the _Static_assert
within qatomic_set that would fail to build a ppc32 host. */
- qatomic_set__nocheck((uint64_t *)jmp_addr, pair);
- flush_icache_range(jmp_addr, jmp_addr + 8);
+ qatomic_set__nocheck((uint64_t *)jmp_rw, pair);
+ flush_idcache_range(jmp_rx, jmp_rw, 8);
} else {
- intptr_t diff = addr - jmp_addr;
+ intptr_t diff = addr - jmp_rx;
tcg_debug_assert(in_range_b(diff));
- qatomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc));
- flush_icache_range(jmp_addr, jmp_addr + 4);
+ qatomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc));
+ flush_idcache_range(jmp_rx, jmp_rw, 4);
}
}
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target)
{
#ifdef _CALL_AIX
/* Look through the descriptor. If the branch is in range, and we
don't have to spend too much effort on building the toc. */
- void *tgt = ((void **)target)[0];
- uintptr_t toc = ((uintptr_t *)target)[1];
+ const void *tgt = ((const void * const *)target)[0];
+ uintptr_t toc = ((const uintptr_t *)target)[1];
intptr_t diff = tcg_pcrel_diff(s, tgt);
if (in_range_b(diff) && toc == (uint32_t)toc) {
@@ -1997,7 +2001,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
label->datahi_reg = datahi_reg;
label->addrlo_reg = addrlo_reg;
label->addrhi_reg = addrhi_reg;
- label->raddr = raddr;
+ label->raddr = tcg_splitwx_to_rx(raddr);
label->label_ptr[0] = lptr;
}
@@ -2007,7 +2011,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp opc = get_memop(oi);
TCGReg hi, lo, arg = TCG_REG_R3;
- if (!reloc_pc14(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc14(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
return false;
}
@@ -2055,7 +2059,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp s_bits = opc & MO_SIZE;
TCGReg hi, lo, arg = TCG_REG_R3;
- if (!reloc_pc14(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc14(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
return false;
}
@@ -2306,10 +2310,10 @@ static void tcg_target_qemu_prologue(TCGContext *s)
int i;
#ifdef _CALL_AIX
- void **desc = (void **)s->code_ptr;
- desc[0] = desc + 2; /* entry point */
- desc[1] = 0; /* environment pointer */
- s->code_ptr = (void *)(desc + 2); /* skip over descriptor */
+ const void **desc = (const void **)s->code_ptr;
+ desc[0] = tcg_splitwx_to_rx(desc + 2); /* entry point */
+ desc[1] = 0; /* environment pointer */
+ s->code_ptr = (void *)(desc + 2); /* skip over descriptor */
#endif
tcg_set_frame(s, TCG_REG_CALL_STACK, REG_SAVE_BOT - CPU_TEMP_BUF_SIZE,
@@ -2341,7 +2345,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
tcg_out32(s, BCCTR | BO_ALWAYS);
/* Epilogue */
- s->code_gen_epilogue = tb_ret_addr = s->code_ptr;
+ tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
@@ -2362,7 +2366,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
switch (opc) {
case INDEX_op_exit_tb:
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
- tcg_out_b(s, 0, tb_ret_addr);
+ tcg_out_b(s, 0, tcg_code_gen_epilogue);
break;
case INDEX_op_goto_tb:
if (s->tb_jmp_insn_offset) {
@@ -2392,9 +2396,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
set_jmp_reset_offset(s, args[0]);
if (USE_REG_TB) {
/* For the unlinked case, need to reset TCG_REG_TB. */
- c = -tcg_current_code_size(s);
- assert(c == (int16_t)c);
- tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, c));
+ tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
+ -tcg_current_code_size(s));
}
break;
case INDEX_op_goto_ptr:
@@ -2411,7 +2414,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
uint32_t insn = B;
if (l->has_value) {
- insn |= reloc_pc24_val(s->code_ptr, l->u.value_ptr);
+ insn |= reloc_pc24_val(tcg_splitwx_to_rx(s->code_ptr),
+ l->u.value_ptr);
} else {
tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, l, 0);
}
@@ -3847,7 +3851,7 @@ static DebugFrame debug_frame = {
}
};
-void tcg_register_jit(void *buf, size_t buf_size)
+void tcg_register_jit(const void *buf, size_t buf_size)
{
uint8_t *p = &debug_frame.fde_reg_ofs[3];
int i;
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index a509a19..d1339af 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -108,6 +108,7 @@ extern bool have_vsx;
#define TCG_TARGET_HAS_mulsh_i32 1
#define TCG_TARGET_HAS_goto_ptr 1
#define TCG_TARGET_HAS_direct_jump 1
+#define TCG_TARGET_HAS_qemu_st8_i32 0
#if TCG_TARGET_REG_BITS == 64
#define TCG_TARGET_HAS_add2_i32 0
@@ -175,7 +176,7 @@ extern bool have_vsx;
#define TCG_TARGET_HAS_bitsel_vec have_vsx
#define TCG_TARGET_HAS_cmpsel_vec 0
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index 4089e29..c60b91b 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -425,39 +425,44 @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
* Relocations
*/
-static bool reloc_sbimm12(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static bool reloc_sbimm12(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
{
- intptr_t offset = (intptr_t)target - (intptr_t)code_ptr;
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ intptr_t offset = (intptr_t)target - (intptr_t)src_rx;
- if (offset == sextreg(offset, 1, 12) << 1) {
- code_ptr[0] |= encode_sbimm12(offset);
+ tcg_debug_assert((offset & 1) == 0);
+ if (offset == sextreg(offset, 0, 12)) {
+ *src_rw |= encode_sbimm12(offset);
return true;
}
return false;
}
-static bool reloc_jimm20(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static bool reloc_jimm20(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
{
- intptr_t offset = (intptr_t)target - (intptr_t)code_ptr;
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ intptr_t offset = (intptr_t)target - (intptr_t)src_rx;
- if (offset == sextreg(offset, 1, 20) << 1) {
- code_ptr[0] |= encode_ujimm20(offset);
+ tcg_debug_assert((offset & 1) == 0);
+ if (offset == sextreg(offset, 0, 20)) {
+ *src_rw |= encode_ujimm20(offset);
return true;
}
return false;
}
-static bool reloc_call(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static bool reloc_call(tcg_insn_unit *src_rw, const tcg_insn_unit *target)
{
- intptr_t offset = (intptr_t)target - (intptr_t)code_ptr;
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ intptr_t offset = (intptr_t)target - (intptr_t)src_rx;
int32_t lo = sextreg(offset, 0, 12);
int32_t hi = offset - lo;
if (offset == hi + lo) {
- code_ptr[0] |= encode_uimm20(hi);
- code_ptr[1] |= encode_imm12(lo);
+ src_rw[0] |= encode_uimm20(hi);
+ src_rw[1] |= encode_imm12(lo);
return true;
}
@@ -467,43 +472,16 @@ static bool reloc_call(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
- uint32_t insn = *code_ptr;
- intptr_t diff;
- bool short_jmp;
-
tcg_debug_assert(addend == 0);
-
switch (type) {
case R_RISCV_BRANCH:
- diff = value - (uintptr_t)code_ptr;
- short_jmp = diff == sextreg(diff, 0, 12);
- if (short_jmp) {
- return reloc_sbimm12(code_ptr, (tcg_insn_unit *)value);
- } else {
- /* Invert the condition */
- insn = insn ^ (1 << 12);
- /* Clear the offset */
- insn &= 0x01fff07f;
- /* Set the offset to the PC + 8 */
- insn |= encode_sbimm12(8);
-
- /* Move forward */
- code_ptr[0] = insn;
-
- /* Overwrite the NOP with jal x0,value */
- diff = value - (uintptr_t)(code_ptr + 1);
- insn = encode_uj(OPC_JAL, TCG_REG_ZERO, diff);
- code_ptr[1] = insn;
-
- return true;
- }
- break;
+ return reloc_sbimm12(code_ptr, (tcg_insn_unit *)value);
case R_RISCV_JAL:
return reloc_jimm20(code_ptr, (tcg_insn_unit *)value);
case R_RISCV_CALL:
return reloc_call(code_ptr, (tcg_insn_unit *)value);
default:
- tcg_abort();
+ g_assert_not_reached();
}
}
@@ -557,7 +535,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
if (tmp == (int32_t)tmp) {
tcg_out_opc_upper(s, OPC_AUIPC, rd, 0);
tcg_out_opc_imm(s, OPC_ADDI, rd, rd, 0);
- ret = reloc_call(s->code_ptr - 2, (tcg_insn_unit *)val);
+ ret = reloc_call(s->code_ptr - 2, (const tcg_insn_unit *)val);
tcg_debug_assert(ret == true);
return;
}
@@ -777,21 +755,8 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1,
arg2 = t;
}
- if (l->has_value) {
- intptr_t diff = tcg_pcrel_diff(s, l->u.value_ptr);
- if (diff == sextreg(diff, 0, 12)) {
- tcg_out_opc_branch(s, op, arg1, arg2, diff);
- } else {
- /* Invert the conditional branch. */
- tcg_out_opc_branch(s, op ^ (1 << 12), arg1, arg2, 8);
- tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, diff - 4);
- }
- } else {
- tcg_out_reloc(s, s->code_ptr, R_RISCV_BRANCH, l, 0);
- tcg_out_opc_branch(s, op, arg1, arg2, 0);
- /* NOP to allow patching later */
- tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0);
- }
+ tcg_out_reloc(s, s->code_ptr, R_RISCV_BRANCH, l, 0);
+ tcg_out_opc_branch(s, op, arg1, arg2, 0);
}
static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
@@ -854,28 +819,21 @@ static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
g_assert_not_reached();
}
-static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target)
-{
- ptrdiff_t offset = tcg_pcrel_diff(s, target);
- tcg_debug_assert(offset == sextreg(offset, 1, 20) << 1);
- tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, offset);
-}
-
-static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
+static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail)
{
TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA;
ptrdiff_t offset = tcg_pcrel_diff(s, arg);
int ret;
- if (offset == sextreg(offset, 1, 20) << 1) {
+ tcg_debug_assert((offset & 1) == 0);
+ if (offset == sextreg(offset, 0, 20)) {
/* short jump: -2097150 to 2097152 */
tcg_out_opc_jump(s, OPC_JAL, link, offset);
- } else if (TCG_TARGET_REG_BITS == 32 ||
- offset == sextreg(offset, 1, 31) << 1) {
+ } else if (TCG_TARGET_REG_BITS == 32 || offset == (int32_t)offset) {
/* long jump: -2147483646 to 2147483648 */
tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0);
tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0);
- ret = reloc_call(s->code_ptr - 2, arg);\
+ ret = reloc_call(s->code_ptr - 2, arg);
tcg_debug_assert(ret == true);
} else if (TCG_TARGET_REG_BITS == 64) {
/* far jump: 64-bit */
@@ -888,7 +846,7 @@ static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
}
}
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg)
{
tcg_out_call_int(s, arg, false);
}
@@ -962,6 +920,13 @@ QEMU_BUILD_BUG_ON(TCG_TARGET_REG_BITS < TARGET_LONG_BITS);
QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11));
+static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target)
+{
+ tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0);
+ bool ok = reloc_jimm20(s->code_ptr - 1, target);
+ tcg_debug_assert(ok);
+}
+
static void tcg_out_tlb_load(TCGContext *s, TCGReg addrl,
TCGReg addrh, TCGMemOpIdx oi,
tcg_insn_unit **label_ptr, bool is_load)
@@ -1007,8 +972,6 @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg addrl,
/* Compare masked address with the TLB entry. */
label_ptr[0] = s->code_ptr;
tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0);
- /* NOP to allow patching later */
- tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0);
/* TLB Hit - translate address using addend. */
if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
@@ -1033,7 +996,7 @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi,
label->datahi_reg = datahi;
label->addrlo_reg = addrlo;
label->addrhi_reg = addrhi;
- label->raddr = raddr;
+ label->raddr = tcg_splitwx_to_rx(raddr);
label->label_ptr[0] = label_ptr[0];
}
@@ -1052,8 +1015,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
}
/* resolve label address */
- if (!patch_reloc(l->label_ptr[0], R_RISCV_BRANCH,
- (intptr_t) s->code_ptr, 0)) {
+ if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
return false;
}
@@ -1087,8 +1049,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
}
/* resolve label address */
- if (!patch_reloc(l->label_ptr[0], R_RISCV_BRANCH,
- (intptr_t) s->code_ptr, 0)) {
+ if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
return false;
}
@@ -1274,7 +1235,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
#endif
}
-static tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *tb_ret_addr;
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg *args, const int *const_args)
@@ -1288,7 +1249,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_exit_tb:
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
- tcg_out_call_int(s, s->code_gen_epilogue, true);
+ tcg_out_call_int(s, tcg_code_gen_epilogue, true);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
tcg_out_call_int(s, tb_ret_addr, true);
@@ -1822,11 +1783,11 @@ static void tcg_target_qemu_prologue(TCGContext *s)
tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, tcg_target_call_iarg_regs[1], 0);
/* Return path for goto_ptr. Set return value to 0 */
- s->code_gen_epilogue = s->code_ptr;
+ tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_A0, TCG_REG_ZERO);
/* TB epilogue */
- tb_ret_addr = s->code_ptr;
+ tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
TCG_REG_SP, SAVE_OFS + i * REG_SIZE);
@@ -1907,7 +1868,7 @@ static const DebugFrame debug_frame = {
}
};
-void tcg_register_jit(void *buf, size_t buf_size)
+void tcg_register_jit(const void *buf, size_t buf_size)
{
tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
}
diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h
index c1bd52b..727c8df 100644
--- a/tcg/riscv/tcg-target.h
+++ b/tcg/riscv/tcg-target.h
@@ -119,6 +119,7 @@ typedef enum {
#define TCG_TARGET_HAS_direct_jump 0
#define TCG_TARGET_HAS_brcond2 1
#define TCG_TARGET_HAS_setcond2 1
+#define TCG_TARGET_HAS_qemu_st8_i32 0
#if TCG_TARGET_REG_BITS == 64
#define TCG_TARGET_HAS_movcond_i64 0
@@ -160,7 +161,7 @@ typedef enum {
#endif
/* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#define TCG_TARGET_DEFAULT_MO (0)
diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc
index c5e0964..d7ef079 100644
--- a/tcg/s390/tcg-target.c.inc
+++ b/tcg/s390/tcg-target.c.inc
@@ -363,36 +363,37 @@ static void * const qemu_st_helpers[16] = {
};
#endif
-static tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *tb_ret_addr;
uint64_t s390_facilities;
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(tcg_insn_unit *src_rw, int type,
intptr_t value, intptr_t addend)
{
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
intptr_t pcrel2;
uint32_t old;
value += addend;
- pcrel2 = (tcg_insn_unit *)value - code_ptr;
+ pcrel2 = (tcg_insn_unit *)value - src_rx;
switch (type) {
case R_390_PC16DBL:
if (pcrel2 == (int16_t)pcrel2) {
- tcg_patch16(code_ptr, pcrel2);
+ tcg_patch16(src_rw, pcrel2);
return true;
}
break;
case R_390_PC32DBL:
if (pcrel2 == (int32_t)pcrel2) {
- tcg_patch32(code_ptr, pcrel2);
+ tcg_patch32(src_rw, pcrel2);
return true;
}
break;
case R_390_20:
if (value == sextract64(value, 0, 20)) {
- old = *(uint32_t *)code_ptr & 0xf00000ff;
+ old = *(uint32_t *)src_rw & 0xf00000ff;
old |= ((value & 0xfff) << 16) | ((value & 0xff000) >> 4);
- tcg_patch32(code_ptr, old);
+ tcg_patch32(src_rw, old);
return true;
}
break;
@@ -630,7 +631,7 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
return;
}
} else if (USE_REG_TB && !in_prologue) {
- ptrdiff_t off = sval - (uintptr_t)s->code_gen_ptr;
+ ptrdiff_t off = tcg_tbrel_diff(s, (void *)sval);
if (off == sextract64(off, 0, 20)) {
/* This is certain to be an address within TB, and therefore
OFF will be negative; don't try RX_LA. */
@@ -655,7 +656,7 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
} else if (USE_REG_TB && !in_prologue) {
tcg_out_insn(s, RXY, LG, ret, TCG_REG_TB, TCG_REG_NONE, 0);
new_pool_label(s, sval, R_390_20, s->code_ptr - 2,
- -(intptr_t)s->code_gen_ptr);
+ tcg_tbrel_diff(s, NULL));
} else {
TCGReg base = ret ? ret : TCG_TMP0;
tcg_out_insn(s, RIL, LARL, base, 0);
@@ -730,7 +731,8 @@ static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
}
/* load data from an absolute host address */
-static void tcg_out_ld_abs(TCGContext *s, TCGType type, TCGReg dest, void *abs)
+static void tcg_out_ld_abs(TCGContext *s, TCGType type,
+ TCGReg dest, const void *abs)
{
intptr_t addr = (intptr_t)abs;
@@ -746,7 +748,7 @@ static void tcg_out_ld_abs(TCGContext *s, TCGType type, TCGReg dest, void *abs)
}
}
if (USE_REG_TB) {
- ptrdiff_t disp = abs - (void *)s->code_gen_ptr;
+ ptrdiff_t disp = tcg_tbrel_diff(s, abs);
if (disp == sextract64(disp, 0, 20)) {
tcg_out_ld(s, type, dest, TCG_REG_TB, disp);
return;
@@ -956,7 +958,7 @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
if (!maybe_out_small_movi(s, type, TCG_TMP0, val)) {
tcg_out_insn(s, RXY, NG, dest, TCG_REG_TB, TCG_REG_NONE, 0);
new_pool_label(s, val & valid, R_390_20, s->code_ptr - 2,
- -(intptr_t)s->code_gen_ptr);
+ tcg_tbrel_diff(s, NULL));
return;
}
} else {
@@ -1015,7 +1017,7 @@ static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
} else if (USE_REG_TB) {
tcg_out_insn(s, RXY, OG, dest, TCG_REG_TB, TCG_REG_NONE, 0);
new_pool_label(s, val, R_390_20, s->code_ptr - 2,
- -(intptr_t)s->code_gen_ptr);
+ tcg_tbrel_diff(s, NULL));
} else {
/* Perform the OR via sequential modifications to the high and
low parts. Do this via recursion to handle 16-bit vs 32-bit
@@ -1050,7 +1052,7 @@ static void tgen_xori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
} else if (USE_REG_TB) {
tcg_out_insn(s, RXY, XG, dest, TCG_REG_TB, TCG_REG_NONE, 0);
new_pool_label(s, val, R_390_20, s->code_ptr - 2,
- -(intptr_t)s->code_gen_ptr);
+ tcg_tbrel_diff(s, NULL));
} else {
/* Perform the xor by parts. */
tcg_debug_assert(s390_facilities & FACILITY_EXT_IMM);
@@ -1108,12 +1110,12 @@ static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
op = (is_unsigned ? RXY_CLY : RXY_CY);
tcg_out_insn_RXY(s, op, r1, TCG_REG_TB, TCG_REG_NONE, 0);
new_pool_label(s, (uint32_t)c2, R_390_20, s->code_ptr - 2,
- 4 - (intptr_t)s->code_gen_ptr);
+ 4 - tcg_tbrel_diff(s, NULL));
} else {
op = (is_unsigned ? RXY_CLG : RXY_CG);
tcg_out_insn_RXY(s, op, r1, TCG_REG_TB, TCG_REG_NONE, 0);
new_pool_label(s, c2, R_390_20, s->code_ptr - 2,
- -(intptr_t)s->code_gen_ptr);
+ tcg_tbrel_diff(s, NULL));
}
goto exit;
} else {
@@ -1302,9 +1304,9 @@ static void tgen_extract(TCGContext *s, TCGReg dest, TCGReg src,
tcg_out_risbg(s, dest, src, 64 - len, 63, 64 - ofs, 1);
}
-static void tgen_gotoi(TCGContext *s, int cc, tcg_insn_unit *dest)
+static void tgen_gotoi(TCGContext *s, int cc, const tcg_insn_unit *dest)
{
- ptrdiff_t off = dest - s->code_ptr;
+ ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
if (off == (int16_t)off) {
tcg_out_insn(s, RI, BRC, cc, off);
} else if (off == (int32_t)off) {
@@ -1333,34 +1335,18 @@ static void tgen_branch(TCGContext *s, int cc, TCGLabel *l)
static void tgen_compare_branch(TCGContext *s, S390Opcode opc, int cc,
TCGReg r1, TCGReg r2, TCGLabel *l)
{
- intptr_t off = 0;
-
- if (l->has_value) {
- off = l->u.value_ptr - s->code_ptr;
- tcg_debug_assert(off == (int16_t)off);
- } else {
- tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
- }
-
+ tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
tcg_out16(s, (opc & 0xff00) | (r1 << 4) | r2);
- tcg_out16(s, off);
+ tcg_out16(s, 0);
tcg_out16(s, cc << 12 | (opc & 0xff));
}
static void tgen_compare_imm_branch(TCGContext *s, S390Opcode opc, int cc,
TCGReg r1, int i2, TCGLabel *l)
{
- tcg_target_long off = 0;
-
- if (l->has_value) {
- off = l->u.value_ptr - s->code_ptr;
- tcg_debug_assert(off == (int16_t)off);
- } else {
- tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
- }
-
+ tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
tcg_out16(s, (opc & 0xff00) | (r1 << 4) | cc);
- tcg_out16(s, off);
+ tcg_out16(s, 0);
tcg_out16(s, (i2 << 8) | (opc & 0xff));
}
@@ -1415,9 +1401,9 @@ static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c,
tgen_branch(s, cc, l);
}
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest)
{
- ptrdiff_t off = dest - s->code_ptr;
+ ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
if (off == (int32_t)off) {
tcg_out_insn(s, RIL, BRASL, TCG_REG_R14, off);
} else {
@@ -1601,7 +1587,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
label->oi = oi;
label->datalo_reg = data;
label->addrlo_reg = addr;
- label->raddr = raddr;
+ label->raddr = tcg_splitwx_to_rx(raddr);
label->label_ptr[0] = label_ptr;
}
@@ -1613,7 +1599,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp opc = get_memop(oi);
if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
- (intptr_t)s->code_ptr, 2)) {
+ (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
return false;
}
@@ -1638,7 +1624,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp opc = get_memop(oi);
if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
- (intptr_t)s->code_ptr, 2)) {
+ (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
return false;
}
@@ -1756,7 +1742,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
/* Reuse the zeroing that exists for goto_ptr. */
a0 = args[0];
if (a0 == 0) {
- tgen_gotoi(s, S390_CC_ALWAYS, s->code_gen_epilogue);
+ tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
@@ -1766,7 +1752,8 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_goto_tb:
a0 = args[0];
if (s->tb_jmp_insn_offset) {
- /* branch displacement must be aligned for atomic patching;
+ /*
+ * branch displacement must be aligned for atomic patching;
* see if we need to add extra nop before branch
*/
if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
@@ -1779,7 +1766,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
} else {
/* load address stored at s->tb_jmp_target_addr + a0 */
tcg_out_ld_abs(s, TCG_TYPE_PTR, TCG_REG_TB,
- s->tb_jmp_target_addr + a0);
+ tcg_splitwx_to_rx(s->tb_jmp_target_addr + a0));
/* and go there */
tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_TB);
}
@@ -1789,8 +1776,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
TCG_REG_TB to the beginning of this TB. */
if (USE_REG_TB) {
int ofs = -tcg_current_code_size(s);
- assert(ofs == (int16_t)ofs);
- tcg_out_insn(s, RI, AGHI, TCG_REG_TB, ofs);
+ /* All TB are restricted to 64KiB by unwind info. */
+ tcg_debug_assert(ofs == sextract64(ofs, 0, 20));
+ tcg_out_insn(s, RXY, LAY, TCG_REG_TB,
+ TCG_REG_TB, TCG_REG_NONE, ofs);
}
break;
@@ -2561,11 +2550,11 @@ static void tcg_target_qemu_prologue(TCGContext *s)
* Return path for goto_ptr. Set return value to 0, a-la exit_tb,
* and fall through to the rest of the epilogue.
*/
- s->code_gen_epilogue = s->code_ptr;
+ tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, 0);
/* TB epilogue */
- tb_ret_addr = s->code_ptr;
+ tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
/* lmg %r6,%r15,fs+48(%r15) (restore registers) */
tcg_out_insn(s, RXY, LMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15,
@@ -2620,7 +2609,7 @@ static const DebugFrame debug_frame = {
}
};
-void tcg_register_jit(void *buf, size_t buf_size)
+void tcg_register_jit(const void *buf, size_t buf_size)
{
tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
}
diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h
index b4feb2f..641464e 100644
--- a/tcg/s390/tcg-target.h
+++ b/tcg/s390/tcg-target.h
@@ -97,6 +97,7 @@ extern uint64_t s390_facilities;
#define TCG_TARGET_HAS_extrh_i64_i32 0
#define TCG_TARGET_HAS_goto_ptr 1
#define TCG_TARGET_HAS_direct_jump (s390_facilities & FACILITY_GEN_INST_EXT)
+#define TCG_TARGET_HAS_qemu_st8_i32 0
#define TCG_TARGET_HAS_div2_i64 1
#define TCG_TARGET_HAS_rot_i64 1
@@ -145,12 +146,12 @@ enum {
TCG_AREG0 = TCG_REG_R10,
};
-static inline void tb_target_set_jmp_target(uintptr_t tc_ptr,
- uintptr_t jmp_addr, uintptr_t addr)
+static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+ uintptr_t jmp_rw, uintptr_t addr)
{
/* patch the branch destination */
- intptr_t disp = addr - (jmp_addr - 2);
- qatomic_set((int32_t *)jmp_addr, disp / 2);
+ intptr_t disp = addr - (jmp_rx - 2);
+ qatomic_set((int32_t *)jmp_rw, disp / 2);
/* no need to flush icache explicitly */
}
diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc
index 6775bd3..922ae96 100644
--- a/tcg/sparc/tcg-target.c.inc
+++ b/tcg/sparc/tcg-target.c.inc
@@ -291,14 +291,15 @@ static inline int check_fit_i32(int32_t val, unsigned int bits)
# define check_fit_ptr check_fit_i32
#endif
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(tcg_insn_unit *src_rw, int type,
intptr_t value, intptr_t addend)
{
- uint32_t insn = *code_ptr;
+ const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
+ uint32_t insn = *src_rw;
intptr_t pcrel;
value += addend;
- pcrel = tcg_ptr_byte_diff((tcg_insn_unit *)value, code_ptr);
+ pcrel = tcg_ptr_byte_diff((tcg_insn_unit *)value, src_rx);
switch (type) {
case R_SPARC_WDISP16:
@@ -315,7 +316,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
g_assert_not_reached();
}
- *code_ptr = insn;
+ *src_rw = insn;
return true;
}
@@ -440,7 +441,7 @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
/* A 13-bit constant relative to the TB. */
if (!in_prologue && USE_REG_TB) {
- test = arg - (uintptr_t)s->code_gen_ptr;
+ test = tcg_tbrel_diff(s, (void *)arg);
if (check_fit_ptr(test, 13)) {
tcg_out_arithi(s, ret, TCG_REG_TB, test, ARITH_ADD);
return;
@@ -537,15 +538,15 @@ static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
return false;
}
-static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, uintptr_t arg)
+static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, const void *arg)
{
- intptr_t diff = arg - (uintptr_t)s->code_gen_ptr;
+ intptr_t diff = tcg_tbrel_diff(s, arg);
if (USE_REG_TB && check_fit_ptr(diff, 13)) {
tcg_out_ld(s, TCG_TYPE_PTR, ret, TCG_REG_TB, diff);
return;
}
- tcg_out_movi(s, TCG_TYPE_PTR, ret, arg & ~0x3ff);
- tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, arg & 0x3ff);
+ tcg_out_movi(s, TCG_TYPE_PTR, ret, (uintptr_t)arg & ~0x3ff);
+ tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, (uintptr_t)arg & 0x3ff);
}
static inline void tcg_out_sety(TCGContext *s, TCGReg rs)
@@ -840,7 +841,7 @@ static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh,
tcg_out_mov(s, TCG_TYPE_I64, rl, tmp);
}
-static void tcg_out_call_nodelay(TCGContext *s, tcg_insn_unit *dest,
+static void tcg_out_call_nodelay(TCGContext *s, const tcg_insn_unit *dest,
bool in_prologue)
{
ptrdiff_t disp = tcg_pcrel_diff(s, dest);
@@ -855,7 +856,7 @@ static void tcg_out_call_nodelay(TCGContext *s, tcg_insn_unit *dest,
}
}
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest)
{
tcg_out_call_nodelay(s, dest, false);
tcg_out_nop(s);
@@ -868,8 +869,8 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
}
#ifdef CONFIG_SOFTMMU
-static tcg_insn_unit *qemu_ld_trampoline[16];
-static tcg_insn_unit *qemu_st_trampoline[16];
+static const tcg_insn_unit *qemu_ld_trampoline[16];
+static const tcg_insn_unit *qemu_st_trampoline[16];
static void emit_extend(TCGContext *s, TCGReg r, int op)
{
@@ -930,7 +931,7 @@ static void build_trampolines(TCGContext *s)
while ((uintptr_t)s->code_ptr & 15) {
tcg_out_nop(s);
}
- qemu_ld_trampoline[i] = s->code_ptr;
+ qemu_ld_trampoline[i] = tcg_splitwx_to_rx(s->code_ptr);
if (SPARC64 || TARGET_LONG_BITS == 32) {
ra = TCG_REG_O3;
@@ -958,7 +959,7 @@ static void build_trampolines(TCGContext *s)
while ((uintptr_t)s->code_ptr & 15) {
tcg_out_nop(s);
}
- qemu_st_trampoline[i] = s->code_ptr;
+ qemu_st_trampoline[i] = tcg_splitwx_to_rx(s->code_ptr);
if (SPARC64) {
emit_extend(s, TCG_REG_O2, i);
@@ -1038,7 +1039,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
tcg_out_nop(s);
/* Epilogue for goto_ptr. */
- s->code_gen_epilogue = s->code_ptr;
+ tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
/* delay slot */
tcg_out_movi_imm13(s, TCG_REG_O0, 0);
@@ -1163,7 +1164,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
#ifdef CONFIG_SOFTMMU
unsigned memi = get_mmuidx(oi);
TCGReg addrz, param;
- tcg_insn_unit *func;
+ const tcg_insn_unit *func;
tcg_insn_unit *label_ptr;
addrz = tcg_out_tlb_load(s, addr, memi, memop,
@@ -1245,7 +1246,7 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
#ifdef CONFIG_SOFTMMU
unsigned memi = get_mmuidx(oi);
TCGReg addrz, param;
- tcg_insn_unit *func;
+ const tcg_insn_unit *func;
tcg_insn_unit *label_ptr;
addrz = tcg_out_tlb_load(s, addr, memi, memop,
@@ -1313,7 +1314,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
tcg_out_movi_imm13(s, TCG_REG_O0, a0);
break;
} else if (USE_REG_TB) {
- intptr_t tb_diff = a0 - (uintptr_t)s->code_gen_ptr;
+ intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0);
if (check_fit_ptr(tb_diff, 13)) {
tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
/* Note that TCG_REG_TB has been unwound to O1. */
@@ -1345,8 +1346,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
}
} else {
/* indirect jump method */
- tcg_out_ld_ptr(s, TCG_REG_TB,
- (uintptr_t)(s->tb_jmp_target_addr + a0));
+ tcg_out_ld_ptr(s, TCG_REG_TB, s->tb_jmp_target_addr + a0);
tcg_out_arithi(s, TCG_REG_G0, TCG_REG_TB, 0, JMPL);
tcg_out_nop(s);
}
@@ -1816,16 +1816,16 @@ static const DebugFrame debug_frame = {
.fde_ret_save = { 9, 15, 31 }, /* DW_CFA_register o7, i7 */
};
-void tcg_register_jit(void *buf, size_t buf_size)
+void tcg_register_jit(const void *buf, size_t buf_size)
{
tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
}
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
- uintptr_t addr)
+void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+ uintptr_t jmp_rw, uintptr_t addr)
{
intptr_t tb_disp = addr - tc_ptr;
- intptr_t br_disp = addr - jmp_addr;
+ intptr_t br_disp = addr - jmp_rx;
tcg_insn_unit i1, i2;
/* We can reach the entire address space for ILP32.
@@ -1834,9 +1834,9 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
tcg_debug_assert(br_disp == (int32_t)br_disp);
if (!USE_REG_TB) {
- qatomic_set((uint32_t *)jmp_addr,
+ qatomic_set((uint32_t *)jmp_rw,
deposit32(CALL, 0, 30, br_disp >> 2));
- flush_icache_range(jmp_addr, jmp_addr + 4);
+ flush_idcache_range(jmp_rx, jmp_rw, 4);
return;
}
@@ -1859,6 +1859,6 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
| INSN_IMM13((tb_disp & 0x3ff) | -0x400));
}
- qatomic_set((uint64_t *)jmp_addr, deposit64(i2, 32, 32, i1));
- flush_icache_range(jmp_addr, jmp_addr + 8);
+ qatomic_set((uint64_t *)jmp_rw, deposit64(i2, 32, 32, i1));
+ flush_idcache_range(jmp_rx, jmp_rw, 8);
}
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index d8b0e32..95ab9af 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -126,6 +126,7 @@ extern bool use_vis3_instructions;
#define TCG_TARGET_HAS_mulsh_i32 0
#define TCG_TARGET_HAS_goto_ptr 1
#define TCG_TARGET_HAS_direct_jump 1
+#define TCG_TARGET_HAS_qemu_st8_i32 0
#define TCG_TARGET_HAS_extrl_i64_i32 1
#define TCG_TARGET_HAS_extrh_i64_i32 1
@@ -168,7 +169,7 @@ extern bool use_vis3_instructions;
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#define TCG_TARGET_NEED_POOL_LABELS
diff --git a/tcg/tcg-ldst.c.inc b/tcg/tcg-ldst.c.inc
index 05f9b3c..c3ce88e 100644
--- a/tcg/tcg-ldst.c.inc
+++ b/tcg/tcg-ldst.c.inc
@@ -28,7 +28,7 @@ typedef struct TCGLabelQemuLdst {
TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */
TCGReg datalo_reg; /* reg index for low word to be loaded or stored */
TCGReg datahi_reg; /* reg index for high word to be loaded or stored */
- tcg_insn_unit *raddr; /* gen code addr of the next IR of qemu_ld/st IR */
+ const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */
tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */
QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next;
} TCGLabelQemuLdst;
diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
index 4b8a473..0374b5d 100644
--- a/tcg/tcg-op.c
+++ b/tcg/tcg-op.c
@@ -2664,9 +2664,20 @@ void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
/* QEMU specific operations. */
-void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)
-{
- uintptr_t val = (uintptr_t)tb + idx;
+void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
+{
+ /*
+ * Let the jit code return the read-only version of the
+ * TranslationBlock, so that we minimize the pc-relative
+ * distance of the address of the exit_tb code to TB.
+ * This will improve utilization of pc-relative address loads.
+ *
+ * TODO: Move this to translator_loop, so that all const
+ * TranslationBlock pointers refer to read-only memory.
+ * This requires coordination with targets that do not use
+ * the translator_loop.
+ */
+ uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
if (tb == NULL) {
tcg_debug_assert(idx == 0);
@@ -2883,7 +2894,11 @@ void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
}
addr = plugin_prep_mem_callbacks(addr);
- gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
+ if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) {
+ gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx);
+ } else {
+ gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
+ }
plugin_gen_mem_callbacks(addr, info);
if (swap) {
diff --git a/tcg/tcg-pool.c.inc b/tcg/tcg-pool.c.inc
index 82cbcc8..90c2e63 100644
--- a/tcg/tcg-pool.c.inc
+++ b/tcg/tcg-pool.c.inc
@@ -140,6 +140,8 @@ static int tcg_out_pool_finalize(TCGContext *s)
for (; p != NULL; p = p->next) {
size_t size = sizeof(tcg_target_ulong) * p->nlong;
+ uintptr_t value;
+
if (!l || l->nlong != p->nlong || memcmp(l->data, p->data, size)) {
if (unlikely(a > s->code_gen_highwater)) {
return -1;
@@ -148,7 +150,9 @@ static int tcg_out_pool_finalize(TCGContext *s)
a += size;
l = p;
}
- if (!patch_reloc(p->label, p->rtype, (intptr_t)a - size, p->addend)) {
+
+ value = (uintptr_t)tcg_splitwx_to_rx(a) - size;
+ if (!patch_reloc(p->label, p->rtype, value, p->addend)) {
return -2;
}
}
diff --git a/tcg/tcg.c b/tcg/tcg.c
index ebb9466..472bf17 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -97,7 +97,7 @@ typedef struct QEMU_PACKED {
DebugFrameFDEHeader fde;
} DebugFrameHeader;
-static void tcg_register_jit_int(void *buf, size_t size,
+static void tcg_register_jit_int(const void *buf, size_t size,
const void *debug_frame,
size_t debug_frame_size)
__attribute__((unused));
@@ -149,7 +149,7 @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
intptr_t arg2);
static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
TCGReg base, intptr_t ofs);
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *target);
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
static int tcg_target_const_match(tcg_target_long val, TCGType type,
const TCGArgConstraint *arg_ct);
#ifdef TCG_TARGET_NEED_LDST_LABELS
@@ -161,6 +161,12 @@ static int tcg_out_ldst_finalize(TCGContext *s);
static TCGContext **tcg_ctxs;
static unsigned int n_tcg_ctxs;
TCGv_env cpu_env = 0;
+const void *tcg_code_gen_epilogue;
+uintptr_t tcg_splitwx_diff;
+
+#ifndef CONFIG_TCG_INTERPRETER
+tcg_prologue_fn *tcg_qemu_tb_exec;
+#endif
struct tcg_region_tree {
QemuMutex lock;
@@ -296,11 +302,11 @@ static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
}
-static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr)
+static void tcg_out_label(TCGContext *s, TCGLabel *l)
{
tcg_debug_assert(!l->has_value);
l->has_value = 1;
- l->u.value_ptr = ptr;
+ l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
}
TCGLabel *gen_new_label(void)
@@ -401,8 +407,9 @@ static void tcg_region_trees_init(void)
}
}
-static struct tcg_region_tree *tc_ptr_to_region_tree(void *p)
+static struct tcg_region_tree *tc_ptr_to_region_tree(const void *cp)
{
+ void *p = tcg_splitwx_to_rw(cp);
size_t region_idx;
if (p < region.start_aligned) {
@@ -696,6 +703,7 @@ void tcg_region_init(void)
size_t region_size;
size_t n_regions;
size_t i;
+ uintptr_t splitwx_diff;
n_regions = tcg_n_regions();
@@ -726,6 +734,7 @@ void tcg_region_init(void)
region.end -= page_size;
/* set guard pages */
+ splitwx_diff = tcg_splitwx_diff;
for (i = 0; i < region.n; i++) {
void *start, *end;
int rc;
@@ -733,6 +742,10 @@ void tcg_region_init(void)
tcg_region_bounds(i, &start, &end);
rc = qemu_mprotect_none(end, page_size);
g_assert(!rc);
+ if (splitwx_diff) {
+ rc = qemu_mprotect_none(end + splitwx_diff, page_size);
+ g_assert(!rc);
+ }
}
tcg_region_trees_init();
@@ -747,6 +760,29 @@ void tcg_region_init(void)
#endif
}
+#ifdef CONFIG_DEBUG_TCG
+const void *tcg_splitwx_to_rx(void *rw)
+{
+ /* Pass NULL pointers unchanged. */
+ if (rw) {
+ g_assert(in_code_gen_buffer(rw));
+ rw += tcg_splitwx_diff;
+ }
+ return rw;
+}
+
+void *tcg_splitwx_to_rw(const void *rx)
+{
+ /* Pass NULL pointers unchanged. */
+ if (rx) {
+ rx -= tcg_splitwx_diff;
+ /* Assert that we end with a pointer in the rw region. */
+ g_assert(in_code_gen_buffer(rx));
+ }
+ return (void *)rx;
+}
+#endif /* CONFIG_DEBUG_TCG */
+
static void alloc_tcg_plugin_context(TCGContext *s)
{
#ifdef CONFIG_PLUGIN
@@ -1055,7 +1091,17 @@ void tcg_prologue_init(TCGContext *s)
s->code_ptr = buf0;
s->code_buf = buf0;
s->data_gen_ptr = NULL;
- s->code_gen_prologue = buf0;
+
+ /*
+ * The region trees are not yet configured, but tcg_splitwx_to_rx
+ * needs the bounds for an assert.
+ */
+ region.start = buf0;
+ region.end = buf0 + total_size;
+
+#ifndef CONFIG_TCG_INTERPRETER
+ tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(buf0);
+#endif
/* Compute a high-water mark, at which we voluntarily flush the buffer
and start over. The size here is arbitrary, significantly larger
@@ -1078,7 +1124,10 @@ void tcg_prologue_init(TCGContext *s)
#endif
buf1 = s->code_ptr;
- flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1);
+#ifndef CONFIG_TCG_INTERPRETER
+ flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(buf0), (uintptr_t)buf0,
+ tcg_ptr_byte_diff(buf1, buf0));
+#endif
/* Deduct the prologue from the buffer. */
prologue_size = tcg_current_code_size(s);
@@ -1088,7 +1137,7 @@ void tcg_prologue_init(TCGContext *s)
total_size -= prologue_size;
s->code_gen_buffer_size = total_size;
- tcg_register_jit(s->code_gen_buffer, total_size);
+ tcg_register_jit(tcg_splitwx_to_rx(s->code_gen_buffer), total_size);
#ifdef DEBUG_DISAS
if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
@@ -1123,7 +1172,7 @@ void tcg_prologue_init(TCGContext *s)
/* Assert that goto_ptr is implemented completely. */
if (TCG_TARGET_HAS_goto_ptr) {
- tcg_debug_assert(s->code_gen_epilogue != NULL);
+ tcg_debug_assert(tcg_code_gen_epilogue != NULL);
}
}
@@ -1427,6 +1476,9 @@ bool tcg_op_supported(TCGOpcode op)
case INDEX_op_qemu_st_i64:
return true;
+ case INDEX_op_qemu_st8_i32:
+ return TCG_TARGET_HAS_qemu_st8_i32;
+
case INDEX_op_goto_ptr:
return TCG_TARGET_HAS_goto_ptr;
@@ -2087,6 +2139,7 @@ static void tcg_dump_ops(TCGContext *s, bool have_prefs)
break;
case INDEX_op_qemu_ld_i32:
case INDEX_op_qemu_st_i32:
+ case INDEX_op_qemu_st8_i32:
case INDEX_op_qemu_ld_i64:
case INDEX_op_qemu_st_i64:
{
@@ -4216,8 +4269,13 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
tcg_reg_alloc_start(s);
- s->code_buf = tb->tc.ptr;
- s->code_ptr = tb->tc.ptr;
+ /*
+ * Reset the buffer pointers when restarting after overflow.
+ * TODO: Move this into translate-all.c with the rest of the
+ * buffer management. Having only this done here is confusing.
+ */
+ s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
+ s->code_ptr = s->code_buf;
#ifdef TCG_TARGET_NEED_LDST_LABELS
QSIMPLEQ_INIT(&s->ldst_labels);
@@ -4271,7 +4329,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
break;
case INDEX_op_set_label:
tcg_reg_alloc_bb_end(s, s->reserved_regs);
- tcg_out_label(s, arg_label(op->args[0]), s->code_ptr);
+ tcg_out_label(s, arg_label(op->args[0]));
break;
case INDEX_op_call:
tcg_reg_alloc_call(s, op);
@@ -4320,8 +4378,12 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
return -2;
}
+#ifndef CONFIG_TCG_INTERPRETER
/* flush instruction cache */
- flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
+ flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
+ (uintptr_t)s->code_buf,
+ tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
+#endif
return tcg_current_code_size(s);
}
@@ -4449,7 +4511,7 @@ static int find_string(const char *strtab, const char *str)
}
}
-static void tcg_register_jit_int(void *buf_ptr, size_t buf_size,
+static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
const void *debug_frame,
size_t debug_frame_size)
{
@@ -4651,13 +4713,13 @@ static void tcg_register_jit_int(void *buf_ptr, size_t buf_size,
/* No support for the feature. Provide the entry point expected by exec.c,
and implement the internal function we declared earlier. */
-static void tcg_register_jit_int(void *buf, size_t size,
+static void tcg_register_jit_int(const void *buf, size_t size,
const void *debug_frame,
size_t debug_frame_size)
{
}
-void tcg_register_jit(void *buf, size_t buf_size)
+void tcg_register_jit(const void *buf, size_t buf_size)
{
}
#endif /* ELF_HOST_MACHINE */
diff --git a/tcg/tci.c b/tcg/tci.c
index 5d97b7c..2bcc440 100644
--- a/tcg/tci.c
+++ b/tcg/tci.c
@@ -164,34 +164,34 @@ static uint64_t tci_uint64(uint32_t high, uint32_t low)
#endif
/* Read constant (native size) from bytecode. */
-static tcg_target_ulong tci_read_i(uint8_t **tb_ptr)
+static tcg_target_ulong tci_read_i(const uint8_t **tb_ptr)
{
- tcg_target_ulong value = *(tcg_target_ulong *)(*tb_ptr);
+ tcg_target_ulong value = *(const tcg_target_ulong *)(*tb_ptr);
*tb_ptr += sizeof(value);
return value;
}
/* Read unsigned constant (32 bit) from bytecode. */
-static uint32_t tci_read_i32(uint8_t **tb_ptr)
+static uint32_t tci_read_i32(const uint8_t **tb_ptr)
{
- uint32_t value = *(uint32_t *)(*tb_ptr);
+ uint32_t value = *(const uint32_t *)(*tb_ptr);
*tb_ptr += sizeof(value);
return value;
}
/* Read signed constant (32 bit) from bytecode. */
-static int32_t tci_read_s32(uint8_t **tb_ptr)
+static int32_t tci_read_s32(const uint8_t **tb_ptr)
{
- int32_t value = *(int32_t *)(*tb_ptr);
+ int32_t value = *(const int32_t *)(*tb_ptr);
*tb_ptr += sizeof(value);
return value;
}
#if TCG_TARGET_REG_BITS == 64
/* Read constant (64 bit) from bytecode. */
-static uint64_t tci_read_i64(uint8_t **tb_ptr)
+static uint64_t tci_read_i64(const uint8_t **tb_ptr)
{
- uint64_t value = *(uint64_t *)(*tb_ptr);
+ uint64_t value = *(const uint64_t *)(*tb_ptr);
*tb_ptr += sizeof(value);
return value;
}
@@ -199,7 +199,7 @@ static uint64_t tci_read_i64(uint8_t **tb_ptr)
/* Read indexed register (native size) from bytecode. */
static tcg_target_ulong
-tci_read_r(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+tci_read_r(const tcg_target_ulong *regs, const uint8_t **tb_ptr)
{
tcg_target_ulong value = tci_read_reg(regs, **tb_ptr);
*tb_ptr += 1;
@@ -207,7 +207,7 @@ tci_read_r(const tcg_target_ulong *regs, uint8_t **tb_ptr)
}
/* Read indexed register (8 bit) from bytecode. */
-static uint8_t tci_read_r8(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static uint8_t tci_read_r8(const tcg_target_ulong *regs, const uint8_t **tb_ptr)
{
uint8_t value = tci_read_reg8(regs, **tb_ptr);
*tb_ptr += 1;
@@ -216,7 +216,7 @@ static uint8_t tci_read_r8(const tcg_target_ulong *regs, uint8_t **tb_ptr)
#if TCG_TARGET_HAS_ext8s_i32 || TCG_TARGET_HAS_ext8s_i64
/* Read indexed register (8 bit signed) from bytecode. */
-static int8_t tci_read_r8s(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static int8_t tci_read_r8s(const tcg_target_ulong *regs, const uint8_t **tb_ptr)
{
int8_t value = tci_read_reg8s(regs, **tb_ptr);
*tb_ptr += 1;
@@ -225,7 +225,8 @@ static int8_t tci_read_r8s(const tcg_target_ulong *regs, uint8_t **tb_ptr)
#endif
/* Read indexed register (16 bit) from bytecode. */
-static uint16_t tci_read_r16(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static uint16_t tci_read_r16(const tcg_target_ulong *regs,
+ const uint8_t **tb_ptr)
{
uint16_t value = tci_read_reg16(regs, **tb_ptr);
*tb_ptr += 1;
@@ -234,7 +235,8 @@ static uint16_t tci_read_r16(const tcg_target_ulong *regs, uint8_t **tb_ptr)
#if TCG_TARGET_HAS_ext16s_i32 || TCG_TARGET_HAS_ext16s_i64
/* Read indexed register (16 bit signed) from bytecode. */
-static int16_t tci_read_r16s(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static int16_t tci_read_r16s(const tcg_target_ulong *regs,
+ const uint8_t **tb_ptr)
{
int16_t value = tci_read_reg16s(regs, **tb_ptr);
*tb_ptr += 1;
@@ -243,7 +245,8 @@ static int16_t tci_read_r16s(const tcg_target_ulong *regs, uint8_t **tb_ptr)
#endif
/* Read indexed register (32 bit) from bytecode. */
-static uint32_t tci_read_r32(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static uint32_t tci_read_r32(const tcg_target_ulong *regs,
+ const uint8_t **tb_ptr)
{
uint32_t value = tci_read_reg32(regs, **tb_ptr);
*tb_ptr += 1;
@@ -252,14 +255,16 @@ static uint32_t tci_read_r32(const tcg_target_ulong *regs, uint8_t **tb_ptr)
#if TCG_TARGET_REG_BITS == 32
/* Read two indexed registers (2 * 32 bit) from bytecode. */
-static uint64_t tci_read_r64(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static uint64_t tci_read_r64(const tcg_target_ulong *regs,
+ const uint8_t **tb_ptr)
{
uint32_t low = tci_read_r32(regs, tb_ptr);
return tci_uint64(tci_read_r32(regs, tb_ptr), low);
}
#elif TCG_TARGET_REG_BITS == 64
/* Read indexed register (32 bit signed) from bytecode. */
-static int32_t tci_read_r32s(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static int32_t tci_read_r32s(const tcg_target_ulong *regs,
+ const uint8_t **tb_ptr)
{
int32_t value = tci_read_reg32s(regs, **tb_ptr);
*tb_ptr += 1;
@@ -267,7 +272,8 @@ static int32_t tci_read_r32s(const tcg_target_ulong *regs, uint8_t **tb_ptr)
}
/* Read indexed register (64 bit) from bytecode. */
-static uint64_t tci_read_r64(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static uint64_t tci_read_r64(const tcg_target_ulong *regs,
+ const uint8_t **tb_ptr)
{
uint64_t value = tci_read_reg64(regs, **tb_ptr);
*tb_ptr += 1;
@@ -277,7 +283,7 @@ static uint64_t tci_read_r64(const tcg_target_ulong *regs, uint8_t **tb_ptr)
/* Read indexed register(s) with target address from bytecode. */
static target_ulong
-tci_read_ulong(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+tci_read_ulong(const tcg_target_ulong *regs, const uint8_t **tb_ptr)
{
target_ulong taddr = tci_read_r(regs, tb_ptr);
#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
@@ -288,7 +294,7 @@ tci_read_ulong(const tcg_target_ulong *regs, uint8_t **tb_ptr)
/* Read indexed register or constant (native size) from bytecode. */
static tcg_target_ulong
-tci_read_ri(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+tci_read_ri(const tcg_target_ulong *regs, const uint8_t **tb_ptr)
{
tcg_target_ulong value;
TCGReg r = **tb_ptr;
@@ -302,7 +308,8 @@ tci_read_ri(const tcg_target_ulong *regs, uint8_t **tb_ptr)
}
/* Read indexed register or constant (32 bit) from bytecode. */
-static uint32_t tci_read_ri32(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static uint32_t tci_read_ri32(const tcg_target_ulong *regs,
+ const uint8_t **tb_ptr)
{
uint32_t value;
TCGReg r = **tb_ptr;
@@ -317,14 +324,16 @@ static uint32_t tci_read_ri32(const tcg_target_ulong *regs, uint8_t **tb_ptr)
#if TCG_TARGET_REG_BITS == 32
/* Read two indexed registers or constants (2 * 32 bit) from bytecode. */
-static uint64_t tci_read_ri64(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static uint64_t tci_read_ri64(const tcg_target_ulong *regs,
+ const uint8_t **tb_ptr)
{
uint32_t low = tci_read_ri32(regs, tb_ptr);
return tci_uint64(tci_read_ri32(regs, tb_ptr), low);
}
#elif TCG_TARGET_REG_BITS == 64
/* Read indexed register or constant (64 bit) from bytecode. */
-static uint64_t tci_read_ri64(const tcg_target_ulong *regs, uint8_t **tb_ptr)
+static uint64_t tci_read_ri64(const tcg_target_ulong *regs,
+ const uint8_t **tb_ptr)
{
uint64_t value;
TCGReg r = **tb_ptr;
@@ -338,7 +347,7 @@ static uint64_t tci_read_ri64(const tcg_target_ulong *regs, uint8_t **tb_ptr)
}
#endif
-static tcg_target_ulong tci_read_label(uint8_t **tb_ptr)
+static tcg_target_ulong tci_read_label(const uint8_t **tb_ptr)
{
tcg_target_ulong label = tci_read_i(tb_ptr);
tci_assert(label != 0);
@@ -481,9 +490,10 @@ static bool tci_compare64(uint64_t u0, uint64_t u1, TCGCond condition)
* One possible operation in the pseudo code is a call to binary code.
* Therefore, disable CFI checks in the interpreter function
*/
-QEMU_DISABLE_CFI
-uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
+uintptr_t QEMU_DISABLE_CFI tcg_qemu_tb_exec(CPUArchState *env,
+ const void *v_tb_ptr)
{
+ const uint8_t *tb_ptr = v_tb_ptr;
tcg_target_ulong regs[TCG_TARGET_NB_REGS];
long tcg_temps[CPU_TEMP_BUF_NLONGS];
uintptr_t sp_value = (uintptr_t)(tcg_temps + CPU_TEMP_BUF_NLONGS);
@@ -497,7 +507,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
TCGOpcode opc = tb_ptr[0];
#if defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)
uint8_t op_size = tb_ptr[1];
- uint8_t *old_code_ptr = tb_ptr;
+ const uint8_t *old_code_ptr = tb_ptr;
#endif
tcg_target_ulong t0;
tcg_target_ulong t1;
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index 231b9b1..d5a4d9d 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -545,7 +545,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type,
old_code_ptr[1] = s->code_ptr - old_code_ptr;
}
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
+static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg)
{
uint8_t *old_code_ptr = s->code_ptr;
tcg_out_op_t(s, INDEX_op_call);
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index b84480f..bb784e0 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -88,6 +88,7 @@
#define TCG_TARGET_HAS_mulsh_i32 0
#define TCG_TARGET_HAS_goto_ptr 0
#define TCG_TARGET_HAS_direct_jump 1
+#define TCG_TARGET_HAS_qemu_st8_i32 0
#if TCG_TARGET_REG_BITS == 64
#define TCG_TARGET_HAS_extrl_i64_i32 0
@@ -198,11 +199,11 @@ void tci_disas(uint8_t opc);
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
-static inline void tb_target_set_jmp_target(uintptr_t tc_ptr,
- uintptr_t jmp_addr, uintptr_t addr)
+static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+ uintptr_t jmp_rw, uintptr_t addr)
{
/* patch the branch destination */
- qatomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
+ qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4));
/* no need to flush icache explicitly */
}
generated by cgit v1.1 at 2024-12-28 19:45:36 +0000