diff options
Diffstat (limited to 'target/arm/tcg/translate-a64.c')
| -rw-r--r-- | target/arm/tcg/translate-a64.c | 754 |
1 files changed, 656 insertions, 98 deletions
diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c index 3901432..3292d7c 100644 --- a/target/arm/tcg/translate-a64.c +++ b/target/arm/tcg/translate-a64.c @@ -17,8 +17,7 @@ * License along with this library; if not, see <http://www.gnu.org/licenses/>. */ #include "qemu/osdep.h" - -#include "exec/exec-all.h" +#include "exec/target_page.h" #include "translate.h" #include "translate-a64.h" #include "qemu/log.h" @@ -27,6 +26,7 @@ #include "cpregs.h" static TCGv_i64 cpu_X[32]; +static TCGv_i64 cpu_gcspr[4]; static TCGv_i64 cpu_pc; /* Load/store exclusive handling */ @@ -78,6 +78,10 @@ static int scale_by_log2_tag_granule(DisasContext *s, int x) /* initialize TCG globals. */ void a64_translate_init(void) { + static const char gcspr_names[4][12] = { + "gcspr_el0", "gcspr_el1", "gcspr_el2", "gcspr_el3" + }; + int i; cpu_pc = tcg_global_mem_new_i64(tcg_env, @@ -91,10 +95,17 @@ void a64_translate_init(void) cpu_exclusive_high = tcg_global_mem_new_i64(tcg_env, offsetof(CPUARMState, exclusive_high), "exclusive_high"); + + for (i = 0; i < 4; i++) { + cpu_gcspr[i] = + tcg_global_mem_new_i64(tcg_env, + offsetof(CPUARMState, cp15.gcspr_el[i]), + gcspr_names[i]); + } } /* - * Return the core mmu_idx to use for A64 load/store insns which + * Return the full arm mmu_idx to use for A64 load/store insns which * have a "unprivileged load/store" variant. Those insns access * EL0 if executed from an EL which has control over EL0 (usually * EL1) but behave like normal loads and stores if executed from @@ -104,7 +115,7 @@ void a64_translate_init(void) * normal encoding (in which case we will return the same * thing as get_mem_index(). */ -static int get_a64_user_mem_index(DisasContext *s, bool unpriv) +static ARMMMUIdx full_a64_user_mem_index(DisasContext *s, bool unpriv) { /* * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL, @@ -131,7 +142,19 @@ static int get_a64_user_mem_index(DisasContext *s, bool unpriv) g_assert_not_reached(); } } - return arm_to_core_mmu_idx(useridx); + return useridx; +} + +/* Return the core mmu_idx per above. */ +static int core_a64_user_mem_index(DisasContext *s, bool unpriv) +{ + return arm_to_core_mmu_idx(full_a64_user_mem_index(s, unpriv)); +} + +/* For a given translation regime, return the core mmu_idx for gcs access. */ +static int core_gcs_mem_index(ARMMMUIdx armidx) +{ + return arm_to_core_mmu_idx(regime_to_gcs(armidx)); } static void set_btype_raw(int val) @@ -409,6 +432,39 @@ static MemOp check_ordered_align(DisasContext *s, int rn, int imm, return finalize_memop(s, mop); } +static void gen_add_gcs_record(DisasContext *s, TCGv_i64 value) +{ + TCGv_i64 addr = tcg_temp_new_i64(); + TCGv_i64 gcspr = cpu_gcspr[s->current_el]; + int mmuidx = core_gcs_mem_index(s->mmu_idx); + MemOp mop = finalize_memop(s, MO_64 | MO_ALIGN); + + tcg_gen_addi_i64(addr, gcspr, -8); + tcg_gen_qemu_st_i64(value, clean_data_tbi(s, addr), mmuidx, mop); + tcg_gen_mov_i64(gcspr, addr); +} + +static void gen_load_check_gcs_record(DisasContext *s, TCGv_i64 target, + GCSInstructionType it, int rt) +{ + TCGv_i64 gcspr = cpu_gcspr[s->current_el]; + int mmuidx = core_gcs_mem_index(s->mmu_idx); + MemOp mop = finalize_memop(s, MO_64 | MO_ALIGN); + TCGv_i64 rec_va = tcg_temp_new_i64(); + + tcg_gen_qemu_ld_i64(rec_va, clean_data_tbi(s, gcspr), mmuidx, mop); + + if (s->gcs_rvcen) { + TCGLabel *fail_label = + delay_exception(s, EXCP_UDEF, syn_gcs_data_check(it, rt)); + + tcg_gen_brcond_i64(TCG_COND_NE, rec_va, target, fail_label); + } + + gen_a64_set_pc(s, rec_va); + tcg_gen_addi_i64(gcspr, gcspr, 8); +} + typedef struct DisasCompare64 { TCGCond cond; TCGv_i64 value; @@ -434,12 +490,6 @@ static void gen_rebuild_hflags(DisasContext *s) gen_helper_rebuild_hflags_a64(tcg_env, tcg_constant_i32(s->current_el)); } -static void gen_exception_internal(int excp) -{ - assert(excp_is_internal(excp)); - gen_helper_exception_internal(tcg_env, tcg_constant_i32(excp)); -} - static void gen_exception_internal_insn(DisasContext *s, int excp) { gen_a64_update_pc(s, 0); @@ -478,7 +528,7 @@ static inline bool use_goto_tb(DisasContext *s, uint64_t dest) return translator_use_goto_tb(&s->base, dest); } -static void gen_goto_tb(DisasContext *s, int n, int64_t diff) +static void gen_goto_tb(DisasContext *s, unsigned tb_slot_idx, int64_t diff) { if (use_goto_tb(s, s->pc_curr + diff)) { /* @@ -491,12 +541,12 @@ static void gen_goto_tb(DisasContext *s, int n, int64_t diff) */ if (tb_cflags(s->base.tb) & CF_PCREL) { gen_a64_update_pc(s, diff); - tcg_gen_goto_tb(n); + tcg_gen_goto_tb(tb_slot_idx); } else { - tcg_gen_goto_tb(n); + tcg_gen_goto_tb(tb_slot_idx); gen_a64_update_pc(s, diff); } - tcg_gen_exit_tb(s->base.tb, n); + tcg_gen_exit_tb(s->base.tb, tb_slot_idx); s->base.is_jmp = DISAS_NORETURN; } else { gen_a64_update_pc(s, diff); @@ -1076,11 +1126,9 @@ static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) TCGv_i64 cf_64 = tcg_temp_new_i64(); TCGv_i64 vf_64 = tcg_temp_new_i64(); TCGv_i64 tmp = tcg_temp_new_i64(); - TCGv_i64 zero = tcg_constant_i64(0); tcg_gen_extu_i32_i64(cf_64, cpu_CF); - tcg_gen_add2_i64(result, cf_64, t0, zero, cf_64, zero); - tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, zero); + tcg_gen_addcio_i64(result, cf_64, t0, t1, cf_64); tcg_gen_extrl_i64_i32(cpu_CF, cf_64); gen_set_NZ64(result); @@ -1094,12 +1142,10 @@ static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) TCGv_i32 t0_32 = tcg_temp_new_i32(); TCGv_i32 t1_32 = tcg_temp_new_i32(); TCGv_i32 tmp = tcg_temp_new_i32(); - TCGv_i32 zero = tcg_constant_i32(0); tcg_gen_extrl_i64_i32(t0_32, t0); tcg_gen_extrl_i64_i32(t1_32, t1); - tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, zero, cpu_CF, zero); - tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, zero); + tcg_gen_addcio_i32(cpu_NF, cpu_CF, t0_32, t1_32, cpu_CF); tcg_gen_mov_i32(cpu_ZF, cpu_NF); tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32); @@ -1392,11 +1438,8 @@ static bool fp_access_check_only(DisasContext *s) return true; } -static bool fp_access_check(DisasContext *s) +static bool nonstreaming_check(DisasContext *s) { - if (!fp_access_check_only(s)) { - return false; - } if (s->sme_trap_nonstreaming && s->is_nonstreaming) { gen_exception_insn(s, 0, EXCP_UDEF, syn_smetrap(SME_ET_Streaming, false)); @@ -1405,6 +1448,11 @@ static bool fp_access_check(DisasContext *s) return true; } +static bool fp_access_check(DisasContext *s) +{ + return fp_access_check_only(s) && nonstreaming_check(s); +} + /* * Return <0 for non-supported element sizes, with MO_16 controlled by * FEAT_FP16; return 0 for fp disabled; otherwise return >0 for success. @@ -1455,14 +1503,24 @@ static int fp_access_check_vector_hsd(DisasContext *s, bool is_q, MemOp esz) */ bool sve_access_check(DisasContext *s) { - if (s->pstate_sm || !dc_isar_feature(aa64_sve, s)) { + if (dc_isar_feature(aa64_sme, s)) { bool ret; - assert(dc_isar_feature(aa64_sme, s)); - ret = sme_sm_enabled_check(s); + if (s->pstate_sm) { + ret = sme_enabled_check(s); + } else if (dc_isar_feature(aa64_sve, s)) { + goto continue_sve; + } else { + ret = sme_sm_enabled_check(s); + } + if (ret) { + ret = nonstreaming_check(s); + } s->sve_access_checked = (ret ? 1 : -1); return ret; } + + continue_sve: if (s->sve_excp_el) { /* Assert that we only raise one exception per instruction. */ assert(!s->sve_access_checked); @@ -1499,7 +1557,8 @@ bool sme_enabled_check(DisasContext *s) * to be zero when fp_excp_el has priority. This is because we need * sme_excp_el by itself for cpregs access checks. */ - if (!s->fp_excp_el || s->sme_excp_el < s->fp_excp_el) { + if (s->sme_excp_el + && (!s->fp_excp_el || s->sme_excp_el <= s->fp_excp_el)) { bool ret = sme_access_check(s); s->fp_access_checked = (ret ? 1 : -1); return ret; @@ -1640,7 +1699,14 @@ static bool trans_B(DisasContext *s, arg_i *a) static bool trans_BL(DisasContext *s, arg_i *a) { - gen_pc_plus_diff(s, cpu_reg(s, 30), curr_insn_len(s)); + TCGv_i64 link = tcg_temp_new_i64(); + + gen_pc_plus_diff(s, link, 4); + if (s->gcs_en) { + gen_add_gcs_record(s, link); + } + tcg_gen_mov_i64(cpu_reg(s, 30), link); + reset_btype(s); gen_goto_tb(s, 0, a->imm); return true; @@ -1737,15 +1803,15 @@ static bool trans_BR(DisasContext *s, arg_r *a) static bool trans_BLR(DisasContext *s, arg_r *a) { - TCGv_i64 dst = cpu_reg(s, a->rn); - TCGv_i64 lr = cpu_reg(s, 30); - if (dst == lr) { - TCGv_i64 tmp = tcg_temp_new_i64(); - tcg_gen_mov_i64(tmp, dst); - dst = tmp; + TCGv_i64 link = tcg_temp_new_i64(); + + gen_pc_plus_diff(s, link, 4); + if (s->gcs_en) { + gen_add_gcs_record(s, link); } - gen_pc_plus_diff(s, lr, curr_insn_len(s)); - gen_a64_set_pc(s, dst); + gen_a64_set_pc(s, cpu_reg(s, a->rn)); + tcg_gen_mov_i64(cpu_reg(s, 30), link); + set_btype_for_blr(s); s->base.is_jmp = DISAS_JUMP; return true; @@ -1753,7 +1819,13 @@ static bool trans_BLR(DisasContext *s, arg_r *a) static bool trans_RET(DisasContext *s, arg_r *a) { - gen_a64_set_pc(s, cpu_reg(s, a->rn)); + TCGv_i64 target = cpu_reg(s, a->rn); + + if (s->gcs_en) { + gen_load_check_gcs_record(s, target, GCS_IT_RET_nPauth, a->rn); + } else { + gen_a64_set_pc(s, target); + } s->base.is_jmp = DISAS_JUMP; return true; } @@ -1797,21 +1869,21 @@ static bool trans_BRAZ(DisasContext *s, arg_braz *a) static bool trans_BLRAZ(DisasContext *s, arg_braz *a) { - TCGv_i64 dst, lr; + TCGv_i64 dst, link; if (!dc_isar_feature(aa64_pauth, s)) { return false; } - dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); - lr = cpu_reg(s, 30); - if (dst == lr) { - TCGv_i64 tmp = tcg_temp_new_i64(); - tcg_gen_mov_i64(tmp, dst); - dst = tmp; + + link = tcg_temp_new_i64(); + gen_pc_plus_diff(s, link, 4); + if (s->gcs_en) { + gen_add_gcs_record(s, link); } - gen_pc_plus_diff(s, lr, curr_insn_len(s)); gen_a64_set_pc(s, dst); + tcg_gen_mov_i64(cpu_reg(s, 30), link); + set_btype_for_blr(s); s->base.is_jmp = DISAS_JUMP; return true; @@ -1821,8 +1893,17 @@ static bool trans_RETA(DisasContext *s, arg_reta *a) { TCGv_i64 dst; + if (!dc_isar_feature(aa64_pauth, s)) { + return false; + } + dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m); - gen_a64_set_pc(s, dst); + if (s->gcs_en) { + GCSInstructionType it = a->m ? GCS_IT_RET_PauthB : GCS_IT_RET_PauthA; + gen_load_check_gcs_record(s, dst, it, 30); + } else { + gen_a64_set_pc(s, dst); + } s->base.is_jmp = DISAS_JUMP; return true; } @@ -1843,20 +1924,21 @@ static bool trans_BRA(DisasContext *s, arg_bra *a) static bool trans_BLRA(DisasContext *s, arg_bra *a) { - TCGv_i64 dst, lr; + TCGv_i64 dst, link; if (!dc_isar_feature(aa64_pauth, s)) { return false; } dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m); - lr = cpu_reg(s, 30); - if (dst == lr) { - TCGv_i64 tmp = tcg_temp_new_i64(); - tcg_gen_mov_i64(tmp, dst); - dst = tmp; + + link = tcg_temp_new_i64(); + gen_pc_plus_diff(s, link, 4); + if (s->gcs_en) { + gen_add_gcs_record(s, link); } - gen_pc_plus_diff(s, lr, curr_insn_len(s)); gen_a64_set_pc(s, dst); + tcg_gen_mov_i64(cpu_reg(s, 30), link); + set_btype_for_blr(s); s->base.is_jmp = DISAS_JUMP; return true; @@ -1864,6 +1946,9 @@ static bool trans_BLRA(DisasContext *s, arg_bra *a) static bool trans_ERET(DisasContext *s, arg_ERET *a) { +#ifdef CONFIG_USER_ONLY + return false; +#else TCGv_i64 dst; if (s->current_el == 0) { @@ -1883,10 +1968,14 @@ static bool trans_ERET(DisasContext *s, arg_ERET *a) /* Must exit loop to check un-masked IRQs */ s->base.is_jmp = DISAS_EXIT; return true; +#endif } static bool trans_ERETA(DisasContext *s, arg_reta *a) { +#ifdef CONFIG_USER_ONLY + return false; +#else TCGv_i64 dst; if (!dc_isar_feature(aa64_pauth, s)) { @@ -1912,6 +2001,7 @@ static bool trans_ERETA(DisasContext *s, arg_reta *a) /* Must exit loop to check un-masked IRQs */ s->base.is_jmp = DISAS_EXIT; return true; +#endif } static bool trans_NOP(DisasContext *s, arg_NOP *a) @@ -2054,6 +2144,14 @@ static bool trans_ESB(DisasContext *s, arg_ESB *a) return true; } +static bool trans_GCSB(DisasContext *s, arg_GCSB *a) +{ + if (dc_isar_feature(aa64_gcs, s)) { + tcg_gen_mb(TCG_BAR_SC | TCG_MO_ALL); + } + return true; +} + static bool trans_PACIAZ(DisasContext *s, arg_PACIAZ *a) { if (s->pauth_active) { @@ -2118,6 +2216,20 @@ static bool trans_AUTIBSP(DisasContext *s, arg_AUTIBSP *a) return true; } +static bool trans_CHKFEAT(DisasContext *s, arg_CHKFEAT *a) +{ + uint64_t feat_en = 0; + + if (s->gcs_en) { + feat_en |= 1 << 0; + } + if (feat_en) { + TCGv_i64 x16 = cpu_reg(s, 16); + tcg_gen_andi_i64(x16, x16, ~feat_en); + } + return true; +} + static bool trans_CLREX(DisasContext *s, arg_CLREX *a) { tcg_gen_movi_i64(cpu_exclusive_addr, -1); @@ -2449,6 +2561,195 @@ static void gen_sysreg_undef(DisasContext *s, bool isread, gen_exception_insn(s, 0, EXCP_UDEF, syndrome); } +static void gen_gcspopm(DisasContext *s, int rt) +{ + TCGv_i64 gcspr = cpu_gcspr[s->current_el]; + int mmuidx = core_gcs_mem_index(s->mmu_idx); + MemOp mop = finalize_memop(s, MO_64 | MO_ALIGN); + TCGv_i64 value = tcg_temp_new_i64(); + TCGLabel *fail_label = + delay_exception(s, EXCP_UDEF, syn_gcs_data_check(GCS_IT_GCSPOPM, rt)); + + /* The value at top-of-stack must have low 2 bits clear. */ + tcg_gen_qemu_ld_i64(value, clean_data_tbi(s, gcspr), mmuidx, mop); + tcg_gen_brcondi_i64(TCG_COND_TSTNE, value, 3, fail_label); + + /* Complete the pop and return the value. */ + tcg_gen_addi_i64(gcspr, gcspr, 8); + tcg_gen_mov_i64(cpu_reg(s, rt), value); +} + +static void gen_gcspushx(DisasContext *s) +{ + TCGv_i64 gcspr = cpu_gcspr[s->current_el]; + int spsr_idx = aarch64_banked_spsr_index(s->current_el); + int spsr_off = offsetof(CPUARMState, banked_spsr[spsr_idx]); + int elr_off = offsetof(CPUARMState, elr_el[s->current_el]); + int mmuidx = core_gcs_mem_index(s->mmu_idx); + MemOp mop = finalize_memop(s, MO_64 | MO_ALIGN); + TCGv_i64 addr = tcg_temp_new_i64(); + TCGv_i64 tmp = tcg_temp_new_i64(); + + tcg_gen_addi_i64(addr, gcspr, -8); + tcg_gen_qemu_st_i64(cpu_reg(s, 30), addr, mmuidx, mop); + + tcg_gen_ld_i64(tmp, tcg_env, spsr_off); + tcg_gen_addi_i64(addr, addr, -8); + tcg_gen_qemu_st_i64(tmp, addr, mmuidx, mop); + + tcg_gen_ld_i64(tmp, tcg_env, elr_off); + tcg_gen_addi_i64(addr, addr, -8); + tcg_gen_qemu_st_i64(tmp, addr, mmuidx, mop); + + tcg_gen_addi_i64(addr, addr, -8); + tcg_gen_qemu_st_i64(tcg_constant_i64(0b1001), addr, mmuidx, mop); + + tcg_gen_mov_i64(gcspr, addr); + clear_pstate_bits(PSTATE_EXLOCK); +} + +static void gen_gcspopcx(DisasContext *s) +{ + TCGv_i64 gcspr = cpu_gcspr[s->current_el]; + int spsr_idx = aarch64_banked_spsr_index(s->current_el); + int spsr_off = offsetof(CPUARMState, banked_spsr[spsr_idx]); + int elr_off = offsetof(CPUARMState, elr_el[s->current_el]); + int gcscr_off = offsetof(CPUARMState, cp15.gcscr_el[s->current_el]); + int pstate_off = offsetof(CPUARMState, pstate); + int mmuidx = core_gcs_mem_index(s->mmu_idx); + MemOp mop = finalize_memop(s, MO_64 | MO_ALIGN); + TCGv_i64 addr = tcg_temp_new_i64(); + TCGv_i64 tmp1 = tcg_temp_new_i64(); + TCGv_i64 tmp2 = tcg_temp_new_i64(); + TCGLabel *fail_label = + delay_exception(s, EXCP_UDEF, syn_gcs_data_check(GCS_IT_GCSPOPCX, 31)); + + /* The value at top-of-stack must be an exception token. */ + tcg_gen_qemu_ld_i64(tmp1, gcspr, mmuidx, mop); + tcg_gen_brcondi_i64(TCG_COND_NE, tmp1, 0b1001, fail_label); + + /* Validate in turn, ELR ... */ + tcg_gen_addi_i64(addr, gcspr, 8); + tcg_gen_qemu_ld_i64(tmp1, addr, mmuidx, mop); + tcg_gen_ld_i64(tmp2, tcg_env, elr_off); + tcg_gen_brcond_i64(TCG_COND_NE, tmp1, tmp2, fail_label); + + /* ... SPSR ... */ + tcg_gen_addi_i64(addr, addr, 8); + tcg_gen_qemu_ld_i64(tmp1, addr, mmuidx, mop); + tcg_gen_ld_i64(tmp2, tcg_env, spsr_off); + tcg_gen_brcond_i64(TCG_COND_NE, tmp1, tmp2, fail_label); + + /* ... and LR. */ + tcg_gen_addi_i64(addr, addr, 8); + tcg_gen_qemu_ld_i64(tmp1, addr, mmuidx, mop); + tcg_gen_brcond_i64(TCG_COND_NE, tmp1, cpu_reg(s, 30), fail_label); + + /* Writeback stack pointer after pop. */ + tcg_gen_addi_i64(gcspr, addr, 8); + + /* PSTATE.EXLOCK = GetCurrentEXLOCKEN(). */ + tcg_gen_ld_i64(tmp1, tcg_env, gcscr_off); + tcg_gen_ld_i64(tmp2, tcg_env, pstate_off); + tcg_gen_shri_i64(tmp1, tmp1, ctz64(GCSCR_EXLOCKEN)); + tcg_gen_deposit_i64(tmp2, tmp2, tmp1, ctz64(PSTATE_EXLOCK), 1); + tcg_gen_st_i64(tmp2, tcg_env, pstate_off); +} + +static void gen_gcspopx(DisasContext *s) +{ + TCGv_i64 gcspr = cpu_gcspr[s->current_el]; + int mmuidx = core_gcs_mem_index(s->mmu_idx); + MemOp mop = finalize_memop(s, MO_64 | MO_ALIGN); + TCGv_i64 addr = tcg_temp_new_i64(); + TCGv_i64 tmp = tcg_temp_new_i64(); + TCGLabel *fail_label = + delay_exception(s, EXCP_UDEF, syn_gcs_data_check(GCS_IT_GCSPOPX, 31)); + + /* The value at top-of-stack must be an exception token. */ + tcg_gen_qemu_ld_i64(tmp, gcspr, mmuidx, mop); + tcg_gen_brcondi_i64(TCG_COND_NE, tmp, 0b1001, fail_label); + + /* + * The other three values in the exception return record + * are ignored, but are loaded anyway to raise faults. + */ + tcg_gen_addi_i64(addr, gcspr, 8); + tcg_gen_qemu_ld_i64(tmp, addr, mmuidx, mop); + tcg_gen_addi_i64(addr, addr, 8); + tcg_gen_qemu_ld_i64(tmp, addr, mmuidx, mop); + tcg_gen_addi_i64(addr, addr, 8); + tcg_gen_qemu_ld_i64(tmp, addr, mmuidx, mop); + tcg_gen_addi_i64(gcspr, addr, 8); +} + +static void gen_gcsss1(DisasContext *s, int rt) +{ + TCGv_i64 gcspr = cpu_gcspr[s->current_el]; + int mmuidx = core_gcs_mem_index(s->mmu_idx); + MemOp mop = finalize_memop(s, MO_64 | MO_ALIGN); + TCGv_i64 inptr = cpu_reg(s, rt); + TCGv_i64 cmp = tcg_temp_new_i64(); + TCGv_i64 new = tcg_temp_new_i64(); + TCGv_i64 old = tcg_temp_new_i64(); + TCGLabel *fail_label = + delay_exception(s, EXCP_UDEF, syn_gcs_data_check(GCS_IT_GCSSS1, rt)); + + /* Compute the valid cap entry that the new stack must have. */ + tcg_gen_deposit_i64(cmp, inptr, tcg_constant_i64(1), 0, 12); + /* Compute the in-progress cap entry for the old stack. */ + tcg_gen_deposit_i64(new, gcspr, tcg_constant_i64(5), 0, 3); + + /* Swap the valid cap the with the in-progress cap. */ + tcg_gen_atomic_cmpxchg_i64(old, inptr, cmp, new, mmuidx, mop); + tcg_gen_brcond_i64(TCG_COND_NE, old, cmp, fail_label); + + /* The new stack had a valid cap: change gcspr. */ + tcg_gen_andi_i64(gcspr, inptr, ~7); +} + +static void gen_gcsss2(DisasContext *s, int rt) +{ + TCGv_i64 gcspr = cpu_gcspr[s->current_el]; + int mmuidx = core_gcs_mem_index(s->mmu_idx); + MemOp mop = finalize_memop(s, MO_64 | MO_ALIGN); + TCGv_i64 outptr = tcg_temp_new_i64(); + TCGv_i64 tmp = tcg_temp_new_i64(); + TCGLabel *fail_label = + delay_exception(s, EXCP_UDEF, syn_gcs_data_check(GCS_IT_GCSSS2, rt)); + + /* Validate that the new stack has an in-progress cap. */ + tcg_gen_qemu_ld_i64(outptr, gcspr, mmuidx, mop); + tcg_gen_andi_i64(tmp, outptr, 7); + tcg_gen_brcondi_i64(TCG_COND_NE, tmp, 5, fail_label); + + /* Push a valid cap to the old stack. */ + tcg_gen_andi_i64(outptr, outptr, ~7); + tcg_gen_addi_i64(outptr, outptr, -8); + tcg_gen_deposit_i64(tmp, outptr, tcg_constant_i64(1), 0, 12); + tcg_gen_qemu_st_i64(tmp, outptr, mmuidx, mop); + tcg_gen_mb(TCG_BAR_SC | TCG_MO_ALL); + + /* Pop the in-progress cap from the new stack. */ + tcg_gen_addi_i64(gcspr, gcspr, 8); + + /* Return a pointer to the old stack cap. */ + tcg_gen_mov_i64(cpu_reg(s, rt), outptr); +} + +/* + * Look up @key, returning the cpreg, which must exist. + * Additionally, the new cpreg must also be accessible. + */ +static const ARMCPRegInfo * +redirect_cpreg(DisasContext *s, uint32_t key, bool isread) +{ + const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key); + assert(ri); + assert(cp_access_ok(s->current_el, ri, isread)); + return ri; +} + /* MRS - move from system register * MSR (register) - move to system register * SYS @@ -2460,8 +2761,7 @@ static void handle_sys(DisasContext *s, bool isread, unsigned int op0, unsigned int op1, unsigned int op2, unsigned int crn, unsigned int crm, unsigned int rt) { - uint32_t key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP, - crn, crm, op0, op1, op2); + uint32_t key = ENCODE_AA64_CP_REG(op0, op1, crn, crm, op2); const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key); bool need_exit_tb = false; bool nv_trap_to_el2 = false; @@ -2555,6 +2855,27 @@ static void handle_sys(DisasContext *s, bool isread, } } + if (ri->vhe_redir_to_el2 && s->current_el == 2 && s->e2h) { + /* + * This one of the FOO_EL1 registers which redirect to FOO_EL2 + * from EL2 when HCR_EL2.E2H is set. + */ + key = ri->vhe_redir_to_el2; + ri = redirect_cpreg(s, key, isread); + } else if (ri->vhe_redir_to_el01 && s->current_el >= 2) { + /* + * This is one of the FOO_EL12 or FOO_EL02 registers. + * With !E2H, they all UNDEF. + * With E2H, from EL2 or EL3, they redirect to FOO_EL1/FOO_EL0. + */ + if (!s->e2h) { + gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt); + return; + } + key = ri->vhe_redir_to_el01; + ri = redirect_cpreg(s, key, isread); + } + if (ri->accessfn || (ri->fgt && s->fgt_active)) { /* Emit code to perform further access permissions checks at * runtime; this may result in an exception. @@ -2597,11 +2918,8 @@ static void handle_sys(DisasContext *s, bool isread, * We don't use the EL1 register's access function, and * fine-grained-traps on EL1 also do not apply here. */ - key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP, - crn, crm, op0, 0, op2); - ri = get_arm_cp_reginfo(s->cp_regs, key); - assert(ri); - assert(cp_access_ok(s->current_el, ri, isread)); + key = ENCODE_AA64_CP_REG(op0, 0, crn, crm, op2); + ri = redirect_cpreg(s, key, isread); /* * We might not have done an update_pc earlier, so check we don't * need it. We could support this in future if necessary. @@ -2725,6 +3043,51 @@ static void handle_sys(DisasContext *s, bool isread, } } return; + case ARM_CP_GCSPUSHM: + if (s->gcs_en) { + gen_add_gcs_record(s, cpu_reg(s, rt)); + } + return; + case ARM_CP_GCSPOPM: + /* Note that X[rt] is unchanged if !GCSEnabled. */ + if (s->gcs_en) { + gen_gcspopm(s, rt); + } + return; + case ARM_CP_GCSPUSHX: + /* Choose the CONSTRAINED UNPREDICTABLE for UNDEF. */ + if (rt != 31) { + unallocated_encoding(s); + } else if (s->gcs_en) { + gen_gcspushx(s); + } + return; + case ARM_CP_GCSPOPCX: + /* Choose the CONSTRAINED UNPREDICTABLE for UNDEF. */ + if (rt != 31) { + unallocated_encoding(s); + } else if (s->gcs_en) { + gen_gcspopcx(s); + } + return; + case ARM_CP_GCSPOPX: + /* Choose the CONSTRAINED UNPREDICTABLE for UNDEF. */ + if (rt != 31) { + unallocated_encoding(s); + } else if (s->gcs_en) { + gen_gcspopx(s); + } + return; + case ARM_CP_GCSSS1: + if (s->gcs_en) { + gen_gcsss1(s, rt); + } + return; + case ARM_CP_GCSSS2: + if (s->gcs_en) { + gen_gcsss2(s, rt); + } + return; default: g_assert_not_reached(); } @@ -3231,7 +3594,7 @@ static bool trans_LDXP(DisasContext *s, arg_stxr *a) static bool trans_CASP(DisasContext *s, arg_CASP *a) { - if (!dc_isar_feature(aa64_atomics, s)) { + if (!dc_isar_feature(aa64_lse, s)) { return false; } if (((a->rt | a->rs) & 1) != 0) { @@ -3244,7 +3607,7 @@ static bool trans_CASP(DisasContext *s, arg_CASP *a) static bool trans_CAS(DisasContext *s, arg_CAS *a) { - if (!dc_isar_feature(aa64_atomics, s)) { + if (!dc_isar_feature(aa64_lse, s)) { return false; } gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz); @@ -3519,7 +3882,7 @@ static void op_addr_ldst_imm_pre(DisasContext *s, arg_ldst_imm *a, if (!a->p) { tcg_gen_addi_i64(*dirty_addr, *dirty_addr, offset); } - memidx = get_a64_user_mem_index(s, a->unpriv); + memidx = core_a64_user_mem_index(s, a->unpriv); *clean_addr = gen_mte_check1_mmuidx(s, *dirty_addr, is_store, a->w || a->rn != 31, mop, a->unpriv, memidx); @@ -3540,7 +3903,7 @@ static bool trans_STR_i(DisasContext *s, arg_ldst_imm *a) { bool iss_sf, iss_valid = !a->w; TCGv_i64 clean_addr, dirty_addr, tcg_rt; - int memidx = get_a64_user_mem_index(s, a->unpriv); + int memidx = core_a64_user_mem_index(s, a->unpriv); MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); @@ -3558,7 +3921,7 @@ static bool trans_LDR_i(DisasContext *s, arg_ldst_imm *a) { bool iss_sf, iss_valid = !a->w; TCGv_i64 clean_addr, dirty_addr, tcg_rt; - int memidx = get_a64_user_mem_index(s, a->unpriv); + int memidx = core_a64_user_mem_index(s, a->unpriv); MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); @@ -3737,15 +4100,64 @@ static bool do_atomic_ld(DisasContext *s, arg_atomic *a, AtomicThreeOpFn *fn, return true; } -TRANS_FEAT(LDADD, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_add_i64, 0, false) -TRANS_FEAT(LDCLR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_and_i64, 0, true) -TRANS_FEAT(LDEOR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_xor_i64, 0, false) -TRANS_FEAT(LDSET, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_or_i64, 0, false) -TRANS_FEAT(LDSMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smax_i64, MO_SIGN, false) -TRANS_FEAT(LDSMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smin_i64, MO_SIGN, false) -TRANS_FEAT(LDUMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umax_i64, 0, false) -TRANS_FEAT(LDUMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umin_i64, 0, false) -TRANS_FEAT(SWP, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_xchg_i64, 0, false) +TRANS_FEAT(LDADD, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_add_i64, 0, false) +TRANS_FEAT(LDCLR, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_and_i64, 0, true) +TRANS_FEAT(LDEOR, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_xor_i64, 0, false) +TRANS_FEAT(LDSET, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_or_i64, 0, false) +TRANS_FEAT(LDSMAX, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_smax_i64, MO_SIGN, false) +TRANS_FEAT(LDSMIN, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_smin_i64, MO_SIGN, false) +TRANS_FEAT(LDUMAX, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_umax_i64, 0, false) +TRANS_FEAT(LDUMIN, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_umin_i64, 0, false) +TRANS_FEAT(SWP, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_xchg_i64, 0, false) + +typedef void Atomic128ThreeOpFn(TCGv_i128, TCGv_i64, TCGv_i128, TCGArg, MemOp); + +static bool do_atomic128_ld(DisasContext *s, arg_atomic128 *a, + Atomic128ThreeOpFn *fn, bool invert) +{ + MemOp mop; + int rlo, rhi; + TCGv_i64 clean_addr, tlo, thi; + TCGv_i128 t16; + + if (a->rt == 31 || a->rt2 == 31 || a->rt == a->rt2) { + return false; + } + if (a->rn == 31) { + gen_check_sp_alignment(s); + } + mop = check_atomic_align(s, a->rn, MO_128); + clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false, + a->rn != 31, mop); + + rlo = (s->be_data == MO_LE ? a->rt : a->rt2); + rhi = (s->be_data == MO_LE ? a->rt2 : a->rt); + + tlo = read_cpu_reg(s, rlo, true); + thi = read_cpu_reg(s, rhi, true); + if (invert) { + tcg_gen_not_i64(tlo, tlo); + tcg_gen_not_i64(thi, thi); + } + /* + * The tcg atomic primitives are all full barriers. Therefore we + * can ignore the Acquire and Release bits of this instruction. + */ + t16 = tcg_temp_new_i128(); + tcg_gen_concat_i64_i128(t16, tlo, thi); + + fn(t16, clean_addr, t16, get_mem_index(s), mop); + + tcg_gen_extr_i128_i64(cpu_reg(s, rlo), cpu_reg(s, rhi), t16); + return true; +} + +TRANS_FEAT(LDCLRP, aa64_lse128, do_atomic128_ld, + a, tcg_gen_atomic_fetch_and_i128, true) +TRANS_FEAT(LDSETP, aa64_lse128, do_atomic128_ld, + a, tcg_gen_atomic_fetch_or_i128, false) +TRANS_FEAT(SWPP, aa64_lse128, do_atomic128_ld, + a, tcg_gen_atomic_xchg_i128, false) static bool trans_LDAPR(DisasContext *s, arg_LDAPR *a) { @@ -3753,7 +4165,7 @@ static bool trans_LDAPR(DisasContext *s, arg_LDAPR *a) TCGv_i64 clean_addr; MemOp mop; - if (!dc_isar_feature(aa64_atomics, s) || + if (!dc_isar_feature(aa64_lse, s) || !dc_isar_feature(aa64_rcpc_8_3, s)) { return false; } @@ -3876,6 +4288,42 @@ static bool trans_STLR_i(DisasContext *s, arg_ldapr_stlr_i *a) return true; } +static bool trans_GCSSTR(DisasContext *s, arg_GCSSTR *a) +{ + ARMMMUIdx armidx; + + if (!dc_isar_feature(aa64_gcs, s)) { + return false; + } + + /* + * The pseudocode for GCSSTTR is + * + * effective_el = AArch64.IsUnprivAccessPriv() ? PSTATE.EL : EL0; + * if (effective_el == PSTATE.EL) CheckGCSSTREnabled(); + * + * We have cached the result of IsUnprivAccessPriv in DisasContext, + * but since we need the result of full_a64_user_mem_index anyway, + * use the mmu_idx test as a proxy for the effective_el test. + */ + armidx = full_a64_user_mem_index(s, a->unpriv); + if (armidx == s->mmu_idx && s->gcsstr_el != 0) { + gen_exception_insn_el(s, 0, EXCP_UDEF, + syn_gcs_gcsstr(a->rn, a->rt), + s->gcsstr_el); + return true; + } + + if (a->rn == 31) { + gen_check_sp_alignment(s); + } + tcg_gen_qemu_st_i64(cpu_reg(s, a->rt), + clean_data_tbi(s, cpu_reg_sp(s, a->rn)), + core_gcs_mem_index(armidx), + finalize_memop(s, MO_64 | MO_ALIGN)); + return true; +} + static bool trans_LD_mult(DisasContext *s, arg_ldst_mult *a) { TCGv_i64 clean_addr, tcg_rn, tcg_ebytes; @@ -4407,7 +4855,7 @@ static bool do_SET(DisasContext *s, arg_set *a, bool is_epilogue, return false; } - memidx = get_a64_user_mem_index(s, a->unpriv); + memidx = core_a64_user_mem_index(s, a->unpriv); /* * We pass option_a == true, matching our implementation; @@ -4461,8 +4909,8 @@ static bool do_CPY(DisasContext *s, arg_cpy *a, bool is_epilogue, CpyFn fn) return false; } - rmemidx = get_a64_user_mem_index(s, runpriv); - wmemidx = get_a64_user_mem_index(s, wunpriv); + rmemidx = core_a64_user_mem_index(s, runpriv); + wmemidx = core_a64_user_mem_index(s, wunpriv); /* * We pass option_a == true, matching our implementation; @@ -4547,6 +4995,50 @@ TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC) TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC) /* + * Min/Max (immediate) + */ + +static void gen_wrap3_i32(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, NeonGenTwoOpFn fn) +{ + TCGv_i32 t1 = tcg_temp_new_i32(); + TCGv_i32 t2 = tcg_temp_new_i32(); + + tcg_gen_extrl_i64_i32(t1, n); + tcg_gen_extrl_i64_i32(t2, m); + fn(t1, t1, t2); + tcg_gen_extu_i32_i64(d, t1); +} + +static void gen_smax32_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) +{ + gen_wrap3_i32(d, n, m, tcg_gen_smax_i32); +} + +static void gen_smin32_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) +{ + gen_wrap3_i32(d, n, m, tcg_gen_smin_i32); +} + +static void gen_umax32_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) +{ + gen_wrap3_i32(d, n, m, tcg_gen_umax_i32); +} + +static void gen_umin32_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) +{ + gen_wrap3_i32(d, n, m, tcg_gen_umin_i32); +} + +TRANS_FEAT(SMAX_i, aa64_cssc, gen_rri, a, 0, 0, + a->sf ? tcg_gen_smax_i64 : gen_smax32_i64) +TRANS_FEAT(SMIN_i, aa64_cssc, gen_rri, a, 0, 0, + a->sf ? tcg_gen_smin_i64 : gen_smin32_i64) +TRANS_FEAT(UMAX_i, aa64_cssc, gen_rri, a, 0, 0, + a->sf ? tcg_gen_umax_i64 : gen_umax32_i64) +TRANS_FEAT(UMIN_i, aa64_cssc, gen_rri, a, 0, 0, + a->sf ? tcg_gen_umin_i64 : gen_umin32_i64) + +/* * Add/subtract (immediate, with tags) */ @@ -6108,9 +6600,9 @@ static bool do_dot_vector_env(DisasContext *s, arg_qrrr_e *a, return true; } -TRANS_FEAT(SDOT_v, aa64_dp, do_dot_vector, a, gen_helper_gvec_sdot_b) -TRANS_FEAT(UDOT_v, aa64_dp, do_dot_vector, a, gen_helper_gvec_udot_b) -TRANS_FEAT(USDOT_v, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_usdot_b) +TRANS_FEAT(SDOT_v, aa64_dp, do_dot_vector, a, gen_helper_gvec_sdot_4b) +TRANS_FEAT(UDOT_v, aa64_dp, do_dot_vector, a, gen_helper_gvec_udot_4b) +TRANS_FEAT(USDOT_v, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_usdot_4b) TRANS_FEAT(BFDOT_v, aa64_bf16, do_dot_vector_env, a, gen_helper_gvec_bfdot) TRANS_FEAT(BFMMLA, aa64_bf16, do_dot_vector_env, a, gen_helper_gvec_bfmmla) TRANS_FEAT(SMMLA, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_smmla_b) @@ -6870,12 +7362,12 @@ static bool do_dot_vector_idx_env(DisasContext *s, arg_qrrx_e *a, return true; } -TRANS_FEAT(SDOT_vi, aa64_dp, do_dot_vector_idx, a, gen_helper_gvec_sdot_idx_b) -TRANS_FEAT(UDOT_vi, aa64_dp, do_dot_vector_idx, a, gen_helper_gvec_udot_idx_b) +TRANS_FEAT(SDOT_vi, aa64_dp, do_dot_vector_idx, a, gen_helper_gvec_sdot_idx_4b) +TRANS_FEAT(UDOT_vi, aa64_dp, do_dot_vector_idx, a, gen_helper_gvec_udot_idx_4b) TRANS_FEAT(SUDOT_vi, aa64_i8mm, do_dot_vector_idx, a, - gen_helper_gvec_sudot_idx_b) + gen_helper_gvec_sudot_idx_4b) TRANS_FEAT(USDOT_vi, aa64_i8mm, do_dot_vector_idx, a, - gen_helper_gvec_usdot_idx_b) + gen_helper_gvec_usdot_idx_4b) TRANS_FEAT(BFDOT_vi, aa64_bf16, do_dot_vector_idx_env, a, gen_helper_gvec_bfdot_idx) @@ -8151,6 +8643,28 @@ static bool trans_PACGA(DisasContext *s, arg_rrr *a) return false; } +static bool gen_rrr(DisasContext *s, arg_rrr_sf *a, ArithTwoOp fn) +{ + TCGv_i64 tcg_rm = cpu_reg(s, a->rm); + TCGv_i64 tcg_rn = cpu_reg(s, a->rn); + TCGv_i64 tcg_rd = cpu_reg(s, a->rd); + + fn(tcg_rd, tcg_rn, tcg_rm); + if (!a->sf) { + tcg_gen_ext32u_i64(tcg_rd, tcg_rd); + } + return true; +} + +TRANS_FEAT(SMAX, aa64_cssc, gen_rrr, a, + a->sf ? tcg_gen_smax_i64 : gen_smax32_i64) +TRANS_FEAT(SMIN, aa64_cssc, gen_rrr, a, + a->sf ? tcg_gen_smin_i64 : gen_smin32_i64) +TRANS_FEAT(UMAX, aa64_cssc, gen_rrr, a, + a->sf ? tcg_gen_umax_i64 : gen_umax32_i64) +TRANS_FEAT(UMIN, aa64_cssc, gen_rrr, a, + a->sf ? tcg_gen_umin_i64 : gen_umin32_i64) + typedef void ArithOneOp(TCGv_i64, TCGv_i64); static bool gen_rr(DisasContext *s, int rd, int rn, ArithOneOp fn) @@ -8159,13 +8673,22 @@ static bool gen_rr(DisasContext *s, int rd, int rn, ArithOneOp fn) return true; } -static void gen_rbit32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +/* + * Perform 32-bit operation fn on the low half of n; + * the high half of the output is zeroed. + */ +static void gen_wrap2_i32(TCGv_i64 d, TCGv_i64 n, NeonGenOneOpFn fn) { - TCGv_i32 t32 = tcg_temp_new_i32(); + TCGv_i32 t = tcg_temp_new_i32(); - tcg_gen_extrl_i64_i32(t32, tcg_rn); - gen_helper_rbit(t32, t32); - tcg_gen_extu_i32_i64(tcg_rd, t32); + tcg_gen_extrl_i64_i32(t, n); + fn(t, t); + tcg_gen_extu_i32_i64(d, t); +} + +static void gen_rbit32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +{ + gen_wrap2_i32(tcg_rd, tcg_rn, gen_helper_rbit); } static void gen_rev16_xx(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn, TCGv_i64 mask) @@ -8221,15 +8744,42 @@ static void gen_clz64(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) static void gen_cls32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) { + gen_wrap2_i32(tcg_rd, tcg_rn, tcg_gen_clrsb_i32); +} + +TRANS(CLZ, gen_rr, a->rd, a->rn, a->sf ? gen_clz64 : gen_clz32) +TRANS(CLS, gen_rr, a->rd, a->rn, a->sf ? tcg_gen_clrsb_i64 : gen_cls32) + +static void gen_ctz32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +{ TCGv_i32 t32 = tcg_temp_new_i32(); tcg_gen_extrl_i64_i32(t32, tcg_rn); - tcg_gen_clrsb_i32(t32, t32); + tcg_gen_ctzi_i32(t32, t32, 32); tcg_gen_extu_i32_i64(tcg_rd, t32); } -TRANS(CLZ, gen_rr, a->rd, a->rn, a->sf ? gen_clz64 : gen_clz32) -TRANS(CLS, gen_rr, a->rd, a->rn, a->sf ? tcg_gen_clrsb_i64 : gen_cls32) +static void gen_ctz64(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +{ + tcg_gen_ctzi_i64(tcg_rd, tcg_rn, 64); +} + +static void gen_cnt32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +{ + gen_wrap2_i32(tcg_rd, tcg_rn, tcg_gen_ctpop_i32); +} + +static void gen_abs32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn) +{ + gen_wrap2_i32(tcg_rd, tcg_rn, tcg_gen_abs_i32); +} + +TRANS_FEAT(CTZ, aa64_cssc, gen_rr, a->rd, a->rn, + a->sf ? gen_ctz64 : gen_ctz32) +TRANS_FEAT(CNT, aa64_cssc, gen_rr, a->rd, a->rn, + a->sf ? tcg_gen_ctpop_i64 : gen_cnt32) +TRANS_FEAT(ABS, aa64_cssc, gen_rr, a->rd, a->rn, + a->sf ? tcg_gen_abs_i64 : gen_abs32) static bool gen_pacaut(DisasContext *s, arg_pacaut *a, NeonGenTwo64OpEnvFn fn) { @@ -8600,7 +9150,7 @@ static bool trans_CCMP(DisasContext *s, arg_CCMP *a) tcg_gen_subi_i32(tcg_t2, tcg_t0, 1); nzcv = a->nzcv; - has_andc = tcg_op_supported(INDEX_op_andc_i32, TCG_TYPE_I32, 0); + has_andc = tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0); if (nzcv & 8) { /* N */ tcg_gen_or_i32(cpu_NF, cpu_NF, tcg_t1); } else { @@ -10133,8 +10683,10 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, dc->trap_eret = EX_TBFLAG_A64(tb_flags, TRAP_ERET); dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL); dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL); + dc->zt0_excp_el = EX_TBFLAG_A64(tb_flags, ZT0EXC_EL); dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16; dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16; + dc->max_svl = arm_cpu->sme_max_vq * 16; dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE); dc->bt = EX_TBFLAG_A64(tb_flags, BT); dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); @@ -10147,13 +10699,17 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA); dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING); dc->naa = EX_TBFLAG_A64(tb_flags, NAA); + dc->e2h = EX_TBFLAG_A64(tb_flags, E2H); dc->nv = EX_TBFLAG_A64(tb_flags, NV); dc->nv1 = EX_TBFLAG_A64(tb_flags, NV1); dc->nv2 = EX_TBFLAG_A64(tb_flags, NV2); - dc->nv2_mem_e20 = EX_TBFLAG_A64(tb_flags, NV2_MEM_E20); + dc->nv2_mem_e20 = dc->nv2 && dc->e2h; dc->nv2_mem_be = EX_TBFLAG_A64(tb_flags, NV2_MEM_BE); dc->fpcr_ah = EX_TBFLAG_A64(tb_flags, AH); dc->fpcr_nep = EX_TBFLAG_A64(tb_flags, NEP); + dc->gcs_en = EX_TBFLAG_A64(tb_flags, GCS_EN); + dc->gcs_rvcen = EX_TBFLAG_A64(tb_flags, GCS_RVCEN); + dc->gcsstr_el = EX_TBFLAG_A64(tb_flags, GCSSTR_EL); dc->vec_len = 0; dc->vec_stride = 0; dc->cp_regs = arm_cpu->cp_regs; @@ -10247,7 +10803,7 @@ static void aarch64_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) * start of the TB. */ assert(s->base.num_insns == 1); - gen_helper_exception_pc_alignment(tcg_env, tcg_constant_tl(pc)); + gen_helper_exception_pc_alignment(tcg_env, tcg_constant_vaddr(pc)); s->base.is_jmp = DISAS_NORETURN; s->base.pc_next = QEMU_ALIGN_UP(pc, 4); return; @@ -10380,6 +10936,8 @@ static void aarch64_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu) break; } } + + emit_delayed_exceptions(dc); } const TranslatorOps aarch64_translator_ops = { |