From 139c1837db7eaee53e1c441629b5bcc159e1deb0 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 4 Feb 2020 12:41:01 +0100 Subject: meson: rename included C source files to .c.inc With Makefiles that have automatically generated dependencies, you generated includes are set as dependencies of the Makefile, so that they are built before everything else and they are available when first building the .c files. Alternatively you can use a fine-grained dependency, e.g. target/arm/translate.o: target/arm/decode-neon-shared.inc.c With Meson you have only one choice and it is a third option, namely "build at the beginning of the corresponding target"; the way you express it is to list the includes in the sources of that target. The problem is that Meson decides if something is a source vs. a generated include by looking at the extension: '.c', '.cc', '.m', '.C' are sources, while everything else is considered an include---including '.inc.c'. Use '.c.inc' to avoid this, as it is consistent with our other convention of using '.rst.inc' for included reStructuredText files. The editorconfig file is adjusted. Signed-off-by: Paolo Bonzini --- target/arm/Makefile.objs | 40 +- target/arm/translate-neon.c.inc | 4161 +++++++++++++++++++++++++++++++++++++++ target/arm/translate-neon.inc.c | 4161 --------------------------------------- target/arm/translate-sve.c | 2 +- target/arm/translate-vfp.c.inc | 2865 +++++++++++++++++++++++++++ target/arm/translate-vfp.inc.c | 2865 --------------------------- target/arm/translate.c | 12 +- 7 files changed, 7053 insertions(+), 7053 deletions(-) create mode 100644 target/arm/translate-neon.c.inc delete mode 100644 target/arm/translate-neon.inc.c create mode 100644 target/arm/translate-vfp.c.inc delete mode 100644 target/arm/translate-vfp.inc.c (limited to 'target/arm') diff --git a/target/arm/Makefile.objs b/target/arm/Makefile.objs index fa39fd7..317eed9 100644 --- a/target/arm/Makefile.objs +++ b/target/arm/Makefile.objs @@ -13,66 +13,66 @@ obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o DECODETREE = $(SRC_PATH)/scripts/decodetree.py -target/arm/decode-sve.inc.c: $(SRC_PATH)/target/arm/sve.decode $(DECODETREE) +target/arm/decode-sve.c.inc: $(SRC_PATH)/target/arm/sve.decode $(DECODETREE) $(call quiet-command,\ $(PYTHON) $(DECODETREE) --decode disas_sve -o $@ $<,\ "GEN", $(TARGET_DIR)$@) -target/arm/decode-neon-shared.inc.c: $(SRC_PATH)/target/arm/neon-shared.decode $(DECODETREE) +target/arm/decode-neon-shared.c.inc: $(SRC_PATH)/target/arm/neon-shared.decode $(DECODETREE) $(call quiet-command,\ $(PYTHON) $(DECODETREE) --static-decode disas_neon_shared -o $@ $<,\ "GEN", $(TARGET_DIR)$@) -target/arm/decode-neon-dp.inc.c: $(SRC_PATH)/target/arm/neon-dp.decode $(DECODETREE) +target/arm/decode-neon-dp.c.inc: $(SRC_PATH)/target/arm/neon-dp.decode $(DECODETREE) $(call quiet-command,\ $(PYTHON) $(DECODETREE) --static-decode disas_neon_dp -o $@ $<,\ "GEN", $(TARGET_DIR)$@) -target/arm/decode-neon-ls.inc.c: $(SRC_PATH)/target/arm/neon-ls.decode $(DECODETREE) +target/arm/decode-neon-ls.c.inc: $(SRC_PATH)/target/arm/neon-ls.decode $(DECODETREE) $(call quiet-command,\ $(PYTHON) $(DECODETREE) --static-decode disas_neon_ls -o $@ $<,\ "GEN", $(TARGET_DIR)$@) -target/arm/decode-vfp.inc.c: $(SRC_PATH)/target/arm/vfp.decode $(DECODETREE) +target/arm/decode-vfp.c.inc: $(SRC_PATH)/target/arm/vfp.decode $(DECODETREE) $(call quiet-command,\ $(PYTHON) $(DECODETREE) --static-decode disas_vfp -o $@ $<,\ "GEN", $(TARGET_DIR)$@) -target/arm/decode-vfp-uncond.inc.c: $(SRC_PATH)/target/arm/vfp-uncond.decode $(DECODETREE) +target/arm/decode-vfp-uncond.c.inc: $(SRC_PATH)/target/arm/vfp-uncond.decode $(DECODETREE) $(call quiet-command,\ $(PYTHON) $(DECODETREE) --static-decode disas_vfp_uncond -o $@ $<,\ "GEN", $(TARGET_DIR)$@) -target/arm/decode-a32.inc.c: $(SRC_PATH)/target/arm/a32.decode $(DECODETREE) +target/arm/decode-a32.c.inc: $(SRC_PATH)/target/arm/a32.decode $(DECODETREE) $(call quiet-command,\ $(PYTHON) $(DECODETREE) --static-decode disas_a32 -o $@ $<,\ "GEN", $(TARGET_DIR)$@) -target/arm/decode-a32-uncond.inc.c: $(SRC_PATH)/target/arm/a32-uncond.decode $(DECODETREE) +target/arm/decode-a32-uncond.c.inc: $(SRC_PATH)/target/arm/a32-uncond.decode $(DECODETREE) $(call quiet-command,\ $(PYTHON) $(DECODETREE) --static-decode disas_a32_uncond -o $@ $<,\ "GEN", $(TARGET_DIR)$@) -target/arm/decode-t32.inc.c: $(SRC_PATH)/target/arm/t32.decode $(DECODETREE) +target/arm/decode-t32.c.inc: $(SRC_PATH)/target/arm/t32.decode $(DECODETREE) $(call quiet-command,\ $(PYTHON) $(DECODETREE) --static-decode disas_t32 -o $@ $<,\ "GEN", $(TARGET_DIR)$@) -target/arm/decode-t16.inc.c: $(SRC_PATH)/target/arm/t16.decode $(DECODETREE) +target/arm/decode-t16.c.inc: $(SRC_PATH)/target/arm/t16.decode $(DECODETREE) $(call quiet-command,\ $(PYTHON) $(DECODETREE) -w 16 --static-decode disas_t16 -o $@ $<,\ "GEN", $(TARGET_DIR)$@) -target/arm/translate-sve.o: target/arm/decode-sve.inc.c -target/arm/translate.o: target/arm/decode-neon-shared.inc.c -target/arm/translate.o: target/arm/decode-neon-dp.inc.c -target/arm/translate.o: target/arm/decode-neon-ls.inc.c -target/arm/translate.o: target/arm/decode-vfp.inc.c -target/arm/translate.o: target/arm/decode-vfp-uncond.inc.c -target/arm/translate.o: target/arm/decode-a32.inc.c -target/arm/translate.o: target/arm/decode-a32-uncond.inc.c -target/arm/translate.o: target/arm/decode-t32.inc.c -target/arm/translate.o: target/arm/decode-t16.inc.c +target/arm/translate-sve.o: target/arm/decode-sve.c.inc +target/arm/translate.o: target/arm/decode-neon-shared.c.inc +target/arm/translate.o: target/arm/decode-neon-dp.c.inc +target/arm/translate.o: target/arm/decode-neon-ls.c.inc +target/arm/translate.o: target/arm/decode-vfp.c.inc +target/arm/translate.o: target/arm/decode-vfp-uncond.c.inc +target/arm/translate.o: target/arm/decode-a32.c.inc +target/arm/translate.o: target/arm/decode-a32-uncond.c.inc +target/arm/translate.o: target/arm/decode-t32.c.inc +target/arm/translate.o: target/arm/decode-t16.c.inc obj-y += tlb_helper.o debug_helper.o obj-y += translate.o op_helper.o diff --git a/target/arm/translate-neon.c.inc b/target/arm/translate-neon.c.inc new file mode 100644 index 0000000..8fbe8ce --- /dev/null +++ b/target/arm/translate-neon.c.inc @@ -0,0 +1,4161 @@ +/* + * ARM translation: AArch32 Neon instructions + * + * Copyright (c) 2003 Fabrice Bellard + * Copyright (c) 2005-2007 CodeSourcery + * Copyright (c) 2007 OpenedHand, Ltd. + * Copyright (c) 2020 Linaro, Ltd. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +/* + * This file is intended to be included from translate.c; it uses + * some macros and definitions provided by that file. + * It might be possible to convert it to a standalone .c file eventually. + */ + +static inline int plus1(DisasContext *s, int x) +{ + return x + 1; +} + +static inline int rsub_64(DisasContext *s, int x) +{ + return 64 - x; +} + +static inline int rsub_32(DisasContext *s, int x) +{ + return 32 - x; +} +static inline int rsub_16(DisasContext *s, int x) +{ + return 16 - x; +} +static inline int rsub_8(DisasContext *s, int x) +{ + return 8 - x; +} + +/* Include the generated Neon decoder */ +#include "decode-neon-dp.c.inc" +#include "decode-neon-ls.c.inc" +#include "decode-neon-shared.c.inc" + +/* Return the offset of a 2**SIZE piece of a NEON register, at index ELE, + * where 0 is the least significant end of the register. + */ +static inline long +neon_element_offset(int reg, int element, MemOp size) +{ + int element_size = 1 << size; + int ofs = element * element_size; +#ifdef HOST_WORDS_BIGENDIAN + /* Calculate the offset assuming fully little-endian, + * then XOR to account for the order of the 8-byte units. + */ + if (element_size < 8) { + ofs ^= 8 - element_size; + } +#endif + return neon_reg_offset(reg, 0) + ofs; +} + +static void neon_load_element(TCGv_i32 var, int reg, int ele, MemOp mop) +{ + long offset = neon_element_offset(reg, ele, mop & MO_SIZE); + + switch (mop) { + case MO_UB: + tcg_gen_ld8u_i32(var, cpu_env, offset); + break; + case MO_UW: + tcg_gen_ld16u_i32(var, cpu_env, offset); + break; + case MO_UL: + tcg_gen_ld_i32(var, cpu_env, offset); + break; + default: + g_assert_not_reached(); + } +} + +static void neon_load_element64(TCGv_i64 var, int reg, int ele, MemOp mop) +{ + long offset = neon_element_offset(reg, ele, mop & MO_SIZE); + + switch (mop) { + case MO_UB: + tcg_gen_ld8u_i64(var, cpu_env, offset); + break; + case MO_UW: + tcg_gen_ld16u_i64(var, cpu_env, offset); + break; + case MO_UL: + tcg_gen_ld32u_i64(var, cpu_env, offset); + break; + case MO_Q: + tcg_gen_ld_i64(var, cpu_env, offset); + break; + default: + g_assert_not_reached(); + } +} + +static void neon_store_element(int reg, int ele, MemOp size, TCGv_i32 var) +{ + long offset = neon_element_offset(reg, ele, size); + + switch (size) { + case MO_8: + tcg_gen_st8_i32(var, cpu_env, offset); + break; + case MO_16: + tcg_gen_st16_i32(var, cpu_env, offset); + break; + case MO_32: + tcg_gen_st_i32(var, cpu_env, offset); + break; + default: + g_assert_not_reached(); + } +} + +static void neon_store_element64(int reg, int ele, MemOp size, TCGv_i64 var) +{ + long offset = neon_element_offset(reg, ele, size); + + switch (size) { + case MO_8: + tcg_gen_st8_i64(var, cpu_env, offset); + break; + case MO_16: + tcg_gen_st16_i64(var, cpu_env, offset); + break; + case MO_32: + tcg_gen_st32_i64(var, cpu_env, offset); + break; + case MO_64: + tcg_gen_st_i64(var, cpu_env, offset); + break; + default: + g_assert_not_reached(); + } +} + +static bool trans_VCMLA(DisasContext *s, arg_VCMLA *a) +{ + int opr_sz; + TCGv_ptr fpst; + gen_helper_gvec_3_ptr *fn_gvec_ptr; + + if (!dc_isar_feature(aa32_vcma, s) + || (!a->size && !dc_isar_feature(aa32_fp16_arith, s))) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if ((a->vn | a->vm | a->vd) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + opr_sz = (1 + a->q) * 8; + fpst = get_fpstatus_ptr(1); + fn_gvec_ptr = a->size ? gen_helper_gvec_fcmlas : gen_helper_gvec_fcmlah; + tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), + vfp_reg_offset(1, a->vn), + vfp_reg_offset(1, a->vm), + fpst, opr_sz, opr_sz, a->rot, + fn_gvec_ptr); + tcg_temp_free_ptr(fpst); + return true; +} + +static bool trans_VCADD(DisasContext *s, arg_VCADD *a) +{ + int opr_sz; + TCGv_ptr fpst; + gen_helper_gvec_3_ptr *fn_gvec_ptr; + + if (!dc_isar_feature(aa32_vcma, s) + || (!a->size && !dc_isar_feature(aa32_fp16_arith, s))) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if ((a->vn | a->vm | a->vd) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + opr_sz = (1 + a->q) * 8; + fpst = get_fpstatus_ptr(1); + fn_gvec_ptr = a->size ? gen_helper_gvec_fcadds : gen_helper_gvec_fcaddh; + tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), + vfp_reg_offset(1, a->vn), + vfp_reg_offset(1, a->vm), + fpst, opr_sz, opr_sz, a->rot, + fn_gvec_ptr); + tcg_temp_free_ptr(fpst); + return true; +} + +static bool trans_VDOT(DisasContext *s, arg_VDOT *a) +{ + int opr_sz; + gen_helper_gvec_3 *fn_gvec; + + if (!dc_isar_feature(aa32_dp, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if ((a->vn | a->vm | a->vd) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + opr_sz = (1 + a->q) * 8; + fn_gvec = a->u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b; + tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd), + vfp_reg_offset(1, a->vn), + vfp_reg_offset(1, a->vm), + opr_sz, opr_sz, 0, fn_gvec); + return true; +} + +static bool trans_VFML(DisasContext *s, arg_VFML *a) +{ + int opr_sz; + + if (!dc_isar_feature(aa32_fhm, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + (a->vd & 0x10)) { + return false; + } + + if (a->vd & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + opr_sz = (1 + a->q) * 8; + tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), + vfp_reg_offset(a->q, a->vn), + vfp_reg_offset(a->q, a->vm), + cpu_env, opr_sz, opr_sz, a->s, /* is_2 == 0 */ + gen_helper_gvec_fmlal_a32); + return true; +} + +static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a) +{ + gen_helper_gvec_3_ptr *fn_gvec_ptr; + int opr_sz; + TCGv_ptr fpst; + + if (!dc_isar_feature(aa32_vcma, s)) { + return false; + } + if (a->size == 0 && !dc_isar_feature(aa32_fp16_arith, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if ((a->vd | a->vn) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fn_gvec_ptr = (a->size ? gen_helper_gvec_fcmlas_idx + : gen_helper_gvec_fcmlah_idx); + opr_sz = (1 + a->q) * 8; + fpst = get_fpstatus_ptr(1); + tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), + vfp_reg_offset(1, a->vn), + vfp_reg_offset(1, a->vm), + fpst, opr_sz, opr_sz, + (a->index << 2) | a->rot, fn_gvec_ptr); + tcg_temp_free_ptr(fpst); + return true; +} + +static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a) +{ + gen_helper_gvec_3 *fn_gvec; + int opr_sz; + TCGv_ptr fpst; + + if (!dc_isar_feature(aa32_dp, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn) & 0x10)) { + return false; + } + + if ((a->vd | a->vn) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fn_gvec = a->u ? gen_helper_gvec_udot_idx_b : gen_helper_gvec_sdot_idx_b; + opr_sz = (1 + a->q) * 8; + fpst = get_fpstatus_ptr(1); + tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd), + vfp_reg_offset(1, a->vn), + vfp_reg_offset(1, a->rm), + opr_sz, opr_sz, a->index, fn_gvec); + tcg_temp_free_ptr(fpst); + return true; +} + +static bool trans_VFML_scalar(DisasContext *s, arg_VFML_scalar *a) +{ + int opr_sz; + + if (!dc_isar_feature(aa32_fhm, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd & 0x10) || (a->q && (a->vn & 0x10)))) { + return false; + } + + if (a->vd & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + opr_sz = (1 + a->q) * 8; + tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), + vfp_reg_offset(a->q, a->vn), + vfp_reg_offset(a->q, a->rm), + cpu_env, opr_sz, opr_sz, + (a->index << 2) | a->s, /* is_2 == 0 */ + gen_helper_gvec_fmlal_idx_a32); + return true; +} + +static struct { + int nregs; + int interleave; + int spacing; +} const neon_ls_element_type[11] = { + {1, 4, 1}, + {1, 4, 2}, + {4, 1, 1}, + {2, 2, 2}, + {1, 3, 1}, + {1, 3, 2}, + {3, 1, 1}, + {1, 1, 1}, + {1, 2, 1}, + {1, 2, 2}, + {2, 1, 1} +}; + +static void gen_neon_ldst_base_update(DisasContext *s, int rm, int rn, + int stride) +{ + if (rm != 15) { + TCGv_i32 base; + + base = load_reg(s, rn); + if (rm == 13) { + tcg_gen_addi_i32(base, base, stride); + } else { + TCGv_i32 index; + index = load_reg(s, rm); + tcg_gen_add_i32(base, base, index); + tcg_temp_free_i32(index); + } + store_reg(s, rn, base); + } +} + +static bool trans_VLDST_multiple(DisasContext *s, arg_VLDST_multiple *a) +{ + /* Neon load/store multiple structures */ + int nregs, interleave, spacing, reg, n; + MemOp endian = s->be_data; + int mmu_idx = get_mem_index(s); + int size = a->size; + TCGv_i64 tmp64; + TCGv_i32 addr, tmp; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { + return false; + } + if (a->itype > 10) { + return false; + } + /* Catch UNDEF cases for bad values of align field */ + switch (a->itype & 0xc) { + case 4: + if (a->align >= 2) { + return false; + } + break; + case 8: + if (a->align == 3) { + return false; + } + break; + default: + break; + } + nregs = neon_ls_element_type[a->itype].nregs; + interleave = neon_ls_element_type[a->itype].interleave; + spacing = neon_ls_element_type[a->itype].spacing; + if (size == 3 && (interleave | spacing) != 1) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + /* For our purposes, bytes are always little-endian. */ + if (size == 0) { + endian = MO_LE; + } + /* + * Consecutive little-endian elements from a single register + * can be promoted to a larger little-endian operation. + */ + if (interleave == 1 && endian == MO_LE) { + size = 3; + } + tmp64 = tcg_temp_new_i64(); + addr = tcg_temp_new_i32(); + tmp = tcg_const_i32(1 << size); + load_reg_var(s, addr, a->rn); + for (reg = 0; reg < nregs; reg++) { + for (n = 0; n < 8 >> size; n++) { + int xs; + for (xs = 0; xs < interleave; xs++) { + int tt = a->vd + reg + spacing * xs; + + if (a->l) { + gen_aa32_ld_i64(s, tmp64, addr, mmu_idx, endian | size); + neon_store_element64(tt, n, size, tmp64); + } else { + neon_load_element64(tmp64, tt, n, size); + gen_aa32_st_i64(s, tmp64, addr, mmu_idx, endian | size); + } + tcg_gen_add_i32(addr, addr, tmp); + } + } + } + tcg_temp_free_i32(addr); + tcg_temp_free_i32(tmp); + tcg_temp_free_i64(tmp64); + + gen_neon_ldst_base_update(s, a->rm, a->rn, nregs * interleave * 8); + return true; +} + +static bool trans_VLD_all_lanes(DisasContext *s, arg_VLD_all_lanes *a) +{ + /* Neon load single structure to all lanes */ + int reg, stride, vec_size; + int vd = a->vd; + int size = a->size; + int nregs = a->n + 1; + TCGv_i32 addr, tmp; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { + return false; + } + + if (size == 3) { + if (nregs != 4 || a->a == 0) { + return false; + } + /* For VLD4 size == 3 a == 1 means 32 bits at 16 byte alignment */ + size = 2; + } + if (nregs == 1 && a->a == 1 && size == 0) { + return false; + } + if (nregs == 3 && a->a == 1) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + /* + * VLD1 to all lanes: T bit indicates how many Dregs to write. + * VLD2/3/4 to all lanes: T bit indicates register stride. + */ + stride = a->t ? 2 : 1; + vec_size = nregs == 1 ? stride * 8 : 8; + + tmp = tcg_temp_new_i32(); + addr = tcg_temp_new_i32(); + load_reg_var(s, addr, a->rn); + for (reg = 0; reg < nregs; reg++) { + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), + s->be_data | size); + if ((vd & 1) && vec_size == 16) { + /* + * We cannot write 16 bytes at once because the + * destination is unaligned. + */ + tcg_gen_gvec_dup_i32(size, neon_reg_offset(vd, 0), + 8, 8, tmp); + tcg_gen_gvec_mov(0, neon_reg_offset(vd + 1, 0), + neon_reg_offset(vd, 0), 8, 8); + } else { + tcg_gen_gvec_dup_i32(size, neon_reg_offset(vd, 0), + vec_size, vec_size, tmp); + } + tcg_gen_addi_i32(addr, addr, 1 << size); + vd += stride; + } + tcg_temp_free_i32(tmp); + tcg_temp_free_i32(addr); + + gen_neon_ldst_base_update(s, a->rm, a->rn, (1 << size) * nregs); + + return true; +} + +static bool trans_VLDST_single(DisasContext *s, arg_VLDST_single *a) +{ + /* Neon load/store single structure to one lane */ + int reg; + int nregs = a->n + 1; + int vd = a->vd; + TCGv_i32 addr, tmp; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { + return false; + } + + /* Catch the UNDEF cases. This is unavoidably a bit messy. */ + switch (nregs) { + case 1: + if (((a->align & (1 << a->size)) != 0) || + (a->size == 2 && ((a->align & 3) == 1 || (a->align & 3) == 2))) { + return false; + } + break; + case 3: + if ((a->align & 1) != 0) { + return false; + } + /* fall through */ + case 2: + if (a->size == 2 && (a->align & 2) != 0) { + return false; + } + break; + case 4: + if ((a->size == 2) && ((a->align & 3) == 3)) { + return false; + } + break; + default: + abort(); + } + if ((vd + a->stride * (nregs - 1)) > 31) { + /* + * Attempts to write off the end of the register file are + * UNPREDICTABLE; we choose to UNDEF because otherwise we would + * access off the end of the array that holds the register data. + */ + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + tmp = tcg_temp_new_i32(); + addr = tcg_temp_new_i32(); + load_reg_var(s, addr, a->rn); + /* + * TODO: if we implemented alignment exceptions, we should check + * addr against the alignment encoded in a->align here. + */ + for (reg = 0; reg < nregs; reg++) { + if (a->l) { + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), + s->be_data | a->size); + neon_store_element(vd, a->reg_idx, a->size, tmp); + } else { /* Store */ + neon_load_element(tmp, vd, a->reg_idx, a->size); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), + s->be_data | a->size); + } + vd += a->stride; + tcg_gen_addi_i32(addr, addr, 1 << a->size); + } + tcg_temp_free_i32(addr); + tcg_temp_free_i32(tmp); + + gen_neon_ldst_base_update(s, a->rm, a->rn, (1 << a->size) * nregs); + + return true; +} + +static bool do_3same(DisasContext *s, arg_3same *a, GVecGen3Fn fn) +{ + int vec_size = a->q ? 16 : 8; + int rd_ofs = neon_reg_offset(a->vd, 0); + int rn_ofs = neon_reg_offset(a->vn, 0); + int rm_ofs = neon_reg_offset(a->vm, 0); + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if ((a->vn | a->vm | a->vd) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fn(a->size, rd_ofs, rn_ofs, rm_ofs, vec_size, vec_size); + return true; +} + +#define DO_3SAME(INSN, FUNC) \ + static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ + { \ + return do_3same(s, a, FUNC); \ + } + +DO_3SAME(VADD, tcg_gen_gvec_add) +DO_3SAME(VSUB, tcg_gen_gvec_sub) +DO_3SAME(VAND, tcg_gen_gvec_and) +DO_3SAME(VBIC, tcg_gen_gvec_andc) +DO_3SAME(VORR, tcg_gen_gvec_or) +DO_3SAME(VORN, tcg_gen_gvec_orc) +DO_3SAME(VEOR, tcg_gen_gvec_xor) +DO_3SAME(VSHL_S, gen_gvec_sshl) +DO_3SAME(VSHL_U, gen_gvec_ushl) +DO_3SAME(VQADD_S, gen_gvec_sqadd_qc) +DO_3SAME(VQADD_U, gen_gvec_uqadd_qc) +DO_3SAME(VQSUB_S, gen_gvec_sqsub_qc) +DO_3SAME(VQSUB_U, gen_gvec_uqsub_qc) + +/* These insns are all gvec_bitsel but with the inputs in various orders. */ +#define DO_3SAME_BITSEL(INSN, O1, O2, O3) \ + static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ + uint32_t rn_ofs, uint32_t rm_ofs, \ + uint32_t oprsz, uint32_t maxsz) \ + { \ + tcg_gen_gvec_bitsel(vece, rd_ofs, O1, O2, O3, oprsz, maxsz); \ + } \ + DO_3SAME(INSN, gen_##INSN##_3s) + +DO_3SAME_BITSEL(VBSL, rd_ofs, rn_ofs, rm_ofs) +DO_3SAME_BITSEL(VBIT, rm_ofs, rn_ofs, rd_ofs) +DO_3SAME_BITSEL(VBIF, rm_ofs, rd_ofs, rn_ofs) + +#define DO_3SAME_NO_SZ_3(INSN, FUNC) \ + static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ + { \ + if (a->size == 3) { \ + return false; \ + } \ + return do_3same(s, a, FUNC); \ + } + +DO_3SAME_NO_SZ_3(VMAX_S, tcg_gen_gvec_smax) +DO_3SAME_NO_SZ_3(VMAX_U, tcg_gen_gvec_umax) +DO_3SAME_NO_SZ_3(VMIN_S, tcg_gen_gvec_smin) +DO_3SAME_NO_SZ_3(VMIN_U, tcg_gen_gvec_umin) +DO_3SAME_NO_SZ_3(VMUL, tcg_gen_gvec_mul) +DO_3SAME_NO_SZ_3(VMLA, gen_gvec_mla) +DO_3SAME_NO_SZ_3(VMLS, gen_gvec_mls) +DO_3SAME_NO_SZ_3(VTST, gen_gvec_cmtst) +DO_3SAME_NO_SZ_3(VABD_S, gen_gvec_sabd) +DO_3SAME_NO_SZ_3(VABA_S, gen_gvec_saba) +DO_3SAME_NO_SZ_3(VABD_U, gen_gvec_uabd) +DO_3SAME_NO_SZ_3(VABA_U, gen_gvec_uaba) + +#define DO_3SAME_CMP(INSN, COND) \ + static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ + uint32_t rn_ofs, uint32_t rm_ofs, \ + uint32_t oprsz, uint32_t maxsz) \ + { \ + tcg_gen_gvec_cmp(COND, vece, rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz); \ + } \ + DO_3SAME_NO_SZ_3(INSN, gen_##INSN##_3s) + +DO_3SAME_CMP(VCGT_S, TCG_COND_GT) +DO_3SAME_CMP(VCGT_U, TCG_COND_GTU) +DO_3SAME_CMP(VCGE_S, TCG_COND_GE) +DO_3SAME_CMP(VCGE_U, TCG_COND_GEU) +DO_3SAME_CMP(VCEQ, TCG_COND_EQ) + +#define WRAP_OOL_FN(WRAPNAME, FUNC) \ + static void WRAPNAME(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, \ + uint32_t rm_ofs, uint32_t oprsz, uint32_t maxsz) \ + { \ + tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, 0, FUNC); \ + } + +WRAP_OOL_FN(gen_VMUL_p_3s, gen_helper_gvec_pmul_b) + +static bool trans_VMUL_p_3s(DisasContext *s, arg_3same *a) +{ + if (a->size != 0) { + return false; + } + return do_3same(s, a, gen_VMUL_p_3s); +} + +#define DO_VQRDMLAH(INSN, FUNC) \ + static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ + { \ + if (!dc_isar_feature(aa32_rdm, s)) { \ + return false; \ + } \ + if (a->size != 1 && a->size != 2) { \ + return false; \ + } \ + return do_3same(s, a, FUNC); \ + } + +DO_VQRDMLAH(VQRDMLAH, gen_gvec_sqrdmlah_qc) +DO_VQRDMLAH(VQRDMLSH, gen_gvec_sqrdmlsh_qc) + +#define DO_SHA1(NAME, FUNC) \ + WRAP_OOL_FN(gen_##NAME##_3s, FUNC) \ + static bool trans_##NAME##_3s(DisasContext *s, arg_3same *a) \ + { \ + if (!dc_isar_feature(aa32_sha1, s)) { \ + return false; \ + } \ + return do_3same(s, a, gen_##NAME##_3s); \ + } + +DO_SHA1(SHA1C, gen_helper_crypto_sha1c) +DO_SHA1(SHA1P, gen_helper_crypto_sha1p) +DO_SHA1(SHA1M, gen_helper_crypto_sha1m) +DO_SHA1(SHA1SU0, gen_helper_crypto_sha1su0) + +#define DO_SHA2(NAME, FUNC) \ + WRAP_OOL_FN(gen_##NAME##_3s, FUNC) \ + static bool trans_##NAME##_3s(DisasContext *s, arg_3same *a) \ + { \ + if (!dc_isar_feature(aa32_sha2, s)) { \ + return false; \ + } \ + return do_3same(s, a, gen_##NAME##_3s); \ + } + +DO_SHA2(SHA256H, gen_helper_crypto_sha256h) +DO_SHA2(SHA256H2, gen_helper_crypto_sha256h2) +DO_SHA2(SHA256SU1, gen_helper_crypto_sha256su1) + +#define DO_3SAME_64(INSN, FUNC) \ + static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ + uint32_t rn_ofs, uint32_t rm_ofs, \ + uint32_t oprsz, uint32_t maxsz) \ + { \ + static const GVecGen3 op = { .fni8 = FUNC }; \ + tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &op); \ + } \ + DO_3SAME(INSN, gen_##INSN##_3s) + +#define DO_3SAME_64_ENV(INSN, FUNC) \ + static void gen_##INSN##_elt(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) \ + { \ + FUNC(d, cpu_env, n, m); \ + } \ + DO_3SAME_64(INSN, gen_##INSN##_elt) + +DO_3SAME_64(VRSHL_S64, gen_helper_neon_rshl_s64) +DO_3SAME_64(VRSHL_U64, gen_helper_neon_rshl_u64) +DO_3SAME_64_ENV(VQSHL_S64, gen_helper_neon_qshl_s64) +DO_3SAME_64_ENV(VQSHL_U64, gen_helper_neon_qshl_u64) +DO_3SAME_64_ENV(VQRSHL_S64, gen_helper_neon_qrshl_s64) +DO_3SAME_64_ENV(VQRSHL_U64, gen_helper_neon_qrshl_u64) + +#define DO_3SAME_32(INSN, FUNC) \ + static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ + uint32_t rn_ofs, uint32_t rm_ofs, \ + uint32_t oprsz, uint32_t maxsz) \ + { \ + static const GVecGen3 ops[4] = { \ + { .fni4 = gen_helper_neon_##FUNC##8 }, \ + { .fni4 = gen_helper_neon_##FUNC##16 }, \ + { .fni4 = gen_helper_neon_##FUNC##32 }, \ + { 0 }, \ + }; \ + tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &ops[vece]); \ + } \ + static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ + { \ + if (a->size > 2) { \ + return false; \ + } \ + return do_3same(s, a, gen_##INSN##_3s); \ + } + +/* + * Some helper functions need to be passed the cpu_env. In order + * to use those with the gvec APIs like tcg_gen_gvec_3() we need + * to create wrapper functions whose prototype is a NeonGenTwoOpFn() + * and which call a NeonGenTwoOpEnvFn(). + */ +#define WRAP_ENV_FN(WRAPNAME, FUNC) \ + static void WRAPNAME(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m) \ + { \ + FUNC(d, cpu_env, n, m); \ + } + +#define DO_3SAME_32_ENV(INSN, FUNC) \ + WRAP_ENV_FN(gen_##INSN##_tramp8, gen_helper_neon_##FUNC##8); \ + WRAP_ENV_FN(gen_##INSN##_tramp16, gen_helper_neon_##FUNC##16); \ + WRAP_ENV_FN(gen_##INSN##_tramp32, gen_helper_neon_##FUNC##32); \ + static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ + uint32_t rn_ofs, uint32_t rm_ofs, \ + uint32_t oprsz, uint32_t maxsz) \ + { \ + static const GVecGen3 ops[4] = { \ + { .fni4 = gen_##INSN##_tramp8 }, \ + { .fni4 = gen_##INSN##_tramp16 }, \ + { .fni4 = gen_##INSN##_tramp32 }, \ + { 0 }, \ + }; \ + tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &ops[vece]); \ + } \ + static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ + { \ + if (a->size > 2) { \ + return false; \ + } \ + return do_3same(s, a, gen_##INSN##_3s); \ + } + +DO_3SAME_32(VHADD_S, hadd_s) +DO_3SAME_32(VHADD_U, hadd_u) +DO_3SAME_32(VHSUB_S, hsub_s) +DO_3SAME_32(VHSUB_U, hsub_u) +DO_3SAME_32(VRHADD_S, rhadd_s) +DO_3SAME_32(VRHADD_U, rhadd_u) +DO_3SAME_32(VRSHL_S, rshl_s) +DO_3SAME_32(VRSHL_U, rshl_u) + +DO_3SAME_32_ENV(VQSHL_S, qshl_s) +DO_3SAME_32_ENV(VQSHL_U, qshl_u) +DO_3SAME_32_ENV(VQRSHL_S, qrshl_s) +DO_3SAME_32_ENV(VQRSHL_U, qrshl_u) + +static bool do_3same_pair(DisasContext *s, arg_3same *a, NeonGenTwoOpFn *fn) +{ + /* Operations handled pairwise 32 bits at a time */ + TCGv_i32 tmp, tmp2, tmp3; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (a->size == 3) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + assert(a->q == 0); /* enforced by decode patterns */ + + /* + * Note that we have to be careful not to clobber the source operands + * in the "vm == vd" case by storing the result of the first pass too + * early. Since Q is 0 there are always just two passes, so instead + * of a complicated loop over each pass we just unroll. + */ + tmp = neon_load_reg(a->vn, 0); + tmp2 = neon_load_reg(a->vn, 1); + fn(tmp, tmp, tmp2); + tcg_temp_free_i32(tmp2); + + tmp3 = neon_load_reg(a->vm, 0); + tmp2 = neon_load_reg(a->vm, 1); + fn(tmp3, tmp3, tmp2); + tcg_temp_free_i32(tmp2); + + neon_store_reg(a->vd, 0, tmp); + neon_store_reg(a->vd, 1, tmp3); + return true; +} + +#define DO_3SAME_PAIR(INSN, func) \ + static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ + { \ + static NeonGenTwoOpFn * const fns[] = { \ + gen_helper_neon_##func##8, \ + gen_helper_neon_##func##16, \ + gen_helper_neon_##func##32, \ + }; \ + if (a->size > 2) { \ + return false; \ + } \ + return do_3same_pair(s, a, fns[a->size]); \ + } + +/* 32-bit pairwise ops end up the same as the elementwise versions. */ +#define gen_helper_neon_pmax_s32 tcg_gen_smax_i32 +#define gen_helper_neon_pmax_u32 tcg_gen_umax_i32 +#define gen_helper_neon_pmin_s32 tcg_gen_smin_i32 +#define gen_helper_neon_pmin_u32 tcg_gen_umin_i32 +#define gen_helper_neon_padd_u32 tcg_gen_add_i32 + +DO_3SAME_PAIR(VPMAX_S, pmax_s) +DO_3SAME_PAIR(VPMIN_S, pmin_s) +DO_3SAME_PAIR(VPMAX_U, pmax_u) +DO_3SAME_PAIR(VPMIN_U, pmin_u) +DO_3SAME_PAIR(VPADD, padd_u) + +#define DO_3SAME_VQDMULH(INSN, FUNC) \ + WRAP_ENV_FN(gen_##INSN##_tramp16, gen_helper_neon_##FUNC##_s16); \ + WRAP_ENV_FN(gen_##INSN##_tramp32, gen_helper_neon_##FUNC##_s32); \ + static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ + uint32_t rn_ofs, uint32_t rm_ofs, \ + uint32_t oprsz, uint32_t maxsz) \ + { \ + static const GVecGen3 ops[2] = { \ + { .fni4 = gen_##INSN##_tramp16 }, \ + { .fni4 = gen_##INSN##_tramp32 }, \ + }; \ + tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &ops[vece - 1]); \ + } \ + static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ + { \ + if (a->size != 1 && a->size != 2) { \ + return false; \ + } \ + return do_3same(s, a, gen_##INSN##_3s); \ + } + +DO_3SAME_VQDMULH(VQDMULH, qdmulh) +DO_3SAME_VQDMULH(VQRDMULH, qrdmulh) + +static bool do_3same_fp(DisasContext *s, arg_3same *a, VFPGen3OpSPFn *fn, + bool reads_vd) +{ + /* + * FP operations handled elementwise 32 bits at a time. + * If reads_vd is true then the old value of Vd will be + * loaded before calling the callback function. This is + * used for multiply-accumulate type operations. + */ + TCGv_i32 tmp, tmp2; + int pass; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if ((a->vn | a->vm | a->vd) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + TCGv_ptr fpstatus = get_fpstatus_ptr(1); + for (pass = 0; pass < (a->q ? 4 : 2); pass++) { + tmp = neon_load_reg(a->vn, pass); + tmp2 = neon_load_reg(a->vm, pass); + if (reads_vd) { + TCGv_i32 tmp_rd = neon_load_reg(a->vd, pass); + fn(tmp_rd, tmp, tmp2, fpstatus); + neon_store_reg(a->vd, pass, tmp_rd); + tcg_temp_free_i32(tmp); + } else { + fn(tmp, tmp, tmp2, fpstatus); + neon_store_reg(a->vd, pass, tmp); + } + tcg_temp_free_i32(tmp2); + } + tcg_temp_free_ptr(fpstatus); + return true; +} + +/* + * For all the functions using this macro, size == 1 means fp16, + * which is an architecture extension we don't implement yet. + */ +#define DO_3S_FP_GVEC(INSN,FUNC) \ + static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ + uint32_t rn_ofs, uint32_t rm_ofs, \ + uint32_t oprsz, uint32_t maxsz) \ + { \ + TCGv_ptr fpst = get_fpstatus_ptr(1); \ + tcg_gen_gvec_3_ptr(rd_ofs, rn_ofs, rm_ofs, fpst, \ + oprsz, maxsz, 0, FUNC); \ + tcg_temp_free_ptr(fpst); \ + } \ + static bool trans_##INSN##_fp_3s(DisasContext *s, arg_3same *a) \ + { \ + if (a->size != 0) { \ + /* TODO fp16 support */ \ + return false; \ + } \ + return do_3same(s, a, gen_##INSN##_3s); \ + } + + +DO_3S_FP_GVEC(VADD, gen_helper_gvec_fadd_s) +DO_3S_FP_GVEC(VSUB, gen_helper_gvec_fsub_s) +DO_3S_FP_GVEC(VABD, gen_helper_gvec_fabd_s) +DO_3S_FP_GVEC(VMUL, gen_helper_gvec_fmul_s) + +/* + * For all the functions using this macro, size == 1 means fp16, + * which is an architecture extension we don't implement yet. + */ +#define DO_3S_FP(INSN,FUNC,READS_VD) \ + static bool trans_##INSN##_fp_3s(DisasContext *s, arg_3same *a) \ + { \ + if (a->size != 0) { \ + /* TODO fp16 support */ \ + return false; \ + } \ + return do_3same_fp(s, a, FUNC, READS_VD); \ + } + +DO_3S_FP(VCEQ, gen_helper_neon_ceq_f32, false) +DO_3S_FP(VCGE, gen_helper_neon_cge_f32, false) +DO_3S_FP(VCGT, gen_helper_neon_cgt_f32, false) +DO_3S_FP(VACGE, gen_helper_neon_acge_f32, false) +DO_3S_FP(VACGT, gen_helper_neon_acgt_f32, false) +DO_3S_FP(VMAX, gen_helper_vfp_maxs, false) +DO_3S_FP(VMIN, gen_helper_vfp_mins, false) + +static void gen_VMLA_fp_3s(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, + TCGv_ptr fpstatus) +{ + gen_helper_vfp_muls(vn, vn, vm, fpstatus); + gen_helper_vfp_adds(vd, vd, vn, fpstatus); +} + +static void gen_VMLS_fp_3s(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, + TCGv_ptr fpstatus) +{ + gen_helper_vfp_muls(vn, vn, vm, fpstatus); + gen_helper_vfp_subs(vd, vd, vn, fpstatus); +} + +DO_3S_FP(VMLA, gen_VMLA_fp_3s, true) +DO_3S_FP(VMLS, gen_VMLS_fp_3s, true) + +static bool trans_VMAXNM_fp_3s(DisasContext *s, arg_3same *a) +{ + if (!arm_dc_feature(s, ARM_FEATURE_V8)) { + return false; + } + + if (a->size != 0) { + /* TODO fp16 support */ + return false; + } + + return do_3same_fp(s, a, gen_helper_vfp_maxnums, false); +} + +static bool trans_VMINNM_fp_3s(DisasContext *s, arg_3same *a) +{ + if (!arm_dc_feature(s, ARM_FEATURE_V8)) { + return false; + } + + if (a->size != 0) { + /* TODO fp16 support */ + return false; + } + + return do_3same_fp(s, a, gen_helper_vfp_minnums, false); +} + +WRAP_ENV_FN(gen_VRECPS_tramp, gen_helper_recps_f32) + +static void gen_VRECPS_fp_3s(unsigned vece, uint32_t rd_ofs, + uint32_t rn_ofs, uint32_t rm_ofs, + uint32_t oprsz, uint32_t maxsz) +{ + static const GVecGen3 ops = { .fni4 = gen_VRECPS_tramp }; + tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &ops); +} + +static bool trans_VRECPS_fp_3s(DisasContext *s, arg_3same *a) +{ + if (a->size != 0) { + /* TODO fp16 support */ + return false; + } + + return do_3same(s, a, gen_VRECPS_fp_3s); +} + +WRAP_ENV_FN(gen_VRSQRTS_tramp, gen_helper_rsqrts_f32) + +static void gen_VRSQRTS_fp_3s(unsigned vece, uint32_t rd_ofs, + uint32_t rn_ofs, uint32_t rm_ofs, + uint32_t oprsz, uint32_t maxsz) +{ + static const GVecGen3 ops = { .fni4 = gen_VRSQRTS_tramp }; + tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &ops); +} + +static bool trans_VRSQRTS_fp_3s(DisasContext *s, arg_3same *a) +{ + if (a->size != 0) { + /* TODO fp16 support */ + return false; + } + + return do_3same(s, a, gen_VRSQRTS_fp_3s); +} + +static void gen_VFMA_fp_3s(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, + TCGv_ptr fpstatus) +{ + gen_helper_vfp_muladds(vd, vn, vm, vd, fpstatus); +} + +static bool trans_VFMA_fp_3s(DisasContext *s, arg_3same *a) +{ + if (!dc_isar_feature(aa32_simdfmac, s)) { + return false; + } + + if (a->size != 0) { + /* TODO fp16 support */ + return false; + } + + return do_3same_fp(s, a, gen_VFMA_fp_3s, true); +} + +static void gen_VFMS_fp_3s(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, + TCGv_ptr fpstatus) +{ + gen_helper_vfp_negs(vn, vn); + gen_helper_vfp_muladds(vd, vn, vm, vd, fpstatus); +} + +static bool trans_VFMS_fp_3s(DisasContext *s, arg_3same *a) +{ + if (!dc_isar_feature(aa32_simdfmac, s)) { + return false; + } + + if (a->size != 0) { + /* TODO fp16 support */ + return false; + } + + return do_3same_fp(s, a, gen_VFMS_fp_3s, true); +} + +static bool do_3same_fp_pair(DisasContext *s, arg_3same *a, VFPGen3OpSPFn *fn) +{ + /* FP operations handled pairwise 32 bits at a time */ + TCGv_i32 tmp, tmp2, tmp3; + TCGv_ptr fpstatus; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + assert(a->q == 0); /* enforced by decode patterns */ + + /* + * Note that we have to be careful not to clobber the source operands + * in the "vm == vd" case by storing the result of the first pass too + * early. Since Q is 0 there are always just two passes, so instead + * of a complicated loop over each pass we just unroll. + */ + fpstatus = get_fpstatus_ptr(1); + tmp = neon_load_reg(a->vn, 0); + tmp2 = neon_load_reg(a->vn, 1); + fn(tmp, tmp, tmp2, fpstatus); + tcg_temp_free_i32(tmp2); + + tmp3 = neon_load_reg(a->vm, 0); + tmp2 = neon_load_reg(a->vm, 1); + fn(tmp3, tmp3, tmp2, fpstatus); + tcg_temp_free_i32(tmp2); + tcg_temp_free_ptr(fpstatus); + + neon_store_reg(a->vd, 0, tmp); + neon_store_reg(a->vd, 1, tmp3); + return true; +} + +/* + * For all the functions using this macro, size == 1 means fp16, + * which is an architecture extension we don't implement yet. + */ +#define DO_3S_FP_PAIR(INSN,FUNC) \ + static bool trans_##INSN##_fp_3s(DisasContext *s, arg_3same *a) \ + { \ + if (a->size != 0) { \ + /* TODO fp16 support */ \ + return false; \ + } \ + return do_3same_fp_pair(s, a, FUNC); \ + } + +DO_3S_FP_PAIR(VPADD, gen_helper_vfp_adds) +DO_3S_FP_PAIR(VPMAX, gen_helper_vfp_maxs) +DO_3S_FP_PAIR(VPMIN, gen_helper_vfp_mins) + +static bool do_vector_2sh(DisasContext *s, arg_2reg_shift *a, GVecGen2iFn *fn) +{ + /* Handle a 2-reg-shift insn which can be vectorized. */ + int vec_size = a->q ? 16 : 8; + int rd_ofs = neon_reg_offset(a->vd, 0); + int rm_ofs = neon_reg_offset(a->vm, 0); + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if ((a->vm | a->vd) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fn(a->size, rd_ofs, rm_ofs, a->shift, vec_size, vec_size); + return true; +} + +#define DO_2SH(INSN, FUNC) \ + static bool trans_##INSN##_2sh(DisasContext *s, arg_2reg_shift *a) \ + { \ + return do_vector_2sh(s, a, FUNC); \ + } \ + +DO_2SH(VSHL, tcg_gen_gvec_shli) +DO_2SH(VSLI, gen_gvec_sli) +DO_2SH(VSRI, gen_gvec_sri) +DO_2SH(VSRA_S, gen_gvec_ssra) +DO_2SH(VSRA_U, gen_gvec_usra) +DO_2SH(VRSHR_S, gen_gvec_srshr) +DO_2SH(VRSHR_U, gen_gvec_urshr) +DO_2SH(VRSRA_S, gen_gvec_srsra) +DO_2SH(VRSRA_U, gen_gvec_ursra) + +static bool trans_VSHR_S_2sh(DisasContext *s, arg_2reg_shift *a) +{ + /* Signed shift out of range results in all-sign-bits */ + a->shift = MIN(a->shift, (8 << a->size) - 1); + return do_vector_2sh(s, a, tcg_gen_gvec_sari); +} + +static void gen_zero_rd_2sh(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs, + int64_t shift, uint32_t oprsz, uint32_t maxsz) +{ + tcg_gen_gvec_dup_imm(vece, rd_ofs, oprsz, maxsz, 0); +} + +static bool trans_VSHR_U_2sh(DisasContext *s, arg_2reg_shift *a) +{ + /* Shift out of range is architecturally valid and results in zero. */ + if (a->shift >= (8 << a->size)) { + return do_vector_2sh(s, a, gen_zero_rd_2sh); + } else { + return do_vector_2sh(s, a, tcg_gen_gvec_shri); + } +} + +static bool do_2shift_env_64(DisasContext *s, arg_2reg_shift *a, + NeonGenTwo64OpEnvFn *fn) +{ + /* + * 2-reg-and-shift operations, size == 3 case, where the + * function needs to be passed cpu_env. + */ + TCGv_i64 constimm; + int pass; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if ((a->vm | a->vd) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + /* + * To avoid excessive duplication of ops we implement shift + * by immediate using the variable shift operations. + */ + constimm = tcg_const_i64(dup_const(a->size, a->shift)); + + for (pass = 0; pass < a->q + 1; pass++) { + TCGv_i64 tmp = tcg_temp_new_i64(); + + neon_load_reg64(tmp, a->vm + pass); + fn(tmp, cpu_env, tmp, constimm); + neon_store_reg64(tmp, a->vd + pass); + tcg_temp_free_i64(tmp); + } + tcg_temp_free_i64(constimm); + return true; +} + +static bool do_2shift_env_32(DisasContext *s, arg_2reg_shift *a, + NeonGenTwoOpEnvFn *fn) +{ + /* + * 2-reg-and-shift operations, size < 3 case, where the + * helper needs to be passed cpu_env. + */ + TCGv_i32 constimm; + int pass; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if ((a->vm | a->vd) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + /* + * To avoid excessive duplication of ops we implement shift + * by immediate using the variable shift operations. + */ + constimm = tcg_const_i32(dup_const(a->size, a->shift)); + + for (pass = 0; pass < (a->q ? 4 : 2); pass++) { + TCGv_i32 tmp = neon_load_reg(a->vm, pass); + fn(tmp, cpu_env, tmp, constimm); + neon_store_reg(a->vd, pass, tmp); + } + tcg_temp_free_i32(constimm); + return true; +} + +#define DO_2SHIFT_ENV(INSN, FUNC) \ + static bool trans_##INSN##_64_2sh(DisasContext *s, arg_2reg_shift *a) \ + { \ + return do_2shift_env_64(s, a, gen_helper_neon_##FUNC##64); \ + } \ + static bool trans_##INSN##_2sh(DisasContext *s, arg_2reg_shift *a) \ + { \ + static NeonGenTwoOpEnvFn * const fns[] = { \ + gen_helper_neon_##FUNC##8, \ + gen_helper_neon_##FUNC##16, \ + gen_helper_neon_##FUNC##32, \ + }; \ + assert(a->size < ARRAY_SIZE(fns)); \ + return do_2shift_env_32(s, a, fns[a->size]); \ + } + +DO_2SHIFT_ENV(VQSHLU, qshlu_s) +DO_2SHIFT_ENV(VQSHL_U, qshl_u) +DO_2SHIFT_ENV(VQSHL_S, qshl_s) + +static bool do_2shift_narrow_64(DisasContext *s, arg_2reg_shift *a, + NeonGenTwo64OpFn *shiftfn, + NeonGenNarrowEnvFn *narrowfn) +{ + /* 2-reg-and-shift narrowing-shift operations, size == 3 case */ + TCGv_i64 constimm, rm1, rm2; + TCGv_i32 rd; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->vm & 1) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + /* + * This is always a right shift, and the shiftfn is always a + * left-shift helper, which thus needs the negated shift count. + */ + constimm = tcg_const_i64(-a->shift); + rm1 = tcg_temp_new_i64(); + rm2 = tcg_temp_new_i64(); + + /* Load both inputs first to avoid potential overwrite if rm == rd */ + neon_load_reg64(rm1, a->vm); + neon_load_reg64(rm2, a->vm + 1); + + shiftfn(rm1, rm1, constimm); + rd = tcg_temp_new_i32(); + narrowfn(rd, cpu_env, rm1); + neon_store_reg(a->vd, 0, rd); + + shiftfn(rm2, rm2, constimm); + rd = tcg_temp_new_i32(); + narrowfn(rd, cpu_env, rm2); + neon_store_reg(a->vd, 1, rd); + + tcg_temp_free_i64(rm1); + tcg_temp_free_i64(rm2); + tcg_temp_free_i64(constimm); + + return true; +} + +static bool do_2shift_narrow_32(DisasContext *s, arg_2reg_shift *a, + NeonGenTwoOpFn *shiftfn, + NeonGenNarrowEnvFn *narrowfn) +{ + /* 2-reg-and-shift narrowing-shift operations, size < 3 case */ + TCGv_i32 constimm, rm1, rm2, rm3, rm4; + TCGv_i64 rtmp; + uint32_t imm; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->vm & 1) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + /* + * This is always a right shift, and the shiftfn is always a + * left-shift helper, which thus needs the negated shift count + * duplicated into each lane of the immediate value. + */ + if (a->size == 1) { + imm = (uint16_t)(-a->shift); + imm |= imm << 16; + } else { + /* size == 2 */ + imm = -a->shift; + } + constimm = tcg_const_i32(imm); + + /* Load all inputs first to avoid potential overwrite */ + rm1 = neon_load_reg(a->vm, 0); + rm2 = neon_load_reg(a->vm, 1); + rm3 = neon_load_reg(a->vm + 1, 0); + rm4 = neon_load_reg(a->vm + 1, 1); + rtmp = tcg_temp_new_i64(); + + shiftfn(rm1, rm1, constimm); + shiftfn(rm2, rm2, constimm); + + tcg_gen_concat_i32_i64(rtmp, rm1, rm2); + tcg_temp_free_i32(rm2); + + narrowfn(rm1, cpu_env, rtmp); + neon_store_reg(a->vd, 0, rm1); + + shiftfn(rm3, rm3, constimm); + shiftfn(rm4, rm4, constimm); + tcg_temp_free_i32(constimm); + + tcg_gen_concat_i32_i64(rtmp, rm3, rm4); + tcg_temp_free_i32(rm4); + + narrowfn(rm3, cpu_env, rtmp); + tcg_temp_free_i64(rtmp); + neon_store_reg(a->vd, 1, rm3); + return true; +} + +#define DO_2SN_64(INSN, FUNC, NARROWFUNC) \ + static bool trans_##INSN##_2sh(DisasContext *s, arg_2reg_shift *a) \ + { \ + return do_2shift_narrow_64(s, a, FUNC, NARROWFUNC); \ + } +#define DO_2SN_32(INSN, FUNC, NARROWFUNC) \ + static bool trans_##INSN##_2sh(DisasContext *s, arg_2reg_shift *a) \ + { \ + return do_2shift_narrow_32(s, a, FUNC, NARROWFUNC); \ + } + +static void gen_neon_narrow_u32(TCGv_i32 dest, TCGv_ptr env, TCGv_i64 src) +{ + tcg_gen_extrl_i64_i32(dest, src); +} + +static void gen_neon_narrow_u16(TCGv_i32 dest, TCGv_ptr env, TCGv_i64 src) +{ + gen_helper_neon_narrow_u16(dest, src); +} + +static void gen_neon_narrow_u8(TCGv_i32 dest, TCGv_ptr env, TCGv_i64 src) +{ + gen_helper_neon_narrow_u8(dest, src); +} + +DO_2SN_64(VSHRN_64, gen_ushl_i64, gen_neon_narrow_u32) +DO_2SN_32(VSHRN_32, gen_ushl_i32, gen_neon_narrow_u16) +DO_2SN_32(VSHRN_16, gen_helper_neon_shl_u16, gen_neon_narrow_u8) + +DO_2SN_64(VRSHRN_64, gen_helper_neon_rshl_u64, gen_neon_narrow_u32) +DO_2SN_32(VRSHRN_32, gen_helper_neon_rshl_u32, gen_neon_narrow_u16) +DO_2SN_32(VRSHRN_16, gen_helper_neon_rshl_u16, gen_neon_narrow_u8) + +DO_2SN_64(VQSHRUN_64, gen_sshl_i64, gen_helper_neon_unarrow_sat32) +DO_2SN_32(VQSHRUN_32, gen_sshl_i32, gen_helper_neon_unarrow_sat16) +DO_2SN_32(VQSHRUN_16, gen_helper_neon_shl_s16, gen_helper_neon_unarrow_sat8) + +DO_2SN_64(VQRSHRUN_64, gen_helper_neon_rshl_s64, gen_helper_neon_unarrow_sat32) +DO_2SN_32(VQRSHRUN_32, gen_helper_neon_rshl_s32, gen_helper_neon_unarrow_sat16) +DO_2SN_32(VQRSHRUN_16, gen_helper_neon_rshl_s16, gen_helper_neon_unarrow_sat8) +DO_2SN_64(VQSHRN_S64, gen_sshl_i64, gen_helper_neon_narrow_sat_s32) +DO_2SN_32(VQSHRN_S32, gen_sshl_i32, gen_helper_neon_narrow_sat_s16) +DO_2SN_32(VQSHRN_S16, gen_helper_neon_shl_s16, gen_helper_neon_narrow_sat_s8) + +DO_2SN_64(VQRSHRN_S64, gen_helper_neon_rshl_s64, gen_helper_neon_narrow_sat_s32) +DO_2SN_32(VQRSHRN_S32, gen_helper_neon_rshl_s32, gen_helper_neon_narrow_sat_s16) +DO_2SN_32(VQRSHRN_S16, gen_helper_neon_rshl_s16, gen_helper_neon_narrow_sat_s8) + +DO_2SN_64(VQSHRN_U64, gen_ushl_i64, gen_helper_neon_narrow_sat_u32) +DO_2SN_32(VQSHRN_U32, gen_ushl_i32, gen_helper_neon_narrow_sat_u16) +DO_2SN_32(VQSHRN_U16, gen_helper_neon_shl_u16, gen_helper_neon_narrow_sat_u8) + +DO_2SN_64(VQRSHRN_U64, gen_helper_neon_rshl_u64, gen_helper_neon_narrow_sat_u32) +DO_2SN_32(VQRSHRN_U32, gen_helper_neon_rshl_u32, gen_helper_neon_narrow_sat_u16) +DO_2SN_32(VQRSHRN_U16, gen_helper_neon_rshl_u16, gen_helper_neon_narrow_sat_u8) + +static bool do_vshll_2sh(DisasContext *s, arg_2reg_shift *a, + NeonGenWidenFn *widenfn, bool u) +{ + TCGv_i64 tmp; + TCGv_i32 rm0, rm1; + uint64_t widen_mask = 0; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->vd & 1) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + /* + * This is a widen-and-shift operation. The shift is always less + * than the width of the source type, so after widening the input + * vector we can simply shift the whole 64-bit widened register, + * and then clear the potential overflow bits resulting from left + * bits of the narrow input appearing as right bits of the left + * neighbour narrow input. Calculate a mask of bits to clear. + */ + if ((a->shift != 0) && (a->size < 2 || u)) { + int esize = 8 << a->size; + widen_mask = MAKE_64BIT_MASK(0, esize); + widen_mask >>= esize - a->shift; + widen_mask = dup_const(a->size + 1, widen_mask); + } + + rm0 = neon_load_reg(a->vm, 0); + rm1 = neon_load_reg(a->vm, 1); + tmp = tcg_temp_new_i64(); + + widenfn(tmp, rm0); + tcg_temp_free_i32(rm0); + if (a->shift != 0) { + tcg_gen_shli_i64(tmp, tmp, a->shift); + tcg_gen_andi_i64(tmp, tmp, ~widen_mask); + } + neon_store_reg64(tmp, a->vd); + + widenfn(tmp, rm1); + tcg_temp_free_i32(rm1); + if (a->shift != 0) { + tcg_gen_shli_i64(tmp, tmp, a->shift); + tcg_gen_andi_i64(tmp, tmp, ~widen_mask); + } + neon_store_reg64(tmp, a->vd + 1); + tcg_temp_free_i64(tmp); + return true; +} + +static bool trans_VSHLL_S_2sh(DisasContext *s, arg_2reg_shift *a) +{ + static NeonGenWidenFn * const widenfn[] = { + gen_helper_neon_widen_s8, + gen_helper_neon_widen_s16, + tcg_gen_ext_i32_i64, + }; + return do_vshll_2sh(s, a, widenfn[a->size], false); +} + +static bool trans_VSHLL_U_2sh(DisasContext *s, arg_2reg_shift *a) +{ + static NeonGenWidenFn * const widenfn[] = { + gen_helper_neon_widen_u8, + gen_helper_neon_widen_u16, + tcg_gen_extu_i32_i64, + }; + return do_vshll_2sh(s, a, widenfn[a->size], true); +} + +static bool do_fp_2sh(DisasContext *s, arg_2reg_shift *a, + NeonGenTwoSingleOpFn *fn) +{ + /* FP operations in 2-reg-and-shift group */ + TCGv_i32 tmp, shiftv; + TCGv_ptr fpstatus; + int pass; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if ((a->vm | a->vd) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpstatus = get_fpstatus_ptr(1); + shiftv = tcg_const_i32(a->shift); + for (pass = 0; pass < (a->q ? 4 : 2); pass++) { + tmp = neon_load_reg(a->vm, pass); + fn(tmp, tmp, shiftv, fpstatus); + neon_store_reg(a->vd, pass, tmp); + } + tcg_temp_free_ptr(fpstatus); + tcg_temp_free_i32(shiftv); + return true; +} + +#define DO_FP_2SH(INSN, FUNC) \ + static bool trans_##INSN##_2sh(DisasContext *s, arg_2reg_shift *a) \ + { \ + return do_fp_2sh(s, a, FUNC); \ + } + +DO_FP_2SH(VCVT_SF, gen_helper_vfp_sltos) +DO_FP_2SH(VCVT_UF, gen_helper_vfp_ultos) +DO_FP_2SH(VCVT_FS, gen_helper_vfp_tosls_round_to_zero) +DO_FP_2SH(VCVT_FU, gen_helper_vfp_touls_round_to_zero) + +static uint64_t asimd_imm_const(uint32_t imm, int cmode, int op) +{ + /* + * Expand the encoded constant. + * Note that cmode = 2,3,4,5,6,7,10,11,12,13 imm=0 is UNPREDICTABLE. + * We choose to not special-case this and will behave as if a + * valid constant encoding of 0 had been given. + * cmode = 15 op = 1 must UNDEF; we assume decode has handled that. + */ + switch (cmode) { + case 0: case 1: + /* no-op */ + break; + case 2: case 3: + imm <<= 8; + break; + case 4: case 5: + imm <<= 16; + break; + case 6: case 7: + imm <<= 24; + break; + case 8: case 9: + imm |= imm << 16; + break; + case 10: case 11: + imm = (imm << 8) | (imm << 24); + break; + case 12: + imm = (imm << 8) | 0xff; + break; + case 13: + imm = (imm << 16) | 0xffff; + break; + case 14: + if (op) { + /* + * This is the only case where the top and bottom 32 bits + * of the encoded constant differ. + */ + uint64_t imm64 = 0; + int n; + + for (n = 0; n < 8; n++) { + if (imm & (1 << n)) { + imm64 |= (0xffULL << (n * 8)); + } + } + return imm64; + } + imm |= (imm << 8) | (imm << 16) | (imm << 24); + break; + case 15: + imm = ((imm & 0x80) << 24) | ((imm & 0x3f) << 19) + | ((imm & 0x40) ? (0x1f << 25) : (1 << 30)); + break; + } + if (op) { + imm = ~imm; + } + return dup_const(MO_32, imm); +} + +static bool do_1reg_imm(DisasContext *s, arg_1reg_imm *a, + GVecGen2iFn *fn) +{ + uint64_t imm; + int reg_ofs, vec_size; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { + return false; + } + + if (a->vd & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + reg_ofs = neon_reg_offset(a->vd, 0); + vec_size = a->q ? 16 : 8; + imm = asimd_imm_const(a->imm, a->cmode, a->op); + + fn(MO_64, reg_ofs, reg_ofs, imm, vec_size, vec_size); + return true; +} + +static void gen_VMOV_1r(unsigned vece, uint32_t dofs, uint32_t aofs, + int64_t c, uint32_t oprsz, uint32_t maxsz) +{ + tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, c); +} + +static bool trans_Vimm_1r(DisasContext *s, arg_1reg_imm *a) +{ + /* Handle decode of cmode/op here between VORR/VBIC/VMOV */ + GVecGen2iFn *fn; + + if ((a->cmode & 1) && a->cmode < 12) { + /* for op=1, the imm will be inverted, so BIC becomes AND. */ + fn = a->op ? tcg_gen_gvec_andi : tcg_gen_gvec_ori; + } else { + /* There is one unallocated cmode/op combination in this space */ + if (a->cmode == 15 && a->op == 1) { + return false; + } + fn = gen_VMOV_1r; + } + return do_1reg_imm(s, a, fn); +} + +static bool do_prewiden_3d(DisasContext *s, arg_3diff *a, + NeonGenWidenFn *widenfn, + NeonGenTwo64OpFn *opfn, + bool src1_wide) +{ + /* 3-regs different lengths, prewidening case (VADDL/VSUBL/VAADW/VSUBW) */ + TCGv_i64 rn0_64, rn1_64, rm_64; + TCGv_i32 rm; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (!widenfn || !opfn) { + /* size == 3 case, which is an entirely different insn group */ + return false; + } + + if ((a->vd & 1) || (src1_wide && (a->vn & 1))) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + rn0_64 = tcg_temp_new_i64(); + rn1_64 = tcg_temp_new_i64(); + rm_64 = tcg_temp_new_i64(); + + if (src1_wide) { + neon_load_reg64(rn0_64, a->vn); + } else { + TCGv_i32 tmp = neon_load_reg(a->vn, 0); + widenfn(rn0_64, tmp); + tcg_temp_free_i32(tmp); + } + rm = neon_load_reg(a->vm, 0); + + widenfn(rm_64, rm); + tcg_temp_free_i32(rm); + opfn(rn0_64, rn0_64, rm_64); + + /* + * Load second pass inputs before storing the first pass result, to + * avoid incorrect results if a narrow input overlaps with the result. + */ + if (src1_wide) { + neon_load_reg64(rn1_64, a->vn + 1); + } else { + TCGv_i32 tmp = neon_load_reg(a->vn, 1); + widenfn(rn1_64, tmp); + tcg_temp_free_i32(tmp); + } + rm = neon_load_reg(a->vm, 1); + + neon_store_reg64(rn0_64, a->vd); + + widenfn(rm_64, rm); + tcg_temp_free_i32(rm); + opfn(rn1_64, rn1_64, rm_64); + neon_store_reg64(rn1_64, a->vd + 1); + + tcg_temp_free_i64(rn0_64); + tcg_temp_free_i64(rn1_64); + tcg_temp_free_i64(rm_64); + + return true; +} + +#define DO_PREWIDEN(INSN, S, EXT, OP, SRC1WIDE) \ + static bool trans_##INSN##_3d(DisasContext *s, arg_3diff *a) \ + { \ + static NeonGenWidenFn * const widenfn[] = { \ + gen_helper_neon_widen_##S##8, \ + gen_helper_neon_widen_##S##16, \ + tcg_gen_##EXT##_i32_i64, \ + NULL, \ + }; \ + static NeonGenTwo64OpFn * const addfn[] = { \ + gen_helper_neon_##OP##l_u16, \ + gen_helper_neon_##OP##l_u32, \ + tcg_gen_##OP##_i64, \ + NULL, \ + }; \ + return do_prewiden_3d(s, a, widenfn[a->size], \ + addfn[a->size], SRC1WIDE); \ + } + +DO_PREWIDEN(VADDL_S, s, ext, add, false) +DO_PREWIDEN(VADDL_U, u, extu, add, false) +DO_PREWIDEN(VSUBL_S, s, ext, sub, false) +DO_PREWIDEN(VSUBL_U, u, extu, sub, false) +DO_PREWIDEN(VADDW_S, s, ext, add, true) +DO_PREWIDEN(VADDW_U, u, extu, add, true) +DO_PREWIDEN(VSUBW_S, s, ext, sub, true) +DO_PREWIDEN(VSUBW_U, u, extu, sub, true) + +static bool do_narrow_3d(DisasContext *s, arg_3diff *a, + NeonGenTwo64OpFn *opfn, NeonGenNarrowFn *narrowfn) +{ + /* 3-regs different lengths, narrowing (VADDHN/VSUBHN/VRADDHN/VRSUBHN) */ + TCGv_i64 rn_64, rm_64; + TCGv_i32 rd0, rd1; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (!opfn || !narrowfn) { + /* size == 3 case, which is an entirely different insn group */ + return false; + } + + if ((a->vn | a->vm) & 1) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + rn_64 = tcg_temp_new_i64(); + rm_64 = tcg_temp_new_i64(); + rd0 = tcg_temp_new_i32(); + rd1 = tcg_temp_new_i32(); + + neon_load_reg64(rn_64, a->vn); + neon_load_reg64(rm_64, a->vm); + + opfn(rn_64, rn_64, rm_64); + + narrowfn(rd0, rn_64); + + neon_load_reg64(rn_64, a->vn + 1); + neon_load_reg64(rm_64, a->vm + 1); + + opfn(rn_64, rn_64, rm_64); + + narrowfn(rd1, rn_64); + + neon_store_reg(a->vd, 0, rd0); + neon_store_reg(a->vd, 1, rd1); + + tcg_temp_free_i64(rn_64); + tcg_temp_free_i64(rm_64); + + return true; +} + +#define DO_NARROW_3D(INSN, OP, NARROWTYPE, EXTOP) \ + static bool trans_##INSN##_3d(DisasContext *s, arg_3diff *a) \ + { \ + static NeonGenTwo64OpFn * const addfn[] = { \ + gen_helper_neon_##OP##l_u16, \ + gen_helper_neon_##OP##l_u32, \ + tcg_gen_##OP##_i64, \ + NULL, \ + }; \ + static NeonGenNarrowFn * const narrowfn[] = { \ + gen_helper_neon_##NARROWTYPE##_high_u8, \ + gen_helper_neon_##NARROWTYPE##_high_u16, \ + EXTOP, \ + NULL, \ + }; \ + return do_narrow_3d(s, a, addfn[a->size], narrowfn[a->size]); \ + } + +static void gen_narrow_round_high_u32(TCGv_i32 rd, TCGv_i64 rn) +{ + tcg_gen_addi_i64(rn, rn, 1u << 31); + tcg_gen_extrh_i64_i32(rd, rn); +} + +DO_NARROW_3D(VADDHN, add, narrow, tcg_gen_extrh_i64_i32) +DO_NARROW_3D(VSUBHN, sub, narrow, tcg_gen_extrh_i64_i32) +DO_NARROW_3D(VRADDHN, add, narrow_round, gen_narrow_round_high_u32) +DO_NARROW_3D(VRSUBHN, sub, narrow_round, gen_narrow_round_high_u32) + +static bool do_long_3d(DisasContext *s, arg_3diff *a, + NeonGenTwoOpWidenFn *opfn, + NeonGenTwo64OpFn *accfn) +{ + /* + * 3-regs different lengths, long operations. + * These perform an operation on two inputs that returns a double-width + * result, and then possibly perform an accumulation operation of + * that result into the double-width destination. + */ + TCGv_i64 rd0, rd1, tmp; + TCGv_i32 rn, rm; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (!opfn) { + /* size == 3 case, which is an entirely different insn group */ + return false; + } + + if (a->vd & 1) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + rd0 = tcg_temp_new_i64(); + rd1 = tcg_temp_new_i64(); + + rn = neon_load_reg(a->vn, 0); + rm = neon_load_reg(a->vm, 0); + opfn(rd0, rn, rm); + tcg_temp_free_i32(rn); + tcg_temp_free_i32(rm); + + rn = neon_load_reg(a->vn, 1); + rm = neon_load_reg(a->vm, 1); + opfn(rd1, rn, rm); + tcg_temp_free_i32(rn); + tcg_temp_free_i32(rm); + + /* Don't store results until after all loads: they might overlap */ + if (accfn) { + tmp = tcg_temp_new_i64(); + neon_load_reg64(tmp, a->vd); + accfn(tmp, tmp, rd0); + neon_store_reg64(tmp, a->vd); + neon_load_reg64(tmp, a->vd + 1); + accfn(tmp, tmp, rd1); + neon_store_reg64(tmp, a->vd + 1); + tcg_temp_free_i64(tmp); + } else { + neon_store_reg64(rd0, a->vd); + neon_store_reg64(rd1, a->vd + 1); + } + + tcg_temp_free_i64(rd0); + tcg_temp_free_i64(rd1); + + return true; +} + +static bool trans_VABDL_S_3d(DisasContext *s, arg_3diff *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + gen_helper_neon_abdl_s16, + gen_helper_neon_abdl_s32, + gen_helper_neon_abdl_s64, + NULL, + }; + + return do_long_3d(s, a, opfn[a->size], NULL); +} + +static bool trans_VABDL_U_3d(DisasContext *s, arg_3diff *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + gen_helper_neon_abdl_u16, + gen_helper_neon_abdl_u32, + gen_helper_neon_abdl_u64, + NULL, + }; + + return do_long_3d(s, a, opfn[a->size], NULL); +} + +static bool trans_VABAL_S_3d(DisasContext *s, arg_3diff *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + gen_helper_neon_abdl_s16, + gen_helper_neon_abdl_s32, + gen_helper_neon_abdl_s64, + NULL, + }; + static NeonGenTwo64OpFn * const addfn[] = { + gen_helper_neon_addl_u16, + gen_helper_neon_addl_u32, + tcg_gen_add_i64, + NULL, + }; + + return do_long_3d(s, a, opfn[a->size], addfn[a->size]); +} + +static bool trans_VABAL_U_3d(DisasContext *s, arg_3diff *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + gen_helper_neon_abdl_u16, + gen_helper_neon_abdl_u32, + gen_helper_neon_abdl_u64, + NULL, + }; + static NeonGenTwo64OpFn * const addfn[] = { + gen_helper_neon_addl_u16, + gen_helper_neon_addl_u32, + tcg_gen_add_i64, + NULL, + }; + + return do_long_3d(s, a, opfn[a->size], addfn[a->size]); +} + +static void gen_mull_s32(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm) +{ + TCGv_i32 lo = tcg_temp_new_i32(); + TCGv_i32 hi = tcg_temp_new_i32(); + + tcg_gen_muls2_i32(lo, hi, rn, rm); + tcg_gen_concat_i32_i64(rd, lo, hi); + + tcg_temp_free_i32(lo); + tcg_temp_free_i32(hi); +} + +static void gen_mull_u32(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm) +{ + TCGv_i32 lo = tcg_temp_new_i32(); + TCGv_i32 hi = tcg_temp_new_i32(); + + tcg_gen_mulu2_i32(lo, hi, rn, rm); + tcg_gen_concat_i32_i64(rd, lo, hi); + + tcg_temp_free_i32(lo); + tcg_temp_free_i32(hi); +} + +static bool trans_VMULL_S_3d(DisasContext *s, arg_3diff *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + gen_helper_neon_mull_s8, + gen_helper_neon_mull_s16, + gen_mull_s32, + NULL, + }; + + return do_long_3d(s, a, opfn[a->size], NULL); +} + +static bool trans_VMULL_U_3d(DisasContext *s, arg_3diff *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + gen_helper_neon_mull_u8, + gen_helper_neon_mull_u16, + gen_mull_u32, + NULL, + }; + + return do_long_3d(s, a, opfn[a->size], NULL); +} + +#define DO_VMLAL(INSN,MULL,ACC) \ + static bool trans_##INSN##_3d(DisasContext *s, arg_3diff *a) \ + { \ + static NeonGenTwoOpWidenFn * const opfn[] = { \ + gen_helper_neon_##MULL##8, \ + gen_helper_neon_##MULL##16, \ + gen_##MULL##32, \ + NULL, \ + }; \ + static NeonGenTwo64OpFn * const accfn[] = { \ + gen_helper_neon_##ACC##l_u16, \ + gen_helper_neon_##ACC##l_u32, \ + tcg_gen_##ACC##_i64, \ + NULL, \ + }; \ + return do_long_3d(s, a, opfn[a->size], accfn[a->size]); \ + } + +DO_VMLAL(VMLAL_S,mull_s,add) +DO_VMLAL(VMLAL_U,mull_u,add) +DO_VMLAL(VMLSL_S,mull_s,sub) +DO_VMLAL(VMLSL_U,mull_u,sub) + +static void gen_VQDMULL_16(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm) +{ + gen_helper_neon_mull_s16(rd, rn, rm); + gen_helper_neon_addl_saturate_s32(rd, cpu_env, rd, rd); +} + +static void gen_VQDMULL_32(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm) +{ + gen_mull_s32(rd, rn, rm); + gen_helper_neon_addl_saturate_s64(rd, cpu_env, rd, rd); +} + +static bool trans_VQDMULL_3d(DisasContext *s, arg_3diff *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + NULL, + gen_VQDMULL_16, + gen_VQDMULL_32, + NULL, + }; + + return do_long_3d(s, a, opfn[a->size], NULL); +} + +static void gen_VQDMLAL_acc_16(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm) +{ + gen_helper_neon_addl_saturate_s32(rd, cpu_env, rn, rm); +} + +static void gen_VQDMLAL_acc_32(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm) +{ + gen_helper_neon_addl_saturate_s64(rd, cpu_env, rn, rm); +} + +static bool trans_VQDMLAL_3d(DisasContext *s, arg_3diff *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + NULL, + gen_VQDMULL_16, + gen_VQDMULL_32, + NULL, + }; + static NeonGenTwo64OpFn * const accfn[] = { + NULL, + gen_VQDMLAL_acc_16, + gen_VQDMLAL_acc_32, + NULL, + }; + + return do_long_3d(s, a, opfn[a->size], accfn[a->size]); +} + +static void gen_VQDMLSL_acc_16(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm) +{ + gen_helper_neon_negl_u32(rm, rm); + gen_helper_neon_addl_saturate_s32(rd, cpu_env, rn, rm); +} + +static void gen_VQDMLSL_acc_32(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm) +{ + tcg_gen_neg_i64(rm, rm); + gen_helper_neon_addl_saturate_s64(rd, cpu_env, rn, rm); +} + +static bool trans_VQDMLSL_3d(DisasContext *s, arg_3diff *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + NULL, + gen_VQDMULL_16, + gen_VQDMULL_32, + NULL, + }; + static NeonGenTwo64OpFn * const accfn[] = { + NULL, + gen_VQDMLSL_acc_16, + gen_VQDMLSL_acc_32, + NULL, + }; + + return do_long_3d(s, a, opfn[a->size], accfn[a->size]); +} + +static bool trans_VMULL_P_3d(DisasContext *s, arg_3diff *a) +{ + gen_helper_gvec_3 *fn_gvec; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (a->vd & 1) { + return false; + } + + switch (a->size) { + case 0: + fn_gvec = gen_helper_neon_pmull_h; + break; + case 2: + if (!dc_isar_feature(aa32_pmull, s)) { + return false; + } + fn_gvec = gen_helper_gvec_pmull_q; + break; + default: + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + tcg_gen_gvec_3_ool(neon_reg_offset(a->vd, 0), + neon_reg_offset(a->vn, 0), + neon_reg_offset(a->vm, 0), + 16, 16, 0, fn_gvec); + return true; +} + +static void gen_neon_dup_low16(TCGv_i32 var) +{ + TCGv_i32 tmp = tcg_temp_new_i32(); + tcg_gen_ext16u_i32(var, var); + tcg_gen_shli_i32(tmp, var, 16); + tcg_gen_or_i32(var, var, tmp); + tcg_temp_free_i32(tmp); +} + +static void gen_neon_dup_high16(TCGv_i32 var) +{ + TCGv_i32 tmp = tcg_temp_new_i32(); + tcg_gen_andi_i32(var, var, 0xffff0000); + tcg_gen_shri_i32(tmp, var, 16); + tcg_gen_or_i32(var, var, tmp); + tcg_temp_free_i32(tmp); +} + +static inline TCGv_i32 neon_get_scalar(int size, int reg) +{ + TCGv_i32 tmp; + if (size == 1) { + tmp = neon_load_reg(reg & 7, reg >> 4); + if (reg & 8) { + gen_neon_dup_high16(tmp); + } else { + gen_neon_dup_low16(tmp); + } + } else { + tmp = neon_load_reg(reg & 15, reg >> 4); + } + return tmp; +} + +static bool do_2scalar(DisasContext *s, arg_2scalar *a, + NeonGenTwoOpFn *opfn, NeonGenTwoOpFn *accfn) +{ + /* + * Two registers and a scalar: perform an operation between + * the input elements and the scalar, and then possibly + * perform an accumulation operation of that result into the + * destination. + */ + TCGv_i32 scalar; + int pass; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (!opfn) { + /* Bad size (including size == 3, which is a different insn group) */ + return false; + } + + if (a->q && ((a->vd | a->vn) & 1)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + scalar = neon_get_scalar(a->size, a->vm); + + for (pass = 0; pass < (a->q ? 4 : 2); pass++) { + TCGv_i32 tmp = neon_load_reg(a->vn, pass); + opfn(tmp, tmp, scalar); + if (accfn) { + TCGv_i32 rd = neon_load_reg(a->vd, pass); + accfn(tmp, rd, tmp); + tcg_temp_free_i32(rd); + } + neon_store_reg(a->vd, pass, tmp); + } + tcg_temp_free_i32(scalar); + return true; +} + +static bool trans_VMUL_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpFn * const opfn[] = { + NULL, + gen_helper_neon_mul_u16, + tcg_gen_mul_i32, + NULL, + }; + + return do_2scalar(s, a, opfn[a->size], NULL); +} + +static bool trans_VMLA_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpFn * const opfn[] = { + NULL, + gen_helper_neon_mul_u16, + tcg_gen_mul_i32, + NULL, + }; + static NeonGenTwoOpFn * const accfn[] = { + NULL, + gen_helper_neon_add_u16, + tcg_gen_add_i32, + NULL, + }; + + return do_2scalar(s, a, opfn[a->size], accfn[a->size]); +} + +static bool trans_VMLS_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpFn * const opfn[] = { + NULL, + gen_helper_neon_mul_u16, + tcg_gen_mul_i32, + NULL, + }; + static NeonGenTwoOpFn * const accfn[] = { + NULL, + gen_helper_neon_sub_u16, + tcg_gen_sub_i32, + NULL, + }; + + return do_2scalar(s, a, opfn[a->size], accfn[a->size]); +} + +/* + * Rather than have a float-specific version of do_2scalar just for + * three insns, we wrap a NeonGenTwoSingleOpFn to turn it into + * a NeonGenTwoOpFn. + */ +#define WRAP_FP_FN(WRAPNAME, FUNC) \ + static void WRAPNAME(TCGv_i32 rd, TCGv_i32 rn, TCGv_i32 rm) \ + { \ + TCGv_ptr fpstatus = get_fpstatus_ptr(1); \ + FUNC(rd, rn, rm, fpstatus); \ + tcg_temp_free_ptr(fpstatus); \ + } + +WRAP_FP_FN(gen_VMUL_F_mul, gen_helper_vfp_muls) +WRAP_FP_FN(gen_VMUL_F_add, gen_helper_vfp_adds) +WRAP_FP_FN(gen_VMUL_F_sub, gen_helper_vfp_subs) + +static bool trans_VMUL_F_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpFn * const opfn[] = { + NULL, + NULL, /* TODO: fp16 support */ + gen_VMUL_F_mul, + NULL, + }; + + return do_2scalar(s, a, opfn[a->size], NULL); +} + +static bool trans_VMLA_F_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpFn * const opfn[] = { + NULL, + NULL, /* TODO: fp16 support */ + gen_VMUL_F_mul, + NULL, + }; + static NeonGenTwoOpFn * const accfn[] = { + NULL, + NULL, /* TODO: fp16 support */ + gen_VMUL_F_add, + NULL, + }; + + return do_2scalar(s, a, opfn[a->size], accfn[a->size]); +} + +static bool trans_VMLS_F_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpFn * const opfn[] = { + NULL, + NULL, /* TODO: fp16 support */ + gen_VMUL_F_mul, + NULL, + }; + static NeonGenTwoOpFn * const accfn[] = { + NULL, + NULL, /* TODO: fp16 support */ + gen_VMUL_F_sub, + NULL, + }; + + return do_2scalar(s, a, opfn[a->size], accfn[a->size]); +} + +WRAP_ENV_FN(gen_VQDMULH_16, gen_helper_neon_qdmulh_s16) +WRAP_ENV_FN(gen_VQDMULH_32, gen_helper_neon_qdmulh_s32) +WRAP_ENV_FN(gen_VQRDMULH_16, gen_helper_neon_qrdmulh_s16) +WRAP_ENV_FN(gen_VQRDMULH_32, gen_helper_neon_qrdmulh_s32) + +static bool trans_VQDMULH_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpFn * const opfn[] = { + NULL, + gen_VQDMULH_16, + gen_VQDMULH_32, + NULL, + }; + + return do_2scalar(s, a, opfn[a->size], NULL); +} + +static bool trans_VQRDMULH_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpFn * const opfn[] = { + NULL, + gen_VQRDMULH_16, + gen_VQRDMULH_32, + NULL, + }; + + return do_2scalar(s, a, opfn[a->size], NULL); +} + +static bool do_vqrdmlah_2sc(DisasContext *s, arg_2scalar *a, + NeonGenThreeOpEnvFn *opfn) +{ + /* + * VQRDMLAH/VQRDMLSH: this is like do_2scalar, but the opfn + * performs a kind of fused op-then-accumulate using a helper + * function that takes all of rd, rn and the scalar at once. + */ + TCGv_i32 scalar; + int pass; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + if (!dc_isar_feature(aa32_rdm, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (!opfn) { + /* Bad size (including size == 3, which is a different insn group) */ + return false; + } + + if (a->q && ((a->vd | a->vn) & 1)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + scalar = neon_get_scalar(a->size, a->vm); + + for (pass = 0; pass < (a->q ? 4 : 2); pass++) { + TCGv_i32 rn = neon_load_reg(a->vn, pass); + TCGv_i32 rd = neon_load_reg(a->vd, pass); + opfn(rd, cpu_env, rn, scalar, rd); + tcg_temp_free_i32(rn); + neon_store_reg(a->vd, pass, rd); + } + tcg_temp_free_i32(scalar); + + return true; +} + +static bool trans_VQRDMLAH_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenThreeOpEnvFn *opfn[] = { + NULL, + gen_helper_neon_qrdmlah_s16, + gen_helper_neon_qrdmlah_s32, + NULL, + }; + return do_vqrdmlah_2sc(s, a, opfn[a->size]); +} + +static bool trans_VQRDMLSH_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenThreeOpEnvFn *opfn[] = { + NULL, + gen_helper_neon_qrdmlsh_s16, + gen_helper_neon_qrdmlsh_s32, + NULL, + }; + return do_vqrdmlah_2sc(s, a, opfn[a->size]); +} + +static bool do_2scalar_long(DisasContext *s, arg_2scalar *a, + NeonGenTwoOpWidenFn *opfn, + NeonGenTwo64OpFn *accfn) +{ + /* + * Two registers and a scalar, long operations: perform an + * operation on the input elements and the scalar which produces + * a double-width result, and then possibly perform an accumulation + * operation of that result into the destination. + */ + TCGv_i32 scalar, rn; + TCGv_i64 rn0_64, rn1_64; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (!opfn) { + /* Bad size (including size == 3, which is a different insn group) */ + return false; + } + + if (a->vd & 1) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + scalar = neon_get_scalar(a->size, a->vm); + + /* Load all inputs before writing any outputs, in case of overlap */ + rn = neon_load_reg(a->vn, 0); + rn0_64 = tcg_temp_new_i64(); + opfn(rn0_64, rn, scalar); + tcg_temp_free_i32(rn); + + rn = neon_load_reg(a->vn, 1); + rn1_64 = tcg_temp_new_i64(); + opfn(rn1_64, rn, scalar); + tcg_temp_free_i32(rn); + tcg_temp_free_i32(scalar); + + if (accfn) { + TCGv_i64 t64 = tcg_temp_new_i64(); + neon_load_reg64(t64, a->vd); + accfn(t64, t64, rn0_64); + neon_store_reg64(t64, a->vd); + neon_load_reg64(t64, a->vd + 1); + accfn(t64, t64, rn1_64); + neon_store_reg64(t64, a->vd + 1); + tcg_temp_free_i64(t64); + } else { + neon_store_reg64(rn0_64, a->vd); + neon_store_reg64(rn1_64, a->vd + 1); + } + tcg_temp_free_i64(rn0_64); + tcg_temp_free_i64(rn1_64); + return true; +} + +static bool trans_VMULL_S_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + NULL, + gen_helper_neon_mull_s16, + gen_mull_s32, + NULL, + }; + + return do_2scalar_long(s, a, opfn[a->size], NULL); +} + +static bool trans_VMULL_U_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + NULL, + gen_helper_neon_mull_u16, + gen_mull_u32, + NULL, + }; + + return do_2scalar_long(s, a, opfn[a->size], NULL); +} + +#define DO_VMLAL_2SC(INSN, MULL, ACC) \ + static bool trans_##INSN##_2sc(DisasContext *s, arg_2scalar *a) \ + { \ + static NeonGenTwoOpWidenFn * const opfn[] = { \ + NULL, \ + gen_helper_neon_##MULL##16, \ + gen_##MULL##32, \ + NULL, \ + }; \ + static NeonGenTwo64OpFn * const accfn[] = { \ + NULL, \ + gen_helper_neon_##ACC##l_u32, \ + tcg_gen_##ACC##_i64, \ + NULL, \ + }; \ + return do_2scalar_long(s, a, opfn[a->size], accfn[a->size]); \ + } + +DO_VMLAL_2SC(VMLAL_S, mull_s, add) +DO_VMLAL_2SC(VMLAL_U, mull_u, add) +DO_VMLAL_2SC(VMLSL_S, mull_s, sub) +DO_VMLAL_2SC(VMLSL_U, mull_u, sub) + +static bool trans_VQDMULL_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + NULL, + gen_VQDMULL_16, + gen_VQDMULL_32, + NULL, + }; + + return do_2scalar_long(s, a, opfn[a->size], NULL); +} + +static bool trans_VQDMLAL_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + NULL, + gen_VQDMULL_16, + gen_VQDMULL_32, + NULL, + }; + static NeonGenTwo64OpFn * const accfn[] = { + NULL, + gen_VQDMLAL_acc_16, + gen_VQDMLAL_acc_32, + NULL, + }; + + return do_2scalar_long(s, a, opfn[a->size], accfn[a->size]); +} + +static bool trans_VQDMLSL_2sc(DisasContext *s, arg_2scalar *a) +{ + static NeonGenTwoOpWidenFn * const opfn[] = { + NULL, + gen_VQDMULL_16, + gen_VQDMULL_32, + NULL, + }; + static NeonGenTwo64OpFn * const accfn[] = { + NULL, + gen_VQDMLSL_acc_16, + gen_VQDMLSL_acc_32, + NULL, + }; + + return do_2scalar_long(s, a, opfn[a->size], accfn[a->size]); +} + +static bool trans_VEXT(DisasContext *s, arg_VEXT *a) +{ + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if ((a->vn | a->vm | a->vd) & a->q) { + return false; + } + + if (a->imm > 7 && !a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + if (!a->q) { + /* Extract 64 bits from */ + TCGv_i64 left, right, dest; + + left = tcg_temp_new_i64(); + right = tcg_temp_new_i64(); + dest = tcg_temp_new_i64(); + + neon_load_reg64(right, a->vn); + neon_load_reg64(left, a->vm); + tcg_gen_extract2_i64(dest, right, left, a->imm * 8); + neon_store_reg64(dest, a->vd); + + tcg_temp_free_i64(left); + tcg_temp_free_i64(right); + tcg_temp_free_i64(dest); + } else { + /* Extract 128 bits from */ + TCGv_i64 left, middle, right, destleft, destright; + + left = tcg_temp_new_i64(); + middle = tcg_temp_new_i64(); + right = tcg_temp_new_i64(); + destleft = tcg_temp_new_i64(); + destright = tcg_temp_new_i64(); + + if (a->imm < 8) { + neon_load_reg64(right, a->vn); + neon_load_reg64(middle, a->vn + 1); + tcg_gen_extract2_i64(destright, right, middle, a->imm * 8); + neon_load_reg64(left, a->vm); + tcg_gen_extract2_i64(destleft, middle, left, a->imm * 8); + } else { + neon_load_reg64(right, a->vn + 1); + neon_load_reg64(middle, a->vm); + tcg_gen_extract2_i64(destright, right, middle, (a->imm - 8) * 8); + neon_load_reg64(left, a->vm + 1); + tcg_gen_extract2_i64(destleft, middle, left, (a->imm - 8) * 8); + } + + neon_store_reg64(destright, a->vd); + neon_store_reg64(destleft, a->vd + 1); + + tcg_temp_free_i64(destright); + tcg_temp_free_i64(destleft); + tcg_temp_free_i64(right); + tcg_temp_free_i64(middle); + tcg_temp_free_i64(left); + } + return true; +} + +static bool trans_VTBL(DisasContext *s, arg_VTBL *a) +{ + int n; + TCGv_i32 tmp, tmp2, tmp3, tmp4; + TCGv_ptr ptr1; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + n = a->len + 1; + if ((a->vn + n) > 32) { + /* + * This is UNPREDICTABLE; we choose to UNDEF to avoid the + * helper function running off the end of the register file. + */ + return false; + } + n <<= 3; + if (a->op) { + tmp = neon_load_reg(a->vd, 0); + } else { + tmp = tcg_temp_new_i32(); + tcg_gen_movi_i32(tmp, 0); + } + tmp2 = neon_load_reg(a->vm, 0); + ptr1 = vfp_reg_ptr(true, a->vn); + tmp4 = tcg_const_i32(n); + gen_helper_neon_tbl(tmp2, tmp2, tmp, ptr1, tmp4); + tcg_temp_free_i32(tmp); + if (a->op) { + tmp = neon_load_reg(a->vd, 1); + } else { + tmp = tcg_temp_new_i32(); + tcg_gen_movi_i32(tmp, 0); + } + tmp3 = neon_load_reg(a->vm, 1); + gen_helper_neon_tbl(tmp3, tmp3, tmp, ptr1, tmp4); + tcg_temp_free_i32(tmp4); + tcg_temp_free_ptr(ptr1); + neon_store_reg(a->vd, 0, tmp2); + neon_store_reg(a->vd, 1, tmp3); + tcg_temp_free_i32(tmp); + return true; +} + +static bool trans_VDUP_scalar(DisasContext *s, arg_VDUP_scalar *a) +{ + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->vd & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + tcg_gen_gvec_dup_mem(a->size, neon_reg_offset(a->vd, 0), + neon_element_offset(a->vm, a->index, a->size), + a->q ? 16 : 8, a->q ? 16 : 8); + return true; +} + +static bool trans_VREV64(DisasContext *s, arg_VREV64 *a) +{ + int pass, half; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if ((a->vd | a->vm) & a->q) { + return false; + } + + if (a->size == 3) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + for (pass = 0; pass < (a->q ? 2 : 1); pass++) { + TCGv_i32 tmp[2]; + + for (half = 0; half < 2; half++) { + tmp[half] = neon_load_reg(a->vm, pass * 2 + half); + switch (a->size) { + case 0: + tcg_gen_bswap32_i32(tmp[half], tmp[half]); + break; + case 1: + gen_swap_half(tmp[half], tmp[half]); + break; + case 2: + break; + default: + g_assert_not_reached(); + } + } + neon_store_reg(a->vd, pass * 2, tmp[1]); + neon_store_reg(a->vd, pass * 2 + 1, tmp[0]); + } + return true; +} + +static bool do_2misc_pairwise(DisasContext *s, arg_2misc *a, + NeonGenWidenFn *widenfn, + NeonGenTwo64OpFn *opfn, + NeonGenTwo64OpFn *accfn) +{ + /* + * Pairwise long operations: widen both halves of the pair, + * combine the pairs with the opfn, and then possibly accumulate + * into the destination with the accfn. + */ + int pass; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if ((a->vd | a->vm) & a->q) { + return false; + } + + if (!widenfn) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + for (pass = 0; pass < a->q + 1; pass++) { + TCGv_i32 tmp; + TCGv_i64 rm0_64, rm1_64, rd_64; + + rm0_64 = tcg_temp_new_i64(); + rm1_64 = tcg_temp_new_i64(); + rd_64 = tcg_temp_new_i64(); + tmp = neon_load_reg(a->vm, pass * 2); + widenfn(rm0_64, tmp); + tcg_temp_free_i32(tmp); + tmp = neon_load_reg(a->vm, pass * 2 + 1); + widenfn(rm1_64, tmp); + tcg_temp_free_i32(tmp); + opfn(rd_64, rm0_64, rm1_64); + tcg_temp_free_i64(rm0_64); + tcg_temp_free_i64(rm1_64); + + if (accfn) { + TCGv_i64 tmp64 = tcg_temp_new_i64(); + neon_load_reg64(tmp64, a->vd + pass); + accfn(rd_64, tmp64, rd_64); + tcg_temp_free_i64(tmp64); + } + neon_store_reg64(rd_64, a->vd + pass); + tcg_temp_free_i64(rd_64); + } + return true; +} + +static bool trans_VPADDL_S(DisasContext *s, arg_2misc *a) +{ + static NeonGenWidenFn * const widenfn[] = { + gen_helper_neon_widen_s8, + gen_helper_neon_widen_s16, + tcg_gen_ext_i32_i64, + NULL, + }; + static NeonGenTwo64OpFn * const opfn[] = { + gen_helper_neon_paddl_u16, + gen_helper_neon_paddl_u32, + tcg_gen_add_i64, + NULL, + }; + + return do_2misc_pairwise(s, a, widenfn[a->size], opfn[a->size], NULL); +} + +static bool trans_VPADDL_U(DisasContext *s, arg_2misc *a) +{ + static NeonGenWidenFn * const widenfn[] = { + gen_helper_neon_widen_u8, + gen_helper_neon_widen_u16, + tcg_gen_extu_i32_i64, + NULL, + }; + static NeonGenTwo64OpFn * const opfn[] = { + gen_helper_neon_paddl_u16, + gen_helper_neon_paddl_u32, + tcg_gen_add_i64, + NULL, + }; + + return do_2misc_pairwise(s, a, widenfn[a->size], opfn[a->size], NULL); +} + +static bool trans_VPADAL_S(DisasContext *s, arg_2misc *a) +{ + static NeonGenWidenFn * const widenfn[] = { + gen_helper_neon_widen_s8, + gen_helper_neon_widen_s16, + tcg_gen_ext_i32_i64, + NULL, + }; + static NeonGenTwo64OpFn * const opfn[] = { + gen_helper_neon_paddl_u16, + gen_helper_neon_paddl_u32, + tcg_gen_add_i64, + NULL, + }; + static NeonGenTwo64OpFn * const accfn[] = { + gen_helper_neon_addl_u16, + gen_helper_neon_addl_u32, + tcg_gen_add_i64, + NULL, + }; + + return do_2misc_pairwise(s, a, widenfn[a->size], opfn[a->size], + accfn[a->size]); +} + +static bool trans_VPADAL_U(DisasContext *s, arg_2misc *a) +{ + static NeonGenWidenFn * const widenfn[] = { + gen_helper_neon_widen_u8, + gen_helper_neon_widen_u16, + tcg_gen_extu_i32_i64, + NULL, + }; + static NeonGenTwo64OpFn * const opfn[] = { + gen_helper_neon_paddl_u16, + gen_helper_neon_paddl_u32, + tcg_gen_add_i64, + NULL, + }; + static NeonGenTwo64OpFn * const accfn[] = { + gen_helper_neon_addl_u16, + gen_helper_neon_addl_u32, + tcg_gen_add_i64, + NULL, + }; + + return do_2misc_pairwise(s, a, widenfn[a->size], opfn[a->size], + accfn[a->size]); +} + +typedef void ZipFn(TCGv_ptr, TCGv_ptr); + +static bool do_zip_uzp(DisasContext *s, arg_2misc *a, + ZipFn *fn) +{ + TCGv_ptr pd, pm; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if ((a->vd | a->vm) & a->q) { + return false; + } + + if (!fn) { + /* Bad size or size/q combination */ + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + pd = vfp_reg_ptr(true, a->vd); + pm = vfp_reg_ptr(true, a->vm); + fn(pd, pm); + tcg_temp_free_ptr(pd); + tcg_temp_free_ptr(pm); + return true; +} + +static bool trans_VUZP(DisasContext *s, arg_2misc *a) +{ + static ZipFn * const fn[2][4] = { + { + gen_helper_neon_unzip8, + gen_helper_neon_unzip16, + NULL, + NULL, + }, { + gen_helper_neon_qunzip8, + gen_helper_neon_qunzip16, + gen_helper_neon_qunzip32, + NULL, + } + }; + return do_zip_uzp(s, a, fn[a->q][a->size]); +} + +static bool trans_VZIP(DisasContext *s, arg_2misc *a) +{ + static ZipFn * const fn[2][4] = { + { + gen_helper_neon_zip8, + gen_helper_neon_zip16, + NULL, + NULL, + }, { + gen_helper_neon_qzip8, + gen_helper_neon_qzip16, + gen_helper_neon_qzip32, + NULL, + } + }; + return do_zip_uzp(s, a, fn[a->q][a->size]); +} + +static bool do_vmovn(DisasContext *s, arg_2misc *a, + NeonGenNarrowEnvFn *narrowfn) +{ + TCGv_i64 rm; + TCGv_i32 rd0, rd1; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->vm & 1) { + return false; + } + + if (!narrowfn) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + rm = tcg_temp_new_i64(); + rd0 = tcg_temp_new_i32(); + rd1 = tcg_temp_new_i32(); + + neon_load_reg64(rm, a->vm); + narrowfn(rd0, cpu_env, rm); + neon_load_reg64(rm, a->vm + 1); + narrowfn(rd1, cpu_env, rm); + neon_store_reg(a->vd, 0, rd0); + neon_store_reg(a->vd, 1, rd1); + tcg_temp_free_i64(rm); + return true; +} + +#define DO_VMOVN(INSN, FUNC) \ + static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ + { \ + static NeonGenNarrowEnvFn * const narrowfn[] = { \ + FUNC##8, \ + FUNC##16, \ + FUNC##32, \ + NULL, \ + }; \ + return do_vmovn(s, a, narrowfn[a->size]); \ + } + +DO_VMOVN(VMOVN, gen_neon_narrow_u) +DO_VMOVN(VQMOVUN, gen_helper_neon_unarrow_sat) +DO_VMOVN(VQMOVN_S, gen_helper_neon_narrow_sat_s) +DO_VMOVN(VQMOVN_U, gen_helper_neon_narrow_sat_u) + +static bool trans_VSHLL(DisasContext *s, arg_2misc *a) +{ + TCGv_i32 rm0, rm1; + TCGv_i64 rd; + static NeonGenWidenFn * const widenfns[] = { + gen_helper_neon_widen_u8, + gen_helper_neon_widen_u16, + tcg_gen_extu_i32_i64, + NULL, + }; + NeonGenWidenFn *widenfn = widenfns[a->size]; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->vd & 1) { + return false; + } + + if (!widenfn) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + rd = tcg_temp_new_i64(); + + rm0 = neon_load_reg(a->vm, 0); + rm1 = neon_load_reg(a->vm, 1); + + widenfn(rd, rm0); + tcg_gen_shli_i64(rd, rd, 8 << a->size); + neon_store_reg64(rd, a->vd); + widenfn(rd, rm1); + tcg_gen_shli_i64(rd, rd, 8 << a->size); + neon_store_reg64(rd, a->vd + 1); + + tcg_temp_free_i64(rd); + tcg_temp_free_i32(rm0); + tcg_temp_free_i32(rm1); + return true; +} + +static bool trans_VCVT_F16_F32(DisasContext *s, arg_2misc *a) +{ + TCGv_ptr fpst; + TCGv_i32 ahp, tmp, tmp2, tmp3; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON) || + !dc_isar_feature(aa32_fp16_spconv, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if ((a->vm & 1) || (a->size != 1)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(true); + ahp = get_ahp_flag(); + tmp = neon_load_reg(a->vm, 0); + gen_helper_vfp_fcvt_f32_to_f16(tmp, tmp, fpst, ahp); + tmp2 = neon_load_reg(a->vm, 1); + gen_helper_vfp_fcvt_f32_to_f16(tmp2, tmp2, fpst, ahp); + tcg_gen_shli_i32(tmp2, tmp2, 16); + tcg_gen_or_i32(tmp2, tmp2, tmp); + tcg_temp_free_i32(tmp); + tmp = neon_load_reg(a->vm, 2); + gen_helper_vfp_fcvt_f32_to_f16(tmp, tmp, fpst, ahp); + tmp3 = neon_load_reg(a->vm, 3); + neon_store_reg(a->vd, 0, tmp2); + gen_helper_vfp_fcvt_f32_to_f16(tmp3, tmp3, fpst, ahp); + tcg_gen_shli_i32(tmp3, tmp3, 16); + tcg_gen_or_i32(tmp3, tmp3, tmp); + neon_store_reg(a->vd, 1, tmp3); + tcg_temp_free_i32(tmp); + tcg_temp_free_i32(ahp); + tcg_temp_free_ptr(fpst); + + return true; +} + +static bool trans_VCVT_F32_F16(DisasContext *s, arg_2misc *a) +{ + TCGv_ptr fpst; + TCGv_i32 ahp, tmp, tmp2, tmp3; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON) || + !dc_isar_feature(aa32_fp16_spconv, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if ((a->vd & 1) || (a->size != 1)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(true); + ahp = get_ahp_flag(); + tmp3 = tcg_temp_new_i32(); + tmp = neon_load_reg(a->vm, 0); + tmp2 = neon_load_reg(a->vm, 1); + tcg_gen_ext16u_i32(tmp3, tmp); + gen_helper_vfp_fcvt_f16_to_f32(tmp3, tmp3, fpst, ahp); + neon_store_reg(a->vd, 0, tmp3); + tcg_gen_shri_i32(tmp, tmp, 16); + gen_helper_vfp_fcvt_f16_to_f32(tmp, tmp, fpst, ahp); + neon_store_reg(a->vd, 1, tmp); + tmp3 = tcg_temp_new_i32(); + tcg_gen_ext16u_i32(tmp3, tmp2); + gen_helper_vfp_fcvt_f16_to_f32(tmp3, tmp3, fpst, ahp); + neon_store_reg(a->vd, 2, tmp3); + tcg_gen_shri_i32(tmp2, tmp2, 16); + gen_helper_vfp_fcvt_f16_to_f32(tmp2, tmp2, fpst, ahp); + neon_store_reg(a->vd, 3, tmp2); + tcg_temp_free_i32(ahp); + tcg_temp_free_ptr(fpst); + + return true; +} + +static bool do_2misc_vec(DisasContext *s, arg_2misc *a, GVecGen2Fn *fn) +{ + int vec_size = a->q ? 16 : 8; + int rd_ofs = neon_reg_offset(a->vd, 0); + int rm_ofs = neon_reg_offset(a->vm, 0); + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->size == 3) { + return false; + } + + if ((a->vd | a->vm) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fn(a->size, rd_ofs, rm_ofs, vec_size, vec_size); + + return true; +} + +#define DO_2MISC_VEC(INSN, FN) \ + static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ + { \ + return do_2misc_vec(s, a, FN); \ + } + +DO_2MISC_VEC(VNEG, tcg_gen_gvec_neg) +DO_2MISC_VEC(VABS, tcg_gen_gvec_abs) +DO_2MISC_VEC(VCEQ0, gen_gvec_ceq0) +DO_2MISC_VEC(VCGT0, gen_gvec_cgt0) +DO_2MISC_VEC(VCLE0, gen_gvec_cle0) +DO_2MISC_VEC(VCGE0, gen_gvec_cge0) +DO_2MISC_VEC(VCLT0, gen_gvec_clt0) + +static bool trans_VMVN(DisasContext *s, arg_2misc *a) +{ + if (a->size != 0) { + return false; + } + return do_2misc_vec(s, a, tcg_gen_gvec_not); +} + +#define WRAP_2M_3_OOL_FN(WRAPNAME, FUNC, DATA) \ + static void WRAPNAME(unsigned vece, uint32_t rd_ofs, \ + uint32_t rm_ofs, uint32_t oprsz, \ + uint32_t maxsz) \ + { \ + tcg_gen_gvec_3_ool(rd_ofs, rd_ofs, rm_ofs, oprsz, maxsz, \ + DATA, FUNC); \ + } + +#define WRAP_2M_2_OOL_FN(WRAPNAME, FUNC, DATA) \ + static void WRAPNAME(unsigned vece, uint32_t rd_ofs, \ + uint32_t rm_ofs, uint32_t oprsz, \ + uint32_t maxsz) \ + { \ + tcg_gen_gvec_2_ool(rd_ofs, rm_ofs, oprsz, maxsz, DATA, FUNC); \ + } + +WRAP_2M_3_OOL_FN(gen_AESE, gen_helper_crypto_aese, 0) +WRAP_2M_3_OOL_FN(gen_AESD, gen_helper_crypto_aese, 1) +WRAP_2M_2_OOL_FN(gen_AESMC, gen_helper_crypto_aesmc, 0) +WRAP_2M_2_OOL_FN(gen_AESIMC, gen_helper_crypto_aesmc, 1) +WRAP_2M_2_OOL_FN(gen_SHA1H, gen_helper_crypto_sha1h, 0) +WRAP_2M_2_OOL_FN(gen_SHA1SU1, gen_helper_crypto_sha1su1, 0) +WRAP_2M_2_OOL_FN(gen_SHA256SU0, gen_helper_crypto_sha256su0, 0) + +#define DO_2M_CRYPTO(INSN, FEATURE, SIZE) \ + static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ + { \ + if (!dc_isar_feature(FEATURE, s) || a->size != SIZE) { \ + return false; \ + } \ + return do_2misc_vec(s, a, gen_##INSN); \ + } + +DO_2M_CRYPTO(AESE, aa32_aes, 0) +DO_2M_CRYPTO(AESD, aa32_aes, 0) +DO_2M_CRYPTO(AESMC, aa32_aes, 0) +DO_2M_CRYPTO(AESIMC, aa32_aes, 0) +DO_2M_CRYPTO(SHA1H, aa32_sha1, 2) +DO_2M_CRYPTO(SHA1SU1, aa32_sha1, 2) +DO_2M_CRYPTO(SHA256SU0, aa32_sha2, 2) + +static bool do_2misc(DisasContext *s, arg_2misc *a, NeonGenOneOpFn *fn) +{ + int pass; + + /* Handle a 2-reg-misc operation by iterating 32 bits at a time */ + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (!fn) { + return false; + } + + if ((a->vd | a->vm) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + for (pass = 0; pass < (a->q ? 4 : 2); pass++) { + TCGv_i32 tmp = neon_load_reg(a->vm, pass); + fn(tmp, tmp); + neon_store_reg(a->vd, pass, tmp); + } + + return true; +} + +static bool trans_VREV32(DisasContext *s, arg_2misc *a) +{ + static NeonGenOneOpFn * const fn[] = { + tcg_gen_bswap32_i32, + gen_swap_half, + NULL, + NULL, + }; + return do_2misc(s, a, fn[a->size]); +} + +static bool trans_VREV16(DisasContext *s, arg_2misc *a) +{ + if (a->size != 0) { + return false; + } + return do_2misc(s, a, gen_rev16); +} + +static bool trans_VCLS(DisasContext *s, arg_2misc *a) +{ + static NeonGenOneOpFn * const fn[] = { + gen_helper_neon_cls_s8, + gen_helper_neon_cls_s16, + gen_helper_neon_cls_s32, + NULL, + }; + return do_2misc(s, a, fn[a->size]); +} + +static void do_VCLZ_32(TCGv_i32 rd, TCGv_i32 rm) +{ + tcg_gen_clzi_i32(rd, rm, 32); +} + +static bool trans_VCLZ(DisasContext *s, arg_2misc *a) +{ + static NeonGenOneOpFn * const fn[] = { + gen_helper_neon_clz_u8, + gen_helper_neon_clz_u16, + do_VCLZ_32, + NULL, + }; + return do_2misc(s, a, fn[a->size]); +} + +static bool trans_VCNT(DisasContext *s, arg_2misc *a) +{ + if (a->size != 0) { + return false; + } + return do_2misc(s, a, gen_helper_neon_cnt_u8); +} + +static bool trans_VABS_F(DisasContext *s, arg_2misc *a) +{ + if (a->size != 2) { + return false; + } + /* TODO: FP16 : size == 1 */ + return do_2misc(s, a, gen_helper_vfp_abss); +} + +static bool trans_VNEG_F(DisasContext *s, arg_2misc *a) +{ + if (a->size != 2) { + return false; + } + /* TODO: FP16 : size == 1 */ + return do_2misc(s, a, gen_helper_vfp_negs); +} + +static bool trans_VRECPE(DisasContext *s, arg_2misc *a) +{ + if (a->size != 2) { + return false; + } + return do_2misc(s, a, gen_helper_recpe_u32); +} + +static bool trans_VRSQRTE(DisasContext *s, arg_2misc *a) +{ + if (a->size != 2) { + return false; + } + return do_2misc(s, a, gen_helper_rsqrte_u32); +} + +#define WRAP_1OP_ENV_FN(WRAPNAME, FUNC) \ + static void WRAPNAME(TCGv_i32 d, TCGv_i32 m) \ + { \ + FUNC(d, cpu_env, m); \ + } + +WRAP_1OP_ENV_FN(gen_VQABS_s8, gen_helper_neon_qabs_s8) +WRAP_1OP_ENV_FN(gen_VQABS_s16, gen_helper_neon_qabs_s16) +WRAP_1OP_ENV_FN(gen_VQABS_s32, gen_helper_neon_qabs_s32) +WRAP_1OP_ENV_FN(gen_VQNEG_s8, gen_helper_neon_qneg_s8) +WRAP_1OP_ENV_FN(gen_VQNEG_s16, gen_helper_neon_qneg_s16) +WRAP_1OP_ENV_FN(gen_VQNEG_s32, gen_helper_neon_qneg_s32) + +static bool trans_VQABS(DisasContext *s, arg_2misc *a) +{ + static NeonGenOneOpFn * const fn[] = { + gen_VQABS_s8, + gen_VQABS_s16, + gen_VQABS_s32, + NULL, + }; + return do_2misc(s, a, fn[a->size]); +} + +static bool trans_VQNEG(DisasContext *s, arg_2misc *a) +{ + static NeonGenOneOpFn * const fn[] = { + gen_VQNEG_s8, + gen_VQNEG_s16, + gen_VQNEG_s32, + NULL, + }; + return do_2misc(s, a, fn[a->size]); +} + +static bool do_2misc_fp(DisasContext *s, arg_2misc *a, + NeonGenOneSingleOpFn *fn) +{ + int pass; + TCGv_ptr fpst; + + /* Handle a 2-reg-misc operation by iterating 32 bits at a time */ + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->size != 2) { + /* TODO: FP16 will be the size == 1 case */ + return false; + } + + if ((a->vd | a->vm) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(1); + for (pass = 0; pass < (a->q ? 4 : 2); pass++) { + TCGv_i32 tmp = neon_load_reg(a->vm, pass); + fn(tmp, tmp, fpst); + neon_store_reg(a->vd, pass, tmp); + } + tcg_temp_free_ptr(fpst); + + return true; +} + +#define DO_2MISC_FP(INSN, FUNC) \ + static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ + { \ + return do_2misc_fp(s, a, FUNC); \ + } + +DO_2MISC_FP(VRECPE_F, gen_helper_recpe_f32) +DO_2MISC_FP(VRSQRTE_F, gen_helper_rsqrte_f32) +DO_2MISC_FP(VCVT_FS, gen_helper_vfp_sitos) +DO_2MISC_FP(VCVT_FU, gen_helper_vfp_uitos) +DO_2MISC_FP(VCVT_SF, gen_helper_vfp_tosizs) +DO_2MISC_FP(VCVT_UF, gen_helper_vfp_touizs) + +static bool trans_VRINTX(DisasContext *s, arg_2misc *a) +{ + if (!arm_dc_feature(s, ARM_FEATURE_V8)) { + return false; + } + return do_2misc_fp(s, a, gen_helper_rints_exact); +} + +#define WRAP_FP_CMP0_FWD(WRAPNAME, FUNC) \ + static void WRAPNAME(TCGv_i32 d, TCGv_i32 m, TCGv_ptr fpst) \ + { \ + TCGv_i32 zero = tcg_const_i32(0); \ + FUNC(d, m, zero, fpst); \ + tcg_temp_free_i32(zero); \ + } +#define WRAP_FP_CMP0_REV(WRAPNAME, FUNC) \ + static void WRAPNAME(TCGv_i32 d, TCGv_i32 m, TCGv_ptr fpst) \ + { \ + TCGv_i32 zero = tcg_const_i32(0); \ + FUNC(d, zero, m, fpst); \ + tcg_temp_free_i32(zero); \ + } + +#define DO_FP_CMP0(INSN, FUNC, REV) \ + WRAP_FP_CMP0_##REV(gen_##INSN, FUNC) \ + static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ + { \ + return do_2misc_fp(s, a, gen_##INSN); \ + } + +DO_FP_CMP0(VCGT0_F, gen_helper_neon_cgt_f32, FWD) +DO_FP_CMP0(VCGE0_F, gen_helper_neon_cge_f32, FWD) +DO_FP_CMP0(VCEQ0_F, gen_helper_neon_ceq_f32, FWD) +DO_FP_CMP0(VCLE0_F, gen_helper_neon_cge_f32, REV) +DO_FP_CMP0(VCLT0_F, gen_helper_neon_cgt_f32, REV) + +static bool do_vrint(DisasContext *s, arg_2misc *a, int rmode) +{ + /* + * Handle a VRINT* operation by iterating 32 bits at a time, + * with a specified rounding mode in operation. + */ + int pass; + TCGv_ptr fpst; + TCGv_i32 tcg_rmode; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON) || + !arm_dc_feature(s, ARM_FEATURE_V8)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->size != 2) { + /* TODO: FP16 will be the size == 1 case */ + return false; + } + + if ((a->vd | a->vm) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(1); + tcg_rmode = tcg_const_i32(arm_rmode_to_sf(rmode)); + gen_helper_set_neon_rmode(tcg_rmode, tcg_rmode, cpu_env); + for (pass = 0; pass < (a->q ? 4 : 2); pass++) { + TCGv_i32 tmp = neon_load_reg(a->vm, pass); + gen_helper_rints(tmp, tmp, fpst); + neon_store_reg(a->vd, pass, tmp); + } + gen_helper_set_neon_rmode(tcg_rmode, tcg_rmode, cpu_env); + tcg_temp_free_i32(tcg_rmode); + tcg_temp_free_ptr(fpst); + + return true; +} + +#define DO_VRINT(INSN, RMODE) \ + static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ + { \ + return do_vrint(s, a, RMODE); \ + } + +DO_VRINT(VRINTN, FPROUNDING_TIEEVEN) +DO_VRINT(VRINTA, FPROUNDING_TIEAWAY) +DO_VRINT(VRINTZ, FPROUNDING_ZERO) +DO_VRINT(VRINTM, FPROUNDING_NEGINF) +DO_VRINT(VRINTP, FPROUNDING_POSINF) + +static bool do_vcvt(DisasContext *s, arg_2misc *a, int rmode, bool is_signed) +{ + /* + * Handle a VCVT* operation by iterating 32 bits at a time, + * with a specified rounding mode in operation. + */ + int pass; + TCGv_ptr fpst; + TCGv_i32 tcg_rmode, tcg_shift; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON) || + !arm_dc_feature(s, ARM_FEATURE_V8)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->size != 2) { + /* TODO: FP16 will be the size == 1 case */ + return false; + } + + if ((a->vd | a->vm) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(1); + tcg_shift = tcg_const_i32(0); + tcg_rmode = tcg_const_i32(arm_rmode_to_sf(rmode)); + gen_helper_set_neon_rmode(tcg_rmode, tcg_rmode, cpu_env); + for (pass = 0; pass < (a->q ? 4 : 2); pass++) { + TCGv_i32 tmp = neon_load_reg(a->vm, pass); + if (is_signed) { + gen_helper_vfp_tosls(tmp, tmp, tcg_shift, fpst); + } else { + gen_helper_vfp_touls(tmp, tmp, tcg_shift, fpst); + } + neon_store_reg(a->vd, pass, tmp); + } + gen_helper_set_neon_rmode(tcg_rmode, tcg_rmode, cpu_env); + tcg_temp_free_i32(tcg_rmode); + tcg_temp_free_i32(tcg_shift); + tcg_temp_free_ptr(fpst); + + return true; +} + +#define DO_VCVT(INSN, RMODE, SIGNED) \ + static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ + { \ + return do_vcvt(s, a, RMODE, SIGNED); \ + } + +DO_VCVT(VCVTAU, FPROUNDING_TIEAWAY, false) +DO_VCVT(VCVTAS, FPROUNDING_TIEAWAY, true) +DO_VCVT(VCVTNU, FPROUNDING_TIEEVEN, false) +DO_VCVT(VCVTNS, FPROUNDING_TIEEVEN, true) +DO_VCVT(VCVTPU, FPROUNDING_POSINF, false) +DO_VCVT(VCVTPS, FPROUNDING_POSINF, true) +DO_VCVT(VCVTMU, FPROUNDING_NEGINF, false) +DO_VCVT(VCVTMS, FPROUNDING_NEGINF, true) + +static bool trans_VSWP(DisasContext *s, arg_2misc *a) +{ + TCGv_i64 rm, rd; + int pass; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (a->size != 0) { + return false; + } + + if ((a->vd | a->vm) & a->q) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + rm = tcg_temp_new_i64(); + rd = tcg_temp_new_i64(); + for (pass = 0; pass < (a->q ? 2 : 1); pass++) { + neon_load_reg64(rm, a->vm + pass); + neon_load_reg64(rd, a->vd + pass); + neon_store_reg64(rm, a->vd + pass); + neon_store_reg64(rd, a->vm + pass); + } + tcg_temp_free_i64(rm); + tcg_temp_free_i64(rd); + + return true; +} +static void gen_neon_trn_u8(TCGv_i32 t0, TCGv_i32 t1) +{ + TCGv_i32 rd, tmp; + + rd = tcg_temp_new_i32(); + tmp = tcg_temp_new_i32(); + + tcg_gen_shli_i32(rd, t0, 8); + tcg_gen_andi_i32(rd, rd, 0xff00ff00); + tcg_gen_andi_i32(tmp, t1, 0x00ff00ff); + tcg_gen_or_i32(rd, rd, tmp); + + tcg_gen_shri_i32(t1, t1, 8); + tcg_gen_andi_i32(t1, t1, 0x00ff00ff); + tcg_gen_andi_i32(tmp, t0, 0xff00ff00); + tcg_gen_or_i32(t1, t1, tmp); + tcg_gen_mov_i32(t0, rd); + + tcg_temp_free_i32(tmp); + tcg_temp_free_i32(rd); +} + +static void gen_neon_trn_u16(TCGv_i32 t0, TCGv_i32 t1) +{ + TCGv_i32 rd, tmp; + + rd = tcg_temp_new_i32(); + tmp = tcg_temp_new_i32(); + + tcg_gen_shli_i32(rd, t0, 16); + tcg_gen_andi_i32(tmp, t1, 0xffff); + tcg_gen_or_i32(rd, rd, tmp); + tcg_gen_shri_i32(t1, t1, 16); + tcg_gen_andi_i32(tmp, t0, 0xffff0000); + tcg_gen_or_i32(t1, t1, tmp); + tcg_gen_mov_i32(t0, rd); + + tcg_temp_free_i32(tmp); + tcg_temp_free_i32(rd); +} + +static bool trans_VTRN(DisasContext *s, arg_2misc *a) +{ + TCGv_i32 tmp, tmp2; + int pass; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vm) & 0x10)) { + return false; + } + + if ((a->vd | a->vm) & a->q) { + return false; + } + + if (a->size == 3) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + if (a->size == 2) { + for (pass = 0; pass < (a->q ? 4 : 2); pass += 2) { + tmp = neon_load_reg(a->vm, pass); + tmp2 = neon_load_reg(a->vd, pass + 1); + neon_store_reg(a->vm, pass, tmp2); + neon_store_reg(a->vd, pass + 1, tmp); + } + } else { + for (pass = 0; pass < (a->q ? 4 : 2); pass++) { + tmp = neon_load_reg(a->vm, pass); + tmp2 = neon_load_reg(a->vd, pass); + if (a->size == 0) { + gen_neon_trn_u8(tmp, tmp2); + } else { + gen_neon_trn_u16(tmp, tmp2); + } + neon_store_reg(a->vm, pass, tmp2); + neon_store_reg(a->vd, pass, tmp); + } + } + return true; +} diff --git a/target/arm/translate-neon.inc.c b/target/arm/translate-neon.inc.c deleted file mode 100644 index f6cb921..0000000 --- a/target/arm/translate-neon.inc.c +++ /dev/null @@ -1,4161 +0,0 @@ -/* - * ARM translation: AArch32 Neon instructions - * - * Copyright (c) 2003 Fabrice Bellard - * Copyright (c) 2005-2007 CodeSourcery - * Copyright (c) 2007 OpenedHand, Ltd. - * Copyright (c) 2020 Linaro, Ltd. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, see . - */ - -/* - * This file is intended to be included from translate.c; it uses - * some macros and definitions provided by that file. - * It might be possible to convert it to a standalone .c file eventually. - */ - -static inline int plus1(DisasContext *s, int x) -{ - return x + 1; -} - -static inline int rsub_64(DisasContext *s, int x) -{ - return 64 - x; -} - -static inline int rsub_32(DisasContext *s, int x) -{ - return 32 - x; -} -static inline int rsub_16(DisasContext *s, int x) -{ - return 16 - x; -} -static inline int rsub_8(DisasContext *s, int x) -{ - return 8 - x; -} - -/* Include the generated Neon decoder */ -#include "decode-neon-dp.inc.c" -#include "decode-neon-ls.inc.c" -#include "decode-neon-shared.inc.c" - -/* Return the offset of a 2**SIZE piece of a NEON register, at index ELE, - * where 0 is the least significant end of the register. - */ -static inline long -neon_element_offset(int reg, int element, MemOp size) -{ - int element_size = 1 << size; - int ofs = element * element_size; -#ifdef HOST_WORDS_BIGENDIAN - /* Calculate the offset assuming fully little-endian, - * then XOR to account for the order of the 8-byte units. - */ - if (element_size < 8) { - ofs ^= 8 - element_size; - } -#endif - return neon_reg_offset(reg, 0) + ofs; -} - -static void neon_load_element(TCGv_i32 var, int reg, int ele, MemOp mop) -{ - long offset = neon_element_offset(reg, ele, mop & MO_SIZE); - - switch (mop) { - case MO_UB: - tcg_gen_ld8u_i32(var, cpu_env, offset); - break; - case MO_UW: - tcg_gen_ld16u_i32(var, cpu_env, offset); - break; - case MO_UL: - tcg_gen_ld_i32(var, cpu_env, offset); - break; - default: - g_assert_not_reached(); - } -} - -static void neon_load_element64(TCGv_i64 var, int reg, int ele, MemOp mop) -{ - long offset = neon_element_offset(reg, ele, mop & MO_SIZE); - - switch (mop) { - case MO_UB: - tcg_gen_ld8u_i64(var, cpu_env, offset); - break; - case MO_UW: - tcg_gen_ld16u_i64(var, cpu_env, offset); - break; - case MO_UL: - tcg_gen_ld32u_i64(var, cpu_env, offset); - break; - case MO_Q: - tcg_gen_ld_i64(var, cpu_env, offset); - break; - default: - g_assert_not_reached(); - } -} - -static void neon_store_element(int reg, int ele, MemOp size, TCGv_i32 var) -{ - long offset = neon_element_offset(reg, ele, size); - - switch (size) { - case MO_8: - tcg_gen_st8_i32(var, cpu_env, offset); - break; - case MO_16: - tcg_gen_st16_i32(var, cpu_env, offset); - break; - case MO_32: - tcg_gen_st_i32(var, cpu_env, offset); - break; - default: - g_assert_not_reached(); - } -} - -static void neon_store_element64(int reg, int ele, MemOp size, TCGv_i64 var) -{ - long offset = neon_element_offset(reg, ele, size); - - switch (size) { - case MO_8: - tcg_gen_st8_i64(var, cpu_env, offset); - break; - case MO_16: - tcg_gen_st16_i64(var, cpu_env, offset); - break; - case MO_32: - tcg_gen_st32_i64(var, cpu_env, offset); - break; - case MO_64: - tcg_gen_st_i64(var, cpu_env, offset); - break; - default: - g_assert_not_reached(); - } -} - -static bool trans_VCMLA(DisasContext *s, arg_VCMLA *a) -{ - int opr_sz; - TCGv_ptr fpst; - gen_helper_gvec_3_ptr *fn_gvec_ptr; - - if (!dc_isar_feature(aa32_vcma, s) - || (!a->size && !dc_isar_feature(aa32_fp16_arith, s))) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if ((a->vn | a->vm | a->vd) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - opr_sz = (1 + a->q) * 8; - fpst = get_fpstatus_ptr(1); - fn_gvec_ptr = a->size ? gen_helper_gvec_fcmlas : gen_helper_gvec_fcmlah; - tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), - vfp_reg_offset(1, a->vn), - vfp_reg_offset(1, a->vm), - fpst, opr_sz, opr_sz, a->rot, - fn_gvec_ptr); - tcg_temp_free_ptr(fpst); - return true; -} - -static bool trans_VCADD(DisasContext *s, arg_VCADD *a) -{ - int opr_sz; - TCGv_ptr fpst; - gen_helper_gvec_3_ptr *fn_gvec_ptr; - - if (!dc_isar_feature(aa32_vcma, s) - || (!a->size && !dc_isar_feature(aa32_fp16_arith, s))) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if ((a->vn | a->vm | a->vd) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - opr_sz = (1 + a->q) * 8; - fpst = get_fpstatus_ptr(1); - fn_gvec_ptr = a->size ? gen_helper_gvec_fcadds : gen_helper_gvec_fcaddh; - tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), - vfp_reg_offset(1, a->vn), - vfp_reg_offset(1, a->vm), - fpst, opr_sz, opr_sz, a->rot, - fn_gvec_ptr); - tcg_temp_free_ptr(fpst); - return true; -} - -static bool trans_VDOT(DisasContext *s, arg_VDOT *a) -{ - int opr_sz; - gen_helper_gvec_3 *fn_gvec; - - if (!dc_isar_feature(aa32_dp, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if ((a->vn | a->vm | a->vd) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - opr_sz = (1 + a->q) * 8; - fn_gvec = a->u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b; - tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd), - vfp_reg_offset(1, a->vn), - vfp_reg_offset(1, a->vm), - opr_sz, opr_sz, 0, fn_gvec); - return true; -} - -static bool trans_VFML(DisasContext *s, arg_VFML *a) -{ - int opr_sz; - - if (!dc_isar_feature(aa32_fhm, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - (a->vd & 0x10)) { - return false; - } - - if (a->vd & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - opr_sz = (1 + a->q) * 8; - tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), - vfp_reg_offset(a->q, a->vn), - vfp_reg_offset(a->q, a->vm), - cpu_env, opr_sz, opr_sz, a->s, /* is_2 == 0 */ - gen_helper_gvec_fmlal_a32); - return true; -} - -static bool trans_VCMLA_scalar(DisasContext *s, arg_VCMLA_scalar *a) -{ - gen_helper_gvec_3_ptr *fn_gvec_ptr; - int opr_sz; - TCGv_ptr fpst; - - if (!dc_isar_feature(aa32_vcma, s)) { - return false; - } - if (a->size == 0 && !dc_isar_feature(aa32_fp16_arith, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if ((a->vd | a->vn) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fn_gvec_ptr = (a->size ? gen_helper_gvec_fcmlas_idx - : gen_helper_gvec_fcmlah_idx); - opr_sz = (1 + a->q) * 8; - fpst = get_fpstatus_ptr(1); - tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), - vfp_reg_offset(1, a->vn), - vfp_reg_offset(1, a->vm), - fpst, opr_sz, opr_sz, - (a->index << 2) | a->rot, fn_gvec_ptr); - tcg_temp_free_ptr(fpst); - return true; -} - -static bool trans_VDOT_scalar(DisasContext *s, arg_VDOT_scalar *a) -{ - gen_helper_gvec_3 *fn_gvec; - int opr_sz; - TCGv_ptr fpst; - - if (!dc_isar_feature(aa32_dp, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn) & 0x10)) { - return false; - } - - if ((a->vd | a->vn) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fn_gvec = a->u ? gen_helper_gvec_udot_idx_b : gen_helper_gvec_sdot_idx_b; - opr_sz = (1 + a->q) * 8; - fpst = get_fpstatus_ptr(1); - tcg_gen_gvec_3_ool(vfp_reg_offset(1, a->vd), - vfp_reg_offset(1, a->vn), - vfp_reg_offset(1, a->rm), - opr_sz, opr_sz, a->index, fn_gvec); - tcg_temp_free_ptr(fpst); - return true; -} - -static bool trans_VFML_scalar(DisasContext *s, arg_VFML_scalar *a) -{ - int opr_sz; - - if (!dc_isar_feature(aa32_fhm, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd & 0x10) || (a->q && (a->vn & 0x10)))) { - return false; - } - - if (a->vd & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - opr_sz = (1 + a->q) * 8; - tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd), - vfp_reg_offset(a->q, a->vn), - vfp_reg_offset(a->q, a->rm), - cpu_env, opr_sz, opr_sz, - (a->index << 2) | a->s, /* is_2 == 0 */ - gen_helper_gvec_fmlal_idx_a32); - return true; -} - -static struct { - int nregs; - int interleave; - int spacing; -} const neon_ls_element_type[11] = { - {1, 4, 1}, - {1, 4, 2}, - {4, 1, 1}, - {2, 2, 2}, - {1, 3, 1}, - {1, 3, 2}, - {3, 1, 1}, - {1, 1, 1}, - {1, 2, 1}, - {1, 2, 2}, - {2, 1, 1} -}; - -static void gen_neon_ldst_base_update(DisasContext *s, int rm, int rn, - int stride) -{ - if (rm != 15) { - TCGv_i32 base; - - base = load_reg(s, rn); - if (rm == 13) { - tcg_gen_addi_i32(base, base, stride); - } else { - TCGv_i32 index; - index = load_reg(s, rm); - tcg_gen_add_i32(base, base, index); - tcg_temp_free_i32(index); - } - store_reg(s, rn, base); - } -} - -static bool trans_VLDST_multiple(DisasContext *s, arg_VLDST_multiple *a) -{ - /* Neon load/store multiple structures */ - int nregs, interleave, spacing, reg, n; - MemOp endian = s->be_data; - int mmu_idx = get_mem_index(s); - int size = a->size; - TCGv_i64 tmp64; - TCGv_i32 addr, tmp; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { - return false; - } - if (a->itype > 10) { - return false; - } - /* Catch UNDEF cases for bad values of align field */ - switch (a->itype & 0xc) { - case 4: - if (a->align >= 2) { - return false; - } - break; - case 8: - if (a->align == 3) { - return false; - } - break; - default: - break; - } - nregs = neon_ls_element_type[a->itype].nregs; - interleave = neon_ls_element_type[a->itype].interleave; - spacing = neon_ls_element_type[a->itype].spacing; - if (size == 3 && (interleave | spacing) != 1) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - /* For our purposes, bytes are always little-endian. */ - if (size == 0) { - endian = MO_LE; - } - /* - * Consecutive little-endian elements from a single register - * can be promoted to a larger little-endian operation. - */ - if (interleave == 1 && endian == MO_LE) { - size = 3; - } - tmp64 = tcg_temp_new_i64(); - addr = tcg_temp_new_i32(); - tmp = tcg_const_i32(1 << size); - load_reg_var(s, addr, a->rn); - for (reg = 0; reg < nregs; reg++) { - for (n = 0; n < 8 >> size; n++) { - int xs; - for (xs = 0; xs < interleave; xs++) { - int tt = a->vd + reg + spacing * xs; - - if (a->l) { - gen_aa32_ld_i64(s, tmp64, addr, mmu_idx, endian | size); - neon_store_element64(tt, n, size, tmp64); - } else { - neon_load_element64(tmp64, tt, n, size); - gen_aa32_st_i64(s, tmp64, addr, mmu_idx, endian | size); - } - tcg_gen_add_i32(addr, addr, tmp); - } - } - } - tcg_temp_free_i32(addr); - tcg_temp_free_i32(tmp); - tcg_temp_free_i64(tmp64); - - gen_neon_ldst_base_update(s, a->rm, a->rn, nregs * interleave * 8); - return true; -} - -static bool trans_VLD_all_lanes(DisasContext *s, arg_VLD_all_lanes *a) -{ - /* Neon load single structure to all lanes */ - int reg, stride, vec_size; - int vd = a->vd; - int size = a->size; - int nregs = a->n + 1; - TCGv_i32 addr, tmp; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { - return false; - } - - if (size == 3) { - if (nregs != 4 || a->a == 0) { - return false; - } - /* For VLD4 size == 3 a == 1 means 32 bits at 16 byte alignment */ - size = 2; - } - if (nregs == 1 && a->a == 1 && size == 0) { - return false; - } - if (nregs == 3 && a->a == 1) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - /* - * VLD1 to all lanes: T bit indicates how many Dregs to write. - * VLD2/3/4 to all lanes: T bit indicates register stride. - */ - stride = a->t ? 2 : 1; - vec_size = nregs == 1 ? stride * 8 : 8; - - tmp = tcg_temp_new_i32(); - addr = tcg_temp_new_i32(); - load_reg_var(s, addr, a->rn); - for (reg = 0; reg < nregs; reg++) { - gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), - s->be_data | size); - if ((vd & 1) && vec_size == 16) { - /* - * We cannot write 16 bytes at once because the - * destination is unaligned. - */ - tcg_gen_gvec_dup_i32(size, neon_reg_offset(vd, 0), - 8, 8, tmp); - tcg_gen_gvec_mov(0, neon_reg_offset(vd + 1, 0), - neon_reg_offset(vd, 0), 8, 8); - } else { - tcg_gen_gvec_dup_i32(size, neon_reg_offset(vd, 0), - vec_size, vec_size, tmp); - } - tcg_gen_addi_i32(addr, addr, 1 << size); - vd += stride; - } - tcg_temp_free_i32(tmp); - tcg_temp_free_i32(addr); - - gen_neon_ldst_base_update(s, a->rm, a->rn, (1 << size) * nregs); - - return true; -} - -static bool trans_VLDST_single(DisasContext *s, arg_VLDST_single *a) -{ - /* Neon load/store single structure to one lane */ - int reg; - int nregs = a->n + 1; - int vd = a->vd; - TCGv_i32 addr, tmp; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { - return false; - } - - /* Catch the UNDEF cases. This is unavoidably a bit messy. */ - switch (nregs) { - case 1: - if (((a->align & (1 << a->size)) != 0) || - (a->size == 2 && ((a->align & 3) == 1 || (a->align & 3) == 2))) { - return false; - } - break; - case 3: - if ((a->align & 1) != 0) { - return false; - } - /* fall through */ - case 2: - if (a->size == 2 && (a->align & 2) != 0) { - return false; - } - break; - case 4: - if ((a->size == 2) && ((a->align & 3) == 3)) { - return false; - } - break; - default: - abort(); - } - if ((vd + a->stride * (nregs - 1)) > 31) { - /* - * Attempts to write off the end of the register file are - * UNPREDICTABLE; we choose to UNDEF because otherwise we would - * access off the end of the array that holds the register data. - */ - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - tmp = tcg_temp_new_i32(); - addr = tcg_temp_new_i32(); - load_reg_var(s, addr, a->rn); - /* - * TODO: if we implemented alignment exceptions, we should check - * addr against the alignment encoded in a->align here. - */ - for (reg = 0; reg < nregs; reg++) { - if (a->l) { - gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), - s->be_data | a->size); - neon_store_element(vd, a->reg_idx, a->size, tmp); - } else { /* Store */ - neon_load_element(tmp, vd, a->reg_idx, a->size); - gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), - s->be_data | a->size); - } - vd += a->stride; - tcg_gen_addi_i32(addr, addr, 1 << a->size); - } - tcg_temp_free_i32(addr); - tcg_temp_free_i32(tmp); - - gen_neon_ldst_base_update(s, a->rm, a->rn, (1 << a->size) * nregs); - - return true; -} - -static bool do_3same(DisasContext *s, arg_3same *a, GVecGen3Fn fn) -{ - int vec_size = a->q ? 16 : 8; - int rd_ofs = neon_reg_offset(a->vd, 0); - int rn_ofs = neon_reg_offset(a->vn, 0); - int rm_ofs = neon_reg_offset(a->vm, 0); - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if ((a->vn | a->vm | a->vd) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fn(a->size, rd_ofs, rn_ofs, rm_ofs, vec_size, vec_size); - return true; -} - -#define DO_3SAME(INSN, FUNC) \ - static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ - { \ - return do_3same(s, a, FUNC); \ - } - -DO_3SAME(VADD, tcg_gen_gvec_add) -DO_3SAME(VSUB, tcg_gen_gvec_sub) -DO_3SAME(VAND, tcg_gen_gvec_and) -DO_3SAME(VBIC, tcg_gen_gvec_andc) -DO_3SAME(VORR, tcg_gen_gvec_or) -DO_3SAME(VORN, tcg_gen_gvec_orc) -DO_3SAME(VEOR, tcg_gen_gvec_xor) -DO_3SAME(VSHL_S, gen_gvec_sshl) -DO_3SAME(VSHL_U, gen_gvec_ushl) -DO_3SAME(VQADD_S, gen_gvec_sqadd_qc) -DO_3SAME(VQADD_U, gen_gvec_uqadd_qc) -DO_3SAME(VQSUB_S, gen_gvec_sqsub_qc) -DO_3SAME(VQSUB_U, gen_gvec_uqsub_qc) - -/* These insns are all gvec_bitsel but with the inputs in various orders. */ -#define DO_3SAME_BITSEL(INSN, O1, O2, O3) \ - static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ - uint32_t rn_ofs, uint32_t rm_ofs, \ - uint32_t oprsz, uint32_t maxsz) \ - { \ - tcg_gen_gvec_bitsel(vece, rd_ofs, O1, O2, O3, oprsz, maxsz); \ - } \ - DO_3SAME(INSN, gen_##INSN##_3s) - -DO_3SAME_BITSEL(VBSL, rd_ofs, rn_ofs, rm_ofs) -DO_3SAME_BITSEL(VBIT, rm_ofs, rn_ofs, rd_ofs) -DO_3SAME_BITSEL(VBIF, rm_ofs, rd_ofs, rn_ofs) - -#define DO_3SAME_NO_SZ_3(INSN, FUNC) \ - static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ - { \ - if (a->size == 3) { \ - return false; \ - } \ - return do_3same(s, a, FUNC); \ - } - -DO_3SAME_NO_SZ_3(VMAX_S, tcg_gen_gvec_smax) -DO_3SAME_NO_SZ_3(VMAX_U, tcg_gen_gvec_umax) -DO_3SAME_NO_SZ_3(VMIN_S, tcg_gen_gvec_smin) -DO_3SAME_NO_SZ_3(VMIN_U, tcg_gen_gvec_umin) -DO_3SAME_NO_SZ_3(VMUL, tcg_gen_gvec_mul) -DO_3SAME_NO_SZ_3(VMLA, gen_gvec_mla) -DO_3SAME_NO_SZ_3(VMLS, gen_gvec_mls) -DO_3SAME_NO_SZ_3(VTST, gen_gvec_cmtst) -DO_3SAME_NO_SZ_3(VABD_S, gen_gvec_sabd) -DO_3SAME_NO_SZ_3(VABA_S, gen_gvec_saba) -DO_3SAME_NO_SZ_3(VABD_U, gen_gvec_uabd) -DO_3SAME_NO_SZ_3(VABA_U, gen_gvec_uaba) - -#define DO_3SAME_CMP(INSN, COND) \ - static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ - uint32_t rn_ofs, uint32_t rm_ofs, \ - uint32_t oprsz, uint32_t maxsz) \ - { \ - tcg_gen_gvec_cmp(COND, vece, rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz); \ - } \ - DO_3SAME_NO_SZ_3(INSN, gen_##INSN##_3s) - -DO_3SAME_CMP(VCGT_S, TCG_COND_GT) -DO_3SAME_CMP(VCGT_U, TCG_COND_GTU) -DO_3SAME_CMP(VCGE_S, TCG_COND_GE) -DO_3SAME_CMP(VCGE_U, TCG_COND_GEU) -DO_3SAME_CMP(VCEQ, TCG_COND_EQ) - -#define WRAP_OOL_FN(WRAPNAME, FUNC) \ - static void WRAPNAME(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, \ - uint32_t rm_ofs, uint32_t oprsz, uint32_t maxsz) \ - { \ - tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, 0, FUNC); \ - } - -WRAP_OOL_FN(gen_VMUL_p_3s, gen_helper_gvec_pmul_b) - -static bool trans_VMUL_p_3s(DisasContext *s, arg_3same *a) -{ - if (a->size != 0) { - return false; - } - return do_3same(s, a, gen_VMUL_p_3s); -} - -#define DO_VQRDMLAH(INSN, FUNC) \ - static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ - { \ - if (!dc_isar_feature(aa32_rdm, s)) { \ - return false; \ - } \ - if (a->size != 1 && a->size != 2) { \ - return false; \ - } \ - return do_3same(s, a, FUNC); \ - } - -DO_VQRDMLAH(VQRDMLAH, gen_gvec_sqrdmlah_qc) -DO_VQRDMLAH(VQRDMLSH, gen_gvec_sqrdmlsh_qc) - -#define DO_SHA1(NAME, FUNC) \ - WRAP_OOL_FN(gen_##NAME##_3s, FUNC) \ - static bool trans_##NAME##_3s(DisasContext *s, arg_3same *a) \ - { \ - if (!dc_isar_feature(aa32_sha1, s)) { \ - return false; \ - } \ - return do_3same(s, a, gen_##NAME##_3s); \ - } - -DO_SHA1(SHA1C, gen_helper_crypto_sha1c) -DO_SHA1(SHA1P, gen_helper_crypto_sha1p) -DO_SHA1(SHA1M, gen_helper_crypto_sha1m) -DO_SHA1(SHA1SU0, gen_helper_crypto_sha1su0) - -#define DO_SHA2(NAME, FUNC) \ - WRAP_OOL_FN(gen_##NAME##_3s, FUNC) \ - static bool trans_##NAME##_3s(DisasContext *s, arg_3same *a) \ - { \ - if (!dc_isar_feature(aa32_sha2, s)) { \ - return false; \ - } \ - return do_3same(s, a, gen_##NAME##_3s); \ - } - -DO_SHA2(SHA256H, gen_helper_crypto_sha256h) -DO_SHA2(SHA256H2, gen_helper_crypto_sha256h2) -DO_SHA2(SHA256SU1, gen_helper_crypto_sha256su1) - -#define DO_3SAME_64(INSN, FUNC) \ - static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ - uint32_t rn_ofs, uint32_t rm_ofs, \ - uint32_t oprsz, uint32_t maxsz) \ - { \ - static const GVecGen3 op = { .fni8 = FUNC }; \ - tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &op); \ - } \ - DO_3SAME(INSN, gen_##INSN##_3s) - -#define DO_3SAME_64_ENV(INSN, FUNC) \ - static void gen_##INSN##_elt(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) \ - { \ - FUNC(d, cpu_env, n, m); \ - } \ - DO_3SAME_64(INSN, gen_##INSN##_elt) - -DO_3SAME_64(VRSHL_S64, gen_helper_neon_rshl_s64) -DO_3SAME_64(VRSHL_U64, gen_helper_neon_rshl_u64) -DO_3SAME_64_ENV(VQSHL_S64, gen_helper_neon_qshl_s64) -DO_3SAME_64_ENV(VQSHL_U64, gen_helper_neon_qshl_u64) -DO_3SAME_64_ENV(VQRSHL_S64, gen_helper_neon_qrshl_s64) -DO_3SAME_64_ENV(VQRSHL_U64, gen_helper_neon_qrshl_u64) - -#define DO_3SAME_32(INSN, FUNC) \ - static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ - uint32_t rn_ofs, uint32_t rm_ofs, \ - uint32_t oprsz, uint32_t maxsz) \ - { \ - static const GVecGen3 ops[4] = { \ - { .fni4 = gen_helper_neon_##FUNC##8 }, \ - { .fni4 = gen_helper_neon_##FUNC##16 }, \ - { .fni4 = gen_helper_neon_##FUNC##32 }, \ - { 0 }, \ - }; \ - tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &ops[vece]); \ - } \ - static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ - { \ - if (a->size > 2) { \ - return false; \ - } \ - return do_3same(s, a, gen_##INSN##_3s); \ - } - -/* - * Some helper functions need to be passed the cpu_env. In order - * to use those with the gvec APIs like tcg_gen_gvec_3() we need - * to create wrapper functions whose prototype is a NeonGenTwoOpFn() - * and which call a NeonGenTwoOpEnvFn(). - */ -#define WRAP_ENV_FN(WRAPNAME, FUNC) \ - static void WRAPNAME(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m) \ - { \ - FUNC(d, cpu_env, n, m); \ - } - -#define DO_3SAME_32_ENV(INSN, FUNC) \ - WRAP_ENV_FN(gen_##INSN##_tramp8, gen_helper_neon_##FUNC##8); \ - WRAP_ENV_FN(gen_##INSN##_tramp16, gen_helper_neon_##FUNC##16); \ - WRAP_ENV_FN(gen_##INSN##_tramp32, gen_helper_neon_##FUNC##32); \ - static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ - uint32_t rn_ofs, uint32_t rm_ofs, \ - uint32_t oprsz, uint32_t maxsz) \ - { \ - static const GVecGen3 ops[4] = { \ - { .fni4 = gen_##INSN##_tramp8 }, \ - { .fni4 = gen_##INSN##_tramp16 }, \ - { .fni4 = gen_##INSN##_tramp32 }, \ - { 0 }, \ - }; \ - tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &ops[vece]); \ - } \ - static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ - { \ - if (a->size > 2) { \ - return false; \ - } \ - return do_3same(s, a, gen_##INSN##_3s); \ - } - -DO_3SAME_32(VHADD_S, hadd_s) -DO_3SAME_32(VHADD_U, hadd_u) -DO_3SAME_32(VHSUB_S, hsub_s) -DO_3SAME_32(VHSUB_U, hsub_u) -DO_3SAME_32(VRHADD_S, rhadd_s) -DO_3SAME_32(VRHADD_U, rhadd_u) -DO_3SAME_32(VRSHL_S, rshl_s) -DO_3SAME_32(VRSHL_U, rshl_u) - -DO_3SAME_32_ENV(VQSHL_S, qshl_s) -DO_3SAME_32_ENV(VQSHL_U, qshl_u) -DO_3SAME_32_ENV(VQRSHL_S, qrshl_s) -DO_3SAME_32_ENV(VQRSHL_U, qrshl_u) - -static bool do_3same_pair(DisasContext *s, arg_3same *a, NeonGenTwoOpFn *fn) -{ - /* Operations handled pairwise 32 bits at a time */ - TCGv_i32 tmp, tmp2, tmp3; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (a->size == 3) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - assert(a->q == 0); /* enforced by decode patterns */ - - /* - * Note that we have to be careful not to clobber the source operands - * in the "vm == vd" case by storing the result of the first pass too - * early. Since Q is 0 there are always just two passes, so instead - * of a complicated loop over each pass we just unroll. - */ - tmp = neon_load_reg(a->vn, 0); - tmp2 = neon_load_reg(a->vn, 1); - fn(tmp, tmp, tmp2); - tcg_temp_free_i32(tmp2); - - tmp3 = neon_load_reg(a->vm, 0); - tmp2 = neon_load_reg(a->vm, 1); - fn(tmp3, tmp3, tmp2); - tcg_temp_free_i32(tmp2); - - neon_store_reg(a->vd, 0, tmp); - neon_store_reg(a->vd, 1, tmp3); - return true; -} - -#define DO_3SAME_PAIR(INSN, func) \ - static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ - { \ - static NeonGenTwoOpFn * const fns[] = { \ - gen_helper_neon_##func##8, \ - gen_helper_neon_##func##16, \ - gen_helper_neon_##func##32, \ - }; \ - if (a->size > 2) { \ - return false; \ - } \ - return do_3same_pair(s, a, fns[a->size]); \ - } - -/* 32-bit pairwise ops end up the same as the elementwise versions. */ -#define gen_helper_neon_pmax_s32 tcg_gen_smax_i32 -#define gen_helper_neon_pmax_u32 tcg_gen_umax_i32 -#define gen_helper_neon_pmin_s32 tcg_gen_smin_i32 -#define gen_helper_neon_pmin_u32 tcg_gen_umin_i32 -#define gen_helper_neon_padd_u32 tcg_gen_add_i32 - -DO_3SAME_PAIR(VPMAX_S, pmax_s) -DO_3SAME_PAIR(VPMIN_S, pmin_s) -DO_3SAME_PAIR(VPMAX_U, pmax_u) -DO_3SAME_PAIR(VPMIN_U, pmin_u) -DO_3SAME_PAIR(VPADD, padd_u) - -#define DO_3SAME_VQDMULH(INSN, FUNC) \ - WRAP_ENV_FN(gen_##INSN##_tramp16, gen_helper_neon_##FUNC##_s16); \ - WRAP_ENV_FN(gen_##INSN##_tramp32, gen_helper_neon_##FUNC##_s32); \ - static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ - uint32_t rn_ofs, uint32_t rm_ofs, \ - uint32_t oprsz, uint32_t maxsz) \ - { \ - static const GVecGen3 ops[2] = { \ - { .fni4 = gen_##INSN##_tramp16 }, \ - { .fni4 = gen_##INSN##_tramp32 }, \ - }; \ - tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &ops[vece - 1]); \ - } \ - static bool trans_##INSN##_3s(DisasContext *s, arg_3same *a) \ - { \ - if (a->size != 1 && a->size != 2) { \ - return false; \ - } \ - return do_3same(s, a, gen_##INSN##_3s); \ - } - -DO_3SAME_VQDMULH(VQDMULH, qdmulh) -DO_3SAME_VQDMULH(VQRDMULH, qrdmulh) - -static bool do_3same_fp(DisasContext *s, arg_3same *a, VFPGen3OpSPFn *fn, - bool reads_vd) -{ - /* - * FP operations handled elementwise 32 bits at a time. - * If reads_vd is true then the old value of Vd will be - * loaded before calling the callback function. This is - * used for multiply-accumulate type operations. - */ - TCGv_i32 tmp, tmp2; - int pass; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if ((a->vn | a->vm | a->vd) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - TCGv_ptr fpstatus = get_fpstatus_ptr(1); - for (pass = 0; pass < (a->q ? 4 : 2); pass++) { - tmp = neon_load_reg(a->vn, pass); - tmp2 = neon_load_reg(a->vm, pass); - if (reads_vd) { - TCGv_i32 tmp_rd = neon_load_reg(a->vd, pass); - fn(tmp_rd, tmp, tmp2, fpstatus); - neon_store_reg(a->vd, pass, tmp_rd); - tcg_temp_free_i32(tmp); - } else { - fn(tmp, tmp, tmp2, fpstatus); - neon_store_reg(a->vd, pass, tmp); - } - tcg_temp_free_i32(tmp2); - } - tcg_temp_free_ptr(fpstatus); - return true; -} - -/* - * For all the functions using this macro, size == 1 means fp16, - * which is an architecture extension we don't implement yet. - */ -#define DO_3S_FP_GVEC(INSN,FUNC) \ - static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \ - uint32_t rn_ofs, uint32_t rm_ofs, \ - uint32_t oprsz, uint32_t maxsz) \ - { \ - TCGv_ptr fpst = get_fpstatus_ptr(1); \ - tcg_gen_gvec_3_ptr(rd_ofs, rn_ofs, rm_ofs, fpst, \ - oprsz, maxsz, 0, FUNC); \ - tcg_temp_free_ptr(fpst); \ - } \ - static bool trans_##INSN##_fp_3s(DisasContext *s, arg_3same *a) \ - { \ - if (a->size != 0) { \ - /* TODO fp16 support */ \ - return false; \ - } \ - return do_3same(s, a, gen_##INSN##_3s); \ - } - - -DO_3S_FP_GVEC(VADD, gen_helper_gvec_fadd_s) -DO_3S_FP_GVEC(VSUB, gen_helper_gvec_fsub_s) -DO_3S_FP_GVEC(VABD, gen_helper_gvec_fabd_s) -DO_3S_FP_GVEC(VMUL, gen_helper_gvec_fmul_s) - -/* - * For all the functions using this macro, size == 1 means fp16, - * which is an architecture extension we don't implement yet. - */ -#define DO_3S_FP(INSN,FUNC,READS_VD) \ - static bool trans_##INSN##_fp_3s(DisasContext *s, arg_3same *a) \ - { \ - if (a->size != 0) { \ - /* TODO fp16 support */ \ - return false; \ - } \ - return do_3same_fp(s, a, FUNC, READS_VD); \ - } - -DO_3S_FP(VCEQ, gen_helper_neon_ceq_f32, false) -DO_3S_FP(VCGE, gen_helper_neon_cge_f32, false) -DO_3S_FP(VCGT, gen_helper_neon_cgt_f32, false) -DO_3S_FP(VACGE, gen_helper_neon_acge_f32, false) -DO_3S_FP(VACGT, gen_helper_neon_acgt_f32, false) -DO_3S_FP(VMAX, gen_helper_vfp_maxs, false) -DO_3S_FP(VMIN, gen_helper_vfp_mins, false) - -static void gen_VMLA_fp_3s(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, - TCGv_ptr fpstatus) -{ - gen_helper_vfp_muls(vn, vn, vm, fpstatus); - gen_helper_vfp_adds(vd, vd, vn, fpstatus); -} - -static void gen_VMLS_fp_3s(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, - TCGv_ptr fpstatus) -{ - gen_helper_vfp_muls(vn, vn, vm, fpstatus); - gen_helper_vfp_subs(vd, vd, vn, fpstatus); -} - -DO_3S_FP(VMLA, gen_VMLA_fp_3s, true) -DO_3S_FP(VMLS, gen_VMLS_fp_3s, true) - -static bool trans_VMAXNM_fp_3s(DisasContext *s, arg_3same *a) -{ - if (!arm_dc_feature(s, ARM_FEATURE_V8)) { - return false; - } - - if (a->size != 0) { - /* TODO fp16 support */ - return false; - } - - return do_3same_fp(s, a, gen_helper_vfp_maxnums, false); -} - -static bool trans_VMINNM_fp_3s(DisasContext *s, arg_3same *a) -{ - if (!arm_dc_feature(s, ARM_FEATURE_V8)) { - return false; - } - - if (a->size != 0) { - /* TODO fp16 support */ - return false; - } - - return do_3same_fp(s, a, gen_helper_vfp_minnums, false); -} - -WRAP_ENV_FN(gen_VRECPS_tramp, gen_helper_recps_f32) - -static void gen_VRECPS_fp_3s(unsigned vece, uint32_t rd_ofs, - uint32_t rn_ofs, uint32_t rm_ofs, - uint32_t oprsz, uint32_t maxsz) -{ - static const GVecGen3 ops = { .fni4 = gen_VRECPS_tramp }; - tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &ops); -} - -static bool trans_VRECPS_fp_3s(DisasContext *s, arg_3same *a) -{ - if (a->size != 0) { - /* TODO fp16 support */ - return false; - } - - return do_3same(s, a, gen_VRECPS_fp_3s); -} - -WRAP_ENV_FN(gen_VRSQRTS_tramp, gen_helper_rsqrts_f32) - -static void gen_VRSQRTS_fp_3s(unsigned vece, uint32_t rd_ofs, - uint32_t rn_ofs, uint32_t rm_ofs, - uint32_t oprsz, uint32_t maxsz) -{ - static const GVecGen3 ops = { .fni4 = gen_VRSQRTS_tramp }; - tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, &ops); -} - -static bool trans_VRSQRTS_fp_3s(DisasContext *s, arg_3same *a) -{ - if (a->size != 0) { - /* TODO fp16 support */ - return false; - } - - return do_3same(s, a, gen_VRSQRTS_fp_3s); -} - -static void gen_VFMA_fp_3s(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, - TCGv_ptr fpstatus) -{ - gen_helper_vfp_muladds(vd, vn, vm, vd, fpstatus); -} - -static bool trans_VFMA_fp_3s(DisasContext *s, arg_3same *a) -{ - if (!dc_isar_feature(aa32_simdfmac, s)) { - return false; - } - - if (a->size != 0) { - /* TODO fp16 support */ - return false; - } - - return do_3same_fp(s, a, gen_VFMA_fp_3s, true); -} - -static void gen_VFMS_fp_3s(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, - TCGv_ptr fpstatus) -{ - gen_helper_vfp_negs(vn, vn); - gen_helper_vfp_muladds(vd, vn, vm, vd, fpstatus); -} - -static bool trans_VFMS_fp_3s(DisasContext *s, arg_3same *a) -{ - if (!dc_isar_feature(aa32_simdfmac, s)) { - return false; - } - - if (a->size != 0) { - /* TODO fp16 support */ - return false; - } - - return do_3same_fp(s, a, gen_VFMS_fp_3s, true); -} - -static bool do_3same_fp_pair(DisasContext *s, arg_3same *a, VFPGen3OpSPFn *fn) -{ - /* FP operations handled pairwise 32 bits at a time */ - TCGv_i32 tmp, tmp2, tmp3; - TCGv_ptr fpstatus; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - assert(a->q == 0); /* enforced by decode patterns */ - - /* - * Note that we have to be careful not to clobber the source operands - * in the "vm == vd" case by storing the result of the first pass too - * early. Since Q is 0 there are always just two passes, so instead - * of a complicated loop over each pass we just unroll. - */ - fpstatus = get_fpstatus_ptr(1); - tmp = neon_load_reg(a->vn, 0); - tmp2 = neon_load_reg(a->vn, 1); - fn(tmp, tmp, tmp2, fpstatus); - tcg_temp_free_i32(tmp2); - - tmp3 = neon_load_reg(a->vm, 0); - tmp2 = neon_load_reg(a->vm, 1); - fn(tmp3, tmp3, tmp2, fpstatus); - tcg_temp_free_i32(tmp2); - tcg_temp_free_ptr(fpstatus); - - neon_store_reg(a->vd, 0, tmp); - neon_store_reg(a->vd, 1, tmp3); - return true; -} - -/* - * For all the functions using this macro, size == 1 means fp16, - * which is an architecture extension we don't implement yet. - */ -#define DO_3S_FP_PAIR(INSN,FUNC) \ - static bool trans_##INSN##_fp_3s(DisasContext *s, arg_3same *a) \ - { \ - if (a->size != 0) { \ - /* TODO fp16 support */ \ - return false; \ - } \ - return do_3same_fp_pair(s, a, FUNC); \ - } - -DO_3S_FP_PAIR(VPADD, gen_helper_vfp_adds) -DO_3S_FP_PAIR(VPMAX, gen_helper_vfp_maxs) -DO_3S_FP_PAIR(VPMIN, gen_helper_vfp_mins) - -static bool do_vector_2sh(DisasContext *s, arg_2reg_shift *a, GVecGen2iFn *fn) -{ - /* Handle a 2-reg-shift insn which can be vectorized. */ - int vec_size = a->q ? 16 : 8; - int rd_ofs = neon_reg_offset(a->vd, 0); - int rm_ofs = neon_reg_offset(a->vm, 0); - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if ((a->vm | a->vd) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fn(a->size, rd_ofs, rm_ofs, a->shift, vec_size, vec_size); - return true; -} - -#define DO_2SH(INSN, FUNC) \ - static bool trans_##INSN##_2sh(DisasContext *s, arg_2reg_shift *a) \ - { \ - return do_vector_2sh(s, a, FUNC); \ - } \ - -DO_2SH(VSHL, tcg_gen_gvec_shli) -DO_2SH(VSLI, gen_gvec_sli) -DO_2SH(VSRI, gen_gvec_sri) -DO_2SH(VSRA_S, gen_gvec_ssra) -DO_2SH(VSRA_U, gen_gvec_usra) -DO_2SH(VRSHR_S, gen_gvec_srshr) -DO_2SH(VRSHR_U, gen_gvec_urshr) -DO_2SH(VRSRA_S, gen_gvec_srsra) -DO_2SH(VRSRA_U, gen_gvec_ursra) - -static bool trans_VSHR_S_2sh(DisasContext *s, arg_2reg_shift *a) -{ - /* Signed shift out of range results in all-sign-bits */ - a->shift = MIN(a->shift, (8 << a->size) - 1); - return do_vector_2sh(s, a, tcg_gen_gvec_sari); -} - -static void gen_zero_rd_2sh(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs, - int64_t shift, uint32_t oprsz, uint32_t maxsz) -{ - tcg_gen_gvec_dup_imm(vece, rd_ofs, oprsz, maxsz, 0); -} - -static bool trans_VSHR_U_2sh(DisasContext *s, arg_2reg_shift *a) -{ - /* Shift out of range is architecturally valid and results in zero. */ - if (a->shift >= (8 << a->size)) { - return do_vector_2sh(s, a, gen_zero_rd_2sh); - } else { - return do_vector_2sh(s, a, tcg_gen_gvec_shri); - } -} - -static bool do_2shift_env_64(DisasContext *s, arg_2reg_shift *a, - NeonGenTwo64OpEnvFn *fn) -{ - /* - * 2-reg-and-shift operations, size == 3 case, where the - * function needs to be passed cpu_env. - */ - TCGv_i64 constimm; - int pass; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if ((a->vm | a->vd) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - /* - * To avoid excessive duplication of ops we implement shift - * by immediate using the variable shift operations. - */ - constimm = tcg_const_i64(dup_const(a->size, a->shift)); - - for (pass = 0; pass < a->q + 1; pass++) { - TCGv_i64 tmp = tcg_temp_new_i64(); - - neon_load_reg64(tmp, a->vm + pass); - fn(tmp, cpu_env, tmp, constimm); - neon_store_reg64(tmp, a->vd + pass); - tcg_temp_free_i64(tmp); - } - tcg_temp_free_i64(constimm); - return true; -} - -static bool do_2shift_env_32(DisasContext *s, arg_2reg_shift *a, - NeonGenTwoOpEnvFn *fn) -{ - /* - * 2-reg-and-shift operations, size < 3 case, where the - * helper needs to be passed cpu_env. - */ - TCGv_i32 constimm; - int pass; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if ((a->vm | a->vd) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - /* - * To avoid excessive duplication of ops we implement shift - * by immediate using the variable shift operations. - */ - constimm = tcg_const_i32(dup_const(a->size, a->shift)); - - for (pass = 0; pass < (a->q ? 4 : 2); pass++) { - TCGv_i32 tmp = neon_load_reg(a->vm, pass); - fn(tmp, cpu_env, tmp, constimm); - neon_store_reg(a->vd, pass, tmp); - } - tcg_temp_free_i32(constimm); - return true; -} - -#define DO_2SHIFT_ENV(INSN, FUNC) \ - static bool trans_##INSN##_64_2sh(DisasContext *s, arg_2reg_shift *a) \ - { \ - return do_2shift_env_64(s, a, gen_helper_neon_##FUNC##64); \ - } \ - static bool trans_##INSN##_2sh(DisasContext *s, arg_2reg_shift *a) \ - { \ - static NeonGenTwoOpEnvFn * const fns[] = { \ - gen_helper_neon_##FUNC##8, \ - gen_helper_neon_##FUNC##16, \ - gen_helper_neon_##FUNC##32, \ - }; \ - assert(a->size < ARRAY_SIZE(fns)); \ - return do_2shift_env_32(s, a, fns[a->size]); \ - } - -DO_2SHIFT_ENV(VQSHLU, qshlu_s) -DO_2SHIFT_ENV(VQSHL_U, qshl_u) -DO_2SHIFT_ENV(VQSHL_S, qshl_s) - -static bool do_2shift_narrow_64(DisasContext *s, arg_2reg_shift *a, - NeonGenTwo64OpFn *shiftfn, - NeonGenNarrowEnvFn *narrowfn) -{ - /* 2-reg-and-shift narrowing-shift operations, size == 3 case */ - TCGv_i64 constimm, rm1, rm2; - TCGv_i32 rd; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->vm & 1) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - /* - * This is always a right shift, and the shiftfn is always a - * left-shift helper, which thus needs the negated shift count. - */ - constimm = tcg_const_i64(-a->shift); - rm1 = tcg_temp_new_i64(); - rm2 = tcg_temp_new_i64(); - - /* Load both inputs first to avoid potential overwrite if rm == rd */ - neon_load_reg64(rm1, a->vm); - neon_load_reg64(rm2, a->vm + 1); - - shiftfn(rm1, rm1, constimm); - rd = tcg_temp_new_i32(); - narrowfn(rd, cpu_env, rm1); - neon_store_reg(a->vd, 0, rd); - - shiftfn(rm2, rm2, constimm); - rd = tcg_temp_new_i32(); - narrowfn(rd, cpu_env, rm2); - neon_store_reg(a->vd, 1, rd); - - tcg_temp_free_i64(rm1); - tcg_temp_free_i64(rm2); - tcg_temp_free_i64(constimm); - - return true; -} - -static bool do_2shift_narrow_32(DisasContext *s, arg_2reg_shift *a, - NeonGenTwoOpFn *shiftfn, - NeonGenNarrowEnvFn *narrowfn) -{ - /* 2-reg-and-shift narrowing-shift operations, size < 3 case */ - TCGv_i32 constimm, rm1, rm2, rm3, rm4; - TCGv_i64 rtmp; - uint32_t imm; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->vm & 1) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - /* - * This is always a right shift, and the shiftfn is always a - * left-shift helper, which thus needs the negated shift count - * duplicated into each lane of the immediate value. - */ - if (a->size == 1) { - imm = (uint16_t)(-a->shift); - imm |= imm << 16; - } else { - /* size == 2 */ - imm = -a->shift; - } - constimm = tcg_const_i32(imm); - - /* Load all inputs first to avoid potential overwrite */ - rm1 = neon_load_reg(a->vm, 0); - rm2 = neon_load_reg(a->vm, 1); - rm3 = neon_load_reg(a->vm + 1, 0); - rm4 = neon_load_reg(a->vm + 1, 1); - rtmp = tcg_temp_new_i64(); - - shiftfn(rm1, rm1, constimm); - shiftfn(rm2, rm2, constimm); - - tcg_gen_concat_i32_i64(rtmp, rm1, rm2); - tcg_temp_free_i32(rm2); - - narrowfn(rm1, cpu_env, rtmp); - neon_store_reg(a->vd, 0, rm1); - - shiftfn(rm3, rm3, constimm); - shiftfn(rm4, rm4, constimm); - tcg_temp_free_i32(constimm); - - tcg_gen_concat_i32_i64(rtmp, rm3, rm4); - tcg_temp_free_i32(rm4); - - narrowfn(rm3, cpu_env, rtmp); - tcg_temp_free_i64(rtmp); - neon_store_reg(a->vd, 1, rm3); - return true; -} - -#define DO_2SN_64(INSN, FUNC, NARROWFUNC) \ - static bool trans_##INSN##_2sh(DisasContext *s, arg_2reg_shift *a) \ - { \ - return do_2shift_narrow_64(s, a, FUNC, NARROWFUNC); \ - } -#define DO_2SN_32(INSN, FUNC, NARROWFUNC) \ - static bool trans_##INSN##_2sh(DisasContext *s, arg_2reg_shift *a) \ - { \ - return do_2shift_narrow_32(s, a, FUNC, NARROWFUNC); \ - } - -static void gen_neon_narrow_u32(TCGv_i32 dest, TCGv_ptr env, TCGv_i64 src) -{ - tcg_gen_extrl_i64_i32(dest, src); -} - -static void gen_neon_narrow_u16(TCGv_i32 dest, TCGv_ptr env, TCGv_i64 src) -{ - gen_helper_neon_narrow_u16(dest, src); -} - -static void gen_neon_narrow_u8(TCGv_i32 dest, TCGv_ptr env, TCGv_i64 src) -{ - gen_helper_neon_narrow_u8(dest, src); -} - -DO_2SN_64(VSHRN_64, gen_ushl_i64, gen_neon_narrow_u32) -DO_2SN_32(VSHRN_32, gen_ushl_i32, gen_neon_narrow_u16) -DO_2SN_32(VSHRN_16, gen_helper_neon_shl_u16, gen_neon_narrow_u8) - -DO_2SN_64(VRSHRN_64, gen_helper_neon_rshl_u64, gen_neon_narrow_u32) -DO_2SN_32(VRSHRN_32, gen_helper_neon_rshl_u32, gen_neon_narrow_u16) -DO_2SN_32(VRSHRN_16, gen_helper_neon_rshl_u16, gen_neon_narrow_u8) - -DO_2SN_64(VQSHRUN_64, gen_sshl_i64, gen_helper_neon_unarrow_sat32) -DO_2SN_32(VQSHRUN_32, gen_sshl_i32, gen_helper_neon_unarrow_sat16) -DO_2SN_32(VQSHRUN_16, gen_helper_neon_shl_s16, gen_helper_neon_unarrow_sat8) - -DO_2SN_64(VQRSHRUN_64, gen_helper_neon_rshl_s64, gen_helper_neon_unarrow_sat32) -DO_2SN_32(VQRSHRUN_32, gen_helper_neon_rshl_s32, gen_helper_neon_unarrow_sat16) -DO_2SN_32(VQRSHRUN_16, gen_helper_neon_rshl_s16, gen_helper_neon_unarrow_sat8) -DO_2SN_64(VQSHRN_S64, gen_sshl_i64, gen_helper_neon_narrow_sat_s32) -DO_2SN_32(VQSHRN_S32, gen_sshl_i32, gen_helper_neon_narrow_sat_s16) -DO_2SN_32(VQSHRN_S16, gen_helper_neon_shl_s16, gen_helper_neon_narrow_sat_s8) - -DO_2SN_64(VQRSHRN_S64, gen_helper_neon_rshl_s64, gen_helper_neon_narrow_sat_s32) -DO_2SN_32(VQRSHRN_S32, gen_helper_neon_rshl_s32, gen_helper_neon_narrow_sat_s16) -DO_2SN_32(VQRSHRN_S16, gen_helper_neon_rshl_s16, gen_helper_neon_narrow_sat_s8) - -DO_2SN_64(VQSHRN_U64, gen_ushl_i64, gen_helper_neon_narrow_sat_u32) -DO_2SN_32(VQSHRN_U32, gen_ushl_i32, gen_helper_neon_narrow_sat_u16) -DO_2SN_32(VQSHRN_U16, gen_helper_neon_shl_u16, gen_helper_neon_narrow_sat_u8) - -DO_2SN_64(VQRSHRN_U64, gen_helper_neon_rshl_u64, gen_helper_neon_narrow_sat_u32) -DO_2SN_32(VQRSHRN_U32, gen_helper_neon_rshl_u32, gen_helper_neon_narrow_sat_u16) -DO_2SN_32(VQRSHRN_U16, gen_helper_neon_rshl_u16, gen_helper_neon_narrow_sat_u8) - -static bool do_vshll_2sh(DisasContext *s, arg_2reg_shift *a, - NeonGenWidenFn *widenfn, bool u) -{ - TCGv_i64 tmp; - TCGv_i32 rm0, rm1; - uint64_t widen_mask = 0; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->vd & 1) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - /* - * This is a widen-and-shift operation. The shift is always less - * than the width of the source type, so after widening the input - * vector we can simply shift the whole 64-bit widened register, - * and then clear the potential overflow bits resulting from left - * bits of the narrow input appearing as right bits of the left - * neighbour narrow input. Calculate a mask of bits to clear. - */ - if ((a->shift != 0) && (a->size < 2 || u)) { - int esize = 8 << a->size; - widen_mask = MAKE_64BIT_MASK(0, esize); - widen_mask >>= esize - a->shift; - widen_mask = dup_const(a->size + 1, widen_mask); - } - - rm0 = neon_load_reg(a->vm, 0); - rm1 = neon_load_reg(a->vm, 1); - tmp = tcg_temp_new_i64(); - - widenfn(tmp, rm0); - tcg_temp_free_i32(rm0); - if (a->shift != 0) { - tcg_gen_shli_i64(tmp, tmp, a->shift); - tcg_gen_andi_i64(tmp, tmp, ~widen_mask); - } - neon_store_reg64(tmp, a->vd); - - widenfn(tmp, rm1); - tcg_temp_free_i32(rm1); - if (a->shift != 0) { - tcg_gen_shli_i64(tmp, tmp, a->shift); - tcg_gen_andi_i64(tmp, tmp, ~widen_mask); - } - neon_store_reg64(tmp, a->vd + 1); - tcg_temp_free_i64(tmp); - return true; -} - -static bool trans_VSHLL_S_2sh(DisasContext *s, arg_2reg_shift *a) -{ - static NeonGenWidenFn * const widenfn[] = { - gen_helper_neon_widen_s8, - gen_helper_neon_widen_s16, - tcg_gen_ext_i32_i64, - }; - return do_vshll_2sh(s, a, widenfn[a->size], false); -} - -static bool trans_VSHLL_U_2sh(DisasContext *s, arg_2reg_shift *a) -{ - static NeonGenWidenFn * const widenfn[] = { - gen_helper_neon_widen_u8, - gen_helper_neon_widen_u16, - tcg_gen_extu_i32_i64, - }; - return do_vshll_2sh(s, a, widenfn[a->size], true); -} - -static bool do_fp_2sh(DisasContext *s, arg_2reg_shift *a, - NeonGenTwoSingleOpFn *fn) -{ - /* FP operations in 2-reg-and-shift group */ - TCGv_i32 tmp, shiftv; - TCGv_ptr fpstatus; - int pass; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if ((a->vm | a->vd) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpstatus = get_fpstatus_ptr(1); - shiftv = tcg_const_i32(a->shift); - for (pass = 0; pass < (a->q ? 4 : 2); pass++) { - tmp = neon_load_reg(a->vm, pass); - fn(tmp, tmp, shiftv, fpstatus); - neon_store_reg(a->vd, pass, tmp); - } - tcg_temp_free_ptr(fpstatus); - tcg_temp_free_i32(shiftv); - return true; -} - -#define DO_FP_2SH(INSN, FUNC) \ - static bool trans_##INSN##_2sh(DisasContext *s, arg_2reg_shift *a) \ - { \ - return do_fp_2sh(s, a, FUNC); \ - } - -DO_FP_2SH(VCVT_SF, gen_helper_vfp_sltos) -DO_FP_2SH(VCVT_UF, gen_helper_vfp_ultos) -DO_FP_2SH(VCVT_FS, gen_helper_vfp_tosls_round_to_zero) -DO_FP_2SH(VCVT_FU, gen_helper_vfp_touls_round_to_zero) - -static uint64_t asimd_imm_const(uint32_t imm, int cmode, int op) -{ - /* - * Expand the encoded constant. - * Note that cmode = 2,3,4,5,6,7,10,11,12,13 imm=0 is UNPREDICTABLE. - * We choose to not special-case this and will behave as if a - * valid constant encoding of 0 had been given. - * cmode = 15 op = 1 must UNDEF; we assume decode has handled that. - */ - switch (cmode) { - case 0: case 1: - /* no-op */ - break; - case 2: case 3: - imm <<= 8; - break; - case 4: case 5: - imm <<= 16; - break; - case 6: case 7: - imm <<= 24; - break; - case 8: case 9: - imm |= imm << 16; - break; - case 10: case 11: - imm = (imm << 8) | (imm << 24); - break; - case 12: - imm = (imm << 8) | 0xff; - break; - case 13: - imm = (imm << 16) | 0xffff; - break; - case 14: - if (op) { - /* - * This is the only case where the top and bottom 32 bits - * of the encoded constant differ. - */ - uint64_t imm64 = 0; - int n; - - for (n = 0; n < 8; n++) { - if (imm & (1 << n)) { - imm64 |= (0xffULL << (n * 8)); - } - } - return imm64; - } - imm |= (imm << 8) | (imm << 16) | (imm << 24); - break; - case 15: - imm = ((imm & 0x80) << 24) | ((imm & 0x3f) << 19) - | ((imm & 0x40) ? (0x1f << 25) : (1 << 30)); - break; - } - if (op) { - imm = ~imm; - } - return dup_const(MO_32, imm); -} - -static bool do_1reg_imm(DisasContext *s, arg_1reg_imm *a, - GVecGen2iFn *fn) -{ - uint64_t imm; - int reg_ofs, vec_size; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { - return false; - } - - if (a->vd & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - reg_ofs = neon_reg_offset(a->vd, 0); - vec_size = a->q ? 16 : 8; - imm = asimd_imm_const(a->imm, a->cmode, a->op); - - fn(MO_64, reg_ofs, reg_ofs, imm, vec_size, vec_size); - return true; -} - -static void gen_VMOV_1r(unsigned vece, uint32_t dofs, uint32_t aofs, - int64_t c, uint32_t oprsz, uint32_t maxsz) -{ - tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, c); -} - -static bool trans_Vimm_1r(DisasContext *s, arg_1reg_imm *a) -{ - /* Handle decode of cmode/op here between VORR/VBIC/VMOV */ - GVecGen2iFn *fn; - - if ((a->cmode & 1) && a->cmode < 12) { - /* for op=1, the imm will be inverted, so BIC becomes AND. */ - fn = a->op ? tcg_gen_gvec_andi : tcg_gen_gvec_ori; - } else { - /* There is one unallocated cmode/op combination in this space */ - if (a->cmode == 15 && a->op == 1) { - return false; - } - fn = gen_VMOV_1r; - } - return do_1reg_imm(s, a, fn); -} - -static bool do_prewiden_3d(DisasContext *s, arg_3diff *a, - NeonGenWidenFn *widenfn, - NeonGenTwo64OpFn *opfn, - bool src1_wide) -{ - /* 3-regs different lengths, prewidening case (VADDL/VSUBL/VAADW/VSUBW) */ - TCGv_i64 rn0_64, rn1_64, rm_64; - TCGv_i32 rm; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (!widenfn || !opfn) { - /* size == 3 case, which is an entirely different insn group */ - return false; - } - - if ((a->vd & 1) || (src1_wide && (a->vn & 1))) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - rn0_64 = tcg_temp_new_i64(); - rn1_64 = tcg_temp_new_i64(); - rm_64 = tcg_temp_new_i64(); - - if (src1_wide) { - neon_load_reg64(rn0_64, a->vn); - } else { - TCGv_i32 tmp = neon_load_reg(a->vn, 0); - widenfn(rn0_64, tmp); - tcg_temp_free_i32(tmp); - } - rm = neon_load_reg(a->vm, 0); - - widenfn(rm_64, rm); - tcg_temp_free_i32(rm); - opfn(rn0_64, rn0_64, rm_64); - - /* - * Load second pass inputs before storing the first pass result, to - * avoid incorrect results if a narrow input overlaps with the result. - */ - if (src1_wide) { - neon_load_reg64(rn1_64, a->vn + 1); - } else { - TCGv_i32 tmp = neon_load_reg(a->vn, 1); - widenfn(rn1_64, tmp); - tcg_temp_free_i32(tmp); - } - rm = neon_load_reg(a->vm, 1); - - neon_store_reg64(rn0_64, a->vd); - - widenfn(rm_64, rm); - tcg_temp_free_i32(rm); - opfn(rn1_64, rn1_64, rm_64); - neon_store_reg64(rn1_64, a->vd + 1); - - tcg_temp_free_i64(rn0_64); - tcg_temp_free_i64(rn1_64); - tcg_temp_free_i64(rm_64); - - return true; -} - -#define DO_PREWIDEN(INSN, S, EXT, OP, SRC1WIDE) \ - static bool trans_##INSN##_3d(DisasContext *s, arg_3diff *a) \ - { \ - static NeonGenWidenFn * const widenfn[] = { \ - gen_helper_neon_widen_##S##8, \ - gen_helper_neon_widen_##S##16, \ - tcg_gen_##EXT##_i32_i64, \ - NULL, \ - }; \ - static NeonGenTwo64OpFn * const addfn[] = { \ - gen_helper_neon_##OP##l_u16, \ - gen_helper_neon_##OP##l_u32, \ - tcg_gen_##OP##_i64, \ - NULL, \ - }; \ - return do_prewiden_3d(s, a, widenfn[a->size], \ - addfn[a->size], SRC1WIDE); \ - } - -DO_PREWIDEN(VADDL_S, s, ext, add, false) -DO_PREWIDEN(VADDL_U, u, extu, add, false) -DO_PREWIDEN(VSUBL_S, s, ext, sub, false) -DO_PREWIDEN(VSUBL_U, u, extu, sub, false) -DO_PREWIDEN(VADDW_S, s, ext, add, true) -DO_PREWIDEN(VADDW_U, u, extu, add, true) -DO_PREWIDEN(VSUBW_S, s, ext, sub, true) -DO_PREWIDEN(VSUBW_U, u, extu, sub, true) - -static bool do_narrow_3d(DisasContext *s, arg_3diff *a, - NeonGenTwo64OpFn *opfn, NeonGenNarrowFn *narrowfn) -{ - /* 3-regs different lengths, narrowing (VADDHN/VSUBHN/VRADDHN/VRSUBHN) */ - TCGv_i64 rn_64, rm_64; - TCGv_i32 rd0, rd1; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (!opfn || !narrowfn) { - /* size == 3 case, which is an entirely different insn group */ - return false; - } - - if ((a->vn | a->vm) & 1) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - rn_64 = tcg_temp_new_i64(); - rm_64 = tcg_temp_new_i64(); - rd0 = tcg_temp_new_i32(); - rd1 = tcg_temp_new_i32(); - - neon_load_reg64(rn_64, a->vn); - neon_load_reg64(rm_64, a->vm); - - opfn(rn_64, rn_64, rm_64); - - narrowfn(rd0, rn_64); - - neon_load_reg64(rn_64, a->vn + 1); - neon_load_reg64(rm_64, a->vm + 1); - - opfn(rn_64, rn_64, rm_64); - - narrowfn(rd1, rn_64); - - neon_store_reg(a->vd, 0, rd0); - neon_store_reg(a->vd, 1, rd1); - - tcg_temp_free_i64(rn_64); - tcg_temp_free_i64(rm_64); - - return true; -} - -#define DO_NARROW_3D(INSN, OP, NARROWTYPE, EXTOP) \ - static bool trans_##INSN##_3d(DisasContext *s, arg_3diff *a) \ - { \ - static NeonGenTwo64OpFn * const addfn[] = { \ - gen_helper_neon_##OP##l_u16, \ - gen_helper_neon_##OP##l_u32, \ - tcg_gen_##OP##_i64, \ - NULL, \ - }; \ - static NeonGenNarrowFn * const narrowfn[] = { \ - gen_helper_neon_##NARROWTYPE##_high_u8, \ - gen_helper_neon_##NARROWTYPE##_high_u16, \ - EXTOP, \ - NULL, \ - }; \ - return do_narrow_3d(s, a, addfn[a->size], narrowfn[a->size]); \ - } - -static void gen_narrow_round_high_u32(TCGv_i32 rd, TCGv_i64 rn) -{ - tcg_gen_addi_i64(rn, rn, 1u << 31); - tcg_gen_extrh_i64_i32(rd, rn); -} - -DO_NARROW_3D(VADDHN, add, narrow, tcg_gen_extrh_i64_i32) -DO_NARROW_3D(VSUBHN, sub, narrow, tcg_gen_extrh_i64_i32) -DO_NARROW_3D(VRADDHN, add, narrow_round, gen_narrow_round_high_u32) -DO_NARROW_3D(VRSUBHN, sub, narrow_round, gen_narrow_round_high_u32) - -static bool do_long_3d(DisasContext *s, arg_3diff *a, - NeonGenTwoOpWidenFn *opfn, - NeonGenTwo64OpFn *accfn) -{ - /* - * 3-regs different lengths, long operations. - * These perform an operation on two inputs that returns a double-width - * result, and then possibly perform an accumulation operation of - * that result into the double-width destination. - */ - TCGv_i64 rd0, rd1, tmp; - TCGv_i32 rn, rm; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (!opfn) { - /* size == 3 case, which is an entirely different insn group */ - return false; - } - - if (a->vd & 1) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - rd0 = tcg_temp_new_i64(); - rd1 = tcg_temp_new_i64(); - - rn = neon_load_reg(a->vn, 0); - rm = neon_load_reg(a->vm, 0); - opfn(rd0, rn, rm); - tcg_temp_free_i32(rn); - tcg_temp_free_i32(rm); - - rn = neon_load_reg(a->vn, 1); - rm = neon_load_reg(a->vm, 1); - opfn(rd1, rn, rm); - tcg_temp_free_i32(rn); - tcg_temp_free_i32(rm); - - /* Don't store results until after all loads: they might overlap */ - if (accfn) { - tmp = tcg_temp_new_i64(); - neon_load_reg64(tmp, a->vd); - accfn(tmp, tmp, rd0); - neon_store_reg64(tmp, a->vd); - neon_load_reg64(tmp, a->vd + 1); - accfn(tmp, tmp, rd1); - neon_store_reg64(tmp, a->vd + 1); - tcg_temp_free_i64(tmp); - } else { - neon_store_reg64(rd0, a->vd); - neon_store_reg64(rd1, a->vd + 1); - } - - tcg_temp_free_i64(rd0); - tcg_temp_free_i64(rd1); - - return true; -} - -static bool trans_VABDL_S_3d(DisasContext *s, arg_3diff *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - gen_helper_neon_abdl_s16, - gen_helper_neon_abdl_s32, - gen_helper_neon_abdl_s64, - NULL, - }; - - return do_long_3d(s, a, opfn[a->size], NULL); -} - -static bool trans_VABDL_U_3d(DisasContext *s, arg_3diff *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - gen_helper_neon_abdl_u16, - gen_helper_neon_abdl_u32, - gen_helper_neon_abdl_u64, - NULL, - }; - - return do_long_3d(s, a, opfn[a->size], NULL); -} - -static bool trans_VABAL_S_3d(DisasContext *s, arg_3diff *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - gen_helper_neon_abdl_s16, - gen_helper_neon_abdl_s32, - gen_helper_neon_abdl_s64, - NULL, - }; - static NeonGenTwo64OpFn * const addfn[] = { - gen_helper_neon_addl_u16, - gen_helper_neon_addl_u32, - tcg_gen_add_i64, - NULL, - }; - - return do_long_3d(s, a, opfn[a->size], addfn[a->size]); -} - -static bool trans_VABAL_U_3d(DisasContext *s, arg_3diff *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - gen_helper_neon_abdl_u16, - gen_helper_neon_abdl_u32, - gen_helper_neon_abdl_u64, - NULL, - }; - static NeonGenTwo64OpFn * const addfn[] = { - gen_helper_neon_addl_u16, - gen_helper_neon_addl_u32, - tcg_gen_add_i64, - NULL, - }; - - return do_long_3d(s, a, opfn[a->size], addfn[a->size]); -} - -static void gen_mull_s32(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm) -{ - TCGv_i32 lo = tcg_temp_new_i32(); - TCGv_i32 hi = tcg_temp_new_i32(); - - tcg_gen_muls2_i32(lo, hi, rn, rm); - tcg_gen_concat_i32_i64(rd, lo, hi); - - tcg_temp_free_i32(lo); - tcg_temp_free_i32(hi); -} - -static void gen_mull_u32(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm) -{ - TCGv_i32 lo = tcg_temp_new_i32(); - TCGv_i32 hi = tcg_temp_new_i32(); - - tcg_gen_mulu2_i32(lo, hi, rn, rm); - tcg_gen_concat_i32_i64(rd, lo, hi); - - tcg_temp_free_i32(lo); - tcg_temp_free_i32(hi); -} - -static bool trans_VMULL_S_3d(DisasContext *s, arg_3diff *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - gen_helper_neon_mull_s8, - gen_helper_neon_mull_s16, - gen_mull_s32, - NULL, - }; - - return do_long_3d(s, a, opfn[a->size], NULL); -} - -static bool trans_VMULL_U_3d(DisasContext *s, arg_3diff *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - gen_helper_neon_mull_u8, - gen_helper_neon_mull_u16, - gen_mull_u32, - NULL, - }; - - return do_long_3d(s, a, opfn[a->size], NULL); -} - -#define DO_VMLAL(INSN,MULL,ACC) \ - static bool trans_##INSN##_3d(DisasContext *s, arg_3diff *a) \ - { \ - static NeonGenTwoOpWidenFn * const opfn[] = { \ - gen_helper_neon_##MULL##8, \ - gen_helper_neon_##MULL##16, \ - gen_##MULL##32, \ - NULL, \ - }; \ - static NeonGenTwo64OpFn * const accfn[] = { \ - gen_helper_neon_##ACC##l_u16, \ - gen_helper_neon_##ACC##l_u32, \ - tcg_gen_##ACC##_i64, \ - NULL, \ - }; \ - return do_long_3d(s, a, opfn[a->size], accfn[a->size]); \ - } - -DO_VMLAL(VMLAL_S,mull_s,add) -DO_VMLAL(VMLAL_U,mull_u,add) -DO_VMLAL(VMLSL_S,mull_s,sub) -DO_VMLAL(VMLSL_U,mull_u,sub) - -static void gen_VQDMULL_16(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm) -{ - gen_helper_neon_mull_s16(rd, rn, rm); - gen_helper_neon_addl_saturate_s32(rd, cpu_env, rd, rd); -} - -static void gen_VQDMULL_32(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm) -{ - gen_mull_s32(rd, rn, rm); - gen_helper_neon_addl_saturate_s64(rd, cpu_env, rd, rd); -} - -static bool trans_VQDMULL_3d(DisasContext *s, arg_3diff *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - NULL, - gen_VQDMULL_16, - gen_VQDMULL_32, - NULL, - }; - - return do_long_3d(s, a, opfn[a->size], NULL); -} - -static void gen_VQDMLAL_acc_16(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm) -{ - gen_helper_neon_addl_saturate_s32(rd, cpu_env, rn, rm); -} - -static void gen_VQDMLAL_acc_32(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm) -{ - gen_helper_neon_addl_saturate_s64(rd, cpu_env, rn, rm); -} - -static bool trans_VQDMLAL_3d(DisasContext *s, arg_3diff *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - NULL, - gen_VQDMULL_16, - gen_VQDMULL_32, - NULL, - }; - static NeonGenTwo64OpFn * const accfn[] = { - NULL, - gen_VQDMLAL_acc_16, - gen_VQDMLAL_acc_32, - NULL, - }; - - return do_long_3d(s, a, opfn[a->size], accfn[a->size]); -} - -static void gen_VQDMLSL_acc_16(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm) -{ - gen_helper_neon_negl_u32(rm, rm); - gen_helper_neon_addl_saturate_s32(rd, cpu_env, rn, rm); -} - -static void gen_VQDMLSL_acc_32(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm) -{ - tcg_gen_neg_i64(rm, rm); - gen_helper_neon_addl_saturate_s64(rd, cpu_env, rn, rm); -} - -static bool trans_VQDMLSL_3d(DisasContext *s, arg_3diff *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - NULL, - gen_VQDMULL_16, - gen_VQDMULL_32, - NULL, - }; - static NeonGenTwo64OpFn * const accfn[] = { - NULL, - gen_VQDMLSL_acc_16, - gen_VQDMLSL_acc_32, - NULL, - }; - - return do_long_3d(s, a, opfn[a->size], accfn[a->size]); -} - -static bool trans_VMULL_P_3d(DisasContext *s, arg_3diff *a) -{ - gen_helper_gvec_3 *fn_gvec; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (a->vd & 1) { - return false; - } - - switch (a->size) { - case 0: - fn_gvec = gen_helper_neon_pmull_h; - break; - case 2: - if (!dc_isar_feature(aa32_pmull, s)) { - return false; - } - fn_gvec = gen_helper_gvec_pmull_q; - break; - default: - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - tcg_gen_gvec_3_ool(neon_reg_offset(a->vd, 0), - neon_reg_offset(a->vn, 0), - neon_reg_offset(a->vm, 0), - 16, 16, 0, fn_gvec); - return true; -} - -static void gen_neon_dup_low16(TCGv_i32 var) -{ - TCGv_i32 tmp = tcg_temp_new_i32(); - tcg_gen_ext16u_i32(var, var); - tcg_gen_shli_i32(tmp, var, 16); - tcg_gen_or_i32(var, var, tmp); - tcg_temp_free_i32(tmp); -} - -static void gen_neon_dup_high16(TCGv_i32 var) -{ - TCGv_i32 tmp = tcg_temp_new_i32(); - tcg_gen_andi_i32(var, var, 0xffff0000); - tcg_gen_shri_i32(tmp, var, 16); - tcg_gen_or_i32(var, var, tmp); - tcg_temp_free_i32(tmp); -} - -static inline TCGv_i32 neon_get_scalar(int size, int reg) -{ - TCGv_i32 tmp; - if (size == 1) { - tmp = neon_load_reg(reg & 7, reg >> 4); - if (reg & 8) { - gen_neon_dup_high16(tmp); - } else { - gen_neon_dup_low16(tmp); - } - } else { - tmp = neon_load_reg(reg & 15, reg >> 4); - } - return tmp; -} - -static bool do_2scalar(DisasContext *s, arg_2scalar *a, - NeonGenTwoOpFn *opfn, NeonGenTwoOpFn *accfn) -{ - /* - * Two registers and a scalar: perform an operation between - * the input elements and the scalar, and then possibly - * perform an accumulation operation of that result into the - * destination. - */ - TCGv_i32 scalar; - int pass; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (!opfn) { - /* Bad size (including size == 3, which is a different insn group) */ - return false; - } - - if (a->q && ((a->vd | a->vn) & 1)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - scalar = neon_get_scalar(a->size, a->vm); - - for (pass = 0; pass < (a->q ? 4 : 2); pass++) { - TCGv_i32 tmp = neon_load_reg(a->vn, pass); - opfn(tmp, tmp, scalar); - if (accfn) { - TCGv_i32 rd = neon_load_reg(a->vd, pass); - accfn(tmp, rd, tmp); - tcg_temp_free_i32(rd); - } - neon_store_reg(a->vd, pass, tmp); - } - tcg_temp_free_i32(scalar); - return true; -} - -static bool trans_VMUL_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpFn * const opfn[] = { - NULL, - gen_helper_neon_mul_u16, - tcg_gen_mul_i32, - NULL, - }; - - return do_2scalar(s, a, opfn[a->size], NULL); -} - -static bool trans_VMLA_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpFn * const opfn[] = { - NULL, - gen_helper_neon_mul_u16, - tcg_gen_mul_i32, - NULL, - }; - static NeonGenTwoOpFn * const accfn[] = { - NULL, - gen_helper_neon_add_u16, - tcg_gen_add_i32, - NULL, - }; - - return do_2scalar(s, a, opfn[a->size], accfn[a->size]); -} - -static bool trans_VMLS_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpFn * const opfn[] = { - NULL, - gen_helper_neon_mul_u16, - tcg_gen_mul_i32, - NULL, - }; - static NeonGenTwoOpFn * const accfn[] = { - NULL, - gen_helper_neon_sub_u16, - tcg_gen_sub_i32, - NULL, - }; - - return do_2scalar(s, a, opfn[a->size], accfn[a->size]); -} - -/* - * Rather than have a float-specific version of do_2scalar just for - * three insns, we wrap a NeonGenTwoSingleOpFn to turn it into - * a NeonGenTwoOpFn. - */ -#define WRAP_FP_FN(WRAPNAME, FUNC) \ - static void WRAPNAME(TCGv_i32 rd, TCGv_i32 rn, TCGv_i32 rm) \ - { \ - TCGv_ptr fpstatus = get_fpstatus_ptr(1); \ - FUNC(rd, rn, rm, fpstatus); \ - tcg_temp_free_ptr(fpstatus); \ - } - -WRAP_FP_FN(gen_VMUL_F_mul, gen_helper_vfp_muls) -WRAP_FP_FN(gen_VMUL_F_add, gen_helper_vfp_adds) -WRAP_FP_FN(gen_VMUL_F_sub, gen_helper_vfp_subs) - -static bool trans_VMUL_F_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpFn * const opfn[] = { - NULL, - NULL, /* TODO: fp16 support */ - gen_VMUL_F_mul, - NULL, - }; - - return do_2scalar(s, a, opfn[a->size], NULL); -} - -static bool trans_VMLA_F_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpFn * const opfn[] = { - NULL, - NULL, /* TODO: fp16 support */ - gen_VMUL_F_mul, - NULL, - }; - static NeonGenTwoOpFn * const accfn[] = { - NULL, - NULL, /* TODO: fp16 support */ - gen_VMUL_F_add, - NULL, - }; - - return do_2scalar(s, a, opfn[a->size], accfn[a->size]); -} - -static bool trans_VMLS_F_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpFn * const opfn[] = { - NULL, - NULL, /* TODO: fp16 support */ - gen_VMUL_F_mul, - NULL, - }; - static NeonGenTwoOpFn * const accfn[] = { - NULL, - NULL, /* TODO: fp16 support */ - gen_VMUL_F_sub, - NULL, - }; - - return do_2scalar(s, a, opfn[a->size], accfn[a->size]); -} - -WRAP_ENV_FN(gen_VQDMULH_16, gen_helper_neon_qdmulh_s16) -WRAP_ENV_FN(gen_VQDMULH_32, gen_helper_neon_qdmulh_s32) -WRAP_ENV_FN(gen_VQRDMULH_16, gen_helper_neon_qrdmulh_s16) -WRAP_ENV_FN(gen_VQRDMULH_32, gen_helper_neon_qrdmulh_s32) - -static bool trans_VQDMULH_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpFn * const opfn[] = { - NULL, - gen_VQDMULH_16, - gen_VQDMULH_32, - NULL, - }; - - return do_2scalar(s, a, opfn[a->size], NULL); -} - -static bool trans_VQRDMULH_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpFn * const opfn[] = { - NULL, - gen_VQRDMULH_16, - gen_VQRDMULH_32, - NULL, - }; - - return do_2scalar(s, a, opfn[a->size], NULL); -} - -static bool do_vqrdmlah_2sc(DisasContext *s, arg_2scalar *a, - NeonGenThreeOpEnvFn *opfn) -{ - /* - * VQRDMLAH/VQRDMLSH: this is like do_2scalar, but the opfn - * performs a kind of fused op-then-accumulate using a helper - * function that takes all of rd, rn and the scalar at once. - */ - TCGv_i32 scalar; - int pass; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - if (!dc_isar_feature(aa32_rdm, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (!opfn) { - /* Bad size (including size == 3, which is a different insn group) */ - return false; - } - - if (a->q && ((a->vd | a->vn) & 1)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - scalar = neon_get_scalar(a->size, a->vm); - - for (pass = 0; pass < (a->q ? 4 : 2); pass++) { - TCGv_i32 rn = neon_load_reg(a->vn, pass); - TCGv_i32 rd = neon_load_reg(a->vd, pass); - opfn(rd, cpu_env, rn, scalar, rd); - tcg_temp_free_i32(rn); - neon_store_reg(a->vd, pass, rd); - } - tcg_temp_free_i32(scalar); - - return true; -} - -static bool trans_VQRDMLAH_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenThreeOpEnvFn *opfn[] = { - NULL, - gen_helper_neon_qrdmlah_s16, - gen_helper_neon_qrdmlah_s32, - NULL, - }; - return do_vqrdmlah_2sc(s, a, opfn[a->size]); -} - -static bool trans_VQRDMLSH_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenThreeOpEnvFn *opfn[] = { - NULL, - gen_helper_neon_qrdmlsh_s16, - gen_helper_neon_qrdmlsh_s32, - NULL, - }; - return do_vqrdmlah_2sc(s, a, opfn[a->size]); -} - -static bool do_2scalar_long(DisasContext *s, arg_2scalar *a, - NeonGenTwoOpWidenFn *opfn, - NeonGenTwo64OpFn *accfn) -{ - /* - * Two registers and a scalar, long operations: perform an - * operation on the input elements and the scalar which produces - * a double-width result, and then possibly perform an accumulation - * operation of that result into the destination. - */ - TCGv_i32 scalar, rn; - TCGv_i64 rn0_64, rn1_64; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (!opfn) { - /* Bad size (including size == 3, which is a different insn group) */ - return false; - } - - if (a->vd & 1) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - scalar = neon_get_scalar(a->size, a->vm); - - /* Load all inputs before writing any outputs, in case of overlap */ - rn = neon_load_reg(a->vn, 0); - rn0_64 = tcg_temp_new_i64(); - opfn(rn0_64, rn, scalar); - tcg_temp_free_i32(rn); - - rn = neon_load_reg(a->vn, 1); - rn1_64 = tcg_temp_new_i64(); - opfn(rn1_64, rn, scalar); - tcg_temp_free_i32(rn); - tcg_temp_free_i32(scalar); - - if (accfn) { - TCGv_i64 t64 = tcg_temp_new_i64(); - neon_load_reg64(t64, a->vd); - accfn(t64, t64, rn0_64); - neon_store_reg64(t64, a->vd); - neon_load_reg64(t64, a->vd + 1); - accfn(t64, t64, rn1_64); - neon_store_reg64(t64, a->vd + 1); - tcg_temp_free_i64(t64); - } else { - neon_store_reg64(rn0_64, a->vd); - neon_store_reg64(rn1_64, a->vd + 1); - } - tcg_temp_free_i64(rn0_64); - tcg_temp_free_i64(rn1_64); - return true; -} - -static bool trans_VMULL_S_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - NULL, - gen_helper_neon_mull_s16, - gen_mull_s32, - NULL, - }; - - return do_2scalar_long(s, a, opfn[a->size], NULL); -} - -static bool trans_VMULL_U_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - NULL, - gen_helper_neon_mull_u16, - gen_mull_u32, - NULL, - }; - - return do_2scalar_long(s, a, opfn[a->size], NULL); -} - -#define DO_VMLAL_2SC(INSN, MULL, ACC) \ - static bool trans_##INSN##_2sc(DisasContext *s, arg_2scalar *a) \ - { \ - static NeonGenTwoOpWidenFn * const opfn[] = { \ - NULL, \ - gen_helper_neon_##MULL##16, \ - gen_##MULL##32, \ - NULL, \ - }; \ - static NeonGenTwo64OpFn * const accfn[] = { \ - NULL, \ - gen_helper_neon_##ACC##l_u32, \ - tcg_gen_##ACC##_i64, \ - NULL, \ - }; \ - return do_2scalar_long(s, a, opfn[a->size], accfn[a->size]); \ - } - -DO_VMLAL_2SC(VMLAL_S, mull_s, add) -DO_VMLAL_2SC(VMLAL_U, mull_u, add) -DO_VMLAL_2SC(VMLSL_S, mull_s, sub) -DO_VMLAL_2SC(VMLSL_U, mull_u, sub) - -static bool trans_VQDMULL_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - NULL, - gen_VQDMULL_16, - gen_VQDMULL_32, - NULL, - }; - - return do_2scalar_long(s, a, opfn[a->size], NULL); -} - -static bool trans_VQDMLAL_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - NULL, - gen_VQDMULL_16, - gen_VQDMULL_32, - NULL, - }; - static NeonGenTwo64OpFn * const accfn[] = { - NULL, - gen_VQDMLAL_acc_16, - gen_VQDMLAL_acc_32, - NULL, - }; - - return do_2scalar_long(s, a, opfn[a->size], accfn[a->size]); -} - -static bool trans_VQDMLSL_2sc(DisasContext *s, arg_2scalar *a) -{ - static NeonGenTwoOpWidenFn * const opfn[] = { - NULL, - gen_VQDMULL_16, - gen_VQDMULL_32, - NULL, - }; - static NeonGenTwo64OpFn * const accfn[] = { - NULL, - gen_VQDMLSL_acc_16, - gen_VQDMLSL_acc_32, - NULL, - }; - - return do_2scalar_long(s, a, opfn[a->size], accfn[a->size]); -} - -static bool trans_VEXT(DisasContext *s, arg_VEXT *a) -{ - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if ((a->vn | a->vm | a->vd) & a->q) { - return false; - } - - if (a->imm > 7 && !a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - if (!a->q) { - /* Extract 64 bits from */ - TCGv_i64 left, right, dest; - - left = tcg_temp_new_i64(); - right = tcg_temp_new_i64(); - dest = tcg_temp_new_i64(); - - neon_load_reg64(right, a->vn); - neon_load_reg64(left, a->vm); - tcg_gen_extract2_i64(dest, right, left, a->imm * 8); - neon_store_reg64(dest, a->vd); - - tcg_temp_free_i64(left); - tcg_temp_free_i64(right); - tcg_temp_free_i64(dest); - } else { - /* Extract 128 bits from */ - TCGv_i64 left, middle, right, destleft, destright; - - left = tcg_temp_new_i64(); - middle = tcg_temp_new_i64(); - right = tcg_temp_new_i64(); - destleft = tcg_temp_new_i64(); - destright = tcg_temp_new_i64(); - - if (a->imm < 8) { - neon_load_reg64(right, a->vn); - neon_load_reg64(middle, a->vn + 1); - tcg_gen_extract2_i64(destright, right, middle, a->imm * 8); - neon_load_reg64(left, a->vm); - tcg_gen_extract2_i64(destleft, middle, left, a->imm * 8); - } else { - neon_load_reg64(right, a->vn + 1); - neon_load_reg64(middle, a->vm); - tcg_gen_extract2_i64(destright, right, middle, (a->imm - 8) * 8); - neon_load_reg64(left, a->vm + 1); - tcg_gen_extract2_i64(destleft, middle, left, (a->imm - 8) * 8); - } - - neon_store_reg64(destright, a->vd); - neon_store_reg64(destleft, a->vd + 1); - - tcg_temp_free_i64(destright); - tcg_temp_free_i64(destleft); - tcg_temp_free_i64(right); - tcg_temp_free_i64(middle); - tcg_temp_free_i64(left); - } - return true; -} - -static bool trans_VTBL(DisasContext *s, arg_VTBL *a) -{ - int n; - TCGv_i32 tmp, tmp2, tmp3, tmp4; - TCGv_ptr ptr1; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - n = a->len + 1; - if ((a->vn + n) > 32) { - /* - * This is UNPREDICTABLE; we choose to UNDEF to avoid the - * helper function running off the end of the register file. - */ - return false; - } - n <<= 3; - if (a->op) { - tmp = neon_load_reg(a->vd, 0); - } else { - tmp = tcg_temp_new_i32(); - tcg_gen_movi_i32(tmp, 0); - } - tmp2 = neon_load_reg(a->vm, 0); - ptr1 = vfp_reg_ptr(true, a->vn); - tmp4 = tcg_const_i32(n); - gen_helper_neon_tbl(tmp2, tmp2, tmp, ptr1, tmp4); - tcg_temp_free_i32(tmp); - if (a->op) { - tmp = neon_load_reg(a->vd, 1); - } else { - tmp = tcg_temp_new_i32(); - tcg_gen_movi_i32(tmp, 0); - } - tmp3 = neon_load_reg(a->vm, 1); - gen_helper_neon_tbl(tmp3, tmp3, tmp, ptr1, tmp4); - tcg_temp_free_i32(tmp4); - tcg_temp_free_ptr(ptr1); - neon_store_reg(a->vd, 0, tmp2); - neon_store_reg(a->vd, 1, tmp3); - tcg_temp_free_i32(tmp); - return true; -} - -static bool trans_VDUP_scalar(DisasContext *s, arg_VDUP_scalar *a) -{ - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->vd & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - tcg_gen_gvec_dup_mem(a->size, neon_reg_offset(a->vd, 0), - neon_element_offset(a->vm, a->index, a->size), - a->q ? 16 : 8, a->q ? 16 : 8); - return true; -} - -static bool trans_VREV64(DisasContext *s, arg_VREV64 *a) -{ - int pass, half; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if ((a->vd | a->vm) & a->q) { - return false; - } - - if (a->size == 3) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - for (pass = 0; pass < (a->q ? 2 : 1); pass++) { - TCGv_i32 tmp[2]; - - for (half = 0; half < 2; half++) { - tmp[half] = neon_load_reg(a->vm, pass * 2 + half); - switch (a->size) { - case 0: - tcg_gen_bswap32_i32(tmp[half], tmp[half]); - break; - case 1: - gen_swap_half(tmp[half], tmp[half]); - break; - case 2: - break; - default: - g_assert_not_reached(); - } - } - neon_store_reg(a->vd, pass * 2, tmp[1]); - neon_store_reg(a->vd, pass * 2 + 1, tmp[0]); - } - return true; -} - -static bool do_2misc_pairwise(DisasContext *s, arg_2misc *a, - NeonGenWidenFn *widenfn, - NeonGenTwo64OpFn *opfn, - NeonGenTwo64OpFn *accfn) -{ - /* - * Pairwise long operations: widen both halves of the pair, - * combine the pairs with the opfn, and then possibly accumulate - * into the destination with the accfn. - */ - int pass; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if ((a->vd | a->vm) & a->q) { - return false; - } - - if (!widenfn) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - for (pass = 0; pass < a->q + 1; pass++) { - TCGv_i32 tmp; - TCGv_i64 rm0_64, rm1_64, rd_64; - - rm0_64 = tcg_temp_new_i64(); - rm1_64 = tcg_temp_new_i64(); - rd_64 = tcg_temp_new_i64(); - tmp = neon_load_reg(a->vm, pass * 2); - widenfn(rm0_64, tmp); - tcg_temp_free_i32(tmp); - tmp = neon_load_reg(a->vm, pass * 2 + 1); - widenfn(rm1_64, tmp); - tcg_temp_free_i32(tmp); - opfn(rd_64, rm0_64, rm1_64); - tcg_temp_free_i64(rm0_64); - tcg_temp_free_i64(rm1_64); - - if (accfn) { - TCGv_i64 tmp64 = tcg_temp_new_i64(); - neon_load_reg64(tmp64, a->vd + pass); - accfn(rd_64, tmp64, rd_64); - tcg_temp_free_i64(tmp64); - } - neon_store_reg64(rd_64, a->vd + pass); - tcg_temp_free_i64(rd_64); - } - return true; -} - -static bool trans_VPADDL_S(DisasContext *s, arg_2misc *a) -{ - static NeonGenWidenFn * const widenfn[] = { - gen_helper_neon_widen_s8, - gen_helper_neon_widen_s16, - tcg_gen_ext_i32_i64, - NULL, - }; - static NeonGenTwo64OpFn * const opfn[] = { - gen_helper_neon_paddl_u16, - gen_helper_neon_paddl_u32, - tcg_gen_add_i64, - NULL, - }; - - return do_2misc_pairwise(s, a, widenfn[a->size], opfn[a->size], NULL); -} - -static bool trans_VPADDL_U(DisasContext *s, arg_2misc *a) -{ - static NeonGenWidenFn * const widenfn[] = { - gen_helper_neon_widen_u8, - gen_helper_neon_widen_u16, - tcg_gen_extu_i32_i64, - NULL, - }; - static NeonGenTwo64OpFn * const opfn[] = { - gen_helper_neon_paddl_u16, - gen_helper_neon_paddl_u32, - tcg_gen_add_i64, - NULL, - }; - - return do_2misc_pairwise(s, a, widenfn[a->size], opfn[a->size], NULL); -} - -static bool trans_VPADAL_S(DisasContext *s, arg_2misc *a) -{ - static NeonGenWidenFn * const widenfn[] = { - gen_helper_neon_widen_s8, - gen_helper_neon_widen_s16, - tcg_gen_ext_i32_i64, - NULL, - }; - static NeonGenTwo64OpFn * const opfn[] = { - gen_helper_neon_paddl_u16, - gen_helper_neon_paddl_u32, - tcg_gen_add_i64, - NULL, - }; - static NeonGenTwo64OpFn * const accfn[] = { - gen_helper_neon_addl_u16, - gen_helper_neon_addl_u32, - tcg_gen_add_i64, - NULL, - }; - - return do_2misc_pairwise(s, a, widenfn[a->size], opfn[a->size], - accfn[a->size]); -} - -static bool trans_VPADAL_U(DisasContext *s, arg_2misc *a) -{ - static NeonGenWidenFn * const widenfn[] = { - gen_helper_neon_widen_u8, - gen_helper_neon_widen_u16, - tcg_gen_extu_i32_i64, - NULL, - }; - static NeonGenTwo64OpFn * const opfn[] = { - gen_helper_neon_paddl_u16, - gen_helper_neon_paddl_u32, - tcg_gen_add_i64, - NULL, - }; - static NeonGenTwo64OpFn * const accfn[] = { - gen_helper_neon_addl_u16, - gen_helper_neon_addl_u32, - tcg_gen_add_i64, - NULL, - }; - - return do_2misc_pairwise(s, a, widenfn[a->size], opfn[a->size], - accfn[a->size]); -} - -typedef void ZipFn(TCGv_ptr, TCGv_ptr); - -static bool do_zip_uzp(DisasContext *s, arg_2misc *a, - ZipFn *fn) -{ - TCGv_ptr pd, pm; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if ((a->vd | a->vm) & a->q) { - return false; - } - - if (!fn) { - /* Bad size or size/q combination */ - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - pd = vfp_reg_ptr(true, a->vd); - pm = vfp_reg_ptr(true, a->vm); - fn(pd, pm); - tcg_temp_free_ptr(pd); - tcg_temp_free_ptr(pm); - return true; -} - -static bool trans_VUZP(DisasContext *s, arg_2misc *a) -{ - static ZipFn * const fn[2][4] = { - { - gen_helper_neon_unzip8, - gen_helper_neon_unzip16, - NULL, - NULL, - }, { - gen_helper_neon_qunzip8, - gen_helper_neon_qunzip16, - gen_helper_neon_qunzip32, - NULL, - } - }; - return do_zip_uzp(s, a, fn[a->q][a->size]); -} - -static bool trans_VZIP(DisasContext *s, arg_2misc *a) -{ - static ZipFn * const fn[2][4] = { - { - gen_helper_neon_zip8, - gen_helper_neon_zip16, - NULL, - NULL, - }, { - gen_helper_neon_qzip8, - gen_helper_neon_qzip16, - gen_helper_neon_qzip32, - NULL, - } - }; - return do_zip_uzp(s, a, fn[a->q][a->size]); -} - -static bool do_vmovn(DisasContext *s, arg_2misc *a, - NeonGenNarrowEnvFn *narrowfn) -{ - TCGv_i64 rm; - TCGv_i32 rd0, rd1; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->vm & 1) { - return false; - } - - if (!narrowfn) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - rm = tcg_temp_new_i64(); - rd0 = tcg_temp_new_i32(); - rd1 = tcg_temp_new_i32(); - - neon_load_reg64(rm, a->vm); - narrowfn(rd0, cpu_env, rm); - neon_load_reg64(rm, a->vm + 1); - narrowfn(rd1, cpu_env, rm); - neon_store_reg(a->vd, 0, rd0); - neon_store_reg(a->vd, 1, rd1); - tcg_temp_free_i64(rm); - return true; -} - -#define DO_VMOVN(INSN, FUNC) \ - static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ - { \ - static NeonGenNarrowEnvFn * const narrowfn[] = { \ - FUNC##8, \ - FUNC##16, \ - FUNC##32, \ - NULL, \ - }; \ - return do_vmovn(s, a, narrowfn[a->size]); \ - } - -DO_VMOVN(VMOVN, gen_neon_narrow_u) -DO_VMOVN(VQMOVUN, gen_helper_neon_unarrow_sat) -DO_VMOVN(VQMOVN_S, gen_helper_neon_narrow_sat_s) -DO_VMOVN(VQMOVN_U, gen_helper_neon_narrow_sat_u) - -static bool trans_VSHLL(DisasContext *s, arg_2misc *a) -{ - TCGv_i32 rm0, rm1; - TCGv_i64 rd; - static NeonGenWidenFn * const widenfns[] = { - gen_helper_neon_widen_u8, - gen_helper_neon_widen_u16, - tcg_gen_extu_i32_i64, - NULL, - }; - NeonGenWidenFn *widenfn = widenfns[a->size]; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->vd & 1) { - return false; - } - - if (!widenfn) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - rd = tcg_temp_new_i64(); - - rm0 = neon_load_reg(a->vm, 0); - rm1 = neon_load_reg(a->vm, 1); - - widenfn(rd, rm0); - tcg_gen_shli_i64(rd, rd, 8 << a->size); - neon_store_reg64(rd, a->vd); - widenfn(rd, rm1); - tcg_gen_shli_i64(rd, rd, 8 << a->size); - neon_store_reg64(rd, a->vd + 1); - - tcg_temp_free_i64(rd); - tcg_temp_free_i32(rm0); - tcg_temp_free_i32(rm1); - return true; -} - -static bool trans_VCVT_F16_F32(DisasContext *s, arg_2misc *a) -{ - TCGv_ptr fpst; - TCGv_i32 ahp, tmp, tmp2, tmp3; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON) || - !dc_isar_feature(aa32_fp16_spconv, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if ((a->vm & 1) || (a->size != 1)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(true); - ahp = get_ahp_flag(); - tmp = neon_load_reg(a->vm, 0); - gen_helper_vfp_fcvt_f32_to_f16(tmp, tmp, fpst, ahp); - tmp2 = neon_load_reg(a->vm, 1); - gen_helper_vfp_fcvt_f32_to_f16(tmp2, tmp2, fpst, ahp); - tcg_gen_shli_i32(tmp2, tmp2, 16); - tcg_gen_or_i32(tmp2, tmp2, tmp); - tcg_temp_free_i32(tmp); - tmp = neon_load_reg(a->vm, 2); - gen_helper_vfp_fcvt_f32_to_f16(tmp, tmp, fpst, ahp); - tmp3 = neon_load_reg(a->vm, 3); - neon_store_reg(a->vd, 0, tmp2); - gen_helper_vfp_fcvt_f32_to_f16(tmp3, tmp3, fpst, ahp); - tcg_gen_shli_i32(tmp3, tmp3, 16); - tcg_gen_or_i32(tmp3, tmp3, tmp); - neon_store_reg(a->vd, 1, tmp3); - tcg_temp_free_i32(tmp); - tcg_temp_free_i32(ahp); - tcg_temp_free_ptr(fpst); - - return true; -} - -static bool trans_VCVT_F32_F16(DisasContext *s, arg_2misc *a) -{ - TCGv_ptr fpst; - TCGv_i32 ahp, tmp, tmp2, tmp3; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON) || - !dc_isar_feature(aa32_fp16_spconv, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if ((a->vd & 1) || (a->size != 1)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(true); - ahp = get_ahp_flag(); - tmp3 = tcg_temp_new_i32(); - tmp = neon_load_reg(a->vm, 0); - tmp2 = neon_load_reg(a->vm, 1); - tcg_gen_ext16u_i32(tmp3, tmp); - gen_helper_vfp_fcvt_f16_to_f32(tmp3, tmp3, fpst, ahp); - neon_store_reg(a->vd, 0, tmp3); - tcg_gen_shri_i32(tmp, tmp, 16); - gen_helper_vfp_fcvt_f16_to_f32(tmp, tmp, fpst, ahp); - neon_store_reg(a->vd, 1, tmp); - tmp3 = tcg_temp_new_i32(); - tcg_gen_ext16u_i32(tmp3, tmp2); - gen_helper_vfp_fcvt_f16_to_f32(tmp3, tmp3, fpst, ahp); - neon_store_reg(a->vd, 2, tmp3); - tcg_gen_shri_i32(tmp2, tmp2, 16); - gen_helper_vfp_fcvt_f16_to_f32(tmp2, tmp2, fpst, ahp); - neon_store_reg(a->vd, 3, tmp2); - tcg_temp_free_i32(ahp); - tcg_temp_free_ptr(fpst); - - return true; -} - -static bool do_2misc_vec(DisasContext *s, arg_2misc *a, GVecGen2Fn *fn) -{ - int vec_size = a->q ? 16 : 8; - int rd_ofs = neon_reg_offset(a->vd, 0); - int rm_ofs = neon_reg_offset(a->vm, 0); - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->size == 3) { - return false; - } - - if ((a->vd | a->vm) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fn(a->size, rd_ofs, rm_ofs, vec_size, vec_size); - - return true; -} - -#define DO_2MISC_VEC(INSN, FN) \ - static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ - { \ - return do_2misc_vec(s, a, FN); \ - } - -DO_2MISC_VEC(VNEG, tcg_gen_gvec_neg) -DO_2MISC_VEC(VABS, tcg_gen_gvec_abs) -DO_2MISC_VEC(VCEQ0, gen_gvec_ceq0) -DO_2MISC_VEC(VCGT0, gen_gvec_cgt0) -DO_2MISC_VEC(VCLE0, gen_gvec_cle0) -DO_2MISC_VEC(VCGE0, gen_gvec_cge0) -DO_2MISC_VEC(VCLT0, gen_gvec_clt0) - -static bool trans_VMVN(DisasContext *s, arg_2misc *a) -{ - if (a->size != 0) { - return false; - } - return do_2misc_vec(s, a, tcg_gen_gvec_not); -} - -#define WRAP_2M_3_OOL_FN(WRAPNAME, FUNC, DATA) \ - static void WRAPNAME(unsigned vece, uint32_t rd_ofs, \ - uint32_t rm_ofs, uint32_t oprsz, \ - uint32_t maxsz) \ - { \ - tcg_gen_gvec_3_ool(rd_ofs, rd_ofs, rm_ofs, oprsz, maxsz, \ - DATA, FUNC); \ - } - -#define WRAP_2M_2_OOL_FN(WRAPNAME, FUNC, DATA) \ - static void WRAPNAME(unsigned vece, uint32_t rd_ofs, \ - uint32_t rm_ofs, uint32_t oprsz, \ - uint32_t maxsz) \ - { \ - tcg_gen_gvec_2_ool(rd_ofs, rm_ofs, oprsz, maxsz, DATA, FUNC); \ - } - -WRAP_2M_3_OOL_FN(gen_AESE, gen_helper_crypto_aese, 0) -WRAP_2M_3_OOL_FN(gen_AESD, gen_helper_crypto_aese, 1) -WRAP_2M_2_OOL_FN(gen_AESMC, gen_helper_crypto_aesmc, 0) -WRAP_2M_2_OOL_FN(gen_AESIMC, gen_helper_crypto_aesmc, 1) -WRAP_2M_2_OOL_FN(gen_SHA1H, gen_helper_crypto_sha1h, 0) -WRAP_2M_2_OOL_FN(gen_SHA1SU1, gen_helper_crypto_sha1su1, 0) -WRAP_2M_2_OOL_FN(gen_SHA256SU0, gen_helper_crypto_sha256su0, 0) - -#define DO_2M_CRYPTO(INSN, FEATURE, SIZE) \ - static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ - { \ - if (!dc_isar_feature(FEATURE, s) || a->size != SIZE) { \ - return false; \ - } \ - return do_2misc_vec(s, a, gen_##INSN); \ - } - -DO_2M_CRYPTO(AESE, aa32_aes, 0) -DO_2M_CRYPTO(AESD, aa32_aes, 0) -DO_2M_CRYPTO(AESMC, aa32_aes, 0) -DO_2M_CRYPTO(AESIMC, aa32_aes, 0) -DO_2M_CRYPTO(SHA1H, aa32_sha1, 2) -DO_2M_CRYPTO(SHA1SU1, aa32_sha1, 2) -DO_2M_CRYPTO(SHA256SU0, aa32_sha2, 2) - -static bool do_2misc(DisasContext *s, arg_2misc *a, NeonGenOneOpFn *fn) -{ - int pass; - - /* Handle a 2-reg-misc operation by iterating 32 bits at a time */ - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (!fn) { - return false; - } - - if ((a->vd | a->vm) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - for (pass = 0; pass < (a->q ? 4 : 2); pass++) { - TCGv_i32 tmp = neon_load_reg(a->vm, pass); - fn(tmp, tmp); - neon_store_reg(a->vd, pass, tmp); - } - - return true; -} - -static bool trans_VREV32(DisasContext *s, arg_2misc *a) -{ - static NeonGenOneOpFn * const fn[] = { - tcg_gen_bswap32_i32, - gen_swap_half, - NULL, - NULL, - }; - return do_2misc(s, a, fn[a->size]); -} - -static bool trans_VREV16(DisasContext *s, arg_2misc *a) -{ - if (a->size != 0) { - return false; - } - return do_2misc(s, a, gen_rev16); -} - -static bool trans_VCLS(DisasContext *s, arg_2misc *a) -{ - static NeonGenOneOpFn * const fn[] = { - gen_helper_neon_cls_s8, - gen_helper_neon_cls_s16, - gen_helper_neon_cls_s32, - NULL, - }; - return do_2misc(s, a, fn[a->size]); -} - -static void do_VCLZ_32(TCGv_i32 rd, TCGv_i32 rm) -{ - tcg_gen_clzi_i32(rd, rm, 32); -} - -static bool trans_VCLZ(DisasContext *s, arg_2misc *a) -{ - static NeonGenOneOpFn * const fn[] = { - gen_helper_neon_clz_u8, - gen_helper_neon_clz_u16, - do_VCLZ_32, - NULL, - }; - return do_2misc(s, a, fn[a->size]); -} - -static bool trans_VCNT(DisasContext *s, arg_2misc *a) -{ - if (a->size != 0) { - return false; - } - return do_2misc(s, a, gen_helper_neon_cnt_u8); -} - -static bool trans_VABS_F(DisasContext *s, arg_2misc *a) -{ - if (a->size != 2) { - return false; - } - /* TODO: FP16 : size == 1 */ - return do_2misc(s, a, gen_helper_vfp_abss); -} - -static bool trans_VNEG_F(DisasContext *s, arg_2misc *a) -{ - if (a->size != 2) { - return false; - } - /* TODO: FP16 : size == 1 */ - return do_2misc(s, a, gen_helper_vfp_negs); -} - -static bool trans_VRECPE(DisasContext *s, arg_2misc *a) -{ - if (a->size != 2) { - return false; - } - return do_2misc(s, a, gen_helper_recpe_u32); -} - -static bool trans_VRSQRTE(DisasContext *s, arg_2misc *a) -{ - if (a->size != 2) { - return false; - } - return do_2misc(s, a, gen_helper_rsqrte_u32); -} - -#define WRAP_1OP_ENV_FN(WRAPNAME, FUNC) \ - static void WRAPNAME(TCGv_i32 d, TCGv_i32 m) \ - { \ - FUNC(d, cpu_env, m); \ - } - -WRAP_1OP_ENV_FN(gen_VQABS_s8, gen_helper_neon_qabs_s8) -WRAP_1OP_ENV_FN(gen_VQABS_s16, gen_helper_neon_qabs_s16) -WRAP_1OP_ENV_FN(gen_VQABS_s32, gen_helper_neon_qabs_s32) -WRAP_1OP_ENV_FN(gen_VQNEG_s8, gen_helper_neon_qneg_s8) -WRAP_1OP_ENV_FN(gen_VQNEG_s16, gen_helper_neon_qneg_s16) -WRAP_1OP_ENV_FN(gen_VQNEG_s32, gen_helper_neon_qneg_s32) - -static bool trans_VQABS(DisasContext *s, arg_2misc *a) -{ - static NeonGenOneOpFn * const fn[] = { - gen_VQABS_s8, - gen_VQABS_s16, - gen_VQABS_s32, - NULL, - }; - return do_2misc(s, a, fn[a->size]); -} - -static bool trans_VQNEG(DisasContext *s, arg_2misc *a) -{ - static NeonGenOneOpFn * const fn[] = { - gen_VQNEG_s8, - gen_VQNEG_s16, - gen_VQNEG_s32, - NULL, - }; - return do_2misc(s, a, fn[a->size]); -} - -static bool do_2misc_fp(DisasContext *s, arg_2misc *a, - NeonGenOneSingleOpFn *fn) -{ - int pass; - TCGv_ptr fpst; - - /* Handle a 2-reg-misc operation by iterating 32 bits at a time */ - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->size != 2) { - /* TODO: FP16 will be the size == 1 case */ - return false; - } - - if ((a->vd | a->vm) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(1); - for (pass = 0; pass < (a->q ? 4 : 2); pass++) { - TCGv_i32 tmp = neon_load_reg(a->vm, pass); - fn(tmp, tmp, fpst); - neon_store_reg(a->vd, pass, tmp); - } - tcg_temp_free_ptr(fpst); - - return true; -} - -#define DO_2MISC_FP(INSN, FUNC) \ - static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ - { \ - return do_2misc_fp(s, a, FUNC); \ - } - -DO_2MISC_FP(VRECPE_F, gen_helper_recpe_f32) -DO_2MISC_FP(VRSQRTE_F, gen_helper_rsqrte_f32) -DO_2MISC_FP(VCVT_FS, gen_helper_vfp_sitos) -DO_2MISC_FP(VCVT_FU, gen_helper_vfp_uitos) -DO_2MISC_FP(VCVT_SF, gen_helper_vfp_tosizs) -DO_2MISC_FP(VCVT_UF, gen_helper_vfp_touizs) - -static bool trans_VRINTX(DisasContext *s, arg_2misc *a) -{ - if (!arm_dc_feature(s, ARM_FEATURE_V8)) { - return false; - } - return do_2misc_fp(s, a, gen_helper_rints_exact); -} - -#define WRAP_FP_CMP0_FWD(WRAPNAME, FUNC) \ - static void WRAPNAME(TCGv_i32 d, TCGv_i32 m, TCGv_ptr fpst) \ - { \ - TCGv_i32 zero = tcg_const_i32(0); \ - FUNC(d, m, zero, fpst); \ - tcg_temp_free_i32(zero); \ - } -#define WRAP_FP_CMP0_REV(WRAPNAME, FUNC) \ - static void WRAPNAME(TCGv_i32 d, TCGv_i32 m, TCGv_ptr fpst) \ - { \ - TCGv_i32 zero = tcg_const_i32(0); \ - FUNC(d, zero, m, fpst); \ - tcg_temp_free_i32(zero); \ - } - -#define DO_FP_CMP0(INSN, FUNC, REV) \ - WRAP_FP_CMP0_##REV(gen_##INSN, FUNC) \ - static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ - { \ - return do_2misc_fp(s, a, gen_##INSN); \ - } - -DO_FP_CMP0(VCGT0_F, gen_helper_neon_cgt_f32, FWD) -DO_FP_CMP0(VCGE0_F, gen_helper_neon_cge_f32, FWD) -DO_FP_CMP0(VCEQ0_F, gen_helper_neon_ceq_f32, FWD) -DO_FP_CMP0(VCLE0_F, gen_helper_neon_cge_f32, REV) -DO_FP_CMP0(VCLT0_F, gen_helper_neon_cgt_f32, REV) - -static bool do_vrint(DisasContext *s, arg_2misc *a, int rmode) -{ - /* - * Handle a VRINT* operation by iterating 32 bits at a time, - * with a specified rounding mode in operation. - */ - int pass; - TCGv_ptr fpst; - TCGv_i32 tcg_rmode; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON) || - !arm_dc_feature(s, ARM_FEATURE_V8)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->size != 2) { - /* TODO: FP16 will be the size == 1 case */ - return false; - } - - if ((a->vd | a->vm) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(1); - tcg_rmode = tcg_const_i32(arm_rmode_to_sf(rmode)); - gen_helper_set_neon_rmode(tcg_rmode, tcg_rmode, cpu_env); - for (pass = 0; pass < (a->q ? 4 : 2); pass++) { - TCGv_i32 tmp = neon_load_reg(a->vm, pass); - gen_helper_rints(tmp, tmp, fpst); - neon_store_reg(a->vd, pass, tmp); - } - gen_helper_set_neon_rmode(tcg_rmode, tcg_rmode, cpu_env); - tcg_temp_free_i32(tcg_rmode); - tcg_temp_free_ptr(fpst); - - return true; -} - -#define DO_VRINT(INSN, RMODE) \ - static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ - { \ - return do_vrint(s, a, RMODE); \ - } - -DO_VRINT(VRINTN, FPROUNDING_TIEEVEN) -DO_VRINT(VRINTA, FPROUNDING_TIEAWAY) -DO_VRINT(VRINTZ, FPROUNDING_ZERO) -DO_VRINT(VRINTM, FPROUNDING_NEGINF) -DO_VRINT(VRINTP, FPROUNDING_POSINF) - -static bool do_vcvt(DisasContext *s, arg_2misc *a, int rmode, bool is_signed) -{ - /* - * Handle a VCVT* operation by iterating 32 bits at a time, - * with a specified rounding mode in operation. - */ - int pass; - TCGv_ptr fpst; - TCGv_i32 tcg_rmode, tcg_shift; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON) || - !arm_dc_feature(s, ARM_FEATURE_V8)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->size != 2) { - /* TODO: FP16 will be the size == 1 case */ - return false; - } - - if ((a->vd | a->vm) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(1); - tcg_shift = tcg_const_i32(0); - tcg_rmode = tcg_const_i32(arm_rmode_to_sf(rmode)); - gen_helper_set_neon_rmode(tcg_rmode, tcg_rmode, cpu_env); - for (pass = 0; pass < (a->q ? 4 : 2); pass++) { - TCGv_i32 tmp = neon_load_reg(a->vm, pass); - if (is_signed) { - gen_helper_vfp_tosls(tmp, tmp, tcg_shift, fpst); - } else { - gen_helper_vfp_touls(tmp, tmp, tcg_shift, fpst); - } - neon_store_reg(a->vd, pass, tmp); - } - gen_helper_set_neon_rmode(tcg_rmode, tcg_rmode, cpu_env); - tcg_temp_free_i32(tcg_rmode); - tcg_temp_free_i32(tcg_shift); - tcg_temp_free_ptr(fpst); - - return true; -} - -#define DO_VCVT(INSN, RMODE, SIGNED) \ - static bool trans_##INSN(DisasContext *s, arg_2misc *a) \ - { \ - return do_vcvt(s, a, RMODE, SIGNED); \ - } - -DO_VCVT(VCVTAU, FPROUNDING_TIEAWAY, false) -DO_VCVT(VCVTAS, FPROUNDING_TIEAWAY, true) -DO_VCVT(VCVTNU, FPROUNDING_TIEEVEN, false) -DO_VCVT(VCVTNS, FPROUNDING_TIEEVEN, true) -DO_VCVT(VCVTPU, FPROUNDING_POSINF, false) -DO_VCVT(VCVTPS, FPROUNDING_POSINF, true) -DO_VCVT(VCVTMU, FPROUNDING_NEGINF, false) -DO_VCVT(VCVTMS, FPROUNDING_NEGINF, true) - -static bool trans_VSWP(DisasContext *s, arg_2misc *a) -{ - TCGv_i64 rm, rd; - int pass; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (a->size != 0) { - return false; - } - - if ((a->vd | a->vm) & a->q) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - rm = tcg_temp_new_i64(); - rd = tcg_temp_new_i64(); - for (pass = 0; pass < (a->q ? 2 : 1); pass++) { - neon_load_reg64(rm, a->vm + pass); - neon_load_reg64(rd, a->vd + pass); - neon_store_reg64(rm, a->vd + pass); - neon_store_reg64(rd, a->vm + pass); - } - tcg_temp_free_i64(rm); - tcg_temp_free_i64(rd); - - return true; -} -static void gen_neon_trn_u8(TCGv_i32 t0, TCGv_i32 t1) -{ - TCGv_i32 rd, tmp; - - rd = tcg_temp_new_i32(); - tmp = tcg_temp_new_i32(); - - tcg_gen_shli_i32(rd, t0, 8); - tcg_gen_andi_i32(rd, rd, 0xff00ff00); - tcg_gen_andi_i32(tmp, t1, 0x00ff00ff); - tcg_gen_or_i32(rd, rd, tmp); - - tcg_gen_shri_i32(t1, t1, 8); - tcg_gen_andi_i32(t1, t1, 0x00ff00ff); - tcg_gen_andi_i32(tmp, t0, 0xff00ff00); - tcg_gen_or_i32(t1, t1, tmp); - tcg_gen_mov_i32(t0, rd); - - tcg_temp_free_i32(tmp); - tcg_temp_free_i32(rd); -} - -static void gen_neon_trn_u16(TCGv_i32 t0, TCGv_i32 t1) -{ - TCGv_i32 rd, tmp; - - rd = tcg_temp_new_i32(); - tmp = tcg_temp_new_i32(); - - tcg_gen_shli_i32(rd, t0, 16); - tcg_gen_andi_i32(tmp, t1, 0xffff); - tcg_gen_or_i32(rd, rd, tmp); - tcg_gen_shri_i32(t1, t1, 16); - tcg_gen_andi_i32(tmp, t0, 0xffff0000); - tcg_gen_or_i32(t1, t1, tmp); - tcg_gen_mov_i32(t0, rd); - - tcg_temp_free_i32(tmp); - tcg_temp_free_i32(rd); -} - -static bool trans_VTRN(DisasContext *s, arg_2misc *a) -{ - TCGv_i32 tmp, tmp2; - int pass; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vm) & 0x10)) { - return false; - } - - if ((a->vd | a->vm) & a->q) { - return false; - } - - if (a->size == 3) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - if (a->size == 2) { - for (pass = 0; pass < (a->q ? 4 : 2); pass += 2) { - tmp = neon_load_reg(a->vm, pass); - tmp2 = neon_load_reg(a->vd, pass + 1); - neon_store_reg(a->vm, pass, tmp2); - neon_store_reg(a->vd, pass + 1, tmp); - } - } else { - for (pass = 0; pass < (a->q ? 4 : 2); pass++) { - tmp = neon_load_reg(a->vm, pass); - tmp2 = neon_load_reg(a->vd, pass); - if (a->size == 0) { - gen_neon_trn_u8(tmp, tmp2); - } else { - gen_neon_trn_u16(tmp, tmp2); - } - neon_store_reg(a->vm, pass, tmp2); - neon_store_reg(a->vd, pass, tmp); - } - } - return true; -} diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index 88a2fb2..8c7fbbd 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -100,7 +100,7 @@ static inline int msz_dtype(DisasContext *s, int msz) * Include the generated decoder. */ -#include "decode-sve.inc.c" +#include "decode-sve.c.inc" /* * Implement all of the translator functions referenced by the decoder. diff --git a/target/arm/translate-vfp.c.inc b/target/arm/translate-vfp.c.inc new file mode 100644 index 0000000..2d63fa0 --- /dev/null +++ b/target/arm/translate-vfp.c.inc @@ -0,0 +1,2865 @@ +/* + * ARM translation: AArch32 VFP instructions + * + * Copyright (c) 2003 Fabrice Bellard + * Copyright (c) 2005-2007 CodeSourcery + * Copyright (c) 2007 OpenedHand, Ltd. + * Copyright (c) 2019 Linaro, Ltd. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +/* + * This file is intended to be included from translate.c; it uses + * some macros and definitions provided by that file. + * It might be possible to convert it to a standalone .c file eventually. + */ + +/* Include the generated VFP decoder */ +#include "decode-vfp.c.inc" +#include "decode-vfp-uncond.c.inc" + +/* + * The imm8 encodes the sign bit, enough bits to represent an exponent in + * the range 01....1xx to 10....0xx, and the most significant 4 bits of + * the mantissa; see VFPExpandImm() in the v8 ARM ARM. + */ +uint64_t vfp_expand_imm(int size, uint8_t imm8) +{ + uint64_t imm; + + switch (size) { + case MO_64: + imm = (extract32(imm8, 7, 1) ? 0x8000 : 0) | + (extract32(imm8, 6, 1) ? 0x3fc0 : 0x4000) | + extract32(imm8, 0, 6); + imm <<= 48; + break; + case MO_32: + imm = (extract32(imm8, 7, 1) ? 0x8000 : 0) | + (extract32(imm8, 6, 1) ? 0x3e00 : 0x4000) | + (extract32(imm8, 0, 6) << 3); + imm <<= 16; + break; + case MO_16: + imm = (extract32(imm8, 7, 1) ? 0x8000 : 0) | + (extract32(imm8, 6, 1) ? 0x3000 : 0x4000) | + (extract32(imm8, 0, 6) << 6); + break; + default: + g_assert_not_reached(); + } + return imm; +} + +/* + * Return the offset of a 16-bit half of the specified VFP single-precision + * register. If top is true, returns the top 16 bits; otherwise the bottom + * 16 bits. + */ +static inline long vfp_f16_offset(unsigned reg, bool top) +{ + long offs = vfp_reg_offset(false, reg); +#ifdef HOST_WORDS_BIGENDIAN + if (!top) { + offs += 2; + } +#else + if (top) { + offs += 2; + } +#endif + return offs; +} + +/* + * Check that VFP access is enabled. If it is, do the necessary + * M-profile lazy-FP handling and then return true. + * If not, emit code to generate an appropriate exception and + * return false. + * The ignore_vfp_enabled argument specifies that we should ignore + * whether VFP is enabled via FPEXC[EN]: this should be true for FMXR/FMRX + * accesses to FPSID, FPEXC, MVFR0, MVFR1, MVFR2, and false for all other insns. + */ +static bool full_vfp_access_check(DisasContext *s, bool ignore_vfp_enabled) +{ + if (s->fp_excp_el) { + if (arm_dc_feature(s, ARM_FEATURE_M)) { + gen_exception_insn(s, s->pc_curr, EXCP_NOCP, syn_uncategorized(), + s->fp_excp_el); + } else { + gen_exception_insn(s, s->pc_curr, EXCP_UDEF, + syn_fp_access_trap(1, 0xe, false), + s->fp_excp_el); + } + return false; + } + + if (!s->vfp_enabled && !ignore_vfp_enabled) { + assert(!arm_dc_feature(s, ARM_FEATURE_M)); + unallocated_encoding(s); + return false; + } + + if (arm_dc_feature(s, ARM_FEATURE_M)) { + /* Handle M-profile lazy FP state mechanics */ + + /* Trigger lazy-state preservation if necessary */ + if (s->v7m_lspact) { + /* + * Lazy state saving affects external memory and also the NVIC, + * so we must mark it as an IO operation for icount (and cause + * this to be the last insn in the TB). + */ + if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { + s->base.is_jmp = DISAS_UPDATE_EXIT; + gen_io_start(); + } + gen_helper_v7m_preserve_fp_state(cpu_env); + /* + * If the preserve_fp_state helper doesn't throw an exception + * then it will clear LSPACT; we don't need to repeat this for + * any further FP insns in this TB. + */ + s->v7m_lspact = false; + } + + /* Update ownership of FP context: set FPCCR.S to match current state */ + if (s->v8m_fpccr_s_wrong) { + TCGv_i32 tmp; + + tmp = load_cpu_field(v7m.fpccr[M_REG_S]); + if (s->v8m_secure) { + tcg_gen_ori_i32(tmp, tmp, R_V7M_FPCCR_S_MASK); + } else { + tcg_gen_andi_i32(tmp, tmp, ~R_V7M_FPCCR_S_MASK); + } + store_cpu_field(tmp, v7m.fpccr[M_REG_S]); + /* Don't need to do this for any further FP insns in this TB */ + s->v8m_fpccr_s_wrong = false; + } + + if (s->v7m_new_fp_ctxt_needed) { + /* + * Create new FP context by updating CONTROL.FPCA, CONTROL.SFPA + * and the FPSCR. + */ + TCGv_i32 control, fpscr; + uint32_t bits = R_V7M_CONTROL_FPCA_MASK; + + fpscr = load_cpu_field(v7m.fpdscr[s->v8m_secure]); + gen_helper_vfp_set_fpscr(cpu_env, fpscr); + tcg_temp_free_i32(fpscr); + /* + * We don't need to arrange to end the TB, because the only + * parts of FPSCR which we cache in the TB flags are the VECLEN + * and VECSTRIDE, and those don't exist for M-profile. + */ + + if (s->v8m_secure) { + bits |= R_V7M_CONTROL_SFPA_MASK; + } + control = load_cpu_field(v7m.control[M_REG_S]); + tcg_gen_ori_i32(control, control, bits); + store_cpu_field(control, v7m.control[M_REG_S]); + /* Don't need to do this for any further FP insns in this TB */ + s->v7m_new_fp_ctxt_needed = false; + } + } + + return true; +} + +/* + * The most usual kind of VFP access check, for everything except + * FMXR/FMRX to the always-available special registers. + */ +static bool vfp_access_check(DisasContext *s) +{ + return full_vfp_access_check(s, false); +} + +static bool trans_VSEL(DisasContext *s, arg_VSEL *a) +{ + uint32_t rd, rn, rm; + bool dp = a->dp; + + if (!dc_isar_feature(aa32_vsel, s)) { + return false; + } + + if (dp && !dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (dp && !dc_isar_feature(aa32_simd_r32, s) && + ((a->vm | a->vn | a->vd) & 0x10)) { + return false; + } + + rd = a->vd; + rn = a->vn; + rm = a->vm; + + if (!vfp_access_check(s)) { + return true; + } + + if (dp) { + TCGv_i64 frn, frm, dest; + TCGv_i64 tmp, zero, zf, nf, vf; + + zero = tcg_const_i64(0); + + frn = tcg_temp_new_i64(); + frm = tcg_temp_new_i64(); + dest = tcg_temp_new_i64(); + + zf = tcg_temp_new_i64(); + nf = tcg_temp_new_i64(); + vf = tcg_temp_new_i64(); + + tcg_gen_extu_i32_i64(zf, cpu_ZF); + tcg_gen_ext_i32_i64(nf, cpu_NF); + tcg_gen_ext_i32_i64(vf, cpu_VF); + + neon_load_reg64(frn, rn); + neon_load_reg64(frm, rm); + switch (a->cc) { + case 0: /* eq: Z */ + tcg_gen_movcond_i64(TCG_COND_EQ, dest, zf, zero, + frn, frm); + break; + case 1: /* vs: V */ + tcg_gen_movcond_i64(TCG_COND_LT, dest, vf, zero, + frn, frm); + break; + case 2: /* ge: N == V -> N ^ V == 0 */ + tmp = tcg_temp_new_i64(); + tcg_gen_xor_i64(tmp, vf, nf); + tcg_gen_movcond_i64(TCG_COND_GE, dest, tmp, zero, + frn, frm); + tcg_temp_free_i64(tmp); + break; + case 3: /* gt: !Z && N == V */ + tcg_gen_movcond_i64(TCG_COND_NE, dest, zf, zero, + frn, frm); + tmp = tcg_temp_new_i64(); + tcg_gen_xor_i64(tmp, vf, nf); + tcg_gen_movcond_i64(TCG_COND_GE, dest, tmp, zero, + dest, frm); + tcg_temp_free_i64(tmp); + break; + } + neon_store_reg64(dest, rd); + tcg_temp_free_i64(frn); + tcg_temp_free_i64(frm); + tcg_temp_free_i64(dest); + + tcg_temp_free_i64(zf); + tcg_temp_free_i64(nf); + tcg_temp_free_i64(vf); + + tcg_temp_free_i64(zero); + } else { + TCGv_i32 frn, frm, dest; + TCGv_i32 tmp, zero; + + zero = tcg_const_i32(0); + + frn = tcg_temp_new_i32(); + frm = tcg_temp_new_i32(); + dest = tcg_temp_new_i32(); + neon_load_reg32(frn, rn); + neon_load_reg32(frm, rm); + switch (a->cc) { + case 0: /* eq: Z */ + tcg_gen_movcond_i32(TCG_COND_EQ, dest, cpu_ZF, zero, + frn, frm); + break; + case 1: /* vs: V */ + tcg_gen_movcond_i32(TCG_COND_LT, dest, cpu_VF, zero, + frn, frm); + break; + case 2: /* ge: N == V -> N ^ V == 0 */ + tmp = tcg_temp_new_i32(); + tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF); + tcg_gen_movcond_i32(TCG_COND_GE, dest, tmp, zero, + frn, frm); + tcg_temp_free_i32(tmp); + break; + case 3: /* gt: !Z && N == V */ + tcg_gen_movcond_i32(TCG_COND_NE, dest, cpu_ZF, zero, + frn, frm); + tmp = tcg_temp_new_i32(); + tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF); + tcg_gen_movcond_i32(TCG_COND_GE, dest, tmp, zero, + dest, frm); + tcg_temp_free_i32(tmp); + break; + } + neon_store_reg32(dest, rd); + tcg_temp_free_i32(frn); + tcg_temp_free_i32(frm); + tcg_temp_free_i32(dest); + + tcg_temp_free_i32(zero); + } + + return true; +} + +/* + * Table for converting the most common AArch32 encoding of + * rounding mode to arm_fprounding order (which matches the + * common AArch64 order); see ARM ARM pseudocode FPDecodeRM(). + */ +static const uint8_t fp_decode_rm[] = { + FPROUNDING_TIEAWAY, + FPROUNDING_TIEEVEN, + FPROUNDING_POSINF, + FPROUNDING_NEGINF, +}; + +static bool trans_VRINT(DisasContext *s, arg_VRINT *a) +{ + uint32_t rd, rm; + bool dp = a->dp; + TCGv_ptr fpst; + TCGv_i32 tcg_rmode; + int rounding = fp_decode_rm[a->rm]; + + if (!dc_isar_feature(aa32_vrint, s)) { + return false; + } + + if (dp && !dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (dp && !dc_isar_feature(aa32_simd_r32, s) && + ((a->vm | a->vd) & 0x10)) { + return false; + } + + rd = a->vd; + rm = a->vm; + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(0); + + tcg_rmode = tcg_const_i32(arm_rmode_to_sf(rounding)); + gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); + + if (dp) { + TCGv_i64 tcg_op; + TCGv_i64 tcg_res; + tcg_op = tcg_temp_new_i64(); + tcg_res = tcg_temp_new_i64(); + neon_load_reg64(tcg_op, rm); + gen_helper_rintd(tcg_res, tcg_op, fpst); + neon_store_reg64(tcg_res, rd); + tcg_temp_free_i64(tcg_op); + tcg_temp_free_i64(tcg_res); + } else { + TCGv_i32 tcg_op; + TCGv_i32 tcg_res; + tcg_op = tcg_temp_new_i32(); + tcg_res = tcg_temp_new_i32(); + neon_load_reg32(tcg_op, rm); + gen_helper_rints(tcg_res, tcg_op, fpst); + neon_store_reg32(tcg_res, rd); + tcg_temp_free_i32(tcg_op); + tcg_temp_free_i32(tcg_res); + } + + gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); + tcg_temp_free_i32(tcg_rmode); + + tcg_temp_free_ptr(fpst); + return true; +} + +static bool trans_VCVT(DisasContext *s, arg_VCVT *a) +{ + uint32_t rd, rm; + bool dp = a->dp; + TCGv_ptr fpst; + TCGv_i32 tcg_rmode, tcg_shift; + int rounding = fp_decode_rm[a->rm]; + bool is_signed = a->op; + + if (!dc_isar_feature(aa32_vcvt_dr, s)) { + return false; + } + + if (dp && !dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (dp && !dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { + return false; + } + + rd = a->vd; + rm = a->vm; + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(0); + + tcg_shift = tcg_const_i32(0); + + tcg_rmode = tcg_const_i32(arm_rmode_to_sf(rounding)); + gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); + + if (dp) { + TCGv_i64 tcg_double, tcg_res; + TCGv_i32 tcg_tmp; + tcg_double = tcg_temp_new_i64(); + tcg_res = tcg_temp_new_i64(); + tcg_tmp = tcg_temp_new_i32(); + neon_load_reg64(tcg_double, rm); + if (is_signed) { + gen_helper_vfp_tosld(tcg_res, tcg_double, tcg_shift, fpst); + } else { + gen_helper_vfp_tould(tcg_res, tcg_double, tcg_shift, fpst); + } + tcg_gen_extrl_i64_i32(tcg_tmp, tcg_res); + neon_store_reg32(tcg_tmp, rd); + tcg_temp_free_i32(tcg_tmp); + tcg_temp_free_i64(tcg_res); + tcg_temp_free_i64(tcg_double); + } else { + TCGv_i32 tcg_single, tcg_res; + tcg_single = tcg_temp_new_i32(); + tcg_res = tcg_temp_new_i32(); + neon_load_reg32(tcg_single, rm); + if (is_signed) { + gen_helper_vfp_tosls(tcg_res, tcg_single, tcg_shift, fpst); + } else { + gen_helper_vfp_touls(tcg_res, tcg_single, tcg_shift, fpst); + } + neon_store_reg32(tcg_res, rd); + tcg_temp_free_i32(tcg_res); + tcg_temp_free_i32(tcg_single); + } + + gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); + tcg_temp_free_i32(tcg_rmode); + + tcg_temp_free_i32(tcg_shift); + + tcg_temp_free_ptr(fpst); + + return true; +} + +static bool trans_VMOV_to_gp(DisasContext *s, arg_VMOV_to_gp *a) +{ + /* VMOV scalar to general purpose register */ + TCGv_i32 tmp; + int pass; + uint32_t offset; + + /* SIZE == 2 is a VFP instruction; otherwise NEON. */ + if (a->size == 2 + ? !dc_isar_feature(aa32_fpsp_v2, s) + : !arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vn & 0x10)) { + return false; + } + + offset = a->index << a->size; + pass = extract32(offset, 2, 1); + offset = extract32(offset, 0, 2) * 8; + + if (!vfp_access_check(s)) { + return true; + } + + tmp = neon_load_reg(a->vn, pass); + switch (a->size) { + case 0: + if (offset) { + tcg_gen_shri_i32(tmp, tmp, offset); + } + if (a->u) { + gen_uxtb(tmp); + } else { + gen_sxtb(tmp); + } + break; + case 1: + if (a->u) { + if (offset) { + tcg_gen_shri_i32(tmp, tmp, 16); + } else { + gen_uxth(tmp); + } + } else { + if (offset) { + tcg_gen_sari_i32(tmp, tmp, 16); + } else { + gen_sxth(tmp); + } + } + break; + case 2: + break; + } + store_reg(s, a->rt, tmp); + + return true; +} + +static bool trans_VMOV_from_gp(DisasContext *s, arg_VMOV_from_gp *a) +{ + /* VMOV general purpose register to scalar */ + TCGv_i32 tmp, tmp2; + int pass; + uint32_t offset; + + /* SIZE == 2 is a VFP instruction; otherwise NEON. */ + if (a->size == 2 + ? !dc_isar_feature(aa32_fpsp_v2, s) + : !arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vn & 0x10)) { + return false; + } + + offset = a->index << a->size; + pass = extract32(offset, 2, 1); + offset = extract32(offset, 0, 2) * 8; + + if (!vfp_access_check(s)) { + return true; + } + + tmp = load_reg(s, a->rt); + switch (a->size) { + case 0: + tmp2 = neon_load_reg(a->vn, pass); + tcg_gen_deposit_i32(tmp, tmp2, tmp, offset, 8); + tcg_temp_free_i32(tmp2); + break; + case 1: + tmp2 = neon_load_reg(a->vn, pass); + tcg_gen_deposit_i32(tmp, tmp2, tmp, offset, 16); + tcg_temp_free_i32(tmp2); + break; + case 2: + break; + } + neon_store_reg(a->vn, pass, tmp); + + return true; +} + +static bool trans_VDUP(DisasContext *s, arg_VDUP *a) +{ + /* VDUP (general purpose register) */ + TCGv_i32 tmp; + int size, vec_size; + + if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vn & 0x10)) { + return false; + } + + if (a->b && a->e) { + return false; + } + + if (a->q && (a->vn & 1)) { + return false; + } + + vec_size = a->q ? 16 : 8; + if (a->b) { + size = 0; + } else if (a->e) { + size = 1; + } else { + size = 2; + } + + if (!vfp_access_check(s)) { + return true; + } + + tmp = load_reg(s, a->rt); + tcg_gen_gvec_dup_i32(size, neon_reg_offset(a->vn, 0), + vec_size, vec_size, tmp); + tcg_temp_free_i32(tmp); + + return true; +} + +static bool trans_VMSR_VMRS(DisasContext *s, arg_VMSR_VMRS *a) +{ + TCGv_i32 tmp; + bool ignore_vfp_enabled = false; + + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + if (arm_dc_feature(s, ARM_FEATURE_M)) { + /* + * The only M-profile VFP vmrs/vmsr sysreg is FPSCR. + * Accesses to R15 are UNPREDICTABLE; we choose to undef. + * (FPSCR -> r15 is a special case which writes to the PSR flags.) + */ + if (a->rt == 15 && (!a->l || a->reg != ARM_VFP_FPSCR)) { + return false; + } + } + + switch (a->reg) { + case ARM_VFP_FPSID: + /* + * VFPv2 allows access to FPSID from userspace; VFPv3 restricts + * all ID registers to privileged access only. + */ + if (IS_USER(s) && dc_isar_feature(aa32_fpsp_v3, s)) { + return false; + } + ignore_vfp_enabled = true; + break; + case ARM_VFP_MVFR0: + case ARM_VFP_MVFR1: + if (IS_USER(s) || !arm_dc_feature(s, ARM_FEATURE_MVFR)) { + return false; + } + ignore_vfp_enabled = true; + break; + case ARM_VFP_MVFR2: + if (IS_USER(s) || !arm_dc_feature(s, ARM_FEATURE_V8)) { + return false; + } + ignore_vfp_enabled = true; + break; + case ARM_VFP_FPSCR: + break; + case ARM_VFP_FPEXC: + if (IS_USER(s)) { + return false; + } + ignore_vfp_enabled = true; + break; + case ARM_VFP_FPINST: + case ARM_VFP_FPINST2: + /* Not present in VFPv3 */ + if (IS_USER(s) || dc_isar_feature(aa32_fpsp_v3, s)) { + return false; + } + break; + default: + return false; + } + + if (!full_vfp_access_check(s, ignore_vfp_enabled)) { + return true; + } + + if (a->l) { + /* VMRS, move VFP special register to gp register */ + switch (a->reg) { + case ARM_VFP_MVFR0: + case ARM_VFP_MVFR1: + case ARM_VFP_MVFR2: + case ARM_VFP_FPSID: + if (s->current_el == 1) { + TCGv_i32 tcg_reg, tcg_rt; + + gen_set_condexec(s); + gen_set_pc_im(s, s->pc_curr); + tcg_reg = tcg_const_i32(a->reg); + tcg_rt = tcg_const_i32(a->rt); + gen_helper_check_hcr_el2_trap(cpu_env, tcg_rt, tcg_reg); + tcg_temp_free_i32(tcg_reg); + tcg_temp_free_i32(tcg_rt); + } + /* fall through */ + case ARM_VFP_FPEXC: + case ARM_VFP_FPINST: + case ARM_VFP_FPINST2: + tmp = load_cpu_field(vfp.xregs[a->reg]); + break; + case ARM_VFP_FPSCR: + if (a->rt == 15) { + tmp = load_cpu_field(vfp.xregs[ARM_VFP_FPSCR]); + tcg_gen_andi_i32(tmp, tmp, 0xf0000000); + } else { + tmp = tcg_temp_new_i32(); + gen_helper_vfp_get_fpscr(tmp, cpu_env); + } + break; + default: + g_assert_not_reached(); + } + + if (a->rt == 15) { + /* Set the 4 flag bits in the CPSR. */ + gen_set_nzcv(tmp); + tcg_temp_free_i32(tmp); + } else { + store_reg(s, a->rt, tmp); + } + } else { + /* VMSR, move gp register to VFP special register */ + switch (a->reg) { + case ARM_VFP_FPSID: + case ARM_VFP_MVFR0: + case ARM_VFP_MVFR1: + case ARM_VFP_MVFR2: + /* Writes are ignored. */ + break; + case ARM_VFP_FPSCR: + tmp = load_reg(s, a->rt); + gen_helper_vfp_set_fpscr(cpu_env, tmp); + tcg_temp_free_i32(tmp); + gen_lookup_tb(s); + break; + case ARM_VFP_FPEXC: + /* + * TODO: VFP subarchitecture support. + * For now, keep the EN bit only + */ + tmp = load_reg(s, a->rt); + tcg_gen_andi_i32(tmp, tmp, 1 << 30); + store_cpu_field(tmp, vfp.xregs[a->reg]); + gen_lookup_tb(s); + break; + case ARM_VFP_FPINST: + case ARM_VFP_FPINST2: + tmp = load_reg(s, a->rt); + store_cpu_field(tmp, vfp.xregs[a->reg]); + break; + default: + g_assert_not_reached(); + } + } + + return true; +} + +static bool trans_VMOV_single(DisasContext *s, arg_VMOV_single *a) +{ + TCGv_i32 tmp; + + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + if (a->l) { + /* VFP to general purpose register */ + tmp = tcg_temp_new_i32(); + neon_load_reg32(tmp, a->vn); + if (a->rt == 15) { + /* Set the 4 flag bits in the CPSR. */ + gen_set_nzcv(tmp); + tcg_temp_free_i32(tmp); + } else { + store_reg(s, a->rt, tmp); + } + } else { + /* general purpose register to VFP */ + tmp = load_reg(s, a->rt); + neon_store_reg32(tmp, a->vn); + tcg_temp_free_i32(tmp); + } + + return true; +} + +static bool trans_VMOV_64_sp(DisasContext *s, arg_VMOV_64_sp *a) +{ + TCGv_i32 tmp; + + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + /* + * VMOV between two general-purpose registers and two single precision + * floating point registers + */ + if (!vfp_access_check(s)) { + return true; + } + + if (a->op) { + /* fpreg to gpreg */ + tmp = tcg_temp_new_i32(); + neon_load_reg32(tmp, a->vm); + store_reg(s, a->rt, tmp); + tmp = tcg_temp_new_i32(); + neon_load_reg32(tmp, a->vm + 1); + store_reg(s, a->rt2, tmp); + } else { + /* gpreg to fpreg */ + tmp = load_reg(s, a->rt); + neon_store_reg32(tmp, a->vm); + tcg_temp_free_i32(tmp); + tmp = load_reg(s, a->rt2); + neon_store_reg32(tmp, a->vm + 1); + tcg_temp_free_i32(tmp); + } + + return true; +} + +static bool trans_VMOV_64_dp(DisasContext *s, arg_VMOV_64_dp *a) +{ + TCGv_i32 tmp; + + /* + * VMOV between two general-purpose registers and one double precision + * floating point register. Note that this does not require support + * for double precision arithmetic. + */ + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + if (a->op) { + /* fpreg to gpreg */ + tmp = tcg_temp_new_i32(); + neon_load_reg32(tmp, a->vm * 2); + store_reg(s, a->rt, tmp); + tmp = tcg_temp_new_i32(); + neon_load_reg32(tmp, a->vm * 2 + 1); + store_reg(s, a->rt2, tmp); + } else { + /* gpreg to fpreg */ + tmp = load_reg(s, a->rt); + neon_store_reg32(tmp, a->vm * 2); + tcg_temp_free_i32(tmp); + tmp = load_reg(s, a->rt2); + neon_store_reg32(tmp, a->vm * 2 + 1); + tcg_temp_free_i32(tmp); + } + + return true; +} + +static bool trans_VLDR_VSTR_sp(DisasContext *s, arg_VLDR_VSTR_sp *a) +{ + uint32_t offset; + TCGv_i32 addr, tmp; + + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + offset = a->imm << 2; + if (!a->u) { + offset = -offset; + } + + /* For thumb, use of PC is UNPREDICTABLE. */ + addr = add_reg_for_lit(s, a->rn, offset); + tmp = tcg_temp_new_i32(); + if (a->l) { + gen_aa32_ld32u(s, tmp, addr, get_mem_index(s)); + neon_store_reg32(tmp, a->vd); + } else { + neon_load_reg32(tmp, a->vd); + gen_aa32_st32(s, tmp, addr, get_mem_index(s)); + } + tcg_temp_free_i32(tmp); + tcg_temp_free_i32(addr); + + return true; +} + +static bool trans_VLDR_VSTR_dp(DisasContext *s, arg_VLDR_VSTR_dp *a) +{ + uint32_t offset; + TCGv_i32 addr; + TCGv_i64 tmp; + + /* Note that this does not require support for double arithmetic. */ + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + offset = a->imm << 2; + if (!a->u) { + offset = -offset; + } + + /* For thumb, use of PC is UNPREDICTABLE. */ + addr = add_reg_for_lit(s, a->rn, offset); + tmp = tcg_temp_new_i64(); + if (a->l) { + gen_aa32_ld64(s, tmp, addr, get_mem_index(s)); + neon_store_reg64(tmp, a->vd); + } else { + neon_load_reg64(tmp, a->vd); + gen_aa32_st64(s, tmp, addr, get_mem_index(s)); + } + tcg_temp_free_i64(tmp); + tcg_temp_free_i32(addr); + + return true; +} + +static bool trans_VLDM_VSTM_sp(DisasContext *s, arg_VLDM_VSTM_sp *a) +{ + uint32_t offset; + TCGv_i32 addr, tmp; + int i, n; + + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + n = a->imm; + + if (n == 0 || (a->vd + n) > 32) { + /* + * UNPREDICTABLE cases for bad immediates: we choose to + * UNDEF to avoid generating huge numbers of TCG ops + */ + return false; + } + if (a->rn == 15 && a->w) { + /* writeback to PC is UNPREDICTABLE, we choose to UNDEF */ + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + /* For thumb, use of PC is UNPREDICTABLE. */ + addr = add_reg_for_lit(s, a->rn, 0); + if (a->p) { + /* pre-decrement */ + tcg_gen_addi_i32(addr, addr, -(a->imm << 2)); + } + + if (s->v8m_stackcheck && a->rn == 13 && a->w) { + /* + * Here 'addr' is the lowest address we will store to, + * and is either the old SP (if post-increment) or + * the new SP (if pre-decrement). For post-increment + * where the old value is below the limit and the new + * value is above, it is UNKNOWN whether the limit check + * triggers; we choose to trigger. + */ + gen_helper_v8m_stackcheck(cpu_env, addr); + } + + offset = 4; + tmp = tcg_temp_new_i32(); + for (i = 0; i < n; i++) { + if (a->l) { + /* load */ + gen_aa32_ld32u(s, tmp, addr, get_mem_index(s)); + neon_store_reg32(tmp, a->vd + i); + } else { + /* store */ + neon_load_reg32(tmp, a->vd + i); + gen_aa32_st32(s, tmp, addr, get_mem_index(s)); + } + tcg_gen_addi_i32(addr, addr, offset); + } + tcg_temp_free_i32(tmp); + if (a->w) { + /* writeback */ + if (a->p) { + offset = -offset * n; + tcg_gen_addi_i32(addr, addr, offset); + } + store_reg(s, a->rn, addr); + } else { + tcg_temp_free_i32(addr); + } + + return true; +} + +static bool trans_VLDM_VSTM_dp(DisasContext *s, arg_VLDM_VSTM_dp *a) +{ + uint32_t offset; + TCGv_i32 addr; + TCGv_i64 tmp; + int i, n; + + /* Note that this does not require support for double arithmetic. */ + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + n = a->imm >> 1; + + if (n == 0 || (a->vd + n) > 32 || n > 16) { + /* + * UNPREDICTABLE cases for bad immediates: we choose to + * UNDEF to avoid generating huge numbers of TCG ops + */ + return false; + } + if (a->rn == 15 && a->w) { + /* writeback to PC is UNPREDICTABLE, we choose to UNDEF */ + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd + n) > 16) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + /* For thumb, use of PC is UNPREDICTABLE. */ + addr = add_reg_for_lit(s, a->rn, 0); + if (a->p) { + /* pre-decrement */ + tcg_gen_addi_i32(addr, addr, -(a->imm << 2)); + } + + if (s->v8m_stackcheck && a->rn == 13 && a->w) { + /* + * Here 'addr' is the lowest address we will store to, + * and is either the old SP (if post-increment) or + * the new SP (if pre-decrement). For post-increment + * where the old value is below the limit and the new + * value is above, it is UNKNOWN whether the limit check + * triggers; we choose to trigger. + */ + gen_helper_v8m_stackcheck(cpu_env, addr); + } + + offset = 8; + tmp = tcg_temp_new_i64(); + for (i = 0; i < n; i++) { + if (a->l) { + /* load */ + gen_aa32_ld64(s, tmp, addr, get_mem_index(s)); + neon_store_reg64(tmp, a->vd + i); + } else { + /* store */ + neon_load_reg64(tmp, a->vd + i); + gen_aa32_st64(s, tmp, addr, get_mem_index(s)); + } + tcg_gen_addi_i32(addr, addr, offset); + } + tcg_temp_free_i64(tmp); + if (a->w) { + /* writeback */ + if (a->p) { + offset = -offset * n; + } else if (a->imm & 1) { + offset = 4; + } else { + offset = 0; + } + + if (offset != 0) { + tcg_gen_addi_i32(addr, addr, offset); + } + store_reg(s, a->rn, addr); + } else { + tcg_temp_free_i32(addr); + } + + return true; +} + +/* + * Types for callbacks for do_vfp_3op_sp() and do_vfp_3op_dp(). + * The callback should emit code to write a value to vd. If + * do_vfp_3op_{sp,dp}() was passed reads_vd then the TCGv vd + * will contain the old value of the relevant VFP register; + * otherwise it must be written to only. + */ +typedef void VFPGen3OpSPFn(TCGv_i32 vd, + TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst); +typedef void VFPGen3OpDPFn(TCGv_i64 vd, + TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst); + +/* + * Types for callbacks for do_vfp_2op_sp() and do_vfp_2op_dp(). + * The callback should emit code to write a value to vd (which + * should be written to only). + */ +typedef void VFPGen2OpSPFn(TCGv_i32 vd, TCGv_i32 vm); +typedef void VFPGen2OpDPFn(TCGv_i64 vd, TCGv_i64 vm); + +/* + * Return true if the specified S reg is in a scalar bank + * (ie if it is s0..s7) + */ +static inline bool vfp_sreg_is_scalar(int reg) +{ + return (reg & 0x18) == 0; +} + +/* + * Return true if the specified D reg is in a scalar bank + * (ie if it is d0..d3 or d16..d19) + */ +static inline bool vfp_dreg_is_scalar(int reg) +{ + return (reg & 0xc) == 0; +} + +/* + * Advance the S reg number forwards by delta within its bank + * (ie increment the low 3 bits but leave the rest the same) + */ +static inline int vfp_advance_sreg(int reg, int delta) +{ + return ((reg + delta) & 0x7) | (reg & ~0x7); +} + +/* + * Advance the D reg number forwards by delta within its bank + * (ie increment the low 2 bits but leave the rest the same) + */ +static inline int vfp_advance_dreg(int reg, int delta) +{ + return ((reg + delta) & 0x3) | (reg & ~0x3); +} + +/* + * Perform a 3-operand VFP data processing instruction. fn is the + * callback to do the actual operation; this function deals with the + * code to handle looping around for VFP vector processing. + */ +static bool do_vfp_3op_sp(DisasContext *s, VFPGen3OpSPFn *fn, + int vd, int vn, int vm, bool reads_vd) +{ + uint32_t delta_m = 0; + uint32_t delta_d = 0; + int veclen = s->vec_len; + TCGv_i32 f0, f1, fd; + TCGv_ptr fpst; + + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + if (!dc_isar_feature(aa32_fpshvec, s) && + (veclen != 0 || s->vec_stride != 0)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + if (veclen > 0) { + /* Figure out what type of vector operation this is. */ + if (vfp_sreg_is_scalar(vd)) { + /* scalar */ + veclen = 0; + } else { + delta_d = s->vec_stride + 1; + + if (vfp_sreg_is_scalar(vm)) { + /* mixed scalar/vector */ + delta_m = 0; + } else { + /* vector */ + delta_m = delta_d; + } + } + } + + f0 = tcg_temp_new_i32(); + f1 = tcg_temp_new_i32(); + fd = tcg_temp_new_i32(); + fpst = get_fpstatus_ptr(0); + + neon_load_reg32(f0, vn); + neon_load_reg32(f1, vm); + + for (;;) { + if (reads_vd) { + neon_load_reg32(fd, vd); + } + fn(fd, f0, f1, fpst); + neon_store_reg32(fd, vd); + + if (veclen == 0) { + break; + } + + /* Set up the operands for the next iteration */ + veclen--; + vd = vfp_advance_sreg(vd, delta_d); + vn = vfp_advance_sreg(vn, delta_d); + neon_load_reg32(f0, vn); + if (delta_m) { + vm = vfp_advance_sreg(vm, delta_m); + neon_load_reg32(f1, vm); + } + } + + tcg_temp_free_i32(f0); + tcg_temp_free_i32(f1); + tcg_temp_free_i32(fd); + tcg_temp_free_ptr(fpst); + + return true; +} + +static bool do_vfp_3op_dp(DisasContext *s, VFPGen3OpDPFn *fn, + int vd, int vn, int vm, bool reads_vd) +{ + uint32_t delta_m = 0; + uint32_t delta_d = 0; + int veclen = s->vec_len; + TCGv_i64 f0, f1, fd; + TCGv_ptr fpst; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && ((vd | vn | vm) & 0x10)) { + return false; + } + + if (!dc_isar_feature(aa32_fpshvec, s) && + (veclen != 0 || s->vec_stride != 0)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + if (veclen > 0) { + /* Figure out what type of vector operation this is. */ + if (vfp_dreg_is_scalar(vd)) { + /* scalar */ + veclen = 0; + } else { + delta_d = (s->vec_stride >> 1) + 1; + + if (vfp_dreg_is_scalar(vm)) { + /* mixed scalar/vector */ + delta_m = 0; + } else { + /* vector */ + delta_m = delta_d; + } + } + } + + f0 = tcg_temp_new_i64(); + f1 = tcg_temp_new_i64(); + fd = tcg_temp_new_i64(); + fpst = get_fpstatus_ptr(0); + + neon_load_reg64(f0, vn); + neon_load_reg64(f1, vm); + + for (;;) { + if (reads_vd) { + neon_load_reg64(fd, vd); + } + fn(fd, f0, f1, fpst); + neon_store_reg64(fd, vd); + + if (veclen == 0) { + break; + } + /* Set up the operands for the next iteration */ + veclen--; + vd = vfp_advance_dreg(vd, delta_d); + vn = vfp_advance_dreg(vn, delta_d); + neon_load_reg64(f0, vn); + if (delta_m) { + vm = vfp_advance_dreg(vm, delta_m); + neon_load_reg64(f1, vm); + } + } + + tcg_temp_free_i64(f0); + tcg_temp_free_i64(f1); + tcg_temp_free_i64(fd); + tcg_temp_free_ptr(fpst); + + return true; +} + +static bool do_vfp_2op_sp(DisasContext *s, VFPGen2OpSPFn *fn, int vd, int vm) +{ + uint32_t delta_m = 0; + uint32_t delta_d = 0; + int veclen = s->vec_len; + TCGv_i32 f0, fd; + + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + if (!dc_isar_feature(aa32_fpshvec, s) && + (veclen != 0 || s->vec_stride != 0)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + if (veclen > 0) { + /* Figure out what type of vector operation this is. */ + if (vfp_sreg_is_scalar(vd)) { + /* scalar */ + veclen = 0; + } else { + delta_d = s->vec_stride + 1; + + if (vfp_sreg_is_scalar(vm)) { + /* mixed scalar/vector */ + delta_m = 0; + } else { + /* vector */ + delta_m = delta_d; + } + } + } + + f0 = tcg_temp_new_i32(); + fd = tcg_temp_new_i32(); + + neon_load_reg32(f0, vm); + + for (;;) { + fn(fd, f0); + neon_store_reg32(fd, vd); + + if (veclen == 0) { + break; + } + + if (delta_m == 0) { + /* single source one-many */ + while (veclen--) { + vd = vfp_advance_sreg(vd, delta_d); + neon_store_reg32(fd, vd); + } + break; + } + + /* Set up the operands for the next iteration */ + veclen--; + vd = vfp_advance_sreg(vd, delta_d); + vm = vfp_advance_sreg(vm, delta_m); + neon_load_reg32(f0, vm); + } + + tcg_temp_free_i32(f0); + tcg_temp_free_i32(fd); + + return true; +} + +static bool do_vfp_2op_dp(DisasContext *s, VFPGen2OpDPFn *fn, int vd, int vm) +{ + uint32_t delta_m = 0; + uint32_t delta_d = 0; + int veclen = s->vec_len; + TCGv_i64 f0, fd; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist */ + if (!dc_isar_feature(aa32_simd_r32, s) && ((vd | vm) & 0x10)) { + return false; + } + + if (!dc_isar_feature(aa32_fpshvec, s) && + (veclen != 0 || s->vec_stride != 0)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + if (veclen > 0) { + /* Figure out what type of vector operation this is. */ + if (vfp_dreg_is_scalar(vd)) { + /* scalar */ + veclen = 0; + } else { + delta_d = (s->vec_stride >> 1) + 1; + + if (vfp_dreg_is_scalar(vm)) { + /* mixed scalar/vector */ + delta_m = 0; + } else { + /* vector */ + delta_m = delta_d; + } + } + } + + f0 = tcg_temp_new_i64(); + fd = tcg_temp_new_i64(); + + neon_load_reg64(f0, vm); + + for (;;) { + fn(fd, f0); + neon_store_reg64(fd, vd); + + if (veclen == 0) { + break; + } + + if (delta_m == 0) { + /* single source one-many */ + while (veclen--) { + vd = vfp_advance_dreg(vd, delta_d); + neon_store_reg64(fd, vd); + } + break; + } + + /* Set up the operands for the next iteration */ + veclen--; + vd = vfp_advance_dreg(vd, delta_d); + vd = vfp_advance_dreg(vm, delta_m); + neon_load_reg64(f0, vm); + } + + tcg_temp_free_i64(f0); + tcg_temp_free_i64(fd); + + return true; +} + +static void gen_VMLA_sp(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst) +{ + /* Note that order of inputs to the add matters for NaNs */ + TCGv_i32 tmp = tcg_temp_new_i32(); + + gen_helper_vfp_muls(tmp, vn, vm, fpst); + gen_helper_vfp_adds(vd, vd, tmp, fpst); + tcg_temp_free_i32(tmp); +} + +static bool trans_VMLA_sp(DisasContext *s, arg_VMLA_sp *a) +{ + return do_vfp_3op_sp(s, gen_VMLA_sp, a->vd, a->vn, a->vm, true); +} + +static void gen_VMLA_dp(TCGv_i64 vd, TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst) +{ + /* Note that order of inputs to the add matters for NaNs */ + TCGv_i64 tmp = tcg_temp_new_i64(); + + gen_helper_vfp_muld(tmp, vn, vm, fpst); + gen_helper_vfp_addd(vd, vd, tmp, fpst); + tcg_temp_free_i64(tmp); +} + +static bool trans_VMLA_dp(DisasContext *s, arg_VMLA_dp *a) +{ + return do_vfp_3op_dp(s, gen_VMLA_dp, a->vd, a->vn, a->vm, true); +} + +static void gen_VMLS_sp(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst) +{ + /* + * VMLS: vd = vd + -(vn * vm) + * Note that order of inputs to the add matters for NaNs. + */ + TCGv_i32 tmp = tcg_temp_new_i32(); + + gen_helper_vfp_muls(tmp, vn, vm, fpst); + gen_helper_vfp_negs(tmp, tmp); + gen_helper_vfp_adds(vd, vd, tmp, fpst); + tcg_temp_free_i32(tmp); +} + +static bool trans_VMLS_sp(DisasContext *s, arg_VMLS_sp *a) +{ + return do_vfp_3op_sp(s, gen_VMLS_sp, a->vd, a->vn, a->vm, true); +} + +static void gen_VMLS_dp(TCGv_i64 vd, TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst) +{ + /* + * VMLS: vd = vd + -(vn * vm) + * Note that order of inputs to the add matters for NaNs. + */ + TCGv_i64 tmp = tcg_temp_new_i64(); + + gen_helper_vfp_muld(tmp, vn, vm, fpst); + gen_helper_vfp_negd(tmp, tmp); + gen_helper_vfp_addd(vd, vd, tmp, fpst); + tcg_temp_free_i64(tmp); +} + +static bool trans_VMLS_dp(DisasContext *s, arg_VMLS_dp *a) +{ + return do_vfp_3op_dp(s, gen_VMLS_dp, a->vd, a->vn, a->vm, true); +} + +static void gen_VNMLS_sp(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst) +{ + /* + * VNMLS: -fd + (fn * fm) + * Note that it isn't valid to replace (-A + B) with (B - A) or similar + * plausible looking simplifications because this will give wrong results + * for NaNs. + */ + TCGv_i32 tmp = tcg_temp_new_i32(); + + gen_helper_vfp_muls(tmp, vn, vm, fpst); + gen_helper_vfp_negs(vd, vd); + gen_helper_vfp_adds(vd, vd, tmp, fpst); + tcg_temp_free_i32(tmp); +} + +static bool trans_VNMLS_sp(DisasContext *s, arg_VNMLS_sp *a) +{ + return do_vfp_3op_sp(s, gen_VNMLS_sp, a->vd, a->vn, a->vm, true); +} + +static void gen_VNMLS_dp(TCGv_i64 vd, TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst) +{ + /* + * VNMLS: -fd + (fn * fm) + * Note that it isn't valid to replace (-A + B) with (B - A) or similar + * plausible looking simplifications because this will give wrong results + * for NaNs. + */ + TCGv_i64 tmp = tcg_temp_new_i64(); + + gen_helper_vfp_muld(tmp, vn, vm, fpst); + gen_helper_vfp_negd(vd, vd); + gen_helper_vfp_addd(vd, vd, tmp, fpst); + tcg_temp_free_i64(tmp); +} + +static bool trans_VNMLS_dp(DisasContext *s, arg_VNMLS_dp *a) +{ + return do_vfp_3op_dp(s, gen_VNMLS_dp, a->vd, a->vn, a->vm, true); +} + +static void gen_VNMLA_sp(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst) +{ + /* VNMLA: -fd + -(fn * fm) */ + TCGv_i32 tmp = tcg_temp_new_i32(); + + gen_helper_vfp_muls(tmp, vn, vm, fpst); + gen_helper_vfp_negs(tmp, tmp); + gen_helper_vfp_negs(vd, vd); + gen_helper_vfp_adds(vd, vd, tmp, fpst); + tcg_temp_free_i32(tmp); +} + +static bool trans_VNMLA_sp(DisasContext *s, arg_VNMLA_sp *a) +{ + return do_vfp_3op_sp(s, gen_VNMLA_sp, a->vd, a->vn, a->vm, true); +} + +static void gen_VNMLA_dp(TCGv_i64 vd, TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst) +{ + /* VNMLA: -fd + (fn * fm) */ + TCGv_i64 tmp = tcg_temp_new_i64(); + + gen_helper_vfp_muld(tmp, vn, vm, fpst); + gen_helper_vfp_negd(tmp, tmp); + gen_helper_vfp_negd(vd, vd); + gen_helper_vfp_addd(vd, vd, tmp, fpst); + tcg_temp_free_i64(tmp); +} + +static bool trans_VNMLA_dp(DisasContext *s, arg_VNMLA_dp *a) +{ + return do_vfp_3op_dp(s, gen_VNMLA_dp, a->vd, a->vn, a->vm, true); +} + +static bool trans_VMUL_sp(DisasContext *s, arg_VMUL_sp *a) +{ + return do_vfp_3op_sp(s, gen_helper_vfp_muls, a->vd, a->vn, a->vm, false); +} + +static bool trans_VMUL_dp(DisasContext *s, arg_VMUL_dp *a) +{ + return do_vfp_3op_dp(s, gen_helper_vfp_muld, a->vd, a->vn, a->vm, false); +} + +static void gen_VNMUL_sp(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst) +{ + /* VNMUL: -(fn * fm) */ + gen_helper_vfp_muls(vd, vn, vm, fpst); + gen_helper_vfp_negs(vd, vd); +} + +static bool trans_VNMUL_sp(DisasContext *s, arg_VNMUL_sp *a) +{ + return do_vfp_3op_sp(s, gen_VNMUL_sp, a->vd, a->vn, a->vm, false); +} + +static void gen_VNMUL_dp(TCGv_i64 vd, TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst) +{ + /* VNMUL: -(fn * fm) */ + gen_helper_vfp_muld(vd, vn, vm, fpst); + gen_helper_vfp_negd(vd, vd); +} + +static bool trans_VNMUL_dp(DisasContext *s, arg_VNMUL_dp *a) +{ + return do_vfp_3op_dp(s, gen_VNMUL_dp, a->vd, a->vn, a->vm, false); +} + +static bool trans_VADD_sp(DisasContext *s, arg_VADD_sp *a) +{ + return do_vfp_3op_sp(s, gen_helper_vfp_adds, a->vd, a->vn, a->vm, false); +} + +static bool trans_VADD_dp(DisasContext *s, arg_VADD_dp *a) +{ + return do_vfp_3op_dp(s, gen_helper_vfp_addd, a->vd, a->vn, a->vm, false); +} + +static bool trans_VSUB_sp(DisasContext *s, arg_VSUB_sp *a) +{ + return do_vfp_3op_sp(s, gen_helper_vfp_subs, a->vd, a->vn, a->vm, false); +} + +static bool trans_VSUB_dp(DisasContext *s, arg_VSUB_dp *a) +{ + return do_vfp_3op_dp(s, gen_helper_vfp_subd, a->vd, a->vn, a->vm, false); +} + +static bool trans_VDIV_sp(DisasContext *s, arg_VDIV_sp *a) +{ + return do_vfp_3op_sp(s, gen_helper_vfp_divs, a->vd, a->vn, a->vm, false); +} + +static bool trans_VDIV_dp(DisasContext *s, arg_VDIV_dp *a) +{ + return do_vfp_3op_dp(s, gen_helper_vfp_divd, a->vd, a->vn, a->vm, false); +} + +static bool trans_VMINNM_sp(DisasContext *s, arg_VMINNM_sp *a) +{ + if (!dc_isar_feature(aa32_vminmaxnm, s)) { + return false; + } + return do_vfp_3op_sp(s, gen_helper_vfp_minnums, + a->vd, a->vn, a->vm, false); +} + +static bool trans_VMAXNM_sp(DisasContext *s, arg_VMAXNM_sp *a) +{ + if (!dc_isar_feature(aa32_vminmaxnm, s)) { + return false; + } + return do_vfp_3op_sp(s, gen_helper_vfp_maxnums, + a->vd, a->vn, a->vm, false); +} + +static bool trans_VMINNM_dp(DisasContext *s, arg_VMINNM_dp *a) +{ + if (!dc_isar_feature(aa32_vminmaxnm, s)) { + return false; + } + return do_vfp_3op_dp(s, gen_helper_vfp_minnumd, + a->vd, a->vn, a->vm, false); +} + +static bool trans_VMAXNM_dp(DisasContext *s, arg_VMAXNM_dp *a) +{ + if (!dc_isar_feature(aa32_vminmaxnm, s)) { + return false; + } + return do_vfp_3op_dp(s, gen_helper_vfp_maxnumd, + a->vd, a->vn, a->vm, false); +} + +static bool do_vfm_sp(DisasContext *s, arg_VFMA_sp *a, bool neg_n, bool neg_d) +{ + /* + * VFNMA : fd = muladd(-fd, fn, fm) + * VFNMS : fd = muladd(-fd, -fn, fm) + * VFMA : fd = muladd( fd, fn, fm) + * VFMS : fd = muladd( fd, -fn, fm) + * + * These are fused multiply-add, and must be done as one floating + * point operation with no rounding between the multiplication and + * addition steps. NB that doing the negations here as separate + * steps is correct : an input NaN should come out with its sign + * bit flipped if it is a negated-input. + */ + TCGv_ptr fpst; + TCGv_i32 vn, vm, vd; + + /* + * Present in VFPv4 only. + * Note that we can't rely on the SIMDFMAC check alone, because + * in a Neon-no-VFP core that ID register field will be non-zero. + */ + if (!dc_isar_feature(aa32_simdfmac, s) || + !dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + /* + * In v7A, UNPREDICTABLE with non-zero vector length/stride; from + * v8A, must UNDEF. We choose to UNDEF for both v7A and v8A. + */ + if (s->vec_len != 0 || s->vec_stride != 0) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + vn = tcg_temp_new_i32(); + vm = tcg_temp_new_i32(); + vd = tcg_temp_new_i32(); + + neon_load_reg32(vn, a->vn); + neon_load_reg32(vm, a->vm); + if (neg_n) { + /* VFNMS, VFMS */ + gen_helper_vfp_negs(vn, vn); + } + neon_load_reg32(vd, a->vd); + if (neg_d) { + /* VFNMA, VFNMS */ + gen_helper_vfp_negs(vd, vd); + } + fpst = get_fpstatus_ptr(0); + gen_helper_vfp_muladds(vd, vn, vm, vd, fpst); + neon_store_reg32(vd, a->vd); + + tcg_temp_free_ptr(fpst); + tcg_temp_free_i32(vn); + tcg_temp_free_i32(vm); + tcg_temp_free_i32(vd); + + return true; +} + +static bool trans_VFMA_sp(DisasContext *s, arg_VFMA_sp *a) +{ + return do_vfm_sp(s, a, false, false); +} + +static bool trans_VFMS_sp(DisasContext *s, arg_VFMS_sp *a) +{ + return do_vfm_sp(s, a, true, false); +} + +static bool trans_VFNMA_sp(DisasContext *s, arg_VFNMA_sp *a) +{ + return do_vfm_sp(s, a, false, true); +} + +static bool trans_VFNMS_sp(DisasContext *s, arg_VFNMS_sp *a) +{ + return do_vfm_sp(s, a, true, true); +} + +static bool do_vfm_dp(DisasContext *s, arg_VFMA_dp *a, bool neg_n, bool neg_d) +{ + /* + * VFNMA : fd = muladd(-fd, fn, fm) + * VFNMS : fd = muladd(-fd, -fn, fm) + * VFMA : fd = muladd( fd, fn, fm) + * VFMS : fd = muladd( fd, -fn, fm) + * + * These are fused multiply-add, and must be done as one floating + * point operation with no rounding between the multiplication and + * addition steps. NB that doing the negations here as separate + * steps is correct : an input NaN should come out with its sign + * bit flipped if it is a negated-input. + */ + TCGv_ptr fpst; + TCGv_i64 vn, vm, vd; + + /* + * Present in VFPv4 only. + * Note that we can't rely on the SIMDFMAC check alone, because + * in a Neon-no-VFP core that ID register field will be non-zero. + */ + if (!dc_isar_feature(aa32_simdfmac, s) || + !dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + /* + * In v7A, UNPREDICTABLE with non-zero vector length/stride; from + * v8A, must UNDEF. We choose to UNDEF for both v7A and v8A. + */ + if (s->vec_len != 0 || s->vec_stride != 0) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && + ((a->vd | a->vn | a->vm) & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + vn = tcg_temp_new_i64(); + vm = tcg_temp_new_i64(); + vd = tcg_temp_new_i64(); + + neon_load_reg64(vn, a->vn); + neon_load_reg64(vm, a->vm); + if (neg_n) { + /* VFNMS, VFMS */ + gen_helper_vfp_negd(vn, vn); + } + neon_load_reg64(vd, a->vd); + if (neg_d) { + /* VFNMA, VFNMS */ + gen_helper_vfp_negd(vd, vd); + } + fpst = get_fpstatus_ptr(0); + gen_helper_vfp_muladdd(vd, vn, vm, vd, fpst); + neon_store_reg64(vd, a->vd); + + tcg_temp_free_ptr(fpst); + tcg_temp_free_i64(vn); + tcg_temp_free_i64(vm); + tcg_temp_free_i64(vd); + + return true; +} + +static bool trans_VFMA_dp(DisasContext *s, arg_VFMA_dp *a) +{ + return do_vfm_dp(s, a, false, false); +} + +static bool trans_VFMS_dp(DisasContext *s, arg_VFMS_dp *a) +{ + return do_vfm_dp(s, a, true, false); +} + +static bool trans_VFNMA_dp(DisasContext *s, arg_VFNMA_dp *a) +{ + return do_vfm_dp(s, a, false, true); +} + +static bool trans_VFNMS_dp(DisasContext *s, arg_VFNMS_dp *a) +{ + return do_vfm_dp(s, a, true, true); +} + +static bool trans_VMOV_imm_sp(DisasContext *s, arg_VMOV_imm_sp *a) +{ + uint32_t delta_d = 0; + int veclen = s->vec_len; + TCGv_i32 fd; + uint32_t vd; + + vd = a->vd; + + if (!dc_isar_feature(aa32_fpsp_v3, s)) { + return false; + } + + if (!dc_isar_feature(aa32_fpshvec, s) && + (veclen != 0 || s->vec_stride != 0)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + if (veclen > 0) { + /* Figure out what type of vector operation this is. */ + if (vfp_sreg_is_scalar(vd)) { + /* scalar */ + veclen = 0; + } else { + delta_d = s->vec_stride + 1; + } + } + + fd = tcg_const_i32(vfp_expand_imm(MO_32, a->imm)); + + for (;;) { + neon_store_reg32(fd, vd); + + if (veclen == 0) { + break; + } + + /* Set up the operands for the next iteration */ + veclen--; + vd = vfp_advance_sreg(vd, delta_d); + } + + tcg_temp_free_i32(fd); + return true; +} + +static bool trans_VMOV_imm_dp(DisasContext *s, arg_VMOV_imm_dp *a) +{ + uint32_t delta_d = 0; + int veclen = s->vec_len; + TCGv_i64 fd; + uint32_t vd; + + vd = a->vd; + + if (!dc_isar_feature(aa32_fpdp_v3, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && (vd & 0x10)) { + return false; + } + + if (!dc_isar_feature(aa32_fpshvec, s) && + (veclen != 0 || s->vec_stride != 0)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + if (veclen > 0) { + /* Figure out what type of vector operation this is. */ + if (vfp_dreg_is_scalar(vd)) { + /* scalar */ + veclen = 0; + } else { + delta_d = (s->vec_stride >> 1) + 1; + } + } + + fd = tcg_const_i64(vfp_expand_imm(MO_64, a->imm)); + + for (;;) { + neon_store_reg64(fd, vd); + + if (veclen == 0) { + break; + } + + /* Set up the operands for the next iteration */ + veclen--; + vd = vfp_advance_dreg(vd, delta_d); + } + + tcg_temp_free_i64(fd); + return true; +} + +static bool trans_VMOV_reg_sp(DisasContext *s, arg_VMOV_reg_sp *a) +{ + return do_vfp_2op_sp(s, tcg_gen_mov_i32, a->vd, a->vm); +} + +static bool trans_VMOV_reg_dp(DisasContext *s, arg_VMOV_reg_dp *a) +{ + return do_vfp_2op_dp(s, tcg_gen_mov_i64, a->vd, a->vm); +} + +static bool trans_VABS_sp(DisasContext *s, arg_VABS_sp *a) +{ + return do_vfp_2op_sp(s, gen_helper_vfp_abss, a->vd, a->vm); +} + +static bool trans_VABS_dp(DisasContext *s, arg_VABS_dp *a) +{ + return do_vfp_2op_dp(s, gen_helper_vfp_absd, a->vd, a->vm); +} + +static bool trans_VNEG_sp(DisasContext *s, arg_VNEG_sp *a) +{ + return do_vfp_2op_sp(s, gen_helper_vfp_negs, a->vd, a->vm); +} + +static bool trans_VNEG_dp(DisasContext *s, arg_VNEG_dp *a) +{ + return do_vfp_2op_dp(s, gen_helper_vfp_negd, a->vd, a->vm); +} + +static void gen_VSQRT_sp(TCGv_i32 vd, TCGv_i32 vm) +{ + gen_helper_vfp_sqrts(vd, vm, cpu_env); +} + +static bool trans_VSQRT_sp(DisasContext *s, arg_VSQRT_sp *a) +{ + return do_vfp_2op_sp(s, gen_VSQRT_sp, a->vd, a->vm); +} + +static void gen_VSQRT_dp(TCGv_i64 vd, TCGv_i64 vm) +{ + gen_helper_vfp_sqrtd(vd, vm, cpu_env); +} + +static bool trans_VSQRT_dp(DisasContext *s, arg_VSQRT_dp *a) +{ + return do_vfp_2op_dp(s, gen_VSQRT_dp, a->vd, a->vm); +} + +static bool trans_VCMP_sp(DisasContext *s, arg_VCMP_sp *a) +{ + TCGv_i32 vd, vm; + + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + /* Vm/M bits must be zero for the Z variant */ + if (a->z && a->vm != 0) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + vd = tcg_temp_new_i32(); + vm = tcg_temp_new_i32(); + + neon_load_reg32(vd, a->vd); + if (a->z) { + tcg_gen_movi_i32(vm, 0); + } else { + neon_load_reg32(vm, a->vm); + } + + if (a->e) { + gen_helper_vfp_cmpes(vd, vm, cpu_env); + } else { + gen_helper_vfp_cmps(vd, vm, cpu_env); + } + + tcg_temp_free_i32(vd); + tcg_temp_free_i32(vm); + + return true; +} + +static bool trans_VCMP_dp(DisasContext *s, arg_VCMP_dp *a) +{ + TCGv_i64 vd, vm; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + /* Vm/M bits must be zero for the Z variant */ + if (a->z && a->vm != 0) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + vd = tcg_temp_new_i64(); + vm = tcg_temp_new_i64(); + + neon_load_reg64(vd, a->vd); + if (a->z) { + tcg_gen_movi_i64(vm, 0); + } else { + neon_load_reg64(vm, a->vm); + } + + if (a->e) { + gen_helper_vfp_cmped(vd, vm, cpu_env); + } else { + gen_helper_vfp_cmpd(vd, vm, cpu_env); + } + + tcg_temp_free_i64(vd); + tcg_temp_free_i64(vm); + + return true; +} + +static bool trans_VCVT_f32_f16(DisasContext *s, arg_VCVT_f32_f16 *a) +{ + TCGv_ptr fpst; + TCGv_i32 ahp_mode; + TCGv_i32 tmp; + + if (!dc_isar_feature(aa32_fp16_spconv, s)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(false); + ahp_mode = get_ahp_flag(); + tmp = tcg_temp_new_i32(); + /* The T bit tells us if we want the low or high 16 bits of Vm */ + tcg_gen_ld16u_i32(tmp, cpu_env, vfp_f16_offset(a->vm, a->t)); + gen_helper_vfp_fcvt_f16_to_f32(tmp, tmp, fpst, ahp_mode); + neon_store_reg32(tmp, a->vd); + tcg_temp_free_i32(ahp_mode); + tcg_temp_free_ptr(fpst); + tcg_temp_free_i32(tmp); + return true; +} + +static bool trans_VCVT_f64_f16(DisasContext *s, arg_VCVT_f64_f16 *a) +{ + TCGv_ptr fpst; + TCGv_i32 ahp_mode; + TCGv_i32 tmp; + TCGv_i64 vd; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + if (!dc_isar_feature(aa32_fp16_dpconv, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(false); + ahp_mode = get_ahp_flag(); + tmp = tcg_temp_new_i32(); + /* The T bit tells us if we want the low or high 16 bits of Vm */ + tcg_gen_ld16u_i32(tmp, cpu_env, vfp_f16_offset(a->vm, a->t)); + vd = tcg_temp_new_i64(); + gen_helper_vfp_fcvt_f16_to_f64(vd, tmp, fpst, ahp_mode); + neon_store_reg64(vd, a->vd); + tcg_temp_free_i32(ahp_mode); + tcg_temp_free_ptr(fpst); + tcg_temp_free_i32(tmp); + tcg_temp_free_i64(vd); + return true; +} + +static bool trans_VCVT_f16_f32(DisasContext *s, arg_VCVT_f16_f32 *a) +{ + TCGv_ptr fpst; + TCGv_i32 ahp_mode; + TCGv_i32 tmp; + + if (!dc_isar_feature(aa32_fp16_spconv, s)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(false); + ahp_mode = get_ahp_flag(); + tmp = tcg_temp_new_i32(); + + neon_load_reg32(tmp, a->vm); + gen_helper_vfp_fcvt_f32_to_f16(tmp, tmp, fpst, ahp_mode); + tcg_gen_st16_i32(tmp, cpu_env, vfp_f16_offset(a->vd, a->t)); + tcg_temp_free_i32(ahp_mode); + tcg_temp_free_ptr(fpst); + tcg_temp_free_i32(tmp); + return true; +} + +static bool trans_VCVT_f16_f64(DisasContext *s, arg_VCVT_f16_f64 *a) +{ + TCGv_ptr fpst; + TCGv_i32 ahp_mode; + TCGv_i32 tmp; + TCGv_i64 vm; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + if (!dc_isar_feature(aa32_fp16_dpconv, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(false); + ahp_mode = get_ahp_flag(); + tmp = tcg_temp_new_i32(); + vm = tcg_temp_new_i64(); + + neon_load_reg64(vm, a->vm); + gen_helper_vfp_fcvt_f64_to_f16(tmp, vm, fpst, ahp_mode); + tcg_temp_free_i64(vm); + tcg_gen_st16_i32(tmp, cpu_env, vfp_f16_offset(a->vd, a->t)); + tcg_temp_free_i32(ahp_mode); + tcg_temp_free_ptr(fpst); + tcg_temp_free_i32(tmp); + return true; +} + +static bool trans_VRINTR_sp(DisasContext *s, arg_VRINTR_sp *a) +{ + TCGv_ptr fpst; + TCGv_i32 tmp; + + if (!dc_isar_feature(aa32_vrint, s)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + tmp = tcg_temp_new_i32(); + neon_load_reg32(tmp, a->vm); + fpst = get_fpstatus_ptr(false); + gen_helper_rints(tmp, tmp, fpst); + neon_store_reg32(tmp, a->vd); + tcg_temp_free_ptr(fpst); + tcg_temp_free_i32(tmp); + return true; +} + +static bool trans_VRINTR_dp(DisasContext *s, arg_VRINTR_dp *a) +{ + TCGv_ptr fpst; + TCGv_i64 tmp; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + if (!dc_isar_feature(aa32_vrint, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + tmp = tcg_temp_new_i64(); + neon_load_reg64(tmp, a->vm); + fpst = get_fpstatus_ptr(false); + gen_helper_rintd(tmp, tmp, fpst); + neon_store_reg64(tmp, a->vd); + tcg_temp_free_ptr(fpst); + tcg_temp_free_i64(tmp); + return true; +} + +static bool trans_VRINTZ_sp(DisasContext *s, arg_VRINTZ_sp *a) +{ + TCGv_ptr fpst; + TCGv_i32 tmp; + TCGv_i32 tcg_rmode; + + if (!dc_isar_feature(aa32_vrint, s)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + tmp = tcg_temp_new_i32(); + neon_load_reg32(tmp, a->vm); + fpst = get_fpstatus_ptr(false); + tcg_rmode = tcg_const_i32(float_round_to_zero); + gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); + gen_helper_rints(tmp, tmp, fpst); + gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); + neon_store_reg32(tmp, a->vd); + tcg_temp_free_ptr(fpst); + tcg_temp_free_i32(tcg_rmode); + tcg_temp_free_i32(tmp); + return true; +} + +static bool trans_VRINTZ_dp(DisasContext *s, arg_VRINTZ_dp *a) +{ + TCGv_ptr fpst; + TCGv_i64 tmp; + TCGv_i32 tcg_rmode; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + if (!dc_isar_feature(aa32_vrint, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + tmp = tcg_temp_new_i64(); + neon_load_reg64(tmp, a->vm); + fpst = get_fpstatus_ptr(false); + tcg_rmode = tcg_const_i32(float_round_to_zero); + gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); + gen_helper_rintd(tmp, tmp, fpst); + gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); + neon_store_reg64(tmp, a->vd); + tcg_temp_free_ptr(fpst); + tcg_temp_free_i64(tmp); + tcg_temp_free_i32(tcg_rmode); + return true; +} + +static bool trans_VRINTX_sp(DisasContext *s, arg_VRINTX_sp *a) +{ + TCGv_ptr fpst; + TCGv_i32 tmp; + + if (!dc_isar_feature(aa32_vrint, s)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + tmp = tcg_temp_new_i32(); + neon_load_reg32(tmp, a->vm); + fpst = get_fpstatus_ptr(false); + gen_helper_rints_exact(tmp, tmp, fpst); + neon_store_reg32(tmp, a->vd); + tcg_temp_free_ptr(fpst); + tcg_temp_free_i32(tmp); + return true; +} + +static bool trans_VRINTX_dp(DisasContext *s, arg_VRINTX_dp *a) +{ + TCGv_ptr fpst; + TCGv_i64 tmp; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + if (!dc_isar_feature(aa32_vrint, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && ((a->vd | a->vm) & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + tmp = tcg_temp_new_i64(); + neon_load_reg64(tmp, a->vm); + fpst = get_fpstatus_ptr(false); + gen_helper_rintd_exact(tmp, tmp, fpst); + neon_store_reg64(tmp, a->vd); + tcg_temp_free_ptr(fpst); + tcg_temp_free_i64(tmp); + return true; +} + +static bool trans_VCVT_sp(DisasContext *s, arg_VCVT_sp *a) +{ + TCGv_i64 vd; + TCGv_i32 vm; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + vm = tcg_temp_new_i32(); + vd = tcg_temp_new_i64(); + neon_load_reg32(vm, a->vm); + gen_helper_vfp_fcvtds(vd, vm, cpu_env); + neon_store_reg64(vd, a->vd); + tcg_temp_free_i32(vm); + tcg_temp_free_i64(vd); + return true; +} + +static bool trans_VCVT_dp(DisasContext *s, arg_VCVT_dp *a) +{ + TCGv_i64 vm; + TCGv_i32 vd; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + vd = tcg_temp_new_i32(); + vm = tcg_temp_new_i64(); + neon_load_reg64(vm, a->vm); + gen_helper_vfp_fcvtsd(vd, vm, cpu_env); + neon_store_reg32(vd, a->vd); + tcg_temp_free_i32(vd); + tcg_temp_free_i64(vm); + return true; +} + +static bool trans_VCVT_int_sp(DisasContext *s, arg_VCVT_int_sp *a) +{ + TCGv_i32 vm; + TCGv_ptr fpst; + + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + vm = tcg_temp_new_i32(); + neon_load_reg32(vm, a->vm); + fpst = get_fpstatus_ptr(false); + if (a->s) { + /* i32 -> f32 */ + gen_helper_vfp_sitos(vm, vm, fpst); + } else { + /* u32 -> f32 */ + gen_helper_vfp_uitos(vm, vm, fpst); + } + neon_store_reg32(vm, a->vd); + tcg_temp_free_i32(vm); + tcg_temp_free_ptr(fpst); + return true; +} + +static bool trans_VCVT_int_dp(DisasContext *s, arg_VCVT_int_dp *a) +{ + TCGv_i32 vm; + TCGv_i64 vd; + TCGv_ptr fpst; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + vm = tcg_temp_new_i32(); + vd = tcg_temp_new_i64(); + neon_load_reg32(vm, a->vm); + fpst = get_fpstatus_ptr(false); + if (a->s) { + /* i32 -> f64 */ + gen_helper_vfp_sitod(vd, vm, fpst); + } else { + /* u32 -> f64 */ + gen_helper_vfp_uitod(vd, vm, fpst); + } + neon_store_reg64(vd, a->vd); + tcg_temp_free_i32(vm); + tcg_temp_free_i64(vd); + tcg_temp_free_ptr(fpst); + return true; +} + +static bool trans_VJCVT(DisasContext *s, arg_VJCVT *a) +{ + TCGv_i32 vd; + TCGv_i64 vm; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + if (!dc_isar_feature(aa32_jscvt, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + vm = tcg_temp_new_i64(); + vd = tcg_temp_new_i32(); + neon_load_reg64(vm, a->vm); + gen_helper_vjcvt(vd, vm, cpu_env); + neon_store_reg32(vd, a->vd); + tcg_temp_free_i64(vm); + tcg_temp_free_i32(vd); + return true; +} + +static bool trans_VCVT_fix_sp(DisasContext *s, arg_VCVT_fix_sp *a) +{ + TCGv_i32 vd, shift; + TCGv_ptr fpst; + int frac_bits; + + if (!dc_isar_feature(aa32_fpsp_v3, s)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + frac_bits = (a->opc & 1) ? (32 - a->imm) : (16 - a->imm); + + vd = tcg_temp_new_i32(); + neon_load_reg32(vd, a->vd); + + fpst = get_fpstatus_ptr(false); + shift = tcg_const_i32(frac_bits); + + /* Switch on op:U:sx bits */ + switch (a->opc) { + case 0: + gen_helper_vfp_shtos(vd, vd, shift, fpst); + break; + case 1: + gen_helper_vfp_sltos(vd, vd, shift, fpst); + break; + case 2: + gen_helper_vfp_uhtos(vd, vd, shift, fpst); + break; + case 3: + gen_helper_vfp_ultos(vd, vd, shift, fpst); + break; + case 4: + gen_helper_vfp_toshs_round_to_zero(vd, vd, shift, fpst); + break; + case 5: + gen_helper_vfp_tosls_round_to_zero(vd, vd, shift, fpst); + break; + case 6: + gen_helper_vfp_touhs_round_to_zero(vd, vd, shift, fpst); + break; + case 7: + gen_helper_vfp_touls_round_to_zero(vd, vd, shift, fpst); + break; + default: + g_assert_not_reached(); + } + + neon_store_reg32(vd, a->vd); + tcg_temp_free_i32(vd); + tcg_temp_free_i32(shift); + tcg_temp_free_ptr(fpst); + return true; +} + +static bool trans_VCVT_fix_dp(DisasContext *s, arg_VCVT_fix_dp *a) +{ + TCGv_i64 vd; + TCGv_i32 shift; + TCGv_ptr fpst; + int frac_bits; + + if (!dc_isar_feature(aa32_fpdp_v3, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + frac_bits = (a->opc & 1) ? (32 - a->imm) : (16 - a->imm); + + vd = tcg_temp_new_i64(); + neon_load_reg64(vd, a->vd); + + fpst = get_fpstatus_ptr(false); + shift = tcg_const_i32(frac_bits); + + /* Switch on op:U:sx bits */ + switch (a->opc) { + case 0: + gen_helper_vfp_shtod(vd, vd, shift, fpst); + break; + case 1: + gen_helper_vfp_sltod(vd, vd, shift, fpst); + break; + case 2: + gen_helper_vfp_uhtod(vd, vd, shift, fpst); + break; + case 3: + gen_helper_vfp_ultod(vd, vd, shift, fpst); + break; + case 4: + gen_helper_vfp_toshd_round_to_zero(vd, vd, shift, fpst); + break; + case 5: + gen_helper_vfp_tosld_round_to_zero(vd, vd, shift, fpst); + break; + case 6: + gen_helper_vfp_touhd_round_to_zero(vd, vd, shift, fpst); + break; + case 7: + gen_helper_vfp_tould_round_to_zero(vd, vd, shift, fpst); + break; + default: + g_assert_not_reached(); + } + + neon_store_reg64(vd, a->vd); + tcg_temp_free_i64(vd); + tcg_temp_free_i32(shift); + tcg_temp_free_ptr(fpst); + return true; +} + +static bool trans_VCVT_sp_int(DisasContext *s, arg_VCVT_sp_int *a) +{ + TCGv_i32 vm; + TCGv_ptr fpst; + + if (!dc_isar_feature(aa32_fpsp_v2, s)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(false); + vm = tcg_temp_new_i32(); + neon_load_reg32(vm, a->vm); + + if (a->s) { + if (a->rz) { + gen_helper_vfp_tosizs(vm, vm, fpst); + } else { + gen_helper_vfp_tosis(vm, vm, fpst); + } + } else { + if (a->rz) { + gen_helper_vfp_touizs(vm, vm, fpst); + } else { + gen_helper_vfp_touis(vm, vm, fpst); + } + } + neon_store_reg32(vm, a->vd); + tcg_temp_free_i32(vm); + tcg_temp_free_ptr(fpst); + return true; +} + +static bool trans_VCVT_dp_int(DisasContext *s, arg_VCVT_dp_int *a) +{ + TCGv_i32 vd; + TCGv_i64 vm; + TCGv_ptr fpst; + + if (!dc_isar_feature(aa32_fpdp_v2, s)) { + return false; + } + + /* UNDEF accesses to D16-D31 if they don't exist. */ + if (!dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { + return false; + } + + if (!vfp_access_check(s)) { + return true; + } + + fpst = get_fpstatus_ptr(false); + vm = tcg_temp_new_i64(); + vd = tcg_temp_new_i32(); + neon_load_reg64(vm, a->vm); + + if (a->s) { + if (a->rz) { + gen_helper_vfp_tosizd(vd, vm, fpst); + } else { + gen_helper_vfp_tosid(vd, vm, fpst); + } + } else { + if (a->rz) { + gen_helper_vfp_touizd(vd, vm, fpst); + } else { + gen_helper_vfp_touid(vd, vm, fpst); + } + } + neon_store_reg32(vd, a->vd); + tcg_temp_free_i32(vd); + tcg_temp_free_i64(vm); + tcg_temp_free_ptr(fpst); + return true; +} + +/* + * Decode VLLDM and VLSTM are nonstandard because: + * * if there is no FPU then these insns must NOP in + * Secure state and UNDEF in Nonsecure state + * * if there is an FPU then these insns do not have + * the usual behaviour that vfp_access_check() provides of + * being controlled by CPACR/NSACR enable bits or the + * lazy-stacking logic. + */ +static bool trans_VLLDM_VLSTM(DisasContext *s, arg_VLLDM_VLSTM *a) +{ + TCGv_i32 fptr; + + if (!arm_dc_feature(s, ARM_FEATURE_M) || + !arm_dc_feature(s, ARM_FEATURE_V8)) { + return false; + } + /* If not secure, UNDEF. */ + if (!s->v8m_secure) { + return false; + } + /* If no fpu, NOP. */ + if (!dc_isar_feature(aa32_vfp, s)) { + return true; + } + + fptr = load_reg(s, a->rn); + if (a->l) { + gen_helper_v7m_vlldm(cpu_env, fptr); + } else { + gen_helper_v7m_vlstm(cpu_env, fptr); + } + tcg_temp_free_i32(fptr); + + /* End the TB, because we have updated FP control bits */ + s->base.is_jmp = DISAS_UPDATE_EXIT; + return true; +} diff --git a/target/arm/translate-vfp.inc.c b/target/arm/translate-vfp.inc.c deleted file mode 100644 index afa8a5f..0000000 --- a/target/arm/translate-vfp.inc.c +++ /dev/null @@ -1,2865 +0,0 @@ -/* - * ARM translation: AArch32 VFP instructions - * - * Copyright (c) 2003 Fabrice Bellard - * Copyright (c) 2005-2007 CodeSourcery - * Copyright (c) 2007 OpenedHand, Ltd. - * Copyright (c) 2019 Linaro, Ltd. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, see . - */ - -/* - * This file is intended to be included from translate.c; it uses - * some macros and definitions provided by that file. - * It might be possible to convert it to a standalone .c file eventually. - */ - -/* Include the generated VFP decoder */ -#include "decode-vfp.inc.c" -#include "decode-vfp-uncond.inc.c" - -/* - * The imm8 encodes the sign bit, enough bits to represent an exponent in - * the range 01....1xx to 10....0xx, and the most significant 4 bits of - * the mantissa; see VFPExpandImm() in the v8 ARM ARM. - */ -uint64_t vfp_expand_imm(int size, uint8_t imm8) -{ - uint64_t imm; - - switch (size) { - case MO_64: - imm = (extract32(imm8, 7, 1) ? 0x8000 : 0) | - (extract32(imm8, 6, 1) ? 0x3fc0 : 0x4000) | - extract32(imm8, 0, 6); - imm <<= 48; - break; - case MO_32: - imm = (extract32(imm8, 7, 1) ? 0x8000 : 0) | - (extract32(imm8, 6, 1) ? 0x3e00 : 0x4000) | - (extract32(imm8, 0, 6) << 3); - imm <<= 16; - break; - case MO_16: - imm = (extract32(imm8, 7, 1) ? 0x8000 : 0) | - (extract32(imm8, 6, 1) ? 0x3000 : 0x4000) | - (extract32(imm8, 0, 6) << 6); - break; - default: - g_assert_not_reached(); - } - return imm; -} - -/* - * Return the offset of a 16-bit half of the specified VFP single-precision - * register. If top is true, returns the top 16 bits; otherwise the bottom - * 16 bits. - */ -static inline long vfp_f16_offset(unsigned reg, bool top) -{ - long offs = vfp_reg_offset(false, reg); -#ifdef HOST_WORDS_BIGENDIAN - if (!top) { - offs += 2; - } -#else - if (top) { - offs += 2; - } -#endif - return offs; -} - -/* - * Check that VFP access is enabled. If it is, do the necessary - * M-profile lazy-FP handling and then return true. - * If not, emit code to generate an appropriate exception and - * return false. - * The ignore_vfp_enabled argument specifies that we should ignore - * whether VFP is enabled via FPEXC[EN]: this should be true for FMXR/FMRX - * accesses to FPSID, FPEXC, MVFR0, MVFR1, MVFR2, and false for all other insns. - */ -static bool full_vfp_access_check(DisasContext *s, bool ignore_vfp_enabled) -{ - if (s->fp_excp_el) { - if (arm_dc_feature(s, ARM_FEATURE_M)) { - gen_exception_insn(s, s->pc_curr, EXCP_NOCP, syn_uncategorized(), - s->fp_excp_el); - } else { - gen_exception_insn(s, s->pc_curr, EXCP_UDEF, - syn_fp_access_trap(1, 0xe, false), - s->fp_excp_el); - } - return false; - } - - if (!s->vfp_enabled && !ignore_vfp_enabled) { - assert(!arm_dc_feature(s, ARM_FEATURE_M)); - unallocated_encoding(s); - return false; - } - - if (arm_dc_feature(s, ARM_FEATURE_M)) { - /* Handle M-profile lazy FP state mechanics */ - - /* Trigger lazy-state preservation if necessary */ - if (s->v7m_lspact) { - /* - * Lazy state saving affects external memory and also the NVIC, - * so we must mark it as an IO operation for icount (and cause - * this to be the last insn in the TB). - */ - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - s->base.is_jmp = DISAS_UPDATE_EXIT; - gen_io_start(); - } - gen_helper_v7m_preserve_fp_state(cpu_env); - /* - * If the preserve_fp_state helper doesn't throw an exception - * then it will clear LSPACT; we don't need to repeat this for - * any further FP insns in this TB. - */ - s->v7m_lspact = false; - } - - /* Update ownership of FP context: set FPCCR.S to match current state */ - if (s->v8m_fpccr_s_wrong) { - TCGv_i32 tmp; - - tmp = load_cpu_field(v7m.fpccr[M_REG_S]); - if (s->v8m_secure) { - tcg_gen_ori_i32(tmp, tmp, R_V7M_FPCCR_S_MASK); - } else { - tcg_gen_andi_i32(tmp, tmp, ~R_V7M_FPCCR_S_MASK); - } - store_cpu_field(tmp, v7m.fpccr[M_REG_S]); - /* Don't need to do this for any further FP insns in this TB */ - s->v8m_fpccr_s_wrong = false; - } - - if (s->v7m_new_fp_ctxt_needed) { - /* - * Create new FP context by updating CONTROL.FPCA, CONTROL.SFPA - * and the FPSCR. - */ - TCGv_i32 control, fpscr; - uint32_t bits = R_V7M_CONTROL_FPCA_MASK; - - fpscr = load_cpu_field(v7m.fpdscr[s->v8m_secure]); - gen_helper_vfp_set_fpscr(cpu_env, fpscr); - tcg_temp_free_i32(fpscr); - /* - * We don't need to arrange to end the TB, because the only - * parts of FPSCR which we cache in the TB flags are the VECLEN - * and VECSTRIDE, and those don't exist for M-profile. - */ - - if (s->v8m_secure) { - bits |= R_V7M_CONTROL_SFPA_MASK; - } - control = load_cpu_field(v7m.control[M_REG_S]); - tcg_gen_ori_i32(control, control, bits); - store_cpu_field(control, v7m.control[M_REG_S]); - /* Don't need to do this for any further FP insns in this TB */ - s->v7m_new_fp_ctxt_needed = false; - } - } - - return true; -} - -/* - * The most usual kind of VFP access check, for everything except - * FMXR/FMRX to the always-available special registers. - */ -static bool vfp_access_check(DisasContext *s) -{ - return full_vfp_access_check(s, false); -} - -static bool trans_VSEL(DisasContext *s, arg_VSEL *a) -{ - uint32_t rd, rn, rm; - bool dp = a->dp; - - if (!dc_isar_feature(aa32_vsel, s)) { - return false; - } - - if (dp && !dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (dp && !dc_isar_feature(aa32_simd_r32, s) && - ((a->vm | a->vn | a->vd) & 0x10)) { - return false; - } - - rd = a->vd; - rn = a->vn; - rm = a->vm; - - if (!vfp_access_check(s)) { - return true; - } - - if (dp) { - TCGv_i64 frn, frm, dest; - TCGv_i64 tmp, zero, zf, nf, vf; - - zero = tcg_const_i64(0); - - frn = tcg_temp_new_i64(); - frm = tcg_temp_new_i64(); - dest = tcg_temp_new_i64(); - - zf = tcg_temp_new_i64(); - nf = tcg_temp_new_i64(); - vf = tcg_temp_new_i64(); - - tcg_gen_extu_i32_i64(zf, cpu_ZF); - tcg_gen_ext_i32_i64(nf, cpu_NF); - tcg_gen_ext_i32_i64(vf, cpu_VF); - - neon_load_reg64(frn, rn); - neon_load_reg64(frm, rm); - switch (a->cc) { - case 0: /* eq: Z */ - tcg_gen_movcond_i64(TCG_COND_EQ, dest, zf, zero, - frn, frm); - break; - case 1: /* vs: V */ - tcg_gen_movcond_i64(TCG_COND_LT, dest, vf, zero, - frn, frm); - break; - case 2: /* ge: N == V -> N ^ V == 0 */ - tmp = tcg_temp_new_i64(); - tcg_gen_xor_i64(tmp, vf, nf); - tcg_gen_movcond_i64(TCG_COND_GE, dest, tmp, zero, - frn, frm); - tcg_temp_free_i64(tmp); - break; - case 3: /* gt: !Z && N == V */ - tcg_gen_movcond_i64(TCG_COND_NE, dest, zf, zero, - frn, frm); - tmp = tcg_temp_new_i64(); - tcg_gen_xor_i64(tmp, vf, nf); - tcg_gen_movcond_i64(TCG_COND_GE, dest, tmp, zero, - dest, frm); - tcg_temp_free_i64(tmp); - break; - } - neon_store_reg64(dest, rd); - tcg_temp_free_i64(frn); - tcg_temp_free_i64(frm); - tcg_temp_free_i64(dest); - - tcg_temp_free_i64(zf); - tcg_temp_free_i64(nf); - tcg_temp_free_i64(vf); - - tcg_temp_free_i64(zero); - } else { - TCGv_i32 frn, frm, dest; - TCGv_i32 tmp, zero; - - zero = tcg_const_i32(0); - - frn = tcg_temp_new_i32(); - frm = tcg_temp_new_i32(); - dest = tcg_temp_new_i32(); - neon_load_reg32(frn, rn); - neon_load_reg32(frm, rm); - switch (a->cc) { - case 0: /* eq: Z */ - tcg_gen_movcond_i32(TCG_COND_EQ, dest, cpu_ZF, zero, - frn, frm); - break; - case 1: /* vs: V */ - tcg_gen_movcond_i32(TCG_COND_LT, dest, cpu_VF, zero, - frn, frm); - break; - case 2: /* ge: N == V -> N ^ V == 0 */ - tmp = tcg_temp_new_i32(); - tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF); - tcg_gen_movcond_i32(TCG_COND_GE, dest, tmp, zero, - frn, frm); - tcg_temp_free_i32(tmp); - break; - case 3: /* gt: !Z && N == V */ - tcg_gen_movcond_i32(TCG_COND_NE, dest, cpu_ZF, zero, - frn, frm); - tmp = tcg_temp_new_i32(); - tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF); - tcg_gen_movcond_i32(TCG_COND_GE, dest, tmp, zero, - dest, frm); - tcg_temp_free_i32(tmp); - break; - } - neon_store_reg32(dest, rd); - tcg_temp_free_i32(frn); - tcg_temp_free_i32(frm); - tcg_temp_free_i32(dest); - - tcg_temp_free_i32(zero); - } - - return true; -} - -/* - * Table for converting the most common AArch32 encoding of - * rounding mode to arm_fprounding order (which matches the - * common AArch64 order); see ARM ARM pseudocode FPDecodeRM(). - */ -static const uint8_t fp_decode_rm[] = { - FPROUNDING_TIEAWAY, - FPROUNDING_TIEEVEN, - FPROUNDING_POSINF, - FPROUNDING_NEGINF, -}; - -static bool trans_VRINT(DisasContext *s, arg_VRINT *a) -{ - uint32_t rd, rm; - bool dp = a->dp; - TCGv_ptr fpst; - TCGv_i32 tcg_rmode; - int rounding = fp_decode_rm[a->rm]; - - if (!dc_isar_feature(aa32_vrint, s)) { - return false; - } - - if (dp && !dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (dp && !dc_isar_feature(aa32_simd_r32, s) && - ((a->vm | a->vd) & 0x10)) { - return false; - } - - rd = a->vd; - rm = a->vm; - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(0); - - tcg_rmode = tcg_const_i32(arm_rmode_to_sf(rounding)); - gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); - - if (dp) { - TCGv_i64 tcg_op; - TCGv_i64 tcg_res; - tcg_op = tcg_temp_new_i64(); - tcg_res = tcg_temp_new_i64(); - neon_load_reg64(tcg_op, rm); - gen_helper_rintd(tcg_res, tcg_op, fpst); - neon_store_reg64(tcg_res, rd); - tcg_temp_free_i64(tcg_op); - tcg_temp_free_i64(tcg_res); - } else { - TCGv_i32 tcg_op; - TCGv_i32 tcg_res; - tcg_op = tcg_temp_new_i32(); - tcg_res = tcg_temp_new_i32(); - neon_load_reg32(tcg_op, rm); - gen_helper_rints(tcg_res, tcg_op, fpst); - neon_store_reg32(tcg_res, rd); - tcg_temp_free_i32(tcg_op); - tcg_temp_free_i32(tcg_res); - } - - gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); - tcg_temp_free_i32(tcg_rmode); - - tcg_temp_free_ptr(fpst); - return true; -} - -static bool trans_VCVT(DisasContext *s, arg_VCVT *a) -{ - uint32_t rd, rm; - bool dp = a->dp; - TCGv_ptr fpst; - TCGv_i32 tcg_rmode, tcg_shift; - int rounding = fp_decode_rm[a->rm]; - bool is_signed = a->op; - - if (!dc_isar_feature(aa32_vcvt_dr, s)) { - return false; - } - - if (dp && !dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (dp && !dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { - return false; - } - - rd = a->vd; - rm = a->vm; - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(0); - - tcg_shift = tcg_const_i32(0); - - tcg_rmode = tcg_const_i32(arm_rmode_to_sf(rounding)); - gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); - - if (dp) { - TCGv_i64 tcg_double, tcg_res; - TCGv_i32 tcg_tmp; - tcg_double = tcg_temp_new_i64(); - tcg_res = tcg_temp_new_i64(); - tcg_tmp = tcg_temp_new_i32(); - neon_load_reg64(tcg_double, rm); - if (is_signed) { - gen_helper_vfp_tosld(tcg_res, tcg_double, tcg_shift, fpst); - } else { - gen_helper_vfp_tould(tcg_res, tcg_double, tcg_shift, fpst); - } - tcg_gen_extrl_i64_i32(tcg_tmp, tcg_res); - neon_store_reg32(tcg_tmp, rd); - tcg_temp_free_i32(tcg_tmp); - tcg_temp_free_i64(tcg_res); - tcg_temp_free_i64(tcg_double); - } else { - TCGv_i32 tcg_single, tcg_res; - tcg_single = tcg_temp_new_i32(); - tcg_res = tcg_temp_new_i32(); - neon_load_reg32(tcg_single, rm); - if (is_signed) { - gen_helper_vfp_tosls(tcg_res, tcg_single, tcg_shift, fpst); - } else { - gen_helper_vfp_touls(tcg_res, tcg_single, tcg_shift, fpst); - } - neon_store_reg32(tcg_res, rd); - tcg_temp_free_i32(tcg_res); - tcg_temp_free_i32(tcg_single); - } - - gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); - tcg_temp_free_i32(tcg_rmode); - - tcg_temp_free_i32(tcg_shift); - - tcg_temp_free_ptr(fpst); - - return true; -} - -static bool trans_VMOV_to_gp(DisasContext *s, arg_VMOV_to_gp *a) -{ - /* VMOV scalar to general purpose register */ - TCGv_i32 tmp; - int pass; - uint32_t offset; - - /* SIZE == 2 is a VFP instruction; otherwise NEON. */ - if (a->size == 2 - ? !dc_isar_feature(aa32_fpsp_v2, s) - : !arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vn & 0x10)) { - return false; - } - - offset = a->index << a->size; - pass = extract32(offset, 2, 1); - offset = extract32(offset, 0, 2) * 8; - - if (!vfp_access_check(s)) { - return true; - } - - tmp = neon_load_reg(a->vn, pass); - switch (a->size) { - case 0: - if (offset) { - tcg_gen_shri_i32(tmp, tmp, offset); - } - if (a->u) { - gen_uxtb(tmp); - } else { - gen_sxtb(tmp); - } - break; - case 1: - if (a->u) { - if (offset) { - tcg_gen_shri_i32(tmp, tmp, 16); - } else { - gen_uxth(tmp); - } - } else { - if (offset) { - tcg_gen_sari_i32(tmp, tmp, 16); - } else { - gen_sxth(tmp); - } - } - break; - case 2: - break; - } - store_reg(s, a->rt, tmp); - - return true; -} - -static bool trans_VMOV_from_gp(DisasContext *s, arg_VMOV_from_gp *a) -{ - /* VMOV general purpose register to scalar */ - TCGv_i32 tmp, tmp2; - int pass; - uint32_t offset; - - /* SIZE == 2 is a VFP instruction; otherwise NEON. */ - if (a->size == 2 - ? !dc_isar_feature(aa32_fpsp_v2, s) - : !arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vn & 0x10)) { - return false; - } - - offset = a->index << a->size; - pass = extract32(offset, 2, 1); - offset = extract32(offset, 0, 2) * 8; - - if (!vfp_access_check(s)) { - return true; - } - - tmp = load_reg(s, a->rt); - switch (a->size) { - case 0: - tmp2 = neon_load_reg(a->vn, pass); - tcg_gen_deposit_i32(tmp, tmp2, tmp, offset, 8); - tcg_temp_free_i32(tmp2); - break; - case 1: - tmp2 = neon_load_reg(a->vn, pass); - tcg_gen_deposit_i32(tmp, tmp2, tmp, offset, 16); - tcg_temp_free_i32(tmp2); - break; - case 2: - break; - } - neon_store_reg(a->vn, pass, tmp); - - return true; -} - -static bool trans_VDUP(DisasContext *s, arg_VDUP *a) -{ - /* VDUP (general purpose register) */ - TCGv_i32 tmp; - int size, vec_size; - - if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vn & 0x10)) { - return false; - } - - if (a->b && a->e) { - return false; - } - - if (a->q && (a->vn & 1)) { - return false; - } - - vec_size = a->q ? 16 : 8; - if (a->b) { - size = 0; - } else if (a->e) { - size = 1; - } else { - size = 2; - } - - if (!vfp_access_check(s)) { - return true; - } - - tmp = load_reg(s, a->rt); - tcg_gen_gvec_dup_i32(size, neon_reg_offset(a->vn, 0), - vec_size, vec_size, tmp); - tcg_temp_free_i32(tmp); - - return true; -} - -static bool trans_VMSR_VMRS(DisasContext *s, arg_VMSR_VMRS *a) -{ - TCGv_i32 tmp; - bool ignore_vfp_enabled = false; - - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - if (arm_dc_feature(s, ARM_FEATURE_M)) { - /* - * The only M-profile VFP vmrs/vmsr sysreg is FPSCR. - * Accesses to R15 are UNPREDICTABLE; we choose to undef. - * (FPSCR -> r15 is a special case which writes to the PSR flags.) - */ - if (a->rt == 15 && (!a->l || a->reg != ARM_VFP_FPSCR)) { - return false; - } - } - - switch (a->reg) { - case ARM_VFP_FPSID: - /* - * VFPv2 allows access to FPSID from userspace; VFPv3 restricts - * all ID registers to privileged access only. - */ - if (IS_USER(s) && dc_isar_feature(aa32_fpsp_v3, s)) { - return false; - } - ignore_vfp_enabled = true; - break; - case ARM_VFP_MVFR0: - case ARM_VFP_MVFR1: - if (IS_USER(s) || !arm_dc_feature(s, ARM_FEATURE_MVFR)) { - return false; - } - ignore_vfp_enabled = true; - break; - case ARM_VFP_MVFR2: - if (IS_USER(s) || !arm_dc_feature(s, ARM_FEATURE_V8)) { - return false; - } - ignore_vfp_enabled = true; - break; - case ARM_VFP_FPSCR: - break; - case ARM_VFP_FPEXC: - if (IS_USER(s)) { - return false; - } - ignore_vfp_enabled = true; - break; - case ARM_VFP_FPINST: - case ARM_VFP_FPINST2: - /* Not present in VFPv3 */ - if (IS_USER(s) || dc_isar_feature(aa32_fpsp_v3, s)) { - return false; - } - break; - default: - return false; - } - - if (!full_vfp_access_check(s, ignore_vfp_enabled)) { - return true; - } - - if (a->l) { - /* VMRS, move VFP special register to gp register */ - switch (a->reg) { - case ARM_VFP_MVFR0: - case ARM_VFP_MVFR1: - case ARM_VFP_MVFR2: - case ARM_VFP_FPSID: - if (s->current_el == 1) { - TCGv_i32 tcg_reg, tcg_rt; - - gen_set_condexec(s); - gen_set_pc_im(s, s->pc_curr); - tcg_reg = tcg_const_i32(a->reg); - tcg_rt = tcg_const_i32(a->rt); - gen_helper_check_hcr_el2_trap(cpu_env, tcg_rt, tcg_reg); - tcg_temp_free_i32(tcg_reg); - tcg_temp_free_i32(tcg_rt); - } - /* fall through */ - case ARM_VFP_FPEXC: - case ARM_VFP_FPINST: - case ARM_VFP_FPINST2: - tmp = load_cpu_field(vfp.xregs[a->reg]); - break; - case ARM_VFP_FPSCR: - if (a->rt == 15) { - tmp = load_cpu_field(vfp.xregs[ARM_VFP_FPSCR]); - tcg_gen_andi_i32(tmp, tmp, 0xf0000000); - } else { - tmp = tcg_temp_new_i32(); - gen_helper_vfp_get_fpscr(tmp, cpu_env); - } - break; - default: - g_assert_not_reached(); - } - - if (a->rt == 15) { - /* Set the 4 flag bits in the CPSR. */ - gen_set_nzcv(tmp); - tcg_temp_free_i32(tmp); - } else { - store_reg(s, a->rt, tmp); - } - } else { - /* VMSR, move gp register to VFP special register */ - switch (a->reg) { - case ARM_VFP_FPSID: - case ARM_VFP_MVFR0: - case ARM_VFP_MVFR1: - case ARM_VFP_MVFR2: - /* Writes are ignored. */ - break; - case ARM_VFP_FPSCR: - tmp = load_reg(s, a->rt); - gen_helper_vfp_set_fpscr(cpu_env, tmp); - tcg_temp_free_i32(tmp); - gen_lookup_tb(s); - break; - case ARM_VFP_FPEXC: - /* - * TODO: VFP subarchitecture support. - * For now, keep the EN bit only - */ - tmp = load_reg(s, a->rt); - tcg_gen_andi_i32(tmp, tmp, 1 << 30); - store_cpu_field(tmp, vfp.xregs[a->reg]); - gen_lookup_tb(s); - break; - case ARM_VFP_FPINST: - case ARM_VFP_FPINST2: - tmp = load_reg(s, a->rt); - store_cpu_field(tmp, vfp.xregs[a->reg]); - break; - default: - g_assert_not_reached(); - } - } - - return true; -} - -static bool trans_VMOV_single(DisasContext *s, arg_VMOV_single *a) -{ - TCGv_i32 tmp; - - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - if (a->l) { - /* VFP to general purpose register */ - tmp = tcg_temp_new_i32(); - neon_load_reg32(tmp, a->vn); - if (a->rt == 15) { - /* Set the 4 flag bits in the CPSR. */ - gen_set_nzcv(tmp); - tcg_temp_free_i32(tmp); - } else { - store_reg(s, a->rt, tmp); - } - } else { - /* general purpose register to VFP */ - tmp = load_reg(s, a->rt); - neon_store_reg32(tmp, a->vn); - tcg_temp_free_i32(tmp); - } - - return true; -} - -static bool trans_VMOV_64_sp(DisasContext *s, arg_VMOV_64_sp *a) -{ - TCGv_i32 tmp; - - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - /* - * VMOV between two general-purpose registers and two single precision - * floating point registers - */ - if (!vfp_access_check(s)) { - return true; - } - - if (a->op) { - /* fpreg to gpreg */ - tmp = tcg_temp_new_i32(); - neon_load_reg32(tmp, a->vm); - store_reg(s, a->rt, tmp); - tmp = tcg_temp_new_i32(); - neon_load_reg32(tmp, a->vm + 1); - store_reg(s, a->rt2, tmp); - } else { - /* gpreg to fpreg */ - tmp = load_reg(s, a->rt); - neon_store_reg32(tmp, a->vm); - tcg_temp_free_i32(tmp); - tmp = load_reg(s, a->rt2); - neon_store_reg32(tmp, a->vm + 1); - tcg_temp_free_i32(tmp); - } - - return true; -} - -static bool trans_VMOV_64_dp(DisasContext *s, arg_VMOV_64_dp *a) -{ - TCGv_i32 tmp; - - /* - * VMOV between two general-purpose registers and one double precision - * floating point register. Note that this does not require support - * for double precision arithmetic. - */ - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - if (a->op) { - /* fpreg to gpreg */ - tmp = tcg_temp_new_i32(); - neon_load_reg32(tmp, a->vm * 2); - store_reg(s, a->rt, tmp); - tmp = tcg_temp_new_i32(); - neon_load_reg32(tmp, a->vm * 2 + 1); - store_reg(s, a->rt2, tmp); - } else { - /* gpreg to fpreg */ - tmp = load_reg(s, a->rt); - neon_store_reg32(tmp, a->vm * 2); - tcg_temp_free_i32(tmp); - tmp = load_reg(s, a->rt2); - neon_store_reg32(tmp, a->vm * 2 + 1); - tcg_temp_free_i32(tmp); - } - - return true; -} - -static bool trans_VLDR_VSTR_sp(DisasContext *s, arg_VLDR_VSTR_sp *a) -{ - uint32_t offset; - TCGv_i32 addr, tmp; - - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - offset = a->imm << 2; - if (!a->u) { - offset = -offset; - } - - /* For thumb, use of PC is UNPREDICTABLE. */ - addr = add_reg_for_lit(s, a->rn, offset); - tmp = tcg_temp_new_i32(); - if (a->l) { - gen_aa32_ld32u(s, tmp, addr, get_mem_index(s)); - neon_store_reg32(tmp, a->vd); - } else { - neon_load_reg32(tmp, a->vd); - gen_aa32_st32(s, tmp, addr, get_mem_index(s)); - } - tcg_temp_free_i32(tmp); - tcg_temp_free_i32(addr); - - return true; -} - -static bool trans_VLDR_VSTR_dp(DisasContext *s, arg_VLDR_VSTR_dp *a) -{ - uint32_t offset; - TCGv_i32 addr; - TCGv_i64 tmp; - - /* Note that this does not require support for double arithmetic. */ - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - offset = a->imm << 2; - if (!a->u) { - offset = -offset; - } - - /* For thumb, use of PC is UNPREDICTABLE. */ - addr = add_reg_for_lit(s, a->rn, offset); - tmp = tcg_temp_new_i64(); - if (a->l) { - gen_aa32_ld64(s, tmp, addr, get_mem_index(s)); - neon_store_reg64(tmp, a->vd); - } else { - neon_load_reg64(tmp, a->vd); - gen_aa32_st64(s, tmp, addr, get_mem_index(s)); - } - tcg_temp_free_i64(tmp); - tcg_temp_free_i32(addr); - - return true; -} - -static bool trans_VLDM_VSTM_sp(DisasContext *s, arg_VLDM_VSTM_sp *a) -{ - uint32_t offset; - TCGv_i32 addr, tmp; - int i, n; - - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - n = a->imm; - - if (n == 0 || (a->vd + n) > 32) { - /* - * UNPREDICTABLE cases for bad immediates: we choose to - * UNDEF to avoid generating huge numbers of TCG ops - */ - return false; - } - if (a->rn == 15 && a->w) { - /* writeback to PC is UNPREDICTABLE, we choose to UNDEF */ - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - /* For thumb, use of PC is UNPREDICTABLE. */ - addr = add_reg_for_lit(s, a->rn, 0); - if (a->p) { - /* pre-decrement */ - tcg_gen_addi_i32(addr, addr, -(a->imm << 2)); - } - - if (s->v8m_stackcheck && a->rn == 13 && a->w) { - /* - * Here 'addr' is the lowest address we will store to, - * and is either the old SP (if post-increment) or - * the new SP (if pre-decrement). For post-increment - * where the old value is below the limit and the new - * value is above, it is UNKNOWN whether the limit check - * triggers; we choose to trigger. - */ - gen_helper_v8m_stackcheck(cpu_env, addr); - } - - offset = 4; - tmp = tcg_temp_new_i32(); - for (i = 0; i < n; i++) { - if (a->l) { - /* load */ - gen_aa32_ld32u(s, tmp, addr, get_mem_index(s)); - neon_store_reg32(tmp, a->vd + i); - } else { - /* store */ - neon_load_reg32(tmp, a->vd + i); - gen_aa32_st32(s, tmp, addr, get_mem_index(s)); - } - tcg_gen_addi_i32(addr, addr, offset); - } - tcg_temp_free_i32(tmp); - if (a->w) { - /* writeback */ - if (a->p) { - offset = -offset * n; - tcg_gen_addi_i32(addr, addr, offset); - } - store_reg(s, a->rn, addr); - } else { - tcg_temp_free_i32(addr); - } - - return true; -} - -static bool trans_VLDM_VSTM_dp(DisasContext *s, arg_VLDM_VSTM_dp *a) -{ - uint32_t offset; - TCGv_i32 addr; - TCGv_i64 tmp; - int i, n; - - /* Note that this does not require support for double arithmetic. */ - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - n = a->imm >> 1; - - if (n == 0 || (a->vd + n) > 32 || n > 16) { - /* - * UNPREDICTABLE cases for bad immediates: we choose to - * UNDEF to avoid generating huge numbers of TCG ops - */ - return false; - } - if (a->rn == 15 && a->w) { - /* writeback to PC is UNPREDICTABLE, we choose to UNDEF */ - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd + n) > 16) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - /* For thumb, use of PC is UNPREDICTABLE. */ - addr = add_reg_for_lit(s, a->rn, 0); - if (a->p) { - /* pre-decrement */ - tcg_gen_addi_i32(addr, addr, -(a->imm << 2)); - } - - if (s->v8m_stackcheck && a->rn == 13 && a->w) { - /* - * Here 'addr' is the lowest address we will store to, - * and is either the old SP (if post-increment) or - * the new SP (if pre-decrement). For post-increment - * where the old value is below the limit and the new - * value is above, it is UNKNOWN whether the limit check - * triggers; we choose to trigger. - */ - gen_helper_v8m_stackcheck(cpu_env, addr); - } - - offset = 8; - tmp = tcg_temp_new_i64(); - for (i = 0; i < n; i++) { - if (a->l) { - /* load */ - gen_aa32_ld64(s, tmp, addr, get_mem_index(s)); - neon_store_reg64(tmp, a->vd + i); - } else { - /* store */ - neon_load_reg64(tmp, a->vd + i); - gen_aa32_st64(s, tmp, addr, get_mem_index(s)); - } - tcg_gen_addi_i32(addr, addr, offset); - } - tcg_temp_free_i64(tmp); - if (a->w) { - /* writeback */ - if (a->p) { - offset = -offset * n; - } else if (a->imm & 1) { - offset = 4; - } else { - offset = 0; - } - - if (offset != 0) { - tcg_gen_addi_i32(addr, addr, offset); - } - store_reg(s, a->rn, addr); - } else { - tcg_temp_free_i32(addr); - } - - return true; -} - -/* - * Types for callbacks for do_vfp_3op_sp() and do_vfp_3op_dp(). - * The callback should emit code to write a value to vd. If - * do_vfp_3op_{sp,dp}() was passed reads_vd then the TCGv vd - * will contain the old value of the relevant VFP register; - * otherwise it must be written to only. - */ -typedef void VFPGen3OpSPFn(TCGv_i32 vd, - TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst); -typedef void VFPGen3OpDPFn(TCGv_i64 vd, - TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst); - -/* - * Types for callbacks for do_vfp_2op_sp() and do_vfp_2op_dp(). - * The callback should emit code to write a value to vd (which - * should be written to only). - */ -typedef void VFPGen2OpSPFn(TCGv_i32 vd, TCGv_i32 vm); -typedef void VFPGen2OpDPFn(TCGv_i64 vd, TCGv_i64 vm); - -/* - * Return true if the specified S reg is in a scalar bank - * (ie if it is s0..s7) - */ -static inline bool vfp_sreg_is_scalar(int reg) -{ - return (reg & 0x18) == 0; -} - -/* - * Return true if the specified D reg is in a scalar bank - * (ie if it is d0..d3 or d16..d19) - */ -static inline bool vfp_dreg_is_scalar(int reg) -{ - return (reg & 0xc) == 0; -} - -/* - * Advance the S reg number forwards by delta within its bank - * (ie increment the low 3 bits but leave the rest the same) - */ -static inline int vfp_advance_sreg(int reg, int delta) -{ - return ((reg + delta) & 0x7) | (reg & ~0x7); -} - -/* - * Advance the D reg number forwards by delta within its bank - * (ie increment the low 2 bits but leave the rest the same) - */ -static inline int vfp_advance_dreg(int reg, int delta) -{ - return ((reg + delta) & 0x3) | (reg & ~0x3); -} - -/* - * Perform a 3-operand VFP data processing instruction. fn is the - * callback to do the actual operation; this function deals with the - * code to handle looping around for VFP vector processing. - */ -static bool do_vfp_3op_sp(DisasContext *s, VFPGen3OpSPFn *fn, - int vd, int vn, int vm, bool reads_vd) -{ - uint32_t delta_m = 0; - uint32_t delta_d = 0; - int veclen = s->vec_len; - TCGv_i32 f0, f1, fd; - TCGv_ptr fpst; - - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - if (!dc_isar_feature(aa32_fpshvec, s) && - (veclen != 0 || s->vec_stride != 0)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - if (veclen > 0) { - /* Figure out what type of vector operation this is. */ - if (vfp_sreg_is_scalar(vd)) { - /* scalar */ - veclen = 0; - } else { - delta_d = s->vec_stride + 1; - - if (vfp_sreg_is_scalar(vm)) { - /* mixed scalar/vector */ - delta_m = 0; - } else { - /* vector */ - delta_m = delta_d; - } - } - } - - f0 = tcg_temp_new_i32(); - f1 = tcg_temp_new_i32(); - fd = tcg_temp_new_i32(); - fpst = get_fpstatus_ptr(0); - - neon_load_reg32(f0, vn); - neon_load_reg32(f1, vm); - - for (;;) { - if (reads_vd) { - neon_load_reg32(fd, vd); - } - fn(fd, f0, f1, fpst); - neon_store_reg32(fd, vd); - - if (veclen == 0) { - break; - } - - /* Set up the operands for the next iteration */ - veclen--; - vd = vfp_advance_sreg(vd, delta_d); - vn = vfp_advance_sreg(vn, delta_d); - neon_load_reg32(f0, vn); - if (delta_m) { - vm = vfp_advance_sreg(vm, delta_m); - neon_load_reg32(f1, vm); - } - } - - tcg_temp_free_i32(f0); - tcg_temp_free_i32(f1); - tcg_temp_free_i32(fd); - tcg_temp_free_ptr(fpst); - - return true; -} - -static bool do_vfp_3op_dp(DisasContext *s, VFPGen3OpDPFn *fn, - int vd, int vn, int vm, bool reads_vd) -{ - uint32_t delta_m = 0; - uint32_t delta_d = 0; - int veclen = s->vec_len; - TCGv_i64 f0, f1, fd; - TCGv_ptr fpst; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && ((vd | vn | vm) & 0x10)) { - return false; - } - - if (!dc_isar_feature(aa32_fpshvec, s) && - (veclen != 0 || s->vec_stride != 0)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - if (veclen > 0) { - /* Figure out what type of vector operation this is. */ - if (vfp_dreg_is_scalar(vd)) { - /* scalar */ - veclen = 0; - } else { - delta_d = (s->vec_stride >> 1) + 1; - - if (vfp_dreg_is_scalar(vm)) { - /* mixed scalar/vector */ - delta_m = 0; - } else { - /* vector */ - delta_m = delta_d; - } - } - } - - f0 = tcg_temp_new_i64(); - f1 = tcg_temp_new_i64(); - fd = tcg_temp_new_i64(); - fpst = get_fpstatus_ptr(0); - - neon_load_reg64(f0, vn); - neon_load_reg64(f1, vm); - - for (;;) { - if (reads_vd) { - neon_load_reg64(fd, vd); - } - fn(fd, f0, f1, fpst); - neon_store_reg64(fd, vd); - - if (veclen == 0) { - break; - } - /* Set up the operands for the next iteration */ - veclen--; - vd = vfp_advance_dreg(vd, delta_d); - vn = vfp_advance_dreg(vn, delta_d); - neon_load_reg64(f0, vn); - if (delta_m) { - vm = vfp_advance_dreg(vm, delta_m); - neon_load_reg64(f1, vm); - } - } - - tcg_temp_free_i64(f0); - tcg_temp_free_i64(f1); - tcg_temp_free_i64(fd); - tcg_temp_free_ptr(fpst); - - return true; -} - -static bool do_vfp_2op_sp(DisasContext *s, VFPGen2OpSPFn *fn, int vd, int vm) -{ - uint32_t delta_m = 0; - uint32_t delta_d = 0; - int veclen = s->vec_len; - TCGv_i32 f0, fd; - - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - if (!dc_isar_feature(aa32_fpshvec, s) && - (veclen != 0 || s->vec_stride != 0)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - if (veclen > 0) { - /* Figure out what type of vector operation this is. */ - if (vfp_sreg_is_scalar(vd)) { - /* scalar */ - veclen = 0; - } else { - delta_d = s->vec_stride + 1; - - if (vfp_sreg_is_scalar(vm)) { - /* mixed scalar/vector */ - delta_m = 0; - } else { - /* vector */ - delta_m = delta_d; - } - } - } - - f0 = tcg_temp_new_i32(); - fd = tcg_temp_new_i32(); - - neon_load_reg32(f0, vm); - - for (;;) { - fn(fd, f0); - neon_store_reg32(fd, vd); - - if (veclen == 0) { - break; - } - - if (delta_m == 0) { - /* single source one-many */ - while (veclen--) { - vd = vfp_advance_sreg(vd, delta_d); - neon_store_reg32(fd, vd); - } - break; - } - - /* Set up the operands for the next iteration */ - veclen--; - vd = vfp_advance_sreg(vd, delta_d); - vm = vfp_advance_sreg(vm, delta_m); - neon_load_reg32(f0, vm); - } - - tcg_temp_free_i32(f0); - tcg_temp_free_i32(fd); - - return true; -} - -static bool do_vfp_2op_dp(DisasContext *s, VFPGen2OpDPFn *fn, int vd, int vm) -{ - uint32_t delta_m = 0; - uint32_t delta_d = 0; - int veclen = s->vec_len; - TCGv_i64 f0, fd; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist */ - if (!dc_isar_feature(aa32_simd_r32, s) && ((vd | vm) & 0x10)) { - return false; - } - - if (!dc_isar_feature(aa32_fpshvec, s) && - (veclen != 0 || s->vec_stride != 0)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - if (veclen > 0) { - /* Figure out what type of vector operation this is. */ - if (vfp_dreg_is_scalar(vd)) { - /* scalar */ - veclen = 0; - } else { - delta_d = (s->vec_stride >> 1) + 1; - - if (vfp_dreg_is_scalar(vm)) { - /* mixed scalar/vector */ - delta_m = 0; - } else { - /* vector */ - delta_m = delta_d; - } - } - } - - f0 = tcg_temp_new_i64(); - fd = tcg_temp_new_i64(); - - neon_load_reg64(f0, vm); - - for (;;) { - fn(fd, f0); - neon_store_reg64(fd, vd); - - if (veclen == 0) { - break; - } - - if (delta_m == 0) { - /* single source one-many */ - while (veclen--) { - vd = vfp_advance_dreg(vd, delta_d); - neon_store_reg64(fd, vd); - } - break; - } - - /* Set up the operands for the next iteration */ - veclen--; - vd = vfp_advance_dreg(vd, delta_d); - vd = vfp_advance_dreg(vm, delta_m); - neon_load_reg64(f0, vm); - } - - tcg_temp_free_i64(f0); - tcg_temp_free_i64(fd); - - return true; -} - -static void gen_VMLA_sp(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst) -{ - /* Note that order of inputs to the add matters for NaNs */ - TCGv_i32 tmp = tcg_temp_new_i32(); - - gen_helper_vfp_muls(tmp, vn, vm, fpst); - gen_helper_vfp_adds(vd, vd, tmp, fpst); - tcg_temp_free_i32(tmp); -} - -static bool trans_VMLA_sp(DisasContext *s, arg_VMLA_sp *a) -{ - return do_vfp_3op_sp(s, gen_VMLA_sp, a->vd, a->vn, a->vm, true); -} - -static void gen_VMLA_dp(TCGv_i64 vd, TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst) -{ - /* Note that order of inputs to the add matters for NaNs */ - TCGv_i64 tmp = tcg_temp_new_i64(); - - gen_helper_vfp_muld(tmp, vn, vm, fpst); - gen_helper_vfp_addd(vd, vd, tmp, fpst); - tcg_temp_free_i64(tmp); -} - -static bool trans_VMLA_dp(DisasContext *s, arg_VMLA_dp *a) -{ - return do_vfp_3op_dp(s, gen_VMLA_dp, a->vd, a->vn, a->vm, true); -} - -static void gen_VMLS_sp(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst) -{ - /* - * VMLS: vd = vd + -(vn * vm) - * Note that order of inputs to the add matters for NaNs. - */ - TCGv_i32 tmp = tcg_temp_new_i32(); - - gen_helper_vfp_muls(tmp, vn, vm, fpst); - gen_helper_vfp_negs(tmp, tmp); - gen_helper_vfp_adds(vd, vd, tmp, fpst); - tcg_temp_free_i32(tmp); -} - -static bool trans_VMLS_sp(DisasContext *s, arg_VMLS_sp *a) -{ - return do_vfp_3op_sp(s, gen_VMLS_sp, a->vd, a->vn, a->vm, true); -} - -static void gen_VMLS_dp(TCGv_i64 vd, TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst) -{ - /* - * VMLS: vd = vd + -(vn * vm) - * Note that order of inputs to the add matters for NaNs. - */ - TCGv_i64 tmp = tcg_temp_new_i64(); - - gen_helper_vfp_muld(tmp, vn, vm, fpst); - gen_helper_vfp_negd(tmp, tmp); - gen_helper_vfp_addd(vd, vd, tmp, fpst); - tcg_temp_free_i64(tmp); -} - -static bool trans_VMLS_dp(DisasContext *s, arg_VMLS_dp *a) -{ - return do_vfp_3op_dp(s, gen_VMLS_dp, a->vd, a->vn, a->vm, true); -} - -static void gen_VNMLS_sp(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst) -{ - /* - * VNMLS: -fd + (fn * fm) - * Note that it isn't valid to replace (-A + B) with (B - A) or similar - * plausible looking simplifications because this will give wrong results - * for NaNs. - */ - TCGv_i32 tmp = tcg_temp_new_i32(); - - gen_helper_vfp_muls(tmp, vn, vm, fpst); - gen_helper_vfp_negs(vd, vd); - gen_helper_vfp_adds(vd, vd, tmp, fpst); - tcg_temp_free_i32(tmp); -} - -static bool trans_VNMLS_sp(DisasContext *s, arg_VNMLS_sp *a) -{ - return do_vfp_3op_sp(s, gen_VNMLS_sp, a->vd, a->vn, a->vm, true); -} - -static void gen_VNMLS_dp(TCGv_i64 vd, TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst) -{ - /* - * VNMLS: -fd + (fn * fm) - * Note that it isn't valid to replace (-A + B) with (B - A) or similar - * plausible looking simplifications because this will give wrong results - * for NaNs. - */ - TCGv_i64 tmp = tcg_temp_new_i64(); - - gen_helper_vfp_muld(tmp, vn, vm, fpst); - gen_helper_vfp_negd(vd, vd); - gen_helper_vfp_addd(vd, vd, tmp, fpst); - tcg_temp_free_i64(tmp); -} - -static bool trans_VNMLS_dp(DisasContext *s, arg_VNMLS_dp *a) -{ - return do_vfp_3op_dp(s, gen_VNMLS_dp, a->vd, a->vn, a->vm, true); -} - -static void gen_VNMLA_sp(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst) -{ - /* VNMLA: -fd + -(fn * fm) */ - TCGv_i32 tmp = tcg_temp_new_i32(); - - gen_helper_vfp_muls(tmp, vn, vm, fpst); - gen_helper_vfp_negs(tmp, tmp); - gen_helper_vfp_negs(vd, vd); - gen_helper_vfp_adds(vd, vd, tmp, fpst); - tcg_temp_free_i32(tmp); -} - -static bool trans_VNMLA_sp(DisasContext *s, arg_VNMLA_sp *a) -{ - return do_vfp_3op_sp(s, gen_VNMLA_sp, a->vd, a->vn, a->vm, true); -} - -static void gen_VNMLA_dp(TCGv_i64 vd, TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst) -{ - /* VNMLA: -fd + (fn * fm) */ - TCGv_i64 tmp = tcg_temp_new_i64(); - - gen_helper_vfp_muld(tmp, vn, vm, fpst); - gen_helper_vfp_negd(tmp, tmp); - gen_helper_vfp_negd(vd, vd); - gen_helper_vfp_addd(vd, vd, tmp, fpst); - tcg_temp_free_i64(tmp); -} - -static bool trans_VNMLA_dp(DisasContext *s, arg_VNMLA_dp *a) -{ - return do_vfp_3op_dp(s, gen_VNMLA_dp, a->vd, a->vn, a->vm, true); -} - -static bool trans_VMUL_sp(DisasContext *s, arg_VMUL_sp *a) -{ - return do_vfp_3op_sp(s, gen_helper_vfp_muls, a->vd, a->vn, a->vm, false); -} - -static bool trans_VMUL_dp(DisasContext *s, arg_VMUL_dp *a) -{ - return do_vfp_3op_dp(s, gen_helper_vfp_muld, a->vd, a->vn, a->vm, false); -} - -static void gen_VNMUL_sp(TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm, TCGv_ptr fpst) -{ - /* VNMUL: -(fn * fm) */ - gen_helper_vfp_muls(vd, vn, vm, fpst); - gen_helper_vfp_negs(vd, vd); -} - -static bool trans_VNMUL_sp(DisasContext *s, arg_VNMUL_sp *a) -{ - return do_vfp_3op_sp(s, gen_VNMUL_sp, a->vd, a->vn, a->vm, false); -} - -static void gen_VNMUL_dp(TCGv_i64 vd, TCGv_i64 vn, TCGv_i64 vm, TCGv_ptr fpst) -{ - /* VNMUL: -(fn * fm) */ - gen_helper_vfp_muld(vd, vn, vm, fpst); - gen_helper_vfp_negd(vd, vd); -} - -static bool trans_VNMUL_dp(DisasContext *s, arg_VNMUL_dp *a) -{ - return do_vfp_3op_dp(s, gen_VNMUL_dp, a->vd, a->vn, a->vm, false); -} - -static bool trans_VADD_sp(DisasContext *s, arg_VADD_sp *a) -{ - return do_vfp_3op_sp(s, gen_helper_vfp_adds, a->vd, a->vn, a->vm, false); -} - -static bool trans_VADD_dp(DisasContext *s, arg_VADD_dp *a) -{ - return do_vfp_3op_dp(s, gen_helper_vfp_addd, a->vd, a->vn, a->vm, false); -} - -static bool trans_VSUB_sp(DisasContext *s, arg_VSUB_sp *a) -{ - return do_vfp_3op_sp(s, gen_helper_vfp_subs, a->vd, a->vn, a->vm, false); -} - -static bool trans_VSUB_dp(DisasContext *s, arg_VSUB_dp *a) -{ - return do_vfp_3op_dp(s, gen_helper_vfp_subd, a->vd, a->vn, a->vm, false); -} - -static bool trans_VDIV_sp(DisasContext *s, arg_VDIV_sp *a) -{ - return do_vfp_3op_sp(s, gen_helper_vfp_divs, a->vd, a->vn, a->vm, false); -} - -static bool trans_VDIV_dp(DisasContext *s, arg_VDIV_dp *a) -{ - return do_vfp_3op_dp(s, gen_helper_vfp_divd, a->vd, a->vn, a->vm, false); -} - -static bool trans_VMINNM_sp(DisasContext *s, arg_VMINNM_sp *a) -{ - if (!dc_isar_feature(aa32_vminmaxnm, s)) { - return false; - } - return do_vfp_3op_sp(s, gen_helper_vfp_minnums, - a->vd, a->vn, a->vm, false); -} - -static bool trans_VMAXNM_sp(DisasContext *s, arg_VMAXNM_sp *a) -{ - if (!dc_isar_feature(aa32_vminmaxnm, s)) { - return false; - } - return do_vfp_3op_sp(s, gen_helper_vfp_maxnums, - a->vd, a->vn, a->vm, false); -} - -static bool trans_VMINNM_dp(DisasContext *s, arg_VMINNM_dp *a) -{ - if (!dc_isar_feature(aa32_vminmaxnm, s)) { - return false; - } - return do_vfp_3op_dp(s, gen_helper_vfp_minnumd, - a->vd, a->vn, a->vm, false); -} - -static bool trans_VMAXNM_dp(DisasContext *s, arg_VMAXNM_dp *a) -{ - if (!dc_isar_feature(aa32_vminmaxnm, s)) { - return false; - } - return do_vfp_3op_dp(s, gen_helper_vfp_maxnumd, - a->vd, a->vn, a->vm, false); -} - -static bool do_vfm_sp(DisasContext *s, arg_VFMA_sp *a, bool neg_n, bool neg_d) -{ - /* - * VFNMA : fd = muladd(-fd, fn, fm) - * VFNMS : fd = muladd(-fd, -fn, fm) - * VFMA : fd = muladd( fd, fn, fm) - * VFMS : fd = muladd( fd, -fn, fm) - * - * These are fused multiply-add, and must be done as one floating - * point operation with no rounding between the multiplication and - * addition steps. NB that doing the negations here as separate - * steps is correct : an input NaN should come out with its sign - * bit flipped if it is a negated-input. - */ - TCGv_ptr fpst; - TCGv_i32 vn, vm, vd; - - /* - * Present in VFPv4 only. - * Note that we can't rely on the SIMDFMAC check alone, because - * in a Neon-no-VFP core that ID register field will be non-zero. - */ - if (!dc_isar_feature(aa32_simdfmac, s) || - !dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - /* - * In v7A, UNPREDICTABLE with non-zero vector length/stride; from - * v8A, must UNDEF. We choose to UNDEF for both v7A and v8A. - */ - if (s->vec_len != 0 || s->vec_stride != 0) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - vn = tcg_temp_new_i32(); - vm = tcg_temp_new_i32(); - vd = tcg_temp_new_i32(); - - neon_load_reg32(vn, a->vn); - neon_load_reg32(vm, a->vm); - if (neg_n) { - /* VFNMS, VFMS */ - gen_helper_vfp_negs(vn, vn); - } - neon_load_reg32(vd, a->vd); - if (neg_d) { - /* VFNMA, VFNMS */ - gen_helper_vfp_negs(vd, vd); - } - fpst = get_fpstatus_ptr(0); - gen_helper_vfp_muladds(vd, vn, vm, vd, fpst); - neon_store_reg32(vd, a->vd); - - tcg_temp_free_ptr(fpst); - tcg_temp_free_i32(vn); - tcg_temp_free_i32(vm); - tcg_temp_free_i32(vd); - - return true; -} - -static bool trans_VFMA_sp(DisasContext *s, arg_VFMA_sp *a) -{ - return do_vfm_sp(s, a, false, false); -} - -static bool trans_VFMS_sp(DisasContext *s, arg_VFMS_sp *a) -{ - return do_vfm_sp(s, a, true, false); -} - -static bool trans_VFNMA_sp(DisasContext *s, arg_VFNMA_sp *a) -{ - return do_vfm_sp(s, a, false, true); -} - -static bool trans_VFNMS_sp(DisasContext *s, arg_VFNMS_sp *a) -{ - return do_vfm_sp(s, a, true, true); -} - -static bool do_vfm_dp(DisasContext *s, arg_VFMA_dp *a, bool neg_n, bool neg_d) -{ - /* - * VFNMA : fd = muladd(-fd, fn, fm) - * VFNMS : fd = muladd(-fd, -fn, fm) - * VFMA : fd = muladd( fd, fn, fm) - * VFMS : fd = muladd( fd, -fn, fm) - * - * These are fused multiply-add, and must be done as one floating - * point operation with no rounding between the multiplication and - * addition steps. NB that doing the negations here as separate - * steps is correct : an input NaN should come out with its sign - * bit flipped if it is a negated-input. - */ - TCGv_ptr fpst; - TCGv_i64 vn, vm, vd; - - /* - * Present in VFPv4 only. - * Note that we can't rely on the SIMDFMAC check alone, because - * in a Neon-no-VFP core that ID register field will be non-zero. - */ - if (!dc_isar_feature(aa32_simdfmac, s) || - !dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - /* - * In v7A, UNPREDICTABLE with non-zero vector length/stride; from - * v8A, must UNDEF. We choose to UNDEF for both v7A and v8A. - */ - if (s->vec_len != 0 || s->vec_stride != 0) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && - ((a->vd | a->vn | a->vm) & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - vn = tcg_temp_new_i64(); - vm = tcg_temp_new_i64(); - vd = tcg_temp_new_i64(); - - neon_load_reg64(vn, a->vn); - neon_load_reg64(vm, a->vm); - if (neg_n) { - /* VFNMS, VFMS */ - gen_helper_vfp_negd(vn, vn); - } - neon_load_reg64(vd, a->vd); - if (neg_d) { - /* VFNMA, VFNMS */ - gen_helper_vfp_negd(vd, vd); - } - fpst = get_fpstatus_ptr(0); - gen_helper_vfp_muladdd(vd, vn, vm, vd, fpst); - neon_store_reg64(vd, a->vd); - - tcg_temp_free_ptr(fpst); - tcg_temp_free_i64(vn); - tcg_temp_free_i64(vm); - tcg_temp_free_i64(vd); - - return true; -} - -static bool trans_VFMA_dp(DisasContext *s, arg_VFMA_dp *a) -{ - return do_vfm_dp(s, a, false, false); -} - -static bool trans_VFMS_dp(DisasContext *s, arg_VFMS_dp *a) -{ - return do_vfm_dp(s, a, true, false); -} - -static bool trans_VFNMA_dp(DisasContext *s, arg_VFNMA_dp *a) -{ - return do_vfm_dp(s, a, false, true); -} - -static bool trans_VFNMS_dp(DisasContext *s, arg_VFNMS_dp *a) -{ - return do_vfm_dp(s, a, true, true); -} - -static bool trans_VMOV_imm_sp(DisasContext *s, arg_VMOV_imm_sp *a) -{ - uint32_t delta_d = 0; - int veclen = s->vec_len; - TCGv_i32 fd; - uint32_t vd; - - vd = a->vd; - - if (!dc_isar_feature(aa32_fpsp_v3, s)) { - return false; - } - - if (!dc_isar_feature(aa32_fpshvec, s) && - (veclen != 0 || s->vec_stride != 0)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - if (veclen > 0) { - /* Figure out what type of vector operation this is. */ - if (vfp_sreg_is_scalar(vd)) { - /* scalar */ - veclen = 0; - } else { - delta_d = s->vec_stride + 1; - } - } - - fd = tcg_const_i32(vfp_expand_imm(MO_32, a->imm)); - - for (;;) { - neon_store_reg32(fd, vd); - - if (veclen == 0) { - break; - } - - /* Set up the operands for the next iteration */ - veclen--; - vd = vfp_advance_sreg(vd, delta_d); - } - - tcg_temp_free_i32(fd); - return true; -} - -static bool trans_VMOV_imm_dp(DisasContext *s, arg_VMOV_imm_dp *a) -{ - uint32_t delta_d = 0; - int veclen = s->vec_len; - TCGv_i64 fd; - uint32_t vd; - - vd = a->vd; - - if (!dc_isar_feature(aa32_fpdp_v3, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && (vd & 0x10)) { - return false; - } - - if (!dc_isar_feature(aa32_fpshvec, s) && - (veclen != 0 || s->vec_stride != 0)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - if (veclen > 0) { - /* Figure out what type of vector operation this is. */ - if (vfp_dreg_is_scalar(vd)) { - /* scalar */ - veclen = 0; - } else { - delta_d = (s->vec_stride >> 1) + 1; - } - } - - fd = tcg_const_i64(vfp_expand_imm(MO_64, a->imm)); - - for (;;) { - neon_store_reg64(fd, vd); - - if (veclen == 0) { - break; - } - - /* Set up the operands for the next iteration */ - veclen--; - vd = vfp_advance_dreg(vd, delta_d); - } - - tcg_temp_free_i64(fd); - return true; -} - -static bool trans_VMOV_reg_sp(DisasContext *s, arg_VMOV_reg_sp *a) -{ - return do_vfp_2op_sp(s, tcg_gen_mov_i32, a->vd, a->vm); -} - -static bool trans_VMOV_reg_dp(DisasContext *s, arg_VMOV_reg_dp *a) -{ - return do_vfp_2op_dp(s, tcg_gen_mov_i64, a->vd, a->vm); -} - -static bool trans_VABS_sp(DisasContext *s, arg_VABS_sp *a) -{ - return do_vfp_2op_sp(s, gen_helper_vfp_abss, a->vd, a->vm); -} - -static bool trans_VABS_dp(DisasContext *s, arg_VABS_dp *a) -{ - return do_vfp_2op_dp(s, gen_helper_vfp_absd, a->vd, a->vm); -} - -static bool trans_VNEG_sp(DisasContext *s, arg_VNEG_sp *a) -{ - return do_vfp_2op_sp(s, gen_helper_vfp_negs, a->vd, a->vm); -} - -static bool trans_VNEG_dp(DisasContext *s, arg_VNEG_dp *a) -{ - return do_vfp_2op_dp(s, gen_helper_vfp_negd, a->vd, a->vm); -} - -static void gen_VSQRT_sp(TCGv_i32 vd, TCGv_i32 vm) -{ - gen_helper_vfp_sqrts(vd, vm, cpu_env); -} - -static bool trans_VSQRT_sp(DisasContext *s, arg_VSQRT_sp *a) -{ - return do_vfp_2op_sp(s, gen_VSQRT_sp, a->vd, a->vm); -} - -static void gen_VSQRT_dp(TCGv_i64 vd, TCGv_i64 vm) -{ - gen_helper_vfp_sqrtd(vd, vm, cpu_env); -} - -static bool trans_VSQRT_dp(DisasContext *s, arg_VSQRT_dp *a) -{ - return do_vfp_2op_dp(s, gen_VSQRT_dp, a->vd, a->vm); -} - -static bool trans_VCMP_sp(DisasContext *s, arg_VCMP_sp *a) -{ - TCGv_i32 vd, vm; - - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - /* Vm/M bits must be zero for the Z variant */ - if (a->z && a->vm != 0) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - vd = tcg_temp_new_i32(); - vm = tcg_temp_new_i32(); - - neon_load_reg32(vd, a->vd); - if (a->z) { - tcg_gen_movi_i32(vm, 0); - } else { - neon_load_reg32(vm, a->vm); - } - - if (a->e) { - gen_helper_vfp_cmpes(vd, vm, cpu_env); - } else { - gen_helper_vfp_cmps(vd, vm, cpu_env); - } - - tcg_temp_free_i32(vd); - tcg_temp_free_i32(vm); - - return true; -} - -static bool trans_VCMP_dp(DisasContext *s, arg_VCMP_dp *a) -{ - TCGv_i64 vd, vm; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - /* Vm/M bits must be zero for the Z variant */ - if (a->z && a->vm != 0) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - vd = tcg_temp_new_i64(); - vm = tcg_temp_new_i64(); - - neon_load_reg64(vd, a->vd); - if (a->z) { - tcg_gen_movi_i64(vm, 0); - } else { - neon_load_reg64(vm, a->vm); - } - - if (a->e) { - gen_helper_vfp_cmped(vd, vm, cpu_env); - } else { - gen_helper_vfp_cmpd(vd, vm, cpu_env); - } - - tcg_temp_free_i64(vd); - tcg_temp_free_i64(vm); - - return true; -} - -static bool trans_VCVT_f32_f16(DisasContext *s, arg_VCVT_f32_f16 *a) -{ - TCGv_ptr fpst; - TCGv_i32 ahp_mode; - TCGv_i32 tmp; - - if (!dc_isar_feature(aa32_fp16_spconv, s)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(false); - ahp_mode = get_ahp_flag(); - tmp = tcg_temp_new_i32(); - /* The T bit tells us if we want the low or high 16 bits of Vm */ - tcg_gen_ld16u_i32(tmp, cpu_env, vfp_f16_offset(a->vm, a->t)); - gen_helper_vfp_fcvt_f16_to_f32(tmp, tmp, fpst, ahp_mode); - neon_store_reg32(tmp, a->vd); - tcg_temp_free_i32(ahp_mode); - tcg_temp_free_ptr(fpst); - tcg_temp_free_i32(tmp); - return true; -} - -static bool trans_VCVT_f64_f16(DisasContext *s, arg_VCVT_f64_f16 *a) -{ - TCGv_ptr fpst; - TCGv_i32 ahp_mode; - TCGv_i32 tmp; - TCGv_i64 vd; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - if (!dc_isar_feature(aa32_fp16_dpconv, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(false); - ahp_mode = get_ahp_flag(); - tmp = tcg_temp_new_i32(); - /* The T bit tells us if we want the low or high 16 bits of Vm */ - tcg_gen_ld16u_i32(tmp, cpu_env, vfp_f16_offset(a->vm, a->t)); - vd = tcg_temp_new_i64(); - gen_helper_vfp_fcvt_f16_to_f64(vd, tmp, fpst, ahp_mode); - neon_store_reg64(vd, a->vd); - tcg_temp_free_i32(ahp_mode); - tcg_temp_free_ptr(fpst); - tcg_temp_free_i32(tmp); - tcg_temp_free_i64(vd); - return true; -} - -static bool trans_VCVT_f16_f32(DisasContext *s, arg_VCVT_f16_f32 *a) -{ - TCGv_ptr fpst; - TCGv_i32 ahp_mode; - TCGv_i32 tmp; - - if (!dc_isar_feature(aa32_fp16_spconv, s)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(false); - ahp_mode = get_ahp_flag(); - tmp = tcg_temp_new_i32(); - - neon_load_reg32(tmp, a->vm); - gen_helper_vfp_fcvt_f32_to_f16(tmp, tmp, fpst, ahp_mode); - tcg_gen_st16_i32(tmp, cpu_env, vfp_f16_offset(a->vd, a->t)); - tcg_temp_free_i32(ahp_mode); - tcg_temp_free_ptr(fpst); - tcg_temp_free_i32(tmp); - return true; -} - -static bool trans_VCVT_f16_f64(DisasContext *s, arg_VCVT_f16_f64 *a) -{ - TCGv_ptr fpst; - TCGv_i32 ahp_mode; - TCGv_i32 tmp; - TCGv_i64 vm; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - if (!dc_isar_feature(aa32_fp16_dpconv, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(false); - ahp_mode = get_ahp_flag(); - tmp = tcg_temp_new_i32(); - vm = tcg_temp_new_i64(); - - neon_load_reg64(vm, a->vm); - gen_helper_vfp_fcvt_f64_to_f16(tmp, vm, fpst, ahp_mode); - tcg_temp_free_i64(vm); - tcg_gen_st16_i32(tmp, cpu_env, vfp_f16_offset(a->vd, a->t)); - tcg_temp_free_i32(ahp_mode); - tcg_temp_free_ptr(fpst); - tcg_temp_free_i32(tmp); - return true; -} - -static bool trans_VRINTR_sp(DisasContext *s, arg_VRINTR_sp *a) -{ - TCGv_ptr fpst; - TCGv_i32 tmp; - - if (!dc_isar_feature(aa32_vrint, s)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - tmp = tcg_temp_new_i32(); - neon_load_reg32(tmp, a->vm); - fpst = get_fpstatus_ptr(false); - gen_helper_rints(tmp, tmp, fpst); - neon_store_reg32(tmp, a->vd); - tcg_temp_free_ptr(fpst); - tcg_temp_free_i32(tmp); - return true; -} - -static bool trans_VRINTR_dp(DisasContext *s, arg_VRINTR_dp *a) -{ - TCGv_ptr fpst; - TCGv_i64 tmp; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - if (!dc_isar_feature(aa32_vrint, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - tmp = tcg_temp_new_i64(); - neon_load_reg64(tmp, a->vm); - fpst = get_fpstatus_ptr(false); - gen_helper_rintd(tmp, tmp, fpst); - neon_store_reg64(tmp, a->vd); - tcg_temp_free_ptr(fpst); - tcg_temp_free_i64(tmp); - return true; -} - -static bool trans_VRINTZ_sp(DisasContext *s, arg_VRINTZ_sp *a) -{ - TCGv_ptr fpst; - TCGv_i32 tmp; - TCGv_i32 tcg_rmode; - - if (!dc_isar_feature(aa32_vrint, s)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - tmp = tcg_temp_new_i32(); - neon_load_reg32(tmp, a->vm); - fpst = get_fpstatus_ptr(false); - tcg_rmode = tcg_const_i32(float_round_to_zero); - gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); - gen_helper_rints(tmp, tmp, fpst); - gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); - neon_store_reg32(tmp, a->vd); - tcg_temp_free_ptr(fpst); - tcg_temp_free_i32(tcg_rmode); - tcg_temp_free_i32(tmp); - return true; -} - -static bool trans_VRINTZ_dp(DisasContext *s, arg_VRINTZ_dp *a) -{ - TCGv_ptr fpst; - TCGv_i64 tmp; - TCGv_i32 tcg_rmode; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - if (!dc_isar_feature(aa32_vrint, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - tmp = tcg_temp_new_i64(); - neon_load_reg64(tmp, a->vm); - fpst = get_fpstatus_ptr(false); - tcg_rmode = tcg_const_i32(float_round_to_zero); - gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); - gen_helper_rintd(tmp, tmp, fpst); - gen_helper_set_rmode(tcg_rmode, tcg_rmode, fpst); - neon_store_reg64(tmp, a->vd); - tcg_temp_free_ptr(fpst); - tcg_temp_free_i64(tmp); - tcg_temp_free_i32(tcg_rmode); - return true; -} - -static bool trans_VRINTX_sp(DisasContext *s, arg_VRINTX_sp *a) -{ - TCGv_ptr fpst; - TCGv_i32 tmp; - - if (!dc_isar_feature(aa32_vrint, s)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - tmp = tcg_temp_new_i32(); - neon_load_reg32(tmp, a->vm); - fpst = get_fpstatus_ptr(false); - gen_helper_rints_exact(tmp, tmp, fpst); - neon_store_reg32(tmp, a->vd); - tcg_temp_free_ptr(fpst); - tcg_temp_free_i32(tmp); - return true; -} - -static bool trans_VRINTX_dp(DisasContext *s, arg_VRINTX_dp *a) -{ - TCGv_ptr fpst; - TCGv_i64 tmp; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - if (!dc_isar_feature(aa32_vrint, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && ((a->vd | a->vm) & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - tmp = tcg_temp_new_i64(); - neon_load_reg64(tmp, a->vm); - fpst = get_fpstatus_ptr(false); - gen_helper_rintd_exact(tmp, tmp, fpst); - neon_store_reg64(tmp, a->vd); - tcg_temp_free_ptr(fpst); - tcg_temp_free_i64(tmp); - return true; -} - -static bool trans_VCVT_sp(DisasContext *s, arg_VCVT_sp *a) -{ - TCGv_i64 vd; - TCGv_i32 vm; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - vm = tcg_temp_new_i32(); - vd = tcg_temp_new_i64(); - neon_load_reg32(vm, a->vm); - gen_helper_vfp_fcvtds(vd, vm, cpu_env); - neon_store_reg64(vd, a->vd); - tcg_temp_free_i32(vm); - tcg_temp_free_i64(vd); - return true; -} - -static bool trans_VCVT_dp(DisasContext *s, arg_VCVT_dp *a) -{ - TCGv_i64 vm; - TCGv_i32 vd; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - vd = tcg_temp_new_i32(); - vm = tcg_temp_new_i64(); - neon_load_reg64(vm, a->vm); - gen_helper_vfp_fcvtsd(vd, vm, cpu_env); - neon_store_reg32(vd, a->vd); - tcg_temp_free_i32(vd); - tcg_temp_free_i64(vm); - return true; -} - -static bool trans_VCVT_int_sp(DisasContext *s, arg_VCVT_int_sp *a) -{ - TCGv_i32 vm; - TCGv_ptr fpst; - - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - vm = tcg_temp_new_i32(); - neon_load_reg32(vm, a->vm); - fpst = get_fpstatus_ptr(false); - if (a->s) { - /* i32 -> f32 */ - gen_helper_vfp_sitos(vm, vm, fpst); - } else { - /* u32 -> f32 */ - gen_helper_vfp_uitos(vm, vm, fpst); - } - neon_store_reg32(vm, a->vd); - tcg_temp_free_i32(vm); - tcg_temp_free_ptr(fpst); - return true; -} - -static bool trans_VCVT_int_dp(DisasContext *s, arg_VCVT_int_dp *a) -{ - TCGv_i32 vm; - TCGv_i64 vd; - TCGv_ptr fpst; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - vm = tcg_temp_new_i32(); - vd = tcg_temp_new_i64(); - neon_load_reg32(vm, a->vm); - fpst = get_fpstatus_ptr(false); - if (a->s) { - /* i32 -> f64 */ - gen_helper_vfp_sitod(vd, vm, fpst); - } else { - /* u32 -> f64 */ - gen_helper_vfp_uitod(vd, vm, fpst); - } - neon_store_reg64(vd, a->vd); - tcg_temp_free_i32(vm); - tcg_temp_free_i64(vd); - tcg_temp_free_ptr(fpst); - return true; -} - -static bool trans_VJCVT(DisasContext *s, arg_VJCVT *a) -{ - TCGv_i32 vd; - TCGv_i64 vm; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - if (!dc_isar_feature(aa32_jscvt, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - vm = tcg_temp_new_i64(); - vd = tcg_temp_new_i32(); - neon_load_reg64(vm, a->vm); - gen_helper_vjcvt(vd, vm, cpu_env); - neon_store_reg32(vd, a->vd); - tcg_temp_free_i64(vm); - tcg_temp_free_i32(vd); - return true; -} - -static bool trans_VCVT_fix_sp(DisasContext *s, arg_VCVT_fix_sp *a) -{ - TCGv_i32 vd, shift; - TCGv_ptr fpst; - int frac_bits; - - if (!dc_isar_feature(aa32_fpsp_v3, s)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - frac_bits = (a->opc & 1) ? (32 - a->imm) : (16 - a->imm); - - vd = tcg_temp_new_i32(); - neon_load_reg32(vd, a->vd); - - fpst = get_fpstatus_ptr(false); - shift = tcg_const_i32(frac_bits); - - /* Switch on op:U:sx bits */ - switch (a->opc) { - case 0: - gen_helper_vfp_shtos(vd, vd, shift, fpst); - break; - case 1: - gen_helper_vfp_sltos(vd, vd, shift, fpst); - break; - case 2: - gen_helper_vfp_uhtos(vd, vd, shift, fpst); - break; - case 3: - gen_helper_vfp_ultos(vd, vd, shift, fpst); - break; - case 4: - gen_helper_vfp_toshs_round_to_zero(vd, vd, shift, fpst); - break; - case 5: - gen_helper_vfp_tosls_round_to_zero(vd, vd, shift, fpst); - break; - case 6: - gen_helper_vfp_touhs_round_to_zero(vd, vd, shift, fpst); - break; - case 7: - gen_helper_vfp_touls_round_to_zero(vd, vd, shift, fpst); - break; - default: - g_assert_not_reached(); - } - - neon_store_reg32(vd, a->vd); - tcg_temp_free_i32(vd); - tcg_temp_free_i32(shift); - tcg_temp_free_ptr(fpst); - return true; -} - -static bool trans_VCVT_fix_dp(DisasContext *s, arg_VCVT_fix_dp *a) -{ - TCGv_i64 vd; - TCGv_i32 shift; - TCGv_ptr fpst; - int frac_bits; - - if (!dc_isar_feature(aa32_fpdp_v3, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vd & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - frac_bits = (a->opc & 1) ? (32 - a->imm) : (16 - a->imm); - - vd = tcg_temp_new_i64(); - neon_load_reg64(vd, a->vd); - - fpst = get_fpstatus_ptr(false); - shift = tcg_const_i32(frac_bits); - - /* Switch on op:U:sx bits */ - switch (a->opc) { - case 0: - gen_helper_vfp_shtod(vd, vd, shift, fpst); - break; - case 1: - gen_helper_vfp_sltod(vd, vd, shift, fpst); - break; - case 2: - gen_helper_vfp_uhtod(vd, vd, shift, fpst); - break; - case 3: - gen_helper_vfp_ultod(vd, vd, shift, fpst); - break; - case 4: - gen_helper_vfp_toshd_round_to_zero(vd, vd, shift, fpst); - break; - case 5: - gen_helper_vfp_tosld_round_to_zero(vd, vd, shift, fpst); - break; - case 6: - gen_helper_vfp_touhd_round_to_zero(vd, vd, shift, fpst); - break; - case 7: - gen_helper_vfp_tould_round_to_zero(vd, vd, shift, fpst); - break; - default: - g_assert_not_reached(); - } - - neon_store_reg64(vd, a->vd); - tcg_temp_free_i64(vd); - tcg_temp_free_i32(shift); - tcg_temp_free_ptr(fpst); - return true; -} - -static bool trans_VCVT_sp_int(DisasContext *s, arg_VCVT_sp_int *a) -{ - TCGv_i32 vm; - TCGv_ptr fpst; - - if (!dc_isar_feature(aa32_fpsp_v2, s)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(false); - vm = tcg_temp_new_i32(); - neon_load_reg32(vm, a->vm); - - if (a->s) { - if (a->rz) { - gen_helper_vfp_tosizs(vm, vm, fpst); - } else { - gen_helper_vfp_tosis(vm, vm, fpst); - } - } else { - if (a->rz) { - gen_helper_vfp_touizs(vm, vm, fpst); - } else { - gen_helper_vfp_touis(vm, vm, fpst); - } - } - neon_store_reg32(vm, a->vd); - tcg_temp_free_i32(vm); - tcg_temp_free_ptr(fpst); - return true; -} - -static bool trans_VCVT_dp_int(DisasContext *s, arg_VCVT_dp_int *a) -{ - TCGv_i32 vd; - TCGv_i64 vm; - TCGv_ptr fpst; - - if (!dc_isar_feature(aa32_fpdp_v2, s)) { - return false; - } - - /* UNDEF accesses to D16-D31 if they don't exist. */ - if (!dc_isar_feature(aa32_simd_r32, s) && (a->vm & 0x10)) { - return false; - } - - if (!vfp_access_check(s)) { - return true; - } - - fpst = get_fpstatus_ptr(false); - vm = tcg_temp_new_i64(); - vd = tcg_temp_new_i32(); - neon_load_reg64(vm, a->vm); - - if (a->s) { - if (a->rz) { - gen_helper_vfp_tosizd(vd, vm, fpst); - } else { - gen_helper_vfp_tosid(vd, vm, fpst); - } - } else { - if (a->rz) { - gen_helper_vfp_touizd(vd, vm, fpst); - } else { - gen_helper_vfp_touid(vd, vm, fpst); - } - } - neon_store_reg32(vd, a->vd); - tcg_temp_free_i32(vd); - tcg_temp_free_i64(vm); - tcg_temp_free_ptr(fpst); - return true; -} - -/* - * Decode VLLDM and VLSTM are nonstandard because: - * * if there is no FPU then these insns must NOP in - * Secure state and UNDEF in Nonsecure state - * * if there is an FPU then these insns do not have - * the usual behaviour that vfp_access_check() provides of - * being controlled by CPACR/NSACR enable bits or the - * lazy-stacking logic. - */ -static bool trans_VLLDM_VLSTM(DisasContext *s, arg_VLLDM_VLSTM *a) -{ - TCGv_i32 fptr; - - if (!arm_dc_feature(s, ARM_FEATURE_M) || - !arm_dc_feature(s, ARM_FEATURE_V8)) { - return false; - } - /* If not secure, UNDEF. */ - if (!s->v8m_secure) { - return false; - } - /* If no fpu, NOP. */ - if (!dc_isar_feature(aa32_vfp, s)) { - return true; - } - - fptr = load_reg(s, a->rn); - if (a->l) { - gen_helper_v7m_vlldm(cpu_env, fptr); - } else { - gen_helper_v7m_vlstm(cpu_env, fptr); - } - tcg_temp_free_i32(fptr); - - /* End the TB, because we have updated FP control bits */ - s->base.is_jmp = DISAS_UPDATE_EXIT; - return true; -} diff --git a/target/arm/translate.c b/target/arm/translate.c index c39a929..556588d 100644 --- a/target/arm/translate.c +++ b/target/arm/translate.c @@ -1176,8 +1176,8 @@ static TCGv_ptr vfp_reg_ptr(bool dp, int reg) #define ARM_CP_RW_BIT (1 << 20) /* Include the VFP and Neon decoders */ -#include "translate-vfp.inc.c" -#include "translate-neon.inc.c" +#include "translate-vfp.c.inc" +#include "translate-neon.c.inc" static inline void iwmmxt_load_reg(TCGv_i64 var, int reg) { @@ -5217,10 +5217,10 @@ static int t16_pop_list(DisasContext *s, int x) * Include the generated decoders. */ -#include "decode-a32.inc.c" -#include "decode-a32-uncond.inc.c" -#include "decode-t32.inc.c" -#include "decode-t16.inc.c" +#include "decode-a32.c.inc" +#include "decode-a32-uncond.c.inc" +#include "decode-t32.c.inc" +#include "decode-t16.c.inc" /* Helpers to swap operands for reverse-subtract. */ static void gen_rsb(TCGv_i32 dst, TCGv_i32 a, TCGv_i32 b) -- cgit v1.1