aboutsummaryrefslogtreecommitdiff
path: root/target/arm/tcg/translate-sve.c
diff options
context:
space:
mode:
Diffstat (limited to 'target/arm/tcg/translate-sve.c')
-rw-r--r--target/arm/tcg/translate-sve.c103
1 files changed, 70 insertions, 33 deletions
diff --git a/target/arm/tcg/translate-sve.c b/target/arm/tcg/translate-sve.c
index 2ed440a..07b827f 100644
--- a/target/arm/tcg/translate-sve.c
+++ b/target/arm/tcg/translate-sve.c
@@ -31,9 +31,9 @@ typedef void gen_helper_gvec_flags_3(TCGv_i32, TCGv_ptr, TCGv_ptr,
typedef void gen_helper_gvec_flags_4(TCGv_i32, TCGv_ptr, TCGv_ptr,
TCGv_ptr, TCGv_ptr, TCGv_i32);
-typedef void gen_helper_gvec_mem(TCGv_env, TCGv_ptr, TCGv_i64, TCGv_i32);
+typedef void gen_helper_gvec_mem(TCGv_env, TCGv_ptr, TCGv_i64, TCGv_i64);
typedef void gen_helper_gvec_mem_scatter(TCGv_env, TCGv_ptr, TCGv_ptr,
- TCGv_ptr, TCGv_i64, TCGv_i32);
+ TCGv_ptr, TCGv_i64, TCGv_i64);
/*
* Helpers for extracting complex instruction fields.
@@ -4883,17 +4883,16 @@ static const uint8_t dtype_esz[19] = {
4, 4, 4,
};
-uint32_t make_svemte_desc(DisasContext *s, unsigned vsz, uint32_t nregs,
+uint64_t make_svemte_desc(DisasContext *s, unsigned vsz, uint32_t nregs,
uint32_t msz, bool is_write, uint32_t data)
{
uint32_t sizem1;
- uint32_t desc = 0;
+ uint64_t desc = 0;
/* Assert all of the data fits, with or without MTE enabled. */
assert(nregs >= 1 && nregs <= 4);
sizem1 = (nregs << msz) - 1;
assert(sizem1 <= R_MTEDESC_SIZEM1_MASK >> R_MTEDESC_SIZEM1_SHIFT);
- assert(data < 1u << SVE_MTEDESC_SHIFT);
if (s->mte_active[0]) {
desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
@@ -4901,9 +4900,9 @@ uint32_t make_svemte_desc(DisasContext *s, unsigned vsz, uint32_t nregs,
desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
desc = FIELD_DP32(desc, MTEDESC, SIZEM1, sizem1);
- desc <<= SVE_MTEDESC_SHIFT;
+ desc <<= 32;
}
- return simd_desc(vsz, vsz, desc | data);
+ return simd_desc(vsz, vsz, data) | desc;
}
static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
@@ -4911,7 +4910,7 @@ static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
gen_helper_gvec_mem *fn)
{
TCGv_ptr t_pg;
- uint32_t desc;
+ uint64_t desc;
if (!s->mte_active[0]) {
addr = clean_data_tbi(s, addr);
@@ -4927,7 +4926,7 @@ static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
t_pg = tcg_temp_new_ptr();
tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg));
- fn(tcg_env, t_pg, addr, tcg_constant_i32(desc));
+ fn(tcg_env, t_pg, addr, tcg_constant_i64(desc));
}
/* Indexed by [mte][be][dtype][nreg] */
@@ -5379,7 +5378,7 @@ static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
unsigned vsz = vec_full_reg_size(s);
TCGv_ptr t_pg;
int poff;
- uint32_t desc;
+ uint64_t desc;
/* Load the first quadword using the normal predicated load helpers. */
if (!s->mte_active[0]) {
@@ -5410,7 +5409,7 @@ static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
gen_helper_gvec_mem *fn
= ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0];
desc = make_svemte_desc(s, 16, 1, dtype_msz(dtype), false, zt);
- fn(tcg_env, t_pg, addr, tcg_constant_i32(desc));
+ fn(tcg_env, t_pg, addr, tcg_constant_i64(desc));
/* Replicate that first quadword. */
if (vsz > 16) {
@@ -5453,7 +5452,7 @@ static void do_ldro(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
unsigned vsz_r32;
TCGv_ptr t_pg;
int poff, doff;
- uint32_t desc;
+ uint64_t desc;
if (vsz < 32) {
/*
@@ -5494,7 +5493,7 @@ static void do_ldro(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
gen_helper_gvec_mem *fn
= ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0];
desc = make_svemte_desc(s, 32, 1, dtype_msz(dtype), false, zt);
- fn(tcg_env, t_pg, addr, tcg_constant_i32(desc));
+ fn(tcg_env, t_pg, addr, tcg_constant_i64(desc));
/*
* Replicate that first octaword.
@@ -5828,14 +5827,14 @@ static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm,
TCGv_ptr t_zm = tcg_temp_new_ptr();
TCGv_ptr t_pg = tcg_temp_new_ptr();
TCGv_ptr t_zt = tcg_temp_new_ptr();
- uint32_t desc;
+ uint64_t desc;
tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg));
tcg_gen_addi_ptr(t_zm, tcg_env, vec_full_reg_offset(s, zm));
tcg_gen_addi_ptr(t_zt, tcg_env, vec_full_reg_offset(s, zt));
desc = make_svemte_desc(s, vec_full_reg_size(s), 1, msz, is_write, scale);
- fn(tcg_env, t_zt, t_pg, t_zm, scalar, tcg_constant_i32(desc));
+ fn(tcg_env, t_zt, t_pg, t_zm, scalar, tcg_constant_i64(desc));
}
/* Indexed by [mte][be][ff][xs][u][msz]. */
@@ -6180,9 +6179,7 @@ static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a)
bool be = s->be_data == MO_BE;
bool mte = s->mte_active[0];
- if (a->esz < MO_128
- ? !dc_isar_feature(aa64_sve, s)
- : !dc_isar_feature(aa64_sve2p1, s)) {
+ if (!dc_isar_feature(aa64_sve, s)) {
return false;
}
s->is_nonstreaming = true;
@@ -6197,10 +6194,6 @@ static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a)
case MO_64:
fn = gather_load_fn64[mte][be][a->ff][a->xs][a->u][a->msz];
break;
- case MO_128:
- assert(!a->ff && a->u && a->xs == 2 && a->msz == MO_128);
- fn = gather_load_fn128[mte][be];
- break;
default:
g_assert_not_reached();
}
@@ -6211,6 +6204,32 @@ static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a)
return true;
}
+static bool trans_LD1Q(DisasContext *s, arg_LD1Q *a)
+{
+ gen_helper_gvec_mem_scatter *fn = NULL;
+ bool be = s->be_data == MO_BE;
+ bool mte = s->mte_active[0];
+
+ if (!dc_isar_feature(aa64_sve2p1, s)) {
+ return false;
+ }
+ s->is_nonstreaming = true;
+ if (!sve_access_check(s)) {
+ return true;
+ }
+
+ fn = gather_load_fn128[mte][be];
+ assert(fn != NULL);
+
+ /*
+ * Unlike LD1_zprz, a->rm is the scalar register and it can be XZR, not XSP.
+ * a->rn is the vector register.
+ */
+ do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
+ cpu_reg(s, a->rm), MO_128, false, fn);
+ return true;
+}
+
static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
{
gen_helper_gvec_mem_scatter *fn = NULL;
@@ -6387,9 +6406,7 @@ static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
if (a->esz < a->msz || (a->msz == 0 && a->scale)) {
return false;
}
- if (a->esz < MO_128
- ? !dc_isar_feature(aa64_sve, s)
- : !dc_isar_feature(aa64_sve2p1, s)) {
+ if (!dc_isar_feature(aa64_sve, s)) {
return false;
}
s->is_nonstreaming = true;
@@ -6403,10 +6420,6 @@ static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
case MO_64:
fn = scatter_store_fn64[mte][be][a->xs][a->msz];
break;
- case MO_128:
- assert(a->xs == 2 && a->msz == MO_128);
- fn = scatter_store_fn128[mte][be];
- break;
default:
g_assert_not_reached();
}
@@ -6415,6 +6428,29 @@ static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
return true;
}
+static bool trans_ST1Q(DisasContext *s, arg_ST1Q *a)
+{
+ gen_helper_gvec_mem_scatter *fn;
+ bool be = s->be_data == MO_BE;
+ bool mte = s->mte_active[0];
+
+ if (!dc_isar_feature(aa64_sve2p1, s)) {
+ return false;
+ }
+ s->is_nonstreaming = true;
+ if (!sve_access_check(s)) {
+ return true;
+ }
+ fn = scatter_store_fn128[mte][be];
+ /*
+ * Unlike ST1_zprz, a->rm is the scalar register, and it
+ * can be XZR, not XSP. a->rn is the vector register.
+ */
+ do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
+ cpu_reg(s, a->rm), MO_128, true, fn);
+ return true;
+}
+
static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
{
gen_helper_gvec_mem_scatter *fn = NULL;
@@ -8079,7 +8115,7 @@ static bool gen_ldst_c(DisasContext *s, TCGv_i64 addr, int zd, int png,
MemOp esz, bool is_write, int n, bool strided)
{
typedef void ldst_c_fn(TCGv_env, TCGv_ptr, TCGv_i64,
- TCGv_i32, TCGv_i32);
+ TCGv_i32, TCGv_i64);
static ldst_c_fn * const f_ldst[2][2][4] = {
{ { gen_helper_sve2p1_ld1bb_c,
gen_helper_sve2p1_ld1hh_le_c,
@@ -8100,9 +8136,10 @@ static bool gen_ldst_c(DisasContext *s, TCGv_i64 addr, int zd, int png,
gen_helper_sve2p1_st1dd_be_c, } }
};
- TCGv_i32 t_png, t_desc;
+ TCGv_i32 t_png;
+ TCGv_i64 t_desc;
TCGv_ptr t_zd;
- uint32_t desc, lg2_rstride = 0;
+ uint64_t desc, lg2_rstride = 0;
bool be = s->be_data == MO_BE;
assert(n == 2 || n == 4);
@@ -8132,7 +8169,7 @@ static bool gen_ldst_c(DisasContext *s, TCGv_i64 addr, int zd, int png,
desc = n == 2 ? 0 : 1;
desc = desc | (lg2_rstride << 1);
desc = make_svemte_desc(s, vec_full_reg_size(s), 1, esz, is_write, desc);
- t_desc = tcg_constant_i32(desc);
+ t_desc = tcg_constant_i64(desc);
t_png = tcg_temp_new_i32();
tcg_gen_ld16u_i32(t_png, tcg_env,
generated by cgit v1.1 at 2025-07-31 02:02:07 +0000