/* * ARM SVE Load/Store Helpers * * Copyright (c) 2018-2022 Linaro * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, see . */ #ifndef TARGET_ARM_SVE_LDST_INTERNAL_H #define TARGET_ARM_SVE_LDST_INTERNAL_H #include "accel/tcg/cpu-ldst.h" /* * Load one element into @vd + @reg_off from @host. * The controlling predicate is known to be true. */ typedef void sve_ldst1_host_fn(void *vd, intptr_t reg_off, void *host); /* * Load one element into @vd + @reg_off from (@env, @vaddr, @ra). * The controlling predicate is known to be true. */ typedef void sve_ldst1_tlb_fn(CPUARMState *env, void *vd, intptr_t reg_off, target_ulong vaddr, uintptr_t retaddr); /* * Generate the above primitives. */ #define DO_LD_HOST(NAME, H, TYPEE, TYPEM, HOST) \ static inline void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ { TYPEM val = HOST(host); *(TYPEE *)(vd + H(reg_off)) = val; } #define DO_ST_HOST(NAME, H, TYPEE, TYPEM, HOST) \ static inline void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ { TYPEM val = *(TYPEE *)(vd + H(reg_off)); HOST(host, val); } #define DO_LD_TLB(NAME, H, TYPEE, TYPEM, TLB) \ static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \ intptr_t reg_off, target_ulong addr, uintptr_t ra) \ { \ TYPEM val = TLB(env, useronly_clean_ptr(addr), ra); \ *(TYPEE *)(vd + H(reg_off)) = val; \ } #define DO_ST_TLB(NAME, H, TYPEE, TYPEM, TLB) \ static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \ intptr_t reg_off, target_ulong addr, uintptr_t ra) \ { \ TYPEM val = *(TYPEE *)(vd + H(reg_off)); \ TLB(env, useronly_clean_ptr(addr), val, ra); \ } #define DO_LD_PRIM_1(NAME, H, TE, TM) \ DO_LD_HOST(NAME, H, TE, TM, ldub_p) \ DO_LD_TLB(NAME, H, TE, TM, cpu_ldub_data_ra) DO_LD_PRIM_1(ld1bb, H1, uint8_t, uint8_t) DO_LD_PRIM_1(ld1bhu, H1_2, uint16_t, uint8_t) DO_LD_PRIM_1(ld1bhs, H1_2, uint16_t, int8_t) DO_LD_PRIM_1(ld1bsu, H1_4, uint32_t, uint8_t) DO_LD_PRIM_1(ld1bss, H1_4, uint32_t, int8_t) DO_LD_PRIM_1(ld1bdu, H1_8, uint64_t, uint8_t) DO_LD_PRIM_1(ld1bds, H1_8, uint64_t, int8_t) #define DO_ST_PRIM_1(NAME, H, TE, TM) \ DO_ST_HOST(st1##NAME, H, TE, TM, stb_p) \ DO_ST_TLB(st1##NAME, H, TE, TM, cpu_stb_data_ra) DO_ST_PRIM_1(bb, H1, uint8_t, uint8_t) DO_ST_PRIM_1(bh, H1_2, uint16_t, uint8_t) DO_ST_PRIM_1(bs, H1_4, uint32_t, uint8_t) DO_ST_PRIM_1(bd, H1_8, uint64_t, uint8_t) #define DO_LD_PRIM_2(NAME, H, TE, TM, LD) \ DO_LD_HOST(ld1##NAME##_be, H, TE, TM, LD##_be_p) \ DO_LD_HOST(ld1##NAME##_le, H, TE, TM, LD##_le_p) \ DO_LD_TLB(ld1##NAME##_be, H, TE, TM, cpu_##LD##_be_data_ra) \ DO_LD_TLB(ld1##NAME##_le, H, TE, TM, cpu_##LD##_le_data_ra) #define DO_ST_PRIM_2(NAME, H, TE, TM, ST) \ DO_ST_HOST(st1##NAME##_be, H, TE, TM, ST##_be_p) \ DO_ST_HOST(st1##NAME##_le, H, TE, TM, ST##_le_p) \ DO_ST_TLB(st1##NAME##_be, H, TE, TM, cpu_##ST##_be_data_ra) \ DO_ST_TLB(st1##NAME##_le, H, TE, TM, cpu_##ST##_le_data_ra) DO_LD_PRIM_2(hh, H1_2, uint16_t, uint16_t, lduw) DO_LD_PRIM_2(hsu, H1_4, uint32_t, uint16_t, lduw) DO_LD_PRIM_2(hss, H1_4, uint32_t, int16_t, lduw) DO_LD_PRIM_2(hdu, H1_8, uint64_t, uint16_t, lduw) DO_LD_PRIM_2(hds, H1_8, uint64_t, int16_t, lduw) DO_ST_PRIM_2(hh, H1_2, uint16_t, uint16_t, stw) DO_ST_PRIM_2(hs, H1_4, uint32_t, uint16_t, stw) DO_ST_PRIM_2(hd, H1_8, uint64_t, uint16_t, stw) DO_LD_PRIM_2(ss, H1_4, uint32_t, uint32_t, ldl) DO_LD_PRIM_2(sdu, H1_8, uint64_t, uint32_t, ldl) DO_LD_PRIM_2(sds, H1_8, uint64_t, int32_t, ldl) DO_ST_PRIM_2(ss, H1_4, uint32_t, uint32_t, stl) DO_ST_PRIM_2(sd, H1_8, uint64_t, uint32_t, stl) DO_LD_PRIM_2(dd, H1_8, uint64_t, uint64_t, ldq) DO_ST_PRIM_2(dd, H1_8, uint64_t, uint64_t, stq) #define DO_LD_PRIM_3(NAME, FUNC) \ static inline void sve_##NAME##_host(void *vd, \ intptr_t reg_off, void *host) \ { sve_##FUNC##_host(vd, reg_off, host); \ *(uint64_t *)(vd + reg_off + 8) = 0; } \ static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \ intptr_t reg_off, target_ulong addr, uintptr_t ra) \ { sve_##FUNC##_tlb(env, vd, reg_off, addr, ra); \ *(uint64_t *)(vd + reg_off + 8) = 0; } DO_LD_PRIM_3(ld1squ_be, ld1sdu_be) DO_LD_PRIM_3(ld1squ_le, ld1sdu_le) DO_LD_PRIM_3(ld1dqu_be, ld1dd_be) DO_LD_PRIM_3(ld1dqu_le, ld1dd_le) #define sve_st1sq_be_host sve_st1sd_be_host #define sve_st1sq_le_host sve_st1sd_le_host #define sve_st1sq_be_tlb sve_st1sd_be_tlb #define sve_st1sq_le_tlb sve_st1sd_le_tlb #define sve_st1dq_be_host sve_st1dd_be_host #define sve_st1dq_le_host sve_st1dd_le_host #define sve_st1dq_be_tlb sve_st1dd_be_tlb #define sve_st1dq_le_tlb sve_st1dd_le_tlb /* * The ARMVectorReg elements are stored in host-endian 64-bit units. * For 128-bit quantities, the sequence defined by the Elem[] pseudocode * corresponds to storing the two 64-bit pieces in little-endian order. */ /* FIXME: Nothing in this file makes any effort at atomicity. */ static inline void sve_ld1qq_be_host(void *vd, intptr_t reg_off, void *host) { sve_ld1dd_be_host(vd, reg_off + 8, host); sve_ld1dd_be_host(vd, reg_off, host + 8); } static inline void sve_ld1qq_le_host(void *vd, intptr_t reg_off, void *host) { sve_ld1dd_le_host(vd, reg_off, host); sve_ld1dd_le_host(vd, reg_off + 8, host + 8); } static inline void sve_ld1qq_be_tlb(CPUARMState *env, void *vd, intptr_t reg_off, target_ulong addr, uintptr_t ra) { sve_ld1dd_be_tlb(env, vd, reg_off + 8, addr, ra); sve_ld1dd_be_tlb(env, vd, reg_off, addr + 8, ra); } static inline void sve_ld1qq_le_tlb(CPUARMState *env, void *vd, intptr_t reg_off, target_ulong addr, uintptr_t ra) { sve_ld1dd_le_tlb(env, vd, reg_off, addr, ra); sve_ld1dd_le_tlb(env, vd, reg_off + 8, addr + 8, ra); } static inline void sve_st1qq_be_host(void *vd, intptr_t reg_off, void *host) { sve_st1dd_be_host(vd, reg_off + 8, host); sve_st1dd_be_host(vd, reg_off, host + 8); } static inline void sve_st1qq_le_host(void *vd, intptr_t reg_off, void *host) { sve_st1dd_le_host(vd, reg_off, host); sve_st1dd_le_host(vd, reg_off + 8, host + 8); } static inline void sve_st1qq_be_tlb(CPUARMState *env, void *vd, intptr_t reg_off, target_ulong addr, uintptr_t ra) { sve_st1dd_be_tlb(env, vd, reg_off + 8, addr, ra); sve_st1dd_be_tlb(env, vd, reg_off, addr + 8, ra); } static inline void sve_st1qq_le_tlb(CPUARMState *env, void *vd, intptr_t reg_off, target_ulong addr, uintptr_t ra) { sve_st1dd_le_tlb(env, vd, reg_off, addr, ra); sve_st1dd_le_tlb(env, vd, reg_off + 8, addr + 8, ra); } #undef DO_LD_TLB #undef DO_ST_TLB #undef DO_LD_HOST #undef DO_LD_PRIM_1 #undef DO_ST_PRIM_1 #undef DO_LD_PRIM_2 #undef DO_ST_PRIM_2 #undef DO_LD_PRIM_3 /* * Resolve the guest virtual address to info->host and info->flags. * If @nofault, return false if the page is invalid, otherwise * exit via page fault exception. */ typedef struct { void *host; int flags; MemTxAttrs attrs; bool tagged; } SVEHostPage; bool sve_probe_page(SVEHostPage *info, bool nofault, CPUARMState *env, target_ulong addr, int mem_off, MMUAccessType access_type, int mmu_idx, uintptr_t retaddr); /* * Analyse contiguous data, protected by a governing predicate. */ typedef enum { FAULT_NO, FAULT_FIRST, FAULT_ALL, } SVEContFault; typedef struct { /* * First and last element wholly contained within the two pages. * mem_off_first[0] and reg_off_first[0] are always set >= 0. * reg_off_last[0] may be < 0 if the first element crosses pages. * All of mem_off_first[1], reg_off_first[1] and reg_off_last[1] * are set >= 0 only if there are complete elements on a second page. * * The reg_off_* offsets are relative to the internal vector register. * The mem_off_first offset is relative to the memory address; the * two offsets are different when a load operation extends, a store * operation truncates, or for multi-register operations. */ int16_t mem_off_first[2]; int16_t reg_off_first[2]; int16_t reg_off_last[2]; /* * One element that is misaligned and spans both pages, * or -1 if there is no such active element. */ int16_t mem_off_split; int16_t reg_off_split; /* * The byte offset at which the entire operation crosses a page boundary. * Set >= 0 if and only if the entire operation spans two pages. */ int16_t page_split; /* TLB data for the two pages. */ SVEHostPage page[2]; } SVEContLdSt; /* * Find first active element on each page, and a loose bound for the * final element on each page. Identify any single element that spans * the page boundary. Return true if there are any active elements. */ bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, uint64_t *vg, intptr_t reg_max, int esz, int msize); /* * Resolve the guest virtual addresses to info->page[]. * Control the generation of page faults with @fault. Return false if * there is no work to do, which can only happen with @fault == FAULT_NO. */ bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault, CPUARMState *env, target_ulong addr, MMUAccessType access_type, uintptr_t retaddr); #ifdef CONFIG_USER_ONLY static inline void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, uint64_t *vg, target_ulong addr, int esize, int msize, int wp_access, uintptr_t retaddr) { } #else void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, uint64_t *vg, target_ulong addr, int esize, int msize, int wp_access, uintptr_t retaddr); #endif void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, uint64_t *vg, target_ulong addr, int esize, int msize, uint32_t mtedesc, uintptr_t ra); #endif /* TARGET_ARM_SVE_LDST_INTERNAL_H */