diff options
Diffstat (limited to 'pk')
| -rw-r--r-- | pk/atomic.h | 72 | ||||
| -rw-r--r-- | pk/bbl.c | 36 | ||||
| -rw-r--r-- | pk/bits.h | 45 | ||||
| -rw-r--r-- | pk/boot.h | 10 | ||||
| -rw-r--r-- | pk/configstring.c | 200 | ||||
| -rw-r--r-- | pk/elf.c | 137 | ||||
| -rw-r--r-- | pk/elf.h | 8 | ||||
| -rw-r--r-- | pk/emulation.c | 313 | ||||
| -rw-r--r-- | pk/emulation.h | 28 | ||||
| -rw-r--r-- | pk/encoding.h | 1037 | ||||
| -rw-r--r-- | pk/file.c | 10 | ||||
| -rw-r--r-- | pk/file.h | 1 | ||||
| -rw-r--r-- | pk/fp_asm.S | 160 | ||||
| -rw-r--r-- | pk/fp_emulation.c | 449 | ||||
| -rw-r--r-- | pk/fp_emulation.h | 81 | ||||
| -rw-r--r-- | pk/frontend.h | 12 | ||||
| -rw-r--r-- | pk/handlers.c | 8 | ||||
| -rw-r--r-- | pk/init.c | 82 | ||||
| -rw-r--r-- | pk/logo.c | 32 | ||||
| -rw-r--r-- | pk/mcall.h | 21 | ||||
| -rw-r--r-- | pk/mentry.S | 244 | ||||
| -rw-r--r-- | pk/minit.c | 108 | ||||
| -rw-r--r-- | pk/mmap.c (renamed from pk/vm.c) | 186 | ||||
| -rw-r--r-- | pk/mmap.h (renamed from pk/vm.h) | 31 | ||||
| -rw-r--r-- | pk/mtrap.c | 281 | ||||
| -rw-r--r-- | pk/mtrap.h | 86 | ||||
| -rw-r--r-- | pk/pk.ac | 10 | ||||
| -rw-r--r-- | pk/pk.c | 95 | ||||
| -rw-r--r-- | pk/pk.h | 17 | ||||
| -rw-r--r-- | pk/pk.lds (renamed from pk/pk.ld) | 5 | ||||
| -rw-r--r-- | pk/pk.mk.in | 30 | ||||
| -rw-r--r-- | pk/sbi.S | 15 | ||||
| -rw-r--r-- | pk/sbi.h | 30 | ||||
| -rw-r--r-- | pk/sbi_entry.S | 109 | ||||
| -rw-r--r-- | pk/sbi_impl.c | 34 | ||||
| -rw-r--r-- | pk/snprintf.c | 98 | ||||
| -rw-r--r-- | pk/string.c | 87 | ||||
| -rw-r--r-- | pk/syscall.c | 2 | ||||
| -rw-r--r-- | pk/unprivileged_memory.h | 79 |
39 files changed, 213 insertions, 4076 deletions
diff --git a/pk/atomic.h b/pk/atomic.h deleted file mode 100644 index e4610de..0000000 --- a/pk/atomic.h +++ /dev/null @@ -1,72 +0,0 @@ -// See LICENSE for license details. - -#ifndef _RISCV_ATOMIC_H -#define _RISCV_ATOMIC_H - -#include "config.h" -#include "encoding.h" - -// Currently, interrupts are always disabled when in pk/bbl. -#define disable_irqsave() (0) -#define enable_irqrestore(flags) ((void) (flags)) - -typedef struct { int lock; } spinlock_t; -#define SPINLOCK_INIT {0} - -#define mb() asm volatile ("fence" ::: "memory") -#define atomic_set(ptr, val) (*(volatile typeof(*(ptr)) *)(ptr) = val) -#define atomic_read(ptr) (*(volatile typeof(*(ptr)) *)(ptr)) - -#ifdef __riscv_atomic -# define atomic_add(ptr, inc) __sync_fetch_and_add(ptr, inc) -# define atomic_or(ptr, inc) __sync_fetch_and_or(ptr, inc) -# define atomic_swap(ptr, swp) __sync_lock_test_and_set(ptr, swp) -# define atomic_cas(ptr, cmp, swp) __sync_val_compare_and_swap(ptr, cmp, swp) -#else -# define atomic_binop(ptr, inc, op) ({ \ - long flags = disable_irqsave(); \ - typeof(*(ptr)) res = atomic_read(ptr); \ - atomic_set(ptr, op); \ - enable_irqrestore(flags); \ - res; }) -# define atomic_add(ptr, inc) atomic_binop(ptr, inc, res + (inc)) -# define atomic_or(ptr, inc) atomic_binop(ptr, inc, res | (inc)) -# define atomic_swap(ptr, inc) atomic_binop(ptr, inc, (inc)) -# define atomic_cas(ptr, cmp, swp) ({ \ - long flags = disable_irqsave(); \ - typeof(*(ptr)) res = *(volatile typeof(*(ptr)) *)(ptr); \ - if (res == (cmp)) *(volatile typeof(ptr))(ptr) = (swp); \ - enable_irqrestore(flags); \ - res; }) -#endif - -static inline void spinlock_lock(spinlock_t* lock) -{ - do - { - while (atomic_read(&lock->lock)) - ; - } while (atomic_swap(&lock->lock, -1)); - mb(); -} - -static inline void spinlock_unlock(spinlock_t* lock) -{ - mb(); - atomic_set(&lock->lock,0); -} - -static inline long spinlock_lock_irqsave(spinlock_t* lock) -{ - long flags = disable_irqsave(); - spinlock_lock(lock); - return flags; -} - -static inline void spinlock_unlock_irqrestore(spinlock_t* lock, long flags) -{ - spinlock_unlock(lock); - enable_irqrestore(flags); -} - -#endif diff --git a/pk/bbl.c b/pk/bbl.c deleted file mode 100644 index 5ace350..0000000 --- a/pk/bbl.c +++ /dev/null @@ -1,36 +0,0 @@ -#include "boot.h" -#include "mtrap.h" -#include "vm.h" -#include "config.h" - -static volatile int elf_loaded; - -static void enter_entry_point() -{ - prepare_supervisor_mode(); - write_csr(mepc, current.entry); - asm volatile("eret"); - __builtin_unreachable(); -} - -void run_loaded_program(size_t argc, char** argv) -{ - if (!current.is_supervisor) - die("bbl can't run user binaries; try using pk instead"); - - supervisor_vm_init(); -#ifdef PK_ENABLE_LOGO - print_logo(); -#endif - mb(); - elf_loaded = 1; - enter_entry_point(); -} - -void boot_other_hart() -{ - while (!elf_loaded) - ; - mb(); - enter_entry_point(); -} diff --git a/pk/bits.h b/pk/bits.h deleted file mode 100644 index 10f9df3..0000000 --- a/pk/bits.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef PK_BITS_H -#define PK_BITS_H - -#define likely(x) __builtin_expect((x), 1) -#define unlikely(x) __builtin_expect((x), 0) - -#define ROUNDUP(a, b) ((((a)-1)/(b)+1)*(b)) -#define ROUNDDOWN(a, b) ((a)/(b)*(b)) - -#define EXTRACT_FIELD(val, which) (((val) & (which)) / ((which) & ~((which)-1))) -#define INSERT_FIELD(val, which, fieldval) (((val) & ~(which)) | ((fieldval) * ((which) & ~((which)-1)))) - -#define CONST_POPCOUNT2(x) ((((x) >> 0) & 1) + (((x) >> 1) & 1)) -#define CONST_POPCOUNT4(x) (CONST_POPCOUNT2(x) + CONST_POPCOUNT2((x)>>2)) -#define CONST_POPCOUNT8(x) (CONST_POPCOUNT4(x) + CONST_POPCOUNT4((x)>>4)) -#define CONST_POPCOUNT16(x) (CONST_POPCOUNT8(x) + CONST_POPCOUNT8((x)>>8)) -#define CONST_POPCOUNT32(x) (CONST_POPCOUNT16(x) + CONST_POPCOUNT16((x)>>16)) -#define CONST_POPCOUNT64(x) (CONST_POPCOUNT32(x) + CONST_POPCOUNT32((x)>>32)) -#define CONST_POPCOUNT(x) CONST_POPCOUNT64(x) - -#define CONST_CTZ2(x) CONST_POPCOUNT2(((x) & -(x))-1) -#define CONST_CTZ4(x) CONST_POPCOUNT4(((x) & -(x))-1) -#define CONST_CTZ8(x) CONST_POPCOUNT8(((x) & -(x))-1) -#define CONST_CTZ16(x) CONST_POPCOUNT16(((x) & -(x))-1) -#define CONST_CTZ32(x) CONST_POPCOUNT32(((x) & -(x))-1) -#define CONST_CTZ64(x) CONST_POPCOUNT64(((x) & -(x))-1) -#define CONST_CTZ(x) CONST_CTZ64(x) - -#define STR(x) XSTR(x) -#define XSTR(x) #x - -#ifdef __riscv64 -# define SLL32 sllw -# define STORE sd -# define LOAD ld -# define LOG_REGBYTES 3 -#else -# define SLL32 sll -# define STORE sw -# define LOAD lw -# define LOG_REGBYTES 2 -#endif -#define REGBYTES (1 << LOG_REGBYTES) - -#endif @@ -5,7 +5,6 @@ #ifndef __ASSEMBLER__ -#include <stdint.h> #include <stddef.h> typedef struct { @@ -14,28 +13,19 @@ typedef struct { int is_supervisor; size_t phdr; size_t phdr_size; - size_t first_free_paddr; - size_t first_user_vaddr; - size_t first_vaddr_after_user; size_t bias; size_t entry; size_t brk_min; size_t brk; size_t brk_max; size_t mmap_max; - size_t stack_bottom; size_t stack_top; size_t t0; } elf_info; extern elf_info current; -void prepare_supervisor_mode(); -void run_loaded_program(size_t argc, char** argv); -void boot_loader(); -void boot_other_hart(); void load_elf(const char* fn, elf_info* info); -void print_logo(); #endif // !__ASSEMBLER__ diff --git a/pk/configstring.c b/pk/configstring.c deleted file mode 100644 index 847cd4d..0000000 --- a/pk/configstring.c +++ /dev/null @@ -1,200 +0,0 @@ -#include "encoding.h" -#include "mtrap.h" -#include <stdio.h> - -static const char* skip_whitespace(const char* str) -{ - while (*str && *str <= ' ') - str++; - return str; -} - -static const char* skip_string(const char* str) -{ - while (*str && *str++ != '"') - ; - return str; -} - -static int is_hex(char ch) -{ - return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); -} - -static int parse_hex(char ch) -{ - return (ch >= '0' && ch <= '9') ? ch - '0' : - (ch >= 'a' && ch <= 'f') ? ch - 'a' + 10 : - ch - 'A' + 10; -} - -static const char* skip_key(const char* str) -{ - while (*str >= 35 && *str <= 122 && *str != ';') - str++; - return str; -} - -typedef struct { - const char* start; - const char* end; -} query_result; - -static query_result query_config_string(const char* str, const char* k) -{ - size_t ksize = 0; - while (k[ksize] && k[ksize] != '{') - ksize++; - int last = !k[ksize]; - - query_result res = {0, 0}; - while (1) { - const char* key_start = str = skip_whitespace(str); - const char* key_end = str = skip_key(str); - int match = (key_end - key_start) == ksize; - if (match) - for (size_t i = 0; i < ksize; i++) - if (key_start[i] != k[i]) - match = 0; - const char* value_start = str = skip_whitespace(str); - while (*str != ';') { - if (!*str) { - return res; - } else if (*str == '"') { - str = skip_string(str+1); - } else if (*str == '{') { - const char* search_key = match && !last ? k + ksize + 1 : ""; - query_result inner_res = query_config_string(str + 1, search_key); - if (inner_res.start) - return inner_res; - str = inner_res.end + 1; - } else { - str = skip_key(str); - } - str = skip_whitespace(str); - } - res.end = str; - if (match && last) { - res.start = value_start; - return res; - } - str = skip_whitespace(str+1); - if (*str == '}') { - res.end = str; - return res; - } - } -} - -static void parse_string(query_result r, char* buf) -{ - if (r.start < r.end) { - if (*r.start == '"') { - for (const char* p = r.start + 1; p < r.end && *p != '"'; p++) { - char ch = p[0]; - if (ch == '\\' && p[1] == 'x' && is_hex(p[2])) { - ch = parse_hex(p[2]); - if (is_hex(p[3])) { - ch = (ch << 4) + parse_hex(p[3]); - p++; - } - p += 2; - } - *buf++ = ch; - } - } else { - for (const char* p = r.start; p < r.end && *p > ' '; p++) - *buf++ = *p; - } - } - *buf = 0; -} - -#define get_string(name, search_res) \ - char name[(search_res).end - (search_res).start + 1]; \ - parse_string(search_res, name) - -static unsigned long __get_uint_hex(const char* s) -{ - unsigned long res = 0; - while (*s) { - if (is_hex(*s)) - res = (res << 4) + parse_hex(*s); - else if (*s != '_') - break; - s++; - } - return res; -} - -static unsigned long __get_uint_dec(const char* s) -{ - unsigned long res = 0; - while (*s) { - if (*s >= '0' && *s <= '9') - res = res * 10 + (*s - '0'); - else - break; - s++; - } - return res; -} - -static unsigned long __get_uint(const char* s) -{ - if (s[0] == '0' && s[1] == 'x') - return __get_uint_hex(s+2); - return __get_uint_dec(s); -} - -static unsigned long get_uint(query_result res) -{ - get_string(s, res); - return __get_uint(s); -} - -static long get_sint(query_result res) -{ - get_string(s, res); - if (s[0] == '-') - return -__get_uint(s+1); - return __get_uint(s); -} - -static void query_mem(const char* config_string) -{ - query_result res = query_config_string(config_string, "ram{0{addr"); - assert(res.start); - uintptr_t base = get_uint(res); - res = query_config_string(config_string, "ram{0{size"); - mem_size = get_uint(res); -} - -static void query_harts(const char* config_string) -{ - for (int core = 0, hart; ; core++) { - for (hart = 0; ; hart++) { - char buf[32]; - snprintf(buf, sizeof buf, "core{%d{%d{addr", core, hart); - query_result res = query_config_string(config_string, buf); - if (!res.start) - break; - csr_t* base = (csr_t*)get_uint(res); - uintptr_t hart_id = base[CSR_MHARTID]; - hls_init(hart_id, base); - num_harts++; - assert(hart_id == num_harts-1); - } - if (!hart) - break; - } - assert(num_harts); - assert(num_harts <= MAX_HARTS); -} - -void parse_config_string() -{ - const char* s = (const char*)read_csr(mcfgaddr); - query_mem(s); - query_harts(s); -} @@ -1,9 +1,9 @@ // See LICENSE for license details. -#include "file.h" -#include "vm.h" +#include "mmap.h" #include "mtrap.h" #include "boot.h" +#include "bits.h" #include <sys/stat.h> #include <fcntl.h> #include <elf.h> @@ -15,100 +15,55 @@ void load_elf(const char* fn, elf_info* info) if (IS_ERR_VALUE(file)) goto fail; - Elf64_Ehdr eh64; - ssize_t ehdr_size = file_pread(file, &eh64, sizeof(eh64), 0); - if (ehdr_size < (ssize_t)sizeof(eh64) || - !(eh64.e_ident[0] == '\177' && eh64.e_ident[1] == 'E' && - eh64.e_ident[2] == 'L' && eh64.e_ident[3] == 'F')) + Elf_Ehdr eh; + ssize_t ehdr_size = file_pread(file, &eh, sizeof(eh), 0); + if (ehdr_size < (ssize_t)sizeof(eh) || + !(eh.e_ident[0] == '\177' && eh.e_ident[1] == 'E' && + eh.e_ident[2] == 'L' && eh.e_ident[3] == 'F')) goto fail; - uintptr_t min_vaddr = -1, max_vaddr = 0; - - #define LOAD_ELF do { \ - eh = (typeof(eh))&eh64; \ - size_t phdr_size = eh->e_phnum*sizeof(*ph); \ - if (phdr_size > info->phdr_size) \ - goto fail; \ - ssize_t ret = file_pread(file, (void*)info->phdr, phdr_size, eh->e_phoff); \ - if (ret < (ssize_t)phdr_size) \ - goto fail; \ - info->phnum = eh->e_phnum; \ - info->phent = sizeof(*ph); \ - ph = (typeof(ph))info->phdr; \ - info->is_supervisor = (eh->e_entry >> (8*sizeof(eh->e_entry)-1)) != 0; \ - if (info->is_supervisor) \ - info->first_free_paddr = ROUNDUP(info->first_free_paddr, SUPERPAGE_SIZE); \ - for (int i = 0; i < eh->e_phnum; i++) \ - if (ph[i].p_type == PT_LOAD && ph[i].p_memsz && ph[i].p_vaddr < min_vaddr) \ - min_vaddr = ph[i].p_vaddr; \ - if (info->is_supervisor) \ - min_vaddr = ROUNDDOWN(min_vaddr, SUPERPAGE_SIZE); \ - else \ - min_vaddr = ROUNDDOWN(min_vaddr, RISCV_PGSIZE); \ - uintptr_t bias = 0; \ - if (info->is_supervisor || eh->e_type == ET_DYN) \ - bias = info->first_free_paddr - min_vaddr; \ - info->entry = eh->e_entry; \ - if (!info->is_supervisor) { \ - info->entry += bias; \ - min_vaddr += bias; \ - } \ - info->bias = bias; \ - int flags = MAP_FIXED | MAP_PRIVATE; \ - if (info->is_supervisor) \ - flags |= MAP_POPULATE; \ - for (int i = eh->e_phnum - 1; i >= 0; i--) { \ - if(ph[i].p_type == PT_LOAD && ph[i].p_memsz) { \ - uintptr_t prepad = ph[i].p_vaddr % RISCV_PGSIZE; \ - uintptr_t vaddr = ph[i].p_vaddr + bias; \ - if (vaddr + ph[i].p_memsz > max_vaddr) \ - max_vaddr = vaddr + ph[i].p_memsz; \ - if (info->is_supervisor) { \ - if (!__valid_user_range(vaddr - prepad, vaddr + ph[i].p_memsz)) \ - goto fail; \ - ret = file_pread(file, (void*)vaddr, ph[i].p_filesz, ph[i].p_offset); \ - if (ret < (ssize_t)ph[i].p_filesz) \ - goto fail; \ - memset((void*)vaddr - prepad, 0, prepad); \ - memset((void*)vaddr + ph[i].p_filesz, 0, ph[i].p_memsz - ph[i].p_filesz); \ - } else { \ - int flags2 = flags | (prepad ? MAP_POPULATE : 0); \ - if (__do_mmap(vaddr - prepad, ph[i].p_filesz + prepad, -1, flags2, file, ph[i].p_offset - prepad) != vaddr - prepad) \ - goto fail; \ - memset((void*)vaddr - prepad, 0, prepad); \ - size_t mapped = ROUNDUP(ph[i].p_filesz + prepad, RISCV_PGSIZE) - prepad; \ - if (ph[i].p_memsz > mapped) \ - if (__do_mmap(vaddr + mapped, ph[i].p_memsz - mapped, -1, flags|MAP_ANONYMOUS, 0, 0) != vaddr + mapped) \ - goto fail; \ - } \ - } \ - } \ - } while(0) - - if (IS_ELF64(eh64)) - { -#ifndef __riscv64 - die("can't run 64-bit ELF on 32-bit arch"); -#endif - Elf64_Ehdr* eh; - Elf64_Phdr* ph; - LOAD_ELF; - } - else if (IS_ELF32(eh64)) - { #ifdef __riscv64 - die("can't run 32-bit ELF on 64-bit arch"); + assert(IS_ELF64(eh)); +#else + assert(IS_ELF32(eh)); #endif - Elf32_Ehdr* eh; - Elf32_Phdr* ph; - LOAD_ELF; - } - else - goto fail; - info->first_user_vaddr = min_vaddr; - info->first_vaddr_after_user = ROUNDUP(max_vaddr - info->bias, RISCV_PGSIZE); - info->brk_min = max_vaddr; + uintptr_t min_vaddr = -1; + size_t phdr_size = eh.e_phnum * sizeof(Elf_Phdr); + if (phdr_size > info->phdr_size) + goto fail; + ssize_t ret = file_pread(file, (void*)info->phdr, phdr_size, eh.e_phoff); + if (ret < (ssize_t)phdr_size) + goto fail; + info->phnum = eh.e_phnum; + info->phent = sizeof(Elf_Phdr); + Elf_Phdr* ph = (typeof(ph))info->phdr; + for (int i = 0; i < eh.e_phnum; i++) + if (ph[i].p_type == PT_LOAD && ph[i].p_memsz && ph[i].p_vaddr < min_vaddr) + min_vaddr = ph[i].p_vaddr; + min_vaddr = ROUNDDOWN(min_vaddr, RISCV_PGSIZE); + uintptr_t bias = 0; + if (eh.e_type == ET_DYN) + bias = first_free_paddr - min_vaddr; + min_vaddr += bias; + info->entry = eh.e_entry + bias; + int flags = MAP_FIXED | MAP_PRIVATE; + for (int i = eh.e_phnum - 1; i >= 0; i--) { + if(ph[i].p_type == PT_LOAD && ph[i].p_memsz) { + uintptr_t prepad = ph[i].p_vaddr % RISCV_PGSIZE; + uintptr_t vaddr = ph[i].p_vaddr + bias; + if (vaddr + ph[i].p_memsz > info->brk_min) + info->brk_min = vaddr + ph[i].p_memsz; + int flags2 = flags | (prepad ? MAP_POPULATE : 0); + if (__do_mmap(vaddr - prepad, ph[i].p_filesz + prepad, -1, flags2, file, ph[i].p_offset - prepad) != vaddr - prepad) + goto fail; + memset((void*)vaddr - prepad, 0, prepad); + size_t mapped = ROUNDUP(ph[i].p_filesz + prepad, RISCV_PGSIZE) - prepad; + if (ph[i].p_memsz > mapped) + if (__do_mmap(vaddr + mapped, ph[i].p_memsz - mapped, -1, flags|MAP_ANONYMOUS, 0, 0) != vaddr + mapped) + goto fail; + } + } file_decref(file); return; @@ -12,6 +12,14 @@ #define IS_ELF32(hdr) (IS_ELF(hdr) && (hdr).e_ident[4] == 1) #define IS_ELF64(hdr) (IS_ELF(hdr) && (hdr).e_ident[4] == 2) +#ifdef __riscv64 +# define Elf_Ehdr Elf64_Ehdr +# define Elf_Phdr Elf64_Phdr +#else +# define Elf_Ehdr Elf32_Ehdr +# define Elf_Phdr Elf32_Phdr +#endif + #define ET_EXEC 2 #define ET_DYN 3 diff --git a/pk/emulation.c b/pk/emulation.c deleted file mode 100644 index 5a66997..0000000 --- a/pk/emulation.c +++ /dev/null @@ -1,313 +0,0 @@ -#include "emulation.h" -#include "fp_emulation.h" -#include "config.h" -#include "unprivileged_memory.h" -#include "mtrap.h" -#include <limits.h> - -void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - asm (".pushsection .rodata\n" - "illegal_insn_trap_table:\n" - " .word truly_illegal_insn\n" -#ifdef PK_ENABLE_FP_EMULATION - " .word emulate_float_load\n" -#else - " .word truly_illegal_insn\n" -#endif - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" -#ifdef PK_ENABLE_FP_EMULATION - " .word emulate_float_store\n" -#else - " .word truly_illegal_insn\n" -#endif - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_mul_div\n" - " .word truly_illegal_insn\n" - " .word emulate_mul_div32\n" - " .word truly_illegal_insn\n" -#ifdef PK_ENABLE_FP_EMULATION - " .word emulate_fmadd\n" - " .word emulate_fmadd\n" - " .word emulate_fmadd\n" - " .word emulate_fmadd\n" - " .word emulate_fp\n" -#else - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" -#endif - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_system\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .popsection"); - - uintptr_t mstatus; - insn_t insn = get_insn(mepc, &mstatus); - - if (unlikely((insn & 3) != 3)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - write_csr(mepc, mepc + 4); - - extern int32_t illegal_insn_trap_table[]; - int32_t* pf = (void*)illegal_insn_trap_table + (insn & 0x7c); - emulation_func f = (emulation_func)(uintptr_t)*pf; - f(regs, mcause, mepc, mstatus, insn); -} - -void __attribute__((noinline)) truly_illegal_insn(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn) -{ - redirect_trap(mepc, mstatus); -} - -void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - union { - uint8_t bytes[8]; - uintptr_t intx; - uint64_t int64; - } val; - uintptr_t mstatus; - insn_t insn = get_insn(mepc, &mstatus); - uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); - - int shift = 0, fp = 0, len; - if ((insn & MASK_LW) == MATCH_LW) - len = 4, shift = 8*(sizeof(uintptr_t) - len); -#ifdef __riscv64 - else if ((insn & MASK_LD) == MATCH_LD) - len = 8, shift = 8*(sizeof(uintptr_t) - len); - else if ((insn & MASK_LWU) == MATCH_LWU) - fp = 0, len = 4, shift = 0; -#endif - else if ((insn & MASK_FLD) == MATCH_FLD) - fp = 1, len = 8; - else if ((insn & MASK_FLW) == MATCH_FLW) - fp = 1, len = 4; - else if ((insn & MASK_LH) == MATCH_LH) - len = 2, shift = 8*(sizeof(uintptr_t) - len); - else if ((insn & MASK_LHU) == MATCH_LHU) - len = 2; - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - val.int64 = 0; - for (intptr_t i = len-1; i >= 0; i--) - val.bytes[i] = load_uint8_t((void *)(addr + i), mepc); - - if (!fp) - SET_RD(insn, regs, (intptr_t)val.intx << shift >> shift); - else if (len == 8) - SET_F64_RD(insn, regs, val.int64); - else - SET_F32_RD(insn, regs, val.intx); - - write_csr(mepc, mepc + 4); -} - -void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - union { - uint8_t bytes[8]; - uintptr_t intx; - uint64_t int64; - } val; - uintptr_t mstatus; - insn_t insn = get_insn(mepc, &mstatus); - int len; - - val.intx = GET_RS2(insn, regs); - if ((insn & MASK_SW) == MATCH_SW) - len = 4; -#ifdef __riscv64 - else if ((insn & MASK_SD) == MATCH_SD) - len = 8; -#endif - else if ((insn & MASK_FSD) == MATCH_FSD) - len = 8, val.int64 = GET_F64_RS2(insn, regs); - else if ((insn & MASK_FSW) == MATCH_FSW) - len = 4, val.intx = GET_F32_RS2(insn, regs); - else if ((insn & MASK_SH) == MATCH_SH) - len = 2; - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); - for (int i = 0; i < len; i++) - store_uint8_t((void *)(addr + i), val.bytes[i], mepc); - - write_csr(mepc, mepc + 4); -} - -#ifdef __riscv64 -typedef __int128 double_int; -#else -typedef int64_t double_int; -#endif - -DECLARE_EMULATION_FUNC(emulate_mul_div) -{ - uintptr_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs), val; - -#ifndef __riscv_muldiv - // If compiled with -mno-multiply, GCC will expand these out - if ((insn & MASK_MUL) == MATCH_MUL) - val = rs1 * rs2; - else if ((insn & MASK_DIV) == MATCH_DIV) - val = (intptr_t)rs1 / (intptr_t)rs2; - else if ((insn & MASK_DIVU) == MATCH_DIVU) - val = rs1 / rs2; - else if ((insn & MASK_REM) == MATCH_REM) - val = (intptr_t)rs1 % (intptr_t)rs2; - else if ((insn & MASK_REMU) == MATCH_REMU) - val = rs1 % rs2; - else if ((insn & MASK_MULH) == MATCH_MULH) - val = ((double_int)(intptr_t)rs1 * (double_int)(intptr_t)rs2) >> (8 * sizeof(rs1)); - else if ((insn & MASK_MULHU) == MATCH_MULHU) - val = ((double_int)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); - else if ((insn & MASK_MULHSU) == MATCH_MULHSU) - val = ((double_int)(intptr_t)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); - else -#endif - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - SET_RD(insn, regs, val); -} - -DECLARE_EMULATION_FUNC(emulate_mul_div32) -{ - uint32_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs); - int32_t val; - -#if defined(__riscv64) && !defined(__riscv_muldiv) - // If compiled with -mno-multiply, GCC will expand these out - if ((insn & MASK_MULW) == MATCH_MULW) - val = rs1 * rs2; - else if ((insn & MASK_DIVW) == MATCH_DIVW) - val = (int32_t)rs1 / (int32_t)rs2; - else if ((insn & MASK_DIVUW) == MATCH_DIVUW) - val = rs1 / rs2; - else if ((insn & MASK_REMW) == MATCH_REMW) - val = (int32_t)rs1 % (int32_t)rs2; - else if ((insn & MASK_REMUW) == MATCH_REMUW) - val = rs1 % rs2; - else -#endif - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - SET_RD(insn, regs, val); -} - -static inline int emulate_read_csr(int num, uintptr_t mstatus, uintptr_t* result) -{ - switch (num) - { - case CSR_TIME: - *result = read_csr(mtime) + HLS()->utime_delta; - return 0; - case CSR_CYCLE: - *result = read_csr(mcycle) + HLS()->ucycle_delta; - return 0; - case CSR_INSTRET: - *result = read_csr(minstret) + HLS()->uinstret_delta; - return 0; - case CSR_STIME: - *result = read_csr(mtime) + HLS()->stime_delta; - return 0; - case CSR_SCYCLE: - *result = read_csr(mcycle) + HLS()->scycle_delta; - return 0; - case CSR_SINSTRET: - *result = read_csr(minstret) + HLS()->sinstret_delta; - return 0; -#ifdef __riscv32 - case CSR_TIMEH: - *result = (((uint64_t)read_csr(mtimeh) << 32) + read_csr(mtime) - + HLS()->stime_delta) >> 32; - return 0; - case CSR_CYCLEH: - *result = (((uint64_t)read_csr(mcycleh) << 32) + read_csr(mcycle) - + HLS()->scycle_delta) >> 32; - return 0; - case CSR_INSTRETH: - *result = (((uint64_t)read_csr(minstreth) << 32) + read_csr(minstret) - + HLS()->sinstret_delta) >> 32; - return 0; -#endif -#ifdef PK_ENABLE_FP_EMULATION - case CSR_FRM: - if ((mstatus & MSTATUS_FS) == 0) break; - *result = GET_FRM(); - return 0; - case CSR_FFLAGS: - if ((mstatus & MSTATUS_FS) == 0) break; - *result = GET_FFLAGS(); - return 0; - case CSR_FCSR: - if ((mstatus & MSTATUS_FS) == 0) break; - *result = GET_FCSR(); - return 0; -#endif - } - return -1; -} - -static inline int emulate_write_csr(int num, uintptr_t value, uintptr_t mstatus) -{ - switch (num) - { -#ifdef PK_ENABLE_FP_EMULATION - case CSR_FRM: SET_FRM(value); return 0; - case CSR_FFLAGS: SET_FFLAGS(value); return 0; - case CSR_FCSR: SET_FCSR(value); return 0; -#endif - } - return -1; -} - -DECLARE_EMULATION_FUNC(emulate_system) -{ - int rs1_num = (insn >> 15) & 0x1f; - uintptr_t rs1_val = GET_RS1(insn, regs); - int csr_num = (uint32_t)insn >> 20; - uintptr_t csr_val, new_csr_val; - - if (emulate_read_csr(csr_num, mstatus, &csr_val)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - int do_write = rs1_num; - switch (GET_RM(insn)) - { - case 0: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - case 1: new_csr_val = rs1_val; do_write = 1; break; - case 2: new_csr_val = csr_val | rs1_val; break; - case 3: new_csr_val = csr_val & ~rs1_val; break; - case 4: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - case 5: new_csr_val = rs1_num; do_write = 1; break; - case 6: new_csr_val = csr_val | rs1_num; break; - case 7: new_csr_val = csr_val & ~rs1_num; break; - } - - if (do_write && emulate_write_csr(csr_num, new_csr_val, mstatus)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - SET_RD(insn, regs, csr_val); -} diff --git a/pk/emulation.h b/pk/emulation.h deleted file mode 100644 index f1a71ec..0000000 --- a/pk/emulation.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef _RISCV_EMULATION_H -#define _RISCV_EMULATION_H - -#include "encoding.h" -#include "bits.h" -#include <stdint.h> - -typedef uint32_t insn_t; -typedef void (*emulation_func)(uintptr_t*, uintptr_t, uintptr_t, uintptr_t, insn_t); -#define DECLARE_EMULATION_FUNC(name) void name(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn) - -void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); -void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); -void redirect_trap(uintptr_t epc, uintptr_t mstatus); -DECLARE_EMULATION_FUNC(truly_illegal_insn); - -#define GET_REG(insn, pos, regs) ({ \ - int mask = (1 << (5+LOG_REGBYTES)) - (1 << LOG_REGBYTES); \ - (uintptr_t*)((uintptr_t)regs + (((insn) >> ((pos) - LOG_REGBYTES)) & mask)); \ -}) -#define GET_RS1(insn, regs) (*GET_REG(insn, 15, regs)) -#define GET_RS2(insn, regs) (*GET_REG(insn, 20, regs)) -#define SET_RD(insn, regs, val) (*GET_REG(insn, 7, regs) = (val)) -#define IMM_I(insn) ((int32_t)(insn) >> 20) -#define IMM_S(insn) (((int32_t)(insn) >> 25 << 5) | (int32_t)(((insn) >> 7) & 0x1f)) -#define MASK_FUNCT3 0x7000 - -#endif diff --git a/pk/encoding.h b/pk/encoding.h deleted file mode 100644 index f2fab36..0000000 --- a/pk/encoding.h +++ /dev/null @@ -1,1037 +0,0 @@ -// See LICENSE for license details. - -#ifndef RISCV_CSR_ENCODING_H -#define RISCV_CSR_ENCODING_H - -#define MSTATUS_UIE 0x00000001 -#define MSTATUS_SIE 0x00000002 -#define MSTATUS_HIE 0x00000004 -#define MSTATUS_MIE 0x00000008 -#define MSTATUS_UPIE 0x00000010 -#define MSTATUS_SPIE 0x00000020 -#define MSTATUS_HPIE 0x00000040 -#define MSTATUS_MPIE 0x00000080 -#define MSTATUS_SPP 0x00000100 -#define MSTATUS_HPP 0x00000600 -#define MSTATUS_MPP 0x00001800 -#define MSTATUS_FS 0x00006000 -#define MSTATUS_XS 0x00018000 -#define MSTATUS_MPRV 0x00020000 -#define MSTATUS_PUM 0x00040000 -#define MSTATUS_VM 0x1F000000 -#define MSTATUS32_SD 0x80000000 -#define MSTATUS64_SD 0x8000000000000000 - -#define SSTATUS_UIE 0x00000001 -#define SSTATUS_SIE 0x00000002 -#define SSTATUS_UPIE 0x00000010 -#define SSTATUS_SPIE 0x00000020 -#define SSTATUS_SPP 0x00000100 -#define SSTATUS_FS 0x00006000 -#define SSTATUS_XS 0x00018000 -#define SSTATUS_PUM 0x00040000 -#define SSTATUS32_SD 0x80000000 -#define SSTATUS64_SD 0x8000000000000000 - -#define MIP_SSIP (1 << IRQ_S_SOFT) -#define MIP_HSIP (1 << IRQ_H_SOFT) -#define MIP_MSIP (1 << IRQ_M_SOFT) -#define MIP_STIP (1 << IRQ_S_TIMER) -#define MIP_HTIP (1 << IRQ_H_TIMER) -#define MIP_MTIP (1 << IRQ_M_TIMER) - -#define SIP_SSIP MIP_SSIP -#define SIP_STIP MIP_STIP - -#define PRV_U 0 -#define PRV_S 1 -#define PRV_H 2 -#define PRV_M 3 - -#define VM_MBARE 0 -#define VM_MBB 1 -#define VM_MBBID 2 -#define VM_SV32 8 -#define VM_SV39 9 -#define VM_SV48 10 - -#define IRQ_S_SOFT 1 -#define IRQ_H_SOFT 2 -#define IRQ_M_SOFT 3 -#define IRQ_S_TIMER 5 -#define IRQ_H_TIMER 6 -#define IRQ_M_TIMER 7 -#define IRQ_S_DEV 9 -#define IRQ_H_DEV 10 -#define IRQ_M_DEV 11 -#define IRQ_COP 12 -#define IRQ_HOST 13 - -#define DEFAULT_RSTVEC 0x0 -#define DEFAULT_NMIVEC 0x4 -#define DEFAULT_MTVEC 0x8 - -// page table entry (PTE) fields -#define PTE_V 0x001 // Valid -#define PTE_TYPE 0x01E // Type -#define PTE_R 0x020 // Referenced -#define PTE_D 0x040 // Dirty -#define PTE_SOFT 0x380 // Reserved for Software - -#define PTE_TYPE_TABLE 0x00 -#define PTE_TYPE_TABLE_GLOBAL 0x02 -#define PTE_TYPE_URX_SR 0x04 -#define PTE_TYPE_URWX_SRW 0x06 -#define PTE_TYPE_UR_SR 0x08 -#define PTE_TYPE_URW_SRW 0x0A -#define PTE_TYPE_URX_SRX 0x0C -#define PTE_TYPE_URWX_SRWX 0x0E -#define PTE_TYPE_SR 0x10 -#define PTE_TYPE_SRW 0x12 -#define PTE_TYPE_SRX 0x14 -#define PTE_TYPE_SRWX 0x16 -#define PTE_TYPE_SR_GLOBAL 0x18 -#define PTE_TYPE_SRW_GLOBAL 0x1A -#define PTE_TYPE_SRX_GLOBAL 0x1C -#define PTE_TYPE_SRWX_GLOBAL 0x1E - -#define PTE_PPN_SHIFT 10 - -#define PTE_TABLE(PTE) ((0x0000000AU >> ((PTE) & 0x1F)) & 1) -#define PTE_UR(PTE) ((0x0000AAA0U >> ((PTE) & 0x1F)) & 1) -#define PTE_UW(PTE) ((0x00008880U >> ((PTE) & 0x1F)) & 1) -#define PTE_UX(PTE) ((0x0000A0A0U >> ((PTE) & 0x1F)) & 1) -#define PTE_SR(PTE) ((0xAAAAAAA0U >> ((PTE) & 0x1F)) & 1) -#define PTE_SW(PTE) ((0x88888880U >> ((PTE) & 0x1F)) & 1) -#define PTE_SX(PTE) ((0xA0A0A000U >> ((PTE) & 0x1F)) & 1) - -#define PTE_CHECK_PERM(PTE, SUPERVISOR, STORE, FETCH) \ - ((STORE) ? ((SUPERVISOR) ? PTE_SW(PTE) : PTE_UW(PTE)) : \ - (FETCH) ? ((SUPERVISOR) ? PTE_SX(PTE) : PTE_UX(PTE)) : \ - ((SUPERVISOR) ? PTE_SR(PTE) : PTE_UR(PTE))) - -#ifdef __riscv - -#ifdef __riscv64 -# define MSTATUS_SD MSTATUS64_SD -# define SSTATUS_SD SSTATUS64_SD -# define RISCV_PGLEVEL_BITS 9 -#else -# define MSTATUS_SD MSTATUS32_SD -# define SSTATUS_SD SSTATUS32_SD -# define RISCV_PGLEVEL_BITS 10 -#endif -#define RISCV_PGSHIFT 12 -#define RISCV_PGSIZE (1 << RISCV_PGSHIFT) - -#ifndef __ASSEMBLER__ - -#ifdef __GNUC__ - -#define read_csr(reg) ({ unsigned long __tmp; \ - asm volatile ("csrr %0, " #reg : "=r"(__tmp)); \ - __tmp; }) - -#define write_csr(reg, val) \ - asm volatile ("csrw " #reg ", %0" :: "r"(val)) - -#define swap_csr(reg, val) ({ long __tmp; \ - asm volatile ("csrrw %0, " #reg ", %1" : "=r"(__tmp) : "r"(val)); \ - __tmp; }) - -#define set_csr(reg, bit) ({ unsigned long __tmp; \ - if (__builtin_constant_p(bit) && (bit) < 32) \ - asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \ - else \ - asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \ - __tmp; }) - -#define clear_csr(reg, bit) ({ unsigned long __tmp; \ - if (__builtin_constant_p(bit) && (bit) < 32) \ - asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \ - else \ - asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \ - __tmp; }) - -#define rdtime() read_csr(time) -#define rdcycle() read_csr(cycle) -#define rdinstret() read_csr(instret) - -#endif - -#endif - -#endif - -#endif -/* Automatically generated by parse-opcodes */ -#ifndef RISCV_ENCODING_H -#define RISCV_ENCODING_H -#define MATCH_BEQ 0x63 -#define MASK_BEQ 0x707f -#define MATCH_BNE 0x1063 -#define MASK_BNE 0x707f -#define MATCH_BLT 0x4063 -#define MASK_BLT 0x707f -#define MATCH_BGE 0x5063 -#define MASK_BGE 0x707f -#define MATCH_BLTU 0x6063 -#define MASK_BLTU 0x707f -#define MATCH_BGEU 0x7063 -#define MASK_BGEU 0x707f -#define MATCH_JALR 0x67 -#define MASK_JALR 0x707f -#define MATCH_JAL 0x6f -#define MASK_JAL 0x7f -#define MATCH_LUI 0x37 -#define MASK_LUI 0x7f -#define MATCH_AUIPC 0x17 -#define MASK_AUIPC 0x7f -#define MATCH_ADDI 0x13 -#define MASK_ADDI 0x707f -#define MATCH_SLLI 0x1013 -#define MASK_SLLI 0xfc00707f -#define MATCH_SLTI 0x2013 -#define MASK_SLTI 0x707f -#define MATCH_SLTIU 0x3013 -#define MASK_SLTIU 0x707f -#define MATCH_XORI 0x4013 -#define MASK_XORI 0x707f -#define MATCH_SRLI 0x5013 -#define MASK_SRLI 0xfc00707f -#define MATCH_SRAI 0x40005013 -#define MASK_SRAI 0xfc00707f -#define MATCH_ORI 0x6013 -#define MASK_ORI 0x707f -#define MATCH_ANDI 0x7013 -#define MASK_ANDI 0x707f -#define MATCH_ADD 0x33 -#define MASK_ADD 0xfe00707f -#define MATCH_SUB 0x40000033 -#define MASK_SUB 0xfe00707f -#define MATCH_SLL 0x1033 -#define MASK_SLL 0xfe00707f -#define MATCH_SLT 0x2033 -#define MASK_SLT 0xfe00707f -#define MATCH_SLTU 0x3033 -#define MASK_SLTU 0xfe00707f -#define MATCH_XOR 0x4033 -#define MASK_XOR 0xfe00707f -#define MATCH_SRL 0x5033 -#define MASK_SRL 0xfe00707f -#define MATCH_SRA 0x40005033 -#define MASK_SRA 0xfe00707f -#define MATCH_OR 0x6033 -#define MASK_OR 0xfe00707f -#define MATCH_AND 0x7033 -#define MASK_AND 0xfe00707f -#define MATCH_ADDIW 0x1b -#define MASK_ADDIW 0x707f -#define MATCH_SLLIW 0x101b -#define MASK_SLLIW 0xfe00707f -#define MATCH_SRLIW 0x501b -#define MASK_SRLIW 0xfe00707f -#define MATCH_SRAIW 0x4000501b -#define MASK_SRAIW 0xfe00707f -#define MATCH_ADDW 0x3b -#define MASK_ADDW 0xfe00707f -#define MATCH_SUBW 0x4000003b -#define MASK_SUBW 0xfe00707f -#define MATCH_SLLW 0x103b -#define MASK_SLLW 0xfe00707f -#define MATCH_SRLW 0x503b -#define MASK_SRLW 0xfe00707f -#define MATCH_SRAW 0x4000503b -#define MASK_SRAW 0xfe00707f -#define MATCH_LB 0x3 -#define MASK_LB 0x707f -#define MATCH_LH 0x1003 -#define MASK_LH 0x707f -#define MATCH_LW 0x2003 -#define MASK_LW 0x707f -#define MATCH_LD 0x3003 -#define MASK_LD 0x707f -#define MATCH_LBU 0x4003 -#define MASK_LBU 0x707f -#define MATCH_LHU 0x5003 -#define MASK_LHU 0x707f -#define MATCH_LWU 0x6003 -#define MASK_LWU 0x707f -#define MATCH_SB 0x23 -#define MASK_SB 0x707f -#define MATCH_SH 0x1023 -#define MASK_SH 0x707f -#define MATCH_SW 0x2023 -#define MASK_SW 0x707f -#define MATCH_SD 0x3023 -#define MASK_SD 0x707f -#define MATCH_FENCE 0xf -#define MASK_FENCE 0x707f -#define MATCH_FENCE_I 0x100f -#define MASK_FENCE_I 0x707f -#define MATCH_MUL 0x2000033 -#define MASK_MUL 0xfe00707f -#define MATCH_MULH 0x2001033 -#define MASK_MULH 0xfe00707f -#define MATCH_MULHSU 0x2002033 -#define MASK_MULHSU 0xfe00707f -#define MATCH_MULHU 0x2003033 -#define MASK_MULHU 0xfe00707f -#define MATCH_DIV 0x2004033 -#define MASK_DIV 0xfe00707f -#define MATCH_DIVU 0x2005033 -#define MASK_DIVU 0xfe00707f -#define MATCH_REM 0x2006033 -#define MASK_REM 0xfe00707f -#define MATCH_REMU 0x2007033 -#define MASK_REMU 0xfe00707f -#define MATCH_MULW 0x200003b -#define MASK_MULW 0xfe00707f -#define MATCH_DIVW 0x200403b -#define MASK_DIVW 0xfe00707f -#define MATCH_DIVUW 0x200503b -#define MASK_DIVUW 0xfe00707f -#define MATCH_REMW 0x200603b -#define MASK_REMW 0xfe00707f -#define MATCH_REMUW 0x200703b -#define MASK_REMUW 0xfe00707f -#define MATCH_AMOADD_W 0x202f -#define MASK_AMOADD_W 0xf800707f -#define MATCH_AMOXOR_W 0x2000202f -#define MASK_AMOXOR_W 0xf800707f -#define MATCH_AMOOR_W 0x4000202f -#define MASK_AMOOR_W 0xf800707f -#define MATCH_AMOAND_W 0x6000202f -#define MASK_AMOAND_W 0xf800707f -#define MATCH_AMOMIN_W 0x8000202f -#define MASK_AMOMIN_W 0xf800707f -#define MATCH_AMOMAX_W 0xa000202f -#define MASK_AMOMAX_W 0xf800707f -#define MATCH_AMOMINU_W 0xc000202f -#define MASK_AMOMINU_W 0xf800707f -#define MATCH_AMOMAXU_W 0xe000202f -#define MASK_AMOMAXU_W 0xf800707f -#define MATCH_AMOSWAP_W 0x800202f -#define MASK_AMOSWAP_W 0xf800707f -#define MATCH_LR_W 0x1000202f -#define MASK_LR_W 0xf9f0707f -#define MATCH_SC_W 0x1800202f -#define MASK_SC_W 0xf800707f -#define MATCH_AMOADD_D 0x302f -#define MASK_AMOADD_D 0xf800707f -#define MATCH_AMOXOR_D 0x2000302f -#define MASK_AMOXOR_D 0xf800707f -#define MATCH_AMOOR_D 0x4000302f -#define MASK_AMOOR_D 0xf800707f -#define MATCH_AMOAND_D 0x6000302f -#define MASK_AMOAND_D 0xf800707f -#define MATCH_AMOMIN_D 0x8000302f -#define MASK_AMOMIN_D 0xf800707f -#define MATCH_AMOMAX_D 0xa000302f -#define MASK_AMOMAX_D 0xf800707f -#define MATCH_AMOMINU_D 0xc000302f -#define MASK_AMOMINU_D 0xf800707f -#define MATCH_AMOMAXU_D 0xe000302f -#define MASK_AMOMAXU_D 0xf800707f -#define MATCH_AMOSWAP_D 0x800302f -#define MASK_AMOSWAP_D 0xf800707f -#define MATCH_LR_D 0x1000302f -#define MASK_LR_D 0xf9f0707f -#define MATCH_SC_D 0x1800302f -#define MASK_SC_D 0xf800707f -#define MATCH_SCALL 0x73 -#define MASK_SCALL 0xffffffff -#define MATCH_SBREAK 0x100073 -#define MASK_SBREAK 0xffffffff -#define MATCH_SRET 0x10200073 -#define MASK_SRET 0xffffffff -#define MATCH_SFENCE_VM 0x10400073 -#define MASK_SFENCE_VM 0xfff07fff -#define MATCH_WFI 0x10500073 -#define MASK_WFI 0xffffffff -#define MATCH_CSRRW 0x1073 -#define MASK_CSRRW 0x707f -#define MATCH_CSRRS 0x2073 -#define MASK_CSRRS 0x707f -#define MATCH_CSRRC 0x3073 -#define MASK_CSRRC 0x707f -#define MATCH_CSRRWI 0x5073 -#define MASK_CSRRWI 0x707f -#define MATCH_CSRRSI 0x6073 -#define MASK_CSRRSI 0x707f -#define MATCH_CSRRCI 0x7073 -#define MASK_CSRRCI 0x707f -#define MATCH_FADD_S 0x53 -#define MASK_FADD_S 0xfe00007f -#define MATCH_FSUB_S 0x8000053 -#define MASK_FSUB_S 0xfe00007f -#define MATCH_FMUL_S 0x10000053 -#define MASK_FMUL_S 0xfe00007f -#define MATCH_FDIV_S 0x18000053 -#define MASK_FDIV_S 0xfe00007f -#define MATCH_FSGNJ_S 0x20000053 -#define MASK_FSGNJ_S 0xfe00707f -#define MATCH_FSGNJN_S 0x20001053 -#define MASK_FSGNJN_S 0xfe00707f -#define MATCH_FSGNJX_S 0x20002053 -#define MASK_FSGNJX_S 0xfe00707f -#define MATCH_FMIN_S 0x28000053 -#define MASK_FMIN_S 0xfe00707f -#define MATCH_FMAX_S 0x28001053 -#define MASK_FMAX_S 0xfe00707f -#define MATCH_FSQRT_S 0x58000053 -#define MASK_FSQRT_S 0xfff0007f -#define MATCH_FADD_D 0x2000053 -#define MASK_FADD_D 0xfe00007f -#define MATCH_FSUB_D 0xa000053 -#define MASK_FSUB_D 0xfe00007f -#define MATCH_FMUL_D 0x12000053 -#define MASK_FMUL_D 0xfe00007f -#define MATCH_FDIV_D 0x1a000053 -#define MASK_FDIV_D 0xfe00007f -#define MATCH_FSGNJ_D 0x22000053 -#define MASK_FSGNJ_D 0xfe00707f -#define MATCH_FSGNJN_D 0x22001053 -#define MASK_FSGNJN_D 0xfe00707f -#define MATCH_FSGNJX_D 0x22002053 -#define MASK_FSGNJX_D 0xfe00707f -#define MATCH_FMIN_D 0x2a000053 -#define MASK_FMIN_D 0xfe00707f -#define MATCH_FMAX_D 0x2a001053 -#define MASK_FMAX_D 0xfe00707f -#define MATCH_FCVT_S_D 0x40100053 -#define MASK_FCVT_S_D 0xfff0007f -#define MATCH_FCVT_D_S 0x42000053 -#define MASK_FCVT_D_S 0xfff0007f -#define MATCH_FSQRT_D 0x5a000053 -#define MASK_FSQRT_D 0xfff0007f -#define MATCH_FLE_S 0xa0000053 -#define MASK_FLE_S 0xfe00707f -#define MATCH_FLT_S 0xa0001053 -#define MASK_FLT_S 0xfe00707f -#define MATCH_FEQ_S 0xa0002053 -#define MASK_FEQ_S 0xfe00707f -#define MATCH_FLE_D 0xa2000053 -#define MASK_FLE_D 0xfe00707f -#define MATCH_FLT_D 0xa2001053 -#define MASK_FLT_D 0xfe00707f -#define MATCH_FEQ_D 0xa2002053 -#define MASK_FEQ_D 0xfe00707f -#define MATCH_FCVT_W_S 0xc0000053 -#define MASK_FCVT_W_S 0xfff0007f -#define MATCH_FCVT_WU_S 0xc0100053 -#define MASK_FCVT_WU_S 0xfff0007f -#define MATCH_FCVT_L_S 0xc0200053 -#define MASK_FCVT_L_S 0xfff0007f -#define MATCH_FCVT_LU_S 0xc0300053 -#define MASK_FCVT_LU_S 0xfff0007f -#define MATCH_FMV_X_S 0xe0000053 -#define MASK_FMV_X_S 0xfff0707f -#define MATCH_FCLASS_S 0xe0001053 -#define MASK_FCLASS_S 0xfff0707f -#define MATCH_FCVT_W_D 0xc2000053 -#define MASK_FCVT_W_D 0xfff0007f -#define MATCH_FCVT_WU_D 0xc2100053 -#define MASK_FCVT_WU_D 0xfff0007f -#define MATCH_FCVT_L_D 0xc2200053 -#define MASK_FCVT_L_D 0xfff0007f -#define MATCH_FCVT_LU_D 0xc2300053 -#define MASK_FCVT_LU_D 0xfff0007f -#define MATCH_FMV_X_D 0xe2000053 -#define MASK_FMV_X_D 0xfff0707f -#define MATCH_FCLASS_D 0xe2001053 -#define MASK_FCLASS_D 0xfff0707f -#define MATCH_FCVT_S_W 0xd0000053 -#define MASK_FCVT_S_W 0xfff0007f -#define MATCH_FCVT_S_WU 0xd0100053 -#define MASK_FCVT_S_WU 0xfff0007f -#define MATCH_FCVT_S_L 0xd0200053 -#define MASK_FCVT_S_L 0xfff0007f -#define MATCH_FCVT_S_LU 0xd0300053 -#define MASK_FCVT_S_LU 0xfff0007f -#define MATCH_FMV_S_X 0xf0000053 -#define MASK_FMV_S_X 0xfff0707f -#define MATCH_FCVT_D_W 0xd2000053 -#define MASK_FCVT_D_W 0xfff0007f -#define MATCH_FCVT_D_WU 0xd2100053 -#define MASK_FCVT_D_WU 0xfff0007f -#define MATCH_FCVT_D_L 0xd2200053 -#define MASK_FCVT_D_L 0xfff0007f -#define MATCH_FCVT_D_LU 0xd2300053 -#define MASK_FCVT_D_LU 0xfff0007f -#define MATCH_FMV_D_X 0xf2000053 -#define MASK_FMV_D_X 0xfff0707f -#define MATCH_FLW 0x2007 -#define MASK_FLW 0x707f -#define MATCH_FLD 0x3007 -#define MASK_FLD 0x707f -#define MATCH_FSW 0x2027 -#define MASK_FSW 0x707f -#define MATCH_FSD 0x3027 -#define MASK_FSD 0x707f -#define MATCH_FMADD_S 0x43 -#define MASK_FMADD_S 0x600007f -#define MATCH_FMSUB_S 0x47 -#define MASK_FMSUB_S 0x600007f -#define MATCH_FNMSUB_S 0x4b -#define MASK_FNMSUB_S 0x600007f -#define MATCH_FNMADD_S 0x4f -#define MASK_FNMADD_S 0x600007f -#define MATCH_FMADD_D 0x2000043 -#define MASK_FMADD_D 0x600007f -#define MATCH_FMSUB_D 0x2000047 -#define MASK_FMSUB_D 0x600007f -#define MATCH_FNMSUB_D 0x200004b -#define MASK_FNMSUB_D 0x600007f -#define MATCH_FNMADD_D 0x200004f -#define MASK_FNMADD_D 0x600007f -#define MATCH_C_NOP 0x1 -#define MASK_C_NOP 0xffff -#define MATCH_C_ADDI16SP 0x6101 -#define MASK_C_ADDI16SP 0xef83 -#define MATCH_C_JR 0x8002 -#define MASK_C_JR 0xf07f -#define MATCH_C_JALR 0x9002 -#define MASK_C_JALR 0xf07f -#define MATCH_C_EBREAK 0x9002 -#define MASK_C_EBREAK 0xffff -#define MATCH_C_LD 0x6000 -#define MASK_C_LD 0xe003 -#define MATCH_C_SD 0xe000 -#define MASK_C_SD 0xe003 -#define MATCH_C_ADDIW 0x2001 -#define MASK_C_ADDIW 0xe003 -#define MATCH_C_LDSP 0x6002 -#define MASK_C_LDSP 0xe003 -#define MATCH_C_SDSP 0xe002 -#define MASK_C_SDSP 0xe003 -#define MATCH_C_ADDI4SPN 0x0 -#define MASK_C_ADDI4SPN 0xe003 -#define MATCH_C_FLD 0x2000 -#define MASK_C_FLD 0xe003 -#define MATCH_C_LW 0x4000 -#define MASK_C_LW 0xe003 -#define MATCH_C_FLW 0x6000 -#define MASK_C_FLW 0xe003 -#define MATCH_C_FSD 0xa000 -#define MASK_C_FSD 0xe003 -#define MATCH_C_SW 0xc000 -#define MASK_C_SW 0xe003 -#define MATCH_C_FSW 0xe000 -#define MASK_C_FSW 0xe003 -#define MATCH_C_ADDI 0x1 -#define MASK_C_ADDI 0xe003 -#define MATCH_C_JAL 0x2001 -#define MASK_C_JAL 0xe003 -#define MATCH_C_LI 0x4001 -#define MASK_C_LI 0xe003 -#define MATCH_C_LUI 0x6001 -#define MASK_C_LUI 0xe003 -#define MATCH_C_SRLI 0x8001 -#define MASK_C_SRLI 0xec03 -#define MATCH_C_SRAI 0x8401 -#define MASK_C_SRAI 0xec03 -#define MATCH_C_ANDI 0x8801 -#define MASK_C_ANDI 0xec03 -#define MATCH_C_SUB 0x8c01 -#define MASK_C_SUB 0xfc63 -#define MATCH_C_XOR 0x8c21 -#define MASK_C_XOR 0xfc63 -#define MATCH_C_OR 0x8c41 -#define MASK_C_OR 0xfc63 -#define MATCH_C_AND 0x8c61 -#define MASK_C_AND 0xfc63 -#define MATCH_C_SUBW 0x9c01 -#define MASK_C_SUBW 0xfc63 -#define MATCH_C_ADDW 0x9c21 -#define MASK_C_ADDW 0xfc63 -#define MATCH_C_J 0xa001 -#define MASK_C_J 0xe003 -#define MATCH_C_BEQZ 0xc001 -#define MASK_C_BEQZ 0xe003 -#define MATCH_C_BNEZ 0xe001 -#define MASK_C_BNEZ 0xe003 -#define MATCH_C_SLLI 0x2 -#define MASK_C_SLLI 0xe003 -#define MATCH_C_FLDSP 0x2002 -#define MASK_C_FLDSP 0xe003 -#define MATCH_C_LWSP 0x4002 -#define MASK_C_LWSP 0xe003 -#define MATCH_C_FLWSP 0x6002 -#define MASK_C_FLWSP 0xe003 -#define MATCH_C_MV 0x8002 -#define MASK_C_MV 0xf003 -#define MATCH_C_ADD 0x9002 -#define MASK_C_ADD 0xf003 -#define MATCH_C_FSDSP 0xa002 -#define MASK_C_FSDSP 0xe003 -#define MATCH_C_SWSP 0xc002 -#define MASK_C_SWSP 0xe003 -#define MATCH_C_FSWSP 0xe002 -#define MASK_C_FSWSP 0xe003 -#define MATCH_CUSTOM0 0xb -#define MASK_CUSTOM0 0x707f -#define MATCH_CUSTOM0_RS1 0x200b -#define MASK_CUSTOM0_RS1 0x707f -#define MATCH_CUSTOM0_RS1_RS2 0x300b -#define MASK_CUSTOM0_RS1_RS2 0x707f -#define MATCH_CUSTOM0_RD 0x400b -#define MASK_CUSTOM0_RD 0x707f -#define MATCH_CUSTOM0_RD_RS1 0x600b -#define MASK_CUSTOM0_RD_RS1 0x707f -#define MATCH_CUSTOM0_RD_RS1_RS2 0x700b -#define MASK_CUSTOM0_RD_RS1_RS2 0x707f -#define MATCH_CUSTOM1 0x2b -#define MASK_CUSTOM1 0x707f -#define MATCH_CUSTOM1_RS1 0x202b -#define MASK_CUSTOM1_RS1 0x707f -#define MATCH_CUSTOM1_RS1_RS2 0x302b -#define MASK_CUSTOM1_RS1_RS2 0x707f -#define MATCH_CUSTOM1_RD 0x402b -#define MASK_CUSTOM1_RD 0x707f -#define MATCH_CUSTOM1_RD_RS1 0x602b -#define MASK_CUSTOM1_RD_RS1 0x707f -#define MATCH_CUSTOM1_RD_RS1_RS2 0x702b -#define MASK_CUSTOM1_RD_RS1_RS2 0x707f -#define MATCH_CUSTOM2 0x5b -#define MASK_CUSTOM2 0x707f -#define MATCH_CUSTOM2_RS1 0x205b -#define MASK_CUSTOM2_RS1 0x707f -#define MATCH_CUSTOM2_RS1_RS2 0x305b -#define MASK_CUSTOM2_RS1_RS2 0x707f -#define MATCH_CUSTOM2_RD 0x405b -#define MASK_CUSTOM2_RD 0x707f -#define MATCH_CUSTOM2_RD_RS1 0x605b -#define MASK_CUSTOM2_RD_RS1 0x707f -#define MATCH_CUSTOM2_RD_RS1_RS2 0x705b -#define MASK_CUSTOM2_RD_RS1_RS2 0x707f -#define MATCH_CUSTOM3 0x7b -#define MASK_CUSTOM3 0x707f -#define MATCH_CUSTOM3_RS1 0x207b -#define MASK_CUSTOM3_RS1 0x707f -#define MATCH_CUSTOM3_RS1_RS2 0x307b -#define MASK_CUSTOM3_RS1_RS2 0x707f -#define MATCH_CUSTOM3_RD 0x407b -#define MASK_CUSTOM3_RD 0x707f -#define MATCH_CUSTOM3_RD_RS1 0x607b -#define MASK_CUSTOM3_RD_RS1 0x707f -#define MATCH_CUSTOM3_RD_RS1_RS2 0x707b -#define MASK_CUSTOM3_RD_RS1_RS2 0x707f -#define CSR_FFLAGS 0x1 -#define CSR_FRM 0x2 -#define CSR_FCSR 0x3 -#define CSR_CYCLE 0xc00 -#define CSR_TIME 0xc01 -#define CSR_INSTRET 0xc02 -#define CSR_STATS 0xc0 -#define CSR_UARCH0 0xcc0 -#define CSR_UARCH1 0xcc1 -#define CSR_UARCH2 0xcc2 -#define CSR_UARCH3 0xcc3 -#define CSR_UARCH4 0xcc4 -#define CSR_UARCH5 0xcc5 -#define CSR_UARCH6 0xcc6 -#define CSR_UARCH7 0xcc7 -#define CSR_UARCH8 0xcc8 -#define CSR_UARCH9 0xcc9 -#define CSR_UARCH10 0xcca -#define CSR_UARCH11 0xccb -#define CSR_UARCH12 0xccc -#define CSR_UARCH13 0xccd -#define CSR_UARCH14 0xcce -#define CSR_UARCH15 0xccf -#define CSR_SSTATUS 0x100 -#define CSR_SIE 0x104 -#define CSR_STVEC 0x105 -#define CSR_SSCRATCH 0x140 -#define CSR_SEPC 0x141 -#define CSR_SCAUSE 0x142 -#define CSR_SBADADDR 0x143 -#define CSR_SIP 0x144 -#define CSR_SPTBR 0x180 -#define CSR_SASID 0x181 -#define CSR_SCYCLE 0xd00 -#define CSR_STIME 0xd01 -#define CSR_SINSTRET 0xd02 -#define CSR_MSTATUS 0x300 -#define CSR_MEDELEG 0x302 -#define CSR_MIDELEG 0x303 -#define CSR_MIE 0x304 -#define CSR_MTVEC 0x305 -#define CSR_MTIMECMP 0x321 -#define CSR_MSCRATCH 0x340 -#define CSR_MEPC 0x341 -#define CSR_MCAUSE 0x342 -#define CSR_MBADADDR 0x343 -#define CSR_MIP 0x344 -#define CSR_MIPI 0x345 -#define CSR_MUCOUNTEREN 0x310 -#define CSR_MSCOUNTEREN 0x311 -#define CSR_MUCYCLE_DELTA 0x700 -#define CSR_MUTIME_DELTA 0x701 -#define CSR_MUINSTRET_DELTA 0x702 -#define CSR_MSCYCLE_DELTA 0x704 -#define CSR_MSTIME_DELTA 0x705 -#define CSR_MSINSTRET_DELTA 0x706 -#define CSR_MCYCLE 0xf00 -#define CSR_MTIME 0xf01 -#define CSR_MINSTRET 0xf02 -#define CSR_MISA 0xf10 -#define CSR_MVENDORID 0xf11 -#define CSR_MARCHID 0xf12 -#define CSR_MIMPID 0xf13 -#define CSR_MCFGADDR 0xf14 -#define CSR_MHARTID 0xf15 -#define CSR_MTOHOST 0x7c0 -#define CSR_MFROMHOST 0x7c1 -#define CSR_MRESET 0x7c2 -#define CSR_CYCLEH 0xc80 -#define CSR_TIMEH 0xc81 -#define CSR_INSTRETH 0xc82 -#define CSR_MTIMECMPH 0x361 -#define CSR_MUCYCLE_DELTAH 0x780 -#define CSR_MUTIME_DELTAH 0x781 -#define CSR_MUINSTRET_DELTAH 0x782 -#define CSR_MSCYCLE_DELTAH 0x784 -#define CSR_MSTIME_DELTAH 0x785 -#define CSR_MSINSTRET_DELTAH 0x786 -#define CSR_MCYCLEH 0xf80 -#define CSR_MTIMEH 0xf81 -#define CSR_MINSTRETH 0xf82 -#define CAUSE_MISALIGNED_FETCH 0x0 -#define CAUSE_FAULT_FETCH 0x1 -#define CAUSE_ILLEGAL_INSTRUCTION 0x2 -#define CAUSE_BREAKPOINT 0x3 -#define CAUSE_MISALIGNED_LOAD 0x4 -#define CAUSE_FAULT_LOAD 0x5 -#define CAUSE_MISALIGNED_STORE 0x6 -#define CAUSE_FAULT_STORE 0x7 -#define CAUSE_USER_ECALL 0x8 -#define CAUSE_SUPERVISOR_ECALL 0x9 -#define CAUSE_HYPERVISOR_ECALL 0xa -#define CAUSE_MACHINE_ECALL 0xb -#endif -#ifdef DECLARE_INSN -DECLARE_INSN(beq, MATCH_BEQ, MASK_BEQ) -DECLARE_INSN(bne, MATCH_BNE, MASK_BNE) -DECLARE_INSN(blt, MATCH_BLT, MASK_BLT) -DECLARE_INSN(bge, MATCH_BGE, MASK_BGE) -DECLARE_INSN(bltu, MATCH_BLTU, MASK_BLTU) -DECLARE_INSN(bgeu, MATCH_BGEU, MASK_BGEU) -DECLARE_INSN(jalr, MATCH_JALR, MASK_JALR) -DECLARE_INSN(jal, MATCH_JAL, MASK_JAL) -DECLARE_INSN(lui, MATCH_LUI, MASK_LUI) -DECLARE_INSN(auipc, MATCH_AUIPC, MASK_AUIPC) -DECLARE_INSN(addi, MATCH_ADDI, MASK_ADDI) -DECLARE_INSN(slli, MATCH_SLLI, MASK_SLLI) -DECLARE_INSN(slti, MATCH_SLTI, MASK_SLTI) -DECLARE_INSN(sltiu, MATCH_SLTIU, MASK_SLTIU) -DECLARE_INSN(xori, MATCH_XORI, MASK_XORI) -DECLARE_INSN(srli, MATCH_SRLI, MASK_SRLI) -DECLARE_INSN(srai, MATCH_SRAI, MASK_SRAI) -DECLARE_INSN(ori, MATCH_ORI, MASK_ORI) -DECLARE_INSN(andi, MATCH_ANDI, MASK_ANDI) -DECLARE_INSN(add, MATCH_ADD, MASK_ADD) -DECLARE_INSN(sub, MATCH_SUB, MASK_SUB) -DECLARE_INSN(sll, MATCH_SLL, MASK_SLL) -DECLARE_INSN(slt, MATCH_SLT, MASK_SLT) -DECLARE_INSN(sltu, MATCH_SLTU, MASK_SLTU) -DECLARE_INSN(xor, MATCH_XOR, MASK_XOR) -DECLARE_INSN(srl, MATCH_SRL, MASK_SRL) -DECLARE_INSN(sra, MATCH_SRA, MASK_SRA) -DECLARE_INSN(or, MATCH_OR, MASK_OR) -DECLARE_INSN(and, MATCH_AND, MASK_AND) -DECLARE_INSN(addiw, MATCH_ADDIW, MASK_ADDIW) -DECLARE_INSN(slliw, MATCH_SLLIW, MASK_SLLIW) -DECLARE_INSN(srliw, MATCH_SRLIW, MASK_SRLIW) -DECLARE_INSN(sraiw, MATCH_SRAIW, MASK_SRAIW) -DECLARE_INSN(addw, MATCH_ADDW, MASK_ADDW) -DECLARE_INSN(subw, MATCH_SUBW, MASK_SUBW) -DECLARE_INSN(sllw, MATCH_SLLW, MASK_SLLW) -DECLARE_INSN(srlw, MATCH_SRLW, MASK_SRLW) -DECLARE_INSN(sraw, MATCH_SRAW, MASK_SRAW) -DECLARE_INSN(lb, MATCH_LB, MASK_LB) -DECLARE_INSN(lh, MATCH_LH, MASK_LH) -DECLARE_INSN(lw, MATCH_LW, MASK_LW) -DECLARE_INSN(ld, MATCH_LD, MASK_LD) -DECLARE_INSN(lbu, MATCH_LBU, MASK_LBU) -DECLARE_INSN(lhu, MATCH_LHU, MASK_LHU) -DECLARE_INSN(lwu, MATCH_LWU, MASK_LWU) -DECLARE_INSN(sb, MATCH_SB, MASK_SB) -DECLARE_INSN(sh, MATCH_SH, MASK_SH) -DECLARE_INSN(sw, MATCH_SW, MASK_SW) -DECLARE_INSN(sd, MATCH_SD, MASK_SD) -DECLARE_INSN(fence, MATCH_FENCE, MASK_FENCE) -DECLARE_INSN(fence_i, MATCH_FENCE_I, MASK_FENCE_I) -DECLARE_INSN(mul, MATCH_MUL, MASK_MUL) -DECLARE_INSN(mulh, MATCH_MULH, MASK_MULH) -DECLARE_INSN(mulhsu, MATCH_MULHSU, MASK_MULHSU) -DECLARE_INSN(mulhu, MATCH_MULHU, MASK_MULHU) -DECLARE_INSN(div, MATCH_DIV, MASK_DIV) -DECLARE_INSN(divu, MATCH_DIVU, MASK_DIVU) -DECLARE_INSN(rem, MATCH_REM, MASK_REM) -DECLARE_INSN(remu, MATCH_REMU, MASK_REMU) -DECLARE_INSN(mulw, MATCH_MULW, MASK_MULW) -DECLARE_INSN(divw, MATCH_DIVW, MASK_DIVW) -DECLARE_INSN(divuw, MATCH_DIVUW, MASK_DIVUW) -DECLARE_INSN(remw, MATCH_REMW, MASK_REMW) -DECLARE_INSN(remuw, MATCH_REMUW, MASK_REMUW) -DECLARE_INSN(amoadd_w, MATCH_AMOADD_W, MASK_AMOADD_W) -DECLARE_INSN(amoxor_w, MATCH_AMOXOR_W, MASK_AMOXOR_W) -DECLARE_INSN(amoor_w, MATCH_AMOOR_W, MASK_AMOOR_W) -DECLARE_INSN(amoand_w, MATCH_AMOAND_W, MASK_AMOAND_W) -DECLARE_INSN(amomin_w, MATCH_AMOMIN_W, MASK_AMOMIN_W) -DECLARE_INSN(amomax_w, MATCH_AMOMAX_W, MASK_AMOMAX_W) -DECLARE_INSN(amominu_w, MATCH_AMOMINU_W, MASK_AMOMINU_W) -DECLARE_INSN(amomaxu_w, MATCH_AMOMAXU_W, MASK_AMOMAXU_W) -DECLARE_INSN(amoswap_w, MATCH_AMOSWAP_W, MASK_AMOSWAP_W) -DECLARE_INSN(lr_w, MATCH_LR_W, MASK_LR_W) -DECLARE_INSN(sc_w, MATCH_SC_W, MASK_SC_W) -DECLARE_INSN(amoadd_d, MATCH_AMOADD_D, MASK_AMOADD_D) -DECLARE_INSN(amoxor_d, MATCH_AMOXOR_D, MASK_AMOXOR_D) -DECLARE_INSN(amoor_d, MATCH_AMOOR_D, MASK_AMOOR_D) -DECLARE_INSN(amoand_d, MATCH_AMOAND_D, MASK_AMOAND_D) -DECLARE_INSN(amomin_d, MATCH_AMOMIN_D, MASK_AMOMIN_D) -DECLARE_INSN(amomax_d, MATCH_AMOMAX_D, MASK_AMOMAX_D) -DECLARE_INSN(amominu_d, MATCH_AMOMINU_D, MASK_AMOMINU_D) -DECLARE_INSN(amomaxu_d, MATCH_AMOMAXU_D, MASK_AMOMAXU_D) -DECLARE_INSN(amoswap_d, MATCH_AMOSWAP_D, MASK_AMOSWAP_D) -DECLARE_INSN(lr_d, MATCH_LR_D, MASK_LR_D) -DECLARE_INSN(sc_d, MATCH_SC_D, MASK_SC_D) -DECLARE_INSN(scall, MATCH_SCALL, MASK_SCALL) -DECLARE_INSN(sbreak, MATCH_SBREAK, MASK_SBREAK) -DECLARE_INSN(sret, MATCH_SRET, MASK_SRET) -DECLARE_INSN(sfence_vm, MATCH_SFENCE_VM, MASK_SFENCE_VM) -DECLARE_INSN(wfi, MATCH_WFI, MASK_WFI) -DECLARE_INSN(csrrw, MATCH_CSRRW, MASK_CSRRW) -DECLARE_INSN(csrrs, MATCH_CSRRS, MASK_CSRRS) -DECLARE_INSN(csrrc, MATCH_CSRRC, MASK_CSRRC) -DECLARE_INSN(csrrwi, MATCH_CSRRWI, MASK_CSRRWI) -DECLARE_INSN(csrrsi, MATCH_CSRRSI, MASK_CSRRSI) -DECLARE_INSN(csrrci, MATCH_CSRRCI, MASK_CSRRCI) -DECLARE_INSN(fadd_s, MATCH_FADD_S, MASK_FADD_S) -DECLARE_INSN(fsub_s, MATCH_FSUB_S, MASK_FSUB_S) -DECLARE_INSN(fmul_s, MATCH_FMUL_S, MASK_FMUL_S) -DECLARE_INSN(fdiv_s, MATCH_FDIV_S, MASK_FDIV_S) -DECLARE_INSN(fsgnj_s, MATCH_FSGNJ_S, MASK_FSGNJ_S) -DECLARE_INSN(fsgnjn_s, MATCH_FSGNJN_S, MASK_FSGNJN_S) -DECLARE_INSN(fsgnjx_s, MATCH_FSGNJX_S, MASK_FSGNJX_S) -DECLARE_INSN(fmin_s, MATCH_FMIN_S, MASK_FMIN_S) -DECLARE_INSN(fmax_s, MATCH_FMAX_S, MASK_FMAX_S) -DECLARE_INSN(fsqrt_s, MATCH_FSQRT_S, MASK_FSQRT_S) -DECLARE_INSN(fadd_d, MATCH_FADD_D, MASK_FADD_D) -DECLARE_INSN(fsub_d, MATCH_FSUB_D, MASK_FSUB_D) -DECLARE_INSN(fmul_d, MATCH_FMUL_D, MASK_FMUL_D) -DECLARE_INSN(fdiv_d, MATCH_FDIV_D, MASK_FDIV_D) -DECLARE_INSN(fsgnj_d, MATCH_FSGNJ_D, MASK_FSGNJ_D) -DECLARE_INSN(fsgnjn_d, MATCH_FSGNJN_D, MASK_FSGNJN_D) -DECLARE_INSN(fsgnjx_d, MATCH_FSGNJX_D, MASK_FSGNJX_D) -DECLARE_INSN(fmin_d, MATCH_FMIN_D, MASK_FMIN_D) -DECLARE_INSN(fmax_d, MATCH_FMAX_D, MASK_FMAX_D) -DECLARE_INSN(fcvt_s_d, MATCH_FCVT_S_D, MASK_FCVT_S_D) -DECLARE_INSN(fcvt_d_s, MATCH_FCVT_D_S, MASK_FCVT_D_S) -DECLARE_INSN(fsqrt_d, MATCH_FSQRT_D, MASK_FSQRT_D) -DECLARE_INSN(fle_s, MATCH_FLE_S, MASK_FLE_S) -DECLARE_INSN(flt_s, MATCH_FLT_S, MASK_FLT_S) -DECLARE_INSN(feq_s, MATCH_FEQ_S, MASK_FEQ_S) -DECLARE_INSN(fle_d, MATCH_FLE_D, MASK_FLE_D) -DECLARE_INSN(flt_d, MATCH_FLT_D, MASK_FLT_D) -DECLARE_INSN(feq_d, MATCH_FEQ_D, MASK_FEQ_D) -DECLARE_INSN(fcvt_w_s, MATCH_FCVT_W_S, MASK_FCVT_W_S) -DECLARE_INSN(fcvt_wu_s, MATCH_FCVT_WU_S, MASK_FCVT_WU_S) -DECLARE_INSN(fcvt_l_s, MATCH_FCVT_L_S, MASK_FCVT_L_S) -DECLARE_INSN(fcvt_lu_s, MATCH_FCVT_LU_S, MASK_FCVT_LU_S) -DECLARE_INSN(fmv_x_s, MATCH_FMV_X_S, MASK_FMV_X_S) -DECLARE_INSN(fclass_s, MATCH_FCLASS_S, MASK_FCLASS_S) -DECLARE_INSN(fcvt_w_d, MATCH_FCVT_W_D, MASK_FCVT_W_D) -DECLARE_INSN(fcvt_wu_d, MATCH_FCVT_WU_D, MASK_FCVT_WU_D) -DECLARE_INSN(fcvt_l_d, MATCH_FCVT_L_D, MASK_FCVT_L_D) -DECLARE_INSN(fcvt_lu_d, MATCH_FCVT_LU_D, MASK_FCVT_LU_D) -DECLARE_INSN(fmv_x_d, MATCH_FMV_X_D, MASK_FMV_X_D) -DECLARE_INSN(fclass_d, MATCH_FCLASS_D, MASK_FCLASS_D) -DECLARE_INSN(fcvt_s_w, MATCH_FCVT_S_W, MASK_FCVT_S_W) -DECLARE_INSN(fcvt_s_wu, MATCH_FCVT_S_WU, MASK_FCVT_S_WU) -DECLARE_INSN(fcvt_s_l, MATCH_FCVT_S_L, MASK_FCVT_S_L) -DECLARE_INSN(fcvt_s_lu, MATCH_FCVT_S_LU, MASK_FCVT_S_LU) -DECLARE_INSN(fmv_s_x, MATCH_FMV_S_X, MASK_FMV_S_X) -DECLARE_INSN(fcvt_d_w, MATCH_FCVT_D_W, MASK_FCVT_D_W) -DECLARE_INSN(fcvt_d_wu, MATCH_FCVT_D_WU, MASK_FCVT_D_WU) -DECLARE_INSN(fcvt_d_l, MATCH_FCVT_D_L, MASK_FCVT_D_L) -DECLARE_INSN(fcvt_d_lu, MATCH_FCVT_D_LU, MASK_FCVT_D_LU) -DECLARE_INSN(fmv_d_x, MATCH_FMV_D_X, MASK_FMV_D_X) -DECLARE_INSN(flw, MATCH_FLW, MASK_FLW) -DECLARE_INSN(fld, MATCH_FLD, MASK_FLD) -DECLARE_INSN(fsw, MATCH_FSW, MASK_FSW) -DECLARE_INSN(fsd, MATCH_FSD, MASK_FSD) -DECLARE_INSN(fmadd_s, MATCH_FMADD_S, MASK_FMADD_S) -DECLARE_INSN(fmsub_s, MATCH_FMSUB_S, MASK_FMSUB_S) -DECLARE_INSN(fnmsub_s, MATCH_FNMSUB_S, MASK_FNMSUB_S) -DECLARE_INSN(fnmadd_s, MATCH_FNMADD_S, MASK_FNMADD_S) -DECLARE_INSN(fmadd_d, MATCH_FMADD_D, MASK_FMADD_D) -DECLARE_INSN(fmsub_d, MATCH_FMSUB_D, MASK_FMSUB_D) -DECLARE_INSN(fnmsub_d, MATCH_FNMSUB_D, MASK_FNMSUB_D) -DECLARE_INSN(fnmadd_d, MATCH_FNMADD_D, MASK_FNMADD_D) -DECLARE_INSN(c_nop, MATCH_C_NOP, MASK_C_NOP) -DECLARE_INSN(c_addi16sp, MATCH_C_ADDI16SP, MASK_C_ADDI16SP) -DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR) -DECLARE_INSN(c_jalr, MATCH_C_JALR, MASK_C_JALR) -DECLARE_INSN(c_ebreak, MATCH_C_EBREAK, MASK_C_EBREAK) -DECLARE_INSN(c_ld, MATCH_C_LD, MASK_C_LD) -DECLARE_INSN(c_sd, MATCH_C_SD, MASK_C_SD) -DECLARE_INSN(c_addiw, MATCH_C_ADDIW, MASK_C_ADDIW) -DECLARE_INSN(c_ldsp, MATCH_C_LDSP, MASK_C_LDSP) -DECLARE_INSN(c_sdsp, MATCH_C_SDSP, MASK_C_SDSP) -DECLARE_INSN(c_addi4spn, MATCH_C_ADDI4SPN, MASK_C_ADDI4SPN) -DECLARE_INSN(c_fld, MATCH_C_FLD, MASK_C_FLD) -DECLARE_INSN(c_lw, MATCH_C_LW, MASK_C_LW) -DECLARE_INSN(c_flw, MATCH_C_FLW, MASK_C_FLW) -DECLARE_INSN(c_fsd, MATCH_C_FSD, MASK_C_FSD) -DECLARE_INSN(c_sw, MATCH_C_SW, MASK_C_SW) -DECLARE_INSN(c_fsw, MATCH_C_FSW, MASK_C_FSW) -DECLARE_INSN(c_addi, MATCH_C_ADDI, MASK_C_ADDI) -DECLARE_INSN(c_jal, MATCH_C_JAL, MASK_C_JAL) -DECLARE_INSN(c_li, MATCH_C_LI, MASK_C_LI) -DECLARE_INSN(c_lui, MATCH_C_LUI, MASK_C_LUI) -DECLARE_INSN(c_srli, MATCH_C_SRLI, MASK_C_SRLI) -DECLARE_INSN(c_srai, MATCH_C_SRAI, MASK_C_SRAI) -DECLARE_INSN(c_andi, MATCH_C_ANDI, MASK_C_ANDI) -DECLARE_INSN(c_sub, MATCH_C_SUB, MASK_C_SUB) -DECLARE_INSN(c_xor, MATCH_C_XOR, MASK_C_XOR) -DECLARE_INSN(c_or, MATCH_C_OR, MASK_C_OR) -DECLARE_INSN(c_and, MATCH_C_AND, MASK_C_AND) -DECLARE_INSN(c_subw, MATCH_C_SUBW, MASK_C_SUBW) -DECLARE_INSN(c_addw, MATCH_C_ADDW, MASK_C_ADDW) -DECLARE_INSN(c_j, MATCH_C_J, MASK_C_J) -DECLARE_INSN(c_beqz, MATCH_C_BEQZ, MASK_C_BEQZ) -DECLARE_INSN(c_bnez, MATCH_C_BNEZ, MASK_C_BNEZ) -DECLARE_INSN(c_slli, MATCH_C_SLLI, MASK_C_SLLI) -DECLARE_INSN(c_fldsp, MATCH_C_FLDSP, MASK_C_FLDSP) -DECLARE_INSN(c_lwsp, MATCH_C_LWSP, MASK_C_LWSP) -DECLARE_INSN(c_flwsp, MATCH_C_FLWSP, MASK_C_FLWSP) -DECLARE_INSN(c_mv, MATCH_C_MV, MASK_C_MV) -DECLARE_INSN(c_add, MATCH_C_ADD, MASK_C_ADD) -DECLARE_INSN(c_fsdsp, MATCH_C_FSDSP, MASK_C_FSDSP) -DECLARE_INSN(c_swsp, MATCH_C_SWSP, MASK_C_SWSP) -DECLARE_INSN(c_fswsp, MATCH_C_FSWSP, MASK_C_FSWSP) -DECLARE_INSN(custom0, MATCH_CUSTOM0, MASK_CUSTOM0) -DECLARE_INSN(custom0_rs1, MATCH_CUSTOM0_RS1, MASK_CUSTOM0_RS1) -DECLARE_INSN(custom0_rs1_rs2, MATCH_CUSTOM0_RS1_RS2, MASK_CUSTOM0_RS1_RS2) -DECLARE_INSN(custom0_rd, MATCH_CUSTOM0_RD, MASK_CUSTOM0_RD) -DECLARE_INSN(custom0_rd_rs1, MATCH_CUSTOM0_RD_RS1, MASK_CUSTOM0_RD_RS1) -DECLARE_INSN(custom0_rd_rs1_rs2, MATCH_CUSTOM0_RD_RS1_RS2, MASK_CUSTOM0_RD_RS1_RS2) -DECLARE_INSN(custom1, MATCH_CUSTOM1, MASK_CUSTOM1) -DECLARE_INSN(custom1_rs1, MATCH_CUSTOM1_RS1, MASK_CUSTOM1_RS1) -DECLARE_INSN(custom1_rs1_rs2, MATCH_CUSTOM1_RS1_RS2, MASK_CUSTOM1_RS1_RS2) -DECLARE_INSN(custom1_rd, MATCH_CUSTOM1_RD, MASK_CUSTOM1_RD) -DECLARE_INSN(custom1_rd_rs1, MATCH_CUSTOM1_RD_RS1, MASK_CUSTOM1_RD_RS1) -DECLARE_INSN(custom1_rd_rs1_rs2, MATCH_CUSTOM1_RD_RS1_RS2, MASK_CUSTOM1_RD_RS1_RS2) -DECLARE_INSN(custom2, MATCH_CUSTOM2, MASK_CUSTOM2) -DECLARE_INSN(custom2_rs1, MATCH_CUSTOM2_RS1, MASK_CUSTOM2_RS1) -DECLARE_INSN(custom2_rs1_rs2, MATCH_CUSTOM2_RS1_RS2, MASK_CUSTOM2_RS1_RS2) -DECLARE_INSN(custom2_rd, MATCH_CUSTOM2_RD, MASK_CUSTOM2_RD) -DECLARE_INSN(custom2_rd_rs1, MATCH_CUSTOM2_RD_RS1, MASK_CUSTOM2_RD_RS1) -DECLARE_INSN(custom2_rd_rs1_rs2, MATCH_CUSTOM2_RD_RS1_RS2, MASK_CUSTOM2_RD_RS1_RS2) -DECLARE_INSN(custom3, MATCH_CUSTOM3, MASK_CUSTOM3) -DECLARE_INSN(custom3_rs1, MATCH_CUSTOM3_RS1, MASK_CUSTOM3_RS1) -DECLARE_INSN(custom3_rs1_rs2, MATCH_CUSTOM3_RS1_RS2, MASK_CUSTOM3_RS1_RS2) -DECLARE_INSN(custom3_rd, MATCH_CUSTOM3_RD, MASK_CUSTOM3_RD) -DECLARE_INSN(custom3_rd_rs1, MATCH_CUSTOM3_RD_RS1, MASK_CUSTOM3_RD_RS1) -DECLARE_INSN(custom3_rd_rs1_rs2, MATCH_CUSTOM3_RD_RS1_RS2, MASK_CUSTOM3_RD_RS1_RS2) -#endif -#ifdef DECLARE_CSR -DECLARE_CSR(fflags, CSR_FFLAGS) -DECLARE_CSR(frm, CSR_FRM) -DECLARE_CSR(fcsr, CSR_FCSR) -DECLARE_CSR(cycle, CSR_CYCLE) -DECLARE_CSR(time, CSR_TIME) -DECLARE_CSR(instret, CSR_INSTRET) -DECLARE_CSR(stats, CSR_STATS) -DECLARE_CSR(uarch0, CSR_UARCH0) -DECLARE_CSR(uarch1, CSR_UARCH1) -DECLARE_CSR(uarch2, CSR_UARCH2) -DECLARE_CSR(uarch3, CSR_UARCH3) -DECLARE_CSR(uarch4, CSR_UARCH4) -DECLARE_CSR(uarch5, CSR_UARCH5) -DECLARE_CSR(uarch6, CSR_UARCH6) -DECLARE_CSR(uarch7, CSR_UARCH7) -DECLARE_CSR(uarch8, CSR_UARCH8) -DECLARE_CSR(uarch9, CSR_UARCH9) -DECLARE_CSR(uarch10, CSR_UARCH10) -DECLARE_CSR(uarch11, CSR_UARCH11) -DECLARE_CSR(uarch12, CSR_UARCH12) -DECLARE_CSR(uarch13, CSR_UARCH13) -DECLARE_CSR(uarch14, CSR_UARCH14) -DECLARE_CSR(uarch15, CSR_UARCH15) -DECLARE_CSR(sstatus, CSR_SSTATUS) -DECLARE_CSR(sie, CSR_SIE) -DECLARE_CSR(stvec, CSR_STVEC) -DECLARE_CSR(sscratch, CSR_SSCRATCH) -DECLARE_CSR(sepc, CSR_SEPC) -DECLARE_CSR(scause, CSR_SCAUSE) -DECLARE_CSR(sbadaddr, CSR_SBADADDR) -DECLARE_CSR(sip, CSR_SIP) -DECLARE_CSR(sptbr, CSR_SPTBR) -DECLARE_CSR(sasid, CSR_SASID) -DECLARE_CSR(scycle, CSR_SCYCLE) -DECLARE_CSR(stime, CSR_STIME) -DECLARE_CSR(sinstret, CSR_SINSTRET) -DECLARE_CSR(mstatus, CSR_MSTATUS) -DECLARE_CSR(medeleg, CSR_MEDELEG) -DECLARE_CSR(mideleg, CSR_MIDELEG) -DECLARE_CSR(mie, CSR_MIE) -DECLARE_CSR(mtvec, CSR_MTVEC) -DECLARE_CSR(mtimecmp, CSR_MTIMECMP) -DECLARE_CSR(mscratch, CSR_MSCRATCH) -DECLARE_CSR(mepc, CSR_MEPC) -DECLARE_CSR(mcause, CSR_MCAUSE) -DECLARE_CSR(mbadaddr, CSR_MBADADDR) -DECLARE_CSR(mip, CSR_MIP) -DECLARE_CSR(mipi, CSR_MIPI) -DECLARE_CSR(mucounteren, CSR_MUCOUNTEREN) -DECLARE_CSR(mscounteren, CSR_MSCOUNTEREN) -DECLARE_CSR(mucycle_delta, CSR_MUCYCLE_DELTA) -DECLARE_CSR(mutime_delta, CSR_MUTIME_DELTA) -DECLARE_CSR(muinstret_delta, CSR_MUINSTRET_DELTA) -DECLARE_CSR(mscycle_delta, CSR_MSCYCLE_DELTA) -DECLARE_CSR(mstime_delta, CSR_MSTIME_DELTA) -DECLARE_CSR(msinstret_delta, CSR_MSINSTRET_DELTA) -DECLARE_CSR(mcycle, CSR_MCYCLE) -DECLARE_CSR(mtime, CSR_MTIME) -DECLARE_CSR(minstret, CSR_MINSTRET) -DECLARE_CSR(misa, CSR_MISA) -DECLARE_CSR(mvendorid, CSR_MVENDORID) -DECLARE_CSR(marchid, CSR_MARCHID) -DECLARE_CSR(mimpid, CSR_MIMPID) -DECLARE_CSR(mcfgaddr, CSR_MCFGADDR) -DECLARE_CSR(mhartid, CSR_MHARTID) -DECLARE_CSR(mtohost, CSR_MTOHOST) -DECLARE_CSR(mfromhost, CSR_MFROMHOST) -DECLARE_CSR(mreset, CSR_MRESET) -DECLARE_CSR(cycleh, CSR_CYCLEH) -DECLARE_CSR(timeh, CSR_TIMEH) -DECLARE_CSR(instreth, CSR_INSTRETH) -DECLARE_CSR(mtimecmph, CSR_MTIMECMPH) -DECLARE_CSR(mucycle_deltah, CSR_MUCYCLE_DELTAH) -DECLARE_CSR(mutime_deltah, CSR_MUTIME_DELTAH) -DECLARE_CSR(muinstret_deltah, CSR_MUINSTRET_DELTAH) -DECLARE_CSR(mscycle_deltah, CSR_MSCYCLE_DELTAH) -DECLARE_CSR(mstime_deltah, CSR_MSTIME_DELTAH) -DECLARE_CSR(msinstret_deltah, CSR_MSINSTRET_DELTAH) -DECLARE_CSR(mcycleh, CSR_MCYCLEH) -DECLARE_CSR(mtimeh, CSR_MTIMEH) -DECLARE_CSR(minstreth, CSR_MINSTRETH) -#endif -#ifdef DECLARE_CAUSE -DECLARE_CAUSE("misaligned fetch", CAUSE_MISALIGNED_FETCH) -DECLARE_CAUSE("fault fetch", CAUSE_FAULT_FETCH) -DECLARE_CAUSE("illegal instruction", CAUSE_ILLEGAL_INSTRUCTION) -DECLARE_CAUSE("breakpoint", CAUSE_BREAKPOINT) -DECLARE_CAUSE("misaligned load", CAUSE_MISALIGNED_LOAD) -DECLARE_CAUSE("fault load", CAUSE_FAULT_LOAD) -DECLARE_CAUSE("misaligned store", CAUSE_MISALIGNED_STORE) -DECLARE_CAUSE("fault store", CAUSE_FAULT_STORE) -DECLARE_CAUSE("user_ecall", CAUSE_USER_ECALL) -DECLARE_CAUSE("supervisor_ecall", CAUSE_SUPERVISOR_ECALL) -DECLARE_CAUSE("hypervisor_ecall", CAUSE_HYPERVISOR_ECALL) -DECLARE_CAUSE("machine_ecall", CAUSE_MACHINE_ECALL) -#endif @@ -1,11 +1,13 @@ // See LICENSE for license details. -#include <string.h> -#include <errno.h> #include "file.h" -#include "pk.h" +#include "atomic.h" +#include "mmap.h" #include "frontend.h" -#include "vm.h" +#include "syscall.h" +#include "pk.h" +#include <string.h> +#include <errno.h> #define MAX_FDS 128 static file_t* fds[MAX_FDS]; @@ -6,7 +6,6 @@ #include <sys/stat.h> #include <unistd.h> #include <stdint.h> -#include "atomic.h" typedef struct file { diff --git a/pk/fp_asm.S b/pk/fp_asm.S deleted file mode 100644 index 4b8dce1..0000000 --- a/pk/fp_asm.S +++ /dev/null @@ -1,160 +0,0 @@ -// See LICENSE for license details. - -#ifdef __riscv_hard_float - -#define get_f32(which) fmv.x.s a0, which; jr t0 -#define put_f32(which) fmv.s.x which, a0; jr t0 -#ifdef __riscv64 -# define get_f64(which) fmv.x.d a0, which; jr t0 -# define put_f64(which) fmv.d.x which, a0; jr t0 -#else -# define get_f64(which) fsd which, 0(a0); jr t0 -# define put_f64(which) fld which, 0(a0); jr t0 -#endif - - .text - .option norvc - .globl get_f32_reg - get_f32_reg: - get_f32(f0) - get_f32(f1) - get_f32(f2) - get_f32(f3) - get_f32(f4) - get_f32(f5) - get_f32(f6) - get_f32(f7) - get_f32(f8) - get_f32(f9) - get_f32(f10) - get_f32(f11) - get_f32(f12) - get_f32(f13) - get_f32(f14) - get_f32(f15) - get_f32(f16) - get_f32(f17) - get_f32(f18) - get_f32(f19) - get_f32(f20) - get_f32(f21) - get_f32(f22) - get_f32(f23) - get_f32(f24) - get_f32(f25) - get_f32(f26) - get_f32(f27) - get_f32(f28) - get_f32(f29) - get_f32(f30) - get_f32(f31) - - .text - .globl put_f32_reg - put_f32_reg: - put_f32(f0) - put_f32(f1) - put_f32(f2) - put_f32(f3) - put_f32(f4) - put_f32(f5) - put_f32(f6) - put_f32(f7) - put_f32(f8) - put_f32(f9) - put_f32(f10) - put_f32(f11) - put_f32(f12) - put_f32(f13) - put_f32(f14) - put_f32(f15) - put_f32(f16) - put_f32(f17) - put_f32(f18) - put_f32(f19) - put_f32(f20) - put_f32(f21) - put_f32(f22) - put_f32(f23) - put_f32(f24) - put_f32(f25) - put_f32(f26) - put_f32(f27) - put_f32(f28) - put_f32(f29) - put_f32(f30) - put_f32(f31) - - .text - .globl get_f64_reg - get_f64_reg: - get_f64(f0) - get_f64(f1) - get_f64(f2) - get_f64(f3) - get_f64(f4) - get_f64(f5) - get_f64(f6) - get_f64(f7) - get_f64(f8) - get_f64(f9) - get_f64(f10) - get_f64(f11) - get_f64(f12) - get_f64(f13) - get_f64(f14) - get_f64(f15) - get_f64(f16) - get_f64(f17) - get_f64(f18) - get_f64(f19) - get_f64(f20) - get_f64(f21) - get_f64(f22) - get_f64(f23) - get_f64(f24) - get_f64(f25) - get_f64(f26) - get_f64(f27) - get_f64(f28) - get_f64(f29) - get_f64(f30) - get_f64(f31) - - .text - .globl put_f64_reg - put_f64_reg: - put_f64(f0) - put_f64(f1) - put_f64(f2) - put_f64(f3) - put_f64(f4) - put_f64(f5) - put_f64(f6) - put_f64(f7) - put_f64(f8) - put_f64(f9) - put_f64(f10) - put_f64(f11) - put_f64(f12) - put_f64(f13) - put_f64(f14) - put_f64(f15) - put_f64(f16) - put_f64(f17) - put_f64(f18) - put_f64(f19) - put_f64(f20) - put_f64(f21) - put_f64(f22) - put_f64(f23) - put_f64(f24) - put_f64(f25) - put_f64(f26) - put_f64(f27) - put_f64(f28) - put_f64(f29) - put_f64(f30) - put_f64(f31) - -#endif diff --git a/pk/fp_emulation.c b/pk/fp_emulation.c deleted file mode 100644 index 536967f..0000000 --- a/pk/fp_emulation.c +++ /dev/null @@ -1,449 +0,0 @@ -#include "fp_emulation.h" -#include "unprivileged_memory.h" -#include "softfloat.h" -#include "config.h" - -DECLARE_EMULATION_FUNC(emulate_float_load) -{ - uint64_t val; - uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); - - switch (insn & MASK_FUNCT3) - { - case MATCH_FLW & MASK_FUNCT3: - if (addr % 4 != 0) - return misaligned_load_trap(regs, mcause, mepc); - - SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc)); - break; - - case MATCH_FLD & MASK_FUNCT3: - if (addr % sizeof(uintptr_t) != 0) - return misaligned_load_trap(regs, mcause, mepc); - -#ifdef __riscv64 - val = load_uint64_t((void *)addr, mepc); -#else - val = load_uint32_t(addr, mepc); - val += (uint64_t)load_uint32_t((void *)(addr + 4), mepc) << 32; -#endif - SET_F64_RD(insn, regs, val); - break; - - default: - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_float_store) -{ - uint64_t val; - uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); - - switch (insn & MASK_FUNCT3) - { - case MATCH_FSW & MASK_FUNCT3: - if (addr % 4 != 0) - return misaligned_store_trap(regs, mcause, mepc); - - store_uint32_t((void *)addr, GET_F32_RS2(insn, regs), mepc); - break; - - case MATCH_FSD & MASK_FUNCT3: - if (addr % sizeof(uintptr_t) != 0) - return misaligned_store_trap(regs, mcause, mepc); - - val = GET_F64_RS2(insn, regs); -#ifdef __riscv64 - store_uint64_t((void *)addr, val, mepc); -#else - store_uint32_t((void *)addr, val, mepc); - store_uint32_t((void *)(addr + 4), val >> 32, mepc); -#endif - break; - - default: - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fp) -{ - asm (".pushsection .rodata\n" - "fp_emulation_table:\n" - " .word emulate_fadd\n" - " .word emulate_fsub\n" - " .word emulate_fmul\n" - " .word emulate_fdiv\n" - " .word emulate_fsgnj\n" - " .word emulate_fmin\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_fcvt_ff\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_fsqrt\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_fcmp\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_fcvt_if\n" - " .word truly_illegal_insn\n" - " .word emulate_fcvt_fi\n" - " .word truly_illegal_insn\n" - " .word emulate_fmv_if\n" - " .word truly_illegal_insn\n" - " .word emulate_fmv_fi\n" - " .word truly_illegal_insn\n" - " .popsection"); - - // if FPU is disabled, punt back to the OS - if (unlikely((mstatus & MSTATUS_FS) == 0)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - extern int32_t fp_emulation_table[]; - int32_t* pf = (void*)fp_emulation_table + ((insn >> 25) & 0x7c); - emulation_func f = (emulation_func)(uintptr_t)*pf; - - SETUP_STATIC_ROUNDING(insn); - return f(regs, mcause, mepc, mstatus, insn); -} - -void emulate_any_fadd(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn, int32_t neg_b) -{ - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs) ^ neg_b; - SET_F32_RD(insn, regs, f32_add(rs1, rs2)); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs) ^ ((uint64_t)neg_b << 32); - SET_F64_RD(insn, regs, f64_add(rs1, rs2)); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fadd) -{ - return emulate_any_fadd(regs, mcause, mepc, mstatus, insn, 0); -} - -DECLARE_EMULATION_FUNC(emulate_fsub) -{ - return emulate_any_fadd(regs, mcause, mepc, mstatus, insn, INT32_MIN); -} - -DECLARE_EMULATION_FUNC(emulate_fmul) -{ - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - SET_F32_RD(insn, regs, f32_mul(rs1, rs2)); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - SET_F64_RD(insn, regs, f64_mul(rs1, rs2)); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fdiv) -{ - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - SET_F32_RD(insn, regs, f32_div(rs1, rs2)); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - SET_F64_RD(insn, regs, f64_div(rs1, rs2)); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fsqrt) -{ - if ((insn >> 20) & 0x1f) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - if (GET_PRECISION(insn) == PRECISION_S) { - SET_F32_RD(insn, regs, f32_sqrt(GET_F32_RS1(insn, regs))); - } else if (GET_PRECISION(insn) == PRECISION_D) { - SET_F64_RD(insn, regs, f64_sqrt(GET_F64_RS1(insn, regs))); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fsgnj) -{ - int rm = GET_RM(insn); - if (rm >= 3) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - #define DO_FSGNJ(rs1, rs2, rm) ({ \ - typeof(rs1) rs1_sign = (rs1) >> (8*sizeof(rs1)-1); \ - typeof(rs1) rs2_sign = (rs2) >> (8*sizeof(rs1)-1); \ - rs1_sign &= (rm) >> 1; \ - rs1_sign ^= (rm) ^ rs2_sign; \ - ((rs1) << 1 >> 1) | (rs1_sign << (8*sizeof(rs1)-1)); }) - - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - SET_F32_RD(insn, regs, DO_FSGNJ(rs1, rs2, rm)); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - SET_F64_RD(insn, regs, DO_FSGNJ(rs1, rs2, rm)); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fmin) -{ - int rm = GET_RM(insn); - if (rm >= 2) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - uint32_t arg1 = rm ? rs2 : rs1; - uint32_t arg2 = rm ? rs1 : rs2; - int use_rs1 = f32_lt_quiet(arg1, arg2) || isNaNF32UI(rs2); - SET_F32_RD(insn, regs, use_rs1 ? rs1 : rs2); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - uint64_t arg1 = rm ? rs2 : rs1; - uint64_t arg2 = rm ? rs1 : rs2; - int use_rs1 = f64_lt_quiet(arg1, arg2) || isNaNF64UI(rs2); - SET_F64_RD(insn, regs, use_rs1 ? rs1 : rs2); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fcvt_ff) -{ - int rs2_num = (insn >> 20) & 0x1f; - if (GET_PRECISION(insn) == PRECISION_S) { - if (rs2_num != 1) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - SET_F32_RD(insn, regs, f64_to_f32(GET_F64_RS1(insn, regs))); - } else if (GET_PRECISION(insn) == PRECISION_D) { - if (rs2_num != 0) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - SET_F64_RD(insn, regs, f32_to_f64(GET_F32_RS1(insn, regs))); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fcvt_fi) -{ - if (GET_PRECISION(insn) != PRECISION_S && GET_PRECISION(insn) != PRECISION_D) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - int negative = 0; - uint64_t uint_val = GET_RS1(insn, regs); - - switch ((insn >> 20) & 0x1f) - { - case 0: // int32 - negative = (int32_t)uint_val < 0; - uint_val = negative ? -(int32_t)uint_val : (int32_t)uint_val; - break; - case 1: // uint32 - uint_val = (uint32_t)uint_val; - break; -#ifdef __riscv64 - case 2: // int64 - negative = (int64_t)uint_val < 0; - uint_val = negative ? -uint_val : uint_val; - case 3: // uint64 - break; -#endif - default: - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } - - uint64_t float64 = ui64_to_f64(uint_val); - if (negative) - float64 ^= INT64_MIN; - - if (GET_PRECISION(insn) == PRECISION_S) - SET_F32_RD(insn, regs, f64_to_f32(float64)); - else - SET_F64_RD(insn, regs, float64); -} - -DECLARE_EMULATION_FUNC(emulate_fcvt_if) -{ - int rs2_num = (insn >> 20) & 0x1f; -#ifdef __riscv64 - if (rs2_num >= 4) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); -#else - if (rs2_num >= 2) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); -#endif - - int64_t float64; - if (GET_PRECISION(insn) == PRECISION_S) - float64 = f32_to_f64(GET_F32_RS1(insn, regs)); - else if (GET_PRECISION(insn) == PRECISION_D) - float64 = GET_F64_RS1(insn, regs); - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - int negative = 0; - if (float64 < 0) { - negative = 1; - float64 ^= INT64_MIN; - } - uint64_t uint_val = f64_to_ui64(float64, softfloat_roundingMode, true); - uint64_t result, limit, limit_result; - - switch (rs2_num) - { - case 0: // int32 - if (negative) { - result = (int32_t)-uint_val; - limit_result = limit = (uint32_t)INT32_MIN; - } else { - result = (int32_t)uint_val; - limit_result = limit = INT32_MAX; - } - break; - - case 1: // uint32 - limit = limit_result = UINT32_MAX; - if (negative) - result = limit = 0; - else - result = (uint32_t)uint_val; - break; - - case 2: // int32 - if (negative) { - result = (int64_t)-uint_val; - limit_result = limit = (uint64_t)INT64_MIN; - } else { - result = (int64_t)uint_val; - limit_result = limit = INT64_MAX; - } - break; - - case 3: // uint64 - limit = limit_result = UINT64_MAX; - if (negative) - result = limit = 0; - else - result = (uint64_t)uint_val; - break; - - default: - __builtin_unreachable(); - } - - if (uint_val > limit) { - result = limit_result; - softfloat_raiseFlags(softfloat_flag_invalid); - } - - SET_FS_DIRTY(); - SET_RD(insn, regs, result); -} - -DECLARE_EMULATION_FUNC(emulate_fcmp) -{ - int rm = GET_RM(insn); - if (rm >= 3) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - uintptr_t result; - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - if (rm != 1) - result = f32_eq(rs1, rs2); - if (rm == 1 || (rm == 0 && !result)) - result = f32_lt(rs1, rs2); - goto success; - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - if (rm != 1) - result = f64_eq(rs1, rs2); - if (rm == 1 || (rm == 0 && !result)) - result = f64_lt(rs1, rs2); - goto success; - } - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); -success: - SET_RD(insn, regs, result); -} - -DECLARE_EMULATION_FUNC(emulate_fmv_if) -{ - uintptr_t result; - if ((insn & MASK_FMV_X_S) == MATCH_FMV_X_S) - result = GET_F32_RS1(insn, regs); -#ifdef __riscv64 - else if ((insn & MASK_FMV_X_D) == MATCH_FMV_X_D) - result = GET_F64_RS1(insn, regs); -#endif - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - SET_RD(insn, regs, result); -} - -DECLARE_EMULATION_FUNC(emulate_fmv_fi) -{ - uintptr_t rs1 = GET_RS1(insn, regs); - - if ((insn & MASK_FMV_S_X) == MATCH_FMV_S_X) - SET_F32_RD(insn, regs, rs1); - else if ((insn & MASK_FMV_D_X) == MATCH_FMV_D_X) - SET_F64_RD(insn, regs, rs1); - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); -} - -DECLARE_EMULATION_FUNC(emulate_fmadd) -{ - // if FPU is disabled, punt back to the OS - if (unlikely((mstatus & MSTATUS_FS) == 0)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - int op = (insn >> 2) & 3; - SETUP_STATIC_ROUNDING(insn); - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - uint32_t rs3 = GET_F32_RS3(insn, regs); - SET_F32_RD(insn, regs, softfloat_mulAddF32(op, rs1, rs2, rs3)); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - uint64_t rs3 = GET_F64_RS3(insn, regs); - SET_F64_RD(insn, regs, softfloat_mulAddF64(op, rs1, rs2, rs3)); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} diff --git a/pk/fp_emulation.h b/pk/fp_emulation.h deleted file mode 100644 index d2357b7..0000000 --- a/pk/fp_emulation.h +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef _RISCV_FP_EMULATION_H -#define _RISCV_FP_EMULATION_H - -#include "emulation.h" - -#define GET_PRECISION(insn) (((insn) >> 25) & 3) -#define GET_RM(insn) (((insn) >> 12) & 7) -#define PRECISION_S 0 -#define PRECISION_D 1 - -#ifdef __riscv_hard_float -# define GET_F32_REG(insn, pos, regs) ({ \ - register int32_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \ - uintptr_t tmp; \ - asm ("1: auipc %0, %%pcrel_hi(get_f32_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \ - value; }) -# define SET_F32_REG(insn, pos, regs, val) ({ \ - register uint32_t value asm("a0") = (val); \ - uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \ - uintptr_t tmp; \ - asm volatile ("1: auipc %0, %%pcrel_hi(put_f32_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); }) -# define init_fp_reg(i) SET_F32_REG((i) << 3, 3, 0, 0) -# define GET_F64_REG(insn, pos, regs) ({ \ - register uintptr_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \ - uintptr_t tmp; \ - asm ("1: auipc %0, %%pcrel_hi(get_f64_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \ - sizeof(uintptr_t) == 4 ? *(int64_t*)value : (int64_t)value; }) -# define SET_F64_REG(insn, pos, regs, val) ({ \ - uint64_t __val = (val); \ - register uintptr_t value asm("a0") = sizeof(uintptr_t) == 4 ? (uintptr_t)&__val : (uintptr_t)__val; \ - uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \ - uintptr_t tmp; \ - asm volatile ("1: auipc %0, %%pcrel_hi(put_f64_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); }) -# define GET_FCSR() read_csr(fcsr) -# define SET_FCSR(value) write_csr(fcsr, (value)) -# define GET_FRM() read_csr(frm) -# define SET_FRM(value) write_csr(frm, (value)) -# define GET_FFLAGS() read_csr(fflags) -# define SET_FFLAGS(value) write_csr(fflags, (value)) - -# define SETUP_STATIC_ROUNDING(insn) ({ \ - register long tp asm("tp") = read_csr(frm); \ - if (likely(((insn) & MASK_FUNCT3) == MASK_FUNCT3)) ; \ - else if (GET_RM(insn) > 4) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); \ - else tp = GET_RM(insn); \ - asm volatile ("":"+r"(tp)); }) -# define softfloat_raiseFlags(which) set_csr(fflags, which) -# define softfloat_roundingMode ({ register int tp asm("tp"); tp; }) -#else -# define GET_F64_REG(insn, pos, regs) (*(int64_t*)((void*)((regs) + 32) + (((insn) >> ((pos)-3)) & 0xf8))) -# define SET_F64_REG(insn, pos, regs, val) (GET_F64_REG(insn, pos, regs) = (val)) -# define GET_F32_REG(insn, pos, regs) (*(int32_t*)&GET_F64_REG(insn, pos, regs)) -# define SET_F32_REG(insn, pos, regs, val) (GET_F32_REG(insn, pos, regs) = (val)) -# define GET_FCSR() ({ register int tp asm("tp"); tp & 0xFF; }) -# define SET_FCSR(value) ({ asm volatile("add tp, x0, %0" :: "rI"((value) & 0xFF)); }) -# define GET_FRM() (GET_FCSR() >> 5) -# define SET_FRM(value) SET_FCSR(GET_FFLAGS() | ((value) << 5)) -# define GET_FFLAGS() (GET_FCSR() & 0x1F) -# define SET_FFLAGS(value) SET_FCSR((GET_FRM() << 5) | ((value) & 0x1F)) - -# define SETUP_STATIC_ROUNDING(insn) ({ \ - register int tp asm("tp"); tp &= 0xFF; \ - if (likely(((insn) & MASK_FUNCT3) == MASK_FUNCT3)) tp |= tp << 8; \ - else if (GET_RM(insn) > 4) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); \ - else tp |= GET_RM(insn) << 13; \ - asm volatile ("":"+r"(tp)); }) -# define softfloat_raiseFlags(which) ({ asm volatile ("or tp, tp, %0" :: "rI"(which)); }) -# define softfloat_roundingMode ({ register int tp asm("tp"); tp >> 13; }) -#endif - -#define GET_F32_RS1(insn, regs) (GET_F32_REG(insn, 15, regs)) -#define GET_F32_RS2(insn, regs) (GET_F32_REG(insn, 20, regs)) -#define GET_F32_RS3(insn, regs) (GET_F32_REG(insn, 27, regs)) -#define GET_F64_RS1(insn, regs) (GET_F64_REG(insn, 15, regs)) -#define GET_F64_RS2(insn, regs) (GET_F64_REG(insn, 20, regs)) -#define GET_F64_RS3(insn, regs) (GET_F64_REG(insn, 27, regs)) -#define SET_F32_RD(insn, regs, val) (SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY()) -#define SET_F64_RD(insn, regs, val) (SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY()) -#define SET_FS_DIRTY() set_csr(mstatus, MSTATUS_FS) - -#endif diff --git a/pk/frontend.h b/pk/frontend.h index 2cf5f81..5b9df31 100644 --- a/pk/frontend.h +++ b/pk/frontend.h @@ -6,18 +6,6 @@ #include <stdint.h> #include <sys/stat.h> -#ifdef __riscv64 -# define TOHOST_CMD(dev, cmd, payload) \ - (((uint64_t)(dev) << 56) | ((uint64_t)(cmd) << 48) | (uint64_t)(payload)) -#else -# define TOHOST_CMD(dev, cmd, payload) ({ \ - if ((dev) || (cmd)) __builtin_trap(); \ - (payload); }) -#endif -#define FROMHOST_DEV(fromhost_value) ((uint64_t)(fromhost_value) >> 56) -#define FROMHOST_CMD(fromhost_value) ((uint64_t)(fromhost_value) << 8 >> 56) -#define FROMHOST_DATA(fromhost_value) ((uint64_t)(fromhost_value) << 16 >> 16) - void shutdown(int) __attribute__((noreturn)); long frontend_syscall(long n, long a0, long a1, long a2, long a3, long a4, long a5, long a6); diff --git a/pk/handlers.c b/pk/handlers.c index bc82b5d..1961852 100644 --- a/pk/handlers.c +++ b/pk/handlers.c @@ -3,7 +3,7 @@ #include "pk.h" #include "config.h" #include "syscall.h" -#include "vm.h" +#include "mmap.h" static void handle_illegal_instruction(trapframe_t* tf) { @@ -31,7 +31,7 @@ static void handle_misaligned_fetch(trapframe_t* tf) panic("Misaligned instruction access!"); } -void handle_misaligned_store(trapframe_t* tf) +static void handle_misaligned_store(trapframe_t* tf) { dump_tf(tf); panic("Misaligned AMO!"); @@ -50,13 +50,13 @@ static void handle_fault_fetch(trapframe_t* tf) segfault(tf, tf->badvaddr, "fetch"); } -void handle_fault_load(trapframe_t* tf) +static void handle_fault_load(trapframe_t* tf) { if (handle_page_fault(tf->badvaddr, PROT_READ) != 0) segfault(tf, tf->badvaddr, "load"); } -void handle_fault_store(trapframe_t* tf) +static void handle_fault_store(trapframe_t* tf) { if (handle_page_fault(tf->badvaddr, PROT_WRITE) != 0) segfault(tf, tf->badvaddr, "store"); diff --git a/pk/init.c b/pk/init.c deleted file mode 100644 index 11e17bd..0000000 --- a/pk/init.c +++ /dev/null @@ -1,82 +0,0 @@ -// See LICENSE for license details. - -#include "pk.h" -#include "boot.h" -#include "file.h" -#include "vm.h" -#include "frontend.h" -#include "elf.h" -#include <stdint.h> -#include <stdlib.h> -#include <string.h> - -elf_info current; -int have_vm = 1; // unless -p flag is given - -int uarch_counters_enabled; -long uarch_counters[NUM_COUNTERS]; -char* uarch_counter_names[NUM_COUNTERS]; - -void init_tf(trapframe_t* tf, long pc, long sp) -{ - memset(tf, 0, sizeof(*tf)); - tf->status = (read_csr(sstatus) &~ SSTATUS_SPP &~ SSTATUS_SIE) | SSTATUS_SPIE; - tf->gpr[2] = sp; - tf->epc = pc; -} - -static void handle_option(const char* s) -{ - switch (s[1]) - { - case 's': // print cycle count upon termination - current.t0 = 1; - break; - - case 'c': // print uarch counters upon termination - // If your HW doesn't support uarch counters, then don't use this flag! - uarch_counters_enabled = 1; - break; - - default: - panic("unrecognized option: `%c'", s[1]); - break; - } -} - -#define MAX_ARGS 64 -typedef union { - uint64_t buf[MAX_ARGS]; - char* argv[MAX_ARGS]; -} arg_buf; - -static size_t parse_args(arg_buf* args) -{ - long r = frontend_syscall(SYS_getmainvars, (uintptr_t)args, sizeof(*args), 0, 0, 0, 0, 0); - kassert(r == 0); - uint64_t* pk_argv = &args->buf[1]; - // pk_argv[0] is the proxy kernel itself. skip it and any flags. - size_t pk_argc = args->buf[0], arg = 1; - for ( ; arg < pk_argc && *(char*)(uintptr_t)pk_argv[arg] == '-'; arg++) - handle_option((const char*)(uintptr_t)pk_argv[arg]); - - for (size_t i = 0; arg + i < pk_argc; i++) - args->argv[i] = (char*)(uintptr_t)pk_argv[arg + i]; - return pk_argc - arg; -} - -void boot_loader() -{ - arg_buf args; - size_t argc = parse_args(&args); - if (!argc) - panic("tell me what ELF to load!"); - - // load program named by argv[0] - long phdrs[128]; - current.phdr = (uintptr_t)phdrs; - current.phdr_size = sizeof(phdrs); - load_elf(args.argv[0], ¤t); - - run_loaded_program(argc, args.argv); -} diff --git a/pk/logo.c b/pk/logo.c deleted file mode 100644 index 2a214c5..0000000 --- a/pk/logo.c +++ /dev/null @@ -1,32 +0,0 @@ -#include <string.h> -#include "file.h" - -static const char logo[] = -" vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" -" vvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvv \n" -"rrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvv \n" -"rr vvvvvvvvvvvvvvvvvvvvvv \n" -"rr vvvvvvvvvvvvvvvvvvvvvvvv rr\n" -"rrrr vvvvvvvvvvvvvvvvvvvvvvvvvv rrrr\n" -"rrrrrr vvvvvvvvvvvvvvvvvvvvvv rrrrrr\n" -"rrrrrrrr vvvvvvvvvvvvvvvvvv rrrrrrrr\n" -"rrrrrrrrrr vvvvvvvvvvvvvv rrrrrrrrrr\n" -"rrrrrrrrrrrr vvvvvvvvvv rrrrrrrrrrrr\n" -"rrrrrrrrrrrrrr vvvvvv rrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrr vv rrrrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrrrrrr\n" -"\n" -" INSTRUCTION SETS WANT TO BE FREE\n"; - -void print_logo() -{ - file_write(stderr, logo, sizeof logo); -} diff --git a/pk/mcall.h b/pk/mcall.h deleted file mode 100644 index e612628..0000000 --- a/pk/mcall.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef _PK_MCALL_H -#define _PK_MCALL_H - -#define MCALL_HART_ID 0 -#define MCALL_CONSOLE_PUTCHAR 1 -#define MCALL_CONSOLE_GETCHAR 2 -#define MCALL_HTIF_SYSCALL 3 -#define MCALL_SEND_IPI 4 -#define MCALL_CLEAR_IPI 5 -#define MCALL_SHUTDOWN 6 -#define MCALL_SET_TIMER 7 -#define MCALL_REMOTE_SFENCE_VM 8 -#define MCALL_REMOTE_FENCE_I 9 - -#ifndef __ASSEMBLER__ - -extern uintptr_t do_mcall(uintptr_t which, ...); - -#endif - -#endif diff --git a/pk/mentry.S b/pk/mentry.S deleted file mode 100644 index 2ded375..0000000 --- a/pk/mentry.S +++ /dev/null @@ -1,244 +0,0 @@ -// See LICENSE for license details. - -#include "mtrap.h" - - .data - .align 6 -trap_table: - .word bad_trap - .word bad_trap - .word illegal_insn_trap - .word bad_trap - .word misaligned_load_trap - .word bad_trap - .word misaligned_store_trap - .word bad_trap - .word bad_trap - .word mcall_trap - .word bad_trap - .word bad_trap -#define HTIF_INTERRUPT_VECTOR 12 - .word htif_interrupt -#define SOFTWARE_INTERRUPT_VECTOR 13 - .word software_interrupt -#define TRAP_FROM_MACHINE_MODE_VECTOR 14 - .word __trap_from_machine_mode - - .option norvc - .section .text.init,"ax",@progbits - .globl reset_vector -reset_vector: - j do_reset - -nmi_vector: -.Lunhandleable_trap: - j bad_trap - -trap_vector: - csrrw sp, mscratch, sp - beqz sp, .Ltrap_from_machine_mode - - STORE a0, 10*REGBYTES(sp) - STORE a1, 11*REGBYTES(sp) - - csrr a1, mcause - bgez a1, .Lhandle_trap_in_machine_mode - - # This is an interrupt. Discard the mcause MSB and decode the rest. - sll a1, a1, 1 - - # Is it a machine timer interrupt? - li a0, IRQ_M_TIMER * 2 - bne a0, a1, 1f - # Yes. Post a supervisor timer interrupt. - li a0, MIP_MTIP - csrc mie, a0 - li a0, MIP_STIP - csrs mip, a0 - -.Leret: - # Go back whence we came. - LOAD a0, 10*REGBYTES(sp) - LOAD a1, 11*REGBYTES(sp) - csrrw sp, mscratch, sp - eret - -1: - # Is it an IPI? - li a0, IRQ_M_SOFT * 2 - bne a0, a1, 1f - li a1, SOFTWARE_INTERRUPT_VECTOR - j .Lhandle_trap_in_machine_mode - -1: - # By process of elimination, it must be an HTIF interrupt. - li a0, IRQ_HOST * 2 - bne a0, a1, .Lunhandleable_trap - li a1, HTIF_INTERRUPT_VECTOR - -.Lhandle_trap_in_machine_mode: - # Preserve the registers. Compute the address of the trap handler. - STORE ra, 1*REGBYTES(sp) - STORE gp, 3*REGBYTES(sp) - STORE tp, 4*REGBYTES(sp) - STORE t0, 5*REGBYTES(sp) -1:auipc t0, %pcrel_hi(trap_table) # t0 <- %hi(trap_table) - STORE t1, 6*REGBYTES(sp) - sll t1, a1, 2 # t1 <- mcause << 2 - STORE t2, 7*REGBYTES(sp) - add t1, t0, t1 # t1 <- %hi(trap_table)[mcause] - STORE s0, 8*REGBYTES(sp) - lw t1, %pcrel_lo(1b)(t1) # t1 <- trap_table[mcause] - STORE s1, 9*REGBYTES(sp) - mv a0, sp # a0 <- regs - STORE a2,12*REGBYTES(sp) - csrr a2, mepc # a2 <- mepc - STORE a3,13*REGBYTES(sp) - csrrw t0, mscratch, x0 # t0 <- user sp - STORE a4,14*REGBYTES(sp) - STORE a5,15*REGBYTES(sp) - STORE a6,16*REGBYTES(sp) - STORE a7,17*REGBYTES(sp) - STORE s2,18*REGBYTES(sp) - STORE s3,19*REGBYTES(sp) - STORE s4,20*REGBYTES(sp) - STORE s5,21*REGBYTES(sp) - STORE s6,22*REGBYTES(sp) - STORE s7,23*REGBYTES(sp) - STORE s8,24*REGBYTES(sp) - STORE s9,25*REGBYTES(sp) - STORE s10,26*REGBYTES(sp) - STORE s11,27*REGBYTES(sp) - STORE t3,28*REGBYTES(sp) - STORE t4,29*REGBYTES(sp) - STORE t5,30*REGBYTES(sp) - STORE t6,31*REGBYTES(sp) - STORE t0, 2*REGBYTES(sp) # sp - -#ifndef __riscv_hard_float - lw tp, (sp) # Move the emulated FCSR from x0's save slot into tp. -#endif - STORE x0, (sp) # Zero x0's save slot. - - # Invoke the handler. - jalr t1 - -#ifndef __riscv_hard_float - sw tp, (sp) # Move the emulated FCSR from tp into x0's save slot. -#endif - -restore_mscratch: - # Restore mscratch, so future traps will know they didn't come from M-mode. - csrw mscratch, sp - -restore_regs: - # Restore all of the registers. - LOAD ra, 1*REGBYTES(sp) - LOAD gp, 3*REGBYTES(sp) - LOAD tp, 4*REGBYTES(sp) - LOAD t0, 5*REGBYTES(sp) - LOAD t1, 6*REGBYTES(sp) - LOAD t2, 7*REGBYTES(sp) - LOAD s0, 8*REGBYTES(sp) - LOAD s1, 9*REGBYTES(sp) - LOAD a0,10*REGBYTES(sp) - LOAD a1,11*REGBYTES(sp) - LOAD a2,12*REGBYTES(sp) - LOAD a3,13*REGBYTES(sp) - LOAD a4,14*REGBYTES(sp) - LOAD a5,15*REGBYTES(sp) - LOAD a6,16*REGBYTES(sp) - LOAD a7,17*REGBYTES(sp) - LOAD s2,18*REGBYTES(sp) - LOAD s3,19*REGBYTES(sp) - LOAD s4,20*REGBYTES(sp) - LOAD s5,21*REGBYTES(sp) - LOAD s6,22*REGBYTES(sp) - LOAD s7,23*REGBYTES(sp) - LOAD s8,24*REGBYTES(sp) - LOAD s9,25*REGBYTES(sp) - LOAD s10,26*REGBYTES(sp) - LOAD s11,27*REGBYTES(sp) - LOAD t3,28*REGBYTES(sp) - LOAD t4,29*REGBYTES(sp) - LOAD t5,30*REGBYTES(sp) - LOAD t6,31*REGBYTES(sp) - LOAD sp, 2*REGBYTES(sp) - eret - -.Ltrap_from_machine_mode: - csrr sp, mscratch - addi sp, sp, -INTEGER_CONTEXT_SIZE - STORE a0,10*REGBYTES(sp) - STORE a1,11*REGBYTES(sp) - li a1, TRAP_FROM_MACHINE_MODE_VECTOR - j .Lhandle_trap_in_machine_mode - - .globl __redirect_trap -__redirect_trap: - # reset sp to top of M-mode stack - li t0, MACHINE_STACK_SIZE - add sp, sp, t0 - neg t0, t0 - and sp, sp, t0 - addi sp, sp, -MENTRY_FRAME_SIZE - j restore_mscratch - -__trap_from_machine_mode: - jal trap_from_machine_mode - j restore_regs - -do_reset: - li x1, 0 - li x2, 0 - li x3, 0 - li x4, 0 - li x5, 0 - li x6, 0 - li x7, 0 - li x8, 0 - li x9, 0 - li x10, 0 - li x11, 0 - li x12, 0 - li x13, 0 - li x14, 0 - li x15, 0 - li x16, 0 - li x17, 0 - li x18, 0 - li x19, 0 - li x20, 0 - li x21, 0 - li x22, 0 - li x23, 0 - li x24, 0 - li x25, 0 - li x26, 0 - li x27, 0 - li x28, 0 - li x29, 0 - li x30, 0 - li x31, 0 - csrw mscratch, x0 - - # sp <- end of first full page after the end of the binary - la sp, _end + 2*RISCV_PGSIZE - 1 - li t0, -RISCV_PGSIZE - and sp, sp, t0 - addi sp, sp, -MENTRY_FRAME_SIZE - - csrr a0, mhartid - slli a1, a0, RISCV_PGSHIFT - add sp, sp, a1 - - beqz a0, init_first_hart - -.LmultiHart: -#if MAX_HARTS > 1 - # make sure our hart id is within a valid range - li a1, MAX_HARTS - bltu a0, a1, init_other_hart - wfi -#endif - j .LmultiHart diff --git a/pk/minit.c b/pk/minit.c deleted file mode 100644 index ad9598b..0000000 --- a/pk/minit.c +++ /dev/null @@ -1,108 +0,0 @@ -#include "vm.h" -#include "mtrap.h" -#include "fp_emulation.h" -#include "boot.h" - -uintptr_t mem_size; -uint32_t num_harts; - -static void mstatus_init() -{ - uintptr_t ms = 0; - ms = INSERT_FIELD(ms, MSTATUS_VM, VM_CHOICE); - ms = INSERT_FIELD(ms, MSTATUS_FS, 1); - write_csr(mstatus, ms); - - ms = read_csr(mstatus); - assert(EXTRACT_FIELD(ms, MSTATUS_VM) == VM_CHOICE); - - write_csr(mtimecmp, 0); - clear_csr(mip, MIP_MSIP); - write_csr(mie, -1); - write_csr(mucounteren, -1); - write_csr(mscounteren, -1); -} - -static void delegate_traps() -{ - uintptr_t interrupts = MIP_SSIP | MIP_STIP; - uintptr_t exceptions = - (1U << CAUSE_MISALIGNED_FETCH) | - (1U << CAUSE_FAULT_FETCH) | - (1U << CAUSE_BREAKPOINT) | - (1U << CAUSE_FAULT_LOAD) | - (1U << CAUSE_FAULT_STORE) | - (1U << CAUSE_BREAKPOINT) | - (1U << CAUSE_USER_ECALL); - - write_csr(mideleg, interrupts); - write_csr(medeleg, exceptions); - assert(read_csr(mideleg) == interrupts); - assert(read_csr(medeleg) == exceptions); -} - -static void fp_init() -{ - assert(read_csr(mstatus) & MSTATUS_FS); - -#ifdef __riscv_hard_float - if (!supports_extension('D')) - die("FPU not found; recompile pk with -msoft-float"); - for (int i = 0; i < 32; i++) - init_fp_reg(i); - write_csr(fcsr, 0); -#else - if (supports_extension('D')) - die("FPU unexpectedly found; recompile with -mhard-float"); -#endif -} - -void hls_init(uint32_t id, csr_t* csrs) -{ - hls_t* hls = OTHER_HLS(id); - memset(hls, 0, sizeof(*hls)); - hls->csrs = csrs; -} - -static void hart_init() -{ - mstatus_init(); - fp_init(); - delegate_traps(); -} - -void init_first_hart() -{ - file_init(); - hart_init(); - - memset(HLS(), 0, sizeof(*HLS())); - parse_config_string(); - vm_init(); - boot_loader(); -} - -void init_other_hart() -{ - hart_init(); - - // wait until virtual memory is enabled - while (*(pte_t* volatile*)&root_page_table == NULL) - ; - mb(); - write_csr(sptbr, (uintptr_t)root_page_table >> RISCV_PGSHIFT); - - // make sure hart 0 has discovered us - assert(HLS()->csrs != NULL); - - boot_other_hart(); -} - -void prepare_supervisor_mode() -{ - uintptr_t mstatus = read_csr(mstatus); - mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S); - mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); - write_csr(mstatus, mstatus); - write_csr(mscratch, MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE); -} @@ -1,8 +1,8 @@ -#include "vm.h" -#include "file.h" +#include "mmap.h" #include "atomic.h" #include "pk.h" #include "boot.h" +#include "bits.h" #include "mtrap.h" #include <stdint.h> #include <errno.h> @@ -17,14 +17,15 @@ typedef struct { } vmr_t; #define MAX_VMR (RISCV_PGSIZE / sizeof(vmr_t)) -spinlock_t vm_lock = SPINLOCK_INIT; +static spinlock_t vm_lock = SPINLOCK_INIT; static vmr_t* vmrs; -pte_t* root_page_table; static uintptr_t first_free_page; static size_t next_free_page; static size_t free_pages; +int have_vm = 1; // unless -p flag is given + static uintptr_t __page_alloc() { kassert(next_free_page != free_pages); @@ -73,41 +74,6 @@ static size_t pte_ppn(pte_t pte) return pte >> PTE_PPN_SHIFT; } -static pte_t ptd_create(uintptr_t ppn) -{ - return (ppn << PTE_PPN_SHIFT) | PTE_V | PTE_TYPE_TABLE; -} - -static inline pte_t pte_create(uintptr_t ppn, int prot, int user) -{ - pte_t pte = (ppn << PTE_PPN_SHIFT) | PTE_V; - prot &= PROT_READ|PROT_WRITE|PROT_EXEC; - if (user) { - switch (prot) { - case PROT_NONE: pte |= PTE_TYPE_SR; break; - case PROT_READ: pte |= PTE_TYPE_UR_SR; break; - case PROT_WRITE: pte |= PTE_TYPE_URW_SRW; break; - case PROT_EXEC: pte |= PTE_TYPE_URX_SRX; break; - case PROT_READ|PROT_WRITE: pte |= PTE_TYPE_URW_SRW; break; - case PROT_READ|PROT_EXEC: pte |= PTE_TYPE_URX_SRX; break; - case PROT_WRITE|PROT_EXEC: pte |= PTE_TYPE_URWX_SRWX; break; - case PROT_READ|PROT_WRITE|PROT_EXEC: pte |= PTE_TYPE_URWX_SRWX; break; - } - } else { - switch (prot) { - case PROT_NONE: kassert(0); break; - case PROT_READ: pte |= PTE_TYPE_SR; break; - case PROT_WRITE: pte |= PTE_TYPE_SRW; break; - case PROT_EXEC: pte |= PTE_TYPE_SRX; break; - case PROT_READ|PROT_WRITE: pte |= PTE_TYPE_SRW; break; - case PROT_READ|PROT_EXEC: pte |= PTE_TYPE_SRX; break; - case PROT_WRITE|PROT_EXEC: pte |= PTE_TYPE_SRWX; break; - case PROT_READ|PROT_WRITE|PROT_EXEC: pte |= PTE_TYPE_SRWX; break; - } - } - return pte; -} - static uintptr_t ppn(uintptr_t addr) { return addr >> RISCV_PGSHIFT; @@ -119,27 +85,19 @@ static size_t pt_idx(uintptr_t addr, int level) return idx & ((1 << RISCV_PGLEVEL_BITS) - 1); } -static void __maybe_create_root_page_table() +static pte_t* __maybe_create_root_page_table() { - if (root_page_table) - return; - root_page_table = (void*)__page_alloc(); - if (have_vm) - write_csr(sptbr, (uintptr_t)root_page_table >> RISCV_PGSHIFT); + if (!root_page_table) + root_page_table = (void*)__page_alloc(); + return root_page_table; } static pte_t* __walk_internal(uintptr_t addr, int create) { - const size_t pte_per_page = RISCV_PGSIZE/sizeof(void*); - __maybe_create_root_page_table(); - pte_t* t = root_page_table; - unsigned levels = (VA_BITS - RISCV_PGSHIFT) / RISCV_PGLEVEL_BITS; - - for (unsigned i = levels-1; i > 0; i--) - { + pte_t* t = __maybe_create_root_page_table(); + for (int i = (VA_BITS - RISCV_PGSHIFT) / RISCV_PGLEVEL_BITS - 1; i > 0; i--) { size_t idx = pt_idx(addr, i); - if (!(t[idx] & PTE_V)) - { + if (!(t[idx] & PTE_V)) { if (!create) return 0; uintptr_t page = __page_alloc(); @@ -185,11 +143,40 @@ static uintptr_t __vm_alloc(size_t npage) return 0; } +static inline pte_t prot_to_type(int prot, int user) +{ + prot &= PROT_READ|PROT_WRITE|PROT_EXEC; + if (user) { + switch (prot) { + case PROT_NONE: return PTE_TYPE_SR; + case PROT_READ: return PTE_TYPE_UR_SR; + case PROT_WRITE: return PTE_TYPE_URW_SRW; + case PROT_EXEC: return PTE_TYPE_URX_SRX; + case PROT_READ|PROT_WRITE: return PTE_TYPE_URW_SRW; + case PROT_READ|PROT_EXEC: return PTE_TYPE_URX_SRX; + case PROT_WRITE|PROT_EXEC: return PTE_TYPE_URWX_SRWX; + case PROT_READ|PROT_WRITE|PROT_EXEC: return PTE_TYPE_URWX_SRWX; + } + } else { + switch (prot) { + case PROT_NONE: + case PROT_READ: return PTE_TYPE_SR; + case PROT_WRITE: return PTE_TYPE_SRW; + case PROT_EXEC: return PTE_TYPE_SRX; + case PROT_READ|PROT_WRITE: return PTE_TYPE_SRW; + case PROT_READ|PROT_EXEC: return PTE_TYPE_SRX; + case PROT_WRITE|PROT_EXEC: return PTE_TYPE_SRWX; + case PROT_READ|PROT_WRITE|PROT_EXEC: return PTE_TYPE_SRWX; + } + } + __builtin_unreachable(); +} + int __valid_user_range(uintptr_t vaddr, size_t len) { if (vaddr + len < vaddr) return 0; - return vaddr >= current.first_free_paddr && vaddr + len <= current.mmap_max; + return vaddr >= first_free_paddr && vaddr + len <= current.mmap_max; } static int __handle_page_fault(uintptr_t vaddr, int prot) @@ -206,7 +193,7 @@ static int __handle_page_fault(uintptr_t vaddr, int prot) uintptr_t ppn = vpn; vmr_t* v = (vmr_t*)*pte; - *pte = pte_create(ppn, PROT_READ|PROT_WRITE, 0); + *pte = pte_create(ppn, prot_to_type(PROT_READ|PROT_WRITE, 0)); flush_tlb(); if (v->file) { @@ -219,10 +206,10 @@ static int __handle_page_fault(uintptr_t vaddr, int prot) else memset((void*)vaddr, 0, RISCV_PGSIZE); __vmr_decref(v, 1); - *pte = pte_create(ppn, v->prot, 1); + *pte = pte_create(ppn, prot_to_type(v->prot, 1)); } - pte_t perms = pte_create(0, prot, 1); + pte_t perms = pte_create(0, prot_to_type(prot, 1)); if ((*pte & perms) != perms) return -1; @@ -384,7 +371,7 @@ uintptr_t do_mprotect(uintptr_t addr, size_t length, int prot) res = -EACCES; break; } - *pte = pte_create(pte_ppn(*pte), prot, 1); + *pte = pte_create(pte_ppn(*pte), prot_to_type(prot, 1)); } } spinlock_unlock(&vm_lock); @@ -395,11 +382,12 @@ uintptr_t do_mprotect(uintptr_t addr, size_t length, int prot) void __map_kernel_range(uintptr_t vaddr, uintptr_t paddr, size_t len, int prot) { uintptr_t n = ROUNDUP(len, RISCV_PGSIZE) / RISCV_PGSIZE; + uintptr_t offset = paddr - vaddr; for (uintptr_t a = vaddr, i = 0; i < n; i++, a += RISCV_PGSIZE) { pte_t* pte = __walk_create(a); kassert(pte); - *pte = pte_create((a - vaddr + paddr) >> RISCV_PGSHIFT, prot, 0); + *pte = pte_create((a + offset) >> RISCV_PGSHIFT, prot_to_type(prot, 0)); } } @@ -415,85 +403,23 @@ void populate_mapping(const void* start, size_t size, int prot) } } -static uintptr_t sbi_top_paddr() -{ - extern char _end; - return ROUNDUP((uintptr_t)&_end, RISCV_PGSIZE); -} - -#define first_free_paddr() (sbi_top_paddr() + num_harts * RISCV_PGSIZE) - -void vm_init() +uintptr_t pk_vm_init() { - mem_size = mem_size / SUPERPAGE_SIZE * SUPERPAGE_SIZE; - current.first_free_paddr = first_free_paddr(); - size_t mem_pages = mem_size >> RISCV_PGSHIFT; free_pages = MAX(8, mem_pages >> (RISCV_PGLEVEL_BITS-1)); first_free_page = mem_size - free_pages * RISCV_PGSIZE; - current.mmap_max = current.brk_max = first_free_page; -} -void supervisor_vm_init() -{ - uintptr_t highest_va = -current.first_free_paddr; - mem_size = MIN(mem_size, highest_va - current.first_user_vaddr) & -SUPERPAGE_SIZE; - - pte_t* sbi_pt = (pte_t*)(current.first_vaddr_after_user + current.bias); - memset(sbi_pt, 0, RISCV_PGSIZE); - pte_t* middle_pt = (void*)sbi_pt + RISCV_PGSIZE; -#ifndef __riscv64 - size_t num_middle_pts = 1; - pte_t* root_pt = middle_pt; - memset(root_pt, 0, RISCV_PGSIZE); -#else - size_t num_middle_pts = (-current.first_user_vaddr - 1) / MEGAPAGE_SIZE + 1; - pte_t* root_pt = (void*)middle_pt + num_middle_pts * RISCV_PGSIZE; - memset(middle_pt, 0, (num_middle_pts + 1) * RISCV_PGSIZE); - for (size_t i = 0; i < num_middle_pts; i++) - root_pt[(1<<RISCV_PGLEVEL_BITS)-num_middle_pts+i] = ptd_create(((uintptr_t)middle_pt >> RISCV_PGSHIFT) + i); -#endif - - for (uintptr_t vaddr = current.first_user_vaddr, paddr = vaddr + current.bias, end = current.first_vaddr_after_user; - paddr < mem_size; vaddr += SUPERPAGE_SIZE, paddr += SUPERPAGE_SIZE) { - int l2_shift = RISCV_PGLEVEL_BITS + RISCV_PGSHIFT; - size_t l2_idx = (current.first_user_vaddr >> l2_shift) & ((1 << RISCV_PGLEVEL_BITS)-1); - l2_idx += ((vaddr - current.first_user_vaddr) >> l2_shift); - middle_pt[l2_idx] = pte_create(paddr >> RISCV_PGSHIFT, PROT_READ|PROT_WRITE|PROT_EXEC, 0); - } - current.first_vaddr_after_user += (void*)root_pt + RISCV_PGSIZE - (void*)sbi_pt; - - // map SBI at top of vaddr space - uintptr_t num_sbi_pages = sbi_top_paddr() / RISCV_PGSIZE; - for (uintptr_t i = 0; i < num_sbi_pages; i++) { - uintptr_t idx = (1 << RISCV_PGLEVEL_BITS) - num_sbi_pages + i; - sbi_pt[idx] = pte_create(i, PROT_READ|PROT_EXEC, 0); - } - pte_t* sbi_pte = middle_pt + ((num_middle_pts << RISCV_PGLEVEL_BITS)-1); - kassert(!*sbi_pte); - *sbi_pte = ptd_create((uintptr_t)sbi_pt >> RISCV_PGSHIFT); - - // disable our allocator - kassert(next_free_page == 0); - free_pages = 0; - - mb(); - root_page_table = root_pt; - write_csr(sptbr, (uintptr_t)root_pt >> RISCV_PGSHIFT); -} + __map_kernel_range(0, 0, first_free_paddr, PROT_READ|PROT_WRITE|PROT_EXEC); + __map_kernel_range(first_free_page, first_free_page, free_pages * RISCV_PGSIZE, PROT_READ|PROT_WRITE); -uintptr_t pk_vm_init() -{ // keep user addresses positive - current.mmap_max = MIN(current.mmap_max, (uintptr_t)INTPTR_MAX + 1); - - __map_kernel_range(0, 0, current.first_free_paddr, PROT_READ|PROT_WRITE|PROT_EXEC); - __map_kernel_range(first_free_page, first_free_page, free_pages * RISCV_PGSIZE, PROT_READ|PROT_WRITE); + current.mmap_max = MIN(first_free_page, (uintptr_t)INTPTR_MAX + 1); + current.brk_max = current.mmap_max; size_t stack_size = RISCV_PGSIZE * CLAMP(mem_size/(RISCV_PGSIZE*32), 1, 256); - current.stack_bottom = __do_mmap(current.mmap_max - stack_size, stack_size, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, 0, 0); - current.stack_top = current.stack_bottom + stack_size; - kassert(current.stack_bottom != (uintptr_t)-1); + size_t stack_bottom = __do_mmap(current.mmap_max - stack_size, stack_size, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, 0, 0); + kassert(stack_bottom != (uintptr_t)-1); + current.stack_top = stack_bottom + stack_size; uintptr_t kernel_stack_top = __page_alloc() + RISCV_PGSIZE; return kernel_stack_top; @@ -1,21 +1,11 @@ -#ifndef _VM_H -#define _VM_H +#ifndef _MMAP_H +#define _MMAP_H +#include "vm.h" #include "syscall.h" +#include "encoding.h" #include "file.h" -#include <string.h> -#include <stdint.h> -#include <sys/types.h> - -#define SUPERPAGE_SIZE ((uintptr_t)(RISCV_PGSIZE << RISCV_PGLEVEL_BITS)) -#ifdef __riscv64 -# define VM_CHOICE VM_SV39 -# define VA_BITS 39 -# define MEGAPAGE_SIZE (SUPERPAGE_SIZE << RISCV_PGLEVEL_BITS) -#else -# define VM_CHOICE VM_SV32 -# define VA_BITS 32 -#endif +#include <stddef.h> #define PROT_NONE 0 #define PROT_READ 1 @@ -28,8 +18,7 @@ #define MAP_POPULATE 0x8000 #define MREMAP_FIXED 0x2 -void vm_init(); -void supervisor_vm_init(); +extern int have_vm; uintptr_t pk_vm_init(); int handle_page_fault(uintptr_t vaddr, int prot); void populate_mapping(const void* start, size_t size, int prot); @@ -42,12 +31,4 @@ uintptr_t do_mremap(uintptr_t addr, size_t old_size, size_t new_size, int flags) uintptr_t do_mprotect(uintptr_t addr, size_t length, int prot); uintptr_t do_brk(uintptr_t addr); -typedef uintptr_t pte_t; -extern pte_t* root_page_table; - -static inline void flush_tlb() -{ - asm volatile("sfence.vm"); -} - #endif diff --git a/pk/mtrap.c b/pk/mtrap.c deleted file mode 100644 index b64360e..0000000 --- a/pk/mtrap.c +++ /dev/null @@ -1,281 +0,0 @@ -#include "mtrap.h" -#include "frontend.h" -#include "mcall.h" -#include "vm.h" -#include <errno.h> -#include <stdarg.h> -#include <stdio.h> - -void __attribute__((noreturn)) bad_trap() -{ - die("machine mode: unhandlable trap %d @ %p", read_csr(mcause), read_csr(mepc)); -} - -static uintptr_t mcall_hart_id() -{ - return read_const_csr(mhartid); -} - -static void request_htif_keyboard_interrupt() -{ - uintptr_t old_tohost = swap_csr(mtohost, TOHOST_CMD(1, 0, 0)); - assert(old_tohost == 0); -} - -void htif_interrupt() -{ - uintptr_t fromhost = swap_csr(mfromhost, 0); - // we should only be interrupted by keypresses - if (!(FROMHOST_DEV(fromhost) == 1 && FROMHOST_CMD(fromhost) == 0)) - die("unexpected htif interrupt"); - HLS()->console_ibuf = 1 + (uint8_t)FROMHOST_DATA(fromhost); - set_csr(mip, MIP_SSIP); -} - -static void do_tohost_fromhost(uintptr_t dev, uintptr_t cmd, uintptr_t data) -{ - while (swap_csr(mtohost, TOHOST_CMD(dev, cmd, data)) != 0) - if (read_csr(mfromhost)) - htif_interrupt(); - - while (1) { - uintptr_t fromhost = read_csr(mfromhost); - if (fromhost) { - if (FROMHOST_DEV(fromhost) == dev && FROMHOST_CMD(fromhost) == cmd) { - write_csr(mfromhost, 0); - break; - } - htif_interrupt(); - } - } -} - -static uintptr_t mcall_console_putchar(uint8_t ch) -{ - do_tohost_fromhost(1, 1, ch); - return 0; -} - -static uintptr_t mcall_htif_syscall(uintptr_t magic_mem) -{ - do_tohost_fromhost(0, 0, magic_mem); - return 0; -} - -void poweroff() -{ - while (1) - write_csr(mtohost, 1); -} - -void printm(const char* s, ...) -{ - char buf[256]; - va_list vl; - - va_start(vl, s); - vsnprintf(buf, sizeof buf, s, vl); - va_end(vl); - - for (const char* p = buf; *p; p++) - mcall_console_putchar(*p); -} - -static void send_ipi(uintptr_t recipient, int event) -{ - if ((atomic_or(&OTHER_HLS(recipient)->mipi_pending, event) & event) == 0) { - mb(); - OTHER_HLS(recipient)->csrs[CSR_MIPI] = 1; - } -} - -static uintptr_t mcall_send_ipi(uintptr_t recipient) -{ - if (recipient >= num_harts) - return -1; - - send_ipi(recipient, IPI_SOFT); - return 0; -} - -static void reset_ssip() -{ - clear_csr(mip, MIP_SSIP); - mb(); - - if (HLS()->sipi_pending || HLS()->console_ibuf > 0) - set_csr(mip, MIP_SSIP); -} - -static uintptr_t mcall_console_getchar() -{ - int ch = atomic_swap(&HLS()->console_ibuf, -1); - if (ch >= 0) - request_htif_keyboard_interrupt(); - reset_ssip(); - return ch - 1; -} - -static uintptr_t mcall_clear_ipi() -{ - int ipi = atomic_swap(&HLS()->sipi_pending, 0); - reset_ssip(); - return ipi; -} - -static uintptr_t mcall_shutdown() -{ - poweroff(); -} - -static uintptr_t mcall_set_timer(unsigned long long when) -{ - // bbl/pk don't use the timer, so there's no need to virtualize it - write_csr(mtimecmp, when); -#ifndef __riscv64 - write_csr(mtimecmph, when >> 32); -#endif - clear_csr(mip, MIP_STIP); - set_csr(mie, MIP_MTIP); - return 0; -} - -void software_interrupt() -{ - clear_csr(mip, MIP_MSIP); - mb(); - int ipi_pending = atomic_swap(&HLS()->mipi_pending, 0); - - if (ipi_pending & IPI_SOFT) { - HLS()->sipi_pending = 1; - set_csr(mip, MIP_SSIP); - } - - if (ipi_pending & IPI_FENCE_I) - asm volatile ("fence.i"); - - if (ipi_pending & IPI_SFENCE_VM) - asm volatile ("sfence.vm"); -} - -static void send_ipi_many(uintptr_t* pmask, int event) -{ - _Static_assert(MAX_HARTS <= 8 * sizeof(*pmask), "# harts > uintptr_t bits"); - uintptr_t mask = -1; - if (pmask) - mask = *pmask; - - // send IPIs to everyone - for (ssize_t i = num_harts-1; i >= 0; i--) - if ((mask >> i) & 1) - send_ipi(i, event); - - // wait until all events have been handled. - // prevent deadlock while spinning by handling any IPIs from other harts. - for (ssize_t i = num_harts-1; i >= 0; i--) - if ((mask >> i) & 1) - while (OTHER_HLS(i)->mipi_pending & event) - software_interrupt(); -} - -static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask, uintptr_t asid) -{ - // ignore the ASID and do a global flush. - // this allows us to avoid queueing a message. - send_ipi_many(hart_mask, IPI_SFENCE_VM); - return 0; -} - -static uintptr_t mcall_remote_fence_i(uintptr_t* hart_mask) -{ - send_ipi_many(hart_mask, IPI_FENCE_I); - return 0; -} - -void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval; - switch (n) - { - case MCALL_HART_ID: - retval = mcall_hart_id(); - break; - case MCALL_CONSOLE_PUTCHAR: - retval = mcall_console_putchar(arg0); - break; - case MCALL_CONSOLE_GETCHAR: - retval = mcall_console_getchar(); - break; - case MCALL_HTIF_SYSCALL: - retval = mcall_htif_syscall(arg0); - break; - case MCALL_SEND_IPI: - retval = mcall_send_ipi(arg0); - break; - case MCALL_CLEAR_IPI: - retval = mcall_clear_ipi(); - break; - case MCALL_SHUTDOWN: - retval = mcall_shutdown(); - break; - case MCALL_SET_TIMER: - retval = mcall_set_timer(arg0); - break; - case MCALL_REMOTE_SFENCE_VM: - retval = mcall_remote_sfence_vm((uintptr_t*)arg0, arg1); - break; - case MCALL_REMOTE_FENCE_I: - retval = mcall_remote_fence_i((uintptr_t*)arg0); - break; - default: - retval = -ENOSYS; - break; - } - regs[10] = retval; - write_csr(mepc, mepc + 4); -} - -void redirect_trap(uintptr_t epc, uintptr_t mstatus) -{ - write_csr(sepc, epc); - write_csr(scause, read_csr(mcause)); - write_csr(mepc, read_csr(stvec)); - - uintptr_t prev_priv = EXTRACT_FIELD(mstatus, MSTATUS_MPP); - uintptr_t prev_ie = EXTRACT_FIELD(mstatus, MSTATUS_MPIE); - mstatus = INSERT_FIELD(mstatus, MSTATUS_SPP, prev_priv); - mstatus = INSERT_FIELD(mstatus, MSTATUS_SPIE, prev_ie); - mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S); - mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); - write_csr(mstatus, mstatus); - - extern void __redirect_trap(); - return __redirect_trap(); -} - -static void machine_page_fault(uintptr_t* regs, uintptr_t mepc) -{ - // MPRV=1 iff this trap occurred while emulating an instruction on behalf - // of a lower privilege level. In that case, a2=epc and a3=mstatus. - if (read_csr(mstatus) & MSTATUS_MPRV) { - write_csr(sbadaddr, read_csr(mbadaddr)); - return redirect_trap(regs[12], regs[13]); - } - bad_trap(); -} - -void trap_from_machine_mode(uintptr_t* regs, uintptr_t dummy, uintptr_t mepc) -{ - uintptr_t mcause = read_csr(mcause); - - switch (mcause) - { - case CAUSE_FAULT_LOAD: - case CAUSE_FAULT_STORE: - return machine_page_fault(regs, mepc); - case CAUSE_MACHINE_ECALL: - return mcall_trap(regs, mcause, mepc); - default: - bad_trap(); - } -} diff --git a/pk/mtrap.h b/pk/mtrap.h deleted file mode 100644 index b6ea1df..0000000 --- a/pk/mtrap.h +++ /dev/null @@ -1,86 +0,0 @@ -#ifndef _PK_MTRAP_H -#define _PK_MTRAP_H - -#include "bits.h" -#include "encoding.h" - -#ifdef __riscv_atomic -# define MAX_HARTS 8 // arbitrary -#else -# define MAX_HARTS 1 -#endif - -#ifndef __ASSEMBLER__ - -#include "sbi.h" -#include <stdint.h> - -#define read_const_csr(reg) ({ unsigned long __tmp; \ - asm ("csrr %0, " #reg : "=r"(__tmp)); \ - __tmp; }) - -static inline int supports_extension(char ext) -{ - return read_const_csr(misa) & (1 << (ext - 'A')); -} - -static inline int xlen() -{ - return read_const_csr(misa) < 0 ? 64 : 32; -} - -extern uintptr_t mem_size; -extern uint32_t num_harts; - -typedef uintptr_t csr_t; // TODO this might become uint128_t for RV128 - -typedef struct { - volatile csr_t* csrs; - volatile int mipi_pending; - volatile int sipi_pending; - int console_ibuf; - - uint64_t utime_delta; - uint64_t ucycle_delta; - uint64_t uinstret_delta; - uint64_t stime_delta; - uint64_t scycle_delta; - uint64_t sinstret_delta; -} hls_t; - -#define IPI_SOFT 0x1 -#define IPI_FENCE_I 0x2 -#define IPI_SFENCE_VM 0x4 - -void hls_init(uint32_t hart_id, csr_t* csrs); - -#define MACHINE_STACK_TOP() ({ \ - register uintptr_t sp asm ("sp"); \ - (void*)((sp + RISCV_PGSIZE) & -RISCV_PGSIZE); }) - -// hart-local storage, at top of stack -#define HLS() ((hls_t*)(MACHINE_STACK_TOP() - HLS_SIZE)) -#define OTHER_HLS(id) ((hls_t*)((void*)HLS() + RISCV_PGSIZE * ((id) - read_const_csr(mhartid)))) - -void parse_config_string(); -void poweroff(void) __attribute((noreturn)); -void printm(const char* s, ...); -#define assert(x) ({ if (!(x)) die("assertion failed: %s", #x); }) -#define die(str, ...) ({ printm("%s:%d: " str "\n", __FILE__, __LINE__, ##__VA_ARGS__); poweroff(); }) -#define printk(...) die("printk") - -#endif // !__ASSEMBLER__ - -#define MACHINE_STACK_SIZE RISCV_PGSIZE -#define MENTRY_FRAME_SIZE (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE \ - + HLS_SIZE) - -#ifdef __riscv_hard_float -# define SOFT_FLOAT_CONTEXT_SIZE 0 -#else -# define SOFT_FLOAT_CONTEXT_SIZE (8 * 32) -#endif -#define HLS_SIZE 64 -#define INTEGER_CONTEXT_SIZE (32 * REGBYTES) - -#endif @@ -2,13 +2,3 @@ AC_ARG_ENABLE([vm], AS_HELP_STRING([--disable-vm], [Disable virtual memory])) AS_IF([test "x$enable_vm" != "xno"], [ AC_DEFINE([PK_ENABLE_VM],,[Define if virtual memory support is enabled]) ]) - -AC_ARG_ENABLE([vm], AS_HELP_STRING([--disable-logo], [Disable boot logo])) -AS_IF([test "x$enable_logo" != "xno"], [ - AC_DEFINE([PK_ENABLE_LOGO],,[Define if the RISC-V logo is to be displayed]) -]) - -AC_ARG_ENABLE([fp-emulation], AS_HELP_STRING([--disable-fp-emulation], [Disable floating-point emulation])) -AS_IF([test "x$enable_fp_emulation" != "xno"], [ - AC_DEFINE([PK_ENABLE_FP_EMULATION],,[Define if floating-point emulation is enabled]) -]) @@ -1,24 +1,66 @@ #include "pk.h" +#include "mmap.h" #include "boot.h" -#include "vm.h" #include "elf.h" +#include "mtrap.h" +#include "frontend.h" -void run_loaded_program(size_t argc, char** argv) +elf_info current; + +int uarch_counters_enabled; +long uarch_counters[NUM_COUNTERS]; +char* uarch_counter_names[NUM_COUNTERS]; + +static void handle_option(const char* s) { - if (current.is_supervisor) - panic("pk can't run kernel binaries; try using bbl instead"); + switch (s[1]) + { + case 's': // print cycle count upon termination + current.t0 = 1; + break; - uintptr_t kernel_stack_top = pk_vm_init(); + case 'c': // print uarch counters upon termination + // If your HW doesn't support uarch counters, then don't use this flag! + uarch_counters_enabled = 1; + break; - extern char trap_entry; - write_csr(stvec, &trap_entry); - write_csr(sscratch, 0); - clear_csr(sie, SIP_STIP | SIP_SSIP); + default: + panic("unrecognized option: `%c'", s[1]); + break; + } +} - // enter supervisor mode - prepare_supervisor_mode(); - asm volatile("la t0, 1f; csrw mepc, t0; eret; 1:" ::: "t0"); +#define MAX_ARGS 64 +typedef union { + uint64_t buf[MAX_ARGS]; + char* argv[MAX_ARGS]; +} arg_buf; + +static size_t parse_args(arg_buf* args) +{ + long r = frontend_syscall(SYS_getmainvars, (uintptr_t)args, sizeof(*args), 0, 0, 0, 0, 0); + kassert(r == 0); + uint64_t* pk_argv = &args->buf[1]; + // pk_argv[0] is the proxy kernel itself. skip it and any flags. + size_t pk_argc = args->buf[0], arg = 1; + for ( ; arg < pk_argc && *(char*)(uintptr_t)pk_argv[arg] == '-'; arg++) + handle_option((const char*)(uintptr_t)pk_argv[arg]); + + for (size_t i = 0; arg + i < pk_argc; i++) + args->argv[i] = (char*)(uintptr_t)pk_argv[arg + i]; + return pk_argc - arg; +} + +static void init_tf(trapframe_t* tf, long pc, long sp) +{ + memset(tf, 0, sizeof(*tf)); + tf->status = (read_csr(sstatus) &~ SSTATUS_SPP &~ SSTATUS_SIE) | SSTATUS_SPIE; + tf->gpr[2] = sp; + tf->epc = pc; +} +static void run_loaded_program(size_t argc, char** argv, uintptr_t kstack_top) +{ // copy phdrs to user stack size_t stack_top = current.stack_top - current.phdr_size; memcpy((void*)stack_top, (void*)current.phdr, current.phdr_size); @@ -94,10 +136,37 @@ void run_loaded_program(size_t argc, char** argv) trapframe_t tf; init_tf(&tf, current.entry, stack_top); __clear_cache(0, 0); - write_csr(sscratch, kernel_stack_top); + write_csr(sscratch, kstack_top); start_user(&tf); } +static void rest_of_boot_loader(uintptr_t kstack_top) +{ + arg_buf args; + size_t argc = parse_args(&args); + if (!argc) + panic("tell me what ELF to load!"); + + // load program named by argv[0] + long phdrs[128]; + current.phdr = (uintptr_t)phdrs; + current.phdr_size = sizeof(phdrs); + load_elf(args.argv[0], ¤t); + + run_loaded_program(argc, args.argv, kstack_top); +} + +void boot_loader() +{ + extern char trap_entry; + write_csr(stvec, &trap_entry); + write_csr(sscratch, 0); + write_csr(sie, 0); + + file_init(); + enter_supervisor_mode(rest_of_boot_loader, pk_vm_init()); +} + void boot_other_hart() { // stall all harts besides hart 0 @@ -24,30 +24,18 @@ typedef struct #define kassert(cond) do { if(!(cond)) kassert_fail(""#cond); } while(0) void do_panic(const char* s, ...) __attribute__((noreturn)); void kassert_fail(const char* s) __attribute__((noreturn)); -#define MAX(a, b) ((a) > (b) ? (a) : (b)) -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define CLAMP(a, lo, hi) MIN(MAX(a, lo), hi) #ifdef __cplusplus extern "C" { #endif -extern int have_vm; - void printk(const char* s, ...); void printm(const char* s, ...); int vsnprintf(char* out, size_t n, const char* s, va_list vl); int snprintf(char* out, size_t n, const char* s, ...); -void init_tf(trapframe_t*, long pc, long sp); void start_user(trapframe_t* tf) __attribute__((noreturn)); void dump_tf(trapframe_t*); -void unhandled_trap(trapframe_t*); -void handle_misaligned_load(trapframe_t*); -void handle_misaligned_store(trapframe_t*); -void handle_fault_load(trapframe_t*); -void handle_fault_store(trapframe_t*); - static inline int insn_len(long insn) { return (insn & 0x3) < 0x3 ? 2 : 4; @@ -60,11 +48,6 @@ extern int uarch_counters_enabled; extern long uarch_counters[NUM_COUNTERS]; extern char* uarch_counter_names[NUM_COUNTERS]; -static inline void wfi() -{ - asm volatile ("wfi" ::: "memory"); -} - #ifdef __cplusplus } #endif @@ -86,10 +86,5 @@ SECTIONS *(COMMON) } - .sbi : - { - sbi.o(.sbi) - } - _end = .; } diff --git a/pk/pk.mk.in b/pk/pk.mk.in index 78f826e..3caa25d 100644 --- a/pk/pk.mk.in +++ b/pk/pk.mk.in @@ -1,52 +1,30 @@ pk_subproject_deps = \ + util \ softfloat \ + machine \ pk_hdrs = \ - atomic.h \ - bits.h \ boot.h \ elf.h \ - emulation.h \ - encoding.h \ file.h \ - fp_emulation.h \ frontend.h \ - mcall.h \ - mtrap.h \ + mmap.h \ pk.h \ - sbi.h \ syscall.h \ - unprivileged_memory.h \ - vm.h \ pk_c_srcs = \ - snprintf.c \ - mtrap.c \ - minit.c \ - emulation.c \ - fp_emulation.c \ - sbi_impl.c \ - init.c \ file.c \ syscall.c \ handlers.c \ frontend.c \ elf.c \ console.c \ - vm.c \ - string.c \ - logo.c \ - configstring.c \ + mmap.c \ pk_asm_srcs = \ - mentry.S \ entry.S \ - fp_asm.S \ - sbi_entry.S \ - sbi.S \ pk_test_srcs = pk_install_prog_srcs = \ pk.c \ - bbl.c \ diff --git a/pk/sbi.S b/pk/sbi.S deleted file mode 100644 index cbea78a..0000000 --- a/pk/sbi.S +++ /dev/null @@ -1,15 +0,0 @@ -.globl sbi_hart_id; sbi_hart_id = -2048 -.globl sbi_num_harts; sbi_num_harts = -2032 -.globl sbi_query_memory; sbi_query_memory = -2016 -.globl sbi_console_putchar; sbi_console_putchar = -2000 -.globl sbi_console_getchar; sbi_console_getchar = -1984 -.globl sbi_send_ipi; sbi_send_ipi = -1952 -.globl sbi_clear_ipi; sbi_clear_ipi = -1936 -.globl sbi_timebase; sbi_timebase = -1920 -.globl sbi_shutdown; sbi_shutdown = -1904 -.globl sbi_set_timer; sbi_set_timer = -1888 -.globl sbi_mask_interrupt; sbi_mask_interrupt = -1872 -.globl sbi_unmask_interrupt; sbi_unmask_interrupt = -1856 -.globl sbi_remote_sfence_vm; sbi_remote_sfence_vm = -1840 -.globl sbi_remote_sfence_vm_range; sbi_remote_sfence_vm_range = -1824 -.globl sbi_remote_fence_i; sbi_remote_fence_i = -1808 diff --git a/pk/sbi.h b/pk/sbi.h deleted file mode 100644 index 4e2fbd8..0000000 --- a/pk/sbi.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef _ASM_RISCV_SBI_H -#define _ASM_RISCV_SBI_H - -typedef struct { - unsigned long base; - unsigned long size; - unsigned long node_id; -} memory_block_info; - -unsigned long sbi_query_memory(unsigned long id, memory_block_info *p); - -unsigned long sbi_hart_id(void); -unsigned long sbi_num_harts(void); -unsigned long sbi_timebase(void); -void sbi_set_timer(unsigned long long stime_value); -void sbi_send_ipi(unsigned long hart_id); -unsigned long sbi_clear_ipi(void); -void sbi_shutdown(void); - -void sbi_console_putchar(unsigned char ch); -int sbi_console_getchar(void); - -void sbi_remote_sfence_vm(unsigned long hart_mask_ptr, unsigned long asid); -void sbi_remote_sfence_vm_range(unsigned long hart_mask_ptr, unsigned long asid, unsigned long start, unsigned long size); -void sbi_remote_fence_i(unsigned long hart_mask_ptr); - -unsigned long sbi_mask_interrupt(unsigned long which); -unsigned long sbi_unmask_interrupt(unsigned long which); - -#endif diff --git a/pk/sbi_entry.S b/pk/sbi_entry.S deleted file mode 100644 index bfb0703..0000000 --- a/pk/sbi_entry.S +++ /dev/null @@ -1,109 +0,0 @@ -#include "encoding.h" -#include "mcall.h" - - .section .sbi,"ax",@progbits - .option norvc - .align RISCV_PGSHIFT - .globl sbi_base -sbi_base: - - # TODO: figure out something better to do with this space. It's not - # protected from the OS, so beware. - .skip RISCV_PGSIZE - 2048 - - # hart_id - .align 4 - li a7, MCALL_HART_ID - ecall - ret - - # num_harts - .align 4 - lw a0, num_harts - ret - - # query_memory - .align 4 - j __sbi_query_memory - - # console_putchar - .align 4 - li a7, MCALL_CONSOLE_PUTCHAR - ecall - ret - - # console_getchar - .align 4 - li a7, MCALL_CONSOLE_GETCHAR - ecall - ret - - # empty - .align 4 - unimp - - # send ipi - .align 4 - li a7, MCALL_SEND_IPI - ecall - ret - - # clear ipi - .align 4 - li a7, MCALL_CLEAR_IPI - ecall - ret - - # timebase - .align 4 - li a0, 10000000 # or, you know, we could provide the correct answer - ret - - # shutdown - .align 4 - li a7, MCALL_SHUTDOWN - ecall - - # set_timer - .align 4 - li a7, MCALL_SET_TIMER - ecall - ret - - # mask_interrupt - .align 4 - j __sbi_mask_interrupt - - # unmask_interrupt - .align 4 - j __sbi_unmask_interrupt - - # remote_sfence_vm - .align 4 - li a7, MCALL_REMOTE_SFENCE_VM - ecall - ret - - # remote_sfence_vm_range - .align 4 - li a7, MCALL_REMOTE_SFENCE_VM - ecall - ret - - # remote_fence_i - .align 4 - li a7, MCALL_REMOTE_FENCE_I - ecall - ret - - # end of SBI trampolines - - .globl do_mcall -do_mcall: - mv a7, a0 - mv a0, a1 - mv a1, a2 - ecall - ret - - .align RISCV_PGSHIFT diff --git a/pk/sbi_impl.c b/pk/sbi_impl.c deleted file mode 100644 index 4af0cd4..0000000 --- a/pk/sbi_impl.c +++ /dev/null @@ -1,34 +0,0 @@ -#include "mtrap.h" -#include "sbi.h" -#include "boot.h" - -uintptr_t __sbi_query_memory(uintptr_t id, memory_block_info *p) -{ - if (id == 0) { - p->base = current.first_free_paddr; - p->size = mem_size - p->base; - return 0; - } - - return -1; -} - -#define LOW_IRQ_OK(n) ((n) == IRQ_S_SOFT || (n) == IRQ_S_TIMER) - -uintptr_t __sbi_mask_interrupt(uintptr_t which) -{ - if (!LOW_IRQ_OK(which)) - return -1; - - clear_csr(sie, 1UL << which); - return 0; -} - -uintptr_t __sbi_unmask_interrupt(uintptr_t which) -{ - if (!LOW_IRQ_OK(which)) - return -1; - - set_csr(sie, 1UL << which); - return 0; -} diff --git a/pk/snprintf.c b/pk/snprintf.c deleted file mode 100644 index 1544a6c..0000000 --- a/pk/snprintf.c +++ /dev/null @@ -1,98 +0,0 @@ -// See LICENSE for license details. - -#include <stdint.h> -#include <string.h> -#include <stdarg.h> -#include <stdbool.h> - -int vsnprintf(char* out, size_t n, const char* s, va_list vl) -{ - bool format = false; - bool longarg = false; - size_t pos = 0; - for( ; *s; s++) - { - if(format) - { - switch(*s) - { - case 'l': - longarg = true; - break; - case 'p': - longarg = true; - if (++pos < n) out[pos-1] = '0'; - if (++pos < n) out[pos-1] = 'x'; - case 'x': - { - long num = longarg ? va_arg(vl, long) : va_arg(vl, int); - for(int i = 2*(longarg ? sizeof(long) : sizeof(int))-1; i >= 0; i--) { - int d = (num >> (4*i)) & 0xF; - if (++pos < n) out[pos-1] = (d < 10 ? '0'+d : 'a'+d-10); - } - longarg = false; - format = false; - break; - } - case 'd': - { - long num = longarg ? va_arg(vl, long) : va_arg(vl, int); - if (num < 0) { - num = -num; - if (++pos < n) out[pos-1] = '-'; - } - long digits = 1; - for (long nn = num; nn /= 10; digits++) - ; - for (int i = digits-1; i >= 0; i--) { - if (pos + i + 1 < n) out[pos + i] = '0' + (num % 10); - num /= 10; - } - pos += digits; - longarg = false; - format = false; - break; - } - case 's': - { - const char* s2 = va_arg(vl, const char*); - while (*s2) { - if (++pos < n) - out[pos-1] = *s2; - s2++; - } - longarg = false; - format = false; - break; - } - case 'c': - { - if (++pos < n) out[pos-1] = (char)va_arg(vl,int); - longarg = false; - format = false; - break; - } - default: - break; - } - } - else if(*s == '%') - format = true; - else - if (++pos < n) out[pos-1] = *s; - } - if (pos < n) - out[pos] = 0; - else if (n) - out[n-1] = 0; - return pos; -} - -int snprintf(char* out, size_t n, const char* s, ...) -{ - va_list vl; - va_start(vl, s); - int res = vsnprintf(out, n, s, vl); - va_end(vl); - return res; -} diff --git a/pk/string.c b/pk/string.c deleted file mode 100644 index e896379..0000000 --- a/pk/string.c +++ /dev/null @@ -1,87 +0,0 @@ -#include <string.h> -#include <stdint.h> -#include <ctype.h> - -void* memcpy(void* dest, const void* src, size_t len) -{ - if ((((uintptr_t)dest | (uintptr_t)src | len) & (sizeof(uintptr_t)-1)) == 0) { - const uintptr_t* s = src; - uintptr_t *d = dest; - while (d < (uintptr_t*)(dest + len)) - *d++ = *s++; - } else { - const char* s = src; - char *d = dest; - while (d < (char*)(dest + len)) - *d++ = *s++; - } - return dest; -} - -void* memset(void* dest, int byte, size_t len) -{ - if ((((uintptr_t)dest | len) & (sizeof(uintptr_t)-1)) == 0) { - uintptr_t word = byte & 0xFF; - word |= word << 8; - word |= word << 16; - word |= word << 16 << 16; - - uintptr_t *d = dest; - while (d < (uintptr_t*)(dest + len)) - *d++ = word; - } else { - char *d = dest; - while (d < (char*)(dest + len)) - *d++ = byte; - } - return dest; -} - -size_t strlen(const char *s) -{ - const char *p = s; - while (*p) - p++; - return p - s; -} - -int strcmp(const char* s1, const char* s2) -{ - unsigned char c1, c2; - - do { - c1 = *s1++; - c2 = *s2++; - } while (c1 != 0 && c1 == c2); - - return c1 - c2; -} - -char* strcpy(char* dest, const char* src) -{ - char* d = dest; - while ((*d++ = *src++)) - ; - return dest; -} - -long atol(const char* str) -{ - long res = 0; - int sign = 0; - - while (*str == ' ') - str++; - - if (*str == '-' || *str == '+') { - sign = *str == '-'; - str++; - } - - while (*str) { - res *= 10; - res += *str++ - '0'; - } - - return sign ? -res : res; -} diff --git a/pk/syscall.c b/pk/syscall.c index 17112d8..de11a0a 100644 --- a/pk/syscall.c +++ b/pk/syscall.c @@ -4,7 +4,7 @@ #include "pk.h" #include "file.h" #include "frontend.h" -#include "vm.h" +#include "mmap.h" #include "boot.h" #include <string.h> #include <errno.h> diff --git a/pk/unprivileged_memory.h b/pk/unprivileged_memory.h deleted file mode 100644 index d03cc5e..0000000 --- a/pk/unprivileged_memory.h +++ /dev/null @@ -1,79 +0,0 @@ -#ifndef _RISCV_MISALIGNED_H -#define _RISCV_MISALIGNED_H - -#include "encoding.h" -#include <stdint.h> - -#define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type, insn) \ - static inline type load_##type(const type* addr, uintptr_t mepc) \ - { \ - register uintptr_t __mepc asm ("a2") = mepc; \ - register uintptr_t __mstatus asm ("a3"); \ - type val; \ - asm ("csrrs %0, mstatus, %3\n" \ - #insn " %1, %2\n" \ - "csrw mstatus, %0" \ - : "+&r" (__mstatus), "=&r" (val) \ - : "m" (*addr), "r" (MSTATUS_MPRV), "r" (__mepc)); \ - return val; \ - } - -#define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type, insn) \ - static inline void store_##type(type* addr, type val, uintptr_t mepc) \ - { \ - register uintptr_t __mepc asm ("a2") = mepc; \ - register uintptr_t __mstatus asm ("a3"); \ - asm volatile ("csrrs %0, mstatus, %3\n" \ - #insn " %1, %2\n" \ - "csrw mstatus, %0" \ - : "+&r" (__mstatus) \ - : "r" (val), "m" (*addr), "r" (MSTATUS_MPRV), \ - "r" (__mepc)); \ - } - -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint8_t, lbu) -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint16_t, lhu) -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int8_t, lb) -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int16_t, lh) -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int32_t, lw) -DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint8_t, sb) -DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint16_t, sh) -DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint32_t, sw) -#ifdef __riscv64 -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lwu) -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint64_t, ld) -DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint64_t, sd) -#else -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lw) -#endif - -static uint32_t __attribute__((always_inline)) get_insn(uintptr_t mepc, uintptr_t* mstatus) -{ - register uintptr_t __mepc asm ("a2") = mepc; - register uintptr_t __mstatus asm ("a3"); - uint32_t val; -#ifndef __riscv_compressed - asm ("csrrs %[mstatus], mstatus, %[mprv]\n" - "lw %[insn], (%[addr])\n" - "csrw mstatus, %[mstatus]" - : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val) - : [mprv] "r" (MSTATUS_MPRV), [addr] "r" (__mepc)); -#else - uintptr_t rvc_mask = 3, tmp; - asm ("csrrs %[mstatus], mstatus, %[mprv]\n" - "lhu %[insn], (%[addr])\n" - "and %[tmp], %[insn], %[rvc_mask]\n" - "bne %[tmp], %[rvc_mask], 1f\n" - "lh %[tmp], 2(%[addr])\n" - "sll %[tmp], %[tmp], 16\n" - "add %[insn], %[insn], %[tmp]\n" - "1: csrw mstatus, %[mstatus]" - : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val), [tmp] "=&r" (tmp) - : [mprv] "r" (MSTATUS_MPRV), [addr] "r" (__mepc), - [rvc_mask] "r" (rvc_mask)); -#endif - *mstatus = __mstatus; - return val; -} - -#endif |