diff options
author | Andrew Waterman <waterman@cs.berkeley.edu> | 2016-03-09 23:58:17 -0800 |
---|---|---|
committer | Andrew Waterman <waterman@cs.berkeley.edu> | 2016-03-09 23:58:17 -0800 |
commit | b94c7a4b07f96f24ae7411780abf874416549f7b (patch) | |
tree | b94ca015e49392f52e5abf1209ee184fcf874db4 /machine | |
parent | f5a96732cb81571a3ba6b081b8556187d564f678 (diff) | |
download | pk-b94c7a4b07f96f24ae7411780abf874416549f7b.zip pk-b94c7a4b07f96f24ae7411780abf874416549f7b.tar.gz pk-b94c7a4b07f96f24ae7411780abf874416549f7b.tar.bz2 |
Refactor pk, bbl, machine into separate libraries
Yuck.
Diffstat (limited to 'machine')
-rw-r--r-- | machine/atomic.h | 72 | ||||
-rw-r--r-- | machine/bits.h | 33 | ||||
-rw-r--r-- | machine/configstring.c | 200 | ||||
-rw-r--r-- | machine/emulation.c | 313 | ||||
-rw-r--r-- | machine/emulation.h | 28 | ||||
-rw-r--r-- | machine/encoding.h | 1037 | ||||
-rw-r--r-- | machine/fp_asm.S | 160 | ||||
-rw-r--r-- | machine/fp_emulation.c | 449 | ||||
-rw-r--r-- | machine/fp_emulation.h | 81 | ||||
-rw-r--r-- | machine/htif.h | 18 | ||||
-rw-r--r-- | machine/machine.ac | 4 | ||||
-rw-r--r-- | machine/machine.mk.in | 29 | ||||
-rw-r--r-- | machine/mcall.h | 21 | ||||
-rw-r--r-- | machine/mentry.S | 245 | ||||
-rw-r--r-- | machine/minit.c | 120 | ||||
-rw-r--r-- | machine/mtrap.c | 287 | ||||
-rw-r--r-- | machine/mtrap.h | 96 | ||||
-rw-r--r-- | machine/sbi.S | 15 | ||||
-rw-r--r-- | machine/sbi.h | 30 | ||||
-rw-r--r-- | machine/sbi_entry.S | 111 | ||||
-rw-r--r-- | machine/sbi_impl.c | 33 | ||||
-rw-r--r-- | machine/unprivileged_memory.h | 79 | ||||
-rw-r--r-- | machine/vm.h | 35 |
23 files changed, 3496 insertions, 0 deletions
diff --git a/machine/atomic.h b/machine/atomic.h new file mode 100644 index 0000000..b23acae --- /dev/null +++ b/machine/atomic.h @@ -0,0 +1,72 @@ +// See LICENSE for license details. + +#ifndef _RISCV_ATOMIC_H +#define _RISCV_ATOMIC_H + +#include "config.h" +#include "encoding.h" + +// Currently, interrupts are always disabled in M-mode. +#define disable_irqsave() (0) +#define enable_irqrestore(flags) ((void) (flags)) + +typedef struct { int lock; } spinlock_t; +#define SPINLOCK_INIT {0} + +#define mb() asm volatile ("fence" ::: "memory") +#define atomic_set(ptr, val) (*(volatile typeof(*(ptr)) *)(ptr) = val) +#define atomic_read(ptr) (*(volatile typeof(*(ptr)) *)(ptr)) + +#ifdef __riscv_atomic +# define atomic_add(ptr, inc) __sync_fetch_and_add(ptr, inc) +# define atomic_or(ptr, inc) __sync_fetch_and_or(ptr, inc) +# define atomic_swap(ptr, swp) __sync_lock_test_and_set(ptr, swp) +# define atomic_cas(ptr, cmp, swp) __sync_val_compare_and_swap(ptr, cmp, swp) +#else +# define atomic_binop(ptr, inc, op) ({ \ + long flags = disable_irqsave(); \ + typeof(*(ptr)) res = atomic_read(ptr); \ + atomic_set(ptr, op); \ + enable_irqrestore(flags); \ + res; }) +# define atomic_add(ptr, inc) atomic_binop(ptr, inc, res + (inc)) +# define atomic_or(ptr, inc) atomic_binop(ptr, inc, res | (inc)) +# define atomic_swap(ptr, inc) atomic_binop(ptr, inc, (inc)) +# define atomic_cas(ptr, cmp, swp) ({ \ + long flags = disable_irqsave(); \ + typeof(*(ptr)) res = *(volatile typeof(*(ptr)) *)(ptr); \ + if (res == (cmp)) *(volatile typeof(ptr))(ptr) = (swp); \ + enable_irqrestore(flags); \ + res; }) +#endif + +static inline void spinlock_lock(spinlock_t* lock) +{ + do + { + while (atomic_read(&lock->lock)) + ; + } while (atomic_swap(&lock->lock, -1)); + mb(); +} + +static inline void spinlock_unlock(spinlock_t* lock) +{ + mb(); + atomic_set(&lock->lock,0); +} + +static inline long spinlock_lock_irqsave(spinlock_t* lock) +{ + long flags = disable_irqsave(); + spinlock_lock(lock); + return flags; +} + +static inline void spinlock_unlock_irqrestore(spinlock_t* lock, long flags) +{ + spinlock_unlock(lock); + enable_irqrestore(flags); +} + +#endif diff --git a/machine/bits.h b/machine/bits.h new file mode 100644 index 0000000..72514ae --- /dev/null +++ b/machine/bits.h @@ -0,0 +1,33 @@ +#ifndef _RISCV_BITS_H +#define _RISCV_BITS_H + +#define likely(x) __builtin_expect((x), 1) +#define unlikely(x) __builtin_expect((x), 0) + +#define ROUNDUP(a, b) ((((a)-1)/(b)+1)*(b)) +#define ROUNDDOWN(a, b) ((a)/(b)*(b)) + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define CLAMP(a, lo, hi) MIN(MAX(a, lo), hi) + +#define EXTRACT_FIELD(val, which) (((val) & (which)) / ((which) & ~((which)-1))) +#define INSERT_FIELD(val, which, fieldval) (((val) & ~(which)) | ((fieldval) * ((which) & ~((which)-1)))) + +#define STR(x) XSTR(x) +#define XSTR(x) #x + +#ifdef __riscv64 +# define SLL32 sllw +# define STORE sd +# define LOAD ld +# define LOG_REGBYTES 3 +#else +# define SLL32 sll +# define STORE sw +# define LOAD lw +# define LOG_REGBYTES 2 +#endif +#define REGBYTES (1 << LOG_REGBYTES) + +#endif diff --git a/machine/configstring.c b/machine/configstring.c new file mode 100644 index 0000000..847cd4d --- /dev/null +++ b/machine/configstring.c @@ -0,0 +1,200 @@ +#include "encoding.h" +#include "mtrap.h" +#include <stdio.h> + +static const char* skip_whitespace(const char* str) +{ + while (*str && *str <= ' ') + str++; + return str; +} + +static const char* skip_string(const char* str) +{ + while (*str && *str++ != '"') + ; + return str; +} + +static int is_hex(char ch) +{ + return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); +} + +static int parse_hex(char ch) +{ + return (ch >= '0' && ch <= '9') ? ch - '0' : + (ch >= 'a' && ch <= 'f') ? ch - 'a' + 10 : + ch - 'A' + 10; +} + +static const char* skip_key(const char* str) +{ + while (*str >= 35 && *str <= 122 && *str != ';') + str++; + return str; +} + +typedef struct { + const char* start; + const char* end; +} query_result; + +static query_result query_config_string(const char* str, const char* k) +{ + size_t ksize = 0; + while (k[ksize] && k[ksize] != '{') + ksize++; + int last = !k[ksize]; + + query_result res = {0, 0}; + while (1) { + const char* key_start = str = skip_whitespace(str); + const char* key_end = str = skip_key(str); + int match = (key_end - key_start) == ksize; + if (match) + for (size_t i = 0; i < ksize; i++) + if (key_start[i] != k[i]) + match = 0; + const char* value_start = str = skip_whitespace(str); + while (*str != ';') { + if (!*str) { + return res; + } else if (*str == '"') { + str = skip_string(str+1); + } else if (*str == '{') { + const char* search_key = match && !last ? k + ksize + 1 : ""; + query_result inner_res = query_config_string(str + 1, search_key); + if (inner_res.start) + return inner_res; + str = inner_res.end + 1; + } else { + str = skip_key(str); + } + str = skip_whitespace(str); + } + res.end = str; + if (match && last) { + res.start = value_start; + return res; + } + str = skip_whitespace(str+1); + if (*str == '}') { + res.end = str; + return res; + } + } +} + +static void parse_string(query_result r, char* buf) +{ + if (r.start < r.end) { + if (*r.start == '"') { + for (const char* p = r.start + 1; p < r.end && *p != '"'; p++) { + char ch = p[0]; + if (ch == '\\' && p[1] == 'x' && is_hex(p[2])) { + ch = parse_hex(p[2]); + if (is_hex(p[3])) { + ch = (ch << 4) + parse_hex(p[3]); + p++; + } + p += 2; + } + *buf++ = ch; + } + } else { + for (const char* p = r.start; p < r.end && *p > ' '; p++) + *buf++ = *p; + } + } + *buf = 0; +} + +#define get_string(name, search_res) \ + char name[(search_res).end - (search_res).start + 1]; \ + parse_string(search_res, name) + +static unsigned long __get_uint_hex(const char* s) +{ + unsigned long res = 0; + while (*s) { + if (is_hex(*s)) + res = (res << 4) + parse_hex(*s); + else if (*s != '_') + break; + s++; + } + return res; +} + +static unsigned long __get_uint_dec(const char* s) +{ + unsigned long res = 0; + while (*s) { + if (*s >= '0' && *s <= '9') + res = res * 10 + (*s - '0'); + else + break; + s++; + } + return res; +} + +static unsigned long __get_uint(const char* s) +{ + if (s[0] == '0' && s[1] == 'x') + return __get_uint_hex(s+2); + return __get_uint_dec(s); +} + +static unsigned long get_uint(query_result res) +{ + get_string(s, res); + return __get_uint(s); +} + +static long get_sint(query_result res) +{ + get_string(s, res); + if (s[0] == '-') + return -__get_uint(s+1); + return __get_uint(s); +} + +static void query_mem(const char* config_string) +{ + query_result res = query_config_string(config_string, "ram{0{addr"); + assert(res.start); + uintptr_t base = get_uint(res); + res = query_config_string(config_string, "ram{0{size"); + mem_size = get_uint(res); +} + +static void query_harts(const char* config_string) +{ + for (int core = 0, hart; ; core++) { + for (hart = 0; ; hart++) { + char buf[32]; + snprintf(buf, sizeof buf, "core{%d{%d{addr", core, hart); + query_result res = query_config_string(config_string, buf); + if (!res.start) + break; + csr_t* base = (csr_t*)get_uint(res); + uintptr_t hart_id = base[CSR_MHARTID]; + hls_init(hart_id, base); + num_harts++; + assert(hart_id == num_harts-1); + } + if (!hart) + break; + } + assert(num_harts); + assert(num_harts <= MAX_HARTS); +} + +void parse_config_string() +{ + const char* s = (const char*)read_csr(mcfgaddr); + query_mem(s); + query_harts(s); +} diff --git a/machine/emulation.c b/machine/emulation.c new file mode 100644 index 0000000..5a66997 --- /dev/null +++ b/machine/emulation.c @@ -0,0 +1,313 @@ +#include "emulation.h" +#include "fp_emulation.h" +#include "config.h" +#include "unprivileged_memory.h" +#include "mtrap.h" +#include <limits.h> + +void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + asm (".pushsection .rodata\n" + "illegal_insn_trap_table:\n" + " .word truly_illegal_insn\n" +#ifdef PK_ENABLE_FP_EMULATION + " .word emulate_float_load\n" +#else + " .word truly_illegal_insn\n" +#endif + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" +#ifdef PK_ENABLE_FP_EMULATION + " .word emulate_float_store\n" +#else + " .word truly_illegal_insn\n" +#endif + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_mul_div\n" + " .word truly_illegal_insn\n" + " .word emulate_mul_div32\n" + " .word truly_illegal_insn\n" +#ifdef PK_ENABLE_FP_EMULATION + " .word emulate_fmadd\n" + " .word emulate_fmadd\n" + " .word emulate_fmadd\n" + " .word emulate_fmadd\n" + " .word emulate_fp\n" +#else + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" +#endif + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_system\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .popsection"); + + uintptr_t mstatus; + insn_t insn = get_insn(mepc, &mstatus); + + if (unlikely((insn & 3) != 3)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + write_csr(mepc, mepc + 4); + + extern int32_t illegal_insn_trap_table[]; + int32_t* pf = (void*)illegal_insn_trap_table + (insn & 0x7c); + emulation_func f = (emulation_func)(uintptr_t)*pf; + f(regs, mcause, mepc, mstatus, insn); +} + +void __attribute__((noinline)) truly_illegal_insn(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn) +{ + redirect_trap(mepc, mstatus); +} + +void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + union { + uint8_t bytes[8]; + uintptr_t intx; + uint64_t int64; + } val; + uintptr_t mstatus; + insn_t insn = get_insn(mepc, &mstatus); + uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); + + int shift = 0, fp = 0, len; + if ((insn & MASK_LW) == MATCH_LW) + len = 4, shift = 8*(sizeof(uintptr_t) - len); +#ifdef __riscv64 + else if ((insn & MASK_LD) == MATCH_LD) + len = 8, shift = 8*(sizeof(uintptr_t) - len); + else if ((insn & MASK_LWU) == MATCH_LWU) + fp = 0, len = 4, shift = 0; +#endif + else if ((insn & MASK_FLD) == MATCH_FLD) + fp = 1, len = 8; + else if ((insn & MASK_FLW) == MATCH_FLW) + fp = 1, len = 4; + else if ((insn & MASK_LH) == MATCH_LH) + len = 2, shift = 8*(sizeof(uintptr_t) - len); + else if ((insn & MASK_LHU) == MATCH_LHU) + len = 2; + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + val.int64 = 0; + for (intptr_t i = len-1; i >= 0; i--) + val.bytes[i] = load_uint8_t((void *)(addr + i), mepc); + + if (!fp) + SET_RD(insn, regs, (intptr_t)val.intx << shift >> shift); + else if (len == 8) + SET_F64_RD(insn, regs, val.int64); + else + SET_F32_RD(insn, regs, val.intx); + + write_csr(mepc, mepc + 4); +} + +void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + union { + uint8_t bytes[8]; + uintptr_t intx; + uint64_t int64; + } val; + uintptr_t mstatus; + insn_t insn = get_insn(mepc, &mstatus); + int len; + + val.intx = GET_RS2(insn, regs); + if ((insn & MASK_SW) == MATCH_SW) + len = 4; +#ifdef __riscv64 + else if ((insn & MASK_SD) == MATCH_SD) + len = 8; +#endif + else if ((insn & MASK_FSD) == MATCH_FSD) + len = 8, val.int64 = GET_F64_RS2(insn, regs); + else if ((insn & MASK_FSW) == MATCH_FSW) + len = 4, val.intx = GET_F32_RS2(insn, regs); + else if ((insn & MASK_SH) == MATCH_SH) + len = 2; + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); + for (int i = 0; i < len; i++) + store_uint8_t((void *)(addr + i), val.bytes[i], mepc); + + write_csr(mepc, mepc + 4); +} + +#ifdef __riscv64 +typedef __int128 double_int; +#else +typedef int64_t double_int; +#endif + +DECLARE_EMULATION_FUNC(emulate_mul_div) +{ + uintptr_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs), val; + +#ifndef __riscv_muldiv + // If compiled with -mno-multiply, GCC will expand these out + if ((insn & MASK_MUL) == MATCH_MUL) + val = rs1 * rs2; + else if ((insn & MASK_DIV) == MATCH_DIV) + val = (intptr_t)rs1 / (intptr_t)rs2; + else if ((insn & MASK_DIVU) == MATCH_DIVU) + val = rs1 / rs2; + else if ((insn & MASK_REM) == MATCH_REM) + val = (intptr_t)rs1 % (intptr_t)rs2; + else if ((insn & MASK_REMU) == MATCH_REMU) + val = rs1 % rs2; + else if ((insn & MASK_MULH) == MATCH_MULH) + val = ((double_int)(intptr_t)rs1 * (double_int)(intptr_t)rs2) >> (8 * sizeof(rs1)); + else if ((insn & MASK_MULHU) == MATCH_MULHU) + val = ((double_int)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); + else if ((insn & MASK_MULHSU) == MATCH_MULHSU) + val = ((double_int)(intptr_t)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); + else +#endif + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + SET_RD(insn, regs, val); +} + +DECLARE_EMULATION_FUNC(emulate_mul_div32) +{ + uint32_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs); + int32_t val; + +#if defined(__riscv64) && !defined(__riscv_muldiv) + // If compiled with -mno-multiply, GCC will expand these out + if ((insn & MASK_MULW) == MATCH_MULW) + val = rs1 * rs2; + else if ((insn & MASK_DIVW) == MATCH_DIVW) + val = (int32_t)rs1 / (int32_t)rs2; + else if ((insn & MASK_DIVUW) == MATCH_DIVUW) + val = rs1 / rs2; + else if ((insn & MASK_REMW) == MATCH_REMW) + val = (int32_t)rs1 % (int32_t)rs2; + else if ((insn & MASK_REMUW) == MATCH_REMUW) + val = rs1 % rs2; + else +#endif + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + SET_RD(insn, regs, val); +} + +static inline int emulate_read_csr(int num, uintptr_t mstatus, uintptr_t* result) +{ + switch (num) + { + case CSR_TIME: + *result = read_csr(mtime) + HLS()->utime_delta; + return 0; + case CSR_CYCLE: + *result = read_csr(mcycle) + HLS()->ucycle_delta; + return 0; + case CSR_INSTRET: + *result = read_csr(minstret) + HLS()->uinstret_delta; + return 0; + case CSR_STIME: + *result = read_csr(mtime) + HLS()->stime_delta; + return 0; + case CSR_SCYCLE: + *result = read_csr(mcycle) + HLS()->scycle_delta; + return 0; + case CSR_SINSTRET: + *result = read_csr(minstret) + HLS()->sinstret_delta; + return 0; +#ifdef __riscv32 + case CSR_TIMEH: + *result = (((uint64_t)read_csr(mtimeh) << 32) + read_csr(mtime) + + HLS()->stime_delta) >> 32; + return 0; + case CSR_CYCLEH: + *result = (((uint64_t)read_csr(mcycleh) << 32) + read_csr(mcycle) + + HLS()->scycle_delta) >> 32; + return 0; + case CSR_INSTRETH: + *result = (((uint64_t)read_csr(minstreth) << 32) + read_csr(minstret) + + HLS()->sinstret_delta) >> 32; + return 0; +#endif +#ifdef PK_ENABLE_FP_EMULATION + case CSR_FRM: + if ((mstatus & MSTATUS_FS) == 0) break; + *result = GET_FRM(); + return 0; + case CSR_FFLAGS: + if ((mstatus & MSTATUS_FS) == 0) break; + *result = GET_FFLAGS(); + return 0; + case CSR_FCSR: + if ((mstatus & MSTATUS_FS) == 0) break; + *result = GET_FCSR(); + return 0; +#endif + } + return -1; +} + +static inline int emulate_write_csr(int num, uintptr_t value, uintptr_t mstatus) +{ + switch (num) + { +#ifdef PK_ENABLE_FP_EMULATION + case CSR_FRM: SET_FRM(value); return 0; + case CSR_FFLAGS: SET_FFLAGS(value); return 0; + case CSR_FCSR: SET_FCSR(value); return 0; +#endif + } + return -1; +} + +DECLARE_EMULATION_FUNC(emulate_system) +{ + int rs1_num = (insn >> 15) & 0x1f; + uintptr_t rs1_val = GET_RS1(insn, regs); + int csr_num = (uint32_t)insn >> 20; + uintptr_t csr_val, new_csr_val; + + if (emulate_read_csr(csr_num, mstatus, &csr_val)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + int do_write = rs1_num; + switch (GET_RM(insn)) + { + case 0: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + case 1: new_csr_val = rs1_val; do_write = 1; break; + case 2: new_csr_val = csr_val | rs1_val; break; + case 3: new_csr_val = csr_val & ~rs1_val; break; + case 4: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + case 5: new_csr_val = rs1_num; do_write = 1; break; + case 6: new_csr_val = csr_val | rs1_num; break; + case 7: new_csr_val = csr_val & ~rs1_num; break; + } + + if (do_write && emulate_write_csr(csr_num, new_csr_val, mstatus)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + SET_RD(insn, regs, csr_val); +} diff --git a/machine/emulation.h b/machine/emulation.h new file mode 100644 index 0000000..f1a71ec --- /dev/null +++ b/machine/emulation.h @@ -0,0 +1,28 @@ +#ifndef _RISCV_EMULATION_H +#define _RISCV_EMULATION_H + +#include "encoding.h" +#include "bits.h" +#include <stdint.h> + +typedef uint32_t insn_t; +typedef void (*emulation_func)(uintptr_t*, uintptr_t, uintptr_t, uintptr_t, insn_t); +#define DECLARE_EMULATION_FUNC(name) void name(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn) + +void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); +void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); +void redirect_trap(uintptr_t epc, uintptr_t mstatus); +DECLARE_EMULATION_FUNC(truly_illegal_insn); + +#define GET_REG(insn, pos, regs) ({ \ + int mask = (1 << (5+LOG_REGBYTES)) - (1 << LOG_REGBYTES); \ + (uintptr_t*)((uintptr_t)regs + (((insn) >> ((pos) - LOG_REGBYTES)) & mask)); \ +}) +#define GET_RS1(insn, regs) (*GET_REG(insn, 15, regs)) +#define GET_RS2(insn, regs) (*GET_REG(insn, 20, regs)) +#define SET_RD(insn, regs, val) (*GET_REG(insn, 7, regs) = (val)) +#define IMM_I(insn) ((int32_t)(insn) >> 20) +#define IMM_S(insn) (((int32_t)(insn) >> 25 << 5) | (int32_t)(((insn) >> 7) & 0x1f)) +#define MASK_FUNCT3 0x7000 + +#endif diff --git a/machine/encoding.h b/machine/encoding.h new file mode 100644 index 0000000..f2fab36 --- /dev/null +++ b/machine/encoding.h @@ -0,0 +1,1037 @@ +// See LICENSE for license details. + +#ifndef RISCV_CSR_ENCODING_H +#define RISCV_CSR_ENCODING_H + +#define MSTATUS_UIE 0x00000001 +#define MSTATUS_SIE 0x00000002 +#define MSTATUS_HIE 0x00000004 +#define MSTATUS_MIE 0x00000008 +#define MSTATUS_UPIE 0x00000010 +#define MSTATUS_SPIE 0x00000020 +#define MSTATUS_HPIE 0x00000040 +#define MSTATUS_MPIE 0x00000080 +#define MSTATUS_SPP 0x00000100 +#define MSTATUS_HPP 0x00000600 +#define MSTATUS_MPP 0x00001800 +#define MSTATUS_FS 0x00006000 +#define MSTATUS_XS 0x00018000 +#define MSTATUS_MPRV 0x00020000 +#define MSTATUS_PUM 0x00040000 +#define MSTATUS_VM 0x1F000000 +#define MSTATUS32_SD 0x80000000 +#define MSTATUS64_SD 0x8000000000000000 + +#define SSTATUS_UIE 0x00000001 +#define SSTATUS_SIE 0x00000002 +#define SSTATUS_UPIE 0x00000010 +#define SSTATUS_SPIE 0x00000020 +#define SSTATUS_SPP 0x00000100 +#define SSTATUS_FS 0x00006000 +#define SSTATUS_XS 0x00018000 +#define SSTATUS_PUM 0x00040000 +#define SSTATUS32_SD 0x80000000 +#define SSTATUS64_SD 0x8000000000000000 + +#define MIP_SSIP (1 << IRQ_S_SOFT) +#define MIP_HSIP (1 << IRQ_H_SOFT) +#define MIP_MSIP (1 << IRQ_M_SOFT) +#define MIP_STIP (1 << IRQ_S_TIMER) +#define MIP_HTIP (1 << IRQ_H_TIMER) +#define MIP_MTIP (1 << IRQ_M_TIMER) + +#define SIP_SSIP MIP_SSIP +#define SIP_STIP MIP_STIP + +#define PRV_U 0 +#define PRV_S 1 +#define PRV_H 2 +#define PRV_M 3 + +#define VM_MBARE 0 +#define VM_MBB 1 +#define VM_MBBID 2 +#define VM_SV32 8 +#define VM_SV39 9 +#define VM_SV48 10 + +#define IRQ_S_SOFT 1 +#define IRQ_H_SOFT 2 +#define IRQ_M_SOFT 3 +#define IRQ_S_TIMER 5 +#define IRQ_H_TIMER 6 +#define IRQ_M_TIMER 7 +#define IRQ_S_DEV 9 +#define IRQ_H_DEV 10 +#define IRQ_M_DEV 11 +#define IRQ_COP 12 +#define IRQ_HOST 13 + +#define DEFAULT_RSTVEC 0x0 +#define DEFAULT_NMIVEC 0x4 +#define DEFAULT_MTVEC 0x8 + +// page table entry (PTE) fields +#define PTE_V 0x001 // Valid +#define PTE_TYPE 0x01E // Type +#define PTE_R 0x020 // Referenced +#define PTE_D 0x040 // Dirty +#define PTE_SOFT 0x380 // Reserved for Software + +#define PTE_TYPE_TABLE 0x00 +#define PTE_TYPE_TABLE_GLOBAL 0x02 +#define PTE_TYPE_URX_SR 0x04 +#define PTE_TYPE_URWX_SRW 0x06 +#define PTE_TYPE_UR_SR 0x08 +#define PTE_TYPE_URW_SRW 0x0A +#define PTE_TYPE_URX_SRX 0x0C +#define PTE_TYPE_URWX_SRWX 0x0E +#define PTE_TYPE_SR 0x10 +#define PTE_TYPE_SRW 0x12 +#define PTE_TYPE_SRX 0x14 +#define PTE_TYPE_SRWX 0x16 +#define PTE_TYPE_SR_GLOBAL 0x18 +#define PTE_TYPE_SRW_GLOBAL 0x1A +#define PTE_TYPE_SRX_GLOBAL 0x1C +#define PTE_TYPE_SRWX_GLOBAL 0x1E + +#define PTE_PPN_SHIFT 10 + +#define PTE_TABLE(PTE) ((0x0000000AU >> ((PTE) & 0x1F)) & 1) +#define PTE_UR(PTE) ((0x0000AAA0U >> ((PTE) & 0x1F)) & 1) +#define PTE_UW(PTE) ((0x00008880U >> ((PTE) & 0x1F)) & 1) +#define PTE_UX(PTE) ((0x0000A0A0U >> ((PTE) & 0x1F)) & 1) +#define PTE_SR(PTE) ((0xAAAAAAA0U >> ((PTE) & 0x1F)) & 1) +#define PTE_SW(PTE) ((0x88888880U >> ((PTE) & 0x1F)) & 1) +#define PTE_SX(PTE) ((0xA0A0A000U >> ((PTE) & 0x1F)) & 1) + +#define PTE_CHECK_PERM(PTE, SUPERVISOR, STORE, FETCH) \ + ((STORE) ? ((SUPERVISOR) ? PTE_SW(PTE) : PTE_UW(PTE)) : \ + (FETCH) ? ((SUPERVISOR) ? PTE_SX(PTE) : PTE_UX(PTE)) : \ + ((SUPERVISOR) ? PTE_SR(PTE) : PTE_UR(PTE))) + +#ifdef __riscv + +#ifdef __riscv64 +# define MSTATUS_SD MSTATUS64_SD +# define SSTATUS_SD SSTATUS64_SD +# define RISCV_PGLEVEL_BITS 9 +#else +# define MSTATUS_SD MSTATUS32_SD +# define SSTATUS_SD SSTATUS32_SD +# define RISCV_PGLEVEL_BITS 10 +#endif +#define RISCV_PGSHIFT 12 +#define RISCV_PGSIZE (1 << RISCV_PGSHIFT) + +#ifndef __ASSEMBLER__ + +#ifdef __GNUC__ + +#define read_csr(reg) ({ unsigned long __tmp; \ + asm volatile ("csrr %0, " #reg : "=r"(__tmp)); \ + __tmp; }) + +#define write_csr(reg, val) \ + asm volatile ("csrw " #reg ", %0" :: "r"(val)) + +#define swap_csr(reg, val) ({ long __tmp; \ + asm volatile ("csrrw %0, " #reg ", %1" : "=r"(__tmp) : "r"(val)); \ + __tmp; }) + +#define set_csr(reg, bit) ({ unsigned long __tmp; \ + if (__builtin_constant_p(bit) && (bit) < 32) \ + asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \ + else \ + asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \ + __tmp; }) + +#define clear_csr(reg, bit) ({ unsigned long __tmp; \ + if (__builtin_constant_p(bit) && (bit) < 32) \ + asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \ + else \ + asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \ + __tmp; }) + +#define rdtime() read_csr(time) +#define rdcycle() read_csr(cycle) +#define rdinstret() read_csr(instret) + +#endif + +#endif + +#endif + +#endif +/* Automatically generated by parse-opcodes */ +#ifndef RISCV_ENCODING_H +#define RISCV_ENCODING_H +#define MATCH_BEQ 0x63 +#define MASK_BEQ 0x707f +#define MATCH_BNE 0x1063 +#define MASK_BNE 0x707f +#define MATCH_BLT 0x4063 +#define MASK_BLT 0x707f +#define MATCH_BGE 0x5063 +#define MASK_BGE 0x707f +#define MATCH_BLTU 0x6063 +#define MASK_BLTU 0x707f +#define MATCH_BGEU 0x7063 +#define MASK_BGEU 0x707f +#define MATCH_JALR 0x67 +#define MASK_JALR 0x707f +#define MATCH_JAL 0x6f +#define MASK_JAL 0x7f +#define MATCH_LUI 0x37 +#define MASK_LUI 0x7f +#define MATCH_AUIPC 0x17 +#define MASK_AUIPC 0x7f +#define MATCH_ADDI 0x13 +#define MASK_ADDI 0x707f +#define MATCH_SLLI 0x1013 +#define MASK_SLLI 0xfc00707f +#define MATCH_SLTI 0x2013 +#define MASK_SLTI 0x707f +#define MATCH_SLTIU 0x3013 +#define MASK_SLTIU 0x707f +#define MATCH_XORI 0x4013 +#define MASK_XORI 0x707f +#define MATCH_SRLI 0x5013 +#define MASK_SRLI 0xfc00707f +#define MATCH_SRAI 0x40005013 +#define MASK_SRAI 0xfc00707f +#define MATCH_ORI 0x6013 +#define MASK_ORI 0x707f +#define MATCH_ANDI 0x7013 +#define MASK_ANDI 0x707f +#define MATCH_ADD 0x33 +#define MASK_ADD 0xfe00707f +#define MATCH_SUB 0x40000033 +#define MASK_SUB 0xfe00707f +#define MATCH_SLL 0x1033 +#define MASK_SLL 0xfe00707f +#define MATCH_SLT 0x2033 +#define MASK_SLT 0xfe00707f +#define MATCH_SLTU 0x3033 +#define MASK_SLTU 0xfe00707f +#define MATCH_XOR 0x4033 +#define MASK_XOR 0xfe00707f +#define MATCH_SRL 0x5033 +#define MASK_SRL 0xfe00707f +#define MATCH_SRA 0x40005033 +#define MASK_SRA 0xfe00707f +#define MATCH_OR 0x6033 +#define MASK_OR 0xfe00707f +#define MATCH_AND 0x7033 +#define MASK_AND 0xfe00707f +#define MATCH_ADDIW 0x1b +#define MASK_ADDIW 0x707f +#define MATCH_SLLIW 0x101b +#define MASK_SLLIW 0xfe00707f +#define MATCH_SRLIW 0x501b +#define MASK_SRLIW 0xfe00707f +#define MATCH_SRAIW 0x4000501b +#define MASK_SRAIW 0xfe00707f +#define MATCH_ADDW 0x3b +#define MASK_ADDW 0xfe00707f +#define MATCH_SUBW 0x4000003b +#define MASK_SUBW 0xfe00707f +#define MATCH_SLLW 0x103b +#define MASK_SLLW 0xfe00707f +#define MATCH_SRLW 0x503b +#define MASK_SRLW 0xfe00707f +#define MATCH_SRAW 0x4000503b +#define MASK_SRAW 0xfe00707f +#define MATCH_LB 0x3 +#define MASK_LB 0x707f +#define MATCH_LH 0x1003 +#define MASK_LH 0x707f +#define MATCH_LW 0x2003 +#define MASK_LW 0x707f +#define MATCH_LD 0x3003 +#define MASK_LD 0x707f +#define MATCH_LBU 0x4003 +#define MASK_LBU 0x707f +#define MATCH_LHU 0x5003 +#define MASK_LHU 0x707f +#define MATCH_LWU 0x6003 +#define MASK_LWU 0x707f +#define MATCH_SB 0x23 +#define MASK_SB 0x707f +#define MATCH_SH 0x1023 +#define MASK_SH 0x707f +#define MATCH_SW 0x2023 +#define MASK_SW 0x707f +#define MATCH_SD 0x3023 +#define MASK_SD 0x707f +#define MATCH_FENCE 0xf +#define MASK_FENCE 0x707f +#define MATCH_FENCE_I 0x100f +#define MASK_FENCE_I 0x707f +#define MATCH_MUL 0x2000033 +#define MASK_MUL 0xfe00707f +#define MATCH_MULH 0x2001033 +#define MASK_MULH 0xfe00707f +#define MATCH_MULHSU 0x2002033 +#define MASK_MULHSU 0xfe00707f +#define MATCH_MULHU 0x2003033 +#define MASK_MULHU 0xfe00707f +#define MATCH_DIV 0x2004033 +#define MASK_DIV 0xfe00707f +#define MATCH_DIVU 0x2005033 +#define MASK_DIVU 0xfe00707f +#define MATCH_REM 0x2006033 +#define MASK_REM 0xfe00707f +#define MATCH_REMU 0x2007033 +#define MASK_REMU 0xfe00707f +#define MATCH_MULW 0x200003b +#define MASK_MULW 0xfe00707f +#define MATCH_DIVW 0x200403b +#define MASK_DIVW 0xfe00707f +#define MATCH_DIVUW 0x200503b +#define MASK_DIVUW 0xfe00707f +#define MATCH_REMW 0x200603b +#define MASK_REMW 0xfe00707f +#define MATCH_REMUW 0x200703b +#define MASK_REMUW 0xfe00707f +#define MATCH_AMOADD_W 0x202f +#define MASK_AMOADD_W 0xf800707f +#define MATCH_AMOXOR_W 0x2000202f +#define MASK_AMOXOR_W 0xf800707f +#define MATCH_AMOOR_W 0x4000202f +#define MASK_AMOOR_W 0xf800707f +#define MATCH_AMOAND_W 0x6000202f +#define MASK_AMOAND_W 0xf800707f +#define MATCH_AMOMIN_W 0x8000202f +#define MASK_AMOMIN_W 0xf800707f +#define MATCH_AMOMAX_W 0xa000202f +#define MASK_AMOMAX_W 0xf800707f +#define MATCH_AMOMINU_W 0xc000202f +#define MASK_AMOMINU_W 0xf800707f +#define MATCH_AMOMAXU_W 0xe000202f +#define MASK_AMOMAXU_W 0xf800707f +#define MATCH_AMOSWAP_W 0x800202f +#define MASK_AMOSWAP_W 0xf800707f +#define MATCH_LR_W 0x1000202f +#define MASK_LR_W 0xf9f0707f +#define MATCH_SC_W 0x1800202f +#define MASK_SC_W 0xf800707f +#define MATCH_AMOADD_D 0x302f +#define MASK_AMOADD_D 0xf800707f +#define MATCH_AMOXOR_D 0x2000302f +#define MASK_AMOXOR_D 0xf800707f +#define MATCH_AMOOR_D 0x4000302f +#define MASK_AMOOR_D 0xf800707f +#define MATCH_AMOAND_D 0x6000302f +#define MASK_AMOAND_D 0xf800707f +#define MATCH_AMOMIN_D 0x8000302f +#define MASK_AMOMIN_D 0xf800707f +#define MATCH_AMOMAX_D 0xa000302f +#define MASK_AMOMAX_D 0xf800707f +#define MATCH_AMOMINU_D 0xc000302f +#define MASK_AMOMINU_D 0xf800707f +#define MATCH_AMOMAXU_D 0xe000302f +#define MASK_AMOMAXU_D 0xf800707f +#define MATCH_AMOSWAP_D 0x800302f +#define MASK_AMOSWAP_D 0xf800707f +#define MATCH_LR_D 0x1000302f +#define MASK_LR_D 0xf9f0707f +#define MATCH_SC_D 0x1800302f +#define MASK_SC_D 0xf800707f +#define MATCH_SCALL 0x73 +#define MASK_SCALL 0xffffffff +#define MATCH_SBREAK 0x100073 +#define MASK_SBREAK 0xffffffff +#define MATCH_SRET 0x10200073 +#define MASK_SRET 0xffffffff +#define MATCH_SFENCE_VM 0x10400073 +#define MASK_SFENCE_VM 0xfff07fff +#define MATCH_WFI 0x10500073 +#define MASK_WFI 0xffffffff +#define MATCH_CSRRW 0x1073 +#define MASK_CSRRW 0x707f +#define MATCH_CSRRS 0x2073 +#define MASK_CSRRS 0x707f +#define MATCH_CSRRC 0x3073 +#define MASK_CSRRC 0x707f +#define MATCH_CSRRWI 0x5073 +#define MASK_CSRRWI 0x707f +#define MATCH_CSRRSI 0x6073 +#define MASK_CSRRSI 0x707f +#define MATCH_CSRRCI 0x7073 +#define MASK_CSRRCI 0x707f +#define MATCH_FADD_S 0x53 +#define MASK_FADD_S 0xfe00007f +#define MATCH_FSUB_S 0x8000053 +#define MASK_FSUB_S 0xfe00007f +#define MATCH_FMUL_S 0x10000053 +#define MASK_FMUL_S 0xfe00007f +#define MATCH_FDIV_S 0x18000053 +#define MASK_FDIV_S 0xfe00007f +#define MATCH_FSGNJ_S 0x20000053 +#define MASK_FSGNJ_S 0xfe00707f +#define MATCH_FSGNJN_S 0x20001053 +#define MASK_FSGNJN_S 0xfe00707f +#define MATCH_FSGNJX_S 0x20002053 +#define MASK_FSGNJX_S 0xfe00707f +#define MATCH_FMIN_S 0x28000053 +#define MASK_FMIN_S 0xfe00707f +#define MATCH_FMAX_S 0x28001053 +#define MASK_FMAX_S 0xfe00707f +#define MATCH_FSQRT_S 0x58000053 +#define MASK_FSQRT_S 0xfff0007f +#define MATCH_FADD_D 0x2000053 +#define MASK_FADD_D 0xfe00007f +#define MATCH_FSUB_D 0xa000053 +#define MASK_FSUB_D 0xfe00007f +#define MATCH_FMUL_D 0x12000053 +#define MASK_FMUL_D 0xfe00007f +#define MATCH_FDIV_D 0x1a000053 +#define MASK_FDIV_D 0xfe00007f +#define MATCH_FSGNJ_D 0x22000053 +#define MASK_FSGNJ_D 0xfe00707f +#define MATCH_FSGNJN_D 0x22001053 +#define MASK_FSGNJN_D 0xfe00707f +#define MATCH_FSGNJX_D 0x22002053 +#define MASK_FSGNJX_D 0xfe00707f +#define MATCH_FMIN_D 0x2a000053 +#define MASK_FMIN_D 0xfe00707f +#define MATCH_FMAX_D 0x2a001053 +#define MASK_FMAX_D 0xfe00707f +#define MATCH_FCVT_S_D 0x40100053 +#define MASK_FCVT_S_D 0xfff0007f +#define MATCH_FCVT_D_S 0x42000053 +#define MASK_FCVT_D_S 0xfff0007f +#define MATCH_FSQRT_D 0x5a000053 +#define MASK_FSQRT_D 0xfff0007f +#define MATCH_FLE_S 0xa0000053 +#define MASK_FLE_S 0xfe00707f +#define MATCH_FLT_S 0xa0001053 +#define MASK_FLT_S 0xfe00707f +#define MATCH_FEQ_S 0xa0002053 +#define MASK_FEQ_S 0xfe00707f +#define MATCH_FLE_D 0xa2000053 +#define MASK_FLE_D 0xfe00707f +#define MATCH_FLT_D 0xa2001053 +#define MASK_FLT_D 0xfe00707f +#define MATCH_FEQ_D 0xa2002053 +#define MASK_FEQ_D 0xfe00707f +#define MATCH_FCVT_W_S 0xc0000053 +#define MASK_FCVT_W_S 0xfff0007f +#define MATCH_FCVT_WU_S 0xc0100053 +#define MASK_FCVT_WU_S 0xfff0007f +#define MATCH_FCVT_L_S 0xc0200053 +#define MASK_FCVT_L_S 0xfff0007f +#define MATCH_FCVT_LU_S 0xc0300053 +#define MASK_FCVT_LU_S 0xfff0007f +#define MATCH_FMV_X_S 0xe0000053 +#define MASK_FMV_X_S 0xfff0707f +#define MATCH_FCLASS_S 0xe0001053 +#define MASK_FCLASS_S 0xfff0707f +#define MATCH_FCVT_W_D 0xc2000053 +#define MASK_FCVT_W_D 0xfff0007f +#define MATCH_FCVT_WU_D 0xc2100053 +#define MASK_FCVT_WU_D 0xfff0007f +#define MATCH_FCVT_L_D 0xc2200053 +#define MASK_FCVT_L_D 0xfff0007f +#define MATCH_FCVT_LU_D 0xc2300053 +#define MASK_FCVT_LU_D 0xfff0007f +#define MATCH_FMV_X_D 0xe2000053 +#define MASK_FMV_X_D 0xfff0707f +#define MATCH_FCLASS_D 0xe2001053 +#define MASK_FCLASS_D 0xfff0707f +#define MATCH_FCVT_S_W 0xd0000053 +#define MASK_FCVT_S_W 0xfff0007f +#define MATCH_FCVT_S_WU 0xd0100053 +#define MASK_FCVT_S_WU 0xfff0007f +#define MATCH_FCVT_S_L 0xd0200053 +#define MASK_FCVT_S_L 0xfff0007f +#define MATCH_FCVT_S_LU 0xd0300053 +#define MASK_FCVT_S_LU 0xfff0007f +#define MATCH_FMV_S_X 0xf0000053 +#define MASK_FMV_S_X 0xfff0707f +#define MATCH_FCVT_D_W 0xd2000053 +#define MASK_FCVT_D_W 0xfff0007f +#define MATCH_FCVT_D_WU 0xd2100053 +#define MASK_FCVT_D_WU 0xfff0007f +#define MATCH_FCVT_D_L 0xd2200053 +#define MASK_FCVT_D_L 0xfff0007f +#define MATCH_FCVT_D_LU 0xd2300053 +#define MASK_FCVT_D_LU 0xfff0007f +#define MATCH_FMV_D_X 0xf2000053 +#define MASK_FMV_D_X 0xfff0707f +#define MATCH_FLW 0x2007 +#define MASK_FLW 0x707f +#define MATCH_FLD 0x3007 +#define MASK_FLD 0x707f +#define MATCH_FSW 0x2027 +#define MASK_FSW 0x707f +#define MATCH_FSD 0x3027 +#define MASK_FSD 0x707f +#define MATCH_FMADD_S 0x43 +#define MASK_FMADD_S 0x600007f +#define MATCH_FMSUB_S 0x47 +#define MASK_FMSUB_S 0x600007f +#define MATCH_FNMSUB_S 0x4b +#define MASK_FNMSUB_S 0x600007f +#define MATCH_FNMADD_S 0x4f +#define MASK_FNMADD_S 0x600007f +#define MATCH_FMADD_D 0x2000043 +#define MASK_FMADD_D 0x600007f +#define MATCH_FMSUB_D 0x2000047 +#define MASK_FMSUB_D 0x600007f +#define MATCH_FNMSUB_D 0x200004b +#define MASK_FNMSUB_D 0x600007f +#define MATCH_FNMADD_D 0x200004f +#define MASK_FNMADD_D 0x600007f +#define MATCH_C_NOP 0x1 +#define MASK_C_NOP 0xffff +#define MATCH_C_ADDI16SP 0x6101 +#define MASK_C_ADDI16SP 0xef83 +#define MATCH_C_JR 0x8002 +#define MASK_C_JR 0xf07f +#define MATCH_C_JALR 0x9002 +#define MASK_C_JALR 0xf07f +#define MATCH_C_EBREAK 0x9002 +#define MASK_C_EBREAK 0xffff +#define MATCH_C_LD 0x6000 +#define MASK_C_LD 0xe003 +#define MATCH_C_SD 0xe000 +#define MASK_C_SD 0xe003 +#define MATCH_C_ADDIW 0x2001 +#define MASK_C_ADDIW 0xe003 +#define MATCH_C_LDSP 0x6002 +#define MASK_C_LDSP 0xe003 +#define MATCH_C_SDSP 0xe002 +#define MASK_C_SDSP 0xe003 +#define MATCH_C_ADDI4SPN 0x0 +#define MASK_C_ADDI4SPN 0xe003 +#define MATCH_C_FLD 0x2000 +#define MASK_C_FLD 0xe003 +#define MATCH_C_LW 0x4000 +#define MASK_C_LW 0xe003 +#define MATCH_C_FLW 0x6000 +#define MASK_C_FLW 0xe003 +#define MATCH_C_FSD 0xa000 +#define MASK_C_FSD 0xe003 +#define MATCH_C_SW 0xc000 +#define MASK_C_SW 0xe003 +#define MATCH_C_FSW 0xe000 +#define MASK_C_FSW 0xe003 +#define MATCH_C_ADDI 0x1 +#define MASK_C_ADDI 0xe003 +#define MATCH_C_JAL 0x2001 +#define MASK_C_JAL 0xe003 +#define MATCH_C_LI 0x4001 +#define MASK_C_LI 0xe003 +#define MATCH_C_LUI 0x6001 +#define MASK_C_LUI 0xe003 +#define MATCH_C_SRLI 0x8001 +#define MASK_C_SRLI 0xec03 +#define MATCH_C_SRAI 0x8401 +#define MASK_C_SRAI 0xec03 +#define MATCH_C_ANDI 0x8801 +#define MASK_C_ANDI 0xec03 +#define MATCH_C_SUB 0x8c01 +#define MASK_C_SUB 0xfc63 +#define MATCH_C_XOR 0x8c21 +#define MASK_C_XOR 0xfc63 +#define MATCH_C_OR 0x8c41 +#define MASK_C_OR 0xfc63 +#define MATCH_C_AND 0x8c61 +#define MASK_C_AND 0xfc63 +#define MATCH_C_SUBW 0x9c01 +#define MASK_C_SUBW 0xfc63 +#define MATCH_C_ADDW 0x9c21 +#define MASK_C_ADDW 0xfc63 +#define MATCH_C_J 0xa001 +#define MASK_C_J 0xe003 +#define MATCH_C_BEQZ 0xc001 +#define MASK_C_BEQZ 0xe003 +#define MATCH_C_BNEZ 0xe001 +#define MASK_C_BNEZ 0xe003 +#define MATCH_C_SLLI 0x2 +#define MASK_C_SLLI 0xe003 +#define MATCH_C_FLDSP 0x2002 +#define MASK_C_FLDSP 0xe003 +#define MATCH_C_LWSP 0x4002 +#define MASK_C_LWSP 0xe003 +#define MATCH_C_FLWSP 0x6002 +#define MASK_C_FLWSP 0xe003 +#define MATCH_C_MV 0x8002 +#define MASK_C_MV 0xf003 +#define MATCH_C_ADD 0x9002 +#define MASK_C_ADD 0xf003 +#define MATCH_C_FSDSP 0xa002 +#define MASK_C_FSDSP 0xe003 +#define MATCH_C_SWSP 0xc002 +#define MASK_C_SWSP 0xe003 +#define MATCH_C_FSWSP 0xe002 +#define MASK_C_FSWSP 0xe003 +#define MATCH_CUSTOM0 0xb +#define MASK_CUSTOM0 0x707f +#define MATCH_CUSTOM0_RS1 0x200b +#define MASK_CUSTOM0_RS1 0x707f +#define MATCH_CUSTOM0_RS1_RS2 0x300b +#define MASK_CUSTOM0_RS1_RS2 0x707f +#define MATCH_CUSTOM0_RD 0x400b +#define MASK_CUSTOM0_RD 0x707f +#define MATCH_CUSTOM0_RD_RS1 0x600b +#define MASK_CUSTOM0_RD_RS1 0x707f +#define MATCH_CUSTOM0_RD_RS1_RS2 0x700b +#define MASK_CUSTOM0_RD_RS1_RS2 0x707f +#define MATCH_CUSTOM1 0x2b +#define MASK_CUSTOM1 0x707f +#define MATCH_CUSTOM1_RS1 0x202b +#define MASK_CUSTOM1_RS1 0x707f +#define MATCH_CUSTOM1_RS1_RS2 0x302b +#define MASK_CUSTOM1_RS1_RS2 0x707f +#define MATCH_CUSTOM1_RD 0x402b +#define MASK_CUSTOM1_RD 0x707f +#define MATCH_CUSTOM1_RD_RS1 0x602b +#define MASK_CUSTOM1_RD_RS1 0x707f +#define MATCH_CUSTOM1_RD_RS1_RS2 0x702b +#define MASK_CUSTOM1_RD_RS1_RS2 0x707f +#define MATCH_CUSTOM2 0x5b +#define MASK_CUSTOM2 0x707f +#define MATCH_CUSTOM2_RS1 0x205b +#define MASK_CUSTOM2_RS1 0x707f +#define MATCH_CUSTOM2_RS1_RS2 0x305b +#define MASK_CUSTOM2_RS1_RS2 0x707f +#define MATCH_CUSTOM2_RD 0x405b +#define MASK_CUSTOM2_RD 0x707f +#define MATCH_CUSTOM2_RD_RS1 0x605b +#define MASK_CUSTOM2_RD_RS1 0x707f +#define MATCH_CUSTOM2_RD_RS1_RS2 0x705b +#define MASK_CUSTOM2_RD_RS1_RS2 0x707f +#define MATCH_CUSTOM3 0x7b +#define MASK_CUSTOM3 0x707f +#define MATCH_CUSTOM3_RS1 0x207b +#define MASK_CUSTOM3_RS1 0x707f +#define MATCH_CUSTOM3_RS1_RS2 0x307b +#define MASK_CUSTOM3_RS1_RS2 0x707f +#define MATCH_CUSTOM3_RD 0x407b +#define MASK_CUSTOM3_RD 0x707f +#define MATCH_CUSTOM3_RD_RS1 0x607b +#define MASK_CUSTOM3_RD_RS1 0x707f +#define MATCH_CUSTOM3_RD_RS1_RS2 0x707b +#define MASK_CUSTOM3_RD_RS1_RS2 0x707f +#define CSR_FFLAGS 0x1 +#define CSR_FRM 0x2 +#define CSR_FCSR 0x3 +#define CSR_CYCLE 0xc00 +#define CSR_TIME 0xc01 +#define CSR_INSTRET 0xc02 +#define CSR_STATS 0xc0 +#define CSR_UARCH0 0xcc0 +#define CSR_UARCH1 0xcc1 +#define CSR_UARCH2 0xcc2 +#define CSR_UARCH3 0xcc3 +#define CSR_UARCH4 0xcc4 +#define CSR_UARCH5 0xcc5 +#define CSR_UARCH6 0xcc6 +#define CSR_UARCH7 0xcc7 +#define CSR_UARCH8 0xcc8 +#define CSR_UARCH9 0xcc9 +#define CSR_UARCH10 0xcca +#define CSR_UARCH11 0xccb +#define CSR_UARCH12 0xccc +#define CSR_UARCH13 0xccd +#define CSR_UARCH14 0xcce +#define CSR_UARCH15 0xccf +#define CSR_SSTATUS 0x100 +#define CSR_SIE 0x104 +#define CSR_STVEC 0x105 +#define CSR_SSCRATCH 0x140 +#define CSR_SEPC 0x141 +#define CSR_SCAUSE 0x142 +#define CSR_SBADADDR 0x143 +#define CSR_SIP 0x144 +#define CSR_SPTBR 0x180 +#define CSR_SASID 0x181 +#define CSR_SCYCLE 0xd00 +#define CSR_STIME 0xd01 +#define CSR_SINSTRET 0xd02 +#define CSR_MSTATUS 0x300 +#define CSR_MEDELEG 0x302 +#define CSR_MIDELEG 0x303 +#define CSR_MIE 0x304 +#define CSR_MTVEC 0x305 +#define CSR_MTIMECMP 0x321 +#define CSR_MSCRATCH 0x340 +#define CSR_MEPC 0x341 +#define CSR_MCAUSE 0x342 +#define CSR_MBADADDR 0x343 +#define CSR_MIP 0x344 +#define CSR_MIPI 0x345 +#define CSR_MUCOUNTEREN 0x310 +#define CSR_MSCOUNTEREN 0x311 +#define CSR_MUCYCLE_DELTA 0x700 +#define CSR_MUTIME_DELTA 0x701 +#define CSR_MUINSTRET_DELTA 0x702 +#define CSR_MSCYCLE_DELTA 0x704 +#define CSR_MSTIME_DELTA 0x705 +#define CSR_MSINSTRET_DELTA 0x706 +#define CSR_MCYCLE 0xf00 +#define CSR_MTIME 0xf01 +#define CSR_MINSTRET 0xf02 +#define CSR_MISA 0xf10 +#define CSR_MVENDORID 0xf11 +#define CSR_MARCHID 0xf12 +#define CSR_MIMPID 0xf13 +#define CSR_MCFGADDR 0xf14 +#define CSR_MHARTID 0xf15 +#define CSR_MTOHOST 0x7c0 +#define CSR_MFROMHOST 0x7c1 +#define CSR_MRESET 0x7c2 +#define CSR_CYCLEH 0xc80 +#define CSR_TIMEH 0xc81 +#define CSR_INSTRETH 0xc82 +#define CSR_MTIMECMPH 0x361 +#define CSR_MUCYCLE_DELTAH 0x780 +#define CSR_MUTIME_DELTAH 0x781 +#define CSR_MUINSTRET_DELTAH 0x782 +#define CSR_MSCYCLE_DELTAH 0x784 +#define CSR_MSTIME_DELTAH 0x785 +#define CSR_MSINSTRET_DELTAH 0x786 +#define CSR_MCYCLEH 0xf80 +#define CSR_MTIMEH 0xf81 +#define CSR_MINSTRETH 0xf82 +#define CAUSE_MISALIGNED_FETCH 0x0 +#define CAUSE_FAULT_FETCH 0x1 +#define CAUSE_ILLEGAL_INSTRUCTION 0x2 +#define CAUSE_BREAKPOINT 0x3 +#define CAUSE_MISALIGNED_LOAD 0x4 +#define CAUSE_FAULT_LOAD 0x5 +#define CAUSE_MISALIGNED_STORE 0x6 +#define CAUSE_FAULT_STORE 0x7 +#define CAUSE_USER_ECALL 0x8 +#define CAUSE_SUPERVISOR_ECALL 0x9 +#define CAUSE_HYPERVISOR_ECALL 0xa +#define CAUSE_MACHINE_ECALL 0xb +#endif +#ifdef DECLARE_INSN +DECLARE_INSN(beq, MATCH_BEQ, MASK_BEQ) +DECLARE_INSN(bne, MATCH_BNE, MASK_BNE) +DECLARE_INSN(blt, MATCH_BLT, MASK_BLT) +DECLARE_INSN(bge, MATCH_BGE, MASK_BGE) +DECLARE_INSN(bltu, MATCH_BLTU, MASK_BLTU) +DECLARE_INSN(bgeu, MATCH_BGEU, MASK_BGEU) +DECLARE_INSN(jalr, MATCH_JALR, MASK_JALR) +DECLARE_INSN(jal, MATCH_JAL, MASK_JAL) +DECLARE_INSN(lui, MATCH_LUI, MASK_LUI) +DECLARE_INSN(auipc, MATCH_AUIPC, MASK_AUIPC) +DECLARE_INSN(addi, MATCH_ADDI, MASK_ADDI) +DECLARE_INSN(slli, MATCH_SLLI, MASK_SLLI) +DECLARE_INSN(slti, MATCH_SLTI, MASK_SLTI) +DECLARE_INSN(sltiu, MATCH_SLTIU, MASK_SLTIU) +DECLARE_INSN(xori, MATCH_XORI, MASK_XORI) +DECLARE_INSN(srli, MATCH_SRLI, MASK_SRLI) +DECLARE_INSN(srai, MATCH_SRAI, MASK_SRAI) +DECLARE_INSN(ori, MATCH_ORI, MASK_ORI) +DECLARE_INSN(andi, MATCH_ANDI, MASK_ANDI) +DECLARE_INSN(add, MATCH_ADD, MASK_ADD) +DECLARE_INSN(sub, MATCH_SUB, MASK_SUB) +DECLARE_INSN(sll, MATCH_SLL, MASK_SLL) +DECLARE_INSN(slt, MATCH_SLT, MASK_SLT) +DECLARE_INSN(sltu, MATCH_SLTU, MASK_SLTU) +DECLARE_INSN(xor, MATCH_XOR, MASK_XOR) +DECLARE_INSN(srl, MATCH_SRL, MASK_SRL) +DECLARE_INSN(sra, MATCH_SRA, MASK_SRA) +DECLARE_INSN(or, MATCH_OR, MASK_OR) +DECLARE_INSN(and, MATCH_AND, MASK_AND) +DECLARE_INSN(addiw, MATCH_ADDIW, MASK_ADDIW) +DECLARE_INSN(slliw, MATCH_SLLIW, MASK_SLLIW) +DECLARE_INSN(srliw, MATCH_SRLIW, MASK_SRLIW) +DECLARE_INSN(sraiw, MATCH_SRAIW, MASK_SRAIW) +DECLARE_INSN(addw, MATCH_ADDW, MASK_ADDW) +DECLARE_INSN(subw, MATCH_SUBW, MASK_SUBW) +DECLARE_INSN(sllw, MATCH_SLLW, MASK_SLLW) +DECLARE_INSN(srlw, MATCH_SRLW, MASK_SRLW) +DECLARE_INSN(sraw, MATCH_SRAW, MASK_SRAW) +DECLARE_INSN(lb, MATCH_LB, MASK_LB) +DECLARE_INSN(lh, MATCH_LH, MASK_LH) +DECLARE_INSN(lw, MATCH_LW, MASK_LW) +DECLARE_INSN(ld, MATCH_LD, MASK_LD) +DECLARE_INSN(lbu, MATCH_LBU, MASK_LBU) +DECLARE_INSN(lhu, MATCH_LHU, MASK_LHU) +DECLARE_INSN(lwu, MATCH_LWU, MASK_LWU) +DECLARE_INSN(sb, MATCH_SB, MASK_SB) +DECLARE_INSN(sh, MATCH_SH, MASK_SH) +DECLARE_INSN(sw, MATCH_SW, MASK_SW) +DECLARE_INSN(sd, MATCH_SD, MASK_SD) +DECLARE_INSN(fence, MATCH_FENCE, MASK_FENCE) +DECLARE_INSN(fence_i, MATCH_FENCE_I, MASK_FENCE_I) +DECLARE_INSN(mul, MATCH_MUL, MASK_MUL) +DECLARE_INSN(mulh, MATCH_MULH, MASK_MULH) +DECLARE_INSN(mulhsu, MATCH_MULHSU, MASK_MULHSU) +DECLARE_INSN(mulhu, MATCH_MULHU, MASK_MULHU) +DECLARE_INSN(div, MATCH_DIV, MASK_DIV) +DECLARE_INSN(divu, MATCH_DIVU, MASK_DIVU) +DECLARE_INSN(rem, MATCH_REM, MASK_REM) +DECLARE_INSN(remu, MATCH_REMU, MASK_REMU) +DECLARE_INSN(mulw, MATCH_MULW, MASK_MULW) +DECLARE_INSN(divw, MATCH_DIVW, MASK_DIVW) +DECLARE_INSN(divuw, MATCH_DIVUW, MASK_DIVUW) +DECLARE_INSN(remw, MATCH_REMW, MASK_REMW) +DECLARE_INSN(remuw, MATCH_REMUW, MASK_REMUW) +DECLARE_INSN(amoadd_w, MATCH_AMOADD_W, MASK_AMOADD_W) +DECLARE_INSN(amoxor_w, MATCH_AMOXOR_W, MASK_AMOXOR_W) +DECLARE_INSN(amoor_w, MATCH_AMOOR_W, MASK_AMOOR_W) +DECLARE_INSN(amoand_w, MATCH_AMOAND_W, MASK_AMOAND_W) +DECLARE_INSN(amomin_w, MATCH_AMOMIN_W, MASK_AMOMIN_W) +DECLARE_INSN(amomax_w, MATCH_AMOMAX_W, MASK_AMOMAX_W) +DECLARE_INSN(amominu_w, MATCH_AMOMINU_W, MASK_AMOMINU_W) +DECLARE_INSN(amomaxu_w, MATCH_AMOMAXU_W, MASK_AMOMAXU_W) +DECLARE_INSN(amoswap_w, MATCH_AMOSWAP_W, MASK_AMOSWAP_W) +DECLARE_INSN(lr_w, MATCH_LR_W, MASK_LR_W) +DECLARE_INSN(sc_w, MATCH_SC_W, MASK_SC_W) +DECLARE_INSN(amoadd_d, MATCH_AMOADD_D, MASK_AMOADD_D) +DECLARE_INSN(amoxor_d, MATCH_AMOXOR_D, MASK_AMOXOR_D) +DECLARE_INSN(amoor_d, MATCH_AMOOR_D, MASK_AMOOR_D) +DECLARE_INSN(amoand_d, MATCH_AMOAND_D, MASK_AMOAND_D) +DECLARE_INSN(amomin_d, MATCH_AMOMIN_D, MASK_AMOMIN_D) +DECLARE_INSN(amomax_d, MATCH_AMOMAX_D, MASK_AMOMAX_D) +DECLARE_INSN(amominu_d, MATCH_AMOMINU_D, MASK_AMOMINU_D) +DECLARE_INSN(amomaxu_d, MATCH_AMOMAXU_D, MASK_AMOMAXU_D) +DECLARE_INSN(amoswap_d, MATCH_AMOSWAP_D, MASK_AMOSWAP_D) +DECLARE_INSN(lr_d, MATCH_LR_D, MASK_LR_D) +DECLARE_INSN(sc_d, MATCH_SC_D, MASK_SC_D) +DECLARE_INSN(scall, MATCH_SCALL, MASK_SCALL) +DECLARE_INSN(sbreak, MATCH_SBREAK, MASK_SBREAK) +DECLARE_INSN(sret, MATCH_SRET, MASK_SRET) +DECLARE_INSN(sfence_vm, MATCH_SFENCE_VM, MASK_SFENCE_VM) +DECLARE_INSN(wfi, MATCH_WFI, MASK_WFI) +DECLARE_INSN(csrrw, MATCH_CSRRW, MASK_CSRRW) +DECLARE_INSN(csrrs, MATCH_CSRRS, MASK_CSRRS) +DECLARE_INSN(csrrc, MATCH_CSRRC, MASK_CSRRC) +DECLARE_INSN(csrrwi, MATCH_CSRRWI, MASK_CSRRWI) +DECLARE_INSN(csrrsi, MATCH_CSRRSI, MASK_CSRRSI) +DECLARE_INSN(csrrci, MATCH_CSRRCI, MASK_CSRRCI) +DECLARE_INSN(fadd_s, MATCH_FADD_S, MASK_FADD_S) +DECLARE_INSN(fsub_s, MATCH_FSUB_S, MASK_FSUB_S) +DECLARE_INSN(fmul_s, MATCH_FMUL_S, MASK_FMUL_S) +DECLARE_INSN(fdiv_s, MATCH_FDIV_S, MASK_FDIV_S) +DECLARE_INSN(fsgnj_s, MATCH_FSGNJ_S, MASK_FSGNJ_S) +DECLARE_INSN(fsgnjn_s, MATCH_FSGNJN_S, MASK_FSGNJN_S) +DECLARE_INSN(fsgnjx_s, MATCH_FSGNJX_S, MASK_FSGNJX_S) +DECLARE_INSN(fmin_s, MATCH_FMIN_S, MASK_FMIN_S) +DECLARE_INSN(fmax_s, MATCH_FMAX_S, MASK_FMAX_S) +DECLARE_INSN(fsqrt_s, MATCH_FSQRT_S, MASK_FSQRT_S) +DECLARE_INSN(fadd_d, MATCH_FADD_D, MASK_FADD_D) +DECLARE_INSN(fsub_d, MATCH_FSUB_D, MASK_FSUB_D) +DECLARE_INSN(fmul_d, MATCH_FMUL_D, MASK_FMUL_D) +DECLARE_INSN(fdiv_d, MATCH_FDIV_D, MASK_FDIV_D) +DECLARE_INSN(fsgnj_d, MATCH_FSGNJ_D, MASK_FSGNJ_D) +DECLARE_INSN(fsgnjn_d, MATCH_FSGNJN_D, MASK_FSGNJN_D) +DECLARE_INSN(fsgnjx_d, MATCH_FSGNJX_D, MASK_FSGNJX_D) +DECLARE_INSN(fmin_d, MATCH_FMIN_D, MASK_FMIN_D) +DECLARE_INSN(fmax_d, MATCH_FMAX_D, MASK_FMAX_D) +DECLARE_INSN(fcvt_s_d, MATCH_FCVT_S_D, MASK_FCVT_S_D) +DECLARE_INSN(fcvt_d_s, MATCH_FCVT_D_S, MASK_FCVT_D_S) +DECLARE_INSN(fsqrt_d, MATCH_FSQRT_D, MASK_FSQRT_D) +DECLARE_INSN(fle_s, MATCH_FLE_S, MASK_FLE_S) +DECLARE_INSN(flt_s, MATCH_FLT_S, MASK_FLT_S) +DECLARE_INSN(feq_s, MATCH_FEQ_S, MASK_FEQ_S) +DECLARE_INSN(fle_d, MATCH_FLE_D, MASK_FLE_D) +DECLARE_INSN(flt_d, MATCH_FLT_D, MASK_FLT_D) +DECLARE_INSN(feq_d, MATCH_FEQ_D, MASK_FEQ_D) +DECLARE_INSN(fcvt_w_s, MATCH_FCVT_W_S, MASK_FCVT_W_S) +DECLARE_INSN(fcvt_wu_s, MATCH_FCVT_WU_S, MASK_FCVT_WU_S) +DECLARE_INSN(fcvt_l_s, MATCH_FCVT_L_S, MASK_FCVT_L_S) +DECLARE_INSN(fcvt_lu_s, MATCH_FCVT_LU_S, MASK_FCVT_LU_S) +DECLARE_INSN(fmv_x_s, MATCH_FMV_X_S, MASK_FMV_X_S) +DECLARE_INSN(fclass_s, MATCH_FCLASS_S, MASK_FCLASS_S) +DECLARE_INSN(fcvt_w_d, MATCH_FCVT_W_D, MASK_FCVT_W_D) +DECLARE_INSN(fcvt_wu_d, MATCH_FCVT_WU_D, MASK_FCVT_WU_D) +DECLARE_INSN(fcvt_l_d, MATCH_FCVT_L_D, MASK_FCVT_L_D) +DECLARE_INSN(fcvt_lu_d, MATCH_FCVT_LU_D, MASK_FCVT_LU_D) +DECLARE_INSN(fmv_x_d, MATCH_FMV_X_D, MASK_FMV_X_D) +DECLARE_INSN(fclass_d, MATCH_FCLASS_D, MASK_FCLASS_D) +DECLARE_INSN(fcvt_s_w, MATCH_FCVT_S_W, MASK_FCVT_S_W) +DECLARE_INSN(fcvt_s_wu, MATCH_FCVT_S_WU, MASK_FCVT_S_WU) +DECLARE_INSN(fcvt_s_l, MATCH_FCVT_S_L, MASK_FCVT_S_L) +DECLARE_INSN(fcvt_s_lu, MATCH_FCVT_S_LU, MASK_FCVT_S_LU) +DECLARE_INSN(fmv_s_x, MATCH_FMV_S_X, MASK_FMV_S_X) +DECLARE_INSN(fcvt_d_w, MATCH_FCVT_D_W, MASK_FCVT_D_W) +DECLARE_INSN(fcvt_d_wu, MATCH_FCVT_D_WU, MASK_FCVT_D_WU) +DECLARE_INSN(fcvt_d_l, MATCH_FCVT_D_L, MASK_FCVT_D_L) +DECLARE_INSN(fcvt_d_lu, MATCH_FCVT_D_LU, MASK_FCVT_D_LU) +DECLARE_INSN(fmv_d_x, MATCH_FMV_D_X, MASK_FMV_D_X) +DECLARE_INSN(flw, MATCH_FLW, MASK_FLW) +DECLARE_INSN(fld, MATCH_FLD, MASK_FLD) +DECLARE_INSN(fsw, MATCH_FSW, MASK_FSW) +DECLARE_INSN(fsd, MATCH_FSD, MASK_FSD) +DECLARE_INSN(fmadd_s, MATCH_FMADD_S, MASK_FMADD_S) +DECLARE_INSN(fmsub_s, MATCH_FMSUB_S, MASK_FMSUB_S) +DECLARE_INSN(fnmsub_s, MATCH_FNMSUB_S, MASK_FNMSUB_S) +DECLARE_INSN(fnmadd_s, MATCH_FNMADD_S, MASK_FNMADD_S) +DECLARE_INSN(fmadd_d, MATCH_FMADD_D, MASK_FMADD_D) +DECLARE_INSN(fmsub_d, MATCH_FMSUB_D, MASK_FMSUB_D) +DECLARE_INSN(fnmsub_d, MATCH_FNMSUB_D, MASK_FNMSUB_D) +DECLARE_INSN(fnmadd_d, MATCH_FNMADD_D, MASK_FNMADD_D) +DECLARE_INSN(c_nop, MATCH_C_NOP, MASK_C_NOP) +DECLARE_INSN(c_addi16sp, MATCH_C_ADDI16SP, MASK_C_ADDI16SP) +DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR) +DECLARE_INSN(c_jalr, MATCH_C_JALR, MASK_C_JALR) +DECLARE_INSN(c_ebreak, MATCH_C_EBREAK, MASK_C_EBREAK) +DECLARE_INSN(c_ld, MATCH_C_LD, MASK_C_LD) +DECLARE_INSN(c_sd, MATCH_C_SD, MASK_C_SD) +DECLARE_INSN(c_addiw, MATCH_C_ADDIW, MASK_C_ADDIW) +DECLARE_INSN(c_ldsp, MATCH_C_LDSP, MASK_C_LDSP) +DECLARE_INSN(c_sdsp, MATCH_C_SDSP, MASK_C_SDSP) +DECLARE_INSN(c_addi4spn, MATCH_C_ADDI4SPN, MASK_C_ADDI4SPN) +DECLARE_INSN(c_fld, MATCH_C_FLD, MASK_C_FLD) +DECLARE_INSN(c_lw, MATCH_C_LW, MASK_C_LW) +DECLARE_INSN(c_flw, MATCH_C_FLW, MASK_C_FLW) +DECLARE_INSN(c_fsd, MATCH_C_FSD, MASK_C_FSD) +DECLARE_INSN(c_sw, MATCH_C_SW, MASK_C_SW) +DECLARE_INSN(c_fsw, MATCH_C_FSW, MASK_C_FSW) +DECLARE_INSN(c_addi, MATCH_C_ADDI, MASK_C_ADDI) +DECLARE_INSN(c_jal, MATCH_C_JAL, MASK_C_JAL) +DECLARE_INSN(c_li, MATCH_C_LI, MASK_C_LI) +DECLARE_INSN(c_lui, MATCH_C_LUI, MASK_C_LUI) +DECLARE_INSN(c_srli, MATCH_C_SRLI, MASK_C_SRLI) +DECLARE_INSN(c_srai, MATCH_C_SRAI, MASK_C_SRAI) +DECLARE_INSN(c_andi, MATCH_C_ANDI, MASK_C_ANDI) +DECLARE_INSN(c_sub, MATCH_C_SUB, MASK_C_SUB) +DECLARE_INSN(c_xor, MATCH_C_XOR, MASK_C_XOR) +DECLARE_INSN(c_or, MATCH_C_OR, MASK_C_OR) +DECLARE_INSN(c_and, MATCH_C_AND, MASK_C_AND) +DECLARE_INSN(c_subw, MATCH_C_SUBW, MASK_C_SUBW) +DECLARE_INSN(c_addw, MATCH_C_ADDW, MASK_C_ADDW) +DECLARE_INSN(c_j, MATCH_C_J, MASK_C_J) +DECLARE_INSN(c_beqz, MATCH_C_BEQZ, MASK_C_BEQZ) +DECLARE_INSN(c_bnez, MATCH_C_BNEZ, MASK_C_BNEZ) +DECLARE_INSN(c_slli, MATCH_C_SLLI, MASK_C_SLLI) +DECLARE_INSN(c_fldsp, MATCH_C_FLDSP, MASK_C_FLDSP) +DECLARE_INSN(c_lwsp, MATCH_C_LWSP, MASK_C_LWSP) +DECLARE_INSN(c_flwsp, MATCH_C_FLWSP, MASK_C_FLWSP) +DECLARE_INSN(c_mv, MATCH_C_MV, MASK_C_MV) +DECLARE_INSN(c_add, MATCH_C_ADD, MASK_C_ADD) +DECLARE_INSN(c_fsdsp, MATCH_C_FSDSP, MASK_C_FSDSP) +DECLARE_INSN(c_swsp, MATCH_C_SWSP, MASK_C_SWSP) +DECLARE_INSN(c_fswsp, MATCH_C_FSWSP, MASK_C_FSWSP) +DECLARE_INSN(custom0, MATCH_CUSTOM0, MASK_CUSTOM0) +DECLARE_INSN(custom0_rs1, MATCH_CUSTOM0_RS1, MASK_CUSTOM0_RS1) +DECLARE_INSN(custom0_rs1_rs2, MATCH_CUSTOM0_RS1_RS2, MASK_CUSTOM0_RS1_RS2) +DECLARE_INSN(custom0_rd, MATCH_CUSTOM0_RD, MASK_CUSTOM0_RD) +DECLARE_INSN(custom0_rd_rs1, MATCH_CUSTOM0_RD_RS1, MASK_CUSTOM0_RD_RS1) +DECLARE_INSN(custom0_rd_rs1_rs2, MATCH_CUSTOM0_RD_RS1_RS2, MASK_CUSTOM0_RD_RS1_RS2) +DECLARE_INSN(custom1, MATCH_CUSTOM1, MASK_CUSTOM1) +DECLARE_INSN(custom1_rs1, MATCH_CUSTOM1_RS1, MASK_CUSTOM1_RS1) +DECLARE_INSN(custom1_rs1_rs2, MATCH_CUSTOM1_RS1_RS2, MASK_CUSTOM1_RS1_RS2) +DECLARE_INSN(custom1_rd, MATCH_CUSTOM1_RD, MASK_CUSTOM1_RD) +DECLARE_INSN(custom1_rd_rs1, MATCH_CUSTOM1_RD_RS1, MASK_CUSTOM1_RD_RS1) +DECLARE_INSN(custom1_rd_rs1_rs2, MATCH_CUSTOM1_RD_RS1_RS2, MASK_CUSTOM1_RD_RS1_RS2) +DECLARE_INSN(custom2, MATCH_CUSTOM2, MASK_CUSTOM2) +DECLARE_INSN(custom2_rs1, MATCH_CUSTOM2_RS1, MASK_CUSTOM2_RS1) +DECLARE_INSN(custom2_rs1_rs2, MATCH_CUSTOM2_RS1_RS2, MASK_CUSTOM2_RS1_RS2) +DECLARE_INSN(custom2_rd, MATCH_CUSTOM2_RD, MASK_CUSTOM2_RD) +DECLARE_INSN(custom2_rd_rs1, MATCH_CUSTOM2_RD_RS1, MASK_CUSTOM2_RD_RS1) +DECLARE_INSN(custom2_rd_rs1_rs2, MATCH_CUSTOM2_RD_RS1_RS2, MASK_CUSTOM2_RD_RS1_RS2) +DECLARE_INSN(custom3, MATCH_CUSTOM3, MASK_CUSTOM3) +DECLARE_INSN(custom3_rs1, MATCH_CUSTOM3_RS1, MASK_CUSTOM3_RS1) +DECLARE_INSN(custom3_rs1_rs2, MATCH_CUSTOM3_RS1_RS2, MASK_CUSTOM3_RS1_RS2) +DECLARE_INSN(custom3_rd, MATCH_CUSTOM3_RD, MASK_CUSTOM3_RD) +DECLARE_INSN(custom3_rd_rs1, MATCH_CUSTOM3_RD_RS1, MASK_CUSTOM3_RD_RS1) +DECLARE_INSN(custom3_rd_rs1_rs2, MATCH_CUSTOM3_RD_RS1_RS2, MASK_CUSTOM3_RD_RS1_RS2) +#endif +#ifdef DECLARE_CSR +DECLARE_CSR(fflags, CSR_FFLAGS) +DECLARE_CSR(frm, CSR_FRM) +DECLARE_CSR(fcsr, CSR_FCSR) +DECLARE_CSR(cycle, CSR_CYCLE) +DECLARE_CSR(time, CSR_TIME) +DECLARE_CSR(instret, CSR_INSTRET) +DECLARE_CSR(stats, CSR_STATS) +DECLARE_CSR(uarch0, CSR_UARCH0) +DECLARE_CSR(uarch1, CSR_UARCH1) +DECLARE_CSR(uarch2, CSR_UARCH2) +DECLARE_CSR(uarch3, CSR_UARCH3) +DECLARE_CSR(uarch4, CSR_UARCH4) +DECLARE_CSR(uarch5, CSR_UARCH5) +DECLARE_CSR(uarch6, CSR_UARCH6) +DECLARE_CSR(uarch7, CSR_UARCH7) +DECLARE_CSR(uarch8, CSR_UARCH8) +DECLARE_CSR(uarch9, CSR_UARCH9) +DECLARE_CSR(uarch10, CSR_UARCH10) +DECLARE_CSR(uarch11, CSR_UARCH11) +DECLARE_CSR(uarch12, CSR_UARCH12) +DECLARE_CSR(uarch13, CSR_UARCH13) +DECLARE_CSR(uarch14, CSR_UARCH14) +DECLARE_CSR(uarch15, CSR_UARCH15) +DECLARE_CSR(sstatus, CSR_SSTATUS) +DECLARE_CSR(sie, CSR_SIE) +DECLARE_CSR(stvec, CSR_STVEC) +DECLARE_CSR(sscratch, CSR_SSCRATCH) +DECLARE_CSR(sepc, CSR_SEPC) +DECLARE_CSR(scause, CSR_SCAUSE) +DECLARE_CSR(sbadaddr, CSR_SBADADDR) +DECLARE_CSR(sip, CSR_SIP) +DECLARE_CSR(sptbr, CSR_SPTBR) +DECLARE_CSR(sasid, CSR_SASID) +DECLARE_CSR(scycle, CSR_SCYCLE) +DECLARE_CSR(stime, CSR_STIME) +DECLARE_CSR(sinstret, CSR_SINSTRET) +DECLARE_CSR(mstatus, CSR_MSTATUS) +DECLARE_CSR(medeleg, CSR_MEDELEG) +DECLARE_CSR(mideleg, CSR_MIDELEG) +DECLARE_CSR(mie, CSR_MIE) +DECLARE_CSR(mtvec, CSR_MTVEC) +DECLARE_CSR(mtimecmp, CSR_MTIMECMP) +DECLARE_CSR(mscratch, CSR_MSCRATCH) +DECLARE_CSR(mepc, CSR_MEPC) +DECLARE_CSR(mcause, CSR_MCAUSE) +DECLARE_CSR(mbadaddr, CSR_MBADADDR) +DECLARE_CSR(mip, CSR_MIP) +DECLARE_CSR(mipi, CSR_MIPI) +DECLARE_CSR(mucounteren, CSR_MUCOUNTEREN) +DECLARE_CSR(mscounteren, CSR_MSCOUNTEREN) +DECLARE_CSR(mucycle_delta, CSR_MUCYCLE_DELTA) +DECLARE_CSR(mutime_delta, CSR_MUTIME_DELTA) +DECLARE_CSR(muinstret_delta, CSR_MUINSTRET_DELTA) +DECLARE_CSR(mscycle_delta, CSR_MSCYCLE_DELTA) +DECLARE_CSR(mstime_delta, CSR_MSTIME_DELTA) +DECLARE_CSR(msinstret_delta, CSR_MSINSTRET_DELTA) +DECLARE_CSR(mcycle, CSR_MCYCLE) +DECLARE_CSR(mtime, CSR_MTIME) +DECLARE_CSR(minstret, CSR_MINSTRET) +DECLARE_CSR(misa, CSR_MISA) +DECLARE_CSR(mvendorid, CSR_MVENDORID) +DECLARE_CSR(marchid, CSR_MARCHID) +DECLARE_CSR(mimpid, CSR_MIMPID) +DECLARE_CSR(mcfgaddr, CSR_MCFGADDR) +DECLARE_CSR(mhartid, CSR_MHARTID) +DECLARE_CSR(mtohost, CSR_MTOHOST) +DECLARE_CSR(mfromhost, CSR_MFROMHOST) +DECLARE_CSR(mreset, CSR_MRESET) +DECLARE_CSR(cycleh, CSR_CYCLEH) +DECLARE_CSR(timeh, CSR_TIMEH) +DECLARE_CSR(instreth, CSR_INSTRETH) +DECLARE_CSR(mtimecmph, CSR_MTIMECMPH) +DECLARE_CSR(mucycle_deltah, CSR_MUCYCLE_DELTAH) +DECLARE_CSR(mutime_deltah, CSR_MUTIME_DELTAH) +DECLARE_CSR(muinstret_deltah, CSR_MUINSTRET_DELTAH) +DECLARE_CSR(mscycle_deltah, CSR_MSCYCLE_DELTAH) +DECLARE_CSR(mstime_deltah, CSR_MSTIME_DELTAH) +DECLARE_CSR(msinstret_deltah, CSR_MSINSTRET_DELTAH) +DECLARE_CSR(mcycleh, CSR_MCYCLEH) +DECLARE_CSR(mtimeh, CSR_MTIMEH) +DECLARE_CSR(minstreth, CSR_MINSTRETH) +#endif +#ifdef DECLARE_CAUSE +DECLARE_CAUSE("misaligned fetch", CAUSE_MISALIGNED_FETCH) +DECLARE_CAUSE("fault fetch", CAUSE_FAULT_FETCH) +DECLARE_CAUSE("illegal instruction", CAUSE_ILLEGAL_INSTRUCTION) +DECLARE_CAUSE("breakpoint", CAUSE_BREAKPOINT) +DECLARE_CAUSE("misaligned load", CAUSE_MISALIGNED_LOAD) +DECLARE_CAUSE("fault load", CAUSE_FAULT_LOAD) +DECLARE_CAUSE("misaligned store", CAUSE_MISALIGNED_STORE) +DECLARE_CAUSE("fault store", CAUSE_FAULT_STORE) +DECLARE_CAUSE("user_ecall", CAUSE_USER_ECALL) +DECLARE_CAUSE("supervisor_ecall", CAUSE_SUPERVISOR_ECALL) +DECLARE_CAUSE("hypervisor_ecall", CAUSE_HYPERVISOR_ECALL) +DECLARE_CAUSE("machine_ecall", CAUSE_MACHINE_ECALL) +#endif diff --git a/machine/fp_asm.S b/machine/fp_asm.S new file mode 100644 index 0000000..4b8dce1 --- /dev/null +++ b/machine/fp_asm.S @@ -0,0 +1,160 @@ +// See LICENSE for license details. + +#ifdef __riscv_hard_float + +#define get_f32(which) fmv.x.s a0, which; jr t0 +#define put_f32(which) fmv.s.x which, a0; jr t0 +#ifdef __riscv64 +# define get_f64(which) fmv.x.d a0, which; jr t0 +# define put_f64(which) fmv.d.x which, a0; jr t0 +#else +# define get_f64(which) fsd which, 0(a0); jr t0 +# define put_f64(which) fld which, 0(a0); jr t0 +#endif + + .text + .option norvc + .globl get_f32_reg + get_f32_reg: + get_f32(f0) + get_f32(f1) + get_f32(f2) + get_f32(f3) + get_f32(f4) + get_f32(f5) + get_f32(f6) + get_f32(f7) + get_f32(f8) + get_f32(f9) + get_f32(f10) + get_f32(f11) + get_f32(f12) + get_f32(f13) + get_f32(f14) + get_f32(f15) + get_f32(f16) + get_f32(f17) + get_f32(f18) + get_f32(f19) + get_f32(f20) + get_f32(f21) + get_f32(f22) + get_f32(f23) + get_f32(f24) + get_f32(f25) + get_f32(f26) + get_f32(f27) + get_f32(f28) + get_f32(f29) + get_f32(f30) + get_f32(f31) + + .text + .globl put_f32_reg + put_f32_reg: + put_f32(f0) + put_f32(f1) + put_f32(f2) + put_f32(f3) + put_f32(f4) + put_f32(f5) + put_f32(f6) + put_f32(f7) + put_f32(f8) + put_f32(f9) + put_f32(f10) + put_f32(f11) + put_f32(f12) + put_f32(f13) + put_f32(f14) + put_f32(f15) + put_f32(f16) + put_f32(f17) + put_f32(f18) + put_f32(f19) + put_f32(f20) + put_f32(f21) + put_f32(f22) + put_f32(f23) + put_f32(f24) + put_f32(f25) + put_f32(f26) + put_f32(f27) + put_f32(f28) + put_f32(f29) + put_f32(f30) + put_f32(f31) + + .text + .globl get_f64_reg + get_f64_reg: + get_f64(f0) + get_f64(f1) + get_f64(f2) + get_f64(f3) + get_f64(f4) + get_f64(f5) + get_f64(f6) + get_f64(f7) + get_f64(f8) + get_f64(f9) + get_f64(f10) + get_f64(f11) + get_f64(f12) + get_f64(f13) + get_f64(f14) + get_f64(f15) + get_f64(f16) + get_f64(f17) + get_f64(f18) + get_f64(f19) + get_f64(f20) + get_f64(f21) + get_f64(f22) + get_f64(f23) + get_f64(f24) + get_f64(f25) + get_f64(f26) + get_f64(f27) + get_f64(f28) + get_f64(f29) + get_f64(f30) + get_f64(f31) + + .text + .globl put_f64_reg + put_f64_reg: + put_f64(f0) + put_f64(f1) + put_f64(f2) + put_f64(f3) + put_f64(f4) + put_f64(f5) + put_f64(f6) + put_f64(f7) + put_f64(f8) + put_f64(f9) + put_f64(f10) + put_f64(f11) + put_f64(f12) + put_f64(f13) + put_f64(f14) + put_f64(f15) + put_f64(f16) + put_f64(f17) + put_f64(f18) + put_f64(f19) + put_f64(f20) + put_f64(f21) + put_f64(f22) + put_f64(f23) + put_f64(f24) + put_f64(f25) + put_f64(f26) + put_f64(f27) + put_f64(f28) + put_f64(f29) + put_f64(f30) + put_f64(f31) + +#endif diff --git a/machine/fp_emulation.c b/machine/fp_emulation.c new file mode 100644 index 0000000..536967f --- /dev/null +++ b/machine/fp_emulation.c @@ -0,0 +1,449 @@ +#include "fp_emulation.h" +#include "unprivileged_memory.h" +#include "softfloat.h" +#include "config.h" + +DECLARE_EMULATION_FUNC(emulate_float_load) +{ + uint64_t val; + uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); + + switch (insn & MASK_FUNCT3) + { + case MATCH_FLW & MASK_FUNCT3: + if (addr % 4 != 0) + return misaligned_load_trap(regs, mcause, mepc); + + SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc)); + break; + + case MATCH_FLD & MASK_FUNCT3: + if (addr % sizeof(uintptr_t) != 0) + return misaligned_load_trap(regs, mcause, mepc); + +#ifdef __riscv64 + val = load_uint64_t((void *)addr, mepc); +#else + val = load_uint32_t(addr, mepc); + val += (uint64_t)load_uint32_t((void *)(addr + 4), mepc) << 32; +#endif + SET_F64_RD(insn, regs, val); + break; + + default: + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_float_store) +{ + uint64_t val; + uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); + + switch (insn & MASK_FUNCT3) + { + case MATCH_FSW & MASK_FUNCT3: + if (addr % 4 != 0) + return misaligned_store_trap(regs, mcause, mepc); + + store_uint32_t((void *)addr, GET_F32_RS2(insn, regs), mepc); + break; + + case MATCH_FSD & MASK_FUNCT3: + if (addr % sizeof(uintptr_t) != 0) + return misaligned_store_trap(regs, mcause, mepc); + + val = GET_F64_RS2(insn, regs); +#ifdef __riscv64 + store_uint64_t((void *)addr, val, mepc); +#else + store_uint32_t((void *)addr, val, mepc); + store_uint32_t((void *)(addr + 4), val >> 32, mepc); +#endif + break; + + default: + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fp) +{ + asm (".pushsection .rodata\n" + "fp_emulation_table:\n" + " .word emulate_fadd\n" + " .word emulate_fsub\n" + " .word emulate_fmul\n" + " .word emulate_fdiv\n" + " .word emulate_fsgnj\n" + " .word emulate_fmin\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_fcvt_ff\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_fsqrt\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_fcmp\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_fcvt_if\n" + " .word truly_illegal_insn\n" + " .word emulate_fcvt_fi\n" + " .word truly_illegal_insn\n" + " .word emulate_fmv_if\n" + " .word truly_illegal_insn\n" + " .word emulate_fmv_fi\n" + " .word truly_illegal_insn\n" + " .popsection"); + + // if FPU is disabled, punt back to the OS + if (unlikely((mstatus & MSTATUS_FS) == 0)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + extern int32_t fp_emulation_table[]; + int32_t* pf = (void*)fp_emulation_table + ((insn >> 25) & 0x7c); + emulation_func f = (emulation_func)(uintptr_t)*pf; + + SETUP_STATIC_ROUNDING(insn); + return f(regs, mcause, mepc, mstatus, insn); +} + +void emulate_any_fadd(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn, int32_t neg_b) +{ + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs) ^ neg_b; + SET_F32_RD(insn, regs, f32_add(rs1, rs2)); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs) ^ ((uint64_t)neg_b << 32); + SET_F64_RD(insn, regs, f64_add(rs1, rs2)); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fadd) +{ + return emulate_any_fadd(regs, mcause, mepc, mstatus, insn, 0); +} + +DECLARE_EMULATION_FUNC(emulate_fsub) +{ + return emulate_any_fadd(regs, mcause, mepc, mstatus, insn, INT32_MIN); +} + +DECLARE_EMULATION_FUNC(emulate_fmul) +{ + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + SET_F32_RD(insn, regs, f32_mul(rs1, rs2)); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + SET_F64_RD(insn, regs, f64_mul(rs1, rs2)); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fdiv) +{ + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + SET_F32_RD(insn, regs, f32_div(rs1, rs2)); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + SET_F64_RD(insn, regs, f64_div(rs1, rs2)); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fsqrt) +{ + if ((insn >> 20) & 0x1f) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + if (GET_PRECISION(insn) == PRECISION_S) { + SET_F32_RD(insn, regs, f32_sqrt(GET_F32_RS1(insn, regs))); + } else if (GET_PRECISION(insn) == PRECISION_D) { + SET_F64_RD(insn, regs, f64_sqrt(GET_F64_RS1(insn, regs))); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fsgnj) +{ + int rm = GET_RM(insn); + if (rm >= 3) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + #define DO_FSGNJ(rs1, rs2, rm) ({ \ + typeof(rs1) rs1_sign = (rs1) >> (8*sizeof(rs1)-1); \ + typeof(rs1) rs2_sign = (rs2) >> (8*sizeof(rs1)-1); \ + rs1_sign &= (rm) >> 1; \ + rs1_sign ^= (rm) ^ rs2_sign; \ + ((rs1) << 1 >> 1) | (rs1_sign << (8*sizeof(rs1)-1)); }) + + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + SET_F32_RD(insn, regs, DO_FSGNJ(rs1, rs2, rm)); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + SET_F64_RD(insn, regs, DO_FSGNJ(rs1, rs2, rm)); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fmin) +{ + int rm = GET_RM(insn); + if (rm >= 2) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + uint32_t arg1 = rm ? rs2 : rs1; + uint32_t arg2 = rm ? rs1 : rs2; + int use_rs1 = f32_lt_quiet(arg1, arg2) || isNaNF32UI(rs2); + SET_F32_RD(insn, regs, use_rs1 ? rs1 : rs2); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + uint64_t arg1 = rm ? rs2 : rs1; + uint64_t arg2 = rm ? rs1 : rs2; + int use_rs1 = f64_lt_quiet(arg1, arg2) || isNaNF64UI(rs2); + SET_F64_RD(insn, regs, use_rs1 ? rs1 : rs2); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fcvt_ff) +{ + int rs2_num = (insn >> 20) & 0x1f; + if (GET_PRECISION(insn) == PRECISION_S) { + if (rs2_num != 1) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + SET_F32_RD(insn, regs, f64_to_f32(GET_F64_RS1(insn, regs))); + } else if (GET_PRECISION(insn) == PRECISION_D) { + if (rs2_num != 0) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + SET_F64_RD(insn, regs, f32_to_f64(GET_F32_RS1(insn, regs))); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fcvt_fi) +{ + if (GET_PRECISION(insn) != PRECISION_S && GET_PRECISION(insn) != PRECISION_D) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + int negative = 0; + uint64_t uint_val = GET_RS1(insn, regs); + + switch ((insn >> 20) & 0x1f) + { + case 0: // int32 + negative = (int32_t)uint_val < 0; + uint_val = negative ? -(int32_t)uint_val : (int32_t)uint_val; + break; + case 1: // uint32 + uint_val = (uint32_t)uint_val; + break; +#ifdef __riscv64 + case 2: // int64 + negative = (int64_t)uint_val < 0; + uint_val = negative ? -uint_val : uint_val; + case 3: // uint64 + break; +#endif + default: + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } + + uint64_t float64 = ui64_to_f64(uint_val); + if (negative) + float64 ^= INT64_MIN; + + if (GET_PRECISION(insn) == PRECISION_S) + SET_F32_RD(insn, regs, f64_to_f32(float64)); + else + SET_F64_RD(insn, regs, float64); +} + +DECLARE_EMULATION_FUNC(emulate_fcvt_if) +{ + int rs2_num = (insn >> 20) & 0x1f; +#ifdef __riscv64 + if (rs2_num >= 4) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); +#else + if (rs2_num >= 2) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); +#endif + + int64_t float64; + if (GET_PRECISION(insn) == PRECISION_S) + float64 = f32_to_f64(GET_F32_RS1(insn, regs)); + else if (GET_PRECISION(insn) == PRECISION_D) + float64 = GET_F64_RS1(insn, regs); + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + int negative = 0; + if (float64 < 0) { + negative = 1; + float64 ^= INT64_MIN; + } + uint64_t uint_val = f64_to_ui64(float64, softfloat_roundingMode, true); + uint64_t result, limit, limit_result; + + switch (rs2_num) + { + case 0: // int32 + if (negative) { + result = (int32_t)-uint_val; + limit_result = limit = (uint32_t)INT32_MIN; + } else { + result = (int32_t)uint_val; + limit_result = limit = INT32_MAX; + } + break; + + case 1: // uint32 + limit = limit_result = UINT32_MAX; + if (negative) + result = limit = 0; + else + result = (uint32_t)uint_val; + break; + + case 2: // int32 + if (negative) { + result = (int64_t)-uint_val; + limit_result = limit = (uint64_t)INT64_MIN; + } else { + result = (int64_t)uint_val; + limit_result = limit = INT64_MAX; + } + break; + + case 3: // uint64 + limit = limit_result = UINT64_MAX; + if (negative) + result = limit = 0; + else + result = (uint64_t)uint_val; + break; + + default: + __builtin_unreachable(); + } + + if (uint_val > limit) { + result = limit_result; + softfloat_raiseFlags(softfloat_flag_invalid); + } + + SET_FS_DIRTY(); + SET_RD(insn, regs, result); +} + +DECLARE_EMULATION_FUNC(emulate_fcmp) +{ + int rm = GET_RM(insn); + if (rm >= 3) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + uintptr_t result; + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + if (rm != 1) + result = f32_eq(rs1, rs2); + if (rm == 1 || (rm == 0 && !result)) + result = f32_lt(rs1, rs2); + goto success; + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + if (rm != 1) + result = f64_eq(rs1, rs2); + if (rm == 1 || (rm == 0 && !result)) + result = f64_lt(rs1, rs2); + goto success; + } + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); +success: + SET_RD(insn, regs, result); +} + +DECLARE_EMULATION_FUNC(emulate_fmv_if) +{ + uintptr_t result; + if ((insn & MASK_FMV_X_S) == MATCH_FMV_X_S) + result = GET_F32_RS1(insn, regs); +#ifdef __riscv64 + else if ((insn & MASK_FMV_X_D) == MATCH_FMV_X_D) + result = GET_F64_RS1(insn, regs); +#endif + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + SET_RD(insn, regs, result); +} + +DECLARE_EMULATION_FUNC(emulate_fmv_fi) +{ + uintptr_t rs1 = GET_RS1(insn, regs); + + if ((insn & MASK_FMV_S_X) == MATCH_FMV_S_X) + SET_F32_RD(insn, regs, rs1); + else if ((insn & MASK_FMV_D_X) == MATCH_FMV_D_X) + SET_F64_RD(insn, regs, rs1); + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); +} + +DECLARE_EMULATION_FUNC(emulate_fmadd) +{ + // if FPU is disabled, punt back to the OS + if (unlikely((mstatus & MSTATUS_FS) == 0)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + int op = (insn >> 2) & 3; + SETUP_STATIC_ROUNDING(insn); + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + uint32_t rs3 = GET_F32_RS3(insn, regs); + SET_F32_RD(insn, regs, softfloat_mulAddF32(op, rs1, rs2, rs3)); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + uint64_t rs3 = GET_F64_RS3(insn, regs); + SET_F64_RD(insn, regs, softfloat_mulAddF64(op, rs1, rs2, rs3)); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} diff --git a/machine/fp_emulation.h b/machine/fp_emulation.h new file mode 100644 index 0000000..d2357b7 --- /dev/null +++ b/machine/fp_emulation.h @@ -0,0 +1,81 @@ +#ifndef _RISCV_FP_EMULATION_H +#define _RISCV_FP_EMULATION_H + +#include "emulation.h" + +#define GET_PRECISION(insn) (((insn) >> 25) & 3) +#define GET_RM(insn) (((insn) >> 12) & 7) +#define PRECISION_S 0 +#define PRECISION_D 1 + +#ifdef __riscv_hard_float +# define GET_F32_REG(insn, pos, regs) ({ \ + register int32_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \ + uintptr_t tmp; \ + asm ("1: auipc %0, %%pcrel_hi(get_f32_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \ + value; }) +# define SET_F32_REG(insn, pos, regs, val) ({ \ + register uint32_t value asm("a0") = (val); \ + uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \ + uintptr_t tmp; \ + asm volatile ("1: auipc %0, %%pcrel_hi(put_f32_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); }) +# define init_fp_reg(i) SET_F32_REG((i) << 3, 3, 0, 0) +# define GET_F64_REG(insn, pos, regs) ({ \ + register uintptr_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \ + uintptr_t tmp; \ + asm ("1: auipc %0, %%pcrel_hi(get_f64_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \ + sizeof(uintptr_t) == 4 ? *(int64_t*)value : (int64_t)value; }) +# define SET_F64_REG(insn, pos, regs, val) ({ \ + uint64_t __val = (val); \ + register uintptr_t value asm("a0") = sizeof(uintptr_t) == 4 ? (uintptr_t)&__val : (uintptr_t)__val; \ + uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \ + uintptr_t tmp; \ + asm volatile ("1: auipc %0, %%pcrel_hi(put_f64_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); }) +# define GET_FCSR() read_csr(fcsr) +# define SET_FCSR(value) write_csr(fcsr, (value)) +# define GET_FRM() read_csr(frm) +# define SET_FRM(value) write_csr(frm, (value)) +# define GET_FFLAGS() read_csr(fflags) +# define SET_FFLAGS(value) write_csr(fflags, (value)) + +# define SETUP_STATIC_ROUNDING(insn) ({ \ + register long tp asm("tp") = read_csr(frm); \ + if (likely(((insn) & MASK_FUNCT3) == MASK_FUNCT3)) ; \ + else if (GET_RM(insn) > 4) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); \ + else tp = GET_RM(insn); \ + asm volatile ("":"+r"(tp)); }) +# define softfloat_raiseFlags(which) set_csr(fflags, which) +# define softfloat_roundingMode ({ register int tp asm("tp"); tp; }) +#else +# define GET_F64_REG(insn, pos, regs) (*(int64_t*)((void*)((regs) + 32) + (((insn) >> ((pos)-3)) & 0xf8))) +# define SET_F64_REG(insn, pos, regs, val) (GET_F64_REG(insn, pos, regs) = (val)) +# define GET_F32_REG(insn, pos, regs) (*(int32_t*)&GET_F64_REG(insn, pos, regs)) +# define SET_F32_REG(insn, pos, regs, val) (GET_F32_REG(insn, pos, regs) = (val)) +# define GET_FCSR() ({ register int tp asm("tp"); tp & 0xFF; }) +# define SET_FCSR(value) ({ asm volatile("add tp, x0, %0" :: "rI"((value) & 0xFF)); }) +# define GET_FRM() (GET_FCSR() >> 5) +# define SET_FRM(value) SET_FCSR(GET_FFLAGS() | ((value) << 5)) +# define GET_FFLAGS() (GET_FCSR() & 0x1F) +# define SET_FFLAGS(value) SET_FCSR((GET_FRM() << 5) | ((value) & 0x1F)) + +# define SETUP_STATIC_ROUNDING(insn) ({ \ + register int tp asm("tp"); tp &= 0xFF; \ + if (likely(((insn) & MASK_FUNCT3) == MASK_FUNCT3)) tp |= tp << 8; \ + else if (GET_RM(insn) > 4) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); \ + else tp |= GET_RM(insn) << 13; \ + asm volatile ("":"+r"(tp)); }) +# define softfloat_raiseFlags(which) ({ asm volatile ("or tp, tp, %0" :: "rI"(which)); }) +# define softfloat_roundingMode ({ register int tp asm("tp"); tp >> 13; }) +#endif + +#define GET_F32_RS1(insn, regs) (GET_F32_REG(insn, 15, regs)) +#define GET_F32_RS2(insn, regs) (GET_F32_REG(insn, 20, regs)) +#define GET_F32_RS3(insn, regs) (GET_F32_REG(insn, 27, regs)) +#define GET_F64_RS1(insn, regs) (GET_F64_REG(insn, 15, regs)) +#define GET_F64_RS2(insn, regs) (GET_F64_REG(insn, 20, regs)) +#define GET_F64_RS3(insn, regs) (GET_F64_REG(insn, 27, regs)) +#define SET_F32_RD(insn, regs, val) (SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY()) +#define SET_F64_RD(insn, regs, val) (SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY()) +#define SET_FS_DIRTY() set_csr(mstatus, MSTATUS_FS) + +#endif diff --git a/machine/htif.h b/machine/htif.h new file mode 100644 index 0000000..e66ad06 --- /dev/null +++ b/machine/htif.h @@ -0,0 +1,18 @@ +#ifndef _RISCV_HTIF_H +#define _RISCV_HTIF_H + +#include <stdint.h> + +#ifdef __riscv64 +# define TOHOST_CMD(dev, cmd, payload) \ + (((uint64_t)(dev) << 56) | ((uint64_t)(cmd) << 48) | (uint64_t)(payload)) +#else +# define TOHOST_CMD(dev, cmd, payload) ({ \ + if ((dev) || (cmd)) __builtin_trap(); \ + (payload); }) +#endif +#define FROMHOST_DEV(fromhost_value) ((uint64_t)(fromhost_value) >> 56) +#define FROMHOST_CMD(fromhost_value) ((uint64_t)(fromhost_value) << 8 >> 56) +#define FROMHOST_DATA(fromhost_value) ((uint64_t)(fromhost_value) << 16 >> 16) + +#endif diff --git a/machine/machine.ac b/machine/machine.ac new file mode 100644 index 0000000..65acf04 --- /dev/null +++ b/machine/machine.ac @@ -0,0 +1,4 @@ +AC_ARG_ENABLE([fp-emulation], AS_HELP_STRING([--disable-fp-emulation], [Disable floating-point emulation])) +AS_IF([test "x$enable_fp_emulation" != "xno"], [ + AC_DEFINE([PK_ENABLE_FP_EMULATION],,[Define if floating-point emulation is enabled]) +]) diff --git a/machine/machine.mk.in b/machine/machine.mk.in new file mode 100644 index 0000000..a0b7ca9 --- /dev/null +++ b/machine/machine.mk.in @@ -0,0 +1,29 @@ +machine_subproject_deps = \ + softfloat \ + +machine_hdrs = \ + atomic.h \ + bits.h \ + emulation.h \ + encoding.h \ + fp_emulation.h \ + htif.h \ + mcall.h \ + mtrap.h \ + sbi.h \ + unprivileged_memory.h \ + vm.h \ + +machine_c_srcs = \ + mtrap.c \ + minit.c \ + emulation.c \ + fp_emulation.c \ + sbi_impl.c \ + configstring.c \ + +machine_asm_srcs = \ + mentry.S \ + fp_asm.S \ + sbi_entry.S \ + sbi.S \ diff --git a/machine/mcall.h b/machine/mcall.h new file mode 100644 index 0000000..2096d16 --- /dev/null +++ b/machine/mcall.h @@ -0,0 +1,21 @@ +#ifndef _RISCV_MCALL_H +#define _RISCV_MCALL_H + +#define MCALL_HART_ID 0 +#define MCALL_CONSOLE_PUTCHAR 1 +#define MCALL_CONSOLE_GETCHAR 2 +#define MCALL_HTIF_SYSCALL 3 +#define MCALL_SEND_IPI 4 +#define MCALL_CLEAR_IPI 5 +#define MCALL_SHUTDOWN 6 +#define MCALL_SET_TIMER 7 +#define MCALL_REMOTE_SFENCE_VM 8 +#define MCALL_REMOTE_FENCE_I 9 + +#ifndef __ASSEMBLER__ + +extern uintptr_t do_mcall(uintptr_t which, ...); + +#endif + +#endif diff --git a/machine/mentry.S b/machine/mentry.S new file mode 100644 index 0000000..50ce9ca --- /dev/null +++ b/machine/mentry.S @@ -0,0 +1,245 @@ +// See LICENSE for license details. + +#include "mtrap.h" +#include "bits.h" + + .data + .align 6 +trap_table: + .word bad_trap + .word bad_trap + .word illegal_insn_trap + .word bad_trap + .word misaligned_load_trap + .word bad_trap + .word misaligned_store_trap + .word bad_trap + .word bad_trap + .word mcall_trap + .word bad_trap + .word bad_trap +#define HTIF_INTERRUPT_VECTOR 12 + .word htif_interrupt +#define SOFTWARE_INTERRUPT_VECTOR 13 + .word software_interrupt +#define TRAP_FROM_MACHINE_MODE_VECTOR 14 + .word __trap_from_machine_mode + + .option norvc + .section .text.init,"ax",@progbits + .globl reset_vector +reset_vector: + j do_reset + +nmi_vector: +.Lunhandleable_trap: + j bad_trap + +trap_vector: + csrrw sp, mscratch, sp + beqz sp, .Ltrap_from_machine_mode + + STORE a0, 10*REGBYTES(sp) + STORE a1, 11*REGBYTES(sp) + + csrr a1, mcause + bgez a1, .Lhandle_trap_in_machine_mode + + # This is an interrupt. Discard the mcause MSB and decode the rest. + sll a1, a1, 1 + + # Is it a machine timer interrupt? + li a0, IRQ_M_TIMER * 2 + bne a0, a1, 1f + # Yes. Post a supervisor timer interrupt. + li a0, MIP_MTIP + csrc mie, a0 + li a0, MIP_STIP + csrs mip, a0 + +.Leret: + # Go back whence we came. + LOAD a0, 10*REGBYTES(sp) + LOAD a1, 11*REGBYTES(sp) + csrrw sp, mscratch, sp + eret + +1: + # Is it an IPI? + li a0, IRQ_M_SOFT * 2 + bne a0, a1, 1f + li a1, SOFTWARE_INTERRUPT_VECTOR + j .Lhandle_trap_in_machine_mode + +1: + # By process of elimination, it must be an HTIF interrupt. + li a0, IRQ_HOST * 2 + bne a0, a1, .Lunhandleable_trap + li a1, HTIF_INTERRUPT_VECTOR + +.Lhandle_trap_in_machine_mode: + # Preserve the registers. Compute the address of the trap handler. + STORE ra, 1*REGBYTES(sp) + STORE gp, 3*REGBYTES(sp) + STORE tp, 4*REGBYTES(sp) + STORE t0, 5*REGBYTES(sp) +1:auipc t0, %pcrel_hi(trap_table) # t0 <- %hi(trap_table) + STORE t1, 6*REGBYTES(sp) + sll t1, a1, 2 # t1 <- mcause << 2 + STORE t2, 7*REGBYTES(sp) + add t1, t0, t1 # t1 <- %hi(trap_table)[mcause] + STORE s0, 8*REGBYTES(sp) + lw t1, %pcrel_lo(1b)(t1) # t1 <- trap_table[mcause] + STORE s1, 9*REGBYTES(sp) + mv a0, sp # a0 <- regs + STORE a2,12*REGBYTES(sp) + csrr a2, mepc # a2 <- mepc + STORE a3,13*REGBYTES(sp) + csrrw t0, mscratch, x0 # t0 <- user sp + STORE a4,14*REGBYTES(sp) + STORE a5,15*REGBYTES(sp) + STORE a6,16*REGBYTES(sp) + STORE a7,17*REGBYTES(sp) + STORE s2,18*REGBYTES(sp) + STORE s3,19*REGBYTES(sp) + STORE s4,20*REGBYTES(sp) + STORE s5,21*REGBYTES(sp) + STORE s6,22*REGBYTES(sp) + STORE s7,23*REGBYTES(sp) + STORE s8,24*REGBYTES(sp) + STORE s9,25*REGBYTES(sp) + STORE s10,26*REGBYTES(sp) + STORE s11,27*REGBYTES(sp) + STORE t3,28*REGBYTES(sp) + STORE t4,29*REGBYTES(sp) + STORE t5,30*REGBYTES(sp) + STORE t6,31*REGBYTES(sp) + STORE t0, 2*REGBYTES(sp) # sp + +#ifndef __riscv_hard_float + lw tp, (sp) # Move the emulated FCSR from x0's save slot into tp. +#endif + STORE x0, (sp) # Zero x0's save slot. + + # Invoke the handler. + jalr t1 + +#ifndef __riscv_hard_float + sw tp, (sp) # Move the emulated FCSR from tp into x0's save slot. +#endif + +restore_mscratch: + # Restore mscratch, so future traps will know they didn't come from M-mode. + csrw mscratch, sp + +restore_regs: + # Restore all of the registers. + LOAD ra, 1*REGBYTES(sp) + LOAD gp, 3*REGBYTES(sp) + LOAD tp, 4*REGBYTES(sp) + LOAD t0, 5*REGBYTES(sp) + LOAD t1, 6*REGBYTES(sp) + LOAD t2, 7*REGBYTES(sp) + LOAD s0, 8*REGBYTES(sp) + LOAD s1, 9*REGBYTES(sp) + LOAD a0,10*REGBYTES(sp) + LOAD a1,11*REGBYTES(sp) + LOAD a2,12*REGBYTES(sp) + LOAD a3,13*REGBYTES(sp) + LOAD a4,14*REGBYTES(sp) + LOAD a5,15*REGBYTES(sp) + LOAD a6,16*REGBYTES(sp) + LOAD a7,17*REGBYTES(sp) + LOAD s2,18*REGBYTES(sp) + LOAD s3,19*REGBYTES(sp) + LOAD s4,20*REGBYTES(sp) + LOAD s5,21*REGBYTES(sp) + LOAD s6,22*REGBYTES(sp) + LOAD s7,23*REGBYTES(sp) + LOAD s8,24*REGBYTES(sp) + LOAD s9,25*REGBYTES(sp) + LOAD s10,26*REGBYTES(sp) + LOAD s11,27*REGBYTES(sp) + LOAD t3,28*REGBYTES(sp) + LOAD t4,29*REGBYTES(sp) + LOAD t5,30*REGBYTES(sp) + LOAD t6,31*REGBYTES(sp) + LOAD sp, 2*REGBYTES(sp) + eret + +.Ltrap_from_machine_mode: + csrr sp, mscratch + addi sp, sp, -INTEGER_CONTEXT_SIZE + STORE a0,10*REGBYTES(sp) + STORE a1,11*REGBYTES(sp) + li a1, TRAP_FROM_MACHINE_MODE_VECTOR + j .Lhandle_trap_in_machine_mode + + .globl __redirect_trap +__redirect_trap: + # reset sp to top of M-mode stack + li t0, MACHINE_STACK_SIZE + add sp, sp, t0 + neg t0, t0 + and sp, sp, t0 + addi sp, sp, -MENTRY_FRAME_SIZE + j restore_mscratch + +__trap_from_machine_mode: + jal trap_from_machine_mode + j restore_regs + +do_reset: + li x1, 0 + li x2, 0 + li x3, 0 + li x4, 0 + li x5, 0 + li x6, 0 + li x7, 0 + li x8, 0 + li x9, 0 + li x10, 0 + li x11, 0 + li x12, 0 + li x13, 0 + li x14, 0 + li x15, 0 + li x16, 0 + li x17, 0 + li x18, 0 + li x19, 0 + li x20, 0 + li x21, 0 + li x22, 0 + li x23, 0 + li x24, 0 + li x25, 0 + li x26, 0 + li x27, 0 + li x28, 0 + li x29, 0 + li x30, 0 + li x31, 0 + csrw mscratch, x0 + + # sp <- end of first full page after the end of the binary + la sp, _end + 2*RISCV_PGSIZE - 1 + li t0, -RISCV_PGSIZE + and sp, sp, t0 + addi sp, sp, -MENTRY_FRAME_SIZE + + csrr a0, mhartid + slli a1, a0, RISCV_PGSHIFT + add sp, sp, a1 + + beqz a0, init_first_hart + +.LmultiHart: +#if MAX_HARTS > 1 + # make sure our hart id is within a valid range + li a1, MAX_HARTS + bltu a0, a1, init_other_hart + wfi +#endif + j .LmultiHart diff --git a/machine/minit.c b/machine/minit.c new file mode 100644 index 0000000..33b94c8 --- /dev/null +++ b/machine/minit.c @@ -0,0 +1,120 @@ +#include "mtrap.h" +#include "atomic.h" +#include "vm.h" +#include "fp_emulation.h" +#include <string.h> + +pte_t* root_page_table; +uintptr_t first_free_paddr; +uintptr_t mem_size; +uint32_t num_harts; + +static void mstatus_init() +{ + uintptr_t ms = 0; + ms = INSERT_FIELD(ms, MSTATUS_VM, VM_CHOICE); + ms = INSERT_FIELD(ms, MSTATUS_FS, 1); + write_csr(mstatus, ms); + + ms = read_csr(mstatus); + assert(EXTRACT_FIELD(ms, MSTATUS_VM) == VM_CHOICE); + + write_csr(mtimecmp, 0); + clear_csr(mip, MIP_MSIP); + write_csr(mie, -1); + write_csr(mucounteren, -1); + write_csr(mscounteren, -1); +} + +static void delegate_traps() +{ + uintptr_t interrupts = MIP_SSIP | MIP_STIP; + uintptr_t exceptions = + (1U << CAUSE_MISALIGNED_FETCH) | + (1U << CAUSE_FAULT_FETCH) | + (1U << CAUSE_BREAKPOINT) | + (1U << CAUSE_FAULT_LOAD) | + (1U << CAUSE_FAULT_STORE) | + (1U << CAUSE_BREAKPOINT) | + (1U << CAUSE_USER_ECALL); + + write_csr(mideleg, interrupts); + write_csr(medeleg, exceptions); + assert(read_csr(mideleg) == interrupts); + assert(read_csr(medeleg) == exceptions); +} + +static void fp_init() +{ + assert(read_csr(mstatus) & MSTATUS_FS); + +#ifdef __riscv_hard_float + if (!supports_extension('D')) + die("FPU not found; recompile pk with -msoft-float"); + for (int i = 0; i < 32; i++) + init_fp_reg(i); + write_csr(fcsr, 0); +#else + if (supports_extension('D')) + die("FPU unexpectedly found; recompile with -mhard-float"); +#endif +} + +void hls_init(uint32_t id, csr_t* csrs) +{ + hls_t* hls = OTHER_HLS(id); + memset(hls, 0, sizeof(*hls)); + hls->csrs = csrs; +} + +static uintptr_t sbi_top_paddr() +{ + extern char _end; + return ROUNDUP((uintptr_t)&_end, RISCV_PGSIZE); +} + +static void memory_init() +{ + mem_size = mem_size / MEGAPAGE_SIZE * MEGAPAGE_SIZE; + first_free_paddr = sbi_top_paddr() + num_harts * RISCV_PGSIZE; +} + +static void hart_init() +{ + mstatus_init(); + fp_init(); + delegate_traps(); +} + +void init_first_hart() +{ + hart_init(); + memset(HLS(), 0, sizeof(*HLS())); + parse_config_string(); + memory_init(); + boot_loader(); +} + +void init_other_hart() +{ + hart_init(); + + // wait until hart 0 discovers us + while (*(csr_t * volatile *)&HLS()->csrs == NULL) + ; + + boot_other_hart(); +} + +void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t stack) +{ + uintptr_t mstatus = read_csr(mstatus); + mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S); + mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); + write_csr(mstatus, mstatus); + write_csr(mscratch, MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE); + write_csr(mepc, fn); + write_csr(sptbr, (uintptr_t)root_page_table >> RISCV_PGSHIFT); + asm volatile ("mv a0, %0; mv sp, %0; eret" : : "r" (stack)); + __builtin_unreachable(); +} diff --git a/machine/mtrap.c b/machine/mtrap.c new file mode 100644 index 0000000..6fc5476 --- /dev/null +++ b/machine/mtrap.c @@ -0,0 +1,287 @@ +#include "mtrap.h" +#include "mcall.h" +#include "htif.h" +#include "atomic.h" +#include "bits.h" +#include <errno.h> +#include <stdarg.h> +#include <stdio.h> + +void __attribute__((noreturn)) bad_trap() +{ + die("machine mode: unhandlable trap %d @ %p", read_csr(mcause), read_csr(mepc)); +} + +static uintptr_t mcall_hart_id() +{ + return read_const_csr(mhartid); +} + +static void request_htif_keyboard_interrupt() +{ + uintptr_t old_tohost = swap_csr(mtohost, TOHOST_CMD(1, 0, 0)); + assert(old_tohost == 0); +} + +void htif_interrupt() +{ + uintptr_t fromhost = swap_csr(mfromhost, 0); + // we should only be interrupted by keypresses + if (!(FROMHOST_DEV(fromhost) == 1 && FROMHOST_CMD(fromhost) == 0)) + die("unexpected htif interrupt"); + HLS()->console_ibuf = 1 + (uint8_t)FROMHOST_DATA(fromhost); + set_csr(mip, MIP_SSIP); +} + +static void do_tohost_fromhost(uintptr_t dev, uintptr_t cmd, uintptr_t data) +{ + while (swap_csr(mtohost, TOHOST_CMD(dev, cmd, data)) != 0) + if (read_csr(mfromhost)) + htif_interrupt(); + + while (1) { + uintptr_t fromhost = read_csr(mfromhost); + if (fromhost) { + if (FROMHOST_DEV(fromhost) == dev && FROMHOST_CMD(fromhost) == cmd) { + write_csr(mfromhost, 0); + break; + } + htif_interrupt(); + } + } +} + +static uintptr_t mcall_console_putchar(uint8_t ch) +{ + do_tohost_fromhost(1, 1, ch); + return 0; +} + +static uintptr_t mcall_htif_syscall(uintptr_t magic_mem) +{ + do_tohost_fromhost(0, 0, magic_mem); + return 0; +} + +void poweroff() +{ + while (1) + write_csr(mtohost, 1); +} + +void putstring(const char* s) +{ + while (*s) + mcall_console_putchar(*s++); +} + +void printm(const char* s, ...) +{ + char buf[256]; + va_list vl; + + va_start(vl, s); + vsnprintf(buf, sizeof buf, s, vl); + va_end(vl); + + putstring(buf); +} + +static void send_ipi(uintptr_t recipient, int event) +{ + if ((atomic_or(&OTHER_HLS(recipient)->mipi_pending, event) & event) == 0) { + mb(); + OTHER_HLS(recipient)->csrs[CSR_MIPI] = 1; + } +} + +static uintptr_t mcall_send_ipi(uintptr_t recipient) +{ + if (recipient >= num_harts) + return -1; + + send_ipi(recipient, IPI_SOFT); + return 0; +} + +static void reset_ssip() +{ + clear_csr(mip, MIP_SSIP); + mb(); + + if (HLS()->sipi_pending || HLS()->console_ibuf > 0) + set_csr(mip, MIP_SSIP); +} + +static uintptr_t mcall_console_getchar() +{ + int ch = atomic_swap(&HLS()->console_ibuf, -1); + if (ch >= 0) + request_htif_keyboard_interrupt(); + reset_ssip(); + return ch - 1; +} + +static uintptr_t mcall_clear_ipi() +{ + int ipi = atomic_swap(&HLS()->sipi_pending, 0); + reset_ssip(); + return ipi; +} + +static uintptr_t mcall_shutdown() +{ + poweroff(); +} + +static uintptr_t mcall_set_timer(unsigned long long when) +{ + // bbl/pk don't use the timer, so there's no need to virtualize it + write_csr(mtimecmp, when); +#ifndef __riscv64 + write_csr(mtimecmph, when >> 32); +#endif + clear_csr(mip, MIP_STIP); + set_csr(mie, MIP_MTIP); + return 0; +} + +void software_interrupt() +{ + clear_csr(mip, MIP_MSIP); + mb(); + int ipi_pending = atomic_swap(&HLS()->mipi_pending, 0); + + if (ipi_pending & IPI_SOFT) { + HLS()->sipi_pending = 1; + set_csr(mip, MIP_SSIP); + } + + if (ipi_pending & IPI_FENCE_I) + asm volatile ("fence.i"); + + if (ipi_pending & IPI_SFENCE_VM) + asm volatile ("sfence.vm"); +} + +static void send_ipi_many(uintptr_t* pmask, int event) +{ + _Static_assert(MAX_HARTS <= 8 * sizeof(*pmask), "# harts > uintptr_t bits"); + uintptr_t mask = -1; + if (pmask) + mask = *pmask; + + // send IPIs to everyone + for (ssize_t i = num_harts-1; i >= 0; i--) + if ((mask >> i) & 1) + send_ipi(i, event); + + // wait until all events have been handled. + // prevent deadlock while spinning by handling any IPIs from other harts. + for (ssize_t i = num_harts-1; i >= 0; i--) + if ((mask >> i) & 1) + while (OTHER_HLS(i)->mipi_pending & event) + software_interrupt(); +} + +static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask, uintptr_t asid) +{ + // ignore the ASID and do a global flush. + // this allows us to avoid queueing a message. + send_ipi_many(hart_mask, IPI_SFENCE_VM); + return 0; +} + +static uintptr_t mcall_remote_fence_i(uintptr_t* hart_mask) +{ + send_ipi_many(hart_mask, IPI_FENCE_I); + return 0; +} + +void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval; + switch (n) + { + case MCALL_HART_ID: + retval = mcall_hart_id(); + break; + case MCALL_CONSOLE_PUTCHAR: + retval = mcall_console_putchar(arg0); + break; + case MCALL_CONSOLE_GETCHAR: + retval = mcall_console_getchar(); + break; + case MCALL_HTIF_SYSCALL: + retval = mcall_htif_syscall(arg0); + break; + case MCALL_SEND_IPI: + retval = mcall_send_ipi(arg0); + break; + case MCALL_CLEAR_IPI: + retval = mcall_clear_ipi(); + break; + case MCALL_SHUTDOWN: + retval = mcall_shutdown(); + break; + case MCALL_SET_TIMER: + retval = mcall_set_timer(arg0); + break; + case MCALL_REMOTE_SFENCE_VM: + retval = mcall_remote_sfence_vm((uintptr_t*)arg0, arg1); + break; + case MCALL_REMOTE_FENCE_I: + retval = mcall_remote_fence_i((uintptr_t*)arg0); + break; + default: + retval = -ENOSYS; + break; + } + regs[10] = retval; + write_csr(mepc, mepc + 4); +} + +void redirect_trap(uintptr_t epc, uintptr_t mstatus) +{ + write_csr(sepc, epc); + write_csr(scause, read_csr(mcause)); + write_csr(mepc, read_csr(stvec)); + + uintptr_t prev_priv = EXTRACT_FIELD(mstatus, MSTATUS_MPP); + uintptr_t prev_ie = EXTRACT_FIELD(mstatus, MSTATUS_MPIE); + mstatus = INSERT_FIELD(mstatus, MSTATUS_SPP, prev_priv); + mstatus = INSERT_FIELD(mstatus, MSTATUS_SPIE, prev_ie); + mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S); + mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); + write_csr(mstatus, mstatus); + + extern void __redirect_trap(); + return __redirect_trap(); +} + +static void machine_page_fault(uintptr_t* regs, uintptr_t mepc) +{ + // MPRV=1 iff this trap occurred while emulating an instruction on behalf + // of a lower privilege level. In that case, a2=epc and a3=mstatus. + if (read_csr(mstatus) & MSTATUS_MPRV) { + write_csr(sbadaddr, read_csr(mbadaddr)); + return redirect_trap(regs[12], regs[13]); + } + bad_trap(); +} + +void trap_from_machine_mode(uintptr_t* regs, uintptr_t dummy, uintptr_t mepc) +{ + uintptr_t mcause = read_csr(mcause); + + switch (mcause) + { + case CAUSE_FAULT_LOAD: + case CAUSE_FAULT_STORE: + return machine_page_fault(regs, mepc); + case CAUSE_MACHINE_ECALL: + return mcall_trap(regs, mcause, mepc); + default: + bad_trap(); + } +} diff --git a/machine/mtrap.h b/machine/mtrap.h new file mode 100644 index 0000000..139f359 --- /dev/null +++ b/machine/mtrap.h @@ -0,0 +1,96 @@ +#ifndef _RISCV_MTRAP_H +#define _RISCV_MTRAP_H + +#include "encoding.h" + +#ifdef __riscv_atomic +# define MAX_HARTS 8 // arbitrary +#else +# define MAX_HARTS 1 +#endif + +#ifndef __ASSEMBLER__ + +#include "sbi.h" +#include <stdint.h> + +#define read_const_csr(reg) ({ unsigned long __tmp; \ + asm ("csrr %0, " #reg : "=r"(__tmp)); \ + __tmp; }) + +static inline int supports_extension(char ext) +{ + return read_const_csr(misa) & (1 << (ext - 'A')); +} + +static inline int xlen() +{ + return read_const_csr(misa) < 0 ? 64 : 32; +} + +extern uintptr_t first_free_paddr; +extern uintptr_t mem_size; +extern uint32_t num_harts; + +typedef uintptr_t csr_t; // TODO this might become uint128_t for RV128 + +typedef struct { + volatile csr_t* csrs; + volatile int mipi_pending; + volatile int sipi_pending; + int console_ibuf; + + uint64_t utime_delta; + uint64_t ucycle_delta; + uint64_t uinstret_delta; + uint64_t stime_delta; + uint64_t scycle_delta; + uint64_t sinstret_delta; +} hls_t; + +#define IPI_SOFT 0x1 +#define IPI_FENCE_I 0x2 +#define IPI_SFENCE_VM 0x4 + +#define MACHINE_STACK_TOP() ({ \ + register uintptr_t sp asm ("sp"); \ + (void*)((sp + RISCV_PGSIZE) & -RISCV_PGSIZE); }) + +// hart-local storage, at top of stack +#define HLS() ((hls_t*)(MACHINE_STACK_TOP() - HLS_SIZE)) +#define OTHER_HLS(id) ((hls_t*)((void*)HLS() + RISCV_PGSIZE * ((id) - read_const_csr(mhartid)))) + +void hls_init(uint32_t hart_id, csr_t* csrs); +void parse_config_string(); +void poweroff(void) __attribute((noreturn)); +void printm(const char* s, ...); +void putstring(const char* s); +#define assert(x) ({ if (!(x)) die("assertion failed: %s", #x); }) +#define die(str, ...) ({ printm("%s:%d: " str "\n", __FILE__, __LINE__, ##__VA_ARGS__); poweroff(); }) +#define printk(...) die("printk") + +void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t stack) + __attribute__((noreturn)); +void boot_loader(); +void boot_other_hart(); + +static inline void wfi() +{ + asm volatile ("wfi" ::: "memory"); +} + +#endif // !__ASSEMBLER__ + +#define MACHINE_STACK_SIZE RISCV_PGSIZE +#define MENTRY_FRAME_SIZE (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE \ + + HLS_SIZE) + +#ifdef __riscv_hard_float +# define SOFT_FLOAT_CONTEXT_SIZE 0 +#else +# define SOFT_FLOAT_CONTEXT_SIZE (8 * 32) +#endif +#define HLS_SIZE 64 +#define INTEGER_CONTEXT_SIZE (32 * REGBYTES) + +#endif diff --git a/machine/sbi.S b/machine/sbi.S new file mode 100644 index 0000000..cbea78a --- /dev/null +++ b/machine/sbi.S @@ -0,0 +1,15 @@ +.globl sbi_hart_id; sbi_hart_id = -2048 +.globl sbi_num_harts; sbi_num_harts = -2032 +.globl sbi_query_memory; sbi_query_memory = -2016 +.globl sbi_console_putchar; sbi_console_putchar = -2000 +.globl sbi_console_getchar; sbi_console_getchar = -1984 +.globl sbi_send_ipi; sbi_send_ipi = -1952 +.globl sbi_clear_ipi; sbi_clear_ipi = -1936 +.globl sbi_timebase; sbi_timebase = -1920 +.globl sbi_shutdown; sbi_shutdown = -1904 +.globl sbi_set_timer; sbi_set_timer = -1888 +.globl sbi_mask_interrupt; sbi_mask_interrupt = -1872 +.globl sbi_unmask_interrupt; sbi_unmask_interrupt = -1856 +.globl sbi_remote_sfence_vm; sbi_remote_sfence_vm = -1840 +.globl sbi_remote_sfence_vm_range; sbi_remote_sfence_vm_range = -1824 +.globl sbi_remote_fence_i; sbi_remote_fence_i = -1808 diff --git a/machine/sbi.h b/machine/sbi.h new file mode 100644 index 0000000..4e2fbd8 --- /dev/null +++ b/machine/sbi.h @@ -0,0 +1,30 @@ +#ifndef _ASM_RISCV_SBI_H +#define _ASM_RISCV_SBI_H + +typedef struct { + unsigned long base; + unsigned long size; + unsigned long node_id; +} memory_block_info; + +unsigned long sbi_query_memory(unsigned long id, memory_block_info *p); + +unsigned long sbi_hart_id(void); +unsigned long sbi_num_harts(void); +unsigned long sbi_timebase(void); +void sbi_set_timer(unsigned long long stime_value); +void sbi_send_ipi(unsigned long hart_id); +unsigned long sbi_clear_ipi(void); +void sbi_shutdown(void); + +void sbi_console_putchar(unsigned char ch); +int sbi_console_getchar(void); + +void sbi_remote_sfence_vm(unsigned long hart_mask_ptr, unsigned long asid); +void sbi_remote_sfence_vm_range(unsigned long hart_mask_ptr, unsigned long asid, unsigned long start, unsigned long size); +void sbi_remote_fence_i(unsigned long hart_mask_ptr); + +unsigned long sbi_mask_interrupt(unsigned long which); +unsigned long sbi_unmask_interrupt(unsigned long which); + +#endif diff --git a/machine/sbi_entry.S b/machine/sbi_entry.S new file mode 100644 index 0000000..a37dd25 --- /dev/null +++ b/machine/sbi_entry.S @@ -0,0 +1,111 @@ +#include "encoding.h" +#include "mcall.h" + + .section .sbi,"ax",@progbits + .option norvc + .align RISCV_PGSHIFT + .globl sbi_base +sbi_base: + + # TODO: figure out something better to do with this space. It's not + # protected from the OS, so beware. + .skip RISCV_PGSIZE - 2048 + + # hart_id + .align 4 + li a7, MCALL_HART_ID + ecall + ret + + # num_harts + .align 4 + lw a0, num_harts + ret + + # query_memory + .align 4 + tail __sbi_query_memory + + # console_putchar + .align 4 + li a7, MCALL_CONSOLE_PUTCHAR + ecall + ret + + # console_getchar + .align 4 + li a7, MCALL_CONSOLE_GETCHAR + ecall + ret + + # empty + .align 4 + unimp + + # send ipi + .align 4 + li a7, MCALL_SEND_IPI + ecall + ret + + # clear ipi + .align 4 + li a7, MCALL_CLEAR_IPI + ecall + ret + + # timebase + .align 4 + li a0, 10000000 # or, you know, we could provide the correct answer + ret + + # shutdown + .align 4 + li a7, MCALL_SHUTDOWN + ecall + + # set_timer + .align 4 + li a7, MCALL_SET_TIMER + ecall + ret + + # mask_interrupt + .align 4 + tail __sbi_mask_interrupt + + # unmask_interrupt + .align 4 + tail __sbi_unmask_interrupt + + # remote_sfence_vm + .align 4 + li a7, MCALL_REMOTE_SFENCE_VM + ecall + ret + + # remote_sfence_vm_range + .align 4 + li a7, MCALL_REMOTE_SFENCE_VM + ecall + ret + + # remote_fence_i + .align 4 + li a7, MCALL_REMOTE_FENCE_I + ecall + ret + + # end of SBI trampolines + + .globl do_mcall +do_mcall: + mv a7, a0 + mv a0, a1 + mv a1, a2 + ecall + ret + + .align RISCV_PGSHIFT + .globl _sbi_end +_sbi_end: diff --git a/machine/sbi_impl.c b/machine/sbi_impl.c new file mode 100644 index 0000000..07844e0 --- /dev/null +++ b/machine/sbi_impl.c @@ -0,0 +1,33 @@ +#include "mtrap.h" +#include "sbi.h" + +uintptr_t __sbi_query_memory(uintptr_t id, memory_block_info *p) +{ + if (id == 0) { + p->base = first_free_paddr; + p->size = mem_size - p->base; + return 0; + } + + return -1; +} + +#define LOW_IRQ_OK(n) ((n) == IRQ_S_SOFT || (n) == IRQ_S_TIMER) + +uintptr_t __sbi_mask_interrupt(uintptr_t which) +{ + if (!LOW_IRQ_OK(which)) + return -1; + + clear_csr(sie, 1UL << which); + return 0; +} + +uintptr_t __sbi_unmask_interrupt(uintptr_t which) +{ + if (!LOW_IRQ_OK(which)) + return -1; + + set_csr(sie, 1UL << which); + return 0; +} diff --git a/machine/unprivileged_memory.h b/machine/unprivileged_memory.h new file mode 100644 index 0000000..d03cc5e --- /dev/null +++ b/machine/unprivileged_memory.h @@ -0,0 +1,79 @@ +#ifndef _RISCV_MISALIGNED_H +#define _RISCV_MISALIGNED_H + +#include "encoding.h" +#include <stdint.h> + +#define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type, insn) \ + static inline type load_##type(const type* addr, uintptr_t mepc) \ + { \ + register uintptr_t __mepc asm ("a2") = mepc; \ + register uintptr_t __mstatus asm ("a3"); \ + type val; \ + asm ("csrrs %0, mstatus, %3\n" \ + #insn " %1, %2\n" \ + "csrw mstatus, %0" \ + : "+&r" (__mstatus), "=&r" (val) \ + : "m" (*addr), "r" (MSTATUS_MPRV), "r" (__mepc)); \ + return val; \ + } + +#define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type, insn) \ + static inline void store_##type(type* addr, type val, uintptr_t mepc) \ + { \ + register uintptr_t __mepc asm ("a2") = mepc; \ + register uintptr_t __mstatus asm ("a3"); \ + asm volatile ("csrrs %0, mstatus, %3\n" \ + #insn " %1, %2\n" \ + "csrw mstatus, %0" \ + : "+&r" (__mstatus) \ + : "r" (val), "m" (*addr), "r" (MSTATUS_MPRV), \ + "r" (__mepc)); \ + } + +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint8_t, lbu) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint16_t, lhu) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int8_t, lb) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int16_t, lh) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int32_t, lw) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint8_t, sb) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint16_t, sh) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint32_t, sw) +#ifdef __riscv64 +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lwu) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint64_t, ld) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint64_t, sd) +#else +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lw) +#endif + +static uint32_t __attribute__((always_inline)) get_insn(uintptr_t mepc, uintptr_t* mstatus) +{ + register uintptr_t __mepc asm ("a2") = mepc; + register uintptr_t __mstatus asm ("a3"); + uint32_t val; +#ifndef __riscv_compressed + asm ("csrrs %[mstatus], mstatus, %[mprv]\n" + "lw %[insn], (%[addr])\n" + "csrw mstatus, %[mstatus]" + : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val) + : [mprv] "r" (MSTATUS_MPRV), [addr] "r" (__mepc)); +#else + uintptr_t rvc_mask = 3, tmp; + asm ("csrrs %[mstatus], mstatus, %[mprv]\n" + "lhu %[insn], (%[addr])\n" + "and %[tmp], %[insn], %[rvc_mask]\n" + "bne %[tmp], %[rvc_mask], 1f\n" + "lh %[tmp], 2(%[addr])\n" + "sll %[tmp], %[tmp], 16\n" + "add %[insn], %[insn], %[tmp]\n" + "1: csrw mstatus, %[mstatus]" + : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val), [tmp] "=&r" (tmp) + : [mprv] "r" (MSTATUS_MPRV), [addr] "r" (__mepc), + [rvc_mask] "r" (rvc_mask)); +#endif + *mstatus = __mstatus; + return val; +} + +#endif diff --git a/machine/vm.h b/machine/vm.h new file mode 100644 index 0000000..f51c639 --- /dev/null +++ b/machine/vm.h @@ -0,0 +1,35 @@ +#ifndef _VM_H +#define _VM_H + +#include "encoding.h" +#include <stdint.h> + +#define MEGAPAGE_SIZE ((uintptr_t)(RISCV_PGSIZE << RISCV_PGLEVEL_BITS)) +#ifdef __riscv64 +# define VM_CHOICE VM_SV39 +# define VA_BITS 39 +# define GIGAPAGE_SIZE (MEGAPAGE_SIZE << RISCV_PGLEVEL_BITS) +#else +# define VM_CHOICE VM_SV32 +# define VA_BITS 32 +#endif + +typedef uintptr_t pte_t; +extern pte_t* root_page_table; + +static inline void flush_tlb() +{ + asm volatile("sfence.vm"); +} + +static inline pte_t pte_create(uintptr_t ppn, int type) +{ + return (ppn << PTE_PPN_SHIFT) | PTE_V | type; +} + +static inline pte_t ptd_create(uintptr_t ppn) +{ + return pte_create(ppn, PTE_TYPE_TABLE); +} + +#endif |