Refactor pk, bbl, machine into separate libraries

Yuck.

23 files changed, 3496 insertions, 0 deletions

diff --git a/machine/atomic.h b/machine/atomic.h
new file mode 100644
index 0000000..b23acae
--- /dev/null
+++ b/machine/atomic.h
@@ -0,0 +1,72 @@
+// See LICENSE for license details.
+
+#ifndef _RISCV_ATOMIC_H
+#define _RISCV_ATOMIC_H
+
+#include "config.h"
+#include "encoding.h"
+
+// Currently, interrupts are always disabled in M-mode.
+#define disable_irqsave() (0)
+#define enable_irqrestore(flags) ((void) (flags))
+
+typedef struct { int lock; } spinlock_t;
+#define SPINLOCK_INIT {0}
+
+#define mb() asm volatile ("fence" ::: "memory")
+#define atomic_set(ptr, val) (*(volatile typeof(*(ptr)) *)(ptr) = val)
+#define atomic_read(ptr) (*(volatile typeof(*(ptr)) *)(ptr))
+
+#ifdef __riscv_atomic
+# define atomic_add(ptr, inc) __sync_fetch_and_add(ptr, inc)
+# define atomic_or(ptr, inc) __sync_fetch_and_or(ptr, inc)
+# define atomic_swap(ptr, swp) __sync_lock_test_and_set(ptr, swp)
+# define atomic_cas(ptr, cmp, swp) __sync_val_compare_and_swap(ptr, cmp, swp)
+#else
+# define atomic_binop(ptr, inc, op) ({ \
+  long flags = disable_irqsave(); \
+  typeof(*(ptr)) res = atomic_read(ptr); \
+  atomic_set(ptr, op); \
+  enable_irqrestore(flags); \
+  res; })
+# define atomic_add(ptr, inc) atomic_binop(ptr, inc, res + (inc))
+# define atomic_or(ptr, inc) atomic_binop(ptr, inc, res | (inc))
+# define atomic_swap(ptr, inc) atomic_binop(ptr, inc, (inc))
+# define atomic_cas(ptr, cmp, swp) ({ \
+  long flags = disable_irqsave(); \
+  typeof(*(ptr)) res = *(volatile typeof(*(ptr)) *)(ptr); \
+  if (res == (cmp)) *(volatile typeof(ptr))(ptr) = (swp); \
+  enable_irqrestore(flags); \
+  res; })
+#endif
+
+static inline void spinlock_lock(spinlock_t* lock)
+{
+  do
+  {
+    while (atomic_read(&lock->lock))
+      ;
+  } while (atomic_swap(&lock->lock, -1));
+  mb();
+}
+
+static inline void spinlock_unlock(spinlock_t* lock)
+{
+  mb();
+  atomic_set(&lock->lock,0);
+}
+
+static inline long spinlock_lock_irqsave(spinlock_t* lock)
+{
+  long flags = disable_irqsave();
+  spinlock_lock(lock);
+  return flags;
+}
+
+static inline void spinlock_unlock_irqrestore(spinlock_t* lock, long flags)
+{
+  spinlock_unlock(lock);
+  enable_irqrestore(flags);
+}
+
+#endif
diff --git a/machine/bits.h b/machine/bits.h
new file mode 100644
index 0000000..72514ae
--- /dev/null
+++ b/machine/bits.h
@@ -0,0 +1,33 @@
+#ifndef _RISCV_BITS_H
+#define _RISCV_BITS_H
+
+#define likely(x) __builtin_expect((x), 1)
+#define unlikely(x) __builtin_expect((x), 0)
+
+#define ROUNDUP(a, b) ((((a)-1)/(b)+1)*(b))
+#define ROUNDDOWN(a, b) ((a)/(b)*(b))
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define CLAMP(a, lo, hi) MIN(MAX(a, lo), hi)
+
+#define EXTRACT_FIELD(val, which) (((val) & (which)) / ((which) & ~((which)-1)))
+#define INSERT_FIELD(val, which, fieldval) (((val) & ~(which)) | ((fieldval) * ((which) & ~((which)-1))))
+
+#define STR(x) XSTR(x)
+#define XSTR(x) #x
+
+#ifdef __riscv64
+# define SLL32    sllw
+# define STORE    sd
+# define LOAD     ld
+# define LOG_REGBYTES 3
+#else
+# define SLL32    sll
+# define STORE    sw
+# define LOAD     lw
+# define LOG_REGBYTES 2
+#endif
+#define REGBYTES (1 << LOG_REGBYTES)
+
+#endif
diff --git a/machine/configstring.c b/machine/configstring.c
new file mode 100644
index 0000000..847cd4d
--- /dev/null
+++ b/machine/configstring.c
@@ -0,0 +1,200 @@
+#include "encoding.h"
+#include "mtrap.h"
+#include <stdio.h>
+
+static const char* skip_whitespace(const char* str)
+{
+  while (*str && *str <= ' ')
+    str++;
+  return str;
+}
+
+static const char* skip_string(const char* str)
+{
+  while (*str && *str++ != '"')
+    ;
+  return str;
+}
+
+static int is_hex(char ch)
+{
+  return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F');
+}
+
+static int parse_hex(char ch)
+{
+  return (ch >= '0' && ch <= '9') ? ch - '0' :
+         (ch >= 'a' && ch <= 'f') ? ch - 'a' + 10 :
+                                    ch - 'A' + 10;
+}
+
+static const char* skip_key(const char* str)
+{
+  while (*str >= 35 && *str <= 122 && *str != ';')
+    str++;
+  return str;
+}
+
+typedef struct {
+  const char* start;
+  const char* end;
+} query_result;
+
+static query_result query_config_string(const char* str, const char* k)
+{
+  size_t ksize = 0;
+  while (k[ksize] && k[ksize] != '{')
+    ksize++;
+  int last = !k[ksize];
+
+  query_result res = {0, 0};
+  while (1) {
+    const char* key_start = str = skip_whitespace(str);
+    const char* key_end = str = skip_key(str);
+    int match = (key_end - key_start) == ksize;
+    if (match)
+      for (size_t i = 0; i < ksize; i++)
+        if (key_start[i] != k[i])
+          match = 0;
+    const char* value_start = str = skip_whitespace(str);
+    while (*str != ';') {
+      if (!*str) {
+        return res;
+      } else if (*str == '"') {
+        str = skip_string(str+1);
+      } else if (*str == '{') {
+        const char* search_key = match && !last ? k + ksize + 1 : "";
+        query_result inner_res = query_config_string(str + 1, search_key);
+        if (inner_res.start)
+          return inner_res;
+        str = inner_res.end + 1;
+      } else {
+        str = skip_key(str);
+      }
+      str = skip_whitespace(str);
+    }
+    res.end = str;
+    if (match && last) {
+      res.start = value_start;
+      return res;
+    }
+    str = skip_whitespace(str+1);
+    if (*str == '}') {
+      res.end = str;
+      return res;
+    }
+  }
+}
+
+static void parse_string(query_result r, char* buf)
+{
+  if (r.start < r.end) {
+    if (*r.start == '"') {
+      for (const char* p = r.start + 1; p < r.end && *p != '"'; p++) {
+        char ch = p[0];
+        if (ch == '\\' && p[1] == 'x' && is_hex(p[2])) {
+          ch = parse_hex(p[2]);
+          if (is_hex(p[3])) {
+            ch = (ch << 4) + parse_hex(p[3]);
+            p++;
+          }
+          p += 2;
+        }
+        *buf++ = ch;
+      }
+    } else {
+      for (const char* p = r.start; p < r.end && *p > ' '; p++)
+        *buf++ = *p;
+    }
+  }
+  *buf = 0;
+}
+
+#define get_string(name, search_res) \
+  char name[(search_res).end - (search_res).start + 1]; \
+  parse_string(search_res, name)
+
+static unsigned long __get_uint_hex(const char* s)
+{
+  unsigned long res = 0;
+  while (*s) {
+    if (is_hex(*s))
+      res = (res << 4) + parse_hex(*s);
+    else if (*s != '_')
+      break;
+    s++;
+  }
+  return res;
+}
+
+static unsigned long __get_uint_dec(const char* s)
+{
+  unsigned long res = 0;
+  while (*s) {
+    if (*s >= '0' && *s <= '9')
+      res = res * 10 + (*s - '0');
+    else
+      break;
+    s++;
+  }
+  return res;
+}
+
+static unsigned long __get_uint(const char* s)
+{
+  if (s[0] == '0' && s[1] == 'x')
+    return __get_uint_hex(s+2);
+  return __get_uint_dec(s);
+}
+
+static unsigned long get_uint(query_result res)
+{
+  get_string(s, res);
+  return __get_uint(s);
+}
+
+static long get_sint(query_result res)
+{
+  get_string(s, res);
+  if (s[0] == '-')
+    return -__get_uint(s+1);
+  return __get_uint(s);
+}
+
+static void query_mem(const char* config_string)
+{
+  query_result res = query_config_string(config_string, "ram{0{addr");
+  assert(res.start);
+  uintptr_t base = get_uint(res);
+  res = query_config_string(config_string, "ram{0{size");
+  mem_size = get_uint(res);
+}
+
+static void query_harts(const char* config_string)
+{
+  for (int core = 0, hart; ; core++) {
+    for (hart = 0; ; hart++) {
+      char buf[32];
+      snprintf(buf, sizeof buf, "core{%d{%d{addr", core, hart);
+      query_result res = query_config_string(config_string, buf);
+      if (!res.start)
+        break;
+      csr_t* base = (csr_t*)get_uint(res);
+      uintptr_t hart_id = base[CSR_MHARTID];
+      hls_init(hart_id, base);
+      num_harts++;
+      assert(hart_id == num_harts-1);
+    }
+    if (!hart)
+      break;
+  }
+  assert(num_harts);
+  assert(num_harts <= MAX_HARTS);
+}
+
+void parse_config_string()
+{
+  const char* s = (const char*)read_csr(mcfgaddr);
+  query_mem(s);
+  query_harts(s);
+}
diff --git a/machine/emulation.c b/machine/emulation.c
new file mode 100644
index 0000000..5a66997
--- /dev/null
+++ b/machine/emulation.c
@@ -0,0 +1,313 @@
+#include "emulation.h"
+#include "fp_emulation.h"
+#include "config.h"
+#include "unprivileged_memory.h"
+#include "mtrap.h"
+#include <limits.h>
+
+void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
+{
+  asm (".pushsection .rodata\n"
+       "illegal_insn_trap_table:\n"
+       "  .word truly_illegal_insn\n"
+#ifdef PK_ENABLE_FP_EMULATION
+       "  .word emulate_float_load\n"
+#else
+       "  .word truly_illegal_insn\n"
+#endif
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+#ifdef PK_ENABLE_FP_EMULATION
+       "  .word emulate_float_store\n"
+#else
+       "  .word truly_illegal_insn\n"
+#endif
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word emulate_mul_div\n"
+       "  .word truly_illegal_insn\n"
+       "  .word emulate_mul_div32\n"
+       "  .word truly_illegal_insn\n"
+#ifdef PK_ENABLE_FP_EMULATION
+       "  .word emulate_fmadd\n"
+       "  .word emulate_fmadd\n"
+       "  .word emulate_fmadd\n"
+       "  .word emulate_fmadd\n"
+       "  .word emulate_fp\n"
+#else
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+#endif
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word emulate_system\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .popsection");
+
+  uintptr_t mstatus;
+  insn_t insn = get_insn(mepc, &mstatus);
+
+  if (unlikely((insn & 3) != 3))
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  write_csr(mepc, mepc + 4);
+
+  extern int32_t illegal_insn_trap_table[];
+  int32_t* pf = (void*)illegal_insn_trap_table + (insn & 0x7c);
+  emulation_func f = (emulation_func)(uintptr_t)*pf;
+  f(regs, mcause, mepc, mstatus, insn);
+}
+
+void __attribute__((noinline)) truly_illegal_insn(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn)
+{
+  redirect_trap(mepc, mstatus);
+}
+
+void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
+{
+  union {
+    uint8_t bytes[8];
+    uintptr_t intx;
+    uint64_t int64;
+  } val;
+  uintptr_t mstatus;
+  insn_t insn = get_insn(mepc, &mstatus);
+  uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn);
+
+  int shift = 0, fp = 0, len;
+  if ((insn & MASK_LW) == MATCH_LW)
+    len = 4, shift = 8*(sizeof(uintptr_t) - len);
+#ifdef __riscv64
+  else if ((insn & MASK_LD) == MATCH_LD)
+    len = 8, shift = 8*(sizeof(uintptr_t) - len);
+  else if ((insn & MASK_LWU) == MATCH_LWU)
+    fp = 0, len = 4, shift = 0;
+#endif
+  else if ((insn & MASK_FLD) == MATCH_FLD)
+    fp = 1, len = 8;
+  else if ((insn & MASK_FLW) == MATCH_FLW)
+    fp = 1, len = 4;
+  else if ((insn & MASK_LH) == MATCH_LH)
+    len = 2, shift = 8*(sizeof(uintptr_t) - len);
+  else if ((insn & MASK_LHU) == MATCH_LHU)
+    len = 2;
+  else
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  val.int64 = 0;
+  for (intptr_t i = len-1; i >= 0; i--)
+    val.bytes[i] = load_uint8_t((void *)(addr + i), mepc);
+
+  if (!fp)
+    SET_RD(insn, regs, (intptr_t)val.intx << shift >> shift);
+  else if (len == 8)
+    SET_F64_RD(insn, regs, val.int64);
+  else
+    SET_F32_RD(insn, regs, val.intx);
+
+  write_csr(mepc, mepc + 4);
+}
+
+void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
+{
+  union {
+    uint8_t bytes[8];
+    uintptr_t intx;
+    uint64_t int64;
+  } val;
+  uintptr_t mstatus;
+  insn_t insn = get_insn(mepc, &mstatus);
+  int len;
+
+  val.intx = GET_RS2(insn, regs);
+  if ((insn & MASK_SW) == MATCH_SW)
+    len = 4;
+#ifdef __riscv64
+  else if ((insn & MASK_SD) == MATCH_SD)
+    len = 8;
+#endif
+  else if ((insn & MASK_FSD) == MATCH_FSD)
+    len = 8, val.int64 = GET_F64_RS2(insn, regs);
+  else if ((insn & MASK_FSW) == MATCH_FSW)
+    len = 4, val.intx = GET_F32_RS2(insn, regs);
+  else if ((insn & MASK_SH) == MATCH_SH)
+    len = 2;
+  else
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn);
+  for (int i = 0; i < len; i++)
+    store_uint8_t((void *)(addr + i), val.bytes[i], mepc);
+
+  write_csr(mepc, mepc + 4);
+}
+
+#ifdef __riscv64
+typedef __int128 double_int;
+#else
+typedef int64_t double_int;
+#endif
+
+DECLARE_EMULATION_FUNC(emulate_mul_div)
+{
+  uintptr_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs), val;
+
+#ifndef __riscv_muldiv
+  // If compiled with -mno-multiply, GCC will expand these out
+  if ((insn & MASK_MUL) == MATCH_MUL)
+    val = rs1 * rs2;
+  else if ((insn & MASK_DIV) == MATCH_DIV)
+    val = (intptr_t)rs1 / (intptr_t)rs2;
+  else if ((insn & MASK_DIVU) == MATCH_DIVU)
+    val = rs1 / rs2;
+  else if ((insn & MASK_REM) == MATCH_REM)
+    val = (intptr_t)rs1 % (intptr_t)rs2;
+  else if ((insn & MASK_REMU) == MATCH_REMU)
+    val = rs1 % rs2;
+  else if ((insn & MASK_MULH) == MATCH_MULH)
+    val = ((double_int)(intptr_t)rs1 * (double_int)(intptr_t)rs2) >> (8 * sizeof(rs1));
+  else if ((insn & MASK_MULHU) == MATCH_MULHU)
+    val = ((double_int)rs1 * (double_int)rs2) >> (8 * sizeof(rs1));
+  else if ((insn & MASK_MULHSU) == MATCH_MULHSU)
+    val = ((double_int)(intptr_t)rs1 * (double_int)rs2) >> (8 * sizeof(rs1));
+  else
+#endif
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  SET_RD(insn, regs, val);
+}
+
+DECLARE_EMULATION_FUNC(emulate_mul_div32)
+{
+  uint32_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs);
+  int32_t val;
+
+#if defined(__riscv64) && !defined(__riscv_muldiv)
+  // If compiled with -mno-multiply, GCC will expand these out
+  if ((insn & MASK_MULW) == MATCH_MULW)
+    val = rs1 * rs2;
+  else if ((insn & MASK_DIVW) == MATCH_DIVW)
+    val = (int32_t)rs1 / (int32_t)rs2;
+  else if ((insn & MASK_DIVUW) == MATCH_DIVUW)
+    val = rs1 / rs2;
+  else if ((insn & MASK_REMW) == MATCH_REMW)
+    val = (int32_t)rs1 % (int32_t)rs2;
+  else if ((insn & MASK_REMUW) == MATCH_REMUW)
+    val = rs1 % rs2;
+  else
+#endif
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  SET_RD(insn, regs, val);
+}
+
+static inline int emulate_read_csr(int num, uintptr_t mstatus, uintptr_t* result)
+{
+  switch (num)
+  {
+    case CSR_TIME:
+      *result = read_csr(mtime) + HLS()->utime_delta;
+      return 0;
+    case CSR_CYCLE:
+      *result = read_csr(mcycle) + HLS()->ucycle_delta;
+      return 0;
+    case CSR_INSTRET:
+      *result = read_csr(minstret) + HLS()->uinstret_delta;
+      return 0;
+    case CSR_STIME:
+      *result = read_csr(mtime) + HLS()->stime_delta;
+      return 0;
+    case CSR_SCYCLE:
+      *result = read_csr(mcycle) + HLS()->scycle_delta;
+      return 0;
+    case CSR_SINSTRET:
+      *result = read_csr(minstret) + HLS()->sinstret_delta;
+      return 0;
+#ifdef __riscv32
+    case CSR_TIMEH:
+      *result = (((uint64_t)read_csr(mtimeh) << 32) + read_csr(mtime)
+                 + HLS()->stime_delta) >> 32;
+      return 0;
+    case CSR_CYCLEH:
+      *result = (((uint64_t)read_csr(mcycleh) << 32) + read_csr(mcycle)
+                 + HLS()->scycle_delta) >> 32;
+      return 0;
+    case CSR_INSTRETH:
+      *result = (((uint64_t)read_csr(minstreth) << 32) + read_csr(minstret)
+                 + HLS()->sinstret_delta) >> 32;
+      return 0;
+#endif
+#ifdef PK_ENABLE_FP_EMULATION
+    case CSR_FRM:
+      if ((mstatus & MSTATUS_FS) == 0) break;
+      *result = GET_FRM();
+      return 0;
+    case CSR_FFLAGS:
+      if ((mstatus & MSTATUS_FS) == 0) break;
+      *result = GET_FFLAGS();
+      return 0;
+    case CSR_FCSR:
+      if ((mstatus & MSTATUS_FS) == 0) break;
+      *result = GET_FCSR();
+      return 0;
+#endif
+  }
+  return -1;
+}
+
+static inline int emulate_write_csr(int num, uintptr_t value, uintptr_t mstatus)
+{
+  switch (num)
+  {
+#ifdef PK_ENABLE_FP_EMULATION
+    case CSR_FRM: SET_FRM(value); return 0;
+    case CSR_FFLAGS: SET_FFLAGS(value); return 0;
+    case CSR_FCSR: SET_FCSR(value); return 0;
+#endif
+  }
+  return -1;
+}
+
+DECLARE_EMULATION_FUNC(emulate_system)
+{
+  int rs1_num = (insn >> 15) & 0x1f;
+  uintptr_t rs1_val = GET_RS1(insn, regs);
+  int csr_num = (uint32_t)insn >> 20;
+  uintptr_t csr_val, new_csr_val;
+
+  if (emulate_read_csr(csr_num, mstatus, &csr_val))
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  int do_write = rs1_num;
+  switch (GET_RM(insn))
+  {
+    case 0: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+    case 1: new_csr_val = rs1_val; do_write = 1; break;
+    case 2: new_csr_val = csr_val | rs1_val; break;
+    case 3: new_csr_val = csr_val & ~rs1_val; break;
+    case 4: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+    case 5: new_csr_val = rs1_num; do_write = 1; break;
+    case 6: new_csr_val = csr_val | rs1_num; break;
+    case 7: new_csr_val = csr_val & ~rs1_num; break;
+  }
+
+  if (do_write && emulate_write_csr(csr_num, new_csr_val, mstatus))
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  SET_RD(insn, regs, csr_val);
+}
diff --git a/machine/emulation.h b/machine/emulation.h
new file mode 100644
index 0000000..f1a71ec
--- /dev/null
+++ b/machine/emulation.h
@@ -0,0 +1,28 @@
+#ifndef _RISCV_EMULATION_H
+#define _RISCV_EMULATION_H
+
+#include "encoding.h"
+#include "bits.h"
+#include <stdint.h>
+
+typedef uint32_t insn_t;
+typedef void (*emulation_func)(uintptr_t*, uintptr_t, uintptr_t, uintptr_t, insn_t);
+#define DECLARE_EMULATION_FUNC(name) void name(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn)
+
+void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc);
+void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc);
+void redirect_trap(uintptr_t epc, uintptr_t mstatus);
+DECLARE_EMULATION_FUNC(truly_illegal_insn);
+
+#define GET_REG(insn, pos, regs) ({ \
+  int mask = (1 << (5+LOG_REGBYTES)) - (1 << LOG_REGBYTES); \
+  (uintptr_t*)((uintptr_t)regs + (((insn) >> ((pos) - LOG_REGBYTES)) & mask)); \
+})
+#define GET_RS1(insn, regs) (*GET_REG(insn, 15, regs))
+#define GET_RS2(insn, regs) (*GET_REG(insn, 20, regs))
+#define SET_RD(insn, regs, val) (*GET_REG(insn, 7, regs) = (val))
+#define IMM_I(insn) ((int32_t)(insn) >> 20)
+#define IMM_S(insn) (((int32_t)(insn) >> 25 << 5) | (int32_t)(((insn) >> 7) & 0x1f))
+#define MASK_FUNCT3 0x7000
+
+#endif
diff --git a/machine/encoding.h b/machine/encoding.h
new file mode 100644
index 0000000..f2fab36
--- /dev/null
+++ b/machine/encoding.h
@@ -0,0 +1,1037 @@
+// See LICENSE for license details.
+
+#ifndef RISCV_CSR_ENCODING_H
+#define RISCV_CSR_ENCODING_H
+
+#define MSTATUS_UIE         0x00000001
+#define MSTATUS_SIE         0x00000002
+#define MSTATUS_HIE         0x00000004
+#define MSTATUS_MIE         0x00000008
+#define MSTATUS_UPIE        0x00000010
+#define MSTATUS_SPIE        0x00000020
+#define MSTATUS_HPIE        0x00000040
+#define MSTATUS_MPIE        0x00000080
+#define MSTATUS_SPP         0x00000100
+#define MSTATUS_HPP         0x00000600
+#define MSTATUS_MPP         0x00001800
+#define MSTATUS_FS          0x00006000
+#define MSTATUS_XS          0x00018000
+#define MSTATUS_MPRV        0x00020000
+#define MSTATUS_PUM         0x00040000
+#define MSTATUS_VM          0x1F000000
+#define MSTATUS32_SD        0x80000000
+#define MSTATUS64_SD        0x8000000000000000
+
+#define SSTATUS_UIE         0x00000001
+#define SSTATUS_SIE         0x00000002
+#define SSTATUS_UPIE        0x00000010
+#define SSTATUS_SPIE        0x00000020
+#define SSTATUS_SPP         0x00000100
+#define SSTATUS_FS          0x00006000
+#define SSTATUS_XS          0x00018000
+#define SSTATUS_PUM         0x00040000
+#define SSTATUS32_SD        0x80000000
+#define SSTATUS64_SD        0x8000000000000000
+
+#define MIP_SSIP            (1 << IRQ_S_SOFT)
+#define MIP_HSIP            (1 << IRQ_H_SOFT)
+#define MIP_MSIP            (1 << IRQ_M_SOFT)
+#define MIP_STIP            (1 << IRQ_S_TIMER)
+#define MIP_HTIP            (1 << IRQ_H_TIMER)
+#define MIP_MTIP            (1 << IRQ_M_TIMER)
+
+#define SIP_SSIP MIP_SSIP
+#define SIP_STIP MIP_STIP
+
+#define PRV_U 0
+#define PRV_S 1
+#define PRV_H 2
+#define PRV_M 3
+
+#define VM_MBARE 0
+#define VM_MBB   1
+#define VM_MBBID 2
+#define VM_SV32  8
+#define VM_SV39  9
+#define VM_SV48  10
+
+#define IRQ_S_SOFT   1
+#define IRQ_H_SOFT   2
+#define IRQ_M_SOFT   3
+#define IRQ_S_TIMER  5
+#define IRQ_H_TIMER  6
+#define IRQ_M_TIMER  7
+#define IRQ_S_DEV    9
+#define IRQ_H_DEV    10
+#define IRQ_M_DEV    11
+#define IRQ_COP      12
+#define IRQ_HOST     13
+
+#define DEFAULT_RSTVEC 0x0
+#define DEFAULT_NMIVEC 0x4
+#define DEFAULT_MTVEC  0x8
+
+// page table entry (PTE) fields
+#define PTE_V     0x001 // Valid
+#define PTE_TYPE  0x01E // Type
+#define PTE_R     0x020 // Referenced
+#define PTE_D     0x040 // Dirty
+#define PTE_SOFT  0x380 // Reserved for Software
+
+#define PTE_TYPE_TABLE        0x00
+#define PTE_TYPE_TABLE_GLOBAL 0x02
+#define PTE_TYPE_URX_SR       0x04
+#define PTE_TYPE_URWX_SRW     0x06
+#define PTE_TYPE_UR_SR        0x08
+#define PTE_TYPE_URW_SRW      0x0A
+#define PTE_TYPE_URX_SRX      0x0C
+#define PTE_TYPE_URWX_SRWX    0x0E
+#define PTE_TYPE_SR           0x10
+#define PTE_TYPE_SRW          0x12
+#define PTE_TYPE_SRX          0x14
+#define PTE_TYPE_SRWX         0x16
+#define PTE_TYPE_SR_GLOBAL    0x18
+#define PTE_TYPE_SRW_GLOBAL   0x1A
+#define PTE_TYPE_SRX_GLOBAL   0x1C
+#define PTE_TYPE_SRWX_GLOBAL  0x1E
+
+#define PTE_PPN_SHIFT 10
+
+#define PTE_TABLE(PTE) ((0x0000000AU >> ((PTE) & 0x1F)) & 1)
+#define PTE_UR(PTE)    ((0x0000AAA0U >> ((PTE) & 0x1F)) & 1)
+#define PTE_UW(PTE)    ((0x00008880U >> ((PTE) & 0x1F)) & 1)
+#define PTE_UX(PTE)    ((0x0000A0A0U >> ((PTE) & 0x1F)) & 1)
+#define PTE_SR(PTE)    ((0xAAAAAAA0U >> ((PTE) & 0x1F)) & 1)
+#define PTE_SW(PTE)    ((0x88888880U >> ((PTE) & 0x1F)) & 1)
+#define PTE_SX(PTE)    ((0xA0A0A000U >> ((PTE) & 0x1F)) & 1)
+
+#define PTE_CHECK_PERM(PTE, SUPERVISOR, STORE, FETCH) \
+  ((STORE) ? ((SUPERVISOR) ? PTE_SW(PTE) : PTE_UW(PTE)) : \
+   (FETCH) ? ((SUPERVISOR) ? PTE_SX(PTE) : PTE_UX(PTE)) : \
+             ((SUPERVISOR) ? PTE_SR(PTE) : PTE_UR(PTE)))
+
+#ifdef __riscv
+
+#ifdef __riscv64
+# define MSTATUS_SD MSTATUS64_SD
+# define SSTATUS_SD SSTATUS64_SD
+# define RISCV_PGLEVEL_BITS 9
+#else
+# define MSTATUS_SD MSTATUS32_SD
+# define SSTATUS_SD SSTATUS32_SD
+# define RISCV_PGLEVEL_BITS 10
+#endif
+#define RISCV_PGSHIFT 12
+#define RISCV_PGSIZE (1 << RISCV_PGSHIFT)
+
+#ifndef __ASSEMBLER__
+
+#ifdef __GNUC__
+
+#define read_csr(reg) ({ unsigned long __tmp; \
+  asm volatile ("csrr %0, " #reg : "=r"(__tmp)); \
+  __tmp; })
+
+#define write_csr(reg, val) \
+  asm volatile ("csrw " #reg ", %0" :: "r"(val))
+
+#define swap_csr(reg, val) ({ long __tmp; \
+  asm volatile ("csrrw %0, " #reg ", %1" : "=r"(__tmp) : "r"(val)); \
+  __tmp; })
+
+#define set_csr(reg, bit) ({ unsigned long __tmp; \
+  if (__builtin_constant_p(bit) && (bit) < 32) \
+    asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \
+  else \
+    asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \
+  __tmp; })
+
+#define clear_csr(reg, bit) ({ unsigned long __tmp; \
+  if (__builtin_constant_p(bit) && (bit) < 32) \
+    asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \
+  else \
+    asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \
+  __tmp; })
+
+#define rdtime() read_csr(time)
+#define rdcycle() read_csr(cycle)
+#define rdinstret() read_csr(instret)
+
+#endif
+
+#endif
+
+#endif
+
+#endif
+/* Automatically generated by parse-opcodes */
+#ifndef RISCV_ENCODING_H
+#define RISCV_ENCODING_H
+#define MATCH_BEQ 0x63
+#define MASK_BEQ  0x707f
+#define MATCH_BNE 0x1063
+#define MASK_BNE  0x707f
+#define MATCH_BLT 0x4063
+#define MASK_BLT  0x707f
+#define MATCH_BGE 0x5063
+#define MASK_BGE  0x707f
+#define MATCH_BLTU 0x6063
+#define MASK_BLTU  0x707f
+#define MATCH_BGEU 0x7063
+#define MASK_BGEU  0x707f
+#define MATCH_JALR 0x67
+#define MASK_JALR  0x707f
+#define MATCH_JAL 0x6f
+#define MASK_JAL  0x7f
+#define MATCH_LUI 0x37
+#define MASK_LUI  0x7f
+#define MATCH_AUIPC 0x17
+#define MASK_AUIPC  0x7f
+#define MATCH_ADDI 0x13
+#define MASK_ADDI  0x707f
+#define MATCH_SLLI 0x1013
+#define MASK_SLLI  0xfc00707f
+#define MATCH_SLTI 0x2013
+#define MASK_SLTI  0x707f
+#define MATCH_SLTIU 0x3013
+#define MASK_SLTIU  0x707f
+#define MATCH_XORI 0x4013
+#define MASK_XORI  0x707f
+#define MATCH_SRLI 0x5013
+#define MASK_SRLI  0xfc00707f
+#define MATCH_SRAI 0x40005013
+#define MASK_SRAI  0xfc00707f
+#define MATCH_ORI 0x6013
+#define MASK_ORI  0x707f
+#define MATCH_ANDI 0x7013
+#define MASK_ANDI  0x707f
+#define MATCH_ADD 0x33
+#define MASK_ADD  0xfe00707f
+#define MATCH_SUB 0x40000033
+#define MASK_SUB  0xfe00707f
+#define MATCH_SLL 0x1033
+#define MASK_SLL  0xfe00707f
+#define MATCH_SLT 0x2033
+#define MASK_SLT  0xfe00707f
+#define MATCH_SLTU 0x3033
+#define MASK_SLTU  0xfe00707f
+#define MATCH_XOR 0x4033
+#define MASK_XOR  0xfe00707f
+#define MATCH_SRL 0x5033
+#define MASK_SRL  0xfe00707f
+#define MATCH_SRA 0x40005033
+#define MASK_SRA  0xfe00707f
+#define MATCH_OR 0x6033
+#define MASK_OR  0xfe00707f
+#define MATCH_AND 0x7033
+#define MASK_AND  0xfe00707f
+#define MATCH_ADDIW 0x1b
+#define MASK_ADDIW  0x707f
+#define MATCH_SLLIW 0x101b
+#define MASK_SLLIW  0xfe00707f
+#define MATCH_SRLIW 0x501b
+#define MASK_SRLIW  0xfe00707f
+#define MATCH_SRAIW 0x4000501b
+#define MASK_SRAIW  0xfe00707f
+#define MATCH_ADDW 0x3b
+#define MASK_ADDW  0xfe00707f
+#define MATCH_SUBW 0x4000003b
+#define MASK_SUBW  0xfe00707f
+#define MATCH_SLLW 0x103b
+#define MASK_SLLW  0xfe00707f
+#define MATCH_SRLW 0x503b
+#define MASK_SRLW  0xfe00707f
+#define MATCH_SRAW 0x4000503b
+#define MASK_SRAW  0xfe00707f
+#define MATCH_LB 0x3
+#define MASK_LB  0x707f
+#define MATCH_LH 0x1003
+#define MASK_LH  0x707f
+#define MATCH_LW 0x2003
+#define MASK_LW  0x707f
+#define MATCH_LD 0x3003
+#define MASK_LD  0x707f
+#define MATCH_LBU 0x4003
+#define MASK_LBU  0x707f
+#define MATCH_LHU 0x5003
+#define MASK_LHU  0x707f
+#define MATCH_LWU 0x6003
+#define MASK_LWU  0x707f
+#define MATCH_SB 0x23
+#define MASK_SB  0x707f
+#define MATCH_SH 0x1023
+#define MASK_SH  0x707f
+#define MATCH_SW 0x2023
+#define MASK_SW  0x707f
+#define MATCH_SD 0x3023
+#define MASK_SD  0x707f
+#define MATCH_FENCE 0xf
+#define MASK_FENCE  0x707f
+#define MATCH_FENCE_I 0x100f
+#define MASK_FENCE_I  0x707f
+#define MATCH_MUL 0x2000033
+#define MASK_MUL  0xfe00707f
+#define MATCH_MULH 0x2001033
+#define MASK_MULH  0xfe00707f
+#define MATCH_MULHSU 0x2002033
+#define MASK_MULHSU  0xfe00707f
+#define MATCH_MULHU 0x2003033
+#define MASK_MULHU  0xfe00707f
+#define MATCH_DIV 0x2004033
+#define MASK_DIV  0xfe00707f
+#define MATCH_DIVU 0x2005033
+#define MASK_DIVU  0xfe00707f
+#define MATCH_REM 0x2006033
+#define MASK_REM  0xfe00707f
+#define MATCH_REMU 0x2007033
+#define MASK_REMU  0xfe00707f
+#define MATCH_MULW 0x200003b
+#define MASK_MULW  0xfe00707f
+#define MATCH_DIVW 0x200403b
+#define MASK_DIVW  0xfe00707f
+#define MATCH_DIVUW 0x200503b
+#define MASK_DIVUW  0xfe00707f
+#define MATCH_REMW 0x200603b
+#define MASK_REMW  0xfe00707f
+#define MATCH_REMUW 0x200703b
+#define MASK_REMUW  0xfe00707f
+#define MATCH_AMOADD_W 0x202f
+#define MASK_AMOADD_W  0xf800707f
+#define MATCH_AMOXOR_W 0x2000202f
+#define MASK_AMOXOR_W  0xf800707f
+#define MATCH_AMOOR_W 0x4000202f
+#define MASK_AMOOR_W  0xf800707f
+#define MATCH_AMOAND_W 0x6000202f
+#define MASK_AMOAND_W  0xf800707f
+#define MATCH_AMOMIN_W 0x8000202f
+#define MASK_AMOMIN_W  0xf800707f
+#define MATCH_AMOMAX_W 0xa000202f
+#define MASK_AMOMAX_W  0xf800707f
+#define MATCH_AMOMINU_W 0xc000202f
+#define MASK_AMOMINU_W  0xf800707f
+#define MATCH_AMOMAXU_W 0xe000202f
+#define MASK_AMOMAXU_W  0xf800707f
+#define MATCH_AMOSWAP_W 0x800202f
+#define MASK_AMOSWAP_W  0xf800707f
+#define MATCH_LR_W 0x1000202f
+#define MASK_LR_W  0xf9f0707f
+#define MATCH_SC_W 0x1800202f
+#define MASK_SC_W  0xf800707f
+#define MATCH_AMOADD_D 0x302f
+#define MASK_AMOADD_D  0xf800707f
+#define MATCH_AMOXOR_D 0x2000302f
+#define MASK_AMOXOR_D  0xf800707f
+#define MATCH_AMOOR_D 0x4000302f
+#define MASK_AMOOR_D  0xf800707f
+#define MATCH_AMOAND_D 0x6000302f
+#define MASK_AMOAND_D  0xf800707f
+#define MATCH_AMOMIN_D 0x8000302f
+#define MASK_AMOMIN_D  0xf800707f
+#define MATCH_AMOMAX_D 0xa000302f
+#define MASK_AMOMAX_D  0xf800707f
+#define MATCH_AMOMINU_D 0xc000302f
+#define MASK_AMOMINU_D  0xf800707f
+#define MATCH_AMOMAXU_D 0xe000302f
+#define MASK_AMOMAXU_D  0xf800707f
+#define MATCH_AMOSWAP_D 0x800302f
+#define MASK_AMOSWAP_D  0xf800707f
+#define MATCH_LR_D 0x1000302f
+#define MASK_LR_D  0xf9f0707f
+#define MATCH_SC_D 0x1800302f
+#define MASK_SC_D  0xf800707f
+#define MATCH_SCALL 0x73
+#define MASK_SCALL  0xffffffff
+#define MATCH_SBREAK 0x100073
+#define MASK_SBREAK  0xffffffff
+#define MATCH_SRET 0x10200073
+#define MASK_SRET  0xffffffff
+#define MATCH_SFENCE_VM 0x10400073
+#define MASK_SFENCE_VM  0xfff07fff
+#define MATCH_WFI 0x10500073
+#define MASK_WFI  0xffffffff
+#define MATCH_CSRRW 0x1073
+#define MASK_CSRRW  0x707f
+#define MATCH_CSRRS 0x2073
+#define MASK_CSRRS  0x707f
+#define MATCH_CSRRC 0x3073
+#define MASK_CSRRC  0x707f
+#define MATCH_CSRRWI 0x5073
+#define MASK_CSRRWI  0x707f
+#define MATCH_CSRRSI 0x6073
+#define MASK_CSRRSI  0x707f
+#define MATCH_CSRRCI 0x7073
+#define MASK_CSRRCI  0x707f
+#define MATCH_FADD_S 0x53
+#define MASK_FADD_S  0xfe00007f
+#define MATCH_FSUB_S 0x8000053
+#define MASK_FSUB_S  0xfe00007f
+#define MATCH_FMUL_S 0x10000053
+#define MASK_FMUL_S  0xfe00007f
+#define MATCH_FDIV_S 0x18000053
+#define MASK_FDIV_S  0xfe00007f
+#define MATCH_FSGNJ_S 0x20000053
+#define MASK_FSGNJ_S  0xfe00707f
+#define MATCH_FSGNJN_S 0x20001053
+#define MASK_FSGNJN_S  0xfe00707f
+#define MATCH_FSGNJX_S 0x20002053
+#define MASK_FSGNJX_S  0xfe00707f
+#define MATCH_FMIN_S 0x28000053
+#define MASK_FMIN_S  0xfe00707f
+#define MATCH_FMAX_S 0x28001053
+#define MASK_FMAX_S  0xfe00707f
+#define MATCH_FSQRT_S 0x58000053
+#define MASK_FSQRT_S  0xfff0007f
+#define MATCH_FADD_D 0x2000053
+#define MASK_FADD_D  0xfe00007f
+#define MATCH_FSUB_D 0xa000053
+#define MASK_FSUB_D  0xfe00007f
+#define MATCH_FMUL_D 0x12000053
+#define MASK_FMUL_D  0xfe00007f
+#define MATCH_FDIV_D 0x1a000053
+#define MASK_FDIV_D  0xfe00007f
+#define MATCH_FSGNJ_D 0x22000053
+#define MASK_FSGNJ_D  0xfe00707f
+#define MATCH_FSGNJN_D 0x22001053
+#define MASK_FSGNJN_D  0xfe00707f
+#define MATCH_FSGNJX_D 0x22002053
+#define MASK_FSGNJX_D  0xfe00707f
+#define MATCH_FMIN_D 0x2a000053
+#define MASK_FMIN_D  0xfe00707f
+#define MATCH_FMAX_D 0x2a001053
+#define MASK_FMAX_D  0xfe00707f
+#define MATCH_FCVT_S_D 0x40100053
+#define MASK_FCVT_S_D  0xfff0007f
+#define MATCH_FCVT_D_S 0x42000053
+#define MASK_FCVT_D_S  0xfff0007f
+#define MATCH_FSQRT_D 0x5a000053
+#define MASK_FSQRT_D  0xfff0007f
+#define MATCH_FLE_S 0xa0000053
+#define MASK_FLE_S  0xfe00707f
+#define MATCH_FLT_S 0xa0001053
+#define MASK_FLT_S  0xfe00707f
+#define MATCH_FEQ_S 0xa0002053
+#define MASK_FEQ_S  0xfe00707f
+#define MATCH_FLE_D 0xa2000053
+#define MASK_FLE_D  0xfe00707f
+#define MATCH_FLT_D 0xa2001053
+#define MASK_FLT_D  0xfe00707f
+#define MATCH_FEQ_D 0xa2002053
+#define MASK_FEQ_D  0xfe00707f
+#define MATCH_FCVT_W_S 0xc0000053
+#define MASK_FCVT_W_S  0xfff0007f
+#define MATCH_FCVT_WU_S 0xc0100053
+#define MASK_FCVT_WU_S  0xfff0007f
+#define MATCH_FCVT_L_S 0xc0200053
+#define MASK_FCVT_L_S  0xfff0007f
+#define MATCH_FCVT_LU_S 0xc0300053
+#define MASK_FCVT_LU_S  0xfff0007f
+#define MATCH_FMV_X_S 0xe0000053
+#define MASK_FMV_X_S  0xfff0707f
+#define MATCH_FCLASS_S 0xe0001053
+#define MASK_FCLASS_S  0xfff0707f
+#define MATCH_FCVT_W_D 0xc2000053
+#define MASK_FCVT_W_D  0xfff0007f
+#define MATCH_FCVT_WU_D 0xc2100053
+#define MASK_FCVT_WU_D  0xfff0007f
+#define MATCH_FCVT_L_D 0xc2200053
+#define MASK_FCVT_L_D  0xfff0007f
+#define MATCH_FCVT_LU_D 0xc2300053
+#define MASK_FCVT_LU_D  0xfff0007f
+#define MATCH_FMV_X_D 0xe2000053
+#define MASK_FMV_X_D  0xfff0707f
+#define MATCH_FCLASS_D 0xe2001053
+#define MASK_FCLASS_D  0xfff0707f
+#define MATCH_FCVT_S_W 0xd0000053
+#define MASK_FCVT_S_W  0xfff0007f
+#define MATCH_FCVT_S_WU 0xd0100053
+#define MASK_FCVT_S_WU  0xfff0007f
+#define MATCH_FCVT_S_L 0xd0200053
+#define MASK_FCVT_S_L  0xfff0007f
+#define MATCH_FCVT_S_LU 0xd0300053
+#define MASK_FCVT_S_LU  0xfff0007f
+#define MATCH_FMV_S_X 0xf0000053
+#define MASK_FMV_S_X  0xfff0707f
+#define MATCH_FCVT_D_W 0xd2000053
+#define MASK_FCVT_D_W  0xfff0007f
+#define MATCH_FCVT_D_WU 0xd2100053
+#define MASK_FCVT_D_WU  0xfff0007f
+#define MATCH_FCVT_D_L 0xd2200053
+#define MASK_FCVT_D_L  0xfff0007f
+#define MATCH_FCVT_D_LU 0xd2300053
+#define MASK_FCVT_D_LU  0xfff0007f
+#define MATCH_FMV_D_X 0xf2000053
+#define MASK_FMV_D_X  0xfff0707f
+#define MATCH_FLW 0x2007
+#define MASK_FLW  0x707f
+#define MATCH_FLD 0x3007
+#define MASK_FLD  0x707f
+#define MATCH_FSW 0x2027
+#define MASK_FSW  0x707f
+#define MATCH_FSD 0x3027
+#define MASK_FSD  0x707f
+#define MATCH_FMADD_S 0x43
+#define MASK_FMADD_S  0x600007f
+#define MATCH_FMSUB_S 0x47
+#define MASK_FMSUB_S  0x600007f
+#define MATCH_FNMSUB_S 0x4b
+#define MASK_FNMSUB_S  0x600007f
+#define MATCH_FNMADD_S 0x4f
+#define MASK_FNMADD_S  0x600007f
+#define MATCH_FMADD_D 0x2000043
+#define MASK_FMADD_D  0x600007f
+#define MATCH_FMSUB_D 0x2000047
+#define MASK_FMSUB_D  0x600007f
+#define MATCH_FNMSUB_D 0x200004b
+#define MASK_FNMSUB_D  0x600007f
+#define MATCH_FNMADD_D 0x200004f
+#define MASK_FNMADD_D  0x600007f
+#define MATCH_C_NOP 0x1
+#define MASK_C_NOP  0xffff
+#define MATCH_C_ADDI16SP 0x6101
+#define MASK_C_ADDI16SP  0xef83
+#define MATCH_C_JR 0x8002
+#define MASK_C_JR  0xf07f
+#define MATCH_C_JALR 0x9002
+#define MASK_C_JALR  0xf07f
+#define MATCH_C_EBREAK 0x9002
+#define MASK_C_EBREAK  0xffff
+#define MATCH_C_LD 0x6000
+#define MASK_C_LD  0xe003
+#define MATCH_C_SD 0xe000
+#define MASK_C_SD  0xe003
+#define MATCH_C_ADDIW 0x2001
+#define MASK_C_ADDIW  0xe003
+#define MATCH_C_LDSP 0x6002
+#define MASK_C_LDSP  0xe003
+#define MATCH_C_SDSP 0xe002
+#define MASK_C_SDSP  0xe003
+#define MATCH_C_ADDI4SPN 0x0
+#define MASK_C_ADDI4SPN  0xe003
+#define MATCH_C_FLD 0x2000
+#define MASK_C_FLD  0xe003
+#define MATCH_C_LW 0x4000
+#define MASK_C_LW  0xe003
+#define MATCH_C_FLW 0x6000
+#define MASK_C_FLW  0xe003
+#define MATCH_C_FSD 0xa000
+#define MASK_C_FSD  0xe003
+#define MATCH_C_SW 0xc000
+#define MASK_C_SW  0xe003
+#define MATCH_C_FSW 0xe000
+#define MASK_C_FSW  0xe003
+#define MATCH_C_ADDI 0x1
+#define MASK_C_ADDI  0xe003
+#define MATCH_C_JAL 0x2001
+#define MASK_C_JAL  0xe003
+#define MATCH_C_LI 0x4001
+#define MASK_C_LI  0xe003
+#define MATCH_C_LUI 0x6001
+#define MASK_C_LUI  0xe003
+#define MATCH_C_SRLI 0x8001
+#define MASK_C_SRLI  0xec03
+#define MATCH_C_SRAI 0x8401
+#define MASK_C_SRAI  0xec03
+#define MATCH_C_ANDI 0x8801
+#define MASK_C_ANDI  0xec03
+#define MATCH_C_SUB 0x8c01
+#define MASK_C_SUB  0xfc63
+#define MATCH_C_XOR 0x8c21
+#define MASK_C_XOR  0xfc63
+#define MATCH_C_OR 0x8c41
+#define MASK_C_OR  0xfc63
+#define MATCH_C_AND 0x8c61
+#define MASK_C_AND  0xfc63
+#define MATCH_C_SUBW 0x9c01
+#define MASK_C_SUBW  0xfc63
+#define MATCH_C_ADDW 0x9c21
+#define MASK_C_ADDW  0xfc63
+#define MATCH_C_J 0xa001
+#define MASK_C_J  0xe003
+#define MATCH_C_BEQZ 0xc001
+#define MASK_C_BEQZ  0xe003
+#define MATCH_C_BNEZ 0xe001
+#define MASK_C_BNEZ  0xe003
+#define MATCH_C_SLLI 0x2
+#define MASK_C_SLLI  0xe003
+#define MATCH_C_FLDSP 0x2002
+#define MASK_C_FLDSP  0xe003
+#define MATCH_C_LWSP 0x4002
+#define MASK_C_LWSP  0xe003
+#define MATCH_C_FLWSP 0x6002
+#define MASK_C_FLWSP  0xe003
+#define MATCH_C_MV 0x8002
+#define MASK_C_MV  0xf003
+#define MATCH_C_ADD 0x9002
+#define MASK_C_ADD  0xf003
+#define MATCH_C_FSDSP 0xa002
+#define MASK_C_FSDSP  0xe003
+#define MATCH_C_SWSP 0xc002
+#define MASK_C_SWSP  0xe003
+#define MATCH_C_FSWSP 0xe002
+#define MASK_C_FSWSP  0xe003
+#define MATCH_CUSTOM0 0xb
+#define MASK_CUSTOM0  0x707f
+#define MATCH_CUSTOM0_RS1 0x200b
+#define MASK_CUSTOM0_RS1  0x707f
+#define MATCH_CUSTOM0_RS1_RS2 0x300b
+#define MASK_CUSTOM0_RS1_RS2  0x707f
+#define MATCH_CUSTOM0_RD 0x400b
+#define MASK_CUSTOM0_RD  0x707f
+#define MATCH_CUSTOM0_RD_RS1 0x600b
+#define MASK_CUSTOM0_RD_RS1  0x707f
+#define MATCH_CUSTOM0_RD_RS1_RS2 0x700b
+#define MASK_CUSTOM0_RD_RS1_RS2  0x707f
+#define MATCH_CUSTOM1 0x2b
+#define MASK_CUSTOM1  0x707f
+#define MATCH_CUSTOM1_RS1 0x202b
+#define MASK_CUSTOM1_RS1  0x707f
+#define MATCH_CUSTOM1_RS1_RS2 0x302b
+#define MASK_CUSTOM1_RS1_RS2  0x707f
+#define MATCH_CUSTOM1_RD 0x402b
+#define MASK_CUSTOM1_RD  0x707f
+#define MATCH_CUSTOM1_RD_RS1 0x602b
+#define MASK_CUSTOM1_RD_RS1  0x707f
+#define MATCH_CUSTOM1_RD_RS1_RS2 0x702b
+#define MASK_CUSTOM1_RD_RS1_RS2  0x707f
+#define MATCH_CUSTOM2 0x5b
+#define MASK_CUSTOM2  0x707f
+#define MATCH_CUSTOM2_RS1 0x205b
+#define MASK_CUSTOM2_RS1  0x707f
+#define MATCH_CUSTOM2_RS1_RS2 0x305b
+#define MASK_CUSTOM2_RS1_RS2  0x707f
+#define MATCH_CUSTOM2_RD 0x405b
+#define MASK_CUSTOM2_RD  0x707f
+#define MATCH_CUSTOM2_RD_RS1 0x605b
+#define MASK_CUSTOM2_RD_RS1  0x707f
+#define MATCH_CUSTOM2_RD_RS1_RS2 0x705b
+#define MASK_CUSTOM2_RD_RS1_RS2  0x707f
+#define MATCH_CUSTOM3 0x7b
+#define MASK_CUSTOM3  0x707f
+#define MATCH_CUSTOM3_RS1 0x207b
+#define MASK_CUSTOM3_RS1  0x707f
+#define MATCH_CUSTOM3_RS1_RS2 0x307b
+#define MASK_CUSTOM3_RS1_RS2  0x707f
+#define MATCH_CUSTOM3_RD 0x407b
+#define MASK_CUSTOM3_RD  0x707f
+#define MATCH_CUSTOM3_RD_RS1 0x607b
+#define MASK_CUSTOM3_RD_RS1  0x707f
+#define MATCH_CUSTOM3_RD_RS1_RS2 0x707b
+#define MASK_CUSTOM3_RD_RS1_RS2  0x707f
+#define CSR_FFLAGS 0x1
+#define CSR_FRM 0x2
+#define CSR_FCSR 0x3
+#define CSR_CYCLE 0xc00
+#define CSR_TIME 0xc01
+#define CSR_INSTRET 0xc02
+#define CSR_STATS 0xc0
+#define CSR_UARCH0 0xcc0
+#define CSR_UARCH1 0xcc1
+#define CSR_UARCH2 0xcc2
+#define CSR_UARCH3 0xcc3
+#define CSR_UARCH4 0xcc4
+#define CSR_UARCH5 0xcc5
+#define CSR_UARCH6 0xcc6
+#define CSR_UARCH7 0xcc7
+#define CSR_UARCH8 0xcc8
+#define CSR_UARCH9 0xcc9
+#define CSR_UARCH10 0xcca
+#define CSR_UARCH11 0xccb
+#define CSR_UARCH12 0xccc
+#define CSR_UARCH13 0xccd
+#define CSR_UARCH14 0xcce
+#define CSR_UARCH15 0xccf
+#define CSR_SSTATUS 0x100
+#define CSR_SIE 0x104
+#define CSR_STVEC 0x105
+#define CSR_SSCRATCH 0x140
+#define CSR_SEPC 0x141
+#define CSR_SCAUSE 0x142
+#define CSR_SBADADDR 0x143
+#define CSR_SIP 0x144
+#define CSR_SPTBR 0x180
+#define CSR_SASID 0x181
+#define CSR_SCYCLE 0xd00
+#define CSR_STIME 0xd01
+#define CSR_SINSTRET 0xd02
+#define CSR_MSTATUS 0x300
+#define CSR_MEDELEG 0x302
+#define CSR_MIDELEG 0x303
+#define CSR_MIE 0x304
+#define CSR_MTVEC 0x305
+#define CSR_MTIMECMP 0x321
+#define CSR_MSCRATCH 0x340
+#define CSR_MEPC 0x341
+#define CSR_MCAUSE 0x342
+#define CSR_MBADADDR 0x343
+#define CSR_MIP 0x344
+#define CSR_MIPI 0x345
+#define CSR_MUCOUNTEREN 0x310
+#define CSR_MSCOUNTEREN 0x311
+#define CSR_MUCYCLE_DELTA 0x700
+#define CSR_MUTIME_DELTA 0x701
+#define CSR_MUINSTRET_DELTA 0x702
+#define CSR_MSCYCLE_DELTA 0x704
+#define CSR_MSTIME_DELTA 0x705
+#define CSR_MSINSTRET_DELTA 0x706
+#define CSR_MCYCLE 0xf00
+#define CSR_MTIME 0xf01
+#define CSR_MINSTRET 0xf02
+#define CSR_MISA 0xf10
+#define CSR_MVENDORID 0xf11
+#define CSR_MARCHID 0xf12
+#define CSR_MIMPID 0xf13
+#define CSR_MCFGADDR 0xf14
+#define CSR_MHARTID 0xf15
+#define CSR_MTOHOST 0x7c0
+#define CSR_MFROMHOST 0x7c1
+#define CSR_MRESET 0x7c2
+#define CSR_CYCLEH 0xc80
+#define CSR_TIMEH 0xc81
+#define CSR_INSTRETH 0xc82
+#define CSR_MTIMECMPH 0x361
+#define CSR_MUCYCLE_DELTAH 0x780
+#define CSR_MUTIME_DELTAH 0x781
+#define CSR_MUINSTRET_DELTAH 0x782
+#define CSR_MSCYCLE_DELTAH 0x784
+#define CSR_MSTIME_DELTAH 0x785
+#define CSR_MSINSTRET_DELTAH 0x786
+#define CSR_MCYCLEH 0xf80
+#define CSR_MTIMEH 0xf81
+#define CSR_MINSTRETH 0xf82
+#define CAUSE_MISALIGNED_FETCH 0x0
+#define CAUSE_FAULT_FETCH 0x1
+#define CAUSE_ILLEGAL_INSTRUCTION 0x2
+#define CAUSE_BREAKPOINT 0x3
+#define CAUSE_MISALIGNED_LOAD 0x4
+#define CAUSE_FAULT_LOAD 0x5
+#define CAUSE_MISALIGNED_STORE 0x6
+#define CAUSE_FAULT_STORE 0x7
+#define CAUSE_USER_ECALL 0x8
+#define CAUSE_SUPERVISOR_ECALL 0x9
+#define CAUSE_HYPERVISOR_ECALL 0xa
+#define CAUSE_MACHINE_ECALL 0xb
+#endif
+#ifdef DECLARE_INSN
+DECLARE_INSN(beq, MATCH_BEQ, MASK_BEQ)
+DECLARE_INSN(bne, MATCH_BNE, MASK_BNE)
+DECLARE_INSN(blt, MATCH_BLT, MASK_BLT)
+DECLARE_INSN(bge, MATCH_BGE, MASK_BGE)
+DECLARE_INSN(bltu, MATCH_BLTU, MASK_BLTU)
+DECLARE_INSN(bgeu, MATCH_BGEU, MASK_BGEU)
+DECLARE_INSN(jalr, MATCH_JALR, MASK_JALR)
+DECLARE_INSN(jal, MATCH_JAL, MASK_JAL)
+DECLARE_INSN(lui, MATCH_LUI, MASK_LUI)
+DECLARE_INSN(auipc, MATCH_AUIPC, MASK_AUIPC)
+DECLARE_INSN(addi, MATCH_ADDI, MASK_ADDI)
+DECLARE_INSN(slli, MATCH_SLLI, MASK_SLLI)
+DECLARE_INSN(slti, MATCH_SLTI, MASK_SLTI)
+DECLARE_INSN(sltiu, MATCH_SLTIU, MASK_SLTIU)
+DECLARE_INSN(xori, MATCH_XORI, MASK_XORI)
+DECLARE_INSN(srli, MATCH_SRLI, MASK_SRLI)
+DECLARE_INSN(srai, MATCH_SRAI, MASK_SRAI)
+DECLARE_INSN(ori, MATCH_ORI, MASK_ORI)
+DECLARE_INSN(andi, MATCH_ANDI, MASK_ANDI)
+DECLARE_INSN(add, MATCH_ADD, MASK_ADD)
+DECLARE_INSN(sub, MATCH_SUB, MASK_SUB)
+DECLARE_INSN(sll, MATCH_SLL, MASK_SLL)
+DECLARE_INSN(slt, MATCH_SLT, MASK_SLT)
+DECLARE_INSN(sltu, MATCH_SLTU, MASK_SLTU)
+DECLARE_INSN(xor, MATCH_XOR, MASK_XOR)
+DECLARE_INSN(srl, MATCH_SRL, MASK_SRL)
+DECLARE_INSN(sra, MATCH_SRA, MASK_SRA)
+DECLARE_INSN(or, MATCH_OR, MASK_OR)
+DECLARE_INSN(and, MATCH_AND, MASK_AND)
+DECLARE_INSN(addiw, MATCH_ADDIW, MASK_ADDIW)
+DECLARE_INSN(slliw, MATCH_SLLIW, MASK_SLLIW)
+DECLARE_INSN(srliw, MATCH_SRLIW, MASK_SRLIW)
+DECLARE_INSN(sraiw, MATCH_SRAIW, MASK_SRAIW)
+DECLARE_INSN(addw, MATCH_ADDW, MASK_ADDW)
+DECLARE_INSN(subw, MATCH_SUBW, MASK_SUBW)
+DECLARE_INSN(sllw, MATCH_SLLW, MASK_SLLW)
+DECLARE_INSN(srlw, MATCH_SRLW, MASK_SRLW)
+DECLARE_INSN(sraw, MATCH_SRAW, MASK_SRAW)
+DECLARE_INSN(lb, MATCH_LB, MASK_LB)
+DECLARE_INSN(lh, MATCH_LH, MASK_LH)
+DECLARE_INSN(lw, MATCH_LW, MASK_LW)
+DECLARE_INSN(ld, MATCH_LD, MASK_LD)
+DECLARE_INSN(lbu, MATCH_LBU, MASK_LBU)
+DECLARE_INSN(lhu, MATCH_LHU, MASK_LHU)
+DECLARE_INSN(lwu, MATCH_LWU, MASK_LWU)
+DECLARE_INSN(sb, MATCH_SB, MASK_SB)
+DECLARE_INSN(sh, MATCH_SH, MASK_SH)
+DECLARE_INSN(sw, MATCH_SW, MASK_SW)
+DECLARE_INSN(sd, MATCH_SD, MASK_SD)
+DECLARE_INSN(fence, MATCH_FENCE, MASK_FENCE)
+DECLARE_INSN(fence_i, MATCH_FENCE_I, MASK_FENCE_I)
+DECLARE_INSN(mul, MATCH_MUL, MASK_MUL)
+DECLARE_INSN(mulh, MATCH_MULH, MASK_MULH)
+DECLARE_INSN(mulhsu, MATCH_MULHSU, MASK_MULHSU)
+DECLARE_INSN(mulhu, MATCH_MULHU, MASK_MULHU)
+DECLARE_INSN(div, MATCH_DIV, MASK_DIV)
+DECLARE_INSN(divu, MATCH_DIVU, MASK_DIVU)
+DECLARE_INSN(rem, MATCH_REM, MASK_REM)
+DECLARE_INSN(remu, MATCH_REMU, MASK_REMU)
+DECLARE_INSN(mulw, MATCH_MULW, MASK_MULW)
+DECLARE_INSN(divw, MATCH_DIVW, MASK_DIVW)
+DECLARE_INSN(divuw, MATCH_DIVUW, MASK_DIVUW)
+DECLARE_INSN(remw, MATCH_REMW, MASK_REMW)
+DECLARE_INSN(remuw, MATCH_REMUW, MASK_REMUW)
+DECLARE_INSN(amoadd_w, MATCH_AMOADD_W, MASK_AMOADD_W)
+DECLARE_INSN(amoxor_w, MATCH_AMOXOR_W, MASK_AMOXOR_W)
+DECLARE_INSN(amoor_w, MATCH_AMOOR_W, MASK_AMOOR_W)
+DECLARE_INSN(amoand_w, MATCH_AMOAND_W, MASK_AMOAND_W)
+DECLARE_INSN(amomin_w, MATCH_AMOMIN_W, MASK_AMOMIN_W)
+DECLARE_INSN(amomax_w, MATCH_AMOMAX_W, MASK_AMOMAX_W)
+DECLARE_INSN(amominu_w, MATCH_AMOMINU_W, MASK_AMOMINU_W)
+DECLARE_INSN(amomaxu_w, MATCH_AMOMAXU_W, MASK_AMOMAXU_W)
+DECLARE_INSN(amoswap_w, MATCH_AMOSWAP_W, MASK_AMOSWAP_W)
+DECLARE_INSN(lr_w, MATCH_LR_W, MASK_LR_W)
+DECLARE_INSN(sc_w, MATCH_SC_W, MASK_SC_W)
+DECLARE_INSN(amoadd_d, MATCH_AMOADD_D, MASK_AMOADD_D)
+DECLARE_INSN(amoxor_d, MATCH_AMOXOR_D, MASK_AMOXOR_D)
+DECLARE_INSN(amoor_d, MATCH_AMOOR_D, MASK_AMOOR_D)
+DECLARE_INSN(amoand_d, MATCH_AMOAND_D, MASK_AMOAND_D)
+DECLARE_INSN(amomin_d, MATCH_AMOMIN_D, MASK_AMOMIN_D)
+DECLARE_INSN(amomax_d, MATCH_AMOMAX_D, MASK_AMOMAX_D)
+DECLARE_INSN(amominu_d, MATCH_AMOMINU_D, MASK_AMOMINU_D)
+DECLARE_INSN(amomaxu_d, MATCH_AMOMAXU_D, MASK_AMOMAXU_D)
+DECLARE_INSN(amoswap_d, MATCH_AMOSWAP_D, MASK_AMOSWAP_D)
+DECLARE_INSN(lr_d, MATCH_LR_D, MASK_LR_D)
+DECLARE_INSN(sc_d, MATCH_SC_D, MASK_SC_D)
+DECLARE_INSN(scall, MATCH_SCALL, MASK_SCALL)
+DECLARE_INSN(sbreak, MATCH_SBREAK, MASK_SBREAK)
+DECLARE_INSN(sret, MATCH_SRET, MASK_SRET)
+DECLARE_INSN(sfence_vm, MATCH_SFENCE_VM, MASK_SFENCE_VM)
+DECLARE_INSN(wfi, MATCH_WFI, MASK_WFI)
+DECLARE_INSN(csrrw, MATCH_CSRRW, MASK_CSRRW)
+DECLARE_INSN(csrrs, MATCH_CSRRS, MASK_CSRRS)
+DECLARE_INSN(csrrc, MATCH_CSRRC, MASK_CSRRC)
+DECLARE_INSN(csrrwi, MATCH_CSRRWI, MASK_CSRRWI)
+DECLARE_INSN(csrrsi, MATCH_CSRRSI, MASK_CSRRSI)
+DECLARE_INSN(csrrci, MATCH_CSRRCI, MASK_CSRRCI)
+DECLARE_INSN(fadd_s, MATCH_FADD_S, MASK_FADD_S)
+DECLARE_INSN(fsub_s, MATCH_FSUB_S, MASK_FSUB_S)
+DECLARE_INSN(fmul_s, MATCH_FMUL_S, MASK_FMUL_S)
+DECLARE_INSN(fdiv_s, MATCH_FDIV_S, MASK_FDIV_S)
+DECLARE_INSN(fsgnj_s, MATCH_FSGNJ_S, MASK_FSGNJ_S)
+DECLARE_INSN(fsgnjn_s, MATCH_FSGNJN_S, MASK_FSGNJN_S)
+DECLARE_INSN(fsgnjx_s, MATCH_FSGNJX_S, MASK_FSGNJX_S)
+DECLARE_INSN(fmin_s, MATCH_FMIN_S, MASK_FMIN_S)
+DECLARE_INSN(fmax_s, MATCH_FMAX_S, MASK_FMAX_S)
+DECLARE_INSN(fsqrt_s, MATCH_FSQRT_S, MASK_FSQRT_S)
+DECLARE_INSN(fadd_d, MATCH_FADD_D, MASK_FADD_D)
+DECLARE_INSN(fsub_d, MATCH_FSUB_D, MASK_FSUB_D)
+DECLARE_INSN(fmul_d, MATCH_FMUL_D, MASK_FMUL_D)
+DECLARE_INSN(fdiv_d, MATCH_FDIV_D, MASK_FDIV_D)
+DECLARE_INSN(fsgnj_d, MATCH_FSGNJ_D, MASK_FSGNJ_D)
+DECLARE_INSN(fsgnjn_d, MATCH_FSGNJN_D, MASK_FSGNJN_D)
+DECLARE_INSN(fsgnjx_d, MATCH_FSGNJX_D, MASK_FSGNJX_D)
+DECLARE_INSN(fmin_d, MATCH_FMIN_D, MASK_FMIN_D)
+DECLARE_INSN(fmax_d, MATCH_FMAX_D, MASK_FMAX_D)
+DECLARE_INSN(fcvt_s_d, MATCH_FCVT_S_D, MASK_FCVT_S_D)
+DECLARE_INSN(fcvt_d_s, MATCH_FCVT_D_S, MASK_FCVT_D_S)
+DECLARE_INSN(fsqrt_d, MATCH_FSQRT_D, MASK_FSQRT_D)
+DECLARE_INSN(fle_s, MATCH_FLE_S, MASK_FLE_S)
+DECLARE_INSN(flt_s, MATCH_FLT_S, MASK_FLT_S)
+DECLARE_INSN(feq_s, MATCH_FEQ_S, MASK_FEQ_S)
+DECLARE_INSN(fle_d, MATCH_FLE_D, MASK_FLE_D)
+DECLARE_INSN(flt_d, MATCH_FLT_D, MASK_FLT_D)
+DECLARE_INSN(feq_d, MATCH_FEQ_D, MASK_FEQ_D)
+DECLARE_INSN(fcvt_w_s, MATCH_FCVT_W_S, MASK_FCVT_W_S)
+DECLARE_INSN(fcvt_wu_s, MATCH_FCVT_WU_S, MASK_FCVT_WU_S)
+DECLARE_INSN(fcvt_l_s, MATCH_FCVT_L_S, MASK_FCVT_L_S)
+DECLARE_INSN(fcvt_lu_s, MATCH_FCVT_LU_S, MASK_FCVT_LU_S)
+DECLARE_INSN(fmv_x_s, MATCH_FMV_X_S, MASK_FMV_X_S)
+DECLARE_INSN(fclass_s, MATCH_FCLASS_S, MASK_FCLASS_S)
+DECLARE_INSN(fcvt_w_d, MATCH_FCVT_W_D, MASK_FCVT_W_D)
+DECLARE_INSN(fcvt_wu_d, MATCH_FCVT_WU_D, MASK_FCVT_WU_D)
+DECLARE_INSN(fcvt_l_d, MATCH_FCVT_L_D, MASK_FCVT_L_D)
+DECLARE_INSN(fcvt_lu_d, MATCH_FCVT_LU_D, MASK_FCVT_LU_D)
+DECLARE_INSN(fmv_x_d, MATCH_FMV_X_D, MASK_FMV_X_D)
+DECLARE_INSN(fclass_d, MATCH_FCLASS_D, MASK_FCLASS_D)
+DECLARE_INSN(fcvt_s_w, MATCH_FCVT_S_W, MASK_FCVT_S_W)
+DECLARE_INSN(fcvt_s_wu, MATCH_FCVT_S_WU, MASK_FCVT_S_WU)
+DECLARE_INSN(fcvt_s_l, MATCH_FCVT_S_L, MASK_FCVT_S_L)
+DECLARE_INSN(fcvt_s_lu, MATCH_FCVT_S_LU, MASK_FCVT_S_LU)
+DECLARE_INSN(fmv_s_x, MATCH_FMV_S_X, MASK_FMV_S_X)
+DECLARE_INSN(fcvt_d_w, MATCH_FCVT_D_W, MASK_FCVT_D_W)
+DECLARE_INSN(fcvt_d_wu, MATCH_FCVT_D_WU, MASK_FCVT_D_WU)
+DECLARE_INSN(fcvt_d_l, MATCH_FCVT_D_L, MASK_FCVT_D_L)
+DECLARE_INSN(fcvt_d_lu, MATCH_FCVT_D_LU, MASK_FCVT_D_LU)
+DECLARE_INSN(fmv_d_x, MATCH_FMV_D_X, MASK_FMV_D_X)
+DECLARE_INSN(flw, MATCH_FLW, MASK_FLW)
+DECLARE_INSN(fld, MATCH_FLD, MASK_FLD)
+DECLARE_INSN(fsw, MATCH_FSW, MASK_FSW)
+DECLARE_INSN(fsd, MATCH_FSD, MASK_FSD)
+DECLARE_INSN(fmadd_s, MATCH_FMADD_S, MASK_FMADD_S)
+DECLARE_INSN(fmsub_s, MATCH_FMSUB_S, MASK_FMSUB_S)
+DECLARE_INSN(fnmsub_s, MATCH_FNMSUB_S, MASK_FNMSUB_S)
+DECLARE_INSN(fnmadd_s, MATCH_FNMADD_S, MASK_FNMADD_S)
+DECLARE_INSN(fmadd_d, MATCH_FMADD_D, MASK_FMADD_D)
+DECLARE_INSN(fmsub_d, MATCH_FMSUB_D, MASK_FMSUB_D)
+DECLARE_INSN(fnmsub_d, MATCH_FNMSUB_D, MASK_FNMSUB_D)
+DECLARE_INSN(fnmadd_d, MATCH_FNMADD_D, MASK_FNMADD_D)
+DECLARE_INSN(c_nop, MATCH_C_NOP, MASK_C_NOP)
+DECLARE_INSN(c_addi16sp, MATCH_C_ADDI16SP, MASK_C_ADDI16SP)
+DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR)
+DECLARE_INSN(c_jalr, MATCH_C_JALR, MASK_C_JALR)
+DECLARE_INSN(c_ebreak, MATCH_C_EBREAK, MASK_C_EBREAK)
+DECLARE_INSN(c_ld, MATCH_C_LD, MASK_C_LD)
+DECLARE_INSN(c_sd, MATCH_C_SD, MASK_C_SD)
+DECLARE_INSN(c_addiw, MATCH_C_ADDIW, MASK_C_ADDIW)
+DECLARE_INSN(c_ldsp, MATCH_C_LDSP, MASK_C_LDSP)
+DECLARE_INSN(c_sdsp, MATCH_C_SDSP, MASK_C_SDSP)
+DECLARE_INSN(c_addi4spn, MATCH_C_ADDI4SPN, MASK_C_ADDI4SPN)
+DECLARE_INSN(c_fld, MATCH_C_FLD, MASK_C_FLD)
+DECLARE_INSN(c_lw, MATCH_C_LW, MASK_C_LW)
+DECLARE_INSN(c_flw, MATCH_C_FLW, MASK_C_FLW)
+DECLARE_INSN(c_fsd, MATCH_C_FSD, MASK_C_FSD)
+DECLARE_INSN(c_sw, MATCH_C_SW, MASK_C_SW)
+DECLARE_INSN(c_fsw, MATCH_C_FSW, MASK_C_FSW)
+DECLARE_INSN(c_addi, MATCH_C_ADDI, MASK_C_ADDI)
+DECLARE_INSN(c_jal, MATCH_C_JAL, MASK_C_JAL)
+DECLARE_INSN(c_li, MATCH_C_LI, MASK_C_LI)
+DECLARE_INSN(c_lui, MATCH_C_LUI, MASK_C_LUI)
+DECLARE_INSN(c_srli, MATCH_C_SRLI, MASK_C_SRLI)
+DECLARE_INSN(c_srai, MATCH_C_SRAI, MASK_C_SRAI)
+DECLARE_INSN(c_andi, MATCH_C_ANDI, MASK_C_ANDI)
+DECLARE_INSN(c_sub, MATCH_C_SUB, MASK_C_SUB)
+DECLARE_INSN(c_xor, MATCH_C_XOR, MASK_C_XOR)
+DECLARE_INSN(c_or, MATCH_C_OR, MASK_C_OR)
+DECLARE_INSN(c_and, MATCH_C_AND, MASK_C_AND)
+DECLARE_INSN(c_subw, MATCH_C_SUBW, MASK_C_SUBW)
+DECLARE_INSN(c_addw, MATCH_C_ADDW, MASK_C_ADDW)
+DECLARE_INSN(c_j, MATCH_C_J, MASK_C_J)
+DECLARE_INSN(c_beqz, MATCH_C_BEQZ, MASK_C_BEQZ)
+DECLARE_INSN(c_bnez, MATCH_C_BNEZ, MASK_C_BNEZ)
+DECLARE_INSN(c_slli, MATCH_C_SLLI, MASK_C_SLLI)
+DECLARE_INSN(c_fldsp, MATCH_C_FLDSP, MASK_C_FLDSP)
+DECLARE_INSN(c_lwsp, MATCH_C_LWSP, MASK_C_LWSP)
+DECLARE_INSN(c_flwsp, MATCH_C_FLWSP, MASK_C_FLWSP)
+DECLARE_INSN(c_mv, MATCH_C_MV, MASK_C_MV)
+DECLARE_INSN(c_add, MATCH_C_ADD, MASK_C_ADD)
+DECLARE_INSN(c_fsdsp, MATCH_C_FSDSP, MASK_C_FSDSP)
+DECLARE_INSN(c_swsp, MATCH_C_SWSP, MASK_C_SWSP)
+DECLARE_INSN(c_fswsp, MATCH_C_FSWSP, MASK_C_FSWSP)
+DECLARE_INSN(custom0, MATCH_CUSTOM0, MASK_CUSTOM0)
+DECLARE_INSN(custom0_rs1, MATCH_CUSTOM0_RS1, MASK_CUSTOM0_RS1)
+DECLARE_INSN(custom0_rs1_rs2, MATCH_CUSTOM0_RS1_RS2, MASK_CUSTOM0_RS1_RS2)
+DECLARE_INSN(custom0_rd, MATCH_CUSTOM0_RD, MASK_CUSTOM0_RD)
+DECLARE_INSN(custom0_rd_rs1, MATCH_CUSTOM0_RD_RS1, MASK_CUSTOM0_RD_RS1)
+DECLARE_INSN(custom0_rd_rs1_rs2, MATCH_CUSTOM0_RD_RS1_RS2, MASK_CUSTOM0_RD_RS1_RS2)
+DECLARE_INSN(custom1, MATCH_CUSTOM1, MASK_CUSTOM1)
+DECLARE_INSN(custom1_rs1, MATCH_CUSTOM1_RS1, MASK_CUSTOM1_RS1)
+DECLARE_INSN(custom1_rs1_rs2, MATCH_CUSTOM1_RS1_RS2, MASK_CUSTOM1_RS1_RS2)
+DECLARE_INSN(custom1_rd, MATCH_CUSTOM1_RD, MASK_CUSTOM1_RD)
+DECLARE_INSN(custom1_rd_rs1, MATCH_CUSTOM1_RD_RS1, MASK_CUSTOM1_RD_RS1)
+DECLARE_INSN(custom1_rd_rs1_rs2, MATCH_CUSTOM1_RD_RS1_RS2, MASK_CUSTOM1_RD_RS1_RS2)
+DECLARE_INSN(custom2, MATCH_CUSTOM2, MASK_CUSTOM2)
+DECLARE_INSN(custom2_rs1, MATCH_CUSTOM2_RS1, MASK_CUSTOM2_RS1)
+DECLARE_INSN(custom2_rs1_rs2, MATCH_CUSTOM2_RS1_RS2, MASK_CUSTOM2_RS1_RS2)
+DECLARE_INSN(custom2_rd, MATCH_CUSTOM2_RD, MASK_CUSTOM2_RD)
+DECLARE_INSN(custom2_rd_rs1, MATCH_CUSTOM2_RD_RS1, MASK_CUSTOM2_RD_RS1)
+DECLARE_INSN(custom2_rd_rs1_rs2, MATCH_CUSTOM2_RD_RS1_RS2, MASK_CUSTOM2_RD_RS1_RS2)
+DECLARE_INSN(custom3, MATCH_CUSTOM3, MASK_CUSTOM3)
+DECLARE_INSN(custom3_rs1, MATCH_CUSTOM3_RS1, MASK_CUSTOM3_RS1)
+DECLARE_INSN(custom3_rs1_rs2, MATCH_CUSTOM3_RS1_RS2, MASK_CUSTOM3_RS1_RS2)
+DECLARE_INSN(custom3_rd, MATCH_CUSTOM3_RD, MASK_CUSTOM3_RD)
+DECLARE_INSN(custom3_rd_rs1, MATCH_CUSTOM3_RD_RS1, MASK_CUSTOM3_RD_RS1)
+DECLARE_INSN(custom3_rd_rs1_rs2, MATCH_CUSTOM3_RD_RS1_RS2, MASK_CUSTOM3_RD_RS1_RS2)
+#endif
+#ifdef DECLARE_CSR
+DECLARE_CSR(fflags, CSR_FFLAGS)
+DECLARE_CSR(frm, CSR_FRM)
+DECLARE_CSR(fcsr, CSR_FCSR)
+DECLARE_CSR(cycle, CSR_CYCLE)
+DECLARE_CSR(time, CSR_TIME)
+DECLARE_CSR(instret, CSR_INSTRET)
+DECLARE_CSR(stats, CSR_STATS)
+DECLARE_CSR(uarch0, CSR_UARCH0)
+DECLARE_CSR(uarch1, CSR_UARCH1)
+DECLARE_CSR(uarch2, CSR_UARCH2)
+DECLARE_CSR(uarch3, CSR_UARCH3)
+DECLARE_CSR(uarch4, CSR_UARCH4)
+DECLARE_CSR(uarch5, CSR_UARCH5)
+DECLARE_CSR(uarch6, CSR_UARCH6)
+DECLARE_CSR(uarch7, CSR_UARCH7)
+DECLARE_CSR(uarch8, CSR_UARCH8)
+DECLARE_CSR(uarch9, CSR_UARCH9)
+DECLARE_CSR(uarch10, CSR_UARCH10)
+DECLARE_CSR(uarch11, CSR_UARCH11)
+DECLARE_CSR(uarch12, CSR_UARCH12)
+DECLARE_CSR(uarch13, CSR_UARCH13)
+DECLARE_CSR(uarch14, CSR_UARCH14)
+DECLARE_CSR(uarch15, CSR_UARCH15)
+DECLARE_CSR(sstatus, CSR_SSTATUS)
+DECLARE_CSR(sie, CSR_SIE)
+DECLARE_CSR(stvec, CSR_STVEC)
+DECLARE_CSR(sscratch, CSR_SSCRATCH)
+DECLARE_CSR(sepc, CSR_SEPC)
+DECLARE_CSR(scause, CSR_SCAUSE)
+DECLARE_CSR(sbadaddr, CSR_SBADADDR)
+DECLARE_CSR(sip, CSR_SIP)
+DECLARE_CSR(sptbr, CSR_SPTBR)
+DECLARE_CSR(sasid, CSR_SASID)
+DECLARE_CSR(scycle, CSR_SCYCLE)
+DECLARE_CSR(stime, CSR_STIME)
+DECLARE_CSR(sinstret, CSR_SINSTRET)
+DECLARE_CSR(mstatus, CSR_MSTATUS)
+DECLARE_CSR(medeleg, CSR_MEDELEG)
+DECLARE_CSR(mideleg, CSR_MIDELEG)
+DECLARE_CSR(mie, CSR_MIE)
+DECLARE_CSR(mtvec, CSR_MTVEC)
+DECLARE_CSR(mtimecmp, CSR_MTIMECMP)
+DECLARE_CSR(mscratch, CSR_MSCRATCH)
+DECLARE_CSR(mepc, CSR_MEPC)
+DECLARE_CSR(mcause, CSR_MCAUSE)
+DECLARE_CSR(mbadaddr, CSR_MBADADDR)
+DECLARE_CSR(mip, CSR_MIP)
+DECLARE_CSR(mipi, CSR_MIPI)
+DECLARE_CSR(mucounteren, CSR_MUCOUNTEREN)
+DECLARE_CSR(mscounteren, CSR_MSCOUNTEREN)
+DECLARE_CSR(mucycle_delta, CSR_MUCYCLE_DELTA)
+DECLARE_CSR(mutime_delta, CSR_MUTIME_DELTA)
+DECLARE_CSR(muinstret_delta, CSR_MUINSTRET_DELTA)
+DECLARE_CSR(mscycle_delta, CSR_MSCYCLE_DELTA)
+DECLARE_CSR(mstime_delta, CSR_MSTIME_DELTA)
+DECLARE_CSR(msinstret_delta, CSR_MSINSTRET_DELTA)
+DECLARE_CSR(mcycle, CSR_MCYCLE)
+DECLARE_CSR(mtime, CSR_MTIME)
+DECLARE_CSR(minstret, CSR_MINSTRET)
+DECLARE_CSR(misa, CSR_MISA)
+DECLARE_CSR(mvendorid, CSR_MVENDORID)
+DECLARE_CSR(marchid, CSR_MARCHID)
+DECLARE_CSR(mimpid, CSR_MIMPID)
+DECLARE_CSR(mcfgaddr, CSR_MCFGADDR)
+DECLARE_CSR(mhartid, CSR_MHARTID)
+DECLARE_CSR(mtohost, CSR_MTOHOST)
+DECLARE_CSR(mfromhost, CSR_MFROMHOST)
+DECLARE_CSR(mreset, CSR_MRESET)
+DECLARE_CSR(cycleh, CSR_CYCLEH)
+DECLARE_CSR(timeh, CSR_TIMEH)
+DECLARE_CSR(instreth, CSR_INSTRETH)
+DECLARE_CSR(mtimecmph, CSR_MTIMECMPH)
+DECLARE_CSR(mucycle_deltah, CSR_MUCYCLE_DELTAH)
+DECLARE_CSR(mutime_deltah, CSR_MUTIME_DELTAH)
+DECLARE_CSR(muinstret_deltah, CSR_MUINSTRET_DELTAH)
+DECLARE_CSR(mscycle_deltah, CSR_MSCYCLE_DELTAH)
+DECLARE_CSR(mstime_deltah, CSR_MSTIME_DELTAH)
+DECLARE_CSR(msinstret_deltah, CSR_MSINSTRET_DELTAH)
+DECLARE_CSR(mcycleh, CSR_MCYCLEH)
+DECLARE_CSR(mtimeh, CSR_MTIMEH)
+DECLARE_CSR(minstreth, CSR_MINSTRETH)
+#endif
+#ifdef DECLARE_CAUSE
+DECLARE_CAUSE("misaligned fetch", CAUSE_MISALIGNED_FETCH)
+DECLARE_CAUSE("fault fetch", CAUSE_FAULT_FETCH)
+DECLARE_CAUSE("illegal instruction", CAUSE_ILLEGAL_INSTRUCTION)
+DECLARE_CAUSE("breakpoint", CAUSE_BREAKPOINT)
+DECLARE_CAUSE("misaligned load", CAUSE_MISALIGNED_LOAD)
+DECLARE_CAUSE("fault load", CAUSE_FAULT_LOAD)
+DECLARE_CAUSE("misaligned store", CAUSE_MISALIGNED_STORE)
+DECLARE_CAUSE("fault store", CAUSE_FAULT_STORE)
+DECLARE_CAUSE("user_ecall", CAUSE_USER_ECALL)
+DECLARE_CAUSE("supervisor_ecall", CAUSE_SUPERVISOR_ECALL)
+DECLARE_CAUSE("hypervisor_ecall", CAUSE_HYPERVISOR_ECALL)
+DECLARE_CAUSE("machine_ecall", CAUSE_MACHINE_ECALL)
+#endif
diff --git a/machine/fp_asm.S b/machine/fp_asm.S
new file mode 100644
index 0000000..4b8dce1
--- /dev/null
+++ b/machine/fp_asm.S
@@ -0,0 +1,160 @@
+// See LICENSE for license details.
+
+#ifdef __riscv_hard_float
+
+#define get_f32(which) fmv.x.s a0, which; jr t0
+#define put_f32(which) fmv.s.x which, a0; jr t0
+#ifdef __riscv64
+# define get_f64(which) fmv.x.d a0, which; jr t0
+# define put_f64(which) fmv.d.x which, a0; jr t0
+#else
+# define get_f64(which) fsd which, 0(a0); jr t0
+# define put_f64(which) fld which, 0(a0); jr t0
+#endif
+
+  .text
+  .option norvc
+  .globl get_f32_reg
+  get_f32_reg:
+    get_f32(f0)
+    get_f32(f1)
+    get_f32(f2)
+    get_f32(f3)
+    get_f32(f4)
+    get_f32(f5)
+    get_f32(f6)
+    get_f32(f7)
+    get_f32(f8)
+    get_f32(f9)
+    get_f32(f10)
+    get_f32(f11)
+    get_f32(f12)
+    get_f32(f13)
+    get_f32(f14)
+    get_f32(f15)
+    get_f32(f16)
+    get_f32(f17)
+    get_f32(f18)
+    get_f32(f19)
+    get_f32(f20)
+    get_f32(f21)
+    get_f32(f22)
+    get_f32(f23)
+    get_f32(f24)
+    get_f32(f25)
+    get_f32(f26)
+    get_f32(f27)
+    get_f32(f28)
+    get_f32(f29)
+    get_f32(f30)
+    get_f32(f31)
+  
+  .text
+  .globl put_f32_reg
+  put_f32_reg:
+    put_f32(f0)
+    put_f32(f1)
+    put_f32(f2)
+    put_f32(f3)
+    put_f32(f4)
+    put_f32(f5)
+    put_f32(f6)
+    put_f32(f7)
+    put_f32(f8)
+    put_f32(f9)
+    put_f32(f10)
+    put_f32(f11)
+    put_f32(f12)
+    put_f32(f13)
+    put_f32(f14)
+    put_f32(f15)
+    put_f32(f16)
+    put_f32(f17)
+    put_f32(f18)
+    put_f32(f19)
+    put_f32(f20)
+    put_f32(f21)
+    put_f32(f22)
+    put_f32(f23)
+    put_f32(f24)
+    put_f32(f25)
+    put_f32(f26)
+    put_f32(f27)
+    put_f32(f28)
+    put_f32(f29)
+    put_f32(f30)
+    put_f32(f31)
+  
+  .text
+  .globl get_f64_reg
+  get_f64_reg:
+    get_f64(f0)
+    get_f64(f1)
+    get_f64(f2)
+    get_f64(f3)
+    get_f64(f4)
+    get_f64(f5)
+    get_f64(f6)
+    get_f64(f7)
+    get_f64(f8)
+    get_f64(f9)
+    get_f64(f10)
+    get_f64(f11)
+    get_f64(f12)
+    get_f64(f13)
+    get_f64(f14)
+    get_f64(f15)
+    get_f64(f16)
+    get_f64(f17)
+    get_f64(f18)
+    get_f64(f19)
+    get_f64(f20)
+    get_f64(f21)
+    get_f64(f22)
+    get_f64(f23)
+    get_f64(f24)
+    get_f64(f25)
+    get_f64(f26)
+    get_f64(f27)
+    get_f64(f28)
+    get_f64(f29)
+    get_f64(f30)
+    get_f64(f31)
+  
+  .text
+  .globl put_f64_reg
+  put_f64_reg:
+    put_f64(f0)
+    put_f64(f1)
+    put_f64(f2)
+    put_f64(f3)
+    put_f64(f4)
+    put_f64(f5)
+    put_f64(f6)
+    put_f64(f7)
+    put_f64(f8)
+    put_f64(f9)
+    put_f64(f10)
+    put_f64(f11)
+    put_f64(f12)
+    put_f64(f13)
+    put_f64(f14)
+    put_f64(f15)
+    put_f64(f16)
+    put_f64(f17)
+    put_f64(f18)
+    put_f64(f19)
+    put_f64(f20)
+    put_f64(f21)
+    put_f64(f22)
+    put_f64(f23)
+    put_f64(f24)
+    put_f64(f25)
+    put_f64(f26)
+    put_f64(f27)
+    put_f64(f28)
+    put_f64(f29)
+    put_f64(f30)
+    put_f64(f31)
+
+#endif
diff --git a/machine/fp_emulation.c b/machine/fp_emulation.c
new file mode 100644
index 0000000..536967f
--- /dev/null
+++ b/machine/fp_emulation.c
@@ -0,0 +1,449 @@
+#include "fp_emulation.h"
+#include "unprivileged_memory.h"
+#include "softfloat.h"
+#include "config.h"
+
+DECLARE_EMULATION_FUNC(emulate_float_load)
+{
+  uint64_t val;
+  uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn);
+  
+  switch (insn & MASK_FUNCT3)
+  {
+    case MATCH_FLW & MASK_FUNCT3:
+      if (addr % 4 != 0)
+        return misaligned_load_trap(regs, mcause, mepc);
+
+      SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc));
+      break;
+
+    case MATCH_FLD & MASK_FUNCT3:
+      if (addr % sizeof(uintptr_t) != 0)
+        return misaligned_load_trap(regs, mcause, mepc);
+
+#ifdef __riscv64
+      val = load_uint64_t((void *)addr, mepc);
+#else
+      val = load_uint32_t(addr, mepc);
+      val += (uint64_t)load_uint32_t((void *)(addr + 4), mepc) << 32;
+#endif
+      SET_F64_RD(insn, regs, val);
+      break;
+
+    default:
+      return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
+DECLARE_EMULATION_FUNC(emulate_float_store)
+{
+  uint64_t val;
+  uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn);
+  
+  switch (insn & MASK_FUNCT3)
+  {
+    case MATCH_FSW & MASK_FUNCT3:
+      if (addr % 4 != 0)
+        return misaligned_store_trap(regs, mcause, mepc);
+
+      store_uint32_t((void *)addr, GET_F32_RS2(insn, regs), mepc);
+      break;
+
+    case MATCH_FSD & MASK_FUNCT3:
+      if (addr % sizeof(uintptr_t) != 0)
+        return misaligned_store_trap(regs, mcause, mepc);
+
+      val = GET_F64_RS2(insn, regs);
+#ifdef __riscv64
+      store_uint64_t((void *)addr, val, mepc);
+#else
+      store_uint32_t((void *)addr, val, mepc);
+      store_uint32_t((void *)(addr + 4), val >> 32, mepc);
+#endif
+      break;
+
+    default:
+      return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
+DECLARE_EMULATION_FUNC(emulate_fp)
+{
+  asm (".pushsection .rodata\n"
+       "fp_emulation_table:\n"
+       "  .word emulate_fadd\n"
+       "  .word emulate_fsub\n"
+       "  .word emulate_fmul\n"
+       "  .word emulate_fdiv\n"
+       "  .word emulate_fsgnj\n"
+       "  .word emulate_fmin\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word emulate_fcvt_ff\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word emulate_fsqrt\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word emulate_fcmp\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word truly_illegal_insn\n"
+       "  .word emulate_fcvt_if\n"
+       "  .word truly_illegal_insn\n"
+       "  .word emulate_fcvt_fi\n"
+       "  .word truly_illegal_insn\n"
+       "  .word emulate_fmv_if\n"
+       "  .word truly_illegal_insn\n"
+       "  .word emulate_fmv_fi\n"
+       "  .word truly_illegal_insn\n"
+       "  .popsection");
+
+  // if FPU is disabled, punt back to the OS
+  if (unlikely((mstatus & MSTATUS_FS) == 0))
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  extern int32_t fp_emulation_table[];
+  int32_t* pf = (void*)fp_emulation_table + ((insn >> 25) & 0x7c);
+  emulation_func f = (emulation_func)(uintptr_t)*pf;
+
+  SETUP_STATIC_ROUNDING(insn);
+  return f(regs, mcause, mepc, mstatus, insn);
+}
+
+void emulate_any_fadd(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn, int32_t neg_b)
+{
+  if (GET_PRECISION(insn) == PRECISION_S) {
+    uint32_t rs1 = GET_F32_RS1(insn, regs);
+    uint32_t rs2 = GET_F32_RS2(insn, regs) ^ neg_b;
+    SET_F32_RD(insn, regs, f32_add(rs1, rs2));
+  } else if (GET_PRECISION(insn) == PRECISION_D) {
+    uint64_t rs1 = GET_F64_RS1(insn, regs);
+    uint64_t rs2 = GET_F64_RS2(insn, regs) ^ ((uint64_t)neg_b << 32);
+    SET_F64_RD(insn, regs, f64_add(rs1, rs2));
+  } else {
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
+DECLARE_EMULATION_FUNC(emulate_fadd)
+{
+  return emulate_any_fadd(regs, mcause, mepc, mstatus, insn, 0);
+}
+
+DECLARE_EMULATION_FUNC(emulate_fsub)
+{
+  return emulate_any_fadd(regs, mcause, mepc, mstatus, insn, INT32_MIN);
+}
+
+DECLARE_EMULATION_FUNC(emulate_fmul)
+{
+  if (GET_PRECISION(insn) == PRECISION_S) {
+    uint32_t rs1 = GET_F32_RS1(insn, regs);
+    uint32_t rs2 = GET_F32_RS2(insn, regs);
+    SET_F32_RD(insn, regs, f32_mul(rs1, rs2));
+  } else if (GET_PRECISION(insn) == PRECISION_D) {
+    uint64_t rs1 = GET_F64_RS1(insn, regs);
+    uint64_t rs2 = GET_F64_RS2(insn, regs);
+    SET_F64_RD(insn, regs, f64_mul(rs1, rs2));
+  } else {
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
+DECLARE_EMULATION_FUNC(emulate_fdiv)
+{
+  if (GET_PRECISION(insn) == PRECISION_S) {
+    uint32_t rs1 = GET_F32_RS1(insn, regs);
+    uint32_t rs2 = GET_F32_RS2(insn, regs);
+    SET_F32_RD(insn, regs, f32_div(rs1, rs2));
+  } else if (GET_PRECISION(insn) == PRECISION_D) {
+    uint64_t rs1 = GET_F64_RS1(insn, regs);
+    uint64_t rs2 = GET_F64_RS2(insn, regs);
+    SET_F64_RD(insn, regs, f64_div(rs1, rs2));
+  } else {
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
+DECLARE_EMULATION_FUNC(emulate_fsqrt)
+{
+  if ((insn >> 20) & 0x1f)
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  if (GET_PRECISION(insn) == PRECISION_S) {
+    SET_F32_RD(insn, regs, f32_sqrt(GET_F32_RS1(insn, regs)));
+  } else if (GET_PRECISION(insn) == PRECISION_D) {
+    SET_F64_RD(insn, regs, f64_sqrt(GET_F64_RS1(insn, regs)));
+  } else {
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
+DECLARE_EMULATION_FUNC(emulate_fsgnj)
+{
+  int rm = GET_RM(insn);
+  if (rm >= 3)
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  #define DO_FSGNJ(rs1, rs2, rm) ({ \
+    typeof(rs1) rs1_sign = (rs1) >> (8*sizeof(rs1)-1); \
+    typeof(rs1) rs2_sign = (rs2) >> (8*sizeof(rs1)-1); \
+    rs1_sign &= (rm) >> 1; \
+    rs1_sign ^= (rm) ^ rs2_sign; \
+    ((rs1) << 1 >> 1) | (rs1_sign << (8*sizeof(rs1)-1)); })
+
+  if (GET_PRECISION(insn) == PRECISION_S) {
+    uint32_t rs1 = GET_F32_RS1(insn, regs);
+    uint32_t rs2 = GET_F32_RS2(insn, regs);
+    SET_F32_RD(insn, regs, DO_FSGNJ(rs1, rs2, rm));
+  } else if (GET_PRECISION(insn) == PRECISION_D) {
+    uint64_t rs1 = GET_F64_RS1(insn, regs);
+    uint64_t rs2 = GET_F64_RS2(insn, regs);
+    SET_F64_RD(insn, regs, DO_FSGNJ(rs1, rs2, rm));
+  } else {
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
+DECLARE_EMULATION_FUNC(emulate_fmin)
+{
+  int rm = GET_RM(insn);
+  if (rm >= 2)
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  if (GET_PRECISION(insn) == PRECISION_S) {
+    uint32_t rs1 = GET_F32_RS1(insn, regs);
+    uint32_t rs2 = GET_F32_RS2(insn, regs);
+    uint32_t arg1 = rm ? rs2 : rs1;
+    uint32_t arg2 = rm ? rs1 : rs2;
+    int use_rs1 = f32_lt_quiet(arg1, arg2) || isNaNF32UI(rs2);
+    SET_F32_RD(insn, regs, use_rs1 ? rs1 : rs2);
+  } else if (GET_PRECISION(insn) == PRECISION_D) {
+    uint64_t rs1 = GET_F64_RS1(insn, regs);
+    uint64_t rs2 = GET_F64_RS2(insn, regs);
+    uint64_t arg1 = rm ? rs2 : rs1;
+    uint64_t arg2 = rm ? rs1 : rs2;
+    int use_rs1 = f64_lt_quiet(arg1, arg2) || isNaNF64UI(rs2);
+    SET_F64_RD(insn, regs, use_rs1 ? rs1 : rs2);
+  } else {
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
+DECLARE_EMULATION_FUNC(emulate_fcvt_ff)
+{
+  int rs2_num = (insn >> 20) & 0x1f;
+  if (GET_PRECISION(insn) == PRECISION_S) {
+    if (rs2_num != 1)
+      return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+    SET_F32_RD(insn, regs, f64_to_f32(GET_F64_RS1(insn, regs)));
+  } else if (GET_PRECISION(insn) == PRECISION_D) {
+    if (rs2_num != 0)
+      return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+    SET_F64_RD(insn, regs, f32_to_f64(GET_F32_RS1(insn, regs)));
+  } else {
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
+DECLARE_EMULATION_FUNC(emulate_fcvt_fi)
+{
+  if (GET_PRECISION(insn) != PRECISION_S && GET_PRECISION(insn) != PRECISION_D)
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  int negative = 0;
+  uint64_t uint_val = GET_RS1(insn, regs);
+
+  switch ((insn >> 20) & 0x1f)
+  {
+    case 0: // int32
+      negative = (int32_t)uint_val < 0;
+      uint_val = negative ? -(int32_t)uint_val : (int32_t)uint_val;
+      break;
+    case 1: // uint32
+      uint_val = (uint32_t)uint_val;
+      break;
+#ifdef __riscv64
+    case 2: // int64
+      negative = (int64_t)uint_val < 0;
+      uint_val = negative ? -uint_val : uint_val;
+    case 3: // uint64
+      break;
+#endif
+    default:
+      return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+
+  uint64_t float64 = ui64_to_f64(uint_val);
+  if (negative)
+    float64 ^= INT64_MIN;
+
+  if (GET_PRECISION(insn) == PRECISION_S)
+    SET_F32_RD(insn, regs, f64_to_f32(float64));
+  else
+    SET_F64_RD(insn, regs, float64);
+}
+
+DECLARE_EMULATION_FUNC(emulate_fcvt_if)
+{
+  int rs2_num = (insn >> 20) & 0x1f;
+#ifdef __riscv64
+  if (rs2_num >= 4)
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+#else
+  if (rs2_num >= 2)
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+#endif
+
+  int64_t float64;
+  if (GET_PRECISION(insn) == PRECISION_S)
+    float64 = f32_to_f64(GET_F32_RS1(insn, regs));
+  else if (GET_PRECISION(insn) == PRECISION_D)
+    float64 = GET_F64_RS1(insn, regs);
+  else
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  int negative = 0;
+  if (float64 < 0) {
+    negative = 1;
+    float64 ^= INT64_MIN;
+  }
+  uint64_t uint_val = f64_to_ui64(float64, softfloat_roundingMode, true);
+  uint64_t result, limit, limit_result;
+
+  switch (rs2_num)
+  {
+    case 0: // int32
+      if (negative) {
+        result = (int32_t)-uint_val;
+        limit_result = limit = (uint32_t)INT32_MIN;
+      } else {
+        result = (int32_t)uint_val;
+        limit_result = limit = INT32_MAX;
+      }
+      break;
+
+    case 1: // uint32
+      limit = limit_result = UINT32_MAX;
+      if (negative)
+        result = limit = 0;
+      else
+        result = (uint32_t)uint_val;
+      break;
+
+    case 2: // int32
+      if (negative) {
+        result = (int64_t)-uint_val;
+        limit_result = limit = (uint64_t)INT64_MIN;
+      } else {
+        result = (int64_t)uint_val;
+        limit_result = limit = INT64_MAX;
+      }
+      break;
+
+    case 3: // uint64
+      limit = limit_result = UINT64_MAX;
+      if (negative)
+        result = limit = 0;
+      else
+        result = (uint64_t)uint_val;
+      break;
+
+    default:
+      __builtin_unreachable();
+  }
+
+  if (uint_val > limit) {
+    result = limit_result;
+    softfloat_raiseFlags(softfloat_flag_invalid);
+  }
+
+  SET_FS_DIRTY();
+  SET_RD(insn, regs, result);
+}
+
+DECLARE_EMULATION_FUNC(emulate_fcmp)
+{
+  int rm = GET_RM(insn);
+  if (rm >= 3)
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  uintptr_t result;
+  if (GET_PRECISION(insn) == PRECISION_S) {
+    uint32_t rs1 = GET_F32_RS1(insn, regs);
+    uint32_t rs2 = GET_F32_RS2(insn, regs);
+    if (rm != 1)
+      result = f32_eq(rs1, rs2);
+    if (rm == 1 || (rm == 0 && !result))
+      result = f32_lt(rs1, rs2);
+    goto success;
+  } else if (GET_PRECISION(insn) == PRECISION_D) {
+    uint64_t rs1 = GET_F64_RS1(insn, regs);
+    uint64_t rs2 = GET_F64_RS2(insn, regs);
+    if (rm != 1)
+      result = f64_eq(rs1, rs2);
+    if (rm == 1 || (rm == 0 && !result))
+      result = f64_lt(rs1, rs2);
+    goto success;
+  }
+  return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+success:
+  SET_RD(insn, regs, result);
+}
+
+DECLARE_EMULATION_FUNC(emulate_fmv_if)
+{
+  uintptr_t result;
+  if ((insn & MASK_FMV_X_S) == MATCH_FMV_X_S)
+    result = GET_F32_RS1(insn, regs);
+#ifdef __riscv64
+  else if ((insn & MASK_FMV_X_D) == MATCH_FMV_X_D)
+    result = GET_F64_RS1(insn, regs);
+#endif
+  else
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  SET_RD(insn, regs, result);
+}
+
+DECLARE_EMULATION_FUNC(emulate_fmv_fi)
+{
+  uintptr_t rs1 = GET_RS1(insn, regs);
+
+  if ((insn & MASK_FMV_S_X) == MATCH_FMV_S_X)
+    SET_F32_RD(insn, regs, rs1);
+  else if ((insn & MASK_FMV_D_X) == MATCH_FMV_D_X)
+    SET_F64_RD(insn, regs, rs1);
+  else
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+}
+
+DECLARE_EMULATION_FUNC(emulate_fmadd)
+{
+  // if FPU is disabled, punt back to the OS
+  if (unlikely((mstatus & MSTATUS_FS) == 0))
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  int op = (insn >> 2) & 3;
+  SETUP_STATIC_ROUNDING(insn);
+  if (GET_PRECISION(insn) == PRECISION_S) {
+    uint32_t rs1 = GET_F32_RS1(insn, regs);
+    uint32_t rs2 = GET_F32_RS2(insn, regs);
+    uint32_t rs3 = GET_F32_RS3(insn, regs);
+    SET_F32_RD(insn, regs, softfloat_mulAddF32(op, rs1, rs2, rs3));
+  } else if (GET_PRECISION(insn) == PRECISION_D) {
+    uint64_t rs1 = GET_F64_RS1(insn, regs);
+    uint64_t rs2 = GET_F64_RS2(insn, regs);
+    uint64_t rs3 = GET_F64_RS3(insn, regs);
+    SET_F64_RD(insn, regs, softfloat_mulAddF64(op, rs1, rs2, rs3));
+  } else {
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
diff --git a/machine/fp_emulation.h b/machine/fp_emulation.h
new file mode 100644
index 0000000..d2357b7
--- /dev/null
+++ b/machine/fp_emulation.h
@@ -0,0 +1,81 @@
+#ifndef _RISCV_FP_EMULATION_H
+#define _RISCV_FP_EMULATION_H
+
+#include "emulation.h"
+
+#define GET_PRECISION(insn) (((insn) >> 25) & 3)
+#define GET_RM(insn) (((insn) >> 12) & 7)
+#define PRECISION_S 0
+#define PRECISION_D 1
+
+#ifdef __riscv_hard_float
+# define GET_F32_REG(insn, pos, regs) ({ \
+  register int32_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \
+  uintptr_t tmp; \
+  asm ("1: auipc %0, %%pcrel_hi(get_f32_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \
+  value; })
+# define SET_F32_REG(insn, pos, regs, val) ({ \
+  register uint32_t value asm("a0") = (val); \
+  uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \
+  uintptr_t tmp; \
+  asm volatile ("1: auipc %0, %%pcrel_hi(put_f32_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); })
+# define init_fp_reg(i) SET_F32_REG((i) << 3, 3, 0, 0)
+# define GET_F64_REG(insn, pos, regs) ({ \
+  register uintptr_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \
+  uintptr_t tmp; \
+  asm ("1: auipc %0, %%pcrel_hi(get_f64_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \
+  sizeof(uintptr_t) == 4 ? *(int64_t*)value : (int64_t)value; })
+# define SET_F64_REG(insn, pos, regs, val) ({ \
+  uint64_t __val = (val); \
+  register uintptr_t value asm("a0") = sizeof(uintptr_t) == 4 ? (uintptr_t)&__val : (uintptr_t)__val; \
+  uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \
+  uintptr_t tmp; \
+  asm volatile ("1: auipc %0, %%pcrel_hi(put_f64_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); })
+# define GET_FCSR() read_csr(fcsr)
+# define SET_FCSR(value) write_csr(fcsr, (value))
+# define GET_FRM() read_csr(frm)
+# define SET_FRM(value) write_csr(frm, (value))
+# define GET_FFLAGS() read_csr(fflags)
+# define SET_FFLAGS(value) write_csr(fflags, (value))
+
+# define SETUP_STATIC_ROUNDING(insn) ({ \
+  register long tp asm("tp") = read_csr(frm); \
+  if (likely(((insn) & MASK_FUNCT3) == MASK_FUNCT3)) ; \
+  else if (GET_RM(insn) > 4) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); \
+  else tp = GET_RM(insn); \
+  asm volatile ("":"+r"(tp)); })
+# define softfloat_raiseFlags(which) set_csr(fflags, which)
+# define softfloat_roundingMode ({ register int tp asm("tp"); tp; })
+#else
+# define GET_F64_REG(insn, pos, regs) (*(int64_t*)((void*)((regs) + 32) + (((insn) >> ((pos)-3)) & 0xf8)))
+# define SET_F64_REG(insn, pos, regs, val) (GET_F64_REG(insn, pos, regs) = (val))
+# define GET_F32_REG(insn, pos, regs) (*(int32_t*)&GET_F64_REG(insn, pos, regs))
+# define SET_F32_REG(insn, pos, regs, val) (GET_F32_REG(insn, pos, regs) = (val))
+# define GET_FCSR() ({ register int tp asm("tp"); tp & 0xFF; })
+# define SET_FCSR(value) ({ asm volatile("add tp, x0, %0" :: "rI"((value) & 0xFF)); })
+# define GET_FRM() (GET_FCSR() >> 5)
+# define SET_FRM(value) SET_FCSR(GET_FFLAGS() | ((value) << 5))
+# define GET_FFLAGS() (GET_FCSR() & 0x1F)
+# define SET_FFLAGS(value) SET_FCSR((GET_FRM() << 5) | ((value) & 0x1F))
+
+# define SETUP_STATIC_ROUNDING(insn) ({ \
+  register int tp asm("tp"); tp &= 0xFF; \
+  if (likely(((insn) & MASK_FUNCT3) == MASK_FUNCT3)) tp |= tp << 8; \
+  else if (GET_RM(insn) > 4) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); \
+  else tp |= GET_RM(insn) << 13; \
+  asm volatile ("":"+r"(tp)); })
+# define softfloat_raiseFlags(which) ({ asm volatile ("or tp, tp, %0" :: "rI"(which)); })
+# define softfloat_roundingMode ({ register int tp asm("tp"); tp >> 13; })
+#endif
+
+#define GET_F32_RS1(insn, regs) (GET_F32_REG(insn, 15, regs))
+#define GET_F32_RS2(insn, regs) (GET_F32_REG(insn, 20, regs))
+#define GET_F32_RS3(insn, regs) (GET_F32_REG(insn, 27, regs))
+#define GET_F64_RS1(insn, regs) (GET_F64_REG(insn, 15, regs))
+#define GET_F64_RS2(insn, regs) (GET_F64_REG(insn, 20, regs))
+#define GET_F64_RS3(insn, regs) (GET_F64_REG(insn, 27, regs))
+#define SET_F32_RD(insn, regs, val) (SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY())
+#define SET_F64_RD(insn, regs, val) (SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY())
+#define SET_FS_DIRTY() set_csr(mstatus, MSTATUS_FS)
+
+#endif
diff --git a/machine/htif.h b/machine/htif.h
new file mode 100644
index 0000000..e66ad06
--- /dev/null
+++ b/machine/htif.h
@@ -0,0 +1,18 @@
+#ifndef _RISCV_HTIF_H
+#define _RISCV_HTIF_H
+
+#include <stdint.h>
+
+#ifdef __riscv64
+# define TOHOST_CMD(dev, cmd, payload) \
+  (((uint64_t)(dev) << 56) | ((uint64_t)(cmd) << 48) | (uint64_t)(payload))
+#else
+# define TOHOST_CMD(dev, cmd, payload) ({ \
+  if ((dev) || (cmd)) __builtin_trap(); \
+  (payload); })
+#endif
+#define FROMHOST_DEV(fromhost_value) ((uint64_t)(fromhost_value) >> 56)
+#define FROMHOST_CMD(fromhost_value) ((uint64_t)(fromhost_value) << 8 >> 56)
+#define FROMHOST_DATA(fromhost_value) ((uint64_t)(fromhost_value) << 16 >> 16)
+
+#endif
diff --git a/machine/machine.ac b/machine/machine.ac
new file mode 100644
index 0000000..65acf04
--- /dev/null
+++ b/machine/machine.ac
@@ -0,0 +1,4 @@
+AC_ARG_ENABLE([fp-emulation], AS_HELP_STRING([--disable-fp-emulation], [Disable floating-point emulation]))
+AS_IF([test "x$enable_fp_emulation" != "xno"], [
+  AC_DEFINE([PK_ENABLE_FP_EMULATION],,[Define if floating-point emulation is enabled])
+])
diff --git a/machine/machine.mk.in b/machine/machine.mk.in
new file mode 100644
index 0000000..a0b7ca9
--- /dev/null
+++ b/machine/machine.mk.in
@@ -0,0 +1,29 @@
+machine_subproject_deps = \
+  softfloat \
+
+machine_hdrs = \
+  atomic.h \
+  bits.h \
+  emulation.h \
+  encoding.h \
+  fp_emulation.h \
+  htif.h \
+  mcall.h \
+  mtrap.h \
+  sbi.h \
+  unprivileged_memory.h \
+  vm.h \
+
+machine_c_srcs = \
+  mtrap.c \
+  minit.c \
+  emulation.c \
+  fp_emulation.c \
+  sbi_impl.c \
+  configstring.c \
+
+machine_asm_srcs = \
+  mentry.S \
+  fp_asm.S \
+  sbi_entry.S \
+  sbi.S \
diff --git a/machine/mcall.h b/machine/mcall.h
new file mode 100644
index 0000000..2096d16
--- /dev/null
+++ b/machine/mcall.h
@@ -0,0 +1,21 @@
+#ifndef _RISCV_MCALL_H
+#define _RISCV_MCALL_H
+
+#define MCALL_HART_ID 0
+#define MCALL_CONSOLE_PUTCHAR 1
+#define MCALL_CONSOLE_GETCHAR 2
+#define MCALL_HTIF_SYSCALL 3
+#define MCALL_SEND_IPI 4
+#define MCALL_CLEAR_IPI 5
+#define MCALL_SHUTDOWN 6
+#define MCALL_SET_TIMER 7
+#define MCALL_REMOTE_SFENCE_VM 8
+#define MCALL_REMOTE_FENCE_I 9
+
+#ifndef __ASSEMBLER__
+
+extern uintptr_t do_mcall(uintptr_t which, ...);
+
+#endif
+
+#endif
diff --git a/machine/mentry.S b/machine/mentry.S
new file mode 100644
index 0000000..50ce9ca
--- /dev/null
+++ b/machine/mentry.S
@@ -0,0 +1,245 @@
+// See LICENSE for license details.
+
+#include "mtrap.h"
+#include "bits.h"
+
+  .data
+  .align 6
+trap_table:
+  .word bad_trap
+  .word bad_trap
+  .word illegal_insn_trap
+  .word bad_trap
+  .word misaligned_load_trap
+  .word bad_trap
+  .word misaligned_store_trap
+  .word bad_trap
+  .word bad_trap
+  .word mcall_trap
+  .word bad_trap
+  .word bad_trap
+#define HTIF_INTERRUPT_VECTOR 12
+  .word htif_interrupt
+#define SOFTWARE_INTERRUPT_VECTOR 13
+  .word software_interrupt
+#define TRAP_FROM_MACHINE_MODE_VECTOR 14
+  .word __trap_from_machine_mode
+
+  .option norvc
+  .section .text.init,"ax",@progbits
+  .globl reset_vector
+reset_vector:
+  j do_reset
+
+nmi_vector:
+.Lunhandleable_trap:
+  j bad_trap
+
+trap_vector:
+  csrrw sp, mscratch, sp
+  beqz sp, .Ltrap_from_machine_mode
+
+  STORE a0, 10*REGBYTES(sp)
+  STORE a1, 11*REGBYTES(sp)
+
+  csrr a1, mcause
+  bgez a1, .Lhandle_trap_in_machine_mode
+
+  # This is an interrupt.  Discard the mcause MSB and decode the rest.
+  sll a1, a1, 1
+
+  # Is it a machine timer interrupt?
+  li a0, IRQ_M_TIMER * 2
+  bne a0, a1, 1f
+  # Yes.  Post a supervisor timer interrupt.
+  li a0, MIP_MTIP
+  csrc mie, a0
+  li a0, MIP_STIP
+  csrs mip, a0
+
+.Leret:
+  # Go back whence we came.
+  LOAD a0, 10*REGBYTES(sp)
+  LOAD a1, 11*REGBYTES(sp)
+  csrrw sp, mscratch, sp
+  eret
+
+1:
+  # Is it an IPI?
+  li a0, IRQ_M_SOFT * 2
+  bne a0, a1, 1f
+  li a1, SOFTWARE_INTERRUPT_VECTOR
+  j .Lhandle_trap_in_machine_mode
+
+1:
+  # By process of elimination, it must be an HTIF interrupt.
+  li a0, IRQ_HOST * 2
+  bne a0, a1, .Lunhandleable_trap
+  li a1, HTIF_INTERRUPT_VECTOR
+
+.Lhandle_trap_in_machine_mode:
+  # Preserve the registers.  Compute the address of the trap handler.
+  STORE ra, 1*REGBYTES(sp)
+  STORE gp, 3*REGBYTES(sp)
+  STORE tp, 4*REGBYTES(sp)
+  STORE t0, 5*REGBYTES(sp)
+1:auipc t0, %pcrel_hi(trap_table)  # t0 <- %hi(trap_table)
+  STORE t1, 6*REGBYTES(sp)
+  sll t1, a1, 2                    # t1 <- mcause << 2
+  STORE t2, 7*REGBYTES(sp)
+  add t1, t0, t1                   # t1 <- %hi(trap_table)[mcause]
+  STORE s0, 8*REGBYTES(sp)
+  lw t1, %pcrel_lo(1b)(t1)         # t1 <- trap_table[mcause]
+  STORE s1, 9*REGBYTES(sp)
+  mv a0, sp                        # a0 <- regs
+  STORE a2,12*REGBYTES(sp)
+  csrr a2, mepc                    # a2 <- mepc
+  STORE a3,13*REGBYTES(sp)
+  csrrw t0, mscratch, x0           # t0 <- user sp
+  STORE a4,14*REGBYTES(sp)
+  STORE a5,15*REGBYTES(sp)
+  STORE a6,16*REGBYTES(sp)
+  STORE a7,17*REGBYTES(sp)
+  STORE s2,18*REGBYTES(sp)
+  STORE s3,19*REGBYTES(sp)
+  STORE s4,20*REGBYTES(sp)
+  STORE s5,21*REGBYTES(sp)
+  STORE s6,22*REGBYTES(sp)
+  STORE s7,23*REGBYTES(sp)
+  STORE s8,24*REGBYTES(sp)
+  STORE s9,25*REGBYTES(sp)
+  STORE s10,26*REGBYTES(sp)
+  STORE s11,27*REGBYTES(sp)
+  STORE t3,28*REGBYTES(sp)
+  STORE t4,29*REGBYTES(sp)
+  STORE t5,30*REGBYTES(sp)
+  STORE t6,31*REGBYTES(sp)
+  STORE t0, 2*REGBYTES(sp)         # sp
+
+#ifndef __riscv_hard_float
+  lw tp, (sp) # Move the emulated FCSR from x0's save slot into tp.
+#endif
+  STORE x0, (sp) # Zero x0's save slot.
+
+  # Invoke the handler.
+  jalr t1
+
+#ifndef __riscv_hard_float
+  sw tp, (sp) # Move the emulated FCSR from tp into x0's save slot.
+#endif
+
+restore_mscratch:
+  # Restore mscratch, so future traps will know they didn't come from M-mode.
+  csrw mscratch, sp
+
+restore_regs:
+  # Restore all of the registers.
+  LOAD ra, 1*REGBYTES(sp)
+  LOAD gp, 3*REGBYTES(sp)
+  LOAD tp, 4*REGBYTES(sp)
+  LOAD t0, 5*REGBYTES(sp)
+  LOAD t1, 6*REGBYTES(sp)
+  LOAD t2, 7*REGBYTES(sp)
+  LOAD s0, 8*REGBYTES(sp)
+  LOAD s1, 9*REGBYTES(sp)
+  LOAD a0,10*REGBYTES(sp)
+  LOAD a1,11*REGBYTES(sp)
+  LOAD a2,12*REGBYTES(sp)
+  LOAD a3,13*REGBYTES(sp)
+  LOAD a4,14*REGBYTES(sp)
+  LOAD a5,15*REGBYTES(sp)
+  LOAD a6,16*REGBYTES(sp)
+  LOAD a7,17*REGBYTES(sp)
+  LOAD s2,18*REGBYTES(sp)
+  LOAD s3,19*REGBYTES(sp)
+  LOAD s4,20*REGBYTES(sp)
+  LOAD s5,21*REGBYTES(sp)
+  LOAD s6,22*REGBYTES(sp)
+  LOAD s7,23*REGBYTES(sp)
+  LOAD s8,24*REGBYTES(sp)
+  LOAD s9,25*REGBYTES(sp)
+  LOAD s10,26*REGBYTES(sp)
+  LOAD s11,27*REGBYTES(sp)
+  LOAD t3,28*REGBYTES(sp)
+  LOAD t4,29*REGBYTES(sp)
+  LOAD t5,30*REGBYTES(sp)
+  LOAD t6,31*REGBYTES(sp)
+  LOAD sp, 2*REGBYTES(sp)
+  eret
+
+.Ltrap_from_machine_mode:
+  csrr sp, mscratch
+  addi sp, sp, -INTEGER_CONTEXT_SIZE
+  STORE a0,10*REGBYTES(sp)
+  STORE a1,11*REGBYTES(sp)
+  li a1, TRAP_FROM_MACHINE_MODE_VECTOR
+  j .Lhandle_trap_in_machine_mode
+
+  .globl __redirect_trap
+__redirect_trap:
+  # reset sp to top of M-mode stack
+  li t0, MACHINE_STACK_SIZE
+  add sp, sp, t0
+  neg t0, t0
+  and sp, sp, t0
+  addi sp, sp, -MENTRY_FRAME_SIZE
+  j restore_mscratch
+
+__trap_from_machine_mode:
+  jal trap_from_machine_mode
+  j restore_regs
+
+do_reset:
+  li x1, 0
+  li x2, 0
+  li x3, 0
+  li x4, 0
+  li x5, 0
+  li x6, 0
+  li x7, 0
+  li x8, 0
+  li x9, 0
+  li x10, 0
+  li x11, 0
+  li x12, 0
+  li x13, 0
+  li x14, 0
+  li x15, 0
+  li x16, 0
+  li x17, 0
+  li x18, 0
+  li x19, 0
+  li x20, 0
+  li x21, 0
+  li x22, 0
+  li x23, 0
+  li x24, 0
+  li x25, 0
+  li x26, 0
+  li x27, 0
+  li x28, 0
+  li x29, 0
+  li x30, 0
+  li x31, 0
+  csrw mscratch, x0
+
+  # sp <- end of first full page after the end of the binary
+  la sp, _end + 2*RISCV_PGSIZE - 1
+  li t0, -RISCV_PGSIZE
+  and sp, sp, t0
+  addi sp, sp, -MENTRY_FRAME_SIZE
+
+  csrr a0, mhartid
+  slli a1, a0, RISCV_PGSHIFT
+  add sp, sp, a1
+
+  beqz a0, init_first_hart
+
+.LmultiHart:
+#if MAX_HARTS > 1
+  # make sure our hart id is within a valid range
+  li a1, MAX_HARTS
+  bltu a0, a1, init_other_hart
+  wfi
+#endif
+  j .LmultiHart
diff --git a/machine/minit.c b/machine/minit.c
new file mode 100644
index 0000000..33b94c8
--- /dev/null
+++ b/machine/minit.c
@@ -0,0 +1,120 @@
+#include "mtrap.h"
+#include "atomic.h"
+#include "vm.h"
+#include "fp_emulation.h"
+#include <string.h>
+
+pte_t* root_page_table;
+uintptr_t first_free_paddr;
+uintptr_t mem_size;
+uint32_t num_harts;
+
+static void mstatus_init()
+{
+  uintptr_t ms = 0;
+  ms = INSERT_FIELD(ms, MSTATUS_VM, VM_CHOICE);
+  ms = INSERT_FIELD(ms, MSTATUS_FS, 1);
+  write_csr(mstatus, ms);
+
+  ms = read_csr(mstatus);
+  assert(EXTRACT_FIELD(ms, MSTATUS_VM) == VM_CHOICE);
+
+  write_csr(mtimecmp, 0);
+  clear_csr(mip, MIP_MSIP);
+  write_csr(mie, -1);
+  write_csr(mucounteren, -1);
+  write_csr(mscounteren, -1);
+}
+
+static void delegate_traps()
+{
+  uintptr_t interrupts = MIP_SSIP | MIP_STIP;
+  uintptr_t exceptions =
+    (1U << CAUSE_MISALIGNED_FETCH) |
+    (1U << CAUSE_FAULT_FETCH) |
+    (1U << CAUSE_BREAKPOINT) |
+    (1U << CAUSE_FAULT_LOAD) |
+    (1U << CAUSE_FAULT_STORE) |
+    (1U << CAUSE_BREAKPOINT) |
+    (1U << CAUSE_USER_ECALL);
+
+  write_csr(mideleg, interrupts);
+  write_csr(medeleg, exceptions);
+  assert(read_csr(mideleg) == interrupts);
+  assert(read_csr(medeleg) == exceptions);
+}
+
+static void fp_init()
+{
+  assert(read_csr(mstatus) & MSTATUS_FS);
+
+#ifdef __riscv_hard_float
+  if (!supports_extension('D'))
+    die("FPU not found; recompile pk with -msoft-float");
+  for (int i = 0; i < 32; i++)
+    init_fp_reg(i);
+  write_csr(fcsr, 0);
+#else
+  if (supports_extension('D'))
+    die("FPU unexpectedly found; recompile with -mhard-float");
+#endif
+}
+
+void hls_init(uint32_t id, csr_t* csrs)
+{
+  hls_t* hls = OTHER_HLS(id);
+  memset(hls, 0, sizeof(*hls));
+  hls->csrs = csrs;
+}
+
+static uintptr_t sbi_top_paddr()
+{
+  extern char _end;
+  return ROUNDUP((uintptr_t)&_end, RISCV_PGSIZE);
+}
+
+static void memory_init()
+{
+  mem_size = mem_size / MEGAPAGE_SIZE * MEGAPAGE_SIZE;
+  first_free_paddr = sbi_top_paddr() + num_harts * RISCV_PGSIZE;
+}
+
+static void hart_init()
+{
+  mstatus_init();
+  fp_init();
+  delegate_traps();
+}
+
+void init_first_hart()
+{
+  hart_init();
+  memset(HLS(), 0, sizeof(*HLS()));
+  parse_config_string();
+  memory_init();
+  boot_loader();
+}
+
+void init_other_hart()
+{
+  hart_init();
+
+  // wait until hart 0 discovers us
+  while (*(csr_t * volatile *)&HLS()->csrs == NULL)
+    ;
+
+  boot_other_hart();
+}
+
+void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t stack)
+{
+  uintptr_t mstatus = read_csr(mstatus);
+  mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S);
+  mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0);
+  write_csr(mstatus, mstatus);
+  write_csr(mscratch, MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE);
+  write_csr(mepc, fn);
+  write_csr(sptbr, (uintptr_t)root_page_table >> RISCV_PGSHIFT);
+  asm volatile ("mv a0, %0; mv sp, %0; eret" : : "r" (stack));
+  __builtin_unreachable();
+}
diff --git a/machine/mtrap.c b/machine/mtrap.c
new file mode 100644
index 0000000..6fc5476
--- /dev/null
+++ b/machine/mtrap.c
@@ -0,0 +1,287 @@
+#include "mtrap.h"
+#include "mcall.h"
+#include "htif.h"
+#include "atomic.h"
+#include "bits.h"
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+void __attribute__((noreturn)) bad_trap()
+{
+  die("machine mode: unhandlable trap %d @ %p", read_csr(mcause), read_csr(mepc));
+}
+
+static uintptr_t mcall_hart_id()
+{
+  return read_const_csr(mhartid);
+}
+
+static void request_htif_keyboard_interrupt()
+{
+  uintptr_t old_tohost = swap_csr(mtohost, TOHOST_CMD(1, 0, 0));
+  assert(old_tohost == 0);
+}
+
+void htif_interrupt()
+{
+  uintptr_t fromhost = swap_csr(mfromhost, 0);
+  // we should only be interrupted by keypresses
+  if (!(FROMHOST_DEV(fromhost) == 1 && FROMHOST_CMD(fromhost) == 0))
+    die("unexpected htif interrupt");
+  HLS()->console_ibuf = 1 + (uint8_t)FROMHOST_DATA(fromhost);
+  set_csr(mip, MIP_SSIP);
+}
+
+static void do_tohost_fromhost(uintptr_t dev, uintptr_t cmd, uintptr_t data)
+{
+  while (swap_csr(mtohost, TOHOST_CMD(dev, cmd, data)) != 0)
+    if (read_csr(mfromhost))
+      htif_interrupt();
+
+  while (1) {
+    uintptr_t fromhost = read_csr(mfromhost);
+    if (fromhost) {
+      if (FROMHOST_DEV(fromhost) == dev && FROMHOST_CMD(fromhost) == cmd) {
+        write_csr(mfromhost, 0);
+        break;
+      }
+      htif_interrupt();
+    }
+  }
+}
+
+static uintptr_t mcall_console_putchar(uint8_t ch)
+{
+  do_tohost_fromhost(1, 1, ch);
+  return 0;
+}
+
+static uintptr_t mcall_htif_syscall(uintptr_t magic_mem)
+{
+  do_tohost_fromhost(0, 0, magic_mem);
+  return 0;
+}
+
+void poweroff()
+{
+  while (1)
+    write_csr(mtohost, 1);
+}
+
+void putstring(const char* s)
+{
+  while (*s)
+    mcall_console_putchar(*s++);
+}
+
+void printm(const char* s, ...)
+{
+  char buf[256];
+  va_list vl;
+
+  va_start(vl, s);
+  vsnprintf(buf, sizeof buf, s, vl);
+  va_end(vl);
+
+  putstring(buf);
+}
+
+static void send_ipi(uintptr_t recipient, int event)
+{
+  if ((atomic_or(&OTHER_HLS(recipient)->mipi_pending, event) & event) == 0) {
+    mb();
+    OTHER_HLS(recipient)->csrs[CSR_MIPI] = 1;
+  }
+}
+
+static uintptr_t mcall_send_ipi(uintptr_t recipient)
+{
+  if (recipient >= num_harts)
+    return -1;
+
+  send_ipi(recipient, IPI_SOFT);
+  return 0;
+}
+
+static void reset_ssip()
+{
+  clear_csr(mip, MIP_SSIP);
+  mb();
+
+  if (HLS()->sipi_pending || HLS()->console_ibuf > 0)
+    set_csr(mip, MIP_SSIP);
+}
+
+static uintptr_t mcall_console_getchar()
+{
+  int ch = atomic_swap(&HLS()->console_ibuf, -1);
+  if (ch >= 0)
+    request_htif_keyboard_interrupt();
+  reset_ssip();
+  return ch - 1;
+}
+
+static uintptr_t mcall_clear_ipi()
+{
+  int ipi = atomic_swap(&HLS()->sipi_pending, 0);
+  reset_ssip();
+  return ipi;
+}
+
+static uintptr_t mcall_shutdown()
+{
+  poweroff();
+}
+
+static uintptr_t mcall_set_timer(unsigned long long when)
+{
+  // bbl/pk don't use the timer, so there's no need to virtualize it
+  write_csr(mtimecmp, when);
+#ifndef __riscv64
+  write_csr(mtimecmph, when >> 32);
+#endif
+  clear_csr(mip, MIP_STIP);
+  set_csr(mie, MIP_MTIP);
+  return 0;
+}
+
+void software_interrupt()
+{
+  clear_csr(mip, MIP_MSIP);
+  mb();
+  int ipi_pending = atomic_swap(&HLS()->mipi_pending, 0);
+
+  if (ipi_pending & IPI_SOFT) {
+    HLS()->sipi_pending = 1;
+    set_csr(mip, MIP_SSIP);
+  }
+
+  if (ipi_pending & IPI_FENCE_I)
+    asm volatile ("fence.i");
+
+  if (ipi_pending & IPI_SFENCE_VM)
+    asm volatile ("sfence.vm");
+}
+
+static void send_ipi_many(uintptr_t* pmask, int event)
+{
+  _Static_assert(MAX_HARTS <= 8 * sizeof(*pmask), "# harts > uintptr_t bits");
+  uintptr_t mask = -1;
+  if (pmask)
+    mask = *pmask;
+
+  // send IPIs to everyone
+  for (ssize_t i = num_harts-1; i >= 0; i--)
+    if ((mask >> i) & 1)
+      send_ipi(i, event);
+
+  // wait until all events have been handled.
+  // prevent deadlock while spinning by handling any IPIs from other harts.
+  for (ssize_t i = num_harts-1; i >= 0; i--)
+    if ((mask >> i) & 1)
+      while (OTHER_HLS(i)->mipi_pending & event)
+        software_interrupt();
+}
+
+static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask, uintptr_t asid)
+{
+  // ignore the ASID and do a global flush.
+  // this allows us to avoid queueing a message.
+  send_ipi_many(hart_mask, IPI_SFENCE_VM);
+  return 0;
+}
+
+static uintptr_t mcall_remote_fence_i(uintptr_t* hart_mask)
+{
+  send_ipi_many(hart_mask, IPI_FENCE_I);
+  return 0;
+}
+
+void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
+{
+  uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval;
+  switch (n)
+  {
+    case MCALL_HART_ID:
+      retval = mcall_hart_id();
+      break;
+    case MCALL_CONSOLE_PUTCHAR:
+      retval = mcall_console_putchar(arg0);
+      break;
+    case MCALL_CONSOLE_GETCHAR:
+      retval = mcall_console_getchar();
+      break;
+    case MCALL_HTIF_SYSCALL:
+      retval = mcall_htif_syscall(arg0);
+      break;
+    case MCALL_SEND_IPI:
+      retval = mcall_send_ipi(arg0);
+      break;
+    case MCALL_CLEAR_IPI:
+      retval = mcall_clear_ipi();
+      break;
+    case MCALL_SHUTDOWN:
+      retval = mcall_shutdown();
+      break;
+    case MCALL_SET_TIMER:
+      retval = mcall_set_timer(arg0);
+      break;
+    case MCALL_REMOTE_SFENCE_VM:
+      retval = mcall_remote_sfence_vm((uintptr_t*)arg0, arg1);
+      break;
+    case MCALL_REMOTE_FENCE_I:
+      retval = mcall_remote_fence_i((uintptr_t*)arg0);
+      break;
+    default:
+      retval = -ENOSYS;
+      break;
+  }
+  regs[10] = retval;
+  write_csr(mepc, mepc + 4);
+}
+
+void redirect_trap(uintptr_t epc, uintptr_t mstatus)
+{
+  write_csr(sepc, epc);
+  write_csr(scause, read_csr(mcause));
+  write_csr(mepc, read_csr(stvec));
+
+  uintptr_t prev_priv = EXTRACT_FIELD(mstatus, MSTATUS_MPP);
+  uintptr_t prev_ie = EXTRACT_FIELD(mstatus, MSTATUS_MPIE);
+  mstatus = INSERT_FIELD(mstatus, MSTATUS_SPP, prev_priv);
+  mstatus = INSERT_FIELD(mstatus, MSTATUS_SPIE, prev_ie);
+  mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S);
+  mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0);
+  write_csr(mstatus, mstatus);
+
+  extern void __redirect_trap();
+  return __redirect_trap();
+}
+
+static void machine_page_fault(uintptr_t* regs, uintptr_t mepc)
+{
+  // MPRV=1 iff this trap occurred while emulating an instruction on behalf
+  // of a lower privilege level. In that case, a2=epc and a3=mstatus.
+  if (read_csr(mstatus) & MSTATUS_MPRV) {
+    write_csr(sbadaddr, read_csr(mbadaddr));
+    return redirect_trap(regs[12], regs[13]);
+  }
+  bad_trap();
+}
+
+void trap_from_machine_mode(uintptr_t* regs, uintptr_t dummy, uintptr_t mepc)
+{
+  uintptr_t mcause = read_csr(mcause);
+
+  switch (mcause)
+  {
+    case CAUSE_FAULT_LOAD:
+    case CAUSE_FAULT_STORE:
+      return machine_page_fault(regs, mepc);
+    case CAUSE_MACHINE_ECALL:
+      return mcall_trap(regs, mcause, mepc);
+    default:
+      bad_trap();
+  }
+}
diff --git a/machine/mtrap.h b/machine/mtrap.h
new file mode 100644
index 0000000..139f359
--- /dev/null
+++ b/machine/mtrap.h
@@ -0,0 +1,96 @@
+#ifndef _RISCV_MTRAP_H
+#define _RISCV_MTRAP_H
+
+#include "encoding.h"
+
+#ifdef __riscv_atomic
+# define MAX_HARTS 8 // arbitrary
+#else
+# define MAX_HARTS 1
+#endif
+
+#ifndef __ASSEMBLER__
+
+#include "sbi.h"
+#include <stdint.h>
+
+#define read_const_csr(reg) ({ unsigned long __tmp; \
+  asm ("csrr %0, " #reg : "=r"(__tmp)); \
+  __tmp; })
+
+static inline int supports_extension(char ext)
+{
+  return read_const_csr(misa) & (1 << (ext - 'A'));
+}
+
+static inline int xlen()
+{
+  return read_const_csr(misa) < 0 ? 64 : 32;
+}
+
+extern uintptr_t first_free_paddr;
+extern uintptr_t mem_size;
+extern uint32_t num_harts;
+
+typedef uintptr_t csr_t; // TODO this might become uint128_t for RV128
+
+typedef struct {
+  volatile csr_t* csrs;
+  volatile int mipi_pending;
+  volatile int sipi_pending;
+  int console_ibuf;
+
+  uint64_t utime_delta;
+  uint64_t ucycle_delta;
+  uint64_t uinstret_delta;
+  uint64_t stime_delta;
+  uint64_t scycle_delta;
+  uint64_t sinstret_delta;
+} hls_t;
+
+#define IPI_SOFT      0x1
+#define IPI_FENCE_I   0x2
+#define IPI_SFENCE_VM 0x4
+
+#define MACHINE_STACK_TOP() ({ \
+  register uintptr_t sp asm ("sp"); \
+  (void*)((sp + RISCV_PGSIZE) & -RISCV_PGSIZE); })
+
+// hart-local storage, at top of stack
+#define HLS() ((hls_t*)(MACHINE_STACK_TOP() - HLS_SIZE))
+#define OTHER_HLS(id) ((hls_t*)((void*)HLS() + RISCV_PGSIZE * ((id) - read_const_csr(mhartid))))
+
+void hls_init(uint32_t hart_id, csr_t* csrs);
+void parse_config_string();
+void poweroff(void) __attribute((noreturn));
+void printm(const char* s, ...);
+void putstring(const char* s);
+#define assert(x) ({ if (!(x)) die("assertion failed: %s", #x); })
+#define die(str, ...) ({ printm("%s:%d: " str "\n", __FILE__, __LINE__, ##__VA_ARGS__); poweroff(); })
+#define printk(...) die("printk")
+
+void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t stack)
+  __attribute__((noreturn));
+void boot_loader();
+void boot_other_hart();
+
+static inline void wfi()
+{
+  asm volatile ("wfi" ::: "memory");
+}
+
+#endif // !__ASSEMBLER__
+
+#define MACHINE_STACK_SIZE RISCV_PGSIZE
+#define MENTRY_FRAME_SIZE (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE \
+                           + HLS_SIZE)
+
+#ifdef __riscv_hard_float
+# define SOFT_FLOAT_CONTEXT_SIZE 0
+#else
+# define SOFT_FLOAT_CONTEXT_SIZE (8 * 32)
+#endif
+#define HLS_SIZE 64
+#define INTEGER_CONTEXT_SIZE (32 * REGBYTES)
+
+#endif
diff --git a/machine/sbi.S b/machine/sbi.S
new file mode 100644
index 0000000..cbea78a
--- /dev/null
+++ b/machine/sbi.S
@@ -0,0 +1,15 @@
+.globl sbi_hart_id; sbi_hart_id = -2048
+.globl sbi_num_harts; sbi_num_harts = -2032
+.globl sbi_query_memory; sbi_query_memory = -2016
+.globl sbi_console_putchar; sbi_console_putchar = -2000
+.globl sbi_console_getchar; sbi_console_getchar = -1984
+.globl sbi_send_ipi; sbi_send_ipi = -1952
+.globl sbi_clear_ipi; sbi_clear_ipi = -1936
+.globl sbi_timebase; sbi_timebase = -1920
+.globl sbi_shutdown; sbi_shutdown = -1904
+.globl sbi_set_timer; sbi_set_timer = -1888
+.globl sbi_mask_interrupt; sbi_mask_interrupt = -1872
+.globl sbi_unmask_interrupt; sbi_unmask_interrupt = -1856
+.globl sbi_remote_sfence_vm; sbi_remote_sfence_vm = -1840
+.globl sbi_remote_sfence_vm_range; sbi_remote_sfence_vm_range = -1824
+.globl sbi_remote_fence_i; sbi_remote_fence_i = -1808
diff --git a/machine/sbi.h b/machine/sbi.h
new file mode 100644
index 0000000..4e2fbd8
--- /dev/null
+++ b/machine/sbi.h
@@ -0,0 +1,30 @@
+#ifndef _ASM_RISCV_SBI_H
+#define _ASM_RISCV_SBI_H
+
+typedef struct {
+  unsigned long base;
+  unsigned long size;
+  unsigned long node_id;
+} memory_block_info;
+
+unsigned long sbi_query_memory(unsigned long id, memory_block_info *p);
+
+unsigned long sbi_hart_id(void);
+unsigned long sbi_num_harts(void);
+unsigned long sbi_timebase(void);
+void sbi_set_timer(unsigned long long stime_value);
+void sbi_send_ipi(unsigned long hart_id);
+unsigned long sbi_clear_ipi(void);
+void sbi_shutdown(void);
+
+void sbi_console_putchar(unsigned char ch);
+int sbi_console_getchar(void);
+
+void sbi_remote_sfence_vm(unsigned long hart_mask_ptr, unsigned long asid);
+void sbi_remote_sfence_vm_range(unsigned long hart_mask_ptr, unsigned long asid, unsigned long start, unsigned long size);
+void sbi_remote_fence_i(unsigned long hart_mask_ptr);
+
+unsigned long sbi_mask_interrupt(unsigned long which);
+unsigned long sbi_unmask_interrupt(unsigned long which);
+
+#endif
diff --git a/machine/sbi_entry.S b/machine/sbi_entry.S
new file mode 100644
index 0000000..a37dd25
--- /dev/null
+++ b/machine/sbi_entry.S
@@ -0,0 +1,111 @@
+#include "encoding.h"
+#include "mcall.h"
+
+  .section .sbi,"ax",@progbits
+  .option norvc
+  .align RISCV_PGSHIFT
+  .globl sbi_base
+sbi_base:
+
+  # TODO: figure out something better to do with this space.  It's not
+  # protected from the OS, so beware.
+  .skip RISCV_PGSIZE - 2048
+
+  # hart_id
+  .align 4
+  li a7, MCALL_HART_ID
+  ecall
+  ret
+
+  # num_harts
+  .align 4
+  lw a0, num_harts
+  ret
+
+  # query_memory
+  .align 4
+  tail __sbi_query_memory
+
+  # console_putchar
+  .align 4
+  li a7, MCALL_CONSOLE_PUTCHAR
+  ecall
+  ret
+
+  # console_getchar
+  .align 4
+  li a7, MCALL_CONSOLE_GETCHAR
+  ecall
+  ret
+
+  # empty
+  .align 4
+  unimp
+
+  # send ipi
+  .align 4
+  li a7, MCALL_SEND_IPI
+  ecall
+  ret
+
+  # clear ipi
+  .align 4
+  li a7, MCALL_CLEAR_IPI
+  ecall
+  ret
+
+  # timebase
+  .align 4
+  li a0, 10000000 # or, you know, we could provide the correct answer
+  ret
+
+  # shutdown
+  .align 4
+  li a7, MCALL_SHUTDOWN
+  ecall
+
+  # set_timer
+  .align 4
+  li a7, MCALL_SET_TIMER
+  ecall
+  ret
+
+  # mask_interrupt
+  .align 4
+  tail __sbi_mask_interrupt
+
+  # unmask_interrupt
+  .align 4
+  tail __sbi_unmask_interrupt
+
+  # remote_sfence_vm
+  .align 4
+  li a7, MCALL_REMOTE_SFENCE_VM
+  ecall
+  ret
+
+  # remote_sfence_vm_range
+  .align 4
+  li a7, MCALL_REMOTE_SFENCE_VM
+  ecall
+  ret
+
+  # remote_fence_i
+  .align 4
+  li a7, MCALL_REMOTE_FENCE_I
+  ecall
+  ret
+
+  # end of SBI trampolines
+
+  .globl do_mcall
+do_mcall:
+  mv a7, a0
+  mv a0, a1
+  mv a1, a2
+  ecall
+  ret
+
+  .align RISCV_PGSHIFT
+  .globl _sbi_end
+_sbi_end:
diff --git a/machine/sbi_impl.c b/machine/sbi_impl.c
new file mode 100644
index 0000000..07844e0
--- /dev/null
+++ b/machine/sbi_impl.c
@@ -0,0 +1,33 @@
+#include "mtrap.h"
+#include "sbi.h"
+
+uintptr_t __sbi_query_memory(uintptr_t id, memory_block_info *p)
+{
+  if (id == 0) {
+    p->base = first_free_paddr;
+    p->size = mem_size - p->base;
+    return 0;
+  }
+
+  return -1;
+}
+
+#define LOW_IRQ_OK(n) ((n) == IRQ_S_SOFT || (n) == IRQ_S_TIMER)
+
+uintptr_t __sbi_mask_interrupt(uintptr_t which)
+{
+  if (!LOW_IRQ_OK(which))
+    return -1;
+
+  clear_csr(sie, 1UL << which);
+  return 0;
+}
+
+uintptr_t __sbi_unmask_interrupt(uintptr_t which)
+{
+  if (!LOW_IRQ_OK(which))
+    return -1;
+
+  set_csr(sie, 1UL << which);
+  return 0;
+}
diff --git a/machine/unprivileged_memory.h b/machine/unprivileged_memory.h
new file mode 100644
index 0000000..d03cc5e
--- /dev/null
+++ b/machine/unprivileged_memory.h
@@ -0,0 +1,79 @@
+#ifndef _RISCV_MISALIGNED_H
+#define _RISCV_MISALIGNED_H
+
+#include "encoding.h"
+#include <stdint.h>
+
+#define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type, insn)              \
+  static inline type load_##type(const type* addr, uintptr_t mepc)  \
+  {                                                                 \
+    register uintptr_t __mepc asm ("a2") = mepc;                    \
+    register uintptr_t __mstatus asm ("a3");                        \
+    type val;                                                       \
+    asm ("csrrs %0, mstatus, %3\n"                                  \
+         #insn " %1, %2\n"                                          \
+         "csrw mstatus, %0"                                         \
+         : "+&r" (__mstatus), "=&r" (val)                           \
+         : "m" (*addr), "r" (MSTATUS_MPRV), "r" (__mepc));          \
+    return val;                                                     \
+  }
+
+#define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type, insn)                 \
+  static inline void store_##type(type* addr, type val, uintptr_t mepc) \
+  {                                                                     \
+    register uintptr_t __mepc asm ("a2") = mepc;                        \
+    register uintptr_t __mstatus asm ("a3");                            \
+    asm volatile ("csrrs %0, mstatus, %3\n"                             \
+                  #insn " %1, %2\n"                                     \
+                  "csrw mstatus, %0"                                    \
+                  : "+&r" (__mstatus)                                   \
+                  : "r" (val), "m" (*addr), "r" (MSTATUS_MPRV),         \
+                    "r" (__mepc));                                      \
+  }
+
+DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint8_t, lbu)
+DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint16_t, lhu)
+DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int8_t, lb)
+DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int16_t, lh)
+DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int32_t, lw)
+DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint8_t, sb)
+DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint16_t, sh)
+DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint32_t, sw)
+#ifdef __riscv64
+DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lwu)
+DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint64_t, ld)
+DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint64_t, sd)
+#else
+DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lw)
+#endif
+
+static uint32_t __attribute__((always_inline)) get_insn(uintptr_t mepc, uintptr_t* mstatus)
+{
+  register uintptr_t __mepc asm ("a2") = mepc;
+  register uintptr_t __mstatus asm ("a3");
+  uint32_t val;
+#ifndef __riscv_compressed
+  asm ("csrrs %[mstatus], mstatus, %[mprv]\n"
+       "lw %[insn], (%[addr])\n"
+       "csrw mstatus, %[mstatus]"
+       : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val)
+       : [mprv] "r" (MSTATUS_MPRV), [addr] "r" (__mepc));
+#else
+  uintptr_t rvc_mask = 3, tmp;
+  asm ("csrrs %[mstatus], mstatus, %[mprv]\n"
+       "lhu %[insn], (%[addr])\n"
+       "and %[tmp], %[insn], %[rvc_mask]\n"
+       "bne %[tmp], %[rvc_mask], 1f\n"
+       "lh %[tmp], 2(%[addr])\n"
+       "sll %[tmp], %[tmp], 16\n"
+       "add %[insn], %[insn], %[tmp]\n"
+       "1: csrw mstatus, %[mstatus]"
+       : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val), [tmp] "=&r" (tmp)
+       : [mprv] "r" (MSTATUS_MPRV), [addr] "r" (__mepc),
+         [rvc_mask] "r" (rvc_mask));
+#endif
+  *mstatus = __mstatus;
+  return val;
+}
+
+#endif
diff --git a/machine/vm.h b/machine/vm.h
new file mode 100644
index 0000000..f51c639
--- /dev/null
+++ b/machine/vm.h
@@ -0,0 +1,35 @@
+#ifndef _VM_H
+#define _VM_H
+
+#include "encoding.h"
+#include <stdint.h>
+
+#define MEGAPAGE_SIZE ((uintptr_t)(RISCV_PGSIZE << RISCV_PGLEVEL_BITS))
+#ifdef __riscv64
+# define VM_CHOICE VM_SV39
+# define VA_BITS 39
+# define GIGAPAGE_SIZE (MEGAPAGE_SIZE << RISCV_PGLEVEL_BITS)
+#else
+# define VM_CHOICE VM_SV32
+# define VA_BITS 32
+#endif
+
+typedef uintptr_t pte_t;
+extern pte_t* root_page_table;
+
+static inline void flush_tlb()
+{
+  asm volatile("sfence.vm");
+}
+
+static inline pte_t pte_create(uintptr_t ppn, int type)
+{
+  return (ppn << PTE_PPN_SHIFT) | PTE_V | type;
+}
+
+static inline pte_t ptd_create(uintptr_t ppn)
+{
+  return pte_create(ppn, PTE_TYPE_TABLE);
+}
+
+#endif