Factor emulation routines into multiple files

7 files changed, 233 insertions, 207 deletions

diff --git a/machine/emulation.c b/machine/emulation.c
index 5a66997..5e3ca2e 100644
--- a/machine/emulation.c
+++ b/machine/emulation.c
@@ -10,7 +10,7 @@ void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
   asm (".pushsection .rodata\n"
        "illegal_insn_trap_table:\n"
        "  .word truly_illegal_insn\n"
-#ifdef PK_ENABLE_FP_EMULATION
+#if !defined(__riscv_hard_float) && defined(PK_ENABLE_FP_EMULATION)
        "  .word emulate_float_load\n"
 #else
        "  .word truly_illegal_insn\n"
@@ -22,16 +22,24 @@ void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
        "  .word truly_illegal_insn\n"
        "  .word truly_illegal_insn\n"
        "  .word truly_illegal_insn\n"
-#ifdef PK_ENABLE_FP_EMULATION
+#if !defined(__riscv_hard_float) && defined(PK_ENABLE_FP_EMULATION)
        "  .word emulate_float_store\n"
 #else
        "  .word truly_illegal_insn\n"
 #endif
        "  .word truly_illegal_insn\n"
        "  .word truly_illegal_insn\n"
+#if !defined(__riscv_muldiv)
        "  .word emulate_mul_div\n"
+#else
        "  .word truly_illegal_insn\n"
+#endif
+       "  .word truly_illegal_insn\n"
+#if !defined(__riscv_muldiv) && defined(__riscv64)
        "  .word emulate_mul_div32\n"
+#else
+       "  .word truly_illegal_insn\n"
+#endif
        "  .word truly_illegal_insn\n"
 #ifdef PK_ENABLE_FP_EMULATION
        "  .word emulate_fmadd\n"
@@ -78,144 +86,6 @@ void __attribute__((noinline)) truly_illegal_insn(uintptr_t* regs, uintptr_t mca
   redirect_trap(mepc, mstatus);
 }
 
-void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
-{
-  union {
-    uint8_t bytes[8];
-    uintptr_t intx;
-    uint64_t int64;
-  } val;
-  uintptr_t mstatus;
-  insn_t insn = get_insn(mepc, &mstatus);
-  uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn);
-
-  int shift = 0, fp = 0, len;
-  if ((insn & MASK_LW) == MATCH_LW)
-    len = 4, shift = 8*(sizeof(uintptr_t) - len);
-#ifdef __riscv64
-  else if ((insn & MASK_LD) == MATCH_LD)
-    len = 8, shift = 8*(sizeof(uintptr_t) - len);
-  else if ((insn & MASK_LWU) == MATCH_LWU)
-    fp = 0, len = 4, shift = 0;
-#endif
-  else if ((insn & MASK_FLD) == MATCH_FLD)
-    fp = 1, len = 8;
-  else if ((insn & MASK_FLW) == MATCH_FLW)
-    fp = 1, len = 4;
-  else if ((insn & MASK_LH) == MATCH_LH)
-    len = 2, shift = 8*(sizeof(uintptr_t) - len);
-  else if ((insn & MASK_LHU) == MATCH_LHU)
-    len = 2;
-  else
-    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
-
-  val.int64 = 0;
-  for (intptr_t i = len-1; i >= 0; i--)
-    val.bytes[i] = load_uint8_t((void *)(addr + i), mepc);
-
-  if (!fp)
-    SET_RD(insn, regs, (intptr_t)val.intx << shift >> shift);
-  else if (len == 8)
-    SET_F64_RD(insn, regs, val.int64);
-  else
-    SET_F32_RD(insn, regs, val.intx);
-
-  write_csr(mepc, mepc + 4);
-}
-
-void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
-{
-  union {
-    uint8_t bytes[8];
-    uintptr_t intx;
-    uint64_t int64;
-  } val;
-  uintptr_t mstatus;
-  insn_t insn = get_insn(mepc, &mstatus);
-  int len;
-
-  val.intx = GET_RS2(insn, regs);
-  if ((insn & MASK_SW) == MATCH_SW)
-    len = 4;
-#ifdef __riscv64
-  else if ((insn & MASK_SD) == MATCH_SD)
-    len = 8;
-#endif
-  else if ((insn & MASK_FSD) == MATCH_FSD)
-    len = 8, val.int64 = GET_F64_RS2(insn, regs);
-  else if ((insn & MASK_FSW) == MATCH_FSW)
-    len = 4, val.intx = GET_F32_RS2(insn, regs);
-  else if ((insn & MASK_SH) == MATCH_SH)
-    len = 2;
-  else
-    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
-
-  uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn);
-  for (int i = 0; i < len; i++)
-    store_uint8_t((void *)(addr + i), val.bytes[i], mepc);
-
-  write_csr(mepc, mepc + 4);
-}
-
-#ifdef __riscv64
-typedef __int128 double_int;
-#else
-typedef int64_t double_int;
-#endif
-
-DECLARE_EMULATION_FUNC(emulate_mul_div)
-{
-  uintptr_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs), val;
-
-#ifndef __riscv_muldiv
-  // If compiled with -mno-multiply, GCC will expand these out
-  if ((insn & MASK_MUL) == MATCH_MUL)
-    val = rs1 * rs2;
-  else if ((insn & MASK_DIV) == MATCH_DIV)
-    val = (intptr_t)rs1 / (intptr_t)rs2;
-  else if ((insn & MASK_DIVU) == MATCH_DIVU)
-    val = rs1 / rs2;
-  else if ((insn & MASK_REM) == MATCH_REM)
-    val = (intptr_t)rs1 % (intptr_t)rs2;
-  else if ((insn & MASK_REMU) == MATCH_REMU)
-    val = rs1 % rs2;
-  else if ((insn & MASK_MULH) == MATCH_MULH)
-    val = ((double_int)(intptr_t)rs1 * (double_int)(intptr_t)rs2) >> (8 * sizeof(rs1));
-  else if ((insn & MASK_MULHU) == MATCH_MULHU)
-    val = ((double_int)rs1 * (double_int)rs2) >> (8 * sizeof(rs1));
-  else if ((insn & MASK_MULHSU) == MATCH_MULHSU)
-    val = ((double_int)(intptr_t)rs1 * (double_int)rs2) >> (8 * sizeof(rs1));
-  else
-#endif
-    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
-
-  SET_RD(insn, regs, val);
-}
-
-DECLARE_EMULATION_FUNC(emulate_mul_div32)
-{
-  uint32_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs);
-  int32_t val;
-
-#if defined(__riscv64) && !defined(__riscv_muldiv)
-  // If compiled with -mno-multiply, GCC will expand these out
-  if ((insn & MASK_MULW) == MATCH_MULW)
-    val = rs1 * rs2;
-  else if ((insn & MASK_DIVW) == MATCH_DIVW)
-    val = (int32_t)rs1 / (int32_t)rs2;
-  else if ((insn & MASK_DIVUW) == MATCH_DIVUW)
-    val = rs1 / rs2;
-  else if ((insn & MASK_REMW) == MATCH_REMW)
-    val = (int32_t)rs1 % (int32_t)rs2;
-  else if ((insn & MASK_REMUW) == MATCH_REMUW)
-    val = rs1 % rs2;
-  else
-#endif
-    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
-
-  SET_RD(insn, regs, val);
-}
-
 static inline int emulate_read_csr(int num, uintptr_t mstatus, uintptr_t* result)
 {
   switch (num)
@@ -252,7 +122,7 @@ static inline int emulate_read_csr(int num, uintptr_t mstatus, uintptr_t* result
                  + HLS()->sinstret_delta) >> 32;
       return 0;
 #endif
-#ifdef PK_ENABLE_FP_EMULATION
+#if !defined(__riscv_hard_float) && defined(PK_ENABLE_FP_EMULATION)
     case CSR_FRM:
       if ((mstatus & MSTATUS_FS) == 0) break;
       *result = GET_FRM();
@@ -274,7 +144,7 @@ static inline int emulate_write_csr(int num, uintptr_t value, uintptr_t mstatus)
 {
   switch (num)
   {
-#ifdef PK_ENABLE_FP_EMULATION
+#if !defined(__riscv_hard_float) && defined(PK_ENABLE_FP_EMULATION)
     case CSR_FRM: SET_FRM(value); return 0;
     case CSR_FFLAGS: SET_FFLAGS(value); return 0;
     case CSR_FCSR: SET_FCSR(value); return 0;
diff --git a/machine/fp_emulation.c b/machine/fp_emulation.c
index 536967f..7523a29 100644
--- a/machine/fp_emulation.c
+++ b/machine/fp_emulation.c
@@ -3,70 +3,6 @@
 #include "softfloat.h"
 #include "config.h"
 
-DECLARE_EMULATION_FUNC(emulate_float_load)
-{
-  uint64_t val;
-  uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn);
-  
-  switch (insn & MASK_FUNCT3)
-  {
-    case MATCH_FLW & MASK_FUNCT3:
-      if (addr % 4 != 0)
-        return misaligned_load_trap(regs, mcause, mepc);
-
-      SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc));
-      break;
-
-    case MATCH_FLD & MASK_FUNCT3:
-      if (addr % sizeof(uintptr_t) != 0)
-        return misaligned_load_trap(regs, mcause, mepc);
-
-#ifdef __riscv64
-      val = load_uint64_t((void *)addr, mepc);
-#else
-      val = load_uint32_t(addr, mepc);
-      val += (uint64_t)load_uint32_t((void *)(addr + 4), mepc) << 32;
-#endif
-      SET_F64_RD(insn, regs, val);
-      break;
-
-    default:
-      return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
-  }
-}
-
-DECLARE_EMULATION_FUNC(emulate_float_store)
-{
-  uint64_t val;
-  uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn);
-  
-  switch (insn & MASK_FUNCT3)
-  {
-    case MATCH_FSW & MASK_FUNCT3:
-      if (addr % 4 != 0)
-        return misaligned_store_trap(regs, mcause, mepc);
-
-      store_uint32_t((void *)addr, GET_F32_RS2(insn, regs), mepc);
-      break;
-
-    case MATCH_FSD & MASK_FUNCT3:
-      if (addr % sizeof(uintptr_t) != 0)
-        return misaligned_store_trap(regs, mcause, mepc);
-
-      val = GET_F64_RS2(insn, regs);
-#ifdef __riscv64
-      store_uint64_t((void *)addr, val, mepc);
-#else
-      store_uint32_t((void *)addr, val, mepc);
-      store_uint32_t((void *)(addr + 4), val >> 32, mepc);
-#endif
-      break;
-
-    default:
-      return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
-  }
-}
-
 DECLARE_EMULATION_FUNC(emulate_fp)
 {
   asm (".pushsection .rodata\n"
diff --git a/machine/fp_emulation.h b/machine/fp_emulation.h
index d2357b7..8d209e1 100644
--- a/machine/fp_emulation.h
+++ b/machine/fp_emulation.h
@@ -46,6 +46,7 @@
   asm volatile ("":"+r"(tp)); })
 # define softfloat_raiseFlags(which) set_csr(fflags, which)
 # define softfloat_roundingMode ({ register int tp asm("tp"); tp; })
+# define SET_FS_DIRTY() ((void) 0)
 #else
 # define GET_F64_REG(insn, pos, regs) (*(int64_t*)((void*)((regs) + 32) + (((insn) >> ((pos)-3)) & 0xf8)))
 # define SET_F64_REG(insn, pos, regs, val) (GET_F64_REG(insn, pos, regs) = (val))
@@ -66,6 +67,7 @@
   asm volatile ("":"+r"(tp)); })
 # define softfloat_raiseFlags(which) ({ asm volatile ("or tp, tp, %0" :: "rI"(which)); })
 # define softfloat_roundingMode ({ register int tp asm("tp"); tp >> 13; })
+# define SET_FS_DIRTY() set_csr(mstatus, MSTATUS_FS)
 #endif
 
 #define GET_F32_RS1(insn, regs) (GET_F32_REG(insn, 15, regs))
@@ -76,6 +78,5 @@
 #define GET_F64_RS3(insn, regs) (GET_F64_REG(insn, 27, regs))
 #define SET_F32_RD(insn, regs, val) (SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY())
 #define SET_F64_RD(insn, regs, val) (SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY())
-#define SET_FS_DIRTY() set_csr(mstatus, MSTATUS_FS)
 
 #endif
diff --git a/machine/fp_ldst.c b/machine/fp_ldst.c
new file mode 100644
index 0000000..9d0a49f
--- /dev/null
+++ b/machine/fp_ldst.c
@@ -0,0 +1,67 @@
+#include "fp_emulation.h"
+#include "unprivileged_memory.h"
+
+DECLARE_EMULATION_FUNC(emulate_float_load)
+{
+  uint64_t val;
+  uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn);
+  
+  switch (insn & MASK_FUNCT3)
+  {
+    case MATCH_FLW & MASK_FUNCT3:
+      if (addr % 4 != 0)
+        return misaligned_load_trap(regs, mcause, mepc);
+
+      SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc));
+      break;
+
+    case MATCH_FLD & MASK_FUNCT3:
+      if (addr % sizeof(uintptr_t) != 0)
+        return misaligned_load_trap(regs, mcause, mepc);
+
+#ifdef __riscv64
+      val = load_uint64_t((void *)addr, mepc);
+#else
+      val = load_uint32_t(addr, mepc);
+      val += (uint64_t)load_uint32_t((void *)(addr + 4), mepc) << 32;
+#endif
+      SET_F64_RD(insn, regs, val);
+      break;
+
+    default:
+      return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
+DECLARE_EMULATION_FUNC(emulate_float_store)
+{
+  uint64_t val;
+  uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn);
+  
+  switch (insn & MASK_FUNCT3)
+  {
+    case MATCH_FSW & MASK_FUNCT3:
+      if (addr % 4 != 0)
+        return misaligned_store_trap(regs, mcause, mepc);
+
+      store_uint32_t((void *)addr, GET_F32_RS2(insn, regs), mepc);
+      break;
+
+    case MATCH_FSD & MASK_FUNCT3:
+      if (addr % sizeof(uintptr_t) != 0)
+        return misaligned_store_trap(regs, mcause, mepc);
+
+      val = GET_F64_RS2(insn, regs);
+#ifdef __riscv64
+      store_uint64_t((void *)addr, val, mepc);
+#else
+      store_uint32_t((void *)addr, val, mepc);
+      store_uint32_t((void *)(addr + 4), val >> 32, mepc);
+#endif
+      break;
+
+    default:
+      return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+  }
+}
+
diff --git a/machine/machine.mk.in b/machine/machine.mk.in
index a0b7ca9..4628954 100644
--- a/machine/machine.mk.in
+++ b/machine/machine.mk.in
@@ -18,7 +18,10 @@ machine_c_srcs = \
   mtrap.c \
   minit.c \
   emulation.c \
+  muldiv_emulation.c \
   fp_emulation.c \
+  fp_ldst.c \
+  misaligned_ldst.c \
   sbi_impl.c \
   configstring.c \
 
diff --git a/machine/misaligned_ldst.c b/machine/misaligned_ldst.c
new file mode 100644
index 0000000..cf6b1d6
--- /dev/null
+++ b/machine/misaligned_ldst.c
@@ -0,0 +1,85 @@
+#include "emulation.h"
+#include "fp_emulation.h"
+#include "unprivileged_memory.h"
+#include "mtrap.h"
+
+union byte_array {
+  uint8_t bytes[8];
+  uintptr_t intx;
+  uint64_t int64;
+};
+
+void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
+{
+  union byte_array val;
+  uintptr_t mstatus;
+  insn_t insn = get_insn(mepc, &mstatus);
+  uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn);
+
+  int shift = 0, fp = 0, len;
+  if ((insn & MASK_LW) == MATCH_LW)
+    len = 4, shift = 8*(sizeof(uintptr_t) - len);
+#ifdef __riscv64
+  else if ((insn & MASK_LD) == MATCH_LD)
+    len = 8, shift = 8*(sizeof(uintptr_t) - len);
+  else if ((insn & MASK_LWU) == MATCH_LWU)
+    len = 4;
+#endif
+#ifdef PK_ENABLE_FP_EMULATION
+  else if ((insn & MASK_FLD) == MATCH_FLD)
+    fp = 1, len = 8;
+  else if ((insn & MASK_FLW) == MATCH_FLW)
+    fp = 1, len = 4;
+#endif
+  else if ((insn & MASK_LH) == MATCH_LH)
+    len = 2, shift = 8*(sizeof(uintptr_t) - len);
+  else if ((insn & MASK_LHU) == MATCH_LHU)
+    len = 2;
+  else
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  val.int64 = 0;
+  for (intptr_t i = len-1; i >= 0; i--)
+    val.bytes[i] = load_uint8_t((void *)(addr + i), mepc);
+
+  if (!fp)
+    SET_RD(insn, regs, (intptr_t)val.intx << shift >> shift);
+  else if (len == 8)
+    SET_F64_RD(insn, regs, val.int64);
+  else
+    SET_F32_RD(insn, regs, val.intx);
+
+  write_csr(mepc, mepc + 4);
+}
+
+void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
+{
+  union byte_array val;
+  uintptr_t mstatus;
+  insn_t insn = get_insn(mepc, &mstatus);
+  int len;
+
+  val.intx = GET_RS2(insn, regs);
+  if ((insn & MASK_SW) == MATCH_SW)
+    len = 4;
+#ifdef __riscv64
+  else if ((insn & MASK_SD) == MATCH_SD)
+    len = 8;
+#endif
+#ifdef PK_ENABLE_FP_EMULATION
+  else if ((insn & MASK_FSD) == MATCH_FSD)
+    len = 8, val.int64 = GET_F64_RS2(insn, regs);
+  else if ((insn & MASK_FSW) == MATCH_FSW)
+    len = 4, val.intx = GET_F32_RS2(insn, regs);
+#endif
+  else if ((insn & MASK_SH) == MATCH_SH)
+    len = 2;
+  else
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn);
+  for (int i = 0; i < len; i++)
+    store_uint8_t((void *)(addr + i), val.bytes[i], mepc);
+
+  write_csr(mepc, mepc + 4);
+}
diff --git a/machine/muldiv_emulation.c b/machine/muldiv_emulation.c
new file mode 100644
index 0000000..d0583c5
--- /dev/null
+++ b/machine/muldiv_emulation.c
@@ -0,0 +1,64 @@
+#include "emulation.h"
+
+#ifndef __riscv_muldiv
+
+#ifdef __riscv64
+typedef __int128 double_int;
+#else
+typedef int64_t double_int;
+#endif
+
+// These routines rely on the compiler to turn these operations into libcalls
+// when not natively supported.  So work on making those go fast.
+
+DECLARE_EMULATION_FUNC(emulate_mul_div)
+{
+  uintptr_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs), val;
+
+  if ((insn & MASK_MUL) == MATCH_MUL)
+    val = rs1 * rs2;
+  else if ((insn & MASK_DIV) == MATCH_DIV)
+    val = (intptr_t)rs1 / (intptr_t)rs2;
+  else if ((insn & MASK_DIVU) == MATCH_DIVU)
+    val = rs1 / rs2;
+  else if ((insn & MASK_REM) == MATCH_REM)
+    val = (intptr_t)rs1 % (intptr_t)rs2;
+  else if ((insn & MASK_REMU) == MATCH_REMU)
+    val = rs1 % rs2;
+  else if ((insn & MASK_MULH) == MATCH_MULH)
+    val = ((double_int)(intptr_t)rs1 * (double_int)(intptr_t)rs2) >> (8 * sizeof(rs1));
+  else if ((insn & MASK_MULHU) == MATCH_MULHU)
+    val = ((double_int)rs1 * (double_int)rs2) >> (8 * sizeof(rs1));
+  else if ((insn & MASK_MULHSU) == MATCH_MULHSU)
+    val = ((double_int)(intptr_t)rs1 * (double_int)rs2) >> (8 * sizeof(rs1));
+  else
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  SET_RD(insn, regs, val);
+}
+
+#ifdef __riscv64
+
+DECLARE_EMULATION_FUNC(emulate_mul_div32)
+{
+  uint32_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs);
+  int32_t val;
+
+  if ((insn & MASK_MULW) == MATCH_MULW)
+    val = rs1 * rs2;
+  else if ((insn & MASK_DIVW) == MATCH_DIVW)
+    val = (int32_t)rs1 / (int32_t)rs2;
+  else if ((insn & MASK_DIVUW) == MATCH_DIVUW)
+    val = rs1 / rs2;
+  else if ((insn & MASK_REMW) == MATCH_REMW)
+    val = (int32_t)rs1 % (int32_t)rs2;
+  else if ((insn & MASK_REMUW) == MATCH_REMUW)
+    val = rs1 % rs2;
+  else
+    return truly_illegal_insn(regs, mcause, mepc, mstatus, insn);
+
+  SET_RD(insn, regs, val);
+}
+
+#endif
+#endif