Incorporate sptbr/sfence.vma changes

5 files changed, 130 insertions, 20 deletions

diff --git a/bbl/bbl.c b/bbl/bbl.c
index 487a416..eacb6c3 100644
--- a/bbl/bbl.c
+++ b/bbl/bbl.c
@@ -51,7 +51,7 @@ static void supervisor_vm_init()
 
   mb();
   root_page_table = root_pt;
-  write_csr(sptbr, (uintptr_t)root_pt >> RISCV_PGSHIFT);
+  write_csr(sptbr, ((uintptr_t)root_pt >> RISCV_PGSHIFT) | SPTBR_MODE);
 }
 
 void boot_loader()
diff --git a/machine/encoding.h b/machine/encoding.h
index 8aeac4c..9a87807 100644
--- a/machine/encoding.h
+++ b/machine/encoding.h
@@ -19,7 +19,6 @@
 #define MSTATUS_MPRV        0x00020000
 #define MSTATUS_PUM         0x00040000
 #define MSTATUS_MXR         0x00080000
-#define MSTATUS_VM          0x1F000000
 #define MSTATUS32_SD        0x80000000
 #define MSTATUS64_SD        0x8000000000000000
 
@@ -114,6 +113,20 @@
 #define VM_SV39  9
 #define VM_SV48  10
 
+#define SPTBR32_MODE 0x80000000
+#define SPTBR32_ASID 0x7FC00000
+#define SPTBR32_PPN  0x003FFFFF
+#define SPTBR64_MODE 0xE000000000000000
+#define SPTBR64_ASID 0x1FFFE00000000000
+#define SPTBR64_PPN  0x0000003FFFFFFFFF
+
+#define SPTBR_MODE_OFF  0
+#define SPTBR_MODE_SV32 1
+#define SPTBR_MODE_SV39 4
+#define SPTBR_MODE_SV48 5
+#define SPTBR_MODE_SV57 6
+#define SPTBR_MODE_SV64 7
+
 #define IRQ_S_SOFT   1
 #define IRQ_H_SOFT   2
 #define IRQ_M_SOFT   3
@@ -208,7 +221,7 @@
 #endif
 
 #endif
-/* Automatically generated by parse-opcodes */
+/* Automatically generated by parse-opcodes.  */
 #ifndef RISCV_ENCODING_H
 #define RISCV_ENCODING_H
 #define MATCH_BEQ 0x63
@@ -397,8 +410,8 @@
 #define MASK_MRET  0xffffffff
 #define MATCH_DRET 0x7b200073
 #define MASK_DRET  0xffffffff
-#define MATCH_SFENCE_VM 0x10400073
-#define MASK_SFENCE_VM  0xfff07fff
+#define MATCH_SFENCE_VMA 0x12000073
+#define MASK_SFENCE_VMA  0xfe007fff
 #define MATCH_WFI 0x10500073
 #define MASK_WFI  0xffffffff
 #define MATCH_CSRRW 0x1073
@@ -457,6 +470,34 @@
 #define MASK_FCVT_D_S  0xfff0007f
 #define MATCH_FSQRT_D 0x5a000053
 #define MASK_FSQRT_D  0xfff0007f
+#define MATCH_FADD_Q 0x6000053
+#define MASK_FADD_Q  0xfe00007f
+#define MATCH_FSUB_Q 0xe000053
+#define MASK_FSUB_Q  0xfe00007f
+#define MATCH_FMUL_Q 0x16000053
+#define MASK_FMUL_Q  0xfe00007f
+#define MATCH_FDIV_Q 0x1e000053
+#define MASK_FDIV_Q  0xfe00007f
+#define MATCH_FSGNJ_Q 0x26000053
+#define MASK_FSGNJ_Q  0xfe00707f
+#define MATCH_FSGNJN_Q 0x26001053
+#define MASK_FSGNJN_Q  0xfe00707f
+#define MATCH_FSGNJX_Q 0x26002053
+#define MASK_FSGNJX_Q  0xfe00707f
+#define MATCH_FMIN_Q 0x2e000053
+#define MASK_FMIN_Q  0xfe00707f
+#define MATCH_FMAX_Q 0x2e001053
+#define MASK_FMAX_Q  0xfe00707f
+#define MATCH_FCVT_S_Q 0x40300053
+#define MASK_FCVT_S_Q  0xfff0007f
+#define MATCH_FCVT_Q_S 0x46000053
+#define MASK_FCVT_Q_S  0xfff0007f
+#define MATCH_FCVT_D_Q 0x42300053
+#define MASK_FCVT_D_Q  0xfff0007f
+#define MATCH_FCVT_Q_D 0x46100053
+#define MASK_FCVT_Q_D  0xfff0007f
+#define MATCH_FSQRT_Q 0x5e000053
+#define MASK_FSQRT_Q  0xfff0007f
 #define MATCH_FLE_S 0xa0000053
 #define MASK_FLE_S  0xfe00707f
 #define MATCH_FLT_S 0xa0001053
@@ -469,6 +510,12 @@
 #define MASK_FLT_D  0xfe00707f
 #define MATCH_FEQ_D 0xa2002053
 #define MASK_FEQ_D  0xfe00707f
+#define MATCH_FLE_Q 0xa6000053
+#define MASK_FLE_Q  0xfe00707f
+#define MATCH_FLT_Q 0xa6001053
+#define MASK_FLT_Q  0xfe00707f
+#define MATCH_FEQ_Q 0xa6002053
+#define MASK_FEQ_Q  0xfe00707f
 #define MATCH_FCVT_W_S 0xc0000053
 #define MASK_FCVT_W_S  0xfff0007f
 #define MATCH_FCVT_WU_S 0xc0100053
@@ -493,6 +540,18 @@
 #define MASK_FMV_X_D  0xfff0707f
 #define MATCH_FCLASS_D 0xe2001053
 #define MASK_FCLASS_D  0xfff0707f
+#define MATCH_FCVT_W_Q 0xc6000053
+#define MASK_FCVT_W_Q  0xfff0007f
+#define MATCH_FCVT_WU_Q 0xc6100053
+#define MASK_FCVT_WU_Q  0xfff0007f
+#define MATCH_FCVT_L_Q 0xc6200053
+#define MASK_FCVT_L_Q  0xfff0007f
+#define MATCH_FCVT_LU_Q 0xc6300053
+#define MASK_FCVT_LU_Q  0xfff0007f
+#define MATCH_FMV_X_Q 0xe6000053
+#define MASK_FMV_X_Q  0xfff0707f
+#define MATCH_FCLASS_Q 0xe6001053
+#define MASK_FCLASS_Q  0xfff0707f
 #define MATCH_FCVT_S_W 0xd0000053
 #define MASK_FCVT_S_W  0xfff0007f
 #define MATCH_FCVT_S_WU 0xd0100053
@@ -513,14 +572,28 @@
 #define MASK_FCVT_D_LU  0xfff0007f
 #define MATCH_FMV_D_X 0xf2000053
 #define MASK_FMV_D_X  0xfff0707f
+#define MATCH_FCVT_Q_W 0xd6000053
+#define MASK_FCVT_Q_W  0xfff0007f
+#define MATCH_FCVT_Q_WU 0xd6100053
+#define MASK_FCVT_Q_WU  0xfff0007f
+#define MATCH_FCVT_Q_L 0xd6200053
+#define MASK_FCVT_Q_L  0xfff0007f
+#define MATCH_FCVT_Q_LU 0xd6300053
+#define MASK_FCVT_Q_LU  0xfff0007f
+#define MATCH_FMV_Q_X 0xf6000053
+#define MASK_FMV_Q_X  0xfff0707f
 #define MATCH_FLW 0x2007
 #define MASK_FLW  0x707f
 #define MATCH_FLD 0x3007
 #define MASK_FLD  0x707f
+#define MATCH_FLQ 0x4007
+#define MASK_FLQ  0x707f
 #define MATCH_FSW 0x2027
 #define MASK_FSW  0x707f
 #define MATCH_FSD 0x3027
 #define MASK_FSD  0x707f
+#define MATCH_FSQ 0x4027
+#define MASK_FSQ  0x707f
 #define MATCH_FMADD_S 0x43
 #define MASK_FMADD_S  0x600007f
 #define MATCH_FMSUB_S 0x47
@@ -537,6 +610,14 @@
 #define MASK_FNMSUB_D  0x600007f
 #define MATCH_FNMADD_D 0x200004f
 #define MASK_FNMADD_D  0x600007f
+#define MATCH_FMADD_Q 0x6000043
+#define MASK_FMADD_Q  0x600007f
+#define MATCH_FMSUB_Q 0x6000047
+#define MASK_FMSUB_Q  0x600007f
+#define MATCH_FNMSUB_Q 0x600004b
+#define MASK_FNMSUB_Q  0x600007f
+#define MATCH_FNMADD_Q 0x600004f
+#define MASK_FNMADD_Q  0x600007f
 #define MATCH_C_NOP 0x1
 #define MASK_C_NOP  0xffff
 #define MATCH_C_ADDI16SP 0x6101
@@ -967,7 +1048,7 @@ DECLARE_INSN(sret, MATCH_SRET, MASK_SRET)
 DECLARE_INSN(hret, MATCH_HRET, MASK_HRET)
 DECLARE_INSN(mret, MATCH_MRET, MASK_MRET)
 DECLARE_INSN(dret, MATCH_DRET, MASK_DRET)
-DECLARE_INSN(sfence_vm, MATCH_SFENCE_VM, MASK_SFENCE_VM)
+DECLARE_INSN(sfence_vma, MATCH_SFENCE_VMA, MASK_SFENCE_VMA)
 DECLARE_INSN(wfi, MATCH_WFI, MASK_WFI)
 DECLARE_INSN(csrrw, MATCH_CSRRW, MASK_CSRRW)
 DECLARE_INSN(csrrs, MATCH_CSRRS, MASK_CSRRS)
@@ -997,12 +1078,29 @@ DECLARE_INSN(fmax_d, MATCH_FMAX_D, MASK_FMAX_D)
 DECLARE_INSN(fcvt_s_d, MATCH_FCVT_S_D, MASK_FCVT_S_D)
 DECLARE_INSN(fcvt_d_s, MATCH_FCVT_D_S, MASK_FCVT_D_S)
 DECLARE_INSN(fsqrt_d, MATCH_FSQRT_D, MASK_FSQRT_D)
+DECLARE_INSN(fadd_q, MATCH_FADD_Q, MASK_FADD_Q)
+DECLARE_INSN(fsub_q, MATCH_FSUB_Q, MASK_FSUB_Q)
+DECLARE_INSN(fmul_q, MATCH_FMUL_Q, MASK_FMUL_Q)
+DECLARE_INSN(fdiv_q, MATCH_FDIV_Q, MASK_FDIV_Q)
+DECLARE_INSN(fsgnj_q, MATCH_FSGNJ_Q, MASK_FSGNJ_Q)
+DECLARE_INSN(fsgnjn_q, MATCH_FSGNJN_Q, MASK_FSGNJN_Q)
+DECLARE_INSN(fsgnjx_q, MATCH_FSGNJX_Q, MASK_FSGNJX_Q)
+DECLARE_INSN(fmin_q, MATCH_FMIN_Q, MASK_FMIN_Q)
+DECLARE_INSN(fmax_q, MATCH_FMAX_Q, MASK_FMAX_Q)
+DECLARE_INSN(fcvt_s_q, MATCH_FCVT_S_Q, MASK_FCVT_S_Q)
+DECLARE_INSN(fcvt_q_s, MATCH_FCVT_Q_S, MASK_FCVT_Q_S)
+DECLARE_INSN(fcvt_d_q, MATCH_FCVT_D_Q, MASK_FCVT_D_Q)
+DECLARE_INSN(fcvt_q_d, MATCH_FCVT_Q_D, MASK_FCVT_Q_D)
+DECLARE_INSN(fsqrt_q, MATCH_FSQRT_Q, MASK_FSQRT_Q)
 DECLARE_INSN(fle_s, MATCH_FLE_S, MASK_FLE_S)
 DECLARE_INSN(flt_s, MATCH_FLT_S, MASK_FLT_S)
 DECLARE_INSN(feq_s, MATCH_FEQ_S, MASK_FEQ_S)
 DECLARE_INSN(fle_d, MATCH_FLE_D, MASK_FLE_D)
 DECLARE_INSN(flt_d, MATCH_FLT_D, MASK_FLT_D)
 DECLARE_INSN(feq_d, MATCH_FEQ_D, MASK_FEQ_D)
+DECLARE_INSN(fle_q, MATCH_FLE_Q, MASK_FLE_Q)
+DECLARE_INSN(flt_q, MATCH_FLT_Q, MASK_FLT_Q)
+DECLARE_INSN(feq_q, MATCH_FEQ_Q, MASK_FEQ_Q)
 DECLARE_INSN(fcvt_w_s, MATCH_FCVT_W_S, MASK_FCVT_W_S)
 DECLARE_INSN(fcvt_wu_s, MATCH_FCVT_WU_S, MASK_FCVT_WU_S)
 DECLARE_INSN(fcvt_l_s, MATCH_FCVT_L_S, MASK_FCVT_L_S)
@@ -1015,6 +1113,12 @@ DECLARE_INSN(fcvt_l_d, MATCH_FCVT_L_D, MASK_FCVT_L_D)
 DECLARE_INSN(fcvt_lu_d, MATCH_FCVT_LU_D, MASK_FCVT_LU_D)
 DECLARE_INSN(fmv_x_d, MATCH_FMV_X_D, MASK_FMV_X_D)
 DECLARE_INSN(fclass_d, MATCH_FCLASS_D, MASK_FCLASS_D)
+DECLARE_INSN(fcvt_w_q, MATCH_FCVT_W_Q, MASK_FCVT_W_Q)
+DECLARE_INSN(fcvt_wu_q, MATCH_FCVT_WU_Q, MASK_FCVT_WU_Q)
+DECLARE_INSN(fcvt_l_q, MATCH_FCVT_L_Q, MASK_FCVT_L_Q)
+DECLARE_INSN(fcvt_lu_q, MATCH_FCVT_LU_Q, MASK_FCVT_LU_Q)
+DECLARE_INSN(fmv_x_q, MATCH_FMV_X_Q, MASK_FMV_X_Q)
+DECLARE_INSN(fclass_q, MATCH_FCLASS_Q, MASK_FCLASS_Q)
 DECLARE_INSN(fcvt_s_w, MATCH_FCVT_S_W, MASK_FCVT_S_W)
 DECLARE_INSN(fcvt_s_wu, MATCH_FCVT_S_WU, MASK_FCVT_S_WU)
 DECLARE_INSN(fcvt_s_l, MATCH_FCVT_S_L, MASK_FCVT_S_L)
@@ -1025,10 +1129,17 @@ DECLARE_INSN(fcvt_d_wu, MATCH_FCVT_D_WU, MASK_FCVT_D_WU)
 DECLARE_INSN(fcvt_d_l, MATCH_FCVT_D_L, MASK_FCVT_D_L)
 DECLARE_INSN(fcvt_d_lu, MATCH_FCVT_D_LU, MASK_FCVT_D_LU)
 DECLARE_INSN(fmv_d_x, MATCH_FMV_D_X, MASK_FMV_D_X)
+DECLARE_INSN(fcvt_q_w, MATCH_FCVT_Q_W, MASK_FCVT_Q_W)
+DECLARE_INSN(fcvt_q_wu, MATCH_FCVT_Q_WU, MASK_FCVT_Q_WU)
+DECLARE_INSN(fcvt_q_l, MATCH_FCVT_Q_L, MASK_FCVT_Q_L)
+DECLARE_INSN(fcvt_q_lu, MATCH_FCVT_Q_LU, MASK_FCVT_Q_LU)
+DECLARE_INSN(fmv_q_x, MATCH_FMV_Q_X, MASK_FMV_Q_X)
 DECLARE_INSN(flw, MATCH_FLW, MASK_FLW)
 DECLARE_INSN(fld, MATCH_FLD, MASK_FLD)
+DECLARE_INSN(flq, MATCH_FLQ, MASK_FLQ)
 DECLARE_INSN(fsw, MATCH_FSW, MASK_FSW)
 DECLARE_INSN(fsd, MATCH_FSD, MASK_FSD)
+DECLARE_INSN(fsq, MATCH_FSQ, MASK_FSQ)
 DECLARE_INSN(fmadd_s, MATCH_FMADD_S, MASK_FMADD_S)
 DECLARE_INSN(fmsub_s, MATCH_FMSUB_S, MASK_FMSUB_S)
 DECLARE_INSN(fnmsub_s, MATCH_FNMSUB_S, MASK_FNMSUB_S)
@@ -1037,6 +1148,10 @@ DECLARE_INSN(fmadd_d, MATCH_FMADD_D, MASK_FMADD_D)
 DECLARE_INSN(fmsub_d, MATCH_FMSUB_D, MASK_FMSUB_D)
 DECLARE_INSN(fnmsub_d, MATCH_FNMSUB_D, MASK_FNMSUB_D)
 DECLARE_INSN(fnmadd_d, MATCH_FNMADD_D, MASK_FNMADD_D)
+DECLARE_INSN(fmadd_q, MATCH_FMADD_Q, MASK_FMADD_Q)
+DECLARE_INSN(fmsub_q, MATCH_FMSUB_Q, MASK_FMSUB_Q)
+DECLARE_INSN(fnmsub_q, MATCH_FNMSUB_Q, MASK_FNMSUB_Q)
+DECLARE_INSN(fnmadd_q, MATCH_FNMADD_Q, MASK_FNMADD_Q)
 DECLARE_INSN(c_nop, MATCH_C_NOP, MASK_C_NOP)
 DECLARE_INSN(c_addi16sp, MATCH_C_ADDI16SP, MASK_C_ADDI16SP)
 DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR)
diff --git a/machine/minit.c b/machine/minit.c
index b3f2c86..06e8d25 100644
--- a/machine/minit.c
+++ b/machine/minit.c
@@ -15,15 +15,8 @@ size_t plic_ndevs;
 
 static void mstatus_init()
 {
-  // Enable FPU and set VM mode
-  uintptr_t ms = 0;
-  ms = INSERT_FIELD(ms, MSTATUS_VM, VM_CHOICE);
-  ms = INSERT_FIELD(ms, MSTATUS_FS, 1);
-  write_csr(mstatus, ms);
-
-  // Make sure the hart actually supports the VM mode we want
-  ms = read_csr(mstatus);
-  assert(EXTRACT_FIELD(ms, MSTATUS_VM) == VM_CHOICE);
+  // Enable FPU
+  write_csr(mstatus, MSTATUS_FS);
 
   // Enable user/supervisor use of perf counters
   write_csr(mucounteren, -1);
@@ -156,7 +149,7 @@ void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t stack)
   write_csr(mstatus, mstatus);
   write_csr(mscratch, MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE);
   write_csr(mepc, fn);
-  write_csr(sptbr, (uintptr_t)root_page_table >> RISCV_PGSHIFT);
+  write_csr(sptbr, ((uintptr_t)root_page_table >> RISCV_PGSHIFT) | SPTBR_MODE);
   asm volatile ("mv a0, %0; mv sp, %0; mret" : : "r" (stack));
   __builtin_unreachable();
 }
diff --git a/machine/mtrap.c b/machine/mtrap.c
index 07f7d10..34772b7 100644
--- a/machine/mtrap.c
+++ b/machine/mtrap.c
@@ -3,6 +3,7 @@
 #include "htif.h"
 #include "atomic.h"
 #include "bits.h"
+#include "vm.h"
 #include <errno.h>
 #include <stdarg.h>
 #include <stdio.h>
@@ -176,6 +177,7 @@ void software_interrupt()
   *HLS()->ipi = 0;
   mb();
   int ipi_pending = atomic_swap(&HLS()->mipi_pending, 0);
+  mb();
 
   if (ipi_pending & IPI_SOFT) {
     HLS()->sipi_pending = 1;
@@ -186,7 +188,7 @@ void software_interrupt()
     asm volatile ("fence.i");
 
   if (ipi_pending & IPI_SFENCE_VM)
-    asm volatile ("sfence.vm");
+    flush_tlb();
 }
 
 static void send_ipi_many(uintptr_t* pmask, int event)
diff --git a/machine/vm.h b/machine/vm.h
index 4e6bcc1..6d3da4e 100644
--- a/machine/vm.h
+++ b/machine/vm.h
@@ -6,11 +6,11 @@
 
 #define MEGAPAGE_SIZE ((uintptr_t)(RISCV_PGSIZE << RISCV_PGLEVEL_BITS))
 #if __riscv_xlen == 64
-# define VM_CHOICE VM_SV39
+# define SPTBR_MODE INSERT_FIELD(0, SPTBR64_MODE, SPTBR_MODE_SV39)
 # define VA_BITS 39
 # define GIGAPAGE_SIZE (MEGAPAGE_SIZE << RISCV_PGLEVEL_BITS)
 #else
-# define VM_CHOICE VM_SV32
+# define SPTBR_MODE INSERT_FIELD(0, SPTBR32_MODE, SPTBR_MODE_SV32)
 # define VA_BITS 32
 #endif
 
@@ -19,7 +19,7 @@ extern pte_t* root_page_table;
 
 static inline void flush_tlb()
 {
-  asm volatile("sfence.vm");
+  asm volatile ("sfence.vma");
 }
 
 static inline pte_t pte_create(uintptr_t ppn, int type)