From 4678e84c040531a48eff2108fd9212660bf527ae Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 15 Feb 2017 15:20:06 -0800 Subject: Incorporate sptbr/sfence.vma changes --- bbl/bbl.c | 2 +- machine/encoding.h | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++--- machine/minit.c | 13 ++---- machine/mtrap.c | 4 +- machine/vm.h | 6 +-- 5 files changed, 130 insertions(+), 20 deletions(-) diff --git a/bbl/bbl.c b/bbl/bbl.c index 487a416..eacb6c3 100644 --- a/bbl/bbl.c +++ b/bbl/bbl.c @@ -51,7 +51,7 @@ static void supervisor_vm_init() mb(); root_page_table = root_pt; - write_csr(sptbr, (uintptr_t)root_pt >> RISCV_PGSHIFT); + write_csr(sptbr, ((uintptr_t)root_pt >> RISCV_PGSHIFT) | SPTBR_MODE); } void boot_loader() diff --git a/machine/encoding.h b/machine/encoding.h index 8aeac4c..9a87807 100644 --- a/machine/encoding.h +++ b/machine/encoding.h @@ -19,7 +19,6 @@ #define MSTATUS_MPRV 0x00020000 #define MSTATUS_PUM 0x00040000 #define MSTATUS_MXR 0x00080000 -#define MSTATUS_VM 0x1F000000 #define MSTATUS32_SD 0x80000000 #define MSTATUS64_SD 0x8000000000000000 @@ -114,6 +113,20 @@ #define VM_SV39 9 #define VM_SV48 10 +#define SPTBR32_MODE 0x80000000 +#define SPTBR32_ASID 0x7FC00000 +#define SPTBR32_PPN 0x003FFFFF +#define SPTBR64_MODE 0xE000000000000000 +#define SPTBR64_ASID 0x1FFFE00000000000 +#define SPTBR64_PPN 0x0000003FFFFFFFFF + +#define SPTBR_MODE_OFF 0 +#define SPTBR_MODE_SV32 1 +#define SPTBR_MODE_SV39 4 +#define SPTBR_MODE_SV48 5 +#define SPTBR_MODE_SV57 6 +#define SPTBR_MODE_SV64 7 + #define IRQ_S_SOFT 1 #define IRQ_H_SOFT 2 #define IRQ_M_SOFT 3 @@ -208,7 +221,7 @@ #endif #endif -/* Automatically generated by parse-opcodes */ +/* Automatically generated by parse-opcodes. */ #ifndef RISCV_ENCODING_H #define RISCV_ENCODING_H #define MATCH_BEQ 0x63 @@ -397,8 +410,8 @@ #define MASK_MRET 0xffffffff #define MATCH_DRET 0x7b200073 #define MASK_DRET 0xffffffff -#define MATCH_SFENCE_VM 0x10400073 -#define MASK_SFENCE_VM 0xfff07fff +#define MATCH_SFENCE_VMA 0x12000073 +#define MASK_SFENCE_VMA 0xfe007fff #define MATCH_WFI 0x10500073 #define MASK_WFI 0xffffffff #define MATCH_CSRRW 0x1073 @@ -457,6 +470,34 @@ #define MASK_FCVT_D_S 0xfff0007f #define MATCH_FSQRT_D 0x5a000053 #define MASK_FSQRT_D 0xfff0007f +#define MATCH_FADD_Q 0x6000053 +#define MASK_FADD_Q 0xfe00007f +#define MATCH_FSUB_Q 0xe000053 +#define MASK_FSUB_Q 0xfe00007f +#define MATCH_FMUL_Q 0x16000053 +#define MASK_FMUL_Q 0xfe00007f +#define MATCH_FDIV_Q 0x1e000053 +#define MASK_FDIV_Q 0xfe00007f +#define MATCH_FSGNJ_Q 0x26000053 +#define MASK_FSGNJ_Q 0xfe00707f +#define MATCH_FSGNJN_Q 0x26001053 +#define MASK_FSGNJN_Q 0xfe00707f +#define MATCH_FSGNJX_Q 0x26002053 +#define MASK_FSGNJX_Q 0xfe00707f +#define MATCH_FMIN_Q 0x2e000053 +#define MASK_FMIN_Q 0xfe00707f +#define MATCH_FMAX_Q 0x2e001053 +#define MASK_FMAX_Q 0xfe00707f +#define MATCH_FCVT_S_Q 0x40300053 +#define MASK_FCVT_S_Q 0xfff0007f +#define MATCH_FCVT_Q_S 0x46000053 +#define MASK_FCVT_Q_S 0xfff0007f +#define MATCH_FCVT_D_Q 0x42300053 +#define MASK_FCVT_D_Q 0xfff0007f +#define MATCH_FCVT_Q_D 0x46100053 +#define MASK_FCVT_Q_D 0xfff0007f +#define MATCH_FSQRT_Q 0x5e000053 +#define MASK_FSQRT_Q 0xfff0007f #define MATCH_FLE_S 0xa0000053 #define MASK_FLE_S 0xfe00707f #define MATCH_FLT_S 0xa0001053 @@ -469,6 +510,12 @@ #define MASK_FLT_D 0xfe00707f #define MATCH_FEQ_D 0xa2002053 #define MASK_FEQ_D 0xfe00707f +#define MATCH_FLE_Q 0xa6000053 +#define MASK_FLE_Q 0xfe00707f +#define MATCH_FLT_Q 0xa6001053 +#define MASK_FLT_Q 0xfe00707f +#define MATCH_FEQ_Q 0xa6002053 +#define MASK_FEQ_Q 0xfe00707f #define MATCH_FCVT_W_S 0xc0000053 #define MASK_FCVT_W_S 0xfff0007f #define MATCH_FCVT_WU_S 0xc0100053 @@ -493,6 +540,18 @@ #define MASK_FMV_X_D 0xfff0707f #define MATCH_FCLASS_D 0xe2001053 #define MASK_FCLASS_D 0xfff0707f +#define MATCH_FCVT_W_Q 0xc6000053 +#define MASK_FCVT_W_Q 0xfff0007f +#define MATCH_FCVT_WU_Q 0xc6100053 +#define MASK_FCVT_WU_Q 0xfff0007f +#define MATCH_FCVT_L_Q 0xc6200053 +#define MASK_FCVT_L_Q 0xfff0007f +#define MATCH_FCVT_LU_Q 0xc6300053 +#define MASK_FCVT_LU_Q 0xfff0007f +#define MATCH_FMV_X_Q 0xe6000053 +#define MASK_FMV_X_Q 0xfff0707f +#define MATCH_FCLASS_Q 0xe6001053 +#define MASK_FCLASS_Q 0xfff0707f #define MATCH_FCVT_S_W 0xd0000053 #define MASK_FCVT_S_W 0xfff0007f #define MATCH_FCVT_S_WU 0xd0100053 @@ -513,14 +572,28 @@ #define MASK_FCVT_D_LU 0xfff0007f #define MATCH_FMV_D_X 0xf2000053 #define MASK_FMV_D_X 0xfff0707f +#define MATCH_FCVT_Q_W 0xd6000053 +#define MASK_FCVT_Q_W 0xfff0007f +#define MATCH_FCVT_Q_WU 0xd6100053 +#define MASK_FCVT_Q_WU 0xfff0007f +#define MATCH_FCVT_Q_L 0xd6200053 +#define MASK_FCVT_Q_L 0xfff0007f +#define MATCH_FCVT_Q_LU 0xd6300053 +#define MASK_FCVT_Q_LU 0xfff0007f +#define MATCH_FMV_Q_X 0xf6000053 +#define MASK_FMV_Q_X 0xfff0707f #define MATCH_FLW 0x2007 #define MASK_FLW 0x707f #define MATCH_FLD 0x3007 #define MASK_FLD 0x707f +#define MATCH_FLQ 0x4007 +#define MASK_FLQ 0x707f #define MATCH_FSW 0x2027 #define MASK_FSW 0x707f #define MATCH_FSD 0x3027 #define MASK_FSD 0x707f +#define MATCH_FSQ 0x4027 +#define MASK_FSQ 0x707f #define MATCH_FMADD_S 0x43 #define MASK_FMADD_S 0x600007f #define MATCH_FMSUB_S 0x47 @@ -537,6 +610,14 @@ #define MASK_FNMSUB_D 0x600007f #define MATCH_FNMADD_D 0x200004f #define MASK_FNMADD_D 0x600007f +#define MATCH_FMADD_Q 0x6000043 +#define MASK_FMADD_Q 0x600007f +#define MATCH_FMSUB_Q 0x6000047 +#define MASK_FMSUB_Q 0x600007f +#define MATCH_FNMSUB_Q 0x600004b +#define MASK_FNMSUB_Q 0x600007f +#define MATCH_FNMADD_Q 0x600004f +#define MASK_FNMADD_Q 0x600007f #define MATCH_C_NOP 0x1 #define MASK_C_NOP 0xffff #define MATCH_C_ADDI16SP 0x6101 @@ -967,7 +1048,7 @@ DECLARE_INSN(sret, MATCH_SRET, MASK_SRET) DECLARE_INSN(hret, MATCH_HRET, MASK_HRET) DECLARE_INSN(mret, MATCH_MRET, MASK_MRET) DECLARE_INSN(dret, MATCH_DRET, MASK_DRET) -DECLARE_INSN(sfence_vm, MATCH_SFENCE_VM, MASK_SFENCE_VM) +DECLARE_INSN(sfence_vma, MATCH_SFENCE_VMA, MASK_SFENCE_VMA) DECLARE_INSN(wfi, MATCH_WFI, MASK_WFI) DECLARE_INSN(csrrw, MATCH_CSRRW, MASK_CSRRW) DECLARE_INSN(csrrs, MATCH_CSRRS, MASK_CSRRS) @@ -997,12 +1078,29 @@ DECLARE_INSN(fmax_d, MATCH_FMAX_D, MASK_FMAX_D) DECLARE_INSN(fcvt_s_d, MATCH_FCVT_S_D, MASK_FCVT_S_D) DECLARE_INSN(fcvt_d_s, MATCH_FCVT_D_S, MASK_FCVT_D_S) DECLARE_INSN(fsqrt_d, MATCH_FSQRT_D, MASK_FSQRT_D) +DECLARE_INSN(fadd_q, MATCH_FADD_Q, MASK_FADD_Q) +DECLARE_INSN(fsub_q, MATCH_FSUB_Q, MASK_FSUB_Q) +DECLARE_INSN(fmul_q, MATCH_FMUL_Q, MASK_FMUL_Q) +DECLARE_INSN(fdiv_q, MATCH_FDIV_Q, MASK_FDIV_Q) +DECLARE_INSN(fsgnj_q, MATCH_FSGNJ_Q, MASK_FSGNJ_Q) +DECLARE_INSN(fsgnjn_q, MATCH_FSGNJN_Q, MASK_FSGNJN_Q) +DECLARE_INSN(fsgnjx_q, MATCH_FSGNJX_Q, MASK_FSGNJX_Q) +DECLARE_INSN(fmin_q, MATCH_FMIN_Q, MASK_FMIN_Q) +DECLARE_INSN(fmax_q, MATCH_FMAX_Q, MASK_FMAX_Q) +DECLARE_INSN(fcvt_s_q, MATCH_FCVT_S_Q, MASK_FCVT_S_Q) +DECLARE_INSN(fcvt_q_s, MATCH_FCVT_Q_S, MASK_FCVT_Q_S) +DECLARE_INSN(fcvt_d_q, MATCH_FCVT_D_Q, MASK_FCVT_D_Q) +DECLARE_INSN(fcvt_q_d, MATCH_FCVT_Q_D, MASK_FCVT_Q_D) +DECLARE_INSN(fsqrt_q, MATCH_FSQRT_Q, MASK_FSQRT_Q) DECLARE_INSN(fle_s, MATCH_FLE_S, MASK_FLE_S) DECLARE_INSN(flt_s, MATCH_FLT_S, MASK_FLT_S) DECLARE_INSN(feq_s, MATCH_FEQ_S, MASK_FEQ_S) DECLARE_INSN(fle_d, MATCH_FLE_D, MASK_FLE_D) DECLARE_INSN(flt_d, MATCH_FLT_D, MASK_FLT_D) DECLARE_INSN(feq_d, MATCH_FEQ_D, MASK_FEQ_D) +DECLARE_INSN(fle_q, MATCH_FLE_Q, MASK_FLE_Q) +DECLARE_INSN(flt_q, MATCH_FLT_Q, MASK_FLT_Q) +DECLARE_INSN(feq_q, MATCH_FEQ_Q, MASK_FEQ_Q) DECLARE_INSN(fcvt_w_s, MATCH_FCVT_W_S, MASK_FCVT_W_S) DECLARE_INSN(fcvt_wu_s, MATCH_FCVT_WU_S, MASK_FCVT_WU_S) DECLARE_INSN(fcvt_l_s, MATCH_FCVT_L_S, MASK_FCVT_L_S) @@ -1015,6 +1113,12 @@ DECLARE_INSN(fcvt_l_d, MATCH_FCVT_L_D, MASK_FCVT_L_D) DECLARE_INSN(fcvt_lu_d, MATCH_FCVT_LU_D, MASK_FCVT_LU_D) DECLARE_INSN(fmv_x_d, MATCH_FMV_X_D, MASK_FMV_X_D) DECLARE_INSN(fclass_d, MATCH_FCLASS_D, MASK_FCLASS_D) +DECLARE_INSN(fcvt_w_q, MATCH_FCVT_W_Q, MASK_FCVT_W_Q) +DECLARE_INSN(fcvt_wu_q, MATCH_FCVT_WU_Q, MASK_FCVT_WU_Q) +DECLARE_INSN(fcvt_l_q, MATCH_FCVT_L_Q, MASK_FCVT_L_Q) +DECLARE_INSN(fcvt_lu_q, MATCH_FCVT_LU_Q, MASK_FCVT_LU_Q) +DECLARE_INSN(fmv_x_q, MATCH_FMV_X_Q, MASK_FMV_X_Q) +DECLARE_INSN(fclass_q, MATCH_FCLASS_Q, MASK_FCLASS_Q) DECLARE_INSN(fcvt_s_w, MATCH_FCVT_S_W, MASK_FCVT_S_W) DECLARE_INSN(fcvt_s_wu, MATCH_FCVT_S_WU, MASK_FCVT_S_WU) DECLARE_INSN(fcvt_s_l, MATCH_FCVT_S_L, MASK_FCVT_S_L) @@ -1025,10 +1129,17 @@ DECLARE_INSN(fcvt_d_wu, MATCH_FCVT_D_WU, MASK_FCVT_D_WU) DECLARE_INSN(fcvt_d_l, MATCH_FCVT_D_L, MASK_FCVT_D_L) DECLARE_INSN(fcvt_d_lu, MATCH_FCVT_D_LU, MASK_FCVT_D_LU) DECLARE_INSN(fmv_d_x, MATCH_FMV_D_X, MASK_FMV_D_X) +DECLARE_INSN(fcvt_q_w, MATCH_FCVT_Q_W, MASK_FCVT_Q_W) +DECLARE_INSN(fcvt_q_wu, MATCH_FCVT_Q_WU, MASK_FCVT_Q_WU) +DECLARE_INSN(fcvt_q_l, MATCH_FCVT_Q_L, MASK_FCVT_Q_L) +DECLARE_INSN(fcvt_q_lu, MATCH_FCVT_Q_LU, MASK_FCVT_Q_LU) +DECLARE_INSN(fmv_q_x, MATCH_FMV_Q_X, MASK_FMV_Q_X) DECLARE_INSN(flw, MATCH_FLW, MASK_FLW) DECLARE_INSN(fld, MATCH_FLD, MASK_FLD) +DECLARE_INSN(flq, MATCH_FLQ, MASK_FLQ) DECLARE_INSN(fsw, MATCH_FSW, MASK_FSW) DECLARE_INSN(fsd, MATCH_FSD, MASK_FSD) +DECLARE_INSN(fsq, MATCH_FSQ, MASK_FSQ) DECLARE_INSN(fmadd_s, MATCH_FMADD_S, MASK_FMADD_S) DECLARE_INSN(fmsub_s, MATCH_FMSUB_S, MASK_FMSUB_S) DECLARE_INSN(fnmsub_s, MATCH_FNMSUB_S, MASK_FNMSUB_S) @@ -1037,6 +1148,10 @@ DECLARE_INSN(fmadd_d, MATCH_FMADD_D, MASK_FMADD_D) DECLARE_INSN(fmsub_d, MATCH_FMSUB_D, MASK_FMSUB_D) DECLARE_INSN(fnmsub_d, MATCH_FNMSUB_D, MASK_FNMSUB_D) DECLARE_INSN(fnmadd_d, MATCH_FNMADD_D, MASK_FNMADD_D) +DECLARE_INSN(fmadd_q, MATCH_FMADD_Q, MASK_FMADD_Q) +DECLARE_INSN(fmsub_q, MATCH_FMSUB_Q, MASK_FMSUB_Q) +DECLARE_INSN(fnmsub_q, MATCH_FNMSUB_Q, MASK_FNMSUB_Q) +DECLARE_INSN(fnmadd_q, MATCH_FNMADD_Q, MASK_FNMADD_Q) DECLARE_INSN(c_nop, MATCH_C_NOP, MASK_C_NOP) DECLARE_INSN(c_addi16sp, MATCH_C_ADDI16SP, MASK_C_ADDI16SP) DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR) diff --git a/machine/minit.c b/machine/minit.c index b3f2c86..06e8d25 100644 --- a/machine/minit.c +++ b/machine/minit.c @@ -15,15 +15,8 @@ size_t plic_ndevs; static void mstatus_init() { - // Enable FPU and set VM mode - uintptr_t ms = 0; - ms = INSERT_FIELD(ms, MSTATUS_VM, VM_CHOICE); - ms = INSERT_FIELD(ms, MSTATUS_FS, 1); - write_csr(mstatus, ms); - - // Make sure the hart actually supports the VM mode we want - ms = read_csr(mstatus); - assert(EXTRACT_FIELD(ms, MSTATUS_VM) == VM_CHOICE); + // Enable FPU + write_csr(mstatus, MSTATUS_FS); // Enable user/supervisor use of perf counters write_csr(mucounteren, -1); @@ -156,7 +149,7 @@ void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t stack) write_csr(mstatus, mstatus); write_csr(mscratch, MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE); write_csr(mepc, fn); - write_csr(sptbr, (uintptr_t)root_page_table >> RISCV_PGSHIFT); + write_csr(sptbr, ((uintptr_t)root_page_table >> RISCV_PGSHIFT) | SPTBR_MODE); asm volatile ("mv a0, %0; mv sp, %0; mret" : : "r" (stack)); __builtin_unreachable(); } diff --git a/machine/mtrap.c b/machine/mtrap.c index 07f7d10..34772b7 100644 --- a/machine/mtrap.c +++ b/machine/mtrap.c @@ -3,6 +3,7 @@ #include "htif.h" #include "atomic.h" #include "bits.h" +#include "vm.h" #include #include #include @@ -176,6 +177,7 @@ void software_interrupt() *HLS()->ipi = 0; mb(); int ipi_pending = atomic_swap(&HLS()->mipi_pending, 0); + mb(); if (ipi_pending & IPI_SOFT) { HLS()->sipi_pending = 1; @@ -186,7 +188,7 @@ void software_interrupt() asm volatile ("fence.i"); if (ipi_pending & IPI_SFENCE_VM) - asm volatile ("sfence.vm"); + flush_tlb(); } static void send_ipi_many(uintptr_t* pmask, int event) diff --git a/machine/vm.h b/machine/vm.h index 4e6bcc1..6d3da4e 100644 --- a/machine/vm.h +++ b/machine/vm.h @@ -6,11 +6,11 @@ #define MEGAPAGE_SIZE ((uintptr_t)(RISCV_PGSIZE << RISCV_PGLEVEL_BITS)) #if __riscv_xlen == 64 -# define VM_CHOICE VM_SV39 +# define SPTBR_MODE INSERT_FIELD(0, SPTBR64_MODE, SPTBR_MODE_SV39) # define VA_BITS 39 # define GIGAPAGE_SIZE (MEGAPAGE_SIZE << RISCV_PGLEVEL_BITS) #else -# define VM_CHOICE VM_SV32 +# define SPTBR_MODE INSERT_FIELD(0, SPTBR32_MODE, SPTBR_MODE_SV32) # define VA_BITS 32 #endif @@ -19,7 +19,7 @@ extern pte_t* root_page_table; static inline void flush_tlb() { - asm volatile("sfence.vm"); + asm volatile ("sfence.vma"); } static inline pte_t pte_create(uintptr_t ppn, int type) -- cgit v1.1