From 28d64a2227ab45b31c71de8366f4c7f965248ca8 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 12 Feb 2016 14:47:17 -0800 Subject: Add SBI calls for remote I$/TLB flushes --- pk/mcall.h | 2 ++ pk/mentry.S | 11 +++++---- pk/mtrap.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- pk/mtrap.h | 6 +++++ pk/sbi.S | 3 +++ pk/sbi.h | 4 ++++ pk/sbi_entry.S | 18 +++++++++++++++ 7 files changed, 110 insertions(+), 6 deletions(-) diff --git a/pk/mcall.h b/pk/mcall.h index 2c56023..c829f21 100644 --- a/pk/mcall.h +++ b/pk/mcall.h @@ -9,6 +9,8 @@ #define MCALL_CLEAR_IPI 5 #define MCALL_SHUTDOWN 6 #define MCALL_SET_TIMER 7 +#define MCALL_REMOTE_SFENCE_VM 8 +#define MCALL_REMOTE_FENCE_I 9 #ifndef __ASSEMBLER__ diff --git a/pk/mentry.S b/pk/mentry.S index ed1d53c..b8e9136 100644 --- a/pk/mentry.S +++ b/pk/mentry.S @@ -19,7 +19,9 @@ trap_table: .word bad_trap #define HTIF_INTERRUPT_VECTOR 12 .word htif_interrupt -#define TRAP_FROM_MACHINE_MODE_VECTOR 13 +#define SOFTWARE_INTERRUPT_VECTOR 13 + .word software_interrupt +#define TRAP_FROM_MACHINE_MODE_VECTOR 14 .word __trap_from_machine_mode .option norvc @@ -65,10 +67,9 @@ trap_vector: # Is it an IPI? li a0, IRQ_M_SOFT * 2 bne a0, a1, 1f - # Yes. Post a supervisor software interrupt. - csrc mip, MIP_MSIP - csrs mip, MIP_SSIP - j .Leret + li a1, SOFTWARE_INTERRUPT_VECTOR + j .Lhandle_trap_in_machine_mode + 1: # By process of elimination, it must be an HTIF interrupt. li a0, IRQ_HOST * 2 diff --git a/pk/mtrap.c b/pk/mtrap.c index cbb0082..ac00069 100644 --- a/pk/mtrap.c +++ b/pk/mtrap.c @@ -248,9 +248,73 @@ static uintptr_t mcall_set_timer(unsigned long long when) return 0; } +void software_interrupt() +{ + clear_csr(mip, MIP_MSIP); + __sync_synchronize(); + + if (HLS()->ipi_pending) + set_csr(mip, MIP_SSIP); + + if (HLS()->rpc_func) { + __sync_synchronize(); + rpc_func_t f = *HLS()->rpc_func; + (f.func)(f.arg); + __sync_synchronize(); + HLS()->rpc_func = (void*)1; + } +} + +static void call_func_all(uintptr_t* mask, void (*func)(uintptr_t), uintptr_t arg) +{ + kassert(MAX_HARTS <= 8*sizeof(*mask)); + kassert(supervisor_paddr_valid((uintptr_t)mask, sizeof(uintptr_t))); + + rpc_func_t f = {func, arg}; + + uintptr_t harts = *mask; + for (ssize_t i = num_harts-1; i >= 0; i--) { + if (((harts >> i) & 1) == 0) + continue; + + if (HLS()->hart_id == i) { + func(arg); + } else { + rpc_func_t** p = &OTHER_HLS(i)->rpc_func; + uintptr_t* mipi = &OTHER_HLS(i)->csrs[CSR_MIPI]; + + while (atomic_cas(p, 0, &f) != &f) + software_interrupt(); + + __sync_synchronize(); + *mipi = 1; + __sync_synchronize(); + + while (atomic_cas(p, (void*)1, 0) != (void*)1) + ; + } + } +} + +static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask, uintptr_t asid) +{ + void sfence_vm(uintptr_t arg) { + asm volatile ("csrrw %0, sasid, %0; sfence.vm; csrw sasid, %0" : "+r"(arg)); + } + call_func_all(hart_mask, &sfence_vm, asid); + return 0; +} + +static uintptr_t mcall_remote_fence_i(uintptr_t* hart_mask) +{ + void fence_i(uintptr_t arg) { asm volatile ("fence.i"); } + call_func_all(hart_mask, &fence_i, 0); + return 0; +} + void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) { - uintptr_t n = regs[17], arg0 = regs[10], retval; + uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval; switch (n) { case MCALL_HART_ID: @@ -277,6 +341,12 @@ void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) case MCALL_SET_TIMER: retval = mcall_set_timer(arg0); break; + case MCALL_REMOTE_SFENCE_VM: + retval = mcall_remote_sfence_vm((uintptr_t*)arg0, arg1); + break; + case MCALL_REMOTE_FENCE_I: + retval = mcall_remote_fence_i((uintptr_t*)arg0); + break; default: retval = -ENOSYS; break; diff --git a/pk/mtrap.h b/pk/mtrap.h index 5998bf8..64caaab 100644 --- a/pk/mtrap.h +++ b/pk/mtrap.h @@ -163,11 +163,17 @@ static inline int xlen() } typedef struct { + void (*func)(uintptr_t); + uintptr_t arg; +} rpc_func_t; + +typedef struct { sbi_device_message* device_request_queue_head; size_t device_request_queue_size; sbi_device_message* device_response_queue_head; sbi_device_message* device_response_queue_tail; + rpc_func_t* rpc_func; uintptr_t* csrs; int hart_id; int ipi_pending; diff --git a/pk/sbi.S b/pk/sbi.S index 59a282e..0c32d06 100644 --- a/pk/sbi.S +++ b/pk/sbi.S @@ -11,3 +11,6 @@ .globl sbi_set_timer; sbi_set_timer = -1888 .globl sbi_mask_interrupt; sbi_mask_interrupt = -1872 .globl sbi_unmask_interrupt; sbi_unmask_interrupt = -1856 +.globl sbi_remote_sfence_vm; sbi_remote_sfence_vm = -1840 +.globl sbi_remote_sfence_vm_range; sbi_remote_sfence_vm_range = -1824 +.globl sbi_remote_fence_i; sbi_remote_fence_i = -1808 diff --git a/pk/sbi.h b/pk/sbi.h index 0a2b880..cc112d9 100644 --- a/pk/sbi.h +++ b/pk/sbi.h @@ -18,6 +18,10 @@ unsigned long sbi_clear_ipi(void); void sbi_console_putchar(unsigned long ch); void sbi_shutdown(void); +void sbi_remote_sfence_vm(unsigned long hart_mask_ptr, unsigned long asid); +void sbi_remote_sfence_vm_range(unsigned long hart_mask_ptr, unsigned long asid, unsigned long start, unsigned long size); +void sbi_remote_fence_i(unsigned long hart_mask_ptr); + typedef struct { unsigned long dev; unsigned long cmd; diff --git a/pk/sbi_entry.S b/pk/sbi_entry.S index de33186..eef456b 100644 --- a/pk/sbi_entry.S +++ b/pk/sbi_entry.S @@ -80,6 +80,24 @@ sbi_base: .align 4 j __sbi_unmask_interrupt + # remote_sfence_vm + .align 4 + li a7, MCALL_REMOTE_SFENCE_VM + ecall + ret + + # remote_sfence_vm_range + .align 4 + li a7, MCALL_REMOTE_SFENCE_VM + ecall + ret + + # remote_fence_i + .align 4 + li a7, MCALL_REMOTE_FENCE_I + ecall + ret + # end of SBI trampolines .globl do_mcall -- cgit v1.1