Add SBI calls for remote I$/TLB flushes

7 files changed, 110 insertions, 6 deletions

diff --git a/pk/mcall.h b/pk/mcall.h
index 2c56023..c829f21 100644
--- a/pk/mcall.h
+++ b/pk/mcall.h
@@ -9,6 +9,8 @@
 #define MCALL_CLEAR_IPI 5
 #define MCALL_SHUTDOWN 6
 #define MCALL_SET_TIMER 7
+#define MCALL_REMOTE_SFENCE_VM 8
+#define MCALL_REMOTE_FENCE_I 9
 
 #ifndef __ASSEMBLER__
 
diff --git a/pk/mentry.S b/pk/mentry.S
index ed1d53c..b8e9136 100644
--- a/pk/mentry.S
+++ b/pk/mentry.S
@@ -19,7 +19,9 @@ trap_table:
   .word bad_trap
 #define HTIF_INTERRUPT_VECTOR 12
   .word htif_interrupt
-#define TRAP_FROM_MACHINE_MODE_VECTOR 13
+#define SOFTWARE_INTERRUPT_VECTOR 13
+  .word software_interrupt
+#define TRAP_FROM_MACHINE_MODE_VECTOR 14
   .word __trap_from_machine_mode
 
   .option norvc
@@ -65,10 +67,9 @@ trap_vector:
   # Is it an IPI?
   li a0, IRQ_M_SOFT * 2
   bne a0, a1, 1f
-  # Yes.  Post a supervisor software interrupt.
-  csrc mip, MIP_MSIP
-  csrs mip, MIP_SSIP
-  j .Leret
+  li a1, SOFTWARE_INTERRUPT_VECTOR
+  j .Lhandle_trap_in_machine_mode
+
 1:
   # By process of elimination, it must be an HTIF interrupt.
   li a0, IRQ_HOST * 2
diff --git a/pk/mtrap.c b/pk/mtrap.c
index cbb0082..ac00069 100644
--- a/pk/mtrap.c
+++ b/pk/mtrap.c
@@ -248,9 +248,73 @@ static uintptr_t mcall_set_timer(unsigned long long when)
   return 0;
 }
 
+void software_interrupt()
+{
+  clear_csr(mip, MIP_MSIP);
+  __sync_synchronize();
+
+  if (HLS()->ipi_pending)
+    set_csr(mip, MIP_SSIP);
+
+  if (HLS()->rpc_func) {
+    __sync_synchronize();
+    rpc_func_t f = *HLS()->rpc_func;
+    (f.func)(f.arg);
+    __sync_synchronize();
+    HLS()->rpc_func = (void*)1;
+  }
+}
+
+static void call_func_all(uintptr_t* mask, void (*func)(uintptr_t), uintptr_t arg)
+{
+  kassert(MAX_HARTS <= 8*sizeof(*mask));
+  kassert(supervisor_paddr_valid((uintptr_t)mask, sizeof(uintptr_t)));
+
+  rpc_func_t f = {func, arg};
+
+  uintptr_t harts = *mask;
+  for (ssize_t i = num_harts-1; i >= 0; i--) {
+    if (((harts >> i) & 1) == 0)
+      continue;
+
+    if (HLS()->hart_id == i) {
+      func(arg);
+    } else {
+      rpc_func_t** p = &OTHER_HLS(i)->rpc_func;
+      uintptr_t* mipi = &OTHER_HLS(i)->csrs[CSR_MIPI];
+
+      while (atomic_cas(p, 0, &f) != &f)
+        software_interrupt();
+
+      __sync_synchronize();
+      *mipi = 1;
+      __sync_synchronize();
+
+      while (atomic_cas(p, (void*)1, 0) != (void*)1)
+        ;
+    }
+  }
+}
+
+static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask, uintptr_t asid)
+{
+  void sfence_vm(uintptr_t arg) {
+    asm volatile ("csrrw %0, sasid, %0; sfence.vm; csrw sasid, %0" : "+r"(arg));
+  }
+  call_func_all(hart_mask, &sfence_vm, asid);
+  return 0;
+}
+
+static uintptr_t mcall_remote_fence_i(uintptr_t* hart_mask)
+{
+  void fence_i(uintptr_t arg) { asm volatile ("fence.i"); }
+  call_func_all(hart_mask, &fence_i, 0);
+  return 0;
+}
+
 void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
 {
-  uintptr_t n = regs[17], arg0 = regs[10], retval;
+  uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval;
   switch (n)
   {
     case MCALL_HART_ID:
@@ -277,6 +341,12 @@ void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc)
     case MCALL_SET_TIMER:
       retval = mcall_set_timer(arg0);
       break;
+    case MCALL_REMOTE_SFENCE_VM:
+      retval = mcall_remote_sfence_vm((uintptr_t*)arg0, arg1);
+      break;
+    case MCALL_REMOTE_FENCE_I:
+      retval = mcall_remote_fence_i((uintptr_t*)arg0);
+      break;
     default:
       retval = -ENOSYS;
       break;
diff --git a/pk/mtrap.h b/pk/mtrap.h
index 5998bf8..64caaab 100644
--- a/pk/mtrap.h
+++ b/pk/mtrap.h
@@ -163,11 +163,17 @@ static inline int xlen()
 }
 
 typedef struct {
+  void (*func)(uintptr_t);
+  uintptr_t arg;
+} rpc_func_t;
+
+typedef struct {
   sbi_device_message* device_request_queue_head;
   size_t device_request_queue_size;
   sbi_device_message* device_response_queue_head;
   sbi_device_message* device_response_queue_tail;
 
+  rpc_func_t* rpc_func;
   uintptr_t* csrs;
   int hart_id;
   int ipi_pending;
diff --git a/pk/sbi.S b/pk/sbi.S
index 59a282e..0c32d06 100644
--- a/pk/sbi.S
+++ b/pk/sbi.S
@@ -11,3 +11,6 @@
 .globl sbi_set_timer; sbi_set_timer = -1888
 .globl sbi_mask_interrupt; sbi_mask_interrupt = -1872
 .globl sbi_unmask_interrupt; sbi_unmask_interrupt = -1856
+.globl sbi_remote_sfence_vm; sbi_remote_sfence_vm = -1840
+.globl sbi_remote_sfence_vm_range; sbi_remote_sfence_vm_range = -1824
+.globl sbi_remote_fence_i; sbi_remote_fence_i = -1808
diff --git a/pk/sbi.h b/pk/sbi.h
index 0a2b880..cc112d9 100644
--- a/pk/sbi.h
+++ b/pk/sbi.h
@@ -18,6 +18,10 @@ unsigned long sbi_clear_ipi(void);
 void sbi_console_putchar(unsigned long ch);
 void sbi_shutdown(void);
 
+void sbi_remote_sfence_vm(unsigned long hart_mask_ptr, unsigned long asid);
+void sbi_remote_sfence_vm_range(unsigned long hart_mask_ptr, unsigned long asid, unsigned long start, unsigned long size);
+void sbi_remote_fence_i(unsigned long hart_mask_ptr);
+
 typedef struct {
   unsigned long dev;
   unsigned long cmd;
diff --git a/pk/sbi_entry.S b/pk/sbi_entry.S
index de33186..eef456b 100644
--- a/pk/sbi_entry.S
+++ b/pk/sbi_entry.S
@@ -80,6 +80,24 @@ sbi_base:
   .align 4
   j __sbi_unmask_interrupt
 
+  # remote_sfence_vm
+  .align 4
+  li a7, MCALL_REMOTE_SFENCE_VM
+  ecall
+  ret
+
+  # remote_sfence_vm_range
+  .align 4
+  li a7, MCALL_REMOTE_SFENCE_VM
+  ecall
+  ret
+
+  # remote_fence_i
+  .align 4
+  li a7, MCALL_REMOTE_FENCE_I
+  ecall
+  ret
+
   # end of SBI trampolines
 
   .globl do_mcall