From 7c4765029820580c0c01b3321fe4310c8416dcb6 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 19 Feb 2017 15:55:02 -0800 Subject: Handle IPIs and timer interrupts more quickly --- machine/mentry.S | 50 +++++++++++++++++++++-------- machine/mtrap.c | 75 +++++++++++++------------------------------ machine/mtrap.h | 18 ++++++----- machine/unprivileged_memory.h | 2 ++ 4 files changed, 71 insertions(+), 74 deletions(-) (limited to 'machine') diff --git a/machine/mentry.S b/machine/mentry.S index 95653c1..6e77d5e 100644 --- a/machine/mentry.S +++ b/machine/mentry.S @@ -18,11 +18,7 @@ trap_table: .word mcall_trap .word bad_trap .word bad_trap -#define SOFTWARE_INTERRUPT_VECTOR 12 - .word software_interrupt -#define TIMER_INTERRUPT_VECTOR 13 - .word timer_interrupt -#define TRAP_FROM_MACHINE_MODE_VECTOR 14 +#define TRAP_FROM_MACHINE_MODE_VECTOR 12 .word __trap_from_machine_mode .option norvc @@ -31,10 +27,6 @@ trap_table: reset_vector: j do_reset -nmi_vector: -.Lunhandleable_trap: - j bad_trap - trap_vector: csrrw sp, mscratch, sp beqz sp, .Ltrap_from_machine_mode @@ -51,8 +43,12 @@ trap_vector: # Is it a machine timer interrupt? li a0, IRQ_M_TIMER * 2 bne a0, a1, 1f - li a1, TIMER_INTERRUPT_VECTOR - j .Lhandle_trap_in_machine_mode + + # Yes. Simply clear MSIE and raise SSIP. + li a0, MIP_MTIP + csrc mie, a0 + li a0, MIP_STIP + csrs mip, a0 .Lmret: # Go back whence we came. @@ -64,9 +60,35 @@ trap_vector: 1: # Is it an IPI? li a0, IRQ_M_SOFT * 2 - bne a0, a1, .Lunhandleable_trap - li a1, SOFTWARE_INTERRUPT_VECTOR - j .Lhandle_trap_in_machine_mode + bne a0, a1, bad_trap + + # Yes. First, clear the MIPI bit. + LOAD a0, MENTRY_IPI_OFFSET(sp) + STORE x0, (a0) + fence + + # Now, decode the cause(s). +#ifdef __riscv_atomic + addi a0, sp, MENTRY_IPI_PENDING_OFFSET + amoswap.w a0, x0, (a0) +#else + lw a0, MENTRY_IPI_PENDING_OFFSET(a0) + sw x0, MENTRY_IPI_PENDING_OFFSET(a0) +#endif + and a1, a0, IPI_SOFT + beqz a1, 1f + csrs mip, MIP_SSIP +1: + andi a1, a0, IPI_FENCE_I + beqz a1, 1f + fence.i +1: + andi a1, a0, IPI_SFENCE_VM + beqz a1, 1f + sfence.vma +1: + j .Lmret + .Lhandle_trap_in_machine_mode: # Preserve the registers. Compute the address of the trap handler. diff --git a/machine/mtrap.c b/machine/mtrap.c index 3cbc064..fb6c3f9 100644 --- a/machine/mtrap.c +++ b/machine/mtrap.c @@ -4,6 +4,7 @@ #include "atomic.h" #include "bits.h" #include "vm.h" +#include "unprivileged_memory.h" #include #include #include @@ -13,15 +14,6 @@ void __attribute__((noreturn)) bad_trap() die("machine mode: unhandlable trap %d @ %p", read_csr(mcause), read_csr(mepc)); } -uintptr_t timer_interrupt() -{ - // just send the timer interrupt to the supervisor - clear_csr(mie, MIP_MTIP); - set_csr(mip, MIP_STIP); - - return 0; -} - static uintptr_t mcall_console_putchar(uint8_t ch) { htif_console_putchar(ch); @@ -59,15 +51,6 @@ static void send_ipi(uintptr_t recipient, int event) } } -static uintptr_t mcall_send_ipi(uintptr_t recipient) -{ - if (recipient >= num_harts) - return -1; - - send_ipi(recipient, IPI_SOFT); - return 0; -} - static uintptr_t mcall_console_getchar() { return htif_console_getchar(); @@ -75,9 +58,7 @@ static uintptr_t mcall_console_getchar() static uintptr_t mcall_clear_ipi() { - clear_csr(mip, MIP_SSIP); - mb(); - return atomic_swap(&HLS()->sipi_pending, 0); + return clear_csr(mip, MIP_SSIP) & MIP_SSIP; } static uintptr_t mcall_shutdown() @@ -93,31 +74,12 @@ static uintptr_t mcall_set_timer(uint64_t when) return 0; } -void software_interrupt() -{ - *HLS()->ipi = 0; - mb(); - int ipi_pending = atomic_swap(&HLS()->mipi_pending, 0); - mb(); - - if (ipi_pending & IPI_SOFT) { - HLS()->sipi_pending = 1; - set_csr(mip, MIP_SSIP); - } - - if (ipi_pending & IPI_FENCE_I) - asm volatile ("fence.i"); - - if (ipi_pending & IPI_SFENCE_VM) - flush_tlb(); -} - static void send_ipi_many(uintptr_t* pmask, int event) { _Static_assert(MAX_HARTS <= 8 * sizeof(*pmask), "# harts > uintptr_t bits"); uintptr_t mask = -1; if (pmask) - mask = *pmask; + mask = load_uintptr_t(pmask, read_csr(mepc)); // send IPIs to everyone for (ssize_t i = num_harts-1; i >= 0; i--) @@ -125,14 +87,21 @@ static void send_ipi_many(uintptr_t* pmask, int event) send_ipi(i, event); // wait until all events have been handled. - // prevent deadlock while spinning by handling any IPIs from other harts. + // prevent deadlock by consuming incoming IPIs. + uint32_t incoming_ipi = 0; for (ssize_t i = num_harts-1; i >= 0; i--) if ((mask >> i) & 1) - while (OTHER_HLS(i)->mipi_pending & event) - software_interrupt(); + while (OTHER_HLS(i)->ipi) + incoming_ipi |= atomic_swap(HLS()->ipi, 0); + + // if we got an IPI, restore it; it will be taken after returning + if (incoming_ipi) { + *HLS()->ipi = incoming_ipi; + mb(); + } } -static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask, uintptr_t asid) +static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask) { // ignore the ASID and do a global flush. // this allows us to avoid queueing a message. @@ -149,6 +118,8 @@ static uintptr_t mcall_remote_fence_i(uintptr_t* hart_mask) void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) { uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval; + uintptr_t ipi_type = 0; + switch (n) { case MCALL_CONSOLE_PUTCHAR: @@ -158,7 +129,13 @@ void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) retval = mcall_console_getchar(); break; case MCALL_SEND_IPI: - retval = mcall_send_ipi(arg0); + ipi_type = IPI_SOFT; + case MCALL_REMOTE_SFENCE_VM: + ipi_type = IPI_SFENCE_VM; + case MCALL_REMOTE_FENCE_I: + ipi_type = IPI_FENCE_I; + send_ipi_many((uintptr_t*)arg0, ipi_type); + retval = 0; break; case MCALL_CLEAR_IPI: retval = mcall_clear_ipi(); @@ -173,12 +150,6 @@ void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) retval = mcall_set_timer(arg0); #endif break; - case MCALL_REMOTE_SFENCE_VM: - retval = mcall_remote_sfence_vm((uintptr_t*)arg0, arg1); - break; - case MCALL_REMOTE_FENCE_I: - retval = mcall_remote_fence_i((uintptr_t*)arg0); - break; default: retval = -ENOSYS; break; diff --git a/machine/mtrap.h b/machine/mtrap.h index 1f692b3..6f38cf6 100644 --- a/machine/mtrap.h +++ b/machine/mtrap.h @@ -36,10 +36,10 @@ extern volatile uint32_t* plic_priorities; extern size_t plic_ndevs; typedef struct { - uint64_t* timecmp; uint32_t* ipi; volatile int mipi_pending; - volatile int sipi_pending; + + uint64_t* timecmp; volatile uint32_t* plic_m_thresh; volatile uintptr_t* plic_m_ie; @@ -47,10 +47,6 @@ typedef struct { volatile uintptr_t* plic_s_ie; } hls_t; -#define IPI_SOFT 0x1 -#define IPI_FENCE_I 0x2 -#define IPI_SFENCE_VM 0x4 - #define MACHINE_STACK_TOP() ({ \ register uintptr_t sp asm ("sp"); \ (void*)((sp + RISCV_PGSIZE) & -RISCV_PGSIZE); }) @@ -79,9 +75,15 @@ static inline void wfi() #endif // !__ASSEMBLER__ +#define IPI_SOFT 0x1 +#define IPI_FENCE_I 0x2 +#define IPI_SFENCE_VM 0x4 + #define MACHINE_STACK_SIZE RISCV_PGSIZE -#define MENTRY_FRAME_SIZE (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE \ - + HLS_SIZE) +#define MENTRY_HLS_OFFSET (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE) +#define MENTRY_FRAME_SIZE (MENTRY_HLS_OFFSET + HLS_SIZE) +#define MENTRY_IPI_OFFSET (MENTRY_HLS_OFFSET) +#define MENTRY_IPI_PENDING_OFFSET (MENTRY_HLS_OFFSET + REGBYTES) #ifdef __riscv_flen # define SOFT_FLOAT_CONTEXT_SIZE 0 diff --git a/machine/unprivileged_memory.h b/machine/unprivileged_memory.h index 39a7722..6b84861 100644 --- a/machine/unprivileged_memory.h +++ b/machine/unprivileged_memory.h @@ -43,8 +43,10 @@ DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint32_t, sw) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lwu) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint64_t, ld) DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint64_t, sd) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uintptr_t, ld) #else DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lw) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uintptr_t, lw) static inline uint64_t load_uint64_t(const uint64_t* addr, uintptr_t mepc) { -- cgit v1.1