From ccf79891f0f1fcb570f9a63c15270b52477fe7c0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 9 Mar 2016 23:54:14 -0800 Subject: Factor emulation routines into multiple files --- machine/emulation.c | 154 ++++----------------------------------------- machine/fp_emulation.c | 64 ------------------- machine/fp_emulation.h | 3 +- machine/fp_ldst.c | 67 ++++++++++++++++++++ machine/machine.mk.in | 3 + machine/misaligned_ldst.c | 85 +++++++++++++++++++++++++ machine/muldiv_emulation.c | 64 +++++++++++++++++++ 7 files changed, 233 insertions(+), 207 deletions(-) create mode 100644 machine/fp_ldst.c create mode 100644 machine/misaligned_ldst.c create mode 100644 machine/muldiv_emulation.c diff --git a/machine/emulation.c b/machine/emulation.c index 5a66997..5e3ca2e 100644 --- a/machine/emulation.c +++ b/machine/emulation.c @@ -10,7 +10,7 @@ void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) asm (".pushsection .rodata\n" "illegal_insn_trap_table:\n" " .word truly_illegal_insn\n" -#ifdef PK_ENABLE_FP_EMULATION +#if !defined(__riscv_hard_float) && defined(PK_ENABLE_FP_EMULATION) " .word emulate_float_load\n" #else " .word truly_illegal_insn\n" @@ -22,16 +22,24 @@ void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) " .word truly_illegal_insn\n" " .word truly_illegal_insn\n" " .word truly_illegal_insn\n" -#ifdef PK_ENABLE_FP_EMULATION +#if !defined(__riscv_hard_float) && defined(PK_ENABLE_FP_EMULATION) " .word emulate_float_store\n" #else " .word truly_illegal_insn\n" #endif " .word truly_illegal_insn\n" " .word truly_illegal_insn\n" +#if !defined(__riscv_muldiv) " .word emulate_mul_div\n" +#else " .word truly_illegal_insn\n" +#endif + " .word truly_illegal_insn\n" +#if !defined(__riscv_muldiv) && defined(__riscv64) " .word emulate_mul_div32\n" +#else + " .word truly_illegal_insn\n" +#endif " .word truly_illegal_insn\n" #ifdef PK_ENABLE_FP_EMULATION " .word emulate_fmadd\n" @@ -78,144 +86,6 @@ void __attribute__((noinline)) truly_illegal_insn(uintptr_t* regs, uintptr_t mca redirect_trap(mepc, mstatus); } -void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - union { - uint8_t bytes[8]; - uintptr_t intx; - uint64_t int64; - } val; - uintptr_t mstatus; - insn_t insn = get_insn(mepc, &mstatus); - uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); - - int shift = 0, fp = 0, len; - if ((insn & MASK_LW) == MATCH_LW) - len = 4, shift = 8*(sizeof(uintptr_t) - len); -#ifdef __riscv64 - else if ((insn & MASK_LD) == MATCH_LD) - len = 8, shift = 8*(sizeof(uintptr_t) - len); - else if ((insn & MASK_LWU) == MATCH_LWU) - fp = 0, len = 4, shift = 0; -#endif - else if ((insn & MASK_FLD) == MATCH_FLD) - fp = 1, len = 8; - else if ((insn & MASK_FLW) == MATCH_FLW) - fp = 1, len = 4; - else if ((insn & MASK_LH) == MATCH_LH) - len = 2, shift = 8*(sizeof(uintptr_t) - len); - else if ((insn & MASK_LHU) == MATCH_LHU) - len = 2; - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - val.int64 = 0; - for (intptr_t i = len-1; i >= 0; i--) - val.bytes[i] = load_uint8_t((void *)(addr + i), mepc); - - if (!fp) - SET_RD(insn, regs, (intptr_t)val.intx << shift >> shift); - else if (len == 8) - SET_F64_RD(insn, regs, val.int64); - else - SET_F32_RD(insn, regs, val.intx); - - write_csr(mepc, mepc + 4); -} - -void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - union { - uint8_t bytes[8]; - uintptr_t intx; - uint64_t int64; - } val; - uintptr_t mstatus; - insn_t insn = get_insn(mepc, &mstatus); - int len; - - val.intx = GET_RS2(insn, regs); - if ((insn & MASK_SW) == MATCH_SW) - len = 4; -#ifdef __riscv64 - else if ((insn & MASK_SD) == MATCH_SD) - len = 8; -#endif - else if ((insn & MASK_FSD) == MATCH_FSD) - len = 8, val.int64 = GET_F64_RS2(insn, regs); - else if ((insn & MASK_FSW) == MATCH_FSW) - len = 4, val.intx = GET_F32_RS2(insn, regs); - else if ((insn & MASK_SH) == MATCH_SH) - len = 2; - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); - for (int i = 0; i < len; i++) - store_uint8_t((void *)(addr + i), val.bytes[i], mepc); - - write_csr(mepc, mepc + 4); -} - -#ifdef __riscv64 -typedef __int128 double_int; -#else -typedef int64_t double_int; -#endif - -DECLARE_EMULATION_FUNC(emulate_mul_div) -{ - uintptr_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs), val; - -#ifndef __riscv_muldiv - // If compiled with -mno-multiply, GCC will expand these out - if ((insn & MASK_MUL) == MATCH_MUL) - val = rs1 * rs2; - else if ((insn & MASK_DIV) == MATCH_DIV) - val = (intptr_t)rs1 / (intptr_t)rs2; - else if ((insn & MASK_DIVU) == MATCH_DIVU) - val = rs1 / rs2; - else if ((insn & MASK_REM) == MATCH_REM) - val = (intptr_t)rs1 % (intptr_t)rs2; - else if ((insn & MASK_REMU) == MATCH_REMU) - val = rs1 % rs2; - else if ((insn & MASK_MULH) == MATCH_MULH) - val = ((double_int)(intptr_t)rs1 * (double_int)(intptr_t)rs2) >> (8 * sizeof(rs1)); - else if ((insn & MASK_MULHU) == MATCH_MULHU) - val = ((double_int)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); - else if ((insn & MASK_MULHSU) == MATCH_MULHSU) - val = ((double_int)(intptr_t)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); - else -#endif - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - SET_RD(insn, regs, val); -} - -DECLARE_EMULATION_FUNC(emulate_mul_div32) -{ - uint32_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs); - int32_t val; - -#if defined(__riscv64) && !defined(__riscv_muldiv) - // If compiled with -mno-multiply, GCC will expand these out - if ((insn & MASK_MULW) == MATCH_MULW) - val = rs1 * rs2; - else if ((insn & MASK_DIVW) == MATCH_DIVW) - val = (int32_t)rs1 / (int32_t)rs2; - else if ((insn & MASK_DIVUW) == MATCH_DIVUW) - val = rs1 / rs2; - else if ((insn & MASK_REMW) == MATCH_REMW) - val = (int32_t)rs1 % (int32_t)rs2; - else if ((insn & MASK_REMUW) == MATCH_REMUW) - val = rs1 % rs2; - else -#endif - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - SET_RD(insn, regs, val); -} - static inline int emulate_read_csr(int num, uintptr_t mstatus, uintptr_t* result) { switch (num) @@ -252,7 +122,7 @@ static inline int emulate_read_csr(int num, uintptr_t mstatus, uintptr_t* result + HLS()->sinstret_delta) >> 32; return 0; #endif -#ifdef PK_ENABLE_FP_EMULATION +#if !defined(__riscv_hard_float) && defined(PK_ENABLE_FP_EMULATION) case CSR_FRM: if ((mstatus & MSTATUS_FS) == 0) break; *result = GET_FRM(); @@ -274,7 +144,7 @@ static inline int emulate_write_csr(int num, uintptr_t value, uintptr_t mstatus) { switch (num) { -#ifdef PK_ENABLE_FP_EMULATION +#if !defined(__riscv_hard_float) && defined(PK_ENABLE_FP_EMULATION) case CSR_FRM: SET_FRM(value); return 0; case CSR_FFLAGS: SET_FFLAGS(value); return 0; case CSR_FCSR: SET_FCSR(value); return 0; diff --git a/machine/fp_emulation.c b/machine/fp_emulation.c index 536967f..7523a29 100644 --- a/machine/fp_emulation.c +++ b/machine/fp_emulation.c @@ -3,70 +3,6 @@ #include "softfloat.h" #include "config.h" -DECLARE_EMULATION_FUNC(emulate_float_load) -{ - uint64_t val; - uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); - - switch (insn & MASK_FUNCT3) - { - case MATCH_FLW & MASK_FUNCT3: - if (addr % 4 != 0) - return misaligned_load_trap(regs, mcause, mepc); - - SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc)); - break; - - case MATCH_FLD & MASK_FUNCT3: - if (addr % sizeof(uintptr_t) != 0) - return misaligned_load_trap(regs, mcause, mepc); - -#ifdef __riscv64 - val = load_uint64_t((void *)addr, mepc); -#else - val = load_uint32_t(addr, mepc); - val += (uint64_t)load_uint32_t((void *)(addr + 4), mepc) << 32; -#endif - SET_F64_RD(insn, regs, val); - break; - - default: - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_float_store) -{ - uint64_t val; - uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); - - switch (insn & MASK_FUNCT3) - { - case MATCH_FSW & MASK_FUNCT3: - if (addr % 4 != 0) - return misaligned_store_trap(regs, mcause, mepc); - - store_uint32_t((void *)addr, GET_F32_RS2(insn, regs), mepc); - break; - - case MATCH_FSD & MASK_FUNCT3: - if (addr % sizeof(uintptr_t) != 0) - return misaligned_store_trap(regs, mcause, mepc); - - val = GET_F64_RS2(insn, regs); -#ifdef __riscv64 - store_uint64_t((void *)addr, val, mepc); -#else - store_uint32_t((void *)addr, val, mepc); - store_uint32_t((void *)(addr + 4), val >> 32, mepc); -#endif - break; - - default: - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - DECLARE_EMULATION_FUNC(emulate_fp) { asm (".pushsection .rodata\n" diff --git a/machine/fp_emulation.h b/machine/fp_emulation.h index d2357b7..8d209e1 100644 --- a/machine/fp_emulation.h +++ b/machine/fp_emulation.h @@ -46,6 +46,7 @@ asm volatile ("":"+r"(tp)); }) # define softfloat_raiseFlags(which) set_csr(fflags, which) # define softfloat_roundingMode ({ register int tp asm("tp"); tp; }) +# define SET_FS_DIRTY() ((void) 0) #else # define GET_F64_REG(insn, pos, regs) (*(int64_t*)((void*)((regs) + 32) + (((insn) >> ((pos)-3)) & 0xf8))) # define SET_F64_REG(insn, pos, regs, val) (GET_F64_REG(insn, pos, regs) = (val)) @@ -66,6 +67,7 @@ asm volatile ("":"+r"(tp)); }) # define softfloat_raiseFlags(which) ({ asm volatile ("or tp, tp, %0" :: "rI"(which)); }) # define softfloat_roundingMode ({ register int tp asm("tp"); tp >> 13; }) +# define SET_FS_DIRTY() set_csr(mstatus, MSTATUS_FS) #endif #define GET_F32_RS1(insn, regs) (GET_F32_REG(insn, 15, regs)) @@ -76,6 +78,5 @@ #define GET_F64_RS3(insn, regs) (GET_F64_REG(insn, 27, regs)) #define SET_F32_RD(insn, regs, val) (SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY()) #define SET_F64_RD(insn, regs, val) (SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY()) -#define SET_FS_DIRTY() set_csr(mstatus, MSTATUS_FS) #endif diff --git a/machine/fp_ldst.c b/machine/fp_ldst.c new file mode 100644 index 0000000..9d0a49f --- /dev/null +++ b/machine/fp_ldst.c @@ -0,0 +1,67 @@ +#include "fp_emulation.h" +#include "unprivileged_memory.h" + +DECLARE_EMULATION_FUNC(emulate_float_load) +{ + uint64_t val; + uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); + + switch (insn & MASK_FUNCT3) + { + case MATCH_FLW & MASK_FUNCT3: + if (addr % 4 != 0) + return misaligned_load_trap(regs, mcause, mepc); + + SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc)); + break; + + case MATCH_FLD & MASK_FUNCT3: + if (addr % sizeof(uintptr_t) != 0) + return misaligned_load_trap(regs, mcause, mepc); + +#ifdef __riscv64 + val = load_uint64_t((void *)addr, mepc); +#else + val = load_uint32_t(addr, mepc); + val += (uint64_t)load_uint32_t((void *)(addr + 4), mepc) << 32; +#endif + SET_F64_RD(insn, regs, val); + break; + + default: + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_float_store) +{ + uint64_t val; + uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); + + switch (insn & MASK_FUNCT3) + { + case MATCH_FSW & MASK_FUNCT3: + if (addr % 4 != 0) + return misaligned_store_trap(regs, mcause, mepc); + + store_uint32_t((void *)addr, GET_F32_RS2(insn, regs), mepc); + break; + + case MATCH_FSD & MASK_FUNCT3: + if (addr % sizeof(uintptr_t) != 0) + return misaligned_store_trap(regs, mcause, mepc); + + val = GET_F64_RS2(insn, regs); +#ifdef __riscv64 + store_uint64_t((void *)addr, val, mepc); +#else + store_uint32_t((void *)addr, val, mepc); + store_uint32_t((void *)(addr + 4), val >> 32, mepc); +#endif + break; + + default: + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + diff --git a/machine/machine.mk.in b/machine/machine.mk.in index a0b7ca9..4628954 100644 --- a/machine/machine.mk.in +++ b/machine/machine.mk.in @@ -18,7 +18,10 @@ machine_c_srcs = \ mtrap.c \ minit.c \ emulation.c \ + muldiv_emulation.c \ fp_emulation.c \ + fp_ldst.c \ + misaligned_ldst.c \ sbi_impl.c \ configstring.c \ diff --git a/machine/misaligned_ldst.c b/machine/misaligned_ldst.c new file mode 100644 index 0000000..cf6b1d6 --- /dev/null +++ b/machine/misaligned_ldst.c @@ -0,0 +1,85 @@ +#include "emulation.h" +#include "fp_emulation.h" +#include "unprivileged_memory.h" +#include "mtrap.h" + +union byte_array { + uint8_t bytes[8]; + uintptr_t intx; + uint64_t int64; +}; + +void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + union byte_array val; + uintptr_t mstatus; + insn_t insn = get_insn(mepc, &mstatus); + uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); + + int shift = 0, fp = 0, len; + if ((insn & MASK_LW) == MATCH_LW) + len = 4, shift = 8*(sizeof(uintptr_t) - len); +#ifdef __riscv64 + else if ((insn & MASK_LD) == MATCH_LD) + len = 8, shift = 8*(sizeof(uintptr_t) - len); + else if ((insn & MASK_LWU) == MATCH_LWU) + len = 4; +#endif +#ifdef PK_ENABLE_FP_EMULATION + else if ((insn & MASK_FLD) == MATCH_FLD) + fp = 1, len = 8; + else if ((insn & MASK_FLW) == MATCH_FLW) + fp = 1, len = 4; +#endif + else if ((insn & MASK_LH) == MATCH_LH) + len = 2, shift = 8*(sizeof(uintptr_t) - len); + else if ((insn & MASK_LHU) == MATCH_LHU) + len = 2; + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + val.int64 = 0; + for (intptr_t i = len-1; i >= 0; i--) + val.bytes[i] = load_uint8_t((void *)(addr + i), mepc); + + if (!fp) + SET_RD(insn, regs, (intptr_t)val.intx << shift >> shift); + else if (len == 8) + SET_F64_RD(insn, regs, val.int64); + else + SET_F32_RD(insn, regs, val.intx); + + write_csr(mepc, mepc + 4); +} + +void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + union byte_array val; + uintptr_t mstatus; + insn_t insn = get_insn(mepc, &mstatus); + int len; + + val.intx = GET_RS2(insn, regs); + if ((insn & MASK_SW) == MATCH_SW) + len = 4; +#ifdef __riscv64 + else if ((insn & MASK_SD) == MATCH_SD) + len = 8; +#endif +#ifdef PK_ENABLE_FP_EMULATION + else if ((insn & MASK_FSD) == MATCH_FSD) + len = 8, val.int64 = GET_F64_RS2(insn, regs); + else if ((insn & MASK_FSW) == MATCH_FSW) + len = 4, val.intx = GET_F32_RS2(insn, regs); +#endif + else if ((insn & MASK_SH) == MATCH_SH) + len = 2; + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); + for (int i = 0; i < len; i++) + store_uint8_t((void *)(addr + i), val.bytes[i], mepc); + + write_csr(mepc, mepc + 4); +} diff --git a/machine/muldiv_emulation.c b/machine/muldiv_emulation.c new file mode 100644 index 0000000..d0583c5 --- /dev/null +++ b/machine/muldiv_emulation.c @@ -0,0 +1,64 @@ +#include "emulation.h" + +#ifndef __riscv_muldiv + +#ifdef __riscv64 +typedef __int128 double_int; +#else +typedef int64_t double_int; +#endif + +// These routines rely on the compiler to turn these operations into libcalls +// when not natively supported. So work on making those go fast. + +DECLARE_EMULATION_FUNC(emulate_mul_div) +{ + uintptr_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs), val; + + if ((insn & MASK_MUL) == MATCH_MUL) + val = rs1 * rs2; + else if ((insn & MASK_DIV) == MATCH_DIV) + val = (intptr_t)rs1 / (intptr_t)rs2; + else if ((insn & MASK_DIVU) == MATCH_DIVU) + val = rs1 / rs2; + else if ((insn & MASK_REM) == MATCH_REM) + val = (intptr_t)rs1 % (intptr_t)rs2; + else if ((insn & MASK_REMU) == MATCH_REMU) + val = rs1 % rs2; + else if ((insn & MASK_MULH) == MATCH_MULH) + val = ((double_int)(intptr_t)rs1 * (double_int)(intptr_t)rs2) >> (8 * sizeof(rs1)); + else if ((insn & MASK_MULHU) == MATCH_MULHU) + val = ((double_int)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); + else if ((insn & MASK_MULHSU) == MATCH_MULHSU) + val = ((double_int)(intptr_t)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + SET_RD(insn, regs, val); +} + +#ifdef __riscv64 + +DECLARE_EMULATION_FUNC(emulate_mul_div32) +{ + uint32_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs); + int32_t val; + + if ((insn & MASK_MULW) == MATCH_MULW) + val = rs1 * rs2; + else if ((insn & MASK_DIVW) == MATCH_DIVW) + val = (int32_t)rs1 / (int32_t)rs2; + else if ((insn & MASK_DIVUW) == MATCH_DIVUW) + val = rs1 / rs2; + else if ((insn & MASK_REMW) == MATCH_REMW) + val = (int32_t)rs1 % (int32_t)rs2; + else if ((insn & MASK_REMUW) == MATCH_REMUW) + val = rs1 % rs2; + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + SET_RD(insn, regs, val); +} + +#endif +#endif -- cgit v1.1