From b94c7a4b07f96f24ae7411780abf874416549f7b Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 9 Mar 2016 23:58:17 -0800 Subject: Refactor pk, bbl, machine into separate libraries Yuck. --- Makefile.in | 7 +- bbl/bbl.ac | 8 + bbl/bbl.c | 76 +++ bbl/bbl.h | 23 + bbl/bbl.lds | 100 ++++ bbl/bbl.mk.in | 22 + bbl/kernel_elf.c | 54 ++ bbl/logo.c | 32 ++ bbl/payload.S | 7 + config.h.in | 12 + configure | 223 +++++++- configure.ac | 2 +- dummy_payload/dummy_payload.ac | 0 dummy_payload/dummy_payload.c | 21 + dummy_payload/dummy_payload.lds | 3 + dummy_payload/dummy_payload.mk.in | 13 + dummy_payload/dummy_sbi.S | 1 + machine/atomic.h | 72 +++ machine/bits.h | 33 ++ machine/configstring.c | 200 +++++++ machine/emulation.c | 313 +++++++++++ machine/emulation.h | 28 + machine/encoding.h | 1037 +++++++++++++++++++++++++++++++++++++ machine/fp_asm.S | 160 ++++++ machine/fp_emulation.c | 449 ++++++++++++++++ machine/fp_emulation.h | 81 +++ machine/htif.h | 18 + machine/machine.ac | 4 + machine/machine.mk.in | 29 ++ machine/mcall.h | 21 + machine/mentry.S | 245 +++++++++ machine/minit.c | 120 +++++ machine/mtrap.c | 287 ++++++++++ machine/mtrap.h | 96 ++++ machine/sbi.S | 15 + machine/sbi.h | 30 ++ machine/sbi_entry.S | 111 ++++ machine/sbi_impl.c | 33 ++ machine/unprivileged_memory.h | 79 +++ machine/vm.h | 35 ++ pk/atomic.h | 72 --- pk/bbl.c | 36 -- pk/bits.h | 45 -- pk/boot.h | 10 - pk/configstring.c | 200 ------- pk/elf.c | 137 ++--- pk/elf.h | 8 + pk/emulation.c | 313 ----------- pk/emulation.h | 28 - pk/encoding.h | 1037 ------------------------------------- pk/file.c | 10 +- pk/file.h | 1 - pk/fp_asm.S | 160 ------ pk/fp_emulation.c | 449 ---------------- pk/fp_emulation.h | 81 --- pk/frontend.h | 12 - pk/handlers.c | 8 +- pk/init.c | 82 --- pk/logo.c | 32 -- pk/mcall.h | 21 - pk/mentry.S | 244 --------- pk/minit.c | 108 ---- pk/mmap.c | 426 +++++++++++++++ pk/mmap.h | 34 ++ pk/mtrap.c | 281 ---------- pk/mtrap.h | 86 --- pk/pk.ac | 10 - pk/pk.c | 95 +++- pk/pk.h | 17 - pk/pk.ld | 95 ---- pk/pk.lds | 90 ++++ pk/pk.mk.in | 30 +- pk/sbi.S | 15 - pk/sbi.h | 30 -- pk/sbi_entry.S | 109 ---- pk/sbi_impl.c | 34 -- pk/snprintf.c | 98 ---- pk/string.c | 87 ---- pk/syscall.c | 2 +- pk/unprivileged_memory.h | 79 --- pk/vm.c | 500 ------------------ pk/vm.h | 53 -- util/snprintf.c | 98 ++++ util/string.c | 87 ++++ util/util.ac | 0 util/util.mk.in | 9 + 86 files changed, 4980 insertions(+), 4579 deletions(-) create mode 100644 bbl/bbl.ac create mode 100644 bbl/bbl.c create mode 100644 bbl/bbl.h create mode 100644 bbl/bbl.lds create mode 100644 bbl/bbl.mk.in create mode 100644 bbl/kernel_elf.c create mode 100644 bbl/logo.c create mode 100644 bbl/payload.S create mode 100644 dummy_payload/dummy_payload.ac create mode 100644 dummy_payload/dummy_payload.c create mode 100644 dummy_payload/dummy_payload.lds create mode 100644 dummy_payload/dummy_payload.mk.in create mode 120000 dummy_payload/dummy_sbi.S create mode 100644 machine/atomic.h create mode 100644 machine/bits.h create mode 100644 machine/configstring.c create mode 100644 machine/emulation.c create mode 100644 machine/emulation.h create mode 100644 machine/encoding.h create mode 100644 machine/fp_asm.S create mode 100644 machine/fp_emulation.c create mode 100644 machine/fp_emulation.h create mode 100644 machine/htif.h create mode 100644 machine/machine.ac create mode 100644 machine/machine.mk.in create mode 100644 machine/mcall.h create mode 100644 machine/mentry.S create mode 100644 machine/minit.c create mode 100644 machine/mtrap.c create mode 100644 machine/mtrap.h create mode 100644 machine/sbi.S create mode 100644 machine/sbi.h create mode 100644 machine/sbi_entry.S create mode 100644 machine/sbi_impl.c create mode 100644 machine/unprivileged_memory.h create mode 100644 machine/vm.h delete mode 100644 pk/atomic.h delete mode 100644 pk/bbl.c delete mode 100644 pk/bits.h delete mode 100644 pk/configstring.c delete mode 100644 pk/emulation.c delete mode 100644 pk/emulation.h delete mode 100644 pk/encoding.h delete mode 100644 pk/fp_asm.S delete mode 100644 pk/fp_emulation.c delete mode 100644 pk/fp_emulation.h delete mode 100644 pk/init.c delete mode 100644 pk/logo.c delete mode 100644 pk/mcall.h delete mode 100644 pk/mentry.S delete mode 100644 pk/minit.c create mode 100644 pk/mmap.c create mode 100644 pk/mmap.h delete mode 100644 pk/mtrap.c delete mode 100644 pk/mtrap.h delete mode 100644 pk/pk.ld create mode 100644 pk/pk.lds delete mode 100644 pk/sbi.S delete mode 100644 pk/sbi.h delete mode 100644 pk/sbi_entry.S delete mode 100644 pk/sbi_impl.c delete mode 100644 pk/snprintf.c delete mode 100644 pk/string.c delete mode 100644 pk/unprivileged_memory.h delete mode 100644 pk/vm.c delete mode 100644 pk/vm.h create mode 100644 util/snprintf.c create mode 100644 util/string.c create mode 100644 util/util.ac create mode 100644 util/util.mk.in diff --git a/Makefile.in b/Makefile.in index 108d19d..8babada 100644 --- a/Makefile.in +++ b/Makefile.in @@ -35,6 +35,7 @@ default : all project_name := @PACKAGE_TARNAME@ src_dir := @srcdir@ scripts_dir := $(src_dir)/scripts +bbl_payload := @BBL_PAYLOAD@ # If the version information is not in the configure script, then we # assume that we are in a working directory. We use the vcs-version.sh @@ -83,7 +84,7 @@ VPATH := $(addprefix $(src_dir)/, $(sprojs_enabled)) # - CXXFLAGS : flags for C++ compiler (eg. -Wall,-g,-O3) CC := @CC@ -CFLAGS := @CFLAGS@ $(CFLAGS) +CFLAGS := @CFLAGS@ $(CFLAGS) -DBBL_PAYLOAD=\"$(bbl_payload)\" COMPILE := $(CC) -MMD -MP $(CFLAGS) \ $(sprojs_include) # Linker @@ -93,7 +94,7 @@ COMPILE := $(CC) -MMD -MP $(CFLAGS) \ LD := $(CC) LDFLAGS := @LDFLAGS@ -nostartfiles -nostdlib -static $(LDFLAGS) LIBS := @LIBS@ -LINK := $(LD) $(LDFLAGS) -T $(src_dir)/pk/pk.ld +LINK := $(LD) $(LDFLAGS) # Library creation @@ -258,7 +259,7 @@ $$($(2)_install_prog_objs) : %.o : %.c $(COMPILE) -c $$< $$($(2)_install_prog_exes) : % : %.o $$($(2)_prog_libnames) - $(LINK) -o $$@ $$< $$($(2)_prog_libarg) $(LIBS) + $(LINK) -o $$@ $$< $$($(2)_prog_libarg) $(LIBS) -T $(src_dir)/$(2)/$(2).lds $(2)_c_deps += $$($(2)_install_prog_deps) $(2)_junk += \ diff --git a/bbl/bbl.ac b/bbl/bbl.ac new file mode 100644 index 0000000..51dcf01 --- /dev/null +++ b/bbl/bbl.ac @@ -0,0 +1,8 @@ +AC_ARG_ENABLE([logo], AS_HELP_STRING([--disable-logo], [Disable boot logo])) +AS_IF([test "x$enable_logo" != "xno"], [ + AC_DEFINE([PK_ENABLE_LOGO],,[Define if the RISC-V logo is to be displayed]) +]) + +AC_ARG_WITH([payload], AS_HELP_STRING([--with-payload], [Set ELF payload for bbl]), + [AC_SUBST([BBL_PAYLOAD], $with_payload, [Kernel payload for bbl])], + [AC_SUBST([BBL_PAYLOAD], [dummy_payload], [Kernel payload for bbl])]) diff --git a/bbl/bbl.c b/bbl/bbl.c new file mode 100644 index 0000000..cd52fc1 --- /dev/null +++ b/bbl/bbl.c @@ -0,0 +1,76 @@ +#include "bbl.h" +#include "mtrap.h" +#include "atomic.h" +#include "vm.h" +#include "bits.h" +#include "config.h" +#include + +static kernel_elf_info info; +static volatile int elf_loaded; + +static void supervisor_vm_init() +{ + uintptr_t highest_va = -first_free_paddr; + mem_size = MIN(mem_size, highest_va - info.first_user_vaddr) & -MEGAPAGE_SIZE; + + pte_t* sbi_pt = (pte_t*)(info.first_vaddr_after_user + info.load_offset); + memset(sbi_pt, 0, RISCV_PGSIZE); + pte_t* middle_pt = (void*)sbi_pt + RISCV_PGSIZE; +#ifndef __riscv64 + size_t num_middle_pts = 1; + pte_t* root_pt = middle_pt; + memset(root_pt, 0, RISCV_PGSIZE); +#else + size_t num_middle_pts = (-info.first_user_vaddr - 1) / GIGAPAGE_SIZE + 1; + pte_t* root_pt = (void*)middle_pt + num_middle_pts * RISCV_PGSIZE; + memset(middle_pt, 0, (num_middle_pts + 1) * RISCV_PGSIZE); + for (size_t i = 0; i < num_middle_pts; i++) + root_pt[(1<> RISCV_PGSHIFT) + i); +#endif + + for (uintptr_t vaddr = info.first_user_vaddr, paddr = vaddr + info.load_offset, end = info.first_vaddr_after_user; + paddr < mem_size; vaddr += MEGAPAGE_SIZE, paddr += MEGAPAGE_SIZE) { + int l2_shift = RISCV_PGLEVEL_BITS + RISCV_PGSHIFT; + size_t l2_idx = (info.first_user_vaddr >> l2_shift) & ((1 << RISCV_PGLEVEL_BITS)-1); + l2_idx += ((vaddr - info.first_user_vaddr) >> l2_shift); + middle_pt[l2_idx] = pte_create(paddr >> RISCV_PGSHIFT, PTE_TYPE_SRWX_GLOBAL); + } + + // map SBI at top of vaddr space + extern char _sbi_end; + uintptr_t num_sbi_pages = ((uintptr_t)&_sbi_end - 1) / RISCV_PGSIZE + 1; + assert(num_sbi_pages <= (1 << RISCV_PGLEVEL_BITS)); + for (uintptr_t i = 0; i < num_sbi_pages; i++) { + uintptr_t idx = (1 << RISCV_PGLEVEL_BITS) - num_sbi_pages + i; + sbi_pt[idx] = pte_create(i, PTE_TYPE_SRX_GLOBAL); + } + pte_t* sbi_pte = middle_pt + ((num_middle_pts << RISCV_PGLEVEL_BITS)-1); + assert(!*sbi_pte); + *sbi_pte = ptd_create((uintptr_t)sbi_pt >> RISCV_PGSHIFT); + + mb(); + root_page_table = root_pt; + write_csr(sptbr, (uintptr_t)root_pt >> RISCV_PGSHIFT); +} + +void boot_loader() +{ + extern char _payload_start, _payload_end; + load_kernel_elf(&_payload_start, &_payload_end - &_payload_start, &info); + supervisor_vm_init(); +#ifdef PK_ENABLE_LOGO + print_logo(); +#endif + mb(); + elf_loaded = 1; + enter_supervisor_mode((void *)info.entry, 0); +} + +void boot_other_hart() +{ + while (!elf_loaded) + ; + mb(); + enter_supervisor_mode((void *)info.entry, 0); +} diff --git a/bbl/bbl.h b/bbl/bbl.h new file mode 100644 index 0000000..e9e1dab --- /dev/null +++ b/bbl/bbl.h @@ -0,0 +1,23 @@ +// See LICENSE for license details. + +#ifndef _BBL_H +#define _BBL_H + +#ifndef __ASSEMBLER__ + +#include +#include + +typedef struct { + uintptr_t entry; + uintptr_t first_user_vaddr; + uintptr_t first_vaddr_after_user; + uintptr_t load_offset; +} kernel_elf_info; + +void load_kernel_elf(void* blob, size_t size, kernel_elf_info* info); +void print_logo(); + +#endif // !__ASSEMBLER__ + +#endif diff --git a/bbl/bbl.lds b/bbl/bbl.lds new file mode 100644 index 0000000..5ccdf22 --- /dev/null +++ b/bbl/bbl.lds @@ -0,0 +1,100 @@ +OUTPUT_ARCH( "riscv" ) + +ENTRY( reset_vector ) + +SECTIONS +{ + + /*--------------------------------------------------------------------*/ + /* Code and read-only segment */ + /*--------------------------------------------------------------------*/ + + /* Begining of code and text segment */ + . = 0; + _ftext = .; + PROVIDE( eprol = . ); + + .text : + { + *(.text.init) + } + + /* text: Program code section */ + .text : + { + *(.text) + *(.text.*) + *(.gnu.linkonce.t.*) + } + + /* rodata: Read-only data */ + .rodata : + { + *(.rdata) + *(.rodata) + *(.rodata.*) + *(.gnu.linkonce.r.*) + } + + /* End of code and read-only segment */ + PROVIDE( etext = . ); + _etext = .; + + /*--------------------------------------------------------------------*/ + /* Initialized data segment */ + /*--------------------------------------------------------------------*/ + + /* Start of initialized data segment */ + . = ALIGN(16); + _fdata = .; + + /* data: Writable data */ + .data : + { + *(.data) + *(.data.*) + *(.srodata*) + *(.gnu.linkonce.d.*) + *(.comment) + } + + /* End of initialized data segment */ + . = ALIGN(4); + PROVIDE( edata = . ); + _edata = .; + + /*--------------------------------------------------------------------*/ + /* Uninitialized data segment */ + /*--------------------------------------------------------------------*/ + + /* Start of uninitialized data segment */ + . = .; + _fbss = .; + + /* sbss: Uninitialized writeable small data section */ + . = .; + + /* bss: Uninitialized writeable data section */ + . = .; + _bss_start = .; + .bss : + { + *(.bss) + *(.bss.*) + *(.sbss*) + *(.gnu.linkonce.b.*) + *(COMMON) + } + + .sbi : + { + *(.sbi) + } + + .payload : + { + *(.payload) + } + + _end = .; +} diff --git a/bbl/bbl.mk.in b/bbl/bbl.mk.in new file mode 100644 index 0000000..1bb4cd1 --- /dev/null +++ b/bbl/bbl.mk.in @@ -0,0 +1,22 @@ +bbl_subproject_deps = \ + util \ + softfloat \ + machine \ + dummy_payload \ + +bbl_hdrs = \ + bbl.h \ + +bbl_c_srcs = \ + kernel_elf.c \ + logo.c \ + +bbl_asm_srcs = \ + payload.S \ + +payload.o: $(bbl_payload) + +bbl_test_srcs = + +bbl_install_prog_srcs = \ + bbl.c \ diff --git a/bbl/kernel_elf.c b/bbl/kernel_elf.c new file mode 100644 index 0000000..e22c35c --- /dev/null +++ b/bbl/kernel_elf.c @@ -0,0 +1,54 @@ +// See LICENSE for license details. + +#include "mtrap.h" +#include "bbl.h" +#include "bits.h" +#include "vm.h" +#include +#include + +void load_kernel_elf(void* blob, size_t size, kernel_elf_info* info) +{ + Elf_Ehdr* eh = blob; + if (sizeof(*eh) > size || + !(eh->e_ident[0] == '\177' && eh->e_ident[1] == 'E' && + eh->e_ident[2] == 'L' && eh->e_ident[3] == 'F')) + goto fail; + + if (IS_ELF64(*eh) != (sizeof(uintptr_t) == 8)) + goto fail; + + uintptr_t min_vaddr = -1, max_vaddr = 0; + size_t phdr_size = eh->e_phnum * sizeof(Elf_Ehdr); + Elf_Phdr* ph = blob + eh->e_phoff; + if (eh->e_phoff + phdr_size > size) + goto fail; + first_free_paddr = ROUNDUP(first_free_paddr, MEGAPAGE_SIZE); + for (int i = 0; i < eh->e_phnum; i++) + if (ph[i].p_type == PT_LOAD && ph[i].p_memsz && ph[i].p_vaddr < min_vaddr) + min_vaddr = ph[i].p_vaddr; + min_vaddr = ROUNDDOWN(min_vaddr, MEGAPAGE_SIZE); + uintptr_t bias = first_free_paddr - min_vaddr; + for (int i = eh->e_phnum - 1; i >= 0; i--) { + if(ph[i].p_type == PT_LOAD && ph[i].p_memsz) { + uintptr_t prepad = ph[i].p_vaddr % RISCV_PGSIZE; + uintptr_t vaddr = ph[i].p_vaddr + bias; + if (vaddr + ph[i].p_memsz > max_vaddr) + max_vaddr = vaddr + ph[i].p_memsz; + if (ph[i].p_offset + ph[i].p_filesz > size) + goto fail; + memcpy((void*)vaddr, blob + ph[i].p_offset, ph[i].p_filesz); + memset((void*)vaddr - prepad, 0, prepad); + memset((void*)vaddr + ph[i].p_filesz, 0, ph[i].p_memsz - ph[i].p_filesz); + } + } + + info->entry = eh->e_entry; + info->load_offset = bias; + info->first_user_vaddr = min_vaddr; + info->first_vaddr_after_user = ROUNDUP(max_vaddr - bias, RISCV_PGSIZE); + return; + +fail: + die("failed to load payload"); +} diff --git a/bbl/logo.c b/bbl/logo.c new file mode 100644 index 0000000..673899d --- /dev/null +++ b/bbl/logo.c @@ -0,0 +1,32 @@ +#include +#include "mtrap.h" + +static const char logo[] = +" vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" +" vvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" +"rrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvvvv\n" +"rrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" +"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" +"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" +"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" +"rrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvv \n" +"rrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvv \n" +"rr vvvvvvvvvvvvvvvvvvvvvv \n" +"rr vvvvvvvvvvvvvvvvvvvvvvvv rr\n" +"rrrr vvvvvvvvvvvvvvvvvvvvvvvvvv rrrr\n" +"rrrrrr vvvvvvvvvvvvvvvvvvvvvv rrrrrr\n" +"rrrrrrrr vvvvvvvvvvvvvvvvvv rrrrrrrr\n" +"rrrrrrrrrr vvvvvvvvvvvvvv rrrrrrrrrr\n" +"rrrrrrrrrrrr vvvvvvvvvv rrrrrrrrrrrr\n" +"rrrrrrrrrrrrrr vvvvvv rrrrrrrrrrrrrr\n" +"rrrrrrrrrrrrrrrr vv rrrrrrrrrrrrrrrr\n" +"rrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrr\n" +"rrrrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrrrr\n" +"rrrrrrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrrrrrr\n" +"\n" +" INSTRUCTION SETS WANT TO BE FREE\n"; + +void print_logo() +{ + putstring(logo); +} diff --git a/bbl/payload.S b/bbl/payload.S new file mode 100644 index 0000000..7ff1e58 --- /dev/null +++ b/bbl/payload.S @@ -0,0 +1,7 @@ +.section ".payload","a",@progbits +.align 3 + +.globl _payload_start, _payload_end +_payload_start: +.incbin BBL_PAYLOAD +_payload_end: diff --git a/config.h.in b/config.h.in index 3424ee6..bc6369e 100644 --- a/config.h.in +++ b/config.h.in @@ -1,5 +1,14 @@ /* config.h.in. Generated from configure.ac by autoheader. */ +/* Define if subproject MCPPBS_SPROJ_NORM is enabled */ +#undef BBL_ENABLED + +/* Define if subproject MCPPBS_SPROJ_NORM is enabled */ +#undef DUMMY_PAYLOAD_ENABLED + +/* Define if subproject MCPPBS_SPROJ_NORM is enabled */ +#undef MACHINE_ENABLED + /* Define to the address where bug reports for this package should be sent. */ #undef PACKAGE_BUGREPORT @@ -35,3 +44,6 @@ /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS + +/* Define if subproject MCPPBS_SPROJ_NORM is enabled */ +#undef UTIL_ENABLED diff --git a/configure b/configure index 0cd8fe2..6386de0 100755 --- a/configure +++ b/configure @@ -590,6 +590,7 @@ ac_subst_vars='LTLIBOBJS LIBOBJS subprojects_enabled subprojects +BBL_PAYLOAD EGREP GREP CPP @@ -667,6 +668,8 @@ enable_option_checking enable_stow enable_optional_subprojects enable_vm +enable_logo +with_payload enable_fp_emulation ' ac_precious_vars='build_alias @@ -1303,6 +1306,11 @@ Optional Features: --disable-logo Disable boot logo --disable-fp-emulation Disable floating-point emulation +Optional Packages: + --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] + --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) + --with-payload Set ELF payload for bbl + Some influential environment variables: CC C compiler command CFLAGS C compiler flags @@ -4046,9 +4054,53 @@ $as_echo "#define PK_ENABLE_VM /**/" >>confdefs.h fi -# Check whether --enable-vm was given. -if test "${enable_vm+set}" = set; then : - enableval=$enable_vm; + + + + + # Determine if this is a required or an optional subproject + + + + # Determine if there is a group with the same name + + + + # Create variations of the subproject name suitable for use as a CPP + # enabled define, a shell enabled variable, and a shell function + + + + + + + + + + + + # Add subproject to our running list + + subprojects="$subprojects bbl" + + # Process the subproject appropriately. If enabled add it to the + # $enabled_subprojects running shell variable, set a + # SUBPROJECT_ENABLED C define, and include the appropriate + # 'subproject.ac'. + + + { $as_echo "$as_me:${as_lineno-$LINENO}: configuring default subproject : bbl" >&5 +$as_echo "$as_me: configuring default subproject : bbl" >&6;} + ac_config_files="$ac_config_files bbl.mk:bbl/bbl.mk.in" + + enable_bbl_sproj="yes" + subprojects_enabled="$subprojects_enabled bbl" + +$as_echo "#define BBL_ENABLED /**/" >>confdefs.h + + # Check whether --enable-logo was given. +if test "${enable_logo+set}" = set; then : + enableval=$enable_logo; fi if test "x$enable_logo" != "xno"; then : @@ -4059,16 +4111,13 @@ $as_echo "#define PK_ENABLE_LOGO /**/" >>confdefs.h fi -# Check whether --enable-fp-emulation was given. -if test "${enable_fp_emulation+set}" = set; then : - enableval=$enable_fp_emulation; -fi -if test "x$enable_fp_emulation" != "xno"; then : - - -$as_echo "#define PK_ENABLE_FP_EMULATION /**/" >>confdefs.h +# Check whether --with-payload was given. +if test "${with_payload+set}" = set; then : + withval=$with_payload; BBL_PAYLOAD=$with_payload +else + BBL_PAYLOAD=dummy_payload fi @@ -4076,6 +4125,7 @@ fi + # Determine if this is a required or an optional subproject @@ -4121,6 +4171,153 @@ $as_echo "#define SOFTFLOAT_ENABLED /**/" >>confdefs.h + # Determine if this is a required or an optional subproject + + + + # Determine if there is a group with the same name + + + + # Create variations of the subproject name suitable for use as a CPP + # enabled define, a shell enabled variable, and a shell function + + + + + + + + + + + + # Add subproject to our running list + + subprojects="$subprojects dummy_payload" + + # Process the subproject appropriately. If enabled add it to the + # $enabled_subprojects running shell variable, set a + # SUBPROJECT_ENABLED C define, and include the appropriate + # 'subproject.ac'. + + + { $as_echo "$as_me:${as_lineno-$LINENO}: configuring default subproject : dummy_payload" >&5 +$as_echo "$as_me: configuring default subproject : dummy_payload" >&6;} + ac_config_files="$ac_config_files dummy_payload.mk:dummy_payload/dummy_payload.mk.in" + + enable_dummy_payload_sproj="yes" + subprojects_enabled="$subprojects_enabled dummy_payload" + +$as_echo "#define DUMMY_PAYLOAD_ENABLED /**/" >>confdefs.h + + + + + + + # Determine if this is a required or an optional subproject + + + + # Determine if there is a group with the same name + + + + # Create variations of the subproject name suitable for use as a CPP + # enabled define, a shell enabled variable, and a shell function + + + + + + + + + + + + # Add subproject to our running list + + subprojects="$subprojects machine" + + # Process the subproject appropriately. If enabled add it to the + # $enabled_subprojects running shell variable, set a + # SUBPROJECT_ENABLED C define, and include the appropriate + # 'subproject.ac'. + + + { $as_echo "$as_me:${as_lineno-$LINENO}: configuring default subproject : machine" >&5 +$as_echo "$as_me: configuring default subproject : machine" >&6;} + ac_config_files="$ac_config_files machine.mk:machine/machine.mk.in" + + enable_machine_sproj="yes" + subprojects_enabled="$subprojects_enabled machine" + +$as_echo "#define MACHINE_ENABLED /**/" >>confdefs.h + + # Check whether --enable-fp-emulation was given. +if test "${enable_fp_emulation+set}" = set; then : + enableval=$enable_fp_emulation; +fi + +if test "x$enable_fp_emulation" != "xno"; then : + + +$as_echo "#define PK_ENABLE_FP_EMULATION /**/" >>confdefs.h + + +fi + + + + + + # Determine if this is a required or an optional subproject + + + + # Determine if there is a group with the same name + + + + # Create variations of the subproject name suitable for use as a CPP + # enabled define, a shell enabled variable, and a shell function + + + + + + + + + + + + # Add subproject to our running list + + subprojects="$subprojects util" + + # Process the subproject appropriately. If enabled add it to the + # $enabled_subprojects running shell variable, set a + # SUBPROJECT_ENABLED C define, and include the appropriate + # 'subproject.ac'. + + + { $as_echo "$as_me:${as_lineno-$LINENO}: configuring default subproject : util" >&5 +$as_echo "$as_me: configuring default subproject : util" >&6;} + ac_config_files="$ac_config_files util.mk:util/util.mk.in" + + enable_util_sproj="yes" + subprojects_enabled="$subprojects_enabled util" + +$as_echo "#define UTIL_ENABLED /**/" >>confdefs.h + + + + + + # Output make variables @@ -4838,7 +5035,11 @@ for ac_config_target in $ac_config_targets do case $ac_config_target in "pk.mk") CONFIG_FILES="$CONFIG_FILES pk.mk:pk/pk.mk.in" ;; + "bbl.mk") CONFIG_FILES="$CONFIG_FILES bbl.mk:bbl/bbl.mk.in" ;; "softfloat.mk") CONFIG_FILES="$CONFIG_FILES softfloat.mk:softfloat/softfloat.mk.in" ;; + "dummy_payload.mk") CONFIG_FILES="$CONFIG_FILES dummy_payload.mk:dummy_payload/dummy_payload.mk.in" ;; + "machine.mk") CONFIG_FILES="$CONFIG_FILES machine.mk:machine/machine.mk.in" ;; + "util.mk") CONFIG_FILES="$CONFIG_FILES util.mk:util/util.mk.in" ;; "config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;; "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; diff --git a/configure.ac b/configure.ac index 64ddc5f..6253228 100644 --- a/configure.ac +++ b/configure.ac @@ -81,7 +81,7 @@ AC_SUBST([LIBS], ["-lgcc"]) # The '*' suffix indicates an optional subproject. The '**' suffix # indicates an optional subproject which is also the name of a group. -MCPPBS_SUBPROJECTS([ pk, softfloat ]) +MCPPBS_SUBPROJECTS([ pk, bbl, softfloat, dummy_payload, machine, util ]) #------------------------------------------------------------------------- # MCPPBS subproject groups diff --git a/dummy_payload/dummy_payload.ac b/dummy_payload/dummy_payload.ac new file mode 100644 index 0000000..e69de29 diff --git a/dummy_payload/dummy_payload.c b/dummy_payload/dummy_payload.c new file mode 100644 index 0000000..61ed59a --- /dev/null +++ b/dummy_payload/dummy_payload.c @@ -0,0 +1,21 @@ +#include +#include "sbi.h" + +asm (".globl _start\n\ + _start: la sp, stack\n\ + j entry\n\ + .pushsection .rodata\n\ + .align 4\n\ + .skip 4096\n\ + stack:\n\ + .popsection"); + +void entry() +{ + const char* message = +"This is bbl's dummy_payload. To boot a real kernel, reconfigure\n\ +bbl with the flag --with-payload=PATH, then rebuild bbl.\n"; + while (*message) + sbi_console_putchar(*message++); + sbi_shutdown(); +} diff --git a/dummy_payload/dummy_payload.lds b/dummy_payload/dummy_payload.lds new file mode 100644 index 0000000..ee9410b --- /dev/null +++ b/dummy_payload/dummy_payload.lds @@ -0,0 +1,3 @@ +SECTIONS { + . = -0x80000000; +} diff --git a/dummy_payload/dummy_payload.mk.in b/dummy_payload/dummy_payload.mk.in new file mode 100644 index 0000000..b1d4ac7 --- /dev/null +++ b/dummy_payload/dummy_payload.mk.in @@ -0,0 +1,13 @@ +dummy_payload_subproject_deps = \ + +dummy_payload_hdrs = \ + +dummy_payload_c_srcs = \ + +dummy_payload_asm_srcs = \ + dummy_sbi.S \ + +dummy_payload_test_srcs = + +dummy_payload_install_prog_srcs = \ + dummy_payload.c \ diff --git a/dummy_payload/dummy_sbi.S b/dummy_payload/dummy_sbi.S new file mode 120000 index 0000000..2978a97 --- /dev/null +++ b/dummy_payload/dummy_sbi.S @@ -0,0 +1 @@ +../machine/sbi.S \ No newline at end of file diff --git a/machine/atomic.h b/machine/atomic.h new file mode 100644 index 0000000..b23acae --- /dev/null +++ b/machine/atomic.h @@ -0,0 +1,72 @@ +// See LICENSE for license details. + +#ifndef _RISCV_ATOMIC_H +#define _RISCV_ATOMIC_H + +#include "config.h" +#include "encoding.h" + +// Currently, interrupts are always disabled in M-mode. +#define disable_irqsave() (0) +#define enable_irqrestore(flags) ((void) (flags)) + +typedef struct { int lock; } spinlock_t; +#define SPINLOCK_INIT {0} + +#define mb() asm volatile ("fence" ::: "memory") +#define atomic_set(ptr, val) (*(volatile typeof(*(ptr)) *)(ptr) = val) +#define atomic_read(ptr) (*(volatile typeof(*(ptr)) *)(ptr)) + +#ifdef __riscv_atomic +# define atomic_add(ptr, inc) __sync_fetch_and_add(ptr, inc) +# define atomic_or(ptr, inc) __sync_fetch_and_or(ptr, inc) +# define atomic_swap(ptr, swp) __sync_lock_test_and_set(ptr, swp) +# define atomic_cas(ptr, cmp, swp) __sync_val_compare_and_swap(ptr, cmp, swp) +#else +# define atomic_binop(ptr, inc, op) ({ \ + long flags = disable_irqsave(); \ + typeof(*(ptr)) res = atomic_read(ptr); \ + atomic_set(ptr, op); \ + enable_irqrestore(flags); \ + res; }) +# define atomic_add(ptr, inc) atomic_binop(ptr, inc, res + (inc)) +# define atomic_or(ptr, inc) atomic_binop(ptr, inc, res | (inc)) +# define atomic_swap(ptr, inc) atomic_binop(ptr, inc, (inc)) +# define atomic_cas(ptr, cmp, swp) ({ \ + long flags = disable_irqsave(); \ + typeof(*(ptr)) res = *(volatile typeof(*(ptr)) *)(ptr); \ + if (res == (cmp)) *(volatile typeof(ptr))(ptr) = (swp); \ + enable_irqrestore(flags); \ + res; }) +#endif + +static inline void spinlock_lock(spinlock_t* lock) +{ + do + { + while (atomic_read(&lock->lock)) + ; + } while (atomic_swap(&lock->lock, -1)); + mb(); +} + +static inline void spinlock_unlock(spinlock_t* lock) +{ + mb(); + atomic_set(&lock->lock,0); +} + +static inline long spinlock_lock_irqsave(spinlock_t* lock) +{ + long flags = disable_irqsave(); + spinlock_lock(lock); + return flags; +} + +static inline void spinlock_unlock_irqrestore(spinlock_t* lock, long flags) +{ + spinlock_unlock(lock); + enable_irqrestore(flags); +} + +#endif diff --git a/machine/bits.h b/machine/bits.h new file mode 100644 index 0000000..72514ae --- /dev/null +++ b/machine/bits.h @@ -0,0 +1,33 @@ +#ifndef _RISCV_BITS_H +#define _RISCV_BITS_H + +#define likely(x) __builtin_expect((x), 1) +#define unlikely(x) __builtin_expect((x), 0) + +#define ROUNDUP(a, b) ((((a)-1)/(b)+1)*(b)) +#define ROUNDDOWN(a, b) ((a)/(b)*(b)) + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define CLAMP(a, lo, hi) MIN(MAX(a, lo), hi) + +#define EXTRACT_FIELD(val, which) (((val) & (which)) / ((which) & ~((which)-1))) +#define INSERT_FIELD(val, which, fieldval) (((val) & ~(which)) | ((fieldval) * ((which) & ~((which)-1)))) + +#define STR(x) XSTR(x) +#define XSTR(x) #x + +#ifdef __riscv64 +# define SLL32 sllw +# define STORE sd +# define LOAD ld +# define LOG_REGBYTES 3 +#else +# define SLL32 sll +# define STORE sw +# define LOAD lw +# define LOG_REGBYTES 2 +#endif +#define REGBYTES (1 << LOG_REGBYTES) + +#endif diff --git a/machine/configstring.c b/machine/configstring.c new file mode 100644 index 0000000..847cd4d --- /dev/null +++ b/machine/configstring.c @@ -0,0 +1,200 @@ +#include "encoding.h" +#include "mtrap.h" +#include + +static const char* skip_whitespace(const char* str) +{ + while (*str && *str <= ' ') + str++; + return str; +} + +static const char* skip_string(const char* str) +{ + while (*str && *str++ != '"') + ; + return str; +} + +static int is_hex(char ch) +{ + return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); +} + +static int parse_hex(char ch) +{ + return (ch >= '0' && ch <= '9') ? ch - '0' : + (ch >= 'a' && ch <= 'f') ? ch - 'a' + 10 : + ch - 'A' + 10; +} + +static const char* skip_key(const char* str) +{ + while (*str >= 35 && *str <= 122 && *str != ';') + str++; + return str; +} + +typedef struct { + const char* start; + const char* end; +} query_result; + +static query_result query_config_string(const char* str, const char* k) +{ + size_t ksize = 0; + while (k[ksize] && k[ksize] != '{') + ksize++; + int last = !k[ksize]; + + query_result res = {0, 0}; + while (1) { + const char* key_start = str = skip_whitespace(str); + const char* key_end = str = skip_key(str); + int match = (key_end - key_start) == ksize; + if (match) + for (size_t i = 0; i < ksize; i++) + if (key_start[i] != k[i]) + match = 0; + const char* value_start = str = skip_whitespace(str); + while (*str != ';') { + if (!*str) { + return res; + } else if (*str == '"') { + str = skip_string(str+1); + } else if (*str == '{') { + const char* search_key = match && !last ? k + ksize + 1 : ""; + query_result inner_res = query_config_string(str + 1, search_key); + if (inner_res.start) + return inner_res; + str = inner_res.end + 1; + } else { + str = skip_key(str); + } + str = skip_whitespace(str); + } + res.end = str; + if (match && last) { + res.start = value_start; + return res; + } + str = skip_whitespace(str+1); + if (*str == '}') { + res.end = str; + return res; + } + } +} + +static void parse_string(query_result r, char* buf) +{ + if (r.start < r.end) { + if (*r.start == '"') { + for (const char* p = r.start + 1; p < r.end && *p != '"'; p++) { + char ch = p[0]; + if (ch == '\\' && p[1] == 'x' && is_hex(p[2])) { + ch = parse_hex(p[2]); + if (is_hex(p[3])) { + ch = (ch << 4) + parse_hex(p[3]); + p++; + } + p += 2; + } + *buf++ = ch; + } + } else { + for (const char* p = r.start; p < r.end && *p > ' '; p++) + *buf++ = *p; + } + } + *buf = 0; +} + +#define get_string(name, search_res) \ + char name[(search_res).end - (search_res).start + 1]; \ + parse_string(search_res, name) + +static unsigned long __get_uint_hex(const char* s) +{ + unsigned long res = 0; + while (*s) { + if (is_hex(*s)) + res = (res << 4) + parse_hex(*s); + else if (*s != '_') + break; + s++; + } + return res; +} + +static unsigned long __get_uint_dec(const char* s) +{ + unsigned long res = 0; + while (*s) { + if (*s >= '0' && *s <= '9') + res = res * 10 + (*s - '0'); + else + break; + s++; + } + return res; +} + +static unsigned long __get_uint(const char* s) +{ + if (s[0] == '0' && s[1] == 'x') + return __get_uint_hex(s+2); + return __get_uint_dec(s); +} + +static unsigned long get_uint(query_result res) +{ + get_string(s, res); + return __get_uint(s); +} + +static long get_sint(query_result res) +{ + get_string(s, res); + if (s[0] == '-') + return -__get_uint(s+1); + return __get_uint(s); +} + +static void query_mem(const char* config_string) +{ + query_result res = query_config_string(config_string, "ram{0{addr"); + assert(res.start); + uintptr_t base = get_uint(res); + res = query_config_string(config_string, "ram{0{size"); + mem_size = get_uint(res); +} + +static void query_harts(const char* config_string) +{ + for (int core = 0, hart; ; core++) { + for (hart = 0; ; hart++) { + char buf[32]; + snprintf(buf, sizeof buf, "core{%d{%d{addr", core, hart); + query_result res = query_config_string(config_string, buf); + if (!res.start) + break; + csr_t* base = (csr_t*)get_uint(res); + uintptr_t hart_id = base[CSR_MHARTID]; + hls_init(hart_id, base); + num_harts++; + assert(hart_id == num_harts-1); + } + if (!hart) + break; + } + assert(num_harts); + assert(num_harts <= MAX_HARTS); +} + +void parse_config_string() +{ + const char* s = (const char*)read_csr(mcfgaddr); + query_mem(s); + query_harts(s); +} diff --git a/machine/emulation.c b/machine/emulation.c new file mode 100644 index 0000000..5a66997 --- /dev/null +++ b/machine/emulation.c @@ -0,0 +1,313 @@ +#include "emulation.h" +#include "fp_emulation.h" +#include "config.h" +#include "unprivileged_memory.h" +#include "mtrap.h" +#include + +void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + asm (".pushsection .rodata\n" + "illegal_insn_trap_table:\n" + " .word truly_illegal_insn\n" +#ifdef PK_ENABLE_FP_EMULATION + " .word emulate_float_load\n" +#else + " .word truly_illegal_insn\n" +#endif + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" +#ifdef PK_ENABLE_FP_EMULATION + " .word emulate_float_store\n" +#else + " .word truly_illegal_insn\n" +#endif + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_mul_div\n" + " .word truly_illegal_insn\n" + " .word emulate_mul_div32\n" + " .word truly_illegal_insn\n" +#ifdef PK_ENABLE_FP_EMULATION + " .word emulate_fmadd\n" + " .word emulate_fmadd\n" + " .word emulate_fmadd\n" + " .word emulate_fmadd\n" + " .word emulate_fp\n" +#else + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" +#endif + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_system\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .popsection"); + + uintptr_t mstatus; + insn_t insn = get_insn(mepc, &mstatus); + + if (unlikely((insn & 3) != 3)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + write_csr(mepc, mepc + 4); + + extern int32_t illegal_insn_trap_table[]; + int32_t* pf = (void*)illegal_insn_trap_table + (insn & 0x7c); + emulation_func f = (emulation_func)(uintptr_t)*pf; + f(regs, mcause, mepc, mstatus, insn); +} + +void __attribute__((noinline)) truly_illegal_insn(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn) +{ + redirect_trap(mepc, mstatus); +} + +void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + union { + uint8_t bytes[8]; + uintptr_t intx; + uint64_t int64; + } val; + uintptr_t mstatus; + insn_t insn = get_insn(mepc, &mstatus); + uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); + + int shift = 0, fp = 0, len; + if ((insn & MASK_LW) == MATCH_LW) + len = 4, shift = 8*(sizeof(uintptr_t) - len); +#ifdef __riscv64 + else if ((insn & MASK_LD) == MATCH_LD) + len = 8, shift = 8*(sizeof(uintptr_t) - len); + else if ((insn & MASK_LWU) == MATCH_LWU) + fp = 0, len = 4, shift = 0; +#endif + else if ((insn & MASK_FLD) == MATCH_FLD) + fp = 1, len = 8; + else if ((insn & MASK_FLW) == MATCH_FLW) + fp = 1, len = 4; + else if ((insn & MASK_LH) == MATCH_LH) + len = 2, shift = 8*(sizeof(uintptr_t) - len); + else if ((insn & MASK_LHU) == MATCH_LHU) + len = 2; + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + val.int64 = 0; + for (intptr_t i = len-1; i >= 0; i--) + val.bytes[i] = load_uint8_t((void *)(addr + i), mepc); + + if (!fp) + SET_RD(insn, regs, (intptr_t)val.intx << shift >> shift); + else if (len == 8) + SET_F64_RD(insn, regs, val.int64); + else + SET_F32_RD(insn, regs, val.intx); + + write_csr(mepc, mepc + 4); +} + +void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + union { + uint8_t bytes[8]; + uintptr_t intx; + uint64_t int64; + } val; + uintptr_t mstatus; + insn_t insn = get_insn(mepc, &mstatus); + int len; + + val.intx = GET_RS2(insn, regs); + if ((insn & MASK_SW) == MATCH_SW) + len = 4; +#ifdef __riscv64 + else if ((insn & MASK_SD) == MATCH_SD) + len = 8; +#endif + else if ((insn & MASK_FSD) == MATCH_FSD) + len = 8, val.int64 = GET_F64_RS2(insn, regs); + else if ((insn & MASK_FSW) == MATCH_FSW) + len = 4, val.intx = GET_F32_RS2(insn, regs); + else if ((insn & MASK_SH) == MATCH_SH) + len = 2; + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); + for (int i = 0; i < len; i++) + store_uint8_t((void *)(addr + i), val.bytes[i], mepc); + + write_csr(mepc, mepc + 4); +} + +#ifdef __riscv64 +typedef __int128 double_int; +#else +typedef int64_t double_int; +#endif + +DECLARE_EMULATION_FUNC(emulate_mul_div) +{ + uintptr_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs), val; + +#ifndef __riscv_muldiv + // If compiled with -mno-multiply, GCC will expand these out + if ((insn & MASK_MUL) == MATCH_MUL) + val = rs1 * rs2; + else if ((insn & MASK_DIV) == MATCH_DIV) + val = (intptr_t)rs1 / (intptr_t)rs2; + else if ((insn & MASK_DIVU) == MATCH_DIVU) + val = rs1 / rs2; + else if ((insn & MASK_REM) == MATCH_REM) + val = (intptr_t)rs1 % (intptr_t)rs2; + else if ((insn & MASK_REMU) == MATCH_REMU) + val = rs1 % rs2; + else if ((insn & MASK_MULH) == MATCH_MULH) + val = ((double_int)(intptr_t)rs1 * (double_int)(intptr_t)rs2) >> (8 * sizeof(rs1)); + else if ((insn & MASK_MULHU) == MATCH_MULHU) + val = ((double_int)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); + else if ((insn & MASK_MULHSU) == MATCH_MULHSU) + val = ((double_int)(intptr_t)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); + else +#endif + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + SET_RD(insn, regs, val); +} + +DECLARE_EMULATION_FUNC(emulate_mul_div32) +{ + uint32_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs); + int32_t val; + +#if defined(__riscv64) && !defined(__riscv_muldiv) + // If compiled with -mno-multiply, GCC will expand these out + if ((insn & MASK_MULW) == MATCH_MULW) + val = rs1 * rs2; + else if ((insn & MASK_DIVW) == MATCH_DIVW) + val = (int32_t)rs1 / (int32_t)rs2; + else if ((insn & MASK_DIVUW) == MATCH_DIVUW) + val = rs1 / rs2; + else if ((insn & MASK_REMW) == MATCH_REMW) + val = (int32_t)rs1 % (int32_t)rs2; + else if ((insn & MASK_REMUW) == MATCH_REMUW) + val = rs1 % rs2; + else +#endif + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + SET_RD(insn, regs, val); +} + +static inline int emulate_read_csr(int num, uintptr_t mstatus, uintptr_t* result) +{ + switch (num) + { + case CSR_TIME: + *result = read_csr(mtime) + HLS()->utime_delta; + return 0; + case CSR_CYCLE: + *result = read_csr(mcycle) + HLS()->ucycle_delta; + return 0; + case CSR_INSTRET: + *result = read_csr(minstret) + HLS()->uinstret_delta; + return 0; + case CSR_STIME: + *result = read_csr(mtime) + HLS()->stime_delta; + return 0; + case CSR_SCYCLE: + *result = read_csr(mcycle) + HLS()->scycle_delta; + return 0; + case CSR_SINSTRET: + *result = read_csr(minstret) + HLS()->sinstret_delta; + return 0; +#ifdef __riscv32 + case CSR_TIMEH: + *result = (((uint64_t)read_csr(mtimeh) << 32) + read_csr(mtime) + + HLS()->stime_delta) >> 32; + return 0; + case CSR_CYCLEH: + *result = (((uint64_t)read_csr(mcycleh) << 32) + read_csr(mcycle) + + HLS()->scycle_delta) >> 32; + return 0; + case CSR_INSTRETH: + *result = (((uint64_t)read_csr(minstreth) << 32) + read_csr(minstret) + + HLS()->sinstret_delta) >> 32; + return 0; +#endif +#ifdef PK_ENABLE_FP_EMULATION + case CSR_FRM: + if ((mstatus & MSTATUS_FS) == 0) break; + *result = GET_FRM(); + return 0; + case CSR_FFLAGS: + if ((mstatus & MSTATUS_FS) == 0) break; + *result = GET_FFLAGS(); + return 0; + case CSR_FCSR: + if ((mstatus & MSTATUS_FS) == 0) break; + *result = GET_FCSR(); + return 0; +#endif + } + return -1; +} + +static inline int emulate_write_csr(int num, uintptr_t value, uintptr_t mstatus) +{ + switch (num) + { +#ifdef PK_ENABLE_FP_EMULATION + case CSR_FRM: SET_FRM(value); return 0; + case CSR_FFLAGS: SET_FFLAGS(value); return 0; + case CSR_FCSR: SET_FCSR(value); return 0; +#endif + } + return -1; +} + +DECLARE_EMULATION_FUNC(emulate_system) +{ + int rs1_num = (insn >> 15) & 0x1f; + uintptr_t rs1_val = GET_RS1(insn, regs); + int csr_num = (uint32_t)insn >> 20; + uintptr_t csr_val, new_csr_val; + + if (emulate_read_csr(csr_num, mstatus, &csr_val)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + int do_write = rs1_num; + switch (GET_RM(insn)) + { + case 0: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + case 1: new_csr_val = rs1_val; do_write = 1; break; + case 2: new_csr_val = csr_val | rs1_val; break; + case 3: new_csr_val = csr_val & ~rs1_val; break; + case 4: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + case 5: new_csr_val = rs1_num; do_write = 1; break; + case 6: new_csr_val = csr_val | rs1_num; break; + case 7: new_csr_val = csr_val & ~rs1_num; break; + } + + if (do_write && emulate_write_csr(csr_num, new_csr_val, mstatus)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + SET_RD(insn, regs, csr_val); +} diff --git a/machine/emulation.h b/machine/emulation.h new file mode 100644 index 0000000..f1a71ec --- /dev/null +++ b/machine/emulation.h @@ -0,0 +1,28 @@ +#ifndef _RISCV_EMULATION_H +#define _RISCV_EMULATION_H + +#include "encoding.h" +#include "bits.h" +#include + +typedef uint32_t insn_t; +typedef void (*emulation_func)(uintptr_t*, uintptr_t, uintptr_t, uintptr_t, insn_t); +#define DECLARE_EMULATION_FUNC(name) void name(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn) + +void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); +void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); +void redirect_trap(uintptr_t epc, uintptr_t mstatus); +DECLARE_EMULATION_FUNC(truly_illegal_insn); + +#define GET_REG(insn, pos, regs) ({ \ + int mask = (1 << (5+LOG_REGBYTES)) - (1 << LOG_REGBYTES); \ + (uintptr_t*)((uintptr_t)regs + (((insn) >> ((pos) - LOG_REGBYTES)) & mask)); \ +}) +#define GET_RS1(insn, regs) (*GET_REG(insn, 15, regs)) +#define GET_RS2(insn, regs) (*GET_REG(insn, 20, regs)) +#define SET_RD(insn, regs, val) (*GET_REG(insn, 7, regs) = (val)) +#define IMM_I(insn) ((int32_t)(insn) >> 20) +#define IMM_S(insn) (((int32_t)(insn) >> 25 << 5) | (int32_t)(((insn) >> 7) & 0x1f)) +#define MASK_FUNCT3 0x7000 + +#endif diff --git a/machine/encoding.h b/machine/encoding.h new file mode 100644 index 0000000..f2fab36 --- /dev/null +++ b/machine/encoding.h @@ -0,0 +1,1037 @@ +// See LICENSE for license details. + +#ifndef RISCV_CSR_ENCODING_H +#define RISCV_CSR_ENCODING_H + +#define MSTATUS_UIE 0x00000001 +#define MSTATUS_SIE 0x00000002 +#define MSTATUS_HIE 0x00000004 +#define MSTATUS_MIE 0x00000008 +#define MSTATUS_UPIE 0x00000010 +#define MSTATUS_SPIE 0x00000020 +#define MSTATUS_HPIE 0x00000040 +#define MSTATUS_MPIE 0x00000080 +#define MSTATUS_SPP 0x00000100 +#define MSTATUS_HPP 0x00000600 +#define MSTATUS_MPP 0x00001800 +#define MSTATUS_FS 0x00006000 +#define MSTATUS_XS 0x00018000 +#define MSTATUS_MPRV 0x00020000 +#define MSTATUS_PUM 0x00040000 +#define MSTATUS_VM 0x1F000000 +#define MSTATUS32_SD 0x80000000 +#define MSTATUS64_SD 0x8000000000000000 + +#define SSTATUS_UIE 0x00000001 +#define SSTATUS_SIE 0x00000002 +#define SSTATUS_UPIE 0x00000010 +#define SSTATUS_SPIE 0x00000020 +#define SSTATUS_SPP 0x00000100 +#define SSTATUS_FS 0x00006000 +#define SSTATUS_XS 0x00018000 +#define SSTATUS_PUM 0x00040000 +#define SSTATUS32_SD 0x80000000 +#define SSTATUS64_SD 0x8000000000000000 + +#define MIP_SSIP (1 << IRQ_S_SOFT) +#define MIP_HSIP (1 << IRQ_H_SOFT) +#define MIP_MSIP (1 << IRQ_M_SOFT) +#define MIP_STIP (1 << IRQ_S_TIMER) +#define MIP_HTIP (1 << IRQ_H_TIMER) +#define MIP_MTIP (1 << IRQ_M_TIMER) + +#define SIP_SSIP MIP_SSIP +#define SIP_STIP MIP_STIP + +#define PRV_U 0 +#define PRV_S 1 +#define PRV_H 2 +#define PRV_M 3 + +#define VM_MBARE 0 +#define VM_MBB 1 +#define VM_MBBID 2 +#define VM_SV32 8 +#define VM_SV39 9 +#define VM_SV48 10 + +#define IRQ_S_SOFT 1 +#define IRQ_H_SOFT 2 +#define IRQ_M_SOFT 3 +#define IRQ_S_TIMER 5 +#define IRQ_H_TIMER 6 +#define IRQ_M_TIMER 7 +#define IRQ_S_DEV 9 +#define IRQ_H_DEV 10 +#define IRQ_M_DEV 11 +#define IRQ_COP 12 +#define IRQ_HOST 13 + +#define DEFAULT_RSTVEC 0x0 +#define DEFAULT_NMIVEC 0x4 +#define DEFAULT_MTVEC 0x8 + +// page table entry (PTE) fields +#define PTE_V 0x001 // Valid +#define PTE_TYPE 0x01E // Type +#define PTE_R 0x020 // Referenced +#define PTE_D 0x040 // Dirty +#define PTE_SOFT 0x380 // Reserved for Software + +#define PTE_TYPE_TABLE 0x00 +#define PTE_TYPE_TABLE_GLOBAL 0x02 +#define PTE_TYPE_URX_SR 0x04 +#define PTE_TYPE_URWX_SRW 0x06 +#define PTE_TYPE_UR_SR 0x08 +#define PTE_TYPE_URW_SRW 0x0A +#define PTE_TYPE_URX_SRX 0x0C +#define PTE_TYPE_URWX_SRWX 0x0E +#define PTE_TYPE_SR 0x10 +#define PTE_TYPE_SRW 0x12 +#define PTE_TYPE_SRX 0x14 +#define PTE_TYPE_SRWX 0x16 +#define PTE_TYPE_SR_GLOBAL 0x18 +#define PTE_TYPE_SRW_GLOBAL 0x1A +#define PTE_TYPE_SRX_GLOBAL 0x1C +#define PTE_TYPE_SRWX_GLOBAL 0x1E + +#define PTE_PPN_SHIFT 10 + +#define PTE_TABLE(PTE) ((0x0000000AU >> ((PTE) & 0x1F)) & 1) +#define PTE_UR(PTE) ((0x0000AAA0U >> ((PTE) & 0x1F)) & 1) +#define PTE_UW(PTE) ((0x00008880U >> ((PTE) & 0x1F)) & 1) +#define PTE_UX(PTE) ((0x0000A0A0U >> ((PTE) & 0x1F)) & 1) +#define PTE_SR(PTE) ((0xAAAAAAA0U >> ((PTE) & 0x1F)) & 1) +#define PTE_SW(PTE) ((0x88888880U >> ((PTE) & 0x1F)) & 1) +#define PTE_SX(PTE) ((0xA0A0A000U >> ((PTE) & 0x1F)) & 1) + +#define PTE_CHECK_PERM(PTE, SUPERVISOR, STORE, FETCH) \ + ((STORE) ? ((SUPERVISOR) ? PTE_SW(PTE) : PTE_UW(PTE)) : \ + (FETCH) ? ((SUPERVISOR) ? PTE_SX(PTE) : PTE_UX(PTE)) : \ + ((SUPERVISOR) ? PTE_SR(PTE) : PTE_UR(PTE))) + +#ifdef __riscv + +#ifdef __riscv64 +# define MSTATUS_SD MSTATUS64_SD +# define SSTATUS_SD SSTATUS64_SD +# define RISCV_PGLEVEL_BITS 9 +#else +# define MSTATUS_SD MSTATUS32_SD +# define SSTATUS_SD SSTATUS32_SD +# define RISCV_PGLEVEL_BITS 10 +#endif +#define RISCV_PGSHIFT 12 +#define RISCV_PGSIZE (1 << RISCV_PGSHIFT) + +#ifndef __ASSEMBLER__ + +#ifdef __GNUC__ + +#define read_csr(reg) ({ unsigned long __tmp; \ + asm volatile ("csrr %0, " #reg : "=r"(__tmp)); \ + __tmp; }) + +#define write_csr(reg, val) \ + asm volatile ("csrw " #reg ", %0" :: "r"(val)) + +#define swap_csr(reg, val) ({ long __tmp; \ + asm volatile ("csrrw %0, " #reg ", %1" : "=r"(__tmp) : "r"(val)); \ + __tmp; }) + +#define set_csr(reg, bit) ({ unsigned long __tmp; \ + if (__builtin_constant_p(bit) && (bit) < 32) \ + asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \ + else \ + asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \ + __tmp; }) + +#define clear_csr(reg, bit) ({ unsigned long __tmp; \ + if (__builtin_constant_p(bit) && (bit) < 32) \ + asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \ + else \ + asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \ + __tmp; }) + +#define rdtime() read_csr(time) +#define rdcycle() read_csr(cycle) +#define rdinstret() read_csr(instret) + +#endif + +#endif + +#endif + +#endif +/* Automatically generated by parse-opcodes */ +#ifndef RISCV_ENCODING_H +#define RISCV_ENCODING_H +#define MATCH_BEQ 0x63 +#define MASK_BEQ 0x707f +#define MATCH_BNE 0x1063 +#define MASK_BNE 0x707f +#define MATCH_BLT 0x4063 +#define MASK_BLT 0x707f +#define MATCH_BGE 0x5063 +#define MASK_BGE 0x707f +#define MATCH_BLTU 0x6063 +#define MASK_BLTU 0x707f +#define MATCH_BGEU 0x7063 +#define MASK_BGEU 0x707f +#define MATCH_JALR 0x67 +#define MASK_JALR 0x707f +#define MATCH_JAL 0x6f +#define MASK_JAL 0x7f +#define MATCH_LUI 0x37 +#define MASK_LUI 0x7f +#define MATCH_AUIPC 0x17 +#define MASK_AUIPC 0x7f +#define MATCH_ADDI 0x13 +#define MASK_ADDI 0x707f +#define MATCH_SLLI 0x1013 +#define MASK_SLLI 0xfc00707f +#define MATCH_SLTI 0x2013 +#define MASK_SLTI 0x707f +#define MATCH_SLTIU 0x3013 +#define MASK_SLTIU 0x707f +#define MATCH_XORI 0x4013 +#define MASK_XORI 0x707f +#define MATCH_SRLI 0x5013 +#define MASK_SRLI 0xfc00707f +#define MATCH_SRAI 0x40005013 +#define MASK_SRAI 0xfc00707f +#define MATCH_ORI 0x6013 +#define MASK_ORI 0x707f +#define MATCH_ANDI 0x7013 +#define MASK_ANDI 0x707f +#define MATCH_ADD 0x33 +#define MASK_ADD 0xfe00707f +#define MATCH_SUB 0x40000033 +#define MASK_SUB 0xfe00707f +#define MATCH_SLL 0x1033 +#define MASK_SLL 0xfe00707f +#define MATCH_SLT 0x2033 +#define MASK_SLT 0xfe00707f +#define MATCH_SLTU 0x3033 +#define MASK_SLTU 0xfe00707f +#define MATCH_XOR 0x4033 +#define MASK_XOR 0xfe00707f +#define MATCH_SRL 0x5033 +#define MASK_SRL 0xfe00707f +#define MATCH_SRA 0x40005033 +#define MASK_SRA 0xfe00707f +#define MATCH_OR 0x6033 +#define MASK_OR 0xfe00707f +#define MATCH_AND 0x7033 +#define MASK_AND 0xfe00707f +#define MATCH_ADDIW 0x1b +#define MASK_ADDIW 0x707f +#define MATCH_SLLIW 0x101b +#define MASK_SLLIW 0xfe00707f +#define MATCH_SRLIW 0x501b +#define MASK_SRLIW 0xfe00707f +#define MATCH_SRAIW 0x4000501b +#define MASK_SRAIW 0xfe00707f +#define MATCH_ADDW 0x3b +#define MASK_ADDW 0xfe00707f +#define MATCH_SUBW 0x4000003b +#define MASK_SUBW 0xfe00707f +#define MATCH_SLLW 0x103b +#define MASK_SLLW 0xfe00707f +#define MATCH_SRLW 0x503b +#define MASK_SRLW 0xfe00707f +#define MATCH_SRAW 0x4000503b +#define MASK_SRAW 0xfe00707f +#define MATCH_LB 0x3 +#define MASK_LB 0x707f +#define MATCH_LH 0x1003 +#define MASK_LH 0x707f +#define MATCH_LW 0x2003 +#define MASK_LW 0x707f +#define MATCH_LD 0x3003 +#define MASK_LD 0x707f +#define MATCH_LBU 0x4003 +#define MASK_LBU 0x707f +#define MATCH_LHU 0x5003 +#define MASK_LHU 0x707f +#define MATCH_LWU 0x6003 +#define MASK_LWU 0x707f +#define MATCH_SB 0x23 +#define MASK_SB 0x707f +#define MATCH_SH 0x1023 +#define MASK_SH 0x707f +#define MATCH_SW 0x2023 +#define MASK_SW 0x707f +#define MATCH_SD 0x3023 +#define MASK_SD 0x707f +#define MATCH_FENCE 0xf +#define MASK_FENCE 0x707f +#define MATCH_FENCE_I 0x100f +#define MASK_FENCE_I 0x707f +#define MATCH_MUL 0x2000033 +#define MASK_MUL 0xfe00707f +#define MATCH_MULH 0x2001033 +#define MASK_MULH 0xfe00707f +#define MATCH_MULHSU 0x2002033 +#define MASK_MULHSU 0xfe00707f +#define MATCH_MULHU 0x2003033 +#define MASK_MULHU 0xfe00707f +#define MATCH_DIV 0x2004033 +#define MASK_DIV 0xfe00707f +#define MATCH_DIVU 0x2005033 +#define MASK_DIVU 0xfe00707f +#define MATCH_REM 0x2006033 +#define MASK_REM 0xfe00707f +#define MATCH_REMU 0x2007033 +#define MASK_REMU 0xfe00707f +#define MATCH_MULW 0x200003b +#define MASK_MULW 0xfe00707f +#define MATCH_DIVW 0x200403b +#define MASK_DIVW 0xfe00707f +#define MATCH_DIVUW 0x200503b +#define MASK_DIVUW 0xfe00707f +#define MATCH_REMW 0x200603b +#define MASK_REMW 0xfe00707f +#define MATCH_REMUW 0x200703b +#define MASK_REMUW 0xfe00707f +#define MATCH_AMOADD_W 0x202f +#define MASK_AMOADD_W 0xf800707f +#define MATCH_AMOXOR_W 0x2000202f +#define MASK_AMOXOR_W 0xf800707f +#define MATCH_AMOOR_W 0x4000202f +#define MASK_AMOOR_W 0xf800707f +#define MATCH_AMOAND_W 0x6000202f +#define MASK_AMOAND_W 0xf800707f +#define MATCH_AMOMIN_W 0x8000202f +#define MASK_AMOMIN_W 0xf800707f +#define MATCH_AMOMAX_W 0xa000202f +#define MASK_AMOMAX_W 0xf800707f +#define MATCH_AMOMINU_W 0xc000202f +#define MASK_AMOMINU_W 0xf800707f +#define MATCH_AMOMAXU_W 0xe000202f +#define MASK_AMOMAXU_W 0xf800707f +#define MATCH_AMOSWAP_W 0x800202f +#define MASK_AMOSWAP_W 0xf800707f +#define MATCH_LR_W 0x1000202f +#define MASK_LR_W 0xf9f0707f +#define MATCH_SC_W 0x1800202f +#define MASK_SC_W 0xf800707f +#define MATCH_AMOADD_D 0x302f +#define MASK_AMOADD_D 0xf800707f +#define MATCH_AMOXOR_D 0x2000302f +#define MASK_AMOXOR_D 0xf800707f +#define MATCH_AMOOR_D 0x4000302f +#define MASK_AMOOR_D 0xf800707f +#define MATCH_AMOAND_D 0x6000302f +#define MASK_AMOAND_D 0xf800707f +#define MATCH_AMOMIN_D 0x8000302f +#define MASK_AMOMIN_D 0xf800707f +#define MATCH_AMOMAX_D 0xa000302f +#define MASK_AMOMAX_D 0xf800707f +#define MATCH_AMOMINU_D 0xc000302f +#define MASK_AMOMINU_D 0xf800707f +#define MATCH_AMOMAXU_D 0xe000302f +#define MASK_AMOMAXU_D 0xf800707f +#define MATCH_AMOSWAP_D 0x800302f +#define MASK_AMOSWAP_D 0xf800707f +#define MATCH_LR_D 0x1000302f +#define MASK_LR_D 0xf9f0707f +#define MATCH_SC_D 0x1800302f +#define MASK_SC_D 0xf800707f +#define MATCH_SCALL 0x73 +#define MASK_SCALL 0xffffffff +#define MATCH_SBREAK 0x100073 +#define MASK_SBREAK 0xffffffff +#define MATCH_SRET 0x10200073 +#define MASK_SRET 0xffffffff +#define MATCH_SFENCE_VM 0x10400073 +#define MASK_SFENCE_VM 0xfff07fff +#define MATCH_WFI 0x10500073 +#define MASK_WFI 0xffffffff +#define MATCH_CSRRW 0x1073 +#define MASK_CSRRW 0x707f +#define MATCH_CSRRS 0x2073 +#define MASK_CSRRS 0x707f +#define MATCH_CSRRC 0x3073 +#define MASK_CSRRC 0x707f +#define MATCH_CSRRWI 0x5073 +#define MASK_CSRRWI 0x707f +#define MATCH_CSRRSI 0x6073 +#define MASK_CSRRSI 0x707f +#define MATCH_CSRRCI 0x7073 +#define MASK_CSRRCI 0x707f +#define MATCH_FADD_S 0x53 +#define MASK_FADD_S 0xfe00007f +#define MATCH_FSUB_S 0x8000053 +#define MASK_FSUB_S 0xfe00007f +#define MATCH_FMUL_S 0x10000053 +#define MASK_FMUL_S 0xfe00007f +#define MATCH_FDIV_S 0x18000053 +#define MASK_FDIV_S 0xfe00007f +#define MATCH_FSGNJ_S 0x20000053 +#define MASK_FSGNJ_S 0xfe00707f +#define MATCH_FSGNJN_S 0x20001053 +#define MASK_FSGNJN_S 0xfe00707f +#define MATCH_FSGNJX_S 0x20002053 +#define MASK_FSGNJX_S 0xfe00707f +#define MATCH_FMIN_S 0x28000053 +#define MASK_FMIN_S 0xfe00707f +#define MATCH_FMAX_S 0x28001053 +#define MASK_FMAX_S 0xfe00707f +#define MATCH_FSQRT_S 0x58000053 +#define MASK_FSQRT_S 0xfff0007f +#define MATCH_FADD_D 0x2000053 +#define MASK_FADD_D 0xfe00007f +#define MATCH_FSUB_D 0xa000053 +#define MASK_FSUB_D 0xfe00007f +#define MATCH_FMUL_D 0x12000053 +#define MASK_FMUL_D 0xfe00007f +#define MATCH_FDIV_D 0x1a000053 +#define MASK_FDIV_D 0xfe00007f +#define MATCH_FSGNJ_D 0x22000053 +#define MASK_FSGNJ_D 0xfe00707f +#define MATCH_FSGNJN_D 0x22001053 +#define MASK_FSGNJN_D 0xfe00707f +#define MATCH_FSGNJX_D 0x22002053 +#define MASK_FSGNJX_D 0xfe00707f +#define MATCH_FMIN_D 0x2a000053 +#define MASK_FMIN_D 0xfe00707f +#define MATCH_FMAX_D 0x2a001053 +#define MASK_FMAX_D 0xfe00707f +#define MATCH_FCVT_S_D 0x40100053 +#define MASK_FCVT_S_D 0xfff0007f +#define MATCH_FCVT_D_S 0x42000053 +#define MASK_FCVT_D_S 0xfff0007f +#define MATCH_FSQRT_D 0x5a000053 +#define MASK_FSQRT_D 0xfff0007f +#define MATCH_FLE_S 0xa0000053 +#define MASK_FLE_S 0xfe00707f +#define MATCH_FLT_S 0xa0001053 +#define MASK_FLT_S 0xfe00707f +#define MATCH_FEQ_S 0xa0002053 +#define MASK_FEQ_S 0xfe00707f +#define MATCH_FLE_D 0xa2000053 +#define MASK_FLE_D 0xfe00707f +#define MATCH_FLT_D 0xa2001053 +#define MASK_FLT_D 0xfe00707f +#define MATCH_FEQ_D 0xa2002053 +#define MASK_FEQ_D 0xfe00707f +#define MATCH_FCVT_W_S 0xc0000053 +#define MASK_FCVT_W_S 0xfff0007f +#define MATCH_FCVT_WU_S 0xc0100053 +#define MASK_FCVT_WU_S 0xfff0007f +#define MATCH_FCVT_L_S 0xc0200053 +#define MASK_FCVT_L_S 0xfff0007f +#define MATCH_FCVT_LU_S 0xc0300053 +#define MASK_FCVT_LU_S 0xfff0007f +#define MATCH_FMV_X_S 0xe0000053 +#define MASK_FMV_X_S 0xfff0707f +#define MATCH_FCLASS_S 0xe0001053 +#define MASK_FCLASS_S 0xfff0707f +#define MATCH_FCVT_W_D 0xc2000053 +#define MASK_FCVT_W_D 0xfff0007f +#define MATCH_FCVT_WU_D 0xc2100053 +#define MASK_FCVT_WU_D 0xfff0007f +#define MATCH_FCVT_L_D 0xc2200053 +#define MASK_FCVT_L_D 0xfff0007f +#define MATCH_FCVT_LU_D 0xc2300053 +#define MASK_FCVT_LU_D 0xfff0007f +#define MATCH_FMV_X_D 0xe2000053 +#define MASK_FMV_X_D 0xfff0707f +#define MATCH_FCLASS_D 0xe2001053 +#define MASK_FCLASS_D 0xfff0707f +#define MATCH_FCVT_S_W 0xd0000053 +#define MASK_FCVT_S_W 0xfff0007f +#define MATCH_FCVT_S_WU 0xd0100053 +#define MASK_FCVT_S_WU 0xfff0007f +#define MATCH_FCVT_S_L 0xd0200053 +#define MASK_FCVT_S_L 0xfff0007f +#define MATCH_FCVT_S_LU 0xd0300053 +#define MASK_FCVT_S_LU 0xfff0007f +#define MATCH_FMV_S_X 0xf0000053 +#define MASK_FMV_S_X 0xfff0707f +#define MATCH_FCVT_D_W 0xd2000053 +#define MASK_FCVT_D_W 0xfff0007f +#define MATCH_FCVT_D_WU 0xd2100053 +#define MASK_FCVT_D_WU 0xfff0007f +#define MATCH_FCVT_D_L 0xd2200053 +#define MASK_FCVT_D_L 0xfff0007f +#define MATCH_FCVT_D_LU 0xd2300053 +#define MASK_FCVT_D_LU 0xfff0007f +#define MATCH_FMV_D_X 0xf2000053 +#define MASK_FMV_D_X 0xfff0707f +#define MATCH_FLW 0x2007 +#define MASK_FLW 0x707f +#define MATCH_FLD 0x3007 +#define MASK_FLD 0x707f +#define MATCH_FSW 0x2027 +#define MASK_FSW 0x707f +#define MATCH_FSD 0x3027 +#define MASK_FSD 0x707f +#define MATCH_FMADD_S 0x43 +#define MASK_FMADD_S 0x600007f +#define MATCH_FMSUB_S 0x47 +#define MASK_FMSUB_S 0x600007f +#define MATCH_FNMSUB_S 0x4b +#define MASK_FNMSUB_S 0x600007f +#define MATCH_FNMADD_S 0x4f +#define MASK_FNMADD_S 0x600007f +#define MATCH_FMADD_D 0x2000043 +#define MASK_FMADD_D 0x600007f +#define MATCH_FMSUB_D 0x2000047 +#define MASK_FMSUB_D 0x600007f +#define MATCH_FNMSUB_D 0x200004b +#define MASK_FNMSUB_D 0x600007f +#define MATCH_FNMADD_D 0x200004f +#define MASK_FNMADD_D 0x600007f +#define MATCH_C_NOP 0x1 +#define MASK_C_NOP 0xffff +#define MATCH_C_ADDI16SP 0x6101 +#define MASK_C_ADDI16SP 0xef83 +#define MATCH_C_JR 0x8002 +#define MASK_C_JR 0xf07f +#define MATCH_C_JALR 0x9002 +#define MASK_C_JALR 0xf07f +#define MATCH_C_EBREAK 0x9002 +#define MASK_C_EBREAK 0xffff +#define MATCH_C_LD 0x6000 +#define MASK_C_LD 0xe003 +#define MATCH_C_SD 0xe000 +#define MASK_C_SD 0xe003 +#define MATCH_C_ADDIW 0x2001 +#define MASK_C_ADDIW 0xe003 +#define MATCH_C_LDSP 0x6002 +#define MASK_C_LDSP 0xe003 +#define MATCH_C_SDSP 0xe002 +#define MASK_C_SDSP 0xe003 +#define MATCH_C_ADDI4SPN 0x0 +#define MASK_C_ADDI4SPN 0xe003 +#define MATCH_C_FLD 0x2000 +#define MASK_C_FLD 0xe003 +#define MATCH_C_LW 0x4000 +#define MASK_C_LW 0xe003 +#define MATCH_C_FLW 0x6000 +#define MASK_C_FLW 0xe003 +#define MATCH_C_FSD 0xa000 +#define MASK_C_FSD 0xe003 +#define MATCH_C_SW 0xc000 +#define MASK_C_SW 0xe003 +#define MATCH_C_FSW 0xe000 +#define MASK_C_FSW 0xe003 +#define MATCH_C_ADDI 0x1 +#define MASK_C_ADDI 0xe003 +#define MATCH_C_JAL 0x2001 +#define MASK_C_JAL 0xe003 +#define MATCH_C_LI 0x4001 +#define MASK_C_LI 0xe003 +#define MATCH_C_LUI 0x6001 +#define MASK_C_LUI 0xe003 +#define MATCH_C_SRLI 0x8001 +#define MASK_C_SRLI 0xec03 +#define MATCH_C_SRAI 0x8401 +#define MASK_C_SRAI 0xec03 +#define MATCH_C_ANDI 0x8801 +#define MASK_C_ANDI 0xec03 +#define MATCH_C_SUB 0x8c01 +#define MASK_C_SUB 0xfc63 +#define MATCH_C_XOR 0x8c21 +#define MASK_C_XOR 0xfc63 +#define MATCH_C_OR 0x8c41 +#define MASK_C_OR 0xfc63 +#define MATCH_C_AND 0x8c61 +#define MASK_C_AND 0xfc63 +#define MATCH_C_SUBW 0x9c01 +#define MASK_C_SUBW 0xfc63 +#define MATCH_C_ADDW 0x9c21 +#define MASK_C_ADDW 0xfc63 +#define MATCH_C_J 0xa001 +#define MASK_C_J 0xe003 +#define MATCH_C_BEQZ 0xc001 +#define MASK_C_BEQZ 0xe003 +#define MATCH_C_BNEZ 0xe001 +#define MASK_C_BNEZ 0xe003 +#define MATCH_C_SLLI 0x2 +#define MASK_C_SLLI 0xe003 +#define MATCH_C_FLDSP 0x2002 +#define MASK_C_FLDSP 0xe003 +#define MATCH_C_LWSP 0x4002 +#define MASK_C_LWSP 0xe003 +#define MATCH_C_FLWSP 0x6002 +#define MASK_C_FLWSP 0xe003 +#define MATCH_C_MV 0x8002 +#define MASK_C_MV 0xf003 +#define MATCH_C_ADD 0x9002 +#define MASK_C_ADD 0xf003 +#define MATCH_C_FSDSP 0xa002 +#define MASK_C_FSDSP 0xe003 +#define MATCH_C_SWSP 0xc002 +#define MASK_C_SWSP 0xe003 +#define MATCH_C_FSWSP 0xe002 +#define MASK_C_FSWSP 0xe003 +#define MATCH_CUSTOM0 0xb +#define MASK_CUSTOM0 0x707f +#define MATCH_CUSTOM0_RS1 0x200b +#define MASK_CUSTOM0_RS1 0x707f +#define MATCH_CUSTOM0_RS1_RS2 0x300b +#define MASK_CUSTOM0_RS1_RS2 0x707f +#define MATCH_CUSTOM0_RD 0x400b +#define MASK_CUSTOM0_RD 0x707f +#define MATCH_CUSTOM0_RD_RS1 0x600b +#define MASK_CUSTOM0_RD_RS1 0x707f +#define MATCH_CUSTOM0_RD_RS1_RS2 0x700b +#define MASK_CUSTOM0_RD_RS1_RS2 0x707f +#define MATCH_CUSTOM1 0x2b +#define MASK_CUSTOM1 0x707f +#define MATCH_CUSTOM1_RS1 0x202b +#define MASK_CUSTOM1_RS1 0x707f +#define MATCH_CUSTOM1_RS1_RS2 0x302b +#define MASK_CUSTOM1_RS1_RS2 0x707f +#define MATCH_CUSTOM1_RD 0x402b +#define MASK_CUSTOM1_RD 0x707f +#define MATCH_CUSTOM1_RD_RS1 0x602b +#define MASK_CUSTOM1_RD_RS1 0x707f +#define MATCH_CUSTOM1_RD_RS1_RS2 0x702b +#define MASK_CUSTOM1_RD_RS1_RS2 0x707f +#define MATCH_CUSTOM2 0x5b +#define MASK_CUSTOM2 0x707f +#define MATCH_CUSTOM2_RS1 0x205b +#define MASK_CUSTOM2_RS1 0x707f +#define MATCH_CUSTOM2_RS1_RS2 0x305b +#define MASK_CUSTOM2_RS1_RS2 0x707f +#define MATCH_CUSTOM2_RD 0x405b +#define MASK_CUSTOM2_RD 0x707f +#define MATCH_CUSTOM2_RD_RS1 0x605b +#define MASK_CUSTOM2_RD_RS1 0x707f +#define MATCH_CUSTOM2_RD_RS1_RS2 0x705b +#define MASK_CUSTOM2_RD_RS1_RS2 0x707f +#define MATCH_CUSTOM3 0x7b +#define MASK_CUSTOM3 0x707f +#define MATCH_CUSTOM3_RS1 0x207b +#define MASK_CUSTOM3_RS1 0x707f +#define MATCH_CUSTOM3_RS1_RS2 0x307b +#define MASK_CUSTOM3_RS1_RS2 0x707f +#define MATCH_CUSTOM3_RD 0x407b +#define MASK_CUSTOM3_RD 0x707f +#define MATCH_CUSTOM3_RD_RS1 0x607b +#define MASK_CUSTOM3_RD_RS1 0x707f +#define MATCH_CUSTOM3_RD_RS1_RS2 0x707b +#define MASK_CUSTOM3_RD_RS1_RS2 0x707f +#define CSR_FFLAGS 0x1 +#define CSR_FRM 0x2 +#define CSR_FCSR 0x3 +#define CSR_CYCLE 0xc00 +#define CSR_TIME 0xc01 +#define CSR_INSTRET 0xc02 +#define CSR_STATS 0xc0 +#define CSR_UARCH0 0xcc0 +#define CSR_UARCH1 0xcc1 +#define CSR_UARCH2 0xcc2 +#define CSR_UARCH3 0xcc3 +#define CSR_UARCH4 0xcc4 +#define CSR_UARCH5 0xcc5 +#define CSR_UARCH6 0xcc6 +#define CSR_UARCH7 0xcc7 +#define CSR_UARCH8 0xcc8 +#define CSR_UARCH9 0xcc9 +#define CSR_UARCH10 0xcca +#define CSR_UARCH11 0xccb +#define CSR_UARCH12 0xccc +#define CSR_UARCH13 0xccd +#define CSR_UARCH14 0xcce +#define CSR_UARCH15 0xccf +#define CSR_SSTATUS 0x100 +#define CSR_SIE 0x104 +#define CSR_STVEC 0x105 +#define CSR_SSCRATCH 0x140 +#define CSR_SEPC 0x141 +#define CSR_SCAUSE 0x142 +#define CSR_SBADADDR 0x143 +#define CSR_SIP 0x144 +#define CSR_SPTBR 0x180 +#define CSR_SASID 0x181 +#define CSR_SCYCLE 0xd00 +#define CSR_STIME 0xd01 +#define CSR_SINSTRET 0xd02 +#define CSR_MSTATUS 0x300 +#define CSR_MEDELEG 0x302 +#define CSR_MIDELEG 0x303 +#define CSR_MIE 0x304 +#define CSR_MTVEC 0x305 +#define CSR_MTIMECMP 0x321 +#define CSR_MSCRATCH 0x340 +#define CSR_MEPC 0x341 +#define CSR_MCAUSE 0x342 +#define CSR_MBADADDR 0x343 +#define CSR_MIP 0x344 +#define CSR_MIPI 0x345 +#define CSR_MUCOUNTEREN 0x310 +#define CSR_MSCOUNTEREN 0x311 +#define CSR_MUCYCLE_DELTA 0x700 +#define CSR_MUTIME_DELTA 0x701 +#define CSR_MUINSTRET_DELTA 0x702 +#define CSR_MSCYCLE_DELTA 0x704 +#define CSR_MSTIME_DELTA 0x705 +#define CSR_MSINSTRET_DELTA 0x706 +#define CSR_MCYCLE 0xf00 +#define CSR_MTIME 0xf01 +#define CSR_MINSTRET 0xf02 +#define CSR_MISA 0xf10 +#define CSR_MVENDORID 0xf11 +#define CSR_MARCHID 0xf12 +#define CSR_MIMPID 0xf13 +#define CSR_MCFGADDR 0xf14 +#define CSR_MHARTID 0xf15 +#define CSR_MTOHOST 0x7c0 +#define CSR_MFROMHOST 0x7c1 +#define CSR_MRESET 0x7c2 +#define CSR_CYCLEH 0xc80 +#define CSR_TIMEH 0xc81 +#define CSR_INSTRETH 0xc82 +#define CSR_MTIMECMPH 0x361 +#define CSR_MUCYCLE_DELTAH 0x780 +#define CSR_MUTIME_DELTAH 0x781 +#define CSR_MUINSTRET_DELTAH 0x782 +#define CSR_MSCYCLE_DELTAH 0x784 +#define CSR_MSTIME_DELTAH 0x785 +#define CSR_MSINSTRET_DELTAH 0x786 +#define CSR_MCYCLEH 0xf80 +#define CSR_MTIMEH 0xf81 +#define CSR_MINSTRETH 0xf82 +#define CAUSE_MISALIGNED_FETCH 0x0 +#define CAUSE_FAULT_FETCH 0x1 +#define CAUSE_ILLEGAL_INSTRUCTION 0x2 +#define CAUSE_BREAKPOINT 0x3 +#define CAUSE_MISALIGNED_LOAD 0x4 +#define CAUSE_FAULT_LOAD 0x5 +#define CAUSE_MISALIGNED_STORE 0x6 +#define CAUSE_FAULT_STORE 0x7 +#define CAUSE_USER_ECALL 0x8 +#define CAUSE_SUPERVISOR_ECALL 0x9 +#define CAUSE_HYPERVISOR_ECALL 0xa +#define CAUSE_MACHINE_ECALL 0xb +#endif +#ifdef DECLARE_INSN +DECLARE_INSN(beq, MATCH_BEQ, MASK_BEQ) +DECLARE_INSN(bne, MATCH_BNE, MASK_BNE) +DECLARE_INSN(blt, MATCH_BLT, MASK_BLT) +DECLARE_INSN(bge, MATCH_BGE, MASK_BGE) +DECLARE_INSN(bltu, MATCH_BLTU, MASK_BLTU) +DECLARE_INSN(bgeu, MATCH_BGEU, MASK_BGEU) +DECLARE_INSN(jalr, MATCH_JALR, MASK_JALR) +DECLARE_INSN(jal, MATCH_JAL, MASK_JAL) +DECLARE_INSN(lui, MATCH_LUI, MASK_LUI) +DECLARE_INSN(auipc, MATCH_AUIPC, MASK_AUIPC) +DECLARE_INSN(addi, MATCH_ADDI, MASK_ADDI) +DECLARE_INSN(slli, MATCH_SLLI, MASK_SLLI) +DECLARE_INSN(slti, MATCH_SLTI, MASK_SLTI) +DECLARE_INSN(sltiu, MATCH_SLTIU, MASK_SLTIU) +DECLARE_INSN(xori, MATCH_XORI, MASK_XORI) +DECLARE_INSN(srli, MATCH_SRLI, MASK_SRLI) +DECLARE_INSN(srai, MATCH_SRAI, MASK_SRAI) +DECLARE_INSN(ori, MATCH_ORI, MASK_ORI) +DECLARE_INSN(andi, MATCH_ANDI, MASK_ANDI) +DECLARE_INSN(add, MATCH_ADD, MASK_ADD) +DECLARE_INSN(sub, MATCH_SUB, MASK_SUB) +DECLARE_INSN(sll, MATCH_SLL, MASK_SLL) +DECLARE_INSN(slt, MATCH_SLT, MASK_SLT) +DECLARE_INSN(sltu, MATCH_SLTU, MASK_SLTU) +DECLARE_INSN(xor, MATCH_XOR, MASK_XOR) +DECLARE_INSN(srl, MATCH_SRL, MASK_SRL) +DECLARE_INSN(sra, MATCH_SRA, MASK_SRA) +DECLARE_INSN(or, MATCH_OR, MASK_OR) +DECLARE_INSN(and, MATCH_AND, MASK_AND) +DECLARE_INSN(addiw, MATCH_ADDIW, MASK_ADDIW) +DECLARE_INSN(slliw, MATCH_SLLIW, MASK_SLLIW) +DECLARE_INSN(srliw, MATCH_SRLIW, MASK_SRLIW) +DECLARE_INSN(sraiw, MATCH_SRAIW, MASK_SRAIW) +DECLARE_INSN(addw, MATCH_ADDW, MASK_ADDW) +DECLARE_INSN(subw, MATCH_SUBW, MASK_SUBW) +DECLARE_INSN(sllw, MATCH_SLLW, MASK_SLLW) +DECLARE_INSN(srlw, MATCH_SRLW, MASK_SRLW) +DECLARE_INSN(sraw, MATCH_SRAW, MASK_SRAW) +DECLARE_INSN(lb, MATCH_LB, MASK_LB) +DECLARE_INSN(lh, MATCH_LH, MASK_LH) +DECLARE_INSN(lw, MATCH_LW, MASK_LW) +DECLARE_INSN(ld, MATCH_LD, MASK_LD) +DECLARE_INSN(lbu, MATCH_LBU, MASK_LBU) +DECLARE_INSN(lhu, MATCH_LHU, MASK_LHU) +DECLARE_INSN(lwu, MATCH_LWU, MASK_LWU) +DECLARE_INSN(sb, MATCH_SB, MASK_SB) +DECLARE_INSN(sh, MATCH_SH, MASK_SH) +DECLARE_INSN(sw, MATCH_SW, MASK_SW) +DECLARE_INSN(sd, MATCH_SD, MASK_SD) +DECLARE_INSN(fence, MATCH_FENCE, MASK_FENCE) +DECLARE_INSN(fence_i, MATCH_FENCE_I, MASK_FENCE_I) +DECLARE_INSN(mul, MATCH_MUL, MASK_MUL) +DECLARE_INSN(mulh, MATCH_MULH, MASK_MULH) +DECLARE_INSN(mulhsu, MATCH_MULHSU, MASK_MULHSU) +DECLARE_INSN(mulhu, MATCH_MULHU, MASK_MULHU) +DECLARE_INSN(div, MATCH_DIV, MASK_DIV) +DECLARE_INSN(divu, MATCH_DIVU, MASK_DIVU) +DECLARE_INSN(rem, MATCH_REM, MASK_REM) +DECLARE_INSN(remu, MATCH_REMU, MASK_REMU) +DECLARE_INSN(mulw, MATCH_MULW, MASK_MULW) +DECLARE_INSN(divw, MATCH_DIVW, MASK_DIVW) +DECLARE_INSN(divuw, MATCH_DIVUW, MASK_DIVUW) +DECLARE_INSN(remw, MATCH_REMW, MASK_REMW) +DECLARE_INSN(remuw, MATCH_REMUW, MASK_REMUW) +DECLARE_INSN(amoadd_w, MATCH_AMOADD_W, MASK_AMOADD_W) +DECLARE_INSN(amoxor_w, MATCH_AMOXOR_W, MASK_AMOXOR_W) +DECLARE_INSN(amoor_w, MATCH_AMOOR_W, MASK_AMOOR_W) +DECLARE_INSN(amoand_w, MATCH_AMOAND_W, MASK_AMOAND_W) +DECLARE_INSN(amomin_w, MATCH_AMOMIN_W, MASK_AMOMIN_W) +DECLARE_INSN(amomax_w, MATCH_AMOMAX_W, MASK_AMOMAX_W) +DECLARE_INSN(amominu_w, MATCH_AMOMINU_W, MASK_AMOMINU_W) +DECLARE_INSN(amomaxu_w, MATCH_AMOMAXU_W, MASK_AMOMAXU_W) +DECLARE_INSN(amoswap_w, MATCH_AMOSWAP_W, MASK_AMOSWAP_W) +DECLARE_INSN(lr_w, MATCH_LR_W, MASK_LR_W) +DECLARE_INSN(sc_w, MATCH_SC_W, MASK_SC_W) +DECLARE_INSN(amoadd_d, MATCH_AMOADD_D, MASK_AMOADD_D) +DECLARE_INSN(amoxor_d, MATCH_AMOXOR_D, MASK_AMOXOR_D) +DECLARE_INSN(amoor_d, MATCH_AMOOR_D, MASK_AMOOR_D) +DECLARE_INSN(amoand_d, MATCH_AMOAND_D, MASK_AMOAND_D) +DECLARE_INSN(amomin_d, MATCH_AMOMIN_D, MASK_AMOMIN_D) +DECLARE_INSN(amomax_d, MATCH_AMOMAX_D, MASK_AMOMAX_D) +DECLARE_INSN(amominu_d, MATCH_AMOMINU_D, MASK_AMOMINU_D) +DECLARE_INSN(amomaxu_d, MATCH_AMOMAXU_D, MASK_AMOMAXU_D) +DECLARE_INSN(amoswap_d, MATCH_AMOSWAP_D, MASK_AMOSWAP_D) +DECLARE_INSN(lr_d, MATCH_LR_D, MASK_LR_D) +DECLARE_INSN(sc_d, MATCH_SC_D, MASK_SC_D) +DECLARE_INSN(scall, MATCH_SCALL, MASK_SCALL) +DECLARE_INSN(sbreak, MATCH_SBREAK, MASK_SBREAK) +DECLARE_INSN(sret, MATCH_SRET, MASK_SRET) +DECLARE_INSN(sfence_vm, MATCH_SFENCE_VM, MASK_SFENCE_VM) +DECLARE_INSN(wfi, MATCH_WFI, MASK_WFI) +DECLARE_INSN(csrrw, MATCH_CSRRW, MASK_CSRRW) +DECLARE_INSN(csrrs, MATCH_CSRRS, MASK_CSRRS) +DECLARE_INSN(csrrc, MATCH_CSRRC, MASK_CSRRC) +DECLARE_INSN(csrrwi, MATCH_CSRRWI, MASK_CSRRWI) +DECLARE_INSN(csrrsi, MATCH_CSRRSI, MASK_CSRRSI) +DECLARE_INSN(csrrci, MATCH_CSRRCI, MASK_CSRRCI) +DECLARE_INSN(fadd_s, MATCH_FADD_S, MASK_FADD_S) +DECLARE_INSN(fsub_s, MATCH_FSUB_S, MASK_FSUB_S) +DECLARE_INSN(fmul_s, MATCH_FMUL_S, MASK_FMUL_S) +DECLARE_INSN(fdiv_s, MATCH_FDIV_S, MASK_FDIV_S) +DECLARE_INSN(fsgnj_s, MATCH_FSGNJ_S, MASK_FSGNJ_S) +DECLARE_INSN(fsgnjn_s, MATCH_FSGNJN_S, MASK_FSGNJN_S) +DECLARE_INSN(fsgnjx_s, MATCH_FSGNJX_S, MASK_FSGNJX_S) +DECLARE_INSN(fmin_s, MATCH_FMIN_S, MASK_FMIN_S) +DECLARE_INSN(fmax_s, MATCH_FMAX_S, MASK_FMAX_S) +DECLARE_INSN(fsqrt_s, MATCH_FSQRT_S, MASK_FSQRT_S) +DECLARE_INSN(fadd_d, MATCH_FADD_D, MASK_FADD_D) +DECLARE_INSN(fsub_d, MATCH_FSUB_D, MASK_FSUB_D) +DECLARE_INSN(fmul_d, MATCH_FMUL_D, MASK_FMUL_D) +DECLARE_INSN(fdiv_d, MATCH_FDIV_D, MASK_FDIV_D) +DECLARE_INSN(fsgnj_d, MATCH_FSGNJ_D, MASK_FSGNJ_D) +DECLARE_INSN(fsgnjn_d, MATCH_FSGNJN_D, MASK_FSGNJN_D) +DECLARE_INSN(fsgnjx_d, MATCH_FSGNJX_D, MASK_FSGNJX_D) +DECLARE_INSN(fmin_d, MATCH_FMIN_D, MASK_FMIN_D) +DECLARE_INSN(fmax_d, MATCH_FMAX_D, MASK_FMAX_D) +DECLARE_INSN(fcvt_s_d, MATCH_FCVT_S_D, MASK_FCVT_S_D) +DECLARE_INSN(fcvt_d_s, MATCH_FCVT_D_S, MASK_FCVT_D_S) +DECLARE_INSN(fsqrt_d, MATCH_FSQRT_D, MASK_FSQRT_D) +DECLARE_INSN(fle_s, MATCH_FLE_S, MASK_FLE_S) +DECLARE_INSN(flt_s, MATCH_FLT_S, MASK_FLT_S) +DECLARE_INSN(feq_s, MATCH_FEQ_S, MASK_FEQ_S) +DECLARE_INSN(fle_d, MATCH_FLE_D, MASK_FLE_D) +DECLARE_INSN(flt_d, MATCH_FLT_D, MASK_FLT_D) +DECLARE_INSN(feq_d, MATCH_FEQ_D, MASK_FEQ_D) +DECLARE_INSN(fcvt_w_s, MATCH_FCVT_W_S, MASK_FCVT_W_S) +DECLARE_INSN(fcvt_wu_s, MATCH_FCVT_WU_S, MASK_FCVT_WU_S) +DECLARE_INSN(fcvt_l_s, MATCH_FCVT_L_S, MASK_FCVT_L_S) +DECLARE_INSN(fcvt_lu_s, MATCH_FCVT_LU_S, MASK_FCVT_LU_S) +DECLARE_INSN(fmv_x_s, MATCH_FMV_X_S, MASK_FMV_X_S) +DECLARE_INSN(fclass_s, MATCH_FCLASS_S, MASK_FCLASS_S) +DECLARE_INSN(fcvt_w_d, MATCH_FCVT_W_D, MASK_FCVT_W_D) +DECLARE_INSN(fcvt_wu_d, MATCH_FCVT_WU_D, MASK_FCVT_WU_D) +DECLARE_INSN(fcvt_l_d, MATCH_FCVT_L_D, MASK_FCVT_L_D) +DECLARE_INSN(fcvt_lu_d, MATCH_FCVT_LU_D, MASK_FCVT_LU_D) +DECLARE_INSN(fmv_x_d, MATCH_FMV_X_D, MASK_FMV_X_D) +DECLARE_INSN(fclass_d, MATCH_FCLASS_D, MASK_FCLASS_D) +DECLARE_INSN(fcvt_s_w, MATCH_FCVT_S_W, MASK_FCVT_S_W) +DECLARE_INSN(fcvt_s_wu, MATCH_FCVT_S_WU, MASK_FCVT_S_WU) +DECLARE_INSN(fcvt_s_l, MATCH_FCVT_S_L, MASK_FCVT_S_L) +DECLARE_INSN(fcvt_s_lu, MATCH_FCVT_S_LU, MASK_FCVT_S_LU) +DECLARE_INSN(fmv_s_x, MATCH_FMV_S_X, MASK_FMV_S_X) +DECLARE_INSN(fcvt_d_w, MATCH_FCVT_D_W, MASK_FCVT_D_W) +DECLARE_INSN(fcvt_d_wu, MATCH_FCVT_D_WU, MASK_FCVT_D_WU) +DECLARE_INSN(fcvt_d_l, MATCH_FCVT_D_L, MASK_FCVT_D_L) +DECLARE_INSN(fcvt_d_lu, MATCH_FCVT_D_LU, MASK_FCVT_D_LU) +DECLARE_INSN(fmv_d_x, MATCH_FMV_D_X, MASK_FMV_D_X) +DECLARE_INSN(flw, MATCH_FLW, MASK_FLW) +DECLARE_INSN(fld, MATCH_FLD, MASK_FLD) +DECLARE_INSN(fsw, MATCH_FSW, MASK_FSW) +DECLARE_INSN(fsd, MATCH_FSD, MASK_FSD) +DECLARE_INSN(fmadd_s, MATCH_FMADD_S, MASK_FMADD_S) +DECLARE_INSN(fmsub_s, MATCH_FMSUB_S, MASK_FMSUB_S) +DECLARE_INSN(fnmsub_s, MATCH_FNMSUB_S, MASK_FNMSUB_S) +DECLARE_INSN(fnmadd_s, MATCH_FNMADD_S, MASK_FNMADD_S) +DECLARE_INSN(fmadd_d, MATCH_FMADD_D, MASK_FMADD_D) +DECLARE_INSN(fmsub_d, MATCH_FMSUB_D, MASK_FMSUB_D) +DECLARE_INSN(fnmsub_d, MATCH_FNMSUB_D, MASK_FNMSUB_D) +DECLARE_INSN(fnmadd_d, MATCH_FNMADD_D, MASK_FNMADD_D) +DECLARE_INSN(c_nop, MATCH_C_NOP, MASK_C_NOP) +DECLARE_INSN(c_addi16sp, MATCH_C_ADDI16SP, MASK_C_ADDI16SP) +DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR) +DECLARE_INSN(c_jalr, MATCH_C_JALR, MASK_C_JALR) +DECLARE_INSN(c_ebreak, MATCH_C_EBREAK, MASK_C_EBREAK) +DECLARE_INSN(c_ld, MATCH_C_LD, MASK_C_LD) +DECLARE_INSN(c_sd, MATCH_C_SD, MASK_C_SD) +DECLARE_INSN(c_addiw, MATCH_C_ADDIW, MASK_C_ADDIW) +DECLARE_INSN(c_ldsp, MATCH_C_LDSP, MASK_C_LDSP) +DECLARE_INSN(c_sdsp, MATCH_C_SDSP, MASK_C_SDSP) +DECLARE_INSN(c_addi4spn, MATCH_C_ADDI4SPN, MASK_C_ADDI4SPN) +DECLARE_INSN(c_fld, MATCH_C_FLD, MASK_C_FLD) +DECLARE_INSN(c_lw, MATCH_C_LW, MASK_C_LW) +DECLARE_INSN(c_flw, MATCH_C_FLW, MASK_C_FLW) +DECLARE_INSN(c_fsd, MATCH_C_FSD, MASK_C_FSD) +DECLARE_INSN(c_sw, MATCH_C_SW, MASK_C_SW) +DECLARE_INSN(c_fsw, MATCH_C_FSW, MASK_C_FSW) +DECLARE_INSN(c_addi, MATCH_C_ADDI, MASK_C_ADDI) +DECLARE_INSN(c_jal, MATCH_C_JAL, MASK_C_JAL) +DECLARE_INSN(c_li, MATCH_C_LI, MASK_C_LI) +DECLARE_INSN(c_lui, MATCH_C_LUI, MASK_C_LUI) +DECLARE_INSN(c_srli, MATCH_C_SRLI, MASK_C_SRLI) +DECLARE_INSN(c_srai, MATCH_C_SRAI, MASK_C_SRAI) +DECLARE_INSN(c_andi, MATCH_C_ANDI, MASK_C_ANDI) +DECLARE_INSN(c_sub, MATCH_C_SUB, MASK_C_SUB) +DECLARE_INSN(c_xor, MATCH_C_XOR, MASK_C_XOR) +DECLARE_INSN(c_or, MATCH_C_OR, MASK_C_OR) +DECLARE_INSN(c_and, MATCH_C_AND, MASK_C_AND) +DECLARE_INSN(c_subw, MATCH_C_SUBW, MASK_C_SUBW) +DECLARE_INSN(c_addw, MATCH_C_ADDW, MASK_C_ADDW) +DECLARE_INSN(c_j, MATCH_C_J, MASK_C_J) +DECLARE_INSN(c_beqz, MATCH_C_BEQZ, MASK_C_BEQZ) +DECLARE_INSN(c_bnez, MATCH_C_BNEZ, MASK_C_BNEZ) +DECLARE_INSN(c_slli, MATCH_C_SLLI, MASK_C_SLLI) +DECLARE_INSN(c_fldsp, MATCH_C_FLDSP, MASK_C_FLDSP) +DECLARE_INSN(c_lwsp, MATCH_C_LWSP, MASK_C_LWSP) +DECLARE_INSN(c_flwsp, MATCH_C_FLWSP, MASK_C_FLWSP) +DECLARE_INSN(c_mv, MATCH_C_MV, MASK_C_MV) +DECLARE_INSN(c_add, MATCH_C_ADD, MASK_C_ADD) +DECLARE_INSN(c_fsdsp, MATCH_C_FSDSP, MASK_C_FSDSP) +DECLARE_INSN(c_swsp, MATCH_C_SWSP, MASK_C_SWSP) +DECLARE_INSN(c_fswsp, MATCH_C_FSWSP, MASK_C_FSWSP) +DECLARE_INSN(custom0, MATCH_CUSTOM0, MASK_CUSTOM0) +DECLARE_INSN(custom0_rs1, MATCH_CUSTOM0_RS1, MASK_CUSTOM0_RS1) +DECLARE_INSN(custom0_rs1_rs2, MATCH_CUSTOM0_RS1_RS2, MASK_CUSTOM0_RS1_RS2) +DECLARE_INSN(custom0_rd, MATCH_CUSTOM0_RD, MASK_CUSTOM0_RD) +DECLARE_INSN(custom0_rd_rs1, MATCH_CUSTOM0_RD_RS1, MASK_CUSTOM0_RD_RS1) +DECLARE_INSN(custom0_rd_rs1_rs2, MATCH_CUSTOM0_RD_RS1_RS2, MASK_CUSTOM0_RD_RS1_RS2) +DECLARE_INSN(custom1, MATCH_CUSTOM1, MASK_CUSTOM1) +DECLARE_INSN(custom1_rs1, MATCH_CUSTOM1_RS1, MASK_CUSTOM1_RS1) +DECLARE_INSN(custom1_rs1_rs2, MATCH_CUSTOM1_RS1_RS2, MASK_CUSTOM1_RS1_RS2) +DECLARE_INSN(custom1_rd, MATCH_CUSTOM1_RD, MASK_CUSTOM1_RD) +DECLARE_INSN(custom1_rd_rs1, MATCH_CUSTOM1_RD_RS1, MASK_CUSTOM1_RD_RS1) +DECLARE_INSN(custom1_rd_rs1_rs2, MATCH_CUSTOM1_RD_RS1_RS2, MASK_CUSTOM1_RD_RS1_RS2) +DECLARE_INSN(custom2, MATCH_CUSTOM2, MASK_CUSTOM2) +DECLARE_INSN(custom2_rs1, MATCH_CUSTOM2_RS1, MASK_CUSTOM2_RS1) +DECLARE_INSN(custom2_rs1_rs2, MATCH_CUSTOM2_RS1_RS2, MASK_CUSTOM2_RS1_RS2) +DECLARE_INSN(custom2_rd, MATCH_CUSTOM2_RD, MASK_CUSTOM2_RD) +DECLARE_INSN(custom2_rd_rs1, MATCH_CUSTOM2_RD_RS1, MASK_CUSTOM2_RD_RS1) +DECLARE_INSN(custom2_rd_rs1_rs2, MATCH_CUSTOM2_RD_RS1_RS2, MASK_CUSTOM2_RD_RS1_RS2) +DECLARE_INSN(custom3, MATCH_CUSTOM3, MASK_CUSTOM3) +DECLARE_INSN(custom3_rs1, MATCH_CUSTOM3_RS1, MASK_CUSTOM3_RS1) +DECLARE_INSN(custom3_rs1_rs2, MATCH_CUSTOM3_RS1_RS2, MASK_CUSTOM3_RS1_RS2) +DECLARE_INSN(custom3_rd, MATCH_CUSTOM3_RD, MASK_CUSTOM3_RD) +DECLARE_INSN(custom3_rd_rs1, MATCH_CUSTOM3_RD_RS1, MASK_CUSTOM3_RD_RS1) +DECLARE_INSN(custom3_rd_rs1_rs2, MATCH_CUSTOM3_RD_RS1_RS2, MASK_CUSTOM3_RD_RS1_RS2) +#endif +#ifdef DECLARE_CSR +DECLARE_CSR(fflags, CSR_FFLAGS) +DECLARE_CSR(frm, CSR_FRM) +DECLARE_CSR(fcsr, CSR_FCSR) +DECLARE_CSR(cycle, CSR_CYCLE) +DECLARE_CSR(time, CSR_TIME) +DECLARE_CSR(instret, CSR_INSTRET) +DECLARE_CSR(stats, CSR_STATS) +DECLARE_CSR(uarch0, CSR_UARCH0) +DECLARE_CSR(uarch1, CSR_UARCH1) +DECLARE_CSR(uarch2, CSR_UARCH2) +DECLARE_CSR(uarch3, CSR_UARCH3) +DECLARE_CSR(uarch4, CSR_UARCH4) +DECLARE_CSR(uarch5, CSR_UARCH5) +DECLARE_CSR(uarch6, CSR_UARCH6) +DECLARE_CSR(uarch7, CSR_UARCH7) +DECLARE_CSR(uarch8, CSR_UARCH8) +DECLARE_CSR(uarch9, CSR_UARCH9) +DECLARE_CSR(uarch10, CSR_UARCH10) +DECLARE_CSR(uarch11, CSR_UARCH11) +DECLARE_CSR(uarch12, CSR_UARCH12) +DECLARE_CSR(uarch13, CSR_UARCH13) +DECLARE_CSR(uarch14, CSR_UARCH14) +DECLARE_CSR(uarch15, CSR_UARCH15) +DECLARE_CSR(sstatus, CSR_SSTATUS) +DECLARE_CSR(sie, CSR_SIE) +DECLARE_CSR(stvec, CSR_STVEC) +DECLARE_CSR(sscratch, CSR_SSCRATCH) +DECLARE_CSR(sepc, CSR_SEPC) +DECLARE_CSR(scause, CSR_SCAUSE) +DECLARE_CSR(sbadaddr, CSR_SBADADDR) +DECLARE_CSR(sip, CSR_SIP) +DECLARE_CSR(sptbr, CSR_SPTBR) +DECLARE_CSR(sasid, CSR_SASID) +DECLARE_CSR(scycle, CSR_SCYCLE) +DECLARE_CSR(stime, CSR_STIME) +DECLARE_CSR(sinstret, CSR_SINSTRET) +DECLARE_CSR(mstatus, CSR_MSTATUS) +DECLARE_CSR(medeleg, CSR_MEDELEG) +DECLARE_CSR(mideleg, CSR_MIDELEG) +DECLARE_CSR(mie, CSR_MIE) +DECLARE_CSR(mtvec, CSR_MTVEC) +DECLARE_CSR(mtimecmp, CSR_MTIMECMP) +DECLARE_CSR(mscratch, CSR_MSCRATCH) +DECLARE_CSR(mepc, CSR_MEPC) +DECLARE_CSR(mcause, CSR_MCAUSE) +DECLARE_CSR(mbadaddr, CSR_MBADADDR) +DECLARE_CSR(mip, CSR_MIP) +DECLARE_CSR(mipi, CSR_MIPI) +DECLARE_CSR(mucounteren, CSR_MUCOUNTEREN) +DECLARE_CSR(mscounteren, CSR_MSCOUNTEREN) +DECLARE_CSR(mucycle_delta, CSR_MUCYCLE_DELTA) +DECLARE_CSR(mutime_delta, CSR_MUTIME_DELTA) +DECLARE_CSR(muinstret_delta, CSR_MUINSTRET_DELTA) +DECLARE_CSR(mscycle_delta, CSR_MSCYCLE_DELTA) +DECLARE_CSR(mstime_delta, CSR_MSTIME_DELTA) +DECLARE_CSR(msinstret_delta, CSR_MSINSTRET_DELTA) +DECLARE_CSR(mcycle, CSR_MCYCLE) +DECLARE_CSR(mtime, CSR_MTIME) +DECLARE_CSR(minstret, CSR_MINSTRET) +DECLARE_CSR(misa, CSR_MISA) +DECLARE_CSR(mvendorid, CSR_MVENDORID) +DECLARE_CSR(marchid, CSR_MARCHID) +DECLARE_CSR(mimpid, CSR_MIMPID) +DECLARE_CSR(mcfgaddr, CSR_MCFGADDR) +DECLARE_CSR(mhartid, CSR_MHARTID) +DECLARE_CSR(mtohost, CSR_MTOHOST) +DECLARE_CSR(mfromhost, CSR_MFROMHOST) +DECLARE_CSR(mreset, CSR_MRESET) +DECLARE_CSR(cycleh, CSR_CYCLEH) +DECLARE_CSR(timeh, CSR_TIMEH) +DECLARE_CSR(instreth, CSR_INSTRETH) +DECLARE_CSR(mtimecmph, CSR_MTIMECMPH) +DECLARE_CSR(mucycle_deltah, CSR_MUCYCLE_DELTAH) +DECLARE_CSR(mutime_deltah, CSR_MUTIME_DELTAH) +DECLARE_CSR(muinstret_deltah, CSR_MUINSTRET_DELTAH) +DECLARE_CSR(mscycle_deltah, CSR_MSCYCLE_DELTAH) +DECLARE_CSR(mstime_deltah, CSR_MSTIME_DELTAH) +DECLARE_CSR(msinstret_deltah, CSR_MSINSTRET_DELTAH) +DECLARE_CSR(mcycleh, CSR_MCYCLEH) +DECLARE_CSR(mtimeh, CSR_MTIMEH) +DECLARE_CSR(minstreth, CSR_MINSTRETH) +#endif +#ifdef DECLARE_CAUSE +DECLARE_CAUSE("misaligned fetch", CAUSE_MISALIGNED_FETCH) +DECLARE_CAUSE("fault fetch", CAUSE_FAULT_FETCH) +DECLARE_CAUSE("illegal instruction", CAUSE_ILLEGAL_INSTRUCTION) +DECLARE_CAUSE("breakpoint", CAUSE_BREAKPOINT) +DECLARE_CAUSE("misaligned load", CAUSE_MISALIGNED_LOAD) +DECLARE_CAUSE("fault load", CAUSE_FAULT_LOAD) +DECLARE_CAUSE("misaligned store", CAUSE_MISALIGNED_STORE) +DECLARE_CAUSE("fault store", CAUSE_FAULT_STORE) +DECLARE_CAUSE("user_ecall", CAUSE_USER_ECALL) +DECLARE_CAUSE("supervisor_ecall", CAUSE_SUPERVISOR_ECALL) +DECLARE_CAUSE("hypervisor_ecall", CAUSE_HYPERVISOR_ECALL) +DECLARE_CAUSE("machine_ecall", CAUSE_MACHINE_ECALL) +#endif diff --git a/machine/fp_asm.S b/machine/fp_asm.S new file mode 100644 index 0000000..4b8dce1 --- /dev/null +++ b/machine/fp_asm.S @@ -0,0 +1,160 @@ +// See LICENSE for license details. + +#ifdef __riscv_hard_float + +#define get_f32(which) fmv.x.s a0, which; jr t0 +#define put_f32(which) fmv.s.x which, a0; jr t0 +#ifdef __riscv64 +# define get_f64(which) fmv.x.d a0, which; jr t0 +# define put_f64(which) fmv.d.x which, a0; jr t0 +#else +# define get_f64(which) fsd which, 0(a0); jr t0 +# define put_f64(which) fld which, 0(a0); jr t0 +#endif + + .text + .option norvc + .globl get_f32_reg + get_f32_reg: + get_f32(f0) + get_f32(f1) + get_f32(f2) + get_f32(f3) + get_f32(f4) + get_f32(f5) + get_f32(f6) + get_f32(f7) + get_f32(f8) + get_f32(f9) + get_f32(f10) + get_f32(f11) + get_f32(f12) + get_f32(f13) + get_f32(f14) + get_f32(f15) + get_f32(f16) + get_f32(f17) + get_f32(f18) + get_f32(f19) + get_f32(f20) + get_f32(f21) + get_f32(f22) + get_f32(f23) + get_f32(f24) + get_f32(f25) + get_f32(f26) + get_f32(f27) + get_f32(f28) + get_f32(f29) + get_f32(f30) + get_f32(f31) + + .text + .globl put_f32_reg + put_f32_reg: + put_f32(f0) + put_f32(f1) + put_f32(f2) + put_f32(f3) + put_f32(f4) + put_f32(f5) + put_f32(f6) + put_f32(f7) + put_f32(f8) + put_f32(f9) + put_f32(f10) + put_f32(f11) + put_f32(f12) + put_f32(f13) + put_f32(f14) + put_f32(f15) + put_f32(f16) + put_f32(f17) + put_f32(f18) + put_f32(f19) + put_f32(f20) + put_f32(f21) + put_f32(f22) + put_f32(f23) + put_f32(f24) + put_f32(f25) + put_f32(f26) + put_f32(f27) + put_f32(f28) + put_f32(f29) + put_f32(f30) + put_f32(f31) + + .text + .globl get_f64_reg + get_f64_reg: + get_f64(f0) + get_f64(f1) + get_f64(f2) + get_f64(f3) + get_f64(f4) + get_f64(f5) + get_f64(f6) + get_f64(f7) + get_f64(f8) + get_f64(f9) + get_f64(f10) + get_f64(f11) + get_f64(f12) + get_f64(f13) + get_f64(f14) + get_f64(f15) + get_f64(f16) + get_f64(f17) + get_f64(f18) + get_f64(f19) + get_f64(f20) + get_f64(f21) + get_f64(f22) + get_f64(f23) + get_f64(f24) + get_f64(f25) + get_f64(f26) + get_f64(f27) + get_f64(f28) + get_f64(f29) + get_f64(f30) + get_f64(f31) + + .text + .globl put_f64_reg + put_f64_reg: + put_f64(f0) + put_f64(f1) + put_f64(f2) + put_f64(f3) + put_f64(f4) + put_f64(f5) + put_f64(f6) + put_f64(f7) + put_f64(f8) + put_f64(f9) + put_f64(f10) + put_f64(f11) + put_f64(f12) + put_f64(f13) + put_f64(f14) + put_f64(f15) + put_f64(f16) + put_f64(f17) + put_f64(f18) + put_f64(f19) + put_f64(f20) + put_f64(f21) + put_f64(f22) + put_f64(f23) + put_f64(f24) + put_f64(f25) + put_f64(f26) + put_f64(f27) + put_f64(f28) + put_f64(f29) + put_f64(f30) + put_f64(f31) + +#endif diff --git a/machine/fp_emulation.c b/machine/fp_emulation.c new file mode 100644 index 0000000..536967f --- /dev/null +++ b/machine/fp_emulation.c @@ -0,0 +1,449 @@ +#include "fp_emulation.h" +#include "unprivileged_memory.h" +#include "softfloat.h" +#include "config.h" + +DECLARE_EMULATION_FUNC(emulate_float_load) +{ + uint64_t val; + uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); + + switch (insn & MASK_FUNCT3) + { + case MATCH_FLW & MASK_FUNCT3: + if (addr % 4 != 0) + return misaligned_load_trap(regs, mcause, mepc); + + SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc)); + break; + + case MATCH_FLD & MASK_FUNCT3: + if (addr % sizeof(uintptr_t) != 0) + return misaligned_load_trap(regs, mcause, mepc); + +#ifdef __riscv64 + val = load_uint64_t((void *)addr, mepc); +#else + val = load_uint32_t(addr, mepc); + val += (uint64_t)load_uint32_t((void *)(addr + 4), mepc) << 32; +#endif + SET_F64_RD(insn, regs, val); + break; + + default: + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_float_store) +{ + uint64_t val; + uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); + + switch (insn & MASK_FUNCT3) + { + case MATCH_FSW & MASK_FUNCT3: + if (addr % 4 != 0) + return misaligned_store_trap(regs, mcause, mepc); + + store_uint32_t((void *)addr, GET_F32_RS2(insn, regs), mepc); + break; + + case MATCH_FSD & MASK_FUNCT3: + if (addr % sizeof(uintptr_t) != 0) + return misaligned_store_trap(regs, mcause, mepc); + + val = GET_F64_RS2(insn, regs); +#ifdef __riscv64 + store_uint64_t((void *)addr, val, mepc); +#else + store_uint32_t((void *)addr, val, mepc); + store_uint32_t((void *)(addr + 4), val >> 32, mepc); +#endif + break; + + default: + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fp) +{ + asm (".pushsection .rodata\n" + "fp_emulation_table:\n" + " .word emulate_fadd\n" + " .word emulate_fsub\n" + " .word emulate_fmul\n" + " .word emulate_fdiv\n" + " .word emulate_fsgnj\n" + " .word emulate_fmin\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_fcvt_ff\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_fsqrt\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_fcmp\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word truly_illegal_insn\n" + " .word emulate_fcvt_if\n" + " .word truly_illegal_insn\n" + " .word emulate_fcvt_fi\n" + " .word truly_illegal_insn\n" + " .word emulate_fmv_if\n" + " .word truly_illegal_insn\n" + " .word emulate_fmv_fi\n" + " .word truly_illegal_insn\n" + " .popsection"); + + // if FPU is disabled, punt back to the OS + if (unlikely((mstatus & MSTATUS_FS) == 0)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + extern int32_t fp_emulation_table[]; + int32_t* pf = (void*)fp_emulation_table + ((insn >> 25) & 0x7c); + emulation_func f = (emulation_func)(uintptr_t)*pf; + + SETUP_STATIC_ROUNDING(insn); + return f(regs, mcause, mepc, mstatus, insn); +} + +void emulate_any_fadd(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn, int32_t neg_b) +{ + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs) ^ neg_b; + SET_F32_RD(insn, regs, f32_add(rs1, rs2)); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs) ^ ((uint64_t)neg_b << 32); + SET_F64_RD(insn, regs, f64_add(rs1, rs2)); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fadd) +{ + return emulate_any_fadd(regs, mcause, mepc, mstatus, insn, 0); +} + +DECLARE_EMULATION_FUNC(emulate_fsub) +{ + return emulate_any_fadd(regs, mcause, mepc, mstatus, insn, INT32_MIN); +} + +DECLARE_EMULATION_FUNC(emulate_fmul) +{ + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + SET_F32_RD(insn, regs, f32_mul(rs1, rs2)); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + SET_F64_RD(insn, regs, f64_mul(rs1, rs2)); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fdiv) +{ + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + SET_F32_RD(insn, regs, f32_div(rs1, rs2)); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + SET_F64_RD(insn, regs, f64_div(rs1, rs2)); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fsqrt) +{ + if ((insn >> 20) & 0x1f) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + if (GET_PRECISION(insn) == PRECISION_S) { + SET_F32_RD(insn, regs, f32_sqrt(GET_F32_RS1(insn, regs))); + } else if (GET_PRECISION(insn) == PRECISION_D) { + SET_F64_RD(insn, regs, f64_sqrt(GET_F64_RS1(insn, regs))); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fsgnj) +{ + int rm = GET_RM(insn); + if (rm >= 3) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + #define DO_FSGNJ(rs1, rs2, rm) ({ \ + typeof(rs1) rs1_sign = (rs1) >> (8*sizeof(rs1)-1); \ + typeof(rs1) rs2_sign = (rs2) >> (8*sizeof(rs1)-1); \ + rs1_sign &= (rm) >> 1; \ + rs1_sign ^= (rm) ^ rs2_sign; \ + ((rs1) << 1 >> 1) | (rs1_sign << (8*sizeof(rs1)-1)); }) + + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + SET_F32_RD(insn, regs, DO_FSGNJ(rs1, rs2, rm)); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + SET_F64_RD(insn, regs, DO_FSGNJ(rs1, rs2, rm)); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fmin) +{ + int rm = GET_RM(insn); + if (rm >= 2) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + uint32_t arg1 = rm ? rs2 : rs1; + uint32_t arg2 = rm ? rs1 : rs2; + int use_rs1 = f32_lt_quiet(arg1, arg2) || isNaNF32UI(rs2); + SET_F32_RD(insn, regs, use_rs1 ? rs1 : rs2); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + uint64_t arg1 = rm ? rs2 : rs1; + uint64_t arg2 = rm ? rs1 : rs2; + int use_rs1 = f64_lt_quiet(arg1, arg2) || isNaNF64UI(rs2); + SET_F64_RD(insn, regs, use_rs1 ? rs1 : rs2); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fcvt_ff) +{ + int rs2_num = (insn >> 20) & 0x1f; + if (GET_PRECISION(insn) == PRECISION_S) { + if (rs2_num != 1) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + SET_F32_RD(insn, regs, f64_to_f32(GET_F64_RS1(insn, regs))); + } else if (GET_PRECISION(insn) == PRECISION_D) { + if (rs2_num != 0) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + SET_F64_RD(insn, regs, f32_to_f64(GET_F32_RS1(insn, regs))); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} + +DECLARE_EMULATION_FUNC(emulate_fcvt_fi) +{ + if (GET_PRECISION(insn) != PRECISION_S && GET_PRECISION(insn) != PRECISION_D) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + int negative = 0; + uint64_t uint_val = GET_RS1(insn, regs); + + switch ((insn >> 20) & 0x1f) + { + case 0: // int32 + negative = (int32_t)uint_val < 0; + uint_val = negative ? -(int32_t)uint_val : (int32_t)uint_val; + break; + case 1: // uint32 + uint_val = (uint32_t)uint_val; + break; +#ifdef __riscv64 + case 2: // int64 + negative = (int64_t)uint_val < 0; + uint_val = negative ? -uint_val : uint_val; + case 3: // uint64 + break; +#endif + default: + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } + + uint64_t float64 = ui64_to_f64(uint_val); + if (negative) + float64 ^= INT64_MIN; + + if (GET_PRECISION(insn) == PRECISION_S) + SET_F32_RD(insn, regs, f64_to_f32(float64)); + else + SET_F64_RD(insn, regs, float64); +} + +DECLARE_EMULATION_FUNC(emulate_fcvt_if) +{ + int rs2_num = (insn >> 20) & 0x1f; +#ifdef __riscv64 + if (rs2_num >= 4) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); +#else + if (rs2_num >= 2) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); +#endif + + int64_t float64; + if (GET_PRECISION(insn) == PRECISION_S) + float64 = f32_to_f64(GET_F32_RS1(insn, regs)); + else if (GET_PRECISION(insn) == PRECISION_D) + float64 = GET_F64_RS1(insn, regs); + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + int negative = 0; + if (float64 < 0) { + negative = 1; + float64 ^= INT64_MIN; + } + uint64_t uint_val = f64_to_ui64(float64, softfloat_roundingMode, true); + uint64_t result, limit, limit_result; + + switch (rs2_num) + { + case 0: // int32 + if (negative) { + result = (int32_t)-uint_val; + limit_result = limit = (uint32_t)INT32_MIN; + } else { + result = (int32_t)uint_val; + limit_result = limit = INT32_MAX; + } + break; + + case 1: // uint32 + limit = limit_result = UINT32_MAX; + if (negative) + result = limit = 0; + else + result = (uint32_t)uint_val; + break; + + case 2: // int32 + if (negative) { + result = (int64_t)-uint_val; + limit_result = limit = (uint64_t)INT64_MIN; + } else { + result = (int64_t)uint_val; + limit_result = limit = INT64_MAX; + } + break; + + case 3: // uint64 + limit = limit_result = UINT64_MAX; + if (negative) + result = limit = 0; + else + result = (uint64_t)uint_val; + break; + + default: + __builtin_unreachable(); + } + + if (uint_val > limit) { + result = limit_result; + softfloat_raiseFlags(softfloat_flag_invalid); + } + + SET_FS_DIRTY(); + SET_RD(insn, regs, result); +} + +DECLARE_EMULATION_FUNC(emulate_fcmp) +{ + int rm = GET_RM(insn); + if (rm >= 3) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + uintptr_t result; + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + if (rm != 1) + result = f32_eq(rs1, rs2); + if (rm == 1 || (rm == 0 && !result)) + result = f32_lt(rs1, rs2); + goto success; + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + if (rm != 1) + result = f64_eq(rs1, rs2); + if (rm == 1 || (rm == 0 && !result)) + result = f64_lt(rs1, rs2); + goto success; + } + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); +success: + SET_RD(insn, regs, result); +} + +DECLARE_EMULATION_FUNC(emulate_fmv_if) +{ + uintptr_t result; + if ((insn & MASK_FMV_X_S) == MATCH_FMV_X_S) + result = GET_F32_RS1(insn, regs); +#ifdef __riscv64 + else if ((insn & MASK_FMV_X_D) == MATCH_FMV_X_D) + result = GET_F64_RS1(insn, regs); +#endif + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + SET_RD(insn, regs, result); +} + +DECLARE_EMULATION_FUNC(emulate_fmv_fi) +{ + uintptr_t rs1 = GET_RS1(insn, regs); + + if ((insn & MASK_FMV_S_X) == MATCH_FMV_S_X) + SET_F32_RD(insn, regs, rs1); + else if ((insn & MASK_FMV_D_X) == MATCH_FMV_D_X) + SET_F64_RD(insn, regs, rs1); + else + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); +} + +DECLARE_EMULATION_FUNC(emulate_fmadd) +{ + // if FPU is disabled, punt back to the OS + if (unlikely((mstatus & MSTATUS_FS) == 0)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + int op = (insn >> 2) & 3; + SETUP_STATIC_ROUNDING(insn); + if (GET_PRECISION(insn) == PRECISION_S) { + uint32_t rs1 = GET_F32_RS1(insn, regs); + uint32_t rs2 = GET_F32_RS2(insn, regs); + uint32_t rs3 = GET_F32_RS3(insn, regs); + SET_F32_RD(insn, regs, softfloat_mulAddF32(op, rs1, rs2, rs3)); + } else if (GET_PRECISION(insn) == PRECISION_D) { + uint64_t rs1 = GET_F64_RS1(insn, regs); + uint64_t rs2 = GET_F64_RS2(insn, regs); + uint64_t rs3 = GET_F64_RS3(insn, regs); + SET_F64_RD(insn, regs, softfloat_mulAddF64(op, rs1, rs2, rs3)); + } else { + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + } +} diff --git a/machine/fp_emulation.h b/machine/fp_emulation.h new file mode 100644 index 0000000..d2357b7 --- /dev/null +++ b/machine/fp_emulation.h @@ -0,0 +1,81 @@ +#ifndef _RISCV_FP_EMULATION_H +#define _RISCV_FP_EMULATION_H + +#include "emulation.h" + +#define GET_PRECISION(insn) (((insn) >> 25) & 3) +#define GET_RM(insn) (((insn) >> 12) & 7) +#define PRECISION_S 0 +#define PRECISION_D 1 + +#ifdef __riscv_hard_float +# define GET_F32_REG(insn, pos, regs) ({ \ + register int32_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \ + uintptr_t tmp; \ + asm ("1: auipc %0, %%pcrel_hi(get_f32_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \ + value; }) +# define SET_F32_REG(insn, pos, regs, val) ({ \ + register uint32_t value asm("a0") = (val); \ + uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \ + uintptr_t tmp; \ + asm volatile ("1: auipc %0, %%pcrel_hi(put_f32_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); }) +# define init_fp_reg(i) SET_F32_REG((i) << 3, 3, 0, 0) +# define GET_F64_REG(insn, pos, regs) ({ \ + register uintptr_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \ + uintptr_t tmp; \ + asm ("1: auipc %0, %%pcrel_hi(get_f64_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \ + sizeof(uintptr_t) == 4 ? *(int64_t*)value : (int64_t)value; }) +# define SET_F64_REG(insn, pos, regs, val) ({ \ + uint64_t __val = (val); \ + register uintptr_t value asm("a0") = sizeof(uintptr_t) == 4 ? (uintptr_t)&__val : (uintptr_t)__val; \ + uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \ + uintptr_t tmp; \ + asm volatile ("1: auipc %0, %%pcrel_hi(put_f64_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); }) +# define GET_FCSR() read_csr(fcsr) +# define SET_FCSR(value) write_csr(fcsr, (value)) +# define GET_FRM() read_csr(frm) +# define SET_FRM(value) write_csr(frm, (value)) +# define GET_FFLAGS() read_csr(fflags) +# define SET_FFLAGS(value) write_csr(fflags, (value)) + +# define SETUP_STATIC_ROUNDING(insn) ({ \ + register long tp asm("tp") = read_csr(frm); \ + if (likely(((insn) & MASK_FUNCT3) == MASK_FUNCT3)) ; \ + else if (GET_RM(insn) > 4) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); \ + else tp = GET_RM(insn); \ + asm volatile ("":"+r"(tp)); }) +# define softfloat_raiseFlags(which) set_csr(fflags, which) +# define softfloat_roundingMode ({ register int tp asm("tp"); tp; }) +#else +# define GET_F64_REG(insn, pos, regs) (*(int64_t*)((void*)((regs) + 32) + (((insn) >> ((pos)-3)) & 0xf8))) +# define SET_F64_REG(insn, pos, regs, val) (GET_F64_REG(insn, pos, regs) = (val)) +# define GET_F32_REG(insn, pos, regs) (*(int32_t*)&GET_F64_REG(insn, pos, regs)) +# define SET_F32_REG(insn, pos, regs, val) (GET_F32_REG(insn, pos, regs) = (val)) +# define GET_FCSR() ({ register int tp asm("tp"); tp & 0xFF; }) +# define SET_FCSR(value) ({ asm volatile("add tp, x0, %0" :: "rI"((value) & 0xFF)); }) +# define GET_FRM() (GET_FCSR() >> 5) +# define SET_FRM(value) SET_FCSR(GET_FFLAGS() | ((value) << 5)) +# define GET_FFLAGS() (GET_FCSR() & 0x1F) +# define SET_FFLAGS(value) SET_FCSR((GET_FRM() << 5) | ((value) & 0x1F)) + +# define SETUP_STATIC_ROUNDING(insn) ({ \ + register int tp asm("tp"); tp &= 0xFF; \ + if (likely(((insn) & MASK_FUNCT3) == MASK_FUNCT3)) tp |= tp << 8; \ + else if (GET_RM(insn) > 4) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); \ + else tp |= GET_RM(insn) << 13; \ + asm volatile ("":"+r"(tp)); }) +# define softfloat_raiseFlags(which) ({ asm volatile ("or tp, tp, %0" :: "rI"(which)); }) +# define softfloat_roundingMode ({ register int tp asm("tp"); tp >> 13; }) +#endif + +#define GET_F32_RS1(insn, regs) (GET_F32_REG(insn, 15, regs)) +#define GET_F32_RS2(insn, regs) (GET_F32_REG(insn, 20, regs)) +#define GET_F32_RS3(insn, regs) (GET_F32_REG(insn, 27, regs)) +#define GET_F64_RS1(insn, regs) (GET_F64_REG(insn, 15, regs)) +#define GET_F64_RS2(insn, regs) (GET_F64_REG(insn, 20, regs)) +#define GET_F64_RS3(insn, regs) (GET_F64_REG(insn, 27, regs)) +#define SET_F32_RD(insn, regs, val) (SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY()) +#define SET_F64_RD(insn, regs, val) (SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY()) +#define SET_FS_DIRTY() set_csr(mstatus, MSTATUS_FS) + +#endif diff --git a/machine/htif.h b/machine/htif.h new file mode 100644 index 0000000..e66ad06 --- /dev/null +++ b/machine/htif.h @@ -0,0 +1,18 @@ +#ifndef _RISCV_HTIF_H +#define _RISCV_HTIF_H + +#include + +#ifdef __riscv64 +# define TOHOST_CMD(dev, cmd, payload) \ + (((uint64_t)(dev) << 56) | ((uint64_t)(cmd) << 48) | (uint64_t)(payload)) +#else +# define TOHOST_CMD(dev, cmd, payload) ({ \ + if ((dev) || (cmd)) __builtin_trap(); \ + (payload); }) +#endif +#define FROMHOST_DEV(fromhost_value) ((uint64_t)(fromhost_value) >> 56) +#define FROMHOST_CMD(fromhost_value) ((uint64_t)(fromhost_value) << 8 >> 56) +#define FROMHOST_DATA(fromhost_value) ((uint64_t)(fromhost_value) << 16 >> 16) + +#endif diff --git a/machine/machine.ac b/machine/machine.ac new file mode 100644 index 0000000..65acf04 --- /dev/null +++ b/machine/machine.ac @@ -0,0 +1,4 @@ +AC_ARG_ENABLE([fp-emulation], AS_HELP_STRING([--disable-fp-emulation], [Disable floating-point emulation])) +AS_IF([test "x$enable_fp_emulation" != "xno"], [ + AC_DEFINE([PK_ENABLE_FP_EMULATION],,[Define if floating-point emulation is enabled]) +]) diff --git a/machine/machine.mk.in b/machine/machine.mk.in new file mode 100644 index 0000000..a0b7ca9 --- /dev/null +++ b/machine/machine.mk.in @@ -0,0 +1,29 @@ +machine_subproject_deps = \ + softfloat \ + +machine_hdrs = \ + atomic.h \ + bits.h \ + emulation.h \ + encoding.h \ + fp_emulation.h \ + htif.h \ + mcall.h \ + mtrap.h \ + sbi.h \ + unprivileged_memory.h \ + vm.h \ + +machine_c_srcs = \ + mtrap.c \ + minit.c \ + emulation.c \ + fp_emulation.c \ + sbi_impl.c \ + configstring.c \ + +machine_asm_srcs = \ + mentry.S \ + fp_asm.S \ + sbi_entry.S \ + sbi.S \ diff --git a/machine/mcall.h b/machine/mcall.h new file mode 100644 index 0000000..2096d16 --- /dev/null +++ b/machine/mcall.h @@ -0,0 +1,21 @@ +#ifndef _RISCV_MCALL_H +#define _RISCV_MCALL_H + +#define MCALL_HART_ID 0 +#define MCALL_CONSOLE_PUTCHAR 1 +#define MCALL_CONSOLE_GETCHAR 2 +#define MCALL_HTIF_SYSCALL 3 +#define MCALL_SEND_IPI 4 +#define MCALL_CLEAR_IPI 5 +#define MCALL_SHUTDOWN 6 +#define MCALL_SET_TIMER 7 +#define MCALL_REMOTE_SFENCE_VM 8 +#define MCALL_REMOTE_FENCE_I 9 + +#ifndef __ASSEMBLER__ + +extern uintptr_t do_mcall(uintptr_t which, ...); + +#endif + +#endif diff --git a/machine/mentry.S b/machine/mentry.S new file mode 100644 index 0000000..50ce9ca --- /dev/null +++ b/machine/mentry.S @@ -0,0 +1,245 @@ +// See LICENSE for license details. + +#include "mtrap.h" +#include "bits.h" + + .data + .align 6 +trap_table: + .word bad_trap + .word bad_trap + .word illegal_insn_trap + .word bad_trap + .word misaligned_load_trap + .word bad_trap + .word misaligned_store_trap + .word bad_trap + .word bad_trap + .word mcall_trap + .word bad_trap + .word bad_trap +#define HTIF_INTERRUPT_VECTOR 12 + .word htif_interrupt +#define SOFTWARE_INTERRUPT_VECTOR 13 + .word software_interrupt +#define TRAP_FROM_MACHINE_MODE_VECTOR 14 + .word __trap_from_machine_mode + + .option norvc + .section .text.init,"ax",@progbits + .globl reset_vector +reset_vector: + j do_reset + +nmi_vector: +.Lunhandleable_trap: + j bad_trap + +trap_vector: + csrrw sp, mscratch, sp + beqz sp, .Ltrap_from_machine_mode + + STORE a0, 10*REGBYTES(sp) + STORE a1, 11*REGBYTES(sp) + + csrr a1, mcause + bgez a1, .Lhandle_trap_in_machine_mode + + # This is an interrupt. Discard the mcause MSB and decode the rest. + sll a1, a1, 1 + + # Is it a machine timer interrupt? + li a0, IRQ_M_TIMER * 2 + bne a0, a1, 1f + # Yes. Post a supervisor timer interrupt. + li a0, MIP_MTIP + csrc mie, a0 + li a0, MIP_STIP + csrs mip, a0 + +.Leret: + # Go back whence we came. + LOAD a0, 10*REGBYTES(sp) + LOAD a1, 11*REGBYTES(sp) + csrrw sp, mscratch, sp + eret + +1: + # Is it an IPI? + li a0, IRQ_M_SOFT * 2 + bne a0, a1, 1f + li a1, SOFTWARE_INTERRUPT_VECTOR + j .Lhandle_trap_in_machine_mode + +1: + # By process of elimination, it must be an HTIF interrupt. + li a0, IRQ_HOST * 2 + bne a0, a1, .Lunhandleable_trap + li a1, HTIF_INTERRUPT_VECTOR + +.Lhandle_trap_in_machine_mode: + # Preserve the registers. Compute the address of the trap handler. + STORE ra, 1*REGBYTES(sp) + STORE gp, 3*REGBYTES(sp) + STORE tp, 4*REGBYTES(sp) + STORE t0, 5*REGBYTES(sp) +1:auipc t0, %pcrel_hi(trap_table) # t0 <- %hi(trap_table) + STORE t1, 6*REGBYTES(sp) + sll t1, a1, 2 # t1 <- mcause << 2 + STORE t2, 7*REGBYTES(sp) + add t1, t0, t1 # t1 <- %hi(trap_table)[mcause] + STORE s0, 8*REGBYTES(sp) + lw t1, %pcrel_lo(1b)(t1) # t1 <- trap_table[mcause] + STORE s1, 9*REGBYTES(sp) + mv a0, sp # a0 <- regs + STORE a2,12*REGBYTES(sp) + csrr a2, mepc # a2 <- mepc + STORE a3,13*REGBYTES(sp) + csrrw t0, mscratch, x0 # t0 <- user sp + STORE a4,14*REGBYTES(sp) + STORE a5,15*REGBYTES(sp) + STORE a6,16*REGBYTES(sp) + STORE a7,17*REGBYTES(sp) + STORE s2,18*REGBYTES(sp) + STORE s3,19*REGBYTES(sp) + STORE s4,20*REGBYTES(sp) + STORE s5,21*REGBYTES(sp) + STORE s6,22*REGBYTES(sp) + STORE s7,23*REGBYTES(sp) + STORE s8,24*REGBYTES(sp) + STORE s9,25*REGBYTES(sp) + STORE s10,26*REGBYTES(sp) + STORE s11,27*REGBYTES(sp) + STORE t3,28*REGBYTES(sp) + STORE t4,29*REGBYTES(sp) + STORE t5,30*REGBYTES(sp) + STORE t6,31*REGBYTES(sp) + STORE t0, 2*REGBYTES(sp) # sp + +#ifndef __riscv_hard_float + lw tp, (sp) # Move the emulated FCSR from x0's save slot into tp. +#endif + STORE x0, (sp) # Zero x0's save slot. + + # Invoke the handler. + jalr t1 + +#ifndef __riscv_hard_float + sw tp, (sp) # Move the emulated FCSR from tp into x0's save slot. +#endif + +restore_mscratch: + # Restore mscratch, so future traps will know they didn't come from M-mode. + csrw mscratch, sp + +restore_regs: + # Restore all of the registers. + LOAD ra, 1*REGBYTES(sp) + LOAD gp, 3*REGBYTES(sp) + LOAD tp, 4*REGBYTES(sp) + LOAD t0, 5*REGBYTES(sp) + LOAD t1, 6*REGBYTES(sp) + LOAD t2, 7*REGBYTES(sp) + LOAD s0, 8*REGBYTES(sp) + LOAD s1, 9*REGBYTES(sp) + LOAD a0,10*REGBYTES(sp) + LOAD a1,11*REGBYTES(sp) + LOAD a2,12*REGBYTES(sp) + LOAD a3,13*REGBYTES(sp) + LOAD a4,14*REGBYTES(sp) + LOAD a5,15*REGBYTES(sp) + LOAD a6,16*REGBYTES(sp) + LOAD a7,17*REGBYTES(sp) + LOAD s2,18*REGBYTES(sp) + LOAD s3,19*REGBYTES(sp) + LOAD s4,20*REGBYTES(sp) + LOAD s5,21*REGBYTES(sp) + LOAD s6,22*REGBYTES(sp) + LOAD s7,23*REGBYTES(sp) + LOAD s8,24*REGBYTES(sp) + LOAD s9,25*REGBYTES(sp) + LOAD s10,26*REGBYTES(sp) + LOAD s11,27*REGBYTES(sp) + LOAD t3,28*REGBYTES(sp) + LOAD t4,29*REGBYTES(sp) + LOAD t5,30*REGBYTES(sp) + LOAD t6,31*REGBYTES(sp) + LOAD sp, 2*REGBYTES(sp) + eret + +.Ltrap_from_machine_mode: + csrr sp, mscratch + addi sp, sp, -INTEGER_CONTEXT_SIZE + STORE a0,10*REGBYTES(sp) + STORE a1,11*REGBYTES(sp) + li a1, TRAP_FROM_MACHINE_MODE_VECTOR + j .Lhandle_trap_in_machine_mode + + .globl __redirect_trap +__redirect_trap: + # reset sp to top of M-mode stack + li t0, MACHINE_STACK_SIZE + add sp, sp, t0 + neg t0, t0 + and sp, sp, t0 + addi sp, sp, -MENTRY_FRAME_SIZE + j restore_mscratch + +__trap_from_machine_mode: + jal trap_from_machine_mode + j restore_regs + +do_reset: + li x1, 0 + li x2, 0 + li x3, 0 + li x4, 0 + li x5, 0 + li x6, 0 + li x7, 0 + li x8, 0 + li x9, 0 + li x10, 0 + li x11, 0 + li x12, 0 + li x13, 0 + li x14, 0 + li x15, 0 + li x16, 0 + li x17, 0 + li x18, 0 + li x19, 0 + li x20, 0 + li x21, 0 + li x22, 0 + li x23, 0 + li x24, 0 + li x25, 0 + li x26, 0 + li x27, 0 + li x28, 0 + li x29, 0 + li x30, 0 + li x31, 0 + csrw mscratch, x0 + + # sp <- end of first full page after the end of the binary + la sp, _end + 2*RISCV_PGSIZE - 1 + li t0, -RISCV_PGSIZE + and sp, sp, t0 + addi sp, sp, -MENTRY_FRAME_SIZE + + csrr a0, mhartid + slli a1, a0, RISCV_PGSHIFT + add sp, sp, a1 + + beqz a0, init_first_hart + +.LmultiHart: +#if MAX_HARTS > 1 + # make sure our hart id is within a valid range + li a1, MAX_HARTS + bltu a0, a1, init_other_hart + wfi +#endif + j .LmultiHart diff --git a/machine/minit.c b/machine/minit.c new file mode 100644 index 0000000..33b94c8 --- /dev/null +++ b/machine/minit.c @@ -0,0 +1,120 @@ +#include "mtrap.h" +#include "atomic.h" +#include "vm.h" +#include "fp_emulation.h" +#include + +pte_t* root_page_table; +uintptr_t first_free_paddr; +uintptr_t mem_size; +uint32_t num_harts; + +static void mstatus_init() +{ + uintptr_t ms = 0; + ms = INSERT_FIELD(ms, MSTATUS_VM, VM_CHOICE); + ms = INSERT_FIELD(ms, MSTATUS_FS, 1); + write_csr(mstatus, ms); + + ms = read_csr(mstatus); + assert(EXTRACT_FIELD(ms, MSTATUS_VM) == VM_CHOICE); + + write_csr(mtimecmp, 0); + clear_csr(mip, MIP_MSIP); + write_csr(mie, -1); + write_csr(mucounteren, -1); + write_csr(mscounteren, -1); +} + +static void delegate_traps() +{ + uintptr_t interrupts = MIP_SSIP | MIP_STIP; + uintptr_t exceptions = + (1U << CAUSE_MISALIGNED_FETCH) | + (1U << CAUSE_FAULT_FETCH) | + (1U << CAUSE_BREAKPOINT) | + (1U << CAUSE_FAULT_LOAD) | + (1U << CAUSE_FAULT_STORE) | + (1U << CAUSE_BREAKPOINT) | + (1U << CAUSE_USER_ECALL); + + write_csr(mideleg, interrupts); + write_csr(medeleg, exceptions); + assert(read_csr(mideleg) == interrupts); + assert(read_csr(medeleg) == exceptions); +} + +static void fp_init() +{ + assert(read_csr(mstatus) & MSTATUS_FS); + +#ifdef __riscv_hard_float + if (!supports_extension('D')) + die("FPU not found; recompile pk with -msoft-float"); + for (int i = 0; i < 32; i++) + init_fp_reg(i); + write_csr(fcsr, 0); +#else + if (supports_extension('D')) + die("FPU unexpectedly found; recompile with -mhard-float"); +#endif +} + +void hls_init(uint32_t id, csr_t* csrs) +{ + hls_t* hls = OTHER_HLS(id); + memset(hls, 0, sizeof(*hls)); + hls->csrs = csrs; +} + +static uintptr_t sbi_top_paddr() +{ + extern char _end; + return ROUNDUP((uintptr_t)&_end, RISCV_PGSIZE); +} + +static void memory_init() +{ + mem_size = mem_size / MEGAPAGE_SIZE * MEGAPAGE_SIZE; + first_free_paddr = sbi_top_paddr() + num_harts * RISCV_PGSIZE; +} + +static void hart_init() +{ + mstatus_init(); + fp_init(); + delegate_traps(); +} + +void init_first_hart() +{ + hart_init(); + memset(HLS(), 0, sizeof(*HLS())); + parse_config_string(); + memory_init(); + boot_loader(); +} + +void init_other_hart() +{ + hart_init(); + + // wait until hart 0 discovers us + while (*(csr_t * volatile *)&HLS()->csrs == NULL) + ; + + boot_other_hart(); +} + +void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t stack) +{ + uintptr_t mstatus = read_csr(mstatus); + mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S); + mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); + write_csr(mstatus, mstatus); + write_csr(mscratch, MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE); + write_csr(mepc, fn); + write_csr(sptbr, (uintptr_t)root_page_table >> RISCV_PGSHIFT); + asm volatile ("mv a0, %0; mv sp, %0; eret" : : "r" (stack)); + __builtin_unreachable(); +} diff --git a/machine/mtrap.c b/machine/mtrap.c new file mode 100644 index 0000000..6fc5476 --- /dev/null +++ b/machine/mtrap.c @@ -0,0 +1,287 @@ +#include "mtrap.h" +#include "mcall.h" +#include "htif.h" +#include "atomic.h" +#include "bits.h" +#include +#include +#include + +void __attribute__((noreturn)) bad_trap() +{ + die("machine mode: unhandlable trap %d @ %p", read_csr(mcause), read_csr(mepc)); +} + +static uintptr_t mcall_hart_id() +{ + return read_const_csr(mhartid); +} + +static void request_htif_keyboard_interrupt() +{ + uintptr_t old_tohost = swap_csr(mtohost, TOHOST_CMD(1, 0, 0)); + assert(old_tohost == 0); +} + +void htif_interrupt() +{ + uintptr_t fromhost = swap_csr(mfromhost, 0); + // we should only be interrupted by keypresses + if (!(FROMHOST_DEV(fromhost) == 1 && FROMHOST_CMD(fromhost) == 0)) + die("unexpected htif interrupt"); + HLS()->console_ibuf = 1 + (uint8_t)FROMHOST_DATA(fromhost); + set_csr(mip, MIP_SSIP); +} + +static void do_tohost_fromhost(uintptr_t dev, uintptr_t cmd, uintptr_t data) +{ + while (swap_csr(mtohost, TOHOST_CMD(dev, cmd, data)) != 0) + if (read_csr(mfromhost)) + htif_interrupt(); + + while (1) { + uintptr_t fromhost = read_csr(mfromhost); + if (fromhost) { + if (FROMHOST_DEV(fromhost) == dev && FROMHOST_CMD(fromhost) == cmd) { + write_csr(mfromhost, 0); + break; + } + htif_interrupt(); + } + } +} + +static uintptr_t mcall_console_putchar(uint8_t ch) +{ + do_tohost_fromhost(1, 1, ch); + return 0; +} + +static uintptr_t mcall_htif_syscall(uintptr_t magic_mem) +{ + do_tohost_fromhost(0, 0, magic_mem); + return 0; +} + +void poweroff() +{ + while (1) + write_csr(mtohost, 1); +} + +void putstring(const char* s) +{ + while (*s) + mcall_console_putchar(*s++); +} + +void printm(const char* s, ...) +{ + char buf[256]; + va_list vl; + + va_start(vl, s); + vsnprintf(buf, sizeof buf, s, vl); + va_end(vl); + + putstring(buf); +} + +static void send_ipi(uintptr_t recipient, int event) +{ + if ((atomic_or(&OTHER_HLS(recipient)->mipi_pending, event) & event) == 0) { + mb(); + OTHER_HLS(recipient)->csrs[CSR_MIPI] = 1; + } +} + +static uintptr_t mcall_send_ipi(uintptr_t recipient) +{ + if (recipient >= num_harts) + return -1; + + send_ipi(recipient, IPI_SOFT); + return 0; +} + +static void reset_ssip() +{ + clear_csr(mip, MIP_SSIP); + mb(); + + if (HLS()->sipi_pending || HLS()->console_ibuf > 0) + set_csr(mip, MIP_SSIP); +} + +static uintptr_t mcall_console_getchar() +{ + int ch = atomic_swap(&HLS()->console_ibuf, -1); + if (ch >= 0) + request_htif_keyboard_interrupt(); + reset_ssip(); + return ch - 1; +} + +static uintptr_t mcall_clear_ipi() +{ + int ipi = atomic_swap(&HLS()->sipi_pending, 0); + reset_ssip(); + return ipi; +} + +static uintptr_t mcall_shutdown() +{ + poweroff(); +} + +static uintptr_t mcall_set_timer(unsigned long long when) +{ + // bbl/pk don't use the timer, so there's no need to virtualize it + write_csr(mtimecmp, when); +#ifndef __riscv64 + write_csr(mtimecmph, when >> 32); +#endif + clear_csr(mip, MIP_STIP); + set_csr(mie, MIP_MTIP); + return 0; +} + +void software_interrupt() +{ + clear_csr(mip, MIP_MSIP); + mb(); + int ipi_pending = atomic_swap(&HLS()->mipi_pending, 0); + + if (ipi_pending & IPI_SOFT) { + HLS()->sipi_pending = 1; + set_csr(mip, MIP_SSIP); + } + + if (ipi_pending & IPI_FENCE_I) + asm volatile ("fence.i"); + + if (ipi_pending & IPI_SFENCE_VM) + asm volatile ("sfence.vm"); +} + +static void send_ipi_many(uintptr_t* pmask, int event) +{ + _Static_assert(MAX_HARTS <= 8 * sizeof(*pmask), "# harts > uintptr_t bits"); + uintptr_t mask = -1; + if (pmask) + mask = *pmask; + + // send IPIs to everyone + for (ssize_t i = num_harts-1; i >= 0; i--) + if ((mask >> i) & 1) + send_ipi(i, event); + + // wait until all events have been handled. + // prevent deadlock while spinning by handling any IPIs from other harts. + for (ssize_t i = num_harts-1; i >= 0; i--) + if ((mask >> i) & 1) + while (OTHER_HLS(i)->mipi_pending & event) + software_interrupt(); +} + +static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask, uintptr_t asid) +{ + // ignore the ASID and do a global flush. + // this allows us to avoid queueing a message. + send_ipi_many(hart_mask, IPI_SFENCE_VM); + return 0; +} + +static uintptr_t mcall_remote_fence_i(uintptr_t* hart_mask) +{ + send_ipi_many(hart_mask, IPI_FENCE_I); + return 0; +} + +void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval; + switch (n) + { + case MCALL_HART_ID: + retval = mcall_hart_id(); + break; + case MCALL_CONSOLE_PUTCHAR: + retval = mcall_console_putchar(arg0); + break; + case MCALL_CONSOLE_GETCHAR: + retval = mcall_console_getchar(); + break; + case MCALL_HTIF_SYSCALL: + retval = mcall_htif_syscall(arg0); + break; + case MCALL_SEND_IPI: + retval = mcall_send_ipi(arg0); + break; + case MCALL_CLEAR_IPI: + retval = mcall_clear_ipi(); + break; + case MCALL_SHUTDOWN: + retval = mcall_shutdown(); + break; + case MCALL_SET_TIMER: + retval = mcall_set_timer(arg0); + break; + case MCALL_REMOTE_SFENCE_VM: + retval = mcall_remote_sfence_vm((uintptr_t*)arg0, arg1); + break; + case MCALL_REMOTE_FENCE_I: + retval = mcall_remote_fence_i((uintptr_t*)arg0); + break; + default: + retval = -ENOSYS; + break; + } + regs[10] = retval; + write_csr(mepc, mepc + 4); +} + +void redirect_trap(uintptr_t epc, uintptr_t mstatus) +{ + write_csr(sepc, epc); + write_csr(scause, read_csr(mcause)); + write_csr(mepc, read_csr(stvec)); + + uintptr_t prev_priv = EXTRACT_FIELD(mstatus, MSTATUS_MPP); + uintptr_t prev_ie = EXTRACT_FIELD(mstatus, MSTATUS_MPIE); + mstatus = INSERT_FIELD(mstatus, MSTATUS_SPP, prev_priv); + mstatus = INSERT_FIELD(mstatus, MSTATUS_SPIE, prev_ie); + mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S); + mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); + write_csr(mstatus, mstatus); + + extern void __redirect_trap(); + return __redirect_trap(); +} + +static void machine_page_fault(uintptr_t* regs, uintptr_t mepc) +{ + // MPRV=1 iff this trap occurred while emulating an instruction on behalf + // of a lower privilege level. In that case, a2=epc and a3=mstatus. + if (read_csr(mstatus) & MSTATUS_MPRV) { + write_csr(sbadaddr, read_csr(mbadaddr)); + return redirect_trap(regs[12], regs[13]); + } + bad_trap(); +} + +void trap_from_machine_mode(uintptr_t* regs, uintptr_t dummy, uintptr_t mepc) +{ + uintptr_t mcause = read_csr(mcause); + + switch (mcause) + { + case CAUSE_FAULT_LOAD: + case CAUSE_FAULT_STORE: + return machine_page_fault(regs, mepc); + case CAUSE_MACHINE_ECALL: + return mcall_trap(regs, mcause, mepc); + default: + bad_trap(); + } +} diff --git a/machine/mtrap.h b/machine/mtrap.h new file mode 100644 index 0000000..139f359 --- /dev/null +++ b/machine/mtrap.h @@ -0,0 +1,96 @@ +#ifndef _RISCV_MTRAP_H +#define _RISCV_MTRAP_H + +#include "encoding.h" + +#ifdef __riscv_atomic +# define MAX_HARTS 8 // arbitrary +#else +# define MAX_HARTS 1 +#endif + +#ifndef __ASSEMBLER__ + +#include "sbi.h" +#include + +#define read_const_csr(reg) ({ unsigned long __tmp; \ + asm ("csrr %0, " #reg : "=r"(__tmp)); \ + __tmp; }) + +static inline int supports_extension(char ext) +{ + return read_const_csr(misa) & (1 << (ext - 'A')); +} + +static inline int xlen() +{ + return read_const_csr(misa) < 0 ? 64 : 32; +} + +extern uintptr_t first_free_paddr; +extern uintptr_t mem_size; +extern uint32_t num_harts; + +typedef uintptr_t csr_t; // TODO this might become uint128_t for RV128 + +typedef struct { + volatile csr_t* csrs; + volatile int mipi_pending; + volatile int sipi_pending; + int console_ibuf; + + uint64_t utime_delta; + uint64_t ucycle_delta; + uint64_t uinstret_delta; + uint64_t stime_delta; + uint64_t scycle_delta; + uint64_t sinstret_delta; +} hls_t; + +#define IPI_SOFT 0x1 +#define IPI_FENCE_I 0x2 +#define IPI_SFENCE_VM 0x4 + +#define MACHINE_STACK_TOP() ({ \ + register uintptr_t sp asm ("sp"); \ + (void*)((sp + RISCV_PGSIZE) & -RISCV_PGSIZE); }) + +// hart-local storage, at top of stack +#define HLS() ((hls_t*)(MACHINE_STACK_TOP() - HLS_SIZE)) +#define OTHER_HLS(id) ((hls_t*)((void*)HLS() + RISCV_PGSIZE * ((id) - read_const_csr(mhartid)))) + +void hls_init(uint32_t hart_id, csr_t* csrs); +void parse_config_string(); +void poweroff(void) __attribute((noreturn)); +void printm(const char* s, ...); +void putstring(const char* s); +#define assert(x) ({ if (!(x)) die("assertion failed: %s", #x); }) +#define die(str, ...) ({ printm("%s:%d: " str "\n", __FILE__, __LINE__, ##__VA_ARGS__); poweroff(); }) +#define printk(...) die("printk") + +void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t stack) + __attribute__((noreturn)); +void boot_loader(); +void boot_other_hart(); + +static inline void wfi() +{ + asm volatile ("wfi" ::: "memory"); +} + +#endif // !__ASSEMBLER__ + +#define MACHINE_STACK_SIZE RISCV_PGSIZE +#define MENTRY_FRAME_SIZE (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE \ + + HLS_SIZE) + +#ifdef __riscv_hard_float +# define SOFT_FLOAT_CONTEXT_SIZE 0 +#else +# define SOFT_FLOAT_CONTEXT_SIZE (8 * 32) +#endif +#define HLS_SIZE 64 +#define INTEGER_CONTEXT_SIZE (32 * REGBYTES) + +#endif diff --git a/machine/sbi.S b/machine/sbi.S new file mode 100644 index 0000000..cbea78a --- /dev/null +++ b/machine/sbi.S @@ -0,0 +1,15 @@ +.globl sbi_hart_id; sbi_hart_id = -2048 +.globl sbi_num_harts; sbi_num_harts = -2032 +.globl sbi_query_memory; sbi_query_memory = -2016 +.globl sbi_console_putchar; sbi_console_putchar = -2000 +.globl sbi_console_getchar; sbi_console_getchar = -1984 +.globl sbi_send_ipi; sbi_send_ipi = -1952 +.globl sbi_clear_ipi; sbi_clear_ipi = -1936 +.globl sbi_timebase; sbi_timebase = -1920 +.globl sbi_shutdown; sbi_shutdown = -1904 +.globl sbi_set_timer; sbi_set_timer = -1888 +.globl sbi_mask_interrupt; sbi_mask_interrupt = -1872 +.globl sbi_unmask_interrupt; sbi_unmask_interrupt = -1856 +.globl sbi_remote_sfence_vm; sbi_remote_sfence_vm = -1840 +.globl sbi_remote_sfence_vm_range; sbi_remote_sfence_vm_range = -1824 +.globl sbi_remote_fence_i; sbi_remote_fence_i = -1808 diff --git a/machine/sbi.h b/machine/sbi.h new file mode 100644 index 0000000..4e2fbd8 --- /dev/null +++ b/machine/sbi.h @@ -0,0 +1,30 @@ +#ifndef _ASM_RISCV_SBI_H +#define _ASM_RISCV_SBI_H + +typedef struct { + unsigned long base; + unsigned long size; + unsigned long node_id; +} memory_block_info; + +unsigned long sbi_query_memory(unsigned long id, memory_block_info *p); + +unsigned long sbi_hart_id(void); +unsigned long sbi_num_harts(void); +unsigned long sbi_timebase(void); +void sbi_set_timer(unsigned long long stime_value); +void sbi_send_ipi(unsigned long hart_id); +unsigned long sbi_clear_ipi(void); +void sbi_shutdown(void); + +void sbi_console_putchar(unsigned char ch); +int sbi_console_getchar(void); + +void sbi_remote_sfence_vm(unsigned long hart_mask_ptr, unsigned long asid); +void sbi_remote_sfence_vm_range(unsigned long hart_mask_ptr, unsigned long asid, unsigned long start, unsigned long size); +void sbi_remote_fence_i(unsigned long hart_mask_ptr); + +unsigned long sbi_mask_interrupt(unsigned long which); +unsigned long sbi_unmask_interrupt(unsigned long which); + +#endif diff --git a/machine/sbi_entry.S b/machine/sbi_entry.S new file mode 100644 index 0000000..a37dd25 --- /dev/null +++ b/machine/sbi_entry.S @@ -0,0 +1,111 @@ +#include "encoding.h" +#include "mcall.h" + + .section .sbi,"ax",@progbits + .option norvc + .align RISCV_PGSHIFT + .globl sbi_base +sbi_base: + + # TODO: figure out something better to do with this space. It's not + # protected from the OS, so beware. + .skip RISCV_PGSIZE - 2048 + + # hart_id + .align 4 + li a7, MCALL_HART_ID + ecall + ret + + # num_harts + .align 4 + lw a0, num_harts + ret + + # query_memory + .align 4 + tail __sbi_query_memory + + # console_putchar + .align 4 + li a7, MCALL_CONSOLE_PUTCHAR + ecall + ret + + # console_getchar + .align 4 + li a7, MCALL_CONSOLE_GETCHAR + ecall + ret + + # empty + .align 4 + unimp + + # send ipi + .align 4 + li a7, MCALL_SEND_IPI + ecall + ret + + # clear ipi + .align 4 + li a7, MCALL_CLEAR_IPI + ecall + ret + + # timebase + .align 4 + li a0, 10000000 # or, you know, we could provide the correct answer + ret + + # shutdown + .align 4 + li a7, MCALL_SHUTDOWN + ecall + + # set_timer + .align 4 + li a7, MCALL_SET_TIMER + ecall + ret + + # mask_interrupt + .align 4 + tail __sbi_mask_interrupt + + # unmask_interrupt + .align 4 + tail __sbi_unmask_interrupt + + # remote_sfence_vm + .align 4 + li a7, MCALL_REMOTE_SFENCE_VM + ecall + ret + + # remote_sfence_vm_range + .align 4 + li a7, MCALL_REMOTE_SFENCE_VM + ecall + ret + + # remote_fence_i + .align 4 + li a7, MCALL_REMOTE_FENCE_I + ecall + ret + + # end of SBI trampolines + + .globl do_mcall +do_mcall: + mv a7, a0 + mv a0, a1 + mv a1, a2 + ecall + ret + + .align RISCV_PGSHIFT + .globl _sbi_end +_sbi_end: diff --git a/machine/sbi_impl.c b/machine/sbi_impl.c new file mode 100644 index 0000000..07844e0 --- /dev/null +++ b/machine/sbi_impl.c @@ -0,0 +1,33 @@ +#include "mtrap.h" +#include "sbi.h" + +uintptr_t __sbi_query_memory(uintptr_t id, memory_block_info *p) +{ + if (id == 0) { + p->base = first_free_paddr; + p->size = mem_size - p->base; + return 0; + } + + return -1; +} + +#define LOW_IRQ_OK(n) ((n) == IRQ_S_SOFT || (n) == IRQ_S_TIMER) + +uintptr_t __sbi_mask_interrupt(uintptr_t which) +{ + if (!LOW_IRQ_OK(which)) + return -1; + + clear_csr(sie, 1UL << which); + return 0; +} + +uintptr_t __sbi_unmask_interrupt(uintptr_t which) +{ + if (!LOW_IRQ_OK(which)) + return -1; + + set_csr(sie, 1UL << which); + return 0; +} diff --git a/machine/unprivileged_memory.h b/machine/unprivileged_memory.h new file mode 100644 index 0000000..d03cc5e --- /dev/null +++ b/machine/unprivileged_memory.h @@ -0,0 +1,79 @@ +#ifndef _RISCV_MISALIGNED_H +#define _RISCV_MISALIGNED_H + +#include "encoding.h" +#include + +#define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type, insn) \ + static inline type load_##type(const type* addr, uintptr_t mepc) \ + { \ + register uintptr_t __mepc asm ("a2") = mepc; \ + register uintptr_t __mstatus asm ("a3"); \ + type val; \ + asm ("csrrs %0, mstatus, %3\n" \ + #insn " %1, %2\n" \ + "csrw mstatus, %0" \ + : "+&r" (__mstatus), "=&r" (val) \ + : "m" (*addr), "r" (MSTATUS_MPRV), "r" (__mepc)); \ + return val; \ + } + +#define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type, insn) \ + static inline void store_##type(type* addr, type val, uintptr_t mepc) \ + { \ + register uintptr_t __mepc asm ("a2") = mepc; \ + register uintptr_t __mstatus asm ("a3"); \ + asm volatile ("csrrs %0, mstatus, %3\n" \ + #insn " %1, %2\n" \ + "csrw mstatus, %0" \ + : "+&r" (__mstatus) \ + : "r" (val), "m" (*addr), "r" (MSTATUS_MPRV), \ + "r" (__mepc)); \ + } + +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint8_t, lbu) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint16_t, lhu) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int8_t, lb) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int16_t, lh) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int32_t, lw) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint8_t, sb) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint16_t, sh) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint32_t, sw) +#ifdef __riscv64 +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lwu) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint64_t, ld) +DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint64_t, sd) +#else +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lw) +#endif + +static uint32_t __attribute__((always_inline)) get_insn(uintptr_t mepc, uintptr_t* mstatus) +{ + register uintptr_t __mepc asm ("a2") = mepc; + register uintptr_t __mstatus asm ("a3"); + uint32_t val; +#ifndef __riscv_compressed + asm ("csrrs %[mstatus], mstatus, %[mprv]\n" + "lw %[insn], (%[addr])\n" + "csrw mstatus, %[mstatus]" + : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val) + : [mprv] "r" (MSTATUS_MPRV), [addr] "r" (__mepc)); +#else + uintptr_t rvc_mask = 3, tmp; + asm ("csrrs %[mstatus], mstatus, %[mprv]\n" + "lhu %[insn], (%[addr])\n" + "and %[tmp], %[insn], %[rvc_mask]\n" + "bne %[tmp], %[rvc_mask], 1f\n" + "lh %[tmp], 2(%[addr])\n" + "sll %[tmp], %[tmp], 16\n" + "add %[insn], %[insn], %[tmp]\n" + "1: csrw mstatus, %[mstatus]" + : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val), [tmp] "=&r" (tmp) + : [mprv] "r" (MSTATUS_MPRV), [addr] "r" (__mepc), + [rvc_mask] "r" (rvc_mask)); +#endif + *mstatus = __mstatus; + return val; +} + +#endif diff --git a/machine/vm.h b/machine/vm.h new file mode 100644 index 0000000..f51c639 --- /dev/null +++ b/machine/vm.h @@ -0,0 +1,35 @@ +#ifndef _VM_H +#define _VM_H + +#include "encoding.h" +#include + +#define MEGAPAGE_SIZE ((uintptr_t)(RISCV_PGSIZE << RISCV_PGLEVEL_BITS)) +#ifdef __riscv64 +# define VM_CHOICE VM_SV39 +# define VA_BITS 39 +# define GIGAPAGE_SIZE (MEGAPAGE_SIZE << RISCV_PGLEVEL_BITS) +#else +# define VM_CHOICE VM_SV32 +# define VA_BITS 32 +#endif + +typedef uintptr_t pte_t; +extern pte_t* root_page_table; + +static inline void flush_tlb() +{ + asm volatile("sfence.vm"); +} + +static inline pte_t pte_create(uintptr_t ppn, int type) +{ + return (ppn << PTE_PPN_SHIFT) | PTE_V | type; +} + +static inline pte_t ptd_create(uintptr_t ppn) +{ + return pte_create(ppn, PTE_TYPE_TABLE); +} + +#endif diff --git a/pk/atomic.h b/pk/atomic.h deleted file mode 100644 index e4610de..0000000 --- a/pk/atomic.h +++ /dev/null @@ -1,72 +0,0 @@ -// See LICENSE for license details. - -#ifndef _RISCV_ATOMIC_H -#define _RISCV_ATOMIC_H - -#include "config.h" -#include "encoding.h" - -// Currently, interrupts are always disabled when in pk/bbl. -#define disable_irqsave() (0) -#define enable_irqrestore(flags) ((void) (flags)) - -typedef struct { int lock; } spinlock_t; -#define SPINLOCK_INIT {0} - -#define mb() asm volatile ("fence" ::: "memory") -#define atomic_set(ptr, val) (*(volatile typeof(*(ptr)) *)(ptr) = val) -#define atomic_read(ptr) (*(volatile typeof(*(ptr)) *)(ptr)) - -#ifdef __riscv_atomic -# define atomic_add(ptr, inc) __sync_fetch_and_add(ptr, inc) -# define atomic_or(ptr, inc) __sync_fetch_and_or(ptr, inc) -# define atomic_swap(ptr, swp) __sync_lock_test_and_set(ptr, swp) -# define atomic_cas(ptr, cmp, swp) __sync_val_compare_and_swap(ptr, cmp, swp) -#else -# define atomic_binop(ptr, inc, op) ({ \ - long flags = disable_irqsave(); \ - typeof(*(ptr)) res = atomic_read(ptr); \ - atomic_set(ptr, op); \ - enable_irqrestore(flags); \ - res; }) -# define atomic_add(ptr, inc) atomic_binop(ptr, inc, res + (inc)) -# define atomic_or(ptr, inc) atomic_binop(ptr, inc, res | (inc)) -# define atomic_swap(ptr, inc) atomic_binop(ptr, inc, (inc)) -# define atomic_cas(ptr, cmp, swp) ({ \ - long flags = disable_irqsave(); \ - typeof(*(ptr)) res = *(volatile typeof(*(ptr)) *)(ptr); \ - if (res == (cmp)) *(volatile typeof(ptr))(ptr) = (swp); \ - enable_irqrestore(flags); \ - res; }) -#endif - -static inline void spinlock_lock(spinlock_t* lock) -{ - do - { - while (atomic_read(&lock->lock)) - ; - } while (atomic_swap(&lock->lock, -1)); - mb(); -} - -static inline void spinlock_unlock(spinlock_t* lock) -{ - mb(); - atomic_set(&lock->lock,0); -} - -static inline long spinlock_lock_irqsave(spinlock_t* lock) -{ - long flags = disable_irqsave(); - spinlock_lock(lock); - return flags; -} - -static inline void spinlock_unlock_irqrestore(spinlock_t* lock, long flags) -{ - spinlock_unlock(lock); - enable_irqrestore(flags); -} - -#endif diff --git a/pk/bbl.c b/pk/bbl.c deleted file mode 100644 index 5ace350..0000000 --- a/pk/bbl.c +++ /dev/null @@ -1,36 +0,0 @@ -#include "boot.h" -#include "mtrap.h" -#include "vm.h" -#include "config.h" - -static volatile int elf_loaded; - -static void enter_entry_point() -{ - prepare_supervisor_mode(); - write_csr(mepc, current.entry); - asm volatile("eret"); - __builtin_unreachable(); -} - -void run_loaded_program(size_t argc, char** argv) -{ - if (!current.is_supervisor) - die("bbl can't run user binaries; try using pk instead"); - - supervisor_vm_init(); -#ifdef PK_ENABLE_LOGO - print_logo(); -#endif - mb(); - elf_loaded = 1; - enter_entry_point(); -} - -void boot_other_hart() -{ - while (!elf_loaded) - ; - mb(); - enter_entry_point(); -} diff --git a/pk/bits.h b/pk/bits.h deleted file mode 100644 index 10f9df3..0000000 --- a/pk/bits.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef PK_BITS_H -#define PK_BITS_H - -#define likely(x) __builtin_expect((x), 1) -#define unlikely(x) __builtin_expect((x), 0) - -#define ROUNDUP(a, b) ((((a)-1)/(b)+1)*(b)) -#define ROUNDDOWN(a, b) ((a)/(b)*(b)) - -#define EXTRACT_FIELD(val, which) (((val) & (which)) / ((which) & ~((which)-1))) -#define INSERT_FIELD(val, which, fieldval) (((val) & ~(which)) | ((fieldval) * ((which) & ~((which)-1)))) - -#define CONST_POPCOUNT2(x) ((((x) >> 0) & 1) + (((x) >> 1) & 1)) -#define CONST_POPCOUNT4(x) (CONST_POPCOUNT2(x) + CONST_POPCOUNT2((x)>>2)) -#define CONST_POPCOUNT8(x) (CONST_POPCOUNT4(x) + CONST_POPCOUNT4((x)>>4)) -#define CONST_POPCOUNT16(x) (CONST_POPCOUNT8(x) + CONST_POPCOUNT8((x)>>8)) -#define CONST_POPCOUNT32(x) (CONST_POPCOUNT16(x) + CONST_POPCOUNT16((x)>>16)) -#define CONST_POPCOUNT64(x) (CONST_POPCOUNT32(x) + CONST_POPCOUNT32((x)>>32)) -#define CONST_POPCOUNT(x) CONST_POPCOUNT64(x) - -#define CONST_CTZ2(x) CONST_POPCOUNT2(((x) & -(x))-1) -#define CONST_CTZ4(x) CONST_POPCOUNT4(((x) & -(x))-1) -#define CONST_CTZ8(x) CONST_POPCOUNT8(((x) & -(x))-1) -#define CONST_CTZ16(x) CONST_POPCOUNT16(((x) & -(x))-1) -#define CONST_CTZ32(x) CONST_POPCOUNT32(((x) & -(x))-1) -#define CONST_CTZ64(x) CONST_POPCOUNT64(((x) & -(x))-1) -#define CONST_CTZ(x) CONST_CTZ64(x) - -#define STR(x) XSTR(x) -#define XSTR(x) #x - -#ifdef __riscv64 -# define SLL32 sllw -# define STORE sd -# define LOAD ld -# define LOG_REGBYTES 3 -#else -# define SLL32 sll -# define STORE sw -# define LOAD lw -# define LOG_REGBYTES 2 -#endif -#define REGBYTES (1 << LOG_REGBYTES) - -#endif diff --git a/pk/boot.h b/pk/boot.h index d66cc11..d2a619d 100644 --- a/pk/boot.h +++ b/pk/boot.h @@ -5,7 +5,6 @@ #ifndef __ASSEMBLER__ -#include #include typedef struct { @@ -14,28 +13,19 @@ typedef struct { int is_supervisor; size_t phdr; size_t phdr_size; - size_t first_free_paddr; - size_t first_user_vaddr; - size_t first_vaddr_after_user; size_t bias; size_t entry; size_t brk_min; size_t brk; size_t brk_max; size_t mmap_max; - size_t stack_bottom; size_t stack_top; size_t t0; } elf_info; extern elf_info current; -void prepare_supervisor_mode(); -void run_loaded_program(size_t argc, char** argv); -void boot_loader(); -void boot_other_hart(); void load_elf(const char* fn, elf_info* info); -void print_logo(); #endif // !__ASSEMBLER__ diff --git a/pk/configstring.c b/pk/configstring.c deleted file mode 100644 index 847cd4d..0000000 --- a/pk/configstring.c +++ /dev/null @@ -1,200 +0,0 @@ -#include "encoding.h" -#include "mtrap.h" -#include - -static const char* skip_whitespace(const char* str) -{ - while (*str && *str <= ' ') - str++; - return str; -} - -static const char* skip_string(const char* str) -{ - while (*str && *str++ != '"') - ; - return str; -} - -static int is_hex(char ch) -{ - return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); -} - -static int parse_hex(char ch) -{ - return (ch >= '0' && ch <= '9') ? ch - '0' : - (ch >= 'a' && ch <= 'f') ? ch - 'a' + 10 : - ch - 'A' + 10; -} - -static const char* skip_key(const char* str) -{ - while (*str >= 35 && *str <= 122 && *str != ';') - str++; - return str; -} - -typedef struct { - const char* start; - const char* end; -} query_result; - -static query_result query_config_string(const char* str, const char* k) -{ - size_t ksize = 0; - while (k[ksize] && k[ksize] != '{') - ksize++; - int last = !k[ksize]; - - query_result res = {0, 0}; - while (1) { - const char* key_start = str = skip_whitespace(str); - const char* key_end = str = skip_key(str); - int match = (key_end - key_start) == ksize; - if (match) - for (size_t i = 0; i < ksize; i++) - if (key_start[i] != k[i]) - match = 0; - const char* value_start = str = skip_whitespace(str); - while (*str != ';') { - if (!*str) { - return res; - } else if (*str == '"') { - str = skip_string(str+1); - } else if (*str == '{') { - const char* search_key = match && !last ? k + ksize + 1 : ""; - query_result inner_res = query_config_string(str + 1, search_key); - if (inner_res.start) - return inner_res; - str = inner_res.end + 1; - } else { - str = skip_key(str); - } - str = skip_whitespace(str); - } - res.end = str; - if (match && last) { - res.start = value_start; - return res; - } - str = skip_whitespace(str+1); - if (*str == '}') { - res.end = str; - return res; - } - } -} - -static void parse_string(query_result r, char* buf) -{ - if (r.start < r.end) { - if (*r.start == '"') { - for (const char* p = r.start + 1; p < r.end && *p != '"'; p++) { - char ch = p[0]; - if (ch == '\\' && p[1] == 'x' && is_hex(p[2])) { - ch = parse_hex(p[2]); - if (is_hex(p[3])) { - ch = (ch << 4) + parse_hex(p[3]); - p++; - } - p += 2; - } - *buf++ = ch; - } - } else { - for (const char* p = r.start; p < r.end && *p > ' '; p++) - *buf++ = *p; - } - } - *buf = 0; -} - -#define get_string(name, search_res) \ - char name[(search_res).end - (search_res).start + 1]; \ - parse_string(search_res, name) - -static unsigned long __get_uint_hex(const char* s) -{ - unsigned long res = 0; - while (*s) { - if (is_hex(*s)) - res = (res << 4) + parse_hex(*s); - else if (*s != '_') - break; - s++; - } - return res; -} - -static unsigned long __get_uint_dec(const char* s) -{ - unsigned long res = 0; - while (*s) { - if (*s >= '0' && *s <= '9') - res = res * 10 + (*s - '0'); - else - break; - s++; - } - return res; -} - -static unsigned long __get_uint(const char* s) -{ - if (s[0] == '0' && s[1] == 'x') - return __get_uint_hex(s+2); - return __get_uint_dec(s); -} - -static unsigned long get_uint(query_result res) -{ - get_string(s, res); - return __get_uint(s); -} - -static long get_sint(query_result res) -{ - get_string(s, res); - if (s[0] == '-') - return -__get_uint(s+1); - return __get_uint(s); -} - -static void query_mem(const char* config_string) -{ - query_result res = query_config_string(config_string, "ram{0{addr"); - assert(res.start); - uintptr_t base = get_uint(res); - res = query_config_string(config_string, "ram{0{size"); - mem_size = get_uint(res); -} - -static void query_harts(const char* config_string) -{ - for (int core = 0, hart; ; core++) { - for (hart = 0; ; hart++) { - char buf[32]; - snprintf(buf, sizeof buf, "core{%d{%d{addr", core, hart); - query_result res = query_config_string(config_string, buf); - if (!res.start) - break; - csr_t* base = (csr_t*)get_uint(res); - uintptr_t hart_id = base[CSR_MHARTID]; - hls_init(hart_id, base); - num_harts++; - assert(hart_id == num_harts-1); - } - if (!hart) - break; - } - assert(num_harts); - assert(num_harts <= MAX_HARTS); -} - -void parse_config_string() -{ - const char* s = (const char*)read_csr(mcfgaddr); - query_mem(s); - query_harts(s); -} diff --git a/pk/elf.c b/pk/elf.c index 7dc783a..7107d1c 100644 --- a/pk/elf.c +++ b/pk/elf.c @@ -1,9 +1,9 @@ // See LICENSE for license details. -#include "file.h" -#include "vm.h" +#include "mmap.h" #include "mtrap.h" #include "boot.h" +#include "bits.h" #include #include #include @@ -15,100 +15,55 @@ void load_elf(const char* fn, elf_info* info) if (IS_ERR_VALUE(file)) goto fail; - Elf64_Ehdr eh64; - ssize_t ehdr_size = file_pread(file, &eh64, sizeof(eh64), 0); - if (ehdr_size < (ssize_t)sizeof(eh64) || - !(eh64.e_ident[0] == '\177' && eh64.e_ident[1] == 'E' && - eh64.e_ident[2] == 'L' && eh64.e_ident[3] == 'F')) + Elf_Ehdr eh; + ssize_t ehdr_size = file_pread(file, &eh, sizeof(eh), 0); + if (ehdr_size < (ssize_t)sizeof(eh) || + !(eh.e_ident[0] == '\177' && eh.e_ident[1] == 'E' && + eh.e_ident[2] == 'L' && eh.e_ident[3] == 'F')) goto fail; - uintptr_t min_vaddr = -1, max_vaddr = 0; - - #define LOAD_ELF do { \ - eh = (typeof(eh))&eh64; \ - size_t phdr_size = eh->e_phnum*sizeof(*ph); \ - if (phdr_size > info->phdr_size) \ - goto fail; \ - ssize_t ret = file_pread(file, (void*)info->phdr, phdr_size, eh->e_phoff); \ - if (ret < (ssize_t)phdr_size) \ - goto fail; \ - info->phnum = eh->e_phnum; \ - info->phent = sizeof(*ph); \ - ph = (typeof(ph))info->phdr; \ - info->is_supervisor = (eh->e_entry >> (8*sizeof(eh->e_entry)-1)) != 0; \ - if (info->is_supervisor) \ - info->first_free_paddr = ROUNDUP(info->first_free_paddr, SUPERPAGE_SIZE); \ - for (int i = 0; i < eh->e_phnum; i++) \ - if (ph[i].p_type == PT_LOAD && ph[i].p_memsz && ph[i].p_vaddr < min_vaddr) \ - min_vaddr = ph[i].p_vaddr; \ - if (info->is_supervisor) \ - min_vaddr = ROUNDDOWN(min_vaddr, SUPERPAGE_SIZE); \ - else \ - min_vaddr = ROUNDDOWN(min_vaddr, RISCV_PGSIZE); \ - uintptr_t bias = 0; \ - if (info->is_supervisor || eh->e_type == ET_DYN) \ - bias = info->first_free_paddr - min_vaddr; \ - info->entry = eh->e_entry; \ - if (!info->is_supervisor) { \ - info->entry += bias; \ - min_vaddr += bias; \ - } \ - info->bias = bias; \ - int flags = MAP_FIXED | MAP_PRIVATE; \ - if (info->is_supervisor) \ - flags |= MAP_POPULATE; \ - for (int i = eh->e_phnum - 1; i >= 0; i--) { \ - if(ph[i].p_type == PT_LOAD && ph[i].p_memsz) { \ - uintptr_t prepad = ph[i].p_vaddr % RISCV_PGSIZE; \ - uintptr_t vaddr = ph[i].p_vaddr + bias; \ - if (vaddr + ph[i].p_memsz > max_vaddr) \ - max_vaddr = vaddr + ph[i].p_memsz; \ - if (info->is_supervisor) { \ - if (!__valid_user_range(vaddr - prepad, vaddr + ph[i].p_memsz)) \ - goto fail; \ - ret = file_pread(file, (void*)vaddr, ph[i].p_filesz, ph[i].p_offset); \ - if (ret < (ssize_t)ph[i].p_filesz) \ - goto fail; \ - memset((void*)vaddr - prepad, 0, prepad); \ - memset((void*)vaddr + ph[i].p_filesz, 0, ph[i].p_memsz - ph[i].p_filesz); \ - } else { \ - int flags2 = flags | (prepad ? MAP_POPULATE : 0); \ - if (__do_mmap(vaddr - prepad, ph[i].p_filesz + prepad, -1, flags2, file, ph[i].p_offset - prepad) != vaddr - prepad) \ - goto fail; \ - memset((void*)vaddr - prepad, 0, prepad); \ - size_t mapped = ROUNDUP(ph[i].p_filesz + prepad, RISCV_PGSIZE) - prepad; \ - if (ph[i].p_memsz > mapped) \ - if (__do_mmap(vaddr + mapped, ph[i].p_memsz - mapped, -1, flags|MAP_ANONYMOUS, 0, 0) != vaddr + mapped) \ - goto fail; \ - } \ - } \ - } \ - } while(0) - - if (IS_ELF64(eh64)) - { -#ifndef __riscv64 - die("can't run 64-bit ELF on 32-bit arch"); -#endif - Elf64_Ehdr* eh; - Elf64_Phdr* ph; - LOAD_ELF; - } - else if (IS_ELF32(eh64)) - { #ifdef __riscv64 - die("can't run 32-bit ELF on 64-bit arch"); + assert(IS_ELF64(eh)); +#else + assert(IS_ELF32(eh)); #endif - Elf32_Ehdr* eh; - Elf32_Phdr* ph; - LOAD_ELF; - } - else - goto fail; - info->first_user_vaddr = min_vaddr; - info->first_vaddr_after_user = ROUNDUP(max_vaddr - info->bias, RISCV_PGSIZE); - info->brk_min = max_vaddr; + uintptr_t min_vaddr = -1; + size_t phdr_size = eh.e_phnum * sizeof(Elf_Phdr); + if (phdr_size > info->phdr_size) + goto fail; + ssize_t ret = file_pread(file, (void*)info->phdr, phdr_size, eh.e_phoff); + if (ret < (ssize_t)phdr_size) + goto fail; + info->phnum = eh.e_phnum; + info->phent = sizeof(Elf_Phdr); + Elf_Phdr* ph = (typeof(ph))info->phdr; + for (int i = 0; i < eh.e_phnum; i++) + if (ph[i].p_type == PT_LOAD && ph[i].p_memsz && ph[i].p_vaddr < min_vaddr) + min_vaddr = ph[i].p_vaddr; + min_vaddr = ROUNDDOWN(min_vaddr, RISCV_PGSIZE); + uintptr_t bias = 0; + if (eh.e_type == ET_DYN) + bias = first_free_paddr - min_vaddr; + min_vaddr += bias; + info->entry = eh.e_entry + bias; + int flags = MAP_FIXED | MAP_PRIVATE; + for (int i = eh.e_phnum - 1; i >= 0; i--) { + if(ph[i].p_type == PT_LOAD && ph[i].p_memsz) { + uintptr_t prepad = ph[i].p_vaddr % RISCV_PGSIZE; + uintptr_t vaddr = ph[i].p_vaddr + bias; + if (vaddr + ph[i].p_memsz > info->brk_min) + info->brk_min = vaddr + ph[i].p_memsz; + int flags2 = flags | (prepad ? MAP_POPULATE : 0); + if (__do_mmap(vaddr - prepad, ph[i].p_filesz + prepad, -1, flags2, file, ph[i].p_offset - prepad) != vaddr - prepad) + goto fail; + memset((void*)vaddr - prepad, 0, prepad); + size_t mapped = ROUNDUP(ph[i].p_filesz + prepad, RISCV_PGSIZE) - prepad; + if (ph[i].p_memsz > mapped) + if (__do_mmap(vaddr + mapped, ph[i].p_memsz - mapped, -1, flags|MAP_ANONYMOUS, 0, 0) != vaddr + mapped) + goto fail; + } + } file_decref(file); return; diff --git a/pk/elf.h b/pk/elf.h index a05ccf2..df876e9 100644 --- a/pk/elf.h +++ b/pk/elf.h @@ -12,6 +12,14 @@ #define IS_ELF32(hdr) (IS_ELF(hdr) && (hdr).e_ident[4] == 1) #define IS_ELF64(hdr) (IS_ELF(hdr) && (hdr).e_ident[4] == 2) +#ifdef __riscv64 +# define Elf_Ehdr Elf64_Ehdr +# define Elf_Phdr Elf64_Phdr +#else +# define Elf_Ehdr Elf32_Ehdr +# define Elf_Phdr Elf32_Phdr +#endif + #define ET_EXEC 2 #define ET_DYN 3 diff --git a/pk/emulation.c b/pk/emulation.c deleted file mode 100644 index 5a66997..0000000 --- a/pk/emulation.c +++ /dev/null @@ -1,313 +0,0 @@ -#include "emulation.h" -#include "fp_emulation.h" -#include "config.h" -#include "unprivileged_memory.h" -#include "mtrap.h" -#include - -void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - asm (".pushsection .rodata\n" - "illegal_insn_trap_table:\n" - " .word truly_illegal_insn\n" -#ifdef PK_ENABLE_FP_EMULATION - " .word emulate_float_load\n" -#else - " .word truly_illegal_insn\n" -#endif - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" -#ifdef PK_ENABLE_FP_EMULATION - " .word emulate_float_store\n" -#else - " .word truly_illegal_insn\n" -#endif - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_mul_div\n" - " .word truly_illegal_insn\n" - " .word emulate_mul_div32\n" - " .word truly_illegal_insn\n" -#ifdef PK_ENABLE_FP_EMULATION - " .word emulate_fmadd\n" - " .word emulate_fmadd\n" - " .word emulate_fmadd\n" - " .word emulate_fmadd\n" - " .word emulate_fp\n" -#else - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" -#endif - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_system\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .popsection"); - - uintptr_t mstatus; - insn_t insn = get_insn(mepc, &mstatus); - - if (unlikely((insn & 3) != 3)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - write_csr(mepc, mepc + 4); - - extern int32_t illegal_insn_trap_table[]; - int32_t* pf = (void*)illegal_insn_trap_table + (insn & 0x7c); - emulation_func f = (emulation_func)(uintptr_t)*pf; - f(regs, mcause, mepc, mstatus, insn); -} - -void __attribute__((noinline)) truly_illegal_insn(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn) -{ - redirect_trap(mepc, mstatus); -} - -void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - union { - uint8_t bytes[8]; - uintptr_t intx; - uint64_t int64; - } val; - uintptr_t mstatus; - insn_t insn = get_insn(mepc, &mstatus); - uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); - - int shift = 0, fp = 0, len; - if ((insn & MASK_LW) == MATCH_LW) - len = 4, shift = 8*(sizeof(uintptr_t) - len); -#ifdef __riscv64 - else if ((insn & MASK_LD) == MATCH_LD) - len = 8, shift = 8*(sizeof(uintptr_t) - len); - else if ((insn & MASK_LWU) == MATCH_LWU) - fp = 0, len = 4, shift = 0; -#endif - else if ((insn & MASK_FLD) == MATCH_FLD) - fp = 1, len = 8; - else if ((insn & MASK_FLW) == MATCH_FLW) - fp = 1, len = 4; - else if ((insn & MASK_LH) == MATCH_LH) - len = 2, shift = 8*(sizeof(uintptr_t) - len); - else if ((insn & MASK_LHU) == MATCH_LHU) - len = 2; - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - val.int64 = 0; - for (intptr_t i = len-1; i >= 0; i--) - val.bytes[i] = load_uint8_t((void *)(addr + i), mepc); - - if (!fp) - SET_RD(insn, regs, (intptr_t)val.intx << shift >> shift); - else if (len == 8) - SET_F64_RD(insn, regs, val.int64); - else - SET_F32_RD(insn, regs, val.intx); - - write_csr(mepc, mepc + 4); -} - -void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - union { - uint8_t bytes[8]; - uintptr_t intx; - uint64_t int64; - } val; - uintptr_t mstatus; - insn_t insn = get_insn(mepc, &mstatus); - int len; - - val.intx = GET_RS2(insn, regs); - if ((insn & MASK_SW) == MATCH_SW) - len = 4; -#ifdef __riscv64 - else if ((insn & MASK_SD) == MATCH_SD) - len = 8; -#endif - else if ((insn & MASK_FSD) == MATCH_FSD) - len = 8, val.int64 = GET_F64_RS2(insn, regs); - else if ((insn & MASK_FSW) == MATCH_FSW) - len = 4, val.intx = GET_F32_RS2(insn, regs); - else if ((insn & MASK_SH) == MATCH_SH) - len = 2; - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); - for (int i = 0; i < len; i++) - store_uint8_t((void *)(addr + i), val.bytes[i], mepc); - - write_csr(mepc, mepc + 4); -} - -#ifdef __riscv64 -typedef __int128 double_int; -#else -typedef int64_t double_int; -#endif - -DECLARE_EMULATION_FUNC(emulate_mul_div) -{ - uintptr_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs), val; - -#ifndef __riscv_muldiv - // If compiled with -mno-multiply, GCC will expand these out - if ((insn & MASK_MUL) == MATCH_MUL) - val = rs1 * rs2; - else if ((insn & MASK_DIV) == MATCH_DIV) - val = (intptr_t)rs1 / (intptr_t)rs2; - else if ((insn & MASK_DIVU) == MATCH_DIVU) - val = rs1 / rs2; - else if ((insn & MASK_REM) == MATCH_REM) - val = (intptr_t)rs1 % (intptr_t)rs2; - else if ((insn & MASK_REMU) == MATCH_REMU) - val = rs1 % rs2; - else if ((insn & MASK_MULH) == MATCH_MULH) - val = ((double_int)(intptr_t)rs1 * (double_int)(intptr_t)rs2) >> (8 * sizeof(rs1)); - else if ((insn & MASK_MULHU) == MATCH_MULHU) - val = ((double_int)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); - else if ((insn & MASK_MULHSU) == MATCH_MULHSU) - val = ((double_int)(intptr_t)rs1 * (double_int)rs2) >> (8 * sizeof(rs1)); - else -#endif - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - SET_RD(insn, regs, val); -} - -DECLARE_EMULATION_FUNC(emulate_mul_div32) -{ - uint32_t rs1 = GET_RS1(insn, regs), rs2 = GET_RS2(insn, regs); - int32_t val; - -#if defined(__riscv64) && !defined(__riscv_muldiv) - // If compiled with -mno-multiply, GCC will expand these out - if ((insn & MASK_MULW) == MATCH_MULW) - val = rs1 * rs2; - else if ((insn & MASK_DIVW) == MATCH_DIVW) - val = (int32_t)rs1 / (int32_t)rs2; - else if ((insn & MASK_DIVUW) == MATCH_DIVUW) - val = rs1 / rs2; - else if ((insn & MASK_REMW) == MATCH_REMW) - val = (int32_t)rs1 % (int32_t)rs2; - else if ((insn & MASK_REMUW) == MATCH_REMUW) - val = rs1 % rs2; - else -#endif - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - SET_RD(insn, regs, val); -} - -static inline int emulate_read_csr(int num, uintptr_t mstatus, uintptr_t* result) -{ - switch (num) - { - case CSR_TIME: - *result = read_csr(mtime) + HLS()->utime_delta; - return 0; - case CSR_CYCLE: - *result = read_csr(mcycle) + HLS()->ucycle_delta; - return 0; - case CSR_INSTRET: - *result = read_csr(minstret) + HLS()->uinstret_delta; - return 0; - case CSR_STIME: - *result = read_csr(mtime) + HLS()->stime_delta; - return 0; - case CSR_SCYCLE: - *result = read_csr(mcycle) + HLS()->scycle_delta; - return 0; - case CSR_SINSTRET: - *result = read_csr(minstret) + HLS()->sinstret_delta; - return 0; -#ifdef __riscv32 - case CSR_TIMEH: - *result = (((uint64_t)read_csr(mtimeh) << 32) + read_csr(mtime) - + HLS()->stime_delta) >> 32; - return 0; - case CSR_CYCLEH: - *result = (((uint64_t)read_csr(mcycleh) << 32) + read_csr(mcycle) - + HLS()->scycle_delta) >> 32; - return 0; - case CSR_INSTRETH: - *result = (((uint64_t)read_csr(minstreth) << 32) + read_csr(minstret) - + HLS()->sinstret_delta) >> 32; - return 0; -#endif -#ifdef PK_ENABLE_FP_EMULATION - case CSR_FRM: - if ((mstatus & MSTATUS_FS) == 0) break; - *result = GET_FRM(); - return 0; - case CSR_FFLAGS: - if ((mstatus & MSTATUS_FS) == 0) break; - *result = GET_FFLAGS(); - return 0; - case CSR_FCSR: - if ((mstatus & MSTATUS_FS) == 0) break; - *result = GET_FCSR(); - return 0; -#endif - } - return -1; -} - -static inline int emulate_write_csr(int num, uintptr_t value, uintptr_t mstatus) -{ - switch (num) - { -#ifdef PK_ENABLE_FP_EMULATION - case CSR_FRM: SET_FRM(value); return 0; - case CSR_FFLAGS: SET_FFLAGS(value); return 0; - case CSR_FCSR: SET_FCSR(value); return 0; -#endif - } - return -1; -} - -DECLARE_EMULATION_FUNC(emulate_system) -{ - int rs1_num = (insn >> 15) & 0x1f; - uintptr_t rs1_val = GET_RS1(insn, regs); - int csr_num = (uint32_t)insn >> 20; - uintptr_t csr_val, new_csr_val; - - if (emulate_read_csr(csr_num, mstatus, &csr_val)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - int do_write = rs1_num; - switch (GET_RM(insn)) - { - case 0: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - case 1: new_csr_val = rs1_val; do_write = 1; break; - case 2: new_csr_val = csr_val | rs1_val; break; - case 3: new_csr_val = csr_val & ~rs1_val; break; - case 4: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - case 5: new_csr_val = rs1_num; do_write = 1; break; - case 6: new_csr_val = csr_val | rs1_num; break; - case 7: new_csr_val = csr_val & ~rs1_num; break; - } - - if (do_write && emulate_write_csr(csr_num, new_csr_val, mstatus)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - SET_RD(insn, regs, csr_val); -} diff --git a/pk/emulation.h b/pk/emulation.h deleted file mode 100644 index f1a71ec..0000000 --- a/pk/emulation.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef _RISCV_EMULATION_H -#define _RISCV_EMULATION_H - -#include "encoding.h" -#include "bits.h" -#include - -typedef uint32_t insn_t; -typedef void (*emulation_func)(uintptr_t*, uintptr_t, uintptr_t, uintptr_t, insn_t); -#define DECLARE_EMULATION_FUNC(name) void name(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn) - -void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); -void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); -void redirect_trap(uintptr_t epc, uintptr_t mstatus); -DECLARE_EMULATION_FUNC(truly_illegal_insn); - -#define GET_REG(insn, pos, regs) ({ \ - int mask = (1 << (5+LOG_REGBYTES)) - (1 << LOG_REGBYTES); \ - (uintptr_t*)((uintptr_t)regs + (((insn) >> ((pos) - LOG_REGBYTES)) & mask)); \ -}) -#define GET_RS1(insn, regs) (*GET_REG(insn, 15, regs)) -#define GET_RS2(insn, regs) (*GET_REG(insn, 20, regs)) -#define SET_RD(insn, regs, val) (*GET_REG(insn, 7, regs) = (val)) -#define IMM_I(insn) ((int32_t)(insn) >> 20) -#define IMM_S(insn) (((int32_t)(insn) >> 25 << 5) | (int32_t)(((insn) >> 7) & 0x1f)) -#define MASK_FUNCT3 0x7000 - -#endif diff --git a/pk/encoding.h b/pk/encoding.h deleted file mode 100644 index f2fab36..0000000 --- a/pk/encoding.h +++ /dev/null @@ -1,1037 +0,0 @@ -// See LICENSE for license details. - -#ifndef RISCV_CSR_ENCODING_H -#define RISCV_CSR_ENCODING_H - -#define MSTATUS_UIE 0x00000001 -#define MSTATUS_SIE 0x00000002 -#define MSTATUS_HIE 0x00000004 -#define MSTATUS_MIE 0x00000008 -#define MSTATUS_UPIE 0x00000010 -#define MSTATUS_SPIE 0x00000020 -#define MSTATUS_HPIE 0x00000040 -#define MSTATUS_MPIE 0x00000080 -#define MSTATUS_SPP 0x00000100 -#define MSTATUS_HPP 0x00000600 -#define MSTATUS_MPP 0x00001800 -#define MSTATUS_FS 0x00006000 -#define MSTATUS_XS 0x00018000 -#define MSTATUS_MPRV 0x00020000 -#define MSTATUS_PUM 0x00040000 -#define MSTATUS_VM 0x1F000000 -#define MSTATUS32_SD 0x80000000 -#define MSTATUS64_SD 0x8000000000000000 - -#define SSTATUS_UIE 0x00000001 -#define SSTATUS_SIE 0x00000002 -#define SSTATUS_UPIE 0x00000010 -#define SSTATUS_SPIE 0x00000020 -#define SSTATUS_SPP 0x00000100 -#define SSTATUS_FS 0x00006000 -#define SSTATUS_XS 0x00018000 -#define SSTATUS_PUM 0x00040000 -#define SSTATUS32_SD 0x80000000 -#define SSTATUS64_SD 0x8000000000000000 - -#define MIP_SSIP (1 << IRQ_S_SOFT) -#define MIP_HSIP (1 << IRQ_H_SOFT) -#define MIP_MSIP (1 << IRQ_M_SOFT) -#define MIP_STIP (1 << IRQ_S_TIMER) -#define MIP_HTIP (1 << IRQ_H_TIMER) -#define MIP_MTIP (1 << IRQ_M_TIMER) - -#define SIP_SSIP MIP_SSIP -#define SIP_STIP MIP_STIP - -#define PRV_U 0 -#define PRV_S 1 -#define PRV_H 2 -#define PRV_M 3 - -#define VM_MBARE 0 -#define VM_MBB 1 -#define VM_MBBID 2 -#define VM_SV32 8 -#define VM_SV39 9 -#define VM_SV48 10 - -#define IRQ_S_SOFT 1 -#define IRQ_H_SOFT 2 -#define IRQ_M_SOFT 3 -#define IRQ_S_TIMER 5 -#define IRQ_H_TIMER 6 -#define IRQ_M_TIMER 7 -#define IRQ_S_DEV 9 -#define IRQ_H_DEV 10 -#define IRQ_M_DEV 11 -#define IRQ_COP 12 -#define IRQ_HOST 13 - -#define DEFAULT_RSTVEC 0x0 -#define DEFAULT_NMIVEC 0x4 -#define DEFAULT_MTVEC 0x8 - -// page table entry (PTE) fields -#define PTE_V 0x001 // Valid -#define PTE_TYPE 0x01E // Type -#define PTE_R 0x020 // Referenced -#define PTE_D 0x040 // Dirty -#define PTE_SOFT 0x380 // Reserved for Software - -#define PTE_TYPE_TABLE 0x00 -#define PTE_TYPE_TABLE_GLOBAL 0x02 -#define PTE_TYPE_URX_SR 0x04 -#define PTE_TYPE_URWX_SRW 0x06 -#define PTE_TYPE_UR_SR 0x08 -#define PTE_TYPE_URW_SRW 0x0A -#define PTE_TYPE_URX_SRX 0x0C -#define PTE_TYPE_URWX_SRWX 0x0E -#define PTE_TYPE_SR 0x10 -#define PTE_TYPE_SRW 0x12 -#define PTE_TYPE_SRX 0x14 -#define PTE_TYPE_SRWX 0x16 -#define PTE_TYPE_SR_GLOBAL 0x18 -#define PTE_TYPE_SRW_GLOBAL 0x1A -#define PTE_TYPE_SRX_GLOBAL 0x1C -#define PTE_TYPE_SRWX_GLOBAL 0x1E - -#define PTE_PPN_SHIFT 10 - -#define PTE_TABLE(PTE) ((0x0000000AU >> ((PTE) & 0x1F)) & 1) -#define PTE_UR(PTE) ((0x0000AAA0U >> ((PTE) & 0x1F)) & 1) -#define PTE_UW(PTE) ((0x00008880U >> ((PTE) & 0x1F)) & 1) -#define PTE_UX(PTE) ((0x0000A0A0U >> ((PTE) & 0x1F)) & 1) -#define PTE_SR(PTE) ((0xAAAAAAA0U >> ((PTE) & 0x1F)) & 1) -#define PTE_SW(PTE) ((0x88888880U >> ((PTE) & 0x1F)) & 1) -#define PTE_SX(PTE) ((0xA0A0A000U >> ((PTE) & 0x1F)) & 1) - -#define PTE_CHECK_PERM(PTE, SUPERVISOR, STORE, FETCH) \ - ((STORE) ? ((SUPERVISOR) ? PTE_SW(PTE) : PTE_UW(PTE)) : \ - (FETCH) ? ((SUPERVISOR) ? PTE_SX(PTE) : PTE_UX(PTE)) : \ - ((SUPERVISOR) ? PTE_SR(PTE) : PTE_UR(PTE))) - -#ifdef __riscv - -#ifdef __riscv64 -# define MSTATUS_SD MSTATUS64_SD -# define SSTATUS_SD SSTATUS64_SD -# define RISCV_PGLEVEL_BITS 9 -#else -# define MSTATUS_SD MSTATUS32_SD -# define SSTATUS_SD SSTATUS32_SD -# define RISCV_PGLEVEL_BITS 10 -#endif -#define RISCV_PGSHIFT 12 -#define RISCV_PGSIZE (1 << RISCV_PGSHIFT) - -#ifndef __ASSEMBLER__ - -#ifdef __GNUC__ - -#define read_csr(reg) ({ unsigned long __tmp; \ - asm volatile ("csrr %0, " #reg : "=r"(__tmp)); \ - __tmp; }) - -#define write_csr(reg, val) \ - asm volatile ("csrw " #reg ", %0" :: "r"(val)) - -#define swap_csr(reg, val) ({ long __tmp; \ - asm volatile ("csrrw %0, " #reg ", %1" : "=r"(__tmp) : "r"(val)); \ - __tmp; }) - -#define set_csr(reg, bit) ({ unsigned long __tmp; \ - if (__builtin_constant_p(bit) && (bit) < 32) \ - asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \ - else \ - asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \ - __tmp; }) - -#define clear_csr(reg, bit) ({ unsigned long __tmp; \ - if (__builtin_constant_p(bit) && (bit) < 32) \ - asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \ - else \ - asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \ - __tmp; }) - -#define rdtime() read_csr(time) -#define rdcycle() read_csr(cycle) -#define rdinstret() read_csr(instret) - -#endif - -#endif - -#endif - -#endif -/* Automatically generated by parse-opcodes */ -#ifndef RISCV_ENCODING_H -#define RISCV_ENCODING_H -#define MATCH_BEQ 0x63 -#define MASK_BEQ 0x707f -#define MATCH_BNE 0x1063 -#define MASK_BNE 0x707f -#define MATCH_BLT 0x4063 -#define MASK_BLT 0x707f -#define MATCH_BGE 0x5063 -#define MASK_BGE 0x707f -#define MATCH_BLTU 0x6063 -#define MASK_BLTU 0x707f -#define MATCH_BGEU 0x7063 -#define MASK_BGEU 0x707f -#define MATCH_JALR 0x67 -#define MASK_JALR 0x707f -#define MATCH_JAL 0x6f -#define MASK_JAL 0x7f -#define MATCH_LUI 0x37 -#define MASK_LUI 0x7f -#define MATCH_AUIPC 0x17 -#define MASK_AUIPC 0x7f -#define MATCH_ADDI 0x13 -#define MASK_ADDI 0x707f -#define MATCH_SLLI 0x1013 -#define MASK_SLLI 0xfc00707f -#define MATCH_SLTI 0x2013 -#define MASK_SLTI 0x707f -#define MATCH_SLTIU 0x3013 -#define MASK_SLTIU 0x707f -#define MATCH_XORI 0x4013 -#define MASK_XORI 0x707f -#define MATCH_SRLI 0x5013 -#define MASK_SRLI 0xfc00707f -#define MATCH_SRAI 0x40005013 -#define MASK_SRAI 0xfc00707f -#define MATCH_ORI 0x6013 -#define MASK_ORI 0x707f -#define MATCH_ANDI 0x7013 -#define MASK_ANDI 0x707f -#define MATCH_ADD 0x33 -#define MASK_ADD 0xfe00707f -#define MATCH_SUB 0x40000033 -#define MASK_SUB 0xfe00707f -#define MATCH_SLL 0x1033 -#define MASK_SLL 0xfe00707f -#define MATCH_SLT 0x2033 -#define MASK_SLT 0xfe00707f -#define MATCH_SLTU 0x3033 -#define MASK_SLTU 0xfe00707f -#define MATCH_XOR 0x4033 -#define MASK_XOR 0xfe00707f -#define MATCH_SRL 0x5033 -#define MASK_SRL 0xfe00707f -#define MATCH_SRA 0x40005033 -#define MASK_SRA 0xfe00707f -#define MATCH_OR 0x6033 -#define MASK_OR 0xfe00707f -#define MATCH_AND 0x7033 -#define MASK_AND 0xfe00707f -#define MATCH_ADDIW 0x1b -#define MASK_ADDIW 0x707f -#define MATCH_SLLIW 0x101b -#define MASK_SLLIW 0xfe00707f -#define MATCH_SRLIW 0x501b -#define MASK_SRLIW 0xfe00707f -#define MATCH_SRAIW 0x4000501b -#define MASK_SRAIW 0xfe00707f -#define MATCH_ADDW 0x3b -#define MASK_ADDW 0xfe00707f -#define MATCH_SUBW 0x4000003b -#define MASK_SUBW 0xfe00707f -#define MATCH_SLLW 0x103b -#define MASK_SLLW 0xfe00707f -#define MATCH_SRLW 0x503b -#define MASK_SRLW 0xfe00707f -#define MATCH_SRAW 0x4000503b -#define MASK_SRAW 0xfe00707f -#define MATCH_LB 0x3 -#define MASK_LB 0x707f -#define MATCH_LH 0x1003 -#define MASK_LH 0x707f -#define MATCH_LW 0x2003 -#define MASK_LW 0x707f -#define MATCH_LD 0x3003 -#define MASK_LD 0x707f -#define MATCH_LBU 0x4003 -#define MASK_LBU 0x707f -#define MATCH_LHU 0x5003 -#define MASK_LHU 0x707f -#define MATCH_LWU 0x6003 -#define MASK_LWU 0x707f -#define MATCH_SB 0x23 -#define MASK_SB 0x707f -#define MATCH_SH 0x1023 -#define MASK_SH 0x707f -#define MATCH_SW 0x2023 -#define MASK_SW 0x707f -#define MATCH_SD 0x3023 -#define MASK_SD 0x707f -#define MATCH_FENCE 0xf -#define MASK_FENCE 0x707f -#define MATCH_FENCE_I 0x100f -#define MASK_FENCE_I 0x707f -#define MATCH_MUL 0x2000033 -#define MASK_MUL 0xfe00707f -#define MATCH_MULH 0x2001033 -#define MASK_MULH 0xfe00707f -#define MATCH_MULHSU 0x2002033 -#define MASK_MULHSU 0xfe00707f -#define MATCH_MULHU 0x2003033 -#define MASK_MULHU 0xfe00707f -#define MATCH_DIV 0x2004033 -#define MASK_DIV 0xfe00707f -#define MATCH_DIVU 0x2005033 -#define MASK_DIVU 0xfe00707f -#define MATCH_REM 0x2006033 -#define MASK_REM 0xfe00707f -#define MATCH_REMU 0x2007033 -#define MASK_REMU 0xfe00707f -#define MATCH_MULW 0x200003b -#define MASK_MULW 0xfe00707f -#define MATCH_DIVW 0x200403b -#define MASK_DIVW 0xfe00707f -#define MATCH_DIVUW 0x200503b -#define MASK_DIVUW 0xfe00707f -#define MATCH_REMW 0x200603b -#define MASK_REMW 0xfe00707f -#define MATCH_REMUW 0x200703b -#define MASK_REMUW 0xfe00707f -#define MATCH_AMOADD_W 0x202f -#define MASK_AMOADD_W 0xf800707f -#define MATCH_AMOXOR_W 0x2000202f -#define MASK_AMOXOR_W 0xf800707f -#define MATCH_AMOOR_W 0x4000202f -#define MASK_AMOOR_W 0xf800707f -#define MATCH_AMOAND_W 0x6000202f -#define MASK_AMOAND_W 0xf800707f -#define MATCH_AMOMIN_W 0x8000202f -#define MASK_AMOMIN_W 0xf800707f -#define MATCH_AMOMAX_W 0xa000202f -#define MASK_AMOMAX_W 0xf800707f -#define MATCH_AMOMINU_W 0xc000202f -#define MASK_AMOMINU_W 0xf800707f -#define MATCH_AMOMAXU_W 0xe000202f -#define MASK_AMOMAXU_W 0xf800707f -#define MATCH_AMOSWAP_W 0x800202f -#define MASK_AMOSWAP_W 0xf800707f -#define MATCH_LR_W 0x1000202f -#define MASK_LR_W 0xf9f0707f -#define MATCH_SC_W 0x1800202f -#define MASK_SC_W 0xf800707f -#define MATCH_AMOADD_D 0x302f -#define MASK_AMOADD_D 0xf800707f -#define MATCH_AMOXOR_D 0x2000302f -#define MASK_AMOXOR_D 0xf800707f -#define MATCH_AMOOR_D 0x4000302f -#define MASK_AMOOR_D 0xf800707f -#define MATCH_AMOAND_D 0x6000302f -#define MASK_AMOAND_D 0xf800707f -#define MATCH_AMOMIN_D 0x8000302f -#define MASK_AMOMIN_D 0xf800707f -#define MATCH_AMOMAX_D 0xa000302f -#define MASK_AMOMAX_D 0xf800707f -#define MATCH_AMOMINU_D 0xc000302f -#define MASK_AMOMINU_D 0xf800707f -#define MATCH_AMOMAXU_D 0xe000302f -#define MASK_AMOMAXU_D 0xf800707f -#define MATCH_AMOSWAP_D 0x800302f -#define MASK_AMOSWAP_D 0xf800707f -#define MATCH_LR_D 0x1000302f -#define MASK_LR_D 0xf9f0707f -#define MATCH_SC_D 0x1800302f -#define MASK_SC_D 0xf800707f -#define MATCH_SCALL 0x73 -#define MASK_SCALL 0xffffffff -#define MATCH_SBREAK 0x100073 -#define MASK_SBREAK 0xffffffff -#define MATCH_SRET 0x10200073 -#define MASK_SRET 0xffffffff -#define MATCH_SFENCE_VM 0x10400073 -#define MASK_SFENCE_VM 0xfff07fff -#define MATCH_WFI 0x10500073 -#define MASK_WFI 0xffffffff -#define MATCH_CSRRW 0x1073 -#define MASK_CSRRW 0x707f -#define MATCH_CSRRS 0x2073 -#define MASK_CSRRS 0x707f -#define MATCH_CSRRC 0x3073 -#define MASK_CSRRC 0x707f -#define MATCH_CSRRWI 0x5073 -#define MASK_CSRRWI 0x707f -#define MATCH_CSRRSI 0x6073 -#define MASK_CSRRSI 0x707f -#define MATCH_CSRRCI 0x7073 -#define MASK_CSRRCI 0x707f -#define MATCH_FADD_S 0x53 -#define MASK_FADD_S 0xfe00007f -#define MATCH_FSUB_S 0x8000053 -#define MASK_FSUB_S 0xfe00007f -#define MATCH_FMUL_S 0x10000053 -#define MASK_FMUL_S 0xfe00007f -#define MATCH_FDIV_S 0x18000053 -#define MASK_FDIV_S 0xfe00007f -#define MATCH_FSGNJ_S 0x20000053 -#define MASK_FSGNJ_S 0xfe00707f -#define MATCH_FSGNJN_S 0x20001053 -#define MASK_FSGNJN_S 0xfe00707f -#define MATCH_FSGNJX_S 0x20002053 -#define MASK_FSGNJX_S 0xfe00707f -#define MATCH_FMIN_S 0x28000053 -#define MASK_FMIN_S 0xfe00707f -#define MATCH_FMAX_S 0x28001053 -#define MASK_FMAX_S 0xfe00707f -#define MATCH_FSQRT_S 0x58000053 -#define MASK_FSQRT_S 0xfff0007f -#define MATCH_FADD_D 0x2000053 -#define MASK_FADD_D 0xfe00007f -#define MATCH_FSUB_D 0xa000053 -#define MASK_FSUB_D 0xfe00007f -#define MATCH_FMUL_D 0x12000053 -#define MASK_FMUL_D 0xfe00007f -#define MATCH_FDIV_D 0x1a000053 -#define MASK_FDIV_D 0xfe00007f -#define MATCH_FSGNJ_D 0x22000053 -#define MASK_FSGNJ_D 0xfe00707f -#define MATCH_FSGNJN_D 0x22001053 -#define MASK_FSGNJN_D 0xfe00707f -#define MATCH_FSGNJX_D 0x22002053 -#define MASK_FSGNJX_D 0xfe00707f -#define MATCH_FMIN_D 0x2a000053 -#define MASK_FMIN_D 0xfe00707f -#define MATCH_FMAX_D 0x2a001053 -#define MASK_FMAX_D 0xfe00707f -#define MATCH_FCVT_S_D 0x40100053 -#define MASK_FCVT_S_D 0xfff0007f -#define MATCH_FCVT_D_S 0x42000053 -#define MASK_FCVT_D_S 0xfff0007f -#define MATCH_FSQRT_D 0x5a000053 -#define MASK_FSQRT_D 0xfff0007f -#define MATCH_FLE_S 0xa0000053 -#define MASK_FLE_S 0xfe00707f -#define MATCH_FLT_S 0xa0001053 -#define MASK_FLT_S 0xfe00707f -#define MATCH_FEQ_S 0xa0002053 -#define MASK_FEQ_S 0xfe00707f -#define MATCH_FLE_D 0xa2000053 -#define MASK_FLE_D 0xfe00707f -#define MATCH_FLT_D 0xa2001053 -#define MASK_FLT_D 0xfe00707f -#define MATCH_FEQ_D 0xa2002053 -#define MASK_FEQ_D 0xfe00707f -#define MATCH_FCVT_W_S 0xc0000053 -#define MASK_FCVT_W_S 0xfff0007f -#define MATCH_FCVT_WU_S 0xc0100053 -#define MASK_FCVT_WU_S 0xfff0007f -#define MATCH_FCVT_L_S 0xc0200053 -#define MASK_FCVT_L_S 0xfff0007f -#define MATCH_FCVT_LU_S 0xc0300053 -#define MASK_FCVT_LU_S 0xfff0007f -#define MATCH_FMV_X_S 0xe0000053 -#define MASK_FMV_X_S 0xfff0707f -#define MATCH_FCLASS_S 0xe0001053 -#define MASK_FCLASS_S 0xfff0707f -#define MATCH_FCVT_W_D 0xc2000053 -#define MASK_FCVT_W_D 0xfff0007f -#define MATCH_FCVT_WU_D 0xc2100053 -#define MASK_FCVT_WU_D 0xfff0007f -#define MATCH_FCVT_L_D 0xc2200053 -#define MASK_FCVT_L_D 0xfff0007f -#define MATCH_FCVT_LU_D 0xc2300053 -#define MASK_FCVT_LU_D 0xfff0007f -#define MATCH_FMV_X_D 0xe2000053 -#define MASK_FMV_X_D 0xfff0707f -#define MATCH_FCLASS_D 0xe2001053 -#define MASK_FCLASS_D 0xfff0707f -#define MATCH_FCVT_S_W 0xd0000053 -#define MASK_FCVT_S_W 0xfff0007f -#define MATCH_FCVT_S_WU 0xd0100053 -#define MASK_FCVT_S_WU 0xfff0007f -#define MATCH_FCVT_S_L 0xd0200053 -#define MASK_FCVT_S_L 0xfff0007f -#define MATCH_FCVT_S_LU 0xd0300053 -#define MASK_FCVT_S_LU 0xfff0007f -#define MATCH_FMV_S_X 0xf0000053 -#define MASK_FMV_S_X 0xfff0707f -#define MATCH_FCVT_D_W 0xd2000053 -#define MASK_FCVT_D_W 0xfff0007f -#define MATCH_FCVT_D_WU 0xd2100053 -#define MASK_FCVT_D_WU 0xfff0007f -#define MATCH_FCVT_D_L 0xd2200053 -#define MASK_FCVT_D_L 0xfff0007f -#define MATCH_FCVT_D_LU 0xd2300053 -#define MASK_FCVT_D_LU 0xfff0007f -#define MATCH_FMV_D_X 0xf2000053 -#define MASK_FMV_D_X 0xfff0707f -#define MATCH_FLW 0x2007 -#define MASK_FLW 0x707f -#define MATCH_FLD 0x3007 -#define MASK_FLD 0x707f -#define MATCH_FSW 0x2027 -#define MASK_FSW 0x707f -#define MATCH_FSD 0x3027 -#define MASK_FSD 0x707f -#define MATCH_FMADD_S 0x43 -#define MASK_FMADD_S 0x600007f -#define MATCH_FMSUB_S 0x47 -#define MASK_FMSUB_S 0x600007f -#define MATCH_FNMSUB_S 0x4b -#define MASK_FNMSUB_S 0x600007f -#define MATCH_FNMADD_S 0x4f -#define MASK_FNMADD_S 0x600007f -#define MATCH_FMADD_D 0x2000043 -#define MASK_FMADD_D 0x600007f -#define MATCH_FMSUB_D 0x2000047 -#define MASK_FMSUB_D 0x600007f -#define MATCH_FNMSUB_D 0x200004b -#define MASK_FNMSUB_D 0x600007f -#define MATCH_FNMADD_D 0x200004f -#define MASK_FNMADD_D 0x600007f -#define MATCH_C_NOP 0x1 -#define MASK_C_NOP 0xffff -#define MATCH_C_ADDI16SP 0x6101 -#define MASK_C_ADDI16SP 0xef83 -#define MATCH_C_JR 0x8002 -#define MASK_C_JR 0xf07f -#define MATCH_C_JALR 0x9002 -#define MASK_C_JALR 0xf07f -#define MATCH_C_EBREAK 0x9002 -#define MASK_C_EBREAK 0xffff -#define MATCH_C_LD 0x6000 -#define MASK_C_LD 0xe003 -#define MATCH_C_SD 0xe000 -#define MASK_C_SD 0xe003 -#define MATCH_C_ADDIW 0x2001 -#define MASK_C_ADDIW 0xe003 -#define MATCH_C_LDSP 0x6002 -#define MASK_C_LDSP 0xe003 -#define MATCH_C_SDSP 0xe002 -#define MASK_C_SDSP 0xe003 -#define MATCH_C_ADDI4SPN 0x0 -#define MASK_C_ADDI4SPN 0xe003 -#define MATCH_C_FLD 0x2000 -#define MASK_C_FLD 0xe003 -#define MATCH_C_LW 0x4000 -#define MASK_C_LW 0xe003 -#define MATCH_C_FLW 0x6000 -#define MASK_C_FLW 0xe003 -#define MATCH_C_FSD 0xa000 -#define MASK_C_FSD 0xe003 -#define MATCH_C_SW 0xc000 -#define MASK_C_SW 0xe003 -#define MATCH_C_FSW 0xe000 -#define MASK_C_FSW 0xe003 -#define MATCH_C_ADDI 0x1 -#define MASK_C_ADDI 0xe003 -#define MATCH_C_JAL 0x2001 -#define MASK_C_JAL 0xe003 -#define MATCH_C_LI 0x4001 -#define MASK_C_LI 0xe003 -#define MATCH_C_LUI 0x6001 -#define MASK_C_LUI 0xe003 -#define MATCH_C_SRLI 0x8001 -#define MASK_C_SRLI 0xec03 -#define MATCH_C_SRAI 0x8401 -#define MASK_C_SRAI 0xec03 -#define MATCH_C_ANDI 0x8801 -#define MASK_C_ANDI 0xec03 -#define MATCH_C_SUB 0x8c01 -#define MASK_C_SUB 0xfc63 -#define MATCH_C_XOR 0x8c21 -#define MASK_C_XOR 0xfc63 -#define MATCH_C_OR 0x8c41 -#define MASK_C_OR 0xfc63 -#define MATCH_C_AND 0x8c61 -#define MASK_C_AND 0xfc63 -#define MATCH_C_SUBW 0x9c01 -#define MASK_C_SUBW 0xfc63 -#define MATCH_C_ADDW 0x9c21 -#define MASK_C_ADDW 0xfc63 -#define MATCH_C_J 0xa001 -#define MASK_C_J 0xe003 -#define MATCH_C_BEQZ 0xc001 -#define MASK_C_BEQZ 0xe003 -#define MATCH_C_BNEZ 0xe001 -#define MASK_C_BNEZ 0xe003 -#define MATCH_C_SLLI 0x2 -#define MASK_C_SLLI 0xe003 -#define MATCH_C_FLDSP 0x2002 -#define MASK_C_FLDSP 0xe003 -#define MATCH_C_LWSP 0x4002 -#define MASK_C_LWSP 0xe003 -#define MATCH_C_FLWSP 0x6002 -#define MASK_C_FLWSP 0xe003 -#define MATCH_C_MV 0x8002 -#define MASK_C_MV 0xf003 -#define MATCH_C_ADD 0x9002 -#define MASK_C_ADD 0xf003 -#define MATCH_C_FSDSP 0xa002 -#define MASK_C_FSDSP 0xe003 -#define MATCH_C_SWSP 0xc002 -#define MASK_C_SWSP 0xe003 -#define MATCH_C_FSWSP 0xe002 -#define MASK_C_FSWSP 0xe003 -#define MATCH_CUSTOM0 0xb -#define MASK_CUSTOM0 0x707f -#define MATCH_CUSTOM0_RS1 0x200b -#define MASK_CUSTOM0_RS1 0x707f -#define MATCH_CUSTOM0_RS1_RS2 0x300b -#define MASK_CUSTOM0_RS1_RS2 0x707f -#define MATCH_CUSTOM0_RD 0x400b -#define MASK_CUSTOM0_RD 0x707f -#define MATCH_CUSTOM0_RD_RS1 0x600b -#define MASK_CUSTOM0_RD_RS1 0x707f -#define MATCH_CUSTOM0_RD_RS1_RS2 0x700b -#define MASK_CUSTOM0_RD_RS1_RS2 0x707f -#define MATCH_CUSTOM1 0x2b -#define MASK_CUSTOM1 0x707f -#define MATCH_CUSTOM1_RS1 0x202b -#define MASK_CUSTOM1_RS1 0x707f -#define MATCH_CUSTOM1_RS1_RS2 0x302b -#define MASK_CUSTOM1_RS1_RS2 0x707f -#define MATCH_CUSTOM1_RD 0x402b -#define MASK_CUSTOM1_RD 0x707f -#define MATCH_CUSTOM1_RD_RS1 0x602b -#define MASK_CUSTOM1_RD_RS1 0x707f -#define MATCH_CUSTOM1_RD_RS1_RS2 0x702b -#define MASK_CUSTOM1_RD_RS1_RS2 0x707f -#define MATCH_CUSTOM2 0x5b -#define MASK_CUSTOM2 0x707f -#define MATCH_CUSTOM2_RS1 0x205b -#define MASK_CUSTOM2_RS1 0x707f -#define MATCH_CUSTOM2_RS1_RS2 0x305b -#define MASK_CUSTOM2_RS1_RS2 0x707f -#define MATCH_CUSTOM2_RD 0x405b -#define MASK_CUSTOM2_RD 0x707f -#define MATCH_CUSTOM2_RD_RS1 0x605b -#define MASK_CUSTOM2_RD_RS1 0x707f -#define MATCH_CUSTOM2_RD_RS1_RS2 0x705b -#define MASK_CUSTOM2_RD_RS1_RS2 0x707f -#define MATCH_CUSTOM3 0x7b -#define MASK_CUSTOM3 0x707f -#define MATCH_CUSTOM3_RS1 0x207b -#define MASK_CUSTOM3_RS1 0x707f -#define MATCH_CUSTOM3_RS1_RS2 0x307b -#define MASK_CUSTOM3_RS1_RS2 0x707f -#define MATCH_CUSTOM3_RD 0x407b -#define MASK_CUSTOM3_RD 0x707f -#define MATCH_CUSTOM3_RD_RS1 0x607b -#define MASK_CUSTOM3_RD_RS1 0x707f -#define MATCH_CUSTOM3_RD_RS1_RS2 0x707b -#define MASK_CUSTOM3_RD_RS1_RS2 0x707f -#define CSR_FFLAGS 0x1 -#define CSR_FRM 0x2 -#define CSR_FCSR 0x3 -#define CSR_CYCLE 0xc00 -#define CSR_TIME 0xc01 -#define CSR_INSTRET 0xc02 -#define CSR_STATS 0xc0 -#define CSR_UARCH0 0xcc0 -#define CSR_UARCH1 0xcc1 -#define CSR_UARCH2 0xcc2 -#define CSR_UARCH3 0xcc3 -#define CSR_UARCH4 0xcc4 -#define CSR_UARCH5 0xcc5 -#define CSR_UARCH6 0xcc6 -#define CSR_UARCH7 0xcc7 -#define CSR_UARCH8 0xcc8 -#define CSR_UARCH9 0xcc9 -#define CSR_UARCH10 0xcca -#define CSR_UARCH11 0xccb -#define CSR_UARCH12 0xccc -#define CSR_UARCH13 0xccd -#define CSR_UARCH14 0xcce -#define CSR_UARCH15 0xccf -#define CSR_SSTATUS 0x100 -#define CSR_SIE 0x104 -#define CSR_STVEC 0x105 -#define CSR_SSCRATCH 0x140 -#define CSR_SEPC 0x141 -#define CSR_SCAUSE 0x142 -#define CSR_SBADADDR 0x143 -#define CSR_SIP 0x144 -#define CSR_SPTBR 0x180 -#define CSR_SASID 0x181 -#define CSR_SCYCLE 0xd00 -#define CSR_STIME 0xd01 -#define CSR_SINSTRET 0xd02 -#define CSR_MSTATUS 0x300 -#define CSR_MEDELEG 0x302 -#define CSR_MIDELEG 0x303 -#define CSR_MIE 0x304 -#define CSR_MTVEC 0x305 -#define CSR_MTIMECMP 0x321 -#define CSR_MSCRATCH 0x340 -#define CSR_MEPC 0x341 -#define CSR_MCAUSE 0x342 -#define CSR_MBADADDR 0x343 -#define CSR_MIP 0x344 -#define CSR_MIPI 0x345 -#define CSR_MUCOUNTEREN 0x310 -#define CSR_MSCOUNTEREN 0x311 -#define CSR_MUCYCLE_DELTA 0x700 -#define CSR_MUTIME_DELTA 0x701 -#define CSR_MUINSTRET_DELTA 0x702 -#define CSR_MSCYCLE_DELTA 0x704 -#define CSR_MSTIME_DELTA 0x705 -#define CSR_MSINSTRET_DELTA 0x706 -#define CSR_MCYCLE 0xf00 -#define CSR_MTIME 0xf01 -#define CSR_MINSTRET 0xf02 -#define CSR_MISA 0xf10 -#define CSR_MVENDORID 0xf11 -#define CSR_MARCHID 0xf12 -#define CSR_MIMPID 0xf13 -#define CSR_MCFGADDR 0xf14 -#define CSR_MHARTID 0xf15 -#define CSR_MTOHOST 0x7c0 -#define CSR_MFROMHOST 0x7c1 -#define CSR_MRESET 0x7c2 -#define CSR_CYCLEH 0xc80 -#define CSR_TIMEH 0xc81 -#define CSR_INSTRETH 0xc82 -#define CSR_MTIMECMPH 0x361 -#define CSR_MUCYCLE_DELTAH 0x780 -#define CSR_MUTIME_DELTAH 0x781 -#define CSR_MUINSTRET_DELTAH 0x782 -#define CSR_MSCYCLE_DELTAH 0x784 -#define CSR_MSTIME_DELTAH 0x785 -#define CSR_MSINSTRET_DELTAH 0x786 -#define CSR_MCYCLEH 0xf80 -#define CSR_MTIMEH 0xf81 -#define CSR_MINSTRETH 0xf82 -#define CAUSE_MISALIGNED_FETCH 0x0 -#define CAUSE_FAULT_FETCH 0x1 -#define CAUSE_ILLEGAL_INSTRUCTION 0x2 -#define CAUSE_BREAKPOINT 0x3 -#define CAUSE_MISALIGNED_LOAD 0x4 -#define CAUSE_FAULT_LOAD 0x5 -#define CAUSE_MISALIGNED_STORE 0x6 -#define CAUSE_FAULT_STORE 0x7 -#define CAUSE_USER_ECALL 0x8 -#define CAUSE_SUPERVISOR_ECALL 0x9 -#define CAUSE_HYPERVISOR_ECALL 0xa -#define CAUSE_MACHINE_ECALL 0xb -#endif -#ifdef DECLARE_INSN -DECLARE_INSN(beq, MATCH_BEQ, MASK_BEQ) -DECLARE_INSN(bne, MATCH_BNE, MASK_BNE) -DECLARE_INSN(blt, MATCH_BLT, MASK_BLT) -DECLARE_INSN(bge, MATCH_BGE, MASK_BGE) -DECLARE_INSN(bltu, MATCH_BLTU, MASK_BLTU) -DECLARE_INSN(bgeu, MATCH_BGEU, MASK_BGEU) -DECLARE_INSN(jalr, MATCH_JALR, MASK_JALR) -DECLARE_INSN(jal, MATCH_JAL, MASK_JAL) -DECLARE_INSN(lui, MATCH_LUI, MASK_LUI) -DECLARE_INSN(auipc, MATCH_AUIPC, MASK_AUIPC) -DECLARE_INSN(addi, MATCH_ADDI, MASK_ADDI) -DECLARE_INSN(slli, MATCH_SLLI, MASK_SLLI) -DECLARE_INSN(slti, MATCH_SLTI, MASK_SLTI) -DECLARE_INSN(sltiu, MATCH_SLTIU, MASK_SLTIU) -DECLARE_INSN(xori, MATCH_XORI, MASK_XORI) -DECLARE_INSN(srli, MATCH_SRLI, MASK_SRLI) -DECLARE_INSN(srai, MATCH_SRAI, MASK_SRAI) -DECLARE_INSN(ori, MATCH_ORI, MASK_ORI) -DECLARE_INSN(andi, MATCH_ANDI, MASK_ANDI) -DECLARE_INSN(add, MATCH_ADD, MASK_ADD) -DECLARE_INSN(sub, MATCH_SUB, MASK_SUB) -DECLARE_INSN(sll, MATCH_SLL, MASK_SLL) -DECLARE_INSN(slt, MATCH_SLT, MASK_SLT) -DECLARE_INSN(sltu, MATCH_SLTU, MASK_SLTU) -DECLARE_INSN(xor, MATCH_XOR, MASK_XOR) -DECLARE_INSN(srl, MATCH_SRL, MASK_SRL) -DECLARE_INSN(sra, MATCH_SRA, MASK_SRA) -DECLARE_INSN(or, MATCH_OR, MASK_OR) -DECLARE_INSN(and, MATCH_AND, MASK_AND) -DECLARE_INSN(addiw, MATCH_ADDIW, MASK_ADDIW) -DECLARE_INSN(slliw, MATCH_SLLIW, MASK_SLLIW) -DECLARE_INSN(srliw, MATCH_SRLIW, MASK_SRLIW) -DECLARE_INSN(sraiw, MATCH_SRAIW, MASK_SRAIW) -DECLARE_INSN(addw, MATCH_ADDW, MASK_ADDW) -DECLARE_INSN(subw, MATCH_SUBW, MASK_SUBW) -DECLARE_INSN(sllw, MATCH_SLLW, MASK_SLLW) -DECLARE_INSN(srlw, MATCH_SRLW, MASK_SRLW) -DECLARE_INSN(sraw, MATCH_SRAW, MASK_SRAW) -DECLARE_INSN(lb, MATCH_LB, MASK_LB) -DECLARE_INSN(lh, MATCH_LH, MASK_LH) -DECLARE_INSN(lw, MATCH_LW, MASK_LW) -DECLARE_INSN(ld, MATCH_LD, MASK_LD) -DECLARE_INSN(lbu, MATCH_LBU, MASK_LBU) -DECLARE_INSN(lhu, MATCH_LHU, MASK_LHU) -DECLARE_INSN(lwu, MATCH_LWU, MASK_LWU) -DECLARE_INSN(sb, MATCH_SB, MASK_SB) -DECLARE_INSN(sh, MATCH_SH, MASK_SH) -DECLARE_INSN(sw, MATCH_SW, MASK_SW) -DECLARE_INSN(sd, MATCH_SD, MASK_SD) -DECLARE_INSN(fence, MATCH_FENCE, MASK_FENCE) -DECLARE_INSN(fence_i, MATCH_FENCE_I, MASK_FENCE_I) -DECLARE_INSN(mul, MATCH_MUL, MASK_MUL) -DECLARE_INSN(mulh, MATCH_MULH, MASK_MULH) -DECLARE_INSN(mulhsu, MATCH_MULHSU, MASK_MULHSU) -DECLARE_INSN(mulhu, MATCH_MULHU, MASK_MULHU) -DECLARE_INSN(div, MATCH_DIV, MASK_DIV) -DECLARE_INSN(divu, MATCH_DIVU, MASK_DIVU) -DECLARE_INSN(rem, MATCH_REM, MASK_REM) -DECLARE_INSN(remu, MATCH_REMU, MASK_REMU) -DECLARE_INSN(mulw, MATCH_MULW, MASK_MULW) -DECLARE_INSN(divw, MATCH_DIVW, MASK_DIVW) -DECLARE_INSN(divuw, MATCH_DIVUW, MASK_DIVUW) -DECLARE_INSN(remw, MATCH_REMW, MASK_REMW) -DECLARE_INSN(remuw, MATCH_REMUW, MASK_REMUW) -DECLARE_INSN(amoadd_w, MATCH_AMOADD_W, MASK_AMOADD_W) -DECLARE_INSN(amoxor_w, MATCH_AMOXOR_W, MASK_AMOXOR_W) -DECLARE_INSN(amoor_w, MATCH_AMOOR_W, MASK_AMOOR_W) -DECLARE_INSN(amoand_w, MATCH_AMOAND_W, MASK_AMOAND_W) -DECLARE_INSN(amomin_w, MATCH_AMOMIN_W, MASK_AMOMIN_W) -DECLARE_INSN(amomax_w, MATCH_AMOMAX_W, MASK_AMOMAX_W) -DECLARE_INSN(amominu_w, MATCH_AMOMINU_W, MASK_AMOMINU_W) -DECLARE_INSN(amomaxu_w, MATCH_AMOMAXU_W, MASK_AMOMAXU_W) -DECLARE_INSN(amoswap_w, MATCH_AMOSWAP_W, MASK_AMOSWAP_W) -DECLARE_INSN(lr_w, MATCH_LR_W, MASK_LR_W) -DECLARE_INSN(sc_w, MATCH_SC_W, MASK_SC_W) -DECLARE_INSN(amoadd_d, MATCH_AMOADD_D, MASK_AMOADD_D) -DECLARE_INSN(amoxor_d, MATCH_AMOXOR_D, MASK_AMOXOR_D) -DECLARE_INSN(amoor_d, MATCH_AMOOR_D, MASK_AMOOR_D) -DECLARE_INSN(amoand_d, MATCH_AMOAND_D, MASK_AMOAND_D) -DECLARE_INSN(amomin_d, MATCH_AMOMIN_D, MASK_AMOMIN_D) -DECLARE_INSN(amomax_d, MATCH_AMOMAX_D, MASK_AMOMAX_D) -DECLARE_INSN(amominu_d, MATCH_AMOMINU_D, MASK_AMOMINU_D) -DECLARE_INSN(amomaxu_d, MATCH_AMOMAXU_D, MASK_AMOMAXU_D) -DECLARE_INSN(amoswap_d, MATCH_AMOSWAP_D, MASK_AMOSWAP_D) -DECLARE_INSN(lr_d, MATCH_LR_D, MASK_LR_D) -DECLARE_INSN(sc_d, MATCH_SC_D, MASK_SC_D) -DECLARE_INSN(scall, MATCH_SCALL, MASK_SCALL) -DECLARE_INSN(sbreak, MATCH_SBREAK, MASK_SBREAK) -DECLARE_INSN(sret, MATCH_SRET, MASK_SRET) -DECLARE_INSN(sfence_vm, MATCH_SFENCE_VM, MASK_SFENCE_VM) -DECLARE_INSN(wfi, MATCH_WFI, MASK_WFI) -DECLARE_INSN(csrrw, MATCH_CSRRW, MASK_CSRRW) -DECLARE_INSN(csrrs, MATCH_CSRRS, MASK_CSRRS) -DECLARE_INSN(csrrc, MATCH_CSRRC, MASK_CSRRC) -DECLARE_INSN(csrrwi, MATCH_CSRRWI, MASK_CSRRWI) -DECLARE_INSN(csrrsi, MATCH_CSRRSI, MASK_CSRRSI) -DECLARE_INSN(csrrci, MATCH_CSRRCI, MASK_CSRRCI) -DECLARE_INSN(fadd_s, MATCH_FADD_S, MASK_FADD_S) -DECLARE_INSN(fsub_s, MATCH_FSUB_S, MASK_FSUB_S) -DECLARE_INSN(fmul_s, MATCH_FMUL_S, MASK_FMUL_S) -DECLARE_INSN(fdiv_s, MATCH_FDIV_S, MASK_FDIV_S) -DECLARE_INSN(fsgnj_s, MATCH_FSGNJ_S, MASK_FSGNJ_S) -DECLARE_INSN(fsgnjn_s, MATCH_FSGNJN_S, MASK_FSGNJN_S) -DECLARE_INSN(fsgnjx_s, MATCH_FSGNJX_S, MASK_FSGNJX_S) -DECLARE_INSN(fmin_s, MATCH_FMIN_S, MASK_FMIN_S) -DECLARE_INSN(fmax_s, MATCH_FMAX_S, MASK_FMAX_S) -DECLARE_INSN(fsqrt_s, MATCH_FSQRT_S, MASK_FSQRT_S) -DECLARE_INSN(fadd_d, MATCH_FADD_D, MASK_FADD_D) -DECLARE_INSN(fsub_d, MATCH_FSUB_D, MASK_FSUB_D) -DECLARE_INSN(fmul_d, MATCH_FMUL_D, MASK_FMUL_D) -DECLARE_INSN(fdiv_d, MATCH_FDIV_D, MASK_FDIV_D) -DECLARE_INSN(fsgnj_d, MATCH_FSGNJ_D, MASK_FSGNJ_D) -DECLARE_INSN(fsgnjn_d, MATCH_FSGNJN_D, MASK_FSGNJN_D) -DECLARE_INSN(fsgnjx_d, MATCH_FSGNJX_D, MASK_FSGNJX_D) -DECLARE_INSN(fmin_d, MATCH_FMIN_D, MASK_FMIN_D) -DECLARE_INSN(fmax_d, MATCH_FMAX_D, MASK_FMAX_D) -DECLARE_INSN(fcvt_s_d, MATCH_FCVT_S_D, MASK_FCVT_S_D) -DECLARE_INSN(fcvt_d_s, MATCH_FCVT_D_S, MASK_FCVT_D_S) -DECLARE_INSN(fsqrt_d, MATCH_FSQRT_D, MASK_FSQRT_D) -DECLARE_INSN(fle_s, MATCH_FLE_S, MASK_FLE_S) -DECLARE_INSN(flt_s, MATCH_FLT_S, MASK_FLT_S) -DECLARE_INSN(feq_s, MATCH_FEQ_S, MASK_FEQ_S) -DECLARE_INSN(fle_d, MATCH_FLE_D, MASK_FLE_D) -DECLARE_INSN(flt_d, MATCH_FLT_D, MASK_FLT_D) -DECLARE_INSN(feq_d, MATCH_FEQ_D, MASK_FEQ_D) -DECLARE_INSN(fcvt_w_s, MATCH_FCVT_W_S, MASK_FCVT_W_S) -DECLARE_INSN(fcvt_wu_s, MATCH_FCVT_WU_S, MASK_FCVT_WU_S) -DECLARE_INSN(fcvt_l_s, MATCH_FCVT_L_S, MASK_FCVT_L_S) -DECLARE_INSN(fcvt_lu_s, MATCH_FCVT_LU_S, MASK_FCVT_LU_S) -DECLARE_INSN(fmv_x_s, MATCH_FMV_X_S, MASK_FMV_X_S) -DECLARE_INSN(fclass_s, MATCH_FCLASS_S, MASK_FCLASS_S) -DECLARE_INSN(fcvt_w_d, MATCH_FCVT_W_D, MASK_FCVT_W_D) -DECLARE_INSN(fcvt_wu_d, MATCH_FCVT_WU_D, MASK_FCVT_WU_D) -DECLARE_INSN(fcvt_l_d, MATCH_FCVT_L_D, MASK_FCVT_L_D) -DECLARE_INSN(fcvt_lu_d, MATCH_FCVT_LU_D, MASK_FCVT_LU_D) -DECLARE_INSN(fmv_x_d, MATCH_FMV_X_D, MASK_FMV_X_D) -DECLARE_INSN(fclass_d, MATCH_FCLASS_D, MASK_FCLASS_D) -DECLARE_INSN(fcvt_s_w, MATCH_FCVT_S_W, MASK_FCVT_S_W) -DECLARE_INSN(fcvt_s_wu, MATCH_FCVT_S_WU, MASK_FCVT_S_WU) -DECLARE_INSN(fcvt_s_l, MATCH_FCVT_S_L, MASK_FCVT_S_L) -DECLARE_INSN(fcvt_s_lu, MATCH_FCVT_S_LU, MASK_FCVT_S_LU) -DECLARE_INSN(fmv_s_x, MATCH_FMV_S_X, MASK_FMV_S_X) -DECLARE_INSN(fcvt_d_w, MATCH_FCVT_D_W, MASK_FCVT_D_W) -DECLARE_INSN(fcvt_d_wu, MATCH_FCVT_D_WU, MASK_FCVT_D_WU) -DECLARE_INSN(fcvt_d_l, MATCH_FCVT_D_L, MASK_FCVT_D_L) -DECLARE_INSN(fcvt_d_lu, MATCH_FCVT_D_LU, MASK_FCVT_D_LU) -DECLARE_INSN(fmv_d_x, MATCH_FMV_D_X, MASK_FMV_D_X) -DECLARE_INSN(flw, MATCH_FLW, MASK_FLW) -DECLARE_INSN(fld, MATCH_FLD, MASK_FLD) -DECLARE_INSN(fsw, MATCH_FSW, MASK_FSW) -DECLARE_INSN(fsd, MATCH_FSD, MASK_FSD) -DECLARE_INSN(fmadd_s, MATCH_FMADD_S, MASK_FMADD_S) -DECLARE_INSN(fmsub_s, MATCH_FMSUB_S, MASK_FMSUB_S) -DECLARE_INSN(fnmsub_s, MATCH_FNMSUB_S, MASK_FNMSUB_S) -DECLARE_INSN(fnmadd_s, MATCH_FNMADD_S, MASK_FNMADD_S) -DECLARE_INSN(fmadd_d, MATCH_FMADD_D, MASK_FMADD_D) -DECLARE_INSN(fmsub_d, MATCH_FMSUB_D, MASK_FMSUB_D) -DECLARE_INSN(fnmsub_d, MATCH_FNMSUB_D, MASK_FNMSUB_D) -DECLARE_INSN(fnmadd_d, MATCH_FNMADD_D, MASK_FNMADD_D) -DECLARE_INSN(c_nop, MATCH_C_NOP, MASK_C_NOP) -DECLARE_INSN(c_addi16sp, MATCH_C_ADDI16SP, MASK_C_ADDI16SP) -DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR) -DECLARE_INSN(c_jalr, MATCH_C_JALR, MASK_C_JALR) -DECLARE_INSN(c_ebreak, MATCH_C_EBREAK, MASK_C_EBREAK) -DECLARE_INSN(c_ld, MATCH_C_LD, MASK_C_LD) -DECLARE_INSN(c_sd, MATCH_C_SD, MASK_C_SD) -DECLARE_INSN(c_addiw, MATCH_C_ADDIW, MASK_C_ADDIW) -DECLARE_INSN(c_ldsp, MATCH_C_LDSP, MASK_C_LDSP) -DECLARE_INSN(c_sdsp, MATCH_C_SDSP, MASK_C_SDSP) -DECLARE_INSN(c_addi4spn, MATCH_C_ADDI4SPN, MASK_C_ADDI4SPN) -DECLARE_INSN(c_fld, MATCH_C_FLD, MASK_C_FLD) -DECLARE_INSN(c_lw, MATCH_C_LW, MASK_C_LW) -DECLARE_INSN(c_flw, MATCH_C_FLW, MASK_C_FLW) -DECLARE_INSN(c_fsd, MATCH_C_FSD, MASK_C_FSD) -DECLARE_INSN(c_sw, MATCH_C_SW, MASK_C_SW) -DECLARE_INSN(c_fsw, MATCH_C_FSW, MASK_C_FSW) -DECLARE_INSN(c_addi, MATCH_C_ADDI, MASK_C_ADDI) -DECLARE_INSN(c_jal, MATCH_C_JAL, MASK_C_JAL) -DECLARE_INSN(c_li, MATCH_C_LI, MASK_C_LI) -DECLARE_INSN(c_lui, MATCH_C_LUI, MASK_C_LUI) -DECLARE_INSN(c_srli, MATCH_C_SRLI, MASK_C_SRLI) -DECLARE_INSN(c_srai, MATCH_C_SRAI, MASK_C_SRAI) -DECLARE_INSN(c_andi, MATCH_C_ANDI, MASK_C_ANDI) -DECLARE_INSN(c_sub, MATCH_C_SUB, MASK_C_SUB) -DECLARE_INSN(c_xor, MATCH_C_XOR, MASK_C_XOR) -DECLARE_INSN(c_or, MATCH_C_OR, MASK_C_OR) -DECLARE_INSN(c_and, MATCH_C_AND, MASK_C_AND) -DECLARE_INSN(c_subw, MATCH_C_SUBW, MASK_C_SUBW) -DECLARE_INSN(c_addw, MATCH_C_ADDW, MASK_C_ADDW) -DECLARE_INSN(c_j, MATCH_C_J, MASK_C_J) -DECLARE_INSN(c_beqz, MATCH_C_BEQZ, MASK_C_BEQZ) -DECLARE_INSN(c_bnez, MATCH_C_BNEZ, MASK_C_BNEZ) -DECLARE_INSN(c_slli, MATCH_C_SLLI, MASK_C_SLLI) -DECLARE_INSN(c_fldsp, MATCH_C_FLDSP, MASK_C_FLDSP) -DECLARE_INSN(c_lwsp, MATCH_C_LWSP, MASK_C_LWSP) -DECLARE_INSN(c_flwsp, MATCH_C_FLWSP, MASK_C_FLWSP) -DECLARE_INSN(c_mv, MATCH_C_MV, MASK_C_MV) -DECLARE_INSN(c_add, MATCH_C_ADD, MASK_C_ADD) -DECLARE_INSN(c_fsdsp, MATCH_C_FSDSP, MASK_C_FSDSP) -DECLARE_INSN(c_swsp, MATCH_C_SWSP, MASK_C_SWSP) -DECLARE_INSN(c_fswsp, MATCH_C_FSWSP, MASK_C_FSWSP) -DECLARE_INSN(custom0, MATCH_CUSTOM0, MASK_CUSTOM0) -DECLARE_INSN(custom0_rs1, MATCH_CUSTOM0_RS1, MASK_CUSTOM0_RS1) -DECLARE_INSN(custom0_rs1_rs2, MATCH_CUSTOM0_RS1_RS2, MASK_CUSTOM0_RS1_RS2) -DECLARE_INSN(custom0_rd, MATCH_CUSTOM0_RD, MASK_CUSTOM0_RD) -DECLARE_INSN(custom0_rd_rs1, MATCH_CUSTOM0_RD_RS1, MASK_CUSTOM0_RD_RS1) -DECLARE_INSN(custom0_rd_rs1_rs2, MATCH_CUSTOM0_RD_RS1_RS2, MASK_CUSTOM0_RD_RS1_RS2) -DECLARE_INSN(custom1, MATCH_CUSTOM1, MASK_CUSTOM1) -DECLARE_INSN(custom1_rs1, MATCH_CUSTOM1_RS1, MASK_CUSTOM1_RS1) -DECLARE_INSN(custom1_rs1_rs2, MATCH_CUSTOM1_RS1_RS2, MASK_CUSTOM1_RS1_RS2) -DECLARE_INSN(custom1_rd, MATCH_CUSTOM1_RD, MASK_CUSTOM1_RD) -DECLARE_INSN(custom1_rd_rs1, MATCH_CUSTOM1_RD_RS1, MASK_CUSTOM1_RD_RS1) -DECLARE_INSN(custom1_rd_rs1_rs2, MATCH_CUSTOM1_RD_RS1_RS2, MASK_CUSTOM1_RD_RS1_RS2) -DECLARE_INSN(custom2, MATCH_CUSTOM2, MASK_CUSTOM2) -DECLARE_INSN(custom2_rs1, MATCH_CUSTOM2_RS1, MASK_CUSTOM2_RS1) -DECLARE_INSN(custom2_rs1_rs2, MATCH_CUSTOM2_RS1_RS2, MASK_CUSTOM2_RS1_RS2) -DECLARE_INSN(custom2_rd, MATCH_CUSTOM2_RD, MASK_CUSTOM2_RD) -DECLARE_INSN(custom2_rd_rs1, MATCH_CUSTOM2_RD_RS1, MASK_CUSTOM2_RD_RS1) -DECLARE_INSN(custom2_rd_rs1_rs2, MATCH_CUSTOM2_RD_RS1_RS2, MASK_CUSTOM2_RD_RS1_RS2) -DECLARE_INSN(custom3, MATCH_CUSTOM3, MASK_CUSTOM3) -DECLARE_INSN(custom3_rs1, MATCH_CUSTOM3_RS1, MASK_CUSTOM3_RS1) -DECLARE_INSN(custom3_rs1_rs2, MATCH_CUSTOM3_RS1_RS2, MASK_CUSTOM3_RS1_RS2) -DECLARE_INSN(custom3_rd, MATCH_CUSTOM3_RD, MASK_CUSTOM3_RD) -DECLARE_INSN(custom3_rd_rs1, MATCH_CUSTOM3_RD_RS1, MASK_CUSTOM3_RD_RS1) -DECLARE_INSN(custom3_rd_rs1_rs2, MATCH_CUSTOM3_RD_RS1_RS2, MASK_CUSTOM3_RD_RS1_RS2) -#endif -#ifdef DECLARE_CSR -DECLARE_CSR(fflags, CSR_FFLAGS) -DECLARE_CSR(frm, CSR_FRM) -DECLARE_CSR(fcsr, CSR_FCSR) -DECLARE_CSR(cycle, CSR_CYCLE) -DECLARE_CSR(time, CSR_TIME) -DECLARE_CSR(instret, CSR_INSTRET) -DECLARE_CSR(stats, CSR_STATS) -DECLARE_CSR(uarch0, CSR_UARCH0) -DECLARE_CSR(uarch1, CSR_UARCH1) -DECLARE_CSR(uarch2, CSR_UARCH2) -DECLARE_CSR(uarch3, CSR_UARCH3) -DECLARE_CSR(uarch4, CSR_UARCH4) -DECLARE_CSR(uarch5, CSR_UARCH5) -DECLARE_CSR(uarch6, CSR_UARCH6) -DECLARE_CSR(uarch7, CSR_UARCH7) -DECLARE_CSR(uarch8, CSR_UARCH8) -DECLARE_CSR(uarch9, CSR_UARCH9) -DECLARE_CSR(uarch10, CSR_UARCH10) -DECLARE_CSR(uarch11, CSR_UARCH11) -DECLARE_CSR(uarch12, CSR_UARCH12) -DECLARE_CSR(uarch13, CSR_UARCH13) -DECLARE_CSR(uarch14, CSR_UARCH14) -DECLARE_CSR(uarch15, CSR_UARCH15) -DECLARE_CSR(sstatus, CSR_SSTATUS) -DECLARE_CSR(sie, CSR_SIE) -DECLARE_CSR(stvec, CSR_STVEC) -DECLARE_CSR(sscratch, CSR_SSCRATCH) -DECLARE_CSR(sepc, CSR_SEPC) -DECLARE_CSR(scause, CSR_SCAUSE) -DECLARE_CSR(sbadaddr, CSR_SBADADDR) -DECLARE_CSR(sip, CSR_SIP) -DECLARE_CSR(sptbr, CSR_SPTBR) -DECLARE_CSR(sasid, CSR_SASID) -DECLARE_CSR(scycle, CSR_SCYCLE) -DECLARE_CSR(stime, CSR_STIME) -DECLARE_CSR(sinstret, CSR_SINSTRET) -DECLARE_CSR(mstatus, CSR_MSTATUS) -DECLARE_CSR(medeleg, CSR_MEDELEG) -DECLARE_CSR(mideleg, CSR_MIDELEG) -DECLARE_CSR(mie, CSR_MIE) -DECLARE_CSR(mtvec, CSR_MTVEC) -DECLARE_CSR(mtimecmp, CSR_MTIMECMP) -DECLARE_CSR(mscratch, CSR_MSCRATCH) -DECLARE_CSR(mepc, CSR_MEPC) -DECLARE_CSR(mcause, CSR_MCAUSE) -DECLARE_CSR(mbadaddr, CSR_MBADADDR) -DECLARE_CSR(mip, CSR_MIP) -DECLARE_CSR(mipi, CSR_MIPI) -DECLARE_CSR(mucounteren, CSR_MUCOUNTEREN) -DECLARE_CSR(mscounteren, CSR_MSCOUNTEREN) -DECLARE_CSR(mucycle_delta, CSR_MUCYCLE_DELTA) -DECLARE_CSR(mutime_delta, CSR_MUTIME_DELTA) -DECLARE_CSR(muinstret_delta, CSR_MUINSTRET_DELTA) -DECLARE_CSR(mscycle_delta, CSR_MSCYCLE_DELTA) -DECLARE_CSR(mstime_delta, CSR_MSTIME_DELTA) -DECLARE_CSR(msinstret_delta, CSR_MSINSTRET_DELTA) -DECLARE_CSR(mcycle, CSR_MCYCLE) -DECLARE_CSR(mtime, CSR_MTIME) -DECLARE_CSR(minstret, CSR_MINSTRET) -DECLARE_CSR(misa, CSR_MISA) -DECLARE_CSR(mvendorid, CSR_MVENDORID) -DECLARE_CSR(marchid, CSR_MARCHID) -DECLARE_CSR(mimpid, CSR_MIMPID) -DECLARE_CSR(mcfgaddr, CSR_MCFGADDR) -DECLARE_CSR(mhartid, CSR_MHARTID) -DECLARE_CSR(mtohost, CSR_MTOHOST) -DECLARE_CSR(mfromhost, CSR_MFROMHOST) -DECLARE_CSR(mreset, CSR_MRESET) -DECLARE_CSR(cycleh, CSR_CYCLEH) -DECLARE_CSR(timeh, CSR_TIMEH) -DECLARE_CSR(instreth, CSR_INSTRETH) -DECLARE_CSR(mtimecmph, CSR_MTIMECMPH) -DECLARE_CSR(mucycle_deltah, CSR_MUCYCLE_DELTAH) -DECLARE_CSR(mutime_deltah, CSR_MUTIME_DELTAH) -DECLARE_CSR(muinstret_deltah, CSR_MUINSTRET_DELTAH) -DECLARE_CSR(mscycle_deltah, CSR_MSCYCLE_DELTAH) -DECLARE_CSR(mstime_deltah, CSR_MSTIME_DELTAH) -DECLARE_CSR(msinstret_deltah, CSR_MSINSTRET_DELTAH) -DECLARE_CSR(mcycleh, CSR_MCYCLEH) -DECLARE_CSR(mtimeh, CSR_MTIMEH) -DECLARE_CSR(minstreth, CSR_MINSTRETH) -#endif -#ifdef DECLARE_CAUSE -DECLARE_CAUSE("misaligned fetch", CAUSE_MISALIGNED_FETCH) -DECLARE_CAUSE("fault fetch", CAUSE_FAULT_FETCH) -DECLARE_CAUSE("illegal instruction", CAUSE_ILLEGAL_INSTRUCTION) -DECLARE_CAUSE("breakpoint", CAUSE_BREAKPOINT) -DECLARE_CAUSE("misaligned load", CAUSE_MISALIGNED_LOAD) -DECLARE_CAUSE("fault load", CAUSE_FAULT_LOAD) -DECLARE_CAUSE("misaligned store", CAUSE_MISALIGNED_STORE) -DECLARE_CAUSE("fault store", CAUSE_FAULT_STORE) -DECLARE_CAUSE("user_ecall", CAUSE_USER_ECALL) -DECLARE_CAUSE("supervisor_ecall", CAUSE_SUPERVISOR_ECALL) -DECLARE_CAUSE("hypervisor_ecall", CAUSE_HYPERVISOR_ECALL) -DECLARE_CAUSE("machine_ecall", CAUSE_MACHINE_ECALL) -#endif diff --git a/pk/file.c b/pk/file.c index e522685..9176f1a 100644 --- a/pk/file.c +++ b/pk/file.c @@ -1,11 +1,13 @@ // See LICENSE for license details. -#include -#include #include "file.h" -#include "pk.h" +#include "atomic.h" +#include "mmap.h" #include "frontend.h" -#include "vm.h" +#include "syscall.h" +#include "pk.h" +#include +#include #define MAX_FDS 128 static file_t* fds[MAX_FDS]; diff --git a/pk/file.h b/pk/file.h index c9f7ce4..3bf7f12 100644 --- a/pk/file.h +++ b/pk/file.h @@ -6,7 +6,6 @@ #include #include #include -#include "atomic.h" typedef struct file { diff --git a/pk/fp_asm.S b/pk/fp_asm.S deleted file mode 100644 index 4b8dce1..0000000 --- a/pk/fp_asm.S +++ /dev/null @@ -1,160 +0,0 @@ -// See LICENSE for license details. - -#ifdef __riscv_hard_float - -#define get_f32(which) fmv.x.s a0, which; jr t0 -#define put_f32(which) fmv.s.x which, a0; jr t0 -#ifdef __riscv64 -# define get_f64(which) fmv.x.d a0, which; jr t0 -# define put_f64(which) fmv.d.x which, a0; jr t0 -#else -# define get_f64(which) fsd which, 0(a0); jr t0 -# define put_f64(which) fld which, 0(a0); jr t0 -#endif - - .text - .option norvc - .globl get_f32_reg - get_f32_reg: - get_f32(f0) - get_f32(f1) - get_f32(f2) - get_f32(f3) - get_f32(f4) - get_f32(f5) - get_f32(f6) - get_f32(f7) - get_f32(f8) - get_f32(f9) - get_f32(f10) - get_f32(f11) - get_f32(f12) - get_f32(f13) - get_f32(f14) - get_f32(f15) - get_f32(f16) - get_f32(f17) - get_f32(f18) - get_f32(f19) - get_f32(f20) - get_f32(f21) - get_f32(f22) - get_f32(f23) - get_f32(f24) - get_f32(f25) - get_f32(f26) - get_f32(f27) - get_f32(f28) - get_f32(f29) - get_f32(f30) - get_f32(f31) - - .text - .globl put_f32_reg - put_f32_reg: - put_f32(f0) - put_f32(f1) - put_f32(f2) - put_f32(f3) - put_f32(f4) - put_f32(f5) - put_f32(f6) - put_f32(f7) - put_f32(f8) - put_f32(f9) - put_f32(f10) - put_f32(f11) - put_f32(f12) - put_f32(f13) - put_f32(f14) - put_f32(f15) - put_f32(f16) - put_f32(f17) - put_f32(f18) - put_f32(f19) - put_f32(f20) - put_f32(f21) - put_f32(f22) - put_f32(f23) - put_f32(f24) - put_f32(f25) - put_f32(f26) - put_f32(f27) - put_f32(f28) - put_f32(f29) - put_f32(f30) - put_f32(f31) - - .text - .globl get_f64_reg - get_f64_reg: - get_f64(f0) - get_f64(f1) - get_f64(f2) - get_f64(f3) - get_f64(f4) - get_f64(f5) - get_f64(f6) - get_f64(f7) - get_f64(f8) - get_f64(f9) - get_f64(f10) - get_f64(f11) - get_f64(f12) - get_f64(f13) - get_f64(f14) - get_f64(f15) - get_f64(f16) - get_f64(f17) - get_f64(f18) - get_f64(f19) - get_f64(f20) - get_f64(f21) - get_f64(f22) - get_f64(f23) - get_f64(f24) - get_f64(f25) - get_f64(f26) - get_f64(f27) - get_f64(f28) - get_f64(f29) - get_f64(f30) - get_f64(f31) - - .text - .globl put_f64_reg - put_f64_reg: - put_f64(f0) - put_f64(f1) - put_f64(f2) - put_f64(f3) - put_f64(f4) - put_f64(f5) - put_f64(f6) - put_f64(f7) - put_f64(f8) - put_f64(f9) - put_f64(f10) - put_f64(f11) - put_f64(f12) - put_f64(f13) - put_f64(f14) - put_f64(f15) - put_f64(f16) - put_f64(f17) - put_f64(f18) - put_f64(f19) - put_f64(f20) - put_f64(f21) - put_f64(f22) - put_f64(f23) - put_f64(f24) - put_f64(f25) - put_f64(f26) - put_f64(f27) - put_f64(f28) - put_f64(f29) - put_f64(f30) - put_f64(f31) - -#endif diff --git a/pk/fp_emulation.c b/pk/fp_emulation.c deleted file mode 100644 index 536967f..0000000 --- a/pk/fp_emulation.c +++ /dev/null @@ -1,449 +0,0 @@ -#include "fp_emulation.h" -#include "unprivileged_memory.h" -#include "softfloat.h" -#include "config.h" - -DECLARE_EMULATION_FUNC(emulate_float_load) -{ - uint64_t val; - uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); - - switch (insn & MASK_FUNCT3) - { - case MATCH_FLW & MASK_FUNCT3: - if (addr % 4 != 0) - return misaligned_load_trap(regs, mcause, mepc); - - SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc)); - break; - - case MATCH_FLD & MASK_FUNCT3: - if (addr % sizeof(uintptr_t) != 0) - return misaligned_load_trap(regs, mcause, mepc); - -#ifdef __riscv64 - val = load_uint64_t((void *)addr, mepc); -#else - val = load_uint32_t(addr, mepc); - val += (uint64_t)load_uint32_t((void *)(addr + 4), mepc) << 32; -#endif - SET_F64_RD(insn, regs, val); - break; - - default: - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_float_store) -{ - uint64_t val; - uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); - - switch (insn & MASK_FUNCT3) - { - case MATCH_FSW & MASK_FUNCT3: - if (addr % 4 != 0) - return misaligned_store_trap(regs, mcause, mepc); - - store_uint32_t((void *)addr, GET_F32_RS2(insn, regs), mepc); - break; - - case MATCH_FSD & MASK_FUNCT3: - if (addr % sizeof(uintptr_t) != 0) - return misaligned_store_trap(regs, mcause, mepc); - - val = GET_F64_RS2(insn, regs); -#ifdef __riscv64 - store_uint64_t((void *)addr, val, mepc); -#else - store_uint32_t((void *)addr, val, mepc); - store_uint32_t((void *)(addr + 4), val >> 32, mepc); -#endif - break; - - default: - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fp) -{ - asm (".pushsection .rodata\n" - "fp_emulation_table:\n" - " .word emulate_fadd\n" - " .word emulate_fsub\n" - " .word emulate_fmul\n" - " .word emulate_fdiv\n" - " .word emulate_fsgnj\n" - " .word emulate_fmin\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_fcvt_ff\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_fsqrt\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_fcmp\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word truly_illegal_insn\n" - " .word emulate_fcvt_if\n" - " .word truly_illegal_insn\n" - " .word emulate_fcvt_fi\n" - " .word truly_illegal_insn\n" - " .word emulate_fmv_if\n" - " .word truly_illegal_insn\n" - " .word emulate_fmv_fi\n" - " .word truly_illegal_insn\n" - " .popsection"); - - // if FPU is disabled, punt back to the OS - if (unlikely((mstatus & MSTATUS_FS) == 0)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - extern int32_t fp_emulation_table[]; - int32_t* pf = (void*)fp_emulation_table + ((insn >> 25) & 0x7c); - emulation_func f = (emulation_func)(uintptr_t)*pf; - - SETUP_STATIC_ROUNDING(insn); - return f(regs, mcause, mepc, mstatus, insn); -} - -void emulate_any_fadd(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn, int32_t neg_b) -{ - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs) ^ neg_b; - SET_F32_RD(insn, regs, f32_add(rs1, rs2)); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs) ^ ((uint64_t)neg_b << 32); - SET_F64_RD(insn, regs, f64_add(rs1, rs2)); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fadd) -{ - return emulate_any_fadd(regs, mcause, mepc, mstatus, insn, 0); -} - -DECLARE_EMULATION_FUNC(emulate_fsub) -{ - return emulate_any_fadd(regs, mcause, mepc, mstatus, insn, INT32_MIN); -} - -DECLARE_EMULATION_FUNC(emulate_fmul) -{ - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - SET_F32_RD(insn, regs, f32_mul(rs1, rs2)); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - SET_F64_RD(insn, regs, f64_mul(rs1, rs2)); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fdiv) -{ - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - SET_F32_RD(insn, regs, f32_div(rs1, rs2)); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - SET_F64_RD(insn, regs, f64_div(rs1, rs2)); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fsqrt) -{ - if ((insn >> 20) & 0x1f) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - if (GET_PRECISION(insn) == PRECISION_S) { - SET_F32_RD(insn, regs, f32_sqrt(GET_F32_RS1(insn, regs))); - } else if (GET_PRECISION(insn) == PRECISION_D) { - SET_F64_RD(insn, regs, f64_sqrt(GET_F64_RS1(insn, regs))); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fsgnj) -{ - int rm = GET_RM(insn); - if (rm >= 3) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - #define DO_FSGNJ(rs1, rs2, rm) ({ \ - typeof(rs1) rs1_sign = (rs1) >> (8*sizeof(rs1)-1); \ - typeof(rs1) rs2_sign = (rs2) >> (8*sizeof(rs1)-1); \ - rs1_sign &= (rm) >> 1; \ - rs1_sign ^= (rm) ^ rs2_sign; \ - ((rs1) << 1 >> 1) | (rs1_sign << (8*sizeof(rs1)-1)); }) - - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - SET_F32_RD(insn, regs, DO_FSGNJ(rs1, rs2, rm)); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - SET_F64_RD(insn, regs, DO_FSGNJ(rs1, rs2, rm)); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fmin) -{ - int rm = GET_RM(insn); - if (rm >= 2) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - uint32_t arg1 = rm ? rs2 : rs1; - uint32_t arg2 = rm ? rs1 : rs2; - int use_rs1 = f32_lt_quiet(arg1, arg2) || isNaNF32UI(rs2); - SET_F32_RD(insn, regs, use_rs1 ? rs1 : rs2); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - uint64_t arg1 = rm ? rs2 : rs1; - uint64_t arg2 = rm ? rs1 : rs2; - int use_rs1 = f64_lt_quiet(arg1, arg2) || isNaNF64UI(rs2); - SET_F64_RD(insn, regs, use_rs1 ? rs1 : rs2); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fcvt_ff) -{ - int rs2_num = (insn >> 20) & 0x1f; - if (GET_PRECISION(insn) == PRECISION_S) { - if (rs2_num != 1) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - SET_F32_RD(insn, regs, f64_to_f32(GET_F64_RS1(insn, regs))); - } else if (GET_PRECISION(insn) == PRECISION_D) { - if (rs2_num != 0) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - SET_F64_RD(insn, regs, f32_to_f64(GET_F32_RS1(insn, regs))); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} - -DECLARE_EMULATION_FUNC(emulate_fcvt_fi) -{ - if (GET_PRECISION(insn) != PRECISION_S && GET_PRECISION(insn) != PRECISION_D) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - int negative = 0; - uint64_t uint_val = GET_RS1(insn, regs); - - switch ((insn >> 20) & 0x1f) - { - case 0: // int32 - negative = (int32_t)uint_val < 0; - uint_val = negative ? -(int32_t)uint_val : (int32_t)uint_val; - break; - case 1: // uint32 - uint_val = (uint32_t)uint_val; - break; -#ifdef __riscv64 - case 2: // int64 - negative = (int64_t)uint_val < 0; - uint_val = negative ? -uint_val : uint_val; - case 3: // uint64 - break; -#endif - default: - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } - - uint64_t float64 = ui64_to_f64(uint_val); - if (negative) - float64 ^= INT64_MIN; - - if (GET_PRECISION(insn) == PRECISION_S) - SET_F32_RD(insn, regs, f64_to_f32(float64)); - else - SET_F64_RD(insn, regs, float64); -} - -DECLARE_EMULATION_FUNC(emulate_fcvt_if) -{ - int rs2_num = (insn >> 20) & 0x1f; -#ifdef __riscv64 - if (rs2_num >= 4) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); -#else - if (rs2_num >= 2) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); -#endif - - int64_t float64; - if (GET_PRECISION(insn) == PRECISION_S) - float64 = f32_to_f64(GET_F32_RS1(insn, regs)); - else if (GET_PRECISION(insn) == PRECISION_D) - float64 = GET_F64_RS1(insn, regs); - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - int negative = 0; - if (float64 < 0) { - negative = 1; - float64 ^= INT64_MIN; - } - uint64_t uint_val = f64_to_ui64(float64, softfloat_roundingMode, true); - uint64_t result, limit, limit_result; - - switch (rs2_num) - { - case 0: // int32 - if (negative) { - result = (int32_t)-uint_val; - limit_result = limit = (uint32_t)INT32_MIN; - } else { - result = (int32_t)uint_val; - limit_result = limit = INT32_MAX; - } - break; - - case 1: // uint32 - limit = limit_result = UINT32_MAX; - if (negative) - result = limit = 0; - else - result = (uint32_t)uint_val; - break; - - case 2: // int32 - if (negative) { - result = (int64_t)-uint_val; - limit_result = limit = (uint64_t)INT64_MIN; - } else { - result = (int64_t)uint_val; - limit_result = limit = INT64_MAX; - } - break; - - case 3: // uint64 - limit = limit_result = UINT64_MAX; - if (negative) - result = limit = 0; - else - result = (uint64_t)uint_val; - break; - - default: - __builtin_unreachable(); - } - - if (uint_val > limit) { - result = limit_result; - softfloat_raiseFlags(softfloat_flag_invalid); - } - - SET_FS_DIRTY(); - SET_RD(insn, regs, result); -} - -DECLARE_EMULATION_FUNC(emulate_fcmp) -{ - int rm = GET_RM(insn); - if (rm >= 3) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - uintptr_t result; - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - if (rm != 1) - result = f32_eq(rs1, rs2); - if (rm == 1 || (rm == 0 && !result)) - result = f32_lt(rs1, rs2); - goto success; - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - if (rm != 1) - result = f64_eq(rs1, rs2); - if (rm == 1 || (rm == 0 && !result)) - result = f64_lt(rs1, rs2); - goto success; - } - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); -success: - SET_RD(insn, regs, result); -} - -DECLARE_EMULATION_FUNC(emulate_fmv_if) -{ - uintptr_t result; - if ((insn & MASK_FMV_X_S) == MATCH_FMV_X_S) - result = GET_F32_RS1(insn, regs); -#ifdef __riscv64 - else if ((insn & MASK_FMV_X_D) == MATCH_FMV_X_D) - result = GET_F64_RS1(insn, regs); -#endif - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - SET_RD(insn, regs, result); -} - -DECLARE_EMULATION_FUNC(emulate_fmv_fi) -{ - uintptr_t rs1 = GET_RS1(insn, regs); - - if ((insn & MASK_FMV_S_X) == MATCH_FMV_S_X) - SET_F32_RD(insn, regs, rs1); - else if ((insn & MASK_FMV_D_X) == MATCH_FMV_D_X) - SET_F64_RD(insn, regs, rs1); - else - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); -} - -DECLARE_EMULATION_FUNC(emulate_fmadd) -{ - // if FPU is disabled, punt back to the OS - if (unlikely((mstatus & MSTATUS_FS) == 0)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - - int op = (insn >> 2) & 3; - SETUP_STATIC_ROUNDING(insn); - if (GET_PRECISION(insn) == PRECISION_S) { - uint32_t rs1 = GET_F32_RS1(insn, regs); - uint32_t rs2 = GET_F32_RS2(insn, regs); - uint32_t rs3 = GET_F32_RS3(insn, regs); - SET_F32_RD(insn, regs, softfloat_mulAddF32(op, rs1, rs2, rs3)); - } else if (GET_PRECISION(insn) == PRECISION_D) { - uint64_t rs1 = GET_F64_RS1(insn, regs); - uint64_t rs2 = GET_F64_RS2(insn, regs); - uint64_t rs3 = GET_F64_RS3(insn, regs); - SET_F64_RD(insn, regs, softfloat_mulAddF64(op, rs1, rs2, rs3)); - } else { - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); - } -} diff --git a/pk/fp_emulation.h b/pk/fp_emulation.h deleted file mode 100644 index d2357b7..0000000 --- a/pk/fp_emulation.h +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef _RISCV_FP_EMULATION_H -#define _RISCV_FP_EMULATION_H - -#include "emulation.h" - -#define GET_PRECISION(insn) (((insn) >> 25) & 3) -#define GET_RM(insn) (((insn) >> 12) & 7) -#define PRECISION_S 0 -#define PRECISION_D 1 - -#ifdef __riscv_hard_float -# define GET_F32_REG(insn, pos, regs) ({ \ - register int32_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \ - uintptr_t tmp; \ - asm ("1: auipc %0, %%pcrel_hi(get_f32_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \ - value; }) -# define SET_F32_REG(insn, pos, regs, val) ({ \ - register uint32_t value asm("a0") = (val); \ - uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \ - uintptr_t tmp; \ - asm volatile ("1: auipc %0, %%pcrel_hi(put_f32_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); }) -# define init_fp_reg(i) SET_F32_REG((i) << 3, 3, 0, 0) -# define GET_F64_REG(insn, pos, regs) ({ \ - register uintptr_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \ - uintptr_t tmp; \ - asm ("1: auipc %0, %%pcrel_hi(get_f64_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \ - sizeof(uintptr_t) == 4 ? *(int64_t*)value : (int64_t)value; }) -# define SET_F64_REG(insn, pos, regs, val) ({ \ - uint64_t __val = (val); \ - register uintptr_t value asm("a0") = sizeof(uintptr_t) == 4 ? (uintptr_t)&__val : (uintptr_t)__val; \ - uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \ - uintptr_t tmp; \ - asm volatile ("1: auipc %0, %%pcrel_hi(put_f64_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); }) -# define GET_FCSR() read_csr(fcsr) -# define SET_FCSR(value) write_csr(fcsr, (value)) -# define GET_FRM() read_csr(frm) -# define SET_FRM(value) write_csr(frm, (value)) -# define GET_FFLAGS() read_csr(fflags) -# define SET_FFLAGS(value) write_csr(fflags, (value)) - -# define SETUP_STATIC_ROUNDING(insn) ({ \ - register long tp asm("tp") = read_csr(frm); \ - if (likely(((insn) & MASK_FUNCT3) == MASK_FUNCT3)) ; \ - else if (GET_RM(insn) > 4) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); \ - else tp = GET_RM(insn); \ - asm volatile ("":"+r"(tp)); }) -# define softfloat_raiseFlags(which) set_csr(fflags, which) -# define softfloat_roundingMode ({ register int tp asm("tp"); tp; }) -#else -# define GET_F64_REG(insn, pos, regs) (*(int64_t*)((void*)((regs) + 32) + (((insn) >> ((pos)-3)) & 0xf8))) -# define SET_F64_REG(insn, pos, regs, val) (GET_F64_REG(insn, pos, regs) = (val)) -# define GET_F32_REG(insn, pos, regs) (*(int32_t*)&GET_F64_REG(insn, pos, regs)) -# define SET_F32_REG(insn, pos, regs, val) (GET_F32_REG(insn, pos, regs) = (val)) -# define GET_FCSR() ({ register int tp asm("tp"); tp & 0xFF; }) -# define SET_FCSR(value) ({ asm volatile("add tp, x0, %0" :: "rI"((value) & 0xFF)); }) -# define GET_FRM() (GET_FCSR() >> 5) -# define SET_FRM(value) SET_FCSR(GET_FFLAGS() | ((value) << 5)) -# define GET_FFLAGS() (GET_FCSR() & 0x1F) -# define SET_FFLAGS(value) SET_FCSR((GET_FRM() << 5) | ((value) & 0x1F)) - -# define SETUP_STATIC_ROUNDING(insn) ({ \ - register int tp asm("tp"); tp &= 0xFF; \ - if (likely(((insn) & MASK_FUNCT3) == MASK_FUNCT3)) tp |= tp << 8; \ - else if (GET_RM(insn) > 4) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); \ - else tp |= GET_RM(insn) << 13; \ - asm volatile ("":"+r"(tp)); }) -# define softfloat_raiseFlags(which) ({ asm volatile ("or tp, tp, %0" :: "rI"(which)); }) -# define softfloat_roundingMode ({ register int tp asm("tp"); tp >> 13; }) -#endif - -#define GET_F32_RS1(insn, regs) (GET_F32_REG(insn, 15, regs)) -#define GET_F32_RS2(insn, regs) (GET_F32_REG(insn, 20, regs)) -#define GET_F32_RS3(insn, regs) (GET_F32_REG(insn, 27, regs)) -#define GET_F64_RS1(insn, regs) (GET_F64_REG(insn, 15, regs)) -#define GET_F64_RS2(insn, regs) (GET_F64_REG(insn, 20, regs)) -#define GET_F64_RS3(insn, regs) (GET_F64_REG(insn, 27, regs)) -#define SET_F32_RD(insn, regs, val) (SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY()) -#define SET_F64_RD(insn, regs, val) (SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY()) -#define SET_FS_DIRTY() set_csr(mstatus, MSTATUS_FS) - -#endif diff --git a/pk/frontend.h b/pk/frontend.h index 2cf5f81..5b9df31 100644 --- a/pk/frontend.h +++ b/pk/frontend.h @@ -6,18 +6,6 @@ #include #include -#ifdef __riscv64 -# define TOHOST_CMD(dev, cmd, payload) \ - (((uint64_t)(dev) << 56) | ((uint64_t)(cmd) << 48) | (uint64_t)(payload)) -#else -# define TOHOST_CMD(dev, cmd, payload) ({ \ - if ((dev) || (cmd)) __builtin_trap(); \ - (payload); }) -#endif -#define FROMHOST_DEV(fromhost_value) ((uint64_t)(fromhost_value) >> 56) -#define FROMHOST_CMD(fromhost_value) ((uint64_t)(fromhost_value) << 8 >> 56) -#define FROMHOST_DATA(fromhost_value) ((uint64_t)(fromhost_value) << 16 >> 16) - void shutdown(int) __attribute__((noreturn)); long frontend_syscall(long n, long a0, long a1, long a2, long a3, long a4, long a5, long a6); diff --git a/pk/handlers.c b/pk/handlers.c index bc82b5d..1961852 100644 --- a/pk/handlers.c +++ b/pk/handlers.c @@ -3,7 +3,7 @@ #include "pk.h" #include "config.h" #include "syscall.h" -#include "vm.h" +#include "mmap.h" static void handle_illegal_instruction(trapframe_t* tf) { @@ -31,7 +31,7 @@ static void handle_misaligned_fetch(trapframe_t* tf) panic("Misaligned instruction access!"); } -void handle_misaligned_store(trapframe_t* tf) +static void handle_misaligned_store(trapframe_t* tf) { dump_tf(tf); panic("Misaligned AMO!"); @@ -50,13 +50,13 @@ static void handle_fault_fetch(trapframe_t* tf) segfault(tf, tf->badvaddr, "fetch"); } -void handle_fault_load(trapframe_t* tf) +static void handle_fault_load(trapframe_t* tf) { if (handle_page_fault(tf->badvaddr, PROT_READ) != 0) segfault(tf, tf->badvaddr, "load"); } -void handle_fault_store(trapframe_t* tf) +static void handle_fault_store(trapframe_t* tf) { if (handle_page_fault(tf->badvaddr, PROT_WRITE) != 0) segfault(tf, tf->badvaddr, "store"); diff --git a/pk/init.c b/pk/init.c deleted file mode 100644 index 11e17bd..0000000 --- a/pk/init.c +++ /dev/null @@ -1,82 +0,0 @@ -// See LICENSE for license details. - -#include "pk.h" -#include "boot.h" -#include "file.h" -#include "vm.h" -#include "frontend.h" -#include "elf.h" -#include -#include -#include - -elf_info current; -int have_vm = 1; // unless -p flag is given - -int uarch_counters_enabled; -long uarch_counters[NUM_COUNTERS]; -char* uarch_counter_names[NUM_COUNTERS]; - -void init_tf(trapframe_t* tf, long pc, long sp) -{ - memset(tf, 0, sizeof(*tf)); - tf->status = (read_csr(sstatus) &~ SSTATUS_SPP &~ SSTATUS_SIE) | SSTATUS_SPIE; - tf->gpr[2] = sp; - tf->epc = pc; -} - -static void handle_option(const char* s) -{ - switch (s[1]) - { - case 's': // print cycle count upon termination - current.t0 = 1; - break; - - case 'c': // print uarch counters upon termination - // If your HW doesn't support uarch counters, then don't use this flag! - uarch_counters_enabled = 1; - break; - - default: - panic("unrecognized option: `%c'", s[1]); - break; - } -} - -#define MAX_ARGS 64 -typedef union { - uint64_t buf[MAX_ARGS]; - char* argv[MAX_ARGS]; -} arg_buf; - -static size_t parse_args(arg_buf* args) -{ - long r = frontend_syscall(SYS_getmainvars, (uintptr_t)args, sizeof(*args), 0, 0, 0, 0, 0); - kassert(r == 0); - uint64_t* pk_argv = &args->buf[1]; - // pk_argv[0] is the proxy kernel itself. skip it and any flags. - size_t pk_argc = args->buf[0], arg = 1; - for ( ; arg < pk_argc && *(char*)(uintptr_t)pk_argv[arg] == '-'; arg++) - handle_option((const char*)(uintptr_t)pk_argv[arg]); - - for (size_t i = 0; arg + i < pk_argc; i++) - args->argv[i] = (char*)(uintptr_t)pk_argv[arg + i]; - return pk_argc - arg; -} - -void boot_loader() -{ - arg_buf args; - size_t argc = parse_args(&args); - if (!argc) - panic("tell me what ELF to load!"); - - // load program named by argv[0] - long phdrs[128]; - current.phdr = (uintptr_t)phdrs; - current.phdr_size = sizeof(phdrs); - load_elf(args.argv[0], ¤t); - - run_loaded_program(argc, args.argv); -} diff --git a/pk/logo.c b/pk/logo.c deleted file mode 100644 index 2a214c5..0000000 --- a/pk/logo.c +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include "file.h" - -static const char logo[] = -" vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" -" vvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvv \n" -"rrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvv \n" -"rr vvvvvvvvvvvvvvvvvvvvvv \n" -"rr vvvvvvvvvvvvvvvvvvvvvvvv rr\n" -"rrrr vvvvvvvvvvvvvvvvvvvvvvvvvv rrrr\n" -"rrrrrr vvvvvvvvvvvvvvvvvvvvvv rrrrrr\n" -"rrrrrrrr vvvvvvvvvvvvvvvvvv rrrrrrrr\n" -"rrrrrrrrrr vvvvvvvvvvvvvv rrrrrrrrrr\n" -"rrrrrrrrrrrr vvvvvvvvvv rrrrrrrrrrrr\n" -"rrrrrrrrrrrrrr vvvvvv rrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrr vv rrrrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrrrrrr\n" -"\n" -" INSTRUCTION SETS WANT TO BE FREE\n"; - -void print_logo() -{ - file_write(stderr, logo, sizeof logo); -} diff --git a/pk/mcall.h b/pk/mcall.h deleted file mode 100644 index e612628..0000000 --- a/pk/mcall.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef _PK_MCALL_H -#define _PK_MCALL_H - -#define MCALL_HART_ID 0 -#define MCALL_CONSOLE_PUTCHAR 1 -#define MCALL_CONSOLE_GETCHAR 2 -#define MCALL_HTIF_SYSCALL 3 -#define MCALL_SEND_IPI 4 -#define MCALL_CLEAR_IPI 5 -#define MCALL_SHUTDOWN 6 -#define MCALL_SET_TIMER 7 -#define MCALL_REMOTE_SFENCE_VM 8 -#define MCALL_REMOTE_FENCE_I 9 - -#ifndef __ASSEMBLER__ - -extern uintptr_t do_mcall(uintptr_t which, ...); - -#endif - -#endif diff --git a/pk/mentry.S b/pk/mentry.S deleted file mode 100644 index 2ded375..0000000 --- a/pk/mentry.S +++ /dev/null @@ -1,244 +0,0 @@ -// See LICENSE for license details. - -#include "mtrap.h" - - .data - .align 6 -trap_table: - .word bad_trap - .word bad_trap - .word illegal_insn_trap - .word bad_trap - .word misaligned_load_trap - .word bad_trap - .word misaligned_store_trap - .word bad_trap - .word bad_trap - .word mcall_trap - .word bad_trap - .word bad_trap -#define HTIF_INTERRUPT_VECTOR 12 - .word htif_interrupt -#define SOFTWARE_INTERRUPT_VECTOR 13 - .word software_interrupt -#define TRAP_FROM_MACHINE_MODE_VECTOR 14 - .word __trap_from_machine_mode - - .option norvc - .section .text.init,"ax",@progbits - .globl reset_vector -reset_vector: - j do_reset - -nmi_vector: -.Lunhandleable_trap: - j bad_trap - -trap_vector: - csrrw sp, mscratch, sp - beqz sp, .Ltrap_from_machine_mode - - STORE a0, 10*REGBYTES(sp) - STORE a1, 11*REGBYTES(sp) - - csrr a1, mcause - bgez a1, .Lhandle_trap_in_machine_mode - - # This is an interrupt. Discard the mcause MSB and decode the rest. - sll a1, a1, 1 - - # Is it a machine timer interrupt? - li a0, IRQ_M_TIMER * 2 - bne a0, a1, 1f - # Yes. Post a supervisor timer interrupt. - li a0, MIP_MTIP - csrc mie, a0 - li a0, MIP_STIP - csrs mip, a0 - -.Leret: - # Go back whence we came. - LOAD a0, 10*REGBYTES(sp) - LOAD a1, 11*REGBYTES(sp) - csrrw sp, mscratch, sp - eret - -1: - # Is it an IPI? - li a0, IRQ_M_SOFT * 2 - bne a0, a1, 1f - li a1, SOFTWARE_INTERRUPT_VECTOR - j .Lhandle_trap_in_machine_mode - -1: - # By process of elimination, it must be an HTIF interrupt. - li a0, IRQ_HOST * 2 - bne a0, a1, .Lunhandleable_trap - li a1, HTIF_INTERRUPT_VECTOR - -.Lhandle_trap_in_machine_mode: - # Preserve the registers. Compute the address of the trap handler. - STORE ra, 1*REGBYTES(sp) - STORE gp, 3*REGBYTES(sp) - STORE tp, 4*REGBYTES(sp) - STORE t0, 5*REGBYTES(sp) -1:auipc t0, %pcrel_hi(trap_table) # t0 <- %hi(trap_table) - STORE t1, 6*REGBYTES(sp) - sll t1, a1, 2 # t1 <- mcause << 2 - STORE t2, 7*REGBYTES(sp) - add t1, t0, t1 # t1 <- %hi(trap_table)[mcause] - STORE s0, 8*REGBYTES(sp) - lw t1, %pcrel_lo(1b)(t1) # t1 <- trap_table[mcause] - STORE s1, 9*REGBYTES(sp) - mv a0, sp # a0 <- regs - STORE a2,12*REGBYTES(sp) - csrr a2, mepc # a2 <- mepc - STORE a3,13*REGBYTES(sp) - csrrw t0, mscratch, x0 # t0 <- user sp - STORE a4,14*REGBYTES(sp) - STORE a5,15*REGBYTES(sp) - STORE a6,16*REGBYTES(sp) - STORE a7,17*REGBYTES(sp) - STORE s2,18*REGBYTES(sp) - STORE s3,19*REGBYTES(sp) - STORE s4,20*REGBYTES(sp) - STORE s5,21*REGBYTES(sp) - STORE s6,22*REGBYTES(sp) - STORE s7,23*REGBYTES(sp) - STORE s8,24*REGBYTES(sp) - STORE s9,25*REGBYTES(sp) - STORE s10,26*REGBYTES(sp) - STORE s11,27*REGBYTES(sp) - STORE t3,28*REGBYTES(sp) - STORE t4,29*REGBYTES(sp) - STORE t5,30*REGBYTES(sp) - STORE t6,31*REGBYTES(sp) - STORE t0, 2*REGBYTES(sp) # sp - -#ifndef __riscv_hard_float - lw tp, (sp) # Move the emulated FCSR from x0's save slot into tp. -#endif - STORE x0, (sp) # Zero x0's save slot. - - # Invoke the handler. - jalr t1 - -#ifndef __riscv_hard_float - sw tp, (sp) # Move the emulated FCSR from tp into x0's save slot. -#endif - -restore_mscratch: - # Restore mscratch, so future traps will know they didn't come from M-mode. - csrw mscratch, sp - -restore_regs: - # Restore all of the registers. - LOAD ra, 1*REGBYTES(sp) - LOAD gp, 3*REGBYTES(sp) - LOAD tp, 4*REGBYTES(sp) - LOAD t0, 5*REGBYTES(sp) - LOAD t1, 6*REGBYTES(sp) - LOAD t2, 7*REGBYTES(sp) - LOAD s0, 8*REGBYTES(sp) - LOAD s1, 9*REGBYTES(sp) - LOAD a0,10*REGBYTES(sp) - LOAD a1,11*REGBYTES(sp) - LOAD a2,12*REGBYTES(sp) - LOAD a3,13*REGBYTES(sp) - LOAD a4,14*REGBYTES(sp) - LOAD a5,15*REGBYTES(sp) - LOAD a6,16*REGBYTES(sp) - LOAD a7,17*REGBYTES(sp) - LOAD s2,18*REGBYTES(sp) - LOAD s3,19*REGBYTES(sp) - LOAD s4,20*REGBYTES(sp) - LOAD s5,21*REGBYTES(sp) - LOAD s6,22*REGBYTES(sp) - LOAD s7,23*REGBYTES(sp) - LOAD s8,24*REGBYTES(sp) - LOAD s9,25*REGBYTES(sp) - LOAD s10,26*REGBYTES(sp) - LOAD s11,27*REGBYTES(sp) - LOAD t3,28*REGBYTES(sp) - LOAD t4,29*REGBYTES(sp) - LOAD t5,30*REGBYTES(sp) - LOAD t6,31*REGBYTES(sp) - LOAD sp, 2*REGBYTES(sp) - eret - -.Ltrap_from_machine_mode: - csrr sp, mscratch - addi sp, sp, -INTEGER_CONTEXT_SIZE - STORE a0,10*REGBYTES(sp) - STORE a1,11*REGBYTES(sp) - li a1, TRAP_FROM_MACHINE_MODE_VECTOR - j .Lhandle_trap_in_machine_mode - - .globl __redirect_trap -__redirect_trap: - # reset sp to top of M-mode stack - li t0, MACHINE_STACK_SIZE - add sp, sp, t0 - neg t0, t0 - and sp, sp, t0 - addi sp, sp, -MENTRY_FRAME_SIZE - j restore_mscratch - -__trap_from_machine_mode: - jal trap_from_machine_mode - j restore_regs - -do_reset: - li x1, 0 - li x2, 0 - li x3, 0 - li x4, 0 - li x5, 0 - li x6, 0 - li x7, 0 - li x8, 0 - li x9, 0 - li x10, 0 - li x11, 0 - li x12, 0 - li x13, 0 - li x14, 0 - li x15, 0 - li x16, 0 - li x17, 0 - li x18, 0 - li x19, 0 - li x20, 0 - li x21, 0 - li x22, 0 - li x23, 0 - li x24, 0 - li x25, 0 - li x26, 0 - li x27, 0 - li x28, 0 - li x29, 0 - li x30, 0 - li x31, 0 - csrw mscratch, x0 - - # sp <- end of first full page after the end of the binary - la sp, _end + 2*RISCV_PGSIZE - 1 - li t0, -RISCV_PGSIZE - and sp, sp, t0 - addi sp, sp, -MENTRY_FRAME_SIZE - - csrr a0, mhartid - slli a1, a0, RISCV_PGSHIFT - add sp, sp, a1 - - beqz a0, init_first_hart - -.LmultiHart: -#if MAX_HARTS > 1 - # make sure our hart id is within a valid range - li a1, MAX_HARTS - bltu a0, a1, init_other_hart - wfi -#endif - j .LmultiHart diff --git a/pk/minit.c b/pk/minit.c deleted file mode 100644 index ad9598b..0000000 --- a/pk/minit.c +++ /dev/null @@ -1,108 +0,0 @@ -#include "vm.h" -#include "mtrap.h" -#include "fp_emulation.h" -#include "boot.h" - -uintptr_t mem_size; -uint32_t num_harts; - -static void mstatus_init() -{ - uintptr_t ms = 0; - ms = INSERT_FIELD(ms, MSTATUS_VM, VM_CHOICE); - ms = INSERT_FIELD(ms, MSTATUS_FS, 1); - write_csr(mstatus, ms); - - ms = read_csr(mstatus); - assert(EXTRACT_FIELD(ms, MSTATUS_VM) == VM_CHOICE); - - write_csr(mtimecmp, 0); - clear_csr(mip, MIP_MSIP); - write_csr(mie, -1); - write_csr(mucounteren, -1); - write_csr(mscounteren, -1); -} - -static void delegate_traps() -{ - uintptr_t interrupts = MIP_SSIP | MIP_STIP; - uintptr_t exceptions = - (1U << CAUSE_MISALIGNED_FETCH) | - (1U << CAUSE_FAULT_FETCH) | - (1U << CAUSE_BREAKPOINT) | - (1U << CAUSE_FAULT_LOAD) | - (1U << CAUSE_FAULT_STORE) | - (1U << CAUSE_BREAKPOINT) | - (1U << CAUSE_USER_ECALL); - - write_csr(mideleg, interrupts); - write_csr(medeleg, exceptions); - assert(read_csr(mideleg) == interrupts); - assert(read_csr(medeleg) == exceptions); -} - -static void fp_init() -{ - assert(read_csr(mstatus) & MSTATUS_FS); - -#ifdef __riscv_hard_float - if (!supports_extension('D')) - die("FPU not found; recompile pk with -msoft-float"); - for (int i = 0; i < 32; i++) - init_fp_reg(i); - write_csr(fcsr, 0); -#else - if (supports_extension('D')) - die("FPU unexpectedly found; recompile with -mhard-float"); -#endif -} - -void hls_init(uint32_t id, csr_t* csrs) -{ - hls_t* hls = OTHER_HLS(id); - memset(hls, 0, sizeof(*hls)); - hls->csrs = csrs; -} - -static void hart_init() -{ - mstatus_init(); - fp_init(); - delegate_traps(); -} - -void init_first_hart() -{ - file_init(); - hart_init(); - - memset(HLS(), 0, sizeof(*HLS())); - parse_config_string(); - vm_init(); - boot_loader(); -} - -void init_other_hart() -{ - hart_init(); - - // wait until virtual memory is enabled - while (*(pte_t* volatile*)&root_page_table == NULL) - ; - mb(); - write_csr(sptbr, (uintptr_t)root_page_table >> RISCV_PGSHIFT); - - // make sure hart 0 has discovered us - assert(HLS()->csrs != NULL); - - boot_other_hart(); -} - -void prepare_supervisor_mode() -{ - uintptr_t mstatus = read_csr(mstatus); - mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S); - mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); - write_csr(mstatus, mstatus); - write_csr(mscratch, MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE); -} diff --git a/pk/mmap.c b/pk/mmap.c new file mode 100644 index 0000000..2f6a202 --- /dev/null +++ b/pk/mmap.c @@ -0,0 +1,426 @@ +#include "mmap.h" +#include "atomic.h" +#include "pk.h" +#include "boot.h" +#include "bits.h" +#include "mtrap.h" +#include +#include + +typedef struct { + uintptr_t addr; + size_t length; + file_t* file; + size_t offset; + unsigned refcnt; + int prot; +} vmr_t; + +#define MAX_VMR (RISCV_PGSIZE / sizeof(vmr_t)) +static spinlock_t vm_lock = SPINLOCK_INIT; +static vmr_t* vmrs; + +static uintptr_t first_free_page; +static size_t next_free_page; +static size_t free_pages; + +int have_vm = 1; // unless -p flag is given + +static uintptr_t __page_alloc() +{ + kassert(next_free_page != free_pages); + uintptr_t addr = first_free_page + RISCV_PGSIZE * next_free_page++; + memset((void*)addr, 0, RISCV_PGSIZE); + return addr; +} + +static vmr_t* __vmr_alloc(uintptr_t addr, size_t length, file_t* file, + size_t offset, unsigned refcnt, int prot) +{ + if (!vmrs) { + spinlock_lock(&vm_lock); + if (!vmrs) + vmrs = (vmr_t*)__page_alloc(); + spinlock_unlock(&vm_lock); + } + + for (vmr_t* v = vmrs; v < vmrs + MAX_VMR; v++) { + if (v->refcnt == 0) { + if (file) + file_incref(file); + v->addr = addr; + v->length = length; + v->file = file; + v->offset = offset; + v->refcnt = refcnt; + v->prot = prot; + return v; + } + } + return NULL; +} + +static void __vmr_decref(vmr_t* v, unsigned dec) +{ + if ((v->refcnt -= dec) == 0) + { + if (v->file) + file_decref(v->file); + } +} + +static size_t pte_ppn(pte_t pte) +{ + return pte >> PTE_PPN_SHIFT; +} + +static uintptr_t ppn(uintptr_t addr) +{ + return addr >> RISCV_PGSHIFT; +} + +static size_t pt_idx(uintptr_t addr, int level) +{ + size_t idx = addr >> (RISCV_PGLEVEL_BITS*level + RISCV_PGSHIFT); + return idx & ((1 << RISCV_PGLEVEL_BITS) - 1); +} + +static pte_t* __maybe_create_root_page_table() +{ + if (!root_page_table) + root_page_table = (void*)__page_alloc(); + return root_page_table; +} + +static pte_t* __walk_internal(uintptr_t addr, int create) +{ + pte_t* t = __maybe_create_root_page_table(); + for (int i = (VA_BITS - RISCV_PGSHIFT) / RISCV_PGLEVEL_BITS - 1; i > 0; i--) { + size_t idx = pt_idx(addr, i); + if (!(t[idx] & PTE_V)) { + if (!create) + return 0; + uintptr_t page = __page_alloc(); + t[idx] = ptd_create(ppn(page)); + } + else + kassert(PTE_TABLE(t[idx])); + t = (pte_t*)(pte_ppn(t[idx]) << RISCV_PGSHIFT); + } + return &t[pt_idx(addr, 0)]; +} + +static pte_t* __walk(uintptr_t addr) +{ + return __walk_internal(addr, 0); +} + +static pte_t* __walk_create(uintptr_t addr) +{ + return __walk_internal(addr, 1); +} + +static int __va_avail(uintptr_t vaddr) +{ + pte_t* pte = __walk(vaddr); + return pte == 0 || *pte == 0; +} + +static uintptr_t __vm_alloc(size_t npage) +{ + uintptr_t start = current.brk, end = current.mmap_max - npage*RISCV_PGSIZE; + for (uintptr_t a = start; a <= end; a += RISCV_PGSIZE) + { + if (!__va_avail(a)) + continue; + uintptr_t first = a, last = a + (npage-1) * RISCV_PGSIZE; + for (a = last; a > first && __va_avail(a); a -= RISCV_PGSIZE) + ; + if (a > first) + continue; + return a; + } + return 0; +} + +static inline pte_t prot_to_type(int prot, int user) +{ + prot &= PROT_READ|PROT_WRITE|PROT_EXEC; + if (user) { + switch (prot) { + case PROT_NONE: return PTE_TYPE_SR; + case PROT_READ: return PTE_TYPE_UR_SR; + case PROT_WRITE: return PTE_TYPE_URW_SRW; + case PROT_EXEC: return PTE_TYPE_URX_SRX; + case PROT_READ|PROT_WRITE: return PTE_TYPE_URW_SRW; + case PROT_READ|PROT_EXEC: return PTE_TYPE_URX_SRX; + case PROT_WRITE|PROT_EXEC: return PTE_TYPE_URWX_SRWX; + case PROT_READ|PROT_WRITE|PROT_EXEC: return PTE_TYPE_URWX_SRWX; + } + } else { + switch (prot) { + case PROT_NONE: + case PROT_READ: return PTE_TYPE_SR; + case PROT_WRITE: return PTE_TYPE_SRW; + case PROT_EXEC: return PTE_TYPE_SRX; + case PROT_READ|PROT_WRITE: return PTE_TYPE_SRW; + case PROT_READ|PROT_EXEC: return PTE_TYPE_SRX; + case PROT_WRITE|PROT_EXEC: return PTE_TYPE_SRWX; + case PROT_READ|PROT_WRITE|PROT_EXEC: return PTE_TYPE_SRWX; + } + } + __builtin_unreachable(); +} + +int __valid_user_range(uintptr_t vaddr, size_t len) +{ + if (vaddr + len < vaddr) + return 0; + return vaddr >= first_free_paddr && vaddr + len <= current.mmap_max; +} + +static int __handle_page_fault(uintptr_t vaddr, int prot) +{ + uintptr_t vpn = vaddr >> RISCV_PGSHIFT; + vaddr = vpn << RISCV_PGSHIFT; + + pte_t* pte = __walk(vaddr); + + if (pte == 0 || *pte == 0 || !__valid_user_range(vaddr, 1)) + return -1; + else if (!(*pte & PTE_V)) + { + uintptr_t ppn = vpn; + + vmr_t* v = (vmr_t*)*pte; + *pte = pte_create(ppn, prot_to_type(PROT_READ|PROT_WRITE, 0)); + flush_tlb(); + if (v->file) + { + size_t flen = MIN(RISCV_PGSIZE, v->length - (vaddr - v->addr)); + ssize_t ret = file_pread(v->file, (void*)vaddr, flen, vaddr - v->addr + v->offset); + kassert(ret > 0); + if (ret < RISCV_PGSIZE) + memset((void*)vaddr + ret, 0, RISCV_PGSIZE - ret); + } + else + memset((void*)vaddr, 0, RISCV_PGSIZE); + __vmr_decref(v, 1); + *pte = pte_create(ppn, prot_to_type(v->prot, 1)); + } + + pte_t perms = pte_create(0, prot_to_type(prot, 1)); + if ((*pte & perms) != perms) + return -1; + + flush_tlb(); + return 0; +} + +int handle_page_fault(uintptr_t vaddr, int prot) +{ + spinlock_lock(&vm_lock); + int ret = __handle_page_fault(vaddr, prot); + spinlock_unlock(&vm_lock); + return ret; +} + +static void __do_munmap(uintptr_t addr, size_t len) +{ + for (uintptr_t a = addr; a < addr + len; a += RISCV_PGSIZE) + { + pte_t* pte = __walk(a); + if (pte == 0 || *pte == 0) + continue; + + if (!(*pte & PTE_V)) + __vmr_decref((vmr_t*)*pte, 1); + + *pte = 0; + } + flush_tlb(); // TODO: shootdown +} + +uintptr_t __do_mmap(uintptr_t addr, size_t length, int prot, int flags, file_t* f, off_t offset) +{ + size_t npage = (length-1)/RISCV_PGSIZE+1; + if (flags & MAP_FIXED) + { + if ((addr & (RISCV_PGSIZE-1)) || !__valid_user_range(addr, length)) + return (uintptr_t)-1; + } + else if ((addr = __vm_alloc(npage)) == 0) + return (uintptr_t)-1; + + vmr_t* v = __vmr_alloc(addr, length, f, offset, npage, prot); + if (!v) + return (uintptr_t)-1; + + for (uintptr_t a = addr; a < addr + length; a += RISCV_PGSIZE) + { + pte_t* pte = __walk_create(a); + kassert(pte); + + if (*pte) + __do_munmap(a, RISCV_PGSIZE); + + *pte = (pte_t)v; + } + + if (!have_vm || (flags & MAP_POPULATE)) + for (uintptr_t a = addr; a < addr + length; a += RISCV_PGSIZE) + kassert(__handle_page_fault(a, prot) == 0); + + return addr; +} + +int do_munmap(uintptr_t addr, size_t length) +{ + if ((addr & (RISCV_PGSIZE-1)) || !__valid_user_range(addr, length)) + return -EINVAL; + + spinlock_lock(&vm_lock); + __do_munmap(addr, length); + spinlock_unlock(&vm_lock); + + return 0; +} + +uintptr_t do_mmap(uintptr_t addr, size_t length, int prot, int flags, int fd, off_t offset) +{ + if (!(flags & MAP_PRIVATE) || length == 0 || (offset & (RISCV_PGSIZE-1))) + return -EINVAL; + + file_t* f = NULL; + if (!(flags & MAP_ANONYMOUS) && (f = file_get(fd)) == NULL) + return -EBADF; + + spinlock_lock(&vm_lock); + addr = __do_mmap(addr, length, prot, flags, f, offset); + + if (addr < current.brk_max) + current.brk_max = addr; + spinlock_unlock(&vm_lock); + + if (f) file_decref(f); + return addr; +} + +uintptr_t __do_brk(size_t addr) +{ + uintptr_t newbrk = addr; + if (addr < current.brk_min) + newbrk = current.brk_min; + else if (addr > current.brk_max) + newbrk = current.brk_max; + + if (current.brk == 0) + current.brk = ROUNDUP(current.brk_min, RISCV_PGSIZE); + + uintptr_t newbrk_page = ROUNDUP(newbrk, RISCV_PGSIZE); + if (current.brk > newbrk_page) + __do_munmap(newbrk_page, current.brk - newbrk_page); + else if (current.brk < newbrk_page) + kassert(__do_mmap(current.brk, newbrk_page - current.brk, -1, MAP_FIXED|MAP_PRIVATE|MAP_ANONYMOUS, 0, 0) == current.brk); + current.brk = newbrk_page; + + return newbrk; +} + +uintptr_t do_brk(size_t addr) +{ + spinlock_lock(&vm_lock); + addr = __do_brk(addr); + spinlock_unlock(&vm_lock); + + return addr; +} + +uintptr_t do_mremap(uintptr_t addr, size_t old_size, size_t new_size, int flags) +{ + return -ENOSYS; +} + +uintptr_t do_mprotect(uintptr_t addr, size_t length, int prot) +{ + uintptr_t res = 0; + if ((addr) & (RISCV_PGSIZE-1)) + return -EINVAL; + + spinlock_lock(&vm_lock); + for (uintptr_t a = addr; a < addr + length; a += RISCV_PGSIZE) + { + pte_t* pte = __walk(a); + if (pte == 0 || *pte == 0) { + res = -ENOMEM; + break; + } + + if (!(*pte & PTE_V)) { + vmr_t* v = (vmr_t*)*pte; + if((v->prot ^ prot) & ~v->prot){ + //TODO:look at file to find perms + res = -EACCES; + break; + } + v->prot = prot; + } else { + if (((prot & PROT_WRITE) && !PTE_UW(*pte)) + || ((prot & PROT_EXEC) && !PTE_UX(*pte))) { + //TODO:look at file to find perms + res = -EACCES; + break; + } + *pte = pte_create(pte_ppn(*pte), prot_to_type(prot, 1)); + } + } + spinlock_unlock(&vm_lock); + + return res; +} + +void __map_kernel_range(uintptr_t vaddr, uintptr_t paddr, size_t len, int prot) +{ + uintptr_t n = ROUNDUP(len, RISCV_PGSIZE) / RISCV_PGSIZE; + uintptr_t offset = paddr - vaddr; + for (uintptr_t a = vaddr, i = 0; i < n; i++, a += RISCV_PGSIZE) + { + pte_t* pte = __walk_create(a); + kassert(pte); + *pte = pte_create((a + offset) >> RISCV_PGSHIFT, prot_to_type(prot, 0)); + } +} + +void populate_mapping(const void* start, size_t size, int prot) +{ + uintptr_t a0 = ROUNDDOWN((uintptr_t)start, RISCV_PGSIZE); + for (uintptr_t a = a0; a < (uintptr_t)start+size; a += RISCV_PGSIZE) + { + if (prot & PROT_WRITE) + atomic_add((int*)a, 0); + else + atomic_read((int*)a); + } +} + +uintptr_t pk_vm_init() +{ + size_t mem_pages = mem_size >> RISCV_PGSHIFT; + free_pages = MAX(8, mem_pages >> (RISCV_PGLEVEL_BITS-1)); + first_free_page = mem_size - free_pages * RISCV_PGSIZE; + + __map_kernel_range(0, 0, first_free_paddr, PROT_READ|PROT_WRITE|PROT_EXEC); + __map_kernel_range(first_free_page, first_free_page, free_pages * RISCV_PGSIZE, PROT_READ|PROT_WRITE); + + // keep user addresses positive + current.mmap_max = MIN(first_free_page, (uintptr_t)INTPTR_MAX + 1); + current.brk_max = current.mmap_max; + + size_t stack_size = RISCV_PGSIZE * CLAMP(mem_size/(RISCV_PGSIZE*32), 1, 256); + size_t stack_bottom = __do_mmap(current.mmap_max - stack_size, stack_size, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, 0, 0); + kassert(stack_bottom != (uintptr_t)-1); + current.stack_top = stack_bottom + stack_size; + + uintptr_t kernel_stack_top = __page_alloc() + RISCV_PGSIZE; + return kernel_stack_top; +} diff --git a/pk/mmap.h b/pk/mmap.h new file mode 100644 index 0000000..e3c2035 --- /dev/null +++ b/pk/mmap.h @@ -0,0 +1,34 @@ +#ifndef _MMAP_H +#define _MMAP_H + +#include "vm.h" +#include "syscall.h" +#include "encoding.h" +#include "file.h" +#include + +#define PROT_NONE 0 +#define PROT_READ 1 +#define PROT_WRITE 2 +#define PROT_EXEC 4 + +#define MAP_PRIVATE 0x2 +#define MAP_FIXED 0x10 +#define MAP_ANONYMOUS 0x20 +#define MAP_POPULATE 0x8000 +#define MREMAP_FIXED 0x2 + +extern int have_vm; +uintptr_t pk_vm_init(); +int handle_page_fault(uintptr_t vaddr, int prot); +void populate_mapping(const void* start, size_t size, int prot); +void __map_kernel_range(uintptr_t va, uintptr_t pa, size_t len, int prot); +int __valid_user_range(uintptr_t vaddr, size_t len); +uintptr_t __do_mmap(uintptr_t addr, size_t length, int prot, int flags, file_t* file, off_t offset); +uintptr_t do_mmap(uintptr_t addr, size_t length, int prot, int flags, int fd, off_t offset); +int do_munmap(uintptr_t addr, size_t length); +uintptr_t do_mremap(uintptr_t addr, size_t old_size, size_t new_size, int flags); +uintptr_t do_mprotect(uintptr_t addr, size_t length, int prot); +uintptr_t do_brk(uintptr_t addr); + +#endif diff --git a/pk/mtrap.c b/pk/mtrap.c deleted file mode 100644 index b64360e..0000000 --- a/pk/mtrap.c +++ /dev/null @@ -1,281 +0,0 @@ -#include "mtrap.h" -#include "frontend.h" -#include "mcall.h" -#include "vm.h" -#include -#include -#include - -void __attribute__((noreturn)) bad_trap() -{ - die("machine mode: unhandlable trap %d @ %p", read_csr(mcause), read_csr(mepc)); -} - -static uintptr_t mcall_hart_id() -{ - return read_const_csr(mhartid); -} - -static void request_htif_keyboard_interrupt() -{ - uintptr_t old_tohost = swap_csr(mtohost, TOHOST_CMD(1, 0, 0)); - assert(old_tohost == 0); -} - -void htif_interrupt() -{ - uintptr_t fromhost = swap_csr(mfromhost, 0); - // we should only be interrupted by keypresses - if (!(FROMHOST_DEV(fromhost) == 1 && FROMHOST_CMD(fromhost) == 0)) - die("unexpected htif interrupt"); - HLS()->console_ibuf = 1 + (uint8_t)FROMHOST_DATA(fromhost); - set_csr(mip, MIP_SSIP); -} - -static void do_tohost_fromhost(uintptr_t dev, uintptr_t cmd, uintptr_t data) -{ - while (swap_csr(mtohost, TOHOST_CMD(dev, cmd, data)) != 0) - if (read_csr(mfromhost)) - htif_interrupt(); - - while (1) { - uintptr_t fromhost = read_csr(mfromhost); - if (fromhost) { - if (FROMHOST_DEV(fromhost) == dev && FROMHOST_CMD(fromhost) == cmd) { - write_csr(mfromhost, 0); - break; - } - htif_interrupt(); - } - } -} - -static uintptr_t mcall_console_putchar(uint8_t ch) -{ - do_tohost_fromhost(1, 1, ch); - return 0; -} - -static uintptr_t mcall_htif_syscall(uintptr_t magic_mem) -{ - do_tohost_fromhost(0, 0, magic_mem); - return 0; -} - -void poweroff() -{ - while (1) - write_csr(mtohost, 1); -} - -void printm(const char* s, ...) -{ - char buf[256]; - va_list vl; - - va_start(vl, s); - vsnprintf(buf, sizeof buf, s, vl); - va_end(vl); - - for (const char* p = buf; *p; p++) - mcall_console_putchar(*p); -} - -static void send_ipi(uintptr_t recipient, int event) -{ - if ((atomic_or(&OTHER_HLS(recipient)->mipi_pending, event) & event) == 0) { - mb(); - OTHER_HLS(recipient)->csrs[CSR_MIPI] = 1; - } -} - -static uintptr_t mcall_send_ipi(uintptr_t recipient) -{ - if (recipient >= num_harts) - return -1; - - send_ipi(recipient, IPI_SOFT); - return 0; -} - -static void reset_ssip() -{ - clear_csr(mip, MIP_SSIP); - mb(); - - if (HLS()->sipi_pending || HLS()->console_ibuf > 0) - set_csr(mip, MIP_SSIP); -} - -static uintptr_t mcall_console_getchar() -{ - int ch = atomic_swap(&HLS()->console_ibuf, -1); - if (ch >= 0) - request_htif_keyboard_interrupt(); - reset_ssip(); - return ch - 1; -} - -static uintptr_t mcall_clear_ipi() -{ - int ipi = atomic_swap(&HLS()->sipi_pending, 0); - reset_ssip(); - return ipi; -} - -static uintptr_t mcall_shutdown() -{ - poweroff(); -} - -static uintptr_t mcall_set_timer(unsigned long long when) -{ - // bbl/pk don't use the timer, so there's no need to virtualize it - write_csr(mtimecmp, when); -#ifndef __riscv64 - write_csr(mtimecmph, when >> 32); -#endif - clear_csr(mip, MIP_STIP); - set_csr(mie, MIP_MTIP); - return 0; -} - -void software_interrupt() -{ - clear_csr(mip, MIP_MSIP); - mb(); - int ipi_pending = atomic_swap(&HLS()->mipi_pending, 0); - - if (ipi_pending & IPI_SOFT) { - HLS()->sipi_pending = 1; - set_csr(mip, MIP_SSIP); - } - - if (ipi_pending & IPI_FENCE_I) - asm volatile ("fence.i"); - - if (ipi_pending & IPI_SFENCE_VM) - asm volatile ("sfence.vm"); -} - -static void send_ipi_many(uintptr_t* pmask, int event) -{ - _Static_assert(MAX_HARTS <= 8 * sizeof(*pmask), "# harts > uintptr_t bits"); - uintptr_t mask = -1; - if (pmask) - mask = *pmask; - - // send IPIs to everyone - for (ssize_t i = num_harts-1; i >= 0; i--) - if ((mask >> i) & 1) - send_ipi(i, event); - - // wait until all events have been handled. - // prevent deadlock while spinning by handling any IPIs from other harts. - for (ssize_t i = num_harts-1; i >= 0; i--) - if ((mask >> i) & 1) - while (OTHER_HLS(i)->mipi_pending & event) - software_interrupt(); -} - -static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask, uintptr_t asid) -{ - // ignore the ASID and do a global flush. - // this allows us to avoid queueing a message. - send_ipi_many(hart_mask, IPI_SFENCE_VM); - return 0; -} - -static uintptr_t mcall_remote_fence_i(uintptr_t* hart_mask) -{ - send_ipi_many(hart_mask, IPI_FENCE_I); - return 0; -} - -void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval; - switch (n) - { - case MCALL_HART_ID: - retval = mcall_hart_id(); - break; - case MCALL_CONSOLE_PUTCHAR: - retval = mcall_console_putchar(arg0); - break; - case MCALL_CONSOLE_GETCHAR: - retval = mcall_console_getchar(); - break; - case MCALL_HTIF_SYSCALL: - retval = mcall_htif_syscall(arg0); - break; - case MCALL_SEND_IPI: - retval = mcall_send_ipi(arg0); - break; - case MCALL_CLEAR_IPI: - retval = mcall_clear_ipi(); - break; - case MCALL_SHUTDOWN: - retval = mcall_shutdown(); - break; - case MCALL_SET_TIMER: - retval = mcall_set_timer(arg0); - break; - case MCALL_REMOTE_SFENCE_VM: - retval = mcall_remote_sfence_vm((uintptr_t*)arg0, arg1); - break; - case MCALL_REMOTE_FENCE_I: - retval = mcall_remote_fence_i((uintptr_t*)arg0); - break; - default: - retval = -ENOSYS; - break; - } - regs[10] = retval; - write_csr(mepc, mepc + 4); -} - -void redirect_trap(uintptr_t epc, uintptr_t mstatus) -{ - write_csr(sepc, epc); - write_csr(scause, read_csr(mcause)); - write_csr(mepc, read_csr(stvec)); - - uintptr_t prev_priv = EXTRACT_FIELD(mstatus, MSTATUS_MPP); - uintptr_t prev_ie = EXTRACT_FIELD(mstatus, MSTATUS_MPIE); - mstatus = INSERT_FIELD(mstatus, MSTATUS_SPP, prev_priv); - mstatus = INSERT_FIELD(mstatus, MSTATUS_SPIE, prev_ie); - mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S); - mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); - write_csr(mstatus, mstatus); - - extern void __redirect_trap(); - return __redirect_trap(); -} - -static void machine_page_fault(uintptr_t* regs, uintptr_t mepc) -{ - // MPRV=1 iff this trap occurred while emulating an instruction on behalf - // of a lower privilege level. In that case, a2=epc and a3=mstatus. - if (read_csr(mstatus) & MSTATUS_MPRV) { - write_csr(sbadaddr, read_csr(mbadaddr)); - return redirect_trap(regs[12], regs[13]); - } - bad_trap(); -} - -void trap_from_machine_mode(uintptr_t* regs, uintptr_t dummy, uintptr_t mepc) -{ - uintptr_t mcause = read_csr(mcause); - - switch (mcause) - { - case CAUSE_FAULT_LOAD: - case CAUSE_FAULT_STORE: - return machine_page_fault(regs, mepc); - case CAUSE_MACHINE_ECALL: - return mcall_trap(regs, mcause, mepc); - default: - bad_trap(); - } -} diff --git a/pk/mtrap.h b/pk/mtrap.h deleted file mode 100644 index b6ea1df..0000000 --- a/pk/mtrap.h +++ /dev/null @@ -1,86 +0,0 @@ -#ifndef _PK_MTRAP_H -#define _PK_MTRAP_H - -#include "bits.h" -#include "encoding.h" - -#ifdef __riscv_atomic -# define MAX_HARTS 8 // arbitrary -#else -# define MAX_HARTS 1 -#endif - -#ifndef __ASSEMBLER__ - -#include "sbi.h" -#include - -#define read_const_csr(reg) ({ unsigned long __tmp; \ - asm ("csrr %0, " #reg : "=r"(__tmp)); \ - __tmp; }) - -static inline int supports_extension(char ext) -{ - return read_const_csr(misa) & (1 << (ext - 'A')); -} - -static inline int xlen() -{ - return read_const_csr(misa) < 0 ? 64 : 32; -} - -extern uintptr_t mem_size; -extern uint32_t num_harts; - -typedef uintptr_t csr_t; // TODO this might become uint128_t for RV128 - -typedef struct { - volatile csr_t* csrs; - volatile int mipi_pending; - volatile int sipi_pending; - int console_ibuf; - - uint64_t utime_delta; - uint64_t ucycle_delta; - uint64_t uinstret_delta; - uint64_t stime_delta; - uint64_t scycle_delta; - uint64_t sinstret_delta; -} hls_t; - -#define IPI_SOFT 0x1 -#define IPI_FENCE_I 0x2 -#define IPI_SFENCE_VM 0x4 - -void hls_init(uint32_t hart_id, csr_t* csrs); - -#define MACHINE_STACK_TOP() ({ \ - register uintptr_t sp asm ("sp"); \ - (void*)((sp + RISCV_PGSIZE) & -RISCV_PGSIZE); }) - -// hart-local storage, at top of stack -#define HLS() ((hls_t*)(MACHINE_STACK_TOP() - HLS_SIZE)) -#define OTHER_HLS(id) ((hls_t*)((void*)HLS() + RISCV_PGSIZE * ((id) - read_const_csr(mhartid)))) - -void parse_config_string(); -void poweroff(void) __attribute((noreturn)); -void printm(const char* s, ...); -#define assert(x) ({ if (!(x)) die("assertion failed: %s", #x); }) -#define die(str, ...) ({ printm("%s:%d: " str "\n", __FILE__, __LINE__, ##__VA_ARGS__); poweroff(); }) -#define printk(...) die("printk") - -#endif // !__ASSEMBLER__ - -#define MACHINE_STACK_SIZE RISCV_PGSIZE -#define MENTRY_FRAME_SIZE (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE \ - + HLS_SIZE) - -#ifdef __riscv_hard_float -# define SOFT_FLOAT_CONTEXT_SIZE 0 -#else -# define SOFT_FLOAT_CONTEXT_SIZE (8 * 32) -#endif -#define HLS_SIZE 64 -#define INTEGER_CONTEXT_SIZE (32 * REGBYTES) - -#endif diff --git a/pk/pk.ac b/pk/pk.ac index aa0c669..f846905 100644 --- a/pk/pk.ac +++ b/pk/pk.ac @@ -2,13 +2,3 @@ AC_ARG_ENABLE([vm], AS_HELP_STRING([--disable-vm], [Disable virtual memory])) AS_IF([test "x$enable_vm" != "xno"], [ AC_DEFINE([PK_ENABLE_VM],,[Define if virtual memory support is enabled]) ]) - -AC_ARG_ENABLE([vm], AS_HELP_STRING([--disable-logo], [Disable boot logo])) -AS_IF([test "x$enable_logo" != "xno"], [ - AC_DEFINE([PK_ENABLE_LOGO],,[Define if the RISC-V logo is to be displayed]) -]) - -AC_ARG_ENABLE([fp-emulation], AS_HELP_STRING([--disable-fp-emulation], [Disable floating-point emulation])) -AS_IF([test "x$enable_fp_emulation" != "xno"], [ - AC_DEFINE([PK_ENABLE_FP_EMULATION],,[Define if floating-point emulation is enabled]) -]) diff --git a/pk/pk.c b/pk/pk.c index 6318963..2737048 100644 --- a/pk/pk.c +++ b/pk/pk.c @@ -1,24 +1,66 @@ #include "pk.h" +#include "mmap.h" #include "boot.h" -#include "vm.h" #include "elf.h" +#include "mtrap.h" +#include "frontend.h" -void run_loaded_program(size_t argc, char** argv) +elf_info current; + +int uarch_counters_enabled; +long uarch_counters[NUM_COUNTERS]; +char* uarch_counter_names[NUM_COUNTERS]; + +static void handle_option(const char* s) { - if (current.is_supervisor) - panic("pk can't run kernel binaries; try using bbl instead"); + switch (s[1]) + { + case 's': // print cycle count upon termination + current.t0 = 1; + break; - uintptr_t kernel_stack_top = pk_vm_init(); + case 'c': // print uarch counters upon termination + // If your HW doesn't support uarch counters, then don't use this flag! + uarch_counters_enabled = 1; + break; - extern char trap_entry; - write_csr(stvec, &trap_entry); - write_csr(sscratch, 0); - clear_csr(sie, SIP_STIP | SIP_SSIP); + default: + panic("unrecognized option: `%c'", s[1]); + break; + } +} - // enter supervisor mode - prepare_supervisor_mode(); - asm volatile("la t0, 1f; csrw mepc, t0; eret; 1:" ::: "t0"); +#define MAX_ARGS 64 +typedef union { + uint64_t buf[MAX_ARGS]; + char* argv[MAX_ARGS]; +} arg_buf; + +static size_t parse_args(arg_buf* args) +{ + long r = frontend_syscall(SYS_getmainvars, (uintptr_t)args, sizeof(*args), 0, 0, 0, 0, 0); + kassert(r == 0); + uint64_t* pk_argv = &args->buf[1]; + // pk_argv[0] is the proxy kernel itself. skip it and any flags. + size_t pk_argc = args->buf[0], arg = 1; + for ( ; arg < pk_argc && *(char*)(uintptr_t)pk_argv[arg] == '-'; arg++) + handle_option((const char*)(uintptr_t)pk_argv[arg]); + + for (size_t i = 0; arg + i < pk_argc; i++) + args->argv[i] = (char*)(uintptr_t)pk_argv[arg + i]; + return pk_argc - arg; +} + +static void init_tf(trapframe_t* tf, long pc, long sp) +{ + memset(tf, 0, sizeof(*tf)); + tf->status = (read_csr(sstatus) &~ SSTATUS_SPP &~ SSTATUS_SIE) | SSTATUS_SPIE; + tf->gpr[2] = sp; + tf->epc = pc; +} +static void run_loaded_program(size_t argc, char** argv, uintptr_t kstack_top) +{ // copy phdrs to user stack size_t stack_top = current.stack_top - current.phdr_size; memcpy((void*)stack_top, (void*)current.phdr, current.phdr_size); @@ -94,10 +136,37 @@ void run_loaded_program(size_t argc, char** argv) trapframe_t tf; init_tf(&tf, current.entry, stack_top); __clear_cache(0, 0); - write_csr(sscratch, kernel_stack_top); + write_csr(sscratch, kstack_top); start_user(&tf); } +static void rest_of_boot_loader(uintptr_t kstack_top) +{ + arg_buf args; + size_t argc = parse_args(&args); + if (!argc) + panic("tell me what ELF to load!"); + + // load program named by argv[0] + long phdrs[128]; + current.phdr = (uintptr_t)phdrs; + current.phdr_size = sizeof(phdrs); + load_elf(args.argv[0], ¤t); + + run_loaded_program(argc, args.argv, kstack_top); +} + +void boot_loader() +{ + extern char trap_entry; + write_csr(stvec, &trap_entry); + write_csr(sscratch, 0); + write_csr(sie, 0); + + file_init(); + enter_supervisor_mode(rest_of_boot_loader, pk_vm_init()); +} + void boot_other_hart() { // stall all harts besides hart 0 diff --git a/pk/pk.h b/pk/pk.h index 280ad6a..c86d596 100644 --- a/pk/pk.h +++ b/pk/pk.h @@ -24,30 +24,18 @@ typedef struct #define kassert(cond) do { if(!(cond)) kassert_fail(""#cond); } while(0) void do_panic(const char* s, ...) __attribute__((noreturn)); void kassert_fail(const char* s) __attribute__((noreturn)); -#define MAX(a, b) ((a) > (b) ? (a) : (b)) -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define CLAMP(a, lo, hi) MIN(MAX(a, lo), hi) #ifdef __cplusplus extern "C" { #endif -extern int have_vm; - void printk(const char* s, ...); void printm(const char* s, ...); int vsnprintf(char* out, size_t n, const char* s, va_list vl); int snprintf(char* out, size_t n, const char* s, ...); -void init_tf(trapframe_t*, long pc, long sp); void start_user(trapframe_t* tf) __attribute__((noreturn)); void dump_tf(trapframe_t*); -void unhandled_trap(trapframe_t*); -void handle_misaligned_load(trapframe_t*); -void handle_misaligned_store(trapframe_t*); -void handle_fault_load(trapframe_t*); -void handle_fault_store(trapframe_t*); - static inline int insn_len(long insn) { return (insn & 0x3) < 0x3 ? 2 : 4; @@ -60,11 +48,6 @@ extern int uarch_counters_enabled; extern long uarch_counters[NUM_COUNTERS]; extern char* uarch_counter_names[NUM_COUNTERS]; -static inline void wfi() -{ - asm volatile ("wfi" ::: "memory"); -} - #ifdef __cplusplus } #endif diff --git a/pk/pk.ld b/pk/pk.ld deleted file mode 100644 index 77ba6cf..0000000 --- a/pk/pk.ld +++ /dev/null @@ -1,95 +0,0 @@ -OUTPUT_ARCH( "riscv" ) - -ENTRY( reset_vector ) - -SECTIONS -{ - - /*--------------------------------------------------------------------*/ - /* Code and read-only segment */ - /*--------------------------------------------------------------------*/ - - /* Begining of code and text segment */ - . = 0; - _ftext = .; - PROVIDE( eprol = . ); - - .text : - { - *(.text.init) - } - - /* text: Program code section */ - .text : - { - *(.text) - *(.text.*) - *(.gnu.linkonce.t.*) - } - - /* rodata: Read-only data */ - .rodata : - { - *(.rdata) - *(.rodata) - *(.rodata.*) - *(.gnu.linkonce.r.*) - } - - /* End of code and read-only segment */ - PROVIDE( etext = . ); - _etext = .; - - /*--------------------------------------------------------------------*/ - /* Initialized data segment */ - /*--------------------------------------------------------------------*/ - - /* Start of initialized data segment */ - . = ALIGN(16); - _fdata = .; - - /* data: Writable data */ - .data : - { - *(.data) - *(.data.*) - *(.srodata*) - *(.gnu.linkonce.d.*) - *(.comment) - } - - /* End of initialized data segment */ - . = ALIGN(4); - PROVIDE( edata = . ); - _edata = .; - - /*--------------------------------------------------------------------*/ - /* Uninitialized data segment */ - /*--------------------------------------------------------------------*/ - - /* Start of uninitialized data segment */ - . = .; - _fbss = .; - - /* sbss: Uninitialized writeable small data section */ - . = .; - - /* bss: Uninitialized writeable data section */ - . = .; - _bss_start = .; - .bss : - { - *(.bss) - *(.bss.*) - *(.sbss*) - *(.gnu.linkonce.b.*) - *(COMMON) - } - - .sbi : - { - sbi.o(.sbi) - } - - _end = .; -} diff --git a/pk/pk.lds b/pk/pk.lds new file mode 100644 index 0000000..0897e50 --- /dev/null +++ b/pk/pk.lds @@ -0,0 +1,90 @@ +OUTPUT_ARCH( "riscv" ) + +ENTRY( reset_vector ) + +SECTIONS +{ + + /*--------------------------------------------------------------------*/ + /* Code and read-only segment */ + /*--------------------------------------------------------------------*/ + + /* Begining of code and text segment */ + . = 0; + _ftext = .; + PROVIDE( eprol = . ); + + .text : + { + *(.text.init) + } + + /* text: Program code section */ + .text : + { + *(.text) + *(.text.*) + *(.gnu.linkonce.t.*) + } + + /* rodata: Read-only data */ + .rodata : + { + *(.rdata) + *(.rodata) + *(.rodata.*) + *(.gnu.linkonce.r.*) + } + + /* End of code and read-only segment */ + PROVIDE( etext = . ); + _etext = .; + + /*--------------------------------------------------------------------*/ + /* Initialized data segment */ + /*--------------------------------------------------------------------*/ + + /* Start of initialized data segment */ + . = ALIGN(16); + _fdata = .; + + /* data: Writable data */ + .data : + { + *(.data) + *(.data.*) + *(.srodata*) + *(.gnu.linkonce.d.*) + *(.comment) + } + + /* End of initialized data segment */ + . = ALIGN(4); + PROVIDE( edata = . ); + _edata = .; + + /*--------------------------------------------------------------------*/ + /* Uninitialized data segment */ + /*--------------------------------------------------------------------*/ + + /* Start of uninitialized data segment */ + . = .; + _fbss = .; + + /* sbss: Uninitialized writeable small data section */ + . = .; + + /* bss: Uninitialized writeable data section */ + . = .; + _bss_start = .; + .bss : + { + *(.bss) + *(.bss.*) + *(.sbss*) + *(.gnu.linkonce.b.*) + *(COMMON) + } + + _end = .; +} diff --git a/pk/pk.mk.in b/pk/pk.mk.in index 78f826e..3caa25d 100644 --- a/pk/pk.mk.in +++ b/pk/pk.mk.in @@ -1,52 +1,30 @@ pk_subproject_deps = \ + util \ softfloat \ + machine \ pk_hdrs = \ - atomic.h \ - bits.h \ boot.h \ elf.h \ - emulation.h \ - encoding.h \ file.h \ - fp_emulation.h \ frontend.h \ - mcall.h \ - mtrap.h \ + mmap.h \ pk.h \ - sbi.h \ syscall.h \ - unprivileged_memory.h \ - vm.h \ pk_c_srcs = \ - snprintf.c \ - mtrap.c \ - minit.c \ - emulation.c \ - fp_emulation.c \ - sbi_impl.c \ - init.c \ file.c \ syscall.c \ handlers.c \ frontend.c \ elf.c \ console.c \ - vm.c \ - string.c \ - logo.c \ - configstring.c \ + mmap.c \ pk_asm_srcs = \ - mentry.S \ entry.S \ - fp_asm.S \ - sbi_entry.S \ - sbi.S \ pk_test_srcs = pk_install_prog_srcs = \ pk.c \ - bbl.c \ diff --git a/pk/sbi.S b/pk/sbi.S deleted file mode 100644 index cbea78a..0000000 --- a/pk/sbi.S +++ /dev/null @@ -1,15 +0,0 @@ -.globl sbi_hart_id; sbi_hart_id = -2048 -.globl sbi_num_harts; sbi_num_harts = -2032 -.globl sbi_query_memory; sbi_query_memory = -2016 -.globl sbi_console_putchar; sbi_console_putchar = -2000 -.globl sbi_console_getchar; sbi_console_getchar = -1984 -.globl sbi_send_ipi; sbi_send_ipi = -1952 -.globl sbi_clear_ipi; sbi_clear_ipi = -1936 -.globl sbi_timebase; sbi_timebase = -1920 -.globl sbi_shutdown; sbi_shutdown = -1904 -.globl sbi_set_timer; sbi_set_timer = -1888 -.globl sbi_mask_interrupt; sbi_mask_interrupt = -1872 -.globl sbi_unmask_interrupt; sbi_unmask_interrupt = -1856 -.globl sbi_remote_sfence_vm; sbi_remote_sfence_vm = -1840 -.globl sbi_remote_sfence_vm_range; sbi_remote_sfence_vm_range = -1824 -.globl sbi_remote_fence_i; sbi_remote_fence_i = -1808 diff --git a/pk/sbi.h b/pk/sbi.h deleted file mode 100644 index 4e2fbd8..0000000 --- a/pk/sbi.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef _ASM_RISCV_SBI_H -#define _ASM_RISCV_SBI_H - -typedef struct { - unsigned long base; - unsigned long size; - unsigned long node_id; -} memory_block_info; - -unsigned long sbi_query_memory(unsigned long id, memory_block_info *p); - -unsigned long sbi_hart_id(void); -unsigned long sbi_num_harts(void); -unsigned long sbi_timebase(void); -void sbi_set_timer(unsigned long long stime_value); -void sbi_send_ipi(unsigned long hart_id); -unsigned long sbi_clear_ipi(void); -void sbi_shutdown(void); - -void sbi_console_putchar(unsigned char ch); -int sbi_console_getchar(void); - -void sbi_remote_sfence_vm(unsigned long hart_mask_ptr, unsigned long asid); -void sbi_remote_sfence_vm_range(unsigned long hart_mask_ptr, unsigned long asid, unsigned long start, unsigned long size); -void sbi_remote_fence_i(unsigned long hart_mask_ptr); - -unsigned long sbi_mask_interrupt(unsigned long which); -unsigned long sbi_unmask_interrupt(unsigned long which); - -#endif diff --git a/pk/sbi_entry.S b/pk/sbi_entry.S deleted file mode 100644 index bfb0703..0000000 --- a/pk/sbi_entry.S +++ /dev/null @@ -1,109 +0,0 @@ -#include "encoding.h" -#include "mcall.h" - - .section .sbi,"ax",@progbits - .option norvc - .align RISCV_PGSHIFT - .globl sbi_base -sbi_base: - - # TODO: figure out something better to do with this space. It's not - # protected from the OS, so beware. - .skip RISCV_PGSIZE - 2048 - - # hart_id - .align 4 - li a7, MCALL_HART_ID - ecall - ret - - # num_harts - .align 4 - lw a0, num_harts - ret - - # query_memory - .align 4 - j __sbi_query_memory - - # console_putchar - .align 4 - li a7, MCALL_CONSOLE_PUTCHAR - ecall - ret - - # console_getchar - .align 4 - li a7, MCALL_CONSOLE_GETCHAR - ecall - ret - - # empty - .align 4 - unimp - - # send ipi - .align 4 - li a7, MCALL_SEND_IPI - ecall - ret - - # clear ipi - .align 4 - li a7, MCALL_CLEAR_IPI - ecall - ret - - # timebase - .align 4 - li a0, 10000000 # or, you know, we could provide the correct answer - ret - - # shutdown - .align 4 - li a7, MCALL_SHUTDOWN - ecall - - # set_timer - .align 4 - li a7, MCALL_SET_TIMER - ecall - ret - - # mask_interrupt - .align 4 - j __sbi_mask_interrupt - - # unmask_interrupt - .align 4 - j __sbi_unmask_interrupt - - # remote_sfence_vm - .align 4 - li a7, MCALL_REMOTE_SFENCE_VM - ecall - ret - - # remote_sfence_vm_range - .align 4 - li a7, MCALL_REMOTE_SFENCE_VM - ecall - ret - - # remote_fence_i - .align 4 - li a7, MCALL_REMOTE_FENCE_I - ecall - ret - - # end of SBI trampolines - - .globl do_mcall -do_mcall: - mv a7, a0 - mv a0, a1 - mv a1, a2 - ecall - ret - - .align RISCV_PGSHIFT diff --git a/pk/sbi_impl.c b/pk/sbi_impl.c deleted file mode 100644 index 4af0cd4..0000000 --- a/pk/sbi_impl.c +++ /dev/null @@ -1,34 +0,0 @@ -#include "mtrap.h" -#include "sbi.h" -#include "boot.h" - -uintptr_t __sbi_query_memory(uintptr_t id, memory_block_info *p) -{ - if (id == 0) { - p->base = current.first_free_paddr; - p->size = mem_size - p->base; - return 0; - } - - return -1; -} - -#define LOW_IRQ_OK(n) ((n) == IRQ_S_SOFT || (n) == IRQ_S_TIMER) - -uintptr_t __sbi_mask_interrupt(uintptr_t which) -{ - if (!LOW_IRQ_OK(which)) - return -1; - - clear_csr(sie, 1UL << which); - return 0; -} - -uintptr_t __sbi_unmask_interrupt(uintptr_t which) -{ - if (!LOW_IRQ_OK(which)) - return -1; - - set_csr(sie, 1UL << which); - return 0; -} diff --git a/pk/snprintf.c b/pk/snprintf.c deleted file mode 100644 index 1544a6c..0000000 --- a/pk/snprintf.c +++ /dev/null @@ -1,98 +0,0 @@ -// See LICENSE for license details. - -#include -#include -#include -#include - -int vsnprintf(char* out, size_t n, const char* s, va_list vl) -{ - bool format = false; - bool longarg = false; - size_t pos = 0; - for( ; *s; s++) - { - if(format) - { - switch(*s) - { - case 'l': - longarg = true; - break; - case 'p': - longarg = true; - if (++pos < n) out[pos-1] = '0'; - if (++pos < n) out[pos-1] = 'x'; - case 'x': - { - long num = longarg ? va_arg(vl, long) : va_arg(vl, int); - for(int i = 2*(longarg ? sizeof(long) : sizeof(int))-1; i >= 0; i--) { - int d = (num >> (4*i)) & 0xF; - if (++pos < n) out[pos-1] = (d < 10 ? '0'+d : 'a'+d-10); - } - longarg = false; - format = false; - break; - } - case 'd': - { - long num = longarg ? va_arg(vl, long) : va_arg(vl, int); - if (num < 0) { - num = -num; - if (++pos < n) out[pos-1] = '-'; - } - long digits = 1; - for (long nn = num; nn /= 10; digits++) - ; - for (int i = digits-1; i >= 0; i--) { - if (pos + i + 1 < n) out[pos + i] = '0' + (num % 10); - num /= 10; - } - pos += digits; - longarg = false; - format = false; - break; - } - case 's': - { - const char* s2 = va_arg(vl, const char*); - while (*s2) { - if (++pos < n) - out[pos-1] = *s2; - s2++; - } - longarg = false; - format = false; - break; - } - case 'c': - { - if (++pos < n) out[pos-1] = (char)va_arg(vl,int); - longarg = false; - format = false; - break; - } - default: - break; - } - } - else if(*s == '%') - format = true; - else - if (++pos < n) out[pos-1] = *s; - } - if (pos < n) - out[pos] = 0; - else if (n) - out[n-1] = 0; - return pos; -} - -int snprintf(char* out, size_t n, const char* s, ...) -{ - va_list vl; - va_start(vl, s); - int res = vsnprintf(out, n, s, vl); - va_end(vl); - return res; -} diff --git a/pk/string.c b/pk/string.c deleted file mode 100644 index e896379..0000000 --- a/pk/string.c +++ /dev/null @@ -1,87 +0,0 @@ -#include -#include -#include - -void* memcpy(void* dest, const void* src, size_t len) -{ - if ((((uintptr_t)dest | (uintptr_t)src | len) & (sizeof(uintptr_t)-1)) == 0) { - const uintptr_t* s = src; - uintptr_t *d = dest; - while (d < (uintptr_t*)(dest + len)) - *d++ = *s++; - } else { - const char* s = src; - char *d = dest; - while (d < (char*)(dest + len)) - *d++ = *s++; - } - return dest; -} - -void* memset(void* dest, int byte, size_t len) -{ - if ((((uintptr_t)dest | len) & (sizeof(uintptr_t)-1)) == 0) { - uintptr_t word = byte & 0xFF; - word |= word << 8; - word |= word << 16; - word |= word << 16 << 16; - - uintptr_t *d = dest; - while (d < (uintptr_t*)(dest + len)) - *d++ = word; - } else { - char *d = dest; - while (d < (char*)(dest + len)) - *d++ = byte; - } - return dest; -} - -size_t strlen(const char *s) -{ - const char *p = s; - while (*p) - p++; - return p - s; -} - -int strcmp(const char* s1, const char* s2) -{ - unsigned char c1, c2; - - do { - c1 = *s1++; - c2 = *s2++; - } while (c1 != 0 && c1 == c2); - - return c1 - c2; -} - -char* strcpy(char* dest, const char* src) -{ - char* d = dest; - while ((*d++ = *src++)) - ; - return dest; -} - -long atol(const char* str) -{ - long res = 0; - int sign = 0; - - while (*str == ' ') - str++; - - if (*str == '-' || *str == '+') { - sign = *str == '-'; - str++; - } - - while (*str) { - res *= 10; - res += *str++ - '0'; - } - - return sign ? -res : res; -} diff --git a/pk/syscall.c b/pk/syscall.c index 17112d8..de11a0a 100644 --- a/pk/syscall.c +++ b/pk/syscall.c @@ -4,7 +4,7 @@ #include "pk.h" #include "file.h" #include "frontend.h" -#include "vm.h" +#include "mmap.h" #include "boot.h" #include #include diff --git a/pk/unprivileged_memory.h b/pk/unprivileged_memory.h deleted file mode 100644 index d03cc5e..0000000 --- a/pk/unprivileged_memory.h +++ /dev/null @@ -1,79 +0,0 @@ -#ifndef _RISCV_MISALIGNED_H -#define _RISCV_MISALIGNED_H - -#include "encoding.h" -#include - -#define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type, insn) \ - static inline type load_##type(const type* addr, uintptr_t mepc) \ - { \ - register uintptr_t __mepc asm ("a2") = mepc; \ - register uintptr_t __mstatus asm ("a3"); \ - type val; \ - asm ("csrrs %0, mstatus, %3\n" \ - #insn " %1, %2\n" \ - "csrw mstatus, %0" \ - : "+&r" (__mstatus), "=&r" (val) \ - : "m" (*addr), "r" (MSTATUS_MPRV), "r" (__mepc)); \ - return val; \ - } - -#define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type, insn) \ - static inline void store_##type(type* addr, type val, uintptr_t mepc) \ - { \ - register uintptr_t __mepc asm ("a2") = mepc; \ - register uintptr_t __mstatus asm ("a3"); \ - asm volatile ("csrrs %0, mstatus, %3\n" \ - #insn " %1, %2\n" \ - "csrw mstatus, %0" \ - : "+&r" (__mstatus) \ - : "r" (val), "m" (*addr), "r" (MSTATUS_MPRV), \ - "r" (__mepc)); \ - } - -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint8_t, lbu) -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint16_t, lhu) -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int8_t, lb) -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int16_t, lh) -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(int32_t, lw) -DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint8_t, sb) -DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint16_t, sh) -DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint32_t, sw) -#ifdef __riscv64 -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lwu) -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint64_t, ld) -DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint64_t, sd) -#else -DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lw) -#endif - -static uint32_t __attribute__((always_inline)) get_insn(uintptr_t mepc, uintptr_t* mstatus) -{ - register uintptr_t __mepc asm ("a2") = mepc; - register uintptr_t __mstatus asm ("a3"); - uint32_t val; -#ifndef __riscv_compressed - asm ("csrrs %[mstatus], mstatus, %[mprv]\n" - "lw %[insn], (%[addr])\n" - "csrw mstatus, %[mstatus]" - : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val) - : [mprv] "r" (MSTATUS_MPRV), [addr] "r" (__mepc)); -#else - uintptr_t rvc_mask = 3, tmp; - asm ("csrrs %[mstatus], mstatus, %[mprv]\n" - "lhu %[insn], (%[addr])\n" - "and %[tmp], %[insn], %[rvc_mask]\n" - "bne %[tmp], %[rvc_mask], 1f\n" - "lh %[tmp], 2(%[addr])\n" - "sll %[tmp], %[tmp], 16\n" - "add %[insn], %[insn], %[tmp]\n" - "1: csrw mstatus, %[mstatus]" - : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val), [tmp] "=&r" (tmp) - : [mprv] "r" (MSTATUS_MPRV), [addr] "r" (__mepc), - [rvc_mask] "r" (rvc_mask)); -#endif - *mstatus = __mstatus; - return val; -} - -#endif diff --git a/pk/vm.c b/pk/vm.c deleted file mode 100644 index c29434d..0000000 --- a/pk/vm.c +++ /dev/null @@ -1,500 +0,0 @@ -#include "vm.h" -#include "file.h" -#include "atomic.h" -#include "pk.h" -#include "boot.h" -#include "mtrap.h" -#include -#include - -typedef struct { - uintptr_t addr; - size_t length; - file_t* file; - size_t offset; - unsigned refcnt; - int prot; -} vmr_t; - -#define MAX_VMR (RISCV_PGSIZE / sizeof(vmr_t)) -spinlock_t vm_lock = SPINLOCK_INIT; -static vmr_t* vmrs; - -pte_t* root_page_table; -static uintptr_t first_free_page; -static size_t next_free_page; -static size_t free_pages; - -static uintptr_t __page_alloc() -{ - kassert(next_free_page != free_pages); - uintptr_t addr = first_free_page + RISCV_PGSIZE * next_free_page++; - memset((void*)addr, 0, RISCV_PGSIZE); - return addr; -} - -static vmr_t* __vmr_alloc(uintptr_t addr, size_t length, file_t* file, - size_t offset, unsigned refcnt, int prot) -{ - if (!vmrs) { - spinlock_lock(&vm_lock); - if (!vmrs) - vmrs = (vmr_t*)__page_alloc(); - spinlock_unlock(&vm_lock); - } - - for (vmr_t* v = vmrs; v < vmrs + MAX_VMR; v++) { - if (v->refcnt == 0) { - if (file) - file_incref(file); - v->addr = addr; - v->length = length; - v->file = file; - v->offset = offset; - v->refcnt = refcnt; - v->prot = prot; - return v; - } - } - return NULL; -} - -static void __vmr_decref(vmr_t* v, unsigned dec) -{ - if ((v->refcnt -= dec) == 0) - { - if (v->file) - file_decref(v->file); - } -} - -static size_t pte_ppn(pte_t pte) -{ - return pte >> PTE_PPN_SHIFT; -} - -static pte_t ptd_create(uintptr_t ppn) -{ - return (ppn << PTE_PPN_SHIFT) | PTE_V | PTE_TYPE_TABLE; -} - -static inline pte_t pte_create(uintptr_t ppn, int prot, int user) -{ - pte_t pte = (ppn << PTE_PPN_SHIFT) | PTE_V; - prot &= PROT_READ|PROT_WRITE|PROT_EXEC; - if (user) { - switch (prot) { - case PROT_NONE: pte |= PTE_TYPE_SR; break; - case PROT_READ: pte |= PTE_TYPE_UR_SR; break; - case PROT_WRITE: pte |= PTE_TYPE_URW_SRW; break; - case PROT_EXEC: pte |= PTE_TYPE_URX_SRX; break; - case PROT_READ|PROT_WRITE: pte |= PTE_TYPE_URW_SRW; break; - case PROT_READ|PROT_EXEC: pte |= PTE_TYPE_URX_SRX; break; - case PROT_WRITE|PROT_EXEC: pte |= PTE_TYPE_URWX_SRWX; break; - case PROT_READ|PROT_WRITE|PROT_EXEC: pte |= PTE_TYPE_URWX_SRWX; break; - } - } else { - switch (prot) { - case PROT_NONE: kassert(0); break; - case PROT_READ: pte |= PTE_TYPE_SR; break; - case PROT_WRITE: pte |= PTE_TYPE_SRW; break; - case PROT_EXEC: pte |= PTE_TYPE_SRX; break; - case PROT_READ|PROT_WRITE: pte |= PTE_TYPE_SRW; break; - case PROT_READ|PROT_EXEC: pte |= PTE_TYPE_SRX; break; - case PROT_WRITE|PROT_EXEC: pte |= PTE_TYPE_SRWX; break; - case PROT_READ|PROT_WRITE|PROT_EXEC: pte |= PTE_TYPE_SRWX; break; - } - } - return pte; -} - -static uintptr_t ppn(uintptr_t addr) -{ - return addr >> RISCV_PGSHIFT; -} - -static size_t pt_idx(uintptr_t addr, int level) -{ - size_t idx = addr >> (RISCV_PGLEVEL_BITS*level + RISCV_PGSHIFT); - return idx & ((1 << RISCV_PGLEVEL_BITS) - 1); -} - -static void __maybe_create_root_page_table() -{ - if (root_page_table) - return; - root_page_table = (void*)__page_alloc(); - if (have_vm) - write_csr(sptbr, (uintptr_t)root_page_table >> RISCV_PGSHIFT); -} - -static pte_t* __walk_internal(uintptr_t addr, int create) -{ - const size_t pte_per_page = RISCV_PGSIZE/sizeof(void*); - __maybe_create_root_page_table(); - pte_t* t = root_page_table; - unsigned levels = (VA_BITS - RISCV_PGSHIFT) / RISCV_PGLEVEL_BITS; - - for (unsigned i = levels-1; i > 0; i--) - { - size_t idx = pt_idx(addr, i); - if (!(t[idx] & PTE_V)) - { - if (!create) - return 0; - uintptr_t page = __page_alloc(); - t[idx] = ptd_create(ppn(page)); - } - else - kassert(PTE_TABLE(t[idx])); - t = (pte_t*)(pte_ppn(t[idx]) << RISCV_PGSHIFT); - } - return &t[pt_idx(addr, 0)]; -} - -static pte_t* __walk(uintptr_t addr) -{ - return __walk_internal(addr, 0); -} - -static pte_t* __walk_create(uintptr_t addr) -{ - return __walk_internal(addr, 1); -} - -static int __va_avail(uintptr_t vaddr) -{ - pte_t* pte = __walk(vaddr); - return pte == 0 || *pte == 0; -} - -static uintptr_t __vm_alloc(size_t npage) -{ - uintptr_t start = current.brk, end = current.mmap_max - npage*RISCV_PGSIZE; - for (uintptr_t a = start; a <= end; a += RISCV_PGSIZE) - { - if (!__va_avail(a)) - continue; - uintptr_t first = a, last = a + (npage-1) * RISCV_PGSIZE; - for (a = last; a > first && __va_avail(a); a -= RISCV_PGSIZE) - ; - if (a > first) - continue; - return a; - } - return 0; -} - -int __valid_user_range(uintptr_t vaddr, size_t len) -{ - if (vaddr + len < vaddr) - return 0; - return vaddr >= current.first_free_paddr && vaddr + len <= current.mmap_max; -} - -static int __handle_page_fault(uintptr_t vaddr, int prot) -{ - uintptr_t vpn = vaddr >> RISCV_PGSHIFT; - vaddr = vpn << RISCV_PGSHIFT; - - pte_t* pte = __walk(vaddr); - - if (pte == 0 || *pte == 0 || !__valid_user_range(vaddr, 1)) - return -1; - else if (!(*pte & PTE_V)) - { - uintptr_t ppn = vpn; - - vmr_t* v = (vmr_t*)*pte; - *pte = pte_create(ppn, PROT_READ|PROT_WRITE, 0); - flush_tlb(); - if (v->file) - { - size_t flen = MIN(RISCV_PGSIZE, v->length - (vaddr - v->addr)); - ssize_t ret = file_pread(v->file, (void*)vaddr, flen, vaddr - v->addr + v->offset); - kassert(ret > 0); - if (ret < RISCV_PGSIZE) - memset((void*)vaddr + ret, 0, RISCV_PGSIZE - ret); - } - else - memset((void*)vaddr, 0, RISCV_PGSIZE); - __vmr_decref(v, 1); - *pte = pte_create(ppn, v->prot, 1); - } - - pte_t perms = pte_create(0, prot, 1); - if ((*pte & perms) != perms) - return -1; - - flush_tlb(); - return 0; -} - -int handle_page_fault(uintptr_t vaddr, int prot) -{ - spinlock_lock(&vm_lock); - int ret = __handle_page_fault(vaddr, prot); - spinlock_unlock(&vm_lock); - return ret; -} - -static void __do_munmap(uintptr_t addr, size_t len) -{ - for (uintptr_t a = addr; a < addr + len; a += RISCV_PGSIZE) - { - pte_t* pte = __walk(a); - if (pte == 0 || *pte == 0) - continue; - - if (!(*pte & PTE_V)) - __vmr_decref((vmr_t*)*pte, 1); - - *pte = 0; - } - flush_tlb(); // TODO: shootdown -} - -uintptr_t __do_mmap(uintptr_t addr, size_t length, int prot, int flags, file_t* f, off_t offset) -{ - size_t npage = (length-1)/RISCV_PGSIZE+1; - if (flags & MAP_FIXED) - { - if ((addr & (RISCV_PGSIZE-1)) || !__valid_user_range(addr, length)) - return (uintptr_t)-1; - } - else if ((addr = __vm_alloc(npage)) == 0) - return (uintptr_t)-1; - - vmr_t* v = __vmr_alloc(addr, length, f, offset, npage, prot); - if (!v) - return (uintptr_t)-1; - - for (uintptr_t a = addr; a < addr + length; a += RISCV_PGSIZE) - { - pte_t* pte = __walk_create(a); - kassert(pte); - - if (*pte) - __do_munmap(a, RISCV_PGSIZE); - - *pte = (pte_t)v; - } - - if (!have_vm || (flags & MAP_POPULATE)) - for (uintptr_t a = addr; a < addr + length; a += RISCV_PGSIZE) - kassert(__handle_page_fault(a, prot) == 0); - - return addr; -} - -int do_munmap(uintptr_t addr, size_t length) -{ - if ((addr & (RISCV_PGSIZE-1)) || !__valid_user_range(addr, length)) - return -EINVAL; - - spinlock_lock(&vm_lock); - __do_munmap(addr, length); - spinlock_unlock(&vm_lock); - - return 0; -} - -uintptr_t do_mmap(uintptr_t addr, size_t length, int prot, int flags, int fd, off_t offset) -{ - if (!(flags & MAP_PRIVATE) || length == 0 || (offset & (RISCV_PGSIZE-1))) - return -EINVAL; - - file_t* f = NULL; - if (!(flags & MAP_ANONYMOUS) && (f = file_get(fd)) == NULL) - return -EBADF; - - spinlock_lock(&vm_lock); - addr = __do_mmap(addr, length, prot, flags, f, offset); - - if (addr < current.brk_max) - current.brk_max = addr; - spinlock_unlock(&vm_lock); - - if (f) file_decref(f); - return addr; -} - -uintptr_t __do_brk(size_t addr) -{ - uintptr_t newbrk = addr; - if (addr < current.brk_min) - newbrk = current.brk_min; - else if (addr > current.brk_max) - newbrk = current.brk_max; - - if (current.brk == 0) - current.brk = ROUNDUP(current.brk_min, RISCV_PGSIZE); - - uintptr_t newbrk_page = ROUNDUP(newbrk, RISCV_PGSIZE); - if (current.brk > newbrk_page) - __do_munmap(newbrk_page, current.brk - newbrk_page); - else if (current.brk < newbrk_page) - kassert(__do_mmap(current.brk, newbrk_page - current.brk, -1, MAP_FIXED|MAP_PRIVATE|MAP_ANONYMOUS, 0, 0) == current.brk); - current.brk = newbrk_page; - - return newbrk; -} - -uintptr_t do_brk(size_t addr) -{ - spinlock_lock(&vm_lock); - addr = __do_brk(addr); - spinlock_unlock(&vm_lock); - - return addr; -} - -uintptr_t do_mremap(uintptr_t addr, size_t old_size, size_t new_size, int flags) -{ - return -ENOSYS; -} - -uintptr_t do_mprotect(uintptr_t addr, size_t length, int prot) -{ - uintptr_t res = 0; - if ((addr) & (RISCV_PGSIZE-1)) - return -EINVAL; - - spinlock_lock(&vm_lock); - for (uintptr_t a = addr; a < addr + length; a += RISCV_PGSIZE) - { - pte_t* pte = __walk(a); - if (pte == 0 || *pte == 0) { - res = -ENOMEM; - break; - } - - if (!(*pte & PTE_V)) { - vmr_t* v = (vmr_t*)*pte; - if((v->prot ^ prot) & ~v->prot){ - //TODO:look at file to find perms - res = -EACCES; - break; - } - v->prot = prot; - } else { - if (((prot & PROT_WRITE) && !PTE_UW(*pte)) - || ((prot & PROT_EXEC) && !PTE_UX(*pte))) { - //TODO:look at file to find perms - res = -EACCES; - break; - } - *pte = pte_create(pte_ppn(*pte), prot, 1); - } - } - spinlock_unlock(&vm_lock); - - return res; -} - -void __map_kernel_range(uintptr_t vaddr, uintptr_t paddr, size_t len, int prot) -{ - uintptr_t n = ROUNDUP(len, RISCV_PGSIZE) / RISCV_PGSIZE; - for (uintptr_t a = vaddr, i = 0; i < n; i++, a += RISCV_PGSIZE) - { - pte_t* pte = __walk_create(a); - kassert(pte); - *pte = pte_create((a - vaddr + paddr) >> RISCV_PGSHIFT, prot, 0); - } -} - -void populate_mapping(const void* start, size_t size, int prot) -{ - uintptr_t a0 = ROUNDDOWN((uintptr_t)start, RISCV_PGSIZE); - for (uintptr_t a = a0; a < (uintptr_t)start+size; a += RISCV_PGSIZE) - { - if (prot & PROT_WRITE) - atomic_add((int*)a, 0); - else - atomic_read((int*)a); - } -} - -static uintptr_t sbi_top_paddr() -{ - extern char _end; - return ROUNDUP((uintptr_t)&_end, RISCV_PGSIZE); -} - -#define first_free_paddr() (sbi_top_paddr() + num_harts * RISCV_PGSIZE) - -void vm_init() -{ - mem_size = mem_size / SUPERPAGE_SIZE * SUPERPAGE_SIZE; - current.first_free_paddr = first_free_paddr(); - - size_t mem_pages = mem_size >> RISCV_PGSHIFT; - free_pages = MAX(8, mem_pages >> (RISCV_PGLEVEL_BITS-1)); - first_free_page = mem_size - free_pages * RISCV_PGSIZE; - current.mmap_max = current.brk_max = first_free_page; -} - -void supervisor_vm_init() -{ - uintptr_t highest_va = -current.first_free_paddr; - mem_size = MIN(mem_size, highest_va - current.first_user_vaddr) & -SUPERPAGE_SIZE; - - pte_t* sbi_pt = (pte_t*)(current.first_vaddr_after_user + current.bias); - memset(sbi_pt, 0, RISCV_PGSIZE); - pte_t* middle_pt = (void*)sbi_pt + RISCV_PGSIZE; -#ifndef __riscv64 - size_t num_middle_pts = 1; - pte_t* root_pt = middle_pt; - memset(root_pt, 0, RISCV_PGSIZE); -#else - size_t num_middle_pts = (-current.first_user_vaddr - 1) / MEGAPAGE_SIZE + 1; - pte_t* root_pt = (void*)middle_pt + num_middle_pts * RISCV_PGSIZE; - memset(middle_pt, 0, (num_middle_pts + 1) * RISCV_PGSIZE); - for (size_t i = 0; i < num_middle_pts; i++) - root_pt[(1<> RISCV_PGSHIFT) + i); -#endif - - for (uintptr_t vaddr = current.first_user_vaddr, paddr = vaddr + current.bias, end = current.first_vaddr_after_user; - paddr < mem_size; vaddr += SUPERPAGE_SIZE, paddr += SUPERPAGE_SIZE) { - int l2_shift = RISCV_PGLEVEL_BITS + RISCV_PGSHIFT; - size_t l2_idx = (current.first_user_vaddr >> l2_shift) & ((1 << RISCV_PGLEVEL_BITS)-1); - l2_idx += ((vaddr - current.first_user_vaddr) >> l2_shift); - middle_pt[l2_idx] = pte_create(paddr >> RISCV_PGSHIFT, PROT_READ|PROT_WRITE|PROT_EXEC, 0); - } - current.first_vaddr_after_user += (void*)root_pt + RISCV_PGSIZE - (void*)sbi_pt; - - // map SBI at top of vaddr space - uintptr_t num_sbi_pages = sbi_top_paddr() / RISCV_PGSIZE; - for (uintptr_t i = 0; i < num_sbi_pages; i++) { - uintptr_t idx = (1 << RISCV_PGLEVEL_BITS) - num_sbi_pages + i; - sbi_pt[idx] = pte_create(i, PROT_READ|PROT_EXEC, 0); - } - pte_t* sbi_pte = middle_pt + ((num_middle_pts << RISCV_PGLEVEL_BITS)-1); - kassert(!*sbi_pte); - *sbi_pte = ptd_create((uintptr_t)sbi_pt >> RISCV_PGSHIFT); - - // disable our allocator - kassert(next_free_page == 0); - free_pages = 0; - - mb(); - root_page_table = root_pt; - write_csr(sptbr, (uintptr_t)root_pt >> RISCV_PGSHIFT); -} - -uintptr_t pk_vm_init() -{ - // keep user addresses positive - current.mmap_max = MIN(current.mmap_max, (uintptr_t)INTPTR_MAX + 1); - - __map_kernel_range(0, 0, current.first_free_paddr, PROT_READ|PROT_WRITE|PROT_EXEC); - __map_kernel_range(first_free_page, first_free_page, free_pages * RISCV_PGSIZE, PROT_READ|PROT_WRITE); - - size_t stack_size = RISCV_PGSIZE * CLAMP(mem_size/(RISCV_PGSIZE*32), 1, 256); - current.stack_bottom = __do_mmap(current.mmap_max - stack_size, stack_size, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, 0, 0); - current.stack_top = current.stack_bottom + stack_size; - kassert(current.stack_bottom != (uintptr_t)-1); - - uintptr_t kernel_stack_top = __page_alloc() + RISCV_PGSIZE; - return kernel_stack_top; -} diff --git a/pk/vm.h b/pk/vm.h deleted file mode 100644 index c05a1ce..0000000 --- a/pk/vm.h +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef _VM_H -#define _VM_H - -#include "syscall.h" -#include "file.h" -#include -#include -#include - -#define SUPERPAGE_SIZE ((uintptr_t)(RISCV_PGSIZE << RISCV_PGLEVEL_BITS)) -#ifdef __riscv64 -# define VM_CHOICE VM_SV39 -# define VA_BITS 39 -# define MEGAPAGE_SIZE (SUPERPAGE_SIZE << RISCV_PGLEVEL_BITS) -#else -# define VM_CHOICE VM_SV32 -# define VA_BITS 32 -#endif - -#define PROT_NONE 0 -#define PROT_READ 1 -#define PROT_WRITE 2 -#define PROT_EXEC 4 - -#define MAP_PRIVATE 0x2 -#define MAP_FIXED 0x10 -#define MAP_ANONYMOUS 0x20 -#define MAP_POPULATE 0x8000 -#define MREMAP_FIXED 0x2 - -void vm_init(); -void supervisor_vm_init(); -uintptr_t pk_vm_init(); -int handle_page_fault(uintptr_t vaddr, int prot); -void populate_mapping(const void* start, size_t size, int prot); -void __map_kernel_range(uintptr_t va, uintptr_t pa, size_t len, int prot); -int __valid_user_range(uintptr_t vaddr, size_t len); -uintptr_t __do_mmap(uintptr_t addr, size_t length, int prot, int flags, file_t* file, off_t offset); -uintptr_t do_mmap(uintptr_t addr, size_t length, int prot, int flags, int fd, off_t offset); -int do_munmap(uintptr_t addr, size_t length); -uintptr_t do_mremap(uintptr_t addr, size_t old_size, size_t new_size, int flags); -uintptr_t do_mprotect(uintptr_t addr, size_t length, int prot); -uintptr_t do_brk(uintptr_t addr); - -typedef uintptr_t pte_t; -extern pte_t* root_page_table; - -static inline void flush_tlb() -{ - asm volatile("sfence.vm"); -} - -#endif diff --git a/util/snprintf.c b/util/snprintf.c new file mode 100644 index 0000000..1544a6c --- /dev/null +++ b/util/snprintf.c @@ -0,0 +1,98 @@ +// See LICENSE for license details. + +#include +#include +#include +#include + +int vsnprintf(char* out, size_t n, const char* s, va_list vl) +{ + bool format = false; + bool longarg = false; + size_t pos = 0; + for( ; *s; s++) + { + if(format) + { + switch(*s) + { + case 'l': + longarg = true; + break; + case 'p': + longarg = true; + if (++pos < n) out[pos-1] = '0'; + if (++pos < n) out[pos-1] = 'x'; + case 'x': + { + long num = longarg ? va_arg(vl, long) : va_arg(vl, int); + for(int i = 2*(longarg ? sizeof(long) : sizeof(int))-1; i >= 0; i--) { + int d = (num >> (4*i)) & 0xF; + if (++pos < n) out[pos-1] = (d < 10 ? '0'+d : 'a'+d-10); + } + longarg = false; + format = false; + break; + } + case 'd': + { + long num = longarg ? va_arg(vl, long) : va_arg(vl, int); + if (num < 0) { + num = -num; + if (++pos < n) out[pos-1] = '-'; + } + long digits = 1; + for (long nn = num; nn /= 10; digits++) + ; + for (int i = digits-1; i >= 0; i--) { + if (pos + i + 1 < n) out[pos + i] = '0' + (num % 10); + num /= 10; + } + pos += digits; + longarg = false; + format = false; + break; + } + case 's': + { + const char* s2 = va_arg(vl, const char*); + while (*s2) { + if (++pos < n) + out[pos-1] = *s2; + s2++; + } + longarg = false; + format = false; + break; + } + case 'c': + { + if (++pos < n) out[pos-1] = (char)va_arg(vl,int); + longarg = false; + format = false; + break; + } + default: + break; + } + } + else if(*s == '%') + format = true; + else + if (++pos < n) out[pos-1] = *s; + } + if (pos < n) + out[pos] = 0; + else if (n) + out[n-1] = 0; + return pos; +} + +int snprintf(char* out, size_t n, const char* s, ...) +{ + va_list vl; + va_start(vl, s); + int res = vsnprintf(out, n, s, vl); + va_end(vl); + return res; +} diff --git a/util/string.c b/util/string.c new file mode 100644 index 0000000..e896379 --- /dev/null +++ b/util/string.c @@ -0,0 +1,87 @@ +#include +#include +#include + +void* memcpy(void* dest, const void* src, size_t len) +{ + if ((((uintptr_t)dest | (uintptr_t)src | len) & (sizeof(uintptr_t)-1)) == 0) { + const uintptr_t* s = src; + uintptr_t *d = dest; + while (d < (uintptr_t*)(dest + len)) + *d++ = *s++; + } else { + const char* s = src; + char *d = dest; + while (d < (char*)(dest + len)) + *d++ = *s++; + } + return dest; +} + +void* memset(void* dest, int byte, size_t len) +{ + if ((((uintptr_t)dest | len) & (sizeof(uintptr_t)-1)) == 0) { + uintptr_t word = byte & 0xFF; + word |= word << 8; + word |= word << 16; + word |= word << 16 << 16; + + uintptr_t *d = dest; + while (d < (uintptr_t*)(dest + len)) + *d++ = word; + } else { + char *d = dest; + while (d < (char*)(dest + len)) + *d++ = byte; + } + return dest; +} + +size_t strlen(const char *s) +{ + const char *p = s; + while (*p) + p++; + return p - s; +} + +int strcmp(const char* s1, const char* s2) +{ + unsigned char c1, c2; + + do { + c1 = *s1++; + c2 = *s2++; + } while (c1 != 0 && c1 == c2); + + return c1 - c2; +} + +char* strcpy(char* dest, const char* src) +{ + char* d = dest; + while ((*d++ = *src++)) + ; + return dest; +} + +long atol(const char* str) +{ + long res = 0; + int sign = 0; + + while (*str == ' ') + str++; + + if (*str == '-' || *str == '+') { + sign = *str == '-'; + str++; + } + + while (*str) { + res *= 10; + res += *str++ - '0'; + } + + return sign ? -res : res; +} diff --git a/util/util.ac b/util/util.ac new file mode 100644 index 0000000..e69de29 diff --git a/util/util.mk.in b/util/util.mk.in new file mode 100644 index 0000000..abbdbd1 --- /dev/null +++ b/util/util.mk.in @@ -0,0 +1,9 @@ +util_subproject_deps = \ + +util_hdrs = \ + +util_c_srcs = \ + snprintf.c \ + string.c \ + +util_asm_srcs = \ -- cgit v1.1