diff options
| author | Colin Schmidt <colins@eecs.berkeley.edu> | 2019-08-26 10:40:08 -0700 |
|---|---|---|
| committer | Colin Schmidt <colins@eecs.berkeley.edu> | 2019-08-26 10:40:08 -0700 |
| commit | 191fc4bfab264bc7aa28c1e07f938e534b474d35 (patch) | |
| tree | f7e9d3928f5425e92b96498ce2c654d52fba12f6 | |
| parent | c53de08b9ba719f3e7b02fc1a029d194a190da48 (diff) | |
| parent | 3d921d3c76db3af7b9ae0b5df0f0790f26222246 (diff) | |
| download | riscv-pk-rocc-enable.zip riscv-pk-rocc-enable.tar.gz riscv-pk-rocc-enable.tar.bz2 | |
Merge commit '3d921d3c76db3af7b9ae0b5df0f0790f26222246' into rocc-enablerocc-enable
59 files changed, 2276 insertions, 1078 deletions
@@ -7,4 +7,5 @@ build/ autom4te.cache/ - +*.swp +*~ diff --git a/Makefile.in b/Makefile.in index b2aca28..0268629 100644 --- a/Makefile.in +++ b/Makefile.in @@ -35,7 +35,6 @@ default : all project_name := @PACKAGE_TARNAME@ src_dir := @srcdir@ scripts_dir := $(src_dir)/scripts -bbl_payload := @BBL_PAYLOAD@ # If the version information is not in the configure script, then we # assume that we are in a working directory. We use the vcs-version.sh @@ -84,7 +83,10 @@ VPATH := $(addprefix $(src_dir)/, $(sprojs_enabled)) # - CXXFLAGS : flags for C++ compiler (eg. -Wall,-g,-O3) CC := @CC@ -CFLAGS := @CFLAGS@ $(CFLAGS) -DBBL_PAYLOAD=\"$(bbl_payload)\" +READELF := @READELF@ +OBJCOPY := @OBJCOPY@ +CFLAGS := @CFLAGS@ $(CFLAGS) -DBBL_PAYLOAD=\"bbl_payload\" -DBBL_LOGO_FILE=\"bbl_logo_file\" +BBL_PAYLOAD := @BBL_PAYLOAD@ COMPILE := $(CC) -MMD -MP $(CFLAGS) \ $(sprojs_include) # Linker @@ -19,6 +19,8 @@ Build Steps We assume that the RISCV environment variable is set to the RISC-V tools install path, and that the riscv-gnu-toolchain package is installed. +Please note that building the binaries directly inside the source +directory is not supported; you need to use a separate build directory. $ mkdir build $ cd build @@ -1,8 +1,12 @@ -AC_ARG_ENABLE([logo], AS_HELP_STRING([--disable-logo], [Disable boot logo])) -AS_IF([test "x$enable_logo" != "xno"], [ +AC_ARG_ENABLE([logo], AS_HELP_STRING([--enable-logo], [Enable boot logo])) +AS_IF([test "x$enable_logo" == "xyes"], [ AC_DEFINE([PK_ENABLE_LOGO],,[Define if the RISC-V logo is to be displayed]) ]) AC_ARG_WITH([payload], AS_HELP_STRING([--with-payload], [Set ELF payload for bbl]), [AC_SUBST([BBL_PAYLOAD], $with_payload, [Kernel payload for bbl])], [AC_SUBST([BBL_PAYLOAD], [dummy_payload], [Kernel payload for bbl])]) + +AC_ARG_WITH([logo], AS_HELP_STRING([--with-logo], [Specify a better logo]), + [AC_SUBST([BBL_LOGO_FILE], $with_logo, [Logo for bbl])], + [AC_SUBST([BBL_LOGO_FILE], [riscv_logo.txt], [Logo for bbl])]) @@ -4,73 +4,64 @@ #include "vm.h" #include "bits.h" #include "config.h" +#include "fdt.h" #include <string.h> -static kernel_elf_info info; -static volatile int elf_loaded; +static const void* entry_point; +long disabled_hart_mask; -static void supervisor_vm_init() +static uintptr_t dtb_output() { - uintptr_t highest_va = DRAM_BASE - first_free_paddr; - mem_size = MIN(mem_size, highest_va - info.first_user_vaddr) & -MEGAPAGE_SIZE; + extern char _payload_end; + uintptr_t end = (uintptr_t) &_payload_end; + return (end + MEGAPAGE_SIZE - 1) / MEGAPAGE_SIZE * MEGAPAGE_SIZE; +} - pte_t* sbi_pt = (pte_t*)(info.first_vaddr_after_user + info.load_offset); - memset(sbi_pt, 0, RISCV_PGSIZE); - pte_t* middle_pt = (void*)sbi_pt + RISCV_PGSIZE; -#if __riscv_xlen == 32 - size_t num_middle_pts = 1; - pte_t* root_pt = middle_pt; - memset(root_pt, 0, RISCV_PGSIZE); -#else - size_t num_middle_pts = (-info.first_user_vaddr - 1) / GIGAPAGE_SIZE + 1; - pte_t* root_pt = (void*)middle_pt + num_middle_pts * RISCV_PGSIZE; - memset(middle_pt, 0, (num_middle_pts + 1) * RISCV_PGSIZE); - for (size_t i = 0; i < num_middle_pts; i++) - root_pt[(1<<RISCV_PGLEVEL_BITS)-num_middle_pts+i] = ptd_create(((uintptr_t)middle_pt >> RISCV_PGSHIFT) + i); -#endif +static void filter_dtb(uintptr_t source) +{ + uintptr_t dest = dtb_output(); + uint32_t size = fdt_size(source); + memcpy((void*)dest, (void*)source, size); - for (uintptr_t vaddr = info.first_user_vaddr, paddr = vaddr + info.load_offset, end = info.first_vaddr_after_user; - paddr < DRAM_BASE + mem_size; vaddr += MEGAPAGE_SIZE, paddr += MEGAPAGE_SIZE) { - int l2_shift = RISCV_PGLEVEL_BITS + RISCV_PGSHIFT; - size_t l2_idx = (info.first_user_vaddr >> l2_shift) & ((1 << RISCV_PGLEVEL_BITS)-1); - l2_idx += ((vaddr - info.first_user_vaddr) >> l2_shift); - middle_pt[l2_idx] = pte_create(paddr >> RISCV_PGSHIFT, PTE_G | PTE_R | PTE_W | PTE_X); - } + // Remove information from the chained FDT + filter_harts(dest, &disabled_hart_mask); + filter_plic(dest); + filter_compat(dest, "riscv,clint0"); + filter_compat(dest, "riscv,debug-013"); +} + +void boot_other_hart(uintptr_t unused __attribute__((unused))) +{ + const void* entry; + do { + entry = entry_point; + mb(); + } while (!entry); - // map SBI at top of vaddr space - extern char _sbi_end; - uintptr_t num_sbi_pages = ((uintptr_t)&_sbi_end - DRAM_BASE - 1) / RISCV_PGSIZE + 1; - assert(num_sbi_pages <= (1 << RISCV_PGLEVEL_BITS)); - for (uintptr_t i = 0; i < num_sbi_pages; i++) { - uintptr_t idx = (1 << RISCV_PGLEVEL_BITS) - num_sbi_pages + i; - sbi_pt[idx] = pte_create((DRAM_BASE / RISCV_PGSIZE) + i, PTE_G | PTE_R | PTE_X); + long hartid = read_csr(mhartid); + if ((1 << hartid) & disabled_hart_mask) { + while (1) { + __asm__ volatile("wfi"); +#ifdef __riscv_div + __asm__ volatile("div x0, x0, x0"); +#endif + } } - pte_t* sbi_pte = middle_pt + ((num_middle_pts << RISCV_PGLEVEL_BITS)-1); - assert(!*sbi_pte); - *sbi_pte = ptd_create((uintptr_t)sbi_pt >> RISCV_PGSHIFT); - mb(); - root_page_table = root_pt; - write_csr(sptbr, (uintptr_t)root_pt >> RISCV_PGSHIFT); + enter_supervisor_mode(entry, hartid, dtb_output()); } -void boot_loader() +void boot_loader(uintptr_t dtb) { - extern char _payload_start, _payload_end; - load_kernel_elf(&_payload_start, &_payload_end - &_payload_start, &info); - supervisor_vm_init(); + extern char _payload_start; + filter_dtb(dtb); #ifdef PK_ENABLE_LOGO print_logo(); #endif +#ifdef PK_PRINT_DEVICE_TREE + fdt_print(dtb_output()); +#endif mb(); - elf_loaded = 1; - enter_supervisor_mode((void *)info.entry, 0); -} - -void boot_other_hart() -{ - while (!elf_loaded) - ; - mb(); - enter_supervisor_mode((void *)info.entry, 0); + entry_point = &_payload_start; + boot_other_hart(0); } @@ -8,14 +8,6 @@ #include <stdint.h> #include <stddef.h> -typedef struct { - uintptr_t entry; - uintptr_t first_user_vaddr; - uintptr_t first_vaddr_after_user; - uintptr_t load_offset; -} kernel_elf_info; - -void load_kernel_elf(void* blob, size_t size, kernel_elf_info* info); void print_logo(); #endif // !__ASSEMBLER__ diff --git a/bbl/bbl.lds b/bbl/bbl.lds index 6833e47..2fd0d7c 100644 --- a/bbl/bbl.lds +++ b/bbl/bbl.lds @@ -44,9 +44,10 @@ SECTIONS /* HTIF, isolated onto separate page */ /*--------------------------------------------------------------------*/ . = ALIGN(0x1000); - htif : + .htif : { - *(htif) + PROVIDE( __htif_base = .); + *(.htif) } . = ALIGN(0x1000); diff --git a/bbl/bbl.mk.in b/bbl/bbl.mk.in index 1bb4cd1..2bc96e1 100644 --- a/bbl/bbl.mk.in +++ b/bbl/bbl.mk.in @@ -8,13 +8,21 @@ bbl_hdrs = \ bbl.h \ bbl_c_srcs = \ - kernel_elf.c \ logo.c \ bbl_asm_srcs = \ payload.S \ + raw_logo.S \ -payload.o: $(bbl_payload) +payload.o: bbl_payload + +bbl_payload: $(BBL_PAYLOAD) + if $(READELF) -h $< 2> /dev/null > /dev/null; then $(OBJCOPY) -O binary $< $@; else cp $< $@; fi + +raw_logo.o: bbl_logo_file + +bbl_logo_file: @BBL_LOGO_FILE@ + cat $^ | sed 's/$$/\r/' > $@ bbl_test_srcs = diff --git a/bbl/kernel_elf.c b/bbl/kernel_elf.c deleted file mode 100644 index e22c35c..0000000 --- a/bbl/kernel_elf.c +++ /dev/null @@ -1,54 +0,0 @@ -// See LICENSE for license details. - -#include "mtrap.h" -#include "bbl.h" -#include "bits.h" -#include "vm.h" -#include <elf.h> -#include <string.h> - -void load_kernel_elf(void* blob, size_t size, kernel_elf_info* info) -{ - Elf_Ehdr* eh = blob; - if (sizeof(*eh) > size || - !(eh->e_ident[0] == '\177' && eh->e_ident[1] == 'E' && - eh->e_ident[2] == 'L' && eh->e_ident[3] == 'F')) - goto fail; - - if (IS_ELF64(*eh) != (sizeof(uintptr_t) == 8)) - goto fail; - - uintptr_t min_vaddr = -1, max_vaddr = 0; - size_t phdr_size = eh->e_phnum * sizeof(Elf_Ehdr); - Elf_Phdr* ph = blob + eh->e_phoff; - if (eh->e_phoff + phdr_size > size) - goto fail; - first_free_paddr = ROUNDUP(first_free_paddr, MEGAPAGE_SIZE); - for (int i = 0; i < eh->e_phnum; i++) - if (ph[i].p_type == PT_LOAD && ph[i].p_memsz && ph[i].p_vaddr < min_vaddr) - min_vaddr = ph[i].p_vaddr; - min_vaddr = ROUNDDOWN(min_vaddr, MEGAPAGE_SIZE); - uintptr_t bias = first_free_paddr - min_vaddr; - for (int i = eh->e_phnum - 1; i >= 0; i--) { - if(ph[i].p_type == PT_LOAD && ph[i].p_memsz) { - uintptr_t prepad = ph[i].p_vaddr % RISCV_PGSIZE; - uintptr_t vaddr = ph[i].p_vaddr + bias; - if (vaddr + ph[i].p_memsz > max_vaddr) - max_vaddr = vaddr + ph[i].p_memsz; - if (ph[i].p_offset + ph[i].p_filesz > size) - goto fail; - memcpy((void*)vaddr, blob + ph[i].p_offset, ph[i].p_filesz); - memset((void*)vaddr - prepad, 0, prepad); - memset((void*)vaddr + ph[i].p_filesz, 0, ph[i].p_memsz - ph[i].p_filesz); - } - } - - info->entry = eh->e_entry; - info->load_offset = bias; - info->first_user_vaddr = min_vaddr; - info->first_vaddr_after_user = ROUNDUP(max_vaddr - bias, RISCV_PGSIZE); - return; - -fail: - die("failed to load payload"); -} @@ -1,30 +1,7 @@ #include <string.h> #include "mtrap.h" -static const char logo[] = -" vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" -" vvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv\n" -"rrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvv \n" -"rrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvv \n" -"rr vvvvvvvvvvvvvvvvvvvvvv \n" -"rr vvvvvvvvvvvvvvvvvvvvvvvv rr\n" -"rrrr vvvvvvvvvvvvvvvvvvvvvvvvvv rrrr\n" -"rrrrrr vvvvvvvvvvvvvvvvvvvvvv rrrrrr\n" -"rrrrrrrr vvvvvvvvvvvvvvvvvv rrrrrrrr\n" -"rrrrrrrrrr vvvvvvvvvvvvvv rrrrrrrrrr\n" -"rrrrrrrrrrrr vvvvvvvvvv rrrrrrrrrrrr\n" -"rrrrrrrrrrrrrr vvvvvv rrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrr vv rrrrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrrrr\n" -"rrrrrrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrrrrrr\n" -"\n" -" INSTRUCTION SETS WANT TO BE FREE\n"; +extern const char logo[]; void print_logo() { diff --git a/bbl/payload.S b/bbl/payload.S index 7ff1e58..6a175aa 100644 --- a/bbl/payload.S +++ b/bbl/payload.S @@ -1,7 +1,9 @@ -.section ".payload","a",@progbits -.align 3 +#include "encoding.h" -.globl _payload_start, _payload_end + .section ".payload","a",@progbits + .align RISCV_PGSHIFT + RISCV_PGLEVEL_BITS + + .globl _payload_start, _payload_end _payload_start: -.incbin BBL_PAYLOAD + .incbin BBL_PAYLOAD _payload_end: diff --git a/bbl/raw_logo.S b/bbl/raw_logo.S new file mode 100644 index 0000000..bddf8f4 --- /dev/null +++ b/bbl/raw_logo.S @@ -0,0 +1,7 @@ +#include "encoding.h" + + .section .rodata + .globl logo +logo: + .incbin BBL_LOGO_FILE + .byte 0 diff --git a/bbl/riscv_logo.txt b/bbl/riscv_logo.txt new file mode 100644 index 0000000..1f4c9f5 --- /dev/null +++ b/bbl/riscv_logo.txt @@ -0,0 +1,23 @@ + vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + vvvvvvvvvvvvvvvvvvvvvvvvvvvv +rrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvvvv +rrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv +rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv +rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv +rrrrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvvvv +rrrrrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvv +rrrrrrrrrrrrr vvvvvvvvvvvvvvvvvvvvvv +rr vvvvvvvvvvvvvvvvvvvvvv +rr vvvvvvvvvvvvvvvvvvvvvvvv rr +rrrr vvvvvvvvvvvvvvvvvvvvvvvvvv rrrr +rrrrrr vvvvvvvvvvvvvvvvvvvvvv rrrrrr +rrrrrrrr vvvvvvvvvvvvvvvvvv rrrrrrrr +rrrrrrrrrr vvvvvvvvvvvvvv rrrrrrrrrr +rrrrrrrrrrrr vvvvvvvvvv rrrrrrrrrrrr +rrrrrrrrrrrrrr vvvvvv rrrrrrrrrrrrrr +rrrrrrrrrrrrrrrr vv rrrrrrrrrrrrrrrr +rrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrr +rrrrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrrrr +rrrrrrrrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrrrrrrrr + + INSTRUCTION SETS WANT TO BE FREE diff --git a/config.h.in b/config.h.in index bc6369e..4674a2e 100644 --- a/config.h.in +++ b/config.h.in @@ -39,6 +39,9 @@ /* Define if virtual memory support is enabled */ #undef PK_ENABLE_VM +/* Define if the DTS is to be displayed */ +#undef PK_PRINT_DEVICE_TREE + /* Define if subproject MCPPBS_SPROJ_NORM is enabled */ #undef SOFTFLOAT_ENABLED @@ -590,6 +590,7 @@ ac_subst_vars='LTLIBOBJS LIBOBJS subprojects_enabled subprojects +BBL_LOGO_FILE BBL_PAYLOAD install_subdir RISCV @@ -603,6 +604,8 @@ INSTALL_PROGRAM STOW_PREFIX STOW_ROOT enable_stow +OBJCOPY +READELF RANLIB AR ac_ct_CXX @@ -642,6 +645,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -666,10 +670,12 @@ ac_user_opts=' enable_option_checking enable_stow enable_32bit +enable_print_device_tree enable_optional_subprojects enable_vm enable_logo with_payload +with_logo enable_fp_emulation ' ac_precious_vars='build_alias @@ -725,6 +731,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -977,6 +984,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1114,7 +1130,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1267,6 +1283,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1302,16 +1319,19 @@ Optional Features: --enable-FEATURE[=ARG] include FEATURE [ARG=yes] --enable-stow Enable stow-based install --enable-32bit Build a 32-bit pk + --enable-print-device-tree + Print DTS when booting --enable-optional-subprojects Enable all optional subprojects --disable-vm Disable virtual memory - --disable-logo Disable boot logo + --enable-logo Enable boot logo --disable-fp-emulation Disable floating-point emulation Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) --with-payload Set ELF payload for bbl + --with-logo Specify a better logo Some influential environment variables: CC C compiler command @@ -3250,6 +3270,190 @@ else RANLIB="$ac_cv_prog_RANLIB" fi +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}readelf", so it can be a program name with args. +set dummy ${ac_tool_prefix}readelf; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_READELF+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$READELF"; then + ac_cv_prog_READELF="$READELF" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_READELF="${ac_tool_prefix}readelf" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +READELF=$ac_cv_prog_READELF +if test -n "$READELF"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $READELF" >&5 +$as_echo "$READELF" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_READELF"; then + ac_ct_READELF=$READELF + # Extract the first word of "readelf", so it can be a program name with args. +set dummy readelf; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_READELF+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_READELF"; then + ac_cv_prog_ac_ct_READELF="$ac_ct_READELF" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_READELF="readelf" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_READELF=$ac_cv_prog_ac_ct_READELF +if test -n "$ac_ct_READELF"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_READELF" >&5 +$as_echo "$ac_ct_READELF" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_READELF" = x; then + READELF="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + READELF=$ac_ct_READELF + fi +else + READELF="$ac_cv_prog_READELF" +fi + +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}objcopy", so it can be a program name with args. +set dummy ${ac_tool_prefix}objcopy; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_OBJCOPY+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$OBJCOPY"; then + ac_cv_prog_OBJCOPY="$OBJCOPY" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_OBJCOPY="${ac_tool_prefix}objcopy" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +OBJCOPY=$ac_cv_prog_OBJCOPY +if test -n "$OBJCOPY"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OBJCOPY" >&5 +$as_echo "$OBJCOPY" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_OBJCOPY"; then + ac_ct_OBJCOPY=$OBJCOPY + # Extract the first word of "objcopy", so it can be a program name with args. +set dummy objcopy; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_ac_ct_OBJCOPY+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$ac_ct_OBJCOPY"; then + ac_cv_prog_ac_ct_OBJCOPY="$ac_ct_OBJCOPY" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_OBJCOPY="objcopy" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_OBJCOPY=$ac_cv_prog_ac_ct_OBJCOPY +if test -n "$ac_ct_OBJCOPY"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_OBJCOPY" >&5 +$as_echo "$ac_ct_OBJCOPY" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_ct_OBJCOPY" = x; then + OBJCOPY="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + OBJCOPY=$ac_ct_OBJCOPY + fi +else + OBJCOPY="$ac_cv_prog_OBJCOPY" +fi + #------------------------------------------------------------------------- # MCPPBS specific program checks @@ -3891,6 +4095,19 @@ case "${BUILD_32BIT}" in ;; esac +# Check whether --enable-print-device-tree was given. +if test "${enable_print_device_tree+set}" = set; then : + enableval=$enable_print_device_tree; +fi + +if test "x$enable_print_device_tree" == "xyes"; then : + + +$as_echo "#define PK_PRINT_DEVICE_TREE /**/" >>confdefs.h + + +fi + LIBS="-lgcc" @@ -4020,7 +4237,7 @@ if test "${enable_logo+set}" = set; then : enableval=$enable_logo; fi -if test "x$enable_logo" != "xno"; then : +if test "x$enable_logo" == "xyes"; then : $as_echo "#define PK_ENABLE_LOGO /**/" >>confdefs.h @@ -4040,6 +4257,17 @@ fi +# Check whether --with-logo was given. +if test "${with_logo+set}" = set; then : + withval=$with_logo; BBL_LOGO_FILE=$with_logo + +else + BBL_LOGO_FILE=riscv_logo.txt + +fi + + + diff --git a/configure.ac b/configure.ac index 1bf0326..4cf9b61 100644 --- a/configure.ac +++ b/configure.ac @@ -50,6 +50,8 @@ AC_PROG_CC AC_PROG_CXX AC_CHECK_TOOL([AR],[ar]) AC_CHECK_TOOL([RANLIB],[ranlib]) +AC_CHECK_TOOL([READELF],[readelf]) +AC_CHECK_TOOL([OBJCOPY],[objcopy]) #------------------------------------------------------------------------- # MCPPBS specific program checks @@ -97,6 +99,11 @@ case "${BUILD_32BIT}" in ;; esac +AC_ARG_ENABLE([print-device-tree], AS_HELP_STRING([--enable-print-device-tree], [Print DTS when booting])) +AS_IF([test "x$enable_print_device_tree" == "xyes"], [ + AC_DEFINE([PK_PRINT_DEVICE_TREE],,[Define if the DTS is to be displayed]) +]) + AC_SUBST(CFLAGS) AC_SUBST(LDFLAGS) AC_SUBST([LIBS], ["-lgcc"]) diff --git a/dummy_payload/dummy_entry.S b/dummy_payload/dummy_entry.S new file mode 100644 index 0000000..92d4652 --- /dev/null +++ b/dummy_payload/dummy_entry.S @@ -0,0 +1,22 @@ +#include "mcall.h" + + .section ".text.init" + .globl _start +_start: + la s0, str +1: + lbu a0, (s0) + beqz a0, 1f + li a7, SBI_CONSOLE_PUTCHAR + ecall + add s0, s0, 1 + j 1b + +1: + li a7, SBI_SHUTDOWN + ecall + + .data +str: + .asciz "This is bbl's dummy_payload. To boot a real kernel, reconfigure\n\ +bbl with the flag --with-payload=PATH, then rebuild bbl.\n" diff --git a/dummy_payload/dummy_payload.c b/dummy_payload/dummy_payload.c index 61ed59a..e69de29 100644 --- a/dummy_payload/dummy_payload.c +++ b/dummy_payload/dummy_payload.c @@ -1,21 +0,0 @@ -#include <stdint.h> -#include "sbi.h" - -asm (".globl _start\n\ - _start: la sp, stack\n\ - j entry\n\ - .pushsection .rodata\n\ - .align 4\n\ - .skip 4096\n\ - stack:\n\ - .popsection"); - -void entry() -{ - const char* message = -"This is bbl's dummy_payload. To boot a real kernel, reconfigure\n\ -bbl with the flag --with-payload=PATH, then rebuild bbl.\n"; - while (*message) - sbi_console_putchar(*message++); - sbi_shutdown(); -} diff --git a/dummy_payload/dummy_payload.lds b/dummy_payload/dummy_payload.lds index ee9410b..0506a8d 100644 --- a/dummy_payload/dummy_payload.lds +++ b/dummy_payload/dummy_payload.lds @@ -1,3 +1,7 @@ +ENTRY(_start) + SECTIONS { . = -0x80000000; + + .text.init : { *(.text.init) } } diff --git a/dummy_payload/dummy_payload.mk.in b/dummy_payload/dummy_payload.mk.in index b1d4ac7..4a133be 100644 --- a/dummy_payload/dummy_payload.mk.in +++ b/dummy_payload/dummy_payload.mk.in @@ -5,7 +5,7 @@ dummy_payload_hdrs = \ dummy_payload_c_srcs = \ dummy_payload_asm_srcs = \ - dummy_sbi.S \ + dummy_entry.S \ dummy_payload_test_srcs = diff --git a/dummy_payload/dummy_sbi.S b/dummy_payload/dummy_sbi.S deleted file mode 120000 index 2978a97..0000000 --- a/dummy_payload/dummy_sbi.S +++ /dev/null @@ -1 +0,0 @@ -../machine/sbi.S
\ No newline at end of file diff --git a/machine/configstring.c b/machine/configstring.c deleted file mode 100644 index 4e72f04..0000000 --- a/machine/configstring.c +++ /dev/null @@ -1,102 +0,0 @@ -#include "configstring.h" -#include "encoding.h" -#include "mtrap.h" -#include "atomic.h" -#include <stdio.h> - -static void query_mem(const char* config_string) -{ - query_result res = query_config_string(config_string, "ram{0{addr"); - assert(res.start); - uintptr_t base = get_uint(res); - assert(base == DRAM_BASE); - res = query_config_string(config_string, "ram{0{size"); - mem_size = get_uint(res); -} - -static void query_rtc(const char* config_string) -{ - query_result res = query_config_string(config_string, "rtc{addr"); - assert(res.start); - mtime = (void*)(uintptr_t)get_uint(res); -} - -static void query_plic(const char* config_string) -{ - query_result res = query_config_string(config_string, "plic{priority"); - if (!res.start) - return; - plic_priorities = (uint32_t*)(uintptr_t)get_uint(res); - - res = query_config_string(config_string, "plic{ndevs"); - if (!res.start) - return; - plic_ndevs = get_uint(res); -} - -static void query_hart_plic(const char* config_string, hls_t* hls, int core, int hart) -{ - char buf[48]; - snprintf(buf, sizeof buf, "core{%d{%d{plic{m{ie", core, hart); - query_result res = query_config_string(config_string, buf); - if (res.start) - hls->plic_m_ie = (void*)(uintptr_t)get_uint(res); - - snprintf(buf, sizeof buf, "core{%d{%d{plic{m{thresh", core, hart); - res = query_config_string(config_string, buf); - if (res.start) - hls->plic_m_thresh = (void*)(uintptr_t)get_uint(res); - - snprintf(buf, sizeof buf, "core{%d{%d{plic{s{ie", core, hart); - res = query_config_string(config_string, buf); - if (res.start) - hls->plic_s_ie = (void*)(uintptr_t)get_uint(res); - - snprintf(buf, sizeof buf, "core{%d{%d{plic{s{thresh", core, hart); - res = query_config_string(config_string, buf); - if (res.start) - hls->plic_s_thresh = (void*)(uintptr_t)get_uint(res); -} - -static void query_harts(const char* config_string) -{ - for (int core = 0, hart; ; core++) { - for (hart = 0; ; hart++) { - char buf[40]; - snprintf(buf, sizeof buf, "core{%d{%d{ipi", core, hart); - query_result res = query_config_string(config_string, buf); - if (!res.start) - break; - hls_t* hls = hls_init(num_harts); - hls->ipi = (void*)(uintptr_t)get_uint(res); - - query_hart_plic(config_string, hls, core, hart); - - snprintf(buf, sizeof buf, "core{%d{%d{timecmp", core, hart); - res = query_config_string(config_string, buf); - assert(res.start); - hls->timecmp = (void*)(uintptr_t)get_uint(res); - - mb(); - - // wake up the hart - *hls->ipi = 1; - - num_harts++; - } - if (!hart) - break; - } - assert(num_harts); - assert(num_harts <= MAX_HARTS); -} - -void parse_config_string() -{ - uint32_t addr = *(uint32_t*)CONFIG_STRING_ADDR; - const char* s = (const char*)(uintptr_t)addr; - query_mem(s); - query_plic(s); - query_rtc(s); - query_harts(s); -} diff --git a/machine/configstring.h b/machine/configstring.h deleted file mode 100644 index 39538c9..0000000 --- a/machine/configstring.h +++ /dev/null @@ -1,166 +0,0 @@ -#ifndef RISCV_CONFIG_STRING_H -#define RISCV_CONFIG_STRING_H - -#include <stddef.h> -#include <stdint.h> - -static const char* skip_whitespace(const char* str) -{ - while (*str && *str <= ' ') - str++; - return str; -} - -static const char* skip_string(const char* str) -{ - while (*str && *str++ != '"') - ; - return str; -} - -static int is_hex(char ch) -{ - return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); -} - -static int parse_hex(char ch) -{ - return (ch >= '0' && ch <= '9') ? ch - '0' : - (ch >= 'a' && ch <= 'f') ? ch - 'a' + 10 : - ch - 'A' + 10; -} - -static const char* skip_key(const char* str) -{ - while (*str >= 35 && *str <= 122 && *str != ';') - str++; - return str; -} - -typedef struct { - const char* start; - const char* end; -} query_result; - -static query_result query_config_string(const char* str, const char* k) -{ - size_t ksize = 0; - while (k[ksize] && k[ksize] != '{') - ksize++; - int last = !k[ksize]; - - query_result res = {0, 0}; - while (1) { - const char* key_start = str = skip_whitespace(str); - const char* key_end = str = skip_key(str); - int match = (size_t)(key_end - key_start) == ksize; - if (match) - for (size_t i = 0; i < ksize; i++) - if (key_start[i] != k[i]) - match = 0; - const char* value_start = str = skip_whitespace(str); - while (*str != ';') { - if (!*str) { - return res; - } else if (*str == '"') { - str = skip_string(str+1); - } else if (*str == '{') { - const char* search_key = match && !last ? k + ksize + 1 : ""; - query_result inner_res = query_config_string(str + 1, search_key); - if (inner_res.start) - return inner_res; - str = inner_res.end + 1; - } else { - str = skip_key(str); - } - str = skip_whitespace(str); - } - res.end = str; - if (match && last) { - res.start = value_start; - return res; - } - str = skip_whitespace(str+1); - if (*str == '}') { - res.end = str; - return res; - } - } -} - -static void parse_string(query_result r, char* buf) -{ - if (r.start < r.end) { - if (*r.start == '"') { - for (const char* p = r.start + 1; p < r.end && *p != '"'; p++) { - char ch = p[0]; - if (ch == '\\' && p[1] == 'x' && is_hex(p[2])) { - ch = parse_hex(p[2]); - if (is_hex(p[3])) { - ch = (ch << 4) + parse_hex(p[3]); - p++; - } - p += 2; - } - *buf++ = ch; - } - } else { - for (const char* p = r.start; p < r.end && *p > ' '; p++) - *buf++ = *p; - } - } - *buf = 0; -} - -#define get_string(name, search_res) \ - char name[(search_res).end - (search_res).start + 1]; \ - parse_string(search_res, name) - -static uint64_t __get_uint_hex(const char* s) -{ - uint64_t res = 0; - while (*s) { - if (is_hex(*s)) - res = (res << 4) + parse_hex(*s); - else if (*s != '_') - break; - s++; - } - return res; -} - -static uint64_t __get_uint_dec(const char* s) -{ - uint64_t res = 0; - while (*s) { - if (*s >= '0' && *s <= '9') - res = res * 10 + (*s - '0'); - else - break; - s++; - } - return res; -} - -static uint64_t __get_uint(const char* s) -{ - if (s[0] == '0' && s[1] == 'x') - return __get_uint_hex(s+2); - return __get_uint_dec(s); -} - -static inline uint64_t get_uint(query_result res) -{ - get_string(s, res); - return __get_uint(s); -} - -static inline int64_t get_sint(query_result res) -{ - get_string(s, res); - if (s[0] == '-') - return -__get_uint(s+1); - return __get_uint(s); -} - -#endif diff --git a/machine/disabled_hart_mask.h b/machine/disabled_hart_mask.h new file mode 100644 index 0000000..2a3a73c --- /dev/null +++ b/machine/disabled_hart_mask.h @@ -0,0 +1,4 @@ +#ifndef DISABLED_HART_MASK_H +#define DISABLED_HART_MASK_H +extern long disabled_hart_mask; +#endif diff --git a/machine/emulation.c b/machine/emulation.c index 882daa1..b86fa36 100644 --- a/machine/emulation.c +++ b/machine/emulation.c @@ -5,6 +5,67 @@ #include "mtrap.h" #include <limits.h> +static DECLARE_EMULATION_FUNC(emulate_rvc) +{ +#ifdef __riscv_compressed + // the only emulable RVC instructions are FP loads and stores. +# if !defined(__riscv_flen) && defined(PK_ENABLE_FP_EMULATION) + write_csr(mepc, mepc + 2); + + // if FPU is disabled, punt back to the OS + if (unlikely((mstatus & MSTATUS_FS) == 0)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + + if ((insn & MASK_C_FLD) == MATCH_C_FLD) { + uintptr_t addr = GET_RS1S(insn, regs) + RVC_LD_IMM(insn); + if (unlikely(addr % sizeof(uintptr_t))) + return misaligned_load_trap(regs, mcause, mepc); + SET_F64_RD(RVC_RS2S(insn) << SH_RD, regs, load_uint64_t((void *)addr, mepc)); + } else if ((insn & MASK_C_FLDSP) == MATCH_C_FLDSP) { + uintptr_t addr = GET_SP(regs) + RVC_LDSP_IMM(insn); + if (unlikely(addr % sizeof(uintptr_t))) + return misaligned_load_trap(regs, mcause, mepc); + SET_F64_RD(insn, regs, load_uint64_t((void *)addr, mepc)); + } else if ((insn & MASK_C_FSD) == MATCH_C_FSD) { + uintptr_t addr = GET_RS1S(insn, regs) + RVC_LD_IMM(insn); + if (unlikely(addr % sizeof(uintptr_t))) + return misaligned_store_trap(regs, mcause, mepc); + store_uint64_t((void *)addr, GET_F64_RS2(RVC_RS2S(insn) << SH_RS2, regs), mepc); + } else if ((insn & MASK_C_FSDSP) == MATCH_C_FSDSP) { + uintptr_t addr = GET_SP(regs) + RVC_SDSP_IMM(insn); + if (unlikely(addr % sizeof(uintptr_t))) + return misaligned_store_trap(regs, mcause, mepc); + store_uint64_t((void *)addr, GET_F64_RS2(RVC_RS2(insn) << SH_RS2, regs), mepc); + } else +# if __riscv_xlen == 32 + if ((insn & MASK_C_FLW) == MATCH_C_FLW) { + uintptr_t addr = GET_RS1S(insn, regs) + RVC_LW_IMM(insn); + if (unlikely(addr % 4)) + return misaligned_load_trap(regs, mcause, mepc); + SET_F32_RD(RVC_RS2S(insn) << SH_RD, regs, load_int32_t((void *)addr, mepc)); + } else if ((insn & MASK_C_FLWSP) == MATCH_C_FLWSP) { + uintptr_t addr = GET_SP(regs) + RVC_LWSP_IMM(insn); + if (unlikely(addr % 4)) + return misaligned_load_trap(regs, mcause, mepc); + SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc)); + } else if ((insn & MASK_C_FSW) == MATCH_C_FSW) { + uintptr_t addr = GET_RS1S(insn, regs) + RVC_LW_IMM(insn); + if (unlikely(addr % 4)) + return misaligned_store_trap(regs, mcause, mepc); + store_uint32_t((void *)addr, GET_F32_RS2(RVC_RS2S(insn) << SH_RS2, regs), mepc); + } else if ((insn & MASK_C_FSWSP) == MATCH_C_FSWSP) { + uintptr_t addr = GET_SP(regs) + RVC_SWSP_IMM(insn); + if (unlikely(addr % 4)) + return misaligned_store_trap(regs, mcause, mepc); + store_uint32_t((void *)addr, GET_F32_RS2(RVC_RS2(insn) << SH_RS2, regs), mepc); + } else +# endif +# endif +#endif + + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); +} + void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) { asm (".pushsection .rodata\n" @@ -67,11 +128,15 @@ void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) " .word truly_illegal_insn\n" " .popsection"); - uintptr_t mstatus; - insn_t insn = get_insn(mepc, &mstatus); + uintptr_t mstatus = read_csr(mstatus); + insn_t insn = read_csr(mbadaddr); - if (unlikely((insn & 3) != 3)) - return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); + if (unlikely((insn & 3) != 3)) { + if (insn == 0) + insn = get_insn(mepc, &mstatus); + if ((insn & 3) != 3) + return emulate_rvc(regs, mcause, mepc, mstatus, insn); + } write_csr(mepc, mepc + 4); @@ -81,16 +146,17 @@ void illegal_insn_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) f(regs, mcause, mepc, mstatus, insn); } -void __attribute__((noinline)) truly_illegal_insn(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn) +__attribute__((noinline)) +DECLARE_EMULATION_FUNC(truly_illegal_insn) { - redirect_trap(mepc, mstatus); + return redirect_trap(mepc, mstatus, insn); } static inline int emulate_read_csr(int num, uintptr_t mstatus, uintptr_t* result) { - uintptr_t counteren = - EXTRACT_FIELD(mstatus, MSTATUS_MPP) == PRV_U ? read_csr(mucounteren) : - read_csr(mscounteren); + uintptr_t counteren = -1; + if (EXTRACT_FIELD(mstatus, MSTATUS_MPP) == PRV_U) + counteren = read_csr(scounteren); switch (num) { diff --git a/machine/emulation.h b/machine/emulation.h index b8712b5..f75173d 100644 --- a/machine/emulation.h +++ b/machine/emulation.h @@ -5,22 +5,45 @@ #include "bits.h" #include <stdint.h> -typedef uint32_t insn_t; +typedef uintptr_t insn_t; typedef void (*emulation_func)(uintptr_t*, uintptr_t, uintptr_t, uintptr_t, insn_t); #define DECLARE_EMULATION_FUNC(name) void name(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc, uintptr_t mstatus, insn_t insn) void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); -void redirect_trap(uintptr_t epc, uintptr_t mstatus) __attribute__((noreturn)); -DECLARE_EMULATION_FUNC(truly_illegal_insn) __attribute__((noreturn)); +void redirect_trap(uintptr_t epc, uintptr_t mstatus, uintptr_t badaddr); +DECLARE_EMULATION_FUNC(truly_illegal_insn); +DECLARE_EMULATION_FUNC(emulate_rvc_0); +DECLARE_EMULATION_FUNC(emulate_rvc_2); +#define SH_RD 7 +#define SH_RS1 15 +#define SH_RS2 20 +#define SH_RS2C 2 + +#define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) +#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | (RV_X(x, 10, 3) << 3) | (RV_X(x, 5, 1) << 6)) +#define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | (RV_X(x, 5, 2) << 6)) +#define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | (RV_X(x, 12, 1) << 5) | (RV_X(x, 2, 2) << 6)) +#define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | (RV_X(x, 12, 1) << 5) | (RV_X(x, 2, 3) << 6)) +#define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | (RV_X(x, 7, 2) << 6)) +#define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | (RV_X(x, 7, 3) << 6)) +#define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) +#define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) +#define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) + +#define SHIFT_RIGHT(x, y) ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) #define GET_REG(insn, pos, regs) ({ \ int mask = (1 << (5+LOG_REGBYTES)) - (1 << LOG_REGBYTES); \ - (uintptr_t*)((uintptr_t)regs + (((insn) >> ((pos) - LOG_REGBYTES)) & mask)); \ + (uintptr_t*)((uintptr_t)regs + (SHIFT_RIGHT(insn, (pos) - LOG_REGBYTES) & (mask))); \ }) -#define GET_RS1(insn, regs) (*GET_REG(insn, 15, regs)) -#define GET_RS2(insn, regs) (*GET_REG(insn, 20, regs)) -#define SET_RD(insn, regs, val) (*GET_REG(insn, 7, regs) = (val)) +#define GET_RS1(insn, regs) (*GET_REG(insn, SH_RS1, regs)) +#define GET_RS2(insn, regs) (*GET_REG(insn, SH_RS2, regs)) +#define GET_RS1S(insn, regs) (*GET_REG(RVC_RS1S(insn), 0, regs)) +#define GET_RS2S(insn, regs) (*GET_REG(RVC_RS2S(insn), 0, regs)) +#define GET_RS2C(insn, regs) (*GET_REG(insn, SH_RS2C, regs)) +#define GET_SP(regs) (*GET_REG(2, 0, regs)) +#define SET_RD(insn, regs, val) (*GET_REG(insn, SH_RD, regs) = (val)) #define IMM_I(insn) ((int32_t)(insn) >> 20) #define IMM_S(insn) (((int32_t)(insn) >> 25 << 5) | (int32_t)(((insn) >> 7) & 0x1f)) #define MASK_FUNCT3 0x7000 diff --git a/machine/encoding.h b/machine/encoding.h index 792d87b..8dbfca4 100644 --- a/machine/encoding.h +++ b/machine/encoding.h @@ -17,10 +17,14 @@ #define MSTATUS_FS 0x00006000 #define MSTATUS_XS 0x00018000 #define MSTATUS_MPRV 0x00020000 -#define MSTATUS_PUM 0x00040000 +#define MSTATUS_SUM 0x00040000 #define MSTATUS_MXR 0x00080000 -#define MSTATUS_VM 0x1F000000 +#define MSTATUS_TVM 0x00100000 +#define MSTATUS_TW 0x00200000 +#define MSTATUS_TSR 0x00400000 #define MSTATUS32_SD 0x80000000 +#define MSTATUS_UXL 0x0000000300000000 +#define MSTATUS_SXL 0x0000000C00000000 #define MSTATUS64_SD 0x8000000000000000 #define SSTATUS_UIE 0x00000001 @@ -30,8 +34,10 @@ #define SSTATUS_SPP 0x00000100 #define SSTATUS_FS 0x00006000 #define SSTATUS_XS 0x00018000 -#define SSTATUS_PUM 0x00040000 +#define SSTATUS_SUM 0x00040000 +#define SSTATUS_MXR 0x00080000 #define SSTATUS32_SD 0x80000000 +#define SSTATUS_UXL 0x0000000300000000 #define SSTATUS64_SD 0x8000000000000000 #define DCSR_XDEBUGVER (3U<<30) @@ -108,12 +114,30 @@ #define PRV_H 2 #define PRV_M 3 -#define VM_MBARE 0 -#define VM_MBB 1 -#define VM_MBBID 2 -#define VM_SV32 8 -#define VM_SV39 9 -#define VM_SV48 10 +#define SATP32_MODE 0x80000000 +#define SATP32_ASID 0x7FC00000 +#define SATP32_PPN 0x003FFFFF +#define SATP64_MODE 0xF000000000000000 +#define SATP64_ASID 0x0FFFF00000000000 +#define SATP64_PPN 0x00000FFFFFFFFFFF + +#define SATP_MODE_OFF 0 +#define SATP_MODE_SV32 1 +#define SATP_MODE_SV39 8 +#define SATP_MODE_SV48 9 +#define SATP_MODE_SV57 10 +#define SATP_MODE_SV64 11 + +#define PMP_R 0x01 +#define PMP_W 0x02 +#define PMP_X 0x04 +#define PMP_A 0x18 +#define PMP_L 0x80 +#define PMP_SHIFT 2 + +#define PMP_TOR 0x08 +#define PMP_NA4 0x10 +#define PMP_NAPOT 0x18 #define IRQ_S_SOFT 1 #define IRQ_H_SOFT 2 @@ -128,9 +152,8 @@ #define IRQ_HOST 13 #define DEFAULT_RSTVEC 0x00001000 -#define DEFAULT_NMIVEC 0x00001004 -#define DEFAULT_MTVEC 0x00001010 -#define CONFIG_STRING_ADDR 0x0000100C +#define CLINT_BASE 0x02000000 +#define CLINT_SIZE 0x000c0000 #define EXT_IO_BASE 0x40000000 #define DRAM_BASE 0x80000000 @@ -155,10 +178,12 @@ # define MSTATUS_SD MSTATUS64_SD # define SSTATUS_SD SSTATUS64_SD # define RISCV_PGLEVEL_BITS 9 +# define SATP_MODE SATP64_MODE #else # define MSTATUS_SD MSTATUS32_SD # define SSTATUS_SD SSTATUS32_SD # define RISCV_PGLEVEL_BITS 10 +# define SATP_MODE SATP32_MODE #endif #define RISCV_PGSHIFT 12 #define RISCV_PGSIZE (1 << RISCV_PGSHIFT) @@ -172,30 +197,18 @@ __tmp; }) #define write_csr(reg, val) ({ \ - if (__builtin_constant_p(val) && (unsigned long)(val) < 32) \ - asm volatile ("csrw " #reg ", %0" :: "i"(val)); \ - else \ - asm volatile ("csrw " #reg ", %0" :: "r"(val)); }) + asm volatile ("csrw " #reg ", %0" :: "rK"(val)); }) #define swap_csr(reg, val) ({ unsigned long __tmp; \ - if (__builtin_constant_p(val) && (unsigned long)(val) < 32) \ - asm volatile ("csrrw %0, " #reg ", %1" : "=r"(__tmp) : "i"(val)); \ - else \ - asm volatile ("csrrw %0, " #reg ", %1" : "=r"(__tmp) : "r"(val)); \ + asm volatile ("csrrw %0, " #reg ", %1" : "=r"(__tmp) : "rK"(val)); \ __tmp; }) #define set_csr(reg, bit) ({ unsigned long __tmp; \ - if (__builtin_constant_p(bit) && (unsigned long)(bit) < 32) \ - asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \ - else \ - asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \ + asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "rK"(bit)); \ __tmp; }) #define clear_csr(reg, bit) ({ unsigned long __tmp; \ - if (__builtin_constant_p(bit) && (unsigned long)(bit) < 32) \ - asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "i"(bit)); \ - else \ - asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "r"(bit)); \ + asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "rK"(bit)); \ __tmp; }) #define rdtime() read_csr(time) @@ -209,7 +222,7 @@ #endif #endif -/* Automatically generated by parse-opcodes */ +/* Automatically generated by parse-opcodes. */ #ifndef RISCV_ENCODING_H #define RISCV_ENCODING_H #define MATCH_BEQ 0x63 @@ -392,14 +405,12 @@ #define MASK_URET 0xffffffff #define MATCH_SRET 0x10200073 #define MASK_SRET 0xffffffff -#define MATCH_HRET 0x20200073 -#define MASK_HRET 0xffffffff #define MATCH_MRET 0x30200073 #define MASK_MRET 0xffffffff #define MATCH_DRET 0x7b200073 #define MASK_DRET 0xffffffff -#define MATCH_SFENCE_VM 0x10400073 -#define MASK_SFENCE_VM 0xfff07fff +#define MATCH_SFENCE_VMA 0x12000073 +#define MASK_SFENCE_VMA 0xfe007fff #define MATCH_WFI 0x10500073 #define MASK_WFI 0xffffffff #define MATCH_CSRRW 0x1073 @@ -458,6 +469,34 @@ #define MASK_FCVT_D_S 0xfff0007f #define MATCH_FSQRT_D 0x5a000053 #define MASK_FSQRT_D 0xfff0007f +#define MATCH_FADD_Q 0x6000053 +#define MASK_FADD_Q 0xfe00007f +#define MATCH_FSUB_Q 0xe000053 +#define MASK_FSUB_Q 0xfe00007f +#define MATCH_FMUL_Q 0x16000053 +#define MASK_FMUL_Q 0xfe00007f +#define MATCH_FDIV_Q 0x1e000053 +#define MASK_FDIV_Q 0xfe00007f +#define MATCH_FSGNJ_Q 0x26000053 +#define MASK_FSGNJ_Q 0xfe00707f +#define MATCH_FSGNJN_Q 0x26001053 +#define MASK_FSGNJN_Q 0xfe00707f +#define MATCH_FSGNJX_Q 0x26002053 +#define MASK_FSGNJX_Q 0xfe00707f +#define MATCH_FMIN_Q 0x2e000053 +#define MASK_FMIN_Q 0xfe00707f +#define MATCH_FMAX_Q 0x2e001053 +#define MASK_FMAX_Q 0xfe00707f +#define MATCH_FCVT_S_Q 0x40300053 +#define MASK_FCVT_S_Q 0xfff0007f +#define MATCH_FCVT_Q_S 0x46000053 +#define MASK_FCVT_Q_S 0xfff0007f +#define MATCH_FCVT_D_Q 0x42300053 +#define MASK_FCVT_D_Q 0xfff0007f +#define MATCH_FCVT_Q_D 0x46100053 +#define MASK_FCVT_Q_D 0xfff0007f +#define MATCH_FSQRT_Q 0x5e000053 +#define MASK_FSQRT_Q 0xfff0007f #define MATCH_FLE_S 0xa0000053 #define MASK_FLE_S 0xfe00707f #define MATCH_FLT_S 0xa0001053 @@ -470,6 +509,12 @@ #define MASK_FLT_D 0xfe00707f #define MATCH_FEQ_D 0xa2002053 #define MASK_FEQ_D 0xfe00707f +#define MATCH_FLE_Q 0xa6000053 +#define MASK_FLE_Q 0xfe00707f +#define MATCH_FLT_Q 0xa6001053 +#define MASK_FLT_Q 0xfe00707f +#define MATCH_FEQ_Q 0xa6002053 +#define MASK_FEQ_Q 0xfe00707f #define MATCH_FCVT_W_S 0xc0000053 #define MASK_FCVT_W_S 0xfff0007f #define MATCH_FCVT_WU_S 0xc0100053 @@ -478,8 +523,8 @@ #define MASK_FCVT_L_S 0xfff0007f #define MATCH_FCVT_LU_S 0xc0300053 #define MASK_FCVT_LU_S 0xfff0007f -#define MATCH_FMV_X_S 0xe0000053 -#define MASK_FMV_X_S 0xfff0707f +#define MATCH_FMV_X_W 0xe0000053 +#define MASK_FMV_X_W 0xfff0707f #define MATCH_FCLASS_S 0xe0001053 #define MASK_FCLASS_S 0xfff0707f #define MATCH_FCVT_W_D 0xc2000053 @@ -494,6 +539,18 @@ #define MASK_FMV_X_D 0xfff0707f #define MATCH_FCLASS_D 0xe2001053 #define MASK_FCLASS_D 0xfff0707f +#define MATCH_FCVT_W_Q 0xc6000053 +#define MASK_FCVT_W_Q 0xfff0007f +#define MATCH_FCVT_WU_Q 0xc6100053 +#define MASK_FCVT_WU_Q 0xfff0007f +#define MATCH_FCVT_L_Q 0xc6200053 +#define MASK_FCVT_L_Q 0xfff0007f +#define MATCH_FCVT_LU_Q 0xc6300053 +#define MASK_FCVT_LU_Q 0xfff0007f +#define MATCH_FMV_X_Q 0xe6000053 +#define MASK_FMV_X_Q 0xfff0707f +#define MATCH_FCLASS_Q 0xe6001053 +#define MASK_FCLASS_Q 0xfff0707f #define MATCH_FCVT_S_W 0xd0000053 #define MASK_FCVT_S_W 0xfff0007f #define MATCH_FCVT_S_WU 0xd0100053 @@ -502,8 +559,8 @@ #define MASK_FCVT_S_L 0xfff0007f #define MATCH_FCVT_S_LU 0xd0300053 #define MASK_FCVT_S_LU 0xfff0007f -#define MATCH_FMV_S_X 0xf0000053 -#define MASK_FMV_S_X 0xfff0707f +#define MATCH_FMV_W_X 0xf0000053 +#define MASK_FMV_W_X 0xfff0707f #define MATCH_FCVT_D_W 0xd2000053 #define MASK_FCVT_D_W 0xfff0007f #define MATCH_FCVT_D_WU 0xd2100053 @@ -514,14 +571,28 @@ #define MASK_FCVT_D_LU 0xfff0007f #define MATCH_FMV_D_X 0xf2000053 #define MASK_FMV_D_X 0xfff0707f +#define MATCH_FCVT_Q_W 0xd6000053 +#define MASK_FCVT_Q_W 0xfff0007f +#define MATCH_FCVT_Q_WU 0xd6100053 +#define MASK_FCVT_Q_WU 0xfff0007f +#define MATCH_FCVT_Q_L 0xd6200053 +#define MASK_FCVT_Q_L 0xfff0007f +#define MATCH_FCVT_Q_LU 0xd6300053 +#define MASK_FCVT_Q_LU 0xfff0007f +#define MATCH_FMV_Q_X 0xf6000053 +#define MASK_FMV_Q_X 0xfff0707f #define MATCH_FLW 0x2007 #define MASK_FLW 0x707f #define MATCH_FLD 0x3007 #define MASK_FLD 0x707f +#define MATCH_FLQ 0x4007 +#define MASK_FLQ 0x707f #define MATCH_FSW 0x2027 #define MASK_FSW 0x707f #define MATCH_FSD 0x3027 #define MASK_FSD 0x707f +#define MATCH_FSQ 0x4027 +#define MASK_FSQ 0x707f #define MATCH_FMADD_S 0x43 #define MASK_FMADD_S 0x600007f #define MATCH_FMSUB_S 0x47 @@ -538,6 +609,14 @@ #define MASK_FNMSUB_D 0x600007f #define MATCH_FNMADD_D 0x200004f #define MASK_FNMADD_D 0x600007f +#define MATCH_FMADD_Q 0x6000043 +#define MASK_FMADD_Q 0x600007f +#define MATCH_FMSUB_Q 0x6000047 +#define MASK_FMSUB_Q 0x600007f +#define MATCH_FNMSUB_Q 0x600004b +#define MASK_FNMSUB_Q 0x600007f +#define MATCH_FNMADD_Q 0x600004f +#define MASK_FNMADD_Q 0x600007f #define MATCH_C_NOP 0x1 #define MASK_C_NOP 0xffff #define MATCH_C_ADDI16SP 0x6101 @@ -708,23 +787,45 @@ #define CSR_SSTATUS 0x100 #define CSR_SIE 0x104 #define CSR_STVEC 0x105 +#define CSR_SCOUNTEREN 0x106 #define CSR_SSCRATCH 0x140 #define CSR_SEPC 0x141 #define CSR_SCAUSE 0x142 -#define CSR_SBADADDR 0x143 +#define CSR_STVAL 0x143 #define CSR_SIP 0x144 -#define CSR_SPTBR 0x180 +#define CSR_SATP 0x180 #define CSR_MSTATUS 0x300 #define CSR_MISA 0x301 #define CSR_MEDELEG 0x302 #define CSR_MIDELEG 0x303 #define CSR_MIE 0x304 #define CSR_MTVEC 0x305 +#define CSR_MCOUNTEREN 0x306 #define CSR_MSCRATCH 0x340 #define CSR_MEPC 0x341 #define CSR_MCAUSE 0x342 -#define CSR_MBADADDR 0x343 +#define CSR_MTVAL 0x343 #define CSR_MIP 0x344 +#define CSR_PMPCFG0 0x3a0 +#define CSR_PMPCFG1 0x3a1 +#define CSR_PMPCFG2 0x3a2 +#define CSR_PMPCFG3 0x3a3 +#define CSR_PMPADDR0 0x3b0 +#define CSR_PMPADDR1 0x3b1 +#define CSR_PMPADDR2 0x3b2 +#define CSR_PMPADDR3 0x3b3 +#define CSR_PMPADDR4 0x3b4 +#define CSR_PMPADDR5 0x3b5 +#define CSR_PMPADDR6 0x3b6 +#define CSR_PMPADDR7 0x3b7 +#define CSR_PMPADDR8 0x3b8 +#define CSR_PMPADDR9 0x3b9 +#define CSR_PMPADDR10 0x3ba +#define CSR_PMPADDR11 0x3bb +#define CSR_PMPADDR12 0x3bc +#define CSR_PMPADDR13 0x3bd +#define CSR_PMPADDR14 0x3be +#define CSR_PMPADDR15 0x3bf #define CSR_TSELECT 0x7a0 #define CSR_TDATA1 0x7a1 #define CSR_TDATA2 0x7a2 @@ -763,8 +864,6 @@ #define CSR_MHPMCOUNTER29 0xb1d #define CSR_MHPMCOUNTER30 0xb1e #define CSR_MHPMCOUNTER31 0xb1f -#define CSR_MUCOUNTEREN 0x320 -#define CSR_MSCOUNTEREN 0x321 #define CSR_MHPMEVENT3 0x323 #define CSR_MHPMEVENT4 0x324 #define CSR_MHPMEVENT5 0x325 @@ -862,17 +961,20 @@ #define CSR_MHPMCOUNTER30H 0xb9e #define CSR_MHPMCOUNTER31H 0xb9f #define CAUSE_MISALIGNED_FETCH 0x0 -#define CAUSE_FAULT_FETCH 0x1 +#define CAUSE_FETCH_ACCESS 0x1 #define CAUSE_ILLEGAL_INSTRUCTION 0x2 #define CAUSE_BREAKPOINT 0x3 #define CAUSE_MISALIGNED_LOAD 0x4 -#define CAUSE_FAULT_LOAD 0x5 +#define CAUSE_LOAD_ACCESS 0x5 #define CAUSE_MISALIGNED_STORE 0x6 -#define CAUSE_FAULT_STORE 0x7 +#define CAUSE_STORE_ACCESS 0x7 #define CAUSE_USER_ECALL 0x8 #define CAUSE_SUPERVISOR_ECALL 0x9 #define CAUSE_HYPERVISOR_ECALL 0xa #define CAUSE_MACHINE_ECALL 0xb +#define CAUSE_FETCH_PAGE_FAULT 0xc +#define CAUSE_LOAD_PAGE_FAULT 0xd +#define CAUSE_STORE_PAGE_FAULT 0xf #endif #ifdef DECLARE_INSN DECLARE_INSN(beq, MATCH_BEQ, MASK_BEQ) @@ -965,10 +1067,9 @@ DECLARE_INSN(ecall, MATCH_ECALL, MASK_ECALL) DECLARE_INSN(ebreak, MATCH_EBREAK, MASK_EBREAK) DECLARE_INSN(uret, MATCH_URET, MASK_URET) DECLARE_INSN(sret, MATCH_SRET, MASK_SRET) -DECLARE_INSN(hret, MATCH_HRET, MASK_HRET) DECLARE_INSN(mret, MATCH_MRET, MASK_MRET) DECLARE_INSN(dret, MATCH_DRET, MASK_DRET) -DECLARE_INSN(sfence_vm, MATCH_SFENCE_VM, MASK_SFENCE_VM) +DECLARE_INSN(sfence_vma, MATCH_SFENCE_VMA, MASK_SFENCE_VMA) DECLARE_INSN(wfi, MATCH_WFI, MASK_WFI) DECLARE_INSN(csrrw, MATCH_CSRRW, MASK_CSRRW) DECLARE_INSN(csrrs, MATCH_CSRRS, MASK_CSRRS) @@ -998,17 +1099,34 @@ DECLARE_INSN(fmax_d, MATCH_FMAX_D, MASK_FMAX_D) DECLARE_INSN(fcvt_s_d, MATCH_FCVT_S_D, MASK_FCVT_S_D) DECLARE_INSN(fcvt_d_s, MATCH_FCVT_D_S, MASK_FCVT_D_S) DECLARE_INSN(fsqrt_d, MATCH_FSQRT_D, MASK_FSQRT_D) +DECLARE_INSN(fadd_q, MATCH_FADD_Q, MASK_FADD_Q) +DECLARE_INSN(fsub_q, MATCH_FSUB_Q, MASK_FSUB_Q) +DECLARE_INSN(fmul_q, MATCH_FMUL_Q, MASK_FMUL_Q) +DECLARE_INSN(fdiv_q, MATCH_FDIV_Q, MASK_FDIV_Q) +DECLARE_INSN(fsgnj_q, MATCH_FSGNJ_Q, MASK_FSGNJ_Q) +DECLARE_INSN(fsgnjn_q, MATCH_FSGNJN_Q, MASK_FSGNJN_Q) +DECLARE_INSN(fsgnjx_q, MATCH_FSGNJX_Q, MASK_FSGNJX_Q) +DECLARE_INSN(fmin_q, MATCH_FMIN_Q, MASK_FMIN_Q) +DECLARE_INSN(fmax_q, MATCH_FMAX_Q, MASK_FMAX_Q) +DECLARE_INSN(fcvt_s_q, MATCH_FCVT_S_Q, MASK_FCVT_S_Q) +DECLARE_INSN(fcvt_q_s, MATCH_FCVT_Q_S, MASK_FCVT_Q_S) +DECLARE_INSN(fcvt_d_q, MATCH_FCVT_D_Q, MASK_FCVT_D_Q) +DECLARE_INSN(fcvt_q_d, MATCH_FCVT_Q_D, MASK_FCVT_Q_D) +DECLARE_INSN(fsqrt_q, MATCH_FSQRT_Q, MASK_FSQRT_Q) DECLARE_INSN(fle_s, MATCH_FLE_S, MASK_FLE_S) DECLARE_INSN(flt_s, MATCH_FLT_S, MASK_FLT_S) DECLARE_INSN(feq_s, MATCH_FEQ_S, MASK_FEQ_S) DECLARE_INSN(fle_d, MATCH_FLE_D, MASK_FLE_D) DECLARE_INSN(flt_d, MATCH_FLT_D, MASK_FLT_D) DECLARE_INSN(feq_d, MATCH_FEQ_D, MASK_FEQ_D) +DECLARE_INSN(fle_q, MATCH_FLE_Q, MASK_FLE_Q) +DECLARE_INSN(flt_q, MATCH_FLT_Q, MASK_FLT_Q) +DECLARE_INSN(feq_q, MATCH_FEQ_Q, MASK_FEQ_Q) DECLARE_INSN(fcvt_w_s, MATCH_FCVT_W_S, MASK_FCVT_W_S) DECLARE_INSN(fcvt_wu_s, MATCH_FCVT_WU_S, MASK_FCVT_WU_S) DECLARE_INSN(fcvt_l_s, MATCH_FCVT_L_S, MASK_FCVT_L_S) DECLARE_INSN(fcvt_lu_s, MATCH_FCVT_LU_S, MASK_FCVT_LU_S) -DECLARE_INSN(fmv_x_s, MATCH_FMV_X_S, MASK_FMV_X_S) +DECLARE_INSN(fmv_x_w, MATCH_FMV_X_W, MASK_FMV_X_W) DECLARE_INSN(fclass_s, MATCH_FCLASS_S, MASK_FCLASS_S) DECLARE_INSN(fcvt_w_d, MATCH_FCVT_W_D, MASK_FCVT_W_D) DECLARE_INSN(fcvt_wu_d, MATCH_FCVT_WU_D, MASK_FCVT_WU_D) @@ -1016,20 +1134,33 @@ DECLARE_INSN(fcvt_l_d, MATCH_FCVT_L_D, MASK_FCVT_L_D) DECLARE_INSN(fcvt_lu_d, MATCH_FCVT_LU_D, MASK_FCVT_LU_D) DECLARE_INSN(fmv_x_d, MATCH_FMV_X_D, MASK_FMV_X_D) DECLARE_INSN(fclass_d, MATCH_FCLASS_D, MASK_FCLASS_D) +DECLARE_INSN(fcvt_w_q, MATCH_FCVT_W_Q, MASK_FCVT_W_Q) +DECLARE_INSN(fcvt_wu_q, MATCH_FCVT_WU_Q, MASK_FCVT_WU_Q) +DECLARE_INSN(fcvt_l_q, MATCH_FCVT_L_Q, MASK_FCVT_L_Q) +DECLARE_INSN(fcvt_lu_q, MATCH_FCVT_LU_Q, MASK_FCVT_LU_Q) +DECLARE_INSN(fmv_x_q, MATCH_FMV_X_Q, MASK_FMV_X_Q) +DECLARE_INSN(fclass_q, MATCH_FCLASS_Q, MASK_FCLASS_Q) DECLARE_INSN(fcvt_s_w, MATCH_FCVT_S_W, MASK_FCVT_S_W) DECLARE_INSN(fcvt_s_wu, MATCH_FCVT_S_WU, MASK_FCVT_S_WU) DECLARE_INSN(fcvt_s_l, MATCH_FCVT_S_L, MASK_FCVT_S_L) DECLARE_INSN(fcvt_s_lu, MATCH_FCVT_S_LU, MASK_FCVT_S_LU) -DECLARE_INSN(fmv_s_x, MATCH_FMV_S_X, MASK_FMV_S_X) +DECLARE_INSN(fmv_w_x, MATCH_FMV_W_X, MASK_FMV_W_X) DECLARE_INSN(fcvt_d_w, MATCH_FCVT_D_W, MASK_FCVT_D_W) DECLARE_INSN(fcvt_d_wu, MATCH_FCVT_D_WU, MASK_FCVT_D_WU) DECLARE_INSN(fcvt_d_l, MATCH_FCVT_D_L, MASK_FCVT_D_L) DECLARE_INSN(fcvt_d_lu, MATCH_FCVT_D_LU, MASK_FCVT_D_LU) DECLARE_INSN(fmv_d_x, MATCH_FMV_D_X, MASK_FMV_D_X) +DECLARE_INSN(fcvt_q_w, MATCH_FCVT_Q_W, MASK_FCVT_Q_W) +DECLARE_INSN(fcvt_q_wu, MATCH_FCVT_Q_WU, MASK_FCVT_Q_WU) +DECLARE_INSN(fcvt_q_l, MATCH_FCVT_Q_L, MASK_FCVT_Q_L) +DECLARE_INSN(fcvt_q_lu, MATCH_FCVT_Q_LU, MASK_FCVT_Q_LU) +DECLARE_INSN(fmv_q_x, MATCH_FMV_Q_X, MASK_FMV_Q_X) DECLARE_INSN(flw, MATCH_FLW, MASK_FLW) DECLARE_INSN(fld, MATCH_FLD, MASK_FLD) +DECLARE_INSN(flq, MATCH_FLQ, MASK_FLQ) DECLARE_INSN(fsw, MATCH_FSW, MASK_FSW) DECLARE_INSN(fsd, MATCH_FSD, MASK_FSD) +DECLARE_INSN(fsq, MATCH_FSQ, MASK_FSQ) DECLARE_INSN(fmadd_s, MATCH_FMADD_S, MASK_FMADD_S) DECLARE_INSN(fmsub_s, MATCH_FMSUB_S, MASK_FMSUB_S) DECLARE_INSN(fnmsub_s, MATCH_FNMSUB_S, MASK_FNMSUB_S) @@ -1038,6 +1169,10 @@ DECLARE_INSN(fmadd_d, MATCH_FMADD_D, MASK_FMADD_D) DECLARE_INSN(fmsub_d, MATCH_FMSUB_D, MASK_FMSUB_D) DECLARE_INSN(fnmsub_d, MATCH_FNMSUB_D, MASK_FNMSUB_D) DECLARE_INSN(fnmadd_d, MATCH_FNMADD_D, MASK_FNMADD_D) +DECLARE_INSN(fmadd_q, MATCH_FMADD_Q, MASK_FMADD_Q) +DECLARE_INSN(fmsub_q, MATCH_FMSUB_Q, MASK_FMSUB_Q) +DECLARE_INSN(fnmsub_q, MATCH_FNMSUB_Q, MASK_FNMSUB_Q) +DECLARE_INSN(fnmadd_q, MATCH_FNMADD_Q, MASK_FNMADD_Q) DECLARE_INSN(c_nop, MATCH_C_NOP, MASK_C_NOP) DECLARE_INSN(c_addi16sp, MATCH_C_ADDI16SP, MASK_C_ADDI16SP) DECLARE_INSN(c_jr, MATCH_C_JR, MASK_C_JR) @@ -1144,23 +1279,45 @@ DECLARE_CSR(hpmcounter31, CSR_HPMCOUNTER31) DECLARE_CSR(sstatus, CSR_SSTATUS) DECLARE_CSR(sie, CSR_SIE) DECLARE_CSR(stvec, CSR_STVEC) +DECLARE_CSR(scounteren, CSR_SCOUNTEREN) DECLARE_CSR(sscratch, CSR_SSCRATCH) DECLARE_CSR(sepc, CSR_SEPC) DECLARE_CSR(scause, CSR_SCAUSE) -DECLARE_CSR(sbadaddr, CSR_SBADADDR) +DECLARE_CSR(stval, CSR_STVAL) DECLARE_CSR(sip, CSR_SIP) -DECLARE_CSR(sptbr, CSR_SPTBR) +DECLARE_CSR(satp, CSR_SATP) DECLARE_CSR(mstatus, CSR_MSTATUS) DECLARE_CSR(misa, CSR_MISA) DECLARE_CSR(medeleg, CSR_MEDELEG) DECLARE_CSR(mideleg, CSR_MIDELEG) DECLARE_CSR(mie, CSR_MIE) DECLARE_CSR(mtvec, CSR_MTVEC) +DECLARE_CSR(mcounteren, CSR_MCOUNTEREN) DECLARE_CSR(mscratch, CSR_MSCRATCH) DECLARE_CSR(mepc, CSR_MEPC) DECLARE_CSR(mcause, CSR_MCAUSE) -DECLARE_CSR(mbadaddr, CSR_MBADADDR) +DECLARE_CSR(mtval, CSR_MTVAL) DECLARE_CSR(mip, CSR_MIP) +DECLARE_CSR(pmpcfg0, CSR_PMPCFG0) +DECLARE_CSR(pmpcfg1, CSR_PMPCFG1) +DECLARE_CSR(pmpcfg2, CSR_PMPCFG2) +DECLARE_CSR(pmpcfg3, CSR_PMPCFG3) +DECLARE_CSR(pmpaddr0, CSR_PMPADDR0) +DECLARE_CSR(pmpaddr1, CSR_PMPADDR1) +DECLARE_CSR(pmpaddr2, CSR_PMPADDR2) +DECLARE_CSR(pmpaddr3, CSR_PMPADDR3) +DECLARE_CSR(pmpaddr4, CSR_PMPADDR4) +DECLARE_CSR(pmpaddr5, CSR_PMPADDR5) +DECLARE_CSR(pmpaddr6, CSR_PMPADDR6) +DECLARE_CSR(pmpaddr7, CSR_PMPADDR7) +DECLARE_CSR(pmpaddr8, CSR_PMPADDR8) +DECLARE_CSR(pmpaddr9, CSR_PMPADDR9) +DECLARE_CSR(pmpaddr10, CSR_PMPADDR10) +DECLARE_CSR(pmpaddr11, CSR_PMPADDR11) +DECLARE_CSR(pmpaddr12, CSR_PMPADDR12) +DECLARE_CSR(pmpaddr13, CSR_PMPADDR13) +DECLARE_CSR(pmpaddr14, CSR_PMPADDR14) +DECLARE_CSR(pmpaddr15, CSR_PMPADDR15) DECLARE_CSR(tselect, CSR_TSELECT) DECLARE_CSR(tdata1, CSR_TDATA1) DECLARE_CSR(tdata2, CSR_TDATA2) @@ -1199,8 +1356,6 @@ DECLARE_CSR(mhpmcounter28, CSR_MHPMCOUNTER28) DECLARE_CSR(mhpmcounter29, CSR_MHPMCOUNTER29) DECLARE_CSR(mhpmcounter30, CSR_MHPMCOUNTER30) DECLARE_CSR(mhpmcounter31, CSR_MHPMCOUNTER31) -DECLARE_CSR(mucounteren, CSR_MUCOUNTEREN) -DECLARE_CSR(mscounteren, CSR_MSCOUNTEREN) DECLARE_CSR(mhpmevent3, CSR_MHPMEVENT3) DECLARE_CSR(mhpmevent4, CSR_MHPMEVENT4) DECLARE_CSR(mhpmevent5, CSR_MHPMEVENT5) @@ -1300,15 +1455,18 @@ DECLARE_CSR(mhpmcounter31h, CSR_MHPMCOUNTER31H) #endif #ifdef DECLARE_CAUSE DECLARE_CAUSE("misaligned fetch", CAUSE_MISALIGNED_FETCH) -DECLARE_CAUSE("fault fetch", CAUSE_FAULT_FETCH) +DECLARE_CAUSE("fetch access", CAUSE_FETCH_ACCESS) DECLARE_CAUSE("illegal instruction", CAUSE_ILLEGAL_INSTRUCTION) DECLARE_CAUSE("breakpoint", CAUSE_BREAKPOINT) DECLARE_CAUSE("misaligned load", CAUSE_MISALIGNED_LOAD) -DECLARE_CAUSE("fault load", CAUSE_FAULT_LOAD) +DECLARE_CAUSE("load access", CAUSE_LOAD_ACCESS) DECLARE_CAUSE("misaligned store", CAUSE_MISALIGNED_STORE) -DECLARE_CAUSE("fault store", CAUSE_FAULT_STORE) +DECLARE_CAUSE("store access", CAUSE_STORE_ACCESS) DECLARE_CAUSE("user_ecall", CAUSE_USER_ECALL) DECLARE_CAUSE("supervisor_ecall", CAUSE_SUPERVISOR_ECALL) DECLARE_CAUSE("hypervisor_ecall", CAUSE_HYPERVISOR_ECALL) DECLARE_CAUSE("machine_ecall", CAUSE_MACHINE_ECALL) +DECLARE_CAUSE("fetch page fault", CAUSE_FETCH_PAGE_FAULT) +DECLARE_CAUSE("load page fault", CAUSE_LOAD_PAGE_FAULT) +DECLARE_CAUSE("store page fault", CAUSE_STORE_PAGE_FAULT) #endif diff --git a/machine/fdt.c b/machine/fdt.c new file mode 100644 index 0000000..061b19e --- /dev/null +++ b/machine/fdt.c @@ -0,0 +1,740 @@ +#include <stdbool.h> +#include <stdint.h> +#include <string.h> +#include "config.h" +#include "fdt.h" +#include "mtrap.h" + +static inline uint32_t bswap(uint32_t x) +{ + uint32_t y = (x & 0x00FF00FF) << 8 | (x & 0xFF00FF00) >> 8; + uint32_t z = (y & 0x0000FFFF) << 16 | (y & 0xFFFF0000) >> 16; + return z; +} + +static inline int isstring(char c) +{ + if (c >= 'A' && c <= 'Z') + return 1; + if (c >= 'a' && c <= 'z') + return 1; + if (c >= '0' && c <= '9') + return 1; + if (c == '\0' || c == ' ' || c == ',' || c == '-') + return 1; + return 0; +} + +static uint32_t *fdt_scan_helper( + uint32_t *lex, + const char *strings, + struct fdt_scan_node *node, + const struct fdt_cb *cb) +{ + struct fdt_scan_node child; + struct fdt_scan_prop prop; + int last = 0; + + child.parent = node; + // these are the default cell counts, as per the FDT spec + child.address_cells = 2; + child.size_cells = 1; + prop.node = node; + + while (1) { + switch (bswap(lex[0])) { + case FDT_NOP: { + lex += 1; + break; + } + case FDT_PROP: { + assert (!last); + prop.name = strings + bswap(lex[2]); + prop.len = bswap(lex[1]); + prop.value = lex + 3; + if (node && !strcmp(prop.name, "#address-cells")) { node->address_cells = bswap(lex[3]); } + if (node && !strcmp(prop.name, "#size-cells")) { node->size_cells = bswap(lex[3]); } + lex += 3 + (prop.len+3)/4; + cb->prop(&prop, cb->extra); + break; + } + case FDT_BEGIN_NODE: { + uint32_t *lex_next; + if (!last && node && cb->done) cb->done(node, cb->extra); + last = 1; + child.name = (const char *)(lex+1); + if (cb->open) cb->open(&child, cb->extra); + lex_next = fdt_scan_helper( + lex + 2 + strlen(child.name)/4, + strings, &child, cb); + if (cb->close && cb->close(&child, cb->extra) == -1) + while (lex != lex_next) *lex++ = bswap(FDT_NOP); + lex = lex_next; + break; + } + case FDT_END_NODE: { + if (!last && node && cb->done) cb->done(node, cb->extra); + return lex + 1; + } + default: { // FDT_END + if (!last && node && cb->done) cb->done(node, cb->extra); + return lex; + } + } + } +} + +void fdt_scan(uintptr_t fdt, const struct fdt_cb *cb) +{ + struct fdt_header *header = (struct fdt_header *)fdt; + + // Only process FDT that we understand + if (bswap(header->magic) != FDT_MAGIC || + bswap(header->last_comp_version) > FDT_VERSION) return; + + const char *strings = (const char *)(fdt + bswap(header->off_dt_strings)); + uint32_t *lex = (uint32_t *)(fdt + bswap(header->off_dt_struct)); + + fdt_scan_helper(lex, strings, 0, cb); +} + +uint32_t fdt_size(uintptr_t fdt) +{ + struct fdt_header *header = (struct fdt_header *)fdt; + + // Only process FDT that we understand + if (bswap(header->magic) != FDT_MAGIC || + bswap(header->last_comp_version) > FDT_VERSION) return 0; + return bswap(header->totalsize); +} + +const uint32_t *fdt_get_address(const struct fdt_scan_node *node, const uint32_t *value, uint64_t *result) +{ + *result = 0; + for (int cells = node->address_cells; cells > 0; --cells) + *result = (*result << 32) + bswap(*value++); + return value; +} + +const uint32_t *fdt_get_size(const struct fdt_scan_node *node, const uint32_t *value, uint64_t *result) +{ + *result = 0; + for (int cells = node->size_cells; cells > 0; --cells) + *result = (*result << 32) + bswap(*value++); + return value; +} + +int fdt_string_list_index(const struct fdt_scan_prop *prop, const char *str) +{ + const char *list = (const char *)prop->value; + const char *end = list + prop->len; + int index = 0; + while (end - list > 0) { + if (!strcmp(list, str)) return index; + ++index; + list += strlen(list) + 1; + } + return -1; +} + +//////////////////////////////////////////// MEMORY SCAN ///////////////////////////////////////// + +struct mem_scan { + int memory; + const uint32_t *reg_value; + int reg_len; +}; + +static void mem_open(const struct fdt_scan_node *node, void *extra) +{ + struct mem_scan *scan = (struct mem_scan *)extra; + memset(scan, 0, sizeof(*scan)); +} + +static void mem_prop(const struct fdt_scan_prop *prop, void *extra) +{ + struct mem_scan *scan = (struct mem_scan *)extra; + if (!strcmp(prop->name, "device_type") && !strcmp((const char*)prop->value, "memory")) { + scan->memory = 1; + } else if (!strcmp(prop->name, "reg")) { + scan->reg_value = prop->value; + scan->reg_len = prop->len; + } +} + +static void mem_done(const struct fdt_scan_node *node, void *extra) +{ + struct mem_scan *scan = (struct mem_scan *)extra; + const uint32_t *value = scan->reg_value; + const uint32_t *end = value + scan->reg_len/4; + uintptr_t self = (uintptr_t)mem_done; + + if (!scan->memory) return; + assert (scan->reg_value && scan->reg_len % 4 == 0); + + while (end - value > 0) { + uint64_t base, size; + value = fdt_get_address(node->parent, value, &base); + value = fdt_get_size (node->parent, value, &size); + if (base <= self && self <= base + size) { mem_size = size; } + } + assert (end == value); +} + +void query_mem(uintptr_t fdt) +{ + struct fdt_cb cb; + struct mem_scan scan; + + memset(&cb, 0, sizeof(cb)); + cb.open = mem_open; + cb.prop = mem_prop; + cb.done = mem_done; + cb.extra = &scan; + + mem_size = 0; + fdt_scan(fdt, &cb); + assert (mem_size > 0); +} + +///////////////////////////////////////////// HART SCAN ////////////////////////////////////////// + +static uint32_t hart_phandles[MAX_HARTS]; +uint64_t hart_mask; + +struct hart_scan { + const struct fdt_scan_node *cpu; + int hart; + const struct fdt_scan_node *controller; + int cells; + uint32_t phandle; +}; + +static void hart_open(const struct fdt_scan_node *node, void *extra) +{ + struct hart_scan *scan = (struct hart_scan *)extra; + if (!scan->cpu) { + scan->hart = -1; + } + if (!scan->controller) { + scan->cells = 0; + scan->phandle = 0; + } +} + +static void hart_prop(const struct fdt_scan_prop *prop, void *extra) +{ + struct hart_scan *scan = (struct hart_scan *)extra; + if (!strcmp(prop->name, "device_type") && !strcmp((const char*)prop->value, "cpu")) { + assert (!scan->cpu); + scan->cpu = prop->node; + } else if (!strcmp(prop->name, "interrupt-controller")) { + assert (!scan->controller); + scan->controller = prop->node; + } else if (!strcmp(prop->name, "#interrupt-cells")) { + scan->cells = bswap(prop->value[0]); + } else if (!strcmp(prop->name, "phandle")) { + scan->phandle = bswap(prop->value[0]); + } else if (!strcmp(prop->name, "reg")) { + uint64_t reg; + fdt_get_address(prop->node->parent, prop->value, ®); + scan->hart = reg; + } +} + +static void hart_done(const struct fdt_scan_node *node, void *extra) +{ + struct hart_scan *scan = (struct hart_scan *)extra; + + if (scan->cpu == node) { + assert (scan->hart >= 0); + } + + if (scan->controller == node && scan->cpu) { + assert (scan->phandle > 0); + assert (scan->cells == 1); + + if (scan->hart < MAX_HARTS) { + hart_phandles[scan->hart] = scan->phandle; + hart_mask |= 1 << scan->hart; + hls_init(scan->hart); + } + } +} + +static int hart_close(const struct fdt_scan_node *node, void *extra) +{ + struct hart_scan *scan = (struct hart_scan *)extra; + if (scan->cpu == node) scan->cpu = 0; + if (scan->controller == node) scan->controller = 0; + return 0; +} + +void query_harts(uintptr_t fdt) +{ + struct fdt_cb cb; + struct hart_scan scan; + + memset(&cb, 0, sizeof(cb)); + memset(&scan, 0, sizeof(scan)); + cb.open = hart_open; + cb.prop = hart_prop; + cb.done = hart_done; + cb.close= hart_close; + cb.extra = &scan; + + fdt_scan(fdt, &cb); + + // The current hart should have been detected + assert ((hart_mask >> read_csr(mhartid)) != 0); +} + +///////////////////////////////////////////// CLINT SCAN ///////////////////////////////////////// + +struct clint_scan +{ + int compat; + uint64_t reg; + const uint32_t *int_value; + int int_len; + int done; +}; + +static void clint_open(const struct fdt_scan_node *node, void *extra) +{ + struct clint_scan *scan = (struct clint_scan *)extra; + scan->compat = 0; + scan->reg = 0; + scan->int_value = 0; +} + +static void clint_prop(const struct fdt_scan_prop *prop, void *extra) +{ + struct clint_scan *scan = (struct clint_scan *)extra; + if (!strcmp(prop->name, "compatible") && fdt_string_list_index(prop, "riscv,clint0") >= 0) { + scan->compat = 1; + } else if (!strcmp(prop->name, "reg")) { + fdt_get_address(prop->node->parent, prop->value, &scan->reg); + } else if (!strcmp(prop->name, "interrupts-extended")) { + scan->int_value = prop->value; + scan->int_len = prop->len; + } +} + +static void clint_done(const struct fdt_scan_node *node, void *extra) +{ + struct clint_scan *scan = (struct clint_scan *)extra; + const uint32_t *value = scan->int_value; + const uint32_t *end = value + scan->int_len/4; + + if (!scan->compat) return; + assert (scan->reg != 0); + assert (scan->int_value && scan->int_len % 16 == 0); + assert (!scan->done); // only one clint + + scan->done = 1; + mtime = (void*)((uintptr_t)scan->reg + 0xbff8); + + for (int index = 0; end - value > 0; ++index) { + uint32_t phandle = bswap(value[0]); + int hart; + for (hart = 0; hart < MAX_HARTS; ++hart) + if (hart_phandles[hart] == phandle) + break; + if (hart < MAX_HARTS) { + hls_t *hls = OTHER_HLS(hart); + hls->ipi = (void*)((uintptr_t)scan->reg + index * 4); + hls->timecmp = (void*)((uintptr_t)scan->reg + 0x4000 + (index * 8)); + } + value += 4; + } +} + +void query_clint(uintptr_t fdt) +{ + struct fdt_cb cb; + struct clint_scan scan; + + memset(&cb, 0, sizeof(cb)); + cb.open = clint_open; + cb.prop = clint_prop; + cb.done = clint_done; + cb.extra = &scan; + + scan.done = 0; + fdt_scan(fdt, &cb); + assert (scan.done); +} + +///////////////////////////////////////////// PLIC SCAN ///////////////////////////////////////// + +struct plic_scan +{ + int compat; + uint64_t reg; + uint32_t *int_value; + int int_len; + int done; + int ndev; +}; + +static void plic_open(const struct fdt_scan_node *node, void *extra) +{ + struct plic_scan *scan = (struct plic_scan *)extra; + scan->compat = 0; + scan->reg = 0; + scan->int_value = 0; +} + +static void plic_prop(const struct fdt_scan_prop *prop, void *extra) +{ + struct plic_scan *scan = (struct plic_scan *)extra; + if (!strcmp(prop->name, "compatible") && fdt_string_list_index(prop, "riscv,plic0") >= 0) { + scan->compat = 1; + } else if (!strcmp(prop->name, "reg")) { + fdt_get_address(prop->node->parent, prop->value, &scan->reg); + } else if (!strcmp(prop->name, "interrupts-extended")) { + scan->int_value = prop->value; + scan->int_len = prop->len; + } else if (!strcmp(prop->name, "riscv,ndev")) { + scan->ndev = bswap(prop->value[0]); + } +} + +#define HART_BASE 0x200000 +#define HART_SIZE 0x1000 +#define ENABLE_BASE 0x2000 +#define ENABLE_SIZE 0x80 + +static void plic_done(const struct fdt_scan_node *node, void *extra) +{ + struct plic_scan *scan = (struct plic_scan *)extra; + const uint32_t *value = scan->int_value; + const uint32_t *end = value + scan->int_len/4; + + if (!scan->compat) return; + assert (scan->reg != 0); + assert (scan->int_value && scan->int_len % 8 == 0); + assert (scan->ndev >= 0 && scan->ndev < 1024); + assert (!scan->done); // only one plic + + scan->done = 1; + plic_priorities = (uint32_t*)(uintptr_t)scan->reg; + plic_ndevs = scan->ndev; + + for (int index = 0; end - value > 0; ++index) { + uint32_t phandle = bswap(value[0]); + uint32_t cpu_int = bswap(value[1]); + int hart; + for (hart = 0; hart < MAX_HARTS; ++hart) + if (hart_phandles[hart] == phandle) + break; + if (hart < MAX_HARTS) { + hls_t *hls = OTHER_HLS(hart); + if (cpu_int == IRQ_M_EXT) { + hls->plic_m_ie = (uintptr_t*)((uintptr_t)scan->reg + ENABLE_BASE + ENABLE_SIZE * index); + hls->plic_m_thresh = (uint32_t*) ((uintptr_t)scan->reg + HART_BASE + HART_SIZE * index); + } else if (cpu_int == IRQ_S_EXT) { + hls->plic_s_ie = (uintptr_t*)((uintptr_t)scan->reg + ENABLE_BASE + ENABLE_SIZE * index); + hls->plic_s_thresh = (uint32_t*) ((uintptr_t)scan->reg + HART_BASE + HART_SIZE * index); + } else { + printm("PLIC wired hart %d to wrong interrupt %d", hart, cpu_int); + } + } + value += 2; + } +#if 0 + printm("PLIC: prio %x devs %d\r\n", (uint32_t)(uintptr_t)plic_priorities, plic_ndevs); + for (int i = 0; i < MAX_HARTS; ++i) { + hls_t *hls = OTHER_HLS(i); + printm("CPU %d: %x %x %x %x\r\n", i, (uint32_t)(uintptr_t)hls->plic_m_ie, (uint32_t)(uintptr_t)hls->plic_m_thresh, (uint32_t)(uintptr_t)hls->plic_s_ie, (uint32_t)(uintptr_t)hls->plic_s_thresh); + } +#endif +} + +void query_plic(uintptr_t fdt) +{ + struct fdt_cb cb; + struct plic_scan scan; + + memset(&cb, 0, sizeof(cb)); + cb.open = plic_open; + cb.prop = plic_prop; + cb.done = plic_done; + cb.extra = &scan; + + scan.done = 0; + fdt_scan(fdt, &cb); +} + +static void plic_redact(const struct fdt_scan_node *node, void *extra) +{ + struct plic_scan *scan = (struct plic_scan *)extra; + uint32_t *value = scan->int_value; + uint32_t *end = value + scan->int_len/4; + + if (!scan->compat) return; + scan->done = 1; + + while (end - value > 0) { + if (bswap(value[1]) == IRQ_M_EXT) value[1] = bswap(-1); + value += 2; + } +} + +void filter_plic(uintptr_t fdt) +{ + struct fdt_cb cb; + struct plic_scan scan; + + memset(&cb, 0, sizeof(cb)); + cb.open = plic_open; + cb.prop = plic_prop; + cb.done = plic_redact; + cb.extra = &scan; + + scan.done = 0; + fdt_scan(fdt, &cb); +} + +//////////////////////////////////////////// COMPAT SCAN //////////////////////////////////////// + +struct compat_scan +{ + const char *compat; + int depth; + int kill; +}; + +static void compat_open(const struct fdt_scan_node *node, void *extra) +{ + struct compat_scan *scan = (struct compat_scan *)extra; + ++scan->depth; +} + +static void compat_prop(const struct fdt_scan_prop *prop, void *extra) +{ + struct compat_scan *scan = (struct compat_scan *)extra; + if (!strcmp(prop->name, "compatible") && fdt_string_list_index(prop, scan->compat) >= 0) + if (scan->depth < scan->kill) + scan->kill = scan->depth; +} + +static int compat_close(const struct fdt_scan_node *node, void *extra) +{ + struct compat_scan *scan = (struct compat_scan *)extra; + if (scan->kill == scan->depth--) { + scan->kill = 999; + return -1; + } else { + return 0; + } +} + +void filter_compat(uintptr_t fdt, const char *compat) +{ + struct fdt_cb cb; + struct compat_scan scan; + + memset(&cb, 0, sizeof(cb)); + cb.open = compat_open; + cb.prop = compat_prop; + cb.close = compat_close; + cb.extra = &scan; + + scan.compat = compat; + scan.depth = 0; + scan.kill = 999; + fdt_scan(fdt, &cb); +} + +//////////////////////////////////////////// HART FILTER //////////////////////////////////////// + +struct hart_filter { + int compat; + int hart; + char *status; + char *mmu_type; + long *disabled_hart_mask; +}; + +static void hart_filter_open(const struct fdt_scan_node *node, void *extra) +{ + struct hart_filter *filter = (struct hart_filter *)extra; + filter->status = NULL; + filter->mmu_type = NULL; + filter->compat = 0; + filter->hart = -1; +} + +static void hart_filter_prop(const struct fdt_scan_prop *prop, void *extra) +{ + struct hart_filter *filter = (struct hart_filter *)extra; + if (!strcmp(prop->name, "device_type") && !strcmp((const char*)prop->value, "cpu")) { + filter->compat = 1; + } else if (!strcmp(prop->name, "reg")) { + uint64_t reg; + fdt_get_address(prop->node->parent, prop->value, ®); + filter->hart = reg; + } else if (!strcmp(prop->name, "status")) { + filter->status = (char*)prop->value; + } else if (!strcmp(prop->name, "mmu-type")) { + filter->mmu_type = (char*)prop->value; + } +} + +static bool hart_filter_mask(const struct hart_filter *filter) +{ + if (filter->mmu_type == NULL) return true; + if (strcmp(filter->status, "okay")) return true; + if (!strcmp(filter->mmu_type, "riscv,sv39")) return false; + if (!strcmp(filter->mmu_type, "riscv,sv48")) return false; + printm("hart_filter_mask saw unknown hart type: status=\"%s\", mmu_type=\"%s\"\n", + filter->status, filter->mmu_type); + return true; +} + +static void hart_filter_done(const struct fdt_scan_node *node, void *extra) +{ + struct hart_filter *filter = (struct hart_filter *)extra; + + if (!filter->compat) return; + assert (filter->status); + assert (filter->hart >= 0); + + if (hart_filter_mask(filter)) { + strcpy(filter->status, "masked"); + uint32_t *len = (uint32_t*)filter->status; + len[-2] = bswap(strlen("masked")+1); + *filter->disabled_hart_mask |= (1 << filter->hart); + } +} + +void filter_harts(uintptr_t fdt, long *disabled_hart_mask) +{ + struct fdt_cb cb; + struct hart_filter filter; + + memset(&cb, 0, sizeof(cb)); + cb.open = hart_filter_open; + cb.prop = hart_filter_prop; + cb.done = hart_filter_done; + cb.extra = &filter; + + filter.disabled_hart_mask = disabled_hart_mask; + *disabled_hart_mask = 0; + fdt_scan(fdt, &cb); +} + +//////////////////////////////////////////// PRINT ////////////////////////////////////////////// + +#ifdef PK_PRINT_DEVICE_TREE +#define FDT_PRINT_MAX_DEPTH 32 + +struct fdt_print_info { + int depth; + const struct fdt_scan_node *stack[FDT_PRINT_MAX_DEPTH]; +}; + +void fdt_print_printm(struct fdt_print_info *info, const char *format, ...) +{ + va_list vl; + + for (int i = 0; i < info->depth; ++i) + printm(" "); + + va_start(vl, format); + vprintm(format, vl); + va_end(vl); +} + +static void fdt_print_open(const struct fdt_scan_node *node, void *extra) +{ + struct fdt_print_info *info = (struct fdt_print_info *)extra; + + while (node->parent != NULL && info->stack[info->depth-1] != node->parent) { + info->depth--; + fdt_print_printm(info, "}\r\n"); + } + + fdt_print_printm(info, "%s {\r\n", node->name); + info->stack[info->depth] = node; + info->depth++; +} + +static void fdt_print_prop(const struct fdt_scan_prop *prop, void *extra) +{ + struct fdt_print_info *info = (struct fdt_print_info *)extra; + int asstring = 1; + char *char_data = (char *)(prop->value); + + fdt_print_printm(info, "%s", prop->name); + + if (prop->len == 0) { + printm(";\r\n"); + return; + } else { + printm(" = "); + } + + /* It appears that dtc uses a hueristic to detect strings so I'm using a + * similar one here. */ + for (int i = 0; i < prop->len; ++i) { + if (!isstring(char_data[i])) + asstring = 0; + if (i > 0 && char_data[i] == '\0' && char_data[i-1] == '\0') + asstring = 0; + } + + if (asstring) { + for (size_t i = 0; i < prop->len; i += strlen(char_data + i) + 1) { + if (i != 0) + printm(", "); + printm("\"%s\"", char_data + i); + } + } else { + printm("<"); + for (size_t i = 0; i < prop->len/4; ++i) { + if (i != 0) + printm(" "); + printm("0x%08x", bswap(prop->value[i])); + } + printm(">"); + } + + printm(";\r\n"); +} + +static void fdt_print_done(const struct fdt_scan_node *node, void *extra) +{ + struct fdt_print_info *info = (struct fdt_print_info *)extra; +} + +static int fdt_print_close(const struct fdt_scan_node *node, void *extra) +{ + struct fdt_print_info *info = (struct fdt_print_info *)extra; + return 0; +} + +void fdt_print(uintptr_t fdt) +{ + struct fdt_print_info info; + struct fdt_cb cb; + + info.depth = 0; + + memset(&cb, 0, sizeof(cb)); + cb.open = fdt_print_open; + cb.prop = fdt_print_prop; + cb.done = fdt_print_done; + cb.close = fdt_print_close; + cb.extra = &info; + + fdt_scan(fdt, &cb); + + while (info.depth > 0) { + info.depth--; + fdt_print_printm(&info, "}\r\n"); + } +} +#endif diff --git a/machine/fdt.h b/machine/fdt.h new file mode 100644 index 0000000..d436778 --- /dev/null +++ b/machine/fdt.h @@ -0,0 +1,76 @@ +#ifndef FDT_H +#define FDT_H + +#define FDT_MAGIC 0xd00dfeed +#define FDT_VERSION 17 + +struct fdt_header { + uint32_t magic; + uint32_t totalsize; + uint32_t off_dt_struct; + uint32_t off_dt_strings; + uint32_t off_mem_rsvmap; + uint32_t version; + uint32_t last_comp_version; /* <= 17 */ + uint32_t boot_cpuid_phys; + uint32_t size_dt_strings; + uint32_t size_dt_struct; +}; + +#define FDT_BEGIN_NODE 1 +#define FDT_END_NODE 2 +#define FDT_PROP 3 +#define FDT_NOP 4 +#define FDT_END 9 + +struct fdt_scan_node { + const struct fdt_scan_node *parent; + const char *name; + int address_cells; + int size_cells; +}; + +struct fdt_scan_prop { + const struct fdt_scan_node *node; + const char *name; + uint32_t *value; + int len; // in bytes of value +}; + +struct fdt_cb { + void (*open)(const struct fdt_scan_node *node, void *extra); + void (*prop)(const struct fdt_scan_prop *prop, void *extra); + void (*done)(const struct fdt_scan_node *node, void *extra); // last property was seen + int (*close)(const struct fdt_scan_node *node, void *extra); // -1 => delete the node + children + void *extra; +}; + +// Scan the contents of FDT +void fdt_scan(uintptr_t fdt, const struct fdt_cb *cb); +uint32_t fdt_size(uintptr_t fdt); + +// Extract fields +const uint32_t *fdt_get_address(const struct fdt_scan_node *node, const uint32_t *base, uint64_t *value); +const uint32_t *fdt_get_size(const struct fdt_scan_node *node, const uint32_t *base, uint64_t *value); +int fdt_string_list_index(const struct fdt_scan_prop *prop, const char *str); // -1 if not found + +// Setup memory+clint+plic +void query_mem(uintptr_t fdt); +void query_harts(uintptr_t fdt); +void query_plic(uintptr_t fdt); +void query_clint(uintptr_t fdt); + +// Remove information from FDT +void filter_harts(uintptr_t fdt, long *disabled_hart_mask); +void filter_plic(uintptr_t fdt); +void filter_compat(uintptr_t fdt, const char *compat); + +// The hartids of available harts +extern uint64_t hart_mask; + +#ifdef PK_PRINT_DEVICE_TREE +// Prints the device tree to the console as a DTS +void fdt_print(uintptr_t fdt); +#endif + +#endif diff --git a/machine/finisher.c b/machine/finisher.c new file mode 100644 index 0000000..d113096 --- /dev/null +++ b/machine/finisher.c @@ -0,0 +1,58 @@ +#include <string.h> +#include "finisher.h" +#include "fdt.h" + +volatile uint32_t* finisher; + +void finisher_exit(uint16_t code) +{ + if (!finisher) return; + if (code == 0) { + *finisher = FINISHER_PASS; + } else { + *finisher = code << 16 | FINISHER_FAIL; + } +} + +struct finisher_scan +{ + int compat; + uint64_t reg; +}; + +static void finisher_open(const struct fdt_scan_node *node, void *extra) +{ + struct finisher_scan *scan = (struct finisher_scan *)extra; + memset(scan, 0, sizeof(*scan)); +} + +static void finisher_prop(const struct fdt_scan_prop *prop, void *extra) +{ + struct finisher_scan *scan = (struct finisher_scan *)extra; + if (!strcmp(prop->name, "compatible") && !strcmp((const char*)prop->value, "sifive,test0")) { + scan->compat = 1; + } else if (!strcmp(prop->name, "reg")) { + fdt_get_address(prop->node->parent, prop->value, &scan->reg); + } +} + +static void finisher_done(const struct fdt_scan_node *node, void *extra) +{ + struct finisher_scan *scan = (struct finisher_scan *)extra; + if (!scan->compat || !scan->reg || finisher) return; + finisher = (uint32_t*)(uintptr_t)scan->reg; +} + +void query_finisher(uintptr_t fdt) +{ + struct fdt_cb cb; + struct finisher_scan scan; + + memset(&cb, 0, sizeof(cb)); + cb.open = finisher_open; + cb.prop = finisher_prop; + cb.done = finisher_done; + cb.extra = &scan; + + fdt_scan(fdt, &cb); +} diff --git a/machine/finisher.h b/machine/finisher.h new file mode 100644 index 0000000..60d10b4 --- /dev/null +++ b/machine/finisher.h @@ -0,0 +1,13 @@ +#ifndef _RISCV_FINISHER_H +#define _RISCV_FINISHER_H + +#include <stdint.h> + +#define FINISHER_REG_FINISH 0 +#define FINISHER_FAIL 0x3333 +#define FINISHER_PASS 0x5555 + +void finisher_exit(uint16_t code); +void query_finisher(uintptr_t fdt); + +#endif diff --git a/machine/fp_emulation.c b/machine/fp_emulation.c index 182567a..c83ef73 100644 --- a/machine/fp_emulation.c +++ b/machine/fp_emulation.c @@ -201,7 +201,7 @@ DECLARE_EMULATION_FUNC(emulate_fcvt_fi) { case 0: // int32 negative = (int32_t)uint_val < 0; - uint_val = negative ? -(int32_t)uint_val : (int32_t)uint_val; + uint_val = (uint32_t)(negative ? -uint_val : uint_val); break; case 1: // uint32 uint_val = (uint32_t)uint_val; @@ -331,6 +331,7 @@ DECLARE_EMULATION_FUNC(emulate_fcmp) } return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); success: + SET_FS_DIRTY(); SET_RD(insn, regs, result); } @@ -343,7 +344,7 @@ DECLARE_EMULATION_FUNC(emulate_fmv_if) if (GET_PRECISION(insn) == PRECISION_S) { result = GET_F32_RS1(insn, regs); switch (GET_RM(insn)) { - case GET_RM(MATCH_FMV_X_S): break; + case GET_RM(MATCH_FMV_X_W): break; case GET_RM(MATCH_FCLASS_S): result = f32_classify(result); break; default: return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); } @@ -358,6 +359,7 @@ DECLARE_EMULATION_FUNC(emulate_fmv_if) return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); } + SET_FS_DIRTY(); SET_RD(insn, regs, result); } @@ -365,7 +367,7 @@ DECLARE_EMULATION_FUNC(emulate_fmv_fi) { uintptr_t rs1 = GET_RS1(insn, regs); - if ((insn & MASK_FMV_S_X) == MATCH_FMV_S_X) + if ((insn & MASK_FMV_W_X) == MATCH_FMV_W_X) SET_F32_RD(insn, regs, rs1); #if __riscv_xlen == 64 else if ((insn & MASK_FMV_D_X) == MATCH_FMV_D_X) diff --git a/machine/fp_emulation.h b/machine/fp_emulation.h index c2177a2..97950df 100644 --- a/machine/fp_emulation.h +++ b/machine/fp_emulation.h @@ -10,25 +10,25 @@ #ifdef __riscv_flen # define GET_F32_REG(insn, pos, regs) ({ \ - register int32_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \ + register int32_t value asm("a0") = SHIFT_RIGHT(insn, (pos)-3) & 0xf8; \ uintptr_t tmp; \ asm ("1: auipc %0, %%pcrel_hi(get_f32_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \ value; }) # define SET_F32_REG(insn, pos, regs, val) ({ \ register uint32_t value asm("a0") = (val); \ - uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \ + uintptr_t offset = SHIFT_RIGHT(insn, (pos)-3) & 0xf8; \ uintptr_t tmp; \ asm volatile ("1: auipc %0, %%pcrel_hi(put_f32_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); }) # define init_fp_reg(i) SET_F32_REG((i) << 3, 3, 0, 0) # define GET_F64_REG(insn, pos, regs) ({ \ - register uintptr_t value asm("a0") = ((insn) >> ((pos)-3)) & 0xf8; \ + register uintptr_t value asm("a0") = SHIFT_RIGHT(insn, (pos)-3) & 0xf8; \ uintptr_t tmp; \ asm ("1: auipc %0, %%pcrel_hi(get_f64_reg); add %0, %0, %1; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp), "+&r"(value) :: "t0"); \ sizeof(uintptr_t) == 4 ? *(int64_t*)value : (int64_t)value; }) # define SET_F64_REG(insn, pos, regs, val) ({ \ uint64_t __val = (val); \ register uintptr_t value asm("a0") = sizeof(uintptr_t) == 4 ? (uintptr_t)&__val : (uintptr_t)__val; \ - uintptr_t offset = ((insn) >> ((pos)-3)) & 0xf8; \ + uintptr_t offset = SHIFT_RIGHT(insn, (pos)-3) & 0xf8; \ uintptr_t tmp; \ asm volatile ("1: auipc %0, %%pcrel_hi(put_f64_reg); add %0, %0, %2; jalr t0, %0, %%pcrel_lo(1b)" : "=&r"(tmp) : "r"(value), "r"(offset) : "t0"); }) # define GET_FCSR() read_csr(fcsr) @@ -48,12 +48,12 @@ # define softfloat_roundingMode ({ register int tp asm("tp"); tp; }) # define SET_FS_DIRTY() ((void) 0) #else -# define GET_F64_REG(insn, pos, regs) (*(int64_t*)((void*)((regs) + 32) + (((insn) >> ((pos)-3)) & 0xf8))) +# define GET_F64_REG(insn, pos, regs) (*(int64_t*)((void*)((regs) + 32) + (SHIFT_RIGHT(insn, (pos)-3) & 0xf8))) # define SET_F64_REG(insn, pos, regs, val) (GET_F64_REG(insn, pos, regs) = (val)) # define GET_F32_REG(insn, pos, regs) (*(int32_t*)&GET_F64_REG(insn, pos, regs)) # define SET_F32_REG(insn, pos, regs, val) (GET_F32_REG(insn, pos, regs) = (val)) # define GET_FCSR() ({ register int tp asm("tp"); tp & 0xFF; }) -# define SET_FCSR(value) ({ asm volatile("add tp, x0, %0" :: "rI"((value) & 0xFF)); }) +# define SET_FCSR(value) ({ asm volatile("add tp, x0, %0" :: "rI"((value) & 0xFF)); SET_FS_DIRTY(); }) # define GET_FRM() (GET_FCSR() >> 5) # define SET_FRM(value) SET_FCSR(GET_FFLAGS() | ((value) << 5)) # define GET_FFLAGS() (GET_FCSR() & 0x1F) @@ -79,4 +79,9 @@ #define SET_F32_RD(insn, regs, val) (SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY()) #define SET_F64_RD(insn, regs, val) (SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY()) +#define GET_F32_RS2C(insn, regs) (GET_F32_REG(insn, 2, regs)) +#define GET_F32_RS2S(insn, regs) (GET_F32_REG(RVC_RS2S(insn), 0, regs)) +#define GET_F64_RS2C(insn, regs) (GET_F64_REG(insn, 2, regs)) +#define GET_F64_RS2S(insn, regs) (GET_F64_REG(RVC_RS2S(insn), 0, regs)) + #endif diff --git a/machine/fp_ldst.c b/machine/fp_ldst.c index 93a4844..e11900c 100644 --- a/machine/fp_ldst.c +++ b/machine/fp_ldst.c @@ -1,31 +1,28 @@ #include "fp_emulation.h" #include "unprivileged_memory.h" +#define punt_to_misaligned_handler(align, handler) \ + if (addr % (align) != 0) \ + return write_csr(mbadaddr, addr), (handler)(regs, mcause, mepc) + DECLARE_EMULATION_FUNC(emulate_float_load) { - uint64_t val; uintptr_t addr = GET_RS1(insn, regs) + IMM_I(insn); + + // if FPU is disabled, punt back to the OS + if (unlikely((mstatus & MSTATUS_FS) == 0)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); switch (insn & MASK_FUNCT3) { case MATCH_FLW & MASK_FUNCT3: - if (addr % 4 != 0) - return misaligned_load_trap(regs, mcause, mepc); - + punt_to_misaligned_handler(4, misaligned_load_trap); SET_F32_RD(insn, regs, load_int32_t((void *)addr, mepc)); break; case MATCH_FLD & MASK_FUNCT3: - if (addr % sizeof(uintptr_t) != 0) - return misaligned_load_trap(regs, mcause, mepc); - -#if __riscv_xlen == 64 - val = load_uint64_t((void *)addr, mepc); -#else - val = load_uint32_t((void *)addr, mepc); - val += (uint64_t)load_uint32_t((void *)(addr + 4), mepc) << 32; -#endif - SET_F64_RD(insn, regs, val); + punt_to_misaligned_handler(sizeof(uintptr_t), misaligned_load_trap); + SET_F64_RD(insn, regs, load_uint64_t((void *)addr, mepc)); break; default: @@ -35,29 +32,22 @@ DECLARE_EMULATION_FUNC(emulate_float_load) DECLARE_EMULATION_FUNC(emulate_float_store) { - uint64_t val; uintptr_t addr = GET_RS1(insn, regs) + IMM_S(insn); + + // if FPU is disabled, punt back to the OS + if (unlikely((mstatus & MSTATUS_FS) == 0)) + return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); switch (insn & MASK_FUNCT3) { case MATCH_FSW & MASK_FUNCT3: - if (addr % 4 != 0) - return misaligned_store_trap(regs, mcause, mepc); - + punt_to_misaligned_handler(4, misaligned_store_trap); store_uint32_t((void *)addr, GET_F32_RS2(insn, regs), mepc); break; case MATCH_FSD & MASK_FUNCT3: - if (addr % sizeof(uintptr_t) != 0) - return misaligned_store_trap(regs, mcause, mepc); - - val = GET_F64_RS2(insn, regs); -#if __riscv_xlen == 64 - store_uint64_t((void *)addr, val, mepc); -#else - store_uint32_t((void *)addr, val, mepc); - store_uint32_t((void *)(addr + 4), val >> 32, mepc); -#endif + punt_to_misaligned_handler(sizeof(uintptr_t), misaligned_store_trap); + store_uint64_t((void *)addr, GET_F64_RS2(insn, regs), mepc); break; default: diff --git a/machine/htif.c b/machine/htif.c new file mode 100644 index 0000000..44ec2dd --- /dev/null +++ b/machine/htif.c @@ -0,0 +1,138 @@ +#include "htif.h" +#include "atomic.h" +#include "mtrap.h" +#include "fdt.h" +#include <string.h> + +extern uint64_t __htif_base; +volatile uint64_t tohost __attribute__((section(".htif"))); +volatile uint64_t fromhost __attribute__((section(".htif"))); +volatile int htif_console_buf; +static spinlock_t htif_lock = SPINLOCK_INIT; +uintptr_t htif; + +#define TOHOST(base_int) (uint64_t *)(base_int + TOHOST_OFFSET) +#define FROMHOST(base_int) (uint64_t *)(base_int + FROMHOST_OFFSET) + +#define TOHOST_OFFSET ((uintptr_t)tohost - (uintptr_t)__htif_base) +#define FROMHOST_OFFSET ((uintptr_t)fromhost - (uintptr_t)__htif_base) + +static void __check_fromhost() +{ + uint64_t fh = fromhost; + if (!fh) + return; + fromhost = 0; + + // this should be from the console + assert(FROMHOST_DEV(fh) == 1); + switch (FROMHOST_CMD(fh)) { + case 0: + htif_console_buf = 1 + (uint8_t)FROMHOST_DATA(fh); + break; + case 1: + break; + default: + assert(0); + } +} + +static void __set_tohost(uintptr_t dev, uintptr_t cmd, uintptr_t data) +{ + while (tohost) + __check_fromhost(); + tohost = TOHOST_CMD(dev, cmd, data); +} + +int htif_console_getchar() +{ + spinlock_lock(&htif_lock); + __check_fromhost(); + int ch = htif_console_buf; + if (ch >= 0) { + htif_console_buf = -1; + __set_tohost(1, 0, 0); + } + spinlock_unlock(&htif_lock); + + return ch - 1; +} + +static void do_tohost_fromhost(uintptr_t dev, uintptr_t cmd, uintptr_t data) +{ + spinlock_lock(&htif_lock); + __set_tohost(dev, cmd, data); + + while (1) { + uint64_t fh = fromhost; + if (fh) { + if (FROMHOST_DEV(fh) == dev && FROMHOST_CMD(fh) == cmd) { + fromhost = 0; + break; + } + __check_fromhost(); + } + } + spinlock_unlock(&htif_lock); +} + +void htif_syscall(uintptr_t arg) +{ + do_tohost_fromhost(0, 0, arg); +} + +void htif_console_putchar(uint8_t ch) +{ + spinlock_lock(&htif_lock); + __set_tohost(1, 1, ch); + spinlock_unlock(&htif_lock); +} + +void htif_poweroff() +{ + while (1) { + fromhost = 0; + tohost = 1; + } +} + +struct htif_scan +{ + int compat; +}; + +static void htif_open(const struct fdt_scan_node *node, void *extra) +{ + struct htif_scan *scan = (struct htif_scan *)extra; + memset(scan, 0, sizeof(*scan)); +} + +static void htif_prop(const struct fdt_scan_prop *prop, void *extra) +{ + struct htif_scan *scan = (struct htif_scan *)extra; + if (!strcmp(prop->name, "compatible") && !strcmp((const char*)prop->value, "ucb,htif0")) { + scan->compat = 1; + } +} + +static void htif_done(const struct fdt_scan_node *node, void *extra) +{ + struct htif_scan *scan = (struct htif_scan *)extra; + if (!scan->compat) return; + + htif = 1; +} + +void query_htif(uintptr_t fdt) +{ + struct fdt_cb cb; + struct htif_scan scan; + + memset(&cb, 0, sizeof(cb)); + cb.open = htif_open; + cb.prop = htif_prop; + cb.done = htif_done; + cb.extra = &scan; + + fdt_scan(fdt, &cb); +} diff --git a/machine/htif.h b/machine/htif.h index 7107ddb..a96bf60 100644 --- a/machine/htif.h +++ b/machine/htif.h @@ -15,4 +15,11 @@ #define FROMHOST_CMD(fromhost_value) ((uint64_t)(fromhost_value) << 8 >> 56) #define FROMHOST_DATA(fromhost_value) ((uint64_t)(fromhost_value) << 16 >> 16) +extern uintptr_t htif; +void query_htif(uintptr_t dtb); +void htif_console_putchar(uint8_t); +int htif_console_getchar(); +void htif_poweroff() __attribute__((noreturn)); +void htif_syscall(uintptr_t); + #endif diff --git a/machine/machine.mk.in b/machine/machine.mk.in index 6568dc5..dc8492f 100644 --- a/machine/machine.mk.in +++ b/machine/machine.mk.in @@ -4,30 +4,31 @@ machine_subproject_deps = \ machine_hdrs = \ atomic.h \ bits.h \ - configstring.h \ + fdt.h \ emulation.h \ encoding.h \ fp_emulation.h \ htif.h \ mcall.h \ mtrap.h \ - sbi.h \ + uart.h \ + finisher.h \ unprivileged_memory.h \ vm.h \ machine_c_srcs = \ + fdt.c \ mtrap.c \ minit.c \ + htif.c \ emulation.c \ muldiv_emulation.c \ fp_emulation.c \ fp_ldst.c \ + uart.c \ + finisher.c \ misaligned_ldst.c \ - sbi_impl.c \ - configstring.c \ machine_asm_srcs = \ mentry.S \ fp_asm.S \ - sbi_entry.S \ - sbi.S \ diff --git a/machine/mcall.h b/machine/mcall.h index 2096d16..6a2c037 100644 --- a/machine/mcall.h +++ b/machine/mcall.h @@ -1,21 +1,14 @@ -#ifndef _RISCV_MCALL_H -#define _RISCV_MCALL_H - -#define MCALL_HART_ID 0 -#define MCALL_CONSOLE_PUTCHAR 1 -#define MCALL_CONSOLE_GETCHAR 2 -#define MCALL_HTIF_SYSCALL 3 -#define MCALL_SEND_IPI 4 -#define MCALL_CLEAR_IPI 5 -#define MCALL_SHUTDOWN 6 -#define MCALL_SET_TIMER 7 -#define MCALL_REMOTE_SFENCE_VM 8 -#define MCALL_REMOTE_FENCE_I 9 - -#ifndef __ASSEMBLER__ - -extern uintptr_t do_mcall(uintptr_t which, ...); - -#endif +#ifndef _RISCV_SBI_H +#define _RISCV_SBI_H + +#define SBI_SET_TIMER 0 +#define SBI_CONSOLE_PUTCHAR 1 +#define SBI_CONSOLE_GETCHAR 2 +#define SBI_CLEAR_IPI 3 +#define SBI_SEND_IPI 4 +#define SBI_REMOTE_FENCE_I 5 +#define SBI_REMOTE_SFENCE_VMA 6 +#define SBI_REMOTE_SFENCE_VMA_ASID 7 +#define SBI_SHUTDOWN 8 #endif diff --git a/machine/mentry.S b/machine/mentry.S index 95653c1..11c053c 100644 --- a/machine/mentry.S +++ b/machine/mentry.S @@ -6,24 +6,24 @@ .data .align 6 trap_table: +#define BAD_TRAP_VECTOR 0 .word bad_trap - .word bad_trap + .word pmp_trap .word illegal_insn_trap .word bad_trap .word misaligned_load_trap - .word bad_trap + .word pmp_trap .word misaligned_store_trap - .word bad_trap + .word pmp_trap .word bad_trap .word mcall_trap .word bad_trap .word bad_trap -#define SOFTWARE_INTERRUPT_VECTOR 12 - .word software_interrupt -#define TIMER_INTERRUPT_VECTOR 13 - .word timer_interrupt -#define TRAP_FROM_MACHINE_MODE_VECTOR 14 + .word bad_trap +#define TRAP_FROM_MACHINE_MODE_VECTOR 13 .word __trap_from_machine_mode + .word bad_trap + .word bad_trap .option norvc .section .text.init,"ax",@progbits @@ -31,10 +31,6 @@ trap_table: reset_vector: j do_reset -nmi_vector: -.Lunhandleable_trap: - j bad_trap - trap_vector: csrrw sp, mscratch, sp beqz sp, .Ltrap_from_machine_mode @@ -51,8 +47,12 @@ trap_vector: # Is it a machine timer interrupt? li a0, IRQ_M_TIMER * 2 bne a0, a1, 1f - li a1, TIMER_INTERRUPT_VECTOR - j .Lhandle_trap_in_machine_mode + + # Yes. Simply clear MSIE and raise SSIP. + li a0, MIP_MTIP + csrc mie, a0 + li a0, MIP_STIP + csrs mip, a0 .Lmret: # Go back whence we came. @@ -64,9 +64,35 @@ trap_vector: 1: # Is it an IPI? li a0, IRQ_M_SOFT * 2 - bne a0, a1, .Lunhandleable_trap - li a1, SOFTWARE_INTERRUPT_VECTOR - j .Lhandle_trap_in_machine_mode + bne a0, a1, .Lbad_trap + + # Yes. First, clear the MIPI bit. + LOAD a0, MENTRY_IPI_OFFSET(sp) + sw x0, (a0) + fence + + # Now, decode the cause(s). +#ifdef __riscv_atomic + addi a0, sp, MENTRY_IPI_PENDING_OFFSET + amoswap.w a0, x0, (a0) +#else + lw a0, MENTRY_IPI_PENDING_OFFSET(a0) + sw x0, MENTRY_IPI_PENDING_OFFSET(a0) +#endif + and a1, a0, IPI_SOFT + beqz a1, 1f + csrs mip, MIP_SSIP +1: + andi a1, a0, IPI_FENCE_I + beqz a1, 1f + fence.i +1: + andi a1, a0, IPI_SFENCE_VMA + beqz a1, 1f + sfence.vma +1: + j .Lmret + .Lhandle_trap_in_machine_mode: # Preserve the registers. Compute the address of the trap handler. @@ -166,6 +192,10 @@ restore_regs: li a1, TRAP_FROM_MACHINE_MODE_VECTOR j .Lhandle_trap_in_machine_mode +.Lbad_trap: + li a1, BAD_TRAP_VECTOR + j .Lhandle_trap_in_machine_mode + .globl __redirect_trap __redirect_trap: # reset sp to top of M-mode stack @@ -190,8 +220,9 @@ do_reset: li x7, 0 li x8, 0 li x9, 0 - li x10, 0 - li x11, 0 +// save a0 and a1; arguments from previous boot loader stage: +// li x10, 0 +// li x11, 0 li x12, 0 li x13, 0 li x14, 0 @@ -220,34 +251,52 @@ do_reset: csrr t1, mtvec 1:bne t0, t1, 1b - # sp <- end of first full page after the end of the binary - la sp, _end + 2*RISCV_PGSIZE - 1 - li t0, -RISCV_PGSIZE - and sp, sp, t0 - addi sp, sp, -MENTRY_FRAME_SIZE + la sp, stacks + RISCV_PGSIZE - MENTRY_FRAME_SIZE - csrr a0, mhartid - slli a1, a0, RISCV_PGSHIFT - add sp, sp, a1 + csrr a3, mhartid + slli a2, a3, RISCV_PGSHIFT + add sp, sp, a2 - beqz a0, init_first_hart + # Boot on the first unmasked hart + la a4, disabled_hart_mask + LOAD a4, 0(a4) + addi a5, a4, 1 + not a4, a4 + and a4, a4, a5 + srl a4, a4, a3 + andi a4, a4, 1 + bnez a4, init_first_hart # set MSIE bit to receive IPI - li a1, MIP_MSIP - csrw mie, a1 + li a2, MIP_MSIP + csrw mie, a2 .LmultiHart: #if MAX_HARTS > 1 # wait for an IPI to signal that it's safe to boot wfi - csrr a1, mip - andi a1, a1, MIP_MSIP - beqz a1, .LmultiHart + + # masked harts never start + la a4, disabled_hart_mask + LOAD a4, 0(a4) + srl a4, a4, a3 + andi a4, a4, 1 + bnez a4, .LmultiHart + + # only start if mip is set + csrr a2, mip + andi a2, a2, MIP_MSIP + beqz a2, .LmultiHart # make sure our hart id is within a valid range fence - li a1, MAX_HARTS - bltu a0, a1, init_other_hart + li a2, MAX_HARTS + bltu a3, a2, init_other_hart #endif wfi j .LmultiHart + + .bss + .align RISCV_PGSHIFT +stacks: + .skip RISCV_PGSIZE * MAX_HARTS diff --git a/machine/minit.c b/machine/minit.c index 54ff88e..456de60 100644 --- a/machine/minit.c +++ b/machine/minit.c @@ -2,46 +2,54 @@ #include "atomic.h" #include "vm.h" #include "fp_emulation.h" +#include "fdt.h" +#include "uart.h" +#include "finisher.h" +#include "disabled_hart_mask.h" +#include "htif.h" #include <string.h> #include <limits.h> pte_t* root_page_table; -uintptr_t first_free_paddr; uintptr_t mem_size; -uintptr_t num_harts; volatile uint64_t* mtime; volatile uint32_t* plic_priorities; size_t plic_ndevs; static void mstatus_init() { - // Enable FPU and set VM mode - uintptr_t ms = 0; - ms = INSERT_FIELD(ms, MSTATUS_VM, VM_CHOICE); - ms = INSERT_FIELD(ms, MSTATUS_FS, 1); - ms = INSERT_FIELD(ms, MSTATUS_XS, 1); - write_csr(mstatus, ms); - - // Make sure the hart actually supports the VM mode we want - ms = read_csr(mstatus); - assert(EXTRACT_FIELD(ms, MSTATUS_VM) == VM_CHOICE); + // Enable FPU + if (supports_extension('D') || supports_extension('F')) + write_csr(mstatus, MSTATUS_FS | MSTATUS_XS); + else + write_csr(mstatus, MSTATUS_XS); // Enable user/supervisor use of perf counters - write_csr(mucounteren, -1); - write_csr(mscounteren, -1); - write_csr(mie, ~MIP_MTIP); // disable timer; enable other interrupts + if (supports_extension('S')) + write_csr(scounteren, -1); + write_csr(mcounteren, -1); + + // Enable software interrupts + write_csr(mie, MIP_MSIP); + + // Disable paging + if (supports_extension('S')) + write_csr(sptbr, 0); } // send S-mode interrupts and most exceptions straight to S-mode static void delegate_traps() { + if (!supports_extension('S')) + return; + uintptr_t interrupts = MIP_SSIP | MIP_STIP | MIP_SEIP; uintptr_t exceptions = (1U << CAUSE_MISALIGNED_FETCH) | - (1U << CAUSE_FAULT_FETCH) | + (1U << CAUSE_FETCH_PAGE_FAULT) | (1U << CAUSE_BREAKPOINT) | - (1U << CAUSE_FAULT_LOAD) | - (1U << CAUSE_FAULT_STORE) | + (1U << CAUSE_LOAD_PAGE_FAULT) | + (1U << CAUSE_STORE_PAGE_FAULT) | (1U << CAUSE_BREAKPOINT) | (1U << CAUSE_USER_ECALL); @@ -53,11 +61,12 @@ static void delegate_traps() static void fp_init() { + if (!supports_extension('D') && !supports_extension('F')) + return; + assert(read_csr(mstatus) & MSTATUS_FS); #ifdef __riscv_flen - if (!supports_extension('D')) - die("FPU not found; recompile pk with -msoft-float"); for (int i = 0; i < 32; i++) init_fp_reg(i); write_csr(fcsr, 0); @@ -75,16 +84,9 @@ hls_t* hls_init(uintptr_t id) return hls; } -static uintptr_t sbi_top_paddr() -{ - extern char _end; - return ROUNDUP((uintptr_t)&_end, RISCV_PGSIZE); -} - static void memory_init() { mem_size = mem_size / MEGAPAGE_SIZE * MEGAPAGE_SIZE; - first_free_paddr = sbi_top_paddr() + num_harts * RISCV_PGSIZE; } static void hart_init() @@ -130,34 +132,68 @@ static void hart_plic_init() *HLS()->plic_s_thresh = 0; } -void init_first_hart() +static void wake_harts() { + for (int hart = 0; hart < MAX_HARTS; ++hart) + if ((((~disabled_hart_mask & hart_mask) >> hart) & 1)) + *OTHER_HLS(hart)->ipi = 1; // wakeup the hart +} + +void init_first_hart(uintptr_t hartid, uintptr_t dtb) +{ + // Confirm console as early as possible + query_uart(dtb); + query_htif(dtb); + hart_init(); hls_init(0); // this might get called again from parse_config_string - parse_config_string(); + + // Find the power button early as well so die() works + query_finisher(dtb); + + query_mem(dtb); + query_harts(dtb); + query_clint(dtb); + query_plic(dtb); + + wake_harts(); + plic_init(); hart_plic_init(); //prci_test(); memory_init(); - boot_loader(); + boot_loader(dtb); } -void init_other_hart() +void init_other_hart(uintptr_t hartid, uintptr_t dtb) { hart_init(); hart_plic_init(); - boot_other_hart(); + boot_other_hart(dtb); } -void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t stack) +void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t arg0, uintptr_t arg1) { + // Set up a PMP to permit access to all of memory. + // Ignore the illegal-instruction trap if PMPs aren't supported. + uintptr_t pmpc = PMP_NAPOT | PMP_R | PMP_W | PMP_X; + asm volatile ("la t0, 1f\n\t" + "csrrw t0, mtvec, t0\n\t" + "csrw pmpaddr0, %1\n\t" + "csrw pmpcfg0, %0\n\t" + ".align 2\n\t" + "1: csrw mtvec, t0" + : : "r" (pmpc), "r" (-1UL) : "t0"); + uintptr_t mstatus = read_csr(mstatus); mstatus = INSERT_FIELD(mstatus, MSTATUS_MPP, PRV_S); mstatus = INSERT_FIELD(mstatus, MSTATUS_MPIE, 0); write_csr(mstatus, mstatus); write_csr(mscratch, MACHINE_STACK_TOP() - MENTRY_FRAME_SIZE); write_csr(mepc, fn); - write_csr(sptbr, (uintptr_t)root_page_table >> RISCV_PGSHIFT); - asm volatile ("mv a0, %0; mv sp, %0; mret" : : "r" (stack)); + + register uintptr_t a0 asm ("a0") = arg0; + register uintptr_t a1 asm ("a1") = arg1; + asm volatile ("mret" : : "r" (a0), "r" (a1)); __builtin_unreachable(); } diff --git a/machine/misaligned_ldst.c b/machine/misaligned_ldst.c index 980aeba..2fccd50 100644 --- a/machine/misaligned_ldst.c +++ b/machine/misaligned_ldst.c @@ -2,6 +2,8 @@ #include "fp_emulation.h" #include "unprivileged_memory.h" #include "mtrap.h" +#include "config.h" +#include "pk.h" union byte_array { uint8_t bytes[8]; @@ -14,6 +16,7 @@ void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) union byte_array val; uintptr_t mstatus; insn_t insn = get_insn(mepc, &mstatus); + uintptr_t npc = mepc + insn_len(insn); uintptr_t addr = read_csr(mbadaddr); int shift = 0, fp = 0, len; @@ -35,6 +38,30 @@ void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) len = 2, shift = 8*(sizeof(uintptr_t) - len); else if ((insn & MASK_LHU) == MATCH_LHU) len = 2; +#ifdef __riscv_compressed +# if __riscv_xlen >= 64 + else if ((insn & MASK_C_LD) == MATCH_C_LD) + len = 8, shift = 8*(sizeof(uintptr_t) - len), insn = RVC_RS2S(insn) << SH_RD; + else if ((insn & MASK_C_LDSP) == MATCH_C_LDSP && ((insn >> SH_RD) & 0x1f)) + len = 8, shift = 8*(sizeof(uintptr_t) - len); +# endif + else if ((insn & MASK_C_LW) == MATCH_C_LW) + len = 4, shift = 8*(sizeof(uintptr_t) - len), insn = RVC_RS2S(insn) << SH_RD; + else if ((insn & MASK_C_LWSP) == MATCH_C_LWSP && ((insn >> SH_RD) & 0x1f)) + len = 4, shift = 8*(sizeof(uintptr_t) - len); +# ifdef PK_ENABLE_FP_EMULATION + else if ((insn & MASK_C_FLD) == MATCH_C_FLD) + fp = 1, len = 8, insn = RVC_RS2S(insn) << SH_RD; + else if ((insn & MASK_C_FLDSP) == MATCH_C_FLDSP) + fp = 1, len = 8; +# if __riscv_xlen == 32 + else if ((insn & MASK_C_FLW) == MATCH_C_FLW) + fp = 1, len = 4, insn = RVC_RS2S(insn) << SH_RD; + else if ((insn & MASK_C_FLWSP) == MATCH_C_FLWSP) + fp = 1, len = 4; +# endif +# endif +#endif else return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); @@ -49,7 +76,7 @@ void misaligned_load_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) else SET_F32_RD(insn, regs, val.intx); - write_csr(mepc, mepc + 4); + write_csr(mepc, npc); } void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) @@ -57,6 +84,7 @@ void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) union byte_array val; uintptr_t mstatus; insn_t insn = get_insn(mepc, &mstatus); + uintptr_t npc = mepc + insn_len(insn); int len; val.intx = GET_RS2(insn, regs); @@ -74,6 +102,30 @@ void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) #endif else if ((insn & MASK_SH) == MATCH_SH) len = 2; +#ifdef __riscv_compressed +# if __riscv_xlen >= 64 + else if ((insn & MASK_C_SD) == MATCH_C_SD) + len = 8, val.intx = GET_RS2S(insn, regs); + else if ((insn & MASK_C_SDSP) == MATCH_C_SDSP && ((insn >> SH_RD) & 0x1f)) + len = 8, val.intx = GET_RS2C(insn, regs); +# endif + else if ((insn & MASK_C_SW) == MATCH_C_SW) + len = 4, val.intx = GET_RS2S(insn, regs); + else if ((insn & MASK_C_SWSP) == MATCH_C_SWSP && ((insn >> SH_RD) & 0x1f)) + len = 4, val.intx = GET_RS2C(insn, regs); +# ifdef PK_ENABLE_FP_EMULATION + else if ((insn & MASK_C_FSD) == MATCH_C_FSD) + len = 8, val.int64 = GET_F64_RS2S(insn, regs); + else if ((insn & MASK_C_FSDSP) == MATCH_C_FSDSP) + len = 8, val.int64 = GET_F64_RS2C(insn, regs); +# if __riscv_xlen == 32 + else if ((insn & MASK_C_FSW) == MATCH_C_FSW) + len = 4, val.intx = GET_F32_RS2S(insn, regs); + else if ((insn & MASK_C_FSWSP) == MATCH_C_FSWSP) + len = 4, val.intx = GET_F32_RS2C(insn, regs); +# endif +# endif +#endif else return truly_illegal_insn(regs, mcause, mepc, mstatus, insn); @@ -81,5 +133,5 @@ void misaligned_store_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) for (int i = 0; i < len; i++) store_uint8_t((void *)(addr + i), val.bytes[i], mepc); - write_csr(mepc, mepc + 4); + write_csr(mepc, npc); } diff --git a/machine/mtrap.c b/machine/mtrap.c index 07f7d10..0f77def 100644 --- a/machine/mtrap.c +++ b/machine/mtrap.c @@ -3,99 +3,40 @@ #include "htif.h" #include "atomic.h" #include "bits.h" +#include "vm.h" +#include "uart.h" +#include "finisher.h" +#include "fdt.h" +#include "unprivileged_memory.h" +#include "disabled_hart_mask.h" #include <errno.h> #include <stdarg.h> #include <stdio.h> -volatile uint64_t tohost __attribute__((aligned(64))) __attribute__((section("htif"))); -volatile uint64_t fromhost __attribute__((aligned(64))) __attribute__((section("htif"))); -static spinlock_t htif_lock = SPINLOCK_INIT; - -void __attribute__((noreturn)) bad_trap() -{ - die("machine mode: unhandlable trap %d @ %p", read_csr(mcause), read_csr(mepc)); -} - -static uintptr_t mcall_hart_id() -{ - return read_const_csr(mhartid); -} - -static void request_htif_keyboard_interrupt() -{ - assert(tohost == 0); - tohost = TOHOST_CMD(1, 0, 0); -} - -static void __htif_interrupt() -{ - // we should only be interrupted by keypresses - uint64_t fh = fromhost; - if (!fh) - return; - if (!(FROMHOST_DEV(fh) == 1 && FROMHOST_CMD(fh) == 0)) - die("unexpected htif interrupt"); - HLS()->console_ibuf = 1 + (uint8_t)FROMHOST_DATA(fh); - fromhost = 0; - set_csr(mip, MIP_SSIP); -} - -static void do_tohost_fromhost(uintptr_t dev, uintptr_t cmd, uintptr_t data) -{ - spinlock_lock(&htif_lock); - while (tohost) - __htif_interrupt(); - tohost = TOHOST_CMD(dev, cmd, data); - - while (1) { - uint64_t fh = fromhost; - if (fh) { - if (FROMHOST_DEV(fh) == dev && FROMHOST_CMD(fh) == cmd) { - fromhost = 0; - break; - } - __htif_interrupt(); - } - } - spinlock_unlock(&htif_lock); -} - -static void htif_interrupt() -{ - if (spinlock_trylock(&htif_lock) == 0) { - __htif_interrupt(); - spinlock_unlock(&htif_lock); - } -} - -uintptr_t timer_interrupt() +void __attribute__((noreturn)) bad_trap(uintptr_t* regs, uintptr_t dummy, uintptr_t mepc) { - // just send the timer interrupt to the supervisor - clear_csr(mie, MIP_MTIP); - set_csr(mip, MIP_STIP); - - // and poll the HTIF console - htif_interrupt(); - - return 0; + die("machine mode: unhandlable trap %d @ %p", read_csr(mcause), mepc); } static uintptr_t mcall_console_putchar(uint8_t ch) { - do_tohost_fromhost(1, 1, ch); - return 0; -} - -static uintptr_t mcall_htif_syscall(uintptr_t magic_mem) -{ - do_tohost_fromhost(0, 0, magic_mem); + if (uart) { + uart_putchar(ch); + } else if (htif) { + htif_console_putchar(ch); + } return 0; } -void poweroff() +void poweroff(uint16_t code) { - while (1) - tohost = 1; + printm("Power off\n"); + finisher_exit(code); + if (htif) { + htif_poweroff(); + } else { + while (1); + } } void putstring(const char* s) @@ -104,63 +45,49 @@ void putstring(const char* s) mcall_console_putchar(*s++); } -void printm(const char* s, ...) +void vprintm(const char* s, va_list vl) { char buf[256]; - va_list vl; - - va_start(vl, s); vsnprintf(buf, sizeof buf, s, vl); - va_end(vl); - putstring(buf); } -static void send_ipi(uintptr_t recipient, int event) -{ - if ((atomic_or(&OTHER_HLS(recipient)->mipi_pending, event) & event) == 0) { - mb(); - *OTHER_HLS(recipient)->ipi = 1; - } -} - -static uintptr_t mcall_send_ipi(uintptr_t recipient) +void printm(const char* s, ...) { - if (recipient >= num_harts) - return -1; + va_list vl; - send_ipi(recipient, IPI_SOFT); - return 0; + va_start(vl, s); + vprintm(s, vl); + va_end(vl); } -static void reset_ssip() +static void send_ipi(uintptr_t recipient, int event) { - clear_csr(mip, MIP_SSIP); + if (((disabled_hart_mask >> recipient) & 1)) return; + atomic_or(&OTHER_HLS(recipient)->mipi_pending, event); mb(); - - if (HLS()->sipi_pending || HLS()->console_ibuf > 0) - set_csr(mip, MIP_SSIP); + *OTHER_HLS(recipient)->ipi = 1; } static uintptr_t mcall_console_getchar() { - int ch = atomic_swap(&HLS()->console_ibuf, -1); - if (ch >= 0) - request_htif_keyboard_interrupt(); - reset_ssip(); - return ch - 1; + if (uart) { + return uart_getchar(); + } else if (htif) { + return htif_console_getchar(); + } else { + return '\0'; + } } static uintptr_t mcall_clear_ipi() { - int ipi = atomic_swap(&HLS()->sipi_pending, 0); - reset_ssip(); - return ipi; + return clear_csr(mip, MIP_SSIP) & MIP_SSIP; } static uintptr_t mcall_shutdown() { - poweroff(); + poweroff(0); } static uintptr_t mcall_set_timer(uint64_t when) @@ -171,114 +98,93 @@ static uintptr_t mcall_set_timer(uint64_t when) return 0; } -void software_interrupt() -{ - *HLS()->ipi = 0; - mb(); - int ipi_pending = atomic_swap(&HLS()->mipi_pending, 0); - - if (ipi_pending & IPI_SOFT) { - HLS()->sipi_pending = 1; - set_csr(mip, MIP_SSIP); - } - - if (ipi_pending & IPI_FENCE_I) - asm volatile ("fence.i"); - - if (ipi_pending & IPI_SFENCE_VM) - asm volatile ("sfence.vm"); -} - static void send_ipi_many(uintptr_t* pmask, int event) { _Static_assert(MAX_HARTS <= 8 * sizeof(*pmask), "# harts > uintptr_t bits"); - uintptr_t mask = -1; + uintptr_t mask = hart_mask; if (pmask) - mask = *pmask; + mask &= load_uintptr_t(pmask, read_csr(mepc)); // send IPIs to everyone - for (ssize_t i = num_harts-1; i >= 0; i--) - if ((mask >> i) & 1) + for (uintptr_t i = 0, m = mask; m; i++, m >>= 1) + if (m & 1) send_ipi(i, event); + if (event == IPI_SOFT) + return; + // wait until all events have been handled. - // prevent deadlock while spinning by handling any IPIs from other harts. - for (ssize_t i = num_harts-1; i >= 0; i--) - if ((mask >> i) & 1) - while (OTHER_HLS(i)->mipi_pending & event) - software_interrupt(); + // prevent deadlock by consuming incoming IPIs. + uint32_t incoming_ipi = 0; + for (uintptr_t i = 0, m = mask; m; i++, m >>= 1) + if (m & 1) + while (*OTHER_HLS(i)->ipi) + incoming_ipi |= atomic_swap(HLS()->ipi, 0); + + // if we got an IPI, restore it; it will be taken after returning + if (incoming_ipi) { + *HLS()->ipi = incoming_ipi; + mb(); + } } -static uintptr_t mcall_remote_sfence_vm(uintptr_t* hart_mask, uintptr_t asid) +void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) { - // ignore the ASID and do a global flush. - // this allows us to avoid queueing a message. - send_ipi_many(hart_mask, IPI_SFENCE_VM); - return 0; -} + write_csr(mepc, mepc + 4); -static uintptr_t mcall_remote_fence_i(uintptr_t* hart_mask) -{ - send_ipi_many(hart_mask, IPI_FENCE_I); - return 0; -} + uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval, ipi_type; -void mcall_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) -{ - uintptr_t n = regs[17], arg0 = regs[10], arg1 = regs[11], retval; switch (n) { - case MCALL_HART_ID: - retval = mcall_hart_id(); - break; - case MCALL_CONSOLE_PUTCHAR: + case SBI_CONSOLE_PUTCHAR: retval = mcall_console_putchar(arg0); break; - case MCALL_CONSOLE_GETCHAR: + case SBI_CONSOLE_GETCHAR: retval = mcall_console_getchar(); break; - case MCALL_HTIF_SYSCALL: - retval = mcall_htif_syscall(arg0); + case SBI_SEND_IPI: + ipi_type = IPI_SOFT; + goto send_ipi; + case SBI_REMOTE_SFENCE_VMA: + case SBI_REMOTE_SFENCE_VMA_ASID: + ipi_type = IPI_SFENCE_VMA; + goto send_ipi; + case SBI_REMOTE_FENCE_I: + ipi_type = IPI_FENCE_I; +send_ipi: + send_ipi_many((uintptr_t*)arg0, ipi_type); + retval = 0; break; - case MCALL_SEND_IPI: - retval = mcall_send_ipi(arg0); - break; - case MCALL_CLEAR_IPI: + case SBI_CLEAR_IPI: retval = mcall_clear_ipi(); break; - case MCALL_SHUTDOWN: + case SBI_SHUTDOWN: retval = mcall_shutdown(); break; - case MCALL_SET_TIMER: + case SBI_SET_TIMER: #if __riscv_xlen == 32 retval = mcall_set_timer(arg0 + ((uint64_t)arg1 << 32)); #else retval = mcall_set_timer(arg0); #endif break; - case MCALL_REMOTE_SFENCE_VM: - retval = mcall_remote_sfence_vm((uintptr_t*)arg0, arg1); - break; - case MCALL_REMOTE_FENCE_I: - retval = mcall_remote_fence_i((uintptr_t*)arg0); - break; default: retval = -ENOSYS; break; } regs[10] = retval; - write_csr(mepc, mepc + 4); } -void redirect_trap(uintptr_t epc, uintptr_t mstatus) +void redirect_trap(uintptr_t epc, uintptr_t mstatus, uintptr_t badaddr) { + write_csr(sbadaddr, badaddr); write_csr(sepc, epc); write_csr(scause, read_csr(mcause)); write_csr(mepc, read_csr(stvec)); - uintptr_t new_mstatus = mstatus & ~(MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_MPIE); + uintptr_t new_mstatus = mstatus & ~(MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE); uintptr_t mpp_s = MSTATUS_MPP & (MSTATUS_MPP >> 1); - new_mstatus |= (mstatus / (MSTATUS_MPIE / MSTATUS_SPIE)) & MSTATUS_SPIE; + new_mstatus |= (mstatus * (MSTATUS_SPIE / MSTATUS_SIE)) & MSTATUS_SPIE; new_mstatus |= (mstatus / (mpp_s / MSTATUS_SPP)) & MSTATUS_SPP; new_mstatus |= mpp_s; write_csr(mstatus, new_mstatus); @@ -287,15 +193,19 @@ void redirect_trap(uintptr_t epc, uintptr_t mstatus) return __redirect_trap(); } -static void machine_page_fault(uintptr_t* regs, uintptr_t mepc) +void pmp_trap(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc) +{ + redirect_trap(mepc, read_csr(mstatus), read_csr(mbadaddr)); +} + +static void machine_page_fault(uintptr_t* regs, uintptr_t dummy, uintptr_t mepc) { // MPRV=1 iff this trap occurred while emulating an instruction on behalf // of a lower privilege level. In that case, a2=epc and a3=mstatus. if (read_csr(mstatus) & MSTATUS_MPRV) { - write_csr(sbadaddr, read_csr(mbadaddr)); - return redirect_trap(regs[12], regs[13]); + return redirect_trap(regs[12], regs[13], read_csr(mbadaddr)); } - bad_trap(); + bad_trap(regs, dummy, mepc); } void trap_from_machine_mode(uintptr_t* regs, uintptr_t dummy, uintptr_t mepc) @@ -304,10 +214,13 @@ void trap_from_machine_mode(uintptr_t* regs, uintptr_t dummy, uintptr_t mepc) switch (mcause) { - case CAUSE_FAULT_LOAD: - case CAUSE_FAULT_STORE: - return machine_page_fault(regs, mepc); + case CAUSE_LOAD_PAGE_FAULT: + case CAUSE_STORE_PAGE_FAULT: + case CAUSE_FETCH_ACCESS: + case CAUSE_LOAD_ACCESS: + case CAUSE_STORE_ACCESS: + return machine_page_fault(regs, dummy, mepc); default: - bad_trap(); + bad_trap(regs, dummy, mepc); } } diff --git a/machine/mtrap.h b/machine/mtrap.h index 9995203..eafdb14 100644 --- a/machine/mtrap.h +++ b/machine/mtrap.h @@ -11,9 +11,9 @@ #ifndef __ASSEMBLER__ -#include "sbi.h" #include <stdint.h> #include <stddef.h> +#include <stdarg.h> #define read_const_csr(reg) ({ unsigned long __tmp; \ asm ("csrr %0, " #reg : "=r"(__tmp)); \ @@ -29,19 +29,16 @@ static inline int xlen() return read_const_csr(misa) < 0 ? 64 : 32; } -extern uintptr_t first_free_paddr; extern uintptr_t mem_size; -extern uintptr_t num_harts; extern volatile uint64_t* mtime; extern volatile uint32_t* plic_priorities; extern size_t plic_ndevs; typedef struct { - uint64_t* timecmp; - uint32_t* ipi; + volatile uint32_t* ipi; volatile int mipi_pending; - volatile int sipi_pending; - int console_ibuf; + + volatile uint64_t* timecmp; volatile uint32_t* plic_m_thresh; volatile uintptr_t* plic_m_ie; @@ -49,10 +46,6 @@ typedef struct { volatile uintptr_t* plic_s_ie; } hls_t; -#define IPI_SOFT 0x1 -#define IPI_FENCE_I 0x2 -#define IPI_SFENCE_VM 0x4 - #define MACHINE_STACK_TOP() ({ \ register uintptr_t sp asm ("sp"); \ (void*)((sp + RISCV_PGSIZE) & -RISCV_PGSIZE); }) @@ -63,16 +56,17 @@ typedef struct { hls_t* hls_init(uintptr_t hart_id); void parse_config_string(); -void poweroff(void) __attribute((noreturn)); +void poweroff(uint16_t code) __attribute((noreturn)); void printm(const char* s, ...); +void vprintm(const char *s, va_list args); void putstring(const char* s); #define assert(x) ({ if (!(x)) die("assertion failed: %s", #x); }) -#define die(str, ...) ({ printm("%s:%d: " str "\n", __FILE__, __LINE__, ##__VA_ARGS__); poweroff(); }) +#define die(str, ...) ({ printm("%s:%d: " str "\n", __FILE__, __LINE__, ##__VA_ARGS__); poweroff(-1); }) -void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t stack) +void enter_supervisor_mode(void (*fn)(uintptr_t), uintptr_t arg0, uintptr_t arg1) __attribute__((noreturn)); -void boot_loader(); -void boot_other_hart(); +void boot_loader(uintptr_t dtb); +void boot_other_hart(uintptr_t dtb); static inline void wfi() { @@ -81,9 +75,15 @@ static inline void wfi() #endif // !__ASSEMBLER__ +#define IPI_SOFT 0x1 +#define IPI_FENCE_I 0x2 +#define IPI_SFENCE_VMA 0x4 + #define MACHINE_STACK_SIZE RISCV_PGSIZE -#define MENTRY_FRAME_SIZE (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE \ - + HLS_SIZE) +#define MENTRY_HLS_OFFSET (INTEGER_CONTEXT_SIZE + SOFT_FLOAT_CONTEXT_SIZE) +#define MENTRY_FRAME_SIZE (MENTRY_HLS_OFFSET + HLS_SIZE) +#define MENTRY_IPI_OFFSET (MENTRY_HLS_OFFSET) +#define MENTRY_IPI_PENDING_OFFSET (MENTRY_HLS_OFFSET + REGBYTES) #ifdef __riscv_flen # define SOFT_FLOAT_CONTEXT_SIZE 0 diff --git a/machine/sbi.S b/machine/sbi.S deleted file mode 100644 index cbea78a..0000000 --- a/machine/sbi.S +++ /dev/null @@ -1,15 +0,0 @@ -.globl sbi_hart_id; sbi_hart_id = -2048 -.globl sbi_num_harts; sbi_num_harts = -2032 -.globl sbi_query_memory; sbi_query_memory = -2016 -.globl sbi_console_putchar; sbi_console_putchar = -2000 -.globl sbi_console_getchar; sbi_console_getchar = -1984 -.globl sbi_send_ipi; sbi_send_ipi = -1952 -.globl sbi_clear_ipi; sbi_clear_ipi = -1936 -.globl sbi_timebase; sbi_timebase = -1920 -.globl sbi_shutdown; sbi_shutdown = -1904 -.globl sbi_set_timer; sbi_set_timer = -1888 -.globl sbi_mask_interrupt; sbi_mask_interrupt = -1872 -.globl sbi_unmask_interrupt; sbi_unmask_interrupt = -1856 -.globl sbi_remote_sfence_vm; sbi_remote_sfence_vm = -1840 -.globl sbi_remote_sfence_vm_range; sbi_remote_sfence_vm_range = -1824 -.globl sbi_remote_fence_i; sbi_remote_fence_i = -1808 diff --git a/machine/sbi.h b/machine/sbi.h deleted file mode 100644 index 4e2fbd8..0000000 --- a/machine/sbi.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef _ASM_RISCV_SBI_H -#define _ASM_RISCV_SBI_H - -typedef struct { - unsigned long base; - unsigned long size; - unsigned long node_id; -} memory_block_info; - -unsigned long sbi_query_memory(unsigned long id, memory_block_info *p); - -unsigned long sbi_hart_id(void); -unsigned long sbi_num_harts(void); -unsigned long sbi_timebase(void); -void sbi_set_timer(unsigned long long stime_value); -void sbi_send_ipi(unsigned long hart_id); -unsigned long sbi_clear_ipi(void); -void sbi_shutdown(void); - -void sbi_console_putchar(unsigned char ch); -int sbi_console_getchar(void); - -void sbi_remote_sfence_vm(unsigned long hart_mask_ptr, unsigned long asid); -void sbi_remote_sfence_vm_range(unsigned long hart_mask_ptr, unsigned long asid, unsigned long start, unsigned long size); -void sbi_remote_fence_i(unsigned long hart_mask_ptr); - -unsigned long sbi_mask_interrupt(unsigned long which); -unsigned long sbi_unmask_interrupt(unsigned long which); - -#endif diff --git a/machine/sbi_entry.S b/machine/sbi_entry.S deleted file mode 100644 index a37dd25..0000000 --- a/machine/sbi_entry.S +++ /dev/null @@ -1,111 +0,0 @@ -#include "encoding.h" -#include "mcall.h" - - .section .sbi,"ax",@progbits - .option norvc - .align RISCV_PGSHIFT - .globl sbi_base -sbi_base: - - # TODO: figure out something better to do with this space. It's not - # protected from the OS, so beware. - .skip RISCV_PGSIZE - 2048 - - # hart_id - .align 4 - li a7, MCALL_HART_ID - ecall - ret - - # num_harts - .align 4 - lw a0, num_harts - ret - - # query_memory - .align 4 - tail __sbi_query_memory - - # console_putchar - .align 4 - li a7, MCALL_CONSOLE_PUTCHAR - ecall - ret - - # console_getchar - .align 4 - li a7, MCALL_CONSOLE_GETCHAR - ecall - ret - - # empty - .align 4 - unimp - - # send ipi - .align 4 - li a7, MCALL_SEND_IPI - ecall - ret - - # clear ipi - .align 4 - li a7, MCALL_CLEAR_IPI - ecall - ret - - # timebase - .align 4 - li a0, 10000000 # or, you know, we could provide the correct answer - ret - - # shutdown - .align 4 - li a7, MCALL_SHUTDOWN - ecall - - # set_timer - .align 4 - li a7, MCALL_SET_TIMER - ecall - ret - - # mask_interrupt - .align 4 - tail __sbi_mask_interrupt - - # unmask_interrupt - .align 4 - tail __sbi_unmask_interrupt - - # remote_sfence_vm - .align 4 - li a7, MCALL_REMOTE_SFENCE_VM - ecall - ret - - # remote_sfence_vm_range - .align 4 - li a7, MCALL_REMOTE_SFENCE_VM - ecall - ret - - # remote_fence_i - .align 4 - li a7, MCALL_REMOTE_FENCE_I - ecall - ret - - # end of SBI trampolines - - .globl do_mcall -do_mcall: - mv a7, a0 - mv a0, a1 - mv a1, a2 - ecall - ret - - .align RISCV_PGSHIFT - .globl _sbi_end -_sbi_end: diff --git a/machine/sbi_impl.c b/machine/sbi_impl.c deleted file mode 100644 index f5ed8c9..0000000 --- a/machine/sbi_impl.c +++ /dev/null @@ -1,33 +0,0 @@ -#include "mtrap.h" -#include "sbi.h" - -uintptr_t __sbi_query_memory(uintptr_t id, memory_block_info *p) -{ - if (id == 0) { - p->base = first_free_paddr; - p->size = mem_size + DRAM_BASE - p->base; - return 0; - } - - return -1; -} - -#define LOW_IRQ_OK(n) ((n) == IRQ_S_SOFT || (n) == IRQ_S_TIMER) - -uintptr_t __sbi_mask_interrupt(uintptr_t which) -{ - if (!LOW_IRQ_OK(which)) - return -1; - - clear_csr(sie, 1UL << which); - return 0; -} - -uintptr_t __sbi_unmask_interrupt(uintptr_t which) -{ - if (!LOW_IRQ_OK(which)) - return -1; - - set_csr(sie, 1UL << which); - return 0; -} diff --git a/machine/uart.c b/machine/uart.c new file mode 100644 index 0000000..0645500 --- /dev/null +++ b/machine/uart.c @@ -0,0 +1,76 @@ +#include <string.h> +#include "uart.h" +#include "fdt.h" + +volatile uint32_t* uart; + +void uart_putchar(uint8_t ch) +{ +#ifdef __riscv_atomic + int32_t r; + do { + __asm__ __volatile__ ( + "amoor.w %0, %2, %1\n" + : "=r" (r), "+A" (uart[UART_REG_TXFIFO]) + : "r" (ch)); + } while (r < 0); +#else + volatile uint32_t *tx = uart + UART_REG_TXFIFO; + while ((int32_t)(*tx) < 0); + *tx = ch; +#endif +} + +int uart_getchar() +{ + int32_t ch = uart[UART_REG_RXFIFO]; + if (ch < 0) return -1; + return ch; +} + +struct uart_scan +{ + int compat; + uint64_t reg; +}; + +static void uart_open(const struct fdt_scan_node *node, void *extra) +{ + struct uart_scan *scan = (struct uart_scan *)extra; + memset(scan, 0, sizeof(*scan)); +} + +static void uart_prop(const struct fdt_scan_prop *prop, void *extra) +{ + struct uart_scan *scan = (struct uart_scan *)extra; + if (!strcmp(prop->name, "compatible") && !strcmp((const char*)prop->value, "sifive,uart0")) { + scan->compat = 1; + } else if (!strcmp(prop->name, "reg")) { + fdt_get_address(prop->node->parent, prop->value, &scan->reg); + } +} + +static void uart_done(const struct fdt_scan_node *node, void *extra) +{ + struct uart_scan *scan = (struct uart_scan *)extra; + if (!scan->compat || !scan->reg || uart) return; + + // Enable Rx/Tx channels + uart = (void*)(uintptr_t)scan->reg; + uart[UART_REG_TXCTRL] = UART_TXEN; + uart[UART_REG_RXCTRL] = UART_RXEN; +} + +void query_uart(uintptr_t fdt) +{ + struct fdt_cb cb; + struct uart_scan scan; + + memset(&cb, 0, sizeof(cb)); + cb.open = uart_open; + cb.prop = uart_prop; + cb.done = uart_done; + cb.extra = &scan; + + fdt_scan(fdt, &cb); +} diff --git a/machine/uart.h b/machine/uart.h new file mode 100644 index 0000000..51e951f --- /dev/null +++ b/machine/uart.h @@ -0,0 +1,21 @@ +#ifndef _RISCV_UART_H +#define _RISCV_UART_H + +#include <stdint.h> + +extern volatile uint32_t* uart; + +#define UART_REG_TXFIFO 0 +#define UART_REG_RXFIFO 1 +#define UART_REG_TXCTRL 2 +#define UART_REG_RXCTRL 3 +#define UART_REG_DIV 4 + +#define UART_TXEN 0x1 +#define UART_RXEN 0x1 + +void uart_putchar(uint8_t ch); +int uart_getchar(); +void query_uart(uintptr_t dtb); + +#endif diff --git a/machine/unprivileged_memory.h b/machine/unprivileged_memory.h index 03f1e27..5cf2727 100644 --- a/machine/unprivileged_memory.h +++ b/machine/unprivileged_memory.h @@ -2,6 +2,7 @@ #define _RISCV_MISALIGNED_H #include "encoding.h" +#include "bits.h" #include <stdint.h> #define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type, insn) \ @@ -43,34 +44,57 @@ DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint32_t, sw) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lwu) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint64_t, ld) DECLARE_UNPRIVILEGED_STORE_FUNCTION(uint64_t, sd) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uintptr_t, ld) #else DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uint32_t, lw) +DECLARE_UNPRIVILEGED_LOAD_FUNCTION(uintptr_t, lw) + +static inline uint64_t load_uint64_t(const uint64_t* addr, uintptr_t mepc) +{ + return load_uint32_t((uint32_t*)addr, mepc) + + ((uint64_t)load_uint32_t((uint32_t*)addr + 1, mepc) << 32); +} + +static inline void store_uint64_t(uint64_t* addr, uint64_t val, uintptr_t mepc) +{ + store_uint32_t((uint32_t*)addr, val, mepc); + store_uint32_t((uint32_t*)addr + 1, val >> 32, mepc); +} #endif -static uint32_t __attribute__((always_inline)) get_insn(uintptr_t mepc, uintptr_t* mstatus) +static uintptr_t __attribute__((always_inline)) get_insn(uintptr_t mepc, uintptr_t* mstatus) { register uintptr_t __mepc asm ("a2") = mepc; register uintptr_t __mstatus asm ("a3"); - uint32_t val; + uintptr_t val; #ifndef __riscv_compressed asm ("csrrs %[mstatus], mstatus, %[mprv]\n" - "lw %[insn], (%[addr])\n" + STR(LWU) " %[insn], (%[addr])\n" "csrw mstatus, %[mstatus]" : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val) : [mprv] "r" (MSTATUS_MPRV | MSTATUS_MXR), [addr] "r" (__mepc)); #else uintptr_t rvc_mask = 3, tmp; asm ("csrrs %[mstatus], mstatus, %[mprv]\n" + "and %[tmp], %[addr], 2\n" + "bnez %[tmp], 1f\n" + STR(LWU) " %[insn], (%[addr])\n" + "and %[tmp], %[insn], %[rvc_mask]\n" + "beq %[tmp], %[rvc_mask], 2f\n" + "sll %[insn], %[insn], %[xlen_minus_16]\n" + "srl %[insn], %[insn], %[xlen_minus_16]\n" + "j 2f\n" + "1:\n" "lhu %[insn], (%[addr])\n" "and %[tmp], %[insn], %[rvc_mask]\n" - "bne %[tmp], %[rvc_mask], 1f\n" - "lh %[tmp], 2(%[addr])\n" + "bne %[tmp], %[rvc_mask], 2f\n" + "lhu %[tmp], 2(%[addr])\n" "sll %[tmp], %[tmp], 16\n" "add %[insn], %[insn], %[tmp]\n" - "1: csrw mstatus, %[mstatus]" + "2: csrw mstatus, %[mstatus]" : [mstatus] "+&r" (__mstatus), [insn] "=&r" (val), [tmp] "=&r" (tmp) : [mprv] "r" (MSTATUS_MPRV | MSTATUS_MXR), [addr] "r" (__mepc), - [rvc_mask] "r" (rvc_mask)); + [rvc_mask] "r" (rvc_mask), [xlen_minus_16] "i" (__riscv_xlen - 16)); #endif *mstatus = __mstatus; return val; diff --git a/machine/vm.h b/machine/vm.h index 4e6bcc1..986f301 100644 --- a/machine/vm.h +++ b/machine/vm.h @@ -6,11 +6,11 @@ #define MEGAPAGE_SIZE ((uintptr_t)(RISCV_PGSIZE << RISCV_PGLEVEL_BITS)) #if __riscv_xlen == 64 -# define VM_CHOICE VM_SV39 +# define SATP_MODE_CHOICE INSERT_FIELD(0, SATP64_MODE, SATP_MODE_SV39) # define VA_BITS 39 # define GIGAPAGE_SIZE (MEGAPAGE_SIZE << RISCV_PGLEVEL_BITS) #else -# define VM_CHOICE VM_SV32 +# define SATP_MODE_CHOICE INSERT_FIELD(0, SATP32_MODE, SATP_MODE_SV32) # define VA_BITS 32 #endif @@ -19,7 +19,7 @@ extern pte_t* root_page_table; static inline void flush_tlb() { - asm volatile("sfence.vm"); + asm volatile ("sfence.vma"); } static inline pte_t pte_create(uintptr_t ppn, int type) diff --git a/pk/frontend.c b/pk/frontend.c index 13fdfcf..e58df26 100644 --- a/pk/frontend.c +++ b/pk/frontend.c @@ -3,9 +3,8 @@ #include "pk.h" #include "atomic.h" #include "frontend.h" -#include "sbi.h" -#include "mcall.h" #include "syscall.h" +#include "htif.h" #include <stdint.h> long frontend_syscall(long n, uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5, uint64_t a6) @@ -24,7 +23,7 @@ long frontend_syscall(long n, uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3 magic_mem[6] = a5; magic_mem[7] = a6; - do_mcall(MCALL_HTIF_SYSCALL, magic_mem); + htif_syscall((uintptr_t)magic_mem); long ret = magic_mem[0]; diff --git a/pk/handlers.c b/pk/handlers.c index 1961852..b0eb76b 100644 --- a/pk/handlers.c +++ b/pk/handlers.c @@ -83,13 +83,13 @@ void handle_trap(trapframe_t* tf) const static trap_handler trap_handlers[] = { [CAUSE_MISALIGNED_FETCH] = handle_misaligned_fetch, - [CAUSE_FAULT_FETCH] = handle_fault_fetch, + [CAUSE_FETCH_PAGE_FAULT] = handle_fault_fetch, [CAUSE_ILLEGAL_INSTRUCTION] = handle_illegal_instruction, [CAUSE_USER_ECALL] = handle_syscall, [CAUSE_BREAKPOINT] = handle_breakpoint, [CAUSE_MISALIGNED_STORE] = handle_misaligned_store, - [CAUSE_FAULT_LOAD] = handle_fault_load, - [CAUSE_FAULT_STORE] = handle_fault_store, + [CAUSE_LOAD_PAGE_FAULT] = handle_fault_load, + [CAUSE_STORE_PAGE_FAULT] = handle_fault_store, }; kassert(tf->cause < ARRAY_SIZE(trap_handlers) && trap_handlers[tf->cause]); @@ -20,6 +20,7 @@ typedef struct { static spinlock_t vm_lock = SPINLOCK_INIT; static vmr_t* vmrs; +uintptr_t first_free_paddr; static uintptr_t first_free_page; static size_t next_free_page; static size_t free_pages; @@ -39,10 +40,14 @@ static vmr_t* __vmr_alloc(uintptr_t addr, size_t length, file_t* file, { if (!vmrs) { spinlock_lock(&vm_lock); - if (!vmrs) - vmrs = (vmr_t*)__page_alloc(); + if (!vmrs) { + vmr_t* page = (vmr_t*)__page_alloc(); + mb(); + vmrs = page; + } spinlock_unlock(&vm_lock); } + mb(); for (vmr_t* v = vmrs; v < vmrs + MAX_VMR; v++) { if (v->refcnt == 0) { @@ -141,9 +146,9 @@ static uintptr_t __vm_alloc(size_t npage) static inline pte_t prot_to_type(int prot, int user) { pte_t pte = 0; - if (prot & PROT_READ) pte |= PTE_R; - if (prot & PROT_WRITE) pte |= PTE_W; - if (prot & PROT_EXEC) pte |= PTE_X; + if (prot & PROT_READ) pte |= PTE_R | PTE_A; + if (prot & PROT_WRITE) pte |= PTE_W | PTE_D; + if (prot & PROT_EXEC) pte |= PTE_X | PTE_A; if (pte == 0) pte = PTE_R; if (user) pte |= PTE_U; return pte; @@ -385,15 +390,14 @@ void populate_mapping(const void* start, size_t size, int prot) uintptr_t pk_vm_init() { -#if __riscv_xlen == 32 - // We can't support more than 2 GiB of memory in RV32 + // HTIF address signedness and va2pa macro both cap memory size to 2 GiB mem_size = MIN(mem_size, 1U << 31); -#endif - size_t mem_pages = mem_size >> RISCV_PGSHIFT; free_pages = MAX(8, mem_pages >> (RISCV_PGLEVEL_BITS-1)); - first_free_page = first_free_paddr; - first_free_paddr += free_pages * RISCV_PGSIZE; + + extern char _end; + first_free_page = ROUNDUP((uintptr_t)&_end, RISCV_PGSIZE); + first_free_paddr = first_free_page + free_pages * RISCV_PGSIZE; root_page_table = (void*)__page_alloc(); __map_kernel_range(DRAM_BASE, DRAM_BASE, first_free_paddr - DRAM_BASE, PROT_READ|PROT_WRITE|PROT_EXEC); @@ -401,11 +405,14 @@ uintptr_t pk_vm_init() current.mmap_max = current.brk_max = MIN(DRAM_BASE, mem_size - (first_free_paddr - DRAM_BASE)); - size_t stack_size = RISCV_PGSIZE * 64; + size_t stack_size = MIN(mem_pages >> 5, 2048) * RISCV_PGSIZE; size_t stack_bottom = __do_mmap(current.mmap_max - stack_size, stack_size, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, 0, 0); kassert(stack_bottom != (uintptr_t)-1); current.stack_top = stack_bottom + stack_size; + flush_tlb(); + write_csr(sptbr, ((uintptr_t)root_page_table >> RISCV_PGSHIFT) | SATP_MODE_CHOICE); + uintptr_t kernel_stack_top = __page_alloc() + RISCV_PGSIZE; return kernel_stack_top; } @@ -33,6 +33,7 @@ uintptr_t do_mprotect(uintptr_t addr, size_t length, int prot); uintptr_t do_brk(uintptr_t addr); #define va2pa(va) ({ uintptr_t __va = (uintptr_t)(va); \ + extern uintptr_t first_free_paddr; \ __va >= DRAM_BASE ? __va : __va + first_free_paddr; }) #endif @@ -7,6 +7,7 @@ #include <stdbool.h> elf_info current; +long disabled_hart_mask; static void handle_option(const char* s) { @@ -154,18 +155,19 @@ static void rest_of_boot_loader(uintptr_t kstack_top) run_loaded_program(argc, args.argv, kstack_top); } -void boot_loader() +void boot_loader(uintptr_t dtb) { extern char trap_entry; write_csr(stvec, &trap_entry); write_csr(sscratch, 0); write_csr(sie, 0); + set_csr(sstatus, SSTATUS_SUM); file_init(); - enter_supervisor_mode(rest_of_boot_loader, pk_vm_init()); + enter_supervisor_mode(rest_of_boot_loader, pk_vm_init(), 0); } -void boot_other_hart() +void boot_other_hart(uintptr_t dtb) { // stall all harts besides hart 0 while (1) @@ -44,9 +44,10 @@ SECTIONS /* HTIF, isolated onto separate page */ /*--------------------------------------------------------------------*/ . = ALIGN(0x1000); - htif : + .htif : { - *(htif) + PROVIDE( __htif_base = . ); + *(.htif) } . = ALIGN(0x1000); diff --git a/pk/syscall.c b/pk/syscall.c index 9f3b739..7f1f196 100644 --- a/pk/syscall.c +++ b/pk/syscall.c @@ -442,6 +442,7 @@ long do_syscall(long a0, long a1, long a2, long a3, long a4, long a5, unsigned l [SYS_munmap] = sys_munmap, [SYS_mremap] = sys_mremap, [SYS_mprotect] = sys_mprotect, + [SYS_prlimit64] = sys_stub_nosys, [SYS_rt_sigaction] = sys_rt_sigaction, [SYS_gettimeofday] = sys_gettimeofday, [SYS_times] = sys_times, diff --git a/pk/syscall.h b/pk/syscall.h index 05360b7..d73af7f 100644 --- a/pk/syscall.h +++ b/pk/syscall.h @@ -33,6 +33,7 @@ #define SYS_munmap 215 #define SYS_mremap 216 #define SYS_mprotect 226 +#define SYS_prlimit64 261 #define SYS_getmainvars 2011 #define SYS_rt_sigaction 134 #define SYS_writev 66 |