diff options
author | Anup Patel <anup.patel@wdc.com> | 2020-06-21 23:30:58 +0530 |
---|---|---|
committer | Anup Patel <anup@brainfault.org> | 2020-07-09 23:04:16 +0530 |
commit | b75aff9e5d132a16d9326bc0b3bbc724ae3c753c (patch) | |
tree | 01500078707b981ec6af402e519090b0e88b1903 | |
parent | 9af85e39a550ba031e4fe9c1913e275959a9927b (diff) | |
download | spike-b75aff9e5d132a16d9326bc0b3bbc724ae3c753c.zip spike-b75aff9e5d132a16d9326bc0b3bbc724ae3c753c.tar.gz spike-b75aff9e5d132a16d9326bc0b3bbc724ae3c753c.tar.bz2 |
Implement hypervisor two-stage MMU
We extend our existing MMU implementation to support two-stage
translation when running VS-mode for RISC-V hypervisor extension.
Signed-off-by: Anup Patel <anup.patel@wdc.com>
-rw-r--r-- | riscv/mmu.cc | 110 | ||||
-rw-r--r-- | riscv/mmu.h | 120 |
2 files changed, 179 insertions, 51 deletions
diff --git a/riscv/mmu.cc b/riscv/mmu.cc index 78abd39..e8dca6a 100644 --- a/riscv/mmu.cc +++ b/riscv/mmu.cc @@ -44,18 +44,30 @@ static void throw_access_exception(reg_t addr, access_type type) } } -reg_t mmu_t::translate(reg_t addr, reg_t len, access_type type) +reg_t mmu_t::translate(reg_t addr, reg_t len, access_type type, uint32_t xlate_flags) { if (!proc) return addr; + bool mxr = get_field(proc->state.mstatus, MSTATUS_MXR); + bool virt = proc->state.v; reg_t mode = proc->state.prv; if (type != FETCH) { - if (!proc->state.debug_mode && get_field(proc->state.mstatus, MSTATUS_MPRV)) + if (!proc->state.debug_mode && get_field(proc->state.mstatus, MSTATUS_MPRV)) { mode = get_field(proc->state.mstatus, MSTATUS_MPP); + if (get_field(proc->state.mstatus, MSTATUS_MPV)) + virt = true; + } + if (!proc->state.debug_mode && (xlate_flags & RISCV_XLATE_VIRT)) { + virt = true; + mode = get_field(proc->state.hstatus, HSTATUS_SPVP); + if (type == LOAD && (xlate_flags & RISCV_XLATE_VIRT_MXR)) { + mxr = true; + } + } } - reg_t paddr = walk(addr, type, mode) | (addr & (PGSIZE-1)); + reg_t paddr = walk(addr, type, mode, virt, mxr) | (addr & (PGSIZE-1)); if (!pmp_ok(paddr, len, type, mode)) throw_access_exception(addr, type); return paddr; @@ -63,7 +75,7 @@ reg_t mmu_t::translate(reg_t addr, reg_t len, access_type type) tlb_entry_t mmu_t::fetch_slow_path(reg_t vaddr) { - reg_t paddr = translate(vaddr, sizeof(fetch_temp), FETCH); + reg_t paddr = translate(vaddr, sizeof(fetch_temp), FETCH, 0); if (auto host_addr = sim->addr_to_mem(paddr)) { return refill_tlb(vaddr, paddr, host_addr, FETCH); @@ -126,9 +138,9 @@ bool mmu_t::mmio_store(reg_t addr, size_t len, const uint8_t* bytes) return sim->mmio_store(addr, len, bytes); } -void mmu_t::load_slow_path(reg_t addr, reg_t len, uint8_t* bytes) +void mmu_t::load_slow_path(reg_t addr, reg_t len, uint8_t* bytes, uint32_t xlate_flags) { - reg_t paddr = translate(addr, len, LOAD); + reg_t paddr = translate(addr, len, LOAD, xlate_flags); if (auto host_addr = sim->addr_to_mem(paddr)) { memcpy(bytes, host_addr, len); @@ -148,9 +160,9 @@ void mmu_t::load_slow_path(reg_t addr, reg_t len, uint8_t* bytes) } } -void mmu_t::store_slow_path(reg_t addr, reg_t len, const uint8_t* bytes) +void mmu_t::store_slow_path(reg_t addr, reg_t len, const uint8_t* bytes, uint32_t xlate_flags) { - reg_t paddr = translate(addr, len, STORE); + reg_t paddr = translate(addr, len, STORE, xlate_flags); if (!matched_trigger) { reg_t data = reg_from_bytes(len, bytes); @@ -285,15 +297,82 @@ reg_t mmu_t::pmp_homogeneous(reg_t addr, reg_t len) return true; } -reg_t mmu_t::walk(reg_t addr, access_type type, reg_t mode) +reg_t mmu_t::s2xlate(reg_t gva, reg_t gpa, access_type type, bool virt, bool mxr) +{ + if (!virt) + return gpa; + + vm_info vm = decode_vm_info(proc->max_xlen, true, 0, proc->get_state()->hgatp); + if (vm.levels == 0) + return gpa; + + reg_t base = vm.ptbase; + for (int i = vm.levels - 1; i >= 0; i--) { + int ptshift = i * vm.idxbits; + int idxbits = (i == (vm.levels - 1)) ? vm.idxbits + vm.widenbits : vm.idxbits; + reg_t idx = (gpa >> (PGSHIFT + ptshift)) & ((reg_t(1) << idxbits) - 1); + + // check that physical address of PTE is legal + auto pte_paddr = base + idx * vm.ptesize; + auto ppte = sim->addr_to_mem(pte_paddr); + if (!ppte || !pmp_ok(pte_paddr, vm.ptesize, LOAD, PRV_S)) { + throw_access_exception(gva, type); + } + + reg_t pte = vm.ptesize == 4 ? from_le(*(uint32_t*)ppte) : from_le(*(uint64_t*)ppte); + reg_t ppn = pte >> PTE_PPN_SHIFT; + + if (PTE_TABLE(pte)) { // next level of page table + base = ppn << PGSHIFT; + } else if (!(pte & PTE_V) || (!(pte & PTE_R) && (pte & PTE_W))) { + break; + } else if (!(pte & PTE_U)) { + break; + } else if (type == FETCH ? !(pte & PTE_X) : + type == LOAD ? !(pte & PTE_R) && !(mxr && (pte & PTE_X)) : + !((pte & PTE_R) && (pte & PTE_W))) { + break; + } else if ((ppn & ((reg_t(1) << ptshift) - 1)) != 0) { + break; + } else { + reg_t ad = PTE_A | ((type == STORE) * PTE_D); +#ifdef RISCV_ENABLE_DIRTY + // set accessed and possibly dirty bits. + if ((pte & ad) != ad) { + if (!pmp_ok(pte_paddr, vm.ptesize, STORE, PRV_S)) + throw_access_exception(gva, type); + *(uint32_t*)ppte |= to_le((uint32_t)ad); + } +#else + // take exception if access or possibly dirty bit is not set. + if ((pte & ad) != ad) + break; +#endif + reg_t vpn = gpa >> PGSHIFT; + reg_t page_mask = (reg_t(1) << PGSHIFT) - 1; + reg_t page_base = (ppn | (vpn & ((reg_t(1) << ptshift) - 1))) << PGSHIFT; + return page_base | (gpa & page_mask); + } + } + + switch (type) { + case FETCH: throw trap_instruction_guest_page_fault(gva, gpa >> 2, 0); + case LOAD: throw trap_load_guest_page_fault(gva, gpa >> 2, 0); + case STORE: throw trap_store_guest_page_fault(gva, gpa >> 2, 0); + default: abort(); + } +} + +reg_t mmu_t::walk(reg_t addr, access_type type, reg_t mode, bool virt, bool mxr) { - vm_info vm = decode_vm_info(proc->max_xlen, mode, proc->get_state()->satp); + reg_t page_mask = (reg_t(1) << PGSHIFT) - 1; + reg_t satp = (virt) ? proc->get_state()->vsatp : proc->get_state()->satp; + vm_info vm = decode_vm_info(proc->max_xlen, false, mode, satp); if (vm.levels == 0) - return addr & ((reg_t(2) << (proc->xlen-1))-1); // zero-extend from xlen + return s2xlate(addr, addr & ((reg_t(2) << (proc->xlen-1))-1), type, virt, mxr) & ~page_mask; // zero-extend from xlen bool s_mode = mode == PRV_S; bool sum = get_field(proc->state.mstatus, MSTATUS_SUM); - bool mxr = get_field(proc->state.mstatus, MSTATUS_MXR); // verify bits xlen-1:va_bits-1 are all equal int va_bits = PGSHIFT + vm.levels * vm.idxbits; @@ -308,7 +387,7 @@ reg_t mmu_t::walk(reg_t addr, access_type type, reg_t mode) reg_t idx = (addr >> (PGSHIFT + ptshift)) & ((1 << vm.idxbits) - 1); // check that physical address of PTE is legal - auto pte_paddr = base + idx * vm.ptesize; + auto pte_paddr = s2xlate(addr, base + idx * vm.ptesize, LOAD, virt, false); auto ppte = sim->addr_to_mem(pte_paddr); if (!ppte || !pmp_ok(pte_paddr, vm.ptesize, LOAD, PRV_S)) throw_access_exception(addr, type); @@ -344,8 +423,9 @@ reg_t mmu_t::walk(reg_t addr, access_type type, reg_t mode) #endif // for superpage mappings, make a fake leaf PTE for the TLB's benefit. reg_t vpn = addr >> PGSHIFT; - reg_t value = (ppn | (vpn & ((reg_t(1) << ptshift) - 1))) << PGSHIFT; - return value; + reg_t page_base = (ppn | (vpn & ((reg_t(1) << ptshift) - 1))) << PGSHIFT; + reg_t phys = page_base | (addr & page_mask); + return s2xlate(addr, phys, type, virt, mxr) & ~page_mask; } } diff --git a/riscv/mmu.h b/riscv/mmu.h index edf38ed..990f137 100644 --- a/riscv/mmu.h +++ b/riscv/mmu.h @@ -87,9 +87,14 @@ public: proc->state.log_mem_read.push_back(std::make_tuple(addr, 0, size)); #endif +#define RISCV_XLATE_VIRT (1U << 0) +#define RISCV_XLATE_VIRT_MXR (1U << 1) + // template for functions that load an aligned value from memory - #define load_func(type) \ - inline type##_t load_##type(reg_t addr) { \ + #define load_func(type, prefix, xlate_flags) \ + inline type##_t prefix##_##type(reg_t addr) { \ + if (xlate_flags) \ + flush_tlb(); \ if (unlikely(addr & (sizeof(type##_t)-1))) \ return misaligned_load(addr, sizeof(type##_t)); \ reg_t vpn = addr >> PGSHIFT; \ @@ -109,22 +114,38 @@ public: return data; \ } \ type##_t res; \ - load_slow_path(addr, sizeof(type##_t), (uint8_t*)&res); \ + load_slow_path(addr, sizeof(type##_t), (uint8_t*)&res, (xlate_flags)); \ if (proc) READ_MEM(addr, size); \ + if (xlate_flags) \ + flush_tlb(); \ return from_le(res); \ } // load value from memory at aligned address; zero extend to register width - load_func(uint8) - load_func(uint16) - load_func(uint32) - load_func(uint64) + load_func(uint8, load, 0) + load_func(uint16, load, 0) + load_func(uint32, load, 0) + load_func(uint64, load, 0) + + // load value from guest memory at aligned address; zero extend to register width + load_func(uint8, guest_load, RISCV_XLATE_VIRT) + load_func(uint16, guest_load, RISCV_XLATE_VIRT) + load_func(uint32, guest_load, RISCV_XLATE_VIRT) + load_func(uint64, guest_load, RISCV_XLATE_VIRT) + load_func(uint16, guest_load_x, RISCV_XLATE_VIRT|RISCV_XLATE_VIRT_MXR) + load_func(uint32, guest_load_x, RISCV_XLATE_VIRT|RISCV_XLATE_VIRT_MXR) // load value from memory at aligned address; sign extend to register width - load_func(int8) - load_func(int16) - load_func(int32) - load_func(int64) + load_func(int8, load, 0) + load_func(int16, load, 0) + load_func(int32, load, 0) + load_func(int64, load, 0) + + // load value from guest memory at aligned address; sign extend to register width + load_func(int8, guest_load, RISCV_XLATE_VIRT) + load_func(int16, guest_load, RISCV_XLATE_VIRT) + load_func(int32, guest_load, RISCV_XLATE_VIRT) + load_func(int64, guest_load, RISCV_XLATE_VIRT) #ifndef RISCV_ENABLE_COMMITLOG # define WRITE_MEM(addr, value, size) ({}) @@ -134,8 +155,10 @@ public: #endif // template for functions that store an aligned value to memory - #define store_func(type) \ - void store_##type(reg_t addr, type##_t val) { \ + #define store_func(type, prefix, xlate_flags) \ + void prefix##_##type(reg_t addr, type##_t val) { \ + if (xlate_flags) \ + flush_tlb(); \ if (unlikely(addr & (sizeof(type##_t)-1))) \ return misaligned_store(addr, val, sizeof(type##_t)); \ reg_t vpn = addr >> PGSHIFT; \ @@ -154,10 +177,12 @@ public: *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = to_le(val); \ } \ else { \ - type##_t le_val = to_le(val); \ - store_slow_path(addr, sizeof(type##_t), (const uint8_t*)&le_val); \ + type##_t le_val = to_le(val); \ + store_slow_path(addr, sizeof(type##_t), (const uint8_t*)&le_val, (xlate_flags)); \ if (proc) WRITE_MEM(addr, val, size); \ } \ + if (xlate_flags) \ + flush_tlb(); \ } // template for functions that perform an atomic memory operation @@ -199,10 +224,16 @@ public: } // store value to memory at aligned address - store_func(uint8) - store_func(uint16) - store_func(uint32) - store_func(uint64) + store_func(uint8, store, 0) + store_func(uint16, store, 0) + store_func(uint32, store, 0) + store_func(uint64, store, 0) + + // store value to guest memory at aligned address + store_func(uint8, guest_store, RISCV_XLATE_VIRT) + store_func(uint16, guest_store, RISCV_XLATE_VIRT) + store_func(uint32, guest_store, RISCV_XLATE_VIRT) + store_func(uint64, guest_store, RISCV_XLATE_VIRT) // perform an atomic memory operation at an aligned address amo_func(uint32) @@ -215,7 +246,7 @@ public: inline void acquire_load_reservation(reg_t vaddr) { - reg_t paddr = translate(vaddr, 1, LOAD); + reg_t paddr = translate(vaddr, 1, LOAD, 0); if (auto host_addr = sim->addr_to_mem(paddr)) load_reservation_address = refill_tlb(vaddr, paddr, host_addr, LOAD).target_offset + vaddr; else @@ -227,7 +258,7 @@ public: if (vaddr & (size-1)) throw trap_store_address_misaligned(vaddr, 0, 0); - reg_t paddr = translate(vaddr, 1, STORE); + reg_t paddr = translate(vaddr, 1, STORE, 0); if (auto host_addr = sim->addr_to_mem(paddr)) return load_reservation_address == refill_tlb(vaddr, paddr, host_addr, STORE).target_offset + vaddr; else @@ -335,17 +366,20 @@ private: tlb_entry_t refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type); const char* fill_from_mmio(reg_t vaddr, reg_t paddr); + // perform a stage2 translation for a given guest address + reg_t s2xlate(reg_t gva, reg_t gpa, access_type type, bool virt, bool mxr); + // perform a page table walk for a given VA; set referenced/dirty bits - reg_t walk(reg_t addr, access_type type, reg_t prv); + reg_t walk(reg_t addr, access_type type, reg_t prv, bool virt, bool mxr); // handle uncommon cases: TLB misses, page faults, MMIO tlb_entry_t fetch_slow_path(reg_t addr); - void load_slow_path(reg_t addr, reg_t len, uint8_t* bytes); - void store_slow_path(reg_t addr, reg_t len, const uint8_t* bytes); + void load_slow_path(reg_t addr, reg_t len, uint8_t* bytes, uint32_t xlate_flags); + void store_slow_path(reg_t addr, reg_t len, const uint8_t* bytes, uint32_t xlate_flags); bool mmio_load(reg_t addr, size_t len, uint8_t* bytes); bool mmio_store(reg_t addr, size_t len, const uint8_t* bytes); bool mmio_ok(reg_t addr, access_type type); - reg_t translate(reg_t addr, reg_t len, access_type type); + reg_t translate(reg_t addr, reg_t len, access_type type, uint32_t xlate_flags); // ITLB lookup inline tlb_entry_t translate_insn_addr(reg_t addr) { @@ -402,27 +436,41 @@ private: struct vm_info { int levels; int idxbits; + int widenbits; int ptesize; reg_t ptbase; }; -inline vm_info decode_vm_info(int xlen, reg_t prv, reg_t satp) +inline vm_info decode_vm_info(int xlen, bool stage2, reg_t prv, reg_t satp) { if (prv == PRV_M) { - return {0, 0, 0, 0}; - } else if (prv <= PRV_S && xlen == 32) { + return {0, 0, 0, 0, 0}; + } else if (!stage2 && prv <= PRV_S && xlen == 32) { switch (get_field(satp, SATP32_MODE)) { - case SATP_MODE_OFF: return {0, 0, 0, 0}; - case SATP_MODE_SV32: return {2, 10, 4, (satp & SATP32_PPN) << PGSHIFT}; + case SATP_MODE_OFF: return {0, 0, 0, 0, 0}; + case SATP_MODE_SV32: return {2, 10, 0, 4, (satp & SATP32_PPN) << PGSHIFT}; default: abort(); } - } else if (prv <= PRV_S && xlen == 64) { + } else if (!stage2 && prv <= PRV_S && xlen == 64) { switch (get_field(satp, SATP64_MODE)) { - case SATP_MODE_OFF: return {0, 0, 0, 0}; - case SATP_MODE_SV39: return {3, 9, 8, (satp & SATP64_PPN) << PGSHIFT}; - case SATP_MODE_SV48: return {4, 9, 8, (satp & SATP64_PPN) << PGSHIFT}; - case SATP_MODE_SV57: return {5, 9, 8, (satp & SATP64_PPN) << PGSHIFT}; - case SATP_MODE_SV64: return {6, 9, 8, (satp & SATP64_PPN) << PGSHIFT}; + case SATP_MODE_OFF: return {0, 0, 0, 0, 0}; + case SATP_MODE_SV39: return {3, 9, 0, 8, (satp & SATP64_PPN) << PGSHIFT}; + case SATP_MODE_SV48: return {4, 9, 0, 8, (satp & SATP64_PPN) << PGSHIFT}; + case SATP_MODE_SV57: return {5, 9, 0, 8, (satp & SATP64_PPN) << PGSHIFT}; + case SATP_MODE_SV64: return {6, 9, 0, 8, (satp & SATP64_PPN) << PGSHIFT}; + default: abort(); + } + } else if (stage2 && xlen == 32) { + switch (get_field(satp, HGATP32_MODE)) { + case HGATP_MODE_OFF: return {0, 0, 0, 0, 0}; + case HGATP_MODE_SV32X4: return {2, 10, 2, 4, (satp & HGATP32_PPN) << PGSHIFT}; + default: abort(); + } + } else if (stage2 && xlen == 64) { + switch (get_field(satp, HGATP64_MODE)) { + case HGATP_MODE_OFF: return {0, 0, 0, 0, 0}; + case HGATP_MODE_SV39X4: return {3, 9, 2, 8, (satp & HGATP64_PPN) << PGSHIFT}; + case HGATP_MODE_SV48X4: return {4, 9, 2, 8, (satp & HGATP64_PPN) << PGSHIFT}; default: abort(); } } else { |