diff options
37 files changed, 669 insertions, 146 deletions
diff --git a/backends/iommufd.c b/backends/iommufd.c index 2a33c7a..fdfb7c9 100644 --- a/backends/iommufd.c +++ b/backends/iommufd.c @@ -197,7 +197,7 @@ void iommufd_backend_free_id(IOMMUFDBackend *be, uint32_t id) } int iommufd_backend_map_dma(IOMMUFDBackend *be, uint32_t ioas_id, hwaddr iova, - ram_addr_t size, void *vaddr, bool readonly) + uint64_t size, void *vaddr, bool readonly) { int ret, fd = be->fd; struct iommu_ioas_map map = { @@ -230,7 +230,7 @@ int iommufd_backend_map_dma(IOMMUFDBackend *be, uint32_t ioas_id, hwaddr iova, } int iommufd_backend_map_file_dma(IOMMUFDBackend *be, uint32_t ioas_id, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, int mfd, unsigned long start, bool readonly) { int ret, fd = be->fd; @@ -268,7 +268,7 @@ int iommufd_backend_map_file_dma(IOMMUFDBackend *be, uint32_t ioas_id, } int iommufd_backend_unmap_dma(IOMMUFDBackend *be, uint32_t ioas_id, - hwaddr iova, ram_addr_t size) + hwaddr iova, uint64_t size) { int ret, fd = be->fd; struct iommu_ioas_unmap unmap = { diff --git a/docs/interop/firmware.json b/docs/interop/firmware.json index 6bbe2cc..ccbfaf8 100644 --- a/docs/interop/firmware.json +++ b/docs/interop/firmware.json @@ -85,12 +85,14 @@ # # @loongarch64: 64-bit LoongArch. (since: 7.1) # +# @riscv64: 64-bit RISC-V. +# # @x86_64: 64-bit x86. # # Since: 3.0 ## { 'enum' : 'FirmwareArchitecture', - 'data' : [ 'aarch64', 'arm', 'i386', 'loongarch64', 'x86_64' ] } + 'data' : [ 'aarch64', 'arm', 'i386', 'loongarch64', 'riscv64', 'x86_64' ] } ## # @FirmwareTarget: diff --git a/docs/specs/riscv-iommu.rst b/docs/specs/riscv-iommu.rst index 991d376..571a6a6 100644 --- a/docs/specs/riscv-iommu.rst +++ b/docs/specs/riscv-iommu.rst @@ -30,15 +30,15 @@ This will add a RISC-V IOMMU PCI device in the board following any additional PCI parameters (like PCI bus address). The behavior of the RISC-V IOMMU is defined by the spec but its operation is OS dependent. -As of this writing the existing Linux kernel support `linux-v8`_, not yet merged, -does not have support for features like VFIO passthrough. The IOMMU emulation -was tested using a public Ventana Micro Systems kernel repository in -`ventana-linux`_. This kernel is based on `linux-v8`_ with additional patches that -enable features like KVM VFIO passthrough with irqbypass. Until the kernel support -is feature complete feel free to use the kernel available in the Ventana Micro Systems -mirror. - -The current Linux kernel support will use the IOMMU device to create IOMMU groups +Linux kernel iommu support was merged in v6.13. QEMU IOMMU emulation can be +used with mainline kernels for simple IOMMU PCIe support. + +As of v6.17, it does not have support for features like VFIO passthrough. +There is a `VFIO`_ RFC series that is not yet merged. The public Ventana Micro +Systems kernel repository in `ventana-linux`_ can be used for testing the VFIO +functions. + +The v6.13+ Linux kernel support uses the IOMMU device to create IOMMU groups with any eligible cards available in the system, regardless of factors such as the order in which the devices are added in the command line. @@ -49,7 +49,7 @@ IOMMU kernel driver behaves: $ qemu-system-riscv64 \ -M virt,aia=aplic-imsic,aia-guests=5 \ - -device riscv-iommu-pci,addr=1.0,vendor-id=0x1efd,device-id=0xedf1 \ + -device riscv-iommu-pci,addr=1.0 \ -device e1000e,netdev=net1 -netdev user,id=net1,net=192.168.0.0/24 \ -device e1000e,netdev=net2 -netdev user,id=net2,net=192.168.200.0/24 \ (...) @@ -58,21 +58,11 @@ IOMMU kernel driver behaves: -M virt,aia=aplic-imsic,aia-guests=5 \ -device e1000e,netdev=net1 -netdev user,id=net1,net=192.168.0.0/24 \ -device e1000e,netdev=net2 -netdev user,id=net2,net=192.168.200.0/24 \ - -device riscv-iommu-pci,addr=1.0,vendor-id=0x1efd,device-id=0xedf1 \ + -device riscv-iommu-pci,addr=3.0 \ (...) Both will create iommu groups for the two e1000e cards. -Another thing to notice on `linux-v8`_ and `ventana-linux`_ is that the kernel driver -considers an IOMMU identified as a Rivos device, i.e. it uses Rivos vendor ID. To -use the riscv-iommu-pci device with the existing kernel support we need to emulate -a Rivos PCI IOMMU by setting 'vendor-id' and 'device-id': - -.. code-block:: bash - - $ qemu-system-riscv64 -M virt \ - -device riscv-iommu-pci,vendor-id=0x1efd,device-id=0xedf1 (...) - Several options are available to control the capabilities of the device, namely: - "bus": the bus that the IOMMU device uses @@ -84,6 +74,7 @@ Several options are available to control the capabilities of the device, namely: - "g-stage": enable g-stage support - "hpm-counters": number of hardware performance counters available. Maximum value is 31. Default value is 31. Use 0 (zero) to disable HPM support +- "vendor-id"/"device-id": pci device ID. Defaults to 1b36:0014 (Redhat) riscv-iommu-sys device ---------------------- @@ -111,6 +102,6 @@ riscv-iommu options: .. _iommu1.0.0: https://github.com/riscv-non-isa/riscv-iommu/releases/download/v1.0.0/riscv-iommu.pdf -.. _linux-v8: https://lore.kernel.org/linux-riscv/cover.1718388908.git.tjeznach@rivosinc.com/ +.. _VFIO: https://lore.kernel.org/linux-riscv/20241114161845.502027-17-ajones@ventanamicro.com/ .. _ventana-linux: https://github.com/ventanamicro/linux/tree/dev-upstream diff --git a/hw/char/sifive_uart.c b/hw/char/sifive_uart.c index 9bc697a..e7357d5 100644 --- a/hw/char/sifive_uart.c +++ b/hw/char/sifive_uart.c @@ -28,23 +28,18 @@ #define TX_INTERRUPT_TRIGGER_DELAY_NS 100 -/* - * Not yet implemented: - * - * Transmit FIFO using "qemu/fifo8.h" - */ - /* Returns the state of the IP (interrupt pending) register */ -static uint64_t sifive_uart_ip(SiFiveUARTState *s) +static uint32_t sifive_uart_ip(SiFiveUARTState *s) { - uint64_t ret = 0; + uint32_t ret = 0; - uint64_t txcnt = SIFIVE_UART_GET_TXCNT(s->txctrl); - uint64_t rxcnt = SIFIVE_UART_GET_RXCNT(s->rxctrl); + uint32_t txcnt = SIFIVE_UART_GET_TXCNT(s->txctrl); + uint32_t rxcnt = SIFIVE_UART_GET_RXCNT(s->rxctrl); - if (txcnt != 0) { + if (fifo8_num_used(&s->tx_fifo) < txcnt) { ret |= SIFIVE_UART_IP_TXWM; } + if (s->rx_fifo_len > rxcnt) { ret |= SIFIVE_UART_IP_RXWM; } @@ -55,15 +50,14 @@ static uint64_t sifive_uart_ip(SiFiveUARTState *s) static void sifive_uart_update_irq(SiFiveUARTState *s) { int cond = 0; - if ((s->ie & SIFIVE_UART_IE_TXWM) || - ((s->ie & SIFIVE_UART_IE_RXWM) && s->rx_fifo_len)) { + uint32_t ip = sifive_uart_ip(s); + + if (((ip & SIFIVE_UART_IP_TXWM) && (s->ie & SIFIVE_UART_IE_TXWM)) || + ((ip & SIFIVE_UART_IP_RXWM) && (s->ie & SIFIVE_UART_IE_RXWM))) { cond = 1; } - if (cond) { - qemu_irq_raise(s->irq); - } else { - qemu_irq_lower(s->irq); - } + + qemu_set_irq(s->irq, cond); } static gboolean sifive_uart_xmit(void *do_not_use, GIOCondition cond, @@ -119,10 +113,12 @@ static void sifive_uart_write_tx_fifo(SiFiveUARTState *s, const uint8_t *buf, if (size > fifo8_num_free(&s->tx_fifo)) { size = fifo8_num_free(&s->tx_fifo); - qemu_log_mask(LOG_GUEST_ERROR, "sifive_uart: TX FIFO overflow"); + qemu_log_mask(LOG_GUEST_ERROR, "sifive_uart: TX FIFO overflow.\n"); } - fifo8_push_all(&s->tx_fifo, buf, size); + if (size > 0) { + fifo8_push_all(&s->tx_fifo, buf, size); + } if (fifo8_is_full(&s->tx_fifo)) { s->txfifo |= SIFIVE_UART_TXFIFO_FULL; diff --git a/hw/intc/riscv_aclint.c b/hw/intc/riscv_aclint.c index 4623cfa0..9f4c36e 100644 --- a/hw/intc/riscv_aclint.c +++ b/hw/intc/riscv_aclint.c @@ -323,12 +323,15 @@ static void riscv_aclint_mtimer_reset_enter(Object *obj, ResetType type) static const VMStateDescription vmstate_riscv_mtimer = { .name = "riscv_mtimer", - .version_id = 1, - .minimum_version_id = 1, + .version_id = 3, + .minimum_version_id = 3, .fields = (const VMStateField[]) { + VMSTATE_UINT64(time_delta, RISCVAclintMTimerState), VMSTATE_VARRAY_UINT32(timecmp, RISCVAclintMTimerState, num_harts, 0, vmstate_info_uint64, uint64_t), + VMSTATE_TIMER_PTR_VARRAY(timers, RISCVAclintMTimerState, + num_harts), VMSTATE_END_OF_LIST() } }; diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c index 96a7fbd..b33c7fe 100644 --- a/hw/riscv/riscv-iommu.c +++ b/hw/riscv/riscv-iommu.c @@ -558,6 +558,7 @@ static MemTxResult riscv_iommu_msi_write(RISCVIOMMUState *s, MemTxResult res; dma_addr_t addr; uint64_t intn; + size_t offset; uint32_t n190; uint64_t pte[2]; int fault_type = RISCV_IOMMU_FQ_TTYPE_UADDR_WR; @@ -565,16 +566,18 @@ static MemTxResult riscv_iommu_msi_write(RISCVIOMMUState *s, /* Interrupt File Number */ intn = riscv_iommu_pext_u64(PPN_DOWN(gpa), ctx->msi_addr_mask); - if (intn >= 256) { + offset = intn * sizeof(pte); + + /* fetch MSI PTE */ + addr = PPN_PHYS(get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_PPN)); + if (addr & offset) { /* Interrupt file number out of range */ res = MEMTX_ACCESS_ERROR; cause = RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT; goto err; } - /* fetch MSI PTE */ - addr = PPN_PHYS(get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_PPN)); - addr = addr | (intn * sizeof(pte)); + addr |= offset; res = dma_memory_read(s->target_as, addr, &pte, sizeof(pte), MEMTXATTRS_UNSPECIFIED); if (res != MEMTX_OK) { @@ -866,6 +869,145 @@ static bool riscv_iommu_validate_process_ctx(RISCVIOMMUState *s, return true; } +/** + * pdt_memory_read: PDT wrapper of dma_memory_read. + * + * @s: IOMMU Device State + * @ctx: Device Translation Context with devid and pasid set + * @addr: address within that address space + * @buf: buffer with the data transferred + * @len: length of the data transferred + * @attrs: memory transaction attributes + */ +static MemTxResult pdt_memory_read(RISCVIOMMUState *s, + RISCVIOMMUContext *ctx, + dma_addr_t addr, + void *buf, dma_addr_t len, + MemTxAttrs attrs) +{ + uint64_t gatp_mode, pte; + struct { + unsigned char step; + unsigned char levels; + unsigned char ptidxbits; + unsigned char ptesize; + } sc; + MemTxResult ret; + dma_addr_t base = addr; + + /* G stages translation mode */ + gatp_mode = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD); + if (gatp_mode == RISCV_IOMMU_DC_IOHGATP_MODE_BARE) { + goto out; + } + + /* G stages translation tables root pointer */ + base = PPN_PHYS(get_field(ctx->gatp, RISCV_IOMMU_ATP_PPN_FIELD)); + + /* Start at step 0 */ + sc.step = 0; + + if (s->fctl & RISCV_IOMMU_FCTL_GXL) { + /* 32bit mode for GXL == 1 */ + switch (gatp_mode) { + case RISCV_IOMMU_DC_IOHGATP_MODE_SV32X4: + if (!(s->cap & RISCV_IOMMU_CAP_SV32X4)) { + return MEMTX_ACCESS_ERROR; + } + sc.levels = 2; + sc.ptidxbits = 10; + sc.ptesize = 4; + break; + default: + return MEMTX_ACCESS_ERROR; + } + } else { + /* 64bit mode for GXL == 0 */ + switch (gatp_mode) { + case RISCV_IOMMU_DC_IOHGATP_MODE_SV39X4: + if (!(s->cap & RISCV_IOMMU_CAP_SV39X4)) { + return MEMTX_ACCESS_ERROR; + } + sc.levels = 3; + sc.ptidxbits = 9; + sc.ptesize = 8; + break; + case RISCV_IOMMU_DC_IOHGATP_MODE_SV48X4: + if (!(s->cap & RISCV_IOMMU_CAP_SV48X4)) { + return MEMTX_ACCESS_ERROR; + } + sc.levels = 4; + sc.ptidxbits = 9; + sc.ptesize = 8; + break; + case RISCV_IOMMU_DC_IOHGATP_MODE_SV57X4: + if (!(s->cap & RISCV_IOMMU_CAP_SV57X4)) { + return MEMTX_ACCESS_ERROR; + } + sc.levels = 5; + sc.ptidxbits = 9; + sc.ptesize = 8; + break; + default: + return MEMTX_ACCESS_ERROR; + } + } + + do { + const unsigned va_bits = (sc.step ? 0 : 2) + sc.ptidxbits; + const unsigned va_skip = TARGET_PAGE_BITS + sc.ptidxbits * + (sc.levels - 1 - sc.step); + const unsigned idx = (addr >> va_skip) & ((1 << va_bits) - 1); + const dma_addr_t pte_addr = base + idx * sc.ptesize; + + /* Address range check before first level lookup */ + if (!sc.step) { + const uint64_t va_mask = (1ULL << (va_skip + va_bits)) - 1; + if ((addr & va_mask) != addr) { + return MEMTX_ACCESS_ERROR; + } + } + + /* Read page table entry */ + if (sc.ptesize == 4) { + uint32_t pte32 = 0; + ret = ldl_le_dma(s->target_as, pte_addr, &pte32, attrs); + pte = pte32; + } else { + ret = ldq_le_dma(s->target_as, pte_addr, &pte, attrs); + } + if (ret != MEMTX_OK) { + return ret; + } + + sc.step++; + hwaddr ppn = pte >> PTE_PPN_SHIFT; + + if (!(pte & PTE_V)) { + return MEMTX_ACCESS_ERROR; /* Invalid PTE */ + } else if (!(pte & (PTE_R | PTE_W | PTE_X))) { + base = PPN_PHYS(ppn); /* Inner PTE, continue walking */ + } else if ((pte & (PTE_R | PTE_W | PTE_X)) == PTE_W) { + return MEMTX_ACCESS_ERROR; /* Reserved leaf PTE flags: PTE_W */ + } else if ((pte & (PTE_R | PTE_W | PTE_X)) == (PTE_W | PTE_X)) { + return MEMTX_ACCESS_ERROR; /* Reserved leaf PTE flags: PTE_W + PTE_X */ + } else if (ppn & ((1ULL << (va_skip - TARGET_PAGE_BITS)) - 1)) { + return MEMTX_ACCESS_ERROR; /* Misaligned PPN */ + } else { + /* Leaf PTE, translation completed. */ + base = PPN_PHYS(ppn) | (addr & ((1ULL << va_skip) - 1)); + break; + } + + if (sc.step == sc.levels) { + return MEMTX_ACCESS_ERROR; /* Can't find leaf PTE */ + } + } while (1); + +out: + return dma_memory_read(s->target_as, base, buf, len, attrs); +} + /* * RISC-V IOMMU Device Context Loopkup - Device Directory Tree Walk * @@ -1038,7 +1180,7 @@ static int riscv_iommu_ctx_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx) */ const int split = depth * 9 + 8; addr |= ((ctx->process_id >> split) << 3) & ~TARGET_PAGE_MASK; - if (dma_memory_read(s->target_as, addr, &de, sizeof(de), + if (pdt_memory_read(s, ctx, addr, &de, sizeof(de), MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) { return RISCV_IOMMU_FQ_CAUSE_PDT_LOAD_FAULT; } @@ -1053,7 +1195,7 @@ static int riscv_iommu_ctx_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx) /* Leaf entry in PDT */ addr |= (ctx->process_id << 4) & ~TARGET_PAGE_MASK; - if (dma_memory_read(s->target_as, addr, &dc.ta, sizeof(uint64_t) * 2, + if (pdt_memory_read(s, ctx, addr, &dc.ta, sizeof(uint64_t) * 2, MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) { return RISCV_IOMMU_FQ_CAUSE_PDT_LOAD_FAULT; } diff --git a/hw/vfio-user/container.c b/hw/vfio-user/container.c index 411eb7b..e45192f 100644 --- a/hw/vfio-user/container.c +++ b/hw/vfio-user/container.c @@ -39,7 +39,7 @@ static void vfio_user_listener_commit(VFIOContainer *bcontainer) } static int vfio_user_dma_unmap(const VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, IOMMUTLBEntry *iotlb, bool unmap_all) { VFIOUserContainer *container = VFIO_IOMMU_USER(bcontainer); @@ -81,7 +81,7 @@ static int vfio_user_dma_unmap(const VFIOContainer *bcontainer, } static int vfio_user_dma_map(const VFIOContainer *bcontainer, hwaddr iova, - ram_addr_t size, void *vaddr, bool readonly, + uint64_t size, void *vaddr, bool readonly, MemoryRegion *mrp) { VFIOUserContainer *container = VFIO_IOMMU_USER(bcontainer); diff --git a/hw/vfio/container-legacy.c b/hw/vfio/container-legacy.c index c0540f2..629ff23 100644 --- a/hw/vfio/container-legacy.c +++ b/hw/vfio/container-legacy.c @@ -69,7 +69,7 @@ static int vfio_ram_block_discard_disable(VFIOLegacyContainer *container, } static int vfio_dma_unmap_bitmap(const VFIOLegacyContainer *container, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, IOMMUTLBEntry *iotlb) { const VFIOContainer *bcontainer = VFIO_IOMMU(container); @@ -122,7 +122,7 @@ unmap_exit: } static int vfio_legacy_dma_unmap_one(const VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, IOMMUTLBEntry *iotlb) { const VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(bcontainer); @@ -147,25 +147,7 @@ static int vfio_legacy_dma_unmap_one(const VFIOContainer *bcontainer, need_dirty_sync = true; } - while (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) { - /* - * The type1 backend has an off-by-one bug in the kernel (71a7d3d78e3c - * v4.15) where an overflow in its wrap-around check prevents us from - * unmapping the last page of the address space. Test for the error - * condition and re-try the unmap excluding the last page. The - * expectation is that we've never mapped the last page anyway and this - * unmap request comes via vIOMMU support which also makes it unlikely - * that this page is used. This bug was introduced well after type1 v2 - * support was introduced, so we shouldn't need to test for v1. A fix - * is queued for kernel v5.0 so this workaround can be removed once - * affected kernels are sufficiently deprecated. - */ - if (errno == EINVAL && unmap.size && !(unmap.iova + unmap.size) && - container->iommu_type == VFIO_TYPE1v2_IOMMU) { - trace_vfio_legacy_dma_unmap_overflow_workaround(); - unmap.size -= 1ULL << ctz64(bcontainer->pgsizes); - continue; - } + if (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) { return -errno; } @@ -185,7 +167,7 @@ static int vfio_legacy_dma_unmap_one(const VFIOContainer *bcontainer, * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86 */ static int vfio_legacy_dma_unmap(const VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, IOMMUTLBEntry *iotlb, bool unmap_all) { int ret; @@ -210,7 +192,7 @@ static int vfio_legacy_dma_unmap(const VFIOContainer *bcontainer, } static int vfio_legacy_dma_map(const VFIOContainer *bcontainer, hwaddr iova, - ram_addr_t size, void *vaddr, bool readonly, + uint64_t size, void *vaddr, bool readonly, MemoryRegion *mr) { const VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(bcontainer); diff --git a/hw/vfio/container.c b/hw/vfio/container.c index 250b20f..41de343 100644 --- a/hw/vfio/container.c +++ b/hw/vfio/container.c @@ -74,7 +74,7 @@ void vfio_address_space_insert(VFIOAddressSpace *space, } int vfio_container_dma_map(VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, void *vaddr, bool readonly, MemoryRegion *mr) { VFIOIOMMUClass *vioc = VFIO_IOMMU_GET_CLASS(bcontainer); @@ -93,7 +93,7 @@ int vfio_container_dma_map(VFIOContainer *bcontainer, } int vfio_container_dma_unmap(VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, IOMMUTLBEntry *iotlb, bool unmap_all) { VFIOIOMMUClass *vioc = VFIO_IOMMU_GET_CLASS(bcontainer); @@ -246,7 +246,7 @@ static int vfio_container_devices_query_dirty_bitmap( int vfio_container_query_dirty_bitmap(const VFIOContainer *bcontainer, uint64_t iova, uint64_t size, - ram_addr_t ram_addr, Error **errp) + hwaddr translated_addr, Error **errp) { bool all_device_dirty_tracking = vfio_container_devices_dirty_tracking_is_supported(bcontainer); @@ -255,7 +255,7 @@ int vfio_container_query_dirty_bitmap(const VFIOContainer *bcontainer, int ret; if (!bcontainer->dirty_pages_supported && !all_device_dirty_tracking) { - cpu_physical_memory_set_dirty_range(ram_addr, size, + cpu_physical_memory_set_dirty_range(translated_addr, size, tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE); return 0; @@ -280,11 +280,12 @@ int vfio_container_query_dirty_bitmap(const VFIOContainer *bcontainer, goto out; } - dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, ram_addr, + dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, + translated_addr, vbmap.pages); - trace_vfio_container_query_dirty_bitmap(iova, size, vbmap.size, ram_addr, - dirty_pages); + trace_vfio_container_query_dirty_bitmap(iova, size, vbmap.size, + translated_addr, dirty_pages); out: g_free(vbmap.bitmap); diff --git a/hw/vfio/cpr-legacy.c b/hw/vfio/cpr-legacy.c index 1a16cb1..80af746 100644 --- a/hw/vfio/cpr-legacy.c +++ b/hw/vfio/cpr-legacy.c @@ -39,7 +39,7 @@ static bool vfio_dma_unmap_vaddr_all(VFIOLegacyContainer *container, * The incoming state is cleared thereafter. */ static int vfio_legacy_cpr_dma_map(const VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, void *vaddr, + hwaddr iova, uint64_t size, void *vaddr, bool readonly, MemoryRegion *mr) { const VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(bcontainer); diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c index f0ffe23..68470d5 100644 --- a/hw/vfio/iommufd.c +++ b/hw/vfio/iommufd.c @@ -35,7 +35,7 @@ TYPE_HOST_IOMMU_DEVICE_IOMMUFD "-vfio" static int iommufd_cdev_map(const VFIOContainer *bcontainer, hwaddr iova, - ram_addr_t size, void *vaddr, bool readonly, + uint64_t size, void *vaddr, bool readonly, MemoryRegion *mr) { const VFIOIOMMUFDContainer *container = VFIO_IOMMU_IOMMUFD(bcontainer); @@ -46,7 +46,7 @@ static int iommufd_cdev_map(const VFIOContainer *bcontainer, hwaddr iova, } static int iommufd_cdev_map_file(const VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, int fd, unsigned long start, bool readonly) { const VFIOIOMMUFDContainer *container = VFIO_IOMMU_IOMMUFD(bcontainer); @@ -57,7 +57,7 @@ static int iommufd_cdev_map_file(const VFIOContainer *bcontainer, } static int iommufd_cdev_unmap(const VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, IOMMUTLBEntry *iotlb, bool unmap_all) { const VFIOIOMMUFDContainer *container = VFIO_IOMMU_IOMMUFD(bcontainer); diff --git a/hw/vfio/listener.c b/hw/vfio/listener.c index 3b6f17f..a2c19a3 100644 --- a/hw/vfio/listener.c +++ b/hw/vfio/listener.c @@ -1059,7 +1059,7 @@ static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) VFIOGuestIOMMU *giommu = gdn->giommu; VFIOContainer *bcontainer = giommu->bcontainer; hwaddr iova = iotlb->iova + giommu->iommu_offset; - ram_addr_t translated_addr; + hwaddr translated_addr; Error *local_err = NULL; int ret = -EINVAL; MemoryRegion *mr; @@ -1108,8 +1108,8 @@ static int vfio_ram_discard_query_dirty_bitmap(MemoryRegionSection *section, { const hwaddr size = int128_get64(section->size); const hwaddr iova = section->offset_within_address_space; - const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) + - section->offset_within_region; + const hwaddr translated_addr = memory_region_get_ram_addr(section->mr) + + section->offset_within_region; VFIORamDiscardListener *vrdl = opaque; Error *local_err = NULL; int ret; @@ -1118,8 +1118,8 @@ static int vfio_ram_discard_query_dirty_bitmap(MemoryRegionSection *section, * Sync the whole mapped region (spanning multiple individual mappings) * in one go. */ - ret = vfio_container_query_dirty_bitmap(vrdl->bcontainer, iova, size, ram_addr, - &local_err); + ret = vfio_container_query_dirty_bitmap(vrdl->bcontainer, iova, size, + translated_addr, &local_err); if (ret) { error_report_err(local_err); } @@ -1183,7 +1183,7 @@ static int vfio_sync_iommu_dirty_bitmap(VFIOContainer *bcontainer, static int vfio_sync_dirty_bitmap(VFIOContainer *bcontainer, MemoryRegionSection *section, Error **errp) { - ram_addr_t ram_addr; + hwaddr translated_addr; if (memory_region_is_iommu(section->mr)) { return vfio_sync_iommu_dirty_bitmap(bcontainer, section); @@ -1198,12 +1198,12 @@ static int vfio_sync_dirty_bitmap(VFIOContainer *bcontainer, return ret; } - ram_addr = memory_region_get_ram_addr(section->mr) + - section->offset_within_region; + translated_addr = memory_region_get_ram_addr(section->mr) + + section->offset_within_region; return vfio_container_query_dirty_bitmap(bcontainer, REAL_HOST_PAGE_ALIGN(section->offset_within_address_space), - int128_get64(section->size), ram_addr, errp); + int128_get64(section->size), translated_addr, errp); } static void vfio_listener_log_sync(MemoryListener *listener, diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events index e3d571f..1e89544 100644 --- a/hw/vfio/trace-events +++ b/hw/vfio/trace-events @@ -104,15 +104,14 @@ vfio_device_dirty_tracking_update(uint64_t start, uint64_t end, uint64_t min, ui vfio_device_dirty_tracking_start(int nr_ranges, uint64_t min32, uint64_t max32, uint64_t min64, uint64_t max64, uint64_t minpci, uint64_t maxpci) "nr_ranges %d 32:[0x%"PRIx64" - 0x%"PRIx64"], 64:[0x%"PRIx64" - 0x%"PRIx64"], pci64:[0x%"PRIx64" - 0x%"PRIx64"]" vfio_iommu_map_dirty_notify(uint64_t iova_start, uint64_t iova_end) "iommu dirty @ 0x%"PRIx64" - 0x%"PRIx64 -# container-base.c -vfio_container_query_dirty_bitmap(uint64_t iova, uint64_t size, uint64_t bitmap_size, uint64_t start, uint64_t dirty_pages) "iova=0x%"PRIx64" size= 0x%"PRIx64" bitmap_size=0x%"PRIx64" start=0x%"PRIx64" dirty_pages=%"PRIu64 - # container.c +vfio_container_query_dirty_bitmap(uint64_t iova, uint64_t size, uint64_t bitmap_size, uint64_t translated_addr, uint64_t dirty_pages) "iova=0x%"PRIx64" size= 0x%"PRIx64" bitmap_size=0x%"PRIx64" gpa=0x%"PRIx64" dirty_pages=%"PRIu64 + +# container-legacy.c vfio_container_disconnect(int fd) "close container->fd=%d" vfio_group_put(int fd) "close group->fd=%d" vfio_device_get(const char * name, unsigned int flags, unsigned int num_regions, unsigned int num_irqs) "Device %s flags: %u, regions: %u, irqs: %u" vfio_device_put(int fd) "close vdev->fd=%d" -vfio_legacy_dma_unmap_overflow_workaround(void) "" # region.c vfio_region_write(const char *name, int index, uint64_t addr, uint64_t data, unsigned size) " (%s:region%d+0x%"PRIx64", 0x%"PRIx64 ", %d)" diff --git a/include/hw/intc/riscv_aclint.h b/include/hw/intc/riscv_aclint.h index 693415e..4b7406e 100644 --- a/include/hw/intc/riscv_aclint.h +++ b/include/hw/intc/riscv_aclint.h @@ -80,4 +80,8 @@ enum { RISCV_ACLINT_SWI_SIZE = 0x4000 }; +#define VMSTATE_TIMER_PTR_VARRAY(_f, _s, _f_n) \ +VMSTATE_VARRAY_OF_POINTER_UINT32(_f, _s, _f_n, 0, vmstate_info_timer, \ + QEMUTimer *) + #endif diff --git a/include/hw/vfio/vfio-container.h b/include/hw/vfio/vfio-container.h index b8fb2b8..c4b58d6 100644 --- a/include/hw/vfio/vfio-container.h +++ b/include/hw/vfio/vfio-container.h @@ -81,10 +81,10 @@ void vfio_address_space_insert(VFIOAddressSpace *space, VFIOContainer *bcontainer); int vfio_container_dma_map(VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, void *vaddr, bool readonly, MemoryRegion *mr); int vfio_container_dma_unmap(VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, IOMMUTLBEntry *iotlb, bool unmap_all); bool vfio_container_add_section_window(VFIOContainer *bcontainer, MemoryRegionSection *section, @@ -98,7 +98,8 @@ bool vfio_container_dirty_tracking_is_started( bool vfio_container_devices_dirty_tracking_is_supported( const VFIOContainer *bcontainer); int vfio_container_query_dirty_bitmap(const VFIOContainer *bcontainer, - uint64_t iova, uint64_t size, ram_addr_t ram_addr, Error **errp); + uint64_t iova, uint64_t size, + hwaddr translated_addr, Error **errp); GList *vfio_container_get_iova_ranges(const VFIOContainer *bcontainer); @@ -166,7 +167,7 @@ struct VFIOIOMMUClass { * Returns 0 to indicate success and -errno otherwise. */ int (*dma_map)(const VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, void *vaddr, bool readonly, MemoryRegion *mr); /** * @dma_map_file @@ -181,7 +182,7 @@ struct VFIOIOMMUClass { * @readonly: map read only if true */ int (*dma_map_file)(const VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, int fd, unsigned long start, bool readonly); /** * @dma_unmap @@ -197,7 +198,7 @@ struct VFIOIOMMUClass { * Returns 0 to indicate success and -errno otherwise. */ int (*dma_unmap)(const VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, + hwaddr iova, uint64_t size, IOMMUTLBEntry *iotlb, bool unmap_all); diff --git a/include/hw/vfio/vfio-cpr.h b/include/hw/vfio/vfio-cpr.h index 26ee0c4..81f4e24 100644 --- a/include/hw/vfio/vfio-cpr.h +++ b/include/hw/vfio/vfio-cpr.h @@ -21,7 +21,7 @@ struct VFIOIOMMUFDContainer; struct IOMMUFDBackend; typedef int (*dma_map_fn)(const struct VFIOContainer *bcontainer, - hwaddr iova, ram_addr_t size, void *vaddr, + hwaddr iova, uint64_t size, void *vaddr, bool readonly, MemoryRegion *mr); typedef struct VFIOContainerCPR { diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h index 6f5a9fe..63ccaee 100644 --- a/include/migration/vmstate.h +++ b/include/migration/vmstate.h @@ -537,6 +537,16 @@ extern const VMStateInfo vmstate_info_qlist; .offset = vmstate_offset_array(_s, _f, _type*, _n), \ } +#define VMSTATE_VARRAY_OF_POINTER_UINT32(_field, _state, _field_num, _version, _info, _type) { \ + .name = (stringify(_field)), \ + .version_id = (_version), \ + .num_offset = vmstate_offset_value(_state, _field_num, uint32_t), \ + .info = &(_info), \ + .size = sizeof(_type), \ + .flags = VMS_VARRAY_UINT32 | VMS_ARRAY_OF_POINTER | VMS_POINTER, \ + .offset = vmstate_offset_pointer(_state, _field, _type), \ +} + #define VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, _num, _version, _vmsd, _type) { \ .name = (stringify(_field)), \ .version_id = (_version), \ diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h index ed3e511..cf8d7cf 100644 --- a/include/qemu/osdep.h +++ b/include/qemu/osdep.h @@ -561,7 +561,7 @@ int madvise(char *, size_t, int); #if defined(__linux__) && \ (defined(__x86_64__) || defined(__arm__) || defined(__aarch64__) \ - || defined(__powerpc64__)) + || defined(__powerpc64__) || defined(__riscv)) /* Use 2 MiB alignment so transparent hugepages can be used by KVM. Valgrind does not support alignments larger than 1 MiB, therefore we need special code which handles running on Valgrind. */ diff --git a/include/system/iommufd.h b/include/system/iommufd.h index c9c72ff..a659f36 100644 --- a/include/system/iommufd.h +++ b/include/system/iommufd.h @@ -45,12 +45,12 @@ bool iommufd_backend_alloc_ioas(IOMMUFDBackend *be, uint32_t *ioas_id, Error **errp); void iommufd_backend_free_id(IOMMUFDBackend *be, uint32_t id); int iommufd_backend_map_file_dma(IOMMUFDBackend *be, uint32_t ioas_id, - hwaddr iova, ram_addr_t size, int fd, + hwaddr iova, uint64_t size, int fd, unsigned long start, bool readonly); int iommufd_backend_map_dma(IOMMUFDBackend *be, uint32_t ioas_id, hwaddr iova, - ram_addr_t size, void *vaddr, bool readonly); + uint64_t size, void *vaddr, bool readonly); int iommufd_backend_unmap_dma(IOMMUFDBackend *be, uint32_t ioas_id, - hwaddr iova, ram_addr_t size); + hwaddr iova, uint64_t size); bool iommufd_backend_get_device_info(IOMMUFDBackend *be, uint32_t devid, uint32_t *type, void *data, uint32_t len, uint64_t *caps, Error **errp); diff --git a/linux-user/syscall.c b/linux-user/syscall.c index 1a5f2a0..d78b202 100644 --- a/linux-user/syscall.c +++ b/linux-user/syscall.c @@ -9023,6 +9023,29 @@ static int do_getdents64(abi_long dirfd, abi_long arg2, abi_long count) #define RISCV_HWPROBE_EXT_ZTSO (1ULL << 33) #define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34) #define RISCV_HWPROBE_EXT_ZICOND (1ULL << 35) +#define RISCV_HWPROBE_EXT_ZIHINTPAUSE (1ULL << 36) +#define RISCV_HWPROBE_EXT_ZVE32X (1ULL << 37) +#define RISCV_HWPROBE_EXT_ZVE32F (1ULL << 38) +#define RISCV_HWPROBE_EXT_ZVE64X (1ULL << 39) +#define RISCV_HWPROBE_EXT_ZVE64F (1ULL << 40) +#define RISCV_HWPROBE_EXT_ZVE64D (1ULL << 41) +#define RISCV_HWPROBE_EXT_ZIMOP (1ULL << 42) +#define RISCV_HWPROBE_EXT_ZCA (1ULL << 43) +#define RISCV_HWPROBE_EXT_ZCB (1ULL << 44) +#define RISCV_HWPROBE_EXT_ZCD (1ULL << 45) +#define RISCV_HWPROBE_EXT_ZCF (1ULL << 46) +#define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47) +#define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48) +#define RISCV_HWPROBE_EXT_SUPM (1ULL << 49) +#define RISCV_HWPROBE_EXT_ZICNTR (1ULL << 50) +#define RISCV_HWPROBE_EXT_ZIHPM (1ULL << 51) +#define RISCV_HWPROBE_EXT_ZFBFMIN (1ULL << 52) +#define RISCV_HWPROBE_EXT_ZVFBFMIN (1ULL << 53) +#define RISCV_HWPROBE_EXT_ZVFBFWMA (1ULL << 54) +#define RISCV_HWPROBE_EXT_ZICBOM (1ULL << 55) +#define RISCV_HWPROBE_EXT_ZAAMO (1ULL << 56) +#define RISCV_HWPROBE_EXT_ZALRSC (1ULL << 57) +#define RISCV_HWPROBE_EXT_ZABHA (1ULL << 58) #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) @@ -9033,6 +9056,22 @@ static int do_getdents64(abi_long dirfd, abi_long arg2, abi_long count) #define RISCV_HWPROBE_MISALIGNED_MASK (7 << 0) #define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6 +#define RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS 7 +#define RISCV_HWPROBE_KEY_TIME_CSR_FREQ 8 +#define RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF 9 +#define RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN 0 +#define RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED 1 +#define RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW 2 +#define RISCV_HWPROBE_MISALIGNED_SCALAR_FAST 3 +#define RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED 4 +#define RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF 10 +#define RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN 0 +#define RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW 2 +#define RISCV_HWPROBE_MISALIGNED_VECTOR_FAST 3 +#define RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED 4 +#define RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0 11 +#define RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE 12 +#define RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0 13 struct riscv_hwprobe { abi_llong key; @@ -9141,6 +9180,52 @@ static void risc_hwprobe_fill_pairs(CPURISCVState *env, RISCV_HWPROBE_EXT_ZACAS : 0; value |= cfg->ext_zicond ? RISCV_HWPROBE_EXT_ZICOND : 0; + value |= cfg->ext_zihintpause ? + RISCV_HWPROBE_EXT_ZIHINTPAUSE : 0; + value |= cfg->ext_zve32x ? + RISCV_HWPROBE_EXT_ZVE32X : 0; + value |= cfg->ext_zve32f ? + RISCV_HWPROBE_EXT_ZVE32F : 0; + value |= cfg->ext_zve64x ? + RISCV_HWPROBE_EXT_ZVE64X : 0; + value |= cfg->ext_zve64f ? + RISCV_HWPROBE_EXT_ZVE64F : 0; + value |= cfg->ext_zve64d ? + RISCV_HWPROBE_EXT_ZVE64D : 0; + value |= cfg->ext_zimop ? + RISCV_HWPROBE_EXT_ZIMOP : 0; + value |= cfg->ext_zca ? + RISCV_HWPROBE_EXT_ZCA : 0; + value |= cfg->ext_zcb ? + RISCV_HWPROBE_EXT_ZCB : 0; + value |= cfg->ext_zcd ? + RISCV_HWPROBE_EXT_ZCD : 0; + value |= cfg->ext_zcf ? + RISCV_HWPROBE_EXT_ZCF : 0; + value |= cfg->ext_zcmop ? + RISCV_HWPROBE_EXT_ZCMOP : 0; + value |= cfg->ext_zawrs ? + RISCV_HWPROBE_EXT_ZAWRS : 0; + value |= cfg->ext_supm ? + RISCV_HWPROBE_EXT_SUPM : 0; + value |= cfg->ext_zicntr ? + RISCV_HWPROBE_EXT_ZICNTR : 0; + value |= cfg->ext_zihpm ? + RISCV_HWPROBE_EXT_ZIHPM : 0; + value |= cfg->ext_zfbfmin ? + RISCV_HWPROBE_EXT_ZFBFMIN : 0; + value |= cfg->ext_zvfbfmin ? + RISCV_HWPROBE_EXT_ZVFBFMIN : 0; + value |= cfg->ext_zvfbfwma ? + RISCV_HWPROBE_EXT_ZVFBFWMA : 0; + value |= cfg->ext_zicbom ? + RISCV_HWPROBE_EXT_ZICBOM : 0; + value |= cfg->ext_zaamo ? + RISCV_HWPROBE_EXT_ZAAMO : 0; + value |= cfg->ext_zalrsc ? + RISCV_HWPROBE_EXT_ZALRSC : 0; + value |= cfg->ext_zabha ? + RISCV_HWPROBE_EXT_ZABHA : 0; __put_user(value, &pair->value); break; case RISCV_HWPROBE_KEY_CPUPERF_0: @@ -9150,6 +9235,10 @@ static void risc_hwprobe_fill_pairs(CPURISCVState *env, value = cfg->ext_zicboz ? cfg->cboz_blocksize : 0; __put_user(value, &pair->value); break; + case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE: + value = cfg->ext_zicbom ? cfg->cbom_blocksize : 0; + __put_user(value, &pair->value); + break; default: __put_user(-1, &pair->key); break; diff --git a/pc-bios/opensbi-riscv32-generic-fw_dynamic.bin b/pc-bios/opensbi-riscv32-generic-fw_dynamic.bin Binary files differindex b2e7400..02be3a7 100644 --- a/pc-bios/opensbi-riscv32-generic-fw_dynamic.bin +++ b/pc-bios/opensbi-riscv32-generic-fw_dynamic.bin diff --git a/pc-bios/opensbi-riscv64-generic-fw_dynamic.bin b/pc-bios/opensbi-riscv64-generic-fw_dynamic.bin Binary files differindex 018b473..cce35c6 100644 --- a/pc-bios/opensbi-riscv64-generic-fw_dynamic.bin +++ b/pc-bios/opensbi-riscv64-generic-fw_dynamic.bin diff --git a/roms/opensbi b/roms/opensbi -Subproject 43cace6c3671e5172d0df0a8963e552bb04b7b2 +Subproject a32a91069119e7a5aa31e6bc51d5e00860be3d8 diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index d055ddf..a877018 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -604,7 +604,7 @@ static void riscv_cpu_dump_state(CPUState *cs, FILE *f, int flags) } } } - if (riscv_has_ext(env, RVV) && (flags & CPU_DUMP_VPU)) { + if (riscv_cpu_cfg(env)->ext_zve32x && (flags & CPU_DUMP_VPU)) { static const int dump_rvv_csrs[] = { CSR_VSTART, CSR_VXSAT, diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h index 4a862da..2c22664 100644 --- a/target/riscv/cpu.h +++ b/target/riscv/cpu.h @@ -592,6 +592,7 @@ static inline int riscv_has_ext(CPURISCVState *env, target_ulong ext) extern const char * const riscv_int_regnames[]; extern const char * const riscv_int_regnamesh[]; extern const char * const riscv_fpr_regnames[]; +extern const char * const riscv_rvv_regnames[]; const char *riscv_cpu_get_trap_name(target_ulong cause, bool async); int riscv_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs, @@ -873,7 +874,7 @@ static inline void riscv_csr_write(CPURISCVState *env, int csrno, static inline target_ulong riscv_csr_read(CPURISCVState *env, int csrno) { target_ulong val = 0; - riscv_csrrw(env, csrno, &val, 0, 0, 0); + riscv_csrr(env, csrno, &val); return val; } diff --git a/target/riscv/csr.c b/target/riscv/csr.c index 3c8989f..5c91658 100644 --- a/target/riscv/csr.c +++ b/target/riscv/csr.c @@ -203,6 +203,8 @@ static RISCVException cfi_ss(CPURISCVState *env, int csrno) #if !defined(CONFIG_USER_ONLY) if (env->debugger) { return RISCV_EXCP_NONE; + } else if (env->virt_enabled) { + return RISCV_EXCP_VIRT_INSTRUCTION_FAULT; } #endif return RISCV_EXCP_ILLEGAL_INST; @@ -2003,7 +2005,8 @@ static RISCVException write_mstatus(CPURISCVState *env, int csrno, if (riscv_has_ext(env, RVF)) { mask |= MSTATUS_FS; } - if (riscv_has_ext(env, RVV)) { + + if (riscv_cpu_cfg(env)->ext_zve32x) { mask |= MSTATUS_VS; } diff --git a/target/riscv/helper.h b/target/riscv/helper.h index f712b1c..b785456 100644 --- a/target/riscv/helper.h +++ b/target/riscv/helper.h @@ -1101,14 +1101,14 @@ DEF_HELPER_6(vslidedown_vx_b, void, ptr, ptr, tl, ptr, env, i32) DEF_HELPER_6(vslidedown_vx_h, void, ptr, ptr, tl, ptr, env, i32) DEF_HELPER_6(vslidedown_vx_w, void, ptr, ptr, tl, ptr, env, i32) DEF_HELPER_6(vslidedown_vx_d, void, ptr, ptr, tl, ptr, env, i32) -DEF_HELPER_6(vslide1up_vx_b, void, ptr, ptr, tl, ptr, env, i32) -DEF_HELPER_6(vslide1up_vx_h, void, ptr, ptr, tl, ptr, env, i32) -DEF_HELPER_6(vslide1up_vx_w, void, ptr, ptr, tl, ptr, env, i32) -DEF_HELPER_6(vslide1up_vx_d, void, ptr, ptr, tl, ptr, env, i32) -DEF_HELPER_6(vslide1down_vx_b, void, ptr, ptr, tl, ptr, env, i32) -DEF_HELPER_6(vslide1down_vx_h, void, ptr, ptr, tl, ptr, env, i32) -DEF_HELPER_6(vslide1down_vx_w, void, ptr, ptr, tl, ptr, env, i32) -DEF_HELPER_6(vslide1down_vx_d, void, ptr, ptr, tl, ptr, env, i32) +DEF_HELPER_6(vslide1up_vx_b, void, ptr, ptr, i64, ptr, env, i32) +DEF_HELPER_6(vslide1up_vx_h, void, ptr, ptr, i64, ptr, env, i32) +DEF_HELPER_6(vslide1up_vx_w, void, ptr, ptr, i64, ptr, env, i32) +DEF_HELPER_6(vslide1up_vx_d, void, ptr, ptr, i64, ptr, env, i32) +DEF_HELPER_6(vslide1down_vx_b, void, ptr, ptr, i64, ptr, env, i32) +DEF_HELPER_6(vslide1down_vx_h, void, ptr, ptr, i64, ptr, env, i32) +DEF_HELPER_6(vslide1down_vx_w, void, ptr, ptr, i64, ptr, env, i32) +DEF_HELPER_6(vslide1down_vx_d, void, ptr, ptr, i64, ptr, env, i32) DEF_HELPER_6(vfslide1up_vf_h, void, ptr, ptr, i64, ptr, env, i32) DEF_HELPER_6(vfslide1up_vf_w, void, ptr, ptr, i64, ptr, env, i32) @@ -1284,3 +1284,8 @@ DEF_HELPER_4(vgmul_vv, void, ptr, ptr, env, i32) DEF_HELPER_5(vsm4k_vi, void, ptr, ptr, i32, env, i32) DEF_HELPER_4(vsm4r_vv, void, ptr, ptr, env, i32) DEF_HELPER_4(vsm4r_vs, void, ptr, ptr, env, i32) + +/* CFI (zicfiss) helpers */ +#ifndef CONFIG_USER_ONLY +DEF_HELPER_1(ssamoswap_disabled, void, env) +#endif diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index 71f98fb..f4b5460 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -3561,7 +3561,6 @@ static bool slideup_check(DisasContext *s, arg_rmrr *a) } GEN_OPIVX_TRANS(vslideup_vx, slideup_check) -GEN_OPIVX_TRANS(vslide1up_vx, slideup_check) GEN_OPIVI_TRANS(vslideup_vi, IMM_ZX, vslideup_vx, slideup_check) static bool slidedown_check(DisasContext *s, arg_rmrr *a) @@ -3572,9 +3571,56 @@ static bool slidedown_check(DisasContext *s, arg_rmrr *a) } GEN_OPIVX_TRANS(vslidedown_vx, slidedown_check) -GEN_OPIVX_TRANS(vslide1down_vx, slidedown_check) GEN_OPIVI_TRANS(vslidedown_vi, IMM_ZX, vslidedown_vx, slidedown_check) +typedef void gen_helper_vslide1_vx(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_ptr, + TCGv_env, TCGv_i32); + +#define GEN_OPIVX_VSLIDE1_TRANS(NAME, CHECK) \ +static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ +{ \ + if (CHECK(s, a)) { \ + static gen_helper_vslide1_vx * const fns[4] = { \ + gen_helper_##NAME##_b, gen_helper_##NAME##_h, \ + gen_helper_##NAME##_w, gen_helper_##NAME##_d, \ + }; \ + \ + TCGv_ptr dest, src2, mask; \ + TCGv_i64 src1; \ + TCGv_i32 desc; \ + uint32_t data = 0; \ + \ + dest = tcg_temp_new_ptr(); \ + mask = tcg_temp_new_ptr(); \ + src2 = tcg_temp_new_ptr(); \ + src1 = tcg_temp_new_i64(); \ + \ + data = FIELD_DP32(data, VDATA, VM, a->vm); \ + data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ + data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s); \ + data = FIELD_DP32(data, VDATA, VMA, s->vma); \ + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, \ + s->cfg_ptr->vlenb, data)); \ + \ + tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd)); \ + tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, a->rs2)); \ + tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0)); \ + tcg_gen_ext_tl_i64(src1, get_gpr(s, a->rs1, EXT_SIGN)); \ + \ + fns[s->sew](dest, mask, src1, src2, tcg_env, desc); \ + \ + tcg_gen_movi_tl(cpu_vstart, 0); \ + finalize_rvv_inst(s); \ + \ + return true; \ + } \ + return false; \ +} + +GEN_OPIVX_VSLIDE1_TRANS(vslide1up_vx, slideup_check) +GEN_OPIVX_VSLIDE1_TRANS(vslide1down_vx, slidedown_check) + /* Vector Floating-Point Slide Instructions */ static bool fslideup_check(DisasContext *s, arg_rmrr *a) { diff --git a/target/riscv/insn_trans/trans_rvzce.c.inc b/target/riscv/insn_trans/trans_rvzce.c.inc index c77c2b9..dd15af0 100644 --- a/target/riscv/insn_trans/trans_rvzce.c.inc +++ b/target/riscv/insn_trans/trans_rvzce.c.inc @@ -88,13 +88,13 @@ static bool trans_c_lbu(DisasContext *ctx, arg_c_lbu *a) static bool trans_c_lhu(DisasContext *ctx, arg_c_lhu *a) { REQUIRE_ZCB(ctx); - return gen_load(ctx, a, MO_UW); + return gen_load(ctx, a, MO_TEUW); } static bool trans_c_lh(DisasContext *ctx, arg_c_lh *a) { REQUIRE_ZCB(ctx); - return gen_load(ctx, a, MO_SW); + return gen_load(ctx, a, MO_TESW); } static bool trans_c_sb(DisasContext *ctx, arg_c_sb *a) @@ -106,7 +106,7 @@ static bool trans_c_sb(DisasContext *ctx, arg_c_sb *a) static bool trans_c_sh(DisasContext *ctx, arg_c_sh *a) { REQUIRE_ZCB(ctx); - return gen_store(ctx, a, MO_UW); + return gen_store(ctx, a, MO_TEUW); } #define X_S0 8 diff --git a/target/riscv/insn_trans/trans_rvzicfiss.c.inc b/target/riscv/insn_trans/trans_rvzicfiss.c.inc index b0096ad..f4a1c12 100644 --- a/target/riscv/insn_trans/trans_rvzicfiss.c.inc +++ b/target/riscv/insn_trans/trans_rvzicfiss.c.inc @@ -40,6 +40,7 @@ static bool trans_sspopchk(DisasContext *ctx, arg_sspopchk *a) tcg_gen_brcond_tl(TCG_COND_EQ, data, rs1, skip); tcg_gen_st_tl(tcg_constant_tl(RISCV_EXCP_SW_CHECK_BCFI_TVAL), tcg_env, offsetof(CPURISCVState, sw_check_code)); + gen_update_pc(ctx, 0); gen_helper_raise_exception(tcg_env, tcg_constant_i32(RISCV_EXCP_SW_CHECK)); gen_set_label(skip); @@ -90,7 +91,11 @@ static bool trans_ssamoswap_w(DisasContext *ctx, arg_amoswap_w *a) } if (!ctx->bcfi_enabled) { +#ifndef CONFIG_USER_ONLY + gen_helper_ssamoswap_disabled(tcg_env); +#else return false; +#endif } TCGv dest = dest_gpr(ctx, a->rd); @@ -115,7 +120,11 @@ static bool trans_ssamoswap_d(DisasContext *ctx, arg_amoswap_w *a) } if (!ctx->bcfi_enabled) { +#ifndef CONFIG_USER_ONLY + gen_helper_ssamoswap_disabled(tcg_env); +#else return false; +#endif } TCGv dest = dest_gpr(ctx, a->rd); diff --git a/target/riscv/kvm/kvm-cpu.c b/target/riscv/kvm/kvm-cpu.c index 5c19062..187c2c9 100644 --- a/target/riscv/kvm/kvm-cpu.c +++ b/target/riscv/kvm/kvm-cpu.c @@ -1588,7 +1588,7 @@ static void kvm_riscv_handle_sbi_dbcn(CPUState *cs, struct kvm_run *run) * Handle the case where a 32 bit CPU is running in a * 64 bit addressing env. */ - if (riscv_cpu_mxl(&cpu->env) == MXL_RV32) { + if (riscv_cpu_is_32bit(cpu)) { addr |= (uint64_t)run->riscv_sbi.args[2] << 32; } diff --git a/target/riscv/machine.c b/target/riscv/machine.c index 1600ec4..18d790a 100644 --- a/target/riscv/machine.c +++ b/target/riscv/machine.c @@ -131,7 +131,8 @@ static bool vector_needed(void *opaque) RISCVCPU *cpu = opaque; CPURISCVState *env = &cpu->env; - return riscv_has_ext(env, RVV); + return kvm_enabled() ? riscv_has_ext(env, RVV) : + riscv_cpu_cfg(env)->ext_zve32x; } static const VMStateDescription vmstate_vector = { @@ -400,6 +401,30 @@ static const VMStateDescription vmstate_ssp = { } }; +static bool sstc_timer_needed(void *opaque) +{ + RISCVCPU *cpu = opaque; + CPURISCVState *env = &cpu->env; + + if (!cpu->cfg.ext_sstc) { + return false; + } + + return env->stimer != NULL || env->vstimer != NULL; +} + +static const VMStateDescription vmstate_sstc = { + .name = "cpu/timer", + .version_id = 1, + .minimum_version_id = 1, + .needed = sstc_timer_needed, + .fields = (const VMStateField[]) { + VMSTATE_TIMER_PTR(env.stimer, RISCVCPU), + VMSTATE_TIMER_PTR(env.vstimer, RISCVCPU), + VMSTATE_END_OF_LIST() + } +}; + const VMStateDescription vmstate_riscv_cpu = { .name = "cpu", .version_id = 10, @@ -476,6 +501,7 @@ const VMStateDescription vmstate_riscv_cpu = { &vmstate_elp, &vmstate_ssp, &vmstate_ctr, + &vmstate_sstc, NULL } }; diff --git a/target/riscv/op_helper.c b/target/riscv/op_helper.c index 110292e..8382aa9 100644 --- a/target/riscv/op_helper.c +++ b/target/riscv/op_helper.c @@ -717,4 +717,53 @@ target_ulong helper_hyp_hlvx_wu(CPURISCVState *env, target_ulong addr) return cpu_ldl_code_mmu(env, addr, oi, ra); } +void helper_ssamoswap_disabled(CPURISCVState *env) +{ + int exception = RISCV_EXCP_ILLEGAL_INST; + + /* + * Here we follow the RISC-V CFI spec [1] to implement the exception type + * of ssamoswap* instruction. + * + * [1] RISC-V CFI spec v1.0, ch2.7 Atomic Swap from a Shadow Stack Location + * + * Note: We have already checked some conditions in trans_* functions: + * 1. The effective priv mode is not M-mode. + * 2. The xSSE specific to the effictive priv mode is disabled. + */ + if (!get_field(env->menvcfg, MENVCFG_SSE)) { + /* + * Disabled M-mode SSE always trigger illegal instruction when + * current priv mode is not M-mode. + */ + exception = RISCV_EXCP_ILLEGAL_INST; + goto done; + } + + if (!riscv_has_ext(env, RVS)) { + /* S-mode is not implemented */ + exception = RISCV_EXCP_ILLEGAL_INST; + goto done; + } else if (env->virt_enabled) { + /* + * VU/VS-mode with disabled xSSE will trigger the virtual instruction + * exception. + */ + exception = RISCV_EXCP_VIRT_INSTRUCTION_FAULT; + goto done; + } else { + /* + * U-mode with disabled S-mode SSE will trigger the illegal instruction + * exception. + * + * Note: S-mode is already handled in the disabled M-mode SSE case. + */ + exception = RISCV_EXCP_ILLEGAL_INST; + goto done; + } + +done: + riscv_raise_exception(env, exception, GETPC()); +} + #endif /* !CONFIG_USER_ONLY */ diff --git a/target/riscv/riscv-qmp-cmds.c b/target/riscv/riscv-qmp-cmds.c index 8a1856c..c499f9b 100644 --- a/target/riscv/riscv-qmp-cmds.c +++ b/target/riscv/riscv-qmp-cmds.c @@ -31,6 +31,10 @@ #include "qapi/qobject-input-visitor.h" #include "qapi/visitor.h" #include "qom/qom-qobject.h" +#include "qemu/ctype.h" +#include "qemu/qemu-print.h" +#include "monitor/hmp.h" +#include "monitor/hmp-target.h" #include "system/kvm.h" #include "system/tcg.h" #include "cpu-qom.h" @@ -240,3 +244,147 @@ CpuModelExpansionInfo *qmp_query_cpu_model_expansion(CpuModelExpansionType type, return expansion_info; } + +/* + * We have way too many potential CSRs and regs being added + * regularly to register them in a static array. + * + * Declare an empty array instead, making get_monitor_def() use + * the target_get_monitor_def() API directly. + */ +const MonitorDef monitor_defs[] = { { } }; +const MonitorDef *target_monitor_defs(void) +{ + return monitor_defs; +} + +static bool reg_is_ulong_integer(CPURISCVState *env, const char *name, + target_ulong *val, bool is_gprh) +{ + const char * const *reg_names; + target_ulong *vals; + + if (is_gprh) { + reg_names = riscv_int_regnamesh; + vals = env->gprh; + } else { + reg_names = riscv_int_regnames; + vals = env->gpr; + } + + for (int i = 0; i < 32; i++) { + g_autofree char *reg_name = g_strdup(reg_names[i]); + char *reg1 = strtok(reg_name, "/"); + char *reg2 = strtok(NULL, "/"); + + if (strcasecmp(reg1, name) == 0 || + (reg2 && strcasecmp(reg2, name) == 0)) { + *val = vals[i]; + return true; + } + } + + return false; +} + +static bool reg_is_u64_fpu(CPURISCVState *env, const char *name, uint64_t *val) +{ + if (qemu_tolower(name[0]) != 'f') { + return false; + } + + for (int i = 0; i < 32; i++) { + g_autofree char *reg_name = g_strdup(riscv_fpr_regnames[i]); + char *reg1 = strtok(reg_name, "/"); + char *reg2 = strtok(NULL, "/"); + + if (strcasecmp(reg1, name) == 0 || + (reg2 && strcasecmp(reg2, name) == 0)) { + *val = env->fpr[i]; + return true; + } + } + + return false; +} + +static bool reg_is_vreg(const char *name) +{ + if (qemu_tolower(name[0]) != 'v' || strlen(name) > 3) { + return false; + } + + for (int i = 0; i < 32; i++) { + if (strcasecmp(name, riscv_rvv_regnames[i]) == 0) { + return true; + } + } + + return false; +} + +int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval) +{ + CPURISCVState *env = &RISCV_CPU(cs)->env; + target_ulong val = 0; + uint64_t val64 = 0; + int i; + + if (reg_is_ulong_integer(env, name, &val, false) || + reg_is_ulong_integer(env, name, &val, true)) { + *pval = val; + return 0; + } + + if (reg_is_u64_fpu(env, name, &val64)) { + *pval = val64; + return 0; + } + + if (reg_is_vreg(name)) { + if (!riscv_cpu_cfg(env)->ext_zve32x) { + return -EINVAL; + } + + qemu_printf("Unable to print the value of vector " + "vreg '%s' from this API\n", name); + + /* + * We're returning 0 because returning -EINVAL triggers + * an 'unknown register' message in exp_unary() later, + * which feels ankward after our own error message. + */ + *pval = 0; + return 0; + } + + for (i = 0; i < ARRAY_SIZE(csr_ops); i++) { + RISCVException res; + int csrno = i; + + /* + * Early skip when possible since we're going + * through a lot of NULL entries. + */ + if (csr_ops[csrno].predicate == NULL) { + continue; + } + + if (strcasecmp(csr_ops[csrno].name, name) != 0) { + continue; + } + + res = riscv_csrrw_debug(env, csrno, &val, 0, 0); + + /* + * Rely on the smode, hmode, etc, predicates within csr.c + * to do the filtering of the registers that are present. + */ + if (res == RISCV_EXCP_NONE) { + *pval = val; + return 0; + } + } + + return -EINVAL; +} diff --git a/target/riscv/tcg/tcg-cpu.c b/target/riscv/tcg/tcg-cpu.c index 143ab07..1150bd1 100644 --- a/target/riscv/tcg/tcg-cpu.c +++ b/target/riscv/tcg/tcg-cpu.c @@ -417,12 +417,21 @@ static void riscv_cpu_validate_misa_priv(CPURISCVState *env, Error **errp) static void riscv_cpu_validate_v(CPURISCVState *env, RISCVCPUConfig *cfg, Error **errp) { + uint32_t min_vlen; uint32_t vlen = cfg->vlenb << 3; - if (vlen > RV_VLEN_MAX || vlen < 128) { + if (riscv_has_ext(env, RVV)) { + min_vlen = 128; + } else if (cfg->ext_zve64x) { + min_vlen = 64; + } else if (cfg->ext_zve32x) { + min_vlen = 32; + } + + if (vlen > RV_VLEN_MAX || vlen < min_vlen) { error_setg(errp, "Vector extension implementation only supports VLEN " - "in the range [128, %d]", RV_VLEN_MAX); + "in the range [%d, %d]", min_vlen, RV_VLEN_MAX); return; } @@ -432,6 +441,12 @@ static void riscv_cpu_validate_v(CPURISCVState *env, RISCVCPUConfig *cfg, "in the range [8, 64]"); return; } + + if (vlen < cfg->elen) { + error_setg(errp, "Vector extension implementation requires VLEN " + "to be greater than or equal to ELEN"); + return; + } } static void riscv_cpu_disable_priv_spec_isa_exts(RISCVCPU *cpu) @@ -661,7 +676,7 @@ void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) return; } - if (riscv_has_ext(env, RVV)) { + if (cpu->cfg.ext_zve32x) { riscv_cpu_validate_v(env, &cpu->cfg, &local_err); if (local_err != NULL) { error_propagate(errp, local_err); diff --git a/target/riscv/translate.c b/target/riscv/translate.c index 9ddef2d..6fc06c7 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -24,6 +24,7 @@ #include "exec/helper-gen.h" #include "exec/target_page.h" #include "exec/translator.h" +#include "accel/tcg/cpu-ldst.h" #include "exec/translation-block.h" #include "exec/log.h" #include "semihosting/semihost.h" @@ -1166,7 +1167,7 @@ static uint32_t opcode_at(DisasContextBase *dcbase, target_ulong pc) CPUState *cpu = ctx->cs; CPURISCVState *env = cpu_env(cpu); - return translator_ldl(env, &ctx->base, pc); + return cpu_ldl_code(env, pc); } #define SS_MMU_INDEX(ctx) (ctx->mem_idx | MMU_IDX_SS_WRITE) diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 7c67d67..41ea223 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -5198,11 +5198,11 @@ GEN_VEXT_VSLIE1UP(16, H2) GEN_VEXT_VSLIE1UP(32, H4) GEN_VEXT_VSLIE1UP(64, H8) -#define GEN_VEXT_VSLIDE1UP_VX(NAME, BITWIDTH) \ -void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ - CPURISCVState *env, uint32_t desc) \ -{ \ - vslide1up_##BITWIDTH(vd, v0, s1, vs2, env, desc); \ +#define GEN_VEXT_VSLIDE1UP_VX(NAME, BITWIDTH) \ +void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ + CPURISCVState *env, uint32_t desc) \ +{ \ + vslide1up_##BITWIDTH(vd, v0, s1, vs2, env, desc); \ } /* vslide1up.vx vd, vs2, rs1, vm # vd[0]=x[rs1], vd[i+1] = vs2[i] */ @@ -5249,11 +5249,11 @@ GEN_VEXT_VSLIDE1DOWN(16, H2) GEN_VEXT_VSLIDE1DOWN(32, H4) GEN_VEXT_VSLIDE1DOWN(64, H8) -#define GEN_VEXT_VSLIDE1DOWN_VX(NAME, BITWIDTH) \ -void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ - CPURISCVState *env, uint32_t desc) \ -{ \ - vslide1down_##BITWIDTH(vd, v0, s1, vs2, env, desc); \ +#define GEN_VEXT_VSLIDE1DOWN_VX(NAME, BITWIDTH) \ +void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ + CPURISCVState *env, uint32_t desc) \ +{ \ + vslide1down_##BITWIDTH(vd, v0, s1, vs2, env, desc); \ } /* vslide1down.vx vd, vs2, rs1, vm # vd[i] = vs2[i+1], vd[vl-1]=x[rs1] */ |