From 7804306c800112390a35af9c777bda86465a7be5 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 29 Mar 2023 22:24:55 +0800 Subject: virtio: Expose VIRTIO_F_IOMMU_PLATFORM in device features If we detect the VIRTIO_F_IOMMU_PLATFORM transport feature for a device, then expose it in the device features. Signed-off-by: Will Deacon [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) Cc: Bin Meng Link: https://android.googlesource.com/platform/external/u-boot/+/9693bd26bfcfe77d6a1295a561420e08c5daf019 Reviewed-by: Simon Glass --- drivers/virtio/virtio-uclass.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/virtio/virtio-uclass.c b/drivers/virtio/virtio-uclass.c index de9bc90..b3fb3db 100644 --- a/drivers/virtio/virtio-uclass.c +++ b/drivers/virtio/virtio-uclass.c @@ -336,7 +336,7 @@ static int virtio_uclass_child_pre_probe(struct udevice *vdev) /* Transport features always preserved to pass to finalize_features */ for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) if ((device_features & (1ULL << i)) && - (i == VIRTIO_F_VERSION_1)) + (i == VIRTIO_F_VERSION_1 || i == VIRTIO_F_IOMMU_PLATFORM)) __virtio_set_bit(vdev->parent, i); debug("(%s) final negotiated features supported %016llx\n", -- cgit v1.1 From da4e8bb09d6ea33618335106ea6a1fc515128123 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 29 Mar 2023 22:24:56 +0800 Subject: virtio: pci: Tear down VQs in virtio_pci_reset() The pages backing the virtqueues for virtio PCI devices are not freed on reset, despite the virtqueue structure being freed as part of the driver '->priv_auto' destruction at ->remove() time. Call virtio_pci_del_vqs() from virtio_pci_reset() to free the virtqueue pages before freeing the virtqueue structure itself. Signed-off-by: Will Deacon [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) Cc: Bin Meng Link: https://android.googlesource.com/platform/external/u-boot/+/5ed54ccd83cbffd0d8719ce650604b4e44b5b0d8 Reviewed-by: Simon Glass --- drivers/virtio/virtio_pci_modern.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index cfde400..3cdc2d2 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -218,25 +218,6 @@ static int virtio_pci_set_status(struct udevice *udev, u8 status) return 0; } -static int virtio_pci_reset(struct udevice *udev) -{ - struct virtio_pci_priv *priv = dev_get_priv(udev); - - /* 0 status means a reset */ - iowrite8(0, &priv->common->device_status); - - /* - * After writing 0 to device_status, the driver MUST wait for a read - * of device_status to return 0 before reinitializing the device. - * This will flush out the status write, and flush in device writes, - * including MSI-X interrupts, if any. - */ - while (ioread8(&priv->common->device_status)) - udelay(1000); - - return 0; -} - static int virtio_pci_get_features(struct udevice *udev, u64 *features) { struct virtio_pci_priv *priv = dev_get_priv(udev); @@ -363,6 +344,25 @@ static int virtio_pci_find_vqs(struct udevice *udev, unsigned int nvqs, return 0; } +static int virtio_pci_reset(struct udevice *udev) +{ + struct virtio_pci_priv *priv = dev_get_priv(udev); + + /* 0 status means a reset */ + iowrite8(0, &priv->common->device_status); + + /* + * After writing 0 to device_status, the driver MUST wait for a read + * of device_status to return 0 before reinitializing the device. + * This will flush out the status write, and flush in device writes, + * including MSI-X interrupts, if any. + */ + while (ioread8(&priv->common->device_status)) + udelay(1000); + + return virtio_pci_del_vqs(udev); +} + static int virtio_pci_notify(struct udevice *udev, struct virtqueue *vq) { struct virtio_pci_priv *priv = dev_get_priv(udev); -- cgit v1.1 From 75582fc2e69ac18f03a144d78c6d136d5594c7f5 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 29 Mar 2023 22:24:57 +0800 Subject: virtio: Allocate virtqueue in page-size units In preparation for explicit bouncing of virtqueue pages for devices advertising the VIRTIO_F_IOMMU_PLATFORM feature, introduce a couple of wrappers around virtqueue allocation and freeing operations, ensuring that buffers are handled in terms of page-size units. Signed-off-by: Will Deacon [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) Cc: Bin Meng Link: https://android.googlesource.com/platform/external/u-boot/+/b4bb5227d4cf4fdfcd8b4e1ff2692d3a54d1482a Reviewed-by: Simon Glass --- drivers/virtio/virtio_ring.c | 24 ++++++++++++++++++++---- include/virtio_ring.h | 16 +++++++++------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index f71bab7..5aeb13f 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -15,6 +15,17 @@ #include #include #include +#include + +static void *virtio_alloc_pages(struct udevice *vdev, u32 npages) +{ + return memalign(PAGE_SIZE, npages * PAGE_SIZE); +} + +static void virtio_free_pages(struct udevice *vdev, void *ptr, u32 npages) +{ + free(ptr); +} static unsigned int virtqueue_attach_desc(struct virtqueue *vq, unsigned int i, struct virtio_sg *sg, u16 flags) @@ -271,6 +282,8 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, unsigned int vring_align, struct udevice *udev) { + struct virtio_dev_priv *uc_priv = dev_get_uclass_priv(udev); + struct udevice *vdev = uc_priv->vdev; struct virtqueue *vq; void *queue = NULL; struct vring vring; @@ -283,7 +296,9 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, /* TODO: allocate each queue chunk individually */ for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) { - queue = memalign(PAGE_SIZE, vring_size(num, vring_align)); + size_t sz = vring_size(num, vring_align); + + queue = virtio_alloc_pages(vdev, DIV_ROUND_UP(sz, PAGE_SIZE)); if (queue) break; } @@ -293,7 +308,7 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, if (!queue) { /* Try to get a single page. You are my only hope! */ - queue = memalign(PAGE_SIZE, vring_size(num, vring_align)); + queue = virtio_alloc_pages(vdev, 1); } if (!queue) return NULL; @@ -303,7 +318,7 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, vq = __vring_new_virtqueue(index, vring, udev); if (!vq) { - free(queue); + virtio_free_pages(vdev, queue, DIV_ROUND_UP(vring.size, PAGE_SIZE)); return NULL; } debug("(%s): created vring @ %p for vq @ %p with num %u\n", udev->name, @@ -314,7 +329,8 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, void vring_del_virtqueue(struct virtqueue *vq) { - free(vq->vring.desc); + virtio_free_pages(vq->vdev, vq->vring.desc, + DIV_ROUND_UP(vq->vring.size, PAGE_SIZE)); free(vq->vring_desc_shadow); list_del(&vq->list); free(vq); diff --git a/include/virtio_ring.h b/include/virtio_ring.h index c77c212..8f8a55c 100644 --- a/include/virtio_ring.h +++ b/include/virtio_ring.h @@ -86,6 +86,7 @@ struct vring_used { struct vring { unsigned int num; + size_t size; struct vring_desc *desc; struct vring_avail *avail; struct vring_used *used; @@ -137,23 +138,24 @@ struct virtqueue { #define vring_used_event(vr) ((vr)->avail->ring[(vr)->num]) #define vring_avail_event(vr) (*(__virtio16 *)&(vr)->used->ring[(vr)->num]) +static inline unsigned int vring_size(unsigned int num, unsigned long align) +{ + return ((sizeof(struct vring_desc) * num + + sizeof(__virtio16) * (3 + num) + align - 1) & ~(align - 1)) + + sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num; +} + static inline void vring_init(struct vring *vr, unsigned int num, void *p, unsigned long align) { vr->num = num; + vr->size = vring_size(num, align); vr->desc = p; vr->avail = p + num * sizeof(struct vring_desc); vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] + sizeof(__virtio16) + align - 1) & ~(align - 1)); } -static inline unsigned int vring_size(unsigned int num, unsigned long align) -{ - return ((sizeof(struct vring_desc) * num + - sizeof(__virtio16) * (3 + num) + align - 1) & ~(align - 1)) + - sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num; -} - /* * The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX. * Assuming a given event_idx value from the other side, if we have just -- cgit v1.1 From a15d67e01abee6f091a4bcde22fd5e0649590944 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 29 Mar 2023 22:24:58 +0800 Subject: virtio: Add helper functions to attach/detach vring descriptors Move the attach and detach logic for manipulating vring descriptors out into their own functions so that we can later extend these to bounce the data for devices with VIRTIO_F_IOMMU_PLATFORM set. Signed-off-by: Will Deacon [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) Cc: Bin Meng Link: https://android.googlesource.com/platform/external/u-boot/+/f73258a4bfe968c5f935db45f2ec5cc0104ee796 Reviewed-by: Simon Glass --- drivers/virtio/virtio_ring.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 5aeb13f..1bd19ad 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -47,6 +47,10 @@ static unsigned int virtqueue_attach_desc(struct virtqueue *vq, unsigned int i, return desc_shadow->next; } +static void virtqueue_detach_desc(struct virtqueue *vq, unsigned int idx) +{ +} + int virtqueue_add(struct virtqueue *vq, struct virtio_sg *sgs[], unsigned int out_sgs, unsigned int in_sgs) { @@ -165,10 +169,12 @@ static void detach_buf(struct virtqueue *vq, unsigned int head) i = head; while (vq->vring_desc_shadow[i].flags & VRING_DESC_F_NEXT) { + virtqueue_detach_desc(vq, i); i = vq->vring_desc_shadow[i].next; vq->num_free++; } + virtqueue_detach_desc(vq, i); vq->vring_desc_shadow[i].next = vq->free_head; vq->free_head = head; -- cgit v1.1 From 37e53db38bdbeb58732e36d1efcdf58f6ce3a138 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 29 Mar 2023 22:24:59 +0800 Subject: virtio: Allocate bounce buffers for devices with VIRTIO_F_IOMMU_PLATFORM In preparation for bouncing virtio data for devices advertising the VIRTIO_F_IOMMU_PLATFORM feature, allocate an array of bounce buffer structures in the vring, one per descriptor. Signed-off-by: Will Deacon [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) Cc: Bin Meng Link: https://android.googlesource.com/platform/external/u-boot/+/3e052749e7c50c4c1a6014e645ae3b9be3710c07 Reviewed-by: Simon Glass --- drivers/virtio/virtio_ring.c | 25 ++++++++++++++++++++----- include/virtio_ring.h | 5 ++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 1bd19ad..de75786 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -6,6 +6,7 @@ * virtio ring implementation */ +#include #include #include #include @@ -292,6 +293,7 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, struct udevice *vdev = uc_priv->vdev; struct virtqueue *vq; void *queue = NULL; + struct bounce_buffer *bbs = NULL; struct vring vring; /* We assume num is a power of 2 */ @@ -320,17 +322,29 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, return NULL; memset(queue, 0, vring_size(num, vring_align)); - vring_init(&vring, num, queue, vring_align); - vq = __vring_new_virtqueue(index, vring, udev); - if (!vq) { - virtio_free_pages(vdev, queue, DIV_ROUND_UP(vring.size, PAGE_SIZE)); - return NULL; + if (virtio_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { + bbs = calloc(num, sizeof(*bbs)); + if (!bbs) + goto err_free_queue; } + + vring_init(&vring, num, queue, vring_align, bbs); + + vq = __vring_new_virtqueue(index, vring, udev); + if (!vq) + goto err_free_bbs; + debug("(%s): created vring @ %p for vq @ %p with num %u\n", udev->name, queue, vq, num); return vq; + +err_free_bbs: + free(bbs); +err_free_queue: + virtio_free_pages(vdev, queue, DIV_ROUND_UP(vring.size, PAGE_SIZE)); + return NULL; } void vring_del_virtqueue(struct virtqueue *vq) @@ -339,6 +353,7 @@ void vring_del_virtqueue(struct virtqueue *vq) DIV_ROUND_UP(vq->vring.size, PAGE_SIZE)); free(vq->vring_desc_shadow); list_del(&vq->list); + free(vq->vring.bouncebufs); free(vq); } diff --git a/include/virtio_ring.h b/include/virtio_ring.h index 8f8a55c..e8e9104 100644 --- a/include/virtio_ring.h +++ b/include/virtio_ring.h @@ -87,6 +87,7 @@ struct vring_used { struct vring { unsigned int num; size_t size; + struct bounce_buffer *bouncebufs; struct vring_desc *desc; struct vring_avail *avail; struct vring_used *used; @@ -146,10 +147,12 @@ static inline unsigned int vring_size(unsigned int num, unsigned long align) } static inline void vring_init(struct vring *vr, unsigned int num, void *p, - unsigned long align) + unsigned long align, + struct bounce_buffer *bouncebufs) { vr->num = num; vr->size = vring_size(num, align); + vr->bouncebufs = bouncebufs; vr->desc = p; vr->avail = p + num * sizeof(struct vring_desc); vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] + -- cgit v1.1 From b0a2fe148a70dc805f1e11702a08a7065418a8dd Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 29 Mar 2023 22:25:00 +0800 Subject: virtio: Use bounce buffers when VIRTIO_F_IOMMU_PLATFORM is set Devices advertising the VIRTIO_F_IOMMU_PLATFORM feature require platform-specific handling to configure their DMA transactions. When handling virtio descriptors for such a device, use bounce buffers to ensure that the underlying buffers are always aligned to and padded to PAGE_SIZE in preparation for platform specific handling at page granularity. Signed-off-by: Will Deacon [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) Cc: Bin Meng Link: https://android.googlesource.com/platform/external/u-boot/+/1eff171e613ee67dca71dbe97be7282e2db17011 Reviewed-by: Simon Glass --- drivers/virtio/virtio_ring.c | 48 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index de75786..c9adcce 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -28,14 +28,51 @@ static void virtio_free_pages(struct udevice *vdev, void *ptr, u32 npages) free(ptr); } +static int __bb_force_page_align(struct bounce_buffer *state) +{ + const ulong align_mask = PAGE_SIZE - 1; + + if ((ulong)state->user_buffer & align_mask) + return 0; + + if (state->len != state->len_aligned) + return 0; + + return 1; +} + static unsigned int virtqueue_attach_desc(struct virtqueue *vq, unsigned int i, struct virtio_sg *sg, u16 flags) { struct vring_desc_shadow *desc_shadow = &vq->vring_desc_shadow[i]; struct vring_desc *desc = &vq->vring.desc[i]; + void *addr; + + if (IS_ENABLED(CONFIG_BOUNCE_BUFFER) && vq->vring.bouncebufs) { + struct bounce_buffer *bb = &vq->vring.bouncebufs[i]; + unsigned int bbflags; + int ret; + + if (flags & VRING_DESC_F_WRITE) + bbflags = GEN_BB_WRITE; + else + bbflags = GEN_BB_READ; + + ret = bounce_buffer_start_extalign(bb, sg->addr, sg->length, + bbflags, PAGE_SIZE, + __bb_force_page_align); + if (ret) { + debug("%s: failed to allocate bounce buffer (length 0x%zx)\n", + vq->vdev->name, sg->length); + } + + addr = bb->bounce_buffer; + } else { + addr = sg->addr; + } /* Update the shadow descriptor. */ - desc_shadow->addr = (u64)(uintptr_t)sg->addr; + desc_shadow->addr = (u64)(uintptr_t)addr; desc_shadow->len = sg->length; desc_shadow->flags = flags; @@ -50,6 +87,15 @@ static unsigned int virtqueue_attach_desc(struct virtqueue *vq, unsigned int i, static void virtqueue_detach_desc(struct virtqueue *vq, unsigned int idx) { + struct vring_desc *desc = &vq->vring.desc[idx]; + struct bounce_buffer *bb; + + if (!IS_ENABLED(CONFIG_BOUNCE_BUFFER) || !vq->vring.bouncebufs) + return; + + bb = &vq->vring.bouncebufs[idx]; + bounce_buffer_stop(bb); + desc->addr = cpu_to_virtio64(vq->vdev, (u64)(uintptr_t)bb->user_buffer); } int virtqueue_add(struct virtqueue *vq, struct virtio_sg *sgs[], -- cgit v1.1