diff options
author | Thanos Makatos <thanos.makatos@nutanix.com> | 2020-11-13 10:17:36 -0500 |
---|---|---|
committer | Thanos Makatos <thanos.makatos@nutanix.com> | 2020-11-13 10:30:39 -0500 |
commit | 1c2b03b25769da6d84ced629c68cf9369a6dfdb8 (patch) | |
tree | de7457820f39573ccaab10dd15acd3e683992cf6 /kmod | |
parent | 847128e5161ea6c1bfcd9da1d9c5bee34d5da0b9 (diff) | |
download | libvfio-user-1c2b03b25769da6d84ced629c68cf9369a6dfdb8.zip libvfio-user-1c2b03b25769da6d84ced629c68cf9369a6dfdb8.tar.gz libvfio-user-1c2b03b25769da6d84ced629c68cf9369a6dfdb8.tar.bz2 |
remove kmod related files
Signed-off-by: Thanos Makatos <thanos.makatos@nutanix.com>
Diffstat (limited to 'kmod')
-rw-r--r-- | kmod/muser.c | 1867 |
1 files changed, 0 insertions, 1867 deletions
diff --git a/kmod/muser.c b/kmod/muser.c deleted file mode 100644 index 8ac2549..0000000 --- a/kmod/muser.c +++ /dev/null @@ -1,1867 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -/* - * Copyright (c) 2019, Nutanix Inc. All rights reserved. - * - * Author: Thanos Makatos <thanos@nutanix.com> - * Swapnil Ingle <swapnil.ingle@nutanix.com> - * Felipe Franciosi <felipe@nutanix.com> - * - */ - -#include <linux/cdev.h> -#include <linux/compat.h> -#include <linux/device.h> -#include <linux/file.h> -#include <linux/idr.h> -#include <linux/list.h> -#include <linux/miscdevice.h> -#include <linux/module.h> -#include <linux/mutex.h> -#include <linux/sched.h> -#include <linux/slab.h> -#include <linux/stat.h> -#include <linux/string.h> -#include <linux/uaccess.h> -#include <linux/wait.h> -#include <linux/vfio.h> -#include <linux/mdev.h> -#include <linux/pagemap.h> -#include <asm-generic/mman-common.h> -#include <linux/device.h> -#include <linux/version.h> -#include <linux/sched/mm.h> - -#include "muser.h" - -#define DRIVER_NAME "muser" - -#define NR_PAGES(x) (((x) + (PAGE_SIZE - 1)) >> PAGE_SHIFT) -#define MIN(a, b) ((a) < (b) ? (a):(b)) - -static struct muser { - struct class *class; - struct list_head mudev_list; - struct idr dev_idr; - struct cdev muser_cdev; - dev_t muser_devt; - struct device dev; - struct mutex muser_lock; -} muser; - -#define muser_log(func, fmt, ...) \ - func(&muser.dev, "%s: " fmt "\n", __func__, ## __VA_ARGS__) - -#define muser_dbg(fmt, ...) muser_log(dev_dbg, fmt, ## __VA_ARGS__) -#define muser_info(fmt, ...) muser_log(dev_info, fmt, ## __VA_ARGS__) -#define muser_warn(fmt, ...) muser_log(dev_warn, fmt, ## __VA_ARGS__) -#define muser_err(fmt, ...) muser_log(dev_err, fmt, ## __VA_ARGS__) -#define muser_alert(fmt, ...) muser_log(dev_alert, fmt, ## __VA_ARGS__) - -/* TODO come up with as better name? */ -/* - * FIXME len and nr_pages are confusing, we user either one or the other however - * they seem to serve the same purpose, fix. - */ -struct page_map { - struct page **pages; - int nr_pages; - size_t len; - int offset; -}; - -struct muser_dma_mapping { - unsigned long iova; - unsigned long length; - unsigned long offset; - union { - struct file *file; /* only valid while operation is ongoing */ - int fd; /* required for unmap */ - }; - struct list_head entry; -}; - -/* - * TODO do we use all members at the same time? Does it make sense to put some - * of them in a union? - */ -struct mudev_cmd { - enum muser_cmd_type type; /* copy of muser_cmd.type */ - struct muser_cmd muser_cmd; - struct page_map pg_map; - struct file **fds; - int *data_fds; - /* - * When libmuser completes an mmap call, we need to know the length - * in order to pass it to do_pin_pages. - */ - unsigned long mmap_len; - struct list_head entry; -}; - -/* - * TODO: - * Reorganise the members of this struct muser_dev - * mucmd_pending should be per filep context - * muser_dev should have a list of filep contexts instead of srv_opened - */ -struct muser_dev { - guid_t uuid; - int minor; - struct device *dev; - struct list_head dlist_entry; - struct list_head cmd_list; - struct mudev_cmd *mucmd_pending; - atomic_t srv_opened; - atomic_t mdev_opened; - struct mutex dev_lock; - struct mdev_device *mdev; - wait_queue_head_t user_wait_q; - struct semaphore sem; - struct notifier_block iommu_notifier; - struct muser_dma_mapping *dma_map; /* Current DMA operation */ - struct list_head dma_list; /* list of dma mappings */ - struct radix_tree_root devmem_tree; /* Device memory */ -}; - -static inline int muser_copyout(void __user *param, const void *address, - unsigned long size) -{ - int err = copy_to_user(param, address, size) ? -EFAULT : 0; - - if (unlikely(err)) - muser_dbg("failed to copy to user: %d", err); - - return err; -} - -static inline int muser_copyin(void *address, void __user *param, - unsigned long size) -{ - int err = copy_from_user(address, param, size) ? -EFAULT : 0; - - if (unlikely(err)) - muser_dbg("failed to copy from user: %d", err); - - return err; -} - -/* called with muser.muser_lock held */ -static struct muser_dev *__muser_search_dev(const guid_t *uuid) -{ - struct muser_dev *mudev; - - list_for_each_entry(mudev, &muser.mudev_list, dlist_entry) { - const uuid_le *u = &mudev->uuid; - - if (uuid_le_cmp(*u, *uuid) == 0) - return mudev; - } - - return NULL; -} - -static int muser_create_dev(const guid_t *uuid, struct mdev_device *mdev) -{ - struct muser_dev *mudev; - char uuid_str[UUID_STRING_LEN + 1]; - int minor; - int err = 0; - - mutex_lock(&muser.muser_lock); - mudev = __muser_search_dev(uuid); - if (mudev) { - err = -EEXIST; - goto out; - } - - mudev = kzalloc(sizeof(*mudev), GFP_KERNEL); - if (!mudev) { - err = -ENOMEM; - goto out; - } - - minor = idr_alloc(&muser.dev_idr, mudev, 0, MINORMASK + 1, GFP_KERNEL); - if (minor < 0) { - err = minor; - kfree(mudev); - goto out; - } - - sprintf(uuid_str, "%pUl", uuid); - mudev->dev = device_create(muser.class, NULL, - MKDEV(MAJOR(muser.muser_devt), minor), - mudev, "%s", uuid_str); - if (IS_ERR(mudev->dev)) { - err = PTR_ERR(mudev->dev); - idr_remove(&muser.dev_idr, minor); - kfree(mudev); - goto out; - } - - memcpy(&mudev->uuid, uuid, sizeof(mudev->uuid)); - mudev->minor = minor; - mudev->mdev = mdev; - mutex_init(&mudev->dev_lock); - sema_init(&mudev->sem, 0); - init_waitqueue_head(&mudev->user_wait_q); - INIT_LIST_HEAD(&mudev->cmd_list); - INIT_LIST_HEAD(&mudev->dma_list); - INIT_RADIX_TREE(&mudev->devmem_tree, GFP_KERNEL); - list_add(&mudev->dlist_entry, &muser.mudev_list); - mdev_set_drvdata(mdev, mudev); - - muser_info("new device %s", uuid_str); - -out: - mutex_unlock(&muser.muser_lock); - return err; -} - -/* called with muser.muser_lock held */ -static void __muser_deinit_dev(struct muser_dev *mudev) -{ - device_destroy(muser.class, - MKDEV(MAJOR(muser.muser_devt), mudev->minor)); - list_del(&mudev->dlist_entry); - idr_remove(&muser.dev_idr, mudev->minor); -} - -/* called with mudev.dev_lock held */ -static void __mudev_page_free(struct muser_dev *mudev, unsigned long pgnr) -{ - struct page *pg; - - pg = radix_tree_delete(&mudev->devmem_tree, pgnr); - if (WARN_ON(!pg)) - return; - - __free_page(pg); -} - -#define NR_INDICES 16 - -/* called with mudev.dev_lock held */ -static void __mudev_free_devmem(struct muser_dev *mudev) -{ - struct radix_tree_iter iter; - struct radix_tree_root *root = &mudev->devmem_tree; - unsigned long indices[NR_INDICES], index = 0; - void __rcu **slot; - int i, nr; - - do { - nr = 0; - radix_tree_for_each_slot(slot, root, &iter, index) { - indices[nr] = iter.index; - if (++nr == NR_INDICES) - break; - } - for (i = 0; i < nr; i++) { - index = indices[i]; - __mudev_page_free(mudev, index); - } - } while (nr > 0); -} - -static int muser_remove_dev(const uuid_le *uuid) -{ - struct muser_dev *mudev; - char uuid_str[UUID_STRING_LEN + 1]; - int err = 0; - - mutex_lock(&muser.muser_lock); - - mudev = __muser_search_dev(uuid); - if (!mudev) { - err = -ENOENT; - goto out; - } - - if (atomic_read(&mudev->mdev_opened) > 0 || - atomic_read(&mudev->srv_opened) > 0) { - err = -EBUSY; - goto out; - } - - mutex_lock(&mudev->dev_lock); - - WARN_ON(!list_empty(&mudev->cmd_list)); - __mudev_free_devmem(mudev); - __muser_deinit_dev(mudev); - - mutex_unlock(&mudev->dev_lock); - kfree(mudev); - - sprintf(uuid_str, "%pUl", uuid); - muser_info("removed muser device %s", uuid_str); - -out: - mutex_unlock(&muser.muser_lock); - return err; -} - -static ssize_t name_show(struct kobject *kobj, struct device *dev, char *buf) -{ - return sprintf(buf, "muser\n"); -} -MDEV_TYPE_ATTR_RO(name); - -static ssize_t device_api_show(struct kobject *kobj, struct device *dev, - char *buf) -{ - return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); -} -MDEV_TYPE_ATTR_RO(device_api); - -static struct attribute *mdev_types_attrs[] = { - &mdev_type_attr_name.attr, - &mdev_type_attr_device_api.attr, - NULL, -}; - -static struct attribute_group mdev_type_group = { - .name = "1", - .attrs = mdev_types_attrs, -}; - -struct attribute_group *mdev_type_groups[] = { - &mdev_type_group, - NULL, -}; - -static int muser_process_cmd(struct muser_dev *mudev, struct mudev_cmd *mucmd) -{ - int err; - - mucmd->type = mucmd->muser_cmd.type; - - /* Add command to mudev list of commands. */ - mutex_lock(&mudev->dev_lock); - list_add_tail(&mucmd->entry, &mudev->cmd_list); - mutex_unlock(&mudev->dev_lock); - - /* Wake up any sleepers */ - wake_up(&mudev->user_wait_q); - - /* - * TODO: decide what to do with timeouts - * Timeouts can happen if: - * 1. No server has attached to mudev - * 2. Processing of cmd takes more time than timeout - * - * Maybe use a while loop instead of goto - */ -retry: - err = down_timeout(&mudev->sem, msecs_to_jiffies(5000)); - if (err) { - struct mudev_cmd *pos, *tmp; - bool found = false; - - mutex_lock(&mudev->dev_lock); - list_for_each_entry_safe(pos, tmp, &mudev->cmd_list, entry) { - if (pos == mucmd) { - list_del(&mucmd->entry); - found = true; - break; - } - } - mutex_unlock(&mudev->dev_lock); - if (found) { - muser_err("giving up, no response for cmd %d", - mucmd->type); - } else { - muser_warn("server taking too long for cmd %d, retry", - mucmd->type); - goto retry; - } - } - - return err; -} - -int muser_create(struct kobject *kobj, struct mdev_device *mdev) -{ - /* XXX this should be taken out when upstreaming */ -#if LINUX_VERSION_CODE <= KERNEL_VERSION(4,19,67) - const uuid_le uuid = mdev_uuid(mdev); - return muser_create_dev(&uuid, mdev); -#else - return muser_create_dev(mdev_uuid(mdev), mdev); -#endif -} - -int muser_remove(struct mdev_device *mdev) -{ - /* XXX this should be taken out when upstreaming */ -#if LINUX_VERSION_CODE <= KERNEL_VERSION(4,19,67) - const uuid_le uuid = mdev_uuid(mdev); - return muser_remove_dev(&uuid); -#else - return muser_remove_dev(mdev_uuid(mdev)); -#endif -} - -static int do_pin_pages(char __user *buf, const size_t count, - int const writeable, struct page_map *const pg_map) -{ - unsigned long start; - unsigned long __user lbuf = (unsigned long __user)buf; - int i; - int err; - - BUG_ON(!buf); - BUG_ON(!pg_map); - - start = round_down(lbuf, PAGE_SIZE); - pg_map->nr_pages = (round_up(lbuf + count, PAGE_SIZE) - start) / - PAGE_SIZE; - pg_map->offset = lbuf - start; - pg_map->pages = kcalloc(pg_map->nr_pages, sizeof *(pg_map->pages), - GFP_KERNEL); - if (unlikely(!pg_map->pages)) { - muser_dbg("failed to allocate %d pages", pg_map->nr_pages); - return -ENOMEM; - } - err = get_user_pages_fast(start, pg_map->nr_pages, writeable, - pg_map->pages); - if (unlikely(err != pg_map->nr_pages)) { - for (i = 0; i < err; i++) - put_page(pg_map->pages[i]); - kfree(pg_map->pages); - muser_dbg("failed to get user pages: %d", err); - return -ENOMEM; - } - - return 0; -} - -static void unpin_pages(struct page_map *const pg_map) -{ - int i; - - if (!pg_map) - return; - - for (i = 0; i < pg_map->nr_pages; i++) - put_page(pg_map->pages[i]); - kfree(pg_map->pages); - pg_map->pages = NULL; -} - -static int _vm_insert_pages(struct vm_area_struct *const vma, - struct page *const pages[], const int nr_pages) -{ - int err = 0, i; - - for (i = 0; i < nr_pages; i++) { - BUG_ON(!pages[i]); - err = vm_insert_page(vma, vma->vm_start + i * PAGE_SIZE, - pages[i]); - if (unlikely(err)) { - muser_dbg("count=%d, anon=%d, slab=%d", - page_count(pages[i]), PageAnon(pages[i]), - PageSlab(pages[i])); - muser_dbg("failed to insert page at %lx: %d", - vma->vm_start + i * PAGE_SIZE, err); - unmap_kernel_range((unsigned long)vma->vm_start, - PAGE_SIZE); - break; - } - } - return err; -} - -static struct page *mudev_page_alloc(struct muser_dev *mudev, - unsigned long pgnr) -{ - struct page *pg; - int ret; - - pg = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_ZERO); - if (unlikely(!pg)) - return NULL; - - ret = radix_tree_insert(&mudev->devmem_tree, pgnr, pg); - if (ret) { - __free_page(pg); - return NULL; - } - - return pg; -} - -static int libmuser_mmap_dev(struct file *fp, struct vm_area_struct *vma) -{ - struct muser_dev *mudev = fp->private_data; - struct page *pg; - unsigned int nr_pages; - unsigned long cur_pgidx, end_pgidx; - unsigned long addr, *new_pgs; - int ret, i; - - WARN_ON(mudev == NULL); - nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; - - /* array to track new alloc'd pages, to be free'd in case of failure */ - new_pgs = kmalloc_array(nr_pages, sizeof(*new_pgs), GFP_KERNEL); - if (new_pgs == NULL) - return -ENOMEM; - - cur_pgidx = vma->vm_pgoff; - end_pgidx = cur_pgidx + nr_pages; - - muser_dbg("mmap_dev: end 0x%lX - start 0x%lX (0x%lX), off = 0x%lX", - vma->vm_end, vma->vm_start, vma->vm_end - vma->vm_start, - cur_pgidx); - - mutex_lock(&mudev->dev_lock); - for (i = 0; cur_pgidx < end_pgidx; cur_pgidx++, i++) { - pg = radix_tree_lookup(&mudev->devmem_tree, cur_pgidx); - if (pg == NULL) { - pg = mudev_page_alloc(mudev, cur_pgidx); - if (pg == NULL) { - muser_dbg("failed to alloc mudev page for index %ld", cur_pgidx); - i--; - ret = -ENOMEM; - goto free_pg; - } - new_pgs[i] = cur_pgidx; - } - - addr = vma->vm_start + (i << PAGE_SHIFT); - ret = vm_insert_page(vma, addr, pg); - if (unlikely(ret != 0)) { - muser_dbg("failed to insert page vma=%p addr=%ld page=%p: %d", - vma, addr, pg, ret); - goto free_pg; - } - } - mutex_unlock(&mudev->dev_lock); - - kfree(new_pgs); - return 0; - -free_pg: - for ( ; i >= 0; i--) - __mudev_page_free(mudev, new_pgs[i]); - mutex_unlock(&mudev->dev_lock); - kfree(new_pgs); - return ret; -} - -static int muser_process_dma_request(struct muser_dev *mudev, - struct muser_dma_mapping *dma_map, - int flags, int type) -{ - int err; - struct mudev_cmd mucmd = { - .type = type, - .muser_cmd = { - .type = type, - .mmap = { - .request = { - .addr = dma_map->iova, - .len = dma_map->length, - .offset = dma_map->offset, - .flags = flags} - } - } - }; - - if (type == MUSER_DMA_MMAP) - mucmd.muser_cmd.mmap.request.file = dma_map->file; - else if (type == MUSER_DMA_MUNMAP) - mucmd.muser_cmd.mmap.request.fd = dma_map->fd; - - err = muser_process_cmd(mudev, &mucmd); - if (unlikely(err)) - return err; - - return mucmd.muser_cmd.err; -} - -static int muser_process_dma_map(struct muser_dev *mudev, int flags) -{ - return muser_process_dma_request(mudev, mudev->dma_map, flags, - MUSER_DMA_MMAP); -} - -static int muser_process_dma_unmap(struct muser_dev *mudev, - struct muser_dma_mapping *dma_map) -{ - return muser_process_dma_request(mudev, dma_map, 0, MUSER_DMA_MUNMAP); -} - -/* - * Returns the struct file backing this VMA and sets the offset. - */ -static struct file* find_file_for_vaddr(unsigned long long vaddr, - unsigned long *offset) -{ - struct vm_area_struct *vma = NULL; - struct mm_struct *mm = get_task_mm(current); - if (unlikely(!mm)) - goto out; - vma = find_vma(mm, vaddr); - if (!vma) { - muser_dbg("no VMA for vaddr=%#llx", vaddr); - goto out; - } - if (!vma->vm_file) { - muser_dbg("no file for vaddr=%#llx", vaddr); - vma = NULL; - goto out; - } - if (IS_PRIVATE(vma->vm_file->f_inode)) { - muser_info("ignore private %pD\n", vma->vm_file); - vma = NULL; - goto out; - } - *offset = vaddr - vma->vm_start; -out: - if (mm) - mmput(mm); - if (vma) - return vma->vm_file; - return NULL; -} - -static int muser_iommu_dma_map(struct muser_dev *mudev, - struct vfio_iommu_type1_dma_map *map) -{ - struct muser_dma_mapping *dma_map; - int ret; - struct file *file; - unsigned long offset; - - muser_dbg("DMA map vaddr=%#llx iova=%#llx-%#llx", map->vaddr, map->iova, - map->iova + map->size); - - /* TODO: support multiple DMA map operations in parallel */ - /* - * TODO I think it's not necessary anymore to block concurrent DMA map - * operations as libmuser will mmap the file and then return in one go. - * Therefore mudev->dma_map can be removed. - */ - mutex_lock(&mudev->dev_lock); - if (mudev->dma_map != NULL) { - mutex_unlock(&mudev->dev_lock); - muser_dbg("another DMA map operation is ongoing"); - return -EBUSY; - } - - file = find_file_for_vaddr(map->vaddr, &offset); - if (file == NULL) { - mutex_unlock(&mudev->dev_lock); - return 0; - } - - dma_map = kmalloc(sizeof(struct muser_dma_mapping), GFP_KERNEL); - if (dma_map == NULL) { - mutex_unlock(&mudev->dev_lock); - return -ENOMEM; - } - mudev->dma_map = dma_map; - mutex_unlock(&mudev->dev_lock); - - dma_map->iova = map->iova; - dma_map->length = map->size; - dma_map->file = file; - dma_map->offset = offset; - - ret = muser_process_dma_map(mudev, map->flags); - if (ret) { - muser_dbg("libmuser failed to DMA map iova=%llx vaddr=%llx size=%llx: %d", - map->iova, map->vaddr, map->size, ret); - goto out; - } - - /* add to the dma_list */ - mutex_lock(&mudev->dev_lock); - list_add_tail(&dma_map->entry, &mudev->dma_list); - mudev->dma_map = NULL; - mutex_unlock(&mudev->dev_lock); - return 0; - -out: - kfree(dma_map); - mutex_lock(&mudev->dev_lock); - mudev->dma_map = NULL; - mutex_unlock(&mudev->dev_lock); - return ret; -} - -/* called with mudev.dev_lock held */ -static struct muser_dma_mapping *__find_dma_map(struct muser_dev *mudev, - unsigned long iova) -{ - struct muser_dma_mapping *dma_map; - - list_for_each_entry(dma_map, &mudev->dma_list, entry) { - if (dma_map->iova == iova) - return dma_map; - } - return NULL; -} - -static int muser_iommu_dma_unmap(struct muser_dev *const mudev, - struct vfio_iommu_type1_dma_unmap *const unmap) -{ - int err; - unsigned long len; - struct muser_dma_mapping *dma_map; - - mutex_lock(&mudev->dev_lock); - dma_map = __find_dma_map(mudev, unmap->iova); - if (!dma_map) { - /* - * XXX We no longer use vfio_pin_pages() so VFIO doesn't send - * DMA unmap events at all. We've patched vfio_iommu_type1 to - * send DMA unmap events even if we haven't pinned any of the - * pages of a particular region (e.g. the VMA is not - * shareable), so we have to ignore requests for such regions. - * This behaviormight be temprorary, depending on whether or - * not this solution gets accepted. For more information see: - * https://www.redhat.com/archives/vfio-users/2020-February/msg00016.html. - */ - mutex_unlock(&mudev->dev_lock); - return 0; - } - list_del(&dma_map->entry); - mutex_unlock(&mudev->dev_lock); - - len = dma_map->length; - err = muser_process_dma_unmap(mudev, dma_map); - if (unlikely(err)) - muser_dbg("failed to request libmuser to munmap iova=%#llx-%#llx: %d", - unmap->iova, unmap->size, err); - - kfree(dma_map); - - /* XXX: Do we need this? */ - unmap->size = len; - return err; -} - -/* - * FIXME There can be multiple DMA map calls per device. If each of these calls - * are serialised (this can be enforced by muser), then we tell libmuser to - * mmap the control device. Do we need to distinguish between the different - * DMA map calls at this stage if we can enforce only one outstanding DMA map - * call? - */ -static int muser_iommu_notifier(struct notifier_block *nb, unsigned long action, - void *data) -{ - struct muser_dev *mudev; - int err; - - BUG_ON(!nb); - BUG_ON(!data); - - mudev = container_of(nb, struct muser_dev, iommu_notifier); - switch (action) { - case VFIO_IOMMU_NOTIFY_DMA_MAP: - err = muser_iommu_dma_map(mudev, - (struct vfio_iommu_type1_dma_map *) - data); - break; - case VFIO_IOMMU_NOTIFY_DMA_UNMAP: - err = muser_iommu_dma_unmap(mudev, - (struct vfio_iommu_type1_dma_unmap - *)data); - break; - default: - muser_dbg("bad action=%lx", action); - err = -EINVAL; - } - - if (unlikely(err)) - return NOTIFY_BAD; - return NOTIFY_OK; -} - -static int register_notifier(struct mdev_device *const mdev) -{ - unsigned long events = - VFIO_IOMMU_NOTIFY_DMA_MAP | VFIO_IOMMU_NOTIFY_DMA_UNMAP; - struct muser_dev *const mudev = mdev_get_drvdata(mdev); - - memset(&mudev->iommu_notifier, 0, sizeof(mudev->iommu_notifier)); - mudev->iommu_notifier.notifier_call = muser_iommu_notifier; - return vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, - &events, &mudev->iommu_notifier); -} - -static int dma_unmap_all(struct muser_dev *mudev) -{ - struct muser_dma_mapping *dma_map; - unsigned long length; - LIST_HEAD(head); - - /* - * TODO: Cleanup - * Use better list functions like: - * list_replace()/list_replace_init() - * list_for_each_entry_safe() - */ - - mutex_lock(&mudev->dev_lock); - while (!list_empty(&mudev->dma_list)) { - dma_map = list_first_entry(&mudev->dma_list, - struct muser_dma_mapping, entry); - list_move(&dma_map->entry, &head); - } - mutex_unlock(&mudev->dev_lock); - - while (!list_empty(&head)) { - dma_map = list_first_entry(&head, struct muser_dma_mapping, - entry); - list_del(&dma_map->entry); - length = dma_map->length; - kfree(dma_map); - } - return 0; -} - -int muser_open(struct mdev_device *mdev) -{ - int err; - struct muser_dev *mudev = mdev_get_drvdata(mdev); - - WARN_ON(mudev == NULL); - - if (atomic_cmpxchg(&mudev->mdev_opened, 0, 1) != 0) { - muser_dbg("device already open"); - return -EBUSY; - } - - if (!try_module_get(THIS_MODULE)) { - atomic_dec(&mudev->mdev_opened); - return -ENODEV; - } - - err = register_notifier(mdev); - if (unlikely(err)) { - int err2; - /* - * TODO we might have triggered some notifiers which will have - * caused libmuser to mmap. If open fails then libmuser dies - * therefore things get automatically cleaned up (e.g. - * vfio_unpin etc.)? - */ - atomic_dec(&mudev->mdev_opened); - module_put(THIS_MODULE); - - muser_dbg("failed to register notifier: %d", err); - err2 = dma_unmap_all(mudev); - if (unlikely(err2)) - muser_dbg("failed to DMA unmap all regions: %d", err2); - err2 = vfio_unregister_notifier(mdev_dev(mdev), - VFIO_IOMMU_NOTIFY, - &mudev->iommu_notifier); - if (unlikely(err2)) - muser_info("failed to unregister notifier: %d", err); - } - - - return err; -} - -void muser_close(struct mdev_device *mdev) -{ - struct muser_dev *mudev = mdev_get_drvdata(mdev); - int err; - - err = dma_unmap_all(mudev); - if (unlikely(err)) - muser_alert("failed to remove one or more DMA maps"); - - err = vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, - &mudev->iommu_notifier); - if (unlikely(err)) - muser_info("failed to unregister notifier: %d", err); - - WARN_ON(atomic_read(&mudev->mdev_opened) == 0); - atomic_dec(&mudev->mdev_opened); - - /* TODO: Replace any pending mucmd back in cmd_list. */ - module_put(THIS_MODULE); -} - -static int -pin_pages(struct mudev_cmd *mucmd, char __user *buf, size_t count, - int writeable) -{ - mucmd->pg_map.len = count; - return do_pin_pages(buf, count, writeable, &mucmd->pg_map); -} - -void dump_buffer(unsigned char const *const buf, uint32_t count) -{ -#if defined(DEBUG) - /* - * TODO would be nice to add an option to print_hex_dump to hide - * repeated lines, e.g. like od(1) - */ - print_hex_dump(KERN_INFO, "", DUMP_PREFIX_NONE, 4, 1, buf, count, - false); -#endif -} - -static ssize_t muser_read(struct mdev_device *mdev, char __user *buf, - size_t count, loff_t *ppos) -{ - struct muser_dev *mudev = mdev_get_drvdata(mdev); - struct mudev_cmd mucmd = { 0 }; - int err; - ssize_t _count; - - WARN_ON(mudev == NULL); - - /* Setup mucmd and pin pages of the calling context. */ - mucmd.type = MUSER_READ; - err = pin_pages(&mucmd, buf, count, 1); - if (err != 0) { - muser_dbg("failed to pin pages: %d", err); - return err; - } - - /* Setup muser_cmd for server context. */ - mucmd.muser_cmd.type = MUSER_READ; - mucmd.muser_cmd.rw.count = count; - mucmd.muser_cmd.rw.pos = *ppos; - - muser_dbg("R %lx@%llx", mucmd.muser_cmd.rw.count, - mucmd.muser_cmd.rw.pos); - - /* TODO: move following into function */ - - /* Process mudev_cmd in libmuser context */ - err = muser_process_cmd(mudev, &mucmd); - if (unlikely(err != 0)) - _count = err; - else - _count = mucmd.muser_cmd.err; - - if (_count < 0) - muser_dbg("failed to process read: %d, %d", err, - mucmd.muser_cmd.err); - - *ppos = mucmd.muser_cmd.rw.pos; - - if (_count > 0) { - muser_dbg("received 0x%lx bytes from user space (0x%lx)", - _count, mucmd.muser_cmd.rw.count); - dump_buffer(buf, _count); - } - - unpin_pages(&mucmd.pg_map); - - return _count; -} - -ssize_t muser_write(struct mdev_device *mdev, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct muser_dev *mudev = mdev_get_drvdata(mdev); - struct mudev_cmd mucmd = { 0 }; - int err; - size_t _count = count; - loff_t _pos = *ppos; - - muser_dbg("W %lx@%llx", count, *ppos); - dump_buffer(buf, count); - - /* Setup mucmd and pin pages of the calling context. */ - mucmd.type = MUSER_WRITE; - err = pin_pages(&mucmd, (char __user *)buf, count, 0); - if (err != 0) - return err; - - /* Setup muser_cmd for libmuser context. */ - mucmd.muser_cmd.type = MUSER_WRITE; - mucmd.muser_cmd.rw.count = count; - mucmd.muser_cmd.rw.pos = *ppos; - - /* Process mudev_cmd in server context. */ - err = muser_process_cmd(mudev, &mucmd); - if (err != 0) - count = -1; - *ppos = mucmd.muser_cmd.rw.pos; - - unpin_pages(&mucmd.pg_map); - - if (mucmd.muser_cmd.err) - muser_info("PCI config write %ld@0x%llx not handled: %d", - _count, _pos, mucmd.muser_cmd.err); - - return count; -} - -static int bounce_fds(struct mudev_cmd *mucmd, void __user *data, - int user_data_size) -{ - int count = mucmd->muser_cmd.ioctl.data.irq_set.count; - int data_size = count * sizeof(int32_t); - int *user_fds; - int i; - int ret = 0; - - if (user_data_size < data_size) - return -EINVAL; - - mucmd->fds = kcalloc(count, sizeof(*mucmd->fds), GFP_KERNEL); - if (mucmd->fds == NULL) - return -ENOMEM; - - user_fds = memdup_user(data, data_size); - if (IS_ERR(user_fds)) { - kfree(mucmd->fds); - mucmd->fds = NULL; - return PTR_ERR(user_fds); - } - - for (i = 0; i < count; i++) { - if (user_fds[i] == -1) - continue; - mucmd->fds[i] = fget(user_fds[i]); - if (mucmd->fds[i] == NULL) { - ret = -EBADF; - goto err; - } - } - - kfree(user_fds); - - return 0; - -err: - for (i--; i >= 0; i--) - fput(mucmd->fds[i]); - kfree(user_fds); - kfree(mucmd->fds); - mucmd->fds = NULL; - - return ret; -} - -static ssize_t get_minsz(unsigned int cmd) -{ - switch (cmd) { - case VFIO_DEVICE_GET_INFO: - return offsetofend(struct vfio_device_info, num_irqs); - case VFIO_DEVICE_GET_REGION_INFO: - return offsetofend(struct vfio_region_info, offset); - case VFIO_DEVICE_GET_IRQ_INFO: - return offsetofend(struct vfio_irq_info, count); - case VFIO_DEVICE_SET_IRQS: - return offsetofend(struct vfio_irq_set, count); - } - return -EOPNOTSUPP; -} - -static ssize_t get_argsz(unsigned int cmd, struct mudev_cmd *mucmd) -{ - switch (cmd) { - case VFIO_DEVICE_GET_INFO: - return mucmd->muser_cmd.ioctl.data.dev_info.argsz; - case VFIO_DEVICE_GET_REGION_INFO: - return mucmd->muser_cmd.ioctl.data.reg_info.argsz; - case VFIO_DEVICE_GET_IRQ_INFO: - return mucmd->muser_cmd.ioctl.data.irq_info.argsz; - case VFIO_DEVICE_SET_IRQS: - return mucmd->muser_cmd.ioctl.data.irq_set.argsz; - } - - return -EOPNOTSUPP; -} - -static int muser_ioctl_setup_cmd(struct mudev_cmd *mucmd, unsigned int cmd, - unsigned long arg) -{ - ssize_t argsz, minsz; - int err; - - /* Determine smallest argsz we need for this command. */ - minsz = get_minsz(cmd); - if (minsz < 0) - return minsz; - - /* Copy caller-provided arg. */ - err = muser_copyin(&mucmd->muser_cmd.ioctl.data, (void __user *)arg, - minsz); - if (unlikely(err)) - return err; - - /* Fetch argsz provided by caller. */ - argsz = get_argsz(cmd, mucmd); - if (argsz < 0) - return argsz; - - /* Ensure provided size is at least the minimum required. */ - if (argsz < minsz) - return -EINVAL; - - /* Fetch potential data provided on SET_IRQS. */ - if (cmd == VFIO_DEVICE_SET_IRQS) { - unsigned int flags = mucmd->muser_cmd.ioctl.data.irq_set.flags; - - switch ((flags & VFIO_IRQ_SET_DATA_TYPE_MASK)) { - case VFIO_IRQ_SET_DATA_NONE: - /* FIXME */ - muser_warn("ignore DATA_NONE index=%d start=%d count=%d", - mucmd->muser_cmd.ioctl.data.irq_set.index, - mucmd->muser_cmd.ioctl.data.irq_set.start, - mucmd->muser_cmd.ioctl.data.irq_set.count); - break; - case VFIO_IRQ_SET_DATA_EVENTFD: - /* Lookup eventfds and bounce references to mucmd. */ - err = bounce_fds(mucmd, (void __user *) (arg + minsz), - argsz - minsz); - if (err) { - muser_dbg("failed to bounce fds: %d", err); - return err; - } - break; - default: - muser_warn("ignore flags=0x%x", flags); - } - } - - /* Pin pages of the calling context. */ - err = pin_pages(mucmd, (char __user *)arg, argsz, 1); - if (unlikely(err)) { - muser_dbg("failed to pin pages: %d", err); - return err; - } - - return err; -} - -static long muser_ioctl(struct mdev_device *mdev, unsigned int cmd, - unsigned long arg) -{ - struct muser_dev *mudev = mdev_get_drvdata(mdev); - struct mudev_cmd mucmd = { 0 }; - int err; - - muser_dbg("mdev=%p, cmd=%u, arg=0x%lX", mdev, cmd, arg); - - if (cmd == VFIO_DEVICE_RESET) { - if (!device_trylock(mudev->dev)) - return -EAGAIN; - } else { - err = muser_ioctl_setup_cmd(&mucmd, cmd, arg); - if (err) - return err; - } - - /* Setup common mucmd records. */ - mucmd.type = MUSER_IOCTL; - mucmd.muser_cmd.type = MUSER_IOCTL; - mucmd.muser_cmd.ioctl.vfio_cmd = cmd; - - /* Process mudev_cmd in server context. */ - err = muser_process_cmd(mudev, &mucmd); - if (err != 0) { - muser_dbg("failed to process command: %d", err); - err = -1; - } - - if (cmd == VFIO_DEVICE_RESET) { - device_unlock(mudev->dev); - } else { - /* Release resources. */ - unpin_pages(&mucmd.pg_map); - - /* maybe allocated for VFIO_IRQ_SET_DATA_EVENTFD */ - kfree(mucmd.fds); - kfree(mucmd.data_fds); - } - - return err; -} - -static int muser_mmap(struct mdev_device *const mdev, - struct vm_area_struct *const vma) -{ - struct muser_dev *mudev = mdev_get_drvdata(mdev); - struct mudev_cmd mucmd = { 0 }; - int err; - - BUG_ON(!mudev); - BUG_ON(!vma); - - /* - * Checking vm_flags cannot be easily done in user space as we can't - * access mm.h, so we have to do it here. Maybe implement the reverse - * of calc_vm_prot_bits/calc_vm_flag_bits? - */ - if ((vma->vm_flags & ~(VM_READ | VM_WRITE | VM_SHARED | VM_MAYREAD | - VM_MAYWRITE | VM_MAYEXEC | VM_MAYSHARE))) { - muser_dbg("bag flags=0x%lx", vma->vm_flags); - return -EINVAL; - } - - mucmd.type = MUSER_MMAP; - mucmd.mmap_len = vma->vm_end - vma->vm_start; - - mucmd.muser_cmd.type = MUSER_MMAP; - mucmd.muser_cmd.mmap.request.addr = vma->vm_pgoff << PAGE_SHIFT; - mucmd.muser_cmd.mmap.request.len = vma->vm_end - vma->vm_start; - - /* Process mudev_cmd in server context. */ - err = muser_process_cmd(mudev, &mucmd); - if (likely(err == 0)) { - err = mucmd.muser_cmd.err; - } - if (unlikely(err != 0)) { - muser_info("failed to mmap %#lx-%#lx: %d", - mucmd.muser_cmd.mmap.request.addr, - mucmd.muser_cmd.mmap.request.addr + mucmd.muser_cmd.mmap.request.len, - err); - return err; - } - - return _vm_insert_pages(vma, mucmd.pg_map.pages, mucmd.pg_map.nr_pages); -} - -struct mdev_parent_ops muser_mdev_fops = { - .owner = THIS_MODULE, - .supported_type_groups = mdev_type_groups, - .create = muser_create, - .remove = muser_remove, - .open = muser_open, - .release = muser_close, - .read = muser_read, - .write = muser_write, - .ioctl = muser_ioctl, - .mmap = muser_mmap, -}; - -/* copy vfio-client pages(mucmd.pg_map) to server(arg) */ -static int bounce_out(void __user *arg, size_t argsz, struct mudev_cmd *mucmd) -{ - unsigned long to_copy, left; - void __user *to; - void *from; - unsigned int offset; - int i, ret = 0; - - left = mucmd->pg_map.len; - if (argsz < left) - return -EINVAL; - - offset = mucmd->pg_map.offset; - - for (i = 0; i < mucmd->pg_map.nr_pages && ret == 0; i++) { - to_copy = min(left, PAGE_SIZE - offset); - to = arg + (mucmd->pg_map.len - left); - from = page_to_virt(mucmd->pg_map.pages[i]) + offset; - - ret = muser_copyout(to, from, to_copy); - if (ret) - return ret; - - left -= to_copy; - - /* Must be zero after first iteration. */ - offset = 0; - } - WARN_ON(left != 0); - - return 0; -} - -/* - * copy from server(ubuf) to vfio-client pages(mucmd.pg_map) - * skip seek bytes from destination before copying. - * - * @page_map: map representing vfio-client pages - * @ubuf : user buffer to copy from - * @bufsz : size of ubuf - * @seek : bytes to be skip from page_map before copy - */ -int bounce_in_seek(struct page_map *page_map, void __user *ubuf, size_t bufsz, - size_t seek) -{ - unsigned long to_copy = 0; - void __user *from = ubuf; - void *to; - size_t total, offset, pgoff; - int pgnr, i, ret; - - if (page_map->len < bufsz) - return -ENOSPC; - - pgnr = NR_PAGES(seek) - 1; - pgoff = seek & ~PAGE_SIZE; - offset = page_map->offset; - - if (!pgnr) - offset += pgoff; - else - offset = pgoff; - - total = bufsz; - for (i = pgnr; i < page_map->nr_pages; i++) { - to = page_to_virt(page_map->pages[i]) + offset; - from += to_copy; - to_copy = min(total, PAGE_SIZE - offset); - - ret = muser_copyin(to, from, to_copy); - if (ret) - return ret; - - total -= to_copy; - offset = 0; - } - - return 0; -} - -/* copy from server(uaddr) to vfio-client pages(mucmd.pg_map) */ -static int bounce_in(struct mudev_cmd *mucmd, void __user *uaddr) -{ - unsigned long to_copy, left; - void __user *from; - void *to; - unsigned int offset; - int i, ret; - - left = mucmd->pg_map.len; - offset = mucmd->pg_map.offset; - - for (i = 0; i < mucmd->pg_map.nr_pages; i++) { - to_copy = min(left, PAGE_SIZE - offset); - from = uaddr + (mucmd->pg_map.len - left); - to = page_to_virt(mucmd->pg_map.pages[i]) + offset; - - ret = muser_copyin(to, from, to_copy); - if (ret) - return ret; - - left -= to_copy; - - /* Must be zero after first iteration. */ - offset = 0; - } - WARN_ON(left != 0); - - return 0; -} - -static long install_fds(struct mudev_cmd *mucmd) -{ - int count = mucmd->muser_cmd.ioctl.data.irq_set.count; - int i; - long ret; - - mucmd->data_fds = kcalloc(count, sizeof(int32_t), GFP_KERNEL); - if (mucmd->data_fds == NULL) - return -ENOMEM; - - for (i = 0; i < count; i++) { - if (mucmd->fds[i] == NULL) { - mucmd->data_fds[i] = -1; - continue; - } - mucmd->data_fds[i] = get_unused_fd_flags(0); - if (mucmd->data_fds[i] < 0) { - ret = mucmd->data_fds[i]; - muser_err("unable to get unused fd: %ld", ret); - goto err; - } - fd_install(mucmd->data_fds[i], mucmd->fds[i]); - } - - return 0; - -err: - for (i--; i >= 0; i--) - put_unused_fd(mucmd->data_fds[i]); - kfree(mucmd->data_fds); - - return ret; -} - -static inline int maybe_install_fds(struct mudev_cmd *mucmd) -{ - unsigned int flags = mucmd->muser_cmd.ioctl.data.irq_set.flags; - long ret = 0; - - if ((mucmd->muser_cmd.type == MUSER_IOCTL) && - (mucmd->muser_cmd.ioctl.vfio_cmd == VFIO_DEVICE_SET_IRQS)) { - ret = -EINVAL; - switch ((flags & VFIO_IRQ_SET_DATA_TYPE_MASK)) { - case VFIO_IRQ_SET_DATA_NONE: - /* FIXME */ - muser_warn("ignore DATA_NONE index=%d start=%d count=%d", - mucmd->muser_cmd.ioctl.data.irq_set.index, - mucmd->muser_cmd.ioctl.data.irq_set.start, - mucmd->muser_cmd.ioctl.data.irq_set.count); - ret = 0; - break; - case VFIO_IRQ_SET_DATA_EVENTFD: - ret = install_fds(mucmd); - if (unlikely(ret)) - muser_dbg("failed to install fds: %ld", ret); - break; - default: - muser_warn("bad flags=0x%x", flags); - /* TODO: SET_DATA_BOOL */ - } - } - - return ret; -} - -static inline int mmap_done(struct mudev_cmd * const mucmd) -{ - struct muser_cmd *cmd = &mucmd->muser_cmd; - char __user *addr = (char __user *) cmd->mmap.response; - int ret; - - ret = do_pin_pages(addr, mucmd->mmap_len, 1, &mucmd->pg_map); - if (ret) { - muser_alert("failed to pin pages: %d", ret); - mucmd->pg_map.pages = NULL; - mucmd->pg_map.nr_pages = 0; - } - - return ret; -} - -static int fd_install_dma_map(struct file *file) -{ - int fd; - fd = get_unused_fd_flags(0); - if (fd < 0) { - muser_err("unable to get unused fd: %d", fd); - return fd; - } - get_file(file); - fd_install(fd, file); - return fd; -} - -static long libmuser_unl_ioctl(struct file *filep, unsigned int cmd, - unsigned long arg) -{ - struct muser_dev *mudev = filep->private_data; - struct mudev_cmd *mucmd; - unsigned long offset; - int ret = -EINVAL, mucmd_err; - - WARN_ON(mudev == NULL); - switch (cmd) { - case MUSER_DEV_CMD_WAIT: - /* Block until a request come from vfio. */ - ret = wait_event_interruptible(mudev->user_wait_q, - !list_empty(&mudev->cmd_list)); - if (unlikely(ret)) { - muser_dbg("failed to wait for user space: %d", ret); - goto out; - } - - /* Pick and remove the mucmd from the cmd_list. */ - mutex_lock(&mudev->dev_lock); - WARN_ON(list_empty(&mudev->cmd_list)); - mucmd = list_first_entry(&mudev->cmd_list, struct mudev_cmd, - entry); - list_del(&mucmd->entry); - mutex_unlock(&mudev->dev_lock); - - /* Keep a reference to mudev_cmd in mudev. */ - WARN_ON(mudev->mucmd_pending != NULL); - mudev->mucmd_pending = mucmd; - /* TODO: These WARN_ON()s should really just detach mudev. */ - - if (mucmd->muser_cmd.type == MUSER_DMA_MMAP) { - ret = fd_install_dma_map(mucmd->muser_cmd.mmap.request.file); - if (ret < 0) { - return ret; - } - mudev->dma_map->fd = mucmd->muser_cmd.mmap.request.fd = ret; - } - - /* Populate userspace with mucmd. */ - ret = muser_copyout((void __user *)arg, &mucmd->muser_cmd, - sizeof(struct muser_cmd)); - if (ret) - return -EFAULT; - - /* Install FDs on VFIO_SET_IRQS */ - ret = maybe_install_fds(mucmd); - if (ret) - return ret; - - break; - case MUSER_DEV_CMD_DONE: - /* This is only called when a command is pending. */ - if (mudev->mucmd_pending == NULL) { - muser_dbg("done but no command pending"); - return -EINVAL; - } - - /* Fetch (and clear) the pending command. */ - mucmd = mudev->mucmd_pending; - mudev->mucmd_pending = NULL; - - /* Fetch response from userspace. */ - ret = muser_copyin(&mucmd->muser_cmd, (void __user *)arg, - sizeof(struct muser_cmd)); - if (ret) - goto out; - - mucmd_err = mucmd->muser_cmd.err; - switch (mucmd->type) { - case MUSER_IOCTL: - offset = offsetof(struct muser_cmd, ioctl); - offset += offsetof(struct muser_cmd_ioctl, data); - ret = bounce_in(mucmd, (void __user *)(arg + offset)); - break; - case MUSER_MMAP: - if (!mucmd_err) - ret = mmap_done(mucmd); - break; - case MUSER_READ: - if (mucmd_err < 0) - muser_alert("read failed: %d", mucmd_err); - break; - case MUSER_WRITE: - case MUSER_DMA_MMAP: - case MUSER_DMA_MUNMAP: - break; - default: - muser_alert("bad command %d", mucmd->type); - ret = -EINVAL; - break; - } - - /* Wake up vfio client. */ - up(&mudev->sem); - break; - - default: - muser_info("bad ioctl 0x%x", cmd); - return -EINVAL; - } - -out: - return ret; -} - -#ifdef CONFIG_COMPAT -static long libmuser_compat_ioctl(struct file *filep, - unsigned int cmd, unsigned long arg) -{ - arg = (unsigned long)compat_ptr(arg); - return libmuser_unl_ioctl(filep, cmd, arg); -} -#endif /* CONFIG_COMPAT */ - -static struct muser_dev *muser_get_dev_from_minor(int minor) -{ - struct muser_dev *mudev; - - /* Locate mudev using idr. */ - mutex_lock(&muser.muser_lock); - mudev = idr_find(&muser.dev_idr, minor); - mutex_unlock(&muser.muser_lock); - - return mudev; -} - -static int libmuser_open(struct inode *inode, struct file *filep) -{ - struct muser_dev *mudev; - int opened; - - /* Fetch corresponding mudev. */ - mudev = muser_get_dev_from_minor(iminor(inode)); - if (!mudev) - return -ENOENT; - - /* Allow only one server for each mudev. */ - opened = atomic_cmpxchg(&mudev->srv_opened, 0, 1); - if (opened) - return -EBUSY; - - WARN_ON(filep->private_data != NULL); - filep->private_data = mudev; - - return 0; -} - -static int libmuser_release(struct inode *inode, struct file *filep) -{ - struct muser_dev *mudev = filep->private_data; - - WARN_ON(mudev == NULL); - mutex_lock(&mudev->dev_lock); - /* - * FIXME must be per filep - */ - if (mudev->mucmd_pending) { - muser_info("moving command back in list"); - list_add_tail(&mudev->mucmd_pending->entry, &mudev->cmd_list); - mudev->mucmd_pending = NULL; - } - mutex_unlock(&mudev->dev_lock); - - filep->private_data = NULL; - atomic_dec(&mudev->srv_opened); - - return 0; -} - -static inline int irq_set_data_eventfd(void __user * const buf, - struct mudev_cmd * const mucmd) -{ - return muser_copyout((void __user *)buf, mucmd->data_fds, - sizeof(__s32) * mucmd->muser_cmd.ioctl.data.irq_set.count); -} - -static inline int irq_set_data_bool(void __user * const buf, - struct mudev_cmd * const mucmd) -{ - return muser_copyout((void __user *)buf, mucmd->data_fds, - sizeof(__u8) * mucmd->muser_cmd.ioctl.data.irq_set.count); -} - -/* - * Called by libmuser for kernel->user transfers. - */ -static ssize_t libmuser_read(struct file *filp, char __user *buf, - size_t bufsz, loff_t *ppos) -{ - struct muser_dev *mudev = filp->private_data; - struct mudev_cmd *mucmd = mudev->mucmd_pending; - int ret = -EINVAL; - uint32_t irq_set_flags; - - if (!mucmd || !mudev) { - muser_dbg("bad arguments"); - return -EINVAL; - } - - /* XXX this should be taken out when upstreaming */ -#if LINUX_VERSION_CODE <= KERNEL_VERSION(4,19,67) - if (!access_ok(VERIFY_WRITE, buf, bufsz)) { -#else - if (!access_ok(buf, bufsz)) { -#endif - muser_dbg("bad permissions"); - return -EFAULT; - } - - switch (mucmd->type) { - case MUSER_WRITE: - ret = bounce_out(buf, bufsz, mucmd); - if (ret) { - muser_dbg("failed to copy to user: %d", ret); - goto err; - } - break; - case MUSER_IOCTL: - /* FIXME move case into separate function */ - if (mucmd->muser_cmd.ioctl.vfio_cmd != VFIO_DEVICE_SET_IRQS) { - muser_dbg("expected VFIO command %d, got %d instead", - VFIO_DEVICE_SET_IRQS, - mucmd->muser_cmd.ioctl.vfio_cmd); - goto err; - } - irq_set_flags = mucmd->muser_cmd.ioctl.data.irq_set.flags & - VFIO_IRQ_SET_DATA_TYPE_MASK; - switch (irq_set_flags) { - case VFIO_IRQ_SET_DATA_EVENTFD: - ret = irq_set_data_eventfd((void __user *)buf, mucmd); - if (unlikely(ret)) { - muser_dbg("failed to set data eventfd: %d", - ret); - goto err; - } - break; - case VFIO_IRQ_SET_DATA_BOOL: - ret = irq_set_data_bool((void __user *)buf, mucmd); - if (unlikely(ret)) - goto err; - break; - default: - muser_dbg("bad VFIO set IRQ flags %d", irq_set_flags); - goto err; - } - break; - default: - muser_dbg("bad muser command %d", mucmd->type); - goto err; - } - return bufsz; - -err: - return ret; -} - -/* - * Called by libmuser for user->kernel transfers. - */ -static ssize_t libmuser_write(struct file *filp, const char __user *buf, - size_t bufsz, loff_t *ppos) -{ - struct muser_dev *mudev = filp->private_data; - struct mudev_cmd *mucmd = mudev->mucmd_pending; - unsigned int seek; - int ret; - - if (!mucmd || !mudev) { - muser_dbg("bad arguments"); - return -EINVAL; - } - /* XXX this should be taken out when upstreaming */ -#if LINUX_VERSION_CODE <= KERNEL_VERSION(4,19,67) - if (!access_ok(VERIFY_READ, buf, bufsz)) { -#else - if (!access_ok(buf, bufsz)) { -#endif - muser_dbg("bad permissions"); - return -EFAULT; - } - - switch (mucmd->type) { - case MUSER_READ: - muser_dbg("received data from libmuser"); - dump_buffer(buf, bufsz); - ret = bounce_in(mucmd, (void __user *)buf); - if (ret) - return ret; - break; - case MUSER_IOCTL: - muser_dbg("received sparse mmap from libmuser"); - /* - * copy the sparse mmap cap information after the - * struct vfio_region_info. - */ - seek = sizeof(struct vfio_region_info); - ret = bounce_in_seek(&mucmd->pg_map, (void __user *)buf, bufsz, - seek); - if (ret) - return ret; - mucmd->pg_map.len -= seek; - break; - default: - muser_dbg("bad command 0x%x", mucmd->type); - return -EINVAL; - } - - return bufsz; -} - -static const struct file_operations libmuser_fops = { - .owner = THIS_MODULE, - .unlocked_ioctl = libmuser_unl_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = libmuser_compat_ioctl, -#endif - .open = libmuser_open, - .release = libmuser_release, - .mmap = libmuser_mmap_dev, - .read = libmuser_read, - .write = libmuser_write, -}; - -static void muser_device_release(struct device *dev) -{ - muser_info("muser dev released"); -} - -static char *muser_devnode(struct device *dev, umode_t *mode) -{ - return kasprintf(GFP_KERNEL, DRIVER_NAME "/%s", dev_name(dev)); -} - -static int __init muser_init(void) -{ - int ret; - - /* Initialise idr. */ - idr_init(&muser.dev_idr); - mutex_init(&muser.muser_lock); - INIT_LIST_HEAD(&muser.mudev_list); - - /* Initialise class. */ - muser.class = class_create(THIS_MODULE, DRIVER_NAME); - if (IS_ERR(muser.class)) - return PTR_ERR(muser.class); - muser.class->devnode = muser_devnode; - - /* Allocate and register a chardev for muser devices. */ - ret = alloc_chrdev_region(&muser.muser_devt, 0, MINORMASK + 1, - DRIVER_NAME); - if (ret) - goto err_alloc_chrdev; - - cdev_init(&muser.muser_cdev, &libmuser_fops); - ret = cdev_add(&muser.muser_cdev, muser.muser_devt, MINORMASK + 1); - if (ret) - goto err_cdev_add; - - muser.dev.class = muser.class; - muser.dev.release = muser_device_release; - dev_set_name(&muser.dev, "%s", DRIVER_NAME); - - ret = device_register(&muser.dev); - if (ret) - goto err_device_register; - - /* Register ourselves with mdev. */ - ret = mdev_register_device(&muser.dev, &muser_mdev_fops); - if (ret) - goto err_mdev_register_device; - - return 0; - -err_mdev_register_device: - device_unregister(&muser.dev); -err_device_register: - cdev_del(&muser.muser_cdev); -err_cdev_add: - unregister_chrdev_region(muser.muser_devt, MINORMASK + 1); -err_alloc_chrdev: - class_destroy(muser.class); - muser.class = NULL; - return ret; -} - -static void __exit muser_cleanup(void) -{ - struct muser_dev *mudev, *tmp; - - /* Remove all devices. */ - mutex_lock(&muser.muser_lock); - list_for_each_entry_safe(mudev, tmp, &muser.mudev_list, dlist_entry) { - WARN_ON(atomic_read(&mudev->mdev_opened) || - atomic_read(&mudev->srv_opened)); - __muser_deinit_dev(mudev); - kfree(mudev); - } - mutex_unlock(&muser.muser_lock); - - /* Unregister with mdev. */ - muser.dev.bus = NULL; - mdev_unregister_device(&muser.dev); - - /* Cleanup everything else. */ - device_unregister(&muser.dev); - idr_destroy(&muser.dev_idr); - cdev_del(&muser.muser_cdev); - unregister_chrdev_region(muser.muser_devt, MINORMASK + 1); - class_destroy(muser.class); - muser.class = NULL; -} - -module_init(muser_init); -module_exit(muser_cleanup); - -MODULE_LICENSE("Dual BSD/GPL"); |