diff options
author | Thanos Makatos <thanos.makatos@nutanix.com> | 2021-02-10 09:08:10 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-02-10 09:08:10 +0000 |
commit | 365ca96a97740332d3633090d850222d10bc9d70 (patch) | |
tree | bd9885cd57351c05ac12d63ef51a4e4e5b3eef7d /include | |
parent | c5d11659c95c995acb77a71fe03c38b240ca43d9 (diff) | |
download | libvfio-user-365ca96a97740332d3633090d850222d10bc9d70.zip libvfio-user-365ca96a97740332d3633090d850222d10bc9d70.tar.gz libvfio-user-365ca96a97740332d3633090d850222d10bc9d70.tar.bz2 |
expose migration region (#305)
This patch exposes the fact that live migration is implemented as a
special device region. Hiding this from the user doesn't offer much
benefit since it only takes just a little bit of extra code for the user
to handle it as a region. We do keep the migration callback
functionality since this feature substantially simplifies supporting
live migration from the device implementation's perspective.
Signed-off-by: Thanos Makatos <thanos.makatos@nutanix.com>
Co-authored-by: John Levon <john.levon@nutanix.com>
Diffstat (limited to 'include')
-rw-r--r-- | include/libvfio-user.h | 81 |
1 files changed, 69 insertions, 12 deletions
diff --git a/include/libvfio-user.h b/include/libvfio-user.h index 75b5943..1cc62d9 100644 --- a/include/libvfio-user.h +++ b/include/libvfio-user.h @@ -227,7 +227,12 @@ typedef ssize_t (vfu_region_access_cb_t)(vfu_ctx_t *vfu_ctx, char *buf, * given callback. However, the callback can still be invoked, even on a * mappable area, if the client chooses to call VFIO_USER_REGION_READ/WRITE. * - * A VFU_PCI_DEV_CFG_REGION_IDX region, corresponding to PCI config space, has + * The following regions are special and are explained below: + * - VFU_PCI_DEV_CFG_REGION_IDX, + * - VFU_PCI_DEV_MIGR_REGION_IDX, and + * - VFU_GENERIC_DEV_MIGR_REG_IDX. + * + * Region VFU_PCI_DEV_CFG_REGION_IDX, corresponding to PCI config space, has * special handling: * * - the @size argument is ignored: the region size is always the size defined @@ -240,6 +245,24 @@ typedef ssize_t (vfu_region_access_cb_t)(vfu_ctx_t *vfu_ctx, char *buf, * and writes are an error * - otherwise, the callback is expected to handle the access * + * Regions VFU_PCI_DEV_MIGR_REGION_IDX and VFU_GENERIC_DEV_MIGR_REG_IDX, + * corresponding to the migration region, enable live migration support for + * the device. The migration region must contain at the beginning the migration + * registers (struct vfio_device_migration_info defined in <linux/vfio.h>) and + * the remaining part of the region can be arbitrarily used by the device + * implementation. The region provided must have at least + * vfu_get_migr_register_area_size() bytes available at the start of the region + * (this size is guaranteed to be page-aligned). If mmap_areas is given, it + * must _not_ include this part of the region. + * + * libvfio-user offers two ways for the migration region to be used: + * 1. natively: the device implementation must handle accesses to the + * migration registers and migration data via the region callbacks. The + * semantics of these registers are explained in <linux/vfio.h>. + * 2. via the vfu_migration_t callbacks: the device implementation registers + * a set of callbacks by calling vfu_setup_device_migration. The region's + * read/write callbacks are never called. + * * @vfu_ctx: the libvfio-user context * @region_idx: region index * @size: size of the region @@ -262,6 +285,13 @@ vfu_setup_region(vfu_ctx_t *vfu_ctx, int region_idx, size_t size, int fd); /* + * Returns the size of the area needed to hold the migration registers at the + * beginning of the migration region; guaranteed to be page aligned. + */ +size_t +vfu_get_migr_register_area_size(void); + +/* * Callback function that is called when the guest resets the device. */ typedef int (vfu_reset_cb_t)(vfu_ctx_t *vfu_ctx); @@ -348,6 +378,8 @@ typedef enum { } vfu_migr_state_t; +#define VFU_MIGR_CALLBACKS_VERS 1 + /* * Callbacks during the pre-copy and stop-and-copy phases. * @@ -363,6 +395,11 @@ typedef enum { */ typedef struct { + /* + * Set it to VFU_MIGR_CALLBACKS_VERS. + */ + int version; + /* migration state transition callback */ /* TODO rename to vfu_migration_state_transition_callback */ /* FIXME maybe we should create a single callback and pass the state? */ @@ -422,21 +459,23 @@ typedef struct { } vfu_migration_callbacks_t; -typedef struct { - size_t size; - vfu_migration_callbacks_t callbacks; - struct iovec *mmap_areas; - uint32_t nr_mmap_areas; -} vfu_migration_t; - -//TODO: Re-visit once migration support is done. /** - * Enable support for device migration. + * vfu_setup_device_migration provides an abstraction over the migration + * protocol: the user specifies a set of callbacks which are called in response + * to client accesses of the migration region; the migration region read/write + * callbacks are not called after this function call. Offsets in callbacks are + * relative to @data_offset. + * * @vfu_ctx: the libvfio-user context - * @migration: information required to migrate device + * @callbacks: migration callbacks + * @data_offset: offset in the migration region where data begins. + * + * @returns 0 on success, -1 on error, sets errno. */ int -vfu_setup_device_migration(vfu_ctx_t *vfu_ctx, vfu_migration_t *migration); +vfu_setup_device_migration_callbacks(vfu_ctx_t *vfu_ctx, + const vfu_migration_callbacks_t *callbacks, + uint64_t data_offset); /** * Triggers an interrupt. @@ -550,6 +589,18 @@ vfu_dma_write(vfu_ctx_t *vfu_ctx, dma_sg_t *sg, void *data); /* * Supported PCI regions. + * + * Note: in VFIO, each region starts at a terabyte offset + * (VFIO_PCI_INDEX_TO_OFFSET) and because Linux supports up to 128 TB of user + * space virtual memory, there can be up to 128 device regions. PCI regions are + * fixed and in retrospect this choice has proven to be problematic because + * devices might contain potentially unused regions. New regions can now be + * positioned anywhere by using the VFIO_REGION_INFO_CAP_TYPE capability. In + * vfio-user we don't have this problem because the region index is just an + * identifier: the VMM memory maps a file descriptor that is passed to it and + * the mapping offset is derived from the mmap_areas offset value, rather than a + * static mapping from region index to offset. Thus, additional regions can + * have static indexes in vfio-user. */ enum { VFU_PCI_DEV_BAR0_REGION_IDX, @@ -561,6 +612,7 @@ enum { VFU_PCI_DEV_ROM_REGION_IDX, VFU_PCI_DEV_CFG_REGION_IDX, VFU_PCI_DEV_VGA_REGION_IDX, + VFU_PCI_DEV_MIGR_REGION_IDX, VFU_PCI_DEV_NUM_REGIONS, }; @@ -571,6 +623,11 @@ typedef enum { VFU_PCI_TYPE_EXPRESS } vfu_pci_type_t; +enum { + VFU_GENERIC_DEV_MIGR_REGION_IDX, + VFU_GENERIC_DEV_NUM_REGIONS +}; + /** * Initialize the context for a PCI device. This function must be called only * once per libvfio-user context. |