Support 64 bits and prefetchable BARs (#792)

* Support 64 bits and prefetchable BARs

Add two new flags for lib user to request 64bits and/or prefetchable
BARs.

Tested with a vfio-user client patched QEMU.

Signed-off-by: Jérémy Fanguède <jfanguede@kalrayinc.com>

5 files changed, 46 insertions, 3 deletions

diff --git a/include/libvfio-user.h b/include/libvfio-user.h
index e4cfa60..c74902b 100644
--- a/include/libvfio-user.h
+++ b/include/libvfio-user.h
@@ -238,8 +238,11 @@ typedef ssize_t (vfu_region_access_cb_t)(vfu_ctx_t *vfu_ctx, char *buf,
 /* If unset, this is an IO region. */
 #define VFU_REGION_FLAG_MEM       (1 << 2)
 #define VFU_REGION_FLAG_ALWAYS_CB (1 << 3)
+#define VFU_REGION_FLAG_64_BITS   (1 << 4)
+#define VFU_REGION_FLAG_PREFETCH  (1 << 5)
 #define VFU_REGION_FLAG_MASK      (VFU_REGION_FLAG_RW | VFU_REGION_FLAG_MEM | \
-                                   VFU_REGION_FLAG_ALWAYS_CB)
+                                   VFU_REGION_FLAG_ALWAYS_CB | VFU_REGION_FLAG_64_BITS | \
+                                   VFU_REGION_FLAG_PREFETCH)
 
 /**
  * Set up a device region.
diff --git a/include/pci_defs.h b/include/pci_defs.h
index 5a77b65..11ab919 100644
--- a/include/pci_defs.h
+++ b/include/pci_defs.h
@@ -48,6 +48,10 @@ extern "C" {
  * instead?
  */
 
+#define PCI_BASE_ADDRESS_MEM_TYPE_LOCATABLE_32 (PCI_BASE_ADDRESS_MEM_TYPE_32 >> 1)
+#define PCI_BASE_ADDRESS_MEM_TYPE_LOCATABLE_1M (PCI_BASE_ADDRESS_MEM_TYPE_1M >> 1)
+#define PCI_BASE_ADDRESS_MEM_TYPE_LOCATABLE_64 (PCI_BASE_ADDRESS_MEM_TYPE_64 >> 1)
+
 typedef union {
     uint32_t raw;
     struct {
diff --git a/lib/libvfio-user.c b/lib/libvfio-user.c
index dc13ea5..62efd30 100644
--- a/lib/libvfio-user.c
+++ b/lib/libvfio-user.c
@@ -1729,6 +1729,17 @@ vfu_realize_ctx(vfu_ctx_t *vfu_ctx)
         if (!(vfu_ctx->reg_info[i].flags & VFU_REGION_FLAG_MEM)) {
             vfu_ctx->pci.config_space->hdr.bars[i].io.region_type |= 0x1;
         }
+        if ((vfu_ctx->reg_info[i].flags & VFU_REGION_FLAG_64_BITS)) {
+                vfu_ctx->pci.config_space->hdr.bars[i].mem.locatable
+			= PCI_BASE_ADDRESS_MEM_TYPE_LOCATABLE_64;
+        }
+        if ((vfu_ctx->reg_info[i].flags & VFU_REGION_FLAG_PREFETCH)) {
+                vfu_ctx->pci.config_space->hdr.bars[i].mem.prefetchable = 1;
+                if (!(vfu_ctx->reg_info[i].flags & VFU_REGION_FLAG_64_BITS)) {
+                        vfu_log(vfu_ctx, LOG_WARNING,
+                                "Region %d has prefetchable flag set, but not 64bits flag", i);
+                }
+        }
     }
 
     if (vfu_ctx->irqs == NULL) {
diff --git a/test/py/libvfio_user.py b/test/py/libvfio_user.py
index 289f10a..976590c 100644
--- a/test/py/libvfio_user.py
+++ b/test/py/libvfio_user.py
@@ -84,6 +84,13 @@ PCI_EXT_CAP_DSN_SIZEOF = 12
 
 PCI_EXT_CAP_VNDR_HDR_SIZEOF = 8
 
+PCI_BASE_ADDRESS_SPACE_IO = 0x01
+PCI_BASE_ADDRESS_SPACE_MEMORY = 0x00
+PCI_BASE_ADDRESS_MEM_TYPE_32 = 0x00
+PCI_BASE_ADDRESS_MEM_TYPE_1M = 0x02
+PCI_BASE_ADDRESS_MEM_TYPE_64 = 0x04
+PCI_BASE_ADDRESS_MEM_PREFETCH = 0x08
+
 # MSI registers
 PCI_MSI_FLAGS = 2  # Message Control offset
 PCI_MSI_ADDRESS_LO = 4  # Message Address offset
@@ -201,6 +208,8 @@ VFU_REGION_FLAG_WRITE = 2
 VFU_REGION_FLAG_RW = (VFU_REGION_FLAG_READ | VFU_REGION_FLAG_WRITE)
 VFU_REGION_FLAG_MEM = 4
 VFU_REGION_FLAG_ALWAYS_CB = 8
+VFU_REGION_FLAG_64_BITS = 16
+VFU_REGION_FLAG_PREFETCH = 32
 
 VFIO_USER_F_DMA_REGION_READ = (1 << 0)
 VFIO_USER_F_DMA_REGION_WRITE = (1 << 1)
diff --git a/test/py/test_vfu_realize_ctx.py b/test/py/test_vfu_realize_ctx.py
index ab0b86a..4b001a4 100644
--- a/test/py/test_vfu_realize_ctx.py
+++ b/test/py/test_vfu_realize_ctx.py
@@ -73,14 +73,30 @@ def test_vfu_realize_ctx_pci_bars():
     ret = vfu_setup_region(ctx, index=VFU_PCI_DEV_BAR1_REGION_IDX, size=4096,
                            flags=(VFU_REGION_FLAG_RW | VFU_REGION_FLAG_MEM))
     assert ret == 0
+    ret = vfu_setup_region(ctx, index=VFU_PCI_DEV_BAR2_REGION_IDX,
+                           size=1048576,
+                           flags=(VFU_REGION_FLAG_RW | VFU_REGION_FLAG_MEM
+                                  | VFU_REGION_FLAG_64_BITS))
+    assert ret == 0
+    ret = vfu_setup_region(ctx, index=VFU_PCI_DEV_BAR4_REGION_IDX,
+                           size=1073741824,
+                           flags=(VFU_REGION_FLAG_RW | VFU_REGION_FLAG_MEM
+                                  | VFU_REGION_FLAG_64_BITS
+                                  | VFU_REGION_FLAG_PREFETCH))
+    assert ret == 0
 
     ret = vfu_realize_ctx(ctx)
     assert ret == 0
 
     # region_type should be set non-MEM BAR, unset otherwise
     hdr = get_pci_header(ctx)
-    assert hdr.bars[0].io == 0x1
-    assert hdr.bars[1].io == 0
+    assert hdr.bars[0].io == PCI_BASE_ADDRESS_SPACE_IO
+    assert hdr.bars[1].mem == PCI_BASE_ADDRESS_SPACE_MEMORY
+    assert hdr.bars[2].mem == (PCI_BASE_ADDRESS_SPACE_MEMORY
+                               | PCI_BASE_ADDRESS_MEM_TYPE_64)
+    assert hdr.bars[4].mem == (PCI_BASE_ADDRESS_SPACE_MEMORY
+                               | PCI_BASE_ADDRESS_MEM_TYPE_64
+                               | PCI_BASE_ADDRESS_MEM_PREFETCH)
 
     vfu_destroy_ctx(ctx)