diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/exec/memory.h | 19 | ||||
| -rw-r--r-- | include/exec/ram_addr.h | 92 | ||||
| -rw-r--r-- | include/qemu/bitmap.h | 9 | ||||
| -rw-r--r-- | include/sysemu/kvm_int.h | 4 |
4 files changed, 119 insertions, 5 deletions
diff --git a/include/exec/memory.h b/include/exec/memory.h index 2c5cdff..bb0961d 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -46,6 +46,8 @@ OBJECT_GET_CLASS(IOMMUMemoryRegionClass, (obj), \ TYPE_IOMMU_MEMORY_REGION) +extern bool global_dirty_log; + typedef struct MemoryRegionOps MemoryRegionOps; typedef struct MemoryRegionMmio MemoryRegionMmio; @@ -414,6 +416,7 @@ struct MemoryListener { void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section, int old, int new); void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section); + void (*log_clear)(MemoryListener *listener, MemoryRegionSection *section); void (*log_global_start)(MemoryListener *listener); void (*log_global_stop)(MemoryListener *listener); void (*eventfd_add)(MemoryListener *listener, MemoryRegionSection *section, @@ -1268,6 +1271,22 @@ void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, hwaddr size); /** + * memory_region_clear_dirty_bitmap - clear dirty bitmap for memory range + * + * This function is called when the caller wants to clear the remote + * dirty bitmap of a memory range within the memory region. This can + * be used by e.g. KVM to manually clear dirty log when + * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT is declared support by the host + * kernel. + * + * @mr: the memory region to clear the dirty log upon + * @start: start address offset within the memory region + * @len: length of the memory region to clear dirty bitmap + */ +void memory_region_clear_dirty_bitmap(MemoryRegion *mr, hwaddr start, + hwaddr len); + +/** * memory_region_snapshot_and_clear_dirty: Get a snapshot of the dirty * bitmap and clear it. * diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h index f96777b..b7b2e60 100644 --- a/include/exec/ram_addr.h +++ b/include/exec/ram_addr.h @@ -51,8 +51,70 @@ struct RAMBlock { unsigned long *unsentmap; /* bitmap of already received pages in postcopy */ unsigned long *receivedmap; + + /* + * bitmap to track already cleared dirty bitmap. When the bit is + * set, it means the corresponding memory chunk needs a log-clear. + * Set this up to non-NULL to enable the capability to postpone + * and split clearing of dirty bitmap on the remote node (e.g., + * KVM). The bitmap will be set only when doing global sync. + * + * NOTE: this bitmap is different comparing to the other bitmaps + * in that one bit can represent multiple guest pages (which is + * decided by the `clear_bmap_shift' variable below). On + * destination side, this should always be NULL, and the variable + * `clear_bmap_shift' is meaningless. + */ + unsigned long *clear_bmap; + uint8_t clear_bmap_shift; }; +/** + * clear_bmap_size: calculate clear bitmap size + * + * @pages: number of guest pages + * @shift: guest page number shift + * + * Returns: number of bits for the clear bitmap + */ +static inline long clear_bmap_size(uint64_t pages, uint8_t shift) +{ + return DIV_ROUND_UP(pages, 1UL << shift); +} + +/** + * clear_bmap_set: set clear bitmap for the page range + * + * @rb: the ramblock to operate on + * @start: the start page number + * @size: number of pages to set in the bitmap + * + * Returns: None + */ +static inline void clear_bmap_set(RAMBlock *rb, uint64_t start, + uint64_t npages) +{ + uint8_t shift = rb->clear_bmap_shift; + + bitmap_set_atomic(rb->clear_bmap, start >> shift, + clear_bmap_size(npages, shift)); +} + +/** + * clear_bmap_test_and_clear: test clear bitmap for the page, clear if set + * + * @rb: the ramblock to operate on + * @page: the page number to check + * + * Returns: true if the bit was set, false otherwise + */ +static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page) +{ + uint8_t shift = rb->clear_bmap_shift; + + return bitmap_test_and_clear_atomic(rb->clear_bmap, page >> shift, 1); +} + static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) { return (b && b->host && offset < b->used_length) ? true : false; @@ -349,8 +411,13 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, if (bitmap[k]) { unsigned long temp = leul_to_cpu(bitmap[k]); - atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp); atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); + + if (global_dirty_log) { + atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], + temp); + } + if (tcg_enabled()) { atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp); } @@ -367,6 +434,11 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); } else { uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; + + if (!global_dirty_log) { + clients &= ~(1 << DIRTY_MEMORY_MIGRATION); + } + /* * bitmap-traveling is faster than memory-traveling (for addr...) * especially when most of the memory is not dirty. @@ -394,7 +466,7 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, unsigned client); DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty - (ram_addr_t start, ram_addr_t length, unsigned client); + (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client); bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, ram_addr_t start, @@ -409,6 +481,7 @@ static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, } +/* Called with RCU critical section */ static inline uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, ram_addr_t start, @@ -432,8 +505,6 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, DIRTY_MEMORY_BLOCK_SIZE); unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); - rcu_read_lock(); - src = atomic_rcu_read( &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; @@ -454,7 +525,18 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, } } - rcu_read_unlock(); + if (rb->clear_bmap) { + /* + * Postpone the dirty bitmap clear to the point before we + * really send the pages, also we will split the clear + * dirty procedure into smaller chunks. + */ + clear_bmap_set(rb, start >> TARGET_PAGE_BITS, + length >> TARGET_PAGE_BITS); + } else { + /* Slow path - still do that in a huge chunk */ + memory_region_clear_dirty_bitmap(rb->mr, start, length); + } } else { ram_addr_t offset = rb->offset; diff --git a/include/qemu/bitmap.h b/include/qemu/bitmap.h index 5c31334..82a1d2f 100644 --- a/include/qemu/bitmap.h +++ b/include/qemu/bitmap.h @@ -41,6 +41,10 @@ * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area * bitmap_to_le(dst, src, nbits) Convert bitmap to little endian * bitmap_from_le(dst, src, nbits) Convert bitmap from little endian + * bitmap_copy_with_src_offset(dst, src, offset, nbits) + * *dst = *src (with an offset into src) + * bitmap_copy_with_dst_offset(dst, src, offset, nbits) + * *dst = *src (with an offset into dst) */ /* @@ -271,4 +275,9 @@ void bitmap_to_le(unsigned long *dst, const unsigned long *src, void bitmap_from_le(unsigned long *dst, const unsigned long *src, long nbits); +void bitmap_copy_with_src_offset(unsigned long *dst, const unsigned long *src, + unsigned long offset, unsigned long nbits); +void bitmap_copy_with_dst_offset(unsigned long *dst, const unsigned long *src, + unsigned long shift, unsigned long nbits); + #endif /* BITMAP_H */ diff --git a/include/sysemu/kvm_int.h b/include/sysemu/kvm_int.h index f838412..31df465 100644 --- a/include/sysemu/kvm_int.h +++ b/include/sysemu/kvm_int.h @@ -21,10 +21,14 @@ typedef struct KVMSlot int slot; int flags; int old_flags; + /* Dirty bitmap cache for the slot */ + unsigned long *dirty_bmap; } KVMSlot; typedef struct KVMMemoryListener { MemoryListener listener; + /* Protects the slots and all inside them */ + QemuMutex slots_lock; KVMSlot *slots; int as_id; } KVMMemoryListener; |