From db94604b20278c1dc227a04e4c564d80230e6c3f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 21 May 2015 15:12:29 +0200 Subject: exec: optimize phys_page_set_level phys_page_set_level is writing zeroes to a struct that has just been filled in by phys_map_node_alloc. Instead, tell phys_map_node_alloc whether to fill in the page "as a leaf" or "as a non-leaf". memcpy is faster than struct assignment, which copies each bitfield individually. A compiler bug (https://gcc.gnu.org/PR66391), and small memcpys like this one are special-cased anyway, and optimized to a register move, so just use the memcpy. This cuts the cost of phys_page_set_level from 25% to 5% when booting qboot. Reviewed-by: Stefan Hajnoczi Signed-off-by: Paolo Bonzini --- exec.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index e19ab22..fc8d05d 100644 --- a/exec.c +++ b/exec.c @@ -173,17 +173,22 @@ static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes) } } -static uint32_t phys_map_node_alloc(PhysPageMap *map) +static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf) { unsigned i; uint32_t ret; + PhysPageEntry e; + PhysPageEntry *p; ret = map->nodes_nb++; + p = map->nodes[ret]; assert(ret != PHYS_MAP_NODE_NIL); assert(ret != map->nodes_nb_alloc); + + e.skip = leaf ? 0 : 1; + e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL; for (i = 0; i < P_L2_SIZE; ++i) { - map->nodes[ret][i].skip = 1; - map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL; + memcpy(&p[i], &e, sizeof(e)); } return ret; } @@ -193,21 +198,12 @@ static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp, int level) { PhysPageEntry *p; - int i; hwaddr step = (hwaddr)1 << (level * P_L2_BITS); if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) { - lp->ptr = phys_map_node_alloc(map); - p = map->nodes[lp->ptr]; - if (level == 0) { - for (i = 0; i < P_L2_SIZE; i++) { - p[i].skip = 0; - p[i].ptr = PHYS_SECTION_UNASSIGNED; - } - } - } else { - p = map->nodes[lp->ptr]; + lp->ptr = phys_map_node_alloc(map, level == 0); } + p = map->nodes[lp->ptr]; lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)]; while (*nb && lp < &p[P_L2_SIZE]) { -- cgit v1.1 From 49dfcec40349245ad365964468b67e132c3cedc7 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 Mar 2015 11:35:19 +0100 Subject: ram_addr: tweaks to xen_modified_memory Invoke xen_modified_memory from cpu_physical_memory_set_dirty_range_nocode; it is akin to DIRTY_MEMORY_MIGRATION, so set it together with that bitmap. The remaining call from invalidate_and_set_dirty's "else" branch will go away soon. Second, fix the second argument to the function in the cpu_physical_memory_set_dirty_lebitmap call site. That function is only used by KVM, but it is better to be clean anyway. Acked-by: Stefano Stabellini Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index fc8d05d..6f6dbc8 100644 --- a/exec.c +++ b/exec.c @@ -2281,8 +2281,9 @@ static void invalidate_and_set_dirty(hwaddr addr, if (cpu_physical_memory_range_includes_clean(addr, length)) { tb_invalidate_phys_range(addr, addr + length, 0); cpu_physical_memory_set_dirty_range_nocode(addr, length); + } else { + xen_modified_memory(addr, length); } - xen_modified_memory(addr, length); } static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr) -- cgit v1.1 From 845b6214a309fa58a4405050bf8313e19fde5c91 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 Mar 2015 11:45:53 +0100 Subject: exec: use memory_region_get_dirty_log_mask to optimize dirty tracking The memory API can now return the exact set of bitmaps that have to be tracked. Use it instead of the in_migration variable. In the next patches, we will also use it to set only DIRTY_MEMORY_VGA or DIRTY_MEMORY_MIGRATION if necessary. This can make a difference for dataplane, especially after the dirty bitmap is changed to use more expensive atomic operations. Of some interest is the change to stl_phys_notdirty. When migration was introduced, stl_phys_notdirty was changed to effectively behave as stl_phys during migration. In fact, if one looks at the function as it was in the beginning (commit 8df1cd0, physical memory access functions, 2005-01-28), at the time the dirty bitmap was the equivalent of DIRTY_MEMORY_CODE nowadays; hence, the function simply should not touch the dirty code bits. This patch changes it to do the intended thing. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 59 +++++++++++++++++++---------------------------------------- 1 file changed, 19 insertions(+), 40 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index 6f6dbc8..950fea4 100644 --- a/exec.c +++ b/exec.c @@ -59,8 +59,6 @@ //#define DEBUG_SUBPAGE #if !defined(CONFIG_USER_ONLY) -static bool in_migration; - /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes * are protected by the ramlist lock. */ @@ -866,11 +864,6 @@ void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length, } } -static void cpu_physical_memory_set_dirty_tracking(bool enable) -{ - in_migration = enable; -} - /* Called from RCU critical section */ hwaddr memory_region_section_get_iotlb(CPUState *cpu, MemoryRegionSection *section, @@ -2161,22 +2154,6 @@ static void tcg_commit(MemoryListener *listener) } } -static void core_log_global_start(MemoryListener *listener) -{ - cpu_physical_memory_set_dirty_tracking(true); -} - -static void core_log_global_stop(MemoryListener *listener) -{ - cpu_physical_memory_set_dirty_tracking(false); -} - -static MemoryListener core_memory_listener = { - .log_global_start = core_log_global_start, - .log_global_stop = core_log_global_stop, - .priority = 1, -}; - void address_space_init_dispatch(AddressSpace *as) { as->dispatch = NULL; @@ -2216,8 +2193,6 @@ static void memory_map_init(void) memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io", 65536); address_space_init(&address_space_io, system_io, "I/O"); - - memory_listener_register(&core_memory_listener, &address_space_memory); } MemoryRegion *get_system_memory(void) @@ -2275,12 +2250,18 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr, #else -static void invalidate_and_set_dirty(hwaddr addr, +static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, hwaddr length) { if (cpu_physical_memory_range_includes_clean(addr, length)) { - tb_invalidate_phys_range(addr, addr + length, 0); - cpu_physical_memory_set_dirty_range_nocode(addr, length); + uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr); + if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { + tb_invalidate_phys_range(addr, addr + length, 0); + dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); + } + if (dirty_log_mask) { + cpu_physical_memory_set_dirty_range_nocode(addr, length); + } } else { xen_modified_memory(addr, length); } @@ -2368,7 +2349,7 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, /* RAM case */ ptr = qemu_get_ram_ptr(addr1); memcpy(ptr, buf, l); - invalidate_and_set_dirty(addr1, l); + invalidate_and_set_dirty(mr, addr1, l); } } else { if (!memory_access_is_direct(mr, is_write)) { @@ -2465,7 +2446,7 @@ static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as, switch (type) { case WRITE_DATA: memcpy(ptr, buf, l); - invalidate_and_set_dirty(addr1, l); + invalidate_and_set_dirty(mr, addr1, l); break; case FLUSH_CACHE: flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l); @@ -2690,7 +2671,7 @@ void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len, mr = qemu_ram_addr_from_host(buffer, &addr1); assert(mr != NULL); if (is_write) { - invalidate_and_set_dirty(addr1, access_len); + invalidate_and_set_dirty(mr, addr1, access_len); } if (xen_enabled()) { xen_invalidate_map_cache_entry(buffer); @@ -3019,6 +3000,7 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val, hwaddr l = 4; hwaddr addr1; MemTxResult r; + uint8_t dirty_log_mask; rcu_read_lock(); mr = address_space_translate(as, addr, &addr1, &l, @@ -3030,13 +3012,10 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val, ptr = qemu_get_ram_ptr(addr1); stl_p(ptr, val); - if (unlikely(in_migration)) { - if (cpu_physical_memory_is_clean(addr1)) { - /* invalidate code */ - tb_invalidate_phys_page_range(addr1, addr1 + 4, 0); - /* set dirty bit */ - cpu_physical_memory_set_dirty_range_nocode(addr1, 4); - } + dirty_log_mask = memory_region_get_dirty_log_mask(mr); + dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); + if (dirty_log_mask) { + cpu_physical_memory_set_dirty_range_nocode(addr1, 4); } r = MEMTX_OK; } @@ -3093,7 +3072,7 @@ static inline void address_space_stl_internal(AddressSpace *as, stl_p(ptr, val); break; } - invalidate_and_set_dirty(addr1, 4); + invalidate_and_set_dirty(mr, addr1, 4); r = MEMTX_OK; } if (result) { @@ -3197,7 +3176,7 @@ static inline void address_space_stw_internal(AddressSpace *as, stw_p(ptr, val); break; } - invalidate_and_set_dirty(addr1, 2); + invalidate_and_set_dirty(mr, addr1, 2); r = MEMTX_OK; } if (result) { -- cgit v1.1 From 358653391b0c0beaa0e3f9e28304e1918cd223b3 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 22 Apr 2015 14:20:35 +0200 Subject: translate-all: remove unnecessary argument to tb_invalidate_phys_range The is_cpu_write_access argument is always 0, remove it. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index 950fea4..650cfa8 100644 --- a/exec.c +++ b/exec.c @@ -2256,7 +2256,7 @@ static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, if (cpu_physical_memory_range_includes_clean(addr, length)) { uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr); if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { - tb_invalidate_phys_range(addr, addr + length, 0); + tb_invalidate_phys_range(addr, addr + length); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); } if (dirty_log_mask) { -- cgit v1.1 From 58d2707e8713ef17b89b8b4c9ce586c76655a385 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 Mar 2015 11:56:01 +0100 Subject: exec: pass client mask to cpu_physical_memory_set_dirty_range This cuts in half the cost of bitmap operations (which will become more expensive when made atomic) during migration on non-VRAM regions. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index 650cfa8..fe137bd 100644 --- a/exec.c +++ b/exec.c @@ -1351,7 +1351,8 @@ int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp) cpu_physical_memory_clear_dirty_range(block->offset, block->used_length); block->used_length = newsize; - cpu_physical_memory_set_dirty_range(block->offset, block->used_length); + cpu_physical_memory_set_dirty_range(block->offset, block->used_length, + DIRTY_CLIENTS_ALL); memory_region_set_size(block->mr, newsize); if (block->resized) { block->resized(block->idstr, newsize, block->host); @@ -1425,7 +1426,8 @@ static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp) } } cpu_physical_memory_set_dirty_range(new_block->offset, - new_block->used_length); + new_block->used_length, + DIRTY_CLIENTS_ALL); if (new_block->host) { qemu_ram_setup_dump(new_block->host, new_block->max_length); @@ -1813,7 +1815,11 @@ static void notdirty_mem_write(void *opaque, hwaddr ram_addr, default: abort(); } - cpu_physical_memory_set_dirty_range_nocode(ram_addr, size); + /* Set both VGA and migration bits for simplicity and to remove + * the notdirty callback faster. + */ + cpu_physical_memory_set_dirty_range(ram_addr, size, + DIRTY_CLIENTS_NOCODE); /* we remove the notdirty callback only if the code has been flushed */ if (!cpu_physical_memory_is_clean(ram_addr)) { @@ -2259,9 +2265,7 @@ static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, tb_invalidate_phys_range(addr, addr + length); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); } - if (dirty_log_mask) { - cpu_physical_memory_set_dirty_range_nocode(addr, length); - } + cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); } else { xen_modified_memory(addr, length); } @@ -3014,9 +3018,7 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val, dirty_log_mask = memory_region_get_dirty_log_mask(mr); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); - if (dirty_log_mask) { - cpu_physical_memory_set_dirty_range_nocode(addr1, 4); - } + cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask); r = MEMTX_OK; } if (result) { -- cgit v1.1 From e87f7778b64d4a6a78e16c288c7fdc6c15317d5f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 25 Mar 2015 15:21:39 +0100 Subject: exec: only check relevant bitmaps for cleanliness Most of the time, not all bitmaps have to be marked as dirty; do not do anything if the interesting ones are already dirty. Previously, any clean bitmap would have cause all the bitmaps to be marked dirty. In fact, unless running TCG most of the time bitmap operations need not be done at all, because memory_region_is_logging returns zero. In this case, skip the call to cpu_physical_memory_range_includes_clean altogether as well. With this patch, cpu_physical_memory_set_dirty_range is called unconditionally, so there need not be anymore a separate call to xen_modified_memory. Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index fe137bd..162c579 100644 --- a/exec.c +++ b/exec.c @@ -2259,16 +2259,20 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr, static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, hwaddr length) { - if (cpu_physical_memory_range_includes_clean(addr, length)) { - uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr); - if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { - tb_invalidate_phys_range(addr, addr + length); - dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); - } - cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); - } else { - xen_modified_memory(addr, length); + uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr); + /* No early return if dirty_log_mask is or becomes 0, because + * cpu_physical_memory_set_dirty_range will still call + * xen_modified_memory. + */ + if (dirty_log_mask) { + dirty_log_mask = + cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask); + } + if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { + tb_invalidate_phys_range(addr, addr + length); + dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); } + cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); } static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr) -- cgit v1.1 From 03eebc9e3246b9b3f5925aa41f7dfd7c1e467875 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Tue, 2 Dec 2014 11:23:18 +0000 Subject: memory: replace cpu_physical_memory_reset_dirty() with test-and-clear The cpu_physical_memory_reset_dirty() function is sometimes used together with cpu_physical_memory_get_dirty(). This is not atomic since two separate accesses to the dirty memory bitmap are made. Turn cpu_physical_memory_reset_dirty() and cpu_physical_memory_clear_dirty_range_type() into the atomic cpu_physical_memory_test_and_clear_dirty(). Signed-off-by: Stefan Hajnoczi Message-Id: <1417519399-3166-6-git-send-email-stefanha@redhat.com> Reviewed-by: Fam Zheng Signed-off-by: Paolo Bonzini --- exec.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'exec.c') diff --git a/exec.c b/exec.c index 162c579..487583b 100644 --- a/exec.c +++ b/exec.c @@ -852,16 +852,27 @@ static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length) } /* Note: start and end must be within the same ram block. */ -void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length, - unsigned client) +bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, + ram_addr_t length, + unsigned client) { - if (length == 0) - return; - cpu_physical_memory_clear_dirty_range_type(start, length, client); + unsigned long end, page; + bool dirty; - if (tcg_enabled()) { + if (length == 0) { + return false; + } + + end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; + page = start >> TARGET_PAGE_BITS; + dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client], + page, end - page); + + if (dirty && tcg_enabled()) { tlb_reset_dirty_range_all(start, length); } + + return dirty; } /* Called from RCU critical section */ -- cgit v1.1