diff options
Diffstat (limited to 'memory.c')
| -rw-r--r-- | memory.c | 378 |
1 files changed, 291 insertions, 87 deletions
@@ -47,6 +47,8 @@ static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners static QTAILQ_HEAD(, AddressSpace) address_spaces = QTAILQ_HEAD_INITIALIZER(address_spaces); +static GHashTable *flat_views; + typedef struct AddrRange AddrRange; /* @@ -154,7 +156,8 @@ enum ListenerDirection { Forward, Reverse }; /* No need to ref/unref .mr, the FlatRange keeps it alive. */ #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback, _args...) \ do { \ - MemoryRegionSection mrs = section_from_flat_range(fr, as); \ + MemoryRegionSection mrs = section_from_flat_range(fr, \ + address_space_to_flatview(as)); \ MEMORY_LISTENER_CALL(as, callback, dir, &mrs, ##_args); \ } while(0) @@ -208,7 +211,6 @@ static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd a, } typedef struct FlatRange FlatRange; -typedef struct FlatView FlatView; /* Range of memory in the global map. Addresses are absolute. */ struct FlatRange { @@ -229,6 +231,8 @@ struct FlatView { FlatRange *ranges; unsigned nr; unsigned nr_allocated; + struct AddressSpaceDispatch *dispatch; + MemoryRegion *root; }; typedef struct AddressSpaceOps AddressSpaceOps; @@ -237,11 +241,11 @@ typedef struct AddressSpaceOps AddressSpaceOps; for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) static inline MemoryRegionSection -section_from_flat_range(FlatRange *fr, AddressSpace *as) +section_from_flat_range(FlatRange *fr, FlatView *fv) { return (MemoryRegionSection) { .mr = fr->mr, - .address_space = as, + .fv = fv, .offset_within_region = fr->offset_in_region, .size = fr->addr.size, .offset_within_address_space = int128_get64(fr->addr.start), @@ -258,12 +262,17 @@ static bool flatrange_equal(FlatRange *a, FlatRange *b) && a->readonly == b->readonly; } -static void flatview_init(FlatView *view) +static FlatView *flatview_new(MemoryRegion *mr_root) { + FlatView *view; + + view = g_new0(FlatView, 1); view->ref = 1; - view->ranges = NULL; - view->nr = 0; - view->nr_allocated = 0; + view->root = mr_root; + memory_region_ref(mr_root); + trace_flatview_new(view, mr_root); + + return view; } /* Insert a range into a given position. Caller is responsible for maintaining @@ -287,25 +296,47 @@ static void flatview_destroy(FlatView *view) { int i; + trace_flatview_destroy(view, view->root); + if (view->dispatch) { + address_space_dispatch_free(view->dispatch); + } for (i = 0; i < view->nr; i++) { memory_region_unref(view->ranges[i].mr); } g_free(view->ranges); + memory_region_unref(view->root); g_free(view); } -static void flatview_ref(FlatView *view) +static bool flatview_ref(FlatView *view) { - atomic_inc(&view->ref); + return atomic_fetch_inc_nonzero(&view->ref) > 0; } static void flatview_unref(FlatView *view) { if (atomic_fetch_dec(&view->ref) == 1) { - flatview_destroy(view); + trace_flatview_destroy_rcu(view, view->root); + assert(view->root); + call_rcu(view, flatview_destroy, rcu); } } +FlatView *address_space_to_flatview(AddressSpace *as) +{ + return atomic_rcu_read(&as->current_map); +} + +AddressSpaceDispatch *flatview_to_dispatch(FlatView *fv) +{ + return fv->dispatch; +} + +AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as) +{ + return flatview_to_dispatch(address_space_to_flatview(as)); +} + static bool can_merge(FlatRange *r1, FlatRange *r2) { return int128_eq(addrrange_end(r1->addr), r2->addr.start) @@ -560,13 +591,14 @@ static MemTxResult access_with_adjusted_size(hwaddr addr, unsigned size, unsigned access_size_min, unsigned access_size_max, - MemTxResult (*access)(MemoryRegion *mr, - hwaddr addr, - uint64_t *value, - unsigned size, - unsigned shift, - uint64_t mask, - MemTxAttrs attrs), + MemTxResult (*access_fn) + (MemoryRegion *mr, + hwaddr addr, + uint64_t *value, + unsigned size, + unsigned shift, + uint64_t mask, + MemTxAttrs attrs), MemoryRegion *mr, MemTxAttrs attrs) { @@ -587,12 +619,12 @@ static MemTxResult access_with_adjusted_size(hwaddr addr, access_mask = -1ULL >> (64 - access_size * 8); if (memory_region_big_endian(mr)) { for (i = 0; i < size; i += access_size) { - r |= access(mr, addr + i, value, access_size, + r |= access_fn(mr, addr + i, value, access_size, (size - access_size - i) * 8, access_mask, attrs); } } else { for (i = 0; i < size; i += access_size) { - r |= access(mr, addr + i, value, access_size, i * 8, + r |= access_fn(mr, addr + i, value, access_size, i * 8, access_mask, attrs); } } @@ -701,13 +733,57 @@ static void render_memory_region(FlatView *view, } } +static MemoryRegion *memory_region_get_flatview_root(MemoryRegion *mr) +{ + while (mr->enabled) { + if (mr->alias) { + if (!mr->alias_offset && int128_ge(mr->size, mr->alias->size)) { + /* The alias is included in its entirety. Use it as + * the "real" root, so that we can share more FlatViews. + */ + mr = mr->alias; + continue; + } + } else if (!mr->terminates) { + unsigned int found = 0; + MemoryRegion *child, *next = NULL; + QTAILQ_FOREACH(child, &mr->subregions, subregions_link) { + if (child->enabled) { + if (++found > 1) { + next = NULL; + break; + } + if (!child->addr && int128_ge(mr->size, child->size)) { + /* A child is included in its entirety. If it's the only + * enabled one, use it in the hope of finding an alias down the + * way. This will also let us share FlatViews. + */ + next = child; + } + } + } + if (found == 0) { + return NULL; + } + if (next) { + mr = next; + continue; + } + } + + return mr; + } + + return NULL; +} + /* Render a memory topology into a list of disjoint absolute ranges. */ static FlatView *generate_memory_topology(MemoryRegion *mr) { + int i; FlatView *view; - view = g_new(FlatView, 1); - flatview_init(view); + view = flatview_new(mr); if (mr) { render_memory_region(view, mr, int128_zero(), @@ -715,6 +791,15 @@ static FlatView *generate_memory_topology(MemoryRegion *mr) } flatview_simplify(view); + view->dispatch = address_space_dispatch_new(view); + for (i = 0; i < view->nr; i++) { + MemoryRegionSection mrs = + section_from_flat_range(&view->ranges[i], view); + flatview_add_to_dispatch(view, &mrs); + } + address_space_dispatch_compact(view->dispatch); + g_hash_table_replace(flat_views, mr, view); + return view; } @@ -740,7 +825,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, fds_new[inew]))) { fd = &fds_old[iold]; section = (MemoryRegionSection) { - .address_space = as, + .fv = address_space_to_flatview(as), .offset_within_address_space = int128_get64(fd->addr.start), .size = fd->addr.size, }; @@ -753,7 +838,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, fds_old[iold]))) { fd = &fds_new[inew]; section = (MemoryRegionSection) { - .address_space = as, + .fv = address_space_to_flatview(as), .offset_within_address_space = int128_get64(fd->addr.start), .size = fd->addr.size, }; @@ -772,8 +857,12 @@ static FlatView *address_space_get_flatview(AddressSpace *as) FlatView *view; rcu_read_lock(); - view = atomic_rcu_read(&as->current_map); - flatview_ref(view); + do { + view = address_space_to_flatview(as); + /* If somebody has replaced as->current_map concurrently, + * flatview_ref returns false. + */ + } while (!flatview_ref(view)); rcu_read_unlock(); return view; } @@ -879,18 +968,81 @@ static void address_space_update_topology_pass(AddressSpace *as, } } +static void flatviews_init(void) +{ + static FlatView *empty_view; + + if (flat_views) { + return; + } -static void address_space_update_topology(AddressSpace *as) + flat_views = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, + (GDestroyNotify) flatview_unref); + if (!empty_view) { + empty_view = generate_memory_topology(NULL); + /* We keep it alive forever in the global variable. */ + flatview_ref(empty_view); + } else { + g_hash_table_replace(flat_views, NULL, empty_view); + flatview_ref(empty_view); + } +} + +static void flatviews_reset(void) +{ + AddressSpace *as; + + if (flat_views) { + g_hash_table_unref(flat_views); + flat_views = NULL; + } + flatviews_init(); + + /* Render unique FVs */ + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + MemoryRegion *physmr = memory_region_get_flatview_root(as->root); + + if (g_hash_table_lookup(flat_views, physmr)) { + continue; + } + + generate_memory_topology(physmr); + } +} + +static void address_space_set_flatview(AddressSpace *as) { - FlatView *old_view = address_space_get_flatview(as); - FlatView *new_view = generate_memory_topology(as->root); + FlatView *old_view = address_space_to_flatview(as); + MemoryRegion *physmr = memory_region_get_flatview_root(as->root); + FlatView *new_view = g_hash_table_lookup(flat_views, physmr); - address_space_update_topology_pass(as, old_view, new_view, false); - address_space_update_topology_pass(as, old_view, new_view, true); + assert(new_view); + + if (old_view == new_view) { + return; + } + + if (old_view) { + flatview_ref(old_view); + } + + flatview_ref(new_view); + + if (!QTAILQ_EMPTY(&as->listeners)) { + FlatView tmpview = { .nr = 0 }, *old_view2 = old_view; + + if (!old_view2) { + old_view2 = &tmpview; + } + address_space_update_topology_pass(as, old_view2, new_view, false); + address_space_update_topology_pass(as, old_view2, new_view, true); + } /* Writes are protected by the BQL. */ atomic_rcu_set(&as->current_map, new_view); - call_rcu(old_view, flatview_unref, rcu); + if (old_view) { + flatview_unref(old_view); + } /* Note that all the old MemoryRegions are still alive up to this * point. This relieves most MemoryListeners from the need to @@ -898,9 +1050,20 @@ static void address_space_update_topology(AddressSpace *as) * outside the iothread mutex, in which case precise reference * counting is necessary. */ - flatview_unref(old_view); + if (old_view) { + flatview_unref(old_view); + } +} - address_space_update_ioeventfds(as); +static void address_space_update_topology(AddressSpace *as) +{ + MemoryRegion *physmr = memory_region_get_flatview_root(as->root); + + flatviews_init(); + if (!g_hash_table_lookup(flat_views, physmr)) { + generate_memory_topology(physmr); + } + address_space_set_flatview(as); } void memory_region_transaction_begin(void) @@ -919,10 +1082,13 @@ void memory_region_transaction_commit(void) --memory_region_transaction_depth; if (!memory_region_transaction_depth) { if (memory_region_update_pending) { + flatviews_reset(); + MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { - address_space_update_topology(as); + address_space_set_flatview(as); + address_space_update_ioeventfds(as); } memory_region_update_pending = false; MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); @@ -1835,7 +2001,7 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr) view = address_space_get_flatview(as); FOR_EACH_FLAT_RANGE(fr, view) { if (fr->mr == mr) { - MemoryRegionSection mrs = section_from_flat_range(fr, as); + MemoryRegionSection mrs = section_from_flat_range(fr, view); listener->log_sync(listener, &mrs); } } @@ -1938,7 +2104,7 @@ static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpa FOR_EACH_FLAT_RANGE(fr, view) { if (fr->mr == mr) { section = (MemoryRegionSection) { - .address_space = as, + .fv = view, .offset_within_address_space = int128_get64(fr->addr.start), .size = fr->addr.size, }; @@ -2289,7 +2455,7 @@ static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr, } range = addrrange_make(int128_make64(addr), int128_make64(size)); - view = atomic_rcu_read(&as->current_map); + view = address_space_to_flatview(as); fr = flatview_lookup(view, range); if (!fr) { return ret; @@ -2300,7 +2466,7 @@ static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr, } ret.mr = fr->mr; - ret.address_space = as; + ret.fv = view; range = addrrange_intersection(range, fr->addr); ret.offset_within_region = fr->offset_in_region; ret.offset_within_region += int128_get64(int128_sub(range.start, @@ -2349,7 +2515,8 @@ void memory_global_dirty_log_sync(void) view = address_space_get_flatview(as); FOR_EACH_FLAT_RANGE(fr, view) { if (fr->dirty_log_mask) { - MemoryRegionSection mrs = section_from_flat_range(fr, as); + MemoryRegionSection mrs = section_from_flat_range(fr, view); + listener->log_sync(listener, &mrs); } } @@ -2434,7 +2601,7 @@ static void listener_add_address_space(MemoryListener *listener, FOR_EACH_FLAT_RANGE(fr, view) { MemoryRegionSection section = { .mr = fr->mr, - .address_space = as, + .fv = view, .offset_within_region = fr->offset_in_region, .size = fr->addr.size, .offset_within_address_space = int128_get64(fr->addr.start), @@ -2610,69 +2777,36 @@ void memory_region_invalidate_mmio_ptr(MemoryRegion *mr, hwaddr offset, void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) { memory_region_ref(root); - memory_region_transaction_begin(); - as->ref_count = 1; as->root = root; - as->malloced = false; - as->current_map = g_new(FlatView, 1); - flatview_init(as->current_map); + as->current_map = NULL; as->ioeventfd_nb = 0; as->ioeventfds = NULL; QTAILQ_INIT(&as->listeners); QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link); as->name = g_strdup(name ? name : "anonymous"); - address_space_init_dispatch(as); - memory_region_update_pending |= root->enabled; - memory_region_transaction_commit(); + address_space_update_topology(as); + address_space_update_ioeventfds(as); } static void do_address_space_destroy(AddressSpace *as) { - bool do_free = as->malloced; - - address_space_destroy_dispatch(as); assert(QTAILQ_EMPTY(&as->listeners)); flatview_unref(as->current_map); g_free(as->name); g_free(as->ioeventfds); memory_region_unref(as->root); - if (do_free) { - g_free(as); - } -} - -AddressSpace *address_space_init_shareable(MemoryRegion *root, const char *name) -{ - AddressSpace *as; - - QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { - if (root == as->root && as->malloced) { - as->ref_count++; - return as; - } - } - - as = g_malloc0(sizeof *as); - address_space_init(as, root, name); - as->malloced = true; - return as; } void address_space_destroy(AddressSpace *as) { MemoryRegion *root = as->root; - as->ref_count--; - if (as->ref_count) { - return; - } /* Flush out anything from MemoryListeners listening in on this */ memory_region_transaction_begin(); as->root = NULL; memory_region_transaction_commit(); QTAILQ_REMOVE(&address_spaces, as, address_spaces_link); - address_space_unregister(as); /* At this point, as->dispatch and as->current_map are dummy * entries that the guest should never use. Wait for the old @@ -2807,18 +2941,44 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, } } -static void mtree_print_flatview(fprintf_function p, void *f, - AddressSpace *as) +struct FlatViewInfo { + fprintf_function mon_printf; + void *f; + int counter; + bool dispatch_tree; +}; + +static void mtree_print_flatview(gpointer key, gpointer value, + gpointer user_data) { - FlatView *view = address_space_get_flatview(as); + FlatView *view = key; + GArray *fv_address_spaces = value; + struct FlatViewInfo *fvi = user_data; + fprintf_function p = fvi->mon_printf; + void *f = fvi->f; FlatRange *range = &view->ranges[0]; MemoryRegion *mr; int n = view->nr; + int i; + AddressSpace *as; + + p(f, "FlatView #%d\n", fvi->counter); + ++fvi->counter; + + for (i = 0; i < fv_address_spaces->len; ++i) { + as = g_array_index(fv_address_spaces, AddressSpace*, i); + p(f, " AS \"%s\", root: %s", as->name, memory_region_name(as->root)); + if (as->root->alias) { + p(f, ", alias %s", memory_region_name(as->root->alias)); + } + p(f, "\n"); + } + + p(f, " Root memory region: %s\n", + view->root ? memory_region_name(view->root) : "(none)"); if (n <= 0) { - p(f, MTREE_INDENT "No rendered FlatView for " - "address space '%s'\n", as->name); - flatview_unref(view); + p(f, MTREE_INDENT "No rendered FlatView\n\n"); return; } @@ -2845,21 +3005,65 @@ static void mtree_print_flatview(fprintf_function p, void *f, range++; } +#if !defined(CONFIG_USER_ONLY) + if (fvi->dispatch_tree && view->root) { + mtree_print_dispatch(p, f, view->dispatch, view->root); + } +#endif + + p(f, "\n"); +} + +static gboolean mtree_info_flatview_free(gpointer key, gpointer value, + gpointer user_data) +{ + FlatView *view = key; + GArray *fv_address_spaces = value; + + g_array_unref(fv_address_spaces); flatview_unref(view); + + return true; } -void mtree_info(fprintf_function mon_printf, void *f, bool flatview) +void mtree_info(fprintf_function mon_printf, void *f, bool flatview, + bool dispatch_tree) { MemoryRegionListHead ml_head; MemoryRegionList *ml, *ml2; AddressSpace *as; if (flatview) { + FlatView *view; + struct FlatViewInfo fvi = { + .mon_printf = mon_printf, + .f = f, + .counter = 0, + .dispatch_tree = dispatch_tree + }; + GArray *fv_address_spaces; + GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal); + + /* Gather all FVs in one table */ QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { - mon_printf(f, "address-space (flat view): %s\n", as->name); - mtree_print_flatview(mon_printf, f, as); - mon_printf(f, "\n"); + view = address_space_get_flatview(as); + + fv_address_spaces = g_hash_table_lookup(views, view); + if (!fv_address_spaces) { + fv_address_spaces = g_array_new(false, false, sizeof(as)); + g_hash_table_insert(views, view, fv_address_spaces); + } + + g_array_append_val(fv_address_spaces, as); } + + /* Print */ + g_hash_table_foreach(views, mtree_print_flatview, &fvi); + + /* Free */ + g_hash_table_foreach_remove(views, mtree_info_flatview_free, 0); + g_hash_table_unref(views); + return; } |