From de58ac72b6a062d1a61478284c0c0f8a0428613e Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 8 Jan 2012 19:46:17 +0200 Subject: ioport: change portio_list not to use memory_region_set_offset() memory_region_set_offset() will be going away soon, so don't use it. Use an alias instead. Signed-off-by: Avi Kivity Reviewed-by: Richard Henderson --- ioport.c | 28 +++++++++++++++++++++------- ioport.h | 1 + 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/ioport.c b/ioport.c index 36fa3a477e..8a474d3492 100644 --- a/ioport.c +++ b/ioport.c @@ -328,6 +328,7 @@ void portio_list_init(PortioList *piolist, piolist->ports = callbacks; piolist->nr = 0; piolist->regions = g_new0(MemoryRegion *, n); + piolist->aliases = g_new0(MemoryRegion *, n); piolist->address_space = NULL; piolist->opaque = opaque; piolist->name = name; @@ -336,6 +337,7 @@ void portio_list_init(PortioList *piolist, void portio_list_destroy(PortioList *piolist) { g_free(piolist->regions); + g_free(piolist->aliases); } static void portio_list_add_1(PortioList *piolist, @@ -345,7 +347,7 @@ static void portio_list_add_1(PortioList *piolist, { MemoryRegionPortio *pio; MemoryRegionOps *ops; - MemoryRegion *region; + MemoryRegion *region, *alias; unsigned i; /* Copy the sub-list and null-terminate it. */ @@ -362,12 +364,20 @@ static void portio_list_add_1(PortioList *piolist, ops->old_portio = pio; region = g_new(MemoryRegion, 1); + alias = g_new(MemoryRegion, 1); + /* + * Use an alias so that the callback is called with an absolute address, + * rather than an offset relative to to start + off_low. + */ memory_region_init_io(region, ops, piolist->opaque, piolist->name, - off_high - off_low); - memory_region_set_offset(region, start + off_low); + UINT64_MAX); + memory_region_init_alias(alias, piolist->name, + region, start + off_low, off_high - off_low); memory_region_add_subregion(piolist->address_space, - start + off_low, region); - piolist->regions[piolist->nr++] = region; + start + off_low, alias); + piolist->regions[piolist->nr] = region; + piolist->aliases[piolist->nr] = alias; + ++piolist->nr; } void portio_list_add(PortioList *piolist, @@ -409,15 +419,19 @@ void portio_list_add(PortioList *piolist, void portio_list_del(PortioList *piolist) { - MemoryRegion *mr; + MemoryRegion *mr, *alias; unsigned i; for (i = 0; i < piolist->nr; ++i) { mr = piolist->regions[i]; - memory_region_del_subregion(piolist->address_space, mr); + alias = piolist->aliases[i]; + memory_region_del_subregion(piolist->address_space, alias); + memory_region_destroy(alias); memory_region_destroy(mr); g_free((MemoryRegionOps *)mr->ops); g_free(mr); + g_free(alias); piolist->regions[i] = NULL; + piolist->aliases[i] = NULL; } } diff --git a/ioport.h b/ioport.h index ae3e9da0b5..ab29c89fb3 100644 --- a/ioport.h +++ b/ioport.h @@ -60,6 +60,7 @@ typedef struct PortioList { struct MemoryRegion *address_space; unsigned nr; struct MemoryRegion **regions; + struct MemoryRegion **aliases; void *opaque; const char *name; } PortioList; -- cgit v1.2.1 From 2b50aa1f14d8e0a40f905ad0c022443c15646914 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 8 Jan 2012 19:51:35 +0200 Subject: memory: remove memory_region_set_offset() memory_region_set_offset() complicates the API, and has been deprecated since its introduction. Now that it is no longer used, remove it. Signed-off-by: Avi Kivity Reviewed-by: Richard Henderson --- memory.c | 26 ++++++++++---------------- memory.h | 9 --------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/memory.c b/memory.c index 22816e20f9..e3a66de9a5 100644 --- a/memory.c +++ b/memory.c @@ -401,17 +401,17 @@ static void memory_region_iorange_read(IORange *iorange, *data = ((uint64_t)1 << (width * 8)) - 1; if (mrp) { - *data = mrp->read(mr->opaque, offset + mr->offset); + *data = mrp->read(mr->opaque, offset); } else if (width == 2) { mrp = find_portio(mr, offset, 1, false); assert(mrp); - *data = mrp->read(mr->opaque, offset + mr->offset) | - (mrp->read(mr->opaque, offset + mr->offset + 1) << 8); + *data = mrp->read(mr->opaque, offset) | + (mrp->read(mr->opaque, offset + 1) << 8); } return; } *data = 0; - access_with_adjusted_size(offset + mr->offset, data, width, + access_with_adjusted_size(offset, data, width, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_read_accessor, mr); @@ -428,16 +428,16 @@ static void memory_region_iorange_write(IORange *iorange, const MemoryRegionPortio *mrp = find_portio(mr, offset, width, true); if (mrp) { - mrp->write(mr->opaque, offset + mr->offset, data); + mrp->write(mr->opaque, offset, data); } else if (width == 2) { mrp = find_portio(mr, offset, 1, false); assert(mrp); - mrp->write(mr->opaque, offset + mr->offset, data & 0xff); - mrp->write(mr->opaque, offset + mr->offset + 1, data >> 8); + mrp->write(mr->opaque, offset, data & 0xff); + mrp->write(mr->opaque, offset + 1, data >> 8); } return; } - access_with_adjusted_size(offset + mr->offset, &data, width, + access_with_adjusted_size(offset, &data, width, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_write_accessor, mr); @@ -863,7 +863,6 @@ void memory_region_init(MemoryRegion *mr, mr->size = int128_2_64(); } mr->addr = 0; - mr->offset = 0; mr->subpage = false; mr->enabled = true; mr->terminates = false; @@ -925,7 +924,7 @@ static uint64_t memory_region_dispatch_read1(MemoryRegion *mr, } /* FIXME: support unaligned access */ - access_with_adjusted_size(addr + mr->offset, &data, size, + access_with_adjusted_size(addr, &data, size, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_read_accessor, mr); @@ -979,7 +978,7 @@ static void memory_region_dispatch_write(MemoryRegion *mr, } /* FIXME: support unaligned access */ - access_with_adjusted_size(addr + mr->offset, &data, size, + access_with_adjusted_size(addr, &data, size, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_write_accessor, mr); @@ -1122,11 +1121,6 @@ bool memory_region_is_rom(MemoryRegion *mr) return mr->ram && mr->readonly; } -void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset) -{ - mr->offset = offset; -} - void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) { uint8_t mask = 1 << client; diff --git a/memory.h b/memory.h index 4cf8d2f27a..4763286528 100644 --- a/memory.h +++ b/memory.h @@ -115,7 +115,6 @@ struct MemoryRegion { MemoryRegion *parent; Int128 size; target_phys_addr_t addr; - target_phys_addr_t offset; void (*destructor)(MemoryRegion *mr); ram_addr_t ram_addr; IORange iorange; @@ -358,14 +357,6 @@ bool memory_region_is_rom(MemoryRegion *mr); */ void *memory_region_get_ram_ptr(MemoryRegion *mr); -/** - * memory_region_set_offset: Sets an offset to be added to MemoryRegionOps - * callbacks. - * - * This function is deprecated and should not be used in new code. - */ -void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset); - /** * memory_region_set_log: Turn dirty logging on or off for a region. * -- cgit v1.2.1 From 946996e9c1b0b7b4a6dc56b253e947ea90d24ec4 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 13:58:44 +0200 Subject: memory: add shorthand for invoking a callback on all listeners Signed-off-by: Avi Kivity Reviewed-by: Richard Henderson --- memory.c | 54 +++++++++++++++++++----------------------------------- 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/memory.c b/memory.c index e3a66de9a5..570125c82f 100644 --- a/memory.c +++ b/memory.c @@ -678,31 +678,23 @@ static void address_space_update_ioeventfds(AddressSpace *as) as->ioeventfd_nb = ioeventfd_nb; } -typedef void ListenerCallback(MemoryListener *listener, - MemoryRegionSection *mrs); - -/* Want "void (&MemoryListener::*callback)(const MemoryRegionSection& s)" */ -static void memory_listener_update_region(FlatRange *fr, AddressSpace *as, - size_t callback_offset) -{ - MemoryRegionSection section = { - .mr = fr->mr, - .address_space = as->root, - .offset_within_region = fr->offset_in_region, - .size = int128_get64(fr->addr.size), - .offset_within_address_space = int128_get64(fr->addr.start), - }; - MemoryListener *listener; - - QLIST_FOREACH(listener, &memory_listeners, link) { - ListenerCallback *callback - = *(ListenerCallback **)((void *)listener + callback_offset); - callback(listener, §ion); - } -} - -#define MEMORY_LISTENER_UPDATE_REGION(fr, as, callback) \ - memory_listener_update_region(fr, as, offsetof(MemoryListener, callback)) +#define MEMORY_LISTENER_CALL(_callback, _args...) \ + do { \ + MemoryListener *_listener; \ + \ + QLIST_FOREACH(_listener, &memory_listeners, link) { \ + _listener->_callback(_listener, ##_args); \ + } \ + } while (0) + +#define MEMORY_LISTENER_UPDATE_REGION(fr, as, callback) \ + MEMORY_LISTENER_CALL(callback, &(MemoryRegionSection) { \ + .mr = (fr)->mr, \ + .address_space = (as)->root, \ + .offset_within_region = (fr)->offset_in_region, \ + .size = int128_get64((fr)->addr.size), \ + .offset_within_address_space = int128_get64((fr)->addr.start), \ + }) static void address_space_update_topology_pass(AddressSpace *as, FlatView old_view, @@ -1483,23 +1475,15 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space) void memory_global_dirty_log_start(void) { - MemoryListener *listener; - cpu_physical_memory_set_dirty_tracking(1); global_dirty_log = true; - QLIST_FOREACH(listener, &memory_listeners, link) { - listener->log_global_start(listener); - } + MEMORY_LISTENER_CALL(log_global_start); } void memory_global_dirty_log_stop(void) { - MemoryListener *listener; - global_dirty_log = false; - QLIST_FOREACH(listener, &memory_listeners, link) { - listener->log_global_stop(listener); - } + MEMORY_LISTENER_CALL(log_global_stop); cpu_physical_memory_set_dirty_tracking(0); } -- cgit v1.2.1 From 72e22d2fe17b85e56b4f0c437c61c6e2de97b308 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 15:05:50 +0200 Subject: memory: switch memory listeners to a QTAILQ This allows reverse iteration, which in turns allows consistent ordering among multiple listeners: l1->add l2->add l2->del l1->del Signed-off-by: Avi Kivity Reviewed-by: Richard Henderson --- hw/vhost.c | 1 + kvm-all.c | 1 + memory.c | 70 +++++++++++++++++++++++++++++++++++++++++++------------------- memory.h | 4 +++- xen-all.c | 1 + 5 files changed, 55 insertions(+), 22 deletions(-) diff --git a/hw/vhost.c b/hw/vhost.c index 5ece659f4a..47371454f4 100644 --- a/hw/vhost.c +++ b/hw/vhost.c @@ -751,6 +751,7 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) .log_sync = vhost_log_sync, .log_global_start = vhost_log_global_start, .log_global_stop = vhost_log_global_stop, + .priority = 10 }; hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions)); hdev->n_mem_sections = 0; diff --git a/kvm-all.c b/kvm-all.c index 0b87658f6a..6835fd46f8 100644 --- a/kvm-all.c +++ b/kvm-all.c @@ -726,6 +726,7 @@ static MemoryListener kvm_memory_listener = { .log_sync = kvm_log_sync, .log_global_start = kvm_log_global_start, .log_global_stop = kvm_log_global_stop, + .priority = 10, }; static void kvm_handle_interrupt(CPUState *env, int mask) diff --git a/memory.c b/memory.c index 570125c82f..a8fb39e367 100644 --- a/memory.c +++ b/memory.c @@ -27,8 +27,8 @@ unsigned memory_region_transaction_depth = 0; static bool memory_region_update_pending = false; static bool global_dirty_log = false; -static QLIST_HEAD(, MemoryListener) memory_listeners - = QLIST_HEAD_INITIALIZER(memory_listeners); +static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners + = QTAILQ_HEAD_INITIALIZER(memory_listeners); typedef struct AddrRange AddrRange; @@ -678,17 +678,31 @@ static void address_space_update_ioeventfds(AddressSpace *as) as->ioeventfd_nb = ioeventfd_nb; } -#define MEMORY_LISTENER_CALL(_callback, _args...) \ - do { \ - MemoryListener *_listener; \ - \ - QLIST_FOREACH(_listener, &memory_listeners, link) { \ - _listener->_callback(_listener, ##_args); \ - } \ +enum ListenerDirection { Forward, Reverse }; + +#define MEMORY_LISTENER_CALL(_callback, _direction, _args...) \ + do { \ + MemoryListener *_listener; \ + \ + switch (_direction) { \ + case Forward: \ + QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ + _listener->_callback(_listener, ##_args); \ + } \ + break; \ + case Reverse: \ + QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ + memory_listeners, link) { \ + _listener->_callback(_listener, ##_args); \ + } \ + break; \ + default: \ + abort(); \ + } \ } while (0) -#define MEMORY_LISTENER_UPDATE_REGION(fr, as, callback) \ - MEMORY_LISTENER_CALL(callback, &(MemoryRegionSection) { \ +#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback) \ + MEMORY_LISTENER_CALL(callback, dir, &(MemoryRegionSection) { \ .mr = (fr)->mr, \ .address_space = (as)->root, \ .offset_within_region = (fr)->offset_in_region, \ @@ -728,7 +742,7 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In old, but (not in new, or in new but attributes changed). */ if (!adding) { - MEMORY_LISTENER_UPDATE_REGION(frold, as, region_del); + MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del); as->ops->range_del(as, frold); } @@ -738,11 +752,11 @@ static void address_space_update_topology_pass(AddressSpace *as, if (adding) { if (frold->dirty_log_mask && !frnew->dirty_log_mask) { - MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_stop); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop); as->ops->log_stop(as, frnew); } else if (frnew->dirty_log_mask && !frold->dirty_log_mask) { as->ops->log_start(as, frnew); - MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_start); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start); } } @@ -753,7 +767,7 @@ static void address_space_update_topology_pass(AddressSpace *as, if (adding) { as->ops->range_add(as, frnew); - MEMORY_LISTENER_UPDATE_REGION(frnew, as, region_add); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add); } ++inew; @@ -1142,7 +1156,8 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr) FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) { if (fr->mr == mr) { - MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, log_sync); + MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, + Forward, log_sync); } } } @@ -1469,7 +1484,7 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space) FlatRange *fr; FOR_EACH_FLAT_RANGE(fr, &as->current_map) { - MEMORY_LISTENER_UPDATE_REGION(fr, as, log_sync); + MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync); } } @@ -1477,13 +1492,13 @@ void memory_global_dirty_log_start(void) { cpu_physical_memory_set_dirty_tracking(1); global_dirty_log = true; - MEMORY_LISTENER_CALL(log_global_start); + MEMORY_LISTENER_CALL(log_global_start, Forward); } void memory_global_dirty_log_stop(void) { global_dirty_log = false; - MEMORY_LISTENER_CALL(log_global_stop); + MEMORY_LISTENER_CALL(log_global_stop, Reverse); cpu_physical_memory_set_dirty_tracking(0); } @@ -1509,14 +1524,27 @@ static void listener_add_address_space(MemoryListener *listener, void memory_listener_register(MemoryListener *listener) { - QLIST_INSERT_HEAD(&memory_listeners, listener, link); + MemoryListener *other = NULL; + + if (QTAILQ_EMPTY(&memory_listeners) + || listener->priority >= QTAILQ_LAST(&memory_listeners, + memory_listeners)->priority) { + QTAILQ_INSERT_TAIL(&memory_listeners, listener, link); + } else { + QTAILQ_FOREACH(other, &memory_listeners, link) { + if (listener->priority < other->priority) { + break; + } + } + QTAILQ_INSERT_BEFORE(other, listener, link); + } listener_add_address_space(listener, &address_space_memory); listener_add_address_space(listener, &address_space_io); } void memory_listener_unregister(MemoryListener *listener) { - QLIST_REMOVE(listener, link); + QTAILQ_REMOVE(&memory_listeners, listener, link); } void set_system_memory_map(MemoryRegion *mr) diff --git a/memory.h b/memory.h index 4763286528..954dc86721 100644 --- a/memory.h +++ b/memory.h @@ -185,7 +185,9 @@ struct MemoryListener { void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section); void (*log_global_start)(MemoryListener *listener); void (*log_global_stop)(MemoryListener *listener); - QLIST_ENTRY(MemoryListener) link; + /* Lower = earlier (during add), later (during del) */ + unsigned priority; + QTAILQ_ENTRY(MemoryListener) link; }; /** diff --git a/xen-all.c b/xen-all.c index fd3916857c..8cb84efa55 100644 --- a/xen-all.c +++ b/xen-all.c @@ -495,6 +495,7 @@ static MemoryListener xen_memory_listener = { .log_sync = xen_log_sync, .log_global_start = xen_log_global_start, .log_global_stop = xen_log_global_stop, + .priority = 10, }; /* VCPU Operations, MMIO, IO ring ... */ -- cgit v1.2.1 From 0e0d36b446bbe913edcf5a7af24590360845f824 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 16:39:45 +0200 Subject: memory: code motion: move MEMORY_LISTENER_CALL() So it can be used in earlier code. Signed-off-by: Avi Kivity Reviewed-by: Richard Henderson --- memory.c | 64 ++++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/memory.c b/memory.c index a8fb39e367..90f5973cdc 100644 --- a/memory.c +++ b/memory.c @@ -82,6 +82,38 @@ static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2) return addrrange_make(start, int128_sub(end, start)); } +enum ListenerDirection { Forward, Reverse }; + +#define MEMORY_LISTENER_CALL(_callback, _direction, _args...) \ + do { \ + MemoryListener *_listener; \ + \ + switch (_direction) { \ + case Forward: \ + QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ + _listener->_callback(_listener, ##_args); \ + } \ + break; \ + case Reverse: \ + QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ + memory_listeners, link) { \ + _listener->_callback(_listener, ##_args); \ + } \ + break; \ + default: \ + abort(); \ + } \ + } while (0) + +#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback) \ + MEMORY_LISTENER_CALL(callback, dir, &(MemoryRegionSection) { \ + .mr = (fr)->mr, \ + .address_space = (as)->root, \ + .offset_within_region = (fr)->offset_in_region, \ + .size = int128_get64((fr)->addr.size), \ + .offset_within_address_space = int128_get64((fr)->addr.start), \ + }) + struct CoalescedMemoryRange { AddrRange addr; QTAILQ_ENTRY(CoalescedMemoryRange) link; @@ -678,38 +710,6 @@ static void address_space_update_ioeventfds(AddressSpace *as) as->ioeventfd_nb = ioeventfd_nb; } -enum ListenerDirection { Forward, Reverse }; - -#define MEMORY_LISTENER_CALL(_callback, _direction, _args...) \ - do { \ - MemoryListener *_listener; \ - \ - switch (_direction) { \ - case Forward: \ - QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ - _listener->_callback(_listener, ##_args); \ - } \ - break; \ - case Reverse: \ - QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ - memory_listeners, link) { \ - _listener->_callback(_listener, ##_args); \ - } \ - break; \ - default: \ - abort(); \ - } \ - } while (0) - -#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback) \ - MEMORY_LISTENER_CALL(callback, dir, &(MemoryRegionSection) { \ - .mr = (fr)->mr, \ - .address_space = (as)->root, \ - .offset_within_region = (fr)->offset_in_region, \ - .size = int128_get64((fr)->addr.size), \ - .offset_within_address_space = int128_get64((fr)->addr.start), \ - }) - static void address_space_update_topology_pass(AddressSpace *as, FlatView old_view, FlatView new_view, -- cgit v1.2.1 From 80a1ea3748203b840d8bad488ada4d6f5bb66c9d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 16:39:06 +0200 Subject: memory: move ioeventfd ops to MemoryListener This way the accelerator (kvm) can handle them directly. Signed-off-by: Avi Kivity Reviewed-by: Richard Henderson --- hw/vhost.c | 14 +++++++++++ kvm-all.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ memory.c | 74 +++++++++++++++------------------------------------------- memory.h | 4 ++++ xen-all.c | 14 +++++++++++ 5 files changed, 128 insertions(+), 56 deletions(-) diff --git a/hw/vhost.c b/hw/vhost.c index 47371454f4..e1e7e01584 100644 --- a/hw/vhost.c +++ b/hw/vhost.c @@ -720,6 +720,18 @@ static void vhost_virtqueue_cleanup(struct vhost_dev *dev, 0, virtio_queue_get_desc_size(vdev, idx)); } +static void vhost_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void vhost_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) { uint64_t features; @@ -751,6 +763,8 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) .log_sync = vhost_log_sync, .log_global_start = vhost_log_global_start, .log_global_stop = vhost_log_global_stop, + .eventfd_add = vhost_eventfd_add, + .eventfd_del = vhost_eventfd_del, .priority = 10 }; hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions)); diff --git a/kvm-all.c b/kvm-all.c index 6835fd46f8..a05e591c59 100644 --- a/kvm-all.c +++ b/kvm-all.c @@ -28,6 +28,7 @@ #include "kvm.h" #include "bswap.h" #include "memory.h" +#include "exec-memory.h" /* This check must be after config-host.h is included */ #ifdef CONFIG_EVENTFD @@ -718,6 +719,81 @@ static void kvm_log_global_stop(struct MemoryListener *listener) assert(r >= 0); } +static void kvm_mem_ioeventfd_add(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + int r; + + assert(match_data && section->size == 4); + + r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space, + data, true); + if (r < 0) { + abort(); + } +} + +static void kvm_mem_ioeventfd_del(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + int r; + + r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space, + data, false); + if (r < 0) { + abort(); + } +} + +static void kvm_io_ioeventfd_add(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + int r; + + assert(match_data && section->size == 2); + + r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space, + data, true); + if (r < 0) { + abort(); + } +} + +static void kvm_io_ioeventfd_del(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) + +{ + int r; + + r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space, + data, false); + if (r < 0) { + abort(); + } +} + +static void kvm_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + if (section->address_space == get_system_memory()) { + kvm_mem_ioeventfd_add(section, match_data, data, fd); + } else { + kvm_io_ioeventfd_add(section, match_data, data, fd); + } +} + +static void kvm_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + if (section->address_space == get_system_memory()) { + kvm_mem_ioeventfd_del(section, match_data, data, fd); + } else { + kvm_io_ioeventfd_del(section, match_data, data, fd); + } +} + static MemoryListener kvm_memory_listener = { .region_add = kvm_region_add, .region_del = kvm_region_del, @@ -726,6 +802,8 @@ static MemoryListener kvm_memory_listener = { .log_sync = kvm_log_sync, .log_global_start = kvm_log_global_start, .log_global_stop = kvm_log_global_stop, + .eventfd_add = kvm_eventfd_add, + .eventfd_del = kvm_eventfd_del, .priority = 10, }; diff --git a/memory.c b/memory.c index 90f5973cdc..a73c5a72e4 100644 --- a/memory.c +++ b/memory.c @@ -202,8 +202,6 @@ struct AddressSpaceOps { void (*range_del)(AddressSpace *as, FlatRange *fr); void (*log_start)(AddressSpace *as, FlatRange *fr); void (*log_stop)(AddressSpace *as, FlatRange *fr); - void (*ioeventfd_add)(AddressSpace *as, MemoryRegionIoeventfd *fd); - void (*ioeventfd_del)(AddressSpace *as, MemoryRegionIoeventfd *fd); }; #define FOR_EACH_FLAT_RANGE(var, view) \ @@ -369,37 +367,11 @@ static void as_memory_log_stop(AddressSpace *as, FlatRange *fr) { } -static void as_memory_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - assert(fd->match_data && int128_get64(fd->addr.size) == 4); - - r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start), - fd->data, true); - if (r < 0) { - abort(); - } -} - -static void as_memory_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start), - fd->data, false); - if (r < 0) { - abort(); - } -} - static const AddressSpaceOps address_space_ops_memory = { .range_add = as_memory_range_add, .range_del = as_memory_range_del, .log_start = as_memory_log_start, .log_stop = as_memory_log_stop, - .ioeventfd_add = as_memory_ioeventfd_add, - .ioeventfd_del = as_memory_ioeventfd_del, }; static AddressSpace address_space_memory = { @@ -493,35 +465,9 @@ static void as_io_range_del(AddressSpace *as, FlatRange *fr) int128_get64(fr->addr.size)); } -static void as_io_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - assert(fd->match_data && int128_get64(fd->addr.size) == 2); - - r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start), - fd->data, true); - if (r < 0) { - abort(); - } -} - -static void as_io_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start), - fd->data, false); - if (r < 0) { - abort(); - } -} - static const AddressSpaceOps address_space_ops_io = { .range_add = as_io_range_add, .range_del = as_io_range_del, - .ioeventfd_add = as_io_ioeventfd_add, - .ioeventfd_del = as_io_ioeventfd_del, }; static AddressSpace address_space_io = { @@ -653,6 +599,8 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, unsigned fds_old_nb) { unsigned iold, inew; + MemoryRegionIoeventfd *fd; + MemoryRegionSection section; /* Generate a symmetric difference of the old and new fd sets, adding * and deleting as necessary. @@ -664,13 +612,27 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, && (inew == fds_new_nb || memory_region_ioeventfd_before(fds_old[iold], fds_new[inew]))) { - as->ops->ioeventfd_del(as, &fds_old[iold]); + fd = &fds_old[iold]; + section = (MemoryRegionSection) { + .address_space = as->root, + .offset_within_address_space = int128_get64(fd->addr.start), + .size = int128_get64(fd->addr.size), + }; + MEMORY_LISTENER_CALL(eventfd_del, Forward, §ion, + fd->match_data, fd->data, fd->fd); ++iold; } else if (inew < fds_new_nb && (iold == fds_old_nb || memory_region_ioeventfd_before(fds_new[inew], fds_old[iold]))) { - as->ops->ioeventfd_add(as, &fds_new[inew]); + fd = &fds_new[inew]; + section = (MemoryRegionSection) { + .address_space = as->root, + .offset_within_address_space = int128_get64(fd->addr.start), + .size = int128_get64(fd->addr.size), + }; + MEMORY_LISTENER_CALL(eventfd_add, Reverse, §ion, + fd->match_data, fd->data, fd->fd); ++inew; } else { ++iold; diff --git a/memory.h b/memory.h index 954dc86721..84bb67c27a 100644 --- a/memory.h +++ b/memory.h @@ -185,6 +185,10 @@ struct MemoryListener { void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section); void (*log_global_start)(MemoryListener *listener); void (*log_global_stop)(MemoryListener *listener); + void (*eventfd_add)(MemoryListener *listener, MemoryRegionSection *section, + bool match_data, uint64_t data, int fd); + void (*eventfd_del)(MemoryListener *listener, MemoryRegionSection *section, + bool match_data, uint64_t data, int fd); /* Lower = earlier (during add), later (during del) */ unsigned priority; QTAILQ_ENTRY(MemoryListener) link; diff --git a/xen-all.c b/xen-all.c index 8cb84efa55..e005b63c7e 100644 --- a/xen-all.c +++ b/xen-all.c @@ -487,6 +487,18 @@ static void xen_log_global_stop(MemoryListener *listener) { } +static void xen_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void xen_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + static MemoryListener xen_memory_listener = { .region_add = xen_region_add, .region_del = xen_region_del, @@ -495,6 +507,8 @@ static MemoryListener xen_memory_listener = { .log_sync = xen_log_sync, .log_global_start = xen_log_global_start, .log_global_stop = xen_log_global_stop, + .eventfd_add = xen_eventfd_add, + .eventfd_del = xen_eventfd_del, .priority = 10, }; -- cgit v1.2.1 From 7a8499e88bffff66d90300d4f2018c938543b1cf Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 17:01:23 +0200 Subject: memory: add a readonly attribute to MemoryRegionSection .readonly cannot be obtained from the MemoryRegion, since it is inherited from aliases (so you can have a MemoryRegion mapped RW at one address and RO at another). Record it in a MemoryRegionSection for listeners. Signed-off-by: Avi Kivity Reviewed-by: Richard Henderson --- memory.c | 5 +++++ memory.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/memory.c b/memory.c index a73c5a72e4..ccc3efbab4 100644 --- a/memory.c +++ b/memory.c @@ -112,6 +112,7 @@ enum ListenerDirection { Forward, Reverse }; .offset_within_region = (fr)->offset_in_region, \ .size = int128_get64((fr)->addr.size), \ .offset_within_address_space = int128_get64((fr)->addr.start), \ + .readonly = (fr)->readonly, \ }) struct CoalescedMemoryRange { @@ -342,6 +343,7 @@ static void as_memory_range_add(AddressSpace *as, FlatRange *fr) .offset_within_address_space = int128_get64(fr->addr.start), .offset_within_region = fr->offset_in_region, .size = int128_get64(fr->addr.size), + .readonly = fr->readonly, }; cpu_register_physical_memory_log(§ion, fr->readable, fr->readonly); @@ -354,6 +356,7 @@ static void as_memory_range_del(AddressSpace *as, FlatRange *fr) .offset_within_address_space = int128_get64(fr->addr.start), .offset_within_region = int128_get64(fr->addr.start), .size = int128_get64(fr->addr.size), + .readonly = fr->readonly, }; cpu_register_physical_memory_log(§ion, true, false); @@ -1437,6 +1440,7 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space, fr->addr.start)); ret.size = int128_get64(range.size); ret.offset_within_address_space = int128_get64(range.start); + ret.readonly = fr->readonly; return ret; } @@ -1479,6 +1483,7 @@ static void listener_add_address_space(MemoryListener *listener, .offset_within_region = fr->offset_in_region, .size = int128_get64(fr->addr.size), .offset_within_address_space = int128_get64(fr->addr.start), + .readonly = fr->readonly, }; listener->region_add(listener, §ion); } diff --git a/memory.h b/memory.h index 84bb67c27a..1d99cee019 100644 --- a/memory.h +++ b/memory.h @@ -160,6 +160,7 @@ typedef struct MemoryRegionSection MemoryRegionSection; * @size: the size of the section; will not exceed @mr's boundaries * @offset_within_address_space: the address of the first byte of the section * relative to the region's address space + * @readonly: writes to this section are ignored */ struct MemoryRegionSection { MemoryRegion *mr; @@ -167,6 +168,7 @@ struct MemoryRegionSection { target_phys_addr_t offset_within_region; uint64_t size; target_phys_addr_t offset_within_address_space; + bool readonly; }; typedef struct MemoryListener MemoryListener; -- cgit v1.2.1 From d7ec83e6b57f48cf07f663e232e4aa2b32bc33c7 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 17:07:26 +0200 Subject: memory: don't pass ->readable attribute to cpu_register_physical_memory_log It can be derived from the MemoryRegion itself (which is why it is not used there). Signed-off-by: Avi Kivity Reviewed-by: Richard Henderson --- exec-obsolete.h | 2 +- exec.c | 2 +- memory.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/exec-obsolete.h b/exec-obsolete.h index 94c23d0951..23ffbaaf4c 100644 --- a/exec-obsolete.h +++ b/exec-obsolete.h @@ -37,7 +37,7 @@ void cpu_unregister_io_memory(int table_address); struct MemoryRegionSection; void cpu_register_physical_memory_log(struct MemoryRegionSection *section, - bool readable, bool readonly); + bool readonly); void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size); void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size); diff --git a/exec.c b/exec.c index b81677ade9..d3020ab57c 100644 --- a/exec.c +++ b/exec.c @@ -2529,7 +2529,7 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, before calculating this offset. This should not be a problem unless the low bits of start_addr and region_offset differ. */ void cpu_register_physical_memory_log(MemoryRegionSection *section, - bool readable, bool readonly) + bool readonly) { target_phys_addr_t start_addr = section->offset_within_address_space; ram_addr_t size = section->size; diff --git a/memory.c b/memory.c index ccc3efbab4..85959ee544 100644 --- a/memory.c +++ b/memory.c @@ -346,7 +346,7 @@ static void as_memory_range_add(AddressSpace *as, FlatRange *fr) .readonly = fr->readonly, }; - cpu_register_physical_memory_log(§ion, fr->readable, fr->readonly); + cpu_register_physical_memory_log(§ion, fr->readonly); } static void as_memory_range_del(AddressSpace *as, FlatRange *fr) @@ -359,7 +359,7 @@ static void as_memory_range_del(AddressSpace *as, FlatRange *fr) .readonly = fr->readonly, }; - cpu_register_physical_memory_log(§ion, true, false); + cpu_register_physical_memory_log(§ion, false); } static void as_memory_log_start(AddressSpace *as, FlatRange *fr) -- cgit v1.2.1 From 9363274709af23c499b6db3eb402b6f9aa46c40b Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 16:54:16 +0200 Subject: memory: use a MemoryListener for core memory map updates too This transforms memory.c into a library which can then be unit tested easily, by feeding it inputs and listening to its outputs. Signed-off-by: Avi Kivity Reviewed-by: Richard Henderson --- exec-obsolete.h | 3 +++ exec.c | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ memory.c | 27 +-------------------- 3 files changed, 79 insertions(+), 26 deletions(-) diff --git a/exec-obsolete.h b/exec-obsolete.h index 23ffbaaf4c..4dbe4768aa 100644 --- a/exec-obsolete.h +++ b/exec-obsolete.h @@ -121,6 +121,9 @@ static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start, void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end, int dirty_flags); + +extern const IORangeOps memory_region_iorange_ops; + #endif #endif diff --git a/exec.c b/exec.c index d3020ab57c..7fb5d4e8ed 100644 --- a/exec.c +++ b/exec.c @@ -3488,6 +3488,79 @@ static void io_mem_init(void) "watch", UINT64_MAX); } +static void core_region_add(MemoryListener *listener, + MemoryRegionSection *section) +{ + if (section->address_space == get_system_memory()) { + cpu_register_physical_memory_log(section, section->readonly); + } else { + iorange_init(§ion->mr->iorange, &memory_region_iorange_ops, + section->offset_within_address_space, section->size); + ioport_register(§ion->mr->iorange); + } +} + +static void core_region_del(MemoryListener *listener, + MemoryRegionSection *section) +{ + if (section->address_space == get_system_memory()) { + cpu_register_physical_memory_log(section, false); + } else { + isa_unassign_ioport(section->offset_within_address_space, + section->size); + } +} + +static void core_log_start(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_log_stop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_log_sync(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_log_global_start(MemoryListener *listener) +{ + cpu_physical_memory_set_dirty_tracking(1); +} + +static void core_log_global_stop(MemoryListener *listener) +{ + cpu_physical_memory_set_dirty_tracking(0); +} + +static void core_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void core_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static MemoryListener core_memory_listener = { + .region_add = core_region_add, + .region_del = core_region_del, + .log_start = core_log_start, + .log_stop = core_log_stop, + .log_sync = core_log_sync, + .log_global_start = core_log_global_start, + .log_global_stop = core_log_global_stop, + .eventfd_add = core_eventfd_add, + .eventfd_del = core_eventfd_del, + .priority = 0, +}; + static void memory_map_init(void) { system_memory = g_malloc(sizeof(*system_memory)); @@ -3497,6 +3570,8 @@ static void memory_map_init(void) system_io = g_malloc(sizeof(*system_io)); memory_region_init(system_io, "io", 65536); set_system_io_map(system_io); + + memory_listener_register(&core_memory_listener); } MemoryRegion *get_system_memory(void) diff --git a/memory.c b/memory.c index 85959ee544..d5591f8c25 100644 --- a/memory.c +++ b/memory.c @@ -338,28 +338,10 @@ static void access_with_adjusted_size(target_phys_addr_t addr, static void as_memory_range_add(AddressSpace *as, FlatRange *fr) { - MemoryRegionSection section = { - .mr = fr->mr, - .offset_within_address_space = int128_get64(fr->addr.start), - .offset_within_region = fr->offset_in_region, - .size = int128_get64(fr->addr.size), - .readonly = fr->readonly, - }; - - cpu_register_physical_memory_log(§ion, fr->readonly); } static void as_memory_range_del(AddressSpace *as, FlatRange *fr) { - MemoryRegionSection section = { - .mr = &io_mem_unassigned, - .offset_within_address_space = int128_get64(fr->addr.start), - .offset_within_region = int128_get64(fr->addr.start), - .size = int128_get64(fr->addr.size), - .readonly = fr->readonly, - }; - - cpu_register_physical_memory_log(§ion, false); } static void as_memory_log_start(AddressSpace *as, FlatRange *fr) @@ -450,22 +432,17 @@ static void memory_region_iorange_write(IORange *iorange, memory_region_write_accessor, mr); } -static const IORangeOps memory_region_iorange_ops = { +const IORangeOps memory_region_iorange_ops = { .read = memory_region_iorange_read, .write = memory_region_iorange_write, }; static void as_io_range_add(AddressSpace *as, FlatRange *fr) { - iorange_init(&fr->mr->iorange, &memory_region_iorange_ops, - int128_get64(fr->addr.start), int128_get64(fr->addr.size)); - ioport_register(&fr->mr->iorange); } static void as_io_range_del(AddressSpace *as, FlatRange *fr) { - isa_unassign_ioport(int128_get64(fr->addr.start), - int128_get64(fr->addr.size)); } static const AddressSpaceOps address_space_ops_io = { @@ -1456,7 +1433,6 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space) void memory_global_dirty_log_start(void) { - cpu_physical_memory_set_dirty_tracking(1); global_dirty_log = true; MEMORY_LISTENER_CALL(log_global_start, Forward); } @@ -1465,7 +1441,6 @@ void memory_global_dirty_log_stop(void) { global_dirty_log = false; MEMORY_LISTENER_CALL(log_global_stop, Reverse); - cpu_physical_memory_set_dirty_tracking(0); } static void listener_add_address_space(MemoryListener *listener, -- cgit v1.2.1 From 8df8a8436fa22442a58b366fbb7e471b4a8d88ce Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 17:19:46 +0200 Subject: memory: drop AddressSpaceOps All functionality has been moved to various MemoryListeners. Signed-off-by: Avi Kivity Reviewed-by: Richard Henderson --- memory.c | 56 ++------------------------------------------------------ 1 file changed, 2 insertions(+), 54 deletions(-) diff --git a/memory.c b/memory.c index d5591f8c25..12ba60f703 100644 --- a/memory.c +++ b/memory.c @@ -191,20 +191,12 @@ typedef struct AddressSpaceOps AddressSpaceOps; /* A system address space - I/O, memory, etc. */ struct AddressSpace { - const AddressSpaceOps *ops; MemoryRegion *root; FlatView current_map; int ioeventfd_nb; MemoryRegionIoeventfd *ioeventfds; }; -struct AddressSpaceOps { - void (*range_add)(AddressSpace *as, FlatRange *fr); - void (*range_del)(AddressSpace *as, FlatRange *fr); - void (*log_start)(AddressSpace *as, FlatRange *fr); - void (*log_stop)(AddressSpace *as, FlatRange *fr); -}; - #define FOR_EACH_FLAT_RANGE(var, view) \ for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) @@ -336,32 +328,7 @@ static void access_with_adjusted_size(target_phys_addr_t addr, } } -static void as_memory_range_add(AddressSpace *as, FlatRange *fr) -{ -} - -static void as_memory_range_del(AddressSpace *as, FlatRange *fr) -{ -} - -static void as_memory_log_start(AddressSpace *as, FlatRange *fr) -{ -} - -static void as_memory_log_stop(AddressSpace *as, FlatRange *fr) -{ -} - -static const AddressSpaceOps address_space_ops_memory = { - .range_add = as_memory_range_add, - .range_del = as_memory_range_del, - .log_start = as_memory_log_start, - .log_stop = as_memory_log_stop, -}; - -static AddressSpace address_space_memory = { - .ops = &address_space_ops_memory, -}; +static AddressSpace address_space_memory; static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset, unsigned width, bool write) @@ -437,22 +404,7 @@ const IORangeOps memory_region_iorange_ops = { .write = memory_region_iorange_write, }; -static void as_io_range_add(AddressSpace *as, FlatRange *fr) -{ -} - -static void as_io_range_del(AddressSpace *as, FlatRange *fr) -{ -} - -static const AddressSpaceOps address_space_ops_io = { - .range_add = as_io_range_add, - .range_del = as_io_range_del, -}; - -static AddressSpace address_space_io = { - .ops = &address_space_ops_io, -}; +static AddressSpace address_space_io; static AddressSpace *memory_region_to_address_space(MemoryRegion *mr) { @@ -685,7 +637,6 @@ static void address_space_update_topology_pass(AddressSpace *as, if (!adding) { MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del); - as->ops->range_del(as, frold); } ++iold; @@ -695,9 +646,7 @@ static void address_space_update_topology_pass(AddressSpace *as, if (adding) { if (frold->dirty_log_mask && !frnew->dirty_log_mask) { MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop); - as->ops->log_stop(as, frnew); } else if (frnew->dirty_log_mask && !frold->dirty_log_mask) { - as->ops->log_start(as, frnew); MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start); } } @@ -708,7 +657,6 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In new */ if (adding) { - as->ops->range_add(as, frnew); MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add); } -- cgit v1.2.1 From 7376e5827a898d8ddf10f929047cc84bf65d9f52 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 21:05:17 +0200 Subject: memory: allow MemoryListeners to observe a specific address space Ignore any regions not belonging to a specified address space. Signed-off-by: Avi Kivity --- exec.c | 2 +- hw/vhost.c | 2 +- kvm-all.c | 2 +- memory.c | 45 +++++++++++++++++++++++++++++++++++++++------ memory.h | 4 +++- xen-all.c | 2 +- 6 files changed, 46 insertions(+), 11 deletions(-) diff --git a/exec.c b/exec.c index 7fb5d4e8ed..aa54eb1d7b 100644 --- a/exec.c +++ b/exec.c @@ -3571,7 +3571,7 @@ static void memory_map_init(void) memory_region_init(system_io, "io", 65536); set_system_io_map(system_io); - memory_listener_register(&core_memory_listener); + memory_listener_register(&core_memory_listener, NULL); } MemoryRegion *get_system_memory(void) diff --git a/hw/vhost.c b/hw/vhost.c index e1e7e01584..01f676a959 100644 --- a/hw/vhost.c +++ b/hw/vhost.c @@ -774,7 +774,7 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) hdev->log_size = 0; hdev->log_enabled = false; hdev->started = false; - memory_listener_register(&hdev->memory_listener); + memory_listener_register(&hdev->memory_listener, NULL); hdev->force = force; return 0; fail: diff --git a/kvm-all.c b/kvm-all.c index a05e591c59..15bc42fcdb 100644 --- a/kvm-all.c +++ b/kvm-all.c @@ -1049,7 +1049,7 @@ int kvm_init(void) } kvm_state = s; - memory_listener_register(&kvm_memory_listener); + memory_listener_register(&kvm_memory_listener, NULL); s->many_ioeventfds = kvm_check_many_ioeventfds(); diff --git a/memory.c b/memory.c index 12ba60f703..35e1483859 100644 --- a/memory.c +++ b/memory.c @@ -84,7 +84,14 @@ static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2) enum ListenerDirection { Forward, Reverse }; -#define MEMORY_LISTENER_CALL(_callback, _direction, _args...) \ +static bool memory_listener_match(MemoryListener *listener, + MemoryRegionSection *section) +{ + return !listener->address_space_filter + || listener->address_space_filter == section->address_space; +} + +#define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \ do { \ MemoryListener *_listener; \ \ @@ -105,15 +112,40 @@ enum ListenerDirection { Forward, Reverse }; } \ } while (0) +#define MEMORY_LISTENER_CALL(_callback, _direction, _section, _args...) \ + do { \ + MemoryListener *_listener; \ + \ + switch (_direction) { \ + case Forward: \ + QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ + if (memory_listener_match(_listener, _section)) { \ + _listener->_callback(_listener, _section, ##_args); \ + } \ + } \ + break; \ + case Reverse: \ + QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ + memory_listeners, link) { \ + if (memory_listener_match(_listener, _section)) { \ + _listener->_callback(_listener, _section, ##_args); \ + } \ + } \ + break; \ + default: \ + abort(); \ + } \ + } while (0) + #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback) \ - MEMORY_LISTENER_CALL(callback, dir, &(MemoryRegionSection) { \ + MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) { \ .mr = (fr)->mr, \ .address_space = (as)->root, \ .offset_within_region = (fr)->offset_in_region, \ .size = int128_get64((fr)->addr.size), \ .offset_within_address_space = int128_get64((fr)->addr.start), \ .readonly = (fr)->readonly, \ - }) + })) struct CoalescedMemoryRange { AddrRange addr; @@ -1382,13 +1414,13 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space) void memory_global_dirty_log_start(void) { global_dirty_log = true; - MEMORY_LISTENER_CALL(log_global_start, Forward); + MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward); } void memory_global_dirty_log_stop(void) { global_dirty_log = false; - MEMORY_LISTENER_CALL(log_global_stop, Reverse); + MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse); } static void listener_add_address_space(MemoryListener *listener, @@ -1412,10 +1444,11 @@ static void listener_add_address_space(MemoryListener *listener, } } -void memory_listener_register(MemoryListener *listener) +void memory_listener_register(MemoryListener *listener, MemoryRegion *filter) { MemoryListener *other = NULL; + listener->address_space_filter = filter; if (QTAILQ_EMPTY(&memory_listeners) || listener->priority >= QTAILQ_LAST(&memory_listeners, memory_listeners)->priority) { diff --git a/memory.h b/memory.h index 1d99cee019..bc9600b402 100644 --- a/memory.h +++ b/memory.h @@ -193,6 +193,7 @@ struct MemoryListener { bool match_data, uint64_t data, int fd); /* Lower = earlier (during add), later (during del) */ unsigned priority; + MemoryRegion *address_space_filter; QTAILQ_ENTRY(MemoryListener) link; }; @@ -685,8 +686,9 @@ void memory_region_transaction_commit(void); * space * * @listener: an object containing the callbacks to be called + * @filter: if non-%NULL, only regions in this address space will be observed */ -void memory_listener_register(MemoryListener *listener); +void memory_listener_register(MemoryListener *listener, MemoryRegion *filter); /** * memory_listener_unregister: undo the effect of memory_listener_register() diff --git a/xen-all.c b/xen-all.c index e005b63c7e..dd52f02573 100644 --- a/xen-all.c +++ b/xen-all.c @@ -989,7 +989,7 @@ int xen_hvm_init(void) state->memory_listener = xen_memory_listener; QLIST_INIT(&state->physmap); - memory_listener_register(&state->memory_listener); + memory_listener_register(&state->memory_listener, NULL); state->log_for_dirtybit = NULL; /* Initialize backend core & drivers */ -- cgit v1.2.1 From 947f562c989d034c6a82920e11f4641f1681583d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 21:10:42 +0200 Subject: xen: ignore I/O memory regions Signed-off-by: Avi Kivity --- xen-all.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xen-all.c b/xen-all.c index dd52f02573..a58a397ca4 100644 --- a/xen-all.c +++ b/xen-all.c @@ -989,7 +989,7 @@ int xen_hvm_init(void) state->memory_listener = xen_memory_listener; QLIST_INIT(&state->physmap); - memory_listener_register(&state->memory_listener, NULL); + memory_listener_register(&state->memory_listener, get_system_memory()); state->log_for_dirtybit = NULL; /* Initialize backend core & drivers */ -- cgit v1.2.1 From 4855d41a61301a04e54c074f1139040e4cb4dd00 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 21:16:05 +0200 Subject: memory: split memory listener for the two address spaces The memory and I/O address spaces do different things, so split them into two memory listeners. Signed-off-by: Avi Kivity --- exec.c | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/exec.c b/exec.c index aa54eb1d7b..16973a675f 100644 --- a/exec.c +++ b/exec.c @@ -3491,24 +3491,13 @@ static void io_mem_init(void) static void core_region_add(MemoryListener *listener, MemoryRegionSection *section) { - if (section->address_space == get_system_memory()) { - cpu_register_physical_memory_log(section, section->readonly); - } else { - iorange_init(§ion->mr->iorange, &memory_region_iorange_ops, - section->offset_within_address_space, section->size); - ioport_register(§ion->mr->iorange); - } + cpu_register_physical_memory_log(section, section->readonly); } static void core_region_del(MemoryListener *listener, MemoryRegionSection *section) { - if (section->address_space == get_system_memory()) { - cpu_register_physical_memory_log(section, false); - } else { - isa_unassign_ioport(section->offset_within_address_space, - section->size); - } + cpu_register_physical_memory_log(section, false); } static void core_log_start(MemoryListener *listener, @@ -3548,6 +3537,55 @@ static void core_eventfd_del(MemoryListener *listener, { } +static void io_region_add(MemoryListener *listener, + MemoryRegionSection *section) +{ + iorange_init(§ion->mr->iorange, &memory_region_iorange_ops, + section->offset_within_address_space, section->size); + ioport_register(§ion->mr->iorange); +} + +static void io_region_del(MemoryListener *listener, + MemoryRegionSection *section) +{ + isa_unassign_ioport(section->offset_within_address_space, section->size); +} + +static void io_log_start(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_stop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_sync(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_global_start(MemoryListener *listener) +{ +} + +static void io_log_global_stop(MemoryListener *listener) +{ +} + +static void io_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void io_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + static MemoryListener core_memory_listener = { .region_add = core_region_add, .region_del = core_region_del, @@ -3561,6 +3599,19 @@ static MemoryListener core_memory_listener = { .priority = 0, }; +static MemoryListener io_memory_listener = { + .region_add = io_region_add, + .region_del = io_region_del, + .log_start = io_log_start, + .log_stop = io_log_stop, + .log_sync = io_log_sync, + .log_global_start = io_log_global_start, + .log_global_stop = io_log_global_stop, + .eventfd_add = io_eventfd_add, + .eventfd_del = io_eventfd_del, + .priority = 0, +}; + static void memory_map_init(void) { system_memory = g_malloc(sizeof(*system_memory)); @@ -3571,7 +3622,8 @@ static void memory_map_init(void) memory_region_init(system_io, "io", 65536); set_system_io_map(system_io); - memory_listener_register(&core_memory_listener, NULL); + memory_listener_register(&core_memory_listener, system_memory); + memory_listener_register(&io_memory_listener, system_io); } MemoryRegion *get_system_memory(void) -- cgit v1.2.1 From 50c1e1491e1981ecba14a477897681d8d0602500 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 8 Feb 2012 21:36:02 +0200 Subject: memory: support stateless memory listeners Current memory listeners are incremental; that is, they are expected to maintain their own state, and receive callbacks for changes to that state. This patch adds support for stateless listeners; these work by receiving a ->begin() callback (which tells them that new state is coming), a sequence of ->region_add() and ->region_nop() callbacks, and then a ->commit() callback which signifies the end of the new state. They should ignore ->region_del() callbacks. Signed-off-by: Avi Kivity --- exec.c | 32 ++++++++++++++++++++++++++++++++ hw/vhost.c | 16 ++++++++++++++++ kvm-all.c | 16 ++++++++++++++++ memory.c | 5 +++++ memory.h | 3 +++ xen-all.c | 16 ++++++++++++++++ 6 files changed, 88 insertions(+) diff --git a/exec.c b/exec.c index 16973a675f..6726afd347 100644 --- a/exec.c +++ b/exec.c @@ -3488,6 +3488,14 @@ static void io_mem_init(void) "watch", UINT64_MAX); } +static void core_begin(MemoryListener *listener) +{ +} + +static void core_commit(MemoryListener *listener) +{ +} + static void core_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -3500,6 +3508,11 @@ static void core_region_del(MemoryListener *listener, cpu_register_physical_memory_log(section, false); } +static void core_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static void core_log_start(MemoryListener *listener, MemoryRegionSection *section) { @@ -3537,6 +3550,14 @@ static void core_eventfd_del(MemoryListener *listener, { } +static void io_begin(MemoryListener *listener) +{ +} + +static void io_commit(MemoryListener *listener) +{ +} + static void io_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -3551,6 +3572,11 @@ static void io_region_del(MemoryListener *listener, isa_unassign_ioport(section->offset_within_address_space, section->size); } +static void io_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static void io_log_start(MemoryListener *listener, MemoryRegionSection *section) { @@ -3587,8 +3613,11 @@ static void io_eventfd_del(MemoryListener *listener, } static MemoryListener core_memory_listener = { + .begin = core_begin, + .commit = core_commit, .region_add = core_region_add, .region_del = core_region_del, + .region_nop = core_region_nop, .log_start = core_log_start, .log_stop = core_log_stop, .log_sync = core_log_sync, @@ -3600,8 +3629,11 @@ static MemoryListener core_memory_listener = { }; static MemoryListener io_memory_listener = { + .begin = io_begin, + .commit = io_commit, .region_add = io_region_add, .region_del = io_region_del, + .region_nop = io_region_nop, .log_start = io_log_start, .log_stop = io_log_stop, .log_sync = io_log_sync, diff --git a/hw/vhost.c b/hw/vhost.c index 01f676a959..8d3ba5b608 100644 --- a/hw/vhost.c +++ b/hw/vhost.c @@ -436,6 +436,14 @@ static bool vhost_section(MemoryRegionSection *section) && memory_region_is_ram(section->mr); } +static void vhost_begin(MemoryListener *listener) +{ +} + +static void vhost_commit(MemoryListener *listener) +{ +} + static void vhost_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -476,6 +484,11 @@ static void vhost_region_del(MemoryListener *listener, } } +static void vhost_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static int vhost_virtqueue_set_addr(struct vhost_dev *dev, struct vhost_virtqueue *vq, unsigned idx, bool enable_log) @@ -756,8 +769,11 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) hdev->features = features; hdev->memory_listener = (MemoryListener) { + .begin = vhost_begin, + .commit = vhost_commit, .region_add = vhost_region_add, .region_del = vhost_region_del, + .region_nop = vhost_region_nop, .log_start = vhost_log_start, .log_stop = vhost_log_stop, .log_sync = vhost_log_sync, diff --git a/kvm-all.c b/kvm-all.c index 15bc42fcdb..c07823df98 100644 --- a/kvm-all.c +++ b/kvm-all.c @@ -680,6 +680,14 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add) } } +static void kvm_begin(MemoryListener *listener) +{ +} + +static void kvm_commit(MemoryListener *listener) +{ +} + static void kvm_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -692,6 +700,11 @@ static void kvm_region_del(MemoryListener *listener, kvm_set_phys_mem(section, false); } +static void kvm_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static void kvm_log_sync(MemoryListener *listener, MemoryRegionSection *section) { @@ -795,8 +808,11 @@ static void kvm_eventfd_del(MemoryListener *listener, } static MemoryListener kvm_memory_listener = { + .begin = kvm_begin, + .commit = kvm_commit, .region_add = kvm_region_add, .region_del = kvm_region_del, + .region_nop = kvm_region_nop, .log_start = kvm_log_start, .log_stop = kvm_log_stop, .log_sync = kvm_log_sync, diff --git a/memory.c b/memory.c index 35e1483859..6565e2e696 100644 --- a/memory.c +++ b/memory.c @@ -676,6 +676,7 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In both (logging may have changed) */ if (adding) { + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop); if (frold->dirty_log_mask && !frnew->dirty_log_mask) { MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop); } else if (frnew->dirty_log_mask && !frold->dirty_log_mask) { @@ -722,6 +723,8 @@ static void memory_region_update_topology(MemoryRegion *mr) return; } + MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); + if (address_space_memory.root) { address_space_update_topology(&address_space_memory); } @@ -729,6 +732,8 @@ static void memory_region_update_topology(MemoryRegion *mr) address_space_update_topology(&address_space_io); } + MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); + memory_region_update_pending = false; } diff --git a/memory.h b/memory.h index bc9600b402..b7bccd1968 100644 --- a/memory.h +++ b/memory.h @@ -180,8 +180,11 @@ typedef struct MemoryListener MemoryListener; * Use with memory_listener_register() and memory_listener_unregister(). */ struct MemoryListener { + void (*begin)(MemoryListener *listener); + void (*commit)(MemoryListener *listener); void (*region_add)(MemoryListener *listener, MemoryRegionSection *section); void (*region_del)(MemoryListener *listener, MemoryRegionSection *section); + void (*region_nop)(MemoryListener *listener, MemoryRegionSection *section); void (*log_start)(MemoryListener *listener, MemoryRegionSection *section); void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section); void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section); diff --git a/xen-all.c b/xen-all.c index a58a397ca4..6a11342665 100644 --- a/xen-all.c +++ b/xen-all.c @@ -394,6 +394,14 @@ static void xen_set_memory(struct MemoryListener *listener, } } +static void xen_begin(MemoryListener *listener) +{ +} + +static void xen_commit(MemoryListener *listener) +{ +} + static void xen_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -406,6 +414,11 @@ static void xen_region_del(MemoryListener *listener, xen_set_memory(listener, section, false); } +static void xen_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static void xen_sync_dirty_bitmap(XenIOState *state, target_phys_addr_t start_addr, ram_addr_t size) @@ -500,8 +513,11 @@ static void xen_eventfd_del(MemoryListener *listener, } static MemoryListener xen_memory_listener = { + .begin = xen_begin, + .commit = xen_commit, .region_add = xen_region_add, .region_del = xen_region_del, + .region_nop = xen_region_nop, .log_start = xen_log_start, .log_stop = xen_log_stop, .log_sync = xen_log_sync, -- cgit v1.2.1 From 54688b1ec1f468c7272b837ff57298068aaedf5f Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 9 Feb 2012 17:34:32 +0200 Subject: memory: change memory registration to rebuild the memory map on each change Instead of incrementally building the memory map, rebuild it every time. This allows later simplification, since the code need not consider overlaying a previous mapping. It is also RCU friendly. With large memory guests this can get expensive, since the operation is O(mem size), but this will be optimized later. As a side effect subpage and L2 leaks are fixed here. Signed-off-by: Avi Kivity --- exec.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/exec.c b/exec.c index 6726afd347..b36c3012f2 100644 --- a/exec.c +++ b/exec.c @@ -2520,6 +2520,53 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, } \ } while (0) +static void destroy_page_desc(PhysPageDesc pd) +{ + unsigned io_index = pd.phys_offset & ~TARGET_PAGE_MASK; + MemoryRegion *mr = io_mem_region[io_index]; + + if (mr->subpage) { + subpage_t *subpage = container_of(mr, subpage_t, iomem); + memory_region_destroy(&subpage->iomem); + g_free(subpage); + } +} + +static void destroy_l2_mapping(void **lp, unsigned level) +{ + unsigned i; + void **p; + PhysPageDesc *pd; + + if (!*lp) { + return; + } + + if (level > 0) { + p = *lp; + for (i = 0; i < L2_SIZE; ++i) { + destroy_l2_mapping(&p[i], level - 1); + } + g_free(p); + } else { + pd = *lp; + for (i = 0; i < L2_SIZE; ++i) { + destroy_page_desc(pd[i]); + } + g_free(pd); + } + *lp = NULL; +} + +static void destroy_all_mappings(void) +{ + unsigned i; + + for (i = 0; i < P_L1_SIZE; ++i) { + destroy_l2_mapping(&l1_phys_map[i], P_L1_SHIFT / L2_BITS - 1); + } +} + /* register physical memory. For RAM, 'size' must be a multiple of the target page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an @@ -3490,6 +3537,7 @@ static void io_mem_init(void) static void core_begin(MemoryListener *listener) { + destroy_all_mappings(); } static void core_commit(MemoryListener *listener) @@ -3505,12 +3553,12 @@ static void core_region_add(MemoryListener *listener, static void core_region_del(MemoryListener *listener, MemoryRegionSection *section) { - cpu_register_physical_memory_log(section, false); } static void core_region_nop(MemoryListener *listener, MemoryRegionSection *section) { + cpu_register_physical_memory_log(section, section->readonly); } static void core_log_start(MemoryListener *listener, -- cgit v1.2.1 From 3eef53df6b0240547bd5a80261091266c6ab1e3c Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Fri, 10 Feb 2012 14:57:31 +0200 Subject: memory: remove first level of l1_phys_map L1 and the lower levels in l1_phys_map are equivalent, except that L1 has a different size, and is always allocated. Simplify the code by removing L1. This leaves us with a tree composed solely of L2 tables, but that problem can be renamed away later. Signed-off-by: Avi Kivity --- exec.c | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/exec.c b/exec.c index b36c3012f2..c541ee7107 100644 --- a/exec.c +++ b/exec.c @@ -160,29 +160,21 @@ typedef struct PageDesc { #define L2_BITS 10 #define L2_SIZE (1 << L2_BITS) +#define P_L2_LEVELS \ + (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1) + /* The bits remaining after N lower levels of page tables. */ -#define P_L1_BITS_REM \ - ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS) #define V_L1_BITS_REM \ ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS) -/* Size of the L1 page table. Avoid silly small sizes. */ -#if P_L1_BITS_REM < 4 -#define P_L1_BITS (P_L1_BITS_REM + L2_BITS) -#else -#define P_L1_BITS P_L1_BITS_REM -#endif - #if V_L1_BITS_REM < 4 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS) #else #define V_L1_BITS V_L1_BITS_REM #endif -#define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS) #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS) -#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS) #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS) unsigned long qemu_real_host_page_size; @@ -202,7 +194,7 @@ typedef struct PhysPageDesc { /* This is a multi-level map on the physical address space. The bottom level has pointers to PhysPageDesc. */ -static void *l1_phys_map[P_L1_SIZE]; +static void *phys_map; static void io_mem_init(void); static void memory_map_init(void); @@ -404,11 +396,10 @@ static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc) void **lp; int i; - /* Level 1. Always allocated. */ - lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1)); + lp = &phys_map; - /* Level 2..N-1. */ - for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) { + /* Level 1..N-1. */ + for (i = P_L2_LEVELS - 1; i > 0; i--) { void **p = *lp; if (p == NULL) { if (!alloc) { @@ -2560,11 +2551,7 @@ static void destroy_l2_mapping(void **lp, unsigned level) static void destroy_all_mappings(void) { - unsigned i; - - for (i = 0; i < P_L1_SIZE; ++i) { - destroy_l2_mapping(&l1_phys_map[i], P_L1_SHIFT / L2_BITS - 1); - } + destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1); } /* register physical memory. -- cgit v1.2.1 From 4346ae3e28fc19e01b3070de169ea9fcef5f54aa Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Fri, 10 Feb 2012 17:00:01 +0200 Subject: memory: unify phys_map last level with intermediate levels This lays the groundwork for storing leaf data in intermediate levels, saving space. Signed-off-by: Avi Kivity --- exec.c | 78 ++++++++++++++++++++++++++++++------------------------------------ 1 file changed, 35 insertions(+), 43 deletions(-) diff --git a/exec.c b/exec.c index c541ee7107..536e70f52c 100644 --- a/exec.c +++ b/exec.c @@ -192,9 +192,18 @@ typedef struct PhysPageDesc { ram_addr_t region_offset; } PhysPageDesc; +typedef struct PhysPageEntry PhysPageEntry; + +struct PhysPageEntry { + union { + PhysPageDesc leaf; + PhysPageEntry *node; + } u; +}; + /* This is a multi-level map on the physical address space. The bottom level has pointers to PhysPageDesc. */ -static void *phys_map; +static PhysPageEntry phys_map; static void io_mem_init(void); static void memory_map_init(void); @@ -392,42 +401,31 @@ static inline PageDesc *page_find(tb_page_addr_t index) #if !defined(CONFIG_USER_ONLY) static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc) { - PhysPageDesc *pd; - void **lp; - int i; + PhysPageEntry *lp, *p; + int i, j; lp = &phys_map; - /* Level 1..N-1. */ - for (i = P_L2_LEVELS - 1; i > 0; i--) { - void **p = *lp; - if (p == NULL) { + /* Level 1..N. */ + for (i = P_L2_LEVELS - 1; i >= 0; i--) { + if (lp->u.node == NULL) { if (!alloc) { return NULL; } - *lp = p = g_malloc0(sizeof(void *) * L2_SIZE); - } - lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1)); - } - - pd = *lp; - if (pd == NULL) { - int i; - int first_index = index & ~(L2_SIZE - 1); - - if (!alloc) { - return NULL; - } - - *lp = pd = g_malloc(sizeof(PhysPageDesc) * L2_SIZE); - - for (i = 0; i < L2_SIZE; i++) { - pd[i].phys_offset = io_mem_unassigned.ram_addr; - pd[i].region_offset = (first_index + i) << TARGET_PAGE_BITS; + lp->u.node = p = g_malloc0(sizeof(PhysPageEntry) * L2_SIZE); + if (i == 0) { + int first_index = index & ~(L2_SIZE - 1); + for (j = 0; j < L2_SIZE; j++) { + p[j].u.leaf.phys_offset = io_mem_unassigned.ram_addr; + p[j].u.leaf.region_offset + = (first_index + j) << TARGET_PAGE_BITS; + } + } } + lp = &lp->u.node[(index >> (i * L2_BITS)) & (L2_SIZE - 1)]; } - return pd + (index & (L2_SIZE - 1)); + return &lp->u.leaf; } static inline PhysPageDesc phys_page_find(target_phys_addr_t index) @@ -2523,30 +2521,24 @@ static void destroy_page_desc(PhysPageDesc pd) } } -static void destroy_l2_mapping(void **lp, unsigned level) +static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level) { unsigned i; - void **p; - PhysPageDesc *pd; + PhysPageEntry *p = lp->u.node; - if (!*lp) { + if (!p) { return; } - if (level > 0) { - p = *lp; - for (i = 0; i < L2_SIZE; ++i) { + for (i = 0; i < L2_SIZE; ++i) { + if (level > 0) { destroy_l2_mapping(&p[i], level - 1); + } else { + destroy_page_desc(p[i].u.leaf); } - g_free(p); - } else { - pd = *lp; - for (i = 0; i < L2_SIZE; ++i) { - destroy_page_desc(pd[i]); - } - g_free(pd); } - *lp = NULL; + g_free(p); + lp->u.node = NULL; } static void destroy_all_mappings(void) -- cgit v1.2.1 From 5312bd8b3152f8d4fcf9389ba54e32b09f4b4093 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 12 Feb 2012 18:32:55 +0200 Subject: memory: store MemoryRegionSection pointers in phys_map Instead of storing PhysPageDesc, store pointers to MemoryRegionSections. The various offsets (phys_offset & ~TARGET_PAGE_MASK, PHYS_OFFSET & TARGET_PAGE_MASK, region_offset) can all be synthesized from the information in a MemoryRegionSection. Adjust phys_page_find() to synthesize a PhysPageDesc. The upshot is that phys_map now contains uniform values, so it's easier to generate and compress. The end result is somewhat clumsy but this will be improved as we we propagate MemoryRegionSections throughout the code instead of transforming them to PhysPageDesc. The MemoryRegionSection pointers are stored as uint16_t offsets in an array. This saves space (when we also compress node pointers) and is more cache friendly. Signed-off-by: Avi Kivity --- exec.c | 187 +++++++++++++++++++++++++++++++++++++---------------------------- 1 file changed, 107 insertions(+), 80 deletions(-) diff --git a/exec.c b/exec.c index 536e70f52c..957bc6d5bf 100644 --- a/exec.c +++ b/exec.c @@ -194,9 +194,13 @@ typedef struct PhysPageDesc { typedef struct PhysPageEntry PhysPageEntry; +static MemoryRegionSection *phys_sections; +static unsigned phys_sections_nb, phys_sections_nb_alloc; +static uint16_t phys_section_unassigned; + struct PhysPageEntry { union { - PhysPageDesc leaf; + uint16_t leaf; /* index into phys_sections */ PhysPageEntry *node; } u; }; @@ -399,7 +403,7 @@ static inline PageDesc *page_find(tb_page_addr_t index) } #if !defined(CONFIG_USER_ONLY) -static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc) +static uint16_t *phys_page_find_alloc(target_phys_addr_t index, int alloc) { PhysPageEntry *lp, *p; int i, j; @@ -414,11 +418,8 @@ static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc) } lp->u.node = p = g_malloc0(sizeof(PhysPageEntry) * L2_SIZE); if (i == 0) { - int first_index = index & ~(L2_SIZE - 1); for (j = 0; j < L2_SIZE; j++) { - p[j].u.leaf.phys_offset = io_mem_unassigned.ram_addr; - p[j].u.leaf.region_offset - = (first_index + j) << TARGET_PAGE_BITS; + p[j].u.leaf = phys_section_unassigned; } } } @@ -430,16 +431,31 @@ static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc) static inline PhysPageDesc phys_page_find(target_phys_addr_t index) { - PhysPageDesc *p = phys_page_find_alloc(index, 0); + uint16_t *p = phys_page_find_alloc(index, 0); + uint16_t s_index = phys_section_unassigned; + MemoryRegionSection *section; + PhysPageDesc pd; if (p) { - return *p; - } else { - return (PhysPageDesc) { - .phys_offset = io_mem_unassigned.ram_addr, - .region_offset = index << TARGET_PAGE_BITS, - }; + s_index = *p; + } + section = &phys_sections[s_index]; + index <<= TARGET_PAGE_BITS; + assert(section->offset_within_address_space <= index + && index <= section->offset_within_address_space + section->size-1); + pd.phys_offset = section->mr->ram_addr; + pd.region_offset = (index - section->offset_within_address_space) + + section->offset_within_region; + if (memory_region_is_ram(section->mr)) { + pd.phys_offset += pd.region_offset; + pd.region_offset = 0; + } else if (section->mr->rom_device) { + pd.phys_offset += pd.region_offset; } + if (section->readonly) { + pd.phys_offset |= io_mem_rom.ram_addr; + } + return pd; } static void tlb_protect_code(ram_addr_t ram_addr); @@ -2480,15 +2496,13 @@ static inline void tlb_set_dirty(CPUState *env, typedef struct subpage_t { MemoryRegion iomem; target_phys_addr_t base; - ram_addr_t sub_io_index[TARGET_PAGE_SIZE]; - ram_addr_t region_offset[TARGET_PAGE_SIZE]; + uint16_t sub_section[TARGET_PAGE_SIZE]; } subpage_t; static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, - ram_addr_t memory, ram_addr_t region_offset); -static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, - ram_addr_t orig_memory, - ram_addr_t region_offset); + uint16_t section); +static subpage_t *subpage_init (target_phys_addr_t base, uint16_t *section, + uint16_t orig_section); #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \ need_subpage) \ do { \ @@ -2509,10 +2523,10 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, } \ } while (0) -static void destroy_page_desc(PhysPageDesc pd) +static void destroy_page_desc(uint16_t section_index) { - unsigned io_index = pd.phys_offset & ~TARGET_PAGE_MASK; - MemoryRegion *mr = io_mem_region[io_index]; + MemoryRegionSection *section = &phys_sections[section_index]; + MemoryRegion *mr = section->mr; if (mr->subpage) { subpage_t *subpage = container_of(mr, subpage_t, iomem); @@ -2546,6 +2560,22 @@ static void destroy_all_mappings(void) destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1); } +static uint16_t phys_section_add(MemoryRegionSection *section) +{ + if (phys_sections_nb == phys_sections_nb_alloc) { + phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16); + phys_sections = g_renew(MemoryRegionSection, phys_sections, + phys_sections_nb_alloc); + } + phys_sections[phys_sections_nb] = *section; + return phys_sections_nb++; +} + +static void phys_sections_clear(void) +{ + phys_sections_nb = 0; +} + /* register physical memory. For RAM, 'size' must be a multiple of the target page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an @@ -2559,67 +2589,46 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section, { target_phys_addr_t start_addr = section->offset_within_address_space; ram_addr_t size = section->size; - ram_addr_t phys_offset = section->mr->ram_addr; - ram_addr_t region_offset = section->offset_within_region; target_phys_addr_t addr, end_addr; - PhysPageDesc *p; + uint16_t *p; CPUState *env; ram_addr_t orig_size = size; subpage_t *subpage; - - if (memory_region_is_ram(section->mr)) { - phys_offset += region_offset; - region_offset = 0; - } - - if (readonly) { - phys_offset |= io_mem_rom.ram_addr; - } + uint16_t section_index = phys_section_add(section); assert(size); - if (phys_offset == io_mem_unassigned.ram_addr) { - region_offset = start_addr; - } - region_offset &= TARGET_PAGE_MASK; size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK; end_addr = start_addr + (target_phys_addr_t)size; addr = start_addr; do { p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 0); - if (p && p->phys_offset != io_mem_unassigned.ram_addr) { - ram_addr_t orig_memory = p->phys_offset; + if (p && *p != phys_section_unassigned) { + uint16_t orig_memory= *p; target_phys_addr_t start_addr2, end_addr2; int need_subpage = 0; - MemoryRegion *mr = io_mem_region[orig_memory & ~TARGET_PAGE_MASK]; + MemoryRegion *mr = phys_sections[orig_memory].mr; CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, need_subpage); if (need_subpage) { if (!(mr->subpage)) { subpage = subpage_init((addr & TARGET_PAGE_MASK), - &p->phys_offset, orig_memory, - p->region_offset); + p, orig_memory); } else { subpage = container_of(mr, subpage_t, iomem); } - subpage_register(subpage, start_addr2, end_addr2, phys_offset, - region_offset); - p->region_offset = 0; + subpage_register(subpage, start_addr2, end_addr2, + section_index); } else { - p->phys_offset = phys_offset; - p->region_offset = region_offset; - if (is_ram_rom_romd(phys_offset)) - phys_offset += TARGET_PAGE_SIZE; + *p = section_index; } } else { + MemoryRegion *mr = section->mr; p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1); - p->phys_offset = phys_offset; - p->region_offset = region_offset; - if (is_ram_rom_romd(phys_offset)) { - phys_offset += TARGET_PAGE_SIZE; - } else { + *p = section_index; + if (!(memory_region_is_ram(mr) || mr->rom_device)) { target_phys_addr_t start_addr2, end_addr2; int need_subpage = 0; @@ -2628,16 +2637,12 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section, if (need_subpage) { subpage = subpage_init((addr & TARGET_PAGE_MASK), - &p->phys_offset, - io_mem_unassigned.ram_addr, - addr & TARGET_PAGE_MASK); + p, phys_section_unassigned); subpage_register(subpage, start_addr2, end_addr2, - phys_offset, region_offset); - p->region_offset = 0; + section_index); } } } - region_offset += TARGET_PAGE_SIZE; addr += TARGET_PAGE_SIZE; } while (addr != end_addr); @@ -3333,14 +3338,17 @@ static uint64_t subpage_read(void *opaque, target_phys_addr_t addr, { subpage_t *mmio = opaque; unsigned int idx = SUBPAGE_IDX(addr); + MemoryRegionSection *section; #if defined(DEBUG_SUBPAGE) printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__, mmio, len, addr, idx); #endif - addr += mmio->region_offset[idx]; - idx = mmio->sub_io_index[idx]; - return io_mem_read(idx, addr, len); + section = &phys_sections[mmio->sub_section[idx]]; + addr += mmio->base; + addr -= section->offset_within_address_space; + addr += section->offset_within_region; + return io_mem_read(section->mr->ram_addr, addr, len); } static void subpage_write(void *opaque, target_phys_addr_t addr, @@ -3348,15 +3356,18 @@ static void subpage_write(void *opaque, target_phys_addr_t addr, { subpage_t *mmio = opaque; unsigned int idx = SUBPAGE_IDX(addr); + MemoryRegionSection *section; #if defined(DEBUG_SUBPAGE) printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %"PRIx64"\n", __func__, mmio, len, addr, idx, value); #endif - addr += mmio->region_offset[idx]; - idx = mmio->sub_io_index[idx]; - io_mem_write(idx, addr, value, len); + section = &phys_sections[mmio->sub_section[idx]]; + addr += mmio->base; + addr -= section->offset_within_address_space; + addr += section->offset_within_region; + io_mem_write(section->mr->ram_addr, addr, value, len); } static const MemoryRegionOps subpage_ops = { @@ -3398,7 +3409,7 @@ static const MemoryRegionOps subpage_ram_ops = { }; static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, - ram_addr_t memory, ram_addr_t region_offset) + uint16_t section) { int idx, eidx; @@ -3410,24 +3421,26 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__, mmio, start, end, idx, eidx, memory); #endif - if ((memory & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr) { - memory = io_mem_subpage_ram.ram_addr; + if (memory_region_is_ram(phys_sections[section].mr)) { + MemoryRegionSection new_section = phys_sections[section]; + new_section.mr = &io_mem_subpage_ram; + section = phys_section_add(&new_section); } - memory &= IO_MEM_NB_ENTRIES - 1; for (; idx <= eidx; idx++) { - mmio->sub_io_index[idx] = memory; - mmio->region_offset[idx] = region_offset; + mmio->sub_section[idx] = section; } return 0; } -static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, - ram_addr_t orig_memory, - ram_addr_t region_offset) +static subpage_t *subpage_init (target_phys_addr_t base, uint16_t *section_ind, + uint16_t orig_section) { subpage_t *mmio; - int subpage_memory; + MemoryRegionSection section = { + .offset_within_address_space = base, + .size = TARGET_PAGE_SIZE, + }; mmio = g_malloc0(sizeof(subpage_t)); @@ -3435,13 +3448,13 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, memory_region_init_io(&mmio->iomem, &subpage_ops, mmio, "subpage", TARGET_PAGE_SIZE); mmio->iomem.subpage = true; - subpage_memory = mmio->iomem.ram_addr; + section.mr = &mmio->iomem; #if defined(DEBUG_SUBPAGE) printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__, mmio, base, TARGET_PAGE_SIZE, subpage_memory); #endif - *phys = subpage_memory; - subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset); + *section_ind = phys_section_add(§ion); + subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_section); return mmio; } @@ -3493,6 +3506,18 @@ void cpu_unregister_io_memory(int io_index) io_mem_used[io_index] = 0; } +static uint16_t dummy_section(MemoryRegion *mr) +{ + MemoryRegionSection section = { + .mr = mr, + .offset_within_address_space = 0, + .offset_within_region = 0, + .size = UINT64_MAX, + }; + + return phys_section_add(§ion); +} + static void io_mem_init(void) { int i; @@ -3517,6 +3542,8 @@ static void io_mem_init(void) static void core_begin(MemoryListener *listener) { destroy_all_mappings(); + phys_sections_clear(); + phys_section_unassigned = dummy_section(&io_mem_unassigned); } static void core_commit(MemoryListener *listener) -- cgit v1.2.1 From d6f2ea22a05b429ba83248b80a625b6fe1d927f3 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 12 Feb 2012 20:12:49 +0200 Subject: memory: compress phys_map node pointers to 16 bits Use an expanding vector to store nodes. Allocation is baroque to g_renew() potentially invalidating pointers; this will be addressed later. Signed-off-by: Avi Kivity --- exec.c | 54 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/exec.c b/exec.c index 957bc6d5bf..0756919e66 100644 --- a/exec.c +++ b/exec.c @@ -201,13 +201,19 @@ static uint16_t phys_section_unassigned; struct PhysPageEntry { union { uint16_t leaf; /* index into phys_sections */ - PhysPageEntry *node; + uint16_t node; /* index into phys_map_nodes */ } u; }; +/* Simple allocator for PhysPageEntry nodes */ +static PhysPageEntry (*phys_map_nodes)[L2_SIZE]; +static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc; + +#define PHYS_MAP_NODE_NIL ((uint16_t)~0) + /* This is a multi-level map on the physical address space. The bottom level has pointers to PhysPageDesc. */ -static PhysPageEntry phys_map; +static PhysPageEntry phys_map = { .u.node = PHYS_MAP_NODE_NIL }; static void io_mem_init(void); static void memory_map_init(void); @@ -403,6 +409,32 @@ static inline PageDesc *page_find(tb_page_addr_t index) } #if !defined(CONFIG_USER_ONLY) + +static PhysPageEntry *phys_map_node_alloc(uint16_t *ptr) +{ + unsigned i; + uint16_t ret; + + /* Assign early to avoid the pointer being invalidated by g_renew() */ + *ptr = ret = phys_map_nodes_nb++; + assert(ret != PHYS_MAP_NODE_NIL); + if (ret == phys_map_nodes_nb_alloc) { + typedef PhysPageEntry Node[L2_SIZE]; + phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16); + phys_map_nodes = g_renew(Node, phys_map_nodes, + phys_map_nodes_nb_alloc); + } + for (i = 0; i < L2_SIZE; ++i) { + phys_map_nodes[ret][i].u.node = PHYS_MAP_NODE_NIL; + } + return phys_map_nodes[ret]; +} + +static void phys_map_nodes_reset(void) +{ + phys_map_nodes_nb = 0; +} + static uint16_t *phys_page_find_alloc(target_phys_addr_t index, int alloc) { PhysPageEntry *lp, *p; @@ -412,18 +444,20 @@ static uint16_t *phys_page_find_alloc(target_phys_addr_t index, int alloc) /* Level 1..N. */ for (i = P_L2_LEVELS - 1; i >= 0; i--) { - if (lp->u.node == NULL) { + if (lp->u.node == PHYS_MAP_NODE_NIL) { if (!alloc) { return NULL; } - lp->u.node = p = g_malloc0(sizeof(PhysPageEntry) * L2_SIZE); + p = phys_map_node_alloc(&lp->u.node); if (i == 0) { for (j = 0; j < L2_SIZE; j++) { p[j].u.leaf = phys_section_unassigned; } } + } else { + p = phys_map_nodes[lp->u.node]; } - lp = &lp->u.node[(index >> (i * L2_BITS)) & (L2_SIZE - 1)]; + lp = &p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)]; } return &lp->u.leaf; @@ -2538,12 +2572,13 @@ static void destroy_page_desc(uint16_t section_index) static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level) { unsigned i; - PhysPageEntry *p = lp->u.node; + PhysPageEntry *p; - if (!p) { + if (lp->u.node == PHYS_MAP_NODE_NIL) { return; } + p = phys_map_nodes[lp->u.node]; for (i = 0; i < L2_SIZE; ++i) { if (level > 0) { destroy_l2_mapping(&p[i], level - 1); @@ -2551,13 +2586,13 @@ static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level) destroy_page_desc(p[i].u.leaf); } } - g_free(p); - lp->u.node = NULL; + lp->u.node = PHYS_MAP_NODE_NIL; } static void destroy_all_mappings(void) { destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1); + phys_map_nodes_reset(); } static uint16_t phys_section_add(MemoryRegionSection *section) @@ -3543,6 +3578,7 @@ static void core_begin(MemoryListener *listener) { destroy_all_mappings(); phys_sections_clear(); + phys_map.u.node = PHYS_MAP_NODE_NIL; phys_section_unassigned = dummy_section(&io_mem_unassigned); } -- cgit v1.2.1 From 8636b9295be46307a1de86942accabea50fa59ae Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 12 Feb 2012 21:10:50 +0200 Subject: memory: fix RAM subpages in newly initialized pages If the first subpage installed in a page is RAM, then we install it as a full page, instead of a subpage. Fix by not special casing RAM. The issue dates to commit db7b5426a4b4242, which introduced subpages. Signed-off-by: Avi Kivity --- exec.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/exec.c b/exec.c index 0756919e66..5d0afdbdeb 100644 --- a/exec.c +++ b/exec.c @@ -2660,22 +2660,20 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section, *p = section_index; } } else { - MemoryRegion *mr = section->mr; + target_phys_addr_t start_addr2, end_addr2; + int need_subpage = 0; + p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1); *p = section_index; - if (!(memory_region_is_ram(mr) || mr->rom_device)) { - target_phys_addr_t start_addr2, end_addr2; - int need_subpage = 0; - CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, - end_addr2, need_subpage); + CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, + end_addr2, need_subpage); - if (need_subpage) { - subpage = subpage_init((addr & TARGET_PAGE_MASK), - p, phys_section_unassigned); - subpage_register(subpage, start_addr2, end_addr2, - section_index); - } + if (need_subpage) { + subpage = subpage_init((addr & TARGET_PAGE_MASK), + p, phys_section_unassigned); + subpage_register(subpage, start_addr2, end_addr2, + section_index); } } addr += TARGET_PAGE_SIZE; -- cgit v1.2.1 From 717cb7b259d7258063f8c4a812098a8d3adfa4b3 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 12 Feb 2012 21:21:21 +0200 Subject: memory: unify the two branches of cpu_register_physical_memory_log() Identical except that the second branch knows its not modifying an existing subpage. Signed-off-by: Avi Kivity --- exec.c | 49 +++++++++++++++---------------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/exec.c b/exec.c index 5d0afdbdeb..6232a39224 100644 --- a/exec.c +++ b/exec.c @@ -2625,7 +2625,6 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section, target_phys_addr_t start_addr = section->offset_within_address_space; ram_addr_t size = section->size; target_phys_addr_t addr, end_addr; - uint16_t *p; CPUState *env; ram_addr_t orig_size = size; subpage_t *subpage; @@ -2638,43 +2637,25 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section, addr = start_addr; do { - p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 0); - if (p && *p != phys_section_unassigned) { - uint16_t orig_memory= *p; - target_phys_addr_t start_addr2, end_addr2; - int need_subpage = 0; - MemoryRegion *mr = phys_sections[orig_memory].mr; - - CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, - need_subpage); - if (need_subpage) { - if (!(mr->subpage)) { - subpage = subpage_init((addr & TARGET_PAGE_MASK), - p, orig_memory); - } else { - subpage = container_of(mr, subpage_t, iomem); - } - subpage_register(subpage, start_addr2, end_addr2, - section_index); + uint16_t *p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1); + uint16_t orig_memory = *p; + target_phys_addr_t start_addr2, end_addr2; + int need_subpage = 0; + MemoryRegion *mr = phys_sections[orig_memory].mr; + + CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, + need_subpage); + if (need_subpage) { + if (!(mr->subpage)) { + subpage = subpage_init((addr & TARGET_PAGE_MASK), + p, orig_memory); } else { - *p = section_index; + subpage = container_of(mr, subpage_t, iomem); } + subpage_register(subpage, start_addr2, end_addr2, + section_index); } else { - target_phys_addr_t start_addr2, end_addr2; - int need_subpage = 0; - - p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1); *p = section_index; - - CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, - end_addr2, need_subpage); - - if (need_subpage) { - subpage = subpage_init((addr & TARGET_PAGE_MASK), - p, phys_section_unassigned); - subpage_register(subpage, start_addr2, end_addr2, - section_index); - } } addr += TARGET_PAGE_SIZE; } while (addr != end_addr); -- cgit v1.2.1 From 117712c3e48e64c763d981b0d72603ef703031fe Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 12 Feb 2012 21:23:17 +0200 Subject: memory: move tlb flush to MemoryListener commit callback This way, if we have several changes in a single transaction, we flush just once. Signed-off-by: Avi Kivity --- exec.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/exec.c b/exec.c index 6232a39224..2171eba487 100644 --- a/exec.c +++ b/exec.c @@ -2625,7 +2625,6 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section, target_phys_addr_t start_addr = section->offset_within_address_space; ram_addr_t size = section->size; target_phys_addr_t addr, end_addr; - CPUState *env; ram_addr_t orig_size = size; subpage_t *subpage; uint16_t section_index = phys_section_add(section); @@ -2659,13 +2658,6 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section, } addr += TARGET_PAGE_SIZE; } while (addr != end_addr); - - /* since each CPU stores ram addresses in its TLB cache, we must - reset the modified entries */ - /* XXX: slow ! */ - for(env = first_cpu; env != NULL; env = env->next_cpu) { - tlb_flush(env, 1); - } } void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size) @@ -3563,6 +3555,14 @@ static void core_begin(MemoryListener *listener) static void core_commit(MemoryListener *listener) { + CPUState *env; + + /* since each CPU stores ram addresses in its TLB cache, we must + reset the modified entries */ + /* XXX: slow ! */ + for(env = first_cpu; env != NULL; env = env->next_cpu) { + tlb_flush(env, 1); + } } static void core_region_add(MemoryListener *listener, -- cgit v1.2.1 From 06ef3525e1f271b6a842781a05eace5cf63b95c2 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 13 Feb 2012 16:11:22 +0200 Subject: memory: make phys_page_find() return a MemoryRegionSection We no longer describe memory in terms of individual pages; use sections throughout instead. PhysPageDesc no longer used - remove. Signed-off-by: Avi Kivity --- exec.c | 299 +++++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 160 insertions(+), 139 deletions(-) diff --git a/exec.c b/exec.c index 2171eba487..bf34dc9e43 100644 --- a/exec.c +++ b/exec.c @@ -186,12 +186,6 @@ unsigned long qemu_host_page_mask; static void *l1_map[V_L1_SIZE]; #if !defined(CONFIG_USER_ONLY) -typedef struct PhysPageDesc { - /* offset in host memory of the page + io_index in the low bits */ - ram_addr_t phys_offset; - ram_addr_t region_offset; -} PhysPageDesc; - typedef struct PhysPageEntry PhysPageEntry; static MemoryRegionSection *phys_sections; @@ -212,7 +206,7 @@ static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc; #define PHYS_MAP_NODE_NIL ((uint16_t)~0) /* This is a multi-level map on the physical address space. - The bottom level has pointers to PhysPageDesc. */ + The bottom level has pointers to MemoryRegionSections. */ static PhysPageEntry phys_map = { .u.node = PHYS_MAP_NODE_NIL }; static void io_mem_init(void); @@ -463,33 +457,25 @@ static uint16_t *phys_page_find_alloc(target_phys_addr_t index, int alloc) return &lp->u.leaf; } -static inline PhysPageDesc phys_page_find(target_phys_addr_t index) +static MemoryRegionSection phys_page_find(target_phys_addr_t index) { uint16_t *p = phys_page_find_alloc(index, 0); uint16_t s_index = phys_section_unassigned; - MemoryRegionSection *section; - PhysPageDesc pd; + MemoryRegionSection section; + target_phys_addr_t delta; if (p) { s_index = *p; } - section = &phys_sections[s_index]; + section = phys_sections[s_index]; index <<= TARGET_PAGE_BITS; - assert(section->offset_within_address_space <= index - && index <= section->offset_within_address_space + section->size-1); - pd.phys_offset = section->mr->ram_addr; - pd.region_offset = (index - section->offset_within_address_space) - + section->offset_within_region; - if (memory_region_is_ram(section->mr)) { - pd.phys_offset += pd.region_offset; - pd.region_offset = 0; - } else if (section->mr->rom_device) { - pd.phys_offset += pd.region_offset; - } - if (section->readonly) { - pd.phys_offset |= io_mem_rom.ram_addr; - } - return pd; + assert(section.offset_within_address_space <= index + && index <= section.offset_within_address_space + section.size-1); + delta = index - section.offset_within_address_space; + section.offset_within_address_space += delta; + section.offset_within_region += delta; + section.size -= delta; + return section; } static void tlb_protect_code(ram_addr_t ram_addr); @@ -1449,14 +1435,18 @@ static void breakpoint_invalidate(CPUState *env, target_ulong pc) static void breakpoint_invalidate(CPUState *env, target_ulong pc) { target_phys_addr_t addr; - target_ulong pd; ram_addr_t ram_addr; - PhysPageDesc p; + MemoryRegionSection section; addr = cpu_get_phys_page_debug(env, pc); - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; - ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK); + section = phys_page_find(addr >> TARGET_PAGE_BITS); + if (!(memory_region_is_ram(section.mr) + || (section.mr->rom_device && section.mr->readable))) { + return; + } + ram_addr = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) & TARGET_PAGE_MASK; + ram_addr |= (pc & ~TARGET_PAGE_MASK); tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); } #endif @@ -2134,24 +2124,21 @@ static void tlb_add_large_page(CPUState *env, target_ulong vaddr, env->tlb_flush_mask = mask; } -static bool is_ram_rom(ram_addr_t pd) +static bool is_ram_rom(MemoryRegionSection *s) { - pd &= ~TARGET_PAGE_MASK; - return pd == io_mem_ram.ram_addr || pd == io_mem_rom.ram_addr; + return memory_region_is_ram(s->mr); } -static bool is_romd(ram_addr_t pd) +static bool is_romd(MemoryRegionSection *s) { - MemoryRegion *mr; + MemoryRegion *mr = s->mr; - pd &= ~TARGET_PAGE_MASK; - mr = io_mem_region[pd]; return mr->rom_device && mr->readable; } -static bool is_ram_rom_romd(ram_addr_t pd) +static bool is_ram_rom_romd(MemoryRegionSection *s) { - return is_ram_rom(pd) || is_romd(pd); + return is_ram_rom(s) || is_romd(s); } /* Add a new TLB entry. At most one entry for a given virtual address @@ -2161,8 +2148,7 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, target_phys_addr_t paddr, int prot, int mmu_idx, target_ulong size) { - PhysPageDesc p; - unsigned long pd; + MemoryRegionSection section; unsigned int index; target_ulong address; target_ulong code_address; @@ -2175,8 +2161,7 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, if (size != TARGET_PAGE_SIZE) { tlb_add_large_page(env, vaddr, size); } - p = phys_page_find(paddr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(paddr >> TARGET_PAGE_BITS); #if defined(DEBUG_TLB) printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx " prot=%x idx=%d pd=0x%08lx\n", @@ -2184,15 +2169,21 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, #endif address = vaddr; - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* IO memory case (romd handled later) */ address |= TLB_MMIO; } - addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK); - if (is_ram_rom(pd)) { + if (is_ram_rom_romd(§ion)) { + addend = (unsigned long)(memory_region_get_ram_ptr(section.mr) + + section.offset_within_region); + } else { + addend = 0; + } + if (is_ram_rom(§ion)) { /* Normal RAM. */ - iotlb = pd & TARGET_PAGE_MASK; - if ((pd & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr) + iotlb = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) & TARGET_PAGE_MASK; + if (!section.readonly) iotlb |= io_mem_notdirty.ram_addr; else iotlb |= io_mem_rom.ram_addr; @@ -2203,8 +2194,8 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, and avoid full address decoding in every device. We can't use the high bits of pd for this because IO_MEM_ROMD uses these as a ram address. */ - iotlb = (pd & ~TARGET_PAGE_MASK); - iotlb += p.region_offset; + iotlb = memory_region_get_ram_addr(section.mr) & ~TARGET_PAGE_MASK; + iotlb += section.offset_within_region; } code_address = address; @@ -2237,11 +2228,14 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, te->addr_code = -1; } if (prot & PAGE_WRITE) { - if ((pd & ~TARGET_PAGE_MASK) == io_mem_rom.ram_addr || is_romd(pd)) { + if ((memory_region_is_ram(section.mr) && section.readonly) + || is_romd(§ion)) { /* Write access calls the I/O callback. */ te->addr_write = address | TLB_MMIO; - } else if ((pd & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr && - !cpu_physical_memory_is_dirty(pd)) { + } else if (memory_region_is_ram(section.mr) + && !cpu_physical_memory_is_dirty( + section.mr->ram_addr + + section.offset_within_region)) { te->addr_write = address | TLB_NOTDIRTY; } else { te->addr_write = address; @@ -3788,22 +3782,22 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, uint8_t *ptr; uint32_t val; target_phys_addr_t page; - ram_addr_t pd; - PhysPageDesc p; + MemoryRegionSection section; while (len > 0) { page = addr & TARGET_PAGE_MASK; l = (page + TARGET_PAGE_SIZE) - addr; if (l > len) l = len; - p = phys_page_find(page >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(page >> TARGET_PAGE_BITS); if (is_write) { - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { + if (!memory_region_is_ram(section.mr)) { target_phys_addr_t addr1; - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr1 = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr1 = (addr & ~TARGET_PAGE_MASK) + + section.offset_within_region; /* XXX: could force cpu_single_env to NULL to avoid potential bugs */ if (l >= 4 && ((addr1 & 3) == 0)) { @@ -3822,9 +3816,11 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, io_mem_write(io_index, addr1, val, 1); l = 1; } - } else { + } else if (!section.readonly) { ram_addr_t addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) + | (addr & ~TARGET_PAGE_MASK); /* RAM case */ ptr = qemu_get_ram_ptr(addr1); memcpy(ptr, buf, l); @@ -3838,11 +3834,13 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, qemu_put_ram_ptr(ptr); } } else { - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { target_phys_addr_t addr1; /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr1 = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr1 = (addr & ~TARGET_PAGE_MASK) + + section.offset_within_region; if (l >= 4 && ((addr1 & 3) == 0)) { /* 32 bit read access */ val = io_mem_read(io_index, addr1, 4); @@ -3861,7 +3859,8 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, } } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr(section.mr->ram_addr + + section.offset_within_region); memcpy(buf, ptr + (addr & ~TARGET_PAGE_MASK), l); qemu_put_ram_ptr(ptr); } @@ -3879,22 +3878,22 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr, int l; uint8_t *ptr; target_phys_addr_t page; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; while (len > 0) { page = addr & TARGET_PAGE_MASK; l = (page + TARGET_PAGE_SIZE) - addr; if (l > len) l = len; - p = phys_page_find(page >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(page >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* do nothing */ } else { unsigned long addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); /* ROM/RAM case */ ptr = qemu_get_ram_ptr(addr1); memcpy(ptr, buf, l); @@ -3967,8 +3966,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, target_phys_addr_t todo = 0; int l; target_phys_addr_t page; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; ram_addr_t raddr = RAM_ADDR_MAX; ram_addr_t rlen; void *ret; @@ -3978,10 +3976,9 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, l = (page + TARGET_PAGE_SIZE) - addr; if (l > len) l = len; - p = phys_page_find(page >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(page >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { + if (!(memory_region_is_ram(section.mr) && !section.readonly)) { if (todo || bounce.buffer) { break; } @@ -3996,7 +3993,9 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, return bounce.buffer; } if (!todo) { - raddr = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + raddr = memory_region_get_ram_addr(section.mr) + + section.offset_within_region + + (addr & ~TARGET_PAGE_MASK); } len -= l; @@ -4055,16 +4054,15 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr, int io_index; uint8_t *ptr; uint32_t val; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; val = io_mem_read(io_index, addr, 4); #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { @@ -4077,7 +4075,9 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr, #endif } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + (addr & ~TARGET_PAGE_MASK); switch (endian) { case DEVICE_LITTLE_ENDIAN: @@ -4116,16 +4116,15 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr, int io_index; uint8_t *ptr; uint64_t val; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; /* XXX This is broken when device endian != cpu endian. Fix and add "endian" variable check */ @@ -4138,8 +4137,10 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr, #endif } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + - (addr & ~TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); switch (endian) { case DEVICE_LITTLE_ENDIAN: val = ldq_le_p(ptr); @@ -4185,16 +4186,15 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr, int io_index; uint8_t *ptr; uint64_t val; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; val = io_mem_read(io_index, addr, 2); #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { @@ -4207,8 +4207,10 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr, #endif } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + - (addr & ~TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); switch (endian) { case DEVICE_LITTLE_ENDIAN: val = lduw_le_p(ptr); @@ -4246,18 +4248,23 @@ void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val) { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; io_mem_write(io_index, addr, val, 4); } else { - unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + unsigned long addr1 = (memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region + + (addr & ~TARGET_PAGE_MASK); ptr = qemu_get_ram_ptr(addr1); stl_p(ptr, val); @@ -4277,15 +4284,18 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val) { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; #ifdef TARGET_WORDS_BIGENDIAN io_mem_write(io_index, addr, val >> 32, 4); io_mem_write(io_index, addr + 4, (uint32_t)val, 4); @@ -4294,8 +4304,10 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val) io_mem_write(io_index, addr + 4, val >> 32, 4); #endif } else { - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + - (addr & ~TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); stq_p(ptr, val); } } @@ -4306,15 +4318,18 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val, { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap32(val); @@ -4327,7 +4342,9 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val, io_mem_write(io_index, addr, val, 4); } else { unsigned long addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) & TARGET_PAGE_MASK) + + section.offset_within_region + + (addr & ~TARGET_PAGE_MASK); /* RAM case */ ptr = qemu_get_ram_ptr(addr1); switch (endian) { @@ -4379,15 +4396,18 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val, { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap16(val); @@ -4400,7 +4420,8 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val, io_mem_write(io_index, addr, val, 2); } else { unsigned long addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) & TARGET_PAGE_MASK) + + section.offset_within_region + (addr & ~TARGET_PAGE_MASK); /* RAM case */ ptr = qemu_get_ram_ptr(addr1); switch (endian) { @@ -4617,7 +4638,7 @@ tb_page_addr_t get_page_addr_code(CPUState *env1, target_ulong addr) } pd = env1->tlb_table[mmu_idx][page_index].addr_code & ~TARGET_PAGE_MASK; if (pd != io_mem_ram.ram_addr && pd != io_mem_rom.ram_addr - && !is_romd(pd)) { + && !io_mem_region[pd]->rom_device) { #if defined(TARGET_ALPHA) || defined(TARGET_MIPS) || defined(TARGET_SPARC) cpu_unassigned_access(env1, addr, 0, 1, 0, 4); #else -- cgit v1.2.1 From 31ab2b4a46eb9761feae9457387eb5ee523f5afd Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 13 Feb 2012 16:44:19 +0200 Subject: memory: give phys_page_find() its own tree search loop We'll change phys_page_find_alloc() soon, but phys_page_find() doesn't need to bear the consequences. Signed-off-by: Avi Kivity --- exec.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/exec.c b/exec.c index bf34dc9e43..24423d5cb6 100644 --- a/exec.c +++ b/exec.c @@ -459,14 +459,23 @@ static uint16_t *phys_page_find_alloc(target_phys_addr_t index, int alloc) static MemoryRegionSection phys_page_find(target_phys_addr_t index) { - uint16_t *p = phys_page_find_alloc(index, 0); - uint16_t s_index = phys_section_unassigned; + PhysPageEntry lp = phys_map; + PhysPageEntry *p; + int i; MemoryRegionSection section; target_phys_addr_t delta; + uint16_t s_index = phys_section_unassigned; - if (p) { - s_index = *p; + for (i = P_L2_LEVELS - 1; i >= 0; i--) { + if (lp.u.node == PHYS_MAP_NODE_NIL) { + goto not_found; + } + p = phys_map_nodes[lp.u.node]; + lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)]; } + + s_index = lp.u.leaf; +not_found: section = phys_sections[s_index]; index <<= TARGET_PAGE_BITS; assert(section.offset_within_address_space <= index -- cgit v1.2.1 From 0f0cb164cc6d75fa3be079b8fc41c75c5fabe402 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 13 Feb 2012 17:14:32 +0200 Subject: memory: simplify multipage/subpage registration Instead of considering subpage on a per-page basis, split each section into a subpage head, multipage body, and subpage tail, and register each separately. This simplifies the registration functions. Signed-off-by: Avi Kivity --- exec.c | 120 +++++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 65 insertions(+), 55 deletions(-) diff --git a/exec.c b/exec.c index 24423d5cb6..e3828583b6 100644 --- a/exec.c +++ b/exec.c @@ -2538,28 +2538,7 @@ typedef struct subpage_t { static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, uint16_t section); -static subpage_t *subpage_init (target_phys_addr_t base, uint16_t *section, - uint16_t orig_section); -#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \ - need_subpage) \ - do { \ - if (addr > start_addr) \ - start_addr2 = 0; \ - else { \ - start_addr2 = start_addr & ~TARGET_PAGE_MASK; \ - if (start_addr2 > 0) \ - need_subpage = 1; \ - } \ - \ - if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \ - end_addr2 = TARGET_PAGE_SIZE - 1; \ - else { \ - end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \ - if (end_addr2 < TARGET_PAGE_SIZE - 1) \ - need_subpage = 1; \ - } \ - } while (0) - +static subpage_t *subpage_init(target_phys_addr_t base); static void destroy_page_desc(uint16_t section_index) { MemoryRegionSection *section = &phys_sections[section_index]; @@ -2622,47 +2601,85 @@ static void phys_sections_clear(void) start_addr and region_offset are rounded down to a page boundary before calculating this offset. This should not be a problem unless the low bits of start_addr and region_offset differ. */ -void cpu_register_physical_memory_log(MemoryRegionSection *section, - bool readonly) +static void register_subpage(MemoryRegionSection *section) +{ + subpage_t *subpage; + target_phys_addr_t base = section->offset_within_address_space + & TARGET_PAGE_MASK; + MemoryRegionSection existing = phys_page_find(base >> TARGET_PAGE_BITS); + MemoryRegionSection subsection = { + .offset_within_address_space = base, + .size = TARGET_PAGE_SIZE, + }; + uint16_t *ptr; + target_phys_addr_t start, end; + + assert(existing.mr->subpage || existing.mr == &io_mem_unassigned); + + if (!(existing.mr->subpage)) { + subpage = subpage_init(base); + subsection.mr = &subpage->iomem; + ptr = phys_page_find_alloc(base >> TARGET_PAGE_BITS, 1); + *ptr = phys_section_add(&subsection); + } else { + subpage = container_of(existing.mr, subpage_t, iomem); + } + start = section->offset_within_address_space & ~TARGET_PAGE_MASK; + end = start + section->size; + subpage_register(subpage, start, end, phys_section_add(section)); +} + + +static void register_multipage(MemoryRegionSection *section) { target_phys_addr_t start_addr = section->offset_within_address_space; ram_addr_t size = section->size; target_phys_addr_t addr, end_addr; - ram_addr_t orig_size = size; - subpage_t *subpage; uint16_t section_index = phys_section_add(section); assert(size); - size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK; end_addr = start_addr + (target_phys_addr_t)size; addr = start_addr; do { uint16_t *p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1); - uint16_t orig_memory = *p; - target_phys_addr_t start_addr2, end_addr2; - int need_subpage = 0; - MemoryRegion *mr = phys_sections[orig_memory].mr; - - CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, - need_subpage); - if (need_subpage) { - if (!(mr->subpage)) { - subpage = subpage_init((addr & TARGET_PAGE_MASK), - p, orig_memory); - } else { - subpage = container_of(mr, subpage_t, iomem); - } - subpage_register(subpage, start_addr2, end_addr2, - section_index); - } else { - *p = section_index; - } + assert(*p == phys_section_unassigned); + *p = section_index; addr += TARGET_PAGE_SIZE; } while (addr != end_addr); } +void cpu_register_physical_memory_log(MemoryRegionSection *section, + bool readonly) +{ + MemoryRegionSection now = *section, remain = *section; + + if ((now.offset_within_address_space & ~TARGET_PAGE_MASK) + || (now.size < TARGET_PAGE_SIZE)) { + now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space) + - now.offset_within_address_space, + now.size); + register_subpage(&now); + remain.size -= now.size; + remain.offset_within_address_space += now.size; + remain.offset_within_region += now.size; + } + now = remain; + now.size &= TARGET_PAGE_MASK; + if (now.size) { + register_multipage(&now); + remain.size -= now.size; + remain.offset_within_address_space += now.size; + remain.offset_within_region += now.size; + } + now = remain; + if (now.size) { + register_subpage(&now); + } +} + + void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size) { if (kvm_enabled()) @@ -3442,14 +3459,9 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, return 0; } -static subpage_t *subpage_init (target_phys_addr_t base, uint16_t *section_ind, - uint16_t orig_section) +static subpage_t *subpage_init(target_phys_addr_t base) { subpage_t *mmio; - MemoryRegionSection section = { - .offset_within_address_space = base, - .size = TARGET_PAGE_SIZE, - }; mmio = g_malloc0(sizeof(subpage_t)); @@ -3457,13 +3469,11 @@ static subpage_t *subpage_init (target_phys_addr_t base, uint16_t *section_ind, memory_region_init_io(&mmio->iomem, &subpage_ops, mmio, "subpage", TARGET_PAGE_SIZE); mmio->iomem.subpage = true; - section.mr = &mmio->iomem; #if defined(DEBUG_SUBPAGE) printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__, mmio, base, TARGET_PAGE_SIZE, subpage_memory); #endif - *section_ind = phys_section_add(§ion); - subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_section); + subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned); return mmio; } -- cgit v1.2.1 From a39184328611abfeb384f99d13d48b1c46d53bae Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 13 Feb 2012 17:19:30 +0200 Subject: memory: replace phys_page_find_alloc() with phys_page_set() By giving the function the value we want to set, we make it more flexible for the next patch. Signed-off-by: Avi Kivity --- exec.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/exec.c b/exec.c index e3828583b6..26e70c3097 100644 --- a/exec.c +++ b/exec.c @@ -429,7 +429,7 @@ static void phys_map_nodes_reset(void) phys_map_nodes_nb = 0; } -static uint16_t *phys_page_find_alloc(target_phys_addr_t index, int alloc) +static void phys_page_set(target_phys_addr_t index, uint16_t leaf) { PhysPageEntry *lp, *p; int i, j; @@ -439,9 +439,6 @@ static uint16_t *phys_page_find_alloc(target_phys_addr_t index, int alloc) /* Level 1..N. */ for (i = P_L2_LEVELS - 1; i >= 0; i--) { if (lp->u.node == PHYS_MAP_NODE_NIL) { - if (!alloc) { - return NULL; - } p = phys_map_node_alloc(&lp->u.node); if (i == 0) { for (j = 0; j < L2_SIZE; j++) { @@ -454,7 +451,7 @@ static uint16_t *phys_page_find_alloc(target_phys_addr_t index, int alloc) lp = &p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)]; } - return &lp->u.leaf; + lp->u.leaf = leaf; } static MemoryRegionSection phys_page_find(target_phys_addr_t index) @@ -2611,7 +2608,6 @@ static void register_subpage(MemoryRegionSection *section) .offset_within_address_space = base, .size = TARGET_PAGE_SIZE, }; - uint16_t *ptr; target_phys_addr_t start, end; assert(existing.mr->subpage || existing.mr == &io_mem_unassigned); @@ -2619,8 +2615,7 @@ static void register_subpage(MemoryRegionSection *section) if (!(existing.mr->subpage)) { subpage = subpage_init(base); subsection.mr = &subpage->iomem; - ptr = phys_page_find_alloc(base >> TARGET_PAGE_BITS, 1); - *ptr = phys_section_add(&subsection); + phys_page_set(base >> TARGET_PAGE_BITS, phys_section_add(&subsection)); } else { subpage = container_of(existing.mr, subpage_t, iomem); } @@ -2643,9 +2638,7 @@ static void register_multipage(MemoryRegionSection *section) addr = start_addr; do { - uint16_t *p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1); - assert(*p == phys_section_unassigned); - *p = section_index; + phys_page_set(addr >> TARGET_PAGE_BITS, section_index); addr += TARGET_PAGE_SIZE; } while (addr != end_addr); } -- cgit v1.2.1 From f7bf546118837a64d4b04dd650e3f64eaa25e623 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 13 Feb 2012 20:12:05 +0200 Subject: memory: switch phys_page_set() to a recursive implementation Setting multiple pages at once requires backtracking to previous nodes; easiest to achieve via recursion. Signed-off-by: Avi Kivity --- exec.c | 67 ++++++++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/exec.c b/exec.c index 26e70c3097..f4cd867d51 100644 --- a/exec.c +++ b/exec.c @@ -404,24 +404,30 @@ static inline PageDesc *page_find(tb_page_addr_t index) #if !defined(CONFIG_USER_ONLY) -static PhysPageEntry *phys_map_node_alloc(uint16_t *ptr) +static void phys_map_node_reserve(unsigned nodes) { - unsigned i; - uint16_t ret; - - /* Assign early to avoid the pointer being invalidated by g_renew() */ - *ptr = ret = phys_map_nodes_nb++; - assert(ret != PHYS_MAP_NODE_NIL); - if (ret == phys_map_nodes_nb_alloc) { + if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) { typedef PhysPageEntry Node[L2_SIZE]; phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16); + phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc, + phys_map_nodes_nb + nodes); phys_map_nodes = g_renew(Node, phys_map_nodes, phys_map_nodes_nb_alloc); } +} + +static uint16_t phys_map_node_alloc(void) +{ + unsigned i; + uint16_t ret; + + ret = phys_map_nodes_nb++; + assert(ret != PHYS_MAP_NODE_NIL); + assert(ret != phys_map_nodes_nb_alloc); for (i = 0; i < L2_SIZE; ++i) { phys_map_nodes[ret][i].u.node = PHYS_MAP_NODE_NIL; } - return phys_map_nodes[ret]; + return ret; } static void phys_map_nodes_reset(void) @@ -429,29 +435,38 @@ static void phys_map_nodes_reset(void) phys_map_nodes_nb = 0; } -static void phys_page_set(target_phys_addr_t index, uint16_t leaf) -{ - PhysPageEntry *lp, *p; - int i, j; - lp = &phys_map; +static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t index, + uint16_t leaf, int level) +{ + PhysPageEntry *p; + int i; - /* Level 1..N. */ - for (i = P_L2_LEVELS - 1; i >= 0; i--) { - if (lp->u.node == PHYS_MAP_NODE_NIL) { - p = phys_map_node_alloc(&lp->u.node); - if (i == 0) { - for (j = 0; j < L2_SIZE; j++) { - p[j].u.leaf = phys_section_unassigned; - } + if (lp->u.node == PHYS_MAP_NODE_NIL) { + lp->u.node = phys_map_node_alloc(); + p = phys_map_nodes[lp->u.node]; + if (level == 0) { + for (i = 0; i < L2_SIZE; i++) { + p[i].u.leaf = phys_section_unassigned; } - } else { - p = phys_map_nodes[lp->u.node]; } - lp = &p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)]; + } else { + p = phys_map_nodes[lp->u.node]; } + lp = &p[(index >> (level * L2_BITS)) & (L2_SIZE - 1)]; + + if (level == 0) { + lp->u.leaf = leaf; + } else { + phys_page_set_level(lp, index, leaf, level - 1); + } +} + +static void phys_page_set(target_phys_addr_t index, uint16_t leaf) +{ + phys_map_node_reserve(P_L2_LEVELS); - lp->u.leaf = leaf; + phys_page_set_level(&phys_map, index, leaf, P_L2_LEVELS - 1); } static MemoryRegionSection phys_page_find(target_phys_addr_t index) -- cgit v1.2.1 From 2999097bf178536188b0f4caf937c4d347e96627 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 13 Feb 2012 20:21:20 +0200 Subject: memory: change phys_page_set() to set multiple pages Signed-off-by: Avi Kivity --- exec.c | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/exec.c b/exec.c index f4cd867d51..98c0056530 100644 --- a/exec.c +++ b/exec.c @@ -436,8 +436,9 @@ static void phys_map_nodes_reset(void) } -static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t index, - uint16_t leaf, int level) +static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index, + target_phys_addr_t *nb, uint16_t leaf, + int level) { PhysPageEntry *p; int i; @@ -453,20 +454,27 @@ static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t index, } else { p = phys_map_nodes[lp->u.node]; } - lp = &p[(index >> (level * L2_BITS)) & (L2_SIZE - 1)]; + lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)]; - if (level == 0) { - lp->u.leaf = leaf; - } else { - phys_page_set_level(lp, index, leaf, level - 1); + while (*nb && lp < &p[L2_SIZE]) { + if (level == 0) { + lp->u.leaf = leaf; + ++*index; + --*nb; + } else { + phys_page_set_level(lp, index, nb, leaf, level - 1); + } + ++lp; } } -static void phys_page_set(target_phys_addr_t index, uint16_t leaf) +static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb, + uint16_t leaf) { - phys_map_node_reserve(P_L2_LEVELS); + /* Wildly overreserve - it doesn't matter much. */ + phys_map_node_reserve((nb + L2_SIZE - 1) / L2_SIZE * P_L2_LEVELS); - phys_page_set_level(&phys_map, index, leaf, P_L2_LEVELS - 1); + phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1); } static MemoryRegionSection phys_page_find(target_phys_addr_t index) @@ -2630,7 +2638,8 @@ static void register_subpage(MemoryRegionSection *section) if (!(existing.mr->subpage)) { subpage = subpage_init(base); subsection.mr = &subpage->iomem; - phys_page_set(base >> TARGET_PAGE_BITS, phys_section_add(&subsection)); + phys_page_set(base >> TARGET_PAGE_BITS, 1, + phys_section_add(&subsection)); } else { subpage = container_of(existing.mr, subpage_t, iomem); } @@ -2644,18 +2653,14 @@ static void register_multipage(MemoryRegionSection *section) { target_phys_addr_t start_addr = section->offset_within_address_space; ram_addr_t size = section->size; - target_phys_addr_t addr, end_addr; + target_phys_addr_t addr; uint16_t section_index = phys_section_add(section); assert(size); - end_addr = start_addr + (target_phys_addr_t)size; - addr = start_addr; - do { - phys_page_set(addr >> TARGET_PAGE_BITS, section_index); - addr += TARGET_PAGE_SIZE; - } while (addr != end_addr); + phys_page_set(addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS, + section_index); } void cpu_register_physical_memory_log(MemoryRegionSection *section, -- cgit v1.2.1 From c19e8800d4a9c019299cc9232b4e628469a81d2d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 13 Feb 2012 20:25:31 +0200 Subject: memory: unify PhysPageEntry::node and ::leaf They have the same type, unify them. Signed-off-by: Avi Kivity --- exec.c | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/exec.c b/exec.c index 98c0056530..a2015f7ef3 100644 --- a/exec.c +++ b/exec.c @@ -193,10 +193,8 @@ static unsigned phys_sections_nb, phys_sections_nb_alloc; static uint16_t phys_section_unassigned; struct PhysPageEntry { - union { - uint16_t leaf; /* index into phys_sections */ - uint16_t node; /* index into phys_map_nodes */ - } u; + /* index into phys_sections (last level) or phys_map_nodes (others) */ + uint16_t ptr; }; /* Simple allocator for PhysPageEntry nodes */ @@ -207,7 +205,7 @@ static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc; /* This is a multi-level map on the physical address space. The bottom level has pointers to MemoryRegionSections. */ -static PhysPageEntry phys_map = { .u.node = PHYS_MAP_NODE_NIL }; +static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL }; static void io_mem_init(void); static void memory_map_init(void); @@ -425,7 +423,7 @@ static uint16_t phys_map_node_alloc(void) assert(ret != PHYS_MAP_NODE_NIL); assert(ret != phys_map_nodes_nb_alloc); for (i = 0; i < L2_SIZE; ++i) { - phys_map_nodes[ret][i].u.node = PHYS_MAP_NODE_NIL; + phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL; } return ret; } @@ -443,22 +441,22 @@ static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index, PhysPageEntry *p; int i; - if (lp->u.node == PHYS_MAP_NODE_NIL) { - lp->u.node = phys_map_node_alloc(); - p = phys_map_nodes[lp->u.node]; + if (lp->ptr == PHYS_MAP_NODE_NIL) { + lp->ptr = phys_map_node_alloc(); + p = phys_map_nodes[lp->ptr]; if (level == 0) { for (i = 0; i < L2_SIZE; i++) { - p[i].u.leaf = phys_section_unassigned; + p[i].ptr = phys_section_unassigned; } } } else { - p = phys_map_nodes[lp->u.node]; + p = phys_map_nodes[lp->ptr]; } lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)]; while (*nb && lp < &p[L2_SIZE]) { if (level == 0) { - lp->u.leaf = leaf; + lp->ptr = leaf; ++*index; --*nb; } else { @@ -487,14 +485,14 @@ static MemoryRegionSection phys_page_find(target_phys_addr_t index) uint16_t s_index = phys_section_unassigned; for (i = P_L2_LEVELS - 1; i >= 0; i--) { - if (lp.u.node == PHYS_MAP_NODE_NIL) { + if (lp.ptr == PHYS_MAP_NODE_NIL) { goto not_found; } - p = phys_map_nodes[lp.u.node]; + p = phys_map_nodes[lp.ptr]; lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)]; } - s_index = lp.u.leaf; + s_index = lp.ptr; not_found: section = phys_sections[s_index]; index <<= TARGET_PAGE_BITS; @@ -2576,19 +2574,19 @@ static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level) unsigned i; PhysPageEntry *p; - if (lp->u.node == PHYS_MAP_NODE_NIL) { + if (lp->ptr == PHYS_MAP_NODE_NIL) { return; } - p = phys_map_nodes[lp->u.node]; + p = phys_map_nodes[lp->ptr]; for (i = 0; i < L2_SIZE; ++i) { if (level > 0) { destroy_l2_mapping(&p[i], level - 1); } else { - destroy_page_desc(p[i].u.leaf); + destroy_page_desc(p[i].ptr); } } - lp->u.node = PHYS_MAP_NODE_NIL; + lp->ptr = PHYS_MAP_NODE_NIL; } static void destroy_all_mappings(void) @@ -3575,7 +3573,7 @@ static void core_begin(MemoryListener *listener) { destroy_all_mappings(); phys_sections_clear(); - phys_map.u.node = PHYS_MAP_NODE_NIL; + phys_map.ptr = PHYS_MAP_NODE_NIL; phys_section_unassigned = dummy_section(&io_mem_unassigned); } -- cgit v1.2.1 From 07f07b31e57efa6e4f563802136ffa1694a2692b Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 13 Feb 2012 20:45:32 +0200 Subject: memory: allow phys_map tree paths to terminate early When storing large contiguous ranges in phys_map, all values tend to be the same pointers to a single MemoryRegionSection. Collapse them by marking nodes with level > 0 as leaves. This reduces tree memory usage dramatically. Signed-off-by: Avi Kivity --- exec.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/exec.c b/exec.c index a2015f7ef3..0c93b2691f 100644 --- a/exec.c +++ b/exec.c @@ -193,19 +193,20 @@ static unsigned phys_sections_nb, phys_sections_nb_alloc; static uint16_t phys_section_unassigned; struct PhysPageEntry { - /* index into phys_sections (last level) or phys_map_nodes (others) */ - uint16_t ptr; + uint16_t is_leaf : 1; + /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */ + uint16_t ptr : 15; }; /* Simple allocator for PhysPageEntry nodes */ static PhysPageEntry (*phys_map_nodes)[L2_SIZE]; static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc; -#define PHYS_MAP_NODE_NIL ((uint16_t)~0) +#define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1) /* This is a multi-level map on the physical address space. The bottom level has pointers to MemoryRegionSections. */ -static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL }; +static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 }; static void io_mem_init(void); static void memory_map_init(void); @@ -423,6 +424,7 @@ static uint16_t phys_map_node_alloc(void) assert(ret != PHYS_MAP_NODE_NIL); assert(ret != phys_map_nodes_nb_alloc); for (i = 0; i < L2_SIZE; ++i) { + phys_map_nodes[ret][i].is_leaf = 0; phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL; } return ret; @@ -440,12 +442,14 @@ static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index, { PhysPageEntry *p; int i; + target_phys_addr_t step = (target_phys_addr_t)1 << (level * L2_BITS); - if (lp->ptr == PHYS_MAP_NODE_NIL) { + if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) { lp->ptr = phys_map_node_alloc(); p = phys_map_nodes[lp->ptr]; if (level == 0) { for (i = 0; i < L2_SIZE; i++) { + p[i].is_leaf = 1; p[i].ptr = phys_section_unassigned; } } @@ -455,10 +459,11 @@ static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index, lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)]; while (*nb && lp < &p[L2_SIZE]) { - if (level == 0) { + if ((*index & (step - 1)) == 0 && *nb >= step) { + lp->is_leaf = true; lp->ptr = leaf; - ++*index; - --*nb; + *index += step; + *nb -= step; } else { phys_page_set_level(lp, index, nb, leaf, level - 1); } @@ -470,7 +475,7 @@ static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb, uint16_t leaf) { /* Wildly overreserve - it doesn't matter much. */ - phys_map_node_reserve((nb + L2_SIZE - 1) / L2_SIZE * P_L2_LEVELS); + phys_map_node_reserve(3 * P_L2_LEVELS); phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1); } @@ -484,7 +489,7 @@ static MemoryRegionSection phys_page_find(target_phys_addr_t index) target_phys_addr_t delta; uint16_t s_index = phys_section_unassigned; - for (i = P_L2_LEVELS - 1; i >= 0; i--) { + for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) { if (lp.ptr == PHYS_MAP_NODE_NIL) { goto not_found; } @@ -2580,12 +2585,13 @@ static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level) p = phys_map_nodes[lp->ptr]; for (i = 0; i < L2_SIZE; ++i) { - if (level > 0) { + if (!p[i].is_leaf) { destroy_l2_mapping(&p[i], level - 1); } else { destroy_page_desc(p[i].ptr); } } + lp->is_leaf = 0; lp->ptr = PHYS_MAP_NODE_NIL; } -- cgit v1.2.1