From 23bfe28fffd6fff12a39c1ff7274b0dfdecbfa38 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Thu, 3 Mar 2011 21:37:55 +0100 Subject: vnc: Fix stack corruption and other bitmap related bugs Commit bc2429b9174ac2d3c56b7fd35884b0d89ec7fb02 introduced a severe bug (stack corruption). bitmap_clear was called with a wrong argument which caused out-of-bound writes to the local variable width_mask. This bug was detected with QEMU running on windows. It also occurs with wine: *** stack smashing detected ***: terminated wine: Unhandled illegal instruction at address 0x6115c7 (thread 0009), starting debugger... The bug is not windows specific! Instead of fixing the wrong parameter value, bitmap_clear(), bitmap_set and width_mask were removed, and bitmap_intersect() was replaced by !bitmap_empty(). The new operation is much shorter and equivalent to the old operations. The declarations of the dirty bitmaps in vnc.h were also wrong for 64 bit hosts because of a rounding effect: for these hosts, VNC_MAX_WIDTH is no longer a multiple of (16 * BITS_PER_LONG), so the rounded value of VNC_DIRTY_WORDS was too small. Fix both declarations by using the macro which is designed for this purpose. Cc: Corentin Chary Cc: Wen Congyang Cc: Gerhard Wiesinger Cc: Anthony Liguori Signed-off-by: Stefan Weil Signed-off-by: Anthony Liguori --- ui/vnc.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'ui/vnc.c') diff --git a/ui/vnc.c b/ui/vnc.c index 610f88485e..34dc0cdc34 100644 --- a/ui/vnc.c +++ b/ui/vnc.c @@ -2383,7 +2383,6 @@ static int vnc_refresh_server_surface(VncDisplay *vd) uint8_t *guest_row; uint8_t *server_row; int cmp_bytes; - unsigned long width_mask[VNC_DIRTY_WORDS]; VncState *vs; int has_dirty = 0; @@ -2399,14 +2398,11 @@ static int vnc_refresh_server_surface(VncDisplay *vd) * Check and copy modified bits from guest to server surface. * Update server dirty map. */ - bitmap_set(width_mask, 0, (ds_get_width(vd->ds) / 16)); - bitmap_clear(width_mask, (ds_get_width(vd->ds) / 16), - VNC_DIRTY_WORDS * BITS_PER_LONG); cmp_bytes = 16 * ds_get_bytes_per_pixel(vd->ds); guest_row = vd->guest.ds->data; server_row = vd->server->data; for (y = 0; y < vd->guest.ds->height; y++) { - if (bitmap_intersects(vd->guest.dirty[y], width_mask, VNC_DIRTY_WORDS)) { + if (!bitmap_empty(vd->guest.dirty[y], VNC_DIRTY_BITS)) { int x; uint8_t *guest_ptr; uint8_t *server_ptr; -- cgit v1.2.1