From eac78a4b0b7da4de2c0a297f4d528ca9cc6256a3 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Mon, 17 Jul 2023 12:39:38 +0200 Subject: linux-user: Fix signed math overflow in brk() syscall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the math overflow when calculating the new_malloc_size. new_host_brk_page and brk_page are unsigned integers. If userspace reduces the heap, new_host_brk_page is lower than brk_page which results in a huge positive number (but should actually be negative). Fix it by adding a proper check and as such make the code more readable. Signed-off-by: Helge Deller Tested-by: "Markus F.X.J. Oberhumer" Reviewed-by: Philippe Mathieu-Daudé Fixes: 86f04735ac ("linux-user: Fix brk() to release pages") Cc: qemu-stable@nongnu.org Buglink: https://github.com/upx/upx/issues/683 --- linux-user/syscall.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'linux-user') diff --git a/linux-user/syscall.c b/linux-user/syscall.c index 125fcbe..95727a8 100644 --- a/linux-user/syscall.c +++ b/linux-user/syscall.c @@ -860,12 +860,13 @@ abi_long do_brk(abi_ulong brk_val) * itself); instead we treat "mapped but at wrong address" as * a failure and unmap again. */ - new_alloc_size = new_host_brk_page - brk_page; - if (new_alloc_size) { + if (new_host_brk_page > brk_page) { + new_alloc_size = new_host_brk_page - brk_page; mapped_addr = get_errno(target_mmap(brk_page, new_alloc_size, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, 0, 0)); } else { + new_alloc_size = 0; mapped_addr = brk_page; } -- cgit v1.1