[gdb/python] Throw MemoryError in inferior.read_memory if malloc fails

PR python/31631 reports a gdb internal error when doing:
...
(gdb) python gdb.selected_inferior().read_memory (0, 0xffffffffffffffff)
utils.c:709: internal-error: virtual memory exhausted.
A problem internal to GDB has been detected,
further debugging may prove unreliable.
...

Fix this by throwing a python MemoryError, such that we have instead:
...
(gdb) python gdb.selected_inferior().read_memory (0, 0xffffffffffffffff)
Python Exception <class 'MemoryError'>:
Error occurred in Python.
(gdb)
...

Likewise for DAP.

Tested on x86_64-linux.

Bug: https://sourceware.org/bugzilla/show_bug.cgi?id=31631

4 files changed, 28 insertions, 3 deletions

diff --git a/gdb/python/lib/gdb/dap/memory.py b/gdb/python/lib/gdb/dap/memory.py
index dd62b0e..4aa4996 100644
--- a/gdb/python/lib/gdb/dap/memory.py
+++ b/gdb/python/lib/gdb/dap/memory.py
@@ -18,13 +18,17 @@ import base64
 import gdb
 
 from .server import capability, request
+from .startup import DAPException
 
 
 @request("readMemory")
 @capability("supportsReadMemoryRequest")
 def read_memory(*, memoryReference: str, offset: int = 0, count: int, **extra):
     addr = int(memoryReference, 0) + offset
-    buf = gdb.selected_inferior().read_memory(addr, count)
+    try:
+        buf = gdb.selected_inferior().read_memory(addr, count)
+    except MemoryError as e:
+        raise DAPException("Out of memory") from e
     return {
         "address": hex(addr),
         "data": base64.b64encode(buf).decode("ASCII"),
diff --git a/gdb/python/py-inferior.c b/gdb/python/py-inferior.c
index 795ac65..a1042ee 100644
--- a/gdb/python/py-inferior.c
+++ b/gdb/python/py-inferior.c
@@ -555,6 +555,18 @@ infpy_read_memory (PyObject *self, PyObject *args, PyObject *kw)
       || get_addr_from_python (length_obj, &length) < 0)
     return NULL;
 
+  if (length == 0)
+    {
+      PyErr_SetString (PyExc_ValueError,
+		       _("Argument 'count' should be greater than zero"));
+      return NULL;
+    }
+
+  void *p = malloc (length);
+  if (p == nullptr)
+    return PyErr_NoMemory ();
+  buffer.reset ((gdb_byte *) p);
+
   try
     {
       /* Use this scoped-restore because we want to be able to read
@@ -562,8 +574,6 @@ infpy_read_memory (PyObject *self, PyObject *args, PyObject *kw)
       scoped_restore_current_inferior_for_memory restore_inferior
 	(inf->inferior);
 
-      buffer.reset ((gdb_byte *) xmalloc (length));
-
       read_memory (addr, buffer.get (), length);
     }
   catch (const gdb_exception &except)
diff --git a/gdb/testsuite/gdb.dap/memory.exp b/gdb/testsuite/gdb.dap/memory.exp
index 2e911f4..4e2e361 100644
--- a/gdb/testsuite/gdb.dap/memory.exp
+++ b/gdb/testsuite/gdb.dap/memory.exp
@@ -55,6 +55,11 @@ set obj [dap_check_request_and_response "evaluate global pointer" \
 	     evaluate {o expression [s thirty_two_p]}]
 set addr [dict get [lindex $obj 0] body memoryReference]
 
+set obj [dap_request_and_response \
+	     readMemory [format {o memoryReference [s %s] count [i 18446744073709551615]} $addr]]
+set response [lindex $obj 0]
+gdb_assert { [dict get $response success] == "false" } "read memory, count to big"
+
 set obj [dap_check_request_and_response "read memory" \
 	     readMemory [format {o memoryReference [s %s] count [i 4]} $addr]]
 
diff --git a/gdb/testsuite/gdb.python/py-inferior.exp b/gdb/testsuite/gdb.python/py-inferior.exp
index c14f2d2..4c19e25 100644
--- a/gdb/testsuite/gdb.python/py-inferior.exp
+++ b/gdb/testsuite/gdb.python/py-inferior.exp
@@ -34,6 +34,12 @@ switch [get_endianness] {
     big { set python_pack_char ">" }
 }
 
+gdb_test \
+    "python gdb.selected_inferior().read_memory (0, 0xffffffffffffffff)" \
+    [multi_line \
+	 [string_to_regexp "Python Exception <class 'MemoryError'>: "] \
+	 [string_to_regexp "Error occurred in Python."]]
+
 # Test memory read operations without execution.
 
 gdb_py_test_silent_cmd "python addr = gdb.lookup_global_symbol ('int8_global').value().address" \