[gdb/python] Throw MemoryError in inferior.read_memory if malloc fails

PR python/31631 reports a gdb internal error when doing:
...
(gdb) python gdb.selected_inferior().read_memory (0, 0xffffffffffffffff)
utils.c:709: internal-error: virtual memory exhausted.
A problem internal to GDB has been detected,
further debugging may prove unreliable.
...

Fix this by throwing a python MemoryError, such that we have instead:
...
(gdb) python gdb.selected_inferior().read_memory (0, 0xffffffffffffffff)
Python Exception <class 'MemoryError'>:
Error occurred in Python.
(gdb)
...

Likewise for DAP.

Tested on x86_64-linux.

Bug: https://sourceware.org/bugzilla/show_bug.cgi?id=31631

2 files changed, 17 insertions, 3 deletions

diff --git a/gdb/python/lib/gdb/dap/memory.py b/gdb/python/lib/gdb/dap/memory.py
index dd62b0e..4aa4996 100644
--- a/gdb/python/lib/gdb/dap/memory.py
+++ b/gdb/python/lib/gdb/dap/memory.py
@@ -18,13 +18,17 @@ import base64
 import gdb
 
 from .server import capability, request
+from .startup import DAPException
 
 
 @request("readMemory")
 @capability("supportsReadMemoryRequest")
 def read_memory(*, memoryReference: str, offset: int = 0, count: int, **extra):
     addr = int(memoryReference, 0) + offset
-    buf = gdb.selected_inferior().read_memory(addr, count)
+    try:
+        buf = gdb.selected_inferior().read_memory(addr, count)
+    except MemoryError as e:
+        raise DAPException("Out of memory") from e
     return {
         "address": hex(addr),
         "data": base64.b64encode(buf).decode("ASCII"),
diff --git a/gdb/python/py-inferior.c b/gdb/python/py-inferior.c
index 795ac65..a1042ee 100644
--- a/gdb/python/py-inferior.c
+++ b/gdb/python/py-inferior.c
@@ -555,6 +555,18 @@ infpy_read_memory (PyObject *self, PyObject *args, PyObject *kw)
       || get_addr_from_python (length_obj, &length) < 0)
     return NULL;
 
+  if (length == 0)
+    {
+      PyErr_SetString (PyExc_ValueError,
+		       _("Argument 'count' should be greater than zero"));
+      return NULL;
+    }
+
+  void *p = malloc (length);
+  if (p == nullptr)
+    return PyErr_NoMemory ();
+  buffer.reset ((gdb_byte *) p);
+
   try
     {
       /* Use this scoped-restore because we want to be able to read
@@ -562,8 +574,6 @@ infpy_read_memory (PyObject *self, PyObject *args, PyObject *kw)
       scoped_restore_current_inferior_for_memory restore_inferior
 	(inf->inferior);
 
-      buffer.reset ((gdb_byte *) xmalloc (length));
-
       read_memory (addr, buffer.get (), length);
     }
   catch (const gdb_exception &except)