From af76484e54f6c5e20452c2b329378026b8f2c59d Mon Sep 17 00:00:00 2001 From: Dongdong Zhang Date: Wed, 30 Nov 2022 09:53:58 +0800 Subject: Fix some typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix some typos in 'python' directory. Signed-off-by: Dongdong Zhang Reviewed-by: Philippe Mathieu-Daudé Message-id: 20221130015358.6998-2-zhangdongdong@eswincomputing.com [Fixed additional typo spotted by Max Filippov. --js] Reviewed-by: John Snow Signed-off-by: John Snow --- python/qemu/machine/console_socket.py | 2 +- python/qemu/machine/qtest.py | 2 +- python/qemu/qmp/protocol.py | 2 +- python/qemu/qmp/qmp_tui.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/qemu/machine/console_socket.py b/python/qemu/machine/console_socket.py index 8c4ff59..4e28ba9 100644 --- a/python/qemu/machine/console_socket.py +++ b/python/qemu/machine/console_socket.py @@ -68,7 +68,7 @@ class ConsoleSocket(socket.socket): """Kick off a thread to drain the socket.""" # Configure socket to not block and timeout. # This allows our drain thread to not block - # on recieve and exit smoothly. + # on receive and exit smoothly. socket.socket.setblocking(self, False) socket.socket.settimeout(self, 1) drain_thread = threading.Thread(target=self._drain_fn) diff --git a/python/qemu/machine/qtest.py b/python/qemu/machine/qtest.py index 1a1fc6c..1c46138 100644 --- a/python/qemu/machine/qtest.py +++ b/python/qemu/machine/qtest.py @@ -42,7 +42,7 @@ class QEMUQtestProtocol: :raise socket.error: on socket connection errors .. note:: - No conection is estabalished by __init__(), this is done + No connection is established by __init__(), this is done by the connect() or accept() methods. """ def __init__(self, address: SocketAddrT, diff --git a/python/qemu/qmp/protocol.py b/python/qemu/qmp/protocol.py index 6ea8665..15909b7 100644 --- a/python/qemu/qmp/protocol.py +++ b/python/qemu/qmp/protocol.py @@ -812,7 +812,7 @@ class AsyncProtocol(Generic[T]): @bottom_half async def _bh_close_stream(self, error_pathway: bool = False) -> None: - # NB: Closing the writer also implcitly closes the reader. + # NB: Closing the writer also implicitly closes the reader. if not self._writer: return diff --git a/python/qemu/qmp/qmp_tui.py b/python/qemu/qmp/qmp_tui.py index ce239d8..8369144 100644 --- a/python/qemu/qmp/qmp_tui.py +++ b/python/qemu/qmp/qmp_tui.py @@ -71,7 +71,7 @@ def format_json(msg: str) -> str: due to an decoding error then a simple string manipulation is done to achieve a single line JSON string. - Converting into single line is more asthetically pleasing when looking + Converting into single line is more aesthetically pleasing when looking along with error messages. Eg: @@ -91,7 +91,7 @@ def format_json(msg: str) -> str: [1, true, 3]: QMP message is not a JSON object. - The single line mode is more asthetically pleasing. + The single line mode is more aesthetically pleasing. :param msg: The message to formatted into single line. @@ -498,7 +498,7 @@ class EditorWidget(urwid.Filler): class HistoryBox(urwid.ListBox): """ This widget is modelled using the ListBox widget, contains the list of - all messages both QMP messages and log messsages to be shown in the TUI. + all messages both QMP messages and log messages to be shown in the TUI. The messages are urwid.Text widgets. On every append of a message, the focus is shifted to the last appended message. -- cgit v1.1 From ada73a492cb29b9c3a9f88c5e6d3407fa0d999e7 Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Fri, 24 Jun 2022 22:52:52 +0300 Subject: python: QEMUMachine: enable qmp accept timeout by default I've spent much time trying to debug hanging pipeline in gitlab. I started from and idea that I have problem in code in my series (which has some timeouts). Finally I found that the problem is that I've used QEMUMachine class directly to avoid qtest, and didn't add necessary arguments. Qemu fails and we wait for qmp accept endlessly. In gitlab it's just stopped by timeout (one hour) with no sign of what's going wrong. With timeout enabled, gitlab don't wait for an hour and prints all needed information. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: John Snow Message-Id: <20220624195252.175249-1-vsementsov@yandex-team.ru> [Fixed typing. --js] Signed-off-by: John Snow --- python/qemu/machine/machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py index 748a0d8..c759db0 100644 --- a/python/qemu/machine/machine.py +++ b/python/qemu/machine/machine.py @@ -131,7 +131,7 @@ class QEMUMachine: drain_console: bool = False, console_log: Optional[str] = None, log_dir: Optional[str] = None, - qmp_timer: Optional[float] = None): + qmp_timer: Optional[float] = 30): ''' Initialize a QEMUMachine -- cgit v1.1 From f9922937d173f50fe59fd1b20fadc445fb6b2564 Mon Sep 17 00:00:00 2001 From: Peter Delevoryas Date: Tue, 10 Jan 2023 00:29:30 -0800 Subject: python/machine: Fix AF_UNIX path too long on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On macOS, private $TMPDIR's are the default. These $TMPDIR's are generated from a user's unix UID and UUID [1], which can create a relatively long path: /var/folders/d7/rz20f6hd709c1ty8f6_6y_z40000gn/T/ QEMU's avocado tests create a temporary directory prefixed by "avo_qemu_sock_", and create QMP sockets within _that_ as well. The QMP socket is unnecessarily long, because a temporary directory is created for every QEMUMachine object. /avo_qemu_sock_uh3w_dgc/qemu-37331-10bacf110-monitor.sock The path limit for unix sockets on macOS is 104: [2] /* * [XSI] Definitions for UNIX IPC domain. */ struct sockaddr_un { unsigned char sun_len; /* sockaddr len including null */ sa_family_t sun_family; /* [XSI] AF_UNIX */ char sun_path[104]; /* [XSI] path name (gag) */ }; This results in avocado tests failing on macOS because the QMP unix socket can't be created, because the path is too long: ERROR| Failed to establish connection: OSError: AF_UNIX path too long This change resolves by reducing the size of the socket directory prefix and the suffix on the QMP and console socket names. The result is paths like this: pdel@pdel-mbp:/var/folders/d7/rz20f6hd709c1ty8f6_6y_z40000gn/T $ tree qemu* qemu_df4evjeq qemu_jbxel3gy qemu_ml9s_gg7 qemu_oc7h7f3u qemu_oqb1yf97 ├── 10a004050.con └── 10a004050.qmp [1] https://apple.stackexchange.com/questions/353832/why-is-mac-osx-temp-directory-in-weird-path [2] /Library/Developer/CommandLineTools/SDKs/MacOSX12.3.sdk/usr/include/sys/un.h Signed-off-by: Peter Delevoryas Reviewed-by: Daniel P. Berrangé Reviewed-by: Philippe Mathieu-Daudé Message-id: 20230110082930.42129-2-peter@pjd.dev Signed-off-by: John Snow --- python/qemu/machine/machine.py | 6 +++--- tests/avocado/avocado_qemu/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py index c759db0..a71d87e 100644 --- a/python/qemu/machine/machine.py +++ b/python/qemu/machine/machine.py @@ -157,7 +157,7 @@ class QEMUMachine: self._wrapper = wrapper self._qmp_timer = qmp_timer - self._name = name or f"qemu-{os.getpid()}-{id(self):02x}" + self._name = name or f"{id(self):x}" self._temp_dir: Optional[str] = None self._base_temp_dir = base_temp_dir self._sock_dir = sock_dir @@ -167,7 +167,7 @@ class QEMUMachine: self._monitor_address = monitor_address else: self._monitor_address = os.path.join( - self.sock_dir, f"{self._name}-monitor.sock" + self.sock_dir, f"{self._name}.qmp" ) self._console_log_path = console_log @@ -192,7 +192,7 @@ class QEMUMachine: self._console_set = False self._console_device_type: Optional[str] = None self._console_address = os.path.join( - self.sock_dir, f"{self._name}-console.sock" + self.sock_dir, f"{self._name}.con" ) self._console_socket: Optional[socket.socket] = None self._remove_files: List[str] = [] diff --git a/tests/avocado/avocado_qemu/__init__.py b/tests/avocado/avocado_qemu/__init__.py index 910f3ba..25a5468 100644 --- a/tests/avocado/avocado_qemu/__init__.py +++ b/tests/avocado/avocado_qemu/__init__.py @@ -306,7 +306,7 @@ class QemuSystemTest(QemuBaseTest): self.cancel('no support for user networking') def _new_vm(self, name, *args): - self._sd = tempfile.TemporaryDirectory(prefix="avo_qemu_sock_") + self._sd = tempfile.TemporaryDirectory(prefix="qemu_") vm = QEMUMachine(self.qemu_bin, base_temp_dir=self.workdir, sock_dir=self._sd.name, log_dir=self.logdir) self.log.debug('QEMUMachine "%s" created', name) -- cgit v1.1 From 166464c6ce82f748e33b24361a72e9d310130fa0 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 12 Jan 2023 18:28:03 +0300 Subject: python/qmp: increase read buffer size Current 256KB is not enough for some real cases. As a possible solution limit can be chosen to be the same as libvirt (10MB) Signed-off-by: Maksim Davydov Reviewed-by: John Snow Message-id: 20230112152805.33109-3-davydov-max@yandex-team.ru Signed-off-by: John Snow --- python/qemu/qmp/qmp_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/qemu/qmp/qmp_client.py b/python/qemu/qmp/qmp_client.py index 5dcda04..b5772e7 100644 --- a/python/qemu/qmp/qmp_client.py +++ b/python/qemu/qmp/qmp_client.py @@ -197,8 +197,8 @@ class QMPClient(AsyncProtocol[Message], Events): #: Logger object used for debugging messages. logger = logging.getLogger(__name__) - # Read buffer limit; large enough to accept query-qmp-schema - _limit = (256 * 1024) + # Read buffer limit; 10MB like libvirt default + _limit = (10 * 1024 * 1024) # Type alias for pending execute() result items _PendingT = Union[Message, ExecInterruptedError] -- cgit v1.1 From a3cfea92e2030926e00a2519d299384ea648e36e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Wed, 11 Jan 2023 12:00:59 +0400 Subject: python/qmp/protocol: add open_with_socket() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of listening for incoming connections with a SocketAddr, add a new method open_with_socket() that accepts an existing socket. Signed-off-by: Marc-André Lureau Reviewed-by: Daniel P. Berrangé Message-id: 20230111080101.969151-2-marcandre.lureau@redhat.com Signed-off-by: John Snow --- python/qemu/qmp/protocol.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/python/qemu/qmp/protocol.py b/python/qemu/qmp/protocol.py index 15909b7..6d3d739 100644 --- a/python/qemu/qmp/protocol.py +++ b/python/qemu/qmp/protocol.py @@ -18,6 +18,7 @@ from asyncio import StreamReader, StreamWriter from enum import Enum from functools import wraps import logging +import socket from ssl import SSLContext from typing import ( Any, @@ -298,6 +299,19 @@ class AsyncProtocol(Generic[T]): @upper_half @require(Runstate.IDLE) + async def open_with_socket(self, sock: socket.socket) -> None: + """ + Start connection with given socket. + + :param sock: A socket. + + :raise StateError: When the `Runstate` is not `IDLE`. + """ + self._reader, self._writer = await asyncio.open_connection(sock=sock) + self._set_state(Runstate.CONNECTING) + + @upper_half + @require(Runstate.IDLE) async def start_server(self, address: SocketAddrT, ssl: Optional[SSLContext] = None) -> None: """ @@ -343,11 +357,12 @@ class AsyncProtocol(Generic[T]): protocol-level failure occurs while establishing a new session, the wrapped error may also be an `QMPError`. """ - if self._accepted is None: - raise QMPError("Cannot call accept() before start_server().") - await self._session_guard( - self._do_accept(), - 'Failed to establish connection') + if not self._reader: + if self._accepted is None: + raise QMPError("Cannot call accept() before start_server().") + await self._session_guard( + self._do_accept(), + 'Failed to establish connection') await self._session_guard( self._establish_session(), 'Failed to establish session') -- cgit v1.1 From 603a3bad4b9a95b524dc8d6a41b1be4d5c5cacdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Wed, 11 Jan 2023 12:01:00 +0400 Subject: python/qmp/legacy: make QEMUMonitorProtocol accept a socket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Teach QEMUMonitorProtocol to accept an exisiting socket. Signed-off-by: Marc-André Lureau Reviewed-by: Daniel P. Berrangé Message-id: 20230111080101.969151-3-marcandre.lureau@redhat.com Signed-off-by: John Snow --- python/qemu/qmp/legacy.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/python/qemu/qmp/legacy.py b/python/qemu/qmp/legacy.py index 1951754..8b09ee7 100644 --- a/python/qemu/qmp/legacy.py +++ b/python/qemu/qmp/legacy.py @@ -22,6 +22,7 @@ old interface. # import asyncio +import socket from types import TracebackType from typing import ( Any, @@ -69,22 +70,32 @@ class QEMUMonitorProtocol: :param address: QEMU address, can be either a unix socket path (string) or a tuple in the form ( address, port ) for a TCP - connection + connection or None + :param sock: a socket or None :param server: Act as the socket server. (See 'accept') :param nickname: Optional nickname used for logging. """ - def __init__(self, address: SocketAddrT, + def __init__(self, + address: Optional[SocketAddrT] = None, + sock: Optional[socket.socket] = None, server: bool = False, nickname: Optional[str] = None): + assert address or sock self._qmp = QMPClient(nickname) self._aloop = asyncio.get_event_loop() self._address = address + self._sock = sock self._timeout: Optional[float] = None if server: - self._sync(self._qmp.start_server(self._address)) + if sock: + assert self._sock is not None + self._sync(self._qmp.open_with_socket(self._sock)) + else: + assert self._address is not None + self._sync(self._qmp.start_server(self._address)) _T = TypeVar('_T') @@ -139,6 +150,7 @@ class QEMUMonitorProtocol: :return: QMP greeting dict, or None if negotiate is false :raise ConnectError: on connection errors """ + assert self._address is not None self._qmp.await_greeting = negotiate self._qmp.negotiate = negotiate -- cgit v1.1 From bd4c0ef409140bd1be393407c04005ac077d4574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Wed, 11 Jan 2023 12:01:01 +0400 Subject: python/qemu/machine: use socketpair() for QMP by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When no monitor address is given, establish the QMP communication through a socketpair() (API is also supported on Windows since Python 3.5) Signed-off-by: Marc-André Lureau Reviewed-by: Daniel P. Berrangé Message-id: 20230111080101.969151-4-marcandre.lureau@redhat.com [Resolved conflicts, fixed typing error. --js] Signed-off-by: John Snow --- python/qemu/machine/machine.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py index a71d87e..e57c254 100644 --- a/python/qemu/machine/machine.py +++ b/python/qemu/machine/machine.py @@ -158,17 +158,13 @@ class QEMUMachine: self._qmp_timer = qmp_timer self._name = name or f"{id(self):x}" + self._sock_pair: Optional[Tuple[socket.socket, socket.socket]] = None self._temp_dir: Optional[str] = None self._base_temp_dir = base_temp_dir self._sock_dir = sock_dir self._log_dir = log_dir - if monitor_address is not None: - self._monitor_address = monitor_address - else: - self._monitor_address = os.path.join( - self.sock_dir, f"{self._name}.qmp" - ) + self._monitor_address = monitor_address self._console_log_path = console_log if self._console_log_path: @@ -303,7 +299,11 @@ class QEMUMachine: args = ['-display', 'none', '-vga', 'none'] if self._qmp_set: - if isinstance(self._monitor_address, tuple): + if self._sock_pair: + fd = self._sock_pair[0].fileno() + os.set_inheritable(fd, True) + moncdev = f"socket,id=mon,fd={fd}" + elif isinstance(self._monitor_address, tuple): moncdev = "socket,id=mon,host={},port={}".format( *self._monitor_address ) @@ -337,10 +337,17 @@ class QEMUMachine: self._remove_files.append(self._console_address) if self._qmp_set: + monitor_address = None + sock = None + if self._monitor_address is None: + self._sock_pair = socket.socketpair() + sock = self._sock_pair[1] if isinstance(self._monitor_address, str): self._remove_files.append(self._monitor_address) + monitor_address = self._monitor_address self._qmp_connection = QEMUMonitorProtocol( - self._monitor_address, + address=monitor_address, + sock=sock, server=True, nickname=self._name ) @@ -360,6 +367,8 @@ class QEMUMachine: )) def _post_launch(self) -> None: + if self._sock_pair: + self._sock_pair[0].close() if self._qmp_connection: self._qmp.accept(self._qmp_timer) -- cgit v1.1