tests: add prefixes to the bare mkdtemp calls
[qemu.git] / python / qemu / machine.py
1 """
2 QEMU machine module:
3
4 The machine module primarily provides the QEMUMachine class,
5 which provides facilities for managing the lifetime of a QEMU VM.
6 """
7
8 # Copyright (C) 2015-2016 Red Hat Inc.
9 # Copyright (C) 2012 IBM Corp.
10 #
11 # Authors:
12 # Fam Zheng <famz@redhat.com>
13 #
14 # This work is licensed under the terms of the GNU GPL, version 2. See
15 # the COPYING file in the top-level directory.
16 #
17 # Based on qmp.py.
18 #
19
20 import errno
21 from itertools import chain
22 import logging
23 import os
24 import shutil
25 import signal
26 import socket
27 import subprocess
28 import tempfile
29 from types import TracebackType
30 from typing import (
31 Any,
32 BinaryIO,
33 Dict,
34 List,
35 Optional,
36 Sequence,
37 Tuple,
38 Type,
39 )
40
41 from . import console_socket, qmp
42 from .qmp import QMPMessage, QMPReturnValue, SocketAddrT
43
44
45 LOG = logging.getLogger(__name__)
46
47
48 class QEMUMachineError(Exception):
49 """
50 Exception called when an error in QEMUMachine happens.
51 """
52
53
54 class QEMUMachineAddDeviceError(QEMUMachineError):
55 """
56 Exception raised when a request to add a device can not be fulfilled
57
58 The failures are caused by limitations, lack of information or conflicting
59 requests on the QEMUMachine methods. This exception does not represent
60 failures reported by the QEMU binary itself.
61 """
62
63
64 class AbnormalShutdown(QEMUMachineError):
65 """
66 Exception raised when a graceful shutdown was requested, but not performed.
67 """
68
69
70 class QEMUMachine:
71 """
72 A QEMU VM.
73
74 Use this object as a context manager to ensure
75 the QEMU process terminates::
76
77 with VM(binary) as vm:
78 ...
79 # vm is guaranteed to be shut down here
80 """
81
82 def __init__(self,
83 binary: str,
84 args: Sequence[str] = (),
85 wrapper: Sequence[str] = (),
86 name: Optional[str] = None,
87 test_dir: str = "/var/tmp",
88 monitor_address: Optional[SocketAddrT] = None,
89 socket_scm_helper: Optional[str] = None,
90 sock_dir: Optional[str] = None,
91 drain_console: bool = False,
92 console_log: Optional[str] = None):
93 '''
94 Initialize a QEMUMachine
95
96 @param binary: path to the qemu binary
97 @param args: list of extra arguments
98 @param wrapper: list of arguments used as prefix to qemu binary
99 @param name: prefix for socket and log file names (default: qemu-PID)
100 @param test_dir: where to create socket and log file
101 @param monitor_address: address for QMP monitor
102 @param socket_scm_helper: helper program, required for send_fd_scm()
103 @param sock_dir: where to create socket (overrides test_dir for sock)
104 @param drain_console: (optional) True to drain console socket to buffer
105 @param console_log: (optional) path to console log file
106 @note: Qemu process is not started until launch() is used.
107 '''
108 # Direct user configuration
109
110 self._binary = binary
111 self._args = list(args)
112 self._wrapper = wrapper
113
114 self._name = name or "qemu-%d" % os.getpid()
115 self._test_dir = test_dir
116 self._sock_dir = sock_dir or self._test_dir
117 self._socket_scm_helper = socket_scm_helper
118
119 if monitor_address is not None:
120 self._monitor_address = monitor_address
121 self._remove_monitor_sockfile = False
122 else:
123 self._monitor_address = os.path.join(
124 self._sock_dir, f"{self._name}-monitor.sock"
125 )
126 self._remove_monitor_sockfile = True
127
128 self._console_log_path = console_log
129 if self._console_log_path:
130 # In order to log the console, buffering needs to be enabled.
131 self._drain_console = True
132 else:
133 self._drain_console = drain_console
134
135 # Runstate
136 self._qemu_log_path: Optional[str] = None
137 self._qemu_log_file: Optional[BinaryIO] = None
138 self._popen: Optional['subprocess.Popen[bytes]'] = None
139 self._events: List[QMPMessage] = []
140 self._iolog: Optional[str] = None
141 self._qmp_set = True # Enable QMP monitor by default.
142 self._qmp_connection: Optional[qmp.QEMUMonitorProtocol] = None
143 self._qemu_full_args: Tuple[str, ...] = ()
144 self._temp_dir: Optional[str] = None
145 self._launched = False
146 self._machine: Optional[str] = None
147 self._console_index = 0
148 self._console_set = False
149 self._console_device_type: Optional[str] = None
150 self._console_address = os.path.join(
151 self._sock_dir, f"{self._name}-console.sock"
152 )
153 self._console_socket: Optional[socket.socket] = None
154 self._remove_files: List[str] = []
155 self._user_killed = False
156
157 def __enter__(self) -> 'QEMUMachine':
158 return self
159
160 def __exit__(self,
161 exc_type: Optional[Type[BaseException]],
162 exc_val: Optional[BaseException],
163 exc_tb: Optional[TracebackType]) -> None:
164 self.shutdown()
165
166 def add_monitor_null(self) -> None:
167 """
168 This can be used to add an unused monitor instance.
169 """
170 self._args.append('-monitor')
171 self._args.append('null')
172
173 def add_fd(self, fd: int, fdset: int,
174 opaque: str, opts: str = '') -> 'QEMUMachine':
175 """
176 Pass a file descriptor to the VM
177 """
178 options = ['fd=%d' % fd,
179 'set=%d' % fdset,
180 'opaque=%s' % opaque]
181 if opts:
182 options.append(opts)
183
184 # This did not exist before 3.4, but since then it is
185 # mandatory for our purpose
186 if hasattr(os, 'set_inheritable'):
187 os.set_inheritable(fd, True)
188
189 self._args.append('-add-fd')
190 self._args.append(','.join(options))
191 return self
192
193 def send_fd_scm(self, fd: Optional[int] = None,
194 file_path: Optional[str] = None) -> int:
195 """
196 Send an fd or file_path to socket_scm_helper.
197
198 Exactly one of fd and file_path must be given.
199 If it is file_path, the helper will open that file and pass its own fd.
200 """
201 # In iotest.py, the qmp should always use unix socket.
202 assert self._qmp.is_scm_available()
203 if self._socket_scm_helper is None:
204 raise QEMUMachineError("No path to socket_scm_helper set")
205 if not os.path.exists(self._socket_scm_helper):
206 raise QEMUMachineError("%s does not exist" %
207 self._socket_scm_helper)
208
209 # This did not exist before 3.4, but since then it is
210 # mandatory for our purpose
211 if hasattr(os, 'set_inheritable'):
212 os.set_inheritable(self._qmp.get_sock_fd(), True)
213 if fd is not None:
214 os.set_inheritable(fd, True)
215
216 fd_param = ["%s" % self._socket_scm_helper,
217 "%d" % self._qmp.get_sock_fd()]
218
219 if file_path is not None:
220 assert fd is None
221 fd_param.append(file_path)
222 else:
223 assert fd is not None
224 fd_param.append(str(fd))
225
226 devnull = open(os.path.devnull, 'rb')
227 proc = subprocess.Popen(
228 fd_param, stdin=devnull, stdout=subprocess.PIPE,
229 stderr=subprocess.STDOUT, close_fds=False
230 )
231 output = proc.communicate()[0]
232 if output:
233 LOG.debug(output)
234
235 return proc.returncode
236
237 @staticmethod
238 def _remove_if_exists(path: str) -> None:
239 """
240 Remove file object at path if it exists
241 """
242 try:
243 os.remove(path)
244 except OSError as exception:
245 if exception.errno == errno.ENOENT:
246 return
247 raise
248
249 def is_running(self) -> bool:
250 """Returns true if the VM is running."""
251 return self._popen is not None and self._popen.poll() is None
252
253 @property
254 def _subp(self) -> 'subprocess.Popen[bytes]':
255 if self._popen is None:
256 raise QEMUMachineError('Subprocess pipe not present')
257 return self._popen
258
259 def exitcode(self) -> Optional[int]:
260 """Returns the exit code if possible, or None."""
261 if self._popen is None:
262 return None
263 return self._popen.poll()
264
265 def get_pid(self) -> Optional[int]:
266 """Returns the PID of the running process, or None."""
267 if not self.is_running():
268 return None
269 return self._subp.pid
270
271 def _load_io_log(self) -> None:
272 if self._qemu_log_path is not None:
273 with open(self._qemu_log_path, "r") as iolog:
274 self._iolog = iolog.read()
275
276 @property
277 def _base_args(self) -> List[str]:
278 args = ['-display', 'none', '-vga', 'none']
279
280 if self._qmp_set:
281 if isinstance(self._monitor_address, tuple):
282 moncdev = "socket,id=mon,host={},port={}".format(
283 *self._monitor_address
284 )
285 else:
286 moncdev = f"socket,id=mon,path={self._monitor_address}"
287 args.extend(['-chardev', moncdev, '-mon',
288 'chardev=mon,mode=control'])
289
290 if self._machine is not None:
291 args.extend(['-machine', self._machine])
292 for _ in range(self._console_index):
293 args.extend(['-serial', 'null'])
294 if self._console_set:
295 chardev = ('socket,id=console,path=%s,server,nowait' %
296 self._console_address)
297 args.extend(['-chardev', chardev])
298 if self._console_device_type is None:
299 args.extend(['-serial', 'chardev:console'])
300 else:
301 device = '%s,chardev=console' % self._console_device_type
302 args.extend(['-device', device])
303 return args
304
305 def _pre_launch(self) -> None:
306 self._temp_dir = tempfile.mkdtemp(prefix="qemu-machine-",
307 dir=self._test_dir)
308 self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log")
309 self._qemu_log_file = open(self._qemu_log_path, 'wb')
310
311 if self._console_set:
312 self._remove_files.append(self._console_address)
313
314 if self._qmp_set:
315 if self._remove_monitor_sockfile:
316 assert isinstance(self._monitor_address, str)
317 self._remove_files.append(self._monitor_address)
318 self._qmp_connection = qmp.QEMUMonitorProtocol(
319 self._monitor_address,
320 server=True,
321 nickname=self._name
322 )
323
324 def _post_launch(self) -> None:
325 if self._qmp_connection:
326 self._qmp.accept()
327
328 def _post_shutdown(self) -> None:
329 """
330 Called to cleanup the VM instance after the process has exited.
331 May also be called after a failed launch.
332 """
333 # Comprehensive reset for the failed launch case:
334 self._early_cleanup()
335
336 if self._qmp_connection:
337 self._qmp.close()
338 self._qmp_connection = None
339
340 self._load_io_log()
341
342 if self._qemu_log_file is not None:
343 self._qemu_log_file.close()
344 self._qemu_log_file = None
345
346 self._qemu_log_path = None
347
348 if self._temp_dir is not None:
349 shutil.rmtree(self._temp_dir)
350 self._temp_dir = None
351
352 while len(self._remove_files) > 0:
353 self._remove_if_exists(self._remove_files.pop())
354
355 exitcode = self.exitcode()
356 if (exitcode is not None and exitcode < 0
357 and not (self._user_killed and exitcode == -signal.SIGKILL)):
358 msg = 'qemu received signal %i; command: "%s"'
359 if self._qemu_full_args:
360 command = ' '.join(self._qemu_full_args)
361 else:
362 command = ''
363 LOG.warning(msg, -int(exitcode), command)
364
365 self._user_killed = False
366 self._launched = False
367
368 def launch(self) -> None:
369 """
370 Launch the VM and make sure we cleanup and expose the
371 command line/output in case of exception
372 """
373
374 if self._launched:
375 raise QEMUMachineError('VM already launched')
376
377 self._iolog = None
378 self._qemu_full_args = ()
379 try:
380 self._launch()
381 self._launched = True
382 except:
383 self._post_shutdown()
384
385 LOG.debug('Error launching VM')
386 if self._qemu_full_args:
387 LOG.debug('Command: %r', ' '.join(self._qemu_full_args))
388 if self._iolog:
389 LOG.debug('Output: %r', self._iolog)
390 raise
391
392 def _launch(self) -> None:
393 """
394 Launch the VM and establish a QMP connection
395 """
396 devnull = open(os.path.devnull, 'rb')
397 self._pre_launch()
398 self._qemu_full_args = tuple(
399 chain(self._wrapper,
400 [self._binary],
401 self._base_args,
402 self._args)
403 )
404 LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args))
405 self._popen = subprocess.Popen(self._qemu_full_args,
406 stdin=devnull,
407 stdout=self._qemu_log_file,
408 stderr=subprocess.STDOUT,
409 shell=False,
410 close_fds=False)
411 self._post_launch()
412
413 def _early_cleanup(self) -> None:
414 """
415 Perform any cleanup that needs to happen before the VM exits.
416
417 May be invoked by both soft and hard shutdown in failover scenarios.
418 Called additionally by _post_shutdown for comprehensive cleanup.
419 """
420 # If we keep the console socket open, we may deadlock waiting
421 # for QEMU to exit, while QEMU is waiting for the socket to
422 # become writeable.
423 if self._console_socket is not None:
424 self._console_socket.close()
425 self._console_socket = None
426
427 def _hard_shutdown(self) -> None:
428 """
429 Perform early cleanup, kill the VM, and wait for it to terminate.
430
431 :raise subprocess.Timeout: When timeout is exceeds 60 seconds
432 waiting for the QEMU process to terminate.
433 """
434 self._early_cleanup()
435 self._subp.kill()
436 self._subp.wait(timeout=60)
437
438 def _soft_shutdown(self, timeout: Optional[int],
439 has_quit: bool = False) -> None:
440 """
441 Perform early cleanup, attempt to gracefully shut down the VM, and wait
442 for it to terminate.
443
444 :param timeout: Timeout in seconds for graceful shutdown.
445 A value of None is an infinite wait.
446 :param has_quit: When True, don't attempt to issue 'quit' QMP command
447
448 :raise ConnectionReset: On QMP communication errors
449 :raise subprocess.TimeoutExpired: When timeout is exceeded waiting for
450 the QEMU process to terminate.
451 """
452 self._early_cleanup()
453
454 if self._qmp_connection:
455 if not has_quit:
456 # Might raise ConnectionReset
457 self._qmp.cmd('quit')
458
459 # May raise subprocess.TimeoutExpired
460 self._subp.wait(timeout=timeout)
461
462 def _do_shutdown(self, timeout: Optional[int],
463 has_quit: bool = False) -> None:
464 """
465 Attempt to shutdown the VM gracefully; fallback to a hard shutdown.
466
467 :param timeout: Timeout in seconds for graceful shutdown.
468 A value of None is an infinite wait.
469 :param has_quit: When True, don't attempt to issue 'quit' QMP command
470
471 :raise AbnormalShutdown: When the VM could not be shut down gracefully.
472 The inner exception will likely be ConnectionReset or
473 subprocess.TimeoutExpired. In rare cases, non-graceful termination
474 may result in its own exceptions, likely subprocess.TimeoutExpired.
475 """
476 try:
477 self._soft_shutdown(timeout, has_quit)
478 except Exception as exc:
479 self._hard_shutdown()
480 raise AbnormalShutdown("Could not perform graceful shutdown") \
481 from exc
482
483 def shutdown(self, has_quit: bool = False,
484 hard: bool = False,
485 timeout: Optional[int] = 30) -> None:
486 """
487 Terminate the VM (gracefully if possible) and perform cleanup.
488 Cleanup will always be performed.
489
490 If the VM has not yet been launched, or shutdown(), wait(), or kill()
491 have already been called, this method does nothing.
492
493 :param has_quit: When true, do not attempt to issue 'quit' QMP command.
494 :param hard: When true, do not attempt graceful shutdown, and
495 suppress the SIGKILL warning log message.
496 :param timeout: Optional timeout in seconds for graceful shutdown.
497 Default 30 seconds, A `None` value is an infinite wait.
498 """
499 if not self._launched:
500 return
501
502 try:
503 if hard:
504 self._user_killed = True
505 self._hard_shutdown()
506 else:
507 self._do_shutdown(timeout, has_quit)
508 finally:
509 self._post_shutdown()
510
511 def kill(self) -> None:
512 """
513 Terminate the VM forcefully, wait for it to exit, and perform cleanup.
514 """
515 self.shutdown(hard=True)
516
517 def wait(self, timeout: Optional[int] = 30) -> None:
518 """
519 Wait for the VM to power off and perform post-shutdown cleanup.
520
521 :param timeout: Optional timeout in seconds. Default 30 seconds.
522 A value of `None` is an infinite wait.
523 """
524 self.shutdown(has_quit=True, timeout=timeout)
525
526 def set_qmp_monitor(self, enabled: bool = True) -> None:
527 """
528 Set the QMP monitor.
529
530 @param enabled: if False, qmp monitor options will be removed from
531 the base arguments of the resulting QEMU command
532 line. Default is True.
533 @note: call this function before launch().
534 """
535 self._qmp_set = enabled
536
537 @property
538 def _qmp(self) -> qmp.QEMUMonitorProtocol:
539 if self._qmp_connection is None:
540 raise QEMUMachineError("Attempt to access QMP with no connection")
541 return self._qmp_connection
542
543 @classmethod
544 def _qmp_args(cls, _conv_keys: bool = True, **args: Any) -> Dict[str, Any]:
545 qmp_args = dict()
546 for key, value in args.items():
547 if _conv_keys:
548 qmp_args[key.replace('_', '-')] = value
549 else:
550 qmp_args[key] = value
551 return qmp_args
552
553 def qmp(self, cmd: str,
554 conv_keys: bool = True,
555 **args: Any) -> QMPMessage:
556 """
557 Invoke a QMP command and return the response dict
558 """
559 qmp_args = self._qmp_args(conv_keys, **args)
560 return self._qmp.cmd(cmd, args=qmp_args)
561
562 def command(self, cmd: str,
563 conv_keys: bool = True,
564 **args: Any) -> QMPReturnValue:
565 """
566 Invoke a QMP command.
567 On success return the response dict.
568 On failure raise an exception.
569 """
570 qmp_args = self._qmp_args(conv_keys, **args)
571 return self._qmp.command(cmd, **qmp_args)
572
573 def get_qmp_event(self, wait: bool = False) -> Optional[QMPMessage]:
574 """
575 Poll for one queued QMP events and return it
576 """
577 if self._events:
578 return self._events.pop(0)
579 return self._qmp.pull_event(wait=wait)
580
581 def get_qmp_events(self, wait: bool = False) -> List[QMPMessage]:
582 """
583 Poll for queued QMP events and return a list of dicts
584 """
585 events = self._qmp.get_events(wait=wait)
586 events.extend(self._events)
587 del self._events[:]
588 self._qmp.clear_events()
589 return events
590
591 @staticmethod
592 def event_match(event: Any, match: Optional[Any]) -> bool:
593 """
594 Check if an event matches optional match criteria.
595
596 The match criteria takes the form of a matching subdict. The event is
597 checked to be a superset of the subdict, recursively, with matching
598 values whenever the subdict values are not None.
599
600 This has a limitation that you cannot explicitly check for None values.
601
602 Examples, with the subdict queries on the left:
603 - None matches any object.
604 - {"foo": None} matches {"foo": {"bar": 1}}
605 - {"foo": None} matches {"foo": 5}
606 - {"foo": {"abc": None}} does not match {"foo": {"bar": 1}}
607 - {"foo": {"rab": 2}} matches {"foo": {"bar": 1, "rab": 2}}
608 """
609 if match is None:
610 return True
611
612 try:
613 for key in match:
614 if key in event:
615 if not QEMUMachine.event_match(event[key], match[key]):
616 return False
617 else:
618 return False
619 return True
620 except TypeError:
621 # either match or event wasn't iterable (not a dict)
622 return bool(match == event)
623
624 def event_wait(self, name: str,
625 timeout: float = 60.0,
626 match: Optional[QMPMessage] = None) -> Optional[QMPMessage]:
627 """
628 event_wait waits for and returns a named event from QMP with a timeout.
629
630 name: The event to wait for.
631 timeout: QEMUMonitorProtocol.pull_event timeout parameter.
632 match: Optional match criteria. See event_match for details.
633 """
634 return self.events_wait([(name, match)], timeout)
635
636 def events_wait(self,
637 events: Sequence[Tuple[str, Any]],
638 timeout: float = 60.0) -> Optional[QMPMessage]:
639 """
640 events_wait waits for and returns a single named event from QMP.
641 In the case of multiple qualifying events, this function returns the
642 first one.
643
644 :param events: A sequence of (name, match_criteria) tuples.
645 The match criteria are optional and may be None.
646 See event_match for details.
647 :param timeout: Optional timeout, in seconds.
648 See QEMUMonitorProtocol.pull_event.
649
650 :raise QMPTimeoutError: If timeout was non-zero and no matching events
651 were found.
652 :return: A QMP event matching the filter criteria.
653 If timeout was 0 and no event matched, None.
654 """
655 def _match(event: QMPMessage) -> bool:
656 for name, match in events:
657 if event['event'] == name and self.event_match(event, match):
658 return True
659 return False
660
661 event: Optional[QMPMessage]
662
663 # Search cached events
664 for event in self._events:
665 if _match(event):
666 self._events.remove(event)
667 return event
668
669 # Poll for new events
670 while True:
671 event = self._qmp.pull_event(wait=timeout)
672 if event is None:
673 # NB: None is only returned when timeout is false-ish.
674 # Timeouts raise QMPTimeoutError instead!
675 break
676 if _match(event):
677 return event
678 self._events.append(event)
679
680 return None
681
682 def get_log(self) -> Optional[str]:
683 """
684 After self.shutdown or failed qemu execution, this returns the output
685 of the qemu process.
686 """
687 return self._iolog
688
689 def add_args(self, *args: str) -> None:
690 """
691 Adds to the list of extra arguments to be given to the QEMU binary
692 """
693 self._args.extend(args)
694
695 def set_machine(self, machine_type: str) -> None:
696 """
697 Sets the machine type
698
699 If set, the machine type will be added to the base arguments
700 of the resulting QEMU command line.
701 """
702 self._machine = machine_type
703
704 def set_console(self,
705 device_type: Optional[str] = None,
706 console_index: int = 0) -> None:
707 """
708 Sets the device type for a console device
709
710 If set, the console device and a backing character device will
711 be added to the base arguments of the resulting QEMU command
712 line.
713
714 This is a convenience method that will either use the provided
715 device type, or default to a "-serial chardev:console" command
716 line argument.
717
718 The actual setting of command line arguments will be be done at
719 machine launch time, as it depends on the temporary directory
720 to be created.
721
722 @param device_type: the device type, such as "isa-serial". If
723 None is given (the default value) a "-serial
724 chardev:console" command line argument will
725 be used instead, resorting to the machine's
726 default device type.
727 @param console_index: the index of the console device to use.
728 If not zero, the command line will create
729 'index - 1' consoles and connect them to
730 the 'null' backing character device.
731 """
732 self._console_set = True
733 self._console_device_type = device_type
734 self._console_index = console_index
735
736 @property
737 def console_socket(self) -> socket.socket:
738 """
739 Returns a socket connected to the console
740 """
741 if self._console_socket is None:
742 self._console_socket = console_socket.ConsoleSocket(
743 self._console_address,
744 file=self._console_log_path,
745 drain=self._drain_console)
746 return self._console_socket