qemu-img: add support for rate limit in qemu-img commit
[qemu.git] / python / qemu / machine.py
1 """
2 QEMU machine module:
3
4 The machine module primarily provides the QEMUMachine class,
5 which provides facilities for managing the lifetime of a QEMU VM.
6 """
7
8 # Copyright (C) 2015-2016 Red Hat Inc.
9 # Copyright (C) 2012 IBM Corp.
10 #
11 # Authors:
12 # Fam Zheng <famz@redhat.com>
13 #
14 # This work is licensed under the terms of the GNU GPL, version 2. See
15 # the COPYING file in the top-level directory.
16 #
17 # Based on qmp.py.
18 #
19
20 import errno
21 from itertools import chain
22 import logging
23 import os
24 import shutil
25 import signal
26 import socket
27 import subprocess
28 import tempfile
29 from types import TracebackType
30 from typing import (
31 Any,
32 BinaryIO,
33 Dict,
34 List,
35 Optional,
36 Sequence,
37 Tuple,
38 Type,
39 )
40
41 from . import console_socket, qmp
42 from .qmp import QMPMessage, QMPReturnValue, SocketAddrT
43
44
45 LOG = logging.getLogger(__name__)
46
47
48 class QEMUMachineError(Exception):
49 """
50 Exception called when an error in QEMUMachine happens.
51 """
52
53
54 class QEMUMachineAddDeviceError(QEMUMachineError):
55 """
56 Exception raised when a request to add a device can not be fulfilled
57
58 The failures are caused by limitations, lack of information or conflicting
59 requests on the QEMUMachine methods. This exception does not represent
60 failures reported by the QEMU binary itself.
61 """
62
63
64 class AbnormalShutdown(QEMUMachineError):
65 """
66 Exception raised when a graceful shutdown was requested, but not performed.
67 """
68
69
70 class QEMUMachine:
71 """
72 A QEMU VM.
73
74 Use this object as a context manager to ensure
75 the QEMU process terminates::
76
77 with VM(binary) as vm:
78 ...
79 # vm is guaranteed to be shut down here
80 """
81
82 def __init__(self,
83 binary: str,
84 args: Sequence[str] = (),
85 wrapper: Sequence[str] = (),
86 name: Optional[str] = None,
87 test_dir: str = "/var/tmp",
88 monitor_address: Optional[SocketAddrT] = None,
89 socket_scm_helper: Optional[str] = None,
90 sock_dir: Optional[str] = None,
91 drain_console: bool = False,
92 console_log: Optional[str] = None):
93 '''
94 Initialize a QEMUMachine
95
96 @param binary: path to the qemu binary
97 @param args: list of extra arguments
98 @param wrapper: list of arguments used as prefix to qemu binary
99 @param name: prefix for socket and log file names (default: qemu-PID)
100 @param test_dir: where to create socket and log file
101 @param monitor_address: address for QMP monitor
102 @param socket_scm_helper: helper program, required for send_fd_scm()
103 @param sock_dir: where to create socket (overrides test_dir for sock)
104 @param drain_console: (optional) True to drain console socket to buffer
105 @param console_log: (optional) path to console log file
106 @note: Qemu process is not started until launch() is used.
107 '''
108 # Direct user configuration
109
110 self._binary = binary
111 self._args = list(args)
112 self._wrapper = wrapper
113
114 self._name = name or "qemu-%d" % os.getpid()
115 self._test_dir = test_dir
116 self._sock_dir = sock_dir or self._test_dir
117 self._socket_scm_helper = socket_scm_helper
118
119 if monitor_address is not None:
120 self._monitor_address = monitor_address
121 self._remove_monitor_sockfile = False
122 else:
123 self._monitor_address = os.path.join(
124 self._sock_dir, f"{self._name}-monitor.sock"
125 )
126 self._remove_monitor_sockfile = True
127
128 self._console_log_path = console_log
129 if self._console_log_path:
130 # In order to log the console, buffering needs to be enabled.
131 self._drain_console = True
132 else:
133 self._drain_console = drain_console
134
135 # Runstate
136 self._qemu_log_path: Optional[str] = None
137 self._qemu_log_file: Optional[BinaryIO] = None
138 self._popen: Optional['subprocess.Popen[bytes]'] = None
139 self._events: List[QMPMessage] = []
140 self._iolog: Optional[str] = None
141 self._qmp_set = True # Enable QMP monitor by default.
142 self._qmp_connection: Optional[qmp.QEMUMonitorProtocol] = None
143 self._qemu_full_args: Tuple[str, ...] = ()
144 self._temp_dir: Optional[str] = None
145 self._launched = False
146 self._machine: Optional[str] = None
147 self._console_index = 0
148 self._console_set = False
149 self._console_device_type: Optional[str] = None
150 self._console_address = os.path.join(
151 self._sock_dir, f"{self._name}-console.sock"
152 )
153 self._console_socket: Optional[socket.socket] = None
154 self._remove_files: List[str] = []
155 self._user_killed = False
156
157 def __enter__(self) -> 'QEMUMachine':
158 return self
159
160 def __exit__(self,
161 exc_type: Optional[Type[BaseException]],
162 exc_val: Optional[BaseException],
163 exc_tb: Optional[TracebackType]) -> None:
164 self.shutdown()
165
166 def add_monitor_null(self) -> None:
167 """
168 This can be used to add an unused monitor instance.
169 """
170 self._args.append('-monitor')
171 self._args.append('null')
172
173 def add_fd(self, fd: int, fdset: int,
174 opaque: str, opts: str = '') -> 'QEMUMachine':
175 """
176 Pass a file descriptor to the VM
177 """
178 options = ['fd=%d' % fd,
179 'set=%d' % fdset,
180 'opaque=%s' % opaque]
181 if opts:
182 options.append(opts)
183
184 # This did not exist before 3.4, but since then it is
185 # mandatory for our purpose
186 if hasattr(os, 'set_inheritable'):
187 os.set_inheritable(fd, True)
188
189 self._args.append('-add-fd')
190 self._args.append(','.join(options))
191 return self
192
193 def send_fd_scm(self, fd: Optional[int] = None,
194 file_path: Optional[str] = None) -> int:
195 """
196 Send an fd or file_path to socket_scm_helper.
197
198 Exactly one of fd and file_path must be given.
199 If it is file_path, the helper will open that file and pass its own fd.
200 """
201 # In iotest.py, the qmp should always use unix socket.
202 assert self._qmp.is_scm_available()
203 if self._socket_scm_helper is None:
204 raise QEMUMachineError("No path to socket_scm_helper set")
205 if not os.path.exists(self._socket_scm_helper):
206 raise QEMUMachineError("%s does not exist" %
207 self._socket_scm_helper)
208
209 # This did not exist before 3.4, but since then it is
210 # mandatory for our purpose
211 if hasattr(os, 'set_inheritable'):
212 os.set_inheritable(self._qmp.get_sock_fd(), True)
213 if fd is not None:
214 os.set_inheritable(fd, True)
215
216 fd_param = ["%s" % self._socket_scm_helper,
217 "%d" % self._qmp.get_sock_fd()]
218
219 if file_path is not None:
220 assert fd is None
221 fd_param.append(file_path)
222 else:
223 assert fd is not None
224 fd_param.append(str(fd))
225
226 devnull = open(os.path.devnull, 'rb')
227 proc = subprocess.Popen(
228 fd_param, stdin=devnull, stdout=subprocess.PIPE,
229 stderr=subprocess.STDOUT, close_fds=False
230 )
231 output = proc.communicate()[0]
232 if output:
233 LOG.debug(output)
234
235 return proc.returncode
236
237 @staticmethod
238 def _remove_if_exists(path: str) -> None:
239 """
240 Remove file object at path if it exists
241 """
242 try:
243 os.remove(path)
244 except OSError as exception:
245 if exception.errno == errno.ENOENT:
246 return
247 raise
248
249 def is_running(self) -> bool:
250 """Returns true if the VM is running."""
251 return self._popen is not None and self._popen.poll() is None
252
253 @property
254 def _subp(self) -> 'subprocess.Popen[bytes]':
255 if self._popen is None:
256 raise QEMUMachineError('Subprocess pipe not present')
257 return self._popen
258
259 def exitcode(self) -> Optional[int]:
260 """Returns the exit code if possible, or None."""
261 if self._popen is None:
262 return None
263 return self._popen.poll()
264
265 def get_pid(self) -> Optional[int]:
266 """Returns the PID of the running process, or None."""
267 if not self.is_running():
268 return None
269 return self._subp.pid
270
271 def _load_io_log(self) -> None:
272 if self._qemu_log_path is not None:
273 with open(self._qemu_log_path, "r") as iolog:
274 self._iolog = iolog.read()
275
276 @property
277 def _base_args(self) -> List[str]:
278 args = ['-display', 'none', '-vga', 'none']
279
280 if self._qmp_set:
281 if isinstance(self._monitor_address, tuple):
282 moncdev = "socket,id=mon,host={},port={}".format(
283 *self._monitor_address
284 )
285 else:
286 moncdev = f"socket,id=mon,path={self._monitor_address}"
287 args.extend(['-chardev', moncdev, '-mon',
288 'chardev=mon,mode=control'])
289
290 if self._machine is not None:
291 args.extend(['-machine', self._machine])
292 for _ in range(self._console_index):
293 args.extend(['-serial', 'null'])
294 if self._console_set:
295 chardev = ('socket,id=console,path=%s,server,nowait' %
296 self._console_address)
297 args.extend(['-chardev', chardev])
298 if self._console_device_type is None:
299 args.extend(['-serial', 'chardev:console'])
300 else:
301 device = '%s,chardev=console' % self._console_device_type
302 args.extend(['-device', device])
303 return args
304
305 def _pre_launch(self) -> None:
306 self._temp_dir = tempfile.mkdtemp(dir=self._test_dir)
307 self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log")
308 self._qemu_log_file = open(self._qemu_log_path, 'wb')
309
310 if self._console_set:
311 self._remove_files.append(self._console_address)
312
313 if self._qmp_set:
314 if self._remove_monitor_sockfile:
315 assert isinstance(self._monitor_address, str)
316 self._remove_files.append(self._monitor_address)
317 self._qmp_connection = qmp.QEMUMonitorProtocol(
318 self._monitor_address,
319 server=True,
320 nickname=self._name
321 )
322
323 def _post_launch(self) -> None:
324 if self._qmp_connection:
325 self._qmp.accept()
326
327 def _post_shutdown(self) -> None:
328 """
329 Called to cleanup the VM instance after the process has exited.
330 May also be called after a failed launch.
331 """
332 # Comprehensive reset for the failed launch case:
333 self._early_cleanup()
334
335 if self._qmp_connection:
336 self._qmp.close()
337 self._qmp_connection = None
338
339 self._load_io_log()
340
341 if self._qemu_log_file is not None:
342 self._qemu_log_file.close()
343 self._qemu_log_file = None
344
345 self._qemu_log_path = None
346
347 if self._temp_dir is not None:
348 shutil.rmtree(self._temp_dir)
349 self._temp_dir = None
350
351 while len(self._remove_files) > 0:
352 self._remove_if_exists(self._remove_files.pop())
353
354 exitcode = self.exitcode()
355 if (exitcode is not None and exitcode < 0
356 and not (self._user_killed and exitcode == -signal.SIGKILL)):
357 msg = 'qemu received signal %i; command: "%s"'
358 if self._qemu_full_args:
359 command = ' '.join(self._qemu_full_args)
360 else:
361 command = ''
362 LOG.warning(msg, -int(exitcode), command)
363
364 self._user_killed = False
365 self._launched = False
366
367 def launch(self) -> None:
368 """
369 Launch the VM and make sure we cleanup and expose the
370 command line/output in case of exception
371 """
372
373 if self._launched:
374 raise QEMUMachineError('VM already launched')
375
376 self._iolog = None
377 self._qemu_full_args = ()
378 try:
379 self._launch()
380 self._launched = True
381 except:
382 self._post_shutdown()
383
384 LOG.debug('Error launching VM')
385 if self._qemu_full_args:
386 LOG.debug('Command: %r', ' '.join(self._qemu_full_args))
387 if self._iolog:
388 LOG.debug('Output: %r', self._iolog)
389 raise
390
391 def _launch(self) -> None:
392 """
393 Launch the VM and establish a QMP connection
394 """
395 devnull = open(os.path.devnull, 'rb')
396 self._pre_launch()
397 self._qemu_full_args = tuple(
398 chain(self._wrapper,
399 [self._binary],
400 self._base_args,
401 self._args)
402 )
403 LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args))
404 self._popen = subprocess.Popen(self._qemu_full_args,
405 stdin=devnull,
406 stdout=self._qemu_log_file,
407 stderr=subprocess.STDOUT,
408 shell=False,
409 close_fds=False)
410 self._post_launch()
411
412 def _early_cleanup(self) -> None:
413 """
414 Perform any cleanup that needs to happen before the VM exits.
415
416 May be invoked by both soft and hard shutdown in failover scenarios.
417 Called additionally by _post_shutdown for comprehensive cleanup.
418 """
419 # If we keep the console socket open, we may deadlock waiting
420 # for QEMU to exit, while QEMU is waiting for the socket to
421 # become writeable.
422 if self._console_socket is not None:
423 self._console_socket.close()
424 self._console_socket = None
425
426 def _hard_shutdown(self) -> None:
427 """
428 Perform early cleanup, kill the VM, and wait for it to terminate.
429
430 :raise subprocess.Timeout: When timeout is exceeds 60 seconds
431 waiting for the QEMU process to terminate.
432 """
433 self._early_cleanup()
434 self._subp.kill()
435 self._subp.wait(timeout=60)
436
437 def _soft_shutdown(self, timeout: Optional[int],
438 has_quit: bool = False) -> None:
439 """
440 Perform early cleanup, attempt to gracefully shut down the VM, and wait
441 for it to terminate.
442
443 :param timeout: Timeout in seconds for graceful shutdown.
444 A value of None is an infinite wait.
445 :param has_quit: When True, don't attempt to issue 'quit' QMP command
446
447 :raise ConnectionReset: On QMP communication errors
448 :raise subprocess.TimeoutExpired: When timeout is exceeded waiting for
449 the QEMU process to terminate.
450 """
451 self._early_cleanup()
452
453 if self._qmp_connection:
454 if not has_quit:
455 # Might raise ConnectionReset
456 self._qmp.cmd('quit')
457
458 # May raise subprocess.TimeoutExpired
459 self._subp.wait(timeout=timeout)
460
461 def _do_shutdown(self, timeout: Optional[int],
462 has_quit: bool = False) -> None:
463 """
464 Attempt to shutdown the VM gracefully; fallback to a hard shutdown.
465
466 :param timeout: Timeout in seconds for graceful shutdown.
467 A value of None is an infinite wait.
468 :param has_quit: When True, don't attempt to issue 'quit' QMP command
469
470 :raise AbnormalShutdown: When the VM could not be shut down gracefully.
471 The inner exception will likely be ConnectionReset or
472 subprocess.TimeoutExpired. In rare cases, non-graceful termination
473 may result in its own exceptions, likely subprocess.TimeoutExpired.
474 """
475 try:
476 self._soft_shutdown(timeout, has_quit)
477 except Exception as exc:
478 self._hard_shutdown()
479 raise AbnormalShutdown("Could not perform graceful shutdown") \
480 from exc
481
482 def shutdown(self, has_quit: bool = False,
483 hard: bool = False,
484 timeout: Optional[int] = 30) -> None:
485 """
486 Terminate the VM (gracefully if possible) and perform cleanup.
487 Cleanup will always be performed.
488
489 If the VM has not yet been launched, or shutdown(), wait(), or kill()
490 have already been called, this method does nothing.
491
492 :param has_quit: When true, do not attempt to issue 'quit' QMP command.
493 :param hard: When true, do not attempt graceful shutdown, and
494 suppress the SIGKILL warning log message.
495 :param timeout: Optional timeout in seconds for graceful shutdown.
496 Default 30 seconds, A `None` value is an infinite wait.
497 """
498 if not self._launched:
499 return
500
501 try:
502 if hard:
503 self._user_killed = True
504 self._hard_shutdown()
505 else:
506 self._do_shutdown(timeout, has_quit)
507 finally:
508 self._post_shutdown()
509
510 def kill(self) -> None:
511 """
512 Terminate the VM forcefully, wait for it to exit, and perform cleanup.
513 """
514 self.shutdown(hard=True)
515
516 def wait(self, timeout: Optional[int] = 30) -> None:
517 """
518 Wait for the VM to power off and perform post-shutdown cleanup.
519
520 :param timeout: Optional timeout in seconds. Default 30 seconds.
521 A value of `None` is an infinite wait.
522 """
523 self.shutdown(has_quit=True, timeout=timeout)
524
525 def set_qmp_monitor(self, enabled: bool = True) -> None:
526 """
527 Set the QMP monitor.
528
529 @param enabled: if False, qmp monitor options will be removed from
530 the base arguments of the resulting QEMU command
531 line. Default is True.
532 @note: call this function before launch().
533 """
534 self._qmp_set = enabled
535
536 @property
537 def _qmp(self) -> qmp.QEMUMonitorProtocol:
538 if self._qmp_connection is None:
539 raise QEMUMachineError("Attempt to access QMP with no connection")
540 return self._qmp_connection
541
542 @classmethod
543 def _qmp_args(cls, _conv_keys: bool = True, **args: Any) -> Dict[str, Any]:
544 qmp_args = dict()
545 for key, value in args.items():
546 if _conv_keys:
547 qmp_args[key.replace('_', '-')] = value
548 else:
549 qmp_args[key] = value
550 return qmp_args
551
552 def qmp(self, cmd: str,
553 conv_keys: bool = True,
554 **args: Any) -> QMPMessage:
555 """
556 Invoke a QMP command and return the response dict
557 """
558 qmp_args = self._qmp_args(conv_keys, **args)
559 return self._qmp.cmd(cmd, args=qmp_args)
560
561 def command(self, cmd: str,
562 conv_keys: bool = True,
563 **args: Any) -> QMPReturnValue:
564 """
565 Invoke a QMP command.
566 On success return the response dict.
567 On failure raise an exception.
568 """
569 qmp_args = self._qmp_args(conv_keys, **args)
570 return self._qmp.command(cmd, **qmp_args)
571
572 def get_qmp_event(self, wait: bool = False) -> Optional[QMPMessage]:
573 """
574 Poll for one queued QMP events and return it
575 """
576 if self._events:
577 return self._events.pop(0)
578 return self._qmp.pull_event(wait=wait)
579
580 def get_qmp_events(self, wait: bool = False) -> List[QMPMessage]:
581 """
582 Poll for queued QMP events and return a list of dicts
583 """
584 events = self._qmp.get_events(wait=wait)
585 events.extend(self._events)
586 del self._events[:]
587 self._qmp.clear_events()
588 return events
589
590 @staticmethod
591 def event_match(event: Any, match: Optional[Any]) -> bool:
592 """
593 Check if an event matches optional match criteria.
594
595 The match criteria takes the form of a matching subdict. The event is
596 checked to be a superset of the subdict, recursively, with matching
597 values whenever the subdict values are not None.
598
599 This has a limitation that you cannot explicitly check for None values.
600
601 Examples, with the subdict queries on the left:
602 - None matches any object.
603 - {"foo": None} matches {"foo": {"bar": 1}}
604 - {"foo": None} matches {"foo": 5}
605 - {"foo": {"abc": None}} does not match {"foo": {"bar": 1}}
606 - {"foo": {"rab": 2}} matches {"foo": {"bar": 1, "rab": 2}}
607 """
608 if match is None:
609 return True
610
611 try:
612 for key in match:
613 if key in event:
614 if not QEMUMachine.event_match(event[key], match[key]):
615 return False
616 else:
617 return False
618 return True
619 except TypeError:
620 # either match or event wasn't iterable (not a dict)
621 return bool(match == event)
622
623 def event_wait(self, name: str,
624 timeout: float = 60.0,
625 match: Optional[QMPMessage] = None) -> Optional[QMPMessage]:
626 """
627 event_wait waits for and returns a named event from QMP with a timeout.
628
629 name: The event to wait for.
630 timeout: QEMUMonitorProtocol.pull_event timeout parameter.
631 match: Optional match criteria. See event_match for details.
632 """
633 return self.events_wait([(name, match)], timeout)
634
635 def events_wait(self,
636 events: Sequence[Tuple[str, Any]],
637 timeout: float = 60.0) -> Optional[QMPMessage]:
638 """
639 events_wait waits for and returns a single named event from QMP.
640 In the case of multiple qualifying events, this function returns the
641 first one.
642
643 :param events: A sequence of (name, match_criteria) tuples.
644 The match criteria are optional and may be None.
645 See event_match for details.
646 :param timeout: Optional timeout, in seconds.
647 See QEMUMonitorProtocol.pull_event.
648
649 :raise QMPTimeoutError: If timeout was non-zero and no matching events
650 were found.
651 :return: A QMP event matching the filter criteria.
652 If timeout was 0 and no event matched, None.
653 """
654 def _match(event: QMPMessage) -> bool:
655 for name, match in events:
656 if event['event'] == name and self.event_match(event, match):
657 return True
658 return False
659
660 event: Optional[QMPMessage]
661
662 # Search cached events
663 for event in self._events:
664 if _match(event):
665 self._events.remove(event)
666 return event
667
668 # Poll for new events
669 while True:
670 event = self._qmp.pull_event(wait=timeout)
671 if event is None:
672 # NB: None is only returned when timeout is false-ish.
673 # Timeouts raise QMPTimeoutError instead!
674 break
675 if _match(event):
676 return event
677 self._events.append(event)
678
679 return None
680
681 def get_log(self) -> Optional[str]:
682 """
683 After self.shutdown or failed qemu execution, this returns the output
684 of the qemu process.
685 """
686 return self._iolog
687
688 def add_args(self, *args: str) -> None:
689 """
690 Adds to the list of extra arguments to be given to the QEMU binary
691 """
692 self._args.extend(args)
693
694 def set_machine(self, machine_type: str) -> None:
695 """
696 Sets the machine type
697
698 If set, the machine type will be added to the base arguments
699 of the resulting QEMU command line.
700 """
701 self._machine = machine_type
702
703 def set_console(self,
704 device_type: Optional[str] = None,
705 console_index: int = 0) -> None:
706 """
707 Sets the device type for a console device
708
709 If set, the console device and a backing character device will
710 be added to the base arguments of the resulting QEMU command
711 line.
712
713 This is a convenience method that will either use the provided
714 device type, or default to a "-serial chardev:console" command
715 line argument.
716
717 The actual setting of command line arguments will be be done at
718 machine launch time, as it depends on the temporary directory
719 to be created.
720
721 @param device_type: the device type, such as "isa-serial". If
722 None is given (the default value) a "-serial
723 chardev:console" command line argument will
724 be used instead, resorting to the machine's
725 default device type.
726 @param console_index: the index of the console device to use.
727 If not zero, the command line will create
728 'index - 1' consoles and connect them to
729 the 'null' backing character device.
730 """
731 self._console_set = True
732 self._console_device_type = device_type
733 self._console_index = console_index
734
735 @property
736 def console_socket(self) -> socket.socket:
737 """
738 Returns a socket connected to the console
739 """
740 if self._console_socket is None:
741 self._console_socket = console_socket.ConsoleSocket(
742 self._console_address,
743 file=self._console_log_path,
744 drain=self._drain_console)
745 return self._console_socket