Update version for v7.1.0-rc3 release
[qemu.git] / migration / migration.c
1 /*
2 * QEMU live migration
3 *
4 * Copyright IBM, Corp. 2008
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
14 */
15
16 #include "qemu/osdep.h"
17 #include "qemu/cutils.h"
18 #include "qemu/error-report.h"
19 #include "qemu/main-loop.h"
20 #include "migration/blocker.h"
21 #include "exec.h"
22 #include "fd.h"
23 #include "socket.h"
24 #include "sysemu/runstate.h"
25 #include "sysemu/sysemu.h"
26 #include "sysemu/cpu-throttle.h"
27 #include "rdma.h"
28 #include "ram.h"
29 #include "migration/global_state.h"
30 #include "migration/misc.h"
31 #include "migration.h"
32 #include "savevm.h"
33 #include "qemu-file-channel.h"
34 #include "qemu-file.h"
35 #include "migration/vmstate.h"
36 #include "block/block.h"
37 #include "qapi/error.h"
38 #include "qapi/clone-visitor.h"
39 #include "qapi/qapi-visit-migration.h"
40 #include "qapi/qapi-visit-sockets.h"
41 #include "qapi/qapi-commands-migration.h"
42 #include "qapi/qapi-events-migration.h"
43 #include "qapi/qmp/qerror.h"
44 #include "qapi/qmp/qnull.h"
45 #include "qemu/rcu.h"
46 #include "block.h"
47 #include "postcopy-ram.h"
48 #include "qemu/thread.h"
49 #include "trace.h"
50 #include "exec/target_page.h"
51 #include "io/channel-buffer.h"
52 #include "migration/colo.h"
53 #include "hw/boards.h"
54 #include "hw/qdev-properties.h"
55 #include "hw/qdev-properties-system.h"
56 #include "monitor/monitor.h"
57 #include "net/announce.h"
58 #include "qemu/queue.h"
59 #include "multifd.h"
60 #include "qemu/yank.h"
61 #include "sysemu/cpus.h"
62 #include "yank_functions.h"
63 #include "sysemu/qtest.h"
64
65 #define MAX_THROTTLE (128 << 20) /* Migration transfer speed throttling */
66
67 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
68 * data. */
69 #define BUFFER_DELAY 100
70 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
71
72 /* Time in milliseconds we are allowed to stop the source,
73 * for sending the last part */
74 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
75
76 /* Maximum migrate downtime set to 2000 seconds */
77 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
78 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
79
80 /* Default compression thread count */
81 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
82 /* Default decompression thread count, usually decompression is at
83 * least 4 times as fast as compression.*/
84 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
85 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
86 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
87 /* Define default autoconverge cpu throttle migration parameters */
88 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
89 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
90 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
91 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
92
93 /* Migration XBZRLE default cache size */
94 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
95
96 /* The delay time (in ms) between two COLO checkpoints */
97 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
98 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
99 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
100 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
101 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
102 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
103 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
104
105 /* Background transfer rate for postcopy, 0 means unlimited, note
106 * that page requests can still exceed this limit.
107 */
108 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
109
110 /*
111 * Parameters for self_announce_delay giving a stream of RARP/ARP
112 * packets after migration.
113 */
114 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL 50
115 #define DEFAULT_MIGRATE_ANNOUNCE_MAX 550
116 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS 5
117 #define DEFAULT_MIGRATE_ANNOUNCE_STEP 100
118
119 static NotifierList migration_state_notifiers =
120 NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
121
122 /* Messages sent on the return path from destination to source */
123 enum mig_rp_message_type {
124 MIG_RP_MSG_INVALID = 0, /* Must be 0 */
125 MIG_RP_MSG_SHUT, /* sibling will not send any more RP messages */
126 MIG_RP_MSG_PONG, /* Response to a PING; data (seq: be32 ) */
127
128 MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
129 MIG_RP_MSG_REQ_PAGES, /* data (start: be64, len: be32) */
130 MIG_RP_MSG_RECV_BITMAP, /* send recved_bitmap back to source */
131 MIG_RP_MSG_RESUME_ACK, /* tell source that we are ready to resume */
132
133 MIG_RP_MSG_MAX
134 };
135
136 /* Migration capabilities set */
137 struct MigrateCapsSet {
138 int size; /* Capability set size */
139 MigrationCapability caps[]; /* Variadic array of capabilities */
140 };
141 typedef struct MigrateCapsSet MigrateCapsSet;
142
143 /* Define and initialize MigrateCapsSet */
144 #define INITIALIZE_MIGRATE_CAPS_SET(_name, ...) \
145 MigrateCapsSet _name = { \
146 .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \
147 .caps = { __VA_ARGS__ } \
148 }
149
150 /* Background-snapshot compatibility check list */
151 static const
152 INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
153 MIGRATION_CAPABILITY_POSTCOPY_RAM,
154 MIGRATION_CAPABILITY_DIRTY_BITMAPS,
155 MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME,
156 MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE,
157 MIGRATION_CAPABILITY_RETURN_PATH,
158 MIGRATION_CAPABILITY_MULTIFD,
159 MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
160 MIGRATION_CAPABILITY_AUTO_CONVERGE,
161 MIGRATION_CAPABILITY_RELEASE_RAM,
162 MIGRATION_CAPABILITY_RDMA_PIN_ALL,
163 MIGRATION_CAPABILITY_COMPRESS,
164 MIGRATION_CAPABILITY_XBZRLE,
165 MIGRATION_CAPABILITY_X_COLO,
166 MIGRATION_CAPABILITY_VALIDATE_UUID);
167
168 /* When we add fault tolerance, we could have several
169 migrations at once. For now we don't need to add
170 dynamic creation of migration */
171
172 static MigrationState *current_migration;
173 static MigrationIncomingState *current_incoming;
174
175 static GSList *migration_blockers;
176
177 static bool migration_object_check(MigrationState *ms, Error **errp);
178 static int migration_maybe_pause(MigrationState *s,
179 int *current_active_state,
180 int new_state);
181 static void migrate_fd_cancel(MigrationState *s);
182
183 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
184 {
185 uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
186
187 return (a > b) - (a < b);
188 }
189
190 void migration_object_init(void)
191 {
192 /* This can only be called once. */
193 assert(!current_migration);
194 current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
195
196 /*
197 * Init the migrate incoming object as well no matter whether
198 * we'll use it or not.
199 */
200 assert(!current_incoming);
201 current_incoming = g_new0(MigrationIncomingState, 1);
202 current_incoming->state = MIGRATION_STATUS_NONE;
203 current_incoming->postcopy_remote_fds =
204 g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
205 qemu_mutex_init(&current_incoming->rp_mutex);
206 qemu_event_init(&current_incoming->main_thread_load_event, false);
207 qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
208 qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
209 qemu_mutex_init(&current_incoming->page_request_mutex);
210 current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
211
212 migration_object_check(current_migration, &error_fatal);
213
214 blk_mig_init();
215 ram_mig_init();
216 dirty_bitmap_mig_init();
217 }
218
219 void migration_cancel(const Error *error)
220 {
221 if (error) {
222 migrate_set_error(current_migration, error);
223 }
224 migrate_fd_cancel(current_migration);
225 }
226
227 void migration_shutdown(void)
228 {
229 /*
230 * When the QEMU main thread exit, the COLO thread
231 * may wait a semaphore. So, we should wakeup the
232 * COLO thread before migration shutdown.
233 */
234 colo_shutdown();
235 /*
236 * Cancel the current migration - that will (eventually)
237 * stop the migration using this structure
238 */
239 migration_cancel(NULL);
240 object_unref(OBJECT(current_migration));
241
242 /*
243 * Cancel outgoing migration of dirty bitmaps. It should
244 * at least unref used block nodes.
245 */
246 dirty_bitmap_mig_cancel_outgoing();
247
248 /*
249 * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
250 * are non-critical data, and their loss never considered as
251 * something serious.
252 */
253 dirty_bitmap_mig_cancel_incoming();
254 }
255
256 /* For outgoing */
257 MigrationState *migrate_get_current(void)
258 {
259 /* This can only be called after the object created. */
260 assert(current_migration);
261 return current_migration;
262 }
263
264 MigrationIncomingState *migration_incoming_get_current(void)
265 {
266 assert(current_incoming);
267 return current_incoming;
268 }
269
270 void migration_incoming_state_destroy(void)
271 {
272 struct MigrationIncomingState *mis = migration_incoming_get_current();
273
274 if (mis->to_src_file) {
275 /* Tell source that we are done */
276 migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
277 qemu_fclose(mis->to_src_file);
278 mis->to_src_file = NULL;
279 }
280
281 if (mis->from_src_file) {
282 migration_ioc_unregister_yank_from_file(mis->from_src_file);
283 qemu_fclose(mis->from_src_file);
284 mis->from_src_file = NULL;
285 }
286 if (mis->postcopy_remote_fds) {
287 g_array_free(mis->postcopy_remote_fds, TRUE);
288 mis->postcopy_remote_fds = NULL;
289 }
290 if (mis->transport_cleanup) {
291 mis->transport_cleanup(mis->transport_data);
292 }
293
294 qemu_event_reset(&mis->main_thread_load_event);
295
296 if (mis->page_requested) {
297 g_tree_destroy(mis->page_requested);
298 mis->page_requested = NULL;
299 }
300
301 if (mis->socket_address_list) {
302 qapi_free_SocketAddressList(mis->socket_address_list);
303 mis->socket_address_list = NULL;
304 }
305
306 yank_unregister_instance(MIGRATION_YANK_INSTANCE);
307 }
308
309 static void migrate_generate_event(int new_state)
310 {
311 if (migrate_use_events()) {
312 qapi_event_send_migration(new_state);
313 }
314 }
315
316 static bool migrate_late_block_activate(void)
317 {
318 MigrationState *s;
319
320 s = migrate_get_current();
321
322 return s->enabled_capabilities[
323 MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
324 }
325
326 /*
327 * Send a message on the return channel back to the source
328 * of the migration.
329 */
330 static int migrate_send_rp_message(MigrationIncomingState *mis,
331 enum mig_rp_message_type message_type,
332 uint16_t len, void *data)
333 {
334 int ret = 0;
335
336 trace_migrate_send_rp_message((int)message_type, len);
337 QEMU_LOCK_GUARD(&mis->rp_mutex);
338
339 /*
340 * It's possible that the file handle got lost due to network
341 * failures.
342 */
343 if (!mis->to_src_file) {
344 ret = -EIO;
345 return ret;
346 }
347
348 qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
349 qemu_put_be16(mis->to_src_file, len);
350 qemu_put_buffer(mis->to_src_file, data, len);
351 qemu_fflush(mis->to_src_file);
352
353 /* It's possible that qemu file got error during sending */
354 ret = qemu_file_get_error(mis->to_src_file);
355
356 return ret;
357 }
358
359 /* Request one page from the source VM at the given start address.
360 * rb: the RAMBlock to request the page in
361 * Start: Address offset within the RB
362 * Len: Length in bytes required - must be a multiple of pagesize
363 */
364 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
365 RAMBlock *rb, ram_addr_t start)
366 {
367 uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
368 size_t msglen = 12; /* start + len */
369 size_t len = qemu_ram_pagesize(rb);
370 enum mig_rp_message_type msg_type;
371 const char *rbname;
372 int rbname_len;
373
374 *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
375 *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
376
377 /*
378 * We maintain the last ramblock that we requested for page. Note that we
379 * don't need locking because this function will only be called within the
380 * postcopy ram fault thread.
381 */
382 if (rb != mis->last_rb) {
383 mis->last_rb = rb;
384
385 rbname = qemu_ram_get_idstr(rb);
386 rbname_len = strlen(rbname);
387
388 assert(rbname_len < 256);
389
390 bufc[msglen++] = rbname_len;
391 memcpy(bufc + msglen, rbname, rbname_len);
392 msglen += rbname_len;
393 msg_type = MIG_RP_MSG_REQ_PAGES_ID;
394 } else {
395 msg_type = MIG_RP_MSG_REQ_PAGES;
396 }
397
398 return migrate_send_rp_message(mis, msg_type, msglen, bufc);
399 }
400
401 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
402 RAMBlock *rb, ram_addr_t start, uint64_t haddr)
403 {
404 void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
405 bool received = false;
406
407 WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
408 received = ramblock_recv_bitmap_test_byte_offset(rb, start);
409 if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
410 /*
411 * The page has not been received, and it's not yet in the page
412 * request list. Queue it. Set the value of element to 1, so that
413 * things like g_tree_lookup() will return TRUE (1) when found.
414 */
415 g_tree_insert(mis->page_requested, aligned, (gpointer)1);
416 mis->page_requested_count++;
417 trace_postcopy_page_req_add(aligned, mis->page_requested_count);
418 }
419 }
420
421 /*
422 * If the page is there, skip sending the message. We don't even need the
423 * lock because as long as the page arrived, it'll be there forever.
424 */
425 if (received) {
426 return 0;
427 }
428
429 return migrate_send_rp_message_req_pages(mis, rb, start);
430 }
431
432 static bool migration_colo_enabled;
433 bool migration_incoming_colo_enabled(void)
434 {
435 return migration_colo_enabled;
436 }
437
438 void migration_incoming_disable_colo(void)
439 {
440 ram_block_discard_disable(false);
441 migration_colo_enabled = false;
442 }
443
444 int migration_incoming_enable_colo(void)
445 {
446 if (ram_block_discard_disable(true)) {
447 error_report("COLO: cannot disable RAM discard");
448 return -EBUSY;
449 }
450 migration_colo_enabled = true;
451 return 0;
452 }
453
454 void migrate_add_address(SocketAddress *address)
455 {
456 MigrationIncomingState *mis = migration_incoming_get_current();
457
458 QAPI_LIST_PREPEND(mis->socket_address_list,
459 QAPI_CLONE(SocketAddress, address));
460 }
461
462 static void qemu_start_incoming_migration(const char *uri, Error **errp)
463 {
464 const char *p = NULL;
465
466 migrate_protocol_allow_multifd(false); /* reset it anyway */
467 qapi_event_send_migration(MIGRATION_STATUS_SETUP);
468 if (strstart(uri, "tcp:", &p) ||
469 strstart(uri, "unix:", NULL) ||
470 strstart(uri, "vsock:", NULL)) {
471 migrate_protocol_allow_multifd(true);
472 socket_start_incoming_migration(p ? p : uri, errp);
473 #ifdef CONFIG_RDMA
474 } else if (strstart(uri, "rdma:", &p)) {
475 rdma_start_incoming_migration(p, errp);
476 #endif
477 } else if (strstart(uri, "exec:", &p)) {
478 exec_start_incoming_migration(p, errp);
479 } else if (strstart(uri, "fd:", &p)) {
480 fd_start_incoming_migration(p, errp);
481 } else {
482 error_setg(errp, "unknown migration protocol: %s", uri);
483 }
484 }
485
486 static void process_incoming_migration_bh(void *opaque)
487 {
488 Error *local_err = NULL;
489 MigrationIncomingState *mis = opaque;
490
491 /* If capability late_block_activate is set:
492 * Only fire up the block code now if we're going to restart the
493 * VM, else 'cont' will do it.
494 * This causes file locking to happen; so we don't want it to happen
495 * unless we really are starting the VM.
496 */
497 if (!migrate_late_block_activate() ||
498 (autostart && (!global_state_received() ||
499 global_state_get_runstate() == RUN_STATE_RUNNING))) {
500 /* Make sure all file formats flush their mutable metadata.
501 * If we get an error here, just don't restart the VM yet. */
502 bdrv_invalidate_cache_all(&local_err);
503 if (local_err) {
504 error_report_err(local_err);
505 local_err = NULL;
506 autostart = false;
507 }
508 }
509
510 /*
511 * This must happen after all error conditions are dealt with and
512 * we're sure the VM is going to be running on this host.
513 */
514 qemu_announce_self(&mis->announce_timer, migrate_announce_params());
515
516 if (multifd_load_cleanup(&local_err) != 0) {
517 error_report_err(local_err);
518 autostart = false;
519 }
520 /* If global state section was not received or we are in running
521 state, we need to obey autostart. Any other state is set with
522 runstate_set. */
523
524 dirty_bitmap_mig_before_vm_start();
525
526 if (!global_state_received() ||
527 global_state_get_runstate() == RUN_STATE_RUNNING) {
528 if (autostart) {
529 vm_start();
530 } else {
531 runstate_set(RUN_STATE_PAUSED);
532 }
533 } else if (migration_incoming_colo_enabled()) {
534 migration_incoming_disable_colo();
535 vm_start();
536 } else {
537 runstate_set(global_state_get_runstate());
538 }
539 /*
540 * This must happen after any state changes since as soon as an external
541 * observer sees this event they might start to prod at the VM assuming
542 * it's ready to use.
543 */
544 migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
545 MIGRATION_STATUS_COMPLETED);
546 qemu_bh_delete(mis->bh);
547 migration_incoming_state_destroy();
548 }
549
550 static void process_incoming_migration_co(void *opaque)
551 {
552 MigrationIncomingState *mis = migration_incoming_get_current();
553 PostcopyState ps;
554 int ret;
555 Error *local_err = NULL;
556
557 assert(mis->from_src_file);
558 mis->migration_incoming_co = qemu_coroutine_self();
559 mis->largest_page_size = qemu_ram_pagesize_largest();
560 postcopy_state_set(POSTCOPY_INCOMING_NONE);
561 migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
562 MIGRATION_STATUS_ACTIVE);
563 ret = qemu_loadvm_state(mis->from_src_file);
564
565 ps = postcopy_state_get();
566 trace_process_incoming_migration_co_end(ret, ps);
567 if (ps != POSTCOPY_INCOMING_NONE) {
568 if (ps == POSTCOPY_INCOMING_ADVISE) {
569 /*
570 * Where a migration had postcopy enabled (and thus went to advise)
571 * but managed to complete within the precopy period, we can use
572 * the normal exit.
573 */
574 postcopy_ram_incoming_cleanup(mis);
575 } else if (ret >= 0) {
576 /*
577 * Postcopy was started, cleanup should happen at the end of the
578 * postcopy thread.
579 */
580 trace_process_incoming_migration_co_postcopy_end_main();
581 return;
582 }
583 /* Else if something went wrong then just fall out of the normal exit */
584 }
585
586 /* we get COLO info, and know if we are in COLO mode */
587 if (!ret && migration_incoming_colo_enabled()) {
588 /* Make sure all file formats flush their mutable metadata */
589 bdrv_invalidate_cache_all(&local_err);
590 if (local_err) {
591 error_report_err(local_err);
592 goto fail;
593 }
594
595 qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
596 colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
597 mis->have_colo_incoming_thread = true;
598 qemu_coroutine_yield();
599
600 qemu_mutex_unlock_iothread();
601 /* Wait checkpoint incoming thread exit before free resource */
602 qemu_thread_join(&mis->colo_incoming_thread);
603 qemu_mutex_lock_iothread();
604 /* We hold the global iothread lock, so it is safe here */
605 colo_release_ram_cache();
606 }
607
608 if (ret < 0) {
609 error_report("load of migration failed: %s", strerror(-ret));
610 goto fail;
611 }
612 mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
613 qemu_bh_schedule(mis->bh);
614 mis->migration_incoming_co = NULL;
615 return;
616 fail:
617 local_err = NULL;
618 migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
619 MIGRATION_STATUS_FAILED);
620 qemu_fclose(mis->from_src_file);
621 if (multifd_load_cleanup(&local_err) != 0) {
622 error_report_err(local_err);
623 }
624 exit(EXIT_FAILURE);
625 }
626
627 /**
628 * migration_incoming_setup: Setup incoming migration
629 * @f: file for main migration channel
630 * @errp: where to put errors
631 *
632 * Returns: %true on success, %false on error.
633 */
634 static bool migration_incoming_setup(QEMUFile *f, Error **errp)
635 {
636 MigrationIncomingState *mis = migration_incoming_get_current();
637
638 if (multifd_load_setup(errp) != 0) {
639 return false;
640 }
641
642 if (!mis->from_src_file) {
643 mis->from_src_file = f;
644 }
645 qemu_file_set_blocking(f, false);
646 return true;
647 }
648
649 void migration_incoming_process(void)
650 {
651 Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
652 qemu_coroutine_enter(co);
653 }
654
655 /* Returns true if recovered from a paused migration, otherwise false */
656 static bool postcopy_try_recover(QEMUFile *f)
657 {
658 MigrationIncomingState *mis = migration_incoming_get_current();
659
660 if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
661 /* Resumed from a paused postcopy migration */
662
663 mis->from_src_file = f;
664 /* Postcopy has standalone thread to do vm load */
665 qemu_file_set_blocking(f, true);
666
667 /* Re-configure the return path */
668 mis->to_src_file = qemu_file_get_return_path(f);
669
670 migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
671 MIGRATION_STATUS_POSTCOPY_RECOVER);
672
673 /*
674 * Here, we only wake up the main loading thread (while the
675 * fault thread will still be waiting), so that we can receive
676 * commands from source now, and answer it if needed. The
677 * fault thread will be woken up afterwards until we are sure
678 * that source is ready to reply to page requests.
679 */
680 qemu_sem_post(&mis->postcopy_pause_sem_dst);
681 return true;
682 }
683
684 return false;
685 }
686
687 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
688 {
689 if (postcopy_try_recover(f)) {
690 return;
691 }
692
693 if (!migration_incoming_setup(f, errp)) {
694 return;
695 }
696 migration_incoming_process();
697 }
698
699 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
700 {
701 MigrationIncomingState *mis = migration_incoming_get_current();
702 Error *local_err = NULL;
703 bool start_migration;
704
705 if (!mis->from_src_file) {
706 /* The first connection (multifd may have multiple) */
707 QEMUFile *f = qemu_fopen_channel_input(ioc);
708
709 /* If it's a recovery, we're done */
710 if (postcopy_try_recover(f)) {
711 return;
712 }
713
714 if (!migration_incoming_setup(f, errp)) {
715 return;
716 }
717
718 /*
719 * Common migration only needs one channel, so we can start
720 * right now. Multifd needs more than one channel, we wait.
721 */
722 start_migration = !migrate_use_multifd();
723 } else {
724 /* Multiple connections */
725 assert(migrate_use_multifd());
726 start_migration = multifd_recv_new_channel(ioc, &local_err);
727 if (local_err) {
728 error_propagate(errp, local_err);
729 return;
730 }
731 }
732
733 if (start_migration) {
734 migration_incoming_process();
735 }
736 }
737
738 /**
739 * @migration_has_all_channels: We have received all channels that we need
740 *
741 * Returns true when we have got connections to all the channels that
742 * we need for migration.
743 */
744 bool migration_has_all_channels(void)
745 {
746 MigrationIncomingState *mis = migration_incoming_get_current();
747 bool all_channels;
748
749 all_channels = multifd_recv_all_channels_created();
750
751 return all_channels && mis->from_src_file != NULL;
752 }
753
754 /*
755 * Send a 'SHUT' message on the return channel with the given value
756 * to indicate that we've finished with the RP. Non-0 value indicates
757 * error.
758 */
759 void migrate_send_rp_shut(MigrationIncomingState *mis,
760 uint32_t value)
761 {
762 uint32_t buf;
763
764 buf = cpu_to_be32(value);
765 migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
766 }
767
768 /*
769 * Send a 'PONG' message on the return channel with the given value
770 * (normally in response to a 'PING')
771 */
772 void migrate_send_rp_pong(MigrationIncomingState *mis,
773 uint32_t value)
774 {
775 uint32_t buf;
776
777 buf = cpu_to_be32(value);
778 migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
779 }
780
781 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
782 char *block_name)
783 {
784 char buf[512];
785 int len;
786 int64_t res;
787
788 /*
789 * First, we send the header part. It contains only the len of
790 * idstr, and the idstr itself.
791 */
792 len = strlen(block_name);
793 buf[0] = len;
794 memcpy(buf + 1, block_name, len);
795
796 if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
797 error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
798 __func__);
799 return;
800 }
801
802 migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
803
804 /*
805 * Next, we dump the received bitmap to the stream.
806 *
807 * TODO: currently we are safe since we are the only one that is
808 * using the to_src_file handle (fault thread is still paused),
809 * and it's ok even not taking the mutex. However the best way is
810 * to take the lock before sending the message header, and release
811 * the lock after sending the bitmap.
812 */
813 qemu_mutex_lock(&mis->rp_mutex);
814 res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
815 qemu_mutex_unlock(&mis->rp_mutex);
816
817 trace_migrate_send_rp_recv_bitmap(block_name, res);
818 }
819
820 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
821 {
822 uint32_t buf;
823
824 buf = cpu_to_be32(value);
825 migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
826 }
827
828 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
829 {
830 MigrationCapabilityStatusList *head = NULL, **tail = &head;
831 MigrationCapabilityStatus *caps;
832 MigrationState *s = migrate_get_current();
833 int i;
834
835 for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
836 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
837 if (i == MIGRATION_CAPABILITY_BLOCK) {
838 continue;
839 }
840 #endif
841 caps = g_malloc0(sizeof(*caps));
842 caps->capability = i;
843 caps->state = s->enabled_capabilities[i];
844 QAPI_LIST_APPEND(tail, caps);
845 }
846
847 return head;
848 }
849
850 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
851 {
852 MigrationParameters *params;
853 MigrationState *s = migrate_get_current();
854
855 /* TODO use QAPI_CLONE() instead of duplicating it inline */
856 params = g_malloc0(sizeof(*params));
857 params->has_compress_level = true;
858 params->compress_level = s->parameters.compress_level;
859 params->has_compress_threads = true;
860 params->compress_threads = s->parameters.compress_threads;
861 params->has_compress_wait_thread = true;
862 params->compress_wait_thread = s->parameters.compress_wait_thread;
863 params->has_decompress_threads = true;
864 params->decompress_threads = s->parameters.decompress_threads;
865 params->has_throttle_trigger_threshold = true;
866 params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
867 params->has_cpu_throttle_initial = true;
868 params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
869 params->has_cpu_throttle_increment = true;
870 params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
871 params->has_cpu_throttle_tailslow = true;
872 params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
873 params->has_tls_creds = true;
874 params->tls_creds = g_strdup(s->parameters.tls_creds);
875 params->has_tls_hostname = true;
876 params->tls_hostname = g_strdup(s->parameters.tls_hostname);
877 params->has_tls_authz = true;
878 params->tls_authz = g_strdup(s->parameters.tls_authz ?
879 s->parameters.tls_authz : "");
880 params->has_max_bandwidth = true;
881 params->max_bandwidth = s->parameters.max_bandwidth;
882 params->has_downtime_limit = true;
883 params->downtime_limit = s->parameters.downtime_limit;
884 params->has_x_checkpoint_delay = true;
885 params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
886 params->has_block_incremental = true;
887 params->block_incremental = s->parameters.block_incremental;
888 params->has_multifd_channels = true;
889 params->multifd_channels = s->parameters.multifd_channels;
890 params->has_multifd_compression = true;
891 params->multifd_compression = s->parameters.multifd_compression;
892 params->has_multifd_zlib_level = true;
893 params->multifd_zlib_level = s->parameters.multifd_zlib_level;
894 params->has_multifd_zstd_level = true;
895 params->multifd_zstd_level = s->parameters.multifd_zstd_level;
896 params->has_xbzrle_cache_size = true;
897 params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
898 params->has_max_postcopy_bandwidth = true;
899 params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
900 params->has_max_cpu_throttle = true;
901 params->max_cpu_throttle = s->parameters.max_cpu_throttle;
902 params->has_announce_initial = true;
903 params->announce_initial = s->parameters.announce_initial;
904 params->has_announce_max = true;
905 params->announce_max = s->parameters.announce_max;
906 params->has_announce_rounds = true;
907 params->announce_rounds = s->parameters.announce_rounds;
908 params->has_announce_step = true;
909 params->announce_step = s->parameters.announce_step;
910
911 if (s->parameters.has_block_bitmap_mapping) {
912 params->has_block_bitmap_mapping = true;
913 params->block_bitmap_mapping =
914 QAPI_CLONE(BitmapMigrationNodeAliasList,
915 s->parameters.block_bitmap_mapping);
916 }
917
918 return params;
919 }
920
921 AnnounceParameters *migrate_announce_params(void)
922 {
923 static AnnounceParameters ap;
924
925 MigrationState *s = migrate_get_current();
926
927 ap.initial = s->parameters.announce_initial;
928 ap.max = s->parameters.announce_max;
929 ap.rounds = s->parameters.announce_rounds;
930 ap.step = s->parameters.announce_step;
931
932 return &ap;
933 }
934
935 /*
936 * Return true if we're already in the middle of a migration
937 * (i.e. any of the active or setup states)
938 */
939 bool migration_is_setup_or_active(int state)
940 {
941 switch (state) {
942 case MIGRATION_STATUS_ACTIVE:
943 case MIGRATION_STATUS_POSTCOPY_ACTIVE:
944 case MIGRATION_STATUS_POSTCOPY_PAUSED:
945 case MIGRATION_STATUS_POSTCOPY_RECOVER:
946 case MIGRATION_STATUS_SETUP:
947 case MIGRATION_STATUS_PRE_SWITCHOVER:
948 case MIGRATION_STATUS_DEVICE:
949 case MIGRATION_STATUS_WAIT_UNPLUG:
950 case MIGRATION_STATUS_COLO:
951 return true;
952
953 default:
954 return false;
955
956 }
957 }
958
959 bool migration_is_running(int state)
960 {
961 switch (state) {
962 case MIGRATION_STATUS_ACTIVE:
963 case MIGRATION_STATUS_POSTCOPY_ACTIVE:
964 case MIGRATION_STATUS_POSTCOPY_PAUSED:
965 case MIGRATION_STATUS_POSTCOPY_RECOVER:
966 case MIGRATION_STATUS_SETUP:
967 case MIGRATION_STATUS_PRE_SWITCHOVER:
968 case MIGRATION_STATUS_DEVICE:
969 case MIGRATION_STATUS_WAIT_UNPLUG:
970 case MIGRATION_STATUS_CANCELLING:
971 return true;
972
973 default:
974 return false;
975
976 }
977 }
978
979 static void populate_time_info(MigrationInfo *info, MigrationState *s)
980 {
981 info->has_status = true;
982 info->has_setup_time = true;
983 info->setup_time = s->setup_time;
984 if (s->state == MIGRATION_STATUS_COMPLETED) {
985 info->has_total_time = true;
986 info->total_time = s->total_time;
987 info->has_downtime = true;
988 info->downtime = s->downtime;
989 } else {
990 info->has_total_time = true;
991 info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
992 s->start_time;
993 info->has_expected_downtime = true;
994 info->expected_downtime = s->expected_downtime;
995 }
996 }
997
998 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
999 {
1000 size_t page_size = qemu_target_page_size();
1001
1002 info->has_ram = true;
1003 info->ram = g_malloc0(sizeof(*info->ram));
1004 info->ram->transferred = ram_counters.transferred;
1005 info->ram->total = ram_bytes_total();
1006 info->ram->duplicate = ram_counters.duplicate;
1007 /* legacy value. It is not used anymore */
1008 info->ram->skipped = 0;
1009 info->ram->normal = ram_counters.normal;
1010 info->ram->normal_bytes = ram_counters.normal * page_size;
1011 info->ram->mbps = s->mbps;
1012 info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
1013 info->ram->postcopy_requests = ram_counters.postcopy_requests;
1014 info->ram->page_size = page_size;
1015 info->ram->multifd_bytes = ram_counters.multifd_bytes;
1016 info->ram->pages_per_second = s->pages_per_second;
1017
1018 if (migrate_use_xbzrle()) {
1019 info->has_xbzrle_cache = true;
1020 info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1021 info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1022 info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1023 info->xbzrle_cache->pages = xbzrle_counters.pages;
1024 info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1025 info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1026 info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1027 info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1028 }
1029
1030 if (migrate_use_compression()) {
1031 info->has_compression = true;
1032 info->compression = g_malloc0(sizeof(*info->compression));
1033 info->compression->pages = compression_counters.pages;
1034 info->compression->busy = compression_counters.busy;
1035 info->compression->busy_rate = compression_counters.busy_rate;
1036 info->compression->compressed_size =
1037 compression_counters.compressed_size;
1038 info->compression->compression_rate =
1039 compression_counters.compression_rate;
1040 }
1041
1042 if (cpu_throttle_active()) {
1043 info->has_cpu_throttle_percentage = true;
1044 info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1045 }
1046
1047 if (s->state != MIGRATION_STATUS_COMPLETED) {
1048 info->ram->remaining = ram_bytes_remaining();
1049 info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1050 }
1051 }
1052
1053 static void populate_disk_info(MigrationInfo *info)
1054 {
1055 if (blk_mig_active()) {
1056 info->has_disk = true;
1057 info->disk = g_malloc0(sizeof(*info->disk));
1058 info->disk->transferred = blk_mig_bytes_transferred();
1059 info->disk->remaining = blk_mig_bytes_remaining();
1060 info->disk->total = blk_mig_bytes_total();
1061 }
1062 }
1063
1064 static void fill_source_migration_info(MigrationInfo *info)
1065 {
1066 MigrationState *s = migrate_get_current();
1067 GSList *cur_blocker = migration_blockers;
1068
1069 info->blocked_reasons = NULL;
1070
1071 /*
1072 * There are two types of reasons a migration might be blocked;
1073 * a) devices marked in VMState as non-migratable, and
1074 * b) Explicit migration blockers
1075 * We need to add both of them here.
1076 */
1077 qemu_savevm_non_migratable_list(&info->blocked_reasons);
1078
1079 while (cur_blocker) {
1080 QAPI_LIST_PREPEND(info->blocked_reasons,
1081 g_strdup(error_get_pretty(cur_blocker->data)));
1082 cur_blocker = g_slist_next(cur_blocker);
1083 }
1084 info->has_blocked_reasons = info->blocked_reasons != NULL;
1085
1086 switch (s->state) {
1087 case MIGRATION_STATUS_NONE:
1088 /* no migration has happened ever */
1089 /* do not overwrite destination migration status */
1090 return;
1091 case MIGRATION_STATUS_SETUP:
1092 info->has_status = true;
1093 info->has_total_time = false;
1094 break;
1095 case MIGRATION_STATUS_ACTIVE:
1096 case MIGRATION_STATUS_CANCELLING:
1097 case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1098 case MIGRATION_STATUS_PRE_SWITCHOVER:
1099 case MIGRATION_STATUS_DEVICE:
1100 case MIGRATION_STATUS_POSTCOPY_PAUSED:
1101 case MIGRATION_STATUS_POSTCOPY_RECOVER:
1102 /* TODO add some postcopy stats */
1103 populate_time_info(info, s);
1104 populate_ram_info(info, s);
1105 populate_disk_info(info);
1106 populate_vfio_info(info);
1107 break;
1108 case MIGRATION_STATUS_COLO:
1109 info->has_status = true;
1110 /* TODO: display COLO specific information (checkpoint info etc.) */
1111 break;
1112 case MIGRATION_STATUS_COMPLETED:
1113 populate_time_info(info, s);
1114 populate_ram_info(info, s);
1115 populate_vfio_info(info);
1116 break;
1117 case MIGRATION_STATUS_FAILED:
1118 info->has_status = true;
1119 if (s->error) {
1120 info->has_error_desc = true;
1121 info->error_desc = g_strdup(error_get_pretty(s->error));
1122 }
1123 break;
1124 case MIGRATION_STATUS_CANCELLED:
1125 info->has_status = true;
1126 break;
1127 case MIGRATION_STATUS_WAIT_UNPLUG:
1128 info->has_status = true;
1129 break;
1130 }
1131 info->status = s->state;
1132 }
1133
1134 typedef enum WriteTrackingSupport {
1135 WT_SUPPORT_UNKNOWN = 0,
1136 WT_SUPPORT_ABSENT,
1137 WT_SUPPORT_AVAILABLE,
1138 WT_SUPPORT_COMPATIBLE
1139 } WriteTrackingSupport;
1140
1141 static
1142 WriteTrackingSupport migrate_query_write_tracking(void)
1143 {
1144 /* Check if kernel supports required UFFD features */
1145 if (!ram_write_tracking_available()) {
1146 return WT_SUPPORT_ABSENT;
1147 }
1148 /*
1149 * Check if current memory configuration is
1150 * compatible with required UFFD features.
1151 */
1152 if (!ram_write_tracking_compatible()) {
1153 return WT_SUPPORT_AVAILABLE;
1154 }
1155
1156 return WT_SUPPORT_COMPATIBLE;
1157 }
1158
1159 /**
1160 * @migration_caps_check - check capability validity
1161 *
1162 * @cap_list: old capability list, array of bool
1163 * @params: new capabilities to be applied soon
1164 * @errp: set *errp if the check failed, with reason
1165 *
1166 * Returns true if check passed, otherwise false.
1167 */
1168 static bool migrate_caps_check(bool *cap_list,
1169 MigrationCapabilityStatusList *params,
1170 Error **errp)
1171 {
1172 MigrationCapabilityStatusList *cap;
1173 bool old_postcopy_cap;
1174 MigrationIncomingState *mis = migration_incoming_get_current();
1175
1176 old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1177
1178 for (cap = params; cap; cap = cap->next) {
1179 cap_list[cap->value->capability] = cap->value->state;
1180 }
1181
1182 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1183 if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1184 error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1185 "block migration");
1186 error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1187 return false;
1188 }
1189 #endif
1190
1191 #ifndef CONFIG_REPLICATION
1192 if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1193 error_setg(errp, "QEMU compiled without replication module"
1194 " can't enable COLO");
1195 error_append_hint(errp, "Please enable replication before COLO.\n");
1196 return false;
1197 }
1198 #endif
1199
1200 if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1201 /* This check is reasonably expensive, so only when it's being
1202 * set the first time, also it's only the destination that needs
1203 * special support.
1204 */
1205 if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1206 !postcopy_ram_supported_by_host(mis)) {
1207 /* postcopy_ram_supported_by_host will have emitted a more
1208 * detailed message
1209 */
1210 error_setg(errp, "Postcopy is not supported");
1211 return false;
1212 }
1213
1214 if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1215 error_setg(errp, "Postcopy is not compatible with ignore-shared");
1216 return false;
1217 }
1218 }
1219
1220 if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) {
1221 WriteTrackingSupport wt_support;
1222 int idx;
1223 /*
1224 * Check if 'background-snapshot' capability is supported by
1225 * host kernel and compatible with guest memory configuration.
1226 */
1227 wt_support = migrate_query_write_tracking();
1228 if (wt_support < WT_SUPPORT_AVAILABLE) {
1229 error_setg(errp, "Background-snapshot is not supported by host kernel");
1230 return false;
1231 }
1232 if (wt_support < WT_SUPPORT_COMPATIBLE) {
1233 error_setg(errp, "Background-snapshot is not compatible "
1234 "with guest memory configuration");
1235 return false;
1236 }
1237
1238 /*
1239 * Check if there are any migration capabilities
1240 * incompatible with 'background-snapshot'.
1241 */
1242 for (idx = 0; idx < check_caps_background_snapshot.size; idx++) {
1243 int incomp_cap = check_caps_background_snapshot.caps[idx];
1244 if (cap_list[incomp_cap]) {
1245 error_setg(errp,
1246 "Background-snapshot is not compatible with %s",
1247 MigrationCapability_str(incomp_cap));
1248 return false;
1249 }
1250 }
1251 }
1252
1253 /* incoming side only */
1254 if (runstate_check(RUN_STATE_INMIGRATE) &&
1255 !migrate_multifd_is_allowed() &&
1256 cap_list[MIGRATION_CAPABILITY_MULTIFD]) {
1257 error_setg(errp, "multifd is not supported by current protocol");
1258 return false;
1259 }
1260
1261 return true;
1262 }
1263
1264 static void fill_destination_migration_info(MigrationInfo *info)
1265 {
1266 MigrationIncomingState *mis = migration_incoming_get_current();
1267
1268 if (mis->socket_address_list) {
1269 info->has_socket_address = true;
1270 info->socket_address =
1271 QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1272 }
1273
1274 switch (mis->state) {
1275 case MIGRATION_STATUS_NONE:
1276 return;
1277 case MIGRATION_STATUS_SETUP:
1278 case MIGRATION_STATUS_CANCELLING:
1279 case MIGRATION_STATUS_CANCELLED:
1280 case MIGRATION_STATUS_ACTIVE:
1281 case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1282 case MIGRATION_STATUS_POSTCOPY_PAUSED:
1283 case MIGRATION_STATUS_POSTCOPY_RECOVER:
1284 case MIGRATION_STATUS_FAILED:
1285 case MIGRATION_STATUS_COLO:
1286 info->has_status = true;
1287 break;
1288 case MIGRATION_STATUS_COMPLETED:
1289 info->has_status = true;
1290 fill_destination_postcopy_migration_info(info);
1291 break;
1292 }
1293 info->status = mis->state;
1294 }
1295
1296 MigrationInfo *qmp_query_migrate(Error **errp)
1297 {
1298 MigrationInfo *info = g_malloc0(sizeof(*info));
1299
1300 fill_destination_migration_info(info);
1301 fill_source_migration_info(info);
1302
1303 return info;
1304 }
1305
1306 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1307 Error **errp)
1308 {
1309 MigrationState *s = migrate_get_current();
1310 MigrationCapabilityStatusList *cap;
1311 bool cap_list[MIGRATION_CAPABILITY__MAX];
1312
1313 if (migration_is_running(s->state)) {
1314 error_setg(errp, QERR_MIGRATION_ACTIVE);
1315 return;
1316 }
1317
1318 memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1319 if (!migrate_caps_check(cap_list, params, errp)) {
1320 return;
1321 }
1322
1323 for (cap = params; cap; cap = cap->next) {
1324 s->enabled_capabilities[cap->value->capability] = cap->value->state;
1325 }
1326 }
1327
1328 /*
1329 * Check whether the parameters are valid. Error will be put into errp
1330 * (if provided). Return true if valid, otherwise false.
1331 */
1332 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1333 {
1334 if (params->has_compress_level &&
1335 (params->compress_level > 9)) {
1336 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1337 "a value between 0 and 9");
1338 return false;
1339 }
1340
1341 if (params->has_compress_threads && (params->compress_threads < 1)) {
1342 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1343 "compress_threads",
1344 "a value between 1 and 255");
1345 return false;
1346 }
1347
1348 if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1349 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1350 "decompress_threads",
1351 "a value between 1 and 255");
1352 return false;
1353 }
1354
1355 if (params->has_throttle_trigger_threshold &&
1356 (params->throttle_trigger_threshold < 1 ||
1357 params->throttle_trigger_threshold > 100)) {
1358 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1359 "throttle_trigger_threshold",
1360 "an integer in the range of 1 to 100");
1361 return false;
1362 }
1363
1364 if (params->has_cpu_throttle_initial &&
1365 (params->cpu_throttle_initial < 1 ||
1366 params->cpu_throttle_initial > 99)) {
1367 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1368 "cpu_throttle_initial",
1369 "an integer in the range of 1 to 99");
1370 return false;
1371 }
1372
1373 if (params->has_cpu_throttle_increment &&
1374 (params->cpu_throttle_increment < 1 ||
1375 params->cpu_throttle_increment > 99)) {
1376 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1377 "cpu_throttle_increment",
1378 "an integer in the range of 1 to 99");
1379 return false;
1380 }
1381
1382 if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1383 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1384 "max_bandwidth",
1385 "an integer in the range of 0 to "stringify(SIZE_MAX)
1386 " bytes/second");
1387 return false;
1388 }
1389
1390 if (params->has_downtime_limit &&
1391 (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1392 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1393 "downtime_limit",
1394 "an integer in the range of 0 to "
1395 stringify(MAX_MIGRATE_DOWNTIME)" ms");
1396 return false;
1397 }
1398
1399 /* x_checkpoint_delay is now always positive */
1400
1401 if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1402 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1403 "multifd_channels",
1404 "a value between 1 and 255");
1405 return false;
1406 }
1407
1408 if (params->has_multifd_zlib_level &&
1409 (params->multifd_zlib_level > 9)) {
1410 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1411 "a value between 0 and 9");
1412 return false;
1413 }
1414
1415 if (params->has_multifd_zstd_level &&
1416 (params->multifd_zstd_level > 20)) {
1417 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1418 "a value between 0 and 20");
1419 return false;
1420 }
1421
1422 if (params->has_xbzrle_cache_size &&
1423 (params->xbzrle_cache_size < qemu_target_page_size() ||
1424 !is_power_of_2(params->xbzrle_cache_size))) {
1425 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1426 "xbzrle_cache_size",
1427 "a power of two no less than the target page size");
1428 return false;
1429 }
1430
1431 if (params->has_max_cpu_throttle &&
1432 (params->max_cpu_throttle < params->cpu_throttle_initial ||
1433 params->max_cpu_throttle > 99)) {
1434 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1435 "max_cpu_throttle",
1436 "an integer in the range of cpu_throttle_initial to 99");
1437 return false;
1438 }
1439
1440 if (params->has_announce_initial &&
1441 params->announce_initial > 100000) {
1442 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1443 "announce_initial",
1444 "a value between 0 and 100000");
1445 return false;
1446 }
1447 if (params->has_announce_max &&
1448 params->announce_max > 100000) {
1449 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1450 "announce_max",
1451 "a value between 0 and 100000");
1452 return false;
1453 }
1454 if (params->has_announce_rounds &&
1455 params->announce_rounds > 1000) {
1456 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1457 "announce_rounds",
1458 "a value between 0 and 1000");
1459 return false;
1460 }
1461 if (params->has_announce_step &&
1462 (params->announce_step < 1 ||
1463 params->announce_step > 10000)) {
1464 error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1465 "announce_step",
1466 "a value between 0 and 10000");
1467 return false;
1468 }
1469
1470 if (params->has_block_bitmap_mapping &&
1471 !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1472 error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1473 return false;
1474 }
1475
1476 return true;
1477 }
1478
1479 static void migrate_params_test_apply(MigrateSetParameters *params,
1480 MigrationParameters *dest)
1481 {
1482 *dest = migrate_get_current()->parameters;
1483
1484 /* TODO use QAPI_CLONE() instead of duplicating it inline */
1485
1486 if (params->has_compress_level) {
1487 dest->compress_level = params->compress_level;
1488 }
1489
1490 if (params->has_compress_threads) {
1491 dest->compress_threads = params->compress_threads;
1492 }
1493
1494 if (params->has_compress_wait_thread) {
1495 dest->compress_wait_thread = params->compress_wait_thread;
1496 }
1497
1498 if (params->has_decompress_threads) {
1499 dest->decompress_threads = params->decompress_threads;
1500 }
1501
1502 if (params->has_throttle_trigger_threshold) {
1503 dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1504 }
1505
1506 if (params->has_cpu_throttle_initial) {
1507 dest->cpu_throttle_initial = params->cpu_throttle_initial;
1508 }
1509
1510 if (params->has_cpu_throttle_increment) {
1511 dest->cpu_throttle_increment = params->cpu_throttle_increment;
1512 }
1513
1514 if (params->has_cpu_throttle_tailslow) {
1515 dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1516 }
1517
1518 if (params->has_tls_creds) {
1519 assert(params->tls_creds->type == QTYPE_QSTRING);
1520 dest->tls_creds = params->tls_creds->u.s;
1521 }
1522
1523 if (params->has_tls_hostname) {
1524 assert(params->tls_hostname->type == QTYPE_QSTRING);
1525 dest->tls_hostname = params->tls_hostname->u.s;
1526 }
1527
1528 if (params->has_max_bandwidth) {
1529 dest->max_bandwidth = params->max_bandwidth;
1530 }
1531
1532 if (params->has_downtime_limit) {
1533 dest->downtime_limit = params->downtime_limit;
1534 }
1535
1536 if (params->has_x_checkpoint_delay) {
1537 dest->x_checkpoint_delay = params->x_checkpoint_delay;
1538 }
1539
1540 if (params->has_block_incremental) {
1541 dest->block_incremental = params->block_incremental;
1542 }
1543 if (params->has_multifd_channels) {
1544 dest->multifd_channels = params->multifd_channels;
1545 }
1546 if (params->has_multifd_compression) {
1547 dest->multifd_compression = params->multifd_compression;
1548 }
1549 if (params->has_xbzrle_cache_size) {
1550 dest->xbzrle_cache_size = params->xbzrle_cache_size;
1551 }
1552 if (params->has_max_postcopy_bandwidth) {
1553 dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1554 }
1555 if (params->has_max_cpu_throttle) {
1556 dest->max_cpu_throttle = params->max_cpu_throttle;
1557 }
1558 if (params->has_announce_initial) {
1559 dest->announce_initial = params->announce_initial;
1560 }
1561 if (params->has_announce_max) {
1562 dest->announce_max = params->announce_max;
1563 }
1564 if (params->has_announce_rounds) {
1565 dest->announce_rounds = params->announce_rounds;
1566 }
1567 if (params->has_announce_step) {
1568 dest->announce_step = params->announce_step;
1569 }
1570
1571 if (params->has_block_bitmap_mapping) {
1572 dest->has_block_bitmap_mapping = true;
1573 dest->block_bitmap_mapping = params->block_bitmap_mapping;
1574 }
1575 }
1576
1577 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1578 {
1579 MigrationState *s = migrate_get_current();
1580
1581 /* TODO use QAPI_CLONE() instead of duplicating it inline */
1582
1583 if (params->has_compress_level) {
1584 s->parameters.compress_level = params->compress_level;
1585 }
1586
1587 if (params->has_compress_threads) {
1588 s->parameters.compress_threads = params->compress_threads;
1589 }
1590
1591 if (params->has_compress_wait_thread) {
1592 s->parameters.compress_wait_thread = params->compress_wait_thread;
1593 }
1594
1595 if (params->has_decompress_threads) {
1596 s->parameters.decompress_threads = params->decompress_threads;
1597 }
1598
1599 if (params->has_throttle_trigger_threshold) {
1600 s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1601 }
1602
1603 if (params->has_cpu_throttle_initial) {
1604 s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1605 }
1606
1607 if (params->has_cpu_throttle_increment) {
1608 s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1609 }
1610
1611 if (params->has_cpu_throttle_tailslow) {
1612 s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1613 }
1614
1615 if (params->has_tls_creds) {
1616 g_free(s->parameters.tls_creds);
1617 assert(params->tls_creds->type == QTYPE_QSTRING);
1618 s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1619 }
1620
1621 if (params->has_tls_hostname) {
1622 g_free(s->parameters.tls_hostname);
1623 assert(params->tls_hostname->type == QTYPE_QSTRING);
1624 s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1625 }
1626
1627 if (params->has_tls_authz) {
1628 g_free(s->parameters.tls_authz);
1629 assert(params->tls_authz->type == QTYPE_QSTRING);
1630 s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1631 }
1632
1633 if (params->has_max_bandwidth) {
1634 s->parameters.max_bandwidth = params->max_bandwidth;
1635 if (s->to_dst_file && !migration_in_postcopy()) {
1636 qemu_file_set_rate_limit(s->to_dst_file,
1637 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1638 }
1639 }
1640
1641 if (params->has_downtime_limit) {
1642 s->parameters.downtime_limit = params->downtime_limit;
1643 }
1644
1645 if (params->has_x_checkpoint_delay) {
1646 s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1647 if (migration_in_colo_state()) {
1648 colo_checkpoint_notify(s);
1649 }
1650 }
1651
1652 if (params->has_block_incremental) {
1653 s->parameters.block_incremental = params->block_incremental;
1654 }
1655 if (params->has_multifd_channels) {
1656 s->parameters.multifd_channels = params->multifd_channels;
1657 }
1658 if (params->has_multifd_compression) {
1659 s->parameters.multifd_compression = params->multifd_compression;
1660 }
1661 if (params->has_xbzrle_cache_size) {
1662 s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1663 xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1664 }
1665 if (params->has_max_postcopy_bandwidth) {
1666 s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1667 if (s->to_dst_file && migration_in_postcopy()) {
1668 qemu_file_set_rate_limit(s->to_dst_file,
1669 s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1670 }
1671 }
1672 if (params->has_max_cpu_throttle) {
1673 s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1674 }
1675 if (params->has_announce_initial) {
1676 s->parameters.announce_initial = params->announce_initial;
1677 }
1678 if (params->has_announce_max) {
1679 s->parameters.announce_max = params->announce_max;
1680 }
1681 if (params->has_announce_rounds) {
1682 s->parameters.announce_rounds = params->announce_rounds;
1683 }
1684 if (params->has_announce_step) {
1685 s->parameters.announce_step = params->announce_step;
1686 }
1687
1688 if (params->has_block_bitmap_mapping) {
1689 qapi_free_BitmapMigrationNodeAliasList(
1690 s->parameters.block_bitmap_mapping);
1691
1692 s->parameters.has_block_bitmap_mapping = true;
1693 s->parameters.block_bitmap_mapping =
1694 QAPI_CLONE(BitmapMigrationNodeAliasList,
1695 params->block_bitmap_mapping);
1696 }
1697 }
1698
1699 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1700 {
1701 MigrationParameters tmp;
1702
1703 /* TODO Rewrite "" to null instead */
1704 if (params->has_tls_creds
1705 && params->tls_creds->type == QTYPE_QNULL) {
1706 qobject_unref(params->tls_creds->u.n);
1707 params->tls_creds->type = QTYPE_QSTRING;
1708 params->tls_creds->u.s = strdup("");
1709 }
1710 /* TODO Rewrite "" to null instead */
1711 if (params->has_tls_hostname
1712 && params->tls_hostname->type == QTYPE_QNULL) {
1713 qobject_unref(params->tls_hostname->u.n);
1714 params->tls_hostname->type = QTYPE_QSTRING;
1715 params->tls_hostname->u.s = strdup("");
1716 }
1717
1718 migrate_params_test_apply(params, &tmp);
1719
1720 if (!migrate_params_check(&tmp, errp)) {
1721 /* Invalid parameter */
1722 return;
1723 }
1724
1725 migrate_params_apply(params, errp);
1726 }
1727
1728
1729 void qmp_migrate_start_postcopy(Error **errp)
1730 {
1731 MigrationState *s = migrate_get_current();
1732
1733 if (!migrate_postcopy()) {
1734 error_setg(errp, "Enable postcopy with migrate_set_capability before"
1735 " the start of migration");
1736 return;
1737 }
1738
1739 if (s->state == MIGRATION_STATUS_NONE) {
1740 error_setg(errp, "Postcopy must be started after migration has been"
1741 " started");
1742 return;
1743 }
1744 /*
1745 * we don't error if migration has finished since that would be racy
1746 * with issuing this command.
1747 */
1748 qatomic_set(&s->start_postcopy, true);
1749 }
1750
1751 /* shared migration helpers */
1752
1753 void migrate_set_state(int *state, int old_state, int new_state)
1754 {
1755 assert(new_state < MIGRATION_STATUS__MAX);
1756 if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1757 trace_migrate_set_state(MigrationStatus_str(new_state));
1758 migrate_generate_event(new_state);
1759 }
1760 }
1761
1762 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1763 bool state)
1764 {
1765 MigrationCapabilityStatus *cap;
1766
1767 cap = g_new0(MigrationCapabilityStatus, 1);
1768 cap->capability = index;
1769 cap->state = state;
1770
1771 return cap;
1772 }
1773
1774 void migrate_set_block_enabled(bool value, Error **errp)
1775 {
1776 MigrationCapabilityStatusList *cap = NULL;
1777
1778 QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1779 qmp_migrate_set_capabilities(cap, errp);
1780 qapi_free_MigrationCapabilityStatusList(cap);
1781 }
1782
1783 static void migrate_set_block_incremental(MigrationState *s, bool value)
1784 {
1785 s->parameters.block_incremental = value;
1786 }
1787
1788 static void block_cleanup_parameters(MigrationState *s)
1789 {
1790 if (s->must_remove_block_options) {
1791 /* setting to false can never fail */
1792 migrate_set_block_enabled(false, &error_abort);
1793 migrate_set_block_incremental(s, false);
1794 s->must_remove_block_options = false;
1795 }
1796 }
1797
1798 static void migrate_fd_cleanup(MigrationState *s)
1799 {
1800 qemu_bh_delete(s->cleanup_bh);
1801 s->cleanup_bh = NULL;
1802
1803 qemu_savevm_state_cleanup();
1804
1805 if (s->to_dst_file) {
1806 QEMUFile *tmp;
1807
1808 trace_migrate_fd_cleanup();
1809 qemu_mutex_unlock_iothread();
1810 if (s->migration_thread_running) {
1811 qemu_thread_join(&s->thread);
1812 s->migration_thread_running = false;
1813 }
1814 qemu_mutex_lock_iothread();
1815
1816 multifd_save_cleanup();
1817 qemu_mutex_lock(&s->qemu_file_lock);
1818 tmp = s->to_dst_file;
1819 s->to_dst_file = NULL;
1820 qemu_mutex_unlock(&s->qemu_file_lock);
1821 /*
1822 * Close the file handle without the lock to make sure the
1823 * critical section won't block for long.
1824 */
1825 migration_ioc_unregister_yank_from_file(tmp);
1826 qemu_fclose(tmp);
1827 }
1828
1829 assert(!migration_is_active(s));
1830
1831 if (s->state == MIGRATION_STATUS_CANCELLING) {
1832 migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1833 MIGRATION_STATUS_CANCELLED);
1834 }
1835
1836 if (s->error) {
1837 /* It is used on info migrate. We can't free it */
1838 error_report_err(error_copy(s->error));
1839 }
1840 notifier_list_notify(&migration_state_notifiers, s);
1841 block_cleanup_parameters(s);
1842 yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1843 }
1844
1845 static void migrate_fd_cleanup_schedule(MigrationState *s)
1846 {
1847 /*
1848 * Ref the state for bh, because it may be called when
1849 * there're already no other refs
1850 */
1851 object_ref(OBJECT(s));
1852 qemu_bh_schedule(s->cleanup_bh);
1853 }
1854
1855 static void migrate_fd_cleanup_bh(void *opaque)
1856 {
1857 MigrationState *s = opaque;
1858 migrate_fd_cleanup(s);
1859 object_unref(OBJECT(s));
1860 }
1861
1862 void migrate_set_error(MigrationState *s, const Error *error)
1863 {
1864 QEMU_LOCK_GUARD(&s->error_mutex);
1865 if (!s->error) {
1866 s->error = error_copy(error);
1867 }
1868 }
1869
1870 static void migrate_error_free(MigrationState *s)
1871 {
1872 QEMU_LOCK_GUARD(&s->error_mutex);
1873 if (s->error) {
1874 error_free(s->error);
1875 s->error = NULL;
1876 }
1877 }
1878
1879 void migrate_fd_error(MigrationState *s, const Error *error)
1880 {
1881 trace_migrate_fd_error(error_get_pretty(error));
1882 assert(s->to_dst_file == NULL);
1883 migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1884 MIGRATION_STATUS_FAILED);
1885 migrate_set_error(s, error);
1886 }
1887
1888 static void migrate_fd_cancel(MigrationState *s)
1889 {
1890 int old_state ;
1891 QEMUFile *f = migrate_get_current()->to_dst_file;
1892 trace_migrate_fd_cancel();
1893
1894 WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1895 if (s->rp_state.from_dst_file) {
1896 /* shutdown the rp socket, so causing the rp thread to shutdown */
1897 qemu_file_shutdown(s->rp_state.from_dst_file);
1898 }
1899 }
1900
1901 do {
1902 old_state = s->state;
1903 if (!migration_is_running(old_state)) {
1904 break;
1905 }
1906 /* If the migration is paused, kick it out of the pause */
1907 if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1908 qemu_sem_post(&s->pause_sem);
1909 }
1910 migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1911 } while (s->state != MIGRATION_STATUS_CANCELLING);
1912
1913 /*
1914 * If we're unlucky the migration code might be stuck somewhere in a
1915 * send/write while the network has failed and is waiting to timeout;
1916 * if we've got shutdown(2) available then we can force it to quit.
1917 * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1918 * called in a bh, so there is no race against this cancel.
1919 */
1920 if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1921 qemu_file_shutdown(f);
1922 }
1923 if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1924 Error *local_err = NULL;
1925
1926 bdrv_invalidate_cache_all(&local_err);
1927 if (local_err) {
1928 error_report_err(local_err);
1929 } else {
1930 s->block_inactive = false;
1931 }
1932 }
1933 }
1934
1935 void add_migration_state_change_notifier(Notifier *notify)
1936 {
1937 notifier_list_add(&migration_state_notifiers, notify);
1938 }
1939
1940 void remove_migration_state_change_notifier(Notifier *notify)
1941 {
1942 notifier_remove(notify);
1943 }
1944
1945 bool migration_in_setup(MigrationState *s)
1946 {
1947 return s->state == MIGRATION_STATUS_SETUP;
1948 }
1949
1950 bool migration_has_finished(MigrationState *s)
1951 {
1952 return s->state == MIGRATION_STATUS_COMPLETED;
1953 }
1954
1955 bool migration_has_failed(MigrationState *s)
1956 {
1957 return (s->state == MIGRATION_STATUS_CANCELLED ||
1958 s->state == MIGRATION_STATUS_FAILED);
1959 }
1960
1961 bool migration_in_postcopy(void)
1962 {
1963 MigrationState *s = migrate_get_current();
1964
1965 switch (s->state) {
1966 case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1967 case MIGRATION_STATUS_POSTCOPY_PAUSED:
1968 case MIGRATION_STATUS_POSTCOPY_RECOVER:
1969 return true;
1970 default:
1971 return false;
1972 }
1973 }
1974
1975 bool migration_in_postcopy_after_devices(MigrationState *s)
1976 {
1977 return migration_in_postcopy() && s->postcopy_after_devices;
1978 }
1979
1980 bool migration_in_incoming_postcopy(void)
1981 {
1982 PostcopyState ps = postcopy_state_get();
1983
1984 return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
1985 }
1986
1987 bool migration_in_bg_snapshot(void)
1988 {
1989 MigrationState *s = migrate_get_current();
1990
1991 return migrate_background_snapshot() &&
1992 migration_is_setup_or_active(s->state);
1993 }
1994
1995 bool migration_is_idle(void)
1996 {
1997 MigrationState *s = current_migration;
1998
1999 if (!s) {
2000 return true;
2001 }
2002
2003 switch (s->state) {
2004 case MIGRATION_STATUS_NONE:
2005 case MIGRATION_STATUS_CANCELLED:
2006 case MIGRATION_STATUS_COMPLETED:
2007 case MIGRATION_STATUS_FAILED:
2008 return true;
2009 case MIGRATION_STATUS_SETUP:
2010 case MIGRATION_STATUS_CANCELLING:
2011 case MIGRATION_STATUS_ACTIVE:
2012 case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2013 case MIGRATION_STATUS_COLO:
2014 case MIGRATION_STATUS_PRE_SWITCHOVER:
2015 case MIGRATION_STATUS_DEVICE:
2016 case MIGRATION_STATUS_WAIT_UNPLUG:
2017 return false;
2018 case MIGRATION_STATUS__MAX:
2019 g_assert_not_reached();
2020 }
2021
2022 return false;
2023 }
2024
2025 bool migration_is_active(MigrationState *s)
2026 {
2027 return (s->state == MIGRATION_STATUS_ACTIVE ||
2028 s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
2029 }
2030
2031 void migrate_init(MigrationState *s)
2032 {
2033 /*
2034 * Reinitialise all migration state, except
2035 * parameters/capabilities that the user set, and
2036 * locks.
2037 */
2038 s->cleanup_bh = 0;
2039 s->vm_start_bh = 0;
2040 s->to_dst_file = NULL;
2041 s->state = MIGRATION_STATUS_NONE;
2042 s->rp_state.from_dst_file = NULL;
2043 s->rp_state.error = false;
2044 s->mbps = 0.0;
2045 s->pages_per_second = 0.0;
2046 s->downtime = 0;
2047 s->expected_downtime = 0;
2048 s->setup_time = 0;
2049 s->start_postcopy = false;
2050 s->postcopy_after_devices = false;
2051 s->migration_thread_running = false;
2052 error_free(s->error);
2053 s->error = NULL;
2054 s->hostname = NULL;
2055
2056 migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
2057
2058 s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2059 s->total_time = 0;
2060 s->vm_was_running = false;
2061 s->iteration_initial_bytes = 0;
2062 s->threshold_size = 0;
2063 }
2064
2065 int migrate_add_blocker_internal(Error *reason, Error **errp)
2066 {
2067 /* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */
2068 if (runstate_check(RUN_STATE_SAVE_VM) || !migration_is_idle()) {
2069 error_propagate_prepend(errp, error_copy(reason),
2070 "disallowing migration blocker "
2071 "(migration/snapshot in progress) for: ");
2072 return -EBUSY;
2073 }
2074
2075 migration_blockers = g_slist_prepend(migration_blockers, reason);
2076 return 0;
2077 }
2078
2079 int migrate_add_blocker(Error *reason, Error **errp)
2080 {
2081 if (only_migratable) {
2082 error_propagate_prepend(errp, error_copy(reason),
2083 "disallowing migration blocker "
2084 "(--only-migratable) for: ");
2085 return -EACCES;
2086 }
2087
2088 return migrate_add_blocker_internal(reason, errp);
2089 }
2090
2091 void migrate_del_blocker(Error *reason)
2092 {
2093 migration_blockers = g_slist_remove(migration_blockers, reason);
2094 }
2095
2096 void qmp_migrate_incoming(const char *uri, Error **errp)
2097 {
2098 Error *local_err = NULL;
2099 static bool once = true;
2100
2101 if (!once) {
2102 error_setg(errp, "The incoming migration has already been started");
2103 return;
2104 }
2105 if (!runstate_check(RUN_STATE_INMIGRATE)) {
2106 error_setg(errp, "'-incoming' was not specified on the command line");
2107 return;
2108 }
2109
2110 if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2111 return;
2112 }
2113
2114 qemu_start_incoming_migration(uri, &local_err);
2115
2116 if (local_err) {
2117 yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2118 error_propagate(errp, local_err);
2119 return;
2120 }
2121
2122 once = false;
2123 }
2124
2125 void qmp_migrate_recover(const char *uri, Error **errp)
2126 {
2127 MigrationIncomingState *mis = migration_incoming_get_current();
2128
2129 /*
2130 * Don't even bother to use ERRP_GUARD() as it _must_ always be set by
2131 * callers (no one should ignore a recover failure); if there is, it's a
2132 * programming error.
2133 */
2134 assert(errp);
2135
2136 if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2137 error_setg(errp, "Migrate recover can only be run "
2138 "when postcopy is paused.");
2139 return;
2140 }
2141
2142 if (qatomic_cmpxchg(&mis->postcopy_recover_triggered,
2143 false, true) == true) {
2144 error_setg(errp, "Migrate recovery is triggered already");
2145 return;
2146 }
2147
2148 /*
2149 * Note that this call will never start a real migration; it will
2150 * only re-setup the migration stream and poke existing migration
2151 * to continue using that newly established channel.
2152 */
2153 qemu_start_incoming_migration(uri, errp);
2154
2155 /* Safe to dereference with the assert above */
2156 if (*errp) {
2157 /* Reset the flag so user could still retry */
2158 qatomic_set(&mis->postcopy_recover_triggered, false);
2159 }
2160 }
2161
2162 void qmp_migrate_pause(Error **errp)
2163 {
2164 MigrationState *ms = migrate_get_current();
2165 MigrationIncomingState *mis = migration_incoming_get_current();
2166 int ret;
2167
2168 if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2169 /* Source side, during postcopy */
2170 qemu_mutex_lock(&ms->qemu_file_lock);
2171 ret = qemu_file_shutdown(ms->to_dst_file);
2172 qemu_mutex_unlock(&ms->qemu_file_lock);
2173 if (ret) {
2174 error_setg(errp, "Failed to pause source migration");
2175 }
2176 return;
2177 }
2178
2179 if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2180 ret = qemu_file_shutdown(mis->from_src_file);
2181 if (ret) {
2182 error_setg(errp, "Failed to pause destination migration");
2183 }
2184 return;
2185 }
2186
2187 error_setg(errp, "migrate-pause is currently only supported "
2188 "during postcopy-active state");
2189 }
2190
2191 bool migration_is_blocked(Error **errp)
2192 {
2193 if (qemu_savevm_state_blocked(errp)) {
2194 return true;
2195 }
2196
2197 if (migration_blockers) {
2198 error_propagate(errp, error_copy(migration_blockers->data));
2199 return true;
2200 }
2201
2202 return false;
2203 }
2204
2205 /* Returns true if continue to migrate, or false if error detected */
2206 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2207 bool resume, Error **errp)
2208 {
2209 Error *local_err = NULL;
2210
2211 if (resume) {
2212 if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2213 error_setg(errp, "Cannot resume if there is no "
2214 "paused migration");
2215 return false;
2216 }
2217
2218 /*
2219 * Postcopy recovery won't work well with release-ram
2220 * capability since release-ram will drop the page buffer as
2221 * long as the page is put into the send buffer. So if there
2222 * is a network failure happened, any page buffers that have
2223 * not yet reached the destination VM but have already been
2224 * sent from the source VM will be lost forever. Let's refuse
2225 * the client from resuming such a postcopy migration.
2226 * Luckily release-ram was designed to only be used when src
2227 * and destination VMs are on the same host, so it should be
2228 * fine.
2229 */
2230 if (migrate_release_ram()) {
2231 error_setg(errp, "Postcopy recovery cannot work "
2232 "when release-ram capability is set");
2233 return false;
2234 }
2235
2236 /* This is a resume, skip init status */
2237 return true;
2238 }
2239
2240 if (migration_is_running(s->state)) {
2241 error_setg(errp, QERR_MIGRATION_ACTIVE);
2242 return false;
2243 }
2244
2245 if (runstate_check(RUN_STATE_INMIGRATE)) {
2246 error_setg(errp, "Guest is waiting for an incoming migration");
2247 return false;
2248 }
2249
2250 if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2251 error_setg(errp, "Can't migrate the vm that was paused due to "
2252 "previous migration");
2253 return false;
2254 }
2255
2256 if (migration_is_blocked(errp)) {
2257 return false;
2258 }
2259
2260 if (blk || blk_inc) {
2261 if (migrate_colo_enabled()) {
2262 error_setg(errp, "No disk migration is required in COLO mode");
2263 return false;
2264 }
2265 if (migrate_use_block() || migrate_use_block_incremental()) {
2266 error_setg(errp, "Command options are incompatible with "
2267 "current migration capabilities");
2268 return false;
2269 }
2270 migrate_set_block_enabled(true, &local_err);
2271 if (local_err) {
2272 error_propagate(errp, local_err);
2273 return false;
2274 }
2275 s->must_remove_block_options = true;
2276 }
2277
2278 if (blk_inc) {
2279 migrate_set_block_incremental(s, true);
2280 }
2281
2282 migrate_init(s);
2283 /*
2284 * set ram_counters compression_counters memory to zero for a
2285 * new migration
2286 */
2287 memset(&ram_counters, 0, sizeof(ram_counters));
2288 memset(&compression_counters, 0, sizeof(compression_counters));
2289
2290 return true;
2291 }
2292
2293 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2294 bool has_inc, bool inc, bool has_detach, bool detach,
2295 bool has_resume, bool resume, Error **errp)
2296 {
2297 Error *local_err = NULL;
2298 MigrationState *s = migrate_get_current();
2299 const char *p = NULL;
2300
2301 if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2302 has_resume && resume, errp)) {
2303 /* Error detected, put into errp */
2304 return;
2305 }
2306
2307 if (!(has_resume && resume)) {
2308 if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2309 return;
2310 }
2311 }
2312
2313 migrate_protocol_allow_multifd(false);
2314 if (strstart(uri, "tcp:", &p) ||
2315 strstart(uri, "unix:", NULL) ||
2316 strstart(uri, "vsock:", NULL)) {
2317 migrate_protocol_allow_multifd(true);
2318 socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2319 #ifdef CONFIG_RDMA
2320 } else if (strstart(uri, "rdma:", &p)) {
2321 rdma_start_outgoing_migration(s, p, &local_err);
2322 #endif
2323 } else if (strstart(uri, "exec:", &p)) {
2324 exec_start_outgoing_migration(s, p, &local_err);
2325 } else if (strstart(uri, "fd:", &p)) {
2326 fd_start_outgoing_migration(s, p, &local_err);
2327 } else {
2328 if (!(has_resume && resume)) {
2329 yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2330 }
2331 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2332 "a valid migration protocol");
2333 migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2334 MIGRATION_STATUS_FAILED);
2335 block_cleanup_parameters(s);
2336 return;
2337 }
2338
2339 if (local_err) {
2340 if (!(has_resume && resume)) {
2341 yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2342 }
2343 migrate_fd_error(s, local_err);
2344 error_propagate(errp, local_err);
2345 return;
2346 }
2347 }
2348
2349 void qmp_migrate_cancel(Error **errp)
2350 {
2351 migration_cancel(NULL);
2352 }
2353
2354 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2355 {
2356 MigrationState *s = migrate_get_current();
2357 if (s->state != state) {
2358 error_setg(errp, "Migration not in expected state: %s",
2359 MigrationStatus_str(s->state));
2360 return;
2361 }
2362 qemu_sem_post(&s->pause_sem);
2363 }
2364
2365 bool migrate_release_ram(void)
2366 {
2367 MigrationState *s;
2368
2369 s = migrate_get_current();
2370
2371 return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2372 }
2373
2374 bool migrate_postcopy_ram(void)
2375 {
2376 MigrationState *s;
2377
2378 s = migrate_get_current();
2379
2380 return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2381 }
2382
2383 bool migrate_postcopy(void)
2384 {
2385 return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2386 }
2387
2388 bool migrate_auto_converge(void)
2389 {
2390 MigrationState *s;
2391
2392 s = migrate_get_current();
2393
2394 return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2395 }
2396
2397 bool migrate_zero_blocks(void)
2398 {
2399 MigrationState *s;
2400
2401 s = migrate_get_current();
2402
2403 return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2404 }
2405
2406 bool migrate_postcopy_blocktime(void)
2407 {
2408 MigrationState *s;
2409
2410 s = migrate_get_current();
2411
2412 return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2413 }
2414
2415 bool migrate_use_compression(void)
2416 {
2417 MigrationState *s;
2418
2419 s = migrate_get_current();
2420
2421 return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2422 }
2423
2424 int migrate_compress_level(void)
2425 {
2426 MigrationState *s;
2427
2428 s = migrate_get_current();
2429
2430 return s->parameters.compress_level;
2431 }
2432
2433 int migrate_compress_threads(void)
2434 {
2435 MigrationState *s;
2436
2437 s = migrate_get_current();
2438
2439 return s->parameters.compress_threads;
2440 }
2441
2442 int migrate_compress_wait_thread(void)
2443 {
2444 MigrationState *s;
2445
2446 s = migrate_get_current();
2447
2448 return s->parameters.compress_wait_thread;
2449 }
2450
2451 int migrate_decompress_threads(void)
2452 {
2453 MigrationState *s;
2454
2455 s = migrate_get_current();
2456
2457 return s->parameters.decompress_threads;
2458 }
2459
2460 bool migrate_dirty_bitmaps(void)
2461 {
2462 MigrationState *s;
2463
2464 s = migrate_get_current();
2465
2466 return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2467 }
2468
2469 bool migrate_ignore_shared(void)
2470 {
2471 MigrationState *s;
2472
2473 s = migrate_get_current();
2474
2475 return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2476 }
2477
2478 bool migrate_validate_uuid(void)
2479 {
2480 MigrationState *s;
2481
2482 s = migrate_get_current();
2483
2484 return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2485 }
2486
2487 bool migrate_use_events(void)
2488 {
2489 MigrationState *s;
2490
2491 s = migrate_get_current();
2492
2493 return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2494 }
2495
2496 bool migrate_use_multifd(void)
2497 {
2498 MigrationState *s;
2499
2500 s = migrate_get_current();
2501
2502 return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2503 }
2504
2505 bool migrate_pause_before_switchover(void)
2506 {
2507 MigrationState *s;
2508
2509 s = migrate_get_current();
2510
2511 return s->enabled_capabilities[
2512 MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2513 }
2514
2515 int migrate_multifd_channels(void)
2516 {
2517 MigrationState *s;
2518
2519 s = migrate_get_current();
2520
2521 return s->parameters.multifd_channels;
2522 }
2523
2524 MultiFDCompression migrate_multifd_compression(void)
2525 {
2526 MigrationState *s;
2527
2528 s = migrate_get_current();
2529
2530 return s->parameters.multifd_compression;
2531 }
2532
2533 int migrate_multifd_zlib_level(void)
2534 {
2535 MigrationState *s;
2536
2537 s = migrate_get_current();
2538
2539 return s->parameters.multifd_zlib_level;
2540 }
2541
2542 int migrate_multifd_zstd_level(void)
2543 {
2544 MigrationState *s;
2545
2546 s = migrate_get_current();
2547
2548 return s->parameters.multifd_zstd_level;
2549 }
2550
2551 int migrate_use_xbzrle(void)
2552 {
2553 MigrationState *s;
2554
2555 s = migrate_get_current();
2556
2557 return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2558 }
2559
2560 uint64_t migrate_xbzrle_cache_size(void)
2561 {
2562 MigrationState *s;
2563
2564 s = migrate_get_current();
2565
2566 return s->parameters.xbzrle_cache_size;
2567 }
2568
2569 static int64_t migrate_max_postcopy_bandwidth(void)
2570 {
2571 MigrationState *s;
2572
2573 s = migrate_get_current();
2574
2575 return s->parameters.max_postcopy_bandwidth;
2576 }
2577
2578 bool migrate_use_block(void)
2579 {
2580 MigrationState *s;
2581
2582 s = migrate_get_current();
2583
2584 return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2585 }
2586
2587 bool migrate_use_return_path(void)
2588 {
2589 MigrationState *s;
2590
2591 s = migrate_get_current();
2592
2593 return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2594 }
2595
2596 bool migrate_use_block_incremental(void)
2597 {
2598 MigrationState *s;
2599
2600 s = migrate_get_current();
2601
2602 return s->parameters.block_incremental;
2603 }
2604
2605 bool migrate_background_snapshot(void)
2606 {
2607 MigrationState *s;
2608
2609 s = migrate_get_current();
2610
2611 return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT];
2612 }
2613
2614 /* migration thread support */
2615 /*
2616 * Something bad happened to the RP stream, mark an error
2617 * The caller shall print or trace something to indicate why
2618 */
2619 static void mark_source_rp_bad(MigrationState *s)
2620 {
2621 s->rp_state.error = true;
2622 }
2623
2624 static struct rp_cmd_args {
2625 ssize_t len; /* -1 = variable */
2626 const char *name;
2627 } rp_cmd_args[] = {
2628 [MIG_RP_MSG_INVALID] = { .len = -1, .name = "INVALID" },
2629 [MIG_RP_MSG_SHUT] = { .len = 4, .name = "SHUT" },
2630 [MIG_RP_MSG_PONG] = { .len = 4, .name = "PONG" },
2631 [MIG_RP_MSG_REQ_PAGES] = { .len = 12, .name = "REQ_PAGES" },
2632 [MIG_RP_MSG_REQ_PAGES_ID] = { .len = -1, .name = "REQ_PAGES_ID" },
2633 [MIG_RP_MSG_RECV_BITMAP] = { .len = -1, .name = "RECV_BITMAP" },
2634 [MIG_RP_MSG_RESUME_ACK] = { .len = 4, .name = "RESUME_ACK" },
2635 [MIG_RP_MSG_MAX] = { .len = -1, .name = "MAX" },
2636 };
2637
2638 /*
2639 * Process a request for pages received on the return path,
2640 * We're allowed to send more than requested (e.g. to round to our page size)
2641 * and we don't need to send pages that have already been sent.
2642 */
2643 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2644 ram_addr_t start, size_t len)
2645 {
2646 long our_host_ps = qemu_real_host_page_size;
2647
2648 trace_migrate_handle_rp_req_pages(rbname, start, len);
2649
2650 /*
2651 * Since we currently insist on matching page sizes, just sanity check
2652 * we're being asked for whole host pages.
2653 */
2654 if (!QEMU_IS_ALIGNED(start, our_host_ps) ||
2655 !QEMU_IS_ALIGNED(len, our_host_ps)) {
2656 error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2657 " len: %zd", __func__, start, len);
2658 mark_source_rp_bad(ms);
2659 return;
2660 }
2661
2662 if (ram_save_queue_pages(rbname, start, len)) {
2663 mark_source_rp_bad(ms);
2664 }
2665 }
2666
2667 /* Return true to retry, false to quit */
2668 static bool postcopy_pause_return_path_thread(MigrationState *s)
2669 {
2670 trace_postcopy_pause_return_path();
2671
2672 qemu_sem_wait(&s->postcopy_pause_rp_sem);
2673
2674 trace_postcopy_pause_return_path_continued();
2675
2676 return true;
2677 }
2678
2679 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2680 {
2681 RAMBlock *block = qemu_ram_block_by_name(block_name);
2682
2683 if (!block) {
2684 error_report("%s: invalid block name '%s'", __func__, block_name);
2685 return -EINVAL;
2686 }
2687
2688 /* Fetch the received bitmap and refresh the dirty bitmap */
2689 return ram_dirty_bitmap_reload(s, block);
2690 }
2691
2692 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2693 {
2694 trace_source_return_path_thread_resume_ack(value);
2695
2696 if (value != MIGRATION_RESUME_ACK_VALUE) {
2697 error_report("%s: illegal resume_ack value %"PRIu32,
2698 __func__, value);
2699 return -1;
2700 }
2701
2702 /* Now both sides are active. */
2703 migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2704 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2705
2706 /* Notify send thread that time to continue send pages */
2707 qemu_sem_post(&s->rp_state.rp_sem);
2708
2709 return 0;
2710 }
2711
2712 /* Release ms->rp_state.from_dst_file in a safe way */
2713 static void migration_release_from_dst_file(MigrationState *ms)
2714 {
2715 QEMUFile *file;
2716
2717 WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
2718 /*
2719 * Reset the from_dst_file pointer first before releasing it, as we
2720 * can't block within lock section
2721 */
2722 file = ms->rp_state.from_dst_file;
2723 ms->rp_state.from_dst_file = NULL;
2724 }
2725
2726 qemu_fclose(file);
2727 }
2728
2729 /*
2730 * Handles messages sent on the return path towards the source VM
2731 *
2732 */
2733 static void *source_return_path_thread(void *opaque)
2734 {
2735 MigrationState *ms = opaque;
2736 QEMUFile *rp = ms->rp_state.from_dst_file;
2737 uint16_t header_len, header_type;
2738 uint8_t buf[512];
2739 uint32_t tmp32, sibling_error;
2740 ram_addr_t start = 0; /* =0 to silence warning */
2741 size_t len = 0, expected_len;
2742 int res;
2743
2744 trace_source_return_path_thread_entry();
2745 rcu_register_thread();
2746
2747 retry:
2748 while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2749 migration_is_setup_or_active(ms->state)) {
2750 trace_source_return_path_thread_loop_top();
2751 header_type = qemu_get_be16(rp);
2752 header_len = qemu_get_be16(rp);
2753
2754 if (qemu_file_get_error(rp)) {
2755 mark_source_rp_bad(ms);
2756 goto out;
2757 }
2758
2759 if (header_type >= MIG_RP_MSG_MAX ||
2760 header_type == MIG_RP_MSG_INVALID) {
2761 error_report("RP: Received invalid message 0x%04x length 0x%04x",
2762 header_type, header_len);
2763 mark_source_rp_bad(ms);
2764 goto out;
2765 }
2766
2767 if ((rp_cmd_args[header_type].len != -1 &&
2768 header_len != rp_cmd_args[header_type].len) ||
2769 header_len > sizeof(buf)) {
2770 error_report("RP: Received '%s' message (0x%04x) with"
2771 "incorrect length %d expecting %zu",
2772 rp_cmd_args[header_type].name, header_type, header_len,
2773 (size_t)rp_cmd_args[header_type].len);
2774 mark_source_rp_bad(ms);
2775 goto out;
2776 }
2777
2778 /* We know we've got a valid header by this point */
2779 res = qemu_get_buffer(rp, buf, header_len);
2780 if (res != header_len) {
2781 error_report("RP: Failed reading data for message 0x%04x"
2782 " read %d expected %d",
2783 header_type, res, header_len);
2784 mark_source_rp_bad(ms);
2785 goto out;
2786 }
2787
2788 /* OK, we have the message and the data */
2789 switch (header_type) {
2790 case MIG_RP_MSG_SHUT:
2791 sibling_error = ldl_be_p(buf);
2792 trace_source_return_path_thread_shut(sibling_error);
2793 if (sibling_error) {
2794 error_report("RP: Sibling indicated error %d", sibling_error);
2795 mark_source_rp_bad(ms);
2796 }
2797 /*
2798 * We'll let the main thread deal with closing the RP
2799 * we could do a shutdown(2) on it, but we're the only user
2800 * anyway, so there's nothing gained.
2801 */
2802 goto out;
2803
2804 case MIG_RP_MSG_PONG:
2805 tmp32 = ldl_be_p(buf);
2806 trace_source_return_path_thread_pong(tmp32);
2807 break;
2808
2809 case MIG_RP_MSG_REQ_PAGES:
2810 start = ldq_be_p(buf);
2811 len = ldl_be_p(buf + 8);
2812 migrate_handle_rp_req_pages(ms, NULL, start, len);
2813 break;
2814
2815 case MIG_RP_MSG_REQ_PAGES_ID:
2816 expected_len = 12 + 1; /* header + termination */
2817
2818 if (header_len >= expected_len) {
2819 start = ldq_be_p(buf);
2820 len = ldl_be_p(buf + 8);
2821 /* Now we expect an idstr */
2822 tmp32 = buf[12]; /* Length of the following idstr */
2823 buf[13 + tmp32] = '\0';
2824 expected_len += tmp32;
2825 }
2826 if (header_len != expected_len) {
2827 error_report("RP: Req_Page_id with length %d expecting %zd",
2828 header_len, expected_len);
2829 mark_source_rp_bad(ms);
2830 goto out;
2831 }
2832 migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2833 break;
2834
2835 case MIG_RP_MSG_RECV_BITMAP:
2836 if (header_len < 1) {
2837 error_report("%s: missing block name", __func__);
2838 mark_source_rp_bad(ms);
2839 goto out;
2840 }
2841 /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2842 buf[buf[0] + 1] = '\0';
2843 if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2844 mark_source_rp_bad(ms);
2845 goto out;
2846 }
2847 break;
2848
2849 case MIG_RP_MSG_RESUME_ACK:
2850 tmp32 = ldl_be_p(buf);
2851 if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2852 mark_source_rp_bad(ms);
2853 goto out;
2854 }
2855 break;
2856
2857 default:
2858 break;
2859 }
2860 }
2861
2862 out:
2863 res = qemu_file_get_error(rp);
2864 if (res) {
2865 if (res == -EIO && migration_in_postcopy()) {
2866 /*
2867 * Maybe there is something we can do: it looks like a
2868 * network down issue, and we pause for a recovery.
2869 */
2870 migration_release_from_dst_file(ms);
2871 rp = NULL;
2872 if (postcopy_pause_return_path_thread(ms)) {
2873 /*
2874 * Reload rp, reset the rest. Referencing it is safe since
2875 * it's reset only by us above, or when migration completes
2876 */
2877 rp = ms->rp_state.from_dst_file;
2878 ms->rp_state.error = false;
2879 goto retry;
2880 }
2881 }
2882
2883 trace_source_return_path_thread_bad_end();
2884 mark_source_rp_bad(ms);
2885 }
2886
2887 trace_source_return_path_thread_end();
2888 migration_release_from_dst_file(ms);
2889 rcu_unregister_thread();
2890 return NULL;
2891 }
2892
2893 static int open_return_path_on_source(MigrationState *ms,
2894 bool create_thread)
2895 {
2896 ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2897 if (!ms->rp_state.from_dst_file) {
2898 return -1;
2899 }
2900
2901 trace_open_return_path_on_source();
2902
2903 if (!create_thread) {
2904 /* We're done */
2905 return 0;
2906 }
2907
2908 qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2909 source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2910 ms->rp_state.rp_thread_created = true;
2911
2912 trace_open_return_path_on_source_continue();
2913
2914 return 0;
2915 }
2916
2917 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2918 static int await_return_path_close_on_source(MigrationState *ms)
2919 {
2920 /*
2921 * If this is a normal exit then the destination will send a SHUT and the
2922 * rp_thread will exit, however if there's an error we need to cause
2923 * it to exit.
2924 */
2925 if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2926 /*
2927 * shutdown(2), if we have it, will cause it to unblock if it's stuck
2928 * waiting for the destination.
2929 */
2930 qemu_file_shutdown(ms->rp_state.from_dst_file);
2931 mark_source_rp_bad(ms);
2932 }
2933 trace_await_return_path_close_on_source_joining();
2934 qemu_thread_join(&ms->rp_state.rp_thread);
2935 ms->rp_state.rp_thread_created = false;
2936 trace_await_return_path_close_on_source_close();
2937 return ms->rp_state.error;
2938 }
2939
2940 /*
2941 * Switch from normal iteration to postcopy
2942 * Returns non-0 on error
2943 */
2944 static int postcopy_start(MigrationState *ms)
2945 {
2946 int ret;
2947 QIOChannelBuffer *bioc;
2948 QEMUFile *fb;
2949 int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2950 int64_t bandwidth = migrate_max_postcopy_bandwidth();
2951 bool restart_block = false;
2952 int cur_state = MIGRATION_STATUS_ACTIVE;
2953 if (!migrate_pause_before_switchover()) {
2954 migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2955 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2956 }
2957
2958 trace_postcopy_start();
2959 qemu_mutex_lock_iothread();
2960 trace_postcopy_start_set_run();
2961
2962 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2963 global_state_store();
2964 ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2965 if (ret < 0) {
2966 goto fail;
2967 }
2968
2969 ret = migration_maybe_pause(ms, &cur_state,
2970 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2971 if (ret < 0) {
2972 goto fail;
2973 }
2974
2975 ret = bdrv_inactivate_all();
2976 if (ret < 0) {
2977 goto fail;
2978 }
2979 restart_block = true;
2980
2981 /*
2982 * Cause any non-postcopiable, but iterative devices to
2983 * send out their final data.
2984 */
2985 qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2986
2987 /*
2988 * in Finish migrate and with the io-lock held everything should
2989 * be quiet, but we've potentially still got dirty pages and we
2990 * need to tell the destination to throw any pages it's already received
2991 * that are dirty
2992 */
2993 if (migrate_postcopy_ram()) {
2994 if (ram_postcopy_send_discard_bitmap(ms)) {
2995 error_report("postcopy send discard bitmap failed");
2996 goto fail;
2997 }
2998 }
2999
3000 /*
3001 * send rest of state - note things that are doing postcopy
3002 * will notice we're in POSTCOPY_ACTIVE and not actually
3003 * wrap their state up here
3004 */
3005 /* 0 max-postcopy-bandwidth means unlimited */
3006 if (!bandwidth) {
3007 qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
3008 } else {
3009 qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
3010 }
3011 if (migrate_postcopy_ram()) {
3012 /* Ping just for debugging, helps line traces up */
3013 qemu_savevm_send_ping(ms->to_dst_file, 2);
3014 }
3015
3016 /*
3017 * While loading the device state we may trigger page transfer
3018 * requests and the fd must be free to process those, and thus
3019 * the destination must read the whole device state off the fd before
3020 * it starts processing it. Unfortunately the ad-hoc migration format
3021 * doesn't allow the destination to know the size to read without fully
3022 * parsing it through each devices load-state code (especially the open
3023 * coded devices that use get/put).
3024 * So we wrap the device state up in a package with a length at the start;
3025 * to do this we use a qemu_buf to hold the whole of the device state.
3026 */
3027 bioc = qio_channel_buffer_new(4096);
3028 qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
3029 fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
3030 object_unref(OBJECT(bioc));
3031
3032 /*
3033 * Make sure the receiver can get incoming pages before we send the rest
3034 * of the state
3035 */
3036 qemu_savevm_send_postcopy_listen(fb);
3037
3038 qemu_savevm_state_complete_precopy(fb, false, false);
3039 if (migrate_postcopy_ram()) {
3040 qemu_savevm_send_ping(fb, 3);
3041 }
3042
3043 qemu_savevm_send_postcopy_run(fb);
3044
3045 /* <><> end of stuff going into the package */
3046
3047 /* Last point of recovery; as soon as we send the package the destination
3048 * can open devices and potentially start running.
3049 * Lets just check again we've not got any errors.
3050 */
3051 ret = qemu_file_get_error(ms->to_dst_file);
3052 if (ret) {
3053 error_report("postcopy_start: Migration stream errored (pre package)");
3054 goto fail_closefb;
3055 }
3056
3057 restart_block = false;
3058
3059 /* Now send that blob */
3060 if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
3061 goto fail_closefb;
3062 }
3063 qemu_fclose(fb);
3064
3065 /* Send a notify to give a chance for anything that needs to happen
3066 * at the transition to postcopy and after the device state; in particular
3067 * spice needs to trigger a transition now
3068 */
3069 ms->postcopy_after_devices = true;
3070 notifier_list_notify(&migration_state_notifiers, ms);
3071
3072 ms->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
3073
3074 qemu_mutex_unlock_iothread();
3075
3076 if (migrate_postcopy_ram()) {
3077 /*
3078 * Although this ping is just for debug, it could potentially be
3079 * used for getting a better measurement of downtime at the source.
3080 */
3081 qemu_savevm_send_ping(ms->to_dst_file, 4);
3082 }
3083
3084 if (migrate_release_ram()) {
3085 ram_postcopy_migrated_memory_release(ms);
3086 }
3087
3088 ret = qemu_file_get_error(ms->to_dst_file);
3089 if (ret) {
3090 error_report("postcopy_start: Migration stream errored");
3091 migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3092 MIGRATION_STATUS_FAILED);
3093 }
3094
3095 return ret;
3096
3097 fail_closefb:
3098 qemu_fclose(fb);
3099 fail:
3100 migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3101 MIGRATION_STATUS_FAILED);
3102 if (restart_block) {
3103 /* A failure happened early enough that we know the destination hasn't
3104 * accessed block devices, so we're safe to recover.
3105 */
3106 Error *local_err = NULL;
3107
3108 bdrv_invalidate_cache_all(&local_err);
3109 if (local_err) {
3110 error_report_err(local_err);
3111 }
3112 }
3113 qemu_mutex_unlock_iothread();
3114 return -1;
3115 }
3116
3117 /**
3118 * migration_maybe_pause: Pause if required to by
3119 * migrate_pause_before_switchover called with the iothread locked
3120 * Returns: 0 on success
3121 */
3122 static int migration_maybe_pause(MigrationState *s,
3123 int *current_active_state,
3124 int new_state)
3125 {
3126 if (!migrate_pause_before_switchover()) {
3127 return 0;
3128 }
3129
3130 /* Since leaving this state is not atomic with posting the semaphore
3131 * it's possible that someone could have issued multiple migrate_continue
3132 * and the semaphore is incorrectly positive at this point;
3133 * the docs say it's undefined to reinit a semaphore that's already
3134 * init'd, so use timedwait to eat up any existing posts.
3135 */
3136 while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
3137 /* This block intentionally left blank */
3138 }
3139
3140 /*
3141 * If the migration is cancelled when it is in the completion phase,
3142 * the migration state is set to MIGRATION_STATUS_CANCELLING.
3143 * So we don't need to wait a semaphore, otherwise we would always
3144 * wait for the 'pause_sem' semaphore.
3145 */
3146 if (s->state != MIGRATION_STATUS_CANCELLING) {
3147 qemu_mutex_unlock_iothread();
3148 migrate_set_state(&s->state, *current_active_state,
3149 MIGRATION_STATUS_PRE_SWITCHOVER);
3150 qemu_sem_wait(&s->pause_sem);
3151 migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
3152 new_state);
3153 *current_active_state = new_state;
3154 qemu_mutex_lock_iothread();
3155 }
3156
3157 return s->state == new_state ? 0 : -EINVAL;
3158 }
3159
3160 /**
3161 * migration_completion: Used by migration_thread when there's not much left.
3162 * The caller 'breaks' the loop when this returns.
3163 *
3164 * @s: Current migration state
3165 */
3166 static void migration_completion(MigrationState *s)
3167 {
3168 int ret;
3169 int current_active_state = s->state;
3170
3171 if (s->state == MIGRATION_STATUS_ACTIVE) {
3172 qemu_mutex_lock_iothread();
3173 s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3174 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3175 s->vm_was_running = runstate_is_running();
3176 ret = global_state_store();
3177
3178 if (!ret) {
3179 bool inactivate = !migrate_colo_enabled();
3180 ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3181 trace_migration_completion_vm_stop(ret);
3182 if (ret >= 0) {
3183 ret = migration_maybe_pause(s, &current_active_state,
3184 MIGRATION_STATUS_DEVICE);
3185 }
3186 if (ret >= 0) {
3187 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3188 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3189 inactivate);
3190 }
3191 if (inactivate && ret >= 0) {
3192 s->block_inactive = true;
3193 }
3194 }
3195 qemu_mutex_unlock_iothread();
3196
3197 if (ret < 0) {
3198 goto fail;
3199 }
3200 } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3201 trace_migration_completion_postcopy_end();
3202
3203 qemu_mutex_lock_iothread();
3204 qemu_savevm_state_complete_postcopy(s->to_dst_file);
3205 qemu_mutex_unlock_iothread();
3206
3207 trace_migration_completion_postcopy_end_after_complete();
3208 } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3209 goto fail;
3210 }
3211
3212 /*
3213 * If rp was opened we must clean up the thread before
3214 * cleaning everything else up (since if there are no failures
3215 * it will wait for the destination to send it's status in
3216 * a SHUT command).
3217 */
3218 if (s->rp_state.rp_thread_created) {
3219 int rp_error;
3220 trace_migration_return_path_end_before();
3221 rp_error = await_return_path_close_on_source(s);
3222 trace_migration_return_path_end_after(rp_error);
3223 if (rp_error) {
3224 goto fail_invalidate;
3225 }
3226 }
3227
3228 if (qemu_file_get_error(s->to_dst_file)) {
3229 trace_migration_completion_file_err();
3230 goto fail_invalidate;
3231 }
3232
3233 if (!migrate_colo_enabled()) {
3234 migrate_set_state(&s->state, current_active_state,
3235 MIGRATION_STATUS_COMPLETED);
3236 }
3237
3238 return;
3239
3240 fail_invalidate:
3241 /* If not doing postcopy, vm_start() will be called: let's regain
3242 * control on images.
3243 */
3244 if (s->state == MIGRATION_STATUS_ACTIVE ||
3245 s->state == MIGRATION_STATUS_DEVICE) {
3246 Error *local_err = NULL;
3247
3248 qemu_mutex_lock_iothread();
3249 bdrv_invalidate_cache_all(&local_err);
3250 if (local_err) {
3251 error_report_err(local_err);
3252 } else {
3253 s->block_inactive = false;
3254 }
3255 qemu_mutex_unlock_iothread();
3256 }
3257
3258 fail:
3259 migrate_set_state(&s->state, current_active_state,
3260 MIGRATION_STATUS_FAILED);
3261 }
3262
3263 /**
3264 * bg_migration_completion: Used by bg_migration_thread when after all the
3265 * RAM has been saved. The caller 'breaks' the loop when this returns.
3266 *
3267 * @s: Current migration state
3268 */
3269 static void bg_migration_completion(MigrationState *s)
3270 {
3271 int current_active_state = s->state;
3272
3273 /*
3274 * Stop tracking RAM writes - un-protect memory, un-register UFFD
3275 * memory ranges, flush kernel wait queues and wake up threads
3276 * waiting for write fault to be resolved.
3277 */
3278 ram_write_tracking_stop();
3279
3280 if (s->state == MIGRATION_STATUS_ACTIVE) {
3281 /*
3282 * By this moment we have RAM content saved into the migration stream.
3283 * The next step is to flush the non-RAM content (device state)
3284 * right after the ram content. The device state has been stored into
3285 * the temporary buffer before RAM saving started.
3286 */
3287 qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
3288 qemu_fflush(s->to_dst_file);
3289 } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3290 goto fail;
3291 }
3292
3293 if (qemu_file_get_error(s->to_dst_file)) {
3294 trace_migration_completion_file_err();
3295 goto fail;
3296 }
3297
3298 migrate_set_state(&s->state, current_active_state,
3299 MIGRATION_STATUS_COMPLETED);
3300 return;
3301
3302 fail:
3303 migrate_set_state(&s->state, current_active_state,
3304 MIGRATION_STATUS_FAILED);
3305 }
3306
3307 bool migrate_colo_enabled(void)
3308 {
3309 MigrationState *s = migrate_get_current();
3310 return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3311 }
3312
3313 typedef enum MigThrError {
3314 /* No error detected */
3315 MIG_THR_ERR_NONE = 0,
3316 /* Detected error, but resumed successfully */
3317 MIG_THR_ERR_RECOVERED = 1,
3318 /* Detected fatal error, need to exit */
3319 MIG_THR_ERR_FATAL = 2,
3320 } MigThrError;
3321
3322 static int postcopy_resume_handshake(MigrationState *s)
3323 {
3324 qemu_savevm_send_postcopy_resume(s->to_dst_file);
3325
3326 while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3327 qemu_sem_wait(&s->rp_state.rp_sem);
3328 }
3329
3330 if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3331 return 0;
3332 }
3333
3334 return -1;
3335 }
3336
3337 /* Return zero if success, or <0 for error */
3338 static int postcopy_do_resume(MigrationState *s)
3339 {
3340 int ret;
3341
3342 /*
3343 * Call all the resume_prepare() hooks, so that modules can be
3344 * ready for the migration resume.
3345 */
3346 ret = qemu_savevm_state_resume_prepare(s);
3347 if (ret) {
3348 error_report("%s: resume_prepare() failure detected: %d",
3349 __func__, ret);
3350 return ret;
3351 }
3352
3353 /*
3354 * Last handshake with destination on the resume (destination will
3355 * switch to postcopy-active afterwards)
3356 */
3357 ret = postcopy_resume_handshake(s);
3358 if (ret) {
3359 error_report("%s: handshake failed: %d", __func__, ret);
3360 return ret;
3361 }
3362
3363 return 0;
3364 }
3365
3366 /*
3367 * We don't return until we are in a safe state to continue current
3368 * postcopy migration. Returns MIG_THR_ERR_RECOVERED if recovered, or
3369 * MIG_THR_ERR_FATAL if unrecovery failure happened.
3370 */
3371 static MigThrError postcopy_pause(MigrationState *s)
3372 {
3373 assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3374
3375 while (true) {
3376 QEMUFile *file;
3377
3378 /*
3379 * Current channel is possibly broken. Release it. Note that this is
3380 * guaranteed even without lock because to_dst_file should only be
3381 * modified by the migration thread. That also guarantees that the
3382 * unregister of yank is safe too without the lock. It should be safe
3383 * even to be within the qemu_file_lock, but we didn't do that to avoid
3384 * taking more mutex (yank_lock) within qemu_file_lock. TL;DR: we make
3385 * the qemu_file_lock critical section as small as possible.
3386 */
3387 assert(s->to_dst_file);
3388 migration_ioc_unregister_yank_from_file(s->to_dst_file);
3389 qemu_mutex_lock(&s->qemu_file_lock);
3390 file = s->to_dst_file;
3391 s->to_dst_file = NULL;
3392 qemu_mutex_unlock(&s->qemu_file_lock);
3393
3394 qemu_file_shutdown(file);
3395 qemu_fclose(file);
3396
3397 migrate_set_state(&s->state, s->state,
3398 MIGRATION_STATUS_POSTCOPY_PAUSED);
3399
3400 error_report("Detected IO failure for postcopy. "
3401 "Migration paused.");
3402
3403 /*
3404 * We wait until things fixed up. Then someone will setup the
3405 * status back for us.
3406 */
3407 while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3408 qemu_sem_wait(&s->postcopy_pause_sem);
3409 }
3410
3411 if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3412 /* Woken up by a recover procedure. Give it a shot */
3413
3414 /*
3415 * Firstly, let's wake up the return path now, with a new
3416 * return path channel.
3417 */
3418 qemu_sem_post(&s->postcopy_pause_rp_sem);
3419
3420 /* Do the resume logic */
3421 if (postcopy_do_resume(s) == 0) {
3422 /* Let's continue! */
3423 trace_postcopy_pause_continued();
3424 return MIG_THR_ERR_RECOVERED;
3425 } else {
3426 /*
3427 * Something wrong happened during the recovery, let's
3428 * pause again. Pause is always better than throwing
3429 * data away.
3430 */
3431 continue;
3432 }
3433 } else {
3434 /* This is not right... Time to quit. */
3435 return MIG_THR_ERR_FATAL;
3436 }
3437 }
3438 }
3439
3440 static MigThrError migration_detect_error(MigrationState *s)
3441 {
3442 int ret;
3443 int state = s->state;
3444 Error *local_error = NULL;
3445
3446 if (state == MIGRATION_STATUS_CANCELLING ||
3447 state == MIGRATION_STATUS_CANCELLED) {
3448 /* End the migration, but don't set the state to failed */
3449 return MIG_THR_ERR_FATAL;
3450 }
3451
3452 /* Try to detect any file errors */
3453 ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3454 if (!ret) {
3455 /* Everything is fine */
3456 assert(!local_error);
3457 return MIG_THR_ERR_NONE;
3458 }
3459
3460 if (local_error) {
3461 migrate_set_error(s, local_error);
3462 error_free(local_error);
3463 }
3464
3465 if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) {
3466 /*
3467 * For postcopy, we allow the network to be down for a
3468 * while. After that, it can be continued by a
3469 * recovery phase.
3470 */
3471 return postcopy_pause(s);
3472 } else {
3473 /*
3474 * For precopy (or postcopy with error outside IO), we fail
3475 * with no time.
3476 */
3477 migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3478 trace_migration_thread_file_err();
3479
3480 /* Time to stop the migration, now. */
3481 return MIG_THR_ERR_FATAL;
3482 }
3483 }
3484
3485 /* How many bytes have we transferred since the beginning of the migration */
3486 static uint64_t migration_total_bytes(MigrationState *s)
3487 {
3488 return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3489 }
3490
3491 static void migration_calculate_complete(MigrationState *s)
3492 {
3493 uint64_t bytes = migration_total_bytes(s);
3494 int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3495 int64_t transfer_time;
3496
3497 s->total_time = end_time - s->start_time;
3498 if (!s->downtime) {
3499 /*
3500 * It's still not set, so we are precopy migration. For
3501 * postcopy, downtime is calculated during postcopy_start().
3502 */
3503 s->downtime = end_time - s->downtime_start;
3504 }
3505
3506 transfer_time = s->total_time - s->setup_time;
3507 if (transfer_time) {
3508 s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3509 }
3510 }
3511
3512 static void update_iteration_initial_status(MigrationState *s)
3513 {
3514 /*
3515 * Update these three fields at the same time to avoid mismatch info lead
3516 * wrong speed calculation.
3517 */
3518 s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3519 s->iteration_initial_bytes = migration_total_bytes(s);
3520 s->iteration_initial_pages = ram_get_total_transferred_pages();
3521 }
3522
3523 static void migration_update_counters(MigrationState *s,
3524 int64_t current_time)
3525 {
3526 uint64_t transferred, transferred_pages, time_spent;
3527 uint64_t current_bytes; /* bytes transferred since the beginning */
3528 double bandwidth;
3529
3530 if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3531 return;
3532 }
3533
3534 current_bytes = migration_total_bytes(s);
3535 transferred = current_bytes - s->iteration_initial_bytes;
3536 time_spent = current_time - s->iteration_start_time;
3537 bandwidth = (double)transferred / time_spent;
3538 s->threshold_size = bandwidth * s->parameters.downtime_limit;
3539
3540 s->mbps = (((double) transferred * 8.0) /
3541 ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3542
3543 transferred_pages = ram_get_total_transferred_pages() -
3544 s->iteration_initial_pages;
3545 s->pages_per_second = (double) transferred_pages /
3546 (((double) time_spent / 1000.0));
3547
3548 /*
3549 * if we haven't sent anything, we don't want to
3550 * recalculate. 10000 is a small enough number for our purposes
3551 */
3552 if (ram_counters.dirty_pages_rate && transferred > 10000) {
3553 s->expected_downtime = ram_counters.remaining / bandwidth;
3554 }
3555
3556 qemu_file_reset_rate_limit(s->to_dst_file);
3557
3558 update_iteration_initial_status(s);
3559
3560 trace_migrate_transferred(transferred, time_spent,
3561 bandwidth, s->threshold_size);
3562 }
3563
3564 /* Migration thread iteration status */
3565 typedef enum {
3566 MIG_ITERATE_RESUME, /* Resume current iteration */
3567 MIG_ITERATE_SKIP, /* Skip current iteration */
3568 MIG_ITERATE_BREAK, /* Break the loop */
3569 } MigIterateState;
3570
3571 /*
3572 * Return true if continue to the next iteration directly, false
3573 * otherwise.
3574 */
3575 static MigIterateState migration_iteration_run(MigrationState *s)
3576 {
3577 uint64_t pending_size, pend_pre, pend_compat, pend_post;
3578 bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3579
3580 qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3581 &pend_compat, &pend_post);
3582 pending_size = pend_pre + pend_compat + pend_post;
3583
3584 trace_migrate_pending(pending_size, s->threshold_size,
3585 pend_pre, pend_compat, pend_post);
3586
3587 if (pending_size && pending_size >= s->threshold_size) {
3588 /* Still a significant amount to transfer */
3589 if (!in_postcopy && pend_pre <= s->threshold_size &&
3590 qatomic_read(&s->start_postcopy)) {
3591 if (postcopy_start(s)) {
3592 error_report("%s: postcopy failed to start", __func__);
3593 }
3594 return MIG_ITERATE_SKIP;
3595 }
3596 /* Just another iteration step */
3597 qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3598 } else {
3599 trace_migration_thread_low_pending(pending_size);
3600 migration_completion(s);
3601 return MIG_ITERATE_BREAK;
3602 }
3603
3604 return MIG_ITERATE_RESUME;
3605 }
3606
3607 static void migration_iteration_finish(MigrationState *s)
3608 {
3609 /* If we enabled cpu throttling for auto-converge, turn it off. */
3610 cpu_throttle_stop();
3611
3612 qemu_mutex_lock_iothread();
3613 switch (s->state) {
3614 case MIGRATION_STATUS_COMPLETED:
3615 migration_calculate_complete(s);
3616 runstate_set(RUN_STATE_POSTMIGRATE);
3617 break;
3618 case MIGRATION_STATUS_COLO:
3619 if (!migrate_colo_enabled()) {
3620 error_report("%s: critical error: calling COLO code without "
3621 "COLO enabled", __func__);
3622 }
3623 migrate_start_colo_process(s);
3624 /*
3625 * Fixme: we will run VM in COLO no matter its old running state.
3626 * After exited COLO, we will keep running.
3627 */
3628 /* Fallthrough */
3629 case MIGRATION_STATUS_ACTIVE:
3630 /*
3631 * We should really assert here, but since it's during
3632 * migration, let's try to reduce the usage of assertions.
3633 */
3634 s->vm_was_running = true;
3635 /* Fallthrough */
3636 case MIGRATION_STATUS_FAILED:
3637 case MIGRATION_STATUS_CANCELLED:
3638 case MIGRATION_STATUS_CANCELLING:
3639 if (s->vm_was_running) {
3640 if (!runstate_check(RUN_STATE_SHUTDOWN)) {
3641 vm_start();
3642 }
3643 } else {
3644 if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3645 runstate_set(RUN_STATE_POSTMIGRATE);
3646 }
3647 }
3648 break;
3649
3650 default:
3651 /* Should not reach here, but if so, forgive the VM. */
3652 error_report("%s: Unknown ending state %d", __func__, s->state);
3653 break;
3654 }
3655 migrate_fd_cleanup_schedule(s);
3656 qemu_mutex_unlock_iothread();
3657 }
3658
3659 static void bg_migration_iteration_finish(MigrationState *s)
3660 {
3661 qemu_mutex_lock_iothread();
3662 switch (s->state) {
3663 case MIGRATION_STATUS_COMPLETED:
3664 migration_calculate_complete(s);
3665 break;
3666
3667 case MIGRATION_STATUS_ACTIVE:
3668 case MIGRATION_STATUS_FAILED:
3669 case MIGRATION_STATUS_CANCELLED:
3670 case MIGRATION_STATUS_CANCELLING:
3671 break;
3672
3673 default:
3674 /* Should not reach here, but if so, forgive the VM. */
3675 error_report("%s: Unknown ending state %d", __func__, s->state);
3676 break;
3677 }
3678
3679 migrate_fd_cleanup_schedule(s);
3680 qemu_mutex_unlock_iothread();
3681 }
3682
3683 /*
3684 * Return true if continue to the next iteration directly, false
3685 * otherwise.
3686 */
3687 static MigIterateState bg_migration_iteration_run(MigrationState *s)
3688 {
3689 int res;
3690
3691 res = qemu_savevm_state_iterate(s->to_dst_file, false);
3692 if (res > 0) {
3693 bg_migration_completion(s);
3694 return MIG_ITERATE_BREAK;
3695 }
3696
3697 return MIG_ITERATE_RESUME;
3698 }
3699
3700 void migration_make_urgent_request(void)
3701 {
3702 qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3703 }
3704
3705 void migration_consume_urgent_request(void)
3706 {
3707 qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3708 }
3709
3710 /* Returns true if the rate limiting was broken by an urgent request */
3711 bool migration_rate_limit(void)
3712 {
3713 int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3714 MigrationState *s = migrate_get_current();
3715
3716 bool urgent = false;
3717 migration_update_counters(s, now);
3718 if (qemu_file_rate_limit(s->to_dst_file)) {
3719
3720 if (qemu_file_get_error(s->to_dst_file)) {
3721 return false;
3722 }
3723 /*
3724 * Wait for a delay to do rate limiting OR
3725 * something urgent to post the semaphore.
3726 */
3727 int ms = s->iteration_start_time + BUFFER_DELAY - now;
3728 trace_migration_rate_limit_pre(ms);
3729 if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3730 /*
3731 * We were woken by one or more urgent things but
3732 * the timedwait will have consumed one of them.
3733 * The service routine for the urgent wake will dec
3734 * the semaphore itself for each item it consumes,
3735 * so add this one we just eat back.
3736 */
3737 qemu_sem_post(&s->rate_limit_sem);
3738 urgent = true;
3739 }
3740 trace_migration_rate_limit_post(urgent);
3741 }
3742 return urgent;
3743 }
3744
3745 /*
3746 * if failover devices are present, wait they are completely
3747 * unplugged
3748 */
3749
3750 static void qemu_savevm_wait_unplug(MigrationState *s, int old_state,
3751 int new_state)
3752 {
3753 if (qemu_savevm_state_guest_unplug_pending()) {
3754 migrate_set_state(&s->state, old_state, MIGRATION_STATUS_WAIT_UNPLUG);
3755
3756 while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3757 qemu_savevm_state_guest_unplug_pending()) {
3758 qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3759 }
3760 if (s->state != MIGRATION_STATUS_WAIT_UNPLUG) {
3761 int timeout = 120; /* 30 seconds */
3762 /*
3763 * migration has been canceled
3764 * but as we have started an unplug we must wait the end
3765 * to be able to plug back the card
3766 */
3767 while (timeout-- && qemu_savevm_state_guest_unplug_pending()) {
3768 qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3769 }
3770 if (qemu_savevm_state_guest_unplug_pending() &&
3771 !qtest_enabled()) {
3772 warn_report("migration: partially unplugged device on "
3773 "failure");
3774 }
3775 }
3776
3777 migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG, new_state);
3778 } else {
3779 migrate_set_state(&s->state, old_state, new_state);
3780 }
3781 }
3782
3783 /*
3784 * Master migration thread on the source VM.
3785 * It drives the migration and pumps the data down the outgoing channel.
3786 */
3787 static void *migration_thread(void *opaque)
3788 {
3789 MigrationState *s = opaque;
3790 int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3791 MigThrError thr_error;
3792 bool urgent = false;
3793
3794 rcu_register_thread();
3795
3796 object_ref(OBJECT(s));
3797 update_iteration_initial_status(s);
3798
3799 qemu_savevm_state_header(s->to_dst_file);
3800
3801 /*
3802 * If we opened the return path, we need to make sure dst has it
3803 * opened as well.
3804 */
3805 if (s->rp_state.rp_thread_created) {
3806 /* Now tell the dest that it should open its end so it can reply */
3807 qemu_savevm_send_open_return_path(s->to_dst_file);
3808
3809 /* And do a ping that will make stuff easier to debug */
3810 qemu_savevm_send_ping(s->to_dst_file, 1);
3811 }
3812
3813 if (migrate_postcopy()) {
3814 /*
3815 * Tell the destination that we *might* want to do postcopy later;
3816 * if the other end can't do postcopy it should fail now, nice and
3817 * early.
3818 */
3819 qemu_savevm_send_postcopy_advise(s->to_dst_file);
3820 }
3821
3822 if (migrate_colo_enabled()) {
3823 /* Notify migration destination that we enable COLO */
3824 qemu_savevm_send_colo_enable(s->to_dst_file);
3825 }
3826
3827 qemu_savevm_state_setup(s->to_dst_file);
3828
3829 qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3830 MIGRATION_STATUS_ACTIVE);
3831
3832 s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3833
3834 trace_migration_thread_setup_complete();
3835
3836 while (migration_is_active(s)) {
3837 if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3838 MigIterateState iter_state = migration_iteration_run(s);
3839 if (iter_state == MIG_ITERATE_SKIP) {
3840 continue;
3841 } else if (iter_state == MIG_ITERATE_BREAK) {
3842 break;
3843 }
3844 }
3845
3846 /*
3847 * Try to detect any kind of failures, and see whether we
3848 * should stop the migration now.
3849 */
3850 thr_error = migration_detect_error(s);
3851 if (thr_error == MIG_THR_ERR_FATAL) {
3852 /* Stop migration */
3853 break;
3854 } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3855 /*
3856 * Just recovered from a e.g. network failure, reset all
3857 * the local variables. This is important to avoid
3858 * breaking transferred_bytes and bandwidth calculation
3859 */
3860 update_iteration_initial_status(s);
3861 }
3862
3863 urgent = migration_rate_limit();
3864 }
3865
3866 trace_migration_thread_after_loop();
3867 migration_iteration_finish(s);
3868 object_unref(OBJECT(s));
3869 rcu_unregister_thread();
3870 return NULL;
3871 }
3872
3873 static void bg_migration_vm_start_bh(void *opaque)
3874 {
3875 MigrationState *s = opaque;
3876
3877 qemu_bh_delete(s->vm_start_bh);
3878 s->vm_start_bh = NULL;
3879
3880 vm_start();
3881 s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start;
3882 }
3883
3884 /**
3885 * Background snapshot thread, based on live migration code.
3886 * This is an alternative implementation of live migration mechanism
3887 * introduced specifically to support background snapshots.
3888 *
3889 * It takes advantage of userfault_fd write protection mechanism introduced
3890 * in v5.7 kernel. Compared to existing dirty page logging migration much
3891 * lesser stream traffic is produced resulting in smaller snapshot images,
3892 * simply cause of no page duplicates can get into the stream.
3893 *
3894 * Another key point is that generated vmstate stream reflects machine state
3895 * 'frozen' at the beginning of snapshot creation compared to dirty page logging
3896 * mechanism, which effectively results in that saved snapshot is the state of VM
3897 * at the end of the process.
3898 */
3899 static void *bg_migration_thread(void *opaque)
3900 {
3901 MigrationState *s = opaque;
3902 int64_t setup_start;
3903 MigThrError thr_error;
3904 QEMUFile *fb;
3905 bool early_fail = true;
3906
3907 rcu_register_thread();
3908 object_ref(OBJECT(s));
3909
3910 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3911
3912 setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3913 /*
3914 * We want to save vmstate for the moment when migration has been
3915 * initiated but also we want to save RAM content while VM is running.
3916 * The RAM content should appear first in the vmstate. So, we first
3917 * stash the non-RAM part of the vmstate to the temporary buffer,
3918 * then write RAM part of the vmstate to the migration stream
3919 * with vCPUs running and, finally, write stashed non-RAM part of
3920 * the vmstate from the buffer to the migration stream.
3921 */