block: Add error handling to bdrv_invalidate_cache()
[qemu.git] / migration.c
1 /*
2 * QEMU live migration
3 *
4 * Copyright IBM, Corp. 2008
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
14 */
15
16 #include "qemu-common.h"
17 #include "qemu/main-loop.h"
18 #include "migration/migration.h"
19 #include "monitor/monitor.h"
20 #include "migration/qemu-file.h"
21 #include "sysemu/sysemu.h"
22 #include "block/block.h"
23 #include "qemu/sockets.h"
24 #include "migration/block.h"
25 #include "qemu/thread.h"
26 #include "qmp-commands.h"
27 #include "trace.h"
28
29 //#define DEBUG_MIGRATION
30
31 #ifdef DEBUG_MIGRATION
32 #define DPRINTF(fmt, ...) \
33 do { printf("migration: " fmt, ## __VA_ARGS__); } while (0)
34 #else
35 #define DPRINTF(fmt, ...) \
36 do { } while (0)
37 #endif
38
39 enum {
40 MIG_STATE_ERROR = -1,
41 MIG_STATE_NONE,
42 MIG_STATE_SETUP,
43 MIG_STATE_CANCELLING,
44 MIG_STATE_CANCELLED,
45 MIG_STATE_ACTIVE,
46 MIG_STATE_COMPLETED,
47 };
48
49 #define MAX_THROTTLE (32 << 20) /* Migration speed throttling */
50
51 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
52 * data. */
53 #define BUFFER_DELAY 100
54 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
55
56 /* Migration XBZRLE default cache size */
57 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
58
59 static NotifierList migration_state_notifiers =
60 NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
61
62 /* When we add fault tolerance, we could have several
63 migrations at once. For now we don't need to add
64 dynamic creation of migration */
65
66 MigrationState *migrate_get_current(void)
67 {
68 static MigrationState current_migration = {
69 .state = MIG_STATE_NONE,
70 .bandwidth_limit = MAX_THROTTLE,
71 .xbzrle_cache_size = DEFAULT_MIGRATE_CACHE_SIZE,
72 .mbps = -1,
73 };
74
75 return &current_migration;
76 }
77
78 void qemu_start_incoming_migration(const char *uri, Error **errp)
79 {
80 const char *p;
81
82 if (strstart(uri, "tcp:", &p))
83 tcp_start_incoming_migration(p, errp);
84 #ifdef CONFIG_RDMA
85 else if (strstart(uri, "rdma:", &p))
86 rdma_start_incoming_migration(p, errp);
87 #endif
88 #if !defined(WIN32)
89 else if (strstart(uri, "exec:", &p))
90 exec_start_incoming_migration(p, errp);
91 else if (strstart(uri, "unix:", &p))
92 unix_start_incoming_migration(p, errp);
93 else if (strstart(uri, "fd:", &p))
94 fd_start_incoming_migration(p, errp);
95 #endif
96 else {
97 error_setg(errp, "unknown migration protocol: %s", uri);
98 }
99 }
100
101 static void process_incoming_migration_co(void *opaque)
102 {
103 QEMUFile *f = opaque;
104 Error *local_err = NULL;
105 int ret;
106
107 ret = qemu_loadvm_state(f);
108 qemu_fclose(f);
109 free_xbzrle_decoded_buf();
110 if (ret < 0) {
111 fprintf(stderr, "load of migration failed\n");
112 exit(EXIT_FAILURE);
113 }
114 qemu_announce_self();
115 DPRINTF("successfully loaded vm state\n");
116
117 bdrv_clear_incoming_migration_all();
118 /* Make sure all file formats flush their mutable metadata */
119 bdrv_invalidate_cache_all(&local_err);
120 if (local_err) {
121 qerror_report_err(local_err);
122 error_free(local_err);
123 exit(EXIT_FAILURE);
124 }
125
126 if (autostart) {
127 vm_start();
128 } else {
129 runstate_set(RUN_STATE_PAUSED);
130 }
131 }
132
133 void process_incoming_migration(QEMUFile *f)
134 {
135 Coroutine *co = qemu_coroutine_create(process_incoming_migration_co);
136 int fd = qemu_get_fd(f);
137
138 assert(fd != -1);
139 qemu_set_nonblock(fd);
140 qemu_coroutine_enter(co, f);
141 }
142
143 /* amount of nanoseconds we are willing to wait for migration to be down.
144 * the choice of nanoseconds is because it is the maximum resolution that
145 * get_clock() can achieve. It is an internal measure. All user-visible
146 * units must be in seconds */
147 static uint64_t max_downtime = 30000000;
148
149 uint64_t migrate_max_downtime(void)
150 {
151 return max_downtime;
152 }
153
154 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
155 {
156 MigrationCapabilityStatusList *head = NULL;
157 MigrationCapabilityStatusList *caps;
158 MigrationState *s = migrate_get_current();
159 int i;
160
161 caps = NULL; /* silence compiler warning */
162 for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) {
163 if (head == NULL) {
164 head = g_malloc0(sizeof(*caps));
165 caps = head;
166 } else {
167 caps->next = g_malloc0(sizeof(*caps));
168 caps = caps->next;
169 }
170 caps->value =
171 g_malloc(sizeof(*caps->value));
172 caps->value->capability = i;
173 caps->value->state = s->enabled_capabilities[i];
174 }
175
176 return head;
177 }
178
179 static void get_xbzrle_cache_stats(MigrationInfo *info)
180 {
181 if (migrate_use_xbzrle()) {
182 info->has_xbzrle_cache = true;
183 info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
184 info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
185 info->xbzrle_cache->bytes = xbzrle_mig_bytes_transferred();
186 info->xbzrle_cache->pages = xbzrle_mig_pages_transferred();
187 info->xbzrle_cache->cache_miss = xbzrle_mig_pages_cache_miss();
188 info->xbzrle_cache->overflow = xbzrle_mig_pages_overflow();
189 }
190 }
191
192 MigrationInfo *qmp_query_migrate(Error **errp)
193 {
194 MigrationInfo *info = g_malloc0(sizeof(*info));
195 MigrationState *s = migrate_get_current();
196
197 switch (s->state) {
198 case MIG_STATE_NONE:
199 /* no migration has happened ever */
200 break;
201 case MIG_STATE_SETUP:
202 info->has_status = true;
203 info->status = g_strdup("setup");
204 info->has_total_time = false;
205 break;
206 case MIG_STATE_ACTIVE:
207 case MIG_STATE_CANCELLING:
208 info->has_status = true;
209 info->status = g_strdup("active");
210 info->has_total_time = true;
211 info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME)
212 - s->total_time;
213 info->has_expected_downtime = true;
214 info->expected_downtime = s->expected_downtime;
215 info->has_setup_time = true;
216 info->setup_time = s->setup_time;
217
218 info->has_ram = true;
219 info->ram = g_malloc0(sizeof(*info->ram));
220 info->ram->transferred = ram_bytes_transferred();
221 info->ram->remaining = ram_bytes_remaining();
222 info->ram->total = ram_bytes_total();
223 info->ram->duplicate = dup_mig_pages_transferred();
224 info->ram->skipped = skipped_mig_pages_transferred();
225 info->ram->normal = norm_mig_pages_transferred();
226 info->ram->normal_bytes = norm_mig_bytes_transferred();
227 info->ram->dirty_pages_rate = s->dirty_pages_rate;
228 info->ram->mbps = s->mbps;
229
230 if (blk_mig_active()) {
231 info->has_disk = true;
232 info->disk = g_malloc0(sizeof(*info->disk));
233 info->disk->transferred = blk_mig_bytes_transferred();
234 info->disk->remaining = blk_mig_bytes_remaining();
235 info->disk->total = blk_mig_bytes_total();
236 }
237
238 get_xbzrle_cache_stats(info);
239 break;
240 case MIG_STATE_COMPLETED:
241 get_xbzrle_cache_stats(info);
242
243 info->has_status = true;
244 info->status = g_strdup("completed");
245 info->has_total_time = true;
246 info->total_time = s->total_time;
247 info->has_downtime = true;
248 info->downtime = s->downtime;
249 info->has_setup_time = true;
250 info->setup_time = s->setup_time;
251
252 info->has_ram = true;
253 info->ram = g_malloc0(sizeof(*info->ram));
254 info->ram->transferred = ram_bytes_transferred();
255 info->ram->remaining = 0;
256 info->ram->total = ram_bytes_total();
257 info->ram->duplicate = dup_mig_pages_transferred();
258 info->ram->skipped = skipped_mig_pages_transferred();
259 info->ram->normal = norm_mig_pages_transferred();
260 info->ram->normal_bytes = norm_mig_bytes_transferred();
261 info->ram->mbps = s->mbps;
262 break;
263 case MIG_STATE_ERROR:
264 info->has_status = true;
265 info->status = g_strdup("failed");
266 break;
267 case MIG_STATE_CANCELLED:
268 info->has_status = true;
269 info->status = g_strdup("cancelled");
270 break;
271 }
272
273 return info;
274 }
275
276 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
277 Error **errp)
278 {
279 MigrationState *s = migrate_get_current();
280 MigrationCapabilityStatusList *cap;
281
282 if (s->state == MIG_STATE_ACTIVE || s->state == MIG_STATE_SETUP) {
283 error_set(errp, QERR_MIGRATION_ACTIVE);
284 return;
285 }
286
287 for (cap = params; cap; cap = cap->next) {
288 s->enabled_capabilities[cap->value->capability] = cap->value->state;
289 }
290 }
291
292 /* shared migration helpers */
293
294 static void migrate_set_state(MigrationState *s, int old_state, int new_state)
295 {
296 if (atomic_cmpxchg(&s->state, old_state, new_state) == new_state) {
297 trace_migrate_set_state(new_state);
298 }
299 }
300
301 static void migrate_fd_cleanup(void *opaque)
302 {
303 MigrationState *s = opaque;
304
305 qemu_bh_delete(s->cleanup_bh);
306 s->cleanup_bh = NULL;
307
308 if (s->file) {
309 DPRINTF("closing file\n");
310 qemu_mutex_unlock_iothread();
311 qemu_thread_join(&s->thread);
312 qemu_mutex_lock_iothread();
313
314 qemu_fclose(s->file);
315 s->file = NULL;
316 }
317
318 assert(s->state != MIG_STATE_ACTIVE);
319
320 if (s->state != MIG_STATE_COMPLETED) {
321 qemu_savevm_state_cancel();
322 if (s->state == MIG_STATE_CANCELLING) {
323 migrate_set_state(s, MIG_STATE_CANCELLING, MIG_STATE_CANCELLED);
324 }
325 }
326
327 notifier_list_notify(&migration_state_notifiers, s);
328 }
329
330 void migrate_fd_error(MigrationState *s)
331 {
332 DPRINTF("setting error state\n");
333 assert(s->file == NULL);
334 s->state = MIG_STATE_ERROR;
335 trace_migrate_set_state(MIG_STATE_ERROR);
336 notifier_list_notify(&migration_state_notifiers, s);
337 }
338
339 static void migrate_fd_cancel(MigrationState *s)
340 {
341 int old_state ;
342 DPRINTF("cancelling migration\n");
343
344 do {
345 old_state = s->state;
346 if (old_state != MIG_STATE_SETUP && old_state != MIG_STATE_ACTIVE) {
347 break;
348 }
349 migrate_set_state(s, old_state, MIG_STATE_CANCELLING);
350 } while (s->state != MIG_STATE_CANCELLING);
351 }
352
353 void add_migration_state_change_notifier(Notifier *notify)
354 {
355 notifier_list_add(&migration_state_notifiers, notify);
356 }
357
358 void remove_migration_state_change_notifier(Notifier *notify)
359 {
360 notifier_remove(notify);
361 }
362
363 bool migration_in_setup(MigrationState *s)
364 {
365 return s->state == MIG_STATE_SETUP;
366 }
367
368 bool migration_has_finished(MigrationState *s)
369 {
370 return s->state == MIG_STATE_COMPLETED;
371 }
372
373 bool migration_has_failed(MigrationState *s)
374 {
375 return (s->state == MIG_STATE_CANCELLED ||
376 s->state == MIG_STATE_ERROR);
377 }
378
379 static MigrationState *migrate_init(const MigrationParams *params)
380 {
381 MigrationState *s = migrate_get_current();
382 int64_t bandwidth_limit = s->bandwidth_limit;
383 bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
384 int64_t xbzrle_cache_size = s->xbzrle_cache_size;
385
386 memcpy(enabled_capabilities, s->enabled_capabilities,
387 sizeof(enabled_capabilities));
388
389 memset(s, 0, sizeof(*s));
390 s->params = *params;
391 memcpy(s->enabled_capabilities, enabled_capabilities,
392 sizeof(enabled_capabilities));
393 s->xbzrle_cache_size = xbzrle_cache_size;
394
395 s->bandwidth_limit = bandwidth_limit;
396 s->state = MIG_STATE_SETUP;
397 trace_migrate_set_state(MIG_STATE_SETUP);
398
399 s->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
400 return s;
401 }
402
403 static GSList *migration_blockers;
404
405 void migrate_add_blocker(Error *reason)
406 {
407 migration_blockers = g_slist_prepend(migration_blockers, reason);
408 }
409
410 void migrate_del_blocker(Error *reason)
411 {
412 migration_blockers = g_slist_remove(migration_blockers, reason);
413 }
414
415 void qmp_migrate(const char *uri, bool has_blk, bool blk,
416 bool has_inc, bool inc, bool has_detach, bool detach,
417 Error **errp)
418 {
419 Error *local_err = NULL;
420 MigrationState *s = migrate_get_current();
421 MigrationParams params;
422 const char *p;
423
424 params.blk = has_blk && blk;
425 params.shared = has_inc && inc;
426
427 if (s->state == MIG_STATE_ACTIVE || s->state == MIG_STATE_SETUP ||
428 s->state == MIG_STATE_CANCELLING) {
429 error_set(errp, QERR_MIGRATION_ACTIVE);
430 return;
431 }
432
433 if (qemu_savevm_state_blocked(errp)) {
434 return;
435 }
436
437 if (migration_blockers) {
438 *errp = error_copy(migration_blockers->data);
439 return;
440 }
441
442 s = migrate_init(&params);
443
444 if (strstart(uri, "tcp:", &p)) {
445 tcp_start_outgoing_migration(s, p, &local_err);
446 #ifdef CONFIG_RDMA
447 } else if (strstart(uri, "rdma:", &p)) {
448 rdma_start_outgoing_migration(s, p, &local_err);
449 #endif
450 #if !defined(WIN32)
451 } else if (strstart(uri, "exec:", &p)) {
452 exec_start_outgoing_migration(s, p, &local_err);
453 } else if (strstart(uri, "unix:", &p)) {
454 unix_start_outgoing_migration(s, p, &local_err);
455 } else if (strstart(uri, "fd:", &p)) {
456 fd_start_outgoing_migration(s, p, &local_err);
457 #endif
458 } else {
459 error_set(errp, QERR_INVALID_PARAMETER_VALUE, "uri", "a valid migration protocol");
460 s->state = MIG_STATE_ERROR;
461 return;
462 }
463
464 if (local_err) {
465 migrate_fd_error(s);
466 error_propagate(errp, local_err);
467 return;
468 }
469 }
470
471 void qmp_migrate_cancel(Error **errp)
472 {
473 migrate_fd_cancel(migrate_get_current());
474 }
475
476 void qmp_migrate_set_cache_size(int64_t value, Error **errp)
477 {
478 MigrationState *s = migrate_get_current();
479 int64_t new_size;
480
481 /* Check for truncation */
482 if (value != (size_t)value) {
483 error_set(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
484 "exceeding address space");
485 return;
486 }
487
488 /* Cache should not be larger than guest ram size */
489 if (value > ram_bytes_total()) {
490 error_set(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
491 "exceeds guest ram size ");
492 return;
493 }
494
495 new_size = xbzrle_cache_resize(value);
496 if (new_size < 0) {
497 error_set(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
498 "is smaller than page size");
499 return;
500 }
501
502 s->xbzrle_cache_size = new_size;
503 }
504
505 int64_t qmp_query_migrate_cache_size(Error **errp)
506 {
507 return migrate_xbzrle_cache_size();
508 }
509
510 void qmp_migrate_set_speed(int64_t value, Error **errp)
511 {
512 MigrationState *s;
513
514 if (value < 0) {
515 value = 0;
516 }
517 if (value > SIZE_MAX) {
518 value = SIZE_MAX;
519 }
520
521 s = migrate_get_current();
522 s->bandwidth_limit = value;
523 if (s->file) {
524 qemu_file_set_rate_limit(s->file, s->bandwidth_limit / XFER_LIMIT_RATIO);
525 }
526 }
527
528 void qmp_migrate_set_downtime(double value, Error **errp)
529 {
530 value *= 1e9;
531 value = MAX(0, MIN(UINT64_MAX, value));
532 max_downtime = (uint64_t)value;
533 }
534
535 bool migrate_rdma_pin_all(void)
536 {
537 MigrationState *s;
538
539 s = migrate_get_current();
540
541 return s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL];
542 }
543
544 bool migrate_auto_converge(void)
545 {
546 MigrationState *s;
547
548 s = migrate_get_current();
549
550 return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
551 }
552
553 bool migrate_zero_blocks(void)
554 {
555 MigrationState *s;
556
557 s = migrate_get_current();
558
559 return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
560 }
561
562 int migrate_use_xbzrle(void)
563 {
564 MigrationState *s;
565
566 s = migrate_get_current();
567
568 return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
569 }
570
571 int64_t migrate_xbzrle_cache_size(void)
572 {
573 MigrationState *s;
574
575 s = migrate_get_current();
576
577 return s->xbzrle_cache_size;
578 }
579
580 /* migration thread support */
581
582 static void *migration_thread(void *opaque)
583 {
584 MigrationState *s = opaque;
585 int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
586 int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
587 int64_t initial_bytes = 0;
588 int64_t max_size = 0;
589 int64_t start_time = initial_time;
590 bool old_vm_running = false;
591
592 DPRINTF("beginning savevm\n");
593 qemu_savevm_state_begin(s->file, &s->params);
594
595 s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
596 migrate_set_state(s, MIG_STATE_SETUP, MIG_STATE_ACTIVE);
597
598 DPRINTF("setup complete\n");
599
600 while (s->state == MIG_STATE_ACTIVE) {
601 int64_t current_time;
602 uint64_t pending_size;
603
604 if (!qemu_file_rate_limit(s->file)) {
605 DPRINTF("iterate\n");
606 pending_size = qemu_savevm_state_pending(s->file, max_size);
607 DPRINTF("pending size %" PRIu64 " max %" PRIu64 "\n",
608 pending_size, max_size);
609 if (pending_size && pending_size >= max_size) {
610 qemu_savevm_state_iterate(s->file);
611 } else {
612 int ret;
613
614 DPRINTF("done iterating\n");
615 qemu_mutex_lock_iothread();
616 start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
617 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
618 old_vm_running = runstate_is_running();
619
620 ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
621 if (ret >= 0) {
622 qemu_file_set_rate_limit(s->file, INT64_MAX);
623 qemu_savevm_state_complete(s->file);
624 }
625 qemu_mutex_unlock_iothread();
626
627 if (ret < 0) {
628 migrate_set_state(s, MIG_STATE_ACTIVE, MIG_STATE_ERROR);
629 break;
630 }
631
632 if (!qemu_file_get_error(s->file)) {
633 migrate_set_state(s, MIG_STATE_ACTIVE, MIG_STATE_COMPLETED);
634 break;
635 }
636 }
637 }
638
639 if (qemu_file_get_error(s->file)) {
640 migrate_set_state(s, MIG_STATE_ACTIVE, MIG_STATE_ERROR);
641 break;
642 }
643 current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
644 if (current_time >= initial_time + BUFFER_DELAY) {
645 uint64_t transferred_bytes = qemu_ftell(s->file) - initial_bytes;
646 uint64_t time_spent = current_time - initial_time;
647 double bandwidth = transferred_bytes / time_spent;
648 max_size = bandwidth * migrate_max_downtime() / 1000000;
649
650 s->mbps = time_spent ? (((double) transferred_bytes * 8.0) /
651 ((double) time_spent / 1000.0)) / 1000.0 / 1000.0 : -1;
652
653 DPRINTF("transferred %" PRIu64 " time_spent %" PRIu64
654 " bandwidth %g max_size %" PRId64 "\n",
655 transferred_bytes, time_spent, bandwidth, max_size);
656 /* if we haven't sent anything, we don't want to recalculate
657 10000 is a small enough number for our purposes */
658 if (s->dirty_bytes_rate && transferred_bytes > 10000) {
659 s->expected_downtime = s->dirty_bytes_rate / bandwidth;
660 }
661
662 qemu_file_reset_rate_limit(s->file);
663 initial_time = current_time;
664 initial_bytes = qemu_ftell(s->file);
665 }
666 if (qemu_file_rate_limit(s->file)) {
667 /* usleep expects microseconds */
668 g_usleep((initial_time + BUFFER_DELAY - current_time)*1000);
669 }
670 }
671
672 qemu_mutex_lock_iothread();
673 if (s->state == MIG_STATE_COMPLETED) {
674 int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
675 s->total_time = end_time - s->total_time;
676 s->downtime = end_time - start_time;
677 runstate_set(RUN_STATE_POSTMIGRATE);
678 } else {
679 if (old_vm_running) {
680 vm_start();
681 }
682 }
683 qemu_bh_schedule(s->cleanup_bh);
684 qemu_mutex_unlock_iothread();
685
686 return NULL;
687 }
688
689 void migrate_fd_connect(MigrationState *s)
690 {
691 s->state = MIG_STATE_SETUP;
692 trace_migrate_set_state(MIG_STATE_SETUP);
693
694 /* This is a best 1st approximation. ns to ms */
695 s->expected_downtime = max_downtime/1000000;
696 s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup, s);
697
698 qemu_file_set_rate_limit(s->file,
699 s->bandwidth_limit / XFER_LIMIT_RATIO);
700
701 /* Notify before starting migration thread */
702 notifier_list_notify(&migration_state_notifiers, s);
703
704 qemu_thread_create(&s->thread, "migration", migration_thread, s,
705 QEMU_THREAD_JOINABLE);
706 }