Merge tag 'linux-user-for-7.1-pull-request' of https://gitlab.com/laurent_vivier...
[qemu.git] / migration / block-dirty-bitmap.c
1 /*
2 * Block dirty bitmap postcopy migration
3 *
4 * Copyright IBM, Corp. 2009
5 * Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
6 *
7 * Authors:
8 * Liran Schour <lirans@il.ibm.com>
9 * Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
10 *
11 * This work is licensed under the terms of the GNU GPL, version 2. See
12 * the COPYING file in the top-level directory.
13 * This file is derived from migration/block.c, so it's author and IBM copyright
14 * are here, although content is quite different.
15 *
16 * Contributions after 2012-01-13 are licensed under the terms of the
17 * GNU GPL, version 2 or (at your option) any later version.
18 *
19 * ***
20 *
21 * Here postcopy migration of dirty bitmaps is realized. Only QMP-addressable
22 * bitmaps are migrated.
23 *
24 * Bitmap migration implies creating bitmap with the same name and granularity
25 * in destination QEMU. If the bitmap with the same name (for the same node)
26 * already exists on destination an error will be generated.
27 *
28 * format of migration:
29 *
30 * # Header (shared for different chunk types)
31 * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags)
32 * [ 1 byte: node alias size ] \ flags & DEVICE_NAME
33 * [ n bytes: node alias ] /
34 * [ 1 byte: bitmap alias size ] \ flags & BITMAP_NAME
35 * [ n bytes: bitmap alias ] /
36 *
37 * # Start of bitmap migration (flags & START)
38 * header
39 * be64: granularity
40 * 1 byte: bitmap flags (corresponds to BdrvDirtyBitmap)
41 * bit 0 - bitmap is enabled
42 * bit 1 - bitmap is persistent
43 * bit 2 - bitmap is autoloading
44 * bits 3-7 - reserved, must be zero
45 *
46 * # Complete of bitmap migration (flags & COMPLETE)
47 * header
48 *
49 * # Data chunk of bitmap migration
50 * header
51 * be64: start sector
52 * be32: number of sectors
53 * [ be64: buffer size ] \ ! (flags & ZEROES)
54 * [ n bytes: buffer ] /
55 *
56 * The last chunk in stream should contain flags & EOS. The chunk may skip
57 * device and/or bitmap names, assuming them to be the same with the previous
58 * chunk.
59 */
60
61 #include "qemu/osdep.h"
62 #include "block/block.h"
63 #include "block/block_int.h"
64 #include "sysemu/block-backend.h"
65 #include "sysemu/runstate.h"
66 #include "qemu/main-loop.h"
67 #include "qemu/error-report.h"
68 #include "migration/misc.h"
69 #include "migration/migration.h"
70 #include "qemu-file.h"
71 #include "migration/vmstate.h"
72 #include "migration/register.h"
73 #include "qemu/hbitmap.h"
74 #include "qemu/cutils.h"
75 #include "qemu/id.h"
76 #include "qapi/error.h"
77 #include "qapi/qapi-commands-migration.h"
78 #include "qapi/qapi-visit-migration.h"
79 #include "qapi/clone-visitor.h"
80 #include "trace.h"
81
82 #define CHUNK_SIZE (1 << 10)
83
84 /* Flags occupy one, two or four bytes (Big Endian). The size is determined as
85 * follows:
86 * in first (most significant) byte bit 8 is clear --> one byte
87 * in first byte bit 8 is set --> two or four bytes, depending on second
88 * byte:
89 * | in second byte bit 8 is clear --> two bytes
90 * | in second byte bit 8 is set --> four bytes
91 */
92 #define DIRTY_BITMAP_MIG_FLAG_EOS 0x01
93 #define DIRTY_BITMAP_MIG_FLAG_ZEROES 0x02
94 #define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME 0x04
95 #define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME 0x08
96 #define DIRTY_BITMAP_MIG_FLAG_START 0x10
97 #define DIRTY_BITMAP_MIG_FLAG_COMPLETE 0x20
98 #define DIRTY_BITMAP_MIG_FLAG_BITS 0x40
99
100 #define DIRTY_BITMAP_MIG_EXTRA_FLAGS 0x80
101
102 #define DIRTY_BITMAP_MIG_START_FLAG_ENABLED 0x01
103 #define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT 0x02
104 /* 0x04 was "AUTOLOAD" flags on older versions, now it is ignored */
105 #define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK 0xf8
106
107 /* State of one bitmap during save process */
108 typedef struct SaveBitmapState {
109 /* Written during setup phase. */
110 BlockDriverState *bs;
111 char *node_alias;
112 char *bitmap_alias;
113 BdrvDirtyBitmap *bitmap;
114 uint64_t total_sectors;
115 uint64_t sectors_per_chunk;
116 QSIMPLEQ_ENTRY(SaveBitmapState) entry;
117 uint8_t flags;
118
119 /* For bulk phase. */
120 bool bulk_completed;
121 uint64_t cur_sector;
122 } SaveBitmapState;
123
124 /* State of the dirty bitmap migration (DBM) during save process */
125 typedef struct DBMSaveState {
126 QSIMPLEQ_HEAD(, SaveBitmapState) dbms_list;
127
128 bool bulk_completed;
129 bool no_bitmaps;
130
131 /* for send_bitmap_bits() */
132 BlockDriverState *prev_bs;
133 BdrvDirtyBitmap *prev_bitmap;
134 } DBMSaveState;
135
136 typedef struct LoadBitmapState {
137 BlockDriverState *bs;
138 BdrvDirtyBitmap *bitmap;
139 bool migrated;
140 bool enabled;
141 } LoadBitmapState;
142
143 /* State of the dirty bitmap migration (DBM) during load process */
144 typedef struct DBMLoadState {
145 uint32_t flags;
146 char node_alias[256];
147 char bitmap_alias[256];
148 char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1];
149 BlockDriverState *bs;
150 BdrvDirtyBitmap *bitmap;
151
152 bool before_vm_start_handled; /* set in dirty_bitmap_mig_before_vm_start */
153 BitmapMigrationBitmapAlias *bmap_inner;
154
155 /*
156 * cancelled
157 * Incoming migration is cancelled for some reason. That means that we
158 * still should read our chunks from migration stream, to not affect other
159 * migration objects (like RAM), but just ignore them and do not touch any
160 * bitmaps or nodes.
161 */
162 bool cancelled;
163
164 GSList *bitmaps;
165 QemuMutex lock; /* protect bitmaps */
166 } DBMLoadState;
167
168 typedef struct DBMState {
169 DBMSaveState save;
170 DBMLoadState load;
171 } DBMState;
172
173 static DBMState dbm_state;
174
175 /* For hash tables that map node/bitmap names to aliases */
176 typedef struct AliasMapInnerNode {
177 char *string;
178 GHashTable *subtree;
179 } AliasMapInnerNode;
180
181 static void free_alias_map_inner_node(void *amin_ptr)
182 {
183 AliasMapInnerNode *amin = amin_ptr;
184
185 g_free(amin->string);
186 g_hash_table_unref(amin->subtree);
187 g_free(amin);
188 }
189
190 /**
191 * Construct an alias map based on the given QMP structure.
192 *
193 * (Note that we cannot store such maps in the MigrationParameters
194 * object, because that struct is defined by the QAPI schema, which
195 * makes it basically impossible to have dicts with arbitrary keys.
196 * Therefore, we instead have to construct these maps when migration
197 * starts.)
198 *
199 * @bbm is the block_bitmap_mapping from the migration parameters.
200 *
201 * If @name_to_alias is true, the returned hash table will map node
202 * and bitmap names to their respective aliases (for outgoing
203 * migration).
204 *
205 * If @name_to_alias is false, the returned hash table will map node
206 * and bitmap aliases to their respective names (for incoming
207 * migration).
208 *
209 * The hash table maps node names/aliases to AliasMapInnerNode
210 * objects, whose .string is the respective node alias/name, and whose
211 * .subtree table maps bitmap names/aliases to the respective bitmap
212 * alias/name.
213 */
214 static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm,
215 bool name_to_alias,
216 Error **errp)
217 {
218 GHashTable *alias_map;
219 size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1;
220
221 alias_map = g_hash_table_new_full(g_str_hash, g_str_equal,
222 g_free, free_alias_map_inner_node);
223
224 for (; bbm; bbm = bbm->next) {
225 const BitmapMigrationNodeAlias *bmna = bbm->value;
226 const BitmapMigrationBitmapAliasList *bmbal;
227 AliasMapInnerNode *amin;
228 GHashTable *bitmaps_map;
229 const char *node_map_from, *node_map_to;
230 GDestroyNotify gdn;
231
232 if (!id_wellformed(bmna->alias)) {
233 error_setg(errp, "The node alias '%s' is not well-formed",
234 bmna->alias);
235 goto fail;
236 }
237
238 if (strlen(bmna->alias) > UINT8_MAX) {
239 error_setg(errp, "The node alias '%s' is longer than %u bytes",
240 bmna->alias, UINT8_MAX);
241 goto fail;
242 }
243
244 if (strlen(bmna->node_name) > max_node_name_len) {
245 error_setg(errp, "The node name '%s' is longer than %zu bytes",
246 bmna->node_name, max_node_name_len);
247 goto fail;
248 }
249
250 if (name_to_alias) {
251 if (g_hash_table_contains(alias_map, bmna->node_name)) {
252 error_setg(errp, "The node name '%s' is mapped twice",
253 bmna->node_name);
254 goto fail;
255 }
256
257 node_map_from = bmna->node_name;
258 node_map_to = bmna->alias;
259 } else {
260 if (g_hash_table_contains(alias_map, bmna->alias)) {
261 error_setg(errp, "The node alias '%s' is used twice",
262 bmna->alias);
263 goto fail;
264 }
265
266 node_map_from = bmna->alias;
267 node_map_to = bmna->node_name;
268 }
269
270 gdn = (GDestroyNotify) qapi_free_BitmapMigrationBitmapAlias;
271 bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal, g_free,
272 gdn);
273
274 amin = g_new(AliasMapInnerNode, 1);
275 *amin = (AliasMapInnerNode){
276 .string = g_strdup(node_map_to),
277 .subtree = bitmaps_map,
278 };
279
280 g_hash_table_insert(alias_map, g_strdup(node_map_from), amin);
281
282 for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) {
283 const BitmapMigrationBitmapAlias *bmba = bmbal->value;
284 const char *bmap_map_from;
285
286 if (strlen(bmba->alias) > UINT8_MAX) {
287 error_setg(errp,
288 "The bitmap alias '%s' is longer than %u bytes",
289 bmba->alias, UINT8_MAX);
290 goto fail;
291 }
292
293 if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) {
294 error_setg(errp, "The bitmap name '%s' is longer than %d bytes",
295 bmba->name, BDRV_BITMAP_MAX_NAME_SIZE);
296 goto fail;
297 }
298
299 if (name_to_alias) {
300 bmap_map_from = bmba->name;
301
302 if (g_hash_table_contains(bitmaps_map, bmba->name)) {
303 error_setg(errp, "The bitmap '%s'/'%s' is mapped twice",
304 bmna->node_name, bmba->name);
305 goto fail;
306 }
307 } else {
308 bmap_map_from = bmba->alias;
309
310 if (g_hash_table_contains(bitmaps_map, bmba->alias)) {
311 error_setg(errp, "The bitmap alias '%s'/'%s' is used twice",
312 bmna->alias, bmba->alias);
313 goto fail;
314 }
315 }
316
317 g_hash_table_insert(bitmaps_map, g_strdup(bmap_map_from),
318 QAPI_CLONE(BitmapMigrationBitmapAlias, bmba));
319 }
320 }
321
322 return alias_map;
323
324 fail:
325 g_hash_table_destroy(alias_map);
326 return NULL;
327 }
328
329 /**
330 * Run construct_alias_map() in both directions to check whether @bbm
331 * is valid.
332 * (This function is to be used by migration/migration.c to validate
333 * the user-specified block-bitmap-mapping migration parameter.)
334 *
335 * Returns true if and only if the mapping is valid.
336 */
337 bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
338 Error **errp)
339 {
340 GHashTable *alias_map;
341
342 alias_map = construct_alias_map(bbm, true, errp);
343 if (!alias_map) {
344 return false;
345 }
346 g_hash_table_destroy(alias_map);
347
348 alias_map = construct_alias_map(bbm, false, errp);
349 if (!alias_map) {
350 return false;
351 }
352 g_hash_table_destroy(alias_map);
353
354 return true;
355 }
356
357 static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
358 {
359 uint8_t flags = qemu_get_byte(f);
360 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
361 flags = flags << 8 | qemu_get_byte(f);
362 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
363 flags = flags << 16 | qemu_get_be16(f);
364 }
365 }
366
367 return flags;
368 }
369
370 static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags)
371 {
372 /* The code currently does not send flags as more than one byte */
373 assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS)));
374
375 qemu_put_byte(f, flags);
376 }
377
378 static void send_bitmap_header(QEMUFile *f, DBMSaveState *s,
379 SaveBitmapState *dbms, uint32_t additional_flags)
380 {
381 BlockDriverState *bs = dbms->bs;
382 BdrvDirtyBitmap *bitmap = dbms->bitmap;
383 uint32_t flags = additional_flags;
384 trace_send_bitmap_header_enter();
385
386 if (bs != s->prev_bs) {
387 s->prev_bs = bs;
388 flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME;
389 }
390
391 if (bitmap != s->prev_bitmap) {
392 s->prev_bitmap = bitmap;
393 flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME;
394 }
395
396 qemu_put_bitmap_flags(f, flags);
397
398 if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
399 qemu_put_counted_string(f, dbms->node_alias);
400 }
401
402 if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
403 qemu_put_counted_string(f, dbms->bitmap_alias);
404 }
405 }
406
407 static void send_bitmap_start(QEMUFile *f, DBMSaveState *s,
408 SaveBitmapState *dbms)
409 {
410 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_START);
411 qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap));
412 qemu_put_byte(f, dbms->flags);
413 }
414
415 static void send_bitmap_complete(QEMUFile *f, DBMSaveState *s,
416 SaveBitmapState *dbms)
417 {
418 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE);
419 }
420
421 static void send_bitmap_bits(QEMUFile *f, DBMSaveState *s,
422 SaveBitmapState *dbms,
423 uint64_t start_sector, uint32_t nr_sectors)
424 {
425 /* align for buffer_is_zero() */
426 uint64_t align = 4 * sizeof(long);
427 uint64_t unaligned_size =
428 bdrv_dirty_bitmap_serialization_size(
429 dbms->bitmap, start_sector << BDRV_SECTOR_BITS,
430 (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
431 uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align);
432 uint8_t *buf = g_malloc0(buf_size);
433 uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS;
434
435 bdrv_dirty_bitmap_serialize_part(
436 dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS,
437 (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
438
439 if (buffer_is_zero(buf, buf_size)) {
440 g_free(buf);
441 buf = NULL;
442 flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES;
443 }
444
445 trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size);
446
447 send_bitmap_header(f, s, dbms, flags);
448
449 qemu_put_be64(f, start_sector);
450 qemu_put_be32(f, nr_sectors);
451
452 /* if a block is zero we need to flush here since the network
453 * bandwidth is now a lot higher than the storage device bandwidth.
454 * thus if we queue zero blocks we slow down the migration. */
455 if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
456 qemu_fflush(f);
457 } else {
458 qemu_put_be64(f, buf_size);
459 qemu_put_buffer(f, buf, buf_size);
460 }
461
462 g_free(buf);
463 }
464
465 /* Called with iothread lock taken. */
466 static void dirty_bitmap_do_save_cleanup(DBMSaveState *s)
467 {
468 SaveBitmapState *dbms;
469
470 while ((dbms = QSIMPLEQ_FIRST(&s->dbms_list)) != NULL) {
471 QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry);
472 bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
473 bdrv_unref(dbms->bs);
474 g_free(dbms->node_alias);
475 g_free(dbms->bitmap_alias);
476 g_free(dbms);
477 }
478 }
479
480 /* Called with iothread lock taken. */
481 static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs,
482 const char *bs_name, GHashTable *alias_map)
483 {
484 BdrvDirtyBitmap *bitmap;
485 SaveBitmapState *dbms;
486 GHashTable *bitmap_aliases;
487 const char *node_alias, *bitmap_name, *bitmap_alias;
488 Error *local_err = NULL;
489
490 /* When an alias map is given, @bs_name must be @bs's node name */
491 assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs)));
492
493 FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
494 if (bdrv_dirty_bitmap_name(bitmap)) {
495 break;
496 }
497 }
498 if (!bitmap) {
499 return 0;
500 }
501
502 bitmap_name = bdrv_dirty_bitmap_name(bitmap);
503
504 if (!bs_name || strcmp(bs_name, "") == 0) {
505 error_report("Bitmap '%s' in unnamed node can't be migrated",
506 bitmap_name);
507 return -1;
508 }
509
510 if (alias_map) {
511 const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name);
512
513 if (!amin) {
514 /* Skip bitmaps on nodes with no alias */
515 return 0;
516 }
517
518 node_alias = amin->string;
519 bitmap_aliases = amin->subtree;
520 } else {
521 node_alias = bs_name;
522 bitmap_aliases = NULL;
523 }
524
525 if (node_alias[0] == '#') {
526 error_report("Bitmap '%s' in a node with auto-generated "
527 "name '%s' can't be migrated",
528 bitmap_name, node_alias);
529 return -1;
530 }
531
532 FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
533 BitmapMigrationBitmapAliasTransform *bitmap_transform = NULL;
534 bitmap_name = bdrv_dirty_bitmap_name(bitmap);
535 if (!bitmap_name) {
536 continue;
537 }
538
539 if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) {
540 error_report_err(local_err);
541 return -1;
542 }
543
544 if (bitmap_aliases) {
545 BitmapMigrationBitmapAlias *bmap_inner;
546
547 bmap_inner = g_hash_table_lookup(bitmap_aliases, bitmap_name);
548 if (!bmap_inner) {
549 /* Skip bitmaps with no alias */
550 continue;
551 }
552
553 bitmap_alias = bmap_inner->alias;
554 if (bmap_inner->has_transform) {
555 bitmap_transform = bmap_inner->transform;
556 }
557 } else {
558 if (strlen(bitmap_name) > UINT8_MAX) {
559 error_report("Cannot migrate bitmap '%s' on node '%s': "
560 "Name is longer than %u bytes",
561 bitmap_name, bs_name, UINT8_MAX);
562 return -1;
563 }
564 bitmap_alias = bitmap_name;
565 }
566
567 bdrv_ref(bs);
568 bdrv_dirty_bitmap_set_busy(bitmap, true);
569
570 dbms = g_new0(SaveBitmapState, 1);
571 dbms->bs = bs;
572 dbms->node_alias = g_strdup(node_alias);
573 dbms->bitmap_alias = g_strdup(bitmap_alias);
574 dbms->bitmap = bitmap;
575 dbms->total_sectors = bdrv_nb_sectors(bs);
576 dbms->sectors_per_chunk = CHUNK_SIZE * 8LLU *
577 (bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS);
578 assert(dbms->sectors_per_chunk != 0);
579 if (bdrv_dirty_bitmap_enabled(bitmap)) {
580 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
581 }
582 if (bitmap_transform &&
583 bitmap_transform->has_persistent) {
584 if (bitmap_transform->persistent) {
585 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
586 }
587 } else {
588 if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
589 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
590 }
591 }
592
593 QSIMPLEQ_INSERT_TAIL(&s->dbms_list, dbms, entry);
594 }
595
596 return 0;
597 }
598
599 /* Called with iothread lock taken. */
600 static int init_dirty_bitmap_migration(DBMSaveState *s)
601 {
602 BlockDriverState *bs;
603 SaveBitmapState *dbms;
604 GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL);
605 BlockBackend *blk;
606 const MigrationParameters *mig_params = &migrate_get_current()->parameters;
607 GHashTable *alias_map = NULL;
608
609 if (mig_params->has_block_bitmap_mapping) {
610 alias_map = construct_alias_map(mig_params->block_bitmap_mapping, true,
611 &error_abort);
612 }
613
614 s->bulk_completed = false;
615 s->prev_bs = NULL;
616 s->prev_bitmap = NULL;
617 s->no_bitmaps = false;
618
619 if (!alias_map) {
620 /*
621 * Use blockdevice name for direct (or filtered) children of named block
622 * backends.
623 */
624 for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
625 const char *name = blk_name(blk);
626
627 if (!name || strcmp(name, "") == 0) {
628 continue;
629 }
630
631 bs = blk_bs(blk);
632
633 /* Skip filters without bitmaps */
634 while (bs && bs->drv && bs->drv->is_filter &&
635 !bdrv_has_named_bitmaps(bs))
636 {
637 bs = bdrv_filter_bs(bs);
638 }
639
640 if (bs && bs->drv && !bs->drv->is_filter) {
641 if (add_bitmaps_to_list(s, bs, name, NULL)) {
642 goto fail;
643 }
644 g_hash_table_add(handled_by_blk, bs);
645 }
646 }
647 }
648
649 for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
650 if (g_hash_table_contains(handled_by_blk, bs)) {
651 continue;
652 }
653
654 if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) {
655 goto fail;
656 }
657 }
658
659 /* unset migration flags here, to not roll back it */
660 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
661 bdrv_dirty_bitmap_skip_store(dbms->bitmap, true);
662 }
663
664 if (QSIMPLEQ_EMPTY(&s->dbms_list)) {
665 s->no_bitmaps = true;
666 }
667
668 g_hash_table_destroy(handled_by_blk);
669 if (alias_map) {
670 g_hash_table_destroy(alias_map);
671 }
672
673 return 0;
674
675 fail:
676 g_hash_table_destroy(handled_by_blk);
677 if (alias_map) {
678 g_hash_table_destroy(alias_map);
679 }
680 dirty_bitmap_do_save_cleanup(s);
681
682 return -1;
683 }
684
685 /* Called with no lock taken. */
686 static void bulk_phase_send_chunk(QEMUFile *f, DBMSaveState *s,
687 SaveBitmapState *dbms)
688 {
689 uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector,
690 dbms->sectors_per_chunk);
691
692 send_bitmap_bits(f, s, dbms, dbms->cur_sector, nr_sectors);
693
694 dbms->cur_sector += nr_sectors;
695 if (dbms->cur_sector >= dbms->total_sectors) {
696 dbms->bulk_completed = true;
697 }
698 }
699
700 /* Called with no lock taken. */
701 static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit)
702 {
703 SaveBitmapState *dbms;
704
705 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
706 while (!dbms->bulk_completed) {
707 bulk_phase_send_chunk(f, s, dbms);
708 if (limit && qemu_file_rate_limit(f)) {
709 return;
710 }
711 }
712 }
713
714 s->bulk_completed = true;
715 }
716
717 /* for SaveVMHandlers */
718 static void dirty_bitmap_save_cleanup(void *opaque)
719 {
720 DBMSaveState *s = &((DBMState *)opaque)->save;
721
722 dirty_bitmap_do_save_cleanup(s);
723 }
724
725 static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque)
726 {
727 DBMSaveState *s = &((DBMState *)opaque)->save;
728
729 trace_dirty_bitmap_save_iterate(migration_in_postcopy());
730
731 if (migration_in_postcopy() && !s->bulk_completed) {
732 bulk_phase(f, s, true);
733 }
734
735 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
736
737 return s->bulk_completed;
738 }
739
740 /* Called with iothread lock taken. */
741
742 static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque)
743 {
744 DBMSaveState *s = &((DBMState *)opaque)->save;
745 SaveBitmapState *dbms;
746 trace_dirty_bitmap_save_complete_enter();
747
748 if (!s->bulk_completed) {
749 bulk_phase(f, s, false);
750 }
751
752 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
753 send_bitmap_complete(f, s, dbms);
754 }
755
756 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
757
758 trace_dirty_bitmap_save_complete_finish();
759
760 dirty_bitmap_save_cleanup(opaque);
761 return 0;
762 }
763
764 static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque,
765 uint64_t max_size,
766 uint64_t *res_precopy_only,
767 uint64_t *res_compatible,
768 uint64_t *res_postcopy_only)
769 {
770 DBMSaveState *s = &((DBMState *)opaque)->save;
771 SaveBitmapState *dbms;
772 uint64_t pending = 0;
773
774 qemu_mutex_lock_iothread();
775
776 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
777 uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap);
778 uint64_t sectors = dbms->bulk_completed ? 0 :
779 dbms->total_sectors - dbms->cur_sector;
780
781 pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran);
782 }
783
784 qemu_mutex_unlock_iothread();
785
786 trace_dirty_bitmap_save_pending(pending, max_size);
787
788 *res_postcopy_only += pending;
789 }
790
791 /* First occurrence of this bitmap. It should be created if doesn't exist */
792 static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s)
793 {
794 Error *local_err = NULL;
795 uint32_t granularity = qemu_get_be32(f);
796 uint8_t flags = qemu_get_byte(f);
797 LoadBitmapState *b;
798 bool persistent;
799
800 if (s->cancelled) {
801 return 0;
802 }
803
804 if (s->bitmap) {
805 error_report("Bitmap with the same name ('%s') already exists on "
806 "destination", bdrv_dirty_bitmap_name(s->bitmap));
807 return -EINVAL;
808 } else {
809 s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity,
810 s->bitmap_name, &local_err);
811 if (!s->bitmap) {
812 error_report_err(local_err);
813 return -EINVAL;
814 }
815 }
816
817 if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) {
818 error_report("Unknown flags in migrated dirty bitmap header: %x",
819 flags);
820 return -EINVAL;
821 }
822
823 if (s->bmap_inner &&
824 s->bmap_inner->has_transform &&
825 s->bmap_inner->transform->has_persistent) {
826 persistent = s->bmap_inner->transform->persistent;
827 } else {
828 persistent = flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
829 }
830
831 if (persistent) {
832 bdrv_dirty_bitmap_set_persistence(s->bitmap, true);
833 }
834
835 bdrv_disable_dirty_bitmap(s->bitmap);
836 if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) {
837 bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err);
838 if (local_err) {
839 error_report_err(local_err);
840 return -EINVAL;
841 }
842 } else {
843 bdrv_dirty_bitmap_set_busy(s->bitmap, true);
844 }
845
846 b = g_new(LoadBitmapState, 1);
847 b->bs = s->bs;
848 b->bitmap = s->bitmap;
849 b->migrated = false;
850 b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
851
852 s->bitmaps = g_slist_prepend(s->bitmaps, b);
853
854 return 0;
855 }
856
857 /*
858 * before_vm_start_handle_item
859 *
860 * g_slist_foreach helper
861 *
862 * item is LoadBitmapState*
863 * opaque is DBMLoadState*
864 */
865 static void before_vm_start_handle_item(void *item, void *opaque)
866 {
867 DBMLoadState *s = opaque;
868 LoadBitmapState *b = item;
869
870 if (b->enabled) {
871 if (b->migrated) {
872 bdrv_enable_dirty_bitmap(b->bitmap);
873 } else {
874 bdrv_dirty_bitmap_enable_successor(b->bitmap);
875 }
876 }
877
878 if (b->migrated) {
879 s->bitmaps = g_slist_remove(s->bitmaps, b);
880 g_free(b);
881 }
882 }
883
884 void dirty_bitmap_mig_before_vm_start(void)
885 {
886 DBMLoadState *s = &dbm_state.load;
887 qemu_mutex_lock(&s->lock);
888
889 assert(!s->before_vm_start_handled);
890 g_slist_foreach(s->bitmaps, before_vm_start_handle_item, s);
891 s->before_vm_start_handled = true;
892
893 qemu_mutex_unlock(&s->lock);
894 }
895
896 static void cancel_incoming_locked(DBMLoadState *s)
897 {
898 GSList *item;
899
900 if (s->cancelled) {
901 return;
902 }
903
904 s->cancelled = true;
905 s->bs = NULL;
906 s->bitmap = NULL;
907
908 /* Drop all unfinished bitmaps */
909 for (item = s->bitmaps; item; item = g_slist_next(item)) {
910 LoadBitmapState *b = item->data;
911
912 /*
913 * Bitmap must be unfinished, as finished bitmaps should already be
914 * removed from the list.
915 */
916 assert(!s->before_vm_start_handled || !b->migrated);
917 if (bdrv_dirty_bitmap_has_successor(b->bitmap)) {
918 bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort);
919 } else {
920 bdrv_dirty_bitmap_set_busy(b->bitmap, false);
921 }
922 bdrv_release_dirty_bitmap(b->bitmap);
923 }
924
925 g_slist_free_full(s->bitmaps, g_free);
926 s->bitmaps = NULL;
927 }
928
929 void dirty_bitmap_mig_cancel_outgoing(void)
930 {
931 dirty_bitmap_do_save_cleanup(&dbm_state.save);
932 }
933
934 void dirty_bitmap_mig_cancel_incoming(void)
935 {
936 DBMLoadState *s = &dbm_state.load;
937
938 qemu_mutex_lock(&s->lock);
939
940 cancel_incoming_locked(s);
941
942 qemu_mutex_unlock(&s->lock);
943 }
944
945 static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s)
946 {
947 GSList *item;
948 trace_dirty_bitmap_load_complete();
949
950 if (s->cancelled) {
951 return;
952 }
953
954 bdrv_dirty_bitmap_deserialize_finish(s->bitmap);
955
956 if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
957 bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort);
958 } else {
959 bdrv_dirty_bitmap_set_busy(s->bitmap, false);
960 }
961
962 for (item = s->bitmaps; item; item = g_slist_next(item)) {
963 LoadBitmapState *b = item->data;
964
965 if (b->bitmap == s->bitmap) {
966 b->migrated = true;
967 if (s->before_vm_start_handled) {
968 s->bitmaps = g_slist_remove(s->bitmaps, b);
969 g_free(b);
970 }
971 break;
972 }
973 }
974 }
975
976 static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s)
977 {
978 uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS;
979 uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS;
980 trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS,
981 nr_bytes >> BDRV_SECTOR_BITS);
982
983 if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
984 trace_dirty_bitmap_load_bits_zeroes();
985 if (!s->cancelled) {
986 bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte,
987 nr_bytes, false);
988 }
989 } else {
990 size_t ret;
991 g_autofree uint8_t *buf = NULL;
992 uint64_t buf_size = qemu_get_be64(f);
993 uint64_t needed_size;
994
995 /*
996 * The actual check for buf_size is done a bit later. We can't do it in
997 * cancelled mode as we don't have the bitmap to check the constraints
998 * (so, we allocate a buffer and read prior to the check). On the other
999 * hand, we shouldn't blindly g_malloc the number from the stream.
1000 * Actually one chunk should not be larger than CHUNK_SIZE. Let's allow
1001 * a bit larger (which means that bitmap migration will fail anyway and
1002 * the whole migration will most probably fail soon due to broken
1003 * stream).
1004 */
1005 if (buf_size > 10 * CHUNK_SIZE) {
1006 error_report("Bitmap migration stream buffer allocation request "
1007 "is too large");
1008 return -EIO;
1009 }
1010
1011 buf = g_malloc(buf_size);
1012 ret = qemu_get_buffer(f, buf, buf_size);
1013 if (ret != buf_size) {
1014 error_report("Failed to read bitmap bits");
1015 return -EIO;
1016 }
1017
1018 if (s->cancelled) {
1019 return 0;
1020 }
1021
1022 needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap,
1023 first_byte,
1024 nr_bytes);
1025
1026 if (needed_size > buf_size ||
1027 buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long))
1028 /* Here used same alignment as in send_bitmap_bits */
1029 ) {
1030 error_report("Migrated bitmap granularity doesn't "
1031 "match the destination bitmap '%s' granularity",
1032 bdrv_dirty_bitmap_name(s->bitmap));
1033 cancel_incoming_locked(s);
1034 return 0;
1035 }
1036
1037 bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes,
1038 false);
1039 }
1040
1041 return 0;
1042 }
1043
1044 static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s,
1045 GHashTable *alias_map)
1046 {
1047 GHashTable *bitmap_alias_map = NULL;
1048 Error *local_err = NULL;
1049 bool nothing;
1050 s->flags = qemu_get_bitmap_flags(f);
1051 trace_dirty_bitmap_load_header(s->flags);
1052
1053 nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS);
1054
1055 if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
1056 if (!qemu_get_counted_string(f, s->node_alias)) {
1057 error_report("Unable to read node alias string");
1058 return -EINVAL;
1059 }
1060
1061 if (!s->cancelled) {
1062 if (alias_map) {
1063 const AliasMapInnerNode *amin;
1064
1065 amin = g_hash_table_lookup(alias_map, s->node_alias);
1066 if (!amin) {
1067 error_setg(&local_err, "Error: Unknown node alias '%s'",
1068 s->node_alias);
1069 s->bs = NULL;
1070 } else {
1071 bitmap_alias_map = amin->subtree;
1072 s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err);
1073 }
1074 } else {
1075 s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias,
1076 &local_err);
1077 }
1078 if (!s->bs) {
1079 error_report_err(local_err);
1080 cancel_incoming_locked(s);
1081 }
1082 }
1083 } else if (s->bs) {
1084 if (alias_map) {
1085 const AliasMapInnerNode *amin;
1086
1087 /* Must be present in the map, or s->bs would not be set */
1088 amin = g_hash_table_lookup(alias_map, s->node_alias);
1089 assert(amin != NULL);
1090
1091 bitmap_alias_map = amin->subtree;
1092 }
1093 } else if (!nothing && !s->cancelled) {
1094 error_report("Error: block device name is not set");
1095 cancel_incoming_locked(s);
1096 }
1097
1098 assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map);
1099
1100 if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
1101 const char *bitmap_name;
1102
1103 if (!qemu_get_counted_string(f, s->bitmap_alias)) {
1104 error_report("Unable to read bitmap alias string");
1105 return -EINVAL;
1106 }
1107
1108 bitmap_name = s->bitmap_alias;
1109 if (!s->cancelled && bitmap_alias_map) {
1110 BitmapMigrationBitmapAlias *bmap_inner;
1111
1112 bmap_inner = g_hash_table_lookup(bitmap_alias_map, s->bitmap_alias);
1113 if (!bmap_inner) {
1114 error_report("Error: Unknown bitmap alias '%s' on node "
1115 "'%s' (alias '%s')", s->bitmap_alias,
1116 s->bs->node_name, s->node_alias);
1117 cancel_incoming_locked(s);
1118 } else {
1119 bitmap_name = bmap_inner->name;
1120 }
1121
1122 s->bmap_inner = bmap_inner;
1123 }
1124
1125 if (!s->cancelled) {
1126 g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name));
1127 s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
1128
1129 /*
1130 * bitmap may be NULL here, it wouldn't be an error if it is the
1131 * first occurrence of the bitmap
1132 */
1133 if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
1134 error_report("Error: unknown dirty bitmap "
1135 "'%s' for block device '%s'",
1136 s->bitmap_name, s->bs->node_name);
1137 cancel_incoming_locked(s);
1138 }
1139 }
1140 } else if (!s->bitmap && !nothing && !s->cancelled) {
1141 error_report("Error: block device name is not set");
1142 cancel_incoming_locked(s);
1143 }
1144
1145 return 0;
1146 }
1147
1148 /*
1149 * dirty_bitmap_load
1150 *
1151 * Load sequence of dirty bitmap chunks. Return error only on fatal io stream
1152 * violations. On other errors just cancel bitmaps incoming migration and return
1153 * 0.
1154 *
1155 * Note, than when incoming bitmap migration is canceled, we still must read all
1156 * our chunks (and just ignore them), to not affect other migration objects.
1157 */
1158 static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
1159 {
1160 GHashTable *alias_map = NULL;
1161 const MigrationParameters *mig_params = &migrate_get_current()->parameters;
1162 DBMLoadState *s = &((DBMState *)opaque)->load;
1163 int ret = 0;
1164
1165 trace_dirty_bitmap_load_enter();
1166
1167 if (version_id != 1) {
1168 QEMU_LOCK_GUARD(&s->lock);
1169 cancel_incoming_locked(s);
1170 return -EINVAL;
1171 }
1172
1173 if (mig_params->has_block_bitmap_mapping) {
1174 alias_map = construct_alias_map(mig_params->block_bitmap_mapping,
1175 false, &error_abort);
1176 }
1177
1178 do {
1179 QEMU_LOCK_GUARD(&s->lock);
1180
1181 ret = dirty_bitmap_load_header(f, s, alias_map);
1182 if (ret < 0) {
1183 cancel_incoming_locked(s);
1184 goto fail;
1185 }
1186
1187 if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) {
1188 ret = dirty_bitmap_load_start(f, s);
1189 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) {
1190 dirty_bitmap_load_complete(f, s);
1191 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITS) {
1192 ret = dirty_bitmap_load_bits(f, s);
1193 }
1194
1195 if (!ret) {
1196 ret = qemu_file_get_error(f);
1197 }
1198
1199 if (ret) {
1200 cancel_incoming_locked(s);
1201 goto fail;
1202 }
1203 } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS));
1204
1205 trace_dirty_bitmap_load_success();
1206 ret = 0;
1207 fail:
1208 if (alias_map) {
1209 g_hash_table_destroy(alias_map);
1210 }
1211 return ret;
1212 }
1213
1214 static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque)
1215 {
1216 DBMSaveState *s = &((DBMState *)opaque)->save;
1217 SaveBitmapState *dbms = NULL;
1218
1219 qemu_mutex_lock_iothread();
1220 if (init_dirty_bitmap_migration(s) < 0) {
1221 qemu_mutex_unlock_iothread();
1222 return -1;
1223 }
1224
1225 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
1226 send_bitmap_start(f, s, dbms);
1227 }
1228 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
1229 qemu_mutex_unlock_iothread();
1230 return 0;
1231 }
1232
1233 static bool dirty_bitmap_is_active(void *opaque)
1234 {
1235 DBMSaveState *s = &((DBMState *)opaque)->save;
1236
1237 return migrate_dirty_bitmaps() && !s->no_bitmaps;
1238 }
1239
1240 static bool dirty_bitmap_is_active_iterate(void *opaque)
1241 {
1242 return dirty_bitmap_is_active(opaque) && !runstate_is_running();
1243 }
1244
1245 static bool dirty_bitmap_has_postcopy(void *opaque)
1246 {
1247 return true;
1248 }
1249
1250 static SaveVMHandlers savevm_dirty_bitmap_handlers = {
1251 .save_setup = dirty_bitmap_save_setup,
1252 .save_live_complete_postcopy = dirty_bitmap_save_complete,
1253 .save_live_complete_precopy = dirty_bitmap_save_complete,
1254 .has_postcopy = dirty_bitmap_has_postcopy,
1255 .save_live_pending = dirty_bitmap_save_pending,
1256 .save_live_iterate = dirty_bitmap_save_iterate,
1257 .is_active_iterate = dirty_bitmap_is_active_iterate,
1258 .load_state = dirty_bitmap_load,
1259 .save_cleanup = dirty_bitmap_save_cleanup,
1260 .is_active = dirty_bitmap_is_active,
1261 };
1262
1263 void dirty_bitmap_mig_init(void)
1264 {
1265 QSIMPLEQ_INIT(&dbm_state.save.dbms_list);
1266 qemu_mutex_init(&dbm_state.load.lock);
1267
1268 register_savevm_live("dirty-bitmap", 0, 1,
1269 &savevm_dirty_bitmap_handlers,
1270 &dbm_state);
1271 }