target/arm: Implement VFP fp16 VCMP
[qemu.git] / migration / block-dirty-bitmap.c
1 /*
2 * Block dirty bitmap postcopy migration
3 *
4 * Copyright IBM, Corp. 2009
5 * Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
6 *
7 * Authors:
8 * Liran Schour <lirans@il.ibm.com>
9 * Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
10 *
11 * This work is licensed under the terms of the GNU GPL, version 2. See
12 * the COPYING file in the top-level directory.
13 * This file is derived from migration/block.c, so it's author and IBM copyright
14 * are here, although content is quite different.
15 *
16 * Contributions after 2012-01-13 are licensed under the terms of the
17 * GNU GPL, version 2 or (at your option) any later version.
18 *
19 * ***
20 *
21 * Here postcopy migration of dirty bitmaps is realized. Only QMP-addressable
22 * bitmaps are migrated.
23 *
24 * Bitmap migration implies creating bitmap with the same name and granularity
25 * in destination QEMU. If the bitmap with the same name (for the same node)
26 * already exists on destination an error will be generated.
27 *
28 * format of migration:
29 *
30 * # Header (shared for different chunk types)
31 * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags)
32 * [ 1 byte: node alias size ] \ flags & DEVICE_NAME
33 * [ n bytes: node alias ] /
34 * [ 1 byte: bitmap alias size ] \ flags & BITMAP_NAME
35 * [ n bytes: bitmap alias ] /
36 *
37 * # Start of bitmap migration (flags & START)
38 * header
39 * be64: granularity
40 * 1 byte: bitmap flags (corresponds to BdrvDirtyBitmap)
41 * bit 0 - bitmap is enabled
42 * bit 1 - bitmap is persistent
43 * bit 2 - bitmap is autoloading
44 * bits 3-7 - reserved, must be zero
45 *
46 * # Complete of bitmap migration (flags & COMPLETE)
47 * header
48 *
49 * # Data chunk of bitmap migration
50 * header
51 * be64: start sector
52 * be32: number of sectors
53 * [ be64: buffer size ] \ ! (flags & ZEROES)
54 * [ n bytes: buffer ] /
55 *
56 * The last chunk in stream should contain flags & EOS. The chunk may skip
57 * device and/or bitmap names, assuming them to be the same with the previous
58 * chunk.
59 */
60
61 #include "qemu/osdep.h"
62 #include "block/block.h"
63 #include "block/block_int.h"
64 #include "sysemu/block-backend.h"
65 #include "sysemu/runstate.h"
66 #include "qemu/main-loop.h"
67 #include "qemu/error-report.h"
68 #include "migration/misc.h"
69 #include "migration/migration.h"
70 #include "qemu-file.h"
71 #include "migration/vmstate.h"
72 #include "migration/register.h"
73 #include "qemu/hbitmap.h"
74 #include "qemu/cutils.h"
75 #include "qemu/id.h"
76 #include "qapi/error.h"
77 #include "qapi/qapi-commands-migration.h"
78 #include "trace.h"
79
80 #define CHUNK_SIZE (1 << 10)
81
82 /* Flags occupy one, two or four bytes (Big Endian). The size is determined as
83 * follows:
84 * in first (most significant) byte bit 8 is clear --> one byte
85 * in first byte bit 8 is set --> two or four bytes, depending on second
86 * byte:
87 * | in second byte bit 8 is clear --> two bytes
88 * | in second byte bit 8 is set --> four bytes
89 */
90 #define DIRTY_BITMAP_MIG_FLAG_EOS 0x01
91 #define DIRTY_BITMAP_MIG_FLAG_ZEROES 0x02
92 #define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME 0x04
93 #define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME 0x08
94 #define DIRTY_BITMAP_MIG_FLAG_START 0x10
95 #define DIRTY_BITMAP_MIG_FLAG_COMPLETE 0x20
96 #define DIRTY_BITMAP_MIG_FLAG_BITS 0x40
97
98 #define DIRTY_BITMAP_MIG_EXTRA_FLAGS 0x80
99
100 #define DIRTY_BITMAP_MIG_START_FLAG_ENABLED 0x01
101 #define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT 0x02
102 /* 0x04 was "AUTOLOAD" flags on older versions, now it is ignored */
103 #define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK 0xf8
104
105 /* State of one bitmap during save process */
106 typedef struct SaveBitmapState {
107 /* Written during setup phase. */
108 BlockDriverState *bs;
109 char *node_alias;
110 char *bitmap_alias;
111 BdrvDirtyBitmap *bitmap;
112 uint64_t total_sectors;
113 uint64_t sectors_per_chunk;
114 QSIMPLEQ_ENTRY(SaveBitmapState) entry;
115 uint8_t flags;
116
117 /* For bulk phase. */
118 bool bulk_completed;
119 uint64_t cur_sector;
120 } SaveBitmapState;
121
122 /* State of the dirty bitmap migration (DBM) during save process */
123 typedef struct DBMSaveState {
124 QSIMPLEQ_HEAD(, SaveBitmapState) dbms_list;
125
126 bool bulk_completed;
127 bool no_bitmaps;
128
129 /* for send_bitmap_bits() */
130 BlockDriverState *prev_bs;
131 BdrvDirtyBitmap *prev_bitmap;
132 } DBMSaveState;
133
134 typedef struct LoadBitmapState {
135 BlockDriverState *bs;
136 BdrvDirtyBitmap *bitmap;
137 bool migrated;
138 bool enabled;
139 } LoadBitmapState;
140
141 /* State of the dirty bitmap migration (DBM) during load process */
142 typedef struct DBMLoadState {
143 uint32_t flags;
144 char node_alias[256];
145 char bitmap_alias[256];
146 char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1];
147 BlockDriverState *bs;
148 BdrvDirtyBitmap *bitmap;
149
150 bool before_vm_start_handled; /* set in dirty_bitmap_mig_before_vm_start */
151
152 /*
153 * cancelled
154 * Incoming migration is cancelled for some reason. That means that we
155 * still should read our chunks from migration stream, to not affect other
156 * migration objects (like RAM), but just ignore them and do not touch any
157 * bitmaps or nodes.
158 */
159 bool cancelled;
160
161 GSList *bitmaps;
162 QemuMutex lock; /* protect bitmaps */
163 } DBMLoadState;
164
165 typedef struct DBMState {
166 DBMSaveState save;
167 DBMLoadState load;
168 } DBMState;
169
170 static DBMState dbm_state;
171
172 /* For hash tables that map node/bitmap names to aliases */
173 typedef struct AliasMapInnerNode {
174 char *string;
175 GHashTable *subtree;
176 } AliasMapInnerNode;
177
178 static void free_alias_map_inner_node(void *amin_ptr)
179 {
180 AliasMapInnerNode *amin = amin_ptr;
181
182 g_free(amin->string);
183 g_hash_table_unref(amin->subtree);
184 g_free(amin);
185 }
186
187 /**
188 * Construct an alias map based on the given QMP structure.
189 *
190 * (Note that we cannot store such maps in the MigrationParameters
191 * object, because that struct is defined by the QAPI schema, which
192 * makes it basically impossible to have dicts with arbitrary keys.
193 * Therefore, we instead have to construct these maps when migration
194 * starts.)
195 *
196 * @bbm is the block_bitmap_mapping from the migration parameters.
197 *
198 * If @name_to_alias is true, the returned hash table will map node
199 * and bitmap names to their respective aliases (for outgoing
200 * migration).
201 *
202 * If @name_to_alias is false, the returned hash table will map node
203 * and bitmap aliases to their respective names (for incoming
204 * migration).
205 *
206 * The hash table maps node names/aliases to AliasMapInnerNode
207 * objects, whose .string is the respective node alias/name, and whose
208 * .subtree table maps bitmap names/aliases to the respective bitmap
209 * alias/name.
210 */
211 static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm,
212 bool name_to_alias,
213 Error **errp)
214 {
215 GHashTable *alias_map;
216 size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1;
217
218 alias_map = g_hash_table_new_full(g_str_hash, g_str_equal,
219 g_free, free_alias_map_inner_node);
220
221 for (; bbm; bbm = bbm->next) {
222 const BitmapMigrationNodeAlias *bmna = bbm->value;
223 const BitmapMigrationBitmapAliasList *bmbal;
224 AliasMapInnerNode *amin;
225 GHashTable *bitmaps_map;
226 const char *node_map_from, *node_map_to;
227
228 if (!id_wellformed(bmna->alias)) {
229 error_setg(errp, "The node alias '%s' is not well-formed",
230 bmna->alias);
231 goto fail;
232 }
233
234 if (strlen(bmna->alias) > UINT8_MAX) {
235 error_setg(errp, "The node alias '%s' is longer than %u bytes",
236 bmna->alias, UINT8_MAX);
237 goto fail;
238 }
239
240 if (strlen(bmna->node_name) > max_node_name_len) {
241 error_setg(errp, "The node name '%s' is longer than %zu bytes",
242 bmna->node_name, max_node_name_len);
243 goto fail;
244 }
245
246 if (name_to_alias) {
247 if (g_hash_table_contains(alias_map, bmna->node_name)) {
248 error_setg(errp, "The node name '%s' is mapped twice",
249 bmna->node_name);
250 goto fail;
251 }
252
253 node_map_from = bmna->node_name;
254 node_map_to = bmna->alias;
255 } else {
256 if (g_hash_table_contains(alias_map, bmna->alias)) {
257 error_setg(errp, "The node alias '%s' is used twice",
258 bmna->alias);
259 goto fail;
260 }
261
262 node_map_from = bmna->alias;
263 node_map_to = bmna->node_name;
264 }
265
266 bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal,
267 g_free, g_free);
268
269 amin = g_new(AliasMapInnerNode, 1);
270 *amin = (AliasMapInnerNode){
271 .string = g_strdup(node_map_to),
272 .subtree = bitmaps_map,
273 };
274
275 g_hash_table_insert(alias_map, g_strdup(node_map_from), amin);
276
277 for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) {
278 const BitmapMigrationBitmapAlias *bmba = bmbal->value;
279 const char *bmap_map_from, *bmap_map_to;
280
281 if (strlen(bmba->alias) > UINT8_MAX) {
282 error_setg(errp,
283 "The bitmap alias '%s' is longer than %u bytes",
284 bmba->alias, UINT8_MAX);
285 goto fail;
286 }
287
288 if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) {
289 error_setg(errp, "The bitmap name '%s' is longer than %d bytes",
290 bmba->name, BDRV_BITMAP_MAX_NAME_SIZE);
291 goto fail;
292 }
293
294 if (name_to_alias) {
295 bmap_map_from = bmba->name;
296 bmap_map_to = bmba->alias;
297
298 if (g_hash_table_contains(bitmaps_map, bmba->name)) {
299 error_setg(errp, "The bitmap '%s'/'%s' is mapped twice",
300 bmna->node_name, bmba->name);
301 goto fail;
302 }
303 } else {
304 bmap_map_from = bmba->alias;
305 bmap_map_to = bmba->name;
306
307 if (g_hash_table_contains(bitmaps_map, bmba->alias)) {
308 error_setg(errp, "The bitmap alias '%s'/'%s' is used twice",
309 bmna->alias, bmba->alias);
310 goto fail;
311 }
312 }
313
314 g_hash_table_insert(bitmaps_map,
315 g_strdup(bmap_map_from), g_strdup(bmap_map_to));
316 }
317 }
318
319 return alias_map;
320
321 fail:
322 g_hash_table_destroy(alias_map);
323 return NULL;
324 }
325
326 /**
327 * Run construct_alias_map() in both directions to check whether @bbm
328 * is valid.
329 * (This function is to be used by migration/migration.c to validate
330 * the user-specified block-bitmap-mapping migration parameter.)
331 *
332 * Returns true if and only if the mapping is valid.
333 */
334 bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
335 Error **errp)
336 {
337 GHashTable *alias_map;
338
339 alias_map = construct_alias_map(bbm, true, errp);
340 if (!alias_map) {
341 return false;
342 }
343 g_hash_table_destroy(alias_map);
344
345 alias_map = construct_alias_map(bbm, false, errp);
346 if (!alias_map) {
347 return false;
348 }
349 g_hash_table_destroy(alias_map);
350
351 return true;
352 }
353
354 static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
355 {
356 uint8_t flags = qemu_get_byte(f);
357 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
358 flags = flags << 8 | qemu_get_byte(f);
359 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
360 flags = flags << 16 | qemu_get_be16(f);
361 }
362 }
363
364 return flags;
365 }
366
367 static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags)
368 {
369 /* The code currently does not send flags as more than one byte */
370 assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS)));
371
372 qemu_put_byte(f, flags);
373 }
374
375 static void send_bitmap_header(QEMUFile *f, DBMSaveState *s,
376 SaveBitmapState *dbms, uint32_t additional_flags)
377 {
378 BlockDriverState *bs = dbms->bs;
379 BdrvDirtyBitmap *bitmap = dbms->bitmap;
380 uint32_t flags = additional_flags;
381 trace_send_bitmap_header_enter();
382
383 if (bs != s->prev_bs) {
384 s->prev_bs = bs;
385 flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME;
386 }
387
388 if (bitmap != s->prev_bitmap) {
389 s->prev_bitmap = bitmap;
390 flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME;
391 }
392
393 qemu_put_bitmap_flags(f, flags);
394
395 if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
396 qemu_put_counted_string(f, dbms->node_alias);
397 }
398
399 if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
400 qemu_put_counted_string(f, dbms->bitmap_alias);
401 }
402 }
403
404 static void send_bitmap_start(QEMUFile *f, DBMSaveState *s,
405 SaveBitmapState *dbms)
406 {
407 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_START);
408 qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap));
409 qemu_put_byte(f, dbms->flags);
410 }
411
412 static void send_bitmap_complete(QEMUFile *f, DBMSaveState *s,
413 SaveBitmapState *dbms)
414 {
415 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE);
416 }
417
418 static void send_bitmap_bits(QEMUFile *f, DBMSaveState *s,
419 SaveBitmapState *dbms,
420 uint64_t start_sector, uint32_t nr_sectors)
421 {
422 /* align for buffer_is_zero() */
423 uint64_t align = 4 * sizeof(long);
424 uint64_t unaligned_size =
425 bdrv_dirty_bitmap_serialization_size(
426 dbms->bitmap, start_sector << BDRV_SECTOR_BITS,
427 (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
428 uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align);
429 uint8_t *buf = g_malloc0(buf_size);
430 uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS;
431
432 bdrv_dirty_bitmap_serialize_part(
433 dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS,
434 (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
435
436 if (buffer_is_zero(buf, buf_size)) {
437 g_free(buf);
438 buf = NULL;
439 flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES;
440 }
441
442 trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size);
443
444 send_bitmap_header(f, s, dbms, flags);
445
446 qemu_put_be64(f, start_sector);
447 qemu_put_be32(f, nr_sectors);
448
449 /* if a block is zero we need to flush here since the network
450 * bandwidth is now a lot higher than the storage device bandwidth.
451 * thus if we queue zero blocks we slow down the migration. */
452 if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
453 qemu_fflush(f);
454 } else {
455 qemu_put_be64(f, buf_size);
456 qemu_put_buffer(f, buf, buf_size);
457 }
458
459 g_free(buf);
460 }
461
462 /* Called with iothread lock taken. */
463 static void dirty_bitmap_do_save_cleanup(DBMSaveState *s)
464 {
465 SaveBitmapState *dbms;
466
467 while ((dbms = QSIMPLEQ_FIRST(&s->dbms_list)) != NULL) {
468 QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry);
469 bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
470 bdrv_unref(dbms->bs);
471 g_free(dbms->node_alias);
472 g_free(dbms->bitmap_alias);
473 g_free(dbms);
474 }
475 }
476
477 /* Called with iothread lock taken. */
478 static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs,
479 const char *bs_name, GHashTable *alias_map)
480 {
481 BdrvDirtyBitmap *bitmap;
482 SaveBitmapState *dbms;
483 GHashTable *bitmap_aliases;
484 const char *node_alias, *bitmap_name, *bitmap_alias;
485 Error *local_err = NULL;
486
487 /* When an alias map is given, @bs_name must be @bs's node name */
488 assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs)));
489
490 FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
491 if (bdrv_dirty_bitmap_name(bitmap)) {
492 break;
493 }
494 }
495 if (!bitmap) {
496 return 0;
497 }
498
499 bitmap_name = bdrv_dirty_bitmap_name(bitmap);
500
501 if (!bs_name || strcmp(bs_name, "") == 0) {
502 error_report("Bitmap '%s' in unnamed node can't be migrated",
503 bitmap_name);
504 return -1;
505 }
506
507 if (alias_map) {
508 const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name);
509
510 if (!amin) {
511 /* Skip bitmaps on nodes with no alias */
512 return 0;
513 }
514
515 node_alias = amin->string;
516 bitmap_aliases = amin->subtree;
517 } else {
518 node_alias = bs_name;
519 bitmap_aliases = NULL;
520 }
521
522 if (node_alias[0] == '#') {
523 error_report("Bitmap '%s' in a node with auto-generated "
524 "name '%s' can't be migrated",
525 bitmap_name, node_alias);
526 return -1;
527 }
528
529 FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
530 bitmap_name = bdrv_dirty_bitmap_name(bitmap);
531 if (!bitmap_name) {
532 continue;
533 }
534
535 if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) {
536 error_report_err(local_err);
537 return -1;
538 }
539
540 if (bitmap_aliases) {
541 bitmap_alias = g_hash_table_lookup(bitmap_aliases, bitmap_name);
542 if (!bitmap_alias) {
543 /* Skip bitmaps with no alias */
544 continue;
545 }
546 } else {
547 if (strlen(bitmap_name) > UINT8_MAX) {
548 error_report("Cannot migrate bitmap '%s' on node '%s': "
549 "Name is longer than %u bytes",
550 bitmap_name, bs_name, UINT8_MAX);
551 return -1;
552 }
553 bitmap_alias = bitmap_name;
554 }
555
556 bdrv_ref(bs);
557 bdrv_dirty_bitmap_set_busy(bitmap, true);
558
559 dbms = g_new0(SaveBitmapState, 1);
560 dbms->bs = bs;
561 dbms->node_alias = g_strdup(node_alias);
562 dbms->bitmap_alias = g_strdup(bitmap_alias);
563 dbms->bitmap = bitmap;
564 dbms->total_sectors = bdrv_nb_sectors(bs);
565 dbms->sectors_per_chunk = CHUNK_SIZE * 8 *
566 bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS;
567 if (bdrv_dirty_bitmap_enabled(bitmap)) {
568 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
569 }
570 if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
571 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
572 }
573
574 QSIMPLEQ_INSERT_TAIL(&s->dbms_list, dbms, entry);
575 }
576
577 return 0;
578 }
579
580 /* Called with iothread lock taken. */
581 static int init_dirty_bitmap_migration(DBMSaveState *s)
582 {
583 BlockDriverState *bs;
584 SaveBitmapState *dbms;
585 GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL);
586 BlockBackend *blk;
587 const MigrationParameters *mig_params = &migrate_get_current()->parameters;
588 GHashTable *alias_map = NULL;
589
590 if (mig_params->has_block_bitmap_mapping) {
591 alias_map = construct_alias_map(mig_params->block_bitmap_mapping, true,
592 &error_abort);
593 }
594
595 s->bulk_completed = false;
596 s->prev_bs = NULL;
597 s->prev_bitmap = NULL;
598 s->no_bitmaps = false;
599
600 if (!alias_map) {
601 /*
602 * Use blockdevice name for direct (or filtered) children of named block
603 * backends.
604 */
605 for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
606 const char *name = blk_name(blk);
607
608 if (!name || strcmp(name, "") == 0) {
609 continue;
610 }
611
612 bs = blk_bs(blk);
613
614 /* Skip filters without bitmaps */
615 while (bs && bs->drv && bs->drv->is_filter &&
616 !bdrv_has_named_bitmaps(bs))
617 {
618 if (bs->backing) {
619 bs = bs->backing->bs;
620 } else if (bs->file) {
621 bs = bs->file->bs;
622 } else {
623 bs = NULL;
624 }
625 }
626
627 if (bs && bs->drv && !bs->drv->is_filter) {
628 if (add_bitmaps_to_list(s, bs, name, NULL)) {
629 goto fail;
630 }
631 g_hash_table_add(handled_by_blk, bs);
632 }
633 }
634 }
635
636 for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
637 if (g_hash_table_contains(handled_by_blk, bs)) {
638 continue;
639 }
640
641 if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) {
642 goto fail;
643 }
644 }
645
646 /* unset migration flags here, to not roll back it */
647 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
648 bdrv_dirty_bitmap_skip_store(dbms->bitmap, true);
649 }
650
651 if (QSIMPLEQ_EMPTY(&s->dbms_list)) {
652 s->no_bitmaps = true;
653 }
654
655 g_hash_table_destroy(handled_by_blk);
656 if (alias_map) {
657 g_hash_table_destroy(alias_map);
658 }
659
660 return 0;
661
662 fail:
663 g_hash_table_destroy(handled_by_blk);
664 if (alias_map) {
665 g_hash_table_destroy(alias_map);
666 }
667 dirty_bitmap_do_save_cleanup(s);
668
669 return -1;
670 }
671
672 /* Called with no lock taken. */
673 static void bulk_phase_send_chunk(QEMUFile *f, DBMSaveState *s,
674 SaveBitmapState *dbms)
675 {
676 uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector,
677 dbms->sectors_per_chunk);
678
679 send_bitmap_bits(f, s, dbms, dbms->cur_sector, nr_sectors);
680
681 dbms->cur_sector += nr_sectors;
682 if (dbms->cur_sector >= dbms->total_sectors) {
683 dbms->bulk_completed = true;
684 }
685 }
686
687 /* Called with no lock taken. */
688 static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit)
689 {
690 SaveBitmapState *dbms;
691
692 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
693 while (!dbms->bulk_completed) {
694 bulk_phase_send_chunk(f, s, dbms);
695 if (limit && qemu_file_rate_limit(f)) {
696 return;
697 }
698 }
699 }
700
701 s->bulk_completed = true;
702 }
703
704 /* for SaveVMHandlers */
705 static void dirty_bitmap_save_cleanup(void *opaque)
706 {
707 DBMSaveState *s = &((DBMState *)opaque)->save;
708
709 dirty_bitmap_do_save_cleanup(s);
710 }
711
712 static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque)
713 {
714 DBMSaveState *s = &((DBMState *)opaque)->save;
715
716 trace_dirty_bitmap_save_iterate(migration_in_postcopy());
717
718 if (migration_in_postcopy() && !s->bulk_completed) {
719 bulk_phase(f, s, true);
720 }
721
722 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
723
724 return s->bulk_completed;
725 }
726
727 /* Called with iothread lock taken. */
728
729 static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque)
730 {
731 DBMSaveState *s = &((DBMState *)opaque)->save;
732 SaveBitmapState *dbms;
733 trace_dirty_bitmap_save_complete_enter();
734
735 if (!s->bulk_completed) {
736 bulk_phase(f, s, false);
737 }
738
739 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
740 send_bitmap_complete(f, s, dbms);
741 }
742
743 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
744
745 trace_dirty_bitmap_save_complete_finish();
746
747 dirty_bitmap_save_cleanup(opaque);
748 return 0;
749 }
750
751 static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque,
752 uint64_t max_size,
753 uint64_t *res_precopy_only,
754 uint64_t *res_compatible,
755 uint64_t *res_postcopy_only)
756 {
757 DBMSaveState *s = &((DBMState *)opaque)->save;
758 SaveBitmapState *dbms;
759 uint64_t pending = 0;
760
761 qemu_mutex_lock_iothread();
762
763 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
764 uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap);
765 uint64_t sectors = dbms->bulk_completed ? 0 :
766 dbms->total_sectors - dbms->cur_sector;
767
768 pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran);
769 }
770
771 qemu_mutex_unlock_iothread();
772
773 trace_dirty_bitmap_save_pending(pending, max_size);
774
775 *res_postcopy_only += pending;
776 }
777
778 /* First occurrence of this bitmap. It should be created if doesn't exist */
779 static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s)
780 {
781 Error *local_err = NULL;
782 uint32_t granularity = qemu_get_be32(f);
783 uint8_t flags = qemu_get_byte(f);
784 LoadBitmapState *b;
785
786 if (s->cancelled) {
787 return 0;
788 }
789
790 if (s->bitmap) {
791 error_report("Bitmap with the same name ('%s') already exists on "
792 "destination", bdrv_dirty_bitmap_name(s->bitmap));
793 return -EINVAL;
794 } else {
795 s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity,
796 s->bitmap_name, &local_err);
797 if (!s->bitmap) {
798 error_report_err(local_err);
799 return -EINVAL;
800 }
801 }
802
803 if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) {
804 error_report("Unknown flags in migrated dirty bitmap header: %x",
805 flags);
806 return -EINVAL;
807 }
808
809 if (flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT) {
810 bdrv_dirty_bitmap_set_persistence(s->bitmap, true);
811 }
812
813 bdrv_disable_dirty_bitmap(s->bitmap);
814 if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) {
815 bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err);
816 if (local_err) {
817 error_report_err(local_err);
818 return -EINVAL;
819 }
820 }
821
822 b = g_new(LoadBitmapState, 1);
823 b->bs = s->bs;
824 b->bitmap = s->bitmap;
825 b->migrated = false;
826 b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
827
828 s->bitmaps = g_slist_prepend(s->bitmaps, b);
829
830 return 0;
831 }
832
833 /*
834 * before_vm_start_handle_item
835 *
836 * g_slist_foreach helper
837 *
838 * item is LoadBitmapState*
839 * opaque is DBMLoadState*
840 */
841 static void before_vm_start_handle_item(void *item, void *opaque)
842 {
843 DBMLoadState *s = opaque;
844 LoadBitmapState *b = item;
845
846 if (b->enabled) {
847 if (b->migrated) {
848 bdrv_enable_dirty_bitmap(b->bitmap);
849 } else {
850 bdrv_dirty_bitmap_enable_successor(b->bitmap);
851 }
852 }
853
854 if (b->migrated) {
855 s->bitmaps = g_slist_remove(s->bitmaps, b);
856 g_free(b);
857 }
858 }
859
860 void dirty_bitmap_mig_before_vm_start(void)
861 {
862 DBMLoadState *s = &dbm_state.load;
863 qemu_mutex_lock(&s->lock);
864
865 assert(!s->before_vm_start_handled);
866 g_slist_foreach(s->bitmaps, before_vm_start_handle_item, s);
867 s->before_vm_start_handled = true;
868
869 qemu_mutex_unlock(&s->lock);
870 }
871
872 static void cancel_incoming_locked(DBMLoadState *s)
873 {
874 GSList *item;
875
876 if (s->cancelled) {
877 return;
878 }
879
880 s->cancelled = true;
881 s->bs = NULL;
882 s->bitmap = NULL;
883
884 /* Drop all unfinished bitmaps */
885 for (item = s->bitmaps; item; item = g_slist_next(item)) {
886 LoadBitmapState *b = item->data;
887
888 /*
889 * Bitmap must be unfinished, as finished bitmaps should already be
890 * removed from the list.
891 */
892 assert(!s->before_vm_start_handled || !b->migrated);
893 if (bdrv_dirty_bitmap_has_successor(b->bitmap)) {
894 bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort);
895 }
896 bdrv_release_dirty_bitmap(b->bitmap);
897 }
898
899 g_slist_free_full(s->bitmaps, g_free);
900 s->bitmaps = NULL;
901 }
902
903 void dirty_bitmap_mig_cancel_outgoing(void)
904 {
905 dirty_bitmap_do_save_cleanup(&dbm_state.save);
906 }
907
908 void dirty_bitmap_mig_cancel_incoming(void)
909 {
910 DBMLoadState *s = &dbm_state.load;
911
912 qemu_mutex_lock(&s->lock);
913
914 cancel_incoming_locked(s);
915
916 qemu_mutex_unlock(&s->lock);
917 }
918
919 static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s)
920 {
921 GSList *item;
922 trace_dirty_bitmap_load_complete();
923
924 if (s->cancelled) {
925 return;
926 }
927
928 bdrv_dirty_bitmap_deserialize_finish(s->bitmap);
929
930 if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
931 bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort);
932 }
933
934 for (item = s->bitmaps; item; item = g_slist_next(item)) {
935 LoadBitmapState *b = item->data;
936
937 if (b->bitmap == s->bitmap) {
938 b->migrated = true;
939 if (s->before_vm_start_handled) {
940 s->bitmaps = g_slist_remove(s->bitmaps, b);
941 g_free(b);
942 }
943 break;
944 }
945 }
946 }
947
948 static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s)
949 {
950 uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS;
951 uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS;
952 trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS,
953 nr_bytes >> BDRV_SECTOR_BITS);
954
955 if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
956 trace_dirty_bitmap_load_bits_zeroes();
957 if (!s->cancelled) {
958 bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte,
959 nr_bytes, false);
960 }
961 } else {
962 size_t ret;
963 g_autofree uint8_t *buf = NULL;
964 uint64_t buf_size = qemu_get_be64(f);
965 uint64_t needed_size;
966
967 /*
968 * The actual check for buf_size is done a bit later. We can't do it in
969 * cancelled mode as we don't have the bitmap to check the constraints
970 * (so, we allocate a buffer and read prior to the check). On the other
971 * hand, we shouldn't blindly g_malloc the number from the stream.
972 * Actually one chunk should not be larger than CHUNK_SIZE. Let's allow
973 * a bit larger (which means that bitmap migration will fail anyway and
974 * the whole migration will most probably fail soon due to broken
975 * stream).
976 */
977 if (buf_size > 10 * CHUNK_SIZE) {
978 error_report("Bitmap migration stream buffer allocation request "
979 "is too large");
980 return -EIO;
981 }
982
983 buf = g_malloc(buf_size);
984 ret = qemu_get_buffer(f, buf, buf_size);
985 if (ret != buf_size) {
986 error_report("Failed to read bitmap bits");
987 return -EIO;
988 }
989
990 if (s->cancelled) {
991 return 0;
992 }
993
994 needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap,
995 first_byte,
996 nr_bytes);
997
998 if (needed_size > buf_size ||
999 buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long))
1000 /* Here used same alignment as in send_bitmap_bits */
1001 ) {
1002 error_report("Migrated bitmap granularity doesn't "
1003 "match the destination bitmap '%s' granularity",
1004 bdrv_dirty_bitmap_name(s->bitmap));
1005 cancel_incoming_locked(s);
1006 return 0;
1007 }
1008
1009 bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes,
1010 false);
1011 }
1012
1013 return 0;
1014 }
1015
1016 static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s,
1017 GHashTable *alias_map)
1018 {
1019 GHashTable *bitmap_alias_map = NULL;
1020 Error *local_err = NULL;
1021 bool nothing;
1022 s->flags = qemu_get_bitmap_flags(f);
1023 trace_dirty_bitmap_load_header(s->flags);
1024
1025 nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS);
1026
1027 if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
1028 if (!qemu_get_counted_string(f, s->node_alias)) {
1029 error_report("Unable to read node alias string");
1030 return -EINVAL;
1031 }
1032
1033 if (!s->cancelled) {
1034 if (alias_map) {
1035 const AliasMapInnerNode *amin;
1036
1037 amin = g_hash_table_lookup(alias_map, s->node_alias);
1038 if (!amin) {
1039 error_setg(&local_err, "Error: Unknown node alias '%s'",
1040 s->node_alias);
1041 s->bs = NULL;
1042 } else {
1043 bitmap_alias_map = amin->subtree;
1044 s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err);
1045 }
1046 } else {
1047 s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias,
1048 &local_err);
1049 }
1050 if (!s->bs) {
1051 error_report_err(local_err);
1052 cancel_incoming_locked(s);
1053 }
1054 }
1055 } else if (s->bs) {
1056 if (alias_map) {
1057 const AliasMapInnerNode *amin;
1058
1059 /* Must be present in the map, or s->bs would not be set */
1060 amin = g_hash_table_lookup(alias_map, s->node_alias);
1061 assert(amin != NULL);
1062
1063 bitmap_alias_map = amin->subtree;
1064 }
1065 } else if (!nothing && !s->cancelled) {
1066 error_report("Error: block device name is not set");
1067 cancel_incoming_locked(s);
1068 }
1069
1070 assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map);
1071
1072 if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
1073 const char *bitmap_name;
1074
1075 if (!qemu_get_counted_string(f, s->bitmap_alias)) {
1076 error_report("Unable to read bitmap alias string");
1077 return -EINVAL;
1078 }
1079
1080 if (!s->cancelled) {
1081 if (bitmap_alias_map) {
1082 bitmap_name = g_hash_table_lookup(bitmap_alias_map,
1083 s->bitmap_alias);
1084 if (!bitmap_name) {
1085 error_report("Error: Unknown bitmap alias '%s' on node "
1086 "'%s' (alias '%s')", s->bitmap_alias,
1087 s->bs->node_name, s->node_alias);
1088 cancel_incoming_locked(s);
1089 }
1090 } else {
1091 bitmap_name = s->bitmap_alias;
1092 }
1093 }
1094
1095 if (!s->cancelled) {
1096 g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name));
1097 s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
1098
1099 /*
1100 * bitmap may be NULL here, it wouldn't be an error if it is the
1101 * first occurrence of the bitmap
1102 */
1103 if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
1104 error_report("Error: unknown dirty bitmap "
1105 "'%s' for block device '%s'",
1106 s->bitmap_name, s->bs->node_name);
1107 cancel_incoming_locked(s);
1108 }
1109 }
1110 } else if (!s->bitmap && !nothing && !s->cancelled) {
1111 error_report("Error: block device name is not set");
1112 cancel_incoming_locked(s);
1113 }
1114
1115 return 0;
1116 }
1117
1118 /*
1119 * dirty_bitmap_load
1120 *
1121 * Load sequence of dirty bitmap chunks. Return error only on fatal io stream
1122 * violations. On other errors just cancel bitmaps incoming migration and return
1123 * 0.
1124 *
1125 * Note, than when incoming bitmap migration is canceled, we still must read all
1126 * our chunks (and just ignore them), to not affect other migration objects.
1127 */
1128 static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
1129 {
1130 GHashTable *alias_map = NULL;
1131 const MigrationParameters *mig_params = &migrate_get_current()->parameters;
1132 DBMLoadState *s = &((DBMState *)opaque)->load;
1133 int ret = 0;
1134
1135 trace_dirty_bitmap_load_enter();
1136
1137 if (version_id != 1) {
1138 QEMU_LOCK_GUARD(&s->lock);
1139 cancel_incoming_locked(s);
1140 return -EINVAL;
1141 }
1142
1143 if (mig_params->has_block_bitmap_mapping) {
1144 alias_map = construct_alias_map(mig_params->block_bitmap_mapping,
1145 false, &error_abort);
1146 }
1147
1148 do {
1149 QEMU_LOCK_GUARD(&s->lock);
1150
1151 ret = dirty_bitmap_load_header(f, s, alias_map);
1152 if (ret < 0) {
1153 cancel_incoming_locked(s);
1154 goto fail;
1155 }
1156
1157 if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) {
1158 ret = dirty_bitmap_load_start(f, s);
1159 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) {
1160 dirty_bitmap_load_complete(f, s);
1161 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITS) {
1162 ret = dirty_bitmap_load_bits(f, s);
1163 }
1164
1165 if (!ret) {
1166 ret = qemu_file_get_error(f);
1167 }
1168
1169 if (ret) {
1170 cancel_incoming_locked(s);
1171 goto fail;
1172 }
1173 } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS));
1174
1175 trace_dirty_bitmap_load_success();
1176 ret = 0;
1177 fail:
1178 if (alias_map) {
1179 g_hash_table_destroy(alias_map);
1180 }
1181 return ret;
1182 }
1183
1184 static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque)
1185 {
1186 DBMSaveState *s = &((DBMState *)opaque)->save;
1187 SaveBitmapState *dbms = NULL;
1188 if (init_dirty_bitmap_migration(s) < 0) {
1189 return -1;
1190 }
1191
1192 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
1193 send_bitmap_start(f, s, dbms);
1194 }
1195 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
1196
1197 return 0;
1198 }
1199
1200 static bool dirty_bitmap_is_active(void *opaque)
1201 {
1202 DBMSaveState *s = &((DBMState *)opaque)->save;
1203
1204 return migrate_dirty_bitmaps() && !s->no_bitmaps;
1205 }
1206
1207 static bool dirty_bitmap_is_active_iterate(void *opaque)
1208 {
1209 return dirty_bitmap_is_active(opaque) && !runstate_is_running();
1210 }
1211
1212 static bool dirty_bitmap_has_postcopy(void *opaque)
1213 {
1214 return true;
1215 }
1216
1217 static SaveVMHandlers savevm_dirty_bitmap_handlers = {
1218 .save_setup = dirty_bitmap_save_setup,
1219 .save_live_complete_postcopy = dirty_bitmap_save_complete,
1220 .save_live_complete_precopy = dirty_bitmap_save_complete,
1221 .has_postcopy = dirty_bitmap_has_postcopy,
1222 .save_live_pending = dirty_bitmap_save_pending,
1223 .save_live_iterate = dirty_bitmap_save_iterate,
1224 .is_active_iterate = dirty_bitmap_is_active_iterate,
1225 .load_state = dirty_bitmap_load,
1226 .save_cleanup = dirty_bitmap_save_cleanup,
1227 .is_active = dirty_bitmap_is_active,
1228 };
1229
1230 void dirty_bitmap_mig_init(void)
1231 {
1232 QSIMPLEQ_INIT(&dbm_state.save.dbms_list);
1233 qemu_mutex_init(&dbm_state.load.lock);
1234
1235 register_savevm_live("dirty-bitmap", 0, 1,
1236 &savevm_dirty_bitmap_handlers,
1237 &dbm_state);
1238 }