Merge remote-tracking branch 'remotes/mcayland/tags/qemu-sparc-20210412' into staging
[qemu.git] / migration / block.c
1 /*
2 * QEMU live block migration
3 *
4 * Copyright IBM, Corp. 2009
5 *
6 * Authors:
7 * Liran Schour <lirans@il.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
14 */
15
16 #include "qemu/osdep.h"
17 #include "qapi/error.h"
18 #include "qemu/error-report.h"
19 #include "qemu/main-loop.h"
20 #include "qemu/cutils.h"
21 #include "qemu/queue.h"
22 #include "block.h"
23 #include "migration/misc.h"
24 #include "migration.h"
25 #include "migration/register.h"
26 #include "qemu-file.h"
27 #include "migration/vmstate.h"
28 #include "sysemu/block-backend.h"
29 #include "trace.h"
30
31 #define BLK_MIG_BLOCK_SIZE (1 << 20)
32 #define BDRV_SECTORS_PER_DIRTY_CHUNK (BLK_MIG_BLOCK_SIZE >> BDRV_SECTOR_BITS)
33
34 #define BLK_MIG_FLAG_DEVICE_BLOCK 0x01
35 #define BLK_MIG_FLAG_EOS 0x02
36 #define BLK_MIG_FLAG_PROGRESS 0x04
37 #define BLK_MIG_FLAG_ZERO_BLOCK 0x08
38
39 #define MAX_IS_ALLOCATED_SEARCH (65536 * BDRV_SECTOR_SIZE)
40
41 #define MAX_IO_BUFFERS 512
42 #define MAX_PARALLEL_IO 16
43
44 /* #define DEBUG_BLK_MIGRATION */
45
46 #ifdef DEBUG_BLK_MIGRATION
47 #define DPRINTF(fmt, ...) \
48 do { printf("blk_migration: " fmt, ## __VA_ARGS__); } while (0)
49 #else
50 #define DPRINTF(fmt, ...) \
51 do { } while (0)
52 #endif
53
54 typedef struct BlkMigDevState {
55 /* Written during setup phase. Can be read without a lock. */
56 BlockBackend *blk;
57 char *blk_name;
58 int shared_base;
59 int64_t total_sectors;
60 QSIMPLEQ_ENTRY(BlkMigDevState) entry;
61 Error *blocker;
62
63 /* Only used by migration thread. Does not need a lock. */
64 int bulk_completed;
65 int64_t cur_sector;
66 int64_t cur_dirty;
67
68 /* Data in the aio_bitmap is protected by block migration lock.
69 * Allocation and free happen during setup and cleanup respectively.
70 */
71 unsigned long *aio_bitmap;
72
73 /* Protected by block migration lock. */
74 int64_t completed_sectors;
75
76 /* During migration this is protected by iothread lock / AioContext.
77 * Allocation and free happen during setup and cleanup respectively.
78 */
79 BdrvDirtyBitmap *dirty_bitmap;
80 } BlkMigDevState;
81
82 typedef struct BlkMigBlock {
83 /* Only used by migration thread. */
84 uint8_t *buf;
85 BlkMigDevState *bmds;
86 int64_t sector;
87 int nr_sectors;
88 QEMUIOVector qiov;
89 BlockAIOCB *aiocb;
90
91 /* Protected by block migration lock. */
92 int ret;
93 QSIMPLEQ_ENTRY(BlkMigBlock) entry;
94 } BlkMigBlock;
95
96 typedef struct BlkMigState {
97 QSIMPLEQ_HEAD(, BlkMigDevState) bmds_list;
98 int64_t total_sector_sum;
99 bool zero_blocks;
100
101 /* Protected by lock. */
102 QSIMPLEQ_HEAD(, BlkMigBlock) blk_list;
103 int submitted;
104 int read_done;
105
106 /* Only used by migration thread. Does not need a lock. */
107 int transferred;
108 int prev_progress;
109 int bulk_completed;
110
111 /* Lock must be taken _inside_ the iothread lock and any AioContexts. */
112 QemuMutex lock;
113 } BlkMigState;
114
115 static BlkMigState block_mig_state;
116
117 static void blk_mig_lock(void)
118 {
119 qemu_mutex_lock(&block_mig_state.lock);
120 }
121
122 static void blk_mig_unlock(void)
123 {
124 qemu_mutex_unlock(&block_mig_state.lock);
125 }
126
127 /* Must run outside of the iothread lock during the bulk phase,
128 * or the VM will stall.
129 */
130
131 static void blk_send(QEMUFile *f, BlkMigBlock * blk)
132 {
133 int len;
134 uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK;
135
136 if (block_mig_state.zero_blocks &&
137 buffer_is_zero(blk->buf, BLK_MIG_BLOCK_SIZE)) {
138 flags |= BLK_MIG_FLAG_ZERO_BLOCK;
139 }
140
141 /* sector number and flags */
142 qemu_put_be64(f, (blk->sector << BDRV_SECTOR_BITS)
143 | flags);
144
145 /* device name */
146 len = strlen(blk->bmds->blk_name);
147 qemu_put_byte(f, len);
148 qemu_put_buffer(f, (uint8_t *) blk->bmds->blk_name, len);
149
150 /* if a block is zero we need to flush here since the network
151 * bandwidth is now a lot higher than the storage device bandwidth.
152 * thus if we queue zero blocks we slow down the migration */
153 if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
154 qemu_fflush(f);
155 return;
156 }
157
158 qemu_put_buffer(f, blk->buf, BLK_MIG_BLOCK_SIZE);
159 }
160
161 int blk_mig_active(void)
162 {
163 return !QSIMPLEQ_EMPTY(&block_mig_state.bmds_list);
164 }
165
166 int blk_mig_bulk_active(void)
167 {
168 return blk_mig_active() && !block_mig_state.bulk_completed;
169 }
170
171 uint64_t blk_mig_bytes_transferred(void)
172 {
173 BlkMigDevState *bmds;
174 uint64_t sum = 0;
175
176 blk_mig_lock();
177 QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
178 sum += bmds->completed_sectors;
179 }
180 blk_mig_unlock();
181 return sum << BDRV_SECTOR_BITS;
182 }
183
184 uint64_t blk_mig_bytes_remaining(void)
185 {
186 return blk_mig_bytes_total() - blk_mig_bytes_transferred();
187 }
188
189 uint64_t blk_mig_bytes_total(void)
190 {
191 BlkMigDevState *bmds;
192 uint64_t sum = 0;
193
194 QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
195 sum += bmds->total_sectors;
196 }
197 return sum << BDRV_SECTOR_BITS;
198 }
199
200
201 /* Called with migration lock held. */
202
203 static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
204 {
205 int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
206
207 if (sector < blk_nb_sectors(bmds->blk)) {
208 return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] &
209 (1UL << (chunk % (sizeof(unsigned long) * 8))));
210 } else {
211 return 0;
212 }
213 }
214
215 /* Called with migration lock held. */
216
217 static void bmds_set_aio_inflight(BlkMigDevState *bmds, int64_t sector_num,
218 int nb_sectors, int set)
219 {
220 int64_t start, end;
221 unsigned long val, idx, bit;
222
223 start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
224 end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;
225
226 for (; start <= end; start++) {
227 idx = start / (sizeof(unsigned long) * 8);
228 bit = start % (sizeof(unsigned long) * 8);
229 val = bmds->aio_bitmap[idx];
230 if (set) {
231 val |= 1UL << bit;
232 } else {
233 val &= ~(1UL << bit);
234 }
235 bmds->aio_bitmap[idx] = val;
236 }
237 }
238
239 static void alloc_aio_bitmap(BlkMigDevState *bmds)
240 {
241 BlockBackend *bb = bmds->blk;
242 int64_t bitmap_size;
243
244 bitmap_size = blk_nb_sectors(bb) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
245 bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;
246
247 bmds->aio_bitmap = g_malloc0(bitmap_size);
248 }
249
250 /* Never hold migration lock when yielding to the main loop! */
251
252 static void blk_mig_read_cb(void *opaque, int ret)
253 {
254 BlkMigBlock *blk = opaque;
255
256 blk_mig_lock();
257 blk->ret = ret;
258
259 QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
260 bmds_set_aio_inflight(blk->bmds, blk->sector, blk->nr_sectors, 0);
261
262 block_mig_state.submitted--;
263 block_mig_state.read_done++;
264 assert(block_mig_state.submitted >= 0);
265 blk_mig_unlock();
266 }
267
268 /* Called with no lock taken. */
269
270 static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
271 {
272 int64_t total_sectors = bmds->total_sectors;
273 int64_t cur_sector = bmds->cur_sector;
274 BlockBackend *bb = bmds->blk;
275 BlkMigBlock *blk;
276 int nr_sectors;
277 int64_t count;
278
279 if (bmds->shared_base) {
280 qemu_mutex_lock_iothread();
281 aio_context_acquire(blk_get_aio_context(bb));
282 /* Skip unallocated sectors; intentionally treats failure or
283 * partial sector as an allocated sector */
284 while (cur_sector < total_sectors &&
285 !bdrv_is_allocated(blk_bs(bb), cur_sector * BDRV_SECTOR_SIZE,
286 MAX_IS_ALLOCATED_SEARCH, &count)) {
287 if (count < BDRV_SECTOR_SIZE) {
288 break;
289 }
290 cur_sector += count >> BDRV_SECTOR_BITS;
291 }
292 aio_context_release(blk_get_aio_context(bb));
293 qemu_mutex_unlock_iothread();
294 }
295
296 if (cur_sector >= total_sectors) {
297 bmds->cur_sector = bmds->completed_sectors = total_sectors;
298 return 1;
299 }
300
301 bmds->completed_sectors = cur_sector;
302
303 cur_sector &= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1);
304
305 /* we are going to transfer a full block even if it is not allocated */
306 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
307
308 if (total_sectors - cur_sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
309 nr_sectors = total_sectors - cur_sector;
310 }
311
312 blk = g_new(BlkMigBlock, 1);
313 blk->buf = g_malloc(BLK_MIG_BLOCK_SIZE);
314 blk->bmds = bmds;
315 blk->sector = cur_sector;
316 blk->nr_sectors = nr_sectors;
317
318 qemu_iovec_init_buf(&blk->qiov, blk->buf, nr_sectors * BDRV_SECTOR_SIZE);
319
320 blk_mig_lock();
321 block_mig_state.submitted++;
322 blk_mig_unlock();
323
324 /* We do not know if bs is under the main thread (and thus does
325 * not acquire the AioContext when doing AIO) or rather under
326 * dataplane. Thus acquire both the iothread mutex and the
327 * AioContext.
328 *
329 * This is ugly and will disappear when we make bdrv_* thread-safe,
330 * without the need to acquire the AioContext.
331 */
332 qemu_mutex_lock_iothread();
333 aio_context_acquire(blk_get_aio_context(bmds->blk));
334 bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, cur_sector * BDRV_SECTOR_SIZE,
335 nr_sectors * BDRV_SECTOR_SIZE);
336 blk->aiocb = blk_aio_preadv(bb, cur_sector * BDRV_SECTOR_SIZE, &blk->qiov,
337 0, blk_mig_read_cb, blk);
338 aio_context_release(blk_get_aio_context(bmds->blk));
339 qemu_mutex_unlock_iothread();
340
341 bmds->cur_sector = cur_sector + nr_sectors;
342 return (bmds->cur_sector >= total_sectors);
343 }
344
345 /* Called with iothread lock taken. */
346
347 static int set_dirty_tracking(void)
348 {
349 BlkMigDevState *bmds;
350 int ret;
351
352 QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
353 bmds->dirty_bitmap = bdrv_create_dirty_bitmap(blk_bs(bmds->blk),
354 BLK_MIG_BLOCK_SIZE,
355 NULL, NULL);
356 if (!bmds->dirty_bitmap) {
357 ret = -errno;
358 goto fail;
359 }
360 }
361 return 0;
362
363 fail:
364 QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
365 if (bmds->dirty_bitmap) {
366 bdrv_release_dirty_bitmap(bmds->dirty_bitmap);
367 }
368 }
369 return ret;
370 }
371
372 /* Called with iothread lock taken. */
373
374 static void unset_dirty_tracking(void)
375 {
376 BlkMigDevState *bmds;
377
378 QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
379 bdrv_release_dirty_bitmap(bmds->dirty_bitmap);
380 }
381 }
382
383 static int init_blk_migration(QEMUFile *f)
384 {
385 BlockDriverState *bs;
386 BlkMigDevState *bmds;
387 int64_t sectors;
388 BdrvNextIterator it;
389 int i, num_bs = 0;
390 struct {
391 BlkMigDevState *bmds;
392 BlockDriverState *bs;
393 } *bmds_bs;
394 Error *local_err = NULL;
395 int ret;
396
397 block_mig_state.submitted = 0;
398 block_mig_state.read_done = 0;
399 block_mig_state.transferred = 0;
400 block_mig_state.total_sector_sum = 0;
401 block_mig_state.prev_progress = -1;
402 block_mig_state.bulk_completed = 0;
403 block_mig_state.zero_blocks = migrate_zero_blocks();
404
405 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
406 num_bs++;
407 }
408 bmds_bs = g_malloc0(num_bs * sizeof(*bmds_bs));
409
410 for (i = 0, bs = bdrv_first(&it); bs; bs = bdrv_next(&it), i++) {
411 if (bdrv_is_read_only(bs)) {
412 continue;
413 }
414
415 sectors = bdrv_nb_sectors(bs);
416 if (sectors <= 0) {
417 ret = sectors;
418 bdrv_next_cleanup(&it);
419 goto out;
420 }
421
422 bmds = g_new0(BlkMigDevState, 1);
423 bmds->blk = blk_new(qemu_get_aio_context(),
424 BLK_PERM_CONSISTENT_READ, BLK_PERM_ALL);
425 bmds->blk_name = g_strdup(bdrv_get_device_name(bs));
426 bmds->bulk_completed = 0;
427 bmds->total_sectors = sectors;
428 bmds->completed_sectors = 0;
429 bmds->shared_base = migrate_use_block_incremental();
430
431 assert(i < num_bs);
432 bmds_bs[i].bmds = bmds;
433 bmds_bs[i].bs = bs;
434
435 block_mig_state.total_sector_sum += sectors;
436
437 if (bmds->shared_base) {
438 trace_migration_block_init_shared(bdrv_get_device_name(bs));
439 } else {
440 trace_migration_block_init_full(bdrv_get_device_name(bs));
441 }
442
443 QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry);
444 }
445
446 /* Can only insert new BDSes now because doing so while iterating block
447 * devices may end up in a deadlock (iterating the new BDSes, too). */
448 for (i = 0; i < num_bs; i++) {
449 BlkMigDevState *bmds = bmds_bs[i].bmds;
450 BlockDriverState *bs = bmds_bs[i].bs;
451
452 if (bmds) {
453 ret = blk_insert_bs(bmds->blk, bs, &local_err);
454 if (ret < 0) {
455 error_report_err(local_err);
456 goto out;
457 }
458
459 alloc_aio_bitmap(bmds);
460 error_setg(&bmds->blocker, "block device is in use by migration");
461 bdrv_op_block_all(bs, bmds->blocker);
462 }
463 }
464
465 ret = 0;
466 out:
467 g_free(bmds_bs);
468 return ret;
469 }
470
471 /* Called with no lock taken. */
472
473 static int blk_mig_save_bulked_block(QEMUFile *f)
474 {
475 int64_t completed_sector_sum = 0;
476 BlkMigDevState *bmds;
477 int progress;
478 int ret = 0;
479
480 QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
481 if (bmds->bulk_completed == 0) {
482 if (mig_save_device_bulk(f, bmds) == 1) {
483 /* completed bulk section for this device */
484 bmds->bulk_completed = 1;
485 }
486 completed_sector_sum += bmds->completed_sectors;
487 ret = 1;
488 break;
489 } else {
490 completed_sector_sum += bmds->completed_sectors;
491 }
492 }
493
494 if (block_mig_state.total_sector_sum != 0) {
495 progress = completed_sector_sum * 100 /
496 block_mig_state.total_sector_sum;
497 } else {
498 progress = 100;
499 }
500 if (progress != block_mig_state.prev_progress) {
501 block_mig_state.prev_progress = progress;
502 qemu_put_be64(f, (progress << BDRV_SECTOR_BITS)
503 | BLK_MIG_FLAG_PROGRESS);
504 DPRINTF("Completed %d %%\r", progress);
505 }
506
507 return ret;
508 }
509
510 static void blk_mig_reset_dirty_cursor(void)
511 {
512 BlkMigDevState *bmds;
513
514 QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
515 bmds->cur_dirty = 0;
516 }
517 }
518
519 /* Called with iothread lock and AioContext taken. */
520
521 static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
522 int is_async)
523 {
524 BlkMigBlock *blk;
525 int64_t total_sectors = bmds->total_sectors;
526 int64_t sector;
527 int nr_sectors;
528 int ret = -EIO;
529
530 for (sector = bmds->cur_dirty; sector < bmds->total_sectors;) {
531 blk_mig_lock();
532 if (bmds_aio_inflight(bmds, sector)) {
533 blk_mig_unlock();
534 blk_drain(bmds->blk);
535 } else {
536 blk_mig_unlock();
537 }
538 bdrv_dirty_bitmap_lock(bmds->dirty_bitmap);
539 if (bdrv_dirty_bitmap_get_locked(bmds->dirty_bitmap,
540 sector * BDRV_SECTOR_SIZE)) {
541 if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
542 nr_sectors = total_sectors - sector;
543 } else {
544 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
545 }
546 bdrv_reset_dirty_bitmap_locked(bmds->dirty_bitmap,
547 sector * BDRV_SECTOR_SIZE,
548 nr_sectors * BDRV_SECTOR_SIZE);
549 bdrv_dirty_bitmap_unlock(bmds->dirty_bitmap);
550
551 blk = g_new(BlkMigBlock, 1);
552 blk->buf = g_malloc(BLK_MIG_BLOCK_SIZE);
553 blk->bmds = bmds;
554 blk->sector = sector;
555 blk->nr_sectors = nr_sectors;
556
557 if (is_async) {
558 qemu_iovec_init_buf(&blk->qiov, blk->buf,
559 nr_sectors * BDRV_SECTOR_SIZE);
560
561 blk->aiocb = blk_aio_preadv(bmds->blk,
562 sector * BDRV_SECTOR_SIZE,
563 &blk->qiov, 0, blk_mig_read_cb,
564 blk);
565
566 blk_mig_lock();
567 block_mig_state.submitted++;
568 bmds_set_aio_inflight(bmds, sector, nr_sectors, 1);
569 blk_mig_unlock();
570 } else {
571 ret = blk_pread(bmds->blk, sector * BDRV_SECTOR_SIZE, blk->buf,
572 nr_sectors * BDRV_SECTOR_SIZE);
573 if (ret < 0) {
574 goto error;
575 }
576 blk_send(f, blk);
577
578 g_free(blk->buf);
579 g_free(blk);
580 }
581
582 sector += nr_sectors;
583 bmds->cur_dirty = sector;
584 break;
585 }
586
587 bdrv_dirty_bitmap_unlock(bmds->dirty_bitmap);
588 sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
589 bmds->cur_dirty = sector;
590 }
591
592 return (bmds->cur_dirty >= bmds->total_sectors);
593
594 error:
595 trace_migration_block_save_device_dirty(sector);
596 g_free(blk->buf);
597 g_free(blk);
598 return ret;
599 }
600
601 /* Called with iothread lock taken.
602 *
603 * return value:
604 * 0: too much data for max_downtime
605 * 1: few enough data for max_downtime
606 */
607 static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
608 {
609 BlkMigDevState *bmds;
610 int ret = 1;
611
612 QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
613 aio_context_acquire(blk_get_aio_context(bmds->blk));
614 ret = mig_save_device_dirty(f, bmds, is_async);
615 aio_context_release(blk_get_aio_context(bmds->blk));
616 if (ret <= 0) {
617 break;
618 }
619 }
620
621 return ret;
622 }
623
624 /* Called with no locks taken. */
625
626 static int flush_blks(QEMUFile *f)
627 {
628 BlkMigBlock *blk;
629 int ret = 0;
630
631 trace_migration_block_flush_blks("Enter", block_mig_state.submitted,
632 block_mig_state.read_done,
633 block_mig_state.transferred);
634
635 blk_mig_lock();
636 while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
637 if (qemu_file_rate_limit(f)) {
638 break;
639 }
640 if (blk->ret < 0) {
641 ret = blk->ret;
642 break;
643 }
644
645 QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
646 blk_mig_unlock();
647 blk_send(f, blk);
648 blk_mig_lock();
649
650 g_free(blk->buf);
651 g_free(blk);
652
653 block_mig_state.read_done--;
654 block_mig_state.transferred++;
655 assert(block_mig_state.read_done >= 0);
656 }
657 blk_mig_unlock();
658
659 trace_migration_block_flush_blks("Exit", block_mig_state.submitted,
660 block_mig_state.read_done,
661 block_mig_state.transferred);
662 return ret;
663 }
664
665 /* Called with iothread lock taken. */
666
667 static int64_t get_remaining_dirty(void)
668 {
669 BlkMigDevState *bmds;
670 int64_t dirty = 0;
671
672 QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
673 aio_context_acquire(blk_get_aio_context(bmds->blk));
674 dirty += bdrv_get_dirty_count(bmds->dirty_bitmap);
675 aio_context_release(blk_get_aio_context(bmds->blk));
676 }
677
678 return dirty;
679 }
680
681
682
683 /* Called with iothread lock taken. */
684 static void block_migration_cleanup_bmds(void)
685 {
686 BlkMigDevState *bmds;
687 AioContext *ctx;
688
689 unset_dirty_tracking();
690
691 while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
692 QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
693 bdrv_op_unblock_all(blk_bs(bmds->blk), bmds->blocker);
694 error_free(bmds->blocker);
695
696 /* Save ctx, because bmds->blk can disappear during blk_unref. */
697 ctx = blk_get_aio_context(bmds->blk);
698 aio_context_acquire(ctx);
699 blk_unref(bmds->blk);
700 aio_context_release(ctx);
701
702 g_free(bmds->blk_name);
703 g_free(bmds->aio_bitmap);
704 g_free(bmds);
705 }
706 }
707
708 /* Called with iothread lock taken. */
709 static void block_migration_cleanup(void *opaque)
710 {
711 BlkMigBlock *blk;
712
713 bdrv_drain_all();
714
715 block_migration_cleanup_bmds();
716
717 blk_mig_lock();
718 while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
719 QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
720 g_free(blk->buf);
721 g_free(blk);
722 }
723 blk_mig_unlock();
724 }
725
726 static int block_save_setup(QEMUFile *f, void *opaque)
727 {
728 int ret;
729
730 trace_migration_block_save("setup", block_mig_state.submitted,
731 block_mig_state.transferred);
732
733 qemu_mutex_lock_iothread();
734 ret = init_blk_migration(f);
735 if (ret < 0) {
736 qemu_mutex_unlock_iothread();
737 return ret;
738 }
739
740 /* start track dirty blocks */
741 ret = set_dirty_tracking();
742
743 qemu_mutex_unlock_iothread();
744
745 if (ret) {
746 return ret;
747 }
748
749 ret = flush_blks(f);
750 blk_mig_reset_dirty_cursor();
751 qemu_put_be64(f, BLK_MIG_FLAG_EOS);
752
753 return ret;
754 }
755
756 static int block_save_iterate(QEMUFile *f, void *opaque)
757 {
758 int ret;
759 int64_t last_ftell = qemu_ftell(f);
760 int64_t delta_ftell;
761
762 trace_migration_block_save("iterate", block_mig_state.submitted,
763 block_mig_state.transferred);
764
765 ret = flush_blks(f);
766 if (ret) {
767 return ret;
768 }
769
770 blk_mig_reset_dirty_cursor();
771
772 /* control the rate of transfer */
773 blk_mig_lock();
774 while (block_mig_state.read_done * BLK_MIG_BLOCK_SIZE <
775 qemu_file_get_rate_limit(f) &&
776 block_mig_state.submitted < MAX_PARALLEL_IO &&
777 (block_mig_state.submitted + block_mig_state.read_done) <
778 MAX_IO_BUFFERS) {
779 blk_mig_unlock();
780 if (block_mig_state.bulk_completed == 0) {
781 /* first finish the bulk phase */
782 if (blk_mig_save_bulked_block(f) == 0) {
783 /* finished saving bulk on all devices */
784 block_mig_state.bulk_completed = 1;
785 }
786 ret = 0;
787 } else {
788 /* Always called with iothread lock taken for
789 * simplicity, block_save_complete also calls it.
790 */
791 qemu_mutex_lock_iothread();
792 ret = blk_mig_save_dirty_block(f, 1);
793 qemu_mutex_unlock_iothread();
794 }
795 if (ret < 0) {
796 return ret;
797 }
798 blk_mig_lock();
799 if (ret != 0) {
800 /* no more dirty blocks */
801 break;
802 }
803 }
804 blk_mig_unlock();
805
806 ret = flush_blks(f);
807 if (ret) {
808 return ret;
809 }
810
811 qemu_put_be64(f, BLK_MIG_FLAG_EOS);
812 delta_ftell = qemu_ftell(f) - last_ftell;
813 if (delta_ftell > 0) {
814 return 1;
815 } else if (delta_ftell < 0) {
816 return -1;
817 } else {
818 return 0;
819 }
820 }
821
822 /* Called with iothread lock taken. */
823
824 static int block_save_complete(QEMUFile *f, void *opaque)
825 {
826 int ret;
827
828 trace_migration_block_save("complete", block_mig_state.submitted,
829 block_mig_state.transferred);
830
831 ret = flush_blks(f);
832 if (ret) {
833 return ret;
834 }
835
836 blk_mig_reset_dirty_cursor();
837
838 /* we know for sure that save bulk is completed and
839 all async read completed */
840 blk_mig_lock();
841 assert(block_mig_state.submitted == 0);
842 blk_mig_unlock();
843
844 do {
845 ret = blk_mig_save_dirty_block(f, 0);
846 if (ret < 0) {
847 return ret;
848 }
849 } while (ret == 0);
850
851 /* report completion */
852 qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
853
854 trace_migration_block_save_complete();
855
856 qemu_put_be64(f, BLK_MIG_FLAG_EOS);
857
858 /* Make sure that our BlockBackends are gone, so that the block driver
859 * nodes can be inactivated. */
860 block_migration_cleanup_bmds();
861
862 return 0;
863 }
864
865 static void block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
866 uint64_t *res_precopy_only,
867 uint64_t *res_compatible,
868 uint64_t *res_postcopy_only)
869 {
870 /* Estimate pending number of bytes to send */
871 uint64_t pending;
872
873 qemu_mutex_lock_iothread();
874 pending = get_remaining_dirty();
875 qemu_mutex_unlock_iothread();
876
877 blk_mig_lock();
878 pending += block_mig_state.submitted * BLK_MIG_BLOCK_SIZE +
879 block_mig_state.read_done * BLK_MIG_BLOCK_SIZE;
880 blk_mig_unlock();
881
882 /* Report at least one block pending during bulk phase */
883 if (pending <= max_size && !block_mig_state.bulk_completed) {
884 pending = max_size + BLK_MIG_BLOCK_SIZE;
885 }
886
887 trace_migration_block_save_pending(pending);
888 /* We don't do postcopy */
889 *res_precopy_only += pending;
890 }
891
892 static int block_load(QEMUFile *f, void *opaque, int version_id)
893 {
894 static int banner_printed;
895 int len, flags;
896 char device_name[256];
897 int64_t addr;
898 BlockBackend *blk, *blk_prev = NULL;
899 Error *local_err = NULL;
900 uint8_t *buf;
901 int64_t total_sectors = 0;
902 int nr_sectors;
903 int ret;
904 BlockDriverInfo bdi;
905 int cluster_size = BLK_MIG_BLOCK_SIZE;
906
907 do {
908 addr = qemu_get_be64(f);
909
910 flags = addr & (BDRV_SECTOR_SIZE - 1);
911 addr >>= BDRV_SECTOR_BITS;
912
913 if (flags & BLK_MIG_FLAG_DEVICE_BLOCK) {
914 /* get device name */
915 len = qemu_get_byte(f);
916 qemu_get_buffer(f, (uint8_t *)device_name, len);
917 device_name[len] = '\0';
918
919 blk = blk_by_name(device_name);
920 if (!blk) {
921 fprintf(stderr, "Error unknown block device %s\n",
922 device_name);
923 return -EINVAL;
924 }
925
926 if (blk != blk_prev) {
927 blk_prev = blk;
928 total_sectors = blk_nb_sectors(blk);
929 if (total_sectors <= 0) {
930 error_report("Error getting length of block device %s",
931 device_name);
932 return -EINVAL;
933 }
934
935 blk_invalidate_cache(blk, &local_err);
936 if (local_err) {
937 error_report_err(local_err);
938 return -EINVAL;
939 }
940
941 ret = bdrv_get_info(blk_bs(blk), &bdi);
942 if (ret == 0 && bdi.cluster_size > 0 &&
943 bdi.cluster_size <= BLK_MIG_BLOCK_SIZE &&
944 BLK_MIG_BLOCK_SIZE % bdi.cluster_size == 0) {
945 cluster_size = bdi.cluster_size;
946 } else {
947 cluster_size = BLK_MIG_BLOCK_SIZE;
948 }
949 }
950
951 if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) {
952 nr_sectors = total_sectors - addr;
953 } else {
954 nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
955 }
956
957 if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
958 ret = blk_pwrite_zeroes(blk, addr * BDRV_SECTOR_SIZE,
959 nr_sectors * BDRV_SECTOR_SIZE,
960 BDRV_REQ_MAY_UNMAP);
961 } else {
962 int i;
963 int64_t cur_addr;
964 uint8_t *cur_buf;
965
966 buf = g_malloc(BLK_MIG_BLOCK_SIZE);
967 qemu_get_buffer(f, buf, BLK_MIG_BLOCK_SIZE);
968 for (i = 0; i < BLK_MIG_BLOCK_SIZE / cluster_size; i++) {
969 cur_addr = addr * BDRV_SECTOR_SIZE + i * cluster_size;
970 cur_buf = buf + i * cluster_size;
971
972 if ((!block_mig_state.zero_blocks ||
973 cluster_size < BLK_MIG_BLOCK_SIZE) &&
974 buffer_is_zero(cur_buf, cluster_size)) {
975 ret = blk_pwrite_zeroes(blk, cur_addr,
976 cluster_size,
977 BDRV_REQ_MAY_UNMAP);
978 } else {
979 ret = blk_pwrite(blk, cur_addr, cur_buf,
980 cluster_size, 0);
981 }
982 if (ret < 0) {
983 break;
984 }
985 }
986 g_free(buf);
987 }
988
989 if (ret < 0) {
990 return ret;
991 }
992 } else if (flags & BLK_MIG_FLAG_PROGRESS) {
993 if (!banner_printed) {
994 printf("Receiving block device images\n");
995 banner_printed = 1;
996 }
997 printf("Completed %d %%%c", (int)addr,
998 (addr == 100) ? '\n' : '\r');
999 fflush(stdout);
1000 } else if (!(flags & BLK_MIG_FLAG_EOS)) {
1001 fprintf(stderr, "Unknown block migration flags: 0x%x\n", flags);
1002 return -EINVAL;
1003 }
1004 ret = qemu_file_get_error(f);
1005 if (ret != 0) {
1006 return ret;
1007 }
1008 } while (!(flags & BLK_MIG_FLAG_EOS));
1009
1010 return 0;
1011 }
1012
1013 static bool block_is_active(void *opaque)
1014 {
1015 return migrate_use_block();
1016 }
1017
1018 static SaveVMHandlers savevm_block_handlers = {
1019 .save_setup = block_save_setup,
1020 .save_live_iterate = block_save_iterate,
1021 .save_live_complete_precopy = block_save_complete,
1022 .save_live_pending = block_save_pending,
1023 .load_state = block_load,
1024 .save_cleanup = block_migration_cleanup,
1025 .is_active = block_is_active,
1026 };
1027
1028 void blk_mig_init(void)
1029 {
1030 QSIMPLEQ_INIT(&block_mig_state.bmds_list);
1031 QSIMPLEQ_INIT(&block_mig_state.blk_list);
1032 qemu_mutex_init(&block_mig_state.lock);
1033
1034 register_savevm_live("block", 0, 1, &savevm_block_handlers,
1035 &block_mig_state);
1036 }