block/io: refactor coroutine wrappers
[qemu.git] / block / io.c
1 /*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/aio-wait.h"
29 #include "block/blockjob.h"
30 #include "block/blockjob_int.h"
31 #include "block/block_int.h"
32 #include "qemu/cutils.h"
33 #include "qapi/error.h"
34 #include "qemu/error-report.h"
35 #include "qemu/main-loop.h"
36 #include "sysemu/replay.h"
37
38 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
39 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
40
41 static void bdrv_parent_cb_resize(BlockDriverState *bs);
42 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
43 int64_t offset, int bytes, BdrvRequestFlags flags);
44
45 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
46 bool ignore_bds_parents)
47 {
48 BdrvChild *c, *next;
49
50 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
51 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
52 continue;
53 }
54 bdrv_parent_drained_begin_single(c, false);
55 }
56 }
57
58 static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c,
59 int *drained_end_counter)
60 {
61 assert(c->parent_quiesce_counter > 0);
62 c->parent_quiesce_counter--;
63 if (c->klass->drained_end) {
64 c->klass->drained_end(c, drained_end_counter);
65 }
66 }
67
68 void bdrv_parent_drained_end_single(BdrvChild *c)
69 {
70 int drained_end_counter = 0;
71 bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter);
72 BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0);
73 }
74
75 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
76 bool ignore_bds_parents,
77 int *drained_end_counter)
78 {
79 BdrvChild *c;
80
81 QLIST_FOREACH(c, &bs->parents, next_parent) {
82 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
83 continue;
84 }
85 bdrv_parent_drained_end_single_no_poll(c, drained_end_counter);
86 }
87 }
88
89 static bool bdrv_parent_drained_poll_single(BdrvChild *c)
90 {
91 if (c->klass->drained_poll) {
92 return c->klass->drained_poll(c);
93 }
94 return false;
95 }
96
97 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
98 bool ignore_bds_parents)
99 {
100 BdrvChild *c, *next;
101 bool busy = false;
102
103 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
104 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
105 continue;
106 }
107 busy |= bdrv_parent_drained_poll_single(c);
108 }
109
110 return busy;
111 }
112
113 void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll)
114 {
115 c->parent_quiesce_counter++;
116 if (c->klass->drained_begin) {
117 c->klass->drained_begin(c);
118 }
119 if (poll) {
120 BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c));
121 }
122 }
123
124 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
125 {
126 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
127 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
128 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
129 src->opt_mem_alignment);
130 dst->min_mem_alignment = MAX(dst->min_mem_alignment,
131 src->min_mem_alignment);
132 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
133 }
134
135 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
136 {
137 BlockDriver *drv = bs->drv;
138 BdrvChild *c;
139 bool have_limits;
140 Error *local_err = NULL;
141
142 memset(&bs->bl, 0, sizeof(bs->bl));
143
144 if (!drv) {
145 return;
146 }
147
148 /* Default alignment based on whether driver has byte interface */
149 bs->bl.request_alignment = (drv->bdrv_co_preadv ||
150 drv->bdrv_aio_preadv ||
151 drv->bdrv_co_preadv_part) ? 1 : 512;
152
153 /* Take some limits from the children as a default */
154 have_limits = false;
155 QLIST_FOREACH(c, &bs->children, next) {
156 if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW))
157 {
158 bdrv_refresh_limits(c->bs, &local_err);
159 if (local_err) {
160 error_propagate(errp, local_err);
161 return;
162 }
163 bdrv_merge_limits(&bs->bl, &c->bs->bl);
164 have_limits = true;
165 }
166 }
167
168 if (!have_limits) {
169 bs->bl.min_mem_alignment = 512;
170 bs->bl.opt_mem_alignment = qemu_real_host_page_size;
171
172 /* Safe default since most protocols use readv()/writev()/etc */
173 bs->bl.max_iov = IOV_MAX;
174 }
175
176 /* Then let the driver override it */
177 if (drv->bdrv_refresh_limits) {
178 drv->bdrv_refresh_limits(bs, errp);
179 }
180 }
181
182 /**
183 * The copy-on-read flag is actually a reference count so multiple users may
184 * use the feature without worrying about clobbering its previous state.
185 * Copy-on-read stays enabled until all users have called to disable it.
186 */
187 void bdrv_enable_copy_on_read(BlockDriverState *bs)
188 {
189 qatomic_inc(&bs->copy_on_read);
190 }
191
192 void bdrv_disable_copy_on_read(BlockDriverState *bs)
193 {
194 int old = qatomic_fetch_dec(&bs->copy_on_read);
195 assert(old >= 1);
196 }
197
198 typedef struct {
199 Coroutine *co;
200 BlockDriverState *bs;
201 bool done;
202 bool begin;
203 bool recursive;
204 bool poll;
205 BdrvChild *parent;
206 bool ignore_bds_parents;
207 int *drained_end_counter;
208 } BdrvCoDrainData;
209
210 static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
211 {
212 BdrvCoDrainData *data = opaque;
213 BlockDriverState *bs = data->bs;
214
215 if (data->begin) {
216 bs->drv->bdrv_co_drain_begin(bs);
217 } else {
218 bs->drv->bdrv_co_drain_end(bs);
219 }
220
221 /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */
222 qatomic_mb_set(&data->done, true);
223 if (!data->begin) {
224 qatomic_dec(data->drained_end_counter);
225 }
226 bdrv_dec_in_flight(bs);
227
228 g_free(data);
229 }
230
231 /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
232 static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
233 int *drained_end_counter)
234 {
235 BdrvCoDrainData *data;
236
237 if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) ||
238 (!begin && !bs->drv->bdrv_co_drain_end)) {
239 return;
240 }
241
242 data = g_new(BdrvCoDrainData, 1);
243 *data = (BdrvCoDrainData) {
244 .bs = bs,
245 .done = false,
246 .begin = begin,
247 .drained_end_counter = drained_end_counter,
248 };
249
250 if (!begin) {
251 qatomic_inc(drained_end_counter);
252 }
253
254 /* Make sure the driver callback completes during the polling phase for
255 * drain_begin. */
256 bdrv_inc_in_flight(bs);
257 data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data);
258 aio_co_schedule(bdrv_get_aio_context(bs), data->co);
259 }
260
261 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
262 bool bdrv_drain_poll(BlockDriverState *bs, bool recursive,
263 BdrvChild *ignore_parent, bool ignore_bds_parents)
264 {
265 BdrvChild *child, *next;
266
267 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
268 return true;
269 }
270
271 if (qatomic_read(&bs->in_flight)) {
272 return true;
273 }
274
275 if (recursive) {
276 assert(!ignore_bds_parents);
277 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
278 if (bdrv_drain_poll(child->bs, recursive, child, false)) {
279 return true;
280 }
281 }
282 }
283
284 return false;
285 }
286
287 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive,
288 BdrvChild *ignore_parent)
289 {
290 return bdrv_drain_poll(bs, recursive, ignore_parent, false);
291 }
292
293 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
294 BdrvChild *parent, bool ignore_bds_parents,
295 bool poll);
296 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
297 BdrvChild *parent, bool ignore_bds_parents,
298 int *drained_end_counter);
299
300 static void bdrv_co_drain_bh_cb(void *opaque)
301 {
302 BdrvCoDrainData *data = opaque;
303 Coroutine *co = data->co;
304 BlockDriverState *bs = data->bs;
305
306 if (bs) {
307 AioContext *ctx = bdrv_get_aio_context(bs);
308 AioContext *co_ctx = qemu_coroutine_get_aio_context(co);
309
310 /*
311 * When the coroutine yielded, the lock for its home context was
312 * released, so we need to re-acquire it here. If it explicitly
313 * acquired a different context, the lock is still held and we don't
314 * want to lock it a second time (or AIO_WAIT_WHILE() would hang).
315 */
316 if (ctx == co_ctx) {
317 aio_context_acquire(ctx);
318 }
319 bdrv_dec_in_flight(bs);
320 if (data->begin) {
321 assert(!data->drained_end_counter);
322 bdrv_do_drained_begin(bs, data->recursive, data->parent,
323 data->ignore_bds_parents, data->poll);
324 } else {
325 assert(!data->poll);
326 bdrv_do_drained_end(bs, data->recursive, data->parent,
327 data->ignore_bds_parents,
328 data->drained_end_counter);
329 }
330 if (ctx == co_ctx) {
331 aio_context_release(ctx);
332 }
333 } else {
334 assert(data->begin);
335 bdrv_drain_all_begin();
336 }
337
338 data->done = true;
339 aio_co_wake(co);
340 }
341
342 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
343 bool begin, bool recursive,
344 BdrvChild *parent,
345 bool ignore_bds_parents,
346 bool poll,
347 int *drained_end_counter)
348 {
349 BdrvCoDrainData data;
350
351 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
352 * other coroutines run if they were queued by aio_co_enter(). */
353
354 assert(qemu_in_coroutine());
355 data = (BdrvCoDrainData) {
356 .co = qemu_coroutine_self(),
357 .bs = bs,
358 .done = false,
359 .begin = begin,
360 .recursive = recursive,
361 .parent = parent,
362 .ignore_bds_parents = ignore_bds_parents,
363 .poll = poll,
364 .drained_end_counter = drained_end_counter,
365 };
366
367 if (bs) {
368 bdrv_inc_in_flight(bs);
369 }
370 replay_bh_schedule_oneshot_event(bdrv_get_aio_context(bs),
371 bdrv_co_drain_bh_cb, &data);
372
373 qemu_coroutine_yield();
374 /* If we are resumed from some other event (such as an aio completion or a
375 * timer callback), it is a bug in the caller that should be fixed. */
376 assert(data.done);
377 }
378
379 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs,
380 BdrvChild *parent, bool ignore_bds_parents)
381 {
382 assert(!qemu_in_coroutine());
383
384 /* Stop things in parent-to-child order */
385 if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) {
386 aio_disable_external(bdrv_get_aio_context(bs));
387 }
388
389 bdrv_parent_drained_begin(bs, parent, ignore_bds_parents);
390 bdrv_drain_invoke(bs, true, NULL);
391 }
392
393 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
394 BdrvChild *parent, bool ignore_bds_parents,
395 bool poll)
396 {
397 BdrvChild *child, *next;
398
399 if (qemu_in_coroutine()) {
400 bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents,
401 poll, NULL);
402 return;
403 }
404
405 bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents);
406
407 if (recursive) {
408 assert(!ignore_bds_parents);
409 bs->recursive_quiesce_counter++;
410 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
411 bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents,
412 false);
413 }
414 }
415
416 /*
417 * Wait for drained requests to finish.
418 *
419 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
420 * call is needed so things in this AioContext can make progress even
421 * though we don't return to the main AioContext loop - this automatically
422 * includes other nodes in the same AioContext and therefore all child
423 * nodes.
424 */
425 if (poll) {
426 assert(!ignore_bds_parents);
427 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent));
428 }
429 }
430
431 void bdrv_drained_begin(BlockDriverState *bs)
432 {
433 bdrv_do_drained_begin(bs, false, NULL, false, true);
434 }
435
436 void bdrv_subtree_drained_begin(BlockDriverState *bs)
437 {
438 bdrv_do_drained_begin(bs, true, NULL, false, true);
439 }
440
441 /**
442 * This function does not poll, nor must any of its recursively called
443 * functions. The *drained_end_counter pointee will be incremented
444 * once for every background operation scheduled, and decremented once
445 * the operation settles. Therefore, the pointer must remain valid
446 * until the pointee reaches 0. That implies that whoever sets up the
447 * pointee has to poll until it is 0.
448 *
449 * We use atomic operations to access *drained_end_counter, because
450 * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of
451 * @bs may contain nodes in different AioContexts,
452 * (2) bdrv_drain_all_end() uses the same counter for all nodes,
453 * regardless of which AioContext they are in.
454 */
455 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
456 BdrvChild *parent, bool ignore_bds_parents,
457 int *drained_end_counter)
458 {
459 BdrvChild *child;
460 int old_quiesce_counter;
461
462 assert(drained_end_counter != NULL);
463
464 if (qemu_in_coroutine()) {
465 bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
466 false, drained_end_counter);
467 return;
468 }
469 assert(bs->quiesce_counter > 0);
470
471 /* Re-enable things in child-to-parent order */
472 bdrv_drain_invoke(bs, false, drained_end_counter);
473 bdrv_parent_drained_end(bs, parent, ignore_bds_parents,
474 drained_end_counter);
475
476 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
477 if (old_quiesce_counter == 1) {
478 aio_enable_external(bdrv_get_aio_context(bs));
479 }
480
481 if (recursive) {
482 assert(!ignore_bds_parents);
483 bs->recursive_quiesce_counter--;
484 QLIST_FOREACH(child, &bs->children, next) {
485 bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents,
486 drained_end_counter);
487 }
488 }
489 }
490
491 void bdrv_drained_end(BlockDriverState *bs)
492 {
493 int drained_end_counter = 0;
494 bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter);
495 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
496 }
497
498 void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter)
499 {
500 bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter);
501 }
502
503 void bdrv_subtree_drained_end(BlockDriverState *bs)
504 {
505 int drained_end_counter = 0;
506 bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter);
507 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
508 }
509
510 void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
511 {
512 int i;
513
514 for (i = 0; i < new_parent->recursive_quiesce_counter; i++) {
515 bdrv_do_drained_begin(child->bs, true, child, false, true);
516 }
517 }
518
519 void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
520 {
521 int drained_end_counter = 0;
522 int i;
523
524 for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
525 bdrv_do_drained_end(child->bs, true, child, false,
526 &drained_end_counter);
527 }
528
529 BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0);
530 }
531
532 /*
533 * Wait for pending requests to complete on a single BlockDriverState subtree,
534 * and suspend block driver's internal I/O until next request arrives.
535 *
536 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
537 * AioContext.
538 */
539 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
540 {
541 assert(qemu_in_coroutine());
542 bdrv_drained_begin(bs);
543 bdrv_drained_end(bs);
544 }
545
546 void bdrv_drain(BlockDriverState *bs)
547 {
548 bdrv_drained_begin(bs);
549 bdrv_drained_end(bs);
550 }
551
552 static void bdrv_drain_assert_idle(BlockDriverState *bs)
553 {
554 BdrvChild *child, *next;
555
556 assert(qatomic_read(&bs->in_flight) == 0);
557 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
558 bdrv_drain_assert_idle(child->bs);
559 }
560 }
561
562 unsigned int bdrv_drain_all_count = 0;
563
564 static bool bdrv_drain_all_poll(void)
565 {
566 BlockDriverState *bs = NULL;
567 bool result = false;
568
569 /* bdrv_drain_poll() can't make changes to the graph and we are holding the
570 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
571 while ((bs = bdrv_next_all_states(bs))) {
572 AioContext *aio_context = bdrv_get_aio_context(bs);
573 aio_context_acquire(aio_context);
574 result |= bdrv_drain_poll(bs, false, NULL, true);
575 aio_context_release(aio_context);
576 }
577
578 return result;
579 }
580
581 /*
582 * Wait for pending requests to complete across all BlockDriverStates
583 *
584 * This function does not flush data to disk, use bdrv_flush_all() for that
585 * after calling this function.
586 *
587 * This pauses all block jobs and disables external clients. It must
588 * be paired with bdrv_drain_all_end().
589 *
590 * NOTE: no new block jobs or BlockDriverStates can be created between
591 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
592 */
593 void bdrv_drain_all_begin(void)
594 {
595 BlockDriverState *bs = NULL;
596
597 if (qemu_in_coroutine()) {
598 bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL);
599 return;
600 }
601
602 /*
603 * bdrv queue is managed by record/replay,
604 * waiting for finishing the I/O requests may
605 * be infinite
606 */
607 if (replay_events_enabled()) {
608 return;
609 }
610
611 /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
612 * loop AioContext, so make sure we're in the main context. */
613 assert(qemu_get_current_aio_context() == qemu_get_aio_context());
614 assert(bdrv_drain_all_count < INT_MAX);
615 bdrv_drain_all_count++;
616
617 /* Quiesce all nodes, without polling in-flight requests yet. The graph
618 * cannot change during this loop. */
619 while ((bs = bdrv_next_all_states(bs))) {
620 AioContext *aio_context = bdrv_get_aio_context(bs);
621
622 aio_context_acquire(aio_context);
623 bdrv_do_drained_begin(bs, false, NULL, true, false);
624 aio_context_release(aio_context);
625 }
626
627 /* Now poll the in-flight requests */
628 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
629
630 while ((bs = bdrv_next_all_states(bs))) {
631 bdrv_drain_assert_idle(bs);
632 }
633 }
634
635 void bdrv_drain_all_end(void)
636 {
637 BlockDriverState *bs = NULL;
638 int drained_end_counter = 0;
639
640 /*
641 * bdrv queue is managed by record/replay,
642 * waiting for finishing the I/O requests may
643 * be endless
644 */
645 if (replay_events_enabled()) {
646 return;
647 }
648
649 while ((bs = bdrv_next_all_states(bs))) {
650 AioContext *aio_context = bdrv_get_aio_context(bs);
651
652 aio_context_acquire(aio_context);
653 bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
654 aio_context_release(aio_context);
655 }
656
657 assert(qemu_get_current_aio_context() == qemu_get_aio_context());
658 AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0);
659
660 assert(bdrv_drain_all_count > 0);
661 bdrv_drain_all_count--;
662 }
663
664 void bdrv_drain_all(void)
665 {
666 bdrv_drain_all_begin();
667 bdrv_drain_all_end();
668 }
669
670 /**
671 * Remove an active request from the tracked requests list
672 *
673 * This function should be called when a tracked request is completing.
674 */
675 static void tracked_request_end(BdrvTrackedRequest *req)
676 {
677 if (req->serialising) {
678 qatomic_dec(&req->bs->serialising_in_flight);
679 }
680
681 qemu_co_mutex_lock(&req->bs->reqs_lock);
682 QLIST_REMOVE(req, list);
683 qemu_co_queue_restart_all(&req->wait_queue);
684 qemu_co_mutex_unlock(&req->bs->reqs_lock);
685 }
686
687 /**
688 * Add an active request to the tracked requests list
689 */
690 static void tracked_request_begin(BdrvTrackedRequest *req,
691 BlockDriverState *bs,
692 int64_t offset,
693 uint64_t bytes,
694 enum BdrvTrackedRequestType type)
695 {
696 assert(bytes <= INT64_MAX && offset <= INT64_MAX - bytes);
697
698 *req = (BdrvTrackedRequest){
699 .bs = bs,
700 .offset = offset,
701 .bytes = bytes,
702 .type = type,
703 .co = qemu_coroutine_self(),
704 .serialising = false,
705 .overlap_offset = offset,
706 .overlap_bytes = bytes,
707 };
708
709 qemu_co_queue_init(&req->wait_queue);
710
711 qemu_co_mutex_lock(&bs->reqs_lock);
712 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
713 qemu_co_mutex_unlock(&bs->reqs_lock);
714 }
715
716 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
717 int64_t offset, uint64_t bytes)
718 {
719 /* aaaa bbbb */
720 if (offset >= req->overlap_offset + req->overlap_bytes) {
721 return false;
722 }
723 /* bbbb aaaa */
724 if (req->overlap_offset >= offset + bytes) {
725 return false;
726 }
727 return true;
728 }
729
730 static bool coroutine_fn
731 bdrv_wait_serialising_requests_locked(BlockDriverState *bs,
732 BdrvTrackedRequest *self)
733 {
734 BdrvTrackedRequest *req;
735 bool retry;
736 bool waited = false;
737
738 do {
739 retry = false;
740 QLIST_FOREACH(req, &bs->tracked_requests, list) {
741 if (req == self || (!req->serialising && !self->serialising)) {
742 continue;
743 }
744 if (tracked_request_overlaps(req, self->overlap_offset,
745 self->overlap_bytes))
746 {
747 /* Hitting this means there was a reentrant request, for
748 * example, a block driver issuing nested requests. This must
749 * never happen since it means deadlock.
750 */
751 assert(qemu_coroutine_self() != req->co);
752
753 /* If the request is already (indirectly) waiting for us, or
754 * will wait for us as soon as it wakes up, then just go on
755 * (instead of producing a deadlock in the former case). */
756 if (!req->waiting_for) {
757 self->waiting_for = req;
758 qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock);
759 self->waiting_for = NULL;
760 retry = true;
761 waited = true;
762 break;
763 }
764 }
765 }
766 } while (retry);
767 return waited;
768 }
769
770 bool bdrv_mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
771 {
772 BlockDriverState *bs = req->bs;
773 int64_t overlap_offset = req->offset & ~(align - 1);
774 uint64_t overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
775 - overlap_offset;
776 bool waited;
777
778 qemu_co_mutex_lock(&bs->reqs_lock);
779 if (!req->serialising) {
780 qatomic_inc(&req->bs->serialising_in_flight);
781 req->serialising = true;
782 }
783
784 req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
785 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
786 waited = bdrv_wait_serialising_requests_locked(bs, req);
787 qemu_co_mutex_unlock(&bs->reqs_lock);
788 return waited;
789 }
790
791 /**
792 * Return the tracked request on @bs for the current coroutine, or
793 * NULL if there is none.
794 */
795 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
796 {
797 BdrvTrackedRequest *req;
798 Coroutine *self = qemu_coroutine_self();
799
800 QLIST_FOREACH(req, &bs->tracked_requests, list) {
801 if (req->co == self) {
802 return req;
803 }
804 }
805
806 return NULL;
807 }
808
809 /**
810 * Round a region to cluster boundaries
811 */
812 void bdrv_round_to_clusters(BlockDriverState *bs,
813 int64_t offset, int64_t bytes,
814 int64_t *cluster_offset,
815 int64_t *cluster_bytes)
816 {
817 BlockDriverInfo bdi;
818
819 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
820 *cluster_offset = offset;
821 *cluster_bytes = bytes;
822 } else {
823 int64_t c = bdi.cluster_size;
824 *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
825 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
826 }
827 }
828
829 static int bdrv_get_cluster_size(BlockDriverState *bs)
830 {
831 BlockDriverInfo bdi;
832 int ret;
833
834 ret = bdrv_get_info(bs, &bdi);
835 if (ret < 0 || bdi.cluster_size == 0) {
836 return bs->bl.request_alignment;
837 } else {
838 return bdi.cluster_size;
839 }
840 }
841
842 void bdrv_inc_in_flight(BlockDriverState *bs)
843 {
844 qatomic_inc(&bs->in_flight);
845 }
846
847 void bdrv_wakeup(BlockDriverState *bs)
848 {
849 aio_wait_kick();
850 }
851
852 void bdrv_dec_in_flight(BlockDriverState *bs)
853 {
854 qatomic_dec(&bs->in_flight);
855 bdrv_wakeup(bs);
856 }
857
858 static bool coroutine_fn bdrv_wait_serialising_requests(BdrvTrackedRequest *self)
859 {
860 BlockDriverState *bs = self->bs;
861 bool waited = false;
862
863 if (!qatomic_read(&bs->serialising_in_flight)) {
864 return false;
865 }
866
867 qemu_co_mutex_lock(&bs->reqs_lock);
868 waited = bdrv_wait_serialising_requests_locked(bs, self);
869 qemu_co_mutex_unlock(&bs->reqs_lock);
870
871 return waited;
872 }
873
874 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
875 size_t size)
876 {
877 if (size > BDRV_REQUEST_MAX_BYTES) {
878 return -EIO;
879 }
880
881 if (!bdrv_is_inserted(bs)) {
882 return -ENOMEDIUM;
883 }
884
885 if (offset < 0) {
886 return -EIO;
887 }
888
889 return 0;
890 }
891
892 typedef int coroutine_fn BdrvRequestEntry(void *opaque);
893 typedef struct BdrvRunCo {
894 BdrvRequestEntry *entry;
895 void *opaque;
896 int ret;
897 bool done;
898 Coroutine *co; /* Coroutine, running bdrv_run_co_entry, for debugging */
899 } BdrvRunCo;
900
901 static void coroutine_fn bdrv_run_co_entry(void *opaque)
902 {
903 BdrvRunCo *arg = opaque;
904
905 arg->ret = arg->entry(arg->opaque);
906 arg->done = true;
907 aio_wait_kick();
908 }
909
910 static int bdrv_run_co(BlockDriverState *bs, BdrvRequestEntry *entry,
911 void *opaque)
912 {
913 if (qemu_in_coroutine()) {
914 /* Fast-path if already in coroutine context */
915 return entry(opaque);
916 } else {
917 BdrvRunCo s = { .entry = entry, .opaque = opaque };
918
919 s.co = qemu_coroutine_create(bdrv_run_co_entry, &s);
920 bdrv_coroutine_enter(bs, s.co);
921
922 BDRV_POLL_WHILE(bs, !s.done);
923
924 return s.ret;
925 }
926 }
927
928 typedef struct RwCo {
929 BdrvChild *child;
930 int64_t offset;
931 QEMUIOVector *qiov;
932 bool is_write;
933 BdrvRequestFlags flags;
934 } RwCo;
935
936 static int coroutine_fn bdrv_co_prwv(BdrvChild *child, int64_t offset,
937 QEMUIOVector *qiov, bool is_write,
938 BdrvRequestFlags flags)
939 {
940 if (is_write) {
941 return bdrv_co_pwritev(child, offset, qiov->size, qiov, flags);
942 } else {
943 return bdrv_co_preadv(child, offset, qiov->size, qiov, flags);
944 }
945 }
946
947 static int coroutine_fn bdrv_rw_co_entry(void *opaque)
948 {
949 RwCo *rwco = opaque;
950
951 return bdrv_co_prwv(rwco->child, rwco->offset, rwco->qiov,
952 rwco->is_write, rwco->flags);
953 }
954
955 /*
956 * Process a vectored synchronous request using coroutines
957 */
958 static int bdrv_prwv(BdrvChild *child, int64_t offset,
959 QEMUIOVector *qiov, bool is_write,
960 BdrvRequestFlags flags)
961 {
962 RwCo rwco = {
963 .child = child,
964 .offset = offset,
965 .qiov = qiov,
966 .is_write = is_write,
967 .flags = flags,
968 };
969
970 return bdrv_run_co(child->bs, bdrv_rw_co_entry, &rwco);
971 }
972
973 int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
974 int bytes, BdrvRequestFlags flags)
975 {
976 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, NULL, bytes);
977
978 return bdrv_prwv(child, offset, &qiov, true, BDRV_REQ_ZERO_WRITE | flags);
979 }
980
981 /*
982 * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
983 * The operation is sped up by checking the block status and only writing
984 * zeroes to the device if they currently do not return zeroes. Optional
985 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
986 * BDRV_REQ_FUA).
987 *
988 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
989 */
990 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
991 {
992 int ret;
993 int64_t target_size, bytes, offset = 0;
994 BlockDriverState *bs = child->bs;
995
996 target_size = bdrv_getlength(bs);
997 if (target_size < 0) {
998 return target_size;
999 }
1000
1001 for (;;) {
1002 bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
1003 if (bytes <= 0) {
1004 return 0;
1005 }
1006 ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
1007 if (ret < 0) {
1008 return ret;
1009 }
1010 if (ret & BDRV_BLOCK_ZERO) {
1011 offset += bytes;
1012 continue;
1013 }
1014 ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
1015 if (ret < 0) {
1016 return ret;
1017 }
1018 offset += bytes;
1019 }
1020 }
1021
1022 /* return < 0 if error. See bdrv_pwrite() for the return codes */
1023 int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
1024 {
1025 int ret;
1026
1027 ret = bdrv_prwv(child, offset, qiov, false, 0);
1028 if (ret < 0) {
1029 return ret;
1030 }
1031
1032 return qiov->size;
1033 }
1034
1035 /* See bdrv_pwrite() for the return codes */
1036 int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes)
1037 {
1038 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
1039
1040 if (bytes < 0) {
1041 return -EINVAL;
1042 }
1043
1044 return bdrv_preadv(child, offset, &qiov);
1045 }
1046
1047 int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
1048 {
1049 int ret;
1050
1051 ret = bdrv_prwv(child, offset, qiov, true, 0);
1052 if (ret < 0) {
1053 return ret;
1054 }
1055
1056 return qiov->size;
1057 }
1058
1059 /* Return no. of bytes on success or < 0 on error. Important errors are:
1060 -EIO generic I/O error (may happen for all errors)
1061 -ENOMEDIUM No media inserted.
1062 -EINVAL Invalid offset or number of bytes
1063 -EACCES Trying to write a read-only device
1064 */
1065 int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes)
1066 {
1067 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
1068
1069 if (bytes < 0) {
1070 return -EINVAL;
1071 }
1072
1073 return bdrv_pwritev(child, offset, &qiov);
1074 }
1075
1076 /*
1077 * Writes to the file and ensures that no writes are reordered across this
1078 * request (acts as a barrier)
1079 *
1080 * Returns 0 on success, -errno in error cases.
1081 */
1082 int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
1083 const void *buf, int count)
1084 {
1085 int ret;
1086
1087 ret = bdrv_pwrite(child, offset, buf, count);
1088 if (ret < 0) {
1089 return ret;
1090 }
1091
1092 ret = bdrv_flush(child->bs);
1093 if (ret < 0) {
1094 return ret;
1095 }
1096
1097 return 0;
1098 }
1099
1100 typedef struct CoroutineIOCompletion {
1101 Coroutine *coroutine;
1102 int ret;
1103 } CoroutineIOCompletion;
1104
1105 static void bdrv_co_io_em_complete(void *opaque, int ret)
1106 {
1107 CoroutineIOCompletion *co = opaque;
1108
1109 co->ret = ret;
1110 aio_co_wake(co->coroutine);
1111 }
1112
1113 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
1114 uint64_t offset, uint64_t bytes,
1115 QEMUIOVector *qiov,
1116 size_t qiov_offset, int flags)
1117 {
1118 BlockDriver *drv = bs->drv;
1119 int64_t sector_num;
1120 unsigned int nb_sectors;
1121 QEMUIOVector local_qiov;
1122 int ret;
1123
1124 assert(!(flags & ~BDRV_REQ_MASK));
1125 assert(!(flags & BDRV_REQ_NO_FALLBACK));
1126
1127 if (!drv) {
1128 return -ENOMEDIUM;
1129 }
1130
1131 if (drv->bdrv_co_preadv_part) {
1132 return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
1133 flags);
1134 }
1135
1136 if (qiov_offset > 0 || bytes != qiov->size) {
1137 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1138 qiov = &local_qiov;
1139 }
1140
1141 if (drv->bdrv_co_preadv) {
1142 ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
1143 goto out;
1144 }
1145
1146 if (drv->bdrv_aio_preadv) {
1147 BlockAIOCB *acb;
1148 CoroutineIOCompletion co = {
1149 .coroutine = qemu_coroutine_self(),
1150 };
1151
1152 acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
1153 bdrv_co_io_em_complete, &co);
1154 if (acb == NULL) {
1155 ret = -EIO;
1156 goto out;
1157 } else {
1158 qemu_coroutine_yield();
1159 ret = co.ret;
1160 goto out;
1161 }
1162 }
1163
1164 sector_num = offset >> BDRV_SECTOR_BITS;
1165 nb_sectors = bytes >> BDRV_SECTOR_BITS;
1166
1167 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1168 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1169 assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1170 assert(drv->bdrv_co_readv);
1171
1172 ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
1173
1174 out:
1175 if (qiov == &local_qiov) {
1176 qemu_iovec_destroy(&local_qiov);
1177 }
1178
1179 return ret;
1180 }
1181
1182 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
1183 uint64_t offset, uint64_t bytes,
1184 QEMUIOVector *qiov,
1185 size_t qiov_offset, int flags)
1186 {
1187 BlockDriver *drv = bs->drv;
1188 int64_t sector_num;
1189 unsigned int nb_sectors;
1190 QEMUIOVector local_qiov;
1191 int ret;
1192
1193 assert(!(flags & ~BDRV_REQ_MASK));
1194 assert(!(flags & BDRV_REQ_NO_FALLBACK));
1195
1196 if (!drv) {
1197 return -ENOMEDIUM;
1198 }
1199
1200 if (drv->bdrv_co_pwritev_part) {
1201 ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
1202 flags & bs->supported_write_flags);
1203 flags &= ~bs->supported_write_flags;
1204 goto emulate_flags;
1205 }
1206
1207 if (qiov_offset > 0 || bytes != qiov->size) {
1208 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1209 qiov = &local_qiov;
1210 }
1211
1212 if (drv->bdrv_co_pwritev) {
1213 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
1214 flags & bs->supported_write_flags);
1215 flags &= ~bs->supported_write_flags;
1216 goto emulate_flags;
1217 }
1218
1219 if (drv->bdrv_aio_pwritev) {
1220 BlockAIOCB *acb;
1221 CoroutineIOCompletion co = {
1222 .coroutine = qemu_coroutine_self(),
1223 };
1224
1225 acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov,
1226 flags & bs->supported_write_flags,
1227 bdrv_co_io_em_complete, &co);
1228 flags &= ~bs->supported_write_flags;
1229 if (acb == NULL) {
1230 ret = -EIO;
1231 } else {
1232 qemu_coroutine_yield();
1233 ret = co.ret;
1234 }
1235 goto emulate_flags;
1236 }
1237
1238 sector_num = offset >> BDRV_SECTOR_BITS;
1239 nb_sectors = bytes >> BDRV_SECTOR_BITS;
1240
1241 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1242 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1243 assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1244
1245 assert(drv->bdrv_co_writev);
1246 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov,
1247 flags & bs->supported_write_flags);
1248 flags &= ~bs->supported_write_flags;
1249
1250 emulate_flags:
1251 if (ret == 0 && (flags & BDRV_REQ_FUA)) {
1252 ret = bdrv_co_flush(bs);
1253 }
1254
1255 if (qiov == &local_qiov) {
1256 qemu_iovec_destroy(&local_qiov);
1257 }
1258
1259 return ret;
1260 }
1261
1262 static int coroutine_fn
1263 bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset,
1264 uint64_t bytes, QEMUIOVector *qiov,
1265 size_t qiov_offset)
1266 {
1267 BlockDriver *drv = bs->drv;
1268 QEMUIOVector local_qiov;
1269 int ret;
1270
1271 if (!drv) {
1272 return -ENOMEDIUM;
1273 }
1274
1275 if (!block_driver_can_compress(drv)) {
1276 return -ENOTSUP;
1277 }
1278
1279 if (drv->bdrv_co_pwritev_compressed_part) {
1280 return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
1281 qiov, qiov_offset);
1282 }
1283
1284 if (qiov_offset == 0) {
1285 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
1286 }
1287
1288 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1289 ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
1290 qemu_iovec_destroy(&local_qiov);
1291
1292 return ret;
1293 }
1294
1295 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
1296 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1297 size_t qiov_offset, int flags)
1298 {
1299 BlockDriverState *bs = child->bs;
1300
1301 /* Perform I/O through a temporary buffer so that users who scribble over
1302 * their read buffer while the operation is in progress do not end up
1303 * modifying the image file. This is critical for zero-copy guest I/O
1304 * where anything might happen inside guest memory.
1305 */
1306 void *bounce_buffer = NULL;
1307
1308 BlockDriver *drv = bs->drv;
1309 int64_t cluster_offset;
1310 int64_t cluster_bytes;
1311 size_t skip_bytes;
1312 int ret;
1313 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1314 BDRV_REQUEST_MAX_BYTES);
1315 unsigned int progress = 0;
1316 bool skip_write;
1317
1318 if (!drv) {
1319 return -ENOMEDIUM;
1320 }
1321
1322 /*
1323 * Do not write anything when the BDS is inactive. That is not
1324 * allowed, and it would not help.
1325 */
1326 skip_write = (bs->open_flags & BDRV_O_INACTIVE);
1327
1328 /* FIXME We cannot require callers to have write permissions when all they
1329 * are doing is a read request. If we did things right, write permissions
1330 * would be obtained anyway, but internally by the copy-on-read code. As
1331 * long as it is implemented here rather than in a separate filter driver,
1332 * the copy-on-read code doesn't have its own BdrvChild, however, for which
1333 * it could request permissions. Therefore we have to bypass the permission
1334 * system for the moment. */
1335 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1336
1337 /* Cover entire cluster so no additional backing file I/O is required when
1338 * allocating cluster in the image file. Note that this value may exceed
1339 * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1340 * is one reason we loop rather than doing it all at once.
1341 */
1342 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
1343 skip_bytes = offset - cluster_offset;
1344
1345 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1346 cluster_offset, cluster_bytes);
1347
1348 while (cluster_bytes) {
1349 int64_t pnum;
1350
1351 if (skip_write) {
1352 ret = 1; /* "already allocated", so nothing will be copied */
1353 pnum = MIN(cluster_bytes, max_transfer);
1354 } else {
1355 ret = bdrv_is_allocated(bs, cluster_offset,
1356 MIN(cluster_bytes, max_transfer), &pnum);
1357 if (ret < 0) {
1358 /*
1359 * Safe to treat errors in querying allocation as if
1360 * unallocated; we'll probably fail again soon on the
1361 * read, but at least that will set a decent errno.
1362 */
1363 pnum = MIN(cluster_bytes, max_transfer);
1364 }
1365
1366 /* Stop at EOF if the image ends in the middle of the cluster */
1367 if (ret == 0 && pnum == 0) {
1368 assert(progress >= bytes);
1369 break;
1370 }
1371
1372 assert(skip_bytes < pnum);
1373 }
1374
1375 if (ret <= 0) {
1376 QEMUIOVector local_qiov;
1377
1378 /* Must copy-on-read; use the bounce buffer */
1379 pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
1380 if (!bounce_buffer) {
1381 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum);
1382 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
1383 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
1384
1385 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
1386 if (!bounce_buffer) {
1387 ret = -ENOMEM;
1388 goto err;
1389 }
1390 }
1391 qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);
1392
1393 ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
1394 &local_qiov, 0, 0);
1395 if (ret < 0) {
1396 goto err;
1397 }
1398
1399 bdrv_debug_event(bs, BLKDBG_COR_WRITE);
1400 if (drv->bdrv_co_pwrite_zeroes &&
1401 buffer_is_zero(bounce_buffer, pnum)) {
1402 /* FIXME: Should we (perhaps conditionally) be setting
1403 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1404 * that still correctly reads as zero? */
1405 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum,
1406 BDRV_REQ_WRITE_UNCHANGED);
1407 } else {
1408 /* This does not change the data on the disk, it is not
1409 * necessary to flush even in cache=writethrough mode.
1410 */
1411 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
1412 &local_qiov, 0,
1413 BDRV_REQ_WRITE_UNCHANGED);
1414 }
1415
1416 if (ret < 0) {
1417 /* It might be okay to ignore write errors for guest
1418 * requests. If this is a deliberate copy-on-read
1419 * then we don't want to ignore the error. Simply
1420 * report it in all cases.
1421 */
1422 goto err;
1423 }
1424
1425 if (!(flags & BDRV_REQ_PREFETCH)) {
1426 qemu_iovec_from_buf(qiov, qiov_offset + progress,
1427 bounce_buffer + skip_bytes,
1428 MIN(pnum - skip_bytes, bytes - progress));
1429 }
1430 } else if (!(flags & BDRV_REQ_PREFETCH)) {
1431 /* Read directly into the destination */
1432 ret = bdrv_driver_preadv(bs, offset + progress,
1433 MIN(pnum - skip_bytes, bytes - progress),
1434 qiov, qiov_offset + progress, 0);
1435 if (ret < 0) {
1436 goto err;
1437 }
1438 }
1439
1440 cluster_offset += pnum;
1441 cluster_bytes -= pnum;
1442 progress += pnum - skip_bytes;
1443 skip_bytes = 0;
1444 }
1445 ret = 0;
1446
1447 err:
1448 qemu_vfree(bounce_buffer);
1449 return ret;
1450 }
1451
1452 /*
1453 * Forwards an already correctly aligned request to the BlockDriver. This
1454 * handles copy on read, zeroing after EOF, and fragmentation of large
1455 * reads; any other features must be implemented by the caller.
1456 */
1457 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
1458 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1459 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
1460 {
1461 BlockDriverState *bs = child->bs;
1462 int64_t total_bytes, max_bytes;
1463 int ret = 0;
1464 uint64_t bytes_remaining = bytes;
1465 int max_transfer;
1466
1467 assert(is_power_of_2(align));
1468 assert((offset & (align - 1)) == 0);
1469 assert((bytes & (align - 1)) == 0);
1470 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1471 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1472 align);
1473
1474 /* TODO: We would need a per-BDS .supported_read_flags and
1475 * potential fallback support, if we ever implement any read flags
1476 * to pass through to drivers. For now, there aren't any
1477 * passthrough flags. */
1478 assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH)));
1479
1480 /* Handle Copy on Read and associated serialisation */
1481 if (flags & BDRV_REQ_COPY_ON_READ) {
1482 /* If we touch the same cluster it counts as an overlap. This
1483 * guarantees that allocating writes will be serialized and not race
1484 * with each other for the same cluster. For example, in copy-on-read
1485 * it ensures that the CoR read and write operations are atomic and
1486 * guest writes cannot interleave between them. */
1487 bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs));
1488 } else {
1489 bdrv_wait_serialising_requests(req);
1490 }
1491
1492 if (flags & BDRV_REQ_COPY_ON_READ) {
1493 int64_t pnum;
1494
1495 ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
1496 if (ret < 0) {
1497 goto out;
1498 }
1499
1500 if (!ret || pnum != bytes) {
1501 ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
1502 qiov, qiov_offset, flags);
1503 goto out;
1504 } else if (flags & BDRV_REQ_PREFETCH) {
1505 goto out;
1506 }
1507 }
1508
1509 /* Forward the request to the BlockDriver, possibly fragmenting it */
1510 total_bytes = bdrv_getlength(bs);
1511 if (total_bytes < 0) {
1512 ret = total_bytes;
1513 goto out;
1514 }
1515
1516 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
1517 if (bytes <= max_bytes && bytes <= max_transfer) {
1518 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, 0);
1519 goto out;
1520 }
1521
1522 while (bytes_remaining) {
1523 int num;
1524
1525 if (max_bytes) {
1526 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
1527 assert(num);
1528
1529 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1530 num, qiov,
1531 qiov_offset + bytes - bytes_remaining, 0);
1532 max_bytes -= num;
1533 } else {
1534 num = bytes_remaining;
1535 ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining,
1536 0, bytes_remaining);
1537 }
1538 if (ret < 0) {
1539 goto out;
1540 }
1541 bytes_remaining -= num;
1542 }
1543
1544 out:
1545 return ret < 0 ? ret : 0;
1546 }
1547
1548 /*
1549 * Request padding
1550 *
1551 * |<---- align ----->| |<----- align ---->|
1552 * |<- head ->|<------------- bytes ------------->|<-- tail -->|
1553 * | | | | | |
1554 * -*----------$-------*-------- ... --------*-----$------------*---
1555 * | | | | | |
1556 * | offset | | end |
1557 * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end)
1558 * [buf ... ) [tail_buf )
1559 *
1560 * @buf is an aligned allocation needed to store @head and @tail paddings. @head
1561 * is placed at the beginning of @buf and @tail at the @end.
1562 *
1563 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
1564 * around tail, if tail exists.
1565 *
1566 * @merge_reads is true for small requests,
1567 * if @buf_len == @head + bytes + @tail. In this case it is possible that both
1568 * head and tail exist but @buf_len == align and @tail_buf == @buf.
1569 */
1570 typedef struct BdrvRequestPadding {
1571 uint8_t *buf;
1572 size_t buf_len;
1573 uint8_t *tail_buf;
1574 size_t head;
1575 size_t tail;
1576 bool merge_reads;
1577 QEMUIOVector local_qiov;
1578 } BdrvRequestPadding;
1579
1580 static bool bdrv_init_padding(BlockDriverState *bs,
1581 int64_t offset, int64_t bytes,
1582 BdrvRequestPadding *pad)
1583 {
1584 uint64_t align = bs->bl.request_alignment;
1585 size_t sum;
1586
1587 memset(pad, 0, sizeof(*pad));
1588
1589 pad->head = offset & (align - 1);
1590 pad->tail = ((offset + bytes) & (align - 1));
1591 if (pad->tail) {
1592 pad->tail = align - pad->tail;
1593 }
1594
1595 if (!pad->head && !pad->tail) {
1596 return false;
1597 }
1598
1599 assert(bytes); /* Nothing good in aligning zero-length requests */
1600
1601 sum = pad->head + bytes + pad->tail;
1602 pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
1603 pad->buf = qemu_blockalign(bs, pad->buf_len);
1604 pad->merge_reads = sum == pad->buf_len;
1605 if (pad->tail) {
1606 pad->tail_buf = pad->buf + pad->buf_len - align;
1607 }
1608
1609 return true;
1610 }
1611
1612 static int bdrv_padding_rmw_read(BdrvChild *child,
1613 BdrvTrackedRequest *req,
1614 BdrvRequestPadding *pad,
1615 bool zero_middle)
1616 {
1617 QEMUIOVector local_qiov;
1618 BlockDriverState *bs = child->bs;
1619 uint64_t align = bs->bl.request_alignment;
1620 int ret;
1621
1622 assert(req->serialising && pad->buf);
1623
1624 if (pad->head || pad->merge_reads) {
1625 uint64_t bytes = pad->merge_reads ? pad->buf_len : align;
1626
1627 qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
1628
1629 if (pad->head) {
1630 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1631 }
1632 if (pad->merge_reads && pad->tail) {
1633 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1634 }
1635 ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
1636 align, &local_qiov, 0, 0);
1637 if (ret < 0) {
1638 return ret;
1639 }
1640 if (pad->head) {
1641 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1642 }
1643 if (pad->merge_reads && pad->tail) {
1644 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1645 }
1646
1647 if (pad->merge_reads) {
1648 goto zero_mem;
1649 }
1650 }
1651
1652 if (pad->tail) {
1653 qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
1654
1655 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1656 ret = bdrv_aligned_preadv(
1657 child, req,
1658 req->overlap_offset + req->overlap_bytes - align,
1659 align, align, &local_qiov, 0, 0);
1660 if (ret < 0) {
1661 return ret;
1662 }
1663 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1664 }
1665
1666 zero_mem:
1667 if (zero_middle) {
1668 memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
1669 }
1670
1671 return 0;
1672 }
1673
1674 static void bdrv_padding_destroy(BdrvRequestPadding *pad)
1675 {
1676 if (pad->buf) {
1677 qemu_vfree(pad->buf);
1678 qemu_iovec_destroy(&pad->local_qiov);
1679 }
1680 }
1681
1682 /*
1683 * bdrv_pad_request
1684 *
1685 * Exchange request parameters with padded request if needed. Don't include RMW
1686 * read of padding, bdrv_padding_rmw_read() should be called separately if
1687 * needed.
1688 *
1689 * All parameters except @bs are in-out: they represent original request at
1690 * function call and padded (if padding needed) at function finish.
1691 *
1692 * Function always succeeds.
1693 */
1694 static bool bdrv_pad_request(BlockDriverState *bs,
1695 QEMUIOVector **qiov, size_t *qiov_offset,
1696 int64_t *offset, unsigned int *bytes,
1697 BdrvRequestPadding *pad)
1698 {
1699 if (!bdrv_init_padding(bs, *offset, *bytes, pad)) {
1700 return false;
1701 }
1702
1703 qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
1704 *qiov, *qiov_offset, *bytes,
1705 pad->buf + pad->buf_len - pad->tail, pad->tail);
1706 *bytes += pad->head + pad->tail;
1707 *offset -= pad->head;
1708 *qiov = &pad->local_qiov;
1709 *qiov_offset = 0;
1710
1711 return true;
1712 }
1713
1714 int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1715 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1716 BdrvRequestFlags flags)
1717 {
1718 return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
1719 }
1720
1721 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
1722 int64_t offset, unsigned int bytes,
1723 QEMUIOVector *qiov, size_t qiov_offset,
1724 BdrvRequestFlags flags)
1725 {
1726 BlockDriverState *bs = child->bs;
1727 BdrvTrackedRequest req;
1728 BdrvRequestPadding pad;
1729 int ret;
1730
1731 trace_bdrv_co_preadv(bs, offset, bytes, flags);
1732
1733 ret = bdrv_check_byte_request(bs, offset, bytes);
1734 if (ret < 0) {
1735 return ret;
1736 }
1737
1738 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
1739 /*
1740 * Aligning zero request is nonsense. Even if driver has special meaning
1741 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
1742 * it to driver due to request_alignment.
1743 *
1744 * Still, no reason to return an error if someone do unaligned
1745 * zero-length read occasionally.
1746 */
1747 return 0;
1748 }
1749
1750 bdrv_inc_in_flight(bs);
1751
1752 /* Don't do copy-on-read if we read data before write operation */
1753 if (qatomic_read(&bs->copy_on_read)) {
1754 flags |= BDRV_REQ_COPY_ON_READ;
1755 }
1756
1757 bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad);
1758
1759 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1760 ret = bdrv_aligned_preadv(child, &req, offset, bytes,
1761 bs->bl.request_alignment,
1762 qiov, qiov_offset, flags);
1763 tracked_request_end(&req);
1764 bdrv_dec_in_flight(bs);
1765
1766 bdrv_padding_destroy(&pad);
1767
1768 return ret;
1769 }
1770
1771 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1772 int64_t offset, int bytes, BdrvRequestFlags flags)
1773 {
1774 BlockDriver *drv = bs->drv;
1775 QEMUIOVector qiov;
1776 void *buf = NULL;
1777 int ret = 0;
1778 bool need_flush = false;
1779 int head = 0;
1780 int tail = 0;
1781
1782 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX);
1783 int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1784 bs->bl.request_alignment);
1785 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1786
1787 if (!drv) {
1788 return -ENOMEDIUM;
1789 }
1790
1791 if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {
1792 return -ENOTSUP;
1793 }
1794
1795 assert(alignment % bs->bl.request_alignment == 0);
1796 head = offset % alignment;
1797 tail = (offset + bytes) % alignment;
1798 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1799 assert(max_write_zeroes >= bs->bl.request_alignment);
1800
1801 while (bytes > 0 && !ret) {
1802 int num = bytes;
1803
1804 /* Align request. Block drivers can expect the "bulk" of the request
1805 * to be aligned, and that unaligned requests do not cross cluster
1806 * boundaries.
1807 */
1808 if (head) {
1809 /* Make a small request up to the first aligned sector. For
1810 * convenience, limit this request to max_transfer even if
1811 * we don't need to fall back to writes. */
1812 num = MIN(MIN(bytes, max_transfer), alignment - head);
1813 head = (head + num) % alignment;
1814 assert(num < max_write_zeroes);
1815 } else if (tail && num > alignment) {
1816 /* Shorten the request to the last aligned sector. */
1817 num -= tail;
1818 }
1819
1820 /* limit request size */
1821 if (num > max_write_zeroes) {
1822 num = max_write_zeroes;
1823 }
1824
1825 ret = -ENOTSUP;
1826 /* First try the efficient write zeroes operation */
1827 if (drv->bdrv_co_pwrite_zeroes) {
1828 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1829 flags & bs->supported_zero_flags);
1830 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1831 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1832 need_flush = true;
1833 }
1834 } else {
1835 assert(!bs->supported_zero_flags);
1836 }
1837
1838 if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {
1839 /* Fall back to bounce buffer if write zeroes is unsupported */
1840 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1841
1842 if ((flags & BDRV_REQ_FUA) &&
1843 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1844 /* No need for bdrv_driver_pwrite() to do a fallback
1845 * flush on each chunk; use just one at the end */
1846 write_flags &= ~BDRV_REQ_FUA;
1847 need_flush = true;
1848 }
1849 num = MIN(num, max_transfer);
1850 if (buf == NULL) {
1851 buf = qemu_try_blockalign0(bs, num);
1852 if (buf == NULL) {
1853 ret = -ENOMEM;
1854 goto fail;
1855 }
1856 }
1857 qemu_iovec_init_buf(&qiov, buf, num);
1858
1859 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
1860
1861 /* Keep bounce buffer around if it is big enough for all
1862 * all future requests.
1863 */
1864 if (num < max_transfer) {
1865 qemu_vfree(buf);
1866 buf = NULL;
1867 }
1868 }
1869
1870 offset += num;
1871 bytes -= num;
1872 }
1873
1874 fail:
1875 if (ret == 0 && need_flush) {
1876 ret = bdrv_co_flush(bs);
1877 }
1878 qemu_vfree(buf);
1879 return ret;
1880 }
1881
1882 static inline int coroutine_fn
1883 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, uint64_t bytes,
1884 BdrvTrackedRequest *req, int flags)
1885 {
1886 BlockDriverState *bs = child->bs;
1887 bool waited;
1888 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1889
1890 if (bs->read_only) {
1891 return -EPERM;
1892 }
1893
1894 assert(!(bs->open_flags & BDRV_O_INACTIVE));
1895 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1896 assert(!(flags & ~BDRV_REQ_MASK));
1897
1898 if (flags & BDRV_REQ_SERIALISING) {
1899 waited = bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs));
1900 /*
1901 * For a misaligned request we should have already waited earlier,
1902 * because we come after bdrv_padding_rmw_read which must be called
1903 * with the request already marked as serialising.
1904 */
1905 assert(!waited ||
1906 (req->offset == req->overlap_offset &&
1907 req->bytes == req->overlap_bytes));
1908 } else {
1909 bdrv_wait_serialising_requests(req);
1910 }
1911
1912 assert(req->overlap_offset <= offset);
1913 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1914 assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE);
1915
1916 switch (req->type) {
1917 case BDRV_TRACKED_WRITE:
1918 case BDRV_TRACKED_DISCARD:
1919 if (flags & BDRV_REQ_WRITE_UNCHANGED) {
1920 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1921 } else {
1922 assert(child->perm & BLK_PERM_WRITE);
1923 }
1924 return notifier_with_return_list_notify(&bs->before_write_notifiers,
1925 req);
1926 case BDRV_TRACKED_TRUNCATE:
1927 assert(child->perm & BLK_PERM_RESIZE);
1928 return 0;
1929 default:
1930 abort();
1931 }
1932 }
1933
1934 static inline void coroutine_fn
1935 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, uint64_t bytes,
1936 BdrvTrackedRequest *req, int ret)
1937 {
1938 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1939 BlockDriverState *bs = child->bs;
1940
1941 qatomic_inc(&bs->write_gen);
1942
1943 /*
1944 * Discard cannot extend the image, but in error handling cases, such as
1945 * when reverting a qcow2 cluster allocation, the discarded range can pass
1946 * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD
1947 * here. Instead, just skip it, since semantically a discard request
1948 * beyond EOF cannot expand the image anyway.
1949 */
1950 if (ret == 0 &&
1951 (req->type == BDRV_TRACKED_TRUNCATE ||
1952 end_sector > bs->total_sectors) &&
1953 req->type != BDRV_TRACKED_DISCARD) {
1954 bs->total_sectors = end_sector;
1955 bdrv_parent_cb_resize(bs);
1956 bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS);
1957 }
1958 if (req->bytes) {
1959 switch (req->type) {
1960 case BDRV_TRACKED_WRITE:
1961 stat64_max(&bs->wr_highest_offset, offset + bytes);
1962 /* fall through, to set dirty bits */
1963 case BDRV_TRACKED_DISCARD:
1964 bdrv_set_dirty(bs, offset, bytes);
1965 break;
1966 default:
1967 break;
1968 }
1969 }
1970 }
1971
1972 /*
1973 * Forwards an already correctly aligned write request to the BlockDriver,
1974 * after possibly fragmenting it.
1975 */
1976 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
1977 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1978 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
1979 {
1980 BlockDriverState *bs = child->bs;
1981 BlockDriver *drv = bs->drv;
1982 int ret;
1983
1984 uint64_t bytes_remaining = bytes;
1985 int max_transfer;
1986
1987 if (!drv) {
1988 return -ENOMEDIUM;
1989 }
1990
1991 if (bdrv_has_readonly_bitmaps(bs)) {
1992 return -EPERM;
1993 }
1994
1995 assert(is_power_of_2(align));
1996 assert((offset & (align - 1)) == 0);
1997 assert((bytes & (align - 1)) == 0);
1998 assert(!qiov || qiov_offset + bytes <= qiov->size);
1999 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
2000 align);
2001
2002 ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags);
2003
2004 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
2005 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
2006 qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
2007 flags |= BDRV_REQ_ZERO_WRITE;
2008 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
2009 flags |= BDRV_REQ_MAY_UNMAP;
2010 }
2011 }
2012
2013 if (ret < 0) {
2014 /* Do nothing, write notifier decided to fail this request */
2015 } else if (flags & BDRV_REQ_ZERO_WRITE) {
2016 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
2017 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
2018 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
2019 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
2020 qiov, qiov_offset);
2021 } else if (bytes <= max_transfer) {
2022 bdrv_debug_event(bs, BLKDBG_PWRITEV);
2023 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
2024 } else {
2025 bdrv_debug_event(bs, BLKDBG_PWRITEV);
2026 while (bytes_remaining) {
2027 int num = MIN(bytes_remaining, max_transfer);
2028 int local_flags = flags;
2029
2030 assert(num);
2031 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
2032 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
2033 /* If FUA is going to be emulated by flush, we only
2034 * need to flush on the last iteration */
2035 local_flags &= ~BDRV_REQ_FUA;
2036 }
2037
2038 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
2039 num, qiov,
2040 qiov_offset + bytes - bytes_remaining,
2041 local_flags);
2042 if (ret < 0) {
2043 break;
2044 }
2045 bytes_remaining -= num;
2046 }
2047 }
2048 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
2049
2050 if (ret >= 0) {
2051 ret = 0;
2052 }
2053 bdrv_co_write_req_finish(child, offset, bytes, req, ret);
2054
2055 return ret;
2056 }
2057
2058 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
2059 int64_t offset,
2060 unsigned int bytes,
2061 BdrvRequestFlags flags,
2062 BdrvTrackedRequest *req)
2063 {
2064 BlockDriverState *bs = child->bs;
2065 QEMUIOVector local_qiov;
2066 uint64_t align = bs->bl.request_alignment;
2067 int ret = 0;
2068 bool padding;
2069 BdrvRequestPadding pad;
2070
2071 padding = bdrv_init_padding(bs, offset, bytes, &pad);
2072 if (padding) {
2073 bdrv_mark_request_serialising(req, align);
2074
2075 bdrv_padding_rmw_read(child, req, &pad, true);
2076
2077 if (pad.head || pad.merge_reads) {
2078 int64_t aligned_offset = offset & ~(align - 1);
2079 int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
2080
2081 qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
2082 ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
2083 align, &local_qiov, 0,
2084 flags & ~BDRV_REQ_ZERO_WRITE);
2085 if (ret < 0 || pad.merge_reads) {
2086 /* Error or all work is done */
2087 goto out;
2088 }
2089 offset += write_bytes - pad.head;
2090 bytes -= write_bytes - pad.head;
2091 }
2092 }
2093
2094 assert(!bytes || (offset & (align - 1)) == 0);
2095 if (bytes >= align) {
2096 /* Write the aligned part in the middle. */
2097 uint64_t aligned_bytes = bytes & ~(align - 1);
2098 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
2099 NULL, 0, flags);
2100 if (ret < 0) {
2101 goto out;
2102 }
2103 bytes -= aligned_bytes;
2104 offset += aligned_bytes;
2105 }
2106
2107 assert(!bytes || (offset & (align - 1)) == 0);
2108 if (bytes) {
2109 assert(align == pad.tail + bytes);
2110
2111 qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
2112 ret = bdrv_aligned_pwritev(child, req, offset, align, align,
2113 &local_qiov, 0,
2114 flags & ~BDRV_REQ_ZERO_WRITE);
2115 }
2116
2117 out:
2118 bdrv_padding_destroy(&pad);
2119
2120 return ret;
2121 }
2122
2123 /*
2124 * Handle a write request in coroutine context
2125 */
2126 int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
2127 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
2128 BdrvRequestFlags flags)
2129 {
2130 return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
2131 }
2132
2133 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
2134 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, size_t qiov_offset,
2135 BdrvRequestFlags flags)
2136 {
2137 BlockDriverState *bs = child->bs;
2138 BdrvTrackedRequest req;
2139 uint64_t align = bs->bl.request_alignment;
2140 BdrvRequestPadding pad;
2141 int ret;
2142
2143 trace_bdrv_co_pwritev(child->bs, offset, bytes, flags);
2144
2145 if (!bs->drv) {
2146 return -ENOMEDIUM;
2147 }
2148
2149 ret = bdrv_check_byte_request(bs, offset, bytes);
2150 if (ret < 0) {
2151 return ret;
2152 }
2153
2154 /* If the request is misaligned then we can't make it efficient */
2155 if ((flags & BDRV_REQ_NO_FALLBACK) &&
2156 !QEMU_IS_ALIGNED(offset | bytes, align))
2157 {
2158 return -ENOTSUP;
2159 }
2160
2161 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
2162 /*
2163 * Aligning zero request is nonsense. Even if driver has special meaning
2164 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
2165 * it to driver due to request_alignment.
2166 *
2167 * Still, no reason to return an error if someone do unaligned
2168 * zero-length write occasionally.
2169 */
2170 return 0;
2171 }
2172
2173 bdrv_inc_in_flight(bs);
2174 /*
2175 * Align write if necessary by performing a read-modify-write cycle.
2176 * Pad qiov with the read parts and be sure to have a tracked request not
2177 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
2178 */
2179 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
2180
2181 if (flags & BDRV_REQ_ZERO_WRITE) {
2182 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
2183 goto out;
2184 }
2185
2186 if (bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad)) {
2187 bdrv_mark_request_serialising(&req, align);
2188 bdrv_padding_rmw_read(child, &req, &pad, false);
2189 }
2190
2191 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
2192 qiov, qiov_offset, flags);
2193
2194 bdrv_padding_destroy(&pad);
2195
2196 out:
2197 tracked_request_end(&req);
2198 bdrv_dec_in_flight(bs);
2199
2200 return ret;
2201 }
2202
2203 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
2204 int bytes, BdrvRequestFlags flags)
2205 {
2206 trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
2207
2208 if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
2209 flags &= ~BDRV_REQ_MAY_UNMAP;
2210 }
2211
2212 return bdrv_co_pwritev(child, offset, bytes, NULL,
2213 BDRV_REQ_ZERO_WRITE | flags);
2214 }
2215
2216 /*
2217 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
2218 */
2219 int bdrv_flush_all(void)
2220 {
2221 BdrvNextIterator it;
2222 BlockDriverState *bs = NULL;
2223 int result = 0;
2224
2225 /*
2226 * bdrv queue is managed by record/replay,
2227 * creating new flush request for stopping
2228 * the VM may break the determinism
2229 */
2230 if (replay_events_enabled()) {
2231 return result;
2232 }
2233
2234 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
2235 AioContext *aio_context = bdrv_get_aio_context(bs);
2236 int ret;
2237
2238 aio_context_acquire(aio_context);
2239 ret = bdrv_flush(bs);
2240 if (ret < 0 && !result) {
2241 result = ret;
2242 }
2243 aio_context_release(aio_context);
2244 }
2245
2246 return result;
2247 }
2248
2249
2250 typedef struct BdrvCoBlockStatusData {
2251 BlockDriverState *bs;
2252 BlockDriverState *base;
2253 bool want_zero;
2254 int64_t offset;
2255 int64_t bytes;
2256 int64_t *pnum;
2257 int64_t *map;
2258 BlockDriverState **file;
2259 } BdrvCoBlockStatusData;
2260
2261 /*
2262 * Returns the allocation status of the specified sectors.
2263 * Drivers not implementing the functionality are assumed to not support
2264 * backing files, hence all their sectors are reported as allocated.
2265 *
2266 * If 'want_zero' is true, the caller is querying for mapping
2267 * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and
2268 * _ZERO where possible; otherwise, the result favors larger 'pnum',
2269 * with a focus on accurate BDRV_BLOCK_ALLOCATED.
2270 *
2271 * If 'offset' is beyond the end of the disk image the return value is
2272 * BDRV_BLOCK_EOF and 'pnum' is set to 0.
2273 *
2274 * 'bytes' is the max value 'pnum' should be set to. If bytes goes
2275 * beyond the end of the disk image it will be clamped; if 'pnum' is set to
2276 * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
2277 *
2278 * 'pnum' is set to the number of bytes (including and immediately
2279 * following the specified offset) that are easily known to be in the
2280 * same allocated/unallocated state. Note that a second call starting
2281 * at the original offset plus returned pnum may have the same status.
2282 * The returned value is non-zero on success except at end-of-file.
2283 *
2284 * Returns negative errno on failure. Otherwise, if the
2285 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
2286 * set to the host mapping and BDS corresponding to the guest offset.
2287 */
2288 static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
2289 bool want_zero,
2290 int64_t offset, int64_t bytes,
2291 int64_t *pnum, int64_t *map,
2292 BlockDriverState **file)
2293 {
2294 int64_t total_size;
2295 int64_t n; /* bytes */
2296 int ret;
2297 int64_t local_map = 0;
2298 BlockDriverState *local_file = NULL;
2299 int64_t aligned_offset, aligned_bytes;
2300 uint32_t align;
2301 bool has_filtered_child;
2302
2303 assert(pnum);
2304 *pnum = 0;
2305 total_size = bdrv_getlength(bs);
2306 if (total_size < 0) {
2307 ret = total_size;
2308 goto early_out;
2309 }
2310
2311 if (offset >= total_size) {
2312 ret = BDRV_BLOCK_EOF;
2313 goto early_out;
2314 }
2315 if (!bytes) {
2316 ret = 0;
2317 goto early_out;
2318 }
2319
2320 n = total_size - offset;
2321 if (n < bytes) {
2322 bytes = n;
2323 }
2324
2325 /* Must be non-NULL or bdrv_getlength() would have failed */
2326 assert(bs->drv);
2327 has_filtered_child = bdrv_filter_child(bs);
2328 if (!bs->drv->bdrv_co_block_status && !has_filtered_child) {
2329 *pnum = bytes;
2330 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
2331 if (offset + bytes == total_size) {
2332 ret |= BDRV_BLOCK_EOF;
2333 }
2334 if (bs->drv->protocol_name) {
2335 ret |= BDRV_BLOCK_OFFSET_VALID;
2336 local_map = offset;
2337 local_file = bs;
2338 }
2339 goto early_out;
2340 }
2341
2342 bdrv_inc_in_flight(bs);
2343
2344 /* Round out to request_alignment boundaries */
2345 align = bs->bl.request_alignment;
2346 aligned_offset = QEMU_ALIGN_DOWN(offset, align);
2347 aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
2348
2349 if (bs->drv->bdrv_co_block_status) {
2350 ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
2351 aligned_bytes, pnum, &local_map,
2352 &local_file);
2353 } else {
2354 /* Default code for filters */
2355
2356 local_file = bdrv_filter_bs(bs);
2357 assert(local_file);
2358
2359 *pnum = aligned_bytes;
2360 local_map = aligned_offset;
2361 ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2362 }
2363 if (ret < 0) {
2364 *pnum = 0;
2365 goto out;
2366 }
2367
2368 /*
2369 * The driver's result must be a non-zero multiple of request_alignment.
2370 * Clamp pnum and adjust map to original request.
2371 */
2372 assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2373 align > offset - aligned_offset);
2374 if (ret & BDRV_BLOCK_RECURSE) {
2375 assert(ret & BDRV_BLOCK_DATA);
2376 assert(ret & BDRV_BLOCK_OFFSET_VALID);
2377 assert(!(ret & BDRV_BLOCK_ZERO));
2378 }
2379
2380 *pnum -= offset - aligned_offset;
2381 if (*pnum > bytes) {
2382 *pnum = bytes;
2383 }
2384 if (ret & BDRV_BLOCK_OFFSET_VALID) {
2385 local_map += offset - aligned_offset;
2386 }
2387
2388 if (ret & BDRV_BLOCK_RAW) {
2389 assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
2390 ret = bdrv_co_block_status(local_file, want_zero, local_map,
2391 *pnum, pnum, &local_map, &local_file);
2392 goto out;
2393 }
2394
2395 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
2396 ret |= BDRV_BLOCK_ALLOCATED;
2397 } else if (want_zero && bs->drv->supports_backing) {
2398 BlockDriverState *cow_bs = bdrv_cow_bs(bs);
2399
2400 if (cow_bs) {
2401 int64_t size2 = bdrv_getlength(cow_bs);
2402
2403 if (size2 >= 0 && offset >= size2) {
2404 ret |= BDRV_BLOCK_ZERO;
2405 }
2406 } else {
2407 ret |= BDRV_BLOCK_ZERO;
2408 }
2409 }
2410
2411 if (want_zero && ret & BDRV_BLOCK_RECURSE &&
2412 local_file && local_file != bs &&
2413 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
2414 (ret & BDRV_BLOCK_OFFSET_VALID)) {
2415 int64_t file_pnum;
2416 int ret2;
2417
2418 ret2 = bdrv_co_block_status(local_file, want_zero, local_map,
2419 *pnum, &file_pnum, NULL, NULL);
2420 if (ret2 >= 0) {
2421 /* Ignore errors. This is just providing extra information, it
2422 * is useful but not necessary.
2423 */
2424 if (ret2 & BDRV_BLOCK_EOF &&
2425 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2426 /*
2427 * It is valid for the format block driver to read
2428 * beyond the end of the underlying file's current
2429 * size; such areas read as zero.
2430 */
2431 ret |= BDRV_BLOCK_ZERO;
2432 } else {
2433 /* Limit request to the range reported by the protocol driver */
2434 *pnum = file_pnum;
2435 ret |= (ret2 & BDRV_BLOCK_ZERO);
2436 }
2437 }
2438 }
2439
2440 out:
2441 bdrv_dec_in_flight(bs);
2442 if (ret >= 0 && offset + *pnum == total_size) {
2443 ret |= BDRV_BLOCK_EOF;
2444 }
2445 early_out:
2446 if (file) {
2447 *file = local_file;
2448 }
2449 if (map) {
2450 *map = local_map;
2451 }
2452 return ret;
2453 }
2454
2455 static int coroutine_fn
2456 bdrv_co_common_block_status_above(BlockDriverState *bs,
2457 BlockDriverState *base,
2458 bool want_zero,
2459 int64_t offset,
2460 int64_t bytes,
2461 int64_t *pnum,
2462 int64_t *map,
2463 BlockDriverState **file)
2464 {
2465 BlockDriverState *p;
2466 int ret = 0;
2467 bool first = true;
2468
2469 assert(bs != base);
2470 for (p = bs; p != base; p = bdrv_filter_or_cow_bs(p)) {
2471 ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map,
2472 file);
2473 if (ret < 0) {
2474 break;
2475 }
2476 if (ret & BDRV_BLOCK_ZERO && ret & BDRV_BLOCK_EOF && !first) {
2477 /*
2478 * Reading beyond the end of the file continues to read
2479 * zeroes, but we can only widen the result to the
2480 * unallocated length we learned from an earlier
2481 * iteration.
2482 */
2483 *pnum = bytes;
2484 }
2485 if (ret & (BDRV_BLOCK_ZERO | BDRV_BLOCK_DATA)) {
2486 break;
2487 }
2488 /* [offset, pnum] unallocated on this layer, which could be only
2489 * the first part of [offset, bytes]. */
2490 bytes = MIN(bytes, *pnum);
2491 first = false;
2492 }
2493 return ret;
2494 }
2495
2496 /* Coroutine wrapper for bdrv_block_status_above() */
2497 static int coroutine_fn bdrv_block_status_above_co_entry(void *opaque)
2498 {
2499 BdrvCoBlockStatusData *data = opaque;
2500
2501 return bdrv_co_common_block_status_above(data->bs, data->base,
2502 data->want_zero,
2503 data->offset, data->bytes,
2504 data->pnum, data->map, data->file);
2505 }
2506
2507 /*
2508 * Synchronous wrapper around bdrv_co_block_status_above().
2509 *
2510 * See bdrv_co_block_status_above() for details.
2511 */
2512 static int bdrv_common_block_status_above(BlockDriverState *bs,
2513 BlockDriverState *base,
2514 bool want_zero, int64_t offset,
2515 int64_t bytes, int64_t *pnum,
2516 int64_t *map,
2517 BlockDriverState **file)
2518 {
2519 BdrvCoBlockStatusData data = {
2520 .bs = bs,
2521 .base = base,
2522 .want_zero = want_zero,
2523 .offset = offset,
2524 .bytes = bytes,
2525 .pnum = pnum,
2526 .map = map,
2527 .file = file,
2528 };
2529
2530 return bdrv_run_co(bs, bdrv_block_status_above_co_entry, &data);
2531 }
2532
2533 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base,
2534 int64_t offset, int64_t bytes, int64_t *pnum,
2535 int64_t *map, BlockDriverState **file)
2536 {
2537 return bdrv_common_block_status_above(bs, base, true, offset, bytes,
2538 pnum, map, file);
2539 }
2540
2541 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes,
2542 int64_t *pnum, int64_t *map, BlockDriverState **file)
2543 {
2544 return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs),
2545 offset, bytes, pnum, map, file);
2546 }
2547
2548 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset,
2549 int64_t bytes, int64_t *pnum)
2550 {
2551 int ret;
2552 int64_t dummy;
2553
2554 ret = bdrv_common_block_status_above(bs, bdrv_filter_or_cow_bs(bs), false,
2555 offset, bytes, pnum ? pnum : &dummy,
2556 NULL, NULL);
2557 if (ret < 0) {
2558 return ret;
2559 }
2560 return !!(ret & BDRV_BLOCK_ALLOCATED);
2561 }
2562
2563 /*
2564 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
2565 *
2566 * Return 1 if (a prefix of) the given range is allocated in any image
2567 * between BASE and TOP (BASE is only included if include_base is set).
2568 * BASE can be NULL to check if the given offset is allocated in any
2569 * image of the chain. Return 0 otherwise, or negative errno on
2570 * failure.
2571 *
2572 * 'pnum' is set to the number of bytes (including and immediately
2573 * following the specified offset) that are known to be in the same
2574 * allocated/unallocated state. Note that a subsequent call starting
2575 * at 'offset + *pnum' may return the same allocation status (in other
2576 * words, the result is not necessarily the maximum possible range);
2577 * but 'pnum' will only be 0 when end of file is reached.
2578 *
2579 */
2580 int bdrv_is_allocated_above(BlockDriverState *top,
2581 BlockDriverState *base,
2582 bool include_base, int64_t offset,
2583 int64_t bytes, int64_t *pnum)
2584 {
2585 BlockDriverState *intermediate;
2586 int ret;
2587 int64_t n = bytes;
2588
2589 assert(base || !include_base);
2590
2591 intermediate = top;
2592 while (include_base || intermediate != base) {
2593 int64_t pnum_inter;
2594 int64_t size_inter;
2595
2596 assert(intermediate);
2597 ret = bdrv_is_allocated(intermediate, offset, bytes, &pnum_inter);
2598 if (ret < 0) {
2599 return ret;
2600 }
2601 if (ret) {
2602 *pnum = pnum_inter;
2603 return 1;
2604 }
2605
2606 size_inter = bdrv_getlength(intermediate);
2607 if (size_inter < 0) {
2608 return size_inter;
2609 }
2610 if (n > pnum_inter &&
2611 (intermediate == top || offset + pnum_inter < size_inter)) {
2612 n = pnum_inter;
2613 }
2614
2615 if (intermediate == base) {
2616 break;
2617 }
2618
2619 intermediate = bdrv_filter_or_cow_bs(intermediate);
2620 }
2621
2622 *pnum = n;
2623 return 0;
2624 }
2625
2626 typedef struct BdrvVmstateCo {
2627 BlockDriverState *bs;
2628 QEMUIOVector *qiov;
2629 int64_t pos;
2630 bool is_read;
2631 } BdrvVmstateCo;
2632
2633 static int coroutine_fn
2634 bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
2635 bool is_read)
2636 {
2637 BlockDriver *drv = bs->drv;
2638 BlockDriverState *child_bs = bdrv_primary_bs(bs);
2639 int ret = -ENOTSUP;
2640
2641 bdrv_inc_in_flight(bs);
2642
2643 if (!drv) {
2644 ret = -ENOMEDIUM;
2645 } else if (drv->bdrv_load_vmstate) {
2646 if (is_read) {
2647 ret = drv->bdrv_load_vmstate(bs, qiov, pos);
2648 } else {
2649 ret = drv->bdrv_save_vmstate(bs, qiov, pos);
2650 }
2651 } else if (child_bs) {
2652 ret = bdrv_co_rw_vmstate(child_bs, qiov, pos, is_read);
2653 }
2654
2655 bdrv_dec_in_flight(bs);
2656 return ret;
2657 }
2658
2659 static int coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque)
2660 {
2661 BdrvVmstateCo *co = opaque;
2662
2663 return bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read);
2664 }
2665
2666 static inline int
2667 bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
2668 bool is_read)
2669 {
2670 BdrvVmstateCo data = {
2671 .bs = bs,
2672 .qiov = qiov,
2673 .pos = pos,
2674 .is_read = is_read,
2675 };
2676
2677 return bdrv_run_co(bs, bdrv_co_rw_vmstate_entry, &data);
2678 }
2679
2680 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
2681 int64_t pos, int size)
2682 {
2683 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2684 int ret;
2685
2686 ret = bdrv_writev_vmstate(bs, &qiov, pos);
2687 if (ret < 0) {
2688 return ret;
2689 }
2690
2691 return size;
2692 }
2693
2694 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2695 {
2696 return bdrv_rw_vmstate(bs, qiov, pos, false);
2697 }
2698
2699 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
2700 int64_t pos, int size)
2701 {
2702 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2703 int ret;
2704
2705 ret = bdrv_readv_vmstate(bs, &qiov, pos);
2706 if (ret < 0) {
2707 return ret;
2708 }
2709
2710 return size;
2711 }
2712
2713 int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2714 {
2715 return bdrv_rw_vmstate(bs, qiov, pos, true);
2716 }
2717
2718 /**************************************************************/
2719 /* async I/Os */
2720
2721 void bdrv_aio_cancel(BlockAIOCB *acb)
2722 {
2723 qemu_aio_ref(acb);
2724 bdrv_aio_cancel_async(acb);
2725 while (acb->refcnt > 1) {
2726 if (acb->aiocb_info->get_aio_context) {
2727 aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2728 } else if (acb->bs) {
2729 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
2730 * assert that we're not using an I/O thread. Thread-safe
2731 * code should use bdrv_aio_cancel_async exclusively.
2732 */
2733 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
2734 aio_poll(bdrv_get_aio_context(acb->bs), true);
2735 } else {
2736 abort();
2737 }
2738 }
2739 qemu_aio_unref(acb);
2740 }
2741
2742 /* Async version of aio cancel. The caller is not blocked if the acb implements
2743 * cancel_async, otherwise we do nothing and let the request normally complete.
2744 * In either case the completion callback must be called. */
2745 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2746 {
2747 if (acb->aiocb_info->cancel_async) {
2748 acb->aiocb_info->cancel_async(acb);
2749 }
2750 }
2751
2752 /**************************************************************/
2753 /* Coroutine block device emulation */
2754
2755 static int coroutine_fn bdrv_flush_co_entry(void *opaque)
2756 {
2757 return bdrv_co_flush(opaque);
2758 }
2759
2760 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2761 {
2762 BdrvChild *primary_child = bdrv_primary_child(bs);
2763 BdrvChild *child;
2764 int current_gen;
2765 int ret = 0;
2766
2767 bdrv_inc_in_flight(bs);
2768
2769 if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2770 bdrv_is_sg(bs)) {
2771 goto early_exit;
2772 }
2773
2774 qemu_co_mutex_lock(&bs->reqs_lock);
2775 current_gen = qatomic_read(&bs->write_gen);
2776
2777 /* Wait until any previous flushes are completed */
2778 while (bs->active_flush_req) {
2779 qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
2780 }
2781
2782 /* Flushes reach this point in nondecreasing current_gen order. */
2783 bs->active_flush_req = true;
2784 qemu_co_mutex_unlock(&bs->reqs_lock);
2785
2786 /* Write back all layers by calling one driver function */
2787 if (bs->drv->bdrv_co_flush) {
2788 ret = bs->drv->bdrv_co_flush(bs);
2789 goto out;
2790 }
2791
2792 /* Write back cached data to the OS even with cache=unsafe */
2793 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS);
2794 if (bs->drv->bdrv_co_flush_to_os) {
2795 ret = bs->drv->bdrv_co_flush_to_os(bs);
2796 if (ret < 0) {
2797 goto out;
2798 }
2799 }
2800
2801 /* But don't actually force it to the disk with cache=unsafe */
2802 if (bs->open_flags & BDRV_O_NO_FLUSH) {
2803 goto flush_children;
2804 }
2805
2806 /* Check if we really need to flush anything */
2807 if (bs->flushed_gen == current_gen) {
2808 goto flush_children;
2809 }
2810
2811 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK);
2812 if (!bs->drv) {
2813 /* bs->drv->bdrv_co_flush() might have ejected the BDS
2814 * (even in case of apparent success) */
2815 ret = -ENOMEDIUM;
2816 goto out;
2817 }
2818 if (bs->drv->bdrv_co_flush_to_disk) {
2819 ret = bs->drv->bdrv_co_flush_to_disk(bs);
2820 } else if (bs->drv->bdrv_aio_flush) {
2821 BlockAIOCB *acb;
2822 CoroutineIOCompletion co = {
2823 .coroutine = qemu_coroutine_self(),
2824 };
2825
2826 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2827 if (acb == NULL) {
2828 ret = -EIO;
2829 } else {
2830 qemu_coroutine_yield();
2831 ret = co.ret;
2832 }
2833 } else {
2834 /*
2835 * Some block drivers always operate in either writethrough or unsafe
2836 * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2837 * know how the server works (because the behaviour is hardcoded or
2838 * depends on server-side configuration), so we can't ensure that
2839 * everything is safe on disk. Returning an error doesn't work because
2840 * that would break guests even if the server operates in writethrough
2841 * mode.
2842 *
2843 * Let's hope the user knows what he's doing.
2844 */
2845 ret = 0;
2846 }
2847
2848 if (ret < 0) {
2849 goto out;
2850 }
2851
2852 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH
2853 * in the case of cache=unsafe, so there are no useless flushes.
2854 */
2855 flush_children:
2856 ret = 0;
2857 QLIST_FOREACH(child, &bs->children, next) {
2858 if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) {
2859 int this_child_ret = bdrv_co_flush(child->bs);
2860 if (!ret) {
2861 ret = this_child_ret;
2862 }
2863 }
2864 }
2865
2866 out:
2867 /* Notify any pending flushes that we have completed */
2868 if (ret == 0) {
2869 bs->flushed_gen = current_gen;
2870 }
2871
2872 qemu_co_mutex_lock(&bs->reqs_lock);
2873 bs->active_flush_req = false;
2874 /* Return value is ignored - it's ok if wait queue is empty */
2875 qemu_co_queue_next(&bs->flush_queue);
2876 qemu_co_mutex_unlock(&bs->reqs_lock);
2877
2878 early_exit:
2879 bdrv_dec_in_flight(bs);
2880 return ret;
2881 }
2882
2883 int bdrv_flush(BlockDriverState *bs)
2884 {
2885 return bdrv_run_co(bs, bdrv_flush_co_entry, bs);
2886 }
2887
2888 typedef struct DiscardCo {
2889 BdrvChild *child;
2890 int64_t offset;
2891 int64_t bytes;
2892 } DiscardCo;
2893
2894 static int coroutine_fn bdrv_pdiscard_co_entry(void *opaque)
2895 {
2896 DiscardCo *rwco = opaque;
2897
2898 return bdrv_co_pdiscard(rwco->child, rwco->offset, rwco->bytes);
2899 }
2900
2901 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
2902 int64_t bytes)
2903 {
2904 BdrvTrackedRequest req;
2905 int max_pdiscard, ret;
2906 int head, tail, align;
2907 BlockDriverState *bs = child->bs;
2908
2909 if (!bs || !bs->drv || !bdrv_is_inserted(bs)) {
2910 return -ENOMEDIUM;
2911 }
2912
2913 if (bdrv_has_readonly_bitmaps(bs)) {
2914 return -EPERM;
2915 }
2916
2917 if (offset < 0 || bytes < 0 || bytes > INT64_MAX - offset) {
2918 return -EIO;
2919 }
2920
2921 /* Do nothing if disabled. */
2922 if (!(bs->open_flags & BDRV_O_UNMAP)) {
2923 return 0;
2924 }
2925
2926 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
2927 return 0;
2928 }
2929
2930 /* Discard is advisory, but some devices track and coalesce
2931 * unaligned requests, so we must pass everything down rather than
2932 * round here. Still, most devices will just silently ignore
2933 * unaligned requests (by returning -ENOTSUP), so we must fragment
2934 * the request accordingly. */
2935 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2936 assert(align % bs->bl.request_alignment == 0);
2937 head = offset % align;
2938 tail = (offset + bytes) % align;
2939
2940 bdrv_inc_in_flight(bs);
2941 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
2942
2943 ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0);
2944 if (ret < 0) {
2945 goto out;
2946 }
2947
2948 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX),
2949 align);
2950 assert(max_pdiscard >= bs->bl.request_alignment);
2951
2952 while (bytes > 0) {
2953 int64_t num = bytes;
2954
2955 if (head) {
2956 /* Make small requests to get to alignment boundaries. */
2957 num = MIN(bytes, align - head);
2958 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
2959 num %= bs->bl.request_alignment;
2960 }
2961 head = (head + num) % align;
2962 assert(num < max_pdiscard);
2963 } else if (tail) {
2964 if (num > align) {
2965 /* Shorten the request to the last aligned cluster. */
2966 num -= tail;
2967 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
2968 tail > bs->bl.request_alignment) {
2969 tail %= bs->bl.request_alignment;
2970 num -= tail;
2971 }
2972 }
2973 /* limit request size */
2974 if (num > max_pdiscard) {
2975 num = max_pdiscard;
2976 }
2977
2978 if (!bs->drv) {
2979 ret = -ENOMEDIUM;
2980 goto out;
2981 }
2982 if (bs->drv->bdrv_co_pdiscard) {
2983 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
2984 } else {
2985 BlockAIOCB *acb;
2986 CoroutineIOCompletion co = {
2987 .coroutine = qemu_coroutine_self(),
2988 };
2989
2990 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
2991 bdrv_co_io_em_complete, &co);
2992 if (acb == NULL) {
2993 ret = -EIO;
2994 goto out;
2995 } else {
2996 qemu_coroutine_yield();
2997 ret = co.ret;
2998 }
2999 }
3000 if (ret && ret != -ENOTSUP) {
3001 goto out;
3002 }
3003
3004 offset += num;
3005 bytes -= num;
3006 }
3007 ret = 0;
3008 out:
3009 bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret);
3010 tracked_request_end(&req);
3011 bdrv_dec_in_flight(bs);
3012 return ret;
3013 }
3014
3015 int bdrv_pdiscard(BdrvChild *child, int64_t offset, int64_t bytes)
3016 {
3017 DiscardCo rwco = {
3018 .child = child,
3019 .offset = offset,
3020 .bytes = bytes,
3021 };
3022
3023 return bdrv_run_co(child->bs, bdrv_pdiscard_co_entry, &rwco);
3024 }
3025
3026 int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
3027 {
3028 BlockDriver *drv = bs->drv;
3029 CoroutineIOCompletion co = {
3030 .coroutine = qemu_coroutine_self(),
3031 };
3032 BlockAIOCB *acb;
3033
3034 bdrv_inc_in_flight(bs);
3035 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
3036 co.ret = -ENOTSUP;
3037 goto out;
3038 }
3039
3040 if (drv->bdrv_co_ioctl) {
3041 co.ret = drv->bdrv_co_ioctl(bs, req, buf);
3042 } else {
3043 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
3044 if (!acb) {
3045 co.ret = -ENOTSUP;
3046 goto out;
3047 }
3048 qemu_coroutine_yield();
3049 }
3050 out:
3051 bdrv_dec_in_flight(bs);
3052 return co.ret;
3053 }
3054
3055 void *qemu_blockalign(BlockDriverState *bs, size_t size)
3056 {
3057 return qemu_memalign(bdrv_opt_mem_align(bs), size);
3058 }
3059
3060 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
3061 {
3062 return memset(qemu_blockalign(bs, size), 0, size);
3063 }
3064
3065 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
3066 {
3067 size_t align = bdrv_opt_mem_align(bs);
3068
3069 /* Ensure that NULL is never returned on success */
3070 assert(align > 0);
3071 if (size == 0) {
3072 size = align;
3073 }
3074
3075 return qemu_try_memalign(align, size);
3076 }
3077
3078 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
3079 {
3080 void *mem = qemu_try_blockalign(bs, size);
3081
3082 if (mem) {
3083 memset(mem, 0, size);
3084 }
3085
3086 return mem;
3087 }
3088
3089 /*
3090 * Check if all memory in this vector is sector aligned.
3091 */
3092 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
3093 {
3094 int i;
3095 size_t alignment = bdrv_min_mem_align(bs);
3096
3097 for (i = 0; i < qiov->niov; i++) {
3098 if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
3099 return false;
3100 }
3101 if (qiov->iov[i].iov_len % alignment) {
3102 return false;
3103 }
3104 }
3105
3106 return true;
3107 }
3108
3109 void bdrv_add_before_write_notifier(BlockDriverState *bs,
3110 NotifierWithReturn *notifier)
3111 {
3112 notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
3113 }
3114
3115 void bdrv_io_plug(BlockDriverState *bs)
3116 {
3117 BdrvChild *child;
3118
3119 QLIST_FOREACH(child, &bs->children, next) {
3120 bdrv_io_plug(child->bs);
3121 }
3122
3123 if (qatomic_fetch_inc(&bs->io_plugged) == 0) {
3124 BlockDriver *drv = bs->drv;
3125 if (drv && drv->bdrv_io_plug) {
3126 drv->bdrv_io_plug(bs);
3127 }
3128 }
3129 }
3130
3131 void bdrv_io_unplug(BlockDriverState *bs)
3132 {
3133 BdrvChild *child;
3134
3135 assert(bs->io_plugged);
3136 if (qatomic_fetch_dec(&bs->io_plugged) == 1) {
3137 BlockDriver *drv = bs->drv;
3138 if (drv && drv->bdrv_io_unplug) {
3139 drv->bdrv_io_unplug(bs);
3140 }
3141 }
3142
3143 QLIST_FOREACH(child, &bs->children, next) {
3144 bdrv_io_unplug(child->bs);
3145 }
3146 }
3147
3148 void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size)
3149 {
3150 BdrvChild *child;
3151
3152 if (bs->drv && bs->drv->bdrv_register_buf) {
3153 bs->drv->bdrv_register_buf(bs, host, size);
3154 }
3155 QLIST_FOREACH(child, &bs->children, next) {
3156 bdrv_register_buf(child->bs, host, size);
3157 }
3158 }
3159
3160 void bdrv_unregister_buf(BlockDriverState *bs, void *host)
3161 {
3162 BdrvChild *child;
3163
3164 if (bs->drv && bs->drv->bdrv_unregister_buf) {
3165 bs->drv->bdrv_unregister_buf(bs, host);
3166 }
3167 QLIST_FOREACH(child, &bs->children, next) {
3168 bdrv_unregister_buf(child->bs, host);
3169 }
3170 }
3171
3172 static int coroutine_fn bdrv_co_copy_range_internal(
3173 BdrvChild *src, uint64_t src_offset, BdrvChild *dst,
3174 uint64_t dst_offset, uint64_t bytes,
3175 BdrvRequestFlags read_flags, BdrvRequestFlags write_flags,
3176 bool recurse_src)
3177 {
3178 BdrvTrackedRequest req;
3179 int ret;
3180
3181 /* TODO We can support BDRV_REQ_NO_FALLBACK here */
3182 assert(!(read_flags & BDRV_REQ_NO_FALLBACK));
3183 assert(!(write_flags & BDRV_REQ_NO_FALLBACK));
3184
3185 if (!dst || !dst->bs) {
3186 return -ENOMEDIUM;
3187 }
3188 ret = bdrv_check_byte_request(dst->bs, dst_offset, bytes);
3189 if (ret) {
3190 return ret;
3191 }
3192 if (write_flags & BDRV_REQ_ZERO_WRITE) {
3193 return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags);
3194 }
3195
3196 if (!src || !src->bs) {
3197 return -ENOMEDIUM;
3198 }
3199 ret = bdrv_check_byte_request(src->bs, src_offset, bytes);
3200 if (ret) {
3201 return ret;
3202 }
3203
3204 if (!src->bs->drv->bdrv_co_copy_range_from
3205 || !dst->bs->drv->bdrv_co_copy_range_to
3206 || src->bs->encrypted || dst->bs->encrypted) {
3207 return -ENOTSUP;
3208 }
3209
3210 if (recurse_src) {
3211 bdrv_inc_in_flight(src->bs);
3212 tracked_request_begin(&req, src->bs, src_offset, bytes,
3213 BDRV_TRACKED_READ);
3214
3215 /* BDRV_REQ_SERIALISING is only for write operation */
3216 assert(!(read_flags & BDRV_REQ_SERIALISING));
3217 bdrv_wait_serialising_requests(&req);
3218
3219 ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
3220 src, src_offset,
3221 dst, dst_offset,
3222 bytes,
3223 read_flags, write_flags);
3224
3225 tracked_request_end(&req);
3226 bdrv_dec_in_flight(src->bs);
3227 } else {
3228 bdrv_inc_in_flight(dst->bs);
3229 tracked_request_begin(&req, dst->bs, dst_offset, bytes,
3230 BDRV_TRACKED_WRITE);
3231 ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req,
3232 write_flags);
3233 if (!ret) {
3234 ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
3235 src, src_offset,
3236 dst, dst_offset,
3237 bytes,
3238 read_flags, write_flags);
3239 }
3240 bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret);
3241 tracked_request_end(&req);
3242 bdrv_dec_in_flight(dst->bs);
3243 }
3244
3245 return ret;
3246 }
3247
3248 /* Copy range from @src to @dst.
3249 *
3250 * See the comment of bdrv_co_copy_range for the parameter and return value
3251 * semantics. */
3252 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, uint64_t src_offset,
3253 BdrvChild *dst, uint64_t dst_offset,
3254 uint64_t bytes,
3255 BdrvRequestFlags read_flags,
3256 BdrvRequestFlags write_flags)
3257 {
3258 trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes,
3259 read_flags, write_flags);
3260 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3261 bytes, read_flags, write_flags, true);
3262 }
3263
3264 /* Copy range from @src to @dst.
3265 *
3266 * See the comment of bdrv_co_copy_range for the parameter and return value
3267 * semantics. */
3268 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, uint64_t src_offset,
3269 BdrvChild *dst, uint64_t dst_offset,
3270 uint64_t bytes,
3271 BdrvRequestFlags read_flags,
3272 BdrvRequestFlags write_flags)
3273 {
3274 trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3275 read_flags, write_flags);
3276 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3277 bytes, read_flags, write_flags, false);
3278 }
3279
3280 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, uint64_t src_offset,
3281 BdrvChild *dst, uint64_t dst_offset,
3282 uint64_t bytes, BdrvRequestFlags read_flags,
3283 BdrvRequestFlags write_flags)
3284 {
3285 return bdrv_co_copy_range_from(src, src_offset,
3286 dst, dst_offset,
3287 bytes, read_flags, write_flags);
3288 }
3289
3290 static void bdrv_parent_cb_resize(BlockDriverState *bs)
3291 {
3292 BdrvChild *c;
3293 QLIST_FOREACH(c, &bs->parents, next_parent) {
3294 if (c->klass->resize) {
3295 c->klass->resize(c);
3296 }
3297 }
3298 }
3299
3300 /**
3301 * Truncate file to 'offset' bytes (needed only for file protocols)
3302 *
3303 * If 'exact' is true, the file must be resized to exactly the given
3304 * 'offset'. Otherwise, it is sufficient for the node to be at least
3305 * 'offset' bytes in length.
3306 */
3307 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
3308 PreallocMode prealloc, BdrvRequestFlags flags,
3309 Error **errp)
3310 {
3311 BlockDriverState *bs = child->bs;
3312 BdrvChild *filtered, *backing;
3313 BlockDriver *drv = bs->drv;
3314 BdrvTrackedRequest req;
3315 int64_t old_size, new_bytes;
3316 int ret;
3317
3318
3319 /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
3320 if (!drv) {
3321 error_setg(errp, "No medium inserted");
3322 return -ENOMEDIUM;
3323 }
3324 if (offset < 0) {
3325 error_setg(errp, "Image size cannot be negative");
3326 return -EINVAL;
3327 }
3328
3329 old_size = bdrv_getlength(bs);
3330 if (old_size < 0) {
3331 error_setg_errno(errp, -old_size, "Failed to get old image size");
3332 return old_size;
3333 }
3334
3335 if (offset > old_size) {
3336 new_bytes = offset - old_size;
3337 } else {
3338 new_bytes = 0;
3339 }
3340
3341 bdrv_inc_in_flight(bs);
3342 tracked_request_begin(&req, bs, offset - new_bytes, new_bytes,
3343 BDRV_TRACKED_TRUNCATE);
3344
3345 /* If we are growing the image and potentially using preallocation for the
3346 * new area, we need to make sure that no write requests are made to it
3347 * concurrently or they might be overwritten by preallocation. */
3348 if (new_bytes) {
3349 bdrv_mark_request_serialising(&req, 1);
3350 }
3351 if (bs->read_only) {
3352 error_setg(errp, "Image is read-only");
3353 ret = -EACCES;
3354 goto out;
3355 }
3356 ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req,
3357 0);
3358 if (ret < 0) {
3359 error_setg_errno(errp, -ret,
3360 "Failed to prepare request for truncation");
3361 goto out;
3362 }
3363
3364 filtered = bdrv_filter_child(bs);
3365 backing = bdrv_cow_child(bs);
3366
3367 /*
3368 * If the image has a backing file that is large enough that it would
3369 * provide data for the new area, we cannot leave it unallocated because
3370 * then the backing file content would become visible. Instead, zero-fill
3371 * the new area.
3372 *
3373 * Note that if the image has a backing file, but was opened without the
3374 * backing file, taking care of keeping things consistent with that backing
3375 * file is the user's responsibility.
3376 */
3377 if (new_bytes && backing) {
3378 int64_t backing_len;
3379
3380 backing_len = bdrv_getlength(backing->bs);
3381 if (backing_len < 0) {
3382 ret = backing_len;
3383 error_setg_errno(errp, -ret, "Could not get backing file size");
3384 goto out;
3385 }
3386
3387 if (backing_len > old_size) {
3388 flags |= BDRV_REQ_ZERO_WRITE;
3389 }
3390 }
3391
3392 if (drv->bdrv_co_truncate) {
3393 if (flags & ~bs->supported_truncate_flags) {
3394 error_setg(errp, "Block driver does not support requested flags");
3395 ret = -ENOTSUP;
3396 goto out;
3397 }
3398 ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp);
3399 } else if (filtered) {
3400 ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp);
3401 } else {
3402 error_setg(errp, "Image format driver does not support resize");
3403 ret = -ENOTSUP;
3404 goto out;
3405 }
3406 if (ret < 0) {
3407 goto out;
3408 }
3409
3410 ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
3411 if (ret < 0) {
3412 error_setg_errno(errp, -ret, "Could not refresh total sector count");
3413 } else {
3414 offset = bs->total_sectors * BDRV_SECTOR_SIZE;
3415 }
3416 /* It's possible that truncation succeeded but refresh_total_sectors
3417 * failed, but the latter doesn't affect how we should finish the request.
3418 * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */
3419 bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0);
3420
3421 out:
3422 tracked_request_end(&req);
3423 bdrv_dec_in_flight(bs);
3424
3425 return ret;
3426 }
3427
3428 typedef struct TruncateCo {
3429 BdrvChild *child;
3430 int64_t offset;
3431 bool exact;
3432 PreallocMode prealloc;
3433 BdrvRequestFlags flags;
3434 Error **errp;
3435 } TruncateCo;
3436
3437 static int coroutine_fn bdrv_truncate_co_entry(void *opaque)
3438 {
3439 TruncateCo *tco = opaque;
3440
3441 return bdrv_co_truncate(tco->child, tco->offset, tco->exact,
3442 tco->prealloc, tco->flags, tco->errp);
3443 }
3444
3445 int bdrv_truncate(BdrvChild *child, int64_t offset, bool exact,
3446 PreallocMode prealloc, BdrvRequestFlags flags, Error **errp)
3447 {
3448 TruncateCo tco = {
3449 .child = child,
3450 .offset = offset,
3451 .exact = exact,
3452 .prealloc = prealloc,
3453 .flags = flags,
3454 .errp = errp,
3455 };
3456
3457 return bdrv_run_co(child->bs, bdrv_truncate_co_entry, &tco);
3458 }