block: drop bdrv_prwv
[qemu.git] / block / io.c
1 /*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/aio-wait.h"
29 #include "block/blockjob.h"
30 #include "block/blockjob_int.h"
31 #include "block/block_int.h"
32 #include "block/coroutines.h"
33 #include "qemu/cutils.h"
34 #include "qapi/error.h"
35 #include "qemu/error-report.h"
36 #include "qemu/main-loop.h"
37 #include "sysemu/replay.h"
38
39 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
40 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
41
42 static void bdrv_parent_cb_resize(BlockDriverState *bs);
43 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
44 int64_t offset, int bytes, BdrvRequestFlags flags);
45
46 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
47 bool ignore_bds_parents)
48 {
49 BdrvChild *c, *next;
50
51 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
52 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
53 continue;
54 }
55 bdrv_parent_drained_begin_single(c, false);
56 }
57 }
58
59 static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c,
60 int *drained_end_counter)
61 {
62 assert(c->parent_quiesce_counter > 0);
63 c->parent_quiesce_counter--;
64 if (c->klass->drained_end) {
65 c->klass->drained_end(c, drained_end_counter);
66 }
67 }
68
69 void bdrv_parent_drained_end_single(BdrvChild *c)
70 {
71 int drained_end_counter = 0;
72 bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter);
73 BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0);
74 }
75
76 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
77 bool ignore_bds_parents,
78 int *drained_end_counter)
79 {
80 BdrvChild *c;
81
82 QLIST_FOREACH(c, &bs->parents, next_parent) {
83 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
84 continue;
85 }
86 bdrv_parent_drained_end_single_no_poll(c, drained_end_counter);
87 }
88 }
89
90 static bool bdrv_parent_drained_poll_single(BdrvChild *c)
91 {
92 if (c->klass->drained_poll) {
93 return c->klass->drained_poll(c);
94 }
95 return false;
96 }
97
98 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
99 bool ignore_bds_parents)
100 {
101 BdrvChild *c, *next;
102 bool busy = false;
103
104 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
105 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
106 continue;
107 }
108 busy |= bdrv_parent_drained_poll_single(c);
109 }
110
111 return busy;
112 }
113
114 void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll)
115 {
116 c->parent_quiesce_counter++;
117 if (c->klass->drained_begin) {
118 c->klass->drained_begin(c);
119 }
120 if (poll) {
121 BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c));
122 }
123 }
124
125 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
126 {
127 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
128 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
129 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
130 src->opt_mem_alignment);
131 dst->min_mem_alignment = MAX(dst->min_mem_alignment,
132 src->min_mem_alignment);
133 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
134 }
135
136 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
137 {
138 BlockDriver *drv = bs->drv;
139 BdrvChild *c;
140 bool have_limits;
141 Error *local_err = NULL;
142
143 memset(&bs->bl, 0, sizeof(bs->bl));
144
145 if (!drv) {
146 return;
147 }
148
149 /* Default alignment based on whether driver has byte interface */
150 bs->bl.request_alignment = (drv->bdrv_co_preadv ||
151 drv->bdrv_aio_preadv ||
152 drv->bdrv_co_preadv_part) ? 1 : 512;
153
154 /* Take some limits from the children as a default */
155 have_limits = false;
156 QLIST_FOREACH(c, &bs->children, next) {
157 if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW))
158 {
159 bdrv_refresh_limits(c->bs, &local_err);
160 if (local_err) {
161 error_propagate(errp, local_err);
162 return;
163 }
164 bdrv_merge_limits(&bs->bl, &c->bs->bl);
165 have_limits = true;
166 }
167 }
168
169 if (!have_limits) {
170 bs->bl.min_mem_alignment = 512;
171 bs->bl.opt_mem_alignment = qemu_real_host_page_size;
172
173 /* Safe default since most protocols use readv()/writev()/etc */
174 bs->bl.max_iov = IOV_MAX;
175 }
176
177 /* Then let the driver override it */
178 if (drv->bdrv_refresh_limits) {
179 drv->bdrv_refresh_limits(bs, errp);
180 }
181 }
182
183 /**
184 * The copy-on-read flag is actually a reference count so multiple users may
185 * use the feature without worrying about clobbering its previous state.
186 * Copy-on-read stays enabled until all users have called to disable it.
187 */
188 void bdrv_enable_copy_on_read(BlockDriverState *bs)
189 {
190 qatomic_inc(&bs->copy_on_read);
191 }
192
193 void bdrv_disable_copy_on_read(BlockDriverState *bs)
194 {
195 int old = qatomic_fetch_dec(&bs->copy_on_read);
196 assert(old >= 1);
197 }
198
199 typedef struct {
200 Coroutine *co;
201 BlockDriverState *bs;
202 bool done;
203 bool begin;
204 bool recursive;
205 bool poll;
206 BdrvChild *parent;
207 bool ignore_bds_parents;
208 int *drained_end_counter;
209 } BdrvCoDrainData;
210
211 static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
212 {
213 BdrvCoDrainData *data = opaque;
214 BlockDriverState *bs = data->bs;
215
216 if (data->begin) {
217 bs->drv->bdrv_co_drain_begin(bs);
218 } else {
219 bs->drv->bdrv_co_drain_end(bs);
220 }
221
222 /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */
223 qatomic_mb_set(&data->done, true);
224 if (!data->begin) {
225 qatomic_dec(data->drained_end_counter);
226 }
227 bdrv_dec_in_flight(bs);
228
229 g_free(data);
230 }
231
232 /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
233 static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
234 int *drained_end_counter)
235 {
236 BdrvCoDrainData *data;
237
238 if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) ||
239 (!begin && !bs->drv->bdrv_co_drain_end)) {
240 return;
241 }
242
243 data = g_new(BdrvCoDrainData, 1);
244 *data = (BdrvCoDrainData) {
245 .bs = bs,
246 .done = false,
247 .begin = begin,
248 .drained_end_counter = drained_end_counter,
249 };
250
251 if (!begin) {
252 qatomic_inc(drained_end_counter);
253 }
254
255 /* Make sure the driver callback completes during the polling phase for
256 * drain_begin. */
257 bdrv_inc_in_flight(bs);
258 data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data);
259 aio_co_schedule(bdrv_get_aio_context(bs), data->co);
260 }
261
262 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
263 bool bdrv_drain_poll(BlockDriverState *bs, bool recursive,
264 BdrvChild *ignore_parent, bool ignore_bds_parents)
265 {
266 BdrvChild *child, *next;
267
268 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
269 return true;
270 }
271
272 if (qatomic_read(&bs->in_flight)) {
273 return true;
274 }
275
276 if (recursive) {
277 assert(!ignore_bds_parents);
278 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
279 if (bdrv_drain_poll(child->bs, recursive, child, false)) {
280 return true;
281 }
282 }
283 }
284
285 return false;
286 }
287
288 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive,
289 BdrvChild *ignore_parent)
290 {
291 return bdrv_drain_poll(bs, recursive, ignore_parent, false);
292 }
293
294 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
295 BdrvChild *parent, bool ignore_bds_parents,
296 bool poll);
297 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
298 BdrvChild *parent, bool ignore_bds_parents,
299 int *drained_end_counter);
300
301 static void bdrv_co_drain_bh_cb(void *opaque)
302 {
303 BdrvCoDrainData *data = opaque;
304 Coroutine *co = data->co;
305 BlockDriverState *bs = data->bs;
306
307 if (bs) {
308 AioContext *ctx = bdrv_get_aio_context(bs);
309 AioContext *co_ctx = qemu_coroutine_get_aio_context(co);
310
311 /*
312 * When the coroutine yielded, the lock for its home context was
313 * released, so we need to re-acquire it here. If it explicitly
314 * acquired a different context, the lock is still held and we don't
315 * want to lock it a second time (or AIO_WAIT_WHILE() would hang).
316 */
317 if (ctx == co_ctx) {
318 aio_context_acquire(ctx);
319 }
320 bdrv_dec_in_flight(bs);
321 if (data->begin) {
322 assert(!data->drained_end_counter);
323 bdrv_do_drained_begin(bs, data->recursive, data->parent,
324 data->ignore_bds_parents, data->poll);
325 } else {
326 assert(!data->poll);
327 bdrv_do_drained_end(bs, data->recursive, data->parent,
328 data->ignore_bds_parents,
329 data->drained_end_counter);
330 }
331 if (ctx == co_ctx) {
332 aio_context_release(ctx);
333 }
334 } else {
335 assert(data->begin);
336 bdrv_drain_all_begin();
337 }
338
339 data->done = true;
340 aio_co_wake(co);
341 }
342
343 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
344 bool begin, bool recursive,
345 BdrvChild *parent,
346 bool ignore_bds_parents,
347 bool poll,
348 int *drained_end_counter)
349 {
350 BdrvCoDrainData data;
351
352 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
353 * other coroutines run if they were queued by aio_co_enter(). */
354
355 assert(qemu_in_coroutine());
356 data = (BdrvCoDrainData) {
357 .co = qemu_coroutine_self(),
358 .bs = bs,
359 .done = false,
360 .begin = begin,
361 .recursive = recursive,
362 .parent = parent,
363 .ignore_bds_parents = ignore_bds_parents,
364 .poll = poll,
365 .drained_end_counter = drained_end_counter,
366 };
367
368 if (bs) {
369 bdrv_inc_in_flight(bs);
370 }
371 replay_bh_schedule_oneshot_event(bdrv_get_aio_context(bs),
372 bdrv_co_drain_bh_cb, &data);
373
374 qemu_coroutine_yield();
375 /* If we are resumed from some other event (such as an aio completion or a
376 * timer callback), it is a bug in the caller that should be fixed. */
377 assert(data.done);
378 }
379
380 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs,
381 BdrvChild *parent, bool ignore_bds_parents)
382 {
383 assert(!qemu_in_coroutine());
384
385 /* Stop things in parent-to-child order */
386 if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) {
387 aio_disable_external(bdrv_get_aio_context(bs));
388 }
389
390 bdrv_parent_drained_begin(bs, parent, ignore_bds_parents);
391 bdrv_drain_invoke(bs, true, NULL);
392 }
393
394 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
395 BdrvChild *parent, bool ignore_bds_parents,
396 bool poll)
397 {
398 BdrvChild *child, *next;
399
400 if (qemu_in_coroutine()) {
401 bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents,
402 poll, NULL);
403 return;
404 }
405
406 bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents);
407
408 if (recursive) {
409 assert(!ignore_bds_parents);
410 bs->recursive_quiesce_counter++;
411 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
412 bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents,
413 false);
414 }
415 }
416
417 /*
418 * Wait for drained requests to finish.
419 *
420 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
421 * call is needed so things in this AioContext can make progress even
422 * though we don't return to the main AioContext loop - this automatically
423 * includes other nodes in the same AioContext and therefore all child
424 * nodes.
425 */
426 if (poll) {
427 assert(!ignore_bds_parents);
428 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent));
429 }
430 }
431
432 void bdrv_drained_begin(BlockDriverState *bs)
433 {
434 bdrv_do_drained_begin(bs, false, NULL, false, true);
435 }
436
437 void bdrv_subtree_drained_begin(BlockDriverState *bs)
438 {
439 bdrv_do_drained_begin(bs, true, NULL, false, true);
440 }
441
442 /**
443 * This function does not poll, nor must any of its recursively called
444 * functions. The *drained_end_counter pointee will be incremented
445 * once for every background operation scheduled, and decremented once
446 * the operation settles. Therefore, the pointer must remain valid
447 * until the pointee reaches 0. That implies that whoever sets up the
448 * pointee has to poll until it is 0.
449 *
450 * We use atomic operations to access *drained_end_counter, because
451 * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of
452 * @bs may contain nodes in different AioContexts,
453 * (2) bdrv_drain_all_end() uses the same counter for all nodes,
454 * regardless of which AioContext they are in.
455 */
456 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
457 BdrvChild *parent, bool ignore_bds_parents,
458 int *drained_end_counter)
459 {
460 BdrvChild *child;
461 int old_quiesce_counter;
462
463 assert(drained_end_counter != NULL);
464
465 if (qemu_in_coroutine()) {
466 bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
467 false, drained_end_counter);
468 return;
469 }
470 assert(bs->quiesce_counter > 0);
471
472 /* Re-enable things in child-to-parent order */
473 bdrv_drain_invoke(bs, false, drained_end_counter);
474 bdrv_parent_drained_end(bs, parent, ignore_bds_parents,
475 drained_end_counter);
476
477 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
478 if (old_quiesce_counter == 1) {
479 aio_enable_external(bdrv_get_aio_context(bs));
480 }
481
482 if (recursive) {
483 assert(!ignore_bds_parents);
484 bs->recursive_quiesce_counter--;
485 QLIST_FOREACH(child, &bs->children, next) {
486 bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents,
487 drained_end_counter);
488 }
489 }
490 }
491
492 void bdrv_drained_end(BlockDriverState *bs)
493 {
494 int drained_end_counter = 0;
495 bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter);
496 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
497 }
498
499 void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter)
500 {
501 bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter);
502 }
503
504 void bdrv_subtree_drained_end(BlockDriverState *bs)
505 {
506 int drained_end_counter = 0;
507 bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter);
508 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
509 }
510
511 void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
512 {
513 int i;
514
515 for (i = 0; i < new_parent->recursive_quiesce_counter; i++) {
516 bdrv_do_drained_begin(child->bs, true, child, false, true);
517 }
518 }
519
520 void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
521 {
522 int drained_end_counter = 0;
523 int i;
524
525 for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
526 bdrv_do_drained_end(child->bs, true, child, false,
527 &drained_end_counter);
528 }
529
530 BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0);
531 }
532
533 /*
534 * Wait for pending requests to complete on a single BlockDriverState subtree,
535 * and suspend block driver's internal I/O until next request arrives.
536 *
537 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
538 * AioContext.
539 */
540 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
541 {
542 assert(qemu_in_coroutine());
543 bdrv_drained_begin(bs);
544 bdrv_drained_end(bs);
545 }
546
547 void bdrv_drain(BlockDriverState *bs)
548 {
549 bdrv_drained_begin(bs);
550 bdrv_drained_end(bs);
551 }
552
553 static void bdrv_drain_assert_idle(BlockDriverState *bs)
554 {
555 BdrvChild *child, *next;
556
557 assert(qatomic_read(&bs->in_flight) == 0);
558 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
559 bdrv_drain_assert_idle(child->bs);
560 }
561 }
562
563 unsigned int bdrv_drain_all_count = 0;
564
565 static bool bdrv_drain_all_poll(void)
566 {
567 BlockDriverState *bs = NULL;
568 bool result = false;
569
570 /* bdrv_drain_poll() can't make changes to the graph and we are holding the
571 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
572 while ((bs = bdrv_next_all_states(bs))) {
573 AioContext *aio_context = bdrv_get_aio_context(bs);
574 aio_context_acquire(aio_context);
575 result |= bdrv_drain_poll(bs, false, NULL, true);
576 aio_context_release(aio_context);
577 }
578
579 return result;
580 }
581
582 /*
583 * Wait for pending requests to complete across all BlockDriverStates
584 *
585 * This function does not flush data to disk, use bdrv_flush_all() for that
586 * after calling this function.
587 *
588 * This pauses all block jobs and disables external clients. It must
589 * be paired with bdrv_drain_all_end().
590 *
591 * NOTE: no new block jobs or BlockDriverStates can be created between
592 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
593 */
594 void bdrv_drain_all_begin(void)
595 {
596 BlockDriverState *bs = NULL;
597
598 if (qemu_in_coroutine()) {
599 bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL);
600 return;
601 }
602
603 /*
604 * bdrv queue is managed by record/replay,
605 * waiting for finishing the I/O requests may
606 * be infinite
607 */
608 if (replay_events_enabled()) {
609 return;
610 }
611
612 /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
613 * loop AioContext, so make sure we're in the main context. */
614 assert(qemu_get_current_aio_context() == qemu_get_aio_context());
615 assert(bdrv_drain_all_count < INT_MAX);
616 bdrv_drain_all_count++;
617
618 /* Quiesce all nodes, without polling in-flight requests yet. The graph
619 * cannot change during this loop. */
620 while ((bs = bdrv_next_all_states(bs))) {
621 AioContext *aio_context = bdrv_get_aio_context(bs);
622
623 aio_context_acquire(aio_context);
624 bdrv_do_drained_begin(bs, false, NULL, true, false);
625 aio_context_release(aio_context);
626 }
627
628 /* Now poll the in-flight requests */
629 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
630
631 while ((bs = bdrv_next_all_states(bs))) {
632 bdrv_drain_assert_idle(bs);
633 }
634 }
635
636 void bdrv_drain_all_end(void)
637 {
638 BlockDriverState *bs = NULL;
639 int drained_end_counter = 0;
640
641 /*
642 * bdrv queue is managed by record/replay,
643 * waiting for finishing the I/O requests may
644 * be endless
645 */
646 if (replay_events_enabled()) {
647 return;
648 }
649
650 while ((bs = bdrv_next_all_states(bs))) {
651 AioContext *aio_context = bdrv_get_aio_context(bs);
652
653 aio_context_acquire(aio_context);
654 bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
655 aio_context_release(aio_context);
656 }
657
658 assert(qemu_get_current_aio_context() == qemu_get_aio_context());
659 AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0);
660
661 assert(bdrv_drain_all_count > 0);
662 bdrv_drain_all_count--;
663 }
664
665 void bdrv_drain_all(void)
666 {
667 bdrv_drain_all_begin();
668 bdrv_drain_all_end();
669 }
670
671 /**
672 * Remove an active request from the tracked requests list
673 *
674 * This function should be called when a tracked request is completing.
675 */
676 static void tracked_request_end(BdrvTrackedRequest *req)
677 {
678 if (req->serialising) {
679 qatomic_dec(&req->bs->serialising_in_flight);
680 }
681
682 qemu_co_mutex_lock(&req->bs->reqs_lock);
683 QLIST_REMOVE(req, list);
684 qemu_co_queue_restart_all(&req->wait_queue);
685 qemu_co_mutex_unlock(&req->bs->reqs_lock);
686 }
687
688 /**
689 * Add an active request to the tracked requests list
690 */
691 static void tracked_request_begin(BdrvTrackedRequest *req,
692 BlockDriverState *bs,
693 int64_t offset,
694 uint64_t bytes,
695 enum BdrvTrackedRequestType type)
696 {
697 assert(bytes <= INT64_MAX && offset <= INT64_MAX - bytes);
698
699 *req = (BdrvTrackedRequest){
700 .bs = bs,
701 .offset = offset,
702 .bytes = bytes,
703 .type = type,
704 .co = qemu_coroutine_self(),
705 .serialising = false,
706 .overlap_offset = offset,
707 .overlap_bytes = bytes,
708 };
709
710 qemu_co_queue_init(&req->wait_queue);
711
712 qemu_co_mutex_lock(&bs->reqs_lock);
713 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
714 qemu_co_mutex_unlock(&bs->reqs_lock);
715 }
716
717 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
718 int64_t offset, uint64_t bytes)
719 {
720 /* aaaa bbbb */
721 if (offset >= req->overlap_offset + req->overlap_bytes) {
722 return false;
723 }
724 /* bbbb aaaa */
725 if (req->overlap_offset >= offset + bytes) {
726 return false;
727 }
728 return true;
729 }
730
731 static bool coroutine_fn
732 bdrv_wait_serialising_requests_locked(BlockDriverState *bs,
733 BdrvTrackedRequest *self)
734 {
735 BdrvTrackedRequest *req;
736 bool retry;
737 bool waited = false;
738
739 do {
740 retry = false;
741 QLIST_FOREACH(req, &bs->tracked_requests, list) {
742 if (req == self || (!req->serialising && !self->serialising)) {
743 continue;
744 }
745 if (tracked_request_overlaps(req, self->overlap_offset,
746 self->overlap_bytes))
747 {
748 /* Hitting this means there was a reentrant request, for
749 * example, a block driver issuing nested requests. This must
750 * never happen since it means deadlock.
751 */
752 assert(qemu_coroutine_self() != req->co);
753
754 /* If the request is already (indirectly) waiting for us, or
755 * will wait for us as soon as it wakes up, then just go on
756 * (instead of producing a deadlock in the former case). */
757 if (!req->waiting_for) {
758 self->waiting_for = req;
759 qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock);
760 self->waiting_for = NULL;
761 retry = true;
762 waited = true;
763 break;
764 }
765 }
766 }
767 } while (retry);
768 return waited;
769 }
770
771 bool bdrv_mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
772 {
773 BlockDriverState *bs = req->bs;
774 int64_t overlap_offset = req->offset & ~(align - 1);
775 uint64_t overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
776 - overlap_offset;
777 bool waited;
778
779 qemu_co_mutex_lock(&bs->reqs_lock);
780 if (!req->serialising) {
781 qatomic_inc(&req->bs->serialising_in_flight);
782 req->serialising = true;
783 }
784
785 req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
786 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
787 waited = bdrv_wait_serialising_requests_locked(bs, req);
788 qemu_co_mutex_unlock(&bs->reqs_lock);
789 return waited;
790 }
791
792 /**
793 * Return the tracked request on @bs for the current coroutine, or
794 * NULL if there is none.
795 */
796 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
797 {
798 BdrvTrackedRequest *req;
799 Coroutine *self = qemu_coroutine_self();
800
801 QLIST_FOREACH(req, &bs->tracked_requests, list) {
802 if (req->co == self) {
803 return req;
804 }
805 }
806
807 return NULL;
808 }
809
810 /**
811 * Round a region to cluster boundaries
812 */
813 void bdrv_round_to_clusters(BlockDriverState *bs,
814 int64_t offset, int64_t bytes,
815 int64_t *cluster_offset,
816 int64_t *cluster_bytes)
817 {
818 BlockDriverInfo bdi;
819
820 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
821 *cluster_offset = offset;
822 *cluster_bytes = bytes;
823 } else {
824 int64_t c = bdi.cluster_size;
825 *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
826 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
827 }
828 }
829
830 static int bdrv_get_cluster_size(BlockDriverState *bs)
831 {
832 BlockDriverInfo bdi;
833 int ret;
834
835 ret = bdrv_get_info(bs, &bdi);
836 if (ret < 0 || bdi.cluster_size == 0) {
837 return bs->bl.request_alignment;
838 } else {
839 return bdi.cluster_size;
840 }
841 }
842
843 void bdrv_inc_in_flight(BlockDriverState *bs)
844 {
845 qatomic_inc(&bs->in_flight);
846 }
847
848 void bdrv_wakeup(BlockDriverState *bs)
849 {
850 aio_wait_kick();
851 }
852
853 void bdrv_dec_in_flight(BlockDriverState *bs)
854 {
855 qatomic_dec(&bs->in_flight);
856 bdrv_wakeup(bs);
857 }
858
859 static bool coroutine_fn bdrv_wait_serialising_requests(BdrvTrackedRequest *self)
860 {
861 BlockDriverState *bs = self->bs;
862 bool waited = false;
863
864 if (!qatomic_read(&bs->serialising_in_flight)) {
865 return false;
866 }
867
868 qemu_co_mutex_lock(&bs->reqs_lock);
869 waited = bdrv_wait_serialising_requests_locked(bs, self);
870 qemu_co_mutex_unlock(&bs->reqs_lock);
871
872 return waited;
873 }
874
875 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
876 size_t size)
877 {
878 if (size > BDRV_REQUEST_MAX_BYTES) {
879 return -EIO;
880 }
881
882 if (!bdrv_is_inserted(bs)) {
883 return -ENOMEDIUM;
884 }
885
886 if (offset < 0) {
887 return -EIO;
888 }
889
890 return 0;
891 }
892
893 int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
894 int bytes, BdrvRequestFlags flags)
895 {
896 return bdrv_pwritev(child, offset, bytes, NULL,
897 BDRV_REQ_ZERO_WRITE | flags);
898 }
899
900 /*
901 * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
902 * The operation is sped up by checking the block status and only writing
903 * zeroes to the device if they currently do not return zeroes. Optional
904 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
905 * BDRV_REQ_FUA).
906 *
907 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
908 */
909 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
910 {
911 int ret;
912 int64_t target_size, bytes, offset = 0;
913 BlockDriverState *bs = child->bs;
914
915 target_size = bdrv_getlength(bs);
916 if (target_size < 0) {
917 return target_size;
918 }
919
920 for (;;) {
921 bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
922 if (bytes <= 0) {
923 return 0;
924 }
925 ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
926 if (ret < 0) {
927 return ret;
928 }
929 if (ret & BDRV_BLOCK_ZERO) {
930 offset += bytes;
931 continue;
932 }
933 ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
934 if (ret < 0) {
935 return ret;
936 }
937 offset += bytes;
938 }
939 }
940
941 /* See bdrv_pwrite() for the return codes */
942 int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes)
943 {
944 int ret;
945 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
946
947 if (bytes < 0) {
948 return -EINVAL;
949 }
950
951 ret = bdrv_preadv(child, offset, bytes, &qiov, 0);
952
953 return ret < 0 ? ret : bytes;
954 }
955
956 /* Return no. of bytes on success or < 0 on error. Important errors are:
957 -EIO generic I/O error (may happen for all errors)
958 -ENOMEDIUM No media inserted.
959 -EINVAL Invalid offset or number of bytes
960 -EACCES Trying to write a read-only device
961 */
962 int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes)
963 {
964 int ret;
965 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
966
967 if (bytes < 0) {
968 return -EINVAL;
969 }
970
971 ret = bdrv_pwritev(child, offset, bytes, &qiov, 0);
972
973 return ret < 0 ? ret : bytes;
974 }
975
976 /*
977 * Writes to the file and ensures that no writes are reordered across this
978 * request (acts as a barrier)
979 *
980 * Returns 0 on success, -errno in error cases.
981 */
982 int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
983 const void *buf, int count)
984 {
985 int ret;
986
987 ret = bdrv_pwrite(child, offset, buf, count);
988 if (ret < 0) {
989 return ret;
990 }
991
992 ret = bdrv_flush(child->bs);
993 if (ret < 0) {
994 return ret;
995 }
996
997 return 0;
998 }
999
1000 typedef struct CoroutineIOCompletion {
1001 Coroutine *coroutine;
1002 int ret;
1003 } CoroutineIOCompletion;
1004
1005 static void bdrv_co_io_em_complete(void *opaque, int ret)
1006 {
1007 CoroutineIOCompletion *co = opaque;
1008
1009 co->ret = ret;
1010 aio_co_wake(co->coroutine);
1011 }
1012
1013 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
1014 uint64_t offset, uint64_t bytes,
1015 QEMUIOVector *qiov,
1016 size_t qiov_offset, int flags)
1017 {
1018 BlockDriver *drv = bs->drv;
1019 int64_t sector_num;
1020 unsigned int nb_sectors;
1021 QEMUIOVector local_qiov;
1022 int ret;
1023
1024 assert(!(flags & ~BDRV_REQ_MASK));
1025 assert(!(flags & BDRV_REQ_NO_FALLBACK));
1026
1027 if (!drv) {
1028 return -ENOMEDIUM;
1029 }
1030
1031 if (drv->bdrv_co_preadv_part) {
1032 return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
1033 flags);
1034 }
1035
1036 if (qiov_offset > 0 || bytes != qiov->size) {
1037 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1038 qiov = &local_qiov;
1039 }
1040
1041 if (drv->bdrv_co_preadv) {
1042 ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
1043 goto out;
1044 }
1045
1046 if (drv->bdrv_aio_preadv) {
1047 BlockAIOCB *acb;
1048 CoroutineIOCompletion co = {
1049 .coroutine = qemu_coroutine_self(),
1050 };
1051
1052 acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
1053 bdrv_co_io_em_complete, &co);
1054 if (acb == NULL) {
1055 ret = -EIO;
1056 goto out;
1057 } else {
1058 qemu_coroutine_yield();
1059 ret = co.ret;
1060 goto out;
1061 }
1062 }
1063
1064 sector_num = offset >> BDRV_SECTOR_BITS;
1065 nb_sectors = bytes >> BDRV_SECTOR_BITS;
1066
1067 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1068 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1069 assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1070 assert(drv->bdrv_co_readv);
1071
1072 ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
1073
1074 out:
1075 if (qiov == &local_qiov) {
1076 qemu_iovec_destroy(&local_qiov);
1077 }
1078
1079 return ret;
1080 }
1081
1082 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
1083 uint64_t offset, uint64_t bytes,
1084 QEMUIOVector *qiov,
1085 size_t qiov_offset, int flags)
1086 {
1087 BlockDriver *drv = bs->drv;
1088 int64_t sector_num;
1089 unsigned int nb_sectors;
1090 QEMUIOVector local_qiov;
1091 int ret;
1092
1093 assert(!(flags & ~BDRV_REQ_MASK));
1094 assert(!(flags & BDRV_REQ_NO_FALLBACK));
1095
1096 if (!drv) {
1097 return -ENOMEDIUM;
1098 }
1099
1100 if (drv->bdrv_co_pwritev_part) {
1101 ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
1102 flags & bs->supported_write_flags);
1103 flags &= ~bs->supported_write_flags;
1104 goto emulate_flags;
1105 }
1106
1107 if (qiov_offset > 0 || bytes != qiov->size) {
1108 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1109 qiov = &local_qiov;
1110 }
1111
1112 if (drv->bdrv_co_pwritev) {
1113 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
1114 flags & bs->supported_write_flags);
1115 flags &= ~bs->supported_write_flags;
1116 goto emulate_flags;
1117 }
1118
1119 if (drv->bdrv_aio_pwritev) {
1120 BlockAIOCB *acb;
1121 CoroutineIOCompletion co = {
1122 .coroutine = qemu_coroutine_self(),
1123 };
1124
1125 acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov,
1126 flags & bs->supported_write_flags,
1127 bdrv_co_io_em_complete, &co);
1128 flags &= ~bs->supported_write_flags;
1129 if (acb == NULL) {
1130 ret = -EIO;
1131 } else {
1132 qemu_coroutine_yield();
1133 ret = co.ret;
1134 }
1135 goto emulate_flags;
1136 }
1137
1138 sector_num = offset >> BDRV_SECTOR_BITS;
1139 nb_sectors = bytes >> BDRV_SECTOR_BITS;
1140
1141 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1142 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1143 assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1144
1145 assert(drv->bdrv_co_writev);
1146 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov,
1147 flags & bs->supported_write_flags);
1148 flags &= ~bs->supported_write_flags;
1149
1150 emulate_flags:
1151 if (ret == 0 && (flags & BDRV_REQ_FUA)) {
1152 ret = bdrv_co_flush(bs);
1153 }
1154
1155 if (qiov == &local_qiov) {
1156 qemu_iovec_destroy(&local_qiov);
1157 }
1158
1159 return ret;
1160 }
1161
1162 static int coroutine_fn
1163 bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset,
1164 uint64_t bytes, QEMUIOVector *qiov,
1165 size_t qiov_offset)
1166 {
1167 BlockDriver *drv = bs->drv;
1168 QEMUIOVector local_qiov;
1169 int ret;
1170
1171 if (!drv) {
1172 return -ENOMEDIUM;
1173 }
1174
1175 if (!block_driver_can_compress(drv)) {
1176 return -ENOTSUP;
1177 }
1178
1179 if (drv->bdrv_co_pwritev_compressed_part) {
1180 return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
1181 qiov, qiov_offset);
1182 }
1183
1184 if (qiov_offset == 0) {
1185 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
1186 }
1187
1188 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1189 ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
1190 qemu_iovec_destroy(&local_qiov);
1191
1192 return ret;
1193 }
1194
1195 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
1196 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1197 size_t qiov_offset, int flags)
1198 {
1199 BlockDriverState *bs = child->bs;
1200
1201 /* Perform I/O through a temporary buffer so that users who scribble over
1202 * their read buffer while the operation is in progress do not end up
1203 * modifying the image file. This is critical for zero-copy guest I/O
1204 * where anything might happen inside guest memory.
1205 */
1206 void *bounce_buffer = NULL;
1207
1208 BlockDriver *drv = bs->drv;
1209 int64_t cluster_offset;
1210 int64_t cluster_bytes;
1211 size_t skip_bytes;
1212 int ret;
1213 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1214 BDRV_REQUEST_MAX_BYTES);
1215 unsigned int progress = 0;
1216 bool skip_write;
1217
1218 if (!drv) {
1219 return -ENOMEDIUM;
1220 }
1221
1222 /*
1223 * Do not write anything when the BDS is inactive. That is not
1224 * allowed, and it would not help.
1225 */
1226 skip_write = (bs->open_flags & BDRV_O_INACTIVE);
1227
1228 /* FIXME We cannot require callers to have write permissions when all they
1229 * are doing is a read request. If we did things right, write permissions
1230 * would be obtained anyway, but internally by the copy-on-read code. As
1231 * long as it is implemented here rather than in a separate filter driver,
1232 * the copy-on-read code doesn't have its own BdrvChild, however, for which
1233 * it could request permissions. Therefore we have to bypass the permission
1234 * system for the moment. */
1235 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1236
1237 /* Cover entire cluster so no additional backing file I/O is required when
1238 * allocating cluster in the image file. Note that this value may exceed
1239 * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1240 * is one reason we loop rather than doing it all at once.
1241 */
1242 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
1243 skip_bytes = offset - cluster_offset;
1244
1245 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1246 cluster_offset, cluster_bytes);
1247
1248 while (cluster_bytes) {
1249 int64_t pnum;
1250
1251 if (skip_write) {
1252 ret = 1; /* "already allocated", so nothing will be copied */
1253 pnum = MIN(cluster_bytes, max_transfer);
1254 } else {
1255 ret = bdrv_is_allocated(bs, cluster_offset,
1256 MIN(cluster_bytes, max_transfer), &pnum);
1257 if (ret < 0) {
1258 /*
1259 * Safe to treat errors in querying allocation as if
1260 * unallocated; we'll probably fail again soon on the
1261 * read, but at least that will set a decent errno.
1262 */
1263 pnum = MIN(cluster_bytes, max_transfer);
1264 }
1265
1266 /* Stop at EOF if the image ends in the middle of the cluster */
1267 if (ret == 0 && pnum == 0) {
1268 assert(progress >= bytes);
1269 break;
1270 }
1271
1272 assert(skip_bytes < pnum);
1273 }
1274
1275 if (ret <= 0) {
1276 QEMUIOVector local_qiov;
1277
1278 /* Must copy-on-read; use the bounce buffer */
1279 pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
1280 if (!bounce_buffer) {
1281 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum);
1282 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
1283 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
1284
1285 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
1286 if (!bounce_buffer) {
1287 ret = -ENOMEM;
1288 goto err;
1289 }
1290 }
1291 qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);
1292
1293 ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
1294 &local_qiov, 0, 0);
1295 if (ret < 0) {
1296 goto err;
1297 }
1298
1299 bdrv_debug_event(bs, BLKDBG_COR_WRITE);
1300 if (drv->bdrv_co_pwrite_zeroes &&
1301 buffer_is_zero(bounce_buffer, pnum)) {
1302 /* FIXME: Should we (perhaps conditionally) be setting
1303 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1304 * that still correctly reads as zero? */
1305 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum,
1306 BDRV_REQ_WRITE_UNCHANGED);
1307 } else {
1308 /* This does not change the data on the disk, it is not
1309 * necessary to flush even in cache=writethrough mode.
1310 */
1311 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
1312 &local_qiov, 0,
1313 BDRV_REQ_WRITE_UNCHANGED);
1314 }
1315
1316 if (ret < 0) {
1317 /* It might be okay to ignore write errors for guest
1318 * requests. If this is a deliberate copy-on-read
1319 * then we don't want to ignore the error. Simply
1320 * report it in all cases.
1321 */
1322 goto err;
1323 }
1324
1325 if (!(flags & BDRV_REQ_PREFETCH)) {
1326 qemu_iovec_from_buf(qiov, qiov_offset + progress,
1327 bounce_buffer + skip_bytes,
1328 MIN(pnum - skip_bytes, bytes - progress));
1329 }
1330 } else if (!(flags & BDRV_REQ_PREFETCH)) {
1331 /* Read directly into the destination */
1332 ret = bdrv_driver_preadv(bs, offset + progress,
1333 MIN(pnum - skip_bytes, bytes - progress),
1334 qiov, qiov_offset + progress, 0);
1335 if (ret < 0) {
1336 goto err;
1337 }
1338 }
1339
1340 cluster_offset += pnum;
1341 cluster_bytes -= pnum;
1342 progress += pnum - skip_bytes;
1343 skip_bytes = 0;
1344 }
1345 ret = 0;
1346
1347 err:
1348 qemu_vfree(bounce_buffer);
1349 return ret;
1350 }
1351
1352 /*
1353 * Forwards an already correctly aligned request to the BlockDriver. This
1354 * handles copy on read, zeroing after EOF, and fragmentation of large
1355 * reads; any other features must be implemented by the caller.
1356 */
1357 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
1358 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1359 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
1360 {
1361 BlockDriverState *bs = child->bs;
1362 int64_t total_bytes, max_bytes;
1363 int ret = 0;
1364 uint64_t bytes_remaining = bytes;
1365 int max_transfer;
1366
1367 assert(is_power_of_2(align));
1368 assert((offset & (align - 1)) == 0);
1369 assert((bytes & (align - 1)) == 0);
1370 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1371 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1372 align);
1373
1374 /* TODO: We would need a per-BDS .supported_read_flags and
1375 * potential fallback support, if we ever implement any read flags
1376 * to pass through to drivers. For now, there aren't any
1377 * passthrough flags. */
1378 assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH)));
1379
1380 /* Handle Copy on Read and associated serialisation */
1381 if (flags & BDRV_REQ_COPY_ON_READ) {
1382 /* If we touch the same cluster it counts as an overlap. This
1383 * guarantees that allocating writes will be serialized and not race
1384 * with each other for the same cluster. For example, in copy-on-read
1385 * it ensures that the CoR read and write operations are atomic and
1386 * guest writes cannot interleave between them. */
1387 bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs));
1388 } else {
1389 bdrv_wait_serialising_requests(req);
1390 }
1391
1392 if (flags & BDRV_REQ_COPY_ON_READ) {
1393 int64_t pnum;
1394
1395 ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
1396 if (ret < 0) {
1397 goto out;
1398 }
1399
1400 if (!ret || pnum != bytes) {
1401 ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
1402 qiov, qiov_offset, flags);
1403 goto out;
1404 } else if (flags & BDRV_REQ_PREFETCH) {
1405 goto out;
1406 }
1407 }
1408
1409 /* Forward the request to the BlockDriver, possibly fragmenting it */
1410 total_bytes = bdrv_getlength(bs);
1411 if (total_bytes < 0) {
1412 ret = total_bytes;
1413 goto out;
1414 }
1415
1416 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
1417 if (bytes <= max_bytes && bytes <= max_transfer) {
1418 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, 0);
1419 goto out;
1420 }
1421
1422 while (bytes_remaining) {
1423 int num;
1424
1425 if (max_bytes) {
1426 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
1427 assert(num);
1428
1429 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1430 num, qiov,
1431 qiov_offset + bytes - bytes_remaining, 0);
1432 max_bytes -= num;
1433 } else {
1434 num = bytes_remaining;
1435 ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining,
1436 0, bytes_remaining);
1437 }
1438 if (ret < 0) {
1439 goto out;
1440 }
1441 bytes_remaining -= num;
1442 }
1443
1444 out:
1445 return ret < 0 ? ret : 0;
1446 }
1447
1448 /*
1449 * Request padding
1450 *
1451 * |<---- align ----->| |<----- align ---->|
1452 * |<- head ->|<------------- bytes ------------->|<-- tail -->|
1453 * | | | | | |
1454 * -*----------$-------*-------- ... --------*-----$------------*---
1455 * | | | | | |
1456 * | offset | | end |
1457 * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end)
1458 * [buf ... ) [tail_buf )
1459 *
1460 * @buf is an aligned allocation needed to store @head and @tail paddings. @head
1461 * is placed at the beginning of @buf and @tail at the @end.
1462 *
1463 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
1464 * around tail, if tail exists.
1465 *
1466 * @merge_reads is true for small requests,
1467 * if @buf_len == @head + bytes + @tail. In this case it is possible that both
1468 * head and tail exist but @buf_len == align and @tail_buf == @buf.
1469 */
1470 typedef struct BdrvRequestPadding {
1471 uint8_t *buf;
1472 size_t buf_len;
1473 uint8_t *tail_buf;
1474 size_t head;
1475 size_t tail;
1476 bool merge_reads;
1477 QEMUIOVector local_qiov;
1478 } BdrvRequestPadding;
1479
1480 static bool bdrv_init_padding(BlockDriverState *bs,
1481 int64_t offset, int64_t bytes,
1482 BdrvRequestPadding *pad)
1483 {
1484 uint64_t align = bs->bl.request_alignment;
1485 size_t sum;
1486
1487 memset(pad, 0, sizeof(*pad));
1488
1489 pad->head = offset & (align - 1);
1490 pad->tail = ((offset + bytes) & (align - 1));
1491 if (pad->tail) {
1492 pad->tail = align - pad->tail;
1493 }
1494
1495 if (!pad->head && !pad->tail) {
1496 return false;
1497 }
1498
1499 assert(bytes); /* Nothing good in aligning zero-length requests */
1500
1501 sum = pad->head + bytes + pad->tail;
1502 pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
1503 pad->buf = qemu_blockalign(bs, pad->buf_len);
1504 pad->merge_reads = sum == pad->buf_len;
1505 if (pad->tail) {
1506 pad->tail_buf = pad->buf + pad->buf_len - align;
1507 }
1508
1509 return true;
1510 }
1511
1512 static int bdrv_padding_rmw_read(BdrvChild *child,
1513 BdrvTrackedRequest *req,
1514 BdrvRequestPadding *pad,
1515 bool zero_middle)
1516 {
1517 QEMUIOVector local_qiov;
1518 BlockDriverState *bs = child->bs;
1519 uint64_t align = bs->bl.request_alignment;
1520 int ret;
1521
1522 assert(req->serialising && pad->buf);
1523
1524 if (pad->head || pad->merge_reads) {
1525 uint64_t bytes = pad->merge_reads ? pad->buf_len : align;
1526
1527 qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
1528
1529 if (pad->head) {
1530 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1531 }
1532 if (pad->merge_reads && pad->tail) {
1533 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1534 }
1535 ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
1536 align, &local_qiov, 0, 0);
1537 if (ret < 0) {
1538 return ret;
1539 }
1540 if (pad->head) {
1541 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1542 }
1543 if (pad->merge_reads && pad->tail) {
1544 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1545 }
1546
1547 if (pad->merge_reads) {
1548 goto zero_mem;
1549 }
1550 }
1551
1552 if (pad->tail) {
1553 qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
1554
1555 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1556 ret = bdrv_aligned_preadv(
1557 child, req,
1558 req->overlap_offset + req->overlap_bytes - align,
1559 align, align, &local_qiov, 0, 0);
1560 if (ret < 0) {
1561 return ret;
1562 }
1563 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1564 }
1565
1566 zero_mem:
1567 if (zero_middle) {
1568 memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
1569 }
1570
1571 return 0;
1572 }
1573
1574 static void bdrv_padding_destroy(BdrvRequestPadding *pad)
1575 {
1576 if (pad->buf) {
1577 qemu_vfree(pad->buf);
1578 qemu_iovec_destroy(&pad->local_qiov);
1579 }
1580 }
1581
1582 /*
1583 * bdrv_pad_request
1584 *
1585 * Exchange request parameters with padded request if needed. Don't include RMW
1586 * read of padding, bdrv_padding_rmw_read() should be called separately if
1587 * needed.
1588 *
1589 * All parameters except @bs are in-out: they represent original request at
1590 * function call and padded (if padding needed) at function finish.
1591 *
1592 * Function always succeeds.
1593 */
1594 static bool bdrv_pad_request(BlockDriverState *bs,
1595 QEMUIOVector **qiov, size_t *qiov_offset,
1596 int64_t *offset, unsigned int *bytes,
1597 BdrvRequestPadding *pad)
1598 {
1599 if (!bdrv_init_padding(bs, *offset, *bytes, pad)) {
1600 return false;
1601 }
1602
1603 qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
1604 *qiov, *qiov_offset, *bytes,
1605 pad->buf + pad->buf_len - pad->tail, pad->tail);
1606 *bytes += pad->head + pad->tail;
1607 *offset -= pad->head;
1608 *qiov = &pad->local_qiov;
1609 *qiov_offset = 0;
1610
1611 return true;
1612 }
1613
1614 int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1615 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1616 BdrvRequestFlags flags)
1617 {
1618 return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
1619 }
1620
1621 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
1622 int64_t offset, unsigned int bytes,
1623 QEMUIOVector *qiov, size_t qiov_offset,
1624 BdrvRequestFlags flags)
1625 {
1626 BlockDriverState *bs = child->bs;
1627 BdrvTrackedRequest req;
1628 BdrvRequestPadding pad;
1629 int ret;
1630
1631 trace_bdrv_co_preadv(bs, offset, bytes, flags);
1632
1633 ret = bdrv_check_byte_request(bs, offset, bytes);
1634 if (ret < 0) {
1635 return ret;
1636 }
1637
1638 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
1639 /*
1640 * Aligning zero request is nonsense. Even if driver has special meaning
1641 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
1642 * it to driver due to request_alignment.
1643 *
1644 * Still, no reason to return an error if someone do unaligned
1645 * zero-length read occasionally.
1646 */
1647 return 0;
1648 }
1649
1650 bdrv_inc_in_flight(bs);
1651
1652 /* Don't do copy-on-read if we read data before write operation */
1653 if (qatomic_read(&bs->copy_on_read)) {
1654 flags |= BDRV_REQ_COPY_ON_READ;
1655 }
1656
1657 bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad);
1658
1659 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1660 ret = bdrv_aligned_preadv(child, &req, offset, bytes,
1661 bs->bl.request_alignment,
1662 qiov, qiov_offset, flags);
1663 tracked_request_end(&req);
1664 bdrv_dec_in_flight(bs);
1665
1666 bdrv_padding_destroy(&pad);
1667
1668 return ret;
1669 }
1670
1671 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1672 int64_t offset, int bytes, BdrvRequestFlags flags)
1673 {
1674 BlockDriver *drv = bs->drv;
1675 QEMUIOVector qiov;
1676 void *buf = NULL;
1677 int ret = 0;
1678 bool need_flush = false;
1679 int head = 0;
1680 int tail = 0;
1681
1682 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX);
1683 int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1684 bs->bl.request_alignment);
1685 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1686
1687 if (!drv) {
1688 return -ENOMEDIUM;
1689 }
1690
1691 if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {
1692 return -ENOTSUP;
1693 }
1694
1695 assert(alignment % bs->bl.request_alignment == 0);
1696 head = offset % alignment;
1697 tail = (offset + bytes) % alignment;
1698 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1699 assert(max_write_zeroes >= bs->bl.request_alignment);
1700
1701 while (bytes > 0 && !ret) {
1702 int num = bytes;
1703
1704 /* Align request. Block drivers can expect the "bulk" of the request
1705 * to be aligned, and that unaligned requests do not cross cluster
1706 * boundaries.
1707 */
1708 if (head) {
1709 /* Make a small request up to the first aligned sector. For
1710 * convenience, limit this request to max_transfer even if
1711 * we don't need to fall back to writes. */
1712 num = MIN(MIN(bytes, max_transfer), alignment - head);
1713 head = (head + num) % alignment;
1714 assert(num < max_write_zeroes);
1715 } else if (tail && num > alignment) {
1716 /* Shorten the request to the last aligned sector. */
1717 num -= tail;
1718 }
1719
1720 /* limit request size */
1721 if (num > max_write_zeroes) {
1722 num = max_write_zeroes;
1723 }
1724
1725 ret = -ENOTSUP;
1726 /* First try the efficient write zeroes operation */
1727 if (drv->bdrv_co_pwrite_zeroes) {
1728 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1729 flags & bs->supported_zero_flags);
1730 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1731 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1732 need_flush = true;
1733 }
1734 } else {
1735 assert(!bs->supported_zero_flags);
1736 }
1737
1738 if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {
1739 /* Fall back to bounce buffer if write zeroes is unsupported */
1740 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1741
1742 if ((flags & BDRV_REQ_FUA) &&
1743 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1744 /* No need for bdrv_driver_pwrite() to do a fallback
1745 * flush on each chunk; use just one at the end */
1746 write_flags &= ~BDRV_REQ_FUA;
1747 need_flush = true;
1748 }
1749 num = MIN(num, max_transfer);
1750 if (buf == NULL) {
1751 buf = qemu_try_blockalign0(bs, num);
1752 if (buf == NULL) {
1753 ret = -ENOMEM;
1754 goto fail;
1755 }
1756 }
1757 qemu_iovec_init_buf(&qiov, buf, num);
1758
1759 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
1760
1761 /* Keep bounce buffer around if it is big enough for all
1762 * all future requests.
1763 */
1764 if (num < max_transfer) {
1765 qemu_vfree(buf);
1766 buf = NULL;
1767 }
1768 }
1769
1770 offset += num;
1771 bytes -= num;
1772 }
1773
1774 fail:
1775 if (ret == 0 && need_flush) {
1776 ret = bdrv_co_flush(bs);
1777 }
1778 qemu_vfree(buf);
1779 return ret;
1780 }
1781
1782 static inline int coroutine_fn
1783 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, uint64_t bytes,
1784 BdrvTrackedRequest *req, int flags)
1785 {
1786 BlockDriverState *bs = child->bs;
1787 bool waited;
1788 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1789
1790 if (bs->read_only) {
1791 return -EPERM;
1792 }
1793
1794 assert(!(bs->open_flags & BDRV_O_INACTIVE));
1795 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1796 assert(!(flags & ~BDRV_REQ_MASK));
1797
1798 if (flags & BDRV_REQ_SERIALISING) {
1799 waited = bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs));
1800 /*
1801 * For a misaligned request we should have already waited earlier,
1802 * because we come after bdrv_padding_rmw_read which must be called
1803 * with the request already marked as serialising.
1804 */
1805 assert(!waited ||
1806 (req->offset == req->overlap_offset &&
1807 req->bytes == req->overlap_bytes));
1808 } else {
1809 bdrv_wait_serialising_requests(req);
1810 }
1811
1812 assert(req->overlap_offset <= offset);
1813 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1814 assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE);
1815
1816 switch (req->type) {
1817 case BDRV_TRACKED_WRITE:
1818 case BDRV_TRACKED_DISCARD:
1819 if (flags & BDRV_REQ_WRITE_UNCHANGED) {
1820 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1821 } else {
1822 assert(child->perm & BLK_PERM_WRITE);
1823 }
1824 return notifier_with_return_list_notify(&bs->before_write_notifiers,
1825 req);
1826 case BDRV_TRACKED_TRUNCATE:
1827 assert(child->perm & BLK_PERM_RESIZE);
1828 return 0;
1829 default:
1830 abort();
1831 }
1832 }
1833
1834 static inline void coroutine_fn
1835 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, uint64_t bytes,
1836 BdrvTrackedRequest *req, int ret)
1837 {
1838 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1839 BlockDriverState *bs = child->bs;
1840
1841 qatomic_inc(&bs->write_gen);
1842
1843 /*
1844 * Discard cannot extend the image, but in error handling cases, such as
1845 * when reverting a qcow2 cluster allocation, the discarded range can pass
1846 * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD
1847 * here. Instead, just skip it, since semantically a discard request
1848 * beyond EOF cannot expand the image anyway.
1849 */
1850 if (ret == 0 &&
1851 (req->type == BDRV_TRACKED_TRUNCATE ||
1852 end_sector > bs->total_sectors) &&
1853 req->type != BDRV_TRACKED_DISCARD) {
1854 bs->total_sectors = end_sector;
1855 bdrv_parent_cb_resize(bs);
1856 bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS);
1857 }
1858 if (req->bytes) {
1859 switch (req->type) {
1860 case BDRV_TRACKED_WRITE:
1861 stat64_max(&bs->wr_highest_offset, offset + bytes);
1862 /* fall through, to set dirty bits */
1863 case BDRV_TRACKED_DISCARD:
1864 bdrv_set_dirty(bs, offset, bytes);
1865 break;
1866 default:
1867 break;
1868 }
1869 }
1870 }
1871
1872 /*
1873 * Forwards an already correctly aligned write request to the BlockDriver,
1874 * after possibly fragmenting it.
1875 */
1876 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
1877 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1878 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
1879 {
1880 BlockDriverState *bs = child->bs;
1881 BlockDriver *drv = bs->drv;
1882 int ret;
1883
1884 uint64_t bytes_remaining = bytes;
1885 int max_transfer;
1886
1887 if (!drv) {
1888 return -ENOMEDIUM;
1889 }
1890
1891 if (bdrv_has_readonly_bitmaps(bs)) {
1892 return -EPERM;
1893 }
1894
1895 assert(is_power_of_2(align));
1896 assert((offset & (align - 1)) == 0);
1897 assert((bytes & (align - 1)) == 0);
1898 assert(!qiov || qiov_offset + bytes <= qiov->size);
1899 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1900 align);
1901
1902 ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags);
1903
1904 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1905 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
1906 qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
1907 flags |= BDRV_REQ_ZERO_WRITE;
1908 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1909 flags |= BDRV_REQ_MAY_UNMAP;
1910 }
1911 }
1912
1913 if (ret < 0) {
1914 /* Do nothing, write notifier decided to fail this request */
1915 } else if (flags & BDRV_REQ_ZERO_WRITE) {
1916 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1917 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
1918 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
1919 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
1920 qiov, qiov_offset);
1921 } else if (bytes <= max_transfer) {
1922 bdrv_debug_event(bs, BLKDBG_PWRITEV);
1923 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
1924 } else {
1925 bdrv_debug_event(bs, BLKDBG_PWRITEV);
1926 while (bytes_remaining) {
1927 int num = MIN(bytes_remaining, max_transfer);
1928 int local_flags = flags;
1929
1930 assert(num);
1931 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
1932 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1933 /* If FUA is going to be emulated by flush, we only
1934 * need to flush on the last iteration */
1935 local_flags &= ~BDRV_REQ_FUA;
1936 }
1937
1938 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
1939 num, qiov,
1940 qiov_offset + bytes - bytes_remaining,
1941 local_flags);
1942 if (ret < 0) {
1943 break;
1944 }
1945 bytes_remaining -= num;
1946 }
1947 }
1948 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1949
1950 if (ret >= 0) {
1951 ret = 0;
1952 }
1953 bdrv_co_write_req_finish(child, offset, bytes, req, ret);
1954
1955 return ret;
1956 }
1957
1958 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
1959 int64_t offset,
1960 unsigned int bytes,
1961 BdrvRequestFlags flags,
1962 BdrvTrackedRequest *req)
1963 {
1964 BlockDriverState *bs = child->bs;
1965 QEMUIOVector local_qiov;
1966 uint64_t align = bs->bl.request_alignment;
1967 int ret = 0;
1968 bool padding;
1969 BdrvRequestPadding pad;
1970
1971 padding = bdrv_init_padding(bs, offset, bytes, &pad);
1972 if (padding) {
1973 bdrv_mark_request_serialising(req, align);
1974
1975 bdrv_padding_rmw_read(child, req, &pad, true);
1976
1977 if (pad.head || pad.merge_reads) {
1978 int64_t aligned_offset = offset & ~(align - 1);
1979 int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
1980
1981 qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
1982 ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
1983 align, &local_qiov, 0,
1984 flags & ~BDRV_REQ_ZERO_WRITE);
1985 if (ret < 0 || pad.merge_reads) {
1986 /* Error or all work is done */
1987 goto out;
1988 }
1989 offset += write_bytes - pad.head;
1990 bytes -= write_bytes - pad.head;
1991 }
1992 }
1993
1994 assert(!bytes || (offset & (align - 1)) == 0);
1995 if (bytes >= align) {
1996 /* Write the aligned part in the middle. */
1997 uint64_t aligned_bytes = bytes & ~(align - 1);
1998 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
1999 NULL, 0, flags);
2000 if (ret < 0) {
2001 goto out;
2002 }
2003 bytes -= aligned_bytes;
2004 offset += aligned_bytes;
2005 }
2006
2007 assert(!bytes || (offset & (align - 1)) == 0);
2008 if (bytes) {
2009 assert(align == pad.tail + bytes);
2010
2011 qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
2012 ret = bdrv_aligned_pwritev(child, req, offset, align, align,
2013 &local_qiov, 0,
2014 flags & ~BDRV_REQ_ZERO_WRITE);
2015 }
2016
2017 out:
2018 bdrv_padding_destroy(&pad);
2019
2020 return ret;
2021 }
2022
2023 /*
2024 * Handle a write request in coroutine context
2025 */
2026 int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
2027 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
2028 BdrvRequestFlags flags)
2029 {
2030 return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
2031 }
2032
2033 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
2034 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, size_t qiov_offset,
2035 BdrvRequestFlags flags)
2036 {
2037 BlockDriverState *bs = child->bs;
2038 BdrvTrackedRequest req;
2039 uint64_t align = bs->bl.request_alignment;
2040 BdrvRequestPadding pad;
2041 int ret;
2042
2043 trace_bdrv_co_pwritev(child->bs, offset, bytes, flags);
2044
2045 if (!bs->drv) {
2046 return -ENOMEDIUM;
2047 }
2048
2049 ret = bdrv_check_byte_request(bs, offset, bytes);
2050 if (ret < 0) {
2051 return ret;
2052 }
2053
2054 /* If the request is misaligned then we can't make it efficient */
2055 if ((flags & BDRV_REQ_NO_FALLBACK) &&
2056 !QEMU_IS_ALIGNED(offset | bytes, align))
2057 {
2058 return -ENOTSUP;
2059 }
2060
2061 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
2062 /*
2063 * Aligning zero request is nonsense. Even if driver has special meaning
2064 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
2065 * it to driver due to request_alignment.
2066 *
2067 * Still, no reason to return an error if someone do unaligned
2068 * zero-length write occasionally.
2069 */
2070 return 0;
2071 }
2072
2073 bdrv_inc_in_flight(bs);
2074 /*
2075 * Align write if necessary by performing a read-modify-write cycle.
2076 * Pad qiov with the read parts and be sure to have a tracked request not
2077 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
2078 */
2079 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
2080
2081 if (flags & BDRV_REQ_ZERO_WRITE) {
2082 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
2083 goto out;
2084 }
2085
2086 if (bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad)) {
2087 bdrv_mark_request_serialising(&req, align);
2088 bdrv_padding_rmw_read(child, &req, &pad, false);
2089 }
2090
2091 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
2092 qiov, qiov_offset, flags);
2093
2094 bdrv_padding_destroy(&pad);
2095
2096 out:
2097 tracked_request_end(&req);
2098 bdrv_dec_in_flight(bs);
2099
2100 return ret;
2101 }
2102
2103 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
2104 int bytes, BdrvRequestFlags flags)
2105 {
2106 trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
2107
2108 if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
2109 flags &= ~BDRV_REQ_MAY_UNMAP;
2110 }
2111
2112 return bdrv_co_pwritev(child, offset, bytes, NULL,
2113 BDRV_REQ_ZERO_WRITE | flags);
2114 }
2115
2116 /*
2117 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
2118 */
2119 int bdrv_flush_all(void)
2120 {
2121 BdrvNextIterator it;
2122 BlockDriverState *bs = NULL;
2123 int result = 0;
2124
2125 /*
2126 * bdrv queue is managed by record/replay,
2127 * creating new flush request for stopping
2128 * the VM may break the determinism
2129 */
2130 if (replay_events_enabled()) {
2131 return result;
2132 }
2133
2134 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
2135 AioContext *aio_context = bdrv_get_aio_context(bs);
2136 int ret;
2137
2138 aio_context_acquire(aio_context);
2139 ret = bdrv_flush(bs);
2140 if (ret < 0 && !result) {
2141 result = ret;
2142 }
2143 aio_context_release(aio_context);
2144 }
2145
2146 return result;
2147 }
2148
2149 /*
2150 * Returns the allocation status of the specified sectors.
2151 * Drivers not implementing the functionality are assumed to not support
2152 * backing files, hence all their sectors are reported as allocated.
2153 *
2154 * If 'want_zero' is true, the caller is querying for mapping
2155 * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and
2156 * _ZERO where possible; otherwise, the result favors larger 'pnum',
2157 * with a focus on accurate BDRV_BLOCK_ALLOCATED.
2158 *
2159 * If 'offset' is beyond the end of the disk image the return value is
2160 * BDRV_BLOCK_EOF and 'pnum' is set to 0.
2161 *
2162 * 'bytes' is the max value 'pnum' should be set to. If bytes goes
2163 * beyond the end of the disk image it will be clamped; if 'pnum' is set to
2164 * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
2165 *
2166 * 'pnum' is set to the number of bytes (including and immediately
2167 * following the specified offset) that are easily known to be in the
2168 * same allocated/unallocated state. Note that a second call starting
2169 * at the original offset plus returned pnum may have the same status.
2170 * The returned value is non-zero on success except at end-of-file.
2171 *
2172 * Returns negative errno on failure. Otherwise, if the
2173 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
2174 * set to the host mapping and BDS corresponding to the guest offset.
2175 */
2176 static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
2177 bool want_zero,
2178 int64_t offset, int64_t bytes,
2179 int64_t *pnum, int64_t *map,
2180 BlockDriverState **file)
2181 {
2182 int64_t total_size;
2183 int64_t n; /* bytes */
2184 int ret;
2185 int64_t local_map = 0;
2186 BlockDriverState *local_file = NULL;
2187 int64_t aligned_offset, aligned_bytes;
2188 uint32_t align;
2189 bool has_filtered_child;
2190
2191 assert(pnum);
2192 *pnum = 0;
2193 total_size = bdrv_getlength(bs);
2194 if (total_size < 0) {
2195 ret = total_size;
2196 goto early_out;
2197 }
2198
2199 if (offset >= total_size) {
2200 ret = BDRV_BLOCK_EOF;
2201 goto early_out;
2202 }
2203 if (!bytes) {
2204 ret = 0;
2205 goto early_out;
2206 }
2207
2208 n = total_size - offset;
2209 if (n < bytes) {
2210 bytes = n;
2211 }
2212
2213 /* Must be non-NULL or bdrv_getlength() would have failed */
2214 assert(bs->drv);
2215 has_filtered_child = bdrv_filter_child(bs);
2216 if (!bs->drv->bdrv_co_block_status && !has_filtered_child) {
2217 *pnum = bytes;
2218 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
2219 if (offset + bytes == total_size) {
2220 ret |= BDRV_BLOCK_EOF;
2221 }
2222 if (bs->drv->protocol_name) {
2223 ret |= BDRV_BLOCK_OFFSET_VALID;
2224 local_map = offset;
2225 local_file = bs;
2226 }
2227 goto early_out;
2228 }
2229
2230 bdrv_inc_in_flight(bs);
2231
2232 /* Round out to request_alignment boundaries */
2233 align = bs->bl.request_alignment;
2234 aligned_offset = QEMU_ALIGN_DOWN(offset, align);
2235 aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
2236
2237 if (bs->drv->bdrv_co_block_status) {
2238 ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
2239 aligned_bytes, pnum, &local_map,
2240 &local_file);
2241 } else {
2242 /* Default code for filters */
2243
2244 local_file = bdrv_filter_bs(bs);
2245 assert(local_file);
2246
2247 *pnum = aligned_bytes;
2248 local_map = aligned_offset;
2249 ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2250 }
2251 if (ret < 0) {
2252 *pnum = 0;
2253 goto out;
2254 }
2255
2256 /*
2257 * The driver's result must be a non-zero multiple of request_alignment.
2258 * Clamp pnum and adjust map to original request.
2259 */
2260 assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2261 align > offset - aligned_offset);
2262 if (ret & BDRV_BLOCK_RECURSE) {
2263 assert(ret & BDRV_BLOCK_DATA);
2264 assert(ret & BDRV_BLOCK_OFFSET_VALID);
2265 assert(!(ret & BDRV_BLOCK_ZERO));
2266 }
2267
2268 *pnum -= offset - aligned_offset;
2269 if (*pnum > bytes) {
2270 *pnum = bytes;
2271 }
2272 if (ret & BDRV_BLOCK_OFFSET_VALID) {
2273 local_map += offset - aligned_offset;
2274 }
2275
2276 if (ret & BDRV_BLOCK_RAW) {
2277 assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
2278 ret = bdrv_co_block_status(local_file, want_zero, local_map,
2279 *pnum, pnum, &local_map, &local_file);
2280 goto out;
2281 }
2282
2283 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
2284 ret |= BDRV_BLOCK_ALLOCATED;
2285 } else if (want_zero && bs->drv->supports_backing) {
2286 BlockDriverState *cow_bs = bdrv_cow_bs(bs);
2287
2288 if (cow_bs) {
2289 int64_t size2 = bdrv_getlength(cow_bs);
2290
2291 if (size2 >= 0 && offset >= size2) {
2292 ret |= BDRV_BLOCK_ZERO;
2293 }
2294 } else {
2295 ret |= BDRV_BLOCK_ZERO;
2296 }
2297 }
2298
2299 if (want_zero && ret & BDRV_BLOCK_RECURSE &&
2300 local_file && local_file != bs &&
2301 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
2302 (ret & BDRV_BLOCK_OFFSET_VALID)) {
2303 int64_t file_pnum;
2304 int ret2;
2305
2306 ret2 = bdrv_co_block_status(local_file, want_zero, local_map,
2307 *pnum, &file_pnum, NULL, NULL);
2308 if (ret2 >= 0) {
2309 /* Ignore errors. This is just providing extra information, it
2310 * is useful but not necessary.
2311 */
2312 if (ret2 & BDRV_BLOCK_EOF &&
2313 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2314 /*
2315 * It is valid for the format block driver to read
2316 * beyond the end of the underlying file's current
2317 * size; such areas read as zero.
2318 */
2319 ret |= BDRV_BLOCK_ZERO;
2320 } else {
2321 /* Limit request to the range reported by the protocol driver */
2322 *pnum = file_pnum;
2323 ret |= (ret2 & BDRV_BLOCK_ZERO);
2324 }
2325 }
2326 }
2327
2328 out:
2329 bdrv_dec_in_flight(bs);
2330 if (ret >= 0 && offset + *pnum == total_size) {
2331 ret |= BDRV_BLOCK_EOF;
2332 }
2333 early_out:
2334 if (file) {
2335 *file = local_file;
2336 }
2337 if (map) {
2338 *map = local_map;
2339 }
2340 return ret;
2341 }
2342
2343 int coroutine_fn
2344 bdrv_co_common_block_status_above(BlockDriverState *bs,
2345 BlockDriverState *base,
2346 bool want_zero,
2347 int64_t offset,
2348 int64_t bytes,
2349 int64_t *pnum,
2350 int64_t *map,
2351 BlockDriverState **file)
2352 {
2353 BlockDriverState *p;
2354 int ret = 0;
2355 bool first = true;
2356
2357 assert(bs != base);
2358 for (p = bs; p != base; p = bdrv_filter_or_cow_bs(p)) {
2359 ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map,
2360 file);
2361 if (ret < 0) {
2362 break;
2363 }
2364 if (ret & BDRV_BLOCK_ZERO && ret & BDRV_BLOCK_EOF && !first) {
2365 /*
2366 * Reading beyond the end of the file continues to read
2367 * zeroes, but we can only widen the result to the
2368 * unallocated length we learned from an earlier
2369 * iteration.
2370 */
2371 *pnum = bytes;
2372 }
2373 if (ret & (BDRV_BLOCK_ZERO | BDRV_BLOCK_DATA)) {
2374 break;
2375 }
2376 /* [offset, pnum] unallocated on this layer, which could be only
2377 * the first part of [offset, bytes]. */
2378 bytes = MIN(bytes, *pnum);
2379 first = false;
2380 }
2381 return ret;
2382 }
2383
2384 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base,
2385 int64_t offset, int64_t bytes, int64_t *pnum,
2386 int64_t *map, BlockDriverState **file)
2387 {
2388 return bdrv_common_block_status_above(bs, base, true, offset, bytes,
2389 pnum, map, file);
2390 }
2391
2392 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes,
2393 int64_t *pnum, int64_t *map, BlockDriverState **file)
2394 {
2395 return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs),
2396 offset, bytes, pnum, map, file);
2397 }
2398
2399 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset,
2400 int64_t bytes, int64_t *pnum)
2401 {
2402 int ret;
2403 int64_t dummy;
2404
2405 ret = bdrv_common_block_status_above(bs, bdrv_filter_or_cow_bs(bs), false,
2406 offset, bytes, pnum ? pnum : &dummy,
2407 NULL, NULL);
2408 if (ret < 0) {
2409 return ret;
2410 }
2411 return !!(ret & BDRV_BLOCK_ALLOCATED);
2412 }
2413
2414 /*
2415 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
2416 *
2417 * Return 1 if (a prefix of) the given range is allocated in any image
2418 * between BASE and TOP (BASE is only included if include_base is set).
2419 * BASE can be NULL to check if the given offset is allocated in any
2420 * image of the chain. Return 0 otherwise, or negative errno on
2421 * failure.
2422 *
2423 * 'pnum' is set to the number of bytes (including and immediately
2424 * following the specified offset) that are known to be in the same
2425 * allocated/unallocated state. Note that a subsequent call starting
2426 * at 'offset + *pnum' may return the same allocation status (in other
2427 * words, the result is not necessarily the maximum possible range);
2428 * but 'pnum' will only be 0 when end of file is reached.
2429 *
2430 */
2431 int bdrv_is_allocated_above(BlockDriverState *top,
2432 BlockDriverState *base,
2433 bool include_base, int64_t offset,
2434 int64_t bytes, int64_t *pnum)
2435 {
2436 BlockDriverState *intermediate;
2437 int ret;
2438 int64_t n = bytes;
2439
2440 assert(base || !include_base);
2441
2442 intermediate = top;
2443 while (include_base || intermediate != base) {
2444 int64_t pnum_inter;
2445 int64_t size_inter;
2446
2447 assert(intermediate);
2448 ret = bdrv_is_allocated(intermediate, offset, bytes, &pnum_inter);
2449 if (ret < 0) {
2450 return ret;
2451 }
2452 if (ret) {
2453 *pnum = pnum_inter;
2454 return 1;
2455 }
2456
2457 size_inter = bdrv_getlength(intermediate);
2458 if (size_inter < 0) {
2459 return size_inter;
2460 }
2461 if (n > pnum_inter &&
2462 (intermediate == top || offset + pnum_inter < size_inter)) {
2463 n = pnum_inter;
2464 }
2465
2466 if (intermediate == base) {
2467 break;
2468 }
2469
2470 intermediate = bdrv_filter_or_cow_bs(intermediate);
2471 }
2472
2473 *pnum = n;
2474 return 0;
2475 }
2476
2477 int coroutine_fn
2478 bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
2479 bool is_read)
2480 {
2481 BlockDriver *drv = bs->drv;
2482 BlockDriverState *child_bs = bdrv_primary_bs(bs);
2483 int ret = -ENOTSUP;
2484
2485 bdrv_inc_in_flight(bs);
2486
2487 if (!drv) {
2488 ret = -ENOMEDIUM;
2489 } else if (drv->bdrv_load_vmstate) {
2490 if (is_read) {
2491 ret = drv->bdrv_load_vmstate(bs, qiov, pos);
2492 } else {
2493 ret = drv->bdrv_save_vmstate(bs, qiov, pos);
2494 }
2495 } else if (child_bs) {
2496 ret = bdrv_co_rw_vmstate(child_bs, qiov, pos, is_read);
2497 }
2498
2499 bdrv_dec_in_flight(bs);
2500 return ret;
2501 }
2502
2503 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
2504 int64_t pos, int size)
2505 {
2506 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2507 int ret;
2508
2509 ret = bdrv_writev_vmstate(bs, &qiov, pos);
2510 if (ret < 0) {
2511 return ret;
2512 }
2513
2514 return size;
2515 }
2516
2517 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2518 {
2519 return bdrv_rw_vmstate(bs, qiov, pos, false);
2520 }
2521
2522 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
2523 int64_t pos, int size)
2524 {
2525 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2526 int ret;
2527
2528 ret = bdrv_readv_vmstate(bs, &qiov, pos);
2529 if (ret < 0) {
2530 return ret;
2531 }
2532
2533 return size;
2534 }
2535
2536 int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2537 {
2538 return bdrv_rw_vmstate(bs, qiov, pos, true);
2539 }
2540
2541 /**************************************************************/
2542 /* async I/Os */
2543
2544 void bdrv_aio_cancel(BlockAIOCB *acb)
2545 {
2546 qemu_aio_ref(acb);
2547 bdrv_aio_cancel_async(acb);
2548 while (acb->refcnt > 1) {
2549 if (acb->aiocb_info->get_aio_context) {
2550 aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2551 } else if (acb->bs) {
2552 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
2553 * assert that we're not using an I/O thread. Thread-safe
2554 * code should use bdrv_aio_cancel_async exclusively.
2555 */
2556 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
2557 aio_poll(bdrv_get_aio_context(acb->bs), true);
2558 } else {
2559 abort();
2560 }
2561 }
2562 qemu_aio_unref(acb);
2563 }
2564
2565 /* Async version of aio cancel. The caller is not blocked if the acb implements
2566 * cancel_async, otherwise we do nothing and let the request normally complete.
2567 * In either case the completion callback must be called. */
2568 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2569 {
2570 if (acb->aiocb_info->cancel_async) {
2571 acb->aiocb_info->cancel_async(acb);
2572 }
2573 }
2574
2575 /**************************************************************/
2576 /* Coroutine block device emulation */
2577
2578 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2579 {
2580 BdrvChild *primary_child = bdrv_primary_child(bs);
2581 BdrvChild *child;
2582 int current_gen;
2583 int ret = 0;
2584
2585 bdrv_inc_in_flight(bs);
2586
2587 if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2588 bdrv_is_sg(bs)) {
2589 goto early_exit;
2590 }
2591
2592 qemu_co_mutex_lock(&bs->reqs_lock);
2593 current_gen = qatomic_read(&bs->write_gen);
2594
2595 /* Wait until any previous flushes are completed */
2596 while (bs->active_flush_req) {
2597 qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
2598 }
2599
2600 /* Flushes reach this point in nondecreasing current_gen order. */
2601 bs->active_flush_req = true;
2602 qemu_co_mutex_unlock(&bs->reqs_lock);
2603
2604 /* Write back all layers by calling one driver function */
2605 if (bs->drv->bdrv_co_flush) {
2606 ret = bs->drv->bdrv_co_flush(bs);
2607 goto out;
2608 }
2609
2610 /* Write back cached data to the OS even with cache=unsafe */
2611 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS);
2612 if (bs->drv->bdrv_co_flush_to_os) {
2613 ret = bs->drv->bdrv_co_flush_to_os(bs);
2614 if (ret < 0) {
2615 goto out;
2616 }
2617 }
2618
2619 /* But don't actually force it to the disk with cache=unsafe */
2620 if (bs->open_flags & BDRV_O_NO_FLUSH) {
2621 goto flush_children;
2622 }
2623
2624 /* Check if we really need to flush anything */
2625 if (bs->flushed_gen == current_gen) {
2626 goto flush_children;
2627 }
2628
2629 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK);
2630 if (!bs->drv) {
2631 /* bs->drv->bdrv_co_flush() might have ejected the BDS
2632 * (even in case of apparent success) */
2633 ret = -ENOMEDIUM;
2634 goto out;
2635 }
2636 if (bs->drv->bdrv_co_flush_to_disk) {
2637 ret = bs->drv->bdrv_co_flush_to_disk(bs);
2638 } else if (bs->drv->bdrv_aio_flush) {
2639 BlockAIOCB *acb;
2640 CoroutineIOCompletion co = {
2641 .coroutine = qemu_coroutine_self(),
2642 };
2643
2644 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2645 if (acb == NULL) {
2646 ret = -EIO;
2647 } else {
2648 qemu_coroutine_yield();
2649 ret = co.ret;
2650 }
2651 } else {
2652 /*
2653 * Some block drivers always operate in either writethrough or unsafe
2654 * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2655 * know how the server works (because the behaviour is hardcoded or
2656 * depends on server-side configuration), so we can't ensure that
2657 * everything is safe on disk. Returning an error doesn't work because
2658 * that would break guests even if the server operates in writethrough
2659 * mode.
2660 *
2661 * Let's hope the user knows what he's doing.
2662 */
2663 ret = 0;
2664 }
2665
2666 if (ret < 0) {
2667 goto out;
2668 }
2669
2670 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH
2671 * in the case of cache=unsafe, so there are no useless flushes.
2672 */
2673 flush_children:
2674 ret = 0;
2675 QLIST_FOREACH(child, &bs->children, next) {
2676 if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) {
2677 int this_child_ret = bdrv_co_flush(child->bs);
2678 if (!ret) {
2679 ret = this_child_ret;
2680 }
2681 }
2682 }
2683
2684 out:
2685 /* Notify any pending flushes that we have completed */
2686 if (ret == 0) {
2687 bs->flushed_gen = current_gen;
2688 }
2689
2690 qemu_co_mutex_lock(&bs->reqs_lock);
2691 bs->active_flush_req = false;
2692 /* Return value is ignored - it's ok if wait queue is empty */
2693 qemu_co_queue_next(&bs->flush_queue);
2694 qemu_co_mutex_unlock(&bs->reqs_lock);
2695
2696 early_exit:
2697 bdrv_dec_in_flight(bs);
2698 return ret;
2699 }
2700
2701 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
2702 int64_t bytes)
2703 {
2704 BdrvTrackedRequest req;
2705 int max_pdiscard, ret;
2706 int head, tail, align;
2707 BlockDriverState *bs = child->bs;
2708
2709 if (!bs || !bs->drv || !bdrv_is_inserted(bs)) {
2710 return -ENOMEDIUM;
2711 }
2712
2713 if (bdrv_has_readonly_bitmaps(bs)) {
2714 return -EPERM;
2715 }
2716
2717 if (offset < 0 || bytes < 0 || bytes > INT64_MAX - offset) {
2718 return -EIO;
2719 }
2720
2721 /* Do nothing if disabled. */
2722 if (!(bs->open_flags & BDRV_O_UNMAP)) {
2723 return 0;
2724 }
2725
2726 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
2727 return 0;
2728 }
2729
2730 /* Discard is advisory, but some devices track and coalesce
2731 * unaligned requests, so we must pass everything down rather than
2732 * round here. Still, most devices will just silently ignore
2733 * unaligned requests (by returning -ENOTSUP), so we must fragment
2734 * the request accordingly. */
2735 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2736 assert(align % bs->bl.request_alignment == 0);
2737 head = offset % align;
2738 tail = (offset + bytes) % align;
2739
2740 bdrv_inc_in_flight(bs);
2741 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
2742
2743 ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0);
2744 if (ret < 0) {
2745 goto out;
2746 }
2747
2748 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX),
2749 align);
2750 assert(max_pdiscard >= bs->bl.request_alignment);
2751
2752 while (bytes > 0) {
2753 int64_t num = bytes;
2754
2755 if (head) {
2756 /* Make small requests to get to alignment boundaries. */
2757 num = MIN(bytes, align - head);
2758 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
2759 num %= bs->bl.request_alignment;
2760 }
2761 head = (head + num) % align;
2762 assert(num < max_pdiscard);
2763 } else if (tail) {
2764 if (num > align) {
2765 /* Shorten the request to the last aligned cluster. */
2766 num -= tail;
2767 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
2768 tail > bs->bl.request_alignment) {
2769 tail %= bs->bl.request_alignment;
2770 num -= tail;
2771 }
2772 }
2773 /* limit request size */
2774 if (num > max_pdiscard) {
2775 num = max_pdiscard;
2776 }
2777
2778 if (!bs->drv) {
2779 ret = -ENOMEDIUM;
2780 goto out;
2781 }
2782 if (bs->drv->bdrv_co_pdiscard) {
2783 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
2784 } else {
2785 BlockAIOCB *acb;
2786 CoroutineIOCompletion co = {
2787 .coroutine = qemu_coroutine_self(),
2788 };
2789
2790 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
2791 bdrv_co_io_em_complete, &co);
2792 if (acb == NULL) {
2793 ret = -EIO;
2794 goto out;
2795 } else {
2796 qemu_coroutine_yield();
2797 ret = co.ret;
2798 }
2799 }
2800 if (ret && ret != -ENOTSUP) {
2801 goto out;
2802 }
2803
2804 offset += num;
2805 bytes -= num;
2806 }
2807 ret = 0;
2808 out:
2809 bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret);
2810 tracked_request_end(&req);
2811 bdrv_dec_in_flight(bs);
2812 return ret;
2813 }
2814
2815 int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
2816 {
2817 BlockDriver *drv = bs->drv;
2818 CoroutineIOCompletion co = {
2819 .coroutine = qemu_coroutine_self(),
2820 };
2821 BlockAIOCB *acb;
2822
2823 bdrv_inc_in_flight(bs);
2824 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
2825 co.ret = -ENOTSUP;
2826 goto out;
2827 }
2828
2829 if (drv->bdrv_co_ioctl) {
2830 co.ret = drv->bdrv_co_ioctl(bs, req, buf);
2831 } else {
2832 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
2833 if (!acb) {
2834 co.ret = -ENOTSUP;
2835 goto out;
2836 }
2837 qemu_coroutine_yield();
2838 }
2839 out:
2840 bdrv_dec_in_flight(bs);
2841 return co.ret;
2842 }
2843
2844 void *qemu_blockalign(BlockDriverState *bs, size_t size)
2845 {
2846 return qemu_memalign(bdrv_opt_mem_align(bs), size);
2847 }
2848
2849 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
2850 {
2851 return memset(qemu_blockalign(bs, size), 0, size);
2852 }
2853
2854 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
2855 {
2856 size_t align = bdrv_opt_mem_align(bs);
2857
2858 /* Ensure that NULL is never returned on success */
2859 assert(align > 0);
2860 if (size == 0) {
2861 size = align;
2862 }
2863
2864 return qemu_try_memalign(align, size);
2865 }
2866
2867 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
2868 {
2869 void *mem = qemu_try_blockalign(bs, size);
2870
2871 if (mem) {
2872 memset(mem, 0, size);
2873 }
2874
2875 return mem;
2876 }
2877
2878 /*
2879 * Check if all memory in this vector is sector aligned.
2880 */
2881 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
2882 {
2883 int i;
2884 size_t alignment = bdrv_min_mem_align(bs);
2885
2886 for (i = 0; i < qiov->niov; i++) {
2887 if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
2888 return false;
2889 }
2890 if (qiov->iov[i].iov_len % alignment) {
2891 return false;
2892 }
2893 }
2894
2895 return true;
2896 }
2897
2898 void bdrv_add_before_write_notifier(BlockDriverState *bs,
2899 NotifierWithReturn *notifier)
2900 {
2901 notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
2902 }
2903
2904 void bdrv_io_plug(BlockDriverState *bs)
2905 {
2906 BdrvChild *child;
2907
2908 QLIST_FOREACH(child, &bs->children, next) {
2909 bdrv_io_plug(child->bs);
2910 }
2911
2912 if (qatomic_fetch_inc(&bs->io_plugged) == 0) {
2913 BlockDriver *drv = bs->drv;
2914 if (drv && drv->bdrv_io_plug) {
2915 drv->bdrv_io_plug(bs);
2916 }
2917 }
2918 }
2919
2920 void bdrv_io_unplug(BlockDriverState *bs)
2921 {
2922 BdrvChild *child;
2923
2924 assert(bs->io_plugged);
2925 if (qatomic_fetch_dec(&bs->io_plugged) == 1) {
2926 BlockDriver *drv = bs->drv;
2927 if (drv && drv->bdrv_io_unplug) {
2928 drv->bdrv_io_unplug(bs);
2929 }
2930 }
2931
2932 QLIST_FOREACH(child, &bs->children, next) {
2933 bdrv_io_unplug(child->bs);
2934 }
2935 }
2936
2937 void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size)
2938 {
2939 BdrvChild *child;
2940
2941 if (bs->drv && bs->drv->bdrv_register_buf) {
2942 bs->drv->bdrv_register_buf(bs, host, size);
2943 }
2944 QLIST_FOREACH(child, &bs->children, next) {
2945 bdrv_register_buf(child->bs, host, size);
2946 }
2947 }
2948
2949 void bdrv_unregister_buf(BlockDriverState *bs, void *host)
2950 {
2951 BdrvChild *child;
2952
2953 if (bs->drv && bs->drv->bdrv_unregister_buf) {
2954 bs->drv->bdrv_unregister_buf(bs, host);
2955 }
2956 QLIST_FOREACH(child, &bs->children, next) {
2957 bdrv_unregister_buf(child->bs, host);
2958 }
2959 }
2960
2961 static int coroutine_fn bdrv_co_copy_range_internal(
2962 BdrvChild *src, uint64_t src_offset, BdrvChild *dst,
2963 uint64_t dst_offset, uint64_t bytes,
2964 BdrvRequestFlags read_flags, BdrvRequestFlags write_flags,
2965 bool recurse_src)
2966 {
2967 BdrvTrackedRequest req;
2968 int ret;
2969
2970 /* TODO We can support BDRV_REQ_NO_FALLBACK here */
2971 assert(!(read_flags & BDRV_REQ_NO_FALLBACK));
2972 assert(!(write_flags & BDRV_REQ_NO_FALLBACK));
2973
2974 if (!dst || !dst->bs) {
2975 return -ENOMEDIUM;
2976 }
2977 ret = bdrv_check_byte_request(dst->bs, dst_offset, bytes);
2978 if (ret) {
2979 return ret;
2980 }
2981 if (write_flags & BDRV_REQ_ZERO_WRITE) {
2982 return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags);
2983 }
2984
2985 if (!src || !src->bs) {
2986 return -ENOMEDIUM;
2987 }
2988 ret = bdrv_check_byte_request(src->bs, src_offset, bytes);
2989 if (ret) {
2990 return ret;
2991 }
2992
2993 if (!src->bs->drv->bdrv_co_copy_range_from
2994 || !dst->bs->drv->bdrv_co_copy_range_to
2995 || src->bs->encrypted || dst->bs->encrypted) {
2996 return -ENOTSUP;
2997 }
2998
2999 if (recurse_src) {
3000 bdrv_inc_in_flight(src->bs);
3001 tracked_request_begin(&req, src->bs, src_offset, bytes,
3002 BDRV_TRACKED_READ);
3003
3004 /* BDRV_REQ_SERIALISING is only for write operation */
3005 assert(!(read_flags & BDRV_REQ_SERIALISING));
3006 bdrv_wait_serialising_requests(&req);
3007
3008 ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
3009 src, src_offset,
3010 dst, dst_offset,
3011 bytes,
3012 read_flags, write_flags);
3013
3014 tracked_request_end(&req);
3015 bdrv_dec_in_flight(src->bs);
3016 } else {
3017 bdrv_inc_in_flight(dst->bs);
3018 tracked_request_begin(&req, dst->bs, dst_offset, bytes,
3019 BDRV_TRACKED_WRITE);
3020 ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req,
3021 write_flags);
3022 if (!ret) {
3023 ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
3024 src, src_offset,
3025 dst, dst_offset,
3026 bytes,
3027 read_flags, write_flags);
3028 }
3029 bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret);
3030 tracked_request_end(&req);
3031 bdrv_dec_in_flight(dst->bs);
3032 }
3033
3034 return ret;
3035 }
3036
3037 /* Copy range from @src to @dst.
3038 *
3039 * See the comment of bdrv_co_copy_range for the parameter and return value
3040 * semantics. */
3041 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, uint64_t src_offset,
3042 BdrvChild *dst, uint64_t dst_offset,
3043 uint64_t bytes,
3044 BdrvRequestFlags read_flags,
3045 BdrvRequestFlags write_flags)
3046 {
3047 trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes,
3048 read_flags, write_flags);
3049 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3050 bytes, read_flags, write_flags, true);
3051 }
3052
3053 /* Copy range from @src to @dst.
3054 *
3055 * See the comment of bdrv_co_copy_range for the parameter and return value
3056 * semantics. */
3057 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, uint64_t src_offset,
3058 BdrvChild *dst, uint64_t dst_offset,
3059 uint64_t bytes,
3060 BdrvRequestFlags read_flags,
3061 BdrvRequestFlags write_flags)
3062 {
3063 trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3064 read_flags, write_flags);
3065 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3066 bytes, read_flags, write_flags, false);
3067 }
3068
3069 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, uint64_t src_offset,
3070 BdrvChild *dst, uint64_t dst_offset,
3071 uint64_t bytes, BdrvRequestFlags read_flags,
3072 BdrvRequestFlags write_flags)
3073 {
3074 return bdrv_co_copy_range_from(src, src_offset,
3075 dst, dst_offset,
3076 bytes, read_flags, write_flags);
3077 }
3078
3079 static void bdrv_parent_cb_resize(BlockDriverState *bs)
3080 {
3081 BdrvChild *c;
3082 QLIST_FOREACH(c, &bs->parents, next_parent) {
3083 if (c->klass->resize) {
3084 c->klass->resize(c);
3085 }
3086 }
3087 }
3088
3089 /**
3090 * Truncate file to 'offset' bytes (needed only for file protocols)
3091 *
3092 * If 'exact' is true, the file must be resized to exactly the given
3093 * 'offset'. Otherwise, it is sufficient for the node to be at least
3094 * 'offset' bytes in length.
3095 */
3096 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
3097 PreallocMode prealloc, BdrvRequestFlags flags,
3098 Error **errp)
3099 {
3100 BlockDriverState *bs = child->bs;
3101 BdrvChild *filtered, *backing;
3102 BlockDriver *drv = bs->drv;
3103 BdrvTrackedRequest req;
3104 int64_t old_size, new_bytes;
3105 int ret;
3106
3107
3108 /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
3109 if (!drv) {
3110 error_setg(errp, "No medium inserted");
3111 return -ENOMEDIUM;
3112 }
3113 if (offset < 0) {
3114 error_setg(errp, "Image size cannot be negative");
3115 return -EINVAL;
3116 }
3117
3118 old_size = bdrv_getlength(bs);
3119 if (old_size < 0) {
3120 error_setg_errno(errp, -old_size, "Failed to get old image size");
3121 return old_size;
3122 }
3123
3124 if (offset > old_size) {
3125 new_bytes = offset - old_size;
3126 } else {
3127 new_bytes = 0;
3128 }
3129
3130 bdrv_inc_in_flight(bs);
3131 tracked_request_begin(&req, bs, offset - new_bytes, new_bytes,
3132 BDRV_TRACKED_TRUNCATE);
3133
3134 /* If we are growing the image and potentially using preallocation for the
3135 * new area, we need to make sure that no write requests are made to it
3136 * concurrently or they might be overwritten by preallocation. */
3137 if (new_bytes) {
3138 bdrv_mark_request_serialising(&req, 1);
3139 }
3140 if (bs->read_only) {
3141 error_setg(errp, "Image is read-only");
3142 ret = -EACCES;
3143 goto out;
3144 }
3145 ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req,
3146 0);
3147 if (ret < 0) {
3148 error_setg_errno(errp, -ret,
3149 "Failed to prepare request for truncation");
3150 goto out;
3151 }
3152
3153 filtered = bdrv_filter_child(bs);
3154 backing = bdrv_cow_child(bs);
3155
3156 /*
3157 * If the image has a backing file that is large enough that it would
3158 * provide data for the new area, we cannot leave it unallocated because
3159 * then the backing file content would become visible. Instead, zero-fill
3160 * the new area.
3161 *
3162 * Note that if the image has a backing file, but was opened without the
3163 * backing file, taking care of keeping things consistent with that backing
3164 * file is the user's responsibility.
3165 */
3166 if (new_bytes && backing) {
3167 int64_t backing_len;
3168
3169 backing_len = bdrv_getlength(backing->bs);
3170 if (backing_len < 0) {
3171 ret = backing_len;
3172 error_setg_errno(errp, -ret, "Could not get backing file size");
3173 goto out;
3174 }
3175
3176 if (backing_len > old_size) {
3177 flags |= BDRV_REQ_ZERO_WRITE;
3178 }
3179 }
3180
3181 if (drv->bdrv_co_truncate) {
3182 if (flags & ~bs->supported_truncate_flags) {
3183 error_setg(errp, "Block driver does not support requested flags");
3184 ret = -ENOTSUP;
3185 goto out;
3186 }
3187 ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp);
3188 } else if (filtered) {
3189 ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp);
3190 } else {
3191 error_setg(errp, "Image format driver does not support resize");
3192 ret = -ENOTSUP;
3193 goto out;
3194 }
3195 if (ret < 0) {
3196 goto out;
3197 }
3198
3199 ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
3200 if (ret < 0) {
3201 error_setg_errno(errp, -ret, "Could not refresh total sector count");
3202 } else {
3203 offset = bs->total_sectors * BDRV_SECTOR_SIZE;
3204 }
3205 /* It's possible that truncation succeeded but refresh_total_sectors
3206 * failed, but the latter doesn't affect how we should finish the request.
3207 * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */
3208 bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0);
3209
3210 out:
3211 tracked_request_end(&req);
3212 bdrv_dec_in_flight(bs);
3213
3214 return ret;
3215 }