block: Drain throttling queue with BdrvChild callback
[qemu.git] / block / io.c
1 /*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/blockjob.h"
29 #include "block/block_int.h"
30 #include "qemu/cutils.h"
31 #include "qapi/error.h"
32 #include "qemu/error-report.h"
33
34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
35
36 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
37 int64_t sector_num,
38 QEMUIOVector *qiov,
39 int nb_sectors,
40 BdrvRequestFlags flags,
41 BlockCompletionFunc *cb,
42 void *opaque,
43 bool is_write);
44 static void coroutine_fn bdrv_co_do_rw(void *opaque);
45 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
46 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags);
47
48 static void bdrv_parent_drained_begin(BlockDriverState *bs)
49 {
50 BdrvChild *c;
51
52 QLIST_FOREACH(c, &bs->parents, next_parent) {
53 if (c->role->drained_begin) {
54 c->role->drained_begin(c);
55 }
56 }
57 }
58
59 static void bdrv_parent_drained_end(BlockDriverState *bs)
60 {
61 BdrvChild *c;
62
63 QLIST_FOREACH(c, &bs->parents, next_parent) {
64 if (c->role->drained_end) {
65 c->role->drained_end(c);
66 }
67 }
68 }
69
70 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
71 {
72 BlockDriver *drv = bs->drv;
73 Error *local_err = NULL;
74
75 memset(&bs->bl, 0, sizeof(bs->bl));
76
77 if (!drv) {
78 return;
79 }
80
81 /* Take some limits from the children as a default */
82 if (bs->file) {
83 bdrv_refresh_limits(bs->file->bs, &local_err);
84 if (local_err) {
85 error_propagate(errp, local_err);
86 return;
87 }
88 bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length;
89 bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length;
90 bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment;
91 bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment;
92 bs->bl.max_iov = bs->file->bs->bl.max_iov;
93 } else {
94 bs->bl.min_mem_alignment = 512;
95 bs->bl.opt_mem_alignment = getpagesize();
96
97 /* Safe default since most protocols use readv()/writev()/etc */
98 bs->bl.max_iov = IOV_MAX;
99 }
100
101 if (bs->backing) {
102 bdrv_refresh_limits(bs->backing->bs, &local_err);
103 if (local_err) {
104 error_propagate(errp, local_err);
105 return;
106 }
107 bs->bl.opt_transfer_length =
108 MAX(bs->bl.opt_transfer_length,
109 bs->backing->bs->bl.opt_transfer_length);
110 bs->bl.max_transfer_length =
111 MIN_NON_ZERO(bs->bl.max_transfer_length,
112 bs->backing->bs->bl.max_transfer_length);
113 bs->bl.opt_mem_alignment =
114 MAX(bs->bl.opt_mem_alignment,
115 bs->backing->bs->bl.opt_mem_alignment);
116 bs->bl.min_mem_alignment =
117 MAX(bs->bl.min_mem_alignment,
118 bs->backing->bs->bl.min_mem_alignment);
119 bs->bl.max_iov =
120 MIN(bs->bl.max_iov,
121 bs->backing->bs->bl.max_iov);
122 }
123
124 /* Then let the driver override it */
125 if (drv->bdrv_refresh_limits) {
126 drv->bdrv_refresh_limits(bs, errp);
127 }
128 }
129
130 /**
131 * The copy-on-read flag is actually a reference count so multiple users may
132 * use the feature without worrying about clobbering its previous state.
133 * Copy-on-read stays enabled until all users have called to disable it.
134 */
135 void bdrv_enable_copy_on_read(BlockDriverState *bs)
136 {
137 bs->copy_on_read++;
138 }
139
140 void bdrv_disable_copy_on_read(BlockDriverState *bs)
141 {
142 assert(bs->copy_on_read > 0);
143 bs->copy_on_read--;
144 }
145
146 /* Check if any requests are in-flight (including throttled requests) */
147 bool bdrv_requests_pending(BlockDriverState *bs)
148 {
149 BdrvChild *child;
150 BlockBackendPublic *blkp = bs->blk ? blk_get_public(bs->blk) : NULL;
151
152 if (!QLIST_EMPTY(&bs->tracked_requests)) {
153 return true;
154 }
155 if (blkp && !qemu_co_queue_empty(&blkp->throttled_reqs[0])) {
156 return true;
157 }
158 if (blkp && !qemu_co_queue_empty(&blkp->throttled_reqs[1])) {
159 return true;
160 }
161
162 QLIST_FOREACH(child, &bs->children, next) {
163 if (bdrv_requests_pending(child->bs)) {
164 return true;
165 }
166 }
167
168 return false;
169 }
170
171 static void bdrv_drain_recurse(BlockDriverState *bs)
172 {
173 BdrvChild *child;
174
175 if (bs->drv && bs->drv->bdrv_drain) {
176 bs->drv->bdrv_drain(bs);
177 }
178 QLIST_FOREACH(child, &bs->children, next) {
179 bdrv_drain_recurse(child->bs);
180 }
181 }
182
183 typedef struct {
184 Coroutine *co;
185 BlockDriverState *bs;
186 QEMUBH *bh;
187 bool done;
188 } BdrvCoDrainData;
189
190 static void bdrv_drain_poll(BlockDriverState *bs)
191 {
192 bool busy = true;
193
194 while (busy) {
195 /* Keep iterating */
196 busy = bdrv_requests_pending(bs);
197 busy |= aio_poll(bdrv_get_aio_context(bs), busy);
198 }
199 }
200
201 static void bdrv_co_drain_bh_cb(void *opaque)
202 {
203 BdrvCoDrainData *data = opaque;
204 Coroutine *co = data->co;
205
206 qemu_bh_delete(data->bh);
207 bdrv_drain_poll(data->bs);
208 data->done = true;
209 qemu_coroutine_enter(co, NULL);
210 }
211
212 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
213 {
214 BdrvCoDrainData data;
215
216 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
217 * other coroutines run if they were queued from
218 * qemu_co_queue_run_restart(). */
219
220 assert(qemu_in_coroutine());
221 data = (BdrvCoDrainData) {
222 .co = qemu_coroutine_self(),
223 .bs = bs,
224 .done = false,
225 .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data),
226 };
227 qemu_bh_schedule(data.bh);
228
229 qemu_coroutine_yield();
230 /* If we are resumed from some other event (such as an aio completion or a
231 * timer callback), it is a bug in the caller that should be fixed. */
232 assert(data.done);
233 }
234
235 /*
236 * Wait for pending requests to complete on a single BlockDriverState subtree,
237 * and suspend block driver's internal I/O until next request arrives.
238 *
239 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
240 * AioContext.
241 *
242 * Only this BlockDriverState's AioContext is run, so in-flight requests must
243 * not depend on events in other AioContexts. In that case, use
244 * bdrv_drain_all() instead.
245 */
246 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
247 {
248 bdrv_parent_drained_begin(bs);
249 bdrv_io_unplugged_begin(bs);
250 bdrv_drain_recurse(bs);
251 bdrv_co_yield_to_drain(bs);
252 bdrv_io_unplugged_end(bs);
253 bdrv_parent_drained_end(bs);
254 }
255
256 void bdrv_drain(BlockDriverState *bs)
257 {
258 bdrv_parent_drained_begin(bs);
259 bdrv_io_unplugged_begin(bs);
260 bdrv_drain_recurse(bs);
261 if (qemu_in_coroutine()) {
262 bdrv_co_yield_to_drain(bs);
263 } else {
264 bdrv_drain_poll(bs);
265 }
266 bdrv_io_unplugged_end(bs);
267 bdrv_parent_drained_end(bs);
268 }
269
270 /*
271 * Wait for pending requests to complete across all BlockDriverStates
272 *
273 * This function does not flush data to disk, use bdrv_flush_all() for that
274 * after calling this function.
275 */
276 void bdrv_drain_all(void)
277 {
278 /* Always run first iteration so any pending completion BHs run */
279 bool busy = true;
280 BlockDriverState *bs = NULL;
281 GSList *aio_ctxs = NULL, *ctx;
282
283 while ((bs = bdrv_next(bs))) {
284 AioContext *aio_context = bdrv_get_aio_context(bs);
285
286 aio_context_acquire(aio_context);
287 if (bs->job) {
288 block_job_pause(bs->job);
289 }
290 bdrv_parent_drained_begin(bs);
291 bdrv_io_unplugged_begin(bs);
292 bdrv_drain_recurse(bs);
293 aio_context_release(aio_context);
294
295 if (!g_slist_find(aio_ctxs, aio_context)) {
296 aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
297 }
298 }
299
300 /* Note that completion of an asynchronous I/O operation can trigger any
301 * number of other I/O operations on other devices---for example a
302 * coroutine can submit an I/O request to another device in response to
303 * request completion. Therefore we must keep looping until there was no
304 * more activity rather than simply draining each device independently.
305 */
306 while (busy) {
307 busy = false;
308
309 for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
310 AioContext *aio_context = ctx->data;
311 bs = NULL;
312
313 aio_context_acquire(aio_context);
314 while ((bs = bdrv_next(bs))) {
315 if (aio_context == bdrv_get_aio_context(bs)) {
316 if (bdrv_requests_pending(bs)) {
317 busy = true;
318 aio_poll(aio_context, busy);
319 }
320 }
321 }
322 busy |= aio_poll(aio_context, false);
323 aio_context_release(aio_context);
324 }
325 }
326
327 bs = NULL;
328 while ((bs = bdrv_next(bs))) {
329 AioContext *aio_context = bdrv_get_aio_context(bs);
330
331 aio_context_acquire(aio_context);
332 bdrv_io_unplugged_end(bs);
333 bdrv_parent_drained_end(bs);
334 if (bs->job) {
335 block_job_resume(bs->job);
336 }
337 aio_context_release(aio_context);
338 }
339 g_slist_free(aio_ctxs);
340 }
341
342 /**
343 * Remove an active request from the tracked requests list
344 *
345 * This function should be called when a tracked request is completing.
346 */
347 static void tracked_request_end(BdrvTrackedRequest *req)
348 {
349 if (req->serialising) {
350 req->bs->serialising_in_flight--;
351 }
352
353 QLIST_REMOVE(req, list);
354 qemu_co_queue_restart_all(&req->wait_queue);
355 }
356
357 /**
358 * Add an active request to the tracked requests list
359 */
360 static void tracked_request_begin(BdrvTrackedRequest *req,
361 BlockDriverState *bs,
362 int64_t offset,
363 unsigned int bytes,
364 enum BdrvTrackedRequestType type)
365 {
366 *req = (BdrvTrackedRequest){
367 .bs = bs,
368 .offset = offset,
369 .bytes = bytes,
370 .type = type,
371 .co = qemu_coroutine_self(),
372 .serialising = false,
373 .overlap_offset = offset,
374 .overlap_bytes = bytes,
375 };
376
377 qemu_co_queue_init(&req->wait_queue);
378
379 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
380 }
381
382 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
383 {
384 int64_t overlap_offset = req->offset & ~(align - 1);
385 unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
386 - overlap_offset;
387
388 if (!req->serialising) {
389 req->bs->serialising_in_flight++;
390 req->serialising = true;
391 }
392
393 req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
394 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
395 }
396
397 /**
398 * Round a region to cluster boundaries
399 */
400 void bdrv_round_to_clusters(BlockDriverState *bs,
401 int64_t sector_num, int nb_sectors,
402 int64_t *cluster_sector_num,
403 int *cluster_nb_sectors)
404 {
405 BlockDriverInfo bdi;
406
407 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
408 *cluster_sector_num = sector_num;
409 *cluster_nb_sectors = nb_sectors;
410 } else {
411 int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
412 *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
413 *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
414 nb_sectors, c);
415 }
416 }
417
418 static int bdrv_get_cluster_size(BlockDriverState *bs)
419 {
420 BlockDriverInfo bdi;
421 int ret;
422
423 ret = bdrv_get_info(bs, &bdi);
424 if (ret < 0 || bdi.cluster_size == 0) {
425 return bs->request_alignment;
426 } else {
427 return bdi.cluster_size;
428 }
429 }
430
431 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
432 int64_t offset, unsigned int bytes)
433 {
434 /* aaaa bbbb */
435 if (offset >= req->overlap_offset + req->overlap_bytes) {
436 return false;
437 }
438 /* bbbb aaaa */
439 if (req->overlap_offset >= offset + bytes) {
440 return false;
441 }
442 return true;
443 }
444
445 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
446 {
447 BlockDriverState *bs = self->bs;
448 BdrvTrackedRequest *req;
449 bool retry;
450 bool waited = false;
451
452 if (!bs->serialising_in_flight) {
453 return false;
454 }
455
456 do {
457 retry = false;
458 QLIST_FOREACH(req, &bs->tracked_requests, list) {
459 if (req == self || (!req->serialising && !self->serialising)) {
460 continue;
461 }
462 if (tracked_request_overlaps(req, self->overlap_offset,
463 self->overlap_bytes))
464 {
465 /* Hitting this means there was a reentrant request, for
466 * example, a block driver issuing nested requests. This must
467 * never happen since it means deadlock.
468 */
469 assert(qemu_coroutine_self() != req->co);
470
471 /* If the request is already (indirectly) waiting for us, or
472 * will wait for us as soon as it wakes up, then just go on
473 * (instead of producing a deadlock in the former case). */
474 if (!req->waiting_for) {
475 self->waiting_for = req;
476 qemu_co_queue_wait(&req->wait_queue);
477 self->waiting_for = NULL;
478 retry = true;
479 waited = true;
480 break;
481 }
482 }
483 }
484 } while (retry);
485
486 return waited;
487 }
488
489 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
490 size_t size)
491 {
492 if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
493 return -EIO;
494 }
495
496 if (!bdrv_is_inserted(bs)) {
497 return -ENOMEDIUM;
498 }
499
500 if (offset < 0) {
501 return -EIO;
502 }
503
504 return 0;
505 }
506
507 static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
508 int nb_sectors)
509 {
510 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
511 return -EIO;
512 }
513
514 return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
515 nb_sectors * BDRV_SECTOR_SIZE);
516 }
517
518 typedef struct RwCo {
519 BlockDriverState *bs;
520 int64_t offset;
521 QEMUIOVector *qiov;
522 bool is_write;
523 int ret;
524 BdrvRequestFlags flags;
525 } RwCo;
526
527 static void coroutine_fn bdrv_rw_co_entry(void *opaque)
528 {
529 RwCo *rwco = opaque;
530
531 if (!rwco->is_write) {
532 rwco->ret = bdrv_co_preadv(rwco->bs, rwco->offset,
533 rwco->qiov->size, rwco->qiov,
534 rwco->flags);
535 } else {
536 rwco->ret = bdrv_co_pwritev(rwco->bs, rwco->offset,
537 rwco->qiov->size, rwco->qiov,
538 rwco->flags);
539 }
540 }
541
542 /*
543 * Process a vectored synchronous request using coroutines
544 */
545 static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset,
546 QEMUIOVector *qiov, bool is_write,
547 BdrvRequestFlags flags)
548 {
549 Coroutine *co;
550 RwCo rwco = {
551 .bs = bs,
552 .offset = offset,
553 .qiov = qiov,
554 .is_write = is_write,
555 .ret = NOT_DONE,
556 .flags = flags,
557 };
558
559 if (qemu_in_coroutine()) {
560 /* Fast-path if already in coroutine context */
561 bdrv_rw_co_entry(&rwco);
562 } else {
563 AioContext *aio_context = bdrv_get_aio_context(bs);
564
565 co = qemu_coroutine_create(bdrv_rw_co_entry);
566 qemu_coroutine_enter(co, &rwco);
567 while (rwco.ret == NOT_DONE) {
568 aio_poll(aio_context, true);
569 }
570 }
571 return rwco.ret;
572 }
573
574 /*
575 * Process a synchronous request using coroutines
576 */
577 static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf,
578 int nb_sectors, bool is_write, BdrvRequestFlags flags)
579 {
580 QEMUIOVector qiov;
581 struct iovec iov = {
582 .iov_base = (void *)buf,
583 .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
584 };
585
586 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
587 return -EINVAL;
588 }
589
590 qemu_iovec_init_external(&qiov, &iov, 1);
591 return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS,
592 &qiov, is_write, flags);
593 }
594
595 /* return < 0 if error. See bdrv_write() for the return codes */
596 int bdrv_read(BlockDriverState *bs, int64_t sector_num,
597 uint8_t *buf, int nb_sectors)
598 {
599 return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0);
600 }
601
602 /* Return < 0 if error. Important errors are:
603 -EIO generic I/O error (may happen for all errors)
604 -ENOMEDIUM No media inserted.
605 -EINVAL Invalid sector number or nb_sectors
606 -EACCES Trying to write a read-only device
607 */
608 int bdrv_write(BlockDriverState *bs, int64_t sector_num,
609 const uint8_t *buf, int nb_sectors)
610 {
611 return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
612 }
613
614 int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
615 int nb_sectors, BdrvRequestFlags flags)
616 {
617 return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true,
618 BDRV_REQ_ZERO_WRITE | flags);
619 }
620
621 /*
622 * Completely zero out a block device with the help of bdrv_write_zeroes.
623 * The operation is sped up by checking the block status and only writing
624 * zeroes to the device if they currently do not return zeroes. Optional
625 * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
626 * BDRV_REQ_FUA).
627 *
628 * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
629 */
630 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
631 {
632 int64_t target_sectors, ret, nb_sectors, sector_num = 0;
633 BlockDriverState *file;
634 int n;
635
636 target_sectors = bdrv_nb_sectors(bs);
637 if (target_sectors < 0) {
638 return target_sectors;
639 }
640
641 for (;;) {
642 nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
643 if (nb_sectors <= 0) {
644 return 0;
645 }
646 ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
647 if (ret < 0) {
648 error_report("error getting block status at sector %" PRId64 ": %s",
649 sector_num, strerror(-ret));
650 return ret;
651 }
652 if (ret & BDRV_BLOCK_ZERO) {
653 sector_num += n;
654 continue;
655 }
656 ret = bdrv_write_zeroes(bs, sector_num, n, flags);
657 if (ret < 0) {
658 error_report("error writing zeroes at sector %" PRId64 ": %s",
659 sector_num, strerror(-ret));
660 return ret;
661 }
662 sector_num += n;
663 }
664 }
665
666 int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes)
667 {
668 QEMUIOVector qiov;
669 struct iovec iov = {
670 .iov_base = (void *)buf,
671 .iov_len = bytes,
672 };
673 int ret;
674
675 if (bytes < 0) {
676 return -EINVAL;
677 }
678
679 qemu_iovec_init_external(&qiov, &iov, 1);
680 ret = bdrv_prwv_co(bs, offset, &qiov, false, 0);
681 if (ret < 0) {
682 return ret;
683 }
684
685 return bytes;
686 }
687
688 int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov)
689 {
690 int ret;
691
692 ret = bdrv_prwv_co(bs, offset, qiov, true, 0);
693 if (ret < 0) {
694 return ret;
695 }
696
697 return qiov->size;
698 }
699
700 int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
701 const void *buf, int bytes)
702 {
703 QEMUIOVector qiov;
704 struct iovec iov = {
705 .iov_base = (void *) buf,
706 .iov_len = bytes,
707 };
708
709 if (bytes < 0) {
710 return -EINVAL;
711 }
712
713 qemu_iovec_init_external(&qiov, &iov, 1);
714 return bdrv_pwritev(bs, offset, &qiov);
715 }
716
717 /*
718 * Writes to the file and ensures that no writes are reordered across this
719 * request (acts as a barrier)
720 *
721 * Returns 0 on success, -errno in error cases.
722 */
723 int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
724 const void *buf, int count)
725 {
726 int ret;
727
728 ret = bdrv_pwrite(bs, offset, buf, count);
729 if (ret < 0) {
730 return ret;
731 }
732
733 ret = bdrv_flush(bs);
734 if (ret < 0) {
735 return ret;
736 }
737
738 return 0;
739 }
740
741 typedef struct CoroutineIOCompletion {
742 Coroutine *coroutine;
743 int ret;
744 } CoroutineIOCompletion;
745
746 static void bdrv_co_io_em_complete(void *opaque, int ret)
747 {
748 CoroutineIOCompletion *co = opaque;
749
750 co->ret = ret;
751 qemu_coroutine_enter(co->coroutine, NULL);
752 }
753
754 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
755 uint64_t offset, uint64_t bytes,
756 QEMUIOVector *qiov, int flags)
757 {
758 BlockDriver *drv = bs->drv;
759 int64_t sector_num;
760 unsigned int nb_sectors;
761
762 if (drv->bdrv_co_preadv) {
763 return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
764 }
765
766 sector_num = offset >> BDRV_SECTOR_BITS;
767 nb_sectors = bytes >> BDRV_SECTOR_BITS;
768
769 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
770 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
771 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
772
773 if (drv->bdrv_co_readv) {
774 return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
775 } else {
776 BlockAIOCB *acb;
777 CoroutineIOCompletion co = {
778 .coroutine = qemu_coroutine_self(),
779 };
780
781 acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
782 bdrv_co_io_em_complete, &co);
783 if (acb == NULL) {
784 return -EIO;
785 } else {
786 qemu_coroutine_yield();
787 return co.ret;
788 }
789 }
790 }
791
792 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
793 uint64_t offset, uint64_t bytes,
794 QEMUIOVector *qiov, int flags)
795 {
796 BlockDriver *drv = bs->drv;
797 int64_t sector_num;
798 unsigned int nb_sectors;
799 int ret;
800
801 if (drv->bdrv_co_pwritev) {
802 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags);
803 goto emulate_flags;
804 }
805
806 sector_num = offset >> BDRV_SECTOR_BITS;
807 nb_sectors = bytes >> BDRV_SECTOR_BITS;
808
809 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
810 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
811 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
812
813 if (drv->bdrv_co_writev_flags) {
814 ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
815 flags & bs->supported_write_flags);
816 flags &= ~bs->supported_write_flags;
817 } else if (drv->bdrv_co_writev) {
818 assert(!bs->supported_write_flags);
819 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
820 } else {
821 BlockAIOCB *acb;
822 CoroutineIOCompletion co = {
823 .coroutine = qemu_coroutine_self(),
824 };
825
826 acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
827 bdrv_co_io_em_complete, &co);
828 if (acb == NULL) {
829 ret = -EIO;
830 } else {
831 qemu_coroutine_yield();
832 ret = co.ret;
833 }
834 }
835
836 emulate_flags:
837 if (ret == 0 && (flags & BDRV_REQ_FUA)) {
838 ret = bdrv_co_flush(bs);
839 }
840
841 return ret;
842 }
843
844 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
845 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
846 {
847 /* Perform I/O through a temporary buffer so that users who scribble over
848 * their read buffer while the operation is in progress do not end up
849 * modifying the image file. This is critical for zero-copy guest I/O
850 * where anything might happen inside guest memory.
851 */
852 void *bounce_buffer;
853
854 BlockDriver *drv = bs->drv;
855 struct iovec iov;
856 QEMUIOVector bounce_qiov;
857 int64_t cluster_sector_num;
858 int cluster_nb_sectors;
859 size_t skip_bytes;
860 int ret;
861
862 /* Cover entire cluster so no additional backing file I/O is required when
863 * allocating cluster in the image file.
864 */
865 bdrv_round_to_clusters(bs, sector_num, nb_sectors,
866 &cluster_sector_num, &cluster_nb_sectors);
867
868 trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
869 cluster_sector_num, cluster_nb_sectors);
870
871 iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
872 iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
873 if (bounce_buffer == NULL) {
874 ret = -ENOMEM;
875 goto err;
876 }
877
878 qemu_iovec_init_external(&bounce_qiov, &iov, 1);
879
880 ret = bdrv_driver_preadv(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
881 cluster_nb_sectors * BDRV_SECTOR_SIZE,
882 &bounce_qiov, 0);
883 if (ret < 0) {
884 goto err;
885 }
886
887 if (drv->bdrv_co_write_zeroes &&
888 buffer_is_zero(bounce_buffer, iov.iov_len)) {
889 ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num,
890 cluster_nb_sectors, 0);
891 } else {
892 /* This does not change the data on the disk, it is not necessary
893 * to flush even in cache=writethrough mode.
894 */
895 ret = bdrv_driver_pwritev(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
896 cluster_nb_sectors * BDRV_SECTOR_SIZE,
897 &bounce_qiov, 0);
898 }
899
900 if (ret < 0) {
901 /* It might be okay to ignore write errors for guest requests. If this
902 * is a deliberate copy-on-read then we don't want to ignore the error.
903 * Simply report it in all cases.
904 */
905 goto err;
906 }
907
908 skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE;
909 qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes,
910 nb_sectors * BDRV_SECTOR_SIZE);
911
912 err:
913 qemu_vfree(bounce_buffer);
914 return ret;
915 }
916
917 /*
918 * Forwards an already correctly aligned request to the BlockDriver. This
919 * handles copy on read and zeroing after EOF; any other features must be
920 * implemented by the caller.
921 */
922 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
923 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
924 int64_t align, QEMUIOVector *qiov, int flags)
925 {
926 int ret;
927
928 int64_t sector_num = offset >> BDRV_SECTOR_BITS;
929 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
930
931 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
932 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
933 assert(!qiov || bytes == qiov->size);
934 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
935
936 /* Handle Copy on Read and associated serialisation */
937 if (flags & BDRV_REQ_COPY_ON_READ) {
938 /* If we touch the same cluster it counts as an overlap. This
939 * guarantees that allocating writes will be serialized and not race
940 * with each other for the same cluster. For example, in copy-on-read
941 * it ensures that the CoR read and write operations are atomic and
942 * guest writes cannot interleave between them. */
943 mark_request_serialising(req, bdrv_get_cluster_size(bs));
944 }
945
946 if (!(flags & BDRV_REQ_NO_SERIALISING)) {
947 wait_serialising_requests(req);
948 }
949
950 if (flags & BDRV_REQ_COPY_ON_READ) {
951 int pnum;
952
953 ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum);
954 if (ret < 0) {
955 goto out;
956 }
957
958 if (!ret || pnum != nb_sectors) {
959 ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov);
960 goto out;
961 }
962 }
963
964 /* Forward the request to the BlockDriver */
965 if (!bs->zero_beyond_eof) {
966 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
967 } else {
968 /* Read zeros after EOF */
969 int64_t total_sectors, max_nb_sectors;
970
971 total_sectors = bdrv_nb_sectors(bs);
972 if (total_sectors < 0) {
973 ret = total_sectors;
974 goto out;
975 }
976
977 max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num),
978 align >> BDRV_SECTOR_BITS);
979 if (nb_sectors < max_nb_sectors) {
980 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
981 } else if (max_nb_sectors > 0) {
982 QEMUIOVector local_qiov;
983
984 qemu_iovec_init(&local_qiov, qiov->niov);
985 qemu_iovec_concat(&local_qiov, qiov, 0,
986 max_nb_sectors * BDRV_SECTOR_SIZE);
987
988 ret = bdrv_driver_preadv(bs, offset,
989 max_nb_sectors * BDRV_SECTOR_SIZE,
990 &local_qiov, 0);
991
992 qemu_iovec_destroy(&local_qiov);
993 } else {
994 ret = 0;
995 }
996
997 /* Reading beyond end of file is supposed to produce zeroes */
998 if (ret == 0 && total_sectors < sector_num + nb_sectors) {
999 uint64_t offset = MAX(0, total_sectors - sector_num);
1000 uint64_t bytes = (sector_num + nb_sectors - offset) *
1001 BDRV_SECTOR_SIZE;
1002 qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes);
1003 }
1004 }
1005
1006 out:
1007 return ret;
1008 }
1009
1010 /*
1011 * Handle a read request in coroutine context
1012 */
1013 int coroutine_fn bdrv_co_preadv(BlockDriverState *bs,
1014 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1015 BdrvRequestFlags flags)
1016 {
1017 BlockDriver *drv = bs->drv;
1018 BdrvTrackedRequest req;
1019
1020 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1021 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1022 uint8_t *head_buf = NULL;
1023 uint8_t *tail_buf = NULL;
1024 QEMUIOVector local_qiov;
1025 bool use_local_qiov = false;
1026 int ret;
1027
1028 if (!drv) {
1029 return -ENOMEDIUM;
1030 }
1031
1032 ret = bdrv_check_byte_request(bs, offset, bytes);
1033 if (ret < 0) {
1034 return ret;
1035 }
1036
1037 /* Don't do copy-on-read if we read data before write operation */
1038 if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
1039 flags |= BDRV_REQ_COPY_ON_READ;
1040 }
1041
1042 /* Align read if necessary by padding qiov */
1043 if (offset & (align - 1)) {
1044 head_buf = qemu_blockalign(bs, align);
1045 qemu_iovec_init(&local_qiov, qiov->niov + 2);
1046 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1047 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1048 use_local_qiov = true;
1049
1050 bytes += offset & (align - 1);
1051 offset = offset & ~(align - 1);
1052 }
1053
1054 if ((offset + bytes) & (align - 1)) {
1055 if (!use_local_qiov) {
1056 qemu_iovec_init(&local_qiov, qiov->niov + 1);
1057 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1058 use_local_qiov = true;
1059 }
1060 tail_buf = qemu_blockalign(bs, align);
1061 qemu_iovec_add(&local_qiov, tail_buf,
1062 align - ((offset + bytes) & (align - 1)));
1063
1064 bytes = ROUND_UP(bytes, align);
1065 }
1066
1067 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1068 ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
1069 use_local_qiov ? &local_qiov : qiov,
1070 flags);
1071 tracked_request_end(&req);
1072
1073 if (use_local_qiov) {
1074 qemu_iovec_destroy(&local_qiov);
1075 qemu_vfree(head_buf);
1076 qemu_vfree(tail_buf);
1077 }
1078
1079 return ret;
1080 }
1081
1082 static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
1083 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1084 BdrvRequestFlags flags)
1085 {
1086 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1087 return -EINVAL;
1088 }
1089
1090 return bdrv_co_preadv(bs, sector_num << BDRV_SECTOR_BITS,
1091 nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1092 }
1093
1094 int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
1095 int nb_sectors, QEMUIOVector *qiov)
1096 {
1097 trace_bdrv_co_readv(bs, sector_num, nb_sectors);
1098
1099 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
1100 }
1101
1102 int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs,
1103 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
1104 {
1105 trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors);
1106
1107 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
1108 BDRV_REQ_NO_SERIALISING);
1109 }
1110
1111 int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
1112 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
1113 {
1114 trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors);
1115
1116 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
1117 BDRV_REQ_COPY_ON_READ);
1118 }
1119
1120 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768
1121
1122 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
1123 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
1124 {
1125 BlockDriver *drv = bs->drv;
1126 QEMUIOVector qiov;
1127 struct iovec iov = {0};
1128 int ret = 0;
1129 bool need_flush = false;
1130
1131 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes,
1132 BDRV_REQUEST_MAX_SECTORS);
1133
1134 while (nb_sectors > 0 && !ret) {
1135 int num = nb_sectors;
1136
1137 /* Align request. Block drivers can expect the "bulk" of the request
1138 * to be aligned.
1139 */
1140 if (bs->bl.write_zeroes_alignment
1141 && num > bs->bl.write_zeroes_alignment) {
1142 if (sector_num % bs->bl.write_zeroes_alignment != 0) {
1143 /* Make a small request up to the first aligned sector. */
1144 num = bs->bl.write_zeroes_alignment;
1145 num -= sector_num % bs->bl.write_zeroes_alignment;
1146 } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) {
1147 /* Shorten the request to the last aligned sector. num cannot
1148 * underflow because num > bs->bl.write_zeroes_alignment.
1149 */
1150 num -= (sector_num + num) % bs->bl.write_zeroes_alignment;
1151 }
1152 }
1153
1154 /* limit request size */
1155 if (num > max_write_zeroes) {
1156 num = max_write_zeroes;
1157 }
1158
1159 ret = -ENOTSUP;
1160 /* First try the efficient write zeroes operation */
1161 if (drv->bdrv_co_write_zeroes) {
1162 ret = drv->bdrv_co_write_zeroes(bs, sector_num, num,
1163 flags & bs->supported_zero_flags);
1164 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1165 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1166 need_flush = true;
1167 }
1168 } else {
1169 assert(!bs->supported_zero_flags);
1170 }
1171
1172 if (ret == -ENOTSUP) {
1173 /* Fall back to bounce buffer if write zeroes is unsupported */
1174 int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length,
1175 MAX_WRITE_ZEROES_BOUNCE_BUFFER);
1176 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1177
1178 if ((flags & BDRV_REQ_FUA) &&
1179 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1180 /* No need for bdrv_driver_pwrite() to do a fallback
1181 * flush on each chunk; use just one at the end */
1182 write_flags &= ~BDRV_REQ_FUA;
1183 need_flush = true;
1184 }
1185 num = MIN(num, max_xfer_len);
1186 iov.iov_len = num * BDRV_SECTOR_SIZE;
1187 if (iov.iov_base == NULL) {
1188 iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE);
1189 if (iov.iov_base == NULL) {
1190 ret = -ENOMEM;
1191 goto fail;
1192 }
1193 memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE);
1194 }
1195 qemu_iovec_init_external(&qiov, &iov, 1);
1196
1197 ret = bdrv_driver_pwritev(bs, sector_num * BDRV_SECTOR_SIZE,
1198 num * BDRV_SECTOR_SIZE, &qiov,
1199 write_flags);
1200
1201 /* Keep bounce buffer around if it is big enough for all
1202 * all future requests.
1203 */
1204 if (num < max_xfer_len) {
1205 qemu_vfree(iov.iov_base);
1206 iov.iov_base = NULL;
1207 }
1208 }
1209
1210 sector_num += num;
1211 nb_sectors -= num;
1212 }
1213
1214 fail:
1215 if (ret == 0 && need_flush) {
1216 ret = bdrv_co_flush(bs);
1217 }
1218 qemu_vfree(iov.iov_base);
1219 return ret;
1220 }
1221
1222 /*
1223 * Forwards an already correctly aligned write request to the BlockDriver.
1224 */
1225 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
1226 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1227 QEMUIOVector *qiov, int flags)
1228 {
1229 BlockDriver *drv = bs->drv;
1230 bool waited;
1231 int ret;
1232
1233 int64_t sector_num = offset >> BDRV_SECTOR_BITS;
1234 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
1235
1236 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
1237 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
1238 assert(!qiov || bytes == qiov->size);
1239 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1240
1241 waited = wait_serialising_requests(req);
1242 assert(!waited || !req->serialising);
1243 assert(req->overlap_offset <= offset);
1244 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1245
1246 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
1247
1248 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1249 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes &&
1250 qemu_iovec_is_zero(qiov)) {
1251 flags |= BDRV_REQ_ZERO_WRITE;
1252 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1253 flags |= BDRV_REQ_MAY_UNMAP;
1254 }
1255 }
1256
1257 if (ret < 0) {
1258 /* Do nothing, write notifier decided to fail this request */
1259 } else if (flags & BDRV_REQ_ZERO_WRITE) {
1260 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1261 ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags);
1262 } else {
1263 bdrv_debug_event(bs, BLKDBG_PWRITEV);
1264 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
1265 }
1266 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1267
1268 bdrv_set_dirty(bs, sector_num, nb_sectors);
1269
1270 if (bs->wr_highest_offset < offset + bytes) {
1271 bs->wr_highest_offset = offset + bytes;
1272 }
1273
1274 if (ret >= 0) {
1275 bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
1276 }
1277
1278 return ret;
1279 }
1280
1281 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
1282 int64_t offset,
1283 unsigned int bytes,
1284 BdrvRequestFlags flags,
1285 BdrvTrackedRequest *req)
1286 {
1287 uint8_t *buf = NULL;
1288 QEMUIOVector local_qiov;
1289 struct iovec iov;
1290 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1291 unsigned int head_padding_bytes, tail_padding_bytes;
1292 int ret = 0;
1293
1294 head_padding_bytes = offset & (align - 1);
1295 tail_padding_bytes = align - ((offset + bytes) & (align - 1));
1296
1297
1298 assert(flags & BDRV_REQ_ZERO_WRITE);
1299 if (head_padding_bytes || tail_padding_bytes) {
1300 buf = qemu_blockalign(bs, align);
1301 iov = (struct iovec) {
1302 .iov_base = buf,
1303 .iov_len = align,
1304 };
1305 qemu_iovec_init_external(&local_qiov, &iov, 1);
1306 }
1307 if (head_padding_bytes) {
1308 uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
1309
1310 /* RMW the unaligned part before head. */
1311 mark_request_serialising(req, align);
1312 wait_serialising_requests(req);
1313 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1314 ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
1315 align, &local_qiov, 0);
1316 if (ret < 0) {
1317 goto fail;
1318 }
1319 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1320
1321 memset(buf + head_padding_bytes, 0, zero_bytes);
1322 ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
1323 &local_qiov,
1324 flags & ~BDRV_REQ_ZERO_WRITE);
1325 if (ret < 0) {
1326 goto fail;
1327 }
1328 offset += zero_bytes;
1329 bytes -= zero_bytes;
1330 }
1331
1332 assert(!bytes || (offset & (align - 1)) == 0);
1333 if (bytes >= align) {
1334 /* Write the aligned part in the middle. */
1335 uint64_t aligned_bytes = bytes & ~(align - 1);
1336 ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes,
1337 NULL, flags);
1338 if (ret < 0) {
1339 goto fail;
1340 }
1341 bytes -= aligned_bytes;
1342 offset += aligned_bytes;
1343 }
1344
1345 assert(!bytes || (offset & (align - 1)) == 0);
1346 if (bytes) {
1347 assert(align == tail_padding_bytes + bytes);
1348 /* RMW the unaligned part after tail. */
1349 mark_request_serialising(req, align);
1350 wait_serialising_requests(req);
1351 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1352 ret = bdrv_aligned_preadv(bs, req, offset, align,
1353 align, &local_qiov, 0);
1354 if (ret < 0) {
1355 goto fail;
1356 }
1357 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1358
1359 memset(buf, 0, bytes);
1360 ret = bdrv_aligned_pwritev(bs, req, offset, align,
1361 &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
1362 }
1363 fail:
1364 qemu_vfree(buf);
1365 return ret;
1366
1367 }
1368
1369 /*
1370 * Handle a write request in coroutine context
1371 */
1372 int coroutine_fn bdrv_co_pwritev(BlockDriverState *bs,
1373 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1374 BdrvRequestFlags flags)
1375 {
1376 BdrvTrackedRequest req;
1377 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1378 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1379 uint8_t *head_buf = NULL;
1380 uint8_t *tail_buf = NULL;
1381 QEMUIOVector local_qiov;
1382 bool use_local_qiov = false;
1383 int ret;
1384
1385 if (!bs->drv) {
1386 return -ENOMEDIUM;
1387 }
1388 if (bs->read_only) {
1389 return -EPERM;
1390 }
1391 assert(!(bs->open_flags & BDRV_O_INACTIVE));
1392
1393 ret = bdrv_check_byte_request(bs, offset, bytes);
1394 if (ret < 0) {
1395 return ret;
1396 }
1397
1398 /*
1399 * Align write if necessary by performing a read-modify-write cycle.
1400 * Pad qiov with the read parts and be sure to have a tracked request not
1401 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
1402 */
1403 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
1404
1405 if (!qiov) {
1406 ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
1407 goto out;
1408 }
1409
1410 if (offset & (align - 1)) {
1411 QEMUIOVector head_qiov;
1412 struct iovec head_iov;
1413
1414 mark_request_serialising(&req, align);
1415 wait_serialising_requests(&req);
1416
1417 head_buf = qemu_blockalign(bs, align);
1418 head_iov = (struct iovec) {
1419 .iov_base = head_buf,
1420 .iov_len = align,
1421 };
1422 qemu_iovec_init_external(&head_qiov, &head_iov, 1);
1423
1424 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1425 ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
1426 align, &head_qiov, 0);
1427 if (ret < 0) {
1428 goto fail;
1429 }
1430 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1431
1432 qemu_iovec_init(&local_qiov, qiov->niov + 2);
1433 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1434 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1435 use_local_qiov = true;
1436
1437 bytes += offset & (align - 1);
1438 offset = offset & ~(align - 1);
1439 }
1440
1441 if ((offset + bytes) & (align - 1)) {
1442 QEMUIOVector tail_qiov;
1443 struct iovec tail_iov;
1444 size_t tail_bytes;
1445 bool waited;
1446
1447 mark_request_serialising(&req, align);
1448 waited = wait_serialising_requests(&req);
1449 assert(!waited || !use_local_qiov);
1450
1451 tail_buf = qemu_blockalign(bs, align);
1452 tail_iov = (struct iovec) {
1453 .iov_base = tail_buf,
1454 .iov_len = align,
1455 };
1456 qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
1457
1458 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1459 ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
1460 align, &tail_qiov, 0);
1461 if (ret < 0) {
1462 goto fail;
1463 }
1464 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1465
1466 if (!use_local_qiov) {
1467 qemu_iovec_init(&local_qiov, qiov->niov + 1);
1468 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1469 use_local_qiov = true;
1470 }
1471
1472 tail_bytes = (offset + bytes) & (align - 1);
1473 qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
1474
1475 bytes = ROUND_UP(bytes, align);
1476 }
1477
1478 ret = bdrv_aligned_pwritev(bs, &req, offset, bytes,
1479 use_local_qiov ? &local_qiov : qiov,
1480 flags);
1481
1482 fail:
1483
1484 if (use_local_qiov) {
1485 qemu_iovec_destroy(&local_qiov);
1486 }
1487 qemu_vfree(head_buf);
1488 qemu_vfree(tail_buf);
1489 out:
1490 tracked_request_end(&req);
1491 return ret;
1492 }
1493
1494 static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
1495 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1496 BdrvRequestFlags flags)
1497 {
1498 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1499 return -EINVAL;
1500 }
1501
1502 return bdrv_co_pwritev(bs, sector_num << BDRV_SECTOR_BITS,
1503 nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1504 }
1505
1506 int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
1507 int nb_sectors, QEMUIOVector *qiov)
1508 {
1509 trace_bdrv_co_writev(bs, sector_num, nb_sectors);
1510
1511 return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0);
1512 }
1513
1514 int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
1515 int64_t sector_num, int nb_sectors,
1516 BdrvRequestFlags flags)
1517 {
1518 trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags);
1519
1520 if (!(bs->open_flags & BDRV_O_UNMAP)) {
1521 flags &= ~BDRV_REQ_MAY_UNMAP;
1522 }
1523
1524 return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL,
1525 BDRV_REQ_ZERO_WRITE | flags);
1526 }
1527
1528 typedef struct BdrvCoGetBlockStatusData {
1529 BlockDriverState *bs;
1530 BlockDriverState *base;
1531 BlockDriverState **file;
1532 int64_t sector_num;
1533 int nb_sectors;
1534 int *pnum;
1535 int64_t ret;
1536 bool done;
1537 } BdrvCoGetBlockStatusData;
1538
1539 /*
1540 * Returns the allocation status of the specified sectors.
1541 * Drivers not implementing the functionality are assumed to not support
1542 * backing files, hence all their sectors are reported as allocated.
1543 *
1544 * If 'sector_num' is beyond the end of the disk image the return value is 0
1545 * and 'pnum' is set to 0.
1546 *
1547 * 'pnum' is set to the number of sectors (including and immediately following
1548 * the specified sector) that are known to be in the same
1549 * allocated/unallocated state.
1550 *
1551 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes
1552 * beyond the end of the disk image it will be clamped.
1553 *
1554 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
1555 * points to the BDS which the sector range is allocated in.
1556 */
1557 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
1558 int64_t sector_num,
1559 int nb_sectors, int *pnum,
1560 BlockDriverState **file)
1561 {
1562 int64_t total_sectors;
1563 int64_t n;
1564 int64_t ret, ret2;
1565
1566 total_sectors = bdrv_nb_sectors(bs);
1567 if (total_sectors < 0) {
1568 return total_sectors;
1569 }
1570
1571 if (sector_num >= total_sectors) {
1572 *pnum = 0;
1573 return 0;
1574 }
1575
1576 n = total_sectors - sector_num;
1577 if (n < nb_sectors) {
1578 nb_sectors = n;
1579 }
1580
1581 if (!bs->drv->bdrv_co_get_block_status) {
1582 *pnum = nb_sectors;
1583 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
1584 if (bs->drv->protocol_name) {
1585 ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
1586 }
1587 return ret;
1588 }
1589
1590 *file = NULL;
1591 ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
1592 file);
1593 if (ret < 0) {
1594 *pnum = 0;
1595 return ret;
1596 }
1597
1598 if (ret & BDRV_BLOCK_RAW) {
1599 assert(ret & BDRV_BLOCK_OFFSET_VALID);
1600 return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
1601 *pnum, pnum, file);
1602 }
1603
1604 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
1605 ret |= BDRV_BLOCK_ALLOCATED;
1606 } else {
1607 if (bdrv_unallocated_blocks_are_zero(bs)) {
1608 ret |= BDRV_BLOCK_ZERO;
1609 } else if (bs->backing) {
1610 BlockDriverState *bs2 = bs->backing->bs;
1611 int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
1612 if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
1613 ret |= BDRV_BLOCK_ZERO;
1614 }
1615 }
1616 }
1617
1618 if (*file && *file != bs &&
1619 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
1620 (ret & BDRV_BLOCK_OFFSET_VALID)) {
1621 BlockDriverState *file2;
1622 int file_pnum;
1623
1624 ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
1625 *pnum, &file_pnum, &file2);
1626 if (ret2 >= 0) {
1627 /* Ignore errors. This is just providing extra information, it
1628 * is useful but not necessary.
1629 */
1630 if (!file_pnum) {
1631 /* !file_pnum indicates an offset at or beyond the EOF; it is
1632 * perfectly valid for the format block driver to point to such
1633 * offsets, so catch it and mark everything as zero */
1634 ret |= BDRV_BLOCK_ZERO;
1635 } else {
1636 /* Limit request to the range reported by the protocol driver */
1637 *pnum = file_pnum;
1638 ret |= (ret2 & BDRV_BLOCK_ZERO);
1639 }
1640 }
1641 }
1642
1643 return ret;
1644 }
1645
1646 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
1647 BlockDriverState *base,
1648 int64_t sector_num,
1649 int nb_sectors,
1650 int *pnum,
1651 BlockDriverState **file)
1652 {
1653 BlockDriverState *p;
1654 int64_t ret = 0;
1655
1656 assert(bs != base);
1657 for (p = bs; p != base; p = backing_bs(p)) {
1658 ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
1659 if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
1660 break;
1661 }
1662 /* [sector_num, pnum] unallocated on this layer, which could be only
1663 * the first part of [sector_num, nb_sectors]. */
1664 nb_sectors = MIN(nb_sectors, *pnum);
1665 }
1666 return ret;
1667 }
1668
1669 /* Coroutine wrapper for bdrv_get_block_status_above() */
1670 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
1671 {
1672 BdrvCoGetBlockStatusData *data = opaque;
1673
1674 data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
1675 data->sector_num,
1676 data->nb_sectors,
1677 data->pnum,
1678 data->file);
1679 data->done = true;
1680 }
1681
1682 /*
1683 * Synchronous wrapper around bdrv_co_get_block_status_above().
1684 *
1685 * See bdrv_co_get_block_status_above() for details.
1686 */
1687 int64_t bdrv_get_block_status_above(BlockDriverState *bs,
1688 BlockDriverState *base,
1689 int64_t sector_num,
1690 int nb_sectors, int *pnum,
1691 BlockDriverState **file)
1692 {
1693 Coroutine *co;
1694 BdrvCoGetBlockStatusData data = {
1695 .bs = bs,
1696 .base = base,
1697 .file = file,
1698 .sector_num = sector_num,
1699 .nb_sectors = nb_sectors,
1700 .pnum = pnum,
1701 .done = false,
1702 };
1703
1704 if (qemu_in_coroutine()) {
1705 /* Fast-path if already in coroutine context */
1706 bdrv_get_block_status_above_co_entry(&data);
1707 } else {
1708 AioContext *aio_context = bdrv_get_aio_context(bs);
1709
1710 co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry);
1711 qemu_coroutine_enter(co, &data);
1712 while (!data.done) {
1713 aio_poll(aio_context, true);
1714 }
1715 }
1716 return data.ret;
1717 }
1718
1719 int64_t bdrv_get_block_status(BlockDriverState *bs,
1720 int64_t sector_num,
1721 int nb_sectors, int *pnum,
1722 BlockDriverState **file)
1723 {
1724 return bdrv_get_block_status_above(bs, backing_bs(bs),
1725 sector_num, nb_sectors, pnum, file);
1726 }
1727
1728 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
1729 int nb_sectors, int *pnum)
1730 {
1731 BlockDriverState *file;
1732 int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
1733 &file);
1734 if (ret < 0) {
1735 return ret;
1736 }
1737 return !!(ret & BDRV_BLOCK_ALLOCATED);
1738 }
1739
1740 /*
1741 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
1742 *
1743 * Return true if the given sector is allocated in any image between
1744 * BASE and TOP (inclusive). BASE can be NULL to check if the given
1745 * sector is allocated in any image of the chain. Return false otherwise.
1746 *
1747 * 'pnum' is set to the number of sectors (including and immediately following
1748 * the specified sector) that are known to be in the same
1749 * allocated/unallocated state.
1750 *
1751 */
1752 int bdrv_is_allocated_above(BlockDriverState *top,
1753 BlockDriverState *base,
1754 int64_t sector_num,
1755 int nb_sectors, int *pnum)
1756 {
1757 BlockDriverState *intermediate;
1758 int ret, n = nb_sectors;
1759
1760 intermediate = top;
1761 while (intermediate && intermediate != base) {
1762 int pnum_inter;
1763 ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
1764 &pnum_inter);
1765 if (ret < 0) {
1766 return ret;
1767 } else if (ret) {
1768 *pnum = pnum_inter;
1769 return 1;
1770 }
1771
1772 /*
1773 * [sector_num, nb_sectors] is unallocated on top but intermediate
1774 * might have
1775 *
1776 * [sector_num+x, nr_sectors] allocated.
1777 */
1778 if (n > pnum_inter &&
1779 (intermediate == top ||
1780 sector_num + pnum_inter < intermediate->total_sectors)) {
1781 n = pnum_inter;
1782 }
1783
1784 intermediate = backing_bs(intermediate);
1785 }
1786
1787 *pnum = n;
1788 return 0;
1789 }
1790
1791 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
1792 const uint8_t *buf, int nb_sectors)
1793 {
1794 BlockDriver *drv = bs->drv;
1795 int ret;
1796
1797 if (!drv) {
1798 return -ENOMEDIUM;
1799 }
1800 if (!drv->bdrv_write_compressed) {
1801 return -ENOTSUP;
1802 }
1803 ret = bdrv_check_request(bs, sector_num, nb_sectors);
1804 if (ret < 0) {
1805 return ret;
1806 }
1807
1808 assert(QLIST_EMPTY(&bs->dirty_bitmaps));
1809
1810 return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
1811 }
1812
1813 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
1814 int64_t pos, int size)
1815 {
1816 QEMUIOVector qiov;
1817 struct iovec iov = {
1818 .iov_base = (void *) buf,
1819 .iov_len = size,
1820 };
1821
1822 qemu_iovec_init_external(&qiov, &iov, 1);
1823 return bdrv_writev_vmstate(bs, &qiov, pos);
1824 }
1825
1826 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
1827 {
1828 BlockDriver *drv = bs->drv;
1829
1830 if (!drv) {
1831 return -ENOMEDIUM;
1832 } else if (drv->bdrv_save_vmstate) {
1833 return drv->bdrv_save_vmstate(bs, qiov, pos);
1834 } else if (bs->file) {
1835 return bdrv_writev_vmstate(bs->file->bs, qiov, pos);
1836 }
1837
1838 return -ENOTSUP;
1839 }
1840
1841 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
1842 int64_t pos, int size)
1843 {
1844 BlockDriver *drv = bs->drv;
1845 if (!drv)
1846 return -ENOMEDIUM;
1847 if (drv->bdrv_load_vmstate)
1848 return drv->bdrv_load_vmstate(bs, buf, pos, size);
1849 if (bs->file)
1850 return bdrv_load_vmstate(bs->file->bs, buf, pos, size);
1851 return -ENOTSUP;
1852 }
1853
1854 /**************************************************************/
1855 /* async I/Os */
1856
1857 BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
1858 QEMUIOVector *qiov, int nb_sectors,
1859 BlockCompletionFunc *cb, void *opaque)
1860 {
1861 trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
1862
1863 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1864 cb, opaque, false);
1865 }
1866
1867 BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
1868 QEMUIOVector *qiov, int nb_sectors,
1869 BlockCompletionFunc *cb, void *opaque)
1870 {
1871 trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
1872
1873 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1874 cb, opaque, true);
1875 }
1876
1877 BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs,
1878 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags,
1879 BlockCompletionFunc *cb, void *opaque)
1880 {
1881 trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque);
1882
1883 return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors,
1884 BDRV_REQ_ZERO_WRITE | flags,
1885 cb, opaque, true);
1886 }
1887
1888
1889 typedef struct MultiwriteCB {
1890 int error;
1891 int num_requests;
1892 int num_callbacks;
1893 struct {
1894 BlockCompletionFunc *cb;
1895 void *opaque;
1896 QEMUIOVector *free_qiov;
1897 } callbacks[];
1898 } MultiwriteCB;
1899
1900 static void multiwrite_user_cb(MultiwriteCB *mcb)
1901 {
1902 int i;
1903
1904 for (i = 0; i < mcb->num_callbacks; i++) {
1905 mcb->callbacks[i].cb(mcb->callbacks[i].opaque, mcb->error);
1906 if (mcb->callbacks[i].free_qiov) {
1907 qemu_iovec_destroy(mcb->callbacks[i].free_qiov);
1908 }
1909 g_free(mcb->callbacks[i].free_qiov);
1910 }
1911 }
1912
1913 static void multiwrite_cb(void *opaque, int ret)
1914 {
1915 MultiwriteCB *mcb = opaque;
1916
1917 trace_multiwrite_cb(mcb, ret);
1918
1919 if (ret < 0 && !mcb->error) {
1920 mcb->error = ret;
1921 }
1922
1923 mcb->num_requests--;
1924 if (mcb->num_requests == 0) {
1925 multiwrite_user_cb(mcb);
1926 g_free(mcb);
1927 }
1928 }
1929
1930 static int multiwrite_req_compare(const void *a, const void *b)
1931 {
1932 const BlockRequest *req1 = a, *req2 = b;
1933
1934 /*
1935 * Note that we can't simply subtract req2->sector from req1->sector
1936 * here as that could overflow the return value.
1937 */
1938 if (req1->sector > req2->sector) {
1939 return 1;
1940 } else if (req1->sector < req2->sector) {
1941 return -1;
1942 } else {
1943 return 0;
1944 }
1945 }
1946
1947 /*
1948 * Takes a bunch of requests and tries to merge them. Returns the number of
1949 * requests that remain after merging.
1950 */
1951 static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
1952 int num_reqs, MultiwriteCB *mcb)
1953 {
1954 int i, outidx;
1955
1956 // Sort requests by start sector
1957 qsort(reqs, num_reqs, sizeof(*reqs), &multiwrite_req_compare);
1958
1959 // Check if adjacent requests touch the same clusters. If so, combine them,
1960 // filling up gaps with zero sectors.
1961 outidx = 0;
1962 for (i = 1; i < num_reqs; i++) {
1963 int merge = 0;
1964 int64_t oldreq_last = reqs[outidx].sector + reqs[outidx].nb_sectors;
1965
1966 // Handle exactly sequential writes and overlapping writes.
1967 if (reqs[i].sector <= oldreq_last) {
1968 merge = 1;
1969 }
1970
1971 if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 >
1972 bs->bl.max_iov) {
1973 merge = 0;
1974 }
1975
1976 if (bs->bl.max_transfer_length && reqs[outidx].nb_sectors +
1977 reqs[i].nb_sectors > bs->bl.max_transfer_length) {
1978 merge = 0;
1979 }
1980
1981 if (merge) {
1982 size_t size;
1983 QEMUIOVector *qiov = g_malloc0(sizeof(*qiov));
1984 qemu_iovec_init(qiov,
1985 reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1);
1986
1987 // Add the first request to the merged one. If the requests are
1988 // overlapping, drop the last sectors of the first request.
1989 size = (reqs[i].sector - reqs[outidx].sector) << 9;
1990 qemu_iovec_concat(qiov, reqs[outidx].qiov, 0, size);
1991
1992 // We should need to add any zeros between the two requests
1993 assert (reqs[i].sector <= oldreq_last);
1994
1995 // Add the second request
1996 qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size);
1997
1998 // Add tail of first request, if necessary
1999 if (qiov->size < reqs[outidx].qiov->size) {
2000 qemu_iovec_concat(qiov, reqs[outidx].qiov, qiov->size,
2001 reqs[outidx].qiov->size - qiov->size);
2002 }
2003
2004 reqs[outidx].nb_sectors = qiov->size >> 9;
2005 reqs[outidx].qiov = qiov;
2006
2007 mcb->callbacks[i].free_qiov = reqs[outidx].qiov;
2008 } else {
2009 outidx++;
2010 reqs[outidx].sector = reqs[i].sector;
2011 reqs[outidx].nb_sectors = reqs[i].nb_sectors;
2012 reqs[outidx].qiov = reqs[i].qiov;
2013 }
2014 }
2015
2016 if (bs->blk) {
2017 block_acct_merge_done(blk_get_stats(bs->blk), BLOCK_ACCT_WRITE,
2018 num_reqs - outidx - 1);
2019 }
2020
2021 return outidx + 1;
2022 }
2023
2024 /*
2025 * Submit multiple AIO write requests at once.
2026 *
2027 * On success, the function returns 0 and all requests in the reqs array have
2028 * been submitted. In error case this function returns -1, and any of the
2029 * requests may or may not be submitted yet. In particular, this means that the
2030 * callback will be called for some of the requests, for others it won't. The
2031 * caller must check the error field of the BlockRequest to wait for the right
2032 * callbacks (if error != 0, no callback will be called).
2033 *
2034 * The implementation may modify the contents of the reqs array, e.g. to merge
2035 * requests. However, the fields opaque and error are left unmodified as they
2036 * are used to signal failure for a single request to the caller.
2037 */
2038 int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs)
2039 {
2040 MultiwriteCB *mcb;
2041 int i;
2042
2043 /* don't submit writes if we don't have a medium */
2044 if (bs->drv == NULL) {
2045 for (i = 0; i < num_reqs; i++) {
2046 reqs[i].error = -ENOMEDIUM;
2047 }
2048 return -1;
2049 }
2050
2051 if (num_reqs == 0) {
2052 return 0;
2053 }
2054
2055 // Create MultiwriteCB structure
2056 mcb = g_malloc0(sizeof(*mcb) + num_reqs * sizeof(*mcb->callbacks));
2057 mcb->num_requests = 0;
2058 mcb->num_callbacks = num_reqs;
2059
2060 for (i = 0; i < num_reqs; i++) {
2061 mcb->callbacks[i].cb = reqs[i].cb;
2062 mcb->callbacks[i].opaque = reqs[i].opaque;
2063 }
2064
2065 // Check for mergable requests
2066 num_reqs = multiwrite_merge(bs, reqs, num_reqs, mcb);
2067
2068 trace_bdrv_aio_multiwrite(mcb, mcb->num_callbacks, num_reqs);
2069
2070 /* Run the aio requests. */
2071 mcb->num_requests = num_reqs;
2072 for (i = 0; i < num_reqs; i++) {
2073 bdrv_co_aio_rw_vector(bs, reqs[i].sector, reqs[i].qiov,
2074 reqs[i].nb_sectors, reqs[i].flags,
2075 multiwrite_cb, mcb,
2076 true);
2077 }
2078
2079 return 0;
2080 }
2081
2082 void bdrv_aio_cancel(BlockAIOCB *acb)
2083 {
2084 qemu_aio_ref(acb);
2085 bdrv_aio_cancel_async(acb);
2086 while (acb->refcnt > 1) {
2087 if (acb->aiocb_info->get_aio_context) {
2088 aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2089 } else if (acb->bs) {
2090 aio_poll(bdrv_get_aio_context(acb->bs), true);
2091 } else {
2092 abort();
2093 }
2094 }
2095 qemu_aio_unref(acb);
2096 }
2097
2098 /* Async version of aio cancel. The caller is not blocked if the acb implements
2099 * cancel_async, otherwise we do nothing and let the request normally complete.
2100 * In either case the completion callback must be called. */
2101 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2102 {
2103 if (acb->aiocb_info->cancel_async) {
2104 acb->aiocb_info->cancel_async(acb);
2105 }
2106 }
2107
2108 /**************************************************************/
2109 /* async block device emulation */
2110
2111 typedef struct BlockAIOCBCoroutine {
2112 BlockAIOCB common;
2113 BlockRequest req;
2114 bool is_write;
2115 bool need_bh;
2116 bool *done;
2117 QEMUBH* bh;
2118 } BlockAIOCBCoroutine;
2119
2120 static const AIOCBInfo bdrv_em_co_aiocb_info = {
2121 .aiocb_size = sizeof(BlockAIOCBCoroutine),
2122 };
2123
2124 static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
2125 {
2126 if (!acb->need_bh) {
2127 acb->common.cb(acb->common.opaque, acb->req.error);
2128 qemu_aio_unref(acb);
2129 }
2130 }
2131
2132 static void bdrv_co_em_bh(void *opaque)
2133 {
2134 BlockAIOCBCoroutine *acb = opaque;
2135
2136 assert(!acb->need_bh);
2137 qemu_bh_delete(acb->bh);
2138 bdrv_co_complete(acb);
2139 }
2140
2141 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
2142 {
2143 acb->need_bh = false;
2144 if (acb->req.error != -EINPROGRESS) {
2145 BlockDriverState *bs = acb->common.bs;
2146
2147 acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
2148 qemu_bh_schedule(acb->bh);
2149 }
2150 }
2151
2152 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
2153 static void coroutine_fn bdrv_co_do_rw(void *opaque)
2154 {
2155 BlockAIOCBCoroutine *acb = opaque;
2156 BlockDriverState *bs = acb->common.bs;
2157
2158 if (!acb->is_write) {
2159 acb->req.error = bdrv_co_do_readv(bs, acb->req.sector,
2160 acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
2161 } else {
2162 acb->req.error = bdrv_co_do_writev(bs, acb->req.sector,
2163 acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
2164 }
2165
2166 bdrv_co_complete(acb);
2167 }
2168
2169 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
2170 int64_t sector_num,
2171 QEMUIOVector *qiov,
2172 int nb_sectors,
2173 BdrvRequestFlags flags,
2174 BlockCompletionFunc *cb,
2175 void *opaque,
2176 bool is_write)
2177 {
2178 Coroutine *co;
2179 BlockAIOCBCoroutine *acb;
2180
2181 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2182 acb->need_bh = true;
2183 acb->req.error = -EINPROGRESS;
2184 acb->req.sector = sector_num;
2185 acb->req.nb_sectors = nb_sectors;
2186 acb->req.qiov = qiov;
2187 acb->req.flags = flags;
2188 acb->is_write = is_write;
2189
2190 co = qemu_coroutine_create(bdrv_co_do_rw);
2191 qemu_coroutine_enter(co, acb);
2192
2193 bdrv_co_maybe_schedule_bh(acb);
2194 return &acb->common;
2195 }
2196
2197 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
2198 {
2199 BlockAIOCBCoroutine *acb = opaque;
2200 BlockDriverState *bs = acb->common.bs;
2201
2202 acb->req.error = bdrv_co_flush(bs);
2203 bdrv_co_complete(acb);
2204 }
2205
2206 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
2207 BlockCompletionFunc *cb, void *opaque)
2208 {
2209 trace_bdrv_aio_flush(bs, opaque);
2210
2211 Coroutine *co;
2212 BlockAIOCBCoroutine *acb;
2213
2214 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2215 acb->need_bh = true;
2216 acb->req.error = -EINPROGRESS;
2217
2218 co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
2219 qemu_coroutine_enter(co, acb);
2220
2221 bdrv_co_maybe_schedule_bh(acb);
2222 return &acb->common;
2223 }
2224
2225 static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
2226 {
2227 BlockAIOCBCoroutine *acb = opaque;
2228 BlockDriverState *bs = acb->common.bs;
2229
2230 acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
2231 bdrv_co_complete(acb);
2232 }
2233
2234 BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
2235 int64_t sector_num, int nb_sectors,
2236 BlockCompletionFunc *cb, void *opaque)
2237 {
2238 Coroutine *co;
2239 BlockAIOCBCoroutine *acb;
2240
2241 trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
2242
2243 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2244 acb->need_bh = true;
2245 acb->req.error = -EINPROGRESS;
2246 acb->req.sector = sector_num;
2247 acb->req.nb_sectors = nb_sectors;
2248 co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
2249 qemu_coroutine_enter(co, acb);
2250
2251 bdrv_co_maybe_schedule_bh(acb);
2252 return &acb->common;
2253 }
2254
2255 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
2256 BlockCompletionFunc *cb, void *opaque)
2257 {
2258 BlockAIOCB *acb;
2259
2260 acb = g_malloc(aiocb_info->aiocb_size);
2261 acb->aiocb_info = aiocb_info;
2262 acb->bs = bs;
2263 acb->cb = cb;
2264 acb->opaque = opaque;
2265 acb->refcnt = 1;
2266 return acb;
2267 }
2268
2269 void qemu_aio_ref(void *p)
2270 {
2271 BlockAIOCB *acb = p;
2272 acb->refcnt++;
2273 }
2274
2275 void qemu_aio_unref(void *p)
2276 {
2277 BlockAIOCB *acb = p;
2278 assert(acb->refcnt > 0);
2279 if (--acb->refcnt == 0) {
2280 g_free(acb);
2281 }
2282 }
2283
2284 /**************************************************************/
2285 /* Coroutine block device emulation */
2286
2287 static void coroutine_fn bdrv_flush_co_entry(void *opaque)
2288 {
2289 RwCo *rwco = opaque;
2290
2291 rwco->ret = bdrv_co_flush(rwco->bs);
2292 }
2293
2294 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2295 {
2296 int ret;
2297 BdrvTrackedRequest req;
2298
2299 if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2300 bdrv_is_sg(bs)) {
2301 return 0;
2302 }
2303
2304 tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
2305
2306 /* Write back all layers by calling one driver function */
2307 if (bs->drv->bdrv_co_flush) {
2308 ret = bs->drv->bdrv_co_flush(bs);
2309 goto out;
2310 }
2311
2312 /* Write back cached data to the OS even with cache=unsafe */
2313 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
2314 if (bs->drv->bdrv_co_flush_to_os) {
2315 ret = bs->drv->bdrv_co_flush_to_os(bs);
2316 if (ret < 0) {
2317 goto out;
2318 }
2319 }
2320
2321 /* But don't actually force it to the disk with cache=unsafe */
2322 if (bs->open_flags & BDRV_O_NO_FLUSH) {
2323 goto flush_parent;
2324 }
2325
2326 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
2327 if (bs->drv->bdrv_co_flush_to_disk) {
2328 ret = bs->drv->bdrv_co_flush_to_disk(bs);
2329 } else if (bs->drv->bdrv_aio_flush) {
2330 BlockAIOCB *acb;
2331 CoroutineIOCompletion co = {
2332 .coroutine = qemu_coroutine_self(),
2333 };
2334
2335 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2336 if (acb == NULL) {
2337 ret = -EIO;
2338 } else {
2339 qemu_coroutine_yield();
2340 ret = co.ret;
2341 }
2342 } else {
2343 /*
2344 * Some block drivers always operate in either writethrough or unsafe
2345 * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2346 * know how the server works (because the behaviour is hardcoded or
2347 * depends on server-side configuration), so we can't ensure that
2348 * everything is safe on disk. Returning an error doesn't work because
2349 * that would break guests even if the server operates in writethrough
2350 * mode.
2351 *
2352 * Let's hope the user knows what he's doing.
2353 */
2354 ret = 0;
2355 }
2356 if (ret < 0) {
2357 goto out;
2358 }
2359
2360 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH
2361 * in the case of cache=unsafe, so there are no useless flushes.
2362 */
2363 flush_parent:
2364 ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2365 out:
2366 tracked_request_end(&req);
2367 return ret;
2368 }
2369
2370 int bdrv_flush(BlockDriverState *bs)
2371 {
2372 Coroutine *co;
2373 RwCo rwco = {
2374 .bs = bs,
2375 .ret = NOT_DONE,
2376 };
2377
2378 if (qemu_in_coroutine()) {
2379 /* Fast-path if already in coroutine context */
2380 bdrv_flush_co_entry(&rwco);
2381 } else {
2382 AioContext *aio_context = bdrv_get_aio_context(bs);
2383
2384 co = qemu_coroutine_create(bdrv_flush_co_entry);
2385 qemu_coroutine_enter(co, &rwco);
2386 while (rwco.ret == NOT_DONE) {
2387 aio_poll(aio_context, true);
2388 }
2389 }
2390
2391 return rwco.ret;
2392 }
2393
2394 typedef struct DiscardCo {
2395 BlockDriverState *bs;
2396 int64_t sector_num;
2397 int nb_sectors;
2398 int ret;
2399 } DiscardCo;
2400 static void coroutine_fn bdrv_discard_co_entry(void *opaque)
2401 {
2402 DiscardCo *rwco = opaque;
2403
2404 rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors);
2405 }
2406
2407 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
2408 int nb_sectors)
2409 {
2410 BdrvTrackedRequest req;
2411 int max_discard, ret;
2412
2413 if (!bs->drv) {
2414 return -ENOMEDIUM;
2415 }
2416
2417 ret = bdrv_check_request(bs, sector_num, nb_sectors);
2418 if (ret < 0) {
2419 return ret;
2420 } else if (bs->read_only) {
2421 return -EPERM;
2422 }
2423 assert(!(bs->open_flags & BDRV_O_INACTIVE));
2424
2425 /* Do nothing if disabled. */
2426 if (!(bs->open_flags & BDRV_O_UNMAP)) {
2427 return 0;
2428 }
2429
2430 if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) {
2431 return 0;
2432 }
2433
2434 tracked_request_begin(&req, bs, sector_num, nb_sectors,
2435 BDRV_TRACKED_DISCARD);
2436 bdrv_set_dirty(bs, sector_num, nb_sectors);
2437
2438 max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS);
2439 while (nb_sectors > 0) {
2440 int ret;
2441 int num = nb_sectors;
2442
2443 /* align request */
2444 if (bs->bl.discard_alignment &&
2445 num >= bs->bl.discard_alignment &&
2446 sector_num % bs->bl.discard_alignment) {
2447 if (num > bs->bl.discard_alignment) {
2448 num = bs->bl.discard_alignment;
2449 }
2450 num -= sector_num % bs->bl.discard_alignment;
2451 }
2452
2453 /* limit request size */
2454 if (num > max_discard) {
2455 num = max_discard;
2456 }
2457
2458 if (bs->drv->bdrv_co_discard) {
2459 ret = bs->drv->bdrv_co_discard(bs, sector_num, num);
2460 } else {
2461 BlockAIOCB *acb;
2462 CoroutineIOCompletion co = {
2463 .coroutine = qemu_coroutine_self(),
2464 };
2465
2466 acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
2467 bdrv_co_io_em_complete, &co);
2468 if (acb == NULL) {
2469 ret = -EIO;
2470 goto out;
2471 } else {
2472 qemu_coroutine_yield();
2473 ret = co.ret;
2474 }
2475 }
2476 if (ret && ret != -ENOTSUP) {
2477 goto out;
2478 }
2479
2480 sector_num += num;
2481 nb_sectors -= num;
2482 }
2483 ret = 0;
2484 out:
2485 tracked_request_end(&req);
2486 return ret;
2487 }
2488
2489 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
2490 {
2491 Coroutine *co;
2492 DiscardCo rwco = {
2493 .bs = bs,
2494 .sector_num = sector_num,
2495 .nb_sectors = nb_sectors,
2496 .ret = NOT_DONE,
2497 };
2498
2499 if (qemu_in_coroutine()) {
2500 /* Fast-path if already in coroutine context */
2501 bdrv_discard_co_entry(&rwco);
2502 } else {
2503 AioContext *aio_context = bdrv_get_aio_context(bs);
2504
2505 co = qemu_coroutine_create(bdrv_discard_co_entry);
2506 qemu_coroutine_enter(co, &rwco);
2507 while (rwco.ret == NOT_DONE) {
2508 aio_poll(aio_context, true);
2509 }
2510 }
2511
2512 return rwco.ret;
2513 }
2514
2515 typedef struct {
2516 CoroutineIOCompletion *co;
2517 QEMUBH *bh;
2518 } BdrvIoctlCompletionData;
2519
2520 static void bdrv_ioctl_bh_cb(void *opaque)
2521 {
2522 BdrvIoctlCompletionData *data = opaque;
2523
2524 bdrv_co_io_em_complete(data->co, -ENOTSUP);
2525 qemu_bh_delete(data->bh);
2526 }
2527
2528 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
2529 {
2530 BlockDriver *drv = bs->drv;
2531 BdrvTrackedRequest tracked_req;
2532 CoroutineIOCompletion co = {
2533 .coroutine = qemu_coroutine_self(),
2534 };
2535 BlockAIOCB *acb;
2536
2537 tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
2538 if (!drv || !drv->bdrv_aio_ioctl) {
2539 co.ret = -ENOTSUP;
2540 goto out;
2541 }
2542
2543 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
2544 if (!acb) {
2545 BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1);
2546 data->bh = aio_bh_new(bdrv_get_aio_context(bs),
2547 bdrv_ioctl_bh_cb, data);
2548 data->co = &co;
2549 qemu_bh_schedule(data->bh);
2550 }
2551 qemu_coroutine_yield();
2552 out:
2553 tracked_request_end(&tracked_req);
2554 return co.ret;
2555 }
2556
2557 typedef struct {
2558 BlockDriverState *bs;
2559 int req;
2560 void *buf;
2561 int ret;
2562 } BdrvIoctlCoData;
2563
2564 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
2565 {
2566 BdrvIoctlCoData *data = opaque;
2567 data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
2568 }
2569
2570 /* needed for generic scsi interface */
2571 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
2572 {
2573 BdrvIoctlCoData data = {
2574 .bs = bs,
2575 .req = req,
2576 .buf = buf,
2577 .ret = -EINPROGRESS,
2578 };
2579
2580 if (qemu_in_coroutine()) {
2581 /* Fast-path if already in coroutine context */
2582 bdrv_co_ioctl_entry(&data);
2583 } else {
2584 Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry);
2585
2586 qemu_coroutine_enter(co, &data);
2587 while (data.ret == -EINPROGRESS) {
2588 aio_poll(bdrv_get_aio_context(bs), true);
2589 }
2590 }
2591 return data.ret;
2592 }
2593
2594 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
2595 {
2596 BlockAIOCBCoroutine *acb = opaque;
2597 acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
2598 acb->req.req, acb->req.buf);
2599 bdrv_co_complete(acb);
2600 }
2601
2602 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
2603 unsigned long int req, void *buf,
2604 BlockCompletionFunc *cb, void *opaque)
2605 {
2606 BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
2607 bs, cb, opaque);
2608 Coroutine *co;
2609
2610 acb->need_bh = true;
2611 acb->req.error = -EINPROGRESS;
2612 acb->req.req = req;
2613 acb->req.buf = buf;
2614 co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry);
2615 qemu_coroutine_enter(co, acb);
2616
2617 bdrv_co_maybe_schedule_bh(acb);
2618 return &acb->common;
2619 }
2620
2621 void *qemu_blockalign(BlockDriverState *bs, size_t size)
2622 {
2623 return qemu_memalign(bdrv_opt_mem_align(bs), size);
2624 }
2625
2626 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
2627 {
2628 return memset(qemu_blockalign(bs, size), 0, size);
2629 }
2630
2631 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
2632 {
2633 size_t align = bdrv_opt_mem_align(bs);
2634
2635 /* Ensure that NULL is never returned on success */
2636 assert(align > 0);
2637 if (size == 0) {
2638 size = align;
2639 }
2640
2641 return qemu_try_memalign(align, size);
2642 }
2643
2644 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
2645 {
2646 void *mem = qemu_try_blockalign(bs, size);
2647
2648 if (mem) {
2649 memset(mem, 0, size);
2650 }
2651
2652 return mem;
2653 }
2654
2655 /*
2656 * Check if all memory in this vector is sector aligned.
2657 */
2658 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
2659 {
2660 int i;
2661 size_t alignment = bdrv_min_mem_align(bs);
2662
2663 for (i = 0; i < qiov->niov; i++) {
2664 if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
2665 return false;
2666 }
2667 if (qiov->iov[i].iov_len % alignment) {
2668 return false;
2669 }
2670 }
2671
2672 return true;
2673 }
2674
2675 void bdrv_add_before_write_notifier(BlockDriverState *bs,
2676 NotifierWithReturn *notifier)
2677 {
2678 notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
2679 }
2680
2681 void bdrv_io_plug(BlockDriverState *bs)
2682 {
2683 BdrvChild *child;
2684
2685 QLIST_FOREACH(child, &bs->children, next) {
2686 bdrv_io_plug(child->bs);
2687 }
2688
2689 if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) {
2690 BlockDriver *drv = bs->drv;
2691 if (drv && drv->bdrv_io_plug) {
2692 drv->bdrv_io_plug(bs);
2693 }
2694 }
2695 }
2696
2697 void bdrv_io_unplug(BlockDriverState *bs)
2698 {
2699 BdrvChild *child;
2700
2701 assert(bs->io_plugged);
2702 if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) {
2703 BlockDriver *drv = bs->drv;
2704 if (drv && drv->bdrv_io_unplug) {
2705 drv->bdrv_io_unplug(bs);
2706 }
2707 }
2708
2709 QLIST_FOREACH(child, &bs->children, next) {
2710 bdrv_io_unplug(child->bs);
2711 }
2712 }
2713
2714 void bdrv_io_unplugged_begin(BlockDriverState *bs)
2715 {
2716 BdrvChild *child;
2717
2718 if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) {
2719 BlockDriver *drv = bs->drv;
2720 if (drv && drv->bdrv_io_unplug) {
2721 drv->bdrv_io_unplug(bs);
2722 }
2723 }
2724
2725 QLIST_FOREACH(child, &bs->children, next) {
2726 bdrv_io_unplugged_begin(child->bs);
2727 }
2728 }
2729
2730 void bdrv_io_unplugged_end(BlockDriverState *bs)
2731 {
2732 BdrvChild *child;
2733
2734 assert(bs->io_plug_disabled);
2735 QLIST_FOREACH(child, &bs->children, next) {
2736 bdrv_io_unplugged_end(child->bs);
2737 }
2738
2739 if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) {
2740 BlockDriver *drv = bs->drv;
2741 if (drv && drv->bdrv_io_plug) {
2742 drv->bdrv_io_plug(bs);
2743 }
2744 }
2745 }
2746
2747 void bdrv_drained_begin(BlockDriverState *bs)
2748 {
2749 if (!bs->quiesce_counter++) {
2750 aio_disable_external(bdrv_get_aio_context(bs));
2751 }
2752 bdrv_drain(bs);
2753 }
2754
2755 void bdrv_drained_end(BlockDriverState *bs)
2756 {
2757 assert(bs->quiesce_counter > 0);
2758 if (--bs->quiesce_counter > 0) {
2759 return;
2760 }
2761 aio_enable_external(bdrv_get_aio_context(bs));
2762 }