block: Don't check throttled reqs in bdrv_requests_pending()
[qemu.git] / block / io.c
1 /*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/blockjob.h"
29 #include "block/block_int.h"
30 #include "qemu/cutils.h"
31 #include "qapi/error.h"
32 #include "qemu/error-report.h"
33
34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
35
36 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
37 int64_t sector_num,
38 QEMUIOVector *qiov,
39 int nb_sectors,
40 BdrvRequestFlags flags,
41 BlockCompletionFunc *cb,
42 void *opaque,
43 bool is_write);
44 static void coroutine_fn bdrv_co_do_rw(void *opaque);
45 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
46 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags);
47
48 static void bdrv_parent_drained_begin(BlockDriverState *bs)
49 {
50 BdrvChild *c;
51
52 QLIST_FOREACH(c, &bs->parents, next_parent) {
53 if (c->role->drained_begin) {
54 c->role->drained_begin(c);
55 }
56 }
57 }
58
59 static void bdrv_parent_drained_end(BlockDriverState *bs)
60 {
61 BdrvChild *c;
62
63 QLIST_FOREACH(c, &bs->parents, next_parent) {
64 if (c->role->drained_end) {
65 c->role->drained_end(c);
66 }
67 }
68 }
69
70 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
71 {
72 BlockDriver *drv = bs->drv;
73 Error *local_err = NULL;
74
75 memset(&bs->bl, 0, sizeof(bs->bl));
76
77 if (!drv) {
78 return;
79 }
80
81 /* Take some limits from the children as a default */
82 if (bs->file) {
83 bdrv_refresh_limits(bs->file->bs, &local_err);
84 if (local_err) {
85 error_propagate(errp, local_err);
86 return;
87 }
88 bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length;
89 bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length;
90 bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment;
91 bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment;
92 bs->bl.max_iov = bs->file->bs->bl.max_iov;
93 } else {
94 bs->bl.min_mem_alignment = 512;
95 bs->bl.opt_mem_alignment = getpagesize();
96
97 /* Safe default since most protocols use readv()/writev()/etc */
98 bs->bl.max_iov = IOV_MAX;
99 }
100
101 if (bs->backing) {
102 bdrv_refresh_limits(bs->backing->bs, &local_err);
103 if (local_err) {
104 error_propagate(errp, local_err);
105 return;
106 }
107 bs->bl.opt_transfer_length =
108 MAX(bs->bl.opt_transfer_length,
109 bs->backing->bs->bl.opt_transfer_length);
110 bs->bl.max_transfer_length =
111 MIN_NON_ZERO(bs->bl.max_transfer_length,
112 bs->backing->bs->bl.max_transfer_length);
113 bs->bl.opt_mem_alignment =
114 MAX(bs->bl.opt_mem_alignment,
115 bs->backing->bs->bl.opt_mem_alignment);
116 bs->bl.min_mem_alignment =
117 MAX(bs->bl.min_mem_alignment,
118 bs->backing->bs->bl.min_mem_alignment);
119 bs->bl.max_iov =
120 MIN(bs->bl.max_iov,
121 bs->backing->bs->bl.max_iov);
122 }
123
124 /* Then let the driver override it */
125 if (drv->bdrv_refresh_limits) {
126 drv->bdrv_refresh_limits(bs, errp);
127 }
128 }
129
130 /**
131 * The copy-on-read flag is actually a reference count so multiple users may
132 * use the feature without worrying about clobbering its previous state.
133 * Copy-on-read stays enabled until all users have called to disable it.
134 */
135 void bdrv_enable_copy_on_read(BlockDriverState *bs)
136 {
137 bs->copy_on_read++;
138 }
139
140 void bdrv_disable_copy_on_read(BlockDriverState *bs)
141 {
142 assert(bs->copy_on_read > 0);
143 bs->copy_on_read--;
144 }
145
146 /* Check if any requests are in-flight (including throttled requests) */
147 bool bdrv_requests_pending(BlockDriverState *bs)
148 {
149 BdrvChild *child;
150
151 if (!QLIST_EMPTY(&bs->tracked_requests)) {
152 return true;
153 }
154
155 QLIST_FOREACH(child, &bs->children, next) {
156 if (bdrv_requests_pending(child->bs)) {
157 return true;
158 }
159 }
160
161 return false;
162 }
163
164 static void bdrv_drain_recurse(BlockDriverState *bs)
165 {
166 BdrvChild *child;
167
168 if (bs->drv && bs->drv->bdrv_drain) {
169 bs->drv->bdrv_drain(bs);
170 }
171 QLIST_FOREACH(child, &bs->children, next) {
172 bdrv_drain_recurse(child->bs);
173 }
174 }
175
176 typedef struct {
177 Coroutine *co;
178 BlockDriverState *bs;
179 QEMUBH *bh;
180 bool done;
181 } BdrvCoDrainData;
182
183 static void bdrv_drain_poll(BlockDriverState *bs)
184 {
185 bool busy = true;
186
187 while (busy) {
188 /* Keep iterating */
189 busy = bdrv_requests_pending(bs);
190 busy |= aio_poll(bdrv_get_aio_context(bs), busy);
191 }
192 }
193
194 static void bdrv_co_drain_bh_cb(void *opaque)
195 {
196 BdrvCoDrainData *data = opaque;
197 Coroutine *co = data->co;
198
199 qemu_bh_delete(data->bh);
200 bdrv_drain_poll(data->bs);
201 data->done = true;
202 qemu_coroutine_enter(co, NULL);
203 }
204
205 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
206 {
207 BdrvCoDrainData data;
208
209 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
210 * other coroutines run if they were queued from
211 * qemu_co_queue_run_restart(). */
212
213 assert(qemu_in_coroutine());
214 data = (BdrvCoDrainData) {
215 .co = qemu_coroutine_self(),
216 .bs = bs,
217 .done = false,
218 .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data),
219 };
220 qemu_bh_schedule(data.bh);
221
222 qemu_coroutine_yield();
223 /* If we are resumed from some other event (such as an aio completion or a
224 * timer callback), it is a bug in the caller that should be fixed. */
225 assert(data.done);
226 }
227
228 /*
229 * Wait for pending requests to complete on a single BlockDriverState subtree,
230 * and suspend block driver's internal I/O until next request arrives.
231 *
232 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
233 * AioContext.
234 *
235 * Only this BlockDriverState's AioContext is run, so in-flight requests must
236 * not depend on events in other AioContexts. In that case, use
237 * bdrv_drain_all() instead.
238 */
239 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
240 {
241 bdrv_parent_drained_begin(bs);
242 bdrv_io_unplugged_begin(bs);
243 bdrv_drain_recurse(bs);
244 bdrv_co_yield_to_drain(bs);
245 bdrv_io_unplugged_end(bs);
246 bdrv_parent_drained_end(bs);
247 }
248
249 void bdrv_drain(BlockDriverState *bs)
250 {
251 bdrv_parent_drained_begin(bs);
252 bdrv_io_unplugged_begin(bs);
253 bdrv_drain_recurse(bs);
254 if (qemu_in_coroutine()) {
255 bdrv_co_yield_to_drain(bs);
256 } else {
257 bdrv_drain_poll(bs);
258 }
259 bdrv_io_unplugged_end(bs);
260 bdrv_parent_drained_end(bs);
261 }
262
263 /*
264 * Wait for pending requests to complete across all BlockDriverStates
265 *
266 * This function does not flush data to disk, use bdrv_flush_all() for that
267 * after calling this function.
268 */
269 void bdrv_drain_all(void)
270 {
271 /* Always run first iteration so any pending completion BHs run */
272 bool busy = true;
273 BlockDriverState *bs = NULL;
274 GSList *aio_ctxs = NULL, *ctx;
275
276 while ((bs = bdrv_next(bs))) {
277 AioContext *aio_context = bdrv_get_aio_context(bs);
278
279 aio_context_acquire(aio_context);
280 if (bs->job) {
281 block_job_pause(bs->job);
282 }
283 bdrv_parent_drained_begin(bs);
284 bdrv_io_unplugged_begin(bs);
285 bdrv_drain_recurse(bs);
286 aio_context_release(aio_context);
287
288 if (!g_slist_find(aio_ctxs, aio_context)) {
289 aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
290 }
291 }
292
293 /* Note that completion of an asynchronous I/O operation can trigger any
294 * number of other I/O operations on other devices---for example a
295 * coroutine can submit an I/O request to another device in response to
296 * request completion. Therefore we must keep looping until there was no
297 * more activity rather than simply draining each device independently.
298 */
299 while (busy) {
300 busy = false;
301
302 for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
303 AioContext *aio_context = ctx->data;
304 bs = NULL;
305
306 aio_context_acquire(aio_context);
307 while ((bs = bdrv_next(bs))) {
308 if (aio_context == bdrv_get_aio_context(bs)) {
309 if (bdrv_requests_pending(bs)) {
310 busy = true;
311 aio_poll(aio_context, busy);
312 }
313 }
314 }
315 busy |= aio_poll(aio_context, false);
316 aio_context_release(aio_context);
317 }
318 }
319
320 bs = NULL;
321 while ((bs = bdrv_next(bs))) {
322 AioContext *aio_context = bdrv_get_aio_context(bs);
323
324 aio_context_acquire(aio_context);
325 bdrv_io_unplugged_end(bs);
326 bdrv_parent_drained_end(bs);
327 if (bs->job) {
328 block_job_resume(bs->job);
329 }
330 aio_context_release(aio_context);
331 }
332 g_slist_free(aio_ctxs);
333 }
334
335 /**
336 * Remove an active request from the tracked requests list
337 *
338 * This function should be called when a tracked request is completing.
339 */
340 static void tracked_request_end(BdrvTrackedRequest *req)
341 {
342 if (req->serialising) {
343 req->bs->serialising_in_flight--;
344 }
345
346 QLIST_REMOVE(req, list);
347 qemu_co_queue_restart_all(&req->wait_queue);
348 }
349
350 /**
351 * Add an active request to the tracked requests list
352 */
353 static void tracked_request_begin(BdrvTrackedRequest *req,
354 BlockDriverState *bs,
355 int64_t offset,
356 unsigned int bytes,
357 enum BdrvTrackedRequestType type)
358 {
359 *req = (BdrvTrackedRequest){
360 .bs = bs,
361 .offset = offset,
362 .bytes = bytes,
363 .type = type,
364 .co = qemu_coroutine_self(),
365 .serialising = false,
366 .overlap_offset = offset,
367 .overlap_bytes = bytes,
368 };
369
370 qemu_co_queue_init(&req->wait_queue);
371
372 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
373 }
374
375 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
376 {
377 int64_t overlap_offset = req->offset & ~(align - 1);
378 unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
379 - overlap_offset;
380
381 if (!req->serialising) {
382 req->bs->serialising_in_flight++;
383 req->serialising = true;
384 }
385
386 req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
387 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
388 }
389
390 /**
391 * Round a region to cluster boundaries
392 */
393 void bdrv_round_to_clusters(BlockDriverState *bs,
394 int64_t sector_num, int nb_sectors,
395 int64_t *cluster_sector_num,
396 int *cluster_nb_sectors)
397 {
398 BlockDriverInfo bdi;
399
400 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
401 *cluster_sector_num = sector_num;
402 *cluster_nb_sectors = nb_sectors;
403 } else {
404 int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
405 *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
406 *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
407 nb_sectors, c);
408 }
409 }
410
411 static int bdrv_get_cluster_size(BlockDriverState *bs)
412 {
413 BlockDriverInfo bdi;
414 int ret;
415
416 ret = bdrv_get_info(bs, &bdi);
417 if (ret < 0 || bdi.cluster_size == 0) {
418 return bs->request_alignment;
419 } else {
420 return bdi.cluster_size;
421 }
422 }
423
424 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
425 int64_t offset, unsigned int bytes)
426 {
427 /* aaaa bbbb */
428 if (offset >= req->overlap_offset + req->overlap_bytes) {
429 return false;
430 }
431 /* bbbb aaaa */
432 if (req->overlap_offset >= offset + bytes) {
433 return false;
434 }
435 return true;
436 }
437
438 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
439 {
440 BlockDriverState *bs = self->bs;
441 BdrvTrackedRequest *req;
442 bool retry;
443 bool waited = false;
444
445 if (!bs->serialising_in_flight) {
446 return false;
447 }
448
449 do {
450 retry = false;
451 QLIST_FOREACH(req, &bs->tracked_requests, list) {
452 if (req == self || (!req->serialising && !self->serialising)) {
453 continue;
454 }
455 if (tracked_request_overlaps(req, self->overlap_offset,
456 self->overlap_bytes))
457 {
458 /* Hitting this means there was a reentrant request, for
459 * example, a block driver issuing nested requests. This must
460 * never happen since it means deadlock.
461 */
462 assert(qemu_coroutine_self() != req->co);
463
464 /* If the request is already (indirectly) waiting for us, or
465 * will wait for us as soon as it wakes up, then just go on
466 * (instead of producing a deadlock in the former case). */
467 if (!req->waiting_for) {
468 self->waiting_for = req;
469 qemu_co_queue_wait(&req->wait_queue);
470 self->waiting_for = NULL;
471 retry = true;
472 waited = true;
473 break;
474 }
475 }
476 }
477 } while (retry);
478
479 return waited;
480 }
481
482 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
483 size_t size)
484 {
485 if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
486 return -EIO;
487 }
488
489 if (!bdrv_is_inserted(bs)) {
490 return -ENOMEDIUM;
491 }
492
493 if (offset < 0) {
494 return -EIO;
495 }
496
497 return 0;
498 }
499
500 static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
501 int nb_sectors)
502 {
503 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
504 return -EIO;
505 }
506
507 return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
508 nb_sectors * BDRV_SECTOR_SIZE);
509 }
510
511 typedef struct RwCo {
512 BlockDriverState *bs;
513 int64_t offset;
514 QEMUIOVector *qiov;
515 bool is_write;
516 int ret;
517 BdrvRequestFlags flags;
518 } RwCo;
519
520 static void coroutine_fn bdrv_rw_co_entry(void *opaque)
521 {
522 RwCo *rwco = opaque;
523
524 if (!rwco->is_write) {
525 rwco->ret = bdrv_co_preadv(rwco->bs, rwco->offset,
526 rwco->qiov->size, rwco->qiov,
527 rwco->flags);
528 } else {
529 rwco->ret = bdrv_co_pwritev(rwco->bs, rwco->offset,
530 rwco->qiov->size, rwco->qiov,
531 rwco->flags);
532 }
533 }
534
535 /*
536 * Process a vectored synchronous request using coroutines
537 */
538 static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset,
539 QEMUIOVector *qiov, bool is_write,
540 BdrvRequestFlags flags)
541 {
542 Coroutine *co;
543 RwCo rwco = {
544 .bs = bs,
545 .offset = offset,
546 .qiov = qiov,
547 .is_write = is_write,
548 .ret = NOT_DONE,
549 .flags = flags,
550 };
551
552 if (qemu_in_coroutine()) {
553 /* Fast-path if already in coroutine context */
554 bdrv_rw_co_entry(&rwco);
555 } else {
556 AioContext *aio_context = bdrv_get_aio_context(bs);
557
558 co = qemu_coroutine_create(bdrv_rw_co_entry);
559 qemu_coroutine_enter(co, &rwco);
560 while (rwco.ret == NOT_DONE) {
561 aio_poll(aio_context, true);
562 }
563 }
564 return rwco.ret;
565 }
566
567 /*
568 * Process a synchronous request using coroutines
569 */
570 static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf,
571 int nb_sectors, bool is_write, BdrvRequestFlags flags)
572 {
573 QEMUIOVector qiov;
574 struct iovec iov = {
575 .iov_base = (void *)buf,
576 .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
577 };
578
579 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
580 return -EINVAL;
581 }
582
583 qemu_iovec_init_external(&qiov, &iov, 1);
584 return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS,
585 &qiov, is_write, flags);
586 }
587
588 /* return < 0 if error. See bdrv_write() for the return codes */
589 int bdrv_read(BlockDriverState *bs, int64_t sector_num,
590 uint8_t *buf, int nb_sectors)
591 {
592 return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0);
593 }
594
595 /* Return < 0 if error. Important errors are:
596 -EIO generic I/O error (may happen for all errors)
597 -ENOMEDIUM No media inserted.
598 -EINVAL Invalid sector number or nb_sectors
599 -EACCES Trying to write a read-only device
600 */
601 int bdrv_write(BlockDriverState *bs, int64_t sector_num,
602 const uint8_t *buf, int nb_sectors)
603 {
604 return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
605 }
606
607 int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
608 int nb_sectors, BdrvRequestFlags flags)
609 {
610 return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true,
611 BDRV_REQ_ZERO_WRITE | flags);
612 }
613
614 /*
615 * Completely zero out a block device with the help of bdrv_write_zeroes.
616 * The operation is sped up by checking the block status and only writing
617 * zeroes to the device if they currently do not return zeroes. Optional
618 * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
619 * BDRV_REQ_FUA).
620 *
621 * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
622 */
623 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
624 {
625 int64_t target_sectors, ret, nb_sectors, sector_num = 0;
626 BlockDriverState *file;
627 int n;
628
629 target_sectors = bdrv_nb_sectors(bs);
630 if (target_sectors < 0) {
631 return target_sectors;
632 }
633
634 for (;;) {
635 nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
636 if (nb_sectors <= 0) {
637 return 0;
638 }
639 ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
640 if (ret < 0) {
641 error_report("error getting block status at sector %" PRId64 ": %s",
642 sector_num, strerror(-ret));
643 return ret;
644 }
645 if (ret & BDRV_BLOCK_ZERO) {
646 sector_num += n;
647 continue;
648 }
649 ret = bdrv_write_zeroes(bs, sector_num, n, flags);
650 if (ret < 0) {
651 error_report("error writing zeroes at sector %" PRId64 ": %s",
652 sector_num, strerror(-ret));
653 return ret;
654 }
655 sector_num += n;
656 }
657 }
658
659 int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes)
660 {
661 QEMUIOVector qiov;
662 struct iovec iov = {
663 .iov_base = (void *)buf,
664 .iov_len = bytes,
665 };
666 int ret;
667
668 if (bytes < 0) {
669 return -EINVAL;
670 }
671
672 qemu_iovec_init_external(&qiov, &iov, 1);
673 ret = bdrv_prwv_co(bs, offset, &qiov, false, 0);
674 if (ret < 0) {
675 return ret;
676 }
677
678 return bytes;
679 }
680
681 int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov)
682 {
683 int ret;
684
685 ret = bdrv_prwv_co(bs, offset, qiov, true, 0);
686 if (ret < 0) {
687 return ret;
688 }
689
690 return qiov->size;
691 }
692
693 int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
694 const void *buf, int bytes)
695 {
696 QEMUIOVector qiov;
697 struct iovec iov = {
698 .iov_base = (void *) buf,
699 .iov_len = bytes,
700 };
701
702 if (bytes < 0) {
703 return -EINVAL;
704 }
705
706 qemu_iovec_init_external(&qiov, &iov, 1);
707 return bdrv_pwritev(bs, offset, &qiov);
708 }
709
710 /*
711 * Writes to the file and ensures that no writes are reordered across this
712 * request (acts as a barrier)
713 *
714 * Returns 0 on success, -errno in error cases.
715 */
716 int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
717 const void *buf, int count)
718 {
719 int ret;
720
721 ret = bdrv_pwrite(bs, offset, buf, count);
722 if (ret < 0) {
723 return ret;
724 }
725
726 ret = bdrv_flush(bs);
727 if (ret < 0) {
728 return ret;
729 }
730
731 return 0;
732 }
733
734 typedef struct CoroutineIOCompletion {
735 Coroutine *coroutine;
736 int ret;
737 } CoroutineIOCompletion;
738
739 static void bdrv_co_io_em_complete(void *opaque, int ret)
740 {
741 CoroutineIOCompletion *co = opaque;
742
743 co->ret = ret;
744 qemu_coroutine_enter(co->coroutine, NULL);
745 }
746
747 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
748 uint64_t offset, uint64_t bytes,
749 QEMUIOVector *qiov, int flags)
750 {
751 BlockDriver *drv = bs->drv;
752 int64_t sector_num;
753 unsigned int nb_sectors;
754
755 if (drv->bdrv_co_preadv) {
756 return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
757 }
758
759 sector_num = offset >> BDRV_SECTOR_BITS;
760 nb_sectors = bytes >> BDRV_SECTOR_BITS;
761
762 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
763 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
764 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
765
766 if (drv->bdrv_co_readv) {
767 return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
768 } else {
769 BlockAIOCB *acb;
770 CoroutineIOCompletion co = {
771 .coroutine = qemu_coroutine_self(),
772 };
773
774 acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
775 bdrv_co_io_em_complete, &co);
776 if (acb == NULL) {
777 return -EIO;
778 } else {
779 qemu_coroutine_yield();
780 return co.ret;
781 }
782 }
783 }
784
785 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
786 uint64_t offset, uint64_t bytes,
787 QEMUIOVector *qiov, int flags)
788 {
789 BlockDriver *drv = bs->drv;
790 int64_t sector_num;
791 unsigned int nb_sectors;
792 int ret;
793
794 if (drv->bdrv_co_pwritev) {
795 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags);
796 goto emulate_flags;
797 }
798
799 sector_num = offset >> BDRV_SECTOR_BITS;
800 nb_sectors = bytes >> BDRV_SECTOR_BITS;
801
802 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
803 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
804 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
805
806 if (drv->bdrv_co_writev_flags) {
807 ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
808 flags & bs->supported_write_flags);
809 flags &= ~bs->supported_write_flags;
810 } else if (drv->bdrv_co_writev) {
811 assert(!bs->supported_write_flags);
812 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
813 } else {
814 BlockAIOCB *acb;
815 CoroutineIOCompletion co = {
816 .coroutine = qemu_coroutine_self(),
817 };
818
819 acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
820 bdrv_co_io_em_complete, &co);
821 if (acb == NULL) {
822 ret = -EIO;
823 } else {
824 qemu_coroutine_yield();
825 ret = co.ret;
826 }
827 }
828
829 emulate_flags:
830 if (ret == 0 && (flags & BDRV_REQ_FUA)) {
831 ret = bdrv_co_flush(bs);
832 }
833
834 return ret;
835 }
836
837 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
838 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
839 {
840 /* Perform I/O through a temporary buffer so that users who scribble over
841 * their read buffer while the operation is in progress do not end up
842 * modifying the image file. This is critical for zero-copy guest I/O
843 * where anything might happen inside guest memory.
844 */
845 void *bounce_buffer;
846
847 BlockDriver *drv = bs->drv;
848 struct iovec iov;
849 QEMUIOVector bounce_qiov;
850 int64_t cluster_sector_num;
851 int cluster_nb_sectors;
852 size_t skip_bytes;
853 int ret;
854
855 /* Cover entire cluster so no additional backing file I/O is required when
856 * allocating cluster in the image file.
857 */
858 bdrv_round_to_clusters(bs, sector_num, nb_sectors,
859 &cluster_sector_num, &cluster_nb_sectors);
860
861 trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
862 cluster_sector_num, cluster_nb_sectors);
863
864 iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
865 iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
866 if (bounce_buffer == NULL) {
867 ret = -ENOMEM;
868 goto err;
869 }
870
871 qemu_iovec_init_external(&bounce_qiov, &iov, 1);
872
873 ret = bdrv_driver_preadv(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
874 cluster_nb_sectors * BDRV_SECTOR_SIZE,
875 &bounce_qiov, 0);
876 if (ret < 0) {
877 goto err;
878 }
879
880 if (drv->bdrv_co_write_zeroes &&
881 buffer_is_zero(bounce_buffer, iov.iov_len)) {
882 ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num,
883 cluster_nb_sectors, 0);
884 } else {
885 /* This does not change the data on the disk, it is not necessary
886 * to flush even in cache=writethrough mode.
887 */
888 ret = bdrv_driver_pwritev(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
889 cluster_nb_sectors * BDRV_SECTOR_SIZE,
890 &bounce_qiov, 0);
891 }
892
893 if (ret < 0) {
894 /* It might be okay to ignore write errors for guest requests. If this
895 * is a deliberate copy-on-read then we don't want to ignore the error.
896 * Simply report it in all cases.
897 */
898 goto err;
899 }
900
901 skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE;
902 qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes,
903 nb_sectors * BDRV_SECTOR_SIZE);
904
905 err:
906 qemu_vfree(bounce_buffer);
907 return ret;
908 }
909
910 /*
911 * Forwards an already correctly aligned request to the BlockDriver. This
912 * handles copy on read and zeroing after EOF; any other features must be
913 * implemented by the caller.
914 */
915 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
916 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
917 int64_t align, QEMUIOVector *qiov, int flags)
918 {
919 int ret;
920
921 int64_t sector_num = offset >> BDRV_SECTOR_BITS;
922 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
923
924 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
925 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
926 assert(!qiov || bytes == qiov->size);
927 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
928
929 /* Handle Copy on Read and associated serialisation */
930 if (flags & BDRV_REQ_COPY_ON_READ) {
931 /* If we touch the same cluster it counts as an overlap. This
932 * guarantees that allocating writes will be serialized and not race
933 * with each other for the same cluster. For example, in copy-on-read
934 * it ensures that the CoR read and write operations are atomic and
935 * guest writes cannot interleave between them. */
936 mark_request_serialising(req, bdrv_get_cluster_size(bs));
937 }
938
939 if (!(flags & BDRV_REQ_NO_SERIALISING)) {
940 wait_serialising_requests(req);
941 }
942
943 if (flags & BDRV_REQ_COPY_ON_READ) {
944 int pnum;
945
946 ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum);
947 if (ret < 0) {
948 goto out;
949 }
950
951 if (!ret || pnum != nb_sectors) {
952 ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov);
953 goto out;
954 }
955 }
956
957 /* Forward the request to the BlockDriver */
958 if (!bs->zero_beyond_eof) {
959 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
960 } else {
961 /* Read zeros after EOF */
962 int64_t total_sectors, max_nb_sectors;
963
964 total_sectors = bdrv_nb_sectors(bs);
965 if (total_sectors < 0) {
966 ret = total_sectors;
967 goto out;
968 }
969
970 max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num),
971 align >> BDRV_SECTOR_BITS);
972 if (nb_sectors < max_nb_sectors) {
973 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
974 } else if (max_nb_sectors > 0) {
975 QEMUIOVector local_qiov;
976
977 qemu_iovec_init(&local_qiov, qiov->niov);
978 qemu_iovec_concat(&local_qiov, qiov, 0,
979 max_nb_sectors * BDRV_SECTOR_SIZE);
980
981 ret = bdrv_driver_preadv(bs, offset,
982 max_nb_sectors * BDRV_SECTOR_SIZE,
983 &local_qiov, 0);
984
985 qemu_iovec_destroy(&local_qiov);
986 } else {
987 ret = 0;
988 }
989
990 /* Reading beyond end of file is supposed to produce zeroes */
991 if (ret == 0 && total_sectors < sector_num + nb_sectors) {
992 uint64_t offset = MAX(0, total_sectors - sector_num);
993 uint64_t bytes = (sector_num + nb_sectors - offset) *
994 BDRV_SECTOR_SIZE;
995 qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes);
996 }
997 }
998
999 out:
1000 return ret;
1001 }
1002
1003 /*
1004 * Handle a read request in coroutine context
1005 */
1006 int coroutine_fn bdrv_co_preadv(BlockDriverState *bs,
1007 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1008 BdrvRequestFlags flags)
1009 {
1010 BlockDriver *drv = bs->drv;
1011 BdrvTrackedRequest req;
1012
1013 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1014 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1015 uint8_t *head_buf = NULL;
1016 uint8_t *tail_buf = NULL;
1017 QEMUIOVector local_qiov;
1018 bool use_local_qiov = false;
1019 int ret;
1020
1021 if (!drv) {
1022 return -ENOMEDIUM;
1023 }
1024
1025 ret = bdrv_check_byte_request(bs, offset, bytes);
1026 if (ret < 0) {
1027 return ret;
1028 }
1029
1030 /* Don't do copy-on-read if we read data before write operation */
1031 if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
1032 flags |= BDRV_REQ_COPY_ON_READ;
1033 }
1034
1035 /* Align read if necessary by padding qiov */
1036 if (offset & (align - 1)) {
1037 head_buf = qemu_blockalign(bs, align);
1038 qemu_iovec_init(&local_qiov, qiov->niov + 2);
1039 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1040 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1041 use_local_qiov = true;
1042
1043 bytes += offset & (align - 1);
1044 offset = offset & ~(align - 1);
1045 }
1046
1047 if ((offset + bytes) & (align - 1)) {
1048 if (!use_local_qiov) {
1049 qemu_iovec_init(&local_qiov, qiov->niov + 1);
1050 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1051 use_local_qiov = true;
1052 }
1053 tail_buf = qemu_blockalign(bs, align);
1054 qemu_iovec_add(&local_qiov, tail_buf,
1055 align - ((offset + bytes) & (align - 1)));
1056
1057 bytes = ROUND_UP(bytes, align);
1058 }
1059
1060 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1061 ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
1062 use_local_qiov ? &local_qiov : qiov,
1063 flags);
1064 tracked_request_end(&req);
1065
1066 if (use_local_qiov) {
1067 qemu_iovec_destroy(&local_qiov);
1068 qemu_vfree(head_buf);
1069 qemu_vfree(tail_buf);
1070 }
1071
1072 return ret;
1073 }
1074
1075 static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
1076 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1077 BdrvRequestFlags flags)
1078 {
1079 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1080 return -EINVAL;
1081 }
1082
1083 return bdrv_co_preadv(bs, sector_num << BDRV_SECTOR_BITS,
1084 nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1085 }
1086
1087 int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
1088 int nb_sectors, QEMUIOVector *qiov)
1089 {
1090 trace_bdrv_co_readv(bs, sector_num, nb_sectors);
1091
1092 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
1093 }
1094
1095 int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs,
1096 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
1097 {
1098 trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors);
1099
1100 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
1101 BDRV_REQ_NO_SERIALISING);
1102 }
1103
1104 int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
1105 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
1106 {
1107 trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors);
1108
1109 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
1110 BDRV_REQ_COPY_ON_READ);
1111 }
1112
1113 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768
1114
1115 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
1116 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
1117 {
1118 BlockDriver *drv = bs->drv;
1119 QEMUIOVector qiov;
1120 struct iovec iov = {0};
1121 int ret = 0;
1122 bool need_flush = false;
1123
1124 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes,
1125 BDRV_REQUEST_MAX_SECTORS);
1126
1127 while (nb_sectors > 0 && !ret) {
1128 int num = nb_sectors;
1129
1130 /* Align request. Block drivers can expect the "bulk" of the request
1131 * to be aligned.
1132 */
1133 if (bs->bl.write_zeroes_alignment
1134 && num > bs->bl.write_zeroes_alignment) {
1135 if (sector_num % bs->bl.write_zeroes_alignment != 0) {
1136 /* Make a small request up to the first aligned sector. */
1137 num = bs->bl.write_zeroes_alignment;
1138 num -= sector_num % bs->bl.write_zeroes_alignment;
1139 } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) {
1140 /* Shorten the request to the last aligned sector. num cannot
1141 * underflow because num > bs->bl.write_zeroes_alignment.
1142 */
1143 num -= (sector_num + num) % bs->bl.write_zeroes_alignment;
1144 }
1145 }
1146
1147 /* limit request size */
1148 if (num > max_write_zeroes) {
1149 num = max_write_zeroes;
1150 }
1151
1152 ret = -ENOTSUP;
1153 /* First try the efficient write zeroes operation */
1154 if (drv->bdrv_co_write_zeroes) {
1155 ret = drv->bdrv_co_write_zeroes(bs, sector_num, num,
1156 flags & bs->supported_zero_flags);
1157 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1158 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1159 need_flush = true;
1160 }
1161 } else {
1162 assert(!bs->supported_zero_flags);
1163 }
1164
1165 if (ret == -ENOTSUP) {
1166 /* Fall back to bounce buffer if write zeroes is unsupported */
1167 int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length,
1168 MAX_WRITE_ZEROES_BOUNCE_BUFFER);
1169 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1170
1171 if ((flags & BDRV_REQ_FUA) &&
1172 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1173 /* No need for bdrv_driver_pwrite() to do a fallback
1174 * flush on each chunk; use just one at the end */
1175 write_flags &= ~BDRV_REQ_FUA;
1176 need_flush = true;
1177 }
1178 num = MIN(num, max_xfer_len);
1179 iov.iov_len = num * BDRV_SECTOR_SIZE;
1180 if (iov.iov_base == NULL) {
1181 iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE);
1182 if (iov.iov_base == NULL) {
1183 ret = -ENOMEM;
1184 goto fail;
1185 }
1186 memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE);
1187 }
1188 qemu_iovec_init_external(&qiov, &iov, 1);
1189
1190 ret = bdrv_driver_pwritev(bs, sector_num * BDRV_SECTOR_SIZE,
1191 num * BDRV_SECTOR_SIZE, &qiov,
1192 write_flags);
1193
1194 /* Keep bounce buffer around if it is big enough for all
1195 * all future requests.
1196 */
1197 if (num < max_xfer_len) {
1198 qemu_vfree(iov.iov_base);
1199 iov.iov_base = NULL;
1200 }
1201 }
1202
1203 sector_num += num;
1204 nb_sectors -= num;
1205 }
1206
1207 fail:
1208 if (ret == 0 && need_flush) {
1209 ret = bdrv_co_flush(bs);
1210 }
1211 qemu_vfree(iov.iov_base);
1212 return ret;
1213 }
1214
1215 /*
1216 * Forwards an already correctly aligned write request to the BlockDriver.
1217 */
1218 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
1219 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1220 QEMUIOVector *qiov, int flags)
1221 {
1222 BlockDriver *drv = bs->drv;
1223 bool waited;
1224 int ret;
1225
1226 int64_t sector_num = offset >> BDRV_SECTOR_BITS;
1227 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
1228
1229 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
1230 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
1231 assert(!qiov || bytes == qiov->size);
1232 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1233
1234 waited = wait_serialising_requests(req);
1235 assert(!waited || !req->serialising);
1236 assert(req->overlap_offset <= offset);
1237 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1238
1239 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
1240
1241 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1242 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes &&
1243 qemu_iovec_is_zero(qiov)) {
1244 flags |= BDRV_REQ_ZERO_WRITE;
1245 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1246 flags |= BDRV_REQ_MAY_UNMAP;
1247 }
1248 }
1249
1250 if (ret < 0) {
1251 /* Do nothing, write notifier decided to fail this request */
1252 } else if (flags & BDRV_REQ_ZERO_WRITE) {
1253 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1254 ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags);
1255 } else {
1256 bdrv_debug_event(bs, BLKDBG_PWRITEV);
1257 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
1258 }
1259 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1260
1261 bdrv_set_dirty(bs, sector_num, nb_sectors);
1262
1263 if (bs->wr_highest_offset < offset + bytes) {
1264 bs->wr_highest_offset = offset + bytes;
1265 }
1266
1267 if (ret >= 0) {
1268 bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
1269 }
1270
1271 return ret;
1272 }
1273
1274 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
1275 int64_t offset,
1276 unsigned int bytes,
1277 BdrvRequestFlags flags,
1278 BdrvTrackedRequest *req)
1279 {
1280 uint8_t *buf = NULL;
1281 QEMUIOVector local_qiov;
1282 struct iovec iov;
1283 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1284 unsigned int head_padding_bytes, tail_padding_bytes;
1285 int ret = 0;
1286
1287 head_padding_bytes = offset & (align - 1);
1288 tail_padding_bytes = align - ((offset + bytes) & (align - 1));
1289
1290
1291 assert(flags & BDRV_REQ_ZERO_WRITE);
1292 if (head_padding_bytes || tail_padding_bytes) {
1293 buf = qemu_blockalign(bs, align);
1294 iov = (struct iovec) {
1295 .iov_base = buf,
1296 .iov_len = align,
1297 };
1298 qemu_iovec_init_external(&local_qiov, &iov, 1);
1299 }
1300 if (head_padding_bytes) {
1301 uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
1302
1303 /* RMW the unaligned part before head. */
1304 mark_request_serialising(req, align);
1305 wait_serialising_requests(req);
1306 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1307 ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
1308 align, &local_qiov, 0);
1309 if (ret < 0) {
1310 goto fail;
1311 }
1312 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1313
1314 memset(buf + head_padding_bytes, 0, zero_bytes);
1315 ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
1316 &local_qiov,
1317 flags & ~BDRV_REQ_ZERO_WRITE);
1318 if (ret < 0) {
1319 goto fail;
1320 }
1321 offset += zero_bytes;
1322 bytes -= zero_bytes;
1323 }
1324
1325 assert(!bytes || (offset & (align - 1)) == 0);
1326 if (bytes >= align) {
1327 /* Write the aligned part in the middle. */
1328 uint64_t aligned_bytes = bytes & ~(align - 1);
1329 ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes,
1330 NULL, flags);
1331 if (ret < 0) {
1332 goto fail;
1333 }
1334 bytes -= aligned_bytes;
1335 offset += aligned_bytes;
1336 }
1337
1338 assert(!bytes || (offset & (align - 1)) == 0);
1339 if (bytes) {
1340 assert(align == tail_padding_bytes + bytes);
1341 /* RMW the unaligned part after tail. */
1342 mark_request_serialising(req, align);
1343 wait_serialising_requests(req);
1344 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1345 ret = bdrv_aligned_preadv(bs, req, offset, align,
1346 align, &local_qiov, 0);
1347 if (ret < 0) {
1348 goto fail;
1349 }
1350 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1351
1352 memset(buf, 0, bytes);
1353 ret = bdrv_aligned_pwritev(bs, req, offset, align,
1354 &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
1355 }
1356 fail:
1357 qemu_vfree(buf);
1358 return ret;
1359
1360 }
1361
1362 /*
1363 * Handle a write request in coroutine context
1364 */
1365 int coroutine_fn bdrv_co_pwritev(BlockDriverState *bs,
1366 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1367 BdrvRequestFlags flags)
1368 {
1369 BdrvTrackedRequest req;
1370 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1371 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1372 uint8_t *head_buf = NULL;
1373 uint8_t *tail_buf = NULL;
1374 QEMUIOVector local_qiov;
1375 bool use_local_qiov = false;
1376 int ret;
1377
1378 if (!bs->drv) {
1379 return -ENOMEDIUM;
1380 }
1381 if (bs->read_only) {
1382 return -EPERM;
1383 }
1384 assert(!(bs->open_flags & BDRV_O_INACTIVE));
1385
1386 ret = bdrv_check_byte_request(bs, offset, bytes);
1387 if (ret < 0) {
1388 return ret;
1389 }
1390
1391 /*
1392 * Align write if necessary by performing a read-modify-write cycle.
1393 * Pad qiov with the read parts and be sure to have a tracked request not
1394 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
1395 */
1396 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
1397
1398 if (!qiov) {
1399 ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
1400 goto out;
1401 }
1402
1403 if (offset & (align - 1)) {
1404 QEMUIOVector head_qiov;
1405 struct iovec head_iov;
1406
1407 mark_request_serialising(&req, align);
1408 wait_serialising_requests(&req);
1409
1410 head_buf = qemu_blockalign(bs, align);
1411 head_iov = (struct iovec) {
1412 .iov_base = head_buf,
1413 .iov_len = align,
1414 };
1415 qemu_iovec_init_external(&head_qiov, &head_iov, 1);
1416
1417 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1418 ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
1419 align, &head_qiov, 0);
1420 if (ret < 0) {
1421 goto fail;
1422 }
1423 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1424
1425 qemu_iovec_init(&local_qiov, qiov->niov + 2);
1426 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1427 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1428 use_local_qiov = true;
1429
1430 bytes += offset & (align - 1);
1431 offset = offset & ~(align - 1);
1432 }
1433
1434 if ((offset + bytes) & (align - 1)) {
1435 QEMUIOVector tail_qiov;
1436 struct iovec tail_iov;
1437 size_t tail_bytes;
1438 bool waited;
1439
1440 mark_request_serialising(&req, align);
1441 waited = wait_serialising_requests(&req);
1442 assert(!waited || !use_local_qiov);
1443
1444 tail_buf = qemu_blockalign(bs, align);
1445 tail_iov = (struct iovec) {
1446 .iov_base = tail_buf,
1447 .iov_len = align,
1448 };
1449 qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
1450
1451 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1452 ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
1453 align, &tail_qiov, 0);
1454 if (ret < 0) {
1455 goto fail;
1456 }
1457 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1458
1459 if (!use_local_qiov) {
1460 qemu_iovec_init(&local_qiov, qiov->niov + 1);
1461 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1462 use_local_qiov = true;
1463 }
1464
1465 tail_bytes = (offset + bytes) & (align - 1);
1466 qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
1467
1468 bytes = ROUND_UP(bytes, align);
1469 }
1470
1471 ret = bdrv_aligned_pwritev(bs, &req, offset, bytes,
1472 use_local_qiov ? &local_qiov : qiov,
1473 flags);
1474
1475 fail:
1476
1477 if (use_local_qiov) {
1478 qemu_iovec_destroy(&local_qiov);
1479 }
1480 qemu_vfree(head_buf);
1481 qemu_vfree(tail_buf);
1482 out:
1483 tracked_request_end(&req);
1484 return ret;
1485 }
1486
1487 static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
1488 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1489 BdrvRequestFlags flags)
1490 {
1491 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1492 return -EINVAL;
1493 }
1494
1495 return bdrv_co_pwritev(bs, sector_num << BDRV_SECTOR_BITS,
1496 nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1497 }
1498
1499 int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
1500 int nb_sectors, QEMUIOVector *qiov)
1501 {
1502 trace_bdrv_co_writev(bs, sector_num, nb_sectors);
1503
1504 return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0);
1505 }
1506
1507 int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
1508 int64_t sector_num, int nb_sectors,
1509 BdrvRequestFlags flags)
1510 {
1511 trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags);
1512
1513 if (!(bs->open_flags & BDRV_O_UNMAP)) {
1514 flags &= ~BDRV_REQ_MAY_UNMAP;
1515 }
1516
1517 return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL,
1518 BDRV_REQ_ZERO_WRITE | flags);
1519 }
1520
1521 typedef struct BdrvCoGetBlockStatusData {
1522 BlockDriverState *bs;
1523 BlockDriverState *base;
1524 BlockDriverState **file;
1525 int64_t sector_num;
1526 int nb_sectors;
1527 int *pnum;
1528 int64_t ret;
1529 bool done;
1530 } BdrvCoGetBlockStatusData;
1531
1532 /*
1533 * Returns the allocation status of the specified sectors.
1534 * Drivers not implementing the functionality are assumed to not support
1535 * backing files, hence all their sectors are reported as allocated.
1536 *
1537 * If 'sector_num' is beyond the end of the disk image the return value is 0
1538 * and 'pnum' is set to 0.
1539 *
1540 * 'pnum' is set to the number of sectors (including and immediately following
1541 * the specified sector) that are known to be in the same
1542 * allocated/unallocated state.
1543 *
1544 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes
1545 * beyond the end of the disk image it will be clamped.
1546 *
1547 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
1548 * points to the BDS which the sector range is allocated in.
1549 */
1550 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
1551 int64_t sector_num,
1552 int nb_sectors, int *pnum,
1553 BlockDriverState **file)
1554 {
1555 int64_t total_sectors;
1556 int64_t n;
1557 int64_t ret, ret2;
1558
1559 total_sectors = bdrv_nb_sectors(bs);
1560 if (total_sectors < 0) {
1561 return total_sectors;
1562 }
1563
1564 if (sector_num >= total_sectors) {
1565 *pnum = 0;
1566 return 0;
1567 }
1568
1569 n = total_sectors - sector_num;
1570 if (n < nb_sectors) {
1571 nb_sectors = n;
1572 }
1573
1574 if (!bs->drv->bdrv_co_get_block_status) {
1575 *pnum = nb_sectors;
1576 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
1577 if (bs->drv->protocol_name) {
1578 ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
1579 }
1580 return ret;
1581 }
1582
1583 *file = NULL;
1584 ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
1585 file);
1586 if (ret < 0) {
1587 *pnum = 0;
1588 return ret;
1589 }
1590
1591 if (ret & BDRV_BLOCK_RAW) {
1592 assert(ret & BDRV_BLOCK_OFFSET_VALID);
1593 return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
1594 *pnum, pnum, file);
1595 }
1596
1597 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
1598 ret |= BDRV_BLOCK_ALLOCATED;
1599 } else {
1600 if (bdrv_unallocated_blocks_are_zero(bs)) {
1601 ret |= BDRV_BLOCK_ZERO;
1602 } else if (bs->backing) {
1603 BlockDriverState *bs2 = bs->backing->bs;
1604 int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
1605 if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
1606 ret |= BDRV_BLOCK_ZERO;
1607 }
1608 }
1609 }
1610
1611 if (*file && *file != bs &&
1612 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
1613 (ret & BDRV_BLOCK_OFFSET_VALID)) {
1614 BlockDriverState *file2;
1615 int file_pnum;
1616
1617 ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
1618 *pnum, &file_pnum, &file2);
1619 if (ret2 >= 0) {
1620 /* Ignore errors. This is just providing extra information, it
1621 * is useful but not necessary.
1622 */
1623 if (!file_pnum) {
1624 /* !file_pnum indicates an offset at or beyond the EOF; it is
1625 * perfectly valid for the format block driver to point to such
1626 * offsets, so catch it and mark everything as zero */
1627 ret |= BDRV_BLOCK_ZERO;
1628 } else {
1629 /* Limit request to the range reported by the protocol driver */
1630 *pnum = file_pnum;
1631 ret |= (ret2 & BDRV_BLOCK_ZERO);
1632 }
1633 }
1634 }
1635
1636 return ret;
1637 }
1638
1639 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
1640 BlockDriverState *base,
1641 int64_t sector_num,
1642 int nb_sectors,
1643 int *pnum,
1644 BlockDriverState **file)
1645 {
1646 BlockDriverState *p;
1647 int64_t ret = 0;
1648
1649 assert(bs != base);
1650 for (p = bs; p != base; p = backing_bs(p)) {
1651 ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
1652 if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
1653 break;
1654 }
1655 /* [sector_num, pnum] unallocated on this layer, which could be only
1656 * the first part of [sector_num, nb_sectors]. */
1657 nb_sectors = MIN(nb_sectors, *pnum);
1658 }
1659 return ret;
1660 }
1661
1662 /* Coroutine wrapper for bdrv_get_block_status_above() */
1663 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
1664 {
1665 BdrvCoGetBlockStatusData *data = opaque;
1666
1667 data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
1668 data->sector_num,
1669 data->nb_sectors,
1670 data->pnum,
1671 data->file);
1672 data->done = true;
1673 }
1674
1675 /*
1676 * Synchronous wrapper around bdrv_co_get_block_status_above().
1677 *
1678 * See bdrv_co_get_block_status_above() for details.
1679 */
1680 int64_t bdrv_get_block_status_above(BlockDriverState *bs,
1681 BlockDriverState *base,
1682 int64_t sector_num,
1683 int nb_sectors, int *pnum,
1684 BlockDriverState **file)
1685 {
1686 Coroutine *co;
1687 BdrvCoGetBlockStatusData data = {
1688 .bs = bs,
1689 .base = base,
1690 .file = file,
1691 .sector_num = sector_num,
1692 .nb_sectors = nb_sectors,
1693 .pnum = pnum,
1694 .done = false,
1695 };
1696
1697 if (qemu_in_coroutine()) {
1698 /* Fast-path if already in coroutine context */
1699 bdrv_get_block_status_above_co_entry(&data);
1700 } else {
1701 AioContext *aio_context = bdrv_get_aio_context(bs);
1702
1703 co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry);
1704 qemu_coroutine_enter(co, &data);
1705 while (!data.done) {
1706 aio_poll(aio_context, true);
1707 }
1708 }
1709 return data.ret;
1710 }
1711
1712 int64_t bdrv_get_block_status(BlockDriverState *bs,
1713 int64_t sector_num,
1714 int nb_sectors, int *pnum,
1715 BlockDriverState **file)
1716 {
1717 return bdrv_get_block_status_above(bs, backing_bs(bs),
1718 sector_num, nb_sectors, pnum, file);
1719 }
1720
1721 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
1722 int nb_sectors, int *pnum)
1723 {
1724 BlockDriverState *file;
1725 int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
1726 &file);
1727 if (ret < 0) {
1728 return ret;
1729 }
1730 return !!(ret & BDRV_BLOCK_ALLOCATED);
1731 }
1732
1733 /*
1734 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
1735 *
1736 * Return true if the given sector is allocated in any image between
1737 * BASE and TOP (inclusive). BASE can be NULL to check if the given
1738 * sector is allocated in any image of the chain. Return false otherwise.
1739 *
1740 * 'pnum' is set to the number of sectors (including and immediately following
1741 * the specified sector) that are known to be in the same
1742 * allocated/unallocated state.
1743 *
1744 */
1745 int bdrv_is_allocated_above(BlockDriverState *top,
1746 BlockDriverState *base,
1747 int64_t sector_num,
1748 int nb_sectors, int *pnum)
1749 {
1750 BlockDriverState *intermediate;
1751 int ret, n = nb_sectors;
1752
1753 intermediate = top;
1754 while (intermediate && intermediate != base) {
1755 int pnum_inter;
1756 ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
1757 &pnum_inter);
1758 if (ret < 0) {
1759 return ret;
1760 } else if (ret) {
1761 *pnum = pnum_inter;
1762 return 1;
1763 }
1764
1765 /*
1766 * [sector_num, nb_sectors] is unallocated on top but intermediate
1767 * might have
1768 *
1769 * [sector_num+x, nr_sectors] allocated.
1770 */
1771 if (n > pnum_inter &&
1772 (intermediate == top ||
1773 sector_num + pnum_inter < intermediate->total_sectors)) {
1774 n = pnum_inter;
1775 }
1776
1777 intermediate = backing_bs(intermediate);
1778 }
1779
1780 *pnum = n;
1781 return 0;
1782 }
1783
1784 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
1785 const uint8_t *buf, int nb_sectors)
1786 {
1787 BlockDriver *drv = bs->drv;
1788 int ret;
1789
1790 if (!drv) {
1791 return -ENOMEDIUM;
1792 }
1793 if (!drv->bdrv_write_compressed) {
1794 return -ENOTSUP;
1795 }
1796 ret = bdrv_check_request(bs, sector_num, nb_sectors);
1797 if (ret < 0) {
1798 return ret;
1799 }
1800
1801 assert(QLIST_EMPTY(&bs->dirty_bitmaps));
1802
1803 return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
1804 }
1805
1806 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
1807 int64_t pos, int size)
1808 {
1809 QEMUIOVector qiov;
1810 struct iovec iov = {
1811 .iov_base = (void *) buf,
1812 .iov_len = size,
1813 };
1814
1815 qemu_iovec_init_external(&qiov, &iov, 1);
1816 return bdrv_writev_vmstate(bs, &qiov, pos);
1817 }
1818
1819 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
1820 {
1821 BlockDriver *drv = bs->drv;
1822
1823 if (!drv) {
1824 return -ENOMEDIUM;
1825 } else if (drv->bdrv_save_vmstate) {
1826 return drv->bdrv_save_vmstate(bs, qiov, pos);
1827 } else if (bs->file) {
1828 return bdrv_writev_vmstate(bs->file->bs, qiov, pos);
1829 }
1830
1831 return -ENOTSUP;
1832 }
1833
1834 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
1835 int64_t pos, int size)
1836 {
1837 BlockDriver *drv = bs->drv;
1838 if (!drv)
1839 return -ENOMEDIUM;
1840 if (drv->bdrv_load_vmstate)
1841 return drv->bdrv_load_vmstate(bs, buf, pos, size);
1842 if (bs->file)
1843 return bdrv_load_vmstate(bs->file->bs, buf, pos, size);
1844 return -ENOTSUP;
1845 }
1846
1847 /**************************************************************/
1848 /* async I/Os */
1849
1850 BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
1851 QEMUIOVector *qiov, int nb_sectors,
1852 BlockCompletionFunc *cb, void *opaque)
1853 {
1854 trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
1855
1856 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1857 cb, opaque, false);
1858 }
1859
1860 BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
1861 QEMUIOVector *qiov, int nb_sectors,
1862 BlockCompletionFunc *cb, void *opaque)
1863 {
1864 trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
1865
1866 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1867 cb, opaque, true);
1868 }
1869
1870 BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs,
1871 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags,
1872 BlockCompletionFunc *cb, void *opaque)
1873 {
1874 trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque);
1875
1876 return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors,
1877 BDRV_REQ_ZERO_WRITE | flags,
1878 cb, opaque, true);
1879 }
1880
1881
1882 typedef struct MultiwriteCB {
1883 int error;
1884 int num_requests;
1885 int num_callbacks;
1886 struct {
1887 BlockCompletionFunc *cb;
1888 void *opaque;
1889 QEMUIOVector *free_qiov;
1890 } callbacks[];
1891 } MultiwriteCB;
1892
1893 static void multiwrite_user_cb(MultiwriteCB *mcb)
1894 {
1895 int i;
1896
1897 for (i = 0; i < mcb->num_callbacks; i++) {
1898 mcb->callbacks[i].cb(mcb->callbacks[i].opaque, mcb->error);
1899 if (mcb->callbacks[i].free_qiov) {
1900 qemu_iovec_destroy(mcb->callbacks[i].free_qiov);
1901 }
1902 g_free(mcb->callbacks[i].free_qiov);
1903 }
1904 }
1905
1906 static void multiwrite_cb(void *opaque, int ret)
1907 {
1908 MultiwriteCB *mcb = opaque;
1909
1910 trace_multiwrite_cb(mcb, ret);
1911
1912 if (ret < 0 && !mcb->error) {
1913 mcb->error = ret;
1914 }
1915
1916 mcb->num_requests--;
1917 if (mcb->num_requests == 0) {
1918 multiwrite_user_cb(mcb);
1919 g_free(mcb);
1920 }
1921 }
1922
1923 static int multiwrite_req_compare(const void *a, const void *b)
1924 {
1925 const BlockRequest *req1 = a, *req2 = b;
1926
1927 /*
1928 * Note that we can't simply subtract req2->sector from req1->sector
1929 * here as that could overflow the return value.
1930 */
1931 if (req1->sector > req2->sector) {
1932 return 1;
1933 } else if (req1->sector < req2->sector) {
1934 return -1;
1935 } else {
1936 return 0;
1937 }
1938 }
1939
1940 /*
1941 * Takes a bunch of requests and tries to merge them. Returns the number of
1942 * requests that remain after merging.
1943 */
1944 static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
1945 int num_reqs, MultiwriteCB *mcb)
1946 {
1947 int i, outidx;
1948
1949 // Sort requests by start sector
1950 qsort(reqs, num_reqs, sizeof(*reqs), &multiwrite_req_compare);
1951
1952 // Check if adjacent requests touch the same clusters. If so, combine them,
1953 // filling up gaps with zero sectors.
1954 outidx = 0;
1955 for (i = 1; i < num_reqs; i++) {
1956 int merge = 0;
1957 int64_t oldreq_last = reqs[outidx].sector + reqs[outidx].nb_sectors;
1958
1959 // Handle exactly sequential writes and overlapping writes.
1960 if (reqs[i].sector <= oldreq_last) {
1961 merge = 1;
1962 }
1963
1964 if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 >
1965 bs->bl.max_iov) {
1966 merge = 0;
1967 }
1968
1969 if (bs->bl.max_transfer_length && reqs[outidx].nb_sectors +
1970 reqs[i].nb_sectors > bs->bl.max_transfer_length) {
1971 merge = 0;
1972 }
1973
1974 if (merge) {
1975 size_t size;
1976 QEMUIOVector *qiov = g_malloc0(sizeof(*qiov));
1977 qemu_iovec_init(qiov,
1978 reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1);
1979
1980 // Add the first request to the merged one. If the requests are
1981 // overlapping, drop the last sectors of the first request.
1982 size = (reqs[i].sector - reqs[outidx].sector) << 9;
1983 qemu_iovec_concat(qiov, reqs[outidx].qiov, 0, size);
1984
1985 // We should need to add any zeros between the two requests
1986 assert (reqs[i].sector <= oldreq_last);
1987
1988 // Add the second request
1989 qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size);
1990
1991 // Add tail of first request, if necessary
1992 if (qiov->size < reqs[outidx].qiov->size) {
1993 qemu_iovec_concat(qiov, reqs[outidx].qiov, qiov->size,
1994 reqs[outidx].qiov->size - qiov->size);
1995 }
1996
1997 reqs[outidx].nb_sectors = qiov->size >> 9;
1998 reqs[outidx].qiov = qiov;
1999
2000 mcb->callbacks[i].free_qiov = reqs[outidx].qiov;
2001 } else {
2002 outidx++;
2003 reqs[outidx].sector = reqs[i].sector;
2004 reqs[outidx].nb_sectors = reqs[i].nb_sectors;
2005 reqs[outidx].qiov = reqs[i].qiov;
2006 }
2007 }
2008
2009 if (bs->blk) {
2010 block_acct_merge_done(blk_get_stats(bs->blk), BLOCK_ACCT_WRITE,
2011 num_reqs - outidx - 1);
2012 }
2013
2014 return outidx + 1;
2015 }
2016
2017 /*
2018 * Submit multiple AIO write requests at once.
2019 *
2020 * On success, the function returns 0 and all requests in the reqs array have
2021 * been submitted. In error case this function returns -1, and any of the
2022 * requests may or may not be submitted yet. In particular, this means that the
2023 * callback will be called for some of the requests, for others it won't. The
2024 * caller must check the error field of the BlockRequest to wait for the right
2025 * callbacks (if error != 0, no callback will be called).
2026 *
2027 * The implementation may modify the contents of the reqs array, e.g. to merge
2028 * requests. However, the fields opaque and error are left unmodified as they
2029 * are used to signal failure for a single request to the caller.
2030 */
2031 int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs)
2032 {
2033 MultiwriteCB *mcb;
2034 int i;
2035
2036 /* don't submit writes if we don't have a medium */
2037 if (bs->drv == NULL) {
2038 for (i = 0; i < num_reqs; i++) {
2039 reqs[i].error = -ENOMEDIUM;
2040 }
2041 return -1;
2042 }
2043
2044 if (num_reqs == 0) {
2045 return 0;
2046 }
2047
2048 // Create MultiwriteCB structure
2049 mcb = g_malloc0(sizeof(*mcb) + num_reqs * sizeof(*mcb->callbacks));
2050 mcb->num_requests = 0;
2051 mcb->num_callbacks = num_reqs;
2052
2053 for (i = 0; i < num_reqs; i++) {
2054 mcb->callbacks[i].cb = reqs[i].cb;
2055 mcb->callbacks[i].opaque = reqs[i].opaque;
2056 }
2057
2058 // Check for mergable requests
2059 num_reqs = multiwrite_merge(bs, reqs, num_reqs, mcb);
2060
2061 trace_bdrv_aio_multiwrite(mcb, mcb->num_callbacks, num_reqs);
2062
2063 /* Run the aio requests. */
2064 mcb->num_requests = num_reqs;
2065 for (i = 0; i < num_reqs; i++) {
2066 bdrv_co_aio_rw_vector(bs, reqs[i].sector, reqs[i].qiov,
2067 reqs[i].nb_sectors, reqs[i].flags,
2068 multiwrite_cb, mcb,
2069 true);
2070 }
2071
2072 return 0;
2073 }
2074
2075 void bdrv_aio_cancel(BlockAIOCB *acb)
2076 {
2077 qemu_aio_ref(acb);
2078 bdrv_aio_cancel_async(acb);
2079 while (acb->refcnt > 1) {
2080 if (acb->aiocb_info->get_aio_context) {
2081 aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2082 } else if (acb->bs) {
2083 aio_poll(bdrv_get_aio_context(acb->bs), true);
2084 } else {
2085 abort();
2086 }
2087 }
2088 qemu_aio_unref(acb);
2089 }
2090
2091 /* Async version of aio cancel. The caller is not blocked if the acb implements
2092 * cancel_async, otherwise we do nothing and let the request normally complete.
2093 * In either case the completion callback must be called. */
2094 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2095 {
2096 if (acb->aiocb_info->cancel_async) {
2097 acb->aiocb_info->cancel_async(acb);
2098 }
2099 }
2100
2101 /**************************************************************/
2102 /* async block device emulation */
2103
2104 typedef struct BlockAIOCBCoroutine {
2105 BlockAIOCB common;
2106 BlockRequest req;
2107 bool is_write;
2108 bool need_bh;
2109 bool *done;
2110 QEMUBH* bh;
2111 } BlockAIOCBCoroutine;
2112
2113 static const AIOCBInfo bdrv_em_co_aiocb_info = {
2114 .aiocb_size = sizeof(BlockAIOCBCoroutine),
2115 };
2116
2117 static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
2118 {
2119 if (!acb->need_bh) {
2120 acb->common.cb(acb->common.opaque, acb->req.error);
2121 qemu_aio_unref(acb);
2122 }
2123 }
2124
2125 static void bdrv_co_em_bh(void *opaque)
2126 {
2127 BlockAIOCBCoroutine *acb = opaque;
2128
2129 assert(!acb->need_bh);
2130 qemu_bh_delete(acb->bh);
2131 bdrv_co_complete(acb);
2132 }
2133
2134 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
2135 {
2136 acb->need_bh = false;
2137 if (acb->req.error != -EINPROGRESS) {
2138 BlockDriverState *bs = acb->common.bs;
2139
2140 acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
2141 qemu_bh_schedule(acb->bh);
2142 }
2143 }
2144
2145 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
2146 static void coroutine_fn bdrv_co_do_rw(void *opaque)
2147 {
2148 BlockAIOCBCoroutine *acb = opaque;
2149 BlockDriverState *bs = acb->common.bs;
2150
2151 if (!acb->is_write) {
2152 acb->req.error = bdrv_co_do_readv(bs, acb->req.sector,
2153 acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
2154 } else {
2155 acb->req.error = bdrv_co_do_writev(bs, acb->req.sector,
2156 acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
2157 }
2158
2159 bdrv_co_complete(acb);
2160 }
2161
2162 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
2163 int64_t sector_num,
2164 QEMUIOVector *qiov,
2165 int nb_sectors,
2166 BdrvRequestFlags flags,
2167 BlockCompletionFunc *cb,
2168 void *opaque,
2169 bool is_write)
2170 {
2171 Coroutine *co;
2172 BlockAIOCBCoroutine *acb;
2173
2174 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2175 acb->need_bh = true;
2176 acb->req.error = -EINPROGRESS;
2177 acb->req.sector = sector_num;
2178 acb->req.nb_sectors = nb_sectors;
2179 acb->req.qiov = qiov;
2180 acb->req.flags = flags;
2181 acb->is_write = is_write;
2182
2183 co = qemu_coroutine_create(bdrv_co_do_rw);
2184 qemu_coroutine_enter(co, acb);
2185
2186 bdrv_co_maybe_schedule_bh(acb);
2187 return &acb->common;
2188 }
2189
2190 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
2191 {
2192 BlockAIOCBCoroutine *acb = opaque;
2193 BlockDriverState *bs = acb->common.bs;
2194
2195 acb->req.error = bdrv_co_flush(bs);
2196 bdrv_co_complete(acb);
2197 }
2198
2199 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
2200 BlockCompletionFunc *cb, void *opaque)
2201 {
2202 trace_bdrv_aio_flush(bs, opaque);
2203
2204 Coroutine *co;
2205 BlockAIOCBCoroutine *acb;
2206
2207 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2208 acb->need_bh = true;
2209 acb->req.error = -EINPROGRESS;
2210
2211 co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
2212 qemu_coroutine_enter(co, acb);
2213
2214 bdrv_co_maybe_schedule_bh(acb);
2215 return &acb->common;
2216 }
2217
2218 static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
2219 {
2220 BlockAIOCBCoroutine *acb = opaque;
2221 BlockDriverState *bs = acb->common.bs;
2222
2223 acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
2224 bdrv_co_complete(acb);
2225 }
2226
2227 BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
2228 int64_t sector_num, int nb_sectors,
2229 BlockCompletionFunc *cb, void *opaque)
2230 {
2231 Coroutine *co;
2232 BlockAIOCBCoroutine *acb;
2233
2234 trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
2235
2236 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2237 acb->need_bh = true;
2238 acb->req.error = -EINPROGRESS;
2239 acb->req.sector = sector_num;
2240 acb->req.nb_sectors = nb_sectors;
2241 co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
2242 qemu_coroutine_enter(co, acb);
2243
2244 bdrv_co_maybe_schedule_bh(acb);
2245 return &acb->common;
2246 }
2247
2248 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
2249 BlockCompletionFunc *cb, void *opaque)
2250 {
2251 BlockAIOCB *acb;
2252
2253 acb = g_malloc(aiocb_info->aiocb_size);
2254 acb->aiocb_info = aiocb_info;
2255 acb->bs = bs;
2256 acb->cb = cb;
2257 acb->opaque = opaque;
2258 acb->refcnt = 1;
2259 return acb;
2260 }
2261
2262 void qemu_aio_ref(void *p)
2263 {
2264 BlockAIOCB *acb = p;
2265 acb->refcnt++;
2266 }
2267
2268 void qemu_aio_unref(void *p)
2269 {
2270 BlockAIOCB *acb = p;
2271 assert(acb->refcnt > 0);
2272 if (--acb->refcnt == 0) {
2273 g_free(acb);
2274 }
2275 }
2276
2277 /**************************************************************/
2278 /* Coroutine block device emulation */
2279
2280 static void coroutine_fn bdrv_flush_co_entry(void *opaque)
2281 {
2282 RwCo *rwco = opaque;
2283
2284 rwco->ret = bdrv_co_flush(rwco->bs);
2285 }
2286
2287 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2288 {
2289 int ret;
2290 BdrvTrackedRequest req;
2291
2292 if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2293 bdrv_is_sg(bs)) {
2294 return 0;
2295 }
2296
2297 tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
2298
2299 /* Write back all layers by calling one driver function */
2300 if (bs->drv->bdrv_co_flush) {
2301 ret = bs->drv->bdrv_co_flush(bs);
2302 goto out;
2303 }
2304
2305 /* Write back cached data to the OS even with cache=unsafe */
2306 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
2307 if (bs->drv->bdrv_co_flush_to_os) {
2308 ret = bs->drv->bdrv_co_flush_to_os(bs);
2309 if (ret < 0) {
2310 goto out;
2311 }
2312 }
2313
2314 /* But don't actually force it to the disk with cache=unsafe */
2315 if (bs->open_flags & BDRV_O_NO_FLUSH) {
2316 goto flush_parent;
2317 }
2318
2319 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
2320 if (bs->drv->bdrv_co_flush_to_disk) {
2321 ret = bs->drv->bdrv_co_flush_to_disk(bs);
2322 } else if (bs->drv->bdrv_aio_flush) {
2323 BlockAIOCB *acb;
2324 CoroutineIOCompletion co = {
2325 .coroutine = qemu_coroutine_self(),
2326 };
2327
2328 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2329 if (acb == NULL) {
2330 ret = -EIO;
2331 } else {
2332 qemu_coroutine_yield();
2333 ret = co.ret;
2334 }
2335 } else {
2336 /*
2337 * Some block drivers always operate in either writethrough or unsafe
2338 * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2339 * know how the server works (because the behaviour is hardcoded or
2340 * depends on server-side configuration), so we can't ensure that
2341 * everything is safe on disk. Returning an error doesn't work because
2342 * that would break guests even if the server operates in writethrough
2343 * mode.
2344 *
2345 * Let's hope the user knows what he's doing.
2346 */
2347 ret = 0;
2348 }
2349 if (ret < 0) {
2350 goto out;
2351 }
2352
2353 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH
2354 * in the case of cache=unsafe, so there are no useless flushes.
2355 */
2356 flush_parent:
2357 ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2358 out:
2359 tracked_request_end(&req);
2360 return ret;
2361 }
2362
2363 int bdrv_flush(BlockDriverState *bs)
2364 {
2365 Coroutine *co;
2366 RwCo rwco = {
2367 .bs = bs,
2368 .ret = NOT_DONE,
2369 };
2370
2371 if (qemu_in_coroutine()) {
2372 /* Fast-path if already in coroutine context */
2373 bdrv_flush_co_entry(&rwco);
2374 } else {
2375 AioContext *aio_context = bdrv_get_aio_context(bs);
2376
2377 co = qemu_coroutine_create(bdrv_flush_co_entry);
2378 qemu_coroutine_enter(co, &rwco);
2379 while (rwco.ret == NOT_DONE) {
2380 aio_poll(aio_context, true);
2381 }
2382 }
2383
2384 return rwco.ret;
2385 }
2386
2387 typedef struct DiscardCo {
2388 BlockDriverState *bs;
2389 int64_t sector_num;
2390 int nb_sectors;
2391 int ret;
2392 } DiscardCo;
2393 static void coroutine_fn bdrv_discard_co_entry(void *opaque)
2394 {
2395 DiscardCo *rwco = opaque;
2396
2397 rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors);
2398 }
2399
2400 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
2401 int nb_sectors)
2402 {
2403 BdrvTrackedRequest req;
2404 int max_discard, ret;
2405
2406 if (!bs->drv) {
2407 return -ENOMEDIUM;
2408 }
2409
2410 ret = bdrv_check_request(bs, sector_num, nb_sectors);
2411 if (ret < 0) {
2412 return ret;
2413 } else if (bs->read_only) {
2414 return -EPERM;
2415 }
2416 assert(!(bs->open_flags & BDRV_O_INACTIVE));
2417
2418 /* Do nothing if disabled. */
2419 if (!(bs->open_flags & BDRV_O_UNMAP)) {
2420 return 0;
2421 }
2422
2423 if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) {
2424 return 0;
2425 }
2426
2427 tracked_request_begin(&req, bs, sector_num, nb_sectors,
2428 BDRV_TRACKED_DISCARD);
2429 bdrv_set_dirty(bs, sector_num, nb_sectors);
2430
2431 max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS);
2432 while (nb_sectors > 0) {
2433 int ret;
2434 int num = nb_sectors;
2435
2436 /* align request */
2437 if (bs->bl.discard_alignment &&
2438 num >= bs->bl.discard_alignment &&
2439 sector_num % bs->bl.discard_alignment) {
2440 if (num > bs->bl.discard_alignment) {
2441 num = bs->bl.discard_alignment;
2442 }
2443 num -= sector_num % bs->bl.discard_alignment;
2444 }
2445
2446 /* limit request size */
2447 if (num > max_discard) {
2448 num = max_discard;
2449 }
2450
2451 if (bs->drv->bdrv_co_discard) {
2452 ret = bs->drv->bdrv_co_discard(bs, sector_num, num);
2453 } else {
2454 BlockAIOCB *acb;
2455 CoroutineIOCompletion co = {
2456 .coroutine = qemu_coroutine_self(),
2457 };
2458
2459 acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
2460 bdrv_co_io_em_complete, &co);
2461 if (acb == NULL) {
2462 ret = -EIO;
2463 goto out;
2464 } else {
2465 qemu_coroutine_yield();
2466 ret = co.ret;
2467 }
2468 }
2469 if (ret && ret != -ENOTSUP) {
2470 goto out;
2471 }
2472
2473 sector_num += num;
2474 nb_sectors -= num;
2475 }
2476 ret = 0;
2477 out:
2478 tracked_request_end(&req);
2479 return ret;
2480 }
2481
2482 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
2483 {
2484 Coroutine *co;
2485 DiscardCo rwco = {
2486 .bs = bs,
2487 .sector_num = sector_num,
2488 .nb_sectors = nb_sectors,
2489 .ret = NOT_DONE,
2490 };
2491
2492 if (qemu_in_coroutine()) {
2493 /* Fast-path if already in coroutine context */
2494 bdrv_discard_co_entry(&rwco);
2495 } else {
2496 AioContext *aio_context = bdrv_get_aio_context(bs);
2497
2498 co = qemu_coroutine_create(bdrv_discard_co_entry);
2499 qemu_coroutine_enter(co, &rwco);
2500 while (rwco.ret == NOT_DONE) {
2501 aio_poll(aio_context, true);
2502 }
2503 }
2504
2505 return rwco.ret;
2506 }
2507
2508 typedef struct {
2509 CoroutineIOCompletion *co;
2510 QEMUBH *bh;
2511 } BdrvIoctlCompletionData;
2512
2513 static void bdrv_ioctl_bh_cb(void *opaque)
2514 {
2515 BdrvIoctlCompletionData *data = opaque;
2516
2517 bdrv_co_io_em_complete(data->co, -ENOTSUP);
2518 qemu_bh_delete(data->bh);
2519 }
2520
2521 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
2522 {
2523 BlockDriver *drv = bs->drv;
2524 BdrvTrackedRequest tracked_req;
2525 CoroutineIOCompletion co = {
2526 .coroutine = qemu_coroutine_self(),
2527 };
2528 BlockAIOCB *acb;
2529
2530 tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
2531 if (!drv || !drv->bdrv_aio_ioctl) {
2532 co.ret = -ENOTSUP;
2533 goto out;
2534 }
2535
2536 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
2537 if (!acb) {
2538 BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1);
2539 data->bh = aio_bh_new(bdrv_get_aio_context(bs),
2540 bdrv_ioctl_bh_cb, data);
2541 data->co = &co;
2542 qemu_bh_schedule(data->bh);
2543 }
2544 qemu_coroutine_yield();
2545 out:
2546 tracked_request_end(&tracked_req);
2547 return co.ret;
2548 }
2549
2550 typedef struct {
2551 BlockDriverState *bs;
2552 int req;
2553 void *buf;
2554 int ret;
2555 } BdrvIoctlCoData;
2556
2557 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
2558 {
2559 BdrvIoctlCoData *data = opaque;
2560 data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
2561 }
2562
2563 /* needed for generic scsi interface */
2564 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
2565 {
2566 BdrvIoctlCoData data = {
2567 .bs = bs,
2568 .req = req,
2569 .buf = buf,
2570 .ret = -EINPROGRESS,
2571 };
2572
2573 if (qemu_in_coroutine()) {
2574 /* Fast-path if already in coroutine context */
2575 bdrv_co_ioctl_entry(&data);
2576 } else {
2577 Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry);
2578
2579 qemu_coroutine_enter(co, &data);
2580 while (data.ret == -EINPROGRESS) {
2581 aio_poll(bdrv_get_aio_context(bs), true);
2582 }
2583 }
2584 return data.ret;
2585 }
2586
2587 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
2588 {
2589 BlockAIOCBCoroutine *acb = opaque;
2590 acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
2591 acb->req.req, acb->req.buf);
2592 bdrv_co_complete(acb);
2593 }
2594
2595 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
2596 unsigned long int req, void *buf,
2597 BlockCompletionFunc *cb, void *opaque)
2598 {
2599 BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
2600 bs, cb, opaque);
2601 Coroutine *co;
2602
2603 acb->need_bh = true;
2604 acb->req.error = -EINPROGRESS;
2605 acb->req.req = req;
2606 acb->req.buf = buf;
2607 co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry);
2608 qemu_coroutine_enter(co, acb);
2609
2610 bdrv_co_maybe_schedule_bh(acb);
2611 return &acb->common;
2612 }
2613
2614 void *qemu_blockalign(BlockDriverState *bs, size_t size)
2615 {
2616 return qemu_memalign(bdrv_opt_mem_align(bs), size);
2617 }
2618
2619 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
2620 {
2621 return memset(qemu_blockalign(bs, size), 0, size);
2622 }
2623
2624 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
2625 {
2626 size_t align = bdrv_opt_mem_align(bs);
2627
2628 /* Ensure that NULL is never returned on success */
2629 assert(align > 0);
2630 if (size == 0) {
2631 size = align;
2632 }
2633
2634 return qemu_try_memalign(align, size);
2635 }
2636
2637 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
2638 {
2639 void *mem = qemu_try_blockalign(bs, size);
2640
2641 if (mem) {
2642 memset(mem, 0, size);
2643 }
2644
2645 return mem;
2646 }
2647
2648 /*
2649 * Check if all memory in this vector is sector aligned.
2650 */
2651 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
2652 {
2653 int i;
2654 size_t alignment = bdrv_min_mem_align(bs);
2655
2656 for (i = 0; i < qiov->niov; i++) {
2657 if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
2658 return false;
2659 }
2660 if (qiov->iov[i].iov_len % alignment) {
2661 return false;
2662 }
2663 }
2664
2665 return true;
2666 }
2667
2668 void bdrv_add_before_write_notifier(BlockDriverState *bs,
2669 NotifierWithReturn *notifier)
2670 {
2671 notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
2672 }
2673
2674 void bdrv_io_plug(BlockDriverState *bs)
2675 {
2676 BdrvChild *child;
2677
2678 QLIST_FOREACH(child, &bs->children, next) {
2679 bdrv_io_plug(child->bs);
2680 }
2681
2682 if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) {
2683 BlockDriver *drv = bs->drv;
2684 if (drv && drv->bdrv_io_plug) {
2685 drv->bdrv_io_plug(bs);
2686 }
2687 }
2688 }
2689
2690 void bdrv_io_unplug(BlockDriverState *bs)
2691 {
2692 BdrvChild *child;
2693
2694 assert(bs->io_plugged);
2695 if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) {
2696 BlockDriver *drv = bs->drv;
2697 if (drv && drv->bdrv_io_unplug) {
2698 drv->bdrv_io_unplug(bs);
2699 }
2700 }
2701
2702 QLIST_FOREACH(child, &bs->children, next) {
2703 bdrv_io_unplug(child->bs);
2704 }
2705 }
2706
2707 void bdrv_io_unplugged_begin(BlockDriverState *bs)
2708 {
2709 BdrvChild *child;
2710
2711 if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) {
2712 BlockDriver *drv = bs->drv;
2713 if (drv && drv->bdrv_io_unplug) {
2714 drv->bdrv_io_unplug(bs);
2715 }
2716 }
2717
2718 QLIST_FOREACH(child, &bs->children, next) {
2719 bdrv_io_unplugged_begin(child->bs);
2720 }
2721 }
2722
2723 void bdrv_io_unplugged_end(BlockDriverState *bs)
2724 {
2725 BdrvChild *child;
2726
2727 assert(bs->io_plug_disabled);
2728 QLIST_FOREACH(child, &bs->children, next) {
2729 bdrv_io_unplugged_end(child->bs);
2730 }
2731
2732 if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) {
2733 BlockDriver *drv = bs->drv;
2734 if (drv && drv->bdrv_io_plug) {
2735 drv->bdrv_io_plug(bs);
2736 }
2737 }
2738 }
2739
2740 void bdrv_drained_begin(BlockDriverState *bs)
2741 {
2742 if (!bs->quiesce_counter++) {
2743 aio_disable_external(bdrv_get_aio_context(bs));
2744 }
2745 bdrv_parent_drained_begin(bs);
2746 bdrv_drain(bs);
2747 }
2748
2749 void bdrv_drained_end(BlockDriverState *bs)
2750 {
2751 bdrv_parent_drained_end(bs);
2752
2753 assert(bs->quiesce_counter > 0);
2754 if (--bs->quiesce_counter > 0) {
2755 return;
2756 }
2757 aio_enable_external(bdrv_get_aio_context(bs));
2758 }