block: throttle-groups: Use BlockBackend pointers internally
[qemu.git] / block / io.c
1 /*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/blockjob.h"
29 #include "block/block_int.h"
30 #include "block/throttle-groups.h"
31 #include "qemu/cutils.h"
32 #include "qapi/error.h"
33 #include "qemu/error-report.h"
34
35 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
36
37 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
38 int64_t sector_num,
39 QEMUIOVector *qiov,
40 int nb_sectors,
41 BdrvRequestFlags flags,
42 BlockCompletionFunc *cb,
43 void *opaque,
44 bool is_write);
45 static void coroutine_fn bdrv_co_do_rw(void *opaque);
46 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
47 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags);
48
49 /* throttling disk I/O limits */
50 void bdrv_set_io_limits(BlockDriverState *bs,
51 ThrottleConfig *cfg)
52 {
53 throttle_group_config(bs, cfg);
54 }
55
56 void bdrv_no_throttling_begin(BlockDriverState *bs)
57 {
58 if (bs->io_limits_disabled++ == 0) {
59 throttle_group_restart_bs(bs);
60 }
61 }
62
63 void bdrv_no_throttling_end(BlockDriverState *bs)
64 {
65 assert(bs->io_limits_disabled);
66 --bs->io_limits_disabled;
67 }
68
69 void bdrv_io_limits_disable(BlockDriverState *bs)
70 {
71 assert(bs->throttle_state);
72 bdrv_no_throttling_begin(bs);
73 throttle_group_unregister_blk(bs->blk);
74 bdrv_no_throttling_end(bs);
75 }
76
77 /* should be called before bdrv_set_io_limits if a limit is set */
78 void bdrv_io_limits_enable(BlockDriverState *bs, const char *group)
79 {
80 assert(!bs->throttle_state);
81 throttle_group_register_blk(bs->blk, group);
82 }
83
84 void bdrv_io_limits_update_group(BlockDriverState *bs, const char *group)
85 {
86 /* this bs is not part of any group */
87 if (!bs->throttle_state) {
88 return;
89 }
90
91 /* this bs is a part of the same group than the one we want */
92 if (!g_strcmp0(throttle_group_get_name(bs), group)) {
93 return;
94 }
95
96 /* need to change the group this bs belong to */
97 bdrv_io_limits_disable(bs);
98 bdrv_io_limits_enable(bs, group);
99 }
100
101 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
102 {
103 BlockDriver *drv = bs->drv;
104 Error *local_err = NULL;
105
106 memset(&bs->bl, 0, sizeof(bs->bl));
107
108 if (!drv) {
109 return;
110 }
111
112 /* Take some limits from the children as a default */
113 if (bs->file) {
114 bdrv_refresh_limits(bs->file->bs, &local_err);
115 if (local_err) {
116 error_propagate(errp, local_err);
117 return;
118 }
119 bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length;
120 bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length;
121 bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment;
122 bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment;
123 bs->bl.max_iov = bs->file->bs->bl.max_iov;
124 } else {
125 bs->bl.min_mem_alignment = 512;
126 bs->bl.opt_mem_alignment = getpagesize();
127
128 /* Safe default since most protocols use readv()/writev()/etc */
129 bs->bl.max_iov = IOV_MAX;
130 }
131
132 if (bs->backing) {
133 bdrv_refresh_limits(bs->backing->bs, &local_err);
134 if (local_err) {
135 error_propagate(errp, local_err);
136 return;
137 }
138 bs->bl.opt_transfer_length =
139 MAX(bs->bl.opt_transfer_length,
140 bs->backing->bs->bl.opt_transfer_length);
141 bs->bl.max_transfer_length =
142 MIN_NON_ZERO(bs->bl.max_transfer_length,
143 bs->backing->bs->bl.max_transfer_length);
144 bs->bl.opt_mem_alignment =
145 MAX(bs->bl.opt_mem_alignment,
146 bs->backing->bs->bl.opt_mem_alignment);
147 bs->bl.min_mem_alignment =
148 MAX(bs->bl.min_mem_alignment,
149 bs->backing->bs->bl.min_mem_alignment);
150 bs->bl.max_iov =
151 MIN(bs->bl.max_iov,
152 bs->backing->bs->bl.max_iov);
153 }
154
155 /* Then let the driver override it */
156 if (drv->bdrv_refresh_limits) {
157 drv->bdrv_refresh_limits(bs, errp);
158 }
159 }
160
161 /**
162 * The copy-on-read flag is actually a reference count so multiple users may
163 * use the feature without worrying about clobbering its previous state.
164 * Copy-on-read stays enabled until all users have called to disable it.
165 */
166 void bdrv_enable_copy_on_read(BlockDriverState *bs)
167 {
168 bs->copy_on_read++;
169 }
170
171 void bdrv_disable_copy_on_read(BlockDriverState *bs)
172 {
173 assert(bs->copy_on_read > 0);
174 bs->copy_on_read--;
175 }
176
177 /* Check if any requests are in-flight (including throttled requests) */
178 bool bdrv_requests_pending(BlockDriverState *bs)
179 {
180 BdrvChild *child;
181
182 if (!QLIST_EMPTY(&bs->tracked_requests)) {
183 return true;
184 }
185 if (!qemu_co_queue_empty(&bs->throttled_reqs[0])) {
186 return true;
187 }
188 if (!qemu_co_queue_empty(&bs->throttled_reqs[1])) {
189 return true;
190 }
191
192 QLIST_FOREACH(child, &bs->children, next) {
193 if (bdrv_requests_pending(child->bs)) {
194 return true;
195 }
196 }
197
198 return false;
199 }
200
201 static void bdrv_drain_recurse(BlockDriverState *bs)
202 {
203 BdrvChild *child;
204
205 if (bs->drv && bs->drv->bdrv_drain) {
206 bs->drv->bdrv_drain(bs);
207 }
208 QLIST_FOREACH(child, &bs->children, next) {
209 bdrv_drain_recurse(child->bs);
210 }
211 }
212
213 typedef struct {
214 Coroutine *co;
215 BlockDriverState *bs;
216 QEMUBH *bh;
217 bool done;
218 } BdrvCoDrainData;
219
220 static void bdrv_drain_poll(BlockDriverState *bs)
221 {
222 bool busy = true;
223
224 while (busy) {
225 /* Keep iterating */
226 busy = bdrv_requests_pending(bs);
227 busy |= aio_poll(bdrv_get_aio_context(bs), busy);
228 }
229 }
230
231 static void bdrv_co_drain_bh_cb(void *opaque)
232 {
233 BdrvCoDrainData *data = opaque;
234 Coroutine *co = data->co;
235
236 qemu_bh_delete(data->bh);
237 bdrv_drain_poll(data->bs);
238 data->done = true;
239 qemu_coroutine_enter(co, NULL);
240 }
241
242 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
243 {
244 BdrvCoDrainData data;
245
246 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
247 * other coroutines run if they were queued from
248 * qemu_co_queue_run_restart(). */
249
250 assert(qemu_in_coroutine());
251 data = (BdrvCoDrainData) {
252 .co = qemu_coroutine_self(),
253 .bs = bs,
254 .done = false,
255 .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data),
256 };
257 qemu_bh_schedule(data.bh);
258
259 qemu_coroutine_yield();
260 /* If we are resumed from some other event (such as an aio completion or a
261 * timer callback), it is a bug in the caller that should be fixed. */
262 assert(data.done);
263 }
264
265 /*
266 * Wait for pending requests to complete on a single BlockDriverState subtree,
267 * and suspend block driver's internal I/O until next request arrives.
268 *
269 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
270 * AioContext.
271 *
272 * Only this BlockDriverState's AioContext is run, so in-flight requests must
273 * not depend on events in other AioContexts. In that case, use
274 * bdrv_drain_all() instead.
275 */
276 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
277 {
278 bdrv_no_throttling_begin(bs);
279 bdrv_io_unplugged_begin(bs);
280 bdrv_drain_recurse(bs);
281 bdrv_co_yield_to_drain(bs);
282 bdrv_io_unplugged_end(bs);
283 bdrv_no_throttling_end(bs);
284 }
285
286 void bdrv_drain(BlockDriverState *bs)
287 {
288 bdrv_no_throttling_begin(bs);
289 bdrv_io_unplugged_begin(bs);
290 bdrv_drain_recurse(bs);
291 if (qemu_in_coroutine()) {
292 bdrv_co_yield_to_drain(bs);
293 } else {
294 bdrv_drain_poll(bs);
295 }
296 bdrv_io_unplugged_end(bs);
297 bdrv_no_throttling_end(bs);
298 }
299
300 /*
301 * Wait for pending requests to complete across all BlockDriverStates
302 *
303 * This function does not flush data to disk, use bdrv_flush_all() for that
304 * after calling this function.
305 */
306 void bdrv_drain_all(void)
307 {
308 /* Always run first iteration so any pending completion BHs run */
309 bool busy = true;
310 BlockDriverState *bs = NULL;
311 GSList *aio_ctxs = NULL, *ctx;
312
313 while ((bs = bdrv_next(bs))) {
314 AioContext *aio_context = bdrv_get_aio_context(bs);
315
316 aio_context_acquire(aio_context);
317 if (bs->job) {
318 block_job_pause(bs->job);
319 }
320 bdrv_no_throttling_begin(bs);
321 bdrv_io_unplugged_begin(bs);
322 bdrv_drain_recurse(bs);
323 aio_context_release(aio_context);
324
325 if (!g_slist_find(aio_ctxs, aio_context)) {
326 aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
327 }
328 }
329
330 /* Note that completion of an asynchronous I/O operation can trigger any
331 * number of other I/O operations on other devices---for example a
332 * coroutine can submit an I/O request to another device in response to
333 * request completion. Therefore we must keep looping until there was no
334 * more activity rather than simply draining each device independently.
335 */
336 while (busy) {
337 busy = false;
338
339 for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
340 AioContext *aio_context = ctx->data;
341 bs = NULL;
342
343 aio_context_acquire(aio_context);
344 while ((bs = bdrv_next(bs))) {
345 if (aio_context == bdrv_get_aio_context(bs)) {
346 if (bdrv_requests_pending(bs)) {
347 busy = true;
348 aio_poll(aio_context, busy);
349 }
350 }
351 }
352 busy |= aio_poll(aio_context, false);
353 aio_context_release(aio_context);
354 }
355 }
356
357 bs = NULL;
358 while ((bs = bdrv_next(bs))) {
359 AioContext *aio_context = bdrv_get_aio_context(bs);
360
361 aio_context_acquire(aio_context);
362 bdrv_io_unplugged_end(bs);
363 bdrv_no_throttling_end(bs);
364 if (bs->job) {
365 block_job_resume(bs->job);
366 }
367 aio_context_release(aio_context);
368 }
369 g_slist_free(aio_ctxs);
370 }
371
372 /**
373 * Remove an active request from the tracked requests list
374 *
375 * This function should be called when a tracked request is completing.
376 */
377 static void tracked_request_end(BdrvTrackedRequest *req)
378 {
379 if (req->serialising) {
380 req->bs->serialising_in_flight--;
381 }
382
383 QLIST_REMOVE(req, list);
384 qemu_co_queue_restart_all(&req->wait_queue);
385 }
386
387 /**
388 * Add an active request to the tracked requests list
389 */
390 static void tracked_request_begin(BdrvTrackedRequest *req,
391 BlockDriverState *bs,
392 int64_t offset,
393 unsigned int bytes,
394 enum BdrvTrackedRequestType type)
395 {
396 *req = (BdrvTrackedRequest){
397 .bs = bs,
398 .offset = offset,
399 .bytes = bytes,
400 .type = type,
401 .co = qemu_coroutine_self(),
402 .serialising = false,
403 .overlap_offset = offset,
404 .overlap_bytes = bytes,
405 };
406
407 qemu_co_queue_init(&req->wait_queue);
408
409 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
410 }
411
412 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
413 {
414 int64_t overlap_offset = req->offset & ~(align - 1);
415 unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
416 - overlap_offset;
417
418 if (!req->serialising) {
419 req->bs->serialising_in_flight++;
420 req->serialising = true;
421 }
422
423 req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
424 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
425 }
426
427 /**
428 * Round a region to cluster boundaries
429 */
430 void bdrv_round_to_clusters(BlockDriverState *bs,
431 int64_t sector_num, int nb_sectors,
432 int64_t *cluster_sector_num,
433 int *cluster_nb_sectors)
434 {
435 BlockDriverInfo bdi;
436
437 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
438 *cluster_sector_num = sector_num;
439 *cluster_nb_sectors = nb_sectors;
440 } else {
441 int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
442 *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
443 *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
444 nb_sectors, c);
445 }
446 }
447
448 static int bdrv_get_cluster_size(BlockDriverState *bs)
449 {
450 BlockDriverInfo bdi;
451 int ret;
452
453 ret = bdrv_get_info(bs, &bdi);
454 if (ret < 0 || bdi.cluster_size == 0) {
455 return bs->request_alignment;
456 } else {
457 return bdi.cluster_size;
458 }
459 }
460
461 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
462 int64_t offset, unsigned int bytes)
463 {
464 /* aaaa bbbb */
465 if (offset >= req->overlap_offset + req->overlap_bytes) {
466 return false;
467 }
468 /* bbbb aaaa */
469 if (req->overlap_offset >= offset + bytes) {
470 return false;
471 }
472 return true;
473 }
474
475 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
476 {
477 BlockDriverState *bs = self->bs;
478 BdrvTrackedRequest *req;
479 bool retry;
480 bool waited = false;
481
482 if (!bs->serialising_in_flight) {
483 return false;
484 }
485
486 do {
487 retry = false;
488 QLIST_FOREACH(req, &bs->tracked_requests, list) {
489 if (req == self || (!req->serialising && !self->serialising)) {
490 continue;
491 }
492 if (tracked_request_overlaps(req, self->overlap_offset,
493 self->overlap_bytes))
494 {
495 /* Hitting this means there was a reentrant request, for
496 * example, a block driver issuing nested requests. This must
497 * never happen since it means deadlock.
498 */
499 assert(qemu_coroutine_self() != req->co);
500
501 /* If the request is already (indirectly) waiting for us, or
502 * will wait for us as soon as it wakes up, then just go on
503 * (instead of producing a deadlock in the former case). */
504 if (!req->waiting_for) {
505 self->waiting_for = req;
506 qemu_co_queue_wait(&req->wait_queue);
507 self->waiting_for = NULL;
508 retry = true;
509 waited = true;
510 break;
511 }
512 }
513 }
514 } while (retry);
515
516 return waited;
517 }
518
519 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
520 size_t size)
521 {
522 if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
523 return -EIO;
524 }
525
526 if (!bdrv_is_inserted(bs)) {
527 return -ENOMEDIUM;
528 }
529
530 if (offset < 0) {
531 return -EIO;
532 }
533
534 return 0;
535 }
536
537 static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
538 int nb_sectors)
539 {
540 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
541 return -EIO;
542 }
543
544 return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
545 nb_sectors * BDRV_SECTOR_SIZE);
546 }
547
548 typedef struct RwCo {
549 BlockDriverState *bs;
550 int64_t offset;
551 QEMUIOVector *qiov;
552 bool is_write;
553 int ret;
554 BdrvRequestFlags flags;
555 } RwCo;
556
557 static void coroutine_fn bdrv_rw_co_entry(void *opaque)
558 {
559 RwCo *rwco = opaque;
560
561 if (!rwco->is_write) {
562 rwco->ret = bdrv_co_preadv(rwco->bs, rwco->offset,
563 rwco->qiov->size, rwco->qiov,
564 rwco->flags);
565 } else {
566 rwco->ret = bdrv_co_pwritev(rwco->bs, rwco->offset,
567 rwco->qiov->size, rwco->qiov,
568 rwco->flags);
569 }
570 }
571
572 /*
573 * Process a vectored synchronous request using coroutines
574 */
575 static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset,
576 QEMUIOVector *qiov, bool is_write,
577 BdrvRequestFlags flags)
578 {
579 Coroutine *co;
580 RwCo rwco = {
581 .bs = bs,
582 .offset = offset,
583 .qiov = qiov,
584 .is_write = is_write,
585 .ret = NOT_DONE,
586 .flags = flags,
587 };
588
589 if (qemu_in_coroutine()) {
590 /* Fast-path if already in coroutine context */
591 bdrv_rw_co_entry(&rwco);
592 } else {
593 AioContext *aio_context = bdrv_get_aio_context(bs);
594
595 co = qemu_coroutine_create(bdrv_rw_co_entry);
596 qemu_coroutine_enter(co, &rwco);
597 while (rwco.ret == NOT_DONE) {
598 aio_poll(aio_context, true);
599 }
600 }
601 return rwco.ret;
602 }
603
604 /*
605 * Process a synchronous request using coroutines
606 */
607 static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf,
608 int nb_sectors, bool is_write, BdrvRequestFlags flags)
609 {
610 QEMUIOVector qiov;
611 struct iovec iov = {
612 .iov_base = (void *)buf,
613 .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
614 };
615
616 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
617 return -EINVAL;
618 }
619
620 qemu_iovec_init_external(&qiov, &iov, 1);
621 return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS,
622 &qiov, is_write, flags);
623 }
624
625 /* return < 0 if error. See bdrv_write() for the return codes */
626 int bdrv_read(BlockDriverState *bs, int64_t sector_num,
627 uint8_t *buf, int nb_sectors)
628 {
629 return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0);
630 }
631
632 /* Return < 0 if error. Important errors are:
633 -EIO generic I/O error (may happen for all errors)
634 -ENOMEDIUM No media inserted.
635 -EINVAL Invalid sector number or nb_sectors
636 -EACCES Trying to write a read-only device
637 */
638 int bdrv_write(BlockDriverState *bs, int64_t sector_num,
639 const uint8_t *buf, int nb_sectors)
640 {
641 return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
642 }
643
644 int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
645 int nb_sectors, BdrvRequestFlags flags)
646 {
647 return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true,
648 BDRV_REQ_ZERO_WRITE | flags);
649 }
650
651 /*
652 * Completely zero out a block device with the help of bdrv_write_zeroes.
653 * The operation is sped up by checking the block status and only writing
654 * zeroes to the device if they currently do not return zeroes. Optional
655 * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
656 * BDRV_REQ_FUA).
657 *
658 * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
659 */
660 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
661 {
662 int64_t target_sectors, ret, nb_sectors, sector_num = 0;
663 BlockDriverState *file;
664 int n;
665
666 target_sectors = bdrv_nb_sectors(bs);
667 if (target_sectors < 0) {
668 return target_sectors;
669 }
670
671 for (;;) {
672 nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
673 if (nb_sectors <= 0) {
674 return 0;
675 }
676 ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
677 if (ret < 0) {
678 error_report("error getting block status at sector %" PRId64 ": %s",
679 sector_num, strerror(-ret));
680 return ret;
681 }
682 if (ret & BDRV_BLOCK_ZERO) {
683 sector_num += n;
684 continue;
685 }
686 ret = bdrv_write_zeroes(bs, sector_num, n, flags);
687 if (ret < 0) {
688 error_report("error writing zeroes at sector %" PRId64 ": %s",
689 sector_num, strerror(-ret));
690 return ret;
691 }
692 sector_num += n;
693 }
694 }
695
696 int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes)
697 {
698 QEMUIOVector qiov;
699 struct iovec iov = {
700 .iov_base = (void *)buf,
701 .iov_len = bytes,
702 };
703 int ret;
704
705 if (bytes < 0) {
706 return -EINVAL;
707 }
708
709 qemu_iovec_init_external(&qiov, &iov, 1);
710 ret = bdrv_prwv_co(bs, offset, &qiov, false, 0);
711 if (ret < 0) {
712 return ret;
713 }
714
715 return bytes;
716 }
717
718 int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov)
719 {
720 int ret;
721
722 ret = bdrv_prwv_co(bs, offset, qiov, true, 0);
723 if (ret < 0) {
724 return ret;
725 }
726
727 return qiov->size;
728 }
729
730 int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
731 const void *buf, int bytes)
732 {
733 QEMUIOVector qiov;
734 struct iovec iov = {
735 .iov_base = (void *) buf,
736 .iov_len = bytes,
737 };
738
739 if (bytes < 0) {
740 return -EINVAL;
741 }
742
743 qemu_iovec_init_external(&qiov, &iov, 1);
744 return bdrv_pwritev(bs, offset, &qiov);
745 }
746
747 /*
748 * Writes to the file and ensures that no writes are reordered across this
749 * request (acts as a barrier)
750 *
751 * Returns 0 on success, -errno in error cases.
752 */
753 int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
754 const void *buf, int count)
755 {
756 int ret;
757
758 ret = bdrv_pwrite(bs, offset, buf, count);
759 if (ret < 0) {
760 return ret;
761 }
762
763 ret = bdrv_flush(bs);
764 if (ret < 0) {
765 return ret;
766 }
767
768 return 0;
769 }
770
771 typedef struct CoroutineIOCompletion {
772 Coroutine *coroutine;
773 int ret;
774 } CoroutineIOCompletion;
775
776 static void bdrv_co_io_em_complete(void *opaque, int ret)
777 {
778 CoroutineIOCompletion *co = opaque;
779
780 co->ret = ret;
781 qemu_coroutine_enter(co->coroutine, NULL);
782 }
783
784 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
785 uint64_t offset, uint64_t bytes,
786 QEMUIOVector *qiov, int flags)
787 {
788 BlockDriver *drv = bs->drv;
789 int64_t sector_num;
790 unsigned int nb_sectors;
791
792 if (drv->bdrv_co_preadv) {
793 return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
794 }
795
796 sector_num = offset >> BDRV_SECTOR_BITS;
797 nb_sectors = bytes >> BDRV_SECTOR_BITS;
798
799 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
800 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
801 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
802
803 if (drv->bdrv_co_readv) {
804 return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
805 } else {
806 BlockAIOCB *acb;
807 CoroutineIOCompletion co = {
808 .coroutine = qemu_coroutine_self(),
809 };
810
811 acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
812 bdrv_co_io_em_complete, &co);
813 if (acb == NULL) {
814 return -EIO;
815 } else {
816 qemu_coroutine_yield();
817 return co.ret;
818 }
819 }
820 }
821
822 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
823 uint64_t offset, uint64_t bytes,
824 QEMUIOVector *qiov, int flags)
825 {
826 BlockDriver *drv = bs->drv;
827 int64_t sector_num;
828 unsigned int nb_sectors;
829 int ret;
830
831 if (drv->bdrv_co_pwritev) {
832 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags);
833 goto emulate_flags;
834 }
835
836 sector_num = offset >> BDRV_SECTOR_BITS;
837 nb_sectors = bytes >> BDRV_SECTOR_BITS;
838
839 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
840 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
841 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
842
843 if (drv->bdrv_co_writev_flags) {
844 ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
845 flags & bs->supported_write_flags);
846 flags &= ~bs->supported_write_flags;
847 } else if (drv->bdrv_co_writev) {
848 assert(!bs->supported_write_flags);
849 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
850 } else {
851 BlockAIOCB *acb;
852 CoroutineIOCompletion co = {
853 .coroutine = qemu_coroutine_self(),
854 };
855
856 acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
857 bdrv_co_io_em_complete, &co);
858 if (acb == NULL) {
859 ret = -EIO;
860 } else {
861 qemu_coroutine_yield();
862 ret = co.ret;
863 }
864 }
865
866 emulate_flags:
867 if (ret == 0 && (flags & BDRV_REQ_FUA)) {
868 ret = bdrv_co_flush(bs);
869 }
870
871 return ret;
872 }
873
874 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
875 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
876 {
877 /* Perform I/O through a temporary buffer so that users who scribble over
878 * their read buffer while the operation is in progress do not end up
879 * modifying the image file. This is critical for zero-copy guest I/O
880 * where anything might happen inside guest memory.
881 */
882 void *bounce_buffer;
883
884 BlockDriver *drv = bs->drv;
885 struct iovec iov;
886 QEMUIOVector bounce_qiov;
887 int64_t cluster_sector_num;
888 int cluster_nb_sectors;
889 size_t skip_bytes;
890 int ret;
891
892 /* Cover entire cluster so no additional backing file I/O is required when
893 * allocating cluster in the image file.
894 */
895 bdrv_round_to_clusters(bs, sector_num, nb_sectors,
896 &cluster_sector_num, &cluster_nb_sectors);
897
898 trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
899 cluster_sector_num, cluster_nb_sectors);
900
901 iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
902 iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
903 if (bounce_buffer == NULL) {
904 ret = -ENOMEM;
905 goto err;
906 }
907
908 qemu_iovec_init_external(&bounce_qiov, &iov, 1);
909
910 ret = bdrv_driver_preadv(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
911 cluster_nb_sectors * BDRV_SECTOR_SIZE,
912 &bounce_qiov, 0);
913 if (ret < 0) {
914 goto err;
915 }
916
917 if (drv->bdrv_co_write_zeroes &&
918 buffer_is_zero(bounce_buffer, iov.iov_len)) {
919 ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num,
920 cluster_nb_sectors, 0);
921 } else {
922 /* This does not change the data on the disk, it is not necessary
923 * to flush even in cache=writethrough mode.
924 */
925 ret = bdrv_driver_pwritev(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
926 cluster_nb_sectors * BDRV_SECTOR_SIZE,
927 &bounce_qiov, 0);
928 }
929
930 if (ret < 0) {
931 /* It might be okay to ignore write errors for guest requests. If this
932 * is a deliberate copy-on-read then we don't want to ignore the error.
933 * Simply report it in all cases.
934 */
935 goto err;
936 }
937
938 skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE;
939 qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes,
940 nb_sectors * BDRV_SECTOR_SIZE);
941
942 err:
943 qemu_vfree(bounce_buffer);
944 return ret;
945 }
946
947 /*
948 * Forwards an already correctly aligned request to the BlockDriver. This
949 * handles copy on read and zeroing after EOF; any other features must be
950 * implemented by the caller.
951 */
952 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
953 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
954 int64_t align, QEMUIOVector *qiov, int flags)
955 {
956 int ret;
957
958 int64_t sector_num = offset >> BDRV_SECTOR_BITS;
959 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
960
961 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
962 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
963 assert(!qiov || bytes == qiov->size);
964 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
965
966 /* Handle Copy on Read and associated serialisation */
967 if (flags & BDRV_REQ_COPY_ON_READ) {
968 /* If we touch the same cluster it counts as an overlap. This
969 * guarantees that allocating writes will be serialized and not race
970 * with each other for the same cluster. For example, in copy-on-read
971 * it ensures that the CoR read and write operations are atomic and
972 * guest writes cannot interleave between them. */
973 mark_request_serialising(req, bdrv_get_cluster_size(bs));
974 }
975
976 if (!(flags & BDRV_REQ_NO_SERIALISING)) {
977 wait_serialising_requests(req);
978 }
979
980 if (flags & BDRV_REQ_COPY_ON_READ) {
981 int pnum;
982
983 ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum);
984 if (ret < 0) {
985 goto out;
986 }
987
988 if (!ret || pnum != nb_sectors) {
989 ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov);
990 goto out;
991 }
992 }
993
994 /* Forward the request to the BlockDriver */
995 if (!bs->zero_beyond_eof) {
996 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
997 } else {
998 /* Read zeros after EOF */
999 int64_t total_sectors, max_nb_sectors;
1000
1001 total_sectors = bdrv_nb_sectors(bs);
1002 if (total_sectors < 0) {
1003 ret = total_sectors;
1004 goto out;
1005 }
1006
1007 max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num),
1008 align >> BDRV_SECTOR_BITS);
1009 if (nb_sectors < max_nb_sectors) {
1010 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
1011 } else if (max_nb_sectors > 0) {
1012 QEMUIOVector local_qiov;
1013
1014 qemu_iovec_init(&local_qiov, qiov->niov);
1015 qemu_iovec_concat(&local_qiov, qiov, 0,
1016 max_nb_sectors * BDRV_SECTOR_SIZE);
1017
1018 ret = bdrv_driver_preadv(bs, offset,
1019 max_nb_sectors * BDRV_SECTOR_SIZE,
1020 &local_qiov, 0);
1021
1022 qemu_iovec_destroy(&local_qiov);
1023 } else {
1024 ret = 0;
1025 }
1026
1027 /* Reading beyond end of file is supposed to produce zeroes */
1028 if (ret == 0 && total_sectors < sector_num + nb_sectors) {
1029 uint64_t offset = MAX(0, total_sectors - sector_num);
1030 uint64_t bytes = (sector_num + nb_sectors - offset) *
1031 BDRV_SECTOR_SIZE;
1032 qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes);
1033 }
1034 }
1035
1036 out:
1037 return ret;
1038 }
1039
1040 /*
1041 * Handle a read request in coroutine context
1042 */
1043 int coroutine_fn bdrv_co_preadv(BlockDriverState *bs,
1044 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1045 BdrvRequestFlags flags)
1046 {
1047 BlockDriver *drv = bs->drv;
1048 BdrvTrackedRequest req;
1049
1050 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1051 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1052 uint8_t *head_buf = NULL;
1053 uint8_t *tail_buf = NULL;
1054 QEMUIOVector local_qiov;
1055 bool use_local_qiov = false;
1056 int ret;
1057
1058 if (!drv) {
1059 return -ENOMEDIUM;
1060 }
1061
1062 ret = bdrv_check_byte_request(bs, offset, bytes);
1063 if (ret < 0) {
1064 return ret;
1065 }
1066
1067 /* Don't do copy-on-read if we read data before write operation */
1068 if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
1069 flags |= BDRV_REQ_COPY_ON_READ;
1070 }
1071
1072 /* throttling disk I/O */
1073 if (bs->throttle_state) {
1074 throttle_group_co_io_limits_intercept(bs, bytes, false);
1075 }
1076
1077 /* Align read if necessary by padding qiov */
1078 if (offset & (align - 1)) {
1079 head_buf = qemu_blockalign(bs, align);
1080 qemu_iovec_init(&local_qiov, qiov->niov + 2);
1081 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1082 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1083 use_local_qiov = true;
1084
1085 bytes += offset & (align - 1);
1086 offset = offset & ~(align - 1);
1087 }
1088
1089 if ((offset + bytes) & (align - 1)) {
1090 if (!use_local_qiov) {
1091 qemu_iovec_init(&local_qiov, qiov->niov + 1);
1092 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1093 use_local_qiov = true;
1094 }
1095 tail_buf = qemu_blockalign(bs, align);
1096 qemu_iovec_add(&local_qiov, tail_buf,
1097 align - ((offset + bytes) & (align - 1)));
1098
1099 bytes = ROUND_UP(bytes, align);
1100 }
1101
1102 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1103 ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
1104 use_local_qiov ? &local_qiov : qiov,
1105 flags);
1106 tracked_request_end(&req);
1107
1108 if (use_local_qiov) {
1109 qemu_iovec_destroy(&local_qiov);
1110 qemu_vfree(head_buf);
1111 qemu_vfree(tail_buf);
1112 }
1113
1114 return ret;
1115 }
1116
1117 static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
1118 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1119 BdrvRequestFlags flags)
1120 {
1121 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1122 return -EINVAL;
1123 }
1124
1125 return bdrv_co_preadv(bs, sector_num << BDRV_SECTOR_BITS,
1126 nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1127 }
1128
1129 int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
1130 int nb_sectors, QEMUIOVector *qiov)
1131 {
1132 trace_bdrv_co_readv(bs, sector_num, nb_sectors);
1133
1134 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
1135 }
1136
1137 int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs,
1138 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
1139 {
1140 trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors);
1141
1142 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
1143 BDRV_REQ_NO_SERIALISING);
1144 }
1145
1146 int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
1147 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
1148 {
1149 trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors);
1150
1151 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
1152 BDRV_REQ_COPY_ON_READ);
1153 }
1154
1155 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768
1156
1157 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
1158 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
1159 {
1160 BlockDriver *drv = bs->drv;
1161 QEMUIOVector qiov;
1162 struct iovec iov = {0};
1163 int ret = 0;
1164 bool need_flush = false;
1165
1166 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes,
1167 BDRV_REQUEST_MAX_SECTORS);
1168
1169 while (nb_sectors > 0 && !ret) {
1170 int num = nb_sectors;
1171
1172 /* Align request. Block drivers can expect the "bulk" of the request
1173 * to be aligned.
1174 */
1175 if (bs->bl.write_zeroes_alignment
1176 && num > bs->bl.write_zeroes_alignment) {
1177 if (sector_num % bs->bl.write_zeroes_alignment != 0) {
1178 /* Make a small request up to the first aligned sector. */
1179 num = bs->bl.write_zeroes_alignment;
1180 num -= sector_num % bs->bl.write_zeroes_alignment;
1181 } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) {
1182 /* Shorten the request to the last aligned sector. num cannot
1183 * underflow because num > bs->bl.write_zeroes_alignment.
1184 */
1185 num -= (sector_num + num) % bs->bl.write_zeroes_alignment;
1186 }
1187 }
1188
1189 /* limit request size */
1190 if (num > max_write_zeroes) {
1191 num = max_write_zeroes;
1192 }
1193
1194 ret = -ENOTSUP;
1195 /* First try the efficient write zeroes operation */
1196 if (drv->bdrv_co_write_zeroes) {
1197 ret = drv->bdrv_co_write_zeroes(bs, sector_num, num,
1198 flags & bs->supported_zero_flags);
1199 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1200 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1201 need_flush = true;
1202 }
1203 } else {
1204 assert(!bs->supported_zero_flags);
1205 }
1206
1207 if (ret == -ENOTSUP) {
1208 /* Fall back to bounce buffer if write zeroes is unsupported */
1209 int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length,
1210 MAX_WRITE_ZEROES_BOUNCE_BUFFER);
1211 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1212
1213 if ((flags & BDRV_REQ_FUA) &&
1214 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1215 /* No need for bdrv_driver_pwrite() to do a fallback
1216 * flush on each chunk; use just one at the end */
1217 write_flags &= ~BDRV_REQ_FUA;
1218 need_flush = true;
1219 }
1220 num = MIN(num, max_xfer_len);
1221 iov.iov_len = num * BDRV_SECTOR_SIZE;
1222 if (iov.iov_base == NULL) {
1223 iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE);
1224 if (iov.iov_base == NULL) {
1225 ret = -ENOMEM;
1226 goto fail;
1227 }
1228 memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE);
1229 }
1230 qemu_iovec_init_external(&qiov, &iov, 1);
1231
1232 ret = bdrv_driver_pwritev(bs, sector_num * BDRV_SECTOR_SIZE,
1233 num * BDRV_SECTOR_SIZE, &qiov,
1234 write_flags);
1235
1236 /* Keep bounce buffer around if it is big enough for all
1237 * all future requests.
1238 */
1239 if (num < max_xfer_len) {
1240 qemu_vfree(iov.iov_base);
1241 iov.iov_base = NULL;
1242 }
1243 }
1244
1245 sector_num += num;
1246 nb_sectors -= num;
1247 }
1248
1249 fail:
1250 if (ret == 0 && need_flush) {
1251 ret = bdrv_co_flush(bs);
1252 }
1253 qemu_vfree(iov.iov_base);
1254 return ret;
1255 }
1256
1257 /*
1258 * Forwards an already correctly aligned write request to the BlockDriver.
1259 */
1260 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
1261 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1262 QEMUIOVector *qiov, int flags)
1263 {
1264 BlockDriver *drv = bs->drv;
1265 bool waited;
1266 int ret;
1267
1268 int64_t sector_num = offset >> BDRV_SECTOR_BITS;
1269 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
1270
1271 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
1272 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
1273 assert(!qiov || bytes == qiov->size);
1274 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1275
1276 waited = wait_serialising_requests(req);
1277 assert(!waited || !req->serialising);
1278 assert(req->overlap_offset <= offset);
1279 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1280
1281 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
1282
1283 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1284 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes &&
1285 qemu_iovec_is_zero(qiov)) {
1286 flags |= BDRV_REQ_ZERO_WRITE;
1287 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1288 flags |= BDRV_REQ_MAY_UNMAP;
1289 }
1290 }
1291
1292 if (ret < 0) {
1293 /* Do nothing, write notifier decided to fail this request */
1294 } else if (flags & BDRV_REQ_ZERO_WRITE) {
1295 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1296 ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags);
1297 } else {
1298 bdrv_debug_event(bs, BLKDBG_PWRITEV);
1299 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
1300 }
1301 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1302
1303 bdrv_set_dirty(bs, sector_num, nb_sectors);
1304
1305 if (bs->wr_highest_offset < offset + bytes) {
1306 bs->wr_highest_offset = offset + bytes;
1307 }
1308
1309 if (ret >= 0) {
1310 bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
1311 }
1312
1313 return ret;
1314 }
1315
1316 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
1317 int64_t offset,
1318 unsigned int bytes,
1319 BdrvRequestFlags flags,
1320 BdrvTrackedRequest *req)
1321 {
1322 uint8_t *buf = NULL;
1323 QEMUIOVector local_qiov;
1324 struct iovec iov;
1325 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1326 unsigned int head_padding_bytes, tail_padding_bytes;
1327 int ret = 0;
1328
1329 head_padding_bytes = offset & (align - 1);
1330 tail_padding_bytes = align - ((offset + bytes) & (align - 1));
1331
1332
1333 assert(flags & BDRV_REQ_ZERO_WRITE);
1334 if (head_padding_bytes || tail_padding_bytes) {
1335 buf = qemu_blockalign(bs, align);
1336 iov = (struct iovec) {
1337 .iov_base = buf,
1338 .iov_len = align,
1339 };
1340 qemu_iovec_init_external(&local_qiov, &iov, 1);
1341 }
1342 if (head_padding_bytes) {
1343 uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
1344
1345 /* RMW the unaligned part before head. */
1346 mark_request_serialising(req, align);
1347 wait_serialising_requests(req);
1348 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1349 ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
1350 align, &local_qiov, 0);
1351 if (ret < 0) {
1352 goto fail;
1353 }
1354 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1355
1356 memset(buf + head_padding_bytes, 0, zero_bytes);
1357 ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
1358 &local_qiov,
1359 flags & ~BDRV_REQ_ZERO_WRITE);
1360 if (ret < 0) {
1361 goto fail;
1362 }
1363 offset += zero_bytes;
1364 bytes -= zero_bytes;
1365 }
1366
1367 assert(!bytes || (offset & (align - 1)) == 0);
1368 if (bytes >= align) {
1369 /* Write the aligned part in the middle. */
1370 uint64_t aligned_bytes = bytes & ~(align - 1);
1371 ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes,
1372 NULL, flags);
1373 if (ret < 0) {
1374 goto fail;
1375 }
1376 bytes -= aligned_bytes;
1377 offset += aligned_bytes;
1378 }
1379
1380 assert(!bytes || (offset & (align - 1)) == 0);
1381 if (bytes) {
1382 assert(align == tail_padding_bytes + bytes);
1383 /* RMW the unaligned part after tail. */
1384 mark_request_serialising(req, align);
1385 wait_serialising_requests(req);
1386 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1387 ret = bdrv_aligned_preadv(bs, req, offset, align,
1388 align, &local_qiov, 0);
1389 if (ret < 0) {
1390 goto fail;
1391 }
1392 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1393
1394 memset(buf, 0, bytes);
1395 ret = bdrv_aligned_pwritev(bs, req, offset, align,
1396 &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
1397 }
1398 fail:
1399 qemu_vfree(buf);
1400 return ret;
1401
1402 }
1403
1404 /*
1405 * Handle a write request in coroutine context
1406 */
1407 int coroutine_fn bdrv_co_pwritev(BlockDriverState *bs,
1408 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1409 BdrvRequestFlags flags)
1410 {
1411 BdrvTrackedRequest req;
1412 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1413 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1414 uint8_t *head_buf = NULL;
1415 uint8_t *tail_buf = NULL;
1416 QEMUIOVector local_qiov;
1417 bool use_local_qiov = false;
1418 int ret;
1419
1420 if (!bs->drv) {
1421 return -ENOMEDIUM;
1422 }
1423 if (bs->read_only) {
1424 return -EPERM;
1425 }
1426 assert(!(bs->open_flags & BDRV_O_INACTIVE));
1427
1428 ret = bdrv_check_byte_request(bs, offset, bytes);
1429 if (ret < 0) {
1430 return ret;
1431 }
1432
1433 /* throttling disk I/O */
1434 if (bs->throttle_state) {
1435 throttle_group_co_io_limits_intercept(bs, bytes, true);
1436 }
1437
1438 /*
1439 * Align write if necessary by performing a read-modify-write cycle.
1440 * Pad qiov with the read parts and be sure to have a tracked request not
1441 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
1442 */
1443 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
1444
1445 if (!qiov) {
1446 ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
1447 goto out;
1448 }
1449
1450 if (offset & (align - 1)) {
1451 QEMUIOVector head_qiov;
1452 struct iovec head_iov;
1453
1454 mark_request_serialising(&req, align);
1455 wait_serialising_requests(&req);
1456
1457 head_buf = qemu_blockalign(bs, align);
1458 head_iov = (struct iovec) {
1459 .iov_base = head_buf,
1460 .iov_len = align,
1461 };
1462 qemu_iovec_init_external(&head_qiov, &head_iov, 1);
1463
1464 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1465 ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
1466 align, &head_qiov, 0);
1467 if (ret < 0) {
1468 goto fail;
1469 }
1470 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1471
1472 qemu_iovec_init(&local_qiov, qiov->niov + 2);
1473 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1474 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1475 use_local_qiov = true;
1476
1477 bytes += offset & (align - 1);
1478 offset = offset & ~(align - 1);
1479 }
1480
1481 if ((offset + bytes) & (align - 1)) {
1482 QEMUIOVector tail_qiov;
1483 struct iovec tail_iov;
1484 size_t tail_bytes;
1485 bool waited;
1486
1487 mark_request_serialising(&req, align);
1488 waited = wait_serialising_requests(&req);
1489 assert(!waited || !use_local_qiov);
1490
1491 tail_buf = qemu_blockalign(bs, align);
1492 tail_iov = (struct iovec) {
1493 .iov_base = tail_buf,
1494 .iov_len = align,
1495 };
1496 qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
1497
1498 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1499 ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
1500 align, &tail_qiov, 0);
1501 if (ret < 0) {
1502 goto fail;
1503 }
1504 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1505
1506 if (!use_local_qiov) {
1507 qemu_iovec_init(&local_qiov, qiov->niov + 1);
1508 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1509 use_local_qiov = true;
1510 }
1511
1512 tail_bytes = (offset + bytes) & (align - 1);
1513 qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
1514
1515 bytes = ROUND_UP(bytes, align);
1516 }
1517
1518 ret = bdrv_aligned_pwritev(bs, &req, offset, bytes,
1519 use_local_qiov ? &local_qiov : qiov,
1520 flags);
1521
1522 fail:
1523
1524 if (use_local_qiov) {
1525 qemu_iovec_destroy(&local_qiov);
1526 }
1527 qemu_vfree(head_buf);
1528 qemu_vfree(tail_buf);
1529 out:
1530 tracked_request_end(&req);
1531 return ret;
1532 }
1533
1534 static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
1535 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1536 BdrvRequestFlags flags)
1537 {
1538 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1539 return -EINVAL;
1540 }
1541
1542 return bdrv_co_pwritev(bs, sector_num << BDRV_SECTOR_BITS,
1543 nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1544 }
1545
1546 int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
1547 int nb_sectors, QEMUIOVector *qiov)
1548 {
1549 trace_bdrv_co_writev(bs, sector_num, nb_sectors);
1550
1551 return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0);
1552 }
1553
1554 int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
1555 int64_t sector_num, int nb_sectors,
1556 BdrvRequestFlags flags)
1557 {
1558 trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags);
1559
1560 if (!(bs->open_flags & BDRV_O_UNMAP)) {
1561 flags &= ~BDRV_REQ_MAY_UNMAP;
1562 }
1563
1564 return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL,
1565 BDRV_REQ_ZERO_WRITE | flags);
1566 }
1567
1568 typedef struct BdrvCoGetBlockStatusData {
1569 BlockDriverState *bs;
1570 BlockDriverState *base;
1571 BlockDriverState **file;
1572 int64_t sector_num;
1573 int nb_sectors;
1574 int *pnum;
1575 int64_t ret;
1576 bool done;
1577 } BdrvCoGetBlockStatusData;
1578
1579 /*
1580 * Returns the allocation status of the specified sectors.
1581 * Drivers not implementing the functionality are assumed to not support
1582 * backing files, hence all their sectors are reported as allocated.
1583 *
1584 * If 'sector_num' is beyond the end of the disk image the return value is 0
1585 * and 'pnum' is set to 0.
1586 *
1587 * 'pnum' is set to the number of sectors (including and immediately following
1588 * the specified sector) that are known to be in the same
1589 * allocated/unallocated state.
1590 *
1591 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes
1592 * beyond the end of the disk image it will be clamped.
1593 *
1594 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
1595 * points to the BDS which the sector range is allocated in.
1596 */
1597 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
1598 int64_t sector_num,
1599 int nb_sectors, int *pnum,
1600 BlockDriverState **file)
1601 {
1602 int64_t total_sectors;
1603 int64_t n;
1604 int64_t ret, ret2;
1605
1606 total_sectors = bdrv_nb_sectors(bs);
1607 if (total_sectors < 0) {
1608 return total_sectors;
1609 }
1610
1611 if (sector_num >= total_sectors) {
1612 *pnum = 0;
1613 return 0;
1614 }
1615
1616 n = total_sectors - sector_num;
1617 if (n < nb_sectors) {
1618 nb_sectors = n;
1619 }
1620
1621 if (!bs->drv->bdrv_co_get_block_status) {
1622 *pnum = nb_sectors;
1623 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
1624 if (bs->drv->protocol_name) {
1625 ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
1626 }
1627 return ret;
1628 }
1629
1630 *file = NULL;
1631 ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
1632 file);
1633 if (ret < 0) {
1634 *pnum = 0;
1635 return ret;
1636 }
1637
1638 if (ret & BDRV_BLOCK_RAW) {
1639 assert(ret & BDRV_BLOCK_OFFSET_VALID);
1640 return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
1641 *pnum, pnum, file);
1642 }
1643
1644 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
1645 ret |= BDRV_BLOCK_ALLOCATED;
1646 } else {
1647 if (bdrv_unallocated_blocks_are_zero(bs)) {
1648 ret |= BDRV_BLOCK_ZERO;
1649 } else if (bs->backing) {
1650 BlockDriverState *bs2 = bs->backing->bs;
1651 int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
1652 if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
1653 ret |= BDRV_BLOCK_ZERO;
1654 }
1655 }
1656 }
1657
1658 if (*file && *file != bs &&
1659 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
1660 (ret & BDRV_BLOCK_OFFSET_VALID)) {
1661 BlockDriverState *file2;
1662 int file_pnum;
1663
1664 ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
1665 *pnum, &file_pnum, &file2);
1666 if (ret2 >= 0) {
1667 /* Ignore errors. This is just providing extra information, it
1668 * is useful but not necessary.
1669 */
1670 if (!file_pnum) {
1671 /* !file_pnum indicates an offset at or beyond the EOF; it is
1672 * perfectly valid for the format block driver to point to such
1673 * offsets, so catch it and mark everything as zero */
1674 ret |= BDRV_BLOCK_ZERO;
1675 } else {
1676 /* Limit request to the range reported by the protocol driver */
1677 *pnum = file_pnum;
1678 ret |= (ret2 & BDRV_BLOCK_ZERO);
1679 }
1680 }
1681 }
1682
1683 return ret;
1684 }
1685
1686 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
1687 BlockDriverState *base,
1688 int64_t sector_num,
1689 int nb_sectors,
1690 int *pnum,
1691 BlockDriverState **file)
1692 {
1693 BlockDriverState *p;
1694 int64_t ret = 0;
1695
1696 assert(bs != base);
1697 for (p = bs; p != base; p = backing_bs(p)) {
1698 ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
1699 if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
1700 break;
1701 }
1702 /* [sector_num, pnum] unallocated on this layer, which could be only
1703 * the first part of [sector_num, nb_sectors]. */
1704 nb_sectors = MIN(nb_sectors, *pnum);
1705 }
1706 return ret;
1707 }
1708
1709 /* Coroutine wrapper for bdrv_get_block_status_above() */
1710 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
1711 {
1712 BdrvCoGetBlockStatusData *data = opaque;
1713
1714 data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
1715 data->sector_num,
1716 data->nb_sectors,
1717 data->pnum,
1718 data->file);
1719 data->done = true;
1720 }
1721
1722 /*
1723 * Synchronous wrapper around bdrv_co_get_block_status_above().
1724 *
1725 * See bdrv_co_get_block_status_above() for details.
1726 */
1727 int64_t bdrv_get_block_status_above(BlockDriverState *bs,
1728 BlockDriverState *base,
1729 int64_t sector_num,
1730 int nb_sectors, int *pnum,
1731 BlockDriverState **file)
1732 {
1733 Coroutine *co;
1734 BdrvCoGetBlockStatusData data = {
1735 .bs = bs,
1736 .base = base,
1737 .file = file,
1738 .sector_num = sector_num,
1739 .nb_sectors = nb_sectors,
1740 .pnum = pnum,
1741 .done = false,
1742 };
1743
1744 if (qemu_in_coroutine()) {
1745 /* Fast-path if already in coroutine context */
1746 bdrv_get_block_status_above_co_entry(&data);
1747 } else {
1748 AioContext *aio_context = bdrv_get_aio_context(bs);
1749
1750 co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry);
1751 qemu_coroutine_enter(co, &data);
1752 while (!data.done) {
1753 aio_poll(aio_context, true);
1754 }
1755 }
1756 return data.ret;
1757 }
1758
1759 int64_t bdrv_get_block_status(BlockDriverState *bs,
1760 int64_t sector_num,
1761 int nb_sectors, int *pnum,
1762 BlockDriverState **file)
1763 {
1764 return bdrv_get_block_status_above(bs, backing_bs(bs),
1765 sector_num, nb_sectors, pnum, file);
1766 }
1767
1768 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
1769 int nb_sectors, int *pnum)
1770 {
1771 BlockDriverState *file;
1772 int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
1773 &file);
1774 if (ret < 0) {
1775 return ret;
1776 }
1777 return !!(ret & BDRV_BLOCK_ALLOCATED);
1778 }
1779
1780 /*
1781 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
1782 *
1783 * Return true if the given sector is allocated in any image between
1784 * BASE and TOP (inclusive). BASE can be NULL to check if the given
1785 * sector is allocated in any image of the chain. Return false otherwise.
1786 *
1787 * 'pnum' is set to the number of sectors (including and immediately following
1788 * the specified sector) that are known to be in the same
1789 * allocated/unallocated state.
1790 *
1791 */
1792 int bdrv_is_allocated_above(BlockDriverState *top,
1793 BlockDriverState *base,
1794 int64_t sector_num,
1795 int nb_sectors, int *pnum)
1796 {
1797 BlockDriverState *intermediate;
1798 int ret, n = nb_sectors;
1799
1800 intermediate = top;
1801 while (intermediate && intermediate != base) {
1802 int pnum_inter;
1803 ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
1804 &pnum_inter);
1805 if (ret < 0) {
1806 return ret;
1807 } else if (ret) {
1808 *pnum = pnum_inter;
1809 return 1;
1810 }
1811
1812 /*
1813 * [sector_num, nb_sectors] is unallocated on top but intermediate
1814 * might have
1815 *
1816 * [sector_num+x, nr_sectors] allocated.
1817 */
1818 if (n > pnum_inter &&
1819 (intermediate == top ||
1820 sector_num + pnum_inter < intermediate->total_sectors)) {
1821 n = pnum_inter;
1822 }
1823
1824 intermediate = backing_bs(intermediate);
1825 }
1826
1827 *pnum = n;
1828 return 0;
1829 }
1830
1831 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
1832 const uint8_t *buf, int nb_sectors)
1833 {
1834 BlockDriver *drv = bs->drv;
1835 int ret;
1836
1837 if (!drv) {
1838 return -ENOMEDIUM;
1839 }
1840 if (!drv->bdrv_write_compressed) {
1841 return -ENOTSUP;
1842 }
1843 ret = bdrv_check_request(bs, sector_num, nb_sectors);
1844 if (ret < 0) {
1845 return ret;
1846 }
1847
1848 assert(QLIST_EMPTY(&bs->dirty_bitmaps));
1849
1850 return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
1851 }
1852
1853 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
1854 int64_t pos, int size)
1855 {
1856 QEMUIOVector qiov;
1857 struct iovec iov = {
1858 .iov_base = (void *) buf,
1859 .iov_len = size,
1860 };
1861
1862 qemu_iovec_init_external(&qiov, &iov, 1);
1863 return bdrv_writev_vmstate(bs, &qiov, pos);
1864 }
1865
1866 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
1867 {
1868 BlockDriver *drv = bs->drv;
1869
1870 if (!drv) {
1871 return -ENOMEDIUM;
1872 } else if (drv->bdrv_save_vmstate) {
1873 return drv->bdrv_save_vmstate(bs, qiov, pos);
1874 } else if (bs->file) {
1875 return bdrv_writev_vmstate(bs->file->bs, qiov, pos);
1876 }
1877
1878 return -ENOTSUP;
1879 }
1880
1881 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
1882 int64_t pos, int size)
1883 {
1884 BlockDriver *drv = bs->drv;
1885 if (!drv)
1886 return -ENOMEDIUM;
1887 if (drv->bdrv_load_vmstate)
1888 return drv->bdrv_load_vmstate(bs, buf, pos, size);
1889 if (bs->file)
1890 return bdrv_load_vmstate(bs->file->bs, buf, pos, size);
1891 return -ENOTSUP;
1892 }
1893
1894 /**************************************************************/
1895 /* async I/Os */
1896
1897 BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
1898 QEMUIOVector *qiov, int nb_sectors,
1899 BlockCompletionFunc *cb, void *opaque)
1900 {
1901 trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
1902
1903 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1904 cb, opaque, false);
1905 }
1906
1907 BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
1908 QEMUIOVector *qiov, int nb_sectors,
1909 BlockCompletionFunc *cb, void *opaque)
1910 {
1911 trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
1912
1913 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1914 cb, opaque, true);
1915 }
1916
1917 BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs,
1918 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags,
1919 BlockCompletionFunc *cb, void *opaque)
1920 {
1921 trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque);
1922
1923 return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors,
1924 BDRV_REQ_ZERO_WRITE | flags,
1925 cb, opaque, true);
1926 }
1927
1928
1929 typedef struct MultiwriteCB {
1930 int error;
1931 int num_requests;
1932 int num_callbacks;
1933 struct {
1934 BlockCompletionFunc *cb;
1935 void *opaque;
1936 QEMUIOVector *free_qiov;
1937 } callbacks[];
1938 } MultiwriteCB;
1939
1940 static void multiwrite_user_cb(MultiwriteCB *mcb)
1941 {
1942 int i;
1943
1944 for (i = 0; i < mcb->num_callbacks; i++) {
1945 mcb->callbacks[i].cb(mcb->callbacks[i].opaque, mcb->error);
1946 if (mcb->callbacks[i].free_qiov) {
1947 qemu_iovec_destroy(mcb->callbacks[i].free_qiov);
1948 }
1949 g_free(mcb->callbacks[i].free_qiov);
1950 }
1951 }
1952
1953 static void multiwrite_cb(void *opaque, int ret)
1954 {
1955 MultiwriteCB *mcb = opaque;
1956
1957 trace_multiwrite_cb(mcb, ret);
1958
1959 if (ret < 0 && !mcb->error) {
1960 mcb->error = ret;
1961 }
1962
1963 mcb->num_requests--;
1964 if (mcb->num_requests == 0) {
1965 multiwrite_user_cb(mcb);
1966 g_free(mcb);
1967 }
1968 }
1969
1970 static int multiwrite_req_compare(const void *a, const void *b)
1971 {
1972 const BlockRequest *req1 = a, *req2 = b;
1973
1974 /*
1975 * Note that we can't simply subtract req2->sector from req1->sector
1976 * here as that could overflow the return value.
1977 */
1978 if (req1->sector > req2->sector) {
1979 return 1;
1980 } else if (req1->sector < req2->sector) {
1981 return -1;
1982 } else {
1983 return 0;
1984 }
1985 }
1986
1987 /*
1988 * Takes a bunch of requests and tries to merge them. Returns the number of
1989 * requests that remain after merging.
1990 */
1991 static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
1992 int num_reqs, MultiwriteCB *mcb)
1993 {
1994 int i, outidx;
1995
1996 // Sort requests by start sector
1997 qsort(reqs, num_reqs, sizeof(*reqs), &multiwrite_req_compare);
1998
1999 // Check if adjacent requests touch the same clusters. If so, combine them,
2000 // filling up gaps with zero sectors.
2001 outidx = 0;
2002 for (i = 1; i < num_reqs; i++) {
2003 int merge = 0;
2004 int64_t oldreq_last = reqs[outidx].sector + reqs[outidx].nb_sectors;
2005
2006 // Handle exactly sequential writes and overlapping writes.
2007 if (reqs[i].sector <= oldreq_last) {
2008 merge = 1;
2009 }
2010
2011 if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 >
2012 bs->bl.max_iov) {
2013 merge = 0;
2014 }
2015
2016 if (bs->bl.max_transfer_length && reqs[outidx].nb_sectors +
2017 reqs[i].nb_sectors > bs->bl.max_transfer_length) {
2018 merge = 0;
2019 }
2020
2021 if (merge) {
2022 size_t size;
2023 QEMUIOVector *qiov = g_malloc0(sizeof(*qiov));
2024 qemu_iovec_init(qiov,
2025 reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1);
2026
2027 // Add the first request to the merged one. If the requests are
2028 // overlapping, drop the last sectors of the first request.
2029 size = (reqs[i].sector - reqs[outidx].sector) << 9;
2030 qemu_iovec_concat(qiov, reqs[outidx].qiov, 0, size);
2031
2032 // We should need to add any zeros between the two requests
2033 assert (reqs[i].sector <= oldreq_last);
2034
2035 // Add the second request
2036 qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size);
2037
2038 // Add tail of first request, if necessary
2039 if (qiov->size < reqs[outidx].qiov->size) {
2040 qemu_iovec_concat(qiov, reqs[outidx].qiov, qiov->size,
2041 reqs[outidx].qiov->size - qiov->size);
2042 }
2043
2044 reqs[outidx].nb_sectors = qiov->size >> 9;
2045 reqs[outidx].qiov = qiov;
2046
2047 mcb->callbacks[i].free_qiov = reqs[outidx].qiov;
2048 } else {
2049 outidx++;
2050 reqs[outidx].sector = reqs[i].sector;
2051 reqs[outidx].nb_sectors = reqs[i].nb_sectors;
2052 reqs[outidx].qiov = reqs[i].qiov;
2053 }
2054 }
2055
2056 if (bs->blk) {
2057 block_acct_merge_done(blk_get_stats(bs->blk), BLOCK_ACCT_WRITE,
2058 num_reqs - outidx - 1);
2059 }
2060
2061 return outidx + 1;
2062 }
2063
2064 /*
2065 * Submit multiple AIO write requests at once.
2066 *
2067 * On success, the function returns 0 and all requests in the reqs array have
2068 * been submitted. In error case this function returns -1, and any of the
2069 * requests may or may not be submitted yet. In particular, this means that the
2070 * callback will be called for some of the requests, for others it won't. The
2071 * caller must check the error field of the BlockRequest to wait for the right
2072 * callbacks (if error != 0, no callback will be called).
2073 *
2074 * The implementation may modify the contents of the reqs array, e.g. to merge
2075 * requests. However, the fields opaque and error are left unmodified as they
2076 * are used to signal failure for a single request to the caller.
2077 */
2078 int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs)
2079 {
2080 MultiwriteCB *mcb;
2081 int i;
2082
2083 /* don't submit writes if we don't have a medium */
2084 if (bs->drv == NULL) {
2085 for (i = 0; i < num_reqs; i++) {
2086 reqs[i].error = -ENOMEDIUM;
2087 }
2088 return -1;
2089 }
2090
2091 if (num_reqs == 0) {
2092 return 0;
2093 }
2094
2095 // Create MultiwriteCB structure
2096 mcb = g_malloc0(sizeof(*mcb) + num_reqs * sizeof(*mcb->callbacks));
2097 mcb->num_requests = 0;
2098 mcb->num_callbacks = num_reqs;
2099
2100 for (i = 0; i < num_reqs; i++) {
2101 mcb->callbacks[i].cb = reqs[i].cb;
2102 mcb->callbacks[i].opaque = reqs[i].opaque;
2103 }
2104
2105 // Check for mergable requests
2106 num_reqs = multiwrite_merge(bs, reqs, num_reqs, mcb);
2107
2108 trace_bdrv_aio_multiwrite(mcb, mcb->num_callbacks, num_reqs);
2109
2110 /* Run the aio requests. */
2111 mcb->num_requests = num_reqs;
2112 for (i = 0; i < num_reqs; i++) {
2113 bdrv_co_aio_rw_vector(bs, reqs[i].sector, reqs[i].qiov,
2114 reqs[i].nb_sectors, reqs[i].flags,
2115 multiwrite_cb, mcb,
2116 true);
2117 }
2118
2119 return 0;
2120 }
2121
2122 void bdrv_aio_cancel(BlockAIOCB *acb)
2123 {
2124 qemu_aio_ref(acb);
2125 bdrv_aio_cancel_async(acb);
2126 while (acb->refcnt > 1) {
2127 if (acb->aiocb_info->get_aio_context) {
2128 aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2129 } else if (acb->bs) {
2130 aio_poll(bdrv_get_aio_context(acb->bs), true);
2131 } else {
2132 abort();
2133 }
2134 }
2135 qemu_aio_unref(acb);
2136 }
2137
2138 /* Async version of aio cancel. The caller is not blocked if the acb implements
2139 * cancel_async, otherwise we do nothing and let the request normally complete.
2140 * In either case the completion callback must be called. */
2141 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2142 {
2143 if (acb->aiocb_info->cancel_async) {
2144 acb->aiocb_info->cancel_async(acb);
2145 }
2146 }
2147
2148 /**************************************************************/
2149 /* async block device emulation */
2150
2151 typedef struct BlockAIOCBCoroutine {
2152 BlockAIOCB common;
2153 BlockRequest req;
2154 bool is_write;
2155 bool need_bh;
2156 bool *done;
2157 QEMUBH* bh;
2158 } BlockAIOCBCoroutine;
2159
2160 static const AIOCBInfo bdrv_em_co_aiocb_info = {
2161 .aiocb_size = sizeof(BlockAIOCBCoroutine),
2162 };
2163
2164 static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
2165 {
2166 if (!acb->need_bh) {
2167 acb->common.cb(acb->common.opaque, acb->req.error);
2168 qemu_aio_unref(acb);
2169 }
2170 }
2171
2172 static void bdrv_co_em_bh(void *opaque)
2173 {
2174 BlockAIOCBCoroutine *acb = opaque;
2175
2176 assert(!acb->need_bh);
2177 qemu_bh_delete(acb->bh);
2178 bdrv_co_complete(acb);
2179 }
2180
2181 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
2182 {
2183 acb->need_bh = false;
2184 if (acb->req.error != -EINPROGRESS) {
2185 BlockDriverState *bs = acb->common.bs;
2186
2187 acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
2188 qemu_bh_schedule(acb->bh);
2189 }
2190 }
2191
2192 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
2193 static void coroutine_fn bdrv_co_do_rw(void *opaque)
2194 {
2195 BlockAIOCBCoroutine *acb = opaque;
2196 BlockDriverState *bs = acb->common.bs;
2197
2198 if (!acb->is_write) {
2199 acb->req.error = bdrv_co_do_readv(bs, acb->req.sector,
2200 acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
2201 } else {
2202 acb->req.error = bdrv_co_do_writev(bs, acb->req.sector,
2203 acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
2204 }
2205
2206 bdrv_co_complete(acb);
2207 }
2208
2209 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
2210 int64_t sector_num,
2211 QEMUIOVector *qiov,
2212 int nb_sectors,
2213 BdrvRequestFlags flags,
2214 BlockCompletionFunc *cb,
2215 void *opaque,
2216 bool is_write)
2217 {
2218 Coroutine *co;
2219 BlockAIOCBCoroutine *acb;
2220
2221 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2222 acb->need_bh = true;
2223 acb->req.error = -EINPROGRESS;
2224 acb->req.sector = sector_num;
2225 acb->req.nb_sectors = nb_sectors;
2226 acb->req.qiov = qiov;
2227 acb->req.flags = flags;
2228 acb->is_write = is_write;
2229
2230 co = qemu_coroutine_create(bdrv_co_do_rw);
2231 qemu_coroutine_enter(co, acb);
2232
2233 bdrv_co_maybe_schedule_bh(acb);
2234 return &acb->common;
2235 }
2236
2237 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
2238 {
2239 BlockAIOCBCoroutine *acb = opaque;
2240 BlockDriverState *bs = acb->common.bs;
2241
2242 acb->req.error = bdrv_co_flush(bs);
2243 bdrv_co_complete(acb);
2244 }
2245
2246 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
2247 BlockCompletionFunc *cb, void *opaque)
2248 {
2249 trace_bdrv_aio_flush(bs, opaque);
2250
2251 Coroutine *co;
2252 BlockAIOCBCoroutine *acb;
2253
2254 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2255 acb->need_bh = true;
2256 acb->req.error = -EINPROGRESS;
2257
2258 co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
2259 qemu_coroutine_enter(co, acb);
2260
2261 bdrv_co_maybe_schedule_bh(acb);
2262 return &acb->common;
2263 }
2264
2265 static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
2266 {
2267 BlockAIOCBCoroutine *acb = opaque;
2268 BlockDriverState *bs = acb->common.bs;
2269
2270 acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
2271 bdrv_co_complete(acb);
2272 }
2273
2274 BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
2275 int64_t sector_num, int nb_sectors,
2276 BlockCompletionFunc *cb, void *opaque)
2277 {
2278 Coroutine *co;
2279 BlockAIOCBCoroutine *acb;
2280
2281 trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
2282
2283 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2284 acb->need_bh = true;
2285 acb->req.error = -EINPROGRESS;
2286 acb->req.sector = sector_num;
2287 acb->req.nb_sectors = nb_sectors;
2288 co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
2289 qemu_coroutine_enter(co, acb);
2290
2291 bdrv_co_maybe_schedule_bh(acb);
2292 return &acb->common;
2293 }
2294
2295 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
2296 BlockCompletionFunc *cb, void *opaque)
2297 {
2298 BlockAIOCB *acb;
2299
2300 acb = g_malloc(aiocb_info->aiocb_size);
2301 acb->aiocb_info = aiocb_info;
2302 acb->bs = bs;
2303 acb->cb = cb;
2304 acb->opaque = opaque;
2305 acb->refcnt = 1;
2306 return acb;
2307 }
2308
2309 void qemu_aio_ref(void *p)
2310 {
2311 BlockAIOCB *acb = p;
2312 acb->refcnt++;
2313 }
2314
2315 void qemu_aio_unref(void *p)
2316 {
2317 BlockAIOCB *acb = p;
2318 assert(acb->refcnt > 0);
2319 if (--acb->refcnt == 0) {
2320 g_free(acb);
2321 }
2322 }
2323
2324 /**************************************************************/
2325 /* Coroutine block device emulation */
2326
2327 static void coroutine_fn bdrv_flush_co_entry(void *opaque)
2328 {
2329 RwCo *rwco = opaque;
2330
2331 rwco->ret = bdrv_co_flush(rwco->bs);
2332 }
2333
2334 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2335 {
2336 int ret;
2337 BdrvTrackedRequest req;
2338
2339 if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2340 bdrv_is_sg(bs)) {
2341 return 0;
2342 }
2343
2344 tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
2345
2346 /* Write back all layers by calling one driver function */
2347 if (bs->drv->bdrv_co_flush) {
2348 ret = bs->drv->bdrv_co_flush(bs);
2349 goto out;
2350 }
2351
2352 /* Write back cached data to the OS even with cache=unsafe */
2353 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
2354 if (bs->drv->bdrv_co_flush_to_os) {
2355 ret = bs->drv->bdrv_co_flush_to_os(bs);
2356 if (ret < 0) {
2357 goto out;
2358 }
2359 }
2360
2361 /* But don't actually force it to the disk with cache=unsafe */
2362 if (bs->open_flags & BDRV_O_NO_FLUSH) {
2363 goto flush_parent;
2364 }
2365
2366 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
2367 if (bs->drv->bdrv_co_flush_to_disk) {
2368 ret = bs->drv->bdrv_co_flush_to_disk(bs);
2369 } else if (bs->drv->bdrv_aio_flush) {
2370 BlockAIOCB *acb;
2371 CoroutineIOCompletion co = {
2372 .coroutine = qemu_coroutine_self(),
2373 };
2374
2375 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2376 if (acb == NULL) {
2377 ret = -EIO;
2378 } else {
2379 qemu_coroutine_yield();
2380 ret = co.ret;
2381 }
2382 } else {
2383 /*
2384 * Some block drivers always operate in either writethrough or unsafe
2385 * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2386 * know how the server works (because the behaviour is hardcoded or
2387 * depends on server-side configuration), so we can't ensure that
2388 * everything is safe on disk. Returning an error doesn't work because
2389 * that would break guests even if the server operates in writethrough
2390 * mode.
2391 *
2392 * Let's hope the user knows what he's doing.
2393 */
2394 ret = 0;
2395 }
2396 if (ret < 0) {
2397 goto out;
2398 }
2399
2400 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH
2401 * in the case of cache=unsafe, so there are no useless flushes.
2402 */
2403 flush_parent:
2404 ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2405 out:
2406 tracked_request_end(&req);
2407 return ret;
2408 }
2409
2410 int bdrv_flush(BlockDriverState *bs)
2411 {
2412 Coroutine *co;
2413 RwCo rwco = {
2414 .bs = bs,
2415 .ret = NOT_DONE,
2416 };
2417
2418 if (qemu_in_coroutine()) {
2419 /* Fast-path if already in coroutine context */
2420 bdrv_flush_co_entry(&rwco);
2421 } else {
2422 AioContext *aio_context = bdrv_get_aio_context(bs);
2423
2424 co = qemu_coroutine_create(bdrv_flush_co_entry);
2425 qemu_coroutine_enter(co, &rwco);
2426 while (rwco.ret == NOT_DONE) {
2427 aio_poll(aio_context, true);
2428 }
2429 }
2430
2431 return rwco.ret;
2432 }
2433
2434 typedef struct DiscardCo {
2435 BlockDriverState *bs;
2436 int64_t sector_num;
2437 int nb_sectors;
2438 int ret;
2439 } DiscardCo;
2440 static void coroutine_fn bdrv_discard_co_entry(void *opaque)
2441 {
2442 DiscardCo *rwco = opaque;
2443
2444 rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors);
2445 }
2446
2447 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
2448 int nb_sectors)
2449 {
2450 BdrvTrackedRequest req;
2451 int max_discard, ret;
2452
2453 if (!bs->drv) {
2454 return -ENOMEDIUM;
2455 }
2456
2457 ret = bdrv_check_request(bs, sector_num, nb_sectors);
2458 if (ret < 0) {
2459 return ret;
2460 } else if (bs->read_only) {
2461 return -EPERM;
2462 }
2463 assert(!(bs->open_flags & BDRV_O_INACTIVE));
2464
2465 /* Do nothing if disabled. */
2466 if (!(bs->open_flags & BDRV_O_UNMAP)) {
2467 return 0;
2468 }
2469
2470 if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) {
2471 return 0;
2472 }
2473
2474 tracked_request_begin(&req, bs, sector_num, nb_sectors,
2475 BDRV_TRACKED_DISCARD);
2476 bdrv_set_dirty(bs, sector_num, nb_sectors);
2477
2478 max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS);
2479 while (nb_sectors > 0) {
2480 int ret;
2481 int num = nb_sectors;
2482
2483 /* align request */
2484 if (bs->bl.discard_alignment &&
2485 num >= bs->bl.discard_alignment &&
2486 sector_num % bs->bl.discard_alignment) {
2487 if (num > bs->bl.discard_alignment) {
2488 num = bs->bl.discard_alignment;
2489 }
2490 num -= sector_num % bs->bl.discard_alignment;
2491 }
2492
2493 /* limit request size */
2494 if (num > max_discard) {
2495 num = max_discard;
2496 }
2497
2498 if (bs->drv->bdrv_co_discard) {
2499 ret = bs->drv->bdrv_co_discard(bs, sector_num, num);
2500 } else {
2501 BlockAIOCB *acb;
2502 CoroutineIOCompletion co = {
2503 .coroutine = qemu_coroutine_self(),
2504 };
2505
2506 acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
2507 bdrv_co_io_em_complete, &co);
2508 if (acb == NULL) {
2509 ret = -EIO;
2510 goto out;
2511 } else {
2512 qemu_coroutine_yield();
2513 ret = co.ret;
2514 }
2515 }
2516 if (ret && ret != -ENOTSUP) {
2517 goto out;
2518 }
2519
2520 sector_num += num;
2521 nb_sectors -= num;
2522 }
2523 ret = 0;
2524 out:
2525 tracked_request_end(&req);
2526 return ret;
2527 }
2528
2529 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
2530 {
2531 Coroutine *co;
2532 DiscardCo rwco = {
2533 .bs = bs,
2534 .sector_num = sector_num,
2535 .nb_sectors = nb_sectors,
2536 .ret = NOT_DONE,
2537 };
2538
2539 if (qemu_in_coroutine()) {
2540 /* Fast-path if already in coroutine context */
2541 bdrv_discard_co_entry(&rwco);
2542 } else {
2543 AioContext *aio_context = bdrv_get_aio_context(bs);
2544
2545 co = qemu_coroutine_create(bdrv_discard_co_entry);
2546 qemu_coroutine_enter(co, &rwco);
2547 while (rwco.ret == NOT_DONE) {
2548 aio_poll(aio_context, true);
2549 }
2550 }
2551
2552 return rwco.ret;
2553 }
2554
2555 typedef struct {
2556 CoroutineIOCompletion *co;
2557 QEMUBH *bh;
2558 } BdrvIoctlCompletionData;
2559
2560 static void bdrv_ioctl_bh_cb(void *opaque)
2561 {
2562 BdrvIoctlCompletionData *data = opaque;
2563
2564 bdrv_co_io_em_complete(data->co, -ENOTSUP);
2565 qemu_bh_delete(data->bh);
2566 }
2567
2568 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
2569 {
2570 BlockDriver *drv = bs->drv;
2571 BdrvTrackedRequest tracked_req;
2572 CoroutineIOCompletion co = {
2573 .coroutine = qemu_coroutine_self(),
2574 };
2575 BlockAIOCB *acb;
2576
2577 tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
2578 if (!drv || !drv->bdrv_aio_ioctl) {
2579 co.ret = -ENOTSUP;
2580 goto out;
2581 }
2582
2583 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
2584 if (!acb) {
2585 BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1);
2586 data->bh = aio_bh_new(bdrv_get_aio_context(bs),
2587 bdrv_ioctl_bh_cb, data);
2588 data->co = &co;
2589 qemu_bh_schedule(data->bh);
2590 }
2591 qemu_coroutine_yield();
2592 out:
2593 tracked_request_end(&tracked_req);
2594 return co.ret;
2595 }
2596
2597 typedef struct {
2598 BlockDriverState *bs;
2599 int req;
2600 void *buf;
2601 int ret;
2602 } BdrvIoctlCoData;
2603
2604 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
2605 {
2606 BdrvIoctlCoData *data = opaque;
2607 data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
2608 }
2609
2610 /* needed for generic scsi interface */
2611 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
2612 {
2613 BdrvIoctlCoData data = {
2614 .bs = bs,
2615 .req = req,
2616 .buf = buf,
2617 .ret = -EINPROGRESS,
2618 };
2619
2620 if (qemu_in_coroutine()) {
2621 /* Fast-path if already in coroutine context */
2622 bdrv_co_ioctl_entry(&data);
2623 } else {
2624 Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry);
2625
2626 qemu_coroutine_enter(co, &data);
2627 while (data.ret == -EINPROGRESS) {
2628 aio_poll(bdrv_get_aio_context(bs), true);
2629 }
2630 }
2631 return data.ret;
2632 }
2633
2634 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
2635 {
2636 BlockAIOCBCoroutine *acb = opaque;
2637 acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
2638 acb->req.req, acb->req.buf);
2639 bdrv_co_complete(acb);
2640 }
2641
2642 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
2643 unsigned long int req, void *buf,
2644 BlockCompletionFunc *cb, void *opaque)
2645 {
2646 BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
2647 bs, cb, opaque);
2648 Coroutine *co;
2649
2650 acb->need_bh = true;
2651 acb->req.error = -EINPROGRESS;
2652 acb->req.req = req;
2653 acb->req.buf = buf;
2654 co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry);
2655 qemu_coroutine_enter(co, acb);
2656
2657 bdrv_co_maybe_schedule_bh(acb);
2658 return &acb->common;
2659 }
2660
2661 void *qemu_blockalign(BlockDriverState *bs, size_t size)
2662 {
2663 return qemu_memalign(bdrv_opt_mem_align(bs), size);
2664 }
2665
2666 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
2667 {
2668 return memset(qemu_blockalign(bs, size), 0, size);
2669 }
2670
2671 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
2672 {
2673 size_t align = bdrv_opt_mem_align(bs);
2674
2675 /* Ensure that NULL is never returned on success */
2676 assert(align > 0);
2677 if (size == 0) {
2678 size = align;
2679 }
2680
2681 return qemu_try_memalign(align, size);
2682 }
2683
2684 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
2685 {
2686 void *mem = qemu_try_blockalign(bs, size);
2687
2688 if (mem) {
2689 memset(mem, 0, size);
2690 }
2691
2692 return mem;
2693 }
2694
2695 /*
2696 * Check if all memory in this vector is sector aligned.
2697 */
2698 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
2699 {
2700 int i;
2701 size_t alignment = bdrv_min_mem_align(bs);
2702
2703 for (i = 0; i < qiov->niov; i++) {
2704 if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
2705 return false;
2706 }
2707 if (qiov->iov[i].iov_len % alignment) {
2708 return false;
2709 }
2710 }
2711
2712 return true;
2713 }
2714
2715 void bdrv_add_before_write_notifier(BlockDriverState *bs,
2716 NotifierWithReturn *notifier)
2717 {
2718 notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
2719 }
2720
2721 void bdrv_io_plug(BlockDriverState *bs)
2722 {
2723 BdrvChild *child;
2724
2725 QLIST_FOREACH(child, &bs->children, next) {
2726 bdrv_io_plug(child->bs);
2727 }
2728
2729 if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) {
2730 BlockDriver *drv = bs->drv;
2731 if (drv && drv->bdrv_io_plug) {
2732 drv->bdrv_io_plug(bs);
2733 }
2734 }
2735 }
2736
2737 void bdrv_io_unplug(BlockDriverState *bs)
2738 {
2739 BdrvChild *child;
2740
2741 assert(bs->io_plugged);
2742 if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) {
2743 BlockDriver *drv = bs->drv;
2744 if (drv && drv->bdrv_io_unplug) {
2745 drv->bdrv_io_unplug(bs);
2746 }
2747 }
2748
2749 QLIST_FOREACH(child, &bs->children, next) {
2750 bdrv_io_unplug(child->bs);
2751 }
2752 }
2753
2754 void bdrv_io_unplugged_begin(BlockDriverState *bs)
2755 {
2756 BdrvChild *child;
2757
2758 if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) {
2759 BlockDriver *drv = bs->drv;
2760 if (drv && drv->bdrv_io_unplug) {
2761 drv->bdrv_io_unplug(bs);
2762 }
2763 }
2764
2765 QLIST_FOREACH(child, &bs->children, next) {
2766 bdrv_io_unplugged_begin(child->bs);
2767 }
2768 }
2769
2770 void bdrv_io_unplugged_end(BlockDriverState *bs)
2771 {
2772 BdrvChild *child;
2773
2774 assert(bs->io_plug_disabled);
2775 QLIST_FOREACH(child, &bs->children, next) {
2776 bdrv_io_unplugged_end(child->bs);
2777 }
2778
2779 if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) {
2780 BlockDriver *drv = bs->drv;
2781 if (drv && drv->bdrv_io_plug) {
2782 drv->bdrv_io_plug(bs);
2783 }
2784 }
2785 }
2786
2787 void bdrv_drained_begin(BlockDriverState *bs)
2788 {
2789 if (!bs->quiesce_counter++) {
2790 aio_disable_external(bdrv_get_aio_context(bs));
2791 }
2792 bdrv_drain(bs);
2793 }
2794
2795 void bdrv_drained_end(BlockDriverState *bs)
2796 {
2797 assert(bs->quiesce_counter > 0);
2798 if (--bs->quiesce_counter > 0) {
2799 return;
2800 }
2801 aio_enable_external(bdrv_get_aio_context(bs));
2802 }