backup: Use BlockBackend for I/O
[qemu.git] / block / io.c
1 /*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/blockjob.h"
29 #include "block/block_int.h"
30 #include "qemu/cutils.h"
31 #include "qapi/error.h"
32 #include "qemu/error-report.h"
33
34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
35
36 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
37 int64_t sector_num,
38 QEMUIOVector *qiov,
39 int nb_sectors,
40 BdrvRequestFlags flags,
41 BlockCompletionFunc *cb,
42 void *opaque,
43 bool is_write);
44 static void coroutine_fn bdrv_co_do_rw(void *opaque);
45 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
46 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags);
47
48 static void bdrv_parent_drained_begin(BlockDriverState *bs)
49 {
50 BdrvChild *c;
51
52 QLIST_FOREACH(c, &bs->parents, next_parent) {
53 if (c->role->drained_begin) {
54 c->role->drained_begin(c);
55 }
56 }
57 }
58
59 static void bdrv_parent_drained_end(BlockDriverState *bs)
60 {
61 BdrvChild *c;
62
63 QLIST_FOREACH(c, &bs->parents, next_parent) {
64 if (c->role->drained_end) {
65 c->role->drained_end(c);
66 }
67 }
68 }
69
70 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
71 {
72 BlockDriver *drv = bs->drv;
73 Error *local_err = NULL;
74
75 memset(&bs->bl, 0, sizeof(bs->bl));
76
77 if (!drv) {
78 return;
79 }
80
81 /* Take some limits from the children as a default */
82 if (bs->file) {
83 bdrv_refresh_limits(bs->file->bs, &local_err);
84 if (local_err) {
85 error_propagate(errp, local_err);
86 return;
87 }
88 bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length;
89 bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length;
90 bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment;
91 bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment;
92 bs->bl.max_iov = bs->file->bs->bl.max_iov;
93 } else {
94 bs->bl.min_mem_alignment = 512;
95 bs->bl.opt_mem_alignment = getpagesize();
96
97 /* Safe default since most protocols use readv()/writev()/etc */
98 bs->bl.max_iov = IOV_MAX;
99 }
100
101 if (bs->backing) {
102 bdrv_refresh_limits(bs->backing->bs, &local_err);
103 if (local_err) {
104 error_propagate(errp, local_err);
105 return;
106 }
107 bs->bl.opt_transfer_length =
108 MAX(bs->bl.opt_transfer_length,
109 bs->backing->bs->bl.opt_transfer_length);
110 bs->bl.max_transfer_length =
111 MIN_NON_ZERO(bs->bl.max_transfer_length,
112 bs->backing->bs->bl.max_transfer_length);
113 bs->bl.opt_mem_alignment =
114 MAX(bs->bl.opt_mem_alignment,
115 bs->backing->bs->bl.opt_mem_alignment);
116 bs->bl.min_mem_alignment =
117 MAX(bs->bl.min_mem_alignment,
118 bs->backing->bs->bl.min_mem_alignment);
119 bs->bl.max_iov =
120 MIN(bs->bl.max_iov,
121 bs->backing->bs->bl.max_iov);
122 }
123
124 /* Then let the driver override it */
125 if (drv->bdrv_refresh_limits) {
126 drv->bdrv_refresh_limits(bs, errp);
127 }
128 }
129
130 /**
131 * The copy-on-read flag is actually a reference count so multiple users may
132 * use the feature without worrying about clobbering its previous state.
133 * Copy-on-read stays enabled until all users have called to disable it.
134 */
135 void bdrv_enable_copy_on_read(BlockDriverState *bs)
136 {
137 bs->copy_on_read++;
138 }
139
140 void bdrv_disable_copy_on_read(BlockDriverState *bs)
141 {
142 assert(bs->copy_on_read > 0);
143 bs->copy_on_read--;
144 }
145
146 /* Check if any requests are in-flight (including throttled requests) */
147 bool bdrv_requests_pending(BlockDriverState *bs)
148 {
149 BdrvChild *child;
150
151 if (!QLIST_EMPTY(&bs->tracked_requests)) {
152 return true;
153 }
154
155 QLIST_FOREACH(child, &bs->children, next) {
156 if (bdrv_requests_pending(child->bs)) {
157 return true;
158 }
159 }
160
161 return false;
162 }
163
164 static void bdrv_drain_recurse(BlockDriverState *bs)
165 {
166 BdrvChild *child;
167
168 if (bs->drv && bs->drv->bdrv_drain) {
169 bs->drv->bdrv_drain(bs);
170 }
171 QLIST_FOREACH(child, &bs->children, next) {
172 bdrv_drain_recurse(child->bs);
173 }
174 }
175
176 typedef struct {
177 Coroutine *co;
178 BlockDriverState *bs;
179 QEMUBH *bh;
180 bool done;
181 } BdrvCoDrainData;
182
183 static void bdrv_drain_poll(BlockDriverState *bs)
184 {
185 bool busy = true;
186
187 while (busy) {
188 /* Keep iterating */
189 busy = bdrv_requests_pending(bs);
190 busy |= aio_poll(bdrv_get_aio_context(bs), busy);
191 }
192 }
193
194 static void bdrv_co_drain_bh_cb(void *opaque)
195 {
196 BdrvCoDrainData *data = opaque;
197 Coroutine *co = data->co;
198
199 qemu_bh_delete(data->bh);
200 bdrv_drain_poll(data->bs);
201 data->done = true;
202 qemu_coroutine_enter(co, NULL);
203 }
204
205 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
206 {
207 BdrvCoDrainData data;
208
209 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
210 * other coroutines run if they were queued from
211 * qemu_co_queue_run_restart(). */
212
213 assert(qemu_in_coroutine());
214 data = (BdrvCoDrainData) {
215 .co = qemu_coroutine_self(),
216 .bs = bs,
217 .done = false,
218 .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data),
219 };
220 qemu_bh_schedule(data.bh);
221
222 qemu_coroutine_yield();
223 /* If we are resumed from some other event (such as an aio completion or a
224 * timer callback), it is a bug in the caller that should be fixed. */
225 assert(data.done);
226 }
227
228 void bdrv_drained_begin(BlockDriverState *bs)
229 {
230 if (!bs->quiesce_counter++) {
231 aio_disable_external(bdrv_get_aio_context(bs));
232 bdrv_parent_drained_begin(bs);
233 }
234
235 bdrv_io_unplugged_begin(bs);
236 bdrv_drain_recurse(bs);
237 if (qemu_in_coroutine()) {
238 bdrv_co_yield_to_drain(bs);
239 } else {
240 bdrv_drain_poll(bs);
241 }
242 bdrv_io_unplugged_end(bs);
243 }
244
245 void bdrv_drained_end(BlockDriverState *bs)
246 {
247 assert(bs->quiesce_counter > 0);
248 if (--bs->quiesce_counter > 0) {
249 return;
250 }
251
252 bdrv_parent_drained_end(bs);
253 aio_enable_external(bdrv_get_aio_context(bs));
254 }
255
256 /*
257 * Wait for pending requests to complete on a single BlockDriverState subtree,
258 * and suspend block driver's internal I/O until next request arrives.
259 *
260 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
261 * AioContext.
262 *
263 * Only this BlockDriverState's AioContext is run, so in-flight requests must
264 * not depend on events in other AioContexts. In that case, use
265 * bdrv_drain_all() instead.
266 */
267 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
268 {
269 assert(qemu_in_coroutine());
270 bdrv_drained_begin(bs);
271 bdrv_drained_end(bs);
272 }
273
274 void bdrv_drain(BlockDriverState *bs)
275 {
276 bdrv_drained_begin(bs);
277 bdrv_drained_end(bs);
278 }
279
280 /*
281 * Wait for pending requests to complete across all BlockDriverStates
282 *
283 * This function does not flush data to disk, use bdrv_flush_all() for that
284 * after calling this function.
285 */
286 void bdrv_drain_all(void)
287 {
288 /* Always run first iteration so any pending completion BHs run */
289 bool busy = true;
290 BlockDriverState *bs;
291 BdrvNextIterator it;
292 GSList *aio_ctxs = NULL, *ctx;
293
294 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
295 AioContext *aio_context = bdrv_get_aio_context(bs);
296
297 aio_context_acquire(aio_context);
298 if (bs->job) {
299 block_job_pause(bs->job);
300 }
301 bdrv_parent_drained_begin(bs);
302 bdrv_io_unplugged_begin(bs);
303 bdrv_drain_recurse(bs);
304 aio_context_release(aio_context);
305
306 if (!g_slist_find(aio_ctxs, aio_context)) {
307 aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
308 }
309 }
310
311 /* Note that completion of an asynchronous I/O operation can trigger any
312 * number of other I/O operations on other devices---for example a
313 * coroutine can submit an I/O request to another device in response to
314 * request completion. Therefore we must keep looping until there was no
315 * more activity rather than simply draining each device independently.
316 */
317 while (busy) {
318 busy = false;
319
320 for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
321 AioContext *aio_context = ctx->data;
322
323 aio_context_acquire(aio_context);
324 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
325 if (aio_context == bdrv_get_aio_context(bs)) {
326 if (bdrv_requests_pending(bs)) {
327 busy = true;
328 aio_poll(aio_context, busy);
329 }
330 }
331 }
332 busy |= aio_poll(aio_context, false);
333 aio_context_release(aio_context);
334 }
335 }
336
337 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
338 AioContext *aio_context = bdrv_get_aio_context(bs);
339
340 aio_context_acquire(aio_context);
341 bdrv_io_unplugged_end(bs);
342 bdrv_parent_drained_end(bs);
343 if (bs->job) {
344 block_job_resume(bs->job);
345 }
346 aio_context_release(aio_context);
347 }
348 g_slist_free(aio_ctxs);
349 }
350
351 /**
352 * Remove an active request from the tracked requests list
353 *
354 * This function should be called when a tracked request is completing.
355 */
356 static void tracked_request_end(BdrvTrackedRequest *req)
357 {
358 if (req->serialising) {
359 req->bs->serialising_in_flight--;
360 }
361
362 QLIST_REMOVE(req, list);
363 qemu_co_queue_restart_all(&req->wait_queue);
364 }
365
366 /**
367 * Add an active request to the tracked requests list
368 */
369 static void tracked_request_begin(BdrvTrackedRequest *req,
370 BlockDriverState *bs,
371 int64_t offset,
372 unsigned int bytes,
373 enum BdrvTrackedRequestType type)
374 {
375 *req = (BdrvTrackedRequest){
376 .bs = bs,
377 .offset = offset,
378 .bytes = bytes,
379 .type = type,
380 .co = qemu_coroutine_self(),
381 .serialising = false,
382 .overlap_offset = offset,
383 .overlap_bytes = bytes,
384 };
385
386 qemu_co_queue_init(&req->wait_queue);
387
388 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
389 }
390
391 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
392 {
393 int64_t overlap_offset = req->offset & ~(align - 1);
394 unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
395 - overlap_offset;
396
397 if (!req->serialising) {
398 req->bs->serialising_in_flight++;
399 req->serialising = true;
400 }
401
402 req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
403 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
404 }
405
406 /**
407 * Round a region to cluster boundaries
408 */
409 void bdrv_round_to_clusters(BlockDriverState *bs,
410 int64_t sector_num, int nb_sectors,
411 int64_t *cluster_sector_num,
412 int *cluster_nb_sectors)
413 {
414 BlockDriverInfo bdi;
415
416 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
417 *cluster_sector_num = sector_num;
418 *cluster_nb_sectors = nb_sectors;
419 } else {
420 int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
421 *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
422 *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
423 nb_sectors, c);
424 }
425 }
426
427 static int bdrv_get_cluster_size(BlockDriverState *bs)
428 {
429 BlockDriverInfo bdi;
430 int ret;
431
432 ret = bdrv_get_info(bs, &bdi);
433 if (ret < 0 || bdi.cluster_size == 0) {
434 return bs->request_alignment;
435 } else {
436 return bdi.cluster_size;
437 }
438 }
439
440 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
441 int64_t offset, unsigned int bytes)
442 {
443 /* aaaa bbbb */
444 if (offset >= req->overlap_offset + req->overlap_bytes) {
445 return false;
446 }
447 /* bbbb aaaa */
448 if (req->overlap_offset >= offset + bytes) {
449 return false;
450 }
451 return true;
452 }
453
454 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
455 {
456 BlockDriverState *bs = self->bs;
457 BdrvTrackedRequest *req;
458 bool retry;
459 bool waited = false;
460
461 if (!bs->serialising_in_flight) {
462 return false;
463 }
464
465 do {
466 retry = false;
467 QLIST_FOREACH(req, &bs->tracked_requests, list) {
468 if (req == self || (!req->serialising && !self->serialising)) {
469 continue;
470 }
471 if (tracked_request_overlaps(req, self->overlap_offset,
472 self->overlap_bytes))
473 {
474 /* Hitting this means there was a reentrant request, for
475 * example, a block driver issuing nested requests. This must
476 * never happen since it means deadlock.
477 */
478 assert(qemu_coroutine_self() != req->co);
479
480 /* If the request is already (indirectly) waiting for us, or
481 * will wait for us as soon as it wakes up, then just go on
482 * (instead of producing a deadlock in the former case). */
483 if (!req->waiting_for) {
484 self->waiting_for = req;
485 qemu_co_queue_wait(&req->wait_queue);
486 self->waiting_for = NULL;
487 retry = true;
488 waited = true;
489 break;
490 }
491 }
492 }
493 } while (retry);
494
495 return waited;
496 }
497
498 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
499 size_t size)
500 {
501 if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
502 return -EIO;
503 }
504
505 if (!bdrv_is_inserted(bs)) {
506 return -ENOMEDIUM;
507 }
508
509 if (offset < 0) {
510 return -EIO;
511 }
512
513 return 0;
514 }
515
516 static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
517 int nb_sectors)
518 {
519 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
520 return -EIO;
521 }
522
523 return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
524 nb_sectors * BDRV_SECTOR_SIZE);
525 }
526
527 typedef struct RwCo {
528 BlockDriverState *bs;
529 int64_t offset;
530 QEMUIOVector *qiov;
531 bool is_write;
532 int ret;
533 BdrvRequestFlags flags;
534 } RwCo;
535
536 static void coroutine_fn bdrv_rw_co_entry(void *opaque)
537 {
538 RwCo *rwco = opaque;
539
540 if (!rwco->is_write) {
541 rwco->ret = bdrv_co_preadv(rwco->bs, rwco->offset,
542 rwco->qiov->size, rwco->qiov,
543 rwco->flags);
544 } else {
545 rwco->ret = bdrv_co_pwritev(rwco->bs, rwco->offset,
546 rwco->qiov->size, rwco->qiov,
547 rwco->flags);
548 }
549 }
550
551 /*
552 * Process a vectored synchronous request using coroutines
553 */
554 static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset,
555 QEMUIOVector *qiov, bool is_write,
556 BdrvRequestFlags flags)
557 {
558 Coroutine *co;
559 RwCo rwco = {
560 .bs = bs,
561 .offset = offset,
562 .qiov = qiov,
563 .is_write = is_write,
564 .ret = NOT_DONE,
565 .flags = flags,
566 };
567
568 if (qemu_in_coroutine()) {
569 /* Fast-path if already in coroutine context */
570 bdrv_rw_co_entry(&rwco);
571 } else {
572 AioContext *aio_context = bdrv_get_aio_context(bs);
573
574 co = qemu_coroutine_create(bdrv_rw_co_entry);
575 qemu_coroutine_enter(co, &rwco);
576 while (rwco.ret == NOT_DONE) {
577 aio_poll(aio_context, true);
578 }
579 }
580 return rwco.ret;
581 }
582
583 /*
584 * Process a synchronous request using coroutines
585 */
586 static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf,
587 int nb_sectors, bool is_write, BdrvRequestFlags flags)
588 {
589 QEMUIOVector qiov;
590 struct iovec iov = {
591 .iov_base = (void *)buf,
592 .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
593 };
594
595 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
596 return -EINVAL;
597 }
598
599 qemu_iovec_init_external(&qiov, &iov, 1);
600 return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS,
601 &qiov, is_write, flags);
602 }
603
604 /* return < 0 if error. See bdrv_write() for the return codes */
605 int bdrv_read(BlockDriverState *bs, int64_t sector_num,
606 uint8_t *buf, int nb_sectors)
607 {
608 return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0);
609 }
610
611 /* Return < 0 if error. Important errors are:
612 -EIO generic I/O error (may happen for all errors)
613 -ENOMEDIUM No media inserted.
614 -EINVAL Invalid sector number or nb_sectors
615 -EACCES Trying to write a read-only device
616 */
617 int bdrv_write(BlockDriverState *bs, int64_t sector_num,
618 const uint8_t *buf, int nb_sectors)
619 {
620 return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
621 }
622
623 int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
624 int nb_sectors, BdrvRequestFlags flags)
625 {
626 return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true,
627 BDRV_REQ_ZERO_WRITE | flags);
628 }
629
630 /*
631 * Completely zero out a block device with the help of bdrv_write_zeroes.
632 * The operation is sped up by checking the block status and only writing
633 * zeroes to the device if they currently do not return zeroes. Optional
634 * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
635 * BDRV_REQ_FUA).
636 *
637 * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
638 */
639 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
640 {
641 int64_t target_sectors, ret, nb_sectors, sector_num = 0;
642 BlockDriverState *file;
643 int n;
644
645 target_sectors = bdrv_nb_sectors(bs);
646 if (target_sectors < 0) {
647 return target_sectors;
648 }
649
650 for (;;) {
651 nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
652 if (nb_sectors <= 0) {
653 return 0;
654 }
655 ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
656 if (ret < 0) {
657 error_report("error getting block status at sector %" PRId64 ": %s",
658 sector_num, strerror(-ret));
659 return ret;
660 }
661 if (ret & BDRV_BLOCK_ZERO) {
662 sector_num += n;
663 continue;
664 }
665 ret = bdrv_write_zeroes(bs, sector_num, n, flags);
666 if (ret < 0) {
667 error_report("error writing zeroes at sector %" PRId64 ": %s",
668 sector_num, strerror(-ret));
669 return ret;
670 }
671 sector_num += n;
672 }
673 }
674
675 int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes)
676 {
677 QEMUIOVector qiov;
678 struct iovec iov = {
679 .iov_base = (void *)buf,
680 .iov_len = bytes,
681 };
682 int ret;
683
684 if (bytes < 0) {
685 return -EINVAL;
686 }
687
688 qemu_iovec_init_external(&qiov, &iov, 1);
689 ret = bdrv_prwv_co(bs, offset, &qiov, false, 0);
690 if (ret < 0) {
691 return ret;
692 }
693
694 return bytes;
695 }
696
697 int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov)
698 {
699 int ret;
700
701 ret = bdrv_prwv_co(bs, offset, qiov, true, 0);
702 if (ret < 0) {
703 return ret;
704 }
705
706 return qiov->size;
707 }
708
709 int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
710 const void *buf, int bytes)
711 {
712 QEMUIOVector qiov;
713 struct iovec iov = {
714 .iov_base = (void *) buf,
715 .iov_len = bytes,
716 };
717
718 if (bytes < 0) {
719 return -EINVAL;
720 }
721
722 qemu_iovec_init_external(&qiov, &iov, 1);
723 return bdrv_pwritev(bs, offset, &qiov);
724 }
725
726 /*
727 * Writes to the file and ensures that no writes are reordered across this
728 * request (acts as a barrier)
729 *
730 * Returns 0 on success, -errno in error cases.
731 */
732 int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
733 const void *buf, int count)
734 {
735 int ret;
736
737 ret = bdrv_pwrite(bs, offset, buf, count);
738 if (ret < 0) {
739 return ret;
740 }
741
742 ret = bdrv_flush(bs);
743 if (ret < 0) {
744 return ret;
745 }
746
747 return 0;
748 }
749
750 typedef struct CoroutineIOCompletion {
751 Coroutine *coroutine;
752 int ret;
753 } CoroutineIOCompletion;
754
755 static void bdrv_co_io_em_complete(void *opaque, int ret)
756 {
757 CoroutineIOCompletion *co = opaque;
758
759 co->ret = ret;
760 qemu_coroutine_enter(co->coroutine, NULL);
761 }
762
763 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
764 uint64_t offset, uint64_t bytes,
765 QEMUIOVector *qiov, int flags)
766 {
767 BlockDriver *drv = bs->drv;
768 int64_t sector_num;
769 unsigned int nb_sectors;
770
771 if (drv->bdrv_co_preadv) {
772 return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
773 }
774
775 sector_num = offset >> BDRV_SECTOR_BITS;
776 nb_sectors = bytes >> BDRV_SECTOR_BITS;
777
778 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
779 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
780 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
781
782 if (drv->bdrv_co_readv) {
783 return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
784 } else {
785 BlockAIOCB *acb;
786 CoroutineIOCompletion co = {
787 .coroutine = qemu_coroutine_self(),
788 };
789
790 acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
791 bdrv_co_io_em_complete, &co);
792 if (acb == NULL) {
793 return -EIO;
794 } else {
795 qemu_coroutine_yield();
796 return co.ret;
797 }
798 }
799 }
800
801 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
802 uint64_t offset, uint64_t bytes,
803 QEMUIOVector *qiov, int flags)
804 {
805 BlockDriver *drv = bs->drv;
806 int64_t sector_num;
807 unsigned int nb_sectors;
808 int ret;
809
810 if (drv->bdrv_co_pwritev) {
811 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags);
812 goto emulate_flags;
813 }
814
815 sector_num = offset >> BDRV_SECTOR_BITS;
816 nb_sectors = bytes >> BDRV_SECTOR_BITS;
817
818 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
819 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
820 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
821
822 if (drv->bdrv_co_writev_flags) {
823 ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
824 flags & bs->supported_write_flags);
825 flags &= ~bs->supported_write_flags;
826 } else if (drv->bdrv_co_writev) {
827 assert(!bs->supported_write_flags);
828 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
829 } else {
830 BlockAIOCB *acb;
831 CoroutineIOCompletion co = {
832 .coroutine = qemu_coroutine_self(),
833 };
834
835 acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
836 bdrv_co_io_em_complete, &co);
837 if (acb == NULL) {
838 ret = -EIO;
839 } else {
840 qemu_coroutine_yield();
841 ret = co.ret;
842 }
843 }
844
845 emulate_flags:
846 if (ret == 0 && (flags & BDRV_REQ_FUA)) {
847 ret = bdrv_co_flush(bs);
848 }
849
850 return ret;
851 }
852
853 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
854 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
855 {
856 /* Perform I/O through a temporary buffer so that users who scribble over
857 * their read buffer while the operation is in progress do not end up
858 * modifying the image file. This is critical for zero-copy guest I/O
859 * where anything might happen inside guest memory.
860 */
861 void *bounce_buffer;
862
863 BlockDriver *drv = bs->drv;
864 struct iovec iov;
865 QEMUIOVector bounce_qiov;
866 int64_t cluster_sector_num;
867 int cluster_nb_sectors;
868 size_t skip_bytes;
869 int ret;
870
871 /* Cover entire cluster so no additional backing file I/O is required when
872 * allocating cluster in the image file.
873 */
874 bdrv_round_to_clusters(bs, sector_num, nb_sectors,
875 &cluster_sector_num, &cluster_nb_sectors);
876
877 trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
878 cluster_sector_num, cluster_nb_sectors);
879
880 iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
881 iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
882 if (bounce_buffer == NULL) {
883 ret = -ENOMEM;
884 goto err;
885 }
886
887 qemu_iovec_init_external(&bounce_qiov, &iov, 1);
888
889 ret = bdrv_driver_preadv(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
890 cluster_nb_sectors * BDRV_SECTOR_SIZE,
891 &bounce_qiov, 0);
892 if (ret < 0) {
893 goto err;
894 }
895
896 if (drv->bdrv_co_write_zeroes &&
897 buffer_is_zero(bounce_buffer, iov.iov_len)) {
898 ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num,
899 cluster_nb_sectors, 0);
900 } else {
901 /* This does not change the data on the disk, it is not necessary
902 * to flush even in cache=writethrough mode.
903 */
904 ret = bdrv_driver_pwritev(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
905 cluster_nb_sectors * BDRV_SECTOR_SIZE,
906 &bounce_qiov, 0);
907 }
908
909 if (ret < 0) {
910 /* It might be okay to ignore write errors for guest requests. If this
911 * is a deliberate copy-on-read then we don't want to ignore the error.
912 * Simply report it in all cases.
913 */
914 goto err;
915 }
916
917 skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE;
918 qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes,
919 nb_sectors * BDRV_SECTOR_SIZE);
920
921 err:
922 qemu_vfree(bounce_buffer);
923 return ret;
924 }
925
926 /*
927 * Forwards an already correctly aligned request to the BlockDriver. This
928 * handles copy on read and zeroing after EOF; any other features must be
929 * implemented by the caller.
930 */
931 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
932 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
933 int64_t align, QEMUIOVector *qiov, int flags)
934 {
935 int ret;
936
937 int64_t sector_num = offset >> BDRV_SECTOR_BITS;
938 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
939
940 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
941 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
942 assert(!qiov || bytes == qiov->size);
943 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
944
945 /* Handle Copy on Read and associated serialisation */
946 if (flags & BDRV_REQ_COPY_ON_READ) {
947 /* If we touch the same cluster it counts as an overlap. This
948 * guarantees that allocating writes will be serialized and not race
949 * with each other for the same cluster. For example, in copy-on-read
950 * it ensures that the CoR read and write operations are atomic and
951 * guest writes cannot interleave between them. */
952 mark_request_serialising(req, bdrv_get_cluster_size(bs));
953 }
954
955 if (!(flags & BDRV_REQ_NO_SERIALISING)) {
956 wait_serialising_requests(req);
957 }
958
959 if (flags & BDRV_REQ_COPY_ON_READ) {
960 int pnum;
961
962 ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum);
963 if (ret < 0) {
964 goto out;
965 }
966
967 if (!ret || pnum != nb_sectors) {
968 ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov);
969 goto out;
970 }
971 }
972
973 /* Forward the request to the BlockDriver */
974 if (!bs->zero_beyond_eof) {
975 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
976 } else {
977 /* Read zeros after EOF */
978 int64_t total_sectors, max_nb_sectors;
979
980 total_sectors = bdrv_nb_sectors(bs);
981 if (total_sectors < 0) {
982 ret = total_sectors;
983 goto out;
984 }
985
986 max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num),
987 align >> BDRV_SECTOR_BITS);
988 if (nb_sectors < max_nb_sectors) {
989 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
990 } else if (max_nb_sectors > 0) {
991 QEMUIOVector local_qiov;
992
993 qemu_iovec_init(&local_qiov, qiov->niov);
994 qemu_iovec_concat(&local_qiov, qiov, 0,
995 max_nb_sectors * BDRV_SECTOR_SIZE);
996
997 ret = bdrv_driver_preadv(bs, offset,
998 max_nb_sectors * BDRV_SECTOR_SIZE,
999 &local_qiov, 0);
1000
1001 qemu_iovec_destroy(&local_qiov);
1002 } else {
1003 ret = 0;
1004 }
1005
1006 /* Reading beyond end of file is supposed to produce zeroes */
1007 if (ret == 0 && total_sectors < sector_num + nb_sectors) {
1008 uint64_t offset = MAX(0, total_sectors - sector_num);
1009 uint64_t bytes = (sector_num + nb_sectors - offset) *
1010 BDRV_SECTOR_SIZE;
1011 qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes);
1012 }
1013 }
1014
1015 out:
1016 return ret;
1017 }
1018
1019 /*
1020 * Handle a read request in coroutine context
1021 */
1022 int coroutine_fn bdrv_co_preadv(BlockDriverState *bs,
1023 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1024 BdrvRequestFlags flags)
1025 {
1026 BlockDriver *drv = bs->drv;
1027 BdrvTrackedRequest req;
1028
1029 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1030 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1031 uint8_t *head_buf = NULL;
1032 uint8_t *tail_buf = NULL;
1033 QEMUIOVector local_qiov;
1034 bool use_local_qiov = false;
1035 int ret;
1036
1037 if (!drv) {
1038 return -ENOMEDIUM;
1039 }
1040
1041 ret = bdrv_check_byte_request(bs, offset, bytes);
1042 if (ret < 0) {
1043 return ret;
1044 }
1045
1046 /* Don't do copy-on-read if we read data before write operation */
1047 if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
1048 flags |= BDRV_REQ_COPY_ON_READ;
1049 }
1050
1051 /* Align read if necessary by padding qiov */
1052 if (offset & (align - 1)) {
1053 head_buf = qemu_blockalign(bs, align);
1054 qemu_iovec_init(&local_qiov, qiov->niov + 2);
1055 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1056 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1057 use_local_qiov = true;
1058
1059 bytes += offset & (align - 1);
1060 offset = offset & ~(align - 1);
1061 }
1062
1063 if ((offset + bytes) & (align - 1)) {
1064 if (!use_local_qiov) {
1065 qemu_iovec_init(&local_qiov, qiov->niov + 1);
1066 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1067 use_local_qiov = true;
1068 }
1069 tail_buf = qemu_blockalign(bs, align);
1070 qemu_iovec_add(&local_qiov, tail_buf,
1071 align - ((offset + bytes) & (align - 1)));
1072
1073 bytes = ROUND_UP(bytes, align);
1074 }
1075
1076 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1077 ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
1078 use_local_qiov ? &local_qiov : qiov,
1079 flags);
1080 tracked_request_end(&req);
1081
1082 if (use_local_qiov) {
1083 qemu_iovec_destroy(&local_qiov);
1084 qemu_vfree(head_buf);
1085 qemu_vfree(tail_buf);
1086 }
1087
1088 return ret;
1089 }
1090
1091 static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
1092 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1093 BdrvRequestFlags flags)
1094 {
1095 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1096 return -EINVAL;
1097 }
1098
1099 return bdrv_co_preadv(bs, sector_num << BDRV_SECTOR_BITS,
1100 nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1101 }
1102
1103 int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
1104 int nb_sectors, QEMUIOVector *qiov)
1105 {
1106 trace_bdrv_co_readv(bs, sector_num, nb_sectors);
1107
1108 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
1109 }
1110
1111 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768
1112
1113 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
1114 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
1115 {
1116 BlockDriver *drv = bs->drv;
1117 QEMUIOVector qiov;
1118 struct iovec iov = {0};
1119 int ret = 0;
1120 bool need_flush = false;
1121
1122 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes,
1123 BDRV_REQUEST_MAX_SECTORS);
1124
1125 while (nb_sectors > 0 && !ret) {
1126 int num = nb_sectors;
1127
1128 /* Align request. Block drivers can expect the "bulk" of the request
1129 * to be aligned.
1130 */
1131 if (bs->bl.write_zeroes_alignment
1132 && num > bs->bl.write_zeroes_alignment) {
1133 if (sector_num % bs->bl.write_zeroes_alignment != 0) {
1134 /* Make a small request up to the first aligned sector. */
1135 num = bs->bl.write_zeroes_alignment;
1136 num -= sector_num % bs->bl.write_zeroes_alignment;
1137 } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) {
1138 /* Shorten the request to the last aligned sector. num cannot
1139 * underflow because num > bs->bl.write_zeroes_alignment.
1140 */
1141 num -= (sector_num + num) % bs->bl.write_zeroes_alignment;
1142 }
1143 }
1144
1145 /* limit request size */
1146 if (num > max_write_zeroes) {
1147 num = max_write_zeroes;
1148 }
1149
1150 ret = -ENOTSUP;
1151 /* First try the efficient write zeroes operation */
1152 if (drv->bdrv_co_write_zeroes) {
1153 ret = drv->bdrv_co_write_zeroes(bs, sector_num, num,
1154 flags & bs->supported_zero_flags);
1155 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1156 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1157 need_flush = true;
1158 }
1159 } else {
1160 assert(!bs->supported_zero_flags);
1161 }
1162
1163 if (ret == -ENOTSUP) {
1164 /* Fall back to bounce buffer if write zeroes is unsupported */
1165 int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length,
1166 MAX_WRITE_ZEROES_BOUNCE_BUFFER);
1167 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1168
1169 if ((flags & BDRV_REQ_FUA) &&
1170 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1171 /* No need for bdrv_driver_pwrite() to do a fallback
1172 * flush on each chunk; use just one at the end */
1173 write_flags &= ~BDRV_REQ_FUA;
1174 need_flush = true;
1175 }
1176 num = MIN(num, max_xfer_len);
1177 iov.iov_len = num * BDRV_SECTOR_SIZE;
1178 if (iov.iov_base == NULL) {
1179 iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE);
1180 if (iov.iov_base == NULL) {
1181 ret = -ENOMEM;
1182 goto fail;
1183 }
1184 memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE);
1185 }
1186 qemu_iovec_init_external(&qiov, &iov, 1);
1187
1188 ret = bdrv_driver_pwritev(bs, sector_num * BDRV_SECTOR_SIZE,
1189 num * BDRV_SECTOR_SIZE, &qiov,
1190 write_flags);
1191
1192 /* Keep bounce buffer around if it is big enough for all
1193 * all future requests.
1194 */
1195 if (num < max_xfer_len) {
1196 qemu_vfree(iov.iov_base);
1197 iov.iov_base = NULL;
1198 }
1199 }
1200
1201 sector_num += num;
1202 nb_sectors -= num;
1203 }
1204
1205 fail:
1206 if (ret == 0 && need_flush) {
1207 ret = bdrv_co_flush(bs);
1208 }
1209 qemu_vfree(iov.iov_base);
1210 return ret;
1211 }
1212
1213 /*
1214 * Forwards an already correctly aligned write request to the BlockDriver.
1215 */
1216 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
1217 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1218 QEMUIOVector *qiov, int flags)
1219 {
1220 BlockDriver *drv = bs->drv;
1221 bool waited;
1222 int ret;
1223
1224 int64_t sector_num = offset >> BDRV_SECTOR_BITS;
1225 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
1226
1227 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
1228 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
1229 assert(!qiov || bytes == qiov->size);
1230 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1231
1232 waited = wait_serialising_requests(req);
1233 assert(!waited || !req->serialising);
1234 assert(req->overlap_offset <= offset);
1235 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1236
1237 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
1238
1239 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1240 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes &&
1241 qemu_iovec_is_zero(qiov)) {
1242 flags |= BDRV_REQ_ZERO_WRITE;
1243 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1244 flags |= BDRV_REQ_MAY_UNMAP;
1245 }
1246 }
1247
1248 if (ret < 0) {
1249 /* Do nothing, write notifier decided to fail this request */
1250 } else if (flags & BDRV_REQ_ZERO_WRITE) {
1251 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1252 ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags);
1253 } else {
1254 bdrv_debug_event(bs, BLKDBG_PWRITEV);
1255 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
1256 }
1257 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1258
1259 bdrv_set_dirty(bs, sector_num, nb_sectors);
1260
1261 if (bs->wr_highest_offset < offset + bytes) {
1262 bs->wr_highest_offset = offset + bytes;
1263 }
1264
1265 if (ret >= 0) {
1266 bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
1267 }
1268
1269 return ret;
1270 }
1271
1272 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
1273 int64_t offset,
1274 unsigned int bytes,
1275 BdrvRequestFlags flags,
1276 BdrvTrackedRequest *req)
1277 {
1278 uint8_t *buf = NULL;
1279 QEMUIOVector local_qiov;
1280 struct iovec iov;
1281 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1282 unsigned int head_padding_bytes, tail_padding_bytes;
1283 int ret = 0;
1284
1285 head_padding_bytes = offset & (align - 1);
1286 tail_padding_bytes = align - ((offset + bytes) & (align - 1));
1287
1288
1289 assert(flags & BDRV_REQ_ZERO_WRITE);
1290 if (head_padding_bytes || tail_padding_bytes) {
1291 buf = qemu_blockalign(bs, align);
1292 iov = (struct iovec) {
1293 .iov_base = buf,
1294 .iov_len = align,
1295 };
1296 qemu_iovec_init_external(&local_qiov, &iov, 1);
1297 }
1298 if (head_padding_bytes) {
1299 uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
1300
1301 /* RMW the unaligned part before head. */
1302 mark_request_serialising(req, align);
1303 wait_serialising_requests(req);
1304 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1305 ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
1306 align, &local_qiov, 0);
1307 if (ret < 0) {
1308 goto fail;
1309 }
1310 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1311
1312 memset(buf + head_padding_bytes, 0, zero_bytes);
1313 ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
1314 &local_qiov,
1315 flags & ~BDRV_REQ_ZERO_WRITE);
1316 if (ret < 0) {
1317 goto fail;
1318 }
1319 offset += zero_bytes;
1320 bytes -= zero_bytes;
1321 }
1322
1323 assert(!bytes || (offset & (align - 1)) == 0);
1324 if (bytes >= align) {
1325 /* Write the aligned part in the middle. */
1326 uint64_t aligned_bytes = bytes & ~(align - 1);
1327 ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes,
1328 NULL, flags);
1329 if (ret < 0) {
1330 goto fail;
1331 }
1332 bytes -= aligned_bytes;
1333 offset += aligned_bytes;
1334 }
1335
1336 assert(!bytes || (offset & (align - 1)) == 0);
1337 if (bytes) {
1338 assert(align == tail_padding_bytes + bytes);
1339 /* RMW the unaligned part after tail. */
1340 mark_request_serialising(req, align);
1341 wait_serialising_requests(req);
1342 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1343 ret = bdrv_aligned_preadv(bs, req, offset, align,
1344 align, &local_qiov, 0);
1345 if (ret < 0) {
1346 goto fail;
1347 }
1348 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1349
1350 memset(buf, 0, bytes);
1351 ret = bdrv_aligned_pwritev(bs, req, offset, align,
1352 &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
1353 }
1354 fail:
1355 qemu_vfree(buf);
1356 return ret;
1357
1358 }
1359
1360 /*
1361 * Handle a write request in coroutine context
1362 */
1363 int coroutine_fn bdrv_co_pwritev(BlockDriverState *bs,
1364 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1365 BdrvRequestFlags flags)
1366 {
1367 BdrvTrackedRequest req;
1368 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1369 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
1370 uint8_t *head_buf = NULL;
1371 uint8_t *tail_buf = NULL;
1372 QEMUIOVector local_qiov;
1373 bool use_local_qiov = false;
1374 int ret;
1375
1376 if (!bs->drv) {
1377 return -ENOMEDIUM;
1378 }
1379 if (bs->read_only) {
1380 return -EPERM;
1381 }
1382 assert(!(bs->open_flags & BDRV_O_INACTIVE));
1383
1384 ret = bdrv_check_byte_request(bs, offset, bytes);
1385 if (ret < 0) {
1386 return ret;
1387 }
1388
1389 /*
1390 * Align write if necessary by performing a read-modify-write cycle.
1391 * Pad qiov with the read parts and be sure to have a tracked request not
1392 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
1393 */
1394 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
1395
1396 if (!qiov) {
1397 ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
1398 goto out;
1399 }
1400
1401 if (offset & (align - 1)) {
1402 QEMUIOVector head_qiov;
1403 struct iovec head_iov;
1404
1405 mark_request_serialising(&req, align);
1406 wait_serialising_requests(&req);
1407
1408 head_buf = qemu_blockalign(bs, align);
1409 head_iov = (struct iovec) {
1410 .iov_base = head_buf,
1411 .iov_len = align,
1412 };
1413 qemu_iovec_init_external(&head_qiov, &head_iov, 1);
1414
1415 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1416 ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
1417 align, &head_qiov, 0);
1418 if (ret < 0) {
1419 goto fail;
1420 }
1421 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1422
1423 qemu_iovec_init(&local_qiov, qiov->niov + 2);
1424 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1425 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1426 use_local_qiov = true;
1427
1428 bytes += offset & (align - 1);
1429 offset = offset & ~(align - 1);
1430 }
1431
1432 if ((offset + bytes) & (align - 1)) {
1433 QEMUIOVector tail_qiov;
1434 struct iovec tail_iov;
1435 size_t tail_bytes;
1436 bool waited;
1437
1438 mark_request_serialising(&req, align);
1439 waited = wait_serialising_requests(&req);
1440 assert(!waited || !use_local_qiov);
1441
1442 tail_buf = qemu_blockalign(bs, align);
1443 tail_iov = (struct iovec) {
1444 .iov_base = tail_buf,
1445 .iov_len = align,
1446 };
1447 qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
1448
1449 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1450 ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
1451 align, &tail_qiov, 0);
1452 if (ret < 0) {
1453 goto fail;
1454 }
1455 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1456
1457 if (!use_local_qiov) {
1458 qemu_iovec_init(&local_qiov, qiov->niov + 1);
1459 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1460 use_local_qiov = true;
1461 }
1462
1463 tail_bytes = (offset + bytes) & (align - 1);
1464 qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
1465
1466 bytes = ROUND_UP(bytes, align);
1467 }
1468
1469 ret = bdrv_aligned_pwritev(bs, &req, offset, bytes,
1470 use_local_qiov ? &local_qiov : qiov,
1471 flags);
1472
1473 fail:
1474
1475 if (use_local_qiov) {
1476 qemu_iovec_destroy(&local_qiov);
1477 }
1478 qemu_vfree(head_buf);
1479 qemu_vfree(tail_buf);
1480 out:
1481 tracked_request_end(&req);
1482 return ret;
1483 }
1484
1485 static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
1486 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1487 BdrvRequestFlags flags)
1488 {
1489 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1490 return -EINVAL;
1491 }
1492
1493 return bdrv_co_pwritev(bs, sector_num << BDRV_SECTOR_BITS,
1494 nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1495 }
1496
1497 int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
1498 int nb_sectors, QEMUIOVector *qiov)
1499 {
1500 trace_bdrv_co_writev(bs, sector_num, nb_sectors);
1501
1502 return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0);
1503 }
1504
1505 int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
1506 int64_t sector_num, int nb_sectors,
1507 BdrvRequestFlags flags)
1508 {
1509 trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags);
1510
1511 if (!(bs->open_flags & BDRV_O_UNMAP)) {
1512 flags &= ~BDRV_REQ_MAY_UNMAP;
1513 }
1514
1515 return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL,
1516 BDRV_REQ_ZERO_WRITE | flags);
1517 }
1518
1519 typedef struct BdrvCoGetBlockStatusData {
1520 BlockDriverState *bs;
1521 BlockDriverState *base;
1522 BlockDriverState **file;
1523 int64_t sector_num;
1524 int nb_sectors;
1525 int *pnum;
1526 int64_t ret;
1527 bool done;
1528 } BdrvCoGetBlockStatusData;
1529
1530 /*
1531 * Returns the allocation status of the specified sectors.
1532 * Drivers not implementing the functionality are assumed to not support
1533 * backing files, hence all their sectors are reported as allocated.
1534 *
1535 * If 'sector_num' is beyond the end of the disk image the return value is 0
1536 * and 'pnum' is set to 0.
1537 *
1538 * 'pnum' is set to the number of sectors (including and immediately following
1539 * the specified sector) that are known to be in the same
1540 * allocated/unallocated state.
1541 *
1542 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes
1543 * beyond the end of the disk image it will be clamped.
1544 *
1545 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
1546 * points to the BDS which the sector range is allocated in.
1547 */
1548 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
1549 int64_t sector_num,
1550 int nb_sectors, int *pnum,
1551 BlockDriverState **file)
1552 {
1553 int64_t total_sectors;
1554 int64_t n;
1555 int64_t ret, ret2;
1556
1557 total_sectors = bdrv_nb_sectors(bs);
1558 if (total_sectors < 0) {
1559 return total_sectors;
1560 }
1561
1562 if (sector_num >= total_sectors) {
1563 *pnum = 0;
1564 return 0;
1565 }
1566
1567 n = total_sectors - sector_num;
1568 if (n < nb_sectors) {
1569 nb_sectors = n;
1570 }
1571
1572 if (!bs->drv->bdrv_co_get_block_status) {
1573 *pnum = nb_sectors;
1574 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
1575 if (bs->drv->protocol_name) {
1576 ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
1577 }
1578 return ret;
1579 }
1580
1581 *file = NULL;
1582 ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
1583 file);
1584 if (ret < 0) {
1585 *pnum = 0;
1586 return ret;
1587 }
1588
1589 if (ret & BDRV_BLOCK_RAW) {
1590 assert(ret & BDRV_BLOCK_OFFSET_VALID);
1591 return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
1592 *pnum, pnum, file);
1593 }
1594
1595 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
1596 ret |= BDRV_BLOCK_ALLOCATED;
1597 } else {
1598 if (bdrv_unallocated_blocks_are_zero(bs)) {
1599 ret |= BDRV_BLOCK_ZERO;
1600 } else if (bs->backing) {
1601 BlockDriverState *bs2 = bs->backing->bs;
1602 int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
1603 if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
1604 ret |= BDRV_BLOCK_ZERO;
1605 }
1606 }
1607 }
1608
1609 if (*file && *file != bs &&
1610 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
1611 (ret & BDRV_BLOCK_OFFSET_VALID)) {
1612 BlockDriverState *file2;
1613 int file_pnum;
1614
1615 ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
1616 *pnum, &file_pnum, &file2);
1617 if (ret2 >= 0) {
1618 /* Ignore errors. This is just providing extra information, it
1619 * is useful but not necessary.
1620 */
1621 if (!file_pnum) {
1622 /* !file_pnum indicates an offset at or beyond the EOF; it is
1623 * perfectly valid for the format block driver to point to such
1624 * offsets, so catch it and mark everything as zero */
1625 ret |= BDRV_BLOCK_ZERO;
1626 } else {
1627 /* Limit request to the range reported by the protocol driver */
1628 *pnum = file_pnum;
1629 ret |= (ret2 & BDRV_BLOCK_ZERO);
1630 }
1631 }
1632 }
1633
1634 return ret;
1635 }
1636
1637 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
1638 BlockDriverState *base,
1639 int64_t sector_num,
1640 int nb_sectors,
1641 int *pnum,
1642 BlockDriverState **file)
1643 {
1644 BlockDriverState *p;
1645 int64_t ret = 0;
1646
1647 assert(bs != base);
1648 for (p = bs; p != base; p = backing_bs(p)) {
1649 ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
1650 if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
1651 break;
1652 }
1653 /* [sector_num, pnum] unallocated on this layer, which could be only
1654 * the first part of [sector_num, nb_sectors]. */
1655 nb_sectors = MIN(nb_sectors, *pnum);
1656 }
1657 return ret;
1658 }
1659
1660 /* Coroutine wrapper for bdrv_get_block_status_above() */
1661 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
1662 {
1663 BdrvCoGetBlockStatusData *data = opaque;
1664
1665 data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
1666 data->sector_num,
1667 data->nb_sectors,
1668 data->pnum,
1669 data->file);
1670 data->done = true;
1671 }
1672
1673 /*
1674 * Synchronous wrapper around bdrv_co_get_block_status_above().
1675 *
1676 * See bdrv_co_get_block_status_above() for details.
1677 */
1678 int64_t bdrv_get_block_status_above(BlockDriverState *bs,
1679 BlockDriverState *base,
1680 int64_t sector_num,
1681 int nb_sectors, int *pnum,
1682 BlockDriverState **file)
1683 {
1684 Coroutine *co;
1685 BdrvCoGetBlockStatusData data = {
1686 .bs = bs,
1687 .base = base,
1688 .file = file,
1689 .sector_num = sector_num,
1690 .nb_sectors = nb_sectors,
1691 .pnum = pnum,
1692 .done = false,
1693 };
1694
1695 if (qemu_in_coroutine()) {
1696 /* Fast-path if already in coroutine context */
1697 bdrv_get_block_status_above_co_entry(&data);
1698 } else {
1699 AioContext *aio_context = bdrv_get_aio_context(bs);
1700
1701 co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry);
1702 qemu_coroutine_enter(co, &data);
1703 while (!data.done) {
1704 aio_poll(aio_context, true);
1705 }
1706 }
1707 return data.ret;
1708 }
1709
1710 int64_t bdrv_get_block_status(BlockDriverState *bs,
1711 int64_t sector_num,
1712 int nb_sectors, int *pnum,
1713 BlockDriverState **file)
1714 {
1715 return bdrv_get_block_status_above(bs, backing_bs(bs),
1716 sector_num, nb_sectors, pnum, file);
1717 }
1718
1719 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
1720 int nb_sectors, int *pnum)
1721 {
1722 BlockDriverState *file;
1723 int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
1724 &file);
1725 if (ret < 0) {
1726 return ret;
1727 }
1728 return !!(ret & BDRV_BLOCK_ALLOCATED);
1729 }
1730
1731 /*
1732 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
1733 *
1734 * Return true if the given sector is allocated in any image between
1735 * BASE and TOP (inclusive). BASE can be NULL to check if the given
1736 * sector is allocated in any image of the chain. Return false otherwise.
1737 *
1738 * 'pnum' is set to the number of sectors (including and immediately following
1739 * the specified sector) that are known to be in the same
1740 * allocated/unallocated state.
1741 *
1742 */
1743 int bdrv_is_allocated_above(BlockDriverState *top,
1744 BlockDriverState *base,
1745 int64_t sector_num,
1746 int nb_sectors, int *pnum)
1747 {
1748 BlockDriverState *intermediate;
1749 int ret, n = nb_sectors;
1750
1751 intermediate = top;
1752 while (intermediate && intermediate != base) {
1753 int pnum_inter;
1754 ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
1755 &pnum_inter);
1756 if (ret < 0) {
1757 return ret;
1758 } else if (ret) {
1759 *pnum = pnum_inter;
1760 return 1;
1761 }
1762
1763 /*
1764 * [sector_num, nb_sectors] is unallocated on top but intermediate
1765 * might have
1766 *
1767 * [sector_num+x, nr_sectors] allocated.
1768 */
1769 if (n > pnum_inter &&
1770 (intermediate == top ||
1771 sector_num + pnum_inter < intermediate->total_sectors)) {
1772 n = pnum_inter;
1773 }
1774
1775 intermediate = backing_bs(intermediate);
1776 }
1777
1778 *pnum = n;
1779 return 0;
1780 }
1781
1782 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
1783 const uint8_t *buf, int nb_sectors)
1784 {
1785 BlockDriver *drv = bs->drv;
1786 int ret;
1787
1788 if (!drv) {
1789 return -ENOMEDIUM;
1790 }
1791 if (!drv->bdrv_write_compressed) {
1792 return -ENOTSUP;
1793 }
1794 ret = bdrv_check_request(bs, sector_num, nb_sectors);
1795 if (ret < 0) {
1796 return ret;
1797 }
1798
1799 assert(QLIST_EMPTY(&bs->dirty_bitmaps));
1800
1801 return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
1802 }
1803
1804 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
1805 int64_t pos, int size)
1806 {
1807 QEMUIOVector qiov;
1808 struct iovec iov = {
1809 .iov_base = (void *) buf,
1810 .iov_len = size,
1811 };
1812
1813 qemu_iovec_init_external(&qiov, &iov, 1);
1814 return bdrv_writev_vmstate(bs, &qiov, pos);
1815 }
1816
1817 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
1818 {
1819 BlockDriver *drv = bs->drv;
1820
1821 if (!drv) {
1822 return -ENOMEDIUM;
1823 } else if (drv->bdrv_save_vmstate) {
1824 return drv->bdrv_save_vmstate(bs, qiov, pos);
1825 } else if (bs->file) {
1826 return bdrv_writev_vmstate(bs->file->bs, qiov, pos);
1827 }
1828
1829 return -ENOTSUP;
1830 }
1831
1832 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
1833 int64_t pos, int size)
1834 {
1835 BlockDriver *drv = bs->drv;
1836 if (!drv)
1837 return -ENOMEDIUM;
1838 if (drv->bdrv_load_vmstate)
1839 return drv->bdrv_load_vmstate(bs, buf, pos, size);
1840 if (bs->file)
1841 return bdrv_load_vmstate(bs->file->bs, buf, pos, size);
1842 return -ENOTSUP;
1843 }
1844
1845 /**************************************************************/
1846 /* async I/Os */
1847
1848 BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
1849 QEMUIOVector *qiov, int nb_sectors,
1850 BlockCompletionFunc *cb, void *opaque)
1851 {
1852 trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
1853
1854 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1855 cb, opaque, false);
1856 }
1857
1858 BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
1859 QEMUIOVector *qiov, int nb_sectors,
1860 BlockCompletionFunc *cb, void *opaque)
1861 {
1862 trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
1863
1864 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
1865 cb, opaque, true);
1866 }
1867
1868 BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs,
1869 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags,
1870 BlockCompletionFunc *cb, void *opaque)
1871 {
1872 trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque);
1873
1874 return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors,
1875 BDRV_REQ_ZERO_WRITE | flags,
1876 cb, opaque, true);
1877 }
1878
1879 void bdrv_aio_cancel(BlockAIOCB *acb)
1880 {
1881 qemu_aio_ref(acb);
1882 bdrv_aio_cancel_async(acb);
1883 while (acb->refcnt > 1) {
1884 if (acb->aiocb_info->get_aio_context) {
1885 aio_poll(acb->aiocb_info->get_aio_context(acb), true);
1886 } else if (acb->bs) {
1887 aio_poll(bdrv_get_aio_context(acb->bs), true);
1888 } else {
1889 abort();
1890 }
1891 }
1892 qemu_aio_unref(acb);
1893 }
1894
1895 /* Async version of aio cancel. The caller is not blocked if the acb implements
1896 * cancel_async, otherwise we do nothing and let the request normally complete.
1897 * In either case the completion callback must be called. */
1898 void bdrv_aio_cancel_async(BlockAIOCB *acb)
1899 {
1900 if (acb->aiocb_info->cancel_async) {
1901 acb->aiocb_info->cancel_async(acb);
1902 }
1903 }
1904
1905 /**************************************************************/
1906 /* async block device emulation */
1907
1908 typedef struct BlockAIOCBCoroutine {
1909 BlockAIOCB common;
1910 BlockRequest req;
1911 bool is_write;
1912 bool need_bh;
1913 bool *done;
1914 QEMUBH* bh;
1915 } BlockAIOCBCoroutine;
1916
1917 static const AIOCBInfo bdrv_em_co_aiocb_info = {
1918 .aiocb_size = sizeof(BlockAIOCBCoroutine),
1919 };
1920
1921 static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
1922 {
1923 if (!acb->need_bh) {
1924 acb->common.cb(acb->common.opaque, acb->req.error);
1925 qemu_aio_unref(acb);
1926 }
1927 }
1928
1929 static void bdrv_co_em_bh(void *opaque)
1930 {
1931 BlockAIOCBCoroutine *acb = opaque;
1932
1933 assert(!acb->need_bh);
1934 qemu_bh_delete(acb->bh);
1935 bdrv_co_complete(acb);
1936 }
1937
1938 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
1939 {
1940 acb->need_bh = false;
1941 if (acb->req.error != -EINPROGRESS) {
1942 BlockDriverState *bs = acb->common.bs;
1943
1944 acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
1945 qemu_bh_schedule(acb->bh);
1946 }
1947 }
1948
1949 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
1950 static void coroutine_fn bdrv_co_do_rw(void *opaque)
1951 {
1952 BlockAIOCBCoroutine *acb = opaque;
1953 BlockDriverState *bs = acb->common.bs;
1954
1955 if (!acb->is_write) {
1956 acb->req.error = bdrv_co_do_readv(bs, acb->req.sector,
1957 acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
1958 } else {
1959 acb->req.error = bdrv_co_do_writev(bs, acb->req.sector,
1960 acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
1961 }
1962
1963 bdrv_co_complete(acb);
1964 }
1965
1966 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
1967 int64_t sector_num,
1968 QEMUIOVector *qiov,
1969 int nb_sectors,
1970 BdrvRequestFlags flags,
1971 BlockCompletionFunc *cb,
1972 void *opaque,
1973 bool is_write)
1974 {
1975 Coroutine *co;
1976 BlockAIOCBCoroutine *acb;
1977
1978 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
1979 acb->need_bh = true;
1980 acb->req.error = -EINPROGRESS;
1981 acb->req.sector = sector_num;
1982 acb->req.nb_sectors = nb_sectors;
1983 acb->req.qiov = qiov;
1984 acb->req.flags = flags;
1985 acb->is_write = is_write;
1986
1987 co = qemu_coroutine_create(bdrv_co_do_rw);
1988 qemu_coroutine_enter(co, acb);
1989
1990 bdrv_co_maybe_schedule_bh(acb);
1991 return &acb->common;
1992 }
1993
1994 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
1995 {
1996 BlockAIOCBCoroutine *acb = opaque;
1997 BlockDriverState *bs = acb->common.bs;
1998
1999 acb->req.error = bdrv_co_flush(bs);
2000 bdrv_co_complete(acb);
2001 }
2002
2003 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
2004 BlockCompletionFunc *cb, void *opaque)
2005 {
2006 trace_bdrv_aio_flush(bs, opaque);
2007
2008 Coroutine *co;
2009 BlockAIOCBCoroutine *acb;
2010
2011 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2012 acb->need_bh = true;
2013 acb->req.error = -EINPROGRESS;
2014
2015 co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
2016 qemu_coroutine_enter(co, acb);
2017
2018 bdrv_co_maybe_schedule_bh(acb);
2019 return &acb->common;
2020 }
2021
2022 static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
2023 {
2024 BlockAIOCBCoroutine *acb = opaque;
2025 BlockDriverState *bs = acb->common.bs;
2026
2027 acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
2028 bdrv_co_complete(acb);
2029 }
2030
2031 BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
2032 int64_t sector_num, int nb_sectors,
2033 BlockCompletionFunc *cb, void *opaque)
2034 {
2035 Coroutine *co;
2036 BlockAIOCBCoroutine *acb;
2037
2038 trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
2039
2040 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2041 acb->need_bh = true;
2042 acb->req.error = -EINPROGRESS;
2043 acb->req.sector = sector_num;
2044 acb->req.nb_sectors = nb_sectors;
2045 co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
2046 qemu_coroutine_enter(co, acb);
2047
2048 bdrv_co_maybe_schedule_bh(acb);
2049 return &acb->common;
2050 }
2051
2052 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
2053 BlockCompletionFunc *cb, void *opaque)
2054 {
2055 BlockAIOCB *acb;
2056
2057 acb = g_malloc(aiocb_info->aiocb_size);
2058 acb->aiocb_info = aiocb_info;
2059 acb->bs = bs;
2060 acb->cb = cb;
2061 acb->opaque = opaque;
2062 acb->refcnt = 1;
2063 return acb;
2064 }
2065
2066 void qemu_aio_ref(void *p)
2067 {
2068 BlockAIOCB *acb = p;
2069 acb->refcnt++;
2070 }
2071
2072 void qemu_aio_unref(void *p)
2073 {
2074 BlockAIOCB *acb = p;
2075 assert(acb->refcnt > 0);
2076 if (--acb->refcnt == 0) {
2077 g_free(acb);
2078 }
2079 }
2080
2081 /**************************************************************/
2082 /* Coroutine block device emulation */
2083
2084 static void coroutine_fn bdrv_flush_co_entry(void *opaque)
2085 {
2086 RwCo *rwco = opaque;
2087
2088 rwco->ret = bdrv_co_flush(rwco->bs);
2089 }
2090
2091 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2092 {
2093 int ret;
2094 BdrvTrackedRequest req;
2095
2096 if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2097 bdrv_is_sg(bs)) {
2098 return 0;
2099 }
2100
2101 tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
2102
2103 /* Write back all layers by calling one driver function */
2104 if (bs->drv->bdrv_co_flush) {
2105 ret = bs->drv->bdrv_co_flush(bs);
2106 goto out;
2107 }
2108
2109 /* Write back cached data to the OS even with cache=unsafe */
2110 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
2111 if (bs->drv->bdrv_co_flush_to_os) {
2112 ret = bs->drv->bdrv_co_flush_to_os(bs);
2113 if (ret < 0) {
2114 goto out;
2115 }
2116 }
2117
2118 /* But don't actually force it to the disk with cache=unsafe */
2119 if (bs->open_flags & BDRV_O_NO_FLUSH) {
2120 goto flush_parent;
2121 }
2122
2123 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
2124 if (bs->drv->bdrv_co_flush_to_disk) {
2125 ret = bs->drv->bdrv_co_flush_to_disk(bs);
2126 } else if (bs->drv->bdrv_aio_flush) {
2127 BlockAIOCB *acb;
2128 CoroutineIOCompletion co = {
2129 .coroutine = qemu_coroutine_self(),
2130 };
2131
2132 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2133 if (acb == NULL) {
2134 ret = -EIO;
2135 } else {
2136 qemu_coroutine_yield();
2137 ret = co.ret;
2138 }
2139 } else {
2140 /*
2141 * Some block drivers always operate in either writethrough or unsafe
2142 * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2143 * know how the server works (because the behaviour is hardcoded or
2144 * depends on server-side configuration), so we can't ensure that
2145 * everything is safe on disk. Returning an error doesn't work because
2146 * that would break guests even if the server operates in writethrough
2147 * mode.
2148 *
2149 * Let's hope the user knows what he's doing.
2150 */
2151 ret = 0;
2152 }
2153 if (ret < 0) {
2154 goto out;
2155 }
2156
2157 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH
2158 * in the case of cache=unsafe, so there are no useless flushes.
2159 */
2160 flush_parent:
2161 ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2162 out:
2163 tracked_request_end(&req);
2164 return ret;
2165 }
2166
2167 int bdrv_flush(BlockDriverState *bs)
2168 {
2169 Coroutine *co;
2170 RwCo rwco = {
2171 .bs = bs,
2172 .ret = NOT_DONE,
2173 };
2174
2175 if (qemu_in_coroutine()) {
2176 /* Fast-path if already in coroutine context */
2177 bdrv_flush_co_entry(&rwco);
2178 } else {
2179 AioContext *aio_context = bdrv_get_aio_context(bs);
2180
2181 co = qemu_coroutine_create(bdrv_flush_co_entry);
2182 qemu_coroutine_enter(co, &rwco);
2183 while (rwco.ret == NOT_DONE) {
2184 aio_poll(aio_context, true);
2185 }
2186 }
2187
2188 return rwco.ret;
2189 }
2190
2191 typedef struct DiscardCo {
2192 BlockDriverState *bs;
2193 int64_t sector_num;
2194 int nb_sectors;
2195 int ret;
2196 } DiscardCo;
2197 static void coroutine_fn bdrv_discard_co_entry(void *opaque)
2198 {
2199 DiscardCo *rwco = opaque;
2200
2201 rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors);
2202 }
2203
2204 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
2205 int nb_sectors)
2206 {
2207 BdrvTrackedRequest req;
2208 int max_discard, ret;
2209
2210 if (!bs->drv) {
2211 return -ENOMEDIUM;
2212 }
2213
2214 ret = bdrv_check_request(bs, sector_num, nb_sectors);
2215 if (ret < 0) {
2216 return ret;
2217 } else if (bs->read_only) {
2218 return -EPERM;
2219 }
2220 assert(!(bs->open_flags & BDRV_O_INACTIVE));
2221
2222 /* Do nothing if disabled. */
2223 if (!(bs->open_flags & BDRV_O_UNMAP)) {
2224 return 0;
2225 }
2226
2227 if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) {
2228 return 0;
2229 }
2230
2231 tracked_request_begin(&req, bs, sector_num, nb_sectors,
2232 BDRV_TRACKED_DISCARD);
2233 bdrv_set_dirty(bs, sector_num, nb_sectors);
2234
2235 max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS);
2236 while (nb_sectors > 0) {
2237 int ret;
2238 int num = nb_sectors;
2239
2240 /* align request */
2241 if (bs->bl.discard_alignment &&
2242 num >= bs->bl.discard_alignment &&
2243 sector_num % bs->bl.discard_alignment) {
2244 if (num > bs->bl.discard_alignment) {
2245 num = bs->bl.discard_alignment;
2246 }
2247 num -= sector_num % bs->bl.discard_alignment;
2248 }
2249
2250 /* limit request size */
2251 if (num > max_discard) {
2252 num = max_discard;
2253 }
2254
2255 if (bs->drv->bdrv_co_discard) {
2256 ret = bs->drv->bdrv_co_discard(bs, sector_num, num);
2257 } else {
2258 BlockAIOCB *acb;
2259 CoroutineIOCompletion co = {
2260 .coroutine = qemu_coroutine_self(),
2261 };
2262
2263 acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
2264 bdrv_co_io_em_complete, &co);
2265 if (acb == NULL) {
2266 ret = -EIO;
2267 goto out;
2268 } else {
2269 qemu_coroutine_yield();
2270 ret = co.ret;
2271 }
2272 }
2273 if (ret && ret != -ENOTSUP) {
2274 goto out;
2275 }
2276
2277 sector_num += num;
2278 nb_sectors -= num;
2279 }
2280 ret = 0;
2281 out:
2282 tracked_request_end(&req);
2283 return ret;
2284 }
2285
2286 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
2287 {
2288 Coroutine *co;
2289 DiscardCo rwco = {
2290 .bs = bs,
2291 .sector_num = sector_num,
2292 .nb_sectors = nb_sectors,
2293 .ret = NOT_DONE,
2294 };
2295
2296 if (qemu_in_coroutine()) {
2297 /* Fast-path if already in coroutine context */
2298 bdrv_discard_co_entry(&rwco);
2299 } else {
2300 AioContext *aio_context = bdrv_get_aio_context(bs);
2301
2302 co = qemu_coroutine_create(bdrv_discard_co_entry);
2303 qemu_coroutine_enter(co, &rwco);
2304 while (rwco.ret == NOT_DONE) {
2305 aio_poll(aio_context, true);
2306 }
2307 }
2308
2309 return rwco.ret;
2310 }
2311
2312 typedef struct {
2313 CoroutineIOCompletion *co;
2314 QEMUBH *bh;
2315 } BdrvIoctlCompletionData;
2316
2317 static void bdrv_ioctl_bh_cb(void *opaque)
2318 {
2319 BdrvIoctlCompletionData *data = opaque;
2320
2321 bdrv_co_io_em_complete(data->co, -ENOTSUP);
2322 qemu_bh_delete(data->bh);
2323 }
2324
2325 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
2326 {
2327 BlockDriver *drv = bs->drv;
2328 BdrvTrackedRequest tracked_req;
2329 CoroutineIOCompletion co = {
2330 .coroutine = qemu_coroutine_self(),
2331 };
2332 BlockAIOCB *acb;
2333
2334 tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
2335 if (!drv || !drv->bdrv_aio_ioctl) {
2336 co.ret = -ENOTSUP;
2337 goto out;
2338 }
2339
2340 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
2341 if (!acb) {
2342 BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1);
2343 data->bh = aio_bh_new(bdrv_get_aio_context(bs),
2344 bdrv_ioctl_bh_cb, data);
2345 data->co = &co;
2346 qemu_bh_schedule(data->bh);
2347 }
2348 qemu_coroutine_yield();
2349 out:
2350 tracked_request_end(&tracked_req);
2351 return co.ret;
2352 }
2353
2354 typedef struct {
2355 BlockDriverState *bs;
2356 int req;
2357 void *buf;
2358 int ret;
2359 } BdrvIoctlCoData;
2360
2361 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
2362 {
2363 BdrvIoctlCoData *data = opaque;
2364 data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
2365 }
2366
2367 /* needed for generic scsi interface */
2368 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
2369 {
2370 BdrvIoctlCoData data = {
2371 .bs = bs,
2372 .req = req,
2373 .buf = buf,
2374 .ret = -EINPROGRESS,
2375 };
2376
2377 if (qemu_in_coroutine()) {
2378 /* Fast-path if already in coroutine context */
2379 bdrv_co_ioctl_entry(&data);
2380 } else {
2381 Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry);
2382
2383 qemu_coroutine_enter(co, &data);
2384 while (data.ret == -EINPROGRESS) {
2385 aio_poll(bdrv_get_aio_context(bs), true);
2386 }
2387 }
2388 return data.ret;
2389 }
2390
2391 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
2392 {
2393 BlockAIOCBCoroutine *acb = opaque;
2394 acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
2395 acb->req.req, acb->req.buf);
2396 bdrv_co_complete(acb);
2397 }
2398
2399 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
2400 unsigned long int req, void *buf,
2401 BlockCompletionFunc *cb, void *opaque)
2402 {
2403 BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
2404 bs, cb, opaque);
2405 Coroutine *co;
2406
2407 acb->need_bh = true;
2408 acb->req.error = -EINPROGRESS;
2409 acb->req.req = req;
2410 acb->req.buf = buf;
2411 co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry);
2412 qemu_coroutine_enter(co, acb);
2413
2414 bdrv_co_maybe_schedule_bh(acb);
2415 return &acb->common;
2416 }
2417
2418 void *qemu_blockalign(BlockDriverState *bs, size_t size)
2419 {
2420 return qemu_memalign(bdrv_opt_mem_align(bs), size);
2421 }
2422
2423 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
2424 {
2425 return memset(qemu_blockalign(bs, size), 0, size);
2426 }
2427
2428 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
2429 {
2430 size_t align = bdrv_opt_mem_align(bs);
2431
2432 /* Ensure that NULL is never returned on success */
2433 assert(align > 0);
2434 if (size == 0) {
2435 size = align;
2436 }
2437
2438 return qemu_try_memalign(align, size);
2439 }
2440
2441 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
2442 {
2443 void *mem = qemu_try_blockalign(bs, size);
2444
2445 if (mem) {
2446 memset(mem, 0, size);
2447 }
2448
2449 return mem;
2450 }
2451
2452 /*
2453 * Check if all memory in this vector is sector aligned.
2454 */
2455 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
2456 {
2457 int i;
2458 size_t alignment = bdrv_min_mem_align(bs);
2459
2460 for (i = 0; i < qiov->niov; i++) {
2461 if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
2462 return false;
2463 }
2464 if (qiov->iov[i].iov_len % alignment) {
2465 return false;
2466 }
2467 }
2468
2469 return true;
2470 }
2471
2472 void bdrv_add_before_write_notifier(BlockDriverState *bs,
2473 NotifierWithReturn *notifier)
2474 {
2475 notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
2476 }
2477
2478 void bdrv_io_plug(BlockDriverState *bs)
2479 {
2480 BdrvChild *child;
2481
2482 QLIST_FOREACH(child, &bs->children, next) {
2483 bdrv_io_plug(child->bs);
2484 }
2485
2486 if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) {
2487 BlockDriver *drv = bs->drv;
2488 if (drv && drv->bdrv_io_plug) {
2489 drv->bdrv_io_plug(bs);
2490 }
2491 }
2492 }
2493
2494 void bdrv_io_unplug(BlockDriverState *bs)
2495 {
2496 BdrvChild *child;
2497
2498 assert(bs->io_plugged);
2499 if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) {
2500 BlockDriver *drv = bs->drv;
2501 if (drv && drv->bdrv_io_unplug) {
2502 drv->bdrv_io_unplug(bs);
2503 }
2504 }
2505
2506 QLIST_FOREACH(child, &bs->children, next) {
2507 bdrv_io_unplug(child->bs);
2508 }
2509 }
2510
2511 void bdrv_io_unplugged_begin(BlockDriverState *bs)
2512 {
2513 BdrvChild *child;
2514
2515 if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) {
2516 BlockDriver *drv = bs->drv;
2517 if (drv && drv->bdrv_io_unplug) {
2518 drv->bdrv_io_unplug(bs);
2519 }
2520 }
2521
2522 QLIST_FOREACH(child, &bs->children, next) {
2523 bdrv_io_unplugged_begin(child->bs);
2524 }
2525 }
2526
2527 void bdrv_io_unplugged_end(BlockDriverState *bs)
2528 {
2529 BdrvChild *child;
2530
2531 assert(bs->io_plug_disabled);
2532 QLIST_FOREACH(child, &bs->children, next) {
2533 bdrv_io_unplugged_end(child->bs);
2534 }
2535
2536 if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) {
2537 BlockDriver *drv = bs->drv;
2538 if (drv && drv->bdrv_io_plug) {
2539 drv->bdrv_io_plug(bs);
2540 }
2541 }
2542 }