Merge remote-tracking branch 'remotes/armbru/tags/pull-qapi-2017-03-16' into staging
[qemu.git] / block / io.c
1 /*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/blockjob.h"
29 #include "block/block_int.h"
30 #include "qemu/cutils.h"
31 #include "qapi/error.h"
32 #include "qemu/error-report.h"
33
34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
35
36 static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child,
37 int64_t offset,
38 QEMUIOVector *qiov,
39 BdrvRequestFlags flags,
40 BlockCompletionFunc *cb,
41 void *opaque,
42 bool is_write);
43 static void coroutine_fn bdrv_co_do_rw(void *opaque);
44 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
45 int64_t offset, int count, BdrvRequestFlags flags);
46
47 static void bdrv_parent_drained_begin(BlockDriverState *bs)
48 {
49 BdrvChild *c;
50
51 QLIST_FOREACH(c, &bs->parents, next_parent) {
52 if (c->role->drained_begin) {
53 c->role->drained_begin(c);
54 }
55 }
56 }
57
58 static void bdrv_parent_drained_end(BlockDriverState *bs)
59 {
60 BdrvChild *c;
61
62 QLIST_FOREACH(c, &bs->parents, next_parent) {
63 if (c->role->drained_end) {
64 c->role->drained_end(c);
65 }
66 }
67 }
68
69 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
70 {
71 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
72 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
73 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
74 src->opt_mem_alignment);
75 dst->min_mem_alignment = MAX(dst->min_mem_alignment,
76 src->min_mem_alignment);
77 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
78 }
79
80 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
81 {
82 BlockDriver *drv = bs->drv;
83 Error *local_err = NULL;
84
85 memset(&bs->bl, 0, sizeof(bs->bl));
86
87 if (!drv) {
88 return;
89 }
90
91 /* Default alignment based on whether driver has byte interface */
92 bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512;
93
94 /* Take some limits from the children as a default */
95 if (bs->file) {
96 bdrv_refresh_limits(bs->file->bs, &local_err);
97 if (local_err) {
98 error_propagate(errp, local_err);
99 return;
100 }
101 bdrv_merge_limits(&bs->bl, &bs->file->bs->bl);
102 } else {
103 bs->bl.min_mem_alignment = 512;
104 bs->bl.opt_mem_alignment = getpagesize();
105
106 /* Safe default since most protocols use readv()/writev()/etc */
107 bs->bl.max_iov = IOV_MAX;
108 }
109
110 if (bs->backing) {
111 bdrv_refresh_limits(bs->backing->bs, &local_err);
112 if (local_err) {
113 error_propagate(errp, local_err);
114 return;
115 }
116 bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl);
117 }
118
119 /* Then let the driver override it */
120 if (drv->bdrv_refresh_limits) {
121 drv->bdrv_refresh_limits(bs, errp);
122 }
123 }
124
125 /**
126 * The copy-on-read flag is actually a reference count so multiple users may
127 * use the feature without worrying about clobbering its previous state.
128 * Copy-on-read stays enabled until all users have called to disable it.
129 */
130 void bdrv_enable_copy_on_read(BlockDriverState *bs)
131 {
132 bs->copy_on_read++;
133 }
134
135 void bdrv_disable_copy_on_read(BlockDriverState *bs)
136 {
137 assert(bs->copy_on_read > 0);
138 bs->copy_on_read--;
139 }
140
141 /* Check if any requests are in-flight (including throttled requests) */
142 bool bdrv_requests_pending(BlockDriverState *bs)
143 {
144 BdrvChild *child;
145
146 if (atomic_read(&bs->in_flight)) {
147 return true;
148 }
149
150 QLIST_FOREACH(child, &bs->children, next) {
151 if (bdrv_requests_pending(child->bs)) {
152 return true;
153 }
154 }
155
156 return false;
157 }
158
159 static bool bdrv_drain_recurse(BlockDriverState *bs)
160 {
161 BdrvChild *child;
162 bool waited;
163
164 waited = BDRV_POLL_WHILE(bs, atomic_read(&bs->in_flight) > 0);
165
166 if (bs->drv && bs->drv->bdrv_drain) {
167 bs->drv->bdrv_drain(bs);
168 }
169
170 QLIST_FOREACH(child, &bs->children, next) {
171 waited |= bdrv_drain_recurse(child->bs);
172 }
173
174 return waited;
175 }
176
177 typedef struct {
178 Coroutine *co;
179 BlockDriverState *bs;
180 bool done;
181 } BdrvCoDrainData;
182
183 static void bdrv_co_drain_bh_cb(void *opaque)
184 {
185 BdrvCoDrainData *data = opaque;
186 Coroutine *co = data->co;
187 BlockDriverState *bs = data->bs;
188
189 bdrv_dec_in_flight(bs);
190 bdrv_drained_begin(bs);
191 data->done = true;
192 aio_co_wake(co);
193 }
194
195 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
196 {
197 BdrvCoDrainData data;
198
199 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
200 * other coroutines run if they were queued from
201 * qemu_co_queue_run_restart(). */
202
203 assert(qemu_in_coroutine());
204 data = (BdrvCoDrainData) {
205 .co = qemu_coroutine_self(),
206 .bs = bs,
207 .done = false,
208 };
209 bdrv_inc_in_flight(bs);
210 aio_bh_schedule_oneshot(bdrv_get_aio_context(bs),
211 bdrv_co_drain_bh_cb, &data);
212
213 qemu_coroutine_yield();
214 /* If we are resumed from some other event (such as an aio completion or a
215 * timer callback), it is a bug in the caller that should be fixed. */
216 assert(data.done);
217 }
218
219 void bdrv_drained_begin(BlockDriverState *bs)
220 {
221 if (qemu_in_coroutine()) {
222 bdrv_co_yield_to_drain(bs);
223 return;
224 }
225
226 if (!bs->quiesce_counter++) {
227 aio_disable_external(bdrv_get_aio_context(bs));
228 bdrv_parent_drained_begin(bs);
229 }
230
231 bdrv_drain_recurse(bs);
232 }
233
234 void bdrv_drained_end(BlockDriverState *bs)
235 {
236 assert(bs->quiesce_counter > 0);
237 if (--bs->quiesce_counter > 0) {
238 return;
239 }
240
241 bdrv_parent_drained_end(bs);
242 aio_enable_external(bdrv_get_aio_context(bs));
243 }
244
245 /*
246 * Wait for pending requests to complete on a single BlockDriverState subtree,
247 * and suspend block driver's internal I/O until next request arrives.
248 *
249 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
250 * AioContext.
251 *
252 * Only this BlockDriverState's AioContext is run, so in-flight requests must
253 * not depend on events in other AioContexts. In that case, use
254 * bdrv_drain_all() instead.
255 */
256 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
257 {
258 assert(qemu_in_coroutine());
259 bdrv_drained_begin(bs);
260 bdrv_drained_end(bs);
261 }
262
263 void bdrv_drain(BlockDriverState *bs)
264 {
265 bdrv_drained_begin(bs);
266 bdrv_drained_end(bs);
267 }
268
269 /*
270 * Wait for pending requests to complete across all BlockDriverStates
271 *
272 * This function does not flush data to disk, use bdrv_flush_all() for that
273 * after calling this function.
274 *
275 * This pauses all block jobs and disables external clients. It must
276 * be paired with bdrv_drain_all_end().
277 *
278 * NOTE: no new block jobs or BlockDriverStates can be created between
279 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
280 */
281 void bdrv_drain_all_begin(void)
282 {
283 /* Always run first iteration so any pending completion BHs run */
284 bool waited = true;
285 BlockDriverState *bs;
286 BdrvNextIterator it;
287 BlockJob *job = NULL;
288 GSList *aio_ctxs = NULL, *ctx;
289
290 while ((job = block_job_next(job))) {
291 AioContext *aio_context = blk_get_aio_context(job->blk);
292
293 aio_context_acquire(aio_context);
294 block_job_pause(job);
295 aio_context_release(aio_context);
296 }
297
298 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
299 AioContext *aio_context = bdrv_get_aio_context(bs);
300
301 aio_context_acquire(aio_context);
302 bdrv_parent_drained_begin(bs);
303 aio_disable_external(aio_context);
304 aio_context_release(aio_context);
305
306 if (!g_slist_find(aio_ctxs, aio_context)) {
307 aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
308 }
309 }
310
311 /* Note that completion of an asynchronous I/O operation can trigger any
312 * number of other I/O operations on other devices---for example a
313 * coroutine can submit an I/O request to another device in response to
314 * request completion. Therefore we must keep looping until there was no
315 * more activity rather than simply draining each device independently.
316 */
317 while (waited) {
318 waited = false;
319
320 for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
321 AioContext *aio_context = ctx->data;
322
323 aio_context_acquire(aio_context);
324 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
325 if (aio_context == bdrv_get_aio_context(bs)) {
326 waited |= bdrv_drain_recurse(bs);
327 }
328 }
329 aio_context_release(aio_context);
330 }
331 }
332
333 g_slist_free(aio_ctxs);
334 }
335
336 void bdrv_drain_all_end(void)
337 {
338 BlockDriverState *bs;
339 BdrvNextIterator it;
340 BlockJob *job = NULL;
341
342 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
343 AioContext *aio_context = bdrv_get_aio_context(bs);
344
345 aio_context_acquire(aio_context);
346 aio_enable_external(aio_context);
347 bdrv_parent_drained_end(bs);
348 aio_context_release(aio_context);
349 }
350
351 while ((job = block_job_next(job))) {
352 AioContext *aio_context = blk_get_aio_context(job->blk);
353
354 aio_context_acquire(aio_context);
355 block_job_resume(job);
356 aio_context_release(aio_context);
357 }
358 }
359
360 void bdrv_drain_all(void)
361 {
362 bdrv_drain_all_begin();
363 bdrv_drain_all_end();
364 }
365
366 /**
367 * Remove an active request from the tracked requests list
368 *
369 * This function should be called when a tracked request is completing.
370 */
371 static void tracked_request_end(BdrvTrackedRequest *req)
372 {
373 if (req->serialising) {
374 req->bs->serialising_in_flight--;
375 }
376
377 QLIST_REMOVE(req, list);
378 qemu_co_queue_restart_all(&req->wait_queue);
379 }
380
381 /**
382 * Add an active request to the tracked requests list
383 */
384 static void tracked_request_begin(BdrvTrackedRequest *req,
385 BlockDriverState *bs,
386 int64_t offset,
387 unsigned int bytes,
388 enum BdrvTrackedRequestType type)
389 {
390 *req = (BdrvTrackedRequest){
391 .bs = bs,
392 .offset = offset,
393 .bytes = bytes,
394 .type = type,
395 .co = qemu_coroutine_self(),
396 .serialising = false,
397 .overlap_offset = offset,
398 .overlap_bytes = bytes,
399 };
400
401 qemu_co_queue_init(&req->wait_queue);
402
403 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
404 }
405
406 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
407 {
408 int64_t overlap_offset = req->offset & ~(align - 1);
409 unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
410 - overlap_offset;
411
412 if (!req->serialising) {
413 req->bs->serialising_in_flight++;
414 req->serialising = true;
415 }
416
417 req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
418 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
419 }
420
421 /**
422 * Round a region to cluster boundaries (sector-based)
423 */
424 void bdrv_round_sectors_to_clusters(BlockDriverState *bs,
425 int64_t sector_num, int nb_sectors,
426 int64_t *cluster_sector_num,
427 int *cluster_nb_sectors)
428 {
429 BlockDriverInfo bdi;
430
431 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
432 *cluster_sector_num = sector_num;
433 *cluster_nb_sectors = nb_sectors;
434 } else {
435 int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
436 *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
437 *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
438 nb_sectors, c);
439 }
440 }
441
442 /**
443 * Round a region to cluster boundaries
444 */
445 void bdrv_round_to_clusters(BlockDriverState *bs,
446 int64_t offset, unsigned int bytes,
447 int64_t *cluster_offset,
448 unsigned int *cluster_bytes)
449 {
450 BlockDriverInfo bdi;
451
452 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
453 *cluster_offset = offset;
454 *cluster_bytes = bytes;
455 } else {
456 int64_t c = bdi.cluster_size;
457 *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
458 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
459 }
460 }
461
462 static int bdrv_get_cluster_size(BlockDriverState *bs)
463 {
464 BlockDriverInfo bdi;
465 int ret;
466
467 ret = bdrv_get_info(bs, &bdi);
468 if (ret < 0 || bdi.cluster_size == 0) {
469 return bs->bl.request_alignment;
470 } else {
471 return bdi.cluster_size;
472 }
473 }
474
475 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
476 int64_t offset, unsigned int bytes)
477 {
478 /* aaaa bbbb */
479 if (offset >= req->overlap_offset + req->overlap_bytes) {
480 return false;
481 }
482 /* bbbb aaaa */
483 if (req->overlap_offset >= offset + bytes) {
484 return false;
485 }
486 return true;
487 }
488
489 void bdrv_inc_in_flight(BlockDriverState *bs)
490 {
491 atomic_inc(&bs->in_flight);
492 }
493
494 static void dummy_bh_cb(void *opaque)
495 {
496 }
497
498 void bdrv_wakeup(BlockDriverState *bs)
499 {
500 if (bs->wakeup) {
501 aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb, NULL);
502 }
503 }
504
505 void bdrv_dec_in_flight(BlockDriverState *bs)
506 {
507 atomic_dec(&bs->in_flight);
508 bdrv_wakeup(bs);
509 }
510
511 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
512 {
513 BlockDriverState *bs = self->bs;
514 BdrvTrackedRequest *req;
515 bool retry;
516 bool waited = false;
517
518 if (!bs->serialising_in_flight) {
519 return false;
520 }
521
522 do {
523 retry = false;
524 QLIST_FOREACH(req, &bs->tracked_requests, list) {
525 if (req == self || (!req->serialising && !self->serialising)) {
526 continue;
527 }
528 if (tracked_request_overlaps(req, self->overlap_offset,
529 self->overlap_bytes))
530 {
531 /* Hitting this means there was a reentrant request, for
532 * example, a block driver issuing nested requests. This must
533 * never happen since it means deadlock.
534 */
535 assert(qemu_coroutine_self() != req->co);
536
537 /* If the request is already (indirectly) waiting for us, or
538 * will wait for us as soon as it wakes up, then just go on
539 * (instead of producing a deadlock in the former case). */
540 if (!req->waiting_for) {
541 self->waiting_for = req;
542 qemu_co_queue_wait(&req->wait_queue, NULL);
543 self->waiting_for = NULL;
544 retry = true;
545 waited = true;
546 break;
547 }
548 }
549 }
550 } while (retry);
551
552 return waited;
553 }
554
555 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
556 size_t size)
557 {
558 if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
559 return -EIO;
560 }
561
562 if (!bdrv_is_inserted(bs)) {
563 return -ENOMEDIUM;
564 }
565
566 if (offset < 0) {
567 return -EIO;
568 }
569
570 return 0;
571 }
572
573 typedef struct RwCo {
574 BdrvChild *child;
575 int64_t offset;
576 QEMUIOVector *qiov;
577 bool is_write;
578 int ret;
579 BdrvRequestFlags flags;
580 } RwCo;
581
582 static void coroutine_fn bdrv_rw_co_entry(void *opaque)
583 {
584 RwCo *rwco = opaque;
585
586 if (!rwco->is_write) {
587 rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset,
588 rwco->qiov->size, rwco->qiov,
589 rwco->flags);
590 } else {
591 rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset,
592 rwco->qiov->size, rwco->qiov,
593 rwco->flags);
594 }
595 }
596
597 /*
598 * Process a vectored synchronous request using coroutines
599 */
600 static int bdrv_prwv_co(BdrvChild *child, int64_t offset,
601 QEMUIOVector *qiov, bool is_write,
602 BdrvRequestFlags flags)
603 {
604 Coroutine *co;
605 RwCo rwco = {
606 .child = child,
607 .offset = offset,
608 .qiov = qiov,
609 .is_write = is_write,
610 .ret = NOT_DONE,
611 .flags = flags,
612 };
613
614 if (qemu_in_coroutine()) {
615 /* Fast-path if already in coroutine context */
616 bdrv_rw_co_entry(&rwco);
617 } else {
618 co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco);
619 qemu_coroutine_enter(co);
620 BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE);
621 }
622 return rwco.ret;
623 }
624
625 /*
626 * Process a synchronous request using coroutines
627 */
628 static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf,
629 int nb_sectors, bool is_write, BdrvRequestFlags flags)
630 {
631 QEMUIOVector qiov;
632 struct iovec iov = {
633 .iov_base = (void *)buf,
634 .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
635 };
636
637 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
638 return -EINVAL;
639 }
640
641 qemu_iovec_init_external(&qiov, &iov, 1);
642 return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS,
643 &qiov, is_write, flags);
644 }
645
646 /* return < 0 if error. See bdrv_write() for the return codes */
647 int bdrv_read(BdrvChild *child, int64_t sector_num,
648 uint8_t *buf, int nb_sectors)
649 {
650 return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0);
651 }
652
653 /* Return < 0 if error. Important errors are:
654 -EIO generic I/O error (may happen for all errors)
655 -ENOMEDIUM No media inserted.
656 -EINVAL Invalid sector number or nb_sectors
657 -EACCES Trying to write a read-only device
658 */
659 int bdrv_write(BdrvChild *child, int64_t sector_num,
660 const uint8_t *buf, int nb_sectors)
661 {
662 return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
663 }
664
665 int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
666 int count, BdrvRequestFlags flags)
667 {
668 QEMUIOVector qiov;
669 struct iovec iov = {
670 .iov_base = NULL,
671 .iov_len = count,
672 };
673
674 qemu_iovec_init_external(&qiov, &iov, 1);
675 return bdrv_prwv_co(child, offset, &qiov, true,
676 BDRV_REQ_ZERO_WRITE | flags);
677 }
678
679 /*
680 * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
681 * The operation is sped up by checking the block status and only writing
682 * zeroes to the device if they currently do not return zeroes. Optional
683 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
684 * BDRV_REQ_FUA).
685 *
686 * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
687 */
688 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
689 {
690 int64_t target_sectors, ret, nb_sectors, sector_num = 0;
691 BlockDriverState *bs = child->bs;
692 BlockDriverState *file;
693 int n;
694
695 target_sectors = bdrv_nb_sectors(bs);
696 if (target_sectors < 0) {
697 return target_sectors;
698 }
699
700 for (;;) {
701 nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
702 if (nb_sectors <= 0) {
703 return 0;
704 }
705 ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
706 if (ret < 0) {
707 error_report("error getting block status at sector %" PRId64 ": %s",
708 sector_num, strerror(-ret));
709 return ret;
710 }
711 if (ret & BDRV_BLOCK_ZERO) {
712 sector_num += n;
713 continue;
714 }
715 ret = bdrv_pwrite_zeroes(child, sector_num << BDRV_SECTOR_BITS,
716 n << BDRV_SECTOR_BITS, flags);
717 if (ret < 0) {
718 error_report("error writing zeroes at sector %" PRId64 ": %s",
719 sector_num, strerror(-ret));
720 return ret;
721 }
722 sector_num += n;
723 }
724 }
725
726 int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
727 {
728 int ret;
729
730 ret = bdrv_prwv_co(child, offset, qiov, false, 0);
731 if (ret < 0) {
732 return ret;
733 }
734
735 return qiov->size;
736 }
737
738 int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes)
739 {
740 QEMUIOVector qiov;
741 struct iovec iov = {
742 .iov_base = (void *)buf,
743 .iov_len = bytes,
744 };
745
746 if (bytes < 0) {
747 return -EINVAL;
748 }
749
750 qemu_iovec_init_external(&qiov, &iov, 1);
751 return bdrv_preadv(child, offset, &qiov);
752 }
753
754 int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
755 {
756 int ret;
757
758 ret = bdrv_prwv_co(child, offset, qiov, true, 0);
759 if (ret < 0) {
760 return ret;
761 }
762
763 return qiov->size;
764 }
765
766 int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes)
767 {
768 QEMUIOVector qiov;
769 struct iovec iov = {
770 .iov_base = (void *) buf,
771 .iov_len = bytes,
772 };
773
774 if (bytes < 0) {
775 return -EINVAL;
776 }
777
778 qemu_iovec_init_external(&qiov, &iov, 1);
779 return bdrv_pwritev(child, offset, &qiov);
780 }
781
782 /*
783 * Writes to the file and ensures that no writes are reordered across this
784 * request (acts as a barrier)
785 *
786 * Returns 0 on success, -errno in error cases.
787 */
788 int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
789 const void *buf, int count)
790 {
791 int ret;
792
793 ret = bdrv_pwrite(child, offset, buf, count);
794 if (ret < 0) {
795 return ret;
796 }
797
798 ret = bdrv_flush(child->bs);
799 if (ret < 0) {
800 return ret;
801 }
802
803 return 0;
804 }
805
806 typedef struct CoroutineIOCompletion {
807 Coroutine *coroutine;
808 int ret;
809 } CoroutineIOCompletion;
810
811 static void bdrv_co_io_em_complete(void *opaque, int ret)
812 {
813 CoroutineIOCompletion *co = opaque;
814
815 co->ret = ret;
816 aio_co_wake(co->coroutine);
817 }
818
819 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
820 uint64_t offset, uint64_t bytes,
821 QEMUIOVector *qiov, int flags)
822 {
823 BlockDriver *drv = bs->drv;
824 int64_t sector_num;
825 unsigned int nb_sectors;
826
827 assert(!(flags & ~BDRV_REQ_MASK));
828
829 if (drv->bdrv_co_preadv) {
830 return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
831 }
832
833 sector_num = offset >> BDRV_SECTOR_BITS;
834 nb_sectors = bytes >> BDRV_SECTOR_BITS;
835
836 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
837 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
838 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
839
840 if (drv->bdrv_co_readv) {
841 return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
842 } else {
843 BlockAIOCB *acb;
844 CoroutineIOCompletion co = {
845 .coroutine = qemu_coroutine_self(),
846 };
847
848 acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
849 bdrv_co_io_em_complete, &co);
850 if (acb == NULL) {
851 return -EIO;
852 } else {
853 qemu_coroutine_yield();
854 return co.ret;
855 }
856 }
857 }
858
859 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
860 uint64_t offset, uint64_t bytes,
861 QEMUIOVector *qiov, int flags)
862 {
863 BlockDriver *drv = bs->drv;
864 int64_t sector_num;
865 unsigned int nb_sectors;
866 int ret;
867
868 assert(!(flags & ~BDRV_REQ_MASK));
869
870 if (drv->bdrv_co_pwritev) {
871 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
872 flags & bs->supported_write_flags);
873 flags &= ~bs->supported_write_flags;
874 goto emulate_flags;
875 }
876
877 sector_num = offset >> BDRV_SECTOR_BITS;
878 nb_sectors = bytes >> BDRV_SECTOR_BITS;
879
880 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
881 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
882 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
883
884 if (drv->bdrv_co_writev_flags) {
885 ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
886 flags & bs->supported_write_flags);
887 flags &= ~bs->supported_write_flags;
888 } else if (drv->bdrv_co_writev) {
889 assert(!bs->supported_write_flags);
890 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
891 } else {
892 BlockAIOCB *acb;
893 CoroutineIOCompletion co = {
894 .coroutine = qemu_coroutine_self(),
895 };
896
897 acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
898 bdrv_co_io_em_complete, &co);
899 if (acb == NULL) {
900 ret = -EIO;
901 } else {
902 qemu_coroutine_yield();
903 ret = co.ret;
904 }
905 }
906
907 emulate_flags:
908 if (ret == 0 && (flags & BDRV_REQ_FUA)) {
909 ret = bdrv_co_flush(bs);
910 }
911
912 return ret;
913 }
914
915 static int coroutine_fn
916 bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset,
917 uint64_t bytes, QEMUIOVector *qiov)
918 {
919 BlockDriver *drv = bs->drv;
920
921 if (!drv->bdrv_co_pwritev_compressed) {
922 return -ENOTSUP;
923 }
924
925 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
926 }
927
928 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
929 int64_t offset, unsigned int bytes, QEMUIOVector *qiov)
930 {
931 BlockDriverState *bs = child->bs;
932
933 /* Perform I/O through a temporary buffer so that users who scribble over
934 * their read buffer while the operation is in progress do not end up
935 * modifying the image file. This is critical for zero-copy guest I/O
936 * where anything might happen inside guest memory.
937 */
938 void *bounce_buffer;
939
940 BlockDriver *drv = bs->drv;
941 struct iovec iov;
942 QEMUIOVector bounce_qiov;
943 int64_t cluster_offset;
944 unsigned int cluster_bytes;
945 size_t skip_bytes;
946 int ret;
947
948 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
949
950 /* Cover entire cluster so no additional backing file I/O is required when
951 * allocating cluster in the image file.
952 */
953 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
954
955 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
956 cluster_offset, cluster_bytes);
957
958 iov.iov_len = cluster_bytes;
959 iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
960 if (bounce_buffer == NULL) {
961 ret = -ENOMEM;
962 goto err;
963 }
964
965 qemu_iovec_init_external(&bounce_qiov, &iov, 1);
966
967 ret = bdrv_driver_preadv(bs, cluster_offset, cluster_bytes,
968 &bounce_qiov, 0);
969 if (ret < 0) {
970 goto err;
971 }
972
973 if (drv->bdrv_co_pwrite_zeroes &&
974 buffer_is_zero(bounce_buffer, iov.iov_len)) {
975 /* FIXME: Should we (perhaps conditionally) be setting
976 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
977 * that still correctly reads as zero? */
978 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, cluster_bytes, 0);
979 } else {
980 /* This does not change the data on the disk, it is not necessary
981 * to flush even in cache=writethrough mode.
982 */
983 ret = bdrv_driver_pwritev(bs, cluster_offset, cluster_bytes,
984 &bounce_qiov, 0);
985 }
986
987 if (ret < 0) {
988 /* It might be okay to ignore write errors for guest requests. If this
989 * is a deliberate copy-on-read then we don't want to ignore the error.
990 * Simply report it in all cases.
991 */
992 goto err;
993 }
994
995 skip_bytes = offset - cluster_offset;
996 qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, bytes);
997
998 err:
999 qemu_vfree(bounce_buffer);
1000 return ret;
1001 }
1002
1003 /*
1004 * Forwards an already correctly aligned request to the BlockDriver. This
1005 * handles copy on read, zeroing after EOF, and fragmentation of large
1006 * reads; any other features must be implemented by the caller.
1007 */
1008 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
1009 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1010 int64_t align, QEMUIOVector *qiov, int flags)
1011 {
1012 BlockDriverState *bs = child->bs;
1013 int64_t total_bytes, max_bytes;
1014 int ret = 0;
1015 uint64_t bytes_remaining = bytes;
1016 int max_transfer;
1017
1018 assert(is_power_of_2(align));
1019 assert((offset & (align - 1)) == 0);
1020 assert((bytes & (align - 1)) == 0);
1021 assert(!qiov || bytes == qiov->size);
1022 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1023 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1024 align);
1025
1026 /* TODO: We would need a per-BDS .supported_read_flags and
1027 * potential fallback support, if we ever implement any read flags
1028 * to pass through to drivers. For now, there aren't any
1029 * passthrough flags. */
1030 assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ)));
1031
1032 /* Handle Copy on Read and associated serialisation */
1033 if (flags & BDRV_REQ_COPY_ON_READ) {
1034 /* If we touch the same cluster it counts as an overlap. This
1035 * guarantees that allocating writes will be serialized and not race
1036 * with each other for the same cluster. For example, in copy-on-read
1037 * it ensures that the CoR read and write operations are atomic and
1038 * guest writes cannot interleave between them. */
1039 mark_request_serialising(req, bdrv_get_cluster_size(bs));
1040 }
1041
1042 if (!(flags & BDRV_REQ_NO_SERIALISING)) {
1043 wait_serialising_requests(req);
1044 }
1045
1046 if (flags & BDRV_REQ_COPY_ON_READ) {
1047 int64_t start_sector = offset >> BDRV_SECTOR_BITS;
1048 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1049 unsigned int nb_sectors = end_sector - start_sector;
1050 int pnum;
1051
1052 ret = bdrv_is_allocated(bs, start_sector, nb_sectors, &pnum);
1053 if (ret < 0) {
1054 goto out;
1055 }
1056
1057 if (!ret || pnum != nb_sectors) {
1058 ret = bdrv_co_do_copy_on_readv(child, offset, bytes, qiov);
1059 goto out;
1060 }
1061 }
1062
1063 /* Forward the request to the BlockDriver, possibly fragmenting it */
1064 total_bytes = bdrv_getlength(bs);
1065 if (total_bytes < 0) {
1066 ret = total_bytes;
1067 goto out;
1068 }
1069
1070 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
1071 if (bytes <= max_bytes && bytes <= max_transfer) {
1072 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
1073 goto out;
1074 }
1075
1076 while (bytes_remaining) {
1077 int num;
1078
1079 if (max_bytes) {
1080 QEMUIOVector local_qiov;
1081
1082 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
1083 assert(num);
1084 qemu_iovec_init(&local_qiov, qiov->niov);
1085 qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);
1086
1087 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1088 num, &local_qiov, 0);
1089 max_bytes -= num;
1090 qemu_iovec_destroy(&local_qiov);
1091 } else {
1092 num = bytes_remaining;
1093 ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0,
1094 bytes_remaining);
1095 }
1096 if (ret < 0) {
1097 goto out;
1098 }
1099 bytes_remaining -= num;
1100 }
1101
1102 out:
1103 return ret < 0 ? ret : 0;
1104 }
1105
1106 /*
1107 * Handle a read request in coroutine context
1108 */
1109 int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1110 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1111 BdrvRequestFlags flags)
1112 {
1113 BlockDriverState *bs = child->bs;
1114 BlockDriver *drv = bs->drv;
1115 BdrvTrackedRequest req;
1116
1117 uint64_t align = bs->bl.request_alignment;
1118 uint8_t *head_buf = NULL;
1119 uint8_t *tail_buf = NULL;
1120 QEMUIOVector local_qiov;
1121 bool use_local_qiov = false;
1122 int ret;
1123
1124 if (!drv) {
1125 return -ENOMEDIUM;
1126 }
1127
1128 ret = bdrv_check_byte_request(bs, offset, bytes);
1129 if (ret < 0) {
1130 return ret;
1131 }
1132
1133 bdrv_inc_in_flight(bs);
1134
1135 /* Don't do copy-on-read if we read data before write operation */
1136 if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
1137 flags |= BDRV_REQ_COPY_ON_READ;
1138 }
1139
1140 /* Align read if necessary by padding qiov */
1141 if (offset & (align - 1)) {
1142 head_buf = qemu_blockalign(bs, align);
1143 qemu_iovec_init(&local_qiov, qiov->niov + 2);
1144 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1145 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1146 use_local_qiov = true;
1147
1148 bytes += offset & (align - 1);
1149 offset = offset & ~(align - 1);
1150 }
1151
1152 if ((offset + bytes) & (align - 1)) {
1153 if (!use_local_qiov) {
1154 qemu_iovec_init(&local_qiov, qiov->niov + 1);
1155 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1156 use_local_qiov = true;
1157 }
1158 tail_buf = qemu_blockalign(bs, align);
1159 qemu_iovec_add(&local_qiov, tail_buf,
1160 align - ((offset + bytes) & (align - 1)));
1161
1162 bytes = ROUND_UP(bytes, align);
1163 }
1164
1165 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1166 ret = bdrv_aligned_preadv(child, &req, offset, bytes, align,
1167 use_local_qiov ? &local_qiov : qiov,
1168 flags);
1169 tracked_request_end(&req);
1170 bdrv_dec_in_flight(bs);
1171
1172 if (use_local_qiov) {
1173 qemu_iovec_destroy(&local_qiov);
1174 qemu_vfree(head_buf);
1175 qemu_vfree(tail_buf);
1176 }
1177
1178 return ret;
1179 }
1180
1181 static int coroutine_fn bdrv_co_do_readv(BdrvChild *child,
1182 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1183 BdrvRequestFlags flags)
1184 {
1185 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1186 return -EINVAL;
1187 }
1188
1189 return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS,
1190 nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1191 }
1192
1193 int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num,
1194 int nb_sectors, QEMUIOVector *qiov)
1195 {
1196 trace_bdrv_co_readv(child->bs, sector_num, nb_sectors);
1197
1198 return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0);
1199 }
1200
1201 /* Maximum buffer for write zeroes fallback, in bytes */
1202 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
1203
1204 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1205 int64_t offset, int count, BdrvRequestFlags flags)
1206 {
1207 BlockDriver *drv = bs->drv;
1208 QEMUIOVector qiov;
1209 struct iovec iov = {0};
1210 int ret = 0;
1211 bool need_flush = false;
1212 int head = 0;
1213 int tail = 0;
1214
1215 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX);
1216 int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1217 bs->bl.request_alignment);
1218 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1219 MAX_WRITE_ZEROES_BOUNCE_BUFFER);
1220
1221 assert(alignment % bs->bl.request_alignment == 0);
1222 head = offset % alignment;
1223 tail = (offset + count) % alignment;
1224 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1225 assert(max_write_zeroes >= bs->bl.request_alignment);
1226
1227 while (count > 0 && !ret) {
1228 int num = count;
1229
1230 /* Align request. Block drivers can expect the "bulk" of the request
1231 * to be aligned, and that unaligned requests do not cross cluster
1232 * boundaries.
1233 */
1234 if (head) {
1235 /* Make a small request up to the first aligned sector. For
1236 * convenience, limit this request to max_transfer even if
1237 * we don't need to fall back to writes. */
1238 num = MIN(MIN(count, max_transfer), alignment - head);
1239 head = (head + num) % alignment;
1240 assert(num < max_write_zeroes);
1241 } else if (tail && num > alignment) {
1242 /* Shorten the request to the last aligned sector. */
1243 num -= tail;
1244 }
1245
1246 /* limit request size */
1247 if (num > max_write_zeroes) {
1248 num = max_write_zeroes;
1249 }
1250
1251 ret = -ENOTSUP;
1252 /* First try the efficient write zeroes operation */
1253 if (drv->bdrv_co_pwrite_zeroes) {
1254 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1255 flags & bs->supported_zero_flags);
1256 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1257 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1258 need_flush = true;
1259 }
1260 } else {
1261 assert(!bs->supported_zero_flags);
1262 }
1263
1264 if (ret == -ENOTSUP) {
1265 /* Fall back to bounce buffer if write zeroes is unsupported */
1266 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1267
1268 if ((flags & BDRV_REQ_FUA) &&
1269 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1270 /* No need for bdrv_driver_pwrite() to do a fallback
1271 * flush on each chunk; use just one at the end */
1272 write_flags &= ~BDRV_REQ_FUA;
1273 need_flush = true;
1274 }
1275 num = MIN(num, max_transfer);
1276 iov.iov_len = num;
1277 if (iov.iov_base == NULL) {
1278 iov.iov_base = qemu_try_blockalign(bs, num);
1279 if (iov.iov_base == NULL) {
1280 ret = -ENOMEM;
1281 goto fail;
1282 }
1283 memset(iov.iov_base, 0, num);
1284 }
1285 qemu_iovec_init_external(&qiov, &iov, 1);
1286
1287 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags);
1288
1289 /* Keep bounce buffer around if it is big enough for all
1290 * all future requests.
1291 */
1292 if (num < max_transfer) {
1293 qemu_vfree(iov.iov_base);
1294 iov.iov_base = NULL;
1295 }
1296 }
1297
1298 offset += num;
1299 count -= num;
1300 }
1301
1302 fail:
1303 if (ret == 0 && need_flush) {
1304 ret = bdrv_co_flush(bs);
1305 }
1306 qemu_vfree(iov.iov_base);
1307 return ret;
1308 }
1309
1310 /*
1311 * Forwards an already correctly aligned write request to the BlockDriver,
1312 * after possibly fragmenting it.
1313 */
1314 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
1315 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1316 int64_t align, QEMUIOVector *qiov, int flags)
1317 {
1318 BlockDriverState *bs = child->bs;
1319 BlockDriver *drv = bs->drv;
1320 bool waited;
1321 int ret;
1322
1323 int64_t start_sector = offset >> BDRV_SECTOR_BITS;
1324 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1325 uint64_t bytes_remaining = bytes;
1326 int max_transfer;
1327
1328 assert(is_power_of_2(align));
1329 assert((offset & (align - 1)) == 0);
1330 assert((bytes & (align - 1)) == 0);
1331 assert(!qiov || bytes == qiov->size);
1332 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1333 assert(!(flags & ~BDRV_REQ_MASK));
1334 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1335 align);
1336
1337 waited = wait_serialising_requests(req);
1338 assert(!waited || !req->serialising);
1339 assert(req->overlap_offset <= offset);
1340 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1341 assert(child->perm & BLK_PERM_WRITE);
1342 assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE);
1343
1344 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
1345
1346 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1347 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
1348 qemu_iovec_is_zero(qiov)) {
1349 flags |= BDRV_REQ_ZERO_WRITE;
1350 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1351 flags |= BDRV_REQ_MAY_UNMAP;
1352 }
1353 }
1354
1355 if (ret < 0) {
1356 /* Do nothing, write notifier decided to fail this request */
1357 } else if (flags & BDRV_REQ_ZERO_WRITE) {
1358 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1359 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
1360 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
1361 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov);
1362 } else if (bytes <= max_transfer) {
1363 bdrv_debug_event(bs, BLKDBG_PWRITEV);
1364 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
1365 } else {
1366 bdrv_debug_event(bs, BLKDBG_PWRITEV);
1367 while (bytes_remaining) {
1368 int num = MIN(bytes_remaining, max_transfer);
1369 QEMUIOVector local_qiov;
1370 int local_flags = flags;
1371
1372 assert(num);
1373 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
1374 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1375 /* If FUA is going to be emulated by flush, we only
1376 * need to flush on the last iteration */
1377 local_flags &= ~BDRV_REQ_FUA;
1378 }
1379 qemu_iovec_init(&local_qiov, qiov->niov);
1380 qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);
1381
1382 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
1383 num, &local_qiov, local_flags);
1384 qemu_iovec_destroy(&local_qiov);
1385 if (ret < 0) {
1386 break;
1387 }
1388 bytes_remaining -= num;
1389 }
1390 }
1391 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1392
1393 ++bs->write_gen;
1394 bdrv_set_dirty(bs, start_sector, end_sector - start_sector);
1395
1396 if (bs->wr_highest_offset < offset + bytes) {
1397 bs->wr_highest_offset = offset + bytes;
1398 }
1399
1400 if (ret >= 0) {
1401 bs->total_sectors = MAX(bs->total_sectors, end_sector);
1402 ret = 0;
1403 }
1404
1405 return ret;
1406 }
1407
1408 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
1409 int64_t offset,
1410 unsigned int bytes,
1411 BdrvRequestFlags flags,
1412 BdrvTrackedRequest *req)
1413 {
1414 BlockDriverState *bs = child->bs;
1415 uint8_t *buf = NULL;
1416 QEMUIOVector local_qiov;
1417 struct iovec iov;
1418 uint64_t align = bs->bl.request_alignment;
1419 unsigned int head_padding_bytes, tail_padding_bytes;
1420 int ret = 0;
1421
1422 head_padding_bytes = offset & (align - 1);
1423 tail_padding_bytes = align - ((offset + bytes) & (align - 1));
1424
1425
1426 assert(flags & BDRV_REQ_ZERO_WRITE);
1427 if (head_padding_bytes || tail_padding_bytes) {
1428 buf = qemu_blockalign(bs, align);
1429 iov = (struct iovec) {
1430 .iov_base = buf,
1431 .iov_len = align,
1432 };
1433 qemu_iovec_init_external(&local_qiov, &iov, 1);
1434 }
1435 if (head_padding_bytes) {
1436 uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
1437
1438 /* RMW the unaligned part before head. */
1439 mark_request_serialising(req, align);
1440 wait_serialising_requests(req);
1441 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1442 ret = bdrv_aligned_preadv(child, req, offset & ~(align - 1), align,
1443 align, &local_qiov, 0);
1444 if (ret < 0) {
1445 goto fail;
1446 }
1447 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1448
1449 memset(buf + head_padding_bytes, 0, zero_bytes);
1450 ret = bdrv_aligned_pwritev(child, req, offset & ~(align - 1), align,
1451 align, &local_qiov,
1452 flags & ~BDRV_REQ_ZERO_WRITE);
1453 if (ret < 0) {
1454 goto fail;
1455 }
1456 offset += zero_bytes;
1457 bytes -= zero_bytes;
1458 }
1459
1460 assert(!bytes || (offset & (align - 1)) == 0);
1461 if (bytes >= align) {
1462 /* Write the aligned part in the middle. */
1463 uint64_t aligned_bytes = bytes & ~(align - 1);
1464 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
1465 NULL, flags);
1466 if (ret < 0) {
1467 goto fail;
1468 }
1469 bytes -= aligned_bytes;
1470 offset += aligned_bytes;
1471 }
1472
1473 assert(!bytes || (offset & (align - 1)) == 0);
1474 if (bytes) {
1475 assert(align == tail_padding_bytes + bytes);
1476 /* RMW the unaligned part after tail. */
1477 mark_request_serialising(req, align);
1478 wait_serialising_requests(req);
1479 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1480 ret = bdrv_aligned_preadv(child, req, offset, align,
1481 align, &local_qiov, 0);
1482 if (ret < 0) {
1483 goto fail;
1484 }
1485 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1486
1487 memset(buf, 0, bytes);
1488 ret = bdrv_aligned_pwritev(child, req, offset, align, align,
1489 &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
1490 }
1491 fail:
1492 qemu_vfree(buf);
1493 return ret;
1494
1495 }
1496
1497 /*
1498 * Handle a write request in coroutine context
1499 */
1500 int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
1501 int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1502 BdrvRequestFlags flags)
1503 {
1504 BlockDriverState *bs = child->bs;
1505 BdrvTrackedRequest req;
1506 uint64_t align = bs->bl.request_alignment;
1507 uint8_t *head_buf = NULL;
1508 uint8_t *tail_buf = NULL;
1509 QEMUIOVector local_qiov;
1510 bool use_local_qiov = false;
1511 int ret;
1512
1513 if (!bs->drv) {
1514 return -ENOMEDIUM;
1515 }
1516 if (bs->read_only) {
1517 return -EPERM;
1518 }
1519 assert(!(bs->open_flags & BDRV_O_INACTIVE));
1520
1521 ret = bdrv_check_byte_request(bs, offset, bytes);
1522 if (ret < 0) {
1523 return ret;
1524 }
1525
1526 bdrv_inc_in_flight(bs);
1527 /*
1528 * Align write if necessary by performing a read-modify-write cycle.
1529 * Pad qiov with the read parts and be sure to have a tracked request not
1530 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
1531 */
1532 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
1533
1534 if (!qiov) {
1535 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
1536 goto out;
1537 }
1538
1539 if (offset & (align - 1)) {
1540 QEMUIOVector head_qiov;
1541 struct iovec head_iov;
1542
1543 mark_request_serialising(&req, align);
1544 wait_serialising_requests(&req);
1545
1546 head_buf = qemu_blockalign(bs, align);
1547 head_iov = (struct iovec) {
1548 .iov_base = head_buf,
1549 .iov_len = align,
1550 };
1551 qemu_iovec_init_external(&head_qiov, &head_iov, 1);
1552
1553 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1554 ret = bdrv_aligned_preadv(child, &req, offset & ~(align - 1), align,
1555 align, &head_qiov, 0);
1556 if (ret < 0) {
1557 goto fail;
1558 }
1559 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1560
1561 qemu_iovec_init(&local_qiov, qiov->niov + 2);
1562 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1563 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1564 use_local_qiov = true;
1565
1566 bytes += offset & (align - 1);
1567 offset = offset & ~(align - 1);
1568
1569 /* We have read the tail already if the request is smaller
1570 * than one aligned block.
1571 */
1572 if (bytes < align) {
1573 qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes);
1574 bytes = align;
1575 }
1576 }
1577
1578 if ((offset + bytes) & (align - 1)) {
1579 QEMUIOVector tail_qiov;
1580 struct iovec tail_iov;
1581 size_t tail_bytes;
1582 bool waited;
1583
1584 mark_request_serialising(&req, align);
1585 waited = wait_serialising_requests(&req);
1586 assert(!waited || !use_local_qiov);
1587
1588 tail_buf = qemu_blockalign(bs, align);
1589 tail_iov = (struct iovec) {
1590 .iov_base = tail_buf,
1591 .iov_len = align,
1592 };
1593 qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
1594
1595 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1596 ret = bdrv_aligned_preadv(child, &req, (offset + bytes) & ~(align - 1),
1597 align, align, &tail_qiov, 0);
1598 if (ret < 0) {
1599 goto fail;
1600 }
1601 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1602
1603 if (!use_local_qiov) {
1604 qemu_iovec_init(&local_qiov, qiov->niov + 1);
1605 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1606 use_local_qiov = true;
1607 }
1608
1609 tail_bytes = (offset + bytes) & (align - 1);
1610 qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
1611
1612 bytes = ROUND_UP(bytes, align);
1613 }
1614
1615 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
1616 use_local_qiov ? &local_qiov : qiov,
1617 flags);
1618
1619 fail:
1620
1621 if (use_local_qiov) {
1622 qemu_iovec_destroy(&local_qiov);
1623 }
1624 qemu_vfree(head_buf);
1625 qemu_vfree(tail_buf);
1626 out:
1627 tracked_request_end(&req);
1628 bdrv_dec_in_flight(bs);
1629 return ret;
1630 }
1631
1632 static int coroutine_fn bdrv_co_do_writev(BdrvChild *child,
1633 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1634 BdrvRequestFlags flags)
1635 {
1636 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1637 return -EINVAL;
1638 }
1639
1640 return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS,
1641 nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1642 }
1643
1644 int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num,
1645 int nb_sectors, QEMUIOVector *qiov)
1646 {
1647 trace_bdrv_co_writev(child->bs, sector_num, nb_sectors);
1648
1649 return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0);
1650 }
1651
1652 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
1653 int count, BdrvRequestFlags flags)
1654 {
1655 trace_bdrv_co_pwrite_zeroes(child->bs, offset, count, flags);
1656
1657 if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
1658 flags &= ~BDRV_REQ_MAY_UNMAP;
1659 }
1660
1661 return bdrv_co_pwritev(child, offset, count, NULL,
1662 BDRV_REQ_ZERO_WRITE | flags);
1663 }
1664
1665 /*
1666 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
1667 */
1668 int bdrv_flush_all(void)
1669 {
1670 BdrvNextIterator it;
1671 BlockDriverState *bs = NULL;
1672 int result = 0;
1673
1674 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
1675 AioContext *aio_context = bdrv_get_aio_context(bs);
1676 int ret;
1677
1678 aio_context_acquire(aio_context);
1679 ret = bdrv_flush(bs);
1680 if (ret < 0 && !result) {
1681 result = ret;
1682 }
1683 aio_context_release(aio_context);
1684 }
1685
1686 return result;
1687 }
1688
1689
1690 typedef struct BdrvCoGetBlockStatusData {
1691 BlockDriverState *bs;
1692 BlockDriverState *base;
1693 BlockDriverState **file;
1694 int64_t sector_num;
1695 int nb_sectors;
1696 int *pnum;
1697 int64_t ret;
1698 bool done;
1699 } BdrvCoGetBlockStatusData;
1700
1701 /*
1702 * Returns the allocation status of the specified sectors.
1703 * Drivers not implementing the functionality are assumed to not support
1704 * backing files, hence all their sectors are reported as allocated.
1705 *
1706 * If 'sector_num' is beyond the end of the disk image the return value is 0
1707 * and 'pnum' is set to 0.
1708 *
1709 * 'pnum' is set to the number of sectors (including and immediately following
1710 * the specified sector) that are known to be in the same
1711 * allocated/unallocated state.
1712 *
1713 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes
1714 * beyond the end of the disk image it will be clamped.
1715 *
1716 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
1717 * points to the BDS which the sector range is allocated in.
1718 */
1719 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
1720 int64_t sector_num,
1721 int nb_sectors, int *pnum,
1722 BlockDriverState **file)
1723 {
1724 int64_t total_sectors;
1725 int64_t n;
1726 int64_t ret, ret2;
1727
1728 total_sectors = bdrv_nb_sectors(bs);
1729 if (total_sectors < 0) {
1730 return total_sectors;
1731 }
1732
1733 if (sector_num >= total_sectors) {
1734 *pnum = 0;
1735 return 0;
1736 }
1737
1738 n = total_sectors - sector_num;
1739 if (n < nb_sectors) {
1740 nb_sectors = n;
1741 }
1742
1743 if (!bs->drv->bdrv_co_get_block_status) {
1744 *pnum = nb_sectors;
1745 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
1746 if (bs->drv->protocol_name) {
1747 ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
1748 }
1749 return ret;
1750 }
1751
1752 *file = NULL;
1753 bdrv_inc_in_flight(bs);
1754 ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
1755 file);
1756 if (ret < 0) {
1757 *pnum = 0;
1758 goto out;
1759 }
1760
1761 if (ret & BDRV_BLOCK_RAW) {
1762 assert(ret & BDRV_BLOCK_OFFSET_VALID);
1763 ret = bdrv_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
1764 *pnum, pnum, file);
1765 goto out;
1766 }
1767
1768 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
1769 ret |= BDRV_BLOCK_ALLOCATED;
1770 } else {
1771 if (bdrv_unallocated_blocks_are_zero(bs)) {
1772 ret |= BDRV_BLOCK_ZERO;
1773 } else if (bs->backing) {
1774 BlockDriverState *bs2 = bs->backing->bs;
1775 int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
1776 if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
1777 ret |= BDRV_BLOCK_ZERO;
1778 }
1779 }
1780 }
1781
1782 if (*file && *file != bs &&
1783 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
1784 (ret & BDRV_BLOCK_OFFSET_VALID)) {
1785 BlockDriverState *file2;
1786 int file_pnum;
1787
1788 ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
1789 *pnum, &file_pnum, &file2);
1790 if (ret2 >= 0) {
1791 /* Ignore errors. This is just providing extra information, it
1792 * is useful but not necessary.
1793 */
1794 if (!file_pnum) {
1795 /* !file_pnum indicates an offset at or beyond the EOF; it is
1796 * perfectly valid for the format block driver to point to such
1797 * offsets, so catch it and mark everything as zero */
1798 ret |= BDRV_BLOCK_ZERO;
1799 } else {
1800 /* Limit request to the range reported by the protocol driver */
1801 *pnum = file_pnum;
1802 ret |= (ret2 & BDRV_BLOCK_ZERO);
1803 }
1804 }
1805 }
1806
1807 out:
1808 bdrv_dec_in_flight(bs);
1809 return ret;
1810 }
1811
1812 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
1813 BlockDriverState *base,
1814 int64_t sector_num,
1815 int nb_sectors,
1816 int *pnum,
1817 BlockDriverState **file)
1818 {
1819 BlockDriverState *p;
1820 int64_t ret = 0;
1821
1822 assert(bs != base);
1823 for (p = bs; p != base; p = backing_bs(p)) {
1824 ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
1825 if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
1826 break;
1827 }
1828 /* [sector_num, pnum] unallocated on this layer, which could be only
1829 * the first part of [sector_num, nb_sectors]. */
1830 nb_sectors = MIN(nb_sectors, *pnum);
1831 }
1832 return ret;
1833 }
1834
1835 /* Coroutine wrapper for bdrv_get_block_status_above() */
1836 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
1837 {
1838 BdrvCoGetBlockStatusData *data = opaque;
1839
1840 data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
1841 data->sector_num,
1842 data->nb_sectors,
1843 data->pnum,
1844 data->file);
1845 data->done = true;
1846 }
1847
1848 /*
1849 * Synchronous wrapper around bdrv_co_get_block_status_above().
1850 *
1851 * See bdrv_co_get_block_status_above() for details.
1852 */
1853 int64_t bdrv_get_block_status_above(BlockDriverState *bs,
1854 BlockDriverState *base,
1855 int64_t sector_num,
1856 int nb_sectors, int *pnum,
1857 BlockDriverState **file)
1858 {
1859 Coroutine *co;
1860 BdrvCoGetBlockStatusData data = {
1861 .bs = bs,
1862 .base = base,
1863 .file = file,
1864 .sector_num = sector_num,
1865 .nb_sectors = nb_sectors,
1866 .pnum = pnum,
1867 .done = false,
1868 };
1869
1870 if (qemu_in_coroutine()) {
1871 /* Fast-path if already in coroutine context */
1872 bdrv_get_block_status_above_co_entry(&data);
1873 } else {
1874 co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry,
1875 &data);
1876 qemu_coroutine_enter(co);
1877 BDRV_POLL_WHILE(bs, !data.done);
1878 }
1879 return data.ret;
1880 }
1881
1882 int64_t bdrv_get_block_status(BlockDriverState *bs,
1883 int64_t sector_num,
1884 int nb_sectors, int *pnum,
1885 BlockDriverState **file)
1886 {
1887 return bdrv_get_block_status_above(bs, backing_bs(bs),
1888 sector_num, nb_sectors, pnum, file);
1889 }
1890
1891 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
1892 int nb_sectors, int *pnum)
1893 {
1894 BlockDriverState *file;
1895 int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
1896 &file);
1897 if (ret < 0) {
1898 return ret;
1899 }
1900 return !!(ret & BDRV_BLOCK_ALLOCATED);
1901 }
1902
1903 /*
1904 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
1905 *
1906 * Return true if the given sector is allocated in any image between
1907 * BASE and TOP (inclusive). BASE can be NULL to check if the given
1908 * sector is allocated in any image of the chain. Return false otherwise.
1909 *
1910 * 'pnum' is set to the number of sectors (including and immediately following
1911 * the specified sector) that are known to be in the same
1912 * allocated/unallocated state.
1913 *
1914 */
1915 int bdrv_is_allocated_above(BlockDriverState *top,
1916 BlockDriverState *base,
1917 int64_t sector_num,
1918 int nb_sectors, int *pnum)
1919 {
1920 BlockDriverState *intermediate;
1921 int ret, n = nb_sectors;
1922
1923 intermediate = top;
1924 while (intermediate && intermediate != base) {
1925 int pnum_inter;
1926 ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
1927 &pnum_inter);
1928 if (ret < 0) {
1929 return ret;
1930 } else if (ret) {
1931 *pnum = pnum_inter;
1932 return 1;
1933 }
1934
1935 /*
1936 * [sector_num, nb_sectors] is unallocated on top but intermediate
1937 * might have
1938 *
1939 * [sector_num+x, nr_sectors] allocated.
1940 */
1941 if (n > pnum_inter &&
1942 (intermediate == top ||
1943 sector_num + pnum_inter < intermediate->total_sectors)) {
1944 n = pnum_inter;
1945 }
1946
1947 intermediate = backing_bs(intermediate);
1948 }
1949
1950 *pnum = n;
1951 return 0;
1952 }
1953
1954 typedef struct BdrvVmstateCo {
1955 BlockDriverState *bs;
1956 QEMUIOVector *qiov;
1957 int64_t pos;
1958 bool is_read;
1959 int ret;
1960 } BdrvVmstateCo;
1961
1962 static int coroutine_fn
1963 bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
1964 bool is_read)
1965 {
1966 BlockDriver *drv = bs->drv;
1967
1968 if (!drv) {
1969 return -ENOMEDIUM;
1970 } else if (drv->bdrv_load_vmstate) {
1971 return is_read ? drv->bdrv_load_vmstate(bs, qiov, pos)
1972 : drv->bdrv_save_vmstate(bs, qiov, pos);
1973 } else if (bs->file) {
1974 return bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read);
1975 }
1976
1977 return -ENOTSUP;
1978 }
1979
1980 static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque)
1981 {
1982 BdrvVmstateCo *co = opaque;
1983 co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read);
1984 }
1985
1986 static inline int
1987 bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
1988 bool is_read)
1989 {
1990 if (qemu_in_coroutine()) {
1991 return bdrv_co_rw_vmstate(bs, qiov, pos, is_read);
1992 } else {
1993 BdrvVmstateCo data = {
1994 .bs = bs,
1995 .qiov = qiov,
1996 .pos = pos,
1997 .is_read = is_read,
1998 .ret = -EINPROGRESS,
1999 };
2000 Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data);
2001
2002 qemu_coroutine_enter(co);
2003 while (data.ret == -EINPROGRESS) {
2004 aio_poll(bdrv_get_aio_context(bs), true);
2005 }
2006 return data.ret;
2007 }
2008 }
2009
2010 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
2011 int64_t pos, int size)
2012 {
2013 QEMUIOVector qiov;
2014 struct iovec iov = {
2015 .iov_base = (void *) buf,
2016 .iov_len = size,
2017 };
2018 int ret;
2019
2020 qemu_iovec_init_external(&qiov, &iov, 1);
2021
2022 ret = bdrv_writev_vmstate(bs, &qiov, pos);
2023 if (ret < 0) {
2024 return ret;
2025 }
2026
2027 return size;
2028 }
2029
2030 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2031 {
2032 return bdrv_rw_vmstate(bs, qiov, pos, false);
2033 }
2034
2035 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
2036 int64_t pos, int size)
2037 {
2038 QEMUIOVector qiov;
2039 struct iovec iov = {
2040 .iov_base = buf,
2041 .iov_len = size,
2042 };
2043 int ret;
2044
2045 qemu_iovec_init_external(&qiov, &iov, 1);
2046 ret = bdrv_readv_vmstate(bs, &qiov, pos);
2047 if (ret < 0) {
2048 return ret;
2049 }
2050
2051 return size;
2052 }
2053
2054 int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2055 {
2056 return bdrv_rw_vmstate(bs, qiov, pos, true);
2057 }
2058
2059 /**************************************************************/
2060 /* async I/Os */
2061
2062 BlockAIOCB *bdrv_aio_readv(BdrvChild *child, int64_t sector_num,
2063 QEMUIOVector *qiov, int nb_sectors,
2064 BlockCompletionFunc *cb, void *opaque)
2065 {
2066 trace_bdrv_aio_readv(child->bs, sector_num, nb_sectors, opaque);
2067
2068 assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size);
2069 return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov,
2070 0, cb, opaque, false);
2071 }
2072
2073 BlockAIOCB *bdrv_aio_writev(BdrvChild *child, int64_t sector_num,
2074 QEMUIOVector *qiov, int nb_sectors,
2075 BlockCompletionFunc *cb, void *opaque)
2076 {
2077 trace_bdrv_aio_writev(child->bs, sector_num, nb_sectors, opaque);
2078
2079 assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size);
2080 return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov,
2081 0, cb, opaque, true);
2082 }
2083
2084 void bdrv_aio_cancel(BlockAIOCB *acb)
2085 {
2086 qemu_aio_ref(acb);
2087 bdrv_aio_cancel_async(acb);
2088 while (acb->refcnt > 1) {
2089 if (acb->aiocb_info->get_aio_context) {
2090 aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2091 } else if (acb->bs) {
2092 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
2093 * assert that we're not using an I/O thread. Thread-safe
2094 * code should use bdrv_aio_cancel_async exclusively.
2095 */
2096 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
2097 aio_poll(bdrv_get_aio_context(acb->bs), true);
2098 } else {
2099 abort();
2100 }
2101 }
2102 qemu_aio_unref(acb);
2103 }
2104
2105 /* Async version of aio cancel. The caller is not blocked if the acb implements
2106 * cancel_async, otherwise we do nothing and let the request normally complete.
2107 * In either case the completion callback must be called. */
2108 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2109 {
2110 if (acb->aiocb_info->cancel_async) {
2111 acb->aiocb_info->cancel_async(acb);
2112 }
2113 }
2114
2115 /**************************************************************/
2116 /* async block device emulation */
2117
2118 typedef struct BlockRequest {
2119 union {
2120 /* Used during read, write, trim */
2121 struct {
2122 int64_t offset;
2123 int bytes;
2124 int flags;
2125 QEMUIOVector *qiov;
2126 };
2127 /* Used during ioctl */
2128 struct {
2129 int req;
2130 void *buf;
2131 };
2132 };
2133 BlockCompletionFunc *cb;
2134 void *opaque;
2135
2136 int error;
2137 } BlockRequest;
2138
2139 typedef struct BlockAIOCBCoroutine {
2140 BlockAIOCB common;
2141 BdrvChild *child;
2142 BlockRequest req;
2143 bool is_write;
2144 bool need_bh;
2145 bool *done;
2146 } BlockAIOCBCoroutine;
2147
2148 static const AIOCBInfo bdrv_em_co_aiocb_info = {
2149 .aiocb_size = sizeof(BlockAIOCBCoroutine),
2150 };
2151
2152 static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
2153 {
2154 if (!acb->need_bh) {
2155 bdrv_dec_in_flight(acb->common.bs);
2156 acb->common.cb(acb->common.opaque, acb->req.error);
2157 qemu_aio_unref(acb);
2158 }
2159 }
2160
2161 static void bdrv_co_em_bh(void *opaque)
2162 {
2163 BlockAIOCBCoroutine *acb = opaque;
2164
2165 assert(!acb->need_bh);
2166 bdrv_co_complete(acb);
2167 }
2168
2169 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
2170 {
2171 acb->need_bh = false;
2172 if (acb->req.error != -EINPROGRESS) {
2173 BlockDriverState *bs = acb->common.bs;
2174
2175 aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
2176 }
2177 }
2178
2179 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
2180 static void coroutine_fn bdrv_co_do_rw(void *opaque)
2181 {
2182 BlockAIOCBCoroutine *acb = opaque;
2183
2184 if (!acb->is_write) {
2185 acb->req.error = bdrv_co_preadv(acb->child, acb->req.offset,
2186 acb->req.qiov->size, acb->req.qiov, acb->req.flags);
2187 } else {
2188 acb->req.error = bdrv_co_pwritev(acb->child, acb->req.offset,
2189 acb->req.qiov->size, acb->req.qiov, acb->req.flags);
2190 }
2191
2192 bdrv_co_complete(acb);
2193 }
2194
2195 static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child,
2196 int64_t offset,
2197 QEMUIOVector *qiov,
2198 BdrvRequestFlags flags,
2199 BlockCompletionFunc *cb,
2200 void *opaque,
2201 bool is_write)
2202 {
2203 Coroutine *co;
2204 BlockAIOCBCoroutine *acb;
2205
2206 /* Matched by bdrv_co_complete's bdrv_dec_in_flight. */
2207 bdrv_inc_in_flight(child->bs);
2208
2209 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, child->bs, cb, opaque);
2210 acb->child = child;
2211 acb->need_bh = true;
2212 acb->req.error = -EINPROGRESS;
2213 acb->req.offset = offset;
2214 acb->req.qiov = qiov;
2215 acb->req.flags = flags;
2216 acb->is_write = is_write;
2217
2218 co = qemu_coroutine_create(bdrv_co_do_rw, acb);
2219 qemu_coroutine_enter(co);
2220
2221 bdrv_co_maybe_schedule_bh(acb);
2222 return &acb->common;
2223 }
2224
2225 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
2226 {
2227 BlockAIOCBCoroutine *acb = opaque;
2228 BlockDriverState *bs = acb->common.bs;
2229
2230 acb->req.error = bdrv_co_flush(bs);
2231 bdrv_co_complete(acb);
2232 }
2233
2234 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
2235 BlockCompletionFunc *cb, void *opaque)
2236 {
2237 trace_bdrv_aio_flush(bs, opaque);
2238
2239 Coroutine *co;
2240 BlockAIOCBCoroutine *acb;
2241
2242 /* Matched by bdrv_co_complete's bdrv_dec_in_flight. */
2243 bdrv_inc_in_flight(bs);
2244
2245 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2246 acb->need_bh = true;
2247 acb->req.error = -EINPROGRESS;
2248
2249 co = qemu_coroutine_create(bdrv_aio_flush_co_entry, acb);
2250 qemu_coroutine_enter(co);
2251
2252 bdrv_co_maybe_schedule_bh(acb);
2253 return &acb->common;
2254 }
2255
2256 /**************************************************************/
2257 /* Coroutine block device emulation */
2258
2259 typedef struct FlushCo {
2260 BlockDriverState *bs;
2261 int ret;
2262 } FlushCo;
2263
2264
2265 static void coroutine_fn bdrv_flush_co_entry(void *opaque)
2266 {
2267 FlushCo *rwco = opaque;
2268
2269 rwco->ret = bdrv_co_flush(rwco->bs);
2270 }
2271
2272 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2273 {
2274 int ret;
2275
2276 if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2277 bdrv_is_sg(bs)) {
2278 return 0;
2279 }
2280
2281 bdrv_inc_in_flight(bs);
2282
2283 int current_gen = bs->write_gen;
2284
2285 /* Wait until any previous flushes are completed */
2286 while (bs->active_flush_req) {
2287 qemu_co_queue_wait(&bs->flush_queue, NULL);
2288 }
2289
2290 bs->active_flush_req = true;
2291
2292 /* Write back all layers by calling one driver function */
2293 if (bs->drv->bdrv_co_flush) {
2294 ret = bs->drv->bdrv_co_flush(bs);
2295 goto out;
2296 }
2297
2298 /* Write back cached data to the OS even with cache=unsafe */
2299 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
2300 if (bs->drv->bdrv_co_flush_to_os) {
2301 ret = bs->drv->bdrv_co_flush_to_os(bs);
2302 if (ret < 0) {
2303 goto out;
2304 }
2305 }
2306
2307 /* But don't actually force it to the disk with cache=unsafe */
2308 if (bs->open_flags & BDRV_O_NO_FLUSH) {
2309 goto flush_parent;
2310 }
2311
2312 /* Check if we really need to flush anything */
2313 if (bs->flushed_gen == current_gen) {
2314 goto flush_parent;
2315 }
2316
2317 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
2318 if (bs->drv->bdrv_co_flush_to_disk) {
2319 ret = bs->drv->bdrv_co_flush_to_disk(bs);
2320 } else if (bs->drv->bdrv_aio_flush) {
2321 BlockAIOCB *acb;
2322 CoroutineIOCompletion co = {
2323 .coroutine = qemu_coroutine_self(),
2324 };
2325
2326 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2327 if (acb == NULL) {
2328 ret = -EIO;
2329 } else {
2330 qemu_coroutine_yield();
2331 ret = co.ret;
2332 }
2333 } else {
2334 /*
2335 * Some block drivers always operate in either writethrough or unsafe
2336 * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2337 * know how the server works (because the behaviour is hardcoded or
2338 * depends on server-side configuration), so we can't ensure that
2339 * everything is safe on disk. Returning an error doesn't work because
2340 * that would break guests even if the server operates in writethrough
2341 * mode.
2342 *
2343 * Let's hope the user knows what he's doing.
2344 */
2345 ret = 0;
2346 }
2347
2348 if (ret < 0) {
2349 goto out;
2350 }
2351
2352 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH
2353 * in the case of cache=unsafe, so there are no useless flushes.
2354 */
2355 flush_parent:
2356 ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2357 out:
2358 /* Notify any pending flushes that we have completed */
2359 if (ret == 0) {
2360 bs->flushed_gen = current_gen;
2361 }
2362 bs->active_flush_req = false;
2363 /* Return value is ignored - it's ok if wait queue is empty */
2364 qemu_co_queue_next(&bs->flush_queue);
2365
2366 bdrv_dec_in_flight(bs);
2367 return ret;
2368 }
2369
2370 int bdrv_flush(BlockDriverState *bs)
2371 {
2372 Coroutine *co;
2373 FlushCo flush_co = {
2374 .bs = bs,
2375 .ret = NOT_DONE,
2376 };
2377
2378 if (qemu_in_coroutine()) {
2379 /* Fast-path if already in coroutine context */
2380 bdrv_flush_co_entry(&flush_co);
2381 } else {
2382 co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co);
2383 qemu_coroutine_enter(co);
2384 BDRV_POLL_WHILE(bs, flush_co.ret == NOT_DONE);
2385 }
2386
2387 return flush_co.ret;
2388 }
2389
2390 typedef struct DiscardCo {
2391 BlockDriverState *bs;
2392 int64_t offset;
2393 int count;
2394 int ret;
2395 } DiscardCo;
2396 static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque)
2397 {
2398 DiscardCo *rwco = opaque;
2399
2400 rwco->ret = bdrv_co_pdiscard(rwco->bs, rwco->offset, rwco->count);
2401 }
2402
2403 int coroutine_fn bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset,
2404 int count)
2405 {
2406 BdrvTrackedRequest req;
2407 int max_pdiscard, ret;
2408 int head, tail, align;
2409
2410 if (!bs->drv) {
2411 return -ENOMEDIUM;
2412 }
2413
2414 ret = bdrv_check_byte_request(bs, offset, count);
2415 if (ret < 0) {
2416 return ret;
2417 } else if (bs->read_only) {
2418 return -EPERM;
2419 }
2420 assert(!(bs->open_flags & BDRV_O_INACTIVE));
2421
2422 /* Do nothing if disabled. */
2423 if (!(bs->open_flags & BDRV_O_UNMAP)) {
2424 return 0;
2425 }
2426
2427 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
2428 return 0;
2429 }
2430
2431 /* Discard is advisory, but some devices track and coalesce
2432 * unaligned requests, so we must pass everything down rather than
2433 * round here. Still, most devices will just silently ignore
2434 * unaligned requests (by returning -ENOTSUP), so we must fragment
2435 * the request accordingly. */
2436 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2437 assert(align % bs->bl.request_alignment == 0);
2438 head = offset % align;
2439 tail = (offset + count) % align;
2440
2441 bdrv_inc_in_flight(bs);
2442 tracked_request_begin(&req, bs, offset, count, BDRV_TRACKED_DISCARD);
2443
2444 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req);
2445 if (ret < 0) {
2446 goto out;
2447 }
2448
2449 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX),
2450 align);
2451 assert(max_pdiscard >= bs->bl.request_alignment);
2452
2453 while (count > 0) {
2454 int ret;
2455 int num = count;
2456
2457 if (head) {
2458 /* Make small requests to get to alignment boundaries. */
2459 num = MIN(count, align - head);
2460 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
2461 num %= bs->bl.request_alignment;
2462 }
2463 head = (head + num) % align;
2464 assert(num < max_pdiscard);
2465 } else if (tail) {
2466 if (num > align) {
2467 /* Shorten the request to the last aligned cluster. */
2468 num -= tail;
2469 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
2470 tail > bs->bl.request_alignment) {
2471 tail %= bs->bl.request_alignment;
2472 num -= tail;
2473 }
2474 }
2475 /* limit request size */
2476 if (num > max_pdiscard) {
2477 num = max_pdiscard;
2478 }
2479
2480 if (bs->drv->bdrv_co_pdiscard) {
2481 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
2482 } else {
2483 BlockAIOCB *acb;
2484 CoroutineIOCompletion co = {
2485 .coroutine = qemu_coroutine_self(),
2486 };
2487
2488 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
2489 bdrv_co_io_em_complete, &co);
2490 if (acb == NULL) {
2491 ret = -EIO;
2492 goto out;
2493 } else {
2494 qemu_coroutine_yield();
2495 ret = co.ret;
2496 }
2497 }
2498 if (ret && ret != -ENOTSUP) {
2499 goto out;
2500 }
2501
2502 offset += num;
2503 count -= num;
2504 }
2505 ret = 0;
2506 out:
2507 ++bs->write_gen;
2508 bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS,
2509 req.bytes >> BDRV_SECTOR_BITS);
2510 tracked_request_end(&req);
2511 bdrv_dec_in_flight(bs);
2512 return ret;
2513 }
2514
2515 int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int count)
2516 {
2517 Coroutine *co;
2518 DiscardCo rwco = {
2519 .bs = bs,
2520 .offset = offset,
2521 .count = count,
2522 .ret = NOT_DONE,
2523 };
2524
2525 if (qemu_in_coroutine()) {
2526 /* Fast-path if already in coroutine context */
2527 bdrv_pdiscard_co_entry(&rwco);
2528 } else {
2529 co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco);
2530 qemu_coroutine_enter(co);
2531 BDRV_POLL_WHILE(bs, rwco.ret == NOT_DONE);
2532 }
2533
2534 return rwco.ret;
2535 }
2536
2537 int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
2538 {
2539 BlockDriver *drv = bs->drv;
2540 CoroutineIOCompletion co = {
2541 .coroutine = qemu_coroutine_self(),
2542 };
2543 BlockAIOCB *acb;
2544
2545 bdrv_inc_in_flight(bs);
2546 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
2547 co.ret = -ENOTSUP;
2548 goto out;
2549 }
2550
2551 if (drv->bdrv_co_ioctl) {
2552 co.ret = drv->bdrv_co_ioctl(bs, req, buf);
2553 } else {
2554 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
2555 if (!acb) {
2556 co.ret = -ENOTSUP;
2557 goto out;
2558 }
2559 qemu_coroutine_yield();
2560 }
2561 out:
2562 bdrv_dec_in_flight(bs);
2563 return co.ret;
2564 }
2565
2566 void *qemu_blockalign(BlockDriverState *bs, size_t size)
2567 {
2568 return qemu_memalign(bdrv_opt_mem_align(bs), size);
2569 }
2570
2571 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
2572 {
2573 return memset(qemu_blockalign(bs, size), 0, size);
2574 }
2575
2576 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
2577 {
2578 size_t align = bdrv_opt_mem_align(bs);
2579
2580 /* Ensure that NULL is never returned on success */
2581 assert(align > 0);
2582 if (size == 0) {
2583 size = align;
2584 }
2585
2586 return qemu_try_memalign(align, size);
2587 }
2588
2589 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
2590 {
2591 void *mem = qemu_try_blockalign(bs, size);
2592
2593 if (mem) {
2594 memset(mem, 0, size);
2595 }
2596
2597 return mem;
2598 }
2599
2600 /*
2601 * Check if all memory in this vector is sector aligned.
2602 */
2603 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
2604 {
2605 int i;
2606 size_t alignment = bdrv_min_mem_align(bs);
2607
2608 for (i = 0; i < qiov->niov; i++) {
2609 if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
2610 return false;
2611 }
2612 if (qiov->iov[i].iov_len % alignment) {
2613 return false;
2614 }
2615 }
2616
2617 return true;
2618 }
2619
2620 void bdrv_add_before_write_notifier(BlockDriverState *bs,
2621 NotifierWithReturn *notifier)
2622 {
2623 notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
2624 }
2625
2626 void bdrv_io_plug(BlockDriverState *bs)
2627 {
2628 BdrvChild *child;
2629
2630 QLIST_FOREACH(child, &bs->children, next) {
2631 bdrv_io_plug(child->bs);
2632 }
2633
2634 if (bs->io_plugged++ == 0) {
2635 BlockDriver *drv = bs->drv;
2636 if (drv && drv->bdrv_io_plug) {
2637 drv->bdrv_io_plug(bs);
2638 }
2639 }
2640 }
2641
2642 void bdrv_io_unplug(BlockDriverState *bs)
2643 {
2644 BdrvChild *child;
2645
2646 assert(bs->io_plugged);
2647 if (--bs->io_plugged == 0) {
2648 BlockDriver *drv = bs->drv;
2649 if (drv && drv->bdrv_io_unplug) {
2650 drv->bdrv_io_unplug(bs);
2651 }
2652 }
2653
2654 QLIST_FOREACH(child, &bs->children, next) {
2655 bdrv_io_unplug(child->bs);
2656 }
2657 }