qcow2: Add subcluster support to zero_in_l2_slice()
[qemu.git] / block / qcow2-cluster.c
1 /*
2 * Block driver for the QCOW version 2 format
3 *
4 * Copyright (c) 2004-2006 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include <zlib.h>
27
28 #include "qapi/error.h"
29 #include "qcow2.h"
30 #include "qemu/bswap.h"
31 #include "trace.h"
32
33 int qcow2_shrink_l1_table(BlockDriverState *bs, uint64_t exact_size)
34 {
35 BDRVQcow2State *s = bs->opaque;
36 int new_l1_size, i, ret;
37
38 if (exact_size >= s->l1_size) {
39 return 0;
40 }
41
42 new_l1_size = exact_size;
43
44 #ifdef DEBUG_ALLOC2
45 fprintf(stderr, "shrink l1_table from %d to %d\n", s->l1_size, new_l1_size);
46 #endif
47
48 BLKDBG_EVENT(bs->file, BLKDBG_L1_SHRINK_WRITE_TABLE);
49 ret = bdrv_pwrite_zeroes(bs->file, s->l1_table_offset +
50 new_l1_size * sizeof(uint64_t),
51 (s->l1_size - new_l1_size) * sizeof(uint64_t), 0);
52 if (ret < 0) {
53 goto fail;
54 }
55
56 ret = bdrv_flush(bs->file->bs);
57 if (ret < 0) {
58 goto fail;
59 }
60
61 BLKDBG_EVENT(bs->file, BLKDBG_L1_SHRINK_FREE_L2_CLUSTERS);
62 for (i = s->l1_size - 1; i > new_l1_size - 1; i--) {
63 if ((s->l1_table[i] & L1E_OFFSET_MASK) == 0) {
64 continue;
65 }
66 qcow2_free_clusters(bs, s->l1_table[i] & L1E_OFFSET_MASK,
67 s->cluster_size, QCOW2_DISCARD_ALWAYS);
68 s->l1_table[i] = 0;
69 }
70 return 0;
71
72 fail:
73 /*
74 * If the write in the l1_table failed the image may contain a partially
75 * overwritten l1_table. In this case it would be better to clear the
76 * l1_table in memory to avoid possible image corruption.
77 */
78 memset(s->l1_table + new_l1_size, 0,
79 (s->l1_size - new_l1_size) * sizeof(uint64_t));
80 return ret;
81 }
82
83 int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
84 bool exact_size)
85 {
86 BDRVQcow2State *s = bs->opaque;
87 int new_l1_size2, ret, i;
88 uint64_t *new_l1_table;
89 int64_t old_l1_table_offset, old_l1_size;
90 int64_t new_l1_table_offset, new_l1_size;
91 uint8_t data[12];
92
93 if (min_size <= s->l1_size)
94 return 0;
95
96 /* Do a sanity check on min_size before trying to calculate new_l1_size
97 * (this prevents overflows during the while loop for the calculation of
98 * new_l1_size) */
99 if (min_size > INT_MAX / sizeof(uint64_t)) {
100 return -EFBIG;
101 }
102
103 if (exact_size) {
104 new_l1_size = min_size;
105 } else {
106 /* Bump size up to reduce the number of times we have to grow */
107 new_l1_size = s->l1_size;
108 if (new_l1_size == 0) {
109 new_l1_size = 1;
110 }
111 while (min_size > new_l1_size) {
112 new_l1_size = DIV_ROUND_UP(new_l1_size * 3, 2);
113 }
114 }
115
116 QEMU_BUILD_BUG_ON(QCOW_MAX_L1_SIZE > INT_MAX);
117 if (new_l1_size > QCOW_MAX_L1_SIZE / sizeof(uint64_t)) {
118 return -EFBIG;
119 }
120
121 #ifdef DEBUG_ALLOC2
122 fprintf(stderr, "grow l1_table from %d to %" PRId64 "\n",
123 s->l1_size, new_l1_size);
124 #endif
125
126 new_l1_size2 = sizeof(uint64_t) * new_l1_size;
127 new_l1_table = qemu_try_blockalign(bs->file->bs, new_l1_size2);
128 if (new_l1_table == NULL) {
129 return -ENOMEM;
130 }
131 memset(new_l1_table, 0, new_l1_size2);
132
133 if (s->l1_size) {
134 memcpy(new_l1_table, s->l1_table, s->l1_size * sizeof(uint64_t));
135 }
136
137 /* write new table (align to cluster) */
138 BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ALLOC_TABLE);
139 new_l1_table_offset = qcow2_alloc_clusters(bs, new_l1_size2);
140 if (new_l1_table_offset < 0) {
141 qemu_vfree(new_l1_table);
142 return new_l1_table_offset;
143 }
144
145 ret = qcow2_cache_flush(bs, s->refcount_block_cache);
146 if (ret < 0) {
147 goto fail;
148 }
149
150 /* the L1 position has not yet been updated, so these clusters must
151 * indeed be completely free */
152 ret = qcow2_pre_write_overlap_check(bs, 0, new_l1_table_offset,
153 new_l1_size2, false);
154 if (ret < 0) {
155 goto fail;
156 }
157
158 BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_WRITE_TABLE);
159 for(i = 0; i < s->l1_size; i++)
160 new_l1_table[i] = cpu_to_be64(new_l1_table[i]);
161 ret = bdrv_pwrite_sync(bs->file, new_l1_table_offset,
162 new_l1_table, new_l1_size2);
163 if (ret < 0)
164 goto fail;
165 for(i = 0; i < s->l1_size; i++)
166 new_l1_table[i] = be64_to_cpu(new_l1_table[i]);
167
168 /* set new table */
169 BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ACTIVATE_TABLE);
170 stl_be_p(data, new_l1_size);
171 stq_be_p(data + 4, new_l1_table_offset);
172 ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_size),
173 data, sizeof(data));
174 if (ret < 0) {
175 goto fail;
176 }
177 qemu_vfree(s->l1_table);
178 old_l1_table_offset = s->l1_table_offset;
179 s->l1_table_offset = new_l1_table_offset;
180 s->l1_table = new_l1_table;
181 old_l1_size = s->l1_size;
182 s->l1_size = new_l1_size;
183 qcow2_free_clusters(bs, old_l1_table_offset, old_l1_size * sizeof(uint64_t),
184 QCOW2_DISCARD_OTHER);
185 return 0;
186 fail:
187 qemu_vfree(new_l1_table);
188 qcow2_free_clusters(bs, new_l1_table_offset, new_l1_size2,
189 QCOW2_DISCARD_OTHER);
190 return ret;
191 }
192
193 /*
194 * l2_load
195 *
196 * @bs: The BlockDriverState
197 * @offset: A guest offset, used to calculate what slice of the L2
198 * table to load.
199 * @l2_offset: Offset to the L2 table in the image file.
200 * @l2_slice: Location to store the pointer to the L2 slice.
201 *
202 * Loads a L2 slice into memory (L2 slices are the parts of L2 tables
203 * that are loaded by the qcow2 cache). If the slice is in the cache,
204 * the cache is used; otherwise the L2 slice is loaded from the image
205 * file.
206 */
207 static int l2_load(BlockDriverState *bs, uint64_t offset,
208 uint64_t l2_offset, uint64_t **l2_slice)
209 {
210 BDRVQcow2State *s = bs->opaque;
211 int start_of_slice = l2_entry_size(s) *
212 (offset_to_l2_index(s, offset) - offset_to_l2_slice_index(s, offset));
213
214 return qcow2_cache_get(bs, s->l2_table_cache, l2_offset + start_of_slice,
215 (void **)l2_slice);
216 }
217
218 /*
219 * Writes an L1 entry to disk (note that depending on the alignment
220 * requirements this function may write more that just one entry in
221 * order to prevent bdrv_pwrite from performing a read-modify-write)
222 */
223 int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
224 {
225 BDRVQcow2State *s = bs->opaque;
226 int l1_start_index;
227 int i, ret;
228 int bufsize = MAX(sizeof(uint64_t),
229 MIN(bs->file->bs->bl.request_alignment, s->cluster_size));
230 int nentries = bufsize / sizeof(uint64_t);
231 g_autofree uint64_t *buf = g_try_new0(uint64_t, nentries);
232
233 if (buf == NULL) {
234 return -ENOMEM;
235 }
236
237 l1_start_index = QEMU_ALIGN_DOWN(l1_index, nentries);
238 for (i = 0; i < MIN(nentries, s->l1_size - l1_start_index); i++) {
239 buf[i] = cpu_to_be64(s->l1_table[l1_start_index + i]);
240 }
241
242 ret = qcow2_pre_write_overlap_check(bs, QCOW2_OL_ACTIVE_L1,
243 s->l1_table_offset + 8 * l1_start_index, bufsize, false);
244 if (ret < 0) {
245 return ret;
246 }
247
248 BLKDBG_EVENT(bs->file, BLKDBG_L1_UPDATE);
249 ret = bdrv_pwrite_sync(bs->file,
250 s->l1_table_offset + 8 * l1_start_index,
251 buf, bufsize);
252 if (ret < 0) {
253 return ret;
254 }
255
256 return 0;
257 }
258
259 /*
260 * l2_allocate
261 *
262 * Allocate a new l2 entry in the file. If l1_index points to an already
263 * used entry in the L2 table (i.e. we are doing a copy on write for the L2
264 * table) copy the contents of the old L2 table into the newly allocated one.
265 * Otherwise the new table is initialized with zeros.
266 *
267 */
268
269 static int l2_allocate(BlockDriverState *bs, int l1_index)
270 {
271 BDRVQcow2State *s = bs->opaque;
272 uint64_t old_l2_offset;
273 uint64_t *l2_slice = NULL;
274 unsigned slice, slice_size2, n_slices;
275 int64_t l2_offset;
276 int ret;
277
278 old_l2_offset = s->l1_table[l1_index];
279
280 trace_qcow2_l2_allocate(bs, l1_index);
281
282 /* allocate a new l2 entry */
283
284 l2_offset = qcow2_alloc_clusters(bs, s->l2_size * l2_entry_size(s));
285 if (l2_offset < 0) {
286 ret = l2_offset;
287 goto fail;
288 }
289
290 /* The offset must fit in the offset field of the L1 table entry */
291 assert((l2_offset & L1E_OFFSET_MASK) == l2_offset);
292
293 /* If we're allocating the table at offset 0 then something is wrong */
294 if (l2_offset == 0) {
295 qcow2_signal_corruption(bs, true, -1, -1, "Preventing invalid "
296 "allocation of L2 table at offset 0");
297 ret = -EIO;
298 goto fail;
299 }
300
301 ret = qcow2_cache_flush(bs, s->refcount_block_cache);
302 if (ret < 0) {
303 goto fail;
304 }
305
306 /* allocate a new entry in the l2 cache */
307
308 slice_size2 = s->l2_slice_size * l2_entry_size(s);
309 n_slices = s->cluster_size / slice_size2;
310
311 trace_qcow2_l2_allocate_get_empty(bs, l1_index);
312 for (slice = 0; slice < n_slices; slice++) {
313 ret = qcow2_cache_get_empty(bs, s->l2_table_cache,
314 l2_offset + slice * slice_size2,
315 (void **) &l2_slice);
316 if (ret < 0) {
317 goto fail;
318 }
319
320 if ((old_l2_offset & L1E_OFFSET_MASK) == 0) {
321 /* if there was no old l2 table, clear the new slice */
322 memset(l2_slice, 0, slice_size2);
323 } else {
324 uint64_t *old_slice;
325 uint64_t old_l2_slice_offset =
326 (old_l2_offset & L1E_OFFSET_MASK) + slice * slice_size2;
327
328 /* if there was an old l2 table, read a slice from the disk */
329 BLKDBG_EVENT(bs->file, BLKDBG_L2_ALLOC_COW_READ);
330 ret = qcow2_cache_get(bs, s->l2_table_cache, old_l2_slice_offset,
331 (void **) &old_slice);
332 if (ret < 0) {
333 goto fail;
334 }
335
336 memcpy(l2_slice, old_slice, slice_size2);
337
338 qcow2_cache_put(s->l2_table_cache, (void **) &old_slice);
339 }
340
341 /* write the l2 slice to the file */
342 BLKDBG_EVENT(bs->file, BLKDBG_L2_ALLOC_WRITE);
343
344 trace_qcow2_l2_allocate_write_l2(bs, l1_index);
345 qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
346 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
347 }
348
349 ret = qcow2_cache_flush(bs, s->l2_table_cache);
350 if (ret < 0) {
351 goto fail;
352 }
353
354 /* update the L1 entry */
355 trace_qcow2_l2_allocate_write_l1(bs, l1_index);
356 s->l1_table[l1_index] = l2_offset | QCOW_OFLAG_COPIED;
357 ret = qcow2_write_l1_entry(bs, l1_index);
358 if (ret < 0) {
359 goto fail;
360 }
361
362 trace_qcow2_l2_allocate_done(bs, l1_index, 0);
363 return 0;
364
365 fail:
366 trace_qcow2_l2_allocate_done(bs, l1_index, ret);
367 if (l2_slice != NULL) {
368 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
369 }
370 s->l1_table[l1_index] = old_l2_offset;
371 if (l2_offset > 0) {
372 qcow2_free_clusters(bs, l2_offset, s->l2_size * l2_entry_size(s),
373 QCOW2_DISCARD_ALWAYS);
374 }
375 return ret;
376 }
377
378 /*
379 * For a given L2 entry, count the number of contiguous subclusters of
380 * the same type starting from @sc_from. Compressed clusters are
381 * treated as if they were divided into subclusters of size
382 * s->subcluster_size.
383 *
384 * Return the number of contiguous subclusters and set @type to the
385 * subcluster type.
386 *
387 * If the L2 entry is invalid return -errno and set @type to
388 * QCOW2_SUBCLUSTER_INVALID.
389 */
390 static int qcow2_get_subcluster_range_type(BlockDriverState *bs,
391 uint64_t l2_entry,
392 uint64_t l2_bitmap,
393 unsigned sc_from,
394 QCow2SubclusterType *type)
395 {
396 BDRVQcow2State *s = bs->opaque;
397 uint32_t val;
398
399 *type = qcow2_get_subcluster_type(bs, l2_entry, l2_bitmap, sc_from);
400
401 if (*type == QCOW2_SUBCLUSTER_INVALID) {
402 return -EINVAL;
403 } else if (!has_subclusters(s) || *type == QCOW2_SUBCLUSTER_COMPRESSED) {
404 return s->subclusters_per_cluster - sc_from;
405 }
406
407 switch (*type) {
408 case QCOW2_SUBCLUSTER_NORMAL:
409 val = l2_bitmap | QCOW_OFLAG_SUB_ALLOC_RANGE(0, sc_from);
410 return cto32(val) - sc_from;
411
412 case QCOW2_SUBCLUSTER_ZERO_PLAIN:
413 case QCOW2_SUBCLUSTER_ZERO_ALLOC:
414 val = (l2_bitmap | QCOW_OFLAG_SUB_ZERO_RANGE(0, sc_from)) >> 32;
415 return cto32(val) - sc_from;
416
417 case QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN:
418 case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC:
419 val = ((l2_bitmap >> 32) | l2_bitmap)
420 & ~QCOW_OFLAG_SUB_ALLOC_RANGE(0, sc_from);
421 return ctz32(val) - sc_from;
422
423 default:
424 g_assert_not_reached();
425 }
426 }
427
428 /*
429 * Return the number of contiguous subclusters of the exact same type
430 * in a given L2 slice, starting from cluster @l2_index, subcluster
431 * @sc_index. Allocated subclusters are required to be contiguous in
432 * the image file.
433 * At most @nb_clusters are checked (note that this means clusters,
434 * not subclusters).
435 * Compressed clusters are always processed one by one but for the
436 * purpose of this count they are treated as if they were divided into
437 * subclusters of size s->subcluster_size.
438 * On failure return -errno and update @l2_index to point to the
439 * invalid entry.
440 */
441 static int count_contiguous_subclusters(BlockDriverState *bs, int nb_clusters,
442 unsigned sc_index, uint64_t *l2_slice,
443 unsigned *l2_index)
444 {
445 BDRVQcow2State *s = bs->opaque;
446 int i, count = 0;
447 bool check_offset = false;
448 uint64_t expected_offset = 0;
449 QCow2SubclusterType expected_type = QCOW2_SUBCLUSTER_NORMAL, type;
450
451 assert(*l2_index + nb_clusters <= s->l2_slice_size);
452
453 for (i = 0; i < nb_clusters; i++) {
454 unsigned first_sc = (i == 0) ? sc_index : 0;
455 uint64_t l2_entry = get_l2_entry(s, l2_slice, *l2_index + i);
456 uint64_t l2_bitmap = get_l2_bitmap(s, l2_slice, *l2_index + i);
457 int ret = qcow2_get_subcluster_range_type(bs, l2_entry, l2_bitmap,
458 first_sc, &type);
459 if (ret < 0) {
460 *l2_index += i; /* Point to the invalid entry */
461 return -EIO;
462 }
463 if (i == 0) {
464 if (type == QCOW2_SUBCLUSTER_COMPRESSED) {
465 /* Compressed clusters are always processed one by one */
466 return ret;
467 }
468 expected_type = type;
469 expected_offset = l2_entry & L2E_OFFSET_MASK;
470 check_offset = (type == QCOW2_SUBCLUSTER_NORMAL ||
471 type == QCOW2_SUBCLUSTER_ZERO_ALLOC ||
472 type == QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC);
473 } else if (type != expected_type) {
474 break;
475 } else if (check_offset) {
476 expected_offset += s->cluster_size;
477 if (expected_offset != (l2_entry & L2E_OFFSET_MASK)) {
478 break;
479 }
480 }
481 count += ret;
482 /* Stop if there are type changes before the end of the cluster */
483 if (first_sc + ret < s->subclusters_per_cluster) {
484 break;
485 }
486 }
487
488 return count;
489 }
490
491 static int coroutine_fn do_perform_cow_read(BlockDriverState *bs,
492 uint64_t src_cluster_offset,
493 unsigned offset_in_cluster,
494 QEMUIOVector *qiov)
495 {
496 int ret;
497
498 if (qiov->size == 0) {
499 return 0;
500 }
501
502 BLKDBG_EVENT(bs->file, BLKDBG_COW_READ);
503
504 if (!bs->drv) {
505 return -ENOMEDIUM;
506 }
507
508 /* Call .bdrv_co_readv() directly instead of using the public block-layer
509 * interface. This avoids double I/O throttling and request tracking,
510 * which can lead to deadlock when block layer copy-on-read is enabled.
511 */
512 ret = bs->drv->bdrv_co_preadv_part(bs,
513 src_cluster_offset + offset_in_cluster,
514 qiov->size, qiov, 0, 0);
515 if (ret < 0) {
516 return ret;
517 }
518
519 return 0;
520 }
521
522 static int coroutine_fn do_perform_cow_write(BlockDriverState *bs,
523 uint64_t cluster_offset,
524 unsigned offset_in_cluster,
525 QEMUIOVector *qiov)
526 {
527 BDRVQcow2State *s = bs->opaque;
528 int ret;
529
530 if (qiov->size == 0) {
531 return 0;
532 }
533
534 ret = qcow2_pre_write_overlap_check(bs, 0,
535 cluster_offset + offset_in_cluster, qiov->size, true);
536 if (ret < 0) {
537 return ret;
538 }
539
540 BLKDBG_EVENT(bs->file, BLKDBG_COW_WRITE);
541 ret = bdrv_co_pwritev(s->data_file, cluster_offset + offset_in_cluster,
542 qiov->size, qiov, 0);
543 if (ret < 0) {
544 return ret;
545 }
546
547 return 0;
548 }
549
550
551 /*
552 * get_host_offset
553 *
554 * For a given offset of the virtual disk find the equivalent host
555 * offset in the qcow2 file and store it in *host_offset. Neither
556 * offset needs to be aligned to a cluster boundary.
557 *
558 * If the cluster is unallocated then *host_offset will be 0.
559 * If the cluster is compressed then *host_offset will contain the
560 * complete compressed cluster descriptor.
561 *
562 * On entry, *bytes is the maximum number of contiguous bytes starting at
563 * offset that we are interested in.
564 *
565 * On exit, *bytes is the number of bytes starting at offset that have the same
566 * subcluster type and (if applicable) are stored contiguously in the image
567 * file. The subcluster type is stored in *subcluster_type.
568 * Compressed clusters are always processed one by one.
569 *
570 * Returns 0 on success, -errno in error cases.
571 */
572 int qcow2_get_host_offset(BlockDriverState *bs, uint64_t offset,
573 unsigned int *bytes, uint64_t *host_offset,
574 QCow2SubclusterType *subcluster_type)
575 {
576 BDRVQcow2State *s = bs->opaque;
577 unsigned int l2_index, sc_index;
578 uint64_t l1_index, l2_offset, *l2_slice, l2_entry, l2_bitmap;
579 int sc;
580 unsigned int offset_in_cluster;
581 uint64_t bytes_available, bytes_needed, nb_clusters;
582 QCow2SubclusterType type;
583 int ret;
584
585 offset_in_cluster = offset_into_cluster(s, offset);
586 bytes_needed = (uint64_t) *bytes + offset_in_cluster;
587
588 /* compute how many bytes there are between the start of the cluster
589 * containing offset and the end of the l2 slice that contains
590 * the entry pointing to it */
591 bytes_available =
592 ((uint64_t) (s->l2_slice_size - offset_to_l2_slice_index(s, offset)))
593 << s->cluster_bits;
594
595 if (bytes_needed > bytes_available) {
596 bytes_needed = bytes_available;
597 }
598
599 *host_offset = 0;
600
601 /* seek to the l2 offset in the l1 table */
602
603 l1_index = offset_to_l1_index(s, offset);
604 if (l1_index >= s->l1_size) {
605 type = QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN;
606 goto out;
607 }
608
609 l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
610 if (!l2_offset) {
611 type = QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN;
612 goto out;
613 }
614
615 if (offset_into_cluster(s, l2_offset)) {
616 qcow2_signal_corruption(bs, true, -1, -1, "L2 table offset %#" PRIx64
617 " unaligned (L1 index: %#" PRIx64 ")",
618 l2_offset, l1_index);
619 return -EIO;
620 }
621
622 /* load the l2 slice in memory */
623
624 ret = l2_load(bs, offset, l2_offset, &l2_slice);
625 if (ret < 0) {
626 return ret;
627 }
628
629 /* find the cluster offset for the given disk offset */
630
631 l2_index = offset_to_l2_slice_index(s, offset);
632 sc_index = offset_to_sc_index(s, offset);
633 l2_entry = get_l2_entry(s, l2_slice, l2_index);
634 l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index);
635
636 nb_clusters = size_to_clusters(s, bytes_needed);
637 /* bytes_needed <= *bytes + offset_in_cluster, both of which are unsigned
638 * integers; the minimum cluster size is 512, so this assertion is always
639 * true */
640 assert(nb_clusters <= INT_MAX);
641
642 type = qcow2_get_subcluster_type(bs, l2_entry, l2_bitmap, sc_index);
643 if (s->qcow_version < 3 && (type == QCOW2_SUBCLUSTER_ZERO_PLAIN ||
644 type == QCOW2_SUBCLUSTER_ZERO_ALLOC)) {
645 qcow2_signal_corruption(bs, true, -1, -1, "Zero cluster entry found"
646 " in pre-v3 image (L2 offset: %#" PRIx64
647 ", L2 index: %#x)", l2_offset, l2_index);
648 ret = -EIO;
649 goto fail;
650 }
651 switch (type) {
652 case QCOW2_SUBCLUSTER_INVALID:
653 break; /* This is handled by count_contiguous_subclusters() below */
654 case QCOW2_SUBCLUSTER_COMPRESSED:
655 if (has_data_file(bs)) {
656 qcow2_signal_corruption(bs, true, -1, -1, "Compressed cluster "
657 "entry found in image with external data "
658 "file (L2 offset: %#" PRIx64 ", L2 index: "
659 "%#x)", l2_offset, l2_index);
660 ret = -EIO;
661 goto fail;
662 }
663 *host_offset = l2_entry & L2E_COMPRESSED_OFFSET_SIZE_MASK;
664 break;
665 case QCOW2_SUBCLUSTER_ZERO_PLAIN:
666 case QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN:
667 break;
668 case QCOW2_SUBCLUSTER_ZERO_ALLOC:
669 case QCOW2_SUBCLUSTER_NORMAL:
670 case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC: {
671 uint64_t host_cluster_offset = l2_entry & L2E_OFFSET_MASK;
672 *host_offset = host_cluster_offset + offset_in_cluster;
673 if (offset_into_cluster(s, host_cluster_offset)) {
674 qcow2_signal_corruption(bs, true, -1, -1,
675 "Cluster allocation offset %#"
676 PRIx64 " unaligned (L2 offset: %#" PRIx64
677 ", L2 index: %#x)", host_cluster_offset,
678 l2_offset, l2_index);
679 ret = -EIO;
680 goto fail;
681 }
682 if (has_data_file(bs) && *host_offset != offset) {
683 qcow2_signal_corruption(bs, true, -1, -1,
684 "External data file host cluster offset %#"
685 PRIx64 " does not match guest cluster "
686 "offset: %#" PRIx64
687 ", L2 index: %#x)", host_cluster_offset,
688 offset - offset_in_cluster, l2_index);
689 ret = -EIO;
690 goto fail;
691 }
692 break;
693 }
694 default:
695 abort();
696 }
697
698 sc = count_contiguous_subclusters(bs, nb_clusters, sc_index,
699 l2_slice, &l2_index);
700 if (sc < 0) {
701 qcow2_signal_corruption(bs, true, -1, -1, "Invalid cluster entry found "
702 " (L2 offset: %#" PRIx64 ", L2 index: %#x)",
703 l2_offset, l2_index);
704 ret = -EIO;
705 goto fail;
706 }
707 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
708
709 bytes_available = ((int64_t)sc + sc_index) << s->subcluster_bits;
710
711 out:
712 if (bytes_available > bytes_needed) {
713 bytes_available = bytes_needed;
714 }
715
716 /* bytes_available <= bytes_needed <= *bytes + offset_in_cluster;
717 * subtracting offset_in_cluster will therefore definitely yield something
718 * not exceeding UINT_MAX */
719 assert(bytes_available - offset_in_cluster <= UINT_MAX);
720 *bytes = bytes_available - offset_in_cluster;
721
722 *subcluster_type = type;
723
724 return 0;
725
726 fail:
727 qcow2_cache_put(s->l2_table_cache, (void **)&l2_slice);
728 return ret;
729 }
730
731 /*
732 * get_cluster_table
733 *
734 * for a given disk offset, load (and allocate if needed)
735 * the appropriate slice of its l2 table.
736 *
737 * the cluster index in the l2 slice is given to the caller.
738 *
739 * Returns 0 on success, -errno in failure case
740 */
741 static int get_cluster_table(BlockDriverState *bs, uint64_t offset,
742 uint64_t **new_l2_slice,
743 int *new_l2_index)
744 {
745 BDRVQcow2State *s = bs->opaque;
746 unsigned int l2_index;
747 uint64_t l1_index, l2_offset;
748 uint64_t *l2_slice = NULL;
749 int ret;
750
751 /* seek to the l2 offset in the l1 table */
752
753 l1_index = offset_to_l1_index(s, offset);
754 if (l1_index >= s->l1_size) {
755 ret = qcow2_grow_l1_table(bs, l1_index + 1, false);
756 if (ret < 0) {
757 return ret;
758 }
759 }
760
761 assert(l1_index < s->l1_size);
762 l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
763 if (offset_into_cluster(s, l2_offset)) {
764 qcow2_signal_corruption(bs, true, -1, -1, "L2 table offset %#" PRIx64
765 " unaligned (L1 index: %#" PRIx64 ")",
766 l2_offset, l1_index);
767 return -EIO;
768 }
769
770 if (!(s->l1_table[l1_index] & QCOW_OFLAG_COPIED)) {
771 /* First allocate a new L2 table (and do COW if needed) */
772 ret = l2_allocate(bs, l1_index);
773 if (ret < 0) {
774 return ret;
775 }
776
777 /* Then decrease the refcount of the old table */
778 if (l2_offset) {
779 qcow2_free_clusters(bs, l2_offset, s->l2_size * l2_entry_size(s),
780 QCOW2_DISCARD_OTHER);
781 }
782
783 /* Get the offset of the newly-allocated l2 table */
784 l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
785 assert(offset_into_cluster(s, l2_offset) == 0);
786 }
787
788 /* load the l2 slice in memory */
789 ret = l2_load(bs, offset, l2_offset, &l2_slice);
790 if (ret < 0) {
791 return ret;
792 }
793
794 /* find the cluster offset for the given disk offset */
795
796 l2_index = offset_to_l2_slice_index(s, offset);
797
798 *new_l2_slice = l2_slice;
799 *new_l2_index = l2_index;
800
801 return 0;
802 }
803
804 /*
805 * alloc_compressed_cluster_offset
806 *
807 * For a given offset on the virtual disk, allocate a new compressed cluster
808 * and put the host offset of the cluster into *host_offset. If a cluster is
809 * already allocated at the offset, return an error.
810 *
811 * Return 0 on success and -errno in error cases
812 */
813 int qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
814 uint64_t offset,
815 int compressed_size,
816 uint64_t *host_offset)
817 {
818 BDRVQcow2State *s = bs->opaque;
819 int l2_index, ret;
820 uint64_t *l2_slice;
821 int64_t cluster_offset;
822 int nb_csectors;
823
824 if (has_data_file(bs)) {
825 return 0;
826 }
827
828 ret = get_cluster_table(bs, offset, &l2_slice, &l2_index);
829 if (ret < 0) {
830 return ret;
831 }
832
833 /* Compression can't overwrite anything. Fail if the cluster was already
834 * allocated. */
835 cluster_offset = get_l2_entry(s, l2_slice, l2_index);
836 if (cluster_offset & L2E_OFFSET_MASK) {
837 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
838 return -EIO;
839 }
840
841 cluster_offset = qcow2_alloc_bytes(bs, compressed_size);
842 if (cluster_offset < 0) {
843 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
844 return cluster_offset;
845 }
846
847 nb_csectors =
848 (cluster_offset + compressed_size - 1) / QCOW2_COMPRESSED_SECTOR_SIZE -
849 (cluster_offset / QCOW2_COMPRESSED_SECTOR_SIZE);
850
851 /* The offset and size must fit in their fields of the L2 table entry */
852 assert((cluster_offset & s->cluster_offset_mask) == cluster_offset);
853 assert((nb_csectors & s->csize_mask) == nb_csectors);
854
855 cluster_offset |= QCOW_OFLAG_COMPRESSED |
856 ((uint64_t)nb_csectors << s->csize_shift);
857
858 /* update L2 table */
859
860 /* compressed clusters never have the copied flag */
861
862 BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE_COMPRESSED);
863 qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
864 set_l2_entry(s, l2_slice, l2_index, cluster_offset);
865 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
866
867 *host_offset = cluster_offset & s->cluster_offset_mask;
868 return 0;
869 }
870
871 static int perform_cow(BlockDriverState *bs, QCowL2Meta *m)
872 {
873 BDRVQcow2State *s = bs->opaque;
874 Qcow2COWRegion *start = &m->cow_start;
875 Qcow2COWRegion *end = &m->cow_end;
876 unsigned buffer_size;
877 unsigned data_bytes = end->offset - (start->offset + start->nb_bytes);
878 bool merge_reads;
879 uint8_t *start_buffer, *end_buffer;
880 QEMUIOVector qiov;
881 int ret;
882
883 assert(start->nb_bytes <= UINT_MAX - end->nb_bytes);
884 assert(start->nb_bytes + end->nb_bytes <= UINT_MAX - data_bytes);
885 assert(start->offset + start->nb_bytes <= end->offset);
886
887 if ((start->nb_bytes == 0 && end->nb_bytes == 0) || m->skip_cow) {
888 return 0;
889 }
890
891 /* If we have to read both the start and end COW regions and the
892 * middle region is not too large then perform just one read
893 * operation */
894 merge_reads = start->nb_bytes && end->nb_bytes && data_bytes <= 16384;
895 if (merge_reads) {
896 buffer_size = start->nb_bytes + data_bytes + end->nb_bytes;
897 } else {
898 /* If we have to do two reads, add some padding in the middle
899 * if necessary to make sure that the end region is optimally
900 * aligned. */
901 size_t align = bdrv_opt_mem_align(bs);
902 assert(align > 0 && align <= UINT_MAX);
903 assert(QEMU_ALIGN_UP(start->nb_bytes, align) <=
904 UINT_MAX - end->nb_bytes);
905 buffer_size = QEMU_ALIGN_UP(start->nb_bytes, align) + end->nb_bytes;
906 }
907
908 /* Reserve a buffer large enough to store all the data that we're
909 * going to read */
910 start_buffer = qemu_try_blockalign(bs, buffer_size);
911 if (start_buffer == NULL) {
912 return -ENOMEM;
913 }
914 /* The part of the buffer where the end region is located */
915 end_buffer = start_buffer + buffer_size - end->nb_bytes;
916
917 qemu_iovec_init(&qiov, 2 + (m->data_qiov ?
918 qemu_iovec_subvec_niov(m->data_qiov,
919 m->data_qiov_offset,
920 data_bytes)
921 : 0));
922
923 qemu_co_mutex_unlock(&s->lock);
924 /* First we read the existing data from both COW regions. We
925 * either read the whole region in one go, or the start and end
926 * regions separately. */
927 if (merge_reads) {
928 qemu_iovec_add(&qiov, start_buffer, buffer_size);
929 ret = do_perform_cow_read(bs, m->offset, start->offset, &qiov);
930 } else {
931 qemu_iovec_add(&qiov, start_buffer, start->nb_bytes);
932 ret = do_perform_cow_read(bs, m->offset, start->offset, &qiov);
933 if (ret < 0) {
934 goto fail;
935 }
936
937 qemu_iovec_reset(&qiov);
938 qemu_iovec_add(&qiov, end_buffer, end->nb_bytes);
939 ret = do_perform_cow_read(bs, m->offset, end->offset, &qiov);
940 }
941 if (ret < 0) {
942 goto fail;
943 }
944
945 /* Encrypt the data if necessary before writing it */
946 if (bs->encrypted) {
947 ret = qcow2_co_encrypt(bs,
948 m->alloc_offset + start->offset,
949 m->offset + start->offset,
950 start_buffer, start->nb_bytes);
951 if (ret < 0) {
952 goto fail;
953 }
954
955 ret = qcow2_co_encrypt(bs,
956 m->alloc_offset + end->offset,
957 m->offset + end->offset,
958 end_buffer, end->nb_bytes);
959 if (ret < 0) {
960 goto fail;
961 }
962 }
963
964 /* And now we can write everything. If we have the guest data we
965 * can write everything in one single operation */
966 if (m->data_qiov) {
967 qemu_iovec_reset(&qiov);
968 if (start->nb_bytes) {
969 qemu_iovec_add(&qiov, start_buffer, start->nb_bytes);
970 }
971 qemu_iovec_concat(&qiov, m->data_qiov, m->data_qiov_offset, data_bytes);
972 if (end->nb_bytes) {
973 qemu_iovec_add(&qiov, end_buffer, end->nb_bytes);
974 }
975 /* NOTE: we have a write_aio blkdebug event here followed by
976 * a cow_write one in do_perform_cow_write(), but there's only
977 * one single I/O operation */
978 BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
979 ret = do_perform_cow_write(bs, m->alloc_offset, start->offset, &qiov);
980 } else {
981 /* If there's no guest data then write both COW regions separately */
982 qemu_iovec_reset(&qiov);
983 qemu_iovec_add(&qiov, start_buffer, start->nb_bytes);
984 ret = do_perform_cow_write(bs, m->alloc_offset, start->offset, &qiov);
985 if (ret < 0) {
986 goto fail;
987 }
988
989 qemu_iovec_reset(&qiov);
990 qemu_iovec_add(&qiov, end_buffer, end->nb_bytes);
991 ret = do_perform_cow_write(bs, m->alloc_offset, end->offset, &qiov);
992 }
993
994 fail:
995 qemu_co_mutex_lock(&s->lock);
996
997 /*
998 * Before we update the L2 table to actually point to the new cluster, we
999 * need to be sure that the refcounts have been increased and COW was
1000 * handled.
1001 */
1002 if (ret == 0) {
1003 qcow2_cache_depends_on_flush(s->l2_table_cache);
1004 }
1005
1006 qemu_vfree(start_buffer);
1007 qemu_iovec_destroy(&qiov);
1008 return ret;
1009 }
1010
1011 int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
1012 {
1013 BDRVQcow2State *s = bs->opaque;
1014 int i, j = 0, l2_index, ret;
1015 uint64_t *old_cluster, *l2_slice;
1016 uint64_t cluster_offset = m->alloc_offset;
1017
1018 trace_qcow2_cluster_link_l2(qemu_coroutine_self(), m->nb_clusters);
1019 assert(m->nb_clusters > 0);
1020
1021 old_cluster = g_try_new(uint64_t, m->nb_clusters);
1022 if (old_cluster == NULL) {
1023 ret = -ENOMEM;
1024 goto err;
1025 }
1026
1027 /* copy content of unmodified sectors */
1028 ret = perform_cow(bs, m);
1029 if (ret < 0) {
1030 goto err;
1031 }
1032
1033 /* Update L2 table. */
1034 if (s->use_lazy_refcounts) {
1035 qcow2_mark_dirty(bs);
1036 }
1037 if (qcow2_need_accurate_refcounts(s)) {
1038 qcow2_cache_set_dependency(bs, s->l2_table_cache,
1039 s->refcount_block_cache);
1040 }
1041
1042 ret = get_cluster_table(bs, m->offset, &l2_slice, &l2_index);
1043 if (ret < 0) {
1044 goto err;
1045 }
1046 qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
1047
1048 assert(l2_index + m->nb_clusters <= s->l2_slice_size);
1049 for (i = 0; i < m->nb_clusters; i++) {
1050 uint64_t offset = cluster_offset + ((uint64_t)i << s->cluster_bits);
1051 /* if two concurrent writes happen to the same unallocated cluster
1052 * each write allocates separate cluster and writes data concurrently.
1053 * The first one to complete updates l2 table with pointer to its
1054 * cluster the second one has to do RMW (which is done above by
1055 * perform_cow()), update l2 table with its cluster pointer and free
1056 * old cluster. This is what this loop does */
1057 if (get_l2_entry(s, l2_slice, l2_index + i) != 0) {
1058 old_cluster[j++] = get_l2_entry(s, l2_slice, l2_index + i);
1059 }
1060
1061 /* The offset must fit in the offset field of the L2 table entry */
1062 assert((offset & L2E_OFFSET_MASK) == offset);
1063
1064 set_l2_entry(s, l2_slice, l2_index + i, offset | QCOW_OFLAG_COPIED);
1065 }
1066
1067
1068 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
1069
1070 /*
1071 * If this was a COW, we need to decrease the refcount of the old cluster.
1072 *
1073 * Don't discard clusters that reach a refcount of 0 (e.g. compressed
1074 * clusters), the next write will reuse them anyway.
1075 */
1076 if (!m->keep_old_clusters && j != 0) {
1077 for (i = 0; i < j; i++) {
1078 qcow2_free_any_clusters(bs, old_cluster[i], 1, QCOW2_DISCARD_NEVER);
1079 }
1080 }
1081
1082 ret = 0;
1083 err:
1084 g_free(old_cluster);
1085 return ret;
1086 }
1087
1088 /**
1089 * Frees the allocated clusters because the request failed and they won't
1090 * actually be linked.
1091 */
1092 void qcow2_alloc_cluster_abort(BlockDriverState *bs, QCowL2Meta *m)
1093 {
1094 BDRVQcow2State *s = bs->opaque;
1095 if (!has_data_file(bs) && !m->keep_old_clusters) {
1096 qcow2_free_clusters(bs, m->alloc_offset,
1097 m->nb_clusters << s->cluster_bits,
1098 QCOW2_DISCARD_NEVER);
1099 }
1100 }
1101
1102 /*
1103 * For a given write request, create a new QCowL2Meta structure, add
1104 * it to @m and the BDRVQcow2State.cluster_allocs list. If the write
1105 * request does not need copy-on-write or changes to the L2 metadata
1106 * then this function does nothing.
1107 *
1108 * @host_cluster_offset points to the beginning of the first cluster.
1109 *
1110 * @guest_offset and @bytes indicate the offset and length of the
1111 * request.
1112 *
1113 * @l2_slice contains the L2 entries of all clusters involved in this
1114 * write request.
1115 *
1116 * If @keep_old is true it means that the clusters were already
1117 * allocated and will be overwritten. If false then the clusters are
1118 * new and we have to decrease the reference count of the old ones.
1119 *
1120 * Returns 0 on success, -errno on failure.
1121 */
1122 static int calculate_l2_meta(BlockDriverState *bs, uint64_t host_cluster_offset,
1123 uint64_t guest_offset, unsigned bytes,
1124 uint64_t *l2_slice, QCowL2Meta **m, bool keep_old)
1125 {
1126 BDRVQcow2State *s = bs->opaque;
1127 int sc_index, l2_index = offset_to_l2_slice_index(s, guest_offset);
1128 uint64_t l2_entry, l2_bitmap;
1129 unsigned cow_start_from, cow_end_to;
1130 unsigned cow_start_to = offset_into_cluster(s, guest_offset);
1131 unsigned cow_end_from = cow_start_to + bytes;
1132 unsigned nb_clusters = size_to_clusters(s, cow_end_from);
1133 QCowL2Meta *old_m = *m;
1134 QCow2SubclusterType type;
1135 int i;
1136 bool skip_cow = keep_old;
1137
1138 assert(nb_clusters <= s->l2_slice_size - l2_index);
1139
1140 /* Check the type of all affected subclusters */
1141 for (i = 0; i < nb_clusters; i++) {
1142 l2_entry = get_l2_entry(s, l2_slice, l2_index + i);
1143 l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index + i);
1144 if (skip_cow) {
1145 unsigned write_from = MAX(cow_start_to, i << s->cluster_bits);
1146 unsigned write_to = MIN(cow_end_from, (i + 1) << s->cluster_bits);
1147 int first_sc = offset_to_sc_index(s, write_from);
1148 int last_sc = offset_to_sc_index(s, write_to - 1);
1149 int cnt = qcow2_get_subcluster_range_type(bs, l2_entry, l2_bitmap,
1150 first_sc, &type);
1151 /* Is any of the subclusters of type != QCOW2_SUBCLUSTER_NORMAL ? */
1152 if (type != QCOW2_SUBCLUSTER_NORMAL || first_sc + cnt <= last_sc) {
1153 skip_cow = false;
1154 }
1155 } else {
1156 /* If we can't skip the cow we can still look for invalid entries */
1157 type = qcow2_get_subcluster_type(bs, l2_entry, l2_bitmap, 0);
1158 }
1159 if (type == QCOW2_SUBCLUSTER_INVALID) {
1160 int l1_index = offset_to_l1_index(s, guest_offset);
1161 uint64_t l2_offset = s->l1_table[l1_index] & L1E_OFFSET_MASK;
1162 qcow2_signal_corruption(bs, true, -1, -1, "Invalid cluster "
1163 "entry found (L2 offset: %#" PRIx64
1164 ", L2 index: %#x)",
1165 l2_offset, l2_index + i);
1166 return -EIO;
1167 }
1168 }
1169
1170 if (skip_cow) {
1171 return 0;
1172 }
1173
1174 /* Get the L2 entry of the first cluster */
1175 l2_entry = get_l2_entry(s, l2_slice, l2_index);
1176 l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index);
1177 sc_index = offset_to_sc_index(s, guest_offset);
1178 type = qcow2_get_subcluster_type(bs, l2_entry, l2_bitmap, sc_index);
1179
1180 if (!keep_old) {
1181 switch (type) {
1182 case QCOW2_SUBCLUSTER_COMPRESSED:
1183 cow_start_from = 0;
1184 break;
1185 case QCOW2_SUBCLUSTER_NORMAL:
1186 case QCOW2_SUBCLUSTER_ZERO_ALLOC:
1187 case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC:
1188 if (has_subclusters(s)) {
1189 /* Skip all leading zero and unallocated subclusters */
1190 uint32_t alloc_bitmap = l2_bitmap & QCOW_L2_BITMAP_ALL_ALLOC;
1191 cow_start_from =
1192 MIN(sc_index, ctz32(alloc_bitmap)) << s->subcluster_bits;
1193 } else {
1194 cow_start_from = 0;
1195 }
1196 break;
1197 case QCOW2_SUBCLUSTER_ZERO_PLAIN:
1198 case QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN:
1199 cow_start_from = sc_index << s->subcluster_bits;
1200 break;
1201 default:
1202 g_assert_not_reached();
1203 }
1204 } else {
1205 switch (type) {
1206 case QCOW2_SUBCLUSTER_NORMAL:
1207 cow_start_from = cow_start_to;
1208 break;
1209 case QCOW2_SUBCLUSTER_ZERO_ALLOC:
1210 case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC:
1211 cow_start_from = sc_index << s->subcluster_bits;
1212 break;
1213 default:
1214 g_assert_not_reached();
1215 }
1216 }
1217
1218 /* Get the L2 entry of the last cluster */
1219 l2_index += nb_clusters - 1;
1220 l2_entry = get_l2_entry(s, l2_slice, l2_index);
1221 l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index);
1222 sc_index = offset_to_sc_index(s, guest_offset + bytes - 1);
1223 type = qcow2_get_subcluster_type(bs, l2_entry, l2_bitmap, sc_index);
1224
1225 if (!keep_old) {
1226 switch (type) {
1227 case QCOW2_SUBCLUSTER_COMPRESSED:
1228 cow_end_to = ROUND_UP(cow_end_from, s->cluster_size);
1229 break;
1230 case QCOW2_SUBCLUSTER_NORMAL:
1231 case QCOW2_SUBCLUSTER_ZERO_ALLOC:
1232 case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC:
1233 cow_end_to = ROUND_UP(cow_end_from, s->cluster_size);
1234 if (has_subclusters(s)) {
1235 /* Skip all trailing zero and unallocated subclusters */
1236 uint32_t alloc_bitmap = l2_bitmap & QCOW_L2_BITMAP_ALL_ALLOC;
1237 cow_end_to -=
1238 MIN(s->subclusters_per_cluster - sc_index - 1,
1239 clz32(alloc_bitmap)) << s->subcluster_bits;
1240 }
1241 break;
1242 case QCOW2_SUBCLUSTER_ZERO_PLAIN:
1243 case QCOW2_SUBCLUSTER_UNALLOCATED_PLAIN:
1244 cow_end_to = ROUND_UP(cow_end_from, s->subcluster_size);
1245 break;
1246 default:
1247 g_assert_not_reached();
1248 }
1249 } else {
1250 switch (type) {
1251 case QCOW2_SUBCLUSTER_NORMAL:
1252 cow_end_to = cow_end_from;
1253 break;
1254 case QCOW2_SUBCLUSTER_ZERO_ALLOC:
1255 case QCOW2_SUBCLUSTER_UNALLOCATED_ALLOC:
1256 cow_end_to = ROUND_UP(cow_end_from, s->subcluster_size);
1257 break;
1258 default:
1259 g_assert_not_reached();
1260 }
1261 }
1262
1263 *m = g_malloc0(sizeof(**m));
1264 **m = (QCowL2Meta) {
1265 .next = old_m,
1266
1267 .alloc_offset = host_cluster_offset,
1268 .offset = start_of_cluster(s, guest_offset),
1269 .nb_clusters = nb_clusters,
1270
1271 .keep_old_clusters = keep_old,
1272
1273 .cow_start = {
1274 .offset = cow_start_from,
1275 .nb_bytes = cow_start_to - cow_start_from,
1276 },
1277 .cow_end = {
1278 .offset = cow_end_from,
1279 .nb_bytes = cow_end_to - cow_end_from,
1280 },
1281 };
1282
1283 qemu_co_queue_init(&(*m)->dependent_requests);
1284 QLIST_INSERT_HEAD(&s->cluster_allocs, *m, next_in_flight);
1285
1286 return 0;
1287 }
1288
1289 /*
1290 * Returns true if writing to the cluster pointed to by @l2_entry
1291 * requires a new allocation (that is, if the cluster is unallocated
1292 * or has refcount > 1 and therefore cannot be written in-place).
1293 */
1294 static bool cluster_needs_new_alloc(BlockDriverState *bs, uint64_t l2_entry)
1295 {
1296 switch (qcow2_get_cluster_type(bs, l2_entry)) {
1297 case QCOW2_CLUSTER_NORMAL:
1298 case QCOW2_CLUSTER_ZERO_ALLOC:
1299 if (l2_entry & QCOW_OFLAG_COPIED) {
1300 return false;
1301 }
1302 case QCOW2_CLUSTER_UNALLOCATED:
1303 case QCOW2_CLUSTER_COMPRESSED:
1304 case QCOW2_CLUSTER_ZERO_PLAIN:
1305 return true;
1306 default:
1307 abort();
1308 }
1309 }
1310
1311 /*
1312 * Returns the number of contiguous clusters that can be written to
1313 * using one single write request, starting from @l2_index.
1314 * At most @nb_clusters are checked.
1315 *
1316 * If @new_alloc is true this counts clusters that are either
1317 * unallocated, or allocated but with refcount > 1 (so they need to be
1318 * newly allocated and COWed).
1319 *
1320 * If @new_alloc is false this counts clusters that are already
1321 * allocated and can be overwritten in-place (this includes clusters
1322 * of type QCOW2_CLUSTER_ZERO_ALLOC).
1323 */
1324 static int count_single_write_clusters(BlockDriverState *bs, int nb_clusters,
1325 uint64_t *l2_slice, int l2_index,
1326 bool new_alloc)
1327 {
1328 BDRVQcow2State *s = bs->opaque;
1329 uint64_t l2_entry = get_l2_entry(s, l2_slice, l2_index);
1330 uint64_t expected_offset = l2_entry & L2E_OFFSET_MASK;
1331 int i;
1332
1333 for (i = 0; i < nb_clusters; i++) {
1334 l2_entry = get_l2_entry(s, l2_slice, l2_index + i);
1335 if (cluster_needs_new_alloc(bs, l2_entry) != new_alloc) {
1336 break;
1337 }
1338 if (!new_alloc) {
1339 if (expected_offset != (l2_entry & L2E_OFFSET_MASK)) {
1340 break;
1341 }
1342 expected_offset += s->cluster_size;
1343 }
1344 }
1345
1346 assert(i <= nb_clusters);
1347 return i;
1348 }
1349
1350 /*
1351 * Check if there already is an AIO write request in flight which allocates
1352 * the same cluster. In this case we need to wait until the previous
1353 * request has completed and updated the L2 table accordingly.
1354 *
1355 * Returns:
1356 * 0 if there was no dependency. *cur_bytes indicates the number of
1357 * bytes from guest_offset that can be read before the next
1358 * dependency must be processed (or the request is complete)
1359 *
1360 * -EAGAIN if we had to wait for another request, previously gathered
1361 * information on cluster allocation may be invalid now. The caller
1362 * must start over anyway, so consider *cur_bytes undefined.
1363 */
1364 static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
1365 uint64_t *cur_bytes, QCowL2Meta **m)
1366 {
1367 BDRVQcow2State *s = bs->opaque;
1368 QCowL2Meta *old_alloc;
1369 uint64_t bytes = *cur_bytes;
1370
1371 QLIST_FOREACH(old_alloc, &s->cluster_allocs, next_in_flight) {
1372
1373 uint64_t start = guest_offset;
1374 uint64_t end = start + bytes;
1375 uint64_t old_start = start_of_cluster(s, l2meta_cow_start(old_alloc));
1376 uint64_t old_end = ROUND_UP(l2meta_cow_end(old_alloc), s->cluster_size);
1377
1378 if (end <= old_start || start >= old_end) {
1379 /* No intersection */
1380 } else {
1381 if (start < old_start) {
1382 /* Stop at the start of a running allocation */
1383 bytes = old_start - start;
1384 } else {
1385 bytes = 0;
1386 }
1387
1388 /* Stop if already an l2meta exists. After yielding, it wouldn't
1389 * be valid any more, so we'd have to clean up the old L2Metas
1390 * and deal with requests depending on them before starting to
1391 * gather new ones. Not worth the trouble. */
1392 if (bytes == 0 && *m) {
1393 *cur_bytes = 0;
1394 return 0;
1395 }
1396
1397 if (bytes == 0) {
1398 /* Wait for the dependency to complete. We need to recheck
1399 * the free/allocated clusters when we continue. */
1400 qemu_co_queue_wait(&old_alloc->dependent_requests, &s->lock);
1401 return -EAGAIN;
1402 }
1403 }
1404 }
1405
1406 /* Make sure that existing clusters and new allocations are only used up to
1407 * the next dependency if we shortened the request above */
1408 *cur_bytes = bytes;
1409
1410 return 0;
1411 }
1412
1413 /*
1414 * Checks how many already allocated clusters that don't require a new
1415 * allocation there are at the given guest_offset (up to *bytes).
1416 * If *host_offset is not INV_OFFSET, only physically contiguous clusters
1417 * beginning at this host offset are counted.
1418 *
1419 * Note that guest_offset may not be cluster aligned. In this case, the
1420 * returned *host_offset points to exact byte referenced by guest_offset and
1421 * therefore isn't cluster aligned as well.
1422 *
1423 * Returns:
1424 * 0: if no allocated clusters are available at the given offset.
1425 * *bytes is normally unchanged. It is set to 0 if the cluster
1426 * is allocated and can be overwritten in-place but doesn't have
1427 * the right physical offset.
1428 *
1429 * 1: if allocated clusters that can be overwritten in place are
1430 * available at the requested offset. *bytes may have decreased
1431 * and describes the length of the area that can be written to.
1432 *
1433 * -errno: in error cases
1434 */
1435 static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,
1436 uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
1437 {
1438 BDRVQcow2State *s = bs->opaque;
1439 int l2_index;
1440 uint64_t l2_entry, cluster_offset;
1441 uint64_t *l2_slice;
1442 uint64_t nb_clusters;
1443 unsigned int keep_clusters;
1444 int ret;
1445
1446 trace_qcow2_handle_copied(qemu_coroutine_self(), guest_offset, *host_offset,
1447 *bytes);
1448
1449 assert(*host_offset == INV_OFFSET || offset_into_cluster(s, guest_offset)
1450 == offset_into_cluster(s, *host_offset));
1451
1452 /*
1453 * Calculate the number of clusters to look for. We stop at L2 slice
1454 * boundaries to keep things simple.
1455 */
1456 nb_clusters =
1457 size_to_clusters(s, offset_into_cluster(s, guest_offset) + *bytes);
1458
1459 l2_index = offset_to_l2_slice_index(s, guest_offset);
1460 nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index);
1461 /* Limit total byte count to BDRV_REQUEST_MAX_BYTES */
1462 nb_clusters = MIN(nb_clusters, BDRV_REQUEST_MAX_BYTES >> s->cluster_bits);
1463
1464 /* Find L2 entry for the first involved cluster */
1465 ret = get_cluster_table(bs, guest_offset, &l2_slice, &l2_index);
1466 if (ret < 0) {
1467 return ret;
1468 }
1469
1470 l2_entry = get_l2_entry(s, l2_slice, l2_index);
1471 cluster_offset = l2_entry & L2E_OFFSET_MASK;
1472
1473 if (!cluster_needs_new_alloc(bs, l2_entry)) {
1474 if (offset_into_cluster(s, cluster_offset)) {
1475 qcow2_signal_corruption(bs, true, -1, -1, "%s cluster offset "
1476 "%#" PRIx64 " unaligned (guest offset: %#"
1477 PRIx64 ")", l2_entry & QCOW_OFLAG_ZERO ?
1478 "Preallocated zero" : "Data",
1479 cluster_offset, guest_offset);
1480 ret = -EIO;
1481 goto out;
1482 }
1483
1484 /* If a specific host_offset is required, check it */
1485 if (*host_offset != INV_OFFSET && cluster_offset != *host_offset) {
1486 *bytes = 0;
1487 ret = 0;
1488 goto out;
1489 }
1490
1491 /* We keep all QCOW_OFLAG_COPIED clusters */
1492 keep_clusters = count_single_write_clusters(bs, nb_clusters, l2_slice,
1493 l2_index, false);
1494 assert(keep_clusters <= nb_clusters);
1495
1496 *bytes = MIN(*bytes,
1497 keep_clusters * s->cluster_size
1498 - offset_into_cluster(s, guest_offset));
1499 assert(*bytes != 0);
1500
1501 ret = calculate_l2_meta(bs, cluster_offset, guest_offset,
1502 *bytes, l2_slice, m, true);
1503 if (ret < 0) {
1504 goto out;
1505 }
1506
1507 ret = 1;
1508 } else {
1509 ret = 0;
1510 }
1511
1512 /* Cleanup */
1513 out:
1514 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
1515
1516 /* Only return a host offset if we actually made progress. Otherwise we
1517 * would make requirements for handle_alloc() that it can't fulfill */
1518 if (ret > 0) {
1519 *host_offset = cluster_offset + offset_into_cluster(s, guest_offset);
1520 }
1521
1522 return ret;
1523 }
1524
1525 /*
1526 * Allocates new clusters for the given guest_offset.
1527 *
1528 * At most *nb_clusters are allocated, and on return *nb_clusters is updated to
1529 * contain the number of clusters that have been allocated and are contiguous
1530 * in the image file.
1531 *
1532 * If *host_offset is not INV_OFFSET, it specifies the offset in the image file
1533 * at which the new clusters must start. *nb_clusters can be 0 on return in
1534 * this case if the cluster at host_offset is already in use. If *host_offset
1535 * is INV_OFFSET, the clusters can be allocated anywhere in the image file.
1536 *
1537 * *host_offset is updated to contain the offset into the image file at which
1538 * the first allocated cluster starts.
1539 *
1540 * Return 0 on success and -errno in error cases. -EAGAIN means that the
1541 * function has been waiting for another request and the allocation must be
1542 * restarted, but the whole request should not be failed.
1543 */
1544 static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
1545 uint64_t *host_offset, uint64_t *nb_clusters)
1546 {
1547 BDRVQcow2State *s = bs->opaque;
1548
1549 trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
1550 *host_offset, *nb_clusters);
1551
1552 if (has_data_file(bs)) {
1553 assert(*host_offset == INV_OFFSET ||
1554 *host_offset == start_of_cluster(s, guest_offset));
1555 *host_offset = start_of_cluster(s, guest_offset);
1556 return 0;
1557 }
1558
1559 /* Allocate new clusters */
1560 trace_qcow2_cluster_alloc_phys(qemu_coroutine_self());
1561 if (*host_offset == INV_OFFSET) {
1562 int64_t cluster_offset =
1563 qcow2_alloc_clusters(bs, *nb_clusters * s->cluster_size);
1564 if (cluster_offset < 0) {
1565 return cluster_offset;
1566 }
1567 *host_offset = cluster_offset;
1568 return 0;
1569 } else {
1570 int64_t ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
1571 if (ret < 0) {
1572 return ret;
1573 }
1574 *nb_clusters = ret;
1575 return 0;
1576 }
1577 }
1578
1579 /*
1580 * Allocates new clusters for an area that is either still unallocated or
1581 * cannot be overwritten in-place. If *host_offset is not INV_OFFSET,
1582 * clusters are only allocated if the new allocation can match the specified
1583 * host offset.
1584 *
1585 * Note that guest_offset may not be cluster aligned. In this case, the
1586 * returned *host_offset points to exact byte referenced by guest_offset and
1587 * therefore isn't cluster aligned as well.
1588 *
1589 * Returns:
1590 * 0: if no clusters could be allocated. *bytes is set to 0,
1591 * *host_offset is left unchanged.
1592 *
1593 * 1: if new clusters were allocated. *bytes may be decreased if the
1594 * new allocation doesn't cover all of the requested area.
1595 * *host_offset is updated to contain the host offset of the first
1596 * newly allocated cluster.
1597 *
1598 * -errno: in error cases
1599 */
1600 static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,
1601 uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
1602 {
1603 BDRVQcow2State *s = bs->opaque;
1604 int l2_index;
1605 uint64_t *l2_slice;
1606 uint64_t nb_clusters;
1607 int ret;
1608
1609 uint64_t alloc_cluster_offset;
1610
1611 trace_qcow2_handle_alloc(qemu_coroutine_self(), guest_offset, *host_offset,
1612 *bytes);
1613 assert(*bytes > 0);
1614
1615 /*
1616 * Calculate the number of clusters to look for. We stop at L2 slice
1617 * boundaries to keep things simple.
1618 */
1619 nb_clusters =
1620 size_to_clusters(s, offset_into_cluster(s, guest_offset) + *bytes);
1621
1622 l2_index = offset_to_l2_slice_index(s, guest_offset);
1623 nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index);
1624 /* Limit total allocation byte count to BDRV_REQUEST_MAX_BYTES */
1625 nb_clusters = MIN(nb_clusters, BDRV_REQUEST_MAX_BYTES >> s->cluster_bits);
1626
1627 /* Find L2 entry for the first involved cluster */
1628 ret = get_cluster_table(bs, guest_offset, &l2_slice, &l2_index);
1629 if (ret < 0) {
1630 return ret;
1631 }
1632
1633 nb_clusters = count_single_write_clusters(bs, nb_clusters,
1634 l2_slice, l2_index, true);
1635
1636 /* This function is only called when there were no non-COW clusters, so if
1637 * we can't find any unallocated or COW clusters either, something is
1638 * wrong with our code. */
1639 assert(nb_clusters > 0);
1640
1641 /* Allocate at a given offset in the image file */
1642 alloc_cluster_offset = *host_offset == INV_OFFSET ? INV_OFFSET :
1643 start_of_cluster(s, *host_offset);
1644 ret = do_alloc_cluster_offset(bs, guest_offset, &alloc_cluster_offset,
1645 &nb_clusters);
1646 if (ret < 0) {
1647 goto out;
1648 }
1649
1650 /* Can't extend contiguous allocation */
1651 if (nb_clusters == 0) {
1652 *bytes = 0;
1653 ret = 0;
1654 goto out;
1655 }
1656
1657 assert(alloc_cluster_offset != INV_OFFSET);
1658
1659 /*
1660 * Save info needed for meta data update.
1661 *
1662 * requested_bytes: Number of bytes from the start of the first
1663 * newly allocated cluster to the end of the (possibly shortened
1664 * before) write request.
1665 *
1666 * avail_bytes: Number of bytes from the start of the first
1667 * newly allocated to the end of the last newly allocated cluster.
1668 *
1669 * nb_bytes: The number of bytes from the start of the first
1670 * newly allocated cluster to the end of the area that the write
1671 * request actually writes to (excluding COW at the end)
1672 */
1673 uint64_t requested_bytes = *bytes + offset_into_cluster(s, guest_offset);
1674 int avail_bytes = nb_clusters << s->cluster_bits;
1675 int nb_bytes = MIN(requested_bytes, avail_bytes);
1676
1677 *host_offset = alloc_cluster_offset + offset_into_cluster(s, guest_offset);
1678 *bytes = MIN(*bytes, nb_bytes - offset_into_cluster(s, guest_offset));
1679 assert(*bytes != 0);
1680
1681 ret = calculate_l2_meta(bs, alloc_cluster_offset, guest_offset, *bytes,
1682 l2_slice, m, false);
1683 if (ret < 0) {
1684 goto out;
1685 }
1686
1687 ret = 1;
1688
1689 out:
1690 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
1691 if (ret < 0 && *m && (*m)->nb_clusters > 0) {
1692 QLIST_REMOVE(*m, next_in_flight);
1693 }
1694 return ret;
1695 }
1696
1697 /*
1698 * alloc_cluster_offset
1699 *
1700 * For a given offset on the virtual disk, find the cluster offset in qcow2
1701 * file. If the offset is not found, allocate a new cluster.
1702 *
1703 * If the cluster was already allocated, m->nb_clusters is set to 0 and
1704 * other fields in m are meaningless.
1705 *
1706 * If the cluster is newly allocated, m->nb_clusters is set to the number of
1707 * contiguous clusters that have been allocated. In this case, the other
1708 * fields of m are valid and contain information about the first allocated
1709 * cluster.
1710 *
1711 * If the request conflicts with another write request in flight, the coroutine
1712 * is queued and will be reentered when the dependency has completed.
1713 *
1714 * Return 0 on success and -errno in error cases
1715 */
1716 int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
1717 unsigned int *bytes, uint64_t *host_offset,
1718 QCowL2Meta **m)
1719 {
1720 BDRVQcow2State *s = bs->opaque;
1721 uint64_t start, remaining;
1722 uint64_t cluster_offset;
1723 uint64_t cur_bytes;
1724 int ret;
1725
1726 trace_qcow2_alloc_clusters_offset(qemu_coroutine_self(), offset, *bytes);
1727
1728 again:
1729 start = offset;
1730 remaining = *bytes;
1731 cluster_offset = INV_OFFSET;
1732 *host_offset = INV_OFFSET;
1733 cur_bytes = 0;
1734 *m = NULL;
1735
1736 while (true) {
1737
1738 if (*host_offset == INV_OFFSET && cluster_offset != INV_OFFSET) {
1739 *host_offset = start_of_cluster(s, cluster_offset);
1740 }
1741
1742 assert(remaining >= cur_bytes);
1743
1744 start += cur_bytes;
1745 remaining -= cur_bytes;
1746
1747 if (cluster_offset != INV_OFFSET) {
1748 cluster_offset += cur_bytes;
1749 }
1750
1751 if (remaining == 0) {
1752 break;
1753 }
1754
1755 cur_bytes = remaining;
1756
1757 /*
1758 * Now start gathering as many contiguous clusters as possible:
1759 *
1760 * 1. Check for overlaps with in-flight allocations
1761 *
1762 * a) Overlap not in the first cluster -> shorten this request and
1763 * let the caller handle the rest in its next loop iteration.
1764 *
1765 * b) Real overlaps of two requests. Yield and restart the search
1766 * for contiguous clusters (the situation could have changed
1767 * while we were sleeping)
1768 *
1769 * c) TODO: Request starts in the same cluster as the in-flight
1770 * allocation ends. Shorten the COW of the in-fight allocation,
1771 * set cluster_offset to write to the same cluster and set up
1772 * the right synchronisation between the in-flight request and
1773 * the new one.
1774 */
1775 ret = handle_dependencies(bs, start, &cur_bytes, m);
1776 if (ret == -EAGAIN) {
1777 /* Currently handle_dependencies() doesn't yield if we already had
1778 * an allocation. If it did, we would have to clean up the L2Meta
1779 * structs before starting over. */
1780 assert(*m == NULL);
1781 goto again;
1782 } else if (ret < 0) {
1783 return ret;
1784 } else if (cur_bytes == 0) {
1785 break;
1786 } else {
1787 /* handle_dependencies() may have decreased cur_bytes (shortened
1788 * the allocations below) so that the next dependency is processed
1789 * correctly during the next loop iteration. */
1790 }
1791
1792 /*
1793 * 2. Count contiguous COPIED clusters.
1794 */
1795 ret = handle_copied(bs, start, &cluster_offset, &cur_bytes, m);
1796 if (ret < 0) {
1797 return ret;
1798 } else if (ret) {
1799 continue;
1800 } else if (cur_bytes == 0) {
1801 break;
1802 }
1803
1804 /*
1805 * 3. If the request still hasn't completed, allocate new clusters,
1806 * considering any cluster_offset of steps 1c or 2.
1807 */
1808 ret = handle_alloc(bs, start, &cluster_offset, &cur_bytes, m);
1809 if (ret < 0) {
1810 return ret;
1811 } else if (ret) {
1812 continue;
1813 } else {
1814 assert(cur_bytes == 0);
1815 break;
1816 }
1817 }
1818
1819 *bytes -= remaining;
1820 assert(*bytes > 0);
1821 assert(*host_offset != INV_OFFSET);
1822
1823 return 0;
1824 }
1825
1826 /*
1827 * This discards as many clusters of nb_clusters as possible at once (i.e.
1828 * all clusters in the same L2 slice) and returns the number of discarded
1829 * clusters.
1830 */
1831 static int discard_in_l2_slice(BlockDriverState *bs, uint64_t offset,
1832 uint64_t nb_clusters,
1833 enum qcow2_discard_type type, bool full_discard)
1834 {
1835 BDRVQcow2State *s = bs->opaque;
1836 uint64_t *l2_slice;
1837 int l2_index;
1838 int ret;
1839 int i;
1840
1841 ret = get_cluster_table(bs, offset, &l2_slice, &l2_index);
1842 if (ret < 0) {
1843 return ret;
1844 }
1845
1846 /* Limit nb_clusters to one L2 slice */
1847 nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index);
1848 assert(nb_clusters <= INT_MAX);
1849
1850 for (i = 0; i < nb_clusters; i++) {
1851 uint64_t old_l2_entry;
1852
1853 old_l2_entry = get_l2_entry(s, l2_slice, l2_index + i);
1854
1855 /*
1856 * If full_discard is false, make sure that a discarded area reads back
1857 * as zeroes for v3 images (we cannot do it for v2 without actually
1858 * writing a zero-filled buffer). We can skip the operation if the
1859 * cluster is already marked as zero, or if it's unallocated and we
1860 * don't have a backing file.
1861 *
1862 * TODO We might want to use bdrv_block_status(bs) here, but we're
1863 * holding s->lock, so that doesn't work today.
1864 *
1865 * If full_discard is true, the sector should not read back as zeroes,
1866 * but rather fall through to the backing file.
1867 */
1868 switch (qcow2_get_cluster_type(bs, old_l2_entry)) {
1869 case QCOW2_CLUSTER_UNALLOCATED:
1870 if (full_discard || !bs->backing) {
1871 continue;
1872 }
1873 break;
1874
1875 case QCOW2_CLUSTER_ZERO_PLAIN:
1876 if (!full_discard) {
1877 continue;
1878 }
1879 break;
1880
1881 case QCOW2_CLUSTER_ZERO_ALLOC:
1882 case QCOW2_CLUSTER_NORMAL:
1883 case QCOW2_CLUSTER_COMPRESSED:
1884 break;
1885
1886 default:
1887 abort();
1888 }
1889
1890 /* First remove L2 entries */
1891 qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
1892 if (!full_discard && s->qcow_version >= 3) {
1893 set_l2_entry(s, l2_slice, l2_index + i, QCOW_OFLAG_ZERO);
1894 } else {
1895 set_l2_entry(s, l2_slice, l2_index + i, 0);
1896 }
1897
1898 /* Then decrease the refcount */
1899 qcow2_free_any_clusters(bs, old_l2_entry, 1, type);
1900 }
1901
1902 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
1903
1904 return nb_clusters;
1905 }
1906
1907 int qcow2_cluster_discard(BlockDriverState *bs, uint64_t offset,
1908 uint64_t bytes, enum qcow2_discard_type type,
1909 bool full_discard)
1910 {
1911 BDRVQcow2State *s = bs->opaque;
1912 uint64_t end_offset = offset + bytes;
1913 uint64_t nb_clusters;
1914 int64_t cleared;
1915 int ret;
1916
1917 /* Caller must pass aligned values, except at image end */
1918 assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
1919 assert(QEMU_IS_ALIGNED(end_offset, s->cluster_size) ||
1920 end_offset == bs->total_sectors << BDRV_SECTOR_BITS);
1921
1922 nb_clusters = size_to_clusters(s, bytes);
1923
1924 s->cache_discards = true;
1925
1926 /* Each L2 slice is handled by its own loop iteration */
1927 while (nb_clusters > 0) {
1928 cleared = discard_in_l2_slice(bs, offset, nb_clusters, type,
1929 full_discard);
1930 if (cleared < 0) {
1931 ret = cleared;
1932 goto fail;
1933 }
1934
1935 nb_clusters -= cleared;
1936 offset += (cleared * s->cluster_size);
1937 }
1938
1939 ret = 0;
1940 fail:
1941 s->cache_discards = false;
1942 qcow2_process_discards(bs, ret);
1943
1944 return ret;
1945 }
1946
1947 /*
1948 * This zeroes as many clusters of nb_clusters as possible at once (i.e.
1949 * all clusters in the same L2 slice) and returns the number of zeroed
1950 * clusters.
1951 */
1952 static int zero_in_l2_slice(BlockDriverState *bs, uint64_t offset,
1953 uint64_t nb_clusters, int flags)
1954 {
1955 BDRVQcow2State *s = bs->opaque;
1956 uint64_t *l2_slice;
1957 int l2_index;
1958 int ret;
1959 int i;
1960
1961 ret = get_cluster_table(bs, offset, &l2_slice, &l2_index);
1962 if (ret < 0) {
1963 return ret;
1964 }
1965
1966 /* Limit nb_clusters to one L2 slice */
1967 nb_clusters = MIN(nb_clusters, s->l2_slice_size - l2_index);
1968 assert(nb_clusters <= INT_MAX);
1969
1970 for (i = 0; i < nb_clusters; i++) {
1971 uint64_t old_l2_entry = get_l2_entry(s, l2_slice, l2_index + i);
1972 uint64_t old_l2_bitmap = get_l2_bitmap(s, l2_slice, l2_index + i);
1973 QCow2ClusterType type = qcow2_get_cluster_type(bs, old_l2_entry);
1974 bool unmap = (type == QCOW2_CLUSTER_COMPRESSED) ||
1975 ((flags & BDRV_REQ_MAY_UNMAP) && qcow2_cluster_is_allocated(type));
1976 uint64_t new_l2_entry = unmap ? 0 : old_l2_entry;
1977 uint64_t new_l2_bitmap = old_l2_bitmap;
1978
1979 if (has_subclusters(s)) {
1980 new_l2_bitmap = QCOW_L2_BITMAP_ALL_ZEROES;
1981 } else {
1982 new_l2_entry |= QCOW_OFLAG_ZERO;
1983 }
1984
1985 if (old_l2_entry == new_l2_entry && old_l2_bitmap == new_l2_bitmap) {
1986 continue;
1987 }
1988
1989 qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
1990 if (unmap) {
1991 qcow2_free_any_clusters(bs, old_l2_entry, 1, QCOW2_DISCARD_REQUEST);
1992 }
1993 set_l2_entry(s, l2_slice, l2_index + i, new_l2_entry);
1994 if (has_subclusters(s)) {
1995 set_l2_bitmap(s, l2_slice, l2_index + i, new_l2_bitmap);
1996 }
1997 }
1998
1999 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
2000
2001 return nb_clusters;
2002 }
2003
2004 int qcow2_cluster_zeroize(BlockDriverState *bs, uint64_t offset,
2005 uint64_t bytes, int flags)
2006 {
2007 BDRVQcow2State *s = bs->opaque;
2008 uint64_t end_offset = offset + bytes;
2009 uint64_t nb_clusters;
2010 int64_t cleared;
2011 int ret;
2012
2013 /* If we have to stay in sync with an external data file, zero out
2014 * s->data_file first. */
2015 if (data_file_is_raw(bs)) {
2016 assert(has_data_file(bs));
2017 ret = bdrv_co_pwrite_zeroes(s->data_file, offset, bytes, flags);
2018 if (ret < 0) {
2019 return ret;
2020 }
2021 }
2022
2023 /* Caller must pass aligned values, except at image end */
2024 assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
2025 assert(QEMU_IS_ALIGNED(end_offset, s->cluster_size) ||
2026 end_offset >= bs->total_sectors << BDRV_SECTOR_BITS);
2027
2028 /*
2029 * The zero flag is only supported by version 3 and newer. However, if we
2030 * have no backing file, we can resort to discard in version 2.
2031 */
2032 if (s->qcow_version < 3) {
2033 if (!bs->backing) {
2034 return qcow2_cluster_discard(bs, offset, bytes,
2035 QCOW2_DISCARD_REQUEST, false);
2036 }
2037 return -ENOTSUP;
2038 }
2039
2040 /* Each L2 slice is handled by its own loop iteration */
2041 nb_clusters = size_to_clusters(s, bytes);
2042
2043 s->cache_discards = true;
2044
2045 while (nb_clusters > 0) {
2046 cleared = zero_in_l2_slice(bs, offset, nb_clusters, flags);
2047 if (cleared < 0) {
2048 ret = cleared;
2049 goto fail;
2050 }
2051
2052 nb_clusters -= cleared;
2053 offset += (cleared * s->cluster_size);
2054 }
2055
2056 ret = 0;
2057 fail:
2058 s->cache_discards = false;
2059 qcow2_process_discards(bs, ret);
2060
2061 return ret;
2062 }
2063
2064 /*
2065 * Expands all zero clusters in a specific L1 table (or deallocates them, for
2066 * non-backed non-pre-allocated zero clusters).
2067 *
2068 * l1_entries and *visited_l1_entries are used to keep track of progress for
2069 * status_cb(). l1_entries contains the total number of L1 entries and
2070 * *visited_l1_entries counts all visited L1 entries.
2071 */
2072 static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
2073 int l1_size, int64_t *visited_l1_entries,
2074 int64_t l1_entries,
2075 BlockDriverAmendStatusCB *status_cb,
2076 void *cb_opaque)
2077 {
2078 BDRVQcow2State *s = bs->opaque;
2079 bool is_active_l1 = (l1_table == s->l1_table);
2080 uint64_t *l2_slice = NULL;
2081 unsigned slice, slice_size2, n_slices;
2082 int ret;
2083 int i, j;
2084
2085 slice_size2 = s->l2_slice_size * l2_entry_size(s);
2086 n_slices = s->cluster_size / slice_size2;
2087
2088 if (!is_active_l1) {
2089 /* inactive L2 tables require a buffer to be stored in when loading
2090 * them from disk */
2091 l2_slice = qemu_try_blockalign(bs->file->bs, slice_size2);
2092 if (l2_slice == NULL) {
2093 return -ENOMEM;
2094 }
2095 }
2096
2097 for (i = 0; i < l1_size; i++) {
2098 uint64_t l2_offset = l1_table[i] & L1E_OFFSET_MASK;
2099 uint64_t l2_refcount;
2100
2101 if (!l2_offset) {
2102 /* unallocated */
2103 (*visited_l1_entries)++;
2104 if (status_cb) {
2105 status_cb(bs, *visited_l1_entries, l1_entries, cb_opaque);
2106 }
2107 continue;
2108 }
2109
2110 if (offset_into_cluster(s, l2_offset)) {
2111 qcow2_signal_corruption(bs, true, -1, -1, "L2 table offset %#"
2112 PRIx64 " unaligned (L1 index: %#x)",
2113 l2_offset, i);
2114 ret = -EIO;
2115 goto fail;
2116 }
2117
2118 ret = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits,
2119 &l2_refcount);
2120 if (ret < 0) {
2121 goto fail;
2122 }
2123
2124 for (slice = 0; slice < n_slices; slice++) {
2125 uint64_t slice_offset = l2_offset + slice * slice_size2;
2126 bool l2_dirty = false;
2127 if (is_active_l1) {
2128 /* get active L2 tables from cache */
2129 ret = qcow2_cache_get(bs, s->l2_table_cache, slice_offset,
2130 (void **)&l2_slice);
2131 } else {
2132 /* load inactive L2 tables from disk */
2133 ret = bdrv_pread(bs->file, slice_offset, l2_slice, slice_size2);
2134 }
2135 if (ret < 0) {
2136 goto fail;
2137 }
2138
2139 for (j = 0; j < s->l2_slice_size; j++) {
2140 uint64_t l2_entry = get_l2_entry(s, l2_slice, j);
2141 int64_t offset = l2_entry & L2E_OFFSET_MASK;
2142 QCow2ClusterType cluster_type =
2143 qcow2_get_cluster_type(bs, l2_entry);
2144
2145 if (cluster_type != QCOW2_CLUSTER_ZERO_PLAIN &&
2146 cluster_type != QCOW2_CLUSTER_ZERO_ALLOC) {
2147 continue;
2148 }
2149
2150 if (cluster_type == QCOW2_CLUSTER_ZERO_PLAIN) {
2151 if (!bs->backing) {
2152 /* not backed; therefore we can simply deallocate the
2153 * cluster */
2154 set_l2_entry(s, l2_slice, j, 0);
2155 l2_dirty = true;
2156 continue;
2157 }
2158
2159 offset = qcow2_alloc_clusters(bs, s->cluster_size);
2160 if (offset < 0) {
2161 ret = offset;
2162 goto fail;
2163 }
2164
2165 /* The offset must fit in the offset field */
2166 assert((offset & L2E_OFFSET_MASK) == offset);
2167
2168 if (l2_refcount > 1) {
2169 /* For shared L2 tables, set the refcount accordingly
2170 * (it is already 1 and needs to be l2_refcount) */
2171 ret = qcow2_update_cluster_refcount(
2172 bs, offset >> s->cluster_bits,
2173 refcount_diff(1, l2_refcount), false,
2174 QCOW2_DISCARD_OTHER);
2175 if (ret < 0) {
2176 qcow2_free_clusters(bs, offset, s->cluster_size,
2177 QCOW2_DISCARD_OTHER);
2178 goto fail;
2179 }
2180 }
2181 }
2182
2183 if (offset_into_cluster(s, offset)) {
2184 int l2_index = slice * s->l2_slice_size + j;
2185 qcow2_signal_corruption(
2186 bs, true, -1, -1,
2187 "Cluster allocation offset "
2188 "%#" PRIx64 " unaligned (L2 offset: %#"
2189 PRIx64 ", L2 index: %#x)", offset,
2190 l2_offset, l2_index);
2191 if (cluster_type == QCOW2_CLUSTER_ZERO_PLAIN) {
2192 qcow2_free_clusters(bs, offset, s->cluster_size,
2193 QCOW2_DISCARD_ALWAYS);
2194 }
2195 ret = -EIO;
2196 goto fail;
2197 }
2198
2199 ret = qcow2_pre_write_overlap_check(bs, 0, offset,
2200 s->cluster_size, true);
2201 if (ret < 0) {
2202 if (cluster_type == QCOW2_CLUSTER_ZERO_PLAIN) {
2203 qcow2_free_clusters(bs, offset, s->cluster_size,
2204 QCOW2_DISCARD_ALWAYS);
2205 }
2206 goto fail;
2207 }
2208
2209 ret = bdrv_pwrite_zeroes(s->data_file, offset,
2210 s->cluster_size, 0);
2211 if (ret < 0) {
2212 if (cluster_type == QCOW2_CLUSTER_ZERO_PLAIN) {
2213 qcow2_free_clusters(bs, offset, s->cluster_size,
2214 QCOW2_DISCARD_ALWAYS);
2215 }
2216 goto fail;
2217 }
2218
2219 if (l2_refcount == 1) {
2220 set_l2_entry(s, l2_slice, j, offset | QCOW_OFLAG_COPIED);
2221 } else {
2222 set_l2_entry(s, l2_slice, j, offset);
2223 }
2224 l2_dirty = true;
2225 }
2226
2227 if (is_active_l1) {
2228 if (l2_dirty) {
2229 qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_slice);
2230 qcow2_cache_depends_on_flush(s->l2_table_cache);
2231 }
2232 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
2233 } else {
2234 if (l2_dirty) {
2235 ret = qcow2_pre_write_overlap_check(
2236 bs, QCOW2_OL_INACTIVE_L2 | QCOW2_OL_ACTIVE_L2,
2237 slice_offset, slice_size2, false);
2238 if (ret < 0) {
2239 goto fail;
2240 }
2241
2242 ret = bdrv_pwrite(bs->file, slice_offset,
2243 l2_slice, slice_size2);
2244 if (ret < 0) {
2245 goto fail;
2246 }
2247 }
2248 }
2249 }
2250
2251 (*visited_l1_entries)++;
2252 if (status_cb) {
2253 status_cb(bs, *visited_l1_entries, l1_entries, cb_opaque);
2254 }
2255 }
2256
2257 ret = 0;
2258
2259 fail:
2260 if (l2_slice) {
2261 if (!is_active_l1) {
2262 qemu_vfree(l2_slice);
2263 } else {
2264 qcow2_cache_put(s->l2_table_cache, (void **) &l2_slice);
2265 }
2266 }
2267 return ret;
2268 }
2269
2270 /*
2271 * For backed images, expands all zero clusters on the image. For non-backed
2272 * images, deallocates all non-pre-allocated zero clusters (and claims the
2273 * allocation for pre-allocated ones). This is important for downgrading to a
2274 * qcow2 version which doesn't yet support metadata zero clusters.
2275 */
2276 int qcow2_expand_zero_clusters(BlockDriverState *bs,
2277 BlockDriverAmendStatusCB *status_cb,
2278 void *cb_opaque)
2279 {
2280 BDRVQcow2State *s = bs->opaque;
2281 uint64_t *l1_table = NULL;
2282 int64_t l1_entries = 0, visited_l1_entries = 0;
2283 int ret;
2284 int i, j;
2285
2286 if (status_cb) {
2287 l1_entries = s->l1_size;
2288 for (i = 0; i < s->nb_snapshots; i++) {
2289 l1_entries += s->snapshots[i].l1_size;
2290 }
2291 }
2292
2293 ret = expand_zero_clusters_in_l1(bs, s->l1_table, s->l1_size,
2294 &visited_l1_entries, l1_entries,
2295 status_cb, cb_opaque);
2296 if (ret < 0) {
2297 goto fail;
2298 }
2299
2300 /* Inactive L1 tables may point to active L2 tables - therefore it is
2301 * necessary to flush the L2 table cache before trying to access the L2
2302 * tables pointed to by inactive L1 entries (else we might try to expand
2303 * zero clusters that have already been expanded); furthermore, it is also
2304 * necessary to empty the L2 table cache, since it may contain tables which
2305 * are now going to be modified directly on disk, bypassing the cache.
2306 * qcow2_cache_empty() does both for us. */
2307 ret = qcow2_cache_empty(bs, s->l2_table_cache);
2308 if (ret < 0) {
2309 goto fail;
2310 }
2311
2312 for (i = 0; i < s->nb_snapshots; i++) {
2313 int l1_size2;
2314 uint64_t *new_l1_table;
2315 Error *local_err = NULL;
2316
2317 ret = qcow2_validate_table(bs, s->snapshots[i].l1_table_offset,
2318 s->snapshots[i].l1_size, sizeof(uint64_t),
2319 QCOW_MAX_L1_SIZE, "Snapshot L1 table",
2320 &local_err);
2321 if (ret < 0) {
2322 error_report_err(local_err);
2323 goto fail;
2324 }
2325
2326 l1_size2 = s->snapshots[i].l1_size * sizeof(uint64_t);
2327 new_l1_table = g_try_realloc(l1_table, l1_size2);
2328
2329 if (!new_l1_table) {
2330 ret = -ENOMEM;
2331 goto fail;
2332 }
2333
2334 l1_table = new_l1_table;
2335
2336 ret = bdrv_pread(bs->file, s->snapshots[i].l1_table_offset,
2337 l1_table, l1_size2);
2338 if (ret < 0) {
2339 goto fail;
2340 }
2341
2342 for (j = 0; j < s->snapshots[i].l1_size; j++) {
2343 be64_to_cpus(&l1_table[j]);
2344 }
2345
2346 ret = expand_zero_clusters_in_l1(bs, l1_table, s->snapshots[i].l1_size,
2347 &visited_l1_entries, l1_entries,
2348 status_cb, cb_opaque);
2349 if (ret < 0) {
2350 goto fail;
2351 }
2352 }
2353
2354 ret = 0;
2355
2356 fail:
2357 g_free(l1_table);
2358 return ret;
2359 }