hw/arm/raspi: fix CPRMAN base address
[qemu.git] / hw / virtio / virtio-mem.c
1 /*
2 * Virtio MEM device
3 *
4 * Copyright (C) 2020 Red Hat, Inc.
5 *
6 * Authors:
7 * David Hildenbrand <david@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2.
10 * See the COPYING file in the top-level directory.
11 */
12
13 #include "qemu/osdep.h"
14 #include "qemu-common.h"
15 #include "qemu/iov.h"
16 #include "qemu/cutils.h"
17 #include "qemu/error-report.h"
18 #include "qemu/units.h"
19 #include "sysemu/numa.h"
20 #include "sysemu/sysemu.h"
21 #include "sysemu/reset.h"
22 #include "hw/virtio/virtio.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "hw/virtio/virtio-access.h"
25 #include "hw/virtio/virtio-mem.h"
26 #include "qapi/error.h"
27 #include "qapi/visitor.h"
28 #include "exec/ram_addr.h"
29 #include "migration/misc.h"
30 #include "hw/boards.h"
31 #include "hw/qdev-properties.h"
32 #include CONFIG_DEVICES
33 #include "trace.h"
34
35 /*
36 * Use QEMU_VMALLOC_ALIGN, so no THP will have to be split when unplugging
37 * memory (e.g., 2MB on x86_64).
38 */
39 #define VIRTIO_MEM_MIN_BLOCK_SIZE ((uint32_t)QEMU_VMALLOC_ALIGN)
40 /*
41 * Size the usable region bigger than the requested size if possible. Esp.
42 * Linux guests will only add (aligned) memory blocks in case they fully
43 * fit into the usable region, but plug+online only a subset of the pages.
44 * The memory block size corresponds mostly to the section size.
45 *
46 * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and
47 * a section size of 1GB on arm64 (as long as the start address is properly
48 * aligned, similar to ordinary DIMMs).
49 *
50 * We can change this at any time and maybe even make it configurable if
51 * necessary (as the section size can change). But it's more likely that the
52 * section size will rather get smaller and not bigger over time.
53 */
54 #if defined(TARGET_X86_64) || defined(TARGET_I386)
55 #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB))
56 #else
57 #error VIRTIO_MEM_USABLE_EXTENT not defined
58 #endif
59
60 static bool virtio_mem_is_busy(void)
61 {
62 /*
63 * Postcopy cannot handle concurrent discards and we don't want to migrate
64 * pages on-demand with stale content when plugging new blocks.
65 *
66 * For precopy, we don't want unplugged blocks in our migration stream, and
67 * when plugging new blocks, the page content might differ between source
68 * and destination (observable by the guest when not initializing pages
69 * after plugging them) until we're running on the destination (as we didn't
70 * migrate these blocks when they were unplugged).
71 */
72 return migration_in_incoming_postcopy() || !migration_is_idle();
73 }
74
75 static bool virtio_mem_test_bitmap(VirtIOMEM *vmem, uint64_t start_gpa,
76 uint64_t size, bool plugged)
77 {
78 const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size;
79 const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1;
80 unsigned long found_bit;
81
82 /* We fake a shorter bitmap to avoid searching too far. */
83 if (plugged) {
84 found_bit = find_next_zero_bit(vmem->bitmap, last_bit + 1, first_bit);
85 } else {
86 found_bit = find_next_bit(vmem->bitmap, last_bit + 1, first_bit);
87 }
88 return found_bit > last_bit;
89 }
90
91 static void virtio_mem_set_bitmap(VirtIOMEM *vmem, uint64_t start_gpa,
92 uint64_t size, bool plugged)
93 {
94 const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size;
95 const unsigned long nbits = size / vmem->block_size;
96
97 if (plugged) {
98 bitmap_set(vmem->bitmap, bit, nbits);
99 } else {
100 bitmap_clear(vmem->bitmap, bit, nbits);
101 }
102 }
103
104 static void virtio_mem_send_response(VirtIOMEM *vmem, VirtQueueElement *elem,
105 struct virtio_mem_resp *resp)
106 {
107 VirtIODevice *vdev = VIRTIO_DEVICE(vmem);
108 VirtQueue *vq = vmem->vq;
109
110 trace_virtio_mem_send_response(le16_to_cpu(resp->type));
111 iov_from_buf(elem->in_sg, elem->in_num, 0, resp, sizeof(*resp));
112
113 virtqueue_push(vq, elem, sizeof(*resp));
114 virtio_notify(vdev, vq);
115 }
116
117 static void virtio_mem_send_response_simple(VirtIOMEM *vmem,
118 VirtQueueElement *elem,
119 uint16_t type)
120 {
121 struct virtio_mem_resp resp = {
122 .type = cpu_to_le16(type),
123 };
124
125 virtio_mem_send_response(vmem, elem, &resp);
126 }
127
128 static bool virtio_mem_valid_range(VirtIOMEM *vmem, uint64_t gpa, uint64_t size)
129 {
130 if (!QEMU_IS_ALIGNED(gpa, vmem->block_size)) {
131 return false;
132 }
133 if (gpa + size < gpa || !size) {
134 return false;
135 }
136 if (gpa < vmem->addr || gpa >= vmem->addr + vmem->usable_region_size) {
137 return false;
138 }
139 if (gpa + size > vmem->addr + vmem->usable_region_size) {
140 return false;
141 }
142 return true;
143 }
144
145 static int virtio_mem_set_block_state(VirtIOMEM *vmem, uint64_t start_gpa,
146 uint64_t size, bool plug)
147 {
148 const uint64_t offset = start_gpa - vmem->addr;
149 int ret;
150
151 if (virtio_mem_is_busy()) {
152 return -EBUSY;
153 }
154
155 if (!plug) {
156 ret = ram_block_discard_range(vmem->memdev->mr.ram_block, offset, size);
157 if (ret) {
158 error_report("Unexpected error discarding RAM: %s",
159 strerror(-ret));
160 return -EBUSY;
161 }
162 }
163 virtio_mem_set_bitmap(vmem, start_gpa, size, plug);
164 return 0;
165 }
166
167 static int virtio_mem_state_change_request(VirtIOMEM *vmem, uint64_t gpa,
168 uint16_t nb_blocks, bool plug)
169 {
170 const uint64_t size = nb_blocks * vmem->block_size;
171 int ret;
172
173 if (!virtio_mem_valid_range(vmem, gpa, size)) {
174 return VIRTIO_MEM_RESP_ERROR;
175 }
176
177 if (plug && (vmem->size + size > vmem->requested_size)) {
178 return VIRTIO_MEM_RESP_NACK;
179 }
180
181 /* test if really all blocks are in the opposite state */
182 if (!virtio_mem_test_bitmap(vmem, gpa, size, !plug)) {
183 return VIRTIO_MEM_RESP_ERROR;
184 }
185
186 ret = virtio_mem_set_block_state(vmem, gpa, size, plug);
187 if (ret) {
188 return VIRTIO_MEM_RESP_BUSY;
189 }
190 if (plug) {
191 vmem->size += size;
192 } else {
193 vmem->size -= size;
194 }
195 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size);
196 return VIRTIO_MEM_RESP_ACK;
197 }
198
199 static void virtio_mem_plug_request(VirtIOMEM *vmem, VirtQueueElement *elem,
200 struct virtio_mem_req *req)
201 {
202 const uint64_t gpa = le64_to_cpu(req->u.plug.addr);
203 const uint16_t nb_blocks = le16_to_cpu(req->u.plug.nb_blocks);
204 uint16_t type;
205
206 trace_virtio_mem_plug_request(gpa, nb_blocks);
207 type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, true);
208 virtio_mem_send_response_simple(vmem, elem, type);
209 }
210
211 static void virtio_mem_unplug_request(VirtIOMEM *vmem, VirtQueueElement *elem,
212 struct virtio_mem_req *req)
213 {
214 const uint64_t gpa = le64_to_cpu(req->u.unplug.addr);
215 const uint16_t nb_blocks = le16_to_cpu(req->u.unplug.nb_blocks);
216 uint16_t type;
217
218 trace_virtio_mem_unplug_request(gpa, nb_blocks);
219 type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, false);
220 virtio_mem_send_response_simple(vmem, elem, type);
221 }
222
223 static void virtio_mem_resize_usable_region(VirtIOMEM *vmem,
224 uint64_t requested_size,
225 bool can_shrink)
226 {
227 uint64_t newsize = MIN(memory_region_size(&vmem->memdev->mr),
228 requested_size + VIRTIO_MEM_USABLE_EXTENT);
229
230 if (!requested_size) {
231 newsize = 0;
232 }
233
234 if (newsize < vmem->usable_region_size && !can_shrink) {
235 return;
236 }
237
238 trace_virtio_mem_resized_usable_region(vmem->usable_region_size, newsize);
239 vmem->usable_region_size = newsize;
240 }
241
242 static int virtio_mem_unplug_all(VirtIOMEM *vmem)
243 {
244 RAMBlock *rb = vmem->memdev->mr.ram_block;
245 int ret;
246
247 if (virtio_mem_is_busy()) {
248 return -EBUSY;
249 }
250
251 ret = ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb));
252 if (ret) {
253 error_report("Unexpected error discarding RAM: %s", strerror(-ret));
254 return -EBUSY;
255 }
256 bitmap_clear(vmem->bitmap, 0, vmem->bitmap_size);
257 if (vmem->size) {
258 vmem->size = 0;
259 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size);
260 }
261 trace_virtio_mem_unplugged_all();
262 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true);
263 return 0;
264 }
265
266 static void virtio_mem_unplug_all_request(VirtIOMEM *vmem,
267 VirtQueueElement *elem)
268 {
269 trace_virtio_mem_unplug_all_request();
270 if (virtio_mem_unplug_all(vmem)) {
271 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_BUSY);
272 } else {
273 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ACK);
274 }
275 }
276
277 static void virtio_mem_state_request(VirtIOMEM *vmem, VirtQueueElement *elem,
278 struct virtio_mem_req *req)
279 {
280 const uint16_t nb_blocks = le16_to_cpu(req->u.state.nb_blocks);
281 const uint64_t gpa = le64_to_cpu(req->u.state.addr);
282 const uint64_t size = nb_blocks * vmem->block_size;
283 struct virtio_mem_resp resp = {
284 .type = cpu_to_le16(VIRTIO_MEM_RESP_ACK),
285 };
286
287 trace_virtio_mem_state_request(gpa, nb_blocks);
288 if (!virtio_mem_valid_range(vmem, gpa, size)) {
289 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ERROR);
290 return;
291 }
292
293 if (virtio_mem_test_bitmap(vmem, gpa, size, true)) {
294 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_PLUGGED);
295 } else if (virtio_mem_test_bitmap(vmem, gpa, size, false)) {
296 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_UNPLUGGED);
297 } else {
298 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_MIXED);
299 }
300 trace_virtio_mem_state_response(le16_to_cpu(resp.u.state.state));
301 virtio_mem_send_response(vmem, elem, &resp);
302 }
303
304 static void virtio_mem_handle_request(VirtIODevice *vdev, VirtQueue *vq)
305 {
306 const int len = sizeof(struct virtio_mem_req);
307 VirtIOMEM *vmem = VIRTIO_MEM(vdev);
308 VirtQueueElement *elem;
309 struct virtio_mem_req req;
310 uint16_t type;
311
312 while (true) {
313 elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
314 if (!elem) {
315 return;
316 }
317
318 if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, len) < len) {
319 virtio_error(vdev, "virtio-mem protocol violation: invalid request"
320 " size: %d", len);
321 virtqueue_detach_element(vq, elem, 0);
322 g_free(elem);
323 return;
324 }
325
326 if (iov_size(elem->in_sg, elem->in_num) <
327 sizeof(struct virtio_mem_resp)) {
328 virtio_error(vdev, "virtio-mem protocol violation: not enough space"
329 " for response: %zu",
330 iov_size(elem->in_sg, elem->in_num));
331 virtqueue_detach_element(vq, elem, 0);
332 g_free(elem);
333 return;
334 }
335
336 type = le16_to_cpu(req.type);
337 switch (type) {
338 case VIRTIO_MEM_REQ_PLUG:
339 virtio_mem_plug_request(vmem, elem, &req);
340 break;
341 case VIRTIO_MEM_REQ_UNPLUG:
342 virtio_mem_unplug_request(vmem, elem, &req);
343 break;
344 case VIRTIO_MEM_REQ_UNPLUG_ALL:
345 virtio_mem_unplug_all_request(vmem, elem);
346 break;
347 case VIRTIO_MEM_REQ_STATE:
348 virtio_mem_state_request(vmem, elem, &req);
349 break;
350 default:
351 virtio_error(vdev, "virtio-mem protocol violation: unknown request"
352 " type: %d", type);
353 virtqueue_detach_element(vq, elem, 0);
354 g_free(elem);
355 return;
356 }
357
358 g_free(elem);
359 }
360 }
361
362 static void virtio_mem_get_config(VirtIODevice *vdev, uint8_t *config_data)
363 {
364 VirtIOMEM *vmem = VIRTIO_MEM(vdev);
365 struct virtio_mem_config *config = (void *) config_data;
366
367 config->block_size = cpu_to_le64(vmem->block_size);
368 config->node_id = cpu_to_le16(vmem->node);
369 config->requested_size = cpu_to_le64(vmem->requested_size);
370 config->plugged_size = cpu_to_le64(vmem->size);
371 config->addr = cpu_to_le64(vmem->addr);
372 config->region_size = cpu_to_le64(memory_region_size(&vmem->memdev->mr));
373 config->usable_region_size = cpu_to_le64(vmem->usable_region_size);
374 }
375
376 static uint64_t virtio_mem_get_features(VirtIODevice *vdev, uint64_t features,
377 Error **errp)
378 {
379 MachineState *ms = MACHINE(qdev_get_machine());
380
381 if (ms->numa_state) {
382 #if defined(CONFIG_ACPI)
383 virtio_add_feature(&features, VIRTIO_MEM_F_ACPI_PXM);
384 #endif
385 }
386 return features;
387 }
388
389 static void virtio_mem_system_reset(void *opaque)
390 {
391 VirtIOMEM *vmem = VIRTIO_MEM(opaque);
392
393 /*
394 * During usual resets, we will unplug all memory and shrink the usable
395 * region size. This is, however, not possible in all scenarios. Then,
396 * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL).
397 */
398 virtio_mem_unplug_all(vmem);
399 }
400
401 static void virtio_mem_device_realize(DeviceState *dev, Error **errp)
402 {
403 MachineState *ms = MACHINE(qdev_get_machine());
404 int nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0;
405 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
406 VirtIOMEM *vmem = VIRTIO_MEM(dev);
407 uint64_t page_size;
408 RAMBlock *rb;
409 int ret;
410
411 if (!vmem->memdev) {
412 error_setg(errp, "'%s' property is not set", VIRTIO_MEM_MEMDEV_PROP);
413 return;
414 } else if (host_memory_backend_is_mapped(vmem->memdev)) {
415 error_setg(errp, "'%s' property specifies a busy memdev: %s",
416 VIRTIO_MEM_MEMDEV_PROP,
417 object_get_canonical_path_component(OBJECT(vmem->memdev)));
418 return;
419 } else if (!memory_region_is_ram(&vmem->memdev->mr) ||
420 memory_region_is_rom(&vmem->memdev->mr) ||
421 !vmem->memdev->mr.ram_block) {
422 error_setg(errp, "'%s' property specifies an unsupported memdev",
423 VIRTIO_MEM_MEMDEV_PROP);
424 return;
425 }
426
427 if ((nb_numa_nodes && vmem->node >= nb_numa_nodes) ||
428 (!nb_numa_nodes && vmem->node)) {
429 error_setg(errp, "'%s' property has value '%" PRIu32 "', which exceeds"
430 "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP,
431 vmem->node, nb_numa_nodes ? nb_numa_nodes : 1);
432 return;
433 }
434
435 if (enable_mlock) {
436 error_setg(errp, "Incompatible with mlock");
437 return;
438 }
439
440 rb = vmem->memdev->mr.ram_block;
441 page_size = qemu_ram_pagesize(rb);
442
443 if (vmem->block_size < page_size) {
444 error_setg(errp, "'%s' property has to be at least the page size (0x%"
445 PRIx64 ")", VIRTIO_MEM_BLOCK_SIZE_PROP, page_size);
446 return;
447 } else if (!QEMU_IS_ALIGNED(vmem->requested_size, vmem->block_size)) {
448 error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64
449 ")", VIRTIO_MEM_REQUESTED_SIZE_PROP,
450 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size);
451 return;
452 } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem->memdev->mr),
453 vmem->block_size)) {
454 error_setg(errp, "'%s' property memdev size has to be multiples of"
455 "'%s' (0x%" PRIx64 ")", VIRTIO_MEM_MEMDEV_PROP,
456 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size);
457 return;
458 }
459
460 if (ram_block_discard_require(true)) {
461 error_setg(errp, "Discarding RAM is disabled");
462 return;
463 }
464
465 ret = ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb));
466 if (ret) {
467 error_setg_errno(errp, -ret, "Unexpected error discarding RAM");
468 ram_block_discard_require(false);
469 return;
470 }
471
472 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true);
473
474 vmem->bitmap_size = memory_region_size(&vmem->memdev->mr) /
475 vmem->block_size;
476 vmem->bitmap = bitmap_new(vmem->bitmap_size);
477
478 virtio_init(vdev, TYPE_VIRTIO_MEM, VIRTIO_ID_MEM,
479 sizeof(struct virtio_mem_config));
480 vmem->vq = virtio_add_queue(vdev, 128, virtio_mem_handle_request);
481
482 host_memory_backend_set_mapped(vmem->memdev, true);
483 vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem));
484 qemu_register_reset(virtio_mem_system_reset, vmem);
485 precopy_add_notifier(&vmem->precopy_notifier);
486 }
487
488 static void virtio_mem_device_unrealize(DeviceState *dev)
489 {
490 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
491 VirtIOMEM *vmem = VIRTIO_MEM(dev);
492
493 precopy_remove_notifier(&vmem->precopy_notifier);
494 qemu_unregister_reset(virtio_mem_system_reset, vmem);
495 vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem));
496 host_memory_backend_set_mapped(vmem->memdev, false);
497 virtio_del_queue(vdev, 0);
498 virtio_cleanup(vdev);
499 g_free(vmem->bitmap);
500 ram_block_discard_require(false);
501 }
502
503 static int virtio_mem_restore_unplugged(VirtIOMEM *vmem)
504 {
505 RAMBlock *rb = vmem->memdev->mr.ram_block;
506 unsigned long first_zero_bit, last_zero_bit;
507 uint64_t offset, length;
508 int ret;
509
510 /* Find consecutive unplugged blocks and discard the consecutive range. */
511 first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size);
512 while (first_zero_bit < vmem->bitmap_size) {
513 offset = first_zero_bit * vmem->block_size;
514 last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size,
515 first_zero_bit + 1) - 1;
516 length = (last_zero_bit - first_zero_bit + 1) * vmem->block_size;
517
518 ret = ram_block_discard_range(rb, offset, length);
519 if (ret) {
520 error_report("Unexpected error discarding RAM: %s",
521 strerror(-ret));
522 return -EINVAL;
523 }
524 first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size,
525 last_zero_bit + 2);
526 }
527 return 0;
528 }
529
530 static int virtio_mem_post_load(void *opaque, int version_id)
531 {
532 if (migration_in_incoming_postcopy()) {
533 return 0;
534 }
535
536 return virtio_mem_restore_unplugged(VIRTIO_MEM(opaque));
537 }
538
539 typedef struct VirtIOMEMMigSanityChecks {
540 VirtIOMEM *parent;
541 uint64_t addr;
542 uint64_t region_size;
543 uint64_t block_size;
544 uint32_t node;
545 } VirtIOMEMMigSanityChecks;
546
547 static int virtio_mem_mig_sanity_checks_pre_save(void *opaque)
548 {
549 VirtIOMEMMigSanityChecks *tmp = opaque;
550 VirtIOMEM *vmem = tmp->parent;
551
552 tmp->addr = vmem->addr;
553 tmp->region_size = memory_region_size(&vmem->memdev->mr);
554 tmp->block_size = vmem->block_size;
555 tmp->node = vmem->node;
556 return 0;
557 }
558
559 static int virtio_mem_mig_sanity_checks_post_load(void *opaque, int version_id)
560 {
561 VirtIOMEMMigSanityChecks *tmp = opaque;
562 VirtIOMEM *vmem = tmp->parent;
563 const uint64_t new_region_size = memory_region_size(&vmem->memdev->mr);
564
565 if (tmp->addr != vmem->addr) {
566 error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64,
567 VIRTIO_MEM_ADDR_PROP, tmp->addr, vmem->addr);
568 return -EINVAL;
569 }
570 /*
571 * Note: Preparation for resizeable memory regions. The maximum size
572 * of the memory region must not change during migration.
573 */
574 if (tmp->region_size != new_region_size) {
575 error_report("Property '%s' size changed from 0x%" PRIx64 " to 0x%"
576 PRIx64, VIRTIO_MEM_MEMDEV_PROP, tmp->region_size,
577 new_region_size);
578 return -EINVAL;
579 }
580 if (tmp->block_size != vmem->block_size) {
581 error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64,
582 VIRTIO_MEM_BLOCK_SIZE_PROP, tmp->block_size,
583 vmem->block_size);
584 return -EINVAL;
585 }
586 if (tmp->node != vmem->node) {
587 error_report("Property '%s' changed from %" PRIu32 " to %" PRIu32,
588 VIRTIO_MEM_NODE_PROP, tmp->node, vmem->node);
589 return -EINVAL;
590 }
591 return 0;
592 }
593
594 static const VMStateDescription vmstate_virtio_mem_sanity_checks = {
595 .name = "virtio-mem-device/sanity-checks",
596 .pre_save = virtio_mem_mig_sanity_checks_pre_save,
597 .post_load = virtio_mem_mig_sanity_checks_post_load,
598 .fields = (VMStateField[]) {
599 VMSTATE_UINT64(addr, VirtIOMEMMigSanityChecks),
600 VMSTATE_UINT64(region_size, VirtIOMEMMigSanityChecks),
601 VMSTATE_UINT64(block_size, VirtIOMEMMigSanityChecks),
602 VMSTATE_UINT32(node, VirtIOMEMMigSanityChecks),
603 VMSTATE_END_OF_LIST(),
604 },
605 };
606
607 static const VMStateDescription vmstate_virtio_mem_device = {
608 .name = "virtio-mem-device",
609 .minimum_version_id = 1,
610 .version_id = 1,
611 .post_load = virtio_mem_post_load,
612 .fields = (VMStateField[]) {
613 VMSTATE_WITH_TMP(VirtIOMEM, VirtIOMEMMigSanityChecks,
614 vmstate_virtio_mem_sanity_checks),
615 VMSTATE_UINT64(usable_region_size, VirtIOMEM),
616 VMSTATE_UINT64(size, VirtIOMEM),
617 VMSTATE_UINT64(requested_size, VirtIOMEM),
618 VMSTATE_BITMAP(bitmap, VirtIOMEM, 0, bitmap_size),
619 VMSTATE_END_OF_LIST()
620 },
621 };
622
623 static const VMStateDescription vmstate_virtio_mem = {
624 .name = "virtio-mem",
625 .minimum_version_id = 1,
626 .version_id = 1,
627 .fields = (VMStateField[]) {
628 VMSTATE_VIRTIO_DEVICE,
629 VMSTATE_END_OF_LIST()
630 },
631 };
632
633 static void virtio_mem_fill_device_info(const VirtIOMEM *vmem,
634 VirtioMEMDeviceInfo *vi)
635 {
636 vi->memaddr = vmem->addr;
637 vi->node = vmem->node;
638 vi->requested_size = vmem->requested_size;
639 vi->size = vmem->size;
640 vi->max_size = memory_region_size(&vmem->memdev->mr);
641 vi->block_size = vmem->block_size;
642 vi->memdev = object_get_canonical_path(OBJECT(vmem->memdev));
643 }
644
645 static MemoryRegion *virtio_mem_get_memory_region(VirtIOMEM *vmem, Error **errp)
646 {
647 if (!vmem->memdev) {
648 error_setg(errp, "'%s' property must be set", VIRTIO_MEM_MEMDEV_PROP);
649 return NULL;
650 }
651
652 return &vmem->memdev->mr;
653 }
654
655 static void virtio_mem_add_size_change_notifier(VirtIOMEM *vmem,
656 Notifier *notifier)
657 {
658 notifier_list_add(&vmem->size_change_notifiers, notifier);
659 }
660
661 static void virtio_mem_remove_size_change_notifier(VirtIOMEM *vmem,
662 Notifier *notifier)
663 {
664 notifier_remove(notifier);
665 }
666
667 static void virtio_mem_get_size(Object *obj, Visitor *v, const char *name,
668 void *opaque, Error **errp)
669 {
670 const VirtIOMEM *vmem = VIRTIO_MEM(obj);
671 uint64_t value = vmem->size;
672
673 visit_type_size(v, name, &value, errp);
674 }
675
676 static void virtio_mem_get_requested_size(Object *obj, Visitor *v,
677 const char *name, void *opaque,
678 Error **errp)
679 {
680 const VirtIOMEM *vmem = VIRTIO_MEM(obj);
681 uint64_t value = vmem->requested_size;
682
683 visit_type_size(v, name, &value, errp);
684 }
685
686 static void virtio_mem_set_requested_size(Object *obj, Visitor *v,
687 const char *name, void *opaque,
688 Error **errp)
689 {
690 VirtIOMEM *vmem = VIRTIO_MEM(obj);
691 Error *err = NULL;
692 uint64_t value;
693
694 visit_type_size(v, name, &value, &err);
695 if (err) {
696 error_propagate(errp, err);
697 return;
698 }
699
700 /*
701 * The block size and memory backend are not fixed until the device was
702 * realized. realize() will verify these properties then.
703 */
704 if (DEVICE(obj)->realized) {
705 if (!QEMU_IS_ALIGNED(value, vmem->block_size)) {
706 error_setg(errp, "'%s' has to be multiples of '%s' (0x%" PRIx64
707 ")", name, VIRTIO_MEM_BLOCK_SIZE_PROP,
708 vmem->block_size);
709 return;
710 } else if (value > memory_region_size(&vmem->memdev->mr)) {
711 error_setg(errp, "'%s' cannot exceed the memory backend size"
712 "(0x%" PRIx64 ")", name,
713 memory_region_size(&vmem->memdev->mr));
714 return;
715 }
716
717 if (value != vmem->requested_size) {
718 virtio_mem_resize_usable_region(vmem, value, false);
719 vmem->requested_size = value;
720 }
721 /*
722 * Trigger a config update so the guest gets notified. We trigger
723 * even if the size didn't change (especially helpful for debugging).
724 */
725 virtio_notify_config(VIRTIO_DEVICE(vmem));
726 } else {
727 vmem->requested_size = value;
728 }
729 }
730
731 static void virtio_mem_get_block_size(Object *obj, Visitor *v, const char *name,
732 void *opaque, Error **errp)
733 {
734 const VirtIOMEM *vmem = VIRTIO_MEM(obj);
735 uint64_t value = vmem->block_size;
736
737 visit_type_size(v, name, &value, errp);
738 }
739
740 static void virtio_mem_set_block_size(Object *obj, Visitor *v, const char *name,
741 void *opaque, Error **errp)
742 {
743 VirtIOMEM *vmem = VIRTIO_MEM(obj);
744 Error *err = NULL;
745 uint64_t value;
746
747 if (DEVICE(obj)->realized) {
748 error_setg(errp, "'%s' cannot be changed", name);
749 return;
750 }
751
752 visit_type_size(v, name, &value, &err);
753 if (err) {
754 error_propagate(errp, err);
755 return;
756 }
757
758 if (value < VIRTIO_MEM_MIN_BLOCK_SIZE) {
759 error_setg(errp, "'%s' property has to be at least 0x%" PRIx32, name,
760 VIRTIO_MEM_MIN_BLOCK_SIZE);
761 return;
762 } else if (!is_power_of_2(value)) {
763 error_setg(errp, "'%s' property has to be a power of two", name);
764 return;
765 }
766 vmem->block_size = value;
767 }
768
769 static void virtio_mem_precopy_exclude_unplugged(VirtIOMEM *vmem)
770 {
771 void * const host = qemu_ram_get_host_addr(vmem->memdev->mr.ram_block);
772 unsigned long first_zero_bit, last_zero_bit;
773 uint64_t offset, length;
774
775 /*
776 * Find consecutive unplugged blocks and exclude them from migration.
777 *
778 * Note: Blocks cannot get (un)plugged during precopy, no locking needed.
779 */
780 first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size);
781 while (first_zero_bit < vmem->bitmap_size) {
782 offset = first_zero_bit * vmem->block_size;
783 last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size,
784 first_zero_bit + 1) - 1;
785 length = (last_zero_bit - first_zero_bit + 1) * vmem->block_size;
786
787 qemu_guest_free_page_hint(host + offset, length);
788 first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size,
789 last_zero_bit + 2);
790 }
791 }
792
793 static int virtio_mem_precopy_notify(NotifierWithReturn *n, void *data)
794 {
795 VirtIOMEM *vmem = container_of(n, VirtIOMEM, precopy_notifier);
796 PrecopyNotifyData *pnd = data;
797
798 switch (pnd->reason) {
799 case PRECOPY_NOTIFY_SETUP:
800 precopy_enable_free_page_optimization();
801 break;
802 case PRECOPY_NOTIFY_AFTER_BITMAP_SYNC:
803 virtio_mem_precopy_exclude_unplugged(vmem);
804 break;
805 default:
806 break;
807 }
808
809 return 0;
810 }
811
812 static void virtio_mem_instance_init(Object *obj)
813 {
814 VirtIOMEM *vmem = VIRTIO_MEM(obj);
815
816 vmem->block_size = VIRTIO_MEM_MIN_BLOCK_SIZE;
817 notifier_list_init(&vmem->size_change_notifiers);
818 vmem->precopy_notifier.notify = virtio_mem_precopy_notify;
819
820 object_property_add(obj, VIRTIO_MEM_SIZE_PROP, "size", virtio_mem_get_size,
821 NULL, NULL, NULL);
822 object_property_add(obj, VIRTIO_MEM_REQUESTED_SIZE_PROP, "size",
823 virtio_mem_get_requested_size,
824 virtio_mem_set_requested_size, NULL, NULL);
825 object_property_add(obj, VIRTIO_MEM_BLOCK_SIZE_PROP, "size",
826 virtio_mem_get_block_size, virtio_mem_set_block_size,
827 NULL, NULL);
828 }
829
830 static Property virtio_mem_properties[] = {
831 DEFINE_PROP_UINT64(VIRTIO_MEM_ADDR_PROP, VirtIOMEM, addr, 0),
832 DEFINE_PROP_UINT32(VIRTIO_MEM_NODE_PROP, VirtIOMEM, node, 0),
833 DEFINE_PROP_LINK(VIRTIO_MEM_MEMDEV_PROP, VirtIOMEM, memdev,
834 TYPE_MEMORY_BACKEND, HostMemoryBackend *),
835 DEFINE_PROP_END_OF_LIST(),
836 };
837
838 static void virtio_mem_class_init(ObjectClass *klass, void *data)
839 {
840 DeviceClass *dc = DEVICE_CLASS(klass);
841 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
842 VirtIOMEMClass *vmc = VIRTIO_MEM_CLASS(klass);
843
844 device_class_set_props(dc, virtio_mem_properties);
845 dc->vmsd = &vmstate_virtio_mem;
846
847 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
848 vdc->realize = virtio_mem_device_realize;
849 vdc->unrealize = virtio_mem_device_unrealize;
850 vdc->get_config = virtio_mem_get_config;
851 vdc->get_features = virtio_mem_get_features;
852 vdc->vmsd = &vmstate_virtio_mem_device;
853
854 vmc->fill_device_info = virtio_mem_fill_device_info;
855 vmc->get_memory_region = virtio_mem_get_memory_region;
856 vmc->add_size_change_notifier = virtio_mem_add_size_change_notifier;
857 vmc->remove_size_change_notifier = virtio_mem_remove_size_change_notifier;
858 }
859
860 static const TypeInfo virtio_mem_info = {
861 .name = TYPE_VIRTIO_MEM,
862 .parent = TYPE_VIRTIO_DEVICE,
863 .instance_size = sizeof(VirtIOMEM),
864 .instance_init = virtio_mem_instance_init,
865 .class_init = virtio_mem_class_init,
866 .class_size = sizeof(VirtIOMEMClass),
867 };
868
869 static void virtio_register_types(void)
870 {
871 type_register_static(&virtio_mem_info);
872 }
873
874 type_init(virtio_register_types)