PPC: e500mc: add missing IVORs to bitmap
[qemu.git] / hw / vhost.c
1 /*
2 * vhost support
3 *
4 * Copyright Red Hat, Inc. 2010
5 *
6 * Authors:
7 * Michael S. Tsirkin <mst@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
14 */
15
16 #include <sys/ioctl.h>
17 #include "vhost.h"
18 #include "hw/hw.h"
19 #include "range.h"
20 #include <linux/vhost.h>
21 #include "exec-memory.h"
22
23 static void vhost_dev_sync_region(struct vhost_dev *dev,
24 MemoryRegionSection *section,
25 uint64_t mfirst, uint64_t mlast,
26 uint64_t rfirst, uint64_t rlast)
27 {
28 uint64_t start = MAX(mfirst, rfirst);
29 uint64_t end = MIN(mlast, rlast);
30 vhost_log_chunk_t *from = dev->log + start / VHOST_LOG_CHUNK;
31 vhost_log_chunk_t *to = dev->log + end / VHOST_LOG_CHUNK + 1;
32 uint64_t addr = (start / VHOST_LOG_CHUNK) * VHOST_LOG_CHUNK;
33
34 assert(end / VHOST_LOG_CHUNK < dev->log_size);
35 assert(start / VHOST_LOG_CHUNK < dev->log_size);
36 if (end < start) {
37 return;
38 }
39 for (;from < to; ++from) {
40 vhost_log_chunk_t log;
41 int bit;
42 /* We first check with non-atomic: much cheaper,
43 * and we expect non-dirty to be the common case. */
44 if (!*from) {
45 addr += VHOST_LOG_CHUNK;
46 continue;
47 }
48 /* Data must be read atomically. We don't really
49 * need the barrier semantics of __sync
50 * builtins, but it's easier to use them than
51 * roll our own. */
52 log = __sync_fetch_and_and(from, 0);
53 while ((bit = sizeof(log) > sizeof(int) ?
54 ffsll(log) : ffs(log))) {
55 ram_addr_t ram_addr;
56 bit -= 1;
57 ram_addr = section->offset_within_region + bit * VHOST_LOG_PAGE;
58 memory_region_set_dirty(section->mr, ram_addr, VHOST_LOG_PAGE);
59 log &= ~(0x1ull << bit);
60 }
61 addr += VHOST_LOG_CHUNK;
62 }
63 }
64
65 static int vhost_sync_dirty_bitmap(struct vhost_dev *dev,
66 MemoryRegionSection *section,
67 target_phys_addr_t start_addr,
68 target_phys_addr_t end_addr)
69 {
70 int i;
71
72 if (!dev->log_enabled || !dev->started) {
73 return 0;
74 }
75 for (i = 0; i < dev->mem->nregions; ++i) {
76 struct vhost_memory_region *reg = dev->mem->regions + i;
77 vhost_dev_sync_region(dev, section, start_addr, end_addr,
78 reg->guest_phys_addr,
79 range_get_last(reg->guest_phys_addr,
80 reg->memory_size));
81 }
82 for (i = 0; i < dev->nvqs; ++i) {
83 struct vhost_virtqueue *vq = dev->vqs + i;
84 vhost_dev_sync_region(dev, section, start_addr, end_addr, vq->used_phys,
85 range_get_last(vq->used_phys, vq->used_size));
86 }
87 return 0;
88 }
89
90 static void vhost_log_sync(MemoryListener *listener,
91 MemoryRegionSection *section)
92 {
93 struct vhost_dev *dev = container_of(listener, struct vhost_dev,
94 memory_listener);
95 target_phys_addr_t start_addr = section->offset_within_address_space;
96 target_phys_addr_t end_addr = start_addr + section->size;
97
98 vhost_sync_dirty_bitmap(dev, section, start_addr, end_addr);
99 }
100
101 /* Assign/unassign. Keep an unsorted array of non-overlapping
102 * memory regions in dev->mem. */
103 static void vhost_dev_unassign_memory(struct vhost_dev *dev,
104 uint64_t start_addr,
105 uint64_t size)
106 {
107 int from, to, n = dev->mem->nregions;
108 /* Track overlapping/split regions for sanity checking. */
109 int overlap_start = 0, overlap_end = 0, overlap_middle = 0, split = 0;
110
111 for (from = 0, to = 0; from < n; ++from, ++to) {
112 struct vhost_memory_region *reg = dev->mem->regions + to;
113 uint64_t reglast;
114 uint64_t memlast;
115 uint64_t change;
116
117 /* clone old region */
118 if (to != from) {
119 memcpy(reg, dev->mem->regions + from, sizeof *reg);
120 }
121
122 /* No overlap is simple */
123 if (!ranges_overlap(reg->guest_phys_addr, reg->memory_size,
124 start_addr, size)) {
125 continue;
126 }
127
128 /* Split only happens if supplied region
129 * is in the middle of an existing one. Thus it can not
130 * overlap with any other existing region. */
131 assert(!split);
132
133 reglast = range_get_last(reg->guest_phys_addr, reg->memory_size);
134 memlast = range_get_last(start_addr, size);
135
136 /* Remove whole region */
137 if (start_addr <= reg->guest_phys_addr && memlast >= reglast) {
138 --dev->mem->nregions;
139 --to;
140 ++overlap_middle;
141 continue;
142 }
143
144 /* Shrink region */
145 if (memlast >= reglast) {
146 reg->memory_size = start_addr - reg->guest_phys_addr;
147 assert(reg->memory_size);
148 assert(!overlap_end);
149 ++overlap_end;
150 continue;
151 }
152
153 /* Shift region */
154 if (start_addr <= reg->guest_phys_addr) {
155 change = memlast + 1 - reg->guest_phys_addr;
156 reg->memory_size -= change;
157 reg->guest_phys_addr += change;
158 reg->userspace_addr += change;
159 assert(reg->memory_size);
160 assert(!overlap_start);
161 ++overlap_start;
162 continue;
163 }
164
165 /* This only happens if supplied region
166 * is in the middle of an existing one. Thus it can not
167 * overlap with any other existing region. */
168 assert(!overlap_start);
169 assert(!overlap_end);
170 assert(!overlap_middle);
171 /* Split region: shrink first part, shift second part. */
172 memcpy(dev->mem->regions + n, reg, sizeof *reg);
173 reg->memory_size = start_addr - reg->guest_phys_addr;
174 assert(reg->memory_size);
175 change = memlast + 1 - reg->guest_phys_addr;
176 reg = dev->mem->regions + n;
177 reg->memory_size -= change;
178 assert(reg->memory_size);
179 reg->guest_phys_addr += change;
180 reg->userspace_addr += change;
181 /* Never add more than 1 region */
182 assert(dev->mem->nregions == n);
183 ++dev->mem->nregions;
184 ++split;
185 }
186 }
187
188 /* Called after unassign, so no regions overlap the given range. */
189 static void vhost_dev_assign_memory(struct vhost_dev *dev,
190 uint64_t start_addr,
191 uint64_t size,
192 uint64_t uaddr)
193 {
194 int from, to;
195 struct vhost_memory_region *merged = NULL;
196 for (from = 0, to = 0; from < dev->mem->nregions; ++from, ++to) {
197 struct vhost_memory_region *reg = dev->mem->regions + to;
198 uint64_t prlast, urlast;
199 uint64_t pmlast, umlast;
200 uint64_t s, e, u;
201
202 /* clone old region */
203 if (to != from) {
204 memcpy(reg, dev->mem->regions + from, sizeof *reg);
205 }
206 prlast = range_get_last(reg->guest_phys_addr, reg->memory_size);
207 pmlast = range_get_last(start_addr, size);
208 urlast = range_get_last(reg->userspace_addr, reg->memory_size);
209 umlast = range_get_last(uaddr, size);
210
211 /* check for overlapping regions: should never happen. */
212 assert(prlast < start_addr || pmlast < reg->guest_phys_addr);
213 /* Not an adjacent or overlapping region - do not merge. */
214 if ((prlast + 1 != start_addr || urlast + 1 != uaddr) &&
215 (pmlast + 1 != reg->guest_phys_addr ||
216 umlast + 1 != reg->userspace_addr)) {
217 continue;
218 }
219
220 if (merged) {
221 --to;
222 assert(to >= 0);
223 } else {
224 merged = reg;
225 }
226 u = MIN(uaddr, reg->userspace_addr);
227 s = MIN(start_addr, reg->guest_phys_addr);
228 e = MAX(pmlast, prlast);
229 uaddr = merged->userspace_addr = u;
230 start_addr = merged->guest_phys_addr = s;
231 size = merged->memory_size = e - s + 1;
232 assert(merged->memory_size);
233 }
234
235 if (!merged) {
236 struct vhost_memory_region *reg = dev->mem->regions + to;
237 memset(reg, 0, sizeof *reg);
238 reg->memory_size = size;
239 assert(reg->memory_size);
240 reg->guest_phys_addr = start_addr;
241 reg->userspace_addr = uaddr;
242 ++to;
243 }
244 assert(to <= dev->mem->nregions + 1);
245 dev->mem->nregions = to;
246 }
247
248 static uint64_t vhost_get_log_size(struct vhost_dev *dev)
249 {
250 uint64_t log_size = 0;
251 int i;
252 for (i = 0; i < dev->mem->nregions; ++i) {
253 struct vhost_memory_region *reg = dev->mem->regions + i;
254 uint64_t last = range_get_last(reg->guest_phys_addr,
255 reg->memory_size);
256 log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
257 }
258 for (i = 0; i < dev->nvqs; ++i) {
259 struct vhost_virtqueue *vq = dev->vqs + i;
260 uint64_t last = vq->used_phys + vq->used_size - 1;
261 log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
262 }
263 return log_size;
264 }
265
266 static inline void vhost_dev_log_resize(struct vhost_dev* dev, uint64_t size)
267 {
268 vhost_log_chunk_t *log;
269 uint64_t log_base;
270 int r, i;
271 if (size) {
272 log = g_malloc0(size * sizeof *log);
273 } else {
274 log = NULL;
275 }
276 log_base = (uint64_t)(unsigned long)log;
277 r = ioctl(dev->control, VHOST_SET_LOG_BASE, &log_base);
278 assert(r >= 0);
279 for (i = 0; i < dev->n_mem_sections; ++i) {
280 vhost_sync_dirty_bitmap(dev, &dev->mem_sections[i],
281 0, (target_phys_addr_t)~0x0ull);
282 }
283 if (dev->log) {
284 g_free(dev->log);
285 }
286 dev->log = log;
287 dev->log_size = size;
288 }
289
290 static int vhost_verify_ring_mappings(struct vhost_dev *dev,
291 uint64_t start_addr,
292 uint64_t size)
293 {
294 int i;
295 for (i = 0; i < dev->nvqs; ++i) {
296 struct vhost_virtqueue *vq = dev->vqs + i;
297 target_phys_addr_t l;
298 void *p;
299
300 if (!ranges_overlap(start_addr, size, vq->ring_phys, vq->ring_size)) {
301 continue;
302 }
303 l = vq->ring_size;
304 p = cpu_physical_memory_map(vq->ring_phys, &l, 1);
305 if (!p || l != vq->ring_size) {
306 fprintf(stderr, "Unable to map ring buffer for ring %d\n", i);
307 return -ENOMEM;
308 }
309 if (p != vq->ring) {
310 fprintf(stderr, "Ring buffer relocated for ring %d\n", i);
311 return -EBUSY;
312 }
313 cpu_physical_memory_unmap(p, l, 0, 0);
314 }
315 return 0;
316 }
317
318 static struct vhost_memory_region *vhost_dev_find_reg(struct vhost_dev *dev,
319 uint64_t start_addr,
320 uint64_t size)
321 {
322 int i, n = dev->mem->nregions;
323 for (i = 0; i < n; ++i) {
324 struct vhost_memory_region *reg = dev->mem->regions + i;
325 if (ranges_overlap(reg->guest_phys_addr, reg->memory_size,
326 start_addr, size)) {
327 return reg;
328 }
329 }
330 return NULL;
331 }
332
333 static bool vhost_dev_cmp_memory(struct vhost_dev *dev,
334 uint64_t start_addr,
335 uint64_t size,
336 uint64_t uaddr)
337 {
338 struct vhost_memory_region *reg = vhost_dev_find_reg(dev, start_addr, size);
339 uint64_t reglast;
340 uint64_t memlast;
341
342 if (!reg) {
343 return true;
344 }
345
346 reglast = range_get_last(reg->guest_phys_addr, reg->memory_size);
347 memlast = range_get_last(start_addr, size);
348
349 /* Need to extend region? */
350 if (start_addr < reg->guest_phys_addr || memlast > reglast) {
351 return true;
352 }
353 /* userspace_addr changed? */
354 return uaddr != reg->userspace_addr + start_addr - reg->guest_phys_addr;
355 }
356
357 static void vhost_set_memory(MemoryListener *listener,
358 MemoryRegionSection *section,
359 bool add)
360 {
361 struct vhost_dev *dev = container_of(listener, struct vhost_dev,
362 memory_listener);
363 target_phys_addr_t start_addr = section->offset_within_address_space;
364 ram_addr_t size = section->size;
365 bool log_dirty = memory_region_is_logging(section->mr);
366 int s = offsetof(struct vhost_memory, regions) +
367 (dev->mem->nregions + 1) * sizeof dev->mem->regions[0];
368 uint64_t log_size;
369 int r;
370 void *ram;
371
372 dev->mem = g_realloc(dev->mem, s);
373
374 if (log_dirty) {
375 add = false;
376 }
377
378 assert(size);
379
380 /* Optimize no-change case. At least cirrus_vga does this a lot at this time. */
381 ram = memory_region_get_ram_ptr(section->mr) + section->offset_within_region;
382 if (add) {
383 if (!vhost_dev_cmp_memory(dev, start_addr, size, (uintptr_t)ram)) {
384 /* Region exists with same address. Nothing to do. */
385 return;
386 }
387 } else {
388 if (!vhost_dev_find_reg(dev, start_addr, size)) {
389 /* Removing region that we don't access. Nothing to do. */
390 return;
391 }
392 }
393
394 vhost_dev_unassign_memory(dev, start_addr, size);
395 if (add) {
396 /* Add given mapping, merging adjacent regions if any */
397 vhost_dev_assign_memory(dev, start_addr, size, (uintptr_t)ram);
398 } else {
399 /* Remove old mapping for this memory, if any. */
400 vhost_dev_unassign_memory(dev, start_addr, size);
401 }
402
403 if (!dev->started) {
404 return;
405 }
406
407 if (dev->started) {
408 r = vhost_verify_ring_mappings(dev, start_addr, size);
409 assert(r >= 0);
410 }
411
412 if (!dev->log_enabled) {
413 r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
414 assert(r >= 0);
415 return;
416 }
417 log_size = vhost_get_log_size(dev);
418 /* We allocate an extra 4K bytes to log,
419 * to reduce the * number of reallocations. */
420 #define VHOST_LOG_BUFFER (0x1000 / sizeof *dev->log)
421 /* To log more, must increase log size before table update. */
422 if (dev->log_size < log_size) {
423 vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
424 }
425 r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
426 assert(r >= 0);
427 /* To log less, can only decrease log size after table update. */
428 if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
429 vhost_dev_log_resize(dev, log_size);
430 }
431 }
432
433 static bool vhost_section(MemoryRegionSection *section)
434 {
435 return section->address_space == get_system_memory()
436 && memory_region_is_ram(section->mr);
437 }
438
439 static void vhost_region_add(MemoryListener *listener,
440 MemoryRegionSection *section)
441 {
442 struct vhost_dev *dev = container_of(listener, struct vhost_dev,
443 memory_listener);
444
445 if (!vhost_section(section)) {
446 return;
447 }
448
449 ++dev->n_mem_sections;
450 dev->mem_sections = g_renew(MemoryRegionSection, dev->mem_sections,
451 dev->n_mem_sections);
452 dev->mem_sections[dev->n_mem_sections - 1] = *section;
453 vhost_set_memory(listener, section, true);
454 }
455
456 static void vhost_region_del(MemoryListener *listener,
457 MemoryRegionSection *section)
458 {
459 struct vhost_dev *dev = container_of(listener, struct vhost_dev,
460 memory_listener);
461 int i;
462
463 if (!vhost_section(section)) {
464 return;
465 }
466
467 vhost_set_memory(listener, section, false);
468 for (i = 0; i < dev->n_mem_sections; ++i) {
469 if (dev->mem_sections[i].offset_within_address_space
470 == section->offset_within_address_space) {
471 --dev->n_mem_sections;
472 memmove(&dev->mem_sections[i], &dev->mem_sections[i+1],
473 (dev->n_mem_sections - i) * sizeof(*dev->mem_sections));
474 break;
475 }
476 }
477 }
478
479 static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
480 struct vhost_virtqueue *vq,
481 unsigned idx, bool enable_log)
482 {
483 struct vhost_vring_addr addr = {
484 .index = idx,
485 .desc_user_addr = (uint64_t)(unsigned long)vq->desc,
486 .avail_user_addr = (uint64_t)(unsigned long)vq->avail,
487 .used_user_addr = (uint64_t)(unsigned long)vq->used,
488 .log_guest_addr = vq->used_phys,
489 .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
490 };
491 int r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr);
492 if (r < 0) {
493 return -errno;
494 }
495 return 0;
496 }
497
498 static int vhost_dev_set_features(struct vhost_dev *dev, bool enable_log)
499 {
500 uint64_t features = dev->acked_features;
501 int r;
502 if (enable_log) {
503 features |= 0x1 << VHOST_F_LOG_ALL;
504 }
505 r = ioctl(dev->control, VHOST_SET_FEATURES, &features);
506 return r < 0 ? -errno : 0;
507 }
508
509 static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
510 {
511 int r, t, i;
512 r = vhost_dev_set_features(dev, enable_log);
513 if (r < 0) {
514 goto err_features;
515 }
516 for (i = 0; i < dev->nvqs; ++i) {
517 r = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
518 enable_log);
519 if (r < 0) {
520 goto err_vq;
521 }
522 }
523 return 0;
524 err_vq:
525 for (; i >= 0; --i) {
526 t = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
527 dev->log_enabled);
528 assert(t >= 0);
529 }
530 t = vhost_dev_set_features(dev, dev->log_enabled);
531 assert(t >= 0);
532 err_features:
533 return r;
534 }
535
536 static int vhost_migration_log(MemoryListener *listener, int enable)
537 {
538 struct vhost_dev *dev = container_of(listener, struct vhost_dev,
539 memory_listener);
540 int r;
541 if (!!enable == dev->log_enabled) {
542 return 0;
543 }
544 if (!dev->started) {
545 dev->log_enabled = enable;
546 return 0;
547 }
548 if (!enable) {
549 r = vhost_dev_set_log(dev, false);
550 if (r < 0) {
551 return r;
552 }
553 if (dev->log) {
554 g_free(dev->log);
555 }
556 dev->log = NULL;
557 dev->log_size = 0;
558 } else {
559 vhost_dev_log_resize(dev, vhost_get_log_size(dev));
560 r = vhost_dev_set_log(dev, true);
561 if (r < 0) {
562 return r;
563 }
564 }
565 dev->log_enabled = enable;
566 return 0;
567 }
568
569 static void vhost_log_global_start(MemoryListener *listener)
570 {
571 int r;
572
573 r = vhost_migration_log(listener, true);
574 if (r < 0) {
575 abort();
576 }
577 }
578
579 static void vhost_log_global_stop(MemoryListener *listener)
580 {
581 int r;
582
583 r = vhost_migration_log(listener, false);
584 if (r < 0) {
585 abort();
586 }
587 }
588
589 static void vhost_log_start(MemoryListener *listener,
590 MemoryRegionSection *section)
591 {
592 /* FIXME: implement */
593 }
594
595 static void vhost_log_stop(MemoryListener *listener,
596 MemoryRegionSection *section)
597 {
598 /* FIXME: implement */
599 }
600
601 static int vhost_virtqueue_init(struct vhost_dev *dev,
602 struct VirtIODevice *vdev,
603 struct vhost_virtqueue *vq,
604 unsigned idx)
605 {
606 target_phys_addr_t s, l, a;
607 int r;
608 struct vhost_vring_file file = {
609 .index = idx,
610 };
611 struct vhost_vring_state state = {
612 .index = idx,
613 };
614 struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
615
616 vq->num = state.num = virtio_queue_get_num(vdev, idx);
617 r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);
618 if (r) {
619 return -errno;
620 }
621
622 state.num = virtio_queue_get_last_avail_idx(vdev, idx);
623 r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state);
624 if (r) {
625 return -errno;
626 }
627
628 s = l = virtio_queue_get_desc_size(vdev, idx);
629 a = virtio_queue_get_desc_addr(vdev, idx);
630 vq->desc = cpu_physical_memory_map(a, &l, 0);
631 if (!vq->desc || l != s) {
632 r = -ENOMEM;
633 goto fail_alloc_desc;
634 }
635 s = l = virtio_queue_get_avail_size(vdev, idx);
636 a = virtio_queue_get_avail_addr(vdev, idx);
637 vq->avail = cpu_physical_memory_map(a, &l, 0);
638 if (!vq->avail || l != s) {
639 r = -ENOMEM;
640 goto fail_alloc_avail;
641 }
642 vq->used_size = s = l = virtio_queue_get_used_size(vdev, idx);
643 vq->used_phys = a = virtio_queue_get_used_addr(vdev, idx);
644 vq->used = cpu_physical_memory_map(a, &l, 1);
645 if (!vq->used || l != s) {
646 r = -ENOMEM;
647 goto fail_alloc_used;
648 }
649
650 vq->ring_size = s = l = virtio_queue_get_ring_size(vdev, idx);
651 vq->ring_phys = a = virtio_queue_get_ring_addr(vdev, idx);
652 vq->ring = cpu_physical_memory_map(a, &l, 1);
653 if (!vq->ring || l != s) {
654 r = -ENOMEM;
655 goto fail_alloc_ring;
656 }
657
658 r = vhost_virtqueue_set_addr(dev, vq, idx, dev->log_enabled);
659 if (r < 0) {
660 r = -errno;
661 goto fail_alloc;
662 }
663 file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
664 r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
665 if (r) {
666 r = -errno;
667 goto fail_kick;
668 }
669
670 file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
671 r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);
672 if (r) {
673 r = -errno;
674 goto fail_call;
675 }
676
677 return 0;
678
679 fail_call:
680 fail_kick:
681 fail_alloc:
682 cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
683 0, 0);
684 fail_alloc_ring:
685 cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx),
686 0, 0);
687 fail_alloc_used:
688 cpu_physical_memory_unmap(vq->avail, virtio_queue_get_avail_size(vdev, idx),
689 0, 0);
690 fail_alloc_avail:
691 cpu_physical_memory_unmap(vq->desc, virtio_queue_get_desc_size(vdev, idx),
692 0, 0);
693 fail_alloc_desc:
694 return r;
695 }
696
697 static void vhost_virtqueue_cleanup(struct vhost_dev *dev,
698 struct VirtIODevice *vdev,
699 struct vhost_virtqueue *vq,
700 unsigned idx)
701 {
702 struct vhost_vring_state state = {
703 .index = idx,
704 };
705 int r;
706 r = ioctl(dev->control, VHOST_GET_VRING_BASE, &state);
707 if (r < 0) {
708 fprintf(stderr, "vhost VQ %d ring restore failed: %d\n", idx, r);
709 fflush(stderr);
710 }
711 virtio_queue_set_last_avail_idx(vdev, idx, state.num);
712 assert (r >= 0);
713 cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
714 0, virtio_queue_get_ring_size(vdev, idx));
715 cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx),
716 1, virtio_queue_get_used_size(vdev, idx));
717 cpu_physical_memory_unmap(vq->avail, virtio_queue_get_avail_size(vdev, idx),
718 0, virtio_queue_get_avail_size(vdev, idx));
719 cpu_physical_memory_unmap(vq->desc, virtio_queue_get_desc_size(vdev, idx),
720 0, virtio_queue_get_desc_size(vdev, idx));
721 }
722
723 int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
724 {
725 uint64_t features;
726 int r;
727 if (devfd >= 0) {
728 hdev->control = devfd;
729 } else {
730 hdev->control = open("/dev/vhost-net", O_RDWR);
731 if (hdev->control < 0) {
732 return -errno;
733 }
734 }
735 r = ioctl(hdev->control, VHOST_SET_OWNER, NULL);
736 if (r < 0) {
737 goto fail;
738 }
739
740 r = ioctl(hdev->control, VHOST_GET_FEATURES, &features);
741 if (r < 0) {
742 goto fail;
743 }
744 hdev->features = features;
745
746 hdev->memory_listener = (MemoryListener) {
747 .region_add = vhost_region_add,
748 .region_del = vhost_region_del,
749 .log_start = vhost_log_start,
750 .log_stop = vhost_log_stop,
751 .log_sync = vhost_log_sync,
752 .log_global_start = vhost_log_global_start,
753 .log_global_stop = vhost_log_global_stop,
754 };
755 hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
756 hdev->n_mem_sections = 0;
757 hdev->mem_sections = NULL;
758 hdev->log = NULL;
759 hdev->log_size = 0;
760 hdev->log_enabled = false;
761 hdev->started = false;
762 memory_listener_register(&hdev->memory_listener);
763 hdev->force = force;
764 return 0;
765 fail:
766 r = -errno;
767 close(hdev->control);
768 return r;
769 }
770
771 void vhost_dev_cleanup(struct vhost_dev *hdev)
772 {
773 memory_listener_unregister(&hdev->memory_listener);
774 g_free(hdev->mem);
775 g_free(hdev->mem_sections);
776 close(hdev->control);
777 }
778
779 bool vhost_dev_query(struct vhost_dev *hdev, VirtIODevice *vdev)
780 {
781 return !vdev->binding->query_guest_notifiers ||
782 vdev->binding->query_guest_notifiers(vdev->binding_opaque) ||
783 hdev->force;
784 }
785
786 /* Stop processing guest IO notifications in qemu.
787 * Start processing them in vhost in kernel.
788 */
789 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
790 {
791 int i, r;
792 if (!vdev->binding->set_host_notifier) {
793 fprintf(stderr, "binding does not support host notifiers\n");
794 r = -ENOSYS;
795 goto fail;
796 }
797
798 for (i = 0; i < hdev->nvqs; ++i) {
799 r = vdev->binding->set_host_notifier(vdev->binding_opaque, i, true);
800 if (r < 0) {
801 fprintf(stderr, "vhost VQ %d notifier binding failed: %d\n", i, -r);
802 goto fail_vq;
803 }
804 }
805
806 return 0;
807 fail_vq:
808 while (--i >= 0) {
809 r = vdev->binding->set_host_notifier(vdev->binding_opaque, i, false);
810 if (r < 0) {
811 fprintf(stderr, "vhost VQ %d notifier cleanup error: %d\n", i, -r);
812 fflush(stderr);
813 }
814 assert (r >= 0);
815 }
816 fail:
817 return r;
818 }
819
820 /* Stop processing guest IO notifications in vhost.
821 * Start processing them in qemu.
822 * This might actually run the qemu handlers right away,
823 * so virtio in qemu must be completely setup when this is called.
824 */
825 void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
826 {
827 int i, r;
828
829 for (i = 0; i < hdev->nvqs; ++i) {
830 r = vdev->binding->set_host_notifier(vdev->binding_opaque, i, false);
831 if (r < 0) {
832 fprintf(stderr, "vhost VQ %d notifier cleanup failed: %d\n", i, -r);
833 fflush(stderr);
834 }
835 assert (r >= 0);
836 }
837 }
838
839 /* Host notifiers must be enabled at this point. */
840 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
841 {
842 int i, r;
843 if (!vdev->binding->set_guest_notifiers) {
844 fprintf(stderr, "binding does not support guest notifiers\n");
845 r = -ENOSYS;
846 goto fail;
847 }
848
849 r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, true);
850 if (r < 0) {
851 fprintf(stderr, "Error binding guest notifier: %d\n", -r);
852 goto fail_notifiers;
853 }
854
855 r = vhost_dev_set_features(hdev, hdev->log_enabled);
856 if (r < 0) {
857 goto fail_features;
858 }
859 r = ioctl(hdev->control, VHOST_SET_MEM_TABLE, hdev->mem);
860 if (r < 0) {
861 r = -errno;
862 goto fail_mem;
863 }
864 for (i = 0; i < hdev->nvqs; ++i) {
865 r = vhost_virtqueue_init(hdev,
866 vdev,
867 hdev->vqs + i,
868 i);
869 if (r < 0) {
870 goto fail_vq;
871 }
872 }
873
874 if (hdev->log_enabled) {
875 hdev->log_size = vhost_get_log_size(hdev);
876 hdev->log = hdev->log_size ?
877 g_malloc0(hdev->log_size * sizeof *hdev->log) : NULL;
878 r = ioctl(hdev->control, VHOST_SET_LOG_BASE,
879 (uint64_t)(unsigned long)hdev->log);
880 if (r < 0) {
881 r = -errno;
882 goto fail_log;
883 }
884 }
885
886 hdev->started = true;
887
888 return 0;
889 fail_log:
890 fail_vq:
891 while (--i >= 0) {
892 vhost_virtqueue_cleanup(hdev,
893 vdev,
894 hdev->vqs + i,
895 i);
896 }
897 fail_mem:
898 fail_features:
899 vdev->binding->set_guest_notifiers(vdev->binding_opaque, false);
900 fail_notifiers:
901 fail:
902 return r;
903 }
904
905 /* Host notifiers must be enabled at this point. */
906 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
907 {
908 int i, r;
909
910 for (i = 0; i < hdev->nvqs; ++i) {
911 vhost_virtqueue_cleanup(hdev,
912 vdev,
913 hdev->vqs + i,
914 i);
915 }
916 for (i = 0; i < hdev->n_mem_sections; ++i) {
917 vhost_sync_dirty_bitmap(hdev, &hdev->mem_sections[i],
918 0, (target_phys_addr_t)~0x0ull);
919 }
920 r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, false);
921 if (r < 0) {
922 fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", r);
923 fflush(stderr);
924 }
925 assert (r >= 0);
926
927 hdev->started = false;
928 g_free(hdev->log);
929 hdev->log = NULL;
930 hdev->log_size = 0;
931 }