virtio: add check for descriptor's mapped address
[qemu.git] / hw / virtio / virtio.c
1 /*
2 * Virtio Support
3 *
4 * Copyright IBM, Corp. 2007
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 */
13
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "qemu-common.h"
17 #include "cpu.h"
18 #include "trace.h"
19 #include "exec/address-spaces.h"
20 #include "qemu/error-report.h"
21 #include "hw/virtio/virtio.h"
22 #include "qemu/atomic.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "migration/migration.h"
25 #include "hw/virtio/virtio-access.h"
26
27 /*
28 * The alignment to use between consumer and producer parts of vring.
29 * x86 pagesize again. This is the default, used by transports like PCI
30 * which don't provide a means for the guest to tell the host the alignment.
31 */
32 #define VIRTIO_PCI_VRING_ALIGN 4096
33
34 typedef struct VRingDesc
35 {
36 uint64_t addr;
37 uint32_t len;
38 uint16_t flags;
39 uint16_t next;
40 } VRingDesc;
41
42 typedef struct VRingAvail
43 {
44 uint16_t flags;
45 uint16_t idx;
46 uint16_t ring[0];
47 } VRingAvail;
48
49 typedef struct VRingUsedElem
50 {
51 uint32_t id;
52 uint32_t len;
53 } VRingUsedElem;
54
55 typedef struct VRingUsed
56 {
57 uint16_t flags;
58 uint16_t idx;
59 VRingUsedElem ring[0];
60 } VRingUsed;
61
62 typedef struct VRing
63 {
64 unsigned int num;
65 unsigned int num_default;
66 unsigned int align;
67 hwaddr desc;
68 hwaddr avail;
69 hwaddr used;
70 } VRing;
71
72 struct VirtQueue
73 {
74 VRing vring;
75
76 /* Next head to pop */
77 uint16_t last_avail_idx;
78
79 /* Last avail_idx read from VQ. */
80 uint16_t shadow_avail_idx;
81
82 uint16_t used_idx;
83
84 /* Last used index value we have signalled on */
85 uint16_t signalled_used;
86
87 /* Last used index value we have signalled on */
88 bool signalled_used_valid;
89
90 /* Notification enabled? */
91 bool notification;
92
93 uint16_t queue_index;
94
95 int inuse;
96
97 uint16_t vector;
98 VirtIOHandleOutput handle_output;
99 VirtIOHandleOutput handle_aio_output;
100 bool use_aio;
101 VirtIODevice *vdev;
102 EventNotifier guest_notifier;
103 EventNotifier host_notifier;
104 QLIST_ENTRY(VirtQueue) node;
105 };
106
107 /* virt queue functions */
108 void virtio_queue_update_rings(VirtIODevice *vdev, int n)
109 {
110 VRing *vring = &vdev->vq[n].vring;
111
112 if (!vring->desc) {
113 /* not yet setup -> nothing to do */
114 return;
115 }
116 vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
117 vring->used = vring_align(vring->avail +
118 offsetof(VRingAvail, ring[vring->num]),
119 vring->align);
120 }
121
122 static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
123 hwaddr desc_pa, int i)
124 {
125 address_space_read(&address_space_memory, desc_pa + i * sizeof(VRingDesc),
126 MEMTXATTRS_UNSPECIFIED, (void *)desc, sizeof(VRingDesc));
127 virtio_tswap64s(vdev, &desc->addr);
128 virtio_tswap32s(vdev, &desc->len);
129 virtio_tswap16s(vdev, &desc->flags);
130 virtio_tswap16s(vdev, &desc->next);
131 }
132
133 static inline uint16_t vring_avail_flags(VirtQueue *vq)
134 {
135 hwaddr pa;
136 pa = vq->vring.avail + offsetof(VRingAvail, flags);
137 return virtio_lduw_phys(vq->vdev, pa);
138 }
139
140 static inline uint16_t vring_avail_idx(VirtQueue *vq)
141 {
142 hwaddr pa;
143 pa = vq->vring.avail + offsetof(VRingAvail, idx);
144 vq->shadow_avail_idx = virtio_lduw_phys(vq->vdev, pa);
145 return vq->shadow_avail_idx;
146 }
147
148 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
149 {
150 hwaddr pa;
151 pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
152 return virtio_lduw_phys(vq->vdev, pa);
153 }
154
155 static inline uint16_t vring_get_used_event(VirtQueue *vq)
156 {
157 return vring_avail_ring(vq, vq->vring.num);
158 }
159
160 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
161 int i)
162 {
163 hwaddr pa;
164 virtio_tswap32s(vq->vdev, &uelem->id);
165 virtio_tswap32s(vq->vdev, &uelem->len);
166 pa = vq->vring.used + offsetof(VRingUsed, ring[i]);
167 address_space_write(&address_space_memory, pa, MEMTXATTRS_UNSPECIFIED,
168 (void *)uelem, sizeof(VRingUsedElem));
169 }
170
171 static uint16_t vring_used_idx(VirtQueue *vq)
172 {
173 hwaddr pa;
174 pa = vq->vring.used + offsetof(VRingUsed, idx);
175 return virtio_lduw_phys(vq->vdev, pa);
176 }
177
178 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
179 {
180 hwaddr pa;
181 pa = vq->vring.used + offsetof(VRingUsed, idx);
182 virtio_stw_phys(vq->vdev, pa, val);
183 vq->used_idx = val;
184 }
185
186 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
187 {
188 VirtIODevice *vdev = vq->vdev;
189 hwaddr pa;
190 pa = vq->vring.used + offsetof(VRingUsed, flags);
191 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) | mask);
192 }
193
194 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
195 {
196 VirtIODevice *vdev = vq->vdev;
197 hwaddr pa;
198 pa = vq->vring.used + offsetof(VRingUsed, flags);
199 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask);
200 }
201
202 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
203 {
204 hwaddr pa;
205 if (!vq->notification) {
206 return;
207 }
208 pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]);
209 virtio_stw_phys(vq->vdev, pa, val);
210 }
211
212 void virtio_queue_set_notification(VirtQueue *vq, int enable)
213 {
214 vq->notification = enable;
215 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
216 vring_set_avail_event(vq, vring_avail_idx(vq));
217 } else if (enable) {
218 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
219 } else {
220 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
221 }
222 if (enable) {
223 /* Expose avail event/used flags before caller checks the avail idx. */
224 smp_mb();
225 }
226 }
227
228 int virtio_queue_ready(VirtQueue *vq)
229 {
230 return vq->vring.avail != 0;
231 }
232
233 /* Fetch avail_idx from VQ memory only when we really need to know if
234 * guest has added some buffers. */
235 int virtio_queue_empty(VirtQueue *vq)
236 {
237 if (vq->shadow_avail_idx != vq->last_avail_idx) {
238 return 0;
239 }
240
241 return vring_avail_idx(vq) == vq->last_avail_idx;
242 }
243
244 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
245 unsigned int len)
246 {
247 unsigned int offset;
248 int i;
249
250 offset = 0;
251 for (i = 0; i < elem->in_num; i++) {
252 size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
253
254 cpu_physical_memory_unmap(elem->in_sg[i].iov_base,
255 elem->in_sg[i].iov_len,
256 1, size);
257
258 offset += size;
259 }
260
261 for (i = 0; i < elem->out_num; i++)
262 cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
263 elem->out_sg[i].iov_len,
264 0, elem->out_sg[i].iov_len);
265 }
266
267 void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
268 unsigned int len)
269 {
270 vq->last_avail_idx--;
271 vq->inuse--;
272 virtqueue_unmap_sg(vq, elem, len);
273 }
274
275 /* virtqueue_rewind:
276 * @vq: The #VirtQueue
277 * @num: Number of elements to push back
278 *
279 * Pretend that elements weren't popped from the virtqueue. The next
280 * virtqueue_pop() will refetch the oldest element.
281 *
282 * Use virtqueue_discard() instead if you have a VirtQueueElement.
283 *
284 * Returns: true on success, false if @num is greater than the number of in use
285 * elements.
286 */
287 bool virtqueue_rewind(VirtQueue *vq, unsigned int num)
288 {
289 if (num > vq->inuse) {
290 return false;
291 }
292 vq->last_avail_idx -= num;
293 vq->inuse -= num;
294 return true;
295 }
296
297 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
298 unsigned int len, unsigned int idx)
299 {
300 VRingUsedElem uelem;
301
302 trace_virtqueue_fill(vq, elem, len, idx);
303
304 virtqueue_unmap_sg(vq, elem, len);
305
306 idx = (idx + vq->used_idx) % vq->vring.num;
307
308 uelem.id = elem->index;
309 uelem.len = len;
310 vring_used_write(vq, &uelem, idx);
311 }
312
313 void virtqueue_flush(VirtQueue *vq, unsigned int count)
314 {
315 uint16_t old, new;
316 /* Make sure buffer is written before we update index. */
317 smp_wmb();
318 trace_virtqueue_flush(vq, count);
319 old = vq->used_idx;
320 new = old + count;
321 vring_used_idx_set(vq, new);
322 vq->inuse -= count;
323 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
324 vq->signalled_used_valid = false;
325 }
326
327 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
328 unsigned int len)
329 {
330 virtqueue_fill(vq, elem, len, 0);
331 virtqueue_flush(vq, 1);
332 }
333
334 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
335 {
336 uint16_t num_heads = vring_avail_idx(vq) - idx;
337
338 /* Check it isn't doing very strange things with descriptor numbers. */
339 if (num_heads > vq->vring.num) {
340 error_report("Guest moved used index from %u to %u",
341 idx, vq->shadow_avail_idx);
342 exit(1);
343 }
344 /* On success, callers read a descriptor at vq->last_avail_idx.
345 * Make sure descriptor read does not bypass avail index read. */
346 if (num_heads) {
347 smp_rmb();
348 }
349
350 return num_heads;
351 }
352
353 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
354 {
355 unsigned int head;
356
357 /* Grab the next descriptor number they're advertising, and increment
358 * the index we've seen. */
359 head = vring_avail_ring(vq, idx % vq->vring.num);
360
361 /* If their number is silly, that's a fatal mistake. */
362 if (head >= vq->vring.num) {
363 error_report("Guest says index %u is available", head);
364 exit(1);
365 }
366
367 return head;
368 }
369
370 static unsigned virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
371 hwaddr desc_pa, unsigned int max)
372 {
373 unsigned int next;
374
375 /* If this descriptor says it doesn't chain, we're done. */
376 if (!(desc->flags & VRING_DESC_F_NEXT)) {
377 return max;
378 }
379
380 /* Check they're not leading us off end of descriptors. */
381 next = desc->next;
382 /* Make sure compiler knows to grab that: we don't want it changing! */
383 smp_wmb();
384
385 if (next >= max) {
386 error_report("Desc next is %u", next);
387 exit(1);
388 }
389
390 vring_desc_read(vdev, desc, desc_pa, next);
391 return next;
392 }
393
394 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
395 unsigned int *out_bytes,
396 unsigned max_in_bytes, unsigned max_out_bytes)
397 {
398 unsigned int idx;
399 unsigned int total_bufs, in_total, out_total;
400
401 idx = vq->last_avail_idx;
402
403 total_bufs = in_total = out_total = 0;
404 while (virtqueue_num_heads(vq, idx)) {
405 VirtIODevice *vdev = vq->vdev;
406 unsigned int max, num_bufs, indirect = 0;
407 VRingDesc desc;
408 hwaddr desc_pa;
409 int i;
410
411 max = vq->vring.num;
412 num_bufs = total_bufs;
413 i = virtqueue_get_head(vq, idx++);
414 desc_pa = vq->vring.desc;
415 vring_desc_read(vdev, &desc, desc_pa, i);
416
417 if (desc.flags & VRING_DESC_F_INDIRECT) {
418 if (desc.len % sizeof(VRingDesc)) {
419 error_report("Invalid size for indirect buffer table");
420 exit(1);
421 }
422
423 /* If we've got too many, that implies a descriptor loop. */
424 if (num_bufs >= max) {
425 error_report("Looped descriptor");
426 exit(1);
427 }
428
429 /* loop over the indirect descriptor table */
430 indirect = 1;
431 max = desc.len / sizeof(VRingDesc);
432 desc_pa = desc.addr;
433 num_bufs = i = 0;
434 vring_desc_read(vdev, &desc, desc_pa, i);
435 }
436
437 do {
438 /* If we've got too many, that implies a descriptor loop. */
439 if (++num_bufs > max) {
440 error_report("Looped descriptor");
441 exit(1);
442 }
443
444 if (desc.flags & VRING_DESC_F_WRITE) {
445 in_total += desc.len;
446 } else {
447 out_total += desc.len;
448 }
449 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
450 goto done;
451 }
452 } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
453
454 if (!indirect)
455 total_bufs = num_bufs;
456 else
457 total_bufs++;
458 }
459 done:
460 if (in_bytes) {
461 *in_bytes = in_total;
462 }
463 if (out_bytes) {
464 *out_bytes = out_total;
465 }
466 }
467
468 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
469 unsigned int out_bytes)
470 {
471 unsigned int in_total, out_total;
472
473 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
474 return in_bytes <= in_total && out_bytes <= out_total;
475 }
476
477 static void virtqueue_map_desc(unsigned int *p_num_sg, hwaddr *addr, struct iovec *iov,
478 unsigned int max_num_sg, bool is_write,
479 hwaddr pa, size_t sz)
480 {
481 unsigned num_sg = *p_num_sg;
482 assert(num_sg <= max_num_sg);
483
484 if (!sz) {
485 error_report("virtio: zero sized buffers are not allowed");
486 exit(1);
487 }
488
489 while (sz) {
490 hwaddr len = sz;
491
492 if (num_sg == max_num_sg) {
493 error_report("virtio: too many write descriptors in indirect table");
494 exit(1);
495 }
496
497 iov[num_sg].iov_base = cpu_physical_memory_map(pa, &len, is_write);
498 if (!iov[num_sg].iov_base) {
499 error_report("virtio: bogus descriptor or out of resources");
500 exit(1);
501 }
502
503 iov[num_sg].iov_len = len;
504 addr[num_sg] = pa;
505
506 sz -= len;
507 pa += len;
508 num_sg++;
509 }
510 *p_num_sg = num_sg;
511 }
512
513 static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr,
514 unsigned int *num_sg, unsigned int max_size,
515 int is_write)
516 {
517 unsigned int i;
518 hwaddr len;
519
520 /* Note: this function MUST validate input, some callers
521 * are passing in num_sg values received over the network.
522 */
523 /* TODO: teach all callers that this can fail, and return failure instead
524 * of asserting here.
525 * When we do, we might be able to re-enable NDEBUG below.
526 */
527 #ifdef NDEBUG
528 #error building with NDEBUG is not supported
529 #endif
530 assert(*num_sg <= max_size);
531
532 for (i = 0; i < *num_sg; i++) {
533 len = sg[i].iov_len;
534 sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
535 if (!sg[i].iov_base) {
536 error_report("virtio: error trying to map MMIO memory");
537 exit(1);
538 }
539 if (len != sg[i].iov_len) {
540 error_report("virtio: unexpected memory split");
541 exit(1);
542 }
543 }
544 }
545
546 void virtqueue_map(VirtQueueElement *elem)
547 {
548 virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num,
549 VIRTQUEUE_MAX_SIZE, 1);
550 virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num,
551 VIRTQUEUE_MAX_SIZE, 0);
552 }
553
554 void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
555 {
556 VirtQueueElement *elem;
557 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
558 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
559 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
560 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
561 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
562 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
563
564 assert(sz >= sizeof(VirtQueueElement));
565 elem = g_malloc(out_sg_end);
566 elem->out_num = out_num;
567 elem->in_num = in_num;
568 elem->in_addr = (void *)elem + in_addr_ofs;
569 elem->out_addr = (void *)elem + out_addr_ofs;
570 elem->in_sg = (void *)elem + in_sg_ofs;
571 elem->out_sg = (void *)elem + out_sg_ofs;
572 return elem;
573 }
574
575 void *virtqueue_pop(VirtQueue *vq, size_t sz)
576 {
577 unsigned int i, head, max;
578 hwaddr desc_pa = vq->vring.desc;
579 VirtIODevice *vdev = vq->vdev;
580 VirtQueueElement *elem;
581 unsigned out_num, in_num;
582 hwaddr addr[VIRTQUEUE_MAX_SIZE];
583 struct iovec iov[VIRTQUEUE_MAX_SIZE];
584 VRingDesc desc;
585
586 if (virtio_queue_empty(vq)) {
587 return NULL;
588 }
589 /* Needed after virtio_queue_empty(), see comment in
590 * virtqueue_num_heads(). */
591 smp_rmb();
592
593 /* When we start there are none of either input nor output. */
594 out_num = in_num = 0;
595
596 max = vq->vring.num;
597
598 if (vq->inuse >= vq->vring.num) {
599 error_report("Virtqueue size exceeded");
600 exit(1);
601 }
602
603 i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
604 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
605 vring_set_avail_event(vq, vq->last_avail_idx);
606 }
607
608 vring_desc_read(vdev, &desc, desc_pa, i);
609 if (desc.flags & VRING_DESC_F_INDIRECT) {
610 if (desc.len % sizeof(VRingDesc)) {
611 error_report("Invalid size for indirect buffer table");
612 exit(1);
613 }
614
615 /* loop over the indirect descriptor table */
616 max = desc.len / sizeof(VRingDesc);
617 desc_pa = desc.addr;
618 i = 0;
619 vring_desc_read(vdev, &desc, desc_pa, i);
620 }
621
622 /* Collect all the descriptors */
623 do {
624 if (desc.flags & VRING_DESC_F_WRITE) {
625 virtqueue_map_desc(&in_num, addr + out_num, iov + out_num,
626 VIRTQUEUE_MAX_SIZE - out_num, true, desc.addr, desc.len);
627 } else {
628 if (in_num) {
629 error_report("Incorrect order for descriptors");
630 exit(1);
631 }
632 virtqueue_map_desc(&out_num, addr, iov,
633 VIRTQUEUE_MAX_SIZE, false, desc.addr, desc.len);
634 }
635
636 /* If we've got too many, that implies a descriptor loop. */
637 if ((in_num + out_num) > max) {
638 error_report("Looped descriptor");
639 exit(1);
640 }
641 } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
642
643 /* Now copy what we have collected and mapped */
644 elem = virtqueue_alloc_element(sz, out_num, in_num);
645 elem->index = head;
646 for (i = 0; i < out_num; i++) {
647 elem->out_addr[i] = addr[i];
648 elem->out_sg[i] = iov[i];
649 }
650 for (i = 0; i < in_num; i++) {
651 elem->in_addr[i] = addr[out_num + i];
652 elem->in_sg[i] = iov[out_num + i];
653 }
654
655 vq->inuse++;
656
657 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
658 return elem;
659 }
660
661 /* Reading and writing a structure directly to QEMUFile is *awful*, but
662 * it is what QEMU has always done by mistake. We can change it sooner
663 * or later by bumping the version number of the affected vm states.
664 * In the meanwhile, since the in-memory layout of VirtQueueElement
665 * has changed, we need to marshal to and from the layout that was
666 * used before the change.
667 */
668 typedef struct VirtQueueElementOld {
669 unsigned int index;
670 unsigned int out_num;
671 unsigned int in_num;
672 hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
673 hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
674 struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
675 struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
676 } VirtQueueElementOld;
677
678 void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz)
679 {
680 VirtQueueElement *elem;
681 VirtQueueElementOld data;
682 int i;
683
684 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
685
686 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
687 elem->index = data.index;
688
689 for (i = 0; i < elem->in_num; i++) {
690 elem->in_addr[i] = data.in_addr[i];
691 }
692
693 for (i = 0; i < elem->out_num; i++) {
694 elem->out_addr[i] = data.out_addr[i];
695 }
696
697 for (i = 0; i < elem->in_num; i++) {
698 /* Base is overwritten by virtqueue_map. */
699 elem->in_sg[i].iov_base = 0;
700 elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
701 }
702
703 for (i = 0; i < elem->out_num; i++) {
704 /* Base is overwritten by virtqueue_map. */
705 elem->out_sg[i].iov_base = 0;
706 elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
707 }
708
709 virtqueue_map(elem);
710 return elem;
711 }
712
713 void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
714 {
715 VirtQueueElementOld data;
716 int i;
717
718 memset(&data, 0, sizeof(data));
719 data.index = elem->index;
720 data.in_num = elem->in_num;
721 data.out_num = elem->out_num;
722
723 for (i = 0; i < elem->in_num; i++) {
724 data.in_addr[i] = elem->in_addr[i];
725 }
726
727 for (i = 0; i < elem->out_num; i++) {
728 data.out_addr[i] = elem->out_addr[i];
729 }
730
731 for (i = 0; i < elem->in_num; i++) {
732 /* Base is overwritten by virtqueue_map when loading. Do not
733 * save it, as it would leak the QEMU address space layout. */
734 data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
735 }
736
737 for (i = 0; i < elem->out_num; i++) {
738 /* Do not save iov_base as above. */
739 data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
740 }
741 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
742 }
743
744 /* virtio device */
745 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
746 {
747 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
748 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
749
750 if (k->notify) {
751 k->notify(qbus->parent, vector);
752 }
753 }
754
755 void virtio_update_irq(VirtIODevice *vdev)
756 {
757 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
758 }
759
760 static int virtio_validate_features(VirtIODevice *vdev)
761 {
762 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
763
764 if (k->validate_features) {
765 return k->validate_features(vdev);
766 } else {
767 return 0;
768 }
769 }
770
771 int virtio_set_status(VirtIODevice *vdev, uint8_t val)
772 {
773 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
774 trace_virtio_set_status(vdev, val);
775
776 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
777 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
778 val & VIRTIO_CONFIG_S_FEATURES_OK) {
779 int ret = virtio_validate_features(vdev);
780
781 if (ret) {
782 return ret;
783 }
784 }
785 }
786 if (k->set_status) {
787 k->set_status(vdev, val);
788 }
789 vdev->status = val;
790 return 0;
791 }
792
793 bool target_words_bigendian(void);
794 static enum virtio_device_endian virtio_default_endian(void)
795 {
796 if (target_words_bigendian()) {
797 return VIRTIO_DEVICE_ENDIAN_BIG;
798 } else {
799 return VIRTIO_DEVICE_ENDIAN_LITTLE;
800 }
801 }
802
803 static enum virtio_device_endian virtio_current_cpu_endian(void)
804 {
805 CPUClass *cc = CPU_GET_CLASS(current_cpu);
806
807 if (cc->virtio_is_big_endian(current_cpu)) {
808 return VIRTIO_DEVICE_ENDIAN_BIG;
809 } else {
810 return VIRTIO_DEVICE_ENDIAN_LITTLE;
811 }
812 }
813
814 void virtio_reset(void *opaque)
815 {
816 VirtIODevice *vdev = opaque;
817 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
818 int i;
819
820 virtio_set_status(vdev, 0);
821 if (current_cpu) {
822 /* Guest initiated reset */
823 vdev->device_endian = virtio_current_cpu_endian();
824 } else {
825 /* System reset */
826 vdev->device_endian = virtio_default_endian();
827 }
828
829 if (k->reset) {
830 k->reset(vdev);
831 }
832
833 vdev->guest_features = 0;
834 vdev->queue_sel = 0;
835 vdev->status = 0;
836 vdev->isr = 0;
837 vdev->config_vector = VIRTIO_NO_VECTOR;
838 virtio_notify_vector(vdev, vdev->config_vector);
839
840 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
841 vdev->vq[i].vring.desc = 0;
842 vdev->vq[i].vring.avail = 0;
843 vdev->vq[i].vring.used = 0;
844 vdev->vq[i].last_avail_idx = 0;
845 vdev->vq[i].shadow_avail_idx = 0;
846 vdev->vq[i].used_idx = 0;
847 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
848 vdev->vq[i].signalled_used = 0;
849 vdev->vq[i].signalled_used_valid = false;
850 vdev->vq[i].notification = true;
851 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
852 vdev->vq[i].inuse = 0;
853 }
854 }
855
856 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
857 {
858 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
859 uint8_t val;
860
861 if (addr + sizeof(val) > vdev->config_len) {
862 return (uint32_t)-1;
863 }
864
865 k->get_config(vdev, vdev->config);
866
867 val = ldub_p(vdev->config + addr);
868 return val;
869 }
870
871 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
872 {
873 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
874 uint16_t val;
875
876 if (addr + sizeof(val) > vdev->config_len) {
877 return (uint32_t)-1;
878 }
879
880 k->get_config(vdev, vdev->config);
881
882 val = lduw_p(vdev->config + addr);
883 return val;
884 }
885
886 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
887 {
888 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
889 uint32_t val;
890
891 if (addr + sizeof(val) > vdev->config_len) {
892 return (uint32_t)-1;
893 }
894
895 k->get_config(vdev, vdev->config);
896
897 val = ldl_p(vdev->config + addr);
898 return val;
899 }
900
901 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
902 {
903 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
904 uint8_t val = data;
905
906 if (addr + sizeof(val) > vdev->config_len) {
907 return;
908 }
909
910 stb_p(vdev->config + addr, val);
911
912 if (k->set_config) {
913 k->set_config(vdev, vdev->config);
914 }
915 }
916
917 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
918 {
919 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
920 uint16_t val = data;
921
922 if (addr + sizeof(val) > vdev->config_len) {
923 return;
924 }
925
926 stw_p(vdev->config + addr, val);
927
928 if (k->set_config) {
929 k->set_config(vdev, vdev->config);
930 }
931 }
932
933 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
934 {
935 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
936 uint32_t val = data;
937
938 if (addr + sizeof(val) > vdev->config_len) {
939 return;
940 }
941
942 stl_p(vdev->config + addr, val);
943
944 if (k->set_config) {
945 k->set_config(vdev, vdev->config);
946 }
947 }
948
949 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
950 {
951 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
952 uint8_t val;
953
954 if (addr + sizeof(val) > vdev->config_len) {
955 return (uint32_t)-1;
956 }
957
958 k->get_config(vdev, vdev->config);
959
960 val = ldub_p(vdev->config + addr);
961 return val;
962 }
963
964 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
965 {
966 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
967 uint16_t val;
968
969 if (addr + sizeof(val) > vdev->config_len) {
970 return (uint32_t)-1;
971 }
972
973 k->get_config(vdev, vdev->config);
974
975 val = lduw_le_p(vdev->config + addr);
976 return val;
977 }
978
979 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
980 {
981 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
982 uint32_t val;
983
984 if (addr + sizeof(val) > vdev->config_len) {
985 return (uint32_t)-1;
986 }
987
988 k->get_config(vdev, vdev->config);
989
990 val = ldl_le_p(vdev->config + addr);
991 return val;
992 }
993
994 void virtio_config_modern_writeb(VirtIODevice *vdev,
995 uint32_t addr, uint32_t data)
996 {
997 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
998 uint8_t val = data;
999
1000 if (addr + sizeof(val) > vdev->config_len) {
1001 return;
1002 }
1003
1004 stb_p(vdev->config + addr, val);
1005
1006 if (k->set_config) {
1007 k->set_config(vdev, vdev->config);
1008 }
1009 }
1010
1011 void virtio_config_modern_writew(VirtIODevice *vdev,
1012 uint32_t addr, uint32_t data)
1013 {
1014 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1015 uint16_t val = data;
1016
1017 if (addr + sizeof(val) > vdev->config_len) {
1018 return;
1019 }
1020
1021 stw_le_p(vdev->config + addr, val);
1022
1023 if (k->set_config) {
1024 k->set_config(vdev, vdev->config);
1025 }
1026 }
1027
1028 void virtio_config_modern_writel(VirtIODevice *vdev,
1029 uint32_t addr, uint32_t data)
1030 {
1031 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1032 uint32_t val = data;
1033
1034 if (addr + sizeof(val) > vdev->config_len) {
1035 return;
1036 }
1037
1038 stl_le_p(vdev->config + addr, val);
1039
1040 if (k->set_config) {
1041 k->set_config(vdev, vdev->config);
1042 }
1043 }
1044
1045 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
1046 {
1047 vdev->vq[n].vring.desc = addr;
1048 virtio_queue_update_rings(vdev, n);
1049 }
1050
1051 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
1052 {
1053 return vdev->vq[n].vring.desc;
1054 }
1055
1056 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
1057 hwaddr avail, hwaddr used)
1058 {
1059 vdev->vq[n].vring.desc = desc;
1060 vdev->vq[n].vring.avail = avail;
1061 vdev->vq[n].vring.used = used;
1062 }
1063
1064 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
1065 {
1066 /* Don't allow guest to flip queue between existent and
1067 * nonexistent states, or to set it to an invalid size.
1068 */
1069 if (!!num != !!vdev->vq[n].vring.num ||
1070 num > VIRTQUEUE_MAX_SIZE ||
1071 num < 0) {
1072 return;
1073 }
1074 vdev->vq[n].vring.num = num;
1075 }
1076
1077 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
1078 {
1079 return QLIST_FIRST(&vdev->vector_queues[vector]);
1080 }
1081
1082 VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
1083 {
1084 return QLIST_NEXT(vq, node);
1085 }
1086
1087 int virtio_queue_get_num(VirtIODevice *vdev, int n)
1088 {
1089 return vdev->vq[n].vring.num;
1090 }
1091
1092 int virtio_get_num_queues(VirtIODevice *vdev)
1093 {
1094 int i;
1095
1096 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1097 if (!virtio_queue_get_num(vdev, i)) {
1098 break;
1099 }
1100 }
1101
1102 return i;
1103 }
1104
1105 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
1106 {
1107 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1108 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1109
1110 /* virtio-1 compliant devices cannot change the alignment */
1111 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1112 error_report("tried to modify queue alignment for virtio-1 device");
1113 return;
1114 }
1115 /* Check that the transport told us it was going to do this
1116 * (so a buggy transport will immediately assert rather than
1117 * silently failing to migrate this state)
1118 */
1119 assert(k->has_variable_vring_alignment);
1120
1121 vdev->vq[n].vring.align = align;
1122 virtio_queue_update_rings(vdev, n);
1123 }
1124
1125 static void virtio_queue_notify_aio_vq(VirtQueue *vq)
1126 {
1127 if (vq->vring.desc && vq->handle_aio_output) {
1128 VirtIODevice *vdev = vq->vdev;
1129
1130 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1131 vq->handle_aio_output(vdev, vq);
1132 }
1133 }
1134
1135 static void virtio_queue_notify_vq(VirtQueue *vq)
1136 {
1137 if (vq->vring.desc && vq->handle_output) {
1138 VirtIODevice *vdev = vq->vdev;
1139
1140 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1141 vq->handle_output(vdev, vq);
1142 }
1143 }
1144
1145 void virtio_queue_notify(VirtIODevice *vdev, int n)
1146 {
1147 virtio_queue_notify_vq(&vdev->vq[n]);
1148 }
1149
1150 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
1151 {
1152 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
1153 VIRTIO_NO_VECTOR;
1154 }
1155
1156 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
1157 {
1158 VirtQueue *vq = &vdev->vq[n];
1159
1160 if (n < VIRTIO_QUEUE_MAX) {
1161 if (vdev->vector_queues &&
1162 vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
1163 QLIST_REMOVE(vq, node);
1164 }
1165 vdev->vq[n].vector = vector;
1166 if (vdev->vector_queues &&
1167 vector != VIRTIO_NO_VECTOR) {
1168 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
1169 }
1170 }
1171 }
1172
1173 static VirtQueue *virtio_add_queue_internal(VirtIODevice *vdev, int queue_size,
1174 VirtIOHandleOutput handle_output,
1175 bool use_aio)
1176 {
1177 int i;
1178
1179 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1180 if (vdev->vq[i].vring.num == 0)
1181 break;
1182 }
1183
1184 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
1185 abort();
1186
1187 vdev->vq[i].vring.num = queue_size;
1188 vdev->vq[i].vring.num_default = queue_size;
1189 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
1190 vdev->vq[i].handle_output = handle_output;
1191 vdev->vq[i].handle_aio_output = NULL;
1192 vdev->vq[i].use_aio = use_aio;
1193
1194 return &vdev->vq[i];
1195 }
1196
1197 /* Add a virt queue and mark AIO.
1198 * An AIO queue will use the AioContext based event interface instead of the
1199 * default IOHandler and EventNotifier interface.
1200 */
1201 VirtQueue *virtio_add_queue_aio(VirtIODevice *vdev, int queue_size,
1202 VirtIOHandleOutput handle_output)
1203 {
1204 return virtio_add_queue_internal(vdev, queue_size, handle_output, true);
1205 }
1206
1207 /* Add a normal virt queue (on the contrary to the AIO version above. */
1208 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
1209 VirtIOHandleOutput handle_output)
1210 {
1211 return virtio_add_queue_internal(vdev, queue_size, handle_output, false);
1212 }
1213
1214 void virtio_del_queue(VirtIODevice *vdev, int n)
1215 {
1216 if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
1217 abort();
1218 }
1219
1220 vdev->vq[n].vring.num = 0;
1221 vdev->vq[n].vring.num_default = 0;
1222 }
1223
1224 void virtio_irq(VirtQueue *vq)
1225 {
1226 trace_virtio_irq(vq);
1227 vq->vdev->isr |= 0x01;
1228 virtio_notify_vector(vq->vdev, vq->vector);
1229 }
1230
1231 bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
1232 {
1233 uint16_t old, new;
1234 bool v;
1235 /* We need to expose used array entries before checking used event. */
1236 smp_mb();
1237 /* Always notify when queue is empty (when feature acknowledge) */
1238 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1239 !vq->inuse && virtio_queue_empty(vq)) {
1240 return true;
1241 }
1242
1243 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1244 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1245 }
1246
1247 v = vq->signalled_used_valid;
1248 vq->signalled_used_valid = true;
1249 old = vq->signalled_used;
1250 new = vq->signalled_used = vq->used_idx;
1251 return !v || vring_need_event(vring_get_used_event(vq), new, old);
1252 }
1253
1254 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
1255 {
1256 if (!virtio_should_notify(vdev, vq)) {
1257 return;
1258 }
1259
1260 trace_virtio_notify(vdev, vq);
1261 vdev->isr |= 0x01;
1262 virtio_notify_vector(vdev, vq->vector);
1263 }
1264
1265 void virtio_notify_config(VirtIODevice *vdev)
1266 {
1267 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1268 return;
1269
1270 vdev->isr |= 0x03;
1271 vdev->generation++;
1272 virtio_notify_vector(vdev, vdev->config_vector);
1273 }
1274
1275 static bool virtio_device_endian_needed(void *opaque)
1276 {
1277 VirtIODevice *vdev = opaque;
1278
1279 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
1280 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1281 return vdev->device_endian != virtio_default_endian();
1282 }
1283 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1284 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
1285 }
1286
1287 static bool virtio_64bit_features_needed(void *opaque)
1288 {
1289 VirtIODevice *vdev = opaque;
1290
1291 return (vdev->host_features >> 32) != 0;
1292 }
1293
1294 static bool virtio_virtqueue_needed(void *opaque)
1295 {
1296 VirtIODevice *vdev = opaque;
1297
1298 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
1299 }
1300
1301 static bool virtio_ringsize_needed(void *opaque)
1302 {
1303 VirtIODevice *vdev = opaque;
1304 int i;
1305
1306 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1307 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
1308 return true;
1309 }
1310 }
1311 return false;
1312 }
1313
1314 static bool virtio_extra_state_needed(void *opaque)
1315 {
1316 VirtIODevice *vdev = opaque;
1317 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1318 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1319
1320 return k->has_extra_state &&
1321 k->has_extra_state(qbus->parent);
1322 }
1323
1324 static const VMStateDescription vmstate_virtqueue = {
1325 .name = "virtqueue_state",
1326 .version_id = 1,
1327 .minimum_version_id = 1,
1328 .fields = (VMStateField[]) {
1329 VMSTATE_UINT64(vring.avail, struct VirtQueue),
1330 VMSTATE_UINT64(vring.used, struct VirtQueue),
1331 VMSTATE_END_OF_LIST()
1332 }
1333 };
1334
1335 static const VMStateDescription vmstate_virtio_virtqueues = {
1336 .name = "virtio/virtqueues",
1337 .version_id = 1,
1338 .minimum_version_id = 1,
1339 .needed = &virtio_virtqueue_needed,
1340 .fields = (VMStateField[]) {
1341 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1342 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
1343 VMSTATE_END_OF_LIST()
1344 }
1345 };
1346
1347 static const VMStateDescription vmstate_ringsize = {
1348 .name = "ringsize_state",
1349 .version_id = 1,
1350 .minimum_version_id = 1,
1351 .fields = (VMStateField[]) {
1352 VMSTATE_UINT32(vring.num_default, struct VirtQueue),
1353 VMSTATE_END_OF_LIST()
1354 }
1355 };
1356
1357 static const VMStateDescription vmstate_virtio_ringsize = {
1358 .name = "virtio/ringsize",
1359 .version_id = 1,
1360 .minimum_version_id = 1,
1361 .needed = &virtio_ringsize_needed,
1362 .fields = (VMStateField[]) {
1363 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1364 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
1365 VMSTATE_END_OF_LIST()
1366 }
1367 };
1368
1369 static int get_extra_state(QEMUFile *f, void *pv, size_t size)
1370 {
1371 VirtIODevice *vdev = pv;
1372 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1373 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1374
1375 if (!k->load_extra_state) {
1376 return -1;
1377 } else {
1378 return k->load_extra_state(qbus->parent, f);
1379 }
1380 }
1381
1382 static void put_extra_state(QEMUFile *f, void *pv, size_t size)
1383 {
1384 VirtIODevice *vdev = pv;
1385 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1386 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1387
1388 k->save_extra_state(qbus->parent, f);
1389 }
1390
1391 static const VMStateInfo vmstate_info_extra_state = {
1392 .name = "virtqueue_extra_state",
1393 .get = get_extra_state,
1394 .put = put_extra_state,
1395 };
1396
1397 static const VMStateDescription vmstate_virtio_extra_state = {
1398 .name = "virtio/extra_state",
1399 .version_id = 1,
1400 .minimum_version_id = 1,
1401 .needed = &virtio_extra_state_needed,
1402 .fields = (VMStateField[]) {
1403 {
1404 .name = "extra_state",
1405 .version_id = 0,
1406 .field_exists = NULL,
1407 .size = 0,
1408 .info = &vmstate_info_extra_state,
1409 .flags = VMS_SINGLE,
1410 .offset = 0,
1411 },
1412 VMSTATE_END_OF_LIST()
1413 }
1414 };
1415
1416 static const VMStateDescription vmstate_virtio_device_endian = {
1417 .name = "virtio/device_endian",
1418 .version_id = 1,
1419 .minimum_version_id = 1,
1420 .needed = &virtio_device_endian_needed,
1421 .fields = (VMStateField[]) {
1422 VMSTATE_UINT8(device_endian, VirtIODevice),
1423 VMSTATE_END_OF_LIST()
1424 }
1425 };
1426
1427 static const VMStateDescription vmstate_virtio_64bit_features = {
1428 .name = "virtio/64bit_features",
1429 .version_id = 1,
1430 .minimum_version_id = 1,
1431 .needed = &virtio_64bit_features_needed,
1432 .fields = (VMStateField[]) {
1433 VMSTATE_UINT64(guest_features, VirtIODevice),
1434 VMSTATE_END_OF_LIST()
1435 }
1436 };
1437
1438 static const VMStateDescription vmstate_virtio = {
1439 .name = "virtio",
1440 .version_id = 1,
1441 .minimum_version_id = 1,
1442 .minimum_version_id_old = 1,
1443 .fields = (VMStateField[]) {
1444 VMSTATE_END_OF_LIST()
1445 },
1446 .subsections = (const VMStateDescription*[]) {
1447 &vmstate_virtio_device_endian,
1448 &vmstate_virtio_64bit_features,
1449 &vmstate_virtio_virtqueues,
1450 &vmstate_virtio_ringsize,
1451 &vmstate_virtio_extra_state,
1452 NULL
1453 }
1454 };
1455
1456 void virtio_save(VirtIODevice *vdev, QEMUFile *f)
1457 {
1458 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1459 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1460 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1461 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
1462 int i;
1463
1464 if (k->save_config) {
1465 k->save_config(qbus->parent, f);
1466 }
1467
1468 qemu_put_8s(f, &vdev->status);
1469 qemu_put_8s(f, &vdev->isr);
1470 qemu_put_be16s(f, &vdev->queue_sel);
1471 qemu_put_be32s(f, &guest_features_lo);
1472 qemu_put_be32(f, vdev->config_len);
1473 qemu_put_buffer(f, vdev->config, vdev->config_len);
1474
1475 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1476 if (vdev->vq[i].vring.num == 0)
1477 break;
1478 }
1479
1480 qemu_put_be32(f, i);
1481
1482 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1483 if (vdev->vq[i].vring.num == 0)
1484 break;
1485
1486 qemu_put_be32(f, vdev->vq[i].vring.num);
1487 if (k->has_variable_vring_alignment) {
1488 qemu_put_be32(f, vdev->vq[i].vring.align);
1489 }
1490 /* XXX virtio-1 devices */
1491 qemu_put_be64(f, vdev->vq[i].vring.desc);
1492 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
1493 if (k->save_queue) {
1494 k->save_queue(qbus->parent, i, f);
1495 }
1496 }
1497
1498 if (vdc->save != NULL) {
1499 vdc->save(vdev, f);
1500 }
1501
1502 /* Subsections */
1503 vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
1504 }
1505
1506 /* A wrapper for use as a VMState .put function */
1507 void virtio_vmstate_save(QEMUFile *f, void *opaque, size_t size)
1508 {
1509 virtio_save(VIRTIO_DEVICE(opaque), f);
1510 }
1511
1512 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
1513 {
1514 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1515 bool bad = (val & ~(vdev->host_features)) != 0;
1516
1517 val &= vdev->host_features;
1518 if (k->set_features) {
1519 k->set_features(vdev, val);
1520 }
1521 vdev->guest_features = val;
1522 return bad ? -1 : 0;
1523 }
1524
1525 int virtio_set_features(VirtIODevice *vdev, uint64_t val)
1526 {
1527 /*
1528 * The driver must not attempt to set features after feature negotiation
1529 * has finished.
1530 */
1531 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
1532 return -EINVAL;
1533 }
1534 return virtio_set_features_nocheck(vdev, val);
1535 }
1536
1537 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
1538 {
1539 int i, ret;
1540 int32_t config_len;
1541 uint32_t num;
1542 uint32_t features;
1543 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1544 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1545 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1546
1547 /*
1548 * We poison the endianness to ensure it does not get used before
1549 * subsections have been loaded.
1550 */
1551 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
1552
1553 if (k->load_config) {
1554 ret = k->load_config(qbus->parent, f);
1555 if (ret)
1556 return ret;
1557 }
1558
1559 qemu_get_8s(f, &vdev->status);
1560 qemu_get_8s(f, &vdev->isr);
1561 qemu_get_be16s(f, &vdev->queue_sel);
1562 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
1563 return -1;
1564 }
1565 qemu_get_be32s(f, &features);
1566
1567 /*
1568 * Temporarily set guest_features low bits - needed by
1569 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
1570 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
1571 *
1572 * Note: devices should always test host features in future - don't create
1573 * new dependencies like this.
1574 */
1575 vdev->guest_features = features;
1576
1577 config_len = qemu_get_be32(f);
1578
1579 /*
1580 * There are cases where the incoming config can be bigger or smaller
1581 * than what we have; so load what we have space for, and skip
1582 * any excess that's in the stream.
1583 */
1584 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
1585
1586 while (config_len > vdev->config_len) {
1587 qemu_get_byte(f);
1588 config_len--;
1589 }
1590
1591 num = qemu_get_be32(f);
1592
1593 if (num > VIRTIO_QUEUE_MAX) {
1594 error_report("Invalid number of virtqueues: 0x%x", num);
1595 return -1;
1596 }
1597
1598 for (i = 0; i < num; i++) {
1599 vdev->vq[i].vring.num = qemu_get_be32(f);
1600 if (k->has_variable_vring_alignment) {
1601 vdev->vq[i].vring.align = qemu_get_be32(f);
1602 }
1603 vdev->vq[i].vring.desc = qemu_get_be64(f);
1604 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
1605 vdev->vq[i].signalled_used_valid = false;
1606 vdev->vq[i].notification = true;
1607
1608 if (vdev->vq[i].vring.desc) {
1609 /* XXX virtio-1 devices */
1610 virtio_queue_update_rings(vdev, i);
1611 } else if (vdev->vq[i].last_avail_idx) {
1612 error_report("VQ %d address 0x0 "
1613 "inconsistent with Host index 0x%x",
1614 i, vdev->vq[i].last_avail_idx);
1615 return -1;
1616 }
1617 if (k->load_queue) {
1618 ret = k->load_queue(qbus->parent, i, f);
1619 if (ret)
1620 return ret;
1621 }
1622 }
1623
1624 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1625
1626 if (vdc->load != NULL) {
1627 ret = vdc->load(vdev, f, version_id);
1628 if (ret) {
1629 return ret;
1630 }
1631 }
1632
1633 /* Subsections */
1634 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
1635 if (ret) {
1636 return ret;
1637 }
1638
1639 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
1640 vdev->device_endian = virtio_default_endian();
1641 }
1642
1643 if (virtio_64bit_features_needed(vdev)) {
1644 /*
1645 * Subsection load filled vdev->guest_features. Run them
1646 * through virtio_set_features to sanity-check them against
1647 * host_features.
1648 */
1649 uint64_t features64 = vdev->guest_features;
1650 if (virtio_set_features_nocheck(vdev, features64) < 0) {
1651 error_report("Features 0x%" PRIx64 " unsupported. "
1652 "Allowed features: 0x%" PRIx64,
1653 features64, vdev->host_features);
1654 return -1;
1655 }
1656 } else {
1657 if (virtio_set_features_nocheck(vdev, features) < 0) {
1658 error_report("Features 0x%x unsupported. "
1659 "Allowed features: 0x%" PRIx64,
1660 features, vdev->host_features);
1661 return -1;
1662 }
1663 }
1664
1665 for (i = 0; i < num; i++) {
1666 if (vdev->vq[i].vring.desc) {
1667 uint16_t nheads;
1668 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
1669 /* Check it isn't doing strange things with descriptor numbers. */
1670 if (nheads > vdev->vq[i].vring.num) {
1671 error_report("VQ %d size 0x%x Guest index 0x%x "
1672 "inconsistent with Host index 0x%x: delta 0x%x",
1673 i, vdev->vq[i].vring.num,
1674 vring_avail_idx(&vdev->vq[i]),
1675 vdev->vq[i].last_avail_idx, nheads);
1676 return -1;
1677 }
1678 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
1679 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
1680
1681 /*
1682 * Some devices migrate VirtQueueElements that have been popped
1683 * from the avail ring but not yet returned to the used ring.
1684 */
1685 vdev->vq[i].inuse = vdev->vq[i].last_avail_idx -
1686 vdev->vq[i].used_idx;
1687 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
1688 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
1689 "used_idx 0x%x",
1690 i, vdev->vq[i].vring.num,
1691 vdev->vq[i].last_avail_idx,
1692 vdev->vq[i].used_idx);
1693 return -1;
1694 }
1695 }
1696 }
1697
1698 return 0;
1699 }
1700
1701 void virtio_cleanup(VirtIODevice *vdev)
1702 {
1703 qemu_del_vm_change_state_handler(vdev->vmstate);
1704 g_free(vdev->config);
1705 g_free(vdev->vq);
1706 g_free(vdev->vector_queues);
1707 }
1708
1709 static void virtio_vmstate_change(void *opaque, int running, RunState state)
1710 {
1711 VirtIODevice *vdev = opaque;
1712 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1713 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1714 bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
1715 vdev->vm_running = running;
1716
1717 if (backend_run) {
1718 virtio_set_status(vdev, vdev->status);
1719 }
1720
1721 if (k->vmstate_change) {
1722 k->vmstate_change(qbus->parent, backend_run);
1723 }
1724
1725 if (!backend_run) {
1726 virtio_set_status(vdev, vdev->status);
1727 }
1728 }
1729
1730 void virtio_instance_init_common(Object *proxy_obj, void *data,
1731 size_t vdev_size, const char *vdev_name)
1732 {
1733 DeviceState *vdev = data;
1734
1735 object_initialize(vdev, vdev_size, vdev_name);
1736 object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL);
1737 object_unref(OBJECT(vdev));
1738 qdev_alias_all_properties(vdev, proxy_obj);
1739 }
1740
1741 void virtio_init(VirtIODevice *vdev, const char *name,
1742 uint16_t device_id, size_t config_size)
1743 {
1744 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1745 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1746 int i;
1747 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
1748
1749 if (nvectors) {
1750 vdev->vector_queues =
1751 g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
1752 }
1753
1754 vdev->device_id = device_id;
1755 vdev->status = 0;
1756 vdev->isr = 0;
1757 vdev->queue_sel = 0;
1758 vdev->config_vector = VIRTIO_NO_VECTOR;
1759 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
1760 vdev->vm_running = runstate_is_running();
1761 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1762 vdev->vq[i].vector = VIRTIO_NO_VECTOR;
1763 vdev->vq[i].vdev = vdev;
1764 vdev->vq[i].queue_index = i;
1765 }
1766
1767 vdev->name = name;
1768 vdev->config_len = config_size;
1769 if (vdev->config_len) {
1770 vdev->config = g_malloc0(config_size);
1771 } else {
1772 vdev->config = NULL;
1773 }
1774 vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
1775 vdev);
1776 vdev->device_endian = virtio_default_endian();
1777 vdev->use_guest_notifier_mask = true;
1778 }
1779
1780 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
1781 {
1782 return vdev->vq[n].vring.desc;
1783 }
1784
1785 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
1786 {
1787 return vdev->vq[n].vring.avail;
1788 }
1789
1790 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
1791 {
1792 return vdev->vq[n].vring.used;
1793 }
1794
1795 hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
1796 {
1797 return vdev->vq[n].vring.desc;
1798 }
1799
1800 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
1801 {
1802 return sizeof(VRingDesc) * vdev->vq[n].vring.num;
1803 }
1804
1805 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
1806 {
1807 return offsetof(VRingAvail, ring) +
1808 sizeof(uint16_t) * vdev->vq[n].vring.num;
1809 }
1810
1811 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
1812 {
1813 return offsetof(VRingUsed, ring) +
1814 sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
1815 }
1816
1817 hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
1818 {
1819 return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
1820 virtio_queue_get_used_size(vdev, n);
1821 }
1822
1823 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
1824 {
1825 return vdev->vq[n].last_avail_idx;
1826 }
1827
1828 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
1829 {
1830 vdev->vq[n].last_avail_idx = idx;
1831 vdev->vq[n].shadow_avail_idx = idx;
1832 }
1833
1834 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
1835 {
1836 vdev->vq[n].signalled_used_valid = false;
1837 }
1838
1839 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
1840 {
1841 return vdev->vq + n;
1842 }
1843
1844 uint16_t virtio_get_queue_index(VirtQueue *vq)
1845 {
1846 return vq->queue_index;
1847 }
1848
1849 static void virtio_queue_guest_notifier_read(EventNotifier *n)
1850 {
1851 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
1852 if (event_notifier_test_and_clear(n)) {
1853 virtio_irq(vq);
1854 }
1855 }
1856
1857 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
1858 bool with_irqfd)
1859 {
1860 if (assign && !with_irqfd) {
1861 event_notifier_set_handler(&vq->guest_notifier, false,
1862 virtio_queue_guest_notifier_read);
1863 } else {
1864 event_notifier_set_handler(&vq->guest_notifier, false, NULL);
1865 }
1866 if (!assign) {
1867 /* Test and clear notifier before closing it,
1868 * in case poll callback didn't have time to run. */
1869 virtio_queue_guest_notifier_read(&vq->guest_notifier);
1870 }
1871 }
1872
1873 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
1874 {
1875 return &vq->guest_notifier;
1876 }
1877
1878 static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
1879 {
1880 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1881 if (event_notifier_test_and_clear(n)) {
1882 virtio_queue_notify_aio_vq(vq);
1883 }
1884 }
1885
1886 void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
1887 VirtIOHandleOutput handle_output)
1888 {
1889 if (handle_output) {
1890 vq->handle_aio_output = handle_output;
1891 aio_set_event_notifier(ctx, &vq->host_notifier, true,
1892 virtio_queue_host_notifier_aio_read);
1893 } else {
1894 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL);
1895 /* Test and clear notifier before after disabling event,
1896 * in case poll callback didn't have time to run. */
1897 virtio_queue_host_notifier_aio_read(&vq->host_notifier);
1898 vq->handle_aio_output = NULL;
1899 }
1900 }
1901
1902 static void virtio_queue_host_notifier_read(EventNotifier *n)
1903 {
1904 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1905 if (event_notifier_test_and_clear(n)) {
1906 virtio_queue_notify_vq(vq);
1907 }
1908 }
1909
1910 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
1911 bool set_handler)
1912 {
1913 AioContext *ctx = qemu_get_aio_context();
1914 if (assign && set_handler) {
1915 if (vq->use_aio) {
1916 aio_set_event_notifier(ctx, &vq->host_notifier, true,
1917 virtio_queue_host_notifier_read);
1918 } else {
1919 event_notifier_set_handler(&vq->host_notifier, true,
1920 virtio_queue_host_notifier_read);
1921 }
1922 } else {
1923 if (vq->use_aio) {
1924 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL);
1925 } else {
1926 event_notifier_set_handler(&vq->host_notifier, true, NULL);
1927 }
1928 }
1929 if (!assign) {
1930 /* Test and clear notifier before after disabling event,
1931 * in case poll callback didn't have time to run. */
1932 virtio_queue_host_notifier_read(&vq->host_notifier);
1933 }
1934 }
1935
1936 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
1937 {
1938 return &vq->host_notifier;
1939 }
1940
1941 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
1942 {
1943 g_free(vdev->bus_name);
1944 vdev->bus_name = g_strdup(bus_name);
1945 }
1946
1947 static void virtio_device_realize(DeviceState *dev, Error **errp)
1948 {
1949 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1950 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1951 Error *err = NULL;
1952
1953 if (vdc->realize != NULL) {
1954 vdc->realize(dev, &err);
1955 if (err != NULL) {
1956 error_propagate(errp, err);
1957 return;
1958 }
1959 }
1960
1961 virtio_bus_device_plugged(vdev, &err);
1962 if (err != NULL) {
1963 error_propagate(errp, err);
1964 return;
1965 }
1966 }
1967
1968 static void virtio_device_unrealize(DeviceState *dev, Error **errp)
1969 {
1970 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1971 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1972 Error *err = NULL;
1973
1974 virtio_bus_device_unplugged(vdev);
1975
1976 if (vdc->unrealize != NULL) {
1977 vdc->unrealize(dev, &err);
1978 if (err != NULL) {
1979 error_propagate(errp, err);
1980 return;
1981 }
1982 }
1983
1984 g_free(vdev->bus_name);
1985 vdev->bus_name = NULL;
1986 }
1987
1988 static Property virtio_properties[] = {
1989 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
1990 DEFINE_PROP_END_OF_LIST(),
1991 };
1992
1993 static void virtio_device_class_init(ObjectClass *klass, void *data)
1994 {
1995 /* Set the default value here. */
1996 DeviceClass *dc = DEVICE_CLASS(klass);
1997
1998 dc->realize = virtio_device_realize;
1999 dc->unrealize = virtio_device_unrealize;
2000 dc->bus_type = TYPE_VIRTIO_BUS;
2001 dc->props = virtio_properties;
2002 }
2003
2004 static const TypeInfo virtio_device_info = {
2005 .name = TYPE_VIRTIO_DEVICE,
2006 .parent = TYPE_DEVICE,
2007 .instance_size = sizeof(VirtIODevice),
2008 .class_init = virtio_device_class_init,
2009 .abstract = true,
2010 .class_size = sizeof(VirtioDeviceClass),
2011 };
2012
2013 static void virtio_register_types(void)
2014 {
2015 type_register_static(&virtio_device_info);
2016 }
2017
2018 type_init(virtio_register_types)