virtio: check vring descriptor buffer length
[qemu.git] / hw / virtio / virtio.c
1 /*
2 * Virtio Support
3 *
4 * Copyright IBM, Corp. 2007
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 */
13
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "qemu-common.h"
17 #include "cpu.h"
18 #include "trace.h"
19 #include "exec/address-spaces.h"
20 #include "qemu/error-report.h"
21 #include "hw/virtio/virtio.h"
22 #include "qemu/atomic.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "migration/migration.h"
25 #include "hw/virtio/virtio-access.h"
26
27 /*
28 * The alignment to use between consumer and producer parts of vring.
29 * x86 pagesize again. This is the default, used by transports like PCI
30 * which don't provide a means for the guest to tell the host the alignment.
31 */
32 #define VIRTIO_PCI_VRING_ALIGN 4096
33
34 typedef struct VRingDesc
35 {
36 uint64_t addr;
37 uint32_t len;
38 uint16_t flags;
39 uint16_t next;
40 } VRingDesc;
41
42 typedef struct VRingAvail
43 {
44 uint16_t flags;
45 uint16_t idx;
46 uint16_t ring[0];
47 } VRingAvail;
48
49 typedef struct VRingUsedElem
50 {
51 uint32_t id;
52 uint32_t len;
53 } VRingUsedElem;
54
55 typedef struct VRingUsed
56 {
57 uint16_t flags;
58 uint16_t idx;
59 VRingUsedElem ring[0];
60 } VRingUsed;
61
62 typedef struct VRing
63 {
64 unsigned int num;
65 unsigned int num_default;
66 unsigned int align;
67 hwaddr desc;
68 hwaddr avail;
69 hwaddr used;
70 } VRing;
71
72 struct VirtQueue
73 {
74 VRing vring;
75
76 /* Next head to pop */
77 uint16_t last_avail_idx;
78
79 /* Last avail_idx read from VQ. */
80 uint16_t shadow_avail_idx;
81
82 uint16_t used_idx;
83
84 /* Last used index value we have signalled on */
85 uint16_t signalled_used;
86
87 /* Last used index value we have signalled on */
88 bool signalled_used_valid;
89
90 /* Notification enabled? */
91 bool notification;
92
93 uint16_t queue_index;
94
95 int inuse;
96
97 uint16_t vector;
98 VirtIOHandleOutput handle_output;
99 VirtIOHandleOutput handle_aio_output;
100 bool use_aio;
101 VirtIODevice *vdev;
102 EventNotifier guest_notifier;
103 EventNotifier host_notifier;
104 QLIST_ENTRY(VirtQueue) node;
105 };
106
107 /* virt queue functions */
108 void virtio_queue_update_rings(VirtIODevice *vdev, int n)
109 {
110 VRing *vring = &vdev->vq[n].vring;
111
112 if (!vring->desc) {
113 /* not yet setup -> nothing to do */
114 return;
115 }
116 vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
117 vring->used = vring_align(vring->avail +
118 offsetof(VRingAvail, ring[vring->num]),
119 vring->align);
120 }
121
122 static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
123 hwaddr desc_pa, int i)
124 {
125 address_space_read(&address_space_memory, desc_pa + i * sizeof(VRingDesc),
126 MEMTXATTRS_UNSPECIFIED, (void *)desc, sizeof(VRingDesc));
127 virtio_tswap64s(vdev, &desc->addr);
128 virtio_tswap32s(vdev, &desc->len);
129 virtio_tswap16s(vdev, &desc->flags);
130 virtio_tswap16s(vdev, &desc->next);
131 }
132
133 static inline uint16_t vring_avail_flags(VirtQueue *vq)
134 {
135 hwaddr pa;
136 pa = vq->vring.avail + offsetof(VRingAvail, flags);
137 return virtio_lduw_phys(vq->vdev, pa);
138 }
139
140 static inline uint16_t vring_avail_idx(VirtQueue *vq)
141 {
142 hwaddr pa;
143 pa = vq->vring.avail + offsetof(VRingAvail, idx);
144 vq->shadow_avail_idx = virtio_lduw_phys(vq->vdev, pa);
145 return vq->shadow_avail_idx;
146 }
147
148 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
149 {
150 hwaddr pa;
151 pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
152 return virtio_lduw_phys(vq->vdev, pa);
153 }
154
155 static inline uint16_t vring_get_used_event(VirtQueue *vq)
156 {
157 return vring_avail_ring(vq, vq->vring.num);
158 }
159
160 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
161 int i)
162 {
163 hwaddr pa;
164 virtio_tswap32s(vq->vdev, &uelem->id);
165 virtio_tswap32s(vq->vdev, &uelem->len);
166 pa = vq->vring.used + offsetof(VRingUsed, ring[i]);
167 address_space_write(&address_space_memory, pa, MEMTXATTRS_UNSPECIFIED,
168 (void *)uelem, sizeof(VRingUsedElem));
169 }
170
171 static uint16_t vring_used_idx(VirtQueue *vq)
172 {
173 hwaddr pa;
174 pa = vq->vring.used + offsetof(VRingUsed, idx);
175 return virtio_lduw_phys(vq->vdev, pa);
176 }
177
178 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
179 {
180 hwaddr pa;
181 pa = vq->vring.used + offsetof(VRingUsed, idx);
182 virtio_stw_phys(vq->vdev, pa, val);
183 vq->used_idx = val;
184 }
185
186 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
187 {
188 VirtIODevice *vdev = vq->vdev;
189 hwaddr pa;
190 pa = vq->vring.used + offsetof(VRingUsed, flags);
191 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) | mask);
192 }
193
194 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
195 {
196 VirtIODevice *vdev = vq->vdev;
197 hwaddr pa;
198 pa = vq->vring.used + offsetof(VRingUsed, flags);
199 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask);
200 }
201
202 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
203 {
204 hwaddr pa;
205 if (!vq->notification) {
206 return;
207 }
208 pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]);
209 virtio_stw_phys(vq->vdev, pa, val);
210 }
211
212 void virtio_queue_set_notification(VirtQueue *vq, int enable)
213 {
214 vq->notification = enable;
215 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
216 vring_set_avail_event(vq, vring_avail_idx(vq));
217 } else if (enable) {
218 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
219 } else {
220 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
221 }
222 if (enable) {
223 /* Expose avail event/used flags before caller checks the avail idx. */
224 smp_mb();
225 }
226 }
227
228 int virtio_queue_ready(VirtQueue *vq)
229 {
230 return vq->vring.avail != 0;
231 }
232
233 /* Fetch avail_idx from VQ memory only when we really need to know if
234 * guest has added some buffers. */
235 int virtio_queue_empty(VirtQueue *vq)
236 {
237 if (vq->shadow_avail_idx != vq->last_avail_idx) {
238 return 0;
239 }
240
241 return vring_avail_idx(vq) == vq->last_avail_idx;
242 }
243
244 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
245 unsigned int len)
246 {
247 unsigned int offset;
248 int i;
249
250 offset = 0;
251 for (i = 0; i < elem->in_num; i++) {
252 size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
253
254 cpu_physical_memory_unmap(elem->in_sg[i].iov_base,
255 elem->in_sg[i].iov_len,
256 1, size);
257
258 offset += size;
259 }
260
261 for (i = 0; i < elem->out_num; i++)
262 cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
263 elem->out_sg[i].iov_len,
264 0, elem->out_sg[i].iov_len);
265 }
266
267 void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
268 unsigned int len)
269 {
270 vq->last_avail_idx--;
271 virtqueue_unmap_sg(vq, elem, len);
272 }
273
274 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
275 unsigned int len, unsigned int idx)
276 {
277 VRingUsedElem uelem;
278
279 trace_virtqueue_fill(vq, elem, len, idx);
280
281 virtqueue_unmap_sg(vq, elem, len);
282
283 idx = (idx + vq->used_idx) % vq->vring.num;
284
285 uelem.id = elem->index;
286 uelem.len = len;
287 vring_used_write(vq, &uelem, idx);
288 }
289
290 void virtqueue_flush(VirtQueue *vq, unsigned int count)
291 {
292 uint16_t old, new;
293 /* Make sure buffer is written before we update index. */
294 smp_wmb();
295 trace_virtqueue_flush(vq, count);
296 old = vq->used_idx;
297 new = old + count;
298 vring_used_idx_set(vq, new);
299 vq->inuse -= count;
300 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
301 vq->signalled_used_valid = false;
302 }
303
304 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
305 unsigned int len)
306 {
307 virtqueue_fill(vq, elem, len, 0);
308 virtqueue_flush(vq, 1);
309 }
310
311 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
312 {
313 uint16_t num_heads = vring_avail_idx(vq) - idx;
314
315 /* Check it isn't doing very strange things with descriptor numbers. */
316 if (num_heads > vq->vring.num) {
317 error_report("Guest moved used index from %u to %u",
318 idx, vq->shadow_avail_idx);
319 exit(1);
320 }
321 /* On success, callers read a descriptor at vq->last_avail_idx.
322 * Make sure descriptor read does not bypass avail index read. */
323 if (num_heads) {
324 smp_rmb();
325 }
326
327 return num_heads;
328 }
329
330 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
331 {
332 unsigned int head;
333
334 /* Grab the next descriptor number they're advertising, and increment
335 * the index we've seen. */
336 head = vring_avail_ring(vq, idx % vq->vring.num);
337
338 /* If their number is silly, that's a fatal mistake. */
339 if (head >= vq->vring.num) {
340 error_report("Guest says index %u is available", head);
341 exit(1);
342 }
343
344 return head;
345 }
346
347 static unsigned virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
348 hwaddr desc_pa, unsigned int max)
349 {
350 unsigned int next;
351
352 /* If this descriptor says it doesn't chain, we're done. */
353 if (!(desc->flags & VRING_DESC_F_NEXT)) {
354 return max;
355 }
356
357 /* Check they're not leading us off end of descriptors. */
358 next = desc->next;
359 /* Make sure compiler knows to grab that: we don't want it changing! */
360 smp_wmb();
361
362 if (next >= max) {
363 error_report("Desc next is %u", next);
364 exit(1);
365 }
366
367 vring_desc_read(vdev, desc, desc_pa, next);
368 return next;
369 }
370
371 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
372 unsigned int *out_bytes,
373 unsigned max_in_bytes, unsigned max_out_bytes)
374 {
375 unsigned int idx;
376 unsigned int total_bufs, in_total, out_total;
377
378 idx = vq->last_avail_idx;
379
380 total_bufs = in_total = out_total = 0;
381 while (virtqueue_num_heads(vq, idx)) {
382 VirtIODevice *vdev = vq->vdev;
383 unsigned int max, num_bufs, indirect = 0;
384 VRingDesc desc;
385 hwaddr desc_pa;
386 int i;
387
388 max = vq->vring.num;
389 num_bufs = total_bufs;
390 i = virtqueue_get_head(vq, idx++);
391 desc_pa = vq->vring.desc;
392 vring_desc_read(vdev, &desc, desc_pa, i);
393
394 if (desc.flags & VRING_DESC_F_INDIRECT) {
395 if (desc.len % sizeof(VRingDesc)) {
396 error_report("Invalid size for indirect buffer table");
397 exit(1);
398 }
399
400 /* If we've got too many, that implies a descriptor loop. */
401 if (num_bufs >= max) {
402 error_report("Looped descriptor");
403 exit(1);
404 }
405
406 /* loop over the indirect descriptor table */
407 indirect = 1;
408 max = desc.len / sizeof(VRingDesc);
409 desc_pa = desc.addr;
410 num_bufs = i = 0;
411 vring_desc_read(vdev, &desc, desc_pa, i);
412 }
413
414 do {
415 /* If we've got too many, that implies a descriptor loop. */
416 if (++num_bufs > max) {
417 error_report("Looped descriptor");
418 exit(1);
419 }
420
421 if (desc.flags & VRING_DESC_F_WRITE) {
422 in_total += desc.len;
423 } else {
424 out_total += desc.len;
425 }
426 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
427 goto done;
428 }
429 } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
430
431 if (!indirect)
432 total_bufs = num_bufs;
433 else
434 total_bufs++;
435 }
436 done:
437 if (in_bytes) {
438 *in_bytes = in_total;
439 }
440 if (out_bytes) {
441 *out_bytes = out_total;
442 }
443 }
444
445 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
446 unsigned int out_bytes)
447 {
448 unsigned int in_total, out_total;
449
450 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
451 return in_bytes <= in_total && out_bytes <= out_total;
452 }
453
454 static void virtqueue_map_desc(unsigned int *p_num_sg, hwaddr *addr, struct iovec *iov,
455 unsigned int max_num_sg, bool is_write,
456 hwaddr pa, size_t sz)
457 {
458 unsigned num_sg = *p_num_sg;
459 assert(num_sg <= max_num_sg);
460
461 if (!sz) {
462 error_report("virtio: zero sized buffers are not allowed");
463 exit(1);
464 }
465
466 while (sz) {
467 hwaddr len = sz;
468
469 if (num_sg == max_num_sg) {
470 error_report("virtio: too many write descriptors in indirect table");
471 exit(1);
472 }
473
474 iov[num_sg].iov_base = cpu_physical_memory_map(pa, &len, is_write);
475 iov[num_sg].iov_len = len;
476 addr[num_sg] = pa;
477
478 sz -= len;
479 pa += len;
480 num_sg++;
481 }
482 *p_num_sg = num_sg;
483 }
484
485 static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr,
486 unsigned int *num_sg, unsigned int max_size,
487 int is_write)
488 {
489 unsigned int i;
490 hwaddr len;
491
492 /* Note: this function MUST validate input, some callers
493 * are passing in num_sg values received over the network.
494 */
495 /* TODO: teach all callers that this can fail, and return failure instead
496 * of asserting here.
497 * When we do, we might be able to re-enable NDEBUG below.
498 */
499 #ifdef NDEBUG
500 #error building with NDEBUG is not supported
501 #endif
502 assert(*num_sg <= max_size);
503
504 for (i = 0; i < *num_sg; i++) {
505 len = sg[i].iov_len;
506 sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
507 if (!sg[i].iov_base) {
508 error_report("virtio: error trying to map MMIO memory");
509 exit(1);
510 }
511 if (len != sg[i].iov_len) {
512 error_report("virtio: unexpected memory split");
513 exit(1);
514 }
515 }
516 }
517
518 void virtqueue_map(VirtQueueElement *elem)
519 {
520 virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num,
521 VIRTQUEUE_MAX_SIZE, 1);
522 virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num,
523 VIRTQUEUE_MAX_SIZE, 0);
524 }
525
526 void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
527 {
528 VirtQueueElement *elem;
529 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
530 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
531 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
532 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
533 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
534 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
535
536 assert(sz >= sizeof(VirtQueueElement));
537 elem = g_malloc(out_sg_end);
538 elem->out_num = out_num;
539 elem->in_num = in_num;
540 elem->in_addr = (void *)elem + in_addr_ofs;
541 elem->out_addr = (void *)elem + out_addr_ofs;
542 elem->in_sg = (void *)elem + in_sg_ofs;
543 elem->out_sg = (void *)elem + out_sg_ofs;
544 return elem;
545 }
546
547 void *virtqueue_pop(VirtQueue *vq, size_t sz)
548 {
549 unsigned int i, head, max;
550 hwaddr desc_pa = vq->vring.desc;
551 VirtIODevice *vdev = vq->vdev;
552 VirtQueueElement *elem;
553 unsigned out_num, in_num;
554 hwaddr addr[VIRTQUEUE_MAX_SIZE];
555 struct iovec iov[VIRTQUEUE_MAX_SIZE];
556 VRingDesc desc;
557
558 if (virtio_queue_empty(vq)) {
559 return NULL;
560 }
561 /* Needed after virtio_queue_empty(), see comment in
562 * virtqueue_num_heads(). */
563 smp_rmb();
564
565 /* When we start there are none of either input nor output. */
566 out_num = in_num = 0;
567
568 max = vq->vring.num;
569
570 i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
571 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
572 vring_set_avail_event(vq, vq->last_avail_idx);
573 }
574
575 vring_desc_read(vdev, &desc, desc_pa, i);
576 if (desc.flags & VRING_DESC_F_INDIRECT) {
577 if (desc.len % sizeof(VRingDesc)) {
578 error_report("Invalid size for indirect buffer table");
579 exit(1);
580 }
581
582 /* loop over the indirect descriptor table */
583 max = desc.len / sizeof(VRingDesc);
584 desc_pa = desc.addr;
585 i = 0;
586 vring_desc_read(vdev, &desc, desc_pa, i);
587 }
588
589 /* Collect all the descriptors */
590 do {
591 if (desc.flags & VRING_DESC_F_WRITE) {
592 virtqueue_map_desc(&in_num, addr + out_num, iov + out_num,
593 VIRTQUEUE_MAX_SIZE - out_num, true, desc.addr, desc.len);
594 } else {
595 if (in_num) {
596 error_report("Incorrect order for descriptors");
597 exit(1);
598 }
599 virtqueue_map_desc(&out_num, addr, iov,
600 VIRTQUEUE_MAX_SIZE, false, desc.addr, desc.len);
601 }
602
603 /* If we've got too many, that implies a descriptor loop. */
604 if ((in_num + out_num) > max) {
605 error_report("Looped descriptor");
606 exit(1);
607 }
608 } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
609
610 /* Now copy what we have collected and mapped */
611 elem = virtqueue_alloc_element(sz, out_num, in_num);
612 elem->index = head;
613 for (i = 0; i < out_num; i++) {
614 elem->out_addr[i] = addr[i];
615 elem->out_sg[i] = iov[i];
616 }
617 for (i = 0; i < in_num; i++) {
618 elem->in_addr[i] = addr[out_num + i];
619 elem->in_sg[i] = iov[out_num + i];
620 }
621
622 vq->inuse++;
623
624 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
625 return elem;
626 }
627
628 /* Reading and writing a structure directly to QEMUFile is *awful*, but
629 * it is what QEMU has always done by mistake. We can change it sooner
630 * or later by bumping the version number of the affected vm states.
631 * In the meanwhile, since the in-memory layout of VirtQueueElement
632 * has changed, we need to marshal to and from the layout that was
633 * used before the change.
634 */
635 typedef struct VirtQueueElementOld {
636 unsigned int index;
637 unsigned int out_num;
638 unsigned int in_num;
639 hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
640 hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
641 struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
642 struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
643 } VirtQueueElementOld;
644
645 void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz)
646 {
647 VirtQueueElement *elem;
648 VirtQueueElementOld data;
649 int i;
650
651 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
652
653 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
654 elem->index = data.index;
655
656 for (i = 0; i < elem->in_num; i++) {
657 elem->in_addr[i] = data.in_addr[i];
658 }
659
660 for (i = 0; i < elem->out_num; i++) {
661 elem->out_addr[i] = data.out_addr[i];
662 }
663
664 for (i = 0; i < elem->in_num; i++) {
665 /* Base is overwritten by virtqueue_map. */
666 elem->in_sg[i].iov_base = 0;
667 elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
668 }
669
670 for (i = 0; i < elem->out_num; i++) {
671 /* Base is overwritten by virtqueue_map. */
672 elem->out_sg[i].iov_base = 0;
673 elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
674 }
675
676 virtqueue_map(elem);
677 return elem;
678 }
679
680 void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
681 {
682 VirtQueueElementOld data;
683 int i;
684
685 memset(&data, 0, sizeof(data));
686 data.index = elem->index;
687 data.in_num = elem->in_num;
688 data.out_num = elem->out_num;
689
690 for (i = 0; i < elem->in_num; i++) {
691 data.in_addr[i] = elem->in_addr[i];
692 }
693
694 for (i = 0; i < elem->out_num; i++) {
695 data.out_addr[i] = elem->out_addr[i];
696 }
697
698 for (i = 0; i < elem->in_num; i++) {
699 /* Base is overwritten by virtqueue_map when loading. Do not
700 * save it, as it would leak the QEMU address space layout. */
701 data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
702 }
703
704 for (i = 0; i < elem->out_num; i++) {
705 /* Do not save iov_base as above. */
706 data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
707 }
708 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
709 }
710
711 /* virtio device */
712 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
713 {
714 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
715 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
716
717 if (k->notify) {
718 k->notify(qbus->parent, vector);
719 }
720 }
721
722 void virtio_update_irq(VirtIODevice *vdev)
723 {
724 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
725 }
726
727 static int virtio_validate_features(VirtIODevice *vdev)
728 {
729 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
730
731 if (k->validate_features) {
732 return k->validate_features(vdev);
733 } else {
734 return 0;
735 }
736 }
737
738 int virtio_set_status(VirtIODevice *vdev, uint8_t val)
739 {
740 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
741 trace_virtio_set_status(vdev, val);
742
743 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
744 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
745 val & VIRTIO_CONFIG_S_FEATURES_OK) {
746 int ret = virtio_validate_features(vdev);
747
748 if (ret) {
749 return ret;
750 }
751 }
752 }
753 if (k->set_status) {
754 k->set_status(vdev, val);
755 }
756 vdev->status = val;
757 return 0;
758 }
759
760 bool target_words_bigendian(void);
761 static enum virtio_device_endian virtio_default_endian(void)
762 {
763 if (target_words_bigendian()) {
764 return VIRTIO_DEVICE_ENDIAN_BIG;
765 } else {
766 return VIRTIO_DEVICE_ENDIAN_LITTLE;
767 }
768 }
769
770 static enum virtio_device_endian virtio_current_cpu_endian(void)
771 {
772 CPUClass *cc = CPU_GET_CLASS(current_cpu);
773
774 if (cc->virtio_is_big_endian(current_cpu)) {
775 return VIRTIO_DEVICE_ENDIAN_BIG;
776 } else {
777 return VIRTIO_DEVICE_ENDIAN_LITTLE;
778 }
779 }
780
781 void virtio_reset(void *opaque)
782 {
783 VirtIODevice *vdev = opaque;
784 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
785 int i;
786
787 virtio_set_status(vdev, 0);
788 if (current_cpu) {
789 /* Guest initiated reset */
790 vdev->device_endian = virtio_current_cpu_endian();
791 } else {
792 /* System reset */
793 vdev->device_endian = virtio_default_endian();
794 }
795
796 if (k->reset) {
797 k->reset(vdev);
798 }
799
800 vdev->guest_features = 0;
801 vdev->queue_sel = 0;
802 vdev->status = 0;
803 vdev->isr = 0;
804 vdev->config_vector = VIRTIO_NO_VECTOR;
805 virtio_notify_vector(vdev, vdev->config_vector);
806
807 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
808 vdev->vq[i].vring.desc = 0;
809 vdev->vq[i].vring.avail = 0;
810 vdev->vq[i].vring.used = 0;
811 vdev->vq[i].last_avail_idx = 0;
812 vdev->vq[i].shadow_avail_idx = 0;
813 vdev->vq[i].used_idx = 0;
814 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
815 vdev->vq[i].signalled_used = 0;
816 vdev->vq[i].signalled_used_valid = false;
817 vdev->vq[i].notification = true;
818 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
819 }
820 }
821
822 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
823 {
824 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
825 uint8_t val;
826
827 if (addr + sizeof(val) > vdev->config_len) {
828 return (uint32_t)-1;
829 }
830
831 k->get_config(vdev, vdev->config);
832
833 val = ldub_p(vdev->config + addr);
834 return val;
835 }
836
837 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
838 {
839 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
840 uint16_t val;
841
842 if (addr + sizeof(val) > vdev->config_len) {
843 return (uint32_t)-1;
844 }
845
846 k->get_config(vdev, vdev->config);
847
848 val = lduw_p(vdev->config + addr);
849 return val;
850 }
851
852 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
853 {
854 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
855 uint32_t val;
856
857 if (addr + sizeof(val) > vdev->config_len) {
858 return (uint32_t)-1;
859 }
860
861 k->get_config(vdev, vdev->config);
862
863 val = ldl_p(vdev->config + addr);
864 return val;
865 }
866
867 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
868 {
869 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
870 uint8_t val = data;
871
872 if (addr + sizeof(val) > vdev->config_len) {
873 return;
874 }
875
876 stb_p(vdev->config + addr, val);
877
878 if (k->set_config) {
879 k->set_config(vdev, vdev->config);
880 }
881 }
882
883 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
884 {
885 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
886 uint16_t val = data;
887
888 if (addr + sizeof(val) > vdev->config_len) {
889 return;
890 }
891
892 stw_p(vdev->config + addr, val);
893
894 if (k->set_config) {
895 k->set_config(vdev, vdev->config);
896 }
897 }
898
899 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
900 {
901 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
902 uint32_t val = data;
903
904 if (addr + sizeof(val) > vdev->config_len) {
905 return;
906 }
907
908 stl_p(vdev->config + addr, val);
909
910 if (k->set_config) {
911 k->set_config(vdev, vdev->config);
912 }
913 }
914
915 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
916 {
917 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
918 uint8_t val;
919
920 if (addr + sizeof(val) > vdev->config_len) {
921 return (uint32_t)-1;
922 }
923
924 k->get_config(vdev, vdev->config);
925
926 val = ldub_p(vdev->config + addr);
927 return val;
928 }
929
930 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
931 {
932 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
933 uint16_t val;
934
935 if (addr + sizeof(val) > vdev->config_len) {
936 return (uint32_t)-1;
937 }
938
939 k->get_config(vdev, vdev->config);
940
941 val = lduw_le_p(vdev->config + addr);
942 return val;
943 }
944
945 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
946 {
947 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
948 uint32_t val;
949
950 if (addr + sizeof(val) > vdev->config_len) {
951 return (uint32_t)-1;
952 }
953
954 k->get_config(vdev, vdev->config);
955
956 val = ldl_le_p(vdev->config + addr);
957 return val;
958 }
959
960 void virtio_config_modern_writeb(VirtIODevice *vdev,
961 uint32_t addr, uint32_t data)
962 {
963 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
964 uint8_t val = data;
965
966 if (addr + sizeof(val) > vdev->config_len) {
967 return;
968 }
969
970 stb_p(vdev->config + addr, val);
971
972 if (k->set_config) {
973 k->set_config(vdev, vdev->config);
974 }
975 }
976
977 void virtio_config_modern_writew(VirtIODevice *vdev,
978 uint32_t addr, uint32_t data)
979 {
980 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
981 uint16_t val = data;
982
983 if (addr + sizeof(val) > vdev->config_len) {
984 return;
985 }
986
987 stw_le_p(vdev->config + addr, val);
988
989 if (k->set_config) {
990 k->set_config(vdev, vdev->config);
991 }
992 }
993
994 void virtio_config_modern_writel(VirtIODevice *vdev,
995 uint32_t addr, uint32_t data)
996 {
997 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
998 uint32_t val = data;
999
1000 if (addr + sizeof(val) > vdev->config_len) {
1001 return;
1002 }
1003
1004 stl_le_p(vdev->config + addr, val);
1005
1006 if (k->set_config) {
1007 k->set_config(vdev, vdev->config);
1008 }
1009 }
1010
1011 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
1012 {
1013 vdev->vq[n].vring.desc = addr;
1014 virtio_queue_update_rings(vdev, n);
1015 }
1016
1017 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
1018 {
1019 return vdev->vq[n].vring.desc;
1020 }
1021
1022 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
1023 hwaddr avail, hwaddr used)
1024 {
1025 vdev->vq[n].vring.desc = desc;
1026 vdev->vq[n].vring.avail = avail;
1027 vdev->vq[n].vring.used = used;
1028 }
1029
1030 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
1031 {
1032 /* Don't allow guest to flip queue between existent and
1033 * nonexistent states, or to set it to an invalid size.
1034 */
1035 if (!!num != !!vdev->vq[n].vring.num ||
1036 num > VIRTQUEUE_MAX_SIZE ||
1037 num < 0) {
1038 return;
1039 }
1040 vdev->vq[n].vring.num = num;
1041 }
1042
1043 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
1044 {
1045 return QLIST_FIRST(&vdev->vector_queues[vector]);
1046 }
1047
1048 VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
1049 {
1050 return QLIST_NEXT(vq, node);
1051 }
1052
1053 int virtio_queue_get_num(VirtIODevice *vdev, int n)
1054 {
1055 return vdev->vq[n].vring.num;
1056 }
1057
1058 int virtio_get_num_queues(VirtIODevice *vdev)
1059 {
1060 int i;
1061
1062 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1063 if (!virtio_queue_get_num(vdev, i)) {
1064 break;
1065 }
1066 }
1067
1068 return i;
1069 }
1070
1071 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
1072 {
1073 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1074 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1075
1076 /* virtio-1 compliant devices cannot change the alignment */
1077 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1078 error_report("tried to modify queue alignment for virtio-1 device");
1079 return;
1080 }
1081 /* Check that the transport told us it was going to do this
1082 * (so a buggy transport will immediately assert rather than
1083 * silently failing to migrate this state)
1084 */
1085 assert(k->has_variable_vring_alignment);
1086
1087 vdev->vq[n].vring.align = align;
1088 virtio_queue_update_rings(vdev, n);
1089 }
1090
1091 static void virtio_queue_notify_aio_vq(VirtQueue *vq)
1092 {
1093 if (vq->vring.desc && vq->handle_aio_output) {
1094 VirtIODevice *vdev = vq->vdev;
1095
1096 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1097 vq->handle_aio_output(vdev, vq);
1098 }
1099 }
1100
1101 static void virtio_queue_notify_vq(VirtQueue *vq)
1102 {
1103 if (vq->vring.desc && vq->handle_output) {
1104 VirtIODevice *vdev = vq->vdev;
1105
1106 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1107 vq->handle_output(vdev, vq);
1108 }
1109 }
1110
1111 void virtio_queue_notify(VirtIODevice *vdev, int n)
1112 {
1113 virtio_queue_notify_vq(&vdev->vq[n]);
1114 }
1115
1116 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
1117 {
1118 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
1119 VIRTIO_NO_VECTOR;
1120 }
1121
1122 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
1123 {
1124 VirtQueue *vq = &vdev->vq[n];
1125
1126 if (n < VIRTIO_QUEUE_MAX) {
1127 if (vdev->vector_queues &&
1128 vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
1129 QLIST_REMOVE(vq, node);
1130 }
1131 vdev->vq[n].vector = vector;
1132 if (vdev->vector_queues &&
1133 vector != VIRTIO_NO_VECTOR) {
1134 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
1135 }
1136 }
1137 }
1138
1139 static VirtQueue *virtio_add_queue_internal(VirtIODevice *vdev, int queue_size,
1140 VirtIOHandleOutput handle_output,
1141 bool use_aio)
1142 {
1143 int i;
1144
1145 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1146 if (vdev->vq[i].vring.num == 0)
1147 break;
1148 }
1149
1150 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
1151 abort();
1152
1153 vdev->vq[i].vring.num = queue_size;
1154 vdev->vq[i].vring.num_default = queue_size;
1155 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
1156 vdev->vq[i].handle_output = handle_output;
1157 vdev->vq[i].handle_aio_output = NULL;
1158 vdev->vq[i].use_aio = use_aio;
1159
1160 return &vdev->vq[i];
1161 }
1162
1163 /* Add a virt queue and mark AIO.
1164 * An AIO queue will use the AioContext based event interface instead of the
1165 * default IOHandler and EventNotifier interface.
1166 */
1167 VirtQueue *virtio_add_queue_aio(VirtIODevice *vdev, int queue_size,
1168 VirtIOHandleOutput handle_output)
1169 {
1170 return virtio_add_queue_internal(vdev, queue_size, handle_output, true);
1171 }
1172
1173 /* Add a normal virt queue (on the contrary to the AIO version above. */
1174 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
1175 VirtIOHandleOutput handle_output)
1176 {
1177 return virtio_add_queue_internal(vdev, queue_size, handle_output, false);
1178 }
1179
1180 void virtio_del_queue(VirtIODevice *vdev, int n)
1181 {
1182 if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
1183 abort();
1184 }
1185
1186 vdev->vq[n].vring.num = 0;
1187 vdev->vq[n].vring.num_default = 0;
1188 }
1189
1190 void virtio_irq(VirtQueue *vq)
1191 {
1192 trace_virtio_irq(vq);
1193 vq->vdev->isr |= 0x01;
1194 virtio_notify_vector(vq->vdev, vq->vector);
1195 }
1196
1197 bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
1198 {
1199 uint16_t old, new;
1200 bool v;
1201 /* We need to expose used array entries before checking used event. */
1202 smp_mb();
1203 /* Always notify when queue is empty (when feature acknowledge) */
1204 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1205 !vq->inuse && virtio_queue_empty(vq)) {
1206 return true;
1207 }
1208
1209 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1210 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1211 }
1212
1213 v = vq->signalled_used_valid;
1214 vq->signalled_used_valid = true;
1215 old = vq->signalled_used;
1216 new = vq->signalled_used = vq->used_idx;
1217 return !v || vring_need_event(vring_get_used_event(vq), new, old);
1218 }
1219
1220 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
1221 {
1222 if (!virtio_should_notify(vdev, vq)) {
1223 return;
1224 }
1225
1226 trace_virtio_notify(vdev, vq);
1227 vdev->isr |= 0x01;
1228 virtio_notify_vector(vdev, vq->vector);
1229 }
1230
1231 void virtio_notify_config(VirtIODevice *vdev)
1232 {
1233 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1234 return;
1235
1236 vdev->isr |= 0x03;
1237 vdev->generation++;
1238 virtio_notify_vector(vdev, vdev->config_vector);
1239 }
1240
1241 static bool virtio_device_endian_needed(void *opaque)
1242 {
1243 VirtIODevice *vdev = opaque;
1244
1245 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
1246 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1247 return vdev->device_endian != virtio_default_endian();
1248 }
1249 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1250 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
1251 }
1252
1253 static bool virtio_64bit_features_needed(void *opaque)
1254 {
1255 VirtIODevice *vdev = opaque;
1256
1257 return (vdev->host_features >> 32) != 0;
1258 }
1259
1260 static bool virtio_virtqueue_needed(void *opaque)
1261 {
1262 VirtIODevice *vdev = opaque;
1263
1264 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
1265 }
1266
1267 static bool virtio_ringsize_needed(void *opaque)
1268 {
1269 VirtIODevice *vdev = opaque;
1270 int i;
1271
1272 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1273 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
1274 return true;
1275 }
1276 }
1277 return false;
1278 }
1279
1280 static bool virtio_extra_state_needed(void *opaque)
1281 {
1282 VirtIODevice *vdev = opaque;
1283 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1284 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1285
1286 return k->has_extra_state &&
1287 k->has_extra_state(qbus->parent);
1288 }
1289
1290 static const VMStateDescription vmstate_virtqueue = {
1291 .name = "virtqueue_state",
1292 .version_id = 1,
1293 .minimum_version_id = 1,
1294 .fields = (VMStateField[]) {
1295 VMSTATE_UINT64(vring.avail, struct VirtQueue),
1296 VMSTATE_UINT64(vring.used, struct VirtQueue),
1297 VMSTATE_END_OF_LIST()
1298 }
1299 };
1300
1301 static const VMStateDescription vmstate_virtio_virtqueues = {
1302 .name = "virtio/virtqueues",
1303 .version_id = 1,
1304 .minimum_version_id = 1,
1305 .needed = &virtio_virtqueue_needed,
1306 .fields = (VMStateField[]) {
1307 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1308 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
1309 VMSTATE_END_OF_LIST()
1310 }
1311 };
1312
1313 static const VMStateDescription vmstate_ringsize = {
1314 .name = "ringsize_state",
1315 .version_id = 1,
1316 .minimum_version_id = 1,
1317 .fields = (VMStateField[]) {
1318 VMSTATE_UINT32(vring.num_default, struct VirtQueue),
1319 VMSTATE_END_OF_LIST()
1320 }
1321 };
1322
1323 static const VMStateDescription vmstate_virtio_ringsize = {
1324 .name = "virtio/ringsize",
1325 .version_id = 1,
1326 .minimum_version_id = 1,
1327 .needed = &virtio_ringsize_needed,
1328 .fields = (VMStateField[]) {
1329 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1330 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
1331 VMSTATE_END_OF_LIST()
1332 }
1333 };
1334
1335 static int get_extra_state(QEMUFile *f, void *pv, size_t size)
1336 {
1337 VirtIODevice *vdev = pv;
1338 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1339 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1340
1341 if (!k->load_extra_state) {
1342 return -1;
1343 } else {
1344 return k->load_extra_state(qbus->parent, f);
1345 }
1346 }
1347
1348 static void put_extra_state(QEMUFile *f, void *pv, size_t size)
1349 {
1350 VirtIODevice *vdev = pv;
1351 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1352 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1353
1354 k->save_extra_state(qbus->parent, f);
1355 }
1356
1357 static const VMStateInfo vmstate_info_extra_state = {
1358 .name = "virtqueue_extra_state",
1359 .get = get_extra_state,
1360 .put = put_extra_state,
1361 };
1362
1363 static const VMStateDescription vmstate_virtio_extra_state = {
1364 .name = "virtio/extra_state",
1365 .version_id = 1,
1366 .minimum_version_id = 1,
1367 .needed = &virtio_extra_state_needed,
1368 .fields = (VMStateField[]) {
1369 {
1370 .name = "extra_state",
1371 .version_id = 0,
1372 .field_exists = NULL,
1373 .size = 0,
1374 .info = &vmstate_info_extra_state,
1375 .flags = VMS_SINGLE,
1376 .offset = 0,
1377 },
1378 VMSTATE_END_OF_LIST()
1379 }
1380 };
1381
1382 static const VMStateDescription vmstate_virtio_device_endian = {
1383 .name = "virtio/device_endian",
1384 .version_id = 1,
1385 .minimum_version_id = 1,
1386 .needed = &virtio_device_endian_needed,
1387 .fields = (VMStateField[]) {
1388 VMSTATE_UINT8(device_endian, VirtIODevice),
1389 VMSTATE_END_OF_LIST()
1390 }
1391 };
1392
1393 static const VMStateDescription vmstate_virtio_64bit_features = {
1394 .name = "virtio/64bit_features",
1395 .version_id = 1,
1396 .minimum_version_id = 1,
1397 .needed = &virtio_64bit_features_needed,
1398 .fields = (VMStateField[]) {
1399 VMSTATE_UINT64(guest_features, VirtIODevice),
1400 VMSTATE_END_OF_LIST()
1401 }
1402 };
1403
1404 static const VMStateDescription vmstate_virtio = {
1405 .name = "virtio",
1406 .version_id = 1,
1407 .minimum_version_id = 1,
1408 .minimum_version_id_old = 1,
1409 .fields = (VMStateField[]) {
1410 VMSTATE_END_OF_LIST()
1411 },
1412 .subsections = (const VMStateDescription*[]) {
1413 &vmstate_virtio_device_endian,
1414 &vmstate_virtio_64bit_features,
1415 &vmstate_virtio_virtqueues,
1416 &vmstate_virtio_ringsize,
1417 &vmstate_virtio_extra_state,
1418 NULL
1419 }
1420 };
1421
1422 void virtio_save(VirtIODevice *vdev, QEMUFile *f)
1423 {
1424 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1425 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1426 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1427 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
1428 int i;
1429
1430 if (k->save_config) {
1431 k->save_config(qbus->parent, f);
1432 }
1433
1434 qemu_put_8s(f, &vdev->status);
1435 qemu_put_8s(f, &vdev->isr);
1436 qemu_put_be16s(f, &vdev->queue_sel);
1437 qemu_put_be32s(f, &guest_features_lo);
1438 qemu_put_be32(f, vdev->config_len);
1439 qemu_put_buffer(f, vdev->config, vdev->config_len);
1440
1441 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1442 if (vdev->vq[i].vring.num == 0)
1443 break;
1444 }
1445
1446 qemu_put_be32(f, i);
1447
1448 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1449 if (vdev->vq[i].vring.num == 0)
1450 break;
1451
1452 qemu_put_be32(f, vdev->vq[i].vring.num);
1453 if (k->has_variable_vring_alignment) {
1454 qemu_put_be32(f, vdev->vq[i].vring.align);
1455 }
1456 /* XXX virtio-1 devices */
1457 qemu_put_be64(f, vdev->vq[i].vring.desc);
1458 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
1459 if (k->save_queue) {
1460 k->save_queue(qbus->parent, i, f);
1461 }
1462 }
1463
1464 if (vdc->save != NULL) {
1465 vdc->save(vdev, f);
1466 }
1467
1468 /* Subsections */
1469 vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
1470 }
1471
1472 /* A wrapper for use as a VMState .put function */
1473 void virtio_vmstate_save(QEMUFile *f, void *opaque, size_t size)
1474 {
1475 virtio_save(VIRTIO_DEVICE(opaque), f);
1476 }
1477
1478 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
1479 {
1480 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1481 bool bad = (val & ~(vdev->host_features)) != 0;
1482
1483 val &= vdev->host_features;
1484 if (k->set_features) {
1485 k->set_features(vdev, val);
1486 }
1487 vdev->guest_features = val;
1488 return bad ? -1 : 0;
1489 }
1490
1491 int virtio_set_features(VirtIODevice *vdev, uint64_t val)
1492 {
1493 /*
1494 * The driver must not attempt to set features after feature negotiation
1495 * has finished.
1496 */
1497 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
1498 return -EINVAL;
1499 }
1500 return virtio_set_features_nocheck(vdev, val);
1501 }
1502
1503 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
1504 {
1505 int i, ret;
1506 int32_t config_len;
1507 uint32_t num;
1508 uint32_t features;
1509 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1510 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1511 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1512
1513 /*
1514 * We poison the endianness to ensure it does not get used before
1515 * subsections have been loaded.
1516 */
1517 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
1518
1519 if (k->load_config) {
1520 ret = k->load_config(qbus->parent, f);
1521 if (ret)
1522 return ret;
1523 }
1524
1525 qemu_get_8s(f, &vdev->status);
1526 qemu_get_8s(f, &vdev->isr);
1527 qemu_get_be16s(f, &vdev->queue_sel);
1528 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
1529 return -1;
1530 }
1531 qemu_get_be32s(f, &features);
1532
1533 /*
1534 * Temporarily set guest_features low bits - needed by
1535 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
1536 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
1537 *
1538 * Note: devices should always test host features in future - don't create
1539 * new dependencies like this.
1540 */
1541 vdev->guest_features = features;
1542
1543 config_len = qemu_get_be32(f);
1544
1545 /*
1546 * There are cases where the incoming config can be bigger or smaller
1547 * than what we have; so load what we have space for, and skip
1548 * any excess that's in the stream.
1549 */
1550 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
1551
1552 while (config_len > vdev->config_len) {
1553 qemu_get_byte(f);
1554 config_len--;
1555 }
1556
1557 num = qemu_get_be32(f);
1558
1559 if (num > VIRTIO_QUEUE_MAX) {
1560 error_report("Invalid number of virtqueues: 0x%x", num);
1561 return -1;
1562 }
1563
1564 for (i = 0; i < num; i++) {
1565 vdev->vq[i].vring.num = qemu_get_be32(f);
1566 if (k->has_variable_vring_alignment) {
1567 vdev->vq[i].vring.align = qemu_get_be32(f);
1568 }
1569 vdev->vq[i].vring.desc = qemu_get_be64(f);
1570 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
1571 vdev->vq[i].signalled_used_valid = false;
1572 vdev->vq[i].notification = true;
1573
1574 if (vdev->vq[i].vring.desc) {
1575 /* XXX virtio-1 devices */
1576 virtio_queue_update_rings(vdev, i);
1577 } else if (vdev->vq[i].last_avail_idx) {
1578 error_report("VQ %d address 0x0 "
1579 "inconsistent with Host index 0x%x",
1580 i, vdev->vq[i].last_avail_idx);
1581 return -1;
1582 }
1583 if (k->load_queue) {
1584 ret = k->load_queue(qbus->parent, i, f);
1585 if (ret)
1586 return ret;
1587 }
1588 }
1589
1590 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1591
1592 if (vdc->load != NULL) {
1593 ret = vdc->load(vdev, f, version_id);
1594 if (ret) {
1595 return ret;
1596 }
1597 }
1598
1599 /* Subsections */
1600 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
1601 if (ret) {
1602 return ret;
1603 }
1604
1605 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
1606 vdev->device_endian = virtio_default_endian();
1607 }
1608
1609 if (virtio_64bit_features_needed(vdev)) {
1610 /*
1611 * Subsection load filled vdev->guest_features. Run them
1612 * through virtio_set_features to sanity-check them against
1613 * host_features.
1614 */
1615 uint64_t features64 = vdev->guest_features;
1616 if (virtio_set_features_nocheck(vdev, features64) < 0) {
1617 error_report("Features 0x%" PRIx64 " unsupported. "
1618 "Allowed features: 0x%" PRIx64,
1619 features64, vdev->host_features);
1620 return -1;
1621 }
1622 } else {
1623 if (virtio_set_features_nocheck(vdev, features) < 0) {
1624 error_report("Features 0x%x unsupported. "
1625 "Allowed features: 0x%" PRIx64,
1626 features, vdev->host_features);
1627 return -1;
1628 }
1629 }
1630
1631 for (i = 0; i < num; i++) {
1632 if (vdev->vq[i].vring.desc) {
1633 uint16_t nheads;
1634 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
1635 /* Check it isn't doing strange things with descriptor numbers. */
1636 if (nheads > vdev->vq[i].vring.num) {
1637 error_report("VQ %d size 0x%x Guest index 0x%x "
1638 "inconsistent with Host index 0x%x: delta 0x%x",
1639 i, vdev->vq[i].vring.num,
1640 vring_avail_idx(&vdev->vq[i]),
1641 vdev->vq[i].last_avail_idx, nheads);
1642 return -1;
1643 }
1644 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
1645 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
1646 }
1647 }
1648
1649 return 0;
1650 }
1651
1652 void virtio_cleanup(VirtIODevice *vdev)
1653 {
1654 qemu_del_vm_change_state_handler(vdev->vmstate);
1655 g_free(vdev->config);
1656 g_free(vdev->vq);
1657 g_free(vdev->vector_queues);
1658 }
1659
1660 static void virtio_vmstate_change(void *opaque, int running, RunState state)
1661 {
1662 VirtIODevice *vdev = opaque;
1663 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1664 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1665 bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
1666 vdev->vm_running = running;
1667
1668 if (backend_run) {
1669 virtio_set_status(vdev, vdev->status);
1670 }
1671
1672 if (k->vmstate_change) {
1673 k->vmstate_change(qbus->parent, backend_run);
1674 }
1675
1676 if (!backend_run) {
1677 virtio_set_status(vdev, vdev->status);
1678 }
1679 }
1680
1681 void virtio_instance_init_common(Object *proxy_obj, void *data,
1682 size_t vdev_size, const char *vdev_name)
1683 {
1684 DeviceState *vdev = data;
1685
1686 object_initialize(vdev, vdev_size, vdev_name);
1687 object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL);
1688 object_unref(OBJECT(vdev));
1689 qdev_alias_all_properties(vdev, proxy_obj);
1690 }
1691
1692 void virtio_init(VirtIODevice *vdev, const char *name,
1693 uint16_t device_id, size_t config_size)
1694 {
1695 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1696 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1697 int i;
1698 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
1699
1700 if (nvectors) {
1701 vdev->vector_queues =
1702 g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
1703 }
1704
1705 vdev->device_id = device_id;
1706 vdev->status = 0;
1707 vdev->isr = 0;
1708 vdev->queue_sel = 0;
1709 vdev->config_vector = VIRTIO_NO_VECTOR;
1710 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
1711 vdev->vm_running = runstate_is_running();
1712 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1713 vdev->vq[i].vector = VIRTIO_NO_VECTOR;
1714 vdev->vq[i].vdev = vdev;
1715 vdev->vq[i].queue_index = i;
1716 }
1717
1718 vdev->name = name;
1719 vdev->config_len = config_size;
1720 if (vdev->config_len) {
1721 vdev->config = g_malloc0(config_size);
1722 } else {
1723 vdev->config = NULL;
1724 }
1725 vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
1726 vdev);
1727 vdev->device_endian = virtio_default_endian();
1728 vdev->use_guest_notifier_mask = true;
1729 }
1730
1731 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
1732 {
1733 return vdev->vq[n].vring.desc;
1734 }
1735
1736 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
1737 {
1738 return vdev->vq[n].vring.avail;
1739 }
1740
1741 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
1742 {
1743 return vdev->vq[n].vring.used;
1744 }
1745
1746 hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
1747 {
1748 return vdev->vq[n].vring.desc;
1749 }
1750
1751 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
1752 {
1753 return sizeof(VRingDesc) * vdev->vq[n].vring.num;
1754 }
1755
1756 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
1757 {
1758 return offsetof(VRingAvail, ring) +
1759 sizeof(uint16_t) * vdev->vq[n].vring.num;
1760 }
1761
1762 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
1763 {
1764 return offsetof(VRingUsed, ring) +
1765 sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
1766 }
1767
1768 hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
1769 {
1770 return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
1771 virtio_queue_get_used_size(vdev, n);
1772 }
1773
1774 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
1775 {
1776 return vdev->vq[n].last_avail_idx;
1777 }
1778
1779 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
1780 {
1781 vdev->vq[n].last_avail_idx = idx;
1782 vdev->vq[n].shadow_avail_idx = idx;
1783 }
1784
1785 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
1786 {
1787 vdev->vq[n].signalled_used_valid = false;
1788 }
1789
1790 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
1791 {
1792 return vdev->vq + n;
1793 }
1794
1795 uint16_t virtio_get_queue_index(VirtQueue *vq)
1796 {
1797 return vq->queue_index;
1798 }
1799
1800 static void virtio_queue_guest_notifier_read(EventNotifier *n)
1801 {
1802 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
1803 if (event_notifier_test_and_clear(n)) {
1804 virtio_irq(vq);
1805 }
1806 }
1807
1808 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
1809 bool with_irqfd)
1810 {
1811 if (assign && !with_irqfd) {
1812 event_notifier_set_handler(&vq->guest_notifier, false,
1813 virtio_queue_guest_notifier_read);
1814 } else {
1815 event_notifier_set_handler(&vq->guest_notifier, false, NULL);
1816 }
1817 if (!assign) {
1818 /* Test and clear notifier before closing it,
1819 * in case poll callback didn't have time to run. */
1820 virtio_queue_guest_notifier_read(&vq->guest_notifier);
1821 }
1822 }
1823
1824 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
1825 {
1826 return &vq->guest_notifier;
1827 }
1828
1829 static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
1830 {
1831 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1832 if (event_notifier_test_and_clear(n)) {
1833 virtio_queue_notify_aio_vq(vq);
1834 }
1835 }
1836
1837 void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
1838 VirtIOHandleOutput handle_output)
1839 {
1840 if (handle_output) {
1841 vq->handle_aio_output = handle_output;
1842 aio_set_event_notifier(ctx, &vq->host_notifier, true,
1843 virtio_queue_host_notifier_aio_read);
1844 } else {
1845 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL);
1846 /* Test and clear notifier before after disabling event,
1847 * in case poll callback didn't have time to run. */
1848 virtio_queue_host_notifier_aio_read(&vq->host_notifier);
1849 vq->handle_aio_output = NULL;
1850 }
1851 }
1852
1853 static void virtio_queue_host_notifier_read(EventNotifier *n)
1854 {
1855 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1856 if (event_notifier_test_and_clear(n)) {
1857 virtio_queue_notify_vq(vq);
1858 }
1859 }
1860
1861 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
1862 bool set_handler)
1863 {
1864 AioContext *ctx = qemu_get_aio_context();
1865 if (assign && set_handler) {
1866 if (vq->use_aio) {
1867 aio_set_event_notifier(ctx, &vq->host_notifier, true,
1868 virtio_queue_host_notifier_read);
1869 } else {
1870 event_notifier_set_handler(&vq->host_notifier, true,
1871 virtio_queue_host_notifier_read);
1872 }
1873 } else {
1874 if (vq->use_aio) {
1875 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL);
1876 } else {
1877 event_notifier_set_handler(&vq->host_notifier, true, NULL);
1878 }
1879 }
1880 if (!assign) {
1881 /* Test and clear notifier before after disabling event,
1882 * in case poll callback didn't have time to run. */
1883 virtio_queue_host_notifier_read(&vq->host_notifier);
1884 }
1885 }
1886
1887 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
1888 {
1889 return &vq->host_notifier;
1890 }
1891
1892 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
1893 {
1894 g_free(vdev->bus_name);
1895 vdev->bus_name = g_strdup(bus_name);
1896 }
1897
1898 static void virtio_device_realize(DeviceState *dev, Error **errp)
1899 {
1900 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1901 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1902 Error *err = NULL;
1903
1904 if (vdc->realize != NULL) {
1905 vdc->realize(dev, &err);
1906 if (err != NULL) {
1907 error_propagate(errp, err);
1908 return;
1909 }
1910 }
1911
1912 virtio_bus_device_plugged(vdev, &err);
1913 if (err != NULL) {
1914 error_propagate(errp, err);
1915 return;
1916 }
1917 }
1918
1919 static void virtio_device_unrealize(DeviceState *dev, Error **errp)
1920 {
1921 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1922 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1923 Error *err = NULL;
1924
1925 virtio_bus_device_unplugged(vdev);
1926
1927 if (vdc->unrealize != NULL) {
1928 vdc->unrealize(dev, &err);
1929 if (err != NULL) {
1930 error_propagate(errp, err);
1931 return;
1932 }
1933 }
1934
1935 g_free(vdev->bus_name);
1936 vdev->bus_name = NULL;
1937 }
1938
1939 static Property virtio_properties[] = {
1940 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
1941 DEFINE_PROP_END_OF_LIST(),
1942 };
1943
1944 static void virtio_device_class_init(ObjectClass *klass, void *data)
1945 {
1946 /* Set the default value here. */
1947 DeviceClass *dc = DEVICE_CLASS(klass);
1948
1949 dc->realize = virtio_device_realize;
1950 dc->unrealize = virtio_device_unrealize;
1951 dc->bus_type = TYPE_VIRTIO_BUS;
1952 dc->props = virtio_properties;
1953 }
1954
1955 static const TypeInfo virtio_device_info = {
1956 .name = TYPE_VIRTIO_DEVICE,
1957 .parent = TYPE_DEVICE,
1958 .instance_size = sizeof(VirtIODevice),
1959 .class_init = virtio_device_class_init,
1960 .abstract = true,
1961 .class_size = sizeof(VirtioDeviceClass),
1962 };
1963
1964 static void virtio_register_types(void)
1965 {
1966 type_register_static(&virtio_device_info);
1967 }
1968
1969 type_init(virtio_register_types)