exec: Make ldl_*_phys input an AddressSpace
[qemu.git] / hw / virtio / virtio.c
1 /*
2 * Virtio Support
3 *
4 * Copyright IBM, Corp. 2007
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 */
13
14 #include <inttypes.h>
15
16 #include "trace.h"
17 #include "exec/address-spaces.h"
18 #include "qemu/error-report.h"
19 #include "hw/virtio/virtio.h"
20 #include "qemu/atomic.h"
21 #include "hw/virtio/virtio-bus.h"
22
23 /*
24 * The alignment to use between consumer and producer parts of vring.
25 * x86 pagesize again. This is the default, used by transports like PCI
26 * which don't provide a means for the guest to tell the host the alignment.
27 */
28 #define VIRTIO_PCI_VRING_ALIGN 4096
29
30 typedef struct VRingDesc
31 {
32 uint64_t addr;
33 uint32_t len;
34 uint16_t flags;
35 uint16_t next;
36 } VRingDesc;
37
38 typedef struct VRingAvail
39 {
40 uint16_t flags;
41 uint16_t idx;
42 uint16_t ring[0];
43 } VRingAvail;
44
45 typedef struct VRingUsedElem
46 {
47 uint32_t id;
48 uint32_t len;
49 } VRingUsedElem;
50
51 typedef struct VRingUsed
52 {
53 uint16_t flags;
54 uint16_t idx;
55 VRingUsedElem ring[0];
56 } VRingUsed;
57
58 typedef struct VRing
59 {
60 unsigned int num;
61 unsigned int align;
62 hwaddr desc;
63 hwaddr avail;
64 hwaddr used;
65 } VRing;
66
67 struct VirtQueue
68 {
69 VRing vring;
70 hwaddr pa;
71 uint16_t last_avail_idx;
72 /* Last used index value we have signalled on */
73 uint16_t signalled_used;
74
75 /* Last used index value we have signalled on */
76 bool signalled_used_valid;
77
78 /* Notification enabled? */
79 bool notification;
80
81 uint16_t queue_index;
82
83 int inuse;
84
85 uint16_t vector;
86 void (*handle_output)(VirtIODevice *vdev, VirtQueue *vq);
87 VirtIODevice *vdev;
88 EventNotifier guest_notifier;
89 EventNotifier host_notifier;
90 };
91
92 /* virt queue functions */
93 static void virtqueue_init(VirtQueue *vq)
94 {
95 hwaddr pa = vq->pa;
96
97 vq->vring.desc = pa;
98 vq->vring.avail = pa + vq->vring.num * sizeof(VRingDesc);
99 vq->vring.used = vring_align(vq->vring.avail +
100 offsetof(VRingAvail, ring[vq->vring.num]),
101 vq->vring.align);
102 }
103
104 static inline uint64_t vring_desc_addr(hwaddr desc_pa, int i)
105 {
106 hwaddr pa;
107 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, addr);
108 return ldq_phys(pa);
109 }
110
111 static inline uint32_t vring_desc_len(hwaddr desc_pa, int i)
112 {
113 hwaddr pa;
114 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, len);
115 return ldl_phys(&address_space_memory, pa);
116 }
117
118 static inline uint16_t vring_desc_flags(hwaddr desc_pa, int i)
119 {
120 hwaddr pa;
121 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, flags);
122 return lduw_phys(pa);
123 }
124
125 static inline uint16_t vring_desc_next(hwaddr desc_pa, int i)
126 {
127 hwaddr pa;
128 pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, next);
129 return lduw_phys(pa);
130 }
131
132 static inline uint16_t vring_avail_flags(VirtQueue *vq)
133 {
134 hwaddr pa;
135 pa = vq->vring.avail + offsetof(VRingAvail, flags);
136 return lduw_phys(pa);
137 }
138
139 static inline uint16_t vring_avail_idx(VirtQueue *vq)
140 {
141 hwaddr pa;
142 pa = vq->vring.avail + offsetof(VRingAvail, idx);
143 return lduw_phys(pa);
144 }
145
146 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
147 {
148 hwaddr pa;
149 pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
150 return lduw_phys(pa);
151 }
152
153 static inline uint16_t vring_used_event(VirtQueue *vq)
154 {
155 return vring_avail_ring(vq, vq->vring.num);
156 }
157
158 static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val)
159 {
160 hwaddr pa;
161 pa = vq->vring.used + offsetof(VRingUsed, ring[i].id);
162 stl_phys(pa, val);
163 }
164
165 static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val)
166 {
167 hwaddr pa;
168 pa = vq->vring.used + offsetof(VRingUsed, ring[i].len);
169 stl_phys(pa, val);
170 }
171
172 static uint16_t vring_used_idx(VirtQueue *vq)
173 {
174 hwaddr pa;
175 pa = vq->vring.used + offsetof(VRingUsed, idx);
176 return lduw_phys(pa);
177 }
178
179 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
180 {
181 hwaddr pa;
182 pa = vq->vring.used + offsetof(VRingUsed, idx);
183 stw_phys(pa, val);
184 }
185
186 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
187 {
188 hwaddr pa;
189 pa = vq->vring.used + offsetof(VRingUsed, flags);
190 stw_phys(pa, lduw_phys(pa) | mask);
191 }
192
193 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
194 {
195 hwaddr pa;
196 pa = vq->vring.used + offsetof(VRingUsed, flags);
197 stw_phys(pa, lduw_phys(pa) & ~mask);
198 }
199
200 static inline void vring_avail_event(VirtQueue *vq, uint16_t val)
201 {
202 hwaddr pa;
203 if (!vq->notification) {
204 return;
205 }
206 pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]);
207 stw_phys(pa, val);
208 }
209
210 void virtio_queue_set_notification(VirtQueue *vq, int enable)
211 {
212 vq->notification = enable;
213 if (vq->vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
214 vring_avail_event(vq, vring_avail_idx(vq));
215 } else if (enable) {
216 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
217 } else {
218 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
219 }
220 if (enable) {
221 /* Expose avail event/used flags before caller checks the avail idx. */
222 smp_mb();
223 }
224 }
225
226 int virtio_queue_ready(VirtQueue *vq)
227 {
228 return vq->vring.avail != 0;
229 }
230
231 int virtio_queue_empty(VirtQueue *vq)
232 {
233 return vring_avail_idx(vq) == vq->last_avail_idx;
234 }
235
236 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
237 unsigned int len, unsigned int idx)
238 {
239 unsigned int offset;
240 int i;
241
242 trace_virtqueue_fill(vq, elem, len, idx);
243
244 offset = 0;
245 for (i = 0; i < elem->in_num; i++) {
246 size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
247
248 cpu_physical_memory_unmap(elem->in_sg[i].iov_base,
249 elem->in_sg[i].iov_len,
250 1, size);
251
252 offset += size;
253 }
254
255 for (i = 0; i < elem->out_num; i++)
256 cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
257 elem->out_sg[i].iov_len,
258 0, elem->out_sg[i].iov_len);
259
260 idx = (idx + vring_used_idx(vq)) % vq->vring.num;
261
262 /* Get a pointer to the next entry in the used ring. */
263 vring_used_ring_id(vq, idx, elem->index);
264 vring_used_ring_len(vq, idx, len);
265 }
266
267 void virtqueue_flush(VirtQueue *vq, unsigned int count)
268 {
269 uint16_t old, new;
270 /* Make sure buffer is written before we update index. */
271 smp_wmb();
272 trace_virtqueue_flush(vq, count);
273 old = vring_used_idx(vq);
274 new = old + count;
275 vring_used_idx_set(vq, new);
276 vq->inuse -= count;
277 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
278 vq->signalled_used_valid = false;
279 }
280
281 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
282 unsigned int len)
283 {
284 virtqueue_fill(vq, elem, len, 0);
285 virtqueue_flush(vq, 1);
286 }
287
288 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
289 {
290 uint16_t num_heads = vring_avail_idx(vq) - idx;
291
292 /* Check it isn't doing very strange things with descriptor numbers. */
293 if (num_heads > vq->vring.num) {
294 error_report("Guest moved used index from %u to %u",
295 idx, vring_avail_idx(vq));
296 exit(1);
297 }
298 /* On success, callers read a descriptor at vq->last_avail_idx.
299 * Make sure descriptor read does not bypass avail index read. */
300 if (num_heads) {
301 smp_rmb();
302 }
303
304 return num_heads;
305 }
306
307 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
308 {
309 unsigned int head;
310
311 /* Grab the next descriptor number they're advertising, and increment
312 * the index we've seen. */
313 head = vring_avail_ring(vq, idx % vq->vring.num);
314
315 /* If their number is silly, that's a fatal mistake. */
316 if (head >= vq->vring.num) {
317 error_report("Guest says index %u is available", head);
318 exit(1);
319 }
320
321 return head;
322 }
323
324 static unsigned virtqueue_next_desc(hwaddr desc_pa,
325 unsigned int i, unsigned int max)
326 {
327 unsigned int next;
328
329 /* If this descriptor says it doesn't chain, we're done. */
330 if (!(vring_desc_flags(desc_pa, i) & VRING_DESC_F_NEXT))
331 return max;
332
333 /* Check they're not leading us off end of descriptors. */
334 next = vring_desc_next(desc_pa, i);
335 /* Make sure compiler knows to grab that: we don't want it changing! */
336 smp_wmb();
337
338 if (next >= max) {
339 error_report("Desc next is %u", next);
340 exit(1);
341 }
342
343 return next;
344 }
345
346 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
347 unsigned int *out_bytes,
348 unsigned max_in_bytes, unsigned max_out_bytes)
349 {
350 unsigned int idx;
351 unsigned int total_bufs, in_total, out_total;
352
353 idx = vq->last_avail_idx;
354
355 total_bufs = in_total = out_total = 0;
356 while (virtqueue_num_heads(vq, idx)) {
357 unsigned int max, num_bufs, indirect = 0;
358 hwaddr desc_pa;
359 int i;
360
361 max = vq->vring.num;
362 num_bufs = total_bufs;
363 i = virtqueue_get_head(vq, idx++);
364 desc_pa = vq->vring.desc;
365
366 if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_INDIRECT) {
367 if (vring_desc_len(desc_pa, i) % sizeof(VRingDesc)) {
368 error_report("Invalid size for indirect buffer table");
369 exit(1);
370 }
371
372 /* If we've got too many, that implies a descriptor loop. */
373 if (num_bufs >= max) {
374 error_report("Looped descriptor");
375 exit(1);
376 }
377
378 /* loop over the indirect descriptor table */
379 indirect = 1;
380 max = vring_desc_len(desc_pa, i) / sizeof(VRingDesc);
381 desc_pa = vring_desc_addr(desc_pa, i);
382 num_bufs = i = 0;
383 }
384
385 do {
386 /* If we've got too many, that implies a descriptor loop. */
387 if (++num_bufs > max) {
388 error_report("Looped descriptor");
389 exit(1);
390 }
391
392 if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_WRITE) {
393 in_total += vring_desc_len(desc_pa, i);
394 } else {
395 out_total += vring_desc_len(desc_pa, i);
396 }
397 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
398 goto done;
399 }
400 } while ((i = virtqueue_next_desc(desc_pa, i, max)) != max);
401
402 if (!indirect)
403 total_bufs = num_bufs;
404 else
405 total_bufs++;
406 }
407 done:
408 if (in_bytes) {
409 *in_bytes = in_total;
410 }
411 if (out_bytes) {
412 *out_bytes = out_total;
413 }
414 }
415
416 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
417 unsigned int out_bytes)
418 {
419 unsigned int in_total, out_total;
420
421 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
422 return in_bytes <= in_total && out_bytes <= out_total;
423 }
424
425 void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
426 size_t num_sg, int is_write)
427 {
428 unsigned int i;
429 hwaddr len;
430
431 for (i = 0; i < num_sg; i++) {
432 len = sg[i].iov_len;
433 sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
434 if (sg[i].iov_base == NULL || len != sg[i].iov_len) {
435 error_report("virtio: trying to map MMIO memory");
436 exit(1);
437 }
438 }
439 }
440
441 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
442 {
443 unsigned int i, head, max;
444 hwaddr desc_pa = vq->vring.desc;
445
446 if (!virtqueue_num_heads(vq, vq->last_avail_idx))
447 return 0;
448
449 /* When we start there are none of either input nor output. */
450 elem->out_num = elem->in_num = 0;
451
452 max = vq->vring.num;
453
454 i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
455 if (vq->vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
456 vring_avail_event(vq, vring_avail_idx(vq));
457 }
458
459 if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_INDIRECT) {
460 if (vring_desc_len(desc_pa, i) % sizeof(VRingDesc)) {
461 error_report("Invalid size for indirect buffer table");
462 exit(1);
463 }
464
465 /* loop over the indirect descriptor table */
466 max = vring_desc_len(desc_pa, i) / sizeof(VRingDesc);
467 desc_pa = vring_desc_addr(desc_pa, i);
468 i = 0;
469 }
470
471 /* Collect all the descriptors */
472 do {
473 struct iovec *sg;
474
475 if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_WRITE) {
476 if (elem->in_num >= ARRAY_SIZE(elem->in_sg)) {
477 error_report("Too many write descriptors in indirect table");
478 exit(1);
479 }
480 elem->in_addr[elem->in_num] = vring_desc_addr(desc_pa, i);
481 sg = &elem->in_sg[elem->in_num++];
482 } else {
483 if (elem->out_num >= ARRAY_SIZE(elem->out_sg)) {
484 error_report("Too many read descriptors in indirect table");
485 exit(1);
486 }
487 elem->out_addr[elem->out_num] = vring_desc_addr(desc_pa, i);
488 sg = &elem->out_sg[elem->out_num++];
489 }
490
491 sg->iov_len = vring_desc_len(desc_pa, i);
492
493 /* If we've got too many, that implies a descriptor loop. */
494 if ((elem->in_num + elem->out_num) > max) {
495 error_report("Looped descriptor");
496 exit(1);
497 }
498 } while ((i = virtqueue_next_desc(desc_pa, i, max)) != max);
499
500 /* Now map what we have collected */
501 virtqueue_map_sg(elem->in_sg, elem->in_addr, elem->in_num, 1);
502 virtqueue_map_sg(elem->out_sg, elem->out_addr, elem->out_num, 0);
503
504 elem->index = head;
505
506 vq->inuse++;
507
508 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
509 return elem->in_num + elem->out_num;
510 }
511
512 /* virtio device */
513 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
514 {
515 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
516 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
517
518 if (k->notify) {
519 k->notify(qbus->parent, vector);
520 }
521 }
522
523 void virtio_update_irq(VirtIODevice *vdev)
524 {
525 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
526 }
527
528 void virtio_set_status(VirtIODevice *vdev, uint8_t val)
529 {
530 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
531 trace_virtio_set_status(vdev, val);
532
533 if (k->set_status) {
534 k->set_status(vdev, val);
535 }
536 vdev->status = val;
537 }
538
539 void virtio_reset(void *opaque)
540 {
541 VirtIODevice *vdev = opaque;
542 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
543 int i;
544
545 virtio_set_status(vdev, 0);
546
547 if (k->reset) {
548 k->reset(vdev);
549 }
550
551 vdev->guest_features = 0;
552 vdev->queue_sel = 0;
553 vdev->status = 0;
554 vdev->isr = 0;
555 vdev->config_vector = VIRTIO_NO_VECTOR;
556 virtio_notify_vector(vdev, vdev->config_vector);
557
558 for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
559 vdev->vq[i].vring.desc = 0;
560 vdev->vq[i].vring.avail = 0;
561 vdev->vq[i].vring.used = 0;
562 vdev->vq[i].last_avail_idx = 0;
563 vdev->vq[i].pa = 0;
564 vdev->vq[i].vector = VIRTIO_NO_VECTOR;
565 vdev->vq[i].signalled_used = 0;
566 vdev->vq[i].signalled_used_valid = false;
567 vdev->vq[i].notification = true;
568 }
569 }
570
571 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
572 {
573 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
574 uint8_t val;
575
576 if (addr + sizeof(val) > vdev->config_len) {
577 return (uint32_t)-1;
578 }
579
580 k->get_config(vdev, vdev->config);
581
582 val = ldub_p(vdev->config + addr);
583 return val;
584 }
585
586 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
587 {
588 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
589 uint16_t val;
590
591 if (addr + sizeof(val) > vdev->config_len) {
592 return (uint32_t)-1;
593 }
594
595 k->get_config(vdev, vdev->config);
596
597 val = lduw_p(vdev->config + addr);
598 return val;
599 }
600
601 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
602 {
603 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
604 uint32_t val;
605
606 if (addr + sizeof(val) > vdev->config_len) {
607 return (uint32_t)-1;
608 }
609
610 k->get_config(vdev, vdev->config);
611
612 val = ldl_p(vdev->config + addr);
613 return val;
614 }
615
616 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
617 {
618 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
619 uint8_t val = data;
620
621 if (addr + sizeof(val) > vdev->config_len) {
622 return;
623 }
624
625 stb_p(vdev->config + addr, val);
626
627 if (k->set_config) {
628 k->set_config(vdev, vdev->config);
629 }
630 }
631
632 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
633 {
634 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
635 uint16_t val = data;
636
637 if (addr + sizeof(val) > vdev->config_len) {
638 return;
639 }
640
641 stw_p(vdev->config + addr, val);
642
643 if (k->set_config) {
644 k->set_config(vdev, vdev->config);
645 }
646 }
647
648 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
649 {
650 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
651 uint32_t val = data;
652
653 if (addr + sizeof(val) > vdev->config_len) {
654 return;
655 }
656
657 stl_p(vdev->config + addr, val);
658
659 if (k->set_config) {
660 k->set_config(vdev, vdev->config);
661 }
662 }
663
664 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
665 {
666 vdev->vq[n].pa = addr;
667 virtqueue_init(&vdev->vq[n]);
668 }
669
670 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
671 {
672 return vdev->vq[n].pa;
673 }
674
675 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
676 {
677 /* Don't allow guest to flip queue between existent and
678 * nonexistent states, or to set it to an invalid size.
679 */
680 if (!!num != !!vdev->vq[n].vring.num ||
681 num > VIRTQUEUE_MAX_SIZE ||
682 num < 0) {
683 return;
684 }
685 vdev->vq[n].vring.num = num;
686 virtqueue_init(&vdev->vq[n]);
687 }
688
689 int virtio_queue_get_num(VirtIODevice *vdev, int n)
690 {
691 return vdev->vq[n].vring.num;
692 }
693
694 int virtio_queue_get_id(VirtQueue *vq)
695 {
696 VirtIODevice *vdev = vq->vdev;
697 assert(vq >= &vdev->vq[0] && vq < &vdev->vq[VIRTIO_PCI_QUEUE_MAX]);
698 return vq - &vdev->vq[0];
699 }
700
701 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
702 {
703 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
704 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
705
706 /* Check that the transport told us it was going to do this
707 * (so a buggy transport will immediately assert rather than
708 * silently failing to migrate this state)
709 */
710 assert(k->has_variable_vring_alignment);
711
712 vdev->vq[n].vring.align = align;
713 virtqueue_init(&vdev->vq[n]);
714 }
715
716 void virtio_queue_notify_vq(VirtQueue *vq)
717 {
718 if (vq->vring.desc) {
719 VirtIODevice *vdev = vq->vdev;
720 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
721 vq->handle_output(vdev, vq);
722 }
723 }
724
725 void virtio_queue_notify(VirtIODevice *vdev, int n)
726 {
727 virtio_queue_notify_vq(&vdev->vq[n]);
728 }
729
730 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
731 {
732 return n < VIRTIO_PCI_QUEUE_MAX ? vdev->vq[n].vector :
733 VIRTIO_NO_VECTOR;
734 }
735
736 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
737 {
738 if (n < VIRTIO_PCI_QUEUE_MAX)
739 vdev->vq[n].vector = vector;
740 }
741
742 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
743 void (*handle_output)(VirtIODevice *, VirtQueue *))
744 {
745 int i;
746
747 for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
748 if (vdev->vq[i].vring.num == 0)
749 break;
750 }
751
752 if (i == VIRTIO_PCI_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
753 abort();
754
755 vdev->vq[i].vring.num = queue_size;
756 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
757 vdev->vq[i].handle_output = handle_output;
758
759 return &vdev->vq[i];
760 }
761
762 void virtio_del_queue(VirtIODevice *vdev, int n)
763 {
764 if (n < 0 || n >= VIRTIO_PCI_QUEUE_MAX) {
765 abort();
766 }
767
768 vdev->vq[n].vring.num = 0;
769 }
770
771 void virtio_irq(VirtQueue *vq)
772 {
773 trace_virtio_irq(vq);
774 vq->vdev->isr |= 0x01;
775 virtio_notify_vector(vq->vdev, vq->vector);
776 }
777
778 /* Assuming a given event_idx value from the other size, if
779 * we have just incremented index from old to new_idx,
780 * should we trigger an event? */
781 static inline int vring_need_event(uint16_t event, uint16_t new, uint16_t old)
782 {
783 /* Note: Xen has similar logic for notification hold-off
784 * in include/xen/interface/io/ring.h with req_event and req_prod
785 * corresponding to event_idx + 1 and new respectively.
786 * Note also that req_event and req_prod in Xen start at 1,
787 * event indexes in virtio start at 0. */
788 return (uint16_t)(new - event - 1) < (uint16_t)(new - old);
789 }
790
791 static bool vring_notify(VirtIODevice *vdev, VirtQueue *vq)
792 {
793 uint16_t old, new;
794 bool v;
795 /* We need to expose used array entries before checking used event. */
796 smp_mb();
797 /* Always notify when queue is empty (when feature acknowledge) */
798 if (((vdev->guest_features & (1 << VIRTIO_F_NOTIFY_ON_EMPTY)) &&
799 !vq->inuse && vring_avail_idx(vq) == vq->last_avail_idx)) {
800 return true;
801 }
802
803 if (!(vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX))) {
804 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
805 }
806
807 v = vq->signalled_used_valid;
808 vq->signalled_used_valid = true;
809 old = vq->signalled_used;
810 new = vq->signalled_used = vring_used_idx(vq);
811 return !v || vring_need_event(vring_used_event(vq), new, old);
812 }
813
814 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
815 {
816 if (!vring_notify(vdev, vq)) {
817 return;
818 }
819
820 trace_virtio_notify(vdev, vq);
821 vdev->isr |= 0x01;
822 virtio_notify_vector(vdev, vq->vector);
823 }
824
825 void virtio_notify_config(VirtIODevice *vdev)
826 {
827 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
828 return;
829
830 vdev->isr |= 0x03;
831 virtio_notify_vector(vdev, vdev->config_vector);
832 }
833
834 void virtio_save(VirtIODevice *vdev, QEMUFile *f)
835 {
836 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
837 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
838 int i;
839
840 if (k->save_config) {
841 k->save_config(qbus->parent, f);
842 }
843
844 qemu_put_8s(f, &vdev->status);
845 qemu_put_8s(f, &vdev->isr);
846 qemu_put_be16s(f, &vdev->queue_sel);
847 qemu_put_be32s(f, &vdev->guest_features);
848 qemu_put_be32(f, vdev->config_len);
849 qemu_put_buffer(f, vdev->config, vdev->config_len);
850
851 for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
852 if (vdev->vq[i].vring.num == 0)
853 break;
854 }
855
856 qemu_put_be32(f, i);
857
858 for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
859 if (vdev->vq[i].vring.num == 0)
860 break;
861
862 qemu_put_be32(f, vdev->vq[i].vring.num);
863 if (k->has_variable_vring_alignment) {
864 qemu_put_be32(f, vdev->vq[i].vring.align);
865 }
866 qemu_put_be64(f, vdev->vq[i].pa);
867 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
868 if (k->save_queue) {
869 k->save_queue(qbus->parent, i, f);
870 }
871 }
872 }
873
874 int virtio_set_features(VirtIODevice *vdev, uint32_t val)
875 {
876 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
877 VirtioBusClass *vbusk = VIRTIO_BUS_GET_CLASS(qbus);
878 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
879 uint32_t supported_features = vbusk->get_features(qbus->parent);
880 bool bad = (val & ~supported_features) != 0;
881
882 val &= supported_features;
883 if (k->set_features) {
884 k->set_features(vdev, val);
885 }
886 vdev->guest_features = val;
887 return bad ? -1 : 0;
888 }
889
890 int virtio_load(VirtIODevice *vdev, QEMUFile *f)
891 {
892 int num, i, ret;
893 uint32_t features;
894 uint32_t supported_features;
895 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
896 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
897
898 if (k->load_config) {
899 ret = k->load_config(qbus->parent, f);
900 if (ret)
901 return ret;
902 }
903
904 qemu_get_8s(f, &vdev->status);
905 qemu_get_8s(f, &vdev->isr);
906 qemu_get_be16s(f, &vdev->queue_sel);
907 qemu_get_be32s(f, &features);
908
909 if (virtio_set_features(vdev, features) < 0) {
910 supported_features = k->get_features(qbus->parent);
911 error_report("Features 0x%x unsupported. Allowed features: 0x%x",
912 features, supported_features);
913 return -1;
914 }
915 vdev->config_len = qemu_get_be32(f);
916 qemu_get_buffer(f, vdev->config, vdev->config_len);
917
918 num = qemu_get_be32(f);
919
920 for (i = 0; i < num; i++) {
921 vdev->vq[i].vring.num = qemu_get_be32(f);
922 if (k->has_variable_vring_alignment) {
923 vdev->vq[i].vring.align = qemu_get_be32(f);
924 }
925 vdev->vq[i].pa = qemu_get_be64(f);
926 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
927 vdev->vq[i].signalled_used_valid = false;
928 vdev->vq[i].notification = true;
929
930 if (vdev->vq[i].pa) {
931 uint16_t nheads;
932 virtqueue_init(&vdev->vq[i]);
933 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
934 /* Check it isn't doing very strange things with descriptor numbers. */
935 if (nheads > vdev->vq[i].vring.num) {
936 error_report("VQ %d size 0x%x Guest index 0x%x "
937 "inconsistent with Host index 0x%x: delta 0x%x",
938 i, vdev->vq[i].vring.num,
939 vring_avail_idx(&vdev->vq[i]),
940 vdev->vq[i].last_avail_idx, nheads);
941 return -1;
942 }
943 } else if (vdev->vq[i].last_avail_idx) {
944 error_report("VQ %d address 0x0 "
945 "inconsistent with Host index 0x%x",
946 i, vdev->vq[i].last_avail_idx);
947 return -1;
948 }
949 if (k->load_queue) {
950 ret = k->load_queue(qbus->parent, i, f);
951 if (ret)
952 return ret;
953 }
954 }
955
956 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
957 return 0;
958 }
959
960 void virtio_cleanup(VirtIODevice *vdev)
961 {
962 qemu_del_vm_change_state_handler(vdev->vmstate);
963 g_free(vdev->config);
964 g_free(vdev->vq);
965 }
966
967 static void virtio_vmstate_change(void *opaque, int running, RunState state)
968 {
969 VirtIODevice *vdev = opaque;
970 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
971 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
972 bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
973 vdev->vm_running = running;
974
975 if (backend_run) {
976 virtio_set_status(vdev, vdev->status);
977 }
978
979 if (k->vmstate_change) {
980 k->vmstate_change(qbus->parent, backend_run);
981 }
982
983 if (!backend_run) {
984 virtio_set_status(vdev, vdev->status);
985 }
986 }
987
988 void virtio_init(VirtIODevice *vdev, const char *name,
989 uint16_t device_id, size_t config_size)
990 {
991 int i;
992 vdev->device_id = device_id;
993 vdev->status = 0;
994 vdev->isr = 0;
995 vdev->queue_sel = 0;
996 vdev->config_vector = VIRTIO_NO_VECTOR;
997 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_PCI_QUEUE_MAX);
998 vdev->vm_running = runstate_is_running();
999 for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
1000 vdev->vq[i].vector = VIRTIO_NO_VECTOR;
1001 vdev->vq[i].vdev = vdev;
1002 vdev->vq[i].queue_index = i;
1003 }
1004
1005 vdev->name = name;
1006 vdev->config_len = config_size;
1007 if (vdev->config_len) {
1008 vdev->config = g_malloc0(config_size);
1009 } else {
1010 vdev->config = NULL;
1011 }
1012 vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
1013 vdev);
1014 }
1015
1016 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
1017 {
1018 return vdev->vq[n].vring.desc;
1019 }
1020
1021 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
1022 {
1023 return vdev->vq[n].vring.avail;
1024 }
1025
1026 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
1027 {
1028 return vdev->vq[n].vring.used;
1029 }
1030
1031 hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
1032 {
1033 return vdev->vq[n].vring.desc;
1034 }
1035
1036 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
1037 {
1038 return sizeof(VRingDesc) * vdev->vq[n].vring.num;
1039 }
1040
1041 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
1042 {
1043 return offsetof(VRingAvail, ring) +
1044 sizeof(uint64_t) * vdev->vq[n].vring.num;
1045 }
1046
1047 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
1048 {
1049 return offsetof(VRingUsed, ring) +
1050 sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
1051 }
1052
1053 hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
1054 {
1055 return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
1056 virtio_queue_get_used_size(vdev, n);
1057 }
1058
1059 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
1060 {
1061 return vdev->vq[n].last_avail_idx;
1062 }
1063
1064 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
1065 {
1066 vdev->vq[n].last_avail_idx = idx;
1067 }
1068
1069 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
1070 {
1071 vdev->vq[n].signalled_used_valid = false;
1072 }
1073
1074 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
1075 {
1076 return vdev->vq + n;
1077 }
1078
1079 uint16_t virtio_get_queue_index(VirtQueue *vq)
1080 {
1081 return vq->queue_index;
1082 }
1083
1084 static void virtio_queue_guest_notifier_read(EventNotifier *n)
1085 {
1086 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
1087 if (event_notifier_test_and_clear(n)) {
1088 virtio_irq(vq);
1089 }
1090 }
1091
1092 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
1093 bool with_irqfd)
1094 {
1095 if (assign && !with_irqfd) {
1096 event_notifier_set_handler(&vq->guest_notifier,
1097 virtio_queue_guest_notifier_read);
1098 } else {
1099 event_notifier_set_handler(&vq->guest_notifier, NULL);
1100 }
1101 if (!assign) {
1102 /* Test and clear notifier before closing it,
1103 * in case poll callback didn't have time to run. */
1104 virtio_queue_guest_notifier_read(&vq->guest_notifier);
1105 }
1106 }
1107
1108 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
1109 {
1110 return &vq->guest_notifier;
1111 }
1112
1113 static void virtio_queue_host_notifier_read(EventNotifier *n)
1114 {
1115 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1116 if (event_notifier_test_and_clear(n)) {
1117 virtio_queue_notify_vq(vq);
1118 }
1119 }
1120
1121 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
1122 bool set_handler)
1123 {
1124 if (assign && set_handler) {
1125 event_notifier_set_handler(&vq->host_notifier,
1126 virtio_queue_host_notifier_read);
1127 } else {
1128 event_notifier_set_handler(&vq->host_notifier, NULL);
1129 }
1130 if (!assign) {
1131 /* Test and clear notifier before after disabling event,
1132 * in case poll callback didn't have time to run. */
1133 virtio_queue_host_notifier_read(&vq->host_notifier);
1134 }
1135 }
1136
1137 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
1138 {
1139 return &vq->host_notifier;
1140 }
1141
1142 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
1143 {
1144 if (vdev->bus_name) {
1145 g_free(vdev->bus_name);
1146 vdev->bus_name = NULL;
1147 }
1148
1149 if (bus_name) {
1150 vdev->bus_name = g_strdup(bus_name);
1151 }
1152 }
1153
1154 static void virtio_device_realize(DeviceState *dev, Error **errp)
1155 {
1156 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1157 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1158 Error *err = NULL;
1159
1160 if (vdc->realize != NULL) {
1161 vdc->realize(dev, &err);
1162 if (err != NULL) {
1163 error_propagate(errp, err);
1164 return;
1165 }
1166 }
1167 virtio_bus_device_plugged(vdev);
1168 }
1169
1170 static void virtio_device_unrealize(DeviceState *dev, Error **errp)
1171 {
1172 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1173 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1174 Error *err = NULL;
1175
1176 virtio_bus_device_unplugged(vdev);
1177
1178 if (vdc->unrealize != NULL) {
1179 vdc->unrealize(dev, &err);
1180 if (err != NULL) {
1181 error_propagate(errp, err);
1182 return;
1183 }
1184 }
1185
1186 if (vdev->bus_name) {
1187 g_free(vdev->bus_name);
1188 vdev->bus_name = NULL;
1189 }
1190 }
1191
1192 static void virtio_device_class_init(ObjectClass *klass, void *data)
1193 {
1194 /* Set the default value here. */
1195 DeviceClass *dc = DEVICE_CLASS(klass);
1196
1197 dc->realize = virtio_device_realize;
1198 dc->unrealize = virtio_device_unrealize;
1199 dc->bus_type = TYPE_VIRTIO_BUS;
1200 }
1201
1202 static const TypeInfo virtio_device_info = {
1203 .name = TYPE_VIRTIO_DEVICE,
1204 .parent = TYPE_DEVICE,
1205 .instance_size = sizeof(VirtIODevice),
1206 .class_init = virtio_device_class_init,
1207 .abstract = true,
1208 .class_size = sizeof(VirtioDeviceClass),
1209 };
1210
1211 static void virtio_register_types(void)
1212 {
1213 type_register_static(&virtio_device_info);
1214 }
1215
1216 type_init(virtio_register_types)