pcie_aer: support configurable AER capa version
[qemu.git] / hw / vfio / pci.c
1 /*
2 * vfio based device assignment support
3 *
4 * Copyright Red Hat, Inc. 2012
5 *
6 * Authors:
7 * Alex Williamson <alex.williamson@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
19 */
20
21 #include "qemu/osdep.h"
22 #include <linux/vfio.h>
23 #include <sys/ioctl.h>
24
25 #include "hw/pci/msi.h"
26 #include "hw/pci/msix.h"
27 #include "hw/pci/pci_bridge.h"
28 #include "qemu/error-report.h"
29 #include "qemu/range.h"
30 #include "sysemu/kvm.h"
31 #include "sysemu/sysemu.h"
32 #include "pci.h"
33 #include "trace.h"
34 #include "qapi/error.h"
35
36 #define MSIX_CAP_LENGTH 12
37
38 static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
39 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
40
41 /*
42 * Disabling BAR mmaping can be slow, but toggling it around INTx can
43 * also be a huge overhead. We try to get the best of both worlds by
44 * waiting until an interrupt to disable mmaps (subsequent transitions
45 * to the same state are effectively no overhead). If the interrupt has
46 * been serviced and the time gap is long enough, we re-enable mmaps for
47 * performance. This works well for things like graphics cards, which
48 * may not use their interrupt at all and are penalized to an unusable
49 * level by read/write BAR traps. Other devices, like NICs, have more
50 * regular interrupts and see much better latency by staying in non-mmap
51 * mode. We therefore set the default mmap_timeout such that a ping
52 * is just enough to keep the mmap disabled. Users can experiment with
53 * other options with the x-intx-mmap-timeout-ms parameter (a value of
54 * zero disables the timer).
55 */
56 static void vfio_intx_mmap_enable(void *opaque)
57 {
58 VFIOPCIDevice *vdev = opaque;
59
60 if (vdev->intx.pending) {
61 timer_mod(vdev->intx.mmap_timer,
62 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
63 return;
64 }
65
66 vfio_mmap_set_enabled(vdev, true);
67 }
68
69 static void vfio_intx_interrupt(void *opaque)
70 {
71 VFIOPCIDevice *vdev = opaque;
72
73 if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
74 return;
75 }
76
77 trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin);
78
79 vdev->intx.pending = true;
80 pci_irq_assert(&vdev->pdev);
81 vfio_mmap_set_enabled(vdev, false);
82 if (vdev->intx.mmap_timeout) {
83 timer_mod(vdev->intx.mmap_timer,
84 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
85 }
86 }
87
88 static void vfio_intx_eoi(VFIODevice *vbasedev)
89 {
90 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
91
92 if (!vdev->intx.pending) {
93 return;
94 }
95
96 trace_vfio_intx_eoi(vbasedev->name);
97
98 vdev->intx.pending = false;
99 pci_irq_deassert(&vdev->pdev);
100 vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
101 }
102
103 static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev, Error **errp)
104 {
105 #ifdef CONFIG_KVM
106 struct kvm_irqfd irqfd = {
107 .fd = event_notifier_get_fd(&vdev->intx.interrupt),
108 .gsi = vdev->intx.route.irq,
109 .flags = KVM_IRQFD_FLAG_RESAMPLE,
110 };
111 struct vfio_irq_set *irq_set;
112 int ret, argsz;
113 int32_t *pfd;
114
115 if (vdev->no_kvm_intx || !kvm_irqfds_enabled() ||
116 vdev->intx.route.mode != PCI_INTX_ENABLED ||
117 !kvm_resamplefds_enabled()) {
118 return;
119 }
120
121 /* Get to a known interrupt state */
122 qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev);
123 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
124 vdev->intx.pending = false;
125 pci_irq_deassert(&vdev->pdev);
126
127 /* Get an eventfd for resample/unmask */
128 if (event_notifier_init(&vdev->intx.unmask, 0)) {
129 error_setg(errp, "event_notifier_init failed eoi");
130 goto fail;
131 }
132
133 /* KVM triggers it, VFIO listens for it */
134 irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
135
136 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
137 error_setg_errno(errp, errno, "failed to setup resample irqfd");
138 goto fail_irqfd;
139 }
140
141 argsz = sizeof(*irq_set) + sizeof(*pfd);
142
143 irq_set = g_malloc0(argsz);
144 irq_set->argsz = argsz;
145 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK;
146 irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
147 irq_set->start = 0;
148 irq_set->count = 1;
149 pfd = (int32_t *)&irq_set->data;
150
151 *pfd = irqfd.resamplefd;
152
153 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
154 g_free(irq_set);
155 if (ret) {
156 error_setg_errno(errp, -ret, "failed to setup INTx unmask fd");
157 goto fail_vfio;
158 }
159
160 /* Let'em rip */
161 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
162
163 vdev->intx.kvm_accel = true;
164
165 trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
166
167 return;
168
169 fail_vfio:
170 irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
171 kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
172 fail_irqfd:
173 event_notifier_cleanup(&vdev->intx.unmask);
174 fail:
175 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
176 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
177 #endif
178 }
179
180 static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
181 {
182 #ifdef CONFIG_KVM
183 struct kvm_irqfd irqfd = {
184 .fd = event_notifier_get_fd(&vdev->intx.interrupt),
185 .gsi = vdev->intx.route.irq,
186 .flags = KVM_IRQFD_FLAG_DEASSIGN,
187 };
188
189 if (!vdev->intx.kvm_accel) {
190 return;
191 }
192
193 /*
194 * Get to a known state, hardware masked, QEMU ready to accept new
195 * interrupts, QEMU IRQ de-asserted.
196 */
197 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
198 vdev->intx.pending = false;
199 pci_irq_deassert(&vdev->pdev);
200
201 /* Tell KVM to stop listening for an INTx irqfd */
202 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
203 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
204 }
205
206 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
207 event_notifier_cleanup(&vdev->intx.unmask);
208
209 /* QEMU starts listening for interrupt events. */
210 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
211
212 vdev->intx.kvm_accel = false;
213
214 /* If we've missed an event, let it re-fire through QEMU */
215 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
216
217 trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
218 #endif
219 }
220
221 static void vfio_intx_update(PCIDevice *pdev)
222 {
223 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
224 PCIINTxRoute route;
225 Error *err = NULL;
226
227 if (vdev->interrupt != VFIO_INT_INTx) {
228 return;
229 }
230
231 route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
232
233 if (!pci_intx_route_changed(&vdev->intx.route, &route)) {
234 return; /* Nothing changed */
235 }
236
237 trace_vfio_intx_update(vdev->vbasedev.name,
238 vdev->intx.route.irq, route.irq);
239
240 vfio_intx_disable_kvm(vdev);
241
242 vdev->intx.route = route;
243
244 if (route.mode != PCI_INTX_ENABLED) {
245 return;
246 }
247
248 vfio_intx_enable_kvm(vdev, &err);
249 if (err) {
250 error_reportf_err(err, WARN_PREFIX, vdev->vbasedev.name);
251 }
252
253 /* Re-enable the interrupt in cased we missed an EOI */
254 vfio_intx_eoi(&vdev->vbasedev);
255 }
256
257 static int vfio_intx_enable(VFIOPCIDevice *vdev, Error **errp)
258 {
259 uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
260 int ret, argsz;
261 struct vfio_irq_set *irq_set;
262 int32_t *pfd;
263 Error *err = NULL;
264
265 if (!pin) {
266 return 0;
267 }
268
269 vfio_disable_interrupts(vdev);
270
271 vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
272 pci_config_set_interrupt_pin(vdev->pdev.config, pin);
273
274 #ifdef CONFIG_KVM
275 /*
276 * Only conditional to avoid generating error messages on platforms
277 * where we won't actually use the result anyway.
278 */
279 if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
280 vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
281 vdev->intx.pin);
282 }
283 #endif
284
285 ret = event_notifier_init(&vdev->intx.interrupt, 0);
286 if (ret) {
287 error_setg_errno(errp, -ret, "event_notifier_init failed");
288 return ret;
289 }
290
291 argsz = sizeof(*irq_set) + sizeof(*pfd);
292
293 irq_set = g_malloc0(argsz);
294 irq_set->argsz = argsz;
295 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
296 irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
297 irq_set->start = 0;
298 irq_set->count = 1;
299 pfd = (int32_t *)&irq_set->data;
300
301 *pfd = event_notifier_get_fd(&vdev->intx.interrupt);
302 qemu_set_fd_handler(*pfd, vfio_intx_interrupt, NULL, vdev);
303
304 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
305 g_free(irq_set);
306 if (ret) {
307 error_setg_errno(errp, -ret, "failed to setup INTx fd");
308 qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
309 event_notifier_cleanup(&vdev->intx.interrupt);
310 return -errno;
311 }
312
313 vfio_intx_enable_kvm(vdev, &err);
314 if (err) {
315 error_reportf_err(err, WARN_PREFIX, vdev->vbasedev.name);
316 }
317
318 vdev->interrupt = VFIO_INT_INTx;
319
320 trace_vfio_intx_enable(vdev->vbasedev.name);
321
322 return 0;
323 }
324
325 static void vfio_intx_disable(VFIOPCIDevice *vdev)
326 {
327 int fd;
328
329 timer_del(vdev->intx.mmap_timer);
330 vfio_intx_disable_kvm(vdev);
331 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
332 vdev->intx.pending = false;
333 pci_irq_deassert(&vdev->pdev);
334 vfio_mmap_set_enabled(vdev, true);
335
336 fd = event_notifier_get_fd(&vdev->intx.interrupt);
337 qemu_set_fd_handler(fd, NULL, NULL, vdev);
338 event_notifier_cleanup(&vdev->intx.interrupt);
339
340 vdev->interrupt = VFIO_INT_NONE;
341
342 trace_vfio_intx_disable(vdev->vbasedev.name);
343 }
344
345 /*
346 * MSI/X
347 */
348 static void vfio_msi_interrupt(void *opaque)
349 {
350 VFIOMSIVector *vector = opaque;
351 VFIOPCIDevice *vdev = vector->vdev;
352 MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector);
353 void (*notify)(PCIDevice *dev, unsigned vector);
354 MSIMessage msg;
355 int nr = vector - vdev->msi_vectors;
356
357 if (!event_notifier_test_and_clear(&vector->interrupt)) {
358 return;
359 }
360
361 if (vdev->interrupt == VFIO_INT_MSIX) {
362 get_msg = msix_get_message;
363 notify = msix_notify;
364
365 /* A masked vector firing needs to use the PBA, enable it */
366 if (msix_is_masked(&vdev->pdev, nr)) {
367 set_bit(nr, vdev->msix->pending);
368 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, true);
369 trace_vfio_msix_pba_enable(vdev->vbasedev.name);
370 }
371 } else if (vdev->interrupt == VFIO_INT_MSI) {
372 get_msg = msi_get_message;
373 notify = msi_notify;
374 } else {
375 abort();
376 }
377
378 msg = get_msg(&vdev->pdev, nr);
379 trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
380 notify(&vdev->pdev, nr);
381 }
382
383 static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
384 {
385 struct vfio_irq_set *irq_set;
386 int ret = 0, i, argsz;
387 int32_t *fds;
388
389 argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
390
391 irq_set = g_malloc0(argsz);
392 irq_set->argsz = argsz;
393 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
394 irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
395 irq_set->start = 0;
396 irq_set->count = vdev->nr_vectors;
397 fds = (int32_t *)&irq_set->data;
398
399 for (i = 0; i < vdev->nr_vectors; i++) {
400 int fd = -1;
401
402 /*
403 * MSI vs MSI-X - The guest has direct access to MSI mask and pending
404 * bits, therefore we always use the KVM signaling path when setup.
405 * MSI-X mask and pending bits are emulated, so we want to use the
406 * KVM signaling path only when configured and unmasked.
407 */
408 if (vdev->msi_vectors[i].use) {
409 if (vdev->msi_vectors[i].virq < 0 ||
410 (msix && msix_is_masked(&vdev->pdev, i))) {
411 fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
412 } else {
413 fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
414 }
415 }
416
417 fds[i] = fd;
418 }
419
420 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
421
422 g_free(irq_set);
423
424 return ret;
425 }
426
427 static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,
428 int vector_n, bool msix)
429 {
430 int virq;
431
432 if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi)) {
433 return;
434 }
435
436 if (event_notifier_init(&vector->kvm_interrupt, 0)) {
437 return;
438 }
439
440 virq = kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev);
441 if (virq < 0) {
442 event_notifier_cleanup(&vector->kvm_interrupt);
443 return;
444 }
445
446 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
447 NULL, virq) < 0) {
448 kvm_irqchip_release_virq(kvm_state, virq);
449 event_notifier_cleanup(&vector->kvm_interrupt);
450 return;
451 }
452
453 vector->virq = virq;
454 }
455
456 static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
457 {
458 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
459 vector->virq);
460 kvm_irqchip_release_virq(kvm_state, vector->virq);
461 vector->virq = -1;
462 event_notifier_cleanup(&vector->kvm_interrupt);
463 }
464
465 static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
466 PCIDevice *pdev)
467 {
468 kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
469 kvm_irqchip_commit_routes(kvm_state);
470 }
471
472 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
473 MSIMessage *msg, IOHandler *handler)
474 {
475 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
476 VFIOMSIVector *vector;
477 int ret;
478
479 trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
480
481 vector = &vdev->msi_vectors[nr];
482
483 if (!vector->use) {
484 vector->vdev = vdev;
485 vector->virq = -1;
486 if (event_notifier_init(&vector->interrupt, 0)) {
487 error_report("vfio: Error: event_notifier_init failed");
488 }
489 vector->use = true;
490 msix_vector_use(pdev, nr);
491 }
492
493 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
494 handler, NULL, vector);
495
496 /*
497 * Attempt to enable route through KVM irqchip,
498 * default to userspace handling if unavailable.
499 */
500 if (vector->virq >= 0) {
501 if (!msg) {
502 vfio_remove_kvm_msi_virq(vector);
503 } else {
504 vfio_update_kvm_msi_virq(vector, *msg, pdev);
505 }
506 } else {
507 if (msg) {
508 vfio_add_kvm_msi_virq(vdev, vector, nr, true);
509 }
510 }
511
512 /*
513 * We don't want to have the host allocate all possible MSI vectors
514 * for a device if they're not in use, so we shutdown and incrementally
515 * increase them as needed.
516 */
517 if (vdev->nr_vectors < nr + 1) {
518 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
519 vdev->nr_vectors = nr + 1;
520 ret = vfio_enable_vectors(vdev, true);
521 if (ret) {
522 error_report("vfio: failed to enable vectors, %d", ret);
523 }
524 } else {
525 int argsz;
526 struct vfio_irq_set *irq_set;
527 int32_t *pfd;
528
529 argsz = sizeof(*irq_set) + sizeof(*pfd);
530
531 irq_set = g_malloc0(argsz);
532 irq_set->argsz = argsz;
533 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
534 VFIO_IRQ_SET_ACTION_TRIGGER;
535 irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
536 irq_set->start = nr;
537 irq_set->count = 1;
538 pfd = (int32_t *)&irq_set->data;
539
540 if (vector->virq >= 0) {
541 *pfd = event_notifier_get_fd(&vector->kvm_interrupt);
542 } else {
543 *pfd = event_notifier_get_fd(&vector->interrupt);
544 }
545
546 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
547 g_free(irq_set);
548 if (ret) {
549 error_report("vfio: failed to modify vector, %d", ret);
550 }
551 }
552
553 /* Disable PBA emulation when nothing more is pending. */
554 clear_bit(nr, vdev->msix->pending);
555 if (find_first_bit(vdev->msix->pending,
556 vdev->nr_vectors) == vdev->nr_vectors) {
557 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
558 trace_vfio_msix_pba_disable(vdev->vbasedev.name);
559 }
560
561 return 0;
562 }
563
564 static int vfio_msix_vector_use(PCIDevice *pdev,
565 unsigned int nr, MSIMessage msg)
566 {
567 return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
568 }
569
570 static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
571 {
572 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
573 VFIOMSIVector *vector = &vdev->msi_vectors[nr];
574
575 trace_vfio_msix_vector_release(vdev->vbasedev.name, nr);
576
577 /*
578 * There are still old guests that mask and unmask vectors on every
579 * interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of
580 * the KVM setup in place, simply switch VFIO to use the non-bypass
581 * eventfd. We'll then fire the interrupt through QEMU and the MSI-X
582 * core will mask the interrupt and set pending bits, allowing it to
583 * be re-asserted on unmask. Nothing to do if already using QEMU mode.
584 */
585 if (vector->virq >= 0) {
586 int argsz;
587 struct vfio_irq_set *irq_set;
588 int32_t *pfd;
589
590 argsz = sizeof(*irq_set) + sizeof(*pfd);
591
592 irq_set = g_malloc0(argsz);
593 irq_set->argsz = argsz;
594 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
595 VFIO_IRQ_SET_ACTION_TRIGGER;
596 irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
597 irq_set->start = nr;
598 irq_set->count = 1;
599 pfd = (int32_t *)&irq_set->data;
600
601 *pfd = event_notifier_get_fd(&vector->interrupt);
602
603 ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
604
605 g_free(irq_set);
606 }
607 }
608
609 static void vfio_msix_enable(VFIOPCIDevice *vdev)
610 {
611 vfio_disable_interrupts(vdev);
612
613 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
614
615 vdev->interrupt = VFIO_INT_MSIX;
616
617 /*
618 * Some communication channels between VF & PF or PF & fw rely on the
619 * physical state of the device and expect that enabling MSI-X from the
620 * guest enables the same on the host. When our guest is Linux, the
621 * guest driver call to pci_enable_msix() sets the enabling bit in the
622 * MSI-X capability, but leaves the vector table masked. We therefore
623 * can't rely on a vector_use callback (from request_irq() in the guest)
624 * to switch the physical device into MSI-X mode because that may come a
625 * long time after pci_enable_msix(). This code enables vector 0 with
626 * triggering to userspace, then immediately release the vector, leaving
627 * the physical device with no vectors enabled, but MSI-X enabled, just
628 * like the guest view.
629 */
630 vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
631 vfio_msix_vector_release(&vdev->pdev, 0);
632
633 if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
634 vfio_msix_vector_release, NULL)) {
635 error_report("vfio: msix_set_vector_notifiers failed");
636 }
637
638 trace_vfio_msix_enable(vdev->vbasedev.name);
639 }
640
641 static void vfio_msi_enable(VFIOPCIDevice *vdev)
642 {
643 int ret, i;
644
645 vfio_disable_interrupts(vdev);
646
647 vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
648 retry:
649 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
650
651 for (i = 0; i < vdev->nr_vectors; i++) {
652 VFIOMSIVector *vector = &vdev->msi_vectors[i];
653
654 vector->vdev = vdev;
655 vector->virq = -1;
656 vector->use = true;
657
658 if (event_notifier_init(&vector->interrupt, 0)) {
659 error_report("vfio: Error: event_notifier_init failed");
660 }
661
662 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
663 vfio_msi_interrupt, NULL, vector);
664
665 /*
666 * Attempt to enable route through KVM irqchip,
667 * default to userspace handling if unavailable.
668 */
669 vfio_add_kvm_msi_virq(vdev, vector, i, false);
670 }
671
672 /* Set interrupt type prior to possible interrupts */
673 vdev->interrupt = VFIO_INT_MSI;
674
675 ret = vfio_enable_vectors(vdev, false);
676 if (ret) {
677 if (ret < 0) {
678 error_report("vfio: Error: Failed to setup MSI fds: %m");
679 } else if (ret != vdev->nr_vectors) {
680 error_report("vfio: Error: Failed to enable %d "
681 "MSI vectors, retry with %d", vdev->nr_vectors, ret);
682 }
683
684 for (i = 0; i < vdev->nr_vectors; i++) {
685 VFIOMSIVector *vector = &vdev->msi_vectors[i];
686 if (vector->virq >= 0) {
687 vfio_remove_kvm_msi_virq(vector);
688 }
689 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
690 NULL, NULL, NULL);
691 event_notifier_cleanup(&vector->interrupt);
692 }
693
694 g_free(vdev->msi_vectors);
695
696 if (ret > 0 && ret != vdev->nr_vectors) {
697 vdev->nr_vectors = ret;
698 goto retry;
699 }
700 vdev->nr_vectors = 0;
701
702 /*
703 * Failing to setup MSI doesn't really fall within any specification.
704 * Let's try leaving interrupts disabled and hope the guest figures
705 * out to fall back to INTx for this device.
706 */
707 error_report("vfio: Error: Failed to enable MSI");
708 vdev->interrupt = VFIO_INT_NONE;
709
710 return;
711 }
712
713 trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
714 }
715
716 static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
717 {
718 Error *err = NULL;
719 int i;
720
721 for (i = 0; i < vdev->nr_vectors; i++) {
722 VFIOMSIVector *vector = &vdev->msi_vectors[i];
723 if (vdev->msi_vectors[i].use) {
724 if (vector->virq >= 0) {
725 vfio_remove_kvm_msi_virq(vector);
726 }
727 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
728 NULL, NULL, NULL);
729 event_notifier_cleanup(&vector->interrupt);
730 }
731 }
732
733 g_free(vdev->msi_vectors);
734 vdev->msi_vectors = NULL;
735 vdev->nr_vectors = 0;
736 vdev->interrupt = VFIO_INT_NONE;
737
738 vfio_intx_enable(vdev, &err);
739 if (err) {
740 error_reportf_err(err, ERR_PREFIX, vdev->vbasedev.name);
741 }
742 }
743
744 static void vfio_msix_disable(VFIOPCIDevice *vdev)
745 {
746 int i;
747
748 msix_unset_vector_notifiers(&vdev->pdev);
749
750 /*
751 * MSI-X will only release vectors if MSI-X is still enabled on the
752 * device, check through the rest and release it ourselves if necessary.
753 */
754 for (i = 0; i < vdev->nr_vectors; i++) {
755 if (vdev->msi_vectors[i].use) {
756 vfio_msix_vector_release(&vdev->pdev, i);
757 msix_vector_unuse(&vdev->pdev, i);
758 }
759 }
760
761 if (vdev->nr_vectors) {
762 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
763 }
764
765 vfio_msi_disable_common(vdev);
766
767 memset(vdev->msix->pending, 0,
768 BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long));
769
770 trace_vfio_msix_disable(vdev->vbasedev.name);
771 }
772
773 static void vfio_msi_disable(VFIOPCIDevice *vdev)
774 {
775 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
776 vfio_msi_disable_common(vdev);
777
778 trace_vfio_msi_disable(vdev->vbasedev.name);
779 }
780
781 static void vfio_update_msi(VFIOPCIDevice *vdev)
782 {
783 int i;
784
785 for (i = 0; i < vdev->nr_vectors; i++) {
786 VFIOMSIVector *vector = &vdev->msi_vectors[i];
787 MSIMessage msg;
788
789 if (!vector->use || vector->virq < 0) {
790 continue;
791 }
792
793 msg = msi_get_message(&vdev->pdev, i);
794 vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev);
795 }
796 }
797
798 static void vfio_pci_load_rom(VFIOPCIDevice *vdev)
799 {
800 struct vfio_region_info *reg_info;
801 uint64_t size;
802 off_t off = 0;
803 ssize_t bytes;
804
805 if (vfio_get_region_info(&vdev->vbasedev,
806 VFIO_PCI_ROM_REGION_INDEX, &reg_info)) {
807 error_report("vfio: Error getting ROM info: %m");
808 return;
809 }
810
811 trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size,
812 (unsigned long)reg_info->offset,
813 (unsigned long)reg_info->flags);
814
815 vdev->rom_size = size = reg_info->size;
816 vdev->rom_offset = reg_info->offset;
817
818 g_free(reg_info);
819
820 if (!vdev->rom_size) {
821 vdev->rom_read_failed = true;
822 error_report("vfio-pci: Cannot read device rom at "
823 "%s", vdev->vbasedev.name);
824 error_printf("Device option ROM contents are probably invalid "
825 "(check dmesg).\nSkip option ROM probe with rombar=0, "
826 "or load from file with romfile=\n");
827 return;
828 }
829
830 vdev->rom = g_malloc(size);
831 memset(vdev->rom, 0xff, size);
832
833 while (size) {
834 bytes = pread(vdev->vbasedev.fd, vdev->rom + off,
835 size, vdev->rom_offset + off);
836 if (bytes == 0) {
837 break;
838 } else if (bytes > 0) {
839 off += bytes;
840 size -= bytes;
841 } else {
842 if (errno == EINTR || errno == EAGAIN) {
843 continue;
844 }
845 error_report("vfio: Error reading device ROM: %m");
846 break;
847 }
848 }
849
850 /*
851 * Test the ROM signature against our device, if the vendor is correct
852 * but the device ID doesn't match, store the correct device ID and
853 * recompute the checksum. Intel IGD devices need this and are known
854 * to have bogus checksums so we can't simply adjust the checksum.
855 */
856 if (pci_get_word(vdev->rom) == 0xaa55 &&
857 pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size &&
858 !memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) {
859 uint16_t vid, did;
860
861 vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4);
862 did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6);
863
864 if (vid == vdev->vendor_id && did != vdev->device_id) {
865 int i;
866 uint8_t csum, *data = vdev->rom;
867
868 pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6,
869 vdev->device_id);
870 data[6] = 0;
871
872 for (csum = 0, i = 0; i < vdev->rom_size; i++) {
873 csum += data[i];
874 }
875
876 data[6] = -csum;
877 }
878 }
879 }
880
881 static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
882 {
883 VFIOPCIDevice *vdev = opaque;
884 union {
885 uint8_t byte;
886 uint16_t word;
887 uint32_t dword;
888 uint64_t qword;
889 } val;
890 uint64_t data = 0;
891
892 /* Load the ROM lazily when the guest tries to read it */
893 if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
894 vfio_pci_load_rom(vdev);
895 }
896
897 memcpy(&val, vdev->rom + addr,
898 (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
899
900 switch (size) {
901 case 1:
902 data = val.byte;
903 break;
904 case 2:
905 data = le16_to_cpu(val.word);
906 break;
907 case 4:
908 data = le32_to_cpu(val.dword);
909 break;
910 default:
911 hw_error("vfio: unsupported read size, %d bytes\n", size);
912 break;
913 }
914
915 trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data);
916
917 return data;
918 }
919
920 static void vfio_rom_write(void *opaque, hwaddr addr,
921 uint64_t data, unsigned size)
922 {
923 }
924
925 static const MemoryRegionOps vfio_rom_ops = {
926 .read = vfio_rom_read,
927 .write = vfio_rom_write,
928 .endianness = DEVICE_LITTLE_ENDIAN,
929 };
930
931 static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
932 {
933 uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
934 off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
935 DeviceState *dev = DEVICE(vdev);
936 char *name;
937 int fd = vdev->vbasedev.fd;
938
939 if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
940 /* Since pci handles romfile, just print a message and return */
941 if (vfio_blacklist_opt_rom(vdev) && vdev->pdev.romfile) {
942 error_printf("Warning : Device at %s is known to cause system instability issues during option rom execution. Proceeding anyway since user specified romfile\n",
943 vdev->vbasedev.name);
944 }
945 return;
946 }
947
948 /*
949 * Use the same size ROM BAR as the physical device. The contents
950 * will get filled in later when the guest tries to read it.
951 */
952 if (pread(fd, &orig, 4, offset) != 4 ||
953 pwrite(fd, &size, 4, offset) != 4 ||
954 pread(fd, &size, 4, offset) != 4 ||
955 pwrite(fd, &orig, 4, offset) != 4) {
956 error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name);
957 return;
958 }
959
960 size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
961
962 if (!size) {
963 return;
964 }
965
966 if (vfio_blacklist_opt_rom(vdev)) {
967 if (dev->opts && qemu_opt_get(dev->opts, "rombar")) {
968 error_printf("Warning : Device at %s is known to cause system instability issues during option rom execution. Proceeding anyway since user specified non zero value for rombar\n",
969 vdev->vbasedev.name);
970 } else {
971 error_printf("Warning : Rom loading for device at %s has been disabled due to system instability issues. Specify rombar=1 or romfile to force\n",
972 vdev->vbasedev.name);
973 return;
974 }
975 }
976
977 trace_vfio_pci_size_rom(vdev->vbasedev.name, size);
978
979 name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name);
980
981 memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
982 &vfio_rom_ops, vdev, name, size);
983 g_free(name);
984
985 pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
986 PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
987
988 vdev->pdev.has_rom = true;
989 vdev->rom_read_failed = false;
990 }
991
992 void vfio_vga_write(void *opaque, hwaddr addr,
993 uint64_t data, unsigned size)
994 {
995 VFIOVGARegion *region = opaque;
996 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
997 union {
998 uint8_t byte;
999 uint16_t word;
1000 uint32_t dword;
1001 uint64_t qword;
1002 } buf;
1003 off_t offset = vga->fd_offset + region->offset + addr;
1004
1005 switch (size) {
1006 case 1:
1007 buf.byte = data;
1008 break;
1009 case 2:
1010 buf.word = cpu_to_le16(data);
1011 break;
1012 case 4:
1013 buf.dword = cpu_to_le32(data);
1014 break;
1015 default:
1016 hw_error("vfio: unsupported write size, %d bytes", size);
1017 break;
1018 }
1019
1020 if (pwrite(vga->fd, &buf, size, offset) != size) {
1021 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
1022 __func__, region->offset + addr, data, size);
1023 }
1024
1025 trace_vfio_vga_write(region->offset + addr, data, size);
1026 }
1027
1028 uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
1029 {
1030 VFIOVGARegion *region = opaque;
1031 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1032 union {
1033 uint8_t byte;
1034 uint16_t word;
1035 uint32_t dword;
1036 uint64_t qword;
1037 } buf;
1038 uint64_t data = 0;
1039 off_t offset = vga->fd_offset + region->offset + addr;
1040
1041 if (pread(vga->fd, &buf, size, offset) != size) {
1042 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1043 __func__, region->offset + addr, size);
1044 return (uint64_t)-1;
1045 }
1046
1047 switch (size) {
1048 case 1:
1049 data = buf.byte;
1050 break;
1051 case 2:
1052 data = le16_to_cpu(buf.word);
1053 break;
1054 case 4:
1055 data = le32_to_cpu(buf.dword);
1056 break;
1057 default:
1058 hw_error("vfio: unsupported read size, %d bytes", size);
1059 break;
1060 }
1061
1062 trace_vfio_vga_read(region->offset + addr, size, data);
1063
1064 return data;
1065 }
1066
1067 static const MemoryRegionOps vfio_vga_ops = {
1068 .read = vfio_vga_read,
1069 .write = vfio_vga_write,
1070 .endianness = DEVICE_LITTLE_ENDIAN,
1071 };
1072
1073 /*
1074 * Expand memory region of sub-page(size < PAGE_SIZE) MMIO BAR to page
1075 * size if the BAR is in an exclusive page in host so that we could map
1076 * this BAR to guest. But this sub-page BAR may not occupy an exclusive
1077 * page in guest. So we should set the priority of the expanded memory
1078 * region to zero in case of overlap with BARs which share the same page
1079 * with the sub-page BAR in guest. Besides, we should also recover the
1080 * size of this sub-page BAR when its base address is changed in guest
1081 * and not page aligned any more.
1082 */
1083 static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar)
1084 {
1085 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
1086 VFIORegion *region = &vdev->bars[bar].region;
1087 MemoryRegion *mmap_mr, *mr;
1088 PCIIORegion *r;
1089 pcibus_t bar_addr;
1090 uint64_t size = region->size;
1091
1092 /* Make sure that the whole region is allowed to be mmapped */
1093 if (region->nr_mmaps != 1 || !region->mmaps[0].mmap ||
1094 region->mmaps[0].size != region->size) {
1095 return;
1096 }
1097
1098 r = &pdev->io_regions[bar];
1099 bar_addr = r->addr;
1100 mr = region->mem;
1101 mmap_mr = &region->mmaps[0].mem;
1102
1103 /* If BAR is mapped and page aligned, update to fill PAGE_SIZE */
1104 if (bar_addr != PCI_BAR_UNMAPPED &&
1105 !(bar_addr & ~qemu_real_host_page_mask)) {
1106 size = qemu_real_host_page_size;
1107 }
1108
1109 memory_region_transaction_begin();
1110
1111 memory_region_set_size(mr, size);
1112 memory_region_set_size(mmap_mr, size);
1113 if (size != region->size && memory_region_is_mapped(mr)) {
1114 memory_region_del_subregion(r->address_space, mr);
1115 memory_region_add_subregion_overlap(r->address_space,
1116 bar_addr, mr, 0);
1117 }
1118
1119 memory_region_transaction_commit();
1120 }
1121
1122 /*
1123 * PCI config space
1124 */
1125 uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
1126 {
1127 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
1128 uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
1129
1130 memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
1131 emu_bits = le32_to_cpu(emu_bits);
1132
1133 if (emu_bits) {
1134 emu_val = pci_default_read_config(pdev, addr, len);
1135 }
1136
1137 if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
1138 ssize_t ret;
1139
1140 ret = pread(vdev->vbasedev.fd, &phys_val, len,
1141 vdev->config_offset + addr);
1142 if (ret != len) {
1143 error_report("%s(%s, 0x%x, 0x%x) failed: %m",
1144 __func__, vdev->vbasedev.name, addr, len);
1145 return -errno;
1146 }
1147 phys_val = le32_to_cpu(phys_val);
1148 }
1149
1150 val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
1151
1152 trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val);
1153
1154 return val;
1155 }
1156
1157 void vfio_pci_write_config(PCIDevice *pdev,
1158 uint32_t addr, uint32_t val, int len)
1159 {
1160 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
1161 uint32_t val_le = cpu_to_le32(val);
1162
1163 trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len);
1164
1165 /* Write everything to VFIO, let it filter out what we can't write */
1166 if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr)
1167 != len) {
1168 error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
1169 __func__, vdev->vbasedev.name, addr, val, len);
1170 }
1171
1172 /* MSI/MSI-X Enabling/Disabling */
1173 if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
1174 ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
1175 int is_enabled, was_enabled = msi_enabled(pdev);
1176
1177 pci_default_write_config(pdev, addr, val, len);
1178
1179 is_enabled = msi_enabled(pdev);
1180
1181 if (!was_enabled) {
1182 if (is_enabled) {
1183 vfio_msi_enable(vdev);
1184 }
1185 } else {
1186 if (!is_enabled) {
1187 vfio_msi_disable(vdev);
1188 } else {
1189 vfio_update_msi(vdev);
1190 }
1191 }
1192 } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
1193 ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
1194 int is_enabled, was_enabled = msix_enabled(pdev);
1195
1196 pci_default_write_config(pdev, addr, val, len);
1197
1198 is_enabled = msix_enabled(pdev);
1199
1200 if (!was_enabled && is_enabled) {
1201 vfio_msix_enable(vdev);
1202 } else if (was_enabled && !is_enabled) {
1203 vfio_msix_disable(vdev);
1204 }
1205 } else if (ranges_overlap(addr, len, PCI_BASE_ADDRESS_0, 24) ||
1206 range_covers_byte(addr, len, PCI_COMMAND)) {
1207 pcibus_t old_addr[PCI_NUM_REGIONS - 1];
1208 int bar;
1209
1210 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1211 old_addr[bar] = pdev->io_regions[bar].addr;
1212 }
1213
1214 pci_default_write_config(pdev, addr, val, len);
1215
1216 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1217 if (old_addr[bar] != pdev->io_regions[bar].addr &&
1218 pdev->io_regions[bar].size > 0 &&
1219 pdev->io_regions[bar].size < qemu_real_host_page_size) {
1220 vfio_sub_page_bar_update_mapping(pdev, bar);
1221 }
1222 }
1223 } else {
1224 /* Write everything to QEMU to keep emulated bits correct */
1225 pci_default_write_config(pdev, addr, val, len);
1226 }
1227 }
1228
1229 /*
1230 * Interrupt setup
1231 */
1232 static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
1233 {
1234 /*
1235 * More complicated than it looks. Disabling MSI/X transitions the
1236 * device to INTx mode (if supported). Therefore we need to first
1237 * disable MSI/X and then cleanup by disabling INTx.
1238 */
1239 if (vdev->interrupt == VFIO_INT_MSIX) {
1240 vfio_msix_disable(vdev);
1241 } else if (vdev->interrupt == VFIO_INT_MSI) {
1242 vfio_msi_disable(vdev);
1243 }
1244
1245 if (vdev->interrupt == VFIO_INT_INTx) {
1246 vfio_intx_disable(vdev);
1247 }
1248 }
1249
1250 static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1251 {
1252 uint16_t ctrl;
1253 bool msi_64bit, msi_maskbit;
1254 int ret, entries;
1255 Error *err = NULL;
1256
1257 if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl),
1258 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
1259 error_setg_errno(errp, errno, "failed reading MSI PCI_CAP_FLAGS");
1260 return -errno;
1261 }
1262 ctrl = le16_to_cpu(ctrl);
1263
1264 msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
1265 msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
1266 entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
1267
1268 trace_vfio_msi_setup(vdev->vbasedev.name, pos);
1269
1270 ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit, &err);
1271 if (ret < 0) {
1272 if (ret == -ENOTSUP) {
1273 return 0;
1274 }
1275 error_prepend(&err, "msi_init failed: ");
1276 error_propagate(errp, err);
1277 return ret;
1278 }
1279 vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
1280
1281 return 0;
1282 }
1283
1284 static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev)
1285 {
1286 off_t start, end;
1287 VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region;
1288
1289 /*
1290 * We expect to find a single mmap covering the whole BAR, anything else
1291 * means it's either unsupported or already setup.
1292 */
1293 if (region->nr_mmaps != 1 || region->mmaps[0].offset ||
1294 region->size != region->mmaps[0].size) {
1295 return;
1296 }
1297
1298 /* MSI-X table start and end aligned to host page size */
1299 start = vdev->msix->table_offset & qemu_real_host_page_mask;
1300 end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
1301 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
1302
1303 /*
1304 * Does the MSI-X table cover the beginning of the BAR? The whole BAR?
1305 * NB - Host page size is necessarily a power of two and so is the PCI
1306 * BAR (not counting EA yet), therefore if we have host page aligned
1307 * @start and @end, then any remainder of the BAR before or after those
1308 * must be at least host page sized and therefore mmap'able.
1309 */
1310 if (!start) {
1311 if (end >= region->size) {
1312 region->nr_mmaps = 0;
1313 g_free(region->mmaps);
1314 region->mmaps = NULL;
1315 trace_vfio_msix_fixup(vdev->vbasedev.name,
1316 vdev->msix->table_bar, 0, 0);
1317 } else {
1318 region->mmaps[0].offset = end;
1319 region->mmaps[0].size = region->size - end;
1320 trace_vfio_msix_fixup(vdev->vbasedev.name,
1321 vdev->msix->table_bar, region->mmaps[0].offset,
1322 region->mmaps[0].offset + region->mmaps[0].size);
1323 }
1324
1325 /* Maybe it's aligned at the end of the BAR */
1326 } else if (end >= region->size) {
1327 region->mmaps[0].size = start;
1328 trace_vfio_msix_fixup(vdev->vbasedev.name,
1329 vdev->msix->table_bar, region->mmaps[0].offset,
1330 region->mmaps[0].offset + region->mmaps[0].size);
1331
1332 /* Otherwise it must split the BAR */
1333 } else {
1334 region->nr_mmaps = 2;
1335 region->mmaps = g_renew(VFIOMmap, region->mmaps, 2);
1336
1337 memcpy(&region->mmaps[1], &region->mmaps[0], sizeof(VFIOMmap));
1338
1339 region->mmaps[0].size = start;
1340 trace_vfio_msix_fixup(vdev->vbasedev.name,
1341 vdev->msix->table_bar, region->mmaps[0].offset,
1342 region->mmaps[0].offset + region->mmaps[0].size);
1343
1344 region->mmaps[1].offset = end;
1345 region->mmaps[1].size = region->size - end;
1346 trace_vfio_msix_fixup(vdev->vbasedev.name,
1347 vdev->msix->table_bar, region->mmaps[1].offset,
1348 region->mmaps[1].offset + region->mmaps[1].size);
1349 }
1350 }
1351
1352 /*
1353 * We don't have any control over how pci_add_capability() inserts
1354 * capabilities into the chain. In order to setup MSI-X we need a
1355 * MemoryRegion for the BAR. In order to setup the BAR and not
1356 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
1357 * need to first look for where the MSI-X table lives. So we
1358 * unfortunately split MSI-X setup across two functions.
1359 */
1360 static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
1361 {
1362 uint8_t pos;
1363 uint16_t ctrl;
1364 uint32_t table, pba;
1365 int fd = vdev->vbasedev.fd;
1366 VFIOMSIXInfo *msix;
1367
1368 pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
1369 if (!pos) {
1370 return;
1371 }
1372
1373 if (pread(fd, &ctrl, sizeof(ctrl),
1374 vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) {
1375 error_setg_errno(errp, errno, "failed to read PCI MSIX FLAGS");
1376 return;
1377 }
1378
1379 if (pread(fd, &table, sizeof(table),
1380 vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
1381 error_setg_errno(errp, errno, "failed to read PCI MSIX TABLE");
1382 return;
1383 }
1384
1385 if (pread(fd, &pba, sizeof(pba),
1386 vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
1387 error_setg_errno(errp, errno, "failed to read PCI MSIX PBA");
1388 return;
1389 }
1390
1391 ctrl = le16_to_cpu(ctrl);
1392 table = le32_to_cpu(table);
1393 pba = le32_to_cpu(pba);
1394
1395 msix = g_malloc0(sizeof(*msix));
1396 msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
1397 msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
1398 msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
1399 msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
1400 msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
1401
1402 /*
1403 * Test the size of the pba_offset variable and catch if it extends outside
1404 * of the specified BAR. If it is the case, we need to apply a hardware
1405 * specific quirk if the device is known or we have a broken configuration.
1406 */
1407 if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
1408 /*
1409 * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
1410 * adapters. The T5 hardware returns an incorrect value of 0x8000 for
1411 * the VF PBA offset while the BAR itself is only 8k. The correct value
1412 * is 0x1000, so we hard code that here.
1413 */
1414 if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
1415 (vdev->device_id & 0xff00) == 0x5800) {
1416 msix->pba_offset = 0x1000;
1417 } else {
1418 error_setg(errp, "hardware reports invalid configuration, "
1419 "MSIX PBA outside of specified BAR");
1420 g_free(msix);
1421 return;
1422 }
1423 }
1424
1425 trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
1426 msix->table_offset, msix->entries);
1427 vdev->msix = msix;
1428
1429 vfio_pci_fixup_msix_region(vdev);
1430 }
1431
1432 static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1433 {
1434 int ret;
1435
1436 vdev->msix->pending = g_malloc0(BITS_TO_LONGS(vdev->msix->entries) *
1437 sizeof(unsigned long));
1438 ret = msix_init(&vdev->pdev, vdev->msix->entries,
1439 vdev->bars[vdev->msix->table_bar].region.mem,
1440 vdev->msix->table_bar, vdev->msix->table_offset,
1441 vdev->bars[vdev->msix->pba_bar].region.mem,
1442 vdev->msix->pba_bar, vdev->msix->pba_offset, pos);
1443 if (ret < 0) {
1444 if (ret == -ENOTSUP) {
1445 return 0;
1446 }
1447 error_setg(errp, "msix_init failed");
1448 return ret;
1449 }
1450
1451 /*
1452 * The PCI spec suggests that devices provide additional alignment for
1453 * MSI-X structures and avoid overlapping non-MSI-X related registers.
1454 * For an assigned device, this hopefully means that emulation of MSI-X
1455 * structures does not affect the performance of the device. If devices
1456 * fail to provide that alignment, a significant performance penalty may
1457 * result, for instance Mellanox MT27500 VFs:
1458 * http://www.spinics.net/lists/kvm/msg125881.html
1459 *
1460 * The PBA is simply not that important for such a serious regression and
1461 * most drivers do not appear to look at it. The solution for this is to
1462 * disable the PBA MemoryRegion unless it's being used. We disable it
1463 * here and only enable it if a masked vector fires through QEMU. As the
1464 * vector-use notifier is called, which occurs on unmask, we test whether
1465 * PBA emulation is needed and again disable if not.
1466 */
1467 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
1468
1469 return 0;
1470 }
1471
1472 static void vfio_teardown_msi(VFIOPCIDevice *vdev)
1473 {
1474 msi_uninit(&vdev->pdev);
1475
1476 if (vdev->msix) {
1477 msix_uninit(&vdev->pdev,
1478 vdev->bars[vdev->msix->table_bar].region.mem,
1479 vdev->bars[vdev->msix->pba_bar].region.mem);
1480 g_free(vdev->msix->pending);
1481 }
1482 }
1483
1484 /*
1485 * Resource setup
1486 */
1487 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
1488 {
1489 int i;
1490
1491 for (i = 0; i < PCI_ROM_SLOT; i++) {
1492 vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled);
1493 }
1494 }
1495
1496 static void vfio_bar_setup(VFIOPCIDevice *vdev, int nr)
1497 {
1498 VFIOBAR *bar = &vdev->bars[nr];
1499
1500 uint32_t pci_bar;
1501 uint8_t type;
1502 int ret;
1503
1504 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
1505 if (!bar->region.size) {
1506 return;
1507 }
1508
1509 /* Determine what type of BAR this is for registration */
1510 ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar),
1511 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
1512 if (ret != sizeof(pci_bar)) {
1513 error_report("vfio: Failed to read BAR %d (%m)", nr);
1514 return;
1515 }
1516
1517 pci_bar = le32_to_cpu(pci_bar);
1518 bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
1519 bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
1520 type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
1521 ~PCI_BASE_ADDRESS_MEM_MASK);
1522
1523 if (vfio_region_mmap(&bar->region)) {
1524 error_report("Failed to mmap %s BAR %d. Performance may be slow",
1525 vdev->vbasedev.name, nr);
1526 }
1527
1528 pci_register_bar(&vdev->pdev, nr, type, bar->region.mem);
1529 }
1530
1531 static void vfio_bars_setup(VFIOPCIDevice *vdev)
1532 {
1533 int i;
1534
1535 for (i = 0; i < PCI_ROM_SLOT; i++) {
1536 vfio_bar_setup(vdev, i);
1537 }
1538 }
1539
1540 static void vfio_bars_exit(VFIOPCIDevice *vdev)
1541 {
1542 int i;
1543
1544 for (i = 0; i < PCI_ROM_SLOT; i++) {
1545 vfio_bar_quirk_exit(vdev, i);
1546 vfio_region_exit(&vdev->bars[i].region);
1547 }
1548
1549 if (vdev->vga) {
1550 pci_unregister_vga(&vdev->pdev);
1551 vfio_vga_quirk_exit(vdev);
1552 }
1553 }
1554
1555 static void vfio_bars_finalize(VFIOPCIDevice *vdev)
1556 {
1557 int i;
1558
1559 for (i = 0; i < PCI_ROM_SLOT; i++) {
1560 vfio_bar_quirk_finalize(vdev, i);
1561 vfio_region_finalize(&vdev->bars[i].region);
1562 }
1563
1564 if (vdev->vga) {
1565 vfio_vga_quirk_finalize(vdev);
1566 for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
1567 object_unparent(OBJECT(&vdev->vga->region[i].mem));
1568 }
1569 g_free(vdev->vga);
1570 }
1571 }
1572
1573 /*
1574 * General setup
1575 */
1576 static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
1577 {
1578 uint8_t tmp;
1579 uint16_t next = PCI_CONFIG_SPACE_SIZE;
1580
1581 for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
1582 tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) {
1583 if (tmp > pos && tmp < next) {
1584 next = tmp;
1585 }
1586 }
1587
1588 return next - pos;
1589 }
1590
1591
1592 static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
1593 {
1594 uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
1595
1596 for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
1597 tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
1598 if (tmp > pos && tmp < next) {
1599 next = tmp;
1600 }
1601 }
1602
1603 return next - pos;
1604 }
1605
1606 static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
1607 {
1608 pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
1609 }
1610
1611 static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos,
1612 uint16_t val, uint16_t mask)
1613 {
1614 vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
1615 vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
1616 vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
1617 }
1618
1619 static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
1620 {
1621 pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
1622 }
1623
1624 static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos,
1625 uint32_t val, uint32_t mask)
1626 {
1627 vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
1628 vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
1629 vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
1630 }
1631
1632 static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size,
1633 Error **errp)
1634 {
1635 uint16_t flags;
1636 uint8_t type;
1637
1638 flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
1639 type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
1640
1641 if (type != PCI_EXP_TYPE_ENDPOINT &&
1642 type != PCI_EXP_TYPE_LEG_END &&
1643 type != PCI_EXP_TYPE_RC_END) {
1644
1645 error_setg(errp, "assignment of PCIe type 0x%x "
1646 "devices is not currently supported", type);
1647 return -EINVAL;
1648 }
1649
1650 if (!pci_bus_is_express(vdev->pdev.bus)) {
1651 PCIBus *bus = vdev->pdev.bus;
1652 PCIDevice *bridge;
1653
1654 /*
1655 * Traditionally PCI device assignment exposes the PCIe capability
1656 * as-is on non-express buses. The reason being that some drivers
1657 * simply assume that it's there, for example tg3. However when
1658 * we're running on a native PCIe machine type, like Q35, we need
1659 * to hide the PCIe capability. The reason for this is twofold;
1660 * first Windows guests get a Code 10 error when the PCIe capability
1661 * is exposed in this configuration. Therefore express devices won't
1662 * work at all unless they're attached to express buses in the VM.
1663 * Second, a native PCIe machine introduces the possibility of fine
1664 * granularity IOMMUs supporting both translation and isolation.
1665 * Guest code to discover the IOMMU visibility of a device, such as
1666 * IOMMU grouping code on Linux, is very aware of device types and
1667 * valid transitions between bus types. An express device on a non-
1668 * express bus is not a valid combination on bare metal systems.
1669 *
1670 * Drivers that require a PCIe capability to make the device
1671 * functional are simply going to need to have their devices placed
1672 * on a PCIe bus in the VM.
1673 */
1674 while (!pci_bus_is_root(bus)) {
1675 bridge = pci_bridge_get_device(bus);
1676 bus = bridge->bus;
1677 }
1678
1679 if (pci_bus_is_express(bus)) {
1680 return 0;
1681 }
1682
1683 } else if (pci_bus_is_root(vdev->pdev.bus)) {
1684 /*
1685 * On a Root Complex bus Endpoints become Root Complex Integrated
1686 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
1687 */
1688 if (type == PCI_EXP_TYPE_ENDPOINT) {
1689 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1690 PCI_EXP_TYPE_RC_END << 4,
1691 PCI_EXP_FLAGS_TYPE);
1692
1693 /* Link Capabilities, Status, and Control goes away */
1694 if (size > PCI_EXP_LNKCTL) {
1695 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
1696 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1697 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
1698
1699 #ifndef PCI_EXP_LNKCAP2
1700 #define PCI_EXP_LNKCAP2 44
1701 #endif
1702 #ifndef PCI_EXP_LNKSTA2
1703 #define PCI_EXP_LNKSTA2 50
1704 #endif
1705 /* Link 2 Capabilities, Status, and Control goes away */
1706 if (size > PCI_EXP_LNKCAP2) {
1707 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
1708 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
1709 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
1710 }
1711 }
1712
1713 } else if (type == PCI_EXP_TYPE_LEG_END) {
1714 /*
1715 * Legacy endpoints don't belong on the root complex. Windows
1716 * seems to be happier with devices if we skip the capability.
1717 */
1718 return 0;
1719 }
1720
1721 } else {
1722 /*
1723 * Convert Root Complex Integrated Endpoints to regular endpoints.
1724 * These devices don't support LNK/LNK2 capabilities, so make them up.
1725 */
1726 if (type == PCI_EXP_TYPE_RC_END) {
1727 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1728 PCI_EXP_TYPE_ENDPOINT << 4,
1729 PCI_EXP_FLAGS_TYPE);
1730 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
1731 PCI_EXP_LNK_MLW_1 | PCI_EXP_LNK_LS_25, ~0);
1732 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1733 }
1734
1735 /* Mark the Link Status bits as emulated to allow virtual negotiation */
1736 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA,
1737 pci_get_word(vdev->pdev.config + pos +
1738 PCI_EXP_LNKSTA),
1739 PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS);
1740 }
1741
1742 pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size);
1743 if (pos >= 0) {
1744 vdev->pdev.exp.exp_cap = pos;
1745 }
1746
1747 return pos;
1748 }
1749
1750 static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos)
1751 {
1752 uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
1753
1754 if (cap & PCI_EXP_DEVCAP_FLR) {
1755 trace_vfio_check_pcie_flr(vdev->vbasedev.name);
1756 vdev->has_flr = true;
1757 }
1758 }
1759
1760 static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos)
1761 {
1762 uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
1763
1764 if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
1765 trace_vfio_check_pm_reset(vdev->vbasedev.name);
1766 vdev->has_pm_reset = true;
1767 }
1768 }
1769
1770 static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos)
1771 {
1772 uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
1773
1774 if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
1775 trace_vfio_check_af_flr(vdev->vbasedev.name);
1776 vdev->has_flr = true;
1777 }
1778 }
1779
1780 static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos, Error **errp)
1781 {
1782 PCIDevice *pdev = &vdev->pdev;
1783 uint8_t cap_id, next, size;
1784 int ret;
1785
1786 cap_id = pdev->config[pos];
1787 next = pdev->config[pos + PCI_CAP_LIST_NEXT];
1788
1789 /*
1790 * If it becomes important to configure capabilities to their actual
1791 * size, use this as the default when it's something we don't recognize.
1792 * Since QEMU doesn't actually handle many of the config accesses,
1793 * exact size doesn't seem worthwhile.
1794 */
1795 size = vfio_std_cap_max_size(pdev, pos);
1796
1797 /*
1798 * pci_add_capability always inserts the new capability at the head
1799 * of the chain. Therefore to end up with a chain that matches the
1800 * physical device, we insert from the end by making this recursive.
1801 * This is also why we pre-calculate size above as cached config space
1802 * will be changed as we unwind the stack.
1803 */
1804 if (next) {
1805 ret = vfio_add_std_cap(vdev, next, errp);
1806 if (ret) {
1807 goto out;
1808 }
1809 } else {
1810 /* Begin the rebuild, use QEMU emulated list bits */
1811 pdev->config[PCI_CAPABILITY_LIST] = 0;
1812 vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
1813 vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
1814 }
1815
1816 /* Use emulated next pointer to allow dropping caps */
1817 pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff);
1818
1819 switch (cap_id) {
1820 case PCI_CAP_ID_MSI:
1821 ret = vfio_msi_setup(vdev, pos, errp);
1822 break;
1823 case PCI_CAP_ID_EXP:
1824 vfio_check_pcie_flr(vdev, pos);
1825 ret = vfio_setup_pcie_cap(vdev, pos, size, errp);
1826 break;
1827 case PCI_CAP_ID_MSIX:
1828 ret = vfio_msix_setup(vdev, pos, errp);
1829 break;
1830 case PCI_CAP_ID_PM:
1831 vfio_check_pm_reset(vdev, pos);
1832 vdev->pm_cap = pos;
1833 ret = pci_add_capability2(pdev, cap_id, pos, size, errp);
1834 break;
1835 case PCI_CAP_ID_AF:
1836 vfio_check_af_flr(vdev, pos);
1837 ret = pci_add_capability2(pdev, cap_id, pos, size, errp);
1838 break;
1839 default:
1840 ret = pci_add_capability2(pdev, cap_id, pos, size, errp);
1841 break;
1842 }
1843 out:
1844 if (ret < 0) {
1845 error_prepend(errp,
1846 "failed to add PCI capability 0x%x[0x%x]@0x%x: ",
1847 cap_id, size, pos);
1848 return ret;
1849 }
1850
1851 return 0;
1852 }
1853
1854 static void vfio_add_ext_cap(VFIOPCIDevice *vdev)
1855 {
1856 PCIDevice *pdev = &vdev->pdev;
1857 uint32_t header;
1858 uint16_t cap_id, next, size;
1859 uint8_t cap_ver;
1860 uint8_t *config;
1861
1862 /* Only add extended caps if we have them and the guest can see them */
1863 if (!pci_is_express(pdev) || !pci_bus_is_express(pdev->bus) ||
1864 !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
1865 return;
1866 }
1867
1868 /*
1869 * pcie_add_capability always inserts the new capability at the tail
1870 * of the chain. Therefore to end up with a chain that matches the
1871 * physical device, we cache the config space to avoid overwriting
1872 * the original config space when we parse the extended capabilities.
1873 */
1874 config = g_memdup(pdev->config, vdev->config_size);
1875
1876 /*
1877 * Extended capabilities are chained with each pointing to the next, so we
1878 * can drop anything other than the head of the chain simply by modifying
1879 * the previous next pointer. For the head of the chain, we can modify the
1880 * capability ID to something that cannot match a valid capability. ID
1881 * 0 is reserved for this since absence of capabilities is indicated by
1882 * 0 for the ID, version, AND next pointer. However, pcie_add_capability()
1883 * uses ID 0 as reserved for list management and will incorrectly match and
1884 * assert if we attempt to pre-load the head of the chain with with this
1885 * ID. Use ID 0xFFFF temporarily since it is also seems to be reserved in
1886 * part for identifying absence of capabilities in a root complex register
1887 * block. If the ID still exists after adding capabilities, switch back to
1888 * zero. We'll mark this entire first dword as emulated for this purpose.
1889 */
1890 pci_set_long(pdev->config + PCI_CONFIG_SPACE_SIZE,
1891 PCI_EXT_CAP(0xFFFF, 0, 0));
1892 pci_set_long(pdev->wmask + PCI_CONFIG_SPACE_SIZE, 0);
1893 pci_set_long(vdev->emulated_config_bits + PCI_CONFIG_SPACE_SIZE, ~0);
1894
1895 for (next = PCI_CONFIG_SPACE_SIZE; next;
1896 next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
1897 header = pci_get_long(config + next);
1898 cap_id = PCI_EXT_CAP_ID(header);
1899 cap_ver = PCI_EXT_CAP_VER(header);
1900
1901 /*
1902 * If it becomes important to configure extended capabilities to their
1903 * actual size, use this as the default when it's something we don't
1904 * recognize. Since QEMU doesn't actually handle many of the config
1905 * accesses, exact size doesn't seem worthwhile.
1906 */
1907 size = vfio_ext_cap_max_size(config, next);
1908
1909 /* Use emulated next pointer to allow dropping extended caps */
1910 pci_long_test_and_set_mask(vdev->emulated_config_bits + next,
1911 PCI_EXT_CAP_NEXT_MASK);
1912
1913 switch (cap_id) {
1914 case PCI_EXT_CAP_ID_SRIOV: /* Read-only VF BARs confuse OVMF */
1915 case PCI_EXT_CAP_ID_ARI: /* XXX Needs next function virtualization */
1916 trace_vfio_add_ext_cap_dropped(vdev->vbasedev.name, cap_id, next);
1917 break;
1918 default:
1919 pcie_add_capability(pdev, cap_id, cap_ver, next, size);
1920 }
1921
1922 }
1923
1924 /* Cleanup chain head ID if necessary */
1925 if (pci_get_word(pdev->config + PCI_CONFIG_SPACE_SIZE) == 0xFFFF) {
1926 pci_set_word(pdev->config + PCI_CONFIG_SPACE_SIZE, 0);
1927 }
1928
1929 g_free(config);
1930 return;
1931 }
1932
1933 static int vfio_add_capabilities(VFIOPCIDevice *vdev, Error **errp)
1934 {
1935 PCIDevice *pdev = &vdev->pdev;
1936 int ret;
1937
1938 if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
1939 !pdev->config[PCI_CAPABILITY_LIST]) {
1940 return 0; /* Nothing to add */
1941 }
1942
1943 ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST], errp);
1944 if (ret) {
1945 return ret;
1946 }
1947
1948 vfio_add_ext_cap(vdev);
1949 return 0;
1950 }
1951
1952 static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
1953 {
1954 PCIDevice *pdev = &vdev->pdev;
1955 uint16_t cmd;
1956
1957 vfio_disable_interrupts(vdev);
1958
1959 /* Make sure the device is in D0 */
1960 if (vdev->pm_cap) {
1961 uint16_t pmcsr;
1962 uint8_t state;
1963
1964 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
1965 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1966 if (state) {
1967 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1968 vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
1969 /* vfio handles the necessary delay here */
1970 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
1971 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1972 if (state) {
1973 error_report("vfio: Unable to power on device, stuck in D%d",
1974 state);
1975 }
1976 }
1977 }
1978
1979 /*
1980 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
1981 * Also put INTx Disable in known state.
1982 */
1983 cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
1984 cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
1985 PCI_COMMAND_INTX_DISABLE);
1986 vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
1987 }
1988
1989 static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
1990 {
1991 Error *err = NULL;
1992 int nr;
1993
1994 vfio_intx_enable(vdev, &err);
1995 if (err) {
1996 error_reportf_err(err, ERR_PREFIX, vdev->vbasedev.name);
1997 }
1998
1999 for (nr = 0; nr < PCI_NUM_REGIONS - 1; ++nr) {
2000 off_t addr = vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr);
2001 uint32_t val = 0;
2002 uint32_t len = sizeof(val);
2003
2004 if (pwrite(vdev->vbasedev.fd, &val, len, addr) != len) {
2005 error_report("%s(%s) reset bar %d failed: %m", __func__,
2006 vdev->vbasedev.name, nr);
2007 }
2008 }
2009 }
2010
2011 static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
2012 {
2013 char tmp[13];
2014
2015 sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain,
2016 addr->bus, addr->slot, addr->function);
2017
2018 return (strcmp(tmp, name) == 0);
2019 }
2020
2021 static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
2022 {
2023 VFIOGroup *group;
2024 struct vfio_pci_hot_reset_info *info;
2025 struct vfio_pci_dependent_device *devices;
2026 struct vfio_pci_hot_reset *reset;
2027 int32_t *fds;
2028 int ret, i, count;
2029 bool multi = false;
2030
2031 trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
2032
2033 if (!single) {
2034 vfio_pci_pre_reset(vdev);
2035 }
2036 vdev->vbasedev.needs_reset = false;
2037
2038 info = g_malloc0(sizeof(*info));
2039 info->argsz = sizeof(*info);
2040
2041 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2042 if (ret && errno != ENOSPC) {
2043 ret = -errno;
2044 if (!vdev->has_pm_reset) {
2045 error_report("vfio: Cannot reset device %s, "
2046 "no available reset mechanism.", vdev->vbasedev.name);
2047 }
2048 goto out_single;
2049 }
2050
2051 count = info->count;
2052 info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
2053 info->argsz = sizeof(*info) + (count * sizeof(*devices));
2054 devices = &info->devices[0];
2055
2056 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2057 if (ret) {
2058 ret = -errno;
2059 error_report("vfio: hot reset info failed: %m");
2060 goto out_single;
2061 }
2062
2063 trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
2064
2065 /* Verify that we have all the groups required */
2066 for (i = 0; i < info->count; i++) {
2067 PCIHostDeviceAddress host;
2068 VFIOPCIDevice *tmp;
2069 VFIODevice *vbasedev_iter;
2070
2071 host.domain = devices[i].segment;
2072 host.bus = devices[i].bus;
2073 host.slot = PCI_SLOT(devices[i].devfn);
2074 host.function = PCI_FUNC(devices[i].devfn);
2075
2076 trace_vfio_pci_hot_reset_dep_devices(host.domain,
2077 host.bus, host.slot, host.function, devices[i].group_id);
2078
2079 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2080 continue;
2081 }
2082
2083 QLIST_FOREACH(group, &vfio_group_list, next) {
2084 if (group->groupid == devices[i].group_id) {
2085 break;
2086 }
2087 }
2088
2089 if (!group) {
2090 if (!vdev->has_pm_reset) {
2091 error_report("vfio: Cannot reset device %s, "
2092 "depends on group %d which is not owned.",
2093 vdev->vbasedev.name, devices[i].group_id);
2094 }
2095 ret = -EPERM;
2096 goto out;
2097 }
2098
2099 /* Prep dependent devices for reset and clear our marker. */
2100 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2101 if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2102 continue;
2103 }
2104 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2105 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2106 if (single) {
2107 ret = -EINVAL;
2108 goto out_single;
2109 }
2110 vfio_pci_pre_reset(tmp);
2111 tmp->vbasedev.needs_reset = false;
2112 multi = true;
2113 break;
2114 }
2115 }
2116 }
2117
2118 if (!single && !multi) {
2119 ret = -EINVAL;
2120 goto out_single;
2121 }
2122
2123 /* Determine how many group fds need to be passed */
2124 count = 0;
2125 QLIST_FOREACH(group, &vfio_group_list, next) {
2126 for (i = 0; i < info->count; i++) {
2127 if (group->groupid == devices[i].group_id) {
2128 count++;
2129 break;
2130 }
2131 }
2132 }
2133
2134 reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
2135 reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
2136 fds = &reset->group_fds[0];
2137
2138 /* Fill in group fds */
2139 QLIST_FOREACH(group, &vfio_group_list, next) {
2140 for (i = 0; i < info->count; i++) {
2141 if (group->groupid == devices[i].group_id) {
2142 fds[reset->count++] = group->fd;
2143 break;
2144 }
2145 }
2146 }
2147
2148 /* Bus reset! */
2149 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
2150 g_free(reset);
2151
2152 trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
2153 ret ? "%m" : "Success");
2154
2155 out:
2156 /* Re-enable INTx on affected devices */
2157 for (i = 0; i < info->count; i++) {
2158 PCIHostDeviceAddress host;
2159 VFIOPCIDevice *tmp;
2160 VFIODevice *vbasedev_iter;
2161
2162 host.domain = devices[i].segment;
2163 host.bus = devices[i].bus;
2164 host.slot = PCI_SLOT(devices[i].devfn);
2165 host.function = PCI_FUNC(devices[i].devfn);
2166
2167 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2168 continue;
2169 }
2170
2171 QLIST_FOREACH(group, &vfio_group_list, next) {
2172 if (group->groupid == devices[i].group_id) {
2173 break;
2174 }
2175 }
2176
2177 if (!group) {
2178 break;
2179 }
2180
2181 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2182 if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2183 continue;
2184 }
2185 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2186 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2187 vfio_pci_post_reset(tmp);
2188 break;
2189 }
2190 }
2191 }
2192 out_single:
2193 if (!single) {
2194 vfio_pci_post_reset(vdev);
2195 }
2196 g_free(info);
2197
2198 return ret;
2199 }
2200
2201 /*
2202 * We want to differentiate hot reset of mulitple in-use devices vs hot reset
2203 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case
2204 * of doing hot resets when there is only a single device per bus. The in-use
2205 * here refers to how many VFIODevices are affected. A hot reset that affects
2206 * multiple devices, but only a single in-use device, means that we can call
2207 * it from our bus ->reset() callback since the extent is effectively a single
2208 * device. This allows us to make use of it in the hotplug path. When there
2209 * are multiple in-use devices, we can only trigger the hot reset during a
2210 * system reset and thus from our reset handler. We separate _one vs _multi
2211 * here so that we don't overlap and do a double reset on the system reset
2212 * path where both our reset handler and ->reset() callback are used. Calling
2213 * _one() will only do a hot reset for the one in-use devices case, calling
2214 * _multi() will do nothing if a _one() would have been sufficient.
2215 */
2216 static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev)
2217 {
2218 return vfio_pci_hot_reset(vdev, true);
2219 }
2220
2221 static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev)
2222 {
2223 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2224 return vfio_pci_hot_reset(vdev, false);
2225 }
2226
2227 static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
2228 {
2229 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2230 if (!vbasedev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
2231 vbasedev->needs_reset = true;
2232 }
2233 }
2234
2235 static VFIODeviceOps vfio_pci_ops = {
2236 .vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
2237 .vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
2238 .vfio_eoi = vfio_intx_eoi,
2239 };
2240
2241 int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp)
2242 {
2243 VFIODevice *vbasedev = &vdev->vbasedev;
2244 struct vfio_region_info *reg_info;
2245 int ret;
2246
2247 ret = vfio_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, &reg_info);
2248 if (ret) {
2249 error_setg_errno(errp, -ret,
2250 "failed getting region info for VGA region index %d",
2251 VFIO_PCI_VGA_REGION_INDEX);
2252 return ret;
2253 }
2254
2255 if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) ||
2256 !(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) ||
2257 reg_info->size < 0xbffff + 1) {
2258 error_setg(errp, "unexpected VGA info, flags 0x%lx, size 0x%lx",
2259 (unsigned long)reg_info->flags,
2260 (unsigned long)reg_info->size);
2261 g_free(reg_info);
2262 return -EINVAL;
2263 }
2264
2265 vdev->vga = g_new0(VFIOVGA, 1);
2266
2267 vdev->vga->fd_offset = reg_info->offset;
2268 vdev->vga->fd = vdev->vbasedev.fd;
2269
2270 g_free(reg_info);
2271
2272 vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
2273 vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
2274 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks);
2275
2276 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2277 OBJECT(vdev), &vfio_vga_ops,
2278 &vdev->vga->region[QEMU_PCI_VGA_MEM],
2279 "vfio-vga-mmio@0xa0000",
2280 QEMU_PCI_VGA_MEM_SIZE);
2281
2282 vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
2283 vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
2284 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks);
2285
2286 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2287 OBJECT(vdev), &vfio_vga_ops,
2288 &vdev->vga->region[QEMU_PCI_VGA_IO_LO],
2289 "vfio-vga-io@0x3b0",
2290 QEMU_PCI_VGA_IO_LO_SIZE);
2291
2292 vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
2293 vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
2294 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks);
2295
2296 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
2297 OBJECT(vdev), &vfio_vga_ops,
2298 &vdev->vga->region[QEMU_PCI_VGA_IO_HI],
2299 "vfio-vga-io@0x3c0",
2300 QEMU_PCI_VGA_IO_HI_SIZE);
2301
2302 pci_register_vga(&vdev->pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2303 &vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2304 &vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem);
2305
2306 return 0;
2307 }
2308
2309 static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
2310 {
2311 VFIODevice *vbasedev = &vdev->vbasedev;
2312 struct vfio_region_info *reg_info;
2313 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
2314 int i, ret = -1;
2315
2316 /* Sanity check device */
2317 if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) {
2318 error_setg(errp, "this isn't a PCI device");
2319 return;
2320 }
2321
2322 if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
2323 error_setg(errp, "unexpected number of io regions %u",
2324 vbasedev->num_regions);
2325 return;
2326 }
2327
2328 if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
2329 error_setg(errp, "unexpected number of irqs %u", vbasedev->num_irqs);
2330 return;
2331 }
2332
2333 for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
2334 char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
2335
2336 ret = vfio_region_setup(OBJECT(vdev), vbasedev,
2337 &vdev->bars[i].region, i, name);
2338 g_free(name);
2339
2340 if (ret) {
2341 error_setg_errno(errp, -ret, "failed to get region %d info", i);
2342 return;
2343 }
2344
2345 QLIST_INIT(&vdev->bars[i].quirks);
2346 }
2347
2348 ret = vfio_get_region_info(vbasedev,
2349 VFIO_PCI_CONFIG_REGION_INDEX, &reg_info);
2350 if (ret) {
2351 error_setg_errno(errp, -ret, "failed to get config info");
2352 return;
2353 }
2354
2355 trace_vfio_populate_device_config(vdev->vbasedev.name,
2356 (unsigned long)reg_info->size,
2357 (unsigned long)reg_info->offset,
2358 (unsigned long)reg_info->flags);
2359
2360 vdev->config_size = reg_info->size;
2361 if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
2362 vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
2363 }
2364 vdev->config_offset = reg_info->offset;
2365
2366 g_free(reg_info);
2367
2368 if (vdev->features & VFIO_FEATURE_ENABLE_VGA) {
2369 ret = vfio_populate_vga(vdev, errp);
2370 if (ret) {
2371 error_append_hint(errp, "device does not support "
2372 "requested feature x-vga\n");
2373 return;
2374 }
2375 }
2376
2377 irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
2378
2379 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
2380 if (ret) {
2381 /* This can fail for an old kernel or legacy PCI dev */
2382 trace_vfio_populate_device_get_irq_info_failure();
2383 } else if (irq_info.count == 1) {
2384 vdev->pci_aer = true;
2385 } else {
2386 error_report(WARN_PREFIX
2387 "Could not enable error recovery for the device",
2388 vbasedev->name);
2389 }
2390 }
2391
2392 static void vfio_put_device(VFIOPCIDevice *vdev)
2393 {
2394 g_free(vdev->vbasedev.name);
2395 g_free(vdev->msix);
2396
2397 vfio_put_base_device(&vdev->vbasedev);
2398 }
2399
2400 static void vfio_err_notifier_handler(void *opaque)
2401 {
2402 VFIOPCIDevice *vdev = opaque;
2403
2404 if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
2405 return;
2406 }
2407
2408 /*
2409 * TBD. Retrieve the error details and decide what action
2410 * needs to be taken. One of the actions could be to pass
2411 * the error to the guest and have the guest driver recover
2412 * from the error. This requires that PCIe capabilities be
2413 * exposed to the guest. For now, we just terminate the
2414 * guest to contain the error.
2415 */
2416
2417 error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name);
2418
2419 vm_stop(RUN_STATE_INTERNAL_ERROR);
2420 }
2421
2422 /*
2423 * Registers error notifier for devices supporting error recovery.
2424 * If we encounter a failure in this function, we report an error
2425 * and continue after disabling error recovery support for the
2426 * device.
2427 */
2428 static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
2429 {
2430 int ret;
2431 int argsz;
2432 struct vfio_irq_set *irq_set;
2433 int32_t *pfd;
2434
2435 if (!vdev->pci_aer) {
2436 return;
2437 }
2438
2439 if (event_notifier_init(&vdev->err_notifier, 0)) {
2440 error_report("vfio: Unable to init event notifier for error detection");
2441 vdev->pci_aer = false;
2442 return;
2443 }
2444
2445 argsz = sizeof(*irq_set) + sizeof(*pfd);
2446
2447 irq_set = g_malloc0(argsz);
2448 irq_set->argsz = argsz;
2449 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
2450 VFIO_IRQ_SET_ACTION_TRIGGER;
2451 irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
2452 irq_set->start = 0;
2453 irq_set->count = 1;
2454 pfd = (int32_t *)&irq_set->data;
2455
2456 *pfd = event_notifier_get_fd(&vdev->err_notifier);
2457 qemu_set_fd_handler(*pfd, vfio_err_notifier_handler, NULL, vdev);
2458
2459 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
2460 if (ret) {
2461 error_report("vfio: Failed to set up error notification");
2462 qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
2463 event_notifier_cleanup(&vdev->err_notifier);
2464 vdev->pci_aer = false;
2465 }
2466 g_free(irq_set);
2467 }
2468
2469 static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev)
2470 {
2471 int argsz;
2472 struct vfio_irq_set *irq_set;
2473 int32_t *pfd;
2474 int ret;
2475
2476 if (!vdev->pci_aer) {
2477 return;
2478 }
2479
2480 argsz = sizeof(*irq_set) + sizeof(*pfd);
2481
2482 irq_set = g_malloc0(argsz);
2483 irq_set->argsz = argsz;
2484 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
2485 VFIO_IRQ_SET_ACTION_TRIGGER;
2486 irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
2487 irq_set->start = 0;
2488 irq_set->count = 1;
2489 pfd = (int32_t *)&irq_set->data;
2490 *pfd = -1;
2491
2492 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
2493 if (ret) {
2494 error_report("vfio: Failed to de-assign error fd: %m");
2495 }
2496 g_free(irq_set);
2497 qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
2498 NULL, NULL, vdev);
2499 event_notifier_cleanup(&vdev->err_notifier);
2500 }
2501
2502 static void vfio_req_notifier_handler(void *opaque)
2503 {
2504 VFIOPCIDevice *vdev = opaque;
2505
2506 if (!event_notifier_test_and_clear(&vdev->req_notifier)) {
2507 return;
2508 }
2509
2510 qdev_unplug(&vdev->pdev.qdev, NULL);
2511 }
2512
2513 static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
2514 {
2515 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
2516 .index = VFIO_PCI_REQ_IRQ_INDEX };
2517 int argsz;
2518 struct vfio_irq_set *irq_set;
2519 int32_t *pfd;
2520
2521 if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) {
2522 return;
2523 }
2524
2525 if (ioctl(vdev->vbasedev.fd,
2526 VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) {
2527 return;
2528 }
2529
2530 if (event_notifier_init(&vdev->req_notifier, 0)) {
2531 error_report("vfio: Unable to init event notifier for device request");
2532 return;
2533 }
2534
2535 argsz = sizeof(*irq_set) + sizeof(*pfd);
2536
2537 irq_set = g_malloc0(argsz);
2538 irq_set->argsz = argsz;
2539 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
2540 VFIO_IRQ_SET_ACTION_TRIGGER;
2541 irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
2542 irq_set->start = 0;
2543 irq_set->count = 1;
2544 pfd = (int32_t *)&irq_set->data;
2545
2546 *pfd = event_notifier_get_fd(&vdev->req_notifier);
2547 qemu_set_fd_handler(*pfd, vfio_req_notifier_handler, NULL, vdev);
2548
2549 if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set)) {
2550 error_report("vfio: Failed to set up device request notification");
2551 qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
2552 event_notifier_cleanup(&vdev->req_notifier);
2553 } else {
2554 vdev->req_enabled = true;
2555 }
2556
2557 g_free(irq_set);
2558 }
2559
2560 static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
2561 {
2562 int argsz;
2563 struct vfio_irq_set *irq_set;
2564 int32_t *pfd;
2565
2566 if (!vdev->req_enabled) {
2567 return;
2568 }
2569
2570 argsz = sizeof(*irq_set) + sizeof(*pfd);
2571
2572 irq_set = g_malloc0(argsz);
2573 irq_set->argsz = argsz;
2574 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
2575 VFIO_IRQ_SET_ACTION_TRIGGER;
2576 irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
2577 irq_set->start = 0;
2578 irq_set->count = 1;
2579 pfd = (int32_t *)&irq_set->data;
2580 *pfd = -1;
2581
2582 if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set)) {
2583 error_report("vfio: Failed to de-assign device request fd: %m");
2584 }
2585 g_free(irq_set);
2586 qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier),
2587 NULL, NULL, vdev);
2588 event_notifier_cleanup(&vdev->req_notifier);
2589
2590 vdev->req_enabled = false;
2591 }
2592
2593 static void vfio_realize(PCIDevice *pdev, Error **errp)
2594 {
2595 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
2596 VFIODevice *vbasedev_iter;
2597 VFIOGroup *group;
2598 char *tmp, group_path[PATH_MAX], *group_name;
2599 Error *err = NULL;
2600 ssize_t len;
2601 struct stat st;
2602 int groupid;
2603 int i, ret;
2604
2605 if (!vdev->vbasedev.sysfsdev) {
2606 if (!(~vdev->host.domain || ~vdev->host.bus ||
2607 ~vdev->host.slot || ~vdev->host.function)) {
2608 error_setg(errp, "No provided host device");
2609 error_append_hint(errp, "Use -vfio-pci,host=DDDD:BB:DD.F "
2610 "or -vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
2611 return;
2612 }
2613 vdev->vbasedev.sysfsdev =
2614 g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
2615 vdev->host.domain, vdev->host.bus,
2616 vdev->host.slot, vdev->host.function);
2617 }
2618
2619 if (stat(vdev->vbasedev.sysfsdev, &st) < 0) {
2620 error_setg_errno(errp, errno, "no such host device");
2621 error_prepend(errp, ERR_PREFIX, vdev->vbasedev.sysfsdev);
2622 return;
2623 }
2624
2625 vdev->vbasedev.name = g_strdup(basename(vdev->vbasedev.sysfsdev));
2626 vdev->vbasedev.ops = &vfio_pci_ops;
2627 vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI;
2628
2629 tmp = g_strdup_printf("%s/iommu_group", vdev->vbasedev.sysfsdev);
2630 len = readlink(tmp, group_path, sizeof(group_path));
2631 g_free(tmp);
2632
2633 if (len <= 0 || len >= sizeof(group_path)) {
2634 error_setg_errno(errp, len < 0 ? errno : ENAMETOOLONG,
2635 "no iommu_group found");
2636 goto error;
2637 }
2638
2639 group_path[len] = 0;
2640
2641 group_name = basename(group_path);
2642 if (sscanf(group_name, "%d", &groupid) != 1) {
2643 error_setg_errno(errp, errno, "failed to read %s", group_path);
2644 goto error;
2645 }
2646
2647 trace_vfio_realize(vdev->vbasedev.name, groupid);
2648
2649 group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev), errp);
2650 if (!group) {
2651 goto error;
2652 }
2653
2654 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2655 if (strcmp(vbasedev_iter->name, vdev->vbasedev.name) == 0) {
2656 error_setg(errp, "device is already attached");
2657 vfio_put_group(group);
2658 goto error;
2659 }
2660 }
2661
2662 ret = vfio_get_device(group, vdev->vbasedev.name, &vdev->vbasedev, errp);
2663 if (ret) {
2664 vfio_put_group(group);
2665 goto error;
2666 }
2667
2668 vfio_populate_device(vdev, &err);
2669 if (err) {
2670 error_propagate(errp, err);
2671 goto error;
2672 }
2673
2674 /* Get a copy of config space */
2675 ret = pread(vdev->vbasedev.fd, vdev->pdev.config,
2676 MIN(pci_config_size(&vdev->pdev), vdev->config_size),
2677 vdev->config_offset);
2678 if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
2679 ret = ret < 0 ? -errno : -EFAULT;
2680 error_setg_errno(errp, -ret, "failed to read device config space");
2681 goto error;
2682 }
2683
2684 /* vfio emulates a lot for us, but some bits need extra love */
2685 vdev->emulated_config_bits = g_malloc0(vdev->config_size);
2686
2687 /* QEMU can choose to expose the ROM or not */
2688 memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
2689
2690 /*
2691 * The PCI spec reserves vendor ID 0xffff as an invalid value. The
2692 * device ID is managed by the vendor and need only be a 16-bit value.
2693 * Allow any 16-bit value for subsystem so they can be hidden or changed.
2694 */
2695 if (vdev->vendor_id != PCI_ANY_ID) {
2696 if (vdev->vendor_id >= 0xffff) {
2697 error_setg(errp, "invalid PCI vendor ID provided");
2698 goto error;
2699 }
2700 vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
2701 trace_vfio_pci_emulated_vendor_id(vdev->vbasedev.name, vdev->vendor_id);
2702 } else {
2703 vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
2704 }
2705
2706 if (vdev->device_id != PCI_ANY_ID) {
2707 if (vdev->device_id > 0xffff) {
2708 error_setg(errp, "invalid PCI device ID provided");
2709 goto error;
2710 }
2711 vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
2712 trace_vfio_pci_emulated_device_id(vdev->vbasedev.name, vdev->device_id);
2713 } else {
2714 vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
2715 }
2716
2717 if (vdev->sub_vendor_id != PCI_ANY_ID) {
2718 if (vdev->sub_vendor_id > 0xffff) {
2719 error_setg(errp, "invalid PCI subsystem vendor ID provided");
2720 goto error;
2721 }
2722 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
2723 vdev->sub_vendor_id, ~0);
2724 trace_vfio_pci_emulated_sub_vendor_id(vdev->vbasedev.name,
2725 vdev->sub_vendor_id);
2726 }
2727
2728 if (vdev->sub_device_id != PCI_ANY_ID) {
2729 if (vdev->sub_device_id > 0xffff) {
2730 error_setg(errp, "invalid PCI subsystem device ID provided");
2731 goto error;
2732 }
2733 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
2734 trace_vfio_pci_emulated_sub_device_id(vdev->vbasedev.name,
2735 vdev->sub_device_id);
2736 }
2737
2738 /* QEMU can change multi-function devices to single function, or reverse */
2739 vdev->emulated_config_bits[PCI_HEADER_TYPE] =
2740 PCI_HEADER_TYPE_MULTI_FUNCTION;
2741
2742 /* Restore or clear multifunction, this is always controlled by QEMU */
2743 if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
2744 vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
2745 } else {
2746 vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
2747 }
2748
2749 /*
2750 * Clear host resource mapping info. If we choose not to register a
2751 * BAR, such as might be the case with the option ROM, we can get
2752 * confusing, unwritable, residual addresses from the host here.
2753 */
2754 memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
2755 memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
2756
2757 vfio_pci_size_rom(vdev);
2758
2759 vfio_msix_early_setup(vdev, &err);
2760 if (err) {
2761 error_propagate(errp, err);
2762 goto error;
2763 }
2764
2765 vfio_bars_setup(vdev);
2766
2767 ret = vfio_add_capabilities(vdev, errp);
2768 if (ret) {
2769 goto out_teardown;
2770 }
2771
2772 if (vdev->vga) {
2773 vfio_vga_quirk_setup(vdev);
2774 }
2775
2776 for (i = 0; i < PCI_ROM_SLOT; i++) {
2777 vfio_bar_quirk_setup(vdev, i);
2778 }
2779
2780 if (!vdev->igd_opregion &&
2781 vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) {
2782 struct vfio_region_info *opregion;
2783
2784 if (vdev->pdev.qdev.hotplugged) {
2785 error_setg(errp,
2786 "cannot support IGD OpRegion feature on hotplugged "
2787 "device");
2788 goto out_teardown;
2789 }
2790
2791 ret = vfio_get_dev_region_info(&vdev->vbasedev,
2792 VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
2793 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion);
2794 if (ret) {
2795 error_setg_errno(errp, -ret,
2796 "does not support requested IGD OpRegion feature");
2797 goto out_teardown;
2798 }
2799
2800 ret = vfio_pci_igd_opregion_init(vdev, opregion, errp);
2801 g_free(opregion);
2802 if (ret) {
2803 goto out_teardown;
2804 }
2805 }
2806
2807 /* QEMU emulates all of MSI & MSIX */
2808 if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
2809 memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
2810 MSIX_CAP_LENGTH);
2811 }
2812
2813 if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
2814 memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
2815 vdev->msi_cap_size);
2816 }
2817
2818 if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
2819 vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
2820 vfio_intx_mmap_enable, vdev);
2821 pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_intx_update);
2822 ret = vfio_intx_enable(vdev, errp);
2823 if (ret) {
2824 goto out_teardown;
2825 }
2826 }
2827
2828 vfio_register_err_notifier(vdev);
2829 vfio_register_req_notifier(vdev);
2830 vfio_setup_resetfn_quirk(vdev);
2831
2832 return;
2833
2834 out_teardown:
2835 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
2836 vfio_teardown_msi(vdev);
2837 vfio_bars_exit(vdev);
2838 error:
2839 error_prepend(errp, ERR_PREFIX, vdev->vbasedev.name);
2840 }
2841
2842 static void vfio_instance_finalize(Object *obj)
2843 {
2844 PCIDevice *pci_dev = PCI_DEVICE(obj);
2845 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pci_dev);
2846 VFIOGroup *group = vdev->vbasedev.group;
2847
2848 vfio_bars_finalize(vdev);
2849 g_free(vdev->emulated_config_bits);
2850 g_free(vdev->rom);
2851 /*
2852 * XXX Leaking igd_opregion is not an oversight, we can't remove the
2853 * fw_cfg entry therefore leaking this allocation seems like the safest
2854 * option.
2855 *
2856 * g_free(vdev->igd_opregion);
2857 */
2858 vfio_put_device(vdev);
2859 vfio_put_group(group);
2860 }
2861
2862 static void vfio_exitfn(PCIDevice *pdev)
2863 {
2864 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
2865
2866 vfio_unregister_req_notifier(vdev);
2867 vfio_unregister_err_notifier(vdev);
2868 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
2869 vfio_disable_interrupts(vdev);
2870 if (vdev->intx.mmap_timer) {
2871 timer_free(vdev->intx.mmap_timer);
2872 }
2873 vfio_teardown_msi(vdev);
2874 vfio_bars_exit(vdev);
2875 }
2876
2877 static void vfio_pci_reset(DeviceState *dev)
2878 {
2879 PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev);
2880 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
2881
2882 trace_vfio_pci_reset(vdev->vbasedev.name);
2883
2884 vfio_pci_pre_reset(vdev);
2885
2886 if (vdev->resetfn && !vdev->resetfn(vdev)) {
2887 goto post_reset;
2888 }
2889
2890 if (vdev->vbasedev.reset_works &&
2891 (vdev->has_flr || !vdev->has_pm_reset) &&
2892 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
2893 trace_vfio_pci_reset_flr(vdev->vbasedev.name);
2894 goto post_reset;
2895 }
2896
2897 /* See if we can do our own bus reset */
2898 if (!vfio_pci_hot_reset_one(vdev)) {
2899 goto post_reset;
2900 }
2901
2902 /* If nothing else works and the device supports PM reset, use it */
2903 if (vdev->vbasedev.reset_works && vdev->has_pm_reset &&
2904 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
2905 trace_vfio_pci_reset_pm(vdev->vbasedev.name);
2906 goto post_reset;
2907 }
2908
2909 post_reset:
2910 vfio_pci_post_reset(vdev);
2911 }
2912
2913 static void vfio_instance_init(Object *obj)
2914 {
2915 PCIDevice *pci_dev = PCI_DEVICE(obj);
2916 VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, PCI_DEVICE(obj));
2917
2918 device_add_bootindex_property(obj, &vdev->bootindex,
2919 "bootindex", NULL,
2920 &pci_dev->qdev, NULL);
2921 vdev->host.domain = ~0U;
2922 vdev->host.bus = ~0U;
2923 vdev->host.slot = ~0U;
2924 vdev->host.function = ~0U;
2925 }
2926
2927 static Property vfio_pci_dev_properties[] = {
2928 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
2929 DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
2930 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
2931 intx.mmap_timeout, 1100),
2932 DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features,
2933 VFIO_FEATURE_ENABLE_VGA_BIT, false),
2934 DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
2935 VFIO_FEATURE_ENABLE_REQ_BIT, true),
2936 DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features,
2937 VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false),
2938 DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
2939 DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
2940 DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
2941 DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
2942 DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
2943 DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
2944 DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
2945 sub_vendor_id, PCI_ANY_ID),
2946 DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
2947 sub_device_id, PCI_ANY_ID),
2948 DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice, igd_gms, 0),
2949 /*
2950 * TODO - support passed fds... is this necessary?
2951 * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
2952 * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
2953 */
2954 DEFINE_PROP_END_OF_LIST(),
2955 };
2956
2957 static const VMStateDescription vfio_pci_vmstate = {
2958 .name = "vfio-pci",
2959 .unmigratable = 1,
2960 };
2961
2962 static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
2963 {
2964 DeviceClass *dc = DEVICE_CLASS(klass);
2965 PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
2966
2967 dc->reset = vfio_pci_reset;
2968 dc->props = vfio_pci_dev_properties;
2969 dc->vmsd = &vfio_pci_vmstate;
2970 dc->desc = "VFIO-based PCI device assignment";
2971 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
2972 pdc->realize = vfio_realize;
2973 pdc->exit = vfio_exitfn;
2974 pdc->config_read = vfio_pci_read_config;
2975 pdc->config_write = vfio_pci_write_config;
2976 pdc->is_express = 1; /* We might be */
2977 }
2978
2979 static const TypeInfo vfio_pci_dev_info = {
2980 .name = "vfio-pci",
2981 .parent = TYPE_PCI_DEVICE,
2982 .instance_size = sizeof(VFIOPCIDevice),
2983 .class_init = vfio_pci_dev_class_init,
2984 .instance_init = vfio_instance_init,
2985 .instance_finalize = vfio_instance_finalize,
2986 };
2987
2988 static void register_vfio_pci_dev_type(void)
2989 {
2990 type_register_static(&vfio_pci_dev_info);
2991 }
2992
2993 type_init(register_vfio_pci_dev_type)