hw/arm/bcm2836: Only provide "enabled-cpus" property to multicore SoCs
[qemu.git] / hw / vfio / pci.c
1 /*
2 * vfio based device assignment support
3 *
4 * Copyright Red Hat, Inc. 2012
5 *
6 * Authors:
7 * Alex Williamson <alex.williamson@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
19 */
20
21 #include "qemu/osdep.h"
22 #include <linux/vfio.h>
23 #include <sys/ioctl.h>
24
25 #include "hw/hw.h"
26 #include "hw/pci/msi.h"
27 #include "hw/pci/msix.h"
28 #include "hw/pci/pci_bridge.h"
29 #include "hw/qdev-properties.h"
30 #include "migration/vmstate.h"
31 #include "qemu/error-report.h"
32 #include "qemu/main-loop.h"
33 #include "qemu/module.h"
34 #include "qemu/option.h"
35 #include "qemu/range.h"
36 #include "qemu/units.h"
37 #include "sysemu/kvm.h"
38 #include "sysemu/runstate.h"
39 #include "sysemu/sysemu.h"
40 #include "pci.h"
41 #include "trace.h"
42 #include "qapi/error.h"
43 #include "migration/blocker.h"
44
45 #define TYPE_VFIO_PCI_NOHOTPLUG "vfio-pci-nohotplug"
46
47 static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
48 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
49
50 /*
51 * Disabling BAR mmaping can be slow, but toggling it around INTx can
52 * also be a huge overhead. We try to get the best of both worlds by
53 * waiting until an interrupt to disable mmaps (subsequent transitions
54 * to the same state are effectively no overhead). If the interrupt has
55 * been serviced and the time gap is long enough, we re-enable mmaps for
56 * performance. This works well for things like graphics cards, which
57 * may not use their interrupt at all and are penalized to an unusable
58 * level by read/write BAR traps. Other devices, like NICs, have more
59 * regular interrupts and see much better latency by staying in non-mmap
60 * mode. We therefore set the default mmap_timeout such that a ping
61 * is just enough to keep the mmap disabled. Users can experiment with
62 * other options with the x-intx-mmap-timeout-ms parameter (a value of
63 * zero disables the timer).
64 */
65 static void vfio_intx_mmap_enable(void *opaque)
66 {
67 VFIOPCIDevice *vdev = opaque;
68
69 if (vdev->intx.pending) {
70 timer_mod(vdev->intx.mmap_timer,
71 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
72 return;
73 }
74
75 vfio_mmap_set_enabled(vdev, true);
76 }
77
78 static void vfio_intx_interrupt(void *opaque)
79 {
80 VFIOPCIDevice *vdev = opaque;
81
82 if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
83 return;
84 }
85
86 trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin);
87
88 vdev->intx.pending = true;
89 pci_irq_assert(&vdev->pdev);
90 vfio_mmap_set_enabled(vdev, false);
91 if (vdev->intx.mmap_timeout) {
92 timer_mod(vdev->intx.mmap_timer,
93 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
94 }
95 }
96
97 static void vfio_intx_eoi(VFIODevice *vbasedev)
98 {
99 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
100
101 if (!vdev->intx.pending) {
102 return;
103 }
104
105 trace_vfio_intx_eoi(vbasedev->name);
106
107 vdev->intx.pending = false;
108 pci_irq_deassert(&vdev->pdev);
109 vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
110 }
111
112 static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev, Error **errp)
113 {
114 #ifdef CONFIG_KVM
115 int irq_fd = event_notifier_get_fd(&vdev->intx.interrupt);
116
117 if (vdev->no_kvm_intx || !kvm_irqfds_enabled() ||
118 vdev->intx.route.mode != PCI_INTX_ENABLED ||
119 !kvm_resamplefds_enabled()) {
120 return;
121 }
122
123 /* Get to a known interrupt state */
124 qemu_set_fd_handler(irq_fd, NULL, NULL, vdev);
125 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
126 vdev->intx.pending = false;
127 pci_irq_deassert(&vdev->pdev);
128
129 /* Get an eventfd for resample/unmask */
130 if (event_notifier_init(&vdev->intx.unmask, 0)) {
131 error_setg(errp, "event_notifier_init failed eoi");
132 goto fail;
133 }
134
135 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state,
136 &vdev->intx.interrupt,
137 &vdev->intx.unmask,
138 vdev->intx.route.irq)) {
139 error_setg_errno(errp, errno, "failed to setup resample irqfd");
140 goto fail_irqfd;
141 }
142
143 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
144 VFIO_IRQ_SET_ACTION_UNMASK,
145 event_notifier_get_fd(&vdev->intx.unmask),
146 errp)) {
147 goto fail_vfio;
148 }
149
150 /* Let'em rip */
151 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
152
153 vdev->intx.kvm_accel = true;
154
155 trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
156
157 return;
158
159 fail_vfio:
160 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vdev->intx.interrupt,
161 vdev->intx.route.irq);
162 fail_irqfd:
163 event_notifier_cleanup(&vdev->intx.unmask);
164 fail:
165 qemu_set_fd_handler(irq_fd, vfio_intx_interrupt, NULL, vdev);
166 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
167 #endif
168 }
169
170 static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
171 {
172 #ifdef CONFIG_KVM
173 if (!vdev->intx.kvm_accel) {
174 return;
175 }
176
177 /*
178 * Get to a known state, hardware masked, QEMU ready to accept new
179 * interrupts, QEMU IRQ de-asserted.
180 */
181 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
182 vdev->intx.pending = false;
183 pci_irq_deassert(&vdev->pdev);
184
185 /* Tell KVM to stop listening for an INTx irqfd */
186 if (kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vdev->intx.interrupt,
187 vdev->intx.route.irq)) {
188 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
189 }
190
191 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
192 event_notifier_cleanup(&vdev->intx.unmask);
193
194 /* QEMU starts listening for interrupt events. */
195 qemu_set_fd_handler(event_notifier_get_fd(&vdev->intx.interrupt),
196 vfio_intx_interrupt, NULL, vdev);
197
198 vdev->intx.kvm_accel = false;
199
200 /* If we've missed an event, let it re-fire through QEMU */
201 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
202
203 trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
204 #endif
205 }
206
207 static void vfio_intx_update(VFIOPCIDevice *vdev, PCIINTxRoute *route)
208 {
209 Error *err = NULL;
210
211 trace_vfio_intx_update(vdev->vbasedev.name,
212 vdev->intx.route.irq, route->irq);
213
214 vfio_intx_disable_kvm(vdev);
215
216 vdev->intx.route = *route;
217
218 if (route->mode != PCI_INTX_ENABLED) {
219 return;
220 }
221
222 vfio_intx_enable_kvm(vdev, &err);
223 if (err) {
224 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
225 }
226
227 /* Re-enable the interrupt in cased we missed an EOI */
228 vfio_intx_eoi(&vdev->vbasedev);
229 }
230
231 static void vfio_intx_routing_notifier(PCIDevice *pdev)
232 {
233 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
234 PCIINTxRoute route;
235
236 if (vdev->interrupt != VFIO_INT_INTx) {
237 return;
238 }
239
240 route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
241
242 if (pci_intx_route_changed(&vdev->intx.route, &route)) {
243 vfio_intx_update(vdev, &route);
244 }
245 }
246
247 static void vfio_irqchip_change(Notifier *notify, void *data)
248 {
249 VFIOPCIDevice *vdev = container_of(notify, VFIOPCIDevice,
250 irqchip_change_notifier);
251
252 vfio_intx_update(vdev, &vdev->intx.route);
253 }
254
255 static int vfio_intx_enable(VFIOPCIDevice *vdev, Error **errp)
256 {
257 uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
258 Error *err = NULL;
259 int32_t fd;
260 int ret;
261
262
263 if (!pin) {
264 return 0;
265 }
266
267 vfio_disable_interrupts(vdev);
268
269 vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
270 pci_config_set_interrupt_pin(vdev->pdev.config, pin);
271
272 #ifdef CONFIG_KVM
273 /*
274 * Only conditional to avoid generating error messages on platforms
275 * where we won't actually use the result anyway.
276 */
277 if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
278 vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
279 vdev->intx.pin);
280 }
281 #endif
282
283 ret = event_notifier_init(&vdev->intx.interrupt, 0);
284 if (ret) {
285 error_setg_errno(errp, -ret, "event_notifier_init failed");
286 return ret;
287 }
288 fd = event_notifier_get_fd(&vdev->intx.interrupt);
289 qemu_set_fd_handler(fd, vfio_intx_interrupt, NULL, vdev);
290
291 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
292 VFIO_IRQ_SET_ACTION_TRIGGER, fd, errp)) {
293 qemu_set_fd_handler(fd, NULL, NULL, vdev);
294 event_notifier_cleanup(&vdev->intx.interrupt);
295 return -errno;
296 }
297
298 vfio_intx_enable_kvm(vdev, &err);
299 if (err) {
300 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
301 }
302
303 vdev->interrupt = VFIO_INT_INTx;
304
305 trace_vfio_intx_enable(vdev->vbasedev.name);
306 return 0;
307 }
308
309 static void vfio_intx_disable(VFIOPCIDevice *vdev)
310 {
311 int fd;
312
313 timer_del(vdev->intx.mmap_timer);
314 vfio_intx_disable_kvm(vdev);
315 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
316 vdev->intx.pending = false;
317 pci_irq_deassert(&vdev->pdev);
318 vfio_mmap_set_enabled(vdev, true);
319
320 fd = event_notifier_get_fd(&vdev->intx.interrupt);
321 qemu_set_fd_handler(fd, NULL, NULL, vdev);
322 event_notifier_cleanup(&vdev->intx.interrupt);
323
324 vdev->interrupt = VFIO_INT_NONE;
325
326 trace_vfio_intx_disable(vdev->vbasedev.name);
327 }
328
329 /*
330 * MSI/X
331 */
332 static void vfio_msi_interrupt(void *opaque)
333 {
334 VFIOMSIVector *vector = opaque;
335 VFIOPCIDevice *vdev = vector->vdev;
336 MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector);
337 void (*notify)(PCIDevice *dev, unsigned vector);
338 MSIMessage msg;
339 int nr = vector - vdev->msi_vectors;
340
341 if (!event_notifier_test_and_clear(&vector->interrupt)) {
342 return;
343 }
344
345 if (vdev->interrupt == VFIO_INT_MSIX) {
346 get_msg = msix_get_message;
347 notify = msix_notify;
348
349 /* A masked vector firing needs to use the PBA, enable it */
350 if (msix_is_masked(&vdev->pdev, nr)) {
351 set_bit(nr, vdev->msix->pending);
352 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, true);
353 trace_vfio_msix_pba_enable(vdev->vbasedev.name);
354 }
355 } else if (vdev->interrupt == VFIO_INT_MSI) {
356 get_msg = msi_get_message;
357 notify = msi_notify;
358 } else {
359 abort();
360 }
361
362 msg = get_msg(&vdev->pdev, nr);
363 trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
364 notify(&vdev->pdev, nr);
365 }
366
367 static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
368 {
369 struct vfio_irq_set *irq_set;
370 int ret = 0, i, argsz;
371 int32_t *fds;
372
373 argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
374
375 irq_set = g_malloc0(argsz);
376 irq_set->argsz = argsz;
377 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
378 irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
379 irq_set->start = 0;
380 irq_set->count = vdev->nr_vectors;
381 fds = (int32_t *)&irq_set->data;
382
383 for (i = 0; i < vdev->nr_vectors; i++) {
384 int fd = -1;
385
386 /*
387 * MSI vs MSI-X - The guest has direct access to MSI mask and pending
388 * bits, therefore we always use the KVM signaling path when setup.
389 * MSI-X mask and pending bits are emulated, so we want to use the
390 * KVM signaling path only when configured and unmasked.
391 */
392 if (vdev->msi_vectors[i].use) {
393 if (vdev->msi_vectors[i].virq < 0 ||
394 (msix && msix_is_masked(&vdev->pdev, i))) {
395 fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
396 } else {
397 fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
398 }
399 }
400
401 fds[i] = fd;
402 }
403
404 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
405
406 g_free(irq_set);
407
408 return ret;
409 }
410
411 static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,
412 int vector_n, bool msix)
413 {
414 int virq;
415
416 if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi)) {
417 return;
418 }
419
420 if (event_notifier_init(&vector->kvm_interrupt, 0)) {
421 return;
422 }
423
424 virq = kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev);
425 if (virq < 0) {
426 event_notifier_cleanup(&vector->kvm_interrupt);
427 return;
428 }
429
430 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
431 NULL, virq) < 0) {
432 kvm_irqchip_release_virq(kvm_state, virq);
433 event_notifier_cleanup(&vector->kvm_interrupt);
434 return;
435 }
436
437 vector->virq = virq;
438 }
439
440 static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
441 {
442 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
443 vector->virq);
444 kvm_irqchip_release_virq(kvm_state, vector->virq);
445 vector->virq = -1;
446 event_notifier_cleanup(&vector->kvm_interrupt);
447 }
448
449 static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
450 PCIDevice *pdev)
451 {
452 kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
453 kvm_irqchip_commit_routes(kvm_state);
454 }
455
456 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
457 MSIMessage *msg, IOHandler *handler)
458 {
459 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
460 VFIOMSIVector *vector;
461 int ret;
462
463 trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
464
465 vector = &vdev->msi_vectors[nr];
466
467 if (!vector->use) {
468 vector->vdev = vdev;
469 vector->virq = -1;
470 if (event_notifier_init(&vector->interrupt, 0)) {
471 error_report("vfio: Error: event_notifier_init failed");
472 }
473 vector->use = true;
474 msix_vector_use(pdev, nr);
475 }
476
477 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
478 handler, NULL, vector);
479
480 /*
481 * Attempt to enable route through KVM irqchip,
482 * default to userspace handling if unavailable.
483 */
484 if (vector->virq >= 0) {
485 if (!msg) {
486 vfio_remove_kvm_msi_virq(vector);
487 } else {
488 vfio_update_kvm_msi_virq(vector, *msg, pdev);
489 }
490 } else {
491 if (msg) {
492 vfio_add_kvm_msi_virq(vdev, vector, nr, true);
493 }
494 }
495
496 /*
497 * We don't want to have the host allocate all possible MSI vectors
498 * for a device if they're not in use, so we shutdown and incrementally
499 * increase them as needed.
500 */
501 if (vdev->nr_vectors < nr + 1) {
502 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
503 vdev->nr_vectors = nr + 1;
504 ret = vfio_enable_vectors(vdev, true);
505 if (ret) {
506 error_report("vfio: failed to enable vectors, %d", ret);
507 }
508 } else {
509 Error *err = NULL;
510 int32_t fd;
511
512 if (vector->virq >= 0) {
513 fd = event_notifier_get_fd(&vector->kvm_interrupt);
514 } else {
515 fd = event_notifier_get_fd(&vector->interrupt);
516 }
517
518 if (vfio_set_irq_signaling(&vdev->vbasedev,
519 VFIO_PCI_MSIX_IRQ_INDEX, nr,
520 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
521 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
522 }
523 }
524
525 /* Disable PBA emulation when nothing more is pending. */
526 clear_bit(nr, vdev->msix->pending);
527 if (find_first_bit(vdev->msix->pending,
528 vdev->nr_vectors) == vdev->nr_vectors) {
529 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
530 trace_vfio_msix_pba_disable(vdev->vbasedev.name);
531 }
532
533 return 0;
534 }
535
536 static int vfio_msix_vector_use(PCIDevice *pdev,
537 unsigned int nr, MSIMessage msg)
538 {
539 return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
540 }
541
542 static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
543 {
544 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
545 VFIOMSIVector *vector = &vdev->msi_vectors[nr];
546
547 trace_vfio_msix_vector_release(vdev->vbasedev.name, nr);
548
549 /*
550 * There are still old guests that mask and unmask vectors on every
551 * interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of
552 * the KVM setup in place, simply switch VFIO to use the non-bypass
553 * eventfd. We'll then fire the interrupt through QEMU and the MSI-X
554 * core will mask the interrupt and set pending bits, allowing it to
555 * be re-asserted on unmask. Nothing to do if already using QEMU mode.
556 */
557 if (vector->virq >= 0) {
558 int32_t fd = event_notifier_get_fd(&vector->interrupt);
559 Error *err = NULL;
560
561 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX, nr,
562 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
563 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
564 }
565 }
566 }
567
568 static void vfio_msix_enable(VFIOPCIDevice *vdev)
569 {
570 vfio_disable_interrupts(vdev);
571
572 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
573
574 vdev->interrupt = VFIO_INT_MSIX;
575
576 /*
577 * Some communication channels between VF & PF or PF & fw rely on the
578 * physical state of the device and expect that enabling MSI-X from the
579 * guest enables the same on the host. When our guest is Linux, the
580 * guest driver call to pci_enable_msix() sets the enabling bit in the
581 * MSI-X capability, but leaves the vector table masked. We therefore
582 * can't rely on a vector_use callback (from request_irq() in the guest)
583 * to switch the physical device into MSI-X mode because that may come a
584 * long time after pci_enable_msix(). This code enables vector 0 with
585 * triggering to userspace, then immediately release the vector, leaving
586 * the physical device with no vectors enabled, but MSI-X enabled, just
587 * like the guest view.
588 */
589 vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
590 vfio_msix_vector_release(&vdev->pdev, 0);
591
592 if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
593 vfio_msix_vector_release, NULL)) {
594 error_report("vfio: msix_set_vector_notifiers failed");
595 }
596
597 trace_vfio_msix_enable(vdev->vbasedev.name);
598 }
599
600 static void vfio_msi_enable(VFIOPCIDevice *vdev)
601 {
602 int ret, i;
603
604 vfio_disable_interrupts(vdev);
605
606 vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
607 retry:
608 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
609
610 for (i = 0; i < vdev->nr_vectors; i++) {
611 VFIOMSIVector *vector = &vdev->msi_vectors[i];
612
613 vector->vdev = vdev;
614 vector->virq = -1;
615 vector->use = true;
616
617 if (event_notifier_init(&vector->interrupt, 0)) {
618 error_report("vfio: Error: event_notifier_init failed");
619 }
620
621 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
622 vfio_msi_interrupt, NULL, vector);
623
624 /*
625 * Attempt to enable route through KVM irqchip,
626 * default to userspace handling if unavailable.
627 */
628 vfio_add_kvm_msi_virq(vdev, vector, i, false);
629 }
630
631 /* Set interrupt type prior to possible interrupts */
632 vdev->interrupt = VFIO_INT_MSI;
633
634 ret = vfio_enable_vectors(vdev, false);
635 if (ret) {
636 if (ret < 0) {
637 error_report("vfio: Error: Failed to setup MSI fds: %m");
638 } else if (ret != vdev->nr_vectors) {
639 error_report("vfio: Error: Failed to enable %d "
640 "MSI vectors, retry with %d", vdev->nr_vectors, ret);
641 }
642
643 for (i = 0; i < vdev->nr_vectors; i++) {
644 VFIOMSIVector *vector = &vdev->msi_vectors[i];
645 if (vector->virq >= 0) {
646 vfio_remove_kvm_msi_virq(vector);
647 }
648 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
649 NULL, NULL, NULL);
650 event_notifier_cleanup(&vector->interrupt);
651 }
652
653 g_free(vdev->msi_vectors);
654 vdev->msi_vectors = NULL;
655
656 if (ret > 0 && ret != vdev->nr_vectors) {
657 vdev->nr_vectors = ret;
658 goto retry;
659 }
660 vdev->nr_vectors = 0;
661
662 /*
663 * Failing to setup MSI doesn't really fall within any specification.
664 * Let's try leaving interrupts disabled and hope the guest figures
665 * out to fall back to INTx for this device.
666 */
667 error_report("vfio: Error: Failed to enable MSI");
668 vdev->interrupt = VFIO_INT_NONE;
669
670 return;
671 }
672
673 trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
674 }
675
676 static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
677 {
678 Error *err = NULL;
679 int i;
680
681 for (i = 0; i < vdev->nr_vectors; i++) {
682 VFIOMSIVector *vector = &vdev->msi_vectors[i];
683 if (vdev->msi_vectors[i].use) {
684 if (vector->virq >= 0) {
685 vfio_remove_kvm_msi_virq(vector);
686 }
687 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
688 NULL, NULL, NULL);
689 event_notifier_cleanup(&vector->interrupt);
690 }
691 }
692
693 g_free(vdev->msi_vectors);
694 vdev->msi_vectors = NULL;
695 vdev->nr_vectors = 0;
696 vdev->interrupt = VFIO_INT_NONE;
697
698 vfio_intx_enable(vdev, &err);
699 if (err) {
700 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
701 }
702 }
703
704 static void vfio_msix_disable(VFIOPCIDevice *vdev)
705 {
706 int i;
707
708 msix_unset_vector_notifiers(&vdev->pdev);
709
710 /*
711 * MSI-X will only release vectors if MSI-X is still enabled on the
712 * device, check through the rest and release it ourselves if necessary.
713 */
714 for (i = 0; i < vdev->nr_vectors; i++) {
715 if (vdev->msi_vectors[i].use) {
716 vfio_msix_vector_release(&vdev->pdev, i);
717 msix_vector_unuse(&vdev->pdev, i);
718 }
719 }
720
721 if (vdev->nr_vectors) {
722 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
723 }
724
725 vfio_msi_disable_common(vdev);
726
727 memset(vdev->msix->pending, 0,
728 BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long));
729
730 trace_vfio_msix_disable(vdev->vbasedev.name);
731 }
732
733 static void vfio_msi_disable(VFIOPCIDevice *vdev)
734 {
735 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
736 vfio_msi_disable_common(vdev);
737
738 trace_vfio_msi_disable(vdev->vbasedev.name);
739 }
740
741 static void vfio_update_msi(VFIOPCIDevice *vdev)
742 {
743 int i;
744
745 for (i = 0; i < vdev->nr_vectors; i++) {
746 VFIOMSIVector *vector = &vdev->msi_vectors[i];
747 MSIMessage msg;
748
749 if (!vector->use || vector->virq < 0) {
750 continue;
751 }
752
753 msg = msi_get_message(&vdev->pdev, i);
754 vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev);
755 }
756 }
757
758 static void vfio_pci_load_rom(VFIOPCIDevice *vdev)
759 {
760 struct vfio_region_info *reg_info;
761 uint64_t size;
762 off_t off = 0;
763 ssize_t bytes;
764
765 if (vfio_get_region_info(&vdev->vbasedev,
766 VFIO_PCI_ROM_REGION_INDEX, &reg_info)) {
767 error_report("vfio: Error getting ROM info: %m");
768 return;
769 }
770
771 trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size,
772 (unsigned long)reg_info->offset,
773 (unsigned long)reg_info->flags);
774
775 vdev->rom_size = size = reg_info->size;
776 vdev->rom_offset = reg_info->offset;
777
778 g_free(reg_info);
779
780 if (!vdev->rom_size) {
781 vdev->rom_read_failed = true;
782 error_report("vfio-pci: Cannot read device rom at "
783 "%s", vdev->vbasedev.name);
784 error_printf("Device option ROM contents are probably invalid "
785 "(check dmesg).\nSkip option ROM probe with rombar=0, "
786 "or load from file with romfile=\n");
787 return;
788 }
789
790 vdev->rom = g_malloc(size);
791 memset(vdev->rom, 0xff, size);
792
793 while (size) {
794 bytes = pread(vdev->vbasedev.fd, vdev->rom + off,
795 size, vdev->rom_offset + off);
796 if (bytes == 0) {
797 break;
798 } else if (bytes > 0) {
799 off += bytes;
800 size -= bytes;
801 } else {
802 if (errno == EINTR || errno == EAGAIN) {
803 continue;
804 }
805 error_report("vfio: Error reading device ROM: %m");
806 break;
807 }
808 }
809
810 /*
811 * Test the ROM signature against our device, if the vendor is correct
812 * but the device ID doesn't match, store the correct device ID and
813 * recompute the checksum. Intel IGD devices need this and are known
814 * to have bogus checksums so we can't simply adjust the checksum.
815 */
816 if (pci_get_word(vdev->rom) == 0xaa55 &&
817 pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size &&
818 !memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) {
819 uint16_t vid, did;
820
821 vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4);
822 did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6);
823
824 if (vid == vdev->vendor_id && did != vdev->device_id) {
825 int i;
826 uint8_t csum, *data = vdev->rom;
827
828 pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6,
829 vdev->device_id);
830 data[6] = 0;
831
832 for (csum = 0, i = 0; i < vdev->rom_size; i++) {
833 csum += data[i];
834 }
835
836 data[6] = -csum;
837 }
838 }
839 }
840
841 static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
842 {
843 VFIOPCIDevice *vdev = opaque;
844 union {
845 uint8_t byte;
846 uint16_t word;
847 uint32_t dword;
848 uint64_t qword;
849 } val;
850 uint64_t data = 0;
851
852 /* Load the ROM lazily when the guest tries to read it */
853 if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
854 vfio_pci_load_rom(vdev);
855 }
856
857 memcpy(&val, vdev->rom + addr,
858 (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
859
860 switch (size) {
861 case 1:
862 data = val.byte;
863 break;
864 case 2:
865 data = le16_to_cpu(val.word);
866 break;
867 case 4:
868 data = le32_to_cpu(val.dword);
869 break;
870 default:
871 hw_error("vfio: unsupported read size, %d bytes\n", size);
872 break;
873 }
874
875 trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data);
876
877 return data;
878 }
879
880 static void vfio_rom_write(void *opaque, hwaddr addr,
881 uint64_t data, unsigned size)
882 {
883 }
884
885 static const MemoryRegionOps vfio_rom_ops = {
886 .read = vfio_rom_read,
887 .write = vfio_rom_write,
888 .endianness = DEVICE_LITTLE_ENDIAN,
889 };
890
891 static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
892 {
893 uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
894 off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
895 DeviceState *dev = DEVICE(vdev);
896 char *name;
897 int fd = vdev->vbasedev.fd;
898
899 if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
900 /* Since pci handles romfile, just print a message and return */
901 if (vfio_blacklist_opt_rom(vdev) && vdev->pdev.romfile) {
902 warn_report("Device at %s is known to cause system instability"
903 " issues during option rom execution",
904 vdev->vbasedev.name);
905 error_printf("Proceeding anyway since user specified romfile\n");
906 }
907 return;
908 }
909
910 /*
911 * Use the same size ROM BAR as the physical device. The contents
912 * will get filled in later when the guest tries to read it.
913 */
914 if (pread(fd, &orig, 4, offset) != 4 ||
915 pwrite(fd, &size, 4, offset) != 4 ||
916 pread(fd, &size, 4, offset) != 4 ||
917 pwrite(fd, &orig, 4, offset) != 4) {
918 error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name);
919 return;
920 }
921
922 size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
923
924 if (!size) {
925 return;
926 }
927
928 if (vfio_blacklist_opt_rom(vdev)) {
929 if (dev->opts && qemu_opt_get(dev->opts, "rombar")) {
930 warn_report("Device at %s is known to cause system instability"
931 " issues during option rom execution",
932 vdev->vbasedev.name);
933 error_printf("Proceeding anyway since user specified"
934 " non zero value for rombar\n");
935 } else {
936 warn_report("Rom loading for device at %s has been disabled"
937 " due to system instability issues",
938 vdev->vbasedev.name);
939 error_printf("Specify rombar=1 or romfile to force\n");
940 return;
941 }
942 }
943
944 trace_vfio_pci_size_rom(vdev->vbasedev.name, size);
945
946 name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name);
947
948 memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
949 &vfio_rom_ops, vdev, name, size);
950 g_free(name);
951
952 pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
953 PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
954
955 vdev->rom_read_failed = false;
956 }
957
958 void vfio_vga_write(void *opaque, hwaddr addr,
959 uint64_t data, unsigned size)
960 {
961 VFIOVGARegion *region = opaque;
962 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
963 union {
964 uint8_t byte;
965 uint16_t word;
966 uint32_t dword;
967 uint64_t qword;
968 } buf;
969 off_t offset = vga->fd_offset + region->offset + addr;
970
971 switch (size) {
972 case 1:
973 buf.byte = data;
974 break;
975 case 2:
976 buf.word = cpu_to_le16(data);
977 break;
978 case 4:
979 buf.dword = cpu_to_le32(data);
980 break;
981 default:
982 hw_error("vfio: unsupported write size, %d bytes", size);
983 break;
984 }
985
986 if (pwrite(vga->fd, &buf, size, offset) != size) {
987 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
988 __func__, region->offset + addr, data, size);
989 }
990
991 trace_vfio_vga_write(region->offset + addr, data, size);
992 }
993
994 uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
995 {
996 VFIOVGARegion *region = opaque;
997 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
998 union {
999 uint8_t byte;
1000 uint16_t word;
1001 uint32_t dword;
1002 uint64_t qword;
1003 } buf;
1004 uint64_t data = 0;
1005 off_t offset = vga->fd_offset + region->offset + addr;
1006
1007 if (pread(vga->fd, &buf, size, offset) != size) {
1008 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1009 __func__, region->offset + addr, size);
1010 return (uint64_t)-1;
1011 }
1012
1013 switch (size) {
1014 case 1:
1015 data = buf.byte;
1016 break;
1017 case 2:
1018 data = le16_to_cpu(buf.word);
1019 break;
1020 case 4:
1021 data = le32_to_cpu(buf.dword);
1022 break;
1023 default:
1024 hw_error("vfio: unsupported read size, %d bytes", size);
1025 break;
1026 }
1027
1028 trace_vfio_vga_read(region->offset + addr, size, data);
1029
1030 return data;
1031 }
1032
1033 static const MemoryRegionOps vfio_vga_ops = {
1034 .read = vfio_vga_read,
1035 .write = vfio_vga_write,
1036 .endianness = DEVICE_LITTLE_ENDIAN,
1037 };
1038
1039 /*
1040 * Expand memory region of sub-page(size < PAGE_SIZE) MMIO BAR to page
1041 * size if the BAR is in an exclusive page in host so that we could map
1042 * this BAR to guest. But this sub-page BAR may not occupy an exclusive
1043 * page in guest. So we should set the priority of the expanded memory
1044 * region to zero in case of overlap with BARs which share the same page
1045 * with the sub-page BAR in guest. Besides, we should also recover the
1046 * size of this sub-page BAR when its base address is changed in guest
1047 * and not page aligned any more.
1048 */
1049 static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar)
1050 {
1051 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
1052 VFIORegion *region = &vdev->bars[bar].region;
1053 MemoryRegion *mmap_mr, *region_mr, *base_mr;
1054 PCIIORegion *r;
1055 pcibus_t bar_addr;
1056 uint64_t size = region->size;
1057
1058 /* Make sure that the whole region is allowed to be mmapped */
1059 if (region->nr_mmaps != 1 || !region->mmaps[0].mmap ||
1060 region->mmaps[0].size != region->size) {
1061 return;
1062 }
1063
1064 r = &pdev->io_regions[bar];
1065 bar_addr = r->addr;
1066 base_mr = vdev->bars[bar].mr;
1067 region_mr = region->mem;
1068 mmap_mr = &region->mmaps[0].mem;
1069
1070 /* If BAR is mapped and page aligned, update to fill PAGE_SIZE */
1071 if (bar_addr != PCI_BAR_UNMAPPED &&
1072 !(bar_addr & ~qemu_real_host_page_mask)) {
1073 size = qemu_real_host_page_size;
1074 }
1075
1076 memory_region_transaction_begin();
1077
1078 if (vdev->bars[bar].size < size) {
1079 memory_region_set_size(base_mr, size);
1080 }
1081 memory_region_set_size(region_mr, size);
1082 memory_region_set_size(mmap_mr, size);
1083 if (size != vdev->bars[bar].size && memory_region_is_mapped(base_mr)) {
1084 memory_region_del_subregion(r->address_space, base_mr);
1085 memory_region_add_subregion_overlap(r->address_space,
1086 bar_addr, base_mr, 0);
1087 }
1088
1089 memory_region_transaction_commit();
1090 }
1091
1092 /*
1093 * PCI config space
1094 */
1095 uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
1096 {
1097 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
1098 uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
1099
1100 memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
1101 emu_bits = le32_to_cpu(emu_bits);
1102
1103 if (emu_bits) {
1104 emu_val = pci_default_read_config(pdev, addr, len);
1105 }
1106
1107 if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
1108 ssize_t ret;
1109
1110 ret = pread(vdev->vbasedev.fd, &phys_val, len,
1111 vdev->config_offset + addr);
1112 if (ret != len) {
1113 error_report("%s(%s, 0x%x, 0x%x) failed: %m",
1114 __func__, vdev->vbasedev.name, addr, len);
1115 return -errno;
1116 }
1117 phys_val = le32_to_cpu(phys_val);
1118 }
1119
1120 val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
1121
1122 trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val);
1123
1124 return val;
1125 }
1126
1127 void vfio_pci_write_config(PCIDevice *pdev,
1128 uint32_t addr, uint32_t val, int len)
1129 {
1130 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
1131 uint32_t val_le = cpu_to_le32(val);
1132
1133 trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len);
1134
1135 /* Write everything to VFIO, let it filter out what we can't write */
1136 if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr)
1137 != len) {
1138 error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
1139 __func__, vdev->vbasedev.name, addr, val, len);
1140 }
1141
1142 /* MSI/MSI-X Enabling/Disabling */
1143 if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
1144 ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
1145 int is_enabled, was_enabled = msi_enabled(pdev);
1146
1147 pci_default_write_config(pdev, addr, val, len);
1148
1149 is_enabled = msi_enabled(pdev);
1150
1151 if (!was_enabled) {
1152 if (is_enabled) {
1153 vfio_msi_enable(vdev);
1154 }
1155 } else {
1156 if (!is_enabled) {
1157 vfio_msi_disable(vdev);
1158 } else {
1159 vfio_update_msi(vdev);
1160 }
1161 }
1162 } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
1163 ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
1164 int is_enabled, was_enabled = msix_enabled(pdev);
1165
1166 pci_default_write_config(pdev, addr, val, len);
1167
1168 is_enabled = msix_enabled(pdev);
1169
1170 if (!was_enabled && is_enabled) {
1171 vfio_msix_enable(vdev);
1172 } else if (was_enabled && !is_enabled) {
1173 vfio_msix_disable(vdev);
1174 }
1175 } else if (ranges_overlap(addr, len, PCI_BASE_ADDRESS_0, 24) ||
1176 range_covers_byte(addr, len, PCI_COMMAND)) {
1177 pcibus_t old_addr[PCI_NUM_REGIONS - 1];
1178 int bar;
1179
1180 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1181 old_addr[bar] = pdev->io_regions[bar].addr;
1182 }
1183
1184 pci_default_write_config(pdev, addr, val, len);
1185
1186 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1187 if (old_addr[bar] != pdev->io_regions[bar].addr &&
1188 vdev->bars[bar].region.size > 0 &&
1189 vdev->bars[bar].region.size < qemu_real_host_page_size) {
1190 vfio_sub_page_bar_update_mapping(pdev, bar);
1191 }
1192 }
1193 } else {
1194 /* Write everything to QEMU to keep emulated bits correct */
1195 pci_default_write_config(pdev, addr, val, len);
1196 }
1197 }
1198
1199 /*
1200 * Interrupt setup
1201 */
1202 static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
1203 {
1204 /*
1205 * More complicated than it looks. Disabling MSI/X transitions the
1206 * device to INTx mode (if supported). Therefore we need to first
1207 * disable MSI/X and then cleanup by disabling INTx.
1208 */
1209 if (vdev->interrupt == VFIO_INT_MSIX) {
1210 vfio_msix_disable(vdev);
1211 } else if (vdev->interrupt == VFIO_INT_MSI) {
1212 vfio_msi_disable(vdev);
1213 }
1214
1215 if (vdev->interrupt == VFIO_INT_INTx) {
1216 vfio_intx_disable(vdev);
1217 }
1218 }
1219
1220 static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1221 {
1222 uint16_t ctrl;
1223 bool msi_64bit, msi_maskbit;
1224 int ret, entries;
1225 Error *err = NULL;
1226
1227 if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl),
1228 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
1229 error_setg_errno(errp, errno, "failed reading MSI PCI_CAP_FLAGS");
1230 return -errno;
1231 }
1232 ctrl = le16_to_cpu(ctrl);
1233
1234 msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
1235 msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
1236 entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
1237
1238 trace_vfio_msi_setup(vdev->vbasedev.name, pos);
1239
1240 ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit, &err);
1241 if (ret < 0) {
1242 if (ret == -ENOTSUP) {
1243 return 0;
1244 }
1245 error_propagate_prepend(errp, err, "msi_init failed: ");
1246 return ret;
1247 }
1248 vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
1249
1250 return 0;
1251 }
1252
1253 static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev)
1254 {
1255 off_t start, end;
1256 VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region;
1257
1258 /*
1259 * If the host driver allows mapping of a MSIX data, we are going to
1260 * do map the entire BAR and emulate MSIX table on top of that.
1261 */
1262 if (vfio_has_region_cap(&vdev->vbasedev, region->nr,
1263 VFIO_REGION_INFO_CAP_MSIX_MAPPABLE)) {
1264 return;
1265 }
1266
1267 /*
1268 * We expect to find a single mmap covering the whole BAR, anything else
1269 * means it's either unsupported or already setup.
1270 */
1271 if (region->nr_mmaps != 1 || region->mmaps[0].offset ||
1272 region->size != region->mmaps[0].size) {
1273 return;
1274 }
1275
1276 /* MSI-X table start and end aligned to host page size */
1277 start = vdev->msix->table_offset & qemu_real_host_page_mask;
1278 end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
1279 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
1280
1281 /*
1282 * Does the MSI-X table cover the beginning of the BAR? The whole BAR?
1283 * NB - Host page size is necessarily a power of two and so is the PCI
1284 * BAR (not counting EA yet), therefore if we have host page aligned
1285 * @start and @end, then any remainder of the BAR before or after those
1286 * must be at least host page sized and therefore mmap'able.
1287 */
1288 if (!start) {
1289 if (end >= region->size) {
1290 region->nr_mmaps = 0;
1291 g_free(region->mmaps);
1292 region->mmaps = NULL;
1293 trace_vfio_msix_fixup(vdev->vbasedev.name,
1294 vdev->msix->table_bar, 0, 0);
1295 } else {
1296 region->mmaps[0].offset = end;
1297 region->mmaps[0].size = region->size - end;
1298 trace_vfio_msix_fixup(vdev->vbasedev.name,
1299 vdev->msix->table_bar, region->mmaps[0].offset,
1300 region->mmaps[0].offset + region->mmaps[0].size);
1301 }
1302
1303 /* Maybe it's aligned at the end of the BAR */
1304 } else if (end >= region->size) {
1305 region->mmaps[0].size = start;
1306 trace_vfio_msix_fixup(vdev->vbasedev.name,
1307 vdev->msix->table_bar, region->mmaps[0].offset,
1308 region->mmaps[0].offset + region->mmaps[0].size);
1309
1310 /* Otherwise it must split the BAR */
1311 } else {
1312 region->nr_mmaps = 2;
1313 region->mmaps = g_renew(VFIOMmap, region->mmaps, 2);
1314
1315 memcpy(&region->mmaps[1], &region->mmaps[0], sizeof(VFIOMmap));
1316
1317 region->mmaps[0].size = start;
1318 trace_vfio_msix_fixup(vdev->vbasedev.name,
1319 vdev->msix->table_bar, region->mmaps[0].offset,
1320 region->mmaps[0].offset + region->mmaps[0].size);
1321
1322 region->mmaps[1].offset = end;
1323 region->mmaps[1].size = region->size - end;
1324 trace_vfio_msix_fixup(vdev->vbasedev.name,
1325 vdev->msix->table_bar, region->mmaps[1].offset,
1326 region->mmaps[1].offset + region->mmaps[1].size);
1327 }
1328 }
1329
1330 static void vfio_pci_relocate_msix(VFIOPCIDevice *vdev, Error **errp)
1331 {
1332 int target_bar = -1;
1333 size_t msix_sz;
1334
1335 if (!vdev->msix || vdev->msix_relo == OFF_AUTOPCIBAR_OFF) {
1336 return;
1337 }
1338
1339 /* The actual minimum size of MSI-X structures */
1340 msix_sz = (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE) +
1341 (QEMU_ALIGN_UP(vdev->msix->entries, 64) / 8);
1342 /* Round up to host pages, we don't want to share a page */
1343 msix_sz = REAL_HOST_PAGE_ALIGN(msix_sz);
1344 /* PCI BARs must be a power of 2 */
1345 msix_sz = pow2ceil(msix_sz);
1346
1347 if (vdev->msix_relo == OFF_AUTOPCIBAR_AUTO) {
1348 /*
1349 * TODO: Lookup table for known devices.
1350 *
1351 * Logically we might use an algorithm here to select the BAR adding
1352 * the least additional MMIO space, but we cannot programatically
1353 * predict the driver dependency on BAR ordering or sizing, therefore
1354 * 'auto' becomes a lookup for combinations reported to work.
1355 */
1356 if (target_bar < 0) {
1357 error_setg(errp, "No automatic MSI-X relocation available for "
1358 "device %04x:%04x", vdev->vendor_id, vdev->device_id);
1359 return;
1360 }
1361 } else {
1362 target_bar = (int)(vdev->msix_relo - OFF_AUTOPCIBAR_BAR0);
1363 }
1364
1365 /* I/O port BARs cannot host MSI-X structures */
1366 if (vdev->bars[target_bar].ioport) {
1367 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1368 "I/O port BAR", target_bar);
1369 return;
1370 }
1371
1372 /* Cannot use a BAR in the "shadow" of a 64-bit BAR */
1373 if (!vdev->bars[target_bar].size &&
1374 target_bar > 0 && vdev->bars[target_bar - 1].mem64) {
1375 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1376 "consumed by 64-bit BAR %d", target_bar, target_bar - 1);
1377 return;
1378 }
1379
1380 /* 2GB max size for 32-bit BARs, cannot double if already > 1G */
1381 if (vdev->bars[target_bar].size > 1 * GiB &&
1382 !vdev->bars[target_bar].mem64) {
1383 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1384 "no space to extend 32-bit BAR", target_bar);
1385 return;
1386 }
1387
1388 /*
1389 * If adding a new BAR, test if we can make it 64bit. We make it
1390 * prefetchable since QEMU MSI-X emulation has no read side effects
1391 * and doing so makes mapping more flexible.
1392 */
1393 if (!vdev->bars[target_bar].size) {
1394 if (target_bar < (PCI_ROM_SLOT - 1) &&
1395 !vdev->bars[target_bar + 1].size) {
1396 vdev->bars[target_bar].mem64 = true;
1397 vdev->bars[target_bar].type = PCI_BASE_ADDRESS_MEM_TYPE_64;
1398 }
1399 vdev->bars[target_bar].type |= PCI_BASE_ADDRESS_MEM_PREFETCH;
1400 vdev->bars[target_bar].size = msix_sz;
1401 vdev->msix->table_offset = 0;
1402 } else {
1403 vdev->bars[target_bar].size = MAX(vdev->bars[target_bar].size * 2,
1404 msix_sz * 2);
1405 /*
1406 * Due to above size calc, MSI-X always starts halfway into the BAR,
1407 * which will always be a separate host page.
1408 */
1409 vdev->msix->table_offset = vdev->bars[target_bar].size / 2;
1410 }
1411
1412 vdev->msix->table_bar = target_bar;
1413 vdev->msix->pba_bar = target_bar;
1414 /* Requires 8-byte alignment, but PCI_MSIX_ENTRY_SIZE guarantees that */
1415 vdev->msix->pba_offset = vdev->msix->table_offset +
1416 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE);
1417
1418 trace_vfio_msix_relo(vdev->vbasedev.name,
1419 vdev->msix->table_bar, vdev->msix->table_offset);
1420 }
1421
1422 /*
1423 * We don't have any control over how pci_add_capability() inserts
1424 * capabilities into the chain. In order to setup MSI-X we need a
1425 * MemoryRegion for the BAR. In order to setup the BAR and not
1426 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
1427 * need to first look for where the MSI-X table lives. So we
1428 * unfortunately split MSI-X setup across two functions.
1429 */
1430 static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
1431 {
1432 uint8_t pos;
1433 uint16_t ctrl;
1434 uint32_t table, pba;
1435 int fd = vdev->vbasedev.fd;
1436 VFIOMSIXInfo *msix;
1437
1438 pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
1439 if (!pos) {
1440 return;
1441 }
1442
1443 if (pread(fd, &ctrl, sizeof(ctrl),
1444 vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) {
1445 error_setg_errno(errp, errno, "failed to read PCI MSIX FLAGS");
1446 return;
1447 }
1448
1449 if (pread(fd, &table, sizeof(table),
1450 vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
1451 error_setg_errno(errp, errno, "failed to read PCI MSIX TABLE");
1452 return;
1453 }
1454
1455 if (pread(fd, &pba, sizeof(pba),
1456 vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
1457 error_setg_errno(errp, errno, "failed to read PCI MSIX PBA");
1458 return;
1459 }
1460
1461 ctrl = le16_to_cpu(ctrl);
1462 table = le32_to_cpu(table);
1463 pba = le32_to_cpu(pba);
1464
1465 msix = g_malloc0(sizeof(*msix));
1466 msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
1467 msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
1468 msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
1469 msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
1470 msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
1471
1472 /*
1473 * Test the size of the pba_offset variable and catch if it extends outside
1474 * of the specified BAR. If it is the case, we need to apply a hardware
1475 * specific quirk if the device is known or we have a broken configuration.
1476 */
1477 if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
1478 /*
1479 * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
1480 * adapters. The T5 hardware returns an incorrect value of 0x8000 for
1481 * the VF PBA offset while the BAR itself is only 8k. The correct value
1482 * is 0x1000, so we hard code that here.
1483 */
1484 if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
1485 (vdev->device_id & 0xff00) == 0x5800) {
1486 msix->pba_offset = 0x1000;
1487 } else if (vdev->msix_relo == OFF_AUTOPCIBAR_OFF) {
1488 error_setg(errp, "hardware reports invalid configuration, "
1489 "MSIX PBA outside of specified BAR");
1490 g_free(msix);
1491 return;
1492 }
1493 }
1494
1495 trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
1496 msix->table_offset, msix->entries);
1497 vdev->msix = msix;
1498
1499 vfio_pci_fixup_msix_region(vdev);
1500
1501 vfio_pci_relocate_msix(vdev, errp);
1502 }
1503
1504 static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1505 {
1506 int ret;
1507 Error *err = NULL;
1508
1509 vdev->msix->pending = g_malloc0(BITS_TO_LONGS(vdev->msix->entries) *
1510 sizeof(unsigned long));
1511 ret = msix_init(&vdev->pdev, vdev->msix->entries,
1512 vdev->bars[vdev->msix->table_bar].mr,
1513 vdev->msix->table_bar, vdev->msix->table_offset,
1514 vdev->bars[vdev->msix->pba_bar].mr,
1515 vdev->msix->pba_bar, vdev->msix->pba_offset, pos,
1516 &err);
1517 if (ret < 0) {
1518 if (ret == -ENOTSUP) {
1519 warn_report_err(err);
1520 return 0;
1521 }
1522
1523 error_propagate(errp, err);
1524 return ret;
1525 }
1526
1527 /*
1528 * The PCI spec suggests that devices provide additional alignment for
1529 * MSI-X structures and avoid overlapping non-MSI-X related registers.
1530 * For an assigned device, this hopefully means that emulation of MSI-X
1531 * structures does not affect the performance of the device. If devices
1532 * fail to provide that alignment, a significant performance penalty may
1533 * result, for instance Mellanox MT27500 VFs:
1534 * http://www.spinics.net/lists/kvm/msg125881.html
1535 *
1536 * The PBA is simply not that important for such a serious regression and
1537 * most drivers do not appear to look at it. The solution for this is to
1538 * disable the PBA MemoryRegion unless it's being used. We disable it
1539 * here and only enable it if a masked vector fires through QEMU. As the
1540 * vector-use notifier is called, which occurs on unmask, we test whether
1541 * PBA emulation is needed and again disable if not.
1542 */
1543 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
1544
1545 /*
1546 * The emulated machine may provide a paravirt interface for MSIX setup
1547 * so it is not strictly necessary to emulate MSIX here. This becomes
1548 * helpful when frequently accessed MMIO registers are located in
1549 * subpages adjacent to the MSIX table but the MSIX data containing page
1550 * cannot be mapped because of a host page size bigger than the MSIX table
1551 * alignment.
1552 */
1553 if (object_property_get_bool(OBJECT(qdev_get_machine()),
1554 "vfio-no-msix-emulation", NULL)) {
1555 memory_region_set_enabled(&vdev->pdev.msix_table_mmio, false);
1556 }
1557
1558 return 0;
1559 }
1560
1561 static void vfio_teardown_msi(VFIOPCIDevice *vdev)
1562 {
1563 msi_uninit(&vdev->pdev);
1564
1565 if (vdev->msix) {
1566 msix_uninit(&vdev->pdev,
1567 vdev->bars[vdev->msix->table_bar].mr,
1568 vdev->bars[vdev->msix->pba_bar].mr);
1569 g_free(vdev->msix->pending);
1570 }
1571 }
1572
1573 /*
1574 * Resource setup
1575 */
1576 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
1577 {
1578 int i;
1579
1580 for (i = 0; i < PCI_ROM_SLOT; i++) {
1581 vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled);
1582 }
1583 }
1584
1585 static void vfio_bar_prepare(VFIOPCIDevice *vdev, int nr)
1586 {
1587 VFIOBAR *bar = &vdev->bars[nr];
1588
1589 uint32_t pci_bar;
1590 int ret;
1591
1592 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
1593 if (!bar->region.size) {
1594 return;
1595 }
1596
1597 /* Determine what type of BAR this is for registration */
1598 ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar),
1599 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
1600 if (ret != sizeof(pci_bar)) {
1601 error_report("vfio: Failed to read BAR %d (%m)", nr);
1602 return;
1603 }
1604
1605 pci_bar = le32_to_cpu(pci_bar);
1606 bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
1607 bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
1608 bar->type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
1609 ~PCI_BASE_ADDRESS_MEM_MASK);
1610 bar->size = bar->region.size;
1611 }
1612
1613 static void vfio_bars_prepare(VFIOPCIDevice *vdev)
1614 {
1615 int i;
1616
1617 for (i = 0; i < PCI_ROM_SLOT; i++) {
1618 vfio_bar_prepare(vdev, i);
1619 }
1620 }
1621
1622 static void vfio_bar_register(VFIOPCIDevice *vdev, int nr)
1623 {
1624 VFIOBAR *bar = &vdev->bars[nr];
1625 char *name;
1626
1627 if (!bar->size) {
1628 return;
1629 }
1630
1631 bar->mr = g_new0(MemoryRegion, 1);
1632 name = g_strdup_printf("%s base BAR %d", vdev->vbasedev.name, nr);
1633 memory_region_init_io(bar->mr, OBJECT(vdev), NULL, NULL, name, bar->size);
1634 g_free(name);
1635
1636 if (bar->region.size) {
1637 memory_region_add_subregion(bar->mr, 0, bar->region.mem);
1638
1639 if (vfio_region_mmap(&bar->region)) {
1640 error_report("Failed to mmap %s BAR %d. Performance may be slow",
1641 vdev->vbasedev.name, nr);
1642 }
1643 }
1644
1645 pci_register_bar(&vdev->pdev, nr, bar->type, bar->mr);
1646 }
1647
1648 static void vfio_bars_register(VFIOPCIDevice *vdev)
1649 {
1650 int i;
1651
1652 for (i = 0; i < PCI_ROM_SLOT; i++) {
1653 vfio_bar_register(vdev, i);
1654 }
1655 }
1656
1657 static void vfio_bars_exit(VFIOPCIDevice *vdev)
1658 {
1659 int i;
1660
1661 for (i = 0; i < PCI_ROM_SLOT; i++) {
1662 VFIOBAR *bar = &vdev->bars[i];
1663
1664 vfio_bar_quirk_exit(vdev, i);
1665 vfio_region_exit(&bar->region);
1666 if (bar->region.size) {
1667 memory_region_del_subregion(bar->mr, bar->region.mem);
1668 }
1669 }
1670
1671 if (vdev->vga) {
1672 pci_unregister_vga(&vdev->pdev);
1673 vfio_vga_quirk_exit(vdev);
1674 }
1675 }
1676
1677 static void vfio_bars_finalize(VFIOPCIDevice *vdev)
1678 {
1679 int i;
1680
1681 for (i = 0; i < PCI_ROM_SLOT; i++) {
1682 VFIOBAR *bar = &vdev->bars[i];
1683
1684 vfio_bar_quirk_finalize(vdev, i);
1685 vfio_region_finalize(&bar->region);
1686 if (bar->size) {
1687 object_unparent(OBJECT(bar->mr));
1688 g_free(bar->mr);
1689 }
1690 }
1691
1692 if (vdev->vga) {
1693 vfio_vga_quirk_finalize(vdev);
1694 for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
1695 object_unparent(OBJECT(&vdev->vga->region[i].mem));
1696 }
1697 g_free(vdev->vga);
1698 }
1699 }
1700
1701 /*
1702 * General setup
1703 */
1704 static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
1705 {
1706 uint8_t tmp;
1707 uint16_t next = PCI_CONFIG_SPACE_SIZE;
1708
1709 for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
1710 tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) {
1711 if (tmp > pos && tmp < next) {
1712 next = tmp;
1713 }
1714 }
1715
1716 return next - pos;
1717 }
1718
1719
1720 static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
1721 {
1722 uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
1723
1724 for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
1725 tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
1726 if (tmp > pos && tmp < next) {
1727 next = tmp;
1728 }
1729 }
1730
1731 return next - pos;
1732 }
1733
1734 static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
1735 {
1736 pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
1737 }
1738
1739 static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos,
1740 uint16_t val, uint16_t mask)
1741 {
1742 vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
1743 vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
1744 vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
1745 }
1746
1747 static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
1748 {
1749 pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
1750 }
1751
1752 static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos,
1753 uint32_t val, uint32_t mask)
1754 {
1755 vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
1756 vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
1757 vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
1758 }
1759
1760 static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size,
1761 Error **errp)
1762 {
1763 uint16_t flags;
1764 uint8_t type;
1765
1766 flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
1767 type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
1768
1769 if (type != PCI_EXP_TYPE_ENDPOINT &&
1770 type != PCI_EXP_TYPE_LEG_END &&
1771 type != PCI_EXP_TYPE_RC_END) {
1772
1773 error_setg(errp, "assignment of PCIe type 0x%x "
1774 "devices is not currently supported", type);
1775 return -EINVAL;
1776 }
1777
1778 if (!pci_bus_is_express(pci_get_bus(&vdev->pdev))) {
1779 PCIBus *bus = pci_get_bus(&vdev->pdev);
1780 PCIDevice *bridge;
1781
1782 /*
1783 * Traditionally PCI device assignment exposes the PCIe capability
1784 * as-is on non-express buses. The reason being that some drivers
1785 * simply assume that it's there, for example tg3. However when
1786 * we're running on a native PCIe machine type, like Q35, we need
1787 * to hide the PCIe capability. The reason for this is twofold;
1788 * first Windows guests get a Code 10 error when the PCIe capability
1789 * is exposed in this configuration. Therefore express devices won't
1790 * work at all unless they're attached to express buses in the VM.
1791 * Second, a native PCIe machine introduces the possibility of fine
1792 * granularity IOMMUs supporting both translation and isolation.
1793 * Guest code to discover the IOMMU visibility of a device, such as
1794 * IOMMU grouping code on Linux, is very aware of device types and
1795 * valid transitions between bus types. An express device on a non-
1796 * express bus is not a valid combination on bare metal systems.
1797 *
1798 * Drivers that require a PCIe capability to make the device
1799 * functional are simply going to need to have their devices placed
1800 * on a PCIe bus in the VM.
1801 */
1802 while (!pci_bus_is_root(bus)) {
1803 bridge = pci_bridge_get_device(bus);
1804 bus = pci_get_bus(bridge);
1805 }
1806
1807 if (pci_bus_is_express(bus)) {
1808 return 0;
1809 }
1810
1811 } else if (pci_bus_is_root(pci_get_bus(&vdev->pdev))) {
1812 /*
1813 * On a Root Complex bus Endpoints become Root Complex Integrated
1814 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
1815 */
1816 if (type == PCI_EXP_TYPE_ENDPOINT) {
1817 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1818 PCI_EXP_TYPE_RC_END << 4,
1819 PCI_EXP_FLAGS_TYPE);
1820
1821 /* Link Capabilities, Status, and Control goes away */
1822 if (size > PCI_EXP_LNKCTL) {
1823 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
1824 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1825 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
1826
1827 #ifndef PCI_EXP_LNKCAP2
1828 #define PCI_EXP_LNKCAP2 44
1829 #endif
1830 #ifndef PCI_EXP_LNKSTA2
1831 #define PCI_EXP_LNKSTA2 50
1832 #endif
1833 /* Link 2 Capabilities, Status, and Control goes away */
1834 if (size > PCI_EXP_LNKCAP2) {
1835 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
1836 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
1837 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
1838 }
1839 }
1840
1841 } else if (type == PCI_EXP_TYPE_LEG_END) {
1842 /*
1843 * Legacy endpoints don't belong on the root complex. Windows
1844 * seems to be happier with devices if we skip the capability.
1845 */
1846 return 0;
1847 }
1848
1849 } else {
1850 /*
1851 * Convert Root Complex Integrated Endpoints to regular endpoints.
1852 * These devices don't support LNK/LNK2 capabilities, so make them up.
1853 */
1854 if (type == PCI_EXP_TYPE_RC_END) {
1855 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1856 PCI_EXP_TYPE_ENDPOINT << 4,
1857 PCI_EXP_FLAGS_TYPE);
1858 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
1859 QEMU_PCI_EXP_LNKCAP_MLW(QEMU_PCI_EXP_LNK_X1) |
1860 QEMU_PCI_EXP_LNKCAP_MLS(QEMU_PCI_EXP_LNK_2_5GT), ~0);
1861 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1862 }
1863 }
1864
1865 /*
1866 * Intel 82599 SR-IOV VFs report an invalid PCIe capability version 0
1867 * (Niantic errate #35) causing Windows to error with a Code 10 for the
1868 * device on Q35. Fixup any such devices to report version 1. If we
1869 * were to remove the capability entirely the guest would lose extended
1870 * config space.
1871 */
1872 if ((flags & PCI_EXP_FLAGS_VERS) == 0) {
1873 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1874 1, PCI_EXP_FLAGS_VERS);
1875 }
1876
1877 pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size,
1878 errp);
1879 if (pos < 0) {
1880 return pos;
1881 }
1882
1883 vdev->pdev.exp.exp_cap = pos;
1884
1885 return pos;
1886 }
1887
1888 static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos)
1889 {
1890 uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
1891
1892 if (cap & PCI_EXP_DEVCAP_FLR) {
1893 trace_vfio_check_pcie_flr(vdev->vbasedev.name);
1894 vdev->has_flr = true;
1895 }
1896 }
1897
1898 static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos)
1899 {
1900 uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
1901
1902 if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
1903 trace_vfio_check_pm_reset(vdev->vbasedev.name);
1904 vdev->has_pm_reset = true;
1905 }
1906 }
1907
1908 static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos)
1909 {
1910 uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
1911
1912 if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
1913 trace_vfio_check_af_flr(vdev->vbasedev.name);
1914 vdev->has_flr = true;
1915 }
1916 }
1917
1918 static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos, Error **errp)
1919 {
1920 PCIDevice *pdev = &vdev->pdev;
1921 uint8_t cap_id, next, size;
1922 int ret;
1923
1924 cap_id = pdev->config[pos];
1925 next = pdev->config[pos + PCI_CAP_LIST_NEXT];
1926
1927 /*
1928 * If it becomes important to configure capabilities to their actual
1929 * size, use this as the default when it's something we don't recognize.
1930 * Since QEMU doesn't actually handle many of the config accesses,
1931 * exact size doesn't seem worthwhile.
1932 */
1933 size = vfio_std_cap_max_size(pdev, pos);
1934
1935 /*
1936 * pci_add_capability always inserts the new capability at the head
1937 * of the chain. Therefore to end up with a chain that matches the
1938 * physical device, we insert from the end by making this recursive.
1939 * This is also why we pre-calculate size above as cached config space
1940 * will be changed as we unwind the stack.
1941 */
1942 if (next) {
1943 ret = vfio_add_std_cap(vdev, next, errp);
1944 if (ret) {
1945 return ret;
1946 }
1947 } else {
1948 /* Begin the rebuild, use QEMU emulated list bits */
1949 pdev->config[PCI_CAPABILITY_LIST] = 0;
1950 vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
1951 vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
1952
1953 ret = vfio_add_virt_caps(vdev, errp);
1954 if (ret) {
1955 return ret;
1956 }
1957 }
1958
1959 /* Scale down size, esp in case virt caps were added above */
1960 size = MIN(size, vfio_std_cap_max_size(pdev, pos));
1961
1962 /* Use emulated next pointer to allow dropping caps */
1963 pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff);
1964
1965 switch (cap_id) {
1966 case PCI_CAP_ID_MSI:
1967 ret = vfio_msi_setup(vdev, pos, errp);
1968 break;
1969 case PCI_CAP_ID_EXP:
1970 vfio_check_pcie_flr(vdev, pos);
1971 ret = vfio_setup_pcie_cap(vdev, pos, size, errp);
1972 break;
1973 case PCI_CAP_ID_MSIX:
1974 ret = vfio_msix_setup(vdev, pos, errp);
1975 break;
1976 case PCI_CAP_ID_PM:
1977 vfio_check_pm_reset(vdev, pos);
1978 vdev->pm_cap = pos;
1979 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
1980 break;
1981 case PCI_CAP_ID_AF:
1982 vfio_check_af_flr(vdev, pos);
1983 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
1984 break;
1985 default:
1986 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
1987 break;
1988 }
1989
1990 if (ret < 0) {
1991 error_prepend(errp,
1992 "failed to add PCI capability 0x%x[0x%x]@0x%x: ",
1993 cap_id, size, pos);
1994 return ret;
1995 }
1996
1997 return 0;
1998 }
1999
2000 static void vfio_add_ext_cap(VFIOPCIDevice *vdev)
2001 {
2002 PCIDevice *pdev = &vdev->pdev;
2003 uint32_t header;
2004 uint16_t cap_id, next, size;
2005 uint8_t cap_ver;
2006 uint8_t *config;
2007
2008 /* Only add extended caps if we have them and the guest can see them */
2009 if (!pci_is_express(pdev) || !pci_bus_is_express(pci_get_bus(pdev)) ||
2010 !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
2011 return;
2012 }
2013
2014 /*
2015 * pcie_add_capability always inserts the new capability at the tail
2016 * of the chain. Therefore to end up with a chain that matches the
2017 * physical device, we cache the config space to avoid overwriting
2018 * the original config space when we parse the extended capabilities.
2019 */
2020 config = g_memdup(pdev->config, vdev->config_size);
2021
2022 /*
2023 * Extended capabilities are chained with each pointing to the next, so we
2024 * can drop anything other than the head of the chain simply by modifying
2025 * the previous next pointer. Seed the head of the chain here such that
2026 * we can simply skip any capabilities we want to drop below, regardless
2027 * of their position in the chain. If this stub capability still exists
2028 * after we add the capabilities we want to expose, update the capability
2029 * ID to zero. Note that we cannot seed with the capability header being
2030 * zero as this conflicts with definition of an absent capability chain
2031 * and prevents capabilities beyond the head of the list from being added.
2032 * By replacing the dummy capability ID with zero after walking the device
2033 * chain, we also transparently mark extended capabilities as absent if
2034 * no capabilities were added. Note that the PCIe spec defines an absence
2035 * of extended capabilities to be determined by a value of zero for the
2036 * capability ID, version, AND next pointer. A non-zero next pointer
2037 * should be sufficient to indicate additional capabilities are present,
2038 * which will occur if we call pcie_add_capability() below. The entire
2039 * first dword is emulated to support this.
2040 *
2041 * NB. The kernel side does similar masking, so be prepared that our
2042 * view of the device may also contain a capability ID zero in the head
2043 * of the chain. Skip it for the same reason that we cannot seed the
2044 * chain with a zero capability.
2045 */
2046 pci_set_long(pdev->config + PCI_CONFIG_SPACE_SIZE,
2047 PCI_EXT_CAP(0xFFFF, 0, 0));
2048 pci_set_long(pdev->wmask + PCI_CONFIG_SPACE_SIZE, 0);
2049 pci_set_long(vdev->emulated_config_bits + PCI_CONFIG_SPACE_SIZE, ~0);
2050
2051 for (next = PCI_CONFIG_SPACE_SIZE; next;
2052 next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
2053 header = pci_get_long(config + next);
2054 cap_id = PCI_EXT_CAP_ID(header);
2055 cap_ver = PCI_EXT_CAP_VER(header);
2056
2057 /*
2058 * If it becomes important to configure extended capabilities to their
2059 * actual size, use this as the default when it's something we don't
2060 * recognize. Since QEMU doesn't actually handle many of the config
2061 * accesses, exact size doesn't seem worthwhile.
2062 */
2063 size = vfio_ext_cap_max_size(config, next);
2064
2065 /* Use emulated next pointer to allow dropping extended caps */
2066 pci_long_test_and_set_mask(vdev->emulated_config_bits + next,
2067 PCI_EXT_CAP_NEXT_MASK);
2068
2069 switch (cap_id) {
2070 case 0: /* kernel masked capability */
2071 case PCI_EXT_CAP_ID_SRIOV: /* Read-only VF BARs confuse OVMF */
2072 case PCI_EXT_CAP_ID_ARI: /* XXX Needs next function virtualization */
2073 case PCI_EXT_CAP_ID_REBAR: /* Can't expose read-only */
2074 trace_vfio_add_ext_cap_dropped(vdev->vbasedev.name, cap_id, next);
2075 break;
2076 default:
2077 pcie_add_capability(pdev, cap_id, cap_ver, next, size);
2078 }
2079
2080 }
2081
2082 /* Cleanup chain head ID if necessary */
2083 if (pci_get_word(pdev->config + PCI_CONFIG_SPACE_SIZE) == 0xFFFF) {
2084 pci_set_word(pdev->config + PCI_CONFIG_SPACE_SIZE, 0);
2085 }
2086
2087 g_free(config);
2088 return;
2089 }
2090
2091 static int vfio_add_capabilities(VFIOPCIDevice *vdev, Error **errp)
2092 {
2093 PCIDevice *pdev = &vdev->pdev;
2094 int ret;
2095
2096 if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
2097 !pdev->config[PCI_CAPABILITY_LIST]) {
2098 return 0; /* Nothing to add */
2099 }
2100
2101 ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST], errp);
2102 if (ret) {
2103 return ret;
2104 }
2105
2106 vfio_add_ext_cap(vdev);
2107 return 0;
2108 }
2109
2110 static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
2111 {
2112 PCIDevice *pdev = &vdev->pdev;
2113 uint16_t cmd;
2114
2115 vfio_disable_interrupts(vdev);
2116
2117 /* Make sure the device is in D0 */
2118 if (vdev->pm_cap) {
2119 uint16_t pmcsr;
2120 uint8_t state;
2121
2122 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2123 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2124 if (state) {
2125 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2126 vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
2127 /* vfio handles the necessary delay here */
2128 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2129 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2130 if (state) {
2131 error_report("vfio: Unable to power on device, stuck in D%d",
2132 state);
2133 }
2134 }
2135 }
2136
2137 /*
2138 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
2139 * Also put INTx Disable in known state.
2140 */
2141 cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
2142 cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
2143 PCI_COMMAND_INTX_DISABLE);
2144 vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
2145 }
2146
2147 static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
2148 {
2149 Error *err = NULL;
2150 int nr;
2151
2152 vfio_intx_enable(vdev, &err);
2153 if (err) {
2154 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2155 }
2156
2157 for (nr = 0; nr < PCI_NUM_REGIONS - 1; ++nr) {
2158 off_t addr = vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr);
2159 uint32_t val = 0;
2160 uint32_t len = sizeof(val);
2161
2162 if (pwrite(vdev->vbasedev.fd, &val, len, addr) != len) {
2163 error_report("%s(%s) reset bar %d failed: %m", __func__,
2164 vdev->vbasedev.name, nr);
2165 }
2166 }
2167
2168 vfio_quirk_reset(vdev);
2169 }
2170
2171 static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
2172 {
2173 char tmp[13];
2174
2175 sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain,
2176 addr->bus, addr->slot, addr->function);
2177
2178 return (strcmp(tmp, name) == 0);
2179 }
2180
2181 static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
2182 {
2183 VFIOGroup *group;
2184 struct vfio_pci_hot_reset_info *info;
2185 struct vfio_pci_dependent_device *devices;
2186 struct vfio_pci_hot_reset *reset;
2187 int32_t *fds;
2188 int ret, i, count;
2189 bool multi = false;
2190
2191 trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
2192
2193 if (!single) {
2194 vfio_pci_pre_reset(vdev);
2195 }
2196 vdev->vbasedev.needs_reset = false;
2197
2198 info = g_malloc0(sizeof(*info));
2199 info->argsz = sizeof(*info);
2200
2201 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2202 if (ret && errno != ENOSPC) {
2203 ret = -errno;
2204 if (!vdev->has_pm_reset) {
2205 error_report("vfio: Cannot reset device %s, "
2206 "no available reset mechanism.", vdev->vbasedev.name);
2207 }
2208 goto out_single;
2209 }
2210
2211 count = info->count;
2212 info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
2213 info->argsz = sizeof(*info) + (count * sizeof(*devices));
2214 devices = &info->devices[0];
2215
2216 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2217 if (ret) {
2218 ret = -errno;
2219 error_report("vfio: hot reset info failed: %m");
2220 goto out_single;
2221 }
2222
2223 trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
2224
2225 /* Verify that we have all the groups required */
2226 for (i = 0; i < info->count; i++) {
2227 PCIHostDeviceAddress host;
2228 VFIOPCIDevice *tmp;
2229 VFIODevice *vbasedev_iter;
2230
2231 host.domain = devices[i].segment;
2232 host.bus = devices[i].bus;
2233 host.slot = PCI_SLOT(devices[i].devfn);
2234 host.function = PCI_FUNC(devices[i].devfn);
2235
2236 trace_vfio_pci_hot_reset_dep_devices(host.domain,
2237 host.bus, host.slot, host.function, devices[i].group_id);
2238
2239 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2240 continue;
2241 }
2242
2243 QLIST_FOREACH(group, &vfio_group_list, next) {
2244 if (group->groupid == devices[i].group_id) {
2245 break;
2246 }
2247 }
2248
2249 if (!group) {
2250 if (!vdev->has_pm_reset) {
2251 error_report("vfio: Cannot reset device %s, "
2252 "depends on group %d which is not owned.",
2253 vdev->vbasedev.name, devices[i].group_id);
2254 }
2255 ret = -EPERM;
2256 goto out;
2257 }
2258
2259 /* Prep dependent devices for reset and clear our marker. */
2260 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2261 if (!vbasedev_iter->dev->realized ||
2262 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2263 continue;
2264 }
2265 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2266 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2267 if (single) {
2268 ret = -EINVAL;
2269 goto out_single;
2270 }
2271 vfio_pci_pre_reset(tmp);
2272 tmp->vbasedev.needs_reset = false;
2273 multi = true;
2274 break;
2275 }
2276 }
2277 }
2278
2279 if (!single && !multi) {
2280 ret = -EINVAL;
2281 goto out_single;
2282 }
2283
2284 /* Determine how many group fds need to be passed */
2285 count = 0;
2286 QLIST_FOREACH(group, &vfio_group_list, next) {
2287 for (i = 0; i < info->count; i++) {
2288 if (group->groupid == devices[i].group_id) {
2289 count++;
2290 break;
2291 }
2292 }
2293 }
2294
2295 reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
2296 reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
2297 fds = &reset->group_fds[0];
2298
2299 /* Fill in group fds */
2300 QLIST_FOREACH(group, &vfio_group_list, next) {
2301 for (i = 0; i < info->count; i++) {
2302 if (group->groupid == devices[i].group_id) {
2303 fds[reset->count++] = group->fd;
2304 break;
2305 }
2306 }
2307 }
2308
2309 /* Bus reset! */
2310 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
2311 g_free(reset);
2312
2313 trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
2314 ret ? "%m" : "Success");
2315
2316 out:
2317 /* Re-enable INTx on affected devices */
2318 for (i = 0; i < info->count; i++) {
2319 PCIHostDeviceAddress host;
2320 VFIOPCIDevice *tmp;
2321 VFIODevice *vbasedev_iter;
2322
2323 host.domain = devices[i].segment;
2324 host.bus = devices[i].bus;
2325 host.slot = PCI_SLOT(devices[i].devfn);
2326 host.function = PCI_FUNC(devices[i].devfn);
2327
2328 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2329 continue;
2330 }
2331
2332 QLIST_FOREACH(group, &vfio_group_list, next) {
2333 if (group->groupid == devices[i].group_id) {
2334 break;
2335 }
2336 }
2337
2338 if (!group) {
2339 break;
2340 }
2341
2342 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2343 if (!vbasedev_iter->dev->realized ||
2344 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2345 continue;
2346 }
2347 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2348 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2349 vfio_pci_post_reset(tmp);
2350 break;
2351 }
2352 }
2353 }
2354 out_single:
2355 if (!single) {
2356 vfio_pci_post_reset(vdev);
2357 }
2358 g_free(info);
2359
2360 return ret;
2361 }
2362
2363 /*
2364 * We want to differentiate hot reset of mulitple in-use devices vs hot reset
2365 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case
2366 * of doing hot resets when there is only a single device per bus. The in-use
2367 * here refers to how many VFIODevices are affected. A hot reset that affects
2368 * multiple devices, but only a single in-use device, means that we can call
2369 * it from our bus ->reset() callback since the extent is effectively a single
2370 * device. This allows us to make use of it in the hotplug path. When there
2371 * are multiple in-use devices, we can only trigger the hot reset during a
2372 * system reset and thus from our reset handler. We separate _one vs _multi
2373 * here so that we don't overlap and do a double reset on the system reset
2374 * path where both our reset handler and ->reset() callback are used. Calling
2375 * _one() will only do a hot reset for the one in-use devices case, calling
2376 * _multi() will do nothing if a _one() would have been sufficient.
2377 */
2378 static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev)
2379 {
2380 return vfio_pci_hot_reset(vdev, true);
2381 }
2382
2383 static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev)
2384 {
2385 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2386 return vfio_pci_hot_reset(vdev, false);
2387 }
2388
2389 static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
2390 {
2391 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2392 if (!vbasedev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
2393 vbasedev->needs_reset = true;
2394 }
2395 }
2396
2397 static VFIODeviceOps vfio_pci_ops = {
2398 .vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
2399 .vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
2400 .vfio_eoi = vfio_intx_eoi,
2401 };
2402
2403 int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp)
2404 {
2405 VFIODevice *vbasedev = &vdev->vbasedev;
2406 struct vfio_region_info *reg_info;
2407 int ret;
2408
2409 ret = vfio_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, &reg_info);
2410 if (ret) {
2411 error_setg_errno(errp, -ret,
2412 "failed getting region info for VGA region index %d",
2413 VFIO_PCI_VGA_REGION_INDEX);
2414 return ret;
2415 }
2416
2417 if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) ||
2418 !(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) ||
2419 reg_info->size < 0xbffff + 1) {
2420 error_setg(errp, "unexpected VGA info, flags 0x%lx, size 0x%lx",
2421 (unsigned long)reg_info->flags,
2422 (unsigned long)reg_info->size);
2423 g_free(reg_info);
2424 return -EINVAL;
2425 }
2426
2427 vdev->vga = g_new0(VFIOVGA, 1);
2428
2429 vdev->vga->fd_offset = reg_info->offset;
2430 vdev->vga->fd = vdev->vbasedev.fd;
2431
2432 g_free(reg_info);
2433
2434 vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
2435 vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
2436 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks);
2437
2438 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2439 OBJECT(vdev), &vfio_vga_ops,
2440 &vdev->vga->region[QEMU_PCI_VGA_MEM],
2441 "vfio-vga-mmio@0xa0000",
2442 QEMU_PCI_VGA_MEM_SIZE);
2443
2444 vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
2445 vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
2446 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks);
2447
2448 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2449 OBJECT(vdev), &vfio_vga_ops,
2450 &vdev->vga->region[QEMU_PCI_VGA_IO_LO],
2451 "vfio-vga-io@0x3b0",
2452 QEMU_PCI_VGA_IO_LO_SIZE);
2453
2454 vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
2455 vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
2456 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks);
2457
2458 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
2459 OBJECT(vdev), &vfio_vga_ops,
2460 &vdev->vga->region[QEMU_PCI_VGA_IO_HI],
2461 "vfio-vga-io@0x3c0",
2462 QEMU_PCI_VGA_IO_HI_SIZE);
2463
2464 pci_register_vga(&vdev->pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2465 &vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2466 &vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem);
2467
2468 return 0;
2469 }
2470
2471 static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
2472 {
2473 VFIODevice *vbasedev = &vdev->vbasedev;
2474 struct vfio_region_info *reg_info;
2475 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
2476 int i, ret = -1;
2477
2478 /* Sanity check device */
2479 if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) {
2480 error_setg(errp, "this isn't a PCI device");
2481 return;
2482 }
2483
2484 if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
2485 error_setg(errp, "unexpected number of io regions %u",
2486 vbasedev->num_regions);
2487 return;
2488 }
2489
2490 if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
2491 error_setg(errp, "unexpected number of irqs %u", vbasedev->num_irqs);
2492 return;
2493 }
2494
2495 for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
2496 char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
2497
2498 ret = vfio_region_setup(OBJECT(vdev), vbasedev,
2499 &vdev->bars[i].region, i, name);
2500 g_free(name);
2501
2502 if (ret) {
2503 error_setg_errno(errp, -ret, "failed to get region %d info", i);
2504 return;
2505 }
2506
2507 QLIST_INIT(&vdev->bars[i].quirks);
2508 }
2509
2510 ret = vfio_get_region_info(vbasedev,
2511 VFIO_PCI_CONFIG_REGION_INDEX, &reg_info);
2512 if (ret) {
2513 error_setg_errno(errp, -ret, "failed to get config info");
2514 return;
2515 }
2516
2517 trace_vfio_populate_device_config(vdev->vbasedev.name,
2518 (unsigned long)reg_info->size,
2519 (unsigned long)reg_info->offset,
2520 (unsigned long)reg_info->flags);
2521
2522 vdev->config_size = reg_info->size;
2523 if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
2524 vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
2525 }
2526 vdev->config_offset = reg_info->offset;
2527
2528 g_free(reg_info);
2529
2530 if (vdev->features & VFIO_FEATURE_ENABLE_VGA) {
2531 ret = vfio_populate_vga(vdev, errp);
2532 if (ret) {
2533 error_append_hint(errp, "device does not support "
2534 "requested feature x-vga\n");
2535 return;
2536 }
2537 }
2538
2539 irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
2540
2541 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
2542 if (ret) {
2543 /* This can fail for an old kernel or legacy PCI dev */
2544 trace_vfio_populate_device_get_irq_info_failure(strerror(errno));
2545 } else if (irq_info.count == 1) {
2546 vdev->pci_aer = true;
2547 } else {
2548 warn_report(VFIO_MSG_PREFIX
2549 "Could not enable error recovery for the device",
2550 vbasedev->name);
2551 }
2552 }
2553
2554 static void vfio_put_device(VFIOPCIDevice *vdev)
2555 {
2556 g_free(vdev->vbasedev.name);
2557 g_free(vdev->msix);
2558
2559 vfio_put_base_device(&vdev->vbasedev);
2560 }
2561
2562 static void vfio_err_notifier_handler(void *opaque)
2563 {
2564 VFIOPCIDevice *vdev = opaque;
2565
2566 if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
2567 return;
2568 }
2569
2570 /*
2571 * TBD. Retrieve the error details and decide what action
2572 * needs to be taken. One of the actions could be to pass
2573 * the error to the guest and have the guest driver recover
2574 * from the error. This requires that PCIe capabilities be
2575 * exposed to the guest. For now, we just terminate the
2576 * guest to contain the error.
2577 */
2578
2579 error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name);
2580
2581 vm_stop(RUN_STATE_INTERNAL_ERROR);
2582 }
2583
2584 /*
2585 * Registers error notifier for devices supporting error recovery.
2586 * If we encounter a failure in this function, we report an error
2587 * and continue after disabling error recovery support for the
2588 * device.
2589 */
2590 static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
2591 {
2592 Error *err = NULL;
2593 int32_t fd;
2594
2595 if (!vdev->pci_aer) {
2596 return;
2597 }
2598
2599 if (event_notifier_init(&vdev->err_notifier, 0)) {
2600 error_report("vfio: Unable to init event notifier for error detection");
2601 vdev->pci_aer = false;
2602 return;
2603 }
2604
2605 fd = event_notifier_get_fd(&vdev->err_notifier);
2606 qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
2607
2608 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
2609 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
2610 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2611 qemu_set_fd_handler(fd, NULL, NULL, vdev);
2612 event_notifier_cleanup(&vdev->err_notifier);
2613 vdev->pci_aer = false;
2614 }
2615 }
2616
2617 static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev)
2618 {
2619 Error *err = NULL;
2620
2621 if (!vdev->pci_aer) {
2622 return;
2623 }
2624
2625 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
2626 VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
2627 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2628 }
2629 qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
2630 NULL, NULL, vdev);
2631 event_notifier_cleanup(&vdev->err_notifier);
2632 }
2633
2634 static void vfio_req_notifier_handler(void *opaque)
2635 {
2636 VFIOPCIDevice *vdev = opaque;
2637 Error *err = NULL;
2638
2639 if (!event_notifier_test_and_clear(&vdev->req_notifier)) {
2640 return;
2641 }
2642
2643 qdev_unplug(DEVICE(vdev), &err);
2644 if (err) {
2645 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2646 }
2647 }
2648
2649 static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
2650 {
2651 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
2652 .index = VFIO_PCI_REQ_IRQ_INDEX };
2653 Error *err = NULL;
2654 int32_t fd;
2655
2656 if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) {
2657 return;
2658 }
2659
2660 if (ioctl(vdev->vbasedev.fd,
2661 VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) {
2662 return;
2663 }
2664
2665 if (event_notifier_init(&vdev->req_notifier, 0)) {
2666 error_report("vfio: Unable to init event notifier for device request");
2667 return;
2668 }
2669
2670 fd = event_notifier_get_fd(&vdev->req_notifier);
2671 qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
2672
2673 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
2674 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
2675 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2676 qemu_set_fd_handler(fd, NULL, NULL, vdev);
2677 event_notifier_cleanup(&vdev->req_notifier);
2678 } else {
2679 vdev->req_enabled = true;
2680 }
2681 }
2682
2683 static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
2684 {
2685 Error *err = NULL;
2686
2687 if (!vdev->req_enabled) {
2688 return;
2689 }
2690
2691 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
2692 VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
2693 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2694 }
2695 qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier),
2696 NULL, NULL, vdev);
2697 event_notifier_cleanup(&vdev->req_notifier);
2698
2699 vdev->req_enabled = false;
2700 }
2701
2702 static void vfio_realize(PCIDevice *pdev, Error **errp)
2703 {
2704 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
2705 VFIODevice *vbasedev_iter;
2706 VFIOGroup *group;
2707 char *tmp, *subsys, group_path[PATH_MAX], *group_name;
2708 Error *err = NULL;
2709 ssize_t len;
2710 struct stat st;
2711 int groupid;
2712 int i, ret;
2713 bool is_mdev;
2714
2715 if (!vdev->vbasedev.sysfsdev) {
2716 if (!(~vdev->host.domain || ~vdev->host.bus ||
2717 ~vdev->host.slot || ~vdev->host.function)) {
2718 error_setg(errp, "No provided host device");
2719 error_append_hint(errp, "Use -device vfio-pci,host=DDDD:BB:DD.F "
2720 "or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
2721 return;
2722 }
2723 vdev->vbasedev.sysfsdev =
2724 g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
2725 vdev->host.domain, vdev->host.bus,
2726 vdev->host.slot, vdev->host.function);
2727 }
2728
2729 if (stat(vdev->vbasedev.sysfsdev, &st) < 0) {
2730 error_setg_errno(errp, errno, "no such host device");
2731 error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.sysfsdev);
2732 return;
2733 }
2734
2735 if (!pdev->failover_pair_id) {
2736 error_setg(&vdev->migration_blocker,
2737 "VFIO device doesn't support migration");
2738 ret = migrate_add_blocker(vdev->migration_blocker, errp);
2739 if (ret) {
2740 error_free(vdev->migration_blocker);
2741 vdev->migration_blocker = NULL;
2742 return;
2743 }
2744 }
2745
2746 vdev->vbasedev.name = g_path_get_basename(vdev->vbasedev.sysfsdev);
2747 vdev->vbasedev.ops = &vfio_pci_ops;
2748 vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI;
2749 vdev->vbasedev.dev = DEVICE(vdev);
2750
2751 tmp = g_strdup_printf("%s/iommu_group", vdev->vbasedev.sysfsdev);
2752 len = readlink(tmp, group_path, sizeof(group_path));
2753 g_free(tmp);
2754
2755 if (len <= 0 || len >= sizeof(group_path)) {
2756 error_setg_errno(errp, len < 0 ? errno : ENAMETOOLONG,
2757 "no iommu_group found");
2758 goto error;
2759 }
2760
2761 group_path[len] = 0;
2762
2763 group_name = basename(group_path);
2764 if (sscanf(group_name, "%d", &groupid) != 1) {
2765 error_setg_errno(errp, errno, "failed to read %s", group_path);
2766 goto error;
2767 }
2768
2769 trace_vfio_realize(vdev->vbasedev.name, groupid);
2770
2771 group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev), errp);
2772 if (!group) {
2773 goto error;
2774 }
2775
2776 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2777 if (strcmp(vbasedev_iter->name, vdev->vbasedev.name) == 0) {
2778 error_setg(errp, "device is already attached");
2779 vfio_put_group(group);
2780 goto error;
2781 }
2782 }
2783
2784 /*
2785 * Mediated devices *might* operate compatibly with discarding of RAM, but
2786 * we cannot know for certain, it depends on whether the mdev vendor driver
2787 * stays in sync with the active working set of the guest driver. Prevent
2788 * the x-balloon-allowed option unless this is minimally an mdev device.
2789 */
2790 tmp = g_strdup_printf("%s/subsystem", vdev->vbasedev.sysfsdev);
2791 subsys = realpath(tmp, NULL);
2792 g_free(tmp);
2793 is_mdev = subsys && (strcmp(subsys, "/sys/bus/mdev") == 0);
2794 free(subsys);
2795
2796 trace_vfio_mdev(vdev->vbasedev.name, is_mdev);
2797
2798 if (vdev->vbasedev.ram_block_discard_allowed && !is_mdev) {
2799 error_setg(errp, "x-balloon-allowed only potentially compatible "
2800 "with mdev devices");
2801 vfio_put_group(group);
2802 goto error;
2803 }
2804
2805 ret = vfio_get_device(group, vdev->vbasedev.name, &vdev->vbasedev, errp);
2806 if (ret) {
2807 vfio_put_group(group);
2808 goto error;
2809 }
2810
2811 vfio_populate_device(vdev, &err);
2812 if (err) {
2813 error_propagate(errp, err);
2814 goto error;
2815 }
2816
2817 /* Get a copy of config space */
2818 ret = pread(vdev->vbasedev.fd, vdev->pdev.config,
2819 MIN(pci_config_size(&vdev->pdev), vdev->config_size),
2820 vdev->config_offset);
2821 if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
2822 ret = ret < 0 ? -errno : -EFAULT;
2823 error_setg_errno(errp, -ret, "failed to read device config space");
2824 goto error;
2825 }
2826
2827 /* vfio emulates a lot for us, but some bits need extra love */
2828 vdev->emulated_config_bits = g_malloc0(vdev->config_size);
2829
2830 /* QEMU can choose to expose the ROM or not */
2831 memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
2832 /* QEMU can also add or extend BARs */
2833 memset(vdev->emulated_config_bits + PCI_BASE_ADDRESS_0, 0xff, 6 * 4);
2834
2835 /*
2836 * The PCI spec reserves vendor ID 0xffff as an invalid value. The
2837 * device ID is managed by the vendor and need only be a 16-bit value.
2838 * Allow any 16-bit value for subsystem so they can be hidden or changed.
2839 */
2840 if (vdev->vendor_id != PCI_ANY_ID) {
2841 if (vdev->vendor_id >= 0xffff) {
2842 error_setg(errp, "invalid PCI vendor ID provided");
2843 goto error;
2844 }
2845 vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
2846 trace_vfio_pci_emulated_vendor_id(vdev->vbasedev.name, vdev->vendor_id);
2847 } else {
2848 vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
2849 }
2850
2851 if (vdev->device_id != PCI_ANY_ID) {
2852 if (vdev->device_id > 0xffff) {
2853 error_setg(errp, "invalid PCI device ID provided");
2854 goto error;
2855 }
2856 vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
2857 trace_vfio_pci_emulated_device_id(vdev->vbasedev.name, vdev->device_id);
2858 } else {
2859 vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
2860 }
2861
2862 if (vdev->sub_vendor_id != PCI_ANY_ID) {
2863 if (vdev->sub_vendor_id > 0xffff) {
2864 error_setg(errp, "invalid PCI subsystem vendor ID provided");
2865 goto error;
2866 }
2867 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
2868 vdev->sub_vendor_id, ~0);
2869 trace_vfio_pci_emulated_sub_vendor_id(vdev->vbasedev.name,
2870 vdev->sub_vendor_id);
2871 }
2872
2873 if (vdev->sub_device_id != PCI_ANY_ID) {
2874 if (vdev->sub_device_id > 0xffff) {
2875 error_setg(errp, "invalid PCI subsystem device ID provided");
2876 goto error;
2877 }
2878 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
2879 trace_vfio_pci_emulated_sub_device_id(vdev->vbasedev.name,
2880 vdev->sub_device_id);
2881 }
2882
2883 /* QEMU can change multi-function devices to single function, or reverse */
2884 vdev->emulated_config_bits[PCI_HEADER_TYPE] =
2885 PCI_HEADER_TYPE_MULTI_FUNCTION;
2886
2887 /* Restore or clear multifunction, this is always controlled by QEMU */
2888 if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
2889 vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
2890 } else {
2891 vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
2892 }
2893
2894 /*
2895 * Clear host resource mapping info. If we choose not to register a
2896 * BAR, such as might be the case with the option ROM, we can get
2897 * confusing, unwritable, residual addresses from the host here.
2898 */
2899 memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
2900 memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
2901
2902 vfio_pci_size_rom(vdev);
2903
2904 vfio_bars_prepare(vdev);
2905
2906 vfio_msix_early_setup(vdev, &err);
2907 if (err) {
2908 error_propagate(errp, err);
2909 goto error;
2910 }
2911
2912 vfio_bars_register(vdev);
2913
2914 ret = vfio_add_capabilities(vdev, errp);
2915 if (ret) {
2916 goto out_teardown;
2917 }
2918
2919 if (vdev->vga) {
2920 vfio_vga_quirk_setup(vdev);
2921 }
2922
2923 for (i = 0; i < PCI_ROM_SLOT; i++) {
2924 vfio_bar_quirk_setup(vdev, i);
2925 }
2926
2927 if (!vdev->igd_opregion &&
2928 vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) {
2929 struct vfio_region_info *opregion;
2930
2931 if (vdev->pdev.qdev.hotplugged) {
2932 error_setg(errp,
2933 "cannot support IGD OpRegion feature on hotplugged "
2934 "device");
2935 goto out_teardown;
2936 }
2937
2938 ret = vfio_get_dev_region_info(&vdev->vbasedev,
2939 VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
2940 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion);
2941 if (ret) {
2942 error_setg_errno(errp, -ret,
2943 "does not support requested IGD OpRegion feature");
2944 goto out_teardown;
2945 }
2946
2947 ret = vfio_pci_igd_opregion_init(vdev, opregion, errp);
2948 g_free(opregion);
2949 if (ret) {
2950 goto out_teardown;
2951 }
2952 }
2953
2954 /* QEMU emulates all of MSI & MSIX */
2955 if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
2956 memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
2957 MSIX_CAP_LENGTH);
2958 }
2959
2960 if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
2961 memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
2962 vdev->msi_cap_size);
2963 }
2964
2965 if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
2966 vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
2967 vfio_intx_mmap_enable, vdev);
2968 pci_device_set_intx_routing_notifier(&vdev->pdev,
2969 vfio_intx_routing_notifier);
2970 vdev->irqchip_change_notifier.notify = vfio_irqchip_change;
2971 kvm_irqchip_add_change_notifier(&vdev->irqchip_change_notifier);
2972 ret = vfio_intx_enable(vdev, errp);
2973 if (ret) {
2974 goto out_deregister;
2975 }
2976 }
2977
2978 if (vdev->display != ON_OFF_AUTO_OFF) {
2979 ret = vfio_display_probe(vdev, errp);
2980 if (ret) {
2981 goto out_deregister;
2982 }
2983 }
2984 if (vdev->enable_ramfb && vdev->dpy == NULL) {
2985 error_setg(errp, "ramfb=on requires display=on");
2986 goto out_deregister;
2987 }
2988 if (vdev->display_xres || vdev->display_yres) {
2989 if (vdev->dpy == NULL) {
2990 error_setg(errp, "xres and yres properties require display=on");
2991 goto out_deregister;
2992 }
2993 if (vdev->dpy->edid_regs == NULL) {
2994 error_setg(errp, "xres and yres properties need edid support");
2995 goto out_deregister;
2996 }
2997 }
2998
2999 if (vdev->vendor_id == PCI_VENDOR_ID_NVIDIA) {
3000 ret = vfio_pci_nvidia_v100_ram_init(vdev, errp);
3001 if (ret && ret != -ENODEV) {
3002 error_report("Failed to setup NVIDIA V100 GPU RAM");
3003 }
3004 }
3005
3006 if (vdev->vendor_id == PCI_VENDOR_ID_IBM) {
3007 ret = vfio_pci_nvlink2_init(vdev, errp);
3008 if (ret && ret != -ENODEV) {
3009 error_report("Failed to setup NVlink2 bridge");
3010 }
3011 }
3012
3013 vfio_register_err_notifier(vdev);
3014 vfio_register_req_notifier(vdev);
3015 vfio_setup_resetfn_quirk(vdev);
3016
3017 return;
3018
3019 out_deregister:
3020 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3021 kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3022 out_teardown:
3023 vfio_teardown_msi(vdev);
3024 vfio_bars_exit(vdev);
3025 error:
3026 error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3027 if (vdev->migration_blocker) {
3028 migrate_del_blocker(vdev->migration_blocker);
3029 error_free(vdev->migration_blocker);
3030 vdev->migration_blocker = NULL;
3031 }
3032 }
3033
3034 static void vfio_instance_finalize(Object *obj)
3035 {
3036 VFIOPCIDevice *vdev = VFIO_PCI(obj);
3037 VFIOGroup *group = vdev->vbasedev.group;
3038
3039 vfio_display_finalize(vdev);
3040 vfio_bars_finalize(vdev);
3041 g_free(vdev->emulated_config_bits);
3042 g_free(vdev->rom);
3043 if (vdev->migration_blocker) {
3044 migrate_del_blocker(vdev->migration_blocker);
3045 error_free(vdev->migration_blocker);
3046 }
3047 /*
3048 * XXX Leaking igd_opregion is not an oversight, we can't remove the
3049 * fw_cfg entry therefore leaking this allocation seems like the safest
3050 * option.
3051 *
3052 * g_free(vdev->igd_opregion);
3053 */
3054 vfio_put_device(vdev);
3055 vfio_put_group(group);
3056 }
3057
3058 static void vfio_exitfn(PCIDevice *pdev)
3059 {
3060 VFIOPCIDevice *vdev = VFIO_PCI(pdev);
3061
3062 vfio_unregister_req_notifier(vdev);
3063 vfio_unregister_err_notifier(vdev);
3064 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3065 if (vdev->irqchip_change_notifier.notify) {
3066 kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3067 }
3068 vfio_disable_interrupts(vdev);
3069 if (vdev->intx.mmap_timer) {
3070 timer_free(vdev->intx.mmap_timer);
3071 }
3072 vfio_teardown_msi(vdev);
3073 vfio_bars_exit(vdev);
3074 }
3075
3076 static void vfio_pci_reset(DeviceState *dev)
3077 {
3078 VFIOPCIDevice *vdev = VFIO_PCI(dev);
3079
3080 trace_vfio_pci_reset(vdev->vbasedev.name);
3081
3082 vfio_pci_pre_reset(vdev);
3083
3084 if (vdev->display != ON_OFF_AUTO_OFF) {
3085 vfio_display_reset(vdev);
3086 }
3087
3088 if (vdev->resetfn && !vdev->resetfn(vdev)) {
3089 goto post_reset;
3090 }
3091
3092 if (vdev->vbasedev.reset_works &&
3093 (vdev->has_flr || !vdev->has_pm_reset) &&
3094 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3095 trace_vfio_pci_reset_flr(vdev->vbasedev.name);
3096 goto post_reset;
3097 }
3098
3099 /* See if we can do our own bus reset */
3100 if (!vfio_pci_hot_reset_one(vdev)) {
3101 goto post_reset;
3102 }
3103
3104 /* If nothing else works and the device supports PM reset, use it */
3105 if (vdev->vbasedev.reset_works && vdev->has_pm_reset &&
3106 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3107 trace_vfio_pci_reset_pm(vdev->vbasedev.name);
3108 goto post_reset;
3109 }
3110
3111 post_reset:
3112 vfio_pci_post_reset(vdev);
3113 }
3114
3115 static void vfio_instance_init(Object *obj)
3116 {
3117 PCIDevice *pci_dev = PCI_DEVICE(obj);
3118 VFIOPCIDevice *vdev = VFIO_PCI(obj);
3119
3120 device_add_bootindex_property(obj, &vdev->bootindex,
3121 "bootindex", NULL,
3122 &pci_dev->qdev);
3123 vdev->host.domain = ~0U;
3124 vdev->host.bus = ~0U;
3125 vdev->host.slot = ~0U;
3126 vdev->host.function = ~0U;
3127
3128 vdev->nv_gpudirect_clique = 0xFF;
3129
3130 /* QEMU_PCI_CAP_EXPRESS initialization does not depend on QEMU command
3131 * line, therefore, no need to wait to realize like other devices */
3132 pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
3133 }
3134
3135 static Property vfio_pci_dev_properties[] = {
3136 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
3137 DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
3138 DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
3139 display, ON_OFF_AUTO_OFF),
3140 DEFINE_PROP_UINT32("xres", VFIOPCIDevice, display_xres, 0),
3141 DEFINE_PROP_UINT32("yres", VFIOPCIDevice, display_yres, 0),
3142 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
3143 intx.mmap_timeout, 1100),
3144 DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features,
3145 VFIO_FEATURE_ENABLE_VGA_BIT, false),
3146 DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
3147 VFIO_FEATURE_ENABLE_REQ_BIT, true),
3148 DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features,
3149 VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false),
3150 DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
3151 DEFINE_PROP_BOOL("x-balloon-allowed", VFIOPCIDevice,
3152 vbasedev.ram_block_discard_allowed, false),
3153 DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
3154 DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
3155 DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
3156 DEFINE_PROP_BOOL("x-no-geforce-quirks", VFIOPCIDevice,
3157 no_geforce_quirks, false),
3158 DEFINE_PROP_BOOL("x-no-kvm-ioeventfd", VFIOPCIDevice, no_kvm_ioeventfd,
3159 false),
3160 DEFINE_PROP_BOOL("x-no-vfio-ioeventfd", VFIOPCIDevice, no_vfio_ioeventfd,
3161 false),
3162 DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
3163 DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
3164 DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
3165 sub_vendor_id, PCI_ANY_ID),
3166 DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
3167 sub_device_id, PCI_ANY_ID),
3168 DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice, igd_gms, 0),
3169 DEFINE_PROP_UNSIGNED_NODEFAULT("x-nv-gpudirect-clique", VFIOPCIDevice,
3170 nv_gpudirect_clique,
3171 qdev_prop_nv_gpudirect_clique, uint8_t),
3172 DEFINE_PROP_OFF_AUTO_PCIBAR("x-msix-relocation", VFIOPCIDevice, msix_relo,
3173 OFF_AUTOPCIBAR_OFF),
3174 /*
3175 * TODO - support passed fds... is this necessary?
3176 * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
3177 * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
3178 */
3179 DEFINE_PROP_END_OF_LIST(),
3180 };
3181
3182 static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
3183 {
3184 DeviceClass *dc = DEVICE_CLASS(klass);
3185 PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
3186
3187 dc->reset = vfio_pci_reset;
3188 device_class_set_props(dc, vfio_pci_dev_properties);
3189 dc->desc = "VFIO-based PCI device assignment";
3190 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
3191 pdc->realize = vfio_realize;
3192 pdc->exit = vfio_exitfn;
3193 pdc->config_read = vfio_pci_read_config;
3194 pdc->config_write = vfio_pci_write_config;
3195 }
3196
3197 static const TypeInfo vfio_pci_dev_info = {
3198 .name = TYPE_VFIO_PCI,
3199 .parent = TYPE_PCI_DEVICE,
3200 .instance_size = sizeof(VFIOPCIDevice),
3201 .class_init = vfio_pci_dev_class_init,
3202 .instance_init = vfio_instance_init,
3203 .instance_finalize = vfio_instance_finalize,
3204 .interfaces = (InterfaceInfo[]) {
3205 { INTERFACE_PCIE_DEVICE },
3206 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
3207 { }
3208 },
3209 };
3210
3211 static Property vfio_pci_dev_nohotplug_properties[] = {
3212 DEFINE_PROP_BOOL("ramfb", VFIOPCIDevice, enable_ramfb, false),
3213 DEFINE_PROP_END_OF_LIST(),
3214 };
3215
3216 static void vfio_pci_nohotplug_dev_class_init(ObjectClass *klass, void *data)
3217 {
3218 DeviceClass *dc = DEVICE_CLASS(klass);
3219
3220 device_class_set_props(dc, vfio_pci_dev_nohotplug_properties);
3221 dc->hotpluggable = false;
3222 }
3223
3224 static const TypeInfo vfio_pci_nohotplug_dev_info = {
3225 .name = TYPE_VFIO_PCI_NOHOTPLUG,
3226 .parent = TYPE_VFIO_PCI,
3227 .instance_size = sizeof(VFIOPCIDevice),
3228 .class_init = vfio_pci_nohotplug_dev_class_init,
3229 };
3230
3231 static void register_vfio_pci_dev_type(void)
3232 {
3233 type_register_static(&vfio_pci_dev_info);
3234 type_register_static(&vfio_pci_nohotplug_dev_info);
3235 }
3236
3237 type_init(register_vfio_pci_dev_type)