meson: convert qapi-specific to meson
[qemu.git] / hw / vfio / pci.c
1 /*
2 * vfio based device assignment support
3 *
4 * Copyright Red Hat, Inc. 2012
5 *
6 * Authors:
7 * Alex Williamson <alex.williamson@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
19 */
20
21 #include "qemu/osdep.h"
22 #include <linux/vfio.h>
23 #include <sys/ioctl.h>
24
25 #include "hw/hw.h"
26 #include "hw/pci/msi.h"
27 #include "hw/pci/msix.h"
28 #include "hw/pci/pci_bridge.h"
29 #include "hw/qdev-properties.h"
30 #include "migration/vmstate.h"
31 #include "qemu/error-report.h"
32 #include "qemu/main-loop.h"
33 #include "qemu/module.h"
34 #include "qemu/option.h"
35 #include "qemu/range.h"
36 #include "qemu/units.h"
37 #include "sysemu/kvm.h"
38 #include "sysemu/runstate.h"
39 #include "sysemu/sysemu.h"
40 #include "pci.h"
41 #include "trace.h"
42 #include "qapi/error.h"
43 #include "migration/blocker.h"
44
45 #define TYPE_VFIO_PCI "vfio-pci"
46 #define PCI_VFIO(obj) OBJECT_CHECK(VFIOPCIDevice, obj, TYPE_VFIO_PCI)
47
48 #define TYPE_VFIO_PCI_NOHOTPLUG "vfio-pci-nohotplug"
49
50 static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
51 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
52
53 /*
54 * Disabling BAR mmaping can be slow, but toggling it around INTx can
55 * also be a huge overhead. We try to get the best of both worlds by
56 * waiting until an interrupt to disable mmaps (subsequent transitions
57 * to the same state are effectively no overhead). If the interrupt has
58 * been serviced and the time gap is long enough, we re-enable mmaps for
59 * performance. This works well for things like graphics cards, which
60 * may not use their interrupt at all and are penalized to an unusable
61 * level by read/write BAR traps. Other devices, like NICs, have more
62 * regular interrupts and see much better latency by staying in non-mmap
63 * mode. We therefore set the default mmap_timeout such that a ping
64 * is just enough to keep the mmap disabled. Users can experiment with
65 * other options with the x-intx-mmap-timeout-ms parameter (a value of
66 * zero disables the timer).
67 */
68 static void vfio_intx_mmap_enable(void *opaque)
69 {
70 VFIOPCIDevice *vdev = opaque;
71
72 if (vdev->intx.pending) {
73 timer_mod(vdev->intx.mmap_timer,
74 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
75 return;
76 }
77
78 vfio_mmap_set_enabled(vdev, true);
79 }
80
81 static void vfio_intx_interrupt(void *opaque)
82 {
83 VFIOPCIDevice *vdev = opaque;
84
85 if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
86 return;
87 }
88
89 trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin);
90
91 vdev->intx.pending = true;
92 pci_irq_assert(&vdev->pdev);
93 vfio_mmap_set_enabled(vdev, false);
94 if (vdev->intx.mmap_timeout) {
95 timer_mod(vdev->intx.mmap_timer,
96 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
97 }
98 }
99
100 static void vfio_intx_eoi(VFIODevice *vbasedev)
101 {
102 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
103
104 if (!vdev->intx.pending) {
105 return;
106 }
107
108 trace_vfio_intx_eoi(vbasedev->name);
109
110 vdev->intx.pending = false;
111 pci_irq_deassert(&vdev->pdev);
112 vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
113 }
114
115 static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev, Error **errp)
116 {
117 #ifdef CONFIG_KVM
118 int irq_fd = event_notifier_get_fd(&vdev->intx.interrupt);
119
120 if (vdev->no_kvm_intx || !kvm_irqfds_enabled() ||
121 vdev->intx.route.mode != PCI_INTX_ENABLED ||
122 !kvm_resamplefds_enabled()) {
123 return;
124 }
125
126 /* Get to a known interrupt state */
127 qemu_set_fd_handler(irq_fd, NULL, NULL, vdev);
128 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
129 vdev->intx.pending = false;
130 pci_irq_deassert(&vdev->pdev);
131
132 /* Get an eventfd for resample/unmask */
133 if (event_notifier_init(&vdev->intx.unmask, 0)) {
134 error_setg(errp, "event_notifier_init failed eoi");
135 goto fail;
136 }
137
138 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state,
139 &vdev->intx.interrupt,
140 &vdev->intx.unmask,
141 vdev->intx.route.irq)) {
142 error_setg_errno(errp, errno, "failed to setup resample irqfd");
143 goto fail_irqfd;
144 }
145
146 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
147 VFIO_IRQ_SET_ACTION_UNMASK,
148 event_notifier_get_fd(&vdev->intx.unmask),
149 errp)) {
150 goto fail_vfio;
151 }
152
153 /* Let'em rip */
154 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
155
156 vdev->intx.kvm_accel = true;
157
158 trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
159
160 return;
161
162 fail_vfio:
163 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vdev->intx.interrupt,
164 vdev->intx.route.irq);
165 fail_irqfd:
166 event_notifier_cleanup(&vdev->intx.unmask);
167 fail:
168 qemu_set_fd_handler(irq_fd, vfio_intx_interrupt, NULL, vdev);
169 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
170 #endif
171 }
172
173 static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
174 {
175 #ifdef CONFIG_KVM
176 if (!vdev->intx.kvm_accel) {
177 return;
178 }
179
180 /*
181 * Get to a known state, hardware masked, QEMU ready to accept new
182 * interrupts, QEMU IRQ de-asserted.
183 */
184 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
185 vdev->intx.pending = false;
186 pci_irq_deassert(&vdev->pdev);
187
188 /* Tell KVM to stop listening for an INTx irqfd */
189 if (kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vdev->intx.interrupt,
190 vdev->intx.route.irq)) {
191 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
192 }
193
194 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
195 event_notifier_cleanup(&vdev->intx.unmask);
196
197 /* QEMU starts listening for interrupt events. */
198 qemu_set_fd_handler(event_notifier_get_fd(&vdev->intx.interrupt),
199 vfio_intx_interrupt, NULL, vdev);
200
201 vdev->intx.kvm_accel = false;
202
203 /* If we've missed an event, let it re-fire through QEMU */
204 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
205
206 trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
207 #endif
208 }
209
210 static void vfio_intx_update(VFIOPCIDevice *vdev, PCIINTxRoute *route)
211 {
212 Error *err = NULL;
213
214 trace_vfio_intx_update(vdev->vbasedev.name,
215 vdev->intx.route.irq, route->irq);
216
217 vfio_intx_disable_kvm(vdev);
218
219 vdev->intx.route = *route;
220
221 if (route->mode != PCI_INTX_ENABLED) {
222 return;
223 }
224
225 vfio_intx_enable_kvm(vdev, &err);
226 if (err) {
227 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
228 }
229
230 /* Re-enable the interrupt in cased we missed an EOI */
231 vfio_intx_eoi(&vdev->vbasedev);
232 }
233
234 static void vfio_intx_routing_notifier(PCIDevice *pdev)
235 {
236 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
237 PCIINTxRoute route;
238
239 if (vdev->interrupt != VFIO_INT_INTx) {
240 return;
241 }
242
243 route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
244
245 if (pci_intx_route_changed(&vdev->intx.route, &route)) {
246 vfio_intx_update(vdev, &route);
247 }
248 }
249
250 static void vfio_irqchip_change(Notifier *notify, void *data)
251 {
252 VFIOPCIDevice *vdev = container_of(notify, VFIOPCIDevice,
253 irqchip_change_notifier);
254
255 vfio_intx_update(vdev, &vdev->intx.route);
256 }
257
258 static int vfio_intx_enable(VFIOPCIDevice *vdev, Error **errp)
259 {
260 uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
261 Error *err = NULL;
262 int32_t fd;
263 int ret;
264
265
266 if (!pin) {
267 return 0;
268 }
269
270 vfio_disable_interrupts(vdev);
271
272 vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
273 pci_config_set_interrupt_pin(vdev->pdev.config, pin);
274
275 #ifdef CONFIG_KVM
276 /*
277 * Only conditional to avoid generating error messages on platforms
278 * where we won't actually use the result anyway.
279 */
280 if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
281 vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
282 vdev->intx.pin);
283 }
284 #endif
285
286 ret = event_notifier_init(&vdev->intx.interrupt, 0);
287 if (ret) {
288 error_setg_errno(errp, -ret, "event_notifier_init failed");
289 return ret;
290 }
291 fd = event_notifier_get_fd(&vdev->intx.interrupt);
292 qemu_set_fd_handler(fd, vfio_intx_interrupt, NULL, vdev);
293
294 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
295 VFIO_IRQ_SET_ACTION_TRIGGER, fd, errp)) {
296 qemu_set_fd_handler(fd, NULL, NULL, vdev);
297 event_notifier_cleanup(&vdev->intx.interrupt);
298 return -errno;
299 }
300
301 vfio_intx_enable_kvm(vdev, &err);
302 if (err) {
303 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
304 }
305
306 vdev->interrupt = VFIO_INT_INTx;
307
308 trace_vfio_intx_enable(vdev->vbasedev.name);
309 return 0;
310 }
311
312 static void vfio_intx_disable(VFIOPCIDevice *vdev)
313 {
314 int fd;
315
316 timer_del(vdev->intx.mmap_timer);
317 vfio_intx_disable_kvm(vdev);
318 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
319 vdev->intx.pending = false;
320 pci_irq_deassert(&vdev->pdev);
321 vfio_mmap_set_enabled(vdev, true);
322
323 fd = event_notifier_get_fd(&vdev->intx.interrupt);
324 qemu_set_fd_handler(fd, NULL, NULL, vdev);
325 event_notifier_cleanup(&vdev->intx.interrupt);
326
327 vdev->interrupt = VFIO_INT_NONE;
328
329 trace_vfio_intx_disable(vdev->vbasedev.name);
330 }
331
332 /*
333 * MSI/X
334 */
335 static void vfio_msi_interrupt(void *opaque)
336 {
337 VFIOMSIVector *vector = opaque;
338 VFIOPCIDevice *vdev = vector->vdev;
339 MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector);
340 void (*notify)(PCIDevice *dev, unsigned vector);
341 MSIMessage msg;
342 int nr = vector - vdev->msi_vectors;
343
344 if (!event_notifier_test_and_clear(&vector->interrupt)) {
345 return;
346 }
347
348 if (vdev->interrupt == VFIO_INT_MSIX) {
349 get_msg = msix_get_message;
350 notify = msix_notify;
351
352 /* A masked vector firing needs to use the PBA, enable it */
353 if (msix_is_masked(&vdev->pdev, nr)) {
354 set_bit(nr, vdev->msix->pending);
355 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, true);
356 trace_vfio_msix_pba_enable(vdev->vbasedev.name);
357 }
358 } else if (vdev->interrupt == VFIO_INT_MSI) {
359 get_msg = msi_get_message;
360 notify = msi_notify;
361 } else {
362 abort();
363 }
364
365 msg = get_msg(&vdev->pdev, nr);
366 trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
367 notify(&vdev->pdev, nr);
368 }
369
370 static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
371 {
372 struct vfio_irq_set *irq_set;
373 int ret = 0, i, argsz;
374 int32_t *fds;
375
376 argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
377
378 irq_set = g_malloc0(argsz);
379 irq_set->argsz = argsz;
380 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
381 irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
382 irq_set->start = 0;
383 irq_set->count = vdev->nr_vectors;
384 fds = (int32_t *)&irq_set->data;
385
386 for (i = 0; i < vdev->nr_vectors; i++) {
387 int fd = -1;
388
389 /*
390 * MSI vs MSI-X - The guest has direct access to MSI mask and pending
391 * bits, therefore we always use the KVM signaling path when setup.
392 * MSI-X mask and pending bits are emulated, so we want to use the
393 * KVM signaling path only when configured and unmasked.
394 */
395 if (vdev->msi_vectors[i].use) {
396 if (vdev->msi_vectors[i].virq < 0 ||
397 (msix && msix_is_masked(&vdev->pdev, i))) {
398 fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
399 } else {
400 fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
401 }
402 }
403
404 fds[i] = fd;
405 }
406
407 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
408
409 g_free(irq_set);
410
411 return ret;
412 }
413
414 static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,
415 int vector_n, bool msix)
416 {
417 int virq;
418
419 if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi)) {
420 return;
421 }
422
423 if (event_notifier_init(&vector->kvm_interrupt, 0)) {
424 return;
425 }
426
427 virq = kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev);
428 if (virq < 0) {
429 event_notifier_cleanup(&vector->kvm_interrupt);
430 return;
431 }
432
433 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
434 NULL, virq) < 0) {
435 kvm_irqchip_release_virq(kvm_state, virq);
436 event_notifier_cleanup(&vector->kvm_interrupt);
437 return;
438 }
439
440 vector->virq = virq;
441 }
442
443 static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
444 {
445 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
446 vector->virq);
447 kvm_irqchip_release_virq(kvm_state, vector->virq);
448 vector->virq = -1;
449 event_notifier_cleanup(&vector->kvm_interrupt);
450 }
451
452 static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
453 PCIDevice *pdev)
454 {
455 kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
456 kvm_irqchip_commit_routes(kvm_state);
457 }
458
459 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
460 MSIMessage *msg, IOHandler *handler)
461 {
462 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
463 VFIOMSIVector *vector;
464 int ret;
465
466 trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
467
468 vector = &vdev->msi_vectors[nr];
469
470 if (!vector->use) {
471 vector->vdev = vdev;
472 vector->virq = -1;
473 if (event_notifier_init(&vector->interrupt, 0)) {
474 error_report("vfio: Error: event_notifier_init failed");
475 }
476 vector->use = true;
477 msix_vector_use(pdev, nr);
478 }
479
480 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
481 handler, NULL, vector);
482
483 /*
484 * Attempt to enable route through KVM irqchip,
485 * default to userspace handling if unavailable.
486 */
487 if (vector->virq >= 0) {
488 if (!msg) {
489 vfio_remove_kvm_msi_virq(vector);
490 } else {
491 vfio_update_kvm_msi_virq(vector, *msg, pdev);
492 }
493 } else {
494 if (msg) {
495 vfio_add_kvm_msi_virq(vdev, vector, nr, true);
496 }
497 }
498
499 /*
500 * We don't want to have the host allocate all possible MSI vectors
501 * for a device if they're not in use, so we shutdown and incrementally
502 * increase them as needed.
503 */
504 if (vdev->nr_vectors < nr + 1) {
505 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
506 vdev->nr_vectors = nr + 1;
507 ret = vfio_enable_vectors(vdev, true);
508 if (ret) {
509 error_report("vfio: failed to enable vectors, %d", ret);
510 }
511 } else {
512 Error *err = NULL;
513 int32_t fd;
514
515 if (vector->virq >= 0) {
516 fd = event_notifier_get_fd(&vector->kvm_interrupt);
517 } else {
518 fd = event_notifier_get_fd(&vector->interrupt);
519 }
520
521 if (vfio_set_irq_signaling(&vdev->vbasedev,
522 VFIO_PCI_MSIX_IRQ_INDEX, nr,
523 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
524 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
525 }
526 }
527
528 /* Disable PBA emulation when nothing more is pending. */
529 clear_bit(nr, vdev->msix->pending);
530 if (find_first_bit(vdev->msix->pending,
531 vdev->nr_vectors) == vdev->nr_vectors) {
532 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
533 trace_vfio_msix_pba_disable(vdev->vbasedev.name);
534 }
535
536 return 0;
537 }
538
539 static int vfio_msix_vector_use(PCIDevice *pdev,
540 unsigned int nr, MSIMessage msg)
541 {
542 return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
543 }
544
545 static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
546 {
547 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
548 VFIOMSIVector *vector = &vdev->msi_vectors[nr];
549
550 trace_vfio_msix_vector_release(vdev->vbasedev.name, nr);
551
552 /*
553 * There are still old guests that mask and unmask vectors on every
554 * interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of
555 * the KVM setup in place, simply switch VFIO to use the non-bypass
556 * eventfd. We'll then fire the interrupt through QEMU and the MSI-X
557 * core will mask the interrupt and set pending bits, allowing it to
558 * be re-asserted on unmask. Nothing to do if already using QEMU mode.
559 */
560 if (vector->virq >= 0) {
561 int32_t fd = event_notifier_get_fd(&vector->interrupt);
562 Error *err = NULL;
563
564 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX, nr,
565 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
566 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
567 }
568 }
569 }
570
571 static void vfio_msix_enable(VFIOPCIDevice *vdev)
572 {
573 vfio_disable_interrupts(vdev);
574
575 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
576
577 vdev->interrupt = VFIO_INT_MSIX;
578
579 /*
580 * Some communication channels between VF & PF or PF & fw rely on the
581 * physical state of the device and expect that enabling MSI-X from the
582 * guest enables the same on the host. When our guest is Linux, the
583 * guest driver call to pci_enable_msix() sets the enabling bit in the
584 * MSI-X capability, but leaves the vector table masked. We therefore
585 * can't rely on a vector_use callback (from request_irq() in the guest)
586 * to switch the physical device into MSI-X mode because that may come a
587 * long time after pci_enable_msix(). This code enables vector 0 with
588 * triggering to userspace, then immediately release the vector, leaving
589 * the physical device with no vectors enabled, but MSI-X enabled, just
590 * like the guest view.
591 */
592 vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
593 vfio_msix_vector_release(&vdev->pdev, 0);
594
595 if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
596 vfio_msix_vector_release, NULL)) {
597 error_report("vfio: msix_set_vector_notifiers failed");
598 }
599
600 trace_vfio_msix_enable(vdev->vbasedev.name);
601 }
602
603 static void vfio_msi_enable(VFIOPCIDevice *vdev)
604 {
605 int ret, i;
606
607 vfio_disable_interrupts(vdev);
608
609 vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
610 retry:
611 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
612
613 for (i = 0; i < vdev->nr_vectors; i++) {
614 VFIOMSIVector *vector = &vdev->msi_vectors[i];
615
616 vector->vdev = vdev;
617 vector->virq = -1;
618 vector->use = true;
619
620 if (event_notifier_init(&vector->interrupt, 0)) {
621 error_report("vfio: Error: event_notifier_init failed");
622 }
623
624 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
625 vfio_msi_interrupt, NULL, vector);
626
627 /*
628 * Attempt to enable route through KVM irqchip,
629 * default to userspace handling if unavailable.
630 */
631 vfio_add_kvm_msi_virq(vdev, vector, i, false);
632 }
633
634 /* Set interrupt type prior to possible interrupts */
635 vdev->interrupt = VFIO_INT_MSI;
636
637 ret = vfio_enable_vectors(vdev, false);
638 if (ret) {
639 if (ret < 0) {
640 error_report("vfio: Error: Failed to setup MSI fds: %m");
641 } else if (ret != vdev->nr_vectors) {
642 error_report("vfio: Error: Failed to enable %d "
643 "MSI vectors, retry with %d", vdev->nr_vectors, ret);
644 }
645
646 for (i = 0; i < vdev->nr_vectors; i++) {
647 VFIOMSIVector *vector = &vdev->msi_vectors[i];
648 if (vector->virq >= 0) {
649 vfio_remove_kvm_msi_virq(vector);
650 }
651 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
652 NULL, NULL, NULL);
653 event_notifier_cleanup(&vector->interrupt);
654 }
655
656 g_free(vdev->msi_vectors);
657 vdev->msi_vectors = NULL;
658
659 if (ret > 0 && ret != vdev->nr_vectors) {
660 vdev->nr_vectors = ret;
661 goto retry;
662 }
663 vdev->nr_vectors = 0;
664
665 /*
666 * Failing to setup MSI doesn't really fall within any specification.
667 * Let's try leaving interrupts disabled and hope the guest figures
668 * out to fall back to INTx for this device.
669 */
670 error_report("vfio: Error: Failed to enable MSI");
671 vdev->interrupt = VFIO_INT_NONE;
672
673 return;
674 }
675
676 trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
677 }
678
679 static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
680 {
681 Error *err = NULL;
682 int i;
683
684 for (i = 0; i < vdev->nr_vectors; i++) {
685 VFIOMSIVector *vector = &vdev->msi_vectors[i];
686 if (vdev->msi_vectors[i].use) {
687 if (vector->virq >= 0) {
688 vfio_remove_kvm_msi_virq(vector);
689 }
690 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
691 NULL, NULL, NULL);
692 event_notifier_cleanup(&vector->interrupt);
693 }
694 }
695
696 g_free(vdev->msi_vectors);
697 vdev->msi_vectors = NULL;
698 vdev->nr_vectors = 0;
699 vdev->interrupt = VFIO_INT_NONE;
700
701 vfio_intx_enable(vdev, &err);
702 if (err) {
703 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
704 }
705 }
706
707 static void vfio_msix_disable(VFIOPCIDevice *vdev)
708 {
709 int i;
710
711 msix_unset_vector_notifiers(&vdev->pdev);
712
713 /*
714 * MSI-X will only release vectors if MSI-X is still enabled on the
715 * device, check through the rest and release it ourselves if necessary.
716 */
717 for (i = 0; i < vdev->nr_vectors; i++) {
718 if (vdev->msi_vectors[i].use) {
719 vfio_msix_vector_release(&vdev->pdev, i);
720 msix_vector_unuse(&vdev->pdev, i);
721 }
722 }
723
724 if (vdev->nr_vectors) {
725 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
726 }
727
728 vfio_msi_disable_common(vdev);
729
730 memset(vdev->msix->pending, 0,
731 BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long));
732
733 trace_vfio_msix_disable(vdev->vbasedev.name);
734 }
735
736 static void vfio_msi_disable(VFIOPCIDevice *vdev)
737 {
738 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
739 vfio_msi_disable_common(vdev);
740
741 trace_vfio_msi_disable(vdev->vbasedev.name);
742 }
743
744 static void vfio_update_msi(VFIOPCIDevice *vdev)
745 {
746 int i;
747
748 for (i = 0; i < vdev->nr_vectors; i++) {
749 VFIOMSIVector *vector = &vdev->msi_vectors[i];
750 MSIMessage msg;
751
752 if (!vector->use || vector->virq < 0) {
753 continue;
754 }
755
756 msg = msi_get_message(&vdev->pdev, i);
757 vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev);
758 }
759 }
760
761 static void vfio_pci_load_rom(VFIOPCIDevice *vdev)
762 {
763 struct vfio_region_info *reg_info;
764 uint64_t size;
765 off_t off = 0;
766 ssize_t bytes;
767
768 if (vfio_get_region_info(&vdev->vbasedev,
769 VFIO_PCI_ROM_REGION_INDEX, &reg_info)) {
770 error_report("vfio: Error getting ROM info: %m");
771 return;
772 }
773
774 trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size,
775 (unsigned long)reg_info->offset,
776 (unsigned long)reg_info->flags);
777
778 vdev->rom_size = size = reg_info->size;
779 vdev->rom_offset = reg_info->offset;
780
781 g_free(reg_info);
782
783 if (!vdev->rom_size) {
784 vdev->rom_read_failed = true;
785 error_report("vfio-pci: Cannot read device rom at "
786 "%s", vdev->vbasedev.name);
787 error_printf("Device option ROM contents are probably invalid "
788 "(check dmesg).\nSkip option ROM probe with rombar=0, "
789 "or load from file with romfile=\n");
790 return;
791 }
792
793 vdev->rom = g_malloc(size);
794 memset(vdev->rom, 0xff, size);
795
796 while (size) {
797 bytes = pread(vdev->vbasedev.fd, vdev->rom + off,
798 size, vdev->rom_offset + off);
799 if (bytes == 0) {
800 break;
801 } else if (bytes > 0) {
802 off += bytes;
803 size -= bytes;
804 } else {
805 if (errno == EINTR || errno == EAGAIN) {
806 continue;
807 }
808 error_report("vfio: Error reading device ROM: %m");
809 break;
810 }
811 }
812
813 /*
814 * Test the ROM signature against our device, if the vendor is correct
815 * but the device ID doesn't match, store the correct device ID and
816 * recompute the checksum. Intel IGD devices need this and are known
817 * to have bogus checksums so we can't simply adjust the checksum.
818 */
819 if (pci_get_word(vdev->rom) == 0xaa55 &&
820 pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size &&
821 !memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) {
822 uint16_t vid, did;
823
824 vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4);
825 did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6);
826
827 if (vid == vdev->vendor_id && did != vdev->device_id) {
828 int i;
829 uint8_t csum, *data = vdev->rom;
830
831 pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6,
832 vdev->device_id);
833 data[6] = 0;
834
835 for (csum = 0, i = 0; i < vdev->rom_size; i++) {
836 csum += data[i];
837 }
838
839 data[6] = -csum;
840 }
841 }
842 }
843
844 static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
845 {
846 VFIOPCIDevice *vdev = opaque;
847 union {
848 uint8_t byte;
849 uint16_t word;
850 uint32_t dword;
851 uint64_t qword;
852 } val;
853 uint64_t data = 0;
854
855 /* Load the ROM lazily when the guest tries to read it */
856 if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
857 vfio_pci_load_rom(vdev);
858 }
859
860 memcpy(&val, vdev->rom + addr,
861 (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
862
863 switch (size) {
864 case 1:
865 data = val.byte;
866 break;
867 case 2:
868 data = le16_to_cpu(val.word);
869 break;
870 case 4:
871 data = le32_to_cpu(val.dword);
872 break;
873 default:
874 hw_error("vfio: unsupported read size, %d bytes\n", size);
875 break;
876 }
877
878 trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data);
879
880 return data;
881 }
882
883 static void vfio_rom_write(void *opaque, hwaddr addr,
884 uint64_t data, unsigned size)
885 {
886 }
887
888 static const MemoryRegionOps vfio_rom_ops = {
889 .read = vfio_rom_read,
890 .write = vfio_rom_write,
891 .endianness = DEVICE_LITTLE_ENDIAN,
892 };
893
894 static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
895 {
896 uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
897 off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
898 DeviceState *dev = DEVICE(vdev);
899 char *name;
900 int fd = vdev->vbasedev.fd;
901
902 if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
903 /* Since pci handles romfile, just print a message and return */
904 if (vfio_blacklist_opt_rom(vdev) && vdev->pdev.romfile) {
905 warn_report("Device at %s is known to cause system instability"
906 " issues during option rom execution",
907 vdev->vbasedev.name);
908 error_printf("Proceeding anyway since user specified romfile\n");
909 }
910 return;
911 }
912
913 /*
914 * Use the same size ROM BAR as the physical device. The contents
915 * will get filled in later when the guest tries to read it.
916 */
917 if (pread(fd, &orig, 4, offset) != 4 ||
918 pwrite(fd, &size, 4, offset) != 4 ||
919 pread(fd, &size, 4, offset) != 4 ||
920 pwrite(fd, &orig, 4, offset) != 4) {
921 error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name);
922 return;
923 }
924
925 size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
926
927 if (!size) {
928 return;
929 }
930
931 if (vfio_blacklist_opt_rom(vdev)) {
932 if (dev->opts && qemu_opt_get(dev->opts, "rombar")) {
933 warn_report("Device at %s is known to cause system instability"
934 " issues during option rom execution",
935 vdev->vbasedev.name);
936 error_printf("Proceeding anyway since user specified"
937 " non zero value for rombar\n");
938 } else {
939 warn_report("Rom loading for device at %s has been disabled"
940 " due to system instability issues",
941 vdev->vbasedev.name);
942 error_printf("Specify rombar=1 or romfile to force\n");
943 return;
944 }
945 }
946
947 trace_vfio_pci_size_rom(vdev->vbasedev.name, size);
948
949 name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name);
950
951 memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
952 &vfio_rom_ops, vdev, name, size);
953 g_free(name);
954
955 pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
956 PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
957
958 vdev->rom_read_failed = false;
959 }
960
961 void vfio_vga_write(void *opaque, hwaddr addr,
962 uint64_t data, unsigned size)
963 {
964 VFIOVGARegion *region = opaque;
965 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
966 union {
967 uint8_t byte;
968 uint16_t word;
969 uint32_t dword;
970 uint64_t qword;
971 } buf;
972 off_t offset = vga->fd_offset + region->offset + addr;
973
974 switch (size) {
975 case 1:
976 buf.byte = data;
977 break;
978 case 2:
979 buf.word = cpu_to_le16(data);
980 break;
981 case 4:
982 buf.dword = cpu_to_le32(data);
983 break;
984 default:
985 hw_error("vfio: unsupported write size, %d bytes", size);
986 break;
987 }
988
989 if (pwrite(vga->fd, &buf, size, offset) != size) {
990 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
991 __func__, region->offset + addr, data, size);
992 }
993
994 trace_vfio_vga_write(region->offset + addr, data, size);
995 }
996
997 uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
998 {
999 VFIOVGARegion *region = opaque;
1000 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1001 union {
1002 uint8_t byte;
1003 uint16_t word;
1004 uint32_t dword;
1005 uint64_t qword;
1006 } buf;
1007 uint64_t data = 0;
1008 off_t offset = vga->fd_offset + region->offset + addr;
1009
1010 if (pread(vga->fd, &buf, size, offset) != size) {
1011 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1012 __func__, region->offset + addr, size);
1013 return (uint64_t)-1;
1014 }
1015
1016 switch (size) {
1017 case 1:
1018 data = buf.byte;
1019 break;
1020 case 2:
1021 data = le16_to_cpu(buf.word);
1022 break;
1023 case 4:
1024 data = le32_to_cpu(buf.dword);
1025 break;
1026 default:
1027 hw_error("vfio: unsupported read size, %d bytes", size);
1028 break;
1029 }
1030
1031 trace_vfio_vga_read(region->offset + addr, size, data);
1032
1033 return data;
1034 }
1035
1036 static const MemoryRegionOps vfio_vga_ops = {
1037 .read = vfio_vga_read,
1038 .write = vfio_vga_write,
1039 .endianness = DEVICE_LITTLE_ENDIAN,
1040 };
1041
1042 /*
1043 * Expand memory region of sub-page(size < PAGE_SIZE) MMIO BAR to page
1044 * size if the BAR is in an exclusive page in host so that we could map
1045 * this BAR to guest. But this sub-page BAR may not occupy an exclusive
1046 * page in guest. So we should set the priority of the expanded memory
1047 * region to zero in case of overlap with BARs which share the same page
1048 * with the sub-page BAR in guest. Besides, we should also recover the
1049 * size of this sub-page BAR when its base address is changed in guest
1050 * and not page aligned any more.
1051 */
1052 static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar)
1053 {
1054 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
1055 VFIORegion *region = &vdev->bars[bar].region;
1056 MemoryRegion *mmap_mr, *region_mr, *base_mr;
1057 PCIIORegion *r;
1058 pcibus_t bar_addr;
1059 uint64_t size = region->size;
1060
1061 /* Make sure that the whole region is allowed to be mmapped */
1062 if (region->nr_mmaps != 1 || !region->mmaps[0].mmap ||
1063 region->mmaps[0].size != region->size) {
1064 return;
1065 }
1066
1067 r = &pdev->io_regions[bar];
1068 bar_addr = r->addr;
1069 base_mr = vdev->bars[bar].mr;
1070 region_mr = region->mem;
1071 mmap_mr = &region->mmaps[0].mem;
1072
1073 /* If BAR is mapped and page aligned, update to fill PAGE_SIZE */
1074 if (bar_addr != PCI_BAR_UNMAPPED &&
1075 !(bar_addr & ~qemu_real_host_page_mask)) {
1076 size = qemu_real_host_page_size;
1077 }
1078
1079 memory_region_transaction_begin();
1080
1081 if (vdev->bars[bar].size < size) {
1082 memory_region_set_size(base_mr, size);
1083 }
1084 memory_region_set_size(region_mr, size);
1085 memory_region_set_size(mmap_mr, size);
1086 if (size != vdev->bars[bar].size && memory_region_is_mapped(base_mr)) {
1087 memory_region_del_subregion(r->address_space, base_mr);
1088 memory_region_add_subregion_overlap(r->address_space,
1089 bar_addr, base_mr, 0);
1090 }
1091
1092 memory_region_transaction_commit();
1093 }
1094
1095 /*
1096 * PCI config space
1097 */
1098 uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
1099 {
1100 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
1101 uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
1102
1103 memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
1104 emu_bits = le32_to_cpu(emu_bits);
1105
1106 if (emu_bits) {
1107 emu_val = pci_default_read_config(pdev, addr, len);
1108 }
1109
1110 if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
1111 ssize_t ret;
1112
1113 ret = pread(vdev->vbasedev.fd, &phys_val, len,
1114 vdev->config_offset + addr);
1115 if (ret != len) {
1116 error_report("%s(%s, 0x%x, 0x%x) failed: %m",
1117 __func__, vdev->vbasedev.name, addr, len);
1118 return -errno;
1119 }
1120 phys_val = le32_to_cpu(phys_val);
1121 }
1122
1123 val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
1124
1125 trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val);
1126
1127 return val;
1128 }
1129
1130 void vfio_pci_write_config(PCIDevice *pdev,
1131 uint32_t addr, uint32_t val, int len)
1132 {
1133 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
1134 uint32_t val_le = cpu_to_le32(val);
1135
1136 trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len);
1137
1138 /* Write everything to VFIO, let it filter out what we can't write */
1139 if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr)
1140 != len) {
1141 error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
1142 __func__, vdev->vbasedev.name, addr, val, len);
1143 }
1144
1145 /* MSI/MSI-X Enabling/Disabling */
1146 if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
1147 ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
1148 int is_enabled, was_enabled = msi_enabled(pdev);
1149
1150 pci_default_write_config(pdev, addr, val, len);
1151
1152 is_enabled = msi_enabled(pdev);
1153
1154 if (!was_enabled) {
1155 if (is_enabled) {
1156 vfio_msi_enable(vdev);
1157 }
1158 } else {
1159 if (!is_enabled) {
1160 vfio_msi_disable(vdev);
1161 } else {
1162 vfio_update_msi(vdev);
1163 }
1164 }
1165 } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
1166 ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
1167 int is_enabled, was_enabled = msix_enabled(pdev);
1168
1169 pci_default_write_config(pdev, addr, val, len);
1170
1171 is_enabled = msix_enabled(pdev);
1172
1173 if (!was_enabled && is_enabled) {
1174 vfio_msix_enable(vdev);
1175 } else if (was_enabled && !is_enabled) {
1176 vfio_msix_disable(vdev);
1177 }
1178 } else if (ranges_overlap(addr, len, PCI_BASE_ADDRESS_0, 24) ||
1179 range_covers_byte(addr, len, PCI_COMMAND)) {
1180 pcibus_t old_addr[PCI_NUM_REGIONS - 1];
1181 int bar;
1182
1183 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1184 old_addr[bar] = pdev->io_regions[bar].addr;
1185 }
1186
1187 pci_default_write_config(pdev, addr, val, len);
1188
1189 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1190 if (old_addr[bar] != pdev->io_regions[bar].addr &&
1191 vdev->bars[bar].region.size > 0 &&
1192 vdev->bars[bar].region.size < qemu_real_host_page_size) {
1193 vfio_sub_page_bar_update_mapping(pdev, bar);
1194 }
1195 }
1196 } else {
1197 /* Write everything to QEMU to keep emulated bits correct */
1198 pci_default_write_config(pdev, addr, val, len);
1199 }
1200 }
1201
1202 /*
1203 * Interrupt setup
1204 */
1205 static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
1206 {
1207 /*
1208 * More complicated than it looks. Disabling MSI/X transitions the
1209 * device to INTx mode (if supported). Therefore we need to first
1210 * disable MSI/X and then cleanup by disabling INTx.
1211 */
1212 if (vdev->interrupt == VFIO_INT_MSIX) {
1213 vfio_msix_disable(vdev);
1214 } else if (vdev->interrupt == VFIO_INT_MSI) {
1215 vfio_msi_disable(vdev);
1216 }
1217
1218 if (vdev->interrupt == VFIO_INT_INTx) {
1219 vfio_intx_disable(vdev);
1220 }
1221 }
1222
1223 static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1224 {
1225 uint16_t ctrl;
1226 bool msi_64bit, msi_maskbit;
1227 int ret, entries;
1228 Error *err = NULL;
1229
1230 if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl),
1231 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
1232 error_setg_errno(errp, errno, "failed reading MSI PCI_CAP_FLAGS");
1233 return -errno;
1234 }
1235 ctrl = le16_to_cpu(ctrl);
1236
1237 msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
1238 msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
1239 entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
1240
1241 trace_vfio_msi_setup(vdev->vbasedev.name, pos);
1242
1243 ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit, &err);
1244 if (ret < 0) {
1245 if (ret == -ENOTSUP) {
1246 return 0;
1247 }
1248 error_propagate_prepend(errp, err, "msi_init failed: ");
1249 return ret;
1250 }
1251 vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
1252
1253 return 0;
1254 }
1255
1256 static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev)
1257 {
1258 off_t start, end;
1259 VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region;
1260
1261 /*
1262 * If the host driver allows mapping of a MSIX data, we are going to
1263 * do map the entire BAR and emulate MSIX table on top of that.
1264 */
1265 if (vfio_has_region_cap(&vdev->vbasedev, region->nr,
1266 VFIO_REGION_INFO_CAP_MSIX_MAPPABLE)) {
1267 return;
1268 }
1269
1270 /*
1271 * We expect to find a single mmap covering the whole BAR, anything else
1272 * means it's either unsupported or already setup.
1273 */
1274 if (region->nr_mmaps != 1 || region->mmaps[0].offset ||
1275 region->size != region->mmaps[0].size) {
1276 return;
1277 }
1278
1279 /* MSI-X table start and end aligned to host page size */
1280 start = vdev->msix->table_offset & qemu_real_host_page_mask;
1281 end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
1282 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
1283
1284 /*
1285 * Does the MSI-X table cover the beginning of the BAR? The whole BAR?
1286 * NB - Host page size is necessarily a power of two and so is the PCI
1287 * BAR (not counting EA yet), therefore if we have host page aligned
1288 * @start and @end, then any remainder of the BAR before or after those
1289 * must be at least host page sized and therefore mmap'able.
1290 */
1291 if (!start) {
1292 if (end >= region->size) {
1293 region->nr_mmaps = 0;
1294 g_free(region->mmaps);
1295 region->mmaps = NULL;
1296 trace_vfio_msix_fixup(vdev->vbasedev.name,
1297 vdev->msix->table_bar, 0, 0);
1298 } else {
1299 region->mmaps[0].offset = end;
1300 region->mmaps[0].size = region->size - end;
1301 trace_vfio_msix_fixup(vdev->vbasedev.name,
1302 vdev->msix->table_bar, region->mmaps[0].offset,
1303 region->mmaps[0].offset + region->mmaps[0].size);
1304 }
1305
1306 /* Maybe it's aligned at the end of the BAR */
1307 } else if (end >= region->size) {
1308 region->mmaps[0].size = start;
1309 trace_vfio_msix_fixup(vdev->vbasedev.name,
1310 vdev->msix->table_bar, region->mmaps[0].offset,
1311 region->mmaps[0].offset + region->mmaps[0].size);
1312
1313 /* Otherwise it must split the BAR */
1314 } else {
1315 region->nr_mmaps = 2;
1316 region->mmaps = g_renew(VFIOMmap, region->mmaps, 2);
1317
1318 memcpy(&region->mmaps[1], &region->mmaps[0], sizeof(VFIOMmap));
1319
1320 region->mmaps[0].size = start;
1321 trace_vfio_msix_fixup(vdev->vbasedev.name,
1322 vdev->msix->table_bar, region->mmaps[0].offset,
1323 region->mmaps[0].offset + region->mmaps[0].size);
1324
1325 region->mmaps[1].offset = end;
1326 region->mmaps[1].size = region->size - end;
1327 trace_vfio_msix_fixup(vdev->vbasedev.name,
1328 vdev->msix->table_bar, region->mmaps[1].offset,
1329 region->mmaps[1].offset + region->mmaps[1].size);
1330 }
1331 }
1332
1333 static void vfio_pci_relocate_msix(VFIOPCIDevice *vdev, Error **errp)
1334 {
1335 int target_bar = -1;
1336 size_t msix_sz;
1337
1338 if (!vdev->msix || vdev->msix_relo == OFF_AUTOPCIBAR_OFF) {
1339 return;
1340 }
1341
1342 /* The actual minimum size of MSI-X structures */
1343 msix_sz = (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE) +
1344 (QEMU_ALIGN_UP(vdev->msix->entries, 64) / 8);
1345 /* Round up to host pages, we don't want to share a page */
1346 msix_sz = REAL_HOST_PAGE_ALIGN(msix_sz);
1347 /* PCI BARs must be a power of 2 */
1348 msix_sz = pow2ceil(msix_sz);
1349
1350 if (vdev->msix_relo == OFF_AUTOPCIBAR_AUTO) {
1351 /*
1352 * TODO: Lookup table for known devices.
1353 *
1354 * Logically we might use an algorithm here to select the BAR adding
1355 * the least additional MMIO space, but we cannot programatically
1356 * predict the driver dependency on BAR ordering or sizing, therefore
1357 * 'auto' becomes a lookup for combinations reported to work.
1358 */
1359 if (target_bar < 0) {
1360 error_setg(errp, "No automatic MSI-X relocation available for "
1361 "device %04x:%04x", vdev->vendor_id, vdev->device_id);
1362 return;
1363 }
1364 } else {
1365 target_bar = (int)(vdev->msix_relo - OFF_AUTOPCIBAR_BAR0);
1366 }
1367
1368 /* I/O port BARs cannot host MSI-X structures */
1369 if (vdev->bars[target_bar].ioport) {
1370 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1371 "I/O port BAR", target_bar);
1372 return;
1373 }
1374
1375 /* Cannot use a BAR in the "shadow" of a 64-bit BAR */
1376 if (!vdev->bars[target_bar].size &&
1377 target_bar > 0 && vdev->bars[target_bar - 1].mem64) {
1378 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1379 "consumed by 64-bit BAR %d", target_bar, target_bar - 1);
1380 return;
1381 }
1382
1383 /* 2GB max size for 32-bit BARs, cannot double if already > 1G */
1384 if (vdev->bars[target_bar].size > 1 * GiB &&
1385 !vdev->bars[target_bar].mem64) {
1386 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1387 "no space to extend 32-bit BAR", target_bar);
1388 return;
1389 }
1390
1391 /*
1392 * If adding a new BAR, test if we can make it 64bit. We make it
1393 * prefetchable since QEMU MSI-X emulation has no read side effects
1394 * and doing so makes mapping more flexible.
1395 */
1396 if (!vdev->bars[target_bar].size) {
1397 if (target_bar < (PCI_ROM_SLOT - 1) &&
1398 !vdev->bars[target_bar + 1].size) {
1399 vdev->bars[target_bar].mem64 = true;
1400 vdev->bars[target_bar].type = PCI_BASE_ADDRESS_MEM_TYPE_64;
1401 }
1402 vdev->bars[target_bar].type |= PCI_BASE_ADDRESS_MEM_PREFETCH;
1403 vdev->bars[target_bar].size = msix_sz;
1404 vdev->msix->table_offset = 0;
1405 } else {
1406 vdev->bars[target_bar].size = MAX(vdev->bars[target_bar].size * 2,
1407 msix_sz * 2);
1408 /*
1409 * Due to above size calc, MSI-X always starts halfway into the BAR,
1410 * which will always be a separate host page.
1411 */
1412 vdev->msix->table_offset = vdev->bars[target_bar].size / 2;
1413 }
1414
1415 vdev->msix->table_bar = target_bar;
1416 vdev->msix->pba_bar = target_bar;
1417 /* Requires 8-byte alignment, but PCI_MSIX_ENTRY_SIZE guarantees that */
1418 vdev->msix->pba_offset = vdev->msix->table_offset +
1419 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE);
1420
1421 trace_vfio_msix_relo(vdev->vbasedev.name,
1422 vdev->msix->table_bar, vdev->msix->table_offset);
1423 }
1424
1425 /*
1426 * We don't have any control over how pci_add_capability() inserts
1427 * capabilities into the chain. In order to setup MSI-X we need a
1428 * MemoryRegion for the BAR. In order to setup the BAR and not
1429 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
1430 * need to first look for where the MSI-X table lives. So we
1431 * unfortunately split MSI-X setup across two functions.
1432 */
1433 static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
1434 {
1435 uint8_t pos;
1436 uint16_t ctrl;
1437 uint32_t table, pba;
1438 int fd = vdev->vbasedev.fd;
1439 VFIOMSIXInfo *msix;
1440
1441 pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
1442 if (!pos) {
1443 return;
1444 }
1445
1446 if (pread(fd, &ctrl, sizeof(ctrl),
1447 vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) {
1448 error_setg_errno(errp, errno, "failed to read PCI MSIX FLAGS");
1449 return;
1450 }
1451
1452 if (pread(fd, &table, sizeof(table),
1453 vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
1454 error_setg_errno(errp, errno, "failed to read PCI MSIX TABLE");
1455 return;
1456 }
1457
1458 if (pread(fd, &pba, sizeof(pba),
1459 vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
1460 error_setg_errno(errp, errno, "failed to read PCI MSIX PBA");
1461 return;
1462 }
1463
1464 ctrl = le16_to_cpu(ctrl);
1465 table = le32_to_cpu(table);
1466 pba = le32_to_cpu(pba);
1467
1468 msix = g_malloc0(sizeof(*msix));
1469 msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
1470 msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
1471 msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
1472 msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
1473 msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
1474
1475 /*
1476 * Test the size of the pba_offset variable and catch if it extends outside
1477 * of the specified BAR. If it is the case, we need to apply a hardware
1478 * specific quirk if the device is known or we have a broken configuration.
1479 */
1480 if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
1481 /*
1482 * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
1483 * adapters. The T5 hardware returns an incorrect value of 0x8000 for
1484 * the VF PBA offset while the BAR itself is only 8k. The correct value
1485 * is 0x1000, so we hard code that here.
1486 */
1487 if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
1488 (vdev->device_id & 0xff00) == 0x5800) {
1489 msix->pba_offset = 0x1000;
1490 } else if (vdev->msix_relo == OFF_AUTOPCIBAR_OFF) {
1491 error_setg(errp, "hardware reports invalid configuration, "
1492 "MSIX PBA outside of specified BAR");
1493 g_free(msix);
1494 return;
1495 }
1496 }
1497
1498 trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
1499 msix->table_offset, msix->entries);
1500 vdev->msix = msix;
1501
1502 vfio_pci_fixup_msix_region(vdev);
1503
1504 vfio_pci_relocate_msix(vdev, errp);
1505 }
1506
1507 static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1508 {
1509 int ret;
1510 Error *err = NULL;
1511
1512 vdev->msix->pending = g_malloc0(BITS_TO_LONGS(vdev->msix->entries) *
1513 sizeof(unsigned long));
1514 ret = msix_init(&vdev->pdev, vdev->msix->entries,
1515 vdev->bars[vdev->msix->table_bar].mr,
1516 vdev->msix->table_bar, vdev->msix->table_offset,
1517 vdev->bars[vdev->msix->pba_bar].mr,
1518 vdev->msix->pba_bar, vdev->msix->pba_offset, pos,
1519 &err);
1520 if (ret < 0) {
1521 if (ret == -ENOTSUP) {
1522 warn_report_err(err);
1523 return 0;
1524 }
1525
1526 error_propagate(errp, err);
1527 return ret;
1528 }
1529
1530 /*
1531 * The PCI spec suggests that devices provide additional alignment for
1532 * MSI-X structures and avoid overlapping non-MSI-X related registers.
1533 * For an assigned device, this hopefully means that emulation of MSI-X
1534 * structures does not affect the performance of the device. If devices
1535 * fail to provide that alignment, a significant performance penalty may
1536 * result, for instance Mellanox MT27500 VFs:
1537 * http://www.spinics.net/lists/kvm/msg125881.html
1538 *
1539 * The PBA is simply not that important for such a serious regression and
1540 * most drivers do not appear to look at it. The solution for this is to
1541 * disable the PBA MemoryRegion unless it's being used. We disable it
1542 * here and only enable it if a masked vector fires through QEMU. As the
1543 * vector-use notifier is called, which occurs on unmask, we test whether
1544 * PBA emulation is needed and again disable if not.
1545 */
1546 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
1547
1548 /*
1549 * The emulated machine may provide a paravirt interface for MSIX setup
1550 * so it is not strictly necessary to emulate MSIX here. This becomes
1551 * helpful when frequently accessed MMIO registers are located in
1552 * subpages adjacent to the MSIX table but the MSIX data containing page
1553 * cannot be mapped because of a host page size bigger than the MSIX table
1554 * alignment.
1555 */
1556 if (object_property_get_bool(OBJECT(qdev_get_machine()),
1557 "vfio-no-msix-emulation", NULL)) {
1558 memory_region_set_enabled(&vdev->pdev.msix_table_mmio, false);
1559 }
1560
1561 return 0;
1562 }
1563
1564 static void vfio_teardown_msi(VFIOPCIDevice *vdev)
1565 {
1566 msi_uninit(&vdev->pdev);
1567
1568 if (vdev->msix) {
1569 msix_uninit(&vdev->pdev,
1570 vdev->bars[vdev->msix->table_bar].mr,
1571 vdev->bars[vdev->msix->pba_bar].mr);
1572 g_free(vdev->msix->pending);
1573 }
1574 }
1575
1576 /*
1577 * Resource setup
1578 */
1579 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
1580 {
1581 int i;
1582
1583 for (i = 0; i < PCI_ROM_SLOT; i++) {
1584 vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled);
1585 }
1586 }
1587
1588 static void vfio_bar_prepare(VFIOPCIDevice *vdev, int nr)
1589 {
1590 VFIOBAR *bar = &vdev->bars[nr];
1591
1592 uint32_t pci_bar;
1593 int ret;
1594
1595 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
1596 if (!bar->region.size) {
1597 return;
1598 }
1599
1600 /* Determine what type of BAR this is for registration */
1601 ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar),
1602 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
1603 if (ret != sizeof(pci_bar)) {
1604 error_report("vfio: Failed to read BAR %d (%m)", nr);
1605 return;
1606 }
1607
1608 pci_bar = le32_to_cpu(pci_bar);
1609 bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
1610 bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
1611 bar->type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
1612 ~PCI_BASE_ADDRESS_MEM_MASK);
1613 bar->size = bar->region.size;
1614 }
1615
1616 static void vfio_bars_prepare(VFIOPCIDevice *vdev)
1617 {
1618 int i;
1619
1620 for (i = 0; i < PCI_ROM_SLOT; i++) {
1621 vfio_bar_prepare(vdev, i);
1622 }
1623 }
1624
1625 static void vfio_bar_register(VFIOPCIDevice *vdev, int nr)
1626 {
1627 VFIOBAR *bar = &vdev->bars[nr];
1628 char *name;
1629
1630 if (!bar->size) {
1631 return;
1632 }
1633
1634 bar->mr = g_new0(MemoryRegion, 1);
1635 name = g_strdup_printf("%s base BAR %d", vdev->vbasedev.name, nr);
1636 memory_region_init_io(bar->mr, OBJECT(vdev), NULL, NULL, name, bar->size);
1637 g_free(name);
1638
1639 if (bar->region.size) {
1640 memory_region_add_subregion(bar->mr, 0, bar->region.mem);
1641
1642 if (vfio_region_mmap(&bar->region)) {
1643 error_report("Failed to mmap %s BAR %d. Performance may be slow",
1644 vdev->vbasedev.name, nr);
1645 }
1646 }
1647
1648 pci_register_bar(&vdev->pdev, nr, bar->type, bar->mr);
1649 }
1650
1651 static void vfio_bars_register(VFIOPCIDevice *vdev)
1652 {
1653 int i;
1654
1655 for (i = 0; i < PCI_ROM_SLOT; i++) {
1656 vfio_bar_register(vdev, i);
1657 }
1658 }
1659
1660 static void vfio_bars_exit(VFIOPCIDevice *vdev)
1661 {
1662 int i;
1663
1664 for (i = 0; i < PCI_ROM_SLOT; i++) {
1665 VFIOBAR *bar = &vdev->bars[i];
1666
1667 vfio_bar_quirk_exit(vdev, i);
1668 vfio_region_exit(&bar->region);
1669 if (bar->region.size) {
1670 memory_region_del_subregion(bar->mr, bar->region.mem);
1671 }
1672 }
1673
1674 if (vdev->vga) {
1675 pci_unregister_vga(&vdev->pdev);
1676 vfio_vga_quirk_exit(vdev);
1677 }
1678 }
1679
1680 static void vfio_bars_finalize(VFIOPCIDevice *vdev)
1681 {
1682 int i;
1683
1684 for (i = 0; i < PCI_ROM_SLOT; i++) {
1685 VFIOBAR *bar = &vdev->bars[i];
1686
1687 vfio_bar_quirk_finalize(vdev, i);
1688 vfio_region_finalize(&bar->region);
1689 if (bar->size) {
1690 object_unparent(OBJECT(bar->mr));
1691 g_free(bar->mr);
1692 }
1693 }
1694
1695 if (vdev->vga) {
1696 vfio_vga_quirk_finalize(vdev);
1697 for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
1698 object_unparent(OBJECT(&vdev->vga->region[i].mem));
1699 }
1700 g_free(vdev->vga);
1701 }
1702 }
1703
1704 /*
1705 * General setup
1706 */
1707 static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
1708 {
1709 uint8_t tmp;
1710 uint16_t next = PCI_CONFIG_SPACE_SIZE;
1711
1712 for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
1713 tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) {
1714 if (tmp > pos && tmp < next) {
1715 next = tmp;
1716 }
1717 }
1718
1719 return next - pos;
1720 }
1721
1722
1723 static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
1724 {
1725 uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
1726
1727 for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
1728 tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
1729 if (tmp > pos && tmp < next) {
1730 next = tmp;
1731 }
1732 }
1733
1734 return next - pos;
1735 }
1736
1737 static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
1738 {
1739 pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
1740 }
1741
1742 static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos,
1743 uint16_t val, uint16_t mask)
1744 {
1745 vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
1746 vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
1747 vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
1748 }
1749
1750 static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
1751 {
1752 pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
1753 }
1754
1755 static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos,
1756 uint32_t val, uint32_t mask)
1757 {
1758 vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
1759 vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
1760 vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
1761 }
1762
1763 static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size,
1764 Error **errp)
1765 {
1766 uint16_t flags;
1767 uint8_t type;
1768
1769 flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
1770 type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
1771
1772 if (type != PCI_EXP_TYPE_ENDPOINT &&
1773 type != PCI_EXP_TYPE_LEG_END &&
1774 type != PCI_EXP_TYPE_RC_END) {
1775
1776 error_setg(errp, "assignment of PCIe type 0x%x "
1777 "devices is not currently supported", type);
1778 return -EINVAL;
1779 }
1780
1781 if (!pci_bus_is_express(pci_get_bus(&vdev->pdev))) {
1782 PCIBus *bus = pci_get_bus(&vdev->pdev);
1783 PCIDevice *bridge;
1784
1785 /*
1786 * Traditionally PCI device assignment exposes the PCIe capability
1787 * as-is on non-express buses. The reason being that some drivers
1788 * simply assume that it's there, for example tg3. However when
1789 * we're running on a native PCIe machine type, like Q35, we need
1790 * to hide the PCIe capability. The reason for this is twofold;
1791 * first Windows guests get a Code 10 error when the PCIe capability
1792 * is exposed in this configuration. Therefore express devices won't
1793 * work at all unless they're attached to express buses in the VM.
1794 * Second, a native PCIe machine introduces the possibility of fine
1795 * granularity IOMMUs supporting both translation and isolation.
1796 * Guest code to discover the IOMMU visibility of a device, such as
1797 * IOMMU grouping code on Linux, is very aware of device types and
1798 * valid transitions between bus types. An express device on a non-
1799 * express bus is not a valid combination on bare metal systems.
1800 *
1801 * Drivers that require a PCIe capability to make the device
1802 * functional are simply going to need to have their devices placed
1803 * on a PCIe bus in the VM.
1804 */
1805 while (!pci_bus_is_root(bus)) {
1806 bridge = pci_bridge_get_device(bus);
1807 bus = pci_get_bus(bridge);
1808 }
1809
1810 if (pci_bus_is_express(bus)) {
1811 return 0;
1812 }
1813
1814 } else if (pci_bus_is_root(pci_get_bus(&vdev->pdev))) {
1815 /*
1816 * On a Root Complex bus Endpoints become Root Complex Integrated
1817 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
1818 */
1819 if (type == PCI_EXP_TYPE_ENDPOINT) {
1820 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1821 PCI_EXP_TYPE_RC_END << 4,
1822 PCI_EXP_FLAGS_TYPE);
1823
1824 /* Link Capabilities, Status, and Control goes away */
1825 if (size > PCI_EXP_LNKCTL) {
1826 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
1827 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1828 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
1829
1830 #ifndef PCI_EXP_LNKCAP2
1831 #define PCI_EXP_LNKCAP2 44
1832 #endif
1833 #ifndef PCI_EXP_LNKSTA2
1834 #define PCI_EXP_LNKSTA2 50
1835 #endif
1836 /* Link 2 Capabilities, Status, and Control goes away */
1837 if (size > PCI_EXP_LNKCAP2) {
1838 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
1839 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
1840 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
1841 }
1842 }
1843
1844 } else if (type == PCI_EXP_TYPE_LEG_END) {
1845 /*
1846 * Legacy endpoints don't belong on the root complex. Windows
1847 * seems to be happier with devices if we skip the capability.
1848 */
1849 return 0;
1850 }
1851
1852 } else {
1853 /*
1854 * Convert Root Complex Integrated Endpoints to regular endpoints.
1855 * These devices don't support LNK/LNK2 capabilities, so make them up.
1856 */
1857 if (type == PCI_EXP_TYPE_RC_END) {
1858 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1859 PCI_EXP_TYPE_ENDPOINT << 4,
1860 PCI_EXP_FLAGS_TYPE);
1861 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
1862 QEMU_PCI_EXP_LNKCAP_MLW(QEMU_PCI_EXP_LNK_X1) |
1863 QEMU_PCI_EXP_LNKCAP_MLS(QEMU_PCI_EXP_LNK_2_5GT), ~0);
1864 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1865 }
1866 }
1867
1868 /*
1869 * Intel 82599 SR-IOV VFs report an invalid PCIe capability version 0
1870 * (Niantic errate #35) causing Windows to error with a Code 10 for the
1871 * device on Q35. Fixup any such devices to report version 1. If we
1872 * were to remove the capability entirely the guest would lose extended
1873 * config space.
1874 */
1875 if ((flags & PCI_EXP_FLAGS_VERS) == 0) {
1876 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1877 1, PCI_EXP_FLAGS_VERS);
1878 }
1879
1880 pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size,
1881 errp);
1882 if (pos < 0) {
1883 return pos;
1884 }
1885
1886 vdev->pdev.exp.exp_cap = pos;
1887
1888 return pos;
1889 }
1890
1891 static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos)
1892 {
1893 uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
1894
1895 if (cap & PCI_EXP_DEVCAP_FLR) {
1896 trace_vfio_check_pcie_flr(vdev->vbasedev.name);
1897 vdev->has_flr = true;
1898 }
1899 }
1900
1901 static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos)
1902 {
1903 uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
1904
1905 if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
1906 trace_vfio_check_pm_reset(vdev->vbasedev.name);
1907 vdev->has_pm_reset = true;
1908 }
1909 }
1910
1911 static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos)
1912 {
1913 uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
1914
1915 if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
1916 trace_vfio_check_af_flr(vdev->vbasedev.name);
1917 vdev->has_flr = true;
1918 }
1919 }
1920
1921 static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos, Error **errp)
1922 {
1923 PCIDevice *pdev = &vdev->pdev;
1924 uint8_t cap_id, next, size;
1925 int ret;
1926
1927 cap_id = pdev->config[pos];
1928 next = pdev->config[pos + PCI_CAP_LIST_NEXT];
1929
1930 /*
1931 * If it becomes important to configure capabilities to their actual
1932 * size, use this as the default when it's something we don't recognize.
1933 * Since QEMU doesn't actually handle many of the config accesses,
1934 * exact size doesn't seem worthwhile.
1935 */
1936 size = vfio_std_cap_max_size(pdev, pos);
1937
1938 /*
1939 * pci_add_capability always inserts the new capability at the head
1940 * of the chain. Therefore to end up with a chain that matches the
1941 * physical device, we insert from the end by making this recursive.
1942 * This is also why we pre-calculate size above as cached config space
1943 * will be changed as we unwind the stack.
1944 */
1945 if (next) {
1946 ret = vfio_add_std_cap(vdev, next, errp);
1947 if (ret) {
1948 return ret;
1949 }
1950 } else {
1951 /* Begin the rebuild, use QEMU emulated list bits */
1952 pdev->config[PCI_CAPABILITY_LIST] = 0;
1953 vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
1954 vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
1955
1956 ret = vfio_add_virt_caps(vdev, errp);
1957 if (ret) {
1958 return ret;
1959 }
1960 }
1961
1962 /* Scale down size, esp in case virt caps were added above */
1963 size = MIN(size, vfio_std_cap_max_size(pdev, pos));
1964
1965 /* Use emulated next pointer to allow dropping caps */
1966 pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff);
1967
1968 switch (cap_id) {
1969 case PCI_CAP_ID_MSI:
1970 ret = vfio_msi_setup(vdev, pos, errp);
1971 break;
1972 case PCI_CAP_ID_EXP:
1973 vfio_check_pcie_flr(vdev, pos);
1974 ret = vfio_setup_pcie_cap(vdev, pos, size, errp);
1975 break;
1976 case PCI_CAP_ID_MSIX:
1977 ret = vfio_msix_setup(vdev, pos, errp);
1978 break;
1979 case PCI_CAP_ID_PM:
1980 vfio_check_pm_reset(vdev, pos);
1981 vdev->pm_cap = pos;
1982 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
1983 break;
1984 case PCI_CAP_ID_AF:
1985 vfio_check_af_flr(vdev, pos);
1986 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
1987 break;
1988 default:
1989 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
1990 break;
1991 }
1992
1993 if (ret < 0) {
1994 error_prepend(errp,
1995 "failed to add PCI capability 0x%x[0x%x]@0x%x: ",
1996 cap_id, size, pos);
1997 return ret;
1998 }
1999
2000 return 0;
2001 }
2002
2003 static void vfio_add_ext_cap(VFIOPCIDevice *vdev)
2004 {
2005 PCIDevice *pdev = &vdev->pdev;
2006 uint32_t header;
2007 uint16_t cap_id, next, size;
2008 uint8_t cap_ver;
2009 uint8_t *config;
2010
2011 /* Only add extended caps if we have them and the guest can see them */
2012 if (!pci_is_express(pdev) || !pci_bus_is_express(pci_get_bus(pdev)) ||
2013 !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
2014 return;
2015 }
2016
2017 /*
2018 * pcie_add_capability always inserts the new capability at the tail
2019 * of the chain. Therefore to end up with a chain that matches the
2020 * physical device, we cache the config space to avoid overwriting
2021 * the original config space when we parse the extended capabilities.
2022 */
2023 config = g_memdup(pdev->config, vdev->config_size);
2024
2025 /*
2026 * Extended capabilities are chained with each pointing to the next, so we
2027 * can drop anything other than the head of the chain simply by modifying
2028 * the previous next pointer. Seed the head of the chain here such that
2029 * we can simply skip any capabilities we want to drop below, regardless
2030 * of their position in the chain. If this stub capability still exists
2031 * after we add the capabilities we want to expose, update the capability
2032 * ID to zero. Note that we cannot seed with the capability header being
2033 * zero as this conflicts with definition of an absent capability chain
2034 * and prevents capabilities beyond the head of the list from being added.
2035 * By replacing the dummy capability ID with zero after walking the device
2036 * chain, we also transparently mark extended capabilities as absent if
2037 * no capabilities were added. Note that the PCIe spec defines an absence
2038 * of extended capabilities to be determined by a value of zero for the
2039 * capability ID, version, AND next pointer. A non-zero next pointer
2040 * should be sufficient to indicate additional capabilities are present,
2041 * which will occur if we call pcie_add_capability() below. The entire
2042 * first dword is emulated to support this.
2043 *
2044 * NB. The kernel side does similar masking, so be prepared that our
2045 * view of the device may also contain a capability ID zero in the head
2046 * of the chain. Skip it for the same reason that we cannot seed the
2047 * chain with a zero capability.
2048 */
2049 pci_set_long(pdev->config + PCI_CONFIG_SPACE_SIZE,
2050 PCI_EXT_CAP(0xFFFF, 0, 0));
2051 pci_set_long(pdev->wmask + PCI_CONFIG_SPACE_SIZE, 0);
2052 pci_set_long(vdev->emulated_config_bits + PCI_CONFIG_SPACE_SIZE, ~0);
2053
2054 for (next = PCI_CONFIG_SPACE_SIZE; next;
2055 next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
2056 header = pci_get_long(config + next);
2057 cap_id = PCI_EXT_CAP_ID(header);
2058 cap_ver = PCI_EXT_CAP_VER(header);
2059
2060 /*
2061 * If it becomes important to configure extended capabilities to their
2062 * actual size, use this as the default when it's something we don't
2063 * recognize. Since QEMU doesn't actually handle many of the config
2064 * accesses, exact size doesn't seem worthwhile.
2065 */
2066 size = vfio_ext_cap_max_size(config, next);
2067
2068 /* Use emulated next pointer to allow dropping extended caps */
2069 pci_long_test_and_set_mask(vdev->emulated_config_bits + next,
2070 PCI_EXT_CAP_NEXT_MASK);
2071
2072 switch (cap_id) {
2073 case 0: /* kernel masked capability */
2074 case PCI_EXT_CAP_ID_SRIOV: /* Read-only VF BARs confuse OVMF */
2075 case PCI_EXT_CAP_ID_ARI: /* XXX Needs next function virtualization */
2076 case PCI_EXT_CAP_ID_REBAR: /* Can't expose read-only */
2077 trace_vfio_add_ext_cap_dropped(vdev->vbasedev.name, cap_id, next);
2078 break;
2079 default:
2080 pcie_add_capability(pdev, cap_id, cap_ver, next, size);
2081 }
2082
2083 }
2084
2085 /* Cleanup chain head ID if necessary */
2086 if (pci_get_word(pdev->config + PCI_CONFIG_SPACE_SIZE) == 0xFFFF) {
2087 pci_set_word(pdev->config + PCI_CONFIG_SPACE_SIZE, 0);
2088 }
2089
2090 g_free(config);
2091 return;
2092 }
2093
2094 static int vfio_add_capabilities(VFIOPCIDevice *vdev, Error **errp)
2095 {
2096 PCIDevice *pdev = &vdev->pdev;
2097 int ret;
2098
2099 if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
2100 !pdev->config[PCI_CAPABILITY_LIST]) {
2101 return 0; /* Nothing to add */
2102 }
2103
2104 ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST], errp);
2105 if (ret) {
2106 return ret;
2107 }
2108
2109 vfio_add_ext_cap(vdev);
2110 return 0;
2111 }
2112
2113 static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
2114 {
2115 PCIDevice *pdev = &vdev->pdev;
2116 uint16_t cmd;
2117
2118 vfio_disable_interrupts(vdev);
2119
2120 /* Make sure the device is in D0 */
2121 if (vdev->pm_cap) {
2122 uint16_t pmcsr;
2123 uint8_t state;
2124
2125 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2126 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2127 if (state) {
2128 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2129 vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
2130 /* vfio handles the necessary delay here */
2131 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2132 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2133 if (state) {
2134 error_report("vfio: Unable to power on device, stuck in D%d",
2135 state);
2136 }
2137 }
2138 }
2139
2140 /*
2141 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
2142 * Also put INTx Disable in known state.
2143 */
2144 cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
2145 cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
2146 PCI_COMMAND_INTX_DISABLE);
2147 vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
2148 }
2149
2150 static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
2151 {
2152 Error *err = NULL;
2153 int nr;
2154
2155 vfio_intx_enable(vdev, &err);
2156 if (err) {
2157 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2158 }
2159
2160 for (nr = 0; nr < PCI_NUM_REGIONS - 1; ++nr) {
2161 off_t addr = vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr);
2162 uint32_t val = 0;
2163 uint32_t len = sizeof(val);
2164
2165 if (pwrite(vdev->vbasedev.fd, &val, len, addr) != len) {
2166 error_report("%s(%s) reset bar %d failed: %m", __func__,
2167 vdev->vbasedev.name, nr);
2168 }
2169 }
2170
2171 vfio_quirk_reset(vdev);
2172 }
2173
2174 static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
2175 {
2176 char tmp[13];
2177
2178 sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain,
2179 addr->bus, addr->slot, addr->function);
2180
2181 return (strcmp(tmp, name) == 0);
2182 }
2183
2184 static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
2185 {
2186 VFIOGroup *group;
2187 struct vfio_pci_hot_reset_info *info;
2188 struct vfio_pci_dependent_device *devices;
2189 struct vfio_pci_hot_reset *reset;
2190 int32_t *fds;
2191 int ret, i, count;
2192 bool multi = false;
2193
2194 trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
2195
2196 if (!single) {
2197 vfio_pci_pre_reset(vdev);
2198 }
2199 vdev->vbasedev.needs_reset = false;
2200
2201 info = g_malloc0(sizeof(*info));
2202 info->argsz = sizeof(*info);
2203
2204 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2205 if (ret && errno != ENOSPC) {
2206 ret = -errno;
2207 if (!vdev->has_pm_reset) {
2208 error_report("vfio: Cannot reset device %s, "
2209 "no available reset mechanism.", vdev->vbasedev.name);
2210 }
2211 goto out_single;
2212 }
2213
2214 count = info->count;
2215 info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
2216 info->argsz = sizeof(*info) + (count * sizeof(*devices));
2217 devices = &info->devices[0];
2218
2219 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2220 if (ret) {
2221 ret = -errno;
2222 error_report("vfio: hot reset info failed: %m");
2223 goto out_single;
2224 }
2225
2226 trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
2227
2228 /* Verify that we have all the groups required */
2229 for (i = 0; i < info->count; i++) {
2230 PCIHostDeviceAddress host;
2231 VFIOPCIDevice *tmp;
2232 VFIODevice *vbasedev_iter;
2233
2234 host.domain = devices[i].segment;
2235 host.bus = devices[i].bus;
2236 host.slot = PCI_SLOT(devices[i].devfn);
2237 host.function = PCI_FUNC(devices[i].devfn);
2238
2239 trace_vfio_pci_hot_reset_dep_devices(host.domain,
2240 host.bus, host.slot, host.function, devices[i].group_id);
2241
2242 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2243 continue;
2244 }
2245
2246 QLIST_FOREACH(group, &vfio_group_list, next) {
2247 if (group->groupid == devices[i].group_id) {
2248 break;
2249 }
2250 }
2251
2252 if (!group) {
2253 if (!vdev->has_pm_reset) {
2254 error_report("vfio: Cannot reset device %s, "
2255 "depends on group %d which is not owned.",
2256 vdev->vbasedev.name, devices[i].group_id);
2257 }
2258 ret = -EPERM;
2259 goto out;
2260 }
2261
2262 /* Prep dependent devices for reset and clear our marker. */
2263 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2264 if (!vbasedev_iter->dev->realized ||
2265 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2266 continue;
2267 }
2268 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2269 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2270 if (single) {
2271 ret = -EINVAL;
2272 goto out_single;
2273 }
2274 vfio_pci_pre_reset(tmp);
2275 tmp->vbasedev.needs_reset = false;
2276 multi = true;
2277 break;
2278 }
2279 }
2280 }
2281
2282 if (!single && !multi) {
2283 ret = -EINVAL;
2284 goto out_single;
2285 }
2286
2287 /* Determine how many group fds need to be passed */
2288 count = 0;
2289 QLIST_FOREACH(group, &vfio_group_list, next) {
2290 for (i = 0; i < info->count; i++) {
2291 if (group->groupid == devices[i].group_id) {
2292 count++;
2293 break;
2294 }
2295 }
2296 }
2297
2298 reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
2299 reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
2300 fds = &reset->group_fds[0];
2301
2302 /* Fill in group fds */
2303 QLIST_FOREACH(group, &vfio_group_list, next) {
2304 for (i = 0; i < info->count; i++) {
2305 if (group->groupid == devices[i].group_id) {
2306 fds[reset->count++] = group->fd;
2307 break;
2308 }
2309 }
2310 }
2311
2312 /* Bus reset! */
2313 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
2314 g_free(reset);
2315
2316 trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
2317 ret ? "%m" : "Success");
2318
2319 out:
2320 /* Re-enable INTx on affected devices */
2321 for (i = 0; i < info->count; i++) {
2322 PCIHostDeviceAddress host;
2323 VFIOPCIDevice *tmp;
2324 VFIODevice *vbasedev_iter;
2325
2326 host.domain = devices[i].segment;
2327 host.bus = devices[i].bus;
2328 host.slot = PCI_SLOT(devices[i].devfn);
2329 host.function = PCI_FUNC(devices[i].devfn);
2330
2331 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2332 continue;
2333 }
2334
2335 QLIST_FOREACH(group, &vfio_group_list, next) {
2336 if (group->groupid == devices[i].group_id) {
2337 break;
2338 }
2339 }
2340
2341 if (!group) {
2342 break;
2343 }
2344
2345 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2346 if (!vbasedev_iter->dev->realized ||
2347 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2348 continue;
2349 }
2350 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2351 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2352 vfio_pci_post_reset(tmp);
2353 break;
2354 }
2355 }
2356 }
2357 out_single:
2358 if (!single) {
2359 vfio_pci_post_reset(vdev);
2360 }
2361 g_free(info);
2362
2363 return ret;
2364 }
2365
2366 /*
2367 * We want to differentiate hot reset of mulitple in-use devices vs hot reset
2368 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case
2369 * of doing hot resets when there is only a single device per bus. The in-use
2370 * here refers to how many VFIODevices are affected. A hot reset that affects
2371 * multiple devices, but only a single in-use device, means that we can call
2372 * it from our bus ->reset() callback since the extent is effectively a single
2373 * device. This allows us to make use of it in the hotplug path. When there
2374 * are multiple in-use devices, we can only trigger the hot reset during a
2375 * system reset and thus from our reset handler. We separate _one vs _multi
2376 * here so that we don't overlap and do a double reset on the system reset
2377 * path where both our reset handler and ->reset() callback are used. Calling
2378 * _one() will only do a hot reset for the one in-use devices case, calling
2379 * _multi() will do nothing if a _one() would have been sufficient.
2380 */
2381 static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev)
2382 {
2383 return vfio_pci_hot_reset(vdev, true);
2384 }
2385
2386 static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev)
2387 {
2388 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2389 return vfio_pci_hot_reset(vdev, false);
2390 }
2391
2392 static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
2393 {
2394 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2395 if (!vbasedev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
2396 vbasedev->needs_reset = true;
2397 }
2398 }
2399
2400 static VFIODeviceOps vfio_pci_ops = {
2401 .vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
2402 .vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
2403 .vfio_eoi = vfio_intx_eoi,
2404 };
2405
2406 int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp)
2407 {
2408 VFIODevice *vbasedev = &vdev->vbasedev;
2409 struct vfio_region_info *reg_info;
2410 int ret;
2411
2412 ret = vfio_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, &reg_info);
2413 if (ret) {
2414 error_setg_errno(errp, -ret,
2415 "failed getting region info for VGA region index %d",
2416 VFIO_PCI_VGA_REGION_INDEX);
2417 return ret;
2418 }
2419
2420 if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) ||
2421 !(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) ||
2422 reg_info->size < 0xbffff + 1) {
2423 error_setg(errp, "unexpected VGA info, flags 0x%lx, size 0x%lx",
2424 (unsigned long)reg_info->flags,
2425 (unsigned long)reg_info->size);
2426 g_free(reg_info);
2427 return -EINVAL;
2428 }
2429
2430 vdev->vga = g_new0(VFIOVGA, 1);
2431
2432 vdev->vga->fd_offset = reg_info->offset;
2433 vdev->vga->fd = vdev->vbasedev.fd;
2434
2435 g_free(reg_info);
2436
2437 vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
2438 vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
2439 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks);
2440
2441 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2442 OBJECT(vdev), &vfio_vga_ops,
2443 &vdev->vga->region[QEMU_PCI_VGA_MEM],
2444 "vfio-vga-mmio@0xa0000",
2445 QEMU_PCI_VGA_MEM_SIZE);
2446
2447 vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
2448 vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
2449 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks);
2450
2451 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2452 OBJECT(vdev), &vfio_vga_ops,
2453 &vdev->vga->region[QEMU_PCI_VGA_IO_LO],
2454 "vfio-vga-io@0x3b0",
2455 QEMU_PCI_VGA_IO_LO_SIZE);
2456
2457 vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
2458 vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
2459 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks);
2460
2461 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
2462 OBJECT(vdev), &vfio_vga_ops,
2463 &vdev->vga->region[QEMU_PCI_VGA_IO_HI],
2464 "vfio-vga-io@0x3c0",
2465 QEMU_PCI_VGA_IO_HI_SIZE);
2466
2467 pci_register_vga(&vdev->pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2468 &vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2469 &vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem);
2470
2471 return 0;
2472 }
2473
2474 static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
2475 {
2476 VFIODevice *vbasedev = &vdev->vbasedev;
2477 struct vfio_region_info *reg_info;
2478 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
2479 int i, ret = -1;
2480
2481 /* Sanity check device */
2482 if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) {
2483 error_setg(errp, "this isn't a PCI device");
2484 return;
2485 }
2486
2487 if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
2488 error_setg(errp, "unexpected number of io regions %u",
2489 vbasedev->num_regions);
2490 return;
2491 }
2492
2493 if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
2494 error_setg(errp, "unexpected number of irqs %u", vbasedev->num_irqs);
2495 return;
2496 }
2497
2498 for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
2499 char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
2500
2501 ret = vfio_region_setup(OBJECT(vdev), vbasedev,
2502 &vdev->bars[i].region, i, name);
2503 g_free(name);
2504
2505 if (ret) {
2506 error_setg_errno(errp, -ret, "failed to get region %d info", i);
2507 return;
2508 }
2509
2510 QLIST_INIT(&vdev->bars[i].quirks);
2511 }
2512
2513 ret = vfio_get_region_info(vbasedev,
2514 VFIO_PCI_CONFIG_REGION_INDEX, &reg_info);
2515 if (ret) {
2516 error_setg_errno(errp, -ret, "failed to get config info");
2517 return;
2518 }
2519
2520 trace_vfio_populate_device_config(vdev->vbasedev.name,
2521 (unsigned long)reg_info->size,
2522 (unsigned long)reg_info->offset,
2523 (unsigned long)reg_info->flags);
2524
2525 vdev->config_size = reg_info->size;
2526 if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
2527 vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
2528 }
2529 vdev->config_offset = reg_info->offset;
2530
2531 g_free(reg_info);
2532
2533 if (vdev->features & VFIO_FEATURE_ENABLE_VGA) {
2534 ret = vfio_populate_vga(vdev, errp);
2535 if (ret) {
2536 error_append_hint(errp, "device does not support "
2537 "requested feature x-vga\n");
2538 return;
2539 }
2540 }
2541
2542 irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
2543
2544 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
2545 if (ret) {
2546 /* This can fail for an old kernel or legacy PCI dev */
2547 trace_vfio_populate_device_get_irq_info_failure(strerror(errno));
2548 } else if (irq_info.count == 1) {
2549 vdev->pci_aer = true;
2550 } else {
2551 warn_report(VFIO_MSG_PREFIX
2552 "Could not enable error recovery for the device",
2553 vbasedev->name);
2554 }
2555 }
2556
2557 static void vfio_put_device(VFIOPCIDevice *vdev)
2558 {
2559 g_free(vdev->vbasedev.name);
2560 g_free(vdev->msix);
2561
2562 vfio_put_base_device(&vdev->vbasedev);
2563 }
2564
2565 static void vfio_err_notifier_handler(void *opaque)
2566 {
2567 VFIOPCIDevice *vdev = opaque;
2568
2569 if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
2570 return;
2571 }
2572
2573 /*
2574 * TBD. Retrieve the error details and decide what action
2575 * needs to be taken. One of the actions could be to pass
2576 * the error to the guest and have the guest driver recover
2577 * from the error. This requires that PCIe capabilities be
2578 * exposed to the guest. For now, we just terminate the
2579 * guest to contain the error.
2580 */
2581
2582 error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name);
2583
2584 vm_stop(RUN_STATE_INTERNAL_ERROR);
2585 }
2586
2587 /*
2588 * Registers error notifier for devices supporting error recovery.
2589 * If we encounter a failure in this function, we report an error
2590 * and continue after disabling error recovery support for the
2591 * device.
2592 */
2593 static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
2594 {
2595 Error *err = NULL;
2596 int32_t fd;
2597
2598 if (!vdev->pci_aer) {
2599 return;
2600 }
2601
2602 if (event_notifier_init(&vdev->err_notifier, 0)) {
2603 error_report("vfio: Unable to init event notifier for error detection");
2604 vdev->pci_aer = false;
2605 return;
2606 }
2607
2608 fd = event_notifier_get_fd(&vdev->err_notifier);
2609 qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
2610
2611 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
2612 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
2613 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2614 qemu_set_fd_handler(fd, NULL, NULL, vdev);
2615 event_notifier_cleanup(&vdev->err_notifier);
2616 vdev->pci_aer = false;
2617 }
2618 }
2619
2620 static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev)
2621 {
2622 Error *err = NULL;
2623
2624 if (!vdev->pci_aer) {
2625 return;
2626 }
2627
2628 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
2629 VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
2630 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2631 }
2632 qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
2633 NULL, NULL, vdev);
2634 event_notifier_cleanup(&vdev->err_notifier);
2635 }
2636
2637 static void vfio_req_notifier_handler(void *opaque)
2638 {
2639 VFIOPCIDevice *vdev = opaque;
2640 Error *err = NULL;
2641
2642 if (!event_notifier_test_and_clear(&vdev->req_notifier)) {
2643 return;
2644 }
2645
2646 qdev_unplug(DEVICE(vdev), &err);
2647 if (err) {
2648 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2649 }
2650 }
2651
2652 static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
2653 {
2654 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
2655 .index = VFIO_PCI_REQ_IRQ_INDEX };
2656 Error *err = NULL;
2657 int32_t fd;
2658
2659 if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) {
2660 return;
2661 }
2662
2663 if (ioctl(vdev->vbasedev.fd,
2664 VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) {
2665 return;
2666 }
2667
2668 if (event_notifier_init(&vdev->req_notifier, 0)) {
2669 error_report("vfio: Unable to init event notifier for device request");
2670 return;
2671 }
2672
2673 fd = event_notifier_get_fd(&vdev->req_notifier);
2674 qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
2675
2676 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
2677 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
2678 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2679 qemu_set_fd_handler(fd, NULL, NULL, vdev);
2680 event_notifier_cleanup(&vdev->req_notifier);
2681 } else {
2682 vdev->req_enabled = true;
2683 }
2684 }
2685
2686 static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
2687 {
2688 Error *err = NULL;
2689
2690 if (!vdev->req_enabled) {
2691 return;
2692 }
2693
2694 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
2695 VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
2696 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2697 }
2698 qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier),
2699 NULL, NULL, vdev);
2700 event_notifier_cleanup(&vdev->req_notifier);
2701
2702 vdev->req_enabled = false;
2703 }
2704
2705 static void vfio_realize(PCIDevice *pdev, Error **errp)
2706 {
2707 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
2708 VFIODevice *vbasedev_iter;
2709 VFIOGroup *group;
2710 char *tmp, *subsys, group_path[PATH_MAX], *group_name;
2711 Error *err = NULL;
2712 ssize_t len;
2713 struct stat st;
2714 int groupid;
2715 int i, ret;
2716 bool is_mdev;
2717
2718 if (!vdev->vbasedev.sysfsdev) {
2719 if (!(~vdev->host.domain || ~vdev->host.bus ||
2720 ~vdev->host.slot || ~vdev->host.function)) {
2721 error_setg(errp, "No provided host device");
2722 error_append_hint(errp, "Use -device vfio-pci,host=DDDD:BB:DD.F "
2723 "or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
2724 return;
2725 }
2726 vdev->vbasedev.sysfsdev =
2727 g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
2728 vdev->host.domain, vdev->host.bus,
2729 vdev->host.slot, vdev->host.function);
2730 }
2731
2732 if (stat(vdev->vbasedev.sysfsdev, &st) < 0) {
2733 error_setg_errno(errp, errno, "no such host device");
2734 error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.sysfsdev);
2735 return;
2736 }
2737
2738 if (!pdev->failover_pair_id) {
2739 error_setg(&vdev->migration_blocker,
2740 "VFIO device doesn't support migration");
2741 ret = migrate_add_blocker(vdev->migration_blocker, errp);
2742 if (ret) {
2743 error_free(vdev->migration_blocker);
2744 vdev->migration_blocker = NULL;
2745 return;
2746 }
2747 }
2748
2749 vdev->vbasedev.name = g_path_get_basename(vdev->vbasedev.sysfsdev);
2750 vdev->vbasedev.ops = &vfio_pci_ops;
2751 vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI;
2752 vdev->vbasedev.dev = DEVICE(vdev);
2753
2754 tmp = g_strdup_printf("%s/iommu_group", vdev->vbasedev.sysfsdev);
2755 len = readlink(tmp, group_path, sizeof(group_path));
2756 g_free(tmp);
2757
2758 if (len <= 0 || len >= sizeof(group_path)) {
2759 error_setg_errno(errp, len < 0 ? errno : ENAMETOOLONG,
2760 "no iommu_group found");
2761 goto error;
2762 }
2763
2764 group_path[len] = 0;
2765
2766 group_name = basename(group_path);
2767 if (sscanf(group_name, "%d", &groupid) != 1) {
2768 error_setg_errno(errp, errno, "failed to read %s", group_path);
2769 goto error;
2770 }
2771
2772 trace_vfio_realize(vdev->vbasedev.name, groupid);
2773
2774 group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev), errp);
2775 if (!group) {
2776 goto error;
2777 }
2778
2779 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2780 if (strcmp(vbasedev_iter->name, vdev->vbasedev.name) == 0) {
2781 error_setg(errp, "device is already attached");
2782 vfio_put_group(group);
2783 goto error;
2784 }
2785 }
2786
2787 /*
2788 * Mediated devices *might* operate compatibly with discarding of RAM, but
2789 * we cannot know for certain, it depends on whether the mdev vendor driver
2790 * stays in sync with the active working set of the guest driver. Prevent
2791 * the x-balloon-allowed option unless this is minimally an mdev device.
2792 */
2793 tmp = g_strdup_printf("%s/subsystem", vdev->vbasedev.sysfsdev);
2794 subsys = realpath(tmp, NULL);
2795 g_free(tmp);
2796 is_mdev = subsys && (strcmp(subsys, "/sys/bus/mdev") == 0);
2797 free(subsys);
2798
2799 trace_vfio_mdev(vdev->vbasedev.name, is_mdev);
2800
2801 if (vdev->vbasedev.ram_block_discard_allowed && !is_mdev) {
2802 error_setg(errp, "x-balloon-allowed only potentially compatible "
2803 "with mdev devices");
2804 vfio_put_group(group);
2805 goto error;
2806 }
2807
2808 ret = vfio_get_device(group, vdev->vbasedev.name, &vdev->vbasedev, errp);
2809 if (ret) {
2810 vfio_put_group(group);
2811 goto error;
2812 }
2813
2814 vfio_populate_device(vdev, &err);
2815 if (err) {
2816 error_propagate(errp, err);
2817 goto error;
2818 }
2819
2820 /* Get a copy of config space */
2821 ret = pread(vdev->vbasedev.fd, vdev->pdev.config,
2822 MIN(pci_config_size(&vdev->pdev), vdev->config_size),
2823 vdev->config_offset);
2824 if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
2825 ret = ret < 0 ? -errno : -EFAULT;
2826 error_setg_errno(errp, -ret, "failed to read device config space");
2827 goto error;
2828 }
2829
2830 /* vfio emulates a lot for us, but some bits need extra love */
2831 vdev->emulated_config_bits = g_malloc0(vdev->config_size);
2832
2833 /* QEMU can choose to expose the ROM or not */
2834 memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
2835 /* QEMU can also add or extend BARs */
2836 memset(vdev->emulated_config_bits + PCI_BASE_ADDRESS_0, 0xff, 6 * 4);
2837
2838 /*
2839 * The PCI spec reserves vendor ID 0xffff as an invalid value. The
2840 * device ID is managed by the vendor and need only be a 16-bit value.
2841 * Allow any 16-bit value for subsystem so they can be hidden or changed.
2842 */
2843 if (vdev->vendor_id != PCI_ANY_ID) {
2844 if (vdev->vendor_id >= 0xffff) {
2845 error_setg(errp, "invalid PCI vendor ID provided");
2846 goto error;
2847 }
2848 vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
2849 trace_vfio_pci_emulated_vendor_id(vdev->vbasedev.name, vdev->vendor_id);
2850 } else {
2851 vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
2852 }
2853
2854 if (vdev->device_id != PCI_ANY_ID) {
2855 if (vdev->device_id > 0xffff) {
2856 error_setg(errp, "invalid PCI device ID provided");
2857 goto error;
2858 }
2859 vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
2860 trace_vfio_pci_emulated_device_id(vdev->vbasedev.name, vdev->device_id);
2861 } else {
2862 vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
2863 }
2864
2865 if (vdev->sub_vendor_id != PCI_ANY_ID) {
2866 if (vdev->sub_vendor_id > 0xffff) {
2867 error_setg(errp, "invalid PCI subsystem vendor ID provided");
2868 goto error;
2869 }
2870 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
2871 vdev->sub_vendor_id, ~0);
2872 trace_vfio_pci_emulated_sub_vendor_id(vdev->vbasedev.name,
2873 vdev->sub_vendor_id);
2874 }
2875
2876 if (vdev->sub_device_id != PCI_ANY_ID) {
2877 if (vdev->sub_device_id > 0xffff) {
2878 error_setg(errp, "invalid PCI subsystem device ID provided");
2879 goto error;
2880 }
2881 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
2882 trace_vfio_pci_emulated_sub_device_id(vdev->vbasedev.name,
2883 vdev->sub_device_id);
2884 }
2885
2886 /* QEMU can change multi-function devices to single function, or reverse */
2887 vdev->emulated_config_bits[PCI_HEADER_TYPE] =
2888 PCI_HEADER_TYPE_MULTI_FUNCTION;
2889
2890 /* Restore or clear multifunction, this is always controlled by QEMU */
2891 if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
2892 vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
2893 } else {
2894 vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
2895 }
2896
2897 /*
2898 * Clear host resource mapping info. If we choose not to register a
2899 * BAR, such as might be the case with the option ROM, we can get
2900 * confusing, unwritable, residual addresses from the host here.
2901 */
2902 memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
2903 memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
2904
2905 vfio_pci_size_rom(vdev);
2906
2907 vfio_bars_prepare(vdev);
2908
2909 vfio_msix_early_setup(vdev, &err);
2910 if (err) {
2911 error_propagate(errp, err);
2912 goto error;
2913 }
2914
2915 vfio_bars_register(vdev);
2916
2917 ret = vfio_add_capabilities(vdev, errp);
2918 if (ret) {
2919 goto out_teardown;
2920 }
2921
2922 if (vdev->vga) {
2923 vfio_vga_quirk_setup(vdev);
2924 }
2925
2926 for (i = 0; i < PCI_ROM_SLOT; i++) {
2927 vfio_bar_quirk_setup(vdev, i);
2928 }
2929
2930 if (!vdev->igd_opregion &&
2931 vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) {
2932 struct vfio_region_info *opregion;
2933
2934 if (vdev->pdev.qdev.hotplugged) {
2935 error_setg(errp,
2936 "cannot support IGD OpRegion feature on hotplugged "
2937 "device");
2938 goto out_teardown;
2939 }
2940
2941 ret = vfio_get_dev_region_info(&vdev->vbasedev,
2942 VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
2943 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion);
2944 if (ret) {
2945 error_setg_errno(errp, -ret,
2946 "does not support requested IGD OpRegion feature");
2947 goto out_teardown;
2948 }
2949
2950 ret = vfio_pci_igd_opregion_init(vdev, opregion, errp);
2951 g_free(opregion);
2952 if (ret) {
2953 goto out_teardown;
2954 }
2955 }
2956
2957 /* QEMU emulates all of MSI & MSIX */
2958 if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
2959 memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
2960 MSIX_CAP_LENGTH);
2961 }
2962
2963 if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
2964 memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
2965 vdev->msi_cap_size);
2966 }
2967
2968 if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
2969 vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
2970 vfio_intx_mmap_enable, vdev);
2971 pci_device_set_intx_routing_notifier(&vdev->pdev,
2972 vfio_intx_routing_notifier);
2973 vdev->irqchip_change_notifier.notify = vfio_irqchip_change;
2974 kvm_irqchip_add_change_notifier(&vdev->irqchip_change_notifier);
2975 ret = vfio_intx_enable(vdev, errp);
2976 if (ret) {
2977 goto out_deregister;
2978 }
2979 }
2980
2981 if (vdev->display != ON_OFF_AUTO_OFF) {
2982 ret = vfio_display_probe(vdev, errp);
2983 if (ret) {
2984 goto out_deregister;
2985 }
2986 }
2987 if (vdev->enable_ramfb && vdev->dpy == NULL) {
2988 error_setg(errp, "ramfb=on requires display=on");
2989 goto out_deregister;
2990 }
2991 if (vdev->display_xres || vdev->display_yres) {
2992 if (vdev->dpy == NULL) {
2993 error_setg(errp, "xres and yres properties require display=on");
2994 goto out_deregister;
2995 }
2996 if (vdev->dpy->edid_regs == NULL) {
2997 error_setg(errp, "xres and yres properties need edid support");
2998 goto out_deregister;
2999 }
3000 }
3001
3002 if (vdev->vendor_id == PCI_VENDOR_ID_NVIDIA) {
3003 ret = vfio_pci_nvidia_v100_ram_init(vdev, errp);
3004 if (ret && ret != -ENODEV) {
3005 error_report("Failed to setup NVIDIA V100 GPU RAM");
3006 }
3007 }
3008
3009 if (vdev->vendor_id == PCI_VENDOR_ID_IBM) {
3010 ret = vfio_pci_nvlink2_init(vdev, errp);
3011 if (ret && ret != -ENODEV) {
3012 error_report("Failed to setup NVlink2 bridge");
3013 }
3014 }
3015
3016 vfio_register_err_notifier(vdev);
3017 vfio_register_req_notifier(vdev);
3018 vfio_setup_resetfn_quirk(vdev);
3019
3020 return;
3021
3022 out_deregister:
3023 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3024 kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3025 out_teardown:
3026 vfio_teardown_msi(vdev);
3027 vfio_bars_exit(vdev);
3028 error:
3029 error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3030 if (vdev->migration_blocker) {
3031 migrate_del_blocker(vdev->migration_blocker);
3032 error_free(vdev->migration_blocker);
3033 vdev->migration_blocker = NULL;
3034 }
3035 }
3036
3037 static void vfio_instance_finalize(Object *obj)
3038 {
3039 VFIOPCIDevice *vdev = PCI_VFIO(obj);
3040 VFIOGroup *group = vdev->vbasedev.group;
3041
3042 vfio_display_finalize(vdev);
3043 vfio_bars_finalize(vdev);
3044 g_free(vdev->emulated_config_bits);
3045 g_free(vdev->rom);
3046 if (vdev->migration_blocker) {
3047 migrate_del_blocker(vdev->migration_blocker);
3048 error_free(vdev->migration_blocker);
3049 }
3050 /*
3051 * XXX Leaking igd_opregion is not an oversight, we can't remove the
3052 * fw_cfg entry therefore leaking this allocation seems like the safest
3053 * option.
3054 *
3055 * g_free(vdev->igd_opregion);
3056 */
3057 vfio_put_device(vdev);
3058 vfio_put_group(group);
3059 }
3060
3061 static void vfio_exitfn(PCIDevice *pdev)
3062 {
3063 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
3064
3065 vfio_unregister_req_notifier(vdev);
3066 vfio_unregister_err_notifier(vdev);
3067 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3068 if (vdev->irqchip_change_notifier.notify) {
3069 kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3070 }
3071 vfio_disable_interrupts(vdev);
3072 if (vdev->intx.mmap_timer) {
3073 timer_free(vdev->intx.mmap_timer);
3074 }
3075 vfio_teardown_msi(vdev);
3076 vfio_bars_exit(vdev);
3077 }
3078
3079 static void vfio_pci_reset(DeviceState *dev)
3080 {
3081 VFIOPCIDevice *vdev = PCI_VFIO(dev);
3082
3083 trace_vfio_pci_reset(vdev->vbasedev.name);
3084
3085 vfio_pci_pre_reset(vdev);
3086
3087 if (vdev->display != ON_OFF_AUTO_OFF) {
3088 vfio_display_reset(vdev);
3089 }
3090
3091 if (vdev->resetfn && !vdev->resetfn(vdev)) {
3092 goto post_reset;
3093 }
3094
3095 if (vdev->vbasedev.reset_works &&
3096 (vdev->has_flr || !vdev->has_pm_reset) &&
3097 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3098 trace_vfio_pci_reset_flr(vdev->vbasedev.name);
3099 goto post_reset;
3100 }
3101
3102 /* See if we can do our own bus reset */
3103 if (!vfio_pci_hot_reset_one(vdev)) {
3104 goto post_reset;
3105 }
3106
3107 /* If nothing else works and the device supports PM reset, use it */
3108 if (vdev->vbasedev.reset_works && vdev->has_pm_reset &&
3109 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3110 trace_vfio_pci_reset_pm(vdev->vbasedev.name);
3111 goto post_reset;
3112 }
3113
3114 post_reset:
3115 vfio_pci_post_reset(vdev);
3116 }
3117
3118 static void vfio_instance_init(Object *obj)
3119 {
3120 PCIDevice *pci_dev = PCI_DEVICE(obj);
3121 VFIOPCIDevice *vdev = PCI_VFIO(obj);
3122
3123 device_add_bootindex_property(obj, &vdev->bootindex,
3124 "bootindex", NULL,
3125 &pci_dev->qdev);
3126 vdev->host.domain = ~0U;
3127 vdev->host.bus = ~0U;
3128 vdev->host.slot = ~0U;
3129 vdev->host.function = ~0U;
3130
3131 vdev->nv_gpudirect_clique = 0xFF;
3132
3133 /* QEMU_PCI_CAP_EXPRESS initialization does not depend on QEMU command
3134 * line, therefore, no need to wait to realize like other devices */
3135 pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
3136 }
3137
3138 static Property vfio_pci_dev_properties[] = {
3139 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
3140 DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
3141 DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
3142 display, ON_OFF_AUTO_OFF),
3143 DEFINE_PROP_UINT32("xres", VFIOPCIDevice, display_xres, 0),
3144 DEFINE_PROP_UINT32("yres", VFIOPCIDevice, display_yres, 0),
3145 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
3146 intx.mmap_timeout, 1100),
3147 DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features,
3148 VFIO_FEATURE_ENABLE_VGA_BIT, false),
3149 DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
3150 VFIO_FEATURE_ENABLE_REQ_BIT, true),
3151 DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features,
3152 VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false),
3153 DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
3154 DEFINE_PROP_BOOL("x-balloon-allowed", VFIOPCIDevice,
3155 vbasedev.ram_block_discard_allowed, false),
3156 DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
3157 DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
3158 DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
3159 DEFINE_PROP_BOOL("x-no-geforce-quirks", VFIOPCIDevice,
3160 no_geforce_quirks, false),
3161 DEFINE_PROP_BOOL("x-no-kvm-ioeventfd", VFIOPCIDevice, no_kvm_ioeventfd,
3162 false),
3163 DEFINE_PROP_BOOL("x-no-vfio-ioeventfd", VFIOPCIDevice, no_vfio_ioeventfd,
3164 false),
3165 DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
3166 DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
3167 DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
3168 sub_vendor_id, PCI_ANY_ID),
3169 DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
3170 sub_device_id, PCI_ANY_ID),
3171 DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice, igd_gms, 0),
3172 DEFINE_PROP_UNSIGNED_NODEFAULT("x-nv-gpudirect-clique", VFIOPCIDevice,
3173 nv_gpudirect_clique,
3174 qdev_prop_nv_gpudirect_clique, uint8_t),
3175 DEFINE_PROP_OFF_AUTO_PCIBAR("x-msix-relocation", VFIOPCIDevice, msix_relo,
3176 OFF_AUTOPCIBAR_OFF),
3177 /*
3178 * TODO - support passed fds... is this necessary?
3179 * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
3180 * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
3181 */
3182 DEFINE_PROP_END_OF_LIST(),
3183 };
3184
3185 static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
3186 {
3187 DeviceClass *dc = DEVICE_CLASS(klass);
3188 PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
3189
3190 dc->reset = vfio_pci_reset;
3191 device_class_set_props(dc, vfio_pci_dev_properties);
3192 dc->desc = "VFIO-based PCI device assignment";
3193 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
3194 pdc->realize = vfio_realize;
3195 pdc->exit = vfio_exitfn;
3196 pdc->config_read = vfio_pci_read_config;
3197 pdc->config_write = vfio_pci_write_config;
3198 }
3199
3200 static const TypeInfo vfio_pci_dev_info = {
3201 .name = TYPE_VFIO_PCI,
3202 .parent = TYPE_PCI_DEVICE,
3203 .instance_size = sizeof(VFIOPCIDevice),
3204 .class_init = vfio_pci_dev_class_init,
3205 .instance_init = vfio_instance_init,
3206 .instance_finalize = vfio_instance_finalize,
3207 .interfaces = (InterfaceInfo[]) {
3208 { INTERFACE_PCIE_DEVICE },
3209 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
3210 { }
3211 },
3212 };
3213
3214 static Property vfio_pci_dev_nohotplug_properties[] = {
3215 DEFINE_PROP_BOOL("ramfb", VFIOPCIDevice, enable_ramfb, false),
3216 DEFINE_PROP_END_OF_LIST(),
3217 };
3218
3219 static void vfio_pci_nohotplug_dev_class_init(ObjectClass *klass, void *data)
3220 {
3221 DeviceClass *dc = DEVICE_CLASS(klass);
3222
3223 device_class_set_props(dc, vfio_pci_dev_nohotplug_properties);
3224 dc->hotpluggable = false;
3225 }
3226
3227 static const TypeInfo vfio_pci_nohotplug_dev_info = {
3228 .name = TYPE_VFIO_PCI_NOHOTPLUG,
3229 .parent = TYPE_VFIO_PCI,
3230 .instance_size = sizeof(VFIOPCIDevice),
3231 .class_init = vfio_pci_nohotplug_dev_class_init,
3232 };
3233
3234 static void register_vfio_pci_dev_type(void)
3235 {
3236 type_register_static(&vfio_pci_dev_info);
3237 type_register_static(&vfio_pci_nohotplug_dev_info);
3238 }
3239
3240 type_init(register_vfio_pci_dev_type)