Merge remote-tracking branch 'remotes/stefanha-gitlab/tags/block-pull-request' into...
[qemu.git] / util / vfio-helpers.c
1 /*
2 * VFIO utility
3 *
4 * Copyright 2016 - 2018 Red Hat, Inc.
5 *
6 * Authors:
7 * Fam Zheng <famz@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
11 */
12
13 #include "qemu/osdep.h"
14 #include <sys/ioctl.h>
15 #include <linux/vfio.h>
16 #include "qapi/error.h"
17 #include "exec/ramlist.h"
18 #include "exec/cpu-common.h"
19 #include "exec/memory.h"
20 #include "trace.h"
21 #include "qemu/error-report.h"
22 #include "standard-headers/linux/pci_regs.h"
23 #include "qemu/event_notifier.h"
24 #include "qemu/vfio-helpers.h"
25 #include "qemu/lockable.h"
26 #include "trace.h"
27
28 #define QEMU_VFIO_DEBUG 0
29
30 #define QEMU_VFIO_IOVA_MIN 0x10000ULL
31 /* XXX: Once VFIO exposes the iova bit width in the IOMMU capability interface,
32 * we can use a runtime limit; alternatively it's also possible to do platform
33 * specific detection by reading sysfs entries. Until then, 39 is a safe bet.
34 **/
35 #define QEMU_VFIO_IOVA_MAX (1ULL << 39)
36
37 typedef struct {
38 /* Page aligned addr. */
39 void *host;
40 size_t size;
41 uint64_t iova;
42 } IOVAMapping;
43
44 struct IOVARange {
45 uint64_t start;
46 uint64_t end;
47 };
48
49 struct QEMUVFIOState {
50 QemuMutex lock;
51
52 /* These fields are protected by BQL */
53 int container;
54 int group;
55 int device;
56 RAMBlockNotifier ram_notifier;
57 struct vfio_region_info config_region_info, bar_region_info[6];
58 struct IOVARange *usable_iova_ranges;
59 uint8_t nb_iova_ranges;
60
61 /* These fields are protected by @lock */
62 /* VFIO's IO virtual address space is managed by splitting into a few
63 * sections:
64 *
65 * --------------- <= 0
66 * |xxxxxxxxxxxxx|
67 * |-------------| <= QEMU_VFIO_IOVA_MIN
68 * | |
69 * | Fixed |
70 * | |
71 * |-------------| <= low_water_mark
72 * | |
73 * | Free |
74 * | |
75 * |-------------| <= high_water_mark
76 * | |
77 * | Temp |
78 * | |
79 * |-------------| <= QEMU_VFIO_IOVA_MAX
80 * |xxxxxxxxxxxxx|
81 * |xxxxxxxxxxxxx|
82 * ---------------
83 *
84 * - Addresses lower than QEMU_VFIO_IOVA_MIN are reserved as invalid;
85 *
86 * - Fixed mappings of HVAs are assigned "low" IOVAs in the range of
87 * [QEMU_VFIO_IOVA_MIN, low_water_mark). Once allocated they will not be
88 * reclaimed - low_water_mark never shrinks;
89 *
90 * - IOVAs in range [low_water_mark, high_water_mark) are free;
91 *
92 * - IOVAs in range [high_water_mark, QEMU_VFIO_IOVA_MAX) are volatile
93 * mappings. At each qemu_vfio_dma_reset_temporary() call, the whole area
94 * is recycled. The caller should make sure I/O's depending on these
95 * mappings are completed before calling.
96 **/
97 uint64_t low_water_mark;
98 uint64_t high_water_mark;
99 IOVAMapping *mappings;
100 int nr_mappings;
101 };
102
103 /**
104 * Find group file by PCI device address as specified @device, and return the
105 * path. The returned string is owned by caller and should be g_free'ed later.
106 */
107 static char *sysfs_find_group_file(const char *device, Error **errp)
108 {
109 char *sysfs_link;
110 char *sysfs_group;
111 char *p;
112 char *path = NULL;
113
114 sysfs_link = g_strdup_printf("/sys/bus/pci/devices/%s/iommu_group", device);
115 sysfs_group = g_malloc0(PATH_MAX);
116 if (readlink(sysfs_link, sysfs_group, PATH_MAX - 1) == -1) {
117 error_setg_errno(errp, errno, "Failed to find iommu group sysfs path");
118 goto out;
119 }
120 p = strrchr(sysfs_group, '/');
121 if (!p) {
122 error_setg(errp, "Failed to find iommu group number");
123 goto out;
124 }
125
126 path = g_strdup_printf("/dev/vfio/%s", p + 1);
127 out:
128 g_free(sysfs_link);
129 g_free(sysfs_group);
130 return path;
131 }
132
133 static inline void assert_bar_index_valid(QEMUVFIOState *s, int index)
134 {
135 assert(index >= 0 && index < ARRAY_SIZE(s->bar_region_info));
136 }
137
138 static int qemu_vfio_pci_init_bar(QEMUVFIOState *s, int index, Error **errp)
139 {
140 g_autofree char *barname = NULL;
141 assert_bar_index_valid(s, index);
142 s->bar_region_info[index] = (struct vfio_region_info) {
143 .index = VFIO_PCI_BAR0_REGION_INDEX + index,
144 .argsz = sizeof(struct vfio_region_info),
145 };
146 if (ioctl(s->device, VFIO_DEVICE_GET_REGION_INFO, &s->bar_region_info[index])) {
147 error_setg_errno(errp, errno, "Failed to get BAR region info");
148 return -errno;
149 }
150 barname = g_strdup_printf("bar[%d]", index);
151 trace_qemu_vfio_region_info(barname, s->bar_region_info[index].offset,
152 s->bar_region_info[index].size,
153 s->bar_region_info[index].cap_offset);
154
155 return 0;
156 }
157
158 /**
159 * Map a PCI bar area.
160 */
161 void *qemu_vfio_pci_map_bar(QEMUVFIOState *s, int index,
162 uint64_t offset, uint64_t size, int prot,
163 Error **errp)
164 {
165 void *p;
166 assert(QEMU_IS_ALIGNED(offset, qemu_real_host_page_size));
167 assert_bar_index_valid(s, index);
168 p = mmap(NULL, MIN(size, s->bar_region_info[index].size - offset),
169 prot, MAP_SHARED,
170 s->device, s->bar_region_info[index].offset + offset);
171 trace_qemu_vfio_pci_map_bar(index, s->bar_region_info[index].offset ,
172 size, offset, p);
173 if (p == MAP_FAILED) {
174 error_setg_errno(errp, errno, "Failed to map BAR region");
175 p = NULL;
176 }
177 return p;
178 }
179
180 /**
181 * Unmap a PCI bar area.
182 */
183 void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar,
184 uint64_t offset, uint64_t size)
185 {
186 if (bar) {
187 munmap(bar, MIN(size, s->bar_region_info[index].size - offset));
188 }
189 }
190
191 /**
192 * Initialize device IRQ with @irq_type and register an event notifier.
193 */
194 int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e,
195 int irq_type, Error **errp)
196 {
197 int r;
198 struct vfio_irq_set *irq_set;
199 size_t irq_set_size;
200 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
201
202 irq_info.index = irq_type;
203 if (ioctl(s->device, VFIO_DEVICE_GET_IRQ_INFO, &irq_info)) {
204 error_setg_errno(errp, errno, "Failed to get device interrupt info");
205 return -errno;
206 }
207 if (!(irq_info.flags & VFIO_IRQ_INFO_EVENTFD)) {
208 error_setg(errp, "Device interrupt doesn't support eventfd");
209 return -EINVAL;
210 }
211
212 irq_set_size = sizeof(*irq_set) + sizeof(int);
213 irq_set = g_malloc0(irq_set_size);
214
215 /* Get to a known IRQ state */
216 *irq_set = (struct vfio_irq_set) {
217 .argsz = irq_set_size,
218 .flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER,
219 .index = irq_info.index,
220 .start = 0,
221 .count = 1,
222 };
223
224 *(int *)&irq_set->data = event_notifier_get_fd(e);
225 r = ioctl(s->device, VFIO_DEVICE_SET_IRQS, irq_set);
226 g_free(irq_set);
227 if (r) {
228 error_setg_errno(errp, errno, "Failed to setup device interrupt");
229 return -errno;
230 }
231 return 0;
232 }
233
234 static int qemu_vfio_pci_read_config(QEMUVFIOState *s, void *buf,
235 int size, int ofs)
236 {
237 int ret;
238
239 trace_qemu_vfio_pci_read_config(buf, ofs, size,
240 s->config_region_info.offset,
241 s->config_region_info.size);
242 assert(QEMU_IS_ALIGNED(s->config_region_info.offset + ofs, size));
243 do {
244 ret = pread(s->device, buf, size, s->config_region_info.offset + ofs);
245 } while (ret == -1 && errno == EINTR);
246 return ret == size ? 0 : -errno;
247 }
248
249 static int qemu_vfio_pci_write_config(QEMUVFIOState *s, void *buf, int size, int ofs)
250 {
251 int ret;
252
253 trace_qemu_vfio_pci_write_config(buf, ofs, size,
254 s->config_region_info.offset,
255 s->config_region_info.size);
256 assert(QEMU_IS_ALIGNED(s->config_region_info.offset + ofs, size));
257 do {
258 ret = pwrite(s->device, buf, size, s->config_region_info.offset + ofs);
259 } while (ret == -1 && errno == EINTR);
260 return ret == size ? 0 : -errno;
261 }
262
263 static void collect_usable_iova_ranges(QEMUVFIOState *s, void *buf)
264 {
265 struct vfio_iommu_type1_info *info = (struct vfio_iommu_type1_info *)buf;
266 struct vfio_info_cap_header *cap = (void *)buf + info->cap_offset;
267 struct vfio_iommu_type1_info_cap_iova_range *cap_iova_range;
268 int i;
269
270 while (cap->id != VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE) {
271 if (!cap->next) {
272 return;
273 }
274 cap = (struct vfio_info_cap_header *)(buf + cap->next);
275 }
276
277 cap_iova_range = (struct vfio_iommu_type1_info_cap_iova_range *)cap;
278
279 s->nb_iova_ranges = cap_iova_range->nr_iovas;
280 if (s->nb_iova_ranges > 1) {
281 s->usable_iova_ranges =
282 g_realloc(s->usable_iova_ranges,
283 s->nb_iova_ranges * sizeof(struct IOVARange));
284 }
285
286 for (i = 0; i < s->nb_iova_ranges; i++) {
287 s->usable_iova_ranges[i].start = cap_iova_range->iova_ranges[i].start;
288 s->usable_iova_ranges[i].end = cap_iova_range->iova_ranges[i].end;
289 }
290 }
291
292 static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device,
293 Error **errp)
294 {
295 int ret;
296 int i;
297 uint16_t pci_cmd;
298 struct vfio_group_status group_status = { .argsz = sizeof(group_status) };
299 struct vfio_iommu_type1_info *iommu_info = NULL;
300 size_t iommu_info_size = sizeof(*iommu_info);
301 struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
302 char *group_file = NULL;
303
304 s->usable_iova_ranges = NULL;
305
306 /* Create a new container */
307 s->container = open("/dev/vfio/vfio", O_RDWR);
308
309 if (s->container == -1) {
310 error_setg_errno(errp, errno, "Failed to open /dev/vfio/vfio");
311 return -errno;
312 }
313 if (ioctl(s->container, VFIO_GET_API_VERSION) != VFIO_API_VERSION) {
314 error_setg(errp, "Invalid VFIO version");
315 ret = -EINVAL;
316 goto fail_container;
317 }
318
319 if (!ioctl(s->container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) {
320 error_setg_errno(errp, errno, "VFIO IOMMU Type1 is not supported");
321 ret = -EINVAL;
322 goto fail_container;
323 }
324
325 /* Open the group */
326 group_file = sysfs_find_group_file(device, errp);
327 if (!group_file) {
328 ret = -EINVAL;
329 goto fail_container;
330 }
331
332 s->group = open(group_file, O_RDWR);
333 if (s->group == -1) {
334 error_setg_errno(errp, errno, "Failed to open VFIO group file: %s",
335 group_file);
336 g_free(group_file);
337 ret = -errno;
338 goto fail_container;
339 }
340 g_free(group_file);
341
342 /* Test the group is viable and available */
343 if (ioctl(s->group, VFIO_GROUP_GET_STATUS, &group_status)) {
344 error_setg_errno(errp, errno, "Failed to get VFIO group status");
345 ret = -errno;
346 goto fail;
347 }
348
349 if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
350 error_setg(errp, "VFIO group is not viable");
351 ret = -EINVAL;
352 goto fail;
353 }
354
355 /* Add the group to the container */
356 if (ioctl(s->group, VFIO_GROUP_SET_CONTAINER, &s->container)) {
357 error_setg_errno(errp, errno, "Failed to add group to VFIO container");
358 ret = -errno;
359 goto fail;
360 }
361
362 /* Enable the IOMMU model we want */
363 if (ioctl(s->container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU)) {
364 error_setg_errno(errp, errno, "Failed to set VFIO IOMMU type");
365 ret = -errno;
366 goto fail;
367 }
368
369 iommu_info = g_malloc0(iommu_info_size);
370 iommu_info->argsz = iommu_info_size;
371
372 /* Get additional IOMMU info */
373 if (ioctl(s->container, VFIO_IOMMU_GET_INFO, iommu_info)) {
374 error_setg_errno(errp, errno, "Failed to get IOMMU info");
375 ret = -errno;
376 goto fail;
377 }
378
379 /*
380 * if the kernel does not report usable IOVA regions, choose
381 * the legacy [QEMU_VFIO_IOVA_MIN, QEMU_VFIO_IOVA_MAX -1] region
382 */
383 s->nb_iova_ranges = 1;
384 s->usable_iova_ranges = g_new0(struct IOVARange, 1);
385 s->usable_iova_ranges[0].start = QEMU_VFIO_IOVA_MIN;
386 s->usable_iova_ranges[0].end = QEMU_VFIO_IOVA_MAX - 1;
387
388 if (iommu_info->argsz > iommu_info_size) {
389 iommu_info_size = iommu_info->argsz;
390 iommu_info = g_realloc(iommu_info, iommu_info_size);
391 if (ioctl(s->container, VFIO_IOMMU_GET_INFO, iommu_info)) {
392 ret = -errno;
393 goto fail;
394 }
395 collect_usable_iova_ranges(s, iommu_info);
396 }
397
398 s->device = ioctl(s->group, VFIO_GROUP_GET_DEVICE_FD, device);
399
400 if (s->device < 0) {
401 error_setg_errno(errp, errno, "Failed to get device fd");
402 ret = -errno;
403 goto fail;
404 }
405
406 /* Test and setup the device */
407 if (ioctl(s->device, VFIO_DEVICE_GET_INFO, &device_info)) {
408 error_setg_errno(errp, errno, "Failed to get device info");
409 ret = -errno;
410 goto fail;
411 }
412
413 if (device_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX) {
414 error_setg(errp, "Invalid device regions");
415 ret = -EINVAL;
416 goto fail;
417 }
418
419 s->config_region_info = (struct vfio_region_info) {
420 .index = VFIO_PCI_CONFIG_REGION_INDEX,
421 .argsz = sizeof(struct vfio_region_info),
422 };
423 if (ioctl(s->device, VFIO_DEVICE_GET_REGION_INFO, &s->config_region_info)) {
424 error_setg_errno(errp, errno, "Failed to get config region info");
425 ret = -errno;
426 goto fail;
427 }
428 trace_qemu_vfio_region_info("config", s->config_region_info.offset,
429 s->config_region_info.size,
430 s->config_region_info.cap_offset);
431
432 for (i = 0; i < ARRAY_SIZE(s->bar_region_info); i++) {
433 ret = qemu_vfio_pci_init_bar(s, i, errp);
434 if (ret) {
435 goto fail;
436 }
437 }
438
439 /* Enable bus master */
440 ret = qemu_vfio_pci_read_config(s, &pci_cmd, sizeof(pci_cmd), PCI_COMMAND);
441 if (ret) {
442 goto fail;
443 }
444 pci_cmd |= PCI_COMMAND_MASTER;
445 ret = qemu_vfio_pci_write_config(s, &pci_cmd, sizeof(pci_cmd), PCI_COMMAND);
446 if (ret) {
447 goto fail;
448 }
449 g_free(iommu_info);
450 return 0;
451 fail:
452 g_free(s->usable_iova_ranges);
453 s->usable_iova_ranges = NULL;
454 s->nb_iova_ranges = 0;
455 g_free(iommu_info);
456 close(s->group);
457 fail_container:
458 close(s->container);
459 return ret;
460 }
461
462 static void qemu_vfio_ram_block_added(RAMBlockNotifier *n,
463 void *host, size_t size)
464 {
465 QEMUVFIOState *s = container_of(n, QEMUVFIOState, ram_notifier);
466 trace_qemu_vfio_ram_block_added(s, host, size);
467 qemu_vfio_dma_map(s, host, size, false, NULL);
468 }
469
470 static void qemu_vfio_ram_block_removed(RAMBlockNotifier *n,
471 void *host, size_t size)
472 {
473 QEMUVFIOState *s = container_of(n, QEMUVFIOState, ram_notifier);
474 if (host) {
475 trace_qemu_vfio_ram_block_removed(s, host, size);
476 qemu_vfio_dma_unmap(s, host);
477 }
478 }
479
480 static int qemu_vfio_init_ramblock(RAMBlock *rb, void *opaque)
481 {
482 void *host_addr = qemu_ram_get_host_addr(rb);
483 ram_addr_t length = qemu_ram_get_used_length(rb);
484 int ret;
485 QEMUVFIOState *s = opaque;
486
487 if (!host_addr) {
488 return 0;
489 }
490 ret = qemu_vfio_dma_map(s, host_addr, length, false, NULL);
491 if (ret) {
492 fprintf(stderr, "qemu_vfio_init_ramblock: failed %p %" PRId64 "\n",
493 host_addr, (uint64_t)length);
494 }
495 return 0;
496 }
497
498 static void qemu_vfio_open_common(QEMUVFIOState *s)
499 {
500 qemu_mutex_init(&s->lock);
501 s->ram_notifier.ram_block_added = qemu_vfio_ram_block_added;
502 s->ram_notifier.ram_block_removed = qemu_vfio_ram_block_removed;
503 ram_block_notifier_add(&s->ram_notifier);
504 s->low_water_mark = QEMU_VFIO_IOVA_MIN;
505 s->high_water_mark = QEMU_VFIO_IOVA_MAX;
506 qemu_ram_foreach_block(qemu_vfio_init_ramblock, s);
507 }
508
509 /**
510 * Open a PCI device, e.g. "0000:00:01.0".
511 */
512 QEMUVFIOState *qemu_vfio_open_pci(const char *device, Error **errp)
513 {
514 int r;
515 QEMUVFIOState *s = g_new0(QEMUVFIOState, 1);
516
517 /*
518 * VFIO may pin all memory inside mappings, resulting it in pinning
519 * all memory inside RAM blocks unconditionally.
520 */
521 r = ram_block_discard_disable(true);
522 if (r) {
523 error_setg_errno(errp, -r, "Cannot set discarding of RAM broken");
524 g_free(s);
525 return NULL;
526 }
527
528 r = qemu_vfio_init_pci(s, device, errp);
529 if (r) {
530 ram_block_discard_disable(false);
531 g_free(s);
532 return NULL;
533 }
534 qemu_vfio_open_common(s);
535 return s;
536 }
537
538 static void qemu_vfio_dump_mappings(QEMUVFIOState *s)
539 {
540 for (int i = 0; i < s->nr_mappings; ++i) {
541 trace_qemu_vfio_dump_mapping(s->mappings[i].host,
542 s->mappings[i].iova,
543 s->mappings[i].size);
544 }
545 }
546
547 /**
548 * Find the mapping entry that contains [host, host + size) and set @index to
549 * the position. If no entry contains it, @index is the position _after_ which
550 * to insert the new mapping. IOW, it is the index of the largest element that
551 * is smaller than @host, or -1 if no entry is.
552 */
553 static IOVAMapping *qemu_vfio_find_mapping(QEMUVFIOState *s, void *host,
554 int *index)
555 {
556 IOVAMapping *p = s->mappings;
557 IOVAMapping *q = p ? p + s->nr_mappings - 1 : NULL;
558 IOVAMapping *mid;
559 trace_qemu_vfio_find_mapping(s, host);
560 if (!p) {
561 *index = -1;
562 return NULL;
563 }
564 while (true) {
565 mid = p + (q - p) / 2;
566 if (mid == p) {
567 break;
568 }
569 if (mid->host > host) {
570 q = mid;
571 } else if (mid->host < host) {
572 p = mid;
573 } else {
574 break;
575 }
576 }
577 if (mid->host > host) {
578 mid--;
579 } else if (mid < &s->mappings[s->nr_mappings - 1]
580 && (mid + 1)->host <= host) {
581 mid++;
582 }
583 *index = mid - &s->mappings[0];
584 if (mid >= &s->mappings[0] &&
585 mid->host <= host && mid->host + mid->size > host) {
586 assert(mid < &s->mappings[s->nr_mappings]);
587 return mid;
588 }
589 /* At this point *index + 1 is the right position to insert the new
590 * mapping.*/
591 return NULL;
592 }
593
594 /**
595 * Allocate IOVA and create a new mapping record and insert it in @s.
596 */
597 static IOVAMapping *qemu_vfio_add_mapping(QEMUVFIOState *s,
598 void *host, size_t size,
599 int index, uint64_t iova)
600 {
601 int shift;
602 IOVAMapping m = {.host = host, .size = size, .iova = iova};
603 IOVAMapping *insert;
604
605 assert(QEMU_IS_ALIGNED(size, qemu_real_host_page_size));
606 assert(QEMU_IS_ALIGNED(s->low_water_mark, qemu_real_host_page_size));
607 assert(QEMU_IS_ALIGNED(s->high_water_mark, qemu_real_host_page_size));
608 trace_qemu_vfio_new_mapping(s, host, size, index, iova);
609
610 assert(index >= 0);
611 s->nr_mappings++;
612 s->mappings = g_renew(IOVAMapping, s->mappings, s->nr_mappings);
613 insert = &s->mappings[index];
614 shift = s->nr_mappings - index - 1;
615 if (shift) {
616 memmove(insert + 1, insert, shift * sizeof(s->mappings[0]));
617 }
618 *insert = m;
619 return insert;
620 }
621
622 /* Do the DMA mapping with VFIO. */
623 static int qemu_vfio_do_mapping(QEMUVFIOState *s, void *host, size_t size,
624 uint64_t iova)
625 {
626 struct vfio_iommu_type1_dma_map dma_map = {
627 .argsz = sizeof(dma_map),
628 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
629 .iova = iova,
630 .vaddr = (uintptr_t)host,
631 .size = size,
632 };
633 trace_qemu_vfio_do_mapping(s, host, iova, size);
634
635 if (ioctl(s->container, VFIO_IOMMU_MAP_DMA, &dma_map)) {
636 error_report("VFIO_MAP_DMA failed: %s", strerror(errno));
637 return -errno;
638 }
639 return 0;
640 }
641
642 /**
643 * Undo the DMA mapping from @s with VFIO, and remove from mapping list.
644 */
645 static void qemu_vfio_undo_mapping(QEMUVFIOState *s, IOVAMapping *mapping,
646 Error **errp)
647 {
648 int index;
649 struct vfio_iommu_type1_dma_unmap unmap = {
650 .argsz = sizeof(unmap),
651 .flags = 0,
652 .iova = mapping->iova,
653 .size = mapping->size,
654 };
655
656 index = mapping - s->mappings;
657 assert(mapping->size > 0);
658 assert(QEMU_IS_ALIGNED(mapping->size, qemu_real_host_page_size));
659 assert(index >= 0 && index < s->nr_mappings);
660 if (ioctl(s->container, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
661 error_setg_errno(errp, errno, "VFIO_UNMAP_DMA failed");
662 }
663 memmove(mapping, &s->mappings[index + 1],
664 sizeof(s->mappings[0]) * (s->nr_mappings - index - 1));
665 s->nr_mappings--;
666 s->mappings = g_renew(IOVAMapping, s->mappings, s->nr_mappings);
667 }
668
669 /* Check if the mapping list is (ascending) ordered. */
670 static bool qemu_vfio_verify_mappings(QEMUVFIOState *s)
671 {
672 int i;
673 if (QEMU_VFIO_DEBUG) {
674 for (i = 0; i < s->nr_mappings - 1; ++i) {
675 if (!(s->mappings[i].host < s->mappings[i + 1].host)) {
676 fprintf(stderr, "item %d not sorted!\n", i);
677 qemu_vfio_dump_mappings(s);
678 return false;
679 }
680 if (!(s->mappings[i].host + s->mappings[i].size <=
681 s->mappings[i + 1].host)) {
682 fprintf(stderr, "item %d overlap with next!\n", i);
683 qemu_vfio_dump_mappings(s);
684 return false;
685 }
686 }
687 }
688 return true;
689 }
690
691 static int
692 qemu_vfio_find_fixed_iova(QEMUVFIOState *s, size_t size, uint64_t *iova)
693 {
694 int i;
695
696 for (i = 0; i < s->nb_iova_ranges; i++) {
697 if (s->usable_iova_ranges[i].end < s->low_water_mark) {
698 continue;
699 }
700 s->low_water_mark =
701 MAX(s->low_water_mark, s->usable_iova_ranges[i].start);
702
703 if (s->usable_iova_ranges[i].end - s->low_water_mark + 1 >= size ||
704 s->usable_iova_ranges[i].end - s->low_water_mark + 1 == 0) {
705 *iova = s->low_water_mark;
706 s->low_water_mark += size;
707 return 0;
708 }
709 }
710 return -ENOMEM;
711 }
712
713 static int
714 qemu_vfio_find_temp_iova(QEMUVFIOState *s, size_t size, uint64_t *iova)
715 {
716 int i;
717
718 for (i = s->nb_iova_ranges - 1; i >= 0; i--) {
719 if (s->usable_iova_ranges[i].start > s->high_water_mark) {
720 continue;
721 }
722 s->high_water_mark =
723 MIN(s->high_water_mark, s->usable_iova_ranges[i].end + 1);
724
725 if (s->high_water_mark - s->usable_iova_ranges[i].start + 1 >= size ||
726 s->high_water_mark - s->usable_iova_ranges[i].start + 1 == 0) {
727 *iova = s->high_water_mark - size;
728 s->high_water_mark = *iova;
729 return 0;
730 }
731 }
732 return -ENOMEM;
733 }
734
735 /* Map [host, host + size) area into a contiguous IOVA address space, and store
736 * the result in @iova if not NULL. The caller need to make sure the area is
737 * aligned to page size, and mustn't overlap with existing mapping areas (split
738 * mapping status within this area is not allowed).
739 */
740 int qemu_vfio_dma_map(QEMUVFIOState *s, void *host, size_t size,
741 bool temporary, uint64_t *iova)
742 {
743 int ret = 0;
744 int index;
745 IOVAMapping *mapping;
746 uint64_t iova0;
747
748 assert(QEMU_PTR_IS_ALIGNED(host, qemu_real_host_page_size));
749 assert(QEMU_IS_ALIGNED(size, qemu_real_host_page_size));
750 trace_qemu_vfio_dma_map(s, host, size, temporary, iova);
751 qemu_mutex_lock(&s->lock);
752 mapping = qemu_vfio_find_mapping(s, host, &index);
753 if (mapping) {
754 iova0 = mapping->iova + ((uint8_t *)host - (uint8_t *)mapping->host);
755 } else {
756 if (s->high_water_mark - s->low_water_mark + 1 < size) {
757 ret = -ENOMEM;
758 goto out;
759 }
760 if (!temporary) {
761 if (qemu_vfio_find_fixed_iova(s, size, &iova0)) {
762 ret = -ENOMEM;
763 goto out;
764 }
765
766 mapping = qemu_vfio_add_mapping(s, host, size, index + 1, iova0);
767 if (!mapping) {
768 ret = -ENOMEM;
769 goto out;
770 }
771 assert(qemu_vfio_verify_mappings(s));
772 ret = qemu_vfio_do_mapping(s, host, size, iova0);
773 if (ret) {
774 qemu_vfio_undo_mapping(s, mapping, NULL);
775 goto out;
776 }
777 qemu_vfio_dump_mappings(s);
778 } else {
779 if (qemu_vfio_find_temp_iova(s, size, &iova0)) {
780 ret = -ENOMEM;
781 goto out;
782 }
783 ret = qemu_vfio_do_mapping(s, host, size, iova0);
784 if (ret) {
785 goto out;
786 }
787 }
788 }
789 trace_qemu_vfio_dma_mapped(s, host, iova0, size);
790 if (iova) {
791 *iova = iova0;
792 }
793 out:
794 qemu_mutex_unlock(&s->lock);
795 return ret;
796 }
797
798 /* Reset the high watermark and free all "temporary" mappings. */
799 int qemu_vfio_dma_reset_temporary(QEMUVFIOState *s)
800 {
801 struct vfio_iommu_type1_dma_unmap unmap = {
802 .argsz = sizeof(unmap),
803 .flags = 0,
804 .iova = s->high_water_mark,
805 .size = QEMU_VFIO_IOVA_MAX - s->high_water_mark,
806 };
807 trace_qemu_vfio_dma_reset_temporary(s);
808 QEMU_LOCK_GUARD(&s->lock);
809 if (ioctl(s->container, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
810 error_report("VFIO_UNMAP_DMA failed: %s", strerror(errno));
811 return -errno;
812 }
813 s->high_water_mark = QEMU_VFIO_IOVA_MAX;
814 return 0;
815 }
816
817 /* Unmapping the whole area that was previously mapped with
818 * qemu_vfio_dma_map(). */
819 void qemu_vfio_dma_unmap(QEMUVFIOState *s, void *host)
820 {
821 int index = 0;
822 IOVAMapping *m;
823
824 if (!host) {
825 return;
826 }
827
828 trace_qemu_vfio_dma_unmap(s, host);
829 qemu_mutex_lock(&s->lock);
830 m = qemu_vfio_find_mapping(s, host, &index);
831 if (!m) {
832 goto out;
833 }
834 qemu_vfio_undo_mapping(s, m, NULL);
835 out:
836 qemu_mutex_unlock(&s->lock);
837 }
838
839 static void qemu_vfio_reset(QEMUVFIOState *s)
840 {
841 ioctl(s->device, VFIO_DEVICE_RESET);
842 }
843
844 /* Close and free the VFIO resources. */
845 void qemu_vfio_close(QEMUVFIOState *s)
846 {
847 int i;
848
849 if (!s) {
850 return;
851 }
852 for (i = 0; i < s->nr_mappings; ++i) {
853 qemu_vfio_undo_mapping(s, &s->mappings[i], NULL);
854 }
855 ram_block_notifier_remove(&s->ram_notifier);
856 g_free(s->usable_iova_ranges);
857 s->nb_iova_ranges = 0;
858 qemu_vfio_reset(s);
859 close(s->device);
860 close(s->group);
861 close(s->container);
862 ram_block_discard_disable(false);
863 }