cpu: Move halted and interrupt_request fields to CPUState
[qemu.git] / kvm-all.c
1 /*
2 * QEMU KVM support
3 *
4 * Copyright IBM, Corp. 2008
5 * Red Hat, Inc. 2008
6 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Glauber Costa <gcosta@redhat.com>
10 *
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
13 *
14 */
15
16 #include <sys/types.h>
17 #include <sys/ioctl.h>
18 #include <sys/mman.h>
19 #include <stdarg.h>
20
21 #include <linux/kvm.h>
22
23 #include "qemu-common.h"
24 #include "qemu/atomic.h"
25 #include "qemu/option.h"
26 #include "qemu/config-file.h"
27 #include "sysemu/sysemu.h"
28 #include "hw/hw.h"
29 #include "hw/pci/msi.h"
30 #include "exec/gdbstub.h"
31 #include "sysemu/kvm.h"
32 #include "qemu/bswap.h"
33 #include "exec/memory.h"
34 #include "exec/address-spaces.h"
35 #include "qemu/event_notifier.h"
36
37 /* This check must be after config-host.h is included */
38 #ifdef CONFIG_EVENTFD
39 #include <sys/eventfd.h>
40 #endif
41
42 #ifdef CONFIG_VALGRIND_H
43 #include <valgrind/memcheck.h>
44 #endif
45
46 /* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
47 #define PAGE_SIZE TARGET_PAGE_SIZE
48
49 //#define DEBUG_KVM
50
51 #ifdef DEBUG_KVM
52 #define DPRINTF(fmt, ...) \
53 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
54 #else
55 #define DPRINTF(fmt, ...) \
56 do { } while (0)
57 #endif
58
59 #define KVM_MSI_HASHTAB_SIZE 256
60
61 typedef struct KVMSlot
62 {
63 hwaddr start_addr;
64 ram_addr_t memory_size;
65 void *ram;
66 int slot;
67 int flags;
68 } KVMSlot;
69
70 typedef struct kvm_dirty_log KVMDirtyLog;
71
72 struct KVMState
73 {
74 KVMSlot slots[32];
75 int fd;
76 int vmfd;
77 int coalesced_mmio;
78 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
79 bool coalesced_flush_in_progress;
80 int broken_set_mem_region;
81 int migration_log;
82 int vcpu_events;
83 int robust_singlestep;
84 int debugregs;
85 #ifdef KVM_CAP_SET_GUEST_DEBUG
86 struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
87 #endif
88 int pit_state2;
89 int xsave, xcrs;
90 int many_ioeventfds;
91 int intx_set_mask;
92 /* The man page (and posix) say ioctl numbers are signed int, but
93 * they're not. Linux, glibc and *BSD all treat ioctl numbers as
94 * unsigned, and treating them as signed here can break things */
95 unsigned irq_set_ioctl;
96 #ifdef KVM_CAP_IRQ_ROUTING
97 struct kvm_irq_routing *irq_routes;
98 int nr_allocated_irq_routes;
99 uint32_t *used_gsi_bitmap;
100 unsigned int gsi_count;
101 QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
102 bool direct_msi;
103 #endif
104 };
105
106 KVMState *kvm_state;
107 bool kvm_kernel_irqchip;
108 bool kvm_async_interrupts_allowed;
109 bool kvm_irqfds_allowed;
110 bool kvm_msi_via_irqfd_allowed;
111 bool kvm_gsi_routing_allowed;
112
113 static const KVMCapabilityInfo kvm_required_capabilites[] = {
114 KVM_CAP_INFO(USER_MEMORY),
115 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
116 KVM_CAP_LAST_INFO
117 };
118
119 static KVMSlot *kvm_alloc_slot(KVMState *s)
120 {
121 int i;
122
123 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
124 if (s->slots[i].memory_size == 0) {
125 return &s->slots[i];
126 }
127 }
128
129 fprintf(stderr, "%s: no free slot available\n", __func__);
130 abort();
131 }
132
133 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
134 hwaddr start_addr,
135 hwaddr end_addr)
136 {
137 int i;
138
139 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
140 KVMSlot *mem = &s->slots[i];
141
142 if (start_addr == mem->start_addr &&
143 end_addr == mem->start_addr + mem->memory_size) {
144 return mem;
145 }
146 }
147
148 return NULL;
149 }
150
151 /*
152 * Find overlapping slot with lowest start address
153 */
154 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
155 hwaddr start_addr,
156 hwaddr end_addr)
157 {
158 KVMSlot *found = NULL;
159 int i;
160
161 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
162 KVMSlot *mem = &s->slots[i];
163
164 if (mem->memory_size == 0 ||
165 (found && found->start_addr < mem->start_addr)) {
166 continue;
167 }
168
169 if (end_addr > mem->start_addr &&
170 start_addr < mem->start_addr + mem->memory_size) {
171 found = mem;
172 }
173 }
174
175 return found;
176 }
177
178 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
179 hwaddr *phys_addr)
180 {
181 int i;
182
183 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
184 KVMSlot *mem = &s->slots[i];
185
186 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
187 *phys_addr = mem->start_addr + (ram - mem->ram);
188 return 1;
189 }
190 }
191
192 return 0;
193 }
194
195 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
196 {
197 struct kvm_userspace_memory_region mem;
198
199 mem.slot = slot->slot;
200 mem.guest_phys_addr = slot->start_addr;
201 mem.memory_size = slot->memory_size;
202 mem.userspace_addr = (unsigned long)slot->ram;
203 mem.flags = slot->flags;
204 if (s->migration_log) {
205 mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
206 }
207 return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
208 }
209
210 static void kvm_reset_vcpu(void *opaque)
211 {
212 CPUState *cpu = opaque;
213
214 kvm_arch_reset_vcpu(cpu);
215 }
216
217 int kvm_init_vcpu(CPUState *cpu)
218 {
219 KVMState *s = kvm_state;
220 long mmap_size;
221 int ret;
222
223 DPRINTF("kvm_init_vcpu\n");
224
225 ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
226 if (ret < 0) {
227 DPRINTF("kvm_create_vcpu failed\n");
228 goto err;
229 }
230
231 cpu->kvm_fd = ret;
232 cpu->kvm_state = s;
233 cpu->kvm_vcpu_dirty = true;
234
235 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
236 if (mmap_size < 0) {
237 ret = mmap_size;
238 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
239 goto err;
240 }
241
242 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
243 cpu->kvm_fd, 0);
244 if (cpu->kvm_run == MAP_FAILED) {
245 ret = -errno;
246 DPRINTF("mmap'ing vcpu state failed\n");
247 goto err;
248 }
249
250 if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
251 s->coalesced_mmio_ring =
252 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
253 }
254
255 ret = kvm_arch_init_vcpu(cpu);
256 if (ret == 0) {
257 qemu_register_reset(kvm_reset_vcpu, cpu);
258 kvm_arch_reset_vcpu(cpu);
259 }
260 err:
261 return ret;
262 }
263
264 /*
265 * dirty pages logging control
266 */
267
268 static int kvm_mem_flags(KVMState *s, bool log_dirty)
269 {
270 return log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
271 }
272
273 static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
274 {
275 KVMState *s = kvm_state;
276 int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
277 int old_flags;
278
279 old_flags = mem->flags;
280
281 flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty);
282 mem->flags = flags;
283
284 /* If nothing changed effectively, no need to issue ioctl */
285 if (s->migration_log) {
286 flags |= KVM_MEM_LOG_DIRTY_PAGES;
287 }
288
289 if (flags == old_flags) {
290 return 0;
291 }
292
293 return kvm_set_user_memory_region(s, mem);
294 }
295
296 static int kvm_dirty_pages_log_change(hwaddr phys_addr,
297 ram_addr_t size, bool log_dirty)
298 {
299 KVMState *s = kvm_state;
300 KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
301
302 if (mem == NULL) {
303 fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
304 TARGET_FMT_plx "\n", __func__, phys_addr,
305 (hwaddr)(phys_addr + size - 1));
306 return -EINVAL;
307 }
308 return kvm_slot_dirty_pages_log_change(mem, log_dirty);
309 }
310
311 static void kvm_log_start(MemoryListener *listener,
312 MemoryRegionSection *section)
313 {
314 int r;
315
316 r = kvm_dirty_pages_log_change(section->offset_within_address_space,
317 section->size, true);
318 if (r < 0) {
319 abort();
320 }
321 }
322
323 static void kvm_log_stop(MemoryListener *listener,
324 MemoryRegionSection *section)
325 {
326 int r;
327
328 r = kvm_dirty_pages_log_change(section->offset_within_address_space,
329 section->size, false);
330 if (r < 0) {
331 abort();
332 }
333 }
334
335 static int kvm_set_migration_log(int enable)
336 {
337 KVMState *s = kvm_state;
338 KVMSlot *mem;
339 int i, err;
340
341 s->migration_log = enable;
342
343 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
344 mem = &s->slots[i];
345
346 if (!mem->memory_size) {
347 continue;
348 }
349 if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
350 continue;
351 }
352 err = kvm_set_user_memory_region(s, mem);
353 if (err) {
354 return err;
355 }
356 }
357 return 0;
358 }
359
360 /* get kvm's dirty pages bitmap and update qemu's */
361 static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
362 unsigned long *bitmap)
363 {
364 unsigned int i, j;
365 unsigned long page_number, c;
366 hwaddr addr, addr1;
367 unsigned int len = ((section->size / getpagesize()) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
368 unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
369
370 /*
371 * bitmap-traveling is faster than memory-traveling (for addr...)
372 * especially when most of the memory is not dirty.
373 */
374 for (i = 0; i < len; i++) {
375 if (bitmap[i] != 0) {
376 c = leul_to_cpu(bitmap[i]);
377 do {
378 j = ffsl(c) - 1;
379 c &= ~(1ul << j);
380 page_number = (i * HOST_LONG_BITS + j) * hpratio;
381 addr1 = page_number * TARGET_PAGE_SIZE;
382 addr = section->offset_within_region + addr1;
383 memory_region_set_dirty(section->mr, addr,
384 TARGET_PAGE_SIZE * hpratio);
385 } while (c != 0);
386 }
387 }
388 return 0;
389 }
390
391 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
392
393 /**
394 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
395 * This function updates qemu's dirty bitmap using
396 * memory_region_set_dirty(). This means all bits are set
397 * to dirty.
398 *
399 * @start_add: start of logged region.
400 * @end_addr: end of logged region.
401 */
402 static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
403 {
404 KVMState *s = kvm_state;
405 unsigned long size, allocated_size = 0;
406 KVMDirtyLog d;
407 KVMSlot *mem;
408 int ret = 0;
409 hwaddr start_addr = section->offset_within_address_space;
410 hwaddr end_addr = start_addr + section->size;
411
412 d.dirty_bitmap = NULL;
413 while (start_addr < end_addr) {
414 mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
415 if (mem == NULL) {
416 break;
417 }
418
419 /* XXX bad kernel interface alert
420 * For dirty bitmap, kernel allocates array of size aligned to
421 * bits-per-long. But for case when the kernel is 64bits and
422 * the userspace is 32bits, userspace can't align to the same
423 * bits-per-long, since sizeof(long) is different between kernel
424 * and user space. This way, userspace will provide buffer which
425 * may be 4 bytes less than the kernel will use, resulting in
426 * userspace memory corruption (which is not detectable by valgrind
427 * too, in most cases).
428 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
429 * a hope that sizeof(long) wont become >8 any time soon.
430 */
431 size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
432 /*HOST_LONG_BITS*/ 64) / 8;
433 if (!d.dirty_bitmap) {
434 d.dirty_bitmap = g_malloc(size);
435 } else if (size > allocated_size) {
436 d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
437 }
438 allocated_size = size;
439 memset(d.dirty_bitmap, 0, allocated_size);
440
441 d.slot = mem->slot;
442
443 if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
444 DPRINTF("ioctl failed %d\n", errno);
445 ret = -1;
446 break;
447 }
448
449 kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
450 start_addr = mem->start_addr + mem->memory_size;
451 }
452 g_free(d.dirty_bitmap);
453
454 return ret;
455 }
456
457 static void kvm_coalesce_mmio_region(MemoryListener *listener,
458 MemoryRegionSection *secion,
459 hwaddr start, hwaddr size)
460 {
461 KVMState *s = kvm_state;
462
463 if (s->coalesced_mmio) {
464 struct kvm_coalesced_mmio_zone zone;
465
466 zone.addr = start;
467 zone.size = size;
468 zone.pad = 0;
469
470 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
471 }
472 }
473
474 static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
475 MemoryRegionSection *secion,
476 hwaddr start, hwaddr size)
477 {
478 KVMState *s = kvm_state;
479
480 if (s->coalesced_mmio) {
481 struct kvm_coalesced_mmio_zone zone;
482
483 zone.addr = start;
484 zone.size = size;
485 zone.pad = 0;
486
487 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
488 }
489 }
490
491 int kvm_check_extension(KVMState *s, unsigned int extension)
492 {
493 int ret;
494
495 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
496 if (ret < 0) {
497 ret = 0;
498 }
499
500 return ret;
501 }
502
503 static int kvm_check_many_ioeventfds(void)
504 {
505 /* Userspace can use ioeventfd for io notification. This requires a host
506 * that supports eventfd(2) and an I/O thread; since eventfd does not
507 * support SIGIO it cannot interrupt the vcpu.
508 *
509 * Older kernels have a 6 device limit on the KVM io bus. Find out so we
510 * can avoid creating too many ioeventfds.
511 */
512 #if defined(CONFIG_EVENTFD)
513 int ioeventfds[7];
514 int i, ret = 0;
515 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
516 ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
517 if (ioeventfds[i] < 0) {
518 break;
519 }
520 ret = kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, true);
521 if (ret < 0) {
522 close(ioeventfds[i]);
523 break;
524 }
525 }
526
527 /* Decide whether many devices are supported or not */
528 ret = i == ARRAY_SIZE(ioeventfds);
529
530 while (i-- > 0) {
531 kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, false);
532 close(ioeventfds[i]);
533 }
534 return ret;
535 #else
536 return 0;
537 #endif
538 }
539
540 static const KVMCapabilityInfo *
541 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
542 {
543 while (list->name) {
544 if (!kvm_check_extension(s, list->value)) {
545 return list;
546 }
547 list++;
548 }
549 return NULL;
550 }
551
552 static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
553 {
554 KVMState *s = kvm_state;
555 KVMSlot *mem, old;
556 int err;
557 MemoryRegion *mr = section->mr;
558 bool log_dirty = memory_region_is_logging(mr);
559 hwaddr start_addr = section->offset_within_address_space;
560 ram_addr_t size = section->size;
561 void *ram = NULL;
562 unsigned delta;
563
564 /* kvm works in page size chunks, but the function may be called
565 with sub-page size and unaligned start address. */
566 delta = TARGET_PAGE_ALIGN(size) - size;
567 if (delta > size) {
568 return;
569 }
570 start_addr += delta;
571 size -= delta;
572 size &= TARGET_PAGE_MASK;
573 if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
574 return;
575 }
576
577 if (!memory_region_is_ram(mr)) {
578 return;
579 }
580
581 ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
582
583 while (1) {
584 mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
585 if (!mem) {
586 break;
587 }
588
589 if (add && start_addr >= mem->start_addr &&
590 (start_addr + size <= mem->start_addr + mem->memory_size) &&
591 (ram - start_addr == mem->ram - mem->start_addr)) {
592 /* The new slot fits into the existing one and comes with
593 * identical parameters - update flags and done. */
594 kvm_slot_dirty_pages_log_change(mem, log_dirty);
595 return;
596 }
597
598 old = *mem;
599
600 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
601 kvm_physical_sync_dirty_bitmap(section);
602 }
603
604 /* unregister the overlapping slot */
605 mem->memory_size = 0;
606 err = kvm_set_user_memory_region(s, mem);
607 if (err) {
608 fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
609 __func__, strerror(-err));
610 abort();
611 }
612
613 /* Workaround for older KVM versions: we can't join slots, even not by
614 * unregistering the previous ones and then registering the larger
615 * slot. We have to maintain the existing fragmentation. Sigh.
616 *
617 * This workaround assumes that the new slot starts at the same
618 * address as the first existing one. If not or if some overlapping
619 * slot comes around later, we will fail (not seen in practice so far)
620 * - and actually require a recent KVM version. */
621 if (s->broken_set_mem_region &&
622 old.start_addr == start_addr && old.memory_size < size && add) {
623 mem = kvm_alloc_slot(s);
624 mem->memory_size = old.memory_size;
625 mem->start_addr = old.start_addr;
626 mem->ram = old.ram;
627 mem->flags = kvm_mem_flags(s, log_dirty);
628
629 err = kvm_set_user_memory_region(s, mem);
630 if (err) {
631 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
632 strerror(-err));
633 abort();
634 }
635
636 start_addr += old.memory_size;
637 ram += old.memory_size;
638 size -= old.memory_size;
639 continue;
640 }
641
642 /* register prefix slot */
643 if (old.start_addr < start_addr) {
644 mem = kvm_alloc_slot(s);
645 mem->memory_size = start_addr - old.start_addr;
646 mem->start_addr = old.start_addr;
647 mem->ram = old.ram;
648 mem->flags = kvm_mem_flags(s, log_dirty);
649
650 err = kvm_set_user_memory_region(s, mem);
651 if (err) {
652 fprintf(stderr, "%s: error registering prefix slot: %s\n",
653 __func__, strerror(-err));
654 #ifdef TARGET_PPC
655 fprintf(stderr, "%s: This is probably because your kernel's " \
656 "PAGE_SIZE is too big. Please try to use 4k " \
657 "PAGE_SIZE!\n", __func__);
658 #endif
659 abort();
660 }
661 }
662
663 /* register suffix slot */
664 if (old.start_addr + old.memory_size > start_addr + size) {
665 ram_addr_t size_delta;
666
667 mem = kvm_alloc_slot(s);
668 mem->start_addr = start_addr + size;
669 size_delta = mem->start_addr - old.start_addr;
670 mem->memory_size = old.memory_size - size_delta;
671 mem->ram = old.ram + size_delta;
672 mem->flags = kvm_mem_flags(s, log_dirty);
673
674 err = kvm_set_user_memory_region(s, mem);
675 if (err) {
676 fprintf(stderr, "%s: error registering suffix slot: %s\n",
677 __func__, strerror(-err));
678 abort();
679 }
680 }
681 }
682
683 /* in case the KVM bug workaround already "consumed" the new slot */
684 if (!size) {
685 return;
686 }
687 if (!add) {
688 return;
689 }
690 mem = kvm_alloc_slot(s);
691 mem->memory_size = size;
692 mem->start_addr = start_addr;
693 mem->ram = ram;
694 mem->flags = kvm_mem_flags(s, log_dirty);
695
696 err = kvm_set_user_memory_region(s, mem);
697 if (err) {
698 fprintf(stderr, "%s: error registering slot: %s\n", __func__,
699 strerror(-err));
700 abort();
701 }
702 }
703
704 static void kvm_region_add(MemoryListener *listener,
705 MemoryRegionSection *section)
706 {
707 kvm_set_phys_mem(section, true);
708 }
709
710 static void kvm_region_del(MemoryListener *listener,
711 MemoryRegionSection *section)
712 {
713 kvm_set_phys_mem(section, false);
714 }
715
716 static void kvm_log_sync(MemoryListener *listener,
717 MemoryRegionSection *section)
718 {
719 int r;
720
721 r = kvm_physical_sync_dirty_bitmap(section);
722 if (r < 0) {
723 abort();
724 }
725 }
726
727 static void kvm_log_global_start(struct MemoryListener *listener)
728 {
729 int r;
730
731 r = kvm_set_migration_log(1);
732 assert(r >= 0);
733 }
734
735 static void kvm_log_global_stop(struct MemoryListener *listener)
736 {
737 int r;
738
739 r = kvm_set_migration_log(0);
740 assert(r >= 0);
741 }
742
743 static void kvm_mem_ioeventfd_add(MemoryListener *listener,
744 MemoryRegionSection *section,
745 bool match_data, uint64_t data,
746 EventNotifier *e)
747 {
748 int fd = event_notifier_get_fd(e);
749 int r;
750
751 assert(match_data && section->size <= 8);
752
753 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
754 data, true, section->size);
755 if (r < 0) {
756 abort();
757 }
758 }
759
760 static void kvm_mem_ioeventfd_del(MemoryListener *listener,
761 MemoryRegionSection *section,
762 bool match_data, uint64_t data,
763 EventNotifier *e)
764 {
765 int fd = event_notifier_get_fd(e);
766 int r;
767
768 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
769 data, false, section->size);
770 if (r < 0) {
771 abort();
772 }
773 }
774
775 static void kvm_io_ioeventfd_add(MemoryListener *listener,
776 MemoryRegionSection *section,
777 bool match_data, uint64_t data,
778 EventNotifier *e)
779 {
780 int fd = event_notifier_get_fd(e);
781 int r;
782
783 assert(match_data && section->size == 2);
784
785 r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
786 data, true);
787 if (r < 0) {
788 abort();
789 }
790 }
791
792 static void kvm_io_ioeventfd_del(MemoryListener *listener,
793 MemoryRegionSection *section,
794 bool match_data, uint64_t data,
795 EventNotifier *e)
796
797 {
798 int fd = event_notifier_get_fd(e);
799 int r;
800
801 r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
802 data, false);
803 if (r < 0) {
804 abort();
805 }
806 }
807
808 static MemoryListener kvm_memory_listener = {
809 .region_add = kvm_region_add,
810 .region_del = kvm_region_del,
811 .log_start = kvm_log_start,
812 .log_stop = kvm_log_stop,
813 .log_sync = kvm_log_sync,
814 .log_global_start = kvm_log_global_start,
815 .log_global_stop = kvm_log_global_stop,
816 .eventfd_add = kvm_mem_ioeventfd_add,
817 .eventfd_del = kvm_mem_ioeventfd_del,
818 .coalesced_mmio_add = kvm_coalesce_mmio_region,
819 .coalesced_mmio_del = kvm_uncoalesce_mmio_region,
820 .priority = 10,
821 };
822
823 static MemoryListener kvm_io_listener = {
824 .eventfd_add = kvm_io_ioeventfd_add,
825 .eventfd_del = kvm_io_ioeventfd_del,
826 .priority = 10,
827 };
828
829 static void kvm_handle_interrupt(CPUArchState *env, int mask)
830 {
831 CPUState *cpu = ENV_GET_CPU(env);
832
833 cpu->interrupt_request |= mask;
834
835 if (!qemu_cpu_is_self(cpu)) {
836 qemu_cpu_kick(cpu);
837 }
838 }
839
840 int kvm_set_irq(KVMState *s, int irq, int level)
841 {
842 struct kvm_irq_level event;
843 int ret;
844
845 assert(kvm_async_interrupts_enabled());
846
847 event.level = level;
848 event.irq = irq;
849 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
850 if (ret < 0) {
851 perror("kvm_set_irq");
852 abort();
853 }
854
855 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
856 }
857
858 #ifdef KVM_CAP_IRQ_ROUTING
859 typedef struct KVMMSIRoute {
860 struct kvm_irq_routing_entry kroute;
861 QTAILQ_ENTRY(KVMMSIRoute) entry;
862 } KVMMSIRoute;
863
864 static void set_gsi(KVMState *s, unsigned int gsi)
865 {
866 s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
867 }
868
869 static void clear_gsi(KVMState *s, unsigned int gsi)
870 {
871 s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
872 }
873
874 static void kvm_init_irq_routing(KVMState *s)
875 {
876 int gsi_count, i;
877
878 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
879 if (gsi_count > 0) {
880 unsigned int gsi_bits, i;
881
882 /* Round up so we can search ints using ffs */
883 gsi_bits = ALIGN(gsi_count, 32);
884 s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
885 s->gsi_count = gsi_count;
886
887 /* Mark any over-allocated bits as already in use */
888 for (i = gsi_count; i < gsi_bits; i++) {
889 set_gsi(s, i);
890 }
891 }
892
893 s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
894 s->nr_allocated_irq_routes = 0;
895
896 if (!s->direct_msi) {
897 for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
898 QTAILQ_INIT(&s->msi_hashtab[i]);
899 }
900 }
901
902 kvm_arch_init_irq_routing(s);
903 }
904
905 static void kvm_irqchip_commit_routes(KVMState *s)
906 {
907 int ret;
908
909 s->irq_routes->flags = 0;
910 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
911 assert(ret == 0);
912 }
913
914 static void kvm_add_routing_entry(KVMState *s,
915 struct kvm_irq_routing_entry *entry)
916 {
917 struct kvm_irq_routing_entry *new;
918 int n, size;
919
920 if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
921 n = s->nr_allocated_irq_routes * 2;
922 if (n < 64) {
923 n = 64;
924 }
925 size = sizeof(struct kvm_irq_routing);
926 size += n * sizeof(*new);
927 s->irq_routes = g_realloc(s->irq_routes, size);
928 s->nr_allocated_irq_routes = n;
929 }
930 n = s->irq_routes->nr++;
931 new = &s->irq_routes->entries[n];
932 memset(new, 0, sizeof(*new));
933 new->gsi = entry->gsi;
934 new->type = entry->type;
935 new->flags = entry->flags;
936 new->u = entry->u;
937
938 set_gsi(s, entry->gsi);
939
940 kvm_irqchip_commit_routes(s);
941 }
942
943 static int kvm_update_routing_entry(KVMState *s,
944 struct kvm_irq_routing_entry *new_entry)
945 {
946 struct kvm_irq_routing_entry *entry;
947 int n;
948
949 for (n = 0; n < s->irq_routes->nr; n++) {
950 entry = &s->irq_routes->entries[n];
951 if (entry->gsi != new_entry->gsi) {
952 continue;
953 }
954
955 entry->type = new_entry->type;
956 entry->flags = new_entry->flags;
957 entry->u = new_entry->u;
958
959 kvm_irqchip_commit_routes(s);
960
961 return 0;
962 }
963
964 return -ESRCH;
965 }
966
967 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
968 {
969 struct kvm_irq_routing_entry e;
970
971 assert(pin < s->gsi_count);
972
973 e.gsi = irq;
974 e.type = KVM_IRQ_ROUTING_IRQCHIP;
975 e.flags = 0;
976 e.u.irqchip.irqchip = irqchip;
977 e.u.irqchip.pin = pin;
978 kvm_add_routing_entry(s, &e);
979 }
980
981 void kvm_irqchip_release_virq(KVMState *s, int virq)
982 {
983 struct kvm_irq_routing_entry *e;
984 int i;
985
986 for (i = 0; i < s->irq_routes->nr; i++) {
987 e = &s->irq_routes->entries[i];
988 if (e->gsi == virq) {
989 s->irq_routes->nr--;
990 *e = s->irq_routes->entries[s->irq_routes->nr];
991 }
992 }
993 clear_gsi(s, virq);
994 }
995
996 static unsigned int kvm_hash_msi(uint32_t data)
997 {
998 /* This is optimized for IA32 MSI layout. However, no other arch shall
999 * repeat the mistake of not providing a direct MSI injection API. */
1000 return data & 0xff;
1001 }
1002
1003 static void kvm_flush_dynamic_msi_routes(KVMState *s)
1004 {
1005 KVMMSIRoute *route, *next;
1006 unsigned int hash;
1007
1008 for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
1009 QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
1010 kvm_irqchip_release_virq(s, route->kroute.gsi);
1011 QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
1012 g_free(route);
1013 }
1014 }
1015 }
1016
1017 static int kvm_irqchip_get_virq(KVMState *s)
1018 {
1019 uint32_t *word = s->used_gsi_bitmap;
1020 int max_words = ALIGN(s->gsi_count, 32) / 32;
1021 int i, bit;
1022 bool retry = true;
1023
1024 again:
1025 /* Return the lowest unused GSI in the bitmap */
1026 for (i = 0; i < max_words; i++) {
1027 bit = ffs(~word[i]);
1028 if (!bit) {
1029 continue;
1030 }
1031
1032 return bit - 1 + i * 32;
1033 }
1034 if (!s->direct_msi && retry) {
1035 retry = false;
1036 kvm_flush_dynamic_msi_routes(s);
1037 goto again;
1038 }
1039 return -ENOSPC;
1040
1041 }
1042
1043 static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
1044 {
1045 unsigned int hash = kvm_hash_msi(msg.data);
1046 KVMMSIRoute *route;
1047
1048 QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
1049 if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
1050 route->kroute.u.msi.address_hi == (msg.address >> 32) &&
1051 route->kroute.u.msi.data == msg.data) {
1052 return route;
1053 }
1054 }
1055 return NULL;
1056 }
1057
1058 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1059 {
1060 struct kvm_msi msi;
1061 KVMMSIRoute *route;
1062
1063 if (s->direct_msi) {
1064 msi.address_lo = (uint32_t)msg.address;
1065 msi.address_hi = msg.address >> 32;
1066 msi.data = msg.data;
1067 msi.flags = 0;
1068 memset(msi.pad, 0, sizeof(msi.pad));
1069
1070 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
1071 }
1072
1073 route = kvm_lookup_msi_route(s, msg);
1074 if (!route) {
1075 int virq;
1076
1077 virq = kvm_irqchip_get_virq(s);
1078 if (virq < 0) {
1079 return virq;
1080 }
1081
1082 route = g_malloc(sizeof(KVMMSIRoute));
1083 route->kroute.gsi = virq;
1084 route->kroute.type = KVM_IRQ_ROUTING_MSI;
1085 route->kroute.flags = 0;
1086 route->kroute.u.msi.address_lo = (uint32_t)msg.address;
1087 route->kroute.u.msi.address_hi = msg.address >> 32;
1088 route->kroute.u.msi.data = msg.data;
1089
1090 kvm_add_routing_entry(s, &route->kroute);
1091
1092 QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
1093 entry);
1094 }
1095
1096 assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
1097
1098 return kvm_set_irq(s, route->kroute.gsi, 1);
1099 }
1100
1101 int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1102 {
1103 struct kvm_irq_routing_entry kroute;
1104 int virq;
1105
1106 if (!kvm_gsi_routing_enabled()) {
1107 return -ENOSYS;
1108 }
1109
1110 virq = kvm_irqchip_get_virq(s);
1111 if (virq < 0) {
1112 return virq;
1113 }
1114
1115 kroute.gsi = virq;
1116 kroute.type = KVM_IRQ_ROUTING_MSI;
1117 kroute.flags = 0;
1118 kroute.u.msi.address_lo = (uint32_t)msg.address;
1119 kroute.u.msi.address_hi = msg.address >> 32;
1120 kroute.u.msi.data = msg.data;
1121
1122 kvm_add_routing_entry(s, &kroute);
1123
1124 return virq;
1125 }
1126
1127 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1128 {
1129 struct kvm_irq_routing_entry kroute;
1130
1131 if (!kvm_irqchip_in_kernel()) {
1132 return -ENOSYS;
1133 }
1134
1135 kroute.gsi = virq;
1136 kroute.type = KVM_IRQ_ROUTING_MSI;
1137 kroute.flags = 0;
1138 kroute.u.msi.address_lo = (uint32_t)msg.address;
1139 kroute.u.msi.address_hi = msg.address >> 32;
1140 kroute.u.msi.data = msg.data;
1141
1142 return kvm_update_routing_entry(s, &kroute);
1143 }
1144
1145 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1146 {
1147 struct kvm_irqfd irqfd = {
1148 .fd = fd,
1149 .gsi = virq,
1150 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1151 };
1152
1153 if (!kvm_irqfds_enabled()) {
1154 return -ENOSYS;
1155 }
1156
1157 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
1158 }
1159
1160 #else /* !KVM_CAP_IRQ_ROUTING */
1161
1162 static void kvm_init_irq_routing(KVMState *s)
1163 {
1164 }
1165
1166 void kvm_irqchip_release_virq(KVMState *s, int virq)
1167 {
1168 }
1169
1170 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1171 {
1172 abort();
1173 }
1174
1175 int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1176 {
1177 return -ENOSYS;
1178 }
1179
1180 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1181 {
1182 abort();
1183 }
1184
1185 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1186 {
1187 return -ENOSYS;
1188 }
1189 #endif /* !KVM_CAP_IRQ_ROUTING */
1190
1191 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
1192 {
1193 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, true);
1194 }
1195
1196 int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
1197 {
1198 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, false);
1199 }
1200
1201 static int kvm_irqchip_create(KVMState *s)
1202 {
1203 QemuOptsList *list = qemu_find_opts("machine");
1204 int ret;
1205
1206 if (QTAILQ_EMPTY(&list->head) ||
1207 !qemu_opt_get_bool(QTAILQ_FIRST(&list->head),
1208 "kernel_irqchip", true) ||
1209 !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
1210 return 0;
1211 }
1212
1213 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
1214 if (ret < 0) {
1215 fprintf(stderr, "Create kernel irqchip failed\n");
1216 return ret;
1217 }
1218
1219 kvm_kernel_irqchip = true;
1220 /* If we have an in-kernel IRQ chip then we must have asynchronous
1221 * interrupt delivery (though the reverse is not necessarily true)
1222 */
1223 kvm_async_interrupts_allowed = true;
1224
1225 kvm_init_irq_routing(s);
1226
1227 return 0;
1228 }
1229
1230 static int kvm_max_vcpus(KVMState *s)
1231 {
1232 int ret;
1233
1234 /* Find number of supported CPUs using the recommended
1235 * procedure from the kernel API documentation to cope with
1236 * older kernels that may be missing capabilities.
1237 */
1238 ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
1239 if (ret) {
1240 return ret;
1241 }
1242 ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
1243 if (ret) {
1244 return ret;
1245 }
1246
1247 return 4;
1248 }
1249
1250 int kvm_init(void)
1251 {
1252 static const char upgrade_note[] =
1253 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
1254 "(see http://sourceforge.net/projects/kvm).\n";
1255 KVMState *s;
1256 const KVMCapabilityInfo *missing_cap;
1257 int ret;
1258 int i;
1259 int max_vcpus;
1260
1261 s = g_malloc0(sizeof(KVMState));
1262
1263 /*
1264 * On systems where the kernel can support different base page
1265 * sizes, host page size may be different from TARGET_PAGE_SIZE,
1266 * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum
1267 * page size for the system though.
1268 */
1269 assert(TARGET_PAGE_SIZE <= getpagesize());
1270
1271 #ifdef KVM_CAP_SET_GUEST_DEBUG
1272 QTAILQ_INIT(&s->kvm_sw_breakpoints);
1273 #endif
1274 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
1275 s->slots[i].slot = i;
1276 }
1277 s->vmfd = -1;
1278 s->fd = qemu_open("/dev/kvm", O_RDWR);
1279 if (s->fd == -1) {
1280 fprintf(stderr, "Could not access KVM kernel module: %m\n");
1281 ret = -errno;
1282 goto err;
1283 }
1284
1285 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
1286 if (ret < KVM_API_VERSION) {
1287 if (ret > 0) {
1288 ret = -EINVAL;
1289 }
1290 fprintf(stderr, "kvm version too old\n");
1291 goto err;
1292 }
1293
1294 if (ret > KVM_API_VERSION) {
1295 ret = -EINVAL;
1296 fprintf(stderr, "kvm version not supported\n");
1297 goto err;
1298 }
1299
1300 max_vcpus = kvm_max_vcpus(s);
1301 if (smp_cpus > max_vcpus) {
1302 ret = -EINVAL;
1303 fprintf(stderr, "Number of SMP cpus requested (%d) exceeds max cpus "
1304 "supported by KVM (%d)\n", smp_cpus, max_vcpus);
1305 goto err;
1306 }
1307
1308 s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
1309 if (s->vmfd < 0) {
1310 #ifdef TARGET_S390X
1311 fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
1312 "your host kernel command line\n");
1313 #endif
1314 ret = s->vmfd;
1315 goto err;
1316 }
1317
1318 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1319 if (!missing_cap) {
1320 missing_cap =
1321 kvm_check_extension_list(s, kvm_arch_required_capabilities);
1322 }
1323 if (missing_cap) {
1324 ret = -EINVAL;
1325 fprintf(stderr, "kvm does not support %s\n%s",
1326 missing_cap->name, upgrade_note);
1327 goto err;
1328 }
1329
1330 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
1331
1332 s->broken_set_mem_region = 1;
1333 ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
1334 if (ret > 0) {
1335 s->broken_set_mem_region = 0;
1336 }
1337
1338 #ifdef KVM_CAP_VCPU_EVENTS
1339 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1340 #endif
1341
1342 s->robust_singlestep =
1343 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
1344
1345 #ifdef KVM_CAP_DEBUGREGS
1346 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1347 #endif
1348
1349 #ifdef KVM_CAP_XSAVE
1350 s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
1351 #endif
1352
1353 #ifdef KVM_CAP_XCRS
1354 s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
1355 #endif
1356
1357 #ifdef KVM_CAP_PIT_STATE2
1358 s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
1359 #endif
1360
1361 #ifdef KVM_CAP_IRQ_ROUTING
1362 s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
1363 #endif
1364
1365 s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
1366
1367 s->irq_set_ioctl = KVM_IRQ_LINE;
1368 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
1369 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
1370 }
1371
1372 ret = kvm_arch_init(s);
1373 if (ret < 0) {
1374 goto err;
1375 }
1376
1377 ret = kvm_irqchip_create(s);
1378 if (ret < 0) {
1379 goto err;
1380 }
1381
1382 kvm_state = s;
1383 memory_listener_register(&kvm_memory_listener, &address_space_memory);
1384 memory_listener_register(&kvm_io_listener, &address_space_io);
1385
1386 s->many_ioeventfds = kvm_check_many_ioeventfds();
1387
1388 cpu_interrupt_handler = kvm_handle_interrupt;
1389
1390 return 0;
1391
1392 err:
1393 if (s->vmfd >= 0) {
1394 close(s->vmfd);
1395 }
1396 if (s->fd != -1) {
1397 close(s->fd);
1398 }
1399 g_free(s);
1400
1401 return ret;
1402 }
1403
1404 static void kvm_handle_io(uint16_t port, void *data, int direction, int size,
1405 uint32_t count)
1406 {
1407 int i;
1408 uint8_t *ptr = data;
1409
1410 for (i = 0; i < count; i++) {
1411 if (direction == KVM_EXIT_IO_IN) {
1412 switch (size) {
1413 case 1:
1414 stb_p(ptr, cpu_inb(port));
1415 break;
1416 case 2:
1417 stw_p(ptr, cpu_inw(port));
1418 break;
1419 case 4:
1420 stl_p(ptr, cpu_inl(port));
1421 break;
1422 }
1423 } else {
1424 switch (size) {
1425 case 1:
1426 cpu_outb(port, ldub_p(ptr));
1427 break;
1428 case 2:
1429 cpu_outw(port, lduw_p(ptr));
1430 break;
1431 case 4:
1432 cpu_outl(port, ldl_p(ptr));
1433 break;
1434 }
1435 }
1436
1437 ptr += size;
1438 }
1439 }
1440
1441 static int kvm_handle_internal_error(CPUArchState *env, struct kvm_run *run)
1442 {
1443 CPUState *cpu = ENV_GET_CPU(env);
1444
1445 fprintf(stderr, "KVM internal error.");
1446 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1447 int i;
1448
1449 fprintf(stderr, " Suberror: %d\n", run->internal.suberror);
1450 for (i = 0; i < run->internal.ndata; ++i) {
1451 fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1452 i, (uint64_t)run->internal.data[i]);
1453 }
1454 } else {
1455 fprintf(stderr, "\n");
1456 }
1457 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1458 fprintf(stderr, "emulation failure\n");
1459 if (!kvm_arch_stop_on_emulation_error(cpu)) {
1460 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1461 return EXCP_INTERRUPT;
1462 }
1463 }
1464 /* FIXME: Should trigger a qmp message to let management know
1465 * something went wrong.
1466 */
1467 return -1;
1468 }
1469
1470 void kvm_flush_coalesced_mmio_buffer(void)
1471 {
1472 KVMState *s = kvm_state;
1473
1474 if (s->coalesced_flush_in_progress) {
1475 return;
1476 }
1477
1478 s->coalesced_flush_in_progress = true;
1479
1480 if (s->coalesced_mmio_ring) {
1481 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
1482 while (ring->first != ring->last) {
1483 struct kvm_coalesced_mmio *ent;
1484
1485 ent = &ring->coalesced_mmio[ring->first];
1486
1487 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
1488 smp_wmb();
1489 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1490 }
1491 }
1492
1493 s->coalesced_flush_in_progress = false;
1494 }
1495
1496 static void do_kvm_cpu_synchronize_state(void *arg)
1497 {
1498 CPUState *cpu = arg;
1499
1500 if (!cpu->kvm_vcpu_dirty) {
1501 kvm_arch_get_registers(cpu);
1502 cpu->kvm_vcpu_dirty = true;
1503 }
1504 }
1505
1506 void kvm_cpu_synchronize_state(CPUArchState *env)
1507 {
1508 CPUState *cpu = ENV_GET_CPU(env);
1509
1510 if (!cpu->kvm_vcpu_dirty) {
1511 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, cpu);
1512 }
1513 }
1514
1515 void kvm_cpu_synchronize_post_reset(CPUArchState *env)
1516 {
1517 CPUState *cpu = ENV_GET_CPU(env);
1518
1519 kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
1520 cpu->kvm_vcpu_dirty = false;
1521 }
1522
1523 void kvm_cpu_synchronize_post_init(CPUArchState *env)
1524 {
1525 CPUState *cpu = ENV_GET_CPU(env);
1526
1527 kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
1528 cpu->kvm_vcpu_dirty = false;
1529 }
1530
1531 int kvm_cpu_exec(CPUArchState *env)
1532 {
1533 CPUState *cpu = ENV_GET_CPU(env);
1534 struct kvm_run *run = cpu->kvm_run;
1535 int ret, run_ret;
1536
1537 DPRINTF("kvm_cpu_exec()\n");
1538
1539 if (kvm_arch_process_async_events(cpu)) {
1540 cpu->exit_request = 0;
1541 return EXCP_HLT;
1542 }
1543
1544 do {
1545 if (cpu->kvm_vcpu_dirty) {
1546 kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
1547 cpu->kvm_vcpu_dirty = false;
1548 }
1549
1550 kvm_arch_pre_run(cpu, run);
1551 if (cpu->exit_request) {
1552 DPRINTF("interrupt exit requested\n");
1553 /*
1554 * KVM requires us to reenter the kernel after IO exits to complete
1555 * instruction emulation. This self-signal will ensure that we
1556 * leave ASAP again.
1557 */
1558 qemu_cpu_kick_self();
1559 }
1560 qemu_mutex_unlock_iothread();
1561
1562 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
1563
1564 qemu_mutex_lock_iothread();
1565 kvm_arch_post_run(cpu, run);
1566
1567 if (run_ret < 0) {
1568 if (run_ret == -EINTR || run_ret == -EAGAIN) {
1569 DPRINTF("io window exit\n");
1570 ret = EXCP_INTERRUPT;
1571 break;
1572 }
1573 fprintf(stderr, "error: kvm run failed %s\n",
1574 strerror(-run_ret));
1575 abort();
1576 }
1577
1578 switch (run->exit_reason) {
1579 case KVM_EXIT_IO:
1580 DPRINTF("handle_io\n");
1581 kvm_handle_io(run->io.port,
1582 (uint8_t *)run + run->io.data_offset,
1583 run->io.direction,
1584 run->io.size,
1585 run->io.count);
1586 ret = 0;
1587 break;
1588 case KVM_EXIT_MMIO:
1589 DPRINTF("handle_mmio\n");
1590 cpu_physical_memory_rw(run->mmio.phys_addr,
1591 run->mmio.data,
1592 run->mmio.len,
1593 run->mmio.is_write);
1594 ret = 0;
1595 break;
1596 case KVM_EXIT_IRQ_WINDOW_OPEN:
1597 DPRINTF("irq_window_open\n");
1598 ret = EXCP_INTERRUPT;
1599 break;
1600 case KVM_EXIT_SHUTDOWN:
1601 DPRINTF("shutdown\n");
1602 qemu_system_reset_request();
1603 ret = EXCP_INTERRUPT;
1604 break;
1605 case KVM_EXIT_UNKNOWN:
1606 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
1607 (uint64_t)run->hw.hardware_exit_reason);
1608 ret = -1;
1609 break;
1610 case KVM_EXIT_INTERNAL_ERROR:
1611 ret = kvm_handle_internal_error(env, run);
1612 break;
1613 default:
1614 DPRINTF("kvm_arch_handle_exit\n");
1615 ret = kvm_arch_handle_exit(cpu, run);
1616 break;
1617 }
1618 } while (ret == 0);
1619
1620 if (ret < 0) {
1621 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1622 vm_stop(RUN_STATE_INTERNAL_ERROR);
1623 }
1624
1625 cpu->exit_request = 0;
1626 return ret;
1627 }
1628
1629 int kvm_ioctl(KVMState *s, int type, ...)
1630 {
1631 int ret;
1632 void *arg;
1633 va_list ap;
1634
1635 va_start(ap, type);
1636 arg = va_arg(ap, void *);
1637 va_end(ap);
1638
1639 ret = ioctl(s->fd, type, arg);
1640 if (ret == -1) {
1641 ret = -errno;
1642 }
1643 return ret;
1644 }
1645
1646 int kvm_vm_ioctl(KVMState *s, int type, ...)
1647 {
1648 int ret;
1649 void *arg;
1650 va_list ap;
1651
1652 va_start(ap, type);
1653 arg = va_arg(ap, void *);
1654 va_end(ap);
1655
1656 ret = ioctl(s->vmfd, type, arg);
1657 if (ret == -1) {
1658 ret = -errno;
1659 }
1660 return ret;
1661 }
1662
1663 int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
1664 {
1665 int ret;
1666 void *arg;
1667 va_list ap;
1668
1669 va_start(ap, type);
1670 arg = va_arg(ap, void *);
1671 va_end(ap);
1672
1673 ret = ioctl(cpu->kvm_fd, type, arg);
1674 if (ret == -1) {
1675 ret = -errno;
1676 }
1677 return ret;
1678 }
1679
1680 int kvm_has_sync_mmu(void)
1681 {
1682 return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
1683 }
1684
1685 int kvm_has_vcpu_events(void)
1686 {
1687 return kvm_state->vcpu_events;
1688 }
1689
1690 int kvm_has_robust_singlestep(void)
1691 {
1692 return kvm_state->robust_singlestep;
1693 }
1694
1695 int kvm_has_debugregs(void)
1696 {
1697 return kvm_state->debugregs;
1698 }
1699
1700 int kvm_has_xsave(void)
1701 {
1702 return kvm_state->xsave;
1703 }
1704
1705 int kvm_has_xcrs(void)
1706 {
1707 return kvm_state->xcrs;
1708 }
1709
1710 int kvm_has_pit_state2(void)
1711 {
1712 return kvm_state->pit_state2;
1713 }
1714
1715 int kvm_has_many_ioeventfds(void)
1716 {
1717 if (!kvm_enabled()) {
1718 return 0;
1719 }
1720 return kvm_state->many_ioeventfds;
1721 }
1722
1723 int kvm_has_gsi_routing(void)
1724 {
1725 #ifdef KVM_CAP_IRQ_ROUTING
1726 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
1727 #else
1728 return false;
1729 #endif
1730 }
1731
1732 int kvm_has_intx_set_mask(void)
1733 {
1734 return kvm_state->intx_set_mask;
1735 }
1736
1737 void *kvm_vmalloc(ram_addr_t size)
1738 {
1739 #ifdef TARGET_S390X
1740 void *mem;
1741
1742 mem = kvm_arch_vmalloc(size);
1743 if (mem) {
1744 return mem;
1745 }
1746 #endif
1747 return qemu_vmalloc(size);
1748 }
1749
1750 void kvm_setup_guest_memory(void *start, size_t size)
1751 {
1752 #ifdef CONFIG_VALGRIND_H
1753 VALGRIND_MAKE_MEM_DEFINED(start, size);
1754 #endif
1755 if (!kvm_has_sync_mmu()) {
1756 int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
1757
1758 if (ret) {
1759 perror("qemu_madvise");
1760 fprintf(stderr,
1761 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1762 exit(1);
1763 }
1764 }
1765 }
1766
1767 #ifdef KVM_CAP_SET_GUEST_DEBUG
1768 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
1769 target_ulong pc)
1770 {
1771 struct kvm_sw_breakpoint *bp;
1772
1773 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
1774 if (bp->pc == pc) {
1775 return bp;
1776 }
1777 }
1778 return NULL;
1779 }
1780
1781 int kvm_sw_breakpoints_active(CPUState *cpu)
1782 {
1783 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
1784 }
1785
1786 struct kvm_set_guest_debug_data {
1787 struct kvm_guest_debug dbg;
1788 CPUState *cpu;
1789 int err;
1790 };
1791
1792 static void kvm_invoke_set_guest_debug(void *data)
1793 {
1794 struct kvm_set_guest_debug_data *dbg_data = data;
1795
1796 dbg_data->err = kvm_vcpu_ioctl(dbg_data->cpu, KVM_SET_GUEST_DEBUG,
1797 &dbg_data->dbg);
1798 }
1799
1800 int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
1801 {
1802 CPUState *cpu = ENV_GET_CPU(env);
1803 struct kvm_set_guest_debug_data data;
1804
1805 data.dbg.control = reinject_trap;
1806
1807 if (env->singlestep_enabled) {
1808 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1809 }
1810 kvm_arch_update_guest_debug(cpu, &data.dbg);
1811 data.cpu = cpu;
1812
1813 run_on_cpu(cpu, kvm_invoke_set_guest_debug, &data);
1814 return data.err;
1815 }
1816
1817 int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
1818 target_ulong len, int type)
1819 {
1820 CPUState *current_cpu = ENV_GET_CPU(current_env);
1821 struct kvm_sw_breakpoint *bp;
1822 CPUArchState *env;
1823 int err;
1824
1825 if (type == GDB_BREAKPOINT_SW) {
1826 bp = kvm_find_sw_breakpoint(current_cpu, addr);
1827 if (bp) {
1828 bp->use_count++;
1829 return 0;
1830 }
1831
1832 bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
1833 if (!bp) {
1834 return -ENOMEM;
1835 }
1836
1837 bp->pc = addr;
1838 bp->use_count = 1;
1839 err = kvm_arch_insert_sw_breakpoint(current_cpu, bp);
1840 if (err) {
1841 g_free(bp);
1842 return err;
1843 }
1844
1845 QTAILQ_INSERT_HEAD(&current_cpu->kvm_state->kvm_sw_breakpoints,
1846 bp, entry);
1847 } else {
1848 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1849 if (err) {
1850 return err;
1851 }
1852 }
1853
1854 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1855 err = kvm_update_guest_debug(env, 0);
1856 if (err) {
1857 return err;
1858 }
1859 }
1860 return 0;
1861 }
1862
1863 int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
1864 target_ulong len, int type)
1865 {
1866 CPUState *current_cpu = ENV_GET_CPU(current_env);
1867 struct kvm_sw_breakpoint *bp;
1868 CPUArchState *env;
1869 int err;
1870
1871 if (type == GDB_BREAKPOINT_SW) {
1872 bp = kvm_find_sw_breakpoint(current_cpu, addr);
1873 if (!bp) {
1874 return -ENOENT;
1875 }
1876
1877 if (bp->use_count > 1) {
1878 bp->use_count--;
1879 return 0;
1880 }
1881
1882 err = kvm_arch_remove_sw_breakpoint(current_cpu, bp);
1883 if (err) {
1884 return err;
1885 }
1886
1887 QTAILQ_REMOVE(&current_cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
1888 g_free(bp);
1889 } else {
1890 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1891 if (err) {
1892 return err;
1893 }
1894 }
1895
1896 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1897 err = kvm_update_guest_debug(env, 0);
1898 if (err) {
1899 return err;
1900 }
1901 }
1902 return 0;
1903 }
1904
1905 void kvm_remove_all_breakpoints(CPUArchState *current_env)
1906 {
1907 CPUState *current_cpu = ENV_GET_CPU(current_env);
1908 struct kvm_sw_breakpoint *bp, *next;
1909 KVMState *s = current_cpu->kvm_state;
1910 CPUArchState *env;
1911 CPUState *cpu;
1912
1913 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1914 if (kvm_arch_remove_sw_breakpoint(current_cpu, bp) != 0) {
1915 /* Try harder to find a CPU that currently sees the breakpoint. */
1916 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1917 cpu = ENV_GET_CPU(env);
1918 if (kvm_arch_remove_sw_breakpoint(cpu, bp) == 0) {
1919 break;
1920 }
1921 }
1922 }
1923 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
1924 g_free(bp);
1925 }
1926 kvm_arch_remove_all_hw_breakpoints();
1927
1928 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1929 kvm_update_guest_debug(env, 0);
1930 }
1931 }
1932
1933 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1934
1935 int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
1936 {
1937 return -EINVAL;
1938 }
1939
1940 int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
1941 target_ulong len, int type)
1942 {
1943 return -EINVAL;
1944 }
1945
1946 int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
1947 target_ulong len, int type)
1948 {
1949 return -EINVAL;
1950 }
1951
1952 void kvm_remove_all_breakpoints(CPUArchState *current_env)
1953 {
1954 }
1955 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1956
1957 int kvm_set_signal_mask(CPUArchState *env, const sigset_t *sigset)
1958 {
1959 CPUState *cpu = ENV_GET_CPU(env);
1960 struct kvm_signal_mask *sigmask;
1961 int r;
1962
1963 if (!sigset) {
1964 return kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, NULL);
1965 }
1966
1967 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
1968
1969 sigmask->len = 8;
1970 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1971 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
1972 g_free(sigmask);
1973
1974 return r;
1975 }
1976
1977 int kvm_set_ioeventfd_mmio(int fd, uint32_t addr, uint32_t val, bool assign,
1978 uint32_t size)
1979 {
1980 int ret;
1981 struct kvm_ioeventfd iofd;
1982
1983 iofd.datamatch = val;
1984 iofd.addr = addr;
1985 iofd.len = size;
1986 iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
1987 iofd.fd = fd;
1988
1989 if (!kvm_enabled()) {
1990 return -ENOSYS;
1991 }
1992
1993 if (!assign) {
1994 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1995 }
1996
1997 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1998
1999 if (ret < 0) {
2000 return -errno;
2001 }
2002
2003 return 0;
2004 }
2005
2006 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
2007 {
2008 struct kvm_ioeventfd kick = {
2009 .datamatch = val,
2010 .addr = addr,
2011 .len = 2,
2012 .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
2013 .fd = fd,
2014 };
2015 int r;
2016 if (!kvm_enabled()) {
2017 return -ENOSYS;
2018 }
2019 if (!assign) {
2020 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
2021 }
2022 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
2023 if (r < 0) {
2024 return r;
2025 }
2026 return 0;
2027 }
2028
2029 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
2030 {
2031 return kvm_arch_on_sigbus_vcpu(cpu, code, addr);
2032 }
2033
2034 int kvm_on_sigbus(int code, void *addr)
2035 {
2036 return kvm_arch_on_sigbus(code, addr);
2037 }