vga: make Cirrus ISA device optional
[qemu.git] / kvm-all.c
1 /*
2 * QEMU KVM support
3 *
4 * Copyright IBM, Corp. 2008
5 * Red Hat, Inc. 2008
6 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Glauber Costa <gcosta@redhat.com>
10 *
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
13 *
14 */
15
16 #include <sys/types.h>
17 #include <sys/ioctl.h>
18 #include <sys/mman.h>
19 #include <stdarg.h>
20
21 #include <linux/kvm.h>
22
23 #include "qemu-common.h"
24 #include "qemu-barrier.h"
25 #include "sysemu.h"
26 #include "hw/hw.h"
27 #include "gdbstub.h"
28 #include "kvm.h"
29 #include "bswap.h"
30 #include "memory.h"
31
32 /* This check must be after config-host.h is included */
33 #ifdef CONFIG_EVENTFD
34 #include <sys/eventfd.h>
35 #endif
36
37 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
38 #define PAGE_SIZE TARGET_PAGE_SIZE
39
40 //#define DEBUG_KVM
41
42 #ifdef DEBUG_KVM
43 #define DPRINTF(fmt, ...) \
44 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
45 #else
46 #define DPRINTF(fmt, ...) \
47 do { } while (0)
48 #endif
49
50 typedef struct KVMSlot
51 {
52 target_phys_addr_t start_addr;
53 ram_addr_t memory_size;
54 void *ram;
55 int slot;
56 int flags;
57 } KVMSlot;
58
59 typedef struct kvm_dirty_log KVMDirtyLog;
60
61 struct KVMState
62 {
63 KVMSlot slots[32];
64 int fd;
65 int vmfd;
66 int coalesced_mmio;
67 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
68 bool coalesced_flush_in_progress;
69 int broken_set_mem_region;
70 int migration_log;
71 int vcpu_events;
72 int robust_singlestep;
73 int debugregs;
74 #ifdef KVM_CAP_SET_GUEST_DEBUG
75 struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
76 #endif
77 int irqchip_in_kernel;
78 int pit_in_kernel;
79 int xsave, xcrs;
80 int many_ioeventfds;
81 };
82
83 KVMState *kvm_state;
84
85 static const KVMCapabilityInfo kvm_required_capabilites[] = {
86 KVM_CAP_INFO(USER_MEMORY),
87 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
88 KVM_CAP_LAST_INFO
89 };
90
91 static KVMSlot *kvm_alloc_slot(KVMState *s)
92 {
93 int i;
94
95 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
96 if (s->slots[i].memory_size == 0) {
97 return &s->slots[i];
98 }
99 }
100
101 fprintf(stderr, "%s: no free slot available\n", __func__);
102 abort();
103 }
104
105 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
106 target_phys_addr_t start_addr,
107 target_phys_addr_t end_addr)
108 {
109 int i;
110
111 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
112 KVMSlot *mem = &s->slots[i];
113
114 if (start_addr == mem->start_addr &&
115 end_addr == mem->start_addr + mem->memory_size) {
116 return mem;
117 }
118 }
119
120 return NULL;
121 }
122
123 /*
124 * Find overlapping slot with lowest start address
125 */
126 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
127 target_phys_addr_t start_addr,
128 target_phys_addr_t end_addr)
129 {
130 KVMSlot *found = NULL;
131 int i;
132
133 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
134 KVMSlot *mem = &s->slots[i];
135
136 if (mem->memory_size == 0 ||
137 (found && found->start_addr < mem->start_addr)) {
138 continue;
139 }
140
141 if (end_addr > mem->start_addr &&
142 start_addr < mem->start_addr + mem->memory_size) {
143 found = mem;
144 }
145 }
146
147 return found;
148 }
149
150 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
151 target_phys_addr_t *phys_addr)
152 {
153 int i;
154
155 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
156 KVMSlot *mem = &s->slots[i];
157
158 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
159 *phys_addr = mem->start_addr + (ram - mem->ram);
160 return 1;
161 }
162 }
163
164 return 0;
165 }
166
167 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
168 {
169 struct kvm_userspace_memory_region mem;
170
171 mem.slot = slot->slot;
172 mem.guest_phys_addr = slot->start_addr;
173 mem.memory_size = slot->memory_size;
174 mem.userspace_addr = (unsigned long)slot->ram;
175 mem.flags = slot->flags;
176 if (s->migration_log) {
177 mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
178 }
179 return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
180 }
181
182 static void kvm_reset_vcpu(void *opaque)
183 {
184 CPUState *env = opaque;
185
186 kvm_arch_reset_vcpu(env);
187 }
188
189 int kvm_irqchip_in_kernel(void)
190 {
191 return kvm_state->irqchip_in_kernel;
192 }
193
194 int kvm_pit_in_kernel(void)
195 {
196 return kvm_state->pit_in_kernel;
197 }
198
199 int kvm_init_vcpu(CPUState *env)
200 {
201 KVMState *s = kvm_state;
202 long mmap_size;
203 int ret;
204
205 DPRINTF("kvm_init_vcpu\n");
206
207 ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
208 if (ret < 0) {
209 DPRINTF("kvm_create_vcpu failed\n");
210 goto err;
211 }
212
213 env->kvm_fd = ret;
214 env->kvm_state = s;
215 env->kvm_vcpu_dirty = 1;
216
217 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
218 if (mmap_size < 0) {
219 ret = mmap_size;
220 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
221 goto err;
222 }
223
224 env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
225 env->kvm_fd, 0);
226 if (env->kvm_run == MAP_FAILED) {
227 ret = -errno;
228 DPRINTF("mmap'ing vcpu state failed\n");
229 goto err;
230 }
231
232 if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
233 s->coalesced_mmio_ring =
234 (void *)env->kvm_run + s->coalesced_mmio * PAGE_SIZE;
235 }
236
237 ret = kvm_arch_init_vcpu(env);
238 if (ret == 0) {
239 qemu_register_reset(kvm_reset_vcpu, env);
240 kvm_arch_reset_vcpu(env);
241 }
242 err:
243 return ret;
244 }
245
246 /*
247 * dirty pages logging control
248 */
249
250 static int kvm_mem_flags(KVMState *s, bool log_dirty)
251 {
252 return log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
253 }
254
255 static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
256 {
257 KVMState *s = kvm_state;
258 int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
259 int old_flags;
260
261 old_flags = mem->flags;
262
263 flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty);
264 mem->flags = flags;
265
266 /* If nothing changed effectively, no need to issue ioctl */
267 if (s->migration_log) {
268 flags |= KVM_MEM_LOG_DIRTY_PAGES;
269 }
270
271 if (flags == old_flags) {
272 return 0;
273 }
274
275 return kvm_set_user_memory_region(s, mem);
276 }
277
278 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
279 ram_addr_t size, bool log_dirty)
280 {
281 KVMState *s = kvm_state;
282 KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
283
284 if (mem == NULL) {
285 fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
286 TARGET_FMT_plx "\n", __func__, phys_addr,
287 (target_phys_addr_t)(phys_addr + size - 1));
288 return -EINVAL;
289 }
290 return kvm_slot_dirty_pages_log_change(mem, log_dirty);
291 }
292
293 static void kvm_log_start(MemoryListener *listener,
294 MemoryRegionSection *section)
295 {
296 int r;
297
298 r = kvm_dirty_pages_log_change(section->offset_within_address_space,
299 section->size, true);
300 if (r < 0) {
301 abort();
302 }
303 }
304
305 static void kvm_log_stop(MemoryListener *listener,
306 MemoryRegionSection *section)
307 {
308 int r;
309
310 r = kvm_dirty_pages_log_change(section->offset_within_address_space,
311 section->size, false);
312 if (r < 0) {
313 abort();
314 }
315 }
316
317 static int kvm_set_migration_log(int enable)
318 {
319 KVMState *s = kvm_state;
320 KVMSlot *mem;
321 int i, err;
322
323 s->migration_log = enable;
324
325 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
326 mem = &s->slots[i];
327
328 if (!mem->memory_size) {
329 continue;
330 }
331 if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
332 continue;
333 }
334 err = kvm_set_user_memory_region(s, mem);
335 if (err) {
336 return err;
337 }
338 }
339 return 0;
340 }
341
342 /* get kvm's dirty pages bitmap and update qemu's */
343 static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
344 unsigned long *bitmap)
345 {
346 unsigned int i, j;
347 unsigned long page_number, c;
348 target_phys_addr_t addr, addr1;
349 unsigned int len = ((section->size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
350
351 /*
352 * bitmap-traveling is faster than memory-traveling (for addr...)
353 * especially when most of the memory is not dirty.
354 */
355 for (i = 0; i < len; i++) {
356 if (bitmap[i] != 0) {
357 c = leul_to_cpu(bitmap[i]);
358 do {
359 j = ffsl(c) - 1;
360 c &= ~(1ul << j);
361 page_number = i * HOST_LONG_BITS + j;
362 addr1 = page_number * TARGET_PAGE_SIZE;
363 addr = section->offset_within_region + addr1;
364 memory_region_set_dirty(section->mr, addr);
365 } while (c != 0);
366 }
367 }
368 return 0;
369 }
370
371 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
372
373 /**
374 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
375 * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
376 * This means all bits are set to dirty.
377 *
378 * @start_add: start of logged region.
379 * @end_addr: end of logged region.
380 */
381 static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
382 {
383 KVMState *s = kvm_state;
384 unsigned long size, allocated_size = 0;
385 KVMDirtyLog d;
386 KVMSlot *mem;
387 int ret = 0;
388 target_phys_addr_t start_addr = section->offset_within_address_space;
389 target_phys_addr_t end_addr = start_addr + section->size;
390
391 d.dirty_bitmap = NULL;
392 while (start_addr < end_addr) {
393 mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
394 if (mem == NULL) {
395 break;
396 }
397
398 /* XXX bad kernel interface alert
399 * For dirty bitmap, kernel allocates array of size aligned to
400 * bits-per-long. But for case when the kernel is 64bits and
401 * the userspace is 32bits, userspace can't align to the same
402 * bits-per-long, since sizeof(long) is different between kernel
403 * and user space. This way, userspace will provide buffer which
404 * may be 4 bytes less than the kernel will use, resulting in
405 * userspace memory corruption (which is not detectable by valgrind
406 * too, in most cases).
407 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
408 * a hope that sizeof(long) wont become >8 any time soon.
409 */
410 size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
411 /*HOST_LONG_BITS*/ 64) / 8;
412 if (!d.dirty_bitmap) {
413 d.dirty_bitmap = g_malloc(size);
414 } else if (size > allocated_size) {
415 d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
416 }
417 allocated_size = size;
418 memset(d.dirty_bitmap, 0, allocated_size);
419
420 d.slot = mem->slot;
421
422 if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
423 DPRINTF("ioctl failed %d\n", errno);
424 ret = -1;
425 break;
426 }
427
428 kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
429 start_addr = mem->start_addr + mem->memory_size;
430 }
431 g_free(d.dirty_bitmap);
432
433 return ret;
434 }
435
436 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
437 {
438 int ret = -ENOSYS;
439 KVMState *s = kvm_state;
440
441 if (s->coalesced_mmio) {
442 struct kvm_coalesced_mmio_zone zone;
443
444 zone.addr = start;
445 zone.size = size;
446
447 ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
448 }
449
450 return ret;
451 }
452
453 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
454 {
455 int ret = -ENOSYS;
456 KVMState *s = kvm_state;
457
458 if (s->coalesced_mmio) {
459 struct kvm_coalesced_mmio_zone zone;
460
461 zone.addr = start;
462 zone.size = size;
463
464 ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
465 }
466
467 return ret;
468 }
469
470 int kvm_check_extension(KVMState *s, unsigned int extension)
471 {
472 int ret;
473
474 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
475 if (ret < 0) {
476 ret = 0;
477 }
478
479 return ret;
480 }
481
482 static int kvm_check_many_ioeventfds(void)
483 {
484 /* Userspace can use ioeventfd for io notification. This requires a host
485 * that supports eventfd(2) and an I/O thread; since eventfd does not
486 * support SIGIO it cannot interrupt the vcpu.
487 *
488 * Older kernels have a 6 device limit on the KVM io bus. Find out so we
489 * can avoid creating too many ioeventfds.
490 */
491 #if defined(CONFIG_EVENTFD)
492 int ioeventfds[7];
493 int i, ret = 0;
494 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
495 ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
496 if (ioeventfds[i] < 0) {
497 break;
498 }
499 ret = kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, true);
500 if (ret < 0) {
501 close(ioeventfds[i]);
502 break;
503 }
504 }
505
506 /* Decide whether many devices are supported or not */
507 ret = i == ARRAY_SIZE(ioeventfds);
508
509 while (i-- > 0) {
510 kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, false);
511 close(ioeventfds[i]);
512 }
513 return ret;
514 #else
515 return 0;
516 #endif
517 }
518
519 static const KVMCapabilityInfo *
520 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
521 {
522 while (list->name) {
523 if (!kvm_check_extension(s, list->value)) {
524 return list;
525 }
526 list++;
527 }
528 return NULL;
529 }
530
531 static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
532 {
533 KVMState *s = kvm_state;
534 KVMSlot *mem, old;
535 int err;
536 MemoryRegion *mr = section->mr;
537 bool log_dirty = memory_region_is_logging(mr);
538 target_phys_addr_t start_addr = section->offset_within_address_space;
539 ram_addr_t size = section->size;
540 void *ram = NULL;
541
542 /* kvm works in page size chunks, but the function may be called
543 with sub-page size and unaligned start address. */
544 size = TARGET_PAGE_ALIGN(size);
545 start_addr = TARGET_PAGE_ALIGN(start_addr);
546
547 if (!memory_region_is_ram(mr)) {
548 return;
549 }
550
551 ram = memory_region_get_ram_ptr(mr) + section->offset_within_region;
552
553 while (1) {
554 mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
555 if (!mem) {
556 break;
557 }
558
559 if (add && start_addr >= mem->start_addr &&
560 (start_addr + size <= mem->start_addr + mem->memory_size) &&
561 (ram - start_addr == mem->ram - mem->start_addr)) {
562 /* The new slot fits into the existing one and comes with
563 * identical parameters - update flags and done. */
564 kvm_slot_dirty_pages_log_change(mem, log_dirty);
565 return;
566 }
567
568 old = *mem;
569
570 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
571 kvm_physical_sync_dirty_bitmap(section);
572 }
573
574 /* unregister the overlapping slot */
575 mem->memory_size = 0;
576 err = kvm_set_user_memory_region(s, mem);
577 if (err) {
578 fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
579 __func__, strerror(-err));
580 abort();
581 }
582
583 /* Workaround for older KVM versions: we can't join slots, even not by
584 * unregistering the previous ones and then registering the larger
585 * slot. We have to maintain the existing fragmentation. Sigh.
586 *
587 * This workaround assumes that the new slot starts at the same
588 * address as the first existing one. If not or if some overlapping
589 * slot comes around later, we will fail (not seen in practice so far)
590 * - and actually require a recent KVM version. */
591 if (s->broken_set_mem_region &&
592 old.start_addr == start_addr && old.memory_size < size && add) {
593 mem = kvm_alloc_slot(s);
594 mem->memory_size = old.memory_size;
595 mem->start_addr = old.start_addr;
596 mem->ram = old.ram;
597 mem->flags = kvm_mem_flags(s, log_dirty);
598
599 err = kvm_set_user_memory_region(s, mem);
600 if (err) {
601 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
602 strerror(-err));
603 abort();
604 }
605
606 start_addr += old.memory_size;
607 ram += old.memory_size;
608 size -= old.memory_size;
609 continue;
610 }
611
612 /* register prefix slot */
613 if (old.start_addr < start_addr) {
614 mem = kvm_alloc_slot(s);
615 mem->memory_size = start_addr - old.start_addr;
616 mem->start_addr = old.start_addr;
617 mem->ram = old.ram;
618 mem->flags = kvm_mem_flags(s, log_dirty);
619
620 err = kvm_set_user_memory_region(s, mem);
621 if (err) {
622 fprintf(stderr, "%s: error registering prefix slot: %s\n",
623 __func__, strerror(-err));
624 #ifdef TARGET_PPC
625 fprintf(stderr, "%s: This is probably because your kernel's " \
626 "PAGE_SIZE is too big. Please try to use 4k " \
627 "PAGE_SIZE!\n", __func__);
628 #endif
629 abort();
630 }
631 }
632
633 /* register suffix slot */
634 if (old.start_addr + old.memory_size > start_addr + size) {
635 ram_addr_t size_delta;
636
637 mem = kvm_alloc_slot(s);
638 mem->start_addr = start_addr + size;
639 size_delta = mem->start_addr - old.start_addr;
640 mem->memory_size = old.memory_size - size_delta;
641 mem->ram = old.ram + size_delta;
642 mem->flags = kvm_mem_flags(s, log_dirty);
643
644 err = kvm_set_user_memory_region(s, mem);
645 if (err) {
646 fprintf(stderr, "%s: error registering suffix slot: %s\n",
647 __func__, strerror(-err));
648 abort();
649 }
650 }
651 }
652
653 /* in case the KVM bug workaround already "consumed" the new slot */
654 if (!size) {
655 return;
656 }
657 if (!add) {
658 return;
659 }
660 mem = kvm_alloc_slot(s);
661 mem->memory_size = size;
662 mem->start_addr = start_addr;
663 mem->ram = ram;
664 mem->flags = kvm_mem_flags(s, log_dirty);
665
666 err = kvm_set_user_memory_region(s, mem);
667 if (err) {
668 fprintf(stderr, "%s: error registering slot: %s\n", __func__,
669 strerror(-err));
670 abort();
671 }
672 }
673
674 static void kvm_region_add(MemoryListener *listener,
675 MemoryRegionSection *section)
676 {
677 kvm_set_phys_mem(section, true);
678 }
679
680 static void kvm_region_del(MemoryListener *listener,
681 MemoryRegionSection *section)
682 {
683 kvm_set_phys_mem(section, false);
684 }
685
686 static void kvm_log_sync(MemoryListener *listener,
687 MemoryRegionSection *section)
688 {
689 int r;
690
691 r = kvm_physical_sync_dirty_bitmap(section);
692 if (r < 0) {
693 abort();
694 }
695 }
696
697 static void kvm_log_global_start(struct MemoryListener *listener)
698 {
699 int r;
700
701 r = kvm_set_migration_log(1);
702 assert(r >= 0);
703 }
704
705 static void kvm_log_global_stop(struct MemoryListener *listener)
706 {
707 int r;
708
709 r = kvm_set_migration_log(0);
710 assert(r >= 0);
711 }
712
713 static MemoryListener kvm_memory_listener = {
714 .region_add = kvm_region_add,
715 .region_del = kvm_region_del,
716 .log_start = kvm_log_start,
717 .log_stop = kvm_log_stop,
718 .log_sync = kvm_log_sync,
719 .log_global_start = kvm_log_global_start,
720 .log_global_stop = kvm_log_global_stop,
721 };
722
723 static void kvm_handle_interrupt(CPUState *env, int mask)
724 {
725 env->interrupt_request |= mask;
726
727 if (!qemu_cpu_is_self(env)) {
728 qemu_cpu_kick(env);
729 }
730 }
731
732 int kvm_init(void)
733 {
734 static const char upgrade_note[] =
735 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
736 "(see http://sourceforge.net/projects/kvm).\n";
737 KVMState *s;
738 const KVMCapabilityInfo *missing_cap;
739 int ret;
740 int i;
741
742 s = g_malloc0(sizeof(KVMState));
743
744 #ifdef KVM_CAP_SET_GUEST_DEBUG
745 QTAILQ_INIT(&s->kvm_sw_breakpoints);
746 #endif
747 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
748 s->slots[i].slot = i;
749 }
750 s->vmfd = -1;
751 s->fd = qemu_open("/dev/kvm", O_RDWR);
752 if (s->fd == -1) {
753 fprintf(stderr, "Could not access KVM kernel module: %m\n");
754 ret = -errno;
755 goto err;
756 }
757
758 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
759 if (ret < KVM_API_VERSION) {
760 if (ret > 0) {
761 ret = -EINVAL;
762 }
763 fprintf(stderr, "kvm version too old\n");
764 goto err;
765 }
766
767 if (ret > KVM_API_VERSION) {
768 ret = -EINVAL;
769 fprintf(stderr, "kvm version not supported\n");
770 goto err;
771 }
772
773 s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
774 if (s->vmfd < 0) {
775 #ifdef TARGET_S390X
776 fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
777 "your host kernel command line\n");
778 #endif
779 ret = s->vmfd;
780 goto err;
781 }
782
783 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
784 if (!missing_cap) {
785 missing_cap =
786 kvm_check_extension_list(s, kvm_arch_required_capabilities);
787 }
788 if (missing_cap) {
789 ret = -EINVAL;
790 fprintf(stderr, "kvm does not support %s\n%s",
791 missing_cap->name, upgrade_note);
792 goto err;
793 }
794
795 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
796
797 s->broken_set_mem_region = 1;
798 ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
799 if (ret > 0) {
800 s->broken_set_mem_region = 0;
801 }
802
803 #ifdef KVM_CAP_VCPU_EVENTS
804 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
805 #endif
806
807 s->robust_singlestep =
808 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
809
810 #ifdef KVM_CAP_DEBUGREGS
811 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
812 #endif
813
814 #ifdef KVM_CAP_XSAVE
815 s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
816 #endif
817
818 #ifdef KVM_CAP_XCRS
819 s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
820 #endif
821
822 ret = kvm_arch_init(s);
823 if (ret < 0) {
824 goto err;
825 }
826
827 kvm_state = s;
828 memory_listener_register(&kvm_memory_listener);
829
830 s->many_ioeventfds = kvm_check_many_ioeventfds();
831
832 cpu_interrupt_handler = kvm_handle_interrupt;
833
834 return 0;
835
836 err:
837 if (s) {
838 if (s->vmfd >= 0) {
839 close(s->vmfd);
840 }
841 if (s->fd != -1) {
842 close(s->fd);
843 }
844 }
845 g_free(s);
846
847 return ret;
848 }
849
850 static void kvm_handle_io(uint16_t port, void *data, int direction, int size,
851 uint32_t count)
852 {
853 int i;
854 uint8_t *ptr = data;
855
856 for (i = 0; i < count; i++) {
857 if (direction == KVM_EXIT_IO_IN) {
858 switch (size) {
859 case 1:
860 stb_p(ptr, cpu_inb(port));
861 break;
862 case 2:
863 stw_p(ptr, cpu_inw(port));
864 break;
865 case 4:
866 stl_p(ptr, cpu_inl(port));
867 break;
868 }
869 } else {
870 switch (size) {
871 case 1:
872 cpu_outb(port, ldub_p(ptr));
873 break;
874 case 2:
875 cpu_outw(port, lduw_p(ptr));
876 break;
877 case 4:
878 cpu_outl(port, ldl_p(ptr));
879 break;
880 }
881 }
882
883 ptr += size;
884 }
885 }
886
887 static int kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
888 {
889 fprintf(stderr, "KVM internal error.");
890 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
891 int i;
892
893 fprintf(stderr, " Suberror: %d\n", run->internal.suberror);
894 for (i = 0; i < run->internal.ndata; ++i) {
895 fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
896 i, (uint64_t)run->internal.data[i]);
897 }
898 } else {
899 fprintf(stderr, "\n");
900 }
901 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
902 fprintf(stderr, "emulation failure\n");
903 if (!kvm_arch_stop_on_emulation_error(env)) {
904 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
905 return EXCP_INTERRUPT;
906 }
907 }
908 /* FIXME: Should trigger a qmp message to let management know
909 * something went wrong.
910 */
911 return -1;
912 }
913
914 void kvm_flush_coalesced_mmio_buffer(void)
915 {
916 KVMState *s = kvm_state;
917
918 if (s->coalesced_flush_in_progress) {
919 return;
920 }
921
922 s->coalesced_flush_in_progress = true;
923
924 if (s->coalesced_mmio_ring) {
925 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
926 while (ring->first != ring->last) {
927 struct kvm_coalesced_mmio *ent;
928
929 ent = &ring->coalesced_mmio[ring->first];
930
931 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
932 smp_wmb();
933 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
934 }
935 }
936
937 s->coalesced_flush_in_progress = false;
938 }
939
940 static void do_kvm_cpu_synchronize_state(void *_env)
941 {
942 CPUState *env = _env;
943
944 if (!env->kvm_vcpu_dirty) {
945 kvm_arch_get_registers(env);
946 env->kvm_vcpu_dirty = 1;
947 }
948 }
949
950 void kvm_cpu_synchronize_state(CPUState *env)
951 {
952 if (!env->kvm_vcpu_dirty) {
953 run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
954 }
955 }
956
957 void kvm_cpu_synchronize_post_reset(CPUState *env)
958 {
959 kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
960 env->kvm_vcpu_dirty = 0;
961 }
962
963 void kvm_cpu_synchronize_post_init(CPUState *env)
964 {
965 kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
966 env->kvm_vcpu_dirty = 0;
967 }
968
969 int kvm_cpu_exec(CPUState *env)
970 {
971 struct kvm_run *run = env->kvm_run;
972 int ret, run_ret;
973
974 DPRINTF("kvm_cpu_exec()\n");
975
976 if (kvm_arch_process_async_events(env)) {
977 env->exit_request = 0;
978 return EXCP_HLT;
979 }
980
981 cpu_single_env = env;
982
983 do {
984 if (env->kvm_vcpu_dirty) {
985 kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
986 env->kvm_vcpu_dirty = 0;
987 }
988
989 kvm_arch_pre_run(env, run);
990 if (env->exit_request) {
991 DPRINTF("interrupt exit requested\n");
992 /*
993 * KVM requires us to reenter the kernel after IO exits to complete
994 * instruction emulation. This self-signal will ensure that we
995 * leave ASAP again.
996 */
997 qemu_cpu_kick_self();
998 }
999 cpu_single_env = NULL;
1000 qemu_mutex_unlock_iothread();
1001
1002 run_ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
1003
1004 qemu_mutex_lock_iothread();
1005 cpu_single_env = env;
1006 kvm_arch_post_run(env, run);
1007
1008 kvm_flush_coalesced_mmio_buffer();
1009
1010 if (run_ret < 0) {
1011 if (run_ret == -EINTR || run_ret == -EAGAIN) {
1012 DPRINTF("io window exit\n");
1013 ret = EXCP_INTERRUPT;
1014 break;
1015 }
1016 fprintf(stderr, "error: kvm run failed %s\n",
1017 strerror(-run_ret));
1018 abort();
1019 }
1020
1021 switch (run->exit_reason) {
1022 case KVM_EXIT_IO:
1023 DPRINTF("handle_io\n");
1024 kvm_handle_io(run->io.port,
1025 (uint8_t *)run + run->io.data_offset,
1026 run->io.direction,
1027 run->io.size,
1028 run->io.count);
1029 ret = 0;
1030 break;
1031 case KVM_EXIT_MMIO:
1032 DPRINTF("handle_mmio\n");
1033 cpu_physical_memory_rw(run->mmio.phys_addr,
1034 run->mmio.data,
1035 run->mmio.len,
1036 run->mmio.is_write);
1037 ret = 0;
1038 break;
1039 case KVM_EXIT_IRQ_WINDOW_OPEN:
1040 DPRINTF("irq_window_open\n");
1041 ret = EXCP_INTERRUPT;
1042 break;
1043 case KVM_EXIT_SHUTDOWN:
1044 DPRINTF("shutdown\n");
1045 qemu_system_reset_request();
1046 ret = EXCP_INTERRUPT;
1047 break;
1048 case KVM_EXIT_UNKNOWN:
1049 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
1050 (uint64_t)run->hw.hardware_exit_reason);
1051 ret = -1;
1052 break;
1053 case KVM_EXIT_INTERNAL_ERROR:
1054 ret = kvm_handle_internal_error(env, run);
1055 break;
1056 default:
1057 DPRINTF("kvm_arch_handle_exit\n");
1058 ret = kvm_arch_handle_exit(env, run);
1059 break;
1060 }
1061 } while (ret == 0);
1062
1063 if (ret < 0) {
1064 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1065 vm_stop(RUN_STATE_INTERNAL_ERROR);
1066 }
1067
1068 env->exit_request = 0;
1069 cpu_single_env = NULL;
1070 return ret;
1071 }
1072
1073 int kvm_ioctl(KVMState *s, int type, ...)
1074 {
1075 int ret;
1076 void *arg;
1077 va_list ap;
1078
1079 va_start(ap, type);
1080 arg = va_arg(ap, void *);
1081 va_end(ap);
1082
1083 ret = ioctl(s->fd, type, arg);
1084 if (ret == -1) {
1085 ret = -errno;
1086 }
1087 return ret;
1088 }
1089
1090 int kvm_vm_ioctl(KVMState *s, int type, ...)
1091 {
1092 int ret;
1093 void *arg;
1094 va_list ap;
1095
1096 va_start(ap, type);
1097 arg = va_arg(ap, void *);
1098 va_end(ap);
1099
1100 ret = ioctl(s->vmfd, type, arg);
1101 if (ret == -1) {
1102 ret = -errno;
1103 }
1104 return ret;
1105 }
1106
1107 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
1108 {
1109 int ret;
1110 void *arg;
1111 va_list ap;
1112
1113 va_start(ap, type);
1114 arg = va_arg(ap, void *);
1115 va_end(ap);
1116
1117 ret = ioctl(env->kvm_fd, type, arg);
1118 if (ret == -1) {
1119 ret = -errno;
1120 }
1121 return ret;
1122 }
1123
1124 int kvm_has_sync_mmu(void)
1125 {
1126 return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
1127 }
1128
1129 int kvm_has_vcpu_events(void)
1130 {
1131 return kvm_state->vcpu_events;
1132 }
1133
1134 int kvm_has_robust_singlestep(void)
1135 {
1136 return kvm_state->robust_singlestep;
1137 }
1138
1139 int kvm_has_debugregs(void)
1140 {
1141 return kvm_state->debugregs;
1142 }
1143
1144 int kvm_has_xsave(void)
1145 {
1146 return kvm_state->xsave;
1147 }
1148
1149 int kvm_has_xcrs(void)
1150 {
1151 return kvm_state->xcrs;
1152 }
1153
1154 int kvm_has_many_ioeventfds(void)
1155 {
1156 if (!kvm_enabled()) {
1157 return 0;
1158 }
1159 return kvm_state->many_ioeventfds;
1160 }
1161
1162 void kvm_setup_guest_memory(void *start, size_t size)
1163 {
1164 if (!kvm_has_sync_mmu()) {
1165 int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
1166
1167 if (ret) {
1168 perror("qemu_madvise");
1169 fprintf(stderr,
1170 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1171 exit(1);
1172 }
1173 }
1174 }
1175
1176 #ifdef KVM_CAP_SET_GUEST_DEBUG
1177 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1178 target_ulong pc)
1179 {
1180 struct kvm_sw_breakpoint *bp;
1181
1182 QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1183 if (bp->pc == pc) {
1184 return bp;
1185 }
1186 }
1187 return NULL;
1188 }
1189
1190 int kvm_sw_breakpoints_active(CPUState *env)
1191 {
1192 return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1193 }
1194
1195 struct kvm_set_guest_debug_data {
1196 struct kvm_guest_debug dbg;
1197 CPUState *env;
1198 int err;
1199 };
1200
1201 static void kvm_invoke_set_guest_debug(void *data)
1202 {
1203 struct kvm_set_guest_debug_data *dbg_data = data;
1204 CPUState *env = dbg_data->env;
1205
1206 dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1207 }
1208
1209 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1210 {
1211 struct kvm_set_guest_debug_data data;
1212
1213 data.dbg.control = reinject_trap;
1214
1215 if (env->singlestep_enabled) {
1216 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1217 }
1218 kvm_arch_update_guest_debug(env, &data.dbg);
1219 data.env = env;
1220
1221 run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1222 return data.err;
1223 }
1224
1225 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1226 target_ulong len, int type)
1227 {
1228 struct kvm_sw_breakpoint *bp;
1229 CPUState *env;
1230 int err;
1231
1232 if (type == GDB_BREAKPOINT_SW) {
1233 bp = kvm_find_sw_breakpoint(current_env, addr);
1234 if (bp) {
1235 bp->use_count++;
1236 return 0;
1237 }
1238
1239 bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
1240 if (!bp) {
1241 return -ENOMEM;
1242 }
1243
1244 bp->pc = addr;
1245 bp->use_count = 1;
1246 err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1247 if (err) {
1248 g_free(bp);
1249 return err;
1250 }
1251
1252 QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1253 bp, entry);
1254 } else {
1255 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1256 if (err) {
1257 return err;
1258 }
1259 }
1260
1261 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1262 err = kvm_update_guest_debug(env, 0);
1263 if (err) {
1264 return err;
1265 }
1266 }
1267 return 0;
1268 }
1269
1270 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1271 target_ulong len, int type)
1272 {
1273 struct kvm_sw_breakpoint *bp;
1274 CPUState *env;
1275 int err;
1276
1277 if (type == GDB_BREAKPOINT_SW) {
1278 bp = kvm_find_sw_breakpoint(current_env, addr);
1279 if (!bp) {
1280 return -ENOENT;
1281 }
1282
1283 if (bp->use_count > 1) {
1284 bp->use_count--;
1285 return 0;
1286 }
1287
1288 err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1289 if (err) {
1290 return err;
1291 }
1292
1293 QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1294 g_free(bp);
1295 } else {
1296 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1297 if (err) {
1298 return err;
1299 }
1300 }
1301
1302 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1303 err = kvm_update_guest_debug(env, 0);
1304 if (err) {
1305 return err;
1306 }
1307 }
1308 return 0;
1309 }
1310
1311 void kvm_remove_all_breakpoints(CPUState *current_env)
1312 {
1313 struct kvm_sw_breakpoint *bp, *next;
1314 KVMState *s = current_env->kvm_state;
1315 CPUState *env;
1316
1317 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1318 if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1319 /* Try harder to find a CPU that currently sees the breakpoint. */
1320 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1321 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0) {
1322 break;
1323 }
1324 }
1325 }
1326 }
1327 kvm_arch_remove_all_hw_breakpoints();
1328
1329 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1330 kvm_update_guest_debug(env, 0);
1331 }
1332 }
1333
1334 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1335
1336 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1337 {
1338 return -EINVAL;
1339 }
1340
1341 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1342 target_ulong len, int type)
1343 {
1344 return -EINVAL;
1345 }
1346
1347 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1348 target_ulong len, int type)
1349 {
1350 return -EINVAL;
1351 }
1352
1353 void kvm_remove_all_breakpoints(CPUState *current_env)
1354 {
1355 }
1356 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1357
1358 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1359 {
1360 struct kvm_signal_mask *sigmask;
1361 int r;
1362
1363 if (!sigset) {
1364 return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1365 }
1366
1367 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
1368
1369 sigmask->len = 8;
1370 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1371 r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1372 g_free(sigmask);
1373
1374 return r;
1375 }
1376
1377 int kvm_set_ioeventfd_mmio_long(int fd, uint32_t addr, uint32_t val, bool assign)
1378 {
1379 int ret;
1380 struct kvm_ioeventfd iofd;
1381
1382 iofd.datamatch = val;
1383 iofd.addr = addr;
1384 iofd.len = 4;
1385 iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
1386 iofd.fd = fd;
1387
1388 if (!kvm_enabled()) {
1389 return -ENOSYS;
1390 }
1391
1392 if (!assign) {
1393 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1394 }
1395
1396 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1397
1398 if (ret < 0) {
1399 return -errno;
1400 }
1401
1402 return 0;
1403 }
1404
1405 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1406 {
1407 struct kvm_ioeventfd kick = {
1408 .datamatch = val,
1409 .addr = addr,
1410 .len = 2,
1411 .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1412 .fd = fd,
1413 };
1414 int r;
1415 if (!kvm_enabled()) {
1416 return -ENOSYS;
1417 }
1418 if (!assign) {
1419 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1420 }
1421 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1422 if (r < 0) {
1423 return r;
1424 }
1425 return 0;
1426 }
1427
1428 int kvm_on_sigbus_vcpu(CPUState *env, int code, void *addr)
1429 {
1430 return kvm_arch_on_sigbus_vcpu(env, code, addr);
1431 }
1432
1433 int kvm_on_sigbus(int code, void *addr)
1434 {
1435 return kvm_arch_on_sigbus(code, addr);
1436 }