migration: increase max-bandwidth to 128 MiB/s (1 Gib/s)
[qemu.git] / target / arm / kvm.c
1 /*
2 * ARM implementation of KVM hooks
3 *
4 * Copyright Christoffer Dall 2009-2010
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
8 *
9 */
10
11 #include "qemu/osdep.h"
12 #include <sys/ioctl.h>
13
14 #include <linux/kvm.h>
15
16 #include "qemu-common.h"
17 #include "qemu/timer.h"
18 #include "qemu/error-report.h"
19 #include "qemu/main-loop.h"
20 #include "qom/object.h"
21 #include "qapi/error.h"
22 #include "sysemu/sysemu.h"
23 #include "sysemu/kvm.h"
24 #include "sysemu/kvm_int.h"
25 #include "kvm_arm.h"
26 #include "cpu.h"
27 #include "trace.h"
28 #include "internals.h"
29 #include "hw/pci/pci.h"
30 #include "exec/memattrs.h"
31 #include "exec/address-spaces.h"
32 #include "hw/boards.h"
33 #include "hw/irq.h"
34 #include "qemu/log.h"
35
36 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
37 KVM_CAP_LAST_INFO
38 };
39
40 static bool cap_has_mp_state;
41 static bool cap_has_inject_serror_esr;
42 static bool cap_has_inject_ext_dabt;
43
44 static ARMHostCPUFeatures arm_host_cpu_features;
45
46 int kvm_arm_vcpu_init(CPUState *cs)
47 {
48 ARMCPU *cpu = ARM_CPU(cs);
49 struct kvm_vcpu_init init;
50
51 init.target = cpu->kvm_target;
52 memcpy(init.features, cpu->kvm_init_features, sizeof(init.features));
53
54 return kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_INIT, &init);
55 }
56
57 int kvm_arm_vcpu_finalize(CPUState *cs, int feature)
58 {
59 return kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_FINALIZE, &feature);
60 }
61
62 void kvm_arm_init_serror_injection(CPUState *cs)
63 {
64 cap_has_inject_serror_esr = kvm_check_extension(cs->kvm_state,
65 KVM_CAP_ARM_INJECT_SERROR_ESR);
66 }
67
68 bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
69 int *fdarray,
70 struct kvm_vcpu_init *init)
71 {
72 int ret = 0, kvmfd = -1, vmfd = -1, cpufd = -1;
73
74 kvmfd = qemu_open_old("/dev/kvm", O_RDWR);
75 if (kvmfd < 0) {
76 goto err;
77 }
78 vmfd = ioctl(kvmfd, KVM_CREATE_VM, 0);
79 if (vmfd < 0) {
80 goto err;
81 }
82 cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0);
83 if (cpufd < 0) {
84 goto err;
85 }
86
87 if (!init) {
88 /* Caller doesn't want the VCPU to be initialized, so skip it */
89 goto finish;
90 }
91
92 if (init->target == -1) {
93 struct kvm_vcpu_init preferred;
94
95 ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, &preferred);
96 if (!ret) {
97 init->target = preferred.target;
98 }
99 }
100 if (ret >= 0) {
101 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
102 if (ret < 0) {
103 goto err;
104 }
105 } else if (cpus_to_try) {
106 /* Old kernel which doesn't know about the
107 * PREFERRED_TARGET ioctl: we know it will only support
108 * creating one kind of guest CPU which is its preferred
109 * CPU type.
110 */
111 struct kvm_vcpu_init try;
112
113 while (*cpus_to_try != QEMU_KVM_ARM_TARGET_NONE) {
114 try.target = *cpus_to_try++;
115 memcpy(try.features, init->features, sizeof(init->features));
116 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, &try);
117 if (ret >= 0) {
118 break;
119 }
120 }
121 if (ret < 0) {
122 goto err;
123 }
124 init->target = try.target;
125 } else {
126 /* Treat a NULL cpus_to_try argument the same as an empty
127 * list, which means we will fail the call since this must
128 * be an old kernel which doesn't support PREFERRED_TARGET.
129 */
130 goto err;
131 }
132
133 finish:
134 fdarray[0] = kvmfd;
135 fdarray[1] = vmfd;
136 fdarray[2] = cpufd;
137
138 return true;
139
140 err:
141 if (cpufd >= 0) {
142 close(cpufd);
143 }
144 if (vmfd >= 0) {
145 close(vmfd);
146 }
147 if (kvmfd >= 0) {
148 close(kvmfd);
149 }
150
151 return false;
152 }
153
154 void kvm_arm_destroy_scratch_host_vcpu(int *fdarray)
155 {
156 int i;
157
158 for (i = 2; i >= 0; i--) {
159 close(fdarray[i]);
160 }
161 }
162
163 void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu)
164 {
165 CPUARMState *env = &cpu->env;
166
167 if (!arm_host_cpu_features.dtb_compatible) {
168 if (!kvm_enabled() ||
169 !kvm_arm_get_host_cpu_features(&arm_host_cpu_features)) {
170 /* We can't report this error yet, so flag that we need to
171 * in arm_cpu_realizefn().
172 */
173 cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE;
174 cpu->host_cpu_probe_failed = true;
175 return;
176 }
177 }
178
179 cpu->kvm_target = arm_host_cpu_features.target;
180 cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible;
181 cpu->isar = arm_host_cpu_features.isar;
182 env->features = arm_host_cpu_features.features;
183 }
184
185 static bool kvm_no_adjvtime_get(Object *obj, Error **errp)
186 {
187 return !ARM_CPU(obj)->kvm_adjvtime;
188 }
189
190 static void kvm_no_adjvtime_set(Object *obj, bool value, Error **errp)
191 {
192 ARM_CPU(obj)->kvm_adjvtime = !value;
193 }
194
195 /* KVM VCPU properties should be prefixed with "kvm-". */
196 void kvm_arm_add_vcpu_properties(Object *obj)
197 {
198 ARMCPU *cpu = ARM_CPU(obj);
199 CPUARMState *env = &cpu->env;
200
201 if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
202 cpu->kvm_adjvtime = true;
203 object_property_add_bool(obj, "kvm-no-adjvtime", kvm_no_adjvtime_get,
204 kvm_no_adjvtime_set);
205 object_property_set_description(obj, "kvm-no-adjvtime",
206 "Set on to disable the adjustment of "
207 "the virtual counter. VM stopped time "
208 "will be counted.");
209 }
210 }
211
212 bool kvm_arm_pmu_supported(void)
213 {
214 return kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3);
215 }
216
217 int kvm_arm_get_max_vm_ipa_size(MachineState *ms)
218 {
219 KVMState *s = KVM_STATE(ms->accelerator);
220 int ret;
221
222 ret = kvm_check_extension(s, KVM_CAP_ARM_VM_IPA_SIZE);
223 return ret > 0 ? ret : 40;
224 }
225
226 int kvm_arch_init(MachineState *ms, KVMState *s)
227 {
228 int ret = 0;
229 /* For ARM interrupt delivery is always asynchronous,
230 * whether we are using an in-kernel VGIC or not.
231 */
232 kvm_async_interrupts_allowed = true;
233
234 /*
235 * PSCI wakes up secondary cores, so we always need to
236 * have vCPUs waiting in kernel space
237 */
238 kvm_halt_in_kernel_allowed = true;
239
240 cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
241
242 if (ms->smp.cpus > 256 &&
243 !kvm_check_extension(s, KVM_CAP_ARM_IRQ_LINE_LAYOUT_2)) {
244 error_report("Using more than 256 vcpus requires a host kernel "
245 "with KVM_CAP_ARM_IRQ_LINE_LAYOUT_2");
246 ret = -EINVAL;
247 }
248
249 if (kvm_check_extension(s, KVM_CAP_ARM_NISV_TO_USER)) {
250 if (kvm_vm_enable_cap(s, KVM_CAP_ARM_NISV_TO_USER, 0)) {
251 error_report("Failed to enable KVM_CAP_ARM_NISV_TO_USER cap");
252 } else {
253 /* Set status for supporting the external dabt injection */
254 cap_has_inject_ext_dabt = kvm_check_extension(s,
255 KVM_CAP_ARM_INJECT_EXT_DABT);
256 }
257 }
258
259 return ret;
260 }
261
262 unsigned long kvm_arch_vcpu_id(CPUState *cpu)
263 {
264 return cpu->cpu_index;
265 }
266
267 /* We track all the KVM devices which need their memory addresses
268 * passing to the kernel in a list of these structures.
269 * When board init is complete we run through the list and
270 * tell the kernel the base addresses of the memory regions.
271 * We use a MemoryListener to track mapping and unmapping of
272 * the regions during board creation, so the board models don't
273 * need to do anything special for the KVM case.
274 *
275 * Sometimes the address must be OR'ed with some other fields
276 * (for example for KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION).
277 * @kda_addr_ormask aims at storing the value of those fields.
278 */
279 typedef struct KVMDevice {
280 struct kvm_arm_device_addr kda;
281 struct kvm_device_attr kdattr;
282 uint64_t kda_addr_ormask;
283 MemoryRegion *mr;
284 QSLIST_ENTRY(KVMDevice) entries;
285 int dev_fd;
286 } KVMDevice;
287
288 static QSLIST_HEAD(, KVMDevice) kvm_devices_head;
289
290 static void kvm_arm_devlistener_add(MemoryListener *listener,
291 MemoryRegionSection *section)
292 {
293 KVMDevice *kd;
294
295 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
296 if (section->mr == kd->mr) {
297 kd->kda.addr = section->offset_within_address_space;
298 }
299 }
300 }
301
302 static void kvm_arm_devlistener_del(MemoryListener *listener,
303 MemoryRegionSection *section)
304 {
305 KVMDevice *kd;
306
307 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
308 if (section->mr == kd->mr) {
309 kd->kda.addr = -1;
310 }
311 }
312 }
313
314 static MemoryListener devlistener = {
315 .region_add = kvm_arm_devlistener_add,
316 .region_del = kvm_arm_devlistener_del,
317 };
318
319 static void kvm_arm_set_device_addr(KVMDevice *kd)
320 {
321 struct kvm_device_attr *attr = &kd->kdattr;
322 int ret;
323
324 /* If the device control API is available and we have a device fd on the
325 * KVMDevice struct, let's use the newer API
326 */
327 if (kd->dev_fd >= 0) {
328 uint64_t addr = kd->kda.addr;
329
330 addr |= kd->kda_addr_ormask;
331 attr->addr = (uintptr_t)&addr;
332 ret = kvm_device_ioctl(kd->dev_fd, KVM_SET_DEVICE_ATTR, attr);
333 } else {
334 ret = kvm_vm_ioctl(kvm_state, KVM_ARM_SET_DEVICE_ADDR, &kd->kda);
335 }
336
337 if (ret < 0) {
338 fprintf(stderr, "Failed to set device address: %s\n",
339 strerror(-ret));
340 abort();
341 }
342 }
343
344 static void kvm_arm_machine_init_done(Notifier *notifier, void *data)
345 {
346 KVMDevice *kd, *tkd;
347
348 QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) {
349 if (kd->kda.addr != -1) {
350 kvm_arm_set_device_addr(kd);
351 }
352 memory_region_unref(kd->mr);
353 QSLIST_REMOVE_HEAD(&kvm_devices_head, entries);
354 g_free(kd);
355 }
356 memory_listener_unregister(&devlistener);
357 }
358
359 static Notifier notify = {
360 .notify = kvm_arm_machine_init_done,
361 };
362
363 void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid, uint64_t group,
364 uint64_t attr, int dev_fd, uint64_t addr_ormask)
365 {
366 KVMDevice *kd;
367
368 if (!kvm_irqchip_in_kernel()) {
369 return;
370 }
371
372 if (QSLIST_EMPTY(&kvm_devices_head)) {
373 memory_listener_register(&devlistener, &address_space_memory);
374 qemu_add_machine_init_done_notifier(&notify);
375 }
376 kd = g_new0(KVMDevice, 1);
377 kd->mr = mr;
378 kd->kda.id = devid;
379 kd->kda.addr = -1;
380 kd->kdattr.flags = 0;
381 kd->kdattr.group = group;
382 kd->kdattr.attr = attr;
383 kd->dev_fd = dev_fd;
384 kd->kda_addr_ormask = addr_ormask;
385 QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries);
386 memory_region_ref(kd->mr);
387 }
388
389 static int compare_u64(const void *a, const void *b)
390 {
391 if (*(uint64_t *)a > *(uint64_t *)b) {
392 return 1;
393 }
394 if (*(uint64_t *)a < *(uint64_t *)b) {
395 return -1;
396 }
397 return 0;
398 }
399
400 /*
401 * cpreg_values are sorted in ascending order by KVM register ID
402 * (see kvm_arm_init_cpreg_list). This allows us to cheaply find
403 * the storage for a KVM register by ID with a binary search.
404 */
405 static uint64_t *kvm_arm_get_cpreg_ptr(ARMCPU *cpu, uint64_t regidx)
406 {
407 uint64_t *res;
408
409 res = bsearch(&regidx, cpu->cpreg_indexes, cpu->cpreg_array_len,
410 sizeof(uint64_t), compare_u64);
411 assert(res);
412
413 return &cpu->cpreg_values[res - cpu->cpreg_indexes];
414 }
415
416 /* Initialize the ARMCPU cpreg list according to the kernel's
417 * definition of what CPU registers it knows about (and throw away
418 * the previous TCG-created cpreg list).
419 */
420 int kvm_arm_init_cpreg_list(ARMCPU *cpu)
421 {
422 struct kvm_reg_list rl;
423 struct kvm_reg_list *rlp;
424 int i, ret, arraylen;
425 CPUState *cs = CPU(cpu);
426
427 rl.n = 0;
428 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
429 if (ret != -E2BIG) {
430 return ret;
431 }
432 rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
433 rlp->n = rl.n;
434 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
435 if (ret) {
436 goto out;
437 }
438 /* Sort the list we get back from the kernel, since cpreg_tuples
439 * must be in strictly ascending order.
440 */
441 qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
442
443 for (i = 0, arraylen = 0; i < rlp->n; i++) {
444 if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
445 continue;
446 }
447 switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
448 case KVM_REG_SIZE_U32:
449 case KVM_REG_SIZE_U64:
450 break;
451 default:
452 fprintf(stderr, "Can't handle size of register in kernel list\n");
453 ret = -EINVAL;
454 goto out;
455 }
456
457 arraylen++;
458 }
459
460 cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
461 cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
462 cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
463 arraylen);
464 cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values,
465 arraylen);
466 cpu->cpreg_array_len = arraylen;
467 cpu->cpreg_vmstate_array_len = arraylen;
468
469 for (i = 0, arraylen = 0; i < rlp->n; i++) {
470 uint64_t regidx = rlp->reg[i];
471 if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
472 continue;
473 }
474 cpu->cpreg_indexes[arraylen] = regidx;
475 arraylen++;
476 }
477 assert(cpu->cpreg_array_len == arraylen);
478
479 if (!write_kvmstate_to_list(cpu)) {
480 /* Shouldn't happen unless kernel is inconsistent about
481 * what registers exist.
482 */
483 fprintf(stderr, "Initial read of kernel register state failed\n");
484 ret = -EINVAL;
485 goto out;
486 }
487
488 out:
489 g_free(rlp);
490 return ret;
491 }
492
493 bool write_kvmstate_to_list(ARMCPU *cpu)
494 {
495 CPUState *cs = CPU(cpu);
496 int i;
497 bool ok = true;
498
499 for (i = 0; i < cpu->cpreg_array_len; i++) {
500 struct kvm_one_reg r;
501 uint64_t regidx = cpu->cpreg_indexes[i];
502 uint32_t v32;
503 int ret;
504
505 r.id = regidx;
506
507 switch (regidx & KVM_REG_SIZE_MASK) {
508 case KVM_REG_SIZE_U32:
509 r.addr = (uintptr_t)&v32;
510 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
511 if (!ret) {
512 cpu->cpreg_values[i] = v32;
513 }
514 break;
515 case KVM_REG_SIZE_U64:
516 r.addr = (uintptr_t)(cpu->cpreg_values + i);
517 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
518 break;
519 default:
520 abort();
521 }
522 if (ret) {
523 ok = false;
524 }
525 }
526 return ok;
527 }
528
529 bool write_list_to_kvmstate(ARMCPU *cpu, int level)
530 {
531 CPUState *cs = CPU(cpu);
532 int i;
533 bool ok = true;
534
535 for (i = 0; i < cpu->cpreg_array_len; i++) {
536 struct kvm_one_reg r;
537 uint64_t regidx = cpu->cpreg_indexes[i];
538 uint32_t v32;
539 int ret;
540
541 if (kvm_arm_cpreg_level(regidx) > level) {
542 continue;
543 }
544
545 r.id = regidx;
546 switch (regidx & KVM_REG_SIZE_MASK) {
547 case KVM_REG_SIZE_U32:
548 v32 = cpu->cpreg_values[i];
549 r.addr = (uintptr_t)&v32;
550 break;
551 case KVM_REG_SIZE_U64:
552 r.addr = (uintptr_t)(cpu->cpreg_values + i);
553 break;
554 default:
555 abort();
556 }
557 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
558 if (ret) {
559 /* We might fail for "unknown register" and also for
560 * "you tried to set a register which is constant with
561 * a different value from what it actually contains".
562 */
563 ok = false;
564 }
565 }
566 return ok;
567 }
568
569 void kvm_arm_cpu_pre_save(ARMCPU *cpu)
570 {
571 /* KVM virtual time adjustment */
572 if (cpu->kvm_vtime_dirty) {
573 *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT) = cpu->kvm_vtime;
574 }
575 }
576
577 void kvm_arm_cpu_post_load(ARMCPU *cpu)
578 {
579 /* KVM virtual time adjustment */
580 if (cpu->kvm_adjvtime) {
581 cpu->kvm_vtime = *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT);
582 cpu->kvm_vtime_dirty = true;
583 }
584 }
585
586 void kvm_arm_reset_vcpu(ARMCPU *cpu)
587 {
588 int ret;
589
590 /* Re-init VCPU so that all registers are set to
591 * their respective reset values.
592 */
593 ret = kvm_arm_vcpu_init(CPU(cpu));
594 if (ret < 0) {
595 fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret));
596 abort();
597 }
598 if (!write_kvmstate_to_list(cpu)) {
599 fprintf(stderr, "write_kvmstate_to_list failed\n");
600 abort();
601 }
602 /*
603 * Sync the reset values also into the CPUState. This is necessary
604 * because the next thing we do will be a kvm_arch_put_registers()
605 * which will update the list values from the CPUState before copying
606 * the list values back to KVM. It's OK to ignore failure returns here
607 * for the same reason we do so in kvm_arch_get_registers().
608 */
609 write_list_to_cpustate(cpu);
610 }
611
612 /*
613 * Update KVM's MP_STATE based on what QEMU thinks it is
614 */
615 int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu)
616 {
617 if (cap_has_mp_state) {
618 struct kvm_mp_state mp_state = {
619 .mp_state = (cpu->power_state == PSCI_OFF) ?
620 KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE
621 };
622 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
623 if (ret) {
624 fprintf(stderr, "%s: failed to set MP_STATE %d/%s\n",
625 __func__, ret, strerror(-ret));
626 return -1;
627 }
628 }
629
630 return 0;
631 }
632
633 /*
634 * Sync the KVM MP_STATE into QEMU
635 */
636 int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu)
637 {
638 if (cap_has_mp_state) {
639 struct kvm_mp_state mp_state;
640 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
641 if (ret) {
642 fprintf(stderr, "%s: failed to get MP_STATE %d/%s\n",
643 __func__, ret, strerror(-ret));
644 abort();
645 }
646 cpu->power_state = (mp_state.mp_state == KVM_MP_STATE_STOPPED) ?
647 PSCI_OFF : PSCI_ON;
648 }
649
650 return 0;
651 }
652
653 void kvm_arm_get_virtual_time(CPUState *cs)
654 {
655 ARMCPU *cpu = ARM_CPU(cs);
656 struct kvm_one_reg reg = {
657 .id = KVM_REG_ARM_TIMER_CNT,
658 .addr = (uintptr_t)&cpu->kvm_vtime,
659 };
660 int ret;
661
662 if (cpu->kvm_vtime_dirty) {
663 return;
664 }
665
666 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
667 if (ret) {
668 error_report("Failed to get KVM_REG_ARM_TIMER_CNT");
669 abort();
670 }
671
672 cpu->kvm_vtime_dirty = true;
673 }
674
675 void kvm_arm_put_virtual_time(CPUState *cs)
676 {
677 ARMCPU *cpu = ARM_CPU(cs);
678 struct kvm_one_reg reg = {
679 .id = KVM_REG_ARM_TIMER_CNT,
680 .addr = (uintptr_t)&cpu->kvm_vtime,
681 };
682 int ret;
683
684 if (!cpu->kvm_vtime_dirty) {
685 return;
686 }
687
688 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
689 if (ret) {
690 error_report("Failed to set KVM_REG_ARM_TIMER_CNT");
691 abort();
692 }
693
694 cpu->kvm_vtime_dirty = false;
695 }
696
697 int kvm_put_vcpu_events(ARMCPU *cpu)
698 {
699 CPUARMState *env = &cpu->env;
700 struct kvm_vcpu_events events;
701 int ret;
702
703 if (!kvm_has_vcpu_events()) {
704 return 0;
705 }
706
707 memset(&events, 0, sizeof(events));
708 events.exception.serror_pending = env->serror.pending;
709
710 /* Inject SError to guest with specified syndrome if host kernel
711 * supports it, otherwise inject SError without syndrome.
712 */
713 if (cap_has_inject_serror_esr) {
714 events.exception.serror_has_esr = env->serror.has_esr;
715 events.exception.serror_esr = env->serror.esr;
716 }
717
718 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
719 if (ret) {
720 error_report("failed to put vcpu events");
721 }
722
723 return ret;
724 }
725
726 int kvm_get_vcpu_events(ARMCPU *cpu)
727 {
728 CPUARMState *env = &cpu->env;
729 struct kvm_vcpu_events events;
730 int ret;
731
732 if (!kvm_has_vcpu_events()) {
733 return 0;
734 }
735
736 memset(&events, 0, sizeof(events));
737 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
738 if (ret) {
739 error_report("failed to get vcpu events");
740 return ret;
741 }
742
743 env->serror.pending = events.exception.serror_pending;
744 env->serror.has_esr = events.exception.serror_has_esr;
745 env->serror.esr = events.exception.serror_esr;
746
747 return 0;
748 }
749
750 void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
751 {
752 ARMCPU *cpu = ARM_CPU(cs);
753 CPUARMState *env = &cpu->env;
754
755 if (unlikely(env->ext_dabt_raised)) {
756 /*
757 * Verifying that the ext DABT has been properly injected,
758 * otherwise risking indefinitely re-running the faulting instruction
759 * Covering a very narrow case for kernels 5.5..5.5.4
760 * when injected abort was misconfigured to be
761 * an IMPLEMENTATION DEFINED exception (for 32-bit EL1)
762 */
763 if (!arm_feature(env, ARM_FEATURE_AARCH64) &&
764 unlikely(!kvm_arm_verify_ext_dabt_pending(cs))) {
765
766 error_report("Data abort exception with no valid ISS generated by "
767 "guest memory access. KVM unable to emulate faulting "
768 "instruction. Failed to inject an external data abort "
769 "into the guest.");
770 abort();
771 }
772 /* Clear the status */
773 env->ext_dabt_raised = 0;
774 }
775 }
776
777 MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
778 {
779 ARMCPU *cpu;
780 uint32_t switched_level;
781
782 if (kvm_irqchip_in_kernel()) {
783 /*
784 * We only need to sync timer states with user-space interrupt
785 * controllers, so return early and save cycles if we don't.
786 */
787 return MEMTXATTRS_UNSPECIFIED;
788 }
789
790 cpu = ARM_CPU(cs);
791
792 /* Synchronize our shadowed in-kernel device irq lines with the kvm ones */
793 if (run->s.regs.device_irq_level != cpu->device_irq_level) {
794 switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
795
796 qemu_mutex_lock_iothread();
797
798 if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
799 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
800 !!(run->s.regs.device_irq_level &
801 KVM_ARM_DEV_EL1_VTIMER));
802 switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
803 }
804
805 if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
806 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
807 !!(run->s.regs.device_irq_level &
808 KVM_ARM_DEV_EL1_PTIMER));
809 switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
810 }
811
812 if (switched_level & KVM_ARM_DEV_PMU) {
813 qemu_set_irq(cpu->pmu_interrupt,
814 !!(run->s.regs.device_irq_level & KVM_ARM_DEV_PMU));
815 switched_level &= ~KVM_ARM_DEV_PMU;
816 }
817
818 if (switched_level) {
819 qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n",
820 __func__, switched_level);
821 }
822
823 /* We also mark unknown levels as processed to not waste cycles */
824 cpu->device_irq_level = run->s.regs.device_irq_level;
825 qemu_mutex_unlock_iothread();
826 }
827
828 return MEMTXATTRS_UNSPECIFIED;
829 }
830
831 void kvm_arm_vm_state_change(void *opaque, int running, RunState state)
832 {
833 CPUState *cs = opaque;
834 ARMCPU *cpu = ARM_CPU(cs);
835
836 if (running) {
837 if (cpu->kvm_adjvtime) {
838 kvm_arm_put_virtual_time(cs);
839 }
840 } else {
841 if (cpu->kvm_adjvtime) {
842 kvm_arm_get_virtual_time(cs);
843 }
844 }
845 }
846
847 /**
848 * kvm_arm_handle_dabt_nisv:
849 * @cs: CPUState
850 * @esr_iss: ISS encoding (limited) for the exception from Data Abort
851 * ISV bit set to '0b0' -> no valid instruction syndrome
852 * @fault_ipa: faulting address for the synchronous data abort
853 *
854 * Returns: 0 if the exception has been handled, < 0 otherwise
855 */
856 static int kvm_arm_handle_dabt_nisv(CPUState *cs, uint64_t esr_iss,
857 uint64_t fault_ipa)
858 {
859 ARMCPU *cpu = ARM_CPU(cs);
860 CPUARMState *env = &cpu->env;
861 /*
862 * Request KVM to inject the external data abort into the guest
863 */
864 if (cap_has_inject_ext_dabt) {
865 struct kvm_vcpu_events events = { };
866 /*
867 * The external data abort event will be handled immediately by KVM
868 * using the address fault that triggered the exit on given VCPU.
869 * Requesting injection of the external data abort does not rely
870 * on any other VCPU state. Therefore, in this particular case, the VCPU
871 * synchronization can be exceptionally skipped.
872 */
873 events.exception.ext_dabt_pending = 1;
874 /* KVM_CAP_ARM_INJECT_EXT_DABT implies KVM_CAP_VCPU_EVENTS */
875 if (!kvm_vcpu_ioctl(cs, KVM_SET_VCPU_EVENTS, &events)) {
876 env->ext_dabt_raised = 1;
877 return 0;
878 }
879 } else {
880 error_report("Data abort exception triggered by guest memory access "
881 "at physical address: 0x" TARGET_FMT_lx,
882 (target_ulong)fault_ipa);
883 error_printf("KVM unable to emulate faulting instruction.\n");
884 }
885 return -1;
886 }
887
888 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
889 {
890 int ret = 0;
891
892 switch (run->exit_reason) {
893 case KVM_EXIT_DEBUG:
894 if (kvm_arm_handle_debug(cs, &run->debug.arch)) {
895 ret = EXCP_DEBUG;
896 } /* otherwise return to guest */
897 break;
898 case KVM_EXIT_ARM_NISV:
899 /* External DABT with no valid iss to decode */
900 ret = kvm_arm_handle_dabt_nisv(cs, run->arm_nisv.esr_iss,
901 run->arm_nisv.fault_ipa);
902 break;
903 default:
904 qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
905 __func__, run->exit_reason);
906 break;
907 }
908 return ret;
909 }
910
911 bool kvm_arch_stop_on_emulation_error(CPUState *cs)
912 {
913 return true;
914 }
915
916 int kvm_arch_process_async_events(CPUState *cs)
917 {
918 return 0;
919 }
920
921 void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
922 {
923 if (kvm_sw_breakpoints_active(cs)) {
924 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
925 }
926 if (kvm_arm_hw_debug_active(cs)) {
927 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW;
928 kvm_arm_copy_hw_debug_data(&dbg->arch);
929 }
930 }
931
932 void kvm_arch_init_irq_routing(KVMState *s)
933 {
934 }
935
936 int kvm_arch_irqchip_create(KVMState *s)
937 {
938 if (kvm_kernel_irqchip_split()) {
939 perror("-machine kernel_irqchip=split is not supported on ARM.");
940 exit(1);
941 }
942
943 /* If we can create the VGIC using the newer device control API, we
944 * let the device do this when it initializes itself, otherwise we
945 * fall back to the old API */
946 return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
947 }
948
949 int kvm_arm_vgic_probe(void)
950 {
951 int val = 0;
952
953 if (kvm_create_device(kvm_state,
954 KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
955 val |= KVM_ARM_VGIC_V3;
956 }
957 if (kvm_create_device(kvm_state,
958 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
959 val |= KVM_ARM_VGIC_V2;
960 }
961 return val;
962 }
963
964 int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level)
965 {
966 int kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq;
967 int cpu_idx1 = cpu % 256;
968 int cpu_idx2 = cpu / 256;
969
970 kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) |
971 (cpu_idx2 << KVM_ARM_IRQ_VCPU2_SHIFT);
972
973 return kvm_set_irq(kvm_state, kvm_irq, !!level);
974 }
975
976 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
977 uint64_t address, uint32_t data, PCIDevice *dev)
978 {
979 AddressSpace *as = pci_device_iommu_address_space(dev);
980 hwaddr xlat, len, doorbell_gpa;
981 MemoryRegionSection mrs;
982 MemoryRegion *mr;
983 int ret = 1;
984
985 if (as == &address_space_memory) {
986 return 0;
987 }
988
989 /* MSI doorbell address is translated by an IOMMU */
990
991 rcu_read_lock();
992 mr = address_space_translate(as, address, &xlat, &len, true,
993 MEMTXATTRS_UNSPECIFIED);
994 if (!mr) {
995 goto unlock;
996 }
997 mrs = memory_region_find(mr, xlat, 1);
998 if (!mrs.mr) {
999 goto unlock;
1000 }
1001
1002 doorbell_gpa = mrs.offset_within_address_space;
1003 memory_region_unref(mrs.mr);
1004
1005 route->u.msi.address_lo = doorbell_gpa;
1006 route->u.msi.address_hi = doorbell_gpa >> 32;
1007
1008 trace_kvm_arm_fixup_msi_route(address, doorbell_gpa);
1009
1010 ret = 0;
1011
1012 unlock:
1013 rcu_read_unlock();
1014 return ret;
1015 }
1016
1017 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
1018 int vector, PCIDevice *dev)
1019 {
1020 return 0;
1021 }
1022
1023 int kvm_arch_release_virq_post(int virq)
1024 {
1025 return 0;
1026 }
1027
1028 int kvm_arch_msi_data_to_gsi(uint32_t data)
1029 {
1030 return (data - 32) & 0xffff;
1031 }