migration: increase max-bandwidth to 128 MiB/s (1 Gib/s)
[qemu.git] / target / arm / kvm64.c
1 /*
2 * ARM implementation of KVM hooks, 64 bit specific code
3 *
4 * Copyright Mian-M. Hamayun 2013, Virtual Open Systems
5 * Copyright Alex Bennée 2014, Linaro
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
9 *
10 */
11
12 #include "qemu/osdep.h"
13 #include <sys/ioctl.h>
14 #include <sys/ptrace.h>
15
16 #include <linux/elf.h>
17 #include <linux/kvm.h>
18
19 #include "qemu-common.h"
20 #include "cpu.h"
21 #include "qemu/timer.h"
22 #include "qemu/error-report.h"
23 #include "qemu/host-utils.h"
24 #include "qemu/main-loop.h"
25 #include "exec/gdbstub.h"
26 #include "sysemu/runstate.h"
27 #include "sysemu/kvm.h"
28 #include "sysemu/kvm_int.h"
29 #include "kvm_arm.h"
30 #include "internals.h"
31 #include "hw/acpi/acpi.h"
32 #include "hw/acpi/ghes.h"
33 #include "hw/arm/virt.h"
34
35 static bool have_guest_debug;
36
37 /*
38 * Although the ARM implementation of hardware assisted debugging
39 * allows for different breakpoints per-core, the current GDB
40 * interface treats them as a global pool of registers (which seems to
41 * be the case for x86, ppc and s390). As a result we store one copy
42 * of registers which is used for all active cores.
43 *
44 * Write access is serialised by virtue of the GDB protocol which
45 * updates things. Read access (i.e. when the values are copied to the
46 * vCPU) is also gated by GDB's run control.
47 *
48 * This is not unreasonable as most of the time debugging kernels you
49 * never know which core will eventually execute your function.
50 */
51
52 typedef struct {
53 uint64_t bcr;
54 uint64_t bvr;
55 } HWBreakpoint;
56
57 /* The watchpoint registers can cover more area than the requested
58 * watchpoint so we need to store the additional information
59 * somewhere. We also need to supply a CPUWatchpoint to the GDB stub
60 * when the watchpoint is hit.
61 */
62 typedef struct {
63 uint64_t wcr;
64 uint64_t wvr;
65 CPUWatchpoint details;
66 } HWWatchpoint;
67
68 /* Maximum and current break/watch point counts */
69 int max_hw_bps, max_hw_wps;
70 GArray *hw_breakpoints, *hw_watchpoints;
71
72 #define cur_hw_wps (hw_watchpoints->len)
73 #define cur_hw_bps (hw_breakpoints->len)
74 #define get_hw_bp(i) (&g_array_index(hw_breakpoints, HWBreakpoint, i))
75 #define get_hw_wp(i) (&g_array_index(hw_watchpoints, HWWatchpoint, i))
76
77 /**
78 * kvm_arm_init_debug() - check for guest debug capabilities
79 * @cs: CPUState
80 *
81 * kvm_check_extension returns the number of debug registers we have
82 * or 0 if we have none.
83 *
84 */
85 static void kvm_arm_init_debug(CPUState *cs)
86 {
87 have_guest_debug = kvm_check_extension(cs->kvm_state,
88 KVM_CAP_SET_GUEST_DEBUG);
89
90 max_hw_wps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_WPS);
91 hw_watchpoints = g_array_sized_new(true, true,
92 sizeof(HWWatchpoint), max_hw_wps);
93
94 max_hw_bps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_BPS);
95 hw_breakpoints = g_array_sized_new(true, true,
96 sizeof(HWBreakpoint), max_hw_bps);
97 return;
98 }
99
100 /**
101 * insert_hw_breakpoint()
102 * @addr: address of breakpoint
103 *
104 * See ARM ARM D2.9.1 for details but here we are only going to create
105 * simple un-linked breakpoints (i.e. we don't chain breakpoints
106 * together to match address and context or vmid). The hardware is
107 * capable of fancier matching but that will require exposing that
108 * fanciness to GDB's interface
109 *
110 * DBGBCR<n>_EL1, Debug Breakpoint Control Registers
111 *
112 * 31 24 23 20 19 16 15 14 13 12 9 8 5 4 3 2 1 0
113 * +------+------+-------+-----+----+------+-----+------+-----+---+
114 * | RES0 | BT | LBN | SSC | HMC| RES0 | BAS | RES0 | PMC | E |
115 * +------+------+-------+-----+----+------+-----+------+-----+---+
116 *
117 * BT: Breakpoint type (0 = unlinked address match)
118 * LBN: Linked BP number (0 = unused)
119 * SSC/HMC/PMC: Security, Higher and Priv access control (Table D-12)
120 * BAS: Byte Address Select (RES1 for AArch64)
121 * E: Enable bit
122 *
123 * DBGBVR<n>_EL1, Debug Breakpoint Value Registers
124 *
125 * 63 53 52 49 48 2 1 0
126 * +------+-----------+----------+-----+
127 * | RESS | VA[52:49] | VA[48:2] | 0 0 |
128 * +------+-----------+----------+-----+
129 *
130 * Depending on the addressing mode bits the top bits of the register
131 * are a sign extension of the highest applicable VA bit. Some
132 * versions of GDB don't do it correctly so we ensure they are correct
133 * here so future PC comparisons will work properly.
134 */
135
136 static int insert_hw_breakpoint(target_ulong addr)
137 {
138 HWBreakpoint brk = {
139 .bcr = 0x1, /* BCR E=1, enable */
140 .bvr = sextract64(addr, 0, 53)
141 };
142
143 if (cur_hw_bps >= max_hw_bps) {
144 return -ENOBUFS;
145 }
146
147 brk.bcr = deposit32(brk.bcr, 1, 2, 0x3); /* PMC = 11 */
148 brk.bcr = deposit32(brk.bcr, 5, 4, 0xf); /* BAS = RES1 */
149
150 g_array_append_val(hw_breakpoints, brk);
151
152 return 0;
153 }
154
155 /**
156 * delete_hw_breakpoint()
157 * @pc: address of breakpoint
158 *
159 * Delete a breakpoint and shuffle any above down
160 */
161
162 static int delete_hw_breakpoint(target_ulong pc)
163 {
164 int i;
165 for (i = 0; i < hw_breakpoints->len; i++) {
166 HWBreakpoint *brk = get_hw_bp(i);
167 if (brk->bvr == pc) {
168 g_array_remove_index(hw_breakpoints, i);
169 return 0;
170 }
171 }
172 return -ENOENT;
173 }
174
175 /**
176 * insert_hw_watchpoint()
177 * @addr: address of watch point
178 * @len: size of area
179 * @type: type of watch point
180 *
181 * See ARM ARM D2.10. As with the breakpoints we can do some advanced
182 * stuff if we want to. The watch points can be linked with the break
183 * points above to make them context aware. However for simplicity
184 * currently we only deal with simple read/write watch points.
185 *
186 * D7.3.11 DBGWCR<n>_EL1, Debug Watchpoint Control Registers
187 *
188 * 31 29 28 24 23 21 20 19 16 15 14 13 12 5 4 3 2 1 0
189 * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+
190 * | RES0 | MASK | RES0 | WT | LBN | SSC | HMC | BAS | LSC | PAC | E |
191 * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+
192 *
193 * MASK: num bits addr mask (0=none,01/10=res,11=3 bits (8 bytes))
194 * WT: 0 - unlinked, 1 - linked (not currently used)
195 * LBN: Linked BP number (not currently used)
196 * SSC/HMC/PAC: Security, Higher and Priv access control (Table D2-11)
197 * BAS: Byte Address Select
198 * LSC: Load/Store control (01: load, 10: store, 11: both)
199 * E: Enable
200 *
201 * The bottom 2 bits of the value register are masked. Therefore to
202 * break on any sizes smaller than an unaligned word you need to set
203 * MASK=0, BAS=bit per byte in question. For larger regions (^2) you
204 * need to ensure you mask the address as required and set BAS=0xff
205 */
206
207 static int insert_hw_watchpoint(target_ulong addr,
208 target_ulong len, int type)
209 {
210 HWWatchpoint wp = {
211 .wcr = 1, /* E=1, enable */
212 .wvr = addr & (~0x7ULL),
213 .details = { .vaddr = addr, .len = len }
214 };
215
216 if (cur_hw_wps >= max_hw_wps) {
217 return -ENOBUFS;
218 }
219
220 /*
221 * HMC=0 SSC=0 PAC=3 will hit EL0 or EL1, any security state,
222 * valid whether EL3 is implemented or not
223 */
224 wp.wcr = deposit32(wp.wcr, 1, 2, 3);
225
226 switch (type) {
227 case GDB_WATCHPOINT_READ:
228 wp.wcr = deposit32(wp.wcr, 3, 2, 1);
229 wp.details.flags = BP_MEM_READ;
230 break;
231 case GDB_WATCHPOINT_WRITE:
232 wp.wcr = deposit32(wp.wcr, 3, 2, 2);
233 wp.details.flags = BP_MEM_WRITE;
234 break;
235 case GDB_WATCHPOINT_ACCESS:
236 wp.wcr = deposit32(wp.wcr, 3, 2, 3);
237 wp.details.flags = BP_MEM_ACCESS;
238 break;
239 default:
240 g_assert_not_reached();
241 break;
242 }
243 if (len <= 8) {
244 /* we align the address and set the bits in BAS */
245 int off = addr & 0x7;
246 int bas = (1 << len) - 1;
247
248 wp.wcr = deposit32(wp.wcr, 5 + off, 8 - off, bas);
249 } else {
250 /* For ranges above 8 bytes we need to be a power of 2 */
251 if (is_power_of_2(len)) {
252 int bits = ctz64(len);
253
254 wp.wvr &= ~((1 << bits) - 1);
255 wp.wcr = deposit32(wp.wcr, 24, 4, bits);
256 wp.wcr = deposit32(wp.wcr, 5, 8, 0xff);
257 } else {
258 return -ENOBUFS;
259 }
260 }
261
262 g_array_append_val(hw_watchpoints, wp);
263 return 0;
264 }
265
266
267 static bool check_watchpoint_in_range(int i, target_ulong addr)
268 {
269 HWWatchpoint *wp = get_hw_wp(i);
270 uint64_t addr_top, addr_bottom = wp->wvr;
271 int bas = extract32(wp->wcr, 5, 8);
272 int mask = extract32(wp->wcr, 24, 4);
273
274 if (mask) {
275 addr_top = addr_bottom + (1 << mask);
276 } else {
277 /* BAS must be contiguous but can offset against the base
278 * address in DBGWVR */
279 addr_bottom = addr_bottom + ctz32(bas);
280 addr_top = addr_bottom + clo32(bas);
281 }
282
283 if (addr >= addr_bottom && addr <= addr_top) {
284 return true;
285 }
286
287 return false;
288 }
289
290 /**
291 * delete_hw_watchpoint()
292 * @addr: address of breakpoint
293 *
294 * Delete a breakpoint and shuffle any above down
295 */
296
297 static int delete_hw_watchpoint(target_ulong addr,
298 target_ulong len, int type)
299 {
300 int i;
301 for (i = 0; i < cur_hw_wps; i++) {
302 if (check_watchpoint_in_range(i, addr)) {
303 g_array_remove_index(hw_watchpoints, i);
304 return 0;
305 }
306 }
307 return -ENOENT;
308 }
309
310
311 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
312 target_ulong len, int type)
313 {
314 switch (type) {
315 case GDB_BREAKPOINT_HW:
316 return insert_hw_breakpoint(addr);
317 break;
318 case GDB_WATCHPOINT_READ:
319 case GDB_WATCHPOINT_WRITE:
320 case GDB_WATCHPOINT_ACCESS:
321 return insert_hw_watchpoint(addr, len, type);
322 default:
323 return -ENOSYS;
324 }
325 }
326
327 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
328 target_ulong len, int type)
329 {
330 switch (type) {
331 case GDB_BREAKPOINT_HW:
332 return delete_hw_breakpoint(addr);
333 case GDB_WATCHPOINT_READ:
334 case GDB_WATCHPOINT_WRITE:
335 case GDB_WATCHPOINT_ACCESS:
336 return delete_hw_watchpoint(addr, len, type);
337 default:
338 return -ENOSYS;
339 }
340 }
341
342
343 void kvm_arch_remove_all_hw_breakpoints(void)
344 {
345 if (cur_hw_wps > 0) {
346 g_array_remove_range(hw_watchpoints, 0, cur_hw_wps);
347 }
348 if (cur_hw_bps > 0) {
349 g_array_remove_range(hw_breakpoints, 0, cur_hw_bps);
350 }
351 }
352
353 void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr)
354 {
355 int i;
356 memset(ptr, 0, sizeof(struct kvm_guest_debug_arch));
357
358 for (i = 0; i < max_hw_wps; i++) {
359 HWWatchpoint *wp = get_hw_wp(i);
360 ptr->dbg_wcr[i] = wp->wcr;
361 ptr->dbg_wvr[i] = wp->wvr;
362 }
363 for (i = 0; i < max_hw_bps; i++) {
364 HWBreakpoint *bp = get_hw_bp(i);
365 ptr->dbg_bcr[i] = bp->bcr;
366 ptr->dbg_bvr[i] = bp->bvr;
367 }
368 }
369
370 bool kvm_arm_hw_debug_active(CPUState *cs)
371 {
372 return ((cur_hw_wps > 0) || (cur_hw_bps > 0));
373 }
374
375 static bool find_hw_breakpoint(CPUState *cpu, target_ulong pc)
376 {
377 int i;
378
379 for (i = 0; i < cur_hw_bps; i++) {
380 HWBreakpoint *bp = get_hw_bp(i);
381 if (bp->bvr == pc) {
382 return true;
383 }
384 }
385 return false;
386 }
387
388 static CPUWatchpoint *find_hw_watchpoint(CPUState *cpu, target_ulong addr)
389 {
390 int i;
391
392 for (i = 0; i < cur_hw_wps; i++) {
393 if (check_watchpoint_in_range(i, addr)) {
394 return &get_hw_wp(i)->details;
395 }
396 }
397 return NULL;
398 }
399
400 static bool kvm_arm_pmu_set_attr(CPUState *cs, struct kvm_device_attr *attr)
401 {
402 int err;
403
404 err = kvm_vcpu_ioctl(cs, KVM_HAS_DEVICE_ATTR, attr);
405 if (err != 0) {
406 error_report("PMU: KVM_HAS_DEVICE_ATTR: %s", strerror(-err));
407 return false;
408 }
409
410 err = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, attr);
411 if (err != 0) {
412 error_report("PMU: KVM_SET_DEVICE_ATTR: %s", strerror(-err));
413 return false;
414 }
415
416 return true;
417 }
418
419 void kvm_arm_pmu_init(CPUState *cs)
420 {
421 struct kvm_device_attr attr = {
422 .group = KVM_ARM_VCPU_PMU_V3_CTRL,
423 .attr = KVM_ARM_VCPU_PMU_V3_INIT,
424 };
425
426 if (!ARM_CPU(cs)->has_pmu) {
427 return;
428 }
429 if (!kvm_arm_pmu_set_attr(cs, &attr)) {
430 error_report("failed to init PMU");
431 abort();
432 }
433 }
434
435 void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
436 {
437 struct kvm_device_attr attr = {
438 .group = KVM_ARM_VCPU_PMU_V3_CTRL,
439 .addr = (intptr_t)&irq,
440 .attr = KVM_ARM_VCPU_PMU_V3_IRQ,
441 };
442
443 if (!ARM_CPU(cs)->has_pmu) {
444 return;
445 }
446 if (!kvm_arm_pmu_set_attr(cs, &attr)) {
447 error_report("failed to set irq for PMU");
448 abort();
449 }
450 }
451
452 static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id)
453 {
454 uint64_t ret;
455 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)&ret };
456 int err;
457
458 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64);
459 err = ioctl(fd, KVM_GET_ONE_REG, &idreg);
460 if (err < 0) {
461 return -1;
462 }
463 *pret = ret;
464 return 0;
465 }
466
467 static int read_sys_reg64(int fd, uint64_t *pret, uint64_t id)
468 {
469 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)pret };
470
471 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64);
472 return ioctl(fd, KVM_GET_ONE_REG, &idreg);
473 }
474
475 bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
476 {
477 /* Identify the feature bits corresponding to the host CPU, and
478 * fill out the ARMHostCPUClass fields accordingly. To do this
479 * we have to create a scratch VM, create a single CPU inside it,
480 * and then query that CPU for the relevant ID registers.
481 */
482 int fdarray[3];
483 bool sve_supported;
484 uint64_t features = 0;
485 uint64_t t;
486 int err;
487
488 /* Old kernels may not know about the PREFERRED_TARGET ioctl: however
489 * we know these will only support creating one kind of guest CPU,
490 * which is its preferred CPU type. Fortunately these old kernels
491 * support only a very limited number of CPUs.
492 */
493 static const uint32_t cpus_to_try[] = {
494 KVM_ARM_TARGET_AEM_V8,
495 KVM_ARM_TARGET_FOUNDATION_V8,
496 KVM_ARM_TARGET_CORTEX_A57,
497 QEMU_KVM_ARM_TARGET_NONE
498 };
499 /*
500 * target = -1 informs kvm_arm_create_scratch_host_vcpu()
501 * to use the preferred target
502 */
503 struct kvm_vcpu_init init = { .target = -1, };
504
505 if (!kvm_arm_create_scratch_host_vcpu(cpus_to_try, fdarray, &init)) {
506 return false;
507 }
508
509 ahcf->target = init.target;
510 ahcf->dtb_compatible = "arm,arm-v8";
511
512 err = read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr0,
513 ARM64_SYS_REG(3, 0, 0, 4, 0));
514 if (unlikely(err < 0)) {
515 /*
516 * Before v4.15, the kernel only exposed a limited number of system
517 * registers, not including any of the interesting AArch64 ID regs.
518 * For the most part we could leave these fields as zero with minimal
519 * effect, since this does not affect the values seen by the guest.
520 *
521 * However, it could cause problems down the line for QEMU,
522 * so provide a minimal v8.0 default.
523 *
524 * ??? Could read MIDR and use knowledge from cpu64.c.
525 * ??? Could map a page of memory into our temp guest and
526 * run the tiniest of hand-crafted kernels to extract
527 * the values seen by the guest.
528 * ??? Either of these sounds like too much effort just
529 * to work around running a modern host kernel.
530 */
531 ahcf->isar.id_aa64pfr0 = 0x00000011; /* EL1&0, AArch64 only */
532 err = 0;
533 } else {
534 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr1,
535 ARM64_SYS_REG(3, 0, 0, 4, 1));
536 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr0,
537 ARM64_SYS_REG(3, 0, 0, 5, 0));
538 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr1,
539 ARM64_SYS_REG(3, 0, 0, 5, 1));
540 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar0,
541 ARM64_SYS_REG(3, 0, 0, 6, 0));
542 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar1,
543 ARM64_SYS_REG(3, 0, 0, 6, 1));
544 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr0,
545 ARM64_SYS_REG(3, 0, 0, 7, 0));
546 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr1,
547 ARM64_SYS_REG(3, 0, 0, 7, 1));
548 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr2,
549 ARM64_SYS_REG(3, 0, 0, 7, 2));
550
551 /*
552 * Note that if AArch32 support is not present in the host,
553 * the AArch32 sysregs are present to be read, but will
554 * return UNKNOWN values. This is neither better nor worse
555 * than skipping the reads and leaving 0, as we must avoid
556 * considering the values in every case.
557 */
558 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_dfr0,
559 ARM64_SYS_REG(3, 0, 0, 1, 2));
560 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr0,
561 ARM64_SYS_REG(3, 0, 0, 1, 4));
562 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr1,
563 ARM64_SYS_REG(3, 0, 0, 1, 5));
564 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr2,
565 ARM64_SYS_REG(3, 0, 0, 1, 6));
566 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr3,
567 ARM64_SYS_REG(3, 0, 0, 1, 7));
568 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar0,
569 ARM64_SYS_REG(3, 0, 0, 2, 0));
570 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar1,
571 ARM64_SYS_REG(3, 0, 0, 2, 1));
572 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar2,
573 ARM64_SYS_REG(3, 0, 0, 2, 2));
574 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar3,
575 ARM64_SYS_REG(3, 0, 0, 2, 3));
576 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar4,
577 ARM64_SYS_REG(3, 0, 0, 2, 4));
578 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar5,
579 ARM64_SYS_REG(3, 0, 0, 2, 5));
580 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr4,
581 ARM64_SYS_REG(3, 0, 0, 2, 6));
582 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar6,
583 ARM64_SYS_REG(3, 0, 0, 2, 7));
584
585 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr0,
586 ARM64_SYS_REG(3, 0, 0, 3, 0));
587 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr1,
588 ARM64_SYS_REG(3, 0, 0, 3, 1));
589 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr2,
590 ARM64_SYS_REG(3, 0, 0, 3, 2));
591
592 /*
593 * DBGDIDR is a bit complicated because the kernel doesn't
594 * provide an accessor for it in 64-bit mode, which is what this
595 * scratch VM is in, and there's no architected "64-bit sysreg
596 * which reads the same as the 32-bit register" the way there is
597 * for other ID registers. Instead we synthesize a value from the
598 * AArch64 ID_AA64DFR0, the same way the kernel code in
599 * arch/arm64/kvm/sys_regs.c:trap_dbgidr() does.
600 * We only do this if the CPU supports AArch32 at EL1.
601 */
602 if (FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL1) >= 2) {
603 int wrps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, WRPS);
604 int brps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, BRPS);
605 int ctx_cmps =
606 FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, CTX_CMPS);
607 int version = 6; /* ARMv8 debug architecture */
608 bool has_el3 =
609 !!FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL3);
610 uint32_t dbgdidr = 0;
611
612 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, WRPS, wrps);
613 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, BRPS, brps);
614 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, CTX_CMPS, ctx_cmps);
615 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, VERSION, version);
616 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, NSUHD_IMP, has_el3);
617 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, SE_IMP, has_el3);
618 dbgdidr |= (1 << 15); /* RES1 bit */
619 ahcf->isar.dbgdidr = dbgdidr;
620 }
621 }
622
623 sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) > 0;
624
625 kvm_arm_destroy_scratch_host_vcpu(fdarray);
626
627 if (err < 0) {
628 return false;
629 }
630
631 /* Add feature bits that can't appear until after VCPU init. */
632 if (sve_supported) {
633 t = ahcf->isar.id_aa64pfr0;
634 t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1);
635 ahcf->isar.id_aa64pfr0 = t;
636 }
637
638 /*
639 * We can assume any KVM supporting CPU is at least a v8
640 * with VFPv4+Neon; this in turn implies most of the other
641 * feature bits.
642 */
643 features |= 1ULL << ARM_FEATURE_V8;
644 features |= 1ULL << ARM_FEATURE_NEON;
645 features |= 1ULL << ARM_FEATURE_AARCH64;
646 features |= 1ULL << ARM_FEATURE_PMU;
647 features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
648
649 ahcf->features = features;
650
651 return true;
652 }
653
654 bool kvm_arm_aarch32_supported(void)
655 {
656 return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT);
657 }
658
659 bool kvm_arm_sve_supported(void)
660 {
661 return kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE);
662 }
663
664 QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1);
665
666 void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map)
667 {
668 /* Only call this function if kvm_arm_sve_supported() returns true. */
669 static uint64_t vls[KVM_ARM64_SVE_VLS_WORDS];
670 static bool probed;
671 uint32_t vq = 0;
672 int i, j;
673
674 bitmap_clear(map, 0, ARM_MAX_VQ);
675
676 /*
677 * KVM ensures all host CPUs support the same set of vector lengths.
678 * So we only need to create the scratch VCPUs once and then cache
679 * the results.
680 */
681 if (!probed) {
682 struct kvm_vcpu_init init = {
683 .target = -1,
684 .features[0] = (1 << KVM_ARM_VCPU_SVE),
685 };
686 struct kvm_one_reg reg = {
687 .id = KVM_REG_ARM64_SVE_VLS,
688 .addr = (uint64_t)&vls[0],
689 };
690 int fdarray[3], ret;
691
692 probed = true;
693
694 if (!kvm_arm_create_scratch_host_vcpu(NULL, fdarray, &init)) {
695 error_report("failed to create scratch VCPU with SVE enabled");
696 abort();
697 }
698 ret = ioctl(fdarray[2], KVM_GET_ONE_REG, &reg);
699 kvm_arm_destroy_scratch_host_vcpu(fdarray);
700 if (ret) {
701 error_report("failed to get KVM_REG_ARM64_SVE_VLS: %s",
702 strerror(errno));
703 abort();
704 }
705
706 for (i = KVM_ARM64_SVE_VLS_WORDS - 1; i >= 0; --i) {
707 if (vls[i]) {
708 vq = 64 - clz64(vls[i]) + i * 64;
709 break;
710 }
711 }
712 if (vq > ARM_MAX_VQ) {
713 warn_report("KVM supports vector lengths larger than "
714 "QEMU can enable");
715 }
716 }
717
718 for (i = 0; i < KVM_ARM64_SVE_VLS_WORDS; ++i) {
719 if (!vls[i]) {
720 continue;
721 }
722 for (j = 1; j <= 64; ++j) {
723 vq = j + i * 64;
724 if (vq > ARM_MAX_VQ) {
725 return;
726 }
727 if (vls[i] & (1UL << (j - 1))) {
728 set_bit(vq - 1, map);
729 }
730 }
731 }
732 }
733
734 static int kvm_arm_sve_set_vls(CPUState *cs)
735 {
736 uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = {0};
737 struct kvm_one_reg reg = {
738 .id = KVM_REG_ARM64_SVE_VLS,
739 .addr = (uint64_t)&vls[0],
740 };
741 ARMCPU *cpu = ARM_CPU(cs);
742 uint32_t vq;
743 int i, j;
744
745 assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX);
746
747 for (vq = 1; vq <= cpu->sve_max_vq; ++vq) {
748 if (test_bit(vq - 1, cpu->sve_vq_map)) {
749 i = (vq - 1) / 64;
750 j = (vq - 1) % 64;
751 vls[i] |= 1UL << j;
752 }
753 }
754
755 return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
756 }
757
758 #define ARM_CPU_ID_MPIDR 3, 0, 0, 0, 5
759
760 int kvm_arch_init_vcpu(CPUState *cs)
761 {
762 int ret;
763 uint64_t mpidr;
764 ARMCPU *cpu = ARM_CPU(cs);
765 CPUARMState *env = &cpu->env;
766
767 if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE ||
768 !object_dynamic_cast(OBJECT(cpu), TYPE_AARCH64_CPU)) {
769 error_report("KVM is not supported for this guest CPU type");
770 return -EINVAL;
771 }
772
773 qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, cs);
774
775 /* Determine init features for this CPU */
776 memset(cpu->kvm_init_features, 0, sizeof(cpu->kvm_init_features));
777 if (cs->start_powered_off) {
778 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_POWER_OFF;
779 }
780 if (kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PSCI_0_2)) {
781 cpu->psci_version = 2;
782 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PSCI_0_2;
783 }
784 if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
785 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_EL1_32BIT;
786 }
787 if (!kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PMU_V3)) {
788 cpu->has_pmu = false;
789 }
790 if (cpu->has_pmu) {
791 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3;
792 } else {
793 env->features &= ~(1ULL << ARM_FEATURE_PMU);
794 }
795 if (cpu_isar_feature(aa64_sve, cpu)) {
796 assert(kvm_arm_sve_supported());
797 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE;
798 }
799
800 /* Do KVM_ARM_VCPU_INIT ioctl */
801 ret = kvm_arm_vcpu_init(cs);
802 if (ret) {
803 return ret;
804 }
805
806 if (cpu_isar_feature(aa64_sve, cpu)) {
807 ret = kvm_arm_sve_set_vls(cs);
808 if (ret) {
809 return ret;
810 }
811 ret = kvm_arm_vcpu_finalize(cs, KVM_ARM_VCPU_SVE);
812 if (ret) {
813 return ret;
814 }
815 }
816
817 /*
818 * When KVM is in use, PSCI is emulated in-kernel and not by qemu.
819 * Currently KVM has its own idea about MPIDR assignment, so we
820 * override our defaults with what we get from KVM.
821 */
822 ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr);
823 if (ret) {
824 return ret;
825 }
826 cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK;
827
828 kvm_arm_init_debug(cs);
829
830 /* Check whether user space can specify guest syndrome value */
831 kvm_arm_init_serror_injection(cs);
832
833 return kvm_arm_init_cpreg_list(cpu);
834 }
835
836 int kvm_arch_destroy_vcpu(CPUState *cs)
837 {
838 return 0;
839 }
840
841 bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx)
842 {
843 /* Return true if the regidx is a register we should synchronize
844 * via the cpreg_tuples array (ie is not a core or sve reg that
845 * we sync by hand in kvm_arch_get/put_registers())
846 */
847 switch (regidx & KVM_REG_ARM_COPROC_MASK) {
848 case KVM_REG_ARM_CORE:
849 case KVM_REG_ARM64_SVE:
850 return false;
851 default:
852 return true;
853 }
854 }
855
856 typedef struct CPRegStateLevel {
857 uint64_t regidx;
858 int level;
859 } CPRegStateLevel;
860
861 /* All system registers not listed in the following table are assumed to be
862 * of the level KVM_PUT_RUNTIME_STATE. If a register should be written less
863 * often, you must add it to this table with a state of either
864 * KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE.
865 */
866 static const CPRegStateLevel non_runtime_cpregs[] = {
867 { KVM_REG_ARM_TIMER_CNT, KVM_PUT_FULL_STATE },
868 };
869
870 int kvm_arm_cpreg_level(uint64_t regidx)
871 {
872 int i;
873
874 for (i = 0; i < ARRAY_SIZE(non_runtime_cpregs); i++) {
875 const CPRegStateLevel *l = &non_runtime_cpregs[i];
876 if (l->regidx == regidx) {
877 return l->level;
878 }
879 }
880
881 return KVM_PUT_RUNTIME_STATE;
882 }
883
884 /* Callers must hold the iothread mutex lock */
885 static void kvm_inject_arm_sea(CPUState *c)
886 {
887 ARMCPU *cpu = ARM_CPU(c);
888 CPUARMState *env = &cpu->env;
889 CPUClass *cc = CPU_GET_CLASS(c);
890 uint32_t esr;
891 bool same_el;
892
893 c->exception_index = EXCP_DATA_ABORT;
894 env->exception.target_el = 1;
895
896 /*
897 * Set the DFSC to synchronous external abort and set FnV to not valid,
898 * this will tell guest the FAR_ELx is UNKNOWN for this abort.
899 */
900 same_el = arm_current_el(env) == env->exception.target_el;
901 esr = syn_data_abort_no_iss(same_el, 1, 0, 0, 0, 0, 0x10);
902
903 env->exception.syndrome = esr;
904
905 cc->do_interrupt(c);
906 }
907
908 #define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \
909 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
910
911 #define AARCH64_SIMD_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \
912 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
913
914 #define AARCH64_SIMD_CTRL_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U32 | \
915 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
916
917 static int kvm_arch_put_fpsimd(CPUState *cs)
918 {
919 CPUARMState *env = &ARM_CPU(cs)->env;
920 struct kvm_one_reg reg;
921 int i, ret;
922
923 for (i = 0; i < 32; i++) {
924 uint64_t *q = aa64_vfp_qreg(env, i);
925 #ifdef HOST_WORDS_BIGENDIAN
926 uint64_t fp_val[2] = { q[1], q[0] };
927 reg.addr = (uintptr_t)fp_val;
928 #else
929 reg.addr = (uintptr_t)q;
930 #endif
931 reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]);
932 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
933 if (ret) {
934 return ret;
935 }
936 }
937
938 return 0;
939 }
940
941 /*
942 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits
943 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard
944 * code the slice index to zero for now as it's unlikely we'll need more than
945 * one slice for quite some time.
946 */
947 static int kvm_arch_put_sve(CPUState *cs)
948 {
949 ARMCPU *cpu = ARM_CPU(cs);
950 CPUARMState *env = &cpu->env;
951 uint64_t tmp[ARM_MAX_VQ * 2];
952 uint64_t *r;
953 struct kvm_one_reg reg;
954 int n, ret;
955
956 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) {
957 r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2);
958 reg.addr = (uintptr_t)r;
959 reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0);
960 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
961 if (ret) {
962 return ret;
963 }
964 }
965
966 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) {
967 r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0],
968 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
969 reg.addr = (uintptr_t)r;
970 reg.id = KVM_REG_ARM64_SVE_PREG(n, 0);
971 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
972 if (ret) {
973 return ret;
974 }
975 }
976
977 r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0],
978 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
979 reg.addr = (uintptr_t)r;
980 reg.id = KVM_REG_ARM64_SVE_FFR(0);
981 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
982 if (ret) {
983 return ret;
984 }
985
986 return 0;
987 }
988
989 int kvm_arch_put_registers(CPUState *cs, int level)
990 {
991 struct kvm_one_reg reg;
992 uint64_t val;
993 uint32_t fpr;
994 int i, ret;
995 unsigned int el;
996
997 ARMCPU *cpu = ARM_CPU(cs);
998 CPUARMState *env = &cpu->env;
999
1000 /* If we are in AArch32 mode then we need to copy the AArch32 regs to the
1001 * AArch64 registers before pushing them out to 64-bit KVM.
1002 */
1003 if (!is_a64(env)) {
1004 aarch64_sync_32_to_64(env);
1005 }
1006
1007 for (i = 0; i < 31; i++) {
1008 reg.id = AARCH64_CORE_REG(regs.regs[i]);
1009 reg.addr = (uintptr_t) &env->xregs[i];
1010 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1011 if (ret) {
1012 return ret;
1013 }
1014 }
1015
1016 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
1017 * QEMU side we keep the current SP in xregs[31] as well.
1018 */
1019 aarch64_save_sp(env, 1);
1020
1021 reg.id = AARCH64_CORE_REG(regs.sp);
1022 reg.addr = (uintptr_t) &env->sp_el[0];
1023 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1024 if (ret) {
1025 return ret;
1026 }
1027
1028 reg.id = AARCH64_CORE_REG(sp_el1);
1029 reg.addr = (uintptr_t) &env->sp_el[1];
1030 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1031 if (ret) {
1032 return ret;
1033 }
1034
1035 /* Note that KVM thinks pstate is 64 bit but we use a uint32_t */
1036 if (is_a64(env)) {
1037 val = pstate_read(env);
1038 } else {
1039 val = cpsr_read(env);
1040 }
1041 reg.id = AARCH64_CORE_REG(regs.pstate);
1042 reg.addr = (uintptr_t) &val;
1043 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1044 if (ret) {
1045 return ret;
1046 }
1047
1048 reg.id = AARCH64_CORE_REG(regs.pc);
1049 reg.addr = (uintptr_t) &env->pc;
1050 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1051 if (ret) {
1052 return ret;
1053 }
1054
1055 reg.id = AARCH64_CORE_REG(elr_el1);
1056 reg.addr = (uintptr_t) &env->elr_el[1];
1057 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1058 if (ret) {
1059 return ret;
1060 }
1061
1062 /* Saved Program State Registers
1063 *
1064 * Before we restore from the banked_spsr[] array we need to
1065 * ensure that any modifications to env->spsr are correctly
1066 * reflected in the banks.
1067 */
1068 el = arm_current_el(env);
1069 if (el > 0 && !is_a64(env)) {
1070 i = bank_number(env->uncached_cpsr & CPSR_M);
1071 env->banked_spsr[i] = env->spsr;
1072 }
1073
1074 /* KVM 0-4 map to QEMU banks 1-5 */
1075 for (i = 0; i < KVM_NR_SPSR; i++) {
1076 reg.id = AARCH64_CORE_REG(spsr[i]);
1077 reg.addr = (uintptr_t) &env->banked_spsr[i + 1];
1078 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1079 if (ret) {
1080 return ret;
1081 }
1082 }
1083
1084 if (cpu_isar_feature(aa64_sve, cpu)) {
1085 ret = kvm_arch_put_sve(cs);
1086 } else {
1087 ret = kvm_arch_put_fpsimd(cs);
1088 }
1089 if (ret) {
1090 return ret;
1091 }
1092
1093 reg.addr = (uintptr_t)(&fpr);
1094 fpr = vfp_get_fpsr(env);
1095 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr);
1096 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1097 if (ret) {
1098 return ret;
1099 }
1100
1101 reg.addr = (uintptr_t)(&fpr);
1102 fpr = vfp_get_fpcr(env);
1103 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr);
1104 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1105 if (ret) {
1106 return ret;
1107 }
1108
1109 write_cpustate_to_list(cpu, true);
1110
1111 if (!write_list_to_kvmstate(cpu, level)) {
1112 return -EINVAL;
1113 }
1114
1115 /*
1116 * Setting VCPU events should be triggered after syncing the registers
1117 * to avoid overwriting potential changes made by KVM upon calling
1118 * KVM_SET_VCPU_EVENTS ioctl
1119 */
1120 ret = kvm_put_vcpu_events(cpu);
1121 if (ret) {
1122 return ret;
1123 }
1124
1125 kvm_arm_sync_mpstate_to_kvm(cpu);
1126
1127 return ret;
1128 }
1129
1130 static int kvm_arch_get_fpsimd(CPUState *cs)
1131 {
1132 CPUARMState *env = &ARM_CPU(cs)->env;
1133 struct kvm_one_reg reg;
1134 int i, ret;
1135
1136 for (i = 0; i < 32; i++) {
1137 uint64_t *q = aa64_vfp_qreg(env, i);
1138 reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]);
1139 reg.addr = (uintptr_t)q;
1140 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1141 if (ret) {
1142 return ret;
1143 } else {
1144 #ifdef HOST_WORDS_BIGENDIAN
1145 uint64_t t;
1146 t = q[0], q[0] = q[1], q[1] = t;
1147 #endif
1148 }
1149 }
1150
1151 return 0;
1152 }
1153
1154 /*
1155 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits
1156 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard
1157 * code the slice index to zero for now as it's unlikely we'll need more than
1158 * one slice for quite some time.
1159 */
1160 static int kvm_arch_get_sve(CPUState *cs)
1161 {
1162 ARMCPU *cpu = ARM_CPU(cs);
1163 CPUARMState *env = &cpu->env;
1164 struct kvm_one_reg reg;
1165 uint64_t *r;
1166 int n, ret;
1167
1168 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) {
1169 r = &env->vfp.zregs[n].d[0];
1170 reg.addr = (uintptr_t)r;
1171 reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0);
1172 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1173 if (ret) {
1174 return ret;
1175 }
1176 sve_bswap64(r, r, cpu->sve_max_vq * 2);
1177 }
1178
1179 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) {
1180 r = &env->vfp.pregs[n].p[0];
1181 reg.addr = (uintptr_t)r;
1182 reg.id = KVM_REG_ARM64_SVE_PREG(n, 0);
1183 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1184 if (ret) {
1185 return ret;
1186 }
1187 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
1188 }
1189
1190 r = &env->vfp.pregs[FFR_PRED_NUM].p[0];
1191 reg.addr = (uintptr_t)r;
1192 reg.id = KVM_REG_ARM64_SVE_FFR(0);
1193 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1194 if (ret) {
1195 return ret;
1196 }
1197 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
1198
1199 return 0;
1200 }
1201
1202 int kvm_arch_get_registers(CPUState *cs)
1203 {
1204 struct kvm_one_reg reg;
1205 uint64_t val;
1206 unsigned int el;
1207 uint32_t fpr;
1208 int i, ret;
1209
1210 ARMCPU *cpu = ARM_CPU(cs);
1211 CPUARMState *env = &cpu->env;
1212
1213 for (i = 0; i < 31; i++) {
1214 reg.id = AARCH64_CORE_REG(regs.regs[i]);
1215 reg.addr = (uintptr_t) &env->xregs[i];
1216 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1217 if (ret) {
1218 return ret;
1219 }
1220 }
1221
1222 reg.id = AARCH64_CORE_REG(regs.sp);
1223 reg.addr = (uintptr_t) &env->sp_el[0];
1224 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1225 if (ret) {
1226 return ret;
1227 }
1228
1229 reg.id = AARCH64_CORE_REG(sp_el1);
1230 reg.addr = (uintptr_t) &env->sp_el[1];
1231 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1232 if (ret) {
1233 return ret;
1234 }
1235
1236 reg.id = AARCH64_CORE_REG(regs.pstate);
1237 reg.addr = (uintptr_t) &val;
1238 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1239 if (ret) {
1240 return ret;
1241 }
1242
1243 env->aarch64 = ((val & PSTATE_nRW) == 0);
1244 if (is_a64(env)) {
1245 pstate_write(env, val);
1246 } else {
1247 cpsr_write(env, val, 0xffffffff, CPSRWriteRaw);
1248 }
1249
1250 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
1251 * QEMU side we keep the current SP in xregs[31] as well.
1252 */
1253 aarch64_restore_sp(env, 1);
1254
1255 reg.id = AARCH64_CORE_REG(regs.pc);
1256 reg.addr = (uintptr_t) &env->pc;
1257 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1258 if (ret) {
1259 return ret;
1260 }
1261
1262 /* If we are in AArch32 mode then we need to sync the AArch32 regs with the
1263 * incoming AArch64 regs received from 64-bit KVM.
1264 * We must perform this after all of the registers have been acquired from
1265 * the kernel.
1266 */
1267 if (!is_a64(env)) {
1268 aarch64_sync_64_to_32(env);
1269 }
1270
1271 reg.id = AARCH64_CORE_REG(elr_el1);
1272 reg.addr = (uintptr_t) &env->elr_el[1];
1273 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1274 if (ret) {
1275 return ret;
1276 }
1277
1278 /* Fetch the SPSR registers
1279 *
1280 * KVM SPSRs 0-4 map to QEMU banks 1-5
1281 */
1282 for (i = 0; i < KVM_NR_SPSR; i++) {
1283 reg.id = AARCH64_CORE_REG(spsr[i]);
1284 reg.addr = (uintptr_t) &env->banked_spsr[i + 1];
1285 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1286 if (ret) {
1287 return ret;
1288 }
1289 }
1290
1291 el = arm_current_el(env);
1292 if (el > 0 && !is_a64(env)) {
1293 i = bank_number(env->uncached_cpsr & CPSR_M);
1294 env->spsr = env->banked_spsr[i];
1295 }
1296
1297 if (cpu_isar_feature(aa64_sve, cpu)) {
1298 ret = kvm_arch_get_sve(cs);
1299 } else {
1300 ret = kvm_arch_get_fpsimd(cs);
1301 }
1302 if (ret) {
1303 return ret;
1304 }
1305
1306 reg.addr = (uintptr_t)(&fpr);
1307 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr);
1308 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1309 if (ret) {
1310 return ret;
1311 }
1312 vfp_set_fpsr(env, fpr);
1313
1314 reg.addr = (uintptr_t)(&fpr);
1315 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr);
1316 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1317 if (ret) {
1318 return ret;
1319 }
1320 vfp_set_fpcr(env, fpr);
1321
1322 ret = kvm_get_vcpu_events(cpu);
1323 if (ret) {
1324 return ret;
1325 }
1326
1327 if (!write_kvmstate_to_list(cpu)) {
1328 return -EINVAL;
1329 }
1330 /* Note that it's OK to have registers which aren't in CPUState,
1331 * so we can ignore a failure return here.
1332 */
1333 write_list_to_cpustate(cpu);
1334
1335 kvm_arm_sync_mpstate_to_qemu(cpu);
1336
1337 /* TODO: other registers */
1338 return ret;
1339 }
1340
1341 void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr)
1342 {
1343 ram_addr_t ram_addr;
1344 hwaddr paddr;
1345 Object *obj = qdev_get_machine();
1346 VirtMachineState *vms = VIRT_MACHINE(obj);
1347 bool acpi_enabled = virt_is_acpi_enabled(vms);
1348
1349 assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO);
1350
1351 if (acpi_enabled && addr &&
1352 object_property_get_bool(obj, "ras", NULL)) {
1353 ram_addr = qemu_ram_addr_from_host(addr);
1354 if (ram_addr != RAM_ADDR_INVALID &&
1355 kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) {
1356 kvm_hwpoison_page_add(ram_addr);
1357 /*
1358 * If this is a BUS_MCEERR_AR, we know we have been called
1359 * synchronously from the vCPU thread, so we can easily
1360 * synchronize the state and inject an error.
1361 *
1362 * TODO: we currently don't tell the guest at all about
1363 * BUS_MCEERR_AO. In that case we might either be being
1364 * called synchronously from the vCPU thread, or a bit
1365 * later from the main thread, so doing the injection of
1366 * the error would be more complicated.
1367 */
1368 if (code == BUS_MCEERR_AR) {
1369 kvm_cpu_synchronize_state(c);
1370 if (!acpi_ghes_record_errors(ACPI_HEST_SRC_ID_SEA, paddr)) {
1371 kvm_inject_arm_sea(c);
1372 } else {
1373 error_report("failed to record the error");
1374 abort();
1375 }
1376 }
1377 return;
1378 }
1379 if (code == BUS_MCEERR_AO) {
1380 error_report("Hardware memory error at addr %p for memory used by "
1381 "QEMU itself instead of guest system!", addr);
1382 }
1383 }
1384
1385 if (code == BUS_MCEERR_AR) {
1386 error_report("Hardware memory error!");
1387 exit(1);
1388 }
1389 }
1390
1391 /* C6.6.29 BRK instruction */
1392 static const uint32_t brk_insn = 0xd4200000;
1393
1394 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
1395 {
1396 if (have_guest_debug) {
1397 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) ||
1398 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) {
1399 return -EINVAL;
1400 }
1401 return 0;
1402 } else {
1403 error_report("guest debug not supported on this kernel");
1404 return -EINVAL;
1405 }
1406 }
1407
1408 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
1409 {
1410 static uint32_t brk;
1411
1412 if (have_guest_debug) {
1413 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) ||
1414 brk != brk_insn ||
1415 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) {
1416 return -EINVAL;
1417 }
1418 return 0;
1419 } else {
1420 error_report("guest debug not supported on this kernel");
1421 return -EINVAL;
1422 }
1423 }
1424
1425 /* See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register
1426 *
1427 * To minimise translating between kernel and user-space the kernel
1428 * ABI just provides user-space with the full exception syndrome
1429 * register value to be decoded in QEMU.
1430 */
1431
1432 bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit)
1433 {
1434 int hsr_ec = syn_get_ec(debug_exit->hsr);
1435 ARMCPU *cpu = ARM_CPU(cs);
1436 CPUClass *cc = CPU_GET_CLASS(cs);
1437 CPUARMState *env = &cpu->env;
1438
1439 /* Ensure PC is synchronised */
1440 kvm_cpu_synchronize_state(cs);
1441
1442 switch (hsr_ec) {
1443 case EC_SOFTWARESTEP:
1444 if (cs->singlestep_enabled) {
1445 return true;
1446 } else {
1447 /*
1448 * The kernel should have suppressed the guest's ability to
1449 * single step at this point so something has gone wrong.
1450 */
1451 error_report("%s: guest single-step while debugging unsupported"
1452 " (%"PRIx64", %"PRIx32")",
1453 __func__, env->pc, debug_exit->hsr);
1454 return false;
1455 }
1456 break;
1457 case EC_AA64_BKPT:
1458 if (kvm_find_sw_breakpoint(cs, env->pc)) {
1459 return true;
1460 }
1461 break;
1462 case EC_BREAKPOINT:
1463 if (find_hw_breakpoint(cs, env->pc)) {
1464 return true;
1465 }
1466 break;
1467 case EC_WATCHPOINT:
1468 {
1469 CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far);
1470 if (wp) {
1471 cs->watchpoint_hit = wp;
1472 return true;
1473 }
1474 break;
1475 }
1476 default:
1477 error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")",
1478 __func__, debug_exit->hsr, env->pc);
1479 }
1480
1481 /* If we are not handling the debug exception it must belong to
1482 * the guest. Let's re-use the existing TCG interrupt code to set
1483 * everything up properly.
1484 */
1485 cs->exception_index = EXCP_BKPT;
1486 env->exception.syndrome = debug_exit->hsr;
1487 env->exception.vaddress = debug_exit->far;
1488 env->exception.target_el = 1;
1489 qemu_mutex_lock_iothread();
1490 cc->do_interrupt(cs);
1491 qemu_mutex_unlock_iothread();
1492
1493 return false;
1494 }
1495
1496 #define ARM64_REG_ESR_EL1 ARM64_SYS_REG(3, 0, 5, 2, 0)
1497 #define ARM64_REG_TCR_EL1 ARM64_SYS_REG(3, 0, 2, 0, 2)
1498
1499 /*
1500 * ESR_EL1
1501 * ISS encoding
1502 * AARCH64: DFSC, bits [5:0]
1503 * AARCH32:
1504 * TTBCR.EAE == 0
1505 * FS[4] - DFSR[10]
1506 * FS[3:0] - DFSR[3:0]
1507 * TTBCR.EAE == 1
1508 * FS, bits [5:0]
1509 */
1510 #define ESR_DFSC(aarch64, lpae, v) \
1511 ((aarch64 || (lpae)) ? ((v) & 0x3F) \
1512 : (((v) >> 6) | ((v) & 0x1F)))
1513
1514 #define ESR_DFSC_EXTABT(aarch64, lpae) \
1515 ((aarch64) ? 0x10 : (lpae) ? 0x10 : 0x8)
1516
1517 bool kvm_arm_verify_ext_dabt_pending(CPUState *cs)
1518 {
1519 uint64_t dfsr_val;
1520
1521 if (!kvm_get_one_reg(cs, ARM64_REG_ESR_EL1, &dfsr_val)) {
1522 ARMCPU *cpu = ARM_CPU(cs);
1523 CPUARMState *env = &cpu->env;
1524 int aarch64_mode = arm_feature(env, ARM_FEATURE_AARCH64);
1525 int lpae = 0;
1526
1527 if (!aarch64_mode) {
1528 uint64_t ttbcr;
1529
1530 if (!kvm_get_one_reg(cs, ARM64_REG_TCR_EL1, &ttbcr)) {
1531 lpae = arm_feature(env, ARM_FEATURE_LPAE)
1532 && (ttbcr & TTBCR_EAE);
1533 }
1534 }
1535 /*
1536 * The verification here is based on the DFSC bits
1537 * of the ESR_EL1 reg only
1538 */
1539 return (ESR_DFSC(aarch64_mode, lpae, dfsr_val) ==
1540 ESR_DFSC_EXTABT(aarch64_mode, lpae));
1541 }
1542 return false;
1543 }