meson: target
[qemu.git] / target / arm / kvm64.c
1 /*
2 * ARM implementation of KVM hooks, 64 bit specific code
3 *
4 * Copyright Mian-M. Hamayun 2013, Virtual Open Systems
5 * Copyright Alex Bennée 2014, Linaro
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
9 *
10 */
11
12 #include "qemu/osdep.h"
13 #include <sys/ioctl.h>
14 #include <sys/ptrace.h>
15
16 #include <linux/elf.h>
17 #include <linux/kvm.h>
18
19 #include "qemu-common.h"
20 #include "cpu.h"
21 #include "qemu/timer.h"
22 #include "qemu/error-report.h"
23 #include "qemu/host-utils.h"
24 #include "qemu/main-loop.h"
25 #include "exec/gdbstub.h"
26 #include "sysemu/runstate.h"
27 #include "sysemu/kvm.h"
28 #include "sysemu/kvm_int.h"
29 #include "kvm_arm.h"
30 #include "internals.h"
31 #include "hw/acpi/acpi.h"
32 #include "hw/acpi/ghes.h"
33 #include "hw/arm/virt.h"
34
35 static bool have_guest_debug;
36
37 /*
38 * Although the ARM implementation of hardware assisted debugging
39 * allows for different breakpoints per-core, the current GDB
40 * interface treats them as a global pool of registers (which seems to
41 * be the case for x86, ppc and s390). As a result we store one copy
42 * of registers which is used for all active cores.
43 *
44 * Write access is serialised by virtue of the GDB protocol which
45 * updates things. Read access (i.e. when the values are copied to the
46 * vCPU) is also gated by GDB's run control.
47 *
48 * This is not unreasonable as most of the time debugging kernels you
49 * never know which core will eventually execute your function.
50 */
51
52 typedef struct {
53 uint64_t bcr;
54 uint64_t bvr;
55 } HWBreakpoint;
56
57 /* The watchpoint registers can cover more area than the requested
58 * watchpoint so we need to store the additional information
59 * somewhere. We also need to supply a CPUWatchpoint to the GDB stub
60 * when the watchpoint is hit.
61 */
62 typedef struct {
63 uint64_t wcr;
64 uint64_t wvr;
65 CPUWatchpoint details;
66 } HWWatchpoint;
67
68 /* Maximum and current break/watch point counts */
69 int max_hw_bps, max_hw_wps;
70 GArray *hw_breakpoints, *hw_watchpoints;
71
72 #define cur_hw_wps (hw_watchpoints->len)
73 #define cur_hw_bps (hw_breakpoints->len)
74 #define get_hw_bp(i) (&g_array_index(hw_breakpoints, HWBreakpoint, i))
75 #define get_hw_wp(i) (&g_array_index(hw_watchpoints, HWWatchpoint, i))
76
77 /**
78 * kvm_arm_init_debug() - check for guest debug capabilities
79 * @cs: CPUState
80 *
81 * kvm_check_extension returns the number of debug registers we have
82 * or 0 if we have none.
83 *
84 */
85 static void kvm_arm_init_debug(CPUState *cs)
86 {
87 have_guest_debug = kvm_check_extension(cs->kvm_state,
88 KVM_CAP_SET_GUEST_DEBUG);
89
90 max_hw_wps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_WPS);
91 hw_watchpoints = g_array_sized_new(true, true,
92 sizeof(HWWatchpoint), max_hw_wps);
93
94 max_hw_bps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_BPS);
95 hw_breakpoints = g_array_sized_new(true, true,
96 sizeof(HWBreakpoint), max_hw_bps);
97 return;
98 }
99
100 /**
101 * insert_hw_breakpoint()
102 * @addr: address of breakpoint
103 *
104 * See ARM ARM D2.9.1 for details but here we are only going to create
105 * simple un-linked breakpoints (i.e. we don't chain breakpoints
106 * together to match address and context or vmid). The hardware is
107 * capable of fancier matching but that will require exposing that
108 * fanciness to GDB's interface
109 *
110 * DBGBCR<n>_EL1, Debug Breakpoint Control Registers
111 *
112 * 31 24 23 20 19 16 15 14 13 12 9 8 5 4 3 2 1 0
113 * +------+------+-------+-----+----+------+-----+------+-----+---+
114 * | RES0 | BT | LBN | SSC | HMC| RES0 | BAS | RES0 | PMC | E |
115 * +------+------+-------+-----+----+------+-----+------+-----+---+
116 *
117 * BT: Breakpoint type (0 = unlinked address match)
118 * LBN: Linked BP number (0 = unused)
119 * SSC/HMC/PMC: Security, Higher and Priv access control (Table D-12)
120 * BAS: Byte Address Select (RES1 for AArch64)
121 * E: Enable bit
122 *
123 * DBGBVR<n>_EL1, Debug Breakpoint Value Registers
124 *
125 * 63 53 52 49 48 2 1 0
126 * +------+-----------+----------+-----+
127 * | RESS | VA[52:49] | VA[48:2] | 0 0 |
128 * +------+-----------+----------+-----+
129 *
130 * Depending on the addressing mode bits the top bits of the register
131 * are a sign extension of the highest applicable VA bit. Some
132 * versions of GDB don't do it correctly so we ensure they are correct
133 * here so future PC comparisons will work properly.
134 */
135
136 static int insert_hw_breakpoint(target_ulong addr)
137 {
138 HWBreakpoint brk = {
139 .bcr = 0x1, /* BCR E=1, enable */
140 .bvr = sextract64(addr, 0, 53)
141 };
142
143 if (cur_hw_bps >= max_hw_bps) {
144 return -ENOBUFS;
145 }
146
147 brk.bcr = deposit32(brk.bcr, 1, 2, 0x3); /* PMC = 11 */
148 brk.bcr = deposit32(brk.bcr, 5, 4, 0xf); /* BAS = RES1 */
149
150 g_array_append_val(hw_breakpoints, brk);
151
152 return 0;
153 }
154
155 /**
156 * delete_hw_breakpoint()
157 * @pc: address of breakpoint
158 *
159 * Delete a breakpoint and shuffle any above down
160 */
161
162 static int delete_hw_breakpoint(target_ulong pc)
163 {
164 int i;
165 for (i = 0; i < hw_breakpoints->len; i++) {
166 HWBreakpoint *brk = get_hw_bp(i);
167 if (brk->bvr == pc) {
168 g_array_remove_index(hw_breakpoints, i);
169 return 0;
170 }
171 }
172 return -ENOENT;
173 }
174
175 /**
176 * insert_hw_watchpoint()
177 * @addr: address of watch point
178 * @len: size of area
179 * @type: type of watch point
180 *
181 * See ARM ARM D2.10. As with the breakpoints we can do some advanced
182 * stuff if we want to. The watch points can be linked with the break
183 * points above to make them context aware. However for simplicity
184 * currently we only deal with simple read/write watch points.
185 *
186 * D7.3.11 DBGWCR<n>_EL1, Debug Watchpoint Control Registers
187 *
188 * 31 29 28 24 23 21 20 19 16 15 14 13 12 5 4 3 2 1 0
189 * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+
190 * | RES0 | MASK | RES0 | WT | LBN | SSC | HMC | BAS | LSC | PAC | E |
191 * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+
192 *
193 * MASK: num bits addr mask (0=none,01/10=res,11=3 bits (8 bytes))
194 * WT: 0 - unlinked, 1 - linked (not currently used)
195 * LBN: Linked BP number (not currently used)
196 * SSC/HMC/PAC: Security, Higher and Priv access control (Table D2-11)
197 * BAS: Byte Address Select
198 * LSC: Load/Store control (01: load, 10: store, 11: both)
199 * E: Enable
200 *
201 * The bottom 2 bits of the value register are masked. Therefore to
202 * break on any sizes smaller than an unaligned word you need to set
203 * MASK=0, BAS=bit per byte in question. For larger regions (^2) you
204 * need to ensure you mask the address as required and set BAS=0xff
205 */
206
207 static int insert_hw_watchpoint(target_ulong addr,
208 target_ulong len, int type)
209 {
210 HWWatchpoint wp = {
211 .wcr = 1, /* E=1, enable */
212 .wvr = addr & (~0x7ULL),
213 .details = { .vaddr = addr, .len = len }
214 };
215
216 if (cur_hw_wps >= max_hw_wps) {
217 return -ENOBUFS;
218 }
219
220 /*
221 * HMC=0 SSC=0 PAC=3 will hit EL0 or EL1, any security state,
222 * valid whether EL3 is implemented or not
223 */
224 wp.wcr = deposit32(wp.wcr, 1, 2, 3);
225
226 switch (type) {
227 case GDB_WATCHPOINT_READ:
228 wp.wcr = deposit32(wp.wcr, 3, 2, 1);
229 wp.details.flags = BP_MEM_READ;
230 break;
231 case GDB_WATCHPOINT_WRITE:
232 wp.wcr = deposit32(wp.wcr, 3, 2, 2);
233 wp.details.flags = BP_MEM_WRITE;
234 break;
235 case GDB_WATCHPOINT_ACCESS:
236 wp.wcr = deposit32(wp.wcr, 3, 2, 3);
237 wp.details.flags = BP_MEM_ACCESS;
238 break;
239 default:
240 g_assert_not_reached();
241 break;
242 }
243 if (len <= 8) {
244 /* we align the address and set the bits in BAS */
245 int off = addr & 0x7;
246 int bas = (1 << len) - 1;
247
248 wp.wcr = deposit32(wp.wcr, 5 + off, 8 - off, bas);
249 } else {
250 /* For ranges above 8 bytes we need to be a power of 2 */
251 if (is_power_of_2(len)) {
252 int bits = ctz64(len);
253
254 wp.wvr &= ~((1 << bits) - 1);
255 wp.wcr = deposit32(wp.wcr, 24, 4, bits);
256 wp.wcr = deposit32(wp.wcr, 5, 8, 0xff);
257 } else {
258 return -ENOBUFS;
259 }
260 }
261
262 g_array_append_val(hw_watchpoints, wp);
263 return 0;
264 }
265
266
267 static bool check_watchpoint_in_range(int i, target_ulong addr)
268 {
269 HWWatchpoint *wp = get_hw_wp(i);
270 uint64_t addr_top, addr_bottom = wp->wvr;
271 int bas = extract32(wp->wcr, 5, 8);
272 int mask = extract32(wp->wcr, 24, 4);
273
274 if (mask) {
275 addr_top = addr_bottom + (1 << mask);
276 } else {
277 /* BAS must be contiguous but can offset against the base
278 * address in DBGWVR */
279 addr_bottom = addr_bottom + ctz32(bas);
280 addr_top = addr_bottom + clo32(bas);
281 }
282
283 if (addr >= addr_bottom && addr <= addr_top) {
284 return true;
285 }
286
287 return false;
288 }
289
290 /**
291 * delete_hw_watchpoint()
292 * @addr: address of breakpoint
293 *
294 * Delete a breakpoint and shuffle any above down
295 */
296
297 static int delete_hw_watchpoint(target_ulong addr,
298 target_ulong len, int type)
299 {
300 int i;
301 for (i = 0; i < cur_hw_wps; i++) {
302 if (check_watchpoint_in_range(i, addr)) {
303 g_array_remove_index(hw_watchpoints, i);
304 return 0;
305 }
306 }
307 return -ENOENT;
308 }
309
310
311 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
312 target_ulong len, int type)
313 {
314 switch (type) {
315 case GDB_BREAKPOINT_HW:
316 return insert_hw_breakpoint(addr);
317 break;
318 case GDB_WATCHPOINT_READ:
319 case GDB_WATCHPOINT_WRITE:
320 case GDB_WATCHPOINT_ACCESS:
321 return insert_hw_watchpoint(addr, len, type);
322 default:
323 return -ENOSYS;
324 }
325 }
326
327 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
328 target_ulong len, int type)
329 {
330 switch (type) {
331 case GDB_BREAKPOINT_HW:
332 return delete_hw_breakpoint(addr);
333 break;
334 case GDB_WATCHPOINT_READ:
335 case GDB_WATCHPOINT_WRITE:
336 case GDB_WATCHPOINT_ACCESS:
337 return delete_hw_watchpoint(addr, len, type);
338 default:
339 return -ENOSYS;
340 }
341 }
342
343
344 void kvm_arch_remove_all_hw_breakpoints(void)
345 {
346 if (cur_hw_wps > 0) {
347 g_array_remove_range(hw_watchpoints, 0, cur_hw_wps);
348 }
349 if (cur_hw_bps > 0) {
350 g_array_remove_range(hw_breakpoints, 0, cur_hw_bps);
351 }
352 }
353
354 void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr)
355 {
356 int i;
357 memset(ptr, 0, sizeof(struct kvm_guest_debug_arch));
358
359 for (i = 0; i < max_hw_wps; i++) {
360 HWWatchpoint *wp = get_hw_wp(i);
361 ptr->dbg_wcr[i] = wp->wcr;
362 ptr->dbg_wvr[i] = wp->wvr;
363 }
364 for (i = 0; i < max_hw_bps; i++) {
365 HWBreakpoint *bp = get_hw_bp(i);
366 ptr->dbg_bcr[i] = bp->bcr;
367 ptr->dbg_bvr[i] = bp->bvr;
368 }
369 }
370
371 bool kvm_arm_hw_debug_active(CPUState *cs)
372 {
373 return ((cur_hw_wps > 0) || (cur_hw_bps > 0));
374 }
375
376 static bool find_hw_breakpoint(CPUState *cpu, target_ulong pc)
377 {
378 int i;
379
380 for (i = 0; i < cur_hw_bps; i++) {
381 HWBreakpoint *bp = get_hw_bp(i);
382 if (bp->bvr == pc) {
383 return true;
384 }
385 }
386 return false;
387 }
388
389 static CPUWatchpoint *find_hw_watchpoint(CPUState *cpu, target_ulong addr)
390 {
391 int i;
392
393 for (i = 0; i < cur_hw_wps; i++) {
394 if (check_watchpoint_in_range(i, addr)) {
395 return &get_hw_wp(i)->details;
396 }
397 }
398 return NULL;
399 }
400
401 static bool kvm_arm_pmu_set_attr(CPUState *cs, struct kvm_device_attr *attr)
402 {
403 int err;
404
405 err = kvm_vcpu_ioctl(cs, KVM_HAS_DEVICE_ATTR, attr);
406 if (err != 0) {
407 error_report("PMU: KVM_HAS_DEVICE_ATTR: %s", strerror(-err));
408 return false;
409 }
410
411 err = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, attr);
412 if (err != 0) {
413 error_report("PMU: KVM_SET_DEVICE_ATTR: %s", strerror(-err));
414 return false;
415 }
416
417 return true;
418 }
419
420 void kvm_arm_pmu_init(CPUState *cs)
421 {
422 struct kvm_device_attr attr = {
423 .group = KVM_ARM_VCPU_PMU_V3_CTRL,
424 .attr = KVM_ARM_VCPU_PMU_V3_INIT,
425 };
426
427 if (!ARM_CPU(cs)->has_pmu) {
428 return;
429 }
430 if (!kvm_arm_pmu_set_attr(cs, &attr)) {
431 error_report("failed to init PMU");
432 abort();
433 }
434 }
435
436 void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
437 {
438 struct kvm_device_attr attr = {
439 .group = KVM_ARM_VCPU_PMU_V3_CTRL,
440 .addr = (intptr_t)&irq,
441 .attr = KVM_ARM_VCPU_PMU_V3_IRQ,
442 };
443
444 if (!ARM_CPU(cs)->has_pmu) {
445 return;
446 }
447 if (!kvm_arm_pmu_set_attr(cs, &attr)) {
448 error_report("failed to set irq for PMU");
449 abort();
450 }
451 }
452
453 static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id)
454 {
455 uint64_t ret;
456 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)&ret };
457 int err;
458
459 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64);
460 err = ioctl(fd, KVM_GET_ONE_REG, &idreg);
461 if (err < 0) {
462 return -1;
463 }
464 *pret = ret;
465 return 0;
466 }
467
468 static int read_sys_reg64(int fd, uint64_t *pret, uint64_t id)
469 {
470 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)pret };
471
472 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64);
473 return ioctl(fd, KVM_GET_ONE_REG, &idreg);
474 }
475
476 bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
477 {
478 /* Identify the feature bits corresponding to the host CPU, and
479 * fill out the ARMHostCPUClass fields accordingly. To do this
480 * we have to create a scratch VM, create a single CPU inside it,
481 * and then query that CPU for the relevant ID registers.
482 */
483 int fdarray[3];
484 bool sve_supported;
485 uint64_t features = 0;
486 uint64_t t;
487 int err;
488
489 /* Old kernels may not know about the PREFERRED_TARGET ioctl: however
490 * we know these will only support creating one kind of guest CPU,
491 * which is its preferred CPU type. Fortunately these old kernels
492 * support only a very limited number of CPUs.
493 */
494 static const uint32_t cpus_to_try[] = {
495 KVM_ARM_TARGET_AEM_V8,
496 KVM_ARM_TARGET_FOUNDATION_V8,
497 KVM_ARM_TARGET_CORTEX_A57,
498 QEMU_KVM_ARM_TARGET_NONE
499 };
500 /*
501 * target = -1 informs kvm_arm_create_scratch_host_vcpu()
502 * to use the preferred target
503 */
504 struct kvm_vcpu_init init = { .target = -1, };
505
506 if (!kvm_arm_create_scratch_host_vcpu(cpus_to_try, fdarray, &init)) {
507 return false;
508 }
509
510 ahcf->target = init.target;
511 ahcf->dtb_compatible = "arm,arm-v8";
512
513 err = read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr0,
514 ARM64_SYS_REG(3, 0, 0, 4, 0));
515 if (unlikely(err < 0)) {
516 /*
517 * Before v4.15, the kernel only exposed a limited number of system
518 * registers, not including any of the interesting AArch64 ID regs.
519 * For the most part we could leave these fields as zero with minimal
520 * effect, since this does not affect the values seen by the guest.
521 *
522 * However, it could cause problems down the line for QEMU,
523 * so provide a minimal v8.0 default.
524 *
525 * ??? Could read MIDR and use knowledge from cpu64.c.
526 * ??? Could map a page of memory into our temp guest and
527 * run the tiniest of hand-crafted kernels to extract
528 * the values seen by the guest.
529 * ??? Either of these sounds like too much effort just
530 * to work around running a modern host kernel.
531 */
532 ahcf->isar.id_aa64pfr0 = 0x00000011; /* EL1&0, AArch64 only */
533 err = 0;
534 } else {
535 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr1,
536 ARM64_SYS_REG(3, 0, 0, 4, 1));
537 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr0,
538 ARM64_SYS_REG(3, 0, 0, 5, 0));
539 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr1,
540 ARM64_SYS_REG(3, 0, 0, 5, 1));
541 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar0,
542 ARM64_SYS_REG(3, 0, 0, 6, 0));
543 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar1,
544 ARM64_SYS_REG(3, 0, 0, 6, 1));
545 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr0,
546 ARM64_SYS_REG(3, 0, 0, 7, 0));
547 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr1,
548 ARM64_SYS_REG(3, 0, 0, 7, 1));
549 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr2,
550 ARM64_SYS_REG(3, 0, 0, 7, 2));
551
552 /*
553 * Note that if AArch32 support is not present in the host,
554 * the AArch32 sysregs are present to be read, but will
555 * return UNKNOWN values. This is neither better nor worse
556 * than skipping the reads and leaving 0, as we must avoid
557 * considering the values in every case.
558 */
559 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_dfr0,
560 ARM64_SYS_REG(3, 0, 0, 1, 2));
561 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr0,
562 ARM64_SYS_REG(3, 0, 0, 1, 4));
563 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr1,
564 ARM64_SYS_REG(3, 0, 0, 1, 5));
565 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr2,
566 ARM64_SYS_REG(3, 0, 0, 1, 6));
567 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr3,
568 ARM64_SYS_REG(3, 0, 0, 1, 7));
569 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar0,
570 ARM64_SYS_REG(3, 0, 0, 2, 0));
571 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar1,
572 ARM64_SYS_REG(3, 0, 0, 2, 1));
573 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar2,
574 ARM64_SYS_REG(3, 0, 0, 2, 2));
575 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar3,
576 ARM64_SYS_REG(3, 0, 0, 2, 3));
577 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar4,
578 ARM64_SYS_REG(3, 0, 0, 2, 4));
579 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar5,
580 ARM64_SYS_REG(3, 0, 0, 2, 5));
581 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr4,
582 ARM64_SYS_REG(3, 0, 0, 2, 6));
583 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar6,
584 ARM64_SYS_REG(3, 0, 0, 2, 7));
585
586 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr0,
587 ARM64_SYS_REG(3, 0, 0, 3, 0));
588 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr1,
589 ARM64_SYS_REG(3, 0, 0, 3, 1));
590 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr2,
591 ARM64_SYS_REG(3, 0, 0, 3, 2));
592
593 /*
594 * DBGDIDR is a bit complicated because the kernel doesn't
595 * provide an accessor for it in 64-bit mode, which is what this
596 * scratch VM is in, and there's no architected "64-bit sysreg
597 * which reads the same as the 32-bit register" the way there is
598 * for other ID registers. Instead we synthesize a value from the
599 * AArch64 ID_AA64DFR0, the same way the kernel code in
600 * arch/arm64/kvm/sys_regs.c:trap_dbgidr() does.
601 * We only do this if the CPU supports AArch32 at EL1.
602 */
603 if (FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL1) >= 2) {
604 int wrps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, WRPS);
605 int brps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, BRPS);
606 int ctx_cmps =
607 FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, CTX_CMPS);
608 int version = 6; /* ARMv8 debug architecture */
609 bool has_el3 =
610 !!FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL3);
611 uint32_t dbgdidr = 0;
612
613 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, WRPS, wrps);
614 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, BRPS, brps);
615 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, CTX_CMPS, ctx_cmps);
616 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, VERSION, version);
617 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, NSUHD_IMP, has_el3);
618 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, SE_IMP, has_el3);
619 dbgdidr |= (1 << 15); /* RES1 bit */
620 ahcf->isar.dbgdidr = dbgdidr;
621 }
622 }
623
624 sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) > 0;
625
626 kvm_arm_destroy_scratch_host_vcpu(fdarray);
627
628 if (err < 0) {
629 return false;
630 }
631
632 /* Add feature bits that can't appear until after VCPU init. */
633 if (sve_supported) {
634 t = ahcf->isar.id_aa64pfr0;
635 t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1);
636 ahcf->isar.id_aa64pfr0 = t;
637 }
638
639 /*
640 * We can assume any KVM supporting CPU is at least a v8
641 * with VFPv4+Neon; this in turn implies most of the other
642 * feature bits.
643 */
644 features |= 1ULL << ARM_FEATURE_V8;
645 features |= 1ULL << ARM_FEATURE_NEON;
646 features |= 1ULL << ARM_FEATURE_AARCH64;
647 features |= 1ULL << ARM_FEATURE_PMU;
648 features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
649
650 ahcf->features = features;
651
652 return true;
653 }
654
655 bool kvm_arm_aarch32_supported(void)
656 {
657 return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT);
658 }
659
660 bool kvm_arm_sve_supported(void)
661 {
662 return kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE);
663 }
664
665 QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1);
666
667 void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map)
668 {
669 /* Only call this function if kvm_arm_sve_supported() returns true. */
670 static uint64_t vls[KVM_ARM64_SVE_VLS_WORDS];
671 static bool probed;
672 uint32_t vq = 0;
673 int i, j;
674
675 bitmap_clear(map, 0, ARM_MAX_VQ);
676
677 /*
678 * KVM ensures all host CPUs support the same set of vector lengths.
679 * So we only need to create the scratch VCPUs once and then cache
680 * the results.
681 */
682 if (!probed) {
683 struct kvm_vcpu_init init = {
684 .target = -1,
685 .features[0] = (1 << KVM_ARM_VCPU_SVE),
686 };
687 struct kvm_one_reg reg = {
688 .id = KVM_REG_ARM64_SVE_VLS,
689 .addr = (uint64_t)&vls[0],
690 };
691 int fdarray[3], ret;
692
693 probed = true;
694
695 if (!kvm_arm_create_scratch_host_vcpu(NULL, fdarray, &init)) {
696 error_report("failed to create scratch VCPU with SVE enabled");
697 abort();
698 }
699 ret = ioctl(fdarray[2], KVM_GET_ONE_REG, &reg);
700 kvm_arm_destroy_scratch_host_vcpu(fdarray);
701 if (ret) {
702 error_report("failed to get KVM_REG_ARM64_SVE_VLS: %s",
703 strerror(errno));
704 abort();
705 }
706
707 for (i = KVM_ARM64_SVE_VLS_WORDS - 1; i >= 0; --i) {
708 if (vls[i]) {
709 vq = 64 - clz64(vls[i]) + i * 64;
710 break;
711 }
712 }
713 if (vq > ARM_MAX_VQ) {
714 warn_report("KVM supports vector lengths larger than "
715 "QEMU can enable");
716 }
717 }
718
719 for (i = 0; i < KVM_ARM64_SVE_VLS_WORDS; ++i) {
720 if (!vls[i]) {
721 continue;
722 }
723 for (j = 1; j <= 64; ++j) {
724 vq = j + i * 64;
725 if (vq > ARM_MAX_VQ) {
726 return;
727 }
728 if (vls[i] & (1UL << (j - 1))) {
729 set_bit(vq - 1, map);
730 }
731 }
732 }
733 }
734
735 static int kvm_arm_sve_set_vls(CPUState *cs)
736 {
737 uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = {0};
738 struct kvm_one_reg reg = {
739 .id = KVM_REG_ARM64_SVE_VLS,
740 .addr = (uint64_t)&vls[0],
741 };
742 ARMCPU *cpu = ARM_CPU(cs);
743 uint32_t vq;
744 int i, j;
745
746 assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX);
747
748 for (vq = 1; vq <= cpu->sve_max_vq; ++vq) {
749 if (test_bit(vq - 1, cpu->sve_vq_map)) {
750 i = (vq - 1) / 64;
751 j = (vq - 1) % 64;
752 vls[i] |= 1UL << j;
753 }
754 }
755
756 return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
757 }
758
759 #define ARM_CPU_ID_MPIDR 3, 0, 0, 0, 5
760
761 int kvm_arch_init_vcpu(CPUState *cs)
762 {
763 int ret;
764 uint64_t mpidr;
765 ARMCPU *cpu = ARM_CPU(cs);
766 CPUARMState *env = &cpu->env;
767
768 if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE ||
769 !object_dynamic_cast(OBJECT(cpu), TYPE_AARCH64_CPU)) {
770 error_report("KVM is not supported for this guest CPU type");
771 return -EINVAL;
772 }
773
774 qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, cs);
775
776 /* Determine init features for this CPU */
777 memset(cpu->kvm_init_features, 0, sizeof(cpu->kvm_init_features));
778 if (cpu->start_powered_off) {
779 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_POWER_OFF;
780 }
781 if (kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PSCI_0_2)) {
782 cpu->psci_version = 2;
783 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PSCI_0_2;
784 }
785 if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
786 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_EL1_32BIT;
787 }
788 if (!kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PMU_V3)) {
789 cpu->has_pmu = false;
790 }
791 if (cpu->has_pmu) {
792 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3;
793 } else {
794 env->features &= ~(1ULL << ARM_FEATURE_PMU);
795 }
796 if (cpu_isar_feature(aa64_sve, cpu)) {
797 assert(kvm_arm_sve_supported());
798 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE;
799 }
800
801 /* Do KVM_ARM_VCPU_INIT ioctl */
802 ret = kvm_arm_vcpu_init(cs);
803 if (ret) {
804 return ret;
805 }
806
807 if (cpu_isar_feature(aa64_sve, cpu)) {
808 ret = kvm_arm_sve_set_vls(cs);
809 if (ret) {
810 return ret;
811 }
812 ret = kvm_arm_vcpu_finalize(cs, KVM_ARM_VCPU_SVE);
813 if (ret) {
814 return ret;
815 }
816 }
817
818 /*
819 * When KVM is in use, PSCI is emulated in-kernel and not by qemu.
820 * Currently KVM has its own idea about MPIDR assignment, so we
821 * override our defaults with what we get from KVM.
822 */
823 ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr);
824 if (ret) {
825 return ret;
826 }
827 cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK;
828
829 kvm_arm_init_debug(cs);
830
831 /* Check whether user space can specify guest syndrome value */
832 kvm_arm_init_serror_injection(cs);
833
834 return kvm_arm_init_cpreg_list(cpu);
835 }
836
837 int kvm_arch_destroy_vcpu(CPUState *cs)
838 {
839 return 0;
840 }
841
842 bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx)
843 {
844 /* Return true if the regidx is a register we should synchronize
845 * via the cpreg_tuples array (ie is not a core or sve reg that
846 * we sync by hand in kvm_arch_get/put_registers())
847 */
848 switch (regidx & KVM_REG_ARM_COPROC_MASK) {
849 case KVM_REG_ARM_CORE:
850 case KVM_REG_ARM64_SVE:
851 return false;
852 default:
853 return true;
854 }
855 }
856
857 typedef struct CPRegStateLevel {
858 uint64_t regidx;
859 int level;
860 } CPRegStateLevel;
861
862 /* All system registers not listed in the following table are assumed to be
863 * of the level KVM_PUT_RUNTIME_STATE. If a register should be written less
864 * often, you must add it to this table with a state of either
865 * KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE.
866 */
867 static const CPRegStateLevel non_runtime_cpregs[] = {
868 { KVM_REG_ARM_TIMER_CNT, KVM_PUT_FULL_STATE },
869 };
870
871 int kvm_arm_cpreg_level(uint64_t regidx)
872 {
873 int i;
874
875 for (i = 0; i < ARRAY_SIZE(non_runtime_cpregs); i++) {
876 const CPRegStateLevel *l = &non_runtime_cpregs[i];
877 if (l->regidx == regidx) {
878 return l->level;
879 }
880 }
881
882 return KVM_PUT_RUNTIME_STATE;
883 }
884
885 /* Callers must hold the iothread mutex lock */
886 static void kvm_inject_arm_sea(CPUState *c)
887 {
888 ARMCPU *cpu = ARM_CPU(c);
889 CPUARMState *env = &cpu->env;
890 CPUClass *cc = CPU_GET_CLASS(c);
891 uint32_t esr;
892 bool same_el;
893
894 c->exception_index = EXCP_DATA_ABORT;
895 env->exception.target_el = 1;
896
897 /*
898 * Set the DFSC to synchronous external abort and set FnV to not valid,
899 * this will tell guest the FAR_ELx is UNKNOWN for this abort.
900 */
901 same_el = arm_current_el(env) == env->exception.target_el;
902 esr = syn_data_abort_no_iss(same_el, 1, 0, 0, 0, 0, 0x10);
903
904 env->exception.syndrome = esr;
905
906 cc->do_interrupt(c);
907 }
908
909 #define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \
910 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
911
912 #define AARCH64_SIMD_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \
913 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
914
915 #define AARCH64_SIMD_CTRL_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U32 | \
916 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
917
918 static int kvm_arch_put_fpsimd(CPUState *cs)
919 {
920 CPUARMState *env = &ARM_CPU(cs)->env;
921 struct kvm_one_reg reg;
922 int i, ret;
923
924 for (i = 0; i < 32; i++) {
925 uint64_t *q = aa64_vfp_qreg(env, i);
926 #ifdef HOST_WORDS_BIGENDIAN
927 uint64_t fp_val[2] = { q[1], q[0] };
928 reg.addr = (uintptr_t)fp_val;
929 #else
930 reg.addr = (uintptr_t)q;
931 #endif
932 reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]);
933 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
934 if (ret) {
935 return ret;
936 }
937 }
938
939 return 0;
940 }
941
942 /*
943 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits
944 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard
945 * code the slice index to zero for now as it's unlikely we'll need more than
946 * one slice for quite some time.
947 */
948 static int kvm_arch_put_sve(CPUState *cs)
949 {
950 ARMCPU *cpu = ARM_CPU(cs);
951 CPUARMState *env = &cpu->env;
952 uint64_t tmp[ARM_MAX_VQ * 2];
953 uint64_t *r;
954 struct kvm_one_reg reg;
955 int n, ret;
956
957 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) {
958 r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2);
959 reg.addr = (uintptr_t)r;
960 reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0);
961 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
962 if (ret) {
963 return ret;
964 }
965 }
966
967 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) {
968 r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0],
969 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
970 reg.addr = (uintptr_t)r;
971 reg.id = KVM_REG_ARM64_SVE_PREG(n, 0);
972 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
973 if (ret) {
974 return ret;
975 }
976 }
977
978 r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0],
979 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
980 reg.addr = (uintptr_t)r;
981 reg.id = KVM_REG_ARM64_SVE_FFR(0);
982 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
983 if (ret) {
984 return ret;
985 }
986
987 return 0;
988 }
989
990 int kvm_arch_put_registers(CPUState *cs, int level)
991 {
992 struct kvm_one_reg reg;
993 uint64_t val;
994 uint32_t fpr;
995 int i, ret;
996 unsigned int el;
997
998 ARMCPU *cpu = ARM_CPU(cs);
999 CPUARMState *env = &cpu->env;
1000
1001 /* If we are in AArch32 mode then we need to copy the AArch32 regs to the
1002 * AArch64 registers before pushing them out to 64-bit KVM.
1003 */
1004 if (!is_a64(env)) {
1005 aarch64_sync_32_to_64(env);
1006 }
1007
1008 for (i = 0; i < 31; i++) {
1009 reg.id = AARCH64_CORE_REG(regs.regs[i]);
1010 reg.addr = (uintptr_t) &env->xregs[i];
1011 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1012 if (ret) {
1013 return ret;
1014 }
1015 }
1016
1017 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
1018 * QEMU side we keep the current SP in xregs[31] as well.
1019 */
1020 aarch64_save_sp(env, 1);
1021
1022 reg.id = AARCH64_CORE_REG(regs.sp);
1023 reg.addr = (uintptr_t) &env->sp_el[0];
1024 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1025 if (ret) {
1026 return ret;
1027 }
1028
1029 reg.id = AARCH64_CORE_REG(sp_el1);
1030 reg.addr = (uintptr_t) &env->sp_el[1];
1031 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1032 if (ret) {
1033 return ret;
1034 }
1035
1036 /* Note that KVM thinks pstate is 64 bit but we use a uint32_t */
1037 if (is_a64(env)) {
1038 val = pstate_read(env);
1039 } else {
1040 val = cpsr_read(env);
1041 }
1042 reg.id = AARCH64_CORE_REG(regs.pstate);
1043 reg.addr = (uintptr_t) &val;
1044 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1045 if (ret) {
1046 return ret;
1047 }
1048
1049 reg.id = AARCH64_CORE_REG(regs.pc);
1050 reg.addr = (uintptr_t) &env->pc;
1051 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1052 if (ret) {
1053 return ret;
1054 }
1055
1056 reg.id = AARCH64_CORE_REG(elr_el1);
1057 reg.addr = (uintptr_t) &env->elr_el[1];
1058 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1059 if (ret) {
1060 return ret;
1061 }
1062
1063 /* Saved Program State Registers
1064 *
1065 * Before we restore from the banked_spsr[] array we need to
1066 * ensure that any modifications to env->spsr are correctly
1067 * reflected in the banks.
1068 */
1069 el = arm_current_el(env);
1070 if (el > 0 && !is_a64(env)) {
1071 i = bank_number(env->uncached_cpsr & CPSR_M);
1072 env->banked_spsr[i] = env->spsr;
1073 }
1074
1075 /* KVM 0-4 map to QEMU banks 1-5 */
1076 for (i = 0; i < KVM_NR_SPSR; i++) {
1077 reg.id = AARCH64_CORE_REG(spsr[i]);
1078 reg.addr = (uintptr_t) &env->banked_spsr[i + 1];
1079 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1080 if (ret) {
1081 return ret;
1082 }
1083 }
1084
1085 if (cpu_isar_feature(aa64_sve, cpu)) {
1086 ret = kvm_arch_put_sve(cs);
1087 } else {
1088 ret = kvm_arch_put_fpsimd(cs);
1089 }
1090 if (ret) {
1091 return ret;
1092 }
1093
1094 reg.addr = (uintptr_t)(&fpr);
1095 fpr = vfp_get_fpsr(env);
1096 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr);
1097 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1098 if (ret) {
1099 return ret;
1100 }
1101
1102 reg.addr = (uintptr_t)(&fpr);
1103 fpr = vfp_get_fpcr(env);
1104 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr);
1105 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1106 if (ret) {
1107 return ret;
1108 }
1109
1110 write_cpustate_to_list(cpu, true);
1111
1112 if (!write_list_to_kvmstate(cpu, level)) {
1113 return -EINVAL;
1114 }
1115
1116 /*
1117 * Setting VCPU events should be triggered after syncing the registers
1118 * to avoid overwriting potential changes made by KVM upon calling
1119 * KVM_SET_VCPU_EVENTS ioctl
1120 */
1121 ret = kvm_put_vcpu_events(cpu);
1122 if (ret) {
1123 return ret;
1124 }
1125
1126 kvm_arm_sync_mpstate_to_kvm(cpu);
1127
1128 return ret;
1129 }
1130
1131 static int kvm_arch_get_fpsimd(CPUState *cs)
1132 {
1133 CPUARMState *env = &ARM_CPU(cs)->env;
1134 struct kvm_one_reg reg;
1135 int i, ret;
1136
1137 for (i = 0; i < 32; i++) {
1138 uint64_t *q = aa64_vfp_qreg(env, i);
1139 reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]);
1140 reg.addr = (uintptr_t)q;
1141 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1142 if (ret) {
1143 return ret;
1144 } else {
1145 #ifdef HOST_WORDS_BIGENDIAN
1146 uint64_t t;
1147 t = q[0], q[0] = q[1], q[1] = t;
1148 #endif
1149 }
1150 }
1151
1152 return 0;
1153 }
1154
1155 /*
1156 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits
1157 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard
1158 * code the slice index to zero for now as it's unlikely we'll need more than
1159 * one slice for quite some time.
1160 */
1161 static int kvm_arch_get_sve(CPUState *cs)
1162 {
1163 ARMCPU *cpu = ARM_CPU(cs);
1164 CPUARMState *env = &cpu->env;
1165 struct kvm_one_reg reg;
1166 uint64_t *r;
1167 int n, ret;
1168
1169 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) {
1170 r = &env->vfp.zregs[n].d[0];
1171 reg.addr = (uintptr_t)r;
1172 reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0);
1173 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1174 if (ret) {
1175 return ret;
1176 }
1177 sve_bswap64(r, r, cpu->sve_max_vq * 2);
1178 }
1179
1180 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) {
1181 r = &env->vfp.pregs[n].p[0];
1182 reg.addr = (uintptr_t)r;
1183 reg.id = KVM_REG_ARM64_SVE_PREG(n, 0);
1184 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1185 if (ret) {
1186 return ret;
1187 }
1188 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
1189 }
1190
1191 r = &env->vfp.pregs[FFR_PRED_NUM].p[0];
1192 reg.addr = (uintptr_t)r;
1193 reg.id = KVM_REG_ARM64_SVE_FFR(0);
1194 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1195 if (ret) {
1196 return ret;
1197 }
1198 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
1199
1200 return 0;
1201 }
1202
1203 int kvm_arch_get_registers(CPUState *cs)
1204 {
1205 struct kvm_one_reg reg;
1206 uint64_t val;
1207 unsigned int el;
1208 uint32_t fpr;
1209 int i, ret;
1210
1211 ARMCPU *cpu = ARM_CPU(cs);
1212 CPUARMState *env = &cpu->env;
1213
1214 for (i = 0; i < 31; i++) {
1215 reg.id = AARCH64_CORE_REG(regs.regs[i]);
1216 reg.addr = (uintptr_t) &env->xregs[i];
1217 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1218 if (ret) {
1219 return ret;
1220 }
1221 }
1222
1223 reg.id = AARCH64_CORE_REG(regs.sp);
1224 reg.addr = (uintptr_t) &env->sp_el[0];
1225 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1226 if (ret) {
1227 return ret;
1228 }
1229
1230 reg.id = AARCH64_CORE_REG(sp_el1);
1231 reg.addr = (uintptr_t) &env->sp_el[1];
1232 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1233 if (ret) {
1234 return ret;
1235 }
1236
1237 reg.id = AARCH64_CORE_REG(regs.pstate);
1238 reg.addr = (uintptr_t) &val;
1239 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1240 if (ret) {
1241 return ret;
1242 }
1243
1244 env->aarch64 = ((val & PSTATE_nRW) == 0);
1245 if (is_a64(env)) {
1246 pstate_write(env, val);
1247 } else {
1248 cpsr_write(env, val, 0xffffffff, CPSRWriteRaw);
1249 }
1250
1251 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
1252 * QEMU side we keep the current SP in xregs[31] as well.
1253 */
1254 aarch64_restore_sp(env, 1);
1255
1256 reg.id = AARCH64_CORE_REG(regs.pc);
1257 reg.addr = (uintptr_t) &env->pc;
1258 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1259 if (ret) {
1260 return ret;
1261 }
1262
1263 /* If we are in AArch32 mode then we need to sync the AArch32 regs with the
1264 * incoming AArch64 regs received from 64-bit KVM.
1265 * We must perform this after all of the registers have been acquired from
1266 * the kernel.
1267 */
1268 if (!is_a64(env)) {
1269 aarch64_sync_64_to_32(env);
1270 }
1271
1272 reg.id = AARCH64_CORE_REG(elr_el1);
1273 reg.addr = (uintptr_t) &env->elr_el[1];
1274 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1275 if (ret) {
1276 return ret;
1277 }
1278
1279 /* Fetch the SPSR registers
1280 *
1281 * KVM SPSRs 0-4 map to QEMU banks 1-5
1282 */
1283 for (i = 0; i < KVM_NR_SPSR; i++) {
1284 reg.id = AARCH64_CORE_REG(spsr[i]);
1285 reg.addr = (uintptr_t) &env->banked_spsr[i + 1];
1286 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1287 if (ret) {
1288 return ret;
1289 }
1290 }
1291
1292 el = arm_current_el(env);
1293 if (el > 0 && !is_a64(env)) {
1294 i = bank_number(env->uncached_cpsr & CPSR_M);
1295 env->spsr = env->banked_spsr[i];
1296 }
1297
1298 if (cpu_isar_feature(aa64_sve, cpu)) {
1299 ret = kvm_arch_get_sve(cs);
1300 } else {
1301 ret = kvm_arch_get_fpsimd(cs);
1302 }
1303 if (ret) {
1304 return ret;
1305 }
1306
1307 reg.addr = (uintptr_t)(&fpr);
1308 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr);
1309 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1310 if (ret) {
1311 return ret;
1312 }
1313 vfp_set_fpsr(env, fpr);
1314
1315 reg.addr = (uintptr_t)(&fpr);
1316 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr);
1317 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1318 if (ret) {
1319 return ret;
1320 }
1321 vfp_set_fpcr(env, fpr);
1322
1323 ret = kvm_get_vcpu_events(cpu);
1324 if (ret) {
1325 return ret;
1326 }
1327
1328 if (!write_kvmstate_to_list(cpu)) {
1329 return -EINVAL;
1330 }
1331 /* Note that it's OK to have registers which aren't in CPUState,
1332 * so we can ignore a failure return here.
1333 */
1334 write_list_to_cpustate(cpu);
1335
1336 kvm_arm_sync_mpstate_to_qemu(cpu);
1337
1338 /* TODO: other registers */
1339 return ret;
1340 }
1341
1342 void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr)
1343 {
1344 ram_addr_t ram_addr;
1345 hwaddr paddr;
1346 Object *obj = qdev_get_machine();
1347 VirtMachineState *vms = VIRT_MACHINE(obj);
1348 bool acpi_enabled = virt_is_acpi_enabled(vms);
1349
1350 assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO);
1351
1352 if (acpi_enabled && addr &&
1353 object_property_get_bool(obj, "ras", NULL)) {
1354 ram_addr = qemu_ram_addr_from_host(addr);
1355 if (ram_addr != RAM_ADDR_INVALID &&
1356 kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) {
1357 kvm_hwpoison_page_add(ram_addr);
1358 /*
1359 * If this is a BUS_MCEERR_AR, we know we have been called
1360 * synchronously from the vCPU thread, so we can easily
1361 * synchronize the state and inject an error.
1362 *
1363 * TODO: we currently don't tell the guest at all about
1364 * BUS_MCEERR_AO. In that case we might either be being
1365 * called synchronously from the vCPU thread, or a bit
1366 * later from the main thread, so doing the injection of
1367 * the error would be more complicated.
1368 */
1369 if (code == BUS_MCEERR_AR) {
1370 kvm_cpu_synchronize_state(c);
1371 if (!acpi_ghes_record_errors(ACPI_HEST_SRC_ID_SEA, paddr)) {
1372 kvm_inject_arm_sea(c);
1373 } else {
1374 error_report("failed to record the error");
1375 abort();
1376 }
1377 }
1378 return;
1379 }
1380 if (code == BUS_MCEERR_AO) {
1381 error_report("Hardware memory error at addr %p for memory used by "
1382 "QEMU itself instead of guest system!", addr);
1383 }
1384 }
1385
1386 if (code == BUS_MCEERR_AR) {
1387 error_report("Hardware memory error!");
1388 exit(1);
1389 }
1390 }
1391
1392 /* C6.6.29 BRK instruction */
1393 static const uint32_t brk_insn = 0xd4200000;
1394
1395 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
1396 {
1397 if (have_guest_debug) {
1398 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) ||
1399 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) {
1400 return -EINVAL;
1401 }
1402 return 0;
1403 } else {
1404 error_report("guest debug not supported on this kernel");
1405 return -EINVAL;
1406 }
1407 }
1408
1409 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
1410 {
1411 static uint32_t brk;
1412
1413 if (have_guest_debug) {
1414 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) ||
1415 brk != brk_insn ||
1416 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) {
1417 return -EINVAL;
1418 }
1419 return 0;
1420 } else {
1421 error_report("guest debug not supported on this kernel");
1422 return -EINVAL;
1423 }
1424 }
1425
1426 /* See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register
1427 *
1428 * To minimise translating between kernel and user-space the kernel
1429 * ABI just provides user-space with the full exception syndrome
1430 * register value to be decoded in QEMU.
1431 */
1432
1433 bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit)
1434 {
1435 int hsr_ec = syn_get_ec(debug_exit->hsr);
1436 ARMCPU *cpu = ARM_CPU(cs);
1437 CPUClass *cc = CPU_GET_CLASS(cs);
1438 CPUARMState *env = &cpu->env;
1439
1440 /* Ensure PC is synchronised */
1441 kvm_cpu_synchronize_state(cs);
1442
1443 switch (hsr_ec) {
1444 case EC_SOFTWARESTEP:
1445 if (cs->singlestep_enabled) {
1446 return true;
1447 } else {
1448 /*
1449 * The kernel should have suppressed the guest's ability to
1450 * single step at this point so something has gone wrong.
1451 */
1452 error_report("%s: guest single-step while debugging unsupported"
1453 " (%"PRIx64", %"PRIx32")",
1454 __func__, env->pc, debug_exit->hsr);
1455 return false;
1456 }
1457 break;
1458 case EC_AA64_BKPT:
1459 if (kvm_find_sw_breakpoint(cs, env->pc)) {
1460 return true;
1461 }
1462 break;
1463 case EC_BREAKPOINT:
1464 if (find_hw_breakpoint(cs, env->pc)) {
1465 return true;
1466 }
1467 break;
1468 case EC_WATCHPOINT:
1469 {
1470 CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far);
1471 if (wp) {
1472 cs->watchpoint_hit = wp;
1473 return true;
1474 }
1475 break;
1476 }
1477 default:
1478 error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")",
1479 __func__, debug_exit->hsr, env->pc);
1480 }
1481
1482 /* If we are not handling the debug exception it must belong to
1483 * the guest. Let's re-use the existing TCG interrupt code to set
1484 * everything up properly.
1485 */
1486 cs->exception_index = EXCP_BKPT;
1487 env->exception.syndrome = debug_exit->hsr;
1488 env->exception.vaddress = debug_exit->far;
1489 env->exception.target_el = 1;
1490 qemu_mutex_lock_iothread();
1491 cc->do_interrupt(cs);
1492 qemu_mutex_unlock_iothread();
1493
1494 return false;
1495 }
1496
1497 #define ARM64_REG_ESR_EL1 ARM64_SYS_REG(3, 0, 5, 2, 0)
1498 #define ARM64_REG_TCR_EL1 ARM64_SYS_REG(3, 0, 2, 0, 2)
1499
1500 /*
1501 * ESR_EL1
1502 * ISS encoding
1503 * AARCH64: DFSC, bits [5:0]
1504 * AARCH32:
1505 * TTBCR.EAE == 0
1506 * FS[4] - DFSR[10]
1507 * FS[3:0] - DFSR[3:0]
1508 * TTBCR.EAE == 1
1509 * FS, bits [5:0]
1510 */
1511 #define ESR_DFSC(aarch64, lpae, v) \
1512 ((aarch64 || (lpae)) ? ((v) & 0x3F) \
1513 : (((v) >> 6) | ((v) & 0x1F)))
1514
1515 #define ESR_DFSC_EXTABT(aarch64, lpae) \
1516 ((aarch64) ? 0x10 : (lpae) ? 0x10 : 0x8)
1517
1518 bool kvm_arm_verify_ext_dabt_pending(CPUState *cs)
1519 {
1520 uint64_t dfsr_val;
1521
1522 if (!kvm_get_one_reg(cs, ARM64_REG_ESR_EL1, &dfsr_val)) {
1523 ARMCPU *cpu = ARM_CPU(cs);
1524 CPUARMState *env = &cpu->env;
1525 int aarch64_mode = arm_feature(env, ARM_FEATURE_AARCH64);
1526 int lpae = 0;
1527
1528 if (!aarch64_mode) {
1529 uint64_t ttbcr;
1530
1531 if (!kvm_get_one_reg(cs, ARM64_REG_TCR_EL1, &ttbcr)) {
1532 lpae = arm_feature(env, ARM_FEATURE_LPAE)
1533 && (ttbcr & TTBCR_EAE);
1534 }
1535 }
1536 /*
1537 * The verification here is based on the DFSC bits
1538 * of the ESR_EL1 reg only
1539 */
1540 return (ESR_DFSC(aarch64_mode, lpae, dfsr_val) ==
1541 ESR_DFSC_EXTABT(aarch64_mode, lpae));
1542 }
1543 return false;
1544 }