stubs: Move qemu_fd_register stub to util/main-loop.c
[qemu.git] / target / riscv / cpu_helper.c
1 /*
2 * RISC-V CPU helpers for qemu.
3 *
4 * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
5 * Copyright (c) 2017-2018 SiFive, Inc.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2 or later, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "qemu/log.h"
22 #include "qemu/main-loop.h"
23 #include "cpu.h"
24 #include "exec/exec-all.h"
25 #include "tcg/tcg-op.h"
26 #include "trace.h"
27
28 int riscv_cpu_mmu_index(CPURISCVState *env, bool ifetch)
29 {
30 #ifdef CONFIG_USER_ONLY
31 return 0;
32 #else
33 return env->priv;
34 #endif
35 }
36
37 #ifndef CONFIG_USER_ONLY
38 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
39 {
40 target_ulong irqs;
41
42 target_ulong mstatus_mie = get_field(env->mstatus, MSTATUS_MIE);
43 target_ulong mstatus_sie = get_field(env->mstatus, MSTATUS_SIE);
44 target_ulong hs_mstatus_sie = get_field(env->mstatus_hs, MSTATUS_SIE);
45
46 target_ulong pending = env->mip & env->mie &
47 ~(MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
48 target_ulong vspending = (env->mip & env->mie &
49 (MIP_VSSIP | MIP_VSTIP | MIP_VSEIP));
50
51 target_ulong mie = env->priv < PRV_M ||
52 (env->priv == PRV_M && mstatus_mie);
53 target_ulong sie = env->priv < PRV_S ||
54 (env->priv == PRV_S && mstatus_sie);
55 target_ulong hs_sie = env->priv < PRV_S ||
56 (env->priv == PRV_S && hs_mstatus_sie);
57
58 if (riscv_cpu_virt_enabled(env)) {
59 target_ulong pending_hs_irq = pending & -hs_sie;
60
61 if (pending_hs_irq) {
62 riscv_cpu_set_force_hs_excep(env, FORCE_HS_EXCEP);
63 return ctz64(pending_hs_irq);
64 }
65
66 pending = vspending;
67 }
68
69 irqs = (pending & ~env->mideleg & -mie) | (pending & env->mideleg & -sie);
70
71 if (irqs) {
72 return ctz64(irqs); /* since non-zero */
73 } else {
74 return EXCP_NONE; /* indicates no pending interrupt */
75 }
76 }
77 #endif
78
79 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
80 {
81 #if !defined(CONFIG_USER_ONLY)
82 if (interrupt_request & CPU_INTERRUPT_HARD) {
83 RISCVCPU *cpu = RISCV_CPU(cs);
84 CPURISCVState *env = &cpu->env;
85 int interruptno = riscv_cpu_local_irq_pending(env);
86 if (interruptno >= 0) {
87 cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
88 riscv_cpu_do_interrupt(cs);
89 return true;
90 }
91 }
92 #endif
93 return false;
94 }
95
96 #if !defined(CONFIG_USER_ONLY)
97
98 /* Return true is floating point support is currently enabled */
99 bool riscv_cpu_fp_enabled(CPURISCVState *env)
100 {
101 if (env->mstatus & MSTATUS_FS) {
102 if (riscv_cpu_virt_enabled(env) && !(env->mstatus_hs & MSTATUS_FS)) {
103 return false;
104 }
105 return true;
106 }
107
108 return false;
109 }
110
111 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
112 {
113 target_ulong mstatus_mask = MSTATUS_MXR | MSTATUS_SUM | MSTATUS_FS |
114 MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE;
115 bool current_virt = riscv_cpu_virt_enabled(env);
116
117 g_assert(riscv_has_ext(env, RVH));
118
119 #if defined(TARGET_RISCV64)
120 mstatus_mask |= MSTATUS64_UXL;
121 #endif
122
123 if (current_virt) {
124 /* Current V=1 and we are about to change to V=0 */
125 env->vsstatus = env->mstatus & mstatus_mask;
126 env->mstatus &= ~mstatus_mask;
127 env->mstatus |= env->mstatus_hs;
128
129 #if defined(TARGET_RISCV32)
130 env->vsstatush = env->mstatush;
131 env->mstatush |= env->mstatush_hs;
132 #endif
133
134 env->vstvec = env->stvec;
135 env->stvec = env->stvec_hs;
136
137 env->vsscratch = env->sscratch;
138 env->sscratch = env->sscratch_hs;
139
140 env->vsepc = env->sepc;
141 env->sepc = env->sepc_hs;
142
143 env->vscause = env->scause;
144 env->scause = env->scause_hs;
145
146 env->vstval = env->sbadaddr;
147 env->sbadaddr = env->stval_hs;
148
149 env->vsatp = env->satp;
150 env->satp = env->satp_hs;
151 } else {
152 /* Current V=0 and we are about to change to V=1 */
153 env->mstatus_hs = env->mstatus & mstatus_mask;
154 env->mstatus &= ~mstatus_mask;
155 env->mstatus |= env->vsstatus;
156
157 #if defined(TARGET_RISCV32)
158 env->mstatush_hs = env->mstatush;
159 env->mstatush |= env->vsstatush;
160 #endif
161
162 env->stvec_hs = env->stvec;
163 env->stvec = env->vstvec;
164
165 env->sscratch_hs = env->sscratch;
166 env->sscratch = env->vsscratch;
167
168 env->sepc_hs = env->sepc;
169 env->sepc = env->vsepc;
170
171 env->scause_hs = env->scause;
172 env->scause = env->vscause;
173
174 env->stval_hs = env->sbadaddr;
175 env->sbadaddr = env->vstval;
176
177 env->satp_hs = env->satp;
178 env->satp = env->vsatp;
179 }
180 }
181
182 bool riscv_cpu_virt_enabled(CPURISCVState *env)
183 {
184 if (!riscv_has_ext(env, RVH)) {
185 return false;
186 }
187
188 return get_field(env->virt, VIRT_ONOFF);
189 }
190
191 void riscv_cpu_set_virt_enabled(CPURISCVState *env, bool enable)
192 {
193 if (!riscv_has_ext(env, RVH)) {
194 return;
195 }
196
197 /* Flush the TLB on all virt mode changes. */
198 if (get_field(env->virt, VIRT_ONOFF) != enable) {
199 tlb_flush(env_cpu(env));
200 }
201
202 env->virt = set_field(env->virt, VIRT_ONOFF, enable);
203 }
204
205 bool riscv_cpu_force_hs_excep_enabled(CPURISCVState *env)
206 {
207 if (!riscv_has_ext(env, RVH)) {
208 return false;
209 }
210
211 return get_field(env->virt, FORCE_HS_EXCEP);
212 }
213
214 void riscv_cpu_set_force_hs_excep(CPURISCVState *env, bool enable)
215 {
216 if (!riscv_has_ext(env, RVH)) {
217 return;
218 }
219
220 env->virt = set_field(env->virt, FORCE_HS_EXCEP, enable);
221 }
222
223 bool riscv_cpu_two_stage_lookup(CPURISCVState *env)
224 {
225 if (!riscv_has_ext(env, RVH)) {
226 return false;
227 }
228
229 return get_field(env->virt, HS_TWO_STAGE);
230 }
231
232 void riscv_cpu_set_two_stage_lookup(CPURISCVState *env, bool enable)
233 {
234 if (!riscv_has_ext(env, RVH)) {
235 return;
236 }
237
238 env->virt = set_field(env->virt, HS_TWO_STAGE, enable);
239 }
240
241 int riscv_cpu_claim_interrupts(RISCVCPU *cpu, uint32_t interrupts)
242 {
243 CPURISCVState *env = &cpu->env;
244 if (env->miclaim & interrupts) {
245 return -1;
246 } else {
247 env->miclaim |= interrupts;
248 return 0;
249 }
250 }
251
252 uint32_t riscv_cpu_update_mip(RISCVCPU *cpu, uint32_t mask, uint32_t value)
253 {
254 CPURISCVState *env = &cpu->env;
255 CPUState *cs = CPU(cpu);
256 uint32_t old = env->mip;
257 bool locked = false;
258
259 if (!qemu_mutex_iothread_locked()) {
260 locked = true;
261 qemu_mutex_lock_iothread();
262 }
263
264 env->mip = (env->mip & ~mask) | (value & mask);
265
266 if (env->mip) {
267 cpu_interrupt(cs, CPU_INTERRUPT_HARD);
268 } else {
269 cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
270 }
271
272 if (locked) {
273 qemu_mutex_unlock_iothread();
274 }
275
276 return old;
277 }
278
279 void riscv_cpu_set_rdtime_fn(CPURISCVState *env, uint64_t (*fn)(void))
280 {
281 env->rdtime_fn = fn;
282 }
283
284 void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
285 {
286 if (newpriv > PRV_M) {
287 g_assert_not_reached();
288 }
289 if (newpriv == PRV_H) {
290 newpriv = PRV_U;
291 }
292 /* tlb_flush is unnecessary as mode is contained in mmu_idx */
293 env->priv = newpriv;
294
295 /*
296 * Clear the load reservation - otherwise a reservation placed in one
297 * context/process can be used by another, resulting in an SC succeeding
298 * incorrectly. Version 2.2 of the ISA specification explicitly requires
299 * this behaviour, while later revisions say that the kernel "should" use
300 * an SC instruction to force the yielding of a load reservation on a
301 * preemptive context switch. As a result, do both.
302 */
303 env->load_res = -1;
304 }
305
306 /* get_physical_address - get the physical address for this virtual address
307 *
308 * Do a page table walk to obtain the physical address corresponding to a
309 * virtual address. Returns 0 if the translation was successful
310 *
311 * Adapted from Spike's mmu_t::translate and mmu_t::walk
312 *
313 * @env: CPURISCVState
314 * @physical: This will be set to the calculated physical address
315 * @prot: The returned protection attributes
316 * @addr: The virtual address to be translated
317 * @access_type: The type of MMU access
318 * @mmu_idx: Indicates current privilege level
319 * @first_stage: Are we in first stage translation?
320 * Second stage is used for hypervisor guest translation
321 * @two_stage: Are we going to perform two stage translation
322 */
323 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
324 int *prot, target_ulong addr,
325 int access_type, int mmu_idx,
326 bool first_stage, bool two_stage)
327 {
328 /* NOTE: the env->pc value visible here will not be
329 * correct, but the value visible to the exception handler
330 * (riscv_cpu_do_interrupt) is correct */
331 MemTxResult res;
332 MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
333 int mode = mmu_idx;
334 bool use_background = false;
335
336 /*
337 * Check if we should use the background registers for the two
338 * stage translation. We don't need to check if we actually need
339 * two stage translation as that happened before this function
340 * was called. Background registers will be used if the guest has
341 * forced a two stage translation to be on (in HS or M mode).
342 */
343 if (riscv_cpu_two_stage_lookup(env) && access_type != MMU_INST_FETCH) {
344 use_background = true;
345 }
346
347 if (mode == PRV_M && access_type != MMU_INST_FETCH) {
348 if (get_field(env->mstatus, MSTATUS_MPRV)) {
349 mode = get_field(env->mstatus, MSTATUS_MPP);
350 }
351 }
352
353 if (first_stage == false) {
354 /* We are in stage 2 translation, this is similar to stage 1. */
355 /* Stage 2 is always taken as U-mode */
356 mode = PRV_U;
357 }
358
359 if (mode == PRV_M || !riscv_feature(env, RISCV_FEATURE_MMU)) {
360 *physical = addr;
361 *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
362 return TRANSLATE_SUCCESS;
363 }
364
365 *prot = 0;
366
367 hwaddr base;
368 int levels, ptidxbits, ptesize, vm, sum, mxr, widened;
369
370 if (first_stage == true) {
371 mxr = get_field(env->mstatus, MSTATUS_MXR);
372 } else {
373 mxr = get_field(env->vsstatus, MSTATUS_MXR);
374 }
375
376 if (first_stage == true) {
377 if (use_background) {
378 base = (hwaddr)get_field(env->vsatp, SATP_PPN) << PGSHIFT;
379 vm = get_field(env->vsatp, SATP_MODE);
380 } else {
381 base = (hwaddr)get_field(env->satp, SATP_PPN) << PGSHIFT;
382 vm = get_field(env->satp, SATP_MODE);
383 }
384 widened = 0;
385 } else {
386 base = (hwaddr)get_field(env->hgatp, HGATP_PPN) << PGSHIFT;
387 vm = get_field(env->hgatp, HGATP_MODE);
388 widened = 2;
389 }
390 sum = get_field(env->mstatus, MSTATUS_SUM);
391 switch (vm) {
392 case VM_1_10_SV32:
393 levels = 2; ptidxbits = 10; ptesize = 4; break;
394 case VM_1_10_SV39:
395 levels = 3; ptidxbits = 9; ptesize = 8; break;
396 case VM_1_10_SV48:
397 levels = 4; ptidxbits = 9; ptesize = 8; break;
398 case VM_1_10_SV57:
399 levels = 5; ptidxbits = 9; ptesize = 8; break;
400 case VM_1_10_MBARE:
401 *physical = addr;
402 *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
403 return TRANSLATE_SUCCESS;
404 default:
405 g_assert_not_reached();
406 }
407
408 CPUState *cs = env_cpu(env);
409 int va_bits = PGSHIFT + levels * ptidxbits + widened;
410 target_ulong mask, masked_msbs;
411
412 if (TARGET_LONG_BITS > (va_bits - 1)) {
413 mask = (1L << (TARGET_LONG_BITS - (va_bits - 1))) - 1;
414 } else {
415 mask = 0;
416 }
417 masked_msbs = (addr >> (va_bits - 1)) & mask;
418
419 if (masked_msbs != 0 && masked_msbs != mask) {
420 return TRANSLATE_FAIL;
421 }
422
423 int ptshift = (levels - 1) * ptidxbits;
424 int i;
425
426 #if !TCG_OVERSIZED_GUEST
427 restart:
428 #endif
429 for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
430 target_ulong idx;
431 if (i == 0) {
432 idx = (addr >> (PGSHIFT + ptshift)) &
433 ((1 << (ptidxbits + widened)) - 1);
434 } else {
435 idx = (addr >> (PGSHIFT + ptshift)) &
436 ((1 << ptidxbits) - 1);
437 }
438
439 /* check that physical address of PTE is legal */
440 hwaddr pte_addr;
441
442 if (two_stage && first_stage) {
443 int vbase_prot;
444 hwaddr vbase;
445
446 /* Do the second stage translation on the base PTE address. */
447 int vbase_ret = get_physical_address(env, &vbase, &vbase_prot,
448 base, MMU_DATA_LOAD,
449 mmu_idx, false, true);
450
451 if (vbase_ret != TRANSLATE_SUCCESS) {
452 return vbase_ret;
453 }
454
455 pte_addr = vbase + idx * ptesize;
456 } else {
457 pte_addr = base + idx * ptesize;
458 }
459
460 if (riscv_feature(env, RISCV_FEATURE_PMP) &&
461 !pmp_hart_has_privs(env, pte_addr, sizeof(target_ulong),
462 1 << MMU_DATA_LOAD, PRV_S)) {
463 return TRANSLATE_PMP_FAIL;
464 }
465
466 #if defined(TARGET_RISCV32)
467 target_ulong pte = address_space_ldl(cs->as, pte_addr, attrs, &res);
468 #elif defined(TARGET_RISCV64)
469 target_ulong pte = address_space_ldq(cs->as, pte_addr, attrs, &res);
470 #endif
471 if (res != MEMTX_OK) {
472 return TRANSLATE_FAIL;
473 }
474
475 hwaddr ppn = pte >> PTE_PPN_SHIFT;
476
477 if (!(pte & PTE_V)) {
478 /* Invalid PTE */
479 return TRANSLATE_FAIL;
480 } else if (!(pte & (PTE_R | PTE_W | PTE_X))) {
481 /* Inner PTE, continue walking */
482 base = ppn << PGSHIFT;
483 } else if ((pte & (PTE_R | PTE_W | PTE_X)) == PTE_W) {
484 /* Reserved leaf PTE flags: PTE_W */
485 return TRANSLATE_FAIL;
486 } else if ((pte & (PTE_R | PTE_W | PTE_X)) == (PTE_W | PTE_X)) {
487 /* Reserved leaf PTE flags: PTE_W + PTE_X */
488 return TRANSLATE_FAIL;
489 } else if ((pte & PTE_U) && ((mode != PRV_U) &&
490 (!sum || access_type == MMU_INST_FETCH))) {
491 /* User PTE flags when not U mode and mstatus.SUM is not set,
492 or the access type is an instruction fetch */
493 return TRANSLATE_FAIL;
494 } else if (!(pte & PTE_U) && (mode != PRV_S)) {
495 /* Supervisor PTE flags when not S mode */
496 return TRANSLATE_FAIL;
497 } else if (ppn & ((1ULL << ptshift) - 1)) {
498 /* Misaligned PPN */
499 return TRANSLATE_FAIL;
500 } else if (access_type == MMU_DATA_LOAD && !((pte & PTE_R) ||
501 ((pte & PTE_X) && mxr))) {
502 /* Read access check failed */
503 return TRANSLATE_FAIL;
504 } else if (access_type == MMU_DATA_STORE && !(pte & PTE_W)) {
505 /* Write access check failed */
506 return TRANSLATE_FAIL;
507 } else if (access_type == MMU_INST_FETCH && !(pte & PTE_X)) {
508 /* Fetch access check failed */
509 return TRANSLATE_FAIL;
510 } else {
511 /* if necessary, set accessed and dirty bits. */
512 target_ulong updated_pte = pte | PTE_A |
513 (access_type == MMU_DATA_STORE ? PTE_D : 0);
514
515 /* Page table updates need to be atomic with MTTCG enabled */
516 if (updated_pte != pte) {
517 /*
518 * - if accessed or dirty bits need updating, and the PTE is
519 * in RAM, then we do so atomically with a compare and swap.
520 * - if the PTE is in IO space or ROM, then it can't be updated
521 * and we return TRANSLATE_FAIL.
522 * - if the PTE changed by the time we went to update it, then
523 * it is no longer valid and we must re-walk the page table.
524 */
525 MemoryRegion *mr;
526 hwaddr l = sizeof(target_ulong), addr1;
527 mr = address_space_translate(cs->as, pte_addr,
528 &addr1, &l, false, MEMTXATTRS_UNSPECIFIED);
529 if (memory_region_is_ram(mr)) {
530 target_ulong *pte_pa =
531 qemu_map_ram_ptr(mr->ram_block, addr1);
532 #if TCG_OVERSIZED_GUEST
533 /* MTTCG is not enabled on oversized TCG guests so
534 * page table updates do not need to be atomic */
535 *pte_pa = pte = updated_pte;
536 #else
537 target_ulong old_pte =
538 atomic_cmpxchg(pte_pa, pte, updated_pte);
539 if (old_pte != pte) {
540 goto restart;
541 } else {
542 pte = updated_pte;
543 }
544 #endif
545 } else {
546 /* misconfigured PTE in ROM (AD bits are not preset) or
547 * PTE is in IO space and can't be updated atomically */
548 return TRANSLATE_FAIL;
549 }
550 }
551
552 /* for superpage mappings, make a fake leaf PTE for the TLB's
553 benefit. */
554 target_ulong vpn = addr >> PGSHIFT;
555 *physical = ((ppn | (vpn & ((1L << ptshift) - 1))) << PGSHIFT) |
556 (addr & ~TARGET_PAGE_MASK);
557
558 /* set permissions on the TLB entry */
559 if ((pte & PTE_R) || ((pte & PTE_X) && mxr)) {
560 *prot |= PAGE_READ;
561 }
562 if ((pte & PTE_X)) {
563 *prot |= PAGE_EXEC;
564 }
565 /* add write permission on stores or if the page is already dirty,
566 so that we TLB miss on later writes to update the dirty bit */
567 if ((pte & PTE_W) &&
568 (access_type == MMU_DATA_STORE || (pte & PTE_D))) {
569 *prot |= PAGE_WRITE;
570 }
571 return TRANSLATE_SUCCESS;
572 }
573 }
574 return TRANSLATE_FAIL;
575 }
576
577 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
578 MMUAccessType access_type, bool pmp_violation,
579 bool first_stage)
580 {
581 CPUState *cs = env_cpu(env);
582 int page_fault_exceptions;
583 if (first_stage) {
584 page_fault_exceptions =
585 get_field(env->satp, SATP_MODE) != VM_1_10_MBARE &&
586 !pmp_violation;
587 } else {
588 page_fault_exceptions =
589 get_field(env->hgatp, HGATP_MODE) != VM_1_10_MBARE &&
590 !pmp_violation;
591 }
592 switch (access_type) {
593 case MMU_INST_FETCH:
594 if (riscv_cpu_virt_enabled(env) && !first_stage) {
595 cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
596 } else {
597 cs->exception_index = page_fault_exceptions ?
598 RISCV_EXCP_INST_PAGE_FAULT : RISCV_EXCP_INST_ACCESS_FAULT;
599 }
600 break;
601 case MMU_DATA_LOAD:
602 if ((riscv_cpu_virt_enabled(env) || riscv_cpu_two_stage_lookup(env)) &&
603 !first_stage) {
604 cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
605 } else {
606 cs->exception_index = page_fault_exceptions ?
607 RISCV_EXCP_LOAD_PAGE_FAULT : RISCV_EXCP_LOAD_ACCESS_FAULT;
608 }
609 break;
610 case MMU_DATA_STORE:
611 if ((riscv_cpu_virt_enabled(env) || riscv_cpu_two_stage_lookup(env)) &&
612 !first_stage) {
613 cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
614 } else {
615 cs->exception_index = page_fault_exceptions ?
616 RISCV_EXCP_STORE_PAGE_FAULT : RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
617 }
618 break;
619 default:
620 g_assert_not_reached();
621 }
622 env->badaddr = address;
623 }
624
625 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
626 {
627 RISCVCPU *cpu = RISCV_CPU(cs);
628 CPURISCVState *env = &cpu->env;
629 hwaddr phys_addr;
630 int prot;
631 int mmu_idx = cpu_mmu_index(&cpu->env, false);
632
633 if (get_physical_address(env, &phys_addr, &prot, addr, 0, mmu_idx,
634 true, riscv_cpu_virt_enabled(env))) {
635 return -1;
636 }
637
638 if (riscv_cpu_virt_enabled(env)) {
639 if (get_physical_address(env, &phys_addr, &prot, phys_addr,
640 0, mmu_idx, false, true)) {
641 return -1;
642 }
643 }
644
645 return phys_addr & TARGET_PAGE_MASK;
646 }
647
648 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
649 vaddr addr, unsigned size,
650 MMUAccessType access_type,
651 int mmu_idx, MemTxAttrs attrs,
652 MemTxResult response, uintptr_t retaddr)
653 {
654 RISCVCPU *cpu = RISCV_CPU(cs);
655 CPURISCVState *env = &cpu->env;
656
657 if (access_type == MMU_DATA_STORE) {
658 cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
659 } else {
660 cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
661 }
662
663 env->badaddr = addr;
664 riscv_raise_exception(&cpu->env, cs->exception_index, retaddr);
665 }
666
667 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
668 MMUAccessType access_type, int mmu_idx,
669 uintptr_t retaddr)
670 {
671 RISCVCPU *cpu = RISCV_CPU(cs);
672 CPURISCVState *env = &cpu->env;
673 switch (access_type) {
674 case MMU_INST_FETCH:
675 cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
676 break;
677 case MMU_DATA_LOAD:
678 cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
679 break;
680 case MMU_DATA_STORE:
681 cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
682 break;
683 default:
684 g_assert_not_reached();
685 }
686 env->badaddr = addr;
687 riscv_raise_exception(env, cs->exception_index, retaddr);
688 }
689 #endif
690
691 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
692 MMUAccessType access_type, int mmu_idx,
693 bool probe, uintptr_t retaddr)
694 {
695 RISCVCPU *cpu = RISCV_CPU(cs);
696 CPURISCVState *env = &cpu->env;
697 #ifndef CONFIG_USER_ONLY
698 vaddr im_address;
699 hwaddr pa = 0;
700 int prot, prot2;
701 bool pmp_violation = false;
702 bool first_stage_error = true;
703 int ret = TRANSLATE_FAIL;
704 int mode = mmu_idx;
705 target_ulong tlb_size = 0;
706
707 env->guest_phys_fault_addr = 0;
708
709 qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
710 __func__, address, access_type, mmu_idx);
711
712 if (mode == PRV_M && access_type != MMU_INST_FETCH) {
713 if (get_field(env->mstatus, MSTATUS_MPRV)) {
714 mode = get_field(env->mstatus, MSTATUS_MPP);
715 }
716 }
717
718 if (riscv_has_ext(env, RVH) && env->priv == PRV_M &&
719 access_type != MMU_INST_FETCH &&
720 get_field(env->mstatus, MSTATUS_MPRV) &&
721 MSTATUS_MPV_ISSET(env)) {
722 riscv_cpu_set_two_stage_lookup(env, true);
723 }
724
725 if (riscv_cpu_virt_enabled(env) ||
726 (riscv_cpu_two_stage_lookup(env) && access_type != MMU_INST_FETCH)) {
727 /* Two stage lookup */
728 ret = get_physical_address(env, &pa, &prot, address, access_type,
729 mmu_idx, true, true);
730
731 qemu_log_mask(CPU_LOG_MMU,
732 "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
733 TARGET_FMT_plx " prot %d\n",
734 __func__, address, ret, pa, prot);
735
736 if (ret != TRANSLATE_FAIL) {
737 /* Second stage lookup */
738 im_address = pa;
739
740 ret = get_physical_address(env, &pa, &prot2, im_address,
741 access_type, mmu_idx, false, true);
742
743 qemu_log_mask(CPU_LOG_MMU,
744 "%s 2nd-stage address=%" VADDR_PRIx " ret %d physical "
745 TARGET_FMT_plx " prot %d\n",
746 __func__, im_address, ret, pa, prot2);
747
748 prot &= prot2;
749
750 if (riscv_feature(env, RISCV_FEATURE_PMP) &&
751 (ret == TRANSLATE_SUCCESS) &&
752 !pmp_hart_has_privs(env, pa, size, 1 << access_type, mode)) {
753 ret = TRANSLATE_PMP_FAIL;
754 }
755
756 if (ret != TRANSLATE_SUCCESS) {
757 /*
758 * Guest physical address translation failed, this is a HS
759 * level exception
760 */
761 first_stage_error = false;
762 env->guest_phys_fault_addr = (im_address |
763 (address &
764 (TARGET_PAGE_SIZE - 1))) >> 2;
765 }
766 }
767 } else {
768 /* Single stage lookup */
769 ret = get_physical_address(env, &pa, &prot, address, access_type,
770 mmu_idx, true, false);
771
772 qemu_log_mask(CPU_LOG_MMU,
773 "%s address=%" VADDR_PRIx " ret %d physical "
774 TARGET_FMT_plx " prot %d\n",
775 __func__, address, ret, pa, prot);
776 }
777
778 /* We did the two stage lookup based on MPRV, unset the lookup */
779 if (riscv_has_ext(env, RVH) && env->priv == PRV_M &&
780 access_type != MMU_INST_FETCH &&
781 get_field(env->mstatus, MSTATUS_MPRV) &&
782 MSTATUS_MPV_ISSET(env)) {
783 riscv_cpu_set_two_stage_lookup(env, false);
784 }
785
786 if (riscv_feature(env, RISCV_FEATURE_PMP) &&
787 (ret == TRANSLATE_SUCCESS) &&
788 !pmp_hart_has_privs(env, pa, size, 1 << access_type, mode)) {
789 ret = TRANSLATE_PMP_FAIL;
790 }
791 if (ret == TRANSLATE_PMP_FAIL) {
792 pmp_violation = true;
793 }
794
795 if (ret == TRANSLATE_SUCCESS) {
796 if (pmp_is_range_in_tlb(env, pa & TARGET_PAGE_MASK, &tlb_size)) {
797 tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1),
798 prot, mmu_idx, tlb_size);
799 } else {
800 tlb_set_page(cs, address & TARGET_PAGE_MASK, pa & TARGET_PAGE_MASK,
801 prot, mmu_idx, TARGET_PAGE_SIZE);
802 }
803 return true;
804 } else if (probe) {
805 return false;
806 } else {
807 raise_mmu_exception(env, address, access_type, pmp_violation, first_stage_error);
808 riscv_raise_exception(env, cs->exception_index, retaddr);
809 }
810
811 return true;
812
813 #else
814 switch (access_type) {
815 case MMU_INST_FETCH:
816 cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
817 break;
818 case MMU_DATA_LOAD:
819 cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
820 break;
821 case MMU_DATA_STORE:
822 cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
823 break;
824 default:
825 g_assert_not_reached();
826 }
827 env->badaddr = address;
828 cpu_loop_exit_restore(cs, retaddr);
829 #endif
830 }
831
832 /*
833 * Handle Traps
834 *
835 * Adapted from Spike's processor_t::take_trap.
836 *
837 */
838 void riscv_cpu_do_interrupt(CPUState *cs)
839 {
840 #if !defined(CONFIG_USER_ONLY)
841
842 RISCVCPU *cpu = RISCV_CPU(cs);
843 CPURISCVState *env = &cpu->env;
844 bool force_hs_execp = riscv_cpu_force_hs_excep_enabled(env);
845 target_ulong s;
846
847 /* cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
848 * so we mask off the MSB and separate into trap type and cause.
849 */
850 bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
851 target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
852 target_ulong deleg = async ? env->mideleg : env->medeleg;
853 target_ulong tval = 0;
854 target_ulong htval = 0;
855 target_ulong mtval2 = 0;
856
857 if (!async) {
858 /* set tval to badaddr for traps with address information */
859 switch (cause) {
860 case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
861 case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
862 case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
863 force_hs_execp = true;
864 /* fallthrough */
865 case RISCV_EXCP_INST_ADDR_MIS:
866 case RISCV_EXCP_INST_ACCESS_FAULT:
867 case RISCV_EXCP_LOAD_ADDR_MIS:
868 case RISCV_EXCP_STORE_AMO_ADDR_MIS:
869 case RISCV_EXCP_LOAD_ACCESS_FAULT:
870 case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
871 case RISCV_EXCP_INST_PAGE_FAULT:
872 case RISCV_EXCP_LOAD_PAGE_FAULT:
873 case RISCV_EXCP_STORE_PAGE_FAULT:
874 tval = env->badaddr;
875 break;
876 default:
877 break;
878 }
879 /* ecall is dispatched as one cause so translate based on mode */
880 if (cause == RISCV_EXCP_U_ECALL) {
881 assert(env->priv <= 3);
882
883 if (env->priv == PRV_M) {
884 cause = RISCV_EXCP_M_ECALL;
885 } else if (env->priv == PRV_S && riscv_cpu_virt_enabled(env)) {
886 cause = RISCV_EXCP_VS_ECALL;
887 } else if (env->priv == PRV_S && !riscv_cpu_virt_enabled(env)) {
888 cause = RISCV_EXCP_S_ECALL;
889 } else if (env->priv == PRV_U) {
890 cause = RISCV_EXCP_U_ECALL;
891 }
892 }
893 }
894
895 trace_riscv_trap(env->mhartid, async, cause, env->pc, tval, cause < 23 ?
896 (async ? riscv_intr_names : riscv_excp_names)[cause] : "(unknown)");
897
898 if (env->priv <= PRV_S &&
899 cause < TARGET_LONG_BITS && ((deleg >> cause) & 1)) {
900 /* handle the trap in S-mode */
901 if (riscv_has_ext(env, RVH)) {
902 target_ulong hdeleg = async ? env->hideleg : env->hedeleg;
903
904 if ((riscv_cpu_virt_enabled(env) ||
905 riscv_cpu_two_stage_lookup(env)) && tval) {
906 /*
907 * If we are writing a guest virtual address to stval, set
908 * this to 1. If we are trapping to VS we will set this to 0
909 * later.
910 */
911 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 1);
912 } else {
913 /* For other HS-mode traps, we set this to 0. */
914 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 0);
915 }
916
917 if (riscv_cpu_virt_enabled(env) && ((hdeleg >> cause) & 1) &&
918 !force_hs_execp) {
919 /* Trap to VS mode */
920 /*
921 * See if we need to adjust cause. Yes if its VS mode interrupt
922 * no if hypervisor has delegated one of hs mode's interrupt
923 */
924 if (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
925 cause == IRQ_VS_EXT) {
926 cause = cause - 1;
927 }
928 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 0);
929 } else if (riscv_cpu_virt_enabled(env)) {
930 /* Trap into HS mode, from virt */
931 riscv_cpu_swap_hypervisor_regs(env);
932 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
933 get_field(env->mstatus, SSTATUS_SPP));
934 env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
935 riscv_cpu_virt_enabled(env));
936
937 htval = env->guest_phys_fault_addr;
938
939 riscv_cpu_set_virt_enabled(env, 0);
940 riscv_cpu_set_force_hs_excep(env, 0);
941 } else {
942 /* Trap into HS mode */
943 if (!riscv_cpu_two_stage_lookup(env)) {
944 env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
945 riscv_cpu_virt_enabled(env));
946 }
947 riscv_cpu_set_two_stage_lookup(env, false);
948 htval = env->guest_phys_fault_addr;
949 }
950 }
951
952 s = env->mstatus;
953 s = set_field(s, MSTATUS_SPIE, get_field(s, MSTATUS_SIE));
954 s = set_field(s, MSTATUS_SPP, env->priv);
955 s = set_field(s, MSTATUS_SIE, 0);
956 env->mstatus = s;
957 env->scause = cause | ((target_ulong)async << (TARGET_LONG_BITS - 1));
958 env->sepc = env->pc;
959 env->sbadaddr = tval;
960 env->htval = htval;
961 env->pc = (env->stvec >> 2 << 2) +
962 ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
963 riscv_cpu_set_mode(env, PRV_S);
964 } else {
965 /* handle the trap in M-mode */
966 if (riscv_has_ext(env, RVH)) {
967 if (riscv_cpu_virt_enabled(env)) {
968 riscv_cpu_swap_hypervisor_regs(env);
969 }
970 #ifdef TARGET_RISCV32
971 env->mstatush = set_field(env->mstatush, MSTATUS_MPV,
972 riscv_cpu_virt_enabled(env));
973 if (riscv_cpu_virt_enabled(env) && tval) {
974 env->mstatush = set_field(env->mstatush, MSTATUS_GVA, 1);
975 }
976 #else
977 env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
978 riscv_cpu_virt_enabled(env));
979 if (riscv_cpu_virt_enabled(env) && tval) {
980 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1);
981 }
982 #endif
983
984 mtval2 = env->guest_phys_fault_addr;
985
986 /* Trapping to M mode, virt is disabled */
987 riscv_cpu_set_virt_enabled(env, 0);
988 riscv_cpu_set_force_hs_excep(env, 0);
989 }
990
991 s = env->mstatus;
992 s = set_field(s, MSTATUS_MPIE, get_field(s, MSTATUS_MIE));
993 s = set_field(s, MSTATUS_MPP, env->priv);
994 s = set_field(s, MSTATUS_MIE, 0);
995 env->mstatus = s;
996 env->mcause = cause | ~(((target_ulong)-1) >> async);
997 env->mepc = env->pc;
998 env->mbadaddr = tval;
999 env->mtval2 = mtval2;
1000 env->pc = (env->mtvec >> 2 << 2) +
1001 ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
1002 riscv_cpu_set_mode(env, PRV_M);
1003 }
1004
1005 /* NOTE: it is not necessary to yield load reservations here. It is only
1006 * necessary for an SC from "another hart" to cause a load reservation
1007 * to be yielded. Refer to the memory consistency model section of the
1008 * RISC-V ISA Specification.
1009 */
1010
1011 #endif
1012 cs->exception_index = EXCP_NONE; /* mark handled to qemu */
1013 }