cirrus: fix off-by-one in cirrus_bitblt_rop_bkwd_transp_*_16
[qemu.git] / cputlb.c
1 /*
2 * Common CPU TLB handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
22 #include "cpu.h"
23 #include "exec/exec-all.h"
24 #include "exec/memory.h"
25 #include "exec/address-spaces.h"
26 #include "exec/cpu_ldst.h"
27 #include "exec/cputlb.h"
28 #include "exec/memory-internal.h"
29 #include "exec/ram_addr.h"
30 #include "tcg/tcg.h"
31 #include "qemu/error-report.h"
32 #include "exec/log.h"
33 #include "exec/helper-proto.h"
34 #include "qemu/atomic.h"
35
36 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
37 /* #define DEBUG_TLB */
38 /* #define DEBUG_TLB_LOG */
39
40 #ifdef DEBUG_TLB
41 # define DEBUG_TLB_GATE 1
42 # ifdef DEBUG_TLB_LOG
43 # define DEBUG_TLB_LOG_GATE 1
44 # else
45 # define DEBUG_TLB_LOG_GATE 0
46 # endif
47 #else
48 # define DEBUG_TLB_GATE 0
49 # define DEBUG_TLB_LOG_GATE 0
50 #endif
51
52 #define tlb_debug(fmt, ...) do { \
53 if (DEBUG_TLB_LOG_GATE) { \
54 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
55 ## __VA_ARGS__); \
56 } else if (DEBUG_TLB_GATE) { \
57 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
58 } \
59 } while (0)
60
61 #define assert_cpu_is_self(this_cpu) do { \
62 if (DEBUG_TLB_GATE) { \
63 g_assert(!cpu->created || qemu_cpu_is_self(cpu)); \
64 } \
65 } while (0)
66
67 /* run_on_cpu_data.target_ptr should always be big enough for a
68 * target_ulong even on 32 bit builds */
69 QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data));
70
71 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
72 */
73 QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
74 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
75
76 /* flush_all_helper: run fn across all cpus
77 *
78 * If the wait flag is set then the src cpu's helper will be queued as
79 * "safe" work and the loop exited creating a synchronisation point
80 * where all queued work will be finished before execution starts
81 * again.
82 */
83 static void flush_all_helper(CPUState *src, run_on_cpu_func fn,
84 run_on_cpu_data d)
85 {
86 CPUState *cpu;
87
88 CPU_FOREACH(cpu) {
89 if (cpu != src) {
90 async_run_on_cpu(cpu, fn, d);
91 }
92 }
93 }
94
95 /* statistics */
96 int tlb_flush_count;
97
98 /* This is OK because CPU architectures generally permit an
99 * implementation to drop entries from the TLB at any time, so
100 * flushing more entries than required is only an efficiency issue,
101 * not a correctness issue.
102 */
103 static void tlb_flush_nocheck(CPUState *cpu)
104 {
105 CPUArchState *env = cpu->env_ptr;
106
107 /* The QOM tests will trigger tlb_flushes without setting up TCG
108 * so we bug out here in that case.
109 */
110 if (!tcg_enabled()) {
111 return;
112 }
113
114 assert_cpu_is_self(cpu);
115 tlb_debug("(count: %d)\n", tlb_flush_count++);
116
117 tb_lock();
118
119 memset(env->tlb_table, -1, sizeof(env->tlb_table));
120 memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
121 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
122
123 env->vtlb_index = 0;
124 env->tlb_flush_addr = -1;
125 env->tlb_flush_mask = 0;
126
127 tb_unlock();
128
129 atomic_mb_set(&cpu->pending_tlb_flush, 0);
130 }
131
132 static void tlb_flush_global_async_work(CPUState *cpu, run_on_cpu_data data)
133 {
134 tlb_flush_nocheck(cpu);
135 }
136
137 void tlb_flush(CPUState *cpu)
138 {
139 if (cpu->created && !qemu_cpu_is_self(cpu)) {
140 if (atomic_mb_read(&cpu->pending_tlb_flush) != ALL_MMUIDX_BITS) {
141 atomic_mb_set(&cpu->pending_tlb_flush, ALL_MMUIDX_BITS);
142 async_run_on_cpu(cpu, tlb_flush_global_async_work,
143 RUN_ON_CPU_NULL);
144 }
145 } else {
146 tlb_flush_nocheck(cpu);
147 }
148 }
149
150 void tlb_flush_all_cpus(CPUState *src_cpu)
151 {
152 const run_on_cpu_func fn = tlb_flush_global_async_work;
153 flush_all_helper(src_cpu, fn, RUN_ON_CPU_NULL);
154 fn(src_cpu, RUN_ON_CPU_NULL);
155 }
156
157 void tlb_flush_all_cpus_synced(CPUState *src_cpu)
158 {
159 const run_on_cpu_func fn = tlb_flush_global_async_work;
160 flush_all_helper(src_cpu, fn, RUN_ON_CPU_NULL);
161 async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_NULL);
162 }
163
164 static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
165 {
166 CPUArchState *env = cpu->env_ptr;
167 unsigned long mmu_idx_bitmask = data.host_int;
168 int mmu_idx;
169
170 assert_cpu_is_self(cpu);
171
172 tb_lock();
173
174 tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask);
175
176 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
177
178 if (test_bit(mmu_idx, &mmu_idx_bitmask)) {
179 tlb_debug("%d\n", mmu_idx);
180
181 memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
182 memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
183 }
184 }
185
186 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
187
188 tlb_debug("done\n");
189
190 tb_unlock();
191 }
192
193 void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap)
194 {
195 tlb_debug("mmu_idx: 0x%" PRIx16 "\n", idxmap);
196
197 if (!qemu_cpu_is_self(cpu)) {
198 uint16_t pending_flushes = idxmap;
199 pending_flushes &= ~atomic_mb_read(&cpu->pending_tlb_flush);
200
201 if (pending_flushes) {
202 tlb_debug("reduced mmu_idx: 0x%" PRIx16 "\n", pending_flushes);
203
204 atomic_or(&cpu->pending_tlb_flush, pending_flushes);
205 async_run_on_cpu(cpu, tlb_flush_by_mmuidx_async_work,
206 RUN_ON_CPU_HOST_INT(pending_flushes));
207 }
208 } else {
209 tlb_flush_by_mmuidx_async_work(cpu,
210 RUN_ON_CPU_HOST_INT(idxmap));
211 }
212 }
213
214 void tlb_flush_by_mmuidx_all_cpus(CPUState *src_cpu, uint16_t idxmap)
215 {
216 const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
217
218 tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
219
220 flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
221 fn(src_cpu, RUN_ON_CPU_HOST_INT(idxmap));
222 }
223
224 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
225 uint16_t idxmap)
226 {
227 const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
228
229 tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
230
231 flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
232 async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
233 }
234
235
236
237 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
238 {
239 if (addr == (tlb_entry->addr_read &
240 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
241 addr == (tlb_entry->addr_write &
242 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
243 addr == (tlb_entry->addr_code &
244 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
245 memset(tlb_entry, -1, sizeof(*tlb_entry));
246 }
247 }
248
249 static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)
250 {
251 CPUArchState *env = cpu->env_ptr;
252 target_ulong addr = (target_ulong) data.target_ptr;
253 int i;
254 int mmu_idx;
255
256 assert_cpu_is_self(cpu);
257
258 tlb_debug("page :" TARGET_FMT_lx "\n", addr);
259
260 /* Check if we need to flush due to large pages. */
261 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
262 tlb_debug("forcing full flush ("
263 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
264 env->tlb_flush_addr, env->tlb_flush_mask);
265
266 tlb_flush(cpu);
267 return;
268 }
269
270 addr &= TARGET_PAGE_MASK;
271 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
272 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
273 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
274 }
275
276 /* check whether there are entries that need to be flushed in the vtlb */
277 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
278 int k;
279 for (k = 0; k < CPU_VTLB_SIZE; k++) {
280 tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
281 }
282 }
283
284 tb_flush_jmp_cache(cpu, addr);
285 }
286
287 void tlb_flush_page(CPUState *cpu, target_ulong addr)
288 {
289 tlb_debug("page :" TARGET_FMT_lx "\n", addr);
290
291 if (!qemu_cpu_is_self(cpu)) {
292 async_run_on_cpu(cpu, tlb_flush_page_async_work,
293 RUN_ON_CPU_TARGET_PTR(addr));
294 } else {
295 tlb_flush_page_async_work(cpu, RUN_ON_CPU_TARGET_PTR(addr));
296 }
297 }
298
299 /* As we are going to hijack the bottom bits of the page address for a
300 * mmuidx bit mask we need to fail to build if we can't do that
301 */
302 QEMU_BUILD_BUG_ON(NB_MMU_MODES > TARGET_PAGE_BITS_MIN);
303
304 static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu,
305 run_on_cpu_data data)
306 {
307 CPUArchState *env = cpu->env_ptr;
308 target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr;
309 target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK;
310 unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS;
311 int page = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
312 int mmu_idx;
313 int i;
314
315 assert_cpu_is_self(cpu);
316
317 tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",
318 page, addr, mmu_idx_bitmap);
319
320 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
321 if (test_bit(mmu_idx, &mmu_idx_bitmap)) {
322 tlb_flush_entry(&env->tlb_table[mmu_idx][page], addr);
323
324 /* check whether there are vltb entries that need to be flushed */
325 for (i = 0; i < CPU_VTLB_SIZE; i++) {
326 tlb_flush_entry(&env->tlb_v_table[mmu_idx][i], addr);
327 }
328 }
329 }
330
331 tb_flush_jmp_cache(cpu, addr);
332 }
333
334 static void tlb_check_page_and_flush_by_mmuidx_async_work(CPUState *cpu,
335 run_on_cpu_data data)
336 {
337 CPUArchState *env = cpu->env_ptr;
338 target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr;
339 target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK;
340 unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS;
341
342 tlb_debug("addr:"TARGET_FMT_lx" mmu_idx: %04lx\n", addr, mmu_idx_bitmap);
343
344 /* Check if we need to flush due to large pages. */
345 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
346 tlb_debug("forced full flush ("
347 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
348 env->tlb_flush_addr, env->tlb_flush_mask);
349
350 tlb_flush_by_mmuidx_async_work(cpu,
351 RUN_ON_CPU_HOST_INT(mmu_idx_bitmap));
352 } else {
353 tlb_flush_page_by_mmuidx_async_work(cpu, data);
354 }
355 }
356
357 void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, uint16_t idxmap)
358 {
359 target_ulong addr_and_mmu_idx;
360
361 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%" PRIx16 "\n", addr, idxmap);
362
363 /* This should already be page aligned */
364 addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
365 addr_and_mmu_idx |= idxmap;
366
367 if (!qemu_cpu_is_self(cpu)) {
368 async_run_on_cpu(cpu, tlb_check_page_and_flush_by_mmuidx_async_work,
369 RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
370 } else {
371 tlb_check_page_and_flush_by_mmuidx_async_work(
372 cpu, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
373 }
374 }
375
376 void tlb_flush_page_by_mmuidx_all_cpus(CPUState *src_cpu, target_ulong addr,
377 uint16_t idxmap)
378 {
379 const run_on_cpu_func fn = tlb_check_page_and_flush_by_mmuidx_async_work;
380 target_ulong addr_and_mmu_idx;
381
382 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap);
383
384 /* This should already be page aligned */
385 addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
386 addr_and_mmu_idx |= idxmap;
387
388 flush_all_helper(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
389 fn(src_cpu, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
390 }
391
392 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
393 target_ulong addr,
394 uint16_t idxmap)
395 {
396 const run_on_cpu_func fn = tlb_check_page_and_flush_by_mmuidx_async_work;
397 target_ulong addr_and_mmu_idx;
398
399 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap);
400
401 /* This should already be page aligned */
402 addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
403 addr_and_mmu_idx |= idxmap;
404
405 flush_all_helper(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
406 async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
407 }
408
409 void tlb_flush_page_all_cpus(CPUState *src, target_ulong addr)
410 {
411 const run_on_cpu_func fn = tlb_flush_page_async_work;
412
413 flush_all_helper(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
414 fn(src, RUN_ON_CPU_TARGET_PTR(addr));
415 }
416
417 void tlb_flush_page_all_cpus_synced(CPUState *src,
418 target_ulong addr)
419 {
420 const run_on_cpu_func fn = tlb_flush_page_async_work;
421
422 flush_all_helper(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
423 async_safe_run_on_cpu(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
424 }
425
426 /* update the TLBs so that writes to code in the virtual page 'addr'
427 can be detected */
428 void tlb_protect_code(ram_addr_t ram_addr)
429 {
430 cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE,
431 DIRTY_MEMORY_CODE);
432 }
433
434 /* update the TLB so that writes in physical page 'phys_addr' are no longer
435 tested for self modifying code */
436 void tlb_unprotect_code(ram_addr_t ram_addr)
437 {
438 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
439 }
440
441
442 /*
443 * Dirty write flag handling
444 *
445 * When the TCG code writes to a location it looks up the address in
446 * the TLB and uses that data to compute the final address. If any of
447 * the lower bits of the address are set then the slow path is forced.
448 * There are a number of reasons to do this but for normal RAM the
449 * most usual is detecting writes to code regions which may invalidate
450 * generated code.
451 *
452 * Because we want other vCPUs to respond to changes straight away we
453 * update the te->addr_write field atomically. If the TLB entry has
454 * been changed by the vCPU in the mean time we skip the update.
455 *
456 * As this function uses atomic accesses we also need to ensure
457 * updates to tlb_entries follow the same access rules. We don't need
458 * to worry about this for oversized guests as MTTCG is disabled for
459 * them.
460 */
461
462 static void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
463 uintptr_t length)
464 {
465 #if TCG_OVERSIZED_GUEST
466 uintptr_t addr = tlb_entry->addr_write;
467
468 if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
469 addr &= TARGET_PAGE_MASK;
470 addr += tlb_entry->addend;
471 if ((addr - start) < length) {
472 tlb_entry->addr_write |= TLB_NOTDIRTY;
473 }
474 }
475 #else
476 /* paired with atomic_mb_set in tlb_set_page_with_attrs */
477 uintptr_t orig_addr = atomic_mb_read(&tlb_entry->addr_write);
478 uintptr_t addr = orig_addr;
479
480 if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
481 addr &= TARGET_PAGE_MASK;
482 addr += atomic_read(&tlb_entry->addend);
483 if ((addr - start) < length) {
484 uintptr_t notdirty_addr = orig_addr | TLB_NOTDIRTY;
485 atomic_cmpxchg(&tlb_entry->addr_write, orig_addr, notdirty_addr);
486 }
487 }
488 #endif
489 }
490
491 /* For atomic correctness when running MTTCG we need to use the right
492 * primitives when copying entries */
493 static inline void copy_tlb_helper(CPUTLBEntry *d, CPUTLBEntry *s,
494 bool atomic_set)
495 {
496 #if TCG_OVERSIZED_GUEST
497 *d = *s;
498 #else
499 if (atomic_set) {
500 d->addr_read = s->addr_read;
501 d->addr_code = s->addr_code;
502 atomic_set(&d->addend, atomic_read(&s->addend));
503 /* Pairs with flag setting in tlb_reset_dirty_range */
504 atomic_mb_set(&d->addr_write, atomic_read(&s->addr_write));
505 } else {
506 d->addr_read = s->addr_read;
507 d->addr_write = atomic_read(&s->addr_write);
508 d->addr_code = s->addr_code;
509 d->addend = atomic_read(&s->addend);
510 }
511 #endif
512 }
513
514 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
515 * the target vCPU). As such care needs to be taken that we don't
516 * dangerously race with another vCPU update. The only thing actually
517 * updated is the target TLB entry ->addr_write flags.
518 */
519 void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
520 {
521 CPUArchState *env;
522
523 int mmu_idx;
524
525 env = cpu->env_ptr;
526 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
527 unsigned int i;
528
529 for (i = 0; i < CPU_TLB_SIZE; i++) {
530 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
531 start1, length);
532 }
533
534 for (i = 0; i < CPU_VTLB_SIZE; i++) {
535 tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
536 start1, length);
537 }
538 }
539 }
540
541 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
542 {
543 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
544 tlb_entry->addr_write = vaddr;
545 }
546 }
547
548 /* update the TLB corresponding to virtual page vaddr
549 so that it is no longer dirty */
550 void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
551 {
552 CPUArchState *env = cpu->env_ptr;
553 int i;
554 int mmu_idx;
555
556 assert_cpu_is_self(cpu);
557
558 vaddr &= TARGET_PAGE_MASK;
559 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
560 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
561 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
562 }
563
564 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
565 int k;
566 for (k = 0; k < CPU_VTLB_SIZE; k++) {
567 tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
568 }
569 }
570 }
571
572 /* Our TLB does not support large pages, so remember the area covered by
573 large pages and trigger a full TLB flush if these are invalidated. */
574 static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr,
575 target_ulong size)
576 {
577 target_ulong mask = ~(size - 1);
578
579 if (env->tlb_flush_addr == (target_ulong)-1) {
580 env->tlb_flush_addr = vaddr & mask;
581 env->tlb_flush_mask = mask;
582 return;
583 }
584 /* Extend the existing region to include the new page.
585 This is a compromise between unnecessary flushes and the cost
586 of maintaining a full variable size TLB. */
587 mask &= env->tlb_flush_mask;
588 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
589 mask <<= 1;
590 }
591 env->tlb_flush_addr &= mask;
592 env->tlb_flush_mask = mask;
593 }
594
595 /* Add a new TLB entry. At most one entry for a given virtual address
596 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
597 * supplied size is only used by tlb_flush_page.
598 *
599 * Called from TCG-generated code, which is under an RCU read-side
600 * critical section.
601 */
602 void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
603 hwaddr paddr, MemTxAttrs attrs, int prot,
604 int mmu_idx, target_ulong size)
605 {
606 CPUArchState *env = cpu->env_ptr;
607 MemoryRegionSection *section;
608 unsigned int index;
609 target_ulong address;
610 target_ulong code_address;
611 uintptr_t addend;
612 CPUTLBEntry *te, *tv, tn;
613 hwaddr iotlb, xlat, sz;
614 unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
615 int asidx = cpu_asidx_from_attrs(cpu, attrs);
616
617 assert_cpu_is_self(cpu);
618 assert(size >= TARGET_PAGE_SIZE);
619 if (size != TARGET_PAGE_SIZE) {
620 tlb_add_large_page(env, vaddr, size);
621 }
622
623 sz = size;
624 section = address_space_translate_for_iotlb(cpu, asidx, paddr, &xlat, &sz);
625 assert(sz >= TARGET_PAGE_SIZE);
626
627 tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
628 " prot=%x idx=%d\n",
629 vaddr, paddr, prot, mmu_idx);
630
631 address = vaddr;
632 if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) {
633 /* IO memory case */
634 address |= TLB_MMIO;
635 addend = 0;
636 } else {
637 /* TLB_MMIO for rom/romd handled below */
638 addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
639 }
640
641 code_address = address;
642 iotlb = memory_region_section_get_iotlb(cpu, section, vaddr, paddr, xlat,
643 prot, &address);
644
645 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
646 te = &env->tlb_table[mmu_idx][index];
647 /* do not discard the translation in te, evict it into a victim tlb */
648 tv = &env->tlb_v_table[mmu_idx][vidx];
649
650 /* addr_write can race with tlb_reset_dirty_range */
651 copy_tlb_helper(tv, te, true);
652
653 env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
654
655 /* refill the tlb */
656 env->iotlb[mmu_idx][index].addr = iotlb - vaddr;
657 env->iotlb[mmu_idx][index].attrs = attrs;
658
659 /* Now calculate the new entry */
660 tn.addend = addend - vaddr;
661 if (prot & PAGE_READ) {
662 tn.addr_read = address;
663 } else {
664 tn.addr_read = -1;
665 }
666
667 if (prot & PAGE_EXEC) {
668 tn.addr_code = code_address;
669 } else {
670 tn.addr_code = -1;
671 }
672
673 tn.addr_write = -1;
674 if (prot & PAGE_WRITE) {
675 if ((memory_region_is_ram(section->mr) && section->readonly)
676 || memory_region_is_romd(section->mr)) {
677 /* Write access calls the I/O callback. */
678 tn.addr_write = address | TLB_MMIO;
679 } else if (memory_region_is_ram(section->mr)
680 && cpu_physical_memory_is_clean(
681 memory_region_get_ram_addr(section->mr) + xlat)) {
682 tn.addr_write = address | TLB_NOTDIRTY;
683 } else {
684 tn.addr_write = address;
685 }
686 }
687
688 /* Pairs with flag setting in tlb_reset_dirty_range */
689 copy_tlb_helper(te, &tn, true);
690 /* atomic_mb_set(&te->addr_write, write_address); */
691 }
692
693 /* Add a new TLB entry, but without specifying the memory
694 * transaction attributes to be used.
695 */
696 void tlb_set_page(CPUState *cpu, target_ulong vaddr,
697 hwaddr paddr, int prot,
698 int mmu_idx, target_ulong size)
699 {
700 tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED,
701 prot, mmu_idx, size);
702 }
703
704 static void report_bad_exec(CPUState *cpu, target_ulong addr)
705 {
706 /* Accidentally executing outside RAM or ROM is quite common for
707 * several user-error situations, so report it in a way that
708 * makes it clear that this isn't a QEMU bug and provide suggestions
709 * about what a user could do to fix things.
710 */
711 error_report("Trying to execute code outside RAM or ROM at 0x"
712 TARGET_FMT_lx, addr);
713 error_printf("This usually means one of the following happened:\n\n"
714 "(1) You told QEMU to execute a kernel for the wrong machine "
715 "type, and it crashed on startup (eg trying to run a "
716 "raspberry pi kernel on a versatilepb QEMU machine)\n"
717 "(2) You didn't give QEMU a kernel or BIOS filename at all, "
718 "and QEMU executed a ROM full of no-op instructions until "
719 "it fell off the end\n"
720 "(3) Your guest kernel has a bug and crashed by jumping "
721 "off into nowhere\n\n"
722 "This is almost always one of the first two, so check your "
723 "command line and that you are using the right type of kernel "
724 "for this machine.\n"
725 "If you think option (3) is likely then you can try debugging "
726 "your guest with the -d debug options; in particular "
727 "-d guest_errors will cause the log to include a dump of the "
728 "guest register state at this point.\n\n"
729 "Execution cannot continue; stopping here.\n\n");
730
731 /* Report also to the logs, with more detail including register dump */
732 qemu_log_mask(LOG_GUEST_ERROR, "qemu: fatal: Trying to execute code "
733 "outside RAM or ROM at 0x" TARGET_FMT_lx "\n", addr);
734 log_cpu_state_mask(LOG_GUEST_ERROR, cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
735 }
736
737 static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
738 {
739 ram_addr_t ram_addr;
740
741 ram_addr = qemu_ram_addr_from_host(ptr);
742 if (ram_addr == RAM_ADDR_INVALID) {
743 error_report("Bad ram pointer %p", ptr);
744 abort();
745 }
746 return ram_addr;
747 }
748
749 /* NOTE: this function can trigger an exception */
750 /* NOTE2: the returned address is not exactly the physical address: it
751 * is actually a ram_addr_t (in system mode; the user mode emulation
752 * version of this function returns a guest virtual address).
753 */
754 tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
755 {
756 int mmu_idx, page_index, pd;
757 void *p;
758 MemoryRegion *mr;
759 CPUState *cpu = ENV_GET_CPU(env1);
760 CPUIOTLBEntry *iotlbentry;
761
762 page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
763 mmu_idx = cpu_mmu_index(env1, true);
764 if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
765 (addr & TARGET_PAGE_MASK))) {
766 cpu_ldub_code(env1, addr);
767 }
768 iotlbentry = &env1->iotlb[mmu_idx][page_index];
769 pd = iotlbentry->addr & ~TARGET_PAGE_MASK;
770 mr = iotlb_to_region(cpu, pd, iotlbentry->attrs);
771 if (memory_region_is_unassigned(mr)) {
772 cpu_unassigned_access(cpu, addr, false, true, 0, 4);
773 /* The CPU's unassigned access hook might have longjumped out
774 * with an exception. If it didn't (or there was no hook) then
775 * we can't proceed further.
776 */
777 report_bad_exec(cpu, addr);
778 exit(1);
779 }
780 p = (void *)((uintptr_t)addr + env1->tlb_table[mmu_idx][page_index].addend);
781 return qemu_ram_addr_from_host_nofail(p);
782 }
783
784 static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
785 target_ulong addr, uintptr_t retaddr, int size)
786 {
787 CPUState *cpu = ENV_GET_CPU(env);
788 hwaddr physaddr = iotlbentry->addr;
789 MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
790 uint64_t val;
791 bool locked = false;
792
793 physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
794 cpu->mem_io_pc = retaddr;
795 if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
796 cpu_io_recompile(cpu, retaddr);
797 }
798
799 cpu->mem_io_vaddr = addr;
800
801 if (mr->global_locking) {
802 qemu_mutex_lock_iothread();
803 locked = true;
804 }
805 memory_region_dispatch_read(mr, physaddr, &val, size, iotlbentry->attrs);
806 if (locked) {
807 qemu_mutex_unlock_iothread();
808 }
809
810 return val;
811 }
812
813 static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
814 uint64_t val, target_ulong addr,
815 uintptr_t retaddr, int size)
816 {
817 CPUState *cpu = ENV_GET_CPU(env);
818 hwaddr physaddr = iotlbentry->addr;
819 MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
820 bool locked = false;
821
822 physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
823 if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
824 cpu_io_recompile(cpu, retaddr);
825 }
826 cpu->mem_io_vaddr = addr;
827 cpu->mem_io_pc = retaddr;
828
829 if (mr->global_locking) {
830 qemu_mutex_lock_iothread();
831 locked = true;
832 }
833 memory_region_dispatch_write(mr, physaddr, val, size, iotlbentry->attrs);
834 if (locked) {
835 qemu_mutex_unlock_iothread();
836 }
837 }
838
839 /* Return true if ADDR is present in the victim tlb, and has been copied
840 back to the main tlb. */
841 static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
842 size_t elt_ofs, target_ulong page)
843 {
844 size_t vidx;
845 for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
846 CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
847 target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
848
849 if (cmp == page) {
850 /* Found entry in victim tlb, swap tlb and iotlb. */
851 CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];
852
853 copy_tlb_helper(&tmptlb, tlb, false);
854 copy_tlb_helper(tlb, vtlb, true);
855 copy_tlb_helper(vtlb, &tmptlb, true);
856
857 CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];
858 CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];
859 tmpio = *io; *io = *vio; *vio = tmpio;
860 return true;
861 }
862 }
863 return false;
864 }
865
866 /* Macro to call the above, with local variables from the use context. */
867 #define VICTIM_TLB_HIT(TY, ADDR) \
868 victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
869 (ADDR) & TARGET_PAGE_MASK)
870
871 /* Probe for whether the specified guest write access is permitted.
872 * If it is not permitted then an exception will be taken in the same
873 * way as if this were a real write access (and we will not return).
874 * Otherwise the function will return, and there will be a valid
875 * entry in the TLB for this access.
876 */
877 void probe_write(CPUArchState *env, target_ulong addr, int mmu_idx,
878 uintptr_t retaddr)
879 {
880 int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
881 target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
882
883 if ((addr & TARGET_PAGE_MASK)
884 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
885 /* TLB entry is for a different page */
886 if (!VICTIM_TLB_HIT(addr_write, addr)) {
887 tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
888 }
889 }
890 }
891
892 /* Probe for a read-modify-write atomic operation. Do not allow unaligned
893 * operations, or io operations to proceed. Return the host address. */
894 static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
895 TCGMemOpIdx oi, uintptr_t retaddr)
896 {
897 size_t mmu_idx = get_mmuidx(oi);
898 size_t index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
899 CPUTLBEntry *tlbe = &env->tlb_table[mmu_idx][index];
900 target_ulong tlb_addr = tlbe->addr_write;
901 TCGMemOp mop = get_memop(oi);
902 int a_bits = get_alignment_bits(mop);
903 int s_bits = mop & MO_SIZE;
904
905 /* Adjust the given return address. */
906 retaddr -= GETPC_ADJ;
907
908 /* Enforce guest required alignment. */
909 if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {
910 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
911 cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
912 mmu_idx, retaddr);
913 }
914
915 /* Enforce qemu required alignment. */
916 if (unlikely(addr & ((1 << s_bits) - 1))) {
917 /* We get here if guest alignment was not requested,
918 or was not enforced by cpu_unaligned_access above.
919 We might widen the access and emulate, but for now
920 mark an exception and exit the cpu loop. */
921 goto stop_the_world;
922 }
923
924 /* Check TLB entry and enforce page permissions. */
925 if ((addr & TARGET_PAGE_MASK)
926 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
927 if (!VICTIM_TLB_HIT(addr_write, addr)) {
928 tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
929 }
930 tlb_addr = tlbe->addr_write;
931 }
932
933 /* Notice an IO access, or a notdirty page. */
934 if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
935 /* There's really nothing that can be done to
936 support this apart from stop-the-world. */
937 goto stop_the_world;
938 }
939
940 /* Let the guest notice RMW on a write-only page. */
941 if (unlikely(tlbe->addr_read != tlb_addr)) {
942 tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_LOAD, mmu_idx, retaddr);
943 /* Since we don't support reads and writes to different addresses,
944 and we do have the proper page loaded for write, this shouldn't
945 ever return. But just in case, handle via stop-the-world. */
946 goto stop_the_world;
947 }
948
949 return (void *)((uintptr_t)addr + tlbe->addend);
950
951 stop_the_world:
952 cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
953 }
954
955 #ifdef TARGET_WORDS_BIGENDIAN
956 # define TGT_BE(X) (X)
957 # define TGT_LE(X) BSWAP(X)
958 #else
959 # define TGT_BE(X) BSWAP(X)
960 # define TGT_LE(X) (X)
961 #endif
962
963 #define MMUSUFFIX _mmu
964
965 #define DATA_SIZE 1
966 #include "softmmu_template.h"
967
968 #define DATA_SIZE 2
969 #include "softmmu_template.h"
970
971 #define DATA_SIZE 4
972 #include "softmmu_template.h"
973
974 #define DATA_SIZE 8
975 #include "softmmu_template.h"
976
977 /* First set of helpers allows passing in of OI and RETADDR. This makes
978 them callable from other helpers. */
979
980 #define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr
981 #define ATOMIC_NAME(X) \
982 HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
983 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr)
984
985 #define DATA_SIZE 1
986 #include "atomic_template.h"
987
988 #define DATA_SIZE 2
989 #include "atomic_template.h"
990
991 #define DATA_SIZE 4
992 #include "atomic_template.h"
993
994 #ifdef CONFIG_ATOMIC64
995 #define DATA_SIZE 8
996 #include "atomic_template.h"
997 #endif
998
999 #ifdef CONFIG_ATOMIC128
1000 #define DATA_SIZE 16
1001 #include "atomic_template.h"
1002 #endif
1003
1004 /* Second set of helpers are directly callable from TCG as helpers. */
1005
1006 #undef EXTRA_ARGS
1007 #undef ATOMIC_NAME
1008 #undef ATOMIC_MMU_LOOKUP
1009 #define EXTRA_ARGS , TCGMemOpIdx oi
1010 #define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
1011 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC())
1012
1013 #define DATA_SIZE 1
1014 #include "atomic_template.h"
1015
1016 #define DATA_SIZE 2
1017 #include "atomic_template.h"
1018
1019 #define DATA_SIZE 4
1020 #include "atomic_template.h"
1021
1022 #ifdef CONFIG_ATOMIC64
1023 #define DATA_SIZE 8
1024 #include "atomic_template.h"
1025 #endif
1026
1027 /* Code access functions. */
1028
1029 #undef MMUSUFFIX
1030 #define MMUSUFFIX _cmmu
1031 #undef GETPC
1032 #define GETPC() ((uintptr_t)0)
1033 #define SOFTMMU_CODE_ACCESS
1034
1035 #define DATA_SIZE 1
1036 #include "softmmu_template.h"
1037
1038 #define DATA_SIZE 2
1039 #include "softmmu_template.h"
1040
1041 #define DATA_SIZE 4
1042 #include "softmmu_template.h"
1043
1044 #define DATA_SIZE 8
1045 #include "softmmu_template.h"