vmsvga: don't process more than 1024 fifo commands at once
[qemu.git] / translate-all.c
1 /*
2 * Host code generation
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/mman.h>
23 #endif
24 #include "qemu/osdep.h"
25
26
27 #include "qemu-common.h"
28 #define NO_CPU_IO_DEFS
29 #include "cpu.h"
30 #include "trace.h"
31 #include "disas/disas.h"
32 #include "exec/exec-all.h"
33 #include "tcg.h"
34 #if defined(CONFIG_USER_ONLY)
35 #include "qemu.h"
36 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
37 #include <sys/param.h>
38 #if __FreeBSD_version >= 700104
39 #define HAVE_KINFO_GETVMMAP
40 #define sigqueue sigqueue_freebsd /* avoid redefinition */
41 #include <sys/proc.h>
42 #include <machine/profile.h>
43 #define _KERNEL
44 #include <sys/user.h>
45 #undef _KERNEL
46 #undef sigqueue
47 #include <libutil.h>
48 #endif
49 #endif
50 #else
51 #include "exec/address-spaces.h"
52 #endif
53
54 #include "exec/cputlb.h"
55 #include "exec/tb-hash.h"
56 #include "translate-all.h"
57 #include "qemu/bitmap.h"
58 #include "qemu/timer.h"
59 #include "exec/log.h"
60
61 //#define DEBUG_TB_INVALIDATE
62 //#define DEBUG_FLUSH
63 /* make various TB consistency checks */
64 //#define DEBUG_TB_CHECK
65
66 #if !defined(CONFIG_USER_ONLY)
67 /* TB consistency checks only implemented for usermode emulation. */
68 #undef DEBUG_TB_CHECK
69 #endif
70
71 #define SMC_BITMAP_USE_THRESHOLD 10
72
73 typedef struct PageDesc {
74 /* list of TBs intersecting this ram page */
75 TranslationBlock *first_tb;
76 #ifdef CONFIG_SOFTMMU
77 /* in order to optimize self modifying code, we count the number
78 of lookups we do to a given page to use a bitmap */
79 unsigned int code_write_count;
80 unsigned long *code_bitmap;
81 #else
82 unsigned long flags;
83 #endif
84 } PageDesc;
85
86 /* In system mode we want L1_MAP to be based on ram offsets,
87 while in user mode we want it to be based on virtual addresses. */
88 #if !defined(CONFIG_USER_ONLY)
89 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
90 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
91 #else
92 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
93 #endif
94 #else
95 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
96 #endif
97
98 /* Size of the L2 (and L3, etc) page tables. */
99 #define V_L2_BITS 10
100 #define V_L2_SIZE (1 << V_L2_BITS)
101
102 /* The bits remaining after N lower levels of page tables. */
103 #define V_L1_BITS_REM \
104 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
105
106 #if V_L1_BITS_REM < 4
107 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
108 #else
109 #define V_L1_BITS V_L1_BITS_REM
110 #endif
111
112 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
113
114 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
115
116 uintptr_t qemu_host_page_size;
117 intptr_t qemu_host_page_mask;
118
119 /* The bottom level has pointers to PageDesc */
120 static void *l1_map[V_L1_SIZE];
121
122 /* code generation context */
123 TCGContext tcg_ctx;
124
125 /* translation block context */
126 #ifdef CONFIG_USER_ONLY
127 __thread int have_tb_lock;
128 #endif
129
130 void tb_lock(void)
131 {
132 #ifdef CONFIG_USER_ONLY
133 assert(!have_tb_lock);
134 qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
135 have_tb_lock++;
136 #endif
137 }
138
139 void tb_unlock(void)
140 {
141 #ifdef CONFIG_USER_ONLY
142 assert(have_tb_lock);
143 have_tb_lock--;
144 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
145 #endif
146 }
147
148 void tb_lock_reset(void)
149 {
150 #ifdef CONFIG_USER_ONLY
151 if (have_tb_lock) {
152 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
153 have_tb_lock = 0;
154 }
155 #endif
156 }
157
158 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
159
160 void cpu_gen_init(void)
161 {
162 tcg_context_init(&tcg_ctx);
163 }
164
165 /* Encode VAL as a signed leb128 sequence at P.
166 Return P incremented past the encoded value. */
167 static uint8_t *encode_sleb128(uint8_t *p, target_long val)
168 {
169 int more, byte;
170
171 do {
172 byte = val & 0x7f;
173 val >>= 7;
174 more = !((val == 0 && (byte & 0x40) == 0)
175 || (val == -1 && (byte & 0x40) != 0));
176 if (more) {
177 byte |= 0x80;
178 }
179 *p++ = byte;
180 } while (more);
181
182 return p;
183 }
184
185 /* Decode a signed leb128 sequence at *PP; increment *PP past the
186 decoded value. Return the decoded value. */
187 static target_long decode_sleb128(uint8_t **pp)
188 {
189 uint8_t *p = *pp;
190 target_long val = 0;
191 int byte, shift = 0;
192
193 do {
194 byte = *p++;
195 val |= (target_ulong)(byte & 0x7f) << shift;
196 shift += 7;
197 } while (byte & 0x80);
198 if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
199 val |= -(target_ulong)1 << shift;
200 }
201
202 *pp = p;
203 return val;
204 }
205
206 /* Encode the data collected about the instructions while compiling TB.
207 Place the data at BLOCK, and return the number of bytes consumed.
208
209 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
210 which come from the target's insn_start data, followed by a uintptr_t
211 which comes from the host pc of the end of the code implementing the insn.
212
213 Each line of the table is encoded as sleb128 deltas from the previous
214 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
215 That is, the first column is seeded with the guest pc, the last column
216 with the host pc, and the middle columns with zeros. */
217
218 static int encode_search(TranslationBlock *tb, uint8_t *block)
219 {
220 uint8_t *highwater = tcg_ctx.code_gen_highwater;
221 uint8_t *p = block;
222 int i, j, n;
223
224 tb->tc_search = block;
225
226 for (i = 0, n = tb->icount; i < n; ++i) {
227 target_ulong prev;
228
229 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
230 if (i == 0) {
231 prev = (j == 0 ? tb->pc : 0);
232 } else {
233 prev = tcg_ctx.gen_insn_data[i - 1][j];
234 }
235 p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev);
236 }
237 prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]);
238 p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev);
239
240 /* Test for (pending) buffer overflow. The assumption is that any
241 one row beginning below the high water mark cannot overrun
242 the buffer completely. Thus we can test for overflow after
243 encoding a row without having to check during encoding. */
244 if (unlikely(p > highwater)) {
245 return -1;
246 }
247 }
248
249 return p - block;
250 }
251
252 /* The cpu state corresponding to 'searched_pc' is restored. */
253 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
254 uintptr_t searched_pc)
255 {
256 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
257 uintptr_t host_pc = (uintptr_t)tb->tc_ptr;
258 CPUArchState *env = cpu->env_ptr;
259 uint8_t *p = tb->tc_search;
260 int i, j, num_insns = tb->icount;
261 #ifdef CONFIG_PROFILER
262 int64_t ti = profile_getclock();
263 #endif
264
265 if (searched_pc < host_pc) {
266 return -1;
267 }
268
269 /* Reconstruct the stored insn data while looking for the point at
270 which the end of the insn exceeds the searched_pc. */
271 for (i = 0; i < num_insns; ++i) {
272 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
273 data[j] += decode_sleb128(&p);
274 }
275 host_pc += decode_sleb128(&p);
276 if (host_pc > searched_pc) {
277 goto found;
278 }
279 }
280 return -1;
281
282 found:
283 if (tb->cflags & CF_USE_ICOUNT) {
284 assert(use_icount);
285 /* Reset the cycle counter to the start of the block. */
286 cpu->icount_decr.u16.low += num_insns;
287 /* Clear the IO flag. */
288 cpu->can_do_io = 0;
289 }
290 cpu->icount_decr.u16.low -= i;
291 restore_state_to_opc(env, tb, data);
292
293 #ifdef CONFIG_PROFILER
294 tcg_ctx.restore_time += profile_getclock() - ti;
295 tcg_ctx.restore_count++;
296 #endif
297 return 0;
298 }
299
300 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
301 {
302 TranslationBlock *tb;
303
304 tb = tb_find_pc(retaddr);
305 if (tb) {
306 cpu_restore_state_from_tb(cpu, tb, retaddr);
307 if (tb->cflags & CF_NOCACHE) {
308 /* one-shot translation, invalidate it immediately */
309 tb_phys_invalidate(tb, -1);
310 tb_free(tb);
311 }
312 return true;
313 }
314 return false;
315 }
316
317 void page_size_init(void)
318 {
319 /* NOTE: we can always suppose that qemu_host_page_size >=
320 TARGET_PAGE_SIZE */
321 qemu_real_host_page_size = getpagesize();
322 qemu_real_host_page_mask = -(intptr_t)qemu_real_host_page_size;
323 if (qemu_host_page_size == 0) {
324 qemu_host_page_size = qemu_real_host_page_size;
325 }
326 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
327 qemu_host_page_size = TARGET_PAGE_SIZE;
328 }
329 qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
330 }
331
332 static void page_init(void)
333 {
334 page_size_init();
335 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
336 {
337 #ifdef HAVE_KINFO_GETVMMAP
338 struct kinfo_vmentry *freep;
339 int i, cnt;
340
341 freep = kinfo_getvmmap(getpid(), &cnt);
342 if (freep) {
343 mmap_lock();
344 for (i = 0; i < cnt; i++) {
345 unsigned long startaddr, endaddr;
346
347 startaddr = freep[i].kve_start;
348 endaddr = freep[i].kve_end;
349 if (h2g_valid(startaddr)) {
350 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
351
352 if (h2g_valid(endaddr)) {
353 endaddr = h2g(endaddr);
354 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
355 } else {
356 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
357 endaddr = ~0ul;
358 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
359 #endif
360 }
361 }
362 }
363 free(freep);
364 mmap_unlock();
365 }
366 #else
367 FILE *f;
368
369 last_brk = (unsigned long)sbrk(0);
370
371 f = fopen("/compat/linux/proc/self/maps", "r");
372 if (f) {
373 mmap_lock();
374
375 do {
376 unsigned long startaddr, endaddr;
377 int n;
378
379 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
380
381 if (n == 2 && h2g_valid(startaddr)) {
382 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
383
384 if (h2g_valid(endaddr)) {
385 endaddr = h2g(endaddr);
386 } else {
387 endaddr = ~0ul;
388 }
389 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
390 }
391 } while (!feof(f));
392
393 fclose(f);
394 mmap_unlock();
395 }
396 #endif
397 }
398 #endif
399 }
400
401 /* If alloc=1:
402 * Called with mmap_lock held for user-mode emulation.
403 */
404 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
405 {
406 PageDesc *pd;
407 void **lp;
408 int i;
409
410 /* Level 1. Always allocated. */
411 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
412
413 /* Level 2..N-1. */
414 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
415 void **p = atomic_rcu_read(lp);
416
417 if (p == NULL) {
418 if (!alloc) {
419 return NULL;
420 }
421 p = g_new0(void *, V_L2_SIZE);
422 atomic_rcu_set(lp, p);
423 }
424
425 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
426 }
427
428 pd = atomic_rcu_read(lp);
429 if (pd == NULL) {
430 if (!alloc) {
431 return NULL;
432 }
433 pd = g_new0(PageDesc, V_L2_SIZE);
434 atomic_rcu_set(lp, pd);
435 }
436
437 return pd + (index & (V_L2_SIZE - 1));
438 }
439
440 static inline PageDesc *page_find(tb_page_addr_t index)
441 {
442 return page_find_alloc(index, 0);
443 }
444
445 #if defined(CONFIG_USER_ONLY)
446 /* Currently it is not recommended to allocate big chunks of data in
447 user mode. It will change when a dedicated libc will be used. */
448 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
449 region in which the guest needs to run. Revisit this. */
450 #define USE_STATIC_CODE_GEN_BUFFER
451 #endif
452
453 /* Minimum size of the code gen buffer. This number is randomly chosen,
454 but not so small that we can't have a fair number of TB's live. */
455 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
456
457 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
458 indicated, this is constrained by the range of direct branches on the
459 host cpu, as used by the TCG implementation of goto_tb. */
460 #if defined(__x86_64__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
462 #elif defined(__sparc__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__powerpc64__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
466 #elif defined(__powerpc__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
468 #elif defined(__aarch64__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
470 #elif defined(__arm__)
471 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
472 #elif defined(__s390x__)
473 /* We have a +- 4GB range on the branches; leave some slop. */
474 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
475 #elif defined(__mips__)
476 /* We have a 256MB branch region, but leave room to make sure the
477 main executable is also within that region. */
478 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
479 #else
480 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
481 #endif
482
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
484
485 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
486 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
487 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
488
489 static inline size_t size_code_gen_buffer(size_t tb_size)
490 {
491 /* Size the buffer. */
492 if (tb_size == 0) {
493 #ifdef USE_STATIC_CODE_GEN_BUFFER
494 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
495 #else
496 /* ??? Needs adjustments. */
497 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
498 static buffer, we could size this on RESERVED_VA, on the text
499 segment size of the executable, or continue to use the default. */
500 tb_size = (unsigned long)(ram_size / 4);
501 #endif
502 }
503 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
504 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
505 }
506 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
507 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
508 }
509 return tb_size;
510 }
511
512 #ifdef __mips__
513 /* In order to use J and JAL within the code_gen_buffer, we require
514 that the buffer not cross a 256MB boundary. */
515 static inline bool cross_256mb(void *addr, size_t size)
516 {
517 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
518 }
519
520 /* We weren't able to allocate a buffer without crossing that boundary,
521 so make do with the larger portion of the buffer that doesn't cross.
522 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
523 static inline void *split_cross_256mb(void *buf1, size_t size1)
524 {
525 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
526 size_t size2 = buf1 + size1 - buf2;
527
528 size1 = buf2 - buf1;
529 if (size1 < size2) {
530 size1 = size2;
531 buf1 = buf2;
532 }
533
534 tcg_ctx.code_gen_buffer_size = size1;
535 return buf1;
536 }
537 #endif
538
539 #ifdef USE_STATIC_CODE_GEN_BUFFER
540 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
541 __attribute__((aligned(CODE_GEN_ALIGN)));
542
543 # ifdef _WIN32
544 static inline void do_protect(void *addr, long size, int prot)
545 {
546 DWORD old_protect;
547 VirtualProtect(addr, size, prot, &old_protect);
548 }
549
550 static inline void map_exec(void *addr, long size)
551 {
552 do_protect(addr, size, PAGE_EXECUTE_READWRITE);
553 }
554
555 static inline void map_none(void *addr, long size)
556 {
557 do_protect(addr, size, PAGE_NOACCESS);
558 }
559 # else
560 static inline void do_protect(void *addr, long size, int prot)
561 {
562 uintptr_t start, end;
563
564 start = (uintptr_t)addr;
565 start &= qemu_real_host_page_mask;
566
567 end = (uintptr_t)addr + size;
568 end = ROUND_UP(end, qemu_real_host_page_size);
569
570 mprotect((void *)start, end - start, prot);
571 }
572
573 static inline void map_exec(void *addr, long size)
574 {
575 do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC);
576 }
577
578 static inline void map_none(void *addr, long size)
579 {
580 do_protect(addr, size, PROT_NONE);
581 }
582 # endif /* WIN32 */
583
584 static inline void *alloc_code_gen_buffer(void)
585 {
586 void *buf = static_code_gen_buffer;
587 size_t full_size, size;
588
589 /* The size of the buffer, rounded down to end on a page boundary. */
590 full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer))
591 & qemu_real_host_page_mask) - (uintptr_t)buf;
592
593 /* Reserve a guard page. */
594 size = full_size - qemu_real_host_page_size;
595
596 /* Honor a command-line option limiting the size of the buffer. */
597 if (size > tcg_ctx.code_gen_buffer_size) {
598 size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size)
599 & qemu_real_host_page_mask) - (uintptr_t)buf;
600 }
601 tcg_ctx.code_gen_buffer_size = size;
602
603 #ifdef __mips__
604 if (cross_256mb(buf, size)) {
605 buf = split_cross_256mb(buf, size);
606 size = tcg_ctx.code_gen_buffer_size;
607 }
608 #endif
609
610 map_exec(buf, size);
611 map_none(buf + size, qemu_real_host_page_size);
612 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
613
614 return buf;
615 }
616 #elif defined(_WIN32)
617 static inline void *alloc_code_gen_buffer(void)
618 {
619 size_t size = tcg_ctx.code_gen_buffer_size;
620 void *buf1, *buf2;
621
622 /* Perform the allocation in two steps, so that the guard page
623 is reserved but uncommitted. */
624 buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size,
625 MEM_RESERVE, PAGE_NOACCESS);
626 if (buf1 != NULL) {
627 buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
628 assert(buf1 == buf2);
629 }
630
631 return buf1;
632 }
633 #else
634 static inline void *alloc_code_gen_buffer(void)
635 {
636 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
637 uintptr_t start = 0;
638 size_t size = tcg_ctx.code_gen_buffer_size;
639 void *buf;
640
641 /* Constrain the position of the buffer based on the host cpu.
642 Note that these addresses are chosen in concert with the
643 addresses assigned in the relevant linker script file. */
644 # if defined(__PIE__) || defined(__PIC__)
645 /* Don't bother setting a preferred location if we're building
646 a position-independent executable. We're more likely to get
647 an address near the main executable if we let the kernel
648 choose the address. */
649 # elif defined(__x86_64__) && defined(MAP_32BIT)
650 /* Force the memory down into low memory with the executable.
651 Leave the choice of exact location with the kernel. */
652 flags |= MAP_32BIT;
653 /* Cannot expect to map more than 800MB in low memory. */
654 if (size > 800u * 1024 * 1024) {
655 tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024;
656 }
657 # elif defined(__sparc__)
658 start = 0x40000000ul;
659 # elif defined(__s390x__)
660 start = 0x90000000ul;
661 # elif defined(__mips__)
662 # if _MIPS_SIM == _ABI64
663 start = 0x128000000ul;
664 # else
665 start = 0x08000000ul;
666 # endif
667 # endif
668
669 buf = mmap((void *)start, size + qemu_real_host_page_size,
670 PROT_NONE, flags, -1, 0);
671 if (buf == MAP_FAILED) {
672 return NULL;
673 }
674
675 #ifdef __mips__
676 if (cross_256mb(buf, size)) {
677 /* Try again, with the original still mapped, to avoid re-acquiring
678 that 256mb crossing. This time don't specify an address. */
679 size_t size2;
680 void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
681 PROT_NONE, flags, -1, 0);
682 switch (buf2 != MAP_FAILED) {
683 case 1:
684 if (!cross_256mb(buf2, size)) {
685 /* Success! Use the new buffer. */
686 munmap(buf, size + qemu_real_host_page_size);
687 break;
688 }
689 /* Failure. Work with what we had. */
690 munmap(buf2, size + qemu_real_host_page_size);
691 /* fallthru */
692 default:
693 /* Split the original buffer. Free the smaller half. */
694 buf2 = split_cross_256mb(buf, size);
695 size2 = tcg_ctx.code_gen_buffer_size;
696 if (buf == buf2) {
697 munmap(buf + size2 + qemu_real_host_page_size, size - size2);
698 } else {
699 munmap(buf, size - size2);
700 }
701 size = size2;
702 break;
703 }
704 buf = buf2;
705 }
706 #endif
707
708 /* Make the final buffer accessible. The guard page at the end
709 will remain inaccessible with PROT_NONE. */
710 mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC);
711
712 /* Request large pages for the buffer. */
713 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
714
715 return buf;
716 }
717 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
718
719 static inline void code_gen_alloc(size_t tb_size)
720 {
721 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
722 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
723 if (tcg_ctx.code_gen_buffer == NULL) {
724 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
725 exit(1);
726 }
727
728 /* Estimate a good size for the number of TBs we can support. We
729 still haven't deducted the prologue from the buffer size here,
730 but that's minimal and won't affect the estimate much. */
731 tcg_ctx.code_gen_max_blocks
732 = tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
733 tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock, tcg_ctx.code_gen_max_blocks);
734
735 qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
736 }
737
738 /* Must be called before using the QEMU cpus. 'tb_size' is the size
739 (in bytes) allocated to the translation buffer. Zero means default
740 size. */
741 void tcg_exec_init(unsigned long tb_size)
742 {
743 cpu_gen_init();
744 page_init();
745 code_gen_alloc(tb_size);
746 #if defined(CONFIG_SOFTMMU)
747 /* There's no guest base to take into account, so go ahead and
748 initialize the prologue now. */
749 tcg_prologue_init(&tcg_ctx);
750 #endif
751 }
752
753 bool tcg_enabled(void)
754 {
755 return tcg_ctx.code_gen_buffer != NULL;
756 }
757
758 /* Allocate a new translation block. Flush the translation buffer if
759 too many translation blocks or too much generated code. */
760 static TranslationBlock *tb_alloc(target_ulong pc)
761 {
762 TranslationBlock *tb;
763
764 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks) {
765 return NULL;
766 }
767 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
768 tb->pc = pc;
769 tb->cflags = 0;
770 return tb;
771 }
772
773 void tb_free(TranslationBlock *tb)
774 {
775 /* In practice this is mostly used for single use temporary TB
776 Ignore the hard cases and just back up if this TB happens to
777 be the last one generated. */
778 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
779 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
780 tcg_ctx.code_gen_ptr = tb->tc_ptr;
781 tcg_ctx.tb_ctx.nb_tbs--;
782 }
783 }
784
785 static inline void invalidate_page_bitmap(PageDesc *p)
786 {
787 #ifdef CONFIG_SOFTMMU
788 g_free(p->code_bitmap);
789 p->code_bitmap = NULL;
790 p->code_write_count = 0;
791 #endif
792 }
793
794 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
795 static void page_flush_tb_1(int level, void **lp)
796 {
797 int i;
798
799 if (*lp == NULL) {
800 return;
801 }
802 if (level == 0) {
803 PageDesc *pd = *lp;
804
805 for (i = 0; i < V_L2_SIZE; ++i) {
806 pd[i].first_tb = NULL;
807 invalidate_page_bitmap(pd + i);
808 }
809 } else {
810 void **pp = *lp;
811
812 for (i = 0; i < V_L2_SIZE; ++i) {
813 page_flush_tb_1(level - 1, pp + i);
814 }
815 }
816 }
817
818 static void page_flush_tb(void)
819 {
820 int i;
821
822 for (i = 0; i < V_L1_SIZE; i++) {
823 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
824 }
825 }
826
827 /* flush all the translation blocks */
828 /* XXX: tb_flush is currently not thread safe */
829 void tb_flush(CPUState *cpu)
830 {
831 #if defined(DEBUG_FLUSH)
832 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
833 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
834 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
835 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
836 tcg_ctx.tb_ctx.nb_tbs : 0);
837 #endif
838 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
839 > tcg_ctx.code_gen_buffer_size) {
840 cpu_abort(cpu, "Internal error: code buffer overflow\n");
841 }
842 tcg_ctx.tb_ctx.nb_tbs = 0;
843
844 CPU_FOREACH(cpu) {
845 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
846 cpu->tb_flushed = true;
847 }
848
849 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
850 page_flush_tb();
851
852 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
853 /* XXX: flush processor icache at this point if cache flush is
854 expensive */
855 tcg_ctx.tb_ctx.tb_flush_count++;
856 }
857
858 #ifdef DEBUG_TB_CHECK
859
860 static void tb_invalidate_check(target_ulong address)
861 {
862 TranslationBlock *tb;
863 int i;
864
865 address &= TARGET_PAGE_MASK;
866 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
867 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
868 tb = tb->phys_hash_next) {
869 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
870 address >= tb->pc + tb->size)) {
871 printf("ERROR invalidate: address=" TARGET_FMT_lx
872 " PC=%08lx size=%04x\n",
873 address, (long)tb->pc, tb->size);
874 }
875 }
876 }
877 }
878
879 /* verify that all the pages have correct rights for code */
880 static void tb_page_check(void)
881 {
882 TranslationBlock *tb;
883 int i, flags1, flags2;
884
885 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
886 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
887 tb = tb->phys_hash_next) {
888 flags1 = page_get_flags(tb->pc);
889 flags2 = page_get_flags(tb->pc + tb->size - 1);
890 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
891 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
892 (long)tb->pc, tb->size, flags1, flags2);
893 }
894 }
895 }
896 }
897
898 #endif
899
900 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
901 {
902 TranslationBlock *tb1;
903
904 for (;;) {
905 tb1 = *ptb;
906 if (tb1 == tb) {
907 *ptb = tb1->phys_hash_next;
908 break;
909 }
910 ptb = &tb1->phys_hash_next;
911 }
912 }
913
914 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
915 {
916 TranslationBlock *tb1;
917 unsigned int n1;
918
919 for (;;) {
920 tb1 = *ptb;
921 n1 = (uintptr_t)tb1 & 3;
922 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
923 if (tb1 == tb) {
924 *ptb = tb1->page_next[n1];
925 break;
926 }
927 ptb = &tb1->page_next[n1];
928 }
929 }
930
931 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
932 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n)
933 {
934 TranslationBlock *tb1;
935 uintptr_t *ptb, ntb;
936 unsigned int n1;
937
938 ptb = &tb->jmp_list_next[n];
939 if (*ptb) {
940 /* find tb(n) in circular list */
941 for (;;) {
942 ntb = *ptb;
943 n1 = ntb & 3;
944 tb1 = (TranslationBlock *)(ntb & ~3);
945 if (n1 == n && tb1 == tb) {
946 break;
947 }
948 if (n1 == 2) {
949 ptb = &tb1->jmp_list_first;
950 } else {
951 ptb = &tb1->jmp_list_next[n1];
952 }
953 }
954 /* now we can suppress tb(n) from the list */
955 *ptb = tb->jmp_list_next[n];
956
957 tb->jmp_list_next[n] = (uintptr_t)NULL;
958 }
959 }
960
961 /* reset the jump entry 'n' of a TB so that it is not chained to
962 another TB */
963 static inline void tb_reset_jump(TranslationBlock *tb, int n)
964 {
965 uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]);
966 tb_set_jmp_target(tb, n, addr);
967 }
968
969 /* remove any jumps to the TB */
970 static inline void tb_jmp_unlink(TranslationBlock *tb)
971 {
972 TranslationBlock *tb1;
973 uintptr_t *ptb, ntb;
974 unsigned int n1;
975
976 ptb = &tb->jmp_list_first;
977 for (;;) {
978 ntb = *ptb;
979 n1 = ntb & 3;
980 tb1 = (TranslationBlock *)(ntb & ~3);
981 if (n1 == 2) {
982 break;
983 }
984 tb_reset_jump(tb1, n1);
985 *ptb = tb1->jmp_list_next[n1];
986 tb1->jmp_list_next[n1] = (uintptr_t)NULL;
987 }
988 }
989
990 /* invalidate one TB */
991 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
992 {
993 CPUState *cpu;
994 PageDesc *p;
995 unsigned int h;
996 tb_page_addr_t phys_pc;
997
998 /* remove the TB from the hash list */
999 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1000 h = tb_phys_hash_func(phys_pc);
1001 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
1002
1003 /* remove the TB from the page list */
1004 if (tb->page_addr[0] != page_addr) {
1005 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
1006 tb_page_remove(&p->first_tb, tb);
1007 invalidate_page_bitmap(p);
1008 }
1009 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
1010 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
1011 tb_page_remove(&p->first_tb, tb);
1012 invalidate_page_bitmap(p);
1013 }
1014
1015 /* remove the TB from the hash list */
1016 h = tb_jmp_cache_hash_func(tb->pc);
1017 CPU_FOREACH(cpu) {
1018 if (cpu->tb_jmp_cache[h] == tb) {
1019 cpu->tb_jmp_cache[h] = NULL;
1020 }
1021 }
1022
1023 /* suppress this TB from the two jump lists */
1024 tb_remove_from_jmp_list(tb, 0);
1025 tb_remove_from_jmp_list(tb, 1);
1026
1027 /* suppress any remaining jumps to this TB */
1028 tb_jmp_unlink(tb);
1029
1030 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
1031 }
1032
1033 #ifdef CONFIG_SOFTMMU
1034 static void build_page_bitmap(PageDesc *p)
1035 {
1036 int n, tb_start, tb_end;
1037 TranslationBlock *tb;
1038
1039 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
1040
1041 tb = p->first_tb;
1042 while (tb != NULL) {
1043 n = (uintptr_t)tb & 3;
1044 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1045 /* NOTE: this is subtle as a TB may span two physical pages */
1046 if (n == 0) {
1047 /* NOTE: tb_end may be after the end of the page, but
1048 it is not a problem */
1049 tb_start = tb->pc & ~TARGET_PAGE_MASK;
1050 tb_end = tb_start + tb->size;
1051 if (tb_end > TARGET_PAGE_SIZE) {
1052 tb_end = TARGET_PAGE_SIZE;
1053 }
1054 } else {
1055 tb_start = 0;
1056 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1057 }
1058 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
1059 tb = tb->page_next[n];
1060 }
1061 }
1062 #endif
1063
1064 /* add the tb in the target page and protect it if necessary
1065 *
1066 * Called with mmap_lock held for user-mode emulation.
1067 */
1068 static inline void tb_alloc_page(TranslationBlock *tb,
1069 unsigned int n, tb_page_addr_t page_addr)
1070 {
1071 PageDesc *p;
1072 #ifndef CONFIG_USER_ONLY
1073 bool page_already_protected;
1074 #endif
1075
1076 tb->page_addr[n] = page_addr;
1077 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1078 tb->page_next[n] = p->first_tb;
1079 #ifndef CONFIG_USER_ONLY
1080 page_already_protected = p->first_tb != NULL;
1081 #endif
1082 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1083 invalidate_page_bitmap(p);
1084
1085 #if defined(CONFIG_USER_ONLY)
1086 if (p->flags & PAGE_WRITE) {
1087 target_ulong addr;
1088 PageDesc *p2;
1089 int prot;
1090
1091 /* force the host page as non writable (writes will have a
1092 page fault + mprotect overhead) */
1093 page_addr &= qemu_host_page_mask;
1094 prot = 0;
1095 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1096 addr += TARGET_PAGE_SIZE) {
1097
1098 p2 = page_find(addr >> TARGET_PAGE_BITS);
1099 if (!p2) {
1100 continue;
1101 }
1102 prot |= p2->flags;
1103 p2->flags &= ~PAGE_WRITE;
1104 }
1105 mprotect(g2h(page_addr), qemu_host_page_size,
1106 (prot & PAGE_BITS) & ~PAGE_WRITE);
1107 #ifdef DEBUG_TB_INVALIDATE
1108 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1109 page_addr);
1110 #endif
1111 }
1112 #else
1113 /* if some code is already present, then the pages are already
1114 protected. So we handle the case where only the first TB is
1115 allocated in a physical page */
1116 if (!page_already_protected) {
1117 tlb_protect_code(page_addr);
1118 }
1119 #endif
1120 }
1121
1122 /* add a new TB and link it to the physical page tables. phys_page2 is
1123 * (-1) to indicate that only one page contains the TB.
1124 *
1125 * Called with mmap_lock held for user-mode emulation.
1126 */
1127 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1128 tb_page_addr_t phys_page2)
1129 {
1130 unsigned int h;
1131 TranslationBlock **ptb;
1132
1133 /* add in the physical hash table */
1134 h = tb_phys_hash_func(phys_pc);
1135 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1136 tb->phys_hash_next = *ptb;
1137 *ptb = tb;
1138
1139 /* add in the page list */
1140 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1141 if (phys_page2 != -1) {
1142 tb_alloc_page(tb, 1, phys_page2);
1143 } else {
1144 tb->page_addr[1] = -1;
1145 }
1146
1147 #ifdef DEBUG_TB_CHECK
1148 tb_page_check();
1149 #endif
1150 }
1151
1152 /* Called with mmap_lock held for user mode emulation. */
1153 TranslationBlock *tb_gen_code(CPUState *cpu,
1154 target_ulong pc, target_ulong cs_base,
1155 uint32_t flags, int cflags)
1156 {
1157 CPUArchState *env = cpu->env_ptr;
1158 TranslationBlock *tb;
1159 tb_page_addr_t phys_pc, phys_page2;
1160 target_ulong virt_page2;
1161 tcg_insn_unit *gen_code_buf;
1162 int gen_code_size, search_size;
1163 #ifdef CONFIG_PROFILER
1164 int64_t ti;
1165 #endif
1166
1167 phys_pc = get_page_addr_code(env, pc);
1168 if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {
1169 cflags |= CF_USE_ICOUNT;
1170 }
1171
1172 tb = tb_alloc(pc);
1173 if (unlikely(!tb)) {
1174 buffer_overflow:
1175 /* flush must be done */
1176 tb_flush(cpu);
1177 /* cannot fail at this point */
1178 tb = tb_alloc(pc);
1179 assert(tb != NULL);
1180 }
1181
1182 gen_code_buf = tcg_ctx.code_gen_ptr;
1183 tb->tc_ptr = gen_code_buf;
1184 tb->cs_base = cs_base;
1185 tb->flags = flags;
1186 tb->cflags = cflags;
1187
1188 #ifdef CONFIG_PROFILER
1189 tcg_ctx.tb_count1++; /* includes aborted translations because of
1190 exceptions */
1191 ti = profile_getclock();
1192 #endif
1193
1194 tcg_func_start(&tcg_ctx);
1195
1196 gen_intermediate_code(env, tb);
1197
1198 trace_translate_block(tb, tb->pc, tb->tc_ptr);
1199
1200 /* generate machine code */
1201 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
1202 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
1203 tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset;
1204 #ifdef USE_DIRECT_JUMP
1205 tcg_ctx.tb_jmp_insn_offset = tb->jmp_insn_offset;
1206 tcg_ctx.tb_jmp_target_addr = NULL;
1207 #else
1208 tcg_ctx.tb_jmp_insn_offset = NULL;
1209 tcg_ctx.tb_jmp_target_addr = tb->jmp_target_addr;
1210 #endif
1211
1212 #ifdef CONFIG_PROFILER
1213 tcg_ctx.tb_count++;
1214 tcg_ctx.interm_time += profile_getclock() - ti;
1215 tcg_ctx.code_time -= profile_getclock();
1216 #endif
1217
1218 /* ??? Overflow could be handled better here. In particular, we
1219 don't need to re-do gen_intermediate_code, nor should we re-do
1220 the tcg optimization currently hidden inside tcg_gen_code. All
1221 that should be required is to flush the TBs, allocate a new TB,
1222 re-initialize it per above, and re-do the actual code generation. */
1223 gen_code_size = tcg_gen_code(&tcg_ctx, tb);
1224 if (unlikely(gen_code_size < 0)) {
1225 goto buffer_overflow;
1226 }
1227 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
1228 if (unlikely(search_size < 0)) {
1229 goto buffer_overflow;
1230 }
1231
1232 #ifdef CONFIG_PROFILER
1233 tcg_ctx.code_time += profile_getclock();
1234 tcg_ctx.code_in_len += tb->size;
1235 tcg_ctx.code_out_len += gen_code_size;
1236 tcg_ctx.search_out_len += search_size;
1237 #endif
1238
1239 #ifdef DEBUG_DISAS
1240 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
1241 qemu_log_in_addr_range(tb->pc)) {
1242 qemu_log("OUT: [size=%d]\n", gen_code_size);
1243 log_disas(tb->tc_ptr, gen_code_size);
1244 qemu_log("\n");
1245 qemu_log_flush();
1246 }
1247 #endif
1248
1249 tcg_ctx.code_gen_ptr = (void *)
1250 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
1251 CODE_GEN_ALIGN);
1252
1253 /* init jump list */
1254 assert(((uintptr_t)tb & 3) == 0);
1255 tb->jmp_list_first = (uintptr_t)tb | 2;
1256 tb->jmp_list_next[0] = (uintptr_t)NULL;
1257 tb->jmp_list_next[1] = (uintptr_t)NULL;
1258
1259 /* init original jump addresses wich has been set during tcg_gen_code() */
1260 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1261 tb_reset_jump(tb, 0);
1262 }
1263 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1264 tb_reset_jump(tb, 1);
1265 }
1266
1267 /* check next page if needed */
1268 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1269 phys_page2 = -1;
1270 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1271 phys_page2 = get_page_addr_code(env, virt_page2);
1272 }
1273 /* As long as consistency of the TB stuff is provided by tb_lock in user
1274 * mode and is implicit in single-threaded softmmu emulation, no explicit
1275 * memory barrier is required before tb_link_page() makes the TB visible
1276 * through the physical hash table and physical page list.
1277 */
1278 tb_link_page(tb, phys_pc, phys_page2);
1279 return tb;
1280 }
1281
1282 /*
1283 * Invalidate all TBs which intersect with the target physical address range
1284 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1285 * 'is_cpu_write_access' should be true if called from a real cpu write
1286 * access: the virtual CPU will exit the current TB if code is modified inside
1287 * this TB.
1288 *
1289 * Called with mmap_lock held for user-mode emulation
1290 */
1291 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
1292 {
1293 while (start < end) {
1294 tb_invalidate_phys_page_range(start, end, 0);
1295 start &= TARGET_PAGE_MASK;
1296 start += TARGET_PAGE_SIZE;
1297 }
1298 }
1299
1300 /*
1301 * Invalidate all TBs which intersect with the target physical address range
1302 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1303 * 'is_cpu_write_access' should be true if called from a real cpu write
1304 * access: the virtual CPU will exit the current TB if code is modified inside
1305 * this TB.
1306 *
1307 * Called with mmap_lock held for user-mode emulation
1308 */
1309 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1310 int is_cpu_write_access)
1311 {
1312 TranslationBlock *tb, *tb_next;
1313 #if defined(TARGET_HAS_PRECISE_SMC)
1314 CPUState *cpu = current_cpu;
1315 CPUArchState *env = NULL;
1316 #endif
1317 tb_page_addr_t tb_start, tb_end;
1318 PageDesc *p;
1319 int n;
1320 #ifdef TARGET_HAS_PRECISE_SMC
1321 int current_tb_not_found = is_cpu_write_access;
1322 TranslationBlock *current_tb = NULL;
1323 int current_tb_modified = 0;
1324 target_ulong current_pc = 0;
1325 target_ulong current_cs_base = 0;
1326 uint32_t current_flags = 0;
1327 #endif /* TARGET_HAS_PRECISE_SMC */
1328
1329 p = page_find(start >> TARGET_PAGE_BITS);
1330 if (!p) {
1331 return;
1332 }
1333 #if defined(TARGET_HAS_PRECISE_SMC)
1334 if (cpu != NULL) {
1335 env = cpu->env_ptr;
1336 }
1337 #endif
1338
1339 /* we remove all the TBs in the range [start, end[ */
1340 /* XXX: see if in some cases it could be faster to invalidate all
1341 the code */
1342 tb = p->first_tb;
1343 while (tb != NULL) {
1344 n = (uintptr_t)tb & 3;
1345 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1346 tb_next = tb->page_next[n];
1347 /* NOTE: this is subtle as a TB may span two physical pages */
1348 if (n == 0) {
1349 /* NOTE: tb_end may be after the end of the page, but
1350 it is not a problem */
1351 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1352 tb_end = tb_start + tb->size;
1353 } else {
1354 tb_start = tb->page_addr[1];
1355 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1356 }
1357 if (!(tb_end <= start || tb_start >= end)) {
1358 #ifdef TARGET_HAS_PRECISE_SMC
1359 if (current_tb_not_found) {
1360 current_tb_not_found = 0;
1361 current_tb = NULL;
1362 if (cpu->mem_io_pc) {
1363 /* now we have a real cpu fault */
1364 current_tb = tb_find_pc(cpu->mem_io_pc);
1365 }
1366 }
1367 if (current_tb == tb &&
1368 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1369 /* If we are modifying the current TB, we must stop
1370 its execution. We could be more precise by checking
1371 that the modification is after the current PC, but it
1372 would require a specialized function to partially
1373 restore the CPU state */
1374
1375 current_tb_modified = 1;
1376 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
1377 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1378 &current_flags);
1379 }
1380 #endif /* TARGET_HAS_PRECISE_SMC */
1381 tb_phys_invalidate(tb, -1);
1382 }
1383 tb = tb_next;
1384 }
1385 #if !defined(CONFIG_USER_ONLY)
1386 /* if no code remaining, no need to continue to use slow writes */
1387 if (!p->first_tb) {
1388 invalidate_page_bitmap(p);
1389 tlb_unprotect_code(start);
1390 }
1391 #endif
1392 #ifdef TARGET_HAS_PRECISE_SMC
1393 if (current_tb_modified) {
1394 /* we generate a block containing just the instruction
1395 modifying the memory. It will ensure that it cannot modify
1396 itself */
1397 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1398 cpu_resume_from_signal(cpu, NULL);
1399 }
1400 #endif
1401 }
1402
1403 #ifdef CONFIG_SOFTMMU
1404 /* len must be <= 8 and start must be a multiple of len */
1405 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1406 {
1407 PageDesc *p;
1408
1409 #if 0
1410 if (1) {
1411 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1412 cpu_single_env->mem_io_vaddr, len,
1413 cpu_single_env->eip,
1414 cpu_single_env->eip +
1415 (intptr_t)cpu_single_env->segs[R_CS].base);
1416 }
1417 #endif
1418 p = page_find(start >> TARGET_PAGE_BITS);
1419 if (!p) {
1420 return;
1421 }
1422 if (!p->code_bitmap &&
1423 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
1424 /* build code bitmap */
1425 build_page_bitmap(p);
1426 }
1427 if (p->code_bitmap) {
1428 unsigned int nr;
1429 unsigned long b;
1430
1431 nr = start & ~TARGET_PAGE_MASK;
1432 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
1433 if (b & ((1 << len) - 1)) {
1434 goto do_invalidate;
1435 }
1436 } else {
1437 do_invalidate:
1438 tb_invalidate_phys_page_range(start, start + len, 1);
1439 }
1440 }
1441 #else
1442 /* Called with mmap_lock held. */
1443 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1444 uintptr_t pc, void *puc,
1445 bool locked)
1446 {
1447 TranslationBlock *tb;
1448 PageDesc *p;
1449 int n;
1450 #ifdef TARGET_HAS_PRECISE_SMC
1451 TranslationBlock *current_tb = NULL;
1452 CPUState *cpu = current_cpu;
1453 CPUArchState *env = NULL;
1454 int current_tb_modified = 0;
1455 target_ulong current_pc = 0;
1456 target_ulong current_cs_base = 0;
1457 uint32_t current_flags = 0;
1458 #endif
1459
1460 addr &= TARGET_PAGE_MASK;
1461 p = page_find(addr >> TARGET_PAGE_BITS);
1462 if (!p) {
1463 return;
1464 }
1465 tb = p->first_tb;
1466 #ifdef TARGET_HAS_PRECISE_SMC
1467 if (tb && pc != 0) {
1468 current_tb = tb_find_pc(pc);
1469 }
1470 if (cpu != NULL) {
1471 env = cpu->env_ptr;
1472 }
1473 #endif
1474 while (tb != NULL) {
1475 n = (uintptr_t)tb & 3;
1476 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1477 #ifdef TARGET_HAS_PRECISE_SMC
1478 if (current_tb == tb &&
1479 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1480 /* If we are modifying the current TB, we must stop
1481 its execution. We could be more precise by checking
1482 that the modification is after the current PC, but it
1483 would require a specialized function to partially
1484 restore the CPU state */
1485
1486 current_tb_modified = 1;
1487 cpu_restore_state_from_tb(cpu, current_tb, pc);
1488 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1489 &current_flags);
1490 }
1491 #endif /* TARGET_HAS_PRECISE_SMC */
1492 tb_phys_invalidate(tb, addr);
1493 tb = tb->page_next[n];
1494 }
1495 p->first_tb = NULL;
1496 #ifdef TARGET_HAS_PRECISE_SMC
1497 if (current_tb_modified) {
1498 /* we generate a block containing just the instruction
1499 modifying the memory. It will ensure that it cannot modify
1500 itself */
1501 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1502 if (locked) {
1503 mmap_unlock();
1504 }
1505 cpu_resume_from_signal(cpu, puc);
1506 }
1507 #endif
1508 }
1509 #endif
1510
1511 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1512 tb[1].tc_ptr. Return NULL if not found */
1513 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1514 {
1515 int m_min, m_max, m;
1516 uintptr_t v;
1517 TranslationBlock *tb;
1518
1519 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1520 return NULL;
1521 }
1522 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1523 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1524 return NULL;
1525 }
1526 /* binary search (cf Knuth) */
1527 m_min = 0;
1528 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1529 while (m_min <= m_max) {
1530 m = (m_min + m_max) >> 1;
1531 tb = &tcg_ctx.tb_ctx.tbs[m];
1532 v = (uintptr_t)tb->tc_ptr;
1533 if (v == tc_ptr) {
1534 return tb;
1535 } else if (tc_ptr < v) {
1536 m_max = m - 1;
1537 } else {
1538 m_min = m + 1;
1539 }
1540 }
1541 return &tcg_ctx.tb_ctx.tbs[m_max];
1542 }
1543
1544 #if !defined(CONFIG_USER_ONLY)
1545 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1546 {
1547 ram_addr_t ram_addr;
1548 MemoryRegion *mr;
1549 hwaddr l = 1;
1550
1551 rcu_read_lock();
1552 mr = address_space_translate(as, addr, &addr, &l, false);
1553 if (!(memory_region_is_ram(mr)
1554 || memory_region_is_romd(mr))) {
1555 rcu_read_unlock();
1556 return;
1557 }
1558 ram_addr = memory_region_get_ram_addr(mr) + addr;
1559 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1560 rcu_read_unlock();
1561 }
1562 #endif /* !defined(CONFIG_USER_ONLY) */
1563
1564 void tb_check_watchpoint(CPUState *cpu)
1565 {
1566 TranslationBlock *tb;
1567
1568 tb = tb_find_pc(cpu->mem_io_pc);
1569 if (tb) {
1570 /* We can use retranslation to find the PC. */
1571 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
1572 tb_phys_invalidate(tb, -1);
1573 } else {
1574 /* The exception probably happened in a helper. The CPU state should
1575 have been saved before calling it. Fetch the PC from there. */
1576 CPUArchState *env = cpu->env_ptr;
1577 target_ulong pc, cs_base;
1578 tb_page_addr_t addr;
1579 uint32_t flags;
1580
1581 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
1582 addr = get_page_addr_code(env, pc);
1583 tb_invalidate_phys_range(addr, addr + 1);
1584 }
1585 }
1586
1587 #ifndef CONFIG_USER_ONLY
1588 /* in deterministic execution mode, instructions doing device I/Os
1589 must be at the end of the TB */
1590 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
1591 {
1592 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1593 CPUArchState *env = cpu->env_ptr;
1594 #endif
1595 TranslationBlock *tb;
1596 uint32_t n, cflags;
1597 target_ulong pc, cs_base;
1598 uint32_t flags;
1599
1600 tb = tb_find_pc(retaddr);
1601 if (!tb) {
1602 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
1603 (void *)retaddr);
1604 }
1605 n = cpu->icount_decr.u16.low + tb->icount;
1606 cpu_restore_state_from_tb(cpu, tb, retaddr);
1607 /* Calculate how many instructions had been executed before the fault
1608 occurred. */
1609 n = n - cpu->icount_decr.u16.low;
1610 /* Generate a new TB ending on the I/O insn. */
1611 n++;
1612 /* On MIPS and SH, delay slot instructions can only be restarted if
1613 they were already the first instruction in the TB. If this is not
1614 the first instruction in a TB then re-execute the preceding
1615 branch. */
1616 #if defined(TARGET_MIPS)
1617 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1618 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
1619 cpu->icount_decr.u16.low++;
1620 env->hflags &= ~MIPS_HFLAG_BMASK;
1621 }
1622 #elif defined(TARGET_SH4)
1623 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1624 && n > 1) {
1625 env->pc -= 2;
1626 cpu->icount_decr.u16.low++;
1627 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1628 }
1629 #endif
1630 /* This should never happen. */
1631 if (n > CF_COUNT_MASK) {
1632 cpu_abort(cpu, "TB too big during recompile");
1633 }
1634
1635 cflags = n | CF_LAST_IO;
1636 pc = tb->pc;
1637 cs_base = tb->cs_base;
1638 flags = tb->flags;
1639 tb_phys_invalidate(tb, -1);
1640 if (tb->cflags & CF_NOCACHE) {
1641 if (tb->orig_tb) {
1642 /* Invalidate original TB if this TB was generated in
1643 * cpu_exec_nocache() */
1644 tb_phys_invalidate(tb->orig_tb, -1);
1645 }
1646 tb_free(tb);
1647 }
1648 /* FIXME: In theory this could raise an exception. In practice
1649 we have already translated the block once so it's probably ok. */
1650 tb_gen_code(cpu, pc, cs_base, flags, cflags);
1651 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1652 the first in the TB) then we end up generating a whole new TB and
1653 repeating the fault, which is horribly inefficient.
1654 Better would be to execute just this insn uncached, or generate a
1655 second new TB. */
1656 cpu_resume_from_signal(cpu, NULL);
1657 }
1658
1659 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
1660 {
1661 unsigned int i;
1662
1663 /* Discard jump cache entries for any tb which might potentially
1664 overlap the flushed page. */
1665 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1666 memset(&cpu->tb_jmp_cache[i], 0,
1667 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1668
1669 i = tb_jmp_cache_hash_page(addr);
1670 memset(&cpu->tb_jmp_cache[i], 0,
1671 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1672 }
1673
1674 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1675 {
1676 int i, target_code_size, max_target_code_size;
1677 int direct_jmp_count, direct_jmp2_count, cross_page;
1678 TranslationBlock *tb;
1679
1680 target_code_size = 0;
1681 max_target_code_size = 0;
1682 cross_page = 0;
1683 direct_jmp_count = 0;
1684 direct_jmp2_count = 0;
1685 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1686 tb = &tcg_ctx.tb_ctx.tbs[i];
1687 target_code_size += tb->size;
1688 if (tb->size > max_target_code_size) {
1689 max_target_code_size = tb->size;
1690 }
1691 if (tb->page_addr[1] != -1) {
1692 cross_page++;
1693 }
1694 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1695 direct_jmp_count++;
1696 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1697 direct_jmp2_count++;
1698 }
1699 }
1700 }
1701 /* XXX: avoid using doubles ? */
1702 cpu_fprintf(f, "Translation buffer state:\n");
1703 cpu_fprintf(f, "gen code size %td/%zd\n",
1704 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1705 tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer);
1706 cpu_fprintf(f, "TB count %d/%d\n",
1707 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1708 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1709 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1710 tcg_ctx.tb_ctx.nb_tbs : 0,
1711 max_target_code_size);
1712 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1713 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1714 tcg_ctx.code_gen_buffer) /
1715 tcg_ctx.tb_ctx.nb_tbs : 0,
1716 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1717 tcg_ctx.code_gen_buffer) /
1718 target_code_size : 0);
1719 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1720 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1721 tcg_ctx.tb_ctx.nb_tbs : 0);
1722 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1723 direct_jmp_count,
1724 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1725 tcg_ctx.tb_ctx.nb_tbs : 0,
1726 direct_jmp2_count,
1727 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1728 tcg_ctx.tb_ctx.nb_tbs : 0);
1729 cpu_fprintf(f, "\nStatistics:\n");
1730 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1731 cpu_fprintf(f, "TB invalidate count %d\n",
1732 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1733 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1734 tcg_dump_info(f, cpu_fprintf);
1735 }
1736
1737 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
1738 {
1739 tcg_dump_op_count(f, cpu_fprintf);
1740 }
1741
1742 #else /* CONFIG_USER_ONLY */
1743
1744 void cpu_interrupt(CPUState *cpu, int mask)
1745 {
1746 cpu->interrupt_request |= mask;
1747 cpu->tcg_exit_req = 1;
1748 }
1749
1750 /*
1751 * Walks guest process memory "regions" one by one
1752 * and calls callback function 'fn' for each region.
1753 */
1754 struct walk_memory_regions_data {
1755 walk_memory_regions_fn fn;
1756 void *priv;
1757 target_ulong start;
1758 int prot;
1759 };
1760
1761 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1762 target_ulong end, int new_prot)
1763 {
1764 if (data->start != -1u) {
1765 int rc = data->fn(data->priv, data->start, end, data->prot);
1766 if (rc != 0) {
1767 return rc;
1768 }
1769 }
1770
1771 data->start = (new_prot ? end : -1u);
1772 data->prot = new_prot;
1773
1774 return 0;
1775 }
1776
1777 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1778 target_ulong base, int level, void **lp)
1779 {
1780 target_ulong pa;
1781 int i, rc;
1782
1783 if (*lp == NULL) {
1784 return walk_memory_regions_end(data, base, 0);
1785 }
1786
1787 if (level == 0) {
1788 PageDesc *pd = *lp;
1789
1790 for (i = 0; i < V_L2_SIZE; ++i) {
1791 int prot = pd[i].flags;
1792
1793 pa = base | (i << TARGET_PAGE_BITS);
1794 if (prot != data->prot) {
1795 rc = walk_memory_regions_end(data, pa, prot);
1796 if (rc != 0) {
1797 return rc;
1798 }
1799 }
1800 }
1801 } else {
1802 void **pp = *lp;
1803
1804 for (i = 0; i < V_L2_SIZE; ++i) {
1805 pa = base | ((target_ulong)i <<
1806 (TARGET_PAGE_BITS + V_L2_BITS * level));
1807 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1808 if (rc != 0) {
1809 return rc;
1810 }
1811 }
1812 }
1813
1814 return 0;
1815 }
1816
1817 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1818 {
1819 struct walk_memory_regions_data data;
1820 uintptr_t i;
1821
1822 data.fn = fn;
1823 data.priv = priv;
1824 data.start = -1u;
1825 data.prot = 0;
1826
1827 for (i = 0; i < V_L1_SIZE; i++) {
1828 int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
1829 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1830 if (rc != 0) {
1831 return rc;
1832 }
1833 }
1834
1835 return walk_memory_regions_end(&data, 0, 0);
1836 }
1837
1838 static int dump_region(void *priv, target_ulong start,
1839 target_ulong end, unsigned long prot)
1840 {
1841 FILE *f = (FILE *)priv;
1842
1843 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
1844 " "TARGET_FMT_lx" %c%c%c\n",
1845 start, end, end - start,
1846 ((prot & PAGE_READ) ? 'r' : '-'),
1847 ((prot & PAGE_WRITE) ? 'w' : '-'),
1848 ((prot & PAGE_EXEC) ? 'x' : '-'));
1849
1850 return 0;
1851 }
1852
1853 /* dump memory mappings */
1854 void page_dump(FILE *f)
1855 {
1856 const int length = sizeof(target_ulong) * 2;
1857 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1858 length, "start", length, "end", length, "size", "prot");
1859 walk_memory_regions(f, dump_region);
1860 }
1861
1862 int page_get_flags(target_ulong address)
1863 {
1864 PageDesc *p;
1865
1866 p = page_find(address >> TARGET_PAGE_BITS);
1867 if (!p) {
1868 return 0;
1869 }
1870 return p->flags;
1871 }
1872
1873 /* Modify the flags of a page and invalidate the code if necessary.
1874 The flag PAGE_WRITE_ORG is positioned automatically depending
1875 on PAGE_WRITE. The mmap_lock should already be held. */
1876 void page_set_flags(target_ulong start, target_ulong end, int flags)
1877 {
1878 target_ulong addr, len;
1879
1880 /* This function should never be called with addresses outside the
1881 guest address space. If this assert fires, it probably indicates
1882 a missing call to h2g_valid. */
1883 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1884 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1885 #endif
1886 assert(start < end);
1887
1888 start = start & TARGET_PAGE_MASK;
1889 end = TARGET_PAGE_ALIGN(end);
1890
1891 if (flags & PAGE_WRITE) {
1892 flags |= PAGE_WRITE_ORG;
1893 }
1894
1895 for (addr = start, len = end - start;
1896 len != 0;
1897 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1898 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1899
1900 /* If the write protection bit is set, then we invalidate
1901 the code inside. */
1902 if (!(p->flags & PAGE_WRITE) &&
1903 (flags & PAGE_WRITE) &&
1904 p->first_tb) {
1905 tb_invalidate_phys_page(addr, 0, NULL, false);
1906 }
1907 p->flags = flags;
1908 }
1909 }
1910
1911 int page_check_range(target_ulong start, target_ulong len, int flags)
1912 {
1913 PageDesc *p;
1914 target_ulong end;
1915 target_ulong addr;
1916
1917 /* This function should never be called with addresses outside the
1918 guest address space. If this assert fires, it probably indicates
1919 a missing call to h2g_valid. */
1920 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1921 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1922 #endif
1923
1924 if (len == 0) {
1925 return 0;
1926 }
1927 if (start + len - 1 < start) {
1928 /* We've wrapped around. */
1929 return -1;
1930 }
1931
1932 /* must do before we loose bits in the next step */
1933 end = TARGET_PAGE_ALIGN(start + len);
1934 start = start & TARGET_PAGE_MASK;
1935
1936 for (addr = start, len = end - start;
1937 len != 0;
1938 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1939 p = page_find(addr >> TARGET_PAGE_BITS);
1940 if (!p) {
1941 return -1;
1942 }
1943 if (!(p->flags & PAGE_VALID)) {
1944 return -1;
1945 }
1946
1947 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1948 return -1;
1949 }
1950 if (flags & PAGE_WRITE) {
1951 if (!(p->flags & PAGE_WRITE_ORG)) {
1952 return -1;
1953 }
1954 /* unprotect the page if it was put read-only because it
1955 contains translated code */
1956 if (!(p->flags & PAGE_WRITE)) {
1957 if (!page_unprotect(addr, 0, NULL)) {
1958 return -1;
1959 }
1960 }
1961 }
1962 }
1963 return 0;
1964 }
1965
1966 /* called from signal handler: invalidate the code and unprotect the
1967 page. Return TRUE if the fault was successfully handled. */
1968 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1969 {
1970 unsigned int prot;
1971 PageDesc *p;
1972 target_ulong host_start, host_end, addr;
1973
1974 /* Technically this isn't safe inside a signal handler. However we
1975 know this only ever happens in a synchronous SEGV handler, so in
1976 practice it seems to be ok. */
1977 mmap_lock();
1978
1979 p = page_find(address >> TARGET_PAGE_BITS);
1980 if (!p) {
1981 mmap_unlock();
1982 return 0;
1983 }
1984
1985 /* if the page was really writable, then we change its
1986 protection back to writable */
1987 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1988 host_start = address & qemu_host_page_mask;
1989 host_end = host_start + qemu_host_page_size;
1990
1991 prot = 0;
1992 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1993 p = page_find(addr >> TARGET_PAGE_BITS);
1994 p->flags |= PAGE_WRITE;
1995 prot |= p->flags;
1996
1997 /* and since the content will be modified, we must invalidate
1998 the corresponding translated code. */
1999 tb_invalidate_phys_page(addr, pc, puc, true);
2000 #ifdef DEBUG_TB_CHECK
2001 tb_invalidate_check(addr);
2002 #endif
2003 }
2004 mprotect((void *)g2h(host_start), qemu_host_page_size,
2005 prot & PAGE_BITS);
2006
2007 mmap_unlock();
2008 return 1;
2009 }
2010 mmap_unlock();
2011 return 0;
2012 }
2013 #endif /* CONFIG_USER_ONLY */