nbd: Don't use *_to_cpup() functions
[qemu.git] / translate-all.c
1 /*
2 * Host code generation
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #ifdef _WIN32
20 #include <windows.h>
21 #endif
22 #include "qemu/osdep.h"
23
24
25 #include "qemu-common.h"
26 #define NO_CPU_IO_DEFS
27 #include "cpu.h"
28 #include "trace.h"
29 #include "disas/disas.h"
30 #include "exec/exec-all.h"
31 #include "tcg.h"
32 #if defined(CONFIG_USER_ONLY)
33 #include "qemu.h"
34 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
35 #include <sys/param.h>
36 #if __FreeBSD_version >= 700104
37 #define HAVE_KINFO_GETVMMAP
38 #define sigqueue sigqueue_freebsd /* avoid redefinition */
39 #include <sys/proc.h>
40 #include <machine/profile.h>
41 #define _KERNEL
42 #include <sys/user.h>
43 #undef _KERNEL
44 #undef sigqueue
45 #include <libutil.h>
46 #endif
47 #endif
48 #else
49 #include "exec/address-spaces.h"
50 #endif
51
52 #include "exec/cputlb.h"
53 #include "exec/tb-hash.h"
54 #include "translate-all.h"
55 #include "qemu/bitmap.h"
56 #include "qemu/timer.h"
57 #include "exec/log.h"
58
59 //#define DEBUG_TB_INVALIDATE
60 //#define DEBUG_FLUSH
61 /* make various TB consistency checks */
62 //#define DEBUG_TB_CHECK
63
64 #if !defined(CONFIG_USER_ONLY)
65 /* TB consistency checks only implemented for usermode emulation. */
66 #undef DEBUG_TB_CHECK
67 #endif
68
69 #define SMC_BITMAP_USE_THRESHOLD 10
70
71 typedef struct PageDesc {
72 /* list of TBs intersecting this ram page */
73 TranslationBlock *first_tb;
74 #ifdef CONFIG_SOFTMMU
75 /* in order to optimize self modifying code, we count the number
76 of lookups we do to a given page to use a bitmap */
77 unsigned int code_write_count;
78 unsigned long *code_bitmap;
79 #else
80 unsigned long flags;
81 #endif
82 } PageDesc;
83
84 /* In system mode we want L1_MAP to be based on ram offsets,
85 while in user mode we want it to be based on virtual addresses. */
86 #if !defined(CONFIG_USER_ONLY)
87 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
88 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
89 #else
90 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
91 #endif
92 #else
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
94 #endif
95
96 /* Size of the L2 (and L3, etc) page tables. */
97 #define V_L2_BITS 10
98 #define V_L2_SIZE (1 << V_L2_BITS)
99
100 /* The bits remaining after N lower levels of page tables. */
101 #define V_L1_BITS_REM \
102 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
103
104 #if V_L1_BITS_REM < 4
105 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
106 #else
107 #define V_L1_BITS V_L1_BITS_REM
108 #endif
109
110 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
111
112 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
113
114 uintptr_t qemu_host_page_size;
115 intptr_t qemu_host_page_mask;
116
117 /* The bottom level has pointers to PageDesc */
118 static void *l1_map[V_L1_SIZE];
119
120 /* code generation context */
121 TCGContext tcg_ctx;
122
123 /* translation block context */
124 #ifdef CONFIG_USER_ONLY
125 __thread int have_tb_lock;
126 #endif
127
128 void tb_lock(void)
129 {
130 #ifdef CONFIG_USER_ONLY
131 assert(!have_tb_lock);
132 qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
133 have_tb_lock++;
134 #endif
135 }
136
137 void tb_unlock(void)
138 {
139 #ifdef CONFIG_USER_ONLY
140 assert(have_tb_lock);
141 have_tb_lock--;
142 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
143 #endif
144 }
145
146 void tb_lock_reset(void)
147 {
148 #ifdef CONFIG_USER_ONLY
149 if (have_tb_lock) {
150 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
151 have_tb_lock = 0;
152 }
153 #endif
154 }
155
156 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
157
158 void cpu_gen_init(void)
159 {
160 tcg_context_init(&tcg_ctx);
161 }
162
163 /* Encode VAL as a signed leb128 sequence at P.
164 Return P incremented past the encoded value. */
165 static uint8_t *encode_sleb128(uint8_t *p, target_long val)
166 {
167 int more, byte;
168
169 do {
170 byte = val & 0x7f;
171 val >>= 7;
172 more = !((val == 0 && (byte & 0x40) == 0)
173 || (val == -1 && (byte & 0x40) != 0));
174 if (more) {
175 byte |= 0x80;
176 }
177 *p++ = byte;
178 } while (more);
179
180 return p;
181 }
182
183 /* Decode a signed leb128 sequence at *PP; increment *PP past the
184 decoded value. Return the decoded value. */
185 static target_long decode_sleb128(uint8_t **pp)
186 {
187 uint8_t *p = *pp;
188 target_long val = 0;
189 int byte, shift = 0;
190
191 do {
192 byte = *p++;
193 val |= (target_ulong)(byte & 0x7f) << shift;
194 shift += 7;
195 } while (byte & 0x80);
196 if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
197 val |= -(target_ulong)1 << shift;
198 }
199
200 *pp = p;
201 return val;
202 }
203
204 /* Encode the data collected about the instructions while compiling TB.
205 Place the data at BLOCK, and return the number of bytes consumed.
206
207 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
208 which come from the target's insn_start data, followed by a uintptr_t
209 which comes from the host pc of the end of the code implementing the insn.
210
211 Each line of the table is encoded as sleb128 deltas from the previous
212 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
213 That is, the first column is seeded with the guest pc, the last column
214 with the host pc, and the middle columns with zeros. */
215
216 static int encode_search(TranslationBlock *tb, uint8_t *block)
217 {
218 uint8_t *highwater = tcg_ctx.code_gen_highwater;
219 uint8_t *p = block;
220 int i, j, n;
221
222 tb->tc_search = block;
223
224 for (i = 0, n = tb->icount; i < n; ++i) {
225 target_ulong prev;
226
227 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
228 if (i == 0) {
229 prev = (j == 0 ? tb->pc : 0);
230 } else {
231 prev = tcg_ctx.gen_insn_data[i - 1][j];
232 }
233 p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev);
234 }
235 prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]);
236 p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev);
237
238 /* Test for (pending) buffer overflow. The assumption is that any
239 one row beginning below the high water mark cannot overrun
240 the buffer completely. Thus we can test for overflow after
241 encoding a row without having to check during encoding. */
242 if (unlikely(p > highwater)) {
243 return -1;
244 }
245 }
246
247 return p - block;
248 }
249
250 /* The cpu state corresponding to 'searched_pc' is restored. */
251 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
252 uintptr_t searched_pc)
253 {
254 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
255 uintptr_t host_pc = (uintptr_t)tb->tc_ptr;
256 CPUArchState *env = cpu->env_ptr;
257 uint8_t *p = tb->tc_search;
258 int i, j, num_insns = tb->icount;
259 #ifdef CONFIG_PROFILER
260 int64_t ti = profile_getclock();
261 #endif
262
263 if (searched_pc < host_pc) {
264 return -1;
265 }
266
267 /* Reconstruct the stored insn data while looking for the point at
268 which the end of the insn exceeds the searched_pc. */
269 for (i = 0; i < num_insns; ++i) {
270 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
271 data[j] += decode_sleb128(&p);
272 }
273 host_pc += decode_sleb128(&p);
274 if (host_pc > searched_pc) {
275 goto found;
276 }
277 }
278 return -1;
279
280 found:
281 if (tb->cflags & CF_USE_ICOUNT) {
282 assert(use_icount);
283 /* Reset the cycle counter to the start of the block. */
284 cpu->icount_decr.u16.low += num_insns;
285 /* Clear the IO flag. */
286 cpu->can_do_io = 0;
287 }
288 cpu->icount_decr.u16.low -= i;
289 restore_state_to_opc(env, tb, data);
290
291 #ifdef CONFIG_PROFILER
292 tcg_ctx.restore_time += profile_getclock() - ti;
293 tcg_ctx.restore_count++;
294 #endif
295 return 0;
296 }
297
298 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
299 {
300 TranslationBlock *tb;
301
302 tb = tb_find_pc(retaddr);
303 if (tb) {
304 cpu_restore_state_from_tb(cpu, tb, retaddr);
305 if (tb->cflags & CF_NOCACHE) {
306 /* one-shot translation, invalidate it immediately */
307 tb_phys_invalidate(tb, -1);
308 tb_free(tb);
309 }
310 return true;
311 }
312 return false;
313 }
314
315 void page_size_init(void)
316 {
317 /* NOTE: we can always suppose that qemu_host_page_size >=
318 TARGET_PAGE_SIZE */
319 qemu_real_host_page_size = getpagesize();
320 qemu_real_host_page_mask = -(intptr_t)qemu_real_host_page_size;
321 if (qemu_host_page_size == 0) {
322 qemu_host_page_size = qemu_real_host_page_size;
323 }
324 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
325 qemu_host_page_size = TARGET_PAGE_SIZE;
326 }
327 qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
328 }
329
330 static void page_init(void)
331 {
332 page_size_init();
333 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
334 {
335 #ifdef HAVE_KINFO_GETVMMAP
336 struct kinfo_vmentry *freep;
337 int i, cnt;
338
339 freep = kinfo_getvmmap(getpid(), &cnt);
340 if (freep) {
341 mmap_lock();
342 for (i = 0; i < cnt; i++) {
343 unsigned long startaddr, endaddr;
344
345 startaddr = freep[i].kve_start;
346 endaddr = freep[i].kve_end;
347 if (h2g_valid(startaddr)) {
348 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
349
350 if (h2g_valid(endaddr)) {
351 endaddr = h2g(endaddr);
352 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
353 } else {
354 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
355 endaddr = ~0ul;
356 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
357 #endif
358 }
359 }
360 }
361 free(freep);
362 mmap_unlock();
363 }
364 #else
365 FILE *f;
366
367 last_brk = (unsigned long)sbrk(0);
368
369 f = fopen("/compat/linux/proc/self/maps", "r");
370 if (f) {
371 mmap_lock();
372
373 do {
374 unsigned long startaddr, endaddr;
375 int n;
376
377 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
378
379 if (n == 2 && h2g_valid(startaddr)) {
380 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
381
382 if (h2g_valid(endaddr)) {
383 endaddr = h2g(endaddr);
384 } else {
385 endaddr = ~0ul;
386 }
387 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
388 }
389 } while (!feof(f));
390
391 fclose(f);
392 mmap_unlock();
393 }
394 #endif
395 }
396 #endif
397 }
398
399 /* If alloc=1:
400 * Called with mmap_lock held for user-mode emulation.
401 */
402 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
403 {
404 PageDesc *pd;
405 void **lp;
406 int i;
407
408 /* Level 1. Always allocated. */
409 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
410
411 /* Level 2..N-1. */
412 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
413 void **p = atomic_rcu_read(lp);
414
415 if (p == NULL) {
416 if (!alloc) {
417 return NULL;
418 }
419 p = g_new0(void *, V_L2_SIZE);
420 atomic_rcu_set(lp, p);
421 }
422
423 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
424 }
425
426 pd = atomic_rcu_read(lp);
427 if (pd == NULL) {
428 if (!alloc) {
429 return NULL;
430 }
431 pd = g_new0(PageDesc, V_L2_SIZE);
432 atomic_rcu_set(lp, pd);
433 }
434
435 return pd + (index & (V_L2_SIZE - 1));
436 }
437
438 static inline PageDesc *page_find(tb_page_addr_t index)
439 {
440 return page_find_alloc(index, 0);
441 }
442
443 #if defined(CONFIG_USER_ONLY)
444 /* Currently it is not recommended to allocate big chunks of data in
445 user mode. It will change when a dedicated libc will be used. */
446 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
447 region in which the guest needs to run. Revisit this. */
448 #define USE_STATIC_CODE_GEN_BUFFER
449 #endif
450
451 /* Minimum size of the code gen buffer. This number is randomly chosen,
452 but not so small that we can't have a fair number of TB's live. */
453 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
454
455 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
456 indicated, this is constrained by the range of direct branches on the
457 host cpu, as used by the TCG implementation of goto_tb. */
458 #if defined(__x86_64__)
459 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
460 #elif defined(__sparc__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
462 #elif defined(__powerpc64__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__powerpc__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
466 #elif defined(__aarch64__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
468 #elif defined(__arm__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
470 #elif defined(__s390x__)
471 /* We have a +- 4GB range on the branches; leave some slop. */
472 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
473 #elif defined(__mips__)
474 /* We have a 256MB branch region, but leave room to make sure the
475 main executable is also within that region. */
476 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
477 #else
478 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
479 #endif
480
481 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
482
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
484 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
485 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
486
487 static inline size_t size_code_gen_buffer(size_t tb_size)
488 {
489 /* Size the buffer. */
490 if (tb_size == 0) {
491 #ifdef USE_STATIC_CODE_GEN_BUFFER
492 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
493 #else
494 /* ??? Needs adjustments. */
495 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
496 static buffer, we could size this on RESERVED_VA, on the text
497 segment size of the executable, or continue to use the default. */
498 tb_size = (unsigned long)(ram_size / 4);
499 #endif
500 }
501 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
502 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
503 }
504 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
505 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
506 }
507 return tb_size;
508 }
509
510 #ifdef __mips__
511 /* In order to use J and JAL within the code_gen_buffer, we require
512 that the buffer not cross a 256MB boundary. */
513 static inline bool cross_256mb(void *addr, size_t size)
514 {
515 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
516 }
517
518 /* We weren't able to allocate a buffer without crossing that boundary,
519 so make do with the larger portion of the buffer that doesn't cross.
520 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
521 static inline void *split_cross_256mb(void *buf1, size_t size1)
522 {
523 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
524 size_t size2 = buf1 + size1 - buf2;
525
526 size1 = buf2 - buf1;
527 if (size1 < size2) {
528 size1 = size2;
529 buf1 = buf2;
530 }
531
532 tcg_ctx.code_gen_buffer_size = size1;
533 return buf1;
534 }
535 #endif
536
537 #ifdef USE_STATIC_CODE_GEN_BUFFER
538 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
539 __attribute__((aligned(CODE_GEN_ALIGN)));
540
541 # ifdef _WIN32
542 static inline void do_protect(void *addr, long size, int prot)
543 {
544 DWORD old_protect;
545 VirtualProtect(addr, size, prot, &old_protect);
546 }
547
548 static inline void map_exec(void *addr, long size)
549 {
550 do_protect(addr, size, PAGE_EXECUTE_READWRITE);
551 }
552
553 static inline void map_none(void *addr, long size)
554 {
555 do_protect(addr, size, PAGE_NOACCESS);
556 }
557 # else
558 static inline void do_protect(void *addr, long size, int prot)
559 {
560 uintptr_t start, end;
561
562 start = (uintptr_t)addr;
563 start &= qemu_real_host_page_mask;
564
565 end = (uintptr_t)addr + size;
566 end = ROUND_UP(end, qemu_real_host_page_size);
567
568 mprotect((void *)start, end - start, prot);
569 }
570
571 static inline void map_exec(void *addr, long size)
572 {
573 do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC);
574 }
575
576 static inline void map_none(void *addr, long size)
577 {
578 do_protect(addr, size, PROT_NONE);
579 }
580 # endif /* WIN32 */
581
582 static inline void *alloc_code_gen_buffer(void)
583 {
584 void *buf = static_code_gen_buffer;
585 size_t full_size, size;
586
587 /* The size of the buffer, rounded down to end on a page boundary. */
588 full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer))
589 & qemu_real_host_page_mask) - (uintptr_t)buf;
590
591 /* Reserve a guard page. */
592 size = full_size - qemu_real_host_page_size;
593
594 /* Honor a command-line option limiting the size of the buffer. */
595 if (size > tcg_ctx.code_gen_buffer_size) {
596 size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size)
597 & qemu_real_host_page_mask) - (uintptr_t)buf;
598 }
599 tcg_ctx.code_gen_buffer_size = size;
600
601 #ifdef __mips__
602 if (cross_256mb(buf, size)) {
603 buf = split_cross_256mb(buf, size);
604 size = tcg_ctx.code_gen_buffer_size;
605 }
606 #endif
607
608 map_exec(buf, size);
609 map_none(buf + size, qemu_real_host_page_size);
610 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
611
612 return buf;
613 }
614 #elif defined(_WIN32)
615 static inline void *alloc_code_gen_buffer(void)
616 {
617 size_t size = tcg_ctx.code_gen_buffer_size;
618 void *buf1, *buf2;
619
620 /* Perform the allocation in two steps, so that the guard page
621 is reserved but uncommitted. */
622 buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size,
623 MEM_RESERVE, PAGE_NOACCESS);
624 if (buf1 != NULL) {
625 buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
626 assert(buf1 == buf2);
627 }
628
629 return buf1;
630 }
631 #else
632 static inline void *alloc_code_gen_buffer(void)
633 {
634 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
635 uintptr_t start = 0;
636 size_t size = tcg_ctx.code_gen_buffer_size;
637 void *buf;
638
639 /* Constrain the position of the buffer based on the host cpu.
640 Note that these addresses are chosen in concert with the
641 addresses assigned in the relevant linker script file. */
642 # if defined(__PIE__) || defined(__PIC__)
643 /* Don't bother setting a preferred location if we're building
644 a position-independent executable. We're more likely to get
645 an address near the main executable if we let the kernel
646 choose the address. */
647 # elif defined(__x86_64__) && defined(MAP_32BIT)
648 /* Force the memory down into low memory with the executable.
649 Leave the choice of exact location with the kernel. */
650 flags |= MAP_32BIT;
651 /* Cannot expect to map more than 800MB in low memory. */
652 if (size > 800u * 1024 * 1024) {
653 tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024;
654 }
655 # elif defined(__sparc__)
656 start = 0x40000000ul;
657 # elif defined(__s390x__)
658 start = 0x90000000ul;
659 # elif defined(__mips__)
660 # if _MIPS_SIM == _ABI64
661 start = 0x128000000ul;
662 # else
663 start = 0x08000000ul;
664 # endif
665 # endif
666
667 buf = mmap((void *)start, size + qemu_real_host_page_size,
668 PROT_NONE, flags, -1, 0);
669 if (buf == MAP_FAILED) {
670 return NULL;
671 }
672
673 #ifdef __mips__
674 if (cross_256mb(buf, size)) {
675 /* Try again, with the original still mapped, to avoid re-acquiring
676 that 256mb crossing. This time don't specify an address. */
677 size_t size2;
678 void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
679 PROT_NONE, flags, -1, 0);
680 switch (buf2 != MAP_FAILED) {
681 case 1:
682 if (!cross_256mb(buf2, size)) {
683 /* Success! Use the new buffer. */
684 munmap(buf, size + qemu_real_host_page_size);
685 break;
686 }
687 /* Failure. Work with what we had. */
688 munmap(buf2, size + qemu_real_host_page_size);
689 /* fallthru */
690 default:
691 /* Split the original buffer. Free the smaller half. */
692 buf2 = split_cross_256mb(buf, size);
693 size2 = tcg_ctx.code_gen_buffer_size;
694 if (buf == buf2) {
695 munmap(buf + size2 + qemu_real_host_page_size, size - size2);
696 } else {
697 munmap(buf, size - size2);
698 }
699 size = size2;
700 break;
701 }
702 buf = buf2;
703 }
704 #endif
705
706 /* Make the final buffer accessible. The guard page at the end
707 will remain inaccessible with PROT_NONE. */
708 mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC);
709
710 /* Request large pages for the buffer. */
711 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
712
713 return buf;
714 }
715 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
716
717 static inline void code_gen_alloc(size_t tb_size)
718 {
719 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
720 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
721 if (tcg_ctx.code_gen_buffer == NULL) {
722 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
723 exit(1);
724 }
725
726 /* Estimate a good size for the number of TBs we can support. We
727 still haven't deducted the prologue from the buffer size here,
728 but that's minimal and won't affect the estimate much. */
729 tcg_ctx.code_gen_max_blocks
730 = tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
731 tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock, tcg_ctx.code_gen_max_blocks);
732
733 qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
734 }
735
736 static void tb_htable_init(void)
737 {
738 unsigned int mode = QHT_MODE_AUTO_RESIZE;
739
740 qht_init(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode);
741 }
742
743 /* Must be called before using the QEMU cpus. 'tb_size' is the size
744 (in bytes) allocated to the translation buffer. Zero means default
745 size. */
746 void tcg_exec_init(unsigned long tb_size)
747 {
748 cpu_gen_init();
749 page_init();
750 tb_htable_init();
751 code_gen_alloc(tb_size);
752 #if defined(CONFIG_SOFTMMU)
753 /* There's no guest base to take into account, so go ahead and
754 initialize the prologue now. */
755 tcg_prologue_init(&tcg_ctx);
756 #endif
757 }
758
759 bool tcg_enabled(void)
760 {
761 return tcg_ctx.code_gen_buffer != NULL;
762 }
763
764 /* Allocate a new translation block. Flush the translation buffer if
765 too many translation blocks or too much generated code. */
766 static TranslationBlock *tb_alloc(target_ulong pc)
767 {
768 TranslationBlock *tb;
769
770 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks) {
771 return NULL;
772 }
773 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
774 tb->pc = pc;
775 tb->cflags = 0;
776 return tb;
777 }
778
779 void tb_free(TranslationBlock *tb)
780 {
781 /* In practice this is mostly used for single use temporary TB
782 Ignore the hard cases and just back up if this TB happens to
783 be the last one generated. */
784 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
785 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
786 tcg_ctx.code_gen_ptr = tb->tc_ptr;
787 tcg_ctx.tb_ctx.nb_tbs--;
788 }
789 }
790
791 static inline void invalidate_page_bitmap(PageDesc *p)
792 {
793 #ifdef CONFIG_SOFTMMU
794 g_free(p->code_bitmap);
795 p->code_bitmap = NULL;
796 p->code_write_count = 0;
797 #endif
798 }
799
800 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
801 static void page_flush_tb_1(int level, void **lp)
802 {
803 int i;
804
805 if (*lp == NULL) {
806 return;
807 }
808 if (level == 0) {
809 PageDesc *pd = *lp;
810
811 for (i = 0; i < V_L2_SIZE; ++i) {
812 pd[i].first_tb = NULL;
813 invalidate_page_bitmap(pd + i);
814 }
815 } else {
816 void **pp = *lp;
817
818 for (i = 0; i < V_L2_SIZE; ++i) {
819 page_flush_tb_1(level - 1, pp + i);
820 }
821 }
822 }
823
824 static void page_flush_tb(void)
825 {
826 int i;
827
828 for (i = 0; i < V_L1_SIZE; i++) {
829 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
830 }
831 }
832
833 /* flush all the translation blocks */
834 /* XXX: tb_flush is currently not thread safe */
835 void tb_flush(CPUState *cpu)
836 {
837 #if defined(DEBUG_FLUSH)
838 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
839 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
840 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
841 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
842 tcg_ctx.tb_ctx.nb_tbs : 0);
843 #endif
844 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
845 > tcg_ctx.code_gen_buffer_size) {
846 cpu_abort(cpu, "Internal error: code buffer overflow\n");
847 }
848 tcg_ctx.tb_ctx.nb_tbs = 0;
849
850 CPU_FOREACH(cpu) {
851 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
852 cpu->tb_flushed = true;
853 }
854
855 qht_reset_size(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
856 page_flush_tb();
857
858 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
859 /* XXX: flush processor icache at this point if cache flush is
860 expensive */
861 tcg_ctx.tb_ctx.tb_flush_count++;
862 }
863
864 #ifdef DEBUG_TB_CHECK
865
866 static void
867 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp)
868 {
869 TranslationBlock *tb = p;
870 target_ulong addr = *(target_ulong *)userp;
871
872 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) {
873 printf("ERROR invalidate: address=" TARGET_FMT_lx
874 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size);
875 }
876 }
877
878 static void tb_invalidate_check(target_ulong address)
879 {
880 address &= TARGET_PAGE_MASK;
881 qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_invalidate_check, &address);
882 }
883
884 static void
885 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp)
886 {
887 TranslationBlock *tb = p;
888 int flags1, flags2;
889
890 flags1 = page_get_flags(tb->pc);
891 flags2 = page_get_flags(tb->pc + tb->size - 1);
892 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
893 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
894 (long)tb->pc, tb->size, flags1, flags2);
895 }
896 }
897
898 /* verify that all the pages have correct rights for code */
899 static void tb_page_check(void)
900 {
901 qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_page_check, NULL);
902 }
903
904 #endif
905
906 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
907 {
908 TranslationBlock *tb1;
909 unsigned int n1;
910
911 for (;;) {
912 tb1 = *ptb;
913 n1 = (uintptr_t)tb1 & 3;
914 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
915 if (tb1 == tb) {
916 *ptb = tb1->page_next[n1];
917 break;
918 }
919 ptb = &tb1->page_next[n1];
920 }
921 }
922
923 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
924 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n)
925 {
926 TranslationBlock *tb1;
927 uintptr_t *ptb, ntb;
928 unsigned int n1;
929
930 ptb = &tb->jmp_list_next[n];
931 if (*ptb) {
932 /* find tb(n) in circular list */
933 for (;;) {
934 ntb = *ptb;
935 n1 = ntb & 3;
936 tb1 = (TranslationBlock *)(ntb & ~3);
937 if (n1 == n && tb1 == tb) {
938 break;
939 }
940 if (n1 == 2) {
941 ptb = &tb1->jmp_list_first;
942 } else {
943 ptb = &tb1->jmp_list_next[n1];
944 }
945 }
946 /* now we can suppress tb(n) from the list */
947 *ptb = tb->jmp_list_next[n];
948
949 tb->jmp_list_next[n] = (uintptr_t)NULL;
950 }
951 }
952
953 /* reset the jump entry 'n' of a TB so that it is not chained to
954 another TB */
955 static inline void tb_reset_jump(TranslationBlock *tb, int n)
956 {
957 uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]);
958 tb_set_jmp_target(tb, n, addr);
959 }
960
961 /* remove any jumps to the TB */
962 static inline void tb_jmp_unlink(TranslationBlock *tb)
963 {
964 TranslationBlock *tb1;
965 uintptr_t *ptb, ntb;
966 unsigned int n1;
967
968 ptb = &tb->jmp_list_first;
969 for (;;) {
970 ntb = *ptb;
971 n1 = ntb & 3;
972 tb1 = (TranslationBlock *)(ntb & ~3);
973 if (n1 == 2) {
974 break;
975 }
976 tb_reset_jump(tb1, n1);
977 *ptb = tb1->jmp_list_next[n1];
978 tb1->jmp_list_next[n1] = (uintptr_t)NULL;
979 }
980 }
981
982 /* invalidate one TB */
983 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
984 {
985 CPUState *cpu;
986 PageDesc *p;
987 uint32_t h;
988 tb_page_addr_t phys_pc;
989
990 /* remove the TB from the hash list */
991 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
992 h = tb_hash_func(phys_pc, tb->pc, tb->flags);
993 qht_remove(&tcg_ctx.tb_ctx.htable, tb, h);
994
995 /* remove the TB from the page list */
996 if (tb->page_addr[0] != page_addr) {
997 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
998 tb_page_remove(&p->first_tb, tb);
999 invalidate_page_bitmap(p);
1000 }
1001 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
1002 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
1003 tb_page_remove(&p->first_tb, tb);
1004 invalidate_page_bitmap(p);
1005 }
1006
1007 /* remove the TB from the hash list */
1008 h = tb_jmp_cache_hash_func(tb->pc);
1009 CPU_FOREACH(cpu) {
1010 if (cpu->tb_jmp_cache[h] == tb) {
1011 cpu->tb_jmp_cache[h] = NULL;
1012 }
1013 }
1014
1015 /* suppress this TB from the two jump lists */
1016 tb_remove_from_jmp_list(tb, 0);
1017 tb_remove_from_jmp_list(tb, 1);
1018
1019 /* suppress any remaining jumps to this TB */
1020 tb_jmp_unlink(tb);
1021
1022 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
1023 }
1024
1025 #ifdef CONFIG_SOFTMMU
1026 static void build_page_bitmap(PageDesc *p)
1027 {
1028 int n, tb_start, tb_end;
1029 TranslationBlock *tb;
1030
1031 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
1032
1033 tb = p->first_tb;
1034 while (tb != NULL) {
1035 n = (uintptr_t)tb & 3;
1036 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1037 /* NOTE: this is subtle as a TB may span two physical pages */
1038 if (n == 0) {
1039 /* NOTE: tb_end may be after the end of the page, but
1040 it is not a problem */
1041 tb_start = tb->pc & ~TARGET_PAGE_MASK;
1042 tb_end = tb_start + tb->size;
1043 if (tb_end > TARGET_PAGE_SIZE) {
1044 tb_end = TARGET_PAGE_SIZE;
1045 }
1046 } else {
1047 tb_start = 0;
1048 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1049 }
1050 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
1051 tb = tb->page_next[n];
1052 }
1053 }
1054 #endif
1055
1056 /* add the tb in the target page and protect it if necessary
1057 *
1058 * Called with mmap_lock held for user-mode emulation.
1059 */
1060 static inline void tb_alloc_page(TranslationBlock *tb,
1061 unsigned int n, tb_page_addr_t page_addr)
1062 {
1063 PageDesc *p;
1064 #ifndef CONFIG_USER_ONLY
1065 bool page_already_protected;
1066 #endif
1067
1068 tb->page_addr[n] = page_addr;
1069 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1070 tb->page_next[n] = p->first_tb;
1071 #ifndef CONFIG_USER_ONLY
1072 page_already_protected = p->first_tb != NULL;
1073 #endif
1074 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1075 invalidate_page_bitmap(p);
1076
1077 #if defined(CONFIG_USER_ONLY)
1078 if (p->flags & PAGE_WRITE) {
1079 target_ulong addr;
1080 PageDesc *p2;
1081 int prot;
1082
1083 /* force the host page as non writable (writes will have a
1084 page fault + mprotect overhead) */
1085 page_addr &= qemu_host_page_mask;
1086 prot = 0;
1087 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1088 addr += TARGET_PAGE_SIZE) {
1089
1090 p2 = page_find(addr >> TARGET_PAGE_BITS);
1091 if (!p2) {
1092 continue;
1093 }
1094 prot |= p2->flags;
1095 p2->flags &= ~PAGE_WRITE;
1096 }
1097 mprotect(g2h(page_addr), qemu_host_page_size,
1098 (prot & PAGE_BITS) & ~PAGE_WRITE);
1099 #ifdef DEBUG_TB_INVALIDATE
1100 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1101 page_addr);
1102 #endif
1103 }
1104 #else
1105 /* if some code is already present, then the pages are already
1106 protected. So we handle the case where only the first TB is
1107 allocated in a physical page */
1108 if (!page_already_protected) {
1109 tlb_protect_code(page_addr);
1110 }
1111 #endif
1112 }
1113
1114 /* add a new TB and link it to the physical page tables. phys_page2 is
1115 * (-1) to indicate that only one page contains the TB.
1116 *
1117 * Called with mmap_lock held for user-mode emulation.
1118 */
1119 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1120 tb_page_addr_t phys_page2)
1121 {
1122 uint32_t h;
1123
1124 /* add in the hash table */
1125 h = tb_hash_func(phys_pc, tb->pc, tb->flags);
1126 qht_insert(&tcg_ctx.tb_ctx.htable, tb, h);
1127
1128 /* add in the page list */
1129 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1130 if (phys_page2 != -1) {
1131 tb_alloc_page(tb, 1, phys_page2);
1132 } else {
1133 tb->page_addr[1] = -1;
1134 }
1135
1136 #ifdef DEBUG_TB_CHECK
1137 tb_page_check();
1138 #endif
1139 }
1140
1141 /* Called with mmap_lock held for user mode emulation. */
1142 TranslationBlock *tb_gen_code(CPUState *cpu,
1143 target_ulong pc, target_ulong cs_base,
1144 uint32_t flags, int cflags)
1145 {
1146 CPUArchState *env = cpu->env_ptr;
1147 TranslationBlock *tb;
1148 tb_page_addr_t phys_pc, phys_page2;
1149 target_ulong virt_page2;
1150 tcg_insn_unit *gen_code_buf;
1151 int gen_code_size, search_size;
1152 #ifdef CONFIG_PROFILER
1153 int64_t ti;
1154 #endif
1155
1156 phys_pc = get_page_addr_code(env, pc);
1157 if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {
1158 cflags |= CF_USE_ICOUNT;
1159 }
1160
1161 tb = tb_alloc(pc);
1162 if (unlikely(!tb)) {
1163 buffer_overflow:
1164 /* flush must be done */
1165 tb_flush(cpu);
1166 /* cannot fail at this point */
1167 tb = tb_alloc(pc);
1168 assert(tb != NULL);
1169 }
1170
1171 gen_code_buf = tcg_ctx.code_gen_ptr;
1172 tb->tc_ptr = gen_code_buf;
1173 tb->cs_base = cs_base;
1174 tb->flags = flags;
1175 tb->cflags = cflags;
1176
1177 #ifdef CONFIG_PROFILER
1178 tcg_ctx.tb_count1++; /* includes aborted translations because of
1179 exceptions */
1180 ti = profile_getclock();
1181 #endif
1182
1183 tcg_func_start(&tcg_ctx);
1184
1185 gen_intermediate_code(env, tb);
1186
1187 trace_translate_block(tb, tb->pc, tb->tc_ptr);
1188
1189 /* generate machine code */
1190 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
1191 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
1192 tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset;
1193 #ifdef USE_DIRECT_JUMP
1194 tcg_ctx.tb_jmp_insn_offset = tb->jmp_insn_offset;
1195 tcg_ctx.tb_jmp_target_addr = NULL;
1196 #else
1197 tcg_ctx.tb_jmp_insn_offset = NULL;
1198 tcg_ctx.tb_jmp_target_addr = tb->jmp_target_addr;
1199 #endif
1200
1201 #ifdef CONFIG_PROFILER
1202 tcg_ctx.tb_count++;
1203 tcg_ctx.interm_time += profile_getclock() - ti;
1204 tcg_ctx.code_time -= profile_getclock();
1205 #endif
1206
1207 /* ??? Overflow could be handled better here. In particular, we
1208 don't need to re-do gen_intermediate_code, nor should we re-do
1209 the tcg optimization currently hidden inside tcg_gen_code. All
1210 that should be required is to flush the TBs, allocate a new TB,
1211 re-initialize it per above, and re-do the actual code generation. */
1212 gen_code_size = tcg_gen_code(&tcg_ctx, tb);
1213 if (unlikely(gen_code_size < 0)) {
1214 goto buffer_overflow;
1215 }
1216 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
1217 if (unlikely(search_size < 0)) {
1218 goto buffer_overflow;
1219 }
1220
1221 #ifdef CONFIG_PROFILER
1222 tcg_ctx.code_time += profile_getclock();
1223 tcg_ctx.code_in_len += tb->size;
1224 tcg_ctx.code_out_len += gen_code_size;
1225 tcg_ctx.search_out_len += search_size;
1226 #endif
1227
1228 #ifdef DEBUG_DISAS
1229 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
1230 qemu_log_in_addr_range(tb->pc)) {
1231 qemu_log("OUT: [size=%d]\n", gen_code_size);
1232 log_disas(tb->tc_ptr, gen_code_size);
1233 qemu_log("\n");
1234 qemu_log_flush();
1235 }
1236 #endif
1237
1238 tcg_ctx.code_gen_ptr = (void *)
1239 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
1240 CODE_GEN_ALIGN);
1241
1242 /* init jump list */
1243 assert(((uintptr_t)tb & 3) == 0);
1244 tb->jmp_list_first = (uintptr_t)tb | 2;
1245 tb->jmp_list_next[0] = (uintptr_t)NULL;
1246 tb->jmp_list_next[1] = (uintptr_t)NULL;
1247
1248 /* init original jump addresses wich has been set during tcg_gen_code() */
1249 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1250 tb_reset_jump(tb, 0);
1251 }
1252 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1253 tb_reset_jump(tb, 1);
1254 }
1255
1256 /* check next page if needed */
1257 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1258 phys_page2 = -1;
1259 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1260 phys_page2 = get_page_addr_code(env, virt_page2);
1261 }
1262 /* As long as consistency of the TB stuff is provided by tb_lock in user
1263 * mode and is implicit in single-threaded softmmu emulation, no explicit
1264 * memory barrier is required before tb_link_page() makes the TB visible
1265 * through the physical hash table and physical page list.
1266 */
1267 tb_link_page(tb, phys_pc, phys_page2);
1268 return tb;
1269 }
1270
1271 /*
1272 * Invalidate all TBs which intersect with the target physical address range
1273 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1274 * 'is_cpu_write_access' should be true if called from a real cpu write
1275 * access: the virtual CPU will exit the current TB if code is modified inside
1276 * this TB.
1277 *
1278 * Called with mmap_lock held for user-mode emulation
1279 */
1280 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
1281 {
1282 while (start < end) {
1283 tb_invalidate_phys_page_range(start, end, 0);
1284 start &= TARGET_PAGE_MASK;
1285 start += TARGET_PAGE_SIZE;
1286 }
1287 }
1288
1289 /*
1290 * Invalidate all TBs which intersect with the target physical address range
1291 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1292 * 'is_cpu_write_access' should be true if called from a real cpu write
1293 * access: the virtual CPU will exit the current TB if code is modified inside
1294 * this TB.
1295 *
1296 * Called with mmap_lock held for user-mode emulation
1297 */
1298 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1299 int is_cpu_write_access)
1300 {
1301 TranslationBlock *tb, *tb_next;
1302 #if defined(TARGET_HAS_PRECISE_SMC)
1303 CPUState *cpu = current_cpu;
1304 CPUArchState *env = NULL;
1305 #endif
1306 tb_page_addr_t tb_start, tb_end;
1307 PageDesc *p;
1308 int n;
1309 #ifdef TARGET_HAS_PRECISE_SMC
1310 int current_tb_not_found = is_cpu_write_access;
1311 TranslationBlock *current_tb = NULL;
1312 int current_tb_modified = 0;
1313 target_ulong current_pc = 0;
1314 target_ulong current_cs_base = 0;
1315 uint32_t current_flags = 0;
1316 #endif /* TARGET_HAS_PRECISE_SMC */
1317
1318 p = page_find(start >> TARGET_PAGE_BITS);
1319 if (!p) {
1320 return;
1321 }
1322 #if defined(TARGET_HAS_PRECISE_SMC)
1323 if (cpu != NULL) {
1324 env = cpu->env_ptr;
1325 }
1326 #endif
1327
1328 /* we remove all the TBs in the range [start, end[ */
1329 /* XXX: see if in some cases it could be faster to invalidate all
1330 the code */
1331 tb = p->first_tb;
1332 while (tb != NULL) {
1333 n = (uintptr_t)tb & 3;
1334 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1335 tb_next = tb->page_next[n];
1336 /* NOTE: this is subtle as a TB may span two physical pages */
1337 if (n == 0) {
1338 /* NOTE: tb_end may be after the end of the page, but
1339 it is not a problem */
1340 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1341 tb_end = tb_start + tb->size;
1342 } else {
1343 tb_start = tb->page_addr[1];
1344 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1345 }
1346 if (!(tb_end <= start || tb_start >= end)) {
1347 #ifdef TARGET_HAS_PRECISE_SMC
1348 if (current_tb_not_found) {
1349 current_tb_not_found = 0;
1350 current_tb = NULL;
1351 if (cpu->mem_io_pc) {
1352 /* now we have a real cpu fault */
1353 current_tb = tb_find_pc(cpu->mem_io_pc);
1354 }
1355 }
1356 if (current_tb == tb &&
1357 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1358 /* If we are modifying the current TB, we must stop
1359 its execution. We could be more precise by checking
1360 that the modification is after the current PC, but it
1361 would require a specialized function to partially
1362 restore the CPU state */
1363
1364 current_tb_modified = 1;
1365 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
1366 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1367 &current_flags);
1368 }
1369 #endif /* TARGET_HAS_PRECISE_SMC */
1370 tb_phys_invalidate(tb, -1);
1371 }
1372 tb = tb_next;
1373 }
1374 #if !defined(CONFIG_USER_ONLY)
1375 /* if no code remaining, no need to continue to use slow writes */
1376 if (!p->first_tb) {
1377 invalidate_page_bitmap(p);
1378 tlb_unprotect_code(start);
1379 }
1380 #endif
1381 #ifdef TARGET_HAS_PRECISE_SMC
1382 if (current_tb_modified) {
1383 /* we generate a block containing just the instruction
1384 modifying the memory. It will ensure that it cannot modify
1385 itself */
1386 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1387 cpu_loop_exit_noexc(cpu);
1388 }
1389 #endif
1390 }
1391
1392 #ifdef CONFIG_SOFTMMU
1393 /* len must be <= 8 and start must be a multiple of len */
1394 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1395 {
1396 PageDesc *p;
1397
1398 #if 0
1399 if (1) {
1400 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1401 cpu_single_env->mem_io_vaddr, len,
1402 cpu_single_env->eip,
1403 cpu_single_env->eip +
1404 (intptr_t)cpu_single_env->segs[R_CS].base);
1405 }
1406 #endif
1407 p = page_find(start >> TARGET_PAGE_BITS);
1408 if (!p) {
1409 return;
1410 }
1411 if (!p->code_bitmap &&
1412 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
1413 /* build code bitmap */
1414 build_page_bitmap(p);
1415 }
1416 if (p->code_bitmap) {
1417 unsigned int nr;
1418 unsigned long b;
1419
1420 nr = start & ~TARGET_PAGE_MASK;
1421 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
1422 if (b & ((1 << len) - 1)) {
1423 goto do_invalidate;
1424 }
1425 } else {
1426 do_invalidate:
1427 tb_invalidate_phys_page_range(start, start + len, 1);
1428 }
1429 }
1430 #else
1431 /* Called with mmap_lock held. If pc is not 0 then it indicates the
1432 * host PC of the faulting store instruction that caused this invalidate.
1433 * Returns true if the caller needs to abort execution of the current
1434 * TB (because it was modified by this store and the guest CPU has
1435 * precise-SMC semantics).
1436 */
1437 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
1438 {
1439 TranslationBlock *tb;
1440 PageDesc *p;
1441 int n;
1442 #ifdef TARGET_HAS_PRECISE_SMC
1443 TranslationBlock *current_tb = NULL;
1444 CPUState *cpu = current_cpu;
1445 CPUArchState *env = NULL;
1446 int current_tb_modified = 0;
1447 target_ulong current_pc = 0;
1448 target_ulong current_cs_base = 0;
1449 uint32_t current_flags = 0;
1450 #endif
1451
1452 addr &= TARGET_PAGE_MASK;
1453 p = page_find(addr >> TARGET_PAGE_BITS);
1454 if (!p) {
1455 return false;
1456 }
1457 tb = p->first_tb;
1458 #ifdef TARGET_HAS_PRECISE_SMC
1459 if (tb && pc != 0) {
1460 current_tb = tb_find_pc(pc);
1461 }
1462 if (cpu != NULL) {
1463 env = cpu->env_ptr;
1464 }
1465 #endif
1466 while (tb != NULL) {
1467 n = (uintptr_t)tb & 3;
1468 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1469 #ifdef TARGET_HAS_PRECISE_SMC
1470 if (current_tb == tb &&
1471 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1472 /* If we are modifying the current TB, we must stop
1473 its execution. We could be more precise by checking
1474 that the modification is after the current PC, but it
1475 would require a specialized function to partially
1476 restore the CPU state */
1477
1478 current_tb_modified = 1;
1479 cpu_restore_state_from_tb(cpu, current_tb, pc);
1480 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1481 &current_flags);
1482 }
1483 #endif /* TARGET_HAS_PRECISE_SMC */
1484 tb_phys_invalidate(tb, addr);
1485 tb = tb->page_next[n];
1486 }
1487 p->first_tb = NULL;
1488 #ifdef TARGET_HAS_PRECISE_SMC
1489 if (current_tb_modified) {
1490 /* we generate a block containing just the instruction
1491 modifying the memory. It will ensure that it cannot modify
1492 itself */
1493 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1494 return true;
1495 }
1496 #endif
1497 return false;
1498 }
1499 #endif
1500
1501 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1502 tb[1].tc_ptr. Return NULL if not found */
1503 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1504 {
1505 int m_min, m_max, m;
1506 uintptr_t v;
1507 TranslationBlock *tb;
1508
1509 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1510 return NULL;
1511 }
1512 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1513 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1514 return NULL;
1515 }
1516 /* binary search (cf Knuth) */
1517 m_min = 0;
1518 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1519 while (m_min <= m_max) {
1520 m = (m_min + m_max) >> 1;
1521 tb = &tcg_ctx.tb_ctx.tbs[m];
1522 v = (uintptr_t)tb->tc_ptr;
1523 if (v == tc_ptr) {
1524 return tb;
1525 } else if (tc_ptr < v) {
1526 m_max = m - 1;
1527 } else {
1528 m_min = m + 1;
1529 }
1530 }
1531 return &tcg_ctx.tb_ctx.tbs[m_max];
1532 }
1533
1534 #if !defined(CONFIG_USER_ONLY)
1535 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1536 {
1537 ram_addr_t ram_addr;
1538 MemoryRegion *mr;
1539 hwaddr l = 1;
1540
1541 rcu_read_lock();
1542 mr = address_space_translate(as, addr, &addr, &l, false);
1543 if (!(memory_region_is_ram(mr)
1544 || memory_region_is_romd(mr))) {
1545 rcu_read_unlock();
1546 return;
1547 }
1548 ram_addr = memory_region_get_ram_addr(mr) + addr;
1549 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1550 rcu_read_unlock();
1551 }
1552 #endif /* !defined(CONFIG_USER_ONLY) */
1553
1554 void tb_check_watchpoint(CPUState *cpu)
1555 {
1556 TranslationBlock *tb;
1557
1558 tb = tb_find_pc(cpu->mem_io_pc);
1559 if (tb) {
1560 /* We can use retranslation to find the PC. */
1561 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
1562 tb_phys_invalidate(tb, -1);
1563 } else {
1564 /* The exception probably happened in a helper. The CPU state should
1565 have been saved before calling it. Fetch the PC from there. */
1566 CPUArchState *env = cpu->env_ptr;
1567 target_ulong pc, cs_base;
1568 tb_page_addr_t addr;
1569 uint32_t flags;
1570
1571 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
1572 addr = get_page_addr_code(env, pc);
1573 tb_invalidate_phys_range(addr, addr + 1);
1574 }
1575 }
1576
1577 #ifndef CONFIG_USER_ONLY
1578 /* in deterministic execution mode, instructions doing device I/Os
1579 must be at the end of the TB */
1580 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
1581 {
1582 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1583 CPUArchState *env = cpu->env_ptr;
1584 #endif
1585 TranslationBlock *tb;
1586 uint32_t n, cflags;
1587 target_ulong pc, cs_base;
1588 uint32_t flags;
1589
1590 tb = tb_find_pc(retaddr);
1591 if (!tb) {
1592 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
1593 (void *)retaddr);
1594 }
1595 n = cpu->icount_decr.u16.low + tb->icount;
1596 cpu_restore_state_from_tb(cpu, tb, retaddr);
1597 /* Calculate how many instructions had been executed before the fault
1598 occurred. */
1599 n = n - cpu->icount_decr.u16.low;
1600 /* Generate a new TB ending on the I/O insn. */
1601 n++;
1602 /* On MIPS and SH, delay slot instructions can only be restarted if
1603 they were already the first instruction in the TB. If this is not
1604 the first instruction in a TB then re-execute the preceding
1605 branch. */
1606 #if defined(TARGET_MIPS)
1607 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1608 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
1609 cpu->icount_decr.u16.low++;
1610 env->hflags &= ~MIPS_HFLAG_BMASK;
1611 }
1612 #elif defined(TARGET_SH4)
1613 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1614 && n > 1) {
1615 env->pc -= 2;
1616 cpu->icount_decr.u16.low++;
1617 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1618 }
1619 #endif
1620 /* This should never happen. */
1621 if (n > CF_COUNT_MASK) {
1622 cpu_abort(cpu, "TB too big during recompile");
1623 }
1624
1625 cflags = n | CF_LAST_IO;
1626 pc = tb->pc;
1627 cs_base = tb->cs_base;
1628 flags = tb->flags;
1629 tb_phys_invalidate(tb, -1);
1630 if (tb->cflags & CF_NOCACHE) {
1631 if (tb->orig_tb) {
1632 /* Invalidate original TB if this TB was generated in
1633 * cpu_exec_nocache() */
1634 tb_phys_invalidate(tb->orig_tb, -1);
1635 }
1636 tb_free(tb);
1637 }
1638 /* FIXME: In theory this could raise an exception. In practice
1639 we have already translated the block once so it's probably ok. */
1640 tb_gen_code(cpu, pc, cs_base, flags, cflags);
1641 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1642 the first in the TB) then we end up generating a whole new TB and
1643 repeating the fault, which is horribly inefficient.
1644 Better would be to execute just this insn uncached, or generate a
1645 second new TB. */
1646 cpu_loop_exit_noexc(cpu);
1647 }
1648
1649 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
1650 {
1651 unsigned int i;
1652
1653 /* Discard jump cache entries for any tb which might potentially
1654 overlap the flushed page. */
1655 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1656 memset(&cpu->tb_jmp_cache[i], 0,
1657 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1658
1659 i = tb_jmp_cache_hash_page(addr);
1660 memset(&cpu->tb_jmp_cache[i], 0,
1661 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1662 }
1663
1664 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1665 {
1666 int i, target_code_size, max_target_code_size;
1667 int direct_jmp_count, direct_jmp2_count, cross_page;
1668 TranslationBlock *tb;
1669 struct qht_stats hst;
1670 uint32_t hgram_opts;
1671 size_t hgram_bins;
1672 char *hgram;
1673
1674 target_code_size = 0;
1675 max_target_code_size = 0;
1676 cross_page = 0;
1677 direct_jmp_count = 0;
1678 direct_jmp2_count = 0;
1679 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1680 tb = &tcg_ctx.tb_ctx.tbs[i];
1681 target_code_size += tb->size;
1682 if (tb->size > max_target_code_size) {
1683 max_target_code_size = tb->size;
1684 }
1685 if (tb->page_addr[1] != -1) {
1686 cross_page++;
1687 }
1688 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1689 direct_jmp_count++;
1690 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1691 direct_jmp2_count++;
1692 }
1693 }
1694 }
1695 /* XXX: avoid using doubles ? */
1696 cpu_fprintf(f, "Translation buffer state:\n");
1697 cpu_fprintf(f, "gen code size %td/%zd\n",
1698 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1699 tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer);
1700 cpu_fprintf(f, "TB count %d/%d\n",
1701 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1702 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1703 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1704 tcg_ctx.tb_ctx.nb_tbs : 0,
1705 max_target_code_size);
1706 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1707 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1708 tcg_ctx.code_gen_buffer) /
1709 tcg_ctx.tb_ctx.nb_tbs : 0,
1710 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1711 tcg_ctx.code_gen_buffer) /
1712 target_code_size : 0);
1713 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1714 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1715 tcg_ctx.tb_ctx.nb_tbs : 0);
1716 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1717 direct_jmp_count,
1718 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1719 tcg_ctx.tb_ctx.nb_tbs : 0,
1720 direct_jmp2_count,
1721 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1722 tcg_ctx.tb_ctx.nb_tbs : 0);
1723
1724 qht_statistics_init(&tcg_ctx.tb_ctx.htable, &hst);
1725
1726 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
1727 hst.used_head_buckets, hst.head_buckets,
1728 (double)hst.used_head_buckets / hst.head_buckets * 100);
1729
1730 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
1731 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT;
1732 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) {
1733 hgram_opts |= QDIST_PR_NODECIMAL;
1734 }
1735 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts);
1736 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
1737 qdist_avg(&hst.occupancy) * 100, hgram);
1738 g_free(hgram);
1739
1740 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
1741 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain);
1742 if (hgram_bins > 10) {
1743 hgram_bins = 10;
1744 } else {
1745 hgram_bins = 0;
1746 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE;
1747 }
1748 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts);
1749 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
1750 qdist_avg(&hst.chain), hgram);
1751 g_free(hgram);
1752
1753 qht_statistics_destroy(&hst);
1754
1755 cpu_fprintf(f, "\nStatistics:\n");
1756 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1757 cpu_fprintf(f, "TB invalidate count %d\n",
1758 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1759 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1760 tcg_dump_info(f, cpu_fprintf);
1761 }
1762
1763 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
1764 {
1765 tcg_dump_op_count(f, cpu_fprintf);
1766 }
1767
1768 #else /* CONFIG_USER_ONLY */
1769
1770 void cpu_interrupt(CPUState *cpu, int mask)
1771 {
1772 cpu->interrupt_request |= mask;
1773 cpu->tcg_exit_req = 1;
1774 }
1775
1776 /*
1777 * Walks guest process memory "regions" one by one
1778 * and calls callback function 'fn' for each region.
1779 */
1780 struct walk_memory_regions_data {
1781 walk_memory_regions_fn fn;
1782 void *priv;
1783 target_ulong start;
1784 int prot;
1785 };
1786
1787 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1788 target_ulong end, int new_prot)
1789 {
1790 if (data->start != -1u) {
1791 int rc = data->fn(data->priv, data->start, end, data->prot);
1792 if (rc != 0) {
1793 return rc;
1794 }
1795 }
1796
1797 data->start = (new_prot ? end : -1u);
1798 data->prot = new_prot;
1799
1800 return 0;
1801 }
1802
1803 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1804 target_ulong base, int level, void **lp)
1805 {
1806 target_ulong pa;
1807 int i, rc;
1808
1809 if (*lp == NULL) {
1810 return walk_memory_regions_end(data, base, 0);
1811 }
1812
1813 if (level == 0) {
1814 PageDesc *pd = *lp;
1815
1816 for (i = 0; i < V_L2_SIZE; ++i) {
1817 int prot = pd[i].flags;
1818
1819 pa = base | (i << TARGET_PAGE_BITS);
1820 if (prot != data->prot) {
1821 rc = walk_memory_regions_end(data, pa, prot);
1822 if (rc != 0) {
1823 return rc;
1824 }
1825 }
1826 }
1827 } else {
1828 void **pp = *lp;
1829
1830 for (i = 0; i < V_L2_SIZE; ++i) {
1831 pa = base | ((target_ulong)i <<
1832 (TARGET_PAGE_BITS + V_L2_BITS * level));
1833 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1834 if (rc != 0) {
1835 return rc;
1836 }
1837 }
1838 }
1839
1840 return 0;
1841 }
1842
1843 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1844 {
1845 struct walk_memory_regions_data data;
1846 uintptr_t i;
1847
1848 data.fn = fn;
1849 data.priv = priv;
1850 data.start = -1u;
1851 data.prot = 0;
1852
1853 for (i = 0; i < V_L1_SIZE; i++) {
1854 int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
1855 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1856 if (rc != 0) {
1857 return rc;
1858 }
1859 }
1860
1861 return walk_memory_regions_end(&data, 0, 0);
1862 }
1863
1864 static int dump_region(void *priv, target_ulong start,
1865 target_ulong end, unsigned long prot)
1866 {
1867 FILE *f = (FILE *)priv;
1868
1869 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
1870 " "TARGET_FMT_lx" %c%c%c\n",
1871 start, end, end - start,
1872 ((prot & PAGE_READ) ? 'r' : '-'),
1873 ((prot & PAGE_WRITE) ? 'w' : '-'),
1874 ((prot & PAGE_EXEC) ? 'x' : '-'));
1875
1876 return 0;
1877 }
1878
1879 /* dump memory mappings */
1880 void page_dump(FILE *f)
1881 {
1882 const int length = sizeof(target_ulong) * 2;
1883 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1884 length, "start", length, "end", length, "size", "prot");
1885 walk_memory_regions(f, dump_region);
1886 }
1887
1888 int page_get_flags(target_ulong address)
1889 {
1890 PageDesc *p;
1891
1892 p = page_find(address >> TARGET_PAGE_BITS);
1893 if (!p) {
1894 return 0;
1895 }
1896 return p->flags;
1897 }
1898
1899 /* Modify the flags of a page and invalidate the code if necessary.
1900 The flag PAGE_WRITE_ORG is positioned automatically depending
1901 on PAGE_WRITE. The mmap_lock should already be held. */
1902 void page_set_flags(target_ulong start, target_ulong end, int flags)
1903 {
1904 target_ulong addr, len;
1905
1906 /* This function should never be called with addresses outside the
1907 guest address space. If this assert fires, it probably indicates
1908 a missing call to h2g_valid. */
1909 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1910 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1911 #endif
1912 assert(start < end);
1913
1914 start = start & TARGET_PAGE_MASK;
1915 end = TARGET_PAGE_ALIGN(end);
1916
1917 if (flags & PAGE_WRITE) {
1918 flags |= PAGE_WRITE_ORG;
1919 }
1920
1921 for (addr = start, len = end - start;
1922 len != 0;
1923 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1924 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1925
1926 /* If the write protection bit is set, then we invalidate
1927 the code inside. */
1928 if (!(p->flags & PAGE_WRITE) &&
1929 (flags & PAGE_WRITE) &&
1930 p->first_tb) {
1931 tb_invalidate_phys_page(addr, 0);
1932 }
1933 p->flags = flags;
1934 }
1935 }
1936
1937 int page_check_range(target_ulong start, target_ulong len, int flags)
1938 {
1939 PageDesc *p;
1940 target_ulong end;
1941 target_ulong addr;
1942
1943 /* This function should never be called with addresses outside the
1944 guest address space. If this assert fires, it probably indicates
1945 a missing call to h2g_valid. */
1946 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1947 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1948 #endif
1949
1950 if (len == 0) {
1951 return 0;
1952 }
1953 if (start + len - 1 < start) {
1954 /* We've wrapped around. */
1955 return -1;
1956 }
1957
1958 /* must do before we loose bits in the next step */
1959 end = TARGET_PAGE_ALIGN(start + len);
1960 start = start & TARGET_PAGE_MASK;
1961
1962 for (addr = start, len = end - start;
1963 len != 0;
1964 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1965 p = page_find(addr >> TARGET_PAGE_BITS);
1966 if (!p) {
1967 return -1;
1968 }
1969 if (!(p->flags & PAGE_VALID)) {
1970 return -1;
1971 }
1972
1973 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1974 return -1;
1975 }
1976 if (flags & PAGE_WRITE) {
1977 if (!(p->flags & PAGE_WRITE_ORG)) {
1978 return -1;
1979 }
1980 /* unprotect the page if it was put read-only because it
1981 contains translated code */
1982 if (!(p->flags & PAGE_WRITE)) {
1983 if (!page_unprotect(addr, 0)) {
1984 return -1;
1985 }
1986 }
1987 }
1988 }
1989 return 0;
1990 }
1991
1992 /* called from signal handler: invalidate the code and unprotect the
1993 * page. Return 0 if the fault was not handled, 1 if it was handled,
1994 * and 2 if it was handled but the caller must cause the TB to be
1995 * immediately exited. (We can only return 2 if the 'pc' argument is
1996 * non-zero.)
1997 */
1998 int page_unprotect(target_ulong address, uintptr_t pc)
1999 {
2000 unsigned int prot;
2001 PageDesc *p;
2002 target_ulong host_start, host_end, addr;
2003
2004 /* Technically this isn't safe inside a signal handler. However we
2005 know this only ever happens in a synchronous SEGV handler, so in
2006 practice it seems to be ok. */
2007 mmap_lock();
2008
2009 p = page_find(address >> TARGET_PAGE_BITS);
2010 if (!p) {
2011 mmap_unlock();
2012 return 0;
2013 }
2014
2015 /* if the page was really writable, then we change its
2016 protection back to writable */
2017 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2018 host_start = address & qemu_host_page_mask;
2019 host_end = host_start + qemu_host_page_size;
2020
2021 prot = 0;
2022 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2023 p = page_find(addr >> TARGET_PAGE_BITS);
2024 p->flags |= PAGE_WRITE;
2025 prot |= p->flags;
2026
2027 /* and since the content will be modified, we must invalidate
2028 the corresponding translated code. */
2029 if (tb_invalidate_phys_page(addr, pc)) {
2030 mmap_unlock();
2031 return 2;
2032 }
2033 #ifdef DEBUG_TB_CHECK
2034 tb_invalidate_check(addr);
2035 #endif
2036 }
2037 mprotect((void *)g2h(host_start), qemu_host_page_size,
2038 prot & PAGE_BITS);
2039
2040 mmap_unlock();
2041 return 1;
2042 }
2043 mmap_unlock();
2044 return 0;
2045 }
2046 #endif /* CONFIG_USER_ONLY */