trap signals for "-serial mon:stdio"
[qemu.git] / translate-all.c
1 /*
2 * Host code generation
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/types.h>
23 #include <sys/mman.h>
24 #endif
25 #include <stdarg.h>
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <inttypes.h>
30
31 #include "config.h"
32
33 #include "qemu-common.h"
34 #define NO_CPU_IO_DEFS
35 #include "cpu.h"
36 #include "disas/disas.h"
37 #include "tcg.h"
38 #if defined(CONFIG_USER_ONLY)
39 #include "qemu.h"
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
45 #include <sys/time.h>
46 #include <sys/proc.h>
47 #include <machine/profile.h>
48 #define _KERNEL
49 #include <sys/user.h>
50 #undef _KERNEL
51 #undef sigqueue
52 #include <libutil.h>
53 #endif
54 #endif
55 #else
56 #include "exec/address-spaces.h"
57 #endif
58
59 #include "exec/cputlb.h"
60 #include "translate-all.h"
61 #include "qemu/timer.h"
62
63 //#define DEBUG_TB_INVALIDATE
64 //#define DEBUG_FLUSH
65 /* make various TB consistency checks */
66 //#define DEBUG_TB_CHECK
67
68 #if !defined(CONFIG_USER_ONLY)
69 /* TB consistency checks only implemented for usermode emulation. */
70 #undef DEBUG_TB_CHECK
71 #endif
72
73 #define SMC_BITMAP_USE_THRESHOLD 10
74
75 typedef struct PageDesc {
76 /* list of TBs intersecting this ram page */
77 TranslationBlock *first_tb;
78 /* in order to optimize self modifying code, we count the number
79 of lookups we do to a given page to use a bitmap */
80 unsigned int code_write_count;
81 uint8_t *code_bitmap;
82 #if defined(CONFIG_USER_ONLY)
83 unsigned long flags;
84 #endif
85 } PageDesc;
86
87 /* In system mode we want L1_MAP to be based on ram offsets,
88 while in user mode we want it to be based on virtual addresses. */
89 #if !defined(CONFIG_USER_ONLY)
90 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
91 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
92 #else
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
94 #endif
95 #else
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
97 #endif
98
99 /* The bits remaining after N lower levels of page tables. */
100 #define V_L1_BITS_REM \
101 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
102
103 #if V_L1_BITS_REM < 4
104 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
105 #else
106 #define V_L1_BITS V_L1_BITS_REM
107 #endif
108
109 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
110
111 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
112
113 uintptr_t qemu_real_host_page_size;
114 uintptr_t qemu_host_page_size;
115 uintptr_t qemu_host_page_mask;
116
117 /* This is a multi-level map on the virtual address space.
118 The bottom level has pointers to PageDesc. */
119 static void *l1_map[V_L1_SIZE];
120
121 /* code generation context */
122 TCGContext tcg_ctx;
123
124 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
125 tb_page_addr_t phys_page2);
126 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
127
128 void cpu_gen_init(void)
129 {
130 tcg_context_init(&tcg_ctx);
131 }
132
133 /* return non zero if the very first instruction is invalid so that
134 the virtual CPU can trigger an exception.
135
136 '*gen_code_size_ptr' contains the size of the generated code (host
137 code).
138 */
139 int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
140 {
141 TCGContext *s = &tcg_ctx;
142 uint8_t *gen_code_buf;
143 int gen_code_size;
144 #ifdef CONFIG_PROFILER
145 int64_t ti;
146 #endif
147
148 #ifdef CONFIG_PROFILER
149 s->tb_count1++; /* includes aborted translations because of
150 exceptions */
151 ti = profile_getclock();
152 #endif
153 tcg_func_start(s);
154
155 gen_intermediate_code(env, tb);
156
157 /* generate machine code */
158 gen_code_buf = tb->tc_ptr;
159 tb->tb_next_offset[0] = 0xffff;
160 tb->tb_next_offset[1] = 0xffff;
161 s->tb_next_offset = tb->tb_next_offset;
162 #ifdef USE_DIRECT_JUMP
163 s->tb_jmp_offset = tb->tb_jmp_offset;
164 s->tb_next = NULL;
165 #else
166 s->tb_jmp_offset = NULL;
167 s->tb_next = tb->tb_next;
168 #endif
169
170 #ifdef CONFIG_PROFILER
171 s->tb_count++;
172 s->interm_time += profile_getclock() - ti;
173 s->code_time -= profile_getclock();
174 #endif
175 gen_code_size = tcg_gen_code(s, gen_code_buf);
176 *gen_code_size_ptr = gen_code_size;
177 #ifdef CONFIG_PROFILER
178 s->code_time += profile_getclock();
179 s->code_in_len += tb->size;
180 s->code_out_len += gen_code_size;
181 #endif
182
183 #ifdef DEBUG_DISAS
184 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
185 qemu_log("OUT: [size=%d]\n", *gen_code_size_ptr);
186 log_disas(tb->tc_ptr, *gen_code_size_ptr);
187 qemu_log("\n");
188 qemu_log_flush();
189 }
190 #endif
191 return 0;
192 }
193
194 /* The cpu state corresponding to 'searched_pc' is restored.
195 */
196 static int cpu_restore_state_from_tb(TranslationBlock *tb, CPUArchState *env,
197 uintptr_t searched_pc)
198 {
199 TCGContext *s = &tcg_ctx;
200 int j;
201 uintptr_t tc_ptr;
202 #ifdef CONFIG_PROFILER
203 int64_t ti;
204 #endif
205
206 #ifdef CONFIG_PROFILER
207 ti = profile_getclock();
208 #endif
209 tcg_func_start(s);
210
211 gen_intermediate_code_pc(env, tb);
212
213 if (use_icount) {
214 /* Reset the cycle counter to the start of the block. */
215 env->icount_decr.u16.low += tb->icount;
216 /* Clear the IO flag. */
217 env->can_do_io = 0;
218 }
219
220 /* find opc index corresponding to search_pc */
221 tc_ptr = (uintptr_t)tb->tc_ptr;
222 if (searched_pc < tc_ptr)
223 return -1;
224
225 s->tb_next_offset = tb->tb_next_offset;
226 #ifdef USE_DIRECT_JUMP
227 s->tb_jmp_offset = tb->tb_jmp_offset;
228 s->tb_next = NULL;
229 #else
230 s->tb_jmp_offset = NULL;
231 s->tb_next = tb->tb_next;
232 #endif
233 j = tcg_gen_code_search_pc(s, (uint8_t *)tc_ptr, searched_pc - tc_ptr);
234 if (j < 0)
235 return -1;
236 /* now find start of instruction before */
237 while (s->gen_opc_instr_start[j] == 0) {
238 j--;
239 }
240 env->icount_decr.u16.low -= s->gen_opc_icount[j];
241
242 restore_state_to_opc(env, tb, j);
243
244 #ifdef CONFIG_PROFILER
245 s->restore_time += profile_getclock() - ti;
246 s->restore_count++;
247 #endif
248 return 0;
249 }
250
251 bool cpu_restore_state(CPUArchState *env, uintptr_t retaddr)
252 {
253 TranslationBlock *tb;
254
255 tb = tb_find_pc(retaddr);
256 if (tb) {
257 cpu_restore_state_from_tb(tb, env, retaddr);
258 return true;
259 }
260 return false;
261 }
262
263 #ifdef _WIN32
264 static inline void map_exec(void *addr, long size)
265 {
266 DWORD old_protect;
267 VirtualProtect(addr, size,
268 PAGE_EXECUTE_READWRITE, &old_protect);
269 }
270 #else
271 static inline void map_exec(void *addr, long size)
272 {
273 unsigned long start, end, page_size;
274
275 page_size = getpagesize();
276 start = (unsigned long)addr;
277 start &= ~(page_size - 1);
278
279 end = (unsigned long)addr + size;
280 end += page_size - 1;
281 end &= ~(page_size - 1);
282
283 mprotect((void *)start, end - start,
284 PROT_READ | PROT_WRITE | PROT_EXEC);
285 }
286 #endif
287
288 static void page_init(void)
289 {
290 /* NOTE: we can always suppose that qemu_host_page_size >=
291 TARGET_PAGE_SIZE */
292 #ifdef _WIN32
293 {
294 SYSTEM_INFO system_info;
295
296 GetSystemInfo(&system_info);
297 qemu_real_host_page_size = system_info.dwPageSize;
298 }
299 #else
300 qemu_real_host_page_size = getpagesize();
301 #endif
302 if (qemu_host_page_size == 0) {
303 qemu_host_page_size = qemu_real_host_page_size;
304 }
305 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
306 qemu_host_page_size = TARGET_PAGE_SIZE;
307 }
308 qemu_host_page_mask = ~(qemu_host_page_size - 1);
309
310 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
311 {
312 #ifdef HAVE_KINFO_GETVMMAP
313 struct kinfo_vmentry *freep;
314 int i, cnt;
315
316 freep = kinfo_getvmmap(getpid(), &cnt);
317 if (freep) {
318 mmap_lock();
319 for (i = 0; i < cnt; i++) {
320 unsigned long startaddr, endaddr;
321
322 startaddr = freep[i].kve_start;
323 endaddr = freep[i].kve_end;
324 if (h2g_valid(startaddr)) {
325 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
326
327 if (h2g_valid(endaddr)) {
328 endaddr = h2g(endaddr);
329 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
330 } else {
331 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
332 endaddr = ~0ul;
333 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
334 #endif
335 }
336 }
337 }
338 free(freep);
339 mmap_unlock();
340 }
341 #else
342 FILE *f;
343
344 last_brk = (unsigned long)sbrk(0);
345
346 f = fopen("/compat/linux/proc/self/maps", "r");
347 if (f) {
348 mmap_lock();
349
350 do {
351 unsigned long startaddr, endaddr;
352 int n;
353
354 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
355
356 if (n == 2 && h2g_valid(startaddr)) {
357 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
358
359 if (h2g_valid(endaddr)) {
360 endaddr = h2g(endaddr);
361 } else {
362 endaddr = ~0ul;
363 }
364 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
365 }
366 } while (!feof(f));
367
368 fclose(f);
369 mmap_unlock();
370 }
371 #endif
372 }
373 #endif
374 }
375
376 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
377 {
378 PageDesc *pd;
379 void **lp;
380 int i;
381
382 #if defined(CONFIG_USER_ONLY)
383 /* We can't use g_malloc because it may recurse into a locked mutex. */
384 # define ALLOC(P, SIZE) \
385 do { \
386 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
387 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
388 } while (0)
389 #else
390 # define ALLOC(P, SIZE) \
391 do { P = g_malloc0(SIZE); } while (0)
392 #endif
393
394 /* Level 1. Always allocated. */
395 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
396
397 /* Level 2..N-1. */
398 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
399 void **p = *lp;
400
401 if (p == NULL) {
402 if (!alloc) {
403 return NULL;
404 }
405 ALLOC(p, sizeof(void *) * L2_SIZE);
406 *lp = p;
407 }
408
409 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
410 }
411
412 pd = *lp;
413 if (pd == NULL) {
414 if (!alloc) {
415 return NULL;
416 }
417 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
418 *lp = pd;
419 }
420
421 #undef ALLOC
422
423 return pd + (index & (L2_SIZE - 1));
424 }
425
426 static inline PageDesc *page_find(tb_page_addr_t index)
427 {
428 return page_find_alloc(index, 0);
429 }
430
431 #if !defined(CONFIG_USER_ONLY)
432 #define mmap_lock() do { } while (0)
433 #define mmap_unlock() do { } while (0)
434 #endif
435
436 #if defined(CONFIG_USER_ONLY)
437 /* Currently it is not recommended to allocate big chunks of data in
438 user mode. It will change when a dedicated libc will be used. */
439 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
440 region in which the guest needs to run. Revisit this. */
441 #define USE_STATIC_CODE_GEN_BUFFER
442 #endif
443
444 /* ??? Should configure for this, not list operating systems here. */
445 #if (defined(__linux__) \
446 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
447 || defined(__DragonFly__) || defined(__OpenBSD__) \
448 || defined(__NetBSD__))
449 # define USE_MMAP
450 #endif
451
452 /* Minimum size of the code gen buffer. This number is randomly chosen,
453 but not so small that we can't have a fair number of TB's live. */
454 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
455
456 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
457 indicated, this is constrained by the range of direct branches on the
458 host cpu, as used by the TCG implementation of goto_tb. */
459 #if defined(__x86_64__)
460 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
461 #elif defined(__sparc__)
462 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
463 #elif defined(__aarch64__)
464 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
465 #elif defined(__arm__)
466 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
467 #elif defined(__s390x__)
468 /* We have a +- 4GB range on the branches; leave some slop. */
469 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
470 #else
471 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
472 #endif
473
474 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
475
476 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
477 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
478 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
479
480 static inline size_t size_code_gen_buffer(size_t tb_size)
481 {
482 /* Size the buffer. */
483 if (tb_size == 0) {
484 #ifdef USE_STATIC_CODE_GEN_BUFFER
485 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
486 #else
487 /* ??? Needs adjustments. */
488 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
489 static buffer, we could size this on RESERVED_VA, on the text
490 segment size of the executable, or continue to use the default. */
491 tb_size = (unsigned long)(ram_size / 4);
492 #endif
493 }
494 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
495 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
496 }
497 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
498 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
499 }
500 tcg_ctx.code_gen_buffer_size = tb_size;
501 return tb_size;
502 }
503
504 #ifdef USE_STATIC_CODE_GEN_BUFFER
505 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
506 __attribute__((aligned(CODE_GEN_ALIGN)));
507
508 static inline void *alloc_code_gen_buffer(void)
509 {
510 map_exec(static_code_gen_buffer, tcg_ctx.code_gen_buffer_size);
511 return static_code_gen_buffer;
512 }
513 #elif defined(USE_MMAP)
514 static inline void *alloc_code_gen_buffer(void)
515 {
516 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
517 uintptr_t start = 0;
518 void *buf;
519
520 /* Constrain the position of the buffer based on the host cpu.
521 Note that these addresses are chosen in concert with the
522 addresses assigned in the relevant linker script file. */
523 # if defined(__PIE__) || defined(__PIC__)
524 /* Don't bother setting a preferred location if we're building
525 a position-independent executable. We're more likely to get
526 an address near the main executable if we let the kernel
527 choose the address. */
528 # elif defined(__x86_64__) && defined(MAP_32BIT)
529 /* Force the memory down into low memory with the executable.
530 Leave the choice of exact location with the kernel. */
531 flags |= MAP_32BIT;
532 /* Cannot expect to map more than 800MB in low memory. */
533 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
534 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
535 }
536 # elif defined(__sparc__)
537 start = 0x40000000ul;
538 # elif defined(__s390x__)
539 start = 0x90000000ul;
540 # endif
541
542 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
543 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
544 return buf == MAP_FAILED ? NULL : buf;
545 }
546 #else
547 static inline void *alloc_code_gen_buffer(void)
548 {
549 void *buf = g_malloc(tcg_ctx.code_gen_buffer_size);
550
551 if (buf) {
552 map_exec(buf, tcg_ctx.code_gen_buffer_size);
553 }
554 return buf;
555 }
556 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
557
558 static inline void code_gen_alloc(size_t tb_size)
559 {
560 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
561 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
562 if (tcg_ctx.code_gen_buffer == NULL) {
563 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
564 exit(1);
565 }
566
567 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
568 QEMU_MADV_HUGEPAGE);
569
570 /* Steal room for the prologue at the end of the buffer. This ensures
571 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
572 from TB's to the prologue are going to be in range. It also means
573 that we don't need to mark (additional) portions of the data segment
574 as executable. */
575 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
576 tcg_ctx.code_gen_buffer_size - 1024;
577 tcg_ctx.code_gen_buffer_size -= 1024;
578
579 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
580 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
581 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
582 CODE_GEN_AVG_BLOCK_SIZE;
583 tcg_ctx.tb_ctx.tbs =
584 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
585 }
586
587 /* Must be called before using the QEMU cpus. 'tb_size' is the size
588 (in bytes) allocated to the translation buffer. Zero means default
589 size. */
590 void tcg_exec_init(unsigned long tb_size)
591 {
592 cpu_gen_init();
593 code_gen_alloc(tb_size);
594 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
595 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
596 page_init();
597 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
598 /* There's no guest base to take into account, so go ahead and
599 initialize the prologue now. */
600 tcg_prologue_init(&tcg_ctx);
601 #endif
602 }
603
604 bool tcg_enabled(void)
605 {
606 return tcg_ctx.code_gen_buffer != NULL;
607 }
608
609 /* Allocate a new translation block. Flush the translation buffer if
610 too many translation blocks or too much generated code. */
611 static TranslationBlock *tb_alloc(target_ulong pc)
612 {
613 TranslationBlock *tb;
614
615 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
616 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
617 tcg_ctx.code_gen_buffer_max_size) {
618 return NULL;
619 }
620 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
621 tb->pc = pc;
622 tb->cflags = 0;
623 return tb;
624 }
625
626 void tb_free(TranslationBlock *tb)
627 {
628 /* In practice this is mostly used for single use temporary TB
629 Ignore the hard cases and just back up if this TB happens to
630 be the last one generated. */
631 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
632 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
633 tcg_ctx.code_gen_ptr = tb->tc_ptr;
634 tcg_ctx.tb_ctx.nb_tbs--;
635 }
636 }
637
638 static inline void invalidate_page_bitmap(PageDesc *p)
639 {
640 if (p->code_bitmap) {
641 g_free(p->code_bitmap);
642 p->code_bitmap = NULL;
643 }
644 p->code_write_count = 0;
645 }
646
647 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
648 static void page_flush_tb_1(int level, void **lp)
649 {
650 int i;
651
652 if (*lp == NULL) {
653 return;
654 }
655 if (level == 0) {
656 PageDesc *pd = *lp;
657
658 for (i = 0; i < L2_SIZE; ++i) {
659 pd[i].first_tb = NULL;
660 invalidate_page_bitmap(pd + i);
661 }
662 } else {
663 void **pp = *lp;
664
665 for (i = 0; i < L2_SIZE; ++i) {
666 page_flush_tb_1(level - 1, pp + i);
667 }
668 }
669 }
670
671 static void page_flush_tb(void)
672 {
673 int i;
674
675 for (i = 0; i < V_L1_SIZE; i++) {
676 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
677 }
678 }
679
680 /* flush all the translation blocks */
681 /* XXX: tb_flush is currently not thread safe */
682 void tb_flush(CPUArchState *env1)
683 {
684 CPUArchState *env;
685
686 #if defined(DEBUG_FLUSH)
687 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
688 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
689 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
690 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
691 tcg_ctx.tb_ctx.nb_tbs : 0);
692 #endif
693 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
694 > tcg_ctx.code_gen_buffer_size) {
695 cpu_abort(env1, "Internal error: code buffer overflow\n");
696 }
697 tcg_ctx.tb_ctx.nb_tbs = 0;
698
699 for (env = first_cpu; env != NULL; env = env->next_cpu) {
700 memset(env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof(void *));
701 }
702
703 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0,
704 CODE_GEN_PHYS_HASH_SIZE * sizeof(void *));
705 page_flush_tb();
706
707 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
708 /* XXX: flush processor icache at this point if cache flush is
709 expensive */
710 tcg_ctx.tb_ctx.tb_flush_count++;
711 }
712
713 #ifdef DEBUG_TB_CHECK
714
715 static void tb_invalidate_check(target_ulong address)
716 {
717 TranslationBlock *tb;
718 int i;
719
720 address &= TARGET_PAGE_MASK;
721 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
722 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
723 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
724 address >= tb->pc + tb->size)) {
725 printf("ERROR invalidate: address=" TARGET_FMT_lx
726 " PC=%08lx size=%04x\n",
727 address, (long)tb->pc, tb->size);
728 }
729 }
730 }
731 }
732
733 /* verify that all the pages have correct rights for code */
734 static void tb_page_check(void)
735 {
736 TranslationBlock *tb;
737 int i, flags1, flags2;
738
739 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
740 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
741 tb = tb->phys_hash_next) {
742 flags1 = page_get_flags(tb->pc);
743 flags2 = page_get_flags(tb->pc + tb->size - 1);
744 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
745 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
746 (long)tb->pc, tb->size, flags1, flags2);
747 }
748 }
749 }
750 }
751
752 #endif
753
754 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
755 {
756 TranslationBlock *tb1;
757
758 for (;;) {
759 tb1 = *ptb;
760 if (tb1 == tb) {
761 *ptb = tb1->phys_hash_next;
762 break;
763 }
764 ptb = &tb1->phys_hash_next;
765 }
766 }
767
768 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
769 {
770 TranslationBlock *tb1;
771 unsigned int n1;
772
773 for (;;) {
774 tb1 = *ptb;
775 n1 = (uintptr_t)tb1 & 3;
776 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
777 if (tb1 == tb) {
778 *ptb = tb1->page_next[n1];
779 break;
780 }
781 ptb = &tb1->page_next[n1];
782 }
783 }
784
785 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
786 {
787 TranslationBlock *tb1, **ptb;
788 unsigned int n1;
789
790 ptb = &tb->jmp_next[n];
791 tb1 = *ptb;
792 if (tb1) {
793 /* find tb(n) in circular list */
794 for (;;) {
795 tb1 = *ptb;
796 n1 = (uintptr_t)tb1 & 3;
797 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
798 if (n1 == n && tb1 == tb) {
799 break;
800 }
801 if (n1 == 2) {
802 ptb = &tb1->jmp_first;
803 } else {
804 ptb = &tb1->jmp_next[n1];
805 }
806 }
807 /* now we can suppress tb(n) from the list */
808 *ptb = tb->jmp_next[n];
809
810 tb->jmp_next[n] = NULL;
811 }
812 }
813
814 /* reset the jump entry 'n' of a TB so that it is not chained to
815 another TB */
816 static inline void tb_reset_jump(TranslationBlock *tb, int n)
817 {
818 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
819 }
820
821 /* invalidate one TB */
822 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
823 {
824 CPUArchState *env;
825 PageDesc *p;
826 unsigned int h, n1;
827 tb_page_addr_t phys_pc;
828 TranslationBlock *tb1, *tb2;
829
830 /* remove the TB from the hash list */
831 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
832 h = tb_phys_hash_func(phys_pc);
833 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
834
835 /* remove the TB from the page list */
836 if (tb->page_addr[0] != page_addr) {
837 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
838 tb_page_remove(&p->first_tb, tb);
839 invalidate_page_bitmap(p);
840 }
841 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
842 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
843 tb_page_remove(&p->first_tb, tb);
844 invalidate_page_bitmap(p);
845 }
846
847 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
848
849 /* remove the TB from the hash list */
850 h = tb_jmp_cache_hash_func(tb->pc);
851 for (env = first_cpu; env != NULL; env = env->next_cpu) {
852 if (env->tb_jmp_cache[h] == tb) {
853 env->tb_jmp_cache[h] = NULL;
854 }
855 }
856
857 /* suppress this TB from the two jump lists */
858 tb_jmp_remove(tb, 0);
859 tb_jmp_remove(tb, 1);
860
861 /* suppress any remaining jumps to this TB */
862 tb1 = tb->jmp_first;
863 for (;;) {
864 n1 = (uintptr_t)tb1 & 3;
865 if (n1 == 2) {
866 break;
867 }
868 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
869 tb2 = tb1->jmp_next[n1];
870 tb_reset_jump(tb1, n1);
871 tb1->jmp_next[n1] = NULL;
872 tb1 = tb2;
873 }
874 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
875
876 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
877 }
878
879 static inline void set_bits(uint8_t *tab, int start, int len)
880 {
881 int end, mask, end1;
882
883 end = start + len;
884 tab += start >> 3;
885 mask = 0xff << (start & 7);
886 if ((start & ~7) == (end & ~7)) {
887 if (start < end) {
888 mask &= ~(0xff << (end & 7));
889 *tab |= mask;
890 }
891 } else {
892 *tab++ |= mask;
893 start = (start + 8) & ~7;
894 end1 = end & ~7;
895 while (start < end1) {
896 *tab++ = 0xff;
897 start += 8;
898 }
899 if (start < end) {
900 mask = ~(0xff << (end & 7));
901 *tab |= mask;
902 }
903 }
904 }
905
906 static void build_page_bitmap(PageDesc *p)
907 {
908 int n, tb_start, tb_end;
909 TranslationBlock *tb;
910
911 p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
912
913 tb = p->first_tb;
914 while (tb != NULL) {
915 n = (uintptr_t)tb & 3;
916 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
917 /* NOTE: this is subtle as a TB may span two physical pages */
918 if (n == 0) {
919 /* NOTE: tb_end may be after the end of the page, but
920 it is not a problem */
921 tb_start = tb->pc & ~TARGET_PAGE_MASK;
922 tb_end = tb_start + tb->size;
923 if (tb_end > TARGET_PAGE_SIZE) {
924 tb_end = TARGET_PAGE_SIZE;
925 }
926 } else {
927 tb_start = 0;
928 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
929 }
930 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
931 tb = tb->page_next[n];
932 }
933 }
934
935 TranslationBlock *tb_gen_code(CPUArchState *env,
936 target_ulong pc, target_ulong cs_base,
937 int flags, int cflags)
938 {
939 TranslationBlock *tb;
940 uint8_t *tc_ptr;
941 tb_page_addr_t phys_pc, phys_page2;
942 target_ulong virt_page2;
943 int code_gen_size;
944
945 phys_pc = get_page_addr_code(env, pc);
946 tb = tb_alloc(pc);
947 if (!tb) {
948 /* flush must be done */
949 tb_flush(env);
950 /* cannot fail at this point */
951 tb = tb_alloc(pc);
952 /* Don't forget to invalidate previous TB info. */
953 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
954 }
955 tc_ptr = tcg_ctx.code_gen_ptr;
956 tb->tc_ptr = tc_ptr;
957 tb->cs_base = cs_base;
958 tb->flags = flags;
959 tb->cflags = cflags;
960 cpu_gen_code(env, tb, &code_gen_size);
961 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
962 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
963
964 /* check next page if needed */
965 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
966 phys_page2 = -1;
967 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
968 phys_page2 = get_page_addr_code(env, virt_page2);
969 }
970 tb_link_page(tb, phys_pc, phys_page2);
971 return tb;
972 }
973
974 /*
975 * Invalidate all TBs which intersect with the target physical address range
976 * [start;end[. NOTE: start and end may refer to *different* physical pages.
977 * 'is_cpu_write_access' should be true if called from a real cpu write
978 * access: the virtual CPU will exit the current TB if code is modified inside
979 * this TB.
980 */
981 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
982 int is_cpu_write_access)
983 {
984 while (start < end) {
985 tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
986 start &= TARGET_PAGE_MASK;
987 start += TARGET_PAGE_SIZE;
988 }
989 }
990
991 /*
992 * Invalidate all TBs which intersect with the target physical address range
993 * [start;end[. NOTE: start and end must refer to the *same* physical page.
994 * 'is_cpu_write_access' should be true if called from a real cpu write
995 * access: the virtual CPU will exit the current TB if code is modified inside
996 * this TB.
997 */
998 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
999 int is_cpu_write_access)
1000 {
1001 TranslationBlock *tb, *tb_next, *saved_tb;
1002 CPUArchState *env = cpu_single_env;
1003 CPUState *cpu = NULL;
1004 tb_page_addr_t tb_start, tb_end;
1005 PageDesc *p;
1006 int n;
1007 #ifdef TARGET_HAS_PRECISE_SMC
1008 int current_tb_not_found = is_cpu_write_access;
1009 TranslationBlock *current_tb = NULL;
1010 int current_tb_modified = 0;
1011 target_ulong current_pc = 0;
1012 target_ulong current_cs_base = 0;
1013 int current_flags = 0;
1014 #endif /* TARGET_HAS_PRECISE_SMC */
1015
1016 p = page_find(start >> TARGET_PAGE_BITS);
1017 if (!p) {
1018 return;
1019 }
1020 if (!p->code_bitmap &&
1021 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1022 is_cpu_write_access) {
1023 /* build code bitmap */
1024 build_page_bitmap(p);
1025 }
1026 if (env != NULL) {
1027 cpu = ENV_GET_CPU(env);
1028 }
1029
1030 /* we remove all the TBs in the range [start, end[ */
1031 /* XXX: see if in some cases it could be faster to invalidate all
1032 the code */
1033 tb = p->first_tb;
1034 while (tb != NULL) {
1035 n = (uintptr_t)tb & 3;
1036 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1037 tb_next = tb->page_next[n];
1038 /* NOTE: this is subtle as a TB may span two physical pages */
1039 if (n == 0) {
1040 /* NOTE: tb_end may be after the end of the page, but
1041 it is not a problem */
1042 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1043 tb_end = tb_start + tb->size;
1044 } else {
1045 tb_start = tb->page_addr[1];
1046 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1047 }
1048 if (!(tb_end <= start || tb_start >= end)) {
1049 #ifdef TARGET_HAS_PRECISE_SMC
1050 if (current_tb_not_found) {
1051 current_tb_not_found = 0;
1052 current_tb = NULL;
1053 if (env->mem_io_pc) {
1054 /* now we have a real cpu fault */
1055 current_tb = tb_find_pc(env->mem_io_pc);
1056 }
1057 }
1058 if (current_tb == tb &&
1059 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1060 /* If we are modifying the current TB, we must stop
1061 its execution. We could be more precise by checking
1062 that the modification is after the current PC, but it
1063 would require a specialized function to partially
1064 restore the CPU state */
1065
1066 current_tb_modified = 1;
1067 cpu_restore_state_from_tb(current_tb, env, env->mem_io_pc);
1068 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1069 &current_flags);
1070 }
1071 #endif /* TARGET_HAS_PRECISE_SMC */
1072 /* we need to do that to handle the case where a signal
1073 occurs while doing tb_phys_invalidate() */
1074 saved_tb = NULL;
1075 if (cpu != NULL) {
1076 saved_tb = cpu->current_tb;
1077 cpu->current_tb = NULL;
1078 }
1079 tb_phys_invalidate(tb, -1);
1080 if (cpu != NULL) {
1081 cpu->current_tb = saved_tb;
1082 if (cpu->interrupt_request && cpu->current_tb) {
1083 cpu_interrupt(cpu, cpu->interrupt_request);
1084 }
1085 }
1086 }
1087 tb = tb_next;
1088 }
1089 #if !defined(CONFIG_USER_ONLY)
1090 /* if no code remaining, no need to continue to use slow writes */
1091 if (!p->first_tb) {
1092 invalidate_page_bitmap(p);
1093 if (is_cpu_write_access) {
1094 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1095 }
1096 }
1097 #endif
1098 #ifdef TARGET_HAS_PRECISE_SMC
1099 if (current_tb_modified) {
1100 /* we generate a block containing just the instruction
1101 modifying the memory. It will ensure that it cannot modify
1102 itself */
1103 cpu->current_tb = NULL;
1104 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1105 cpu_resume_from_signal(env, NULL);
1106 }
1107 #endif
1108 }
1109
1110 /* len must be <= 8 and start must be a multiple of len */
1111 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1112 {
1113 PageDesc *p;
1114 int offset, b;
1115
1116 #if 0
1117 if (1) {
1118 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1119 cpu_single_env->mem_io_vaddr, len,
1120 cpu_single_env->eip,
1121 cpu_single_env->eip +
1122 (intptr_t)cpu_single_env->segs[R_CS].base);
1123 }
1124 #endif
1125 p = page_find(start >> TARGET_PAGE_BITS);
1126 if (!p) {
1127 return;
1128 }
1129 if (p->code_bitmap) {
1130 offset = start & ~TARGET_PAGE_MASK;
1131 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1132 if (b & ((1 << len) - 1)) {
1133 goto do_invalidate;
1134 }
1135 } else {
1136 do_invalidate:
1137 tb_invalidate_phys_page_range(start, start + len, 1);
1138 }
1139 }
1140
1141 #if !defined(CONFIG_SOFTMMU)
1142 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1143 uintptr_t pc, void *puc)
1144 {
1145 TranslationBlock *tb;
1146 PageDesc *p;
1147 int n;
1148 #ifdef TARGET_HAS_PRECISE_SMC
1149 TranslationBlock *current_tb = NULL;
1150 CPUArchState *env = cpu_single_env;
1151 CPUState *cpu = NULL;
1152 int current_tb_modified = 0;
1153 target_ulong current_pc = 0;
1154 target_ulong current_cs_base = 0;
1155 int current_flags = 0;
1156 #endif
1157
1158 addr &= TARGET_PAGE_MASK;
1159 p = page_find(addr >> TARGET_PAGE_BITS);
1160 if (!p) {
1161 return;
1162 }
1163 tb = p->first_tb;
1164 #ifdef TARGET_HAS_PRECISE_SMC
1165 if (tb && pc != 0) {
1166 current_tb = tb_find_pc(pc);
1167 }
1168 if (env != NULL) {
1169 cpu = ENV_GET_CPU(env);
1170 }
1171 #endif
1172 while (tb != NULL) {
1173 n = (uintptr_t)tb & 3;
1174 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1175 #ifdef TARGET_HAS_PRECISE_SMC
1176 if (current_tb == tb &&
1177 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1178 /* If we are modifying the current TB, we must stop
1179 its execution. We could be more precise by checking
1180 that the modification is after the current PC, but it
1181 would require a specialized function to partially
1182 restore the CPU state */
1183
1184 current_tb_modified = 1;
1185 cpu_restore_state_from_tb(current_tb, env, pc);
1186 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1187 &current_flags);
1188 }
1189 #endif /* TARGET_HAS_PRECISE_SMC */
1190 tb_phys_invalidate(tb, addr);
1191 tb = tb->page_next[n];
1192 }
1193 p->first_tb = NULL;
1194 #ifdef TARGET_HAS_PRECISE_SMC
1195 if (current_tb_modified) {
1196 /* we generate a block containing just the instruction
1197 modifying the memory. It will ensure that it cannot modify
1198 itself */
1199 cpu->current_tb = NULL;
1200 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1201 cpu_resume_from_signal(env, puc);
1202 }
1203 #endif
1204 }
1205 #endif
1206
1207 /* add the tb in the target page and protect it if necessary */
1208 static inline void tb_alloc_page(TranslationBlock *tb,
1209 unsigned int n, tb_page_addr_t page_addr)
1210 {
1211 PageDesc *p;
1212 #ifndef CONFIG_USER_ONLY
1213 bool page_already_protected;
1214 #endif
1215
1216 tb->page_addr[n] = page_addr;
1217 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1218 tb->page_next[n] = p->first_tb;
1219 #ifndef CONFIG_USER_ONLY
1220 page_already_protected = p->first_tb != NULL;
1221 #endif
1222 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1223 invalidate_page_bitmap(p);
1224
1225 #if defined(TARGET_HAS_SMC) || 1
1226
1227 #if defined(CONFIG_USER_ONLY)
1228 if (p->flags & PAGE_WRITE) {
1229 target_ulong addr;
1230 PageDesc *p2;
1231 int prot;
1232
1233 /* force the host page as non writable (writes will have a
1234 page fault + mprotect overhead) */
1235 page_addr &= qemu_host_page_mask;
1236 prot = 0;
1237 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1238 addr += TARGET_PAGE_SIZE) {
1239
1240 p2 = page_find(addr >> TARGET_PAGE_BITS);
1241 if (!p2) {
1242 continue;
1243 }
1244 prot |= p2->flags;
1245 p2->flags &= ~PAGE_WRITE;
1246 }
1247 mprotect(g2h(page_addr), qemu_host_page_size,
1248 (prot & PAGE_BITS) & ~PAGE_WRITE);
1249 #ifdef DEBUG_TB_INVALIDATE
1250 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1251 page_addr);
1252 #endif
1253 }
1254 #else
1255 /* if some code is already present, then the pages are already
1256 protected. So we handle the case where only the first TB is
1257 allocated in a physical page */
1258 if (!page_already_protected) {
1259 tlb_protect_code(page_addr);
1260 }
1261 #endif
1262
1263 #endif /* TARGET_HAS_SMC */
1264 }
1265
1266 /* add a new TB and link it to the physical page tables. phys_page2 is
1267 (-1) to indicate that only one page contains the TB. */
1268 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1269 tb_page_addr_t phys_page2)
1270 {
1271 unsigned int h;
1272 TranslationBlock **ptb;
1273
1274 /* Grab the mmap lock to stop another thread invalidating this TB
1275 before we are done. */
1276 mmap_lock();
1277 /* add in the physical hash table */
1278 h = tb_phys_hash_func(phys_pc);
1279 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1280 tb->phys_hash_next = *ptb;
1281 *ptb = tb;
1282
1283 /* add in the page list */
1284 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1285 if (phys_page2 != -1) {
1286 tb_alloc_page(tb, 1, phys_page2);
1287 } else {
1288 tb->page_addr[1] = -1;
1289 }
1290
1291 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1292 tb->jmp_next[0] = NULL;
1293 tb->jmp_next[1] = NULL;
1294
1295 /* init original jump addresses */
1296 if (tb->tb_next_offset[0] != 0xffff) {
1297 tb_reset_jump(tb, 0);
1298 }
1299 if (tb->tb_next_offset[1] != 0xffff) {
1300 tb_reset_jump(tb, 1);
1301 }
1302
1303 #ifdef DEBUG_TB_CHECK
1304 tb_page_check();
1305 #endif
1306 mmap_unlock();
1307 }
1308
1309 #if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
1310 /* check whether the given addr is in TCG generated code buffer or not */
1311 bool is_tcg_gen_code(uintptr_t tc_ptr)
1312 {
1313 /* This can be called during code generation, code_gen_buffer_size
1314 is used instead of code_gen_ptr for upper boundary checking */
1315 return (tc_ptr >= (uintptr_t)tcg_ctx.code_gen_buffer &&
1316 tc_ptr < (uintptr_t)(tcg_ctx.code_gen_buffer +
1317 tcg_ctx.code_gen_buffer_size));
1318 }
1319 #endif
1320
1321 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1322 tb[1].tc_ptr. Return NULL if not found */
1323 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1324 {
1325 int m_min, m_max, m;
1326 uintptr_t v;
1327 TranslationBlock *tb;
1328
1329 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1330 return NULL;
1331 }
1332 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1333 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1334 return NULL;
1335 }
1336 /* binary search (cf Knuth) */
1337 m_min = 0;
1338 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1339 while (m_min <= m_max) {
1340 m = (m_min + m_max) >> 1;
1341 tb = &tcg_ctx.tb_ctx.tbs[m];
1342 v = (uintptr_t)tb->tc_ptr;
1343 if (v == tc_ptr) {
1344 return tb;
1345 } else if (tc_ptr < v) {
1346 m_max = m - 1;
1347 } else {
1348 m_min = m + 1;
1349 }
1350 }
1351 return &tcg_ctx.tb_ctx.tbs[m_max];
1352 }
1353
1354 #if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
1355 void tb_invalidate_phys_addr(hwaddr addr)
1356 {
1357 ram_addr_t ram_addr;
1358 MemoryRegion *mr;
1359 hwaddr l = 1;
1360
1361 mr = address_space_translate(&address_space_memory, addr, &addr, &l, false);
1362 if (!(memory_region_is_ram(mr)
1363 || memory_region_is_romd(mr))) {
1364 return;
1365 }
1366 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
1367 + addr;
1368 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1369 }
1370 #endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
1371
1372 void tb_check_watchpoint(CPUArchState *env)
1373 {
1374 TranslationBlock *tb;
1375
1376 tb = tb_find_pc(env->mem_io_pc);
1377 if (!tb) {
1378 cpu_abort(env, "check_watchpoint: could not find TB for pc=%p",
1379 (void *)env->mem_io_pc);
1380 }
1381 cpu_restore_state_from_tb(tb, env, env->mem_io_pc);
1382 tb_phys_invalidate(tb, -1);
1383 }
1384
1385 #ifndef CONFIG_USER_ONLY
1386 /* mask must never be zero, except for A20 change call */
1387 static void tcg_handle_interrupt(CPUState *cpu, int mask)
1388 {
1389 CPUArchState *env = cpu->env_ptr;
1390 int old_mask;
1391
1392 old_mask = cpu->interrupt_request;
1393 cpu->interrupt_request |= mask;
1394
1395 /*
1396 * If called from iothread context, wake the target cpu in
1397 * case its halted.
1398 */
1399 if (!qemu_cpu_is_self(cpu)) {
1400 qemu_cpu_kick(cpu);
1401 return;
1402 }
1403
1404 if (use_icount) {
1405 env->icount_decr.u16.high = 0xffff;
1406 if (!can_do_io(env)
1407 && (mask & ~old_mask) != 0) {
1408 cpu_abort(env, "Raised interrupt while not in I/O function");
1409 }
1410 } else {
1411 cpu->tcg_exit_req = 1;
1412 }
1413 }
1414
1415 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1416
1417 /* in deterministic execution mode, instructions doing device I/Os
1418 must be at the end of the TB */
1419 void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
1420 {
1421 TranslationBlock *tb;
1422 uint32_t n, cflags;
1423 target_ulong pc, cs_base;
1424 uint64_t flags;
1425
1426 tb = tb_find_pc(retaddr);
1427 if (!tb) {
1428 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
1429 (void *)retaddr);
1430 }
1431 n = env->icount_decr.u16.low + tb->icount;
1432 cpu_restore_state_from_tb(tb, env, retaddr);
1433 /* Calculate how many instructions had been executed before the fault
1434 occurred. */
1435 n = n - env->icount_decr.u16.low;
1436 /* Generate a new TB ending on the I/O insn. */
1437 n++;
1438 /* On MIPS and SH, delay slot instructions can only be restarted if
1439 they were already the first instruction in the TB. If this is not
1440 the first instruction in a TB then re-execute the preceding
1441 branch. */
1442 #if defined(TARGET_MIPS)
1443 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1444 env->active_tc.PC -= 4;
1445 env->icount_decr.u16.low++;
1446 env->hflags &= ~MIPS_HFLAG_BMASK;
1447 }
1448 #elif defined(TARGET_SH4)
1449 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1450 && n > 1) {
1451 env->pc -= 2;
1452 env->icount_decr.u16.low++;
1453 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1454 }
1455 #endif
1456 /* This should never happen. */
1457 if (n > CF_COUNT_MASK) {
1458 cpu_abort(env, "TB too big during recompile");
1459 }
1460
1461 cflags = n | CF_LAST_IO;
1462 pc = tb->pc;
1463 cs_base = tb->cs_base;
1464 flags = tb->flags;
1465 tb_phys_invalidate(tb, -1);
1466 /* FIXME: In theory this could raise an exception. In practice
1467 we have already translated the block once so it's probably ok. */
1468 tb_gen_code(env, pc, cs_base, flags, cflags);
1469 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1470 the first in the TB) then we end up generating a whole new TB and
1471 repeating the fault, which is horribly inefficient.
1472 Better would be to execute just this insn uncached, or generate a
1473 second new TB. */
1474 cpu_resume_from_signal(env, NULL);
1475 }
1476
1477 void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
1478 {
1479 unsigned int i;
1480
1481 /* Discard jump cache entries for any tb which might potentially
1482 overlap the flushed page. */
1483 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1484 memset(&env->tb_jmp_cache[i], 0,
1485 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1486
1487 i = tb_jmp_cache_hash_page(addr);
1488 memset(&env->tb_jmp_cache[i], 0,
1489 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1490 }
1491
1492 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1493 {
1494 int i, target_code_size, max_target_code_size;
1495 int direct_jmp_count, direct_jmp2_count, cross_page;
1496 TranslationBlock *tb;
1497
1498 target_code_size = 0;
1499 max_target_code_size = 0;
1500 cross_page = 0;
1501 direct_jmp_count = 0;
1502 direct_jmp2_count = 0;
1503 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1504 tb = &tcg_ctx.tb_ctx.tbs[i];
1505 target_code_size += tb->size;
1506 if (tb->size > max_target_code_size) {
1507 max_target_code_size = tb->size;
1508 }
1509 if (tb->page_addr[1] != -1) {
1510 cross_page++;
1511 }
1512 if (tb->tb_next_offset[0] != 0xffff) {
1513 direct_jmp_count++;
1514 if (tb->tb_next_offset[1] != 0xffff) {
1515 direct_jmp2_count++;
1516 }
1517 }
1518 }
1519 /* XXX: avoid using doubles ? */
1520 cpu_fprintf(f, "Translation buffer state:\n");
1521 cpu_fprintf(f, "gen code size %td/%zd\n",
1522 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1523 tcg_ctx.code_gen_buffer_max_size);
1524 cpu_fprintf(f, "TB count %d/%d\n",
1525 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1526 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1527 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1528 tcg_ctx.tb_ctx.nb_tbs : 0,
1529 max_target_code_size);
1530 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1531 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1532 tcg_ctx.code_gen_buffer) /
1533 tcg_ctx.tb_ctx.nb_tbs : 0,
1534 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1535 tcg_ctx.code_gen_buffer) /
1536 target_code_size : 0);
1537 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1538 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1539 tcg_ctx.tb_ctx.nb_tbs : 0);
1540 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1541 direct_jmp_count,
1542 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1543 tcg_ctx.tb_ctx.nb_tbs : 0,
1544 direct_jmp2_count,
1545 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1546 tcg_ctx.tb_ctx.nb_tbs : 0);
1547 cpu_fprintf(f, "\nStatistics:\n");
1548 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1549 cpu_fprintf(f, "TB invalidate count %d\n",
1550 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1551 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1552 tcg_dump_info(f, cpu_fprintf);
1553 }
1554
1555 #else /* CONFIG_USER_ONLY */
1556
1557 void cpu_interrupt(CPUState *cpu, int mask)
1558 {
1559 cpu->interrupt_request |= mask;
1560 cpu->tcg_exit_req = 1;
1561 }
1562
1563 /*
1564 * Walks guest process memory "regions" one by one
1565 * and calls callback function 'fn' for each region.
1566 */
1567 struct walk_memory_regions_data {
1568 walk_memory_regions_fn fn;
1569 void *priv;
1570 uintptr_t start;
1571 int prot;
1572 };
1573
1574 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1575 abi_ulong end, int new_prot)
1576 {
1577 if (data->start != -1ul) {
1578 int rc = data->fn(data->priv, data->start, end, data->prot);
1579 if (rc != 0) {
1580 return rc;
1581 }
1582 }
1583
1584 data->start = (new_prot ? end : -1ul);
1585 data->prot = new_prot;
1586
1587 return 0;
1588 }
1589
1590 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1591 abi_ulong base, int level, void **lp)
1592 {
1593 abi_ulong pa;
1594 int i, rc;
1595
1596 if (*lp == NULL) {
1597 return walk_memory_regions_end(data, base, 0);
1598 }
1599
1600 if (level == 0) {
1601 PageDesc *pd = *lp;
1602
1603 for (i = 0; i < L2_SIZE; ++i) {
1604 int prot = pd[i].flags;
1605
1606 pa = base | (i << TARGET_PAGE_BITS);
1607 if (prot != data->prot) {
1608 rc = walk_memory_regions_end(data, pa, prot);
1609 if (rc != 0) {
1610 return rc;
1611 }
1612 }
1613 }
1614 } else {
1615 void **pp = *lp;
1616
1617 for (i = 0; i < L2_SIZE; ++i) {
1618 pa = base | ((abi_ulong)i <<
1619 (TARGET_PAGE_BITS + L2_BITS * level));
1620 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1621 if (rc != 0) {
1622 return rc;
1623 }
1624 }
1625 }
1626
1627 return 0;
1628 }
1629
1630 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1631 {
1632 struct walk_memory_regions_data data;
1633 uintptr_t i;
1634
1635 data.fn = fn;
1636 data.priv = priv;
1637 data.start = -1ul;
1638 data.prot = 0;
1639
1640 for (i = 0; i < V_L1_SIZE; i++) {
1641 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
1642 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
1643
1644 if (rc != 0) {
1645 return rc;
1646 }
1647 }
1648
1649 return walk_memory_regions_end(&data, 0, 0);
1650 }
1651
1652 static int dump_region(void *priv, abi_ulong start,
1653 abi_ulong end, unsigned long prot)
1654 {
1655 FILE *f = (FILE *)priv;
1656
1657 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
1658 " "TARGET_ABI_FMT_lx" %c%c%c\n",
1659 start, end, end - start,
1660 ((prot & PAGE_READ) ? 'r' : '-'),
1661 ((prot & PAGE_WRITE) ? 'w' : '-'),
1662 ((prot & PAGE_EXEC) ? 'x' : '-'));
1663
1664 return 0;
1665 }
1666
1667 /* dump memory mappings */
1668 void page_dump(FILE *f)
1669 {
1670 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
1671 "start", "end", "size", "prot");
1672 walk_memory_regions(f, dump_region);
1673 }
1674
1675 int page_get_flags(target_ulong address)
1676 {
1677 PageDesc *p;
1678
1679 p = page_find(address >> TARGET_PAGE_BITS);
1680 if (!p) {
1681 return 0;
1682 }
1683 return p->flags;
1684 }
1685
1686 /* Modify the flags of a page and invalidate the code if necessary.
1687 The flag PAGE_WRITE_ORG is positioned automatically depending
1688 on PAGE_WRITE. The mmap_lock should already be held. */
1689 void page_set_flags(target_ulong start, target_ulong end, int flags)
1690 {
1691 target_ulong addr, len;
1692
1693 /* This function should never be called with addresses outside the
1694 guest address space. If this assert fires, it probably indicates
1695 a missing call to h2g_valid. */
1696 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1697 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1698 #endif
1699 assert(start < end);
1700
1701 start = start & TARGET_PAGE_MASK;
1702 end = TARGET_PAGE_ALIGN(end);
1703
1704 if (flags & PAGE_WRITE) {
1705 flags |= PAGE_WRITE_ORG;
1706 }
1707
1708 for (addr = start, len = end - start;
1709 len != 0;
1710 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1711 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1712
1713 /* If the write protection bit is set, then we invalidate
1714 the code inside. */
1715 if (!(p->flags & PAGE_WRITE) &&
1716 (flags & PAGE_WRITE) &&
1717 p->first_tb) {
1718 tb_invalidate_phys_page(addr, 0, NULL);
1719 }
1720 p->flags = flags;
1721 }
1722 }
1723
1724 int page_check_range(target_ulong start, target_ulong len, int flags)
1725 {
1726 PageDesc *p;
1727 target_ulong end;
1728 target_ulong addr;
1729
1730 /* This function should never be called with addresses outside the
1731 guest address space. If this assert fires, it probably indicates
1732 a missing call to h2g_valid. */
1733 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1734 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1735 #endif
1736
1737 if (len == 0) {
1738 return 0;
1739 }
1740 if (start + len - 1 < start) {
1741 /* We've wrapped around. */
1742 return -1;
1743 }
1744
1745 /* must do before we loose bits in the next step */
1746 end = TARGET_PAGE_ALIGN(start + len);
1747 start = start & TARGET_PAGE_MASK;
1748
1749 for (addr = start, len = end - start;
1750 len != 0;
1751 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1752 p = page_find(addr >> TARGET_PAGE_BITS);
1753 if (!p) {
1754 return -1;
1755 }
1756 if (!(p->flags & PAGE_VALID)) {
1757 return -1;
1758 }
1759
1760 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1761 return -1;
1762 }
1763 if (flags & PAGE_WRITE) {
1764 if (!(p->flags & PAGE_WRITE_ORG)) {
1765 return -1;
1766 }
1767 /* unprotect the page if it was put read-only because it
1768 contains translated code */
1769 if (!(p->flags & PAGE_WRITE)) {
1770 if (!page_unprotect(addr, 0, NULL)) {
1771 return -1;
1772 }
1773 }
1774 return 0;
1775 }
1776 }
1777 return 0;
1778 }
1779
1780 /* called from signal handler: invalidate the code and unprotect the
1781 page. Return TRUE if the fault was successfully handled. */
1782 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1783 {
1784 unsigned int prot;
1785 PageDesc *p;
1786 target_ulong host_start, host_end, addr;
1787
1788 /* Technically this isn't safe inside a signal handler. However we
1789 know this only ever happens in a synchronous SEGV handler, so in
1790 practice it seems to be ok. */
1791 mmap_lock();
1792
1793 p = page_find(address >> TARGET_PAGE_BITS);
1794 if (!p) {
1795 mmap_unlock();
1796 return 0;
1797 }
1798
1799 /* if the page was really writable, then we change its
1800 protection back to writable */
1801 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1802 host_start = address & qemu_host_page_mask;
1803 host_end = host_start + qemu_host_page_size;
1804
1805 prot = 0;
1806 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1807 p = page_find(addr >> TARGET_PAGE_BITS);
1808 p->flags |= PAGE_WRITE;
1809 prot |= p->flags;
1810
1811 /* and since the content will be modified, we must invalidate
1812 the corresponding translated code. */
1813 tb_invalidate_phys_page(addr, pc, puc);
1814 #ifdef DEBUG_TB_CHECK
1815 tb_invalidate_check(addr);
1816 #endif
1817 }
1818 mprotect((void *)g2h(host_start), qemu_host_page_size,
1819 prot & PAGE_BITS);
1820
1821 mmap_unlock();
1822 return 1;
1823 }
1824 mmap_unlock();
1825 return 0;
1826 }
1827 #endif /* CONFIG_USER_ONLY */