translate-all: Change cpu_restore_state_from_tb() argument to CPUState
[qemu.git] / translate-all.c
1 /*
2 * Host code generation
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/types.h>
23 #include <sys/mman.h>
24 #endif
25 #include <stdarg.h>
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <inttypes.h>
30
31 #include "config.h"
32
33 #include "qemu-common.h"
34 #define NO_CPU_IO_DEFS
35 #include "cpu.h"
36 #include "disas/disas.h"
37 #include "tcg.h"
38 #if defined(CONFIG_USER_ONLY)
39 #include "qemu.h"
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
45 #include <sys/time.h>
46 #include <sys/proc.h>
47 #include <machine/profile.h>
48 #define _KERNEL
49 #include <sys/user.h>
50 #undef _KERNEL
51 #undef sigqueue
52 #include <libutil.h>
53 #endif
54 #endif
55 #else
56 #include "exec/address-spaces.h"
57 #endif
58
59 #include "exec/cputlb.h"
60 #include "translate-all.h"
61 #include "qemu/timer.h"
62
63 //#define DEBUG_TB_INVALIDATE
64 //#define DEBUG_FLUSH
65 /* make various TB consistency checks */
66 //#define DEBUG_TB_CHECK
67
68 #if !defined(CONFIG_USER_ONLY)
69 /* TB consistency checks only implemented for usermode emulation. */
70 #undef DEBUG_TB_CHECK
71 #endif
72
73 #define SMC_BITMAP_USE_THRESHOLD 10
74
75 typedef struct PageDesc {
76 /* list of TBs intersecting this ram page */
77 TranslationBlock *first_tb;
78 /* in order to optimize self modifying code, we count the number
79 of lookups we do to a given page to use a bitmap */
80 unsigned int code_write_count;
81 uint8_t *code_bitmap;
82 #if defined(CONFIG_USER_ONLY)
83 unsigned long flags;
84 #endif
85 } PageDesc;
86
87 /* In system mode we want L1_MAP to be based on ram offsets,
88 while in user mode we want it to be based on virtual addresses. */
89 #if !defined(CONFIG_USER_ONLY)
90 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
91 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
92 #else
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
94 #endif
95 #else
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
97 #endif
98
99 /* Size of the L2 (and L3, etc) page tables. */
100 #define V_L2_BITS 10
101 #define V_L2_SIZE (1 << V_L2_BITS)
102
103 /* The bits remaining after N lower levels of page tables. */
104 #define V_L1_BITS_REM \
105 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
106
107 #if V_L1_BITS_REM < 4
108 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
109 #else
110 #define V_L1_BITS V_L1_BITS_REM
111 #endif
112
113 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
114
115 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
116
117 uintptr_t qemu_real_host_page_size;
118 uintptr_t qemu_host_page_size;
119 uintptr_t qemu_host_page_mask;
120
121 /* This is a multi-level map on the virtual address space.
122 The bottom level has pointers to PageDesc. */
123 static void *l1_map[V_L1_SIZE];
124
125 /* code generation context */
126 TCGContext tcg_ctx;
127
128 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
129 tb_page_addr_t phys_page2);
130 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
131
132 void cpu_gen_init(void)
133 {
134 tcg_context_init(&tcg_ctx);
135 }
136
137 /* return non zero if the very first instruction is invalid so that
138 the virtual CPU can trigger an exception.
139
140 '*gen_code_size_ptr' contains the size of the generated code (host
141 code).
142 */
143 int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
144 {
145 TCGContext *s = &tcg_ctx;
146 uint8_t *gen_code_buf;
147 int gen_code_size;
148 #ifdef CONFIG_PROFILER
149 int64_t ti;
150 #endif
151
152 #ifdef CONFIG_PROFILER
153 s->tb_count1++; /* includes aborted translations because of
154 exceptions */
155 ti = profile_getclock();
156 #endif
157 tcg_func_start(s);
158
159 gen_intermediate_code(env, tb);
160
161 /* generate machine code */
162 gen_code_buf = tb->tc_ptr;
163 tb->tb_next_offset[0] = 0xffff;
164 tb->tb_next_offset[1] = 0xffff;
165 s->tb_next_offset = tb->tb_next_offset;
166 #ifdef USE_DIRECT_JUMP
167 s->tb_jmp_offset = tb->tb_jmp_offset;
168 s->tb_next = NULL;
169 #else
170 s->tb_jmp_offset = NULL;
171 s->tb_next = tb->tb_next;
172 #endif
173
174 #ifdef CONFIG_PROFILER
175 s->tb_count++;
176 s->interm_time += profile_getclock() - ti;
177 s->code_time -= profile_getclock();
178 #endif
179 gen_code_size = tcg_gen_code(s, gen_code_buf);
180 *gen_code_size_ptr = gen_code_size;
181 #ifdef CONFIG_PROFILER
182 s->code_time += profile_getclock();
183 s->code_in_len += tb->size;
184 s->code_out_len += gen_code_size;
185 #endif
186
187 #ifdef DEBUG_DISAS
188 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
189 qemu_log("OUT: [size=%d]\n", *gen_code_size_ptr);
190 log_disas(tb->tc_ptr, *gen_code_size_ptr);
191 qemu_log("\n");
192 qemu_log_flush();
193 }
194 #endif
195 return 0;
196 }
197
198 /* The cpu state corresponding to 'searched_pc' is restored.
199 */
200 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
201 uintptr_t searched_pc)
202 {
203 CPUArchState *env = cpu->env_ptr;
204 TCGContext *s = &tcg_ctx;
205 int j;
206 uintptr_t tc_ptr;
207 #ifdef CONFIG_PROFILER
208 int64_t ti;
209 #endif
210
211 #ifdef CONFIG_PROFILER
212 ti = profile_getclock();
213 #endif
214 tcg_func_start(s);
215
216 gen_intermediate_code_pc(env, tb);
217
218 if (use_icount) {
219 /* Reset the cycle counter to the start of the block. */
220 cpu->icount_decr.u16.low += tb->icount;
221 /* Clear the IO flag. */
222 cpu->can_do_io = 0;
223 }
224
225 /* find opc index corresponding to search_pc */
226 tc_ptr = (uintptr_t)tb->tc_ptr;
227 if (searched_pc < tc_ptr)
228 return -1;
229
230 s->tb_next_offset = tb->tb_next_offset;
231 #ifdef USE_DIRECT_JUMP
232 s->tb_jmp_offset = tb->tb_jmp_offset;
233 s->tb_next = NULL;
234 #else
235 s->tb_jmp_offset = NULL;
236 s->tb_next = tb->tb_next;
237 #endif
238 j = tcg_gen_code_search_pc(s, (uint8_t *)tc_ptr, searched_pc - tc_ptr);
239 if (j < 0)
240 return -1;
241 /* now find start of instruction before */
242 while (s->gen_opc_instr_start[j] == 0) {
243 j--;
244 }
245 cpu->icount_decr.u16.low -= s->gen_opc_icount[j];
246
247 restore_state_to_opc(env, tb, j);
248
249 #ifdef CONFIG_PROFILER
250 s->restore_time += profile_getclock() - ti;
251 s->restore_count++;
252 #endif
253 return 0;
254 }
255
256 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
257 {
258 TranslationBlock *tb;
259
260 tb = tb_find_pc(retaddr);
261 if (tb) {
262 cpu_restore_state_from_tb(cpu, tb, retaddr);
263 return true;
264 }
265 return false;
266 }
267
268 #ifdef _WIN32
269 static inline void map_exec(void *addr, long size)
270 {
271 DWORD old_protect;
272 VirtualProtect(addr, size,
273 PAGE_EXECUTE_READWRITE, &old_protect);
274 }
275 #else
276 static inline void map_exec(void *addr, long size)
277 {
278 unsigned long start, end, page_size;
279
280 page_size = getpagesize();
281 start = (unsigned long)addr;
282 start &= ~(page_size - 1);
283
284 end = (unsigned long)addr + size;
285 end += page_size - 1;
286 end &= ~(page_size - 1);
287
288 mprotect((void *)start, end - start,
289 PROT_READ | PROT_WRITE | PROT_EXEC);
290 }
291 #endif
292
293 void page_size_init(void)
294 {
295 /* NOTE: we can always suppose that qemu_host_page_size >=
296 TARGET_PAGE_SIZE */
297 #ifdef _WIN32
298 SYSTEM_INFO system_info;
299
300 GetSystemInfo(&system_info);
301 qemu_real_host_page_size = system_info.dwPageSize;
302 #else
303 qemu_real_host_page_size = getpagesize();
304 #endif
305 if (qemu_host_page_size == 0) {
306 qemu_host_page_size = qemu_real_host_page_size;
307 }
308 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
309 qemu_host_page_size = TARGET_PAGE_SIZE;
310 }
311 qemu_host_page_mask = ~(qemu_host_page_size - 1);
312 }
313
314 static void page_init(void)
315 {
316 page_size_init();
317 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
318 {
319 #ifdef HAVE_KINFO_GETVMMAP
320 struct kinfo_vmentry *freep;
321 int i, cnt;
322
323 freep = kinfo_getvmmap(getpid(), &cnt);
324 if (freep) {
325 mmap_lock();
326 for (i = 0; i < cnt; i++) {
327 unsigned long startaddr, endaddr;
328
329 startaddr = freep[i].kve_start;
330 endaddr = freep[i].kve_end;
331 if (h2g_valid(startaddr)) {
332 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
333
334 if (h2g_valid(endaddr)) {
335 endaddr = h2g(endaddr);
336 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
337 } else {
338 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
339 endaddr = ~0ul;
340 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
341 #endif
342 }
343 }
344 }
345 free(freep);
346 mmap_unlock();
347 }
348 #else
349 FILE *f;
350
351 last_brk = (unsigned long)sbrk(0);
352
353 f = fopen("/compat/linux/proc/self/maps", "r");
354 if (f) {
355 mmap_lock();
356
357 do {
358 unsigned long startaddr, endaddr;
359 int n;
360
361 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
362
363 if (n == 2 && h2g_valid(startaddr)) {
364 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
365
366 if (h2g_valid(endaddr)) {
367 endaddr = h2g(endaddr);
368 } else {
369 endaddr = ~0ul;
370 }
371 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
372 }
373 } while (!feof(f));
374
375 fclose(f);
376 mmap_unlock();
377 }
378 #endif
379 }
380 #endif
381 }
382
383 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
384 {
385 PageDesc *pd;
386 void **lp;
387 int i;
388
389 #if defined(CONFIG_USER_ONLY)
390 /* We can't use g_malloc because it may recurse into a locked mutex. */
391 # define ALLOC(P, SIZE) \
392 do { \
393 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
394 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
395 } while (0)
396 #else
397 # define ALLOC(P, SIZE) \
398 do { P = g_malloc0(SIZE); } while (0)
399 #endif
400
401 /* Level 1. Always allocated. */
402 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
403
404 /* Level 2..N-1. */
405 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
406 void **p = *lp;
407
408 if (p == NULL) {
409 if (!alloc) {
410 return NULL;
411 }
412 ALLOC(p, sizeof(void *) * V_L2_SIZE);
413 *lp = p;
414 }
415
416 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
417 }
418
419 pd = *lp;
420 if (pd == NULL) {
421 if (!alloc) {
422 return NULL;
423 }
424 ALLOC(pd, sizeof(PageDesc) * V_L2_SIZE);
425 *lp = pd;
426 }
427
428 #undef ALLOC
429
430 return pd + (index & (V_L2_SIZE - 1));
431 }
432
433 static inline PageDesc *page_find(tb_page_addr_t index)
434 {
435 return page_find_alloc(index, 0);
436 }
437
438 #if !defined(CONFIG_USER_ONLY)
439 #define mmap_lock() do { } while (0)
440 #define mmap_unlock() do { } while (0)
441 #endif
442
443 #if defined(CONFIG_USER_ONLY)
444 /* Currently it is not recommended to allocate big chunks of data in
445 user mode. It will change when a dedicated libc will be used. */
446 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
447 region in which the guest needs to run. Revisit this. */
448 #define USE_STATIC_CODE_GEN_BUFFER
449 #endif
450
451 /* ??? Should configure for this, not list operating systems here. */
452 #if (defined(__linux__) \
453 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
454 || defined(__DragonFly__) || defined(__OpenBSD__) \
455 || defined(__NetBSD__))
456 # define USE_MMAP
457 #endif
458
459 /* Minimum size of the code gen buffer. This number is randomly chosen,
460 but not so small that we can't have a fair number of TB's live. */
461 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
462
463 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
464 indicated, this is constrained by the range of direct branches on the
465 host cpu, as used by the TCG implementation of goto_tb. */
466 #if defined(__x86_64__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
468 #elif defined(__sparc__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
470 #elif defined(__aarch64__)
471 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
472 #elif defined(__arm__)
473 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
474 #elif defined(__s390x__)
475 /* We have a +- 4GB range on the branches; leave some slop. */
476 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
477 #else
478 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
479 #endif
480
481 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
482
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
484 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
485 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
486
487 static inline size_t size_code_gen_buffer(size_t tb_size)
488 {
489 /* Size the buffer. */
490 if (tb_size == 0) {
491 #ifdef USE_STATIC_CODE_GEN_BUFFER
492 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
493 #else
494 /* ??? Needs adjustments. */
495 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
496 static buffer, we could size this on RESERVED_VA, on the text
497 segment size of the executable, or continue to use the default. */
498 tb_size = (unsigned long)(ram_size / 4);
499 #endif
500 }
501 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
502 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
503 }
504 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
505 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
506 }
507 tcg_ctx.code_gen_buffer_size = tb_size;
508 return tb_size;
509 }
510
511 #ifdef USE_STATIC_CODE_GEN_BUFFER
512 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
513 __attribute__((aligned(CODE_GEN_ALIGN)));
514
515 static inline void *alloc_code_gen_buffer(void)
516 {
517 map_exec(static_code_gen_buffer, tcg_ctx.code_gen_buffer_size);
518 return static_code_gen_buffer;
519 }
520 #elif defined(USE_MMAP)
521 static inline void *alloc_code_gen_buffer(void)
522 {
523 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
524 uintptr_t start = 0;
525 void *buf;
526
527 /* Constrain the position of the buffer based on the host cpu.
528 Note that these addresses are chosen in concert with the
529 addresses assigned in the relevant linker script file. */
530 # if defined(__PIE__) || defined(__PIC__)
531 /* Don't bother setting a preferred location if we're building
532 a position-independent executable. We're more likely to get
533 an address near the main executable if we let the kernel
534 choose the address. */
535 # elif defined(__x86_64__) && defined(MAP_32BIT)
536 /* Force the memory down into low memory with the executable.
537 Leave the choice of exact location with the kernel. */
538 flags |= MAP_32BIT;
539 /* Cannot expect to map more than 800MB in low memory. */
540 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
541 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
542 }
543 # elif defined(__sparc__)
544 start = 0x40000000ul;
545 # elif defined(__s390x__)
546 start = 0x90000000ul;
547 # endif
548
549 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
550 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
551 return buf == MAP_FAILED ? NULL : buf;
552 }
553 #else
554 static inline void *alloc_code_gen_buffer(void)
555 {
556 void *buf = g_malloc(tcg_ctx.code_gen_buffer_size);
557
558 if (buf) {
559 map_exec(buf, tcg_ctx.code_gen_buffer_size);
560 }
561 return buf;
562 }
563 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
564
565 static inline void code_gen_alloc(size_t tb_size)
566 {
567 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
568 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
569 if (tcg_ctx.code_gen_buffer == NULL) {
570 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
571 exit(1);
572 }
573
574 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
575 QEMU_MADV_HUGEPAGE);
576
577 /* Steal room for the prologue at the end of the buffer. This ensures
578 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
579 from TB's to the prologue are going to be in range. It also means
580 that we don't need to mark (additional) portions of the data segment
581 as executable. */
582 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
583 tcg_ctx.code_gen_buffer_size - 1024;
584 tcg_ctx.code_gen_buffer_size -= 1024;
585
586 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
587 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
588 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
589 CODE_GEN_AVG_BLOCK_SIZE;
590 tcg_ctx.tb_ctx.tbs =
591 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
592 }
593
594 /* Must be called before using the QEMU cpus. 'tb_size' is the size
595 (in bytes) allocated to the translation buffer. Zero means default
596 size. */
597 void tcg_exec_init(unsigned long tb_size)
598 {
599 cpu_gen_init();
600 code_gen_alloc(tb_size);
601 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
602 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
603 page_init();
604 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
605 /* There's no guest base to take into account, so go ahead and
606 initialize the prologue now. */
607 tcg_prologue_init(&tcg_ctx);
608 #endif
609 }
610
611 bool tcg_enabled(void)
612 {
613 return tcg_ctx.code_gen_buffer != NULL;
614 }
615
616 /* Allocate a new translation block. Flush the translation buffer if
617 too many translation blocks or too much generated code. */
618 static TranslationBlock *tb_alloc(target_ulong pc)
619 {
620 TranslationBlock *tb;
621
622 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
623 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
624 tcg_ctx.code_gen_buffer_max_size) {
625 return NULL;
626 }
627 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
628 tb->pc = pc;
629 tb->cflags = 0;
630 return tb;
631 }
632
633 void tb_free(TranslationBlock *tb)
634 {
635 /* In practice this is mostly used for single use temporary TB
636 Ignore the hard cases and just back up if this TB happens to
637 be the last one generated. */
638 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
639 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
640 tcg_ctx.code_gen_ptr = tb->tc_ptr;
641 tcg_ctx.tb_ctx.nb_tbs--;
642 }
643 }
644
645 static inline void invalidate_page_bitmap(PageDesc *p)
646 {
647 if (p->code_bitmap) {
648 g_free(p->code_bitmap);
649 p->code_bitmap = NULL;
650 }
651 p->code_write_count = 0;
652 }
653
654 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
655 static void page_flush_tb_1(int level, void **lp)
656 {
657 int i;
658
659 if (*lp == NULL) {
660 return;
661 }
662 if (level == 0) {
663 PageDesc *pd = *lp;
664
665 for (i = 0; i < V_L2_SIZE; ++i) {
666 pd[i].first_tb = NULL;
667 invalidate_page_bitmap(pd + i);
668 }
669 } else {
670 void **pp = *lp;
671
672 for (i = 0; i < V_L2_SIZE; ++i) {
673 page_flush_tb_1(level - 1, pp + i);
674 }
675 }
676 }
677
678 static void page_flush_tb(void)
679 {
680 int i;
681
682 for (i = 0; i < V_L1_SIZE; i++) {
683 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
684 }
685 }
686
687 /* flush all the translation blocks */
688 /* XXX: tb_flush is currently not thread safe */
689 void tb_flush(CPUArchState *env1)
690 {
691 CPUState *cpu;
692
693 #if defined(DEBUG_FLUSH)
694 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
695 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
696 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
697 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
698 tcg_ctx.tb_ctx.nb_tbs : 0);
699 #endif
700 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
701 > tcg_ctx.code_gen_buffer_size) {
702 cpu_abort(env1, "Internal error: code buffer overflow\n");
703 }
704 tcg_ctx.tb_ctx.nb_tbs = 0;
705
706 CPU_FOREACH(cpu) {
707 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
708 }
709
710 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
711 page_flush_tb();
712
713 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
714 /* XXX: flush processor icache at this point if cache flush is
715 expensive */
716 tcg_ctx.tb_ctx.tb_flush_count++;
717 }
718
719 #ifdef DEBUG_TB_CHECK
720
721 static void tb_invalidate_check(target_ulong address)
722 {
723 TranslationBlock *tb;
724 int i;
725
726 address &= TARGET_PAGE_MASK;
727 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
728 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
729 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
730 address >= tb->pc + tb->size)) {
731 printf("ERROR invalidate: address=" TARGET_FMT_lx
732 " PC=%08lx size=%04x\n",
733 address, (long)tb->pc, tb->size);
734 }
735 }
736 }
737 }
738
739 /* verify that all the pages have correct rights for code */
740 static void tb_page_check(void)
741 {
742 TranslationBlock *tb;
743 int i, flags1, flags2;
744
745 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
746 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
747 tb = tb->phys_hash_next) {
748 flags1 = page_get_flags(tb->pc);
749 flags2 = page_get_flags(tb->pc + tb->size - 1);
750 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
751 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
752 (long)tb->pc, tb->size, flags1, flags2);
753 }
754 }
755 }
756 }
757
758 #endif
759
760 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
761 {
762 TranslationBlock *tb1;
763
764 for (;;) {
765 tb1 = *ptb;
766 if (tb1 == tb) {
767 *ptb = tb1->phys_hash_next;
768 break;
769 }
770 ptb = &tb1->phys_hash_next;
771 }
772 }
773
774 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
775 {
776 TranslationBlock *tb1;
777 unsigned int n1;
778
779 for (;;) {
780 tb1 = *ptb;
781 n1 = (uintptr_t)tb1 & 3;
782 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
783 if (tb1 == tb) {
784 *ptb = tb1->page_next[n1];
785 break;
786 }
787 ptb = &tb1->page_next[n1];
788 }
789 }
790
791 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
792 {
793 TranslationBlock *tb1, **ptb;
794 unsigned int n1;
795
796 ptb = &tb->jmp_next[n];
797 tb1 = *ptb;
798 if (tb1) {
799 /* find tb(n) in circular list */
800 for (;;) {
801 tb1 = *ptb;
802 n1 = (uintptr_t)tb1 & 3;
803 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
804 if (n1 == n && tb1 == tb) {
805 break;
806 }
807 if (n1 == 2) {
808 ptb = &tb1->jmp_first;
809 } else {
810 ptb = &tb1->jmp_next[n1];
811 }
812 }
813 /* now we can suppress tb(n) from the list */
814 *ptb = tb->jmp_next[n];
815
816 tb->jmp_next[n] = NULL;
817 }
818 }
819
820 /* reset the jump entry 'n' of a TB so that it is not chained to
821 another TB */
822 static inline void tb_reset_jump(TranslationBlock *tb, int n)
823 {
824 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
825 }
826
827 /* invalidate one TB */
828 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
829 {
830 CPUState *cpu;
831 PageDesc *p;
832 unsigned int h, n1;
833 tb_page_addr_t phys_pc;
834 TranslationBlock *tb1, *tb2;
835
836 /* remove the TB from the hash list */
837 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
838 h = tb_phys_hash_func(phys_pc);
839 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
840
841 /* remove the TB from the page list */
842 if (tb->page_addr[0] != page_addr) {
843 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
844 tb_page_remove(&p->first_tb, tb);
845 invalidate_page_bitmap(p);
846 }
847 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
848 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
849 tb_page_remove(&p->first_tb, tb);
850 invalidate_page_bitmap(p);
851 }
852
853 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
854
855 /* remove the TB from the hash list */
856 h = tb_jmp_cache_hash_func(tb->pc);
857 CPU_FOREACH(cpu) {
858 if (cpu->tb_jmp_cache[h] == tb) {
859 cpu->tb_jmp_cache[h] = NULL;
860 }
861 }
862
863 /* suppress this TB from the two jump lists */
864 tb_jmp_remove(tb, 0);
865 tb_jmp_remove(tb, 1);
866
867 /* suppress any remaining jumps to this TB */
868 tb1 = tb->jmp_first;
869 for (;;) {
870 n1 = (uintptr_t)tb1 & 3;
871 if (n1 == 2) {
872 break;
873 }
874 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
875 tb2 = tb1->jmp_next[n1];
876 tb_reset_jump(tb1, n1);
877 tb1->jmp_next[n1] = NULL;
878 tb1 = tb2;
879 }
880 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
881
882 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
883 }
884
885 static inline void set_bits(uint8_t *tab, int start, int len)
886 {
887 int end, mask, end1;
888
889 end = start + len;
890 tab += start >> 3;
891 mask = 0xff << (start & 7);
892 if ((start & ~7) == (end & ~7)) {
893 if (start < end) {
894 mask &= ~(0xff << (end & 7));
895 *tab |= mask;
896 }
897 } else {
898 *tab++ |= mask;
899 start = (start + 8) & ~7;
900 end1 = end & ~7;
901 while (start < end1) {
902 *tab++ = 0xff;
903 start += 8;
904 }
905 if (start < end) {
906 mask = ~(0xff << (end & 7));
907 *tab |= mask;
908 }
909 }
910 }
911
912 static void build_page_bitmap(PageDesc *p)
913 {
914 int n, tb_start, tb_end;
915 TranslationBlock *tb;
916
917 p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
918
919 tb = p->first_tb;
920 while (tb != NULL) {
921 n = (uintptr_t)tb & 3;
922 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
923 /* NOTE: this is subtle as a TB may span two physical pages */
924 if (n == 0) {
925 /* NOTE: tb_end may be after the end of the page, but
926 it is not a problem */
927 tb_start = tb->pc & ~TARGET_PAGE_MASK;
928 tb_end = tb_start + tb->size;
929 if (tb_end > TARGET_PAGE_SIZE) {
930 tb_end = TARGET_PAGE_SIZE;
931 }
932 } else {
933 tb_start = 0;
934 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
935 }
936 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
937 tb = tb->page_next[n];
938 }
939 }
940
941 TranslationBlock *tb_gen_code(CPUArchState *env,
942 target_ulong pc, target_ulong cs_base,
943 int flags, int cflags)
944 {
945 TranslationBlock *tb;
946 uint8_t *tc_ptr;
947 tb_page_addr_t phys_pc, phys_page2;
948 target_ulong virt_page2;
949 int code_gen_size;
950
951 phys_pc = get_page_addr_code(env, pc);
952 tb = tb_alloc(pc);
953 if (!tb) {
954 /* flush must be done */
955 tb_flush(env);
956 /* cannot fail at this point */
957 tb = tb_alloc(pc);
958 /* Don't forget to invalidate previous TB info. */
959 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
960 }
961 tc_ptr = tcg_ctx.code_gen_ptr;
962 tb->tc_ptr = tc_ptr;
963 tb->cs_base = cs_base;
964 tb->flags = flags;
965 tb->cflags = cflags;
966 cpu_gen_code(env, tb, &code_gen_size);
967 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
968 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
969
970 /* check next page if needed */
971 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
972 phys_page2 = -1;
973 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
974 phys_page2 = get_page_addr_code(env, virt_page2);
975 }
976 tb_link_page(tb, phys_pc, phys_page2);
977 return tb;
978 }
979
980 /*
981 * Invalidate all TBs which intersect with the target physical address range
982 * [start;end[. NOTE: start and end may refer to *different* physical pages.
983 * 'is_cpu_write_access' should be true if called from a real cpu write
984 * access: the virtual CPU will exit the current TB if code is modified inside
985 * this TB.
986 */
987 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
988 int is_cpu_write_access)
989 {
990 while (start < end) {
991 tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
992 start &= TARGET_PAGE_MASK;
993 start += TARGET_PAGE_SIZE;
994 }
995 }
996
997 /*
998 * Invalidate all TBs which intersect with the target physical address range
999 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1000 * 'is_cpu_write_access' should be true if called from a real cpu write
1001 * access: the virtual CPU will exit the current TB if code is modified inside
1002 * this TB.
1003 */
1004 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1005 int is_cpu_write_access)
1006 {
1007 TranslationBlock *tb, *tb_next, *saved_tb;
1008 CPUState *cpu = current_cpu;
1009 #if defined(TARGET_HAS_PRECISE_SMC) || !defined(CONFIG_USER_ONLY)
1010 CPUArchState *env = NULL;
1011 #endif
1012 tb_page_addr_t tb_start, tb_end;
1013 PageDesc *p;
1014 int n;
1015 #ifdef TARGET_HAS_PRECISE_SMC
1016 int current_tb_not_found = is_cpu_write_access;
1017 TranslationBlock *current_tb = NULL;
1018 int current_tb_modified = 0;
1019 target_ulong current_pc = 0;
1020 target_ulong current_cs_base = 0;
1021 int current_flags = 0;
1022 #endif /* TARGET_HAS_PRECISE_SMC */
1023
1024 p = page_find(start >> TARGET_PAGE_BITS);
1025 if (!p) {
1026 return;
1027 }
1028 if (!p->code_bitmap &&
1029 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1030 is_cpu_write_access) {
1031 /* build code bitmap */
1032 build_page_bitmap(p);
1033 }
1034 #if defined(TARGET_HAS_PRECISE_SMC) || !defined(CONFIG_USER_ONLY)
1035 if (cpu != NULL) {
1036 env = cpu->env_ptr;
1037 }
1038 #endif
1039
1040 /* we remove all the TBs in the range [start, end[ */
1041 /* XXX: see if in some cases it could be faster to invalidate all
1042 the code */
1043 tb = p->first_tb;
1044 while (tb != NULL) {
1045 n = (uintptr_t)tb & 3;
1046 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1047 tb_next = tb->page_next[n];
1048 /* NOTE: this is subtle as a TB may span two physical pages */
1049 if (n == 0) {
1050 /* NOTE: tb_end may be after the end of the page, but
1051 it is not a problem */
1052 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1053 tb_end = tb_start + tb->size;
1054 } else {
1055 tb_start = tb->page_addr[1];
1056 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1057 }
1058 if (!(tb_end <= start || tb_start >= end)) {
1059 #ifdef TARGET_HAS_PRECISE_SMC
1060 if (current_tb_not_found) {
1061 current_tb_not_found = 0;
1062 current_tb = NULL;
1063 if (cpu->mem_io_pc) {
1064 /* now we have a real cpu fault */
1065 current_tb = tb_find_pc(cpu->mem_io_pc);
1066 }
1067 }
1068 if (current_tb == tb &&
1069 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1070 /* If we are modifying the current TB, we must stop
1071 its execution. We could be more precise by checking
1072 that the modification is after the current PC, but it
1073 would require a specialized function to partially
1074 restore the CPU state */
1075
1076 current_tb_modified = 1;
1077 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
1078 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1079 &current_flags);
1080 }
1081 #endif /* TARGET_HAS_PRECISE_SMC */
1082 /* we need to do that to handle the case where a signal
1083 occurs while doing tb_phys_invalidate() */
1084 saved_tb = NULL;
1085 if (cpu != NULL) {
1086 saved_tb = cpu->current_tb;
1087 cpu->current_tb = NULL;
1088 }
1089 tb_phys_invalidate(tb, -1);
1090 if (cpu != NULL) {
1091 cpu->current_tb = saved_tb;
1092 if (cpu->interrupt_request && cpu->current_tb) {
1093 cpu_interrupt(cpu, cpu->interrupt_request);
1094 }
1095 }
1096 }
1097 tb = tb_next;
1098 }
1099 #if !defined(CONFIG_USER_ONLY)
1100 /* if no code remaining, no need to continue to use slow writes */
1101 if (!p->first_tb) {
1102 invalidate_page_bitmap(p);
1103 if (is_cpu_write_access) {
1104 tlb_unprotect_code_phys(env, start, cpu->mem_io_vaddr);
1105 }
1106 }
1107 #endif
1108 #ifdef TARGET_HAS_PRECISE_SMC
1109 if (current_tb_modified) {
1110 /* we generate a block containing just the instruction
1111 modifying the memory. It will ensure that it cannot modify
1112 itself */
1113 cpu->current_tb = NULL;
1114 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1115 cpu_resume_from_signal(env, NULL);
1116 }
1117 #endif
1118 }
1119
1120 /* len must be <= 8 and start must be a multiple of len */
1121 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1122 {
1123 PageDesc *p;
1124 int offset, b;
1125
1126 #if 0
1127 if (1) {
1128 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1129 cpu_single_env->mem_io_vaddr, len,
1130 cpu_single_env->eip,
1131 cpu_single_env->eip +
1132 (intptr_t)cpu_single_env->segs[R_CS].base);
1133 }
1134 #endif
1135 p = page_find(start >> TARGET_PAGE_BITS);
1136 if (!p) {
1137 return;
1138 }
1139 if (p->code_bitmap) {
1140 offset = start & ~TARGET_PAGE_MASK;
1141 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1142 if (b & ((1 << len) - 1)) {
1143 goto do_invalidate;
1144 }
1145 } else {
1146 do_invalidate:
1147 tb_invalidate_phys_page_range(start, start + len, 1);
1148 }
1149 }
1150
1151 #if !defined(CONFIG_SOFTMMU)
1152 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1153 uintptr_t pc, void *puc,
1154 bool locked)
1155 {
1156 TranslationBlock *tb;
1157 PageDesc *p;
1158 int n;
1159 #ifdef TARGET_HAS_PRECISE_SMC
1160 TranslationBlock *current_tb = NULL;
1161 CPUState *cpu = current_cpu;
1162 CPUArchState *env = NULL;
1163 int current_tb_modified = 0;
1164 target_ulong current_pc = 0;
1165 target_ulong current_cs_base = 0;
1166 int current_flags = 0;
1167 #endif
1168
1169 addr &= TARGET_PAGE_MASK;
1170 p = page_find(addr >> TARGET_PAGE_BITS);
1171 if (!p) {
1172 return;
1173 }
1174 tb = p->first_tb;
1175 #ifdef TARGET_HAS_PRECISE_SMC
1176 if (tb && pc != 0) {
1177 current_tb = tb_find_pc(pc);
1178 }
1179 if (cpu != NULL) {
1180 env = cpu->env_ptr;
1181 }
1182 #endif
1183 while (tb != NULL) {
1184 n = (uintptr_t)tb & 3;
1185 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1186 #ifdef TARGET_HAS_PRECISE_SMC
1187 if (current_tb == tb &&
1188 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1189 /* If we are modifying the current TB, we must stop
1190 its execution. We could be more precise by checking
1191 that the modification is after the current PC, but it
1192 would require a specialized function to partially
1193 restore the CPU state */
1194
1195 current_tb_modified = 1;
1196 cpu_restore_state_from_tb(cpu, current_tb, pc);
1197 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1198 &current_flags);
1199 }
1200 #endif /* TARGET_HAS_PRECISE_SMC */
1201 tb_phys_invalidate(tb, addr);
1202 tb = tb->page_next[n];
1203 }
1204 p->first_tb = NULL;
1205 #ifdef TARGET_HAS_PRECISE_SMC
1206 if (current_tb_modified) {
1207 /* we generate a block containing just the instruction
1208 modifying the memory. It will ensure that it cannot modify
1209 itself */
1210 cpu->current_tb = NULL;
1211 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1212 if (locked) {
1213 mmap_unlock();
1214 }
1215 cpu_resume_from_signal(env, puc);
1216 }
1217 #endif
1218 }
1219 #endif
1220
1221 /* add the tb in the target page and protect it if necessary */
1222 static inline void tb_alloc_page(TranslationBlock *tb,
1223 unsigned int n, tb_page_addr_t page_addr)
1224 {
1225 PageDesc *p;
1226 #ifndef CONFIG_USER_ONLY
1227 bool page_already_protected;
1228 #endif
1229
1230 tb->page_addr[n] = page_addr;
1231 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1232 tb->page_next[n] = p->first_tb;
1233 #ifndef CONFIG_USER_ONLY
1234 page_already_protected = p->first_tb != NULL;
1235 #endif
1236 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1237 invalidate_page_bitmap(p);
1238
1239 #if defined(TARGET_HAS_SMC) || 1
1240
1241 #if defined(CONFIG_USER_ONLY)
1242 if (p->flags & PAGE_WRITE) {
1243 target_ulong addr;
1244 PageDesc *p2;
1245 int prot;
1246
1247 /* force the host page as non writable (writes will have a
1248 page fault + mprotect overhead) */
1249 page_addr &= qemu_host_page_mask;
1250 prot = 0;
1251 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1252 addr += TARGET_PAGE_SIZE) {
1253
1254 p2 = page_find(addr >> TARGET_PAGE_BITS);
1255 if (!p2) {
1256 continue;
1257 }
1258 prot |= p2->flags;
1259 p2->flags &= ~PAGE_WRITE;
1260 }
1261 mprotect(g2h(page_addr), qemu_host_page_size,
1262 (prot & PAGE_BITS) & ~PAGE_WRITE);
1263 #ifdef DEBUG_TB_INVALIDATE
1264 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1265 page_addr);
1266 #endif
1267 }
1268 #else
1269 /* if some code is already present, then the pages are already
1270 protected. So we handle the case where only the first TB is
1271 allocated in a physical page */
1272 if (!page_already_protected) {
1273 tlb_protect_code(page_addr);
1274 }
1275 #endif
1276
1277 #endif /* TARGET_HAS_SMC */
1278 }
1279
1280 /* add a new TB and link it to the physical page tables. phys_page2 is
1281 (-1) to indicate that only one page contains the TB. */
1282 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1283 tb_page_addr_t phys_page2)
1284 {
1285 unsigned int h;
1286 TranslationBlock **ptb;
1287
1288 /* Grab the mmap lock to stop another thread invalidating this TB
1289 before we are done. */
1290 mmap_lock();
1291 /* add in the physical hash table */
1292 h = tb_phys_hash_func(phys_pc);
1293 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1294 tb->phys_hash_next = *ptb;
1295 *ptb = tb;
1296
1297 /* add in the page list */
1298 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1299 if (phys_page2 != -1) {
1300 tb_alloc_page(tb, 1, phys_page2);
1301 } else {
1302 tb->page_addr[1] = -1;
1303 }
1304
1305 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1306 tb->jmp_next[0] = NULL;
1307 tb->jmp_next[1] = NULL;
1308
1309 /* init original jump addresses */
1310 if (tb->tb_next_offset[0] != 0xffff) {
1311 tb_reset_jump(tb, 0);
1312 }
1313 if (tb->tb_next_offset[1] != 0xffff) {
1314 tb_reset_jump(tb, 1);
1315 }
1316
1317 #ifdef DEBUG_TB_CHECK
1318 tb_page_check();
1319 #endif
1320 mmap_unlock();
1321 }
1322
1323 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1324 tb[1].tc_ptr. Return NULL if not found */
1325 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1326 {
1327 int m_min, m_max, m;
1328 uintptr_t v;
1329 TranslationBlock *tb;
1330
1331 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1332 return NULL;
1333 }
1334 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1335 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1336 return NULL;
1337 }
1338 /* binary search (cf Knuth) */
1339 m_min = 0;
1340 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1341 while (m_min <= m_max) {
1342 m = (m_min + m_max) >> 1;
1343 tb = &tcg_ctx.tb_ctx.tbs[m];
1344 v = (uintptr_t)tb->tc_ptr;
1345 if (v == tc_ptr) {
1346 return tb;
1347 } else if (tc_ptr < v) {
1348 m_max = m - 1;
1349 } else {
1350 m_min = m + 1;
1351 }
1352 }
1353 return &tcg_ctx.tb_ctx.tbs[m_max];
1354 }
1355
1356 #if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
1357 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1358 {
1359 ram_addr_t ram_addr;
1360 MemoryRegion *mr;
1361 hwaddr l = 1;
1362
1363 mr = address_space_translate(as, addr, &addr, &l, false);
1364 if (!(memory_region_is_ram(mr)
1365 || memory_region_is_romd(mr))) {
1366 return;
1367 }
1368 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
1369 + addr;
1370 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1371 }
1372 #endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
1373
1374 void tb_check_watchpoint(CPUArchState *env)
1375 {
1376 CPUState *cpu = ENV_GET_CPU(env);
1377 TranslationBlock *tb;
1378
1379 tb = tb_find_pc(cpu->mem_io_pc);
1380 if (!tb) {
1381 cpu_abort(env, "check_watchpoint: could not find TB for pc=%p",
1382 (void *)cpu->mem_io_pc);
1383 }
1384 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
1385 tb_phys_invalidate(tb, -1);
1386 }
1387
1388 #ifndef CONFIG_USER_ONLY
1389 /* mask must never be zero, except for A20 change call */
1390 static void tcg_handle_interrupt(CPUState *cpu, int mask)
1391 {
1392 CPUArchState *env = cpu->env_ptr;
1393 int old_mask;
1394
1395 old_mask = cpu->interrupt_request;
1396 cpu->interrupt_request |= mask;
1397
1398 /*
1399 * If called from iothread context, wake the target cpu in
1400 * case its halted.
1401 */
1402 if (!qemu_cpu_is_self(cpu)) {
1403 qemu_cpu_kick(cpu);
1404 return;
1405 }
1406
1407 if (use_icount) {
1408 cpu->icount_decr.u16.high = 0xffff;
1409 if (!cpu_can_do_io(cpu)
1410 && (mask & ~old_mask) != 0) {
1411 cpu_abort(env, "Raised interrupt while not in I/O function");
1412 }
1413 } else {
1414 cpu->tcg_exit_req = 1;
1415 }
1416 }
1417
1418 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1419
1420 /* in deterministic execution mode, instructions doing device I/Os
1421 must be at the end of the TB */
1422 void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
1423 {
1424 CPUState *cpu = ENV_GET_CPU(env);
1425 TranslationBlock *tb;
1426 uint32_t n, cflags;
1427 target_ulong pc, cs_base;
1428 uint64_t flags;
1429
1430 tb = tb_find_pc(retaddr);
1431 if (!tb) {
1432 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
1433 (void *)retaddr);
1434 }
1435 n = cpu->icount_decr.u16.low + tb->icount;
1436 cpu_restore_state_from_tb(cpu, tb, retaddr);
1437 /* Calculate how many instructions had been executed before the fault
1438 occurred. */
1439 n = n - cpu->icount_decr.u16.low;
1440 /* Generate a new TB ending on the I/O insn. */
1441 n++;
1442 /* On MIPS and SH, delay slot instructions can only be restarted if
1443 they were already the first instruction in the TB. If this is not
1444 the first instruction in a TB then re-execute the preceding
1445 branch. */
1446 #if defined(TARGET_MIPS)
1447 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1448 env->active_tc.PC -= 4;
1449 cpu->icount_decr.u16.low++;
1450 env->hflags &= ~MIPS_HFLAG_BMASK;
1451 }
1452 #elif defined(TARGET_SH4)
1453 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1454 && n > 1) {
1455 env->pc -= 2;
1456 cpu->icount_decr.u16.low++;
1457 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1458 }
1459 #endif
1460 /* This should never happen. */
1461 if (n > CF_COUNT_MASK) {
1462 cpu_abort(env, "TB too big during recompile");
1463 }
1464
1465 cflags = n | CF_LAST_IO;
1466 pc = tb->pc;
1467 cs_base = tb->cs_base;
1468 flags = tb->flags;
1469 tb_phys_invalidate(tb, -1);
1470 /* FIXME: In theory this could raise an exception. In practice
1471 we have already translated the block once so it's probably ok. */
1472 tb_gen_code(env, pc, cs_base, flags, cflags);
1473 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1474 the first in the TB) then we end up generating a whole new TB and
1475 repeating the fault, which is horribly inefficient.
1476 Better would be to execute just this insn uncached, or generate a
1477 second new TB. */
1478 cpu_resume_from_signal(env, NULL);
1479 }
1480
1481 void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
1482 {
1483 CPUState *cpu = ENV_GET_CPU(env);
1484 unsigned int i;
1485
1486 /* Discard jump cache entries for any tb which might potentially
1487 overlap the flushed page. */
1488 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1489 memset(&cpu->tb_jmp_cache[i], 0,
1490 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1491
1492 i = tb_jmp_cache_hash_page(addr);
1493 memset(&cpu->tb_jmp_cache[i], 0,
1494 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1495 }
1496
1497 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1498 {
1499 int i, target_code_size, max_target_code_size;
1500 int direct_jmp_count, direct_jmp2_count, cross_page;
1501 TranslationBlock *tb;
1502
1503 target_code_size = 0;
1504 max_target_code_size = 0;
1505 cross_page = 0;
1506 direct_jmp_count = 0;
1507 direct_jmp2_count = 0;
1508 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1509 tb = &tcg_ctx.tb_ctx.tbs[i];
1510 target_code_size += tb->size;
1511 if (tb->size > max_target_code_size) {
1512 max_target_code_size = tb->size;
1513 }
1514 if (tb->page_addr[1] != -1) {
1515 cross_page++;
1516 }
1517 if (tb->tb_next_offset[0] != 0xffff) {
1518 direct_jmp_count++;
1519 if (tb->tb_next_offset[1] != 0xffff) {
1520 direct_jmp2_count++;
1521 }
1522 }
1523 }
1524 /* XXX: avoid using doubles ? */
1525 cpu_fprintf(f, "Translation buffer state:\n");
1526 cpu_fprintf(f, "gen code size %td/%zd\n",
1527 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1528 tcg_ctx.code_gen_buffer_max_size);
1529 cpu_fprintf(f, "TB count %d/%d\n",
1530 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1531 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1532 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1533 tcg_ctx.tb_ctx.nb_tbs : 0,
1534 max_target_code_size);
1535 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1536 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1537 tcg_ctx.code_gen_buffer) /
1538 tcg_ctx.tb_ctx.nb_tbs : 0,
1539 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1540 tcg_ctx.code_gen_buffer) /
1541 target_code_size : 0);
1542 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1543 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1544 tcg_ctx.tb_ctx.nb_tbs : 0);
1545 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1546 direct_jmp_count,
1547 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1548 tcg_ctx.tb_ctx.nb_tbs : 0,
1549 direct_jmp2_count,
1550 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1551 tcg_ctx.tb_ctx.nb_tbs : 0);
1552 cpu_fprintf(f, "\nStatistics:\n");
1553 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1554 cpu_fprintf(f, "TB invalidate count %d\n",
1555 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1556 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1557 tcg_dump_info(f, cpu_fprintf);
1558 }
1559
1560 #else /* CONFIG_USER_ONLY */
1561
1562 void cpu_interrupt(CPUState *cpu, int mask)
1563 {
1564 cpu->interrupt_request |= mask;
1565 cpu->tcg_exit_req = 1;
1566 }
1567
1568 /*
1569 * Walks guest process memory "regions" one by one
1570 * and calls callback function 'fn' for each region.
1571 */
1572 struct walk_memory_regions_data {
1573 walk_memory_regions_fn fn;
1574 void *priv;
1575 uintptr_t start;
1576 int prot;
1577 };
1578
1579 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1580 abi_ulong end, int new_prot)
1581 {
1582 if (data->start != -1ul) {
1583 int rc = data->fn(data->priv, data->start, end, data->prot);
1584 if (rc != 0) {
1585 return rc;
1586 }
1587 }
1588
1589 data->start = (new_prot ? end : -1ul);
1590 data->prot = new_prot;
1591
1592 return 0;
1593 }
1594
1595 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1596 abi_ulong base, int level, void **lp)
1597 {
1598 abi_ulong pa;
1599 int i, rc;
1600
1601 if (*lp == NULL) {
1602 return walk_memory_regions_end(data, base, 0);
1603 }
1604
1605 if (level == 0) {
1606 PageDesc *pd = *lp;
1607
1608 for (i = 0; i < V_L2_SIZE; ++i) {
1609 int prot = pd[i].flags;
1610
1611 pa = base | (i << TARGET_PAGE_BITS);
1612 if (prot != data->prot) {
1613 rc = walk_memory_regions_end(data, pa, prot);
1614 if (rc != 0) {
1615 return rc;
1616 }
1617 }
1618 }
1619 } else {
1620 void **pp = *lp;
1621
1622 for (i = 0; i < V_L2_SIZE; ++i) {
1623 pa = base | ((abi_ulong)i <<
1624 (TARGET_PAGE_BITS + V_L2_BITS * level));
1625 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1626 if (rc != 0) {
1627 return rc;
1628 }
1629 }
1630 }
1631
1632 return 0;
1633 }
1634
1635 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1636 {
1637 struct walk_memory_regions_data data;
1638 uintptr_t i;
1639
1640 data.fn = fn;
1641 data.priv = priv;
1642 data.start = -1ul;
1643 data.prot = 0;
1644
1645 for (i = 0; i < V_L1_SIZE; i++) {
1646 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
1647 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1648
1649 if (rc != 0) {
1650 return rc;
1651 }
1652 }
1653
1654 return walk_memory_regions_end(&data, 0, 0);
1655 }
1656
1657 static int dump_region(void *priv, abi_ulong start,
1658 abi_ulong end, unsigned long prot)
1659 {
1660 FILE *f = (FILE *)priv;
1661
1662 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
1663 " "TARGET_ABI_FMT_lx" %c%c%c\n",
1664 start, end, end - start,
1665 ((prot & PAGE_READ) ? 'r' : '-'),
1666 ((prot & PAGE_WRITE) ? 'w' : '-'),
1667 ((prot & PAGE_EXEC) ? 'x' : '-'));
1668
1669 return 0;
1670 }
1671
1672 /* dump memory mappings */
1673 void page_dump(FILE *f)
1674 {
1675 const int length = sizeof(abi_ulong) * 2;
1676 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1677 length, "start", length, "end", length, "size", "prot");
1678 walk_memory_regions(f, dump_region);
1679 }
1680
1681 int page_get_flags(target_ulong address)
1682 {
1683 PageDesc *p;
1684
1685 p = page_find(address >> TARGET_PAGE_BITS);
1686 if (!p) {
1687 return 0;
1688 }
1689 return p->flags;
1690 }
1691
1692 /* Modify the flags of a page and invalidate the code if necessary.
1693 The flag PAGE_WRITE_ORG is positioned automatically depending
1694 on PAGE_WRITE. The mmap_lock should already be held. */
1695 void page_set_flags(target_ulong start, target_ulong end, int flags)
1696 {
1697 target_ulong addr, len;
1698
1699 /* This function should never be called with addresses outside the
1700 guest address space. If this assert fires, it probably indicates
1701 a missing call to h2g_valid. */
1702 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1703 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1704 #endif
1705 assert(start < end);
1706
1707 start = start & TARGET_PAGE_MASK;
1708 end = TARGET_PAGE_ALIGN(end);
1709
1710 if (flags & PAGE_WRITE) {
1711 flags |= PAGE_WRITE_ORG;
1712 }
1713
1714 for (addr = start, len = end - start;
1715 len != 0;
1716 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1717 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1718
1719 /* If the write protection bit is set, then we invalidate
1720 the code inside. */
1721 if (!(p->flags & PAGE_WRITE) &&
1722 (flags & PAGE_WRITE) &&
1723 p->first_tb) {
1724 tb_invalidate_phys_page(addr, 0, NULL, false);
1725 }
1726 p->flags = flags;
1727 }
1728 }
1729
1730 int page_check_range(target_ulong start, target_ulong len, int flags)
1731 {
1732 PageDesc *p;
1733 target_ulong end;
1734 target_ulong addr;
1735
1736 /* This function should never be called with addresses outside the
1737 guest address space. If this assert fires, it probably indicates
1738 a missing call to h2g_valid. */
1739 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1740 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1741 #endif
1742
1743 if (len == 0) {
1744 return 0;
1745 }
1746 if (start + len - 1 < start) {
1747 /* We've wrapped around. */
1748 return -1;
1749 }
1750
1751 /* must do before we loose bits in the next step */
1752 end = TARGET_PAGE_ALIGN(start + len);
1753 start = start & TARGET_PAGE_MASK;
1754
1755 for (addr = start, len = end - start;
1756 len != 0;
1757 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1758 p = page_find(addr >> TARGET_PAGE_BITS);
1759 if (!p) {
1760 return -1;
1761 }
1762 if (!(p->flags & PAGE_VALID)) {
1763 return -1;
1764 }
1765
1766 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1767 return -1;
1768 }
1769 if (flags & PAGE_WRITE) {
1770 if (!(p->flags & PAGE_WRITE_ORG)) {
1771 return -1;
1772 }
1773 /* unprotect the page if it was put read-only because it
1774 contains translated code */
1775 if (!(p->flags & PAGE_WRITE)) {
1776 if (!page_unprotect(addr, 0, NULL)) {
1777 return -1;
1778 }
1779 }
1780 return 0;
1781 }
1782 }
1783 return 0;
1784 }
1785
1786 /* called from signal handler: invalidate the code and unprotect the
1787 page. Return TRUE if the fault was successfully handled. */
1788 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1789 {
1790 unsigned int prot;
1791 PageDesc *p;
1792 target_ulong host_start, host_end, addr;
1793
1794 /* Technically this isn't safe inside a signal handler. However we
1795 know this only ever happens in a synchronous SEGV handler, so in
1796 practice it seems to be ok. */
1797 mmap_lock();
1798
1799 p = page_find(address >> TARGET_PAGE_BITS);
1800 if (!p) {
1801 mmap_unlock();
1802 return 0;
1803 }
1804
1805 /* if the page was really writable, then we change its
1806 protection back to writable */
1807 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1808 host_start = address & qemu_host_page_mask;
1809 host_end = host_start + qemu_host_page_size;
1810
1811 prot = 0;
1812 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1813 p = page_find(addr >> TARGET_PAGE_BITS);
1814 p->flags |= PAGE_WRITE;
1815 prot |= p->flags;
1816
1817 /* and since the content will be modified, we must invalidate
1818 the corresponding translated code. */
1819 tb_invalidate_phys_page(addr, pc, puc, true);
1820 #ifdef DEBUG_TB_CHECK
1821 tb_invalidate_check(addr);
1822 #endif
1823 }
1824 mprotect((void *)g2h(host_start), qemu_host_page_size,
1825 prot & PAGE_BITS);
1826
1827 mmap_unlock();
1828 return 1;
1829 }
1830 mmap_unlock();
1831 return 0;
1832 }
1833 #endif /* CONFIG_USER_ONLY */