cpu: Move can_do_io field from CPU_COMMON to CPUState
[qemu.git] / translate-all.c
1 /*
2 * Host code generation
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/types.h>
23 #include <sys/mman.h>
24 #endif
25 #include <stdarg.h>
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <inttypes.h>
30
31 #include "config.h"
32
33 #include "qemu-common.h"
34 #define NO_CPU_IO_DEFS
35 #include "cpu.h"
36 #include "disas/disas.h"
37 #include "tcg.h"
38 #if defined(CONFIG_USER_ONLY)
39 #include "qemu.h"
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
45 #include <sys/time.h>
46 #include <sys/proc.h>
47 #include <machine/profile.h>
48 #define _KERNEL
49 #include <sys/user.h>
50 #undef _KERNEL
51 #undef sigqueue
52 #include <libutil.h>
53 #endif
54 #endif
55 #else
56 #include "exec/address-spaces.h"
57 #endif
58
59 #include "exec/cputlb.h"
60 #include "translate-all.h"
61 #include "qemu/timer.h"
62
63 //#define DEBUG_TB_INVALIDATE
64 //#define DEBUG_FLUSH
65 /* make various TB consistency checks */
66 //#define DEBUG_TB_CHECK
67
68 #if !defined(CONFIG_USER_ONLY)
69 /* TB consistency checks only implemented for usermode emulation. */
70 #undef DEBUG_TB_CHECK
71 #endif
72
73 #define SMC_BITMAP_USE_THRESHOLD 10
74
75 typedef struct PageDesc {
76 /* list of TBs intersecting this ram page */
77 TranslationBlock *first_tb;
78 /* in order to optimize self modifying code, we count the number
79 of lookups we do to a given page to use a bitmap */
80 unsigned int code_write_count;
81 uint8_t *code_bitmap;
82 #if defined(CONFIG_USER_ONLY)
83 unsigned long flags;
84 #endif
85 } PageDesc;
86
87 /* In system mode we want L1_MAP to be based on ram offsets,
88 while in user mode we want it to be based on virtual addresses. */
89 #if !defined(CONFIG_USER_ONLY)
90 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
91 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
92 #else
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
94 #endif
95 #else
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
97 #endif
98
99 /* Size of the L2 (and L3, etc) page tables. */
100 #define V_L2_BITS 10
101 #define V_L2_SIZE (1 << V_L2_BITS)
102
103 /* The bits remaining after N lower levels of page tables. */
104 #define V_L1_BITS_REM \
105 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
106
107 #if V_L1_BITS_REM < 4
108 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
109 #else
110 #define V_L1_BITS V_L1_BITS_REM
111 #endif
112
113 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
114
115 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
116
117 uintptr_t qemu_real_host_page_size;
118 uintptr_t qemu_host_page_size;
119 uintptr_t qemu_host_page_mask;
120
121 /* This is a multi-level map on the virtual address space.
122 The bottom level has pointers to PageDesc. */
123 static void *l1_map[V_L1_SIZE];
124
125 /* code generation context */
126 TCGContext tcg_ctx;
127
128 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
129 tb_page_addr_t phys_page2);
130 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
131
132 void cpu_gen_init(void)
133 {
134 tcg_context_init(&tcg_ctx);
135 }
136
137 /* return non zero if the very first instruction is invalid so that
138 the virtual CPU can trigger an exception.
139
140 '*gen_code_size_ptr' contains the size of the generated code (host
141 code).
142 */
143 int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
144 {
145 TCGContext *s = &tcg_ctx;
146 uint8_t *gen_code_buf;
147 int gen_code_size;
148 #ifdef CONFIG_PROFILER
149 int64_t ti;
150 #endif
151
152 #ifdef CONFIG_PROFILER
153 s->tb_count1++; /* includes aborted translations because of
154 exceptions */
155 ti = profile_getclock();
156 #endif
157 tcg_func_start(s);
158
159 gen_intermediate_code(env, tb);
160
161 /* generate machine code */
162 gen_code_buf = tb->tc_ptr;
163 tb->tb_next_offset[0] = 0xffff;
164 tb->tb_next_offset[1] = 0xffff;
165 s->tb_next_offset = tb->tb_next_offset;
166 #ifdef USE_DIRECT_JUMP
167 s->tb_jmp_offset = tb->tb_jmp_offset;
168 s->tb_next = NULL;
169 #else
170 s->tb_jmp_offset = NULL;
171 s->tb_next = tb->tb_next;
172 #endif
173
174 #ifdef CONFIG_PROFILER
175 s->tb_count++;
176 s->interm_time += profile_getclock() - ti;
177 s->code_time -= profile_getclock();
178 #endif
179 gen_code_size = tcg_gen_code(s, gen_code_buf);
180 *gen_code_size_ptr = gen_code_size;
181 #ifdef CONFIG_PROFILER
182 s->code_time += profile_getclock();
183 s->code_in_len += tb->size;
184 s->code_out_len += gen_code_size;
185 #endif
186
187 #ifdef DEBUG_DISAS
188 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
189 qemu_log("OUT: [size=%d]\n", *gen_code_size_ptr);
190 log_disas(tb->tc_ptr, *gen_code_size_ptr);
191 qemu_log("\n");
192 qemu_log_flush();
193 }
194 #endif
195 return 0;
196 }
197
198 /* The cpu state corresponding to 'searched_pc' is restored.
199 */
200 static int cpu_restore_state_from_tb(TranslationBlock *tb, CPUArchState *env,
201 uintptr_t searched_pc)
202 {
203 CPUState *cpu = ENV_GET_CPU(env);
204 TCGContext *s = &tcg_ctx;
205 int j;
206 uintptr_t tc_ptr;
207 #ifdef CONFIG_PROFILER
208 int64_t ti;
209 #endif
210
211 #ifdef CONFIG_PROFILER
212 ti = profile_getclock();
213 #endif
214 tcg_func_start(s);
215
216 gen_intermediate_code_pc(env, tb);
217
218 if (use_icount) {
219 /* Reset the cycle counter to the start of the block. */
220 env->icount_decr.u16.low += tb->icount;
221 /* Clear the IO flag. */
222 cpu->can_do_io = 0;
223 }
224
225 /* find opc index corresponding to search_pc */
226 tc_ptr = (uintptr_t)tb->tc_ptr;
227 if (searched_pc < tc_ptr)
228 return -1;
229
230 s->tb_next_offset = tb->tb_next_offset;
231 #ifdef USE_DIRECT_JUMP
232 s->tb_jmp_offset = tb->tb_jmp_offset;
233 s->tb_next = NULL;
234 #else
235 s->tb_jmp_offset = NULL;
236 s->tb_next = tb->tb_next;
237 #endif
238 j = tcg_gen_code_search_pc(s, (uint8_t *)tc_ptr, searched_pc - tc_ptr);
239 if (j < 0)
240 return -1;
241 /* now find start of instruction before */
242 while (s->gen_opc_instr_start[j] == 0) {
243 j--;
244 }
245 env->icount_decr.u16.low -= s->gen_opc_icount[j];
246
247 restore_state_to_opc(env, tb, j);
248
249 #ifdef CONFIG_PROFILER
250 s->restore_time += profile_getclock() - ti;
251 s->restore_count++;
252 #endif
253 return 0;
254 }
255
256 bool cpu_restore_state(CPUArchState *env, uintptr_t retaddr)
257 {
258 TranslationBlock *tb;
259
260 tb = tb_find_pc(retaddr);
261 if (tb) {
262 cpu_restore_state_from_tb(tb, env, retaddr);
263 return true;
264 }
265 return false;
266 }
267
268 #ifdef _WIN32
269 static inline void map_exec(void *addr, long size)
270 {
271 DWORD old_protect;
272 VirtualProtect(addr, size,
273 PAGE_EXECUTE_READWRITE, &old_protect);
274 }
275 #else
276 static inline void map_exec(void *addr, long size)
277 {
278 unsigned long start, end, page_size;
279
280 page_size = getpagesize();
281 start = (unsigned long)addr;
282 start &= ~(page_size - 1);
283
284 end = (unsigned long)addr + size;
285 end += page_size - 1;
286 end &= ~(page_size - 1);
287
288 mprotect((void *)start, end - start,
289 PROT_READ | PROT_WRITE | PROT_EXEC);
290 }
291 #endif
292
293 void page_size_init(void)
294 {
295 /* NOTE: we can always suppose that qemu_host_page_size >=
296 TARGET_PAGE_SIZE */
297 #ifdef _WIN32
298 SYSTEM_INFO system_info;
299
300 GetSystemInfo(&system_info);
301 qemu_real_host_page_size = system_info.dwPageSize;
302 #else
303 qemu_real_host_page_size = getpagesize();
304 #endif
305 if (qemu_host_page_size == 0) {
306 qemu_host_page_size = qemu_real_host_page_size;
307 }
308 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
309 qemu_host_page_size = TARGET_PAGE_SIZE;
310 }
311 qemu_host_page_mask = ~(qemu_host_page_size - 1);
312 }
313
314 static void page_init(void)
315 {
316 page_size_init();
317 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
318 {
319 #ifdef HAVE_KINFO_GETVMMAP
320 struct kinfo_vmentry *freep;
321 int i, cnt;
322
323 freep = kinfo_getvmmap(getpid(), &cnt);
324 if (freep) {
325 mmap_lock();
326 for (i = 0; i < cnt; i++) {
327 unsigned long startaddr, endaddr;
328
329 startaddr = freep[i].kve_start;
330 endaddr = freep[i].kve_end;
331 if (h2g_valid(startaddr)) {
332 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
333
334 if (h2g_valid(endaddr)) {
335 endaddr = h2g(endaddr);
336 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
337 } else {
338 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
339 endaddr = ~0ul;
340 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
341 #endif
342 }
343 }
344 }
345 free(freep);
346 mmap_unlock();
347 }
348 #else
349 FILE *f;
350
351 last_brk = (unsigned long)sbrk(0);
352
353 f = fopen("/compat/linux/proc/self/maps", "r");
354 if (f) {
355 mmap_lock();
356
357 do {
358 unsigned long startaddr, endaddr;
359 int n;
360
361 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
362
363 if (n == 2 && h2g_valid(startaddr)) {
364 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
365
366 if (h2g_valid(endaddr)) {
367 endaddr = h2g(endaddr);
368 } else {
369 endaddr = ~0ul;
370 }
371 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
372 }
373 } while (!feof(f));
374
375 fclose(f);
376 mmap_unlock();
377 }
378 #endif
379 }
380 #endif
381 }
382
383 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
384 {
385 PageDesc *pd;
386 void **lp;
387 int i;
388
389 #if defined(CONFIG_USER_ONLY)
390 /* We can't use g_malloc because it may recurse into a locked mutex. */
391 # define ALLOC(P, SIZE) \
392 do { \
393 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
394 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
395 } while (0)
396 #else
397 # define ALLOC(P, SIZE) \
398 do { P = g_malloc0(SIZE); } while (0)
399 #endif
400
401 /* Level 1. Always allocated. */
402 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
403
404 /* Level 2..N-1. */
405 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
406 void **p = *lp;
407
408 if (p == NULL) {
409 if (!alloc) {
410 return NULL;
411 }
412 ALLOC(p, sizeof(void *) * V_L2_SIZE);
413 *lp = p;
414 }
415
416 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
417 }
418
419 pd = *lp;
420 if (pd == NULL) {
421 if (!alloc) {
422 return NULL;
423 }
424 ALLOC(pd, sizeof(PageDesc) * V_L2_SIZE);
425 *lp = pd;
426 }
427
428 #undef ALLOC
429
430 return pd + (index & (V_L2_SIZE - 1));
431 }
432
433 static inline PageDesc *page_find(tb_page_addr_t index)
434 {
435 return page_find_alloc(index, 0);
436 }
437
438 #if !defined(CONFIG_USER_ONLY)
439 #define mmap_lock() do { } while (0)
440 #define mmap_unlock() do { } while (0)
441 #endif
442
443 #if defined(CONFIG_USER_ONLY)
444 /* Currently it is not recommended to allocate big chunks of data in
445 user mode. It will change when a dedicated libc will be used. */
446 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
447 region in which the guest needs to run. Revisit this. */
448 #define USE_STATIC_CODE_GEN_BUFFER
449 #endif
450
451 /* ??? Should configure for this, not list operating systems here. */
452 #if (defined(__linux__) \
453 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
454 || defined(__DragonFly__) || defined(__OpenBSD__) \
455 || defined(__NetBSD__))
456 # define USE_MMAP
457 #endif
458
459 /* Minimum size of the code gen buffer. This number is randomly chosen,
460 but not so small that we can't have a fair number of TB's live. */
461 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
462
463 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
464 indicated, this is constrained by the range of direct branches on the
465 host cpu, as used by the TCG implementation of goto_tb. */
466 #if defined(__x86_64__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
468 #elif defined(__sparc__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
470 #elif defined(__aarch64__)
471 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
472 #elif defined(__arm__)
473 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
474 #elif defined(__s390x__)
475 /* We have a +- 4GB range on the branches; leave some slop. */
476 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
477 #else
478 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
479 #endif
480
481 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
482
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
484 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
485 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
486
487 static inline size_t size_code_gen_buffer(size_t tb_size)
488 {
489 /* Size the buffer. */
490 if (tb_size == 0) {
491 #ifdef USE_STATIC_CODE_GEN_BUFFER
492 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
493 #else
494 /* ??? Needs adjustments. */
495 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
496 static buffer, we could size this on RESERVED_VA, on the text
497 segment size of the executable, or continue to use the default. */
498 tb_size = (unsigned long)(ram_size / 4);
499 #endif
500 }
501 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
502 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
503 }
504 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
505 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
506 }
507 tcg_ctx.code_gen_buffer_size = tb_size;
508 return tb_size;
509 }
510
511 #ifdef USE_STATIC_CODE_GEN_BUFFER
512 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
513 __attribute__((aligned(CODE_GEN_ALIGN)));
514
515 static inline void *alloc_code_gen_buffer(void)
516 {
517 map_exec(static_code_gen_buffer, tcg_ctx.code_gen_buffer_size);
518 return static_code_gen_buffer;
519 }
520 #elif defined(USE_MMAP)
521 static inline void *alloc_code_gen_buffer(void)
522 {
523 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
524 uintptr_t start = 0;
525 void *buf;
526
527 /* Constrain the position of the buffer based on the host cpu.
528 Note that these addresses are chosen in concert with the
529 addresses assigned in the relevant linker script file. */
530 # if defined(__PIE__) || defined(__PIC__)
531 /* Don't bother setting a preferred location if we're building
532 a position-independent executable. We're more likely to get
533 an address near the main executable if we let the kernel
534 choose the address. */
535 # elif defined(__x86_64__) && defined(MAP_32BIT)
536 /* Force the memory down into low memory with the executable.
537 Leave the choice of exact location with the kernel. */
538 flags |= MAP_32BIT;
539 /* Cannot expect to map more than 800MB in low memory. */
540 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
541 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
542 }
543 # elif defined(__sparc__)
544 start = 0x40000000ul;
545 # elif defined(__s390x__)
546 start = 0x90000000ul;
547 # endif
548
549 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
550 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
551 return buf == MAP_FAILED ? NULL : buf;
552 }
553 #else
554 static inline void *alloc_code_gen_buffer(void)
555 {
556 void *buf = g_malloc(tcg_ctx.code_gen_buffer_size);
557
558 if (buf) {
559 map_exec(buf, tcg_ctx.code_gen_buffer_size);
560 }
561 return buf;
562 }
563 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
564
565 static inline void code_gen_alloc(size_t tb_size)
566 {
567 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
568 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
569 if (tcg_ctx.code_gen_buffer == NULL) {
570 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
571 exit(1);
572 }
573
574 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
575 QEMU_MADV_HUGEPAGE);
576
577 /* Steal room for the prologue at the end of the buffer. This ensures
578 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
579 from TB's to the prologue are going to be in range. It also means
580 that we don't need to mark (additional) portions of the data segment
581 as executable. */
582 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
583 tcg_ctx.code_gen_buffer_size - 1024;
584 tcg_ctx.code_gen_buffer_size -= 1024;
585
586 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
587 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
588 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
589 CODE_GEN_AVG_BLOCK_SIZE;
590 tcg_ctx.tb_ctx.tbs =
591 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
592 }
593
594 /* Must be called before using the QEMU cpus. 'tb_size' is the size
595 (in bytes) allocated to the translation buffer. Zero means default
596 size. */
597 void tcg_exec_init(unsigned long tb_size)
598 {
599 cpu_gen_init();
600 code_gen_alloc(tb_size);
601 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
602 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
603 page_init();
604 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
605 /* There's no guest base to take into account, so go ahead and
606 initialize the prologue now. */
607 tcg_prologue_init(&tcg_ctx);
608 #endif
609 }
610
611 bool tcg_enabled(void)
612 {
613 return tcg_ctx.code_gen_buffer != NULL;
614 }
615
616 /* Allocate a new translation block. Flush the translation buffer if
617 too many translation blocks or too much generated code. */
618 static TranslationBlock *tb_alloc(target_ulong pc)
619 {
620 TranslationBlock *tb;
621
622 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
623 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
624 tcg_ctx.code_gen_buffer_max_size) {
625 return NULL;
626 }
627 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
628 tb->pc = pc;
629 tb->cflags = 0;
630 return tb;
631 }
632
633 void tb_free(TranslationBlock *tb)
634 {
635 /* In practice this is mostly used for single use temporary TB
636 Ignore the hard cases and just back up if this TB happens to
637 be the last one generated. */
638 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
639 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
640 tcg_ctx.code_gen_ptr = tb->tc_ptr;
641 tcg_ctx.tb_ctx.nb_tbs--;
642 }
643 }
644
645 static inline void invalidate_page_bitmap(PageDesc *p)
646 {
647 if (p->code_bitmap) {
648 g_free(p->code_bitmap);
649 p->code_bitmap = NULL;
650 }
651 p->code_write_count = 0;
652 }
653
654 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
655 static void page_flush_tb_1(int level, void **lp)
656 {
657 int i;
658
659 if (*lp == NULL) {
660 return;
661 }
662 if (level == 0) {
663 PageDesc *pd = *lp;
664
665 for (i = 0; i < V_L2_SIZE; ++i) {
666 pd[i].first_tb = NULL;
667 invalidate_page_bitmap(pd + i);
668 }
669 } else {
670 void **pp = *lp;
671
672 for (i = 0; i < V_L2_SIZE; ++i) {
673 page_flush_tb_1(level - 1, pp + i);
674 }
675 }
676 }
677
678 static void page_flush_tb(void)
679 {
680 int i;
681
682 for (i = 0; i < V_L1_SIZE; i++) {
683 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
684 }
685 }
686
687 /* flush all the translation blocks */
688 /* XXX: tb_flush is currently not thread safe */
689 void tb_flush(CPUArchState *env1)
690 {
691 CPUState *cpu;
692
693 #if defined(DEBUG_FLUSH)
694 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
695 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
696 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
697 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
698 tcg_ctx.tb_ctx.nb_tbs : 0);
699 #endif
700 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
701 > tcg_ctx.code_gen_buffer_size) {
702 cpu_abort(env1, "Internal error: code buffer overflow\n");
703 }
704 tcg_ctx.tb_ctx.nb_tbs = 0;
705
706 CPU_FOREACH(cpu) {
707 CPUArchState *env = cpu->env_ptr;
708
709 memset(env->tb_jmp_cache, 0, sizeof(env->tb_jmp_cache));
710 }
711
712 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
713 page_flush_tb();
714
715 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
716 /* XXX: flush processor icache at this point if cache flush is
717 expensive */
718 tcg_ctx.tb_ctx.tb_flush_count++;
719 }
720
721 #ifdef DEBUG_TB_CHECK
722
723 static void tb_invalidate_check(target_ulong address)
724 {
725 TranslationBlock *tb;
726 int i;
727
728 address &= TARGET_PAGE_MASK;
729 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
730 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
731 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
732 address >= tb->pc + tb->size)) {
733 printf("ERROR invalidate: address=" TARGET_FMT_lx
734 " PC=%08lx size=%04x\n",
735 address, (long)tb->pc, tb->size);
736 }
737 }
738 }
739 }
740
741 /* verify that all the pages have correct rights for code */
742 static void tb_page_check(void)
743 {
744 TranslationBlock *tb;
745 int i, flags1, flags2;
746
747 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
748 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
749 tb = tb->phys_hash_next) {
750 flags1 = page_get_flags(tb->pc);
751 flags2 = page_get_flags(tb->pc + tb->size - 1);
752 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
753 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
754 (long)tb->pc, tb->size, flags1, flags2);
755 }
756 }
757 }
758 }
759
760 #endif
761
762 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
763 {
764 TranslationBlock *tb1;
765
766 for (;;) {
767 tb1 = *ptb;
768 if (tb1 == tb) {
769 *ptb = tb1->phys_hash_next;
770 break;
771 }
772 ptb = &tb1->phys_hash_next;
773 }
774 }
775
776 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
777 {
778 TranslationBlock *tb1;
779 unsigned int n1;
780
781 for (;;) {
782 tb1 = *ptb;
783 n1 = (uintptr_t)tb1 & 3;
784 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
785 if (tb1 == tb) {
786 *ptb = tb1->page_next[n1];
787 break;
788 }
789 ptb = &tb1->page_next[n1];
790 }
791 }
792
793 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
794 {
795 TranslationBlock *tb1, **ptb;
796 unsigned int n1;
797
798 ptb = &tb->jmp_next[n];
799 tb1 = *ptb;
800 if (tb1) {
801 /* find tb(n) in circular list */
802 for (;;) {
803 tb1 = *ptb;
804 n1 = (uintptr_t)tb1 & 3;
805 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
806 if (n1 == n && tb1 == tb) {
807 break;
808 }
809 if (n1 == 2) {
810 ptb = &tb1->jmp_first;
811 } else {
812 ptb = &tb1->jmp_next[n1];
813 }
814 }
815 /* now we can suppress tb(n) from the list */
816 *ptb = tb->jmp_next[n];
817
818 tb->jmp_next[n] = NULL;
819 }
820 }
821
822 /* reset the jump entry 'n' of a TB so that it is not chained to
823 another TB */
824 static inline void tb_reset_jump(TranslationBlock *tb, int n)
825 {
826 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
827 }
828
829 /* invalidate one TB */
830 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
831 {
832 CPUState *cpu;
833 PageDesc *p;
834 unsigned int h, n1;
835 tb_page_addr_t phys_pc;
836 TranslationBlock *tb1, *tb2;
837
838 /* remove the TB from the hash list */
839 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
840 h = tb_phys_hash_func(phys_pc);
841 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
842
843 /* remove the TB from the page list */
844 if (tb->page_addr[0] != page_addr) {
845 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
846 tb_page_remove(&p->first_tb, tb);
847 invalidate_page_bitmap(p);
848 }
849 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
850 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
851 tb_page_remove(&p->first_tb, tb);
852 invalidate_page_bitmap(p);
853 }
854
855 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
856
857 /* remove the TB from the hash list */
858 h = tb_jmp_cache_hash_func(tb->pc);
859 CPU_FOREACH(cpu) {
860 CPUArchState *env = cpu->env_ptr;
861
862 if (env->tb_jmp_cache[h] == tb) {
863 env->tb_jmp_cache[h] = NULL;
864 }
865 }
866
867 /* suppress this TB from the two jump lists */
868 tb_jmp_remove(tb, 0);
869 tb_jmp_remove(tb, 1);
870
871 /* suppress any remaining jumps to this TB */
872 tb1 = tb->jmp_first;
873 for (;;) {
874 n1 = (uintptr_t)tb1 & 3;
875 if (n1 == 2) {
876 break;
877 }
878 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
879 tb2 = tb1->jmp_next[n1];
880 tb_reset_jump(tb1, n1);
881 tb1->jmp_next[n1] = NULL;
882 tb1 = tb2;
883 }
884 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
885
886 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
887 }
888
889 static inline void set_bits(uint8_t *tab, int start, int len)
890 {
891 int end, mask, end1;
892
893 end = start + len;
894 tab += start >> 3;
895 mask = 0xff << (start & 7);
896 if ((start & ~7) == (end & ~7)) {
897 if (start < end) {
898 mask &= ~(0xff << (end & 7));
899 *tab |= mask;
900 }
901 } else {
902 *tab++ |= mask;
903 start = (start + 8) & ~7;
904 end1 = end & ~7;
905 while (start < end1) {
906 *tab++ = 0xff;
907 start += 8;
908 }
909 if (start < end) {
910 mask = ~(0xff << (end & 7));
911 *tab |= mask;
912 }
913 }
914 }
915
916 static void build_page_bitmap(PageDesc *p)
917 {
918 int n, tb_start, tb_end;
919 TranslationBlock *tb;
920
921 p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
922
923 tb = p->first_tb;
924 while (tb != NULL) {
925 n = (uintptr_t)tb & 3;
926 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
927 /* NOTE: this is subtle as a TB may span two physical pages */
928 if (n == 0) {
929 /* NOTE: tb_end may be after the end of the page, but
930 it is not a problem */
931 tb_start = tb->pc & ~TARGET_PAGE_MASK;
932 tb_end = tb_start + tb->size;
933 if (tb_end > TARGET_PAGE_SIZE) {
934 tb_end = TARGET_PAGE_SIZE;
935 }
936 } else {
937 tb_start = 0;
938 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
939 }
940 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
941 tb = tb->page_next[n];
942 }
943 }
944
945 TranslationBlock *tb_gen_code(CPUArchState *env,
946 target_ulong pc, target_ulong cs_base,
947 int flags, int cflags)
948 {
949 TranslationBlock *tb;
950 uint8_t *tc_ptr;
951 tb_page_addr_t phys_pc, phys_page2;
952 target_ulong virt_page2;
953 int code_gen_size;
954
955 phys_pc = get_page_addr_code(env, pc);
956 tb = tb_alloc(pc);
957 if (!tb) {
958 /* flush must be done */
959 tb_flush(env);
960 /* cannot fail at this point */
961 tb = tb_alloc(pc);
962 /* Don't forget to invalidate previous TB info. */
963 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
964 }
965 tc_ptr = tcg_ctx.code_gen_ptr;
966 tb->tc_ptr = tc_ptr;
967 tb->cs_base = cs_base;
968 tb->flags = flags;
969 tb->cflags = cflags;
970 cpu_gen_code(env, tb, &code_gen_size);
971 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
972 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
973
974 /* check next page if needed */
975 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
976 phys_page2 = -1;
977 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
978 phys_page2 = get_page_addr_code(env, virt_page2);
979 }
980 tb_link_page(tb, phys_pc, phys_page2);
981 return tb;
982 }
983
984 /*
985 * Invalidate all TBs which intersect with the target physical address range
986 * [start;end[. NOTE: start and end may refer to *different* physical pages.
987 * 'is_cpu_write_access' should be true if called from a real cpu write
988 * access: the virtual CPU will exit the current TB if code is modified inside
989 * this TB.
990 */
991 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
992 int is_cpu_write_access)
993 {
994 while (start < end) {
995 tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
996 start &= TARGET_PAGE_MASK;
997 start += TARGET_PAGE_SIZE;
998 }
999 }
1000
1001 /*
1002 * Invalidate all TBs which intersect with the target physical address range
1003 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1004 * 'is_cpu_write_access' should be true if called from a real cpu write
1005 * access: the virtual CPU will exit the current TB if code is modified inside
1006 * this TB.
1007 */
1008 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1009 int is_cpu_write_access)
1010 {
1011 TranslationBlock *tb, *tb_next, *saved_tb;
1012 CPUState *cpu = current_cpu;
1013 #if defined(TARGET_HAS_PRECISE_SMC) || !defined(CONFIG_USER_ONLY)
1014 CPUArchState *env = NULL;
1015 #endif
1016 tb_page_addr_t tb_start, tb_end;
1017 PageDesc *p;
1018 int n;
1019 #ifdef TARGET_HAS_PRECISE_SMC
1020 int current_tb_not_found = is_cpu_write_access;
1021 TranslationBlock *current_tb = NULL;
1022 int current_tb_modified = 0;
1023 target_ulong current_pc = 0;
1024 target_ulong current_cs_base = 0;
1025 int current_flags = 0;
1026 #endif /* TARGET_HAS_PRECISE_SMC */
1027
1028 p = page_find(start >> TARGET_PAGE_BITS);
1029 if (!p) {
1030 return;
1031 }
1032 if (!p->code_bitmap &&
1033 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1034 is_cpu_write_access) {
1035 /* build code bitmap */
1036 build_page_bitmap(p);
1037 }
1038 #if defined(TARGET_HAS_PRECISE_SMC) || !defined(CONFIG_USER_ONLY)
1039 if (cpu != NULL) {
1040 env = cpu->env_ptr;
1041 }
1042 #endif
1043
1044 /* we remove all the TBs in the range [start, end[ */
1045 /* XXX: see if in some cases it could be faster to invalidate all
1046 the code */
1047 tb = p->first_tb;
1048 while (tb != NULL) {
1049 n = (uintptr_t)tb & 3;
1050 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1051 tb_next = tb->page_next[n];
1052 /* NOTE: this is subtle as a TB may span two physical pages */
1053 if (n == 0) {
1054 /* NOTE: tb_end may be after the end of the page, but
1055 it is not a problem */
1056 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1057 tb_end = tb_start + tb->size;
1058 } else {
1059 tb_start = tb->page_addr[1];
1060 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1061 }
1062 if (!(tb_end <= start || tb_start >= end)) {
1063 #ifdef TARGET_HAS_PRECISE_SMC
1064 if (current_tb_not_found) {
1065 current_tb_not_found = 0;
1066 current_tb = NULL;
1067 if (cpu->mem_io_pc) {
1068 /* now we have a real cpu fault */
1069 current_tb = tb_find_pc(cpu->mem_io_pc);
1070 }
1071 }
1072 if (current_tb == tb &&
1073 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1074 /* If we are modifying the current TB, we must stop
1075 its execution. We could be more precise by checking
1076 that the modification is after the current PC, but it
1077 would require a specialized function to partially
1078 restore the CPU state */
1079
1080 current_tb_modified = 1;
1081 cpu_restore_state_from_tb(current_tb, env, cpu->mem_io_pc);
1082 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1083 &current_flags);
1084 }
1085 #endif /* TARGET_HAS_PRECISE_SMC */
1086 /* we need to do that to handle the case where a signal
1087 occurs while doing tb_phys_invalidate() */
1088 saved_tb = NULL;
1089 if (cpu != NULL) {
1090 saved_tb = cpu->current_tb;
1091 cpu->current_tb = NULL;
1092 }
1093 tb_phys_invalidate(tb, -1);
1094 if (cpu != NULL) {
1095 cpu->current_tb = saved_tb;
1096 if (cpu->interrupt_request && cpu->current_tb) {
1097 cpu_interrupt(cpu, cpu->interrupt_request);
1098 }
1099 }
1100 }
1101 tb = tb_next;
1102 }
1103 #if !defined(CONFIG_USER_ONLY)
1104 /* if no code remaining, no need to continue to use slow writes */
1105 if (!p->first_tb) {
1106 invalidate_page_bitmap(p);
1107 if (is_cpu_write_access) {
1108 tlb_unprotect_code_phys(env, start, cpu->mem_io_vaddr);
1109 }
1110 }
1111 #endif
1112 #ifdef TARGET_HAS_PRECISE_SMC
1113 if (current_tb_modified) {
1114 /* we generate a block containing just the instruction
1115 modifying the memory. It will ensure that it cannot modify
1116 itself */
1117 cpu->current_tb = NULL;
1118 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1119 cpu_resume_from_signal(env, NULL);
1120 }
1121 #endif
1122 }
1123
1124 /* len must be <= 8 and start must be a multiple of len */
1125 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1126 {
1127 PageDesc *p;
1128 int offset, b;
1129
1130 #if 0
1131 if (1) {
1132 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1133 cpu_single_env->mem_io_vaddr, len,
1134 cpu_single_env->eip,
1135 cpu_single_env->eip +
1136 (intptr_t)cpu_single_env->segs[R_CS].base);
1137 }
1138 #endif
1139 p = page_find(start >> TARGET_PAGE_BITS);
1140 if (!p) {
1141 return;
1142 }
1143 if (p->code_bitmap) {
1144 offset = start & ~TARGET_PAGE_MASK;
1145 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1146 if (b & ((1 << len) - 1)) {
1147 goto do_invalidate;
1148 }
1149 } else {
1150 do_invalidate:
1151 tb_invalidate_phys_page_range(start, start + len, 1);
1152 }
1153 }
1154
1155 #if !defined(CONFIG_SOFTMMU)
1156 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1157 uintptr_t pc, void *puc,
1158 bool locked)
1159 {
1160 TranslationBlock *tb;
1161 PageDesc *p;
1162 int n;
1163 #ifdef TARGET_HAS_PRECISE_SMC
1164 TranslationBlock *current_tb = NULL;
1165 CPUState *cpu = current_cpu;
1166 CPUArchState *env = NULL;
1167 int current_tb_modified = 0;
1168 target_ulong current_pc = 0;
1169 target_ulong current_cs_base = 0;
1170 int current_flags = 0;
1171 #endif
1172
1173 addr &= TARGET_PAGE_MASK;
1174 p = page_find(addr >> TARGET_PAGE_BITS);
1175 if (!p) {
1176 return;
1177 }
1178 tb = p->first_tb;
1179 #ifdef TARGET_HAS_PRECISE_SMC
1180 if (tb && pc != 0) {
1181 current_tb = tb_find_pc(pc);
1182 }
1183 if (cpu != NULL) {
1184 env = cpu->env_ptr;
1185 }
1186 #endif
1187 while (tb != NULL) {
1188 n = (uintptr_t)tb & 3;
1189 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1190 #ifdef TARGET_HAS_PRECISE_SMC
1191 if (current_tb == tb &&
1192 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1193 /* If we are modifying the current TB, we must stop
1194 its execution. We could be more precise by checking
1195 that the modification is after the current PC, but it
1196 would require a specialized function to partially
1197 restore the CPU state */
1198
1199 current_tb_modified = 1;
1200 cpu_restore_state_from_tb(current_tb, env, pc);
1201 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1202 &current_flags);
1203 }
1204 #endif /* TARGET_HAS_PRECISE_SMC */
1205 tb_phys_invalidate(tb, addr);
1206 tb = tb->page_next[n];
1207 }
1208 p->first_tb = NULL;
1209 #ifdef TARGET_HAS_PRECISE_SMC
1210 if (current_tb_modified) {
1211 /* we generate a block containing just the instruction
1212 modifying the memory. It will ensure that it cannot modify
1213 itself */
1214 cpu->current_tb = NULL;
1215 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1216 if (locked) {
1217 mmap_unlock();
1218 }
1219 cpu_resume_from_signal(env, puc);
1220 }
1221 #endif
1222 }
1223 #endif
1224
1225 /* add the tb in the target page and protect it if necessary */
1226 static inline void tb_alloc_page(TranslationBlock *tb,
1227 unsigned int n, tb_page_addr_t page_addr)
1228 {
1229 PageDesc *p;
1230 #ifndef CONFIG_USER_ONLY
1231 bool page_already_protected;
1232 #endif
1233
1234 tb->page_addr[n] = page_addr;
1235 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1236 tb->page_next[n] = p->first_tb;
1237 #ifndef CONFIG_USER_ONLY
1238 page_already_protected = p->first_tb != NULL;
1239 #endif
1240 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1241 invalidate_page_bitmap(p);
1242
1243 #if defined(TARGET_HAS_SMC) || 1
1244
1245 #if defined(CONFIG_USER_ONLY)
1246 if (p->flags & PAGE_WRITE) {
1247 target_ulong addr;
1248 PageDesc *p2;
1249 int prot;
1250
1251 /* force the host page as non writable (writes will have a
1252 page fault + mprotect overhead) */
1253 page_addr &= qemu_host_page_mask;
1254 prot = 0;
1255 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1256 addr += TARGET_PAGE_SIZE) {
1257
1258 p2 = page_find(addr >> TARGET_PAGE_BITS);
1259 if (!p2) {
1260 continue;
1261 }
1262 prot |= p2->flags;
1263 p2->flags &= ~PAGE_WRITE;
1264 }
1265 mprotect(g2h(page_addr), qemu_host_page_size,
1266 (prot & PAGE_BITS) & ~PAGE_WRITE);
1267 #ifdef DEBUG_TB_INVALIDATE
1268 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1269 page_addr);
1270 #endif
1271 }
1272 #else
1273 /* if some code is already present, then the pages are already
1274 protected. So we handle the case where only the first TB is
1275 allocated in a physical page */
1276 if (!page_already_protected) {
1277 tlb_protect_code(page_addr);
1278 }
1279 #endif
1280
1281 #endif /* TARGET_HAS_SMC */
1282 }
1283
1284 /* add a new TB and link it to the physical page tables. phys_page2 is
1285 (-1) to indicate that only one page contains the TB. */
1286 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1287 tb_page_addr_t phys_page2)
1288 {
1289 unsigned int h;
1290 TranslationBlock **ptb;
1291
1292 /* Grab the mmap lock to stop another thread invalidating this TB
1293 before we are done. */
1294 mmap_lock();
1295 /* add in the physical hash table */
1296 h = tb_phys_hash_func(phys_pc);
1297 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1298 tb->phys_hash_next = *ptb;
1299 *ptb = tb;
1300
1301 /* add in the page list */
1302 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1303 if (phys_page2 != -1) {
1304 tb_alloc_page(tb, 1, phys_page2);
1305 } else {
1306 tb->page_addr[1] = -1;
1307 }
1308
1309 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1310 tb->jmp_next[0] = NULL;
1311 tb->jmp_next[1] = NULL;
1312
1313 /* init original jump addresses */
1314 if (tb->tb_next_offset[0] != 0xffff) {
1315 tb_reset_jump(tb, 0);
1316 }
1317 if (tb->tb_next_offset[1] != 0xffff) {
1318 tb_reset_jump(tb, 1);
1319 }
1320
1321 #ifdef DEBUG_TB_CHECK
1322 tb_page_check();
1323 #endif
1324 mmap_unlock();
1325 }
1326
1327 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1328 tb[1].tc_ptr. Return NULL if not found */
1329 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1330 {
1331 int m_min, m_max, m;
1332 uintptr_t v;
1333 TranslationBlock *tb;
1334
1335 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1336 return NULL;
1337 }
1338 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1339 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1340 return NULL;
1341 }
1342 /* binary search (cf Knuth) */
1343 m_min = 0;
1344 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1345 while (m_min <= m_max) {
1346 m = (m_min + m_max) >> 1;
1347 tb = &tcg_ctx.tb_ctx.tbs[m];
1348 v = (uintptr_t)tb->tc_ptr;
1349 if (v == tc_ptr) {
1350 return tb;
1351 } else if (tc_ptr < v) {
1352 m_max = m - 1;
1353 } else {
1354 m_min = m + 1;
1355 }
1356 }
1357 return &tcg_ctx.tb_ctx.tbs[m_max];
1358 }
1359
1360 #if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
1361 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1362 {
1363 ram_addr_t ram_addr;
1364 MemoryRegion *mr;
1365 hwaddr l = 1;
1366
1367 mr = address_space_translate(as, addr, &addr, &l, false);
1368 if (!(memory_region_is_ram(mr)
1369 || memory_region_is_romd(mr))) {
1370 return;
1371 }
1372 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
1373 + addr;
1374 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1375 }
1376 #endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
1377
1378 void tb_check_watchpoint(CPUArchState *env)
1379 {
1380 CPUState *cpu = ENV_GET_CPU(env);
1381 TranslationBlock *tb;
1382
1383 tb = tb_find_pc(cpu->mem_io_pc);
1384 if (!tb) {
1385 cpu_abort(env, "check_watchpoint: could not find TB for pc=%p",
1386 (void *)cpu->mem_io_pc);
1387 }
1388 cpu_restore_state_from_tb(tb, env, cpu->mem_io_pc);
1389 tb_phys_invalidate(tb, -1);
1390 }
1391
1392 #ifndef CONFIG_USER_ONLY
1393 /* mask must never be zero, except for A20 change call */
1394 static void tcg_handle_interrupt(CPUState *cpu, int mask)
1395 {
1396 CPUArchState *env = cpu->env_ptr;
1397 int old_mask;
1398
1399 old_mask = cpu->interrupt_request;
1400 cpu->interrupt_request |= mask;
1401
1402 /*
1403 * If called from iothread context, wake the target cpu in
1404 * case its halted.
1405 */
1406 if (!qemu_cpu_is_self(cpu)) {
1407 qemu_cpu_kick(cpu);
1408 return;
1409 }
1410
1411 if (use_icount) {
1412 env->icount_decr.u16.high = 0xffff;
1413 if (!cpu_can_do_io(cpu)
1414 && (mask & ~old_mask) != 0) {
1415 cpu_abort(env, "Raised interrupt while not in I/O function");
1416 }
1417 } else {
1418 cpu->tcg_exit_req = 1;
1419 }
1420 }
1421
1422 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1423
1424 /* in deterministic execution mode, instructions doing device I/Os
1425 must be at the end of the TB */
1426 void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
1427 {
1428 TranslationBlock *tb;
1429 uint32_t n, cflags;
1430 target_ulong pc, cs_base;
1431 uint64_t flags;
1432
1433 tb = tb_find_pc(retaddr);
1434 if (!tb) {
1435 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
1436 (void *)retaddr);
1437 }
1438 n = env->icount_decr.u16.low + tb->icount;
1439 cpu_restore_state_from_tb(tb, env, retaddr);
1440 /* Calculate how many instructions had been executed before the fault
1441 occurred. */
1442 n = n - env->icount_decr.u16.low;
1443 /* Generate a new TB ending on the I/O insn. */
1444 n++;
1445 /* On MIPS and SH, delay slot instructions can only be restarted if
1446 they were already the first instruction in the TB. If this is not
1447 the first instruction in a TB then re-execute the preceding
1448 branch. */
1449 #if defined(TARGET_MIPS)
1450 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1451 env->active_tc.PC -= 4;
1452 env->icount_decr.u16.low++;
1453 env->hflags &= ~MIPS_HFLAG_BMASK;
1454 }
1455 #elif defined(TARGET_SH4)
1456 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1457 && n > 1) {
1458 env->pc -= 2;
1459 env->icount_decr.u16.low++;
1460 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1461 }
1462 #endif
1463 /* This should never happen. */
1464 if (n > CF_COUNT_MASK) {
1465 cpu_abort(env, "TB too big during recompile");
1466 }
1467
1468 cflags = n | CF_LAST_IO;
1469 pc = tb->pc;
1470 cs_base = tb->cs_base;
1471 flags = tb->flags;
1472 tb_phys_invalidate(tb, -1);
1473 /* FIXME: In theory this could raise an exception. In practice
1474 we have already translated the block once so it's probably ok. */
1475 tb_gen_code(env, pc, cs_base, flags, cflags);
1476 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1477 the first in the TB) then we end up generating a whole new TB and
1478 repeating the fault, which is horribly inefficient.
1479 Better would be to execute just this insn uncached, or generate a
1480 second new TB. */
1481 cpu_resume_from_signal(env, NULL);
1482 }
1483
1484 void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
1485 {
1486 unsigned int i;
1487
1488 /* Discard jump cache entries for any tb which might potentially
1489 overlap the flushed page. */
1490 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1491 memset(&env->tb_jmp_cache[i], 0,
1492 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1493
1494 i = tb_jmp_cache_hash_page(addr);
1495 memset(&env->tb_jmp_cache[i], 0,
1496 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1497 }
1498
1499 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1500 {
1501 int i, target_code_size, max_target_code_size;
1502 int direct_jmp_count, direct_jmp2_count, cross_page;
1503 TranslationBlock *tb;
1504
1505 target_code_size = 0;
1506 max_target_code_size = 0;
1507 cross_page = 0;
1508 direct_jmp_count = 0;
1509 direct_jmp2_count = 0;
1510 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1511 tb = &tcg_ctx.tb_ctx.tbs[i];
1512 target_code_size += tb->size;
1513 if (tb->size > max_target_code_size) {
1514 max_target_code_size = tb->size;
1515 }
1516 if (tb->page_addr[1] != -1) {
1517 cross_page++;
1518 }
1519 if (tb->tb_next_offset[0] != 0xffff) {
1520 direct_jmp_count++;
1521 if (tb->tb_next_offset[1] != 0xffff) {
1522 direct_jmp2_count++;
1523 }
1524 }
1525 }
1526 /* XXX: avoid using doubles ? */
1527 cpu_fprintf(f, "Translation buffer state:\n");
1528 cpu_fprintf(f, "gen code size %td/%zd\n",
1529 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1530 tcg_ctx.code_gen_buffer_max_size);
1531 cpu_fprintf(f, "TB count %d/%d\n",
1532 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1533 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1534 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1535 tcg_ctx.tb_ctx.nb_tbs : 0,
1536 max_target_code_size);
1537 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1538 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1539 tcg_ctx.code_gen_buffer) /
1540 tcg_ctx.tb_ctx.nb_tbs : 0,
1541 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1542 tcg_ctx.code_gen_buffer) /
1543 target_code_size : 0);
1544 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1545 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1546 tcg_ctx.tb_ctx.nb_tbs : 0);
1547 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1548 direct_jmp_count,
1549 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1550 tcg_ctx.tb_ctx.nb_tbs : 0,
1551 direct_jmp2_count,
1552 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1553 tcg_ctx.tb_ctx.nb_tbs : 0);
1554 cpu_fprintf(f, "\nStatistics:\n");
1555 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1556 cpu_fprintf(f, "TB invalidate count %d\n",
1557 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1558 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1559 tcg_dump_info(f, cpu_fprintf);
1560 }
1561
1562 #else /* CONFIG_USER_ONLY */
1563
1564 void cpu_interrupt(CPUState *cpu, int mask)
1565 {
1566 cpu->interrupt_request |= mask;
1567 cpu->tcg_exit_req = 1;
1568 }
1569
1570 /*
1571 * Walks guest process memory "regions" one by one
1572 * and calls callback function 'fn' for each region.
1573 */
1574 struct walk_memory_regions_data {
1575 walk_memory_regions_fn fn;
1576 void *priv;
1577 uintptr_t start;
1578 int prot;
1579 };
1580
1581 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1582 abi_ulong end, int new_prot)
1583 {
1584 if (data->start != -1ul) {
1585 int rc = data->fn(data->priv, data->start, end, data->prot);
1586 if (rc != 0) {
1587 return rc;
1588 }
1589 }
1590
1591 data->start = (new_prot ? end : -1ul);
1592 data->prot = new_prot;
1593
1594 return 0;
1595 }
1596
1597 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1598 abi_ulong base, int level, void **lp)
1599 {
1600 abi_ulong pa;
1601 int i, rc;
1602
1603 if (*lp == NULL) {
1604 return walk_memory_regions_end(data, base, 0);
1605 }
1606
1607 if (level == 0) {
1608 PageDesc *pd = *lp;
1609
1610 for (i = 0; i < V_L2_SIZE; ++i) {
1611 int prot = pd[i].flags;
1612
1613 pa = base | (i << TARGET_PAGE_BITS);
1614 if (prot != data->prot) {
1615 rc = walk_memory_regions_end(data, pa, prot);
1616 if (rc != 0) {
1617 return rc;
1618 }
1619 }
1620 }
1621 } else {
1622 void **pp = *lp;
1623
1624 for (i = 0; i < V_L2_SIZE; ++i) {
1625 pa = base | ((abi_ulong)i <<
1626 (TARGET_PAGE_BITS + V_L2_BITS * level));
1627 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1628 if (rc != 0) {
1629 return rc;
1630 }
1631 }
1632 }
1633
1634 return 0;
1635 }
1636
1637 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1638 {
1639 struct walk_memory_regions_data data;
1640 uintptr_t i;
1641
1642 data.fn = fn;
1643 data.priv = priv;
1644 data.start = -1ul;
1645 data.prot = 0;
1646
1647 for (i = 0; i < V_L1_SIZE; i++) {
1648 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
1649 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1650
1651 if (rc != 0) {
1652 return rc;
1653 }
1654 }
1655
1656 return walk_memory_regions_end(&data, 0, 0);
1657 }
1658
1659 static int dump_region(void *priv, abi_ulong start,
1660 abi_ulong end, unsigned long prot)
1661 {
1662 FILE *f = (FILE *)priv;
1663
1664 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
1665 " "TARGET_ABI_FMT_lx" %c%c%c\n",
1666 start, end, end - start,
1667 ((prot & PAGE_READ) ? 'r' : '-'),
1668 ((prot & PAGE_WRITE) ? 'w' : '-'),
1669 ((prot & PAGE_EXEC) ? 'x' : '-'));
1670
1671 return 0;
1672 }
1673
1674 /* dump memory mappings */
1675 void page_dump(FILE *f)
1676 {
1677 const int length = sizeof(abi_ulong) * 2;
1678 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1679 length, "start", length, "end", length, "size", "prot");
1680 walk_memory_regions(f, dump_region);
1681 }
1682
1683 int page_get_flags(target_ulong address)
1684 {
1685 PageDesc *p;
1686
1687 p = page_find(address >> TARGET_PAGE_BITS);
1688 if (!p) {
1689 return 0;
1690 }
1691 return p->flags;
1692 }
1693
1694 /* Modify the flags of a page and invalidate the code if necessary.
1695 The flag PAGE_WRITE_ORG is positioned automatically depending
1696 on PAGE_WRITE. The mmap_lock should already be held. */
1697 void page_set_flags(target_ulong start, target_ulong end, int flags)
1698 {
1699 target_ulong addr, len;
1700
1701 /* This function should never be called with addresses outside the
1702 guest address space. If this assert fires, it probably indicates
1703 a missing call to h2g_valid. */
1704 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1705 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1706 #endif
1707 assert(start < end);
1708
1709 start = start & TARGET_PAGE_MASK;
1710 end = TARGET_PAGE_ALIGN(end);
1711
1712 if (flags & PAGE_WRITE) {
1713 flags |= PAGE_WRITE_ORG;
1714 }
1715
1716 for (addr = start, len = end - start;
1717 len != 0;
1718 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1719 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1720
1721 /* If the write protection bit is set, then we invalidate
1722 the code inside. */
1723 if (!(p->flags & PAGE_WRITE) &&
1724 (flags & PAGE_WRITE) &&
1725 p->first_tb) {
1726 tb_invalidate_phys_page(addr, 0, NULL, false);
1727 }
1728 p->flags = flags;
1729 }
1730 }
1731
1732 int page_check_range(target_ulong start, target_ulong len, int flags)
1733 {
1734 PageDesc *p;
1735 target_ulong end;
1736 target_ulong addr;
1737
1738 /* This function should never be called with addresses outside the
1739 guest address space. If this assert fires, it probably indicates
1740 a missing call to h2g_valid. */
1741 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1742 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1743 #endif
1744
1745 if (len == 0) {
1746 return 0;
1747 }
1748 if (start + len - 1 < start) {
1749 /* We've wrapped around. */
1750 return -1;
1751 }
1752
1753 /* must do before we loose bits in the next step */
1754 end = TARGET_PAGE_ALIGN(start + len);
1755 start = start & TARGET_PAGE_MASK;
1756
1757 for (addr = start, len = end - start;
1758 len != 0;
1759 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1760 p = page_find(addr >> TARGET_PAGE_BITS);
1761 if (!p) {
1762 return -1;
1763 }
1764 if (!(p->flags & PAGE_VALID)) {
1765 return -1;
1766 }
1767
1768 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1769 return -1;
1770 }
1771 if (flags & PAGE_WRITE) {
1772 if (!(p->flags & PAGE_WRITE_ORG)) {
1773 return -1;
1774 }
1775 /* unprotect the page if it was put read-only because it
1776 contains translated code */
1777 if (!(p->flags & PAGE_WRITE)) {
1778 if (!page_unprotect(addr, 0, NULL)) {
1779 return -1;
1780 }
1781 }
1782 return 0;
1783 }
1784 }
1785 return 0;
1786 }
1787
1788 /* called from signal handler: invalidate the code and unprotect the
1789 page. Return TRUE if the fault was successfully handled. */
1790 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1791 {
1792 unsigned int prot;
1793 PageDesc *p;
1794 target_ulong host_start, host_end, addr;
1795
1796 /* Technically this isn't safe inside a signal handler. However we
1797 know this only ever happens in a synchronous SEGV handler, so in
1798 practice it seems to be ok. */
1799 mmap_lock();
1800
1801 p = page_find(address >> TARGET_PAGE_BITS);
1802 if (!p) {
1803 mmap_unlock();
1804 return 0;
1805 }
1806
1807 /* if the page was really writable, then we change its
1808 protection back to writable */
1809 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1810 host_start = address & qemu_host_page_mask;
1811 host_end = host_start + qemu_host_page_size;
1812
1813 prot = 0;
1814 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1815 p = page_find(addr >> TARGET_PAGE_BITS);
1816 p->flags |= PAGE_WRITE;
1817 prot |= p->flags;
1818
1819 /* and since the content will be modified, we must invalidate
1820 the corresponding translated code. */
1821 tb_invalidate_phys_page(addr, pc, puc, true);
1822 #ifdef DEBUG_TB_CHECK
1823 tb_invalidate_check(addr);
1824 #endif
1825 }
1826 mprotect((void *)g2h(host_start), qemu_host_page_size,
1827 prot & PAGE_BITS);
1828
1829 mmap_unlock();
1830 return 1;
1831 }
1832 mmap_unlock();
1833 return 0;
1834 }
1835 #endif /* CONFIG_USER_ONLY */