cpu: Move icount_decr field from CPU_COMMON to CPUState
[qemu.git] / translate-all.c
1 /*
2 * Host code generation
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/types.h>
23 #include <sys/mman.h>
24 #endif
25 #include <stdarg.h>
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <inttypes.h>
30
31 #include "config.h"
32
33 #include "qemu-common.h"
34 #define NO_CPU_IO_DEFS
35 #include "cpu.h"
36 #include "disas/disas.h"
37 #include "tcg.h"
38 #if defined(CONFIG_USER_ONLY)
39 #include "qemu.h"
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
45 #include <sys/time.h>
46 #include <sys/proc.h>
47 #include <machine/profile.h>
48 #define _KERNEL
49 #include <sys/user.h>
50 #undef _KERNEL
51 #undef sigqueue
52 #include <libutil.h>
53 #endif
54 #endif
55 #else
56 #include "exec/address-spaces.h"
57 #endif
58
59 #include "exec/cputlb.h"
60 #include "translate-all.h"
61 #include "qemu/timer.h"
62
63 //#define DEBUG_TB_INVALIDATE
64 //#define DEBUG_FLUSH
65 /* make various TB consistency checks */
66 //#define DEBUG_TB_CHECK
67
68 #if !defined(CONFIG_USER_ONLY)
69 /* TB consistency checks only implemented for usermode emulation. */
70 #undef DEBUG_TB_CHECK
71 #endif
72
73 #define SMC_BITMAP_USE_THRESHOLD 10
74
75 typedef struct PageDesc {
76 /* list of TBs intersecting this ram page */
77 TranslationBlock *first_tb;
78 /* in order to optimize self modifying code, we count the number
79 of lookups we do to a given page to use a bitmap */
80 unsigned int code_write_count;
81 uint8_t *code_bitmap;
82 #if defined(CONFIG_USER_ONLY)
83 unsigned long flags;
84 #endif
85 } PageDesc;
86
87 /* In system mode we want L1_MAP to be based on ram offsets,
88 while in user mode we want it to be based on virtual addresses. */
89 #if !defined(CONFIG_USER_ONLY)
90 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
91 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
92 #else
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
94 #endif
95 #else
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
97 #endif
98
99 /* Size of the L2 (and L3, etc) page tables. */
100 #define V_L2_BITS 10
101 #define V_L2_SIZE (1 << V_L2_BITS)
102
103 /* The bits remaining after N lower levels of page tables. */
104 #define V_L1_BITS_REM \
105 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
106
107 #if V_L1_BITS_REM < 4
108 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
109 #else
110 #define V_L1_BITS V_L1_BITS_REM
111 #endif
112
113 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
114
115 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
116
117 uintptr_t qemu_real_host_page_size;
118 uintptr_t qemu_host_page_size;
119 uintptr_t qemu_host_page_mask;
120
121 /* This is a multi-level map on the virtual address space.
122 The bottom level has pointers to PageDesc. */
123 static void *l1_map[V_L1_SIZE];
124
125 /* code generation context */
126 TCGContext tcg_ctx;
127
128 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
129 tb_page_addr_t phys_page2);
130 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
131
132 void cpu_gen_init(void)
133 {
134 tcg_context_init(&tcg_ctx);
135 }
136
137 /* return non zero if the very first instruction is invalid so that
138 the virtual CPU can trigger an exception.
139
140 '*gen_code_size_ptr' contains the size of the generated code (host
141 code).
142 */
143 int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
144 {
145 TCGContext *s = &tcg_ctx;
146 uint8_t *gen_code_buf;
147 int gen_code_size;
148 #ifdef CONFIG_PROFILER
149 int64_t ti;
150 #endif
151
152 #ifdef CONFIG_PROFILER
153 s->tb_count1++; /* includes aborted translations because of
154 exceptions */
155 ti = profile_getclock();
156 #endif
157 tcg_func_start(s);
158
159 gen_intermediate_code(env, tb);
160
161 /* generate machine code */
162 gen_code_buf = tb->tc_ptr;
163 tb->tb_next_offset[0] = 0xffff;
164 tb->tb_next_offset[1] = 0xffff;
165 s->tb_next_offset = tb->tb_next_offset;
166 #ifdef USE_DIRECT_JUMP
167 s->tb_jmp_offset = tb->tb_jmp_offset;
168 s->tb_next = NULL;
169 #else
170 s->tb_jmp_offset = NULL;
171 s->tb_next = tb->tb_next;
172 #endif
173
174 #ifdef CONFIG_PROFILER
175 s->tb_count++;
176 s->interm_time += profile_getclock() - ti;
177 s->code_time -= profile_getclock();
178 #endif
179 gen_code_size = tcg_gen_code(s, gen_code_buf);
180 *gen_code_size_ptr = gen_code_size;
181 #ifdef CONFIG_PROFILER
182 s->code_time += profile_getclock();
183 s->code_in_len += tb->size;
184 s->code_out_len += gen_code_size;
185 #endif
186
187 #ifdef DEBUG_DISAS
188 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
189 qemu_log("OUT: [size=%d]\n", *gen_code_size_ptr);
190 log_disas(tb->tc_ptr, *gen_code_size_ptr);
191 qemu_log("\n");
192 qemu_log_flush();
193 }
194 #endif
195 return 0;
196 }
197
198 /* The cpu state corresponding to 'searched_pc' is restored.
199 */
200 static int cpu_restore_state_from_tb(TranslationBlock *tb, CPUArchState *env,
201 uintptr_t searched_pc)
202 {
203 CPUState *cpu = ENV_GET_CPU(env);
204 TCGContext *s = &tcg_ctx;
205 int j;
206 uintptr_t tc_ptr;
207 #ifdef CONFIG_PROFILER
208 int64_t ti;
209 #endif
210
211 #ifdef CONFIG_PROFILER
212 ti = profile_getclock();
213 #endif
214 tcg_func_start(s);
215
216 gen_intermediate_code_pc(env, tb);
217
218 if (use_icount) {
219 /* Reset the cycle counter to the start of the block. */
220 cpu->icount_decr.u16.low += tb->icount;
221 /* Clear the IO flag. */
222 cpu->can_do_io = 0;
223 }
224
225 /* find opc index corresponding to search_pc */
226 tc_ptr = (uintptr_t)tb->tc_ptr;
227 if (searched_pc < tc_ptr)
228 return -1;
229
230 s->tb_next_offset = tb->tb_next_offset;
231 #ifdef USE_DIRECT_JUMP
232 s->tb_jmp_offset = tb->tb_jmp_offset;
233 s->tb_next = NULL;
234 #else
235 s->tb_jmp_offset = NULL;
236 s->tb_next = tb->tb_next;
237 #endif
238 j = tcg_gen_code_search_pc(s, (uint8_t *)tc_ptr, searched_pc - tc_ptr);
239 if (j < 0)
240 return -1;
241 /* now find start of instruction before */
242 while (s->gen_opc_instr_start[j] == 0) {
243 j--;
244 }
245 cpu->icount_decr.u16.low -= s->gen_opc_icount[j];
246
247 restore_state_to_opc(env, tb, j);
248
249 #ifdef CONFIG_PROFILER
250 s->restore_time += profile_getclock() - ti;
251 s->restore_count++;
252 #endif
253 return 0;
254 }
255
256 bool cpu_restore_state(CPUArchState *env, uintptr_t retaddr)
257 {
258 TranslationBlock *tb;
259
260 tb = tb_find_pc(retaddr);
261 if (tb) {
262 cpu_restore_state_from_tb(tb, env, retaddr);
263 return true;
264 }
265 return false;
266 }
267
268 #ifdef _WIN32
269 static inline void map_exec(void *addr, long size)
270 {
271 DWORD old_protect;
272 VirtualProtect(addr, size,
273 PAGE_EXECUTE_READWRITE, &old_protect);
274 }
275 #else
276 static inline void map_exec(void *addr, long size)
277 {
278 unsigned long start, end, page_size;
279
280 page_size = getpagesize();
281 start = (unsigned long)addr;
282 start &= ~(page_size - 1);
283
284 end = (unsigned long)addr + size;
285 end += page_size - 1;
286 end &= ~(page_size - 1);
287
288 mprotect((void *)start, end - start,
289 PROT_READ | PROT_WRITE | PROT_EXEC);
290 }
291 #endif
292
293 void page_size_init(void)
294 {
295 /* NOTE: we can always suppose that qemu_host_page_size >=
296 TARGET_PAGE_SIZE */
297 #ifdef _WIN32
298 SYSTEM_INFO system_info;
299
300 GetSystemInfo(&system_info);
301 qemu_real_host_page_size = system_info.dwPageSize;
302 #else
303 qemu_real_host_page_size = getpagesize();
304 #endif
305 if (qemu_host_page_size == 0) {
306 qemu_host_page_size = qemu_real_host_page_size;
307 }
308 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
309 qemu_host_page_size = TARGET_PAGE_SIZE;
310 }
311 qemu_host_page_mask = ~(qemu_host_page_size - 1);
312 }
313
314 static void page_init(void)
315 {
316 page_size_init();
317 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
318 {
319 #ifdef HAVE_KINFO_GETVMMAP
320 struct kinfo_vmentry *freep;
321 int i, cnt;
322
323 freep = kinfo_getvmmap(getpid(), &cnt);
324 if (freep) {
325 mmap_lock();
326 for (i = 0; i < cnt; i++) {
327 unsigned long startaddr, endaddr;
328
329 startaddr = freep[i].kve_start;
330 endaddr = freep[i].kve_end;
331 if (h2g_valid(startaddr)) {
332 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
333
334 if (h2g_valid(endaddr)) {
335 endaddr = h2g(endaddr);
336 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
337 } else {
338 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
339 endaddr = ~0ul;
340 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
341 #endif
342 }
343 }
344 }
345 free(freep);
346 mmap_unlock();
347 }
348 #else
349 FILE *f;
350
351 last_brk = (unsigned long)sbrk(0);
352
353 f = fopen("/compat/linux/proc/self/maps", "r");
354 if (f) {
355 mmap_lock();
356
357 do {
358 unsigned long startaddr, endaddr;
359 int n;
360
361 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
362
363 if (n == 2 && h2g_valid(startaddr)) {
364 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
365
366 if (h2g_valid(endaddr)) {
367 endaddr = h2g(endaddr);
368 } else {
369 endaddr = ~0ul;
370 }
371 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
372 }
373 } while (!feof(f));
374
375 fclose(f);
376 mmap_unlock();
377 }
378 #endif
379 }
380 #endif
381 }
382
383 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
384 {
385 PageDesc *pd;
386 void **lp;
387 int i;
388
389 #if defined(CONFIG_USER_ONLY)
390 /* We can't use g_malloc because it may recurse into a locked mutex. */
391 # define ALLOC(P, SIZE) \
392 do { \
393 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
394 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
395 } while (0)
396 #else
397 # define ALLOC(P, SIZE) \
398 do { P = g_malloc0(SIZE); } while (0)
399 #endif
400
401 /* Level 1. Always allocated. */
402 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
403
404 /* Level 2..N-1. */
405 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
406 void **p = *lp;
407
408 if (p == NULL) {
409 if (!alloc) {
410 return NULL;
411 }
412 ALLOC(p, sizeof(void *) * V_L2_SIZE);
413 *lp = p;
414 }
415
416 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
417 }
418
419 pd = *lp;
420 if (pd == NULL) {
421 if (!alloc) {
422 return NULL;
423 }
424 ALLOC(pd, sizeof(PageDesc) * V_L2_SIZE);
425 *lp = pd;
426 }
427
428 #undef ALLOC
429
430 return pd + (index & (V_L2_SIZE - 1));
431 }
432
433 static inline PageDesc *page_find(tb_page_addr_t index)
434 {
435 return page_find_alloc(index, 0);
436 }
437
438 #if !defined(CONFIG_USER_ONLY)
439 #define mmap_lock() do { } while (0)
440 #define mmap_unlock() do { } while (0)
441 #endif
442
443 #if defined(CONFIG_USER_ONLY)
444 /* Currently it is not recommended to allocate big chunks of data in
445 user mode. It will change when a dedicated libc will be used. */
446 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
447 region in which the guest needs to run. Revisit this. */
448 #define USE_STATIC_CODE_GEN_BUFFER
449 #endif
450
451 /* ??? Should configure for this, not list operating systems here. */
452 #if (defined(__linux__) \
453 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
454 || defined(__DragonFly__) || defined(__OpenBSD__) \
455 || defined(__NetBSD__))
456 # define USE_MMAP
457 #endif
458
459 /* Minimum size of the code gen buffer. This number is randomly chosen,
460 but not so small that we can't have a fair number of TB's live. */
461 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
462
463 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
464 indicated, this is constrained by the range of direct branches on the
465 host cpu, as used by the TCG implementation of goto_tb. */
466 #if defined(__x86_64__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
468 #elif defined(__sparc__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
470 #elif defined(__aarch64__)
471 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
472 #elif defined(__arm__)
473 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
474 #elif defined(__s390x__)
475 /* We have a +- 4GB range on the branches; leave some slop. */
476 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
477 #else
478 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
479 #endif
480
481 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
482
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
484 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
485 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
486
487 static inline size_t size_code_gen_buffer(size_t tb_size)
488 {
489 /* Size the buffer. */
490 if (tb_size == 0) {
491 #ifdef USE_STATIC_CODE_GEN_BUFFER
492 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
493 #else
494 /* ??? Needs adjustments. */
495 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
496 static buffer, we could size this on RESERVED_VA, on the text
497 segment size of the executable, or continue to use the default. */
498 tb_size = (unsigned long)(ram_size / 4);
499 #endif
500 }
501 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
502 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
503 }
504 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
505 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
506 }
507 tcg_ctx.code_gen_buffer_size = tb_size;
508 return tb_size;
509 }
510
511 #ifdef USE_STATIC_CODE_GEN_BUFFER
512 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
513 __attribute__((aligned(CODE_GEN_ALIGN)));
514
515 static inline void *alloc_code_gen_buffer(void)
516 {
517 map_exec(static_code_gen_buffer, tcg_ctx.code_gen_buffer_size);
518 return static_code_gen_buffer;
519 }
520 #elif defined(USE_MMAP)
521 static inline void *alloc_code_gen_buffer(void)
522 {
523 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
524 uintptr_t start = 0;
525 void *buf;
526
527 /* Constrain the position of the buffer based on the host cpu.
528 Note that these addresses are chosen in concert with the
529 addresses assigned in the relevant linker script file. */
530 # if defined(__PIE__) || defined(__PIC__)
531 /* Don't bother setting a preferred location if we're building
532 a position-independent executable. We're more likely to get
533 an address near the main executable if we let the kernel
534 choose the address. */
535 # elif defined(__x86_64__) && defined(MAP_32BIT)
536 /* Force the memory down into low memory with the executable.
537 Leave the choice of exact location with the kernel. */
538 flags |= MAP_32BIT;
539 /* Cannot expect to map more than 800MB in low memory. */
540 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
541 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
542 }
543 # elif defined(__sparc__)
544 start = 0x40000000ul;
545 # elif defined(__s390x__)
546 start = 0x90000000ul;
547 # endif
548
549 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
550 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
551 return buf == MAP_FAILED ? NULL : buf;
552 }
553 #else
554 static inline void *alloc_code_gen_buffer(void)
555 {
556 void *buf = g_malloc(tcg_ctx.code_gen_buffer_size);
557
558 if (buf) {
559 map_exec(buf, tcg_ctx.code_gen_buffer_size);
560 }
561 return buf;
562 }
563 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
564
565 static inline void code_gen_alloc(size_t tb_size)
566 {
567 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
568 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
569 if (tcg_ctx.code_gen_buffer == NULL) {
570 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
571 exit(1);
572 }
573
574 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
575 QEMU_MADV_HUGEPAGE);
576
577 /* Steal room for the prologue at the end of the buffer. This ensures
578 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
579 from TB's to the prologue are going to be in range. It also means
580 that we don't need to mark (additional) portions of the data segment
581 as executable. */
582 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
583 tcg_ctx.code_gen_buffer_size - 1024;
584 tcg_ctx.code_gen_buffer_size -= 1024;
585
586 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
587 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
588 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
589 CODE_GEN_AVG_BLOCK_SIZE;
590 tcg_ctx.tb_ctx.tbs =
591 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
592 }
593
594 /* Must be called before using the QEMU cpus. 'tb_size' is the size
595 (in bytes) allocated to the translation buffer. Zero means default
596 size. */
597 void tcg_exec_init(unsigned long tb_size)
598 {
599 cpu_gen_init();
600 code_gen_alloc(tb_size);
601 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
602 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
603 page_init();
604 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
605 /* There's no guest base to take into account, so go ahead and
606 initialize the prologue now. */
607 tcg_prologue_init(&tcg_ctx);
608 #endif
609 }
610
611 bool tcg_enabled(void)
612 {
613 return tcg_ctx.code_gen_buffer != NULL;
614 }
615
616 /* Allocate a new translation block. Flush the translation buffer if
617 too many translation blocks or too much generated code. */
618 static TranslationBlock *tb_alloc(target_ulong pc)
619 {
620 TranslationBlock *tb;
621
622 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
623 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
624 tcg_ctx.code_gen_buffer_max_size) {
625 return NULL;
626 }
627 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
628 tb->pc = pc;
629 tb->cflags = 0;
630 return tb;
631 }
632
633 void tb_free(TranslationBlock *tb)
634 {
635 /* In practice this is mostly used for single use temporary TB
636 Ignore the hard cases and just back up if this TB happens to
637 be the last one generated. */
638 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
639 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
640 tcg_ctx.code_gen_ptr = tb->tc_ptr;
641 tcg_ctx.tb_ctx.nb_tbs--;
642 }
643 }
644
645 static inline void invalidate_page_bitmap(PageDesc *p)
646 {
647 if (p->code_bitmap) {
648 g_free(p->code_bitmap);
649 p->code_bitmap = NULL;
650 }
651 p->code_write_count = 0;
652 }
653
654 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
655 static void page_flush_tb_1(int level, void **lp)
656 {
657 int i;
658
659 if (*lp == NULL) {
660 return;
661 }
662 if (level == 0) {
663 PageDesc *pd = *lp;
664
665 for (i = 0; i < V_L2_SIZE; ++i) {
666 pd[i].first_tb = NULL;
667 invalidate_page_bitmap(pd + i);
668 }
669 } else {
670 void **pp = *lp;
671
672 for (i = 0; i < V_L2_SIZE; ++i) {
673 page_flush_tb_1(level - 1, pp + i);
674 }
675 }
676 }
677
678 static void page_flush_tb(void)
679 {
680 int i;
681
682 for (i = 0; i < V_L1_SIZE; i++) {
683 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
684 }
685 }
686
687 /* flush all the translation blocks */
688 /* XXX: tb_flush is currently not thread safe */
689 void tb_flush(CPUArchState *env1)
690 {
691 CPUState *cpu;
692
693 #if defined(DEBUG_FLUSH)
694 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
695 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
696 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
697 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
698 tcg_ctx.tb_ctx.nb_tbs : 0);
699 #endif
700 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
701 > tcg_ctx.code_gen_buffer_size) {
702 cpu_abort(env1, "Internal error: code buffer overflow\n");
703 }
704 tcg_ctx.tb_ctx.nb_tbs = 0;
705
706 CPU_FOREACH(cpu) {
707 CPUArchState *env = cpu->env_ptr;
708
709 memset(env->tb_jmp_cache, 0, sizeof(env->tb_jmp_cache));
710 }
711
712 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
713 page_flush_tb();
714
715 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
716 /* XXX: flush processor icache at this point if cache flush is
717 expensive */
718 tcg_ctx.tb_ctx.tb_flush_count++;
719 }
720
721 #ifdef DEBUG_TB_CHECK
722
723 static void tb_invalidate_check(target_ulong address)
724 {
725 TranslationBlock *tb;
726 int i;
727
728 address &= TARGET_PAGE_MASK;
729 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
730 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
731 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
732 address >= tb->pc + tb->size)) {
733 printf("ERROR invalidate: address=" TARGET_FMT_lx
734 " PC=%08lx size=%04x\n",
735 address, (long)tb->pc, tb->size);
736 }
737 }
738 }
739 }
740
741 /* verify that all the pages have correct rights for code */
742 static void tb_page_check(void)
743 {
744 TranslationBlock *tb;
745 int i, flags1, flags2;
746
747 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
748 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
749 tb = tb->phys_hash_next) {
750 flags1 = page_get_flags(tb->pc);
751 flags2 = page_get_flags(tb->pc + tb->size - 1);
752 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
753 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
754 (long)tb->pc, tb->size, flags1, flags2);
755 }
756 }
757 }
758 }
759
760 #endif
761
762 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
763 {
764 TranslationBlock *tb1;
765
766 for (;;) {
767 tb1 = *ptb;
768 if (tb1 == tb) {
769 *ptb = tb1->phys_hash_next;
770 break;
771 }
772 ptb = &tb1->phys_hash_next;
773 }
774 }
775
776 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
777 {
778 TranslationBlock *tb1;
779 unsigned int n1;
780
781 for (;;) {
782 tb1 = *ptb;
783 n1 = (uintptr_t)tb1 & 3;
784 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
785 if (tb1 == tb) {
786 *ptb = tb1->page_next[n1];
787 break;
788 }
789 ptb = &tb1->page_next[n1];
790 }
791 }
792
793 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
794 {
795 TranslationBlock *tb1, **ptb;
796 unsigned int n1;
797
798 ptb = &tb->jmp_next[n];
799 tb1 = *ptb;
800 if (tb1) {
801 /* find tb(n) in circular list */
802 for (;;) {
803 tb1 = *ptb;
804 n1 = (uintptr_t)tb1 & 3;
805 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
806 if (n1 == n && tb1 == tb) {
807 break;
808 }
809 if (n1 == 2) {
810 ptb = &tb1->jmp_first;
811 } else {
812 ptb = &tb1->jmp_next[n1];
813 }
814 }
815 /* now we can suppress tb(n) from the list */
816 *ptb = tb->jmp_next[n];
817
818 tb->jmp_next[n] = NULL;
819 }
820 }
821
822 /* reset the jump entry 'n' of a TB so that it is not chained to
823 another TB */
824 static inline void tb_reset_jump(TranslationBlock *tb, int n)
825 {
826 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
827 }
828
829 /* invalidate one TB */
830 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
831 {
832 CPUState *cpu;
833 PageDesc *p;
834 unsigned int h, n1;
835 tb_page_addr_t phys_pc;
836 TranslationBlock *tb1, *tb2;
837
838 /* remove the TB from the hash list */
839 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
840 h = tb_phys_hash_func(phys_pc);
841 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
842
843 /* remove the TB from the page list */
844 if (tb->page_addr[0] != page_addr) {
845 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
846 tb_page_remove(&p->first_tb, tb);
847 invalidate_page_bitmap(p);
848 }
849 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
850 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
851 tb_page_remove(&p->first_tb, tb);
852 invalidate_page_bitmap(p);
853 }
854
855 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
856
857 /* remove the TB from the hash list */
858 h = tb_jmp_cache_hash_func(tb->pc);
859 CPU_FOREACH(cpu) {
860 CPUArchState *env = cpu->env_ptr;
861
862 if (env->tb_jmp_cache[h] == tb) {
863 env->tb_jmp_cache[h] = NULL;
864 }
865 }
866
867 /* suppress this TB from the two jump lists */
868 tb_jmp_remove(tb, 0);
869 tb_jmp_remove(tb, 1);
870
871 /* suppress any remaining jumps to this TB */
872 tb1 = tb->jmp_first;
873 for (;;) {
874 n1 = (uintptr_t)tb1 & 3;
875 if (n1 == 2) {
876 break;
877 }
878 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
879 tb2 = tb1->jmp_next[n1];
880 tb_reset_jump(tb1, n1);
881 tb1->jmp_next[n1] = NULL;
882 tb1 = tb2;
883 }
884 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
885
886 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
887 }
888
889 static inline void set_bits(uint8_t *tab, int start, int len)
890 {
891 int end, mask, end1;
892
893 end = start + len;
894 tab += start >> 3;
895 mask = 0xff << (start & 7);
896 if ((start & ~7) == (end & ~7)) {
897 if (start < end) {
898 mask &= ~(0xff << (end & 7));
899 *tab |= mask;
900 }
901 } else {
902 *tab++ |= mask;
903 start = (start + 8) & ~7;
904 end1 = end & ~7;
905 while (start < end1) {
906 *tab++ = 0xff;
907 start += 8;
908 }
909 if (start < end) {
910 mask = ~(0xff << (end & 7));
911 *tab |= mask;
912 }
913 }
914 }
915
916 static void build_page_bitmap(PageDesc *p)
917 {
918 int n, tb_start, tb_end;
919 TranslationBlock *tb;
920
921 p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
922
923 tb = p->first_tb;
924 while (tb != NULL) {
925 n = (uintptr_t)tb & 3;
926 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
927 /* NOTE: this is subtle as a TB may span two physical pages */
928 if (n == 0) {
929 /* NOTE: tb_end may be after the end of the page, but
930 it is not a problem */
931 tb_start = tb->pc & ~TARGET_PAGE_MASK;
932 tb_end = tb_start + tb->size;
933 if (tb_end > TARGET_PAGE_SIZE) {
934 tb_end = TARGET_PAGE_SIZE;
935 }
936 } else {
937 tb_start = 0;
938 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
939 }
940 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
941 tb = tb->page_next[n];
942 }
943 }
944
945 TranslationBlock *tb_gen_code(CPUArchState *env,
946 target_ulong pc, target_ulong cs_base,
947 int flags, int cflags)
948 {
949 TranslationBlock *tb;
950 uint8_t *tc_ptr;
951 tb_page_addr_t phys_pc, phys_page2;
952 target_ulong virt_page2;
953 int code_gen_size;
954
955 phys_pc = get_page_addr_code(env, pc);
956 tb = tb_alloc(pc);
957 if (!tb) {
958 /* flush must be done */
959 tb_flush(env);
960 /* cannot fail at this point */
961 tb = tb_alloc(pc);
962 /* Don't forget to invalidate previous TB info. */
963 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
964 }
965 tc_ptr = tcg_ctx.code_gen_ptr;
966 tb->tc_ptr = tc_ptr;
967 tb->cs_base = cs_base;
968 tb->flags = flags;
969 tb->cflags = cflags;
970 cpu_gen_code(env, tb, &code_gen_size);
971 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
972 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
973
974 /* check next page if needed */
975 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
976 phys_page2 = -1;
977 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
978 phys_page2 = get_page_addr_code(env, virt_page2);
979 }
980 tb_link_page(tb, phys_pc, phys_page2);
981 return tb;
982 }
983
984 /*
985 * Invalidate all TBs which intersect with the target physical address range
986 * [start;end[. NOTE: start and end may refer to *different* physical pages.
987 * 'is_cpu_write_access' should be true if called from a real cpu write
988 * access: the virtual CPU will exit the current TB if code is modified inside
989 * this TB.
990 */
991 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
992 int is_cpu_write_access)
993 {
994 while (start < end) {
995 tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
996 start &= TARGET_PAGE_MASK;
997 start += TARGET_PAGE_SIZE;
998 }
999 }
1000
1001 /*
1002 * Invalidate all TBs which intersect with the target physical address range
1003 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1004 * 'is_cpu_write_access' should be true if called from a real cpu write
1005 * access: the virtual CPU will exit the current TB if code is modified inside
1006 * this TB.
1007 */
1008 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1009 int is_cpu_write_access)
1010 {
1011 TranslationBlock *tb, *tb_next, *saved_tb;
1012 CPUState *cpu = current_cpu;
1013 #if defined(TARGET_HAS_PRECISE_SMC) || !defined(CONFIG_USER_ONLY)
1014 CPUArchState *env = NULL;
1015 #endif
1016 tb_page_addr_t tb_start, tb_end;
1017 PageDesc *p;
1018 int n;
1019 #ifdef TARGET_HAS_PRECISE_SMC
1020 int current_tb_not_found = is_cpu_write_access;
1021 TranslationBlock *current_tb = NULL;
1022 int current_tb_modified = 0;
1023 target_ulong current_pc = 0;
1024 target_ulong current_cs_base = 0;
1025 int current_flags = 0;
1026 #endif /* TARGET_HAS_PRECISE_SMC */
1027
1028 p = page_find(start >> TARGET_PAGE_BITS);
1029 if (!p) {
1030 return;
1031 }
1032 if (!p->code_bitmap &&
1033 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1034 is_cpu_write_access) {
1035 /* build code bitmap */
1036 build_page_bitmap(p);
1037 }
1038 #if defined(TARGET_HAS_PRECISE_SMC) || !defined(CONFIG_USER_ONLY)
1039 if (cpu != NULL) {
1040 env = cpu->env_ptr;
1041 }
1042 #endif
1043
1044 /* we remove all the TBs in the range [start, end[ */
1045 /* XXX: see if in some cases it could be faster to invalidate all
1046 the code */
1047 tb = p->first_tb;
1048 while (tb != NULL) {
1049 n = (uintptr_t)tb & 3;
1050 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1051 tb_next = tb->page_next[n];
1052 /* NOTE: this is subtle as a TB may span two physical pages */
1053 if (n == 0) {
1054 /* NOTE: tb_end may be after the end of the page, but
1055 it is not a problem */
1056 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1057 tb_end = tb_start + tb->size;
1058 } else {
1059 tb_start = tb->page_addr[1];
1060 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1061 }
1062 if (!(tb_end <= start || tb_start >= end)) {
1063 #ifdef TARGET_HAS_PRECISE_SMC
1064 if (current_tb_not_found) {
1065 current_tb_not_found = 0;
1066 current_tb = NULL;
1067 if (cpu->mem_io_pc) {
1068 /* now we have a real cpu fault */
1069 current_tb = tb_find_pc(cpu->mem_io_pc);
1070 }
1071 }
1072 if (current_tb == tb &&
1073 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1074 /* If we are modifying the current TB, we must stop
1075 its execution. We could be more precise by checking
1076 that the modification is after the current PC, but it
1077 would require a specialized function to partially
1078 restore the CPU state */
1079
1080 current_tb_modified = 1;
1081 cpu_restore_state_from_tb(current_tb, env, cpu->mem_io_pc);
1082 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1083 &current_flags);
1084 }
1085 #endif /* TARGET_HAS_PRECISE_SMC */
1086 /* we need to do that to handle the case where a signal
1087 occurs while doing tb_phys_invalidate() */
1088 saved_tb = NULL;
1089 if (cpu != NULL) {
1090 saved_tb = cpu->current_tb;
1091 cpu->current_tb = NULL;
1092 }
1093 tb_phys_invalidate(tb, -1);
1094 if (cpu != NULL) {
1095 cpu->current_tb = saved_tb;
1096 if (cpu->interrupt_request && cpu->current_tb) {
1097 cpu_interrupt(cpu, cpu->interrupt_request);
1098 }
1099 }
1100 }
1101 tb = tb_next;
1102 }
1103 #if !defined(CONFIG_USER_ONLY)
1104 /* if no code remaining, no need to continue to use slow writes */
1105 if (!p->first_tb) {
1106 invalidate_page_bitmap(p);
1107 if (is_cpu_write_access) {
1108 tlb_unprotect_code_phys(env, start, cpu->mem_io_vaddr);
1109 }
1110 }
1111 #endif
1112 #ifdef TARGET_HAS_PRECISE_SMC
1113 if (current_tb_modified) {
1114 /* we generate a block containing just the instruction
1115 modifying the memory. It will ensure that it cannot modify
1116 itself */
1117 cpu->current_tb = NULL;
1118 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1119 cpu_resume_from_signal(env, NULL);
1120 }
1121 #endif
1122 }
1123
1124 /* len must be <= 8 and start must be a multiple of len */
1125 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1126 {
1127 PageDesc *p;
1128 int offset, b;
1129
1130 #if 0
1131 if (1) {
1132 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1133 cpu_single_env->mem_io_vaddr, len,
1134 cpu_single_env->eip,
1135 cpu_single_env->eip +
1136 (intptr_t)cpu_single_env->segs[R_CS].base);
1137 }
1138 #endif
1139 p = page_find(start >> TARGET_PAGE_BITS);
1140 if (!p) {
1141 return;
1142 }
1143 if (p->code_bitmap) {
1144 offset = start & ~TARGET_PAGE_MASK;
1145 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1146 if (b & ((1 << len) - 1)) {
1147 goto do_invalidate;
1148 }
1149 } else {
1150 do_invalidate:
1151 tb_invalidate_phys_page_range(start, start + len, 1);
1152 }
1153 }
1154
1155 #if !defined(CONFIG_SOFTMMU)
1156 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1157 uintptr_t pc, void *puc,
1158 bool locked)
1159 {
1160 TranslationBlock *tb;
1161 PageDesc *p;
1162 int n;
1163 #ifdef TARGET_HAS_PRECISE_SMC
1164 TranslationBlock *current_tb = NULL;
1165 CPUState *cpu = current_cpu;
1166 CPUArchState *env = NULL;
1167 int current_tb_modified = 0;
1168 target_ulong current_pc = 0;
1169 target_ulong current_cs_base = 0;
1170 int current_flags = 0;
1171 #endif
1172
1173 addr &= TARGET_PAGE_MASK;
1174 p = page_find(addr >> TARGET_PAGE_BITS);
1175 if (!p) {
1176 return;
1177 }
1178 tb = p->first_tb;
1179 #ifdef TARGET_HAS_PRECISE_SMC
1180 if (tb && pc != 0) {
1181 current_tb = tb_find_pc(pc);
1182 }
1183 if (cpu != NULL) {
1184 env = cpu->env_ptr;
1185 }
1186 #endif
1187 while (tb != NULL) {
1188 n = (uintptr_t)tb & 3;
1189 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1190 #ifdef TARGET_HAS_PRECISE_SMC
1191 if (current_tb == tb &&
1192 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1193 /* If we are modifying the current TB, we must stop
1194 its execution. We could be more precise by checking
1195 that the modification is after the current PC, but it
1196 would require a specialized function to partially
1197 restore the CPU state */
1198
1199 current_tb_modified = 1;
1200 cpu_restore_state_from_tb(current_tb, env, pc);
1201 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1202 &current_flags);
1203 }
1204 #endif /* TARGET_HAS_PRECISE_SMC */
1205 tb_phys_invalidate(tb, addr);
1206 tb = tb->page_next[n];
1207 }
1208 p->first_tb = NULL;
1209 #ifdef TARGET_HAS_PRECISE_SMC
1210 if (current_tb_modified) {
1211 /* we generate a block containing just the instruction
1212 modifying the memory. It will ensure that it cannot modify
1213 itself */
1214 cpu->current_tb = NULL;
1215 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1216 if (locked) {
1217 mmap_unlock();
1218 }
1219 cpu_resume_from_signal(env, puc);
1220 }
1221 #endif
1222 }
1223 #endif
1224
1225 /* add the tb in the target page and protect it if necessary */
1226 static inline void tb_alloc_page(TranslationBlock *tb,
1227 unsigned int n, tb_page_addr_t page_addr)
1228 {
1229 PageDesc *p;
1230 #ifndef CONFIG_USER_ONLY
1231 bool page_already_protected;
1232 #endif
1233
1234 tb->page_addr[n] = page_addr;
1235 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1236 tb->page_next[n] = p->first_tb;
1237 #ifndef CONFIG_USER_ONLY
1238 page_already_protected = p->first_tb != NULL;
1239 #endif
1240 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1241 invalidate_page_bitmap(p);
1242
1243 #if defined(TARGET_HAS_SMC) || 1
1244
1245 #if defined(CONFIG_USER_ONLY)
1246 if (p->flags & PAGE_WRITE) {
1247 target_ulong addr;
1248 PageDesc *p2;
1249 int prot;
1250
1251 /* force the host page as non writable (writes will have a
1252 page fault + mprotect overhead) */
1253 page_addr &= qemu_host_page_mask;
1254 prot = 0;
1255 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1256 addr += TARGET_PAGE_SIZE) {
1257
1258 p2 = page_find(addr >> TARGET_PAGE_BITS);
1259 if (!p2) {
1260 continue;
1261 }
1262 prot |= p2->flags;
1263 p2->flags &= ~PAGE_WRITE;
1264 }
1265 mprotect(g2h(page_addr), qemu_host_page_size,
1266 (prot & PAGE_BITS) & ~PAGE_WRITE);
1267 #ifdef DEBUG_TB_INVALIDATE
1268 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1269 page_addr);
1270 #endif
1271 }
1272 #else
1273 /* if some code is already present, then the pages are already
1274 protected. So we handle the case where only the first TB is
1275 allocated in a physical page */
1276 if (!page_already_protected) {
1277 tlb_protect_code(page_addr);
1278 }
1279 #endif
1280
1281 #endif /* TARGET_HAS_SMC */
1282 }
1283
1284 /* add a new TB and link it to the physical page tables. phys_page2 is
1285 (-1) to indicate that only one page contains the TB. */
1286 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1287 tb_page_addr_t phys_page2)
1288 {
1289 unsigned int h;
1290 TranslationBlock **ptb;
1291
1292 /* Grab the mmap lock to stop another thread invalidating this TB
1293 before we are done. */
1294 mmap_lock();
1295 /* add in the physical hash table */
1296 h = tb_phys_hash_func(phys_pc);
1297 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1298 tb->phys_hash_next = *ptb;
1299 *ptb = tb;
1300
1301 /* add in the page list */
1302 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1303 if (phys_page2 != -1) {
1304 tb_alloc_page(tb, 1, phys_page2);
1305 } else {
1306 tb->page_addr[1] = -1;
1307 }
1308
1309 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1310 tb->jmp_next[0] = NULL;
1311 tb->jmp_next[1] = NULL;
1312
1313 /* init original jump addresses */
1314 if (tb->tb_next_offset[0] != 0xffff) {
1315 tb_reset_jump(tb, 0);
1316 }
1317 if (tb->tb_next_offset[1] != 0xffff) {
1318 tb_reset_jump(tb, 1);
1319 }
1320
1321 #ifdef DEBUG_TB_CHECK
1322 tb_page_check();
1323 #endif
1324 mmap_unlock();
1325 }
1326
1327 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1328 tb[1].tc_ptr. Return NULL if not found */
1329 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1330 {
1331 int m_min, m_max, m;
1332 uintptr_t v;
1333 TranslationBlock *tb;
1334
1335 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1336 return NULL;
1337 }
1338 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1339 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1340 return NULL;
1341 }
1342 /* binary search (cf Knuth) */
1343 m_min = 0;
1344 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1345 while (m_min <= m_max) {
1346 m = (m_min + m_max) >> 1;
1347 tb = &tcg_ctx.tb_ctx.tbs[m];
1348 v = (uintptr_t)tb->tc_ptr;
1349 if (v == tc_ptr) {
1350 return tb;
1351 } else if (tc_ptr < v) {
1352 m_max = m - 1;
1353 } else {
1354 m_min = m + 1;
1355 }
1356 }
1357 return &tcg_ctx.tb_ctx.tbs[m_max];
1358 }
1359
1360 #if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
1361 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1362 {
1363 ram_addr_t ram_addr;
1364 MemoryRegion *mr;
1365 hwaddr l = 1;
1366
1367 mr = address_space_translate(as, addr, &addr, &l, false);
1368 if (!(memory_region_is_ram(mr)
1369 || memory_region_is_romd(mr))) {
1370 return;
1371 }
1372 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
1373 + addr;
1374 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1375 }
1376 #endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
1377
1378 void tb_check_watchpoint(CPUArchState *env)
1379 {
1380 CPUState *cpu = ENV_GET_CPU(env);
1381 TranslationBlock *tb;
1382
1383 tb = tb_find_pc(cpu->mem_io_pc);
1384 if (!tb) {
1385 cpu_abort(env, "check_watchpoint: could not find TB for pc=%p",
1386 (void *)cpu->mem_io_pc);
1387 }
1388 cpu_restore_state_from_tb(tb, env, cpu->mem_io_pc);
1389 tb_phys_invalidate(tb, -1);
1390 }
1391
1392 #ifndef CONFIG_USER_ONLY
1393 /* mask must never be zero, except for A20 change call */
1394 static void tcg_handle_interrupt(CPUState *cpu, int mask)
1395 {
1396 CPUArchState *env = cpu->env_ptr;
1397 int old_mask;
1398
1399 old_mask = cpu->interrupt_request;
1400 cpu->interrupt_request |= mask;
1401
1402 /*
1403 * If called from iothread context, wake the target cpu in
1404 * case its halted.
1405 */
1406 if (!qemu_cpu_is_self(cpu)) {
1407 qemu_cpu_kick(cpu);
1408 return;
1409 }
1410
1411 if (use_icount) {
1412 cpu->icount_decr.u16.high = 0xffff;
1413 if (!cpu_can_do_io(cpu)
1414 && (mask & ~old_mask) != 0) {
1415 cpu_abort(env, "Raised interrupt while not in I/O function");
1416 }
1417 } else {
1418 cpu->tcg_exit_req = 1;
1419 }
1420 }
1421
1422 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1423
1424 /* in deterministic execution mode, instructions doing device I/Os
1425 must be at the end of the TB */
1426 void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
1427 {
1428 CPUState *cpu = ENV_GET_CPU(env);
1429 TranslationBlock *tb;
1430 uint32_t n, cflags;
1431 target_ulong pc, cs_base;
1432 uint64_t flags;
1433
1434 tb = tb_find_pc(retaddr);
1435 if (!tb) {
1436 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
1437 (void *)retaddr);
1438 }
1439 n = cpu->icount_decr.u16.low + tb->icount;
1440 cpu_restore_state_from_tb(tb, env, retaddr);
1441 /* Calculate how many instructions had been executed before the fault
1442 occurred. */
1443 n = n - cpu->icount_decr.u16.low;
1444 /* Generate a new TB ending on the I/O insn. */
1445 n++;
1446 /* On MIPS and SH, delay slot instructions can only be restarted if
1447 they were already the first instruction in the TB. If this is not
1448 the first instruction in a TB then re-execute the preceding
1449 branch. */
1450 #if defined(TARGET_MIPS)
1451 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1452 env->active_tc.PC -= 4;
1453 cpu->icount_decr.u16.low++;
1454 env->hflags &= ~MIPS_HFLAG_BMASK;
1455 }
1456 #elif defined(TARGET_SH4)
1457 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1458 && n > 1) {
1459 env->pc -= 2;
1460 cpu->icount_decr.u16.low++;
1461 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1462 }
1463 #endif
1464 /* This should never happen. */
1465 if (n > CF_COUNT_MASK) {
1466 cpu_abort(env, "TB too big during recompile");
1467 }
1468
1469 cflags = n | CF_LAST_IO;
1470 pc = tb->pc;
1471 cs_base = tb->cs_base;
1472 flags = tb->flags;
1473 tb_phys_invalidate(tb, -1);
1474 /* FIXME: In theory this could raise an exception. In practice
1475 we have already translated the block once so it's probably ok. */
1476 tb_gen_code(env, pc, cs_base, flags, cflags);
1477 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1478 the first in the TB) then we end up generating a whole new TB and
1479 repeating the fault, which is horribly inefficient.
1480 Better would be to execute just this insn uncached, or generate a
1481 second new TB. */
1482 cpu_resume_from_signal(env, NULL);
1483 }
1484
1485 void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
1486 {
1487 unsigned int i;
1488
1489 /* Discard jump cache entries for any tb which might potentially
1490 overlap the flushed page. */
1491 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1492 memset(&env->tb_jmp_cache[i], 0,
1493 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1494
1495 i = tb_jmp_cache_hash_page(addr);
1496 memset(&env->tb_jmp_cache[i], 0,
1497 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1498 }
1499
1500 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1501 {
1502 int i, target_code_size, max_target_code_size;
1503 int direct_jmp_count, direct_jmp2_count, cross_page;
1504 TranslationBlock *tb;
1505
1506 target_code_size = 0;
1507 max_target_code_size = 0;
1508 cross_page = 0;
1509 direct_jmp_count = 0;
1510 direct_jmp2_count = 0;
1511 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1512 tb = &tcg_ctx.tb_ctx.tbs[i];
1513 target_code_size += tb->size;
1514 if (tb->size > max_target_code_size) {
1515 max_target_code_size = tb->size;
1516 }
1517 if (tb->page_addr[1] != -1) {
1518 cross_page++;
1519 }
1520 if (tb->tb_next_offset[0] != 0xffff) {
1521 direct_jmp_count++;
1522 if (tb->tb_next_offset[1] != 0xffff) {
1523 direct_jmp2_count++;
1524 }
1525 }
1526 }
1527 /* XXX: avoid using doubles ? */
1528 cpu_fprintf(f, "Translation buffer state:\n");
1529 cpu_fprintf(f, "gen code size %td/%zd\n",
1530 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1531 tcg_ctx.code_gen_buffer_max_size);
1532 cpu_fprintf(f, "TB count %d/%d\n",
1533 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1534 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1535 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1536 tcg_ctx.tb_ctx.nb_tbs : 0,
1537 max_target_code_size);
1538 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1539 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1540 tcg_ctx.code_gen_buffer) /
1541 tcg_ctx.tb_ctx.nb_tbs : 0,
1542 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1543 tcg_ctx.code_gen_buffer) /
1544 target_code_size : 0);
1545 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1546 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1547 tcg_ctx.tb_ctx.nb_tbs : 0);
1548 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1549 direct_jmp_count,
1550 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1551 tcg_ctx.tb_ctx.nb_tbs : 0,
1552 direct_jmp2_count,
1553 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1554 tcg_ctx.tb_ctx.nb_tbs : 0);
1555 cpu_fprintf(f, "\nStatistics:\n");
1556 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1557 cpu_fprintf(f, "TB invalidate count %d\n",
1558 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1559 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1560 tcg_dump_info(f, cpu_fprintf);
1561 }
1562
1563 #else /* CONFIG_USER_ONLY */
1564
1565 void cpu_interrupt(CPUState *cpu, int mask)
1566 {
1567 cpu->interrupt_request |= mask;
1568 cpu->tcg_exit_req = 1;
1569 }
1570
1571 /*
1572 * Walks guest process memory "regions" one by one
1573 * and calls callback function 'fn' for each region.
1574 */
1575 struct walk_memory_regions_data {
1576 walk_memory_regions_fn fn;
1577 void *priv;
1578 uintptr_t start;
1579 int prot;
1580 };
1581
1582 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1583 abi_ulong end, int new_prot)
1584 {
1585 if (data->start != -1ul) {
1586 int rc = data->fn(data->priv, data->start, end, data->prot);
1587 if (rc != 0) {
1588 return rc;
1589 }
1590 }
1591
1592 data->start = (new_prot ? end : -1ul);
1593 data->prot = new_prot;
1594
1595 return 0;
1596 }
1597
1598 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1599 abi_ulong base, int level, void **lp)
1600 {
1601 abi_ulong pa;
1602 int i, rc;
1603
1604 if (*lp == NULL) {
1605 return walk_memory_regions_end(data, base, 0);
1606 }
1607
1608 if (level == 0) {
1609 PageDesc *pd = *lp;
1610
1611 for (i = 0; i < V_L2_SIZE; ++i) {
1612 int prot = pd[i].flags;
1613
1614 pa = base | (i << TARGET_PAGE_BITS);
1615 if (prot != data->prot) {
1616 rc = walk_memory_regions_end(data, pa, prot);
1617 if (rc != 0) {
1618 return rc;
1619 }
1620 }
1621 }
1622 } else {
1623 void **pp = *lp;
1624
1625 for (i = 0; i < V_L2_SIZE; ++i) {
1626 pa = base | ((abi_ulong)i <<
1627 (TARGET_PAGE_BITS + V_L2_BITS * level));
1628 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1629 if (rc != 0) {
1630 return rc;
1631 }
1632 }
1633 }
1634
1635 return 0;
1636 }
1637
1638 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1639 {
1640 struct walk_memory_regions_data data;
1641 uintptr_t i;
1642
1643 data.fn = fn;
1644 data.priv = priv;
1645 data.start = -1ul;
1646 data.prot = 0;
1647
1648 for (i = 0; i < V_L1_SIZE; i++) {
1649 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
1650 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1651
1652 if (rc != 0) {
1653 return rc;
1654 }
1655 }
1656
1657 return walk_memory_regions_end(&data, 0, 0);
1658 }
1659
1660 static int dump_region(void *priv, abi_ulong start,
1661 abi_ulong end, unsigned long prot)
1662 {
1663 FILE *f = (FILE *)priv;
1664
1665 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
1666 " "TARGET_ABI_FMT_lx" %c%c%c\n",
1667 start, end, end - start,
1668 ((prot & PAGE_READ) ? 'r' : '-'),
1669 ((prot & PAGE_WRITE) ? 'w' : '-'),
1670 ((prot & PAGE_EXEC) ? 'x' : '-'));
1671
1672 return 0;
1673 }
1674
1675 /* dump memory mappings */
1676 void page_dump(FILE *f)
1677 {
1678 const int length = sizeof(abi_ulong) * 2;
1679 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1680 length, "start", length, "end", length, "size", "prot");
1681 walk_memory_regions(f, dump_region);
1682 }
1683
1684 int page_get_flags(target_ulong address)
1685 {
1686 PageDesc *p;
1687
1688 p = page_find(address >> TARGET_PAGE_BITS);
1689 if (!p) {
1690 return 0;
1691 }
1692 return p->flags;
1693 }
1694
1695 /* Modify the flags of a page and invalidate the code if necessary.
1696 The flag PAGE_WRITE_ORG is positioned automatically depending
1697 on PAGE_WRITE. The mmap_lock should already be held. */
1698 void page_set_flags(target_ulong start, target_ulong end, int flags)
1699 {
1700 target_ulong addr, len;
1701
1702 /* This function should never be called with addresses outside the
1703 guest address space. If this assert fires, it probably indicates
1704 a missing call to h2g_valid. */
1705 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1706 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1707 #endif
1708 assert(start < end);
1709
1710 start = start & TARGET_PAGE_MASK;
1711 end = TARGET_PAGE_ALIGN(end);
1712
1713 if (flags & PAGE_WRITE) {
1714 flags |= PAGE_WRITE_ORG;
1715 }
1716
1717 for (addr = start, len = end - start;
1718 len != 0;
1719 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1720 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1721
1722 /* If the write protection bit is set, then we invalidate
1723 the code inside. */
1724 if (!(p->flags & PAGE_WRITE) &&
1725 (flags & PAGE_WRITE) &&
1726 p->first_tb) {
1727 tb_invalidate_phys_page(addr, 0, NULL, false);
1728 }
1729 p->flags = flags;
1730 }
1731 }
1732
1733 int page_check_range(target_ulong start, target_ulong len, int flags)
1734 {
1735 PageDesc *p;
1736 target_ulong end;
1737 target_ulong addr;
1738
1739 /* This function should never be called with addresses outside the
1740 guest address space. If this assert fires, it probably indicates
1741 a missing call to h2g_valid. */
1742 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1743 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1744 #endif
1745
1746 if (len == 0) {
1747 return 0;
1748 }
1749 if (start + len - 1 < start) {
1750 /* We've wrapped around. */
1751 return -1;
1752 }
1753
1754 /* must do before we loose bits in the next step */
1755 end = TARGET_PAGE_ALIGN(start + len);
1756 start = start & TARGET_PAGE_MASK;
1757
1758 for (addr = start, len = end - start;
1759 len != 0;
1760 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1761 p = page_find(addr >> TARGET_PAGE_BITS);
1762 if (!p) {
1763 return -1;
1764 }
1765 if (!(p->flags & PAGE_VALID)) {
1766 return -1;
1767 }
1768
1769 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1770 return -1;
1771 }
1772 if (flags & PAGE_WRITE) {
1773 if (!(p->flags & PAGE_WRITE_ORG)) {
1774 return -1;
1775 }
1776 /* unprotect the page if it was put read-only because it
1777 contains translated code */
1778 if (!(p->flags & PAGE_WRITE)) {
1779 if (!page_unprotect(addr, 0, NULL)) {
1780 return -1;
1781 }
1782 }
1783 return 0;
1784 }
1785 }
1786 return 0;
1787 }
1788
1789 /* called from signal handler: invalidate the code and unprotect the
1790 page. Return TRUE if the fault was successfully handled. */
1791 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1792 {
1793 unsigned int prot;
1794 PageDesc *p;
1795 target_ulong host_start, host_end, addr;
1796
1797 /* Technically this isn't safe inside a signal handler. However we
1798 know this only ever happens in a synchronous SEGV handler, so in
1799 practice it seems to be ok. */
1800 mmap_lock();
1801
1802 p = page_find(address >> TARGET_PAGE_BITS);
1803 if (!p) {
1804 mmap_unlock();
1805 return 0;
1806 }
1807
1808 /* if the page was really writable, then we change its
1809 protection back to writable */
1810 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1811 host_start = address & qemu_host_page_mask;
1812 host_end = host_start + qemu_host_page_size;
1813
1814 prot = 0;
1815 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1816 p = page_find(addr >> TARGET_PAGE_BITS);
1817 p->flags |= PAGE_WRITE;
1818 prot |= p->flags;
1819
1820 /* and since the content will be modified, we must invalidate
1821 the corresponding translated code. */
1822 tb_invalidate_phys_page(addr, pc, puc, true);
1823 #ifdef DEBUG_TB_CHECK
1824 tb_invalidate_check(addr);
1825 #endif
1826 }
1827 mprotect((void *)g2h(host_start), qemu_host_page_size,
1828 prot & PAGE_BITS);
1829
1830 mmap_unlock();
1831 return 1;
1832 }
1833 mmap_unlock();
1834 return 0;
1835 }
1836 #endif /* CONFIG_USER_ONLY */