hw/net: fix mcf_fec driver receiver
[qemu.git] / translate-all.c
1 /*
2 * Host code generation
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/types.h>
23 #include <sys/mman.h>
24 #endif
25 #include <stdarg.h>
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <inttypes.h>
30
31 #include "config.h"
32
33 #include "qemu-common.h"
34 #define NO_CPU_IO_DEFS
35 #include "cpu.h"
36 #include "trace.h"
37 #include "disas/disas.h"
38 #include "tcg.h"
39 #if defined(CONFIG_USER_ONLY)
40 #include "qemu.h"
41 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
42 #include <sys/param.h>
43 #if __FreeBSD_version >= 700104
44 #define HAVE_KINFO_GETVMMAP
45 #define sigqueue sigqueue_freebsd /* avoid redefinition */
46 #include <sys/time.h>
47 #include <sys/proc.h>
48 #include <machine/profile.h>
49 #define _KERNEL
50 #include <sys/user.h>
51 #undef _KERNEL
52 #undef sigqueue
53 #include <libutil.h>
54 #endif
55 #endif
56 #else
57 #include "exec/address-spaces.h"
58 #endif
59
60 #include "exec/cputlb.h"
61 #include "exec/tb-hash.h"
62 #include "translate-all.h"
63 #include "qemu/bitmap.h"
64 #include "qemu/timer.h"
65
66 //#define DEBUG_TB_INVALIDATE
67 //#define DEBUG_FLUSH
68 /* make various TB consistency checks */
69 //#define DEBUG_TB_CHECK
70
71 #if !defined(CONFIG_USER_ONLY)
72 /* TB consistency checks only implemented for usermode emulation. */
73 #undef DEBUG_TB_CHECK
74 #endif
75
76 #define SMC_BITMAP_USE_THRESHOLD 10
77
78 typedef struct PageDesc {
79 /* list of TBs intersecting this ram page */
80 TranslationBlock *first_tb;
81 /* in order to optimize self modifying code, we count the number
82 of lookups we do to a given page to use a bitmap */
83 unsigned int code_write_count;
84 unsigned long *code_bitmap;
85 #if defined(CONFIG_USER_ONLY)
86 unsigned long flags;
87 #endif
88 } PageDesc;
89
90 /* In system mode we want L1_MAP to be based on ram offsets,
91 while in user mode we want it to be based on virtual addresses. */
92 #if !defined(CONFIG_USER_ONLY)
93 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
94 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
95 #else
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
97 #endif
98 #else
99 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
100 #endif
101
102 /* Size of the L2 (and L3, etc) page tables. */
103 #define V_L2_BITS 10
104 #define V_L2_SIZE (1 << V_L2_BITS)
105
106 /* The bits remaining after N lower levels of page tables. */
107 #define V_L1_BITS_REM \
108 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
109
110 #if V_L1_BITS_REM < 4
111 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
112 #else
113 #define V_L1_BITS V_L1_BITS_REM
114 #endif
115
116 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
117
118 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
119
120 uintptr_t qemu_real_host_page_size;
121 uintptr_t qemu_real_host_page_mask;
122 uintptr_t qemu_host_page_size;
123 uintptr_t qemu_host_page_mask;
124
125 /* This is a multi-level map on the virtual address space.
126 The bottom level has pointers to PageDesc. */
127 static void *l1_map[V_L1_SIZE];
128
129 /* code generation context */
130 TCGContext tcg_ctx;
131
132 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
133 tb_page_addr_t phys_page2);
134 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
135
136 void cpu_gen_init(void)
137 {
138 tcg_context_init(&tcg_ctx);
139 }
140
141 /* return non zero if the very first instruction is invalid so that
142 the virtual CPU can trigger an exception.
143
144 '*gen_code_size_ptr' contains the size of the generated code (host
145 code).
146 */
147 int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
148 {
149 TCGContext *s = &tcg_ctx;
150 tcg_insn_unit *gen_code_buf;
151 int gen_code_size;
152 #ifdef CONFIG_PROFILER
153 int64_t ti;
154 #endif
155
156 #ifdef CONFIG_PROFILER
157 s->tb_count1++; /* includes aborted translations because of
158 exceptions */
159 ti = profile_getclock();
160 #endif
161 tcg_func_start(s);
162
163 gen_intermediate_code(env, tb);
164
165 trace_translate_block(tb, tb->pc, tb->tc_ptr);
166
167 /* generate machine code */
168 gen_code_buf = tb->tc_ptr;
169 tb->tb_next_offset[0] = 0xffff;
170 tb->tb_next_offset[1] = 0xffff;
171 s->tb_next_offset = tb->tb_next_offset;
172 #ifdef USE_DIRECT_JUMP
173 s->tb_jmp_offset = tb->tb_jmp_offset;
174 s->tb_next = NULL;
175 #else
176 s->tb_jmp_offset = NULL;
177 s->tb_next = tb->tb_next;
178 #endif
179
180 #ifdef CONFIG_PROFILER
181 s->tb_count++;
182 s->interm_time += profile_getclock() - ti;
183 s->code_time -= profile_getclock();
184 #endif
185 gen_code_size = tcg_gen_code(s, gen_code_buf);
186 *gen_code_size_ptr = gen_code_size;
187 #ifdef CONFIG_PROFILER
188 s->code_time += profile_getclock();
189 s->code_in_len += tb->size;
190 s->code_out_len += gen_code_size;
191 #endif
192
193 #ifdef DEBUG_DISAS
194 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
195 qemu_log("OUT: [size=%d]\n", gen_code_size);
196 log_disas(tb->tc_ptr, gen_code_size);
197 qemu_log("\n");
198 qemu_log_flush();
199 }
200 #endif
201 return 0;
202 }
203
204 /* The cpu state corresponding to 'searched_pc' is restored.
205 */
206 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
207 uintptr_t searched_pc)
208 {
209 CPUArchState *env = cpu->env_ptr;
210 TCGContext *s = &tcg_ctx;
211 int j;
212 uintptr_t tc_ptr;
213 #ifdef CONFIG_PROFILER
214 int64_t ti;
215 #endif
216
217 #ifdef CONFIG_PROFILER
218 ti = profile_getclock();
219 #endif
220 tcg_func_start(s);
221
222 gen_intermediate_code_pc(env, tb);
223
224 if (tb->cflags & CF_USE_ICOUNT) {
225 /* Reset the cycle counter to the start of the block. */
226 cpu->icount_decr.u16.low += tb->icount;
227 /* Clear the IO flag. */
228 cpu->can_do_io = 0;
229 }
230
231 /* find opc index corresponding to search_pc */
232 tc_ptr = (uintptr_t)tb->tc_ptr;
233 if (searched_pc < tc_ptr)
234 return -1;
235
236 s->tb_next_offset = tb->tb_next_offset;
237 #ifdef USE_DIRECT_JUMP
238 s->tb_jmp_offset = tb->tb_jmp_offset;
239 s->tb_next = NULL;
240 #else
241 s->tb_jmp_offset = NULL;
242 s->tb_next = tb->tb_next;
243 #endif
244 j = tcg_gen_code_search_pc(s, (tcg_insn_unit *)tc_ptr,
245 searched_pc - tc_ptr);
246 if (j < 0)
247 return -1;
248 /* now find start of instruction before */
249 while (s->gen_opc_instr_start[j] == 0) {
250 j--;
251 }
252 cpu->icount_decr.u16.low -= s->gen_opc_icount[j];
253
254 restore_state_to_opc(env, tb, j);
255
256 #ifdef CONFIG_PROFILER
257 s->restore_time += profile_getclock() - ti;
258 s->restore_count++;
259 #endif
260 return 0;
261 }
262
263 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
264 {
265 TranslationBlock *tb;
266
267 tb = tb_find_pc(retaddr);
268 if (tb) {
269 cpu_restore_state_from_tb(cpu, tb, retaddr);
270 if (tb->cflags & CF_NOCACHE) {
271 /* one-shot translation, invalidate it immediately */
272 cpu->current_tb = NULL;
273 tb_phys_invalidate(tb, -1);
274 tb_free(tb);
275 }
276 return true;
277 }
278 return false;
279 }
280
281 #ifdef _WIN32
282 static __attribute__((unused)) void map_exec(void *addr, long size)
283 {
284 DWORD old_protect;
285 VirtualProtect(addr, size,
286 PAGE_EXECUTE_READWRITE, &old_protect);
287 }
288 #else
289 static __attribute__((unused)) void map_exec(void *addr, long size)
290 {
291 unsigned long start, end, page_size;
292
293 page_size = getpagesize();
294 start = (unsigned long)addr;
295 start &= ~(page_size - 1);
296
297 end = (unsigned long)addr + size;
298 end += page_size - 1;
299 end &= ~(page_size - 1);
300
301 mprotect((void *)start, end - start,
302 PROT_READ | PROT_WRITE | PROT_EXEC);
303 }
304 #endif
305
306 void page_size_init(void)
307 {
308 /* NOTE: we can always suppose that qemu_host_page_size >=
309 TARGET_PAGE_SIZE */
310 qemu_real_host_page_size = getpagesize();
311 qemu_real_host_page_mask = ~(qemu_real_host_page_size - 1);
312 if (qemu_host_page_size == 0) {
313 qemu_host_page_size = qemu_real_host_page_size;
314 }
315 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
316 qemu_host_page_size = TARGET_PAGE_SIZE;
317 }
318 qemu_host_page_mask = ~(qemu_host_page_size - 1);
319 }
320
321 static void page_init(void)
322 {
323 page_size_init();
324 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
325 {
326 #ifdef HAVE_KINFO_GETVMMAP
327 struct kinfo_vmentry *freep;
328 int i, cnt;
329
330 freep = kinfo_getvmmap(getpid(), &cnt);
331 if (freep) {
332 mmap_lock();
333 for (i = 0; i < cnt; i++) {
334 unsigned long startaddr, endaddr;
335
336 startaddr = freep[i].kve_start;
337 endaddr = freep[i].kve_end;
338 if (h2g_valid(startaddr)) {
339 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
340
341 if (h2g_valid(endaddr)) {
342 endaddr = h2g(endaddr);
343 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
344 } else {
345 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
346 endaddr = ~0ul;
347 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
348 #endif
349 }
350 }
351 }
352 free(freep);
353 mmap_unlock();
354 }
355 #else
356 FILE *f;
357
358 last_brk = (unsigned long)sbrk(0);
359
360 f = fopen("/compat/linux/proc/self/maps", "r");
361 if (f) {
362 mmap_lock();
363
364 do {
365 unsigned long startaddr, endaddr;
366 int n;
367
368 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
369
370 if (n == 2 && h2g_valid(startaddr)) {
371 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
372
373 if (h2g_valid(endaddr)) {
374 endaddr = h2g(endaddr);
375 } else {
376 endaddr = ~0ul;
377 }
378 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
379 }
380 } while (!feof(f));
381
382 fclose(f);
383 mmap_unlock();
384 }
385 #endif
386 }
387 #endif
388 }
389
390 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
391 {
392 PageDesc *pd;
393 void **lp;
394 int i;
395
396 /* Level 1. Always allocated. */
397 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
398
399 /* Level 2..N-1. */
400 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
401 void **p = *lp;
402
403 if (p == NULL) {
404 if (!alloc) {
405 return NULL;
406 }
407 p = g_new0(void *, V_L2_SIZE);
408 *lp = p;
409 }
410
411 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
412 }
413
414 pd = *lp;
415 if (pd == NULL) {
416 if (!alloc) {
417 return NULL;
418 }
419 pd = g_new0(PageDesc, V_L2_SIZE);
420 *lp = pd;
421 }
422
423 return pd + (index & (V_L2_SIZE - 1));
424 }
425
426 static inline PageDesc *page_find(tb_page_addr_t index)
427 {
428 return page_find_alloc(index, 0);
429 }
430
431 #if !defined(CONFIG_USER_ONLY)
432 #define mmap_lock() do { } while (0)
433 #define mmap_unlock() do { } while (0)
434 #endif
435
436 #if defined(CONFIG_USER_ONLY)
437 /* Currently it is not recommended to allocate big chunks of data in
438 user mode. It will change when a dedicated libc will be used. */
439 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
440 region in which the guest needs to run. Revisit this. */
441 #define USE_STATIC_CODE_GEN_BUFFER
442 #endif
443
444 /* ??? Should configure for this, not list operating systems here. */
445 #if (defined(__linux__) \
446 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
447 || defined(__DragonFly__) || defined(__OpenBSD__) \
448 || defined(__NetBSD__))
449 # define USE_MMAP
450 #endif
451
452 /* Minimum size of the code gen buffer. This number is randomly chosen,
453 but not so small that we can't have a fair number of TB's live. */
454 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
455
456 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
457 indicated, this is constrained by the range of direct branches on the
458 host cpu, as used by the TCG implementation of goto_tb. */
459 #if defined(__x86_64__)
460 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
461 #elif defined(__sparc__)
462 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
463 #elif defined(__aarch64__)
464 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
465 #elif defined(__arm__)
466 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
467 #elif defined(__s390x__)
468 /* We have a +- 4GB range on the branches; leave some slop. */
469 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
470 #elif defined(__mips__)
471 /* We have a 256MB branch region, but leave room to make sure the
472 main executable is also within that region. */
473 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
474 #else
475 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
476 #endif
477
478 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
479
480 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
481 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
482 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
483
484 static inline size_t size_code_gen_buffer(size_t tb_size)
485 {
486 /* Size the buffer. */
487 if (tb_size == 0) {
488 #ifdef USE_STATIC_CODE_GEN_BUFFER
489 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
490 #else
491 /* ??? Needs adjustments. */
492 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
493 static buffer, we could size this on RESERVED_VA, on the text
494 segment size of the executable, or continue to use the default. */
495 tb_size = (unsigned long)(ram_size / 4);
496 #endif
497 }
498 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
499 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
500 }
501 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
502 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
503 }
504 tcg_ctx.code_gen_buffer_size = tb_size;
505 return tb_size;
506 }
507
508 #ifdef __mips__
509 /* In order to use J and JAL within the code_gen_buffer, we require
510 that the buffer not cross a 256MB boundary. */
511 static inline bool cross_256mb(void *addr, size_t size)
512 {
513 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & 0xf0000000;
514 }
515
516 /* We weren't able to allocate a buffer without crossing that boundary,
517 so make do with the larger portion of the buffer that doesn't cross.
518 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
519 static inline void *split_cross_256mb(void *buf1, size_t size1)
520 {
521 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & 0xf0000000);
522 size_t size2 = buf1 + size1 - buf2;
523
524 size1 = buf2 - buf1;
525 if (size1 < size2) {
526 size1 = size2;
527 buf1 = buf2;
528 }
529
530 tcg_ctx.code_gen_buffer_size = size1;
531 return buf1;
532 }
533 #endif
534
535 #ifdef USE_STATIC_CODE_GEN_BUFFER
536 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
537 __attribute__((aligned(CODE_GEN_ALIGN)));
538
539 static inline void *alloc_code_gen_buffer(void)
540 {
541 void *buf = static_code_gen_buffer;
542 #ifdef __mips__
543 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
544 buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
545 }
546 #endif
547 map_exec(buf, tcg_ctx.code_gen_buffer_size);
548 return buf;
549 }
550 #elif defined(USE_MMAP)
551 static inline void *alloc_code_gen_buffer(void)
552 {
553 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
554 uintptr_t start = 0;
555 void *buf;
556
557 /* Constrain the position of the buffer based on the host cpu.
558 Note that these addresses are chosen in concert with the
559 addresses assigned in the relevant linker script file. */
560 # if defined(__PIE__) || defined(__PIC__)
561 /* Don't bother setting a preferred location if we're building
562 a position-independent executable. We're more likely to get
563 an address near the main executable if we let the kernel
564 choose the address. */
565 # elif defined(__x86_64__) && defined(MAP_32BIT)
566 /* Force the memory down into low memory with the executable.
567 Leave the choice of exact location with the kernel. */
568 flags |= MAP_32BIT;
569 /* Cannot expect to map more than 800MB in low memory. */
570 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
571 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
572 }
573 # elif defined(__sparc__)
574 start = 0x40000000ul;
575 # elif defined(__s390x__)
576 start = 0x90000000ul;
577 # elif defined(__mips__)
578 /* ??? We ought to more explicitly manage layout for softmmu too. */
579 # ifdef CONFIG_USER_ONLY
580 start = 0x68000000ul;
581 # elif _MIPS_SIM == _ABI64
582 start = 0x128000000ul;
583 # else
584 start = 0x08000000ul;
585 # endif
586 # endif
587
588 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
589 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
590 if (buf == MAP_FAILED) {
591 return NULL;
592 }
593
594 #ifdef __mips__
595 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
596 /* Try again, with the original still mapped, to avoid re-acquiring
597 that 256mb crossing. This time don't specify an address. */
598 size_t size2, size1 = tcg_ctx.code_gen_buffer_size;
599 void *buf2 = mmap(NULL, size1, PROT_WRITE | PROT_READ | PROT_EXEC,
600 flags, -1, 0);
601 if (buf2 != MAP_FAILED) {
602 if (!cross_256mb(buf2, size1)) {
603 /* Success! Use the new buffer. */
604 munmap(buf, size1);
605 return buf2;
606 }
607 /* Failure. Work with what we had. */
608 munmap(buf2, size1);
609 }
610
611 /* Split the original buffer. Free the smaller half. */
612 buf2 = split_cross_256mb(buf, size1);
613 size2 = tcg_ctx.code_gen_buffer_size;
614 munmap(buf + (buf == buf2 ? size2 : 0), size1 - size2);
615 return buf2;
616 }
617 #endif
618
619 return buf;
620 }
621 #else
622 static inline void *alloc_code_gen_buffer(void)
623 {
624 void *buf = g_try_malloc(tcg_ctx.code_gen_buffer_size);
625
626 if (buf == NULL) {
627 return NULL;
628 }
629
630 #ifdef __mips__
631 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
632 void *buf2 = g_malloc(tcg_ctx.code_gen_buffer_size);
633 if (buf2 != NULL && !cross_256mb(buf2, size1)) {
634 /* Success! Use the new buffer. */
635 free(buf);
636 buf = buf2;
637 } else {
638 /* Failure. Work with what we had. Since this is malloc
639 and not mmap, we can't free the other half. */
640 free(buf2);
641 buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
642 }
643 }
644 #endif
645
646 map_exec(buf, tcg_ctx.code_gen_buffer_size);
647 return buf;
648 }
649 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
650
651 static inline void code_gen_alloc(size_t tb_size)
652 {
653 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
654 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
655 if (tcg_ctx.code_gen_buffer == NULL) {
656 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
657 exit(1);
658 }
659
660 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
661 QEMU_MADV_HUGEPAGE);
662
663 /* Steal room for the prologue at the end of the buffer. This ensures
664 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
665 from TB's to the prologue are going to be in range. It also means
666 that we don't need to mark (additional) portions of the data segment
667 as executable. */
668 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
669 tcg_ctx.code_gen_buffer_size - 1024;
670 tcg_ctx.code_gen_buffer_size -= 1024;
671
672 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
673 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
674 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
675 CODE_GEN_AVG_BLOCK_SIZE;
676 tcg_ctx.tb_ctx.tbs =
677 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
678 }
679
680 /* Must be called before using the QEMU cpus. 'tb_size' is the size
681 (in bytes) allocated to the translation buffer. Zero means default
682 size. */
683 void tcg_exec_init(unsigned long tb_size)
684 {
685 cpu_gen_init();
686 code_gen_alloc(tb_size);
687 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
688 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
689 page_init();
690 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
691 /* There's no guest base to take into account, so go ahead and
692 initialize the prologue now. */
693 tcg_prologue_init(&tcg_ctx);
694 #endif
695 }
696
697 bool tcg_enabled(void)
698 {
699 return tcg_ctx.code_gen_buffer != NULL;
700 }
701
702 /* Allocate a new translation block. Flush the translation buffer if
703 too many translation blocks or too much generated code. */
704 static TranslationBlock *tb_alloc(target_ulong pc)
705 {
706 TranslationBlock *tb;
707
708 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
709 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
710 tcg_ctx.code_gen_buffer_max_size) {
711 return NULL;
712 }
713 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
714 tb->pc = pc;
715 tb->cflags = 0;
716 return tb;
717 }
718
719 void tb_free(TranslationBlock *tb)
720 {
721 /* In practice this is mostly used for single use temporary TB
722 Ignore the hard cases and just back up if this TB happens to
723 be the last one generated. */
724 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
725 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
726 tcg_ctx.code_gen_ptr = tb->tc_ptr;
727 tcg_ctx.tb_ctx.nb_tbs--;
728 }
729 }
730
731 static inline void invalidate_page_bitmap(PageDesc *p)
732 {
733 if (p->code_bitmap) {
734 g_free(p->code_bitmap);
735 p->code_bitmap = NULL;
736 }
737 p->code_write_count = 0;
738 }
739
740 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
741 static void page_flush_tb_1(int level, void **lp)
742 {
743 int i;
744
745 if (*lp == NULL) {
746 return;
747 }
748 if (level == 0) {
749 PageDesc *pd = *lp;
750
751 for (i = 0; i < V_L2_SIZE; ++i) {
752 pd[i].first_tb = NULL;
753 invalidate_page_bitmap(pd + i);
754 }
755 } else {
756 void **pp = *lp;
757
758 for (i = 0; i < V_L2_SIZE; ++i) {
759 page_flush_tb_1(level - 1, pp + i);
760 }
761 }
762 }
763
764 static void page_flush_tb(void)
765 {
766 int i;
767
768 for (i = 0; i < V_L1_SIZE; i++) {
769 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
770 }
771 }
772
773 /* flush all the translation blocks */
774 /* XXX: tb_flush is currently not thread safe */
775 void tb_flush(CPUState *cpu)
776 {
777 #if defined(DEBUG_FLUSH)
778 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
779 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
780 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
781 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
782 tcg_ctx.tb_ctx.nb_tbs : 0);
783 #endif
784 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
785 > tcg_ctx.code_gen_buffer_size) {
786 cpu_abort(cpu, "Internal error: code buffer overflow\n");
787 }
788 tcg_ctx.tb_ctx.nb_tbs = 0;
789
790 CPU_FOREACH(cpu) {
791 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
792 }
793
794 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
795 page_flush_tb();
796
797 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
798 /* XXX: flush processor icache at this point if cache flush is
799 expensive */
800 tcg_ctx.tb_ctx.tb_flush_count++;
801 }
802
803 #ifdef DEBUG_TB_CHECK
804
805 static void tb_invalidate_check(target_ulong address)
806 {
807 TranslationBlock *tb;
808 int i;
809
810 address &= TARGET_PAGE_MASK;
811 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
812 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
813 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
814 address >= tb->pc + tb->size)) {
815 printf("ERROR invalidate: address=" TARGET_FMT_lx
816 " PC=%08lx size=%04x\n",
817 address, (long)tb->pc, tb->size);
818 }
819 }
820 }
821 }
822
823 /* verify that all the pages have correct rights for code */
824 static void tb_page_check(void)
825 {
826 TranslationBlock *tb;
827 int i, flags1, flags2;
828
829 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
830 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
831 tb = tb->phys_hash_next) {
832 flags1 = page_get_flags(tb->pc);
833 flags2 = page_get_flags(tb->pc + tb->size - 1);
834 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
835 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
836 (long)tb->pc, tb->size, flags1, flags2);
837 }
838 }
839 }
840 }
841
842 #endif
843
844 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
845 {
846 TranslationBlock *tb1;
847
848 for (;;) {
849 tb1 = *ptb;
850 if (tb1 == tb) {
851 *ptb = tb1->phys_hash_next;
852 break;
853 }
854 ptb = &tb1->phys_hash_next;
855 }
856 }
857
858 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
859 {
860 TranslationBlock *tb1;
861 unsigned int n1;
862
863 for (;;) {
864 tb1 = *ptb;
865 n1 = (uintptr_t)tb1 & 3;
866 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
867 if (tb1 == tb) {
868 *ptb = tb1->page_next[n1];
869 break;
870 }
871 ptb = &tb1->page_next[n1];
872 }
873 }
874
875 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
876 {
877 TranslationBlock *tb1, **ptb;
878 unsigned int n1;
879
880 ptb = &tb->jmp_next[n];
881 tb1 = *ptb;
882 if (tb1) {
883 /* find tb(n) in circular list */
884 for (;;) {
885 tb1 = *ptb;
886 n1 = (uintptr_t)tb1 & 3;
887 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
888 if (n1 == n && tb1 == tb) {
889 break;
890 }
891 if (n1 == 2) {
892 ptb = &tb1->jmp_first;
893 } else {
894 ptb = &tb1->jmp_next[n1];
895 }
896 }
897 /* now we can suppress tb(n) from the list */
898 *ptb = tb->jmp_next[n];
899
900 tb->jmp_next[n] = NULL;
901 }
902 }
903
904 /* reset the jump entry 'n' of a TB so that it is not chained to
905 another TB */
906 static inline void tb_reset_jump(TranslationBlock *tb, int n)
907 {
908 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
909 }
910
911 /* invalidate one TB */
912 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
913 {
914 CPUState *cpu;
915 PageDesc *p;
916 unsigned int h, n1;
917 tb_page_addr_t phys_pc;
918 TranslationBlock *tb1, *tb2;
919
920 /* remove the TB from the hash list */
921 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
922 h = tb_phys_hash_func(phys_pc);
923 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
924
925 /* remove the TB from the page list */
926 if (tb->page_addr[0] != page_addr) {
927 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
928 tb_page_remove(&p->first_tb, tb);
929 invalidate_page_bitmap(p);
930 }
931 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
932 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
933 tb_page_remove(&p->first_tb, tb);
934 invalidate_page_bitmap(p);
935 }
936
937 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
938
939 /* remove the TB from the hash list */
940 h = tb_jmp_cache_hash_func(tb->pc);
941 CPU_FOREACH(cpu) {
942 if (cpu->tb_jmp_cache[h] == tb) {
943 cpu->tb_jmp_cache[h] = NULL;
944 }
945 }
946
947 /* suppress this TB from the two jump lists */
948 tb_jmp_remove(tb, 0);
949 tb_jmp_remove(tb, 1);
950
951 /* suppress any remaining jumps to this TB */
952 tb1 = tb->jmp_first;
953 for (;;) {
954 n1 = (uintptr_t)tb1 & 3;
955 if (n1 == 2) {
956 break;
957 }
958 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
959 tb2 = tb1->jmp_next[n1];
960 tb_reset_jump(tb1, n1);
961 tb1->jmp_next[n1] = NULL;
962 tb1 = tb2;
963 }
964 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
965
966 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
967 }
968
969 static void build_page_bitmap(PageDesc *p)
970 {
971 int n, tb_start, tb_end;
972 TranslationBlock *tb;
973
974 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
975
976 tb = p->first_tb;
977 while (tb != NULL) {
978 n = (uintptr_t)tb & 3;
979 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
980 /* NOTE: this is subtle as a TB may span two physical pages */
981 if (n == 0) {
982 /* NOTE: tb_end may be after the end of the page, but
983 it is not a problem */
984 tb_start = tb->pc & ~TARGET_PAGE_MASK;
985 tb_end = tb_start + tb->size;
986 if (tb_end > TARGET_PAGE_SIZE) {
987 tb_end = TARGET_PAGE_SIZE;
988 }
989 } else {
990 tb_start = 0;
991 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
992 }
993 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
994 tb = tb->page_next[n];
995 }
996 }
997
998 TranslationBlock *tb_gen_code(CPUState *cpu,
999 target_ulong pc, target_ulong cs_base,
1000 int flags, int cflags)
1001 {
1002 CPUArchState *env = cpu->env_ptr;
1003 TranslationBlock *tb;
1004 tb_page_addr_t phys_pc, phys_page2;
1005 target_ulong virt_page2;
1006 int code_gen_size;
1007
1008 phys_pc = get_page_addr_code(env, pc);
1009 if (use_icount) {
1010 cflags |= CF_USE_ICOUNT;
1011 }
1012 tb = tb_alloc(pc);
1013 if (!tb) {
1014 /* flush must be done */
1015 tb_flush(cpu);
1016 /* cannot fail at this point */
1017 tb = tb_alloc(pc);
1018 /* Don't forget to invalidate previous TB info. */
1019 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
1020 }
1021 tb->tc_ptr = tcg_ctx.code_gen_ptr;
1022 tb->cs_base = cs_base;
1023 tb->flags = flags;
1024 tb->cflags = cflags;
1025 cpu_gen_code(env, tb, &code_gen_size);
1026 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
1027 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
1028
1029 /* check next page if needed */
1030 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1031 phys_page2 = -1;
1032 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1033 phys_page2 = get_page_addr_code(env, virt_page2);
1034 }
1035 tb_link_page(tb, phys_pc, phys_page2);
1036 return tb;
1037 }
1038
1039 /*
1040 * Invalidate all TBs which intersect with the target physical address range
1041 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1042 * 'is_cpu_write_access' should be true if called from a real cpu write
1043 * access: the virtual CPU will exit the current TB if code is modified inside
1044 * this TB.
1045 */
1046 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
1047 {
1048 while (start < end) {
1049 tb_invalidate_phys_page_range(start, end, 0);
1050 start &= TARGET_PAGE_MASK;
1051 start += TARGET_PAGE_SIZE;
1052 }
1053 }
1054
1055 /*
1056 * Invalidate all TBs which intersect with the target physical address range
1057 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1058 * 'is_cpu_write_access' should be true if called from a real cpu write
1059 * access: the virtual CPU will exit the current TB if code is modified inside
1060 * this TB.
1061 */
1062 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1063 int is_cpu_write_access)
1064 {
1065 TranslationBlock *tb, *tb_next, *saved_tb;
1066 CPUState *cpu = current_cpu;
1067 #if defined(TARGET_HAS_PRECISE_SMC)
1068 CPUArchState *env = NULL;
1069 #endif
1070 tb_page_addr_t tb_start, tb_end;
1071 PageDesc *p;
1072 int n;
1073 #ifdef TARGET_HAS_PRECISE_SMC
1074 int current_tb_not_found = is_cpu_write_access;
1075 TranslationBlock *current_tb = NULL;
1076 int current_tb_modified = 0;
1077 target_ulong current_pc = 0;
1078 target_ulong current_cs_base = 0;
1079 int current_flags = 0;
1080 #endif /* TARGET_HAS_PRECISE_SMC */
1081
1082 p = page_find(start >> TARGET_PAGE_BITS);
1083 if (!p) {
1084 return;
1085 }
1086 #if defined(TARGET_HAS_PRECISE_SMC)
1087 if (cpu != NULL) {
1088 env = cpu->env_ptr;
1089 }
1090 #endif
1091
1092 /* we remove all the TBs in the range [start, end[ */
1093 /* XXX: see if in some cases it could be faster to invalidate all
1094 the code */
1095 tb = p->first_tb;
1096 while (tb != NULL) {
1097 n = (uintptr_t)tb & 3;
1098 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1099 tb_next = tb->page_next[n];
1100 /* NOTE: this is subtle as a TB may span two physical pages */
1101 if (n == 0) {
1102 /* NOTE: tb_end may be after the end of the page, but
1103 it is not a problem */
1104 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1105 tb_end = tb_start + tb->size;
1106 } else {
1107 tb_start = tb->page_addr[1];
1108 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1109 }
1110 if (!(tb_end <= start || tb_start >= end)) {
1111 #ifdef TARGET_HAS_PRECISE_SMC
1112 if (current_tb_not_found) {
1113 current_tb_not_found = 0;
1114 current_tb = NULL;
1115 if (cpu->mem_io_pc) {
1116 /* now we have a real cpu fault */
1117 current_tb = tb_find_pc(cpu->mem_io_pc);
1118 }
1119 }
1120 if (current_tb == tb &&
1121 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1122 /* If we are modifying the current TB, we must stop
1123 its execution. We could be more precise by checking
1124 that the modification is after the current PC, but it
1125 would require a specialized function to partially
1126 restore the CPU state */
1127
1128 current_tb_modified = 1;
1129 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
1130 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1131 &current_flags);
1132 }
1133 #endif /* TARGET_HAS_PRECISE_SMC */
1134 /* we need to do that to handle the case where a signal
1135 occurs while doing tb_phys_invalidate() */
1136 saved_tb = NULL;
1137 if (cpu != NULL) {
1138 saved_tb = cpu->current_tb;
1139 cpu->current_tb = NULL;
1140 }
1141 tb_phys_invalidate(tb, -1);
1142 if (cpu != NULL) {
1143 cpu->current_tb = saved_tb;
1144 if (cpu->interrupt_request && cpu->current_tb) {
1145 cpu_interrupt(cpu, cpu->interrupt_request);
1146 }
1147 }
1148 }
1149 tb = tb_next;
1150 }
1151 #if !defined(CONFIG_USER_ONLY)
1152 /* if no code remaining, no need to continue to use slow writes */
1153 if (!p->first_tb) {
1154 invalidate_page_bitmap(p);
1155 tlb_unprotect_code(start);
1156 }
1157 #endif
1158 #ifdef TARGET_HAS_PRECISE_SMC
1159 if (current_tb_modified) {
1160 /* we generate a block containing just the instruction
1161 modifying the memory. It will ensure that it cannot modify
1162 itself */
1163 cpu->current_tb = NULL;
1164 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1165 cpu_resume_from_signal(cpu, NULL);
1166 }
1167 #endif
1168 }
1169
1170 /* len must be <= 8 and start must be a multiple of len */
1171 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1172 {
1173 PageDesc *p;
1174
1175 #if 0
1176 if (1) {
1177 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1178 cpu_single_env->mem_io_vaddr, len,
1179 cpu_single_env->eip,
1180 cpu_single_env->eip +
1181 (intptr_t)cpu_single_env->segs[R_CS].base);
1182 }
1183 #endif
1184 p = page_find(start >> TARGET_PAGE_BITS);
1185 if (!p) {
1186 return;
1187 }
1188 if (!p->code_bitmap &&
1189 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
1190 /* build code bitmap */
1191 build_page_bitmap(p);
1192 }
1193 if (p->code_bitmap) {
1194 unsigned int nr;
1195 unsigned long b;
1196
1197 nr = start & ~TARGET_PAGE_MASK;
1198 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
1199 if (b & ((1 << len) - 1)) {
1200 goto do_invalidate;
1201 }
1202 } else {
1203 do_invalidate:
1204 tb_invalidate_phys_page_range(start, start + len, 1);
1205 }
1206 }
1207
1208 #if !defined(CONFIG_SOFTMMU)
1209 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1210 uintptr_t pc, void *puc,
1211 bool locked)
1212 {
1213 TranslationBlock *tb;
1214 PageDesc *p;
1215 int n;
1216 #ifdef TARGET_HAS_PRECISE_SMC
1217 TranslationBlock *current_tb = NULL;
1218 CPUState *cpu = current_cpu;
1219 CPUArchState *env = NULL;
1220 int current_tb_modified = 0;
1221 target_ulong current_pc = 0;
1222 target_ulong current_cs_base = 0;
1223 int current_flags = 0;
1224 #endif
1225
1226 addr &= TARGET_PAGE_MASK;
1227 p = page_find(addr >> TARGET_PAGE_BITS);
1228 if (!p) {
1229 return;
1230 }
1231 tb = p->first_tb;
1232 #ifdef TARGET_HAS_PRECISE_SMC
1233 if (tb && pc != 0) {
1234 current_tb = tb_find_pc(pc);
1235 }
1236 if (cpu != NULL) {
1237 env = cpu->env_ptr;
1238 }
1239 #endif
1240 while (tb != NULL) {
1241 n = (uintptr_t)tb & 3;
1242 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1243 #ifdef TARGET_HAS_PRECISE_SMC
1244 if (current_tb == tb &&
1245 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1246 /* If we are modifying the current TB, we must stop
1247 its execution. We could be more precise by checking
1248 that the modification is after the current PC, but it
1249 would require a specialized function to partially
1250 restore the CPU state */
1251
1252 current_tb_modified = 1;
1253 cpu_restore_state_from_tb(cpu, current_tb, pc);
1254 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1255 &current_flags);
1256 }
1257 #endif /* TARGET_HAS_PRECISE_SMC */
1258 tb_phys_invalidate(tb, addr);
1259 tb = tb->page_next[n];
1260 }
1261 p->first_tb = NULL;
1262 #ifdef TARGET_HAS_PRECISE_SMC
1263 if (current_tb_modified) {
1264 /* we generate a block containing just the instruction
1265 modifying the memory. It will ensure that it cannot modify
1266 itself */
1267 cpu->current_tb = NULL;
1268 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1269 if (locked) {
1270 mmap_unlock();
1271 }
1272 cpu_resume_from_signal(cpu, puc);
1273 }
1274 #endif
1275 }
1276 #endif
1277
1278 /* add the tb in the target page and protect it if necessary */
1279 static inline void tb_alloc_page(TranslationBlock *tb,
1280 unsigned int n, tb_page_addr_t page_addr)
1281 {
1282 PageDesc *p;
1283 #ifndef CONFIG_USER_ONLY
1284 bool page_already_protected;
1285 #endif
1286
1287 tb->page_addr[n] = page_addr;
1288 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1289 tb->page_next[n] = p->first_tb;
1290 #ifndef CONFIG_USER_ONLY
1291 page_already_protected = p->first_tb != NULL;
1292 #endif
1293 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1294 invalidate_page_bitmap(p);
1295
1296 #if defined(CONFIG_USER_ONLY)
1297 if (p->flags & PAGE_WRITE) {
1298 target_ulong addr;
1299 PageDesc *p2;
1300 int prot;
1301
1302 /* force the host page as non writable (writes will have a
1303 page fault + mprotect overhead) */
1304 page_addr &= qemu_host_page_mask;
1305 prot = 0;
1306 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1307 addr += TARGET_PAGE_SIZE) {
1308
1309 p2 = page_find(addr >> TARGET_PAGE_BITS);
1310 if (!p2) {
1311 continue;
1312 }
1313 prot |= p2->flags;
1314 p2->flags &= ~PAGE_WRITE;
1315 }
1316 mprotect(g2h(page_addr), qemu_host_page_size,
1317 (prot & PAGE_BITS) & ~PAGE_WRITE);
1318 #ifdef DEBUG_TB_INVALIDATE
1319 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1320 page_addr);
1321 #endif
1322 }
1323 #else
1324 /* if some code is already present, then the pages are already
1325 protected. So we handle the case where only the first TB is
1326 allocated in a physical page */
1327 if (!page_already_protected) {
1328 tlb_protect_code(page_addr);
1329 }
1330 #endif
1331 }
1332
1333 /* add a new TB and link it to the physical page tables. phys_page2 is
1334 (-1) to indicate that only one page contains the TB. */
1335 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1336 tb_page_addr_t phys_page2)
1337 {
1338 unsigned int h;
1339 TranslationBlock **ptb;
1340
1341 /* Grab the mmap lock to stop another thread invalidating this TB
1342 before we are done. */
1343 mmap_lock();
1344 /* add in the physical hash table */
1345 h = tb_phys_hash_func(phys_pc);
1346 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1347 tb->phys_hash_next = *ptb;
1348 *ptb = tb;
1349
1350 /* add in the page list */
1351 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1352 if (phys_page2 != -1) {
1353 tb_alloc_page(tb, 1, phys_page2);
1354 } else {
1355 tb->page_addr[1] = -1;
1356 }
1357
1358 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1359 tb->jmp_next[0] = NULL;
1360 tb->jmp_next[1] = NULL;
1361
1362 /* init original jump addresses */
1363 if (tb->tb_next_offset[0] != 0xffff) {
1364 tb_reset_jump(tb, 0);
1365 }
1366 if (tb->tb_next_offset[1] != 0xffff) {
1367 tb_reset_jump(tb, 1);
1368 }
1369
1370 #ifdef DEBUG_TB_CHECK
1371 tb_page_check();
1372 #endif
1373 mmap_unlock();
1374 }
1375
1376 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1377 tb[1].tc_ptr. Return NULL if not found */
1378 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1379 {
1380 int m_min, m_max, m;
1381 uintptr_t v;
1382 TranslationBlock *tb;
1383
1384 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1385 return NULL;
1386 }
1387 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1388 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1389 return NULL;
1390 }
1391 /* binary search (cf Knuth) */
1392 m_min = 0;
1393 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1394 while (m_min <= m_max) {
1395 m = (m_min + m_max) >> 1;
1396 tb = &tcg_ctx.tb_ctx.tbs[m];
1397 v = (uintptr_t)tb->tc_ptr;
1398 if (v == tc_ptr) {
1399 return tb;
1400 } else if (tc_ptr < v) {
1401 m_max = m - 1;
1402 } else {
1403 m_min = m + 1;
1404 }
1405 }
1406 return &tcg_ctx.tb_ctx.tbs[m_max];
1407 }
1408
1409 #if !defined(CONFIG_USER_ONLY)
1410 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1411 {
1412 ram_addr_t ram_addr;
1413 MemoryRegion *mr;
1414 hwaddr l = 1;
1415
1416 rcu_read_lock();
1417 mr = address_space_translate(as, addr, &addr, &l, false);
1418 if (!(memory_region_is_ram(mr)
1419 || memory_region_is_romd(mr))) {
1420 rcu_read_unlock();
1421 return;
1422 }
1423 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
1424 + addr;
1425 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1426 rcu_read_unlock();
1427 }
1428 #endif /* !defined(CONFIG_USER_ONLY) */
1429
1430 void tb_check_watchpoint(CPUState *cpu)
1431 {
1432 TranslationBlock *tb;
1433
1434 tb = tb_find_pc(cpu->mem_io_pc);
1435 if (tb) {
1436 /* We can use retranslation to find the PC. */
1437 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
1438 tb_phys_invalidate(tb, -1);
1439 } else {
1440 /* The exception probably happened in a helper. The CPU state should
1441 have been saved before calling it. Fetch the PC from there. */
1442 CPUArchState *env = cpu->env_ptr;
1443 target_ulong pc, cs_base;
1444 tb_page_addr_t addr;
1445 int flags;
1446
1447 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
1448 addr = get_page_addr_code(env, pc);
1449 tb_invalidate_phys_range(addr, addr + 1);
1450 }
1451 }
1452
1453 #ifndef CONFIG_USER_ONLY
1454 /* mask must never be zero, except for A20 change call */
1455 static void tcg_handle_interrupt(CPUState *cpu, int mask)
1456 {
1457 int old_mask;
1458
1459 old_mask = cpu->interrupt_request;
1460 cpu->interrupt_request |= mask;
1461
1462 /*
1463 * If called from iothread context, wake the target cpu in
1464 * case its halted.
1465 */
1466 if (!qemu_cpu_is_self(cpu)) {
1467 qemu_cpu_kick(cpu);
1468 return;
1469 }
1470
1471 if (use_icount) {
1472 cpu->icount_decr.u16.high = 0xffff;
1473 if (!cpu_can_do_io(cpu)
1474 && (mask & ~old_mask) != 0) {
1475 cpu_abort(cpu, "Raised interrupt while not in I/O function");
1476 }
1477 } else {
1478 cpu->tcg_exit_req = 1;
1479 }
1480 }
1481
1482 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1483
1484 /* in deterministic execution mode, instructions doing device I/Os
1485 must be at the end of the TB */
1486 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
1487 {
1488 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1489 CPUArchState *env = cpu->env_ptr;
1490 #endif
1491 TranslationBlock *tb;
1492 uint32_t n, cflags;
1493 target_ulong pc, cs_base;
1494 uint64_t flags;
1495
1496 tb = tb_find_pc(retaddr);
1497 if (!tb) {
1498 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
1499 (void *)retaddr);
1500 }
1501 n = cpu->icount_decr.u16.low + tb->icount;
1502 cpu_restore_state_from_tb(cpu, tb, retaddr);
1503 /* Calculate how many instructions had been executed before the fault
1504 occurred. */
1505 n = n - cpu->icount_decr.u16.low;
1506 /* Generate a new TB ending on the I/O insn. */
1507 n++;
1508 /* On MIPS and SH, delay slot instructions can only be restarted if
1509 they were already the first instruction in the TB. If this is not
1510 the first instruction in a TB then re-execute the preceding
1511 branch. */
1512 #if defined(TARGET_MIPS)
1513 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1514 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
1515 cpu->icount_decr.u16.low++;
1516 env->hflags &= ~MIPS_HFLAG_BMASK;
1517 }
1518 #elif defined(TARGET_SH4)
1519 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1520 && n > 1) {
1521 env->pc -= 2;
1522 cpu->icount_decr.u16.low++;
1523 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1524 }
1525 #endif
1526 /* This should never happen. */
1527 if (n > CF_COUNT_MASK) {
1528 cpu_abort(cpu, "TB too big during recompile");
1529 }
1530
1531 cflags = n | CF_LAST_IO;
1532 pc = tb->pc;
1533 cs_base = tb->cs_base;
1534 flags = tb->flags;
1535 tb_phys_invalidate(tb, -1);
1536 /* FIXME: In theory this could raise an exception. In practice
1537 we have already translated the block once so it's probably ok. */
1538 tb_gen_code(cpu, pc, cs_base, flags, cflags);
1539 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1540 the first in the TB) then we end up generating a whole new TB and
1541 repeating the fault, which is horribly inefficient.
1542 Better would be to execute just this insn uncached, or generate a
1543 second new TB. */
1544 cpu_resume_from_signal(cpu, NULL);
1545 }
1546
1547 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
1548 {
1549 unsigned int i;
1550
1551 /* Discard jump cache entries for any tb which might potentially
1552 overlap the flushed page. */
1553 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1554 memset(&cpu->tb_jmp_cache[i], 0,
1555 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1556
1557 i = tb_jmp_cache_hash_page(addr);
1558 memset(&cpu->tb_jmp_cache[i], 0,
1559 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1560 }
1561
1562 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1563 {
1564 int i, target_code_size, max_target_code_size;
1565 int direct_jmp_count, direct_jmp2_count, cross_page;
1566 TranslationBlock *tb;
1567
1568 target_code_size = 0;
1569 max_target_code_size = 0;
1570 cross_page = 0;
1571 direct_jmp_count = 0;
1572 direct_jmp2_count = 0;
1573 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1574 tb = &tcg_ctx.tb_ctx.tbs[i];
1575 target_code_size += tb->size;
1576 if (tb->size > max_target_code_size) {
1577 max_target_code_size = tb->size;
1578 }
1579 if (tb->page_addr[1] != -1) {
1580 cross_page++;
1581 }
1582 if (tb->tb_next_offset[0] != 0xffff) {
1583 direct_jmp_count++;
1584 if (tb->tb_next_offset[1] != 0xffff) {
1585 direct_jmp2_count++;
1586 }
1587 }
1588 }
1589 /* XXX: avoid using doubles ? */
1590 cpu_fprintf(f, "Translation buffer state:\n");
1591 cpu_fprintf(f, "gen code size %td/%zd\n",
1592 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1593 tcg_ctx.code_gen_buffer_max_size);
1594 cpu_fprintf(f, "TB count %d/%d\n",
1595 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1596 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1597 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1598 tcg_ctx.tb_ctx.nb_tbs : 0,
1599 max_target_code_size);
1600 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1601 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1602 tcg_ctx.code_gen_buffer) /
1603 tcg_ctx.tb_ctx.nb_tbs : 0,
1604 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1605 tcg_ctx.code_gen_buffer) /
1606 target_code_size : 0);
1607 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1608 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1609 tcg_ctx.tb_ctx.nb_tbs : 0);
1610 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1611 direct_jmp_count,
1612 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1613 tcg_ctx.tb_ctx.nb_tbs : 0,
1614 direct_jmp2_count,
1615 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1616 tcg_ctx.tb_ctx.nb_tbs : 0);
1617 cpu_fprintf(f, "\nStatistics:\n");
1618 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1619 cpu_fprintf(f, "TB invalidate count %d\n",
1620 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1621 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1622 tcg_dump_info(f, cpu_fprintf);
1623 }
1624
1625 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
1626 {
1627 tcg_dump_op_count(f, cpu_fprintf);
1628 }
1629
1630 #else /* CONFIG_USER_ONLY */
1631
1632 void cpu_interrupt(CPUState *cpu, int mask)
1633 {
1634 cpu->interrupt_request |= mask;
1635 cpu->tcg_exit_req = 1;
1636 }
1637
1638 /*
1639 * Walks guest process memory "regions" one by one
1640 * and calls callback function 'fn' for each region.
1641 */
1642 struct walk_memory_regions_data {
1643 walk_memory_regions_fn fn;
1644 void *priv;
1645 target_ulong start;
1646 int prot;
1647 };
1648
1649 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1650 target_ulong end, int new_prot)
1651 {
1652 if (data->start != -1u) {
1653 int rc = data->fn(data->priv, data->start, end, data->prot);
1654 if (rc != 0) {
1655 return rc;
1656 }
1657 }
1658
1659 data->start = (new_prot ? end : -1u);
1660 data->prot = new_prot;
1661
1662 return 0;
1663 }
1664
1665 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1666 target_ulong base, int level, void **lp)
1667 {
1668 target_ulong pa;
1669 int i, rc;
1670
1671 if (*lp == NULL) {
1672 return walk_memory_regions_end(data, base, 0);
1673 }
1674
1675 if (level == 0) {
1676 PageDesc *pd = *lp;
1677
1678 for (i = 0; i < V_L2_SIZE; ++i) {
1679 int prot = pd[i].flags;
1680
1681 pa = base | (i << TARGET_PAGE_BITS);
1682 if (prot != data->prot) {
1683 rc = walk_memory_regions_end(data, pa, prot);
1684 if (rc != 0) {
1685 return rc;
1686 }
1687 }
1688 }
1689 } else {
1690 void **pp = *lp;
1691
1692 for (i = 0; i < V_L2_SIZE; ++i) {
1693 pa = base | ((target_ulong)i <<
1694 (TARGET_PAGE_BITS + V_L2_BITS * level));
1695 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1696 if (rc != 0) {
1697 return rc;
1698 }
1699 }
1700 }
1701
1702 return 0;
1703 }
1704
1705 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1706 {
1707 struct walk_memory_regions_data data;
1708 uintptr_t i;
1709
1710 data.fn = fn;
1711 data.priv = priv;
1712 data.start = -1u;
1713 data.prot = 0;
1714
1715 for (i = 0; i < V_L1_SIZE; i++) {
1716 int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
1717 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1718 if (rc != 0) {
1719 return rc;
1720 }
1721 }
1722
1723 return walk_memory_regions_end(&data, 0, 0);
1724 }
1725
1726 static int dump_region(void *priv, target_ulong start,
1727 target_ulong end, unsigned long prot)
1728 {
1729 FILE *f = (FILE *)priv;
1730
1731 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
1732 " "TARGET_FMT_lx" %c%c%c\n",
1733 start, end, end - start,
1734 ((prot & PAGE_READ) ? 'r' : '-'),
1735 ((prot & PAGE_WRITE) ? 'w' : '-'),
1736 ((prot & PAGE_EXEC) ? 'x' : '-'));
1737
1738 return 0;
1739 }
1740
1741 /* dump memory mappings */
1742 void page_dump(FILE *f)
1743 {
1744 const int length = sizeof(target_ulong) * 2;
1745 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1746 length, "start", length, "end", length, "size", "prot");
1747 walk_memory_regions(f, dump_region);
1748 }
1749
1750 int page_get_flags(target_ulong address)
1751 {
1752 PageDesc *p;
1753
1754 p = page_find(address >> TARGET_PAGE_BITS);
1755 if (!p) {
1756 return 0;
1757 }
1758 return p->flags;
1759 }
1760
1761 /* Modify the flags of a page and invalidate the code if necessary.
1762 The flag PAGE_WRITE_ORG is positioned automatically depending
1763 on PAGE_WRITE. The mmap_lock should already be held. */
1764 void page_set_flags(target_ulong start, target_ulong end, int flags)
1765 {
1766 target_ulong addr, len;
1767
1768 /* This function should never be called with addresses outside the
1769 guest address space. If this assert fires, it probably indicates
1770 a missing call to h2g_valid. */
1771 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1772 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1773 #endif
1774 assert(start < end);
1775
1776 start = start & TARGET_PAGE_MASK;
1777 end = TARGET_PAGE_ALIGN(end);
1778
1779 if (flags & PAGE_WRITE) {
1780 flags |= PAGE_WRITE_ORG;
1781 }
1782
1783 for (addr = start, len = end - start;
1784 len != 0;
1785 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1786 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1787
1788 /* If the write protection bit is set, then we invalidate
1789 the code inside. */
1790 if (!(p->flags & PAGE_WRITE) &&
1791 (flags & PAGE_WRITE) &&
1792 p->first_tb) {
1793 tb_invalidate_phys_page(addr, 0, NULL, false);
1794 }
1795 p->flags = flags;
1796 }
1797 }
1798
1799 int page_check_range(target_ulong start, target_ulong len, int flags)
1800 {
1801 PageDesc *p;
1802 target_ulong end;
1803 target_ulong addr;
1804
1805 /* This function should never be called with addresses outside the
1806 guest address space. If this assert fires, it probably indicates
1807 a missing call to h2g_valid. */
1808 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1809 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1810 #endif
1811
1812 if (len == 0) {
1813 return 0;
1814 }
1815 if (start + len - 1 < start) {
1816 /* We've wrapped around. */
1817 return -1;
1818 }
1819
1820 /* must do before we loose bits in the next step */
1821 end = TARGET_PAGE_ALIGN(start + len);
1822 start = start & TARGET_PAGE_MASK;
1823
1824 for (addr = start, len = end - start;
1825 len != 0;
1826 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1827 p = page_find(addr >> TARGET_PAGE_BITS);
1828 if (!p) {
1829 return -1;
1830 }
1831 if (!(p->flags & PAGE_VALID)) {
1832 return -1;
1833 }
1834
1835 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1836 return -1;
1837 }
1838 if (flags & PAGE_WRITE) {
1839 if (!(p->flags & PAGE_WRITE_ORG)) {
1840 return -1;
1841 }
1842 /* unprotect the page if it was put read-only because it
1843 contains translated code */
1844 if (!(p->flags & PAGE_WRITE)) {
1845 if (!page_unprotect(addr, 0, NULL)) {
1846 return -1;
1847 }
1848 }
1849 }
1850 }
1851 return 0;
1852 }
1853
1854 /* called from signal handler: invalidate the code and unprotect the
1855 page. Return TRUE if the fault was successfully handled. */
1856 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1857 {
1858 unsigned int prot;
1859 PageDesc *p;
1860 target_ulong host_start, host_end, addr;
1861
1862 /* Technically this isn't safe inside a signal handler. However we
1863 know this only ever happens in a synchronous SEGV handler, so in
1864 practice it seems to be ok. */
1865 mmap_lock();
1866
1867 p = page_find(address >> TARGET_PAGE_BITS);
1868 if (!p) {
1869 mmap_unlock();
1870 return 0;
1871 }
1872
1873 /* if the page was really writable, then we change its
1874 protection back to writable */
1875 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1876 host_start = address & qemu_host_page_mask;
1877 host_end = host_start + qemu_host_page_size;
1878
1879 prot = 0;
1880 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1881 p = page_find(addr >> TARGET_PAGE_BITS);
1882 p->flags |= PAGE_WRITE;
1883 prot |= p->flags;
1884
1885 /* and since the content will be modified, we must invalidate
1886 the corresponding translated code. */
1887 tb_invalidate_phys_page(addr, pc, puc, true);
1888 #ifdef DEBUG_TB_CHECK
1889 tb_invalidate_check(addr);
1890 #endif
1891 }
1892 mprotect((void *)g2h(host_start), qemu_host_page_size,
1893 prot & PAGE_BITS);
1894
1895 mmap_unlock();
1896 return 1;
1897 }
1898 mmap_unlock();
1899 return 0;
1900 }
1901 #endif /* CONFIG_USER_ONLY */