translate-all: make have_tb_lock static
[qemu.git] / accel / tcg / translate-all.c
1 /*
2 * Host code generation
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #ifdef _WIN32
20 #include <windows.h>
21 #endif
22 #include "qemu/osdep.h"
23
24
25 #include "qemu-common.h"
26 #define NO_CPU_IO_DEFS
27 #include "cpu.h"
28 #include "trace.h"
29 #include "disas/disas.h"
30 #include "exec/exec-all.h"
31 #include "tcg.h"
32 #if defined(CONFIG_USER_ONLY)
33 #include "qemu.h"
34 #include "exec/exec-all.h"
35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
36 #include <sys/param.h>
37 #if __FreeBSD_version >= 700104
38 #define HAVE_KINFO_GETVMMAP
39 #define sigqueue sigqueue_freebsd /* avoid redefinition */
40 #include <sys/proc.h>
41 #include <machine/profile.h>
42 #define _KERNEL
43 #include <sys/user.h>
44 #undef _KERNEL
45 #undef sigqueue
46 #include <libutil.h>
47 #endif
48 #endif
49 #else
50 #include "exec/address-spaces.h"
51 #endif
52
53 #include "exec/cputlb.h"
54 #include "exec/tb-hash.h"
55 #include "translate-all.h"
56 #include "qemu/bitmap.h"
57 #include "qemu/error-report.h"
58 #include "qemu/timer.h"
59 #include "qemu/main-loop.h"
60 #include "exec/log.h"
61 #include "sysemu/cpus.h"
62
63 /* #define DEBUG_TB_INVALIDATE */
64 /* #define DEBUG_TB_FLUSH */
65 /* make various TB consistency checks */
66 /* #define DEBUG_TB_CHECK */
67
68 #if !defined(CONFIG_USER_ONLY)
69 /* TB consistency checks only implemented for usermode emulation. */
70 #undef DEBUG_TB_CHECK
71 #endif
72
73 /* Access to the various translations structures need to be serialised via locks
74 * for consistency. This is automatic for SoftMMU based system
75 * emulation due to its single threaded nature. In user-mode emulation
76 * access to the memory related structures are protected with the
77 * mmap_lock.
78 */
79 #ifdef CONFIG_SOFTMMU
80 #define assert_memory_lock() tcg_debug_assert(have_tb_lock)
81 #else
82 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
83 #endif
84
85 #define SMC_BITMAP_USE_THRESHOLD 10
86
87 typedef struct PageDesc {
88 /* list of TBs intersecting this ram page */
89 TranslationBlock *first_tb;
90 #ifdef CONFIG_SOFTMMU
91 /* in order to optimize self modifying code, we count the number
92 of lookups we do to a given page to use a bitmap */
93 unsigned int code_write_count;
94 unsigned long *code_bitmap;
95 #else
96 unsigned long flags;
97 #endif
98 } PageDesc;
99
100 /* In system mode we want L1_MAP to be based on ram offsets,
101 while in user mode we want it to be based on virtual addresses. */
102 #if !defined(CONFIG_USER_ONLY)
103 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
104 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
105 #else
106 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
107 #endif
108 #else
109 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
110 #endif
111
112 /* Size of the L2 (and L3, etc) page tables. */
113 #define V_L2_BITS 10
114 #define V_L2_SIZE (1 << V_L2_BITS)
115
116 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
117 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS >
118 sizeof(((TranslationBlock *)0)->trace_vcpu_dstate)
119 * BITS_PER_BYTE);
120
121 /*
122 * L1 Mapping properties
123 */
124 static int v_l1_size;
125 static int v_l1_shift;
126 static int v_l2_levels;
127
128 /* The bottom level has pointers to PageDesc, and is indexed by
129 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
130 */
131 #define V_L1_MIN_BITS 4
132 #define V_L1_MAX_BITS (V_L2_BITS + 3)
133 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
134
135 static void *l1_map[V_L1_MAX_SIZE];
136
137 /* code generation context */
138 TCGContext tcg_ctx;
139 bool parallel_cpus;
140
141 /* translation block context */
142 static __thread int have_tb_lock;
143
144 static void page_table_config_init(void)
145 {
146 uint32_t v_l1_bits;
147
148 assert(TARGET_PAGE_BITS);
149 /* The bits remaining after N lower levels of page tables. */
150 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS;
151 if (v_l1_bits < V_L1_MIN_BITS) {
152 v_l1_bits += V_L2_BITS;
153 }
154
155 v_l1_size = 1 << v_l1_bits;
156 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits;
157 v_l2_levels = v_l1_shift / V_L2_BITS - 1;
158
159 assert(v_l1_bits <= V_L1_MAX_BITS);
160 assert(v_l1_shift % V_L2_BITS == 0);
161 assert(v_l2_levels >= 0);
162 }
163
164 #define assert_tb_locked() tcg_debug_assert(have_tb_lock)
165 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock)
166
167 void tb_lock(void)
168 {
169 assert_tb_unlocked();
170 qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
171 have_tb_lock++;
172 }
173
174 void tb_unlock(void)
175 {
176 assert_tb_locked();
177 have_tb_lock--;
178 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
179 }
180
181 void tb_lock_reset(void)
182 {
183 if (have_tb_lock) {
184 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
185 have_tb_lock = 0;
186 }
187 }
188
189 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
190
191 void cpu_gen_init(void)
192 {
193 tcg_context_init(&tcg_ctx);
194 }
195
196 /* Encode VAL as a signed leb128 sequence at P.
197 Return P incremented past the encoded value. */
198 static uint8_t *encode_sleb128(uint8_t *p, target_long val)
199 {
200 int more, byte;
201
202 do {
203 byte = val & 0x7f;
204 val >>= 7;
205 more = !((val == 0 && (byte & 0x40) == 0)
206 || (val == -1 && (byte & 0x40) != 0));
207 if (more) {
208 byte |= 0x80;
209 }
210 *p++ = byte;
211 } while (more);
212
213 return p;
214 }
215
216 /* Decode a signed leb128 sequence at *PP; increment *PP past the
217 decoded value. Return the decoded value. */
218 static target_long decode_sleb128(uint8_t **pp)
219 {
220 uint8_t *p = *pp;
221 target_long val = 0;
222 int byte, shift = 0;
223
224 do {
225 byte = *p++;
226 val |= (target_ulong)(byte & 0x7f) << shift;
227 shift += 7;
228 } while (byte & 0x80);
229 if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
230 val |= -(target_ulong)1 << shift;
231 }
232
233 *pp = p;
234 return val;
235 }
236
237 /* Encode the data collected about the instructions while compiling TB.
238 Place the data at BLOCK, and return the number of bytes consumed.
239
240 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
241 which come from the target's insn_start data, followed by a uintptr_t
242 which comes from the host pc of the end of the code implementing the insn.
243
244 Each line of the table is encoded as sleb128 deltas from the previous
245 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
246 That is, the first column is seeded with the guest pc, the last column
247 with the host pc, and the middle columns with zeros. */
248
249 static int encode_search(TranslationBlock *tb, uint8_t *block)
250 {
251 uint8_t *highwater = tcg_ctx.code_gen_highwater;
252 uint8_t *p = block;
253 int i, j, n;
254
255 tb->tc_search = block;
256
257 for (i = 0, n = tb->icount; i < n; ++i) {
258 target_ulong prev;
259
260 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
261 if (i == 0) {
262 prev = (j == 0 ? tb->pc : 0);
263 } else {
264 prev = tcg_ctx.gen_insn_data[i - 1][j];
265 }
266 p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev);
267 }
268 prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]);
269 p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev);
270
271 /* Test for (pending) buffer overflow. The assumption is that any
272 one row beginning below the high water mark cannot overrun
273 the buffer completely. Thus we can test for overflow after
274 encoding a row without having to check during encoding. */
275 if (unlikely(p > highwater)) {
276 return -1;
277 }
278 }
279
280 return p - block;
281 }
282
283 /* The cpu state corresponding to 'searched_pc' is restored.
284 * Called with tb_lock held.
285 */
286 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
287 uintptr_t searched_pc)
288 {
289 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
290 uintptr_t host_pc = (uintptr_t)tb->tc_ptr;
291 CPUArchState *env = cpu->env_ptr;
292 uint8_t *p = tb->tc_search;
293 int i, j, num_insns = tb->icount;
294 #ifdef CONFIG_PROFILER
295 int64_t ti = profile_getclock();
296 #endif
297
298 searched_pc -= GETPC_ADJ;
299
300 if (searched_pc < host_pc) {
301 return -1;
302 }
303
304 /* Reconstruct the stored insn data while looking for the point at
305 which the end of the insn exceeds the searched_pc. */
306 for (i = 0; i < num_insns; ++i) {
307 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
308 data[j] += decode_sleb128(&p);
309 }
310 host_pc += decode_sleb128(&p);
311 if (host_pc > searched_pc) {
312 goto found;
313 }
314 }
315 return -1;
316
317 found:
318 if (tb->cflags & CF_USE_ICOUNT) {
319 assert(use_icount);
320 /* Reset the cycle counter to the start of the block. */
321 cpu->icount_decr.u16.low += num_insns;
322 /* Clear the IO flag. */
323 cpu->can_do_io = 0;
324 }
325 cpu->icount_decr.u16.low -= i;
326 restore_state_to_opc(env, tb, data);
327
328 #ifdef CONFIG_PROFILER
329 tcg_ctx.restore_time += profile_getclock() - ti;
330 tcg_ctx.restore_count++;
331 #endif
332 return 0;
333 }
334
335 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
336 {
337 TranslationBlock *tb;
338 bool r = false;
339
340 /* A retaddr of zero is invalid so we really shouldn't have ended
341 * up here. The target code has likely forgotten to check retaddr
342 * != 0 before attempting to restore state. We return early to
343 * avoid blowing up on a recursive tb_lock(). The target must have
344 * previously survived a failed cpu_restore_state because
345 * tb_find_pc(0) would have failed anyway. It still should be
346 * fixed though.
347 */
348
349 if (!retaddr) {
350 return r;
351 }
352
353 tb_lock();
354 tb = tb_find_pc(retaddr);
355 if (tb) {
356 cpu_restore_state_from_tb(cpu, tb, retaddr);
357 if (tb->cflags & CF_NOCACHE) {
358 /* one-shot translation, invalidate it immediately */
359 tb_phys_invalidate(tb, -1);
360 tb_free(tb);
361 }
362 r = true;
363 }
364 tb_unlock();
365
366 return r;
367 }
368
369 static void page_init(void)
370 {
371 page_size_init();
372 page_table_config_init();
373
374 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
375 {
376 #ifdef HAVE_KINFO_GETVMMAP
377 struct kinfo_vmentry *freep;
378 int i, cnt;
379
380 freep = kinfo_getvmmap(getpid(), &cnt);
381 if (freep) {
382 mmap_lock();
383 for (i = 0; i < cnt; i++) {
384 unsigned long startaddr, endaddr;
385
386 startaddr = freep[i].kve_start;
387 endaddr = freep[i].kve_end;
388 if (h2g_valid(startaddr)) {
389 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
390
391 if (h2g_valid(endaddr)) {
392 endaddr = h2g(endaddr);
393 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
394 } else {
395 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
396 endaddr = ~0ul;
397 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
398 #endif
399 }
400 }
401 }
402 free(freep);
403 mmap_unlock();
404 }
405 #else
406 FILE *f;
407
408 last_brk = (unsigned long)sbrk(0);
409
410 f = fopen("/compat/linux/proc/self/maps", "r");
411 if (f) {
412 mmap_lock();
413
414 do {
415 unsigned long startaddr, endaddr;
416 int n;
417
418 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
419
420 if (n == 2 && h2g_valid(startaddr)) {
421 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
422
423 if (h2g_valid(endaddr)) {
424 endaddr = h2g(endaddr);
425 } else {
426 endaddr = ~0ul;
427 }
428 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
429 }
430 } while (!feof(f));
431
432 fclose(f);
433 mmap_unlock();
434 }
435 #endif
436 }
437 #endif
438 }
439
440 /* If alloc=1:
441 * Called with tb_lock held for system emulation.
442 * Called with mmap_lock held for user-mode emulation.
443 */
444 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
445 {
446 PageDesc *pd;
447 void **lp;
448 int i;
449
450 if (alloc) {
451 assert_memory_lock();
452 }
453
454 /* Level 1. Always allocated. */
455 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1));
456
457 /* Level 2..N-1. */
458 for (i = v_l2_levels; i > 0; i--) {
459 void **p = atomic_rcu_read(lp);
460
461 if (p == NULL) {
462 if (!alloc) {
463 return NULL;
464 }
465 p = g_new0(void *, V_L2_SIZE);
466 atomic_rcu_set(lp, p);
467 }
468
469 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
470 }
471
472 pd = atomic_rcu_read(lp);
473 if (pd == NULL) {
474 if (!alloc) {
475 return NULL;
476 }
477 pd = g_new0(PageDesc, V_L2_SIZE);
478 atomic_rcu_set(lp, pd);
479 }
480
481 return pd + (index & (V_L2_SIZE - 1));
482 }
483
484 static inline PageDesc *page_find(tb_page_addr_t index)
485 {
486 return page_find_alloc(index, 0);
487 }
488
489 #if defined(CONFIG_USER_ONLY)
490 /* Currently it is not recommended to allocate big chunks of data in
491 user mode. It will change when a dedicated libc will be used. */
492 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
493 region in which the guest needs to run. Revisit this. */
494 #define USE_STATIC_CODE_GEN_BUFFER
495 #endif
496
497 /* Minimum size of the code gen buffer. This number is randomly chosen,
498 but not so small that we can't have a fair number of TB's live. */
499 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
500
501 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
502 indicated, this is constrained by the range of direct branches on the
503 host cpu, as used by the TCG implementation of goto_tb. */
504 #if defined(__x86_64__)
505 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
506 #elif defined(__sparc__)
507 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
508 #elif defined(__powerpc64__)
509 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
510 #elif defined(__powerpc__)
511 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
512 #elif defined(__aarch64__)
513 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
514 #elif defined(__s390x__)
515 /* We have a +- 4GB range on the branches; leave some slop. */
516 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
517 #elif defined(__mips__)
518 /* We have a 256MB branch region, but leave room to make sure the
519 main executable is also within that region. */
520 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
521 #else
522 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
523 #endif
524
525 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
526
527 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
528 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
529 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
530
531 static inline size_t size_code_gen_buffer(size_t tb_size)
532 {
533 /* Size the buffer. */
534 if (tb_size == 0) {
535 #ifdef USE_STATIC_CODE_GEN_BUFFER
536 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
537 #else
538 /* ??? Needs adjustments. */
539 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
540 static buffer, we could size this on RESERVED_VA, on the text
541 segment size of the executable, or continue to use the default. */
542 tb_size = (unsigned long)(ram_size / 4);
543 #endif
544 }
545 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
546 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
547 }
548 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
549 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
550 }
551 return tb_size;
552 }
553
554 #ifdef __mips__
555 /* In order to use J and JAL within the code_gen_buffer, we require
556 that the buffer not cross a 256MB boundary. */
557 static inline bool cross_256mb(void *addr, size_t size)
558 {
559 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
560 }
561
562 /* We weren't able to allocate a buffer without crossing that boundary,
563 so make do with the larger portion of the buffer that doesn't cross.
564 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
565 static inline void *split_cross_256mb(void *buf1, size_t size1)
566 {
567 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
568 size_t size2 = buf1 + size1 - buf2;
569
570 size1 = buf2 - buf1;
571 if (size1 < size2) {
572 size1 = size2;
573 buf1 = buf2;
574 }
575
576 tcg_ctx.code_gen_buffer_size = size1;
577 return buf1;
578 }
579 #endif
580
581 #ifdef USE_STATIC_CODE_GEN_BUFFER
582 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
583 __attribute__((aligned(CODE_GEN_ALIGN)));
584
585 # ifdef _WIN32
586 static inline void do_protect(void *addr, long size, int prot)
587 {
588 DWORD old_protect;
589 VirtualProtect(addr, size, prot, &old_protect);
590 }
591
592 static inline void map_exec(void *addr, long size)
593 {
594 do_protect(addr, size, PAGE_EXECUTE_READWRITE);
595 }
596
597 static inline void map_none(void *addr, long size)
598 {
599 do_protect(addr, size, PAGE_NOACCESS);
600 }
601 # else
602 static inline void do_protect(void *addr, long size, int prot)
603 {
604 uintptr_t start, end;
605
606 start = (uintptr_t)addr;
607 start &= qemu_real_host_page_mask;
608
609 end = (uintptr_t)addr + size;
610 end = ROUND_UP(end, qemu_real_host_page_size);
611
612 mprotect((void *)start, end - start, prot);
613 }
614
615 static inline void map_exec(void *addr, long size)
616 {
617 do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC);
618 }
619
620 static inline void map_none(void *addr, long size)
621 {
622 do_protect(addr, size, PROT_NONE);
623 }
624 # endif /* WIN32 */
625
626 static inline void *alloc_code_gen_buffer(void)
627 {
628 void *buf = static_code_gen_buffer;
629 size_t full_size, size;
630
631 /* The size of the buffer, rounded down to end on a page boundary. */
632 full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer))
633 & qemu_real_host_page_mask) - (uintptr_t)buf;
634
635 /* Reserve a guard page. */
636 size = full_size - qemu_real_host_page_size;
637
638 /* Honor a command-line option limiting the size of the buffer. */
639 if (size > tcg_ctx.code_gen_buffer_size) {
640 size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size)
641 & qemu_real_host_page_mask) - (uintptr_t)buf;
642 }
643 tcg_ctx.code_gen_buffer_size = size;
644
645 #ifdef __mips__
646 if (cross_256mb(buf, size)) {
647 buf = split_cross_256mb(buf, size);
648 size = tcg_ctx.code_gen_buffer_size;
649 }
650 #endif
651
652 map_exec(buf, size);
653 map_none(buf + size, qemu_real_host_page_size);
654 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
655
656 return buf;
657 }
658 #elif defined(_WIN32)
659 static inline void *alloc_code_gen_buffer(void)
660 {
661 size_t size = tcg_ctx.code_gen_buffer_size;
662 void *buf1, *buf2;
663
664 /* Perform the allocation in two steps, so that the guard page
665 is reserved but uncommitted. */
666 buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size,
667 MEM_RESERVE, PAGE_NOACCESS);
668 if (buf1 != NULL) {
669 buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
670 assert(buf1 == buf2);
671 }
672
673 return buf1;
674 }
675 #else
676 static inline void *alloc_code_gen_buffer(void)
677 {
678 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
679 uintptr_t start = 0;
680 size_t size = tcg_ctx.code_gen_buffer_size;
681 void *buf;
682
683 /* Constrain the position of the buffer based on the host cpu.
684 Note that these addresses are chosen in concert with the
685 addresses assigned in the relevant linker script file. */
686 # if defined(__PIE__) || defined(__PIC__)
687 /* Don't bother setting a preferred location if we're building
688 a position-independent executable. We're more likely to get
689 an address near the main executable if we let the kernel
690 choose the address. */
691 # elif defined(__x86_64__) && defined(MAP_32BIT)
692 /* Force the memory down into low memory with the executable.
693 Leave the choice of exact location with the kernel. */
694 flags |= MAP_32BIT;
695 /* Cannot expect to map more than 800MB in low memory. */
696 if (size > 800u * 1024 * 1024) {
697 tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024;
698 }
699 # elif defined(__sparc__)
700 start = 0x40000000ul;
701 # elif defined(__s390x__)
702 start = 0x90000000ul;
703 # elif defined(__mips__)
704 # if _MIPS_SIM == _ABI64
705 start = 0x128000000ul;
706 # else
707 start = 0x08000000ul;
708 # endif
709 # endif
710
711 buf = mmap((void *)start, size + qemu_real_host_page_size,
712 PROT_NONE, flags, -1, 0);
713 if (buf == MAP_FAILED) {
714 return NULL;
715 }
716
717 #ifdef __mips__
718 if (cross_256mb(buf, size)) {
719 /* Try again, with the original still mapped, to avoid re-acquiring
720 that 256mb crossing. This time don't specify an address. */
721 size_t size2;
722 void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
723 PROT_NONE, flags, -1, 0);
724 switch ((int)(buf2 != MAP_FAILED)) {
725 case 1:
726 if (!cross_256mb(buf2, size)) {
727 /* Success! Use the new buffer. */
728 munmap(buf, size + qemu_real_host_page_size);
729 break;
730 }
731 /* Failure. Work with what we had. */
732 munmap(buf2, size + qemu_real_host_page_size);
733 /* fallthru */
734 default:
735 /* Split the original buffer. Free the smaller half. */
736 buf2 = split_cross_256mb(buf, size);
737 size2 = tcg_ctx.code_gen_buffer_size;
738 if (buf == buf2) {
739 munmap(buf + size2 + qemu_real_host_page_size, size - size2);
740 } else {
741 munmap(buf, size - size2);
742 }
743 size = size2;
744 break;
745 }
746 buf = buf2;
747 }
748 #endif
749
750 /* Make the final buffer accessible. The guard page at the end
751 will remain inaccessible with PROT_NONE. */
752 mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC);
753
754 /* Request large pages for the buffer. */
755 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
756
757 return buf;
758 }
759 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
760
761 static inline void code_gen_alloc(size_t tb_size)
762 {
763 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
764 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
765 if (tcg_ctx.code_gen_buffer == NULL) {
766 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
767 exit(1);
768 }
769
770 /* size this conservatively -- realloc later if needed */
771 tcg_ctx.tb_ctx.tbs_size =
772 tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE / 8;
773 if (unlikely(!tcg_ctx.tb_ctx.tbs_size)) {
774 tcg_ctx.tb_ctx.tbs_size = 64 * 1024;
775 }
776 tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock *, tcg_ctx.tb_ctx.tbs_size);
777
778 qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
779 }
780
781 static void tb_htable_init(void)
782 {
783 unsigned int mode = QHT_MODE_AUTO_RESIZE;
784
785 qht_init(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode);
786 }
787
788 /* Must be called before using the QEMU cpus. 'tb_size' is the size
789 (in bytes) allocated to the translation buffer. Zero means default
790 size. */
791 void tcg_exec_init(unsigned long tb_size)
792 {
793 tcg_allowed = true;
794 cpu_gen_init();
795 page_init();
796 tb_htable_init();
797 code_gen_alloc(tb_size);
798 #if defined(CONFIG_SOFTMMU)
799 /* There's no guest base to take into account, so go ahead and
800 initialize the prologue now. */
801 tcg_prologue_init(&tcg_ctx);
802 #endif
803 }
804
805 /*
806 * Allocate a new translation block. Flush the translation buffer if
807 * too many translation blocks or too much generated code.
808 *
809 * Called with tb_lock held.
810 */
811 static TranslationBlock *tb_alloc(target_ulong pc)
812 {
813 TranslationBlock *tb;
814 TBContext *ctx;
815
816 assert_tb_locked();
817
818 tb = tcg_tb_alloc(&tcg_ctx);
819 if (unlikely(tb == NULL)) {
820 return NULL;
821 }
822 ctx = &tcg_ctx.tb_ctx;
823 if (unlikely(ctx->nb_tbs == ctx->tbs_size)) {
824 ctx->tbs_size *= 2;
825 ctx->tbs = g_renew(TranslationBlock *, ctx->tbs, ctx->tbs_size);
826 }
827 ctx->tbs[ctx->nb_tbs++] = tb;
828 return tb;
829 }
830
831 /* Called with tb_lock held. */
832 void tb_free(TranslationBlock *tb)
833 {
834 assert_tb_locked();
835
836 /* In practice this is mostly used for single use temporary TB
837 Ignore the hard cases and just back up if this TB happens to
838 be the last one generated. */
839 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
840 tb == tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
841 size_t struct_size = ROUND_UP(sizeof(*tb), qemu_icache_linesize);
842
843 tcg_ctx.code_gen_ptr = tb->tc_ptr - struct_size;
844 tcg_ctx.tb_ctx.nb_tbs--;
845 }
846 }
847
848 static inline void invalidate_page_bitmap(PageDesc *p)
849 {
850 #ifdef CONFIG_SOFTMMU
851 g_free(p->code_bitmap);
852 p->code_bitmap = NULL;
853 p->code_write_count = 0;
854 #endif
855 }
856
857 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
858 static void page_flush_tb_1(int level, void **lp)
859 {
860 int i;
861
862 if (*lp == NULL) {
863 return;
864 }
865 if (level == 0) {
866 PageDesc *pd = *lp;
867
868 for (i = 0; i < V_L2_SIZE; ++i) {
869 pd[i].first_tb = NULL;
870 invalidate_page_bitmap(pd + i);
871 }
872 } else {
873 void **pp = *lp;
874
875 for (i = 0; i < V_L2_SIZE; ++i) {
876 page_flush_tb_1(level - 1, pp + i);
877 }
878 }
879 }
880
881 static void page_flush_tb(void)
882 {
883 int i, l1_sz = v_l1_size;
884
885 for (i = 0; i < l1_sz; i++) {
886 page_flush_tb_1(v_l2_levels, l1_map + i);
887 }
888 }
889
890 /* flush all the translation blocks */
891 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
892 {
893 tb_lock();
894
895 /* If it is already been done on request of another CPU,
896 * just retry.
897 */
898 if (tcg_ctx.tb_ctx.tb_flush_count != tb_flush_count.host_int) {
899 goto done;
900 }
901
902 #if defined(DEBUG_TB_FLUSH)
903 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
904 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
905 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
906 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
907 tcg_ctx.tb_ctx.nb_tbs : 0);
908 #endif
909 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
910 > tcg_ctx.code_gen_buffer_size) {
911 cpu_abort(cpu, "Internal error: code buffer overflow\n");
912 }
913
914 CPU_FOREACH(cpu) {
915 cpu_tb_jmp_cache_clear(cpu);
916 }
917
918 tcg_ctx.tb_ctx.nb_tbs = 0;
919 qht_reset_size(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
920 page_flush_tb();
921
922 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
923 /* XXX: flush processor icache at this point if cache flush is
924 expensive */
925 atomic_mb_set(&tcg_ctx.tb_ctx.tb_flush_count,
926 tcg_ctx.tb_ctx.tb_flush_count + 1);
927
928 done:
929 tb_unlock();
930 }
931
932 void tb_flush(CPUState *cpu)
933 {
934 if (tcg_enabled()) {
935 unsigned tb_flush_count = atomic_mb_read(&tcg_ctx.tb_ctx.tb_flush_count);
936 async_safe_run_on_cpu(cpu, do_tb_flush,
937 RUN_ON_CPU_HOST_INT(tb_flush_count));
938 }
939 }
940
941 #ifdef DEBUG_TB_CHECK
942
943 static void
944 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp)
945 {
946 TranslationBlock *tb = p;
947 target_ulong addr = *(target_ulong *)userp;
948
949 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) {
950 printf("ERROR invalidate: address=" TARGET_FMT_lx
951 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size);
952 }
953 }
954
955 /* verify that all the pages have correct rights for code
956 *
957 * Called with tb_lock held.
958 */
959 static void tb_invalidate_check(target_ulong address)
960 {
961 address &= TARGET_PAGE_MASK;
962 qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_invalidate_check, &address);
963 }
964
965 static void
966 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp)
967 {
968 TranslationBlock *tb = p;
969 int flags1, flags2;
970
971 flags1 = page_get_flags(tb->pc);
972 flags2 = page_get_flags(tb->pc + tb->size - 1);
973 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
974 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
975 (long)tb->pc, tb->size, flags1, flags2);
976 }
977 }
978
979 /* verify that all the pages have correct rights for code */
980 static void tb_page_check(void)
981 {
982 qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_page_check, NULL);
983 }
984
985 #endif
986
987 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
988 {
989 TranslationBlock *tb1;
990 unsigned int n1;
991
992 for (;;) {
993 tb1 = *ptb;
994 n1 = (uintptr_t)tb1 & 3;
995 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
996 if (tb1 == tb) {
997 *ptb = tb1->page_next[n1];
998 break;
999 }
1000 ptb = &tb1->page_next[n1];
1001 }
1002 }
1003
1004 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
1005 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n)
1006 {
1007 TranslationBlock *tb1;
1008 uintptr_t *ptb, ntb;
1009 unsigned int n1;
1010
1011 ptb = &tb->jmp_list_next[n];
1012 if (*ptb) {
1013 /* find tb(n) in circular list */
1014 for (;;) {
1015 ntb = *ptb;
1016 n1 = ntb & 3;
1017 tb1 = (TranslationBlock *)(ntb & ~3);
1018 if (n1 == n && tb1 == tb) {
1019 break;
1020 }
1021 if (n1 == 2) {
1022 ptb = &tb1->jmp_list_first;
1023 } else {
1024 ptb = &tb1->jmp_list_next[n1];
1025 }
1026 }
1027 /* now we can suppress tb(n) from the list */
1028 *ptb = tb->jmp_list_next[n];
1029
1030 tb->jmp_list_next[n] = (uintptr_t)NULL;
1031 }
1032 }
1033
1034 /* reset the jump entry 'n' of a TB so that it is not chained to
1035 another TB */
1036 static inline void tb_reset_jump(TranslationBlock *tb, int n)
1037 {
1038 uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]);
1039 tb_set_jmp_target(tb, n, addr);
1040 }
1041
1042 /* remove any jumps to the TB */
1043 static inline void tb_jmp_unlink(TranslationBlock *tb)
1044 {
1045 TranslationBlock *tb1;
1046 uintptr_t *ptb, ntb;
1047 unsigned int n1;
1048
1049 ptb = &tb->jmp_list_first;
1050 for (;;) {
1051 ntb = *ptb;
1052 n1 = ntb & 3;
1053 tb1 = (TranslationBlock *)(ntb & ~3);
1054 if (n1 == 2) {
1055 break;
1056 }
1057 tb_reset_jump(tb1, n1);
1058 *ptb = tb1->jmp_list_next[n1];
1059 tb1->jmp_list_next[n1] = (uintptr_t)NULL;
1060 }
1061 }
1062
1063 /* invalidate one TB
1064 *
1065 * Called with tb_lock held.
1066 */
1067 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
1068 {
1069 CPUState *cpu;
1070 PageDesc *p;
1071 uint32_t h;
1072 tb_page_addr_t phys_pc;
1073
1074 assert_tb_locked();
1075
1076 atomic_set(&tb->invalid, true);
1077
1078 /* remove the TB from the hash list */
1079 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1080 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->trace_vcpu_dstate);
1081 qht_remove(&tcg_ctx.tb_ctx.htable, tb, h);
1082
1083 /* remove the TB from the page list */
1084 if (tb->page_addr[0] != page_addr) {
1085 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
1086 tb_page_remove(&p->first_tb, tb);
1087 invalidate_page_bitmap(p);
1088 }
1089 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
1090 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
1091 tb_page_remove(&p->first_tb, tb);
1092 invalidate_page_bitmap(p);
1093 }
1094
1095 /* remove the TB from the hash list */
1096 h = tb_jmp_cache_hash_func(tb->pc);
1097 CPU_FOREACH(cpu) {
1098 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) {
1099 atomic_set(&cpu->tb_jmp_cache[h], NULL);
1100 }
1101 }
1102
1103 /* suppress this TB from the two jump lists */
1104 tb_remove_from_jmp_list(tb, 0);
1105 tb_remove_from_jmp_list(tb, 1);
1106
1107 /* suppress any remaining jumps to this TB */
1108 tb_jmp_unlink(tb);
1109
1110 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
1111 }
1112
1113 #ifdef CONFIG_SOFTMMU
1114 static void build_page_bitmap(PageDesc *p)
1115 {
1116 int n, tb_start, tb_end;
1117 TranslationBlock *tb;
1118
1119 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
1120
1121 tb = p->first_tb;
1122 while (tb != NULL) {
1123 n = (uintptr_t)tb & 3;
1124 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1125 /* NOTE: this is subtle as a TB may span two physical pages */
1126 if (n == 0) {
1127 /* NOTE: tb_end may be after the end of the page, but
1128 it is not a problem */
1129 tb_start = tb->pc & ~TARGET_PAGE_MASK;
1130 tb_end = tb_start + tb->size;
1131 if (tb_end > TARGET_PAGE_SIZE) {
1132 tb_end = TARGET_PAGE_SIZE;
1133 }
1134 } else {
1135 tb_start = 0;
1136 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1137 }
1138 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
1139 tb = tb->page_next[n];
1140 }
1141 }
1142 #endif
1143
1144 /* add the tb in the target page and protect it if necessary
1145 *
1146 * Called with mmap_lock held for user-mode emulation.
1147 */
1148 static inline void tb_alloc_page(TranslationBlock *tb,
1149 unsigned int n, tb_page_addr_t page_addr)
1150 {
1151 PageDesc *p;
1152 #ifndef CONFIG_USER_ONLY
1153 bool page_already_protected;
1154 #endif
1155
1156 assert_memory_lock();
1157
1158 tb->page_addr[n] = page_addr;
1159 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1160 tb->page_next[n] = p->first_tb;
1161 #ifndef CONFIG_USER_ONLY
1162 page_already_protected = p->first_tb != NULL;
1163 #endif
1164 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1165 invalidate_page_bitmap(p);
1166
1167 #if defined(CONFIG_USER_ONLY)
1168 if (p->flags & PAGE_WRITE) {
1169 target_ulong addr;
1170 PageDesc *p2;
1171 int prot;
1172
1173 /* force the host page as non writable (writes will have a
1174 page fault + mprotect overhead) */
1175 page_addr &= qemu_host_page_mask;
1176 prot = 0;
1177 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1178 addr += TARGET_PAGE_SIZE) {
1179
1180 p2 = page_find(addr >> TARGET_PAGE_BITS);
1181 if (!p2) {
1182 continue;
1183 }
1184 prot |= p2->flags;
1185 p2->flags &= ~PAGE_WRITE;
1186 }
1187 mprotect(g2h(page_addr), qemu_host_page_size,
1188 (prot & PAGE_BITS) & ~PAGE_WRITE);
1189 #ifdef DEBUG_TB_INVALIDATE
1190 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1191 page_addr);
1192 #endif
1193 }
1194 #else
1195 /* if some code is already present, then the pages are already
1196 protected. So we handle the case where only the first TB is
1197 allocated in a physical page */
1198 if (!page_already_protected) {
1199 tlb_protect_code(page_addr);
1200 }
1201 #endif
1202 }
1203
1204 /* add a new TB and link it to the physical page tables. phys_page2 is
1205 * (-1) to indicate that only one page contains the TB.
1206 *
1207 * Called with mmap_lock held for user-mode emulation.
1208 */
1209 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1210 tb_page_addr_t phys_page2)
1211 {
1212 uint32_t h;
1213
1214 assert_memory_lock();
1215
1216 /* add in the page list */
1217 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1218 if (phys_page2 != -1) {
1219 tb_alloc_page(tb, 1, phys_page2);
1220 } else {
1221 tb->page_addr[1] = -1;
1222 }
1223
1224 /* add in the hash table */
1225 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->trace_vcpu_dstate);
1226 qht_insert(&tcg_ctx.tb_ctx.htable, tb, h);
1227
1228 #ifdef DEBUG_TB_CHECK
1229 tb_page_check();
1230 #endif
1231 }
1232
1233 /* Called with mmap_lock held for user mode emulation. */
1234 TranslationBlock *tb_gen_code(CPUState *cpu,
1235 target_ulong pc, target_ulong cs_base,
1236 uint32_t flags, int cflags)
1237 {
1238 CPUArchState *env = cpu->env_ptr;
1239 TranslationBlock *tb;
1240 tb_page_addr_t phys_pc, phys_page2;
1241 target_ulong virt_page2;
1242 tcg_insn_unit *gen_code_buf;
1243 int gen_code_size, search_size;
1244 #ifdef CONFIG_PROFILER
1245 int64_t ti;
1246 #endif
1247 assert_memory_lock();
1248
1249 phys_pc = get_page_addr_code(env, pc);
1250 if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {
1251 cflags |= CF_USE_ICOUNT;
1252 }
1253
1254 tb = tb_alloc(pc);
1255 if (unlikely(!tb)) {
1256 buffer_overflow:
1257 /* flush must be done */
1258 tb_flush(cpu);
1259 mmap_unlock();
1260 /* Make the execution loop process the flush as soon as possible. */
1261 cpu->exception_index = EXCP_INTERRUPT;
1262 cpu_loop_exit(cpu);
1263 }
1264
1265 gen_code_buf = tcg_ctx.code_gen_ptr;
1266 tb->tc_ptr = gen_code_buf;
1267 tb->pc = pc;
1268 tb->cs_base = cs_base;
1269 tb->flags = flags;
1270 tb->cflags = cflags;
1271 tb->trace_vcpu_dstate = *cpu->trace_dstate;
1272 tb->invalid = false;
1273
1274 #ifdef CONFIG_PROFILER
1275 tcg_ctx.tb_count1++; /* includes aborted translations because of
1276 exceptions */
1277 ti = profile_getclock();
1278 #endif
1279
1280 tcg_func_start(&tcg_ctx);
1281
1282 tcg_ctx.cpu = ENV_GET_CPU(env);
1283 gen_intermediate_code(cpu, tb);
1284 tcg_ctx.cpu = NULL;
1285
1286 trace_translate_block(tb, tb->pc, tb->tc_ptr);
1287
1288 /* generate machine code */
1289 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
1290 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
1291 tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset;
1292 if (TCG_TARGET_HAS_direct_jump) {
1293 tcg_ctx.tb_jmp_insn_offset = tb->jmp_target_arg;
1294 tcg_ctx.tb_jmp_target_addr = NULL;
1295 } else {
1296 tcg_ctx.tb_jmp_insn_offset = NULL;
1297 tcg_ctx.tb_jmp_target_addr = tb->jmp_target_arg;
1298 }
1299
1300 #ifdef CONFIG_PROFILER
1301 tcg_ctx.tb_count++;
1302 tcg_ctx.interm_time += profile_getclock() - ti;
1303 ti = profile_getclock();
1304 #endif
1305
1306 /* ??? Overflow could be handled better here. In particular, we
1307 don't need to re-do gen_intermediate_code, nor should we re-do
1308 the tcg optimization currently hidden inside tcg_gen_code. All
1309 that should be required is to flush the TBs, allocate a new TB,
1310 re-initialize it per above, and re-do the actual code generation. */
1311 gen_code_size = tcg_gen_code(&tcg_ctx, tb);
1312 if (unlikely(gen_code_size < 0)) {
1313 goto buffer_overflow;
1314 }
1315 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
1316 if (unlikely(search_size < 0)) {
1317 goto buffer_overflow;
1318 }
1319
1320 #ifdef CONFIG_PROFILER
1321 tcg_ctx.code_time += profile_getclock() - ti;
1322 tcg_ctx.code_in_len += tb->size;
1323 tcg_ctx.code_out_len += gen_code_size;
1324 tcg_ctx.search_out_len += search_size;
1325 #endif
1326
1327 #ifdef DEBUG_DISAS
1328 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
1329 qemu_log_in_addr_range(tb->pc)) {
1330 qemu_log_lock();
1331 qemu_log("OUT: [size=%d]\n", gen_code_size);
1332 if (tcg_ctx.data_gen_ptr) {
1333 size_t code_size = tcg_ctx.data_gen_ptr - tb->tc_ptr;
1334 size_t data_size = gen_code_size - code_size;
1335 size_t i;
1336
1337 log_disas(tb->tc_ptr, code_size);
1338
1339 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1340 if (sizeof(tcg_target_ulong) == 8) {
1341 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1342 (uintptr_t)tcg_ctx.data_gen_ptr + i,
1343 *(uint64_t *)(tcg_ctx.data_gen_ptr + i));
1344 } else {
1345 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
1346 (uintptr_t)tcg_ctx.data_gen_ptr + i,
1347 *(uint32_t *)(tcg_ctx.data_gen_ptr + i));
1348 }
1349 }
1350 } else {
1351 log_disas(tb->tc_ptr, gen_code_size);
1352 }
1353 qemu_log("\n");
1354 qemu_log_flush();
1355 qemu_log_unlock();
1356 }
1357 #endif
1358
1359 tcg_ctx.code_gen_ptr = (void *)
1360 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
1361 CODE_GEN_ALIGN);
1362
1363 /* init jump list */
1364 assert(((uintptr_t)tb & 3) == 0);
1365 tb->jmp_list_first = (uintptr_t)tb | 2;
1366 tb->jmp_list_next[0] = (uintptr_t)NULL;
1367 tb->jmp_list_next[1] = (uintptr_t)NULL;
1368
1369 /* init original jump addresses wich has been set during tcg_gen_code() */
1370 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1371 tb_reset_jump(tb, 0);
1372 }
1373 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1374 tb_reset_jump(tb, 1);
1375 }
1376
1377 /* check next page if needed */
1378 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1379 phys_page2 = -1;
1380 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1381 phys_page2 = get_page_addr_code(env, virt_page2);
1382 }
1383 /* As long as consistency of the TB stuff is provided by tb_lock in user
1384 * mode and is implicit in single-threaded softmmu emulation, no explicit
1385 * memory barrier is required before tb_link_page() makes the TB visible
1386 * through the physical hash table and physical page list.
1387 */
1388 tb_link_page(tb, phys_pc, phys_page2);
1389 return tb;
1390 }
1391
1392 /*
1393 * Invalidate all TBs which intersect with the target physical address range
1394 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1395 * 'is_cpu_write_access' should be true if called from a real cpu write
1396 * access: the virtual CPU will exit the current TB if code is modified inside
1397 * this TB.
1398 *
1399 * Called with mmap_lock held for user-mode emulation, grabs tb_lock
1400 * Called with tb_lock held for system-mode emulation
1401 */
1402 static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end)
1403 {
1404 while (start < end) {
1405 tb_invalidate_phys_page_range(start, end, 0);
1406 start &= TARGET_PAGE_MASK;
1407 start += TARGET_PAGE_SIZE;
1408 }
1409 }
1410
1411 #ifdef CONFIG_SOFTMMU
1412 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
1413 {
1414 assert_tb_locked();
1415 tb_invalidate_phys_range_1(start, end);
1416 }
1417 #else
1418 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
1419 {
1420 assert_memory_lock();
1421 tb_lock();
1422 tb_invalidate_phys_range_1(start, end);
1423 tb_unlock();
1424 }
1425 #endif
1426 /*
1427 * Invalidate all TBs which intersect with the target physical address range
1428 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1429 * 'is_cpu_write_access' should be true if called from a real cpu write
1430 * access: the virtual CPU will exit the current TB if code is modified inside
1431 * this TB.
1432 *
1433 * Called with tb_lock/mmap_lock held for user-mode emulation
1434 * Called with tb_lock held for system-mode emulation
1435 */
1436 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1437 int is_cpu_write_access)
1438 {
1439 TranslationBlock *tb, *tb_next;
1440 #if defined(TARGET_HAS_PRECISE_SMC)
1441 CPUState *cpu = current_cpu;
1442 CPUArchState *env = NULL;
1443 #endif
1444 tb_page_addr_t tb_start, tb_end;
1445 PageDesc *p;
1446 int n;
1447 #ifdef TARGET_HAS_PRECISE_SMC
1448 int current_tb_not_found = is_cpu_write_access;
1449 TranslationBlock *current_tb = NULL;
1450 int current_tb_modified = 0;
1451 target_ulong current_pc = 0;
1452 target_ulong current_cs_base = 0;
1453 uint32_t current_flags = 0;
1454 #endif /* TARGET_HAS_PRECISE_SMC */
1455
1456 assert_memory_lock();
1457 assert_tb_locked();
1458
1459 p = page_find(start >> TARGET_PAGE_BITS);
1460 if (!p) {
1461 return;
1462 }
1463 #if defined(TARGET_HAS_PRECISE_SMC)
1464 if (cpu != NULL) {
1465 env = cpu->env_ptr;
1466 }
1467 #endif
1468
1469 /* we remove all the TBs in the range [start, end[ */
1470 /* XXX: see if in some cases it could be faster to invalidate all
1471 the code */
1472 tb = p->first_tb;
1473 while (tb != NULL) {
1474 n = (uintptr_t)tb & 3;
1475 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1476 tb_next = tb->page_next[n];
1477 /* NOTE: this is subtle as a TB may span two physical pages */
1478 if (n == 0) {
1479 /* NOTE: tb_end may be after the end of the page, but
1480 it is not a problem */
1481 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1482 tb_end = tb_start + tb->size;
1483 } else {
1484 tb_start = tb->page_addr[1];
1485 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1486 }
1487 if (!(tb_end <= start || tb_start >= end)) {
1488 #ifdef TARGET_HAS_PRECISE_SMC
1489 if (current_tb_not_found) {
1490 current_tb_not_found = 0;
1491 current_tb = NULL;
1492 if (cpu->mem_io_pc) {
1493 /* now we have a real cpu fault */
1494 current_tb = tb_find_pc(cpu->mem_io_pc);
1495 }
1496 }
1497 if (current_tb == tb &&
1498 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1499 /* If we are modifying the current TB, we must stop
1500 its execution. We could be more precise by checking
1501 that the modification is after the current PC, but it
1502 would require a specialized function to partially
1503 restore the CPU state */
1504
1505 current_tb_modified = 1;
1506 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
1507 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1508 &current_flags);
1509 }
1510 #endif /* TARGET_HAS_PRECISE_SMC */
1511 tb_phys_invalidate(tb, -1);
1512 }
1513 tb = tb_next;
1514 }
1515 #if !defined(CONFIG_USER_ONLY)
1516 /* if no code remaining, no need to continue to use slow writes */
1517 if (!p->first_tb) {
1518 invalidate_page_bitmap(p);
1519 tlb_unprotect_code(start);
1520 }
1521 #endif
1522 #ifdef TARGET_HAS_PRECISE_SMC
1523 if (current_tb_modified) {
1524 /* we generate a block containing just the instruction
1525 modifying the memory. It will ensure that it cannot modify
1526 itself */
1527 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1528 cpu_loop_exit_noexc(cpu);
1529 }
1530 #endif
1531 }
1532
1533 #ifdef CONFIG_SOFTMMU
1534 /* len must be <= 8 and start must be a multiple of len.
1535 * Called via softmmu_template.h when code areas are written to with
1536 * iothread mutex not held.
1537 */
1538 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1539 {
1540 PageDesc *p;
1541
1542 #if 0
1543 if (1) {
1544 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1545 cpu_single_env->mem_io_vaddr, len,
1546 cpu_single_env->eip,
1547 cpu_single_env->eip +
1548 (intptr_t)cpu_single_env->segs[R_CS].base);
1549 }
1550 #endif
1551 assert_memory_lock();
1552
1553 p = page_find(start >> TARGET_PAGE_BITS);
1554 if (!p) {
1555 return;
1556 }
1557 if (!p->code_bitmap &&
1558 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
1559 /* build code bitmap. FIXME: writes should be protected by
1560 * tb_lock, reads by tb_lock or RCU.
1561 */
1562 build_page_bitmap(p);
1563 }
1564 if (p->code_bitmap) {
1565 unsigned int nr;
1566 unsigned long b;
1567
1568 nr = start & ~TARGET_PAGE_MASK;
1569 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
1570 if (b & ((1 << len) - 1)) {
1571 goto do_invalidate;
1572 }
1573 } else {
1574 do_invalidate:
1575 tb_invalidate_phys_page_range(start, start + len, 1);
1576 }
1577 }
1578 #else
1579 /* Called with mmap_lock held. If pc is not 0 then it indicates the
1580 * host PC of the faulting store instruction that caused this invalidate.
1581 * Returns true if the caller needs to abort execution of the current
1582 * TB (because it was modified by this store and the guest CPU has
1583 * precise-SMC semantics).
1584 */
1585 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
1586 {
1587 TranslationBlock *tb;
1588 PageDesc *p;
1589 int n;
1590 #ifdef TARGET_HAS_PRECISE_SMC
1591 TranslationBlock *current_tb = NULL;
1592 CPUState *cpu = current_cpu;
1593 CPUArchState *env = NULL;
1594 int current_tb_modified = 0;
1595 target_ulong current_pc = 0;
1596 target_ulong current_cs_base = 0;
1597 uint32_t current_flags = 0;
1598 #endif
1599
1600 assert_memory_lock();
1601
1602 addr &= TARGET_PAGE_MASK;
1603 p = page_find(addr >> TARGET_PAGE_BITS);
1604 if (!p) {
1605 return false;
1606 }
1607
1608 tb_lock();
1609 tb = p->first_tb;
1610 #ifdef TARGET_HAS_PRECISE_SMC
1611 if (tb && pc != 0) {
1612 current_tb = tb_find_pc(pc);
1613 }
1614 if (cpu != NULL) {
1615 env = cpu->env_ptr;
1616 }
1617 #endif
1618 while (tb != NULL) {
1619 n = (uintptr_t)tb & 3;
1620 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1621 #ifdef TARGET_HAS_PRECISE_SMC
1622 if (current_tb == tb &&
1623 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1624 /* If we are modifying the current TB, we must stop
1625 its execution. We could be more precise by checking
1626 that the modification is after the current PC, but it
1627 would require a specialized function to partially
1628 restore the CPU state */
1629
1630 current_tb_modified = 1;
1631 cpu_restore_state_from_tb(cpu, current_tb, pc);
1632 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1633 &current_flags);
1634 }
1635 #endif /* TARGET_HAS_PRECISE_SMC */
1636 tb_phys_invalidate(tb, addr);
1637 tb = tb->page_next[n];
1638 }
1639 p->first_tb = NULL;
1640 #ifdef TARGET_HAS_PRECISE_SMC
1641 if (current_tb_modified) {
1642 /* we generate a block containing just the instruction
1643 modifying the memory. It will ensure that it cannot modify
1644 itself */
1645 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1646 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps
1647 * back into the cpu_exec loop. */
1648 return true;
1649 }
1650 #endif
1651 tb_unlock();
1652
1653 return false;
1654 }
1655 #endif
1656
1657 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1658 tb[1].tc_ptr. Return NULL if not found */
1659 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1660 {
1661 int m_min, m_max, m;
1662 uintptr_t v;
1663 TranslationBlock *tb;
1664
1665 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1666 return NULL;
1667 }
1668 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1669 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1670 return NULL;
1671 }
1672 /* binary search (cf Knuth) */
1673 m_min = 0;
1674 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1675 while (m_min <= m_max) {
1676 m = (m_min + m_max) >> 1;
1677 tb = tcg_ctx.tb_ctx.tbs[m];
1678 v = (uintptr_t)tb->tc_ptr;
1679 if (v == tc_ptr) {
1680 return tb;
1681 } else if (tc_ptr < v) {
1682 m_max = m - 1;
1683 } else {
1684 m_min = m + 1;
1685 }
1686 }
1687 return tcg_ctx.tb_ctx.tbs[m_max];
1688 }
1689
1690 #if !defined(CONFIG_USER_ONLY)
1691 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1692 {
1693 ram_addr_t ram_addr;
1694 MemoryRegion *mr;
1695 hwaddr l = 1;
1696
1697 rcu_read_lock();
1698 mr = address_space_translate(as, addr, &addr, &l, false);
1699 if (!(memory_region_is_ram(mr)
1700 || memory_region_is_romd(mr))) {
1701 rcu_read_unlock();
1702 return;
1703 }
1704 ram_addr = memory_region_get_ram_addr(mr) + addr;
1705 tb_lock();
1706 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1707 tb_unlock();
1708 rcu_read_unlock();
1709 }
1710 #endif /* !defined(CONFIG_USER_ONLY) */
1711
1712 /* Called with tb_lock held. */
1713 void tb_check_watchpoint(CPUState *cpu)
1714 {
1715 TranslationBlock *tb;
1716
1717 tb = tb_find_pc(cpu->mem_io_pc);
1718 if (tb) {
1719 /* We can use retranslation to find the PC. */
1720 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
1721 tb_phys_invalidate(tb, -1);
1722 } else {
1723 /* The exception probably happened in a helper. The CPU state should
1724 have been saved before calling it. Fetch the PC from there. */
1725 CPUArchState *env = cpu->env_ptr;
1726 target_ulong pc, cs_base;
1727 tb_page_addr_t addr;
1728 uint32_t flags;
1729
1730 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
1731 addr = get_page_addr_code(env, pc);
1732 tb_invalidate_phys_range(addr, addr + 1);
1733 }
1734 }
1735
1736 #ifndef CONFIG_USER_ONLY
1737 /* in deterministic execution mode, instructions doing device I/Os
1738 * must be at the end of the TB.
1739 *
1740 * Called by softmmu_template.h, with iothread mutex not held.
1741 */
1742 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
1743 {
1744 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1745 CPUArchState *env = cpu->env_ptr;
1746 #endif
1747 TranslationBlock *tb;
1748 uint32_t n, cflags;
1749 target_ulong pc, cs_base;
1750 uint32_t flags;
1751
1752 tb_lock();
1753 tb = tb_find_pc(retaddr);
1754 if (!tb) {
1755 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
1756 (void *)retaddr);
1757 }
1758 n = cpu->icount_decr.u16.low + tb->icount;
1759 cpu_restore_state_from_tb(cpu, tb, retaddr);
1760 /* Calculate how many instructions had been executed before the fault
1761 occurred. */
1762 n = n - cpu->icount_decr.u16.low;
1763 /* Generate a new TB ending on the I/O insn. */
1764 n++;
1765 /* On MIPS and SH, delay slot instructions can only be restarted if
1766 they were already the first instruction in the TB. If this is not
1767 the first instruction in a TB then re-execute the preceding
1768 branch. */
1769 #if defined(TARGET_MIPS)
1770 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1771 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
1772 cpu->icount_decr.u16.low++;
1773 env->hflags &= ~MIPS_HFLAG_BMASK;
1774 }
1775 #elif defined(TARGET_SH4)
1776 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1777 && n > 1) {
1778 env->pc -= 2;
1779 cpu->icount_decr.u16.low++;
1780 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1781 }
1782 #endif
1783 /* This should never happen. */
1784 if (n > CF_COUNT_MASK) {
1785 cpu_abort(cpu, "TB too big during recompile");
1786 }
1787
1788 cflags = n | CF_LAST_IO;
1789 pc = tb->pc;
1790 cs_base = tb->cs_base;
1791 flags = tb->flags;
1792 tb_phys_invalidate(tb, -1);
1793 if (tb->cflags & CF_NOCACHE) {
1794 if (tb->orig_tb) {
1795 /* Invalidate original TB if this TB was generated in
1796 * cpu_exec_nocache() */
1797 tb_phys_invalidate(tb->orig_tb, -1);
1798 }
1799 tb_free(tb);
1800 }
1801 /* FIXME: In theory this could raise an exception. In practice
1802 we have already translated the block once so it's probably ok. */
1803 tb_gen_code(cpu, pc, cs_base, flags, cflags);
1804
1805 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1806 * the first in the TB) then we end up generating a whole new TB and
1807 * repeating the fault, which is horribly inefficient.
1808 * Better would be to execute just this insn uncached, or generate a
1809 * second new TB.
1810 *
1811 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the
1812 * tb_lock gets reset.
1813 */
1814 cpu_loop_exit_noexc(cpu);
1815 }
1816
1817 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr)
1818 {
1819 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr);
1820
1821 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) {
1822 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL);
1823 }
1824 }
1825
1826 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
1827 {
1828 /* Discard jump cache entries for any tb which might potentially
1829 overlap the flushed page. */
1830 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE);
1831 tb_jmp_cache_clear_page(cpu, addr);
1832 }
1833
1834 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf,
1835 struct qht_stats hst)
1836 {
1837 uint32_t hgram_opts;
1838 size_t hgram_bins;
1839 char *hgram;
1840
1841 if (!hst.head_buckets) {
1842 return;
1843 }
1844 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
1845 hst.used_head_buckets, hst.head_buckets,
1846 (double)hst.used_head_buckets / hst.head_buckets * 100);
1847
1848 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
1849 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT;
1850 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) {
1851 hgram_opts |= QDIST_PR_NODECIMAL;
1852 }
1853 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts);
1854 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
1855 qdist_avg(&hst.occupancy) * 100, hgram);
1856 g_free(hgram);
1857
1858 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
1859 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain);
1860 if (hgram_bins > 10) {
1861 hgram_bins = 10;
1862 } else {
1863 hgram_bins = 0;
1864 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE;
1865 }
1866 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts);
1867 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
1868 qdist_avg(&hst.chain), hgram);
1869 g_free(hgram);
1870 }
1871
1872 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1873 {
1874 int i, target_code_size, max_target_code_size;
1875 int direct_jmp_count, direct_jmp2_count, cross_page;
1876 TranslationBlock *tb;
1877 struct qht_stats hst;
1878
1879 tb_lock();
1880
1881 target_code_size = 0;
1882 max_target_code_size = 0;
1883 cross_page = 0;
1884 direct_jmp_count = 0;
1885 direct_jmp2_count = 0;
1886 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1887 tb = tcg_ctx.tb_ctx.tbs[i];
1888 target_code_size += tb->size;
1889 if (tb->size > max_target_code_size) {
1890 max_target_code_size = tb->size;
1891 }
1892 if (tb->page_addr[1] != -1) {
1893 cross_page++;
1894 }
1895 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1896 direct_jmp_count++;
1897 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1898 direct_jmp2_count++;
1899 }
1900 }
1901 }
1902 /* XXX: avoid using doubles ? */
1903 cpu_fprintf(f, "Translation buffer state:\n");
1904 cpu_fprintf(f, "gen code size %td/%zd\n",
1905 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1906 tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer);
1907 cpu_fprintf(f, "TB count %d\n", tcg_ctx.tb_ctx.nb_tbs);
1908 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1909 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1910 tcg_ctx.tb_ctx.nb_tbs : 0,
1911 max_target_code_size);
1912 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1913 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1914 tcg_ctx.code_gen_buffer) /
1915 tcg_ctx.tb_ctx.nb_tbs : 0,
1916 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1917 tcg_ctx.code_gen_buffer) /
1918 target_code_size : 0);
1919 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1920 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1921 tcg_ctx.tb_ctx.nb_tbs : 0);
1922 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1923 direct_jmp_count,
1924 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1925 tcg_ctx.tb_ctx.nb_tbs : 0,
1926 direct_jmp2_count,
1927 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1928 tcg_ctx.tb_ctx.nb_tbs : 0);
1929
1930 qht_statistics_init(&tcg_ctx.tb_ctx.htable, &hst);
1931 print_qht_statistics(f, cpu_fprintf, hst);
1932 qht_statistics_destroy(&hst);
1933
1934 cpu_fprintf(f, "\nStatistics:\n");
1935 cpu_fprintf(f, "TB flush count %u\n",
1936 atomic_read(&tcg_ctx.tb_ctx.tb_flush_count));
1937 cpu_fprintf(f, "TB invalidate count %d\n",
1938 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1939 cpu_fprintf(f, "TLB flush count %zu\n", tlb_flush_count());
1940 tcg_dump_info(f, cpu_fprintf);
1941
1942 tb_unlock();
1943 }
1944
1945 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
1946 {
1947 tcg_dump_op_count(f, cpu_fprintf);
1948 }
1949
1950 #else /* CONFIG_USER_ONLY */
1951
1952 void cpu_interrupt(CPUState *cpu, int mask)
1953 {
1954 g_assert(qemu_mutex_iothread_locked());
1955 cpu->interrupt_request |= mask;
1956 cpu->icount_decr.u16.high = -1;
1957 }
1958
1959 /*
1960 * Walks guest process memory "regions" one by one
1961 * and calls callback function 'fn' for each region.
1962 */
1963 struct walk_memory_regions_data {
1964 walk_memory_regions_fn fn;
1965 void *priv;
1966 target_ulong start;
1967 int prot;
1968 };
1969
1970 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1971 target_ulong end, int new_prot)
1972 {
1973 if (data->start != -1u) {
1974 int rc = data->fn(data->priv, data->start, end, data->prot);
1975 if (rc != 0) {
1976 return rc;
1977 }
1978 }
1979
1980 data->start = (new_prot ? end : -1u);
1981 data->prot = new_prot;
1982
1983 return 0;
1984 }
1985
1986 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1987 target_ulong base, int level, void **lp)
1988 {
1989 target_ulong pa;
1990 int i, rc;
1991
1992 if (*lp == NULL) {
1993 return walk_memory_regions_end(data, base, 0);
1994 }
1995
1996 if (level == 0) {
1997 PageDesc *pd = *lp;
1998
1999 for (i = 0; i < V_L2_SIZE; ++i) {
2000 int prot = pd[i].flags;
2001
2002 pa = base | (i << TARGET_PAGE_BITS);
2003 if (prot != data->prot) {
2004 rc = walk_memory_regions_end(data, pa, prot);
2005 if (rc != 0) {
2006 return rc;
2007 }
2008 }
2009 }
2010 } else {
2011 void **pp = *lp;
2012
2013 for (i = 0; i < V_L2_SIZE; ++i) {
2014 pa = base | ((target_ulong)i <<
2015 (TARGET_PAGE_BITS + V_L2_BITS * level));
2016 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2017 if (rc != 0) {
2018 return rc;
2019 }
2020 }
2021 }
2022
2023 return 0;
2024 }
2025
2026 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2027 {
2028 struct walk_memory_regions_data data;
2029 uintptr_t i, l1_sz = v_l1_size;
2030
2031 data.fn = fn;
2032 data.priv = priv;
2033 data.start = -1u;
2034 data.prot = 0;
2035
2036 for (i = 0; i < l1_sz; i++) {
2037 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS);
2038 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i);
2039 if (rc != 0) {
2040 return rc;
2041 }
2042 }
2043
2044 return walk_memory_regions_end(&data, 0, 0);
2045 }
2046
2047 static int dump_region(void *priv, target_ulong start,
2048 target_ulong end, unsigned long prot)
2049 {
2050 FILE *f = (FILE *)priv;
2051
2052 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
2053 " "TARGET_FMT_lx" %c%c%c\n",
2054 start, end, end - start,
2055 ((prot & PAGE_READ) ? 'r' : '-'),
2056 ((prot & PAGE_WRITE) ? 'w' : '-'),
2057 ((prot & PAGE_EXEC) ? 'x' : '-'));
2058
2059 return 0;
2060 }
2061
2062 /* dump memory mappings */
2063 void page_dump(FILE *f)
2064 {
2065 const int length = sizeof(target_ulong) * 2;
2066 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
2067 length, "start", length, "end", length, "size", "prot");
2068 walk_memory_regions(f, dump_region);
2069 }
2070
2071 int page_get_flags(target_ulong address)
2072 {
2073 PageDesc *p;
2074
2075 p = page_find(address >> TARGET_PAGE_BITS);
2076 if (!p) {
2077 return 0;
2078 }
2079 return p->flags;
2080 }
2081
2082 /* Modify the flags of a page and invalidate the code if necessary.
2083 The flag PAGE_WRITE_ORG is positioned automatically depending
2084 on PAGE_WRITE. The mmap_lock should already be held. */
2085 void page_set_flags(target_ulong start, target_ulong end, int flags)
2086 {
2087 target_ulong addr, len;
2088
2089 /* This function should never be called with addresses outside the
2090 guest address space. If this assert fires, it probably indicates
2091 a missing call to h2g_valid. */
2092 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2093 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2094 #endif
2095 assert(start < end);
2096 assert_memory_lock();
2097
2098 start = start & TARGET_PAGE_MASK;
2099 end = TARGET_PAGE_ALIGN(end);
2100
2101 if (flags & PAGE_WRITE) {
2102 flags |= PAGE_WRITE_ORG;
2103 }
2104
2105 for (addr = start, len = end - start;
2106 len != 0;
2107 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2108 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2109
2110 /* If the write protection bit is set, then we invalidate
2111 the code inside. */
2112 if (!(p->flags & PAGE_WRITE) &&
2113 (flags & PAGE_WRITE) &&
2114 p->first_tb) {
2115 tb_invalidate_phys_page(addr, 0);
2116 }
2117 p->flags = flags;
2118 }
2119 }
2120
2121 int page_check_range(target_ulong start, target_ulong len, int flags)
2122 {
2123 PageDesc *p;
2124 target_ulong end;
2125 target_ulong addr;
2126
2127 /* This function should never be called with addresses outside the
2128 guest address space. If this assert fires, it probably indicates
2129 a missing call to h2g_valid. */
2130 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2131 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2132 #endif
2133
2134 if (len == 0) {
2135 return 0;
2136 }
2137 if (start + len - 1 < start) {
2138 /* We've wrapped around. */
2139 return -1;
2140 }
2141
2142 /* must do before we loose bits in the next step */
2143 end = TARGET_PAGE_ALIGN(start + len);
2144 start = start & TARGET_PAGE_MASK;
2145
2146 for (addr = start, len = end - start;
2147 len != 0;
2148 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2149 p = page_find(addr >> TARGET_PAGE_BITS);
2150 if (!p) {
2151 return -1;
2152 }
2153 if (!(p->flags & PAGE_VALID)) {
2154 return -1;
2155 }
2156
2157 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
2158 return -1;
2159 }
2160 if (flags & PAGE_WRITE) {
2161 if (!(p->flags & PAGE_WRITE_ORG)) {
2162 return -1;
2163 }
2164 /* unprotect the page if it was put read-only because it
2165 contains translated code */
2166 if (!(p->flags & PAGE_WRITE)) {
2167 if (!page_unprotect(addr, 0)) {
2168 return -1;
2169 }
2170 }
2171 }
2172 }
2173 return 0;
2174 }
2175
2176 /* called from signal handler: invalidate the code and unprotect the
2177 * page. Return 0 if the fault was not handled, 1 if it was handled,
2178 * and 2 if it was handled but the caller must cause the TB to be
2179 * immediately exited. (We can only return 2 if the 'pc' argument is
2180 * non-zero.)
2181 */
2182 int page_unprotect(target_ulong address, uintptr_t pc)
2183 {
2184 unsigned int prot;
2185 bool current_tb_invalidated;
2186 PageDesc *p;
2187 target_ulong host_start, host_end, addr;
2188
2189 /* Technically this isn't safe inside a signal handler. However we
2190 know this only ever happens in a synchronous SEGV handler, so in
2191 practice it seems to be ok. */
2192 mmap_lock();
2193
2194 p = page_find(address >> TARGET_PAGE_BITS);
2195 if (!p) {
2196 mmap_unlock();
2197 return 0;
2198 }
2199
2200 /* if the page was really writable, then we change its
2201 protection back to writable */
2202 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2203 host_start = address & qemu_host_page_mask;
2204 host_end = host_start + qemu_host_page_size;
2205
2206 prot = 0;
2207 current_tb_invalidated = false;
2208 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2209 p = page_find(addr >> TARGET_PAGE_BITS);
2210 p->flags |= PAGE_WRITE;
2211 prot |= p->flags;
2212
2213 /* and since the content will be modified, we must invalidate
2214 the corresponding translated code. */
2215 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc);
2216 #ifdef DEBUG_TB_CHECK
2217 tb_invalidate_check(addr);
2218 #endif
2219 }
2220 mprotect((void *)g2h(host_start), qemu_host_page_size,
2221 prot & PAGE_BITS);
2222
2223 mmap_unlock();
2224 /* If current TB was invalidated return to main loop */
2225 return current_tb_invalidated ? 2 : 1;
2226 }
2227 mmap_unlock();
2228 return 0;
2229 }
2230 #endif /* CONFIG_USER_ONLY */
2231
2232 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */
2233 void tcg_flush_softmmu_tlb(CPUState *cs)
2234 {
2235 #ifdef CONFIG_SOFTMMU
2236 tlb_flush(cs);
2237 #endif
2238 }