Merge remote-tracking branch 'remotes/marcel/tags/rdma-pull-request' into staging
[qemu.git] / accel / tcg / translate-all.c
1 /*
2 * Host code generation
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "qemu/units.h"
22 #include "qemu-common.h"
23
24 #define NO_CPU_IO_DEFS
25 #include "cpu.h"
26 #include "trace.h"
27 #include "disas/disas.h"
28 #include "exec/exec-all.h"
29 #include "tcg/tcg.h"
30 #if defined(CONFIG_USER_ONLY)
31 #include "qemu.h"
32 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
33 #include <sys/param.h>
34 #if __FreeBSD_version >= 700104
35 #define HAVE_KINFO_GETVMMAP
36 #define sigqueue sigqueue_freebsd /* avoid redefinition */
37 #include <sys/proc.h>
38 #include <machine/profile.h>
39 #define _KERNEL
40 #include <sys/user.h>
41 #undef _KERNEL
42 #undef sigqueue
43 #include <libutil.h>
44 #endif
45 #endif
46 #else
47 #include "exec/ram_addr.h"
48 #endif
49
50 #include "exec/cputlb.h"
51 #include "exec/tb-hash.h"
52 #include "translate-all.h"
53 #include "qemu/bitmap.h"
54 #include "qemu/error-report.h"
55 #include "qemu/qemu-print.h"
56 #include "qemu/timer.h"
57 #include "qemu/main-loop.h"
58 #include "exec/log.h"
59 #include "sysemu/cpus.h"
60 #include "sysemu/tcg.h"
61
62 /* #define DEBUG_TB_INVALIDATE */
63 /* #define DEBUG_TB_FLUSH */
64 /* make various TB consistency checks */
65 /* #define DEBUG_TB_CHECK */
66
67 #ifdef DEBUG_TB_INVALIDATE
68 #define DEBUG_TB_INVALIDATE_GATE 1
69 #else
70 #define DEBUG_TB_INVALIDATE_GATE 0
71 #endif
72
73 #ifdef DEBUG_TB_FLUSH
74 #define DEBUG_TB_FLUSH_GATE 1
75 #else
76 #define DEBUG_TB_FLUSH_GATE 0
77 #endif
78
79 #if !defined(CONFIG_USER_ONLY)
80 /* TB consistency checks only implemented for usermode emulation. */
81 #undef DEBUG_TB_CHECK
82 #endif
83
84 #ifdef DEBUG_TB_CHECK
85 #define DEBUG_TB_CHECK_GATE 1
86 #else
87 #define DEBUG_TB_CHECK_GATE 0
88 #endif
89
90 /* Access to the various translations structures need to be serialised via locks
91 * for consistency.
92 * In user-mode emulation access to the memory related structures are protected
93 * with mmap_lock.
94 * In !user-mode we use per-page locks.
95 */
96 #ifdef CONFIG_SOFTMMU
97 #define assert_memory_lock()
98 #else
99 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
100 #endif
101
102 #define SMC_BITMAP_USE_THRESHOLD 10
103
104 typedef struct PageDesc {
105 /* list of TBs intersecting this ram page */
106 uintptr_t first_tb;
107 #ifdef CONFIG_SOFTMMU
108 /* in order to optimize self modifying code, we count the number
109 of lookups we do to a given page to use a bitmap */
110 unsigned long *code_bitmap;
111 unsigned int code_write_count;
112 #else
113 unsigned long flags;
114 #endif
115 #ifndef CONFIG_USER_ONLY
116 QemuSpin lock;
117 #endif
118 } PageDesc;
119
120 /**
121 * struct page_entry - page descriptor entry
122 * @pd: pointer to the &struct PageDesc of the page this entry represents
123 * @index: page index of the page
124 * @locked: whether the page is locked
125 *
126 * This struct helps us keep track of the locked state of a page, without
127 * bloating &struct PageDesc.
128 *
129 * A page lock protects accesses to all fields of &struct PageDesc.
130 *
131 * See also: &struct page_collection.
132 */
133 struct page_entry {
134 PageDesc *pd;
135 tb_page_addr_t index;
136 bool locked;
137 };
138
139 /**
140 * struct page_collection - tracks a set of pages (i.e. &struct page_entry's)
141 * @tree: Binary search tree (BST) of the pages, with key == page index
142 * @max: Pointer to the page in @tree with the highest page index
143 *
144 * To avoid deadlock we lock pages in ascending order of page index.
145 * When operating on a set of pages, we need to keep track of them so that
146 * we can lock them in order and also unlock them later. For this we collect
147 * pages (i.e. &struct page_entry's) in a binary search @tree. Given that the
148 * @tree implementation we use does not provide an O(1) operation to obtain the
149 * highest-ranked element, we use @max to keep track of the inserted page
150 * with the highest index. This is valuable because if a page is not in
151 * the tree and its index is higher than @max's, then we can lock it
152 * without breaking the locking order rule.
153 *
154 * Note on naming: 'struct page_set' would be shorter, but we already have a few
155 * page_set_*() helpers, so page_collection is used instead to avoid confusion.
156 *
157 * See also: page_collection_lock().
158 */
159 struct page_collection {
160 GTree *tree;
161 struct page_entry *max;
162 };
163
164 /* list iterators for lists of tagged pointers in TranslationBlock */
165 #define TB_FOR_EACH_TAGGED(head, tb, n, field) \
166 for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \
167 tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \
168 tb = (TranslationBlock *)((uintptr_t)tb & ~1))
169
170 #define PAGE_FOR_EACH_TB(pagedesc, tb, n) \
171 TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
172
173 #define TB_FOR_EACH_JMP(head_tb, tb, n) \
174 TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next)
175
176 /* In system mode we want L1_MAP to be based on ram offsets,
177 while in user mode we want it to be based on virtual addresses. */
178 #if !defined(CONFIG_USER_ONLY)
179 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
180 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
181 #else
182 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
183 #endif
184 #else
185 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
186 #endif
187
188 /* Size of the L2 (and L3, etc) page tables. */
189 #define V_L2_BITS 10
190 #define V_L2_SIZE (1 << V_L2_BITS)
191
192 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
193 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS >
194 sizeof_field(TranslationBlock, trace_vcpu_dstate)
195 * BITS_PER_BYTE);
196
197 /*
198 * L1 Mapping properties
199 */
200 static int v_l1_size;
201 static int v_l1_shift;
202 static int v_l2_levels;
203
204 /* The bottom level has pointers to PageDesc, and is indexed by
205 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
206 */
207 #define V_L1_MIN_BITS 4
208 #define V_L1_MAX_BITS (V_L2_BITS + 3)
209 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
210
211 static void *l1_map[V_L1_MAX_SIZE];
212
213 /* code generation context */
214 TCGContext tcg_init_ctx;
215 __thread TCGContext *tcg_ctx;
216 TBContext tb_ctx;
217 bool parallel_cpus;
218
219 static void page_table_config_init(void)
220 {
221 uint32_t v_l1_bits;
222
223 assert(TARGET_PAGE_BITS);
224 /* The bits remaining after N lower levels of page tables. */
225 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS;
226 if (v_l1_bits < V_L1_MIN_BITS) {
227 v_l1_bits += V_L2_BITS;
228 }
229
230 v_l1_size = 1 << v_l1_bits;
231 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits;
232 v_l2_levels = v_l1_shift / V_L2_BITS - 1;
233
234 assert(v_l1_bits <= V_L1_MAX_BITS);
235 assert(v_l1_shift % V_L2_BITS == 0);
236 assert(v_l2_levels >= 0);
237 }
238
239 void cpu_gen_init(void)
240 {
241 tcg_context_init(&tcg_init_ctx);
242 }
243
244 /* Encode VAL as a signed leb128 sequence at P.
245 Return P incremented past the encoded value. */
246 static uint8_t *encode_sleb128(uint8_t *p, target_long val)
247 {
248 int more, byte;
249
250 do {
251 byte = val & 0x7f;
252 val >>= 7;
253 more = !((val == 0 && (byte & 0x40) == 0)
254 || (val == -1 && (byte & 0x40) != 0));
255 if (more) {
256 byte |= 0x80;
257 }
258 *p++ = byte;
259 } while (more);
260
261 return p;
262 }
263
264 /* Decode a signed leb128 sequence at *PP; increment *PP past the
265 decoded value. Return the decoded value. */
266 static target_long decode_sleb128(uint8_t **pp)
267 {
268 uint8_t *p = *pp;
269 target_long val = 0;
270 int byte, shift = 0;
271
272 do {
273 byte = *p++;
274 val |= (target_ulong)(byte & 0x7f) << shift;
275 shift += 7;
276 } while (byte & 0x80);
277 if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
278 val |= -(target_ulong)1 << shift;
279 }
280
281 *pp = p;
282 return val;
283 }
284
285 /* Encode the data collected about the instructions while compiling TB.
286 Place the data at BLOCK, and return the number of bytes consumed.
287
288 The logical table consists of TARGET_INSN_START_WORDS target_ulong's,
289 which come from the target's insn_start data, followed by a uintptr_t
290 which comes from the host pc of the end of the code implementing the insn.
291
292 Each line of the table is encoded as sleb128 deltas from the previous
293 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }.
294 That is, the first column is seeded with the guest pc, the last column
295 with the host pc, and the middle columns with zeros. */
296
297 static int encode_search(TranslationBlock *tb, uint8_t *block)
298 {
299 uint8_t *highwater = tcg_ctx->code_gen_highwater;
300 uint8_t *p = block;
301 int i, j, n;
302
303 for (i = 0, n = tb->icount; i < n; ++i) {
304 target_ulong prev;
305
306 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
307 if (i == 0) {
308 prev = (j == 0 ? tb->pc : 0);
309 } else {
310 prev = tcg_ctx->gen_insn_data[i - 1][j];
311 }
312 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev);
313 }
314 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]);
315 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev);
316
317 /* Test for (pending) buffer overflow. The assumption is that any
318 one row beginning below the high water mark cannot overrun
319 the buffer completely. Thus we can test for overflow after
320 encoding a row without having to check during encoding. */
321 if (unlikely(p > highwater)) {
322 return -1;
323 }
324 }
325
326 return p - block;
327 }
328
329 /* The cpu state corresponding to 'searched_pc' is restored.
330 * When reset_icount is true, current TB will be interrupted and
331 * icount should be recalculated.
332 */
333 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
334 uintptr_t searched_pc, bool reset_icount)
335 {
336 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
337 uintptr_t host_pc = (uintptr_t)tb->tc.ptr;
338 CPUArchState *env = cpu->env_ptr;
339 uint8_t *p = tb->tc.ptr + tb->tc.size;
340 int i, j, num_insns = tb->icount;
341 #ifdef CONFIG_PROFILER
342 TCGProfile *prof = &tcg_ctx->prof;
343 int64_t ti = profile_getclock();
344 #endif
345
346 searched_pc -= GETPC_ADJ;
347
348 if (searched_pc < host_pc) {
349 return -1;
350 }
351
352 /* Reconstruct the stored insn data while looking for the point at
353 which the end of the insn exceeds the searched_pc. */
354 for (i = 0; i < num_insns; ++i) {
355 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
356 data[j] += decode_sleb128(&p);
357 }
358 host_pc += decode_sleb128(&p);
359 if (host_pc > searched_pc) {
360 goto found;
361 }
362 }
363 return -1;
364
365 found:
366 if (reset_icount && (tb_cflags(tb) & CF_USE_ICOUNT)) {
367 assert(use_icount);
368 /* Reset the cycle counter to the start of the block
369 and shift if to the number of actually executed instructions */
370 cpu_neg(cpu)->icount_decr.u16.low += num_insns - i;
371 }
372 restore_state_to_opc(env, tb, data);
373
374 #ifdef CONFIG_PROFILER
375 atomic_set(&prof->restore_time,
376 prof->restore_time + profile_getclock() - ti);
377 atomic_set(&prof->restore_count, prof->restore_count + 1);
378 #endif
379 return 0;
380 }
381
382 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit)
383 {
384 TranslationBlock *tb;
385 bool r = false;
386 uintptr_t check_offset;
387
388 /* The host_pc has to be in the region of current code buffer. If
389 * it is not we will not be able to resolve it here. The two cases
390 * where host_pc will not be correct are:
391 *
392 * - fault during translation (instruction fetch)
393 * - fault from helper (not using GETPC() macro)
394 *
395 * Either way we need return early as we can't resolve it here.
396 *
397 * We are using unsigned arithmetic so if host_pc <
398 * tcg_init_ctx.code_gen_buffer check_offset will wrap to way
399 * above the code_gen_buffer_size
400 */
401 check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer;
402
403 if (check_offset < tcg_init_ctx.code_gen_buffer_size) {
404 tb = tcg_tb_lookup(host_pc);
405 if (tb) {
406 cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit);
407 if (tb_cflags(tb) & CF_NOCACHE) {
408 /* one-shot translation, invalidate it immediately */
409 tb_phys_invalidate(tb, -1);
410 tcg_tb_remove(tb);
411 }
412 r = true;
413 }
414 }
415
416 return r;
417 }
418
419 static void page_init(void)
420 {
421 page_size_init();
422 page_table_config_init();
423
424 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
425 {
426 #ifdef HAVE_KINFO_GETVMMAP
427 struct kinfo_vmentry *freep;
428 int i, cnt;
429
430 freep = kinfo_getvmmap(getpid(), &cnt);
431 if (freep) {
432 mmap_lock();
433 for (i = 0; i < cnt; i++) {
434 unsigned long startaddr, endaddr;
435
436 startaddr = freep[i].kve_start;
437 endaddr = freep[i].kve_end;
438 if (h2g_valid(startaddr)) {
439 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
440
441 if (h2g_valid(endaddr)) {
442 endaddr = h2g(endaddr);
443 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
444 } else {
445 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
446 endaddr = ~0ul;
447 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
448 #endif
449 }
450 }
451 }
452 free(freep);
453 mmap_unlock();
454 }
455 #else
456 FILE *f;
457
458 last_brk = (unsigned long)sbrk(0);
459
460 f = fopen("/compat/linux/proc/self/maps", "r");
461 if (f) {
462 mmap_lock();
463
464 do {
465 unsigned long startaddr, endaddr;
466 int n;
467
468 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
469
470 if (n == 2 && h2g_valid(startaddr)) {
471 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
472
473 if (h2g_valid(endaddr)) {
474 endaddr = h2g(endaddr);
475 } else {
476 endaddr = ~0ul;
477 }
478 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
479 }
480 } while (!feof(f));
481
482 fclose(f);
483 mmap_unlock();
484 }
485 #endif
486 }
487 #endif
488 }
489
490 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
491 {
492 PageDesc *pd;
493 void **lp;
494 int i;
495
496 /* Level 1. Always allocated. */
497 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1));
498
499 /* Level 2..N-1. */
500 for (i = v_l2_levels; i > 0; i--) {
501 void **p = atomic_rcu_read(lp);
502
503 if (p == NULL) {
504 void *existing;
505
506 if (!alloc) {
507 return NULL;
508 }
509 p = g_new0(void *, V_L2_SIZE);
510 existing = atomic_cmpxchg(lp, NULL, p);
511 if (unlikely(existing)) {
512 g_free(p);
513 p = existing;
514 }
515 }
516
517 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
518 }
519
520 pd = atomic_rcu_read(lp);
521 if (pd == NULL) {
522 void *existing;
523
524 if (!alloc) {
525 return NULL;
526 }
527 pd = g_new0(PageDesc, V_L2_SIZE);
528 #ifndef CONFIG_USER_ONLY
529 {
530 int i;
531
532 for (i = 0; i < V_L2_SIZE; i++) {
533 qemu_spin_init(&pd[i].lock);
534 }
535 }
536 #endif
537 existing = atomic_cmpxchg(lp, NULL, pd);
538 if (unlikely(existing)) {
539 g_free(pd);
540 pd = existing;
541 }
542 }
543
544 return pd + (index & (V_L2_SIZE - 1));
545 }
546
547 static inline PageDesc *page_find(tb_page_addr_t index)
548 {
549 return page_find_alloc(index, 0);
550 }
551
552 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
553 PageDesc **ret_p2, tb_page_addr_t phys2, int alloc);
554
555 /* In user-mode page locks aren't used; mmap_lock is enough */
556 #ifdef CONFIG_USER_ONLY
557
558 #define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock())
559
560 static inline void page_lock(PageDesc *pd)
561 { }
562
563 static inline void page_unlock(PageDesc *pd)
564 { }
565
566 static inline void page_lock_tb(const TranslationBlock *tb)
567 { }
568
569 static inline void page_unlock_tb(const TranslationBlock *tb)
570 { }
571
572 struct page_collection *
573 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
574 {
575 return NULL;
576 }
577
578 void page_collection_unlock(struct page_collection *set)
579 { }
580 #else /* !CONFIG_USER_ONLY */
581
582 #ifdef CONFIG_DEBUG_TCG
583
584 static __thread GHashTable *ht_pages_locked_debug;
585
586 static void ht_pages_locked_debug_init(void)
587 {
588 if (ht_pages_locked_debug) {
589 return;
590 }
591 ht_pages_locked_debug = g_hash_table_new(NULL, NULL);
592 }
593
594 static bool page_is_locked(const PageDesc *pd)
595 {
596 PageDesc *found;
597
598 ht_pages_locked_debug_init();
599 found = g_hash_table_lookup(ht_pages_locked_debug, pd);
600 return !!found;
601 }
602
603 static void page_lock__debug(PageDesc *pd)
604 {
605 ht_pages_locked_debug_init();
606 g_assert(!page_is_locked(pd));
607 g_hash_table_insert(ht_pages_locked_debug, pd, pd);
608 }
609
610 static void page_unlock__debug(const PageDesc *pd)
611 {
612 bool removed;
613
614 ht_pages_locked_debug_init();
615 g_assert(page_is_locked(pd));
616 removed = g_hash_table_remove(ht_pages_locked_debug, pd);
617 g_assert(removed);
618 }
619
620 static void
621 do_assert_page_locked(const PageDesc *pd, const char *file, int line)
622 {
623 if (unlikely(!page_is_locked(pd))) {
624 error_report("assert_page_lock: PageDesc %p not locked @ %s:%d",
625 pd, file, line);
626 abort();
627 }
628 }
629
630 #define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__)
631
632 void assert_no_pages_locked(void)
633 {
634 ht_pages_locked_debug_init();
635 g_assert(g_hash_table_size(ht_pages_locked_debug) == 0);
636 }
637
638 #else /* !CONFIG_DEBUG_TCG */
639
640 #define assert_page_locked(pd)
641
642 static inline void page_lock__debug(const PageDesc *pd)
643 {
644 }
645
646 static inline void page_unlock__debug(const PageDesc *pd)
647 {
648 }
649
650 #endif /* CONFIG_DEBUG_TCG */
651
652 static inline void page_lock(PageDesc *pd)
653 {
654 page_lock__debug(pd);
655 qemu_spin_lock(&pd->lock);
656 }
657
658 static inline void page_unlock(PageDesc *pd)
659 {
660 qemu_spin_unlock(&pd->lock);
661 page_unlock__debug(pd);
662 }
663
664 /* lock the page(s) of a TB in the correct acquisition order */
665 static inline void page_lock_tb(const TranslationBlock *tb)
666 {
667 page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], 0);
668 }
669
670 static inline void page_unlock_tb(const TranslationBlock *tb)
671 {
672 PageDesc *p1 = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
673
674 page_unlock(p1);
675 if (unlikely(tb->page_addr[1] != -1)) {
676 PageDesc *p2 = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
677
678 if (p2 != p1) {
679 page_unlock(p2);
680 }
681 }
682 }
683
684 static inline struct page_entry *
685 page_entry_new(PageDesc *pd, tb_page_addr_t index)
686 {
687 struct page_entry *pe = g_malloc(sizeof(*pe));
688
689 pe->index = index;
690 pe->pd = pd;
691 pe->locked = false;
692 return pe;
693 }
694
695 static void page_entry_destroy(gpointer p)
696 {
697 struct page_entry *pe = p;
698
699 g_assert(pe->locked);
700 page_unlock(pe->pd);
701 g_free(pe);
702 }
703
704 /* returns false on success */
705 static bool page_entry_trylock(struct page_entry *pe)
706 {
707 bool busy;
708
709 busy = qemu_spin_trylock(&pe->pd->lock);
710 if (!busy) {
711 g_assert(!pe->locked);
712 pe->locked = true;
713 page_lock__debug(pe->pd);
714 }
715 return busy;
716 }
717
718 static void do_page_entry_lock(struct page_entry *pe)
719 {
720 page_lock(pe->pd);
721 g_assert(!pe->locked);
722 pe->locked = true;
723 }
724
725 static gboolean page_entry_lock(gpointer key, gpointer value, gpointer data)
726 {
727 struct page_entry *pe = value;
728
729 do_page_entry_lock(pe);
730 return FALSE;
731 }
732
733 static gboolean page_entry_unlock(gpointer key, gpointer value, gpointer data)
734 {
735 struct page_entry *pe = value;
736
737 if (pe->locked) {
738 pe->locked = false;
739 page_unlock(pe->pd);
740 }
741 return FALSE;
742 }
743
744 /*
745 * Trylock a page, and if successful, add the page to a collection.
746 * Returns true ("busy") if the page could not be locked; false otherwise.
747 */
748 static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr)
749 {
750 tb_page_addr_t index = addr >> TARGET_PAGE_BITS;
751 struct page_entry *pe;
752 PageDesc *pd;
753
754 pe = g_tree_lookup(set->tree, &index);
755 if (pe) {
756 return false;
757 }
758
759 pd = page_find(index);
760 if (pd == NULL) {
761 return false;
762 }
763
764 pe = page_entry_new(pd, index);
765 g_tree_insert(set->tree, &pe->index, pe);
766
767 /*
768 * If this is either (1) the first insertion or (2) a page whose index
769 * is higher than any other so far, just lock the page and move on.
770 */
771 if (set->max == NULL || pe->index > set->max->index) {
772 set->max = pe;
773 do_page_entry_lock(pe);
774 return false;
775 }
776 /*
777 * Try to acquire out-of-order lock; if busy, return busy so that we acquire
778 * locks in order.
779 */
780 return page_entry_trylock(pe);
781 }
782
783 static gint tb_page_addr_cmp(gconstpointer ap, gconstpointer bp, gpointer udata)
784 {
785 tb_page_addr_t a = *(const tb_page_addr_t *)ap;
786 tb_page_addr_t b = *(const tb_page_addr_t *)bp;
787
788 if (a == b) {
789 return 0;
790 } else if (a < b) {
791 return -1;
792 }
793 return 1;
794 }
795
796 /*
797 * Lock a range of pages ([@start,@end[) as well as the pages of all
798 * intersecting TBs.
799 * Locking order: acquire locks in ascending order of page index.
800 */
801 struct page_collection *
802 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
803 {
804 struct page_collection *set = g_malloc(sizeof(*set));
805 tb_page_addr_t index;
806 PageDesc *pd;
807
808 start >>= TARGET_PAGE_BITS;
809 end >>= TARGET_PAGE_BITS;
810 g_assert(start <= end);
811
812 set->tree = g_tree_new_full(tb_page_addr_cmp, NULL, NULL,
813 page_entry_destroy);
814 set->max = NULL;
815 assert_no_pages_locked();
816
817 retry:
818 g_tree_foreach(set->tree, page_entry_lock, NULL);
819
820 for (index = start; index <= end; index++) {
821 TranslationBlock *tb;
822 int n;
823
824 pd = page_find(index);
825 if (pd == NULL) {
826 continue;
827 }
828 if (page_trylock_add(set, index << TARGET_PAGE_BITS)) {
829 g_tree_foreach(set->tree, page_entry_unlock, NULL);
830 goto retry;
831 }
832 assert_page_locked(pd);
833 PAGE_FOR_EACH_TB(pd, tb, n) {
834 if (page_trylock_add(set, tb->page_addr[0]) ||
835 (tb->page_addr[1] != -1 &&
836 page_trylock_add(set, tb->page_addr[1]))) {
837 /* drop all locks, and reacquire in order */
838 g_tree_foreach(set->tree, page_entry_unlock, NULL);
839 goto retry;
840 }
841 }
842 }
843 return set;
844 }
845
846 void page_collection_unlock(struct page_collection *set)
847 {
848 /* entries are unlocked and freed via page_entry_destroy */
849 g_tree_destroy(set->tree);
850 g_free(set);
851 }
852
853 #endif /* !CONFIG_USER_ONLY */
854
855 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
856 PageDesc **ret_p2, tb_page_addr_t phys2, int alloc)
857 {
858 PageDesc *p1, *p2;
859 tb_page_addr_t page1;
860 tb_page_addr_t page2;
861
862 assert_memory_lock();
863 g_assert(phys1 != -1);
864
865 page1 = phys1 >> TARGET_PAGE_BITS;
866 page2 = phys2 >> TARGET_PAGE_BITS;
867
868 p1 = page_find_alloc(page1, alloc);
869 if (ret_p1) {
870 *ret_p1 = p1;
871 }
872 if (likely(phys2 == -1)) {
873 page_lock(p1);
874 return;
875 } else if (page1 == page2) {
876 page_lock(p1);
877 if (ret_p2) {
878 *ret_p2 = p1;
879 }
880 return;
881 }
882 p2 = page_find_alloc(page2, alloc);
883 if (ret_p2) {
884 *ret_p2 = p2;
885 }
886 if (page1 < page2) {
887 page_lock(p1);
888 page_lock(p2);
889 } else {
890 page_lock(p2);
891 page_lock(p1);
892 }
893 }
894
895 /* Minimum size of the code gen buffer. This number is randomly chosen,
896 but not so small that we can't have a fair number of TB's live. */
897 #define MIN_CODE_GEN_BUFFER_SIZE (1 * MiB)
898
899 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
900 indicated, this is constrained by the range of direct branches on the
901 host cpu, as used by the TCG implementation of goto_tb. */
902 #if defined(__x86_64__)
903 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
904 #elif defined(__sparc__)
905 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
906 #elif defined(__powerpc64__)
907 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
908 #elif defined(__powerpc__)
909 # define MAX_CODE_GEN_BUFFER_SIZE (32 * MiB)
910 #elif defined(__aarch64__)
911 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
912 #elif defined(__s390x__)
913 /* We have a +- 4GB range on the branches; leave some slop. */
914 # define MAX_CODE_GEN_BUFFER_SIZE (3 * GiB)
915 #elif defined(__mips__)
916 /* We have a 256MB branch region, but leave room to make sure the
917 main executable is also within that region. */
918 # define MAX_CODE_GEN_BUFFER_SIZE (128 * MiB)
919 #else
920 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
921 #endif
922
923 #if TCG_TARGET_REG_BITS == 32
924 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32 * MiB)
925 #ifdef CONFIG_USER_ONLY
926 /*
927 * For user mode on smaller 32 bit systems we may run into trouble
928 * allocating big chunks of data in the right place. On these systems
929 * we utilise a static code generation buffer directly in the binary.
930 */
931 #define USE_STATIC_CODE_GEN_BUFFER
932 #endif
933 #else /* TCG_TARGET_REG_BITS == 64 */
934 #ifdef CONFIG_USER_ONLY
935 /*
936 * As user-mode emulation typically means running multiple instances
937 * of the translator don't go too nuts with our default code gen
938 * buffer lest we make things too hard for the OS.
939 */
940 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (128 * MiB)
941 #else
942 /*
943 * We expect most system emulation to run one or two guests per host.
944 * Users running large scale system emulation may want to tweak their
945 * runtime setup via the tb-size control on the command line.
946 */
947 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (1 * GiB)
948 #endif
949 #endif
950
951 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
952 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
953 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
954
955 static inline size_t size_code_gen_buffer(size_t tb_size)
956 {
957 /* Size the buffer. */
958 if (tb_size == 0) {
959 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
960 }
961 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
962 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
963 }
964 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
965 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
966 }
967 return tb_size;
968 }
969
970 #ifdef __mips__
971 /* In order to use J and JAL within the code_gen_buffer, we require
972 that the buffer not cross a 256MB boundary. */
973 static inline bool cross_256mb(void *addr, size_t size)
974 {
975 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
976 }
977
978 /* We weren't able to allocate a buffer without crossing that boundary,
979 so make do with the larger portion of the buffer that doesn't cross.
980 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
981 static inline void *split_cross_256mb(void *buf1, size_t size1)
982 {
983 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
984 size_t size2 = buf1 + size1 - buf2;
985
986 size1 = buf2 - buf1;
987 if (size1 < size2) {
988 size1 = size2;
989 buf1 = buf2;
990 }
991
992 tcg_ctx->code_gen_buffer_size = size1;
993 return buf1;
994 }
995 #endif
996
997 #ifdef USE_STATIC_CODE_GEN_BUFFER
998 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
999 __attribute__((aligned(CODE_GEN_ALIGN)));
1000
1001 static inline void *alloc_code_gen_buffer(void)
1002 {
1003 void *buf = static_code_gen_buffer;
1004 void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer);
1005 size_t size;
1006
1007 /* page-align the beginning and end of the buffer */
1008 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size);
1009 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size);
1010
1011 size = end - buf;
1012
1013 /* Honor a command-line option limiting the size of the buffer. */
1014 if (size > tcg_ctx->code_gen_buffer_size) {
1015 size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size,
1016 qemu_real_host_page_size);
1017 }
1018 tcg_ctx->code_gen_buffer_size = size;
1019
1020 #ifdef __mips__
1021 if (cross_256mb(buf, size)) {
1022 buf = split_cross_256mb(buf, size);
1023 size = tcg_ctx->code_gen_buffer_size;
1024 }
1025 #endif
1026
1027 if (qemu_mprotect_rwx(buf, size)) {
1028 abort();
1029 }
1030 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
1031
1032 return buf;
1033 }
1034 #elif defined(_WIN32)
1035 static inline void *alloc_code_gen_buffer(void)
1036 {
1037 size_t size = tcg_ctx->code_gen_buffer_size;
1038 return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT,
1039 PAGE_EXECUTE_READWRITE);
1040 }
1041 #else
1042 static inline void *alloc_code_gen_buffer(void)
1043 {
1044 int prot = PROT_WRITE | PROT_READ | PROT_EXEC;
1045 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
1046 uintptr_t start = 0;
1047 size_t size = tcg_ctx->code_gen_buffer_size;
1048 void *buf;
1049
1050 /* Constrain the position of the buffer based on the host cpu.
1051 Note that these addresses are chosen in concert with the
1052 addresses assigned in the relevant linker script file. */
1053 # if defined(__PIE__) || defined(__PIC__)
1054 /* Don't bother setting a preferred location if we're building
1055 a position-independent executable. We're more likely to get
1056 an address near the main executable if we let the kernel
1057 choose the address. */
1058 # elif defined(__x86_64__) && defined(MAP_32BIT)
1059 /* Force the memory down into low memory with the executable.
1060 Leave the choice of exact location with the kernel. */
1061 flags |= MAP_32BIT;
1062 /* Cannot expect to map more than 800MB in low memory. */
1063 if (size > 800u * 1024 * 1024) {
1064 tcg_ctx->code_gen_buffer_size = size = 800u * 1024 * 1024;
1065 }
1066 # elif defined(__sparc__)
1067 start = 0x40000000ul;
1068 # elif defined(__s390x__)
1069 start = 0x90000000ul;
1070 # elif defined(__mips__)
1071 # if _MIPS_SIM == _ABI64
1072 start = 0x128000000ul;
1073 # else
1074 start = 0x08000000ul;
1075 # endif
1076 # endif
1077
1078 buf = mmap((void *)start, size, prot, flags, -1, 0);
1079 if (buf == MAP_FAILED) {
1080 return NULL;
1081 }
1082
1083 #ifdef __mips__
1084 if (cross_256mb(buf, size)) {
1085 /* Try again, with the original still mapped, to avoid re-acquiring
1086 that 256mb crossing. This time don't specify an address. */
1087 size_t size2;
1088 void *buf2 = mmap(NULL, size, prot, flags, -1, 0);
1089 switch ((int)(buf2 != MAP_FAILED)) {
1090 case 1:
1091 if (!cross_256mb(buf2, size)) {
1092 /* Success! Use the new buffer. */
1093 munmap(buf, size);
1094 break;
1095 }
1096 /* Failure. Work with what we had. */
1097 munmap(buf2, size);
1098 /* fallthru */
1099 default:
1100 /* Split the original buffer. Free the smaller half. */
1101 buf2 = split_cross_256mb(buf, size);
1102 size2 = tcg_ctx->code_gen_buffer_size;
1103 if (buf == buf2) {
1104 munmap(buf + size2, size - size2);
1105 } else {
1106 munmap(buf, size - size2);
1107 }
1108 size = size2;
1109 break;
1110 }
1111 buf = buf2;
1112 }
1113 #endif
1114
1115 /* Request large pages for the buffer. */
1116 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
1117
1118 return buf;
1119 }
1120 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
1121
1122 static inline void code_gen_alloc(size_t tb_size)
1123 {
1124 tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size);
1125 tcg_ctx->code_gen_buffer = alloc_code_gen_buffer();
1126 if (tcg_ctx->code_gen_buffer == NULL) {
1127 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
1128 exit(1);
1129 }
1130 }
1131
1132 static bool tb_cmp(const void *ap, const void *bp)
1133 {
1134 const TranslationBlock *a = ap;
1135 const TranslationBlock *b = bp;
1136
1137 return a->pc == b->pc &&
1138 a->cs_base == b->cs_base &&
1139 a->flags == b->flags &&
1140 (tb_cflags(a) & CF_HASH_MASK) == (tb_cflags(b) & CF_HASH_MASK) &&
1141 a->trace_vcpu_dstate == b->trace_vcpu_dstate &&
1142 a->page_addr[0] == b->page_addr[0] &&
1143 a->page_addr[1] == b->page_addr[1];
1144 }
1145
1146 static void tb_htable_init(void)
1147 {
1148 unsigned int mode = QHT_MODE_AUTO_RESIZE;
1149
1150 qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode);
1151 }
1152
1153 /* Must be called before using the QEMU cpus. 'tb_size' is the size
1154 (in bytes) allocated to the translation buffer. Zero means default
1155 size. */
1156 void tcg_exec_init(unsigned long tb_size)
1157 {
1158 tcg_allowed = true;
1159 cpu_gen_init();
1160 page_init();
1161 tb_htable_init();
1162 code_gen_alloc(tb_size);
1163 #if defined(CONFIG_SOFTMMU)
1164 /* There's no guest base to take into account, so go ahead and
1165 initialize the prologue now. */
1166 tcg_prologue_init(tcg_ctx);
1167 #endif
1168 }
1169
1170 /* call with @p->lock held */
1171 static inline void invalidate_page_bitmap(PageDesc *p)
1172 {
1173 assert_page_locked(p);
1174 #ifdef CONFIG_SOFTMMU
1175 g_free(p->code_bitmap);
1176 p->code_bitmap = NULL;
1177 p->code_write_count = 0;
1178 #endif
1179 }
1180
1181 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
1182 static void page_flush_tb_1(int level, void **lp)
1183 {
1184 int i;
1185
1186 if (*lp == NULL) {
1187 return;
1188 }
1189 if (level == 0) {
1190 PageDesc *pd = *lp;
1191
1192 for (i = 0; i < V_L2_SIZE; ++i) {
1193 page_lock(&pd[i]);
1194 pd[i].first_tb = (uintptr_t)NULL;
1195 invalidate_page_bitmap(pd + i);
1196 page_unlock(&pd[i]);
1197 }
1198 } else {
1199 void **pp = *lp;
1200
1201 for (i = 0; i < V_L2_SIZE; ++i) {
1202 page_flush_tb_1(level - 1, pp + i);
1203 }
1204 }
1205 }
1206
1207 static void page_flush_tb(void)
1208 {
1209 int i, l1_sz = v_l1_size;
1210
1211 for (i = 0; i < l1_sz; i++) {
1212 page_flush_tb_1(v_l2_levels, l1_map + i);
1213 }
1214 }
1215
1216 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data)
1217 {
1218 const TranslationBlock *tb = value;
1219 size_t *size = data;
1220
1221 *size += tb->tc.size;
1222 return false;
1223 }
1224
1225 /* flush all the translation blocks */
1226 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
1227 {
1228 bool did_flush = false;
1229
1230 mmap_lock();
1231 /* If it is already been done on request of another CPU,
1232 * just retry.
1233 */
1234 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) {
1235 goto done;
1236 }
1237 did_flush = true;
1238
1239 if (DEBUG_TB_FLUSH_GATE) {
1240 size_t nb_tbs = tcg_nb_tbs();
1241 size_t host_size = 0;
1242
1243 tcg_tb_foreach(tb_host_size_iter, &host_size);
1244 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n",
1245 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0);
1246 }
1247
1248 CPU_FOREACH(cpu) {
1249 cpu_tb_jmp_cache_clear(cpu);
1250 }
1251
1252 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
1253 page_flush_tb();
1254
1255 tcg_region_reset_all();
1256 /* XXX: flush processor icache at this point if cache flush is
1257 expensive */
1258 atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1);
1259
1260 done:
1261 mmap_unlock();
1262 if (did_flush) {
1263 qemu_plugin_flush_cb();
1264 }
1265 }
1266
1267 void tb_flush(CPUState *cpu)
1268 {
1269 if (tcg_enabled()) {
1270 unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count);
1271
1272 if (cpu_in_exclusive_context(cpu)) {
1273 do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
1274 } else {
1275 async_safe_run_on_cpu(cpu, do_tb_flush,
1276 RUN_ON_CPU_HOST_INT(tb_flush_count));
1277 }
1278 }
1279 }
1280
1281 /*
1282 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only,
1283 * so in order to prevent bit rot we compile them unconditionally in user-mode,
1284 * and let the optimizer get rid of them by wrapping their user-only callers
1285 * with if (DEBUG_TB_CHECK_GATE).
1286 */
1287 #ifdef CONFIG_USER_ONLY
1288
1289 static void do_tb_invalidate_check(void *p, uint32_t hash, void *userp)
1290 {
1291 TranslationBlock *tb = p;
1292 target_ulong addr = *(target_ulong *)userp;
1293
1294 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) {
1295 printf("ERROR invalidate: address=" TARGET_FMT_lx
1296 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size);
1297 }
1298 }
1299
1300 /* verify that all the pages have correct rights for code
1301 *
1302 * Called with mmap_lock held.
1303 */
1304 static void tb_invalidate_check(target_ulong address)
1305 {
1306 address &= TARGET_PAGE_MASK;
1307 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address);
1308 }
1309
1310 static void do_tb_page_check(void *p, uint32_t hash, void *userp)
1311 {
1312 TranslationBlock *tb = p;
1313 int flags1, flags2;
1314
1315 flags1 = page_get_flags(tb->pc);
1316 flags2 = page_get_flags(tb->pc + tb->size - 1);
1317 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
1318 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
1319 (long)tb->pc, tb->size, flags1, flags2);
1320 }
1321 }
1322
1323 /* verify that all the pages have correct rights for code */
1324 static void tb_page_check(void)
1325 {
1326 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL);
1327 }
1328
1329 #endif /* CONFIG_USER_ONLY */
1330
1331 /*
1332 * user-mode: call with mmap_lock held
1333 * !user-mode: call with @pd->lock held
1334 */
1335 static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb)
1336 {
1337 TranslationBlock *tb1;
1338 uintptr_t *pprev;
1339 unsigned int n1;
1340
1341 assert_page_locked(pd);
1342 pprev = &pd->first_tb;
1343 PAGE_FOR_EACH_TB(pd, tb1, n1) {
1344 if (tb1 == tb) {
1345 *pprev = tb1->page_next[n1];
1346 return;
1347 }
1348 pprev = &tb1->page_next[n1];
1349 }
1350 g_assert_not_reached();
1351 }
1352
1353 /* remove @orig from its @n_orig-th jump list */
1354 static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig)
1355 {
1356 uintptr_t ptr, ptr_locked;
1357 TranslationBlock *dest;
1358 TranslationBlock *tb;
1359 uintptr_t *pprev;
1360 int n;
1361
1362 /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
1363 ptr = atomic_or_fetch(&orig->jmp_dest[n_orig], 1);
1364 dest = (TranslationBlock *)(ptr & ~1);
1365 if (dest == NULL) {
1366 return;
1367 }
1368
1369 qemu_spin_lock(&dest->jmp_lock);
1370 /*
1371 * While acquiring the lock, the jump might have been removed if the
1372 * destination TB was invalidated; check again.
1373 */
1374 ptr_locked = atomic_read(&orig->jmp_dest[n_orig]);
1375 if (ptr_locked != ptr) {
1376 qemu_spin_unlock(&dest->jmp_lock);
1377 /*
1378 * The only possibility is that the jump was unlinked via
1379 * tb_jump_unlink(dest). Seeing here another destination would be a bug,
1380 * because we set the LSB above.
1381 */
1382 g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID);
1383 return;
1384 }
1385 /*
1386 * We first acquired the lock, and since the destination pointer matches,
1387 * we know for sure that @orig is in the jmp list.
1388 */
1389 pprev = &dest->jmp_list_head;
1390 TB_FOR_EACH_JMP(dest, tb, n) {
1391 if (tb == orig && n == n_orig) {
1392 *pprev = tb->jmp_list_next[n];
1393 /* no need to set orig->jmp_dest[n]; setting the LSB was enough */
1394 qemu_spin_unlock(&dest->jmp_lock);
1395 return;
1396 }
1397 pprev = &tb->jmp_list_next[n];
1398 }
1399 g_assert_not_reached();
1400 }
1401
1402 /* reset the jump entry 'n' of a TB so that it is not chained to
1403 another TB */
1404 static inline void tb_reset_jump(TranslationBlock *tb, int n)
1405 {
1406 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]);
1407 tb_set_jmp_target(tb, n, addr);
1408 }
1409
1410 /* remove any jumps to the TB */
1411 static inline void tb_jmp_unlink(TranslationBlock *dest)
1412 {
1413 TranslationBlock *tb;
1414 int n;
1415
1416 qemu_spin_lock(&dest->jmp_lock);
1417
1418 TB_FOR_EACH_JMP(dest, tb, n) {
1419 tb_reset_jump(tb, n);
1420 atomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1);
1421 /* No need to clear the list entry; setting the dest ptr is enough */
1422 }
1423 dest->jmp_list_head = (uintptr_t)NULL;
1424
1425 qemu_spin_unlock(&dest->jmp_lock);
1426 }
1427
1428 /*
1429 * In user-mode, call with mmap_lock held.
1430 * In !user-mode, if @rm_from_page_list is set, call with the TB's pages'
1431 * locks held.
1432 */
1433 static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list)
1434 {
1435 CPUState *cpu;
1436 PageDesc *p;
1437 uint32_t h;
1438 tb_page_addr_t phys_pc;
1439
1440 assert_memory_lock();
1441
1442 /* make sure no further incoming jumps will be chained to this TB */
1443 qemu_spin_lock(&tb->jmp_lock);
1444 atomic_set(&tb->cflags, tb->cflags | CF_INVALID);
1445 qemu_spin_unlock(&tb->jmp_lock);
1446
1447 /* remove the TB from the hash list */
1448 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1449 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb_cflags(tb) & CF_HASH_MASK,
1450 tb->trace_vcpu_dstate);
1451 if (!(tb->cflags & CF_NOCACHE) &&
1452 !qht_remove(&tb_ctx.htable, tb, h)) {
1453 return;
1454 }
1455
1456 /* remove the TB from the page list */
1457 if (rm_from_page_list) {
1458 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
1459 tb_page_remove(p, tb);
1460 invalidate_page_bitmap(p);
1461 if (tb->page_addr[1] != -1) {
1462 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
1463 tb_page_remove(p, tb);
1464 invalidate_page_bitmap(p);
1465 }
1466 }
1467
1468 /* remove the TB from the hash list */
1469 h = tb_jmp_cache_hash_func(tb->pc);
1470 CPU_FOREACH(cpu) {
1471 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) {
1472 atomic_set(&cpu->tb_jmp_cache[h], NULL);
1473 }
1474 }
1475
1476 /* suppress this TB from the two jump lists */
1477 tb_remove_from_jmp_list(tb, 0);
1478 tb_remove_from_jmp_list(tb, 1);
1479
1480 /* suppress any remaining jumps to this TB */
1481 tb_jmp_unlink(tb);
1482
1483 atomic_set(&tcg_ctx->tb_phys_invalidate_count,
1484 tcg_ctx->tb_phys_invalidate_count + 1);
1485 }
1486
1487 static void tb_phys_invalidate__locked(TranslationBlock *tb)
1488 {
1489 do_tb_phys_invalidate(tb, true);
1490 }
1491
1492 /* invalidate one TB
1493 *
1494 * Called with mmap_lock held in user-mode.
1495 */
1496 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
1497 {
1498 if (page_addr == -1 && tb->page_addr[0] != -1) {
1499 page_lock_tb(tb);
1500 do_tb_phys_invalidate(tb, true);
1501 page_unlock_tb(tb);
1502 } else {
1503 do_tb_phys_invalidate(tb, false);
1504 }
1505 }
1506
1507 #ifdef CONFIG_SOFTMMU
1508 /* call with @p->lock held */
1509 static void build_page_bitmap(PageDesc *p)
1510 {
1511 int n, tb_start, tb_end;
1512 TranslationBlock *tb;
1513
1514 assert_page_locked(p);
1515 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
1516
1517 PAGE_FOR_EACH_TB(p, tb, n) {
1518 /* NOTE: this is subtle as a TB may span two physical pages */
1519 if (n == 0) {
1520 /* NOTE: tb_end may be after the end of the page, but
1521 it is not a problem */
1522 tb_start = tb->pc & ~TARGET_PAGE_MASK;
1523 tb_end = tb_start + tb->size;
1524 if (tb_end > TARGET_PAGE_SIZE) {
1525 tb_end = TARGET_PAGE_SIZE;
1526 }
1527 } else {
1528 tb_start = 0;
1529 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1530 }
1531 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
1532 }
1533 }
1534 #endif
1535
1536 /* add the tb in the target page and protect it if necessary
1537 *
1538 * Called with mmap_lock held for user-mode emulation.
1539 * Called with @p->lock held in !user-mode.
1540 */
1541 static inline void tb_page_add(PageDesc *p, TranslationBlock *tb,
1542 unsigned int n, tb_page_addr_t page_addr)
1543 {
1544 #ifndef CONFIG_USER_ONLY
1545 bool page_already_protected;
1546 #endif
1547
1548 assert_page_locked(p);
1549
1550 tb->page_addr[n] = page_addr;
1551 tb->page_next[n] = p->first_tb;
1552 #ifndef CONFIG_USER_ONLY
1553 page_already_protected = p->first_tb != (uintptr_t)NULL;
1554 #endif
1555 p->first_tb = (uintptr_t)tb | n;
1556 invalidate_page_bitmap(p);
1557
1558 #if defined(CONFIG_USER_ONLY)
1559 if (p->flags & PAGE_WRITE) {
1560 target_ulong addr;
1561 PageDesc *p2;
1562 int prot;
1563
1564 /* force the host page as non writable (writes will have a
1565 page fault + mprotect overhead) */
1566 page_addr &= qemu_host_page_mask;
1567 prot = 0;
1568 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1569 addr += TARGET_PAGE_SIZE) {
1570
1571 p2 = page_find(addr >> TARGET_PAGE_BITS);
1572 if (!p2) {
1573 continue;
1574 }
1575 prot |= p2->flags;
1576 p2->flags &= ~PAGE_WRITE;
1577 }
1578 mprotect(g2h(page_addr), qemu_host_page_size,
1579 (prot & PAGE_BITS) & ~PAGE_WRITE);
1580 if (DEBUG_TB_INVALIDATE_GATE) {
1581 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr);
1582 }
1583 }
1584 #else
1585 /* if some code is already present, then the pages are already
1586 protected. So we handle the case where only the first TB is
1587 allocated in a physical page */
1588 if (!page_already_protected) {
1589 tlb_protect_code(page_addr);
1590 }
1591 #endif
1592 }
1593
1594 /* add a new TB and link it to the physical page tables. phys_page2 is
1595 * (-1) to indicate that only one page contains the TB.
1596 *
1597 * Called with mmap_lock held for user-mode emulation.
1598 *
1599 * Returns a pointer @tb, or a pointer to an existing TB that matches @tb.
1600 * Note that in !user-mode, another thread might have already added a TB
1601 * for the same block of guest code that @tb corresponds to. In that case,
1602 * the caller should discard the original @tb, and use instead the returned TB.
1603 */
1604 static TranslationBlock *
1605 tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1606 tb_page_addr_t phys_page2)
1607 {
1608 PageDesc *p;
1609 PageDesc *p2 = NULL;
1610
1611 assert_memory_lock();
1612
1613 if (phys_pc == -1) {
1614 /*
1615 * If the TB is not associated with a physical RAM page then
1616 * it must be a temporary one-insn TB, and we have nothing to do
1617 * except fill in the page_addr[] fields.
1618 */
1619 assert(tb->cflags & CF_NOCACHE);
1620 tb->page_addr[0] = tb->page_addr[1] = -1;
1621 return tb;
1622 }
1623
1624 /*
1625 * Add the TB to the page list, acquiring first the pages's locks.
1626 * We keep the locks held until after inserting the TB in the hash table,
1627 * so that if the insertion fails we know for sure that the TBs are still
1628 * in the page descriptors.
1629 * Note that inserting into the hash table first isn't an option, since
1630 * we can only insert TBs that are fully initialized.
1631 */
1632 page_lock_pair(&p, phys_pc, &p2, phys_page2, 1);
1633 tb_page_add(p, tb, 0, phys_pc & TARGET_PAGE_MASK);
1634 if (p2) {
1635 tb_page_add(p2, tb, 1, phys_page2);
1636 } else {
1637 tb->page_addr[1] = -1;
1638 }
1639
1640 if (!(tb->cflags & CF_NOCACHE)) {
1641 void *existing_tb = NULL;
1642 uint32_t h;
1643
1644 /* add in the hash table */
1645 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK,
1646 tb->trace_vcpu_dstate);
1647 qht_insert(&tb_ctx.htable, tb, h, &existing_tb);
1648
1649 /* remove TB from the page(s) if we couldn't insert it */
1650 if (unlikely(existing_tb)) {
1651 tb_page_remove(p, tb);
1652 invalidate_page_bitmap(p);
1653 if (p2) {
1654 tb_page_remove(p2, tb);
1655 invalidate_page_bitmap(p2);
1656 }
1657 tb = existing_tb;
1658 }
1659 }
1660
1661 if (p2 && p2 != p) {
1662 page_unlock(p2);
1663 }
1664 page_unlock(p);
1665
1666 #ifdef CONFIG_USER_ONLY
1667 if (DEBUG_TB_CHECK_GATE) {
1668 tb_page_check();
1669 }
1670 #endif
1671 return tb;
1672 }
1673
1674 /* Called with mmap_lock held for user mode emulation. */
1675 TranslationBlock *tb_gen_code(CPUState *cpu,
1676 target_ulong pc, target_ulong cs_base,
1677 uint32_t flags, int cflags)
1678 {
1679 CPUArchState *env = cpu->env_ptr;
1680 TranslationBlock *tb, *existing_tb;
1681 tb_page_addr_t phys_pc, phys_page2;
1682 target_ulong virt_page2;
1683 tcg_insn_unit *gen_code_buf;
1684 int gen_code_size, search_size, max_insns;
1685 #ifdef CONFIG_PROFILER
1686 TCGProfile *prof = &tcg_ctx->prof;
1687 int64_t ti;
1688 #endif
1689
1690 assert_memory_lock();
1691
1692 phys_pc = get_page_addr_code(env, pc);
1693
1694 if (phys_pc == -1) {
1695 /* Generate a temporary TB with 1 insn in it */
1696 cflags &= ~CF_COUNT_MASK;
1697 cflags |= CF_NOCACHE | 1;
1698 }
1699
1700 cflags &= ~CF_CLUSTER_MASK;
1701 cflags |= cpu->cluster_index << CF_CLUSTER_SHIFT;
1702
1703 max_insns = cflags & CF_COUNT_MASK;
1704 if (max_insns == 0) {
1705 max_insns = CF_COUNT_MASK;
1706 }
1707 if (max_insns > TCG_MAX_INSNS) {
1708 max_insns = TCG_MAX_INSNS;
1709 }
1710 if (cpu->singlestep_enabled || singlestep) {
1711 max_insns = 1;
1712 }
1713
1714 buffer_overflow:
1715 tb = tcg_tb_alloc(tcg_ctx);
1716 if (unlikely(!tb)) {
1717 /* flush must be done */
1718 tb_flush(cpu);
1719 mmap_unlock();
1720 /* Make the execution loop process the flush as soon as possible. */
1721 cpu->exception_index = EXCP_INTERRUPT;
1722 cpu_loop_exit(cpu);
1723 }
1724
1725 gen_code_buf = tcg_ctx->code_gen_ptr;
1726 tb->tc.ptr = gen_code_buf;
1727 tb->pc = pc;
1728 tb->cs_base = cs_base;
1729 tb->flags = flags;
1730 tb->cflags = cflags;
1731 tb->orig_tb = NULL;
1732 tb->trace_vcpu_dstate = *cpu->trace_dstate;
1733 tcg_ctx->tb_cflags = cflags;
1734 tb_overflow:
1735
1736 #ifdef CONFIG_PROFILER
1737 /* includes aborted translations because of exceptions */
1738 atomic_set(&prof->tb_count1, prof->tb_count1 + 1);
1739 ti = profile_getclock();
1740 #endif
1741
1742 tcg_func_start(tcg_ctx);
1743
1744 tcg_ctx->cpu = env_cpu(env);
1745 gen_intermediate_code(cpu, tb, max_insns);
1746 tcg_ctx->cpu = NULL;
1747
1748 trace_translate_block(tb, tb->pc, tb->tc.ptr);
1749
1750 /* generate machine code */
1751 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
1752 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
1753 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
1754 if (TCG_TARGET_HAS_direct_jump) {
1755 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
1756 tcg_ctx->tb_jmp_target_addr = NULL;
1757 } else {
1758 tcg_ctx->tb_jmp_insn_offset = NULL;
1759 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
1760 }
1761
1762 #ifdef CONFIG_PROFILER
1763 atomic_set(&prof->tb_count, prof->tb_count + 1);
1764 atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti);
1765 ti = profile_getclock();
1766 #endif
1767
1768 gen_code_size = tcg_gen_code(tcg_ctx, tb);
1769 if (unlikely(gen_code_size < 0)) {
1770 switch (gen_code_size) {
1771 case -1:
1772 /*
1773 * Overflow of code_gen_buffer, or the current slice of it.
1774 *
1775 * TODO: We don't need to re-do gen_intermediate_code, nor
1776 * should we re-do the tcg optimization currently hidden
1777 * inside tcg_gen_code. All that should be required is to
1778 * flush the TBs, allocate a new TB, re-initialize it per
1779 * above, and re-do the actual code generation.
1780 */
1781 goto buffer_overflow;
1782
1783 case -2:
1784 /*
1785 * The code generated for the TranslationBlock is too large.
1786 * The maximum size allowed by the unwind info is 64k.
1787 * There may be stricter constraints from relocations
1788 * in the tcg backend.
1789 *
1790 * Try again with half as many insns as we attempted this time.
1791 * If a single insn overflows, there's a bug somewhere...
1792 */
1793 max_insns = tb->icount;
1794 assert(max_insns > 1);
1795 max_insns /= 2;
1796 goto tb_overflow;
1797
1798 default:
1799 g_assert_not_reached();
1800 }
1801 }
1802 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
1803 if (unlikely(search_size < 0)) {
1804 goto buffer_overflow;
1805 }
1806 tb->tc.size = gen_code_size;
1807
1808 #ifdef CONFIG_PROFILER
1809 atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti);
1810 atomic_set(&prof->code_in_len, prof->code_in_len + tb->size);
1811 atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size);
1812 atomic_set(&prof->search_out_len, prof->search_out_len + search_size);
1813 #endif
1814
1815 #ifdef DEBUG_DISAS
1816 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
1817 qemu_log_in_addr_range(tb->pc)) {
1818 FILE *logfile = qemu_log_lock();
1819 qemu_log("OUT: [size=%d]\n", gen_code_size);
1820 if (tcg_ctx->data_gen_ptr) {
1821 size_t code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr;
1822 size_t data_size = gen_code_size - code_size;
1823 size_t i;
1824
1825 log_disas(tb->tc.ptr, code_size);
1826
1827 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1828 if (sizeof(tcg_target_ulong) == 8) {
1829 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1830 (uintptr_t)tcg_ctx->data_gen_ptr + i,
1831 *(uint64_t *)(tcg_ctx->data_gen_ptr + i));
1832 } else {
1833 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
1834 (uintptr_t)tcg_ctx->data_gen_ptr + i,
1835 *(uint32_t *)(tcg_ctx->data_gen_ptr + i));
1836 }
1837 }
1838 } else {
1839 log_disas(tb->tc.ptr, gen_code_size);
1840 }
1841 qemu_log("\n");
1842 qemu_log_flush();
1843 qemu_log_unlock(logfile);
1844 }
1845 #endif
1846
1847 atomic_set(&tcg_ctx->code_gen_ptr, (void *)
1848 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
1849 CODE_GEN_ALIGN));
1850
1851 /* init jump list */
1852 qemu_spin_init(&tb->jmp_lock);
1853 tb->jmp_list_head = (uintptr_t)NULL;
1854 tb->jmp_list_next[0] = (uintptr_t)NULL;
1855 tb->jmp_list_next[1] = (uintptr_t)NULL;
1856 tb->jmp_dest[0] = (uintptr_t)NULL;
1857 tb->jmp_dest[1] = (uintptr_t)NULL;
1858
1859 /* init original jump addresses which have been set during tcg_gen_code() */
1860 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1861 tb_reset_jump(tb, 0);
1862 }
1863 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1864 tb_reset_jump(tb, 1);
1865 }
1866
1867 /* check next page if needed */
1868 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1869 phys_page2 = -1;
1870 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1871 phys_page2 = get_page_addr_code(env, virt_page2);
1872 }
1873 /*
1874 * No explicit memory barrier is required -- tb_link_page() makes the
1875 * TB visible in a consistent state.
1876 */
1877 existing_tb = tb_link_page(tb, phys_pc, phys_page2);
1878 /* if the TB already exists, discard what we just translated */
1879 if (unlikely(existing_tb != tb)) {
1880 uintptr_t orig_aligned = (uintptr_t)gen_code_buf;
1881
1882 orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize);
1883 atomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned);
1884 return existing_tb;
1885 }
1886 tcg_tb_insert(tb);
1887 return tb;
1888 }
1889
1890 /*
1891 * @p must be non-NULL.
1892 * user-mode: call with mmap_lock held.
1893 * !user-mode: call with all @pages locked.
1894 */
1895 static void
1896 tb_invalidate_phys_page_range__locked(struct page_collection *pages,
1897 PageDesc *p, tb_page_addr_t start,
1898 tb_page_addr_t end,
1899 uintptr_t retaddr)
1900 {
1901 TranslationBlock *tb;
1902 tb_page_addr_t tb_start, tb_end;
1903 int n;
1904 #ifdef TARGET_HAS_PRECISE_SMC
1905 CPUState *cpu = current_cpu;
1906 CPUArchState *env = NULL;
1907 bool current_tb_not_found = retaddr != 0;
1908 bool current_tb_modified = false;
1909 TranslationBlock *current_tb = NULL;
1910 target_ulong current_pc = 0;
1911 target_ulong current_cs_base = 0;
1912 uint32_t current_flags = 0;
1913 #endif /* TARGET_HAS_PRECISE_SMC */
1914
1915 assert_page_locked(p);
1916
1917 #if defined(TARGET_HAS_PRECISE_SMC)
1918 if (cpu != NULL) {
1919 env = cpu->env_ptr;
1920 }
1921 #endif
1922
1923 /* we remove all the TBs in the range [start, end[ */
1924 /* XXX: see if in some cases it could be faster to invalidate all
1925 the code */
1926 PAGE_FOR_EACH_TB(p, tb, n) {
1927 assert_page_locked(p);
1928 /* NOTE: this is subtle as a TB may span two physical pages */
1929 if (n == 0) {
1930 /* NOTE: tb_end may be after the end of the page, but
1931 it is not a problem */
1932 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1933 tb_end = tb_start + tb->size;
1934 } else {
1935 tb_start = tb->page_addr[1];
1936 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1937 }
1938 if (!(tb_end <= start || tb_start >= end)) {
1939 #ifdef TARGET_HAS_PRECISE_SMC
1940 if (current_tb_not_found) {
1941 current_tb_not_found = false;
1942 /* now we have a real cpu fault */
1943 current_tb = tcg_tb_lookup(retaddr);
1944 }
1945 if (current_tb == tb &&
1946 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
1947 /*
1948 * If we are modifying the current TB, we must stop
1949 * its execution. We could be more precise by checking
1950 * that the modification is after the current PC, but it
1951 * would require a specialized function to partially
1952 * restore the CPU state.
1953 */
1954 current_tb_modified = true;
1955 cpu_restore_state_from_tb(cpu, current_tb, retaddr, true);
1956 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1957 &current_flags);
1958 }
1959 #endif /* TARGET_HAS_PRECISE_SMC */
1960 tb_phys_invalidate__locked(tb);
1961 }
1962 }
1963 #if !defined(CONFIG_USER_ONLY)
1964 /* if no code remaining, no need to continue to use slow writes */
1965 if (!p->first_tb) {
1966 invalidate_page_bitmap(p);
1967 tlb_unprotect_code(start);
1968 }
1969 #endif
1970 #ifdef TARGET_HAS_PRECISE_SMC
1971 if (current_tb_modified) {
1972 page_collection_unlock(pages);
1973 /* Force execution of one insn next time. */
1974 cpu->cflags_next_tb = 1 | curr_cflags();
1975 mmap_unlock();
1976 cpu_loop_exit_noexc(cpu);
1977 }
1978 #endif
1979 }
1980
1981 /*
1982 * Invalidate all TBs which intersect with the target physical address range
1983 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1984 * 'is_cpu_write_access' should be true if called from a real cpu write
1985 * access: the virtual CPU will exit the current TB if code is modified inside
1986 * this TB.
1987 *
1988 * Called with mmap_lock held for user-mode emulation
1989 */
1990 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end)
1991 {
1992 struct page_collection *pages;
1993 PageDesc *p;
1994
1995 assert_memory_lock();
1996
1997 p = page_find(start >> TARGET_PAGE_BITS);
1998 if (p == NULL) {
1999 return;
2000 }
2001 pages = page_collection_lock(start, end);
2002 tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
2003 page_collection_unlock(pages);
2004 }
2005
2006 /*
2007 * Invalidate all TBs which intersect with the target physical address range
2008 * [start;end[. NOTE: start and end may refer to *different* physical pages.
2009 * 'is_cpu_write_access' should be true if called from a real cpu write
2010 * access: the virtual CPU will exit the current TB if code is modified inside
2011 * this TB.
2012 *
2013 * Called with mmap_lock held for user-mode emulation.
2014 */
2015 #ifdef CONFIG_SOFTMMU
2016 void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end)
2017 #else
2018 void tb_invalidate_phys_range(target_ulong start, target_ulong end)
2019 #endif
2020 {
2021 struct page_collection *pages;
2022 tb_page_addr_t next;
2023
2024 assert_memory_lock();
2025
2026 pages = page_collection_lock(start, end);
2027 for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
2028 start < end;
2029 start = next, next += TARGET_PAGE_SIZE) {
2030 PageDesc *pd = page_find(start >> TARGET_PAGE_BITS);
2031 tb_page_addr_t bound = MIN(next, end);
2032
2033 if (pd == NULL) {
2034 continue;
2035 }
2036 tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
2037 }
2038 page_collection_unlock(pages);
2039 }
2040
2041 #ifdef CONFIG_SOFTMMU
2042 /* len must be <= 8 and start must be a multiple of len.
2043 * Called via softmmu_template.h when code areas are written to with
2044 * iothread mutex not held.
2045 *
2046 * Call with all @pages in the range [@start, @start + len[ locked.
2047 */
2048 void tb_invalidate_phys_page_fast(struct page_collection *pages,
2049 tb_page_addr_t start, int len,
2050 uintptr_t retaddr)
2051 {
2052 PageDesc *p;
2053
2054 assert_memory_lock();
2055
2056 p = page_find(start >> TARGET_PAGE_BITS);
2057 if (!p) {
2058 return;
2059 }
2060
2061 assert_page_locked(p);
2062 if (!p->code_bitmap &&
2063 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
2064 build_page_bitmap(p);
2065 }
2066 if (p->code_bitmap) {
2067 unsigned int nr;
2068 unsigned long b;
2069
2070 nr = start & ~TARGET_PAGE_MASK;
2071 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
2072 if (b & ((1 << len) - 1)) {
2073 goto do_invalidate;
2074 }
2075 } else {
2076 do_invalidate:
2077 tb_invalidate_phys_page_range__locked(pages, p, start, start + len,
2078 retaddr);
2079 }
2080 }
2081 #else
2082 /* Called with mmap_lock held. If pc is not 0 then it indicates the
2083 * host PC of the faulting store instruction that caused this invalidate.
2084 * Returns true if the caller needs to abort execution of the current
2085 * TB (because it was modified by this store and the guest CPU has
2086 * precise-SMC semantics).
2087 */
2088 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
2089 {
2090 TranslationBlock *tb;
2091 PageDesc *p;
2092 int n;
2093 #ifdef TARGET_HAS_PRECISE_SMC
2094 TranslationBlock *current_tb = NULL;
2095 CPUState *cpu = current_cpu;
2096 CPUArchState *env = NULL;
2097 int current_tb_modified = 0;
2098 target_ulong current_pc = 0;
2099 target_ulong current_cs_base = 0;
2100 uint32_t current_flags = 0;
2101 #endif
2102
2103 assert_memory_lock();
2104
2105 addr &= TARGET_PAGE_MASK;
2106 p = page_find(addr >> TARGET_PAGE_BITS);
2107 if (!p) {
2108 return false;
2109 }
2110
2111 #ifdef TARGET_HAS_PRECISE_SMC
2112 if (p->first_tb && pc != 0) {
2113 current_tb = tcg_tb_lookup(pc);
2114 }
2115 if (cpu != NULL) {
2116 env = cpu->env_ptr;
2117 }
2118 #endif
2119 assert_page_locked(p);
2120 PAGE_FOR_EACH_TB(p, tb, n) {
2121 #ifdef TARGET_HAS_PRECISE_SMC
2122 if (current_tb == tb &&
2123 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
2124 /* If we are modifying the current TB, we must stop
2125 its execution. We could be more precise by checking
2126 that the modification is after the current PC, but it
2127 would require a specialized function to partially
2128 restore the CPU state */
2129
2130 current_tb_modified = 1;
2131 cpu_restore_state_from_tb(cpu, current_tb, pc, true);
2132 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
2133 &current_flags);
2134 }
2135 #endif /* TARGET_HAS_PRECISE_SMC */
2136 tb_phys_invalidate(tb, addr);
2137 }
2138 p->first_tb = (uintptr_t)NULL;
2139 #ifdef TARGET_HAS_PRECISE_SMC
2140 if (current_tb_modified) {
2141 /* Force execution of one insn next time. */
2142 cpu->cflags_next_tb = 1 | curr_cflags();
2143 return true;
2144 }
2145 #endif
2146
2147 return false;
2148 }
2149 #endif
2150
2151 /* user-mode: call with mmap_lock held */
2152 void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)
2153 {
2154 TranslationBlock *tb;
2155
2156 assert_memory_lock();
2157
2158 tb = tcg_tb_lookup(retaddr);
2159 if (tb) {
2160 /* We can use retranslation to find the PC. */
2161 cpu_restore_state_from_tb(cpu, tb, retaddr, true);
2162 tb_phys_invalidate(tb, -1);
2163 } else {
2164 /* The exception probably happened in a helper. The CPU state should
2165 have been saved before calling it. Fetch the PC from there. */
2166 CPUArchState *env = cpu->env_ptr;
2167 target_ulong pc, cs_base;
2168 tb_page_addr_t addr;
2169 uint32_t flags;
2170
2171 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
2172 addr = get_page_addr_code(env, pc);
2173 if (addr != -1) {
2174 tb_invalidate_phys_range(addr, addr + 1);
2175 }
2176 }
2177 }
2178
2179 #ifndef CONFIG_USER_ONLY
2180 /* in deterministic execution mode, instructions doing device I/Os
2181 * must be at the end of the TB.
2182 *
2183 * Called by softmmu_template.h, with iothread mutex not held.
2184 */
2185 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
2186 {
2187 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
2188 CPUArchState *env = cpu->env_ptr;
2189 #endif
2190 TranslationBlock *tb;
2191 uint32_t n;
2192
2193 tb = tcg_tb_lookup(retaddr);
2194 if (!tb) {
2195 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
2196 (void *)retaddr);
2197 }
2198 cpu_restore_state_from_tb(cpu, tb, retaddr, true);
2199
2200 /* On MIPS and SH, delay slot instructions can only be restarted if
2201 they were already the first instruction in the TB. If this is not
2202 the first instruction in a TB then re-execute the preceding
2203 branch. */
2204 n = 1;
2205 #if defined(TARGET_MIPS)
2206 if ((env->hflags & MIPS_HFLAG_BMASK) != 0
2207 && env->active_tc.PC != tb->pc) {
2208 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
2209 cpu_neg(cpu)->icount_decr.u16.low++;
2210 env->hflags &= ~MIPS_HFLAG_BMASK;
2211 n = 2;
2212 }
2213 #elif defined(TARGET_SH4)
2214 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
2215 && env->pc != tb->pc) {
2216 env->pc -= 2;
2217 cpu_neg(cpu)->icount_decr.u16.low++;
2218 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
2219 n = 2;
2220 }
2221 #endif
2222
2223 /* Generate a new TB executing the I/O insn. */
2224 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n;
2225
2226 if (tb_cflags(tb) & CF_NOCACHE) {
2227 if (tb->orig_tb) {
2228 /* Invalidate original TB if this TB was generated in
2229 * cpu_exec_nocache() */
2230 tb_phys_invalidate(tb->orig_tb, -1);
2231 }
2232 tcg_tb_remove(tb);
2233 }
2234
2235 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
2236 * the first in the TB) then we end up generating a whole new TB and
2237 * repeating the fault, which is horribly inefficient.
2238 * Better would be to execute just this insn uncached, or generate a
2239 * second new TB.
2240 */
2241 cpu_loop_exit_noexc(cpu);
2242 }
2243
2244 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr)
2245 {
2246 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr);
2247
2248 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) {
2249 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL);
2250 }
2251 }
2252
2253 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
2254 {
2255 /* Discard jump cache entries for any tb which might potentially
2256 overlap the flushed page. */
2257 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE);
2258 tb_jmp_cache_clear_page(cpu, addr);
2259 }
2260
2261 static void print_qht_statistics(struct qht_stats hst)
2262 {
2263 uint32_t hgram_opts;
2264 size_t hgram_bins;
2265 char *hgram;
2266
2267 if (!hst.head_buckets) {
2268 return;
2269 }
2270 qemu_printf("TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
2271 hst.used_head_buckets, hst.head_buckets,
2272 (double)hst.used_head_buckets / hst.head_buckets * 100);
2273
2274 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
2275 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT;
2276 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) {
2277 hgram_opts |= QDIST_PR_NODECIMAL;
2278 }
2279 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts);
2280 qemu_printf("TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
2281 qdist_avg(&hst.occupancy) * 100, hgram);
2282 g_free(hgram);
2283
2284 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
2285 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain);
2286 if (hgram_bins > 10) {
2287 hgram_bins = 10;
2288 } else {
2289 hgram_bins = 0;
2290 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE;
2291 }
2292 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts);
2293 qemu_printf("TB hash avg chain %0.3f buckets. Histogram: %s\n",
2294 qdist_avg(&hst.chain), hgram);
2295 g_free(hgram);
2296 }
2297
2298 struct tb_tree_stats {
2299 size_t nb_tbs;
2300 size_t host_size;
2301 size_t target_size;
2302 size_t max_target_size;
2303 size_t direct_jmp_count;
2304 size_t direct_jmp2_count;
2305 size_t cross_page;
2306 };
2307
2308 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data)
2309 {
2310 const TranslationBlock *tb = value;
2311 struct tb_tree_stats *tst = data;
2312
2313 tst->nb_tbs++;
2314 tst->host_size += tb->tc.size;
2315 tst->target_size += tb->size;
2316 if (tb->size > tst->max_target_size) {
2317 tst->max_target_size = tb->size;
2318 }
2319 if (tb->page_addr[1] != -1) {
2320 tst->cross_page++;
2321 }
2322 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
2323 tst->direct_jmp_count++;
2324 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
2325 tst->direct_jmp2_count++;
2326 }
2327 }
2328 return false;
2329 }
2330
2331 void dump_exec_info(void)
2332 {
2333 struct tb_tree_stats tst = {};
2334 struct qht_stats hst;
2335 size_t nb_tbs, flush_full, flush_part, flush_elide;
2336
2337 tcg_tb_foreach(tb_tree_stats_iter, &tst);
2338 nb_tbs = tst.nb_tbs;
2339 /* XXX: avoid using doubles ? */
2340 qemu_printf("Translation buffer state:\n");
2341 /*
2342 * Report total code size including the padding and TB structs;
2343 * otherwise users might think "-tb-size" is not honoured.
2344 * For avg host size we use the precise numbers from tb_tree_stats though.
2345 */
2346 qemu_printf("gen code size %zu/%zu\n",
2347 tcg_code_size(), tcg_code_capacity());
2348 qemu_printf("TB count %zu\n", nb_tbs);
2349 qemu_printf("TB avg target size %zu max=%zu bytes\n",
2350 nb_tbs ? tst.target_size / nb_tbs : 0,
2351 tst.max_target_size);
2352 qemu_printf("TB avg host size %zu bytes (expansion ratio: %0.1f)\n",
2353 nb_tbs ? tst.host_size / nb_tbs : 0,
2354 tst.target_size ? (double)tst.host_size / tst.target_size : 0);
2355 qemu_printf("cross page TB count %zu (%zu%%)\n", tst.cross_page,
2356 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0);
2357 qemu_printf("direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n",
2358 tst.direct_jmp_count,
2359 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0,
2360 tst.direct_jmp2_count,
2361 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0);
2362
2363 qht_statistics_init(&tb_ctx.htable, &hst);
2364 print_qht_statistics(hst);
2365 qht_statistics_destroy(&hst);
2366
2367 qemu_printf("\nStatistics:\n");
2368 qemu_printf("TB flush count %u\n",
2369 atomic_read(&tb_ctx.tb_flush_count));
2370 qemu_printf("TB invalidate count %zu\n",
2371 tcg_tb_phys_invalidate_count());
2372
2373 tlb_flush_counts(&flush_full, &flush_part, &flush_elide);
2374 qemu_printf("TLB full flushes %zu\n", flush_full);
2375 qemu_printf("TLB partial flushes %zu\n", flush_part);
2376 qemu_printf("TLB elided flushes %zu\n", flush_elide);
2377 tcg_dump_info();
2378 }
2379
2380 void dump_opcount_info(void)
2381 {
2382 tcg_dump_op_count();
2383 }
2384
2385 #else /* CONFIG_USER_ONLY */
2386
2387 void cpu_interrupt(CPUState *cpu, int mask)
2388 {
2389 g_assert(qemu_mutex_iothread_locked());
2390 cpu->interrupt_request |= mask;
2391 atomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1);
2392 }
2393
2394 /*
2395 * Walks guest process memory "regions" one by one
2396 * and calls callback function 'fn' for each region.
2397 */
2398 struct walk_memory_regions_data {
2399 walk_memory_regions_fn fn;
2400 void *priv;
2401 target_ulong start;
2402 int prot;
2403 };
2404
2405 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2406 target_ulong end, int new_prot)
2407 {
2408 if (data->start != -1u) {
2409 int rc = data->fn(data->priv, data->start, end, data->prot);
2410 if (rc != 0) {
2411 return rc;
2412 }
2413 }
2414
2415 data->start = (new_prot ? end : -1u);
2416 data->prot = new_prot;
2417
2418 return 0;
2419 }
2420
2421 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2422 target_ulong base, int level, void **lp)
2423 {
2424 target_ulong pa;
2425 int i, rc;
2426
2427 if (*lp == NULL) {
2428 return walk_memory_regions_end(data, base, 0);
2429 }
2430
2431 if (level == 0) {
2432 PageDesc *pd = *lp;
2433
2434 for (i = 0; i < V_L2_SIZE; ++i) {
2435 int prot = pd[i].flags;
2436
2437 pa = base | (i << TARGET_PAGE_BITS);
2438 if (prot != data->prot) {
2439 rc = walk_memory_regions_end(data, pa, prot);
2440 if (rc != 0) {
2441 return rc;
2442 }
2443 }
2444 }
2445 } else {
2446 void **pp = *lp;
2447
2448 for (i = 0; i < V_L2_SIZE; ++i) {
2449 pa = base | ((target_ulong)i <<
2450 (TARGET_PAGE_BITS + V_L2_BITS * level));
2451 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2452 if (rc != 0) {
2453 return rc;
2454 }
2455 }
2456 }
2457
2458 return 0;
2459 }
2460
2461 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2462 {
2463 struct walk_memory_regions_data data;
2464 uintptr_t i, l1_sz = v_l1_size;
2465
2466 data.fn = fn;
2467 data.priv = priv;
2468 data.start = -1u;
2469 data.prot = 0;
2470
2471 for (i = 0; i < l1_sz; i++) {
2472 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS);
2473 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i);
2474 if (rc != 0) {
2475 return rc;
2476 }
2477 }
2478
2479 return walk_memory_regions_end(&data, 0, 0);
2480 }
2481
2482 static int dump_region(void *priv, target_ulong start,
2483 target_ulong end, unsigned long prot)
2484 {
2485 FILE *f = (FILE *)priv;
2486
2487 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
2488 " "TARGET_FMT_lx" %c%c%c\n",
2489 start, end, end - start,
2490 ((prot & PAGE_READ) ? 'r' : '-'),
2491 ((prot & PAGE_WRITE) ? 'w' : '-'),
2492 ((prot & PAGE_EXEC) ? 'x' : '-'));
2493
2494 return 0;
2495 }
2496
2497 /* dump memory mappings */
2498 void page_dump(FILE *f)
2499 {
2500 const int length = sizeof(target_ulong) * 2;
2501 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
2502 length, "start", length, "end", length, "size", "prot");
2503 walk_memory_regions(f, dump_region);
2504 }
2505
2506 int page_get_flags(target_ulong address)
2507 {
2508 PageDesc *p;
2509
2510 p = page_find(address >> TARGET_PAGE_BITS);
2511 if (!p) {
2512 return 0;
2513 }
2514 return p->flags;
2515 }
2516
2517 /* Modify the flags of a page and invalidate the code if necessary.
2518 The flag PAGE_WRITE_ORG is positioned automatically depending
2519 on PAGE_WRITE. The mmap_lock should already be held. */
2520 void page_set_flags(target_ulong start, target_ulong end, int flags)
2521 {
2522 target_ulong addr, len;
2523
2524 /* This function should never be called with addresses outside the
2525 guest address space. If this assert fires, it probably indicates
2526 a missing call to h2g_valid. */
2527 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2528 assert(end <= ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2529 #endif
2530 assert(start < end);
2531 assert_memory_lock();
2532
2533 start = start & TARGET_PAGE_MASK;
2534 end = TARGET_PAGE_ALIGN(end);
2535
2536 if (flags & PAGE_WRITE) {
2537 flags |= PAGE_WRITE_ORG;
2538 }
2539
2540 for (addr = start, len = end - start;
2541 len != 0;
2542 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2543 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2544
2545 /* If the write protection bit is set, then we invalidate
2546 the code inside. */
2547 if (!(p->flags & PAGE_WRITE) &&
2548 (flags & PAGE_WRITE) &&
2549 p->first_tb) {
2550 tb_invalidate_phys_page(addr, 0);
2551 }
2552 p->flags = flags;
2553 }
2554 }
2555
2556 int page_check_range(target_ulong start, target_ulong len, int flags)
2557 {
2558 PageDesc *p;
2559 target_ulong end;
2560 target_ulong addr;
2561
2562 /* This function should never be called with addresses outside the
2563 guest address space. If this assert fires, it probably indicates
2564 a missing call to h2g_valid. */
2565 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2566 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2567 #endif
2568
2569 if (len == 0) {
2570 return 0;
2571 }
2572 if (start + len - 1 < start) {
2573 /* We've wrapped around. */
2574 return -1;
2575 }
2576
2577 /* must do before we loose bits in the next step */
2578 end = TARGET_PAGE_ALIGN(start + len);
2579 start = start & TARGET_PAGE_MASK;
2580
2581 for (addr = start, len = end - start;
2582 len != 0;
2583 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2584 p = page_find(addr >> TARGET_PAGE_BITS);
2585 if (!p) {
2586 return -1;
2587 }
2588 if (!(p->flags & PAGE_VALID)) {
2589 return -1;
2590 }
2591
2592 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
2593 return -1;
2594 }
2595 if (flags & PAGE_WRITE) {
2596 if (!(p->flags & PAGE_WRITE_ORG)) {
2597 return -1;
2598 }
2599 /* unprotect the page if it was put read-only because it
2600 contains translated code */
2601 if (!(p->flags & PAGE_WRITE)) {
2602 if (!page_unprotect(addr, 0)) {
2603 return -1;
2604 }
2605 }
2606 }
2607 }
2608 return 0;
2609 }
2610
2611 /* called from signal handler: invalidate the code and unprotect the
2612 * page. Return 0 if the fault was not handled, 1 if it was handled,
2613 * and 2 if it was handled but the caller must cause the TB to be
2614 * immediately exited. (We can only return 2 if the 'pc' argument is
2615 * non-zero.)
2616 */
2617 int page_unprotect(target_ulong address, uintptr_t pc)
2618 {
2619 unsigned int prot;
2620 bool current_tb_invalidated;
2621 PageDesc *p;
2622 target_ulong host_start, host_end, addr;
2623
2624 /* Technically this isn't safe inside a signal handler. However we
2625 know this only ever happens in a synchronous SEGV handler, so in
2626 practice it seems to be ok. */
2627 mmap_lock();
2628
2629 p = page_find(address >> TARGET_PAGE_BITS);
2630 if (!p) {
2631 mmap_unlock();
2632 return 0;
2633 }
2634
2635 /* if the page was really writable, then we change its
2636 protection back to writable */
2637 if (p->flags & PAGE_WRITE_ORG) {
2638 current_tb_invalidated = false;
2639 if (p->flags & PAGE_WRITE) {
2640 /* If the page is actually marked WRITE then assume this is because
2641 * this thread raced with another one which got here first and
2642 * set the page to PAGE_WRITE and did the TB invalidate for us.
2643 */
2644 #ifdef TARGET_HAS_PRECISE_SMC
2645 TranslationBlock *current_tb = tcg_tb_lookup(pc);
2646 if (current_tb) {
2647 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID;
2648 }
2649 #endif
2650 } else {
2651 host_start = address & qemu_host_page_mask;
2652 host_end = host_start + qemu_host_page_size;
2653
2654 prot = 0;
2655 for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) {
2656 p = page_find(addr >> TARGET_PAGE_BITS);
2657 p->flags |= PAGE_WRITE;
2658 prot |= p->flags;
2659
2660 /* and since the content will be modified, we must invalidate
2661 the corresponding translated code. */
2662 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc);
2663 #ifdef CONFIG_USER_ONLY
2664 if (DEBUG_TB_CHECK_GATE) {
2665 tb_invalidate_check(addr);
2666 }
2667 #endif
2668 }
2669 mprotect((void *)g2h(host_start), qemu_host_page_size,
2670 prot & PAGE_BITS);
2671 }
2672 mmap_unlock();
2673 /* If current TB was invalidated return to main loop */
2674 return current_tb_invalidated ? 2 : 1;
2675 }
2676 mmap_unlock();
2677 return 0;
2678 }
2679 #endif /* CONFIG_USER_ONLY */
2680
2681 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */
2682 void tcg_flush_softmmu_tlb(CPUState *cs)
2683 {
2684 #ifdef CONFIG_SOFTMMU
2685 tlb_flush(cs);
2686 #endif
2687 }