stubs: Move qemu_fd_register stub to util/main-loop.c
[qemu.git] / tcg / tcg.c
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 /* define it to use liveness analysis (better code) */
26 #define USE_TCG_OPTIMIZATIONS
27
28 #include "qemu/osdep.h"
29
30 /* Define to jump the ELF file used to communicate with GDB. */
31 #undef DEBUG_JIT
32
33 #include "qemu/error-report.h"
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/qemu-print.h"
37 #include "qemu/timer.h"
38
39 /* Note: the long term plan is to reduce the dependencies on the QEMU
40 CPU definitions. Currently they are used for qemu_ld/st
41 instructions */
42 #define NO_CPU_IO_DEFS
43 #include "cpu.h"
44
45 #include "exec/exec-all.h"
46
47 #if !defined(CONFIG_USER_ONLY)
48 #include "hw/boards.h"
49 #endif
50
51 #include "tcg/tcg-op.h"
52
53 #if UINTPTR_MAX == UINT32_MAX
54 # define ELF_CLASS ELFCLASS32
55 #else
56 # define ELF_CLASS ELFCLASS64
57 #endif
58 #ifdef HOST_WORDS_BIGENDIAN
59 # define ELF_DATA ELFDATA2MSB
60 #else
61 # define ELF_DATA ELFDATA2LSB
62 #endif
63
64 #include "elf.h"
65 #include "exec/log.h"
66 #include "sysemu/sysemu.h"
67
68 /* Forward declarations for functions declared in tcg-target.c.inc and
69 used here. */
70 static void tcg_target_init(TCGContext *s);
71 static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode);
72 static void tcg_target_qemu_prologue(TCGContext *s);
73 static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
74 intptr_t value, intptr_t addend);
75
76 /* The CIE and FDE header definitions will be common to all hosts. */
77 typedef struct {
78 uint32_t len __attribute__((aligned((sizeof(void *)))));
79 uint32_t id;
80 uint8_t version;
81 char augmentation[1];
82 uint8_t code_align;
83 uint8_t data_align;
84 uint8_t return_column;
85 } DebugFrameCIE;
86
87 typedef struct QEMU_PACKED {
88 uint32_t len __attribute__((aligned((sizeof(void *)))));
89 uint32_t cie_offset;
90 uintptr_t func_start;
91 uintptr_t func_len;
92 } DebugFrameFDEHeader;
93
94 typedef struct QEMU_PACKED {
95 DebugFrameCIE cie;
96 DebugFrameFDEHeader fde;
97 } DebugFrameHeader;
98
99 static void tcg_register_jit_int(void *buf, size_t size,
100 const void *debug_frame,
101 size_t debug_frame_size)
102 __attribute__((unused));
103
104 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
105 static const char *target_parse_constraint(TCGArgConstraint *ct,
106 const char *ct_str, TCGType type);
107 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
108 intptr_t arg2);
109 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
110 static void tcg_out_movi(TCGContext *s, TCGType type,
111 TCGReg ret, tcg_target_long arg);
112 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
113 const int *const_args);
114 #if TCG_TARGET_MAYBE_vec
115 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
116 TCGReg dst, TCGReg src);
117 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
118 TCGReg dst, TCGReg base, intptr_t offset);
119 static void tcg_out_dupi_vec(TCGContext *s, TCGType type,
120 TCGReg dst, tcg_target_long arg);
121 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl,
122 unsigned vece, const TCGArg *args,
123 const int *const_args);
124 #else
125 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
126 TCGReg dst, TCGReg src)
127 {
128 g_assert_not_reached();
129 }
130 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
131 TCGReg dst, TCGReg base, intptr_t offset)
132 {
133 g_assert_not_reached();
134 }
135 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type,
136 TCGReg dst, tcg_target_long arg)
137 {
138 g_assert_not_reached();
139 }
140 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl,
141 unsigned vece, const TCGArg *args,
142 const int *const_args)
143 {
144 g_assert_not_reached();
145 }
146 #endif
147 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
148 intptr_t arg2);
149 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
150 TCGReg base, intptr_t ofs);
151 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target);
152 static int tcg_target_const_match(tcg_target_long val, TCGType type,
153 const TCGArgConstraint *arg_ct);
154 #ifdef TCG_TARGET_NEED_LDST_LABELS
155 static int tcg_out_ldst_finalize(TCGContext *s);
156 #endif
157
158 #define TCG_HIGHWATER 1024
159
160 static TCGContext **tcg_ctxs;
161 static unsigned int n_tcg_ctxs;
162 TCGv_env cpu_env = 0;
163
164 struct tcg_region_tree {
165 QemuMutex lock;
166 GTree *tree;
167 /* padding to avoid false sharing is computed at run-time */
168 };
169
170 /*
171 * We divide code_gen_buffer into equally-sized "regions" that TCG threads
172 * dynamically allocate from as demand dictates. Given appropriate region
173 * sizing, this minimizes flushes even when some TCG threads generate a lot
174 * more code than others.
175 */
176 struct tcg_region_state {
177 QemuMutex lock;
178
179 /* fields set at init time */
180 void *start;
181 void *start_aligned;
182 void *end;
183 size_t n;
184 size_t size; /* size of one region */
185 size_t stride; /* .size + guard size */
186
187 /* fields protected by the lock */
188 size_t current; /* current region index */
189 size_t agg_size_full; /* aggregate size of full regions */
190 };
191
192 static struct tcg_region_state region;
193 /*
194 * This is an array of struct tcg_region_tree's, with padding.
195 * We use void * to simplify the computation of region_trees[i]; each
196 * struct is found every tree_size bytes.
197 */
198 static void *region_trees;
199 static size_t tree_size;
200 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
201 static TCGRegSet tcg_target_call_clobber_regs;
202
203 #if TCG_TARGET_INSN_UNIT_SIZE == 1
204 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
205 {
206 *s->code_ptr++ = v;
207 }
208
209 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
210 uint8_t v)
211 {
212 *p = v;
213 }
214 #endif
215
216 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
217 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
218 {
219 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
220 *s->code_ptr++ = v;
221 } else {
222 tcg_insn_unit *p = s->code_ptr;
223 memcpy(p, &v, sizeof(v));
224 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
225 }
226 }
227
228 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
229 uint16_t v)
230 {
231 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
232 *p = v;
233 } else {
234 memcpy(p, &v, sizeof(v));
235 }
236 }
237 #endif
238
239 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
240 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
241 {
242 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
243 *s->code_ptr++ = v;
244 } else {
245 tcg_insn_unit *p = s->code_ptr;
246 memcpy(p, &v, sizeof(v));
247 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
248 }
249 }
250
251 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
252 uint32_t v)
253 {
254 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
255 *p = v;
256 } else {
257 memcpy(p, &v, sizeof(v));
258 }
259 }
260 #endif
261
262 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
263 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
264 {
265 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
266 *s->code_ptr++ = v;
267 } else {
268 tcg_insn_unit *p = s->code_ptr;
269 memcpy(p, &v, sizeof(v));
270 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
271 }
272 }
273
274 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
275 uint64_t v)
276 {
277 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
278 *p = v;
279 } else {
280 memcpy(p, &v, sizeof(v));
281 }
282 }
283 #endif
284
285 /* label relocation processing */
286
287 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
288 TCGLabel *l, intptr_t addend)
289 {
290 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
291
292 r->type = type;
293 r->ptr = code_ptr;
294 r->addend = addend;
295 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
296 }
297
298 static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr)
299 {
300 tcg_debug_assert(!l->has_value);
301 l->has_value = 1;
302 l->u.value_ptr = ptr;
303 }
304
305 TCGLabel *gen_new_label(void)
306 {
307 TCGContext *s = tcg_ctx;
308 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
309
310 memset(l, 0, sizeof(TCGLabel));
311 l->id = s->nb_labels++;
312 QSIMPLEQ_INIT(&l->relocs);
313
314 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
315
316 return l;
317 }
318
319 static bool tcg_resolve_relocs(TCGContext *s)
320 {
321 TCGLabel *l;
322
323 QSIMPLEQ_FOREACH(l, &s->labels, next) {
324 TCGRelocation *r;
325 uintptr_t value = l->u.value;
326
327 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
328 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
329 return false;
330 }
331 }
332 }
333 return true;
334 }
335
336 static void set_jmp_reset_offset(TCGContext *s, int which)
337 {
338 size_t off = tcg_current_code_size(s);
339 s->tb_jmp_reset_offset[which] = off;
340 /* Make sure that we didn't overflow the stored offset. */
341 assert(s->tb_jmp_reset_offset[which] == off);
342 }
343
344 #include "tcg-target.c.inc"
345
346 /* compare a pointer @ptr and a tb_tc @s */
347 static int ptr_cmp_tb_tc(const void *ptr, const struct tb_tc *s)
348 {
349 if (ptr >= s->ptr + s->size) {
350 return 1;
351 } else if (ptr < s->ptr) {
352 return -1;
353 }
354 return 0;
355 }
356
357 static gint tb_tc_cmp(gconstpointer ap, gconstpointer bp)
358 {
359 const struct tb_tc *a = ap;
360 const struct tb_tc *b = bp;
361
362 /*
363 * When both sizes are set, we know this isn't a lookup.
364 * This is the most likely case: every TB must be inserted; lookups
365 * are a lot less frequent.
366 */
367 if (likely(a->size && b->size)) {
368 if (a->ptr > b->ptr) {
369 return 1;
370 } else if (a->ptr < b->ptr) {
371 return -1;
372 }
373 /* a->ptr == b->ptr should happen only on deletions */
374 g_assert(a->size == b->size);
375 return 0;
376 }
377 /*
378 * All lookups have either .size field set to 0.
379 * From the glib sources we see that @ap is always the lookup key. However
380 * the docs provide no guarantee, so we just mark this case as likely.
381 */
382 if (likely(a->size == 0)) {
383 return ptr_cmp_tb_tc(a->ptr, b);
384 }
385 return ptr_cmp_tb_tc(b->ptr, a);
386 }
387
388 static void tcg_region_trees_init(void)
389 {
390 size_t i;
391
392 tree_size = ROUND_UP(sizeof(struct tcg_region_tree), qemu_dcache_linesize);
393 region_trees = qemu_memalign(qemu_dcache_linesize, region.n * tree_size);
394 for (i = 0; i < region.n; i++) {
395 struct tcg_region_tree *rt = region_trees + i * tree_size;
396
397 qemu_mutex_init(&rt->lock);
398 rt->tree = g_tree_new(tb_tc_cmp);
399 }
400 }
401
402 static struct tcg_region_tree *tc_ptr_to_region_tree(void *p)
403 {
404 size_t region_idx;
405
406 if (p < region.start_aligned) {
407 region_idx = 0;
408 } else {
409 ptrdiff_t offset = p - region.start_aligned;
410
411 if (offset > region.stride * (region.n - 1)) {
412 region_idx = region.n - 1;
413 } else {
414 region_idx = offset / region.stride;
415 }
416 }
417 return region_trees + region_idx * tree_size;
418 }
419
420 void tcg_tb_insert(TranslationBlock *tb)
421 {
422 struct tcg_region_tree *rt = tc_ptr_to_region_tree(tb->tc.ptr);
423
424 qemu_mutex_lock(&rt->lock);
425 g_tree_insert(rt->tree, &tb->tc, tb);
426 qemu_mutex_unlock(&rt->lock);
427 }
428
429 void tcg_tb_remove(TranslationBlock *tb)
430 {
431 struct tcg_region_tree *rt = tc_ptr_to_region_tree(tb->tc.ptr);
432
433 qemu_mutex_lock(&rt->lock);
434 g_tree_remove(rt->tree, &tb->tc);
435 qemu_mutex_unlock(&rt->lock);
436 }
437
438 /*
439 * Find the TB 'tb' such that
440 * tb->tc.ptr <= tc_ptr < tb->tc.ptr + tb->tc.size
441 * Return NULL if not found.
442 */
443 TranslationBlock *tcg_tb_lookup(uintptr_t tc_ptr)
444 {
445 struct tcg_region_tree *rt = tc_ptr_to_region_tree((void *)tc_ptr);
446 TranslationBlock *tb;
447 struct tb_tc s = { .ptr = (void *)tc_ptr };
448
449 qemu_mutex_lock(&rt->lock);
450 tb = g_tree_lookup(rt->tree, &s);
451 qemu_mutex_unlock(&rt->lock);
452 return tb;
453 }
454
455 static void tcg_region_tree_lock_all(void)
456 {
457 size_t i;
458
459 for (i = 0; i < region.n; i++) {
460 struct tcg_region_tree *rt = region_trees + i * tree_size;
461
462 qemu_mutex_lock(&rt->lock);
463 }
464 }
465
466 static void tcg_region_tree_unlock_all(void)
467 {
468 size_t i;
469
470 for (i = 0; i < region.n; i++) {
471 struct tcg_region_tree *rt = region_trees + i * tree_size;
472
473 qemu_mutex_unlock(&rt->lock);
474 }
475 }
476
477 void tcg_tb_foreach(GTraverseFunc func, gpointer user_data)
478 {
479 size_t i;
480
481 tcg_region_tree_lock_all();
482 for (i = 0; i < region.n; i++) {
483 struct tcg_region_tree *rt = region_trees + i * tree_size;
484
485 g_tree_foreach(rt->tree, func, user_data);
486 }
487 tcg_region_tree_unlock_all();
488 }
489
490 size_t tcg_nb_tbs(void)
491 {
492 size_t nb_tbs = 0;
493 size_t i;
494
495 tcg_region_tree_lock_all();
496 for (i = 0; i < region.n; i++) {
497 struct tcg_region_tree *rt = region_trees + i * tree_size;
498
499 nb_tbs += g_tree_nnodes(rt->tree);
500 }
501 tcg_region_tree_unlock_all();
502 return nb_tbs;
503 }
504
505 static gboolean tcg_region_tree_traverse(gpointer k, gpointer v, gpointer data)
506 {
507 TranslationBlock *tb = v;
508
509 tb_destroy(tb);
510 return FALSE;
511 }
512
513 static void tcg_region_tree_reset_all(void)
514 {
515 size_t i;
516
517 tcg_region_tree_lock_all();
518 for (i = 0; i < region.n; i++) {
519 struct tcg_region_tree *rt = region_trees + i * tree_size;
520
521 g_tree_foreach(rt->tree, tcg_region_tree_traverse, NULL);
522 /* Increment the refcount first so that destroy acts as a reset */
523 g_tree_ref(rt->tree);
524 g_tree_destroy(rt->tree);
525 }
526 tcg_region_tree_unlock_all();
527 }
528
529 static void tcg_region_bounds(size_t curr_region, void **pstart, void **pend)
530 {
531 void *start, *end;
532
533 start = region.start_aligned + curr_region * region.stride;
534 end = start + region.size;
535
536 if (curr_region == 0) {
537 start = region.start;
538 }
539 if (curr_region == region.n - 1) {
540 end = region.end;
541 }
542
543 *pstart = start;
544 *pend = end;
545 }
546
547 static void tcg_region_assign(TCGContext *s, size_t curr_region)
548 {
549 void *start, *end;
550
551 tcg_region_bounds(curr_region, &start, &end);
552
553 s->code_gen_buffer = start;
554 s->code_gen_ptr = start;
555 s->code_gen_buffer_size = end - start;
556 s->code_gen_highwater = end - TCG_HIGHWATER;
557 }
558
559 static bool tcg_region_alloc__locked(TCGContext *s)
560 {
561 if (region.current == region.n) {
562 return true;
563 }
564 tcg_region_assign(s, region.current);
565 region.current++;
566 return false;
567 }
568
569 /*
570 * Request a new region once the one in use has filled up.
571 * Returns true on error.
572 */
573 static bool tcg_region_alloc(TCGContext *s)
574 {
575 bool err;
576 /* read the region size now; alloc__locked will overwrite it on success */
577 size_t size_full = s->code_gen_buffer_size;
578
579 qemu_mutex_lock(&region.lock);
580 err = tcg_region_alloc__locked(s);
581 if (!err) {
582 region.agg_size_full += size_full - TCG_HIGHWATER;
583 }
584 qemu_mutex_unlock(&region.lock);
585 return err;
586 }
587
588 /*
589 * Perform a context's first region allocation.
590 * This function does _not_ increment region.agg_size_full.
591 */
592 static inline bool tcg_region_initial_alloc__locked(TCGContext *s)
593 {
594 return tcg_region_alloc__locked(s);
595 }
596
597 /* Call from a safe-work context */
598 void tcg_region_reset_all(void)
599 {
600 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs);
601 unsigned int i;
602
603 qemu_mutex_lock(&region.lock);
604 region.current = 0;
605 region.agg_size_full = 0;
606
607 for (i = 0; i < n_ctxs; i++) {
608 TCGContext *s = atomic_read(&tcg_ctxs[i]);
609 bool err = tcg_region_initial_alloc__locked(s);
610
611 g_assert(!err);
612 }
613 qemu_mutex_unlock(&region.lock);
614
615 tcg_region_tree_reset_all();
616 }
617
618 #ifdef CONFIG_USER_ONLY
619 static size_t tcg_n_regions(void)
620 {
621 return 1;
622 }
623 #else
624 /*
625 * It is likely that some vCPUs will translate more code than others, so we
626 * first try to set more regions than max_cpus, with those regions being of
627 * reasonable size. If that's not possible we make do by evenly dividing
628 * the code_gen_buffer among the vCPUs.
629 */
630 static size_t tcg_n_regions(void)
631 {
632 size_t i;
633
634 /* Use a single region if all we have is one vCPU thread */
635 #if !defined(CONFIG_USER_ONLY)
636 MachineState *ms = MACHINE(qdev_get_machine());
637 unsigned int max_cpus = ms->smp.max_cpus;
638 #endif
639 if (max_cpus == 1 || !qemu_tcg_mttcg_enabled()) {
640 return 1;
641 }
642
643 /* Try to have more regions than max_cpus, with each region being >= 2 MB */
644 for (i = 8; i > 0; i--) {
645 size_t regions_per_thread = i;
646 size_t region_size;
647
648 region_size = tcg_init_ctx.code_gen_buffer_size;
649 region_size /= max_cpus * regions_per_thread;
650
651 if (region_size >= 2 * 1024u * 1024) {
652 return max_cpus * regions_per_thread;
653 }
654 }
655 /* If we can't, then just allocate one region per vCPU thread */
656 return max_cpus;
657 }
658 #endif
659
660 /*
661 * Initializes region partitioning.
662 *
663 * Called at init time from the parent thread (i.e. the one calling
664 * tcg_context_init), after the target's TCG globals have been set.
665 *
666 * Region partitioning works by splitting code_gen_buffer into separate regions,
667 * and then assigning regions to TCG threads so that the threads can translate
668 * code in parallel without synchronization.
669 *
670 * In softmmu the number of TCG threads is bounded by max_cpus, so we use at
671 * least max_cpus regions in MTTCG. In !MTTCG we use a single region.
672 * Note that the TCG options from the command-line (i.e. -accel accel=tcg,[...])
673 * must have been parsed before calling this function, since it calls
674 * qemu_tcg_mttcg_enabled().
675 *
676 * In user-mode we use a single region. Having multiple regions in user-mode
677 * is not supported, because the number of vCPU threads (recall that each thread
678 * spawned by the guest corresponds to a vCPU thread) is only bounded by the
679 * OS, and usually this number is huge (tens of thousands is not uncommon).
680 * Thus, given this large bound on the number of vCPU threads and the fact
681 * that code_gen_buffer is allocated at compile-time, we cannot guarantee
682 * that the availability of at least one region per vCPU thread.
683 *
684 * However, this user-mode limitation is unlikely to be a significant problem
685 * in practice. Multi-threaded guests share most if not all of their translated
686 * code, which makes parallel code generation less appealing than in softmmu.
687 */
688 void tcg_region_init(void)
689 {
690 void *buf = tcg_init_ctx.code_gen_buffer;
691 void *aligned;
692 size_t size = tcg_init_ctx.code_gen_buffer_size;
693 size_t page_size = qemu_real_host_page_size;
694 size_t region_size;
695 size_t n_regions;
696 size_t i;
697
698 n_regions = tcg_n_regions();
699
700 /* The first region will be 'aligned - buf' bytes larger than the others */
701 aligned = QEMU_ALIGN_PTR_UP(buf, page_size);
702 g_assert(aligned < tcg_init_ctx.code_gen_buffer + size);
703 /*
704 * Make region_size a multiple of page_size, using aligned as the start.
705 * As a result of this we might end up with a few extra pages at the end of
706 * the buffer; we will assign those to the last region.
707 */
708 region_size = (size - (aligned - buf)) / n_regions;
709 region_size = QEMU_ALIGN_DOWN(region_size, page_size);
710
711 /* A region must have at least 2 pages; one code, one guard */
712 g_assert(region_size >= 2 * page_size);
713
714 /* init the region struct */
715 qemu_mutex_init(&region.lock);
716 region.n = n_regions;
717 region.size = region_size - page_size;
718 region.stride = region_size;
719 region.start = buf;
720 region.start_aligned = aligned;
721 /* page-align the end, since its last page will be a guard page */
722 region.end = QEMU_ALIGN_PTR_DOWN(buf + size, page_size);
723 /* account for that last guard page */
724 region.end -= page_size;
725
726 /* set guard pages */
727 for (i = 0; i < region.n; i++) {
728 void *start, *end;
729 int rc;
730
731 tcg_region_bounds(i, &start, &end);
732 rc = qemu_mprotect_none(end, page_size);
733 g_assert(!rc);
734 }
735
736 tcg_region_trees_init();
737
738 /* In user-mode we support only one ctx, so do the initial allocation now */
739 #ifdef CONFIG_USER_ONLY
740 {
741 bool err = tcg_region_initial_alloc__locked(tcg_ctx);
742
743 g_assert(!err);
744 }
745 #endif
746 }
747
748 static void alloc_tcg_plugin_context(TCGContext *s)
749 {
750 #ifdef CONFIG_PLUGIN
751 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
752 s->plugin_tb->insns =
753 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
754 #endif
755 }
756
757 /*
758 * All TCG threads except the parent (i.e. the one that called tcg_context_init
759 * and registered the target's TCG globals) must register with this function
760 * before initiating translation.
761 *
762 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
763 * of tcg_region_init() for the reasoning behind this.
764 *
765 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
766 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
767 * is not used anymore for translation once this function is called.
768 *
769 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
770 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
771 */
772 #ifdef CONFIG_USER_ONLY
773 void tcg_register_thread(void)
774 {
775 tcg_ctx = &tcg_init_ctx;
776 }
777 #else
778 void tcg_register_thread(void)
779 {
780 MachineState *ms = MACHINE(qdev_get_machine());
781 TCGContext *s = g_malloc(sizeof(*s));
782 unsigned int i, n;
783 bool err;
784
785 *s = tcg_init_ctx;
786
787 /* Relink mem_base. */
788 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
789 if (tcg_init_ctx.temps[i].mem_base) {
790 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
791 tcg_debug_assert(b >= 0 && b < n);
792 s->temps[i].mem_base = &s->temps[b];
793 }
794 }
795
796 /* Claim an entry in tcg_ctxs */
797 n = atomic_fetch_inc(&n_tcg_ctxs);
798 g_assert(n < ms->smp.max_cpus);
799 atomic_set(&tcg_ctxs[n], s);
800
801 if (n > 0) {
802 alloc_tcg_plugin_context(s);
803 }
804
805 tcg_ctx = s;
806 qemu_mutex_lock(&region.lock);
807 err = tcg_region_initial_alloc__locked(tcg_ctx);
808 g_assert(!err);
809 qemu_mutex_unlock(&region.lock);
810 }
811 #endif /* !CONFIG_USER_ONLY */
812
813 /*
814 * Returns the size (in bytes) of all translated code (i.e. from all regions)
815 * currently in the cache.
816 * See also: tcg_code_capacity()
817 * Do not confuse with tcg_current_code_size(); that one applies to a single
818 * TCG context.
819 */
820 size_t tcg_code_size(void)
821 {
822 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs);
823 unsigned int i;
824 size_t total;
825
826 qemu_mutex_lock(&region.lock);
827 total = region.agg_size_full;
828 for (i = 0; i < n_ctxs; i++) {
829 const TCGContext *s = atomic_read(&tcg_ctxs[i]);
830 size_t size;
831
832 size = atomic_read(&s->code_gen_ptr) - s->code_gen_buffer;
833 g_assert(size <= s->code_gen_buffer_size);
834 total += size;
835 }
836 qemu_mutex_unlock(&region.lock);
837 return total;
838 }
839
840 /*
841 * Returns the code capacity (in bytes) of the entire cache, i.e. including all
842 * regions.
843 * See also: tcg_code_size()
844 */
845 size_t tcg_code_capacity(void)
846 {
847 size_t guard_size, capacity;
848
849 /* no need for synchronization; these variables are set at init time */
850 guard_size = region.stride - region.size;
851 capacity = region.end + guard_size - region.start;
852 capacity -= region.n * (guard_size + TCG_HIGHWATER);
853 return capacity;
854 }
855
856 size_t tcg_tb_phys_invalidate_count(void)
857 {
858 unsigned int n_ctxs = atomic_read(&n_tcg_ctxs);
859 unsigned int i;
860 size_t total = 0;
861
862 for (i = 0; i < n_ctxs; i++) {
863 const TCGContext *s = atomic_read(&tcg_ctxs[i]);
864
865 total += atomic_read(&s->tb_phys_invalidate_count);
866 }
867 return total;
868 }
869
870 /* pool based memory allocation */
871 void *tcg_malloc_internal(TCGContext *s, int size)
872 {
873 TCGPool *p;
874 int pool_size;
875
876 if (size > TCG_POOL_CHUNK_SIZE) {
877 /* big malloc: insert a new pool (XXX: could optimize) */
878 p = g_malloc(sizeof(TCGPool) + size);
879 p->size = size;
880 p->next = s->pool_first_large;
881 s->pool_first_large = p;
882 return p->data;
883 } else {
884 p = s->pool_current;
885 if (!p) {
886 p = s->pool_first;
887 if (!p)
888 goto new_pool;
889 } else {
890 if (!p->next) {
891 new_pool:
892 pool_size = TCG_POOL_CHUNK_SIZE;
893 p = g_malloc(sizeof(TCGPool) + pool_size);
894 p->size = pool_size;
895 p->next = NULL;
896 if (s->pool_current)
897 s->pool_current->next = p;
898 else
899 s->pool_first = p;
900 } else {
901 p = p->next;
902 }
903 }
904 }
905 s->pool_current = p;
906 s->pool_cur = p->data + size;
907 s->pool_end = p->data + p->size;
908 return p->data;
909 }
910
911 void tcg_pool_reset(TCGContext *s)
912 {
913 TCGPool *p, *t;
914 for (p = s->pool_first_large; p; p = t) {
915 t = p->next;
916 g_free(p);
917 }
918 s->pool_first_large = NULL;
919 s->pool_cur = s->pool_end = NULL;
920 s->pool_current = NULL;
921 }
922
923 typedef struct TCGHelperInfo {
924 void *func;
925 const char *name;
926 unsigned flags;
927 unsigned sizemask;
928 } TCGHelperInfo;
929
930 #include "exec/helper-proto.h"
931
932 static const TCGHelperInfo all_helpers[] = {
933 #include "exec/helper-tcg.h"
934 };
935 static GHashTable *helper_table;
936
937 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
938 static void process_op_defs(TCGContext *s);
939 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
940 TCGReg reg, const char *name);
941
942 void tcg_context_init(TCGContext *s)
943 {
944 int op, total_args, n, i;
945 TCGOpDef *def;
946 TCGArgConstraint *args_ct;
947 int *sorted_args;
948 TCGTemp *ts;
949
950 memset(s, 0, sizeof(*s));
951 s->nb_globals = 0;
952
953 /* Count total number of arguments and allocate the corresponding
954 space */
955 total_args = 0;
956 for(op = 0; op < NB_OPS; op++) {
957 def = &tcg_op_defs[op];
958 n = def->nb_iargs + def->nb_oargs;
959 total_args += n;
960 }
961
962 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args);
963 sorted_args = g_malloc(sizeof(int) * total_args);
964
965 for(op = 0; op < NB_OPS; op++) {
966 def = &tcg_op_defs[op];
967 def->args_ct = args_ct;
968 def->sorted_args = sorted_args;
969 n = def->nb_iargs + def->nb_oargs;
970 sorted_args += n;
971 args_ct += n;
972 }
973
974 /* Register helpers. */
975 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
976 helper_table = g_hash_table_new(NULL, NULL);
977
978 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
979 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
980 (gpointer)&all_helpers[i]);
981 }
982
983 tcg_target_init(s);
984 process_op_defs(s);
985
986 /* Reverse the order of the saved registers, assuming they're all at
987 the start of tcg_target_reg_alloc_order. */
988 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
989 int r = tcg_target_reg_alloc_order[n];
990 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
991 break;
992 }
993 }
994 for (i = 0; i < n; ++i) {
995 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
996 }
997 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
998 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
999 }
1000
1001 alloc_tcg_plugin_context(s);
1002
1003 tcg_ctx = s;
1004 /*
1005 * In user-mode we simply share the init context among threads, since we
1006 * use a single region. See the documentation tcg_region_init() for the
1007 * reasoning behind this.
1008 * In softmmu we will have at most max_cpus TCG threads.
1009 */
1010 #ifdef CONFIG_USER_ONLY
1011 tcg_ctxs = &tcg_ctx;
1012 n_tcg_ctxs = 1;
1013 #else
1014 MachineState *ms = MACHINE(qdev_get_machine());
1015 unsigned int max_cpus = ms->smp.max_cpus;
1016 tcg_ctxs = g_new(TCGContext *, max_cpus);
1017 #endif
1018
1019 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1020 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
1021 cpu_env = temp_tcgv_ptr(ts);
1022 }
1023
1024 /*
1025 * Allocate TBs right before their corresponding translated code, making
1026 * sure that TBs and code are on different cache lines.
1027 */
1028 TranslationBlock *tcg_tb_alloc(TCGContext *s)
1029 {
1030 uintptr_t align = qemu_icache_linesize;
1031 TranslationBlock *tb;
1032 void *next;
1033
1034 retry:
1035 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1036 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
1037
1038 if (unlikely(next > s->code_gen_highwater)) {
1039 if (tcg_region_alloc(s)) {
1040 return NULL;
1041 }
1042 goto retry;
1043 }
1044 atomic_set(&s->code_gen_ptr, next);
1045 s->data_gen_ptr = NULL;
1046 return tb;
1047 }
1048
1049 void tcg_prologue_init(TCGContext *s)
1050 {
1051 size_t prologue_size, total_size;
1052 void *buf0, *buf1;
1053
1054 /* Put the prologue at the beginning of code_gen_buffer. */
1055 buf0 = s->code_gen_buffer;
1056 total_size = s->code_gen_buffer_size;
1057 s->code_ptr = buf0;
1058 s->code_buf = buf0;
1059 s->data_gen_ptr = NULL;
1060 s->code_gen_prologue = buf0;
1061
1062 /* Compute a high-water mark, at which we voluntarily flush the buffer
1063 and start over. The size here is arbitrary, significantly larger
1064 than we expect the code generation for any one opcode to require. */
1065 s->code_gen_highwater = s->code_gen_buffer + (total_size - TCG_HIGHWATER);
1066
1067 #ifdef TCG_TARGET_NEED_POOL_LABELS
1068 s->pool_labels = NULL;
1069 #endif
1070
1071 /* Generate the prologue. */
1072 tcg_target_qemu_prologue(s);
1073
1074 #ifdef TCG_TARGET_NEED_POOL_LABELS
1075 /* Allow the prologue to put e.g. guest_base into a pool entry. */
1076 {
1077 int result = tcg_out_pool_finalize(s);
1078 tcg_debug_assert(result == 0);
1079 }
1080 #endif
1081
1082 buf1 = s->code_ptr;
1083 flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1);
1084
1085 /* Deduct the prologue from the buffer. */
1086 prologue_size = tcg_current_code_size(s);
1087 s->code_gen_ptr = buf1;
1088 s->code_gen_buffer = buf1;
1089 s->code_buf = buf1;
1090 total_size -= prologue_size;
1091 s->code_gen_buffer_size = total_size;
1092
1093 tcg_register_jit(s->code_gen_buffer, total_size);
1094
1095 #ifdef DEBUG_DISAS
1096 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
1097 FILE *logfile = qemu_log_lock();
1098 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
1099 if (s->data_gen_ptr) {
1100 size_t code_size = s->data_gen_ptr - buf0;
1101 size_t data_size = prologue_size - code_size;
1102 size_t i;
1103
1104 log_disas(buf0, code_size, NULL);
1105
1106 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1107 if (sizeof(tcg_target_ulong) == 8) {
1108 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1109 (uintptr_t)s->data_gen_ptr + i,
1110 *(uint64_t *)(s->data_gen_ptr + i));
1111 } else {
1112 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
1113 (uintptr_t)s->data_gen_ptr + i,
1114 *(uint32_t *)(s->data_gen_ptr + i));
1115 }
1116 }
1117 } else {
1118 log_disas(buf0, prologue_size, NULL);
1119 }
1120 qemu_log("\n");
1121 qemu_log_flush();
1122 qemu_log_unlock(logfile);
1123 }
1124 #endif
1125
1126 /* Assert that goto_ptr is implemented completely. */
1127 if (TCG_TARGET_HAS_goto_ptr) {
1128 tcg_debug_assert(s->code_gen_epilogue != NULL);
1129 }
1130 }
1131
1132 void tcg_func_start(TCGContext *s)
1133 {
1134 tcg_pool_reset(s);
1135 s->nb_temps = s->nb_globals;
1136
1137 /* No temps have been previously allocated for size or locality. */
1138 memset(s->free_temps, 0, sizeof(s->free_temps));
1139
1140 s->nb_ops = 0;
1141 s->nb_labels = 0;
1142 s->current_frame_offset = s->frame_start;
1143
1144 #ifdef CONFIG_DEBUG_TCG
1145 s->goto_tb_issue_mask = 0;
1146 #endif
1147
1148 QTAILQ_INIT(&s->ops);
1149 QTAILQ_INIT(&s->free_ops);
1150 QSIMPLEQ_INIT(&s->labels);
1151 }
1152
1153 static inline TCGTemp *tcg_temp_alloc(TCGContext *s)
1154 {
1155 int n = s->nb_temps++;
1156 tcg_debug_assert(n < TCG_MAX_TEMPS);
1157 return memset(&s->temps[n], 0, sizeof(TCGTemp));
1158 }
1159
1160 static inline TCGTemp *tcg_global_alloc(TCGContext *s)
1161 {
1162 TCGTemp *ts;
1163
1164 tcg_debug_assert(s->nb_globals == s->nb_temps);
1165 s->nb_globals++;
1166 ts = tcg_temp_alloc(s);
1167 ts->temp_global = 1;
1168
1169 return ts;
1170 }
1171
1172 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1173 TCGReg reg, const char *name)
1174 {
1175 TCGTemp *ts;
1176
1177 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
1178 tcg_abort();
1179 }
1180
1181 ts = tcg_global_alloc(s);
1182 ts->base_type = type;
1183 ts->type = type;
1184 ts->fixed_reg = 1;
1185 ts->reg = reg;
1186 ts->name = name;
1187 tcg_regset_set_reg(s->reserved_regs, reg);
1188
1189 return ts;
1190 }
1191
1192 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
1193 {
1194 s->frame_start = start;
1195 s->frame_end = start + size;
1196 s->frame_temp
1197 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
1198 }
1199
1200 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
1201 intptr_t offset, const char *name)
1202 {
1203 TCGContext *s = tcg_ctx;
1204 TCGTemp *base_ts = tcgv_ptr_temp(base);
1205 TCGTemp *ts = tcg_global_alloc(s);
1206 int indirect_reg = 0, bigendian = 0;
1207 #ifdef HOST_WORDS_BIGENDIAN
1208 bigendian = 1;
1209 #endif
1210
1211 if (!base_ts->fixed_reg) {
1212 /* We do not support double-indirect registers. */
1213 tcg_debug_assert(!base_ts->indirect_reg);
1214 base_ts->indirect_base = 1;
1215 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
1216 ? 2 : 1);
1217 indirect_reg = 1;
1218 }
1219
1220 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1221 TCGTemp *ts2 = tcg_global_alloc(s);
1222 char buf[64];
1223
1224 ts->base_type = TCG_TYPE_I64;
1225 ts->type = TCG_TYPE_I32;
1226 ts->indirect_reg = indirect_reg;
1227 ts->mem_allocated = 1;
1228 ts->mem_base = base_ts;
1229 ts->mem_offset = offset + bigendian * 4;
1230 pstrcpy(buf, sizeof(buf), name);
1231 pstrcat(buf, sizeof(buf), "_0");
1232 ts->name = strdup(buf);
1233
1234 tcg_debug_assert(ts2 == ts + 1);
1235 ts2->base_type = TCG_TYPE_I64;
1236 ts2->type = TCG_TYPE_I32;
1237 ts2->indirect_reg = indirect_reg;
1238 ts2->mem_allocated = 1;
1239 ts2->mem_base = base_ts;
1240 ts2->mem_offset = offset + (1 - bigendian) * 4;
1241 pstrcpy(buf, sizeof(buf), name);
1242 pstrcat(buf, sizeof(buf), "_1");
1243 ts2->name = strdup(buf);
1244 } else {
1245 ts->base_type = type;
1246 ts->type = type;
1247 ts->indirect_reg = indirect_reg;
1248 ts->mem_allocated = 1;
1249 ts->mem_base = base_ts;
1250 ts->mem_offset = offset;
1251 ts->name = name;
1252 }
1253 return ts;
1254 }
1255
1256 TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
1257 {
1258 TCGContext *s = tcg_ctx;
1259 TCGTemp *ts;
1260 int idx, k;
1261
1262 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
1263 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
1264 if (idx < TCG_MAX_TEMPS) {
1265 /* There is already an available temp with the right type. */
1266 clear_bit(idx, s->free_temps[k].l);
1267
1268 ts = &s->temps[idx];
1269 ts->temp_allocated = 1;
1270 tcg_debug_assert(ts->base_type == type);
1271 tcg_debug_assert(ts->temp_local == temp_local);
1272 } else {
1273 ts = tcg_temp_alloc(s);
1274 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1275 TCGTemp *ts2 = tcg_temp_alloc(s);
1276
1277 ts->base_type = type;
1278 ts->type = TCG_TYPE_I32;
1279 ts->temp_allocated = 1;
1280 ts->temp_local = temp_local;
1281
1282 tcg_debug_assert(ts2 == ts + 1);
1283 ts2->base_type = TCG_TYPE_I64;
1284 ts2->type = TCG_TYPE_I32;
1285 ts2->temp_allocated = 1;
1286 ts2->temp_local = temp_local;
1287 } else {
1288 ts->base_type = type;
1289 ts->type = type;
1290 ts->temp_allocated = 1;
1291 ts->temp_local = temp_local;
1292 }
1293 }
1294
1295 #if defined(CONFIG_DEBUG_TCG)
1296 s->temps_in_use++;
1297 #endif
1298 return ts;
1299 }
1300
1301 TCGv_vec tcg_temp_new_vec(TCGType type)
1302 {
1303 TCGTemp *t;
1304
1305 #ifdef CONFIG_DEBUG_TCG
1306 switch (type) {
1307 case TCG_TYPE_V64:
1308 assert(TCG_TARGET_HAS_v64);
1309 break;
1310 case TCG_TYPE_V128:
1311 assert(TCG_TARGET_HAS_v128);
1312 break;
1313 case TCG_TYPE_V256:
1314 assert(TCG_TARGET_HAS_v256);
1315 break;
1316 default:
1317 g_assert_not_reached();
1318 }
1319 #endif
1320
1321 t = tcg_temp_new_internal(type, 0);
1322 return temp_tcgv_vec(t);
1323 }
1324
1325 /* Create a new temp of the same type as an existing temp. */
1326 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1327 {
1328 TCGTemp *t = tcgv_vec_temp(match);
1329
1330 tcg_debug_assert(t->temp_allocated != 0);
1331
1332 t = tcg_temp_new_internal(t->base_type, 0);
1333 return temp_tcgv_vec(t);
1334 }
1335
1336 void tcg_temp_free_internal(TCGTemp *ts)
1337 {
1338 TCGContext *s = tcg_ctx;
1339 int k, idx;
1340
1341 #if defined(CONFIG_DEBUG_TCG)
1342 s->temps_in_use--;
1343 if (s->temps_in_use < 0) {
1344 fprintf(stderr, "More temporaries freed than allocated!\n");
1345 }
1346 #endif
1347
1348 tcg_debug_assert(ts->temp_global == 0);
1349 tcg_debug_assert(ts->temp_allocated != 0);
1350 ts->temp_allocated = 0;
1351
1352 idx = temp_idx(ts);
1353 k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0);
1354 set_bit(idx, s->free_temps[k].l);
1355 }
1356
1357 TCGv_i32 tcg_const_i32(int32_t val)
1358 {
1359 TCGv_i32 t0;
1360 t0 = tcg_temp_new_i32();
1361 tcg_gen_movi_i32(t0, val);
1362 return t0;
1363 }
1364
1365 TCGv_i64 tcg_const_i64(int64_t val)
1366 {
1367 TCGv_i64 t0;
1368 t0 = tcg_temp_new_i64();
1369 tcg_gen_movi_i64(t0, val);
1370 return t0;
1371 }
1372
1373 TCGv_i32 tcg_const_local_i32(int32_t val)
1374 {
1375 TCGv_i32 t0;
1376 t0 = tcg_temp_local_new_i32();
1377 tcg_gen_movi_i32(t0, val);
1378 return t0;
1379 }
1380
1381 TCGv_i64 tcg_const_local_i64(int64_t val)
1382 {
1383 TCGv_i64 t0;
1384 t0 = tcg_temp_local_new_i64();
1385 tcg_gen_movi_i64(t0, val);
1386 return t0;
1387 }
1388
1389 #if defined(CONFIG_DEBUG_TCG)
1390 void tcg_clear_temp_count(void)
1391 {
1392 TCGContext *s = tcg_ctx;
1393 s->temps_in_use = 0;
1394 }
1395
1396 int tcg_check_temp_count(void)
1397 {
1398 TCGContext *s = tcg_ctx;
1399 if (s->temps_in_use) {
1400 /* Clear the count so that we don't give another
1401 * warning immediately next time around.
1402 */
1403 s->temps_in_use = 0;
1404 return 1;
1405 }
1406 return 0;
1407 }
1408 #endif
1409
1410 /* Return true if OP may appear in the opcode stream.
1411 Test the runtime variable that controls each opcode. */
1412 bool tcg_op_supported(TCGOpcode op)
1413 {
1414 const bool have_vec
1415 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1416
1417 switch (op) {
1418 case INDEX_op_discard:
1419 case INDEX_op_set_label:
1420 case INDEX_op_call:
1421 case INDEX_op_br:
1422 case INDEX_op_mb:
1423 case INDEX_op_insn_start:
1424 case INDEX_op_exit_tb:
1425 case INDEX_op_goto_tb:
1426 case INDEX_op_qemu_ld_i32:
1427 case INDEX_op_qemu_st_i32:
1428 case INDEX_op_qemu_ld_i64:
1429 case INDEX_op_qemu_st_i64:
1430 return true;
1431
1432 case INDEX_op_goto_ptr:
1433 return TCG_TARGET_HAS_goto_ptr;
1434
1435 case INDEX_op_mov_i32:
1436 case INDEX_op_movi_i32:
1437 case INDEX_op_setcond_i32:
1438 case INDEX_op_brcond_i32:
1439 case INDEX_op_ld8u_i32:
1440 case INDEX_op_ld8s_i32:
1441 case INDEX_op_ld16u_i32:
1442 case INDEX_op_ld16s_i32:
1443 case INDEX_op_ld_i32:
1444 case INDEX_op_st8_i32:
1445 case INDEX_op_st16_i32:
1446 case INDEX_op_st_i32:
1447 case INDEX_op_add_i32:
1448 case INDEX_op_sub_i32:
1449 case INDEX_op_mul_i32:
1450 case INDEX_op_and_i32:
1451 case INDEX_op_or_i32:
1452 case INDEX_op_xor_i32:
1453 case INDEX_op_shl_i32:
1454 case INDEX_op_shr_i32:
1455 case INDEX_op_sar_i32:
1456 return true;
1457
1458 case INDEX_op_movcond_i32:
1459 return TCG_TARGET_HAS_movcond_i32;
1460 case INDEX_op_div_i32:
1461 case INDEX_op_divu_i32:
1462 return TCG_TARGET_HAS_div_i32;
1463 case INDEX_op_rem_i32:
1464 case INDEX_op_remu_i32:
1465 return TCG_TARGET_HAS_rem_i32;
1466 case INDEX_op_div2_i32:
1467 case INDEX_op_divu2_i32:
1468 return TCG_TARGET_HAS_div2_i32;
1469 case INDEX_op_rotl_i32:
1470 case INDEX_op_rotr_i32:
1471 return TCG_TARGET_HAS_rot_i32;
1472 case INDEX_op_deposit_i32:
1473 return TCG_TARGET_HAS_deposit_i32;
1474 case INDEX_op_extract_i32:
1475 return TCG_TARGET_HAS_extract_i32;
1476 case INDEX_op_sextract_i32:
1477 return TCG_TARGET_HAS_sextract_i32;
1478 case INDEX_op_extract2_i32:
1479 return TCG_TARGET_HAS_extract2_i32;
1480 case INDEX_op_add2_i32:
1481 return TCG_TARGET_HAS_add2_i32;
1482 case INDEX_op_sub2_i32:
1483 return TCG_TARGET_HAS_sub2_i32;
1484 case INDEX_op_mulu2_i32:
1485 return TCG_TARGET_HAS_mulu2_i32;
1486 case INDEX_op_muls2_i32:
1487 return TCG_TARGET_HAS_muls2_i32;
1488 case INDEX_op_muluh_i32:
1489 return TCG_TARGET_HAS_muluh_i32;
1490 case INDEX_op_mulsh_i32:
1491 return TCG_TARGET_HAS_mulsh_i32;
1492 case INDEX_op_ext8s_i32:
1493 return TCG_TARGET_HAS_ext8s_i32;
1494 case INDEX_op_ext16s_i32:
1495 return TCG_TARGET_HAS_ext16s_i32;
1496 case INDEX_op_ext8u_i32:
1497 return TCG_TARGET_HAS_ext8u_i32;
1498 case INDEX_op_ext16u_i32:
1499 return TCG_TARGET_HAS_ext16u_i32;
1500 case INDEX_op_bswap16_i32:
1501 return TCG_TARGET_HAS_bswap16_i32;
1502 case INDEX_op_bswap32_i32:
1503 return TCG_TARGET_HAS_bswap32_i32;
1504 case INDEX_op_not_i32:
1505 return TCG_TARGET_HAS_not_i32;
1506 case INDEX_op_neg_i32:
1507 return TCG_TARGET_HAS_neg_i32;
1508 case INDEX_op_andc_i32:
1509 return TCG_TARGET_HAS_andc_i32;
1510 case INDEX_op_orc_i32:
1511 return TCG_TARGET_HAS_orc_i32;
1512 case INDEX_op_eqv_i32:
1513 return TCG_TARGET_HAS_eqv_i32;
1514 case INDEX_op_nand_i32:
1515 return TCG_TARGET_HAS_nand_i32;
1516 case INDEX_op_nor_i32:
1517 return TCG_TARGET_HAS_nor_i32;
1518 case INDEX_op_clz_i32:
1519 return TCG_TARGET_HAS_clz_i32;
1520 case INDEX_op_ctz_i32:
1521 return TCG_TARGET_HAS_ctz_i32;
1522 case INDEX_op_ctpop_i32:
1523 return TCG_TARGET_HAS_ctpop_i32;
1524
1525 case INDEX_op_brcond2_i32:
1526 case INDEX_op_setcond2_i32:
1527 return TCG_TARGET_REG_BITS == 32;
1528
1529 case INDEX_op_mov_i64:
1530 case INDEX_op_movi_i64:
1531 case INDEX_op_setcond_i64:
1532 case INDEX_op_brcond_i64:
1533 case INDEX_op_ld8u_i64:
1534 case INDEX_op_ld8s_i64:
1535 case INDEX_op_ld16u_i64:
1536 case INDEX_op_ld16s_i64:
1537 case INDEX_op_ld32u_i64:
1538 case INDEX_op_ld32s_i64:
1539 case INDEX_op_ld_i64:
1540 case INDEX_op_st8_i64:
1541 case INDEX_op_st16_i64:
1542 case INDEX_op_st32_i64:
1543 case INDEX_op_st_i64:
1544 case INDEX_op_add_i64:
1545 case INDEX_op_sub_i64:
1546 case INDEX_op_mul_i64:
1547 case INDEX_op_and_i64:
1548 case INDEX_op_or_i64:
1549 case INDEX_op_xor_i64:
1550 case INDEX_op_shl_i64:
1551 case INDEX_op_shr_i64:
1552 case INDEX_op_sar_i64:
1553 case INDEX_op_ext_i32_i64:
1554 case INDEX_op_extu_i32_i64:
1555 return TCG_TARGET_REG_BITS == 64;
1556
1557 case INDEX_op_movcond_i64:
1558 return TCG_TARGET_HAS_movcond_i64;
1559 case INDEX_op_div_i64:
1560 case INDEX_op_divu_i64:
1561 return TCG_TARGET_HAS_div_i64;
1562 case INDEX_op_rem_i64:
1563 case INDEX_op_remu_i64:
1564 return TCG_TARGET_HAS_rem_i64;
1565 case INDEX_op_div2_i64:
1566 case INDEX_op_divu2_i64:
1567 return TCG_TARGET_HAS_div2_i64;
1568 case INDEX_op_rotl_i64:
1569 case INDEX_op_rotr_i64:
1570 return TCG_TARGET_HAS_rot_i64;
1571 case INDEX_op_deposit_i64:
1572 return TCG_TARGET_HAS_deposit_i64;
1573 case INDEX_op_extract_i64:
1574 return TCG_TARGET_HAS_extract_i64;
1575 case INDEX_op_sextract_i64:
1576 return TCG_TARGET_HAS_sextract_i64;
1577 case INDEX_op_extract2_i64:
1578 return TCG_TARGET_HAS_extract2_i64;
1579 case INDEX_op_extrl_i64_i32:
1580 return TCG_TARGET_HAS_extrl_i64_i32;
1581 case INDEX_op_extrh_i64_i32:
1582 return TCG_TARGET_HAS_extrh_i64_i32;
1583 case INDEX_op_ext8s_i64:
1584 return TCG_TARGET_HAS_ext8s_i64;
1585 case INDEX_op_ext16s_i64:
1586 return TCG_TARGET_HAS_ext16s_i64;
1587 case INDEX_op_ext32s_i64:
1588 return TCG_TARGET_HAS_ext32s_i64;
1589 case INDEX_op_ext8u_i64:
1590 return TCG_TARGET_HAS_ext8u_i64;
1591 case INDEX_op_ext16u_i64:
1592 return TCG_TARGET_HAS_ext16u_i64;
1593 case INDEX_op_ext32u_i64:
1594 return TCG_TARGET_HAS_ext32u_i64;
1595 case INDEX_op_bswap16_i64:
1596 return TCG_TARGET_HAS_bswap16_i64;
1597 case INDEX_op_bswap32_i64:
1598 return TCG_TARGET_HAS_bswap32_i64;
1599 case INDEX_op_bswap64_i64:
1600 return TCG_TARGET_HAS_bswap64_i64;
1601 case INDEX_op_not_i64:
1602 return TCG_TARGET_HAS_not_i64;
1603 case INDEX_op_neg_i64:
1604 return TCG_TARGET_HAS_neg_i64;
1605 case INDEX_op_andc_i64:
1606 return TCG_TARGET_HAS_andc_i64;
1607 case INDEX_op_orc_i64:
1608 return TCG_TARGET_HAS_orc_i64;
1609 case INDEX_op_eqv_i64:
1610 return TCG_TARGET_HAS_eqv_i64;
1611 case INDEX_op_nand_i64:
1612 return TCG_TARGET_HAS_nand_i64;
1613 case INDEX_op_nor_i64:
1614 return TCG_TARGET_HAS_nor_i64;
1615 case INDEX_op_clz_i64:
1616 return TCG_TARGET_HAS_clz_i64;
1617 case INDEX_op_ctz_i64:
1618 return TCG_TARGET_HAS_ctz_i64;
1619 case INDEX_op_ctpop_i64:
1620 return TCG_TARGET_HAS_ctpop_i64;
1621 case INDEX_op_add2_i64:
1622 return TCG_TARGET_HAS_add2_i64;
1623 case INDEX_op_sub2_i64:
1624 return TCG_TARGET_HAS_sub2_i64;
1625 case INDEX_op_mulu2_i64:
1626 return TCG_TARGET_HAS_mulu2_i64;
1627 case INDEX_op_muls2_i64:
1628 return TCG_TARGET_HAS_muls2_i64;
1629 case INDEX_op_muluh_i64:
1630 return TCG_TARGET_HAS_muluh_i64;
1631 case INDEX_op_mulsh_i64:
1632 return TCG_TARGET_HAS_mulsh_i64;
1633
1634 case INDEX_op_mov_vec:
1635 case INDEX_op_dup_vec:
1636 case INDEX_op_dupi_vec:
1637 case INDEX_op_dupm_vec:
1638 case INDEX_op_ld_vec:
1639 case INDEX_op_st_vec:
1640 case INDEX_op_add_vec:
1641 case INDEX_op_sub_vec:
1642 case INDEX_op_and_vec:
1643 case INDEX_op_or_vec:
1644 case INDEX_op_xor_vec:
1645 case INDEX_op_cmp_vec:
1646 return have_vec;
1647 case INDEX_op_dup2_vec:
1648 return have_vec && TCG_TARGET_REG_BITS == 32;
1649 case INDEX_op_not_vec:
1650 return have_vec && TCG_TARGET_HAS_not_vec;
1651 case INDEX_op_neg_vec:
1652 return have_vec && TCG_TARGET_HAS_neg_vec;
1653 case INDEX_op_abs_vec:
1654 return have_vec && TCG_TARGET_HAS_abs_vec;
1655 case INDEX_op_andc_vec:
1656 return have_vec && TCG_TARGET_HAS_andc_vec;
1657 case INDEX_op_orc_vec:
1658 return have_vec && TCG_TARGET_HAS_orc_vec;
1659 case INDEX_op_mul_vec:
1660 return have_vec && TCG_TARGET_HAS_mul_vec;
1661 case INDEX_op_shli_vec:
1662 case INDEX_op_shri_vec:
1663 case INDEX_op_sari_vec:
1664 return have_vec && TCG_TARGET_HAS_shi_vec;
1665 case INDEX_op_shls_vec:
1666 case INDEX_op_shrs_vec:
1667 case INDEX_op_sars_vec:
1668 return have_vec && TCG_TARGET_HAS_shs_vec;
1669 case INDEX_op_shlv_vec:
1670 case INDEX_op_shrv_vec:
1671 case INDEX_op_sarv_vec:
1672 return have_vec && TCG_TARGET_HAS_shv_vec;
1673 case INDEX_op_rotli_vec:
1674 return have_vec && TCG_TARGET_HAS_roti_vec;
1675 case INDEX_op_rotls_vec:
1676 return have_vec && TCG_TARGET_HAS_rots_vec;
1677 case INDEX_op_rotlv_vec:
1678 case INDEX_op_rotrv_vec:
1679 return have_vec && TCG_TARGET_HAS_rotv_vec;
1680 case INDEX_op_ssadd_vec:
1681 case INDEX_op_usadd_vec:
1682 case INDEX_op_sssub_vec:
1683 case INDEX_op_ussub_vec:
1684 return have_vec && TCG_TARGET_HAS_sat_vec;
1685 case INDEX_op_smin_vec:
1686 case INDEX_op_umin_vec:
1687 case INDEX_op_smax_vec:
1688 case INDEX_op_umax_vec:
1689 return have_vec && TCG_TARGET_HAS_minmax_vec;
1690 case INDEX_op_bitsel_vec:
1691 return have_vec && TCG_TARGET_HAS_bitsel_vec;
1692 case INDEX_op_cmpsel_vec:
1693 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
1694
1695 default:
1696 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1697 return true;
1698 }
1699 }
1700
1701 /* Note: we convert the 64 bit args to 32 bit and do some alignment
1702 and endian swap. Maybe it would be better to do the alignment
1703 and endian swap in tcg_reg_alloc_call(). */
1704 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
1705 {
1706 int i, real_args, nb_rets, pi;
1707 unsigned sizemask, flags;
1708 TCGHelperInfo *info;
1709 TCGOp *op;
1710
1711 info = g_hash_table_lookup(helper_table, (gpointer)func);
1712 flags = info->flags;
1713 sizemask = info->sizemask;
1714
1715 #ifdef CONFIG_PLUGIN
1716 /* detect non-plugin helpers */
1717 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1718 tcg_ctx->plugin_insn->calls_helpers = true;
1719 }
1720 #endif
1721
1722 #if defined(__sparc__) && !defined(__arch64__) \
1723 && !defined(CONFIG_TCG_INTERPRETER)
1724 /* We have 64-bit values in one register, but need to pass as two
1725 separate parameters. Split them. */
1726 int orig_sizemask = sizemask;
1727 int orig_nargs = nargs;
1728 TCGv_i64 retl, reth;
1729 TCGTemp *split_args[MAX_OPC_PARAM];
1730
1731 retl = NULL;
1732 reth = NULL;
1733 if (sizemask != 0) {
1734 for (i = real_args = 0; i < nargs; ++i) {
1735 int is_64bit = sizemask & (1 << (i+1)*2);
1736 if (is_64bit) {
1737 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1738 TCGv_i32 h = tcg_temp_new_i32();
1739 TCGv_i32 l = tcg_temp_new_i32();
1740 tcg_gen_extr_i64_i32(l, h, orig);
1741 split_args[real_args++] = tcgv_i32_temp(h);
1742 split_args[real_args++] = tcgv_i32_temp(l);
1743 } else {
1744 split_args[real_args++] = args[i];
1745 }
1746 }
1747 nargs = real_args;
1748 args = split_args;
1749 sizemask = 0;
1750 }
1751 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1752 for (i = 0; i < nargs; ++i) {
1753 int is_64bit = sizemask & (1 << (i+1)*2);
1754 int is_signed = sizemask & (2 << (i+1)*2);
1755 if (!is_64bit) {
1756 TCGv_i64 temp = tcg_temp_new_i64();
1757 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1758 if (is_signed) {
1759 tcg_gen_ext32s_i64(temp, orig);
1760 } else {
1761 tcg_gen_ext32u_i64(temp, orig);
1762 }
1763 args[i] = tcgv_i64_temp(temp);
1764 }
1765 }
1766 #endif /* TCG_TARGET_EXTEND_ARGS */
1767
1768 op = tcg_emit_op(INDEX_op_call);
1769
1770 pi = 0;
1771 if (ret != NULL) {
1772 #if defined(__sparc__) && !defined(__arch64__) \
1773 && !defined(CONFIG_TCG_INTERPRETER)
1774 if (orig_sizemask & 1) {
1775 /* The 32-bit ABI is going to return the 64-bit value in
1776 the %o0/%o1 register pair. Prepare for this by using
1777 two return temporaries, and reassemble below. */
1778 retl = tcg_temp_new_i64();
1779 reth = tcg_temp_new_i64();
1780 op->args[pi++] = tcgv_i64_arg(reth);
1781 op->args[pi++] = tcgv_i64_arg(retl);
1782 nb_rets = 2;
1783 } else {
1784 op->args[pi++] = temp_arg(ret);
1785 nb_rets = 1;
1786 }
1787 #else
1788 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) {
1789 #ifdef HOST_WORDS_BIGENDIAN
1790 op->args[pi++] = temp_arg(ret + 1);
1791 op->args[pi++] = temp_arg(ret);
1792 #else
1793 op->args[pi++] = temp_arg(ret);
1794 op->args[pi++] = temp_arg(ret + 1);
1795 #endif
1796 nb_rets = 2;
1797 } else {
1798 op->args[pi++] = temp_arg(ret);
1799 nb_rets = 1;
1800 }
1801 #endif
1802 } else {
1803 nb_rets = 0;
1804 }
1805 TCGOP_CALLO(op) = nb_rets;
1806
1807 real_args = 0;
1808 for (i = 0; i < nargs; i++) {
1809 int is_64bit = sizemask & (1 << (i+1)*2);
1810 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
1811 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
1812 /* some targets want aligned 64 bit args */
1813 if (real_args & 1) {
1814 op->args[pi++] = TCG_CALL_DUMMY_ARG;
1815 real_args++;
1816 }
1817 #endif
1818 /* If stack grows up, then we will be placing successive
1819 arguments at lower addresses, which means we need to
1820 reverse the order compared to how we would normally
1821 treat either big or little-endian. For those arguments
1822 that will wind up in registers, this still works for
1823 HPPA (the only current STACK_GROWSUP target) since the
1824 argument registers are *also* allocated in decreasing
1825 order. If another such target is added, this logic may
1826 have to get more complicated to differentiate between
1827 stack arguments and register arguments. */
1828 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
1829 op->args[pi++] = temp_arg(args[i] + 1);
1830 op->args[pi++] = temp_arg(args[i]);
1831 #else
1832 op->args[pi++] = temp_arg(args[i]);
1833 op->args[pi++] = temp_arg(args[i] + 1);
1834 #endif
1835 real_args += 2;
1836 continue;
1837 }
1838
1839 op->args[pi++] = temp_arg(args[i]);
1840 real_args++;
1841 }
1842 op->args[pi++] = (uintptr_t)func;
1843 op->args[pi++] = flags;
1844 TCGOP_CALLI(op) = real_args;
1845
1846 /* Make sure the fields didn't overflow. */
1847 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
1848 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
1849
1850 #if defined(__sparc__) && !defined(__arch64__) \
1851 && !defined(CONFIG_TCG_INTERPRETER)
1852 /* Free all of the parts we allocated above. */
1853 for (i = real_args = 0; i < orig_nargs; ++i) {
1854 int is_64bit = orig_sizemask & (1 << (i+1)*2);
1855 if (is_64bit) {
1856 tcg_temp_free_internal(args[real_args++]);
1857 tcg_temp_free_internal(args[real_args++]);
1858 } else {
1859 real_args++;
1860 }
1861 }
1862 if (orig_sizemask & 1) {
1863 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1864 Note that describing these as TCGv_i64 eliminates an unnecessary
1865 zero-extension that tcg_gen_concat_i32_i64 would create. */
1866 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth);
1867 tcg_temp_free_i64(retl);
1868 tcg_temp_free_i64(reth);
1869 }
1870 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
1871 for (i = 0; i < nargs; ++i) {
1872 int is_64bit = sizemask & (1 << (i+1)*2);
1873 if (!is_64bit) {
1874 tcg_temp_free_internal(args[i]);
1875 }
1876 }
1877 #endif /* TCG_TARGET_EXTEND_ARGS */
1878 }
1879
1880 static void tcg_reg_alloc_start(TCGContext *s)
1881 {
1882 int i, n;
1883 TCGTemp *ts;
1884
1885 for (i = 0, n = s->nb_globals; i < n; i++) {
1886 ts = &s->temps[i];
1887 ts->val_type = (ts->fixed_reg ? TEMP_VAL_REG : TEMP_VAL_MEM);
1888 }
1889 for (n = s->nb_temps; i < n; i++) {
1890 ts = &s->temps[i];
1891 ts->val_type = (ts->temp_local ? TEMP_VAL_MEM : TEMP_VAL_DEAD);
1892 ts->mem_allocated = 0;
1893 ts->fixed_reg = 0;
1894 }
1895
1896 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
1897 }
1898
1899 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1900 TCGTemp *ts)
1901 {
1902 int idx = temp_idx(ts);
1903
1904 if (ts->temp_global) {
1905 pstrcpy(buf, buf_size, ts->name);
1906 } else if (ts->temp_local) {
1907 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
1908 } else {
1909 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
1910 }
1911 return buf;
1912 }
1913
1914 static char *tcg_get_arg_str(TCGContext *s, char *buf,
1915 int buf_size, TCGArg arg)
1916 {
1917 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
1918 }
1919
1920 /* Find helper name. */
1921 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val)
1922 {
1923 const char *ret = NULL;
1924 if (helper_table) {
1925 TCGHelperInfo *info = g_hash_table_lookup(helper_table, (gpointer)val);
1926 if (info) {
1927 ret = info->name;
1928 }
1929 }
1930 return ret;
1931 }
1932
1933 static const char * const cond_name[] =
1934 {
1935 [TCG_COND_NEVER] = "never",
1936 [TCG_COND_ALWAYS] = "always",
1937 [TCG_COND_EQ] = "eq",
1938 [TCG_COND_NE] = "ne",
1939 [TCG_COND_LT] = "lt",
1940 [TCG_COND_GE] = "ge",
1941 [TCG_COND_LE] = "le",
1942 [TCG_COND_GT] = "gt",
1943 [TCG_COND_LTU] = "ltu",
1944 [TCG_COND_GEU] = "geu",
1945 [TCG_COND_LEU] = "leu",
1946 [TCG_COND_GTU] = "gtu"
1947 };
1948
1949 static const char * const ldst_name[] =
1950 {
1951 [MO_UB] = "ub",
1952 [MO_SB] = "sb",
1953 [MO_LEUW] = "leuw",
1954 [MO_LESW] = "lesw",
1955 [MO_LEUL] = "leul",
1956 [MO_LESL] = "lesl",
1957 [MO_LEQ] = "leq",
1958 [MO_BEUW] = "beuw",
1959 [MO_BESW] = "besw",
1960 [MO_BEUL] = "beul",
1961 [MO_BESL] = "besl",
1962 [MO_BEQ] = "beq",
1963 };
1964
1965 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
1966 #ifdef TARGET_ALIGNED_ONLY
1967 [MO_UNALN >> MO_ASHIFT] = "un+",
1968 [MO_ALIGN >> MO_ASHIFT] = "",
1969 #else
1970 [MO_UNALN >> MO_ASHIFT] = "",
1971 [MO_ALIGN >> MO_ASHIFT] = "al+",
1972 #endif
1973 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1974 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1975 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1976 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1977 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1978 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1979 };
1980
1981 static inline bool tcg_regset_single(TCGRegSet d)
1982 {
1983 return (d & (d - 1)) == 0;
1984 }
1985
1986 static inline TCGReg tcg_regset_first(TCGRegSet d)
1987 {
1988 if (TCG_TARGET_NB_REGS <= 32) {
1989 return ctz32(d);
1990 } else {
1991 return ctz64(d);
1992 }
1993 }
1994
1995 static void tcg_dump_ops(TCGContext *s, bool have_prefs)
1996 {
1997 char buf[128];
1998 TCGOp *op;
1999
2000 QTAILQ_FOREACH(op, &s->ops, link) {
2001 int i, k, nb_oargs, nb_iargs, nb_cargs;
2002 const TCGOpDef *def;
2003 TCGOpcode c;
2004 int col = 0;
2005
2006 c = op->opc;
2007 def = &tcg_op_defs[c];
2008
2009 if (c == INDEX_op_insn_start) {
2010 nb_oargs = 0;
2011 col += qemu_log("\n ----");
2012
2013 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
2014 target_ulong a;
2015 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
2016 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
2017 #else
2018 a = op->args[i];
2019 #endif
2020 col += qemu_log(" " TARGET_FMT_lx, a);
2021 }
2022 } else if (c == INDEX_op_call) {
2023 /* variable number of arguments */
2024 nb_oargs = TCGOP_CALLO(op);
2025 nb_iargs = TCGOP_CALLI(op);
2026 nb_cargs = def->nb_cargs;
2027
2028 /* function name, flags, out args */
2029 col += qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name,
2030 tcg_find_helper(s, op->args[nb_oargs + nb_iargs]),
2031 op->args[nb_oargs + nb_iargs + 1], nb_oargs);
2032 for (i = 0; i < nb_oargs; i++) {
2033 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2034 op->args[i]));
2035 }
2036 for (i = 0; i < nb_iargs; i++) {
2037 TCGArg arg = op->args[nb_oargs + i];
2038 const char *t = "<dummy>";
2039 if (arg != TCG_CALL_DUMMY_ARG) {
2040 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
2041 }
2042 col += qemu_log(",%s", t);
2043 }
2044 } else {
2045 col += qemu_log(" %s ", def->name);
2046
2047 nb_oargs = def->nb_oargs;
2048 nb_iargs = def->nb_iargs;
2049 nb_cargs = def->nb_cargs;
2050
2051 if (def->flags & TCG_OPF_VECTOR) {
2052 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op),
2053 8 << TCGOP_VECE(op));
2054 }
2055
2056 k = 0;
2057 for (i = 0; i < nb_oargs; i++) {
2058 if (k != 0) {
2059 col += qemu_log(",");
2060 }
2061 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
2062 op->args[k++]));
2063 }
2064 for (i = 0; i < nb_iargs; i++) {
2065 if (k != 0) {
2066 col += qemu_log(",");
2067 }
2068 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
2069 op->args[k++]));
2070 }
2071 switch (c) {
2072 case INDEX_op_brcond_i32:
2073 case INDEX_op_setcond_i32:
2074 case INDEX_op_movcond_i32:
2075 case INDEX_op_brcond2_i32:
2076 case INDEX_op_setcond2_i32:
2077 case INDEX_op_brcond_i64:
2078 case INDEX_op_setcond_i64:
2079 case INDEX_op_movcond_i64:
2080 case INDEX_op_cmp_vec:
2081 case INDEX_op_cmpsel_vec:
2082 if (op->args[k] < ARRAY_SIZE(cond_name)
2083 && cond_name[op->args[k]]) {
2084 col += qemu_log(",%s", cond_name[op->args[k++]]);
2085 } else {
2086 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]);
2087 }
2088 i = 1;
2089 break;
2090 case INDEX_op_qemu_ld_i32:
2091 case INDEX_op_qemu_st_i32:
2092 case INDEX_op_qemu_ld_i64:
2093 case INDEX_op_qemu_st_i64:
2094 {
2095 TCGMemOpIdx oi = op->args[k++];
2096 MemOp op = get_memop(oi);
2097 unsigned ix = get_mmuidx(oi);
2098
2099 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
2100 col += qemu_log(",$0x%x,%u", op, ix);
2101 } else {
2102 const char *s_al, *s_op;
2103 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
2104 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
2105 col += qemu_log(",%s%s,%u", s_al, s_op, ix);
2106 }
2107 i = 1;
2108 }
2109 break;
2110 default:
2111 i = 0;
2112 break;
2113 }
2114 switch (c) {
2115 case INDEX_op_set_label:
2116 case INDEX_op_br:
2117 case INDEX_op_brcond_i32:
2118 case INDEX_op_brcond_i64:
2119 case INDEX_op_brcond2_i32:
2120 col += qemu_log("%s$L%d", k ? "," : "",
2121 arg_label(op->args[k])->id);
2122 i++, k++;
2123 break;
2124 default:
2125 break;
2126 }
2127 for (; i < nb_cargs; i++, k++) {
2128 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]);
2129 }
2130 }
2131
2132 if (have_prefs || op->life) {
2133
2134 QemuLogFile *logfile;
2135
2136 rcu_read_lock();
2137 logfile = atomic_rcu_read(&qemu_logfile);
2138 if (logfile) {
2139 for (; col < 40; ++col) {
2140 putc(' ', logfile->fd);
2141 }
2142 }
2143 rcu_read_unlock();
2144 }
2145
2146 if (op->life) {
2147 unsigned life = op->life;
2148
2149 if (life & (SYNC_ARG * 3)) {
2150 qemu_log(" sync:");
2151 for (i = 0; i < 2; ++i) {
2152 if (life & (SYNC_ARG << i)) {
2153 qemu_log(" %d", i);
2154 }
2155 }
2156 }
2157 life /= DEAD_ARG;
2158 if (life) {
2159 qemu_log(" dead:");
2160 for (i = 0; life; ++i, life >>= 1) {
2161 if (life & 1) {
2162 qemu_log(" %d", i);
2163 }
2164 }
2165 }
2166 }
2167
2168 if (have_prefs) {
2169 for (i = 0; i < nb_oargs; ++i) {
2170 TCGRegSet set = op->output_pref[i];
2171
2172 if (i == 0) {
2173 qemu_log(" pref=");
2174 } else {
2175 qemu_log(",");
2176 }
2177 if (set == 0) {
2178 qemu_log("none");
2179 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
2180 qemu_log("all");
2181 #ifdef CONFIG_DEBUG_TCG
2182 } else if (tcg_regset_single(set)) {
2183 TCGReg reg = tcg_regset_first(set);
2184 qemu_log("%s", tcg_target_reg_names[reg]);
2185 #endif
2186 } else if (TCG_TARGET_NB_REGS <= 32) {
2187 qemu_log("%#x", (uint32_t)set);
2188 } else {
2189 qemu_log("%#" PRIx64, (uint64_t)set);
2190 }
2191 }
2192 }
2193
2194 qemu_log("\n");
2195 }
2196 }
2197
2198 /* we give more priority to constraints with less registers */
2199 static int get_constraint_priority(const TCGOpDef *def, int k)
2200 {
2201 const TCGArgConstraint *arg_ct;
2202
2203 int i, n;
2204 arg_ct = &def->args_ct[k];
2205 if (arg_ct->ct & TCG_CT_ALIAS) {
2206 /* an alias is equivalent to a single register */
2207 n = 1;
2208 } else {
2209 if (!(arg_ct->ct & TCG_CT_REG))
2210 return 0;
2211 n = 0;
2212 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
2213 if (tcg_regset_test_reg(arg_ct->u.regs, i))
2214 n++;
2215 }
2216 }
2217 return TCG_TARGET_NB_REGS - n + 1;
2218 }
2219
2220 /* sort from highest priority to lowest */
2221 static void sort_constraints(TCGOpDef *def, int start, int n)
2222 {
2223 int i, j, p1, p2, tmp;
2224
2225 for(i = 0; i < n; i++)
2226 def->sorted_args[start + i] = start + i;
2227 if (n <= 1)
2228 return;
2229 for(i = 0; i < n - 1; i++) {
2230 for(j = i + 1; j < n; j++) {
2231 p1 = get_constraint_priority(def, def->sorted_args[start + i]);
2232 p2 = get_constraint_priority(def, def->sorted_args[start + j]);
2233 if (p1 < p2) {
2234 tmp = def->sorted_args[start + i];
2235 def->sorted_args[start + i] = def->sorted_args[start + j];
2236 def->sorted_args[start + j] = tmp;
2237 }
2238 }
2239 }
2240 }
2241
2242 static void process_op_defs(TCGContext *s)
2243 {
2244 TCGOpcode op;
2245
2246 for (op = 0; op < NB_OPS; op++) {
2247 TCGOpDef *def = &tcg_op_defs[op];
2248 const TCGTargetOpDef *tdefs;
2249 TCGType type;
2250 int i, nb_args;
2251
2252 if (def->flags & TCG_OPF_NOT_PRESENT) {
2253 continue;
2254 }
2255
2256 nb_args = def->nb_iargs + def->nb_oargs;
2257 if (nb_args == 0) {
2258 continue;
2259 }
2260
2261 tdefs = tcg_target_op_def(op);
2262 /* Missing TCGTargetOpDef entry. */
2263 tcg_debug_assert(tdefs != NULL);
2264
2265 type = (def->flags & TCG_OPF_64BIT ? TCG_TYPE_I64 : TCG_TYPE_I32);
2266 for (i = 0; i < nb_args; i++) {
2267 const char *ct_str = tdefs->args_ct_str[i];
2268 /* Incomplete TCGTargetOpDef entry. */
2269 tcg_debug_assert(ct_str != NULL);
2270
2271 def->args_ct[i].u.regs = 0;
2272 def->args_ct[i].ct = 0;
2273 while (*ct_str != '\0') {
2274 switch(*ct_str) {
2275 case '0' ... '9':
2276 {
2277 int oarg = *ct_str - '0';
2278 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2279 tcg_debug_assert(oarg < def->nb_oargs);
2280 tcg_debug_assert(def->args_ct[oarg].ct & TCG_CT_REG);
2281 /* TCG_CT_ALIAS is for the output arguments.
2282 The input is tagged with TCG_CT_IALIAS. */
2283 def->args_ct[i] = def->args_ct[oarg];
2284 def->args_ct[oarg].ct |= TCG_CT_ALIAS;
2285 def->args_ct[oarg].alias_index = i;
2286 def->args_ct[i].ct |= TCG_CT_IALIAS;
2287 def->args_ct[i].alias_index = oarg;
2288 }
2289 ct_str++;
2290 break;
2291 case '&':
2292 def->args_ct[i].ct |= TCG_CT_NEWREG;
2293 ct_str++;
2294 break;
2295 case 'i':
2296 def->args_ct[i].ct |= TCG_CT_CONST;
2297 ct_str++;
2298 break;
2299 default:
2300 ct_str = target_parse_constraint(&def->args_ct[i],
2301 ct_str, type);
2302 /* Typo in TCGTargetOpDef constraint. */
2303 tcg_debug_assert(ct_str != NULL);
2304 }
2305 }
2306 }
2307
2308 /* TCGTargetOpDef entry with too much information? */
2309 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
2310
2311 /* sort the constraints (XXX: this is just an heuristic) */
2312 sort_constraints(def, 0, def->nb_oargs);
2313 sort_constraints(def, def->nb_oargs, def->nb_iargs);
2314 }
2315 }
2316
2317 void tcg_op_remove(TCGContext *s, TCGOp *op)
2318 {
2319 TCGLabel *label;
2320
2321 switch (op->opc) {
2322 case INDEX_op_br:
2323 label = arg_label(op->args[0]);
2324 label->refs--;
2325 break;
2326 case INDEX_op_brcond_i32:
2327 case INDEX_op_brcond_i64:
2328 label = arg_label(op->args[3]);
2329 label->refs--;
2330 break;
2331 case INDEX_op_brcond2_i32:
2332 label = arg_label(op->args[5]);
2333 label->refs--;
2334 break;
2335 default:
2336 break;
2337 }
2338
2339 QTAILQ_REMOVE(&s->ops, op, link);
2340 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
2341 s->nb_ops--;
2342
2343 #ifdef CONFIG_PROFILER
2344 atomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
2345 #endif
2346 }
2347
2348 static TCGOp *tcg_op_alloc(TCGOpcode opc)
2349 {
2350 TCGContext *s = tcg_ctx;
2351 TCGOp *op;
2352
2353 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2354 op = tcg_malloc(sizeof(TCGOp));
2355 } else {
2356 op = QTAILQ_FIRST(&s->free_ops);
2357 QTAILQ_REMOVE(&s->free_ops, op, link);
2358 }
2359 memset(op, 0, offsetof(TCGOp, link));
2360 op->opc = opc;
2361 s->nb_ops++;
2362
2363 return op;
2364 }
2365
2366 TCGOp *tcg_emit_op(TCGOpcode opc)
2367 {
2368 TCGOp *op = tcg_op_alloc(opc);
2369 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2370 return op;
2371 }
2372
2373 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
2374 {
2375 TCGOp *new_op = tcg_op_alloc(opc);
2376 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
2377 return new_op;
2378 }
2379
2380 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
2381 {
2382 TCGOp *new_op = tcg_op_alloc(opc);
2383 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
2384 return new_op;
2385 }
2386
2387 /* Reachable analysis : remove unreachable code. */
2388 static void reachable_code_pass(TCGContext *s)
2389 {
2390 TCGOp *op, *op_next;
2391 bool dead = false;
2392
2393 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2394 bool remove = dead;
2395 TCGLabel *label;
2396 int call_flags;
2397
2398 switch (op->opc) {
2399 case INDEX_op_set_label:
2400 label = arg_label(op->args[0]);
2401 if (label->refs == 0) {
2402 /*
2403 * While there is an occasional backward branch, virtually
2404 * all branches generated by the translators are forward.
2405 * Which means that generally we will have already removed
2406 * all references to the label that will be, and there is
2407 * little to be gained by iterating.
2408 */
2409 remove = true;
2410 } else {
2411 /* Once we see a label, insns become live again. */
2412 dead = false;
2413 remove = false;
2414
2415 /*
2416 * Optimization can fold conditional branches to unconditional.
2417 * If we find a label with one reference which is preceded by
2418 * an unconditional branch to it, remove both. This needed to
2419 * wait until the dead code in between them was removed.
2420 */
2421 if (label->refs == 1) {
2422 TCGOp *op_prev = QTAILQ_PREV(op, link);
2423 if (op_prev->opc == INDEX_op_br &&
2424 label == arg_label(op_prev->args[0])) {
2425 tcg_op_remove(s, op_prev);
2426 remove = true;
2427 }
2428 }
2429 }
2430 break;
2431
2432 case INDEX_op_br:
2433 case INDEX_op_exit_tb:
2434 case INDEX_op_goto_ptr:
2435 /* Unconditional branches; everything following is dead. */
2436 dead = true;
2437 break;
2438
2439 case INDEX_op_call:
2440 /* Notice noreturn helper calls, raising exceptions. */
2441 call_flags = op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op) + 1];
2442 if (call_flags & TCG_CALL_NO_RETURN) {
2443 dead = true;
2444 }
2445 break;
2446
2447 case INDEX_op_insn_start:
2448 /* Never remove -- we need to keep these for unwind. */
2449 remove = false;
2450 break;
2451
2452 default:
2453 break;
2454 }
2455
2456 if (remove) {
2457 tcg_op_remove(s, op);
2458 }
2459 }
2460 }
2461
2462 #define TS_DEAD 1
2463 #define TS_MEM 2
2464
2465 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2466 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2467
2468 /* For liveness_pass_1, the register preferences for a given temp. */
2469 static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2470 {
2471 return ts->state_ptr;
2472 }
2473
2474 /* For liveness_pass_1, reset the preferences for a given temp to the
2475 * maximal regset for its type.
2476 */
2477 static inline void la_reset_pref(TCGTemp *ts)
2478 {
2479 *la_temp_pref(ts)
2480 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2481 }
2482
2483 /* liveness analysis: end of function: all temps are dead, and globals
2484 should be in memory. */
2485 static void la_func_end(TCGContext *s, int ng, int nt)
2486 {
2487 int i;
2488
2489 for (i = 0; i < ng; ++i) {
2490 s->temps[i].state = TS_DEAD | TS_MEM;
2491 la_reset_pref(&s->temps[i]);
2492 }
2493 for (i = ng; i < nt; ++i) {
2494 s->temps[i].state = TS_DEAD;
2495 la_reset_pref(&s->temps[i]);
2496 }
2497 }
2498
2499 /* liveness analysis: end of basic block: all temps are dead, globals
2500 and local temps should be in memory. */
2501 static void la_bb_end(TCGContext *s, int ng, int nt)
2502 {
2503 int i;
2504
2505 for (i = 0; i < ng; ++i) {
2506 s->temps[i].state = TS_DEAD | TS_MEM;
2507 la_reset_pref(&s->temps[i]);
2508 }
2509 for (i = ng; i < nt; ++i) {
2510 s->temps[i].state = (s->temps[i].temp_local
2511 ? TS_DEAD | TS_MEM
2512 : TS_DEAD);
2513 la_reset_pref(&s->temps[i]);
2514 }
2515 }
2516
2517 /* liveness analysis: sync globals back to memory. */
2518 static void la_global_sync(TCGContext *s, int ng)
2519 {
2520 int i;
2521
2522 for (i = 0; i < ng; ++i) {
2523 int state = s->temps[i].state;
2524 s->temps[i].state = state | TS_MEM;
2525 if (state == TS_DEAD) {
2526 /* If the global was previously dead, reset prefs. */
2527 la_reset_pref(&s->temps[i]);
2528 }
2529 }
2530 }
2531
2532 /* liveness analysis: sync globals back to memory and kill. */
2533 static void la_global_kill(TCGContext *s, int ng)
2534 {
2535 int i;
2536
2537 for (i = 0; i < ng; i++) {
2538 s->temps[i].state = TS_DEAD | TS_MEM;
2539 la_reset_pref(&s->temps[i]);
2540 }
2541 }
2542
2543 /* liveness analysis: note live globals crossing calls. */
2544 static void la_cross_call(TCGContext *s, int nt)
2545 {
2546 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2547 int i;
2548
2549 for (i = 0; i < nt; i++) {
2550 TCGTemp *ts = &s->temps[i];
2551 if (!(ts->state & TS_DEAD)) {
2552 TCGRegSet *pset = la_temp_pref(ts);
2553 TCGRegSet set = *pset;
2554
2555 set &= mask;
2556 /* If the combination is not possible, restart. */
2557 if (set == 0) {
2558 set = tcg_target_available_regs[ts->type] & mask;
2559 }
2560 *pset = set;
2561 }
2562 }
2563 }
2564
2565 /* Liveness analysis : update the opc_arg_life array to tell if a
2566 given input arguments is dead. Instructions updating dead
2567 temporaries are removed. */
2568 static void liveness_pass_1(TCGContext *s)
2569 {
2570 int nb_globals = s->nb_globals;
2571 int nb_temps = s->nb_temps;
2572 TCGOp *op, *op_prev;
2573 TCGRegSet *prefs;
2574 int i;
2575
2576 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2577 for (i = 0; i < nb_temps; ++i) {
2578 s->temps[i].state_ptr = prefs + i;
2579 }
2580
2581 /* ??? Should be redundant with the exit_tb that ends the TB. */
2582 la_func_end(s, nb_globals, nb_temps);
2583
2584 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
2585 int nb_iargs, nb_oargs;
2586 TCGOpcode opc_new, opc_new2;
2587 bool have_opc_new2;
2588 TCGLifeData arg_life = 0;
2589 TCGTemp *ts;
2590 TCGOpcode opc = op->opc;
2591 const TCGOpDef *def = &tcg_op_defs[opc];
2592
2593 switch (opc) {
2594 case INDEX_op_call:
2595 {
2596 int call_flags;
2597 int nb_call_regs;
2598
2599 nb_oargs = TCGOP_CALLO(op);
2600 nb_iargs = TCGOP_CALLI(op);
2601 call_flags = op->args[nb_oargs + nb_iargs + 1];
2602
2603 /* pure functions can be removed if their result is unused */
2604 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
2605 for (i = 0; i < nb_oargs; i++) {
2606 ts = arg_temp(op->args[i]);
2607 if (ts->state != TS_DEAD) {
2608 goto do_not_remove_call;
2609 }
2610 }
2611 goto do_remove;
2612 }
2613 do_not_remove_call:
2614
2615 /* Output args are dead. */
2616 for (i = 0; i < nb_oargs; i++) {
2617 ts = arg_temp(op->args[i]);
2618 if (ts->state & TS_DEAD) {
2619 arg_life |= DEAD_ARG << i;
2620 }
2621 if (ts->state & TS_MEM) {
2622 arg_life |= SYNC_ARG << i;
2623 }
2624 ts->state = TS_DEAD;
2625 la_reset_pref(ts);
2626
2627 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2628 op->output_pref[i] = 0;
2629 }
2630
2631 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2632 TCG_CALL_NO_READ_GLOBALS))) {
2633 la_global_kill(s, nb_globals);
2634 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
2635 la_global_sync(s, nb_globals);
2636 }
2637
2638 /* Record arguments that die in this helper. */
2639 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2640 ts = arg_temp(op->args[i]);
2641 if (ts && ts->state & TS_DEAD) {
2642 arg_life |= DEAD_ARG << i;
2643 }
2644 }
2645
2646 /* For all live registers, remove call-clobbered prefs. */
2647 la_cross_call(s, nb_temps);
2648
2649 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2650
2651 /* Input arguments are live for preceding opcodes. */
2652 for (i = 0; i < nb_iargs; i++) {
2653 ts = arg_temp(op->args[i + nb_oargs]);
2654 if (ts && ts->state & TS_DEAD) {
2655 /* For those arguments that die, and will be allocated
2656 * in registers, clear the register set for that arg,
2657 * to be filled in below. For args that will be on
2658 * the stack, reset to any available reg.
2659 */
2660 *la_temp_pref(ts)
2661 = (i < nb_call_regs ? 0 :
2662 tcg_target_available_regs[ts->type]);
2663 ts->state &= ~TS_DEAD;
2664 }
2665 }
2666
2667 /* For each input argument, add its input register to prefs.
2668 If a temp is used once, this produces a single set bit. */
2669 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2670 ts = arg_temp(op->args[i + nb_oargs]);
2671 if (ts) {
2672 tcg_regset_set_reg(*la_temp_pref(ts),
2673 tcg_target_call_iarg_regs[i]);
2674 }
2675 }
2676 }
2677 break;
2678 case INDEX_op_insn_start:
2679 break;
2680 case INDEX_op_discard:
2681 /* mark the temporary as dead */
2682 ts = arg_temp(op->args[0]);
2683 ts->state = TS_DEAD;
2684 la_reset_pref(ts);
2685 break;
2686
2687 case INDEX_op_add2_i32:
2688 opc_new = INDEX_op_add_i32;
2689 goto do_addsub2;
2690 case INDEX_op_sub2_i32:
2691 opc_new = INDEX_op_sub_i32;
2692 goto do_addsub2;
2693 case INDEX_op_add2_i64:
2694 opc_new = INDEX_op_add_i64;
2695 goto do_addsub2;
2696 case INDEX_op_sub2_i64:
2697 opc_new = INDEX_op_sub_i64;
2698 do_addsub2:
2699 nb_iargs = 4;
2700 nb_oargs = 2;
2701 /* Test if the high part of the operation is dead, but not
2702 the low part. The result can be optimized to a simple
2703 add or sub. This happens often for x86_64 guest when the
2704 cpu mode is set to 32 bit. */
2705 if (arg_temp(op->args[1])->state == TS_DEAD) {
2706 if (arg_temp(op->args[0])->state == TS_DEAD) {
2707 goto do_remove;
2708 }
2709 /* Replace the opcode and adjust the args in place,
2710 leaving 3 unused args at the end. */
2711 op->opc = opc = opc_new;
2712 op->args[1] = op->args[2];
2713 op->args[2] = op->args[4];
2714 /* Fall through and mark the single-word operation live. */
2715 nb_iargs = 2;
2716 nb_oargs = 1;
2717 }
2718 goto do_not_remove;
2719
2720 case INDEX_op_mulu2_i32:
2721 opc_new = INDEX_op_mul_i32;
2722 opc_new2 = INDEX_op_muluh_i32;
2723 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
2724 goto do_mul2;
2725 case INDEX_op_muls2_i32:
2726 opc_new = INDEX_op_mul_i32;
2727 opc_new2 = INDEX_op_mulsh_i32;
2728 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
2729 goto do_mul2;
2730 case INDEX_op_mulu2_i64:
2731 opc_new = INDEX_op_mul_i64;
2732 opc_new2 = INDEX_op_muluh_i64;
2733 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
2734 goto do_mul2;
2735 case INDEX_op_muls2_i64:
2736 opc_new = INDEX_op_mul_i64;
2737 opc_new2 = INDEX_op_mulsh_i64;
2738 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
2739 goto do_mul2;
2740 do_mul2:
2741 nb_iargs = 2;
2742 nb_oargs = 2;
2743 if (arg_temp(op->args[1])->state == TS_DEAD) {
2744 if (arg_temp(op->args[0])->state == TS_DEAD) {
2745 /* Both parts of the operation are dead. */
2746 goto do_remove;
2747 }
2748 /* The high part of the operation is dead; generate the low. */
2749 op->opc = opc = opc_new;
2750 op->args[1] = op->args[2];
2751 op->args[2] = op->args[3];
2752 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
2753 /* The low part of the operation is dead; generate the high. */
2754 op->opc = opc = opc_new2;
2755 op->args[0] = op->args[1];
2756 op->args[1] = op->args[2];
2757 op->args[2] = op->args[3];
2758 } else {
2759 goto do_not_remove;
2760 }
2761 /* Mark the single-word operation live. */
2762 nb_oargs = 1;
2763 goto do_not_remove;
2764
2765 default:
2766 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
2767 nb_iargs = def->nb_iargs;
2768 nb_oargs = def->nb_oargs;
2769
2770 /* Test if the operation can be removed because all
2771 its outputs are dead. We assume that nb_oargs == 0
2772 implies side effects */
2773 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
2774 for (i = 0; i < nb_oargs; i++) {
2775 if (arg_temp(op->args[i])->state != TS_DEAD) {
2776 goto do_not_remove;
2777 }
2778 }
2779 goto do_remove;
2780 }
2781 goto do_not_remove;
2782
2783 do_remove:
2784 tcg_op_remove(s, op);
2785 break;
2786
2787 do_not_remove:
2788 for (i = 0; i < nb_oargs; i++) {
2789 ts = arg_temp(op->args[i]);
2790
2791 /* Remember the preference of the uses that followed. */
2792 op->output_pref[i] = *la_temp_pref(ts);
2793
2794 /* Output args are dead. */
2795 if (ts->state & TS_DEAD) {
2796 arg_life |= DEAD_ARG << i;
2797 }
2798 if (ts->state & TS_MEM) {
2799 arg_life |= SYNC_ARG << i;
2800 }
2801 ts->state = TS_DEAD;
2802 la_reset_pref(ts);
2803 }
2804
2805 /* If end of basic block, update. */
2806 if (def->flags & TCG_OPF_BB_EXIT) {
2807 la_func_end(s, nb_globals, nb_temps);
2808 } else if (def->flags & TCG_OPF_BB_END) {
2809 la_bb_end(s, nb_globals, nb_temps);
2810 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2811 la_global_sync(s, nb_globals);
2812 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2813 la_cross_call(s, nb_temps);
2814 }
2815 }
2816
2817 /* Record arguments that die in this opcode. */
2818 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2819 ts = arg_temp(op->args[i]);
2820 if (ts->state & TS_DEAD) {
2821 arg_life |= DEAD_ARG << i;
2822 }
2823 }
2824
2825 /* Input arguments are live for preceding opcodes. */
2826 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2827 ts = arg_temp(op->args[i]);
2828 if (ts->state & TS_DEAD) {
2829 /* For operands that were dead, initially allow
2830 all regs for the type. */
2831 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2832 ts->state &= ~TS_DEAD;
2833 }
2834 }
2835
2836 /* Incorporate constraints for this operand. */
2837 switch (opc) {
2838 case INDEX_op_mov_i32:
2839 case INDEX_op_mov_i64:
2840 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2841 have proper constraints. That said, special case
2842 moves to propagate preferences backward. */
2843 if (IS_DEAD_ARG(1)) {
2844 *la_temp_pref(arg_temp(op->args[0]))
2845 = *la_temp_pref(arg_temp(op->args[1]));
2846 }
2847 break;
2848
2849 default:
2850 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2851 const TCGArgConstraint *ct = &def->args_ct[i];
2852 TCGRegSet set, *pset;
2853
2854 ts = arg_temp(op->args[i]);
2855 pset = la_temp_pref(ts);
2856 set = *pset;
2857
2858 set &= ct->u.regs;
2859 if (ct->ct & TCG_CT_IALIAS) {
2860 set &= op->output_pref[ct->alias_index];
2861 }
2862 /* If the combination is not possible, restart. */
2863 if (set == 0) {
2864 set = ct->u.regs;
2865 }
2866 *pset = set;
2867 }
2868 break;
2869 }
2870 break;
2871 }
2872 op->life = arg_life;
2873 }
2874 }
2875
2876 /* Liveness analysis: Convert indirect regs to direct temporaries. */
2877 static bool liveness_pass_2(TCGContext *s)
2878 {
2879 int nb_globals = s->nb_globals;
2880 int nb_temps, i;
2881 bool changes = false;
2882 TCGOp *op, *op_next;
2883
2884 /* Create a temporary for each indirect global. */
2885 for (i = 0; i < nb_globals; ++i) {
2886 TCGTemp *its = &s->temps[i];
2887 if (its->indirect_reg) {
2888 TCGTemp *dts = tcg_temp_alloc(s);
2889 dts->type = its->type;
2890 dts->base_type = its->base_type;
2891 its->state_ptr = dts;
2892 } else {
2893 its->state_ptr = NULL;
2894 }
2895 /* All globals begin dead. */
2896 its->state = TS_DEAD;
2897 }
2898 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2899 TCGTemp *its = &s->temps[i];
2900 its->state_ptr = NULL;
2901 its->state = TS_DEAD;
2902 }
2903
2904 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2905 TCGOpcode opc = op->opc;
2906 const TCGOpDef *def = &tcg_op_defs[opc];
2907 TCGLifeData arg_life = op->life;
2908 int nb_iargs, nb_oargs, call_flags;
2909 TCGTemp *arg_ts, *dir_ts;
2910
2911 if (opc == INDEX_op_call) {
2912 nb_oargs = TCGOP_CALLO(op);
2913 nb_iargs = TCGOP_CALLI(op);
2914 call_flags = op->args[nb_oargs + nb_iargs + 1];
2915 } else {
2916 nb_iargs = def->nb_iargs;
2917 nb_oargs = def->nb_oargs;
2918
2919 /* Set flags similar to how calls require. */
2920 if (def->flags & TCG_OPF_BB_END) {
2921 /* Like writing globals: save_globals */
2922 call_flags = 0;
2923 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2924 /* Like reading globals: sync_globals */
2925 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2926 } else {
2927 /* No effect on globals. */
2928 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2929 TCG_CALL_NO_WRITE_GLOBALS);
2930 }
2931 }
2932
2933 /* Make sure that input arguments are available. */
2934 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2935 arg_ts = arg_temp(op->args[i]);
2936 if (arg_ts) {
2937 dir_ts = arg_ts->state_ptr;
2938 if (dir_ts && arg_ts->state == TS_DEAD) {
2939 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
2940 ? INDEX_op_ld_i32
2941 : INDEX_op_ld_i64);
2942 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
2943
2944 lop->args[0] = temp_arg(dir_ts);
2945 lop->args[1] = temp_arg(arg_ts->mem_base);
2946 lop->args[2] = arg_ts->mem_offset;
2947
2948 /* Loaded, but synced with memory. */
2949 arg_ts->state = TS_MEM;
2950 }
2951 }
2952 }
2953
2954 /* Perform input replacement, and mark inputs that became dead.
2955 No action is required except keeping temp_state up to date
2956 so that we reload when needed. */
2957 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2958 arg_ts = arg_temp(op->args[i]);
2959 if (arg_ts) {
2960 dir_ts = arg_ts->state_ptr;
2961 if (dir_ts) {
2962 op->args[i] = temp_arg(dir_ts);
2963 changes = true;
2964 if (IS_DEAD_ARG(i)) {
2965 arg_ts->state = TS_DEAD;
2966 }
2967 }
2968 }
2969 }
2970
2971 /* Liveness analysis should ensure that the following are
2972 all correct, for call sites and basic block end points. */
2973 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2974 /* Nothing to do */
2975 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2976 for (i = 0; i < nb_globals; ++i) {
2977 /* Liveness should see that globals are synced back,
2978 that is, either TS_DEAD or TS_MEM. */
2979 arg_ts = &s->temps[i];
2980 tcg_debug_assert(arg_ts->state_ptr == 0
2981 || arg_ts->state != 0);
2982 }
2983 } else {
2984 for (i = 0; i < nb_globals; ++i) {
2985 /* Liveness should see that globals are saved back,
2986 that is, TS_DEAD, waiting to be reloaded. */
2987 arg_ts = &s->temps[i];
2988 tcg_debug_assert(arg_ts->state_ptr == 0
2989 || arg_ts->state == TS_DEAD);
2990 }
2991 }
2992
2993 /* Outputs become available. */
2994 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2995 arg_ts = arg_temp(op->args[0]);
2996 dir_ts = arg_ts->state_ptr;
2997 if (dir_ts) {
2998 op->args[0] = temp_arg(dir_ts);
2999 changes = true;
3000
3001 /* The output is now live and modified. */
3002 arg_ts->state = 0;
3003
3004 if (NEED_SYNC_ARG(0)) {
3005 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3006 ? INDEX_op_st_i32
3007 : INDEX_op_st_i64);
3008 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
3009 TCGTemp *out_ts = dir_ts;
3010
3011 if (IS_DEAD_ARG(0)) {
3012 out_ts = arg_temp(op->args[1]);
3013 arg_ts->state = TS_DEAD;
3014 tcg_op_remove(s, op);
3015 } else {
3016 arg_ts->state = TS_MEM;
3017 }
3018
3019 sop->args[0] = temp_arg(out_ts);
3020 sop->args[1] = temp_arg(arg_ts->mem_base);
3021 sop->args[2] = arg_ts->mem_offset;
3022 } else {
3023 tcg_debug_assert(!IS_DEAD_ARG(0));
3024 }
3025 }
3026 } else {
3027 for (i = 0; i < nb_oargs; i++) {
3028 arg_ts = arg_temp(op->args[i]);
3029 dir_ts = arg_ts->state_ptr;
3030 if (!dir_ts) {
3031 continue;
3032 }
3033 op->args[i] = temp_arg(dir_ts);
3034 changes = true;
3035
3036 /* The output is now live and modified. */
3037 arg_ts->state = 0;
3038
3039 /* Sync outputs upon their last write. */
3040 if (NEED_SYNC_ARG(i)) {
3041 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
3042 ? INDEX_op_st_i32
3043 : INDEX_op_st_i64);
3044 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
3045
3046 sop->args[0] = temp_arg(dir_ts);
3047 sop->args[1] = temp_arg(arg_ts->mem_base);
3048 sop->args[2] = arg_ts->mem_offset;
3049
3050 arg_ts->state = TS_MEM;
3051 }
3052 /* Drop outputs that are dead. */
3053 if (IS_DEAD_ARG(i)) {
3054 arg_ts->state = TS_DEAD;
3055 }
3056 }
3057 }
3058 }
3059
3060 return changes;
3061 }
3062
3063 #ifdef CONFIG_DEBUG_TCG
3064 static void dump_regs(TCGContext *s)
3065 {
3066 TCGTemp *ts;
3067 int i;
3068 char buf[64];
3069
3070 for(i = 0; i < s->nb_temps; i++) {
3071 ts = &s->temps[i];
3072 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
3073 switch(ts->val_type) {
3074 case TEMP_VAL_REG:
3075 printf("%s", tcg_target_reg_names[ts->reg]);
3076 break;
3077 case TEMP_VAL_MEM:
3078 printf("%d(%s)", (int)ts->mem_offset,
3079 tcg_target_reg_names[ts->mem_base->reg]);
3080 break;
3081 case TEMP_VAL_CONST:
3082 printf("$0x%" TCG_PRIlx, ts->val);
3083 break;
3084 case TEMP_VAL_DEAD:
3085 printf("D");
3086 break;
3087 default:
3088 printf("???");
3089 break;
3090 }
3091 printf("\n");
3092 }
3093
3094 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
3095 if (s->reg_to_temp[i] != NULL) {
3096 printf("%s: %s\n",
3097 tcg_target_reg_names[i],
3098 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
3099 }
3100 }
3101 }
3102
3103 static void check_regs(TCGContext *s)
3104 {
3105 int reg;
3106 int k;
3107 TCGTemp *ts;
3108 char buf[64];
3109
3110 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
3111 ts = s->reg_to_temp[reg];
3112 if (ts != NULL) {
3113 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
3114 printf("Inconsistency for register %s:\n",
3115 tcg_target_reg_names[reg]);
3116 goto fail;
3117 }
3118 }
3119 }
3120 for (k = 0; k < s->nb_temps; k++) {
3121 ts = &s->temps[k];
3122 if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg
3123 && s->reg_to_temp[ts->reg] != ts) {
3124 printf("Inconsistency for temp %s:\n",
3125 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
3126 fail:
3127 printf("reg state:\n");
3128 dump_regs(s);
3129 tcg_abort();
3130 }
3131 }
3132 }
3133 #endif
3134
3135 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
3136 {
3137 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
3138 /* Sparc64 stack is accessed with offset of 2047 */
3139 s->current_frame_offset = (s->current_frame_offset +
3140 (tcg_target_long)sizeof(tcg_target_long) - 1) &
3141 ~(sizeof(tcg_target_long) - 1);
3142 #endif
3143 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) >
3144 s->frame_end) {
3145 tcg_abort();
3146 }
3147 ts->mem_offset = s->current_frame_offset;
3148 ts->mem_base = s->frame_temp;
3149 ts->mem_allocated = 1;
3150 s->current_frame_offset += sizeof(tcg_target_long);
3151 }
3152
3153 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
3154
3155 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3156 mark it free; otherwise mark it dead. */
3157 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
3158 {
3159 if (ts->fixed_reg) {
3160 return;
3161 }
3162 if (ts->val_type == TEMP_VAL_REG) {
3163 s->reg_to_temp[ts->reg] = NULL;
3164 }
3165 ts->val_type = (free_or_dead < 0
3166 || ts->temp_local
3167 || ts->temp_global
3168 ? TEMP_VAL_MEM : TEMP_VAL_DEAD);
3169 }
3170
3171 /* Mark a temporary as dead. */
3172 static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3173 {
3174 temp_free_or_dead(s, ts, 1);
3175 }
3176
3177 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3178 registers needs to be allocated to store a constant. If 'free_or_dead'
3179 is non-zero, subsequently release the temporary; if it is positive, the
3180 temp is dead; if it is negative, the temp is free. */
3181 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3182 TCGRegSet preferred_regs, int free_or_dead)
3183 {
3184 if (ts->fixed_reg) {
3185 return;
3186 }
3187 if (!ts->mem_coherent) {
3188 if (!ts->mem_allocated) {
3189 temp_allocate_frame(s, ts);
3190 }
3191 switch (ts->val_type) {
3192 case TEMP_VAL_CONST:
3193 /* If we're going to free the temp immediately, then we won't
3194 require it later in a register, so attempt to store the
3195 constant to memory directly. */
3196 if (free_or_dead
3197 && tcg_out_sti(s, ts->type, ts->val,
3198 ts->mem_base->reg, ts->mem_offset)) {
3199 break;
3200 }
3201 temp_load(s, ts, tcg_target_available_regs[ts->type],
3202 allocated_regs, preferred_regs);
3203 /* fallthrough */
3204
3205 case TEMP_VAL_REG:
3206 tcg_out_st(s, ts->type, ts->reg,
3207 ts->mem_base->reg, ts->mem_offset);
3208 break;
3209
3210 case TEMP_VAL_MEM:
3211 break;
3212
3213 case TEMP_VAL_DEAD:
3214 default:
3215 tcg_abort();
3216 }
3217 ts->mem_coherent = 1;
3218 }
3219 if (free_or_dead) {
3220 temp_free_or_dead(s, ts, free_or_dead);
3221 }
3222 }
3223
3224 /* free register 'reg' by spilling the corresponding temporary if necessary */
3225 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
3226 {
3227 TCGTemp *ts = s->reg_to_temp[reg];
3228 if (ts != NULL) {
3229 temp_sync(s, ts, allocated_regs, 0, -1);
3230 }
3231 }
3232
3233 /**
3234 * tcg_reg_alloc:
3235 * @required_regs: Set of registers in which we must allocate.
3236 * @allocated_regs: Set of registers which must be avoided.
3237 * @preferred_regs: Set of registers we should prefer.
3238 * @rev: True if we search the registers in "indirect" order.
3239 *
3240 * The allocated register must be in @required_regs & ~@allocated_regs,
3241 * but if we can put it in @preferred_regs we may save a move later.
3242 */
3243 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3244 TCGRegSet allocated_regs,
3245 TCGRegSet preferred_regs, bool rev)
3246 {
3247 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3248 TCGRegSet reg_ct[2];
3249 const int *order;
3250
3251 reg_ct[1] = required_regs & ~allocated_regs;
3252 tcg_debug_assert(reg_ct[1] != 0);
3253 reg_ct[0] = reg_ct[1] & preferred_regs;
3254
3255 /* Skip the preferred_regs option if it cannot be satisfied,
3256 or if the preference made no difference. */
3257 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3258
3259 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
3260
3261 /* Try free registers, preferences first. */
3262 for (j = f; j < 2; j++) {
3263 TCGRegSet set = reg_ct[j];
3264
3265 if (tcg_regset_single(set)) {
3266 /* One register in the set. */
3267 TCGReg reg = tcg_regset_first(set);
3268 if (s->reg_to_temp[reg] == NULL) {
3269 return reg;
3270 }
3271 } else {
3272 for (i = 0; i < n; i++) {
3273 TCGReg reg = order[i];
3274 if (s->reg_to_temp[reg] == NULL &&
3275 tcg_regset_test_reg(set, reg)) {
3276 return reg;
3277 }
3278 }
3279 }
3280 }
3281
3282 /* We must spill something. */
3283 for (j = f; j < 2; j++) {
3284 TCGRegSet set = reg_ct[j];
3285
3286 if (tcg_regset_single(set)) {
3287 /* One register in the set. */
3288 TCGReg reg = tcg_regset_first(set);
3289 tcg_reg_free(s, reg, allocated_regs);
3290 return reg;
3291 } else {
3292 for (i = 0; i < n; i++) {
3293 TCGReg reg = order[i];
3294 if (tcg_regset_test_reg(set, reg)) {
3295 tcg_reg_free(s, reg, allocated_regs);
3296 return reg;
3297 }
3298 }
3299 }
3300 }
3301
3302 tcg_abort();
3303 }
3304
3305 /* Make sure the temporary is in a register. If needed, allocate the register
3306 from DESIRED while avoiding ALLOCATED. */
3307 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
3308 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
3309 {
3310 TCGReg reg;
3311
3312 switch (ts->val_type) {
3313 case TEMP_VAL_REG:
3314 return;
3315 case TEMP_VAL_CONST:
3316 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3317 preferred_regs, ts->indirect_base);
3318 tcg_out_movi(s, ts->type, reg, ts->val);
3319 ts->mem_coherent = 0;
3320 break;
3321 case TEMP_VAL_MEM:
3322 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
3323 preferred_regs, ts->indirect_base);
3324 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3325 ts->mem_coherent = 1;
3326 break;
3327 case TEMP_VAL_DEAD:
3328 default:
3329 tcg_abort();
3330 }
3331 ts->reg = reg;
3332 ts->val_type = TEMP_VAL_REG;
3333 s->reg_to_temp[reg] = ts;
3334 }
3335
3336 /* Save a temporary to memory. 'allocated_regs' is used in case a
3337 temporary registers needs to be allocated to store a constant. */
3338 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
3339 {
3340 /* The liveness analysis already ensures that globals are back
3341 in memory. Keep an tcg_debug_assert for safety. */
3342 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg);
3343 }
3344
3345 /* save globals to their canonical location and assume they can be
3346 modified be the following code. 'allocated_regs' is used in case a
3347 temporary registers needs to be allocated to store a constant. */
3348 static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
3349 {
3350 int i, n;
3351
3352 for (i = 0, n = s->nb_globals; i < n; i++) {
3353 temp_save(s, &s->temps[i], allocated_regs);
3354 }
3355 }
3356
3357 /* sync globals to their canonical location and assume they can be
3358 read by the following code. 'allocated_regs' is used in case a
3359 temporary registers needs to be allocated to store a constant. */
3360 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3361 {
3362 int i, n;
3363
3364 for (i = 0, n = s->nb_globals; i < n; i++) {
3365 TCGTemp *ts = &s->temps[i];
3366 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
3367 || ts->fixed_reg
3368 || ts->mem_coherent);
3369 }
3370 }
3371
3372 /* at the end of a basic block, we assume all temporaries are dead and
3373 all globals are stored at their canonical location. */
3374 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
3375 {
3376 int i;
3377
3378 for (i = s->nb_globals; i < s->nb_temps; i++) {
3379 TCGTemp *ts = &s->temps[i];
3380 if (ts->temp_local) {
3381 temp_save(s, ts, allocated_regs);
3382 } else {
3383 /* The liveness analysis already ensures that temps are dead.
3384 Keep an tcg_debug_assert for safety. */
3385 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
3386 }
3387 }
3388
3389 save_globals(s, allocated_regs);
3390 }
3391
3392 /*
3393 * Specialized code generation for INDEX_op_movi_*.
3394 */
3395 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
3396 tcg_target_ulong val, TCGLifeData arg_life,
3397 TCGRegSet preferred_regs)
3398 {
3399 /* ENV should not be modified. */
3400 tcg_debug_assert(!ots->fixed_reg);
3401
3402 /* The movi is not explicitly generated here. */
3403 if (ots->val_type == TEMP_VAL_REG) {
3404 s->reg_to_temp[ots->reg] = NULL;
3405 }
3406 ots->val_type = TEMP_VAL_CONST;
3407 ots->val = val;
3408 ots->mem_coherent = 0;
3409 if (NEED_SYNC_ARG(0)) {
3410 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
3411 } else if (IS_DEAD_ARG(0)) {
3412 temp_dead(s, ots);
3413 }
3414 }
3415
3416 static void tcg_reg_alloc_movi(TCGContext *s, const TCGOp *op)
3417 {
3418 TCGTemp *ots = arg_temp(op->args[0]);
3419 tcg_target_ulong val = op->args[1];
3420
3421 tcg_reg_alloc_do_movi(s, ots, val, op->life, op->output_pref[0]);
3422 }
3423
3424 /*
3425 * Specialized code generation for INDEX_op_mov_*.
3426 */
3427 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
3428 {
3429 const TCGLifeData arg_life = op->life;
3430 TCGRegSet allocated_regs, preferred_regs;
3431 TCGTemp *ts, *ots;
3432 TCGType otype, itype;
3433
3434 allocated_regs = s->reserved_regs;
3435 preferred_regs = op->output_pref[0];
3436 ots = arg_temp(op->args[0]);
3437 ts = arg_temp(op->args[1]);
3438
3439 /* ENV should not be modified. */
3440 tcg_debug_assert(!ots->fixed_reg);
3441
3442 /* Note that otype != itype for no-op truncation. */
3443 otype = ots->type;
3444 itype = ts->type;
3445
3446 if (ts->val_type == TEMP_VAL_CONST) {
3447 /* propagate constant or generate sti */
3448 tcg_target_ulong val = ts->val;