migration: split migration hooks out of QEMUFileOps
[qemu.git] / tcg / tcg.c
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 /* define it to use liveness analysis (better code) */
26 #define USE_LIVENESS_ANALYSIS
27 #define USE_TCG_OPTIMIZATIONS
28
29 #include "qemu/osdep.h"
30
31 /* Define to jump the ELF file used to communicate with GDB. */
32 #undef DEBUG_JIT
33
34 #include "qemu/cutils.h"
35 #include "qemu/host-utils.h"
36 #include "qemu/timer.h"
37
38 /* Note: the long term plan is to reduce the dependencies on the QEMU
39 CPU definitions. Currently they are used for qemu_ld/st
40 instructions */
41 #define NO_CPU_IO_DEFS
42 #include "cpu.h"
43
44 #include "qemu/host-utils.h"
45 #include "qemu/timer.h"
46 #include "exec/cpu-common.h"
47 #include "exec/exec-all.h"
48
49 #include "tcg-op.h"
50
51 #if UINTPTR_MAX == UINT32_MAX
52 # define ELF_CLASS ELFCLASS32
53 #else
54 # define ELF_CLASS ELFCLASS64
55 #endif
56 #ifdef HOST_WORDS_BIGENDIAN
57 # define ELF_DATA ELFDATA2MSB
58 #else
59 # define ELF_DATA ELFDATA2LSB
60 #endif
61
62 #include "elf.h"
63 #include "exec/log.h"
64
65 /* Forward declarations for functions declared in tcg-target.inc.c and
66 used here. */
67 static void tcg_target_init(TCGContext *s);
68 static void tcg_target_qemu_prologue(TCGContext *s);
69 static void patch_reloc(tcg_insn_unit *code_ptr, int type,
70 intptr_t value, intptr_t addend);
71
72 /* The CIE and FDE header definitions will be common to all hosts. */
73 typedef struct {
74 uint32_t len __attribute__((aligned((sizeof(void *)))));
75 uint32_t id;
76 uint8_t version;
77 char augmentation[1];
78 uint8_t code_align;
79 uint8_t data_align;
80 uint8_t return_column;
81 } DebugFrameCIE;
82
83 typedef struct QEMU_PACKED {
84 uint32_t len __attribute__((aligned((sizeof(void *)))));
85 uint32_t cie_offset;
86 uintptr_t func_start;
87 uintptr_t func_len;
88 } DebugFrameFDEHeader;
89
90 typedef struct QEMU_PACKED {
91 DebugFrameCIE cie;
92 DebugFrameFDEHeader fde;
93 } DebugFrameHeader;
94
95 static void tcg_register_jit_int(void *buf, size_t size,
96 const void *debug_frame,
97 size_t debug_frame_size)
98 __attribute__((unused));
99
100 /* Forward declarations for functions declared and used in tcg-target.inc.c. */
101 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str);
102 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
103 intptr_t arg2);
104 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
105 static void tcg_out_movi(TCGContext *s, TCGType type,
106 TCGReg ret, tcg_target_long arg);
107 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
108 const int *const_args);
109 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
110 intptr_t arg2);
111 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target);
112 static int tcg_target_const_match(tcg_target_long val, TCGType type,
113 const TCGArgConstraint *arg_ct);
114 static void tcg_out_tb_init(TCGContext *s);
115 static bool tcg_out_tb_finalize(TCGContext *s);
116
117
118
119 static TCGRegSet tcg_target_available_regs[2];
120 static TCGRegSet tcg_target_call_clobber_regs;
121
122 #if TCG_TARGET_INSN_UNIT_SIZE == 1
123 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
124 {
125 *s->code_ptr++ = v;
126 }
127
128 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
129 uint8_t v)
130 {
131 *p = v;
132 }
133 #endif
134
135 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
136 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
137 {
138 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
139 *s->code_ptr++ = v;
140 } else {
141 tcg_insn_unit *p = s->code_ptr;
142 memcpy(p, &v, sizeof(v));
143 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
144 }
145 }
146
147 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
148 uint16_t v)
149 {
150 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
151 *p = v;
152 } else {
153 memcpy(p, &v, sizeof(v));
154 }
155 }
156 #endif
157
158 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
159 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
160 {
161 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
162 *s->code_ptr++ = v;
163 } else {
164 tcg_insn_unit *p = s->code_ptr;
165 memcpy(p, &v, sizeof(v));
166 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
167 }
168 }
169
170 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
171 uint32_t v)
172 {
173 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
174 *p = v;
175 } else {
176 memcpy(p, &v, sizeof(v));
177 }
178 }
179 #endif
180
181 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
182 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
183 {
184 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
185 *s->code_ptr++ = v;
186 } else {
187 tcg_insn_unit *p = s->code_ptr;
188 memcpy(p, &v, sizeof(v));
189 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
190 }
191 }
192
193 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
194 uint64_t v)
195 {
196 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
197 *p = v;
198 } else {
199 memcpy(p, &v, sizeof(v));
200 }
201 }
202 #endif
203
204 /* label relocation processing */
205
206 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
207 TCGLabel *l, intptr_t addend)
208 {
209 TCGRelocation *r;
210
211 if (l->has_value) {
212 /* FIXME: This may break relocations on RISC targets that
213 modify instruction fields in place. The caller may not have
214 written the initial value. */
215 patch_reloc(code_ptr, type, l->u.value, addend);
216 } else {
217 /* add a new relocation entry */
218 r = tcg_malloc(sizeof(TCGRelocation));
219 r->type = type;
220 r->ptr = code_ptr;
221 r->addend = addend;
222 r->next = l->u.first_reloc;
223 l->u.first_reloc = r;
224 }
225 }
226
227 static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr)
228 {
229 intptr_t value = (intptr_t)ptr;
230 TCGRelocation *r;
231
232 tcg_debug_assert(!l->has_value);
233
234 for (r = l->u.first_reloc; r != NULL; r = r->next) {
235 patch_reloc(r->ptr, r->type, value, r->addend);
236 }
237
238 l->has_value = 1;
239 l->u.value_ptr = ptr;
240 }
241
242 TCGLabel *gen_new_label(void)
243 {
244 TCGContext *s = &tcg_ctx;
245 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
246
247 *l = (TCGLabel){
248 .id = s->nb_labels++
249 };
250
251 return l;
252 }
253
254 #include "tcg-target.inc.c"
255
256 /* pool based memory allocation */
257 void *tcg_malloc_internal(TCGContext *s, int size)
258 {
259 TCGPool *p;
260 int pool_size;
261
262 if (size > TCG_POOL_CHUNK_SIZE) {
263 /* big malloc: insert a new pool (XXX: could optimize) */
264 p = g_malloc(sizeof(TCGPool) + size);
265 p->size = size;
266 p->next = s->pool_first_large;
267 s->pool_first_large = p;
268 return p->data;
269 } else {
270 p = s->pool_current;
271 if (!p) {
272 p = s->pool_first;
273 if (!p)
274 goto new_pool;
275 } else {
276 if (!p->next) {
277 new_pool:
278 pool_size = TCG_POOL_CHUNK_SIZE;
279 p = g_malloc(sizeof(TCGPool) + pool_size);
280 p->size = pool_size;
281 p->next = NULL;
282 if (s->pool_current)
283 s->pool_current->next = p;
284 else
285 s->pool_first = p;
286 } else {
287 p = p->next;
288 }
289 }
290 }
291 s->pool_current = p;
292 s->pool_cur = p->data + size;
293 s->pool_end = p->data + p->size;
294 return p->data;
295 }
296
297 void tcg_pool_reset(TCGContext *s)
298 {
299 TCGPool *p, *t;
300 for (p = s->pool_first_large; p; p = t) {
301 t = p->next;
302 g_free(p);
303 }
304 s->pool_first_large = NULL;
305 s->pool_cur = s->pool_end = NULL;
306 s->pool_current = NULL;
307 }
308
309 typedef struct TCGHelperInfo {
310 void *func;
311 const char *name;
312 unsigned flags;
313 unsigned sizemask;
314 } TCGHelperInfo;
315
316 #include "exec/helper-proto.h"
317
318 static const TCGHelperInfo all_helpers[] = {
319 #include "exec/helper-tcg.h"
320 };
321
322 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
323
324 void tcg_context_init(TCGContext *s)
325 {
326 int op, total_args, n, i;
327 TCGOpDef *def;
328 TCGArgConstraint *args_ct;
329 int *sorted_args;
330 GHashTable *helper_table;
331
332 memset(s, 0, sizeof(*s));
333 s->nb_globals = 0;
334
335 /* Count total number of arguments and allocate the corresponding
336 space */
337 total_args = 0;
338 for(op = 0; op < NB_OPS; op++) {
339 def = &tcg_op_defs[op];
340 n = def->nb_iargs + def->nb_oargs;
341 total_args += n;
342 }
343
344 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args);
345 sorted_args = g_malloc(sizeof(int) * total_args);
346
347 for(op = 0; op < NB_OPS; op++) {
348 def = &tcg_op_defs[op];
349 def->args_ct = args_ct;
350 def->sorted_args = sorted_args;
351 n = def->nb_iargs + def->nb_oargs;
352 sorted_args += n;
353 args_ct += n;
354 }
355
356 /* Register helpers. */
357 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
358 s->helpers = helper_table = g_hash_table_new(NULL, NULL);
359
360 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
361 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
362 (gpointer)&all_helpers[i]);
363 }
364
365 tcg_target_init(s);
366
367 /* Reverse the order of the saved registers, assuming they're all at
368 the start of tcg_target_reg_alloc_order. */
369 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
370 int r = tcg_target_reg_alloc_order[n];
371 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
372 break;
373 }
374 }
375 for (i = 0; i < n; ++i) {
376 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
377 }
378 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
379 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
380 }
381 }
382
383 void tcg_prologue_init(TCGContext *s)
384 {
385 size_t prologue_size, total_size;
386 void *buf0, *buf1;
387
388 /* Put the prologue at the beginning of code_gen_buffer. */
389 buf0 = s->code_gen_buffer;
390 s->code_ptr = buf0;
391 s->code_buf = buf0;
392 s->code_gen_prologue = buf0;
393
394 /* Generate the prologue. */
395 tcg_target_qemu_prologue(s);
396 buf1 = s->code_ptr;
397 flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1);
398
399 /* Deduct the prologue from the buffer. */
400 prologue_size = tcg_current_code_size(s);
401 s->code_gen_ptr = buf1;
402 s->code_gen_buffer = buf1;
403 s->code_buf = buf1;
404 total_size = s->code_gen_buffer_size - prologue_size;
405 s->code_gen_buffer_size = total_size;
406
407 /* Compute a high-water mark, at which we voluntarily flush the buffer
408 and start over. The size here is arbitrary, significantly larger
409 than we expect the code generation for any one opcode to require. */
410 s->code_gen_highwater = s->code_gen_buffer + (total_size - 1024);
411
412 tcg_register_jit(s->code_gen_buffer, total_size);
413
414 #ifdef DEBUG_DISAS
415 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
416 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
417 log_disas(buf0, prologue_size);
418 qemu_log("\n");
419 qemu_log_flush();
420 }
421 #endif
422 }
423
424 void tcg_func_start(TCGContext *s)
425 {
426 tcg_pool_reset(s);
427 s->nb_temps = s->nb_globals;
428
429 /* No temps have been previously allocated for size or locality. */
430 memset(s->free_temps, 0, sizeof(s->free_temps));
431
432 s->nb_labels = 0;
433 s->current_frame_offset = s->frame_start;
434
435 #ifdef CONFIG_DEBUG_TCG
436 s->goto_tb_issue_mask = 0;
437 #endif
438
439 s->gen_first_op_idx = 0;
440 s->gen_last_op_idx = -1;
441 s->gen_next_op_idx = 0;
442 s->gen_next_parm_idx = 0;
443
444 s->be = tcg_malloc(sizeof(TCGBackendData));
445 }
446
447 static inline int temp_idx(TCGContext *s, TCGTemp *ts)
448 {
449 ptrdiff_t n = ts - s->temps;
450 tcg_debug_assert(n >= 0 && n < s->nb_temps);
451 return n;
452 }
453
454 static inline TCGTemp *tcg_temp_alloc(TCGContext *s)
455 {
456 int n = s->nb_temps++;
457 tcg_debug_assert(n < TCG_MAX_TEMPS);
458 return memset(&s->temps[n], 0, sizeof(TCGTemp));
459 }
460
461 static inline TCGTemp *tcg_global_alloc(TCGContext *s)
462 {
463 tcg_debug_assert(s->nb_globals == s->nb_temps);
464 s->nb_globals++;
465 return tcg_temp_alloc(s);
466 }
467
468 static int tcg_global_reg_new_internal(TCGContext *s, TCGType type,
469 TCGReg reg, const char *name)
470 {
471 TCGTemp *ts;
472
473 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
474 tcg_abort();
475 }
476
477 ts = tcg_global_alloc(s);
478 ts->base_type = type;
479 ts->type = type;
480 ts->fixed_reg = 1;
481 ts->reg = reg;
482 ts->name = name;
483 tcg_regset_set_reg(s->reserved_regs, reg);
484
485 return temp_idx(s, ts);
486 }
487
488 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
489 {
490 int idx;
491 s->frame_start = start;
492 s->frame_end = start + size;
493 idx = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
494 s->frame_temp = &s->temps[idx];
495 }
496
497 TCGv_i32 tcg_global_reg_new_i32(TCGReg reg, const char *name)
498 {
499 TCGContext *s = &tcg_ctx;
500 int idx;
501
502 if (tcg_regset_test_reg(s->reserved_regs, reg)) {
503 tcg_abort();
504 }
505 idx = tcg_global_reg_new_internal(s, TCG_TYPE_I32, reg, name);
506 return MAKE_TCGV_I32(idx);
507 }
508
509 TCGv_i64 tcg_global_reg_new_i64(TCGReg reg, const char *name)
510 {
511 TCGContext *s = &tcg_ctx;
512 int idx;
513
514 if (tcg_regset_test_reg(s->reserved_regs, reg)) {
515 tcg_abort();
516 }
517 idx = tcg_global_reg_new_internal(s, TCG_TYPE_I64, reg, name);
518 return MAKE_TCGV_I64(idx);
519 }
520
521 int tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
522 intptr_t offset, const char *name)
523 {
524 TCGContext *s = &tcg_ctx;
525 TCGTemp *base_ts = &s->temps[GET_TCGV_PTR(base)];
526 TCGTemp *ts = tcg_global_alloc(s);
527 int indirect_reg = 0, bigendian = 0;
528 #ifdef HOST_WORDS_BIGENDIAN
529 bigendian = 1;
530 #endif
531
532 if (!base_ts->fixed_reg) {
533 indirect_reg = 1;
534 base_ts->indirect_base = 1;
535 }
536
537 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
538 TCGTemp *ts2 = tcg_global_alloc(s);
539 char buf[64];
540
541 ts->base_type = TCG_TYPE_I64;
542 ts->type = TCG_TYPE_I32;
543 ts->indirect_reg = indirect_reg;
544 ts->mem_allocated = 1;
545 ts->mem_base = base_ts;
546 ts->mem_offset = offset + bigendian * 4;
547 pstrcpy(buf, sizeof(buf), name);
548 pstrcat(buf, sizeof(buf), "_0");
549 ts->name = strdup(buf);
550
551 tcg_debug_assert(ts2 == ts + 1);
552 ts2->base_type = TCG_TYPE_I64;
553 ts2->type = TCG_TYPE_I32;
554 ts2->indirect_reg = indirect_reg;
555 ts2->mem_allocated = 1;
556 ts2->mem_base = base_ts;
557 ts2->mem_offset = offset + (1 - bigendian) * 4;
558 pstrcpy(buf, sizeof(buf), name);
559 pstrcat(buf, sizeof(buf), "_1");
560 ts->name = strdup(buf);
561 } else {
562 ts->base_type = type;
563 ts->type = type;
564 ts->indirect_reg = indirect_reg;
565 ts->mem_allocated = 1;
566 ts->mem_base = base_ts;
567 ts->mem_offset = offset;
568 ts->name = name;
569 }
570 return temp_idx(s, ts);
571 }
572
573 static int tcg_temp_new_internal(TCGType type, int temp_local)
574 {
575 TCGContext *s = &tcg_ctx;
576 TCGTemp *ts;
577 int idx, k;
578
579 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
580 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
581 if (idx < TCG_MAX_TEMPS) {
582 /* There is already an available temp with the right type. */
583 clear_bit(idx, s->free_temps[k].l);
584
585 ts = &s->temps[idx];
586 ts->temp_allocated = 1;
587 tcg_debug_assert(ts->base_type == type);
588 tcg_debug_assert(ts->temp_local == temp_local);
589 } else {
590 ts = tcg_temp_alloc(s);
591 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
592 TCGTemp *ts2 = tcg_temp_alloc(s);
593
594 ts->base_type = type;
595 ts->type = TCG_TYPE_I32;
596 ts->temp_allocated = 1;
597 ts->temp_local = temp_local;
598
599 tcg_debug_assert(ts2 == ts + 1);
600 ts2->base_type = TCG_TYPE_I64;
601 ts2->type = TCG_TYPE_I32;
602 ts2->temp_allocated = 1;
603 ts2->temp_local = temp_local;
604 } else {
605 ts->base_type = type;
606 ts->type = type;
607 ts->temp_allocated = 1;
608 ts->temp_local = temp_local;
609 }
610 idx = temp_idx(s, ts);
611 }
612
613 #if defined(CONFIG_DEBUG_TCG)
614 s->temps_in_use++;
615 #endif
616 return idx;
617 }
618
619 TCGv_i32 tcg_temp_new_internal_i32(int temp_local)
620 {
621 int idx;
622
623 idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local);
624 return MAKE_TCGV_I32(idx);
625 }
626
627 TCGv_i64 tcg_temp_new_internal_i64(int temp_local)
628 {
629 int idx;
630
631 idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local);
632 return MAKE_TCGV_I64(idx);
633 }
634
635 static void tcg_temp_free_internal(int idx)
636 {
637 TCGContext *s = &tcg_ctx;
638 TCGTemp *ts;
639 int k;
640
641 #if defined(CONFIG_DEBUG_TCG)
642 s->temps_in_use--;
643 if (s->temps_in_use < 0) {
644 fprintf(stderr, "More temporaries freed than allocated!\n");
645 }
646 #endif
647
648 tcg_debug_assert(idx >= s->nb_globals && idx < s->nb_temps);
649 ts = &s->temps[idx];
650 tcg_debug_assert(ts->temp_allocated != 0);
651 ts->temp_allocated = 0;
652
653 k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0);
654 set_bit(idx, s->free_temps[k].l);
655 }
656
657 void tcg_temp_free_i32(TCGv_i32 arg)
658 {
659 tcg_temp_free_internal(GET_TCGV_I32(arg));
660 }
661
662 void tcg_temp_free_i64(TCGv_i64 arg)
663 {
664 tcg_temp_free_internal(GET_TCGV_I64(arg));
665 }
666
667 TCGv_i32 tcg_const_i32(int32_t val)
668 {
669 TCGv_i32 t0;
670 t0 = tcg_temp_new_i32();
671 tcg_gen_movi_i32(t0, val);
672 return t0;
673 }
674
675 TCGv_i64 tcg_const_i64(int64_t val)
676 {
677 TCGv_i64 t0;
678 t0 = tcg_temp_new_i64();
679 tcg_gen_movi_i64(t0, val);
680 return t0;
681 }
682
683 TCGv_i32 tcg_const_local_i32(int32_t val)
684 {
685 TCGv_i32 t0;
686 t0 = tcg_temp_local_new_i32();
687 tcg_gen_movi_i32(t0, val);
688 return t0;
689 }
690
691 TCGv_i64 tcg_const_local_i64(int64_t val)
692 {
693 TCGv_i64 t0;
694 t0 = tcg_temp_local_new_i64();
695 tcg_gen_movi_i64(t0, val);
696 return t0;
697 }
698
699 #if defined(CONFIG_DEBUG_TCG)
700 void tcg_clear_temp_count(void)
701 {
702 TCGContext *s = &tcg_ctx;
703 s->temps_in_use = 0;
704 }
705
706 int tcg_check_temp_count(void)
707 {
708 TCGContext *s = &tcg_ctx;
709 if (s->temps_in_use) {
710 /* Clear the count so that we don't give another
711 * warning immediately next time around.
712 */
713 s->temps_in_use = 0;
714 return 1;
715 }
716 return 0;
717 }
718 #endif
719
720 /* Note: we convert the 64 bit args to 32 bit and do some alignment
721 and endian swap. Maybe it would be better to do the alignment
722 and endian swap in tcg_reg_alloc_call(). */
723 void tcg_gen_callN(TCGContext *s, void *func, TCGArg ret,
724 int nargs, TCGArg *args)
725 {
726 int i, real_args, nb_rets, pi, pi_first;
727 unsigned sizemask, flags;
728 TCGHelperInfo *info;
729
730 info = g_hash_table_lookup(s->helpers, (gpointer)func);
731 flags = info->flags;
732 sizemask = info->sizemask;
733
734 #if defined(__sparc__) && !defined(__arch64__) \
735 && !defined(CONFIG_TCG_INTERPRETER)
736 /* We have 64-bit values in one register, but need to pass as two
737 separate parameters. Split them. */
738 int orig_sizemask = sizemask;
739 int orig_nargs = nargs;
740 TCGv_i64 retl, reth;
741
742 TCGV_UNUSED_I64(retl);
743 TCGV_UNUSED_I64(reth);
744 if (sizemask != 0) {
745 TCGArg *split_args = __builtin_alloca(sizeof(TCGArg) * nargs * 2);
746 for (i = real_args = 0; i < nargs; ++i) {
747 int is_64bit = sizemask & (1 << (i+1)*2);
748 if (is_64bit) {
749 TCGv_i64 orig = MAKE_TCGV_I64(args[i]);
750 TCGv_i32 h = tcg_temp_new_i32();
751 TCGv_i32 l = tcg_temp_new_i32();
752 tcg_gen_extr_i64_i32(l, h, orig);
753 split_args[real_args++] = GET_TCGV_I32(h);
754 split_args[real_args++] = GET_TCGV_I32(l);
755 } else {
756 split_args[real_args++] = args[i];
757 }
758 }
759 nargs = real_args;
760 args = split_args;
761 sizemask = 0;
762 }
763 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
764 for (i = 0; i < nargs; ++i) {
765 int is_64bit = sizemask & (1 << (i+1)*2);
766 int is_signed = sizemask & (2 << (i+1)*2);
767 if (!is_64bit) {
768 TCGv_i64 temp = tcg_temp_new_i64();
769 TCGv_i64 orig = MAKE_TCGV_I64(args[i]);
770 if (is_signed) {
771 tcg_gen_ext32s_i64(temp, orig);
772 } else {
773 tcg_gen_ext32u_i64(temp, orig);
774 }
775 args[i] = GET_TCGV_I64(temp);
776 }
777 }
778 #endif /* TCG_TARGET_EXTEND_ARGS */
779
780 pi_first = pi = s->gen_next_parm_idx;
781 if (ret != TCG_CALL_DUMMY_ARG) {
782 #if defined(__sparc__) && !defined(__arch64__) \
783 && !defined(CONFIG_TCG_INTERPRETER)
784 if (orig_sizemask & 1) {
785 /* The 32-bit ABI is going to return the 64-bit value in
786 the %o0/%o1 register pair. Prepare for this by using
787 two return temporaries, and reassemble below. */
788 retl = tcg_temp_new_i64();
789 reth = tcg_temp_new_i64();
790 s->gen_opparam_buf[pi++] = GET_TCGV_I64(reth);
791 s->gen_opparam_buf[pi++] = GET_TCGV_I64(retl);
792 nb_rets = 2;
793 } else {
794 s->gen_opparam_buf[pi++] = ret;
795 nb_rets = 1;
796 }
797 #else
798 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) {
799 #ifdef HOST_WORDS_BIGENDIAN
800 s->gen_opparam_buf[pi++] = ret + 1;
801 s->gen_opparam_buf[pi++] = ret;
802 #else
803 s->gen_opparam_buf[pi++] = ret;
804 s->gen_opparam_buf[pi++] = ret + 1;
805 #endif
806 nb_rets = 2;
807 } else {
808 s->gen_opparam_buf[pi++] = ret;
809 nb_rets = 1;
810 }
811 #endif
812 } else {
813 nb_rets = 0;
814 }
815 real_args = 0;
816 for (i = 0; i < nargs; i++) {
817 int is_64bit = sizemask & (1 << (i+1)*2);
818 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
819 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
820 /* some targets want aligned 64 bit args */
821 if (real_args & 1) {
822 s->gen_opparam_buf[pi++] = TCG_CALL_DUMMY_ARG;
823 real_args++;
824 }
825 #endif
826 /* If stack grows up, then we will be placing successive
827 arguments at lower addresses, which means we need to
828 reverse the order compared to how we would normally
829 treat either big or little-endian. For those arguments
830 that will wind up in registers, this still works for
831 HPPA (the only current STACK_GROWSUP target) since the
832 argument registers are *also* allocated in decreasing
833 order. If another such target is added, this logic may
834 have to get more complicated to differentiate between
835 stack arguments and register arguments. */
836 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
837 s->gen_opparam_buf[pi++] = args[i] + 1;
838 s->gen_opparam_buf[pi++] = args[i];
839 #else
840 s->gen_opparam_buf[pi++] = args[i];
841 s->gen_opparam_buf[pi++] = args[i] + 1;
842 #endif
843 real_args += 2;
844 continue;
845 }
846
847 s->gen_opparam_buf[pi++] = args[i];
848 real_args++;
849 }
850 s->gen_opparam_buf[pi++] = (uintptr_t)func;
851 s->gen_opparam_buf[pi++] = flags;
852
853 i = s->gen_next_op_idx;
854 tcg_debug_assert(i < OPC_BUF_SIZE);
855 tcg_debug_assert(pi <= OPPARAM_BUF_SIZE);
856
857 /* Set links for sequential allocation during translation. */
858 s->gen_op_buf[i] = (TCGOp){
859 .opc = INDEX_op_call,
860 .callo = nb_rets,
861 .calli = real_args,
862 .args = pi_first,
863 .prev = i - 1,
864 .next = i + 1
865 };
866
867 /* Make sure the calli field didn't overflow. */
868 tcg_debug_assert(s->gen_op_buf[i].calli == real_args);
869
870 s->gen_last_op_idx = i;
871 s->gen_next_op_idx = i + 1;
872 s->gen_next_parm_idx = pi;
873
874 #if defined(__sparc__) && !defined(__arch64__) \
875 && !defined(CONFIG_TCG_INTERPRETER)
876 /* Free all of the parts we allocated above. */
877 for (i = real_args = 0; i < orig_nargs; ++i) {
878 int is_64bit = orig_sizemask & (1 << (i+1)*2);
879 if (is_64bit) {
880 TCGv_i32 h = MAKE_TCGV_I32(args[real_args++]);
881 TCGv_i32 l = MAKE_TCGV_I32(args[real_args++]);
882 tcg_temp_free_i32(h);
883 tcg_temp_free_i32(l);
884 } else {
885 real_args++;
886 }
887 }
888 if (orig_sizemask & 1) {
889 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
890 Note that describing these as TCGv_i64 eliminates an unnecessary
891 zero-extension that tcg_gen_concat_i32_i64 would create. */
892 tcg_gen_concat32_i64(MAKE_TCGV_I64(ret), retl, reth);
893 tcg_temp_free_i64(retl);
894 tcg_temp_free_i64(reth);
895 }
896 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
897 for (i = 0; i < nargs; ++i) {
898 int is_64bit = sizemask & (1 << (i+1)*2);
899 if (!is_64bit) {
900 TCGv_i64 temp = MAKE_TCGV_I64(args[i]);
901 tcg_temp_free_i64(temp);
902 }
903 }
904 #endif /* TCG_TARGET_EXTEND_ARGS */
905 }
906
907 static void tcg_reg_alloc_start(TCGContext *s)
908 {
909 int i;
910 TCGTemp *ts;
911 for(i = 0; i < s->nb_globals; i++) {
912 ts = &s->temps[i];
913 if (ts->fixed_reg) {
914 ts->val_type = TEMP_VAL_REG;
915 } else {
916 ts->val_type = TEMP_VAL_MEM;
917 }
918 }
919 for(i = s->nb_globals; i < s->nb_temps; i++) {
920 ts = &s->temps[i];
921 if (ts->temp_local) {
922 ts->val_type = TEMP_VAL_MEM;
923 } else {
924 ts->val_type = TEMP_VAL_DEAD;
925 }
926 ts->mem_allocated = 0;
927 ts->fixed_reg = 0;
928 }
929
930 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
931 }
932
933 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
934 TCGTemp *ts)
935 {
936 int idx = temp_idx(s, ts);
937
938 if (idx < s->nb_globals) {
939 pstrcpy(buf, buf_size, ts->name);
940 } else if (ts->temp_local) {
941 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
942 } else {
943 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
944 }
945 return buf;
946 }
947
948 static char *tcg_get_arg_str_idx(TCGContext *s, char *buf,
949 int buf_size, int idx)
950 {
951 tcg_debug_assert(idx >= 0 && idx < s->nb_temps);
952 return tcg_get_arg_str_ptr(s, buf, buf_size, &s->temps[idx]);
953 }
954
955 /* Find helper name. */
956 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val)
957 {
958 const char *ret = NULL;
959 if (s->helpers) {
960 TCGHelperInfo *info = g_hash_table_lookup(s->helpers, (gpointer)val);
961 if (info) {
962 ret = info->name;
963 }
964 }
965 return ret;
966 }
967
968 static const char * const cond_name[] =
969 {
970 [TCG_COND_NEVER] = "never",
971 [TCG_COND_ALWAYS] = "always",
972 [TCG_COND_EQ] = "eq",
973 [TCG_COND_NE] = "ne",
974 [TCG_COND_LT] = "lt",
975 [TCG_COND_GE] = "ge",
976 [TCG_COND_LE] = "le",
977 [TCG_COND_GT] = "gt",
978 [TCG_COND_LTU] = "ltu",
979 [TCG_COND_GEU] = "geu",
980 [TCG_COND_LEU] = "leu",
981 [TCG_COND_GTU] = "gtu"
982 };
983
984 static const char * const ldst_name[] =
985 {
986 [MO_UB] = "ub",
987 [MO_SB] = "sb",
988 [MO_LEUW] = "leuw",
989 [MO_LESW] = "lesw",
990 [MO_LEUL] = "leul",
991 [MO_LESL] = "lesl",
992 [MO_LEQ] = "leq",
993 [MO_BEUW] = "beuw",
994 [MO_BESW] = "besw",
995 [MO_BEUL] = "beul",
996 [MO_BESL] = "besl",
997 [MO_BEQ] = "beq",
998 };
999
1000 void tcg_dump_ops(TCGContext *s)
1001 {
1002 char buf[128];
1003 TCGOp *op;
1004 int oi;
1005
1006 for (oi = s->gen_first_op_idx; oi >= 0; oi = op->next) {
1007 int i, k, nb_oargs, nb_iargs, nb_cargs;
1008 const TCGOpDef *def;
1009 const TCGArg *args;
1010 TCGOpcode c;
1011
1012 op = &s->gen_op_buf[oi];
1013 c = op->opc;
1014 def = &tcg_op_defs[c];
1015 args = &s->gen_opparam_buf[op->args];
1016
1017 if (c == INDEX_op_insn_start) {
1018 qemu_log("%s ----", oi != s->gen_first_op_idx ? "\n" : "");
1019
1020 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1021 target_ulong a;
1022 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1023 a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2];
1024 #else
1025 a = args[i];
1026 #endif
1027 qemu_log(" " TARGET_FMT_lx, a);
1028 }
1029 } else if (c == INDEX_op_call) {
1030 /* variable number of arguments */
1031 nb_oargs = op->callo;
1032 nb_iargs = op->calli;
1033 nb_cargs = def->nb_cargs;
1034
1035 /* function name, flags, out args */
1036 qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name,
1037 tcg_find_helper(s, args[nb_oargs + nb_iargs]),
1038 args[nb_oargs + nb_iargs + 1], nb_oargs);
1039 for (i = 0; i < nb_oargs; i++) {
1040 qemu_log(",%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
1041 args[i]));
1042 }
1043 for (i = 0; i < nb_iargs; i++) {
1044 TCGArg arg = args[nb_oargs + i];
1045 const char *t = "<dummy>";
1046 if (arg != TCG_CALL_DUMMY_ARG) {
1047 t = tcg_get_arg_str_idx(s, buf, sizeof(buf), arg);
1048 }
1049 qemu_log(",%s", t);
1050 }
1051 } else {
1052 qemu_log(" %s ", def->name);
1053
1054 nb_oargs = def->nb_oargs;
1055 nb_iargs = def->nb_iargs;
1056 nb_cargs = def->nb_cargs;
1057
1058 k = 0;
1059 for (i = 0; i < nb_oargs; i++) {
1060 if (k != 0) {
1061 qemu_log(",");
1062 }
1063 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
1064 args[k++]));
1065 }
1066 for (i = 0; i < nb_iargs; i++) {
1067 if (k != 0) {
1068 qemu_log(",");
1069 }
1070 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
1071 args[k++]));
1072 }
1073 switch (c) {
1074 case INDEX_op_brcond_i32:
1075 case INDEX_op_setcond_i32:
1076 case INDEX_op_movcond_i32:
1077 case INDEX_op_brcond2_i32:
1078 case INDEX_op_setcond2_i32:
1079 case INDEX_op_brcond_i64:
1080 case INDEX_op_setcond_i64:
1081 case INDEX_op_movcond_i64:
1082 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) {
1083 qemu_log(",%s", cond_name[args[k++]]);
1084 } else {
1085 qemu_log(",$0x%" TCG_PRIlx, args[k++]);
1086 }
1087 i = 1;
1088 break;
1089 case INDEX_op_qemu_ld_i32:
1090 case INDEX_op_qemu_st_i32:
1091 case INDEX_op_qemu_ld_i64:
1092 case INDEX_op_qemu_st_i64:
1093 {
1094 TCGMemOpIdx oi = args[k++];
1095 TCGMemOp op = get_memop(oi);
1096 unsigned ix = get_mmuidx(oi);
1097
1098 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
1099 qemu_log(",$0x%x,%u", op, ix);
1100 } else {
1101 const char *s_al = "", *s_op;
1102 if (op & MO_AMASK) {
1103 if ((op & MO_AMASK) == MO_ALIGN) {
1104 s_al = "al+";
1105 } else {
1106 s_al = "un+";
1107 }
1108 }
1109 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
1110 qemu_log(",%s%s,%u", s_al, s_op, ix);
1111 }
1112 i = 1;
1113 }
1114 break;
1115 default:
1116 i = 0;
1117 break;
1118 }
1119 switch (c) {
1120 case INDEX_op_set_label:
1121 case INDEX_op_br:
1122 case INDEX_op_brcond_i32:
1123 case INDEX_op_brcond_i64:
1124 case INDEX_op_brcond2_i32:
1125 qemu_log("%s$L%d", k ? "," : "", arg_label(args[k])->id);
1126 i++, k++;
1127 break;
1128 default:
1129 break;
1130 }
1131 for (; i < nb_cargs; i++, k++) {
1132 qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", args[k]);
1133 }
1134 }
1135 qemu_log("\n");
1136 }
1137 }
1138
1139 /* we give more priority to constraints with less registers */
1140 static int get_constraint_priority(const TCGOpDef *def, int k)
1141 {
1142 const TCGArgConstraint *arg_ct;
1143
1144 int i, n;
1145 arg_ct = &def->args_ct[k];
1146 if (arg_ct->ct & TCG_CT_ALIAS) {
1147 /* an alias is equivalent to a single register */
1148 n = 1;
1149 } else {
1150 if (!(arg_ct->ct & TCG_CT_REG))
1151 return 0;
1152 n = 0;
1153 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
1154 if (tcg_regset_test_reg(arg_ct->u.regs, i))
1155 n++;
1156 }
1157 }
1158 return TCG_TARGET_NB_REGS - n + 1;
1159 }
1160
1161 /* sort from highest priority to lowest */
1162 static void sort_constraints(TCGOpDef *def, int start, int n)
1163 {
1164 int i, j, p1, p2, tmp;
1165
1166 for(i = 0; i < n; i++)
1167 def->sorted_args[start + i] = start + i;
1168 if (n <= 1)
1169 return;
1170 for(i = 0; i < n - 1; i++) {
1171 for(j = i + 1; j < n; j++) {
1172 p1 = get_constraint_priority(def, def->sorted_args[start + i]);
1173 p2 = get_constraint_priority(def, def->sorted_args[start + j]);
1174 if (p1 < p2) {
1175 tmp = def->sorted_args[start + i];
1176 def->sorted_args[start + i] = def->sorted_args[start + j];
1177 def->sorted_args[start + j] = tmp;
1178 }
1179 }
1180 }
1181 }
1182
1183 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs)
1184 {
1185 TCGOpcode op;
1186 TCGOpDef *def;
1187 const char *ct_str;
1188 int i, nb_args;
1189
1190 for(;;) {
1191 if (tdefs->op == (TCGOpcode)-1)
1192 break;
1193 op = tdefs->op;
1194 tcg_debug_assert((unsigned)op < NB_OPS);
1195 def = &tcg_op_defs[op];
1196 #if defined(CONFIG_DEBUG_TCG)
1197 /* Duplicate entry in op definitions? */
1198 tcg_debug_assert(!def->used);
1199 def->used = 1;
1200 #endif
1201 nb_args = def->nb_iargs + def->nb_oargs;
1202 for(i = 0; i < nb_args; i++) {
1203 ct_str = tdefs->args_ct_str[i];
1204 /* Incomplete TCGTargetOpDef entry? */
1205 tcg_debug_assert(ct_str != NULL);
1206 tcg_regset_clear(def->args_ct[i].u.regs);
1207 def->args_ct[i].ct = 0;
1208 if (ct_str[0] >= '0' && ct_str[0] <= '9') {
1209 int oarg;
1210 oarg = ct_str[0] - '0';
1211 tcg_debug_assert(oarg < def->nb_oargs);
1212 tcg_debug_assert(def->args_ct[oarg].ct & TCG_CT_REG);
1213 /* TCG_CT_ALIAS is for the output arguments. The input
1214 argument is tagged with TCG_CT_IALIAS. */
1215 def->args_ct[i] = def->args_ct[oarg];
1216 def->args_ct[oarg].ct = TCG_CT_ALIAS;
1217 def->args_ct[oarg].alias_index = i;
1218 def->args_ct[i].ct |= TCG_CT_IALIAS;
1219 def->args_ct[i].alias_index = oarg;
1220 } else {
1221 for(;;) {
1222 if (*ct_str == '\0')
1223 break;
1224 switch(*ct_str) {
1225 case 'i':
1226 def->args_ct[i].ct |= TCG_CT_CONST;
1227 ct_str++;
1228 break;
1229 default:
1230 if (target_parse_constraint(&def->args_ct[i], &ct_str) < 0) {
1231 fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n",
1232 ct_str, i, def->name);
1233 exit(1);
1234 }
1235 }
1236 }
1237 }
1238 }
1239
1240 /* TCGTargetOpDef entry with too much information? */
1241 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
1242
1243 /* sort the constraints (XXX: this is just an heuristic) */
1244 sort_constraints(def, 0, def->nb_oargs);
1245 sort_constraints(def, def->nb_oargs, def->nb_iargs);
1246
1247 #if 0
1248 {
1249 int i;
1250
1251 printf("%s: sorted=", def->name);
1252 for(i = 0; i < def->nb_oargs + def->nb_iargs; i++)
1253 printf(" %d", def->sorted_args[i]);
1254 printf("\n");
1255 }
1256 #endif
1257 tdefs++;
1258 }
1259
1260 #if defined(CONFIG_DEBUG_TCG)
1261 i = 0;
1262 for (op = 0; op < tcg_op_defs_max; op++) {
1263 const TCGOpDef *def = &tcg_op_defs[op];
1264 if (def->flags & TCG_OPF_NOT_PRESENT) {
1265 /* Wrong entry in op definitions? */
1266 if (def->used) {
1267 fprintf(stderr, "Invalid op definition for %s\n", def->name);
1268 i = 1;
1269 }
1270 } else {
1271 /* Missing entry in op definitions? */
1272 if (!def->used) {
1273 fprintf(stderr, "Missing op definition for %s\n", def->name);
1274 i = 1;
1275 }
1276 }
1277 }
1278 if (i == 1) {
1279 tcg_abort();
1280 }
1281 #endif
1282 }
1283
1284 void tcg_op_remove(TCGContext *s, TCGOp *op)
1285 {
1286 int next = op->next;
1287 int prev = op->prev;
1288
1289 if (next >= 0) {
1290 s->gen_op_buf[next].prev = prev;
1291 } else {
1292 s->gen_last_op_idx = prev;
1293 }
1294 if (prev >= 0) {
1295 s->gen_op_buf[prev].next = next;
1296 } else {
1297 s->gen_first_op_idx = next;
1298 }
1299
1300 memset(op, -1, sizeof(*op));
1301
1302 #ifdef CONFIG_PROFILER
1303 s->del_op_count++;
1304 #endif
1305 }
1306
1307 #ifdef USE_LIVENESS_ANALYSIS
1308 /* liveness analysis: end of function: all temps are dead, and globals
1309 should be in memory. */
1310 static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps,
1311 uint8_t *mem_temps)
1312 {
1313 memset(dead_temps, 1, s->nb_temps);
1314 memset(mem_temps, 1, s->nb_globals);
1315 memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals);
1316 }
1317
1318 /* liveness analysis: end of basic block: all temps are dead, globals
1319 and local temps should be in memory. */
1320 static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps,
1321 uint8_t *mem_temps)
1322 {
1323 int i;
1324
1325 memset(dead_temps, 1, s->nb_temps);
1326 memset(mem_temps, 1, s->nb_globals);
1327 for(i = s->nb_globals; i < s->nb_temps; i++) {
1328 mem_temps[i] = s->temps[i].temp_local;
1329 }
1330 }
1331
1332 /* Liveness analysis : update the opc_dead_args array to tell if a
1333 given input arguments is dead. Instructions updating dead
1334 temporaries are removed. */
1335 static void tcg_liveness_analysis(TCGContext *s)
1336 {
1337 uint8_t *dead_temps, *mem_temps;
1338 int oi, oi_prev, nb_ops;
1339
1340 nb_ops = s->gen_next_op_idx;
1341 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
1342 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
1343
1344 dead_temps = tcg_malloc(s->nb_temps);
1345 mem_temps = tcg_malloc(s->nb_temps);
1346 tcg_la_func_end(s, dead_temps, mem_temps);
1347
1348 for (oi = s->gen_last_op_idx; oi >= 0; oi = oi_prev) {
1349 int i, nb_iargs, nb_oargs;
1350 TCGOpcode opc_new, opc_new2;
1351 bool have_opc_new2;
1352 uint16_t dead_args;
1353 uint8_t sync_args;
1354 TCGArg arg;
1355
1356 TCGOp * const op = &s->gen_op_buf[oi];
1357 TCGArg * const args = &s->gen_opparam_buf[op->args];
1358 TCGOpcode opc = op->opc;
1359 const TCGOpDef *def = &tcg_op_defs[opc];
1360
1361 oi_prev = op->prev;
1362
1363 switch (opc) {
1364 case INDEX_op_call:
1365 {
1366 int call_flags;
1367
1368 nb_oargs = op->callo;
1369 nb_iargs = op->calli;
1370 call_flags = args[nb_oargs + nb_iargs + 1];
1371
1372 /* pure functions can be removed if their result is unused */
1373 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
1374 for (i = 0; i < nb_oargs; i++) {
1375 arg = args[i];
1376 if (!dead_temps[arg] || mem_temps[arg]) {
1377 goto do_not_remove_call;
1378 }
1379 }
1380 goto do_remove;
1381 } else {
1382 do_not_remove_call:
1383
1384 /* output args are dead */
1385 dead_args = 0;
1386 sync_args = 0;
1387 for (i = 0; i < nb_oargs; i++) {
1388 arg = args[i];
1389 if (dead_temps[arg]) {
1390 dead_args |= (1 << i);
1391 }
1392 if (mem_temps[arg]) {
1393 sync_args |= (1 << i);
1394 }
1395 dead_temps[arg] = 1;
1396 mem_temps[arg] = 0;
1397 }
1398
1399 if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
1400 /* globals should be synced to memory */
1401 memset(mem_temps, 1, s->nb_globals);
1402 }
1403 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
1404 TCG_CALL_NO_READ_GLOBALS))) {
1405 /* globals should go back to memory */
1406 memset(dead_temps, 1, s->nb_globals);
1407 }
1408
1409 /* record arguments that die in this helper */
1410 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
1411 arg = args[i];
1412 if (arg != TCG_CALL_DUMMY_ARG) {
1413 if (dead_temps[arg]) {
1414 dead_args |= (1 << i);
1415 }
1416 }
1417 }
1418 /* input arguments are live for preceding opcodes */
1419 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
1420 arg = args[i];
1421 dead_temps[arg] = 0;
1422 }
1423 s->op_dead_args[oi] = dead_args;
1424 s->op_sync_args[oi] = sync_args;
1425 }
1426 }
1427 break;
1428 case INDEX_op_insn_start:
1429 break;
1430 case INDEX_op_discard:
1431 /* mark the temporary as dead */
1432 dead_temps[args[0]] = 1;
1433 mem_temps[args[0]] = 0;
1434 break;
1435
1436 case INDEX_op_add2_i32:
1437 opc_new = INDEX_op_add_i32;
1438 goto do_addsub2;
1439 case INDEX_op_sub2_i32:
1440 opc_new = INDEX_op_sub_i32;
1441 goto do_addsub2;
1442 case INDEX_op_add2_i64:
1443 opc_new = INDEX_op_add_i64;
1444 goto do_addsub2;
1445 case INDEX_op_sub2_i64:
1446 opc_new = INDEX_op_sub_i64;
1447 do_addsub2:
1448 nb_iargs = 4;
1449 nb_oargs = 2;
1450 /* Test if the high part of the operation is dead, but not
1451 the low part. The result can be optimized to a simple
1452 add or sub. This happens often for x86_64 guest when the
1453 cpu mode is set to 32 bit. */
1454 if (dead_temps[args[1]] && !mem_temps[args[1]]) {
1455 if (dead_temps[args[0]] && !mem_temps[args[0]]) {
1456 goto do_remove;
1457 }
1458 /* Replace the opcode and adjust the args in place,
1459 leaving 3 unused args at the end. */
1460 op->opc = opc = opc_new;
1461 args[1] = args[2];
1462 args[2] = args[4];
1463 /* Fall through and mark the single-word operation live. */
1464 nb_iargs = 2;
1465 nb_oargs = 1;
1466 }
1467 goto do_not_remove;
1468
1469 case INDEX_op_mulu2_i32:
1470 opc_new = INDEX_op_mul_i32;
1471 opc_new2 = INDEX_op_muluh_i32;
1472 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
1473 goto do_mul2;
1474 case INDEX_op_muls2_i32:
1475 opc_new = INDEX_op_mul_i32;
1476 opc_new2 = INDEX_op_mulsh_i32;
1477 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
1478 goto do_mul2;
1479 case INDEX_op_mulu2_i64:
1480 opc_new = INDEX_op_mul_i64;
1481 opc_new2 = INDEX_op_muluh_i64;
1482 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
1483 goto do_mul2;
1484 case INDEX_op_muls2_i64:
1485 opc_new = INDEX_op_mul_i64;
1486 opc_new2 = INDEX_op_mulsh_i64;
1487 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
1488 goto do_mul2;
1489 do_mul2:
1490 nb_iargs = 2;
1491 nb_oargs = 2;
1492 if (dead_temps[args[1]] && !mem_temps[args[1]]) {
1493 if (dead_temps[args[0]] && !mem_temps[args[0]]) {
1494 /* Both parts of the operation are dead. */
1495 goto do_remove;
1496 }
1497 /* The high part of the operation is dead; generate the low. */
1498 op->opc = opc = opc_new;
1499 args[1] = args[2];
1500 args[2] = args[3];
1501 } else if (have_opc_new2 && dead_temps[args[0]]
1502 && !mem_temps[args[0]]) {
1503 /* The low part of the operation is dead; generate the high. */
1504 op->opc = opc = opc_new2;
1505 args[0] = args[1];
1506 args[1] = args[2];
1507 args[2] = args[3];
1508 } else {
1509 goto do_not_remove;
1510 }
1511 /* Mark the single-word operation live. */
1512 nb_oargs = 1;
1513 goto do_not_remove;
1514
1515 default:
1516 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
1517 nb_iargs = def->nb_iargs;
1518 nb_oargs = def->nb_oargs;
1519
1520 /* Test if the operation can be removed because all
1521 its outputs are dead. We assume that nb_oargs == 0
1522 implies side effects */
1523 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
1524 for (i = 0; i < nb_oargs; i++) {
1525 arg = args[i];
1526 if (!dead_temps[arg] || mem_temps[arg]) {
1527 goto do_not_remove;
1528 }
1529 }
1530 do_remove:
1531 tcg_op_remove(s, op);
1532 } else {
1533 do_not_remove:
1534 /* output args are dead */
1535 dead_args = 0;
1536 sync_args = 0;
1537 for (i = 0; i < nb_oargs; i++) {
1538 arg = args[i];
1539 if (dead_temps[arg]) {
1540 dead_args |= (1 << i);
1541 }
1542 if (mem_temps[arg]) {
1543 sync_args |= (1 << i);
1544 }
1545 dead_temps[arg] = 1;
1546 mem_temps[arg] = 0;
1547 }
1548
1549 /* if end of basic block, update */
1550 if (def->flags & TCG_OPF_BB_END) {
1551 tcg_la_bb_end(s, dead_temps, mem_temps);
1552 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
1553 /* globals should be synced to memory */
1554 memset(mem_temps, 1, s->nb_globals);
1555 }
1556
1557 /* record arguments that die in this opcode */
1558 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
1559 arg = args[i];
1560 if (dead_temps[arg]) {
1561 dead_args |= (1 << i);
1562 }
1563 }
1564 /* input arguments are live for preceding opcodes */
1565 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
1566 arg = args[i];
1567 dead_temps[arg] = 0;
1568 }
1569 s->op_dead_args[oi] = dead_args;
1570 s->op_sync_args[oi] = sync_args;
1571 }
1572 break;
1573 }
1574 }
1575 }
1576 #else
1577 /* dummy liveness analysis */
1578 static void tcg_liveness_analysis(TCGContext *s)
1579 {
1580 int nb_ops = s->gen_next_op_idx;
1581
1582 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
1583 memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t));
1584 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
1585 memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t));
1586 }
1587 #endif
1588
1589 #ifdef CONFIG_DEBUG_TCG
1590 static void dump_regs(TCGContext *s)
1591 {
1592 TCGTemp *ts;
1593 int i;
1594 char buf[64];
1595
1596 for(i = 0; i < s->nb_temps; i++) {
1597 ts = &s->temps[i];
1598 printf(" %10s: ", tcg_get_arg_str_idx(s, buf, sizeof(buf), i));
1599 switch(ts->val_type) {
1600 case TEMP_VAL_REG:
1601 printf("%s", tcg_target_reg_names[ts->reg]);
1602 break;
1603 case TEMP_VAL_MEM:
1604 printf("%d(%s)", (int)ts->mem_offset,
1605 tcg_target_reg_names[ts->mem_base->reg]);
1606 break;
1607 case TEMP_VAL_CONST:
1608 printf("$0x%" TCG_PRIlx, ts->val);
1609 break;
1610 case TEMP_VAL_DEAD:
1611 printf("D");
1612 break;
1613 default:
1614 printf("???");
1615 break;
1616 }
1617 printf("\n");
1618 }
1619
1620 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
1621 if (s->reg_to_temp[i] != NULL) {
1622 printf("%s: %s\n",
1623 tcg_target_reg_names[i],
1624 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
1625 }
1626 }
1627 }
1628
1629 static void check_regs(TCGContext *s)
1630 {
1631 int reg;
1632 int k;
1633 TCGTemp *ts;
1634 char buf[64];
1635
1636 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
1637 ts = s->reg_to_temp[reg];
1638 if (ts != NULL) {
1639 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
1640 printf("Inconsistency for register %s:\n",
1641 tcg_target_reg_names[reg]);
1642 goto fail;
1643 }
1644 }
1645 }
1646 for (k = 0; k < s->nb_temps; k++) {
1647 ts = &s->temps[k];
1648 if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg
1649 && s->reg_to_temp[ts->reg] != ts) {
1650 printf("Inconsistency for temp %s:\n",
1651 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
1652 fail:
1653 printf("reg state:\n");
1654 dump_regs(s);
1655 tcg_abort();
1656 }
1657 }
1658 }
1659 #endif
1660
1661 static void temp_allocate_frame(TCGContext *s, int temp)
1662 {
1663 TCGTemp *ts;
1664 ts = &s->temps[temp];
1665 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
1666 /* Sparc64 stack is accessed with offset of 2047 */
1667 s->current_frame_offset = (s->current_frame_offset +
1668 (tcg_target_long)sizeof(tcg_target_long) - 1) &
1669 ~(sizeof(tcg_target_long) - 1);
1670 #endif
1671 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) >
1672 s->frame_end) {
1673 tcg_abort();
1674 }
1675 ts->mem_offset = s->current_frame_offset;
1676 ts->mem_base = s->frame_temp;
1677 ts->mem_allocated = 1;
1678 s->current_frame_offset += sizeof(tcg_target_long);
1679 }
1680
1681 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet);
1682
1683 /* sync register 'reg' by saving it to the corresponding temporary */
1684 static void tcg_reg_sync(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
1685 {
1686 TCGTemp *ts = s->reg_to_temp[reg];
1687
1688 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
1689 if (!ts->mem_coherent && !ts->fixed_reg) {
1690 if (!ts->mem_allocated) {
1691 temp_allocate_frame(s, temp_idx(s, ts));
1692 } else if (ts->indirect_reg) {
1693 tcg_regset_set_reg(allocated_regs, ts->reg);
1694 temp_load(s, ts->mem_base,
1695 tcg_target_available_regs[TCG_TYPE_PTR],
1696 allocated_regs);
1697 }
1698 tcg_out_st(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
1699 }
1700 ts->mem_coherent = 1;
1701 }
1702
1703 /* free register 'reg' by spilling the corresponding temporary if necessary */
1704 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
1705 {
1706 TCGTemp *ts = s->reg_to_temp[reg];
1707
1708 if (ts != NULL) {
1709 tcg_reg_sync(s, reg, allocated_regs);
1710 ts->val_type = TEMP_VAL_MEM;
1711 s->reg_to_temp[reg] = NULL;
1712 }
1713 }
1714
1715 /* Allocate a register belonging to reg1 & ~reg2 */
1716 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet desired_regs,
1717 TCGRegSet allocated_regs, bool rev)
1718 {
1719 int i, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
1720 const int *order;
1721 TCGReg reg;
1722 TCGRegSet reg_ct;
1723
1724 tcg_regset_andnot(reg_ct, desired_regs, allocated_regs);
1725 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
1726
1727 /* first try free registers */
1728 for(i = 0; i < n; i++) {
1729 reg = order[i];
1730 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == NULL)
1731 return reg;
1732 }
1733
1734 /* XXX: do better spill choice */
1735 for(i = 0; i < n; i++) {
1736 reg = order[i];
1737 if (tcg_regset_test_reg(reg_ct, reg)) {
1738 tcg_reg_free(s, reg, allocated_regs);
1739 return reg;
1740 }
1741 }
1742
1743 tcg_abort();
1744 }
1745
1746 /* Make sure the temporary is in a register. If needed, allocate the register
1747 from DESIRED while avoiding ALLOCATED. */
1748 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
1749 TCGRegSet allocated_regs)
1750 {
1751 TCGReg reg;
1752
1753 switch (ts->val_type) {
1754 case TEMP_VAL_REG:
1755 return;
1756 case TEMP_VAL_CONST:
1757 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base);
1758 tcg_out_movi(s, ts->type, reg, ts->val);
1759 ts->mem_coherent = 0;
1760 break;
1761 case TEMP_VAL_MEM:
1762 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base);
1763 if (ts->indirect_reg) {
1764 tcg_regset_set_reg(allocated_regs, reg);
1765 temp_load(s, ts->mem_base,
1766 tcg_target_available_regs[TCG_TYPE_PTR],
1767 allocated_regs);
1768 }
1769 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
1770 ts->mem_coherent = 1;
1771 break;
1772 case TEMP_VAL_DEAD:
1773 default:
1774 tcg_abort();
1775 }
1776 ts->reg = reg;
1777 ts->val_type = TEMP_VAL_REG;
1778 s->reg_to_temp[reg] = ts;
1779 }
1780
1781 /* mark a temporary as dead. */
1782 static inline void temp_dead(TCGContext *s, TCGTemp *ts)
1783 {
1784 if (ts->fixed_reg) {
1785 return;
1786 }
1787 if (ts->val_type == TEMP_VAL_REG) {
1788 s->reg_to_temp[ts->reg] = NULL;
1789 }
1790 ts->val_type = (temp_idx(s, ts) < s->nb_globals || ts->temp_local
1791 ? TEMP_VAL_MEM : TEMP_VAL_DEAD);
1792 }
1793
1794 /* sync a temporary to memory. 'allocated_regs' is used in case a
1795 temporary registers needs to be allocated to store a constant. */
1796 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1797 {
1798 if (ts->fixed_reg) {
1799 return;
1800 }
1801 switch (ts->val_type) {
1802 case TEMP_VAL_CONST:
1803 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs);
1804 /* fallthrough */
1805 case TEMP_VAL_REG:
1806 tcg_reg_sync(s, ts->reg, allocated_regs);
1807 break;
1808 case TEMP_VAL_DEAD:
1809 case TEMP_VAL_MEM:
1810 break;
1811 default:
1812 tcg_abort();
1813 }
1814 }
1815
1816 /* save a temporary to memory. 'allocated_regs' is used in case a
1817 temporary registers needs to be allocated to store a constant. */
1818 static inline void temp_save(TCGContext *s, TCGTemp *ts,
1819 TCGRegSet allocated_regs)
1820 {
1821 #ifdef USE_LIVENESS_ANALYSIS
1822 /* ??? Liveness does not yet incorporate indirect bases. */
1823 if (!ts->indirect_base) {
1824 /* The liveness analysis already ensures that globals are back
1825 in memory. Keep an tcg_debug_assert for safety. */
1826 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg);
1827 return;
1828 }
1829 #endif
1830 temp_sync(s, ts, allocated_regs);
1831 temp_dead(s, ts);
1832 }
1833
1834 /* save globals to their canonical location and assume they can be
1835 modified be the following code. 'allocated_regs' is used in case a
1836 temporary registers needs to be allocated to store a constant. */
1837 static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
1838 {
1839 int i;
1840
1841 for (i = 0; i < s->nb_globals; i++) {
1842 temp_save(s, &s->temps[i], allocated_regs);
1843 }
1844 }
1845
1846 /* sync globals to their canonical location and assume they can be
1847 read by the following code. 'allocated_regs' is used in case a
1848 temporary registers needs to be allocated to store a constant. */
1849 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
1850 {
1851 int i;
1852
1853 for (i = 0; i < s->nb_globals; i++) {
1854 TCGTemp *ts = &s->temps[i];
1855 #ifdef USE_LIVENESS_ANALYSIS
1856 /* ??? Liveness does not yet incorporate indirect bases. */
1857 if (!ts->indirect_base) {
1858 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
1859 || ts->fixed_reg
1860 || ts->mem_coherent);
1861 continue;
1862 }
1863 #endif
1864 temp_sync(s, ts, allocated_regs);
1865 }
1866 }
1867
1868 /* at the end of a basic block, we assume all temporaries are dead and
1869 all globals are stored at their canonical location. */
1870 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
1871 {
1872 int i;
1873
1874 for (i = s->nb_globals; i < s->nb_temps; i++) {
1875 TCGTemp *ts = &s->temps[i];
1876 if (ts->temp_local) {
1877 temp_save(s, ts, allocated_regs);
1878 } else {
1879 #ifdef USE_LIVENESS_ANALYSIS
1880 /* ??? Liveness does not yet incorporate indirect bases. */
1881 if (!ts->indirect_base) {
1882 /* The liveness analysis already ensures that temps are dead.
1883 Keep an tcg_debug_assert for safety. */
1884 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
1885 continue;
1886 }
1887 #endif
1888 temp_dead(s, ts);
1889 }
1890 }
1891
1892 save_globals(s, allocated_regs);
1893 }
1894
1895 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1)
1896 #define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1)
1897
1898 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args,
1899 uint16_t dead_args, uint8_t sync_args)
1900 {
1901 TCGTemp *ots;
1902 tcg_target_ulong val;
1903
1904 ots = &s->temps[args[0]];
1905 val = args[1];
1906
1907 if (ots->fixed_reg) {
1908 /* for fixed registers, we do not do any constant
1909 propagation */
1910 tcg_out_movi(s, ots->type, ots->reg, val);
1911 } else {
1912 /* The movi is not explicitly generated here */
1913 if (ots->val_type == TEMP_VAL_REG) {
1914 s->reg_to_temp[ots->reg] = NULL;
1915 }
1916 ots->val_type = TEMP_VAL_CONST;
1917 ots->val = val;
1918 }
1919 if (NEED_SYNC_ARG(0)) {
1920 temp_sync(s, ots, s->reserved_regs);
1921 }
1922 if (IS_DEAD_ARG(0)) {
1923 temp_dead(s, ots);
1924 }
1925 }
1926
1927 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def,
1928 const TCGArg *args, uint16_t dead_args,
1929 uint8_t sync_args)
1930 {
1931 TCGRegSet allocated_regs;
1932 TCGTemp *ts, *ots;
1933 TCGType otype, itype;
1934
1935 tcg_regset_set(allocated_regs, s->reserved_regs);
1936 ots = &s->temps[args[0]];
1937 ts = &s->temps[args[1]];
1938
1939 /* Note that otype != itype for no-op truncation. */
1940 otype = ots->type;
1941 itype = ts->type;
1942
1943 /* If the source value is not in a register, and we're going to be
1944 forced to have it in a register in order to perform the copy,
1945 then copy the SOURCE value into its own register first. That way
1946 we don't have to reload SOURCE the next time it is used. */
1947 if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG)
1948 || ts->val_type == TEMP_VAL_MEM) {
1949 temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs);
1950 }
1951
1952 if (IS_DEAD_ARG(0) && !ots->fixed_reg) {
1953 /* mov to a non-saved dead register makes no sense (even with
1954 liveness analysis disabled). */
1955 tcg_debug_assert(NEED_SYNC_ARG(0));
1956 /* The code above should have moved the temp to a register. */
1957 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
1958 if (!ots->mem_allocated) {
1959 temp_allocate_frame(s, args[0]);
1960 }
1961 if (ots->indirect_reg) {
1962 tcg_regset_set_reg(allocated_regs, ts->reg);
1963 temp_load(s, ots->mem_base,
1964 tcg_target_available_regs[TCG_TYPE_PTR],
1965 allocated_regs);
1966 }
1967 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
1968 if (IS_DEAD_ARG(1)) {
1969 temp_dead(s, ts);
1970 }
1971 temp_dead(s, ots);
1972 } else if (ts->val_type == TEMP_VAL_CONST) {
1973 /* propagate constant */
1974 if (ots->val_type == TEMP_VAL_REG) {
1975 s->reg_to_temp[ots->reg] = NULL;
1976 }
1977 ots->val_type = TEMP_VAL_CONST;
1978 ots->val = ts->val;
1979 if (IS_DEAD_ARG(1)) {
1980 temp_dead(s, ts);
1981 }
1982 } else {
1983 /* The code in the first if block should have moved the
1984 temp to a register. */
1985 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
1986 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) {
1987 /* the mov can be suppressed */
1988 if (ots->val_type == TEMP_VAL_REG) {
1989 s->reg_to_temp[ots->reg] = NULL;
1990 }
1991 ots->reg = ts->reg;
1992 temp_dead(s, ts);
1993 } else {
1994 if (ots->val_type != TEMP_VAL_REG) {
1995 /* When allocating a new register, make sure to not spill the
1996 input one. */
1997 tcg_regset_set_reg(allocated_regs, ts->reg);
1998 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
1999 allocated_regs, ots->indirect_base);
2000 }
2001 tcg_out_mov(s, otype, ots->reg, ts->reg);
2002 }
2003 ots->val_type = TEMP_VAL_REG;
2004 ots->mem_coherent = 0;
2005 s->reg_to_temp[ots->reg] = ots;
2006 if (NEED_SYNC_ARG(0)) {
2007 tcg_reg_sync(s, ots->reg, allocated_regs);
2008 }
2009 }
2010 }
2011
2012 static void tcg_reg_alloc_op(TCGContext *s,
2013 const TCGOpDef *def, TCGOpcode opc,
2014 const TCGArg *args, uint16_t dead_args,
2015 uint8_t sync_args)
2016 {
2017 TCGRegSet allocated_regs;
2018 int i, k, nb_iargs, nb_oargs;
2019 TCGReg reg;
2020 TCGArg arg;
2021 const TCGArgConstraint *arg_ct;
2022 TCGTemp *ts;
2023 TCGArg new_args[TCG_MAX_OP_ARGS];
2024 int const_args[TCG_MAX_OP_ARGS];
2025
2026 nb_oargs = def->nb_oargs;
2027 nb_iargs = def->nb_iargs;
2028
2029 /* copy constants */
2030 memcpy(new_args + nb_oargs + nb_iargs,
2031 args + nb_oargs + nb_iargs,
2032 sizeof(TCGArg) * def->nb_cargs);
2033
2034 /* satisfy input constraints */
2035 tcg_regset_set(allocated_regs, s->reserved_regs);
2036 for(k = 0; k < nb_iargs; k++) {
2037 i = def->sorted_args[nb_oargs + k];
2038 arg = args[i];
2039 arg_ct = &def->args_ct[i];
2040 ts = &s->temps[arg];
2041
2042 if (ts->val_type == TEMP_VAL_CONST
2043 && tcg_target_const_match(ts->val, ts->type, arg_ct)) {
2044 /* constant is OK for instruction */
2045 const_args[i] = 1;
2046 new_args[i] = ts->val;
2047 goto iarg_end;
2048 }
2049
2050 temp_load(s, ts, arg_ct->u.regs, allocated_regs);
2051
2052 if (arg_ct->ct & TCG_CT_IALIAS) {
2053 if (ts->fixed_reg) {
2054 /* if fixed register, we must allocate a new register
2055 if the alias is not the same register */
2056 if (arg != args[arg_ct->alias_index])
2057 goto allocate_in_reg;
2058 } else {
2059 /* if the input is aliased to an output and if it is
2060 not dead after the instruction, we must allocate
2061 a new register and move it */
2062 if (!IS_DEAD_ARG(i)) {
2063 goto allocate_in_reg;
2064 }
2065 /* check if the current register has already been allocated
2066 for another input aliased to an output */
2067 int k2, i2;
2068 for (k2 = 0 ; k2 < k ; k2++) {
2069 i2 = def->sorted_args[nb_oargs + k2];
2070 if ((def->args_ct[i2].ct & TCG_CT_IALIAS) &&
2071 (new_args[i2] == ts->reg)) {
2072 goto allocate_in_reg;
2073 }
2074 }
2075 }
2076 }
2077 reg = ts->reg;
2078 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) {
2079 /* nothing to do : the constraint is satisfied */
2080 } else {
2081 allocate_in_reg:
2082 /* allocate a new register matching the constraint
2083 and move the temporary register into it */
2084 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs,
2085 ts->indirect_base);
2086 tcg_out_mov(s, ts->type, reg, ts->reg);
2087 }
2088 new_args[i] = reg;
2089 const_args[i] = 0;
2090 tcg_regset_set_reg(allocated_regs, reg);
2091 iarg_end: ;
2092 }
2093
2094 /* mark dead temporaries and free the associated registers */
2095 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2096 if (IS_DEAD_ARG(i)) {
2097 temp_dead(s, &s->temps[args[i]]);
2098 }
2099 }
2100
2101 if (def->flags & TCG_OPF_BB_END) {
2102 tcg_reg_alloc_bb_end(s, allocated_regs);
2103 } else {
2104 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2105 /* XXX: permit generic clobber register list ? */
2106 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
2107 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
2108 tcg_reg_free(s, i, allocated_regs);
2109 }
2110 }
2111 }
2112 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2113 /* sync globals if the op has side effects and might trigger
2114 an exception. */
2115 sync_globals(s, allocated_regs);
2116 }
2117
2118 /* satisfy the output constraints */
2119 tcg_regset_set(allocated_regs, s->reserved_regs);
2120 for(k = 0; k < nb_oargs; k++) {
2121 i = def->sorted_args[k];
2122 arg = args[i];
2123 arg_ct = &def->args_ct[i];
2124 ts = &s->temps[arg];
2125 if (arg_ct->ct & TCG_CT_ALIAS) {
2126 reg = new_args[arg_ct->alias_index];
2127 } else {
2128 /* if fixed register, we try to use it */
2129 reg = ts->reg;
2130 if (ts->fixed_reg &&
2131 tcg_regset_test_reg(arg_ct->u.regs, reg)) {
2132 goto oarg_end;
2133 }
2134 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs,
2135 ts->indirect_base);
2136 }
2137 tcg_regset_set_reg(allocated_regs, reg);
2138 /* if a fixed register is used, then a move will be done afterwards */
2139 if (!ts->fixed_reg) {
2140 if (ts->val_type == TEMP_VAL_REG) {
2141 s->reg_to_temp[ts->reg] = NULL;
2142 }
2143 ts->val_type = TEMP_VAL_REG;
2144 ts->reg = reg;
2145 /* temp value is modified, so the value kept in memory is
2146 potentially not the same */
2147 ts->mem_coherent = 0;
2148 s->reg_to_temp[reg] = ts;
2149 }
2150 oarg_end:
2151 new_args[i] = reg;
2152 }
2153 }
2154
2155 /* emit instruction */
2156 tcg_out_op(s, opc, new_args, const_args);
2157
2158 /* move the outputs in the correct register if needed */
2159 for(i = 0; i < nb_oargs; i++) {
2160 ts = &s->temps[args[i]];
2161 reg = new_args[i];
2162 if (ts->fixed_reg && ts->reg != reg) {
2163 tcg_out_mov(s, ts->type, ts->reg, reg);
2164 }
2165 if (NEED_SYNC_ARG(i)) {
2166 tcg_reg_sync(s, reg, allocated_regs);
2167 }
2168 if (IS_DEAD_ARG(i)) {
2169 temp_dead(s, ts);
2170 }
2171 }
2172 }
2173
2174 #ifdef TCG_TARGET_STACK_GROWSUP
2175 #define STACK_DIR(x) (-(x))
2176 #else
2177 #define STACK_DIR(x) (x)
2178 #endif
2179
2180 static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs,
2181 const TCGArg * const args, uint16_t dead_args,
2182 uint8_t sync_args)
2183 {
2184 int flags, nb_regs, i;
2185 TCGReg reg;
2186 TCGArg arg;
2187 TCGTemp *ts;
2188 intptr_t stack_offset;
2189 size_t call_stack_size;
2190 tcg_insn_unit *func_addr;
2191 int allocate_args;
2192 TCGRegSet allocated_regs;
2193
2194 func_addr = (tcg_insn_unit *)(intptr_t)args[nb_oargs + nb_iargs];
2195 flags = args[nb_oargs + nb_iargs + 1];
2196
2197 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2198 if (nb_regs > nb_iargs) {
2199 nb_regs = nb_iargs;
2200 }
2201
2202 /* assign stack slots first */
2203 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
2204 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
2205 ~(TCG_TARGET_STACK_ALIGN - 1);
2206 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
2207 if (allocate_args) {
2208 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
2209 preallocate call stack */
2210 tcg_abort();
2211 }
2212
2213 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
2214 for(i = nb_regs; i < nb_iargs; i++) {
2215 arg = args[nb_oargs + i];
2216 #ifdef TCG_TARGET_STACK_GROWSUP
2217 stack_offset -= sizeof(tcg_target_long);
2218 #endif
2219 if (arg != TCG_CALL_DUMMY_ARG) {
2220 ts = &s->temps[arg];
2221 temp_load(s, ts, tcg_target_available_regs[ts->type],
2222 s->reserved_regs);
2223 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
2224 }
2225 #ifndef TCG_TARGET_STACK_GROWSUP
2226 stack_offset += sizeof(tcg_target_long);
2227 #endif
2228 }
2229
2230 /* assign input registers */
2231 tcg_regset_set(allocated_regs, s->reserved_regs);
2232 for(i = 0; i < nb_regs; i++) {
2233 arg = args[nb_oargs + i];
2234 if (arg != TCG_CALL_DUMMY_ARG) {
2235 ts = &s->temps[arg];
2236 reg = tcg_target_call_iarg_regs[i];
2237 tcg_reg_free(s, reg, allocated_regs);
2238
2239 if (ts->val_type == TEMP_VAL_REG) {
2240 if (ts->reg != reg) {
2241 tcg_out_mov(s, ts->type, reg, ts->reg);
2242 }
2243 } else {
2244 TCGRegSet arg_set;
2245
2246 tcg_regset_clear(arg_set);
2247 tcg_regset_set_reg(arg_set, reg);
2248 temp_load(s, ts, arg_set, allocated_regs);
2249 }
2250
2251 tcg_regset_set_reg(allocated_regs, reg);
2252 }
2253 }
2254
2255 /* mark dead temporaries and free the associated registers */
2256 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2257 if (IS_DEAD_ARG(i)) {
2258 temp_dead(s, &s->temps[args[i]]);
2259 }
2260 }
2261
2262 /* clobber call registers */
2263 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
2264 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
2265 tcg_reg_free(s, i, allocated_regs);
2266 }
2267 }
2268
2269 /* Save globals if they might be written by the helper, sync them if
2270 they might be read. */
2271 if (flags & TCG_CALL_NO_READ_GLOBALS) {
2272 /* Nothing to do */
2273 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
2274 sync_globals(s, allocated_regs);
2275 } else {
2276 save_globals(s, allocated_regs);
2277 }
2278
2279 tcg_out_call(s, func_addr);
2280
2281 /* assign output registers and emit moves if needed */
2282 for(i = 0; i < nb_oargs; i++) {
2283 arg = args[i];
2284 ts = &s->temps[arg];
2285 reg = tcg_target_call_oarg_regs[i];
2286 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
2287
2288 if (ts->fixed_reg) {
2289 if (ts->reg != reg) {
2290 tcg_out_mov(s, ts->type, ts->reg, reg);
2291 }
2292 } else {
2293 if (ts->val_type == TEMP_VAL_REG) {
2294 s->reg_to_temp[ts->reg] = NULL;
2295 }
2296 ts->val_type = TEMP_VAL_REG;
2297 ts->reg = reg;
2298 ts->mem_coherent = 0;
2299 s->reg_to_temp[reg] = ts;
2300 if (NEED_SYNC_ARG(i)) {
2301 tcg_reg_sync(s, reg, allocated_regs);
2302 }
2303 if (IS_DEAD_ARG(i)) {
2304 temp_dead(s, ts);
2305 }
2306 }
2307 }
2308 }
2309
2310 #ifdef CONFIG_PROFILER
2311
2312 static int64_t tcg_table_op_count[NB_OPS];
2313
2314 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
2315 {
2316 int i;
2317
2318 for (i = 0; i < NB_OPS; i++) {
2319 cpu_fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name,
2320 tcg_table_op_count[i]);
2321 }
2322 }
2323 #else
2324 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
2325 {
2326 cpu_fprintf(f, "[TCG profiler not compiled]\n");
2327 }
2328 #endif
2329
2330
2331 int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
2332 {
2333 int i, oi, oi_next, num_insns;
2334
2335 #ifdef CONFIG_PROFILER
2336 {
2337 int n;
2338
2339 n = s->gen_last_op_idx + 1;
2340 s->op_count += n;
2341 if (n > s->op_count_max) {
2342 s->op_count_max = n;
2343 }
2344
2345 n = s->nb_temps;
2346 s->temp_count += n;
2347 if (n > s->temp_count_max) {
2348 s->temp_count_max = n;
2349 }
2350 }
2351 #endif
2352
2353 #ifdef DEBUG_DISAS
2354 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
2355 && qemu_log_in_addr_range(tb->pc))) {
2356 qemu_log("OP:\n");
2357 tcg_dump_ops(s);
2358 qemu_log("\n");
2359 }
2360 #endif
2361
2362 #ifdef CONFIG_PROFILER
2363 s->opt_time -= profile_getclock();
2364 #endif
2365
2366 #ifdef USE_TCG_OPTIMIZATIONS
2367 tcg_optimize(s);
2368 #endif
2369
2370 #ifdef CONFIG_PROFILER
2371 s->opt_time += profile_getclock();
2372 s->la_time -= profile_getclock();
2373 #endif
2374
2375 tcg_liveness_analysis(s);
2376
2377 #ifdef CONFIG_PROFILER
2378 s->la_time += profile_getclock();
2379 #endif
2380
2381 #ifdef DEBUG_DISAS
2382 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
2383 && qemu_log_in_addr_range(tb->pc))) {
2384 qemu_log("OP after optimization and liveness analysis:\n");
2385 tcg_dump_ops(s);
2386 qemu_log("\n");
2387 }
2388 #endif
2389
2390 tcg_reg_alloc_start(s);
2391
2392 s->code_buf = tb->tc_ptr;
2393 s->code_ptr = tb->tc_ptr;
2394
2395 tcg_out_tb_init(s);
2396
2397 num_insns = -1;
2398 for (oi = s->gen_first_op_idx; oi >= 0; oi = oi_next) {
2399 TCGOp * const op = &s->gen_op_buf[oi];
2400 TCGArg * const args = &s->gen_opparam_buf[op->args];
2401 TCGOpcode opc = op->opc;
2402 const TCGOpDef *def = &tcg_op_defs[opc];
2403 uint16_t dead_args = s->op_dead_args[oi];
2404 uint8_t sync_args = s->op_sync_args[oi];
2405
2406 oi_next = op->next;
2407 #ifdef CONFIG_PROFILER
2408 tcg_table_op_count[opc]++;
2409 #endif
2410
2411 switch (opc) {
2412 case INDEX_op_mov_i32:
2413 case INDEX_op_mov_i64:
2414 tcg_reg_alloc_mov(s, def, args, dead_args, sync_args);
2415 break;
2416 case INDEX_op_movi_i32:
2417 case INDEX_op_movi_i64:
2418 tcg_reg_alloc_movi(s, args, dead_args, sync_args);
2419 break;
2420 case INDEX_op_insn_start:
2421 if (num_insns >= 0) {
2422 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
2423 }
2424 num_insns++;
2425 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
2426 target_ulong a;
2427 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
2428 a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2];
2429 #else
2430 a = args[i];
2431 #endif
2432 s->gen_insn_data[num_insns][i] = a;
2433 }
2434 break;
2435 case INDEX_op_discard:
2436 temp_dead(s, &s->temps[args[0]]);
2437 break;
2438 case INDEX_op_set_label:
2439 tcg_reg_alloc_bb_end(s, s->reserved_regs);
2440 tcg_out_label(s, arg_label(args[0]), s->code_ptr);
2441 break;
2442 case INDEX_op_call:
2443 tcg_reg_alloc_call(s, op->callo, op->calli, args,
2444 dead_args, sync_args);
2445 break;
2446 default:
2447 /* Sanity check that we've not introduced any unhandled opcodes. */
2448 if (def->flags & TCG_OPF_NOT_PRESENT) {
2449 tcg_abort();
2450 }
2451 /* Note: in order to speed up the code, it would be much
2452 faster to have specialized register allocator functions for
2453 some common argument patterns */
2454 tcg_reg_alloc_op(s, def, opc, args, dead_args, sync_args);
2455 break;
2456 }
2457 #ifdef CONFIG_DEBUG_TCG
2458 check_regs(s);
2459 #endif
2460 /* Test for (pending) buffer overflow. The assumption is that any
2461 one operation beginning below the high water mark cannot overrun
2462 the buffer completely. Thus we can test for overflow after
2463 generating code without having to check during generation. */
2464 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
2465 return -1;
2466 }
2467 }
2468 tcg_debug_assert(num_insns >= 0);
2469 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
2470
2471 /* Generate TB finalization at the end of block */
2472 if (!tcg_out_tb_finalize(s)) {
2473 return -1;
2474 }
2475
2476 /* flush instruction cache */
2477 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
2478
2479 return tcg_current_code_size(s);
2480 }
2481
2482 #ifdef CONFIG_PROFILER
2483 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
2484 {
2485 TCGContext *s = &tcg_ctx;
2486 int64_t tb_count = s->tb_count;
2487 int64_t tb_div_count = tb_count ? tb_count : 1;
2488 int64_t tot = s->interm_time + s->code_time;
2489
2490 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n",
2491 tot, tot / 2.4e9);
2492 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n",
2493 tb_count, s->tb_count1 - tb_count,
2494 (double)(s->tb_count1 - s->tb_count)
2495 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
2496 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n",
2497 (double)s->op_count / tb_div_count, s->op_count_max);
2498 cpu_fprintf(f, "deleted ops/TB %0.2f\n",
2499 (double)s->del_op_count / tb_div_count);
2500 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n",
2501 (double)s->temp_count / tb_div_count, s->temp_count_max);
2502 cpu_fprintf(f, "avg host code/TB %0.1f\n",
2503 (double)s->code_out_len / tb_div_count);
2504 cpu_fprintf(f, "avg search data/TB %0.1f\n",
2505 (double)s->search_out_len / tb_div_count);
2506
2507 cpu_fprintf(f, "cycles/op %0.1f\n",
2508 s->op_count ? (double)tot / s->op_count : 0);
2509 cpu_fprintf(f, "cycles/in byte %0.1f\n",
2510 s->code_in_len ? (double)tot / s->code_in_len : 0);
2511 cpu_fprintf(f, "cycles/out byte %0.1f\n",
2512 s->code_out_len ? (double)tot / s->code_out_len : 0);
2513 cpu_fprintf(f, "cycles/search byte %0.1f\n",
2514 s->search_out_len ? (double)tot / s->search_out_len : 0);
2515 if (tot == 0) {
2516 tot = 1;
2517 }
2518 cpu_fprintf(f, " gen_interm time %0.1f%%\n",
2519 (double)s->interm_time / tot * 100.0);
2520 cpu_fprintf(f, " gen_code time %0.1f%%\n",
2521 (double)s->code_time / tot * 100.0);
2522 cpu_fprintf(f, "optim./code time %0.1f%%\n",
2523 (double)s->opt_time / (s->code_time ? s->code_time : 1)
2524 * 100.0);
2525 cpu_fprintf(f, "liveness/code time %0.1f%%\n",
2526 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
2527 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n",
2528 s->restore_count);
2529 cpu_fprintf(f, " avg cycles %0.1f\n",
2530 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
2531 }
2532 #else
2533 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
2534 {
2535 cpu_fprintf(f, "[TCG profiler not compiled]\n");
2536 }
2537 #endif
2538
2539 #ifdef ELF_HOST_MACHINE
2540 /* In order to use this feature, the backend needs to do three things:
2541
2542 (1) Define ELF_HOST_MACHINE to indicate both what value to
2543 put into the ELF image and to indicate support for the feature.
2544
2545 (2) Define tcg_register_jit. This should create a buffer containing
2546 the contents of a .debug_frame section that describes the post-
2547 prologue unwind info for the tcg machine.
2548
2549 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
2550 */
2551
2552 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
2553 typedef enum {
2554 JIT_NOACTION = 0,
2555 JIT_REGISTER_FN,
2556 JIT_UNREGISTER_FN
2557 } jit_actions_t;
2558
2559 struct jit_code_entry {
2560 struct jit_code_entry *next_entry;
2561 struct jit_code_entry *prev_entry;
2562 const void *symfile_addr;
2563 uint64_t symfile_size;
2564 };
2565
2566 struct jit_descriptor {
2567 uint32_t version;
2568 uint32_t action_flag;
2569 struct jit_code_entry *relevant_entry;
2570 struct jit_code_entry *first_entry;
2571 };
2572
2573 void __jit_debug_register_code(void) __attribute__((noinline));
2574 void __jit_debug_register_code(void)
2575 {
2576 asm("");
2577 }
2578
2579 /* Must statically initialize the version, because GDB may check
2580 the version before we can set it. */
2581 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
2582
2583 /* End GDB interface. */
2584
2585 static int find_string(const char *strtab, const char *str)
2586 {
2587 const char *p = strtab + 1;
2588
2589 while (1) {
2590 if (strcmp(p, str) == 0) {
2591 return p - strtab;
2592 }
2593 p += strlen(p) + 1;
2594 }
2595 }
2596
2597 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size,
2598 const void *debug_frame,
2599 size_t debug_frame_size)
2600 {
2601 struct __attribute__((packed)) DebugInfo {
2602 uint32_t len;
2603 uint16_t version;
2604 uint32_t abbrev;
2605 uint8_t ptr_size;
2606 uint8_t cu_die;
2607 uint16_t cu_lang;
2608 uintptr_t cu_low_pc;
2609 uintptr_t cu_high_pc;
2610 uint8_t fn_die;
2611 char fn_name[16];
2612 uintptr_t fn_low_pc;
2613 uintptr_t fn_high_pc;
2614 uint8_t cu_eoc;
2615 };
2616
2617 struct ElfImage {
2618 ElfW(Ehdr) ehdr;
2619 ElfW(Phdr) phdr;
2620 ElfW(Shdr) shdr[7];
2621 ElfW(Sym) sym[2];
2622 struct DebugInfo di;
2623 uint8_t da[24];
2624 char str[80];
2625 };
2626
2627 struct ElfImage *img;
2628
2629 static const struct ElfImage img_template = {
2630 .ehdr = {
2631 .e_ident[EI_MAG0] = ELFMAG0,
2632 .e_ident[EI_MAG1] = ELFMAG1,
2633 .e_ident[EI_MAG2] = ELFMAG2,
2634 .e_ident[EI_MAG3] = ELFMAG3,
2635 .e_ident[EI_CLASS] = ELF_CLASS,
2636 .e_ident[EI_DATA] = ELF_DATA,
2637 .e_ident[EI_VERSION] = EV_CURRENT,
2638 .e_type = ET_EXEC,
2639 .e_machine = ELF_HOST_MACHINE,
2640 .e_version = EV_CURRENT,
2641 .e_phoff = offsetof(struct ElfImage, phdr),
2642 .e_shoff = offsetof(struct ElfImage, shdr),
2643 .e_ehsize = sizeof(ElfW(Shdr)),
2644 .e_phentsize = sizeof(ElfW(Phdr)),
2645 .e_phnum = 1,
2646 .e_shentsize = sizeof(ElfW(Shdr)),
2647 .e_shnum = ARRAY_SIZE(img->shdr),
2648 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
2649 #ifdef ELF_HOST_FLAGS
2650 .e_flags = ELF_HOST_FLAGS,
2651 #endif
2652 #ifdef ELF_OSABI
2653 .e_ident[EI_OSABI] = ELF_OSABI,
2654 #endif
2655 },
2656 .phdr = {
2657 .p_type = PT_LOAD,
2658 .p_flags = PF_X,
2659 },
2660 .shdr = {
2661 [0] = { .sh_type = SHT_NULL },
2662 /* Trick: The contents of code_gen_buffer are not present in
2663 this fake ELF file; that got allocated elsewhere. Therefore
2664 we mark .text as SHT_NOBITS (similar to .bss) so that readers
2665 will not look for contents. We can record any address. */
2666 [1] = { /* .text */
2667 .sh_type = SHT_NOBITS,
2668 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
2669 },
2670 [2] = { /* .debug_info */
2671 .sh_type = SHT_PROGBITS,
2672 .sh_offset = offsetof(struct ElfImage, di),
2673 .sh_size = sizeof(struct DebugInfo),
2674 },
2675 [3] = { /* .debug_abbrev */
2676 .sh_type = SHT_PROGBITS,
2677 .sh_offset = offsetof(struct ElfImage, da),
2678 .sh_size = sizeof(img->da),
2679 },
2680 [4] = { /* .debug_frame */
2681 .sh_type = SHT_PROGBITS,
2682 .sh_offset = sizeof(struct ElfImage),
2683 },
2684 [5] = { /* .symtab */
2685 .sh_type = SHT_SYMTAB,
2686 .sh_offset = offsetof(struct ElfImage, sym),
2687 .sh_size = sizeof(img->sym),
2688 .sh_info = 1,
2689 .sh_link = ARRAY_SIZE(img->shdr) - 1,
2690 .sh_entsize = sizeof(ElfW(Sym)),
2691 },
2692 [6] = { /* .strtab */
2693 .sh_type = SHT_STRTAB,
2694 .sh_offset = offsetof(struct ElfImage, str),
2695 .sh_size = sizeof(img->str),
2696 }
2697 },
2698 .sym = {
2699 [1] = { /* code_gen_buffer */
2700 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
2701 .st_shndx = 1,
2702 }
2703 },
2704 .di = {
2705 .len = sizeof(struct DebugInfo) - 4,
2706 .version = 2,
2707 .ptr_size = sizeof(void *),
2708 .cu_die = 1,
2709 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
2710 .fn_die = 2,
2711 .fn_name = "code_gen_buffer"
2712 },
2713 .da = {
2714 1, /* abbrev number (the cu) */
2715 0x11, 1, /* DW_TAG_compile_unit, has children */
2716 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
2717 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
2718 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
2719 0, 0, /* end of abbrev */
2720 2, /* abbrev number (the fn) */
2721 0x2e, 0, /* DW_TAG_subprogram, no children */
2722 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
2723 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
2724 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
2725 0, 0, /* end of abbrev */
2726 0 /* no more abbrev */
2727 },
2728 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
2729 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
2730 };
2731
2732 /* We only need a single jit entry; statically allocate it. */
2733 static struct jit_code_entry one_entry;
2734
2735 uintptr_t buf = (uintptr_t)buf_ptr;
2736 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2737 DebugFrameHeader *dfh;
2738
2739 img = g_malloc(img_size);
2740 *img = img_template;
2741
2742 img->phdr.p_vaddr = buf;
2743 img->phdr.p_paddr = buf;
2744 img->phdr.p_memsz = buf_size;
2745
2746 img->shdr[1].sh_name = find_string(img->str, ".text");
2747 img->shdr[1].sh_addr = buf;
2748 img->shdr[1].sh_size = buf_size;
2749
2750 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
2751 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
2752
2753 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
2754 img->shdr[4].sh_size = debug_frame_size;
2755
2756 img->shdr[5].sh_name = find_string(img->str, ".symtab");
2757 img->shdr[6].sh_name = find_string(img->str, ".strtab");
2758
2759 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
2760 img->sym[1].st_value = buf;
2761 img->sym[1].st_size = buf_size;
2762
2763 img->di.cu_low_pc = buf;
2764 img->di.cu_high_pc = buf + buf_size;
2765 img->di.fn_low_pc = buf;
2766 img->di.fn_high_pc = buf + buf_size;
2767
2768 dfh = (DebugFrameHeader *)(img + 1);
2769 memcpy(dfh, debug_frame, debug_frame_size);
2770 dfh->fde.func_start = buf;
2771 dfh->fde.func_len = buf_size;
2772
2773 #ifdef DEBUG_JIT
2774 /* Enable this block to be able to debug the ELF image file creation.
2775 One can use readelf, objdump, or other inspection utilities. */
2776 {
2777 FILE *f = fopen("/tmp/qemu.jit", "w+b");
2778 if (f) {
2779 if (fwrite(img, img_size, 1, f) != img_size) {
2780 /* Avoid stupid unused return value warning for fwrite. */
2781 }
2782 fclose(f);
2783 }
2784 }
2785 #endif
2786
2787 one_entry.symfile_addr = img;
2788 one_entry.symfile_size = img_size;
2789
2790 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
2791 __jit_debug_descriptor.relevant_entry = &one_entry;
2792 __jit_debug_descriptor.first_entry = &one_entry;
2793 __jit_debug_register_code();
2794 }
2795 #else
2796 /* No support for the feature. Provide the entry point expected by exec.c,
2797 and implement the internal function we declared earlier. */
2798
2799 static void tcg_register_jit_int(void *buf, size_t size,
2800 const void *debug_frame,
2801 size_t debug_frame_size)
2802 {
2803 }
2804
2805 void tcg_register_jit(void *buf, size_t buf_size)
2806 {
2807 }
2808 #endif /* ELF_HOST_MACHINE */