target/arm: Avoid maybe-uninitialized warning with gcc 4.9
[qemu.git] / target / arm / translate-a64.c
1 /*
2 * AArch64 translation
3 *
4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "qemu/osdep.h"
20
21 #include "cpu.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "qemu/log.h"
26 #include "arm_ldst.h"
27 #include "translate.h"
28 #include "internals.h"
29 #include "qemu/host-utils.h"
30
31 #include "hw/semihosting/semihost.h"
32 #include "exec/gen-icount.h"
33
34 #include "exec/helper-proto.h"
35 #include "exec/helper-gen.h"
36 #include "exec/log.h"
37
38 #include "trace-tcg.h"
39 #include "translate-a64.h"
40 #include "qemu/atomic128.h"
41
42 static TCGv_i64 cpu_X[32];
43 static TCGv_i64 cpu_pc;
44
45 /* Load/store exclusive handling */
46 static TCGv_i64 cpu_exclusive_high;
47
48 static const char *regnames[] = {
49 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
50 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
51 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
52 "x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp"
53 };
54
55 enum a64_shift_type {
56 A64_SHIFT_TYPE_LSL = 0,
57 A64_SHIFT_TYPE_LSR = 1,
58 A64_SHIFT_TYPE_ASR = 2,
59 A64_SHIFT_TYPE_ROR = 3
60 };
61
62 /* Table based decoder typedefs - used when the relevant bits for decode
63 * are too awkwardly scattered across the instruction (eg SIMD).
64 */
65 typedef void AArch64DecodeFn(DisasContext *s, uint32_t insn);
66
67 typedef struct AArch64DecodeTable {
68 uint32_t pattern;
69 uint32_t mask;
70 AArch64DecodeFn *disas_fn;
71 } AArch64DecodeTable;
72
73 /* initialize TCG globals. */
74 void a64_translate_init(void)
75 {
76 int i;
77
78 cpu_pc = tcg_global_mem_new_i64(cpu_env,
79 offsetof(CPUARMState, pc),
80 "pc");
81 for (i = 0; i < 32; i++) {
82 cpu_X[i] = tcg_global_mem_new_i64(cpu_env,
83 offsetof(CPUARMState, xregs[i]),
84 regnames[i]);
85 }
86
87 cpu_exclusive_high = tcg_global_mem_new_i64(cpu_env,
88 offsetof(CPUARMState, exclusive_high), "exclusive_high");
89 }
90
91 /*
92 * Return the core mmu_idx to use for A64 "unprivileged load/store" insns
93 */
94 static int get_a64_user_mem_index(DisasContext *s)
95 {
96 /*
97 * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL,
98 * which is the usual mmu_idx for this cpu state.
99 */
100 ARMMMUIdx useridx = s->mmu_idx;
101
102 if (s->unpriv) {
103 /*
104 * We have pre-computed the condition for AccType_UNPRIV.
105 * Therefore we should never get here with a mmu_idx for
106 * which we do not know the corresponding user mmu_idx.
107 */
108 switch (useridx) {
109 case ARMMMUIdx_E10_1:
110 case ARMMMUIdx_E10_1_PAN:
111 useridx = ARMMMUIdx_E10_0;
112 break;
113 case ARMMMUIdx_E20_2:
114 case ARMMMUIdx_E20_2_PAN:
115 useridx = ARMMMUIdx_E20_0;
116 break;
117 case ARMMMUIdx_SE10_1:
118 case ARMMMUIdx_SE10_1_PAN:
119 useridx = ARMMMUIdx_SE10_0;
120 break;
121 default:
122 g_assert_not_reached();
123 }
124 }
125 return arm_to_core_mmu_idx(useridx);
126 }
127
128 static void reset_btype(DisasContext *s)
129 {
130 if (s->btype != 0) {
131 TCGv_i32 zero = tcg_const_i32(0);
132 tcg_gen_st_i32(zero, cpu_env, offsetof(CPUARMState, btype));
133 tcg_temp_free_i32(zero);
134 s->btype = 0;
135 }
136 }
137
138 static void set_btype(DisasContext *s, int val)
139 {
140 TCGv_i32 tcg_val;
141
142 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */
143 tcg_debug_assert(val >= 1 && val <= 3);
144
145 tcg_val = tcg_const_i32(val);
146 tcg_gen_st_i32(tcg_val, cpu_env, offsetof(CPUARMState, btype));
147 tcg_temp_free_i32(tcg_val);
148 s->btype = -1;
149 }
150
151 void gen_a64_set_pc_im(uint64_t val)
152 {
153 tcg_gen_movi_i64(cpu_pc, val);
154 }
155
156 /*
157 * Handle Top Byte Ignore (TBI) bits.
158 *
159 * If address tagging is enabled via the TCR TBI bits:
160 * + for EL2 and EL3 there is only one TBI bit, and if it is set
161 * then the address is zero-extended, clearing bits [63:56]
162 * + for EL0 and EL1, TBI0 controls addresses with bit 55 == 0
163 * and TBI1 controls addressses with bit 55 == 1.
164 * If the appropriate TBI bit is set for the address then
165 * the address is sign-extended from bit 55 into bits [63:56]
166 *
167 * Here We have concatenated TBI{1,0} into tbi.
168 */
169 static void gen_top_byte_ignore(DisasContext *s, TCGv_i64 dst,
170 TCGv_i64 src, int tbi)
171 {
172 if (tbi == 0) {
173 /* Load unmodified address */
174 tcg_gen_mov_i64(dst, src);
175 } else if (!regime_has_2_ranges(s->mmu_idx)) {
176 /* Force tag byte to all zero */
177 tcg_gen_extract_i64(dst, src, 0, 56);
178 } else {
179 /* Sign-extend from bit 55. */
180 tcg_gen_sextract_i64(dst, src, 0, 56);
181
182 if (tbi != 3) {
183 TCGv_i64 tcg_zero = tcg_const_i64(0);
184
185 /*
186 * The two TBI bits differ.
187 * If tbi0, then !tbi1: only use the extension if positive.
188 * if !tbi0, then tbi1: only use the extension if negative.
189 */
190 tcg_gen_movcond_i64(tbi == 1 ? TCG_COND_GE : TCG_COND_LT,
191 dst, dst, tcg_zero, dst, src);
192 tcg_temp_free_i64(tcg_zero);
193 }
194 }
195 }
196
197 static void gen_a64_set_pc(DisasContext *s, TCGv_i64 src)
198 {
199 /*
200 * If address tagging is enabled for instructions via the TCR TBI bits,
201 * then loading an address into the PC will clear out any tag.
202 */
203 gen_top_byte_ignore(s, cpu_pc, src, s->tbii);
204 }
205
206 /*
207 * Handle MTE and/or TBI.
208 *
209 * For TBI, ideally, we would do nothing. Proper behaviour on fault is
210 * for the tag to be present in the FAR_ELx register. But for user-only
211 * mode we do not have a TLB with which to implement this, so we must
212 * remove the top byte now.
213 *
214 * Always return a fresh temporary that we can increment independently
215 * of the write-back address.
216 */
217
218 TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr)
219 {
220 TCGv_i64 clean = new_tmp_a64(s);
221 #ifdef CONFIG_USER_ONLY
222 gen_top_byte_ignore(s, clean, addr, s->tbid);
223 #else
224 tcg_gen_mov_i64(clean, addr);
225 #endif
226 return clean;
227 }
228
229 /* Insert a zero tag into src, with the result at dst. */
230 static void gen_address_with_allocation_tag0(TCGv_i64 dst, TCGv_i64 src)
231 {
232 tcg_gen_andi_i64(dst, src, ~MAKE_64BIT_MASK(56, 4));
233 }
234
235 static void gen_probe_access(DisasContext *s, TCGv_i64 ptr,
236 MMUAccessType acc, int log2_size)
237 {
238 TCGv_i32 t_acc = tcg_const_i32(acc);
239 TCGv_i32 t_idx = tcg_const_i32(get_mem_index(s));
240 TCGv_i32 t_size = tcg_const_i32(1 << log2_size);
241
242 gen_helper_probe_access(cpu_env, ptr, t_acc, t_idx, t_size);
243 tcg_temp_free_i32(t_acc);
244 tcg_temp_free_i32(t_idx);
245 tcg_temp_free_i32(t_size);
246 }
247
248 /*
249 * For MTE, check a single logical or atomic access. This probes a single
250 * address, the exact one specified. The size and alignment of the access
251 * is not relevant to MTE, per se, but watchpoints do require the size,
252 * and we want to recognize those before making any other changes to state.
253 */
254 static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr,
255 bool is_write, bool tag_checked,
256 int log2_size, bool is_unpriv,
257 int core_idx)
258 {
259 if (tag_checked && s->mte_active[is_unpriv]) {
260 TCGv_i32 tcg_desc;
261 TCGv_i64 ret;
262 int desc = 0;
263
264 desc = FIELD_DP32(desc, MTEDESC, MIDX, core_idx);
265 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
266 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
267 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
268 desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << log2_size);
269 tcg_desc = tcg_const_i32(desc);
270
271 ret = new_tmp_a64(s);
272 gen_helper_mte_check1(ret, cpu_env, tcg_desc, addr);
273 tcg_temp_free_i32(tcg_desc);
274
275 return ret;
276 }
277 return clean_data_tbi(s, addr);
278 }
279
280 TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write,
281 bool tag_checked, int log2_size)
282 {
283 return gen_mte_check1_mmuidx(s, addr, is_write, tag_checked, log2_size,
284 false, get_mem_index(s));
285 }
286
287 /*
288 * For MTE, check multiple logical sequential accesses.
289 */
290 TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
291 bool tag_checked, int log2_esize, int total_size)
292 {
293 if (tag_checked && s->mte_active[0] && total_size != (1 << log2_esize)) {
294 TCGv_i32 tcg_desc;
295 TCGv_i64 ret;
296 int desc = 0;
297
298 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
299 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
300 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
301 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
302 desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << log2_esize);
303 desc = FIELD_DP32(desc, MTEDESC, TSIZE, total_size);
304 tcg_desc = tcg_const_i32(desc);
305
306 ret = new_tmp_a64(s);
307 gen_helper_mte_checkN(ret, cpu_env, tcg_desc, addr);
308 tcg_temp_free_i32(tcg_desc);
309
310 return ret;
311 }
312 return gen_mte_check1(s, addr, is_write, tag_checked, log2_esize);
313 }
314
315 typedef struct DisasCompare64 {
316 TCGCond cond;
317 TCGv_i64 value;
318 } DisasCompare64;
319
320 static void a64_test_cc(DisasCompare64 *c64, int cc)
321 {
322 DisasCompare c32;
323
324 arm_test_cc(&c32, cc);
325
326 /* Sign-extend the 32-bit value so that the GE/LT comparisons work
327 * properly. The NE/EQ comparisons are also fine with this choice. */
328 c64->cond = c32.cond;
329 c64->value = tcg_temp_new_i64();
330 tcg_gen_ext_i32_i64(c64->value, c32.value);
331
332 arm_free_cc(&c32);
333 }
334
335 static void a64_free_cc(DisasCompare64 *c64)
336 {
337 tcg_temp_free_i64(c64->value);
338 }
339
340 static void gen_exception_internal(int excp)
341 {
342 TCGv_i32 tcg_excp = tcg_const_i32(excp);
343
344 assert(excp_is_internal(excp));
345 gen_helper_exception_internal(cpu_env, tcg_excp);
346 tcg_temp_free_i32(tcg_excp);
347 }
348
349 static void gen_exception_internal_insn(DisasContext *s, uint64_t pc, int excp)
350 {
351 gen_a64_set_pc_im(pc);
352 gen_exception_internal(excp);
353 s->base.is_jmp = DISAS_NORETURN;
354 }
355
356 static void gen_exception_insn(DisasContext *s, uint64_t pc, int excp,
357 uint32_t syndrome, uint32_t target_el)
358 {
359 gen_a64_set_pc_im(pc);
360 gen_exception(excp, syndrome, target_el);
361 s->base.is_jmp = DISAS_NORETURN;
362 }
363
364 static void gen_exception_bkpt_insn(DisasContext *s, uint32_t syndrome)
365 {
366 TCGv_i32 tcg_syn;
367
368 gen_a64_set_pc_im(s->pc_curr);
369 tcg_syn = tcg_const_i32(syndrome);
370 gen_helper_exception_bkpt_insn(cpu_env, tcg_syn);
371 tcg_temp_free_i32(tcg_syn);
372 s->base.is_jmp = DISAS_NORETURN;
373 }
374
375 static void gen_step_complete_exception(DisasContext *s)
376 {
377 /* We just completed step of an insn. Move from Active-not-pending
378 * to Active-pending, and then also take the swstep exception.
379 * This corresponds to making the (IMPDEF) choice to prioritize
380 * swstep exceptions over asynchronous exceptions taken to an exception
381 * level where debug is disabled. This choice has the advantage that
382 * we do not need to maintain internal state corresponding to the
383 * ISV/EX syndrome bits between completion of the step and generation
384 * of the exception, and our syndrome information is always correct.
385 */
386 gen_ss_advance(s);
387 gen_swstep_exception(s, 1, s->is_ldex);
388 s->base.is_jmp = DISAS_NORETURN;
389 }
390
391 static inline bool use_goto_tb(DisasContext *s, int n, uint64_t dest)
392 {
393 /* No direct tb linking with singlestep (either QEMU's or the ARM
394 * debug architecture kind) or deterministic io
395 */
396 if (s->base.singlestep_enabled || s->ss_active ||
397 (tb_cflags(s->base.tb) & CF_LAST_IO)) {
398 return false;
399 }
400
401 #ifndef CONFIG_USER_ONLY
402 /* Only link tbs from inside the same guest page */
403 if ((s->base.tb->pc & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) {
404 return false;
405 }
406 #endif
407
408 return true;
409 }
410
411 static inline void gen_goto_tb(DisasContext *s, int n, uint64_t dest)
412 {
413 TranslationBlock *tb;
414
415 tb = s->base.tb;
416 if (use_goto_tb(s, n, dest)) {
417 tcg_gen_goto_tb(n);
418 gen_a64_set_pc_im(dest);
419 tcg_gen_exit_tb(tb, n);
420 s->base.is_jmp = DISAS_NORETURN;
421 } else {
422 gen_a64_set_pc_im(dest);
423 if (s->ss_active) {
424 gen_step_complete_exception(s);
425 } else if (s->base.singlestep_enabled) {
426 gen_exception_internal(EXCP_DEBUG);
427 } else {
428 tcg_gen_lookup_and_goto_ptr();
429 s->base.is_jmp = DISAS_NORETURN;
430 }
431 }
432 }
433
434 void unallocated_encoding(DisasContext *s)
435 {
436 /* Unallocated and reserved encodings are uncategorized */
437 gen_exception_insn(s, s->pc_curr, EXCP_UDEF, syn_uncategorized(),
438 default_exception_el(s));
439 }
440
441 static void init_tmp_a64_array(DisasContext *s)
442 {
443 #ifdef CONFIG_DEBUG_TCG
444 memset(s->tmp_a64, 0, sizeof(s->tmp_a64));
445 #endif
446 s->tmp_a64_count = 0;
447 }
448
449 static void free_tmp_a64(DisasContext *s)
450 {
451 int i;
452 for (i = 0; i < s->tmp_a64_count; i++) {
453 tcg_temp_free_i64(s->tmp_a64[i]);
454 }
455 init_tmp_a64_array(s);
456 }
457
458 TCGv_i64 new_tmp_a64(DisasContext *s)
459 {
460 assert(s->tmp_a64_count < TMP_A64_MAX);
461 return s->tmp_a64[s->tmp_a64_count++] = tcg_temp_new_i64();
462 }
463
464 TCGv_i64 new_tmp_a64_local(DisasContext *s)
465 {
466 assert(s->tmp_a64_count < TMP_A64_MAX);
467 return s->tmp_a64[s->tmp_a64_count++] = tcg_temp_local_new_i64();
468 }
469
470 TCGv_i64 new_tmp_a64_zero(DisasContext *s)
471 {
472 TCGv_i64 t = new_tmp_a64(s);
473 tcg_gen_movi_i64(t, 0);
474 return t;
475 }
476
477 /*
478 * Register access functions
479 *
480 * These functions are used for directly accessing a register in where
481 * changes to the final register value are likely to be made. If you
482 * need to use a register for temporary calculation (e.g. index type
483 * operations) use the read_* form.
484 *
485 * B1.2.1 Register mappings
486 *
487 * In instruction register encoding 31 can refer to ZR (zero register) or
488 * the SP (stack pointer) depending on context. In QEMU's case we map SP
489 * to cpu_X[31] and ZR accesses to a temporary which can be discarded.
490 * This is the point of the _sp forms.
491 */
492 TCGv_i64 cpu_reg(DisasContext *s, int reg)
493 {
494 if (reg == 31) {
495 return new_tmp_a64_zero(s);
496 } else {
497 return cpu_X[reg];
498 }
499 }
500
501 /* register access for when 31 == SP */
502 TCGv_i64 cpu_reg_sp(DisasContext *s, int reg)
503 {
504 return cpu_X[reg];
505 }
506
507 /* read a cpu register in 32bit/64bit mode. Returns a TCGv_i64
508 * representing the register contents. This TCGv is an auto-freed
509 * temporary so it need not be explicitly freed, and may be modified.
510 */
511 TCGv_i64 read_cpu_reg(DisasContext *s, int reg, int sf)
512 {
513 TCGv_i64 v = new_tmp_a64(s);
514 if (reg != 31) {
515 if (sf) {
516 tcg_gen_mov_i64(v, cpu_X[reg]);
517 } else {
518 tcg_gen_ext32u_i64(v, cpu_X[reg]);
519 }
520 } else {
521 tcg_gen_movi_i64(v, 0);
522 }
523 return v;
524 }
525
526 TCGv_i64 read_cpu_reg_sp(DisasContext *s, int reg, int sf)
527 {
528 TCGv_i64 v = new_tmp_a64(s);
529 if (sf) {
530 tcg_gen_mov_i64(v, cpu_X[reg]);
531 } else {
532 tcg_gen_ext32u_i64(v, cpu_X[reg]);
533 }
534 return v;
535 }
536
537 /* Return the offset into CPUARMState of a slice (from
538 * the least significant end) of FP register Qn (ie
539 * Dn, Sn, Hn or Bn).
540 * (Note that this is not the same mapping as for A32; see cpu.h)
541 */
542 static inline int fp_reg_offset(DisasContext *s, int regno, MemOp size)
543 {
544 return vec_reg_offset(s, regno, 0, size);
545 }
546
547 /* Offset of the high half of the 128 bit vector Qn */
548 static inline int fp_reg_hi_offset(DisasContext *s, int regno)
549 {
550 return vec_reg_offset(s, regno, 1, MO_64);
551 }
552
553 /* Convenience accessors for reading and writing single and double
554 * FP registers. Writing clears the upper parts of the associated
555 * 128 bit vector register, as required by the architecture.
556 * Note that unlike the GP register accessors, the values returned
557 * by the read functions must be manually freed.
558 */
559 static TCGv_i64 read_fp_dreg(DisasContext *s, int reg)
560 {
561 TCGv_i64 v = tcg_temp_new_i64();
562
563 tcg_gen_ld_i64(v, cpu_env, fp_reg_offset(s, reg, MO_64));
564 return v;
565 }
566
567 static TCGv_i32 read_fp_sreg(DisasContext *s, int reg)
568 {
569 TCGv_i32 v = tcg_temp_new_i32();
570
571 tcg_gen_ld_i32(v, cpu_env, fp_reg_offset(s, reg, MO_32));
572 return v;
573 }
574
575 static TCGv_i32 read_fp_hreg(DisasContext *s, int reg)
576 {
577 TCGv_i32 v = tcg_temp_new_i32();
578
579 tcg_gen_ld16u_i32(v, cpu_env, fp_reg_offset(s, reg, MO_16));
580 return v;
581 }
582
583 /* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
584 * If SVE is not enabled, then there are only 128 bits in the vector.
585 */
586 static void clear_vec_high(DisasContext *s, bool is_q, int rd)
587 {
588 unsigned ofs = fp_reg_offset(s, rd, MO_64);
589 unsigned vsz = vec_full_reg_size(s);
590
591 /* Nop move, with side effect of clearing the tail. */
592 tcg_gen_gvec_mov(MO_64, ofs, ofs, is_q ? 16 : 8, vsz);
593 }
594
595 void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v)
596 {
597 unsigned ofs = fp_reg_offset(s, reg, MO_64);
598
599 tcg_gen_st_i64(v, cpu_env, ofs);
600 clear_vec_high(s, false, reg);
601 }
602
603 static void write_fp_sreg(DisasContext *s, int reg, TCGv_i32 v)
604 {
605 TCGv_i64 tmp = tcg_temp_new_i64();
606
607 tcg_gen_extu_i32_i64(tmp, v);
608 write_fp_dreg(s, reg, tmp);
609 tcg_temp_free_i64(tmp);
610 }
611
612 TCGv_ptr get_fpstatus_ptr(bool is_f16)
613 {
614 TCGv_ptr statusptr = tcg_temp_new_ptr();
615 int offset;
616
617 /* In A64 all instructions (both FP and Neon) use the FPCR; there
618 * is no equivalent of the A32 Neon "standard FPSCR value".
619 * However half-precision operations operate under a different
620 * FZ16 flag and use vfp.fp_status_f16 instead of vfp.fp_status.
621 */
622 if (is_f16) {
623 offset = offsetof(CPUARMState, vfp.fp_status_f16);
624 } else {
625 offset = offsetof(CPUARMState, vfp.fp_status);
626 }
627 tcg_gen_addi_ptr(statusptr, cpu_env, offset);
628 return statusptr;
629 }
630
631 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
632 static void gen_gvec_fn2(DisasContext *s, bool is_q, int rd, int rn,
633 GVecGen2Fn *gvec_fn, int vece)
634 {
635 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
636 is_q ? 16 : 8, vec_full_reg_size(s));
637 }
638
639 /* Expand a 2-operand + immediate AdvSIMD vector operation using
640 * an expander function.
641 */
642 static void gen_gvec_fn2i(DisasContext *s, bool is_q, int rd, int rn,
643 int64_t imm, GVecGen2iFn *gvec_fn, int vece)
644 {
645 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
646 imm, is_q ? 16 : 8, vec_full_reg_size(s));
647 }
648
649 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
650 static void gen_gvec_fn3(DisasContext *s, bool is_q, int rd, int rn, int rm,
651 GVecGen3Fn *gvec_fn, int vece)
652 {
653 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
654 vec_full_reg_offset(s, rm), is_q ? 16 : 8, vec_full_reg_size(s));
655 }
656
657 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
658 static void gen_gvec_fn4(DisasContext *s, bool is_q, int rd, int rn, int rm,
659 int rx, GVecGen4Fn *gvec_fn, int vece)
660 {
661 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
662 vec_full_reg_offset(s, rm), vec_full_reg_offset(s, rx),
663 is_q ? 16 : 8, vec_full_reg_size(s));
664 }
665
666 /* Expand a 2-operand operation using an out-of-line helper. */
667 static void gen_gvec_op2_ool(DisasContext *s, bool is_q, int rd,
668 int rn, int data, gen_helper_gvec_2 *fn)
669 {
670 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd),
671 vec_full_reg_offset(s, rn),
672 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
673 }
674
675 /* Expand a 3-operand operation using an out-of-line helper. */
676 static void gen_gvec_op3_ool(DisasContext *s, bool is_q, int rd,
677 int rn, int rm, int data, gen_helper_gvec_3 *fn)
678 {
679 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
680 vec_full_reg_offset(s, rn),
681 vec_full_reg_offset(s, rm),
682 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
683 }
684
685 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
686 * an out-of-line helper.
687 */
688 static void gen_gvec_op3_fpst(DisasContext *s, bool is_q, int rd, int rn,
689 int rm, bool is_fp16, int data,
690 gen_helper_gvec_3_ptr *fn)
691 {
692 TCGv_ptr fpst = get_fpstatus_ptr(is_fp16);
693 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
694 vec_full_reg_offset(s, rn),
695 vec_full_reg_offset(s, rm), fpst,
696 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
697 tcg_temp_free_ptr(fpst);
698 }
699
700 /* Set ZF and NF based on a 64 bit result. This is alas fiddlier
701 * than the 32 bit equivalent.
702 */
703 static inline void gen_set_NZ64(TCGv_i64 result)
704 {
705 tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result);
706 tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF);
707 }
708
709 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
710 static inline void gen_logic_CC(int sf, TCGv_i64 result)
711 {
712 if (sf) {
713 gen_set_NZ64(result);
714 } else {
715 tcg_gen_extrl_i64_i32(cpu_ZF, result);
716 tcg_gen_mov_i32(cpu_NF, cpu_ZF);
717 }
718 tcg_gen_movi_i32(cpu_CF, 0);
719 tcg_gen_movi_i32(cpu_VF, 0);
720 }
721
722 /* dest = T0 + T1; compute C, N, V and Z flags */
723 static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
724 {
725 if (sf) {
726 TCGv_i64 result, flag, tmp;
727 result = tcg_temp_new_i64();
728 flag = tcg_temp_new_i64();
729 tmp = tcg_temp_new_i64();
730
731 tcg_gen_movi_i64(tmp, 0);
732 tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);
733
734 tcg_gen_extrl_i64_i32(cpu_CF, flag);
735
736 gen_set_NZ64(result);
737
738 tcg_gen_xor_i64(flag, result, t0);
739 tcg_gen_xor_i64(tmp, t0, t1);
740 tcg_gen_andc_i64(flag, flag, tmp);
741 tcg_temp_free_i64(tmp);
742 tcg_gen_extrh_i64_i32(cpu_VF, flag);
743
744 tcg_gen_mov_i64(dest, result);
745 tcg_temp_free_i64(result);
746 tcg_temp_free_i64(flag);
747 } else {
748 /* 32 bit arithmetic */
749 TCGv_i32 t0_32 = tcg_temp_new_i32();
750 TCGv_i32 t1_32 = tcg_temp_new_i32();
751 TCGv_i32 tmp = tcg_temp_new_i32();
752
753 tcg_gen_movi_i32(tmp, 0);
754 tcg_gen_extrl_i64_i32(t0_32, t0);
755 tcg_gen_extrl_i64_i32(t1_32, t1);
756 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);
757 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
758 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
759 tcg_gen_xor_i32(tmp, t0_32, t1_32);
760 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
761 tcg_gen_extu_i32_i64(dest, cpu_NF);
762
763 tcg_temp_free_i32(tmp);
764 tcg_temp_free_i32(t0_32);
765 tcg_temp_free_i32(t1_32);
766 }
767 }
768
769 /* dest = T0 - T1; compute C, N, V and Z flags */
770 static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
771 {
772 if (sf) {
773 /* 64 bit arithmetic */
774 TCGv_i64 result, flag, tmp;
775
776 result = tcg_temp_new_i64();
777 flag = tcg_temp_new_i64();
778 tcg_gen_sub_i64(result, t0, t1);
779
780 gen_set_NZ64(result);
781
782 tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);
783 tcg_gen_extrl_i64_i32(cpu_CF, flag);
784
785 tcg_gen_xor_i64(flag, result, t0);
786 tmp = tcg_temp_new_i64();
787 tcg_gen_xor_i64(tmp, t0, t1);
788 tcg_gen_and_i64(flag, flag, tmp);
789 tcg_temp_free_i64(tmp);
790 tcg_gen_extrh_i64_i32(cpu_VF, flag);
791 tcg_gen_mov_i64(dest, result);
792 tcg_temp_free_i64(flag);
793 tcg_temp_free_i64(result);
794 } else {
795 /* 32 bit arithmetic */
796 TCGv_i32 t0_32 = tcg_temp_new_i32();
797 TCGv_i32 t1_32 = tcg_temp_new_i32();
798 TCGv_i32 tmp;
799
800 tcg_gen_extrl_i64_i32(t0_32, t0);
801 tcg_gen_extrl_i64_i32(t1_32, t1);
802 tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);
803 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
804 tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);
805 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
806 tmp = tcg_temp_new_i32();
807 tcg_gen_xor_i32(tmp, t0_32, t1_32);
808 tcg_temp_free_i32(t0_32);
809 tcg_temp_free_i32(t1_32);
810 tcg_gen_and_i32(cpu_VF, cpu_VF, tmp);
811 tcg_temp_free_i32(tmp);
812 tcg_gen_extu_i32_i64(dest, cpu_NF);
813 }
814 }
815
816 /* dest = T0 + T1 + CF; do not compute flags. */
817 static void gen_adc(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
818 {
819 TCGv_i64 flag = tcg_temp_new_i64();
820 tcg_gen_extu_i32_i64(flag, cpu_CF);
821 tcg_gen_add_i64(dest, t0, t1);
822 tcg_gen_add_i64(dest, dest, flag);
823 tcg_temp_free_i64(flag);
824
825 if (!sf) {
826 tcg_gen_ext32u_i64(dest, dest);
827 }
828 }
829
830 /* dest = T0 + T1 + CF; compute C, N, V and Z flags. */
831 static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
832 {
833 if (sf) {
834 TCGv_i64 result, cf_64, vf_64, tmp;
835 result = tcg_temp_new_i64();
836 cf_64 = tcg_temp_new_i64();
837 vf_64 = tcg_temp_new_i64();
838 tmp = tcg_const_i64(0);
839
840 tcg_gen_extu_i32_i64(cf_64, cpu_CF);
841 tcg_gen_add2_i64(result, cf_64, t0, tmp, cf_64, tmp);
842 tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, tmp);
843 tcg_gen_extrl_i64_i32(cpu_CF, cf_64);
844 gen_set_NZ64(result);
845
846 tcg_gen_xor_i64(vf_64, result, t0);
847 tcg_gen_xor_i64(tmp, t0, t1);
848 tcg_gen_andc_i64(vf_64, vf_64, tmp);
849 tcg_gen_extrh_i64_i32(cpu_VF, vf_64);
850
851 tcg_gen_mov_i64(dest, result);
852
853 tcg_temp_free_i64(tmp);
854 tcg_temp_free_i64(vf_64);
855 tcg_temp_free_i64(cf_64);
856 tcg_temp_free_i64(result);
857 } else {
858 TCGv_i32 t0_32, t1_32, tmp;
859 t0_32 = tcg_temp_new_i32();
860 t1_32 = tcg_temp_new_i32();
861 tmp = tcg_const_i32(0);
862
863 tcg_gen_extrl_i64_i32(t0_32, t0);
864 tcg_gen_extrl_i64_i32(t1_32, t1);
865 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, cpu_CF, tmp);
866 tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, tmp);
867
868 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
869 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
870 tcg_gen_xor_i32(tmp, t0_32, t1_32);
871 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
872 tcg_gen_extu_i32_i64(dest, cpu_NF);
873
874 tcg_temp_free_i32(tmp);
875 tcg_temp_free_i32(t1_32);
876 tcg_temp_free_i32(t0_32);
877 }
878 }
879
880 /*
881 * Load/Store generators
882 */
883
884 /*
885 * Store from GPR register to memory.
886 */
887 static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
888 TCGv_i64 tcg_addr, int size, int memidx,
889 bool iss_valid,
890 unsigned int iss_srt,
891 bool iss_sf, bool iss_ar)
892 {
893 g_assert(size <= 3);
894 tcg_gen_qemu_st_i64(source, tcg_addr, memidx, s->be_data + size);
895
896 if (iss_valid) {
897 uint32_t syn;
898
899 syn = syn_data_abort_with_iss(0,
900 size,
901 false,
902 iss_srt,
903 iss_sf,
904 iss_ar,
905 0, 0, 0, 0, 0, false);
906 disas_set_insn_syndrome(s, syn);
907 }
908 }
909
910 static void do_gpr_st(DisasContext *s, TCGv_i64 source,
911 TCGv_i64 tcg_addr, int size,
912 bool iss_valid,
913 unsigned int iss_srt,
914 bool iss_sf, bool iss_ar)
915 {
916 do_gpr_st_memidx(s, source, tcg_addr, size, get_mem_index(s),
917 iss_valid, iss_srt, iss_sf, iss_ar);
918 }
919
920 /*
921 * Load from memory to GPR register
922 */
923 static void do_gpr_ld_memidx(DisasContext *s,
924 TCGv_i64 dest, TCGv_i64 tcg_addr,
925 int size, bool is_signed,
926 bool extend, int memidx,
927 bool iss_valid, unsigned int iss_srt,
928 bool iss_sf, bool iss_ar)
929 {
930 MemOp memop = s->be_data + size;
931
932 g_assert(size <= 3);
933
934 if (is_signed) {
935 memop += MO_SIGN;
936 }
937
938 tcg_gen_qemu_ld_i64(dest, tcg_addr, memidx, memop);
939
940 if (extend && is_signed) {
941 g_assert(size < 3);
942 tcg_gen_ext32u_i64(dest, dest);
943 }
944
945 if (iss_valid) {
946 uint32_t syn;
947
948 syn = syn_data_abort_with_iss(0,
949 size,
950 is_signed,
951 iss_srt,
952 iss_sf,
953 iss_ar,
954 0, 0, 0, 0, 0, false);
955 disas_set_insn_syndrome(s, syn);
956 }
957 }
958
959 static void do_gpr_ld(DisasContext *s,
960 TCGv_i64 dest, TCGv_i64 tcg_addr,
961 int size, bool is_signed, bool extend,
962 bool iss_valid, unsigned int iss_srt,
963 bool iss_sf, bool iss_ar)
964 {
965 do_gpr_ld_memidx(s, dest, tcg_addr, size, is_signed, extend,
966 get_mem_index(s),
967 iss_valid, iss_srt, iss_sf, iss_ar);
968 }
969
970 /*
971 * Store from FP register to memory
972 */
973 static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, int size)
974 {
975 /* This writes the bottom N bits of a 128 bit wide vector to memory */
976 TCGv_i64 tmp = tcg_temp_new_i64();
977 tcg_gen_ld_i64(tmp, cpu_env, fp_reg_offset(s, srcidx, MO_64));
978 if (size < 4) {
979 tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s),
980 s->be_data + size);
981 } else {
982 bool be = s->be_data == MO_BE;
983 TCGv_i64 tcg_hiaddr = tcg_temp_new_i64();
984
985 tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);
986 tcg_gen_qemu_st_i64(tmp, be ? tcg_hiaddr : tcg_addr, get_mem_index(s),
987 s->be_data | MO_Q);
988 tcg_gen_ld_i64(tmp, cpu_env, fp_reg_hi_offset(s, srcidx));
989 tcg_gen_qemu_st_i64(tmp, be ? tcg_addr : tcg_hiaddr, get_mem_index(s),
990 s->be_data | MO_Q);
991 tcg_temp_free_i64(tcg_hiaddr);
992 }
993
994 tcg_temp_free_i64(tmp);
995 }
996
997 /*
998 * Load from memory to FP register
999 */
1000 static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, int size)
1001 {
1002 /* This always zero-extends and writes to a full 128 bit wide vector */
1003 TCGv_i64 tmplo = tcg_temp_new_i64();
1004 TCGv_i64 tmphi = NULL;
1005
1006 if (size < 4) {
1007 MemOp memop = s->be_data + size;
1008 tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), memop);
1009 } else {
1010 bool be = s->be_data == MO_BE;
1011 TCGv_i64 tcg_hiaddr;
1012
1013 tmphi = tcg_temp_new_i64();
1014 tcg_hiaddr = tcg_temp_new_i64();
1015
1016 tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);
1017 tcg_gen_qemu_ld_i64(tmplo, be ? tcg_hiaddr : tcg_addr, get_mem_index(s),
1018 s->be_data | MO_Q);
1019 tcg_gen_qemu_ld_i64(tmphi, be ? tcg_addr : tcg_hiaddr, get_mem_index(s),
1020 s->be_data | MO_Q);
1021 tcg_temp_free_i64(tcg_hiaddr);
1022 }
1023
1024 tcg_gen_st_i64(tmplo, cpu_env, fp_reg_offset(s, destidx, MO_64));
1025 tcg_temp_free_i64(tmplo);
1026
1027 if (tmphi) {
1028 tcg_gen_st_i64(tmphi, cpu_env, fp_reg_hi_offset(s, destidx));
1029 tcg_temp_free_i64(tmphi);
1030 }
1031 clear_vec_high(s, tmphi != NULL, destidx);
1032 }
1033
1034 /*
1035 * Vector load/store helpers.
1036 *
1037 * The principal difference between this and a FP load is that we don't
1038 * zero extend as we are filling a partial chunk of the vector register.
1039 * These functions don't support 128 bit loads/stores, which would be
1040 * normal load/store operations.
1041 *
1042 * The _i32 versions are useful when operating on 32 bit quantities
1043 * (eg for floating point single or using Neon helper functions).
1044 */
1045
1046 /* Get value of an element within a vector register */
1047 static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx,
1048 int element, MemOp memop)
1049 {
1050 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1051 switch (memop) {
1052 case MO_8:
1053 tcg_gen_ld8u_i64(tcg_dest, cpu_env, vect_off);
1054 break;
1055 case MO_16:
1056 tcg_gen_ld16u_i64(tcg_dest, cpu_env, vect_off);
1057 break;
1058 case MO_32:
1059 tcg_gen_ld32u_i64(tcg_dest, cpu_env, vect_off);
1060 break;
1061 case MO_8|MO_SIGN:
1062 tcg_gen_ld8s_i64(tcg_dest, cpu_env, vect_off);
1063 break;
1064 case MO_16|MO_SIGN:
1065 tcg_gen_ld16s_i64(tcg_dest, cpu_env, vect_off);
1066 break;
1067 case MO_32|MO_SIGN:
1068 tcg_gen_ld32s_i64(tcg_dest, cpu_env, vect_off);
1069 break;
1070 case MO_64:
1071 case MO_64|MO_SIGN:
1072 tcg_gen_ld_i64(tcg_dest, cpu_env, vect_off);
1073 break;
1074 default:
1075 g_assert_not_reached();
1076 }
1077 }
1078
1079 static void read_vec_element_i32(DisasContext *s, TCGv_i32 tcg_dest, int srcidx,
1080 int element, MemOp memop)
1081 {
1082 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1083 switch (memop) {
1084 case MO_8:
1085 tcg_gen_ld8u_i32(tcg_dest, cpu_env, vect_off);
1086 break;
1087 case MO_16:
1088 tcg_gen_ld16u_i32(tcg_dest, cpu_env, vect_off);
1089 break;
1090 case MO_8|MO_SIGN:
1091 tcg_gen_ld8s_i32(tcg_dest, cpu_env, vect_off);
1092 break;
1093 case MO_16|MO_SIGN:
1094 tcg_gen_ld16s_i32(tcg_dest, cpu_env, vect_off);
1095 break;
1096 case MO_32:
1097 case MO_32|MO_SIGN:
1098 tcg_gen_ld_i32(tcg_dest, cpu_env, vect_off);
1099 break;
1100 default:
1101 g_assert_not_reached();
1102 }
1103 }
1104
1105 /* Set value of an element within a vector register */
1106 static void write_vec_element(DisasContext *s, TCGv_i64 tcg_src, int destidx,
1107 int element, MemOp memop)
1108 {
1109 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1110 switch (memop) {
1111 case MO_8:
1112 tcg_gen_st8_i64(tcg_src, cpu_env, vect_off);
1113 break;
1114 case MO_16:
1115 tcg_gen_st16_i64(tcg_src, cpu_env, vect_off);
1116 break;
1117 case MO_32:
1118 tcg_gen_st32_i64(tcg_src, cpu_env, vect_off);
1119 break;
1120 case MO_64:
1121 tcg_gen_st_i64(tcg_src, cpu_env, vect_off);
1122 break;
1123 default:
1124 g_assert_not_reached();
1125 }
1126 }
1127
1128 static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src,
1129 int destidx, int element, MemOp memop)
1130 {
1131 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1132 switch (memop) {
1133 case MO_8:
1134 tcg_gen_st8_i32(tcg_src, cpu_env, vect_off);
1135 break;
1136 case MO_16:
1137 tcg_gen_st16_i32(tcg_src, cpu_env, vect_off);
1138 break;
1139 case MO_32:
1140 tcg_gen_st_i32(tcg_src, cpu_env, vect_off);
1141 break;
1142 default:
1143 g_assert_not_reached();
1144 }
1145 }
1146
1147 /* Store from vector register to memory */
1148 static void do_vec_st(DisasContext *s, int srcidx, int element,
1149 TCGv_i64 tcg_addr, int size, MemOp endian)
1150 {
1151 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1152
1153 read_vec_element(s, tcg_tmp, srcidx, element, size);
1154 tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), endian | size);
1155
1156 tcg_temp_free_i64(tcg_tmp);
1157 }
1158
1159 /* Load from memory to vector register */
1160 static void do_vec_ld(DisasContext *s, int destidx, int element,
1161 TCGv_i64 tcg_addr, int size, MemOp endian)
1162 {
1163 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1164
1165 tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), endian | size);
1166 write_vec_element(s, tcg_tmp, destidx, element, size);
1167
1168 tcg_temp_free_i64(tcg_tmp);
1169 }
1170
1171 /* Check that FP/Neon access is enabled. If it is, return
1172 * true. If not, emit code to generate an appropriate exception,
1173 * and return false; the caller should not emit any code for
1174 * the instruction. Note that this check must happen after all
1175 * unallocated-encoding checks (otherwise the syndrome information
1176 * for the resulting exception will be incorrect).
1177 */
1178 static inline bool fp_access_check(DisasContext *s)
1179 {
1180 assert(!s->fp_access_checked);
1181 s->fp_access_checked = true;
1182
1183 if (!s->fp_excp_el) {
1184 return true;
1185 }
1186
1187 gen_exception_insn(s, s->pc_curr, EXCP_UDEF,
1188 syn_fp_access_trap(1, 0xe, false), s->fp_excp_el);
1189 return false;
1190 }
1191
1192 /* Check that SVE access is enabled. If it is, return true.
1193 * If not, emit code to generate an appropriate exception and return false.
1194 */
1195 bool sve_access_check(DisasContext *s)
1196 {
1197 if (s->sve_excp_el) {
1198 gen_exception_insn(s, s->pc_curr, EXCP_UDEF, syn_sve_access_trap(),
1199 s->sve_excp_el);
1200 return false;
1201 }
1202 return fp_access_check(s);
1203 }
1204
1205 /*
1206 * This utility function is for doing register extension with an
1207 * optional shift. You will likely want to pass a temporary for the
1208 * destination register. See DecodeRegExtend() in the ARM ARM.
1209 */
1210 static void ext_and_shift_reg(TCGv_i64 tcg_out, TCGv_i64 tcg_in,
1211 int option, unsigned int shift)
1212 {
1213 int extsize = extract32(option, 0, 2);
1214 bool is_signed = extract32(option, 2, 1);
1215
1216 if (is_signed) {
1217 switch (extsize) {
1218 case 0:
1219 tcg_gen_ext8s_i64(tcg_out, tcg_in);
1220 break;
1221 case 1:
1222 tcg_gen_ext16s_i64(tcg_out, tcg_in);
1223 break;
1224 case 2:
1225 tcg_gen_ext32s_i64(tcg_out, tcg_in);
1226 break;
1227 case 3:
1228 tcg_gen_mov_i64(tcg_out, tcg_in);
1229 break;
1230 }
1231 } else {
1232 switch (extsize) {
1233 case 0:
1234 tcg_gen_ext8u_i64(tcg_out, tcg_in);
1235 break;
1236 case 1:
1237 tcg_gen_ext16u_i64(tcg_out, tcg_in);
1238 break;
1239 case 2:
1240 tcg_gen_ext32u_i64(tcg_out, tcg_in);
1241 break;
1242 case 3:
1243 tcg_gen_mov_i64(tcg_out, tcg_in);
1244 break;
1245 }
1246 }
1247
1248 if (shift) {
1249 tcg_gen_shli_i64(tcg_out, tcg_out, shift);
1250 }
1251 }
1252
1253 static inline void gen_check_sp_alignment(DisasContext *s)
1254 {
1255 /* The AArch64 architecture mandates that (if enabled via PSTATE
1256 * or SCTLR bits) there is a check that SP is 16-aligned on every
1257 * SP-relative load or store (with an exception generated if it is not).
1258 * In line with general QEMU practice regarding misaligned accesses,
1259 * we omit these checks for the sake of guest program performance.
1260 * This function is provided as a hook so we can more easily add these
1261 * checks in future (possibly as a "favour catching guest program bugs
1262 * over speed" user selectable option).
1263 */
1264 }
1265
1266 /*
1267 * This provides a simple table based table lookup decoder. It is
1268 * intended to be used when the relevant bits for decode are too
1269 * awkwardly placed and switch/if based logic would be confusing and
1270 * deeply nested. Since it's a linear search through the table, tables
1271 * should be kept small.
1272 *
1273 * It returns the first handler where insn & mask == pattern, or
1274 * NULL if there is no match.
1275 * The table is terminated by an empty mask (i.e. 0)
1276 */
1277 static inline AArch64DecodeFn *lookup_disas_fn(const AArch64DecodeTable *table,
1278 uint32_t insn)
1279 {
1280 const AArch64DecodeTable *tptr = table;
1281
1282 while (tptr->mask) {
1283 if ((insn & tptr->mask) == tptr->pattern) {
1284 return tptr->disas_fn;
1285 }
1286 tptr++;
1287 }
1288 return NULL;
1289 }
1290
1291 /*
1292 * The instruction disassembly implemented here matches
1293 * the instruction encoding classifications in chapter C4
1294 * of the ARM Architecture Reference Manual (DDI0487B_a);
1295 * classification names and decode diagrams here should generally
1296 * match up with those in the manual.
1297 */
1298
1299 /* Unconditional branch (immediate)
1300 * 31 30 26 25 0
1301 * +----+-----------+-------------------------------------+
1302 * | op | 0 0 1 0 1 | imm26 |
1303 * +----+-----------+-------------------------------------+
1304 */
1305 static void disas_uncond_b_imm(DisasContext *s, uint32_t insn)
1306 {
1307 uint64_t addr = s->pc_curr + sextract32(insn, 0, 26) * 4;
1308
1309 if (insn & (1U << 31)) {
1310 /* BL Branch with link */
1311 tcg_gen_movi_i64(cpu_reg(s, 30), s->base.pc_next);
1312 }
1313
1314 /* B Branch / BL Branch with link */
1315 reset_btype(s);
1316 gen_goto_tb(s, 0, addr);
1317 }
1318
1319 /* Compare and branch (immediate)
1320 * 31 30 25 24 23 5 4 0
1321 * +----+-------------+----+---------------------+--------+
1322 * | sf | 0 1 1 0 1 0 | op | imm19 | Rt |
1323 * +----+-------------+----+---------------------+--------+
1324 */
1325 static void disas_comp_b_imm(DisasContext *s, uint32_t insn)
1326 {
1327 unsigned int sf, op, rt;
1328 uint64_t addr;
1329 TCGLabel *label_match;
1330 TCGv_i64 tcg_cmp;
1331
1332 sf = extract32(insn, 31, 1);
1333 op = extract32(insn, 24, 1); /* 0: CBZ; 1: CBNZ */
1334 rt = extract32(insn, 0, 5);
1335 addr = s->pc_curr + sextract32(insn, 5, 19) * 4;
1336
1337 tcg_cmp = read_cpu_reg(s, rt, sf);
1338 label_match = gen_new_label();
1339
1340 reset_btype(s);
1341 tcg_gen_brcondi_i64(op ? TCG_COND_NE : TCG_COND_EQ,
1342 tcg_cmp, 0, label_match);
1343
1344 gen_goto_tb(s, 0, s->base.pc_next);
1345 gen_set_label(label_match);
1346 gen_goto_tb(s, 1, addr);
1347 }
1348
1349 /* Test and branch (immediate)
1350 * 31 30 25 24 23 19 18 5 4 0
1351 * +----+-------------+----+-------+-------------+------+
1352 * | b5 | 0 1 1 0 1 1 | op | b40 | imm14 | Rt |
1353 * +----+-------------+----+-------+-------------+------+
1354 */
1355 static void disas_test_b_imm(DisasContext *s, uint32_t insn)
1356 {
1357 unsigned int bit_pos, op, rt;
1358 uint64_t addr;
1359 TCGLabel *label_match;
1360 TCGv_i64 tcg_cmp;
1361
1362 bit_pos = (extract32(insn, 31, 1) << 5) | extract32(insn, 19, 5);
1363 op = extract32(insn, 24, 1); /* 0: TBZ; 1: TBNZ */
1364 addr = s->pc_curr + sextract32(insn, 5, 14) * 4;
1365 rt = extract32(insn, 0, 5);
1366
1367 tcg_cmp = tcg_temp_new_i64();
1368 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, rt), (1ULL << bit_pos));
1369 label_match = gen_new_label();
1370
1371 reset_btype(s);
1372 tcg_gen_brcondi_i64(op ? TCG_COND_NE : TCG_COND_EQ,
1373 tcg_cmp, 0, label_match);
1374 tcg_temp_free_i64(tcg_cmp);
1375 gen_goto_tb(s, 0, s->base.pc_next);
1376 gen_set_label(label_match);
1377 gen_goto_tb(s, 1, addr);
1378 }
1379
1380 /* Conditional branch (immediate)
1381 * 31 25 24 23 5 4 3 0
1382 * +---------------+----+---------------------+----+------+
1383 * | 0 1 0 1 0 1 0 | o1 | imm19 | o0 | cond |
1384 * +---------------+----+---------------------+----+------+
1385 */
1386 static void disas_cond_b_imm(DisasContext *s, uint32_t insn)
1387 {
1388 unsigned int cond;
1389 uint64_t addr;
1390
1391 if ((insn & (1 << 4)) || (insn & (1 << 24))) {
1392 unallocated_encoding(s);
1393 return;
1394 }
1395 addr = s->pc_curr + sextract32(insn, 5, 19) * 4;
1396 cond = extract32(insn, 0, 4);
1397
1398 reset_btype(s);
1399 if (cond < 0x0e) {
1400 /* genuinely conditional branches */
1401 TCGLabel *label_match = gen_new_label();
1402 arm_gen_test_cc(cond, label_match);
1403 gen_goto_tb(s, 0, s->base.pc_next);
1404 gen_set_label(label_match);
1405 gen_goto_tb(s, 1, addr);
1406 } else {
1407 /* 0xe and 0xf are both "always" conditions */
1408 gen_goto_tb(s, 0, addr);
1409 }
1410 }
1411
1412 /* HINT instruction group, including various allocated HINTs */
1413 static void handle_hint(DisasContext *s, uint32_t insn,
1414 unsigned int op1, unsigned int op2, unsigned int crm)
1415 {
1416 unsigned int selector = crm << 3 | op2;
1417
1418 if (op1 != 3) {
1419 unallocated_encoding(s);
1420 return;
1421 }
1422
1423 switch (selector) {
1424 case 0b00000: /* NOP */
1425 break;
1426 case 0b00011: /* WFI */
1427 s->base.is_jmp = DISAS_WFI;
1428 break;
1429 case 0b00001: /* YIELD */
1430 /* When running in MTTCG we don't generate jumps to the yield and
1431 * WFE helpers as it won't affect the scheduling of other vCPUs.
1432 * If we wanted to more completely model WFE/SEV so we don't busy
1433 * spin unnecessarily we would need to do something more involved.
1434 */
1435 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1436 s->base.is_jmp = DISAS_YIELD;
1437 }
1438 break;
1439 case 0b00010: /* WFE */
1440 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1441 s->base.is_jmp = DISAS_WFE;
1442 }
1443 break;
1444 case 0b00100: /* SEV */
1445 case 0b00101: /* SEVL */
1446 /* we treat all as NOP at least for now */
1447 break;
1448 case 0b00111: /* XPACLRI */
1449 if (s->pauth_active) {
1450 gen_helper_xpaci(cpu_X[30], cpu_env, cpu_X[30]);
1451 }
1452 break;
1453 case 0b01000: /* PACIA1716 */
1454 if (s->pauth_active) {
1455 gen_helper_pacia(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]);
1456 }
1457 break;
1458 case 0b01010: /* PACIB1716 */
1459 if (s->pauth_active) {
1460 gen_helper_pacib(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]);
1461 }
1462 break;
1463 case 0b01100: /* AUTIA1716 */
1464 if (s->pauth_active) {
1465 gen_helper_autia(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]);
1466 }
1467 break;
1468 case 0b01110: /* AUTIB1716 */
1469 if (s->pauth_active) {
1470 gen_helper_autib(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]);
1471 }
1472 break;
1473 case 0b11000: /* PACIAZ */
1474 if (s->pauth_active) {
1475 gen_helper_pacia(cpu_X[30], cpu_env, cpu_X[30],
1476 new_tmp_a64_zero(s));
1477 }
1478 break;
1479 case 0b11001: /* PACIASP */
1480 if (s->pauth_active) {
1481 gen_helper_pacia(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]);
1482 }
1483 break;
1484 case 0b11010: /* PACIBZ */
1485 if (s->pauth_active) {
1486 gen_helper_pacib(cpu_X[30], cpu_env, cpu_X[30],
1487 new_tmp_a64_zero(s));
1488 }
1489 break;
1490 case 0b11011: /* PACIBSP */
1491 if (s->pauth_active) {
1492 gen_helper_pacib(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]);
1493 }
1494 break;
1495 case 0b11100: /* AUTIAZ */
1496 if (s->pauth_active) {
1497 gen_helper_autia(cpu_X[30], cpu_env, cpu_X[30],
1498 new_tmp_a64_zero(s));
1499 }
1500 break;
1501 case 0b11101: /* AUTIASP */
1502 if (s->pauth_active) {
1503 gen_helper_autia(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]);
1504 }
1505 break;
1506 case 0b11110: /* AUTIBZ */
1507 if (s->pauth_active) {
1508 gen_helper_autib(cpu_X[30], cpu_env, cpu_X[30],
1509 new_tmp_a64_zero(s));
1510 }
1511 break;
1512 case 0b11111: /* AUTIBSP */
1513 if (s->pauth_active) {
1514 gen_helper_autib(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]);
1515 }
1516 break;
1517 default:
1518 /* default specified as NOP equivalent */
1519 break;
1520 }
1521 }
1522
1523 static void gen_clrex(DisasContext *s, uint32_t insn)
1524 {
1525 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
1526 }
1527
1528 /* CLREX, DSB, DMB, ISB */
1529 static void handle_sync(DisasContext *s, uint32_t insn,
1530 unsigned int op1, unsigned int op2, unsigned int crm)
1531 {
1532 TCGBar bar;
1533
1534 if (op1 != 3) {
1535 unallocated_encoding(s);
1536 return;
1537 }
1538
1539 switch (op2) {
1540 case 2: /* CLREX */
1541 gen_clrex(s, insn);
1542 return;
1543 case 4: /* DSB */
1544 case 5: /* DMB */
1545 switch (crm & 3) {
1546 case 1: /* MBReqTypes_Reads */
1547 bar = TCG_BAR_SC | TCG_MO_LD_LD | TCG_MO_LD_ST;
1548 break;
1549 case 2: /* MBReqTypes_Writes */
1550 bar = TCG_BAR_SC | TCG_MO_ST_ST;
1551 break;
1552 default: /* MBReqTypes_All */
1553 bar = TCG_BAR_SC | TCG_MO_ALL;
1554 break;
1555 }
1556 tcg_gen_mb(bar);
1557 return;
1558 case 6: /* ISB */
1559 /* We need to break the TB after this insn to execute
1560 * a self-modified code correctly and also to take
1561 * any pending interrupts immediately.
1562 */
1563 reset_btype(s);
1564 gen_goto_tb(s, 0, s->base.pc_next);
1565 return;
1566
1567 case 7: /* SB */
1568 if (crm != 0 || !dc_isar_feature(aa64_sb, s)) {
1569 goto do_unallocated;
1570 }
1571 /*
1572 * TODO: There is no speculation barrier opcode for TCG;
1573 * MB and end the TB instead.
1574 */
1575 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC);
1576 gen_goto_tb(s, 0, s->base.pc_next);
1577 return;
1578
1579 default:
1580 do_unallocated:
1581 unallocated_encoding(s);
1582 return;
1583 }
1584 }
1585
1586 static void gen_xaflag(void)
1587 {
1588 TCGv_i32 z = tcg_temp_new_i32();
1589
1590 tcg_gen_setcondi_i32(TCG_COND_EQ, z, cpu_ZF, 0);
1591
1592 /*
1593 * (!C & !Z) << 31
1594 * (!(C | Z)) << 31
1595 * ~((C | Z) << 31)
1596 * ~-(C | Z)
1597 * (C | Z) - 1
1598 */
1599 tcg_gen_or_i32(cpu_NF, cpu_CF, z);
1600 tcg_gen_subi_i32(cpu_NF, cpu_NF, 1);
1601
1602 /* !(Z & C) */
1603 tcg_gen_and_i32(cpu_ZF, z, cpu_CF);
1604 tcg_gen_xori_i32(cpu_ZF, cpu_ZF, 1);
1605
1606 /* (!C & Z) << 31 -> -(Z & ~C) */
1607 tcg_gen_andc_i32(cpu_VF, z, cpu_CF);
1608 tcg_gen_neg_i32(cpu_VF, cpu_VF);
1609
1610 /* C | Z */
1611 tcg_gen_or_i32(cpu_CF, cpu_CF, z);
1612
1613 tcg_temp_free_i32(z);
1614 }
1615
1616 static void gen_axflag(void)
1617 {
1618 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */
1619 tcg_gen_andc_i32(cpu_CF, cpu_CF, cpu_VF); /* C & !V */
1620
1621 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */
1622 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, cpu_VF);
1623
1624 tcg_gen_movi_i32(cpu_NF, 0);
1625 tcg_gen_movi_i32(cpu_VF, 0);
1626 }
1627
1628 /* MSR (immediate) - move immediate to processor state field */
1629 static void handle_msr_i(DisasContext *s, uint32_t insn,
1630 unsigned int op1, unsigned int op2, unsigned int crm)
1631 {
1632 TCGv_i32 t1;
1633 int op = op1 << 3 | op2;
1634
1635 /* End the TB by default, chaining is ok. */
1636 s->base.is_jmp = DISAS_TOO_MANY;
1637
1638 switch (op) {
1639 case 0x00: /* CFINV */
1640 if (crm != 0 || !dc_isar_feature(aa64_condm_4, s)) {
1641 goto do_unallocated;
1642 }
1643 tcg_gen_xori_i32(cpu_CF, cpu_CF, 1);
1644 s->base.is_jmp = DISAS_NEXT;
1645 break;
1646
1647 case 0x01: /* XAFlag */
1648 if (crm != 0 || !dc_isar_feature(aa64_condm_5, s)) {
1649 goto do_unallocated;
1650 }
1651 gen_xaflag();
1652 s->base.is_jmp = DISAS_NEXT;
1653 break;
1654
1655 case 0x02: /* AXFlag */
1656 if (crm != 0 || !dc_isar_feature(aa64_condm_5, s)) {
1657 goto do_unallocated;
1658 }
1659 gen_axflag();
1660 s->base.is_jmp = DISAS_NEXT;
1661 break;
1662
1663 case 0x03: /* UAO */
1664 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) {
1665 goto do_unallocated;
1666 }
1667 if (crm & 1) {
1668 set_pstate_bits(PSTATE_UAO);
1669 } else {
1670 clear_pstate_bits(PSTATE_UAO);
1671 }
1672 t1 = tcg_const_i32(s->current_el);
1673 gen_helper_rebuild_hflags_a64(cpu_env, t1);
1674 tcg_temp_free_i32(t1);
1675 break;
1676
1677 case 0x04: /* PAN */
1678 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) {
1679 goto do_unallocated;
1680 }
1681 if (crm & 1) {
1682 set_pstate_bits(PSTATE_PAN);
1683 } else {
1684 clear_pstate_bits(PSTATE_PAN);
1685 }
1686 t1 = tcg_const_i32(s->current_el);
1687 gen_helper_rebuild_hflags_a64(cpu_env, t1);
1688 tcg_temp_free_i32(t1);
1689 break;
1690
1691 case 0x05: /* SPSel */
1692 if (s->current_el == 0) {
1693 goto do_unallocated;
1694 }
1695 t1 = tcg_const_i32(crm & PSTATE_SP);
1696 gen_helper_msr_i_spsel(cpu_env, t1);
1697 tcg_temp_free_i32(t1);
1698 break;
1699
1700 case 0x1e: /* DAIFSet */
1701 t1 = tcg_const_i32(crm);
1702 gen_helper_msr_i_daifset(cpu_env, t1);
1703 tcg_temp_free_i32(t1);
1704 break;
1705
1706 case 0x1f: /* DAIFClear */
1707 t1 = tcg_const_i32(crm);
1708 gen_helper_msr_i_daifclear(cpu_env, t1);
1709 tcg_temp_free_i32(t1);
1710 /* For DAIFClear, exit the cpu loop to re-evaluate pending IRQs. */
1711 s->base.is_jmp = DISAS_UPDATE_EXIT;
1712 break;
1713
1714 case 0x1c: /* TCO */
1715 if (dc_isar_feature(aa64_mte, s)) {
1716 /* Full MTE is enabled -- set the TCO bit as directed. */
1717 if (crm & 1) {
1718 set_pstate_bits(PSTATE_TCO);
1719 } else {
1720 clear_pstate_bits(PSTATE_TCO);
1721 }
1722 t1 = tcg_const_i32(s->current_el);
1723 gen_helper_rebuild_hflags_a64(cpu_env, t1);
1724 tcg_temp_free_i32(t1);
1725 /* Many factors, including TCO, go into MTE_ACTIVE. */
1726 s->base.is_jmp = DISAS_UPDATE_NOCHAIN;
1727 } else if (dc_isar_feature(aa64_mte_insn_reg, s)) {
1728 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */
1729 s->base.is_jmp = DISAS_NEXT;
1730 } else {
1731 goto do_unallocated;
1732 }
1733 break;
1734
1735 default:
1736 do_unallocated:
1737 unallocated_encoding(s);
1738 return;
1739 }
1740 }
1741
1742 static void gen_get_nzcv(TCGv_i64 tcg_rt)
1743 {
1744 TCGv_i32 tmp = tcg_temp_new_i32();
1745 TCGv_i32 nzcv = tcg_temp_new_i32();
1746
1747 /* build bit 31, N */
1748 tcg_gen_andi_i32(nzcv, cpu_NF, (1U << 31));
1749 /* build bit 30, Z */
1750 tcg_gen_setcondi_i32(TCG_COND_EQ, tmp, cpu_ZF, 0);
1751 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 30, 1);
1752 /* build bit 29, C */
1753 tcg_gen_deposit_i32(nzcv, nzcv, cpu_CF, 29, 1);
1754 /* build bit 28, V */
1755 tcg_gen_shri_i32(tmp, cpu_VF, 31);
1756 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 28, 1);
1757 /* generate result */
1758 tcg_gen_extu_i32_i64(tcg_rt, nzcv);
1759
1760 tcg_temp_free_i32(nzcv);
1761 tcg_temp_free_i32(tmp);
1762 }
1763
1764 static void gen_set_nzcv(TCGv_i64 tcg_rt)
1765 {
1766 TCGv_i32 nzcv = tcg_temp_new_i32();
1767
1768 /* take NZCV from R[t] */
1769 tcg_gen_extrl_i64_i32(nzcv, tcg_rt);
1770
1771 /* bit 31, N */
1772 tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31));
1773 /* bit 30, Z */
1774 tcg_gen_andi_i32(cpu_ZF, nzcv, (1 << 30));
1775 tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_ZF, cpu_ZF, 0);
1776 /* bit 29, C */
1777 tcg_gen_andi_i32(cpu_CF, nzcv, (1 << 29));
1778 tcg_gen_shri_i32(cpu_CF, cpu_CF, 29);
1779 /* bit 28, V */
1780 tcg_gen_andi_i32(cpu_VF, nzcv, (1 << 28));
1781 tcg_gen_shli_i32(cpu_VF, cpu_VF, 3);
1782 tcg_temp_free_i32(nzcv);
1783 }
1784
1785 /* MRS - move from system register
1786 * MSR (register) - move to system register
1787 * SYS
1788 * SYSL
1789 * These are all essentially the same insn in 'read' and 'write'
1790 * versions, with varying op0 fields.
1791 */
1792 static void handle_sys(DisasContext *s, uint32_t insn, bool isread,
1793 unsigned int op0, unsigned int op1, unsigned int op2,
1794 unsigned int crn, unsigned int crm, unsigned int rt)
1795 {
1796 const ARMCPRegInfo *ri;
1797 TCGv_i64 tcg_rt;
1798
1799 ri = get_arm_cp_reginfo(s->cp_regs,
1800 ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP,
1801 crn, crm, op0, op1, op2));
1802
1803 if (!ri) {
1804 /* Unknown register; this might be a guest error or a QEMU
1805 * unimplemented feature.
1806 */
1807 qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch64 "
1808 "system register op0:%d op1:%d crn:%d crm:%d op2:%d\n",
1809 isread ? "read" : "write", op0, op1, crn, crm, op2);
1810 unallocated_encoding(s);
1811 return;
1812 }
1813
1814 /* Check access permissions */
1815 if (!cp_access_ok(s->current_el, ri, isread)) {
1816 unallocated_encoding(s);
1817 return;
1818 }
1819
1820 if (ri->accessfn) {
1821 /* Emit code to perform further access permissions checks at
1822 * runtime; this may result in an exception.
1823 */
1824 TCGv_ptr tmpptr;
1825 TCGv_i32 tcg_syn, tcg_isread;
1826 uint32_t syndrome;
1827
1828 gen_a64_set_pc_im(s->pc_curr);
1829 tmpptr = tcg_const_ptr(ri);
1830 syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
1831 tcg_syn = tcg_const_i32(syndrome);
1832 tcg_isread = tcg_const_i32(isread);
1833 gen_helper_access_check_cp_reg(cpu_env, tmpptr, tcg_syn, tcg_isread);
1834 tcg_temp_free_ptr(tmpptr);
1835 tcg_temp_free_i32(tcg_syn);
1836 tcg_temp_free_i32(tcg_isread);
1837 } else if (ri->type & ARM_CP_RAISES_EXC) {
1838 /*
1839 * The readfn or writefn might raise an exception;
1840 * synchronize the CPU state in case it does.
1841 */
1842 gen_a64_set_pc_im(s->pc_curr);
1843 }
1844
1845 /* Handle special cases first */
1846 switch (ri->type & ~(ARM_CP_FLAG_MASK & ~ARM_CP_SPECIAL)) {
1847 case ARM_CP_NOP:
1848 return;
1849 case ARM_CP_NZCV:
1850 tcg_rt = cpu_reg(s, rt);
1851 if (isread) {
1852 gen_get_nzcv(tcg_rt);
1853 } else {
1854 gen_set_nzcv(tcg_rt);
1855 }
1856 return;
1857 case ARM_CP_CURRENTEL:
1858 /* Reads as current EL value from pstate, which is
1859 * guaranteed to be constant by the tb flags.
1860 */
1861 tcg_rt = cpu_reg(s, rt);
1862 tcg_gen_movi_i64(tcg_rt, s->current_el << 2);
1863 return;
1864 case ARM_CP_DC_ZVA:
1865 /* Writes clear the aligned block of memory which rt points into. */
1866 if (s->mte_active[0]) {
1867 TCGv_i32 t_desc;
1868 int desc = 0;
1869
1870 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
1871 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
1872 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
1873 t_desc = tcg_const_i32(desc);
1874
1875 tcg_rt = new_tmp_a64(s);
1876 gen_helper_mte_check_zva(tcg_rt, cpu_env, t_desc, cpu_reg(s, rt));
1877 tcg_temp_free_i32(t_desc);
1878 } else {
1879 tcg_rt = clean_data_tbi(s, cpu_reg(s, rt));
1880 }
1881 gen_helper_dc_zva(cpu_env, tcg_rt);
1882 return;
1883 case ARM_CP_DC_GVA:
1884 {
1885 TCGv_i64 clean_addr, tag;
1886
1887 /*
1888 * DC_GVA, like DC_ZVA, requires that we supply the original
1889 * pointer for an invalid page. Probe that address first.
1890 */
1891 tcg_rt = cpu_reg(s, rt);
1892 clean_addr = clean_data_tbi(s, tcg_rt);
1893 gen_probe_access(s, clean_addr, MMU_DATA_STORE, MO_8);
1894
1895 if (s->ata) {
1896 /* Extract the tag from the register to match STZGM. */
1897 tag = tcg_temp_new_i64();
1898 tcg_gen_shri_i64(tag, tcg_rt, 56);
1899 gen_helper_stzgm_tags(cpu_env, clean_addr, tag);
1900 tcg_temp_free_i64(tag);
1901 }
1902 }
1903 return;
1904 case ARM_CP_DC_GZVA:
1905 {
1906 TCGv_i64 clean_addr, tag;
1907
1908 /* For DC_GZVA, we can rely on DC_ZVA for the proper fault. */
1909 tcg_rt = cpu_reg(s, rt);
1910 clean_addr = clean_data_tbi(s, tcg_rt);
1911 gen_helper_dc_zva(cpu_env, clean_addr);
1912
1913 if (s->ata) {
1914 /* Extract the tag from the register to match STZGM. */
1915 tag = tcg_temp_new_i64();
1916 tcg_gen_shri_i64(tag, tcg_rt, 56);
1917 gen_helper_stzgm_tags(cpu_env, clean_addr, tag);
1918 tcg_temp_free_i64(tag);
1919 }
1920 }
1921 return;
1922 default:
1923 break;
1924 }
1925 if ((ri->type & ARM_CP_FPU) && !fp_access_check(s)) {
1926 return;
1927 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
1928 return;
1929 }
1930
1931 if ((tb_cflags(s->base.tb) & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) {
1932 gen_io_start();
1933 }
1934
1935 tcg_rt = cpu_reg(s, rt);
1936
1937 if (isread) {
1938 if (ri->type & ARM_CP_CONST) {
1939 tcg_gen_movi_i64(tcg_rt, ri->resetvalue);
1940 } else if (ri->readfn) {
1941 TCGv_ptr tmpptr;
1942 tmpptr = tcg_const_ptr(ri);
1943 gen_helper_get_cp_reg64(tcg_rt, cpu_env, tmpptr);
1944 tcg_temp_free_ptr(tmpptr);
1945 } else {
1946 tcg_gen_ld_i64(tcg_rt, cpu_env, ri->fieldoffset);
1947 }
1948 } else {
1949 if (ri->type & ARM_CP_CONST) {
1950 /* If not forbidden by access permissions, treat as WI */
1951 return;
1952 } else if (ri->writefn) {
1953 TCGv_ptr tmpptr;
1954 tmpptr = tcg_const_ptr(ri);
1955 gen_helper_set_cp_reg64(cpu_env, tmpptr, tcg_rt);
1956 tcg_temp_free_ptr(tmpptr);
1957 } else {
1958 tcg_gen_st_i64(tcg_rt, cpu_env, ri->fieldoffset);
1959 }
1960 }
1961
1962 if ((tb_cflags(s->base.tb) & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) {
1963 /* I/O operations must end the TB here (whether read or write) */
1964 s->base.is_jmp = DISAS_UPDATE_EXIT;
1965 }
1966 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) {
1967 /*
1968 * A write to any coprocessor regiser that ends a TB
1969 * must rebuild the hflags for the next TB.
1970 */
1971 TCGv_i32 tcg_el = tcg_const_i32(s->current_el);
1972 gen_helper_rebuild_hflags_a64(cpu_env, tcg_el);
1973 tcg_temp_free_i32(tcg_el);
1974 /*
1975 * We default to ending the TB on a coprocessor register write,
1976 * but allow this to be suppressed by the register definition
1977 * (usually only necessary to work around guest bugs).
1978 */
1979 s->base.is_jmp = DISAS_UPDATE_EXIT;
1980 }
1981 }
1982
1983 /* System
1984 * 31 22 21 20 19 18 16 15 12 11 8 7 5 4 0
1985 * +---------------------+---+-----+-----+-------+-------+-----+------+
1986 * | 1 1 0 1 0 1 0 1 0 0 | L | op0 | op1 | CRn | CRm | op2 | Rt |
1987 * +---------------------+---+-----+-----+-------+-------+-----+------+
1988 */
1989 static void disas_system(DisasContext *s, uint32_t insn)
1990 {
1991 unsigned int l, op0, op1, crn, crm, op2, rt;
1992 l = extract32(insn, 21, 1);
1993 op0 = extract32(insn, 19, 2);
1994 op1 = extract32(insn, 16, 3);
1995 crn = extract32(insn, 12, 4);
1996 crm = extract32(insn, 8, 4);
1997 op2 = extract32(insn, 5, 3);
1998 rt = extract32(insn, 0, 5);
1999
2000 if (op0 == 0) {
2001 if (l || rt != 31) {
2002 unallocated_encoding(s);
2003 return;
2004 }
2005 switch (crn) {
2006 case 2: /* HINT (including allocated hints like NOP, YIELD, etc) */
2007 handle_hint(s, insn, op1, op2, crm);
2008 break;
2009 case 3: /* CLREX, DSB, DMB, ISB */
2010 handle_sync(s, insn, op1, op2, crm);
2011 break;
2012 case 4: /* MSR (immediate) */
2013 handle_msr_i(s, insn, op1, op2, crm);
2014 break;
2015 default:
2016 unallocated_encoding(s);
2017 break;
2018 }
2019 return;
2020 }
2021 handle_sys(s, insn, l, op0, op1, op2, crn, crm, rt);
2022 }
2023
2024 /* Exception generation
2025 *
2026 * 31 24 23 21 20 5 4 2 1 0
2027 * +-----------------+-----+------------------------+-----+----+
2028 * | 1 1 0 1 0 1 0 0 | opc | imm16 | op2 | LL |
2029 * +-----------------------+------------------------+----------+
2030 */
2031 static void disas_exc(DisasContext *s, uint32_t insn)
2032 {
2033 int opc = extract32(insn, 21, 3);
2034 int op2_ll = extract32(insn, 0, 5);
2035 int imm16 = extract32(insn, 5, 16);
2036 TCGv_i32 tmp;
2037
2038 switch (opc) {
2039 case 0:
2040 /* For SVC, HVC and SMC we advance the single-step state
2041 * machine before taking the exception. This is architecturally
2042 * mandated, to ensure that single-stepping a system call
2043 * instruction works properly.
2044 */
2045 switch (op2_ll) {
2046 case 1: /* SVC */
2047 gen_ss_advance(s);
2048 gen_exception_insn(s, s->base.pc_next, EXCP_SWI,
2049 syn_aa64_svc(imm16), default_exception_el(s));
2050 break;
2051 case 2: /* HVC */
2052 if (s->current_el == 0) {
2053 unallocated_encoding(s);
2054 break;
2055 }
2056 /* The pre HVC helper handles cases when HVC gets trapped
2057 * as an undefined insn by runtime configuration.
2058 */
2059 gen_a64_set_pc_im(s->pc_curr);
2060 gen_helper_pre_hvc(cpu_env);
2061 gen_ss_advance(s);
2062 gen_exception_insn(s, s->base.pc_next, EXCP_HVC,
2063 syn_aa64_hvc(imm16), 2);
2064 break;
2065 case 3: /* SMC */
2066 if (s->current_el == 0) {
2067 unallocated_encoding(s);
2068 break;
2069 }
2070 gen_a64_set_pc_im(s->pc_curr);
2071 tmp = tcg_const_i32(syn_aa64_smc(imm16));
2072 gen_helper_pre_smc(cpu_env, tmp);
2073 tcg_temp_free_i32(tmp);
2074 gen_ss_advance(s);
2075 gen_exception_insn(s, s->base.pc_next, EXCP_SMC,
2076 syn_aa64_smc(imm16), 3);
2077 break;
2078 default:
2079 unallocated_encoding(s);
2080 break;
2081 }
2082 break;
2083 case 1:
2084 if (op2_ll != 0) {
2085 unallocated_encoding(s);
2086 break;
2087 }
2088 /* BRK */
2089 gen_exception_bkpt_insn(s, syn_aa64_bkpt(imm16));
2090 break;
2091 case 2:
2092 if (op2_ll != 0) {
2093 unallocated_encoding(s);
2094 break;
2095 }
2096 /* HLT. This has two purposes.
2097 * Architecturally, it is an external halting debug instruction.
2098 * Since QEMU doesn't implement external debug, we treat this as
2099 * it is required for halting debug disabled: it will UNDEF.
2100 * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
2101 */
2102 if (semihosting_enabled() && imm16 == 0xf000) {
2103 #ifndef CONFIG_USER_ONLY
2104 /* In system mode, don't allow userspace access to semihosting,
2105 * to provide some semblance of security (and for consistency
2106 * with our 32-bit semihosting).
2107 */
2108 if (s->current_el == 0) {
2109 unsupported_encoding(s, insn);
2110 break;
2111 }
2112 #endif
2113 gen_exception_internal_insn(s, s->pc_curr, EXCP_SEMIHOST);
2114 } else {
2115 unsupported_encoding(s, insn);
2116 }
2117 break;
2118 case 5:
2119 if (op2_ll < 1 || op2_ll > 3) {
2120 unallocated_encoding(s);
2121 break;
2122 }
2123 /* DCPS1, DCPS2, DCPS3 */
2124 unsupported_encoding(s, insn);
2125 break;
2126 default:
2127 unallocated_encoding(s);
2128 break;
2129 }
2130 }
2131
2132 /* Unconditional branch (register)
2133 * 31 25 24 21 20 16 15 10 9 5 4 0
2134 * +---------------+-------+-------+-------+------+-------+
2135 * | 1 1 0 1 0 1 1 | opc | op2 | op3 | Rn | op4 |
2136 * +---------------+-------+-------+-------+------+-------+
2137 */
2138 static void disas_uncond_b_reg(DisasContext *s, uint32_t insn)
2139 {
2140 unsigned int opc, op2, op3, rn, op4;
2141 unsigned btype_mod = 2; /* 0: BR, 1: BLR, 2: other */
2142 TCGv_i64 dst;
2143 TCGv_i64 modifier;
2144
2145 opc = extract32(insn, 21, 4);
2146 op2 = extract32(insn, 16, 5);
2147 op3 = extract32(insn, 10, 6);
2148 rn = extract32(insn, 5, 5);
2149 op4 = extract32(insn, 0, 5);
2150
2151 if (op2 != 0x1f) {
2152 goto do_unallocated;
2153 }
2154
2155 switch (opc) {
2156 case 0: /* BR */
2157 case 1: /* BLR */
2158 case 2: /* RET */
2159 btype_mod = opc;
2160 switch (op3) {
2161 case 0:
2162 /* BR, BLR, RET */
2163 if (op4 != 0) {
2164 goto do_unallocated;
2165 }
2166 dst = cpu_reg(s, rn);
2167 break;
2168
2169 case 2:
2170 case 3:
2171 if (!dc_isar_feature(aa64_pauth, s)) {
2172 goto do_unallocated;
2173 }
2174 if (opc == 2) {
2175 /* RETAA, RETAB */
2176 if (rn != 0x1f || op4 != 0x1f) {
2177 goto do_unallocated;
2178 }
2179 rn = 30;
2180 modifier = cpu_X[31];
2181 } else {
2182 /* BRAAZ, BRABZ, BLRAAZ, BLRABZ */
2183 if (op4 != 0x1f) {
2184 goto do_unallocated;
2185 }
2186 modifier = new_tmp_a64_zero(s);
2187 }
2188 if (s->pauth_active) {
2189 dst = new_tmp_a64(s);
2190 if (op3 == 2) {
2191 gen_helper_autia(dst, cpu_env, cpu_reg(s, rn), modifier);
2192 } else {
2193 gen_helper_autib(dst, cpu_env, cpu_reg(s, rn), modifier);
2194 }
2195 } else {
2196 dst = cpu_reg(s, rn);
2197 }
2198 break;
2199
2200 default:
2201 goto do_unallocated;
2202 }
2203 gen_a64_set_pc(s, dst);
2204 /* BLR also needs to load return address */
2205 if (opc == 1) {
2206 tcg_gen_movi_i64(cpu_reg(s, 30), s->base.pc_next);
2207 }
2208 break;
2209
2210 case 8: /* BRAA */
2211 case 9: /* BLRAA */
2212 if (!dc_isar_feature(aa64_pauth, s)) {
2213 goto do_unallocated;
2214 }
2215 if ((op3 & ~1) != 2) {
2216 goto do_unallocated;
2217 }
2218 btype_mod = opc & 1;
2219 if (s->pauth_active) {
2220 dst = new_tmp_a64(s);
2221 modifier = cpu_reg_sp(s, op4);
2222 if (op3 == 2) {
2223 gen_helper_autia(dst, cpu_env, cpu_reg(s, rn), modifier);
2224 } else {
2225 gen_helper_autib(dst, cpu_env, cpu_reg(s, rn), modifier);
2226 }
2227 } else {
2228 dst = cpu_reg(s, rn);
2229 }
2230 gen_a64_set_pc(s, dst);
2231 /* BLRAA also needs to load return address */
2232 if (opc == 9) {
2233 tcg_gen_movi_i64(cpu_reg(s, 30), s->base.pc_next);
2234 }
2235 break;
2236
2237 case 4: /* ERET */
2238 if (s->current_el == 0) {
2239 goto do_unallocated;
2240 }
2241 switch (op3) {
2242 case 0: /* ERET */
2243 if (op4 != 0) {
2244 goto do_unallocated;
2245 }
2246 dst = tcg_temp_new_i64();
2247 tcg_gen_ld_i64(dst, cpu_env,
2248 offsetof(CPUARMState, elr_el[s->current_el]));
2249 break;
2250
2251 case 2: /* ERETAA */
2252 case 3: /* ERETAB */
2253 if (!dc_isar_feature(aa64_pauth, s)) {
2254 goto do_unallocated;
2255 }
2256 if (rn != 0x1f || op4 != 0x1f) {
2257 goto do_unallocated;
2258 }
2259 dst = tcg_temp_new_i64();
2260 tcg_gen_ld_i64(dst, cpu_env,
2261 offsetof(CPUARMState, elr_el[s->current_el]));
2262 if (s->pauth_active) {
2263 modifier = cpu_X[31];
2264 if (op3 == 2) {
2265 gen_helper_autia(dst, cpu_env, dst, modifier);
2266 } else {
2267 gen_helper_autib(dst, cpu_env, dst, modifier);
2268 }
2269 }
2270 break;
2271
2272 default:
2273 goto do_unallocated;
2274 }
2275 if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
2276 gen_io_start();
2277 }
2278
2279 gen_helper_exception_return(cpu_env, dst);
2280 tcg_temp_free_i64(dst);
2281 /* Must exit loop to check un-masked IRQs */
2282 s->base.is_jmp = DISAS_EXIT;
2283 return;
2284
2285 case 5: /* DRPS */
2286 if (op3 != 0 || op4 != 0 || rn != 0x1f) {
2287 goto do_unallocated;
2288 } else {
2289 unsupported_encoding(s, insn);
2290 }
2291 return;
2292
2293 default:
2294 do_unallocated:
2295 unallocated_encoding(s);
2296 return;
2297 }
2298
2299 switch (btype_mod) {
2300 case 0: /* BR */
2301 if (dc_isar_feature(aa64_bti, s)) {
2302 /* BR to {x16,x17} or !guard -> 1, else 3. */
2303 set_btype(s, rn == 16 || rn == 17 || !s->guarded_page ? 1 : 3);
2304 }
2305 break;
2306
2307 case 1: /* BLR */
2308 if (dc_isar_feature(aa64_bti, s)) {
2309 /* BLR sets BTYPE to 2, regardless of source guarded page. */
2310 set_btype(s, 2);
2311 }
2312 break;
2313
2314 default: /* RET or none of the above. */
2315 /* BTYPE will be set to 0 by normal end-of-insn processing. */
2316 break;
2317 }
2318
2319 s->base.is_jmp = DISAS_JUMP;
2320 }
2321
2322 /* Branches, exception generating and system instructions */
2323 static void disas_b_exc_sys(DisasContext *s, uint32_t insn)
2324 {
2325 switch (extract32(insn, 25, 7)) {
2326 case 0x0a: case 0x0b:
2327 case 0x4a: case 0x4b: /* Unconditional branch (immediate) */
2328 disas_uncond_b_imm(s, insn);
2329 break;
2330 case 0x1a: case 0x5a: /* Compare & branch (immediate) */
2331 disas_comp_b_imm(s, insn);
2332 break;
2333 case 0x1b: case 0x5b: /* Test & branch (immediate) */
2334 disas_test_b_imm(s, insn);
2335 break;
2336 case 0x2a: /* Conditional branch (immediate) */
2337 disas_cond_b_imm(s, insn);
2338 break;
2339 case 0x6a: /* Exception generation / System */
2340 if (insn & (1 << 24)) {
2341 if (extract32(insn, 22, 2) == 0) {
2342 disas_system(s, insn);
2343 } else {
2344 unallocated_encoding(s);
2345 }
2346 } else {
2347 disas_exc(s, insn);
2348 }
2349 break;
2350 case 0x6b: /* Unconditional branch (register) */
2351 disas_uncond_b_reg(s, insn);
2352 break;
2353 default:
2354 unallocated_encoding(s);
2355 break;
2356 }
2357 }
2358
2359 /*
2360 * Load/Store exclusive instructions are implemented by remembering
2361 * the value/address loaded, and seeing if these are the same
2362 * when the store is performed. This is not actually the architecturally
2363 * mandated semantics, but it works for typical guest code sequences
2364 * and avoids having to monitor regular stores.
2365 *
2366 * The store exclusive uses the atomic cmpxchg primitives to avoid
2367 * races in multi-threaded linux-user and when MTTCG softmmu is
2368 * enabled.
2369 */
2370 static void gen_load_exclusive(DisasContext *s, int rt, int rt2,
2371 TCGv_i64 addr, int size, bool is_pair)
2372 {
2373 int idx = get_mem_index(s);
2374 MemOp memop = s->be_data;
2375
2376 g_assert(size <= 3);
2377 if (is_pair) {
2378 g_assert(size >= 2);
2379 if (size == 2) {
2380 /* The pair must be single-copy atomic for the doubleword. */
2381 memop |= MO_64 | MO_ALIGN;
2382 tcg_gen_qemu_ld_i64(cpu_exclusive_val, addr, idx, memop);
2383 if (s->be_data == MO_LE) {
2384 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 0, 32);
2385 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 32, 32);
2386 } else {
2387 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 32, 32);
2388 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 0, 32);
2389 }
2390 } else {
2391 /* The pair must be single-copy atomic for *each* doubleword, not
2392 the entire quadword, however it must be quadword aligned. */
2393 memop |= MO_64;
2394 tcg_gen_qemu_ld_i64(cpu_exclusive_val, addr, idx,
2395 memop | MO_ALIGN_16);
2396
2397 TCGv_i64 addr2 = tcg_temp_new_i64();
2398 tcg_gen_addi_i64(addr2, addr, 8);
2399 tcg_gen_qemu_ld_i64(cpu_exclusive_high, addr2, idx, memop);
2400 tcg_temp_free_i64(addr2);
2401
2402 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2403 tcg_gen_mov_i64(cpu_reg(s, rt2), cpu_exclusive_high);
2404 }
2405 } else {
2406 memop |= size | MO_ALIGN;
2407 tcg_gen_qemu_ld_i64(cpu_exclusive_val, addr, idx, memop);
2408 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2409 }
2410 tcg_gen_mov_i64(cpu_exclusive_addr, addr);
2411 }
2412
2413 static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
2414 TCGv_i64 addr, int size, int is_pair)
2415 {
2416 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr]
2417 * && (!is_pair || env->exclusive_high == [addr + datasize])) {
2418 * [addr] = {Rt};
2419 * if (is_pair) {
2420 * [addr + datasize] = {Rt2};
2421 * }
2422 * {Rd} = 0;
2423 * } else {
2424 * {Rd} = 1;
2425 * }
2426 * env->exclusive_addr = -1;
2427 */
2428 TCGLabel *fail_label = gen_new_label();
2429 TCGLabel *done_label = gen_new_label();
2430 TCGv_i64 tmp;
2431
2432 tcg_gen_brcond_i64(TCG_COND_NE, addr, cpu_exclusive_addr, fail_label);
2433
2434 tmp = tcg_temp_new_i64();
2435 if (is_pair) {
2436 if (size == 2) {
2437 if (s->be_data == MO_LE) {
2438 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt), cpu_reg(s, rt2));
2439 } else {
2440 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt2), cpu_reg(s, rt));
2441 }
2442 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr,
2443 cpu_exclusive_val, tmp,
2444 get_mem_index(s),
2445 MO_64 | MO_ALIGN | s->be_data);
2446 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2447 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
2448 if (!HAVE_CMPXCHG128) {
2449 gen_helper_exit_atomic(cpu_env);
2450 s->base.is_jmp = DISAS_NORETURN;
2451 } else if (s->be_data == MO_LE) {
2452 gen_helper_paired_cmpxchg64_le_parallel(tmp, cpu_env,
2453 cpu_exclusive_addr,
2454 cpu_reg(s, rt),
2455 cpu_reg(s, rt2));
2456 } else {
2457 gen_helper_paired_cmpxchg64_be_parallel(tmp, cpu_env,
2458 cpu_exclusive_addr,
2459 cpu_reg(s, rt),
2460 cpu_reg(s, rt2));
2461 }
2462 } else if (s->be_data == MO_LE) {
2463 gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr,
2464 cpu_reg(s, rt), cpu_reg(s, rt2));
2465 } else {
2466 gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr,
2467 cpu_reg(s, rt), cpu_reg(s, rt2));
2468 }
2469 } else {
2470 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,
2471 cpu_reg(s, rt), get_mem_index(s),
2472 size | MO_ALIGN | s->be_data);
2473 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2474 }
2475 tcg_gen_mov_i64(cpu_reg(s, rd), tmp);
2476 tcg_temp_free_i64(tmp);
2477 tcg_gen_br(done_label);
2478
2479 gen_set_label(fail_label);
2480 tcg_gen_movi_i64(cpu_reg(s, rd), 1);
2481 gen_set_label(done_label);
2482 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
2483 }
2484
2485 static void gen_compare_and_swap(DisasContext *s, int rs, int rt,
2486 int rn, int size)
2487 {
2488 TCGv_i64 tcg_rs = cpu_reg(s, rs);
2489 TCGv_i64 tcg_rt = cpu_reg(s, rt);
2490 int memidx = get_mem_index(s);
2491 TCGv_i64 clean_addr;
2492
2493 if (rn == 31) {
2494 gen_check_sp_alignment(s);
2495 }
2496 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, size);
2497 tcg_gen_atomic_cmpxchg_i64(tcg_rs, clean_addr, tcg_rs, tcg_rt, memidx,
2498 size | MO_ALIGN | s->be_data);
2499 }
2500
2501 static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt,
2502 int rn, int size)
2503 {
2504 TCGv_i64 s1 = cpu_reg(s, rs);
2505 TCGv_i64 s2 = cpu_reg(s, rs + 1);
2506 TCGv_i64 t1 = cpu_reg(s, rt);
2507 TCGv_i64 t2 = cpu_reg(s, rt + 1);
2508 TCGv_i64 clean_addr;
2509 int memidx = get_mem_index(s);
2510
2511 if (rn == 31) {
2512 gen_check_sp_alignment(s);
2513 }
2514
2515 /* This is a single atomic access, despite the "pair". */
2516 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, size + 1);
2517
2518 if (size == 2) {
2519 TCGv_i64 cmp = tcg_temp_new_i64();
2520 TCGv_i64 val = tcg_temp_new_i64();
2521
2522 if (s->be_data == MO_LE) {
2523 tcg_gen_concat32_i64(val, t1, t2);
2524 tcg_gen_concat32_i64(cmp, s1, s2);
2525 } else {
2526 tcg_gen_concat32_i64(val, t2, t1);
2527 tcg_gen_concat32_i64(cmp, s2, s1);
2528 }
2529
2530 tcg_gen_atomic_cmpxchg_i64(cmp, clean_addr, cmp, val, memidx,
2531 MO_64 | MO_ALIGN | s->be_data);
2532 tcg_temp_free_i64(val);
2533
2534 if (s->be_data == MO_LE) {
2535 tcg_gen_extr32_i64(s1, s2, cmp);
2536 } else {
2537 tcg_gen_extr32_i64(s2, s1, cmp);
2538 }
2539 tcg_temp_free_i64(cmp);
2540 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
2541 if (HAVE_CMPXCHG128) {
2542 TCGv_i32 tcg_rs = tcg_const_i32(rs);
2543 if (s->be_data == MO_LE) {
2544 gen_helper_casp_le_parallel(cpu_env, tcg_rs,
2545 clean_addr, t1, t2);
2546 } else {
2547 gen_helper_casp_be_parallel(cpu_env, tcg_rs,
2548 clean_addr, t1, t2);
2549 }
2550 tcg_temp_free_i32(tcg_rs);
2551 } else {
2552 gen_helper_exit_atomic(cpu_env);
2553 s->base.is_jmp = DISAS_NORETURN;
2554 }
2555 } else {
2556 TCGv_i64 d1 = tcg_temp_new_i64();
2557 TCGv_i64 d2 = tcg_temp_new_i64();
2558 TCGv_i64 a2 = tcg_temp_new_i64();
2559 TCGv_i64 c1 = tcg_temp_new_i64();
2560 TCGv_i64 c2 = tcg_temp_new_i64();
2561 TCGv_i64 zero = tcg_const_i64(0);
2562
2563 /* Load the two words, in memory order. */
2564 tcg_gen_qemu_ld_i64(d1, clean_addr, memidx,
2565 MO_64 | MO_ALIGN_16 | s->be_data);
2566 tcg_gen_addi_i64(a2, clean_addr, 8);
2567 tcg_gen_qemu_ld_i64(d2, a2, memidx, MO_64 | s->be_data);
2568
2569 /* Compare the two words, also in memory order. */
2570 tcg_gen_setcond_i64(TCG_COND_EQ, c1, d1, s1);
2571 tcg_gen_setcond_i64(TCG_COND_EQ, c2, d2, s2);
2572 tcg_gen_and_i64(c2, c2, c1);
2573
2574 /* If compare equal, write back new data, else write back old data. */
2575 tcg_gen_movcond_i64(TCG_COND_NE, c1, c2, zero, t1, d1);
2576 tcg_gen_movcond_i64(TCG_COND_NE, c2, c2, zero, t2, d2);
2577 tcg_gen_qemu_st_i64(c1, clean_addr, memidx, MO_64 | s->be_data);
2578 tcg_gen_qemu_st_i64(c2, a2, memidx, MO_64 | s->be_data);
2579 tcg_temp_free_i64(a2);
2580 tcg_temp_free_i64(c1);
2581 tcg_temp_free_i64(c2);
2582 tcg_temp_free_i64(zero);
2583
2584 /* Write back the data from memory to Rs. */
2585 tcg_gen_mov_i64(s1, d1);
2586 tcg_gen_mov_i64(s2, d2);
2587 tcg_temp_free_i64(d1);
2588 tcg_temp_free_i64(d2);
2589 }
2590 }
2591
2592 /* Update the Sixty-Four bit (SF) registersize. This logic is derived
2593 * from the ARMv8 specs for LDR (Shared decode for all encodings).
2594 */
2595 static bool disas_ldst_compute_iss_sf(int size, bool is_signed, int opc)
2596 {
2597 int opc0 = extract32(opc, 0, 1);
2598 int regsize;
2599
2600 if (is_signed) {
2601 regsize = opc0 ? 32 : 64;
2602 } else {
2603 regsize = size == 3 ? 64 : 32;
2604 }
2605 return regsize == 64;
2606 }
2607
2608 /* Load/store exclusive
2609 *
2610 * 31 30 29 24 23 22 21 20 16 15 14 10 9 5 4 0
2611 * +-----+-------------+----+---+----+------+----+-------+------+------+
2612 * | sz | 0 0 1 0 0 0 | o2 | L | o1 | Rs | o0 | Rt2 | Rn | Rt |
2613 * +-----+-------------+----+---+----+------+----+-------+------+------+
2614 *
2615 * sz: 00 -> 8 bit, 01 -> 16 bit, 10 -> 32 bit, 11 -> 64 bit
2616 * L: 0 -> store, 1 -> load
2617 * o2: 0 -> exclusive, 1 -> not
2618 * o1: 0 -> single register, 1 -> register pair
2619 * o0: 1 -> load-acquire/store-release, 0 -> not
2620 */
2621 static void disas_ldst_excl(DisasContext *s, uint32_t insn)
2622 {
2623 int rt = extract32(insn, 0, 5);
2624 int rn = extract32(insn, 5, 5);
2625 int rt2 = extract32(insn, 10, 5);
2626 int rs = extract32(insn, 16, 5);
2627 int is_lasr = extract32(insn, 15, 1);
2628 int o2_L_o1_o0 = extract32(insn, 21, 3) * 2 | is_lasr;
2629 int size = extract32(insn, 30, 2);
2630 TCGv_i64 clean_addr;
2631
2632 switch (o2_L_o1_o0) {
2633 case 0x0: /* STXR */
2634 case 0x1: /* STLXR */
2635 if (rn == 31) {
2636 gen_check_sp_alignment(s);
2637 }
2638 if (is_lasr) {
2639 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2640 }
2641 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn),
2642 true, rn != 31, size);
2643 gen_store_exclusive(s, rs, rt, rt2, clean_addr, size, false);
2644 return;
2645
2646 case 0x4: /* LDXR */
2647 case 0x5: /* LDAXR */
2648 if (rn == 31) {
2649 gen_check_sp_alignment(s);
2650 }
2651 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn),
2652 false, rn != 31, size);
2653 s->is_ldex = true;
2654 gen_load_exclusive(s, rt, rt2, clean_addr, size, false);
2655 if (is_lasr) {
2656 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2657 }
2658 return;
2659
2660 case 0x8: /* STLLR */
2661 if (!dc_isar_feature(aa64_lor, s)) {
2662 break;
2663 }
2664 /* StoreLORelease is the same as Store-Release for QEMU. */
2665 /* fall through */
2666 case 0x9: /* STLR */
2667 /* Generate ISS for non-exclusive accesses including LASR. */
2668 if (rn == 31) {
2669 gen_check_sp_alignment(s);
2670 }
2671 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2672 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn),
2673 true, rn != 31, size);
2674 do_gpr_st(s, cpu_reg(s, rt), clean_addr, size, true, rt,
2675 disas_ldst_compute_iss_sf(size, false, 0), is_lasr);
2676 return;
2677
2678 case 0xc: /* LDLAR */
2679 if (!dc_isar_feature(aa64_lor, s)) {
2680 break;
2681 }
2682 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */
2683 /* fall through */
2684 case 0xd: /* LDAR */
2685 /* Generate ISS for non-exclusive accesses including LASR. */
2686 if (rn == 31) {
2687 gen_check_sp_alignment(s);
2688 }
2689 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn),
2690 false, rn != 31, size);
2691 do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, false, false, true, rt,
2692 disas_ldst_compute_iss_sf(size, false, 0), is_lasr);
2693 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2694 return;
2695
2696 case 0x2: case 0x3: /* CASP / STXP */
2697 if (size & 2) { /* STXP / STLXP */
2698 if (rn == 31) {
2699 gen_check_sp_alignment(s);
2700 }
2701 if (is_lasr) {
2702 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2703 }
2704 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn),
2705 true, rn != 31, size);
2706 gen_store_exclusive(s, rs, rt, rt2, clean_addr, size, true);
2707 return;
2708 }
2709 if (rt2 == 31
2710 && ((rt | rs) & 1) == 0
2711 && dc_isar_feature(aa64_atomics, s)) {
2712 /* CASP / CASPL */
2713 gen_compare_and_swap_pair(s, rs, rt, rn, size | 2);
2714 return;
2715 }
2716 break;
2717
2718 case 0x6: case 0x7: /* CASPA / LDXP */
2719 if (size & 2) { /* LDXP / LDAXP */
2720 if (rn == 31) {
2721 gen_check_sp_alignment(s);
2722 }
2723 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn),
2724 false, rn != 31, size);
2725 s->is_ldex = true;
2726 gen_load_exclusive(s, rt, rt2, clean_addr, size, true);
2727 if (is_lasr) {
2728 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2729 }
2730 return;
2731 }
2732 if (rt2 == 31
2733 && ((rt | rs) & 1) == 0
2734 && dc_isar_feature(aa64_atomics, s)) {
2735 /* CASPA / CASPAL */
2736 gen_compare_and_swap_pair(s, rs, rt, rn, size | 2);
2737 return;
2738 }
2739 break;
2740
2741 case 0xa: /* CAS */
2742 case 0xb: /* CASL */
2743 case 0xe: /* CASA */
2744 case 0xf: /* CASAL */
2745 if (rt2 == 31 && dc_isar_feature(aa64_atomics, s)) {
2746 gen_compare_and_swap(s, rs, rt, rn, size);
2747 return;
2748 }
2749 break;
2750 }
2751 unallocated_encoding(s);
2752 }
2753
2754 /*
2755 * Load register (literal)
2756 *
2757 * 31 30 29 27 26 25 24 23 5 4 0
2758 * +-----+-------+---+-----+-------------------+-------+
2759 * | opc | 0 1 1 | V | 0 0 | imm19 | Rt |
2760 * +-----+-------+---+-----+-------------------+-------+
2761 *
2762 * V: 1 -> vector (simd/fp)
2763 * opc (non-vector): 00 -> 32 bit, 01 -> 64 bit,
2764 * 10-> 32 bit signed, 11 -> prefetch
2765 * opc (vector): 00 -> 32 bit, 01 -> 64 bit, 10 -> 128 bit (11 unallocated)
2766 */
2767 static void disas_ld_lit(DisasContext *s, uint32_t insn)
2768 {
2769 int rt = extract32(insn, 0, 5);
2770 int64_t imm = sextract32(insn, 5, 19) << 2;
2771 bool is_vector = extract32(insn, 26, 1);
2772 int opc = extract32(insn, 30, 2);
2773 bool is_signed = false;
2774 int size = 2;
2775 TCGv_i64 tcg_rt, clean_addr;
2776
2777 if (is_vector) {
2778 if (opc == 3) {
2779 unallocated_encoding(s);
2780 return;
2781 }
2782 size = 2 + opc;
2783 if (!fp_access_check(s)) {
2784 return;
2785 }
2786 } else {
2787 if (opc == 3) {
2788 /* PRFM (literal) : prefetch */
2789 return;
2790 }
2791 size = 2 + extract32(opc, 0, 1);
2792 is_signed = extract32(opc, 1, 1);
2793 }
2794
2795 tcg_rt = cpu_reg(s, rt);
2796
2797 clean_addr = tcg_const_i64(s->pc_curr + imm);
2798 if (is_vector) {
2799 do_fp_ld(s, rt, clean_addr, size);
2800 } else {
2801 /* Only unsigned 32bit loads target 32bit registers. */
2802 bool iss_sf = opc != 0;
2803
2804 do_gpr_ld(s, tcg_rt, clean_addr, size, is_signed, false,
2805 true, rt, iss_sf, false);
2806 }
2807 tcg_temp_free_i64(clean_addr);
2808 }
2809
2810 /*
2811 * LDNP (Load Pair - non-temporal hint)
2812 * LDP (Load Pair - non vector)
2813 * LDPSW (Load Pair Signed Word - non vector)
2814 * STNP (Store Pair - non-temporal hint)
2815 * STP (Store Pair - non vector)
2816 * LDNP (Load Pair of SIMD&FP - non-temporal hint)
2817 * LDP (Load Pair of SIMD&FP)
2818 * STNP (Store Pair of SIMD&FP - non-temporal hint)
2819 * STP (Store Pair of SIMD&FP)
2820 *
2821 * 31 30 29 27 26 25 24 23 22 21 15 14 10 9 5 4 0
2822 * +-----+-------+---+---+-------+---+-----------------------------+
2823 * | opc | 1 0 1 | V | 0 | index | L | imm7 | Rt2 | Rn | Rt |
2824 * +-----+-------+---+---+-------+---+-------+-------+------+------+
2825 *
2826 * opc: LDP/STP/LDNP/STNP 00 -> 32 bit, 10 -> 64 bit
2827 * LDPSW/STGP 01
2828 * LDP/STP/LDNP/STNP (SIMD) 00 -> 32 bit, 01 -> 64 bit, 10 -> 128 bit
2829 * V: 0 -> GPR, 1 -> Vector
2830 * idx: 00 -> signed offset with non-temporal hint, 01 -> post-index,
2831 * 10 -> signed offset, 11 -> pre-index
2832 * L: 0 -> Store 1 -> Load
2833 *
2834 * Rt, Rt2 = GPR or SIMD registers to be stored
2835 * Rn = general purpose register containing address
2836 * imm7 = signed offset (multiple of 4 or 8 depending on size)
2837 */
2838 static void disas_ldst_pair(DisasContext *s, uint32_t insn)
2839 {
2840 int rt = extract32(insn, 0, 5);
2841 int rn = extract32(insn, 5, 5);
2842 int rt2 = extract32(insn, 10, 5);
2843 uint64_t offset = sextract64(insn, 15, 7);
2844 int index = extract32(insn, 23, 2);
2845 bool is_vector = extract32(insn, 26, 1);
2846 bool is_load = extract32(insn, 22, 1);
2847 int opc = extract32(insn, 30, 2);
2848
2849 bool is_signed = false;
2850 bool postindex = false;
2851 bool wback = false;
2852 bool set_tag = false;
2853
2854 TCGv_i64 clean_addr, dirty_addr;
2855
2856 int size;
2857
2858 if (opc == 3) {
2859 unallocated_encoding(s);
2860 return;
2861 }
2862
2863 if (is_vector) {
2864 size = 2 + opc;
2865 } else if (opc == 1 && !is_load) {
2866 /* STGP */
2867 if (!dc_isar_feature(aa64_mte_insn_reg, s) || index == 0) {
2868 unallocated_encoding(s);
2869 return;
2870 }
2871 size = 3;
2872 set_tag = true;
2873 } else {
2874 size = 2 + extract32(opc, 1, 1);
2875 is_signed = extract32(opc, 0, 1);
2876 if (!is_load && is_signed) {
2877 unallocated_encoding(s);
2878 return;
2879 }
2880 }
2881
2882 switch (index) {
2883 case 1: /* post-index */
2884 postindex = true;
2885 wback = true;
2886 break;
2887 case 0:
2888 /* signed offset with "non-temporal" hint. Since we don't emulate
2889 * caches we don't care about hints to the cache system about
2890 * data access patterns, and handle this identically to plain
2891 * signed offset.
2892 */
2893 if (is_signed) {
2894 /* There is no non-temporal-hint version of LDPSW */
2895 unallocated_encoding(s);
2896 return;
2897 }
2898 postindex = false;
2899 break;
2900 case 2: /* signed offset, rn not updated */
2901 postindex = false;
2902 break;
2903 case 3: /* pre-index */
2904 postindex = false;
2905 wback = true;
2906 break;
2907 }
2908
2909 if (is_vector && !fp_access_check(s)) {
2910 return;
2911 }
2912
2913 offset <<= (set_tag ? LOG2_TAG_GRANULE : size);
2914
2915 if (rn == 31) {
2916 gen_check_sp_alignment(s);
2917 }
2918
2919 dirty_addr = read_cpu_reg_sp(s, rn, 1);
2920 if (!postindex) {
2921 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
2922 }
2923
2924 if (set_tag) {
2925 if (!s->ata) {
2926 /*
2927 * TODO: We could rely on the stores below, at least for
2928 * system mode, if we arrange to add MO_ALIGN_16.
2929 */
2930 gen_helper_stg_stub(cpu_env, dirty_addr);
2931 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
2932 gen_helper_stg_parallel(cpu_env, dirty_addr, dirty_addr);
2933 } else {
2934 gen_helper_stg(cpu_env, dirty_addr, dirty_addr);
2935 }
2936 }
2937
2938 clean_addr = gen_mte_checkN(s, dirty_addr, !is_load,
2939 (wback || rn != 31) && !set_tag,
2940 size, 2 << size);
2941
2942 if (is_vector) {
2943 if (is_load) {
2944 do_fp_ld(s, rt, clean_addr, size);
2945 } else {
2946 do_fp_st(s, rt, clean_addr, size);
2947 }
2948 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << size);
2949 if (is_load) {
2950 do_fp_ld(s, rt2, clean_addr, size);
2951 } else {
2952 do_fp_st(s, rt2, clean_addr, size);
2953 }
2954 } else {
2955 TCGv_i64 tcg_rt = cpu_reg(s, rt);
2956 TCGv_i64 tcg_rt2 = cpu_reg(s, rt2);
2957
2958 if (is_load) {
2959 TCGv_i64 tmp = tcg_temp_new_i64();
2960
2961 /* Do not modify tcg_rt before recognizing any exception
2962 * from the second load.
2963 */
2964 do_gpr_ld(s, tmp, clean_addr, size, is_signed, false,
2965 false, 0, false, false);
2966 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << size);
2967 do_gpr_ld(s, tcg_rt2, clean_addr, size, is_signed, false,
2968 false, 0, false, false);
2969
2970 tcg_gen_mov_i64(tcg_rt, tmp);
2971 tcg_temp_free_i64(tmp);
2972 } else {
2973 do_gpr_st(s, tcg_rt, clean_addr, size,
2974 false, 0, false, false);
2975 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << size);
2976 do_gpr_st(s, tcg_rt2, clean_addr, size,
2977 false, 0, false, false);
2978 }
2979 }
2980
2981 if (wback) {
2982 if (postindex) {
2983 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
2984 }
2985 tcg_gen_mov_i64(cpu_reg_sp(s, rn), dirty_addr);
2986 }
2987 }
2988
2989 /*
2990 * Load/store (immediate post-indexed)
2991 * Load/store (immediate pre-indexed)
2992 * Load/store (unscaled immediate)
2993 *
2994 * 31 30 29 27 26 25 24 23 22 21 20 12 11 10 9 5 4 0
2995 * +----+-------+---+-----+-----+---+--------+-----+------+------+
2996 * |size| 1 1 1 | V | 0 0 | opc | 0 | imm9 | idx | Rn | Rt |
2997 * +----+-------+---+-----+-----+---+--------+-----+------+------+
2998 *
2999 * idx = 01 -> post-indexed, 11 pre-indexed, 00 unscaled imm. (no writeback)
3000 10 -> unprivileged
3001 * V = 0 -> non-vector
3002 * size: 00 -> 8 bit, 01 -> 16 bit, 10 -> 32 bit, 11 -> 64bit
3003 * opc: 00 -> store, 01 -> loadu, 10 -> loads 64, 11 -> loads 32
3004 */
3005 static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn,
3006 int opc,
3007 int size,
3008 int rt,
3009 bool is_vector)
3010 {
3011 int rn = extract32(insn, 5, 5);
3012 int imm9 = sextract32(insn, 12, 9);
3013 int idx = extract32(insn, 10, 2);
3014 bool is_signed = false;
3015 bool is_store = false;
3016 bool is_extended = false;
3017 bool is_unpriv = (idx == 2);
3018 bool iss_valid = !is_vector;
3019 bool post_index;
3020 bool writeback;
3021 int memidx;
3022
3023 TCGv_i64 clean_addr, dirty_addr;
3024
3025 if (is_vector) {
3026 size |= (opc & 2) << 1;
3027 if (size > 4 || is_unpriv) {
3028 unallocated_encoding(s);
3029 return;
3030 }
3031 is_store = ((opc & 1) == 0);
3032 if (!fp_access_check(s)) {
3033 return;
3034 }
3035 } else {
3036 if (size == 3 && opc == 2) {
3037 /* PRFM - prefetch */
3038 if (idx != 0) {
3039 unallocated_encoding(s);
3040 return;
3041 }
3042 return;
3043 }
3044 if (opc == 3 && size > 1) {
3045 unallocated_encoding(s);
3046 return;
3047 }
3048 is_store = (opc == 0);
3049 is_signed = extract32(opc, 1, 1);
3050 is_extended = (size < 3) && extract32(opc, 0, 1);
3051 }
3052
3053 switch (idx) {
3054 case 0:
3055 case 2:
3056 post_index = false;
3057 writeback = false;
3058 break;
3059 case 1:
3060 post_index = true;
3061 writeback = true;
3062 break;
3063 case 3:
3064 post_index = false;
3065 writeback = true;
3066 break;
3067 default:
3068 g_assert_not_reached();
3069 }
3070
3071 if (rn == 31) {
3072 gen_check_sp_alignment(s);
3073 }
3074
3075 dirty_addr = read_cpu_reg_sp(s, rn, 1);
3076 if (!post_index) {
3077 tcg_gen_addi_i64(dirty_addr, dirty_addr, imm9);
3078 }
3079
3080 memidx = is_unpriv ? get_a64_user_mem_index(s) : get_mem_index(s);
3081 clean_addr = gen_mte_check1_mmuidx(s, dirty_addr, is_store,
3082 writeback || rn != 31,
3083 size, is_unpriv, memidx);
3084
3085 if (is_vector) {
3086 if (is_store) {
3087 do_fp_st(s, rt, clean_addr, size);
3088 } else {
3089 do_fp_ld(s, rt, clean_addr, size);
3090 }
3091 } else {
3092 TCGv_i64 tcg_rt = cpu_reg(s, rt);
3093 bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc);
3094
3095 if (is_store) {
3096 do_gpr_st_memidx(s, tcg_rt, clean_addr, size, memidx,
3097 iss_valid, rt, iss_sf, false);
3098 } else {
3099 do_gpr_ld_memidx(s, tcg_rt, clean_addr, size,
3100 is_signed, is_extended, memidx,
3101 iss_valid, rt, iss_sf, false);
3102 }
3103 }
3104
3105 if (writeback) {
3106 TCGv_i64 tcg_rn = cpu_reg_sp(s, rn);
3107 if (post_index) {
3108 tcg_gen_addi_i64(dirty_addr, dirty_addr, imm9);
3109 }
3110 tcg_gen_mov_i64(tcg_rn, dirty_addr);
3111 }
3112 }
3113
3114 /*
3115 * Load/store (register offset)
3116 *
3117 * 31 30 29 27 26 25 24 23 22 21 20 16 15 13 12 11 10 9 5 4 0
3118 * +----+-------+---+-----+-----+---+------+-----+--+-----+----+----+
3119 * |size| 1 1 1 | V | 0 0 | opc | 1 | Rm | opt | S| 1 0 | Rn | Rt |
3120 * +----+-------+---+-----+-----+---+------+-----+--+-----+----+----+
3121 *
3122 * For non-vector:
3123 * size: 00-> byte, 01 -> 16 bit, 10 -> 32bit, 11 -> 64bit
3124 * opc: 00 -> store, 01 -> loadu, 10 -> loads 64, 11 -> loads 32
3125 * For vector:
3126 * size is opc<1>:size<1:0> so 100 -> 128 bit; 110 and 111 unallocated
3127 * opc<0>: 0 -> store, 1 -> load
3128 * V: 1 -> vector/simd
3129 * opt: extend encoding (see DecodeRegExtend)
3130 * S: if S=1 then scale (essentially index by sizeof(size))
3131 * Rt: register to transfer into/out of
3132 * Rn: address register or SP for base
3133 * Rm: offset register or ZR for offset
3134 */
3135 static void disas_ldst_reg_roffset(DisasContext *s, uint32_t insn,
3136 int opc,
3137 int size,
3138 int rt,
3139 bool is_vector)
3140 {
3141 int rn = extract32(insn, 5, 5);
3142 int shift = extract32(insn, 12, 1);
3143 int rm = extract32(insn, 16, 5);
3144 int opt = extract32(insn, 13, 3);
3145 bool is_signed = false;
3146 bool is_store = false;
3147 bool is_extended = false;
3148
3149 TCGv_i64 tcg_rm, clean_addr, dirty_addr;
3150
3151 if (extract32(opt, 1, 1) == 0) {
3152 unallocated_encoding(s);
3153 return;
3154 }
3155
3156 if (is_vector) {
3157 size |= (opc & 2) << 1;
3158 if (size > 4) {
3159 unallocated_encoding(s);
3160 return;
3161 }
3162 is_store = !extract32(opc, 0, 1);
3163 if (!fp_access_check(s)) {
3164 return;
3165 }
3166 } else {
3167 if (size == 3 && opc == 2) {
3168 /* PRFM - prefetch */
3169 return;
3170 }
3171 if (opc == 3 && size > 1) {
3172 unallocated_encoding(s);
3173 return;
3174 }
3175 is_store = (opc == 0);
3176 is_signed = extract32(opc, 1, 1);
3177 is_extended = (size < 3) && extract32(opc, 0, 1);
3178 }
3179
3180 if (rn == 31) {
3181 gen_check_sp_alignment(s);
3182 }
3183 dirty_addr = read_cpu_reg_sp(s, rn, 1);
3184
3185 tcg_rm = read_cpu_reg(s, rm, 1);
3186 ext_and_shift_reg(tcg_rm, tcg_rm, opt, shift ? size : 0);
3187
3188 tcg_gen_add_i64(dirty_addr, dirty_addr, tcg_rm);
3189 clean_addr = gen_mte_check1(s, dirty_addr, is_store, true, size);
3190
3191 if (is_vector) {
3192 if (is_store) {
3193 do_fp_st(s, rt, clean_addr, size);
3194 } else {
3195 do_fp_ld(s, rt, clean_addr, size);
3196 }
3197 } else {
3198 TCGv_i64 tcg_rt = cpu_reg(s, rt);
3199 bool iss_sf =<