target/arm: Move computation of index in handle_simd_dupe
[qemu.git] / target / arm / translate-a64.c
1 /*
2 * AArch64 translation
3 *
4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "qemu/osdep.h"
20
21 #include "cpu.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "qemu/log.h"
26 #include "arm_ldst.h"
27 #include "translate.h"
28 #include "internals.h"
29 #include "qemu/host-utils.h"
30
31 #include "hw/semihosting/semihost.h"
32 #include "exec/gen-icount.h"
33
34 #include "exec/helper-proto.h"
35 #include "exec/helper-gen.h"
36 #include "exec/log.h"
37
38 #include "trace-tcg.h"
39 #include "translate-a64.h"
40 #include "qemu/atomic128.h"
41
42 static TCGv_i64 cpu_X[32];
43 static TCGv_i64 cpu_pc;
44
45 /* Load/store exclusive handling */
46 static TCGv_i64 cpu_exclusive_high;
47
48 static const char *regnames[] = {
49 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
50 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
51 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
52 "x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp"
53 };
54
55 enum a64_shift_type {
56 A64_SHIFT_TYPE_LSL = 0,
57 A64_SHIFT_TYPE_LSR = 1,
58 A64_SHIFT_TYPE_ASR = 2,
59 A64_SHIFT_TYPE_ROR = 3
60 };
61
62 /* Table based decoder typedefs - used when the relevant bits for decode
63 * are too awkwardly scattered across the instruction (eg SIMD).
64 */
65 typedef void AArch64DecodeFn(DisasContext *s, uint32_t insn);
66
67 typedef struct AArch64DecodeTable {
68 uint32_t pattern;
69 uint32_t mask;
70 AArch64DecodeFn *disas_fn;
71 } AArch64DecodeTable;
72
73 /* Function prototype for gen_ functions for calling Neon helpers */
74 typedef void NeonGenOneOpEnvFn(TCGv_i32, TCGv_ptr, TCGv_i32);
75 typedef void NeonGenTwoOpFn(TCGv_i32, TCGv_i32, TCGv_i32);
76 typedef void NeonGenTwoOpEnvFn(TCGv_i32, TCGv_ptr, TCGv_i32, TCGv_i32);
77 typedef void NeonGenTwo64OpFn(TCGv_i64, TCGv_i64, TCGv_i64);
78 typedef void NeonGenTwo64OpEnvFn(TCGv_i64, TCGv_ptr, TCGv_i64, TCGv_i64);
79 typedef void NeonGenNarrowFn(TCGv_i32, TCGv_i64);
80 typedef void NeonGenNarrowEnvFn(TCGv_i32, TCGv_ptr, TCGv_i64);
81 typedef void NeonGenWidenFn(TCGv_i64, TCGv_i32);
82 typedef void NeonGenTwoSingleOPFn(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_ptr);
83 typedef void NeonGenTwoDoubleOPFn(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_ptr);
84 typedef void NeonGenOneOpFn(TCGv_i64, TCGv_i64);
85 typedef void CryptoTwoOpFn(TCGv_ptr, TCGv_ptr);
86 typedef void CryptoThreeOpIntFn(TCGv_ptr, TCGv_ptr, TCGv_i32);
87 typedef void CryptoThreeOpFn(TCGv_ptr, TCGv_ptr, TCGv_ptr);
88 typedef void AtomicThreeOpFn(TCGv_i64, TCGv_i64, TCGv_i64, TCGArg, MemOp);
89
90 /* initialize TCG globals. */
91 void a64_translate_init(void)
92 {
93 int i;
94
95 cpu_pc = tcg_global_mem_new_i64(cpu_env,
96 offsetof(CPUARMState, pc),
97 "pc");
98 for (i = 0; i < 32; i++) {
99 cpu_X[i] = tcg_global_mem_new_i64(cpu_env,
100 offsetof(CPUARMState, xregs[i]),
101 regnames[i]);
102 }
103
104 cpu_exclusive_high = tcg_global_mem_new_i64(cpu_env,
105 offsetof(CPUARMState, exclusive_high), "exclusive_high");
106 }
107
108 /*
109 * Return the core mmu_idx to use for A64 "unprivileged load/store" insns
110 */
111 static int get_a64_user_mem_index(DisasContext *s)
112 {
113 /*
114 * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL,
115 * which is the usual mmu_idx for this cpu state.
116 */
117 ARMMMUIdx useridx = s->mmu_idx;
118
119 if (s->unpriv) {
120 /*
121 * We have pre-computed the condition for AccType_UNPRIV.
122 * Therefore we should never get here with a mmu_idx for
123 * which we do not know the corresponding user mmu_idx.
124 */
125 switch (useridx) {
126 case ARMMMUIdx_E10_1:
127 case ARMMMUIdx_E10_1_PAN:
128 useridx = ARMMMUIdx_E10_0;
129 break;
130 case ARMMMUIdx_E20_2:
131 case ARMMMUIdx_E20_2_PAN:
132 useridx = ARMMMUIdx_E20_0;
133 break;
134 case ARMMMUIdx_SE10_1:
135 case ARMMMUIdx_SE10_1_PAN:
136 useridx = ARMMMUIdx_SE10_0;
137 break;
138 default:
139 g_assert_not_reached();
140 }
141 }
142 return arm_to_core_mmu_idx(useridx);
143 }
144
145 static void reset_btype(DisasContext *s)
146 {
147 if (s->btype != 0) {
148 TCGv_i32 zero = tcg_const_i32(0);
149 tcg_gen_st_i32(zero, cpu_env, offsetof(CPUARMState, btype));
150 tcg_temp_free_i32(zero);
151 s->btype = 0;
152 }
153 }
154
155 static void set_btype(DisasContext *s, int val)
156 {
157 TCGv_i32 tcg_val;
158
159 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */
160 tcg_debug_assert(val >= 1 && val <= 3);
161
162 tcg_val = tcg_const_i32(val);
163 tcg_gen_st_i32(tcg_val, cpu_env, offsetof(CPUARMState, btype));
164 tcg_temp_free_i32(tcg_val);
165 s->btype = -1;
166 }
167
168 void gen_a64_set_pc_im(uint64_t val)
169 {
170 tcg_gen_movi_i64(cpu_pc, val);
171 }
172
173 /*
174 * Handle Top Byte Ignore (TBI) bits.
175 *
176 * If address tagging is enabled via the TCR TBI bits:
177 * + for EL2 and EL3 there is only one TBI bit, and if it is set
178 * then the address is zero-extended, clearing bits [63:56]
179 * + for EL0 and EL1, TBI0 controls addresses with bit 55 == 0
180 * and TBI1 controls addressses with bit 55 == 1.
181 * If the appropriate TBI bit is set for the address then
182 * the address is sign-extended from bit 55 into bits [63:56]
183 *
184 * Here We have concatenated TBI{1,0} into tbi.
185 */
186 static void gen_top_byte_ignore(DisasContext *s, TCGv_i64 dst,
187 TCGv_i64 src, int tbi)
188 {
189 if (tbi == 0) {
190 /* Load unmodified address */
191 tcg_gen_mov_i64(dst, src);
192 } else if (!regime_has_2_ranges(s->mmu_idx)) {
193 /* Force tag byte to all zero */
194 tcg_gen_extract_i64(dst, src, 0, 56);
195 } else {
196 /* Sign-extend from bit 55. */
197 tcg_gen_sextract_i64(dst, src, 0, 56);
198
199 if (tbi != 3) {
200 TCGv_i64 tcg_zero = tcg_const_i64(0);
201
202 /*
203 * The two TBI bits differ.
204 * If tbi0, then !tbi1: only use the extension if positive.
205 * if !tbi0, then tbi1: only use the extension if negative.
206 */
207 tcg_gen_movcond_i64(tbi == 1 ? TCG_COND_GE : TCG_COND_LT,
208 dst, dst, tcg_zero, dst, src);
209 tcg_temp_free_i64(tcg_zero);
210 }
211 }
212 }
213
214 static void gen_a64_set_pc(DisasContext *s, TCGv_i64 src)
215 {
216 /*
217 * If address tagging is enabled for instructions via the TCR TBI bits,
218 * then loading an address into the PC will clear out any tag.
219 */
220 gen_top_byte_ignore(s, cpu_pc, src, s->tbii);
221 }
222
223 /*
224 * Return a "clean" address for ADDR according to TBID.
225 * This is always a fresh temporary, as we need to be able to
226 * increment this independently of a dirty write-back address.
227 */
228 static TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr)
229 {
230 TCGv_i64 clean = new_tmp_a64(s);
231 /*
232 * In order to get the correct value in the FAR_ELx register,
233 * we must present the memory subsystem with the "dirty" address
234 * including the TBI. In system mode we can make this work via
235 * the TLB, dropping the TBI during translation. But for user-only
236 * mode we don't have that option, and must remove the top byte now.
237 */
238 #ifdef CONFIG_USER_ONLY
239 gen_top_byte_ignore(s, clean, addr, s->tbid);
240 #else
241 tcg_gen_mov_i64(clean, addr);
242 #endif
243 return clean;
244 }
245
246 typedef struct DisasCompare64 {
247 TCGCond cond;
248 TCGv_i64 value;
249 } DisasCompare64;
250
251 static void a64_test_cc(DisasCompare64 *c64, int cc)
252 {
253 DisasCompare c32;
254
255 arm_test_cc(&c32, cc);
256
257 /* Sign-extend the 32-bit value so that the GE/LT comparisons work
258 * properly. The NE/EQ comparisons are also fine with this choice. */
259 c64->cond = c32.cond;
260 c64->value = tcg_temp_new_i64();
261 tcg_gen_ext_i32_i64(c64->value, c32.value);
262
263 arm_free_cc(&c32);
264 }
265
266 static void a64_free_cc(DisasCompare64 *c64)
267 {
268 tcg_temp_free_i64(c64->value);
269 }
270
271 static void gen_exception_internal(int excp)
272 {
273 TCGv_i32 tcg_excp = tcg_const_i32(excp);
274
275 assert(excp_is_internal(excp));
276 gen_helper_exception_internal(cpu_env, tcg_excp);
277 tcg_temp_free_i32(tcg_excp);
278 }
279
280 static void gen_exception_internal_insn(DisasContext *s, uint64_t pc, int excp)
281 {
282 gen_a64_set_pc_im(pc);
283 gen_exception_internal(excp);
284 s->base.is_jmp = DISAS_NORETURN;
285 }
286
287 static void gen_exception_insn(DisasContext *s, uint64_t pc, int excp,
288 uint32_t syndrome, uint32_t target_el)
289 {
290 gen_a64_set_pc_im(pc);
291 gen_exception(excp, syndrome, target_el);
292 s->base.is_jmp = DISAS_NORETURN;
293 }
294
295 static void gen_exception_bkpt_insn(DisasContext *s, uint32_t syndrome)
296 {
297 TCGv_i32 tcg_syn;
298
299 gen_a64_set_pc_im(s->pc_curr);
300 tcg_syn = tcg_const_i32(syndrome);
301 gen_helper_exception_bkpt_insn(cpu_env, tcg_syn);
302 tcg_temp_free_i32(tcg_syn);
303 s->base.is_jmp = DISAS_NORETURN;
304 }
305
306 static void gen_step_complete_exception(DisasContext *s)
307 {
308 /* We just completed step of an insn. Move from Active-not-pending
309 * to Active-pending, and then also take the swstep exception.
310 * This corresponds to making the (IMPDEF) choice to prioritize
311 * swstep exceptions over asynchronous exceptions taken to an exception
312 * level where debug is disabled. This choice has the advantage that
313 * we do not need to maintain internal state corresponding to the
314 * ISV/EX syndrome bits between completion of the step and generation
315 * of the exception, and our syndrome information is always correct.
316 */
317 gen_ss_advance(s);
318 gen_swstep_exception(s, 1, s->is_ldex);
319 s->base.is_jmp = DISAS_NORETURN;
320 }
321
322 static inline bool use_goto_tb(DisasContext *s, int n, uint64_t dest)
323 {
324 /* No direct tb linking with singlestep (either QEMU's or the ARM
325 * debug architecture kind) or deterministic io
326 */
327 if (s->base.singlestep_enabled || s->ss_active ||
328 (tb_cflags(s->base.tb) & CF_LAST_IO)) {
329 return false;
330 }
331
332 #ifndef CONFIG_USER_ONLY
333 /* Only link tbs from inside the same guest page */
334 if ((s->base.tb->pc & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) {
335 return false;
336 }
337 #endif
338
339 return true;
340 }
341
342 static inline void gen_goto_tb(DisasContext *s, int n, uint64_t dest)
343 {
344 TranslationBlock *tb;
345
346 tb = s->base.tb;
347 if (use_goto_tb(s, n, dest)) {
348 tcg_gen_goto_tb(n);
349 gen_a64_set_pc_im(dest);
350 tcg_gen_exit_tb(tb, n);
351 s->base.is_jmp = DISAS_NORETURN;
352 } else {
353 gen_a64_set_pc_im(dest);
354 if (s->ss_active) {
355 gen_step_complete_exception(s);
356 } else if (s->base.singlestep_enabled) {
357 gen_exception_internal(EXCP_DEBUG);
358 } else {
359 tcg_gen_lookup_and_goto_ptr();
360 s->base.is_jmp = DISAS_NORETURN;
361 }
362 }
363 }
364
365 void unallocated_encoding(DisasContext *s)
366 {
367 /* Unallocated and reserved encodings are uncategorized */
368 gen_exception_insn(s, s->pc_curr, EXCP_UDEF, syn_uncategorized(),
369 default_exception_el(s));
370 }
371
372 static void init_tmp_a64_array(DisasContext *s)
373 {
374 #ifdef CONFIG_DEBUG_TCG
375 memset(s->tmp_a64, 0, sizeof(s->tmp_a64));
376 #endif
377 s->tmp_a64_count = 0;
378 }
379
380 static void free_tmp_a64(DisasContext *s)
381 {
382 int i;
383 for (i = 0; i < s->tmp_a64_count; i++) {
384 tcg_temp_free_i64(s->tmp_a64[i]);
385 }
386 init_tmp_a64_array(s);
387 }
388
389 TCGv_i64 new_tmp_a64(DisasContext *s)
390 {
391 assert(s->tmp_a64_count < TMP_A64_MAX);
392 return s->tmp_a64[s->tmp_a64_count++] = tcg_temp_new_i64();
393 }
394
395 TCGv_i64 new_tmp_a64_zero(DisasContext *s)
396 {
397 TCGv_i64 t = new_tmp_a64(s);
398 tcg_gen_movi_i64(t, 0);
399 return t;
400 }
401
402 /*
403 * Register access functions
404 *
405 * These functions are used for directly accessing a register in where
406 * changes to the final register value are likely to be made. If you
407 * need to use a register for temporary calculation (e.g. index type
408 * operations) use the read_* form.
409 *
410 * B1.2.1 Register mappings
411 *
412 * In instruction register encoding 31 can refer to ZR (zero register) or
413 * the SP (stack pointer) depending on context. In QEMU's case we map SP
414 * to cpu_X[31] and ZR accesses to a temporary which can be discarded.
415 * This is the point of the _sp forms.
416 */
417 TCGv_i64 cpu_reg(DisasContext *s, int reg)
418 {
419 if (reg == 31) {
420 return new_tmp_a64_zero(s);
421 } else {
422 return cpu_X[reg];
423 }
424 }
425
426 /* register access for when 31 == SP */
427 TCGv_i64 cpu_reg_sp(DisasContext *s, int reg)
428 {
429 return cpu_X[reg];
430 }
431
432 /* read a cpu register in 32bit/64bit mode. Returns a TCGv_i64
433 * representing the register contents. This TCGv is an auto-freed
434 * temporary so it need not be explicitly freed, and may be modified.
435 */
436 TCGv_i64 read_cpu_reg(DisasContext *s, int reg, int sf)
437 {
438 TCGv_i64 v = new_tmp_a64(s);
439 if (reg != 31) {
440 if (sf) {
441 tcg_gen_mov_i64(v, cpu_X[reg]);
442 } else {
443 tcg_gen_ext32u_i64(v, cpu_X[reg]);
444 }
445 } else {
446 tcg_gen_movi_i64(v, 0);
447 }
448 return v;
449 }
450
451 TCGv_i64 read_cpu_reg_sp(DisasContext *s, int reg, int sf)
452 {
453 TCGv_i64 v = new_tmp_a64(s);
454 if (sf) {
455 tcg_gen_mov_i64(v, cpu_X[reg]);
456 } else {
457 tcg_gen_ext32u_i64(v, cpu_X[reg]);
458 }
459 return v;
460 }
461
462 /* Return the offset into CPUARMState of a slice (from
463 * the least significant end) of FP register Qn (ie
464 * Dn, Sn, Hn or Bn).
465 * (Note that this is not the same mapping as for A32; see cpu.h)
466 */
467 static inline int fp_reg_offset(DisasContext *s, int regno, MemOp size)
468 {
469 return vec_reg_offset(s, regno, 0, size);
470 }
471
472 /* Offset of the high half of the 128 bit vector Qn */
473 static inline int fp_reg_hi_offset(DisasContext *s, int regno)
474 {
475 return vec_reg_offset(s, regno, 1, MO_64);
476 }
477
478 /* Convenience accessors for reading and writing single and double
479 * FP registers. Writing clears the upper parts of the associated
480 * 128 bit vector register, as required by the architecture.
481 * Note that unlike the GP register accessors, the values returned
482 * by the read functions must be manually freed.
483 */
484 static TCGv_i64 read_fp_dreg(DisasContext *s, int reg)
485 {
486 TCGv_i64 v = tcg_temp_new_i64();
487
488 tcg_gen_ld_i64(v, cpu_env, fp_reg_offset(s, reg, MO_64));
489 return v;
490 }
491
492 static TCGv_i32 read_fp_sreg(DisasContext *s, int reg)
493 {
494 TCGv_i32 v = tcg_temp_new_i32();
495
496 tcg_gen_ld_i32(v, cpu_env, fp_reg_offset(s, reg, MO_32));
497 return v;
498 }
499
500 static TCGv_i32 read_fp_hreg(DisasContext *s, int reg)
501 {
502 TCGv_i32 v = tcg_temp_new_i32();
503
504 tcg_gen_ld16u_i32(v, cpu_env, fp_reg_offset(s, reg, MO_16));
505 return v;
506 }
507
508 /* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
509 * If SVE is not enabled, then there are only 128 bits in the vector.
510 */
511 static void clear_vec_high(DisasContext *s, bool is_q, int rd)
512 {
513 unsigned ofs = fp_reg_offset(s, rd, MO_64);
514 unsigned vsz = vec_full_reg_size(s);
515
516 if (!is_q) {
517 TCGv_i64 tcg_zero = tcg_const_i64(0);
518 tcg_gen_st_i64(tcg_zero, cpu_env, ofs + 8);
519 tcg_temp_free_i64(tcg_zero);
520 }
521 if (vsz > 16) {
522 tcg_gen_gvec_dup8i(ofs + 16, vsz - 16, vsz - 16, 0);
523 }
524 }
525
526 void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v)
527 {
528 unsigned ofs = fp_reg_offset(s, reg, MO_64);
529
530 tcg_gen_st_i64(v, cpu_env, ofs);
531 clear_vec_high(s, false, reg);
532 }
533
534 static void write_fp_sreg(DisasContext *s, int reg, TCGv_i32 v)
535 {
536 TCGv_i64 tmp = tcg_temp_new_i64();
537
538 tcg_gen_extu_i32_i64(tmp, v);
539 write_fp_dreg(s, reg, tmp);
540 tcg_temp_free_i64(tmp);
541 }
542
543 TCGv_ptr get_fpstatus_ptr(bool is_f16)
544 {
545 TCGv_ptr statusptr = tcg_temp_new_ptr();
546 int offset;
547
548 /* In A64 all instructions (both FP and Neon) use the FPCR; there
549 * is no equivalent of the A32 Neon "standard FPSCR value".
550 * However half-precision operations operate under a different
551 * FZ16 flag and use vfp.fp_status_f16 instead of vfp.fp_status.
552 */
553 if (is_f16) {
554 offset = offsetof(CPUARMState, vfp.fp_status_f16);
555 } else {
556 offset = offsetof(CPUARMState, vfp.fp_status);
557 }
558 tcg_gen_addi_ptr(statusptr, cpu_env, offset);
559 return statusptr;
560 }
561
562 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
563 static void gen_gvec_fn2(DisasContext *s, bool is_q, int rd, int rn,
564 GVecGen2Fn *gvec_fn, int vece)
565 {
566 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
567 is_q ? 16 : 8, vec_full_reg_size(s));
568 }
569
570 /* Expand a 2-operand + immediate AdvSIMD vector operation using
571 * an expander function.
572 */
573 static void gen_gvec_fn2i(DisasContext *s, bool is_q, int rd, int rn,
574 int64_t imm, GVecGen2iFn *gvec_fn, int vece)
575 {
576 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
577 imm, is_q ? 16 : 8, vec_full_reg_size(s));
578 }
579
580 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
581 static void gen_gvec_fn3(DisasContext *s, bool is_q, int rd, int rn, int rm,
582 GVecGen3Fn *gvec_fn, int vece)
583 {
584 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
585 vec_full_reg_offset(s, rm), is_q ? 16 : 8, vec_full_reg_size(s));
586 }
587
588 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
589 static void gen_gvec_fn4(DisasContext *s, bool is_q, int rd, int rn, int rm,
590 int rx, GVecGen4Fn *gvec_fn, int vece)
591 {
592 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
593 vec_full_reg_offset(s, rm), vec_full_reg_offset(s, rx),
594 is_q ? 16 : 8, vec_full_reg_size(s));
595 }
596
597 /* Expand a 2-operand + immediate AdvSIMD vector operation using
598 * an op descriptor.
599 */
600 static void gen_gvec_op2i(DisasContext *s, bool is_q, int rd,
601 int rn, int64_t imm, const GVecGen2i *gvec_op)
602 {
603 tcg_gen_gvec_2i(vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
604 is_q ? 16 : 8, vec_full_reg_size(s), imm, gvec_op);
605 }
606
607 /* Expand a 3-operand AdvSIMD vector operation using an op descriptor. */
608 static void gen_gvec_op3(DisasContext *s, bool is_q, int rd,
609 int rn, int rm, const GVecGen3 *gvec_op)
610 {
611 tcg_gen_gvec_3(vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
612 vec_full_reg_offset(s, rm), is_q ? 16 : 8,
613 vec_full_reg_size(s), gvec_op);
614 }
615
616 /* Expand a 3-operand operation using an out-of-line helper. */
617 static void gen_gvec_op3_ool(DisasContext *s, bool is_q, int rd,
618 int rn, int rm, int data, gen_helper_gvec_3 *fn)
619 {
620 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
621 vec_full_reg_offset(s, rn),
622 vec_full_reg_offset(s, rm),
623 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
624 }
625
626 /* Expand a 3-operand + env pointer operation using
627 * an out-of-line helper.
628 */
629 static void gen_gvec_op3_env(DisasContext *s, bool is_q, int rd,
630 int rn, int rm, gen_helper_gvec_3_ptr *fn)
631 {
632 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
633 vec_full_reg_offset(s, rn),
634 vec_full_reg_offset(s, rm), cpu_env,
635 is_q ? 16 : 8, vec_full_reg_size(s), 0, fn);
636 }
637
638 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
639 * an out-of-line helper.
640 */
641 static void gen_gvec_op3_fpst(DisasContext *s, bool is_q, int rd, int rn,
642 int rm, bool is_fp16, int data,
643 gen_helper_gvec_3_ptr *fn)
644 {
645 TCGv_ptr fpst = get_fpstatus_ptr(is_fp16);
646 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
647 vec_full_reg_offset(s, rn),
648 vec_full_reg_offset(s, rm), fpst,
649 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
650 tcg_temp_free_ptr(fpst);
651 }
652
653 /* Set ZF and NF based on a 64 bit result. This is alas fiddlier
654 * than the 32 bit equivalent.
655 */
656 static inline void gen_set_NZ64(TCGv_i64 result)
657 {
658 tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result);
659 tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF);
660 }
661
662 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
663 static inline void gen_logic_CC(int sf, TCGv_i64 result)
664 {
665 if (sf) {
666 gen_set_NZ64(result);
667 } else {
668 tcg_gen_extrl_i64_i32(cpu_ZF, result);
669 tcg_gen_mov_i32(cpu_NF, cpu_ZF);
670 }
671 tcg_gen_movi_i32(cpu_CF, 0);
672 tcg_gen_movi_i32(cpu_VF, 0);
673 }
674
675 /* dest = T0 + T1; compute C, N, V and Z flags */
676 static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
677 {
678 if (sf) {
679 TCGv_i64 result, flag, tmp;
680 result = tcg_temp_new_i64();
681 flag = tcg_temp_new_i64();
682 tmp = tcg_temp_new_i64();
683
684 tcg_gen_movi_i64(tmp, 0);
685 tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);
686
687 tcg_gen_extrl_i64_i32(cpu_CF, flag);
688
689 gen_set_NZ64(result);
690
691 tcg_gen_xor_i64(flag, result, t0);
692 tcg_gen_xor_i64(tmp, t0, t1);
693 tcg_gen_andc_i64(flag, flag, tmp);
694 tcg_temp_free_i64(tmp);
695 tcg_gen_extrh_i64_i32(cpu_VF, flag);
696
697 tcg_gen_mov_i64(dest, result);
698 tcg_temp_free_i64(result);
699 tcg_temp_free_i64(flag);
700 } else {
701 /* 32 bit arithmetic */
702 TCGv_i32 t0_32 = tcg_temp_new_i32();
703 TCGv_i32 t1_32 = tcg_temp_new_i32();
704 TCGv_i32 tmp = tcg_temp_new_i32();
705
706 tcg_gen_movi_i32(tmp, 0);
707 tcg_gen_extrl_i64_i32(t0_32, t0);
708 tcg_gen_extrl_i64_i32(t1_32, t1);
709 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);
710 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
711 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
712 tcg_gen_xor_i32(tmp, t0_32, t1_32);
713 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
714 tcg_gen_extu_i32_i64(dest, cpu_NF);
715
716 tcg_temp_free_i32(tmp);
717 tcg_temp_free_i32(t0_32);
718 tcg_temp_free_i32(t1_32);
719 }
720 }
721
722 /* dest = T0 - T1; compute C, N, V and Z flags */
723 static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
724 {
725 if (sf) {
726 /* 64 bit arithmetic */
727 TCGv_i64 result, flag, tmp;
728
729 result = tcg_temp_new_i64();
730 flag = tcg_temp_new_i64();
731 tcg_gen_sub_i64(result, t0, t1);
732
733 gen_set_NZ64(result);
734
735 tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);
736 tcg_gen_extrl_i64_i32(cpu_CF, flag);
737
738 tcg_gen_xor_i64(flag, result, t0);
739 tmp = tcg_temp_new_i64();
740 tcg_gen_xor_i64(tmp, t0, t1);
741 tcg_gen_and_i64(flag, flag, tmp);
742 tcg_temp_free_i64(tmp);
743 tcg_gen_extrh_i64_i32(cpu_VF, flag);
744 tcg_gen_mov_i64(dest, result);
745 tcg_temp_free_i64(flag);
746 tcg_temp_free_i64(result);
747 } else {
748 /* 32 bit arithmetic */
749 TCGv_i32 t0_32 = tcg_temp_new_i32();
750 TCGv_i32 t1_32 = tcg_temp_new_i32();
751 TCGv_i32 tmp;
752
753 tcg_gen_extrl_i64_i32(t0_32, t0);
754 tcg_gen_extrl_i64_i32(t1_32, t1);
755 tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);
756 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
757 tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);
758 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
759 tmp = tcg_temp_new_i32();
760 tcg_gen_xor_i32(tmp, t0_32, t1_32);
761 tcg_temp_free_i32(t0_32);
762 tcg_temp_free_i32(t1_32);
763 tcg_gen_and_i32(cpu_VF, cpu_VF, tmp);
764 tcg_temp_free_i32(tmp);
765 tcg_gen_extu_i32_i64(dest, cpu_NF);
766 }
767 }
768
769 /* dest = T0 + T1 + CF; do not compute flags. */
770 static void gen_adc(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
771 {
772 TCGv_i64 flag = tcg_temp_new_i64();
773 tcg_gen_extu_i32_i64(flag, cpu_CF);
774 tcg_gen_add_i64(dest, t0, t1);
775 tcg_gen_add_i64(dest, dest, flag);
776 tcg_temp_free_i64(flag);
777
778 if (!sf) {
779 tcg_gen_ext32u_i64(dest, dest);
780 }
781 }
782
783 /* dest = T0 + T1 + CF; compute C, N, V and Z flags. */
784 static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
785 {
786 if (sf) {
787 TCGv_i64 result, cf_64, vf_64, tmp;
788 result = tcg_temp_new_i64();
789 cf_64 = tcg_temp_new_i64();
790 vf_64 = tcg_temp_new_i64();
791 tmp = tcg_const_i64(0);
792
793 tcg_gen_extu_i32_i64(cf_64, cpu_CF);
794 tcg_gen_add2_i64(result, cf_64, t0, tmp, cf_64, tmp);
795 tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, tmp);
796 tcg_gen_extrl_i64_i32(cpu_CF, cf_64);
797 gen_set_NZ64(result);
798
799 tcg_gen_xor_i64(vf_64, result, t0);
800 tcg_gen_xor_i64(tmp, t0, t1);
801 tcg_gen_andc_i64(vf_64, vf_64, tmp);
802 tcg_gen_extrh_i64_i32(cpu_VF, vf_64);
803
804 tcg_gen_mov_i64(dest, result);
805
806 tcg_temp_free_i64(tmp);
807 tcg_temp_free_i64(vf_64);
808 tcg_temp_free_i64(cf_64);
809 tcg_temp_free_i64(result);
810 } else {
811 TCGv_i32 t0_32, t1_32, tmp;
812 t0_32 = tcg_temp_new_i32();
813 t1_32 = tcg_temp_new_i32();
814 tmp = tcg_const_i32(0);
815
816 tcg_gen_extrl_i64_i32(t0_32, t0);
817 tcg_gen_extrl_i64_i32(t1_32, t1);
818 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, cpu_CF, tmp);
819 tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, tmp);
820
821 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
822 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
823 tcg_gen_xor_i32(tmp, t0_32, t1_32);
824 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
825 tcg_gen_extu_i32_i64(dest, cpu_NF);
826
827 tcg_temp_free_i32(tmp);
828 tcg_temp_free_i32(t1_32);
829 tcg_temp_free_i32(t0_32);
830 }
831 }
832
833 /*
834 * Load/Store generators
835 */
836
837 /*
838 * Store from GPR register to memory.
839 */
840 static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
841 TCGv_i64 tcg_addr, int size, int memidx,
842 bool iss_valid,
843 unsigned int iss_srt,
844 bool iss_sf, bool iss_ar)
845 {
846 g_assert(size <= 3);
847 tcg_gen_qemu_st_i64(source, tcg_addr, memidx, s->be_data + size);
848
849 if (iss_valid) {
850 uint32_t syn;
851
852 syn = syn_data_abort_with_iss(0,
853 size,
854 false,
855 iss_srt,
856 iss_sf,
857 iss_ar,
858 0, 0, 0, 0, 0, false);
859 disas_set_insn_syndrome(s, syn);
860 }
861 }
862
863 static void do_gpr_st(DisasContext *s, TCGv_i64 source,
864 TCGv_i64 tcg_addr, int size,
865 bool iss_valid,
866 unsigned int iss_srt,
867 bool iss_sf, bool iss_ar)
868 {
869 do_gpr_st_memidx(s, source, tcg_addr, size, get_mem_index(s),
870 iss_valid, iss_srt, iss_sf, iss_ar);
871 }
872
873 /*
874 * Load from memory to GPR register
875 */
876 static void do_gpr_ld_memidx(DisasContext *s,
877 TCGv_i64 dest, TCGv_i64 tcg_addr,
878 int size, bool is_signed,
879 bool extend, int memidx,
880 bool iss_valid, unsigned int iss_srt,
881 bool iss_sf, bool iss_ar)
882 {
883 MemOp memop = s->be_data + size;
884
885 g_assert(size <= 3);
886
887 if (is_signed) {
888 memop += MO_SIGN;
889 }
890
891 tcg_gen_qemu_ld_i64(dest, tcg_addr, memidx, memop);
892
893 if (extend && is_signed) {
894 g_assert(size < 3);
895 tcg_gen_ext32u_i64(dest, dest);
896 }
897
898 if (iss_valid) {
899 uint32_t syn;
900
901 syn = syn_data_abort_with_iss(0,
902 size,
903 is_signed,
904 iss_srt,
905 iss_sf,
906 iss_ar,
907 0, 0, 0, 0, 0, false);
908 disas_set_insn_syndrome(s, syn);
909 }
910 }
911
912 static void do_gpr_ld(DisasContext *s,
913 TCGv_i64 dest, TCGv_i64 tcg_addr,
914 int size, bool is_signed, bool extend,
915 bool iss_valid, unsigned int iss_srt,
916 bool iss_sf, bool iss_ar)
917 {
918 do_gpr_ld_memidx(s, dest, tcg_addr, size, is_signed, extend,
919 get_mem_index(s),
920 iss_valid, iss_srt, iss_sf, iss_ar);
921 }
922
923 /*
924 * Store from FP register to memory
925 */
926 static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, int size)
927 {
928 /* This writes the bottom N bits of a 128 bit wide vector to memory */
929 TCGv_i64 tmp = tcg_temp_new_i64();
930 tcg_gen_ld_i64(tmp, cpu_env, fp_reg_offset(s, srcidx, MO_64));
931 if (size < 4) {
932 tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s),
933 s->be_data + size);
934 } else {
935 bool be = s->be_data == MO_BE;
936 TCGv_i64 tcg_hiaddr = tcg_temp_new_i64();
937
938 tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);
939 tcg_gen_qemu_st_i64(tmp, be ? tcg_hiaddr : tcg_addr, get_mem_index(s),
940 s->be_data | MO_Q);
941 tcg_gen_ld_i64(tmp, cpu_env, fp_reg_hi_offset(s, srcidx));
942 tcg_gen_qemu_st_i64(tmp, be ? tcg_addr : tcg_hiaddr, get_mem_index(s),
943 s->be_data | MO_Q);
944 tcg_temp_free_i64(tcg_hiaddr);
945 }
946
947 tcg_temp_free_i64(tmp);
948 }
949
950 /*
951 * Load from memory to FP register
952 */
953 static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, int size)
954 {
955 /* This always zero-extends and writes to a full 128 bit wide vector */
956 TCGv_i64 tmplo = tcg_temp_new_i64();
957 TCGv_i64 tmphi;
958
959 if (size < 4) {
960 MemOp memop = s->be_data + size;
961 tmphi = tcg_const_i64(0);
962 tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), memop);
963 } else {
964 bool be = s->be_data == MO_BE;
965 TCGv_i64 tcg_hiaddr;
966
967 tmphi = tcg_temp_new_i64();
968 tcg_hiaddr = tcg_temp_new_i64();
969
970 tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);
971 tcg_gen_qemu_ld_i64(tmplo, be ? tcg_hiaddr : tcg_addr, get_mem_index(s),
972 s->be_data | MO_Q);
973 tcg_gen_qemu_ld_i64(tmphi, be ? tcg_addr : tcg_hiaddr, get_mem_index(s),
974 s->be_data | MO_Q);
975 tcg_temp_free_i64(tcg_hiaddr);
976 }
977
978 tcg_gen_st_i64(tmplo, cpu_env, fp_reg_offset(s, destidx, MO_64));
979 tcg_gen_st_i64(tmphi, cpu_env, fp_reg_hi_offset(s, destidx));
980
981 tcg_temp_free_i64(tmplo);
982 tcg_temp_free_i64(tmphi);
983
984 clear_vec_high(s, true, destidx);
985 }
986
987 /*
988 * Vector load/store helpers.
989 *
990 * The principal difference between this and a FP load is that we don't
991 * zero extend as we are filling a partial chunk of the vector register.
992 * These functions don't support 128 bit loads/stores, which would be
993 * normal load/store operations.
994 *
995 * The _i32 versions are useful when operating on 32 bit quantities
996 * (eg for floating point single or using Neon helper functions).
997 */
998
999 /* Get value of an element within a vector register */
1000 static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx,
1001 int element, MemOp memop)
1002 {
1003 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1004 switch (memop) {
1005 case MO_8:
1006 tcg_gen_ld8u_i64(tcg_dest, cpu_env, vect_off);
1007 break;
1008 case MO_16:
1009 tcg_gen_ld16u_i64(tcg_dest, cpu_env, vect_off);
1010 break;
1011 case MO_32:
1012 tcg_gen_ld32u_i64(tcg_dest, cpu_env, vect_off);
1013 break;
1014 case MO_8|MO_SIGN:
1015 tcg_gen_ld8s_i64(tcg_dest, cpu_env, vect_off);
1016 break;
1017 case MO_16|MO_SIGN:
1018 tcg_gen_ld16s_i64(tcg_dest, cpu_env, vect_off);
1019 break;
1020 case MO_32|MO_SIGN:
1021 tcg_gen_ld32s_i64(tcg_dest, cpu_env, vect_off);
1022 break;
1023 case MO_64:
1024 case MO_64|MO_SIGN:
1025 tcg_gen_ld_i64(tcg_dest, cpu_env, vect_off);
1026 break;
1027 default:
1028 g_assert_not_reached();
1029 }
1030 }
1031
1032 static void read_vec_element_i32(DisasContext *s, TCGv_i32 tcg_dest, int srcidx,
1033 int element, MemOp memop)
1034 {
1035 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1036 switch (memop) {
1037 case MO_8:
1038 tcg_gen_ld8u_i32(tcg_dest, cpu_env, vect_off);
1039 break;
1040 case MO_16:
1041 tcg_gen_ld16u_i32(tcg_dest, cpu_env, vect_off);
1042 break;
1043 case MO_8|MO_SIGN:
1044 tcg_gen_ld8s_i32(tcg_dest, cpu_env, vect_off);
1045 break;
1046 case MO_16|MO_SIGN:
1047 tcg_gen_ld16s_i32(tcg_dest, cpu_env, vect_off);
1048 break;
1049 case MO_32:
1050 case MO_32|MO_SIGN:
1051 tcg_gen_ld_i32(tcg_dest, cpu_env, vect_off);
1052 break;
1053 default:
1054 g_assert_not_reached();
1055 }
1056 }
1057
1058 /* Set value of an element within a vector register */
1059 static void write_vec_element(DisasContext *s, TCGv_i64 tcg_src, int destidx,
1060 int element, MemOp memop)
1061 {
1062 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1063 switch (memop) {
1064 case MO_8:
1065 tcg_gen_st8_i64(tcg_src, cpu_env, vect_off);
1066 break;
1067 case MO_16:
1068 tcg_gen_st16_i64(tcg_src, cpu_env, vect_off);
1069 break;
1070 case MO_32:
1071 tcg_gen_st32_i64(tcg_src, cpu_env, vect_off);
1072 break;
1073 case MO_64:
1074 tcg_gen_st_i64(tcg_src, cpu_env, vect_off);
1075 break;
1076 default:
1077 g_assert_not_reached();
1078 }
1079 }
1080
1081 static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src,
1082 int destidx, int element, MemOp memop)
1083 {
1084 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1085 switch (memop) {
1086 case MO_8:
1087 tcg_gen_st8_i32(tcg_src, cpu_env, vect_off);
1088 break;
1089 case MO_16:
1090 tcg_gen_st16_i32(tcg_src, cpu_env, vect_off);
1091 break;
1092 case MO_32:
1093 tcg_gen_st_i32(tcg_src, cpu_env, vect_off);
1094 break;
1095 default:
1096 g_assert_not_reached();
1097 }
1098 }
1099
1100 /* Store from vector register to memory */
1101 static void do_vec_st(DisasContext *s, int srcidx, int element,
1102 TCGv_i64 tcg_addr, int size, MemOp endian)
1103 {
1104 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1105
1106 read_vec_element(s, tcg_tmp, srcidx, element, size);
1107 tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), endian | size);
1108
1109 tcg_temp_free_i64(tcg_tmp);
1110 }
1111
1112 /* Load from memory to vector register */
1113 static void do_vec_ld(DisasContext *s, int destidx, int element,
1114 TCGv_i64 tcg_addr, int size, MemOp endian)
1115 {
1116 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1117
1118 tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), endian | size);
1119 write_vec_element(s, tcg_tmp, destidx, element, size);
1120
1121 tcg_temp_free_i64(tcg_tmp);
1122 }
1123
1124 /* Check that FP/Neon access is enabled. If it is, return
1125 * true. If not, emit code to generate an appropriate exception,
1126 * and return false; the caller should not emit any code for
1127 * the instruction. Note that this check must happen after all
1128 * unallocated-encoding checks (otherwise the syndrome information
1129 * for the resulting exception will be incorrect).
1130 */
1131 static inline bool fp_access_check(DisasContext *s)
1132 {
1133 assert(!s->fp_access_checked);
1134 s->fp_access_checked = true;
1135
1136 if (!s->fp_excp_el) {
1137 return true;
1138 }
1139
1140 gen_exception_insn(s, s->pc_curr, EXCP_UDEF,
1141 syn_fp_access_trap(1, 0xe, false), s->fp_excp_el);
1142 return false;
1143 }
1144
1145 /* Check that SVE access is enabled. If it is, return true.
1146 * If not, emit code to generate an appropriate exception and return false.
1147 */
1148 bool sve_access_check(DisasContext *s)
1149 {
1150 if (s->sve_excp_el) {
1151 gen_exception_insn(s, s->pc_curr, EXCP_UDEF, syn_sve_access_trap(),
1152 s->sve_excp_el);
1153 return false;
1154 }
1155 return fp_access_check(s);
1156 }
1157
1158 /*
1159 * This utility function is for doing register extension with an
1160 * optional shift. You will likely want to pass a temporary for the
1161 * destination register. See DecodeRegExtend() in the ARM ARM.
1162 */
1163 static void ext_and_shift_reg(TCGv_i64 tcg_out, TCGv_i64 tcg_in,
1164 int option, unsigned int shift)
1165 {
1166 int extsize = extract32(option, 0, 2);
1167 bool is_signed = extract32(option, 2, 1);
1168
1169 if (is_signed) {
1170 switch (extsize) {
1171 case 0:
1172 tcg_gen_ext8s_i64(tcg_out, tcg_in);
1173 break;
1174 case 1:
1175 tcg_gen_ext16s_i64(tcg_out, tcg_in);
1176 break;
1177 case 2:
1178 tcg_gen_ext32s_i64(tcg_out, tcg_in);
1179 break;
1180 case 3:
1181 tcg_gen_mov_i64(tcg_out, tcg_in);
1182 break;
1183 }
1184 } else {
1185 switch (extsize) {
1186 case 0:
1187 tcg_gen_ext8u_i64(tcg_out, tcg_in);
1188 break;
1189 case 1:
1190 tcg_gen_ext16u_i64(tcg_out, tcg_in);
1191 break;
1192 case 2:
1193 tcg_gen_ext32u_i64(tcg_out, tcg_in);
1194 break;
1195 case 3:
1196 tcg_gen_mov_i64(tcg_out, tcg_in);
1197 break;
1198 }
1199 }
1200
1201 if (shift) {
1202 tcg_gen_shli_i64(tcg_out, tcg_out, shift);
1203 }
1204 }
1205
1206 static inline void gen_check_sp_alignment(DisasContext *s)
1207 {
1208 /* The AArch64 architecture mandates that (if enabled via PSTATE
1209 * or SCTLR bits) there is a check that SP is 16-aligned on every
1210 * SP-relative load or store (with an exception generated if it is not).
1211 * In line with general QEMU practice regarding misaligned accesses,
1212 * we omit these checks for the sake of guest program performance.
1213 * This function is provided as a hook so we can more easily add these
1214 * checks in future (possibly as a "favour catching guest program bugs
1215 * over speed" user selectable option).
1216 */
1217 }
1218
1219 /*
1220 * This provides a simple table based table lookup decoder. It is
1221 * intended to be used when the relevant bits for decode are too
1222 * awkwardly placed and switch/if based logic would be confusing and
1223 * deeply nested. Since it's a linear search through the table, tables
1224 * should be kept small.
1225 *
1226 * It returns the first handler where insn & mask == pattern, or
1227 * NULL if there is no match.
1228 * The table is terminated by an empty mask (i.e. 0)
1229 */
1230 static inline AArch64DecodeFn *lookup_disas_fn(const AArch64DecodeTable *table,
1231 uint32_t insn)
1232 {
1233 const AArch64DecodeTable *tptr = table;
1234
1235 while (tptr->mask) {
1236 if ((insn & tptr->mask) == tptr->pattern) {
1237 return tptr->disas_fn;
1238 }
1239 tptr++;
1240 }
1241 return NULL;
1242 }
1243
1244 /*
1245 * The instruction disassembly implemented here matches
1246 * the instruction encoding classifications in chapter C4
1247 * of the ARM Architecture Reference Manual (DDI0487B_a);
1248 * classification names and decode diagrams here should generally
1249 * match up with those in the manual.
1250 */
1251
1252 /* Unconditional branch (immediate)
1253 * 31 30 26 25 0
1254 * +----+-----------+-------------------------------------+
1255 * | op | 0 0 1 0 1 | imm26 |
1256 * +----+-----------+-------------------------------------+
1257 */
1258 static void disas_uncond_b_imm(DisasContext *s, uint32_t insn)
1259 {
1260 uint64_t addr = s->pc_curr + sextract32(insn, 0, 26) * 4;
1261
1262 if (insn & (1U << 31)) {
1263 /* BL Branch with link */
1264 tcg_gen_movi_i64(cpu_reg(s, 30), s->base.pc_next);
1265 }
1266
1267 /* B Branch / BL Branch with link */
1268 reset_btype(s);
1269 gen_goto_tb(s, 0, addr);
1270 }
1271
1272 /* Compare and branch (immediate)
1273 * 31 30 25 24 23 5 4 0
1274 * +----+-------------+----+---------------------+--------+
1275 * | sf | 0 1 1 0 1 0 | op | imm19 | Rt |
1276 * +----+-------------+----+---------------------+--------+
1277 */
1278 static void disas_comp_b_imm(DisasContext *s, uint32_t insn)
1279 {
1280 unsigned int sf, op, rt;
1281 uint64_t addr;
1282 TCGLabel *label_match;
1283 TCGv_i64 tcg_cmp;
1284
1285 sf = extract32(insn, 31, 1);
1286 op = extract32(insn, 24, 1); /* 0: CBZ; 1: CBNZ */
1287 rt = extract32(insn, 0, 5);
1288 addr = s->pc_curr + sextract32(insn, 5, 19) * 4;
1289
1290 tcg_cmp = read_cpu_reg(s, rt, sf);
1291 label_match = gen_new_label();
1292
1293 reset_btype(s);
1294 tcg_gen_brcondi_i64(op ? TCG_COND_NE : TCG_COND_EQ,
1295 tcg_cmp, 0, label_match);
1296
1297 gen_goto_tb(s, 0, s->base.pc_next);
1298 gen_set_label(label_match);
1299 gen_goto_tb(s, 1, addr);
1300 }
1301
1302 /* Test and branch (immediate)
1303 * 31 30 25 24 23 19 18 5 4 0
1304 * +----+-------------+----+-------+-------------+------+
1305 * | b5 | 0 1 1 0 1 1 | op | b40 | imm14 | Rt |
1306 * +----+-------------+----+-------+-------------+------+
1307 */
1308 static void disas_test_b_imm(DisasContext *s, uint32_t insn)
1309 {
1310 unsigned int bit_pos, op, rt;
1311 uint64_t addr;
1312 TCGLabel *label_match;
1313 TCGv_i64 tcg_cmp;
1314
1315 bit_pos = (extract32(insn, 31, 1) << 5) | extract32(insn, 19, 5);
1316 op = extract32(insn, 24, 1); /* 0: TBZ; 1: TBNZ */
1317 addr = s->pc_curr + sextract32(insn, 5, 14) * 4;
1318 rt = extract32(insn, 0, 5);
1319
1320 tcg_cmp = tcg_temp_new_i64();
1321 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, rt), (1ULL << bit_pos));
1322 label_match = gen_new_label();
1323
1324 reset_btype(s);
1325 tcg_gen_brcondi_i64(op ? TCG_COND_NE : TCG_COND_EQ,
1326 tcg_cmp, 0, label_match);
1327 tcg_temp_free_i64(tcg_cmp);
1328 gen_goto_tb(s, 0, s->base.pc_next);
1329 gen_set_label(label_match);
1330 gen_goto_tb(s, 1, addr);
1331 }
1332
1333 /* Conditional branch (immediate)
1334 * 31 25 24 23 5 4 3 0
1335 * +---------------+----+---------------------+----+------+
1336 * | 0 1 0 1 0 1 0 | o1 | imm19 | o0 | cond |
1337 * +---------------+----+---------------------+----+------+
1338 */
1339 static void disas_cond_b_imm(DisasContext *s, uint32_t insn)
1340 {
1341 unsigned int cond;
1342 uint64_t addr;
1343
1344 if ((insn & (1 << 4)) || (insn & (1 << 24))) {
1345 unallocated_encoding(s);
1346 return;
1347 }
1348 addr = s->pc_curr + sextract32(insn, 5, 19) * 4;
1349 cond = extract32(insn, 0, 4);
1350
1351 reset_btype(s);
1352 if (cond < 0x0e) {
1353 /* genuinely conditional branches */
1354 TCGLabel *label_match = gen_new_label();
1355 arm_gen_test_cc(cond, label_match);
1356 gen_goto_tb(s, 0, s->base.pc_next);
1357 gen_set_label(label_match);
1358 gen_goto_tb(s, 1, addr);
1359 } else {
1360 /* 0xe and 0xf are both "always" conditions */
1361 gen_goto_tb(s, 0, addr);
1362 }
1363 }
1364
1365 /* HINT instruction group, including various allocated HINTs */
1366 static void handle_hint(DisasContext *s, uint32_t insn,
1367 unsigned int op1, unsigned int op2, unsigned int crm)
1368 {
1369 unsigned int selector = crm << 3 | op2;
1370
1371 if (op1 != 3) {
1372 unallocated_encoding(s);
1373 return;
1374 }
1375
1376 switch (selector) {
1377 case 0b00000: /* NOP */
1378 break;
1379 case 0b00011: /* WFI */
1380 s->base.is_jmp = DISAS_WFI;
1381 break;
1382 case 0b00001: /* YIELD */
1383 /* When running in MTTCG we don't generate jumps to the yield and
1384 * WFE helpers as it won't affect the scheduling of other vCPUs.
1385 * If we wanted to more completely model WFE/SEV so we don't busy
1386 * spin unnecessarily we would need to do something more involved.
1387 */
1388 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1389 s->base.is_jmp = DISAS_YIELD;
1390 }
1391 break;
1392 case 0b00010: /* WFE */
1393 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1394 s->base.is_jmp = DISAS_WFE;
1395 }
1396 break;
1397 case 0b00100: /* SEV */
1398 case 0b00101: /* SEVL */
1399 /* we treat all as NOP at least for now */
1400 break;
1401 case 0b00111: /* XPACLRI */
1402 if (s->pauth_active) {
1403 gen_helper_xpaci(cpu_X[30], cpu_env, cpu_X[30]);
1404 }
1405 break;
1406 case 0b01000: /* PACIA1716 */
1407 if (s->pauth_active) {
1408 gen_helper_pacia(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]);
1409 }
1410 break;
1411 case 0b01010: /* PACIB1716 */
1412 if (s->pauth_active) {
1413 gen_helper_pacib(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]);
1414 }
1415 break;
1416 case 0b01100: /* AUTIA1716 */
1417 if (s->pauth_active) {
1418 gen_helper_autia(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]);
1419 }
1420 break;
1421 case 0b01110: /* AUTIB1716 */
1422 if (s->pauth_active) {
1423 gen_helper_autib(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]);
1424 }
1425 break;
1426 case 0b11000: /* PACIAZ */
1427 if (s->pauth_active) {
1428 gen_helper_pacia(cpu_X[30], cpu_env, cpu_X[30],
1429 new_tmp_a64_zero(s));
1430 }
1431 break;
1432 case 0b11001: /* PACIASP */
1433 if (s->pauth_active) {
1434 gen_helper_pacia(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]);
1435 }
1436 break;
1437 case 0b11010: /* PACIBZ */
1438 if (s->pauth_active) {
1439 gen_helper_pacib(cpu_X[30], cpu_env, cpu_X[30],
1440 new_tmp_a64_zero(s));
1441 }
1442 break;
1443 case 0b11011: /* PACIBSP */
1444 if (s->pauth_active) {
1445 gen_helper_pacib(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]);
1446 }
1447 break;
1448 case 0b11100: /* AUTIAZ */
1449 if (s->pauth_active) {
1450 gen_helper_autia(cpu_X[30], cpu_env, cpu_X[30],
1451 new_tmp_a64_zero(s));
1452 }
1453 break;
1454 case 0b11101: /* AUTIASP */
1455 if (s->pauth_active) {
1456 gen_helper_autia(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]);
1457 }
1458 break;
1459 case 0b11110: /* AUTIBZ */
1460 if (s->pauth_active) {
1461 gen_helper_autib(cpu_X[30], cpu_env, cpu_X[30],
1462 new_tmp_a64_zero(s));
1463 }
1464 break;
1465 case 0b11111: /* AUTIBSP */
1466 if (s->pauth_active) {
1467 gen_helper_autib(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]);
1468 }
1469 break;
1470 default:
1471 /* default specified as NOP equivalent */
1472 break;
1473 }
1474 }
1475
1476 static void gen_clrex(DisasContext *s, uint32_t insn)
1477 {
1478 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
1479 }
1480
1481 /* CLREX, DSB, DMB, ISB */
1482 static void handle_sync(DisasContext *s, uint32_t insn,
1483 unsigned int op1, unsigned int op2, unsigned int crm)
1484 {
1485 TCGBar bar;
1486
1487 if (op1 != 3) {
1488 unallocated_encoding(s);
1489 return;
1490 }
1491
1492 switch (op2) {
1493 case 2: /* CLREX */
1494 gen_clrex(s, insn);
1495 return;
1496 case 4: /* DSB */
1497 case 5: /* DMB */
1498 switch (crm & 3) {
1499 case 1: /* MBReqTypes_Reads */
1500 bar = TCG_BAR_SC | TCG_MO_LD_LD | TCG_MO_LD_ST;
1501 break;
1502 case 2: /* MBReqTypes_Writes */
1503 bar = TCG_BAR_SC | TCG_MO_ST_ST;
1504 break;
1505 default: /* MBReqTypes_All */
1506 bar = TCG_BAR_SC | TCG_MO_ALL;
1507 break;
1508 }
1509 tcg_gen_mb(bar);
1510 return;
1511 case 6: /* ISB */
1512 /* We need to break the TB after this insn to execute
1513 * a self-modified code correctly and also to take
1514 * any pending interrupts immediately.
1515 */
1516 reset_btype(s);
1517 gen_goto_tb(s, 0, s->base.pc_next);
1518 return;
1519
1520 case 7: /* SB */
1521 if (crm != 0 || !dc_isar_feature(aa64_sb, s)) {
1522 goto do_unallocated;
1523 }
1524 /*
1525 * TODO: There is no speculation barrier opcode for TCG;
1526 * MB and end the TB instead.
1527 */
1528 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC);
1529 gen_goto_tb(s, 0, s->base.pc_next);
1530 return;
1531
1532 default:
1533 do_unallocated:
1534 unallocated_encoding(s);
1535 return;
1536 }
1537 }
1538
1539 static void gen_xaflag(void)
1540 {
1541 TCGv_i32 z = tcg_temp_new_i32();
1542
1543 tcg_gen_setcondi_i32(TCG_COND_EQ, z, cpu_ZF, 0);
1544
1545 /*
1546 * (!C & !Z) << 31
1547 * (!(C | Z)) << 31
1548 * ~((C | Z) << 31)
1549 * ~-(C | Z)
1550 * (C | Z) - 1
1551 */
1552 tcg_gen_or_i32(cpu_NF, cpu_CF, z);
1553 tcg_gen_subi_i32(cpu_NF, cpu_NF, 1);
1554
1555 /* !(Z & C) */
1556 tcg_gen_and_i32(cpu_ZF, z, cpu_CF);
1557 tcg_gen_xori_i32(cpu_ZF, cpu_ZF, 1);
1558
1559 /* (!C & Z) << 31 -> -(Z & ~C) */
1560 tcg_gen_andc_i32(cpu_VF, z, cpu_CF);
1561 tcg_gen_neg_i32(cpu_VF, cpu_VF);
1562
1563 /* C | Z */
1564 tcg_gen_or_i32(cpu_CF, cpu_CF, z);
1565
1566 tcg_temp_free_i32(z);
1567 }
1568
1569 static void gen_axflag(void)
1570 {
1571 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */
1572 tcg_gen_andc_i32(cpu_CF, cpu_CF, cpu_VF); /* C & !V */
1573
1574 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */
1575 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, cpu_VF);
1576
1577 tcg_gen_movi_i32(cpu_NF, 0);
1578 tcg_gen_movi_i32(cpu_VF, 0);
1579 }
1580
1581 /* MSR (immediate) - move immediate to processor state field */
1582 static void handle_msr_i(DisasContext *s, uint32_t insn,
1583 unsigned int op1, unsigned int op2, unsigned int crm)
1584 {
1585 TCGv_i32 t1;
1586 int op = op1 << 3 | op2;
1587
1588 /* End the TB by default, chaining is ok. */
1589 s->base.is_jmp = DISAS_TOO_MANY;
1590
1591 switch (op) {
1592 case 0x00: /* CFINV */
1593 if (crm != 0 || !dc_isar_feature(aa64_condm_4, s)) {
1594 goto do_unallocated;
1595 }
1596 tcg_gen_xori_i32(cpu_CF, cpu_CF, 1);
1597 s->base.is_jmp = DISAS_NEXT;
1598 break;
1599
1600 case 0x01: /* XAFlag */
1601 if (crm != 0 || !dc_isar_feature(aa64_condm_5, s)) {
1602 goto do_unallocated;
1603 }
1604 gen_xaflag();
1605 s->base.is_jmp = DISAS_NEXT;
1606 break;
1607
1608 case 0x02: /* AXFlag */
1609 if (crm != 0 || !dc_isar_feature(aa64_condm_5, s)) {
1610 goto do_unallocated;
1611 }
1612 gen_axflag();
1613 s->base.is_jmp = DISAS_NEXT;
1614 break;
1615
1616 case 0x03: /* UAO */
1617 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) {
1618 goto do_unallocated;
1619 }
1620 if (crm & 1) {
1621 set_pstate_bits(PSTATE_UAO);
1622 } else {
1623 clear_pstate_bits(PSTATE_UAO);
1624 }
1625 t1 = tcg_const_i32(s->current_el);
1626 gen_helper_rebuild_hflags_a64(cpu_env, t1);
1627 tcg_temp_free_i32(t1);
1628 break;
1629
1630 case 0x04: /* PAN */
1631 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) {
1632 goto do_unallocated;
1633 }
1634 if (crm & 1) {
1635 set_pstate_bits(PSTATE_PAN);
1636 } else {
1637 clear_pstate_bits(PSTATE_PAN);
1638 }
1639 t1 = tcg_const_i32(s->current_el);
1640 gen_helper_rebuild_hflags_a64(cpu_env, t1);
1641 tcg_temp_free_i32(t1);
1642 break;
1643
1644 case 0x05: /* SPSel */
1645 if (s->current_el == 0) {
1646 goto do_unallocated;
1647 }
1648 t1 = tcg_const_i32(crm & PSTATE_SP);
1649 gen_helper_msr_i_spsel(cpu_env, t1);
1650 tcg_temp_free_i32(t1);
1651 break;
1652
1653 case 0x1e: /* DAIFSet */
1654 t1 = tcg_const_i32(crm);
1655 gen_helper_msr_i_daifset(cpu_env, t1);
1656 tcg_temp_free_i32(t1);
1657 break;
1658
1659 case 0x1f: /* DAIFClear */
1660 t1 = tcg_const_i32(crm);
1661 gen_helper_msr_i_daifclear(cpu_env, t1);
1662 tcg_temp_free_i32(t1);
1663 /* For DAIFClear, exit the cpu loop to re-evaluate pending IRQs. */
1664 s->base.is_jmp = DISAS_UPDATE;
1665 break;
1666
1667 default:
1668 do_unallocated:
1669 unallocated_encoding(s);
1670 return;
1671 }
1672 }
1673
1674 static void gen_get_nzcv(TCGv_i64 tcg_rt)
1675 {
1676 TCGv_i32 tmp = tcg_temp_new_i32();
1677 TCGv_i32 nzcv = tcg_temp_new_i32();
1678
1679 /* build bit 31, N */
1680 tcg_gen_andi_i32(nzcv, cpu_NF, (1U << 31));
1681 /* build bit 30, Z */
1682 tcg_gen_setcondi_i32(TCG_COND_EQ, tmp, cpu_ZF, 0);
1683 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 30, 1);
1684 /* build bit 29, C */
1685 tcg_gen_deposit_i32(nzcv, nzcv, cpu_CF, 29, 1);
1686 /* build bit 28, V */
1687 tcg_gen_shri_i32(tmp, cpu_VF, 31);
1688 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 28, 1);
1689 /* generate result */
1690 tcg_gen_extu_i32_i64(tcg_rt, nzcv);
1691
1692 tcg_temp_free_i32(nzcv);
1693 tcg_temp_free_i32(tmp);
1694 }
1695
1696 static void gen_set_nzcv(TCGv_i64 tcg_rt)
1697 {
1698 TCGv_i32 nzcv = tcg_temp_new_i32();
1699
1700 /* take NZCV from R[t] */
1701 tcg_gen_extrl_i64_i32(nzcv, tcg_rt);
1702
1703 /* bit 31, N */
1704 tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31));
1705 /* bit 30, Z */
1706 tcg_gen_andi_i32(cpu_ZF, nzcv, (1 << 30));
1707 tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_ZF, cpu_ZF, 0);
1708 /* bit 29, C */
1709 tcg_gen_andi_i32(cpu_CF, nzcv, (1 << 29));
1710 tcg_gen_shri_i32(cpu_CF, cpu_CF, 29);
1711 /* bit 28, V */
1712 tcg_gen_andi_i32(cpu_VF, nzcv, (1 << 28));
1713 tcg_gen_shli_i32(cpu_VF, cpu_VF, 3);
1714 tcg_temp_free_i32(nzcv);
1715 }
1716
1717 /* MRS - move from system register
1718 * MSR (register) - move to system register
1719 * SYS
1720 * SYSL
1721 * These are all essentially the same insn in 'read' and 'write'
1722 * versions, with varying op0 fields.
1723 */
1724 static void handle_sys(DisasContext *s, uint32_t insn, bool isread,
1725 unsigned int op0, unsigned int op1, unsigned int op2,
1726 unsigned int crn, unsigned int crm, unsigned int rt)
1727 {
1728 const ARMCPRegInfo *ri;
1729 TCGv_i64 tcg_rt;
1730
1731 ri = get_arm_cp_reginfo(s->cp_regs,
1732 ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP,
1733 crn, crm, op0, op1, op2));
1734
1735 if (!ri) {
1736 /* Unknown register; this might be a guest error or a QEMU
1737 * unimplemented feature.
1738 */
1739 qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch64 "
1740 "system register op0:%d op1:%d crn:%d crm:%d op2:%d\n",
1741 isread ? "read" : "write", op0, op1, crn, crm, op2);
1742 unallocated_encoding(s);
1743 return;
1744 }
1745
1746 /* Check access permissions */
1747 if (!cp_access_ok(s->current_el, ri, isread)) {
1748 unallocated_encoding(s);
1749 return;
1750 }
1751
1752 if (ri->accessfn) {
1753 /* Emit code to perform further access permissions checks at
1754 * runtime; this may result in an exception.
1755 */
1756 TCGv_ptr tmpptr;
1757 TCGv_i32 tcg_syn, tcg_isread;
1758 uint32_t syndrome;
1759
1760 gen_a64_set_pc_im(s->pc_curr);
1761 tmpptr = tcg_const_ptr(ri);
1762 syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
1763 tcg_syn = tcg_const_i32(syndrome);
1764 tcg_isread = tcg_const_i32(isread);
1765 gen_helper_access_check_cp_reg(cpu_env, tmpptr, tcg_syn, tcg_isread);
1766 tcg_temp_free_ptr(tmpptr);
1767 tcg_temp_free_i32(tcg_syn);
1768 tcg_temp_free_i32(tcg_isread);
1769 } else if (ri->type & ARM_CP_RAISES_EXC) {
1770 /*
1771 * The readfn or writefn might raise an exception;
1772 * synchronize the CPU state in case it does.
1773 */
1774 gen_a64_set_pc_im(s->pc_curr);
1775 }
1776
1777 /* Handle special cases first */
1778 switch (ri->type & ~(ARM_CP_FLAG_MASK & ~ARM_CP_SPECIAL)) {
1779 case ARM_CP_NOP:
1780 return;
1781 case ARM_CP_NZCV:
1782 tcg_rt = cpu_reg(s, rt);
1783 if (isread) {
1784 gen_get_nzcv(tcg_rt);
1785 } else {
1786 gen_set_nzcv(tcg_rt);
1787 }
1788 return;
1789 case ARM_CP_CURRENTEL:
1790 /* Reads as current EL value from pstate, which is
1791 * guaranteed to be constant by the tb flags.
1792 */
1793 tcg_rt = cpu_reg(s, rt);
1794 tcg_gen_movi_i64(tcg_rt, s->current_el << 2);
1795 return;
1796 case ARM_CP_DC_ZVA:
1797 /* Writes clear the aligned block of memory which rt points into. */
1798 tcg_rt = clean_data_tbi(s, cpu_reg(s, rt));
1799 gen_helper_dc_zva(cpu_env, tcg_rt);
1800 return;
1801 default:
1802 break;
1803 }
1804 if ((ri->type & ARM_CP_FPU) && !fp_access_check(s)) {
1805 return;
1806 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
1807 return;
1808 }
1809
1810 if ((tb_cflags(s->base.tb) & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) {
1811 gen_io_start();
1812 }
1813
1814 tcg_rt = cpu_reg(s, rt);
1815
1816 if (isread) {
1817 if (ri->type & ARM_CP_CONST) {
1818 tcg_gen_movi_i64(tcg_rt, ri->resetvalue);
1819 } else if (ri->readfn) {
1820 TCGv_ptr tmpptr;
1821 tmpptr = tcg_const_ptr(ri);
1822 gen_helper_get_cp_reg64(tcg_rt, cpu_env, tmpptr);
1823 tcg_temp_free_ptr(tmpptr);
1824 } else {
1825 tcg_gen_ld_i64(tcg_rt, cpu_env, ri->fieldoffset);
1826 }
1827 } else {
1828 if (ri->type & ARM_CP_CONST) {
1829 /* If not forbidden by access permissions, treat as WI */
1830 return;
1831 } else if (ri->writefn) {
1832 TCGv_ptr tmpptr;
1833 tmpptr = tcg_const_ptr(ri);
1834 gen_helper_set_cp_reg64(cpu_env, tmpptr, tcg_rt);
1835 tcg_temp_free_ptr(tmpptr);
1836 } else {
1837 tcg_gen_st_i64(tcg_rt, cpu_env, ri->fieldoffset);
1838 }
1839 }
1840
1841 if ((tb_cflags(s->base.tb) & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) {
1842 /* I/O operations must end the TB here (whether read or write) */
1843 s->base.is_jmp = DISAS_UPDATE;
1844 }
1845 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) {
1846 /*
1847 * A write to any coprocessor regiser that ends a TB
1848 * must rebuild the hflags for the next TB.
1849 */
1850 TCGv_i32 tcg_el = tcg_const_i32(s->current_el);
1851 gen_helper_rebuild_hflags_a64(cpu_env, tcg_el);
1852 tcg_temp_free_i32(tcg_el);
1853 /*
1854 * We default to ending the TB on a coprocessor register write,
1855 * but allow this to be suppressed by the register definition
1856 * (usually only necessary to work around guest bugs).
1857 */
1858 s->base.is_jmp = DISAS_UPDATE;
1859 }
1860 }
1861
1862 /* System
1863 * 31 22 21 20 19 18 16 15 12 11 8 7 5 4 0
1864 * +---------------------+---+-----+-----+-------+-------+-----+------+
1865 * | 1 1 0 1 0 1 0 1 0 0 | L | op0 | op1 | CRn | CRm | op2 | Rt |
1866 * +---------------------+---+-----+-----+-------+-------+-----+------+
1867 */
1868 static void disas_system(DisasContext *s, uint32_t insn)
1869 {
1870 unsigned int l, op0, op1, crn, crm, op2, rt;
1871 l = extract32(insn, 21, 1);
1872 op0 = extract32(insn, 19, 2);
1873 op1 = extract32(insn, 16, 3);
1874 crn = extract32(insn, 12, 4);
1875 crm = extract32(insn, 8, 4);
1876 op2 = extract32(insn, 5, 3);
1877 rt = extract32(insn, 0, 5);
1878
1879 if (op0 == 0) {
1880 if (l || rt != 31) {
1881 unallocated_encoding(s);
1882 return;
1883 }
1884 switch (crn) {
1885 case 2: /* HINT (including allocated hints like NOP, YIELD, etc) */
1886 handle_hint(s, insn, op1, op2, crm);
1887 break;
1888 case 3: /* CLREX, DSB, DMB, ISB */
1889 handle_sync(s, insn, op1, op2, crm);
1890 break;
1891 case 4: /* MSR (immediate) */
1892 handle_msr_i(s, insn, op1, op2, crm);
1893 break;
1894 default:
1895 unallocated_encoding(s);
1896 break;
1897 }
1898 return;
1899 }
1900 handle_sys(s, insn, l, op0, op1, op2, crn, crm, rt);
1901 }
1902
1903 /* Exception generation
1904 *
1905 * 31 24 23 21 20 5 4 2 1 0
1906 * +-----------------+-----+------------------------+-----+----+
1907 * | 1 1 0 1 0 1 0 0 | opc | imm16 | op2 | LL |
1908 * +-----------------------+------------------------+----------+
1909 */
1910 static void disas_exc(DisasContext *s, uint32_t insn)
1911 {
1912 int opc = extract32(insn, 21, 3);
1913 int op2_ll = extract32(insn, 0, 5);
1914 int imm16 = extract32(insn, 5, 16);
1915 TCGv_i32 tmp;
1916
1917 switch (opc) {
1918 case 0:
1919 /* For SVC, HVC and SMC we advance the single-step state
1920 * machine before taking the exception. This is architecturally
1921 * mandated, to ensure that single-stepping a system call
1922 * instruction works properly.
1923 */
1924 switch (op2_ll) {
1925 case 1: /* SVC */
1926 gen_ss_advance(s);
1927 gen_exception_insn(s, s->base.pc_next, EXCP_SWI,
1928 syn_aa64_svc(imm16), default_exception_el(s));
1929 break;
1930 case 2: /* HVC */
1931 if (s->current_el == 0) {
1932 unallocated_encoding(s);
1933 break;
1934 }
1935 /* The pre HVC helper handles cases when HVC gets trapped
1936 * as an undefined insn by runtime configuration.
1937 */
1938 gen_a64_set_pc_im(s->pc_curr);
1939 gen_helper_pre_hvc(cpu_env);
1940 gen_ss_advance(s);
1941 gen_exception_insn(s, s->base.pc_next, EXCP_HVC,
1942 syn_aa64_hvc(imm16), 2);
1943 break;
1944 case 3: /* SMC */
1945 if (s->current_el == 0) {
1946 unallocated_encoding(s);
1947 break;
1948 }
1949 gen_a64_set_pc_im(s->pc_curr);
1950 tmp = tcg_const_i32(syn_aa64_smc(imm16));
1951 gen_helper_pre_smc(cpu_env, tmp);
1952 tcg_temp_free_i32(tmp);
1953 gen_ss_advance(s);
1954 gen_exception_insn(s, s->base.pc_next, EXCP_SMC,
1955 syn_aa64_smc(imm16), 3);
1956 break;
1957 default:
1958 unallocated_encoding(s);
1959 break;
1960 }
1961 break;
1962 case 1:
1963 if (op2_ll != 0) {
1964 unallocated_encoding(s);
1965 break;
1966 }
1967 /* BRK */
1968 gen_exception_bkpt_insn(s, syn_aa64_bkpt(imm16));
1969 break;
1970 case 2:
1971 if (op2_ll != 0) {
1972 unallocated_encoding(s);
1973 break;
1974 }
1975 /* HLT. This has two purposes.
1976 * Architecturally, it is an external halting debug instruction.
1977 * Since QEMU doesn't implement external debug, we treat this as
1978 * it is required for halting debug disabled: it will UNDEF.
1979 * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
1980 */
1981 if (semihosting_enabled() && imm16 == 0xf000) {
1982 #ifndef CONFIG_USER_ONLY
1983 /* In system mode, don't allow userspace access to semihosting,
1984 * to provide some semblance of security (and for consistency
1985 * with our 32-bit semihosting).
1986 */
1987 if (s->current_el == 0) {
1988 unsupported_encoding(s, insn);
1989 break;
1990 }
1991 #endif
1992 gen_exception_internal_insn(s, s->pc_curr, EXCP_SEMIHOST);
1993 } else {
1994 unsupported_encoding(s, insn);
1995 }
1996 break;
1997 case 5:
1998 if (op2_ll < 1 || op2_ll > 3) {
1999 unallocated_encoding(s);
2000 break;
2001 }
2002 /* DCPS1, DCPS2, DCPS3 */
2003 unsupported_encoding(s, insn);
2004 break;
2005 default:
2006 unallocated_encoding(s);
2007 break;
2008 }
2009 }
2010
2011 /* Unconditional branch (register)
2012 * 31 25 24 21 20 16 15 10 9 5 4 0
2013 * +---------------+-------+-------+-------+------+-------+
2014 * | 1 1 0 1 0 1 1 | opc | op2 | op3 | Rn | op4 |
2015 * +---------------+-------+-------+-------+------+-------+
2016 */
2017 static void disas_uncond_b_reg(DisasContext *s, uint32_t insn)
2018 {
2019 unsigned int opc, op2, op3, rn, op4;
2020 unsigned btype_mod = 2; /* 0: BR, 1: BLR, 2: other */
2021 TCGv_i64 dst;
2022 TCGv_i64 modifier;
2023
2024 opc = extract32(insn, 21, 4);
2025 op2 = extract32(insn, 16, 5);
2026 op3 = extract32(insn, 10, 6);
2027 rn = extract32(insn, 5, 5);
2028 op4 = extract32(insn, 0, 5);
2029
2030 if (op2 != 0x1f) {
2031 goto do_unallocated;
2032 }
2033
2034 switch (opc) {
2035 case 0: /* BR */
2036 case 1: /* BLR */
2037 case 2: /* RET */
2038 btype_mod = opc;
2039 switch (op3) {
2040 case 0:
2041 /* BR, BLR, RET */
2042 if (op4 != 0) {
2043 goto do_unallocated;
2044 }
2045 dst = cpu_reg(s, rn);
2046 break;
2047
2048 case 2:
2049 case 3:
2050 if (!dc_isar_feature(aa64_pauth, s)) {
2051 goto do_unallocated;
2052 }
2053 if (opc == 2) {
2054 /* RETAA, RETAB */
2055 if (rn != 0x1f || op4 != 0x1f) {
2056 goto do_unallocated;
2057 }
2058 rn = 30;
2059 modifier = cpu_X[31];
2060 } else {
2061 /* BRAAZ, BRABZ, BLRAAZ, BLRABZ */
2062 if (op4 != 0x1f) {
2063 goto do_unallocated;
2064 }
2065 modifier = new_tmp_a64_zero(s);
2066 }
2067 if (s->pauth_active) {
2068 dst = new_tmp_a64(s);
2069 if (op3 == 2) {
2070 gen_helper_autia(dst, cpu_env, cpu_reg(s, rn), modifier);
2071 } else {
2072 gen_helper_autib(dst, cpu_env, cpu_reg(s, rn), modifier);
2073 }
2074 } else {
2075 dst = cpu_reg(s, rn);
2076 }
2077 break;
2078
2079 default:
2080 goto do_unallocated;
2081 }
2082 gen_a64_set_pc(s, dst);
2083 /* BLR also needs to load return address */
2084 if (opc == 1) {
2085 tcg_gen_movi_i64(cpu_reg(s, 30), s->base.pc_next);
2086 }
2087 break;
2088
2089 case 8: /* BRAA */
2090 case 9: /* BLRAA */
2091 if (!dc_isar_feature(aa64_pauth, s)) {
2092 goto do_unallocated;
2093 }
2094 if ((op3 & ~1) != 2) {
2095 goto do_unallocated;
2096 }
2097 btype_mod = opc & 1;
2098 if (s->pauth_active) {
2099 dst = new_tmp_a64(s);
2100 modifier = cpu_reg_sp(s, op4);
2101 if (op3 == 2) {
2102 gen_helper_autia(dst, cpu_env, cpu_reg(s, rn), modifier);
2103 } else {
2104 gen_helper_autib(dst, cpu_env, cpu_reg(s, rn), modifier);
2105 }
2106 } else {
2107 dst = cpu_reg(s, rn);
2108 }
2109 gen_a64_set_pc(s, dst);
2110 /* BLRAA also needs to load return address */
2111 if (opc == 9) {
2112 tcg_gen_movi_i64(cpu_reg(s, 30), s->base.pc_next);
2113 }
2114 break;
2115
2116 case 4: /* ERET */
2117 if (s->current_el == 0) {
2118 goto do_unallocated;
2119 }
2120 switch (op3) {
2121 case 0: /* ERET */
2122 if (op4 != 0) {
2123 goto do_unallocated;
2124 }
2125 dst = tcg_temp_new_i64();
2126 tcg_gen_ld_i64(dst, cpu_env,
2127 offsetof(CPUARMState, elr_el[s->current_el]));
2128 break;
2129
2130 case 2: /* ERETAA */
2131 case 3: /* ERETAB */
2132 if (!dc_isar_feature(aa64_pauth, s)) {
2133 goto do_unallocated;
2134 }
2135 if (rn != 0x1f || op4 != 0x1f) {
2136 goto do_unallocated;
2137 }
2138 dst = tcg_temp_new_i64();
2139 tcg_gen_ld_i64(dst, cpu_env,
2140 offsetof(CPUARMState, elr_el[s->current_el]));
2141 if (s->pauth_active) {
2142 modifier = cpu_X[31];
2143 if (op3 == 2) {
2144 gen_helper_autia(dst, cpu_env, dst, modifier);
2145 } else {
2146 gen_helper_autib(dst, cpu_env, dst, modifier);
2147 }
2148 }
2149 break;
2150
2151 default:
2152 goto do_unallocated;
2153 }
2154 if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
2155 gen_io_start();
2156 }
2157
2158 gen_helper_exception_return(cpu_env, dst);
2159 tcg_temp_free_i64(dst);
2160 /* Must exit loop to check un-masked IRQs */
2161 s->base.is_jmp = DISAS_EXIT;
2162 return;
2163
2164 case 5: /* DRPS */
2165 if (op3 != 0 || op4 != 0 || rn != 0x1f) {
2166 goto do_unallocated;
2167 } else {
2168 unsupported_encoding(s, insn);
2169 }
2170 return;
2171
2172 default:
2173 do_unallocated:
2174 unallocated_encoding(s);
2175 return;
2176 }
2177
2178 switch (btype_mod) {
2179 case 0: /* BR */
2180 if (dc_isar_feature(aa64_bti, s)) {
2181 /* BR to {x16,x17} or !guard -> 1, else 3. */
2182 set_btype(s, rn == 16 || rn == 17 || !s->guarded_page ? 1 : 3);
2183 }
2184 break;
2185
2186 case 1: /* BLR */
2187 if (dc_isar_feature(aa64_bti, s)) {
2188 /* BLR sets BTYPE to 2, regardless of source guarded page. */
2189 set_btype(s, 2);
2190 }
2191 break;
2192
2193 default: /* RET or none of the above. */
2194 /* BTYPE will be set to 0 by normal end-of-insn processing. */
2195 break;
2196 }
2197
2198 s->base.is_jmp = DISAS_JUMP;
2199 }
2200
2201 /* Branches, exception generating and system instructions */
2202 static void disas_b_exc_sys(DisasContext *s, uint32_t insn)
2203 {
2204 switch (extract32(insn, 25, 7)) {
2205 case 0x0a: case 0x0b:
2206 case 0x4a: case 0x4b: /* Unconditional branch (immediate) */
2207 disas_uncond_b_imm(s, insn);
2208 break;
2209 case 0x1a: case 0x5a: /* Compare & branch (immediate) */
2210 disas_comp_b_imm(s, insn);
2211 break;
2212 case 0x1b: case 0x5b: /* Test & branch (immediate) */
2213 disas_test_b_imm(s, insn);
2214 break;
2215 case 0x2a: /* Conditional branch (immediate) */
2216 disas_cond_b_imm(s, insn);
2217 break;
2218 case 0x6a: /* Exception generation / System */
2219 if (insn & (1 << 24)) {
2220 if (extract32(insn, 22, 2) == 0) {
2221 disas_system(s, insn);
2222 } else {
2223 unallocated_encoding(s);
2224 }
2225 } else {
2226 disas_exc(s, insn);
2227 }
2228 break;
2229 case 0x6b: /* Unconditional branch (register) */
2230 disas_uncond_b_reg(s, insn);
2231 break;
2232 default:
2233 unallocated_encoding(s);
2234 break;
2235 }
2236 }
2237
2238 /*
2239 * Load/Store exclusive instructions are implemented by remembering
2240 * the value/address loaded, and seeing if these are the same
2241 * when the store is performed. This is not actually the architecturally
2242 * mandated semantics, but it works for typical guest code sequences
2243 * and avoids having to monitor regular stores.
2244 *
2245 * The store exclusive uses the atomic cmpxchg primitives to avoid
2246 * races in multi-threaded linux-user and when MTTCG softmmu is
2247 * enabled.
2248 */
2249 static void gen_load_exclusive(DisasContext *s, int rt, int rt2,
2250 TCGv_i64 addr, int size, bool is_pair)
2251 {
2252 int idx = get_mem_index(s);
2253 MemOp memop = s->be_data;
2254
2255 g_assert(size <= 3);
2256 if (is_pair) {
2257 g_assert(size >= 2);
2258 if (size == 2) {
2259 /* The pair must be single-copy atomic for the doubleword. */
2260 memop |= MO_64 | MO_ALIGN;
2261 tcg_gen_qemu_ld_i64(cpu_exclusive_val, addr, idx, memop);
2262 if (s->be_data == MO_LE) {
2263 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 0, 32);
2264 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 32, 32);
2265 } else {
2266 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 32, 32);
2267 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 0, 32);
2268 }
2269 } else {
2270 /* The pair must be single-copy atomic for *each* doubleword, not
2271 the entire quadword, however it must be quadword aligned. */
2272 memop |= MO_64;
2273 tcg_gen_qemu_ld_i64(cpu_exclusive_val, addr, idx,
2274 memop | MO_ALIGN_16);
2275
2276 TCGv_i64 addr2 = tcg_temp_new_i64();
2277 tcg_gen_addi_i64(addr2, addr, 8);
2278 tcg_gen_qemu_ld_i64(cpu_exclusive_high, addr2, idx, memop);
2279 tcg_temp_free_i64(addr2);
2280
2281 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2282 tcg_gen_mov_i64(cpu_reg(s, rt2), cpu_exclusive_high);
2283 }
2284 } else {
2285 memop |= size | MO_ALIGN;
2286 tcg_gen_qemu_ld_i64(cpu_exclusive_val, addr, idx, memop);
2287 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2288 }
2289 tcg_gen_mov_i64(cpu_exclusive_addr, addr);
2290 }
2291
2292 static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
2293 TCGv_i64 addr, int size, int is_pair)
2294 {
2295 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr]
2296 * && (!is_pair || env->exclusive_high == [addr + datasize])) {
2297 * [addr] = {Rt};
2298 * if (is_pair) {
2299 * [addr + datasize] = {Rt2};
2300 * }
2301 * {Rd} = 0;
2302 * } else {
2303 * {Rd} = 1;
2304 * }
2305 * env->exclusive_addr = -1;
2306 */
2307 TCGLabel *fail_label = gen_new_label();
2308 TCGLabel *done_label = gen_new_label();
2309 TCGv_i64 tmp;
2310
2311 tcg_gen_brcond_i64(TCG_COND_NE, addr, cpu_exclusive_addr, fail_label);
2312
2313 tmp = tcg_temp_new_i64();
2314 if (is_pair) {
2315 if (size == 2) {
2316 if (s->be_data == MO_LE) {
2317 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt), cpu_reg(s, rt2));
2318 } else {
2319 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt2), cpu_reg(s, rt));
2320 }
2321 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr,
2322 cpu_exclusive_val, tmp,
2323 get_mem_index(s),
2324 MO_64 | MO_ALIGN | s->be_data);
2325 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2326 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
2327 if (!HAVE_CMPXCHG128) {
2328 gen_helper_exit_atomic(cpu_env);
2329 s->base.is_jmp = DISAS_NORETURN;
2330 } else if (s->be_data == MO_LE) {
2331 gen_helper_paired_cmpxchg64_le_parallel(tmp, cpu_env,
2332 cpu_exclusive_addr,
2333 cpu_reg(s, rt),
2334 cpu_reg(s, rt2));
2335 } else {
2336 gen_helper_paired_cmpxchg64_be_parallel(tmp, cpu_env,
2337 cpu_exclusive_addr,
2338 cpu_reg(s, rt),
2339 cpu_reg(s, rt2));
2340 }
2341 } else if (s->be_data == MO_LE) {
2342 gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr,
2343 cpu_reg(s, rt), cpu_reg(s, rt2));
2344 } else {
2345 gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr,
2346 cpu_reg(s, rt), cpu_reg(s, rt2));
2347 }
2348 } else {
2349 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,
2350 cpu_reg(s, rt), get_mem_index(s),
2351 size | MO_ALIGN | s->be_data);
2352 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2353 }
2354 tcg_gen_mov_i64(cpu_reg(s, rd), tmp);
2355 tcg_temp_free_i64(tmp);
2356 tcg_gen_br(done_label);
2357
2358 gen_set_label(fail_label);
2359 tcg_gen_movi_i64(cpu_reg(s, rd), 1);
2360 gen_set_label(done_label);
2361 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
2362 }
2363
2364 static void gen_compare_and_swap(DisasContext *s, int rs, int rt,
2365 int rn, int size)
2366 {
2367 TCGv_i64 tcg_rs = cpu_reg(s, rs);
2368 TCGv_i64 tcg_rt = cpu_reg(s, rt);
2369 int memidx = get_mem_index(s);
2370 TCGv_i64 clean_addr;
2371
2372 if (rn == 31) {
2373 gen_check_sp_alignment(s);
2374 }
2375 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2376 tcg_gen_atomic_cmpxchg_i64(tcg_rs, clean_addr, tcg_rs, tcg_rt, memidx,
2377 size | MO_ALIGN | s->be_data);
2378 }
2379
2380 static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt,
2381 int rn, int size)
2382 {
2383 TCGv_i64 s1 = cpu_reg(s, rs);
2384 TCGv_i64 s2 = cpu_reg(s, rs + 1);
2385 TCGv_i64 t1 = cpu_reg(s, rt);
2386 TCGv_i64 t2 = cpu_reg(s, rt + 1);
2387 TCGv_i64 clean_addr;
2388 int memidx = get_mem_index(s);
2389
2390 if (rn == 31) {
2391 gen_check_sp_alignment(s);
2392 }
2393 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2394
2395 if (size == 2) {
2396 TCGv_i64 cmp = tcg_temp_new_i64();
2397 TCGv_i64 val = tcg_temp_new_i64();
2398
2399 if (s->be_data == MO_LE) {
2400 tcg_gen_concat32_i64(val, t1, t2);
2401 tcg_gen_concat32_i64(cmp, s1, s2);
2402 } else {
2403 tcg_gen_concat32_i64(val, t2, t1);
2404 tcg_gen_concat32_i64(cmp, s2, s1);
2405 }
2406
2407 tcg_gen_atomic_cmpxchg_i64(cmp, clean_addr, cmp, val, memidx,
2408 MO_64 | MO_ALIGN | s->be_data);
2409 tcg_temp_free_i64(val);
2410
2411 if (s->be_data == MO_LE) {
2412 tcg_gen_extr32_i64(s1, s2, cmp);
2413 } else {
2414 tcg_gen_extr32_i64(s2, s1, cmp);
2415 }
2416 tcg_temp_free_i64(cmp);
2417 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
2418 if (HAVE_CMPXCHG128) {
2419 TCGv_i32 tcg_rs = tcg_const_i32(rs);
2420 if (s->be_data == MO_LE) {
2421 gen_helper_casp_le_parallel(cpu_env, tcg_rs,
2422 clean_addr, t1, t2);
2423 } else {
2424 gen_helper_casp_be_parallel(cpu_env, tcg_rs,
2425 clean_addr, t1, t2);
2426 }
2427 tcg_temp_free_i32(tcg_rs);
2428 } else {
2429 gen_helper_exit_atomic(cpu_env);
2430 s->base.is_jmp = DISAS_NORETURN;
2431 }
2432 } else {
2433 TCGv_i64 d1 = tcg_temp_new_i64();
2434 TCGv_i64 d2 = tcg_temp_new_i64();
2435 TCGv_i64 a2 = tcg_temp_new_i64();
2436 TCGv_i64 c1 = tcg_temp_new_i64();
2437 TCGv_i64 c2 = tcg_temp_new_i64();
2438 TCGv_i64 zero = tcg_const_i64(0);
2439
2440 /* Load the two words, in memory order. */
2441 tcg_gen_qemu_ld_i64(d1, clean_addr, memidx,
2442 MO_64 | MO_ALIGN_16 | s->be_data);
2443 tcg_gen_addi_i64(a2, clean_addr, 8);
2444 tcg_gen_qemu_ld_i64(d2, a2, memidx, MO_64 | s->be_data);
2445
2446 /* Compare the two words, also in memory order. */
2447 tcg_gen_setcond_i64(TCG_COND_EQ, c1, d1, s1);
2448 tcg_gen_setcond_i64(TCG_COND_EQ, c2, d2, s2);
2449 tcg_gen_and_i64(c2, c2, c1);
2450
2451 /* If compare equal, write back new data, else write back old data. */
2452 tcg_gen_movcond_i64(TCG_COND_NE, c1, c2, zero, t1, d1);
2453 tcg_gen_movcond_i64(TCG_COND_NE, c2, c2, zero, t2, d2);
2454 tcg_gen_qemu_st_i64(c1, clean_addr, memidx, MO_64 | s->be_data);
2455 tcg_gen_qemu_st_i64(c2, a2, memidx, MO_64 | s->be_data);
2456 tcg_temp_free_i64(a2);
2457 tcg_temp_free_i64(c1);
2458 tcg_temp_free_i64(c2);
2459 tcg_temp_free_i64(zero);
2460
2461 /* Write back the data from memory to Rs. */
2462 tcg_gen_mov_i64(s1, d1);
2463 tcg_gen_mov_i64(s2, d2);
2464 tcg_temp_free_i64(d1);
2465 tcg_temp_free_i64(d2);
2466 }
2467 }
2468
2469 /* Update the Sixty-Four bit (SF) registersize. This logic is derived
2470 * from the ARMv8 specs for LDR (Shared decode for all encodings).
2471 */
2472 static bool disas_ldst_compute_iss_sf(int size, bool is_signed, int opc)
2473 {
2474 int opc0 = extract32(opc, 0, 1);
2475 int regsize;
2476
2477 if (is_signed) {
2478 regsize = opc0 ? 32 : 64;
2479 } else {
2480 regsize = size == 3 ? 64 : 32;
2481 }
2482 return regsize == 64;
2483 }
2484
2485 /* Load/store exclusive
2486 *
2487 * 31 30 29 24 23 22 21 20 16 15 14 10 9 5 4 0
2488 * +-----+-------------+----+---+----+------+----+-------+------+------+
2489 * | sz | 0 0 1 0 0 0 | o2 | L | o1 | Rs | o0 | Rt2 | Rn | Rt |
2490 * +-----+-------------+----+---+----+------+----+-------+------+------+
2491 *
2492 * sz: 00 -> 8 bit, 01 -> 16 bit, 10 -> 32 bit, 11 -> 64 bit
2493 * L: 0 -> store, 1 -> load
2494 * o2: 0 -> exclusive, 1 -> not
2495 * o1: 0 -> single register, 1 -> register pair
2496 * o0: 1 -> load-acquire/store-release, 0 -> not
2497 */
2498 static void disas_ldst_excl(DisasContext *s, uint32_t insn)
2499 {
2500 int rt = extract32(insn, 0, 5);
2501 int rn = extract32(insn, 5, 5);
2502 int rt2 = extract32(insn, 10, 5);
2503 int rs = extract32(insn, 16, 5);
2504 int is_lasr = extract32(insn, 15, 1);
2505 int o2_L_o1_o0 = extract32(insn, 21, 3) * 2 | is_lasr;
2506 int size = extract32(insn, 30, 2);
2507 TCGv_i64 clean_addr;
2508
2509 switch (o2_L_o1_o0) {
2510 case 0x0: /* STXR */
2511 case 0x1: /* STLXR */
2512 if (rn == 31) {
2513 gen_check_sp_alignment(s);
2514 }
2515 if (is_lasr) {
2516 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2517 }
2518 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2519 gen_store_exclusive(s, rs, rt, rt2, clean_addr, size, false);
2520 return;
2521
2522 case 0x4: /* LDXR */
2523 case 0x5: /* LDAXR */
2524 if (rn == 31) {
2525 gen_check_sp_alignment(s);
2526 }
2527 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2528 s->is_ldex = true;
2529 gen_load_exclusive(s, rt, rt2, clean_addr, size, false);
2530 if (is_lasr) {
2531 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2532 }
2533 return;
2534
2535 case 0x8: /* STLLR */
2536 if (!dc_isar_feature(aa64_lor, s)) {
2537 break;
2538 }
2539 /* StoreLORelease is the same as Store-Release for QEMU. */
2540 /* fall through */
2541 case 0x9: /* STLR */
2542 /* Generate ISS for non-exclusive accesses including LASR. */
2543 if (rn == 31) {
2544 gen_check_sp_alignment(s);
2545 }
2546 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2547 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2548 do_gpr_st(s, cpu_reg(s, rt), clean_addr, size, true, rt,
2549 disas_ldst_compute_iss_sf(size, false, 0), is_lasr);
2550 return;
2551
2552 case 0xc: /* LDLAR */
2553 if (!dc_isar_feature(aa64_lor, s)) {
2554 break;
2555 }
2556 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */
2557 /* fall through */
2558 case 0xd: /* LDAR */
2559 /* Generate ISS for non-exclusive accesses including LASR. */
2560 if (rn == 31) {
2561 gen_check_sp_alignment(s);
2562 }
2563 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2564 do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, false, false, true, rt,
2565 disas_ldst_compute_iss_sf(size, false, 0), is_lasr);
2566 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2567 return;
2568
2569 case 0x2: case 0x3: /* CASP / STXP */
2570 if (size & 2) { /* STXP / STLXP */
2571 if (rn == 31) {
2572 gen_check_sp_alignment(s);
2573 }
2574 if (is_lasr) {
2575 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2576 }
2577 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2578 gen_store_exclusive(s, rs, rt, rt2, clean_addr, size, true);
2579 return;
2580 }
2581 if (rt2 == 31
2582 && ((rt | rs) & 1) == 0
2583 && dc_isar_feature(aa64_atomics, s)) {
2584 /* CASP / CASPL */
2585 gen_compare_and_swap_pair(s, rs, rt, rn, size | 2);
2586 return;
2587 }
2588 break;
2589
2590 case 0x6: case 0x7: /* CASPA / LDXP */
2591 if (size & 2) { /* LDXP / LDAXP */
2592 if (rn == 31) {
2593 gen_check_sp_alignment(s);
2594 }
2595 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2596 s->is_ldex = true;
2597 gen_load_exclusive(s, rt, rt2, clean_addr, size, true);
2598 if (is_lasr) {
2599 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2600 }
2601 return;
2602 }
2603 if (rt2 == 31
2604 && ((rt | rs) & 1) == 0
2605 && dc_isar_feature(aa64_atomics, s)) {
2606 /* CASPA / CASPAL */
2607 gen_compare_and_swap_pair(s, rs, rt, rn, size | 2);
2608 return;
2609 }
2610 break;
2611
2612 case 0xa: /* CAS */
2613 case 0xb: /* CASL */
2614 case 0xe: /* CASA */
2615 case 0xf: /* CASAL */
2616 if (rt2 == 31 && dc_isar_feature(aa64_atomics, s)) {
2617 gen_compare_and_swap(s, rs, rt, rn, size);
2618 return;
2619 }
2620 break;
2621 }
2622 unallocated_encoding(s);
2623 }
2624
2625 /*
2626 * Load register (literal)
2627 *
2628 * 31 30 29 27 26 25 24 23 5 4 0
2629 * +-----+-------+---+-----+-------------------+-------+
2630 * | opc | 0 1 1 | V | 0 0 | imm19 | Rt |
2631 * +-----+-------+---+-----+-------------------+-------+
2632 *
2633 * V: 1 -> vector (simd/fp)
2634 * opc (non-vector): 00 -> 32 bit, 01 -> 64 bit,
2635 * 10-> 32 bit signed, 11 -> prefetch
2636 * opc (vector): 00 -> 32 bit, 01 -> 64 bit, 10 -> 128 bit (11 unallocated)
2637 */
2638 static void disas_ld_lit(DisasContext *s, uint32_t insn)
2639 {
2640 int rt = extract32(insn, 0, 5);
2641 int64_t imm = sextract32(insn, 5, 19) << 2;
2642 bool is_vector = extract32(insn, 26, 1);
2643 int opc = extract32(insn, 30, 2);
2644 bool is_signed = false;
2645 int size = 2;
2646 TCGv_i64 tcg_rt, clean_addr;
2647
2648 if (is_vector) {
2649 if (opc == 3) {
2650 unallocated_encoding(s);
2651 return;
2652 }
2653 size = 2 + opc;
2654 if (!fp_access_check(s)) {
2655 return;
2656 }
2657 } else {
2658 if (opc == 3) {
2659 /* PRFM (literal) : prefetch */
2660 return;
2661 }
2662 size = 2 + extract32(opc, 0, 1);
2663 is_signed = extract32(opc, 1, 1);
2664 }
2665
2666 tcg_rt = cpu_reg(s, rt);
2667
2668 clean_addr = tcg_const_i64(s->pc_curr + imm);
2669 if (is_vector) {
2670 do_fp_ld(s, rt, clean_addr, size);
2671 } else {
2672 /* Only unsigned 32bit loads target 32bit registers. */
2673 bool iss_sf = opc != 0;
2674
2675 do_gpr_ld(s, tcg_rt, clean_addr, size, is_signed, false,
2676 true, rt, iss_sf, false);
2677 }
2678 tcg_temp_free_i64(clean_addr);
2679 }
2680
2681 /*
2682 * LDNP (Load Pair - non-temporal hint)
2683 * LDP (Load Pair - non vector)
2684 * LDPSW (Load Pair Signed Word - non vector)
2685 * STNP (Store Pair - non-temporal hint)
2686 * STP (Store Pair - non vector)
2687 * LDNP (Load Pair of SIMD&FP - non-temporal hint)
2688 * LDP (Load Pair of SIMD&FP)
2689 * STNP (Store Pair of SIMD&FP - non-temporal hint)
2690 * STP (Store Pair of SIMD&FP)
2691 *
2692 * 31 30 29 27 26 25 24 23 22 21 15 14 10 9 5 4 0
2693 * +-----+-------+---+---+-------+---+-----------------------------+
2694 * | opc | 1 0 1 | V | 0 | index | L | imm7 | Rt2 | Rn | Rt |
2695 * +-----+-------+---+---+-------+---+-------+-------+------+------+
2696 *
2697 * opc: LDP/STP/LDNP/STNP 00 -> 32 bit, 10 -> 64 bit
2698 * LDPSW 01
2699 * LDP/STP/LDNP/STNP (SIMD) 00 -> 32 bit, 01 -> 64 bit, 10 -> 128 bit
2700 * V: 0 -> GPR, 1 -> Vector
2701 * idx: 00 -> signed offset with non-temporal hint, 01 -> post-index,
2702 * 10 -> signed offset, 11 -> pre-index
2703 * L: 0 -> Store 1 -> Load
2704 *
2705 * Rt, Rt2 = GPR or SIMD registers to be stored
2706 * Rn = general purpose register containing address
2707 * imm7 = signed offset (multiple of 4 or 8 depending on size)
2708 */
2709 static void disas_ldst_pair(DisasContext *s, uint32_t insn)
2710 {
2711 int rt = extract32(insn, 0, 5);
2712 int rn = extract32(insn, 5, 5);
2713 int rt2 = extract32(insn, 10, 5);
2714 uint64_t offset = sextract64(insn, 15, 7);
2715 int index = extract32(insn, 23, 2);
2716 bool is_vector = extract32(insn, 26, 1);
2717 bool is_load = extract32(insn, 22, 1);
2718 int opc = extract32(insn, 30, 2);
2719
2720 bool is_signed = false;
2721 bool postindex = false;
2722 bool wback = false;
2723
2724 TCGv_i64 clean_addr, dirty_addr;
2725
2726 int size;
2727
2728 if (opc == 3) {
2729 unallocated_encoding(s);
2730 return;
2731 }
2732
2733 if (is_vector) {
2734 size = 2 + opc;
2735 } else {
2736 size = 2 + extract32(opc, 1, 1);
2737 is_signed = extract32(opc, 0, 1);
2738 if (!is_load && is_signed) {
2739 unallocated_encoding(s);
2740 return;
2741 }
2742 }
2743
2744 switch (index) {
2745 case 1: /* post-index */
2746 postindex = true;
2747 wback = true;
2748 break;
2749 case 0:
2750 /* signed offset with "non-temporal" hint. Since we don't emulate
2751 * caches we don't care about hints to the cache system about
2752 * data access patterns, and handle this identically to plain
2753 * signed offset.
2754 */
2755 if (is_signed) {
2756 /* There is no non-temporal-hint version of LDPSW */
2757 unallocated_encoding(s);
2758 return;
2759 }
2760 postindex = false;
2761 break;
2762 case 2: /* signed offset, rn not updated */
2763 postindex = false;
2764 break;
2765 case 3: /* pre-index */
2766 postindex = false;
2767 wback = true;
2768 break;
2769 }
2770
2771 if (is_vector && !fp_access_check(s)) {
2772 return;
2773 }
2774
2775 offset <<= size;
2776
2777 if (rn == 31) {
2778 gen_check_sp_alignment(s);
2779 }
2780
2781 dirty_addr = read_cpu_reg_sp(s, rn, 1);
2782 if (!postindex) {
2783 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
2784 }
2785 clean_addr = clean_data_tbi(s, dirty_addr);
2786
2787 if (is_vector) {
2788 if (is_load) {
2789 do_fp_ld(s, rt, clean_addr, size);
2790 } else {
2791 do_fp_st(s, rt, clean_addr, size);
2792 }
2793 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << size);
2794 if (is_load) {
2795 do_fp_ld(s, rt2, clean_addr, size);
2796 } else {
2797 do_fp_st(s, rt2, clean_addr, size);
2798 }
2799 } else {
2800 TCGv_i64 tcg_rt = cpu_reg(s, rt);
2801 TCGv_i64 tcg_rt2 = cpu_reg(s, rt2);
2802
2803 if (is_load) {
2804 TCGv_i64 tmp = tcg_temp_new_i64();
2805
2806 /* Do not modify tcg_rt before recognizing any exception
2807 * from the second load.
2808 */
2809 do_gpr_ld(s, tmp, clean_addr, size, is_signed, false,
2810 false, 0, false, false);
2811 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << size);
2812 do_gpr_ld(s, tcg_rt2, clean_addr, size, is_signed, false,
2813 false, 0, false, false);
2814
2815 tcg_gen_mov_i64(tcg_rt, tmp);
2816 tcg_temp_free_i64(tmp);
2817 } else {
2818 do_gpr_st(s, tcg_rt, clean_addr, size,
2819 false, 0, false, false);
2820 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << size);
2821 do_gpr_st(s, tcg_rt2, clean_addr, size,
2822 false, 0, false, false);
2823 }
2824 }
2825
2826 if (wback) {
2827 if (postindex) {
2828 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
2829 }
2830 tcg_gen_mov_i64(cpu_reg_sp(s, rn), dirty_addr);
2831 }
2832 }
2833
2834 /*
2835 * Load/store (immediate post-indexed)
2836 * Load/store (immediate pre-indexed)
2837 * Load/store (unscaled immediate)
2838 *
2839 * 31 30 29 27 26 25 24 23 22 21 20 12 11 10 9 5 4 0
2840 * +----+-------+---+-----+-----+---+--------+-----+------+------+
2841 * |size| 1 1 1 | V | 0 0 | opc | 0 | imm9 | idx | Rn | Rt |
2842 * +----+-------+---+-----+-----+---+--------+-----+------+------+
2843 *
2844 * idx = 01 -> post-indexed, 11 pre-indexed, 00 unscaled imm. (no writeback)
2845 10 -> unprivileged
2846 * V = 0 -> non-vector
2847 * size: 00 -> 8 bit, 01 -> 16 bit, 10 -> 32 bit, 11 -> 64bit
2848 * opc: 00 -> store, 01 -> loadu, 10 -> loads 64, 11 -> loads 32
2849 */
2850 static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn,
2851 int opc,
2852 int size,
2853 int rt,
2854 bool is_vector)
2855 {
2856 int rn = extract32(insn, 5, 5);
2857 int imm9 = sextract32(insn, 12, 9);
2858 int idx = extract32(insn, 10, 2);
2859 bool is_signed = false;
2860 bool is_store = false;
2861 bool is_extended = false;
2862 bool is_unpriv = (idx == 2);
2863 bool iss_valid = !is_vector;
2864 bool post_index;
2865 bool writeback;
2866
2867 TCGv_i64 clean_addr, dirty_addr;
2868
2869 if (is_vector) {
2870 size |= (opc & 2) << 1;
2871 if (size > 4 || is_unpriv) {
2872 unallocated_encoding(s);
2873 return;
2874 }
2875 is_store = ((opc & 1) == 0);
2876 if (!fp_access_check(s)) {
2877 return;
2878 }
2879 } else {
2880 if (size == 3 && opc == 2) {
2881 /* PRFM - prefetch */
2882 if (idx != 0) {
2883 unallocated_encoding(s);
2884 return;
2885 }
2886 return;
2887 }
2888 if (opc == 3 && size > 1) {
2889 unallocated_encoding(s);
2890 return;
2891 }
2892 is_store = (opc == 0);
2893 is_signed = extract32(opc, 1, 1);
2894 is_extended = (size < 3) && extract32(opc, 0, 1);
2895 }
2896
2897 switch (idx) {
2898 case 0:
2899 case 2:
2900 post_index = false;
2901 writeback = false;
2902 break;
2903 case 1:
2904 post_index = true;
2905 writeback = true;
2906 break;
2907 case 3:
2908 post_index = false;
2909 writeback = true;
2910 break;
2911 default:
2912 g_assert_not_reached();
2913 }
2914
2915 if (rn == 31) {
2916 gen_check_sp_alignment(s);
2917 }
2918
2919 dirty_addr = read_cpu_reg_sp(s, rn, 1);
2920 if (!post_index) {
2921 tcg_gen_addi_i64(dirty_addr, dirty_addr, imm9);
2922 }
2923 clean_addr = clean_data_tbi(s, dirty_addr);
2924
2925 if (is_vector) {
2926 if (is_store) {
2927 do_fp_st(s, rt, clean_addr, size);
2928 } else {
2929 do_fp_ld(s, rt, clean_addr, size);
2930 }
2931 } else {
2932 TCGv_i64 tcg_rt = cpu_reg(s, rt);
2933 int memidx = is_unpriv ? get_a64_user_mem_index(s) : get_mem_index(s);
2934 bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc);
2935
2936 if (is_store) {
2937 do_gpr_st_memidx(s, tcg_rt, clean_addr, size, memidx,
2938 iss_valid, rt, iss_sf, false);
2939 } else {
2940 do_gpr_ld_memidx(s, tcg_rt, clean_addr, size,
2941 is_signed, is_extended, memidx,
2942 iss_valid, rt, iss_sf, false);
2943 }
2944 }
2945
2946 if (writeback) {
2947 TCGv_i64 tcg_rn = cpu_reg_sp(s, rn);
2948 if (post_index) {
2949 tcg_gen_addi_i64(dirty_addr, dirty_addr, imm9);
2950 }
2951 tcg_gen_mov_i64(tcg_rn, dirty_addr);
2952 }
2953 }
2954
2955 /*
2956 * Load/store (register offset)
2957 *
2958 * 31 30 29 27 26 25 24 23 22 21 20 16 15 13 12 11 10 9 5 4 0
2959 * +----+-------+---+-----+-----+---+------+-----+--+-----+----+----+
2960 * |size| 1 1 1 | V | 0 0 | opc | 1 | Rm | opt | S| 1 0 | Rn | Rt |
2961 * +----+-------+---+-----+-----+---+------+-----+--+-----+----+----+
2962 *
2963 * For non-vector:
2964 * size: 00-> byte, 01 -> 16 bit, 10 -> 32bit, 11 -> 64bit
2965 * opc: 00 -> store, 01 -> loadu, 10 -> loads 64, 11 -> loads 32
2966 * For vector:
2967 * size is opc<1>:size<1:0> so 100 -> 128 bit; 110 and 111 unallocated
2968 * opc<0>: 0 -> store, 1 -> load
2969 * V: 1 -> vector/simd
2970 * opt: extend encoding (see DecodeRegExtend)
2971 * S: if S=1 then scale (essentially index by sizeof(size))
2972 * Rt: register to transfer into/out of
2973 * Rn: address register or SP for base
2974 * Rm: offset register or ZR for offset
2975 */
2976 static void disas_ldst_reg_roffset(DisasContext *s, uint32_t insn,
2977 int opc,
2978 int size,
2979 int rt,
2980 bool is_vector)
2981 {
2982 int rn = extract32(insn, 5, 5);
2983 int shift = extract32(insn, 12, 1);
2984 int rm = extract32(insn, 16, 5);
2985 int opt = extract32(insn, 13, 3);
2986 bool is_signed = false;
2987 bool is_store = false;
2988 bool is_extended = false;
2989
2990 TCGv_i64 tcg_rm, clean_addr, dirty_addr;
2991
2992 if (extract32(opt, 1, 1) == 0) {
2993 unallocated_encoding(s);
2994 return;
2995 }
2996
2997 if (is_vector) {
2998 size |= (opc & 2) << 1;
2999 if (size > 4) {
3000 unallocated_encoding(s);
3001 return;
3002 }
3003 is_store = !extract32(opc, 0, 1);
3004 if (!fp_access_check(s)) {
3005 return;
3006 }
3007 } else {
3008 if (size == 3 && opc == 2) {
3009 /* PRFM - prefetch */
3010 return;
3011 }
3012 if (opc == 3 && size > 1) {
3013 unallocated_encoding(s);
3014 return;
3015 }
3016 is_store = (opc == 0);
3017 is_signed = extract32(opc, 1, 1);
3018 is_extended = (size < 3) && extract32(opc, 0, 1);
3019 }
3020
3021 if (rn == 31) {
3022 gen_check_sp_alignment(s);
3023 }
3024 dirty_addr = read_cpu_reg_sp(s, rn, 1);
3025
3026 tcg_rm = read_cpu_reg(s, rm, 1);
3027 ext_and_shift_reg(tcg_rm, tcg_rm, opt, shift ? size : 0);
3028
3029 tcg_gen_add_i64(dirty_addr, dirty_addr, tcg_rm);
3030 clean_addr = clean_data_tbi(s, dirty_addr);
3031
3032 if (is_vector) {
3033 if (is_store) {
3034 do_fp_st(s, rt, clean_addr, size);
3035 } else {
3036 do_fp_ld(s, rt, clean_addr, size);
3037 }
3038 } else {
3039 TCGv_i64 tcg_rt = cpu_reg(s, rt);
3040 bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc);
3041 if (is_store) {
3042 do_gpr_st(s, tcg_rt, clean_addr, size,
3043 true, rt, iss_sf, false);
3044 } else {
3045 do_gpr_ld(s, tcg_rt, clean_addr, size,
3046 is_signed, is_extended,
3047 true, rt, iss_sf, false);
3048 }
3049 }
3050 }
3051
3052 /*
3053 * Load/store (unsigned immediate)
3054 *
3055 * 31 30 29 27 26 25 24 23 22 21 10 9 5
3056 * +----+-------+---+-----+-----+------------+-------+------+
3057 * |size| 1 1 1 | V | 0 1 | opc | imm12 | Rn | Rt |
3058 * +----+-------+---+-----+-----+------------+-------+------+
3059 *
3060 * For non-vector:
3061 * size: 00-> byte, 01 -> 16 bit, 10 -> 32bit, 11 -> 64bit
3062 * opc: 00 -> store, 01 -> loadu, 10 -> loads 64, 11 -> loads 32
3063 * For vector:
3064 * size is opc<1>:size<1:0> so 100 -> 128 bit; 110 and 111 unallocated
3065 * opc<0>: 0 -> store, 1 -> load
3066 * Rn: base address register (inc SP)
3067 * Rt: target register
3068 */
3069 static void disas_ldst_reg_unsigned_imm(DisasContext *s, uint32_t insn,
3070 int opc,
3071 int size,
3072 int rt,
3073 bool is_vector)
3074 {
3075 int rn = extract32(insn, 5, 5);
3076 unsigned int imm12 = extract32(insn, 10, 12);
3077 unsigned int offset;
3078
3079 TCGv_i64 clean_addr, dirty_addr;
3080
3081 bool is_store;
3082 bool is_signed = false;
3083 bool is_extended = false;
3084
3085 if (is_vector) {
3086 size |= (opc & 2) << 1;
3087 if (size > 4) {
3088 unallocated_encoding(s);
3089 return;
3090 }
3091 is_store = !extract32(opc, 0, 1);
3092 if (!fp_access_check(s)) {
3093 return;
3094 }
3095 } else {
3096 if (size == 3 && opc == 2) {
3097 /* PRFM - prefetch */
3098 return;
3099 }
3100 if (opc == 3 && size > 1) {
3101 unallocated_encoding(s);
3102 return;
3103 }
3104 is_store = (opc == 0);
3105 is_signed = extract32(opc, 1, 1);
3106 is_extended = (size < 3) && extract32(opc, 0, 1);
3107 }
3108
3109 if (rn == 31) {
3110 gen_check_sp_alignment(s);
3111 }
3112 dirty_addr = read_cpu_reg_sp(s, rn, 1);
3113 offset = imm12 << size;
3114 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3115 clean_addr = clean_data_tbi(s, dirty_addr);
3116
3117 if (is_vector) {
3118 if (is_store) {
3119 do_fp_st(s, rt, clean_addr, size);
3120 } else {
3121 do_fp_ld(s, rt, clean_addr, size);
3122 }
3123 } else {
3124 TCGv_i64 tcg_rt = cpu_reg(s, rt);
3125 bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc);
3126 if (is_store) {
3127 do_gpr_st(s, tcg_rt, clean_addr, size,
3128 true, rt, iss_sf, false);
3129 } else {
3130 do_gpr_ld(s, tcg_rt, clean_addr, size, is_signed, is_extended,
3131 true, rt, iss_sf, false);
3132 }
3133 }
3134 }
3135
3136 /* Atomic memory operations
3137 *
3138 * 31 30 27 26 24 22 21 16 15 12 10 5 0
3139 * +------+-------+---+-----+-----+---+----+----+-----+-----+----+-----+
3140 * | size | 1 1 1 | V | 0 0 | A R | 1 | Rs | o3 | opc | 0 0 | Rn | Rt |
3141 * +------+-------+---+-----+-----+--------+----+-----+-----+----+-----+
3142 *
3143 * Rt: the result register
3144 * Rn: base address or SP
3145 * Rs: the source register for the operation
3146 * V: vector flag (always 0 as of v8.3)
3147 * A: acquire flag
3148 * R: release flag
3149 */
3150 static void disas_ldst_atomic(DisasContext *s, uint32_t insn,
3151 int size, int rt, bool is_vector)
3152 {
3153 int rs = extract32(insn, 16, 5);
3154 int rn = extract32(insn, 5, 5);
3155 int o3_opc = extract32(insn, 12, 4);
3156 bool r = extract32(insn, 22, 1);
3157 bool a = extract32(insn, 23, 1);
3158 TCGv_i64 tcg_rs, clean_addr;
3159 AtomicThreeOpFn *fn;
3160
3161 if (is_vector || !dc_isar_feature(aa64_atomics, s)) {
3162 unallocated_encoding(s);
3163 return;
3164 }
3165 switch (o3_opc) {
3166 case 000: /* LDADD */
3167 fn = tcg_gen_atomic_fetch_add_i64;
3168 break;
3169 case 001: /* LDCLR */
3170 fn = tcg_gen_atomic_fetch_and_i64;
3171 break;
3172 case 002: /* LDEOR */
3173 fn = tcg_gen_atomic_fetch_xor_i64;
3174 break;
3175 case 003: /* LDSET */
3176 fn = tcg_gen_atomic_fetch_or_i64;
3177 break;
3178 case 004: /* LDSMAX */
3179 fn = tcg_gen_atomic_fetch_smax_i64;
3180 break;
3181 case 005: /* LDSMIN */
3182 fn = tcg_gen_atomic_fetch_smin_i64;
3183 break;
3184 case 006: /* LDUMAX */
3185 fn = tcg_gen_atomic_fetch_umax_i64;
3186 break;
3187 case 007: /* LDUMIN */
3188 fn = tcg_gen_atomic_fetch_umin_i64;
3189 break;
3190 case 010: /* SWP */
3191 fn = tcg_gen_atomic_xchg_i64;
3192 break;
3193 case 014: /* LDAPR, LDAPRH, LDAPRB */
3194 if (!dc_isar_feature(aa64_rcpc_8_3, s) ||
3195 rs != 31 || a != 1 || r != 0) {
3196 unallocated_encoding(s);
3197 return;
3198