migration: increase max-bandwidth to 128 MiB/s (1 Gib/s)
[qemu.git] / target / arm / helper-a64.c
1 /*
2 * AArch64 specific helpers
3 *
4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "qemu/units.h"
22 #include "cpu.h"
23 #include "exec/gdbstub.h"
24 #include "exec/helper-proto.h"
25 #include "qemu/host-utils.h"
26 #include "qemu/log.h"
27 #include "qemu/main-loop.h"
28 #include "qemu/bitops.h"
29 #include "internals.h"
30 #include "qemu/crc32c.h"
31 #include "exec/exec-all.h"
32 #include "exec/cpu_ldst.h"
33 #include "qemu/int128.h"
34 #include "qemu/atomic128.h"
35 #include "tcg/tcg.h"
36 #include "fpu/softfloat.h"
37 #include <zlib.h> /* For crc32 */
38
39 /* C2.4.7 Multiply and divide */
40 /* special cases for 0 and LLONG_MIN are mandated by the standard */
41 uint64_t HELPER(udiv64)(uint64_t num, uint64_t den)
42 {
43 if (den == 0) {
44 return 0;
45 }
46 return num / den;
47 }
48
49 int64_t HELPER(sdiv64)(int64_t num, int64_t den)
50 {
51 if (den == 0) {
52 return 0;
53 }
54 if (num == LLONG_MIN && den == -1) {
55 return LLONG_MIN;
56 }
57 return num / den;
58 }
59
60 uint64_t HELPER(rbit64)(uint64_t x)
61 {
62 return revbit64(x);
63 }
64
65 void HELPER(msr_i_spsel)(CPUARMState *env, uint32_t imm)
66 {
67 update_spsel(env, imm);
68 }
69
70 static void daif_check(CPUARMState *env, uint32_t op,
71 uint32_t imm, uintptr_t ra)
72 {
73 /* DAIF update to PSTATE. This is OK from EL0 only if UMA is set. */
74 if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) {
75 raise_exception_ra(env, EXCP_UDEF,
76 syn_aa64_sysregtrap(0, extract32(op, 0, 3),
77 extract32(op, 3, 3), 4,
78 imm, 0x1f, 0),
79 exception_target_el(env), ra);
80 }
81 }
82
83 void HELPER(msr_i_daifset)(CPUARMState *env, uint32_t imm)
84 {
85 daif_check(env, 0x1e, imm, GETPC());
86 env->daif |= (imm << 6) & PSTATE_DAIF;
87 }
88
89 void HELPER(msr_i_daifclear)(CPUARMState *env, uint32_t imm)
90 {
91 daif_check(env, 0x1f, imm, GETPC());
92 env->daif &= ~((imm << 6) & PSTATE_DAIF);
93 }
94
95 /* Convert a softfloat float_relation_ (as returned by
96 * the float*_compare functions) to the correct ARM
97 * NZCV flag state.
98 */
99 static inline uint32_t float_rel_to_flags(int res)
100 {
101 uint64_t flags;
102 switch (res) {
103 case float_relation_equal:
104 flags = PSTATE_Z | PSTATE_C;
105 break;
106 case float_relation_less:
107 flags = PSTATE_N;
108 break;
109 case float_relation_greater:
110 flags = PSTATE_C;
111 break;
112 case float_relation_unordered:
113 default:
114 flags = PSTATE_C | PSTATE_V;
115 break;
116 }
117 return flags;
118 }
119
120 uint64_t HELPER(vfp_cmph_a64)(uint32_t x, uint32_t y, void *fp_status)
121 {
122 return float_rel_to_flags(float16_compare_quiet(x, y, fp_status));
123 }
124
125 uint64_t HELPER(vfp_cmpeh_a64)(uint32_t x, uint32_t y, void *fp_status)
126 {
127 return float_rel_to_flags(float16_compare(x, y, fp_status));
128 }
129
130 uint64_t HELPER(vfp_cmps_a64)(float32 x, float32 y, void *fp_status)
131 {
132 return float_rel_to_flags(float32_compare_quiet(x, y, fp_status));
133 }
134
135 uint64_t HELPER(vfp_cmpes_a64)(float32 x, float32 y, void *fp_status)
136 {
137 return float_rel_to_flags(float32_compare(x, y, fp_status));
138 }
139
140 uint64_t HELPER(vfp_cmpd_a64)(float64 x, float64 y, void *fp_status)
141 {
142 return float_rel_to_flags(float64_compare_quiet(x, y, fp_status));
143 }
144
145 uint64_t HELPER(vfp_cmped_a64)(float64 x, float64 y, void *fp_status)
146 {
147 return float_rel_to_flags(float64_compare(x, y, fp_status));
148 }
149
150 float32 HELPER(vfp_mulxs)(float32 a, float32 b, void *fpstp)
151 {
152 float_status *fpst = fpstp;
153
154 a = float32_squash_input_denormal(a, fpst);
155 b = float32_squash_input_denormal(b, fpst);
156
157 if ((float32_is_zero(a) && float32_is_infinity(b)) ||
158 (float32_is_infinity(a) && float32_is_zero(b))) {
159 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
160 return make_float32((1U << 30) |
161 ((float32_val(a) ^ float32_val(b)) & (1U << 31)));
162 }
163 return float32_mul(a, b, fpst);
164 }
165
166 float64 HELPER(vfp_mulxd)(float64 a, float64 b, void *fpstp)
167 {
168 float_status *fpst = fpstp;
169
170 a = float64_squash_input_denormal(a, fpst);
171 b = float64_squash_input_denormal(b, fpst);
172
173 if ((float64_is_zero(a) && float64_is_infinity(b)) ||
174 (float64_is_infinity(a) && float64_is_zero(b))) {
175 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
176 return make_float64((1ULL << 62) |
177 ((float64_val(a) ^ float64_val(b)) & (1ULL << 63)));
178 }
179 return float64_mul(a, b, fpst);
180 }
181
182 uint64_t HELPER(simd_tbl)(CPUARMState *env, uint64_t result, uint64_t indices,
183 uint32_t rn, uint32_t numregs)
184 {
185 /* Helper function for SIMD TBL and TBX. We have to do the table
186 * lookup part for the 64 bits worth of indices we're passed in.
187 * result is the initial results vector (either zeroes for TBL
188 * or some guest values for TBX), rn the register number where
189 * the table starts, and numregs the number of registers in the table.
190 * We return the results of the lookups.
191 */
192 int shift;
193
194 for (shift = 0; shift < 64; shift += 8) {
195 int index = extract64(indices, shift, 8);
196 if (index < 16 * numregs) {
197 /* Convert index (a byte offset into the virtual table
198 * which is a series of 128-bit vectors concatenated)
199 * into the correct register element plus a bit offset
200 * into that element, bearing in mind that the table
201 * can wrap around from V31 to V0.
202 */
203 int elt = (rn * 2 + (index >> 3)) % 64;
204 int bitidx = (index & 7) * 8;
205 uint64_t *q = aa64_vfp_qreg(env, elt >> 1);
206 uint64_t val = extract64(q[elt & 1], bitidx, 8);
207
208 result = deposit64(result, shift, 8, val);
209 }
210 }
211 return result;
212 }
213
214 /* 64bit/double versions of the neon float compare functions */
215 uint64_t HELPER(neon_ceq_f64)(float64 a, float64 b, void *fpstp)
216 {
217 float_status *fpst = fpstp;
218 return -float64_eq_quiet(a, b, fpst);
219 }
220
221 uint64_t HELPER(neon_cge_f64)(float64 a, float64 b, void *fpstp)
222 {
223 float_status *fpst = fpstp;
224 return -float64_le(b, a, fpst);
225 }
226
227 uint64_t HELPER(neon_cgt_f64)(float64 a, float64 b, void *fpstp)
228 {
229 float_status *fpst = fpstp;
230 return -float64_lt(b, a, fpst);
231 }
232
233 /* Reciprocal step and sqrt step. Note that unlike the A32/T32
234 * versions, these do a fully fused multiply-add or
235 * multiply-add-and-halve.
236 */
237
238 uint32_t HELPER(recpsf_f16)(uint32_t a, uint32_t b, void *fpstp)
239 {
240 float_status *fpst = fpstp;
241
242 a = float16_squash_input_denormal(a, fpst);
243 b = float16_squash_input_denormal(b, fpst);
244
245 a = float16_chs(a);
246 if ((float16_is_infinity(a) && float16_is_zero(b)) ||
247 (float16_is_infinity(b) && float16_is_zero(a))) {
248 return float16_two;
249 }
250 return float16_muladd(a, b, float16_two, 0, fpst);
251 }
252
253 float32 HELPER(recpsf_f32)(float32 a, float32 b, void *fpstp)
254 {
255 float_status *fpst = fpstp;
256
257 a = float32_squash_input_denormal(a, fpst);
258 b = float32_squash_input_denormal(b, fpst);
259
260 a = float32_chs(a);
261 if ((float32_is_infinity(a) && float32_is_zero(b)) ||
262 (float32_is_infinity(b) && float32_is_zero(a))) {
263 return float32_two;
264 }
265 return float32_muladd(a, b, float32_two, 0, fpst);
266 }
267
268 float64 HELPER(recpsf_f64)(float64 a, float64 b, void *fpstp)
269 {
270 float_status *fpst = fpstp;
271
272 a = float64_squash_input_denormal(a, fpst);
273 b = float64_squash_input_denormal(b, fpst);
274
275 a = float64_chs(a);
276 if ((float64_is_infinity(a) && float64_is_zero(b)) ||
277 (float64_is_infinity(b) && float64_is_zero(a))) {
278 return float64_two;
279 }
280 return float64_muladd(a, b, float64_two, 0, fpst);
281 }
282
283 uint32_t HELPER(rsqrtsf_f16)(uint32_t a, uint32_t b, void *fpstp)
284 {
285 float_status *fpst = fpstp;
286
287 a = float16_squash_input_denormal(a, fpst);
288 b = float16_squash_input_denormal(b, fpst);
289
290 a = float16_chs(a);
291 if ((float16_is_infinity(a) && float16_is_zero(b)) ||
292 (float16_is_infinity(b) && float16_is_zero(a))) {
293 return float16_one_point_five;
294 }
295 return float16_muladd(a, b, float16_three, float_muladd_halve_result, fpst);
296 }
297
298 float32 HELPER(rsqrtsf_f32)(float32 a, float32 b, void *fpstp)
299 {
300 float_status *fpst = fpstp;
301
302 a = float32_squash_input_denormal(a, fpst);
303 b = float32_squash_input_denormal(b, fpst);
304
305 a = float32_chs(a);
306 if ((float32_is_infinity(a) && float32_is_zero(b)) ||
307 (float32_is_infinity(b) && float32_is_zero(a))) {
308 return float32_one_point_five;
309 }
310 return float32_muladd(a, b, float32_three, float_muladd_halve_result, fpst);
311 }
312
313 float64 HELPER(rsqrtsf_f64)(float64 a, float64 b, void *fpstp)
314 {
315 float_status *fpst = fpstp;
316
317 a = float64_squash_input_denormal(a, fpst);
318 b = float64_squash_input_denormal(b, fpst);
319
320 a = float64_chs(a);
321 if ((float64_is_infinity(a) && float64_is_zero(b)) ||
322 (float64_is_infinity(b) && float64_is_zero(a))) {
323 return float64_one_point_five;
324 }
325 return float64_muladd(a, b, float64_three, float_muladd_halve_result, fpst);
326 }
327
328 /* Pairwise long add: add pairs of adjacent elements into
329 * double-width elements in the result (eg _s8 is an 8x8->16 op)
330 */
331 uint64_t HELPER(neon_addlp_s8)(uint64_t a)
332 {
333 uint64_t nsignmask = 0x0080008000800080ULL;
334 uint64_t wsignmask = 0x8000800080008000ULL;
335 uint64_t elementmask = 0x00ff00ff00ff00ffULL;
336 uint64_t tmp1, tmp2;
337 uint64_t res, signres;
338
339 /* Extract odd elements, sign extend each to a 16 bit field */
340 tmp1 = a & elementmask;
341 tmp1 ^= nsignmask;
342 tmp1 |= wsignmask;
343 tmp1 = (tmp1 - nsignmask) ^ wsignmask;
344 /* Ditto for the even elements */
345 tmp2 = (a >> 8) & elementmask;
346 tmp2 ^= nsignmask;
347 tmp2 |= wsignmask;
348 tmp2 = (tmp2 - nsignmask) ^ wsignmask;
349
350 /* calculate the result by summing bits 0..14, 16..22, etc,
351 * and then adjusting the sign bits 15, 23, etc manually.
352 * This ensures the addition can't overflow the 16 bit field.
353 */
354 signres = (tmp1 ^ tmp2) & wsignmask;
355 res = (tmp1 & ~wsignmask) + (tmp2 & ~wsignmask);
356 res ^= signres;
357
358 return res;
359 }
360
361 uint64_t HELPER(neon_addlp_u8)(uint64_t a)
362 {
363 uint64_t tmp;
364
365 tmp = a & 0x00ff00ff00ff00ffULL;
366 tmp += (a >> 8) & 0x00ff00ff00ff00ffULL;
367 return tmp;
368 }
369
370 uint64_t HELPER(neon_addlp_s16)(uint64_t a)
371 {
372 int32_t reslo, reshi;
373
374 reslo = (int32_t)(int16_t)a + (int32_t)(int16_t)(a >> 16);
375 reshi = (int32_t)(int16_t)(a >> 32) + (int32_t)(int16_t)(a >> 48);
376
377 return (uint32_t)reslo | (((uint64_t)reshi) << 32);
378 }
379
380 uint64_t HELPER(neon_addlp_u16)(uint64_t a)
381 {
382 uint64_t tmp;
383
384 tmp = a & 0x0000ffff0000ffffULL;
385 tmp += (a >> 16) & 0x0000ffff0000ffffULL;
386 return tmp;
387 }
388
389 /* Floating-point reciprocal exponent - see FPRecpX in ARM ARM */
390 uint32_t HELPER(frecpx_f16)(uint32_t a, void *fpstp)
391 {
392 float_status *fpst = fpstp;
393 uint16_t val16, sbit;
394 int16_t exp;
395
396 if (float16_is_any_nan(a)) {
397 float16 nan = a;
398 if (float16_is_signaling_nan(a, fpst)) {
399 float_raise(float_flag_invalid, fpst);
400 nan = float16_silence_nan(a, fpst);
401 }
402 if (fpst->default_nan_mode) {
403 nan = float16_default_nan(fpst);
404 }
405 return nan;
406 }
407
408 a = float16_squash_input_denormal(a, fpst);
409
410 val16 = float16_val(a);
411 sbit = 0x8000 & val16;
412 exp = extract32(val16, 10, 5);
413
414 if (exp == 0) {
415 return make_float16(deposit32(sbit, 10, 5, 0x1e));
416 } else {
417 return make_float16(deposit32(sbit, 10, 5, ~exp));
418 }
419 }
420
421 float32 HELPER(frecpx_f32)(float32 a, void *fpstp)
422 {
423 float_status *fpst = fpstp;
424 uint32_t val32, sbit;
425 int32_t exp;
426
427 if (float32_is_any_nan(a)) {
428 float32 nan = a;
429 if (float32_is_signaling_nan(a, fpst)) {
430 float_raise(float_flag_invalid, fpst);
431 nan = float32_silence_nan(a, fpst);
432 }
433 if (fpst->default_nan_mode) {
434 nan = float32_default_nan(fpst);
435 }
436 return nan;
437 }
438
439 a = float32_squash_input_denormal(a, fpst);
440
441 val32 = float32_val(a);
442 sbit = 0x80000000ULL & val32;
443 exp = extract32(val32, 23, 8);
444
445 if (exp == 0) {
446 return make_float32(sbit | (0xfe << 23));
447 } else {
448 return make_float32(sbit | (~exp & 0xff) << 23);
449 }
450 }
451
452 float64 HELPER(frecpx_f64)(float64 a, void *fpstp)
453 {
454 float_status *fpst = fpstp;
455 uint64_t val64, sbit;
456 int64_t exp;
457
458 if (float64_is_any_nan(a)) {
459 float64 nan = a;
460 if (float64_is_signaling_nan(a, fpst)) {
461 float_raise(float_flag_invalid, fpst);
462 nan = float64_silence_nan(a, fpst);
463 }
464 if (fpst->default_nan_mode) {
465 nan = float64_default_nan(fpst);
466 }
467 return nan;
468 }
469
470 a = float64_squash_input_denormal(a, fpst);
471
472 val64 = float64_val(a);
473 sbit = 0x8000000000000000ULL & val64;
474 exp = extract64(float64_val(a), 52, 11);
475
476 if (exp == 0) {
477 return make_float64(sbit | (0x7feULL << 52));
478 } else {
479 return make_float64(sbit | (~exp & 0x7ffULL) << 52);
480 }
481 }
482
483 float32 HELPER(fcvtx_f64_to_f32)(float64 a, CPUARMState *env)
484 {
485 /* Von Neumann rounding is implemented by using round-to-zero
486 * and then setting the LSB of the result if Inexact was raised.
487 */
488 float32 r;
489 float_status *fpst = &env->vfp.fp_status;
490 float_status tstat = *fpst;
491 int exflags;
492
493 set_float_rounding_mode(float_round_to_zero, &tstat);
494 set_float_exception_flags(0, &tstat);
495 r = float64_to_float32(a, &tstat);
496 exflags = get_float_exception_flags(&tstat);
497 if (exflags & float_flag_inexact) {
498 r = make_float32(float32_val(r) | 1);
499 }
500 exflags |= get_float_exception_flags(fpst);
501 set_float_exception_flags(exflags, fpst);
502 return r;
503 }
504
505 /* 64-bit versions of the CRC helpers. Note that although the operation
506 * (and the prototypes of crc32c() and crc32() mean that only the bottom
507 * 32 bits of the accumulator and result are used, we pass and return
508 * uint64_t for convenience of the generated code. Unlike the 32-bit
509 * instruction set versions, val may genuinely have 64 bits of data in it.
510 * The upper bytes of val (above the number specified by 'bytes') must have
511 * been zeroed out by the caller.
512 */
513 uint64_t HELPER(crc32_64)(uint64_t acc, uint64_t val, uint32_t bytes)
514 {
515 uint8_t buf[8];
516
517 stq_le_p(buf, val);
518
519 /* zlib crc32 converts the accumulator and output to one's complement. */
520 return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
521 }
522
523 uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
524 {
525 uint8_t buf[8];
526
527 stq_le_p(buf, val);
528
529 /* Linux crc32c converts the output to one's complement. */
530 return crc32c(acc, buf, bytes) ^ 0xffffffff;
531 }
532
533 uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
534 uint64_t new_lo, uint64_t new_hi)
535 {
536 Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
537 Int128 newv = int128_make128(new_lo, new_hi);
538 Int128 oldv;
539 uintptr_t ra = GETPC();
540 uint64_t o0, o1;
541 bool success;
542
543 #ifdef CONFIG_USER_ONLY
544 /* ??? Enforce alignment. */
545 uint64_t *haddr = g2h(addr);
546
547 set_helper_retaddr(ra);
548 o0 = ldq_le_p(haddr + 0);
549 o1 = ldq_le_p(haddr + 1);
550 oldv = int128_make128(o0, o1);
551
552 success = int128_eq(oldv, cmpv);
553 if (success) {
554 stq_le_p(haddr + 0, int128_getlo(newv));
555 stq_le_p(haddr + 1, int128_gethi(newv));
556 }
557 clear_helper_retaddr();
558 #else
559 int mem_idx = cpu_mmu_index(env, false);
560 TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
561 TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx);
562
563 o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra);
564 o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra);
565 oldv = int128_make128(o0, o1);
566
567 success = int128_eq(oldv, cmpv);
568 if (success) {
569 helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
570 helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
571 }
572 #endif
573
574 return !success;
575 }
576
577 uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
578 uint64_t new_lo, uint64_t new_hi)
579 {
580 Int128 oldv, cmpv, newv;
581 uintptr_t ra = GETPC();
582 bool success;
583 int mem_idx;
584 TCGMemOpIdx oi;
585
586 assert(HAVE_CMPXCHG128);
587
588 mem_idx = cpu_mmu_index(env, false);
589 oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
590
591 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
592 newv = int128_make128(new_lo, new_hi);
593 oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
594
595 success = int128_eq(oldv, cmpv);
596 return !success;
597 }
598
599 uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr,
600 uint64_t new_lo, uint64_t new_hi)
601 {
602 /*
603 * High and low need to be switched here because this is not actually a
604 * 128bit store but two doublewords stored consecutively
605 */
606 Int128 cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
607 Int128 newv = int128_make128(new_hi, new_lo);
608 Int128 oldv;
609 uintptr_t ra = GETPC();
610 uint64_t o0, o1;
611 bool success;
612
613 #ifdef CONFIG_USER_ONLY
614 /* ??? Enforce alignment. */
615 uint64_t *haddr = g2h(addr);
616
617 set_helper_retaddr(ra);
618 o1 = ldq_be_p(haddr + 0);
619 o0 = ldq_be_p(haddr + 1);
620 oldv = int128_make128(o0, o1);
621
622 success = int128_eq(oldv, cmpv);
623 if (success) {
624 stq_be_p(haddr + 0, int128_gethi(newv));
625 stq_be_p(haddr + 1, int128_getlo(newv));
626 }
627 clear_helper_retaddr();
628 #else
629 int mem_idx = cpu_mmu_index(env, false);
630 TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
631 TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx);
632
633 o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra);
634 o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra);
635 oldv = int128_make128(o0, o1);
636
637 success = int128_eq(oldv, cmpv);
638 if (success) {
639 helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
640 helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
641 }
642 #endif
643
644 return !success;
645 }
646
647 uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
648 uint64_t new_lo, uint64_t new_hi)
649 {
650 Int128 oldv, cmpv, newv;
651 uintptr_t ra = GETPC();
652 bool success;
653 int mem_idx;
654 TCGMemOpIdx oi;
655
656 assert(HAVE_CMPXCHG128);
657
658 mem_idx = cpu_mmu_index(env, false);
659 oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
660
661 /*
662 * High and low need to be switched here because this is not actually a
663 * 128bit store but two doublewords stored consecutively
664 */
665 cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
666 newv = int128_make128(new_hi, new_lo);
667 oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
668
669 success = int128_eq(oldv, cmpv);
670 return !success;
671 }
672
673 /* Writes back the old data into Rs. */
674 void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
675 uint64_t new_lo, uint64_t new_hi)
676 {
677 Int128 oldv, cmpv, newv;
678 uintptr_t ra = GETPC();
679 int mem_idx;
680 TCGMemOpIdx oi;
681
682 assert(HAVE_CMPXCHG128);
683
684 mem_idx = cpu_mmu_index(env, false);
685 oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
686
687 cmpv = int128_make128(env->xregs[rs], env->xregs[rs + 1]);
688 newv = int128_make128(new_lo, new_hi);
689 oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
690
691 env->xregs[rs] = int128_getlo(oldv);
692 env->xregs[rs + 1] = int128_gethi(oldv);
693 }
694
695 void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
696 uint64_t new_hi, uint64_t new_lo)
697 {
698 Int128 oldv, cmpv, newv;
699 uintptr_t ra = GETPC();
700 int mem_idx;
701 TCGMemOpIdx oi;
702
703 assert(HAVE_CMPXCHG128);
704
705 mem_idx = cpu_mmu_index(env, false);
706 oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
707
708 cmpv = int128_make128(env->xregs[rs + 1], env->xregs[rs]);
709 newv = int128_make128(new_lo, new_hi);
710 oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
711
712 env->xregs[rs + 1] = int128_getlo(oldv);
713 env->xregs[rs] = int128_gethi(oldv);
714 }
715
716 /*
717 * AdvSIMD half-precision
718 */
719
720 #define ADVSIMD_HELPER(name, suffix) HELPER(glue(glue(advsimd_, name), suffix))
721
722 #define ADVSIMD_HALFOP(name) \
723 uint32_t ADVSIMD_HELPER(name, h)(uint32_t a, uint32_t b, void *fpstp) \
724 { \
725 float_status *fpst = fpstp; \
726 return float16_ ## name(a, b, fpst); \
727 }
728
729 ADVSIMD_HALFOP(add)
730 ADVSIMD_HALFOP(sub)
731 ADVSIMD_HALFOP(mul)
732 ADVSIMD_HALFOP(div)
733 ADVSIMD_HALFOP(min)
734 ADVSIMD_HALFOP(max)
735 ADVSIMD_HALFOP(minnum)
736 ADVSIMD_HALFOP(maxnum)
737
738 #define ADVSIMD_TWOHALFOP(name) \
739 uint32_t ADVSIMD_HELPER(name, 2h)(uint32_t two_a, uint32_t two_b, void *fpstp) \
740 { \
741 float16 a1, a2, b1, b2; \
742 uint32_t r1, r2; \
743 float_status *fpst = fpstp; \
744 a1 = extract32(two_a, 0, 16); \
745 a2 = extract32(two_a, 16, 16); \
746 b1 = extract32(two_b, 0, 16); \
747 b2 = extract32(two_b, 16, 16); \
748 r1 = float16_ ## name(a1, b1, fpst); \
749 r2 = float16_ ## name(a2, b2, fpst); \
750 return deposit32(r1, 16, 16, r2); \
751 }
752
753 ADVSIMD_TWOHALFOP(add)
754 ADVSIMD_TWOHALFOP(sub)
755 ADVSIMD_TWOHALFOP(mul)
756 ADVSIMD_TWOHALFOP(div)
757 ADVSIMD_TWOHALFOP(min)
758 ADVSIMD_TWOHALFOP(max)
759 ADVSIMD_TWOHALFOP(minnum)
760 ADVSIMD_TWOHALFOP(maxnum)
761
762 /* Data processing - scalar floating-point and advanced SIMD */
763 static float16 float16_mulx(float16 a, float16 b, void *fpstp)
764 {
765 float_status *fpst = fpstp;
766
767 a = float16_squash_input_denormal(a, fpst);
768 b = float16_squash_input_denormal(b, fpst);
769
770 if ((float16_is_zero(a) && float16_is_infinity(b)) ||
771 (float16_is_infinity(a) && float16_is_zero(b))) {
772 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
773 return make_float16((1U << 14) |
774 ((float16_val(a) ^ float16_val(b)) & (1U << 15)));
775 }
776 return float16_mul(a, b, fpst);
777 }
778
779 ADVSIMD_HALFOP(mulx)
780 ADVSIMD_TWOHALFOP(mulx)
781
782 /* fused multiply-accumulate */
783 uint32_t HELPER(advsimd_muladdh)(uint32_t a, uint32_t b, uint32_t c,
784 void *fpstp)
785 {
786 float_status *fpst = fpstp;
787 return float16_muladd(a, b, c, 0, fpst);
788 }
789
790 uint32_t HELPER(advsimd_muladd2h)(uint32_t two_a, uint32_t two_b,
791 uint32_t two_c, void *fpstp)
792 {
793 float_status *fpst = fpstp;
794 float16 a1, a2, b1, b2, c1, c2;
795 uint32_t r1, r2;
796 a1 = extract32(two_a, 0, 16);
797 a2 = extract32(two_a, 16, 16);
798 b1 = extract32(two_b, 0, 16);
799 b2 = extract32(two_b, 16, 16);
800 c1 = extract32(two_c, 0, 16);
801 c2 = extract32(two_c, 16, 16);
802 r1 = float16_muladd(a1, b1, c1, 0, fpst);
803 r2 = float16_muladd(a2, b2, c2, 0, fpst);
804 return deposit32(r1, 16, 16, r2);
805 }
806
807 /*
808 * Floating point comparisons produce an integer result. Softfloat
809 * routines return float_relation types which we convert to the 0/-1
810 * Neon requires.
811 */
812
813 #define ADVSIMD_CMPRES(test) (test) ? 0xffff : 0
814
815 uint32_t HELPER(advsimd_ceq_f16)(uint32_t a, uint32_t b, void *fpstp)
816 {
817 float_status *fpst = fpstp;
818 int compare = float16_compare_quiet(a, b, fpst);
819 return ADVSIMD_CMPRES(compare == float_relation_equal);
820 }
821
822 uint32_t HELPER(advsimd_cge_f16)(uint32_t a, uint32_t b, void *fpstp)
823 {
824 float_status *fpst = fpstp;
825 int compare = float16_compare(a, b, fpst);
826 return ADVSIMD_CMPRES(compare == float_relation_greater ||
827 compare == float_relation_equal);
828 }
829
830 uint32_t HELPER(advsimd_cgt_f16)(uint32_t a, uint32_t b, void *fpstp)
831 {
832 float_status *fpst = fpstp;
833 int compare = float16_compare(a, b, fpst);
834 return ADVSIMD_CMPRES(compare == float_relation_greater);
835 }
836
837 uint32_t HELPER(advsimd_acge_f16)(uint32_t a, uint32_t b, void *fpstp)
838 {
839 float_status *fpst = fpstp;
840 float16 f0 = float16_abs(a);
841 float16 f1 = float16_abs(b);
842 int compare = float16_compare(f0, f1, fpst);
843 return ADVSIMD_CMPRES(compare == float_relation_greater ||
844 compare == float_relation_equal);
845 }
846
847 uint32_t HELPER(advsimd_acgt_f16)(uint32_t a, uint32_t b, void *fpstp)
848 {
849 float_status *fpst = fpstp;
850 float16 f0 = float16_abs(a);
851 float16 f1 = float16_abs(b);
852 int compare = float16_compare(f0, f1, fpst);
853 return ADVSIMD_CMPRES(compare == float_relation_greater);
854 }
855
856 /* round to integral */
857 uint32_t HELPER(advsimd_rinth_exact)(uint32_t x, void *fp_status)
858 {
859 return float16_round_to_int(x, fp_status);
860 }
861
862 uint32_t HELPER(advsimd_rinth)(uint32_t x, void *fp_status)
863 {
864 int old_flags = get_float_exception_flags(fp_status), new_flags;
865 float16 ret;
866
867 ret = float16_round_to_int(x, fp_status);
868
869 /* Suppress any inexact exceptions the conversion produced */
870 if (!(old_flags & float_flag_inexact)) {
871 new_flags = get_float_exception_flags(fp_status);
872 set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
873 }
874
875 return ret;
876 }
877
878 /*
879 * Half-precision floating point conversion functions
880 *
881 * There are a multitude of conversion functions with various
882 * different rounding modes. This is dealt with by the calling code
883 * setting the mode appropriately before calling the helper.
884 */
885
886 uint32_t HELPER(advsimd_f16tosinth)(uint32_t a, void *fpstp)
887 {
888 float_status *fpst = fpstp;
889
890 /* Invalid if we are passed a NaN */
891 if (float16_is_any_nan(a)) {
892 float_raise(float_flag_invalid, fpst);
893 return 0;
894 }
895 return float16_to_int16(a, fpst);
896 }
897
898 uint32_t HELPER(advsimd_f16touinth)(uint32_t a, void *fpstp)
899 {
900 float_status *fpst = fpstp;
901
902 /* Invalid if we are passed a NaN */
903 if (float16_is_any_nan(a)) {
904 float_raise(float_flag_invalid, fpst);
905 return 0;
906 }
907 return float16_to_uint16(a, fpst);
908 }
909
910 static int el_from_spsr(uint32_t spsr)
911 {
912 /* Return the exception level that this SPSR is requesting a return to,
913 * or -1 if it is invalid (an illegal return)
914 */
915 if (spsr & PSTATE_nRW) {
916 switch (spsr & CPSR_M) {
917 case ARM_CPU_MODE_USR:
918 return 0;
919 case ARM_CPU_MODE_HYP:
920 return 2;
921 case ARM_CPU_MODE_FIQ:
922 case ARM_CPU_MODE_IRQ:
923 case ARM_CPU_MODE_SVC:
924 case ARM_CPU_MODE_ABT:
925 case ARM_CPU_MODE_UND:
926 case ARM_CPU_MODE_SYS:
927 return 1;
928 case ARM_CPU_MODE_MON:
929 /* Returning to Mon from AArch64 is never possible,
930 * so this is an illegal return.
931 */
932 default:
933 return -1;
934 }
935 } else {
936 if (extract32(spsr, 1, 1)) {
937 /* Return with reserved M[1] bit set */
938 return -1;
939 }
940 if (extract32(spsr, 0, 4) == 1) {
941 /* return to EL0 with M[0] bit set */
942 return -1;
943 }
944 return extract32(spsr, 2, 2);
945 }
946 }
947
948 void HELPER(exception_return)(CPUARMState *env, uint64_t new_pc)
949 {
950 int cur_el = arm_current_el(env);
951 unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
952 uint32_t mask, spsr = env->banked_spsr[spsr_idx];
953 int new_el;
954 bool return_to_aa64 = (spsr & PSTATE_nRW) == 0;
955
956 aarch64_save_sp(env, cur_el);
957
958 arm_clear_exclusive(env);
959
960 /* We must squash the PSTATE.SS bit to zero unless both of the
961 * following hold:
962 * 1. debug exceptions are currently disabled
963 * 2. singlestep will be active in the EL we return to
964 * We check 1 here and 2 after we've done the pstate/cpsr write() to
965 * transition to the EL we're going to.
966 */
967 if (arm_generate_debug_exceptions(env)) {
968 spsr &= ~PSTATE_SS;
969 }
970
971 new_el = el_from_spsr(spsr);
972 if (new_el == -1) {
973 goto illegal_return;
974 }
975 if (new_el > cur_el
976 || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
977 /* Disallow return to an EL which is unimplemented or higher
978 * than the current one.
979 */
980 goto illegal_return;
981 }
982
983 if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) {
984 /* Return to an EL which is configured for a different register width */
985 goto illegal_return;
986 }
987
988 if (new_el == 2 && arm_is_secure_below_el3(env)) {
989 /* Return to the non-existent secure-EL2 */
990 goto illegal_return;
991 }
992
993 if (new_el == 1 && (arm_hcr_el2_eff(env) & HCR_TGE)) {
994 goto illegal_return;
995 }
996
997 qemu_mutex_lock_iothread();
998 arm_call_pre_el_change_hook(env_archcpu(env));
999 qemu_mutex_unlock_iothread();
1000
1001 if (!return_to_aa64) {
1002 env->aarch64 = 0;
1003 /* We do a raw CPSR write because aarch64_sync_64_to_32()
1004 * will sort the register banks out for us, and we've already
1005 * caught all the bad-mode cases in el_from_spsr().
1006 */
1007 mask = aarch32_cpsr_valid_mask(env->features, &env_archcpu(env)->isar);
1008 cpsr_write(env, spsr, mask, CPSRWriteRaw);
1009 if (!arm_singlestep_active(env)) {
1010 env->uncached_cpsr &= ~PSTATE_SS;
1011 }
1012 aarch64_sync_64_to_32(env);
1013
1014 if (spsr & CPSR_T) {
1015 env->regs[15] = new_pc & ~0x1;
1016 } else {
1017 env->regs[15] = new_pc & ~0x3;
1018 }
1019 helper_rebuild_hflags_a32(env, new_el);
1020 qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to "
1021 "AArch32 EL%d PC 0x%" PRIx32 "\n",
1022 cur_el, new_el, env->regs[15]);
1023 } else {
1024 int tbii;
1025
1026 env->aarch64 = 1;
1027 spsr &= aarch64_pstate_valid_mask(&env_archcpu(env)->isar);
1028 pstate_write(env, spsr);
1029 if (!arm_singlestep_active(env)) {
1030 env->pstate &= ~PSTATE_SS;
1031 }
1032 aarch64_restore_sp(env, new_el);
1033 helper_rebuild_hflags_a64(env, new_el);
1034
1035 /*
1036 * Apply TBI to the exception return address. We had to delay this
1037 * until after we selected the new EL, so that we could select the
1038 * correct TBI+TBID bits. This is made easier by waiting until after
1039 * the hflags rebuild, since we can pull the composite TBII field
1040 * from there.
1041 */
1042 tbii = FIELD_EX32(env->hflags, TBFLAG_A64, TBII);
1043 if ((tbii >> extract64(new_pc, 55, 1)) & 1) {
1044 /* TBI is enabled. */
1045 int core_mmu_idx = cpu_mmu_index(env, false);
1046 if (regime_has_2_ranges(core_to_aa64_mmu_idx(core_mmu_idx))) {
1047 new_pc = sextract64(new_pc, 0, 56);
1048 } else {
1049 new_pc = extract64(new_pc, 0, 56);
1050 }
1051 }
1052 env->pc = new_pc;
1053
1054 qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to "
1055 "AArch64 EL%d PC 0x%" PRIx64 "\n",
1056 cur_el, new_el, env->pc);
1057 }
1058
1059 /*
1060 * Note that cur_el can never be 0. If new_el is 0, then
1061 * el0_a64 is return_to_aa64, else el0_a64 is ignored.
1062 */
1063 aarch64_sve_change_el(env, cur_el, new_el, return_to_aa64);
1064
1065 qemu_mutex_lock_iothread();
1066 arm_call_el_change_hook(env_archcpu(env));
1067 qemu_mutex_unlock_iothread();
1068
1069 return;
1070
1071 illegal_return:
1072 /* Illegal return events of various kinds have architecturally
1073 * mandated behaviour:
1074 * restore NZCV and DAIF from SPSR_ELx
1075 * set PSTATE.IL
1076 * restore PC from ELR_ELx
1077 * no change to exception level, execution state or stack pointer
1078 */
1079 env->pstate |= PSTATE_IL;
1080 env->pc = new_pc;
1081 spsr &= PSTATE_NZCV | PSTATE_DAIF;
1082 spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF);
1083 pstate_write(env, spsr);
1084 if (!arm_singlestep_active(env)) {
1085 env->pstate &= ~PSTATE_SS;
1086 }
1087 qemu_log_mask(LOG_GUEST_ERROR, "Illegal exception return at EL%d: "
1088 "resuming execution at 0x%" PRIx64 "\n", cur_el, env->pc);
1089 }
1090
1091 /*
1092 * Square Root and Reciprocal square root
1093 */
1094
1095 uint32_t HELPER(sqrt_f16)(uint32_t a, void *fpstp)
1096 {
1097 float_status *s = fpstp;
1098
1099 return float16_sqrt(a, s);
1100 }
1101
1102 void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
1103 {
1104 /*
1105 * Implement DC ZVA, which zeroes a fixed-length block of memory.
1106 * Note that we do not implement the (architecturally mandated)
1107 * alignment fault for attempts to use this on Device memory
1108 * (which matches the usual QEMU behaviour of not implementing either
1109 * alignment faults or any memory attribute handling).
1110 */
1111 int blocklen = 4 << env_archcpu(env)->dcz_blocksize;
1112 uint64_t vaddr = vaddr_in & ~(blocklen - 1);
1113 int mmu_idx = cpu_mmu_index(env, false);
1114 void *mem;
1115
1116 /*
1117 * Trapless lookup. In addition to actual invalid page, may
1118 * return NULL for I/O, watchpoints, clean pages, etc.
1119 */
1120 mem = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx);
1121
1122 #ifndef CONFIG_USER_ONLY
1123 if (unlikely(!mem)) {
1124 uintptr_t ra = GETPC();
1125
1126 /*
1127 * Trap if accessing an invalid page. DC_ZVA requires that we supply
1128 * the original pointer for an invalid page. But watchpoints require
1129 * that we probe the actual space. So do both.
1130 */
1131 (void) probe_write(env, vaddr_in, 1, mmu_idx, ra);
1132 mem = probe_write(env, vaddr, blocklen, mmu_idx, ra);
1133
1134 if (unlikely(!mem)) {
1135 /*
1136 * The only remaining reason for mem == NULL is I/O.
1137 * Just do a series of byte writes as the architecture demands.
1138 */
1139 for (int i = 0; i < blocklen; i++) {
1140 cpu_stb_mmuidx_ra(env, vaddr + i, 0, mmu_idx, ra);
1141 }
1142 return;
1143 }
1144 }
1145 #endif
1146
1147 memset(mem, 0, blocklen);
1148 }