meson: target
[qemu.git] / target / arm / helper-a64.c
1 /*
2 * AArch64 specific helpers
3 *
4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "qemu/units.h"
22 #include "cpu.h"
23 #include "exec/gdbstub.h"
24 #include "exec/helper-proto.h"
25 #include "qemu/host-utils.h"
26 #include "qemu/log.h"
27 #include "qemu/main-loop.h"
28 #include "qemu/bitops.h"
29 #include "internals.h"
30 #include "qemu/crc32c.h"
31 #include "exec/exec-all.h"
32 #include "exec/cpu_ldst.h"
33 #include "qemu/int128.h"
34 #include "qemu/atomic128.h"
35 #include "tcg/tcg.h"
36 #include "fpu/softfloat.h"
37 #include <zlib.h> /* For crc32 */
38
39 /* C2.4.7 Multiply and divide */
40 /* special cases for 0 and LLONG_MIN are mandated by the standard */
41 uint64_t HELPER(udiv64)(uint64_t num, uint64_t den)
42 {
43 if (den == 0) {
44 return 0;
45 }
46 return num / den;
47 }
48
49 int64_t HELPER(sdiv64)(int64_t num, int64_t den)
50 {
51 if (den == 0) {
52 return 0;
53 }
54 if (num == LLONG_MIN && den == -1) {
55 return LLONG_MIN;
56 }
57 return num / den;
58 }
59
60 uint64_t HELPER(rbit64)(uint64_t x)
61 {
62 return revbit64(x);
63 }
64
65 void HELPER(msr_i_spsel)(CPUARMState *env, uint32_t imm)
66 {
67 update_spsel(env, imm);
68 }
69
70 static void daif_check(CPUARMState *env, uint32_t op,
71 uint32_t imm, uintptr_t ra)
72 {
73 /* DAIF update to PSTATE. This is OK from EL0 only if UMA is set. */
74 if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) {
75 raise_exception_ra(env, EXCP_UDEF,
76 syn_aa64_sysregtrap(0, extract32(op, 0, 3),
77 extract32(op, 3, 3), 4,
78 imm, 0x1f, 0),
79 exception_target_el(env), ra);
80 }
81 }
82
83 void HELPER(msr_i_daifset)(CPUARMState *env, uint32_t imm)
84 {
85 daif_check(env, 0x1e, imm, GETPC());
86 env->daif |= (imm << 6) & PSTATE_DAIF;
87 }
88
89 void HELPER(msr_i_daifclear)(CPUARMState *env, uint32_t imm)
90 {
91 daif_check(env, 0x1f, imm, GETPC());
92 env->daif &= ~((imm << 6) & PSTATE_DAIF);
93 }
94
95 /* Convert a softfloat float_relation_ (as returned by
96 * the float*_compare functions) to the correct ARM
97 * NZCV flag state.
98 */
99 static inline uint32_t float_rel_to_flags(int res)
100 {
101 uint64_t flags;
102 switch (res) {
103 case float_relation_equal:
104 flags = PSTATE_Z | PSTATE_C;
105 break;
106 case float_relation_less:
107 flags = PSTATE_N;
108 break;
109 case float_relation_greater:
110 flags = PSTATE_C;
111 break;
112 case float_relation_unordered:
113 default:
114 flags = PSTATE_C | PSTATE_V;
115 break;
116 }
117 return flags;
118 }
119
120 uint64_t HELPER(vfp_cmph_a64)(uint32_t x, uint32_t y, void *fp_status)
121 {
122 return float_rel_to_flags(float16_compare_quiet(x, y, fp_status));
123 }
124
125 uint64_t HELPER(vfp_cmpeh_a64)(uint32_t x, uint32_t y, void *fp_status)
126 {
127 return float_rel_to_flags(float16_compare(x, y, fp_status));
128 }
129
130 uint64_t HELPER(vfp_cmps_a64)(float32 x, float32 y, void *fp_status)
131 {
132 return float_rel_to_flags(float32_compare_quiet(x, y, fp_status));
133 }
134
135 uint64_t HELPER(vfp_cmpes_a64)(float32 x, float32 y, void *fp_status)
136 {
137 return float_rel_to_flags(float32_compare(x, y, fp_status));
138 }
139
140 uint64_t HELPER(vfp_cmpd_a64)(float64 x, float64 y, void *fp_status)
141 {
142 return float_rel_to_flags(float64_compare_quiet(x, y, fp_status));
143 }
144
145 uint64_t HELPER(vfp_cmped_a64)(float64 x, float64 y, void *fp_status)
146 {
147 return float_rel_to_flags(float64_compare(x, y, fp_status));
148 }
149
150 float32 HELPER(vfp_mulxs)(float32 a, float32 b, void *fpstp)
151 {
152 float_status *fpst = fpstp;
153
154 a = float32_squash_input_denormal(a, fpst);
155 b = float32_squash_input_denormal(b, fpst);
156
157 if ((float32_is_zero(a) && float32_is_infinity(b)) ||
158 (float32_is_infinity(a) && float32_is_zero(b))) {
159 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
160 return make_float32((1U << 30) |
161 ((float32_val(a) ^ float32_val(b)) & (1U << 31)));
162 }
163 return float32_mul(a, b, fpst);
164 }
165
166 float64 HELPER(vfp_mulxd)(float64 a, float64 b, void *fpstp)
167 {
168 float_status *fpst = fpstp;
169
170 a = float64_squash_input_denormal(a, fpst);
171 b = float64_squash_input_denormal(b, fpst);
172
173 if ((float64_is_zero(a) && float64_is_infinity(b)) ||
174 (float64_is_infinity(a) && float64_is_zero(b))) {
175 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
176 return make_float64((1ULL << 62) |
177 ((float64_val(a) ^ float64_val(b)) & (1ULL << 63)));
178 }
179 return float64_mul(a, b, fpst);
180 }
181
182 uint64_t HELPER(simd_tbl)(CPUARMState *env, uint64_t result, uint64_t indices,
183 uint32_t rn, uint32_t numregs)
184 {
185 /* Helper function for SIMD TBL and TBX. We have to do the table
186 * lookup part for the 64 bits worth of indices we're passed in.
187 * result is the initial results vector (either zeroes for TBL
188 * or some guest values for TBX), rn the register number where
189 * the table starts, and numregs the number of registers in the table.
190 * We return the results of the lookups.
191 */
192 int shift;
193
194 for (shift = 0; shift < 64; shift += 8) {
195 int index = extract64(indices, shift, 8);
196 if (index < 16 * numregs) {
197 /* Convert index (a byte offset into the virtual table
198 * which is a series of 128-bit vectors concatenated)
199 * into the correct register element plus a bit offset
200 * into that element, bearing in mind that the table
201 * can wrap around from V31 to V0.
202 */
203 int elt = (rn * 2 + (index >> 3)) % 64;
204 int bitidx = (index & 7) * 8;
205 uint64_t *q = aa64_vfp_qreg(env, elt >> 1);
206 uint64_t val = extract64(q[elt & 1], bitidx, 8);
207
208 result = deposit64(result, shift, 8, val);
209 }
210 }
211 return result;
212 }
213
214 /* 64bit/double versions of the neon float compare functions */
215 uint64_t HELPER(neon_ceq_f64)(float64 a, float64 b, void *fpstp)
216 {
217 float_status *fpst = fpstp;
218 return -float64_eq_quiet(a, b, fpst);
219 }
220
221 uint64_t HELPER(neon_cge_f64)(float64 a, float64 b, void *fpstp)
222 {
223 float_status *fpst = fpstp;
224 return -float64_le(b, a, fpst);
225 }
226
227 uint64_t HELPER(neon_cgt_f64)(float64 a, float64 b, void *fpstp)
228 {
229 float_status *fpst = fpstp;
230 return -float64_lt(b, a, fpst);
231 }
232
233 /* Reciprocal step and sqrt step. Note that unlike the A32/T32
234 * versions, these do a fully fused multiply-add or
235 * multiply-add-and-halve.
236 */
237 #define float16_two make_float16(0x4000)
238 #define float16_three make_float16(0x4200)
239 #define float16_one_point_five make_float16(0x3e00)
240
241 #define float32_two make_float32(0x40000000)
242 #define float32_three make_float32(0x40400000)
243 #define float32_one_point_five make_float32(0x3fc00000)
244
245 #define float64_two make_float64(0x4000000000000000ULL)
246 #define float64_three make_float64(0x4008000000000000ULL)
247 #define float64_one_point_five make_float64(0x3FF8000000000000ULL)
248
249 uint32_t HELPER(recpsf_f16)(uint32_t a, uint32_t b, void *fpstp)
250 {
251 float_status *fpst = fpstp;
252
253 a = float16_squash_input_denormal(a, fpst);
254 b = float16_squash_input_denormal(b, fpst);
255
256 a = float16_chs(a);
257 if ((float16_is_infinity(a) && float16_is_zero(b)) ||
258 (float16_is_infinity(b) && float16_is_zero(a))) {
259 return float16_two;
260 }
261 return float16_muladd(a, b, float16_two, 0, fpst);
262 }
263
264 float32 HELPER(recpsf_f32)(float32 a, float32 b, void *fpstp)
265 {
266 float_status *fpst = fpstp;
267
268 a = float32_squash_input_denormal(a, fpst);
269 b = float32_squash_input_denormal(b, fpst);
270
271 a = float32_chs(a);
272 if ((float32_is_infinity(a) && float32_is_zero(b)) ||
273 (float32_is_infinity(b) && float32_is_zero(a))) {
274 return float32_two;
275 }
276 return float32_muladd(a, b, float32_two, 0, fpst);
277 }
278
279 float64 HELPER(recpsf_f64)(float64 a, float64 b, void *fpstp)
280 {
281 float_status *fpst = fpstp;
282
283 a = float64_squash_input_denormal(a, fpst);
284 b = float64_squash_input_denormal(b, fpst);
285
286 a = float64_chs(a);
287 if ((float64_is_infinity(a) && float64_is_zero(b)) ||
288 (float64_is_infinity(b) && float64_is_zero(a))) {
289 return float64_two;
290 }
291 return float64_muladd(a, b, float64_two, 0, fpst);
292 }
293
294 uint32_t HELPER(rsqrtsf_f16)(uint32_t a, uint32_t b, void *fpstp)
295 {
296 float_status *fpst = fpstp;
297
298 a = float16_squash_input_denormal(a, fpst);
299 b = float16_squash_input_denormal(b, fpst);
300
301 a = float16_chs(a);
302 if ((float16_is_infinity(a) && float16_is_zero(b)) ||
303 (float16_is_infinity(b) && float16_is_zero(a))) {
304 return float16_one_point_five;
305 }
306 return float16_muladd(a, b, float16_three, float_muladd_halve_result, fpst);
307 }
308
309 float32 HELPER(rsqrtsf_f32)(float32 a, float32 b, void *fpstp)
310 {
311 float_status *fpst = fpstp;
312
313 a = float32_squash_input_denormal(a, fpst);
314 b = float32_squash_input_denormal(b, fpst);
315
316 a = float32_chs(a);
317 if ((float32_is_infinity(a) && float32_is_zero(b)) ||
318 (float32_is_infinity(b) && float32_is_zero(a))) {
319 return float32_one_point_five;
320 }
321 return float32_muladd(a, b, float32_three, float_muladd_halve_result, fpst);
322 }
323
324 float64 HELPER(rsqrtsf_f64)(float64 a, float64 b, void *fpstp)
325 {
326 float_status *fpst = fpstp;
327
328 a = float64_squash_input_denormal(a, fpst);
329 b = float64_squash_input_denormal(b, fpst);
330
331 a = float64_chs(a);
332 if ((float64_is_infinity(a) && float64_is_zero(b)) ||
333 (float64_is_infinity(b) && float64_is_zero(a))) {
334 return float64_one_point_five;
335 }
336 return float64_muladd(a, b, float64_three, float_muladd_halve_result, fpst);
337 }
338
339 /* Pairwise long add: add pairs of adjacent elements into
340 * double-width elements in the result (eg _s8 is an 8x8->16 op)
341 */
342 uint64_t HELPER(neon_addlp_s8)(uint64_t a)
343 {
344 uint64_t nsignmask = 0x0080008000800080ULL;
345 uint64_t wsignmask = 0x8000800080008000ULL;
346 uint64_t elementmask = 0x00ff00ff00ff00ffULL;
347 uint64_t tmp1, tmp2;
348 uint64_t res, signres;
349
350 /* Extract odd elements, sign extend each to a 16 bit field */
351 tmp1 = a & elementmask;
352 tmp1 ^= nsignmask;
353 tmp1 |= wsignmask;
354 tmp1 = (tmp1 - nsignmask) ^ wsignmask;
355 /* Ditto for the even elements */
356 tmp2 = (a >> 8) & elementmask;
357 tmp2 ^= nsignmask;
358 tmp2 |= wsignmask;
359 tmp2 = (tmp2 - nsignmask) ^ wsignmask;
360
361 /* calculate the result by summing bits 0..14, 16..22, etc,
362 * and then adjusting the sign bits 15, 23, etc manually.
363 * This ensures the addition can't overflow the 16 bit field.
364 */
365 signres = (tmp1 ^ tmp2) & wsignmask;
366 res = (tmp1 & ~wsignmask) + (tmp2 & ~wsignmask);
367 res ^= signres;
368
369 return res;
370 }
371
372 uint64_t HELPER(neon_addlp_u8)(uint64_t a)
373 {
374 uint64_t tmp;
375
376 tmp = a & 0x00ff00ff00ff00ffULL;
377 tmp += (a >> 8) & 0x00ff00ff00ff00ffULL;
378 return tmp;
379 }
380
381 uint64_t HELPER(neon_addlp_s16)(uint64_t a)
382 {
383 int32_t reslo, reshi;
384
385 reslo = (int32_t)(int16_t)a + (int32_t)(int16_t)(a >> 16);
386 reshi = (int32_t)(int16_t)(a >> 32) + (int32_t)(int16_t)(a >> 48);
387
388 return (uint32_t)reslo | (((uint64_t)reshi) << 32);
389 }
390
391 uint64_t HELPER(neon_addlp_u16)(uint64_t a)
392 {
393 uint64_t tmp;
394
395 tmp = a & 0x0000ffff0000ffffULL;
396 tmp += (a >> 16) & 0x0000ffff0000ffffULL;
397 return tmp;
398 }
399
400 /* Floating-point reciprocal exponent - see FPRecpX in ARM ARM */
401 uint32_t HELPER(frecpx_f16)(uint32_t a, void *fpstp)
402 {
403 float_status *fpst = fpstp;
404 uint16_t val16, sbit;
405 int16_t exp;
406
407 if (float16_is_any_nan(a)) {
408 float16 nan = a;
409 if (float16_is_signaling_nan(a, fpst)) {
410 float_raise(float_flag_invalid, fpst);
411 nan = float16_silence_nan(a, fpst);
412 }
413 if (fpst->default_nan_mode) {
414 nan = float16_default_nan(fpst);
415 }
416 return nan;
417 }
418
419 a = float16_squash_input_denormal(a, fpst);
420
421 val16 = float16_val(a);
422 sbit = 0x8000 & val16;
423 exp = extract32(val16, 10, 5);
424
425 if (exp == 0) {
426 return make_float16(deposit32(sbit, 10, 5, 0x1e));
427 } else {
428 return make_float16(deposit32(sbit, 10, 5, ~exp));
429 }
430 }
431
432 float32 HELPER(frecpx_f32)(float32 a, void *fpstp)
433 {
434 float_status *fpst = fpstp;
435 uint32_t val32, sbit;
436 int32_t exp;
437
438 if (float32_is_any_nan(a)) {
439 float32 nan = a;
440 if (float32_is_signaling_nan(a, fpst)) {
441 float_raise(float_flag_invalid, fpst);
442 nan = float32_silence_nan(a, fpst);
443 }
444 if (fpst->default_nan_mode) {
445 nan = float32_default_nan(fpst);
446 }
447 return nan;
448 }
449
450 a = float32_squash_input_denormal(a, fpst);
451
452 val32 = float32_val(a);
453 sbit = 0x80000000ULL & val32;
454 exp = extract32(val32, 23, 8);
455
456 if (exp == 0) {
457 return make_float32(sbit | (0xfe << 23));
458 } else {
459 return make_float32(sbit | (~exp & 0xff) << 23);
460 }
461 }
462
463 float64 HELPER(frecpx_f64)(float64 a, void *fpstp)
464 {
465 float_status *fpst = fpstp;
466 uint64_t val64, sbit;
467 int64_t exp;
468
469 if (float64_is_any_nan(a)) {
470 float64 nan = a;
471 if (float64_is_signaling_nan(a, fpst)) {
472 float_raise(float_flag_invalid, fpst);
473 nan = float64_silence_nan(a, fpst);
474 }
475 if (fpst->default_nan_mode) {
476 nan = float64_default_nan(fpst);
477 }
478 return nan;
479 }
480
481 a = float64_squash_input_denormal(a, fpst);
482
483 val64 = float64_val(a);
484 sbit = 0x8000000000000000ULL & val64;
485 exp = extract64(float64_val(a), 52, 11);
486
487 if (exp == 0) {
488 return make_float64(sbit | (0x7feULL << 52));
489 } else {
490 return make_float64(sbit | (~exp & 0x7ffULL) << 52);
491 }
492 }
493
494 float32 HELPER(fcvtx_f64_to_f32)(float64 a, CPUARMState *env)
495 {
496 /* Von Neumann rounding is implemented by using round-to-zero
497 * and then setting the LSB of the result if Inexact was raised.
498 */
499 float32 r;
500 float_status *fpst = &env->vfp.fp_status;
501 float_status tstat = *fpst;
502 int exflags;
503
504 set_float_rounding_mode(float_round_to_zero, &tstat);
505 set_float_exception_flags(0, &tstat);
506 r = float64_to_float32(a, &tstat);
507 exflags = get_float_exception_flags(&tstat);
508 if (exflags & float_flag_inexact) {
509 r = make_float32(float32_val(r) | 1);
510 }
511 exflags |= get_float_exception_flags(fpst);
512 set_float_exception_flags(exflags, fpst);
513 return r;
514 }
515
516 /* 64-bit versions of the CRC helpers. Note that although the operation
517 * (and the prototypes of crc32c() and crc32() mean that only the bottom
518 * 32 bits of the accumulator and result are used, we pass and return
519 * uint64_t for convenience of the generated code. Unlike the 32-bit
520 * instruction set versions, val may genuinely have 64 bits of data in it.
521 * The upper bytes of val (above the number specified by 'bytes') must have
522 * been zeroed out by the caller.
523 */
524 uint64_t HELPER(crc32_64)(uint64_t acc, uint64_t val, uint32_t bytes)
525 {
526 uint8_t buf[8];
527
528 stq_le_p(buf, val);
529
530 /* zlib crc32 converts the accumulator and output to one's complement. */
531 return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
532 }
533
534 uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
535 {
536 uint8_t buf[8];
537
538 stq_le_p(buf, val);
539
540 /* Linux crc32c converts the output to one's complement. */
541 return crc32c(acc, buf, bytes) ^ 0xffffffff;
542 }
543
544 uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
545 uint64_t new_lo, uint64_t new_hi)
546 {
547 Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
548 Int128 newv = int128_make128(new_lo, new_hi);
549 Int128 oldv;
550 uintptr_t ra = GETPC();
551 uint64_t o0, o1;
552 bool success;
553
554 #ifdef CONFIG_USER_ONLY
555 /* ??? Enforce alignment. */
556 uint64_t *haddr = g2h(addr);
557
558 set_helper_retaddr(ra);
559 o0 = ldq_le_p(haddr + 0);
560 o1 = ldq_le_p(haddr + 1);
561 oldv = int128_make128(o0, o1);
562
563 success = int128_eq(oldv, cmpv);
564 if (success) {
565 stq_le_p(haddr + 0, int128_getlo(newv));
566 stq_le_p(haddr + 1, int128_gethi(newv));
567 }
568 clear_helper_retaddr();
569 #else
570 int mem_idx = cpu_mmu_index(env, false);
571 TCGMemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
572 TCGMemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx);
573
574 o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra);
575 o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra);
576 oldv = int128_make128(o0, o1);
577
578 success = int128_eq(oldv, cmpv);
579 if (success) {
580 helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
581 helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
582 }
583 #endif
584
585 return !success;
586 }
587
588 uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
589 uint64_t new_lo, uint64_t new_hi)
590 {
591 Int128 oldv, cmpv, newv;
592 uintptr_t ra = GETPC();
593 bool success;
594 int mem_idx;
595 TCGMemOpIdx oi;
596
597 assert(HAVE_CMPXCHG128);
598
599 mem_idx = cpu_mmu_index(env, false);
600 oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
601
602 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
603 newv = int128_make128(new_lo, new_hi);
604 oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
605
606 success = int128_eq(oldv, cmpv);
607 return !success;
608 }
609
610 uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr,
611 uint64_t new_lo, uint64_t new_hi)
612 {
613 /*
614 * High and low need to be switched here because this is not actually a
615 * 128bit store but two doublewords stored consecutively
616 */
617 Int128 cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
618 Int128 newv = int128_make128(new_hi, new_lo);
619 Int128 oldv;
620 uintptr_t ra = GETPC();
621 uint64_t o0, o1;
622 bool success;
623
624 #ifdef CONFIG_USER_ONLY
625 /* ??? Enforce alignment. */
626 uint64_t *haddr = g2h(addr);
627
628 set_helper_retaddr(ra);
629 o1 = ldq_be_p(haddr + 0);
630 o0 = ldq_be_p(haddr + 1);
631 oldv = int128_make128(o0, o1);
632
633 success = int128_eq(oldv, cmpv);
634 if (success) {
635 stq_be_p(haddr + 0, int128_gethi(newv));
636 stq_be_p(haddr + 1, int128_getlo(newv));
637 }
638 clear_helper_retaddr();
639 #else
640 int mem_idx = cpu_mmu_index(env, false);
641 TCGMemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
642 TCGMemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx);
643
644 o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra);
645 o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra);
646 oldv = int128_make128(o0, o1);
647
648 success = int128_eq(oldv, cmpv);
649 if (success) {
650 helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
651 helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
652 }
653 #endif
654
655 return !success;
656 }
657
658 uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
659 uint64_t new_lo, uint64_t new_hi)
660 {
661 Int128 oldv, cmpv, newv;
662 uintptr_t ra = GETPC();
663 bool success;
664 int mem_idx;
665 TCGMemOpIdx oi;
666
667 assert(HAVE_CMPXCHG128);
668
669 mem_idx = cpu_mmu_index(env, false);
670 oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
671
672 /*
673 * High and low need to be switched here because this is not actually a
674 * 128bit store but two doublewords stored consecutively
675 */
676 cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
677 newv = int128_make128(new_hi, new_lo);
678 oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
679
680 success = int128_eq(oldv, cmpv);
681 return !success;
682 }
683
684 /* Writes back the old data into Rs. */
685 void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
686 uint64_t new_lo, uint64_t new_hi)
687 {
688 Int128 oldv, cmpv, newv;
689 uintptr_t ra = GETPC();
690 int mem_idx;
691 TCGMemOpIdx oi;
692
693 assert(HAVE_CMPXCHG128);
694
695 mem_idx = cpu_mmu_index(env, false);
696 oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
697
698 cmpv = int128_make128(env->xregs[rs], env->xregs[rs + 1]);
699 newv = int128_make128(new_lo, new_hi);
700 oldv = helper_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
701
702 env->xregs[rs] = int128_getlo(oldv);
703 env->xregs[rs + 1] = int128_gethi(oldv);
704 }
705
706 void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
707 uint64_t new_hi, uint64_t new_lo)
708 {
709 Int128 oldv, cmpv, newv;
710 uintptr_t ra = GETPC();
711 int mem_idx;
712 TCGMemOpIdx oi;
713
714 assert(HAVE_CMPXCHG128);
715
716 mem_idx = cpu_mmu_index(env, false);
717 oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
718
719 cmpv = int128_make128(env->xregs[rs + 1], env->xregs[rs]);
720 newv = int128_make128(new_lo, new_hi);
721 oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
722
723 env->xregs[rs + 1] = int128_getlo(oldv);
724 env->xregs[rs] = int128_gethi(oldv);
725 }
726
727 /*
728 * AdvSIMD half-precision
729 */
730
731 #define ADVSIMD_HELPER(name, suffix) HELPER(glue(glue(advsimd_, name), suffix))
732
733 #define ADVSIMD_HALFOP(name) \
734 uint32_t ADVSIMD_HELPER(name, h)(uint32_t a, uint32_t b, void *fpstp) \
735 { \
736 float_status *fpst = fpstp; \
737 return float16_ ## name(a, b, fpst); \
738 }
739
740 ADVSIMD_HALFOP(add)
741 ADVSIMD_HALFOP(sub)
742 ADVSIMD_HALFOP(mul)
743 ADVSIMD_HALFOP(div)
744 ADVSIMD_HALFOP(min)
745 ADVSIMD_HALFOP(max)
746 ADVSIMD_HALFOP(minnum)
747 ADVSIMD_HALFOP(maxnum)
748
749 #define ADVSIMD_TWOHALFOP(name) \
750 uint32_t ADVSIMD_HELPER(name, 2h)(uint32_t two_a, uint32_t two_b, void *fpstp) \
751 { \
752 float16 a1, a2, b1, b2; \
753 uint32_t r1, r2; \
754 float_status *fpst = fpstp; \
755 a1 = extract32(two_a, 0, 16); \
756 a2 = extract32(two_a, 16, 16); \
757 b1 = extract32(two_b, 0, 16); \
758 b2 = extract32(two_b, 16, 16); \
759 r1 = float16_ ## name(a1, b1, fpst); \
760 r2 = float16_ ## name(a2, b2, fpst); \
761 return deposit32(r1, 16, 16, r2); \
762 }
763
764 ADVSIMD_TWOHALFOP(add)
765 ADVSIMD_TWOHALFOP(sub)
766 ADVSIMD_TWOHALFOP(mul)
767 ADVSIMD_TWOHALFOP(div)
768 ADVSIMD_TWOHALFOP(min)
769 ADVSIMD_TWOHALFOP(max)
770 ADVSIMD_TWOHALFOP(minnum)
771 ADVSIMD_TWOHALFOP(maxnum)
772
773 /* Data processing - scalar floating-point and advanced SIMD */
774 static float16 float16_mulx(float16 a, float16 b, void *fpstp)
775 {
776 float_status *fpst = fpstp;
777
778 a = float16_squash_input_denormal(a, fpst);
779 b = float16_squash_input_denormal(b, fpst);
780
781 if ((float16_is_zero(a) && float16_is_infinity(b)) ||
782 (float16_is_infinity(a) && float16_is_zero(b))) {
783 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
784 return make_float16((1U << 14) |
785 ((float16_val(a) ^ float16_val(b)) & (1U << 15)));
786 }
787 return float16_mul(a, b, fpst);
788 }
789
790 ADVSIMD_HALFOP(mulx)
791 ADVSIMD_TWOHALFOP(mulx)
792
793 /* fused multiply-accumulate */
794 uint32_t HELPER(advsimd_muladdh)(uint32_t a, uint32_t b, uint32_t c,
795 void *fpstp)
796 {
797 float_status *fpst = fpstp;
798 return float16_muladd(a, b, c, 0, fpst);
799 }
800
801 uint32_t HELPER(advsimd_muladd2h)(uint32_t two_a, uint32_t two_b,
802 uint32_t two_c, void *fpstp)
803 {
804 float_status *fpst = fpstp;
805 float16 a1, a2, b1, b2, c1, c2;
806 uint32_t r1, r2;
807 a1 = extract32(two_a, 0, 16);
808 a2 = extract32(two_a, 16, 16);
809 b1 = extract32(two_b, 0, 16);
810 b2 = extract32(two_b, 16, 16);
811 c1 = extract32(two_c, 0, 16);
812 c2 = extract32(two_c, 16, 16);
813 r1 = float16_muladd(a1, b1, c1, 0, fpst);
814 r2 = float16_muladd(a2, b2, c2, 0, fpst);
815 return deposit32(r1, 16, 16, r2);
816 }
817
818 /*
819 * Floating point comparisons produce an integer result. Softfloat
820 * routines return float_relation types which we convert to the 0/-1
821 * Neon requires.
822 */
823
824 #define ADVSIMD_CMPRES(test) (test) ? 0xffff : 0
825
826 uint32_t HELPER(advsimd_ceq_f16)(uint32_t a, uint32_t b, void *fpstp)
827 {
828 float_status *fpst = fpstp;
829 int compare = float16_compare_quiet(a, b, fpst);
830 return ADVSIMD_CMPRES(compare == float_relation_equal);
831 }
832
833 uint32_t HELPER(advsimd_cge_f16)(uint32_t a, uint32_t b, void *fpstp)
834 {
835 float_status *fpst = fpstp;
836 int compare = float16_compare(a, b, fpst);
837 return ADVSIMD_CMPRES(compare == float_relation_greater ||
838 compare == float_relation_equal);
839 }
840
841 uint32_t HELPER(advsimd_cgt_f16)(uint32_t a, uint32_t b, void *fpstp)
842 {
843 float_status *fpst = fpstp;
844 int compare = float16_compare(a, b, fpst);
845 return ADVSIMD_CMPRES(compare == float_relation_greater);
846 }
847
848 uint32_t HELPER(advsimd_acge_f16)(uint32_t a, uint32_t b, void *fpstp)
849 {
850 float_status *fpst = fpstp;
851 float16 f0 = float16_abs(a);
852 float16 f1 = float16_abs(b);
853 int compare = float16_compare(f0, f1, fpst);
854 return ADVSIMD_CMPRES(compare == float_relation_greater ||
855 compare == float_relation_equal);
856 }
857
858 uint32_t HELPER(advsimd_acgt_f16)(uint32_t a, uint32_t b, void *fpstp)
859 {
860 float_status *fpst = fpstp;
861 float16 f0 = float16_abs(a);
862 float16 f1 = float16_abs(b);
863 int compare = float16_compare(f0, f1, fpst);
864 return ADVSIMD_CMPRES(compare == float_relation_greater);
865 }
866
867 /* round to integral */
868 uint32_t HELPER(advsimd_rinth_exact)(uint32_t x, void *fp_status)
869 {
870 return float16_round_to_int(x, fp_status);
871 }
872
873 uint32_t HELPER(advsimd_rinth)(uint32_t x, void *fp_status)
874 {
875 int old_flags = get_float_exception_flags(fp_status), new_flags;
876 float16 ret;
877
878 ret = float16_round_to_int(x, fp_status);
879
880 /* Suppress any inexact exceptions the conversion produced */
881 if (!(old_flags & float_flag_inexact)) {
882 new_flags = get_float_exception_flags(fp_status);
883 set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
884 }
885
886 return ret;
887 }
888
889 /*
890 * Half-precision floating point conversion functions
891 *
892 * There are a multitude of conversion functions with various
893 * different rounding modes. This is dealt with by the calling code
894 * setting the mode appropriately before calling the helper.
895 */
896
897 uint32_t HELPER(advsimd_f16tosinth)(uint32_t a, void *fpstp)
898 {
899 float_status *fpst = fpstp;
900
901 /* Invalid if we are passed a NaN */
902 if (float16_is_any_nan(a)) {
903 float_raise(float_flag_invalid, fpst);
904 return 0;
905 }
906 return float16_to_int16(a, fpst);
907 }
908
909 uint32_t HELPER(advsimd_f16touinth)(uint32_t a, void *fpstp)
910 {
911 float_status *fpst = fpstp;
912
913 /* Invalid if we are passed a NaN */
914 if (float16_is_any_nan(a)) {
915 float_raise(float_flag_invalid, fpst);
916 return 0;
917 }
918 return float16_to_uint16(a, fpst);
919 }
920
921 static int el_from_spsr(uint32_t spsr)
922 {
923 /* Return the exception level that this SPSR is requesting a return to,
924 * or -1 if it is invalid (an illegal return)
925 */
926 if (spsr & PSTATE_nRW) {
927 switch (spsr & CPSR_M) {
928 case ARM_CPU_MODE_USR:
929 return 0;
930 case ARM_CPU_MODE_HYP:
931 return 2;
932 case ARM_CPU_MODE_FIQ:
933 case ARM_CPU_MODE_IRQ:
934 case ARM_CPU_MODE_SVC:
935 case ARM_CPU_MODE_ABT:
936 case ARM_CPU_MODE_UND:
937 case ARM_CPU_MODE_SYS:
938 return 1;
939 case ARM_CPU_MODE_MON:
940 /* Returning to Mon from AArch64 is never possible,
941 * so this is an illegal return.
942 */
943 default:
944 return -1;
945 }
946 } else {
947 if (extract32(spsr, 1, 1)) {
948 /* Return with reserved M[1] bit set */
949 return -1;
950 }
951 if (extract32(spsr, 0, 4) == 1) {
952 /* return to EL0 with M[0] bit set */
953 return -1;
954 }
955 return extract32(spsr, 2, 2);
956 }
957 }
958
959 void HELPER(exception_return)(CPUARMState *env, uint64_t new_pc)
960 {
961 int cur_el = arm_current_el(env);
962 unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
963 uint32_t mask, spsr = env->banked_spsr[spsr_idx];
964 int new_el;
965 bool return_to_aa64 = (spsr & PSTATE_nRW) == 0;
966
967 aarch64_save_sp(env, cur_el);
968
969 arm_clear_exclusive(env);
970
971 /* We must squash the PSTATE.SS bit to zero unless both of the
972 * following hold:
973 * 1. debug exceptions are currently disabled
974 * 2. singlestep will be active in the EL we return to
975 * We check 1 here and 2 after we've done the pstate/cpsr write() to
976 * transition to the EL we're going to.
977 */
978 if (arm_generate_debug_exceptions(env)) {
979 spsr &= ~PSTATE_SS;
980 }
981
982 new_el = el_from_spsr(spsr);
983 if (new_el == -1) {
984 goto illegal_return;
985 }
986 if (new_el > cur_el
987 || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
988 /* Disallow return to an EL which is unimplemented or higher
989 * than the current one.
990 */
991 goto illegal_return;
992 }
993
994 if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) {
995 /* Return to an EL which is configured for a different register width */
996 goto illegal_return;
997 }
998
999 if (new_el == 2 && arm_is_secure_below_el3(env)) {
1000 /* Return to the non-existent secure-EL2 */
1001 goto illegal_return;
1002 }
1003
1004 if (new_el == 1 && (arm_hcr_el2_eff(env) & HCR_TGE)) {
1005 goto illegal_return;
1006 }
1007
1008 qemu_mutex_lock_iothread();
1009 arm_call_pre_el_change_hook(env_archcpu(env));
1010 qemu_mutex_unlock_iothread();
1011
1012 if (!return_to_aa64) {
1013 env->aarch64 = 0;
1014 /* We do a raw CPSR write because aarch64_sync_64_to_32()
1015 * will sort the register banks out for us, and we've already
1016 * caught all the bad-mode cases in el_from_spsr().
1017 */
1018 mask = aarch32_cpsr_valid_mask(env->features, &env_archcpu(env)->isar);
1019 cpsr_write(env, spsr, mask, CPSRWriteRaw);
1020 if (!arm_singlestep_active(env)) {
1021 env->uncached_cpsr &= ~PSTATE_SS;
1022 }
1023 aarch64_sync_64_to_32(env);
1024
1025 if (spsr & CPSR_T) {
1026 env->regs[15] = new_pc & ~0x1;
1027 } else {
1028 env->regs[15] = new_pc & ~0x3;
1029 }
1030 helper_rebuild_hflags_a32(env, new_el);
1031 qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to "
1032 "AArch32 EL%d PC 0x%" PRIx32 "\n",
1033 cur_el, new_el, env->regs[15]);
1034 } else {
1035 int tbii;
1036
1037 env->aarch64 = 1;
1038 spsr &= aarch64_pstate_valid_mask(&env_archcpu(env)->isar);
1039 pstate_write(env, spsr);
1040 if (!arm_singlestep_active(env)) {
1041 env->pstate &= ~PSTATE_SS;
1042 }
1043 aarch64_restore_sp(env, new_el);
1044 helper_rebuild_hflags_a64(env, new_el);
1045
1046 /*
1047 * Apply TBI to the exception return address. We had to delay this
1048 * until after we selected the new EL, so that we could select the
1049 * correct TBI+TBID bits. This is made easier by waiting until after
1050 * the hflags rebuild, since we can pull the composite TBII field
1051 * from there.
1052 */
1053 tbii = FIELD_EX32(env->hflags, TBFLAG_A64, TBII);
1054 if ((tbii >> extract64(new_pc, 55, 1)) & 1) {
1055 /* TBI is enabled. */
1056 int core_mmu_idx = cpu_mmu_index(env, false);
1057 if (regime_has_2_ranges(core_to_aa64_mmu_idx(core_mmu_idx))) {
1058 new_pc = sextract64(new_pc, 0, 56);
1059 } else {
1060 new_pc = extract64(new_pc, 0, 56);
1061 }
1062 }
1063 env->pc = new_pc;
1064
1065 qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to "
1066 "AArch64 EL%d PC 0x%" PRIx64 "\n",
1067 cur_el, new_el, env->pc);
1068 }
1069
1070 /*
1071 * Note that cur_el can never be 0. If new_el is 0, then
1072 * el0_a64 is return_to_aa64, else el0_a64 is ignored.
1073 */
1074 aarch64_sve_change_el(env, cur_el, new_el, return_to_aa64);
1075
1076 qemu_mutex_lock_iothread();
1077 arm_call_el_change_hook(env_archcpu(env));
1078 qemu_mutex_unlock_iothread();
1079
1080 return;
1081
1082 illegal_return:
1083 /* Illegal return events of various kinds have architecturally
1084 * mandated behaviour:
1085 * restore NZCV and DAIF from SPSR_ELx
1086 * set PSTATE.IL
1087 * restore PC from ELR_ELx
1088 * no change to exception level, execution state or stack pointer
1089 */
1090 env->pstate |= PSTATE_IL;
1091 env->pc = new_pc;
1092 spsr &= PSTATE_NZCV | PSTATE_DAIF;
1093 spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF);
1094 pstate_write(env, spsr);
1095 if (!arm_singlestep_active(env)) {
1096 env->pstate &= ~PSTATE_SS;
1097 }
1098 qemu_log_mask(LOG_GUEST_ERROR, "Illegal exception return at EL%d: "
1099 "resuming execution at 0x%" PRIx64 "\n", cur_el, env->pc);
1100 }
1101
1102 /*
1103 * Square Root and Reciprocal square root
1104 */
1105
1106 uint32_t HELPER(sqrt_f16)(uint32_t a, void *fpstp)
1107 {
1108 float_status *s = fpstp;
1109
1110 return float16_sqrt(a, s);
1111 }
1112
1113 void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
1114 {
1115 /*
1116 * Implement DC ZVA, which zeroes a fixed-length block of memory.
1117 * Note that we do not implement the (architecturally mandated)
1118 * alignment fault for attempts to use this on Device memory
1119 * (which matches the usual QEMU behaviour of not implementing either
1120 * alignment faults or any memory attribute handling).
1121 */
1122 int blocklen = 4 << env_archcpu(env)->dcz_blocksize;
1123 uint64_t vaddr = vaddr_in & ~(blocklen - 1);
1124 int mmu_idx = cpu_mmu_index(env, false);
1125 void *mem;
1126
1127 /*
1128 * Trapless lookup. In addition to actual invalid page, may
1129 * return NULL for I/O, watchpoints, clean pages, etc.
1130 */
1131 mem = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx);
1132
1133 #ifndef CONFIG_USER_ONLY
1134 if (unlikely(!mem)) {
1135 uintptr_t ra = GETPC();
1136
1137 /*
1138 * Trap if accessing an invalid page. DC_ZVA requires that we supply
1139 * the original pointer for an invalid page. But watchpoints require
1140 * that we probe the actual space. So do both.
1141 */
1142 (void) probe_write(env, vaddr_in, 1, mmu_idx, ra);
1143 mem = probe_write(env, vaddr, blocklen, mmu_idx, ra);
1144
1145 if (unlikely(!mem)) {
1146 /*
1147 * The only remaining reason for mem == NULL is I/O.
1148 * Just do a series of byte writes as the architecture demands.
1149 */
1150 for (int i = 0; i < blocklen; i++) {
1151 cpu_stb_mmuidx_ra(env, vaddr + i, 0, mmu_idx, ra);
1152 }
1153 return;
1154 }
1155 }
1156 #endif
1157
1158 memset(mem, 0, blocklen);
1159 }