qemu-img: Omit error_report() after img_open()
[qemu.git] / target-arm / neon_helper.c
1 /*
2 * ARM NEON vector operations.
3 *
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
6 *
7 * This code is licensed under the GNU GPL v2.
8 */
9 #include <stdlib.h>
10 #include <stdio.h>
11
12 #include "cpu.h"
13 #include "exec/exec-all.h"
14 #include "exec/helper-proto.h"
15
16 #define SIGNBIT (uint32_t)0x80000000
17 #define SIGNBIT64 ((uint64_t)1 << 63)
18
19 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] |= CPSR_Q
20
21 #define NEON_TYPE1(name, type) \
22 typedef struct \
23 { \
24 type v1; \
25 } neon_##name;
26 #ifdef HOST_WORDS_BIGENDIAN
27 #define NEON_TYPE2(name, type) \
28 typedef struct \
29 { \
30 type v2; \
31 type v1; \
32 } neon_##name;
33 #define NEON_TYPE4(name, type) \
34 typedef struct \
35 { \
36 type v4; \
37 type v3; \
38 type v2; \
39 type v1; \
40 } neon_##name;
41 #else
42 #define NEON_TYPE2(name, type) \
43 typedef struct \
44 { \
45 type v1; \
46 type v2; \
47 } neon_##name;
48 #define NEON_TYPE4(name, type) \
49 typedef struct \
50 { \
51 type v1; \
52 type v2; \
53 type v3; \
54 type v4; \
55 } neon_##name;
56 #endif
57
58 NEON_TYPE4(s8, int8_t)
59 NEON_TYPE4(u8, uint8_t)
60 NEON_TYPE2(s16, int16_t)
61 NEON_TYPE2(u16, uint16_t)
62 NEON_TYPE1(s32, int32_t)
63 NEON_TYPE1(u32, uint32_t)
64 #undef NEON_TYPE4
65 #undef NEON_TYPE2
66 #undef NEON_TYPE1
67
68 /* Copy from a uint32_t to a vector structure type. */
69 #define NEON_UNPACK(vtype, dest, val) do { \
70 union { \
71 vtype v; \
72 uint32_t i; \
73 } conv_u; \
74 conv_u.i = (val); \
75 dest = conv_u.v; \
76 } while(0)
77
78 /* Copy from a vector structure type to a uint32_t. */
79 #define NEON_PACK(vtype, dest, val) do { \
80 union { \
81 vtype v; \
82 uint32_t i; \
83 } conv_u; \
84 conv_u.v = (val); \
85 dest = conv_u.i; \
86 } while(0)
87
88 #define NEON_DO1 \
89 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
90 #define NEON_DO2 \
91 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
92 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
93 #define NEON_DO4 \
94 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
95 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
96 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
97 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
98
99 #define NEON_VOP_BODY(vtype, n) \
100 { \
101 uint32_t res; \
102 vtype vsrc1; \
103 vtype vsrc2; \
104 vtype vdest; \
105 NEON_UNPACK(vtype, vsrc1, arg1); \
106 NEON_UNPACK(vtype, vsrc2, arg2); \
107 NEON_DO##n; \
108 NEON_PACK(vtype, res, vdest); \
109 return res; \
110 }
111
112 #define NEON_VOP(name, vtype, n) \
113 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
114 NEON_VOP_BODY(vtype, n)
115
116 #define NEON_VOP_ENV(name, vtype, n) \
117 uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
118 NEON_VOP_BODY(vtype, n)
119
120 /* Pairwise operations. */
121 /* For 32-bit elements each segment only contains a single element, so
122 the elementwise and pairwise operations are the same. */
123 #define NEON_PDO2 \
124 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
125 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
126 #define NEON_PDO4 \
127 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
128 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
129 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
130 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
131
132 #define NEON_POP(name, vtype, n) \
133 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
134 { \
135 uint32_t res; \
136 vtype vsrc1; \
137 vtype vsrc2; \
138 vtype vdest; \
139 NEON_UNPACK(vtype, vsrc1, arg1); \
140 NEON_UNPACK(vtype, vsrc2, arg2); \
141 NEON_PDO##n; \
142 NEON_PACK(vtype, res, vdest); \
143 return res; \
144 }
145
146 /* Unary operators. */
147 #define NEON_VOP1(name, vtype, n) \
148 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
149 { \
150 vtype vsrc1; \
151 vtype vdest; \
152 NEON_UNPACK(vtype, vsrc1, arg); \
153 NEON_DO##n; \
154 NEON_PACK(vtype, arg, vdest); \
155 return arg; \
156 }
157
158
159 #define NEON_USAT(dest, src1, src2, type) do { \
160 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
161 if (tmp != (type)tmp) { \
162 SET_QC(); \
163 dest = ~0; \
164 } else { \
165 dest = tmp; \
166 }} while(0)
167 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
168 NEON_VOP_ENV(qadd_u8, neon_u8, 4)
169 #undef NEON_FN
170 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
171 NEON_VOP_ENV(qadd_u16, neon_u16, 2)
172 #undef NEON_FN
173 #undef NEON_USAT
174
175 uint32_t HELPER(neon_qadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
176 {
177 uint32_t res = a + b;
178 if (res < a) {
179 SET_QC();
180 res = ~0;
181 }
182 return res;
183 }
184
185 uint64_t HELPER(neon_qadd_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
186 {
187 uint64_t res;
188
189 res = src1 + src2;
190 if (res < src1) {
191 SET_QC();
192 res = ~(uint64_t)0;
193 }
194 return res;
195 }
196
197 #define NEON_SSAT(dest, src1, src2, type) do { \
198 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
199 if (tmp != (type)tmp) { \
200 SET_QC(); \
201 if (src2 > 0) { \
202 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
203 } else { \
204 tmp = 1 << (sizeof(type) * 8 - 1); \
205 } \
206 } \
207 dest = tmp; \
208 } while(0)
209 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
210 NEON_VOP_ENV(qadd_s8, neon_s8, 4)
211 #undef NEON_FN
212 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
213 NEON_VOP_ENV(qadd_s16, neon_s16, 2)
214 #undef NEON_FN
215 #undef NEON_SSAT
216
217 uint32_t HELPER(neon_qadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
218 {
219 uint32_t res = a + b;
220 if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
221 SET_QC();
222 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
223 }
224 return res;
225 }
226
227 uint64_t HELPER(neon_qadd_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
228 {
229 uint64_t res;
230
231 res = src1 + src2;
232 if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
233 SET_QC();
234 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
235 }
236 return res;
237 }
238
239 /* Unsigned saturating accumulate of signed value
240 *
241 * Op1/Rn is treated as signed
242 * Op2/Rd is treated as unsigned
243 *
244 * Explicit casting is used to ensure the correct sign extension of
245 * inputs. The result is treated as a unsigned value and saturated as such.
246 *
247 * We use a macro for the 8/16 bit cases which expects signed integers of va,
248 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
249 */
250
251 #define USATACC(bits, shift) \
252 do { \
253 va = sextract32(a, shift, bits); \
254 vb = extract32(b, shift, bits); \
255 vr = va + vb; \
256 if (vr > UINT##bits##_MAX) { \
257 SET_QC(); \
258 vr = UINT##bits##_MAX; \
259 } else if (vr < 0) { \
260 SET_QC(); \
261 vr = 0; \
262 } \
263 r = deposit32(r, shift, bits, vr); \
264 } while (0)
265
266 uint32_t HELPER(neon_uqadd_s8)(CPUARMState *env, uint32_t a, uint32_t b)
267 {
268 int16_t va, vb, vr;
269 uint32_t r = 0;
270
271 USATACC(8, 0);
272 USATACC(8, 8);
273 USATACC(8, 16);
274 USATACC(8, 24);
275 return r;
276 }
277
278 uint32_t HELPER(neon_uqadd_s16)(CPUARMState *env, uint32_t a, uint32_t b)
279 {
280 int32_t va, vb, vr;
281 uint64_t r = 0;
282
283 USATACC(16, 0);
284 USATACC(16, 16);
285 return r;
286 }
287
288 #undef USATACC
289
290 uint32_t HELPER(neon_uqadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
291 {
292 int64_t va = (int32_t)a;
293 int64_t vb = (uint32_t)b;
294 int64_t vr = va + vb;
295 if (vr > UINT32_MAX) {
296 SET_QC();
297 vr = UINT32_MAX;
298 } else if (vr < 0) {
299 SET_QC();
300 vr = 0;
301 }
302 return vr;
303 }
304
305 uint64_t HELPER(neon_uqadd_s64)(CPUARMState *env, uint64_t a, uint64_t b)
306 {
307 uint64_t res;
308 res = a + b;
309 /* We only need to look at the pattern of SIGN bits to detect
310 * +ve/-ve saturation
311 */
312 if (~a & b & ~res & SIGNBIT64) {
313 SET_QC();
314 res = UINT64_MAX;
315 } else if (a & ~b & res & SIGNBIT64) {
316 SET_QC();
317 res = 0;
318 }
319 return res;
320 }
321
322 /* Signed saturating accumulate of unsigned value
323 *
324 * Op1/Rn is treated as unsigned
325 * Op2/Rd is treated as signed
326 *
327 * The result is treated as a signed value and saturated as such
328 *
329 * We use a macro for the 8/16 bit cases which expects signed integers of va,
330 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
331 */
332
333 #define SSATACC(bits, shift) \
334 do { \
335 va = extract32(a, shift, bits); \
336 vb = sextract32(b, shift, bits); \
337 vr = va + vb; \
338 if (vr > INT##bits##_MAX) { \
339 SET_QC(); \
340 vr = INT##bits##_MAX; \
341 } else if (vr < INT##bits##_MIN) { \
342 SET_QC(); \
343 vr = INT##bits##_MIN; \
344 } \
345 r = deposit32(r, shift, bits, vr); \
346 } while (0)
347
348 uint32_t HELPER(neon_sqadd_u8)(CPUARMState *env, uint32_t a, uint32_t b)
349 {
350 int16_t va, vb, vr;
351 uint32_t r = 0;
352
353 SSATACC(8, 0);
354 SSATACC(8, 8);
355 SSATACC(8, 16);
356 SSATACC(8, 24);
357 return r;
358 }
359
360 uint32_t HELPER(neon_sqadd_u16)(CPUARMState *env, uint32_t a, uint32_t b)
361 {
362 int32_t va, vb, vr;
363 uint32_t r = 0;
364
365 SSATACC(16, 0);
366 SSATACC(16, 16);
367
368 return r;
369 }
370
371 #undef SSATACC
372
373 uint32_t HELPER(neon_sqadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
374 {
375 int64_t res;
376 int64_t op1 = (uint32_t)a;
377 int64_t op2 = (int32_t)b;
378 res = op1 + op2;
379 if (res > INT32_MAX) {
380 SET_QC();
381 res = INT32_MAX;
382 } else if (res < INT32_MIN) {
383 SET_QC();
384 res = INT32_MIN;
385 }
386 return res;
387 }
388
389 uint64_t HELPER(neon_sqadd_u64)(CPUARMState *env, uint64_t a, uint64_t b)
390 {
391 uint64_t res;
392 res = a + b;
393 /* We only need to look at the pattern of SIGN bits to detect an overflow */
394 if (((a & res)
395 | (~b & res)
396 | (a & ~b)) & SIGNBIT64) {
397 SET_QC();
398 res = INT64_MAX;
399 }
400 return res;
401 }
402
403
404 #define NEON_USAT(dest, src1, src2, type) do { \
405 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
406 if (tmp != (type)tmp) { \
407 SET_QC(); \
408 dest = 0; \
409 } else { \
410 dest = tmp; \
411 }} while(0)
412 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
413 NEON_VOP_ENV(qsub_u8, neon_u8, 4)
414 #undef NEON_FN
415 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
416 NEON_VOP_ENV(qsub_u16, neon_u16, 2)
417 #undef NEON_FN
418 #undef NEON_USAT
419
420 uint32_t HELPER(neon_qsub_u32)(CPUARMState *env, uint32_t a, uint32_t b)
421 {
422 uint32_t res = a - b;
423 if (res > a) {
424 SET_QC();
425 res = 0;
426 }
427 return res;
428 }
429
430 uint64_t HELPER(neon_qsub_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
431 {
432 uint64_t res;
433
434 if (src1 < src2) {
435 SET_QC();
436 res = 0;
437 } else {
438 res = src1 - src2;
439 }
440 return res;
441 }
442
443 #define NEON_SSAT(dest, src1, src2, type) do { \
444 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
445 if (tmp != (type)tmp) { \
446 SET_QC(); \
447 if (src2 < 0) { \
448 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
449 } else { \
450 tmp = 1 << (sizeof(type) * 8 - 1); \
451 } \
452 } \
453 dest = tmp; \
454 } while(0)
455 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
456 NEON_VOP_ENV(qsub_s8, neon_s8, 4)
457 #undef NEON_FN
458 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
459 NEON_VOP_ENV(qsub_s16, neon_s16, 2)
460 #undef NEON_FN
461 #undef NEON_SSAT
462
463 uint32_t HELPER(neon_qsub_s32)(CPUARMState *env, uint32_t a, uint32_t b)
464 {
465 uint32_t res = a - b;
466 if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
467 SET_QC();
468 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
469 }
470 return res;
471 }
472
473 uint64_t HELPER(neon_qsub_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
474 {
475 uint64_t res;
476
477 res = src1 - src2;
478 if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
479 SET_QC();
480 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
481 }
482 return res;
483 }
484
485 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
486 NEON_VOP(hadd_s8, neon_s8, 4)
487 NEON_VOP(hadd_u8, neon_u8, 4)
488 NEON_VOP(hadd_s16, neon_s16, 2)
489 NEON_VOP(hadd_u16, neon_u16, 2)
490 #undef NEON_FN
491
492 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
493 {
494 int32_t dest;
495
496 dest = (src1 >> 1) + (src2 >> 1);
497 if (src1 & src2 & 1)
498 dest++;
499 return dest;
500 }
501
502 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
503 {
504 uint32_t dest;
505
506 dest = (src1 >> 1) + (src2 >> 1);
507 if (src1 & src2 & 1)
508 dest++;
509 return dest;
510 }
511
512 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
513 NEON_VOP(rhadd_s8, neon_s8, 4)
514 NEON_VOP(rhadd_u8, neon_u8, 4)
515 NEON_VOP(rhadd_s16, neon_s16, 2)
516 NEON_VOP(rhadd_u16, neon_u16, 2)
517 #undef NEON_FN
518
519 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
520 {
521 int32_t dest;
522
523 dest = (src1 >> 1) + (src2 >> 1);
524 if ((src1 | src2) & 1)
525 dest++;
526 return dest;
527 }
528
529 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
530 {
531 uint32_t dest;
532
533 dest = (src1 >> 1) + (src2 >> 1);
534 if ((src1 | src2) & 1)
535 dest++;
536 return dest;
537 }
538
539 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
540 NEON_VOP(hsub_s8, neon_s8, 4)
541 NEON_VOP(hsub_u8, neon_u8, 4)
542 NEON_VOP(hsub_s16, neon_s16, 2)
543 NEON_VOP(hsub_u16, neon_u16, 2)
544 #undef NEON_FN
545
546 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
547 {
548 int32_t dest;
549
550 dest = (src1 >> 1) - (src2 >> 1);
551 if ((~src1) & src2 & 1)
552 dest--;
553 return dest;
554 }
555
556 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
557 {
558 uint32_t dest;
559
560 dest = (src1 >> 1) - (src2 >> 1);
561 if ((~src1) & src2 & 1)
562 dest--;
563 return dest;
564 }
565
566 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
567 NEON_VOP(cgt_s8, neon_s8, 4)
568 NEON_VOP(cgt_u8, neon_u8, 4)
569 NEON_VOP(cgt_s16, neon_s16, 2)
570 NEON_VOP(cgt_u16, neon_u16, 2)
571 NEON_VOP(cgt_s32, neon_s32, 1)
572 NEON_VOP(cgt_u32, neon_u32, 1)
573 #undef NEON_FN
574
575 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
576 NEON_VOP(cge_s8, neon_s8, 4)
577 NEON_VOP(cge_u8, neon_u8, 4)
578 NEON_VOP(cge_s16, neon_s16, 2)
579 NEON_VOP(cge_u16, neon_u16, 2)
580 NEON_VOP(cge_s32, neon_s32, 1)
581 NEON_VOP(cge_u32, neon_u32, 1)
582 #undef NEON_FN
583
584 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
585 NEON_VOP(min_s8, neon_s8, 4)
586 NEON_VOP(min_u8, neon_u8, 4)
587 NEON_VOP(min_s16, neon_s16, 2)
588 NEON_VOP(min_u16, neon_u16, 2)
589 NEON_VOP(min_s32, neon_s32, 1)
590 NEON_VOP(min_u32, neon_u32, 1)
591 NEON_POP(pmin_s8, neon_s8, 4)
592 NEON_POP(pmin_u8, neon_u8, 4)
593 NEON_POP(pmin_s16, neon_s16, 2)
594 NEON_POP(pmin_u16, neon_u16, 2)
595 #undef NEON_FN
596
597 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
598 NEON_VOP(max_s8, neon_s8, 4)
599 NEON_VOP(max_u8, neon_u8, 4)
600 NEON_VOP(max_s16, neon_s16, 2)
601 NEON_VOP(max_u16, neon_u16, 2)
602 NEON_VOP(max_s32, neon_s32, 1)
603 NEON_VOP(max_u32, neon_u32, 1)
604 NEON_POP(pmax_s8, neon_s8, 4)
605 NEON_POP(pmax_u8, neon_u8, 4)
606 NEON_POP(pmax_s16, neon_s16, 2)
607 NEON_POP(pmax_u16, neon_u16, 2)
608 #undef NEON_FN
609
610 #define NEON_FN(dest, src1, src2) \
611 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
612 NEON_VOP(abd_s8, neon_s8, 4)
613 NEON_VOP(abd_u8, neon_u8, 4)
614 NEON_VOP(abd_s16, neon_s16, 2)
615 NEON_VOP(abd_u16, neon_u16, 2)
616 NEON_VOP(abd_s32, neon_s32, 1)
617 NEON_VOP(abd_u32, neon_u32, 1)
618 #undef NEON_FN
619
620 #define NEON_FN(dest, src1, src2) do { \
621 int8_t tmp; \
622 tmp = (int8_t)src2; \
623 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
624 tmp <= -(ssize_t)sizeof(src1) * 8) { \
625 dest = 0; \
626 } else if (tmp < 0) { \
627 dest = src1 >> -tmp; \
628 } else { \
629 dest = src1 << tmp; \
630 }} while (0)
631 NEON_VOP(shl_u8, neon_u8, 4)
632 NEON_VOP(shl_u16, neon_u16, 2)
633 NEON_VOP(shl_u32, neon_u32, 1)
634 #undef NEON_FN
635
636 uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
637 {
638 int8_t shift = (int8_t)shiftop;
639 if (shift >= 64 || shift <= -64) {
640 val = 0;
641 } else if (shift < 0) {
642 val >>= -shift;
643 } else {
644 val <<= shift;
645 }
646 return val;
647 }
648
649 #define NEON_FN(dest, src1, src2) do { \
650 int8_t tmp; \
651 tmp = (int8_t)src2; \
652 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
653 dest = 0; \
654 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
655 dest = src1 >> (sizeof(src1) * 8 - 1); \
656 } else if (tmp < 0) { \
657 dest = src1 >> -tmp; \
658 } else { \
659 dest = src1 << tmp; \
660 }} while (0)
661 NEON_VOP(shl_s8, neon_s8, 4)
662 NEON_VOP(shl_s16, neon_s16, 2)
663 NEON_VOP(shl_s32, neon_s32, 1)
664 #undef NEON_FN
665
666 uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
667 {
668 int8_t shift = (int8_t)shiftop;
669 int64_t val = valop;
670 if (shift >= 64) {
671 val = 0;
672 } else if (shift <= -64) {
673 val >>= 63;
674 } else if (shift < 0) {
675 val >>= -shift;
676 } else {
677 val <<= shift;
678 }
679 return val;
680 }
681
682 #define NEON_FN(dest, src1, src2) do { \
683 int8_t tmp; \
684 tmp = (int8_t)src2; \
685 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
686 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
687 dest = 0; \
688 } else if (tmp < 0) { \
689 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
690 } else { \
691 dest = src1 << tmp; \
692 }} while (0)
693 NEON_VOP(rshl_s8, neon_s8, 4)
694 NEON_VOP(rshl_s16, neon_s16, 2)
695 #undef NEON_FN
696
697 /* The addition of the rounding constant may overflow, so we use an
698 * intermediate 64 bit accumulator. */
699 uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
700 {
701 int32_t dest;
702 int32_t val = (int32_t)valop;
703 int8_t shift = (int8_t)shiftop;
704 if ((shift >= 32) || (shift <= -32)) {
705 dest = 0;
706 } else if (shift < 0) {
707 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
708 dest = big_dest >> -shift;
709 } else {
710 dest = val << shift;
711 }
712 return dest;
713 }
714
715 /* Handling addition overflow with 64 bit input values is more
716 * tricky than with 32 bit values. */
717 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
718 {
719 int8_t shift = (int8_t)shiftop;
720 int64_t val = valop;
721 if ((shift >= 64) || (shift <= -64)) {
722 val = 0;
723 } else if (shift < 0) {
724 val >>= (-shift - 1);
725 if (val == INT64_MAX) {
726 /* In this case, it means that the rounding constant is 1,
727 * and the addition would overflow. Return the actual
728 * result directly. */
729 val = 0x4000000000000000LL;
730 } else {
731 val++;
732 val >>= 1;
733 }
734 } else {
735 val <<= shift;
736 }
737 return val;
738 }
739
740 #define NEON_FN(dest, src1, src2) do { \
741 int8_t tmp; \
742 tmp = (int8_t)src2; \
743 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
744 tmp < -(ssize_t)sizeof(src1) * 8) { \
745 dest = 0; \
746 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
747 dest = src1 >> (-tmp - 1); \
748 } else if (tmp < 0) { \
749 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
750 } else { \
751 dest = src1 << tmp; \
752 }} while (0)
753 NEON_VOP(rshl_u8, neon_u8, 4)
754 NEON_VOP(rshl_u16, neon_u16, 2)
755 #undef NEON_FN
756
757 /* The addition of the rounding constant may overflow, so we use an
758 * intermediate 64 bit accumulator. */
759 uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
760 {
761 uint32_t dest;
762 int8_t shift = (int8_t)shiftop;
763 if (shift >= 32 || shift < -32) {
764 dest = 0;
765 } else if (shift == -32) {
766 dest = val >> 31;
767 } else if (shift < 0) {
768 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
769 dest = big_dest >> -shift;
770 } else {
771 dest = val << shift;
772 }
773 return dest;
774 }
775
776 /* Handling addition overflow with 64 bit input values is more
777 * tricky than with 32 bit values. */
778 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
779 {
780 int8_t shift = (uint8_t)shiftop;
781 if (shift >= 64 || shift < -64) {
782 val = 0;
783 } else if (shift == -64) {
784 /* Rounding a 1-bit result just preserves that bit. */
785 val >>= 63;
786 } else if (shift < 0) {
787 val >>= (-shift - 1);
788 if (val == UINT64_MAX) {
789 /* In this case, it means that the rounding constant is 1,
790 * and the addition would overflow. Return the actual
791 * result directly. */
792 val = 0x8000000000000000ULL;
793 } else {
794 val++;
795 val >>= 1;
796 }
797 } else {
798 val <<= shift;
799 }
800 return val;
801 }
802
803 #define NEON_FN(dest, src1, src2) do { \
804 int8_t tmp; \
805 tmp = (int8_t)src2; \
806 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
807 if (src1) { \
808 SET_QC(); \
809 dest = ~0; \
810 } else { \
811 dest = 0; \
812 } \
813 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
814 dest = 0; \
815 } else if (tmp < 0) { \
816 dest = src1 >> -tmp; \
817 } else { \
818 dest = src1 << tmp; \
819 if ((dest >> tmp) != src1) { \
820 SET_QC(); \
821 dest = ~0; \
822 } \
823 }} while (0)
824 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
825 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
826 NEON_VOP_ENV(qshl_u32, neon_u32, 1)
827 #undef NEON_FN
828
829 uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
830 {
831 int8_t shift = (int8_t)shiftop;
832 if (shift >= 64) {
833 if (val) {
834 val = ~(uint64_t)0;
835 SET_QC();
836 }
837 } else if (shift <= -64) {
838 val = 0;
839 } else if (shift < 0) {
840 val >>= -shift;
841 } else {
842 uint64_t tmp = val;
843 val <<= shift;
844 if ((val >> shift) != tmp) {
845 SET_QC();
846 val = ~(uint64_t)0;
847 }
848 }
849 return val;
850 }
851
852 #define NEON_FN(dest, src1, src2) do { \
853 int8_t tmp; \
854 tmp = (int8_t)src2; \
855 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
856 if (src1) { \
857 SET_QC(); \
858 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
859 if (src1 > 0) { \
860 dest--; \
861 } \
862 } else { \
863 dest = src1; \
864 } \
865 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
866 dest = src1 >> 31; \
867 } else if (tmp < 0) { \
868 dest = src1 >> -tmp; \
869 } else { \
870 dest = src1 << tmp; \
871 if ((dest >> tmp) != src1) { \
872 SET_QC(); \
873 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
874 if (src1 > 0) { \
875 dest--; \
876 } \
877 } \
878 }} while (0)
879 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
880 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
881 NEON_VOP_ENV(qshl_s32, neon_s32, 1)
882 #undef NEON_FN
883
884 uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
885 {
886 int8_t shift = (uint8_t)shiftop;
887 int64_t val = valop;
888 if (shift >= 64) {
889 if (val) {
890 SET_QC();
891 val = (val >> 63) ^ ~SIGNBIT64;
892 }
893 } else if (shift <= -64) {
894 val >>= 63;
895 } else if (shift < 0) {
896 val >>= -shift;
897 } else {
898 int64_t tmp = val;
899 val <<= shift;
900 if ((val >> shift) != tmp) {
901 SET_QC();
902 val = (tmp >> 63) ^ ~SIGNBIT64;
903 }
904 }
905 return val;
906 }
907
908 #define NEON_FN(dest, src1, src2) do { \
909 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
910 SET_QC(); \
911 dest = 0; \
912 } else { \
913 int8_t tmp; \
914 tmp = (int8_t)src2; \
915 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
916 if (src1) { \
917 SET_QC(); \
918 dest = ~0; \
919 } else { \
920 dest = 0; \
921 } \
922 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
923 dest = 0; \
924 } else if (tmp < 0) { \
925 dest = src1 >> -tmp; \
926 } else { \
927 dest = src1 << tmp; \
928 if ((dest >> tmp) != src1) { \
929 SET_QC(); \
930 dest = ~0; \
931 } \
932 } \
933 }} while (0)
934 NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
935 NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
936 #undef NEON_FN
937
938 uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
939 {
940 if ((int32_t)valop < 0) {
941 SET_QC();
942 return 0;
943 }
944 return helper_neon_qshl_u32(env, valop, shiftop);
945 }
946
947 uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
948 {
949 if ((int64_t)valop < 0) {
950 SET_QC();
951 return 0;
952 }
953 return helper_neon_qshl_u64(env, valop, shiftop);
954 }
955
956 #define NEON_FN(dest, src1, src2) do { \
957 int8_t tmp; \
958 tmp = (int8_t)src2; \
959 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
960 if (src1) { \
961 SET_QC(); \
962 dest = ~0; \
963 } else { \
964 dest = 0; \
965 } \
966 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
967 dest = 0; \
968 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
969 dest = src1 >> (sizeof(src1) * 8 - 1); \
970 } else if (tmp < 0) { \
971 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
972 } else { \
973 dest = src1 << tmp; \
974 if ((dest >> tmp) != src1) { \
975 SET_QC(); \
976 dest = ~0; \
977 } \
978 }} while (0)
979 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
980 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
981 #undef NEON_FN
982
983 /* The addition of the rounding constant may overflow, so we use an
984 * intermediate 64 bit accumulator. */
985 uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
986 {
987 uint32_t dest;
988 int8_t shift = (int8_t)shiftop;
989 if (shift >= 32) {
990 if (val) {
991 SET_QC();
992 dest = ~0;
993 } else {
994 dest = 0;
995 }
996 } else if (shift < -32) {
997 dest = 0;
998 } else if (shift == -32) {
999 dest = val >> 31;
1000 } else if (shift < 0) {
1001 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
1002 dest = big_dest >> -shift;
1003 } else {
1004 dest = val << shift;
1005 if ((dest >> shift) != val) {
1006 SET_QC();
1007 dest = ~0;
1008 }
1009 }
1010 return dest;
1011 }
1012
1013 /* Handling addition overflow with 64 bit input values is more
1014 * tricky than with 32 bit values. */
1015 uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
1016 {
1017 int8_t shift = (int8_t)shiftop;
1018 if (shift >= 64) {
1019 if (val) {
1020 SET_QC();
1021 val = ~0;
1022 }
1023 } else if (shift < -64) {
1024 val = 0;
1025 } else if (shift == -64) {
1026 val >>= 63;
1027 } else if (shift < 0) {
1028 val >>= (-shift - 1);
1029 if (val == UINT64_MAX) {
1030 /* In this case, it means that the rounding constant is 1,
1031 * and the addition would overflow. Return the actual
1032 * result directly. */
1033 val = 0x8000000000000000ULL;
1034 } else {
1035 val++;
1036 val >>= 1;
1037 }
1038 } else { \
1039 uint64_t tmp = val;
1040 val <<= shift;
1041 if ((val >> shift) != tmp) {
1042 SET_QC();
1043 val = ~0;
1044 }
1045 }
1046 return val;
1047 }
1048
1049 #define NEON_FN(dest, src1, src2) do { \
1050 int8_t tmp; \
1051 tmp = (int8_t)src2; \
1052 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
1053 if (src1) { \
1054 SET_QC(); \
1055 dest = (1 << (sizeof(src1) * 8 - 1)); \
1056 if (src1 > 0) { \
1057 dest--; \
1058 } \
1059 } else { \
1060 dest = 0; \
1061 } \
1062 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
1063 dest = 0; \
1064 } else if (tmp < 0) { \
1065 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
1066 } else { \
1067 dest = src1 << tmp; \
1068 if ((dest >> tmp) != src1) { \
1069 SET_QC(); \
1070 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
1071 if (src1 > 0) { \
1072 dest--; \
1073 } \
1074 } \
1075 }} while (0)
1076 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
1077 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
1078 #undef NEON_FN
1079
1080 /* The addition of the rounding constant may overflow, so we use an
1081 * intermediate 64 bit accumulator. */
1082 uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
1083 {
1084 int32_t dest;
1085 int32_t val = (int32_t)valop;
1086 int8_t shift = (int8_t)shiftop;
1087 if (shift >= 32) {
1088 if (val) {
1089 SET_QC();
1090 dest = (val >> 31) ^ ~SIGNBIT;
1091 } else {
1092 dest = 0;
1093 }
1094 } else if (shift <= -32) {
1095 dest = 0;
1096 } else if (shift < 0) {
1097 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
1098 dest = big_dest >> -shift;
1099 } else {
1100 dest = val << shift;
1101 if ((dest >> shift) != val) {
1102 SET_QC();
1103 dest = (val >> 31) ^ ~SIGNBIT;
1104 }
1105 }
1106 return dest;
1107 }
1108
1109 /* Handling addition overflow with 64 bit input values is more
1110 * tricky than with 32 bit values. */
1111 uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
1112 {
1113 int8_t shift = (uint8_t)shiftop;
1114 int64_t val = valop;
1115
1116 if (shift >= 64) {
1117 if (val) {
1118 SET_QC();
1119 val = (val >> 63) ^ ~SIGNBIT64;
1120 }
1121 } else if (shift <= -64) {
1122 val = 0;
1123 } else if (shift < 0) {
1124 val >>= (-shift - 1);
1125 if (val == INT64_MAX) {
1126 /* In this case, it means that the rounding constant is 1,
1127 * and the addition would overflow. Return the actual
1128 * result directly. */
1129 val = 0x4000000000000000ULL;
1130 } else {
1131 val++;
1132 val >>= 1;
1133 }
1134 } else {
1135 int64_t tmp = val;
1136 val <<= shift;
1137 if ((val >> shift) != tmp) {
1138 SET_QC();
1139 val = (tmp >> 63) ^ ~SIGNBIT64;
1140 }
1141 }
1142 return val;
1143 }
1144
1145 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
1146 {
1147 uint32_t mask;
1148 mask = (a ^ b) & 0x80808080u;
1149 a &= ~0x80808080u;
1150 b &= ~0x80808080u;
1151 return (a + b) ^ mask;
1152 }
1153
1154 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
1155 {
1156 uint32_t mask;
1157 mask = (a ^ b) & 0x80008000u;
1158 a &= ~0x80008000u;
1159 b &= ~0x80008000u;
1160 return (a + b) ^ mask;
1161 }
1162
1163 #define NEON_FN(dest, src1, src2) dest = src1 + src2
1164 NEON_POP(padd_u8, neon_u8, 4)
1165 NEON_POP(padd_u16, neon_u16, 2)
1166 #undef NEON_FN
1167
1168 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1169 NEON_VOP(sub_u8, neon_u8, 4)
1170 NEON_VOP(sub_u16, neon_u16, 2)
1171 #undef NEON_FN
1172
1173 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1174 NEON_VOP(mul_u8, neon_u8, 4)
1175 NEON_VOP(mul_u16, neon_u16, 2)
1176 #undef NEON_FN
1177
1178 /* Polynomial multiplication is like integer multiplication except the
1179 partial products are XORed, not added. */
1180 uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
1181 {
1182 uint32_t mask;
1183 uint32_t result;
1184 result = 0;
1185 while (op1) {
1186 mask = 0;
1187 if (op1 & 1)
1188 mask |= 0xff;
1189 if (op1 & (1 << 8))
1190 mask |= (0xff << 8);
1191 if (op1 & (1 << 16))
1192 mask |= (0xff << 16);
1193 if (op1 & (1 << 24))
1194 mask |= (0xff << 24);
1195 result ^= op2 & mask;
1196 op1 = (op1 >> 1) & 0x7f7f7f7f;
1197 op2 = (op2 << 1) & 0xfefefefe;
1198 }
1199 return result;
1200 }
1201
1202 uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
1203 {
1204 uint64_t result = 0;
1205 uint64_t mask;
1206 uint64_t op2ex = op2;
1207 op2ex = (op2ex & 0xff) |
1208 ((op2ex & 0xff00) << 8) |
1209 ((op2ex & 0xff0000) << 16) |
1210 ((op2ex & 0xff000000) << 24);
1211 while (op1) {
1212 mask = 0;
1213 if (op1 & 1) {
1214 mask |= 0xffff;
1215 }
1216 if (op1 & (1 << 8)) {
1217 mask |= (0xffffU << 16);
1218 }
1219 if (op1 & (1 << 16)) {
1220 mask |= (0xffffULL << 32);
1221 }
1222 if (op1 & (1 << 24)) {
1223 mask |= (0xffffULL << 48);
1224 }
1225 result ^= op2ex & mask;
1226 op1 = (op1 >> 1) & 0x7f7f7f7f;
1227 op2ex <<= 1;
1228 }
1229 return result;
1230 }
1231
1232 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1233 NEON_VOP(tst_u8, neon_u8, 4)
1234 NEON_VOP(tst_u16, neon_u16, 2)
1235 NEON_VOP(tst_u32, neon_u32, 1)
1236 #undef NEON_FN
1237
1238 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1239 NEON_VOP(ceq_u8, neon_u8, 4)
1240 NEON_VOP(ceq_u16, neon_u16, 2)
1241 NEON_VOP(ceq_u32, neon_u32, 1)
1242 #undef NEON_FN
1243
1244 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
1245 NEON_VOP1(abs_s8, neon_s8, 4)
1246 NEON_VOP1(abs_s16, neon_s16, 2)
1247 #undef NEON_FN
1248
1249 /* Count Leading Sign/Zero Bits. */
1250 static inline int do_clz8(uint8_t x)
1251 {
1252 int n;
1253 for (n = 8; x; n--)
1254 x >>= 1;
1255 return n;
1256 }
1257
1258 static inline int do_clz16(uint16_t x)
1259 {
1260 int n;
1261 for (n = 16; x; n--)
1262 x >>= 1;
1263 return n;
1264 }
1265
1266 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1267 NEON_VOP1(clz_u8, neon_u8, 4)
1268 #undef NEON_FN
1269
1270 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1271 NEON_VOP1(clz_u16, neon_u16, 2)
1272 #undef NEON_FN
1273
1274 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1275 NEON_VOP1(cls_s8, neon_s8, 4)
1276 #undef NEON_FN
1277
1278 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1279 NEON_VOP1(cls_s16, neon_s16, 2)
1280 #undef NEON_FN
1281
1282 uint32_t HELPER(neon_cls_s32)(uint32_t x)
1283 {
1284 int count;
1285 if ((int32_t)x < 0)
1286 x = ~x;
1287 for (count = 32; x; count--)
1288 x = x >> 1;
1289 return count - 1;
1290 }
1291
1292 /* Bit count. */
1293 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1294 {
1295 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
1296 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
1297 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
1298 return x;
1299 }
1300
1301 /* Reverse bits in each 8 bit word */
1302 uint32_t HELPER(neon_rbit_u8)(uint32_t x)
1303 {
1304 x = ((x & 0xf0f0f0f0) >> 4)
1305 | ((x & 0x0f0f0f0f) << 4);
1306 x = ((x & 0x88888888) >> 3)
1307 | ((x & 0x44444444) >> 1)
1308 | ((x & 0x22222222) << 1)
1309 | ((x & 0x11111111) << 3);
1310 return x;
1311 }
1312
1313 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1314 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1315 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1316 SET_QC(); \
1317 tmp = (tmp >> 31) ^ ~SIGNBIT; \
1318 } else { \
1319 tmp <<= 1; \
1320 } \
1321 if (round) { \
1322 int32_t old = tmp; \
1323 tmp += 1 << 15; \
1324 if ((int32_t)tmp < old) { \
1325 SET_QC(); \
1326 tmp = SIGNBIT - 1; \
1327 } \
1328 } \
1329 dest = tmp >> 16; \
1330 } while(0)
1331 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1332 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
1333 #undef NEON_FN
1334 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1335 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
1336 #undef NEON_FN
1337 #undef NEON_QDMULH16
1338
1339 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1340 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1341 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1342 SET_QC(); \
1343 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1344 } else { \
1345 tmp <<= 1; \
1346 } \
1347 if (round) { \
1348 int64_t old = tmp; \
1349 tmp += (int64_t)1 << 31; \
1350 if ((int64_t)tmp < old) { \
1351 SET_QC(); \
1352 tmp = SIGNBIT64 - 1; \
1353 } \
1354 } \
1355 dest = tmp >> 32; \
1356 } while(0)
1357 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1358 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
1359 #undef NEON_FN
1360 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1361 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
1362 #undef NEON_FN
1363 #undef NEON_QDMULH32
1364
1365 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1366 {
1367 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1368 | ((x >> 24) & 0xff000000u);
1369 }
1370
1371 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1372 {
1373 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1374 }
1375
1376 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1377 {
1378 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1379 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1380 }
1381
1382 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1383 {
1384 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1385 }
1386
1387 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1388 {
1389 x &= 0xff80ff80ff80ff80ull;
1390 x += 0x0080008000800080ull;
1391 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1392 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1393 }
1394
1395 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1396 {
1397 x &= 0xffff8000ffff8000ull;
1398 x += 0x0000800000008000ull;
1399 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1400 }
1401
1402 uint32_t HELPER(neon_unarrow_sat8)(CPUARMState *env, uint64_t x)
1403 {
1404 uint16_t s;
1405 uint8_t d;
1406 uint32_t res = 0;
1407 #define SAT8(n) \
1408 s = x >> n; \
1409 if (s & 0x8000) { \
1410 SET_QC(); \
1411 } else { \
1412 if (s > 0xff) { \
1413 d = 0xff; \
1414 SET_QC(); \
1415 } else { \
1416 d = s; \
1417 } \
1418 res |= (uint32_t)d << (n / 2); \
1419 }
1420
1421 SAT8(0);
1422 SAT8(16);
1423 SAT8(32);
1424 SAT8(48);
1425 #undef SAT8
1426 return res;
1427 }
1428
1429 uint32_t HELPER(neon_narrow_sat_u8)(CPUARMState *env, uint64_t x)
1430 {
1431 uint16_t s;
1432 uint8_t d;
1433 uint32_t res = 0;
1434 #define SAT8(n) \
1435 s = x >> n; \
1436 if (s > 0xff) { \
1437 d = 0xff; \
1438 SET_QC(); \
1439 } else { \
1440 d = s; \
1441 } \
1442 res |= (uint32_t)d << (n / 2);
1443
1444 SAT8(0);
1445 SAT8(16);
1446 SAT8(32);
1447 SAT8(48);
1448 #undef SAT8
1449 return res;
1450 }
1451
1452 uint32_t HELPER(neon_narrow_sat_s8)(CPUARMState *env, uint64_t x)
1453 {
1454 int16_t s;
1455 uint8_t d;
1456 uint32_t res = 0;
1457 #define SAT8(n) \
1458 s = x >> n; \
1459 if (s != (int8_t)s) { \
1460 d = (s >> 15) ^ 0x7f; \
1461 SET_QC(); \
1462 } else { \
1463 d = s; \
1464 } \
1465 res |= (uint32_t)d << (n / 2);
1466
1467 SAT8(0);
1468 SAT8(16);
1469 SAT8(32);
1470 SAT8(48);
1471 #undef SAT8
1472 return res;
1473 }
1474
1475 uint32_t HELPER(neon_unarrow_sat16)(CPUARMState *env, uint64_t x)
1476 {
1477 uint32_t high;
1478 uint32_t low;
1479 low = x;
1480 if (low & 0x80000000) {
1481 low = 0;
1482 SET_QC();
1483 } else if (low > 0xffff) {
1484 low = 0xffff;
1485 SET_QC();
1486 }
1487 high = x >> 32;
1488 if (high & 0x80000000) {
1489 high = 0;
1490 SET_QC();
1491 } else if (high > 0xffff) {
1492 high = 0xffff;
1493 SET_QC();
1494 }
1495 return low | (high << 16);
1496 }
1497
1498 uint32_t HELPER(neon_narrow_sat_u16)(CPUARMState *env, uint64_t x)
1499 {
1500 uint32_t high;
1501 uint32_t low;
1502 low = x;
1503 if (low > 0xffff) {
1504 low = 0xffff;
1505 SET_QC();
1506 }
1507 high = x >> 32;
1508 if (high > 0xffff) {
1509 high = 0xffff;
1510 SET_QC();
1511 }
1512 return low | (high << 16);
1513 }
1514
1515 uint32_t HELPER(neon_narrow_sat_s16)(CPUARMState *env, uint64_t x)
1516 {
1517 int32_t low;
1518 int32_t high;
1519 low = x;
1520 if (low != (int16_t)low) {
1521 low = (low >> 31) ^ 0x7fff;
1522 SET_QC();
1523 }
1524 high = x >> 32;
1525 if (high != (int16_t)high) {
1526 high = (high >> 31) ^ 0x7fff;
1527 SET_QC();
1528 }
1529 return (uint16_t)low | (high << 16);
1530 }
1531
1532 uint32_t HELPER(neon_unarrow_sat32)(CPUARMState *env, uint64_t x)
1533 {
1534 if (x & 0x8000000000000000ull) {
1535 SET_QC();
1536 return 0;
1537 }
1538 if (x > 0xffffffffu) {
1539 SET_QC();
1540 return 0xffffffffu;
1541 }
1542 return x;
1543 }
1544
1545 uint32_t HELPER(neon_narrow_sat_u32)(CPUARMState *env, uint64_t x)
1546 {
1547 if (x > 0xffffffffu) {
1548 SET_QC();
1549 return 0xffffffffu;
1550 }
1551 return x;
1552 }
1553
1554 uint32_t HELPER(neon_narrow_sat_s32)(CPUARMState *env, uint64_t x)
1555 {
1556 if ((int64_t)x != (int32_t)x) {
1557 SET_QC();
1558 return ((int64_t)x >> 63) ^ 0x7fffffff;
1559 }
1560 return x;
1561 }
1562
1563 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1564 {
1565 uint64_t tmp;
1566 uint64_t ret;
1567 ret = (uint8_t)x;
1568 tmp = (uint8_t)(x >> 8);
1569 ret |= tmp << 16;
1570 tmp = (uint8_t)(x >> 16);
1571 ret |= tmp << 32;
1572 tmp = (uint8_t)(x >> 24);
1573 ret |= tmp << 48;
1574 return ret;
1575 }
1576
1577 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1578 {
1579 uint64_t tmp;
1580 uint64_t ret;
1581 ret = (uint16_t)(int8_t)x;
1582 tmp = (uint16_t)(int8_t)(x >> 8);
1583 ret |= tmp << 16;
1584 tmp = (uint16_t)(int8_t)(x >> 16);
1585 ret |= tmp << 32;
1586 tmp = (uint16_t)(int8_t)(x >> 24);
1587 ret |= tmp << 48;
1588 return ret;
1589 }
1590
1591 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1592 {
1593 uint64_t high = (uint16_t)(x >> 16);
1594 return ((uint16_t)x) | (high << 32);
1595 }
1596
1597 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1598 {
1599 uint64_t high = (int16_t)(x >> 16);
1600 return ((uint32_t)(int16_t)x) | (high << 32);
1601 }
1602
1603 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1604 {
1605 uint64_t mask;
1606 mask = (a ^ b) & 0x8000800080008000ull;
1607 a &= ~0x8000800080008000ull;
1608 b &= ~0x8000800080008000ull;
1609 return (a + b) ^ mask;
1610 }
1611
1612 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1613 {
1614 uint64_t mask;
1615 mask = (a ^ b) & 0x8000000080000000ull;
1616 a &= ~0x8000000080000000ull;
1617 b &= ~0x8000000080000000ull;
1618 return (a + b) ^ mask;
1619 }
1620
1621 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1622 {
1623 uint64_t tmp;
1624 uint64_t tmp2;
1625
1626 tmp = a & 0x0000ffff0000ffffull;
1627 tmp += (a >> 16) & 0x0000ffff0000ffffull;
1628 tmp2 = b & 0xffff0000ffff0000ull;
1629 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1630 return ( tmp & 0xffff)
1631 | ((tmp >> 16) & 0xffff0000ull)
1632 | ((tmp2 << 16) & 0xffff00000000ull)
1633 | ( tmp2 & 0xffff000000000000ull);
1634 }
1635
1636 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1637 {
1638 uint32_t low = a + (a >> 32);
1639 uint32_t high = b + (b >> 32);
1640 return low + ((uint64_t)high << 32);
1641 }
1642
1643 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1644 {
1645 uint64_t mask;
1646 mask = (a ^ ~b) & 0x8000800080008000ull;
1647 a |= 0x8000800080008000ull;
1648 b &= ~0x8000800080008000ull;
1649 return (a - b) ^ mask;
1650 }
1651
1652 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1653 {
1654 uint64_t mask;
1655 mask = (a ^ ~b) & 0x8000000080000000ull;
1656 a |= 0x8000000080000000ull;
1657 b &= ~0x8000000080000000ull;
1658 return (a - b) ^ mask;
1659 }
1660
1661 uint64_t HELPER(neon_addl_saturate_s32)(CPUARMState *env, uint64_t a, uint64_t b)
1662 {
1663 uint32_t x, y;
1664 uint32_t low, high;
1665
1666 x = a;
1667 y = b;
1668 low = x + y;
1669 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1670 SET_QC();
1671 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1672 }
1673 x = a >> 32;
1674 y = b >> 32;
1675 high = x + y;
1676 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1677 SET_QC();
1678 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1679 }
1680 return low | ((uint64_t)high << 32);
1681 }
1682
1683 uint64_t HELPER(neon_addl_saturate_s64)(CPUARMState *env, uint64_t a, uint64_t b)
1684 {
1685 uint64_t result;
1686
1687 result = a + b;
1688 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1689 SET_QC();
1690 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1691 }
1692 return result;
1693 }
1694
1695 /* We have to do the arithmetic in a larger type than
1696 * the input type, because for example with a signed 32 bit
1697 * op the absolute difference can overflow a signed 32 bit value.
1698 */
1699 #define DO_ABD(dest, x, y, intype, arithtype) do { \
1700 arithtype tmp_x = (intype)(x); \
1701 arithtype tmp_y = (intype)(y); \
1702 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1703 } while(0)
1704
1705 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1706 {
1707 uint64_t tmp;
1708 uint64_t result;
1709 DO_ABD(result, a, b, uint8_t, uint32_t);
1710 DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
1711 result |= tmp << 16;
1712 DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
1713 result |= tmp << 32;
1714 DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
1715 result |= tmp << 48;
1716 return result;
1717 }
1718
1719 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1720 {
1721 uint64_t tmp;
1722 uint64_t result;
1723 DO_ABD(result, a, b, int8_t, int32_t);
1724 DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
1725 result |= tmp << 16;
1726 DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
1727 result |= tmp << 32;
1728 DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
1729 result |= tmp << 48;
1730 return result;
1731 }
1732
1733 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1734 {
1735 uint64_t tmp;
1736 uint64_t result;
1737 DO_ABD(result, a, b, uint16_t, uint32_t);
1738 DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1739 return result | (tmp << 32);
1740 }
1741
1742 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1743 {
1744 uint64_t tmp;
1745 uint64_t result;
1746 DO_ABD(result, a, b, int16_t, int32_t);
1747 DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
1748 return result | (tmp << 32);
1749 }
1750
1751 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1752 {
1753 uint64_t result;
1754 DO_ABD(result, a, b, uint32_t, uint64_t);
1755 return result;
1756 }
1757
1758 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1759 {
1760 uint64_t result;
1761 DO_ABD(result, a, b, int32_t, int64_t);
1762 return result;
1763 }
1764 #undef DO_ABD
1765
1766 /* Widening multiply. Named type is the source type. */
1767 #define DO_MULL(dest, x, y, type1, type2) do { \
1768 type1 tmp_x = x; \
1769 type1 tmp_y = y; \
1770 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1771 } while(0)
1772
1773 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1774 {
1775 uint64_t tmp;
1776 uint64_t result;
1777
1778 DO_MULL(result, a, b, uint8_t, uint16_t);
1779 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1780 result |= tmp << 16;
1781 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1782 result |= tmp << 32;
1783 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1784 result |= tmp << 48;
1785 return result;
1786 }
1787
1788 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1789 {
1790 uint64_t tmp;
1791 uint64_t result;
1792
1793 DO_MULL(result, a, b, int8_t, uint16_t);
1794 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1795 result |= tmp << 16;
1796 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1797 result |= tmp << 32;
1798 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1799 result |= tmp << 48;
1800 return result;
1801 }
1802
1803 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1804 {
1805 uint64_t tmp;
1806 uint64_t result;
1807
1808 DO_MULL(result, a, b, uint16_t, uint32_t);
1809 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1810 return result | (tmp << 32);
1811 }
1812
1813 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1814 {
1815 uint64_t tmp;
1816 uint64_t result;
1817
1818 DO_MULL(result, a, b, int16_t, uint32_t);
1819 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1820 return result | (tmp << 32);
1821 }
1822
1823 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1824 {
1825 uint16_t tmp;
1826 uint64_t result;
1827 result = (uint16_t)-x;
1828 tmp = -(x >> 16);
1829 result |= (uint64_t)tmp << 16;
1830 tmp = -(x >> 32);
1831 result |= (uint64_t)tmp << 32;
1832 tmp = -(x >> 48);
1833 result |= (uint64_t)tmp << 48;
1834 return result;
1835 }
1836
1837 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1838 {
1839 uint32_t low = -x;
1840 uint32_t high = -(x >> 32);
1841 return low | ((uint64_t)high << 32);
1842 }
1843
1844 /* Saturating sign manipulation. */
1845 /* ??? Make these use NEON_VOP1 */
1846 #define DO_QABS8(x) do { \
1847 if (x == (int8_t)0x80) { \
1848 x = 0x7f; \
1849 SET_QC(); \
1850 } else if (x < 0) { \
1851 x = -x; \
1852 }} while (0)
1853 uint32_t HELPER(neon_qabs_s8)(CPUARMState *env, uint32_t x)
1854 {
1855 neon_s8 vec;
1856 NEON_UNPACK(neon_s8, vec, x);
1857 DO_QABS8(vec.v1);
1858 DO_QABS8(vec.v2);
1859 DO_QABS8(vec.v3);
1860 DO_QABS8(vec.v4);
1861 NEON_PACK(neon_s8, x, vec);
1862 return x;
1863 }
1864 #undef DO_QABS8
1865
1866 #define DO_QNEG8(x) do { \
1867 if (x == (int8_t)0x80) { \
1868 x = 0x7f; \
1869 SET_QC(); \
1870 } else { \
1871 x = -x; \
1872 }} while (0)
1873 uint32_t HELPER(neon_qneg_s8)(CPUARMState *env, uint32_t x)
1874 {
1875 neon_s8 vec;
1876 NEON_UNPACK(neon_s8, vec, x);
1877 DO_QNEG8(vec.v1);
1878 DO_QNEG8(vec.v2);
1879 DO_QNEG8(vec.v3);
1880 DO_QNEG8(vec.v4);
1881 NEON_PACK(neon_s8, x, vec);
1882 return x;
1883 }
1884 #undef DO_QNEG8
1885
1886 #define DO_QABS16(x) do { \
1887 if (x == (int16_t)0x8000) { \
1888 x = 0x7fff; \
1889 SET_QC(); \
1890 } else if (x < 0) { \
1891 x = -x; \
1892 }} while (0)
1893 uint32_t HELPER(neon_qabs_s16)(CPUARMState *env, uint32_t x)
1894 {
1895 neon_s16 vec;
1896 NEON_UNPACK(neon_s16, vec, x);
1897 DO_QABS16(vec.v1);
1898 DO_QABS16(vec.v2);
1899 NEON_PACK(neon_s16, x, vec);
1900 return x;
1901 }
1902 #undef DO_QABS16
1903
1904 #define DO_QNEG16(x) do { \
1905 if (x == (int16_t)0x8000) { \
1906 x = 0x7fff; \
1907 SET_QC(); \
1908 } else { \
1909 x = -x; \
1910 }} while (0)
1911 uint32_t HELPER(neon_qneg_s16)(CPUARMState *env, uint32_t x)
1912 {
1913 neon_s16 vec;
1914 NEON_UNPACK(neon_s16, vec, x);
1915 DO_QNEG16(vec.v1);
1916 DO_QNEG16(vec.v2);
1917 NEON_PACK(neon_s16, x, vec);
1918 return x;
1919 }
1920 #undef DO_QNEG16
1921
1922 uint32_t HELPER(neon_qabs_s32)(CPUARMState *env, uint32_t x)
1923 {
1924 if (x == SIGNBIT) {
1925 SET_QC();
1926 x = ~SIGNBIT;
1927 } else if ((int32_t)x < 0) {
1928 x = -x;
1929 }
1930 return x;
1931 }
1932
1933 uint32_t HELPER(neon_qneg_s32)(CPUARMState *env, uint32_t x)
1934 {
1935 if (x == SIGNBIT) {
1936 SET_QC();
1937 x = ~SIGNBIT;
1938 } else {
1939 x = -x;
1940 }
1941 return x;
1942 }
1943
1944 uint64_t HELPER(neon_qabs_s64)(CPUARMState *env, uint64_t x)
1945 {
1946 if (x == SIGNBIT64) {
1947 SET_QC();
1948 x = ~SIGNBIT64;
1949 } else if ((int64_t)x < 0) {
1950 x = -x;
1951 }
1952 return x;
1953 }
1954
1955 uint64_t HELPER(neon_qneg_s64)(CPUARMState *env, uint64_t x)
1956 {
1957 if (x == SIGNBIT64) {
1958 SET_QC();
1959 x = ~SIGNBIT64;
1960 } else {
1961 x = -x;
1962 }
1963 return x;
1964 }
1965
1966 /* NEON Float helpers. */
1967 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b, void *fpstp)
1968 {
1969 float_status *fpst = fpstp;
1970 float32 f0 = make_float32(a);
1971 float32 f1 = make_float32(b);
1972 return float32_val(float32_abs(float32_sub(f0, f1, fpst)));
1973 }
1974
1975 /* Floating point comparisons produce an integer result.
1976 * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1977 * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1978 */
1979 uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
1980 {
1981 float_status *fpst = fpstp;
1982 return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
1983 }
1984
1985 uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
1986 {
1987 float_status *fpst = fpstp;
1988 return -float32_le(make_float32(b), make_float32(a), fpst);
1989 }
1990
1991 uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1992 {
1993 float_status *fpst = fpstp;
1994 return -float32_lt(make_float32(b), make_float32(a), fpst);
1995 }
1996
1997 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
1998 {
1999 float_status *fpst = fpstp;
2000 float32 f0 = float32_abs(make_float32(a));
2001 float32 f1 = float32_abs(make_float32(b));
2002 return -float32_le(f1, f0, fpst);
2003 }
2004
2005 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
2006 {
2007 float_status *fpst = fpstp;
2008 float32 f0 = float32_abs(make_float32(a));
2009 float32 f1 = float32_abs(make_float32(b));
2010 return -float32_lt(f1, f0, fpst);
2011 }
2012
2013 uint64_t HELPER(neon_acge_f64)(uint64_t a, uint64_t b, void *fpstp)
2014 {
2015 float_status *fpst = fpstp;
2016 float64 f0 = float64_abs(make_float64(a));
2017 float64 f1 = float64_abs(make_float64(b));
2018 return -float64_le(f1, f0, fpst);
2019 }
2020
2021 uint64_t HELPER(neon_acgt_f64)(uint64_t a, uint64_t b, void *fpstp)
2022 {
2023 float_status *fpst = fpstp;
2024 float64 f0 = float64_abs(make_float64(a));
2025 float64 f1 = float64_abs(make_float64(b));
2026 return -float64_lt(f1, f0, fpst);
2027 }
2028
2029 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
2030
2031 void HELPER(neon_qunzip8)(CPUARMState *env, uint32_t rd, uint32_t rm)
2032 {
2033 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
2034 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
2035 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
2036 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
2037 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
2038 | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
2039 | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
2040 | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
2041 uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
2042 | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
2043 | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2044 | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
2045 uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
2046 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
2047 | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
2048 | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
2049 uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
2050 | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
2051 | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
2052 | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2053 env->vfp.regs[rm] = make_float64(m0);
2054 env->vfp.regs[rm + 1] = make_float64(m1);
2055 env->vfp.regs[rd] = make_float64(d0);
2056 env->vfp.regs[rd + 1] = make_float64(d1);
2057 }
2058
2059 void HELPER(neon_qunzip16)(CPUARMState *env, uint32_t rd, uint32_t rm)
2060 {
2061 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
2062 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
2063 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
2064 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
2065 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
2066 | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
2067 uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
2068 | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
2069 uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
2070 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
2071 uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
2072 | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2073 env->vfp.regs[rm] = make_float64(m0);
2074 env->vfp.regs[rm + 1] = make_float64(m1);
2075 env->vfp.regs[rd] = make_float64(d0);
2076 env->vfp.regs[rd + 1] = make_float64(d1);
2077 }
2078
2079 void HELPER(neon_qunzip32)(CPUARMState *env, uint32_t rd, uint32_t rm)
2080 {
2081 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
2082 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
2083 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
2084 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
2085 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
2086 uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2087 uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
2088 uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2089 env->vfp.regs[rm] = make_float64(m0);
2090 env->vfp.regs[rm + 1] = make_float64(m1);
2091 env->vfp.regs[rd] = make_float64(d0);
2092 env->vfp.regs[rd + 1] = make_float64(d1);
2093 }
2094
2095 void HELPER(neon_unzip8)(CPUARMState *env, uint32_t rd, uint32_t rm)
2096 {
2097 uint64_t zm = float64_val(env->vfp.regs[rm]);
2098 uint64_t zd = float64_val(env->vfp.regs[rd]);
2099 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
2100 | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
2101 | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2102 | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
2103 uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
2104 | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
2105 | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
2106 | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2107 env->vfp.regs[rm] = make_float64(m0);
2108 env->vfp.regs[rd] = make_float64(d0);
2109 }
2110
2111 void HELPER(neon_unzip16)(CPUARMState *env, uint32_t rd, uint32_t rm)
2112 {
2113 uint64_t zm = float64_val(env->vfp.regs[rm]);
2114 uint64_t zd = float64_val(env->vfp.regs[rd]);
2115 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
2116 | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
2117 uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
2118 | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2119 env->vfp.regs[rm] = make_float64(m0);
2120 env->vfp.regs[rd] = make_float64(d0);
2121 }
2122
2123 void HELPER(neon_qzip8)(CPUARMState *env, uint32_t rd, uint32_t rm)
2124 {
2125 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
2126 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
2127 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
2128 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
2129 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
2130 | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
2131 | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
2132 | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
2133 uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
2134 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
2135 | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
2136 | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
2137 uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
2138 | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
2139 | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2140 | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
2141 uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
2142 | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
2143 | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
2144 | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2145 env->vfp.regs[rm] = make_float64(m0);
2146 env->vfp.regs[rm + 1] = make_float64(m1);
2147 env->vfp.regs[rd] = make_float64(d0);
2148 env->vfp.regs[rd + 1] = make_float64(d1);
2149 }
2150
2151 void HELPER(neon_qzip16)(CPUARMState *env, uint32_t rd, uint32_t rm)
2152 {
2153 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
2154 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
2155 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
2156 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
2157 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
2158 | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
2159 uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
2160 | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
2161 uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
2162 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
2163 uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
2164 | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2165 env->vfp.regs[rm] = make_float64(m0);
2166 env->vfp.regs[rm + 1] = make_float64(m1);
2167 env->vfp.regs[rd] = make_float64(d0);
2168 env->vfp.regs[rd + 1] = make_float64(d1);
2169 }
2170
2171 void HELPER(neon_qzip32)(CPUARMState *env, uint32_t rd, uint32_t rm)
2172 {
2173 uint64_t zm0 = float64_val(env->vfp.regs[rm]);
2174 uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]);
2175 uint64_t zd0 = float64_val(env->vfp.regs[rd]);
2176 uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]);
2177 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
2178 uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
2179 uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2180 uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2181 env->vfp.regs[rm] = make_float64(m0);
2182 env->vfp.regs[rm + 1] = make_float64(m1);
2183 env->vfp.regs[rd] = make_float64(d0);
2184 env->vfp.regs[rd + 1] = make_float64(d1);
2185 }
2186
2187 void HELPER(neon_zip8)(CPUARMState *env, uint32_t rd, uint32_t rm)
2188 {
2189 uint64_t zm = float64_val(env->vfp.regs[rm]);
2190 uint64_t zd = float64_val(env->vfp.regs[rd]);
2191 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2192 | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2193 | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2194 | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2195 uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2196 | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2197 | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2198 | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2199 env->vfp.regs[rm] = make_float64(m0);
2200 env->vfp.regs[rd] = make_float64(d0);
2201 }
2202
2203 void HELPER(neon_zip16)(CPUARMState *env, uint32_t rd, uint32_t rm)
2204 {
2205 uint64_t zm = float64_val(env->vfp.regs[rm]);
2206 uint64_t zd = float64_val(env->vfp.regs[rd]);
2207 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2208 | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2209 uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2210 | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2211 env->vfp.regs[rm] = make_float64(m0);
2212 env->vfp.regs[rd] = make_float64(d0);
2213 }
2214
2215 /* Helper function for 64 bit polynomial multiply case:
2216 * perform PolynomialMult(op1, op2) and return either the top or
2217 * bottom half of the 128 bit result.
2218 */
2219 uint64_t HELPER(neon_pmull_64_lo)(uint64_t op1, uint64_t op2)
2220 {
2221 int bitnum;
2222 uint64_t res = 0;
2223
2224 for (bitnum = 0; bitnum < 64; bitnum++) {
2225 if (op1 & (1ULL << bitnum)) {
2226 res ^= op2 << bitnum;
2227 }
2228 }
2229 return res;
2230 }
2231 uint64_t HELPER(neon_pmull_64_hi)(uint64_t op1, uint64_t op2)
2232 {
2233 int bitnum;
2234 uint64_t res = 0;
2235
2236 /* bit 0 of op1 can't influence the high 64 bits at all */
2237 for (bitnum = 1; bitnum < 64; bitnum++) {
2238 if (op1 & (1ULL << bitnum)) {
2239 res ^= op2 >> (64 - bitnum);
2240 }
2241 }
2242 return res;
2243 }