migration: increase max-bandwidth to 128 MiB/s (1 Gib/s)
[qemu.git] / target / arm / neon_helper.c
1 /*
2 * ARM NEON vector operations.
3 *
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
6 *
7 * This code is licensed under the GNU GPL v2.
8 */
9 #include "qemu/osdep.h"
10
11 #include "cpu.h"
12 #include "exec/helper-proto.h"
13 #include "fpu/softfloat.h"
14
15 #define SIGNBIT (uint32_t)0x80000000
16 #define SIGNBIT64 ((uint64_t)1 << 63)
17
18 #define SET_QC() env->vfp.qc[0] = 1
19
20 #define NEON_TYPE1(name, type) \
21 typedef struct \
22 { \
23 type v1; \
24 } neon_##name;
25 #ifdef HOST_WORDS_BIGENDIAN
26 #define NEON_TYPE2(name, type) \
27 typedef struct \
28 { \
29 type v2; \
30 type v1; \
31 } neon_##name;
32 #define NEON_TYPE4(name, type) \
33 typedef struct \
34 { \
35 type v4; \
36 type v3; \
37 type v2; \
38 type v1; \
39 } neon_##name;
40 #else
41 #define NEON_TYPE2(name, type) \
42 typedef struct \
43 { \
44 type v1; \
45 type v2; \
46 } neon_##name;
47 #define NEON_TYPE4(name, type) \
48 typedef struct \
49 { \
50 type v1; \
51 type v2; \
52 type v3; \
53 type v4; \
54 } neon_##name;
55 #endif
56
57 NEON_TYPE4(s8, int8_t)
58 NEON_TYPE4(u8, uint8_t)
59 NEON_TYPE2(s16, int16_t)
60 NEON_TYPE2(u16, uint16_t)
61 NEON_TYPE1(s32, int32_t)
62 NEON_TYPE1(u32, uint32_t)
63 #undef NEON_TYPE4
64 #undef NEON_TYPE2
65 #undef NEON_TYPE1
66
67 /* Copy from a uint32_t to a vector structure type. */
68 #define NEON_UNPACK(vtype, dest, val) do { \
69 union { \
70 vtype v; \
71 uint32_t i; \
72 } conv_u; \
73 conv_u.i = (val); \
74 dest = conv_u.v; \
75 } while(0)
76
77 /* Copy from a vector structure type to a uint32_t. */
78 #define NEON_PACK(vtype, dest, val) do { \
79 union { \
80 vtype v; \
81 uint32_t i; \
82 } conv_u; \
83 conv_u.v = (val); \
84 dest = conv_u.i; \
85 } while(0)
86
87 #define NEON_DO1 \
88 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
89 #define NEON_DO2 \
90 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
91 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
92 #define NEON_DO4 \
93 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
94 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
95 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
96 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
97
98 #define NEON_VOP_BODY(vtype, n) \
99 { \
100 uint32_t res; \
101 vtype vsrc1; \
102 vtype vsrc2; \
103 vtype vdest; \
104 NEON_UNPACK(vtype, vsrc1, arg1); \
105 NEON_UNPACK(vtype, vsrc2, arg2); \
106 NEON_DO##n; \
107 NEON_PACK(vtype, res, vdest); \
108 return res; \
109 }
110
111 #define NEON_VOP(name, vtype, n) \
112 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
113 NEON_VOP_BODY(vtype, n)
114
115 #define NEON_VOP_ENV(name, vtype, n) \
116 uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
117 NEON_VOP_BODY(vtype, n)
118
119 /* Pairwise operations. */
120 /* For 32-bit elements each segment only contains a single element, so
121 the elementwise and pairwise operations are the same. */
122 #define NEON_PDO2 \
123 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
124 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
125 #define NEON_PDO4 \
126 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
127 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
128 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
129 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
130
131 #define NEON_POP(name, vtype, n) \
132 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
133 { \
134 uint32_t res; \
135 vtype vsrc1; \
136 vtype vsrc2; \
137 vtype vdest; \
138 NEON_UNPACK(vtype, vsrc1, arg1); \
139 NEON_UNPACK(vtype, vsrc2, arg2); \
140 NEON_PDO##n; \
141 NEON_PACK(vtype, res, vdest); \
142 return res; \
143 }
144
145 /* Unary operators. */
146 #define NEON_VOP1(name, vtype, n) \
147 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
148 { \
149 vtype vsrc1; \
150 vtype vdest; \
151 NEON_UNPACK(vtype, vsrc1, arg); \
152 NEON_DO##n; \
153 NEON_PACK(vtype, arg, vdest); \
154 return arg; \
155 }
156
157
158 #define NEON_USAT(dest, src1, src2, type) do { \
159 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
160 if (tmp != (type)tmp) { \
161 SET_QC(); \
162 dest = ~0; \
163 } else { \
164 dest = tmp; \
165 }} while(0)
166 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
167 NEON_VOP_ENV(qadd_u8, neon_u8, 4)
168 #undef NEON_FN
169 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
170 NEON_VOP_ENV(qadd_u16, neon_u16, 2)
171 #undef NEON_FN
172 #undef NEON_USAT
173
174 uint32_t HELPER(neon_qadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
175 {
176 uint32_t res = a + b;
177 if (res < a) {
178 SET_QC();
179 res = ~0;
180 }
181 return res;
182 }
183
184 uint64_t HELPER(neon_qadd_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
185 {
186 uint64_t res;
187
188 res = src1 + src2;
189 if (res < src1) {
190 SET_QC();
191 res = ~(uint64_t)0;
192 }
193 return res;
194 }
195
196 #define NEON_SSAT(dest, src1, src2, type) do { \
197 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
198 if (tmp != (type)tmp) { \
199 SET_QC(); \
200 if (src2 > 0) { \
201 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
202 } else { \
203 tmp = 1 << (sizeof(type) * 8 - 1); \
204 } \
205 } \
206 dest = tmp; \
207 } while(0)
208 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
209 NEON_VOP_ENV(qadd_s8, neon_s8, 4)
210 #undef NEON_FN
211 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
212 NEON_VOP_ENV(qadd_s16, neon_s16, 2)
213 #undef NEON_FN
214 #undef NEON_SSAT
215
216 uint32_t HELPER(neon_qadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
217 {
218 uint32_t res = a + b;
219 if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
220 SET_QC();
221 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
222 }
223 return res;
224 }
225
226 uint64_t HELPER(neon_qadd_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
227 {
228 uint64_t res;
229
230 res = src1 + src2;
231 if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
232 SET_QC();
233 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
234 }
235 return res;
236 }
237
238 /* Unsigned saturating accumulate of signed value
239 *
240 * Op1/Rn is treated as signed
241 * Op2/Rd is treated as unsigned
242 *
243 * Explicit casting is used to ensure the correct sign extension of
244 * inputs. The result is treated as a unsigned value and saturated as such.
245 *
246 * We use a macro for the 8/16 bit cases which expects signed integers of va,
247 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
248 */
249
250 #define USATACC(bits, shift) \
251 do { \
252 va = sextract32(a, shift, bits); \
253 vb = extract32(b, shift, bits); \
254 vr = va + vb; \
255 if (vr > UINT##bits##_MAX) { \
256 SET_QC(); \
257 vr = UINT##bits##_MAX; \
258 } else if (vr < 0) { \
259 SET_QC(); \
260 vr = 0; \
261 } \
262 r = deposit32(r, shift, bits, vr); \
263 } while (0)
264
265 uint32_t HELPER(neon_uqadd_s8)(CPUARMState *env, uint32_t a, uint32_t b)
266 {
267 int16_t va, vb, vr;
268 uint32_t r = 0;
269
270 USATACC(8, 0);
271 USATACC(8, 8);
272 USATACC(8, 16);
273 USATACC(8, 24);
274 return r;
275 }
276
277 uint32_t HELPER(neon_uqadd_s16)(CPUARMState *env, uint32_t a, uint32_t b)
278 {
279 int32_t va, vb, vr;
280 uint64_t r = 0;
281
282 USATACC(16, 0);
283 USATACC(16, 16);
284 return r;
285 }
286
287 #undef USATACC
288
289 uint32_t HELPER(neon_uqadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
290 {
291 int64_t va = (int32_t)a;
292 int64_t vb = (uint32_t)b;
293 int64_t vr = va + vb;
294 if (vr > UINT32_MAX) {
295 SET_QC();
296 vr = UINT32_MAX;
297 } else if (vr < 0) {
298 SET_QC();
299 vr = 0;
300 }
301 return vr;
302 }
303
304 uint64_t HELPER(neon_uqadd_s64)(CPUARMState *env, uint64_t a, uint64_t b)
305 {
306 uint64_t res;
307 res = a + b;
308 /* We only need to look at the pattern of SIGN bits to detect
309 * +ve/-ve saturation
310 */
311 if (~a & b & ~res & SIGNBIT64) {
312 SET_QC();
313 res = UINT64_MAX;
314 } else if (a & ~b & res & SIGNBIT64) {
315 SET_QC();
316 res = 0;
317 }
318 return res;
319 }
320
321 /* Signed saturating accumulate of unsigned value
322 *
323 * Op1/Rn is treated as unsigned
324 * Op2/Rd is treated as signed
325 *
326 * The result is treated as a signed value and saturated as such
327 *
328 * We use a macro for the 8/16 bit cases which expects signed integers of va,
329 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
330 */
331
332 #define SSATACC(bits, shift) \
333 do { \
334 va = extract32(a, shift, bits); \
335 vb = sextract32(b, shift, bits); \
336 vr = va + vb; \
337 if (vr > INT##bits##_MAX) { \
338 SET_QC(); \
339 vr = INT##bits##_MAX; \
340 } else if (vr < INT##bits##_MIN) { \
341 SET_QC(); \
342 vr = INT##bits##_MIN; \
343 } \
344 r = deposit32(r, shift, bits, vr); \
345 } while (0)
346
347 uint32_t HELPER(neon_sqadd_u8)(CPUARMState *env, uint32_t a, uint32_t b)
348 {
349 int16_t va, vb, vr;
350 uint32_t r = 0;
351
352 SSATACC(8, 0);
353 SSATACC(8, 8);
354 SSATACC(8, 16);
355 SSATACC(8, 24);
356 return r;
357 }
358
359 uint32_t HELPER(neon_sqadd_u16)(CPUARMState *env, uint32_t a, uint32_t b)
360 {
361 int32_t va, vb, vr;
362 uint32_t r = 0;
363
364 SSATACC(16, 0);
365 SSATACC(16, 16);
366
367 return r;
368 }
369
370 #undef SSATACC
371
372 uint32_t HELPER(neon_sqadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
373 {
374 int64_t res;
375 int64_t op1 = (uint32_t)a;
376 int64_t op2 = (int32_t)b;
377 res = op1 + op2;
378 if (res > INT32_MAX) {
379 SET_QC();
380 res = INT32_MAX;
381 } else if (res < INT32_MIN) {
382 SET_QC();
383 res = INT32_MIN;
384 }
385 return res;
386 }
387
388 uint64_t HELPER(neon_sqadd_u64)(CPUARMState *env, uint64_t a, uint64_t b)
389 {
390 uint64_t res;
391 res = a + b;
392 /* We only need to look at the pattern of SIGN bits to detect an overflow */
393 if (((a & res)
394 | (~b & res)
395 | (a & ~b)) & SIGNBIT64) {
396 SET_QC();
397 res = INT64_MAX;
398 }
399 return res;
400 }
401
402
403 #define NEON_USAT(dest, src1, src2, type) do { \
404 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
405 if (tmp != (type)tmp) { \
406 SET_QC(); \
407 dest = 0; \
408 } else { \
409 dest = tmp; \
410 }} while(0)
411 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
412 NEON_VOP_ENV(qsub_u8, neon_u8, 4)
413 #undef NEON_FN
414 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
415 NEON_VOP_ENV(qsub_u16, neon_u16, 2)
416 #undef NEON_FN
417 #undef NEON_USAT
418
419 uint32_t HELPER(neon_qsub_u32)(CPUARMState *env, uint32_t a, uint32_t b)
420 {
421 uint32_t res = a - b;
422 if (res > a) {
423 SET_QC();
424 res = 0;
425 }
426 return res;
427 }
428
429 uint64_t HELPER(neon_qsub_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
430 {
431 uint64_t res;
432
433 if (src1 < src2) {
434 SET_QC();
435 res = 0;
436 } else {
437 res = src1 - src2;
438 }
439 return res;
440 }
441
442 #define NEON_SSAT(dest, src1, src2, type) do { \
443 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
444 if (tmp != (type)tmp) { \
445 SET_QC(); \
446 if (src2 < 0) { \
447 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
448 } else { \
449 tmp = 1 << (sizeof(type) * 8 - 1); \
450 } \
451 } \
452 dest = tmp; \
453 } while(0)
454 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
455 NEON_VOP_ENV(qsub_s8, neon_s8, 4)
456 #undef NEON_FN
457 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
458 NEON_VOP_ENV(qsub_s16, neon_s16, 2)
459 #undef NEON_FN
460 #undef NEON_SSAT
461
462 uint32_t HELPER(neon_qsub_s32)(CPUARMState *env, uint32_t a, uint32_t b)
463 {
464 uint32_t res = a - b;
465 if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
466 SET_QC();
467 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
468 }
469 return res;
470 }
471
472 uint64_t HELPER(neon_qsub_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
473 {
474 uint64_t res;
475
476 res = src1 - src2;
477 if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
478 SET_QC();
479 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
480 }
481 return res;
482 }
483
484 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
485 NEON_VOP(hadd_s8, neon_s8, 4)
486 NEON_VOP(hadd_u8, neon_u8, 4)
487 NEON_VOP(hadd_s16, neon_s16, 2)
488 NEON_VOP(hadd_u16, neon_u16, 2)
489 #undef NEON_FN
490
491 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
492 {
493 int32_t dest;
494
495 dest = (src1 >> 1) + (src2 >> 1);
496 if (src1 & src2 & 1)
497 dest++;
498 return dest;
499 }
500
501 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
502 {
503 uint32_t dest;
504
505 dest = (src1 >> 1) + (src2 >> 1);
506 if (src1 & src2 & 1)
507 dest++;
508 return dest;
509 }
510
511 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
512 NEON_VOP(rhadd_s8, neon_s8, 4)
513 NEON_VOP(rhadd_u8, neon_u8, 4)
514 NEON_VOP(rhadd_s16, neon_s16, 2)
515 NEON_VOP(rhadd_u16, neon_u16, 2)
516 #undef NEON_FN
517
518 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
519 {
520 int32_t dest;
521
522 dest = (src1 >> 1) + (src2 >> 1);
523 if ((src1 | src2) & 1)
524 dest++;
525 return dest;
526 }
527
528 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
529 {
530 uint32_t dest;
531
532 dest = (src1 >> 1) + (src2 >> 1);
533 if ((src1 | src2) & 1)
534 dest++;
535 return dest;
536 }
537
538 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
539 NEON_VOP(hsub_s8, neon_s8, 4)
540 NEON_VOP(hsub_u8, neon_u8, 4)
541 NEON_VOP(hsub_s16, neon_s16, 2)
542 NEON_VOP(hsub_u16, neon_u16, 2)
543 #undef NEON_FN
544
545 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
546 {
547 int32_t dest;
548
549 dest = (src1 >> 1) - (src2 >> 1);
550 if ((~src1) & src2 & 1)
551 dest--;
552 return dest;
553 }
554
555 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
556 {
557 uint32_t dest;
558
559 dest = (src1 >> 1) - (src2 >> 1);
560 if ((~src1) & src2 & 1)
561 dest--;
562 return dest;
563 }
564
565 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
566 NEON_POP(pmin_s8, neon_s8, 4)
567 NEON_POP(pmin_u8, neon_u8, 4)
568 NEON_POP(pmin_s16, neon_s16, 2)
569 NEON_POP(pmin_u16, neon_u16, 2)
570 #undef NEON_FN
571
572 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
573 NEON_POP(pmax_s8, neon_s8, 4)
574 NEON_POP(pmax_u8, neon_u8, 4)
575 NEON_POP(pmax_s16, neon_s16, 2)
576 NEON_POP(pmax_u16, neon_u16, 2)
577 #undef NEON_FN
578
579 #define NEON_FN(dest, src1, src2) do { \
580 int8_t tmp; \
581 tmp = (int8_t)src2; \
582 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
583 tmp <= -(ssize_t)sizeof(src1) * 8) { \
584 dest = 0; \
585 } else if (tmp < 0) { \
586 dest = src1 >> -tmp; \
587 } else { \
588 dest = src1 << tmp; \
589 }} while (0)
590 NEON_VOP(shl_u16, neon_u16, 2)
591 #undef NEON_FN
592
593 #define NEON_FN(dest, src1, src2) do { \
594 int8_t tmp; \
595 tmp = (int8_t)src2; \
596 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
597 dest = 0; \
598 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
599 dest = src1 >> (sizeof(src1) * 8 - 1); \
600 } else if (tmp < 0) { \
601 dest = src1 >> -tmp; \
602 } else { \
603 dest = src1 << tmp; \
604 }} while (0)
605 NEON_VOP(shl_s16, neon_s16, 2)
606 #undef NEON_FN
607
608 #define NEON_FN(dest, src1, src2) do { \
609 int8_t tmp; \
610 tmp = (int8_t)src2; \
611 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
612 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
613 dest = 0; \
614 } else if (tmp < 0) { \
615 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
616 } else { \
617 dest = src1 << tmp; \
618 }} while (0)
619 NEON_VOP(rshl_s8, neon_s8, 4)
620 NEON_VOP(rshl_s16, neon_s16, 2)
621 #undef NEON_FN
622
623 /* The addition of the rounding constant may overflow, so we use an
624 * intermediate 64 bit accumulator. */
625 uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
626 {
627 int32_t dest;
628 int32_t val = (int32_t)valop;
629 int8_t shift = (int8_t)shiftop;
630 if ((shift >= 32) || (shift <= -32)) {
631 dest = 0;
632 } else if (shift < 0) {
633 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
634 dest = big_dest >> -shift;
635 } else {
636 dest = val << shift;
637 }
638 return dest;
639 }
640
641 /* Handling addition overflow with 64 bit input values is more
642 * tricky than with 32 bit values. */
643 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
644 {
645 int8_t shift = (int8_t)shiftop;
646 int64_t val = valop;
647 if ((shift >= 64) || (shift <= -64)) {
648 val = 0;
649 } else if (shift < 0) {
650 val >>= (-shift - 1);
651 if (val == INT64_MAX) {
652 /* In this case, it means that the rounding constant is 1,
653 * and the addition would overflow. Return the actual
654 * result directly. */
655 val = 0x4000000000000000LL;
656 } else {
657 val++;
658 val >>= 1;
659 }
660 } else {
661 val <<= shift;
662 }
663 return val;
664 }
665
666 #define NEON_FN(dest, src1, src2) do { \
667 int8_t tmp; \
668 tmp = (int8_t)src2; \
669 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
670 tmp < -(ssize_t)sizeof(src1) * 8) { \
671 dest = 0; \
672 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
673 dest = src1 >> (-tmp - 1); \
674 } else if (tmp < 0) { \
675 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
676 } else { \
677 dest = src1 << tmp; \
678 }} while (0)
679 NEON_VOP(rshl_u8, neon_u8, 4)
680 NEON_VOP(rshl_u16, neon_u16, 2)
681 #undef NEON_FN
682
683 /* The addition of the rounding constant may overflow, so we use an
684 * intermediate 64 bit accumulator. */
685 uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
686 {
687 uint32_t dest;
688 int8_t shift = (int8_t)shiftop;
689 if (shift >= 32 || shift < -32) {
690 dest = 0;
691 } else if (shift == -32) {
692 dest = val >> 31;
693 } else if (shift < 0) {
694 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
695 dest = big_dest >> -shift;
696 } else {
697 dest = val << shift;
698 }
699 return dest;
700 }
701
702 /* Handling addition overflow with 64 bit input values is more
703 * tricky than with 32 bit values. */
704 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
705 {
706 int8_t shift = (uint8_t)shiftop;
707 if (shift >= 64 || shift < -64) {
708 val = 0;
709 } else if (shift == -64) {
710 /* Rounding a 1-bit result just preserves that bit. */
711 val >>= 63;
712 } else if (shift < 0) {
713 val >>= (-shift - 1);
714 if (val == UINT64_MAX) {
715 /* In this case, it means that the rounding constant is 1,
716 * and the addition would overflow. Return the actual
717 * result directly. */
718 val = 0x8000000000000000ULL;
719 } else {
720 val++;
721 val >>= 1;
722 }
723 } else {
724 val <<= shift;
725 }
726 return val;
727 }
728
729 #define NEON_FN(dest, src1, src2) do { \
730 int8_t tmp; \
731 tmp = (int8_t)src2; \
732 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
733 if (src1) { \
734 SET_QC(); \
735 dest = ~0; \
736 } else { \
737 dest = 0; \
738 } \
739 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
740 dest = 0; \
741 } else if (tmp < 0) { \
742 dest = src1 >> -tmp; \
743 } else { \
744 dest = src1 << tmp; \
745 if ((dest >> tmp) != src1) { \
746 SET_QC(); \
747 dest = ~0; \
748 } \
749 }} while (0)
750 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
751 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
752 NEON_VOP_ENV(qshl_u32, neon_u32, 1)
753 #undef NEON_FN
754
755 uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
756 {
757 int8_t shift = (int8_t)shiftop;
758 if (shift >= 64) {
759 if (val) {
760 val = ~(uint64_t)0;
761 SET_QC();
762 }
763 } else if (shift <= -64) {
764 val = 0;
765 } else if (shift < 0) {
766 val >>= -shift;
767 } else {
768 uint64_t tmp = val;
769 val <<= shift;
770 if ((val >> shift) != tmp) {
771 SET_QC();
772 val = ~(uint64_t)0;
773 }
774 }
775 return val;
776 }
777
778 #define NEON_FN(dest, src1, src2) do { \
779 int8_t tmp; \
780 tmp = (int8_t)src2; \
781 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
782 if (src1) { \
783 SET_QC(); \
784 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
785 if (src1 > 0) { \
786 dest--; \
787 } \
788 } else { \
789 dest = src1; \
790 } \
791 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
792 dest = src1 >> 31; \
793 } else if (tmp < 0) { \
794 dest = src1 >> -tmp; \
795 } else { \
796 dest = src1 << tmp; \
797 if ((dest >> tmp) != src1) { \
798 SET_QC(); \
799 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
800 if (src1 > 0) { \
801 dest--; \
802 } \
803 } \
804 }} while (0)
805 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
806 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
807 NEON_VOP_ENV(qshl_s32, neon_s32, 1)
808 #undef NEON_FN
809
810 uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
811 {
812 int8_t shift = (uint8_t)shiftop;
813 int64_t val = valop;
814 if (shift >= 64) {
815 if (val) {
816 SET_QC();
817 val = (val >> 63) ^ ~SIGNBIT64;
818 }
819 } else if (shift <= -64) {
820 val >>= 63;
821 } else if (shift < 0) {
822 val >>= -shift;
823 } else {
824 int64_t tmp = val;
825 val <<= shift;
826 if ((val >> shift) != tmp) {
827 SET_QC();
828 val = (tmp >> 63) ^ ~SIGNBIT64;
829 }
830 }
831 return val;
832 }
833
834 #define NEON_FN(dest, src1, src2) do { \
835 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
836 SET_QC(); \
837 dest = 0; \
838 } else { \
839 int8_t tmp; \
840 tmp = (int8_t)src2; \
841 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
842 if (src1) { \
843 SET_QC(); \
844 dest = ~0; \
845 } else { \
846 dest = 0; \
847 } \
848 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
849 dest = 0; \
850 } else if (tmp < 0) { \
851 dest = src1 >> -tmp; \
852 } else { \
853 dest = src1 << tmp; \
854 if ((dest >> tmp) != src1) { \
855 SET_QC(); \
856 dest = ~0; \
857 } \
858 } \
859 }} while (0)
860 NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
861 NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
862 #undef NEON_FN
863
864 uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
865 {
866 if ((int32_t)valop < 0) {
867 SET_QC();
868 return 0;
869 }
870 return helper_neon_qshl_u32(env, valop, shiftop);
871 }
872
873 uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
874 {
875 if ((int64_t)valop < 0) {
876 SET_QC();
877 return 0;
878 }
879 return helper_neon_qshl_u64(env, valop, shiftop);
880 }
881
882 #define NEON_FN(dest, src1, src2) do { \
883 int8_t tmp; \
884 tmp = (int8_t)src2; \
885 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
886 if (src1) { \
887 SET_QC(); \
888 dest = ~0; \
889 } else { \
890 dest = 0; \
891 } \
892 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
893 dest = 0; \
894 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
895 dest = src1 >> (sizeof(src1) * 8 - 1); \
896 } else if (tmp < 0) { \
897 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
898 } else { \
899 dest = src1 << tmp; \
900 if ((dest >> tmp) != src1) { \
901 SET_QC(); \
902 dest = ~0; \
903 } \
904 }} while (0)
905 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
906 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
907 #undef NEON_FN
908
909 /* The addition of the rounding constant may overflow, so we use an
910 * intermediate 64 bit accumulator. */
911 uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
912 {
913 uint32_t dest;
914 int8_t shift = (int8_t)shiftop;
915 if (shift >= 32) {
916 if (val) {
917 SET_QC();
918 dest = ~0;
919 } else {
920 dest = 0;
921 }
922 } else if (shift < -32) {
923 dest = 0;
924 } else if (shift == -32) {
925 dest = val >> 31;
926 } else if (shift < 0) {
927 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
928 dest = big_dest >> -shift;
929 } else {
930 dest = val << shift;
931 if ((dest >> shift) != val) {
932 SET_QC();
933 dest = ~0;
934 }
935 }
936 return dest;
937 }
938
939 /* Handling addition overflow with 64 bit input values is more
940 * tricky than with 32 bit values. */
941 uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
942 {
943 int8_t shift = (int8_t)shiftop;
944 if (shift >= 64) {
945 if (val) {
946 SET_QC();
947 val = ~0;
948 }
949 } else if (shift < -64) {
950 val = 0;
951 } else if (shift == -64) {
952 val >>= 63;
953 } else if (shift < 0) {
954 val >>= (-shift - 1);
955 if (val == UINT64_MAX) {
956 /* In this case, it means that the rounding constant is 1,
957 * and the addition would overflow. Return the actual
958 * result directly. */
959 val = 0x8000000000000000ULL;
960 } else {
961 val++;
962 val >>= 1;
963 }
964 } else { \
965 uint64_t tmp = val;
966 val <<= shift;
967 if ((val >> shift) != tmp) {
968 SET_QC();
969 val = ~0;
970 }
971 }
972 return val;
973 }
974
975 #define NEON_FN(dest, src1, src2) do { \
976 int8_t tmp; \
977 tmp = (int8_t)src2; \
978 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
979 if (src1) { \
980 SET_QC(); \
981 dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
982 if (src1 > 0) { \
983 dest--; \
984 } \
985 } else { \
986 dest = 0; \
987 } \
988 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
989 dest = 0; \
990 } else if (tmp < 0) { \
991 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
992 } else { \
993 dest = src1 << tmp; \
994 if ((dest >> tmp) != src1) { \
995 SET_QC(); \
996 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
997 if (src1 > 0) { \
998 dest--; \
999 } \
1000 } \
1001 }} while (0)
1002 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
1003 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
1004 #undef NEON_FN
1005
1006 /* The addition of the rounding constant may overflow, so we use an
1007 * intermediate 64 bit accumulator. */
1008 uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
1009 {
1010 int32_t dest;
1011 int32_t val = (int32_t)valop;
1012 int8_t shift = (int8_t)shiftop;
1013 if (shift >= 32) {
1014 if (val) {
1015 SET_QC();
1016 dest = (val >> 31) ^ ~SIGNBIT;
1017 } else {
1018 dest = 0;
1019 }
1020 } else if (shift <= -32) {
1021 dest = 0;
1022 } else if (shift < 0) {
1023 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
1024 dest = big_dest >> -shift;
1025 } else {
1026 dest = val << shift;
1027 if ((dest >> shift) != val) {
1028 SET_QC();
1029 dest = (val >> 31) ^ ~SIGNBIT;
1030 }
1031 }
1032 return dest;
1033 }
1034
1035 /* Handling addition overflow with 64 bit input values is more
1036 * tricky than with 32 bit values. */
1037 uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
1038 {
1039 int8_t shift = (uint8_t)shiftop;
1040 int64_t val = valop;
1041
1042 if (shift >= 64) {
1043 if (val) {
1044 SET_QC();
1045 val = (val >> 63) ^ ~SIGNBIT64;
1046 }
1047 } else if (shift <= -64) {
1048 val = 0;
1049 } else if (shift < 0) {
1050 val >>= (-shift - 1);
1051 if (val == INT64_MAX) {
1052 /* In this case, it means that the rounding constant is 1,
1053 * and the addition would overflow. Return the actual
1054 * result directly. */
1055 val = 0x4000000000000000ULL;
1056 } else {
1057 val++;
1058 val >>= 1;
1059 }
1060 } else {
1061 int64_t tmp = val;
1062 val <<= shift;
1063 if ((val >> shift) != tmp) {
1064 SET_QC();
1065 val = (tmp >> 63) ^ ~SIGNBIT64;
1066 }
1067 }
1068 return val;
1069 }
1070
1071 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
1072 {
1073 uint32_t mask;
1074 mask = (a ^ b) & 0x80808080u;
1075 a &= ~0x80808080u;
1076 b &= ~0x80808080u;
1077 return (a + b) ^ mask;
1078 }
1079
1080 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
1081 {
1082 uint32_t mask;
1083 mask = (a ^ b) & 0x80008000u;
1084 a &= ~0x80008000u;
1085 b &= ~0x80008000u;
1086 return (a + b) ^ mask;
1087 }
1088
1089 #define NEON_FN(dest, src1, src2) dest = src1 + src2
1090 NEON_POP(padd_u8, neon_u8, 4)
1091 NEON_POP(padd_u16, neon_u16, 2)
1092 #undef NEON_FN
1093
1094 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1095 NEON_VOP(sub_u8, neon_u8, 4)
1096 NEON_VOP(sub_u16, neon_u16, 2)
1097 #undef NEON_FN
1098
1099 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1100 NEON_VOP(mul_u8, neon_u8, 4)
1101 NEON_VOP(mul_u16, neon_u16, 2)
1102 #undef NEON_FN
1103
1104 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1105 NEON_VOP(tst_u8, neon_u8, 4)
1106 NEON_VOP(tst_u16, neon_u16, 2)
1107 NEON_VOP(tst_u32, neon_u32, 1)
1108 #undef NEON_FN
1109
1110 /* Count Leading Sign/Zero Bits. */
1111 static inline int do_clz8(uint8_t x)
1112 {
1113 int n;
1114 for (n = 8; x; n--)
1115 x >>= 1;
1116 return n;
1117 }
1118
1119 static inline int do_clz16(uint16_t x)
1120 {
1121 int n;
1122 for (n = 16; x; n--)
1123 x >>= 1;
1124 return n;
1125 }
1126
1127 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1128 NEON_VOP1(clz_u8, neon_u8, 4)
1129 #undef NEON_FN
1130
1131 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1132 NEON_VOP1(clz_u16, neon_u16, 2)
1133 #undef NEON_FN
1134
1135 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1136 NEON_VOP1(cls_s8, neon_s8, 4)
1137 #undef NEON_FN
1138
1139 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1140 NEON_VOP1(cls_s16, neon_s16, 2)
1141 #undef NEON_FN
1142
1143 uint32_t HELPER(neon_cls_s32)(uint32_t x)
1144 {
1145 int count;
1146 if ((int32_t)x < 0)
1147 x = ~x;
1148 for (count = 32; x; count--)
1149 x = x >> 1;
1150 return count - 1;
1151 }
1152
1153 /* Bit count. */
1154 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1155 {
1156 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
1157 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
1158 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
1159 return x;
1160 }
1161
1162 /* Reverse bits in each 8 bit word */
1163 uint32_t HELPER(neon_rbit_u8)(uint32_t x)
1164 {
1165 x = ((x & 0xf0f0f0f0) >> 4)
1166 | ((x & 0x0f0f0f0f) << 4);
1167 x = ((x & 0x88888888) >> 3)
1168 | ((x & 0x44444444) >> 1)
1169 | ((x & 0x22222222) << 1)
1170 | ((x & 0x11111111) << 3);
1171 return x;
1172 }
1173
1174 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1175 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1176 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1177 SET_QC(); \
1178 tmp = (tmp >> 31) ^ ~SIGNBIT; \
1179 } else { \
1180 tmp <<= 1; \
1181 } \
1182 if (round) { \
1183 int32_t old = tmp; \
1184 tmp += 1 << 15; \
1185 if ((int32_t)tmp < old) { \
1186 SET_QC(); \
1187 tmp = SIGNBIT - 1; \
1188 } \
1189 } \
1190 dest = tmp >> 16; \
1191 } while(0)
1192 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1193 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
1194 #undef NEON_FN
1195 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1196 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
1197 #undef NEON_FN
1198 #undef NEON_QDMULH16
1199
1200 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1201 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1202 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1203 SET_QC(); \
1204 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1205 } else { \
1206 tmp <<= 1; \
1207 } \
1208 if (round) { \
1209 int64_t old = tmp; \
1210 tmp += (int64_t)1 << 31; \
1211 if ((int64_t)tmp < old) { \
1212 SET_QC(); \
1213 tmp = SIGNBIT64 - 1; \
1214 } \
1215 } \
1216 dest = tmp >> 32; \
1217 } while(0)
1218 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1219 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
1220 #undef NEON_FN
1221 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1222 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
1223 #undef NEON_FN
1224 #undef NEON_QDMULH32
1225
1226 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1227 {
1228 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1229 | ((x >> 24) & 0xff000000u);
1230 }
1231
1232 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1233 {
1234 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1235 }
1236
1237 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1238 {
1239 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1240 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1241 }
1242
1243 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1244 {
1245 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1246 }
1247
1248 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1249 {
1250 x &= 0xff80ff80ff80ff80ull;
1251 x += 0x0080008000800080ull;
1252 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1253 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1254 }
1255
1256 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1257 {
1258 x &= 0xffff8000ffff8000ull;
1259 x += 0x0000800000008000ull;
1260 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1261 }
1262
1263 uint32_t HELPER(neon_unarrow_sat8)(CPUARMState *env, uint64_t x)
1264 {
1265 uint16_t s;
1266 uint8_t d;
1267 uint32_t res = 0;
1268 #define SAT8(n) \
1269 s = x >> n; \
1270 if (s & 0x8000) { \
1271 SET_QC(); \
1272 } else { \
1273 if (s > 0xff) { \
1274 d = 0xff; \
1275 SET_QC(); \
1276 } else { \
1277 d = s; \
1278 } \
1279 res |= (uint32_t)d << (n / 2); \
1280 }
1281
1282 SAT8(0);
1283 SAT8(16);
1284 SAT8(32);
1285 SAT8(48);
1286 #undef SAT8
1287 return res;
1288 }
1289
1290 uint32_t HELPER(neon_narrow_sat_u8)(CPUARMState *env, uint64_t x)
1291 {
1292 uint16_t s;
1293 uint8_t d;
1294 uint32_t res = 0;
1295 #define SAT8(n) \
1296 s = x >> n; \
1297 if (s > 0xff) { \
1298 d = 0xff; \
1299 SET_QC(); \
1300 } else { \
1301 d = s; \
1302 } \
1303 res |= (uint32_t)d << (n / 2);
1304
1305 SAT8(0);
1306 SAT8(16);
1307 SAT8(32);
1308 SAT8(48);
1309 #undef SAT8
1310 return res;
1311 }
1312
1313 uint32_t HELPER(neon_narrow_sat_s8)(CPUARMState *env, uint64_t x)
1314 {
1315 int16_t s;
1316 uint8_t d;
1317 uint32_t res = 0;
1318 #define SAT8(n) \
1319 s = x >> n; \
1320 if (s != (int8_t)s) { \
1321 d = (s >> 15) ^ 0x7f; \
1322 SET_QC(); \
1323 } else { \
1324 d = s; \
1325 } \
1326 res |= (uint32_t)d << (n / 2);
1327
1328 SAT8(0);
1329 SAT8(16);
1330 SAT8(32);
1331 SAT8(48);
1332 #undef SAT8
1333 return res;
1334 }
1335
1336 uint32_t HELPER(neon_unarrow_sat16)(CPUARMState *env, uint64_t x)
1337 {
1338 uint32_t high;
1339 uint32_t low;
1340 low = x;
1341 if (low & 0x80000000) {
1342 low = 0;
1343 SET_QC();
1344 } else if (low > 0xffff) {
1345 low = 0xffff;
1346 SET_QC();
1347 }
1348 high = x >> 32;
1349 if (high & 0x80000000) {
1350 high = 0;
1351 SET_QC();
1352 } else if (high > 0xffff) {
1353 high = 0xffff;
1354 SET_QC();
1355 }
1356 return low | (high << 16);
1357 }
1358
1359 uint32_t HELPER(neon_narrow_sat_u16)(CPUARMState *env, uint64_t x)
1360 {
1361 uint32_t high;
1362 uint32_t low;
1363 low = x;
1364 if (low > 0xffff) {
1365 low = 0xffff;
1366 SET_QC();
1367 }
1368 high = x >> 32;
1369 if (high > 0xffff) {
1370 high = 0xffff;
1371 SET_QC();
1372 }
1373 return low | (high << 16);
1374 }
1375
1376 uint32_t HELPER(neon_narrow_sat_s16)(CPUARMState *env, uint64_t x)
1377 {
1378 int32_t low;
1379 int32_t high;
1380 low = x;
1381 if (low != (int16_t)low) {
1382 low = (low >> 31) ^ 0x7fff;
1383 SET_QC();
1384 }
1385 high = x >> 32;
1386 if (high != (int16_t)high) {
1387 high = (high >> 31) ^ 0x7fff;
1388 SET_QC();
1389 }
1390 return (uint16_t)low | (high << 16);
1391 }
1392
1393 uint32_t HELPER(neon_unarrow_sat32)(CPUARMState *env, uint64_t x)
1394 {
1395 if (x & 0x8000000000000000ull) {
1396 SET_QC();
1397 return 0;
1398 }
1399 if (x > 0xffffffffu) {
1400 SET_QC();
1401 return 0xffffffffu;
1402 }
1403 return x;
1404 }
1405
1406 uint32_t HELPER(neon_narrow_sat_u32)(CPUARMState *env, uint64_t x)
1407 {
1408 if (x > 0xffffffffu) {
1409 SET_QC();
1410 return 0xffffffffu;
1411 }
1412 return x;
1413 }
1414
1415 uint32_t HELPER(neon_narrow_sat_s32)(CPUARMState *env, uint64_t x)
1416 {
1417 if ((int64_t)x != (int32_t)x) {
1418 SET_QC();
1419 return ((int64_t)x >> 63) ^ 0x7fffffff;
1420 }
1421 return x;
1422 }
1423
1424 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1425 {
1426 uint64_t tmp;
1427 uint64_t ret;
1428 ret = (uint8_t)x;
1429 tmp = (uint8_t)(x >> 8);
1430 ret |= tmp << 16;
1431 tmp = (uint8_t)(x >> 16);
1432 ret |= tmp << 32;
1433 tmp = (uint8_t)(x >> 24);
1434 ret |= tmp << 48;
1435 return ret;
1436 }
1437
1438 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1439 {
1440 uint64_t tmp;
1441 uint64_t ret;
1442 ret = (uint16_t)(int8_t)x;
1443 tmp = (uint16_t)(int8_t)(x >> 8);
1444 ret |= tmp << 16;
1445 tmp = (uint16_t)(int8_t)(x >> 16);
1446 ret |= tmp << 32;
1447 tmp = (uint16_t)(int8_t)(x >> 24);
1448 ret |= tmp << 48;
1449 return ret;
1450 }
1451
1452 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1453 {
1454 uint64_t high = (uint16_t)(x >> 16);
1455 return ((uint16_t)x) | (high << 32);
1456 }
1457
1458 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1459 {
1460 uint64_t high = (int16_t)(x >> 16);
1461 return ((uint32_t)(int16_t)x) | (high << 32);
1462 }
1463
1464 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1465 {
1466 uint64_t mask;
1467 mask = (a ^ b) & 0x8000800080008000ull;
1468 a &= ~0x8000800080008000ull;
1469 b &= ~0x8000800080008000ull;
1470 return (a + b) ^ mask;
1471 }
1472
1473 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1474 {
1475 uint64_t mask;
1476 mask = (a ^ b) & 0x8000000080000000ull;
1477 a &= ~0x8000000080000000ull;
1478 b &= ~0x8000000080000000ull;
1479 return (a + b) ^ mask;
1480 }
1481
1482 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1483 {
1484 uint64_t tmp;
1485 uint64_t tmp2;
1486
1487 tmp = a & 0x0000ffff0000ffffull;
1488 tmp += (a >> 16) & 0x0000ffff0000ffffull;
1489 tmp2 = b & 0xffff0000ffff0000ull;
1490 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1491 return ( tmp & 0xffff)
1492 | ((tmp >> 16) & 0xffff0000ull)
1493 | ((tmp2 << 16) & 0xffff00000000ull)
1494 | ( tmp2 & 0xffff000000000000ull);
1495 }
1496
1497 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1498 {
1499 uint32_t low = a + (a >> 32);
1500 uint32_t high = b + (b >> 32);
1501 return low + ((uint64_t)high << 32);
1502 }
1503
1504 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1505 {
1506 uint64_t mask;
1507 mask = (a ^ ~b) & 0x8000800080008000ull;
1508 a |= 0x8000800080008000ull;
1509 b &= ~0x8000800080008000ull;
1510 return (a - b) ^ mask;
1511 }
1512
1513 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1514 {
1515 uint64_t mask;
1516 mask = (a ^ ~b) & 0x8000000080000000ull;
1517 a |= 0x8000000080000000ull;
1518 b &= ~0x8000000080000000ull;
1519 return (a - b) ^ mask;
1520 }
1521
1522 uint64_t HELPER(neon_addl_saturate_s32)(CPUARMState *env, uint64_t a, uint64_t b)
1523 {
1524 uint32_t x, y;
1525 uint32_t low, high;
1526
1527 x = a;
1528 y = b;
1529 low = x + y;
1530 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1531 SET_QC();
1532 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1533 }
1534 x = a >> 32;
1535 y = b >> 32;
1536 high = x + y;
1537 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1538 SET_QC();
1539 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1540 }
1541 return low | ((uint64_t)high << 32);
1542 }
1543
1544 uint64_t HELPER(neon_addl_saturate_s64)(CPUARMState *env, uint64_t a, uint64_t b)
1545 {
1546 uint64_t result;
1547
1548 result = a + b;
1549 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1550 SET_QC();
1551 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1552 }
1553 return result;
1554 }
1555
1556 /* We have to do the arithmetic in a larger type than
1557 * the input type, because for example with a signed 32 bit
1558 * op the absolute difference can overflow a signed 32 bit value.
1559 */
1560 #define DO_ABD(dest, x, y, intype, arithtype) do { \
1561 arithtype tmp_x = (intype)(x); \
1562 arithtype tmp_y = (intype)(y); \
1563 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1564 } while(0)
1565
1566 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1567 {
1568 uint64_t tmp;
1569 uint64_t result;
1570 DO_ABD(result, a, b, uint8_t, uint32_t);
1571 DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
1572 result |= tmp << 16;
1573 DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
1574 result |= tmp << 32;
1575 DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
1576 result |= tmp << 48;
1577 return result;
1578 }
1579
1580 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1581 {
1582 uint64_t tmp;
1583 uint64_t result;
1584 DO_ABD(result, a, b, int8_t, int32_t);
1585 DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
1586 result |= tmp << 16;
1587 DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
1588 result |= tmp << 32;
1589 DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
1590 result |= tmp << 48;
1591 return result;
1592 }
1593
1594 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1595 {
1596 uint64_t tmp;
1597 uint64_t result;
1598 DO_ABD(result, a, b, uint16_t, uint32_t);
1599 DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1600 return result | (tmp << 32);
1601 }
1602
1603 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1604 {
1605 uint64_t tmp;
1606 uint64_t result;
1607 DO_ABD(result, a, b, int16_t, int32_t);
1608 DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
1609 return result | (tmp << 32);
1610 }
1611
1612 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1613 {
1614 uint64_t result;
1615 DO_ABD(result, a, b, uint32_t, uint64_t);
1616 return result;
1617 }
1618
1619 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1620 {
1621 uint64_t result;
1622 DO_ABD(result, a, b, int32_t, int64_t);
1623 return result;
1624 }
1625 #undef DO_ABD
1626
1627 /* Widening multiply. Named type is the source type. */
1628 #define DO_MULL(dest, x, y, type1, type2) do { \
1629 type1 tmp_x = x; \
1630 type1 tmp_y = y; \
1631 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1632 } while(0)
1633
1634 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1635 {
1636 uint64_t tmp;
1637 uint64_t result;
1638
1639 DO_MULL(result, a, b, uint8_t, uint16_t);
1640 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1641 result |= tmp << 16;
1642 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1643 result |= tmp << 32;
1644 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1645 result |= tmp << 48;
1646 return result;
1647 }
1648
1649 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1650 {
1651 uint64_t tmp;
1652 uint64_t result;
1653
1654 DO_MULL(result, a, b, int8_t, uint16_t);
1655 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1656 result |= tmp << 16;
1657 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1658 result |= tmp << 32;
1659 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1660 result |= tmp << 48;
1661 return result;
1662 }
1663
1664 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1665 {
1666 uint64_t tmp;
1667 uint64_t result;
1668
1669 DO_MULL(result, a, b, uint16_t, uint32_t);
1670 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1671 return result | (tmp << 32);
1672 }
1673
1674 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1675 {
1676 uint64_t tmp;
1677 uint64_t result;
1678
1679 DO_MULL(result, a, b, int16_t, uint32_t);
1680 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1681 return result | (tmp << 32);
1682 }
1683
1684 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1685 {
1686 uint16_t tmp;
1687 uint64_t result;
1688 result = (uint16_t)-x;
1689 tmp = -(x >> 16);
1690 result |= (uint64_t)tmp << 16;
1691 tmp = -(x >> 32);
1692 result |= (uint64_t)tmp << 32;
1693 tmp = -(x >> 48);
1694 result |= (uint64_t)tmp << 48;
1695 return result;
1696 }
1697
1698 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1699 {
1700 uint32_t low = -x;
1701 uint32_t high = -(x >> 32);
1702 return low | ((uint64_t)high << 32);
1703 }
1704
1705 /* Saturating sign manipulation. */
1706 /* ??? Make these use NEON_VOP1 */
1707 #define DO_QABS8(x) do { \
1708 if (x == (int8_t)0x80) { \
1709 x = 0x7f; \
1710 SET_QC(); \
1711 } else if (x < 0) { \
1712 x = -x; \
1713 }} while (0)
1714 uint32_t HELPER(neon_qabs_s8)(CPUARMState *env, uint32_t x)
1715 {
1716 neon_s8 vec;
1717 NEON_UNPACK(neon_s8, vec, x);
1718 DO_QABS8(vec.v1);
1719 DO_QABS8(vec.v2);
1720 DO_QABS8(vec.v3);
1721 DO_QABS8(vec.v4);
1722 NEON_PACK(neon_s8, x, vec);
1723 return x;
1724 }
1725 #undef DO_QABS8
1726
1727 #define DO_QNEG8(x) do { \
1728 if (x == (int8_t)0x80) { \
1729 x = 0x7f; \
1730 SET_QC(); \
1731 } else { \
1732 x = -x; \
1733 }} while (0)
1734 uint32_t HELPER(neon_qneg_s8)(CPUARMState *env, uint32_t x)
1735 {
1736 neon_s8 vec;
1737 NEON_UNPACK(neon_s8, vec, x);
1738 DO_QNEG8(vec.v1);
1739 DO_QNEG8(vec.v2);
1740 DO_QNEG8(vec.v3);
1741 DO_QNEG8(vec.v4);
1742 NEON_PACK(neon_s8, x, vec);
1743 return x;
1744 }
1745 #undef DO_QNEG8
1746
1747 #define DO_QABS16(x) do { \
1748 if (x == (int16_t)0x8000) { \
1749 x = 0x7fff; \
1750 SET_QC(); \
1751 } else if (x < 0) { \
1752 x = -x; \
1753 }} while (0)
1754 uint32_t HELPER(neon_qabs_s16)(CPUARMState *env, uint32_t x)
1755 {
1756 neon_s16 vec;
1757 NEON_UNPACK(neon_s16, vec, x);
1758 DO_QABS16(vec.v1);
1759 DO_QABS16(vec.v2);
1760 NEON_PACK(neon_s16, x, vec);
1761 return x;
1762 }
1763 #undef DO_QABS16
1764
1765 #define DO_QNEG16(x) do { \
1766 if (x == (int16_t)0x8000) { \
1767 x = 0x7fff; \
1768 SET_QC(); \
1769 } else { \
1770 x = -x; \
1771 }} while (0)
1772 uint32_t HELPER(neon_qneg_s16)(CPUARMState *env, uint32_t x)
1773 {
1774 neon_s16 vec;
1775 NEON_UNPACK(neon_s16, vec, x);
1776 DO_QNEG16(vec.v1);
1777 DO_QNEG16(vec.v2);
1778 NEON_PACK(neon_s16, x, vec);
1779 return x;
1780 }
1781 #undef DO_QNEG16
1782
1783 uint32_t HELPER(neon_qabs_s32)(CPUARMState *env, uint32_t x)
1784 {
1785 if (x == SIGNBIT) {
1786 SET_QC();
1787 x = ~SIGNBIT;
1788 } else if ((int32_t)x < 0) {
1789 x = -x;
1790 }
1791 return x;
1792 }
1793
1794 uint32_t HELPER(neon_qneg_s32)(CPUARMState *env, uint32_t x)
1795 {
1796 if (x == SIGNBIT) {
1797 SET_QC();
1798 x = ~SIGNBIT;
1799 } else {
1800 x = -x;
1801 }
1802 return x;
1803 }
1804
1805 uint64_t HELPER(neon_qabs_s64)(CPUARMState *env, uint64_t x)
1806 {
1807 if (x == SIGNBIT64) {
1808 SET_QC();
1809 x = ~SIGNBIT64;
1810 } else if ((int64_t)x < 0) {
1811 x = -x;
1812 }
1813 return x;
1814 }
1815
1816 uint64_t HELPER(neon_qneg_s64)(CPUARMState *env, uint64_t x)
1817 {
1818 if (x == SIGNBIT64) {
1819 SET_QC();
1820 x = ~SIGNBIT64;
1821 } else {
1822 x = -x;
1823 }
1824 return x;
1825 }
1826
1827 /* NEON Float helpers. */
1828
1829 /* Floating point comparisons produce an integer result.
1830 * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1831 * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1832 */
1833 uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
1834 {
1835 float_status *fpst = fpstp;
1836 return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
1837 }
1838
1839 uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
1840 {
1841 float_status *fpst = fpstp;
1842 return -float32_le(make_float32(b), make_float32(a), fpst);
1843 }
1844
1845 uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1846 {
1847 float_status *fpst = fpstp;
1848 return -float32_lt(make_float32(b), make_float32(a), fpst);
1849 }
1850
1851 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
1852 {
1853 float_status *fpst = fpstp;
1854 float32 f0 = float32_abs(make_float32(a));
1855 float32 f1 = float32_abs(make_float32(b));
1856 return -float32_le(f1, f0, fpst);
1857 }
1858
1859 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1860 {
1861 float_status *fpst = fpstp;
1862 float32 f0 = float32_abs(make_float32(a));
1863 float32 f1 = float32_abs(make_float32(b));
1864 return -float32_lt(f1, f0, fpst);
1865 }
1866
1867 uint64_t HELPER(neon_acge_f64)(uint64_t a, uint64_t b, void *fpstp)
1868 {
1869 float_status *fpst = fpstp;
1870 float64 f0 = float64_abs(make_float64(a));
1871 float64 f1 = float64_abs(make_float64(b));
1872 return -float64_le(f1, f0, fpst);
1873 }
1874
1875 uint64_t HELPER(neon_acgt_f64)(uint64_t a, uint64_t b, void *fpstp)
1876 {
1877 float_status *fpst = fpstp;
1878 float64 f0 = float64_abs(make_float64(a));
1879 float64 f1 = float64_abs(make_float64(b));
1880 return -float64_lt(f1, f0, fpst);
1881 }
1882
1883 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1884
1885 void HELPER(neon_qunzip8)(void *vd, void *vm)
1886 {
1887 uint64_t *rd = vd, *rm = vm;
1888 uint64_t zd0 = rd[0], zd1 = rd[1];
1889 uint64_t zm0 = rm[0], zm1 = rm[1];
1890
1891 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
1892 | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
1893 | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
1894 | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
1895 uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
1896 | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
1897 | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1898 | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
1899 uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
1900 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
1901 | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
1902 | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
1903 uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
1904 | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
1905 | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
1906 | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
1907
1908 rm[0] = m0;
1909 rm[1] = m1;
1910 rd[0] = d0;
1911 rd[1] = d1;
1912 }
1913
1914 void HELPER(neon_qunzip16)(void *vd, void *vm)
1915 {
1916 uint64_t *rd = vd, *rm = vm;
1917 uint64_t zd0 = rd[0], zd1 = rd[1];
1918 uint64_t zm0 = rm[0], zm1 = rm[1];
1919
1920 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
1921 | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
1922 uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
1923 | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
1924 uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
1925 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
1926 uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
1927 | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
1928
1929 rm[0] = m0;
1930 rm[1] = m1;
1931 rd[0] = d0;
1932 rd[1] = d1;
1933 }
1934
1935 void HELPER(neon_qunzip32)(void *vd, void *vm)
1936 {
1937 uint64_t *rd = vd, *rm = vm;
1938 uint64_t zd0 = rd[0], zd1 = rd[1];
1939 uint64_t zm0 = rm[0], zm1 = rm[1];
1940
1941 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
1942 uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1943 uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
1944 uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
1945
1946 rm[0] = m0;
1947 rm[1] = m1;
1948 rd[0] = d0;
1949 rd[1] = d1;
1950 }
1951
1952 void HELPER(neon_unzip8)(void *vd, void *vm)
1953 {
1954 uint64_t *rd = vd, *rm = vm;
1955 uint64_t zd = rd[0], zm = rm[0];
1956
1957 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
1958 | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
1959 | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
1960 | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
1961 uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
1962 | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
1963 | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
1964 | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
1965
1966 rm[0] = m0;
1967 rd[0] = d0;
1968 }
1969
1970 void HELPER(neon_unzip16)(void *vd, void *vm)
1971 {
1972 uint64_t *rd = vd, *rm = vm;
1973 uint64_t zd = rd[0], zm = rm[0];
1974
1975 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
1976 | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
1977 uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
1978 | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
1979
1980 rm[0] = m0;
1981 rd[0] = d0;
1982 }
1983
1984 void HELPER(neon_qzip8)(void *vd, void *vm)
1985 {
1986 uint64_t *rd = vd, *rm = vm;
1987 uint64_t zd0 = rd[0], zd1 = rd[1];
1988 uint64_t zm0 = rm[0], zm1 = rm[1];
1989
1990 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
1991 | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
1992 | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
1993 | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
1994 uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
1995 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
1996 | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
1997 | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
1998 uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
1999 | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
2000 | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2001 | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
2002 uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
2003 | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
2004 | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
2005 | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2006
2007 rm[0] = m0;
2008 rm[1] = m1;
2009 rd[0] = d0;
2010 rd[1] = d1;
2011 }
2012
2013 void HELPER(neon_qzip16)(void *vd, void *vm)
2014 {
2015 uint64_t *rd = vd, *rm = vm;
2016 uint64_t zd0 = rd[0], zd1 = rd[1];
2017 uint64_t zm0 = rm[0], zm1 = rm[1];
2018
2019 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
2020 | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
2021 uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
2022 | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
2023 uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
2024 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
2025 uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
2026 | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2027
2028 rm[0] = m0;
2029 rm[1] = m1;
2030 rd[0] = d0;
2031 rd[1] = d1;
2032 }
2033
2034 void HELPER(neon_qzip32)(void *vd, void *vm)
2035 {
2036 uint64_t *rd = vd, *rm = vm;
2037 uint64_t zd0 = rd[0], zd1 = rd[1];
2038 uint64_t zm0 = rm[0], zm1 = rm[1];
2039
2040 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
2041 uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
2042 uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2043 uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2044
2045 rm[0] = m0;
2046 rm[1] = m1;
2047 rd[0] = d0;
2048 rd[1] = d1;
2049 }
2050
2051 void HELPER(neon_zip8)(void *vd, void *vm)
2052 {
2053 uint64_t *rd = vd, *rm = vm;
2054 uint64_t zd = rd[0], zm = rm[0];
2055
2056 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2057 | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2058 | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2059 | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2060 uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2061 | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2062 | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2063 | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2064
2065 rm[0] = m0;
2066 rd[0] = d0;
2067 }
2068
2069 void HELPER(neon_zip16)(void *vd, void *vm)
2070 {
2071 uint64_t *rd = vd, *rm = vm;
2072 uint64_t zd = rd[0], zm = rm[0];
2073
2074 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2075 | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2076 uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2077 | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2078
2079 rm[0] = m0;
2080 rd[0] = d0;
2081 }