Merge tag 'for-upstream' of git://repo.or.cz/qemu/kevin into staging
[qemu.git] / tcg / tcg-op-vec.c
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2018 Linaro, Inc.
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "tcg/tcg.h"
22 #include "tcg/tcg-op.h"
23 #include "tcg/tcg-mo.h"
24
25 /* Reduce the number of ifdefs below. This assumes that all uses of
26 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
27 the compiler can eliminate. */
28 #if TCG_TARGET_REG_BITS == 64
29 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
30 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
31 #define TCGV_LOW TCGV_LOW_link_error
32 #define TCGV_HIGH TCGV_HIGH_link_error
33 #endif
34
35 /*
36 * Vector optional opcode tracking.
37 * Except for the basic logical operations (and, or, xor), and
38 * data movement (mov, ld, st, dupi), many vector opcodes are
39 * optional and may not be supported on the host. Thank Intel
40 * for the irregularity in their instruction set.
41 *
42 * The gvec expanders allow custom vector operations to be composed,
43 * generally via the .fniv callback in the GVecGen* structures. At
44 * the same time, in deciding whether to use this hook we need to
45 * know if the host supports the required operations. This is
46 * presented as an array of opcodes, terminated by 0. Each opcode
47 * is assumed to be expanded with the given VECE.
48 *
49 * For debugging, we want to validate this array. Therefore, when
50 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
51 * will validate that their opcode is present in the list.
52 */
53 #ifdef CONFIG_DEBUG_TCG
54 void tcg_assert_listed_vecop(TCGOpcode op)
55 {
56 const TCGOpcode *p = tcg_ctx->vecop_list;
57 if (p) {
58 for (; *p; ++p) {
59 if (*p == op) {
60 return;
61 }
62 }
63 g_assert_not_reached();
64 }
65 }
66 #endif
67
68 bool tcg_can_emit_vecop_list(const TCGOpcode *list,
69 TCGType type, unsigned vece)
70 {
71 if (list == NULL) {
72 return true;
73 }
74
75 for (; *list; ++list) {
76 TCGOpcode opc = *list;
77
78 #ifdef CONFIG_DEBUG_TCG
79 switch (opc) {
80 case INDEX_op_and_vec:
81 case INDEX_op_or_vec:
82 case INDEX_op_xor_vec:
83 case INDEX_op_mov_vec:
84 case INDEX_op_dup_vec:
85 case INDEX_op_dup2_vec:
86 case INDEX_op_ld_vec:
87 case INDEX_op_st_vec:
88 case INDEX_op_bitsel_vec:
89 /* These opcodes are mandatory and should not be listed. */
90 g_assert_not_reached();
91 case INDEX_op_not_vec:
92 /* These opcodes have generic expansions using the above. */
93 g_assert_not_reached();
94 default:
95 break;
96 }
97 #endif
98
99 if (tcg_can_emit_vec_op(opc, type, vece)) {
100 continue;
101 }
102
103 /*
104 * The opcode list is created by front ends based on what they
105 * actually invoke. We must mirror the logic in the routines
106 * below for generic expansions using other opcodes.
107 */
108 switch (opc) {
109 case INDEX_op_neg_vec:
110 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) {
111 continue;
112 }
113 break;
114 case INDEX_op_abs_vec:
115 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)
116 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0
117 || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0
118 || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) {
119 continue;
120 }
121 break;
122 case INDEX_op_usadd_vec:
123 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) ||
124 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
125 continue;
126 }
127 break;
128 case INDEX_op_ussub_vec:
129 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) ||
130 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
131 continue;
132 }
133 break;
134 case INDEX_op_cmpsel_vec:
135 case INDEX_op_smin_vec:
136 case INDEX_op_smax_vec:
137 case INDEX_op_umin_vec:
138 case INDEX_op_umax_vec:
139 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
140 continue;
141 }
142 break;
143 default:
144 break;
145 }
146 return false;
147 }
148 return true;
149 }
150
151 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
152 {
153 TCGOp *op = tcg_emit_op(opc);
154 TCGOP_VECL(op) = type - TCG_TYPE_V64;
155 TCGOP_VECE(op) = vece;
156 op->args[0] = r;
157 op->args[1] = a;
158 }
159
160 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
161 TCGArg r, TCGArg a, TCGArg b)
162 {
163 TCGOp *op = tcg_emit_op(opc);
164 TCGOP_VECL(op) = type - TCG_TYPE_V64;
165 TCGOP_VECE(op) = vece;
166 op->args[0] = r;
167 op->args[1] = a;
168 op->args[2] = b;
169 }
170
171 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
172 TCGArg r, TCGArg a, TCGArg b, TCGArg c)
173 {
174 TCGOp *op = tcg_emit_op(opc);
175 TCGOP_VECL(op) = type - TCG_TYPE_V64;
176 TCGOP_VECE(op) = vece;
177 op->args[0] = r;
178 op->args[1] = a;
179 op->args[2] = b;
180 op->args[3] = c;
181 }
182
183 static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r,
184 TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e)
185 {
186 TCGOp *op = tcg_emit_op(opc);
187 TCGOP_VECL(op) = type - TCG_TYPE_V64;
188 TCGOP_VECE(op) = vece;
189 op->args[0] = r;
190 op->args[1] = a;
191 op->args[2] = b;
192 op->args[3] = c;
193 op->args[4] = d;
194 op->args[5] = e;
195 }
196
197 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
198 {
199 TCGTemp *rt = tcgv_vec_temp(r);
200 TCGTemp *at = tcgv_vec_temp(a);
201 TCGType type = rt->base_type;
202
203 /* Must enough inputs for the output. */
204 tcg_debug_assert(at->base_type >= type);
205 vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
206 }
207
208 static void vec_gen_op3(TCGOpcode opc, unsigned vece,
209 TCGv_vec r, TCGv_vec a, TCGv_vec b)
210 {
211 TCGTemp *rt = tcgv_vec_temp(r);
212 TCGTemp *at = tcgv_vec_temp(a);
213 TCGTemp *bt = tcgv_vec_temp(b);
214 TCGType type = rt->base_type;
215
216 /* Must enough inputs for the output. */
217 tcg_debug_assert(at->base_type >= type);
218 tcg_debug_assert(bt->base_type >= type);
219 vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
220 }
221
222 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
223 {
224 if (r != a) {
225 vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
226 }
227 }
228
229 TCGv_vec tcg_const_zeros_vec(TCGType type)
230 {
231 TCGv_vec ret = tcg_temp_new_vec(type);
232 tcg_gen_dupi_vec(MO_64, ret, 0);
233 return ret;
234 }
235
236 TCGv_vec tcg_const_ones_vec(TCGType type)
237 {
238 TCGv_vec ret = tcg_temp_new_vec(type);
239 tcg_gen_dupi_vec(MO_64, ret, -1);
240 return ret;
241 }
242
243 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m)
244 {
245 TCGTemp *t = tcgv_vec_temp(m);
246 return tcg_const_zeros_vec(t->base_type);
247 }
248
249 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m)
250 {
251 TCGTemp *t = tcgv_vec_temp(m);
252 return tcg_const_ones_vec(t->base_type);
253 }
254
255 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
256 {
257 TCGTemp *rt = tcgv_vec_temp(r);
258 tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a));
259 }
260
261 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
262 {
263 TCGArg ri = tcgv_vec_arg(r);
264 TCGTemp *rt = arg_temp(ri);
265 TCGType type = rt->base_type;
266
267 if (TCG_TARGET_REG_BITS == 64) {
268 TCGArg ai = tcgv_i64_arg(a);
269 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
270 } else if (vece == MO_64) {
271 TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
272 TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
273 vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
274 } else {
275 TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
276 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
277 }
278 }
279
280 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
281 {
282 TCGArg ri = tcgv_vec_arg(r);
283 TCGArg ai = tcgv_i32_arg(a);
284 TCGTemp *rt = arg_temp(ri);
285 TCGType type = rt->base_type;
286
287 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
288 }
289
290 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b,
291 tcg_target_long ofs)
292 {
293 TCGArg ri = tcgv_vec_arg(r);
294 TCGArg bi = tcgv_ptr_arg(b);
295 TCGTemp *rt = arg_temp(ri);
296 TCGType type = rt->base_type;
297
298 vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs);
299 }
300
301 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
302 {
303 TCGArg ri = tcgv_vec_arg(r);
304 TCGArg bi = tcgv_ptr_arg(b);
305 TCGTemp *rt = arg_temp(ri);
306 TCGType type = rt->base_type;
307
308 vec_gen_3(opc, type, 0, ri, bi, o);
309 }
310
311 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
312 {
313 vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
314 }
315
316 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
317 {
318 vec_gen_ldst(INDEX_op_st_vec, r, b, o);
319 }
320
321 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
322 {
323 TCGArg ri = tcgv_vec_arg(r);
324 TCGArg bi = tcgv_ptr_arg(b);
325 TCGTemp *rt = arg_temp(ri);
326 TCGType type = rt->base_type;
327
328 tcg_debug_assert(low_type >= TCG_TYPE_V64);
329 tcg_debug_assert(low_type <= type);
330 vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
331 }
332
333 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
334 {
335 vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
336 }
337
338 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
339 {
340 vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
341 }
342
343 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
344 {
345 vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
346 }
347
348 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
349 {
350 if (TCG_TARGET_HAS_andc_vec) {
351 vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
352 } else {
353 TCGv_vec t = tcg_temp_new_vec_matching(r);
354 tcg_gen_not_vec(0, t, b);
355 tcg_gen_and_vec(0, r, a, t);
356 tcg_temp_free_vec(t);
357 }
358 }
359
360 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
361 {
362 if (TCG_TARGET_HAS_orc_vec) {
363 vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
364 } else {
365 TCGv_vec t = tcg_temp_new_vec_matching(r);
366 tcg_gen_not_vec(0, t, b);
367 tcg_gen_or_vec(0, r, a, t);
368 tcg_temp_free_vec(t);
369 }
370 }
371
372 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
373 {
374 if (TCG_TARGET_HAS_nand_vec) {
375 vec_gen_op3(INDEX_op_nand_vec, 0, r, a, b);
376 } else {
377 tcg_gen_and_vec(0, r, a, b);
378 tcg_gen_not_vec(0, r, r);
379 }
380 }
381
382 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
383 {
384 if (TCG_TARGET_HAS_nor_vec) {
385 vec_gen_op3(INDEX_op_nor_vec, 0, r, a, b);
386 } else {
387 tcg_gen_or_vec(0, r, a, b);
388 tcg_gen_not_vec(0, r, r);
389 }
390 }
391
392 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
393 {
394 if (TCG_TARGET_HAS_eqv_vec) {
395 vec_gen_op3(INDEX_op_eqv_vec, 0, r, a, b);
396 } else {
397 tcg_gen_xor_vec(0, r, a, b);
398 tcg_gen_not_vec(0, r, r);
399 }
400 }
401
402 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc)
403 {
404 TCGTemp *rt = tcgv_vec_temp(r);
405 TCGTemp *at = tcgv_vec_temp(a);
406 TCGArg ri = temp_arg(rt);
407 TCGArg ai = temp_arg(at);
408 TCGType type = rt->base_type;
409 int can;
410
411 tcg_debug_assert(at->base_type >= type);
412 tcg_assert_listed_vecop(opc);
413 can = tcg_can_emit_vec_op(opc, type, vece);
414 if (can > 0) {
415 vec_gen_2(opc, type, vece, ri, ai);
416 } else if (can < 0) {
417 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
418 tcg_expand_vec_op(opc, type, vece, ri, ai);
419 tcg_swap_vecop_list(hold_list);
420 } else {
421 return false;
422 }
423 return true;
424 }
425
426 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
427 {
428 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
429
430 if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) {
431 TCGv_vec t = tcg_const_ones_vec_matching(r);
432 tcg_gen_xor_vec(0, r, a, t);
433 tcg_temp_free_vec(t);
434 }
435 tcg_swap_vecop_list(hold_list);
436 }
437
438 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
439 {
440 const TCGOpcode *hold_list;
441
442 tcg_assert_listed_vecop(INDEX_op_neg_vec);
443 hold_list = tcg_swap_vecop_list(NULL);
444
445 if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) {
446 TCGv_vec t = tcg_const_zeros_vec_matching(r);
447 tcg_gen_sub_vec(vece, r, t, a);
448 tcg_temp_free_vec(t);
449 }
450 tcg_swap_vecop_list(hold_list);
451 }
452
453 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
454 {
455 const TCGOpcode *hold_list;
456
457 tcg_assert_listed_vecop(INDEX_op_abs_vec);
458 hold_list = tcg_swap_vecop_list(NULL);
459
460 if (!do_op2(vece, r, a, INDEX_op_abs_vec)) {
461 TCGType type = tcgv_vec_temp(r)->base_type;
462 TCGv_vec t = tcg_temp_new_vec(type);
463
464 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece));
465 if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) {
466 tcg_gen_neg_vec(vece, t, a);
467 tcg_gen_smax_vec(vece, r, a, t);
468 } else {
469 if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) {
470 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1);
471 } else {
472 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a,
473 tcg_constant_vec(type, vece, 0));
474 }
475 tcg_gen_xor_vec(vece, r, a, t);
476 tcg_gen_sub_vec(vece, r, r, t);
477 }
478
479 tcg_temp_free_vec(t);
480 }
481 tcg_swap_vecop_list(hold_list);
482 }
483
484 static void do_shifti(TCGOpcode opc, unsigned vece,
485 TCGv_vec r, TCGv_vec a, int64_t i)
486 {
487 TCGTemp *rt = tcgv_vec_temp(r);
488 TCGTemp *at = tcgv_vec_temp(a);
489 TCGArg ri = temp_arg(rt);
490 TCGArg ai = temp_arg(at);
491 TCGType type = rt->base_type;
492 int can;
493
494 tcg_debug_assert(at->base_type == type);
495 tcg_debug_assert(i >= 0 && i < (8 << vece));
496 tcg_assert_listed_vecop(opc);
497
498 if (i == 0) {
499 tcg_gen_mov_vec(r, a);
500 return;
501 }
502
503 can = tcg_can_emit_vec_op(opc, type, vece);
504 if (can > 0) {
505 vec_gen_3(opc, type, vece, ri, ai, i);
506 } else {
507 /* We leave the choice of expansion via scalar or vector shift
508 to the target. Often, but not always, dupi can feed a vector
509 shift easier than a scalar. */
510 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
511 tcg_debug_assert(can < 0);
512 tcg_expand_vec_op(opc, type, vece, ri, ai, i);
513 tcg_swap_vecop_list(hold_list);
514 }
515 }
516
517 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
518 {
519 do_shifti(INDEX_op_shli_vec, vece, r, a, i);
520 }
521
522 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
523 {
524 do_shifti(INDEX_op_shri_vec, vece, r, a, i);
525 }
526
527 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
528 {
529 do_shifti(INDEX_op_sari_vec, vece, r, a, i);
530 }
531
532 void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
533 {
534 do_shifti(INDEX_op_rotli_vec, vece, r, a, i);
535 }
536
537 void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
538 {
539 int bits = 8 << vece;
540 tcg_debug_assert(i >= 0 && i < bits);
541 do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1));
542 }
543
544 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
545 TCGv_vec r, TCGv_vec a, TCGv_vec b)
546 {
547 TCGTemp *rt = tcgv_vec_temp(r);
548 TCGTemp *at = tcgv_vec_temp(a);
549 TCGTemp *bt = tcgv_vec_temp(b);
550 TCGArg ri = temp_arg(rt);
551 TCGArg ai = temp_arg(at);
552 TCGArg bi = temp_arg(bt);
553 TCGType type = rt->base_type;
554 int can;
555
556 tcg_debug_assert(at->base_type >= type);
557 tcg_debug_assert(bt->base_type >= type);
558 tcg_assert_listed_vecop(INDEX_op_cmp_vec);
559 can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
560 if (can > 0) {
561 vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
562 } else {
563 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
564 tcg_debug_assert(can < 0);
565 tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
566 tcg_swap_vecop_list(hold_list);
567 }
568 }
569
570 static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a,
571 TCGv_vec b, TCGOpcode opc)
572 {
573 TCGTemp *rt = tcgv_vec_temp(r);
574 TCGTemp *at = tcgv_vec_temp(a);
575 TCGTemp *bt = tcgv_vec_temp(b);
576 TCGArg ri = temp_arg(rt);
577 TCGArg ai = temp_arg(at);
578 TCGArg bi = temp_arg(bt);
579 TCGType type = rt->base_type;
580 int can;
581
582 tcg_debug_assert(at->base_type >= type);
583 tcg_debug_assert(bt->base_type >= type);
584 tcg_assert_listed_vecop(opc);
585 can = tcg_can_emit_vec_op(opc, type, vece);
586 if (can > 0) {
587 vec_gen_3(opc, type, vece, ri, ai, bi);
588 } else if (can < 0) {
589 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
590 tcg_expand_vec_op(opc, type, vece, ri, ai, bi);
591 tcg_swap_vecop_list(hold_list);
592 } else {
593 return false;
594 }
595 return true;
596 }
597
598 static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a,
599 TCGv_vec b, TCGOpcode opc)
600 {
601 bool ok = do_op3(vece, r, a, b, opc);
602 tcg_debug_assert(ok);
603 }
604
605 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
606 {
607 do_op3_nofail(vece, r, a, b, INDEX_op_add_vec);
608 }
609
610 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
611 {
612 do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec);
613 }
614
615 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
616 {
617 do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec);
618 }
619
620 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
621 {
622 do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec);
623 }
624
625 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
626 {
627 if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) {
628 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
629 TCGv_vec t = tcg_temp_new_vec_matching(r);
630
631 /* usadd(a, b) = min(a, ~b) + b */
632 tcg_gen_not_vec(vece, t, b);
633 tcg_gen_umin_vec(vece, t, t, a);
634 tcg_gen_add_vec(vece, r, t, b);
635
636 tcg_temp_free_vec(t);
637 tcg_swap_vecop_list(hold_list);
638 }
639 }
640
641 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
642 {
643 do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec);
644 }
645
646 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
647 {
648 if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) {
649 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
650 TCGv_vec t = tcg_temp_new_vec_matching(r);
651
652 /* ussub(a, b) = max(a, b) - b */
653 tcg_gen_umax_vec(vece, t, a, b);
654 tcg_gen_sub_vec(vece, r, t, b);
655
656 tcg_temp_free_vec(t);
657 tcg_swap_vecop_list(hold_list);
658 }
659 }
660
661 static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a,
662 TCGv_vec b, TCGOpcode opc, TCGCond cond)
663 {
664 if (!do_op3(vece, r, a, b, opc)) {
665 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
666 tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b);
667 tcg_swap_vecop_list(hold_list);
668 }
669 }
670
671 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
672 {
673 do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT);
674 }
675
676 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
677 {
678 do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU);
679 }
680
681 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
682 {
683 do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT);
684 }
685
686 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
687 {
688 do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU);
689 }
690
691 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
692 {
693 do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec);
694 }
695
696 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
697 {
698 do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec);
699 }
700
701 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
702 {
703 do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec);
704 }
705
706 void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
707 {
708 do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec);
709 }
710
711 void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
712 {
713 do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec);
714 }
715
716 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a,
717 TCGv_i32 s, TCGOpcode opc)
718 {
719 TCGTemp *rt = tcgv_vec_temp(r);
720 TCGTemp *at = tcgv_vec_temp(a);
721 TCGTemp *st = tcgv_i32_temp(s);
722 TCGArg ri = temp_arg(rt);
723 TCGArg ai = temp_arg(at);
724 TCGArg si = temp_arg(st);
725 TCGType type = rt->base_type;
726 int can;
727
728 tcg_debug_assert(at->base_type >= type);
729 tcg_assert_listed_vecop(opc);
730 can = tcg_can_emit_vec_op(opc, type, vece);
731 if (can > 0) {
732 vec_gen_3(opc, type, vece, ri, ai, si);
733 } else if (can < 0) {
734 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
735 tcg_expand_vec_op(opc, type, vece, ri, ai, si);
736 tcg_swap_vecop_list(hold_list);
737 } else {
738 g_assert_not_reached();
739 }
740 }
741
742 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
743 {
744 do_shifts(vece, r, a, b, INDEX_op_shls_vec);
745 }
746
747 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
748 {
749 do_shifts(vece, r, a, b, INDEX_op_shrs_vec);
750 }
751
752 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
753 {
754 do_shifts(vece, r, a, b, INDEX_op_sars_vec);
755 }
756
757 void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s)
758 {
759 do_shifts(vece, r, a, s, INDEX_op_rotls_vec);
760 }
761
762 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a,
763 TCGv_vec b, TCGv_vec c)
764 {
765 TCGTemp *rt = tcgv_vec_temp(r);
766 TCGTemp *at = tcgv_vec_temp(a);
767 TCGTemp *bt = tcgv_vec_temp(b);
768 TCGTemp *ct = tcgv_vec_temp(c);
769 TCGType type = rt->base_type;
770
771 tcg_debug_assert(at->base_type >= type);
772 tcg_debug_assert(bt->base_type >= type);
773 tcg_debug_assert(ct->base_type >= type);
774
775 if (TCG_TARGET_HAS_bitsel_vec) {
776 vec_gen_4(INDEX_op_bitsel_vec, type, MO_8,
777 temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct));
778 } else {
779 TCGv_vec t = tcg_temp_new_vec(type);
780 tcg_gen_and_vec(MO_8, t, a, b);
781 tcg_gen_andc_vec(MO_8, r, c, a);
782 tcg_gen_or_vec(MO_8, r, r, t);
783 tcg_temp_free_vec(t);
784 }
785 }
786
787 void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r,
788 TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d)
789 {
790 TCGTemp *rt = tcgv_vec_temp(r);
791 TCGTemp *at = tcgv_vec_temp(a);
792 TCGTemp *bt = tcgv_vec_temp(b);
793 TCGTemp *ct = tcgv_vec_temp(c);
794 TCGTemp *dt = tcgv_vec_temp(d);
795 TCGArg ri = temp_arg(rt);
796 TCGArg ai = temp_arg(at);
797 TCGArg bi = temp_arg(bt);
798 TCGArg ci = temp_arg(ct);
799 TCGArg di = temp_arg(dt);
800 TCGType type = rt->base_type;
801 const TCGOpcode *hold_list;
802 int can;
803
804 tcg_debug_assert(at->base_type >= type);
805 tcg_debug_assert(bt->base_type >= type);
806 tcg_debug_assert(ct->base_type >= type);
807 tcg_debug_assert(dt->base_type >= type);
808
809 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec);
810 hold_list = tcg_swap_vecop_list(NULL);
811 can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece);
812
813 if (can > 0) {
814 vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond);
815 } else if (can < 0) {
816 tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece,
817 ri, ai, bi, ci, di, cond);
818 } else {
819 TCGv_vec t = tcg_temp_new_vec(type);
820 tcg_gen_cmp_vec(cond, vece, t, a, b);
821 tcg_gen_bitsel_vec(vece, r, t, c, d);
822 tcg_temp_free_vec(t);
823 }
824 tcg_swap_vecop_list(hold_list);
825 }