Merge remote-tracking branch 'remotes/rth/tags/pull-arm-20211021' into staging
[qemu.git] / tcg / tcg-op-vec.c
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2018 Linaro, Inc.
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "tcg/tcg.h"
22 #include "tcg/tcg-op.h"
23 #include "tcg/tcg-mo.h"
24
25 /* Reduce the number of ifdefs below. This assumes that all uses of
26 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
27 the compiler can eliminate. */
28 #if TCG_TARGET_REG_BITS == 64
29 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
30 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
31 #define TCGV_LOW TCGV_LOW_link_error
32 #define TCGV_HIGH TCGV_HIGH_link_error
33 #endif
34
35 /*
36 * Vector optional opcode tracking.
37 * Except for the basic logical operations (and, or, xor), and
38 * data movement (mov, ld, st, dupi), many vector opcodes are
39 * optional and may not be supported on the host. Thank Intel
40 * for the irregularity in their instruction set.
41 *
42 * The gvec expanders allow custom vector operations to be composed,
43 * generally via the .fniv callback in the GVecGen* structures. At
44 * the same time, in deciding whether to use this hook we need to
45 * know if the host supports the required operations. This is
46 * presented as an array of opcodes, terminated by 0. Each opcode
47 * is assumed to be expanded with the given VECE.
48 *
49 * For debugging, we want to validate this array. Therefore, when
50 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
51 * will validate that their opcode is present in the list.
52 */
53 #ifdef CONFIG_DEBUG_TCG
54 void tcg_assert_listed_vecop(TCGOpcode op)
55 {
56 const TCGOpcode *p = tcg_ctx->vecop_list;
57 if (p) {
58 for (; *p; ++p) {
59 if (*p == op) {
60 return;
61 }
62 }
63 g_assert_not_reached();
64 }
65 }
66 #endif
67
68 bool tcg_can_emit_vecop_list(const TCGOpcode *list,
69 TCGType type, unsigned vece)
70 {
71 if (list == NULL) {
72 return true;
73 }
74
75 for (; *list; ++list) {
76 TCGOpcode opc = *list;
77
78 #ifdef CONFIG_DEBUG_TCG
79 switch (opc) {
80 case INDEX_op_and_vec:
81 case INDEX_op_or_vec:
82 case INDEX_op_xor_vec:
83 case INDEX_op_mov_vec:
84 case INDEX_op_dup_vec:
85 case INDEX_op_dup2_vec:
86 case INDEX_op_ld_vec:
87 case INDEX_op_st_vec:
88 case INDEX_op_bitsel_vec:
89 /* These opcodes are mandatory and should not be listed. */
90 g_assert_not_reached();
91 case INDEX_op_not_vec:
92 /* These opcodes have generic expansions using the above. */
93 g_assert_not_reached();
94 default:
95 break;
96 }
97 #endif
98
99 if (tcg_can_emit_vec_op(opc, type, vece)) {
100 continue;
101 }
102
103 /*
104 * The opcode list is created by front ends based on what they
105 * actually invoke. We must mirror the logic in the routines
106 * below for generic expansions using other opcodes.
107 */
108 switch (opc) {
109 case INDEX_op_neg_vec:
110 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) {
111 continue;
112 }
113 break;
114 case INDEX_op_abs_vec:
115 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)
116 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0
117 || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0
118 || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) {
119 continue;
120 }
121 break;
122 case INDEX_op_cmpsel_vec:
123 case INDEX_op_smin_vec:
124 case INDEX_op_smax_vec:
125 case INDEX_op_umin_vec:
126 case INDEX_op_umax_vec:
127 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
128 continue;
129 }
130 break;
131 default:
132 break;
133 }
134 return false;
135 }
136 return true;
137 }
138
139 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
140 {
141 TCGOp *op = tcg_emit_op(opc);
142 TCGOP_VECL(op) = type - TCG_TYPE_V64;
143 TCGOP_VECE(op) = vece;
144 op->args[0] = r;
145 op->args[1] = a;
146 }
147
148 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
149 TCGArg r, TCGArg a, TCGArg b)
150 {
151 TCGOp *op = tcg_emit_op(opc);
152 TCGOP_VECL(op) = type - TCG_TYPE_V64;
153 TCGOP_VECE(op) = vece;
154 op->args[0] = r;
155 op->args[1] = a;
156 op->args[2] = b;
157 }
158
159 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
160 TCGArg r, TCGArg a, TCGArg b, TCGArg c)
161 {
162 TCGOp *op = tcg_emit_op(opc);
163 TCGOP_VECL(op) = type - TCG_TYPE_V64;
164 TCGOP_VECE(op) = vece;
165 op->args[0] = r;
166 op->args[1] = a;
167 op->args[2] = b;
168 op->args[3] = c;
169 }
170
171 static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r,
172 TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e)
173 {
174 TCGOp *op = tcg_emit_op(opc);
175 TCGOP_VECL(op) = type - TCG_TYPE_V64;
176 TCGOP_VECE(op) = vece;
177 op->args[0] = r;
178 op->args[1] = a;
179 op->args[2] = b;
180 op->args[3] = c;
181 op->args[4] = d;
182 op->args[5] = e;
183 }
184
185 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
186 {
187 TCGTemp *rt = tcgv_vec_temp(r);
188 TCGTemp *at = tcgv_vec_temp(a);
189 TCGType type = rt->base_type;
190
191 /* Must enough inputs for the output. */
192 tcg_debug_assert(at->base_type >= type);
193 vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
194 }
195
196 static void vec_gen_op3(TCGOpcode opc, unsigned vece,
197 TCGv_vec r, TCGv_vec a, TCGv_vec b)
198 {
199 TCGTemp *rt = tcgv_vec_temp(r);
200 TCGTemp *at = tcgv_vec_temp(a);
201 TCGTemp *bt = tcgv_vec_temp(b);
202 TCGType type = rt->base_type;
203
204 /* Must enough inputs for the output. */
205 tcg_debug_assert(at->base_type >= type);
206 tcg_debug_assert(bt->base_type >= type);
207 vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
208 }
209
210 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
211 {
212 if (r != a) {
213 vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
214 }
215 }
216
217 TCGv_vec tcg_const_zeros_vec(TCGType type)
218 {
219 TCGv_vec ret = tcg_temp_new_vec(type);
220 tcg_gen_dupi_vec(MO_64, ret, 0);
221 return ret;
222 }
223
224 TCGv_vec tcg_const_ones_vec(TCGType type)
225 {
226 TCGv_vec ret = tcg_temp_new_vec(type);
227 tcg_gen_dupi_vec(MO_64, ret, -1);
228 return ret;
229 }
230
231 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m)
232 {
233 TCGTemp *t = tcgv_vec_temp(m);
234 return tcg_const_zeros_vec(t->base_type);
235 }
236
237 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m)
238 {
239 TCGTemp *t = tcgv_vec_temp(m);
240 return tcg_const_ones_vec(t->base_type);
241 }
242
243 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
244 {
245 TCGTemp *rt = tcgv_vec_temp(r);
246 tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a));
247 }
248
249 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
250 {
251 TCGArg ri = tcgv_vec_arg(r);
252 TCGTemp *rt = arg_temp(ri);
253 TCGType type = rt->base_type;
254
255 if (TCG_TARGET_REG_BITS == 64) {
256 TCGArg ai = tcgv_i64_arg(a);
257 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
258 } else if (vece == MO_64) {
259 TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
260 TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
261 vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
262 } else {
263 TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
264 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
265 }
266 }
267
268 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
269 {
270 TCGArg ri = tcgv_vec_arg(r);
271 TCGArg ai = tcgv_i32_arg(a);
272 TCGTemp *rt = arg_temp(ri);
273 TCGType type = rt->base_type;
274
275 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
276 }
277
278 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b,
279 tcg_target_long ofs)
280 {
281 TCGArg ri = tcgv_vec_arg(r);
282 TCGArg bi = tcgv_ptr_arg(b);
283 TCGTemp *rt = arg_temp(ri);
284 TCGType type = rt->base_type;
285
286 vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs);
287 }
288
289 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
290 {
291 TCGArg ri = tcgv_vec_arg(r);
292 TCGArg bi = tcgv_ptr_arg(b);
293 TCGTemp *rt = arg_temp(ri);
294 TCGType type = rt->base_type;
295
296 vec_gen_3(opc, type, 0, ri, bi, o);
297 }
298
299 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
300 {
301 vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
302 }
303
304 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
305 {
306 vec_gen_ldst(INDEX_op_st_vec, r, b, o);
307 }
308
309 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
310 {
311 TCGArg ri = tcgv_vec_arg(r);
312 TCGArg bi = tcgv_ptr_arg(b);
313 TCGTemp *rt = arg_temp(ri);
314 TCGType type = rt->base_type;
315
316 tcg_debug_assert(low_type >= TCG_TYPE_V64);
317 tcg_debug_assert(low_type <= type);
318 vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
319 }
320
321 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
322 {
323 vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
324 }
325
326 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
327 {
328 vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
329 }
330
331 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
332 {
333 vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
334 }
335
336 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
337 {
338 if (TCG_TARGET_HAS_andc_vec) {
339 vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
340 } else {
341 TCGv_vec t = tcg_temp_new_vec_matching(r);
342 tcg_gen_not_vec(0, t, b);
343 tcg_gen_and_vec(0, r, a, t);
344 tcg_temp_free_vec(t);
345 }
346 }
347
348 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
349 {
350 if (TCG_TARGET_HAS_orc_vec) {
351 vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
352 } else {
353 TCGv_vec t = tcg_temp_new_vec_matching(r);
354 tcg_gen_not_vec(0, t, b);
355 tcg_gen_or_vec(0, r, a, t);
356 tcg_temp_free_vec(t);
357 }
358 }
359
360 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
361 {
362 /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */
363 tcg_gen_and_vec(0, r, a, b);
364 tcg_gen_not_vec(0, r, r);
365 }
366
367 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
368 {
369 /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */
370 tcg_gen_or_vec(0, r, a, b);
371 tcg_gen_not_vec(0, r, r);
372 }
373
374 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
375 {
376 /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */
377 tcg_gen_xor_vec(0, r, a, b);
378 tcg_gen_not_vec(0, r, r);
379 }
380
381 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc)
382 {
383 TCGTemp *rt = tcgv_vec_temp(r);
384 TCGTemp *at = tcgv_vec_temp(a);
385 TCGArg ri = temp_arg(rt);
386 TCGArg ai = temp_arg(at);
387 TCGType type = rt->base_type;
388 int can;
389
390 tcg_debug_assert(at->base_type >= type);
391 tcg_assert_listed_vecop(opc);
392 can = tcg_can_emit_vec_op(opc, type, vece);
393 if (can > 0) {
394 vec_gen_2(opc, type, vece, ri, ai);
395 } else if (can < 0) {
396 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
397 tcg_expand_vec_op(opc, type, vece, ri, ai);
398 tcg_swap_vecop_list(hold_list);
399 } else {
400 return false;
401 }
402 return true;
403 }
404
405 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
406 {
407 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
408
409 if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) {
410 TCGv_vec t = tcg_const_ones_vec_matching(r);
411 tcg_gen_xor_vec(0, r, a, t);
412 tcg_temp_free_vec(t);
413 }
414 tcg_swap_vecop_list(hold_list);
415 }
416
417 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
418 {
419 const TCGOpcode *hold_list;
420
421 tcg_assert_listed_vecop(INDEX_op_neg_vec);
422 hold_list = tcg_swap_vecop_list(NULL);
423
424 if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) {
425 TCGv_vec t = tcg_const_zeros_vec_matching(r);
426 tcg_gen_sub_vec(vece, r, t, a);
427 tcg_temp_free_vec(t);
428 }
429 tcg_swap_vecop_list(hold_list);
430 }
431
432 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
433 {
434 const TCGOpcode *hold_list;
435
436 tcg_assert_listed_vecop(INDEX_op_abs_vec);
437 hold_list = tcg_swap_vecop_list(NULL);
438
439 if (!do_op2(vece, r, a, INDEX_op_abs_vec)) {
440 TCGType type = tcgv_vec_temp(r)->base_type;
441 TCGv_vec t = tcg_temp_new_vec(type);
442
443 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece));
444 if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) {
445 tcg_gen_neg_vec(vece, t, a);
446 tcg_gen_smax_vec(vece, r, a, t);
447 } else {
448 if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) {
449 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1);
450 } else {
451 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a,
452 tcg_constant_vec(type, vece, 0));
453 }
454 tcg_gen_xor_vec(vece, r, a, t);
455 tcg_gen_sub_vec(vece, r, r, t);
456 }
457
458 tcg_temp_free_vec(t);
459 }
460 tcg_swap_vecop_list(hold_list);
461 }
462
463 static void do_shifti(TCGOpcode opc, unsigned vece,
464 TCGv_vec r, TCGv_vec a, int64_t i)
465 {
466 TCGTemp *rt = tcgv_vec_temp(r);
467 TCGTemp *at = tcgv_vec_temp(a);
468 TCGArg ri = temp_arg(rt);
469 TCGArg ai = temp_arg(at);
470 TCGType type = rt->base_type;
471 int can;
472
473 tcg_debug_assert(at->base_type == type);
474 tcg_debug_assert(i >= 0 && i < (8 << vece));
475 tcg_assert_listed_vecop(opc);
476
477 if (i == 0) {
478 tcg_gen_mov_vec(r, a);
479 return;
480 }
481
482 can = tcg_can_emit_vec_op(opc, type, vece);
483 if (can > 0) {
484 vec_gen_3(opc, type, vece, ri, ai, i);
485 } else {
486 /* We leave the choice of expansion via scalar or vector shift
487 to the target. Often, but not always, dupi can feed a vector
488 shift easier than a scalar. */
489 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
490 tcg_debug_assert(can < 0);
491 tcg_expand_vec_op(opc, type, vece, ri, ai, i);
492 tcg_swap_vecop_list(hold_list);
493 }
494 }
495
496 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
497 {
498 do_shifti(INDEX_op_shli_vec, vece, r, a, i);
499 }
500
501 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
502 {
503 do_shifti(INDEX_op_shri_vec, vece, r, a, i);
504 }
505
506 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
507 {
508 do_shifti(INDEX_op_sari_vec, vece, r, a, i);
509 }
510
511 void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
512 {
513 do_shifti(INDEX_op_rotli_vec, vece, r, a, i);
514 }
515
516 void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
517 {
518 int bits = 8 << vece;
519 tcg_debug_assert(i >= 0 && i < bits);
520 do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1));
521 }
522
523 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
524 TCGv_vec r, TCGv_vec a, TCGv_vec b)
525 {
526 TCGTemp *rt = tcgv_vec_temp(r);
527 TCGTemp *at = tcgv_vec_temp(a);
528 TCGTemp *bt = tcgv_vec_temp(b);
529 TCGArg ri = temp_arg(rt);
530 TCGArg ai = temp_arg(at);
531 TCGArg bi = temp_arg(bt);
532 TCGType type = rt->base_type;
533 int can;
534
535 tcg_debug_assert(at->base_type >= type);
536 tcg_debug_assert(bt->base_type >= type);
537 tcg_assert_listed_vecop(INDEX_op_cmp_vec);
538 can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
539 if (can > 0) {
540 vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
541 } else {
542 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
543 tcg_debug_assert(can < 0);
544 tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
545 tcg_swap_vecop_list(hold_list);
546 }
547 }
548
549 static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a,
550 TCGv_vec b, TCGOpcode opc)
551 {
552 TCGTemp *rt = tcgv_vec_temp(r);
553 TCGTemp *at = tcgv_vec_temp(a);
554 TCGTemp *bt = tcgv_vec_temp(b);
555 TCGArg ri = temp_arg(rt);
556 TCGArg ai = temp_arg(at);
557 TCGArg bi = temp_arg(bt);
558 TCGType type = rt->base_type;
559 int can;
560
561 tcg_debug_assert(at->base_type >= type);
562 tcg_debug_assert(bt->base_type >= type);
563 tcg_assert_listed_vecop(opc);
564 can = tcg_can_emit_vec_op(opc, type, vece);
565 if (can > 0) {
566 vec_gen_3(opc, type, vece, ri, ai, bi);
567 } else if (can < 0) {
568 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
569 tcg_expand_vec_op(opc, type, vece, ri, ai, bi);
570 tcg_swap_vecop_list(hold_list);
571 } else {
572 return false;
573 }
574 return true;
575 }
576
577 static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a,
578 TCGv_vec b, TCGOpcode opc)
579 {
580 bool ok = do_op3(vece, r, a, b, opc);
581 tcg_debug_assert(ok);
582 }
583
584 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
585 {
586 do_op3_nofail(vece, r, a, b, INDEX_op_add_vec);
587 }
588
589 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
590 {
591 do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec);
592 }
593
594 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
595 {
596 do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec);
597 }
598
599 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
600 {
601 do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec);
602 }
603
604 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
605 {
606 do_op3_nofail(vece, r, a, b, INDEX_op_usadd_vec);
607 }
608
609 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
610 {
611 do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec);
612 }
613
614 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
615 {
616 do_op3_nofail(vece, r, a, b, INDEX_op_ussub_vec);
617 }
618
619 static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a,
620 TCGv_vec b, TCGOpcode opc, TCGCond cond)
621 {
622 if (!do_op3(vece, r, a, b, opc)) {
623 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
624 tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b);
625 tcg_swap_vecop_list(hold_list);
626 }
627 }
628
629 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
630 {
631 do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT);
632 }
633
634 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
635 {
636 do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU);
637 }
638
639 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
640 {
641 do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT);
642 }
643
644 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
645 {
646 do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU);
647 }
648
649 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
650 {
651 do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec);
652 }
653
654 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
655 {
656 do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec);
657 }
658
659 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
660 {
661 do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec);
662 }
663
664 void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
665 {
666 do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec);
667 }
668
669 void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
670 {
671 do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec);
672 }
673
674 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a,
675 TCGv_i32 s, TCGOpcode opc)
676 {
677 TCGTemp *rt = tcgv_vec_temp(r);
678 TCGTemp *at = tcgv_vec_temp(a);
679 TCGTemp *st = tcgv_i32_temp(s);
680 TCGArg ri = temp_arg(rt);
681 TCGArg ai = temp_arg(at);
682 TCGArg si = temp_arg(st);
683 TCGType type = rt->base_type;
684 int can;
685
686 tcg_debug_assert(at->base_type >= type);
687 tcg_assert_listed_vecop(opc);
688 can = tcg_can_emit_vec_op(opc, type, vece);
689 if (can > 0) {
690 vec_gen_3(opc, type, vece, ri, ai, si);
691 } else if (can < 0) {
692 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
693 tcg_expand_vec_op(opc, type, vece, ri, ai, si);
694 tcg_swap_vecop_list(hold_list);
695 } else {
696 g_assert_not_reached();
697 }
698 }
699
700 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
701 {
702 do_shifts(vece, r, a, b, INDEX_op_shls_vec);
703 }
704
705 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
706 {
707 do_shifts(vece, r, a, b, INDEX_op_shrs_vec);
708 }
709
710 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
711 {
712 do_shifts(vece, r, a, b, INDEX_op_sars_vec);
713 }
714
715 void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s)
716 {
717 do_shifts(vece, r, a, s, INDEX_op_rotls_vec);
718 }
719
720 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a,
721 TCGv_vec b, TCGv_vec c)
722 {
723 TCGTemp *rt = tcgv_vec_temp(r);
724 TCGTemp *at = tcgv_vec_temp(a);
725 TCGTemp *bt = tcgv_vec_temp(b);
726 TCGTemp *ct = tcgv_vec_temp(c);
727 TCGType type = rt->base_type;
728
729 tcg_debug_assert(at->base_type >= type);
730 tcg_debug_assert(bt->base_type >= type);
731 tcg_debug_assert(ct->base_type >= type);
732
733 if (TCG_TARGET_HAS_bitsel_vec) {
734 vec_gen_4(INDEX_op_bitsel_vec, type, MO_8,
735 temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct));
736 } else {
737 TCGv_vec t = tcg_temp_new_vec(type);
738 tcg_gen_and_vec(MO_8, t, a, b);
739 tcg_gen_andc_vec(MO_8, r, c, a);
740 tcg_gen_or_vec(MO_8, r, r, t);
741 tcg_temp_free_vec(t);
742 }
743 }
744
745 void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r,
746 TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d)
747 {
748 TCGTemp *rt = tcgv_vec_temp(r);
749 TCGTemp *at = tcgv_vec_temp(a);
750 TCGTemp *bt = tcgv_vec_temp(b);
751 TCGTemp *ct = tcgv_vec_temp(c);
752 TCGTemp *dt = tcgv_vec_temp(d);
753 TCGArg ri = temp_arg(rt);
754 TCGArg ai = temp_arg(at);
755 TCGArg bi = temp_arg(bt);
756 TCGArg ci = temp_arg(ct);
757 TCGArg di = temp_arg(dt);
758 TCGType type = rt->base_type;
759 const TCGOpcode *hold_list;
760 int can;
761
762 tcg_debug_assert(at->base_type >= type);
763 tcg_debug_assert(bt->base_type >= type);
764 tcg_debug_assert(ct->base_type >= type);
765 tcg_debug_assert(dt->base_type >= type);
766
767 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec);
768 hold_list = tcg_swap_vecop_list(NULL);
769 can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece);
770
771 if (can > 0) {
772 vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond);
773 } else if (can < 0) {
774 tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece,
775 ri, ai, bi, ci, di, cond);
776 } else {
777 TCGv_vec t = tcg_temp_new_vec(type);
778 tcg_gen_cmp_vec(cond, vece, t, a, b);
779 tcg_gen_bitsel_vec(vece, r, t, c, d);
780 tcg_temp_free_vec(t);
781 }
782 tcg_swap_vecop_list(hold_list);
783 }