Update version for v7.1.0-rc3 release
[qemu.git] / tcg / tcg-op-vec.c
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2018 Linaro, Inc.
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "tcg/tcg.h"
22 #include "tcg/tcg-op.h"
23 #include "tcg/tcg-mo.h"
24
25 /* Reduce the number of ifdefs below. This assumes that all uses of
26 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
27 the compiler can eliminate. */
28 #if TCG_TARGET_REG_BITS == 64
29 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
30 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
31 #define TCGV_LOW TCGV_LOW_link_error
32 #define TCGV_HIGH TCGV_HIGH_link_error
33 #endif
34
35 /*
36 * Vector optional opcode tracking.
37 * Except for the basic logical operations (and, or, xor), and
38 * data movement (mov, ld, st, dupi), many vector opcodes are
39 * optional and may not be supported on the host. Thank Intel
40 * for the irregularity in their instruction set.
41 *
42 * The gvec expanders allow custom vector operations to be composed,
43 * generally via the .fniv callback in the GVecGen* structures. At
44 * the same time, in deciding whether to use this hook we need to
45 * know if the host supports the required operations. This is
46 * presented as an array of opcodes, terminated by 0. Each opcode
47 * is assumed to be expanded with the given VECE.
48 *
49 * For debugging, we want to validate this array. Therefore, when
50 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
51 * will validate that their opcode is present in the list.
52 */
53 #ifdef CONFIG_DEBUG_TCG
54 void tcg_assert_listed_vecop(TCGOpcode op)
55 {
56 const TCGOpcode *p = tcg_ctx->vecop_list;
57 if (p) {
58 for (; *p; ++p) {
59 if (*p == op) {
60 return;
61 }
62 }
63 g_assert_not_reached();
64 }
65 }
66 #endif
67
68 bool tcg_can_emit_vecop_list(const TCGOpcode *list,
69 TCGType type, unsigned vece)
70 {
71 if (list == NULL) {
72 return true;
73 }
74
75 for (; *list; ++list) {
76 TCGOpcode opc = *list;
77
78 #ifdef CONFIG_DEBUG_TCG
79 switch (opc) {
80 case INDEX_op_and_vec:
81 case INDEX_op_or_vec:
82 case INDEX_op_xor_vec:
83 case INDEX_op_mov_vec:
84 case INDEX_op_dup_vec:
85 case INDEX_op_dup2_vec:
86 case INDEX_op_ld_vec:
87 case INDEX_op_st_vec:
88 case INDEX_op_bitsel_vec:
89 /* These opcodes are mandatory and should not be listed. */
90 g_assert_not_reached();
91 case INDEX_op_not_vec:
92 /* These opcodes have generic expansions using the above. */
93 g_assert_not_reached();
94 default:
95 break;
96 }
97 #endif
98
99 if (tcg_can_emit_vec_op(opc, type, vece)) {
100 continue;
101 }
102
103 /*
104 * The opcode list is created by front ends based on what they
105 * actually invoke. We must mirror the logic in the routines
106 * below for generic expansions using other opcodes.
107 */
108 switch (opc) {
109 case INDEX_op_neg_vec:
110 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) {
111 continue;
112 }
113 break;
114 case INDEX_op_abs_vec:
115 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)
116 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0
117 || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0
118 || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) {
119 continue;
120 }
121 break;
122 case INDEX_op_usadd_vec:
123 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) ||
124 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
125 continue;
126 }
127 break;
128 case INDEX_op_ussub_vec:
129 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) ||
130 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
131 continue;
132 }
133 break;
134 case INDEX_op_cmpsel_vec:
135 case INDEX_op_smin_vec:
136 case INDEX_op_smax_vec:
137 case INDEX_op_umin_vec:
138 case INDEX_op_umax_vec:
139 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
140 continue;
141 }
142 break;
143 default:
144 break;
145 }
146 return false;
147 }
148 return true;
149 }
150
151 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
152 {
153 TCGOp *op = tcg_emit_op(opc);
154 TCGOP_VECL(op) = type - TCG_TYPE_V64;
155 TCGOP_VECE(op) = vece;
156 op->args[0] = r;
157 op->args[1] = a;
158 }
159
160 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
161 TCGArg r, TCGArg a, TCGArg b)
162 {
163 TCGOp *op = tcg_emit_op(opc);
164 TCGOP_VECL(op) = type - TCG_TYPE_V64;
165 TCGOP_VECE(op) = vece;
166 op->args[0] = r;
167 op->args[1] = a;
168 op->args[2] = b;
169 }
170
171 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
172 TCGArg r, TCGArg a, TCGArg b, TCGArg c)
173 {
174 TCGOp *op = tcg_emit_op(opc);
175 TCGOP_VECL(op) = type - TCG_TYPE_V64;
176 TCGOP_VECE(op) = vece;
177 op->args[0] = r;
178 op->args[1] = a;
179 op->args[2] = b;
180 op->args[3] = c;
181 }
182
183 static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r,
184 TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e)
185 {
186 TCGOp *op = tcg_emit_op(opc);
187 TCGOP_VECL(op) = type - TCG_TYPE_V64;
188 TCGOP_VECE(op) = vece;
189 op->args[0] = r;
190 op->args[1] = a;
191 op->args[2] = b;
192 op->args[3] = c;
193 op->args[4] = d;
194 op->args[5] = e;
195 }
196
197 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
198 {
199 TCGTemp *rt = tcgv_vec_temp(r);
200 TCGTemp *at = tcgv_vec_temp(a);
201 TCGType type = rt->base_type;
202
203 /* Must enough inputs for the output. */
204 tcg_debug_assert(at->base_type >= type);
205 vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
206 }
207
208 static void vec_gen_op3(TCGOpcode opc, unsigned vece,
209 TCGv_vec r, TCGv_vec a, TCGv_vec b)
210 {
211 TCGTemp *rt = tcgv_vec_temp(r);
212 TCGTemp *at = tcgv_vec_temp(a);
213 TCGTemp *bt = tcgv_vec_temp(b);
214 TCGType type = rt->base_type;
215
216 /* Must enough inputs for the output. */
217 tcg_debug_assert(at->base_type >= type);
218 tcg_debug_assert(bt->base_type >= type);
219 vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
220 }
221
222 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
223 {
224 if (r != a) {
225 vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
226 }
227 }
228
229 TCGv_vec tcg_const_zeros_vec(TCGType type)
230 {
231 TCGv_vec ret = tcg_temp_new_vec(type);
232 tcg_gen_dupi_vec(MO_64, ret, 0);
233 return ret;
234 }
235
236 TCGv_vec tcg_const_ones_vec(TCGType type)
237 {
238 TCGv_vec ret = tcg_temp_new_vec(type);
239 tcg_gen_dupi_vec(MO_64, ret, -1);
240 return ret;
241 }
242
243 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m)
244 {
245 TCGTemp *t = tcgv_vec_temp(m);
246 return tcg_const_zeros_vec(t->base_type);
247 }
248
249 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m)
250 {
251 TCGTemp *t = tcgv_vec_temp(m);
252 return tcg_const_ones_vec(t->base_type);
253 }
254
255 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
256 {
257 TCGTemp *rt = tcgv_vec_temp(r);
258 tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a));
259 }
260
261 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
262 {
263 TCGArg ri = tcgv_vec_arg(r);
264 TCGTemp *rt = arg_temp(ri);
265 TCGType type = rt->base_type;
266
267 if (TCG_TARGET_REG_BITS == 64) {
268 TCGArg ai = tcgv_i64_arg(a);
269 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
270 } else if (vece == MO_64) {
271 TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
272 TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
273 vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
274 } else {
275 TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
276 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
277 }
278 }
279
280 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
281 {
282 TCGArg ri = tcgv_vec_arg(r);
283 TCGArg ai = tcgv_i32_arg(a);
284 TCGTemp *rt = arg_temp(ri);
285 TCGType type = rt->base_type;
286
287 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
288 }
289
290 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b,
291 tcg_target_long ofs)
292 {
293 TCGArg ri = tcgv_vec_arg(r);
294 TCGArg bi = tcgv_ptr_arg(b);
295 TCGTemp *rt = arg_temp(ri);
296 TCGType type = rt->base_type;
297
298 vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs);
299 }
300
301 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
302 {
303 TCGArg ri = tcgv_vec_arg(r);
304 TCGArg bi = tcgv_ptr_arg(b);
305 TCGTemp *rt = arg_temp(ri);
306 TCGType type = rt->base_type;
307
308 vec_gen_3(opc, type, 0, ri, bi, o);
309 }
310
311 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
312 {
313 vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
314 }
315
316 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
317 {
318 vec_gen_ldst(INDEX_op_st_vec, r, b, o);
319 }
320
321 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
322 {
323 TCGArg ri = tcgv_vec_arg(r);
324 TCGArg bi = tcgv_ptr_arg(b);
325 TCGTemp *rt = arg_temp(ri);
326 TCGType type = rt->base_type;
327
328 tcg_debug_assert(low_type >= TCG_TYPE_V64);
329 tcg_debug_assert(low_type <= type);
330 vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
331 }
332
333 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
334 {
335 vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
336 }
337
338 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
339 {
340 vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
341 }
342
343 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
344 {
345 vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
346 }
347
348 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
349 {
350 if (TCG_TARGET_HAS_andc_vec) {
351 vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
352 } else {
353 TCGv_vec t = tcg_temp_new_vec_matching(r);
354 tcg_gen_not_vec(0, t, b);
355 tcg_gen_and_vec(0, r, a, t);
356 tcg_temp_free_vec(t);
357 }
358 }
359
360 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
361 {
362 if (TCG_TARGET_HAS_orc_vec) {
363 vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
364 } else {
365 TCGv_vec t = tcg_temp_new_vec_matching(r);
366 tcg_gen_not_vec(0, t, b);
367 tcg_gen_or_vec(0, r, a, t);
368 tcg_temp_free_vec(t);
369 }
370 }
371
372 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
373 {
374 /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */
375 tcg_gen_and_vec(0, r, a, b);
376 tcg_gen_not_vec(0, r, r);
377 }
378
379 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
380 {
381 /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */
382 tcg_gen_or_vec(0, r, a, b);
383 tcg_gen_not_vec(0, r, r);
384 }
385
386 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
387 {
388 /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */
389 tcg_gen_xor_vec(0, r, a, b);
390 tcg_gen_not_vec(0, r, r);
391 }
392
393 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc)
394 {
395 TCGTemp *rt = tcgv_vec_temp(r);
396 TCGTemp *at = tcgv_vec_temp(a);
397 TCGArg ri = temp_arg(rt);
398 TCGArg ai = temp_arg(at);
399 TCGType type = rt->base_type;
400 int can;
401
402 tcg_debug_assert(at->base_type >= type);
403 tcg_assert_listed_vecop(opc);
404 can = tcg_can_emit_vec_op(opc, type, vece);
405 if (can > 0) {
406 vec_gen_2(opc, type, vece, ri, ai);
407 } else if (can < 0) {
408 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
409 tcg_expand_vec_op(opc, type, vece, ri, ai);
410 tcg_swap_vecop_list(hold_list);
411 } else {
412 return false;
413 }
414 return true;
415 }
416
417 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
418 {
419 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
420
421 if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) {
422 TCGv_vec t = tcg_const_ones_vec_matching(r);
423 tcg_gen_xor_vec(0, r, a, t);
424 tcg_temp_free_vec(t);
425 }
426 tcg_swap_vecop_list(hold_list);
427 }
428
429 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
430 {
431 const TCGOpcode *hold_list;
432
433 tcg_assert_listed_vecop(INDEX_op_neg_vec);
434 hold_list = tcg_swap_vecop_list(NULL);
435
436 if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) {
437 TCGv_vec t = tcg_const_zeros_vec_matching(r);
438 tcg_gen_sub_vec(vece, r, t, a);
439 tcg_temp_free_vec(t);
440 }
441 tcg_swap_vecop_list(hold_list);
442 }
443
444 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
445 {
446 const TCGOpcode *hold_list;
447
448 tcg_assert_listed_vecop(INDEX_op_abs_vec);
449 hold_list = tcg_swap_vecop_list(NULL);
450
451 if (!do_op2(vece, r, a, INDEX_op_abs_vec)) {
452 TCGType type = tcgv_vec_temp(r)->base_type;
453 TCGv_vec t = tcg_temp_new_vec(type);
454
455 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece));
456 if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) {
457 tcg_gen_neg_vec(vece, t, a);
458 tcg_gen_smax_vec(vece, r, a, t);
459 } else {
460 if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) {
461 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1);
462 } else {
463 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a,
464 tcg_constant_vec(type, vece, 0));
465 }
466 tcg_gen_xor_vec(vece, r, a, t);
467 tcg_gen_sub_vec(vece, r, r, t);
468 }
469
470 tcg_temp_free_vec(t);
471 }
472 tcg_swap_vecop_list(hold_list);
473 }
474
475 static void do_shifti(TCGOpcode opc, unsigned vece,
476 TCGv_vec r, TCGv_vec a, int64_t i)
477 {
478 TCGTemp *rt = tcgv_vec_temp(r);
479 TCGTemp *at = tcgv_vec_temp(a);
480 TCGArg ri = temp_arg(rt);
481 TCGArg ai = temp_arg(at);
482 TCGType type = rt->base_type;
483 int can;
484
485 tcg_debug_assert(at->base_type == type);
486 tcg_debug_assert(i >= 0 && i < (8 << vece));
487 tcg_assert_listed_vecop(opc);
488
489 if (i == 0) {
490 tcg_gen_mov_vec(r, a);
491 return;
492 }
493
494 can = tcg_can_emit_vec_op(opc, type, vece);
495 if (can > 0) {
496 vec_gen_3(opc, type, vece, ri, ai, i);
497 } else {
498 /* We leave the choice of expansion via scalar or vector shift
499 to the target. Often, but not always, dupi can feed a vector
500 shift easier than a scalar. */
501 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
502 tcg_debug_assert(can < 0);
503 tcg_expand_vec_op(opc, type, vece, ri, ai, i);
504 tcg_swap_vecop_list(hold_list);
505 }
506 }
507
508 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
509 {
510 do_shifti(INDEX_op_shli_vec, vece, r, a, i);
511 }
512
513 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
514 {
515 do_shifti(INDEX_op_shri_vec, vece, r, a, i);
516 }
517
518 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
519 {
520 do_shifti(INDEX_op_sari_vec, vece, r, a, i);
521 }
522
523 void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
524 {
525 do_shifti(INDEX_op_rotli_vec, vece, r, a, i);
526 }
527
528 void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
529 {
530 int bits = 8 << vece;
531 tcg_debug_assert(i >= 0 && i < bits);
532 do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1));
533 }
534
535 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
536 TCGv_vec r, TCGv_vec a, TCGv_vec b)
537 {
538 TCGTemp *rt = tcgv_vec_temp(r);
539 TCGTemp *at = tcgv_vec_temp(a);
540 TCGTemp *bt = tcgv_vec_temp(b);
541 TCGArg ri = temp_arg(rt);
542 TCGArg ai = temp_arg(at);
543 TCGArg bi = temp_arg(bt);
544 TCGType type = rt->base_type;
545 int can;
546
547 tcg_debug_assert(at->base_type >= type);
548 tcg_debug_assert(bt->base_type >= type);
549 tcg_assert_listed_vecop(INDEX_op_cmp_vec);
550 can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
551 if (can > 0) {
552 vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
553 } else {
554 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
555 tcg_debug_assert(can < 0);
556 tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
557 tcg_swap_vecop_list(hold_list);
558 }
559 }
560
561 static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a,
562 TCGv_vec b, TCGOpcode opc)
563 {
564 TCGTemp *rt = tcgv_vec_temp(r);
565 TCGTemp *at = tcgv_vec_temp(a);
566 TCGTemp *bt = tcgv_vec_temp(b);
567 TCGArg ri = temp_arg(rt);
568 TCGArg ai = temp_arg(at);
569 TCGArg bi = temp_arg(bt);
570 TCGType type = rt->base_type;
571 int can;
572
573 tcg_debug_assert(at->base_type >= type);
574 tcg_debug_assert(bt->base_type >= type);
575 tcg_assert_listed_vecop(opc);
576 can = tcg_can_emit_vec_op(opc, type, vece);
577 if (can > 0) {
578 vec_gen_3(opc, type, vece, ri, ai, bi);
579 } else if (can < 0) {
580 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
581 tcg_expand_vec_op(opc, type, vece, ri, ai, bi);
582 tcg_swap_vecop_list(hold_list);
583 } else {
584 return false;
585 }
586 return true;
587 }
588
589 static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a,
590 TCGv_vec b, TCGOpcode opc)
591 {
592 bool ok = do_op3(vece, r, a, b, opc);
593 tcg_debug_assert(ok);
594 }
595
596 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
597 {
598 do_op3_nofail(vece, r, a, b, INDEX_op_add_vec);
599 }
600
601 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
602 {
603 do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec);
604 }
605
606 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
607 {
608 do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec);
609 }
610
611 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
612 {
613 do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec);
614 }
615
616 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
617 {
618 if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) {
619 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
620 TCGv_vec t = tcg_temp_new_vec_matching(r);
621
622 /* usadd(a, b) = min(a, ~b) + b */
623 tcg_gen_not_vec(vece, t, b);
624 tcg_gen_umin_vec(vece, t, t, a);
625 tcg_gen_add_vec(vece, r, t, b);
626
627 tcg_temp_free_vec(t);
628 tcg_swap_vecop_list(hold_list);
629 }
630 }
631
632 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
633 {
634 do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec);
635 }
636
637 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
638 {
639 if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) {
640 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
641 TCGv_vec t = tcg_temp_new_vec_matching(r);
642
643 /* ussub(a, b) = max(a, b) - b */
644 tcg_gen_umax_vec(vece, t, a, b);
645 tcg_gen_sub_vec(vece, r, t, b);
646
647 tcg_temp_free_vec(t);
648 tcg_swap_vecop_list(hold_list);
649 }
650 }
651
652 static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a,
653 TCGv_vec b, TCGOpcode opc, TCGCond cond)
654 {
655 if (!do_op3(vece, r, a, b, opc)) {
656 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
657 tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b);
658 tcg_swap_vecop_list(hold_list);
659 }
660 }
661
662 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
663 {
664 do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT);
665 }
666
667 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
668 {
669 do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU);
670 }
671
672 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
673 {
674 do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT);
675 }
676
677 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
678 {
679 do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU);
680 }
681
682 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
683 {
684 do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec);
685 }
686
687 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
688 {
689 do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec);
690 }
691
692 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
693 {
694 do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec);
695 }
696
697 void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
698 {
699 do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec);
700 }
701
702 void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
703 {
704 do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec);
705 }
706
707 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a,
708 TCGv_i32 s, TCGOpcode opc)
709 {
710 TCGTemp *rt = tcgv_vec_temp(r);
711 TCGTemp *at = tcgv_vec_temp(a);
712 TCGTemp *st = tcgv_i32_temp(s);
713 TCGArg ri = temp_arg(rt);
714 TCGArg ai = temp_arg(at);
715 TCGArg si = temp_arg(st);
716 TCGType type = rt->base_type;
717 int can;
718
719 tcg_debug_assert(at->base_type >= type);
720 tcg_assert_listed_vecop(opc);
721 can = tcg_can_emit_vec_op(opc, type, vece);
722 if (can > 0) {
723 vec_gen_3(opc, type, vece, ri, ai, si);
724 } else if (can < 0) {
725 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
726 tcg_expand_vec_op(opc, type, vece, ri, ai, si);
727 tcg_swap_vecop_list(hold_list);
728 } else {
729 g_assert_not_reached();
730 }
731 }
732
733 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
734 {
735 do_shifts(vece, r, a, b, INDEX_op_shls_vec);
736 }
737
738 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
739 {
740 do_shifts(vece, r, a, b, INDEX_op_shrs_vec);
741 }
742
743 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
744 {
745 do_shifts(vece, r, a, b, INDEX_op_sars_vec);
746 }
747
748 void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s)
749 {
750 do_shifts(vece, r, a, s, INDEX_op_rotls_vec);
751 }
752
753 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a,
754 TCGv_vec b, TCGv_vec c)
755 {
756 TCGTemp *rt = tcgv_vec_temp(r);
757 TCGTemp *at = tcgv_vec_temp(a);
758 TCGTemp *bt = tcgv_vec_temp(b);
759 TCGTemp *ct = tcgv_vec_temp(c);
760 TCGType type = rt->base_type;
761
762 tcg_debug_assert(at->base_type >= type);
763 tcg_debug_assert(bt->base_type >= type);
764 tcg_debug_assert(ct->base_type >= type);
765
766 if (TCG_TARGET_HAS_bitsel_vec) {
767 vec_gen_4(INDEX_op_bitsel_vec, type, MO_8,
768 temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct));
769 } else {
770 TCGv_vec t = tcg_temp_new_vec(type);
771 tcg_gen_and_vec(MO_8, t, a, b);
772 tcg_gen_andc_vec(MO_8, r, c, a);
773 tcg_gen_or_vec(MO_8, r, r, t);
774 tcg_temp_free_vec(t);
775 }
776 }
777
778 void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r,
779 TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d)
780 {
781 TCGTemp *rt = tcgv_vec_temp(r);
782 TCGTemp *at = tcgv_vec_temp(a);
783 TCGTemp *bt = tcgv_vec_temp(b);
784 TCGTemp *ct = tcgv_vec_temp(c);
785 TCGTemp *dt = tcgv_vec_temp(d);
786 TCGArg ri = temp_arg(rt);
787 TCGArg ai = temp_arg(at);
788 TCGArg bi = temp_arg(bt);
789 TCGArg ci = temp_arg(ct);
790 TCGArg di = temp_arg(dt);
791 TCGType type = rt->base_type;
792 const TCGOpcode *hold_list;
793 int can;
794
795 tcg_debug_assert(at->base_type >= type);
796 tcg_debug_assert(bt->base_type >= type);
797 tcg_debug_assert(ct->base_type >= type);
798 tcg_debug_assert(dt->base_type >= type);
799
800 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec);
801 hold_list = tcg_swap_vecop_list(NULL);
802 can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece);
803
804 if (can > 0) {
805 vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond);
806 } else if (can < 0) {
807 tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece,
808 ri, ai, bi, ci, di, cond);
809 } else {
810 TCGv_vec t = tcg_temp_new_vec(type);
811 tcg_gen_cmp_vec(cond, vece, t, a, b);
812 tcg_gen_bitsel_vec(vece, r, t, c, d);
813 tcg_temp_free_vec(t);
814 }
815 tcg_swap_vecop_list(hold_list);
816 }