target/arm: Implement VFP fp16 for VABS, VNEG, VSQRT
[qemu.git] / accel / tcg / atomic_template.h
1 /*
2 * Atomic helper templates
3 * Included from tcg-runtime.c and cputlb.c.
4 *
5 * Copyright (c) 2016 Red Hat, Inc
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "qemu/plugin.h"
22 #include "trace/mem.h"
23
24 #if DATA_SIZE == 16
25 # define SUFFIX o
26 # define DATA_TYPE Int128
27 # define BSWAP bswap128
28 # define SHIFT 4
29 #elif DATA_SIZE == 8
30 # define SUFFIX q
31 # define DATA_TYPE uint64_t
32 # define SDATA_TYPE int64_t
33 # define BSWAP bswap64
34 # define SHIFT 3
35 #elif DATA_SIZE == 4
36 # define SUFFIX l
37 # define DATA_TYPE uint32_t
38 # define SDATA_TYPE int32_t
39 # define BSWAP bswap32
40 # define SHIFT 2
41 #elif DATA_SIZE == 2
42 # define SUFFIX w
43 # define DATA_TYPE uint16_t
44 # define SDATA_TYPE int16_t
45 # define BSWAP bswap16
46 # define SHIFT 1
47 #elif DATA_SIZE == 1
48 # define SUFFIX b
49 # define DATA_TYPE uint8_t
50 # define SDATA_TYPE int8_t
51 # define BSWAP
52 # define SHIFT 0
53 #else
54 # error unsupported data size
55 #endif
56
57 #if DATA_SIZE >= 4
58 # define ABI_TYPE DATA_TYPE
59 #else
60 # define ABI_TYPE uint32_t
61 #endif
62
63 /* Define host-endian atomic operations. Note that END is used within
64 the ATOMIC_NAME macro, and redefined below. */
65 #if DATA_SIZE == 1
66 # define END
67 #elif defined(HOST_WORDS_BIGENDIAN)
68 # define END _be
69 #else
70 # define END _le
71 #endif
72
73 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
74 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
75 {
76 ATOMIC_MMU_DECLS;
77 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
78 DATA_TYPE ret;
79 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
80 ATOMIC_MMU_IDX);
81
82 atomic_trace_rmw_pre(env, addr, info);
83 #if DATA_SIZE == 16
84 ret = atomic16_cmpxchg(haddr, cmpv, newv);
85 #else
86 ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
87 #endif
88 ATOMIC_MMU_CLEANUP;
89 atomic_trace_rmw_post(env, addr, info);
90 return ret;
91 }
92
93 #if DATA_SIZE >= 16
94 #if HAVE_ATOMIC128
95 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
96 {
97 ATOMIC_MMU_DECLS;
98 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
99 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
100 ATOMIC_MMU_IDX);
101
102 atomic_trace_ld_pre(env, addr, info);
103 val = atomic16_read(haddr);
104 ATOMIC_MMU_CLEANUP;
105 atomic_trace_ld_post(env, addr, info);
106 return val;
107 }
108
109 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
110 ABI_TYPE val EXTRA_ARGS)
111 {
112 ATOMIC_MMU_DECLS;
113 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
114 uint16_t info = trace_mem_build_info(SHIFT, false, 0, true,
115 ATOMIC_MMU_IDX);
116
117 atomic_trace_st_pre(env, addr, info);
118 atomic16_set(haddr, val);
119 ATOMIC_MMU_CLEANUP;
120 atomic_trace_st_post(env, addr, info);
121 }
122 #endif
123 #else
124 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
125 ABI_TYPE val EXTRA_ARGS)
126 {
127 ATOMIC_MMU_DECLS;
128 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
129 DATA_TYPE ret;
130 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false,
131 ATOMIC_MMU_IDX);
132
133 atomic_trace_rmw_pre(env, addr, info);
134 ret = atomic_xchg__nocheck(haddr, val);
135 ATOMIC_MMU_CLEANUP;
136 atomic_trace_rmw_post(env, addr, info);
137 return ret;
138 }
139
140 #define GEN_ATOMIC_HELPER(X) \
141 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
142 ABI_TYPE val EXTRA_ARGS) \
143 { \
144 ATOMIC_MMU_DECLS; \
145 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
146 DATA_TYPE ret; \
147 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \
148 ATOMIC_MMU_IDX); \
149 atomic_trace_rmw_pre(env, addr, info); \
150 ret = atomic_##X(haddr, val); \
151 ATOMIC_MMU_CLEANUP; \
152 atomic_trace_rmw_post(env, addr, info); \
153 return ret; \
154 }
155
156 GEN_ATOMIC_HELPER(fetch_add)
157 GEN_ATOMIC_HELPER(fetch_and)
158 GEN_ATOMIC_HELPER(fetch_or)
159 GEN_ATOMIC_HELPER(fetch_xor)
160 GEN_ATOMIC_HELPER(add_fetch)
161 GEN_ATOMIC_HELPER(and_fetch)
162 GEN_ATOMIC_HELPER(or_fetch)
163 GEN_ATOMIC_HELPER(xor_fetch)
164
165 #undef GEN_ATOMIC_HELPER
166
167 /* These helpers are, as a whole, full barriers. Within the helper,
168 * the leading barrier is explicit and the trailing barrier is within
169 * cmpxchg primitive.
170 *
171 * Trace this load + RMW loop as a single RMW op. This way, regardless
172 * of CF_PARALLEL's value, we'll trace just a read and a write.
173 */
174 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
175 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
176 ABI_TYPE xval EXTRA_ARGS) \
177 { \
178 ATOMIC_MMU_DECLS; \
179 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
180 XDATA_TYPE cmp, old, new, val = xval; \
181 uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \
182 ATOMIC_MMU_IDX); \
183 atomic_trace_rmw_pre(env, addr, info); \
184 smp_mb(); \
185 cmp = atomic_read__nocheck(haddr); \
186 do { \
187 old = cmp; new = FN(old, val); \
188 cmp = atomic_cmpxchg__nocheck(haddr, old, new); \
189 } while (cmp != old); \
190 ATOMIC_MMU_CLEANUP; \
191 atomic_trace_rmw_post(env, addr, info); \
192 return RET; \
193 }
194
195 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
196 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
197 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
198 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
199
200 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
201 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
202 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
203 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
204
205 #undef GEN_ATOMIC_HELPER_FN
206 #endif /* DATA SIZE >= 16 */
207
208 #undef END
209
210 #if DATA_SIZE > 1
211
212 /* Define reverse-host-endian atomic operations. Note that END is used
213 within the ATOMIC_NAME macro. */
214 #ifdef HOST_WORDS_BIGENDIAN
215 # define END _le
216 #else
217 # define END _be
218 #endif
219
220 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
221 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
222 {
223 ATOMIC_MMU_DECLS;
224 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
225 DATA_TYPE ret;
226 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
227 ATOMIC_MMU_IDX);
228
229 atomic_trace_rmw_pre(env, addr, info);
230 #if DATA_SIZE == 16
231 ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
232 #else
233 ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
234 #endif
235 ATOMIC_MMU_CLEANUP;
236 atomic_trace_rmw_post(env, addr, info);
237 return BSWAP(ret);
238 }
239
240 #if DATA_SIZE >= 16
241 #if HAVE_ATOMIC128
242 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
243 {
244 ATOMIC_MMU_DECLS;
245 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
246 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
247 ATOMIC_MMU_IDX);
248
249 atomic_trace_ld_pre(env, addr, info);
250 val = atomic16_read(haddr);
251 ATOMIC_MMU_CLEANUP;
252 atomic_trace_ld_post(env, addr, info);
253 return BSWAP(val);
254 }
255
256 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
257 ABI_TYPE val EXTRA_ARGS)
258 {
259 ATOMIC_MMU_DECLS;
260 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
261 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, true,
262 ATOMIC_MMU_IDX);
263
264 val = BSWAP(val);
265 atomic_trace_st_pre(env, addr, info);
266 val = BSWAP(val);
267 atomic16_set(haddr, val);
268 ATOMIC_MMU_CLEANUP;
269 atomic_trace_st_post(env, addr, info);
270 }
271 #endif
272 #else
273 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
274 ABI_TYPE val EXTRA_ARGS)
275 {
276 ATOMIC_MMU_DECLS;
277 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
278 ABI_TYPE ret;
279 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, false,
280 ATOMIC_MMU_IDX);
281
282 atomic_trace_rmw_pre(env, addr, info);
283 ret = atomic_xchg__nocheck(haddr, BSWAP(val));
284 ATOMIC_MMU_CLEANUP;
285 atomic_trace_rmw_post(env, addr, info);
286 return BSWAP(ret);
287 }
288
289 #define GEN_ATOMIC_HELPER(X) \
290 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
291 ABI_TYPE val EXTRA_ARGS) \
292 { \
293 ATOMIC_MMU_DECLS; \
294 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
295 DATA_TYPE ret; \
296 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \
297 false, ATOMIC_MMU_IDX); \
298 atomic_trace_rmw_pre(env, addr, info); \
299 ret = atomic_##X(haddr, BSWAP(val)); \
300 ATOMIC_MMU_CLEANUP; \
301 atomic_trace_rmw_post(env, addr, info); \
302 return BSWAP(ret); \
303 }
304
305 GEN_ATOMIC_HELPER(fetch_and)
306 GEN_ATOMIC_HELPER(fetch_or)
307 GEN_ATOMIC_HELPER(fetch_xor)
308 GEN_ATOMIC_HELPER(and_fetch)
309 GEN_ATOMIC_HELPER(or_fetch)
310 GEN_ATOMIC_HELPER(xor_fetch)
311
312 #undef GEN_ATOMIC_HELPER
313
314 /* These helpers are, as a whole, full barriers. Within the helper,
315 * the leading barrier is explicit and the trailing barrier is within
316 * cmpxchg primitive.
317 *
318 * Trace this load + RMW loop as a single RMW op. This way, regardless
319 * of CF_PARALLEL's value, we'll trace just a read and a write.
320 */
321 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
322 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
323 ABI_TYPE xval EXTRA_ARGS) \
324 { \
325 ATOMIC_MMU_DECLS; \
326 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
327 XDATA_TYPE ldo, ldn, old, new, val = xval; \
328 uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \
329 false, ATOMIC_MMU_IDX); \
330 atomic_trace_rmw_pre(env, addr, info); \
331 smp_mb(); \
332 ldn = atomic_read__nocheck(haddr); \
333 do { \
334 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
335 ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
336 } while (ldo != ldn); \
337 ATOMIC_MMU_CLEANUP; \
338 atomic_trace_rmw_post(env, addr, info); \
339 return RET; \
340 }
341
342 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
343 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
344 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
345 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
346
347 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
348 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
349 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
350 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
351
352 /* Note that for addition, we need to use a separate cmpxchg loop instead
353 of bswaps for the reverse-host-endian helpers. */
354 #define ADD(X, Y) (X + Y)
355 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
356 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
357 #undef ADD
358
359 #undef GEN_ATOMIC_HELPER_FN
360 #endif /* DATA_SIZE >= 16 */
361
362 #undef END
363 #endif /* DATA_SIZE > 1 */
364
365 #undef BSWAP
366 #undef ABI_TYPE
367 #undef DATA_TYPE
368 #undef SDATA_TYPE
369 #undef SUFFIX
370 #undef DATA_SIZE
371 #undef SHIFT