Merge remote-tracking branch 'remotes/stefanha/tags/block-pull-request' into staging
[qemu.git] / atomic_template.h
1 /*
2 * Atomic helper templates
3 * Included from tcg-runtime.c and cputlb.c.
4 *
5 * Copyright (c) 2016 Red Hat, Inc
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #if DATA_SIZE == 16
22 # define SUFFIX o
23 # define DATA_TYPE Int128
24 # define BSWAP bswap128
25 #elif DATA_SIZE == 8
26 # define SUFFIX q
27 # define DATA_TYPE uint64_t
28 # define BSWAP bswap64
29 #elif DATA_SIZE == 4
30 # define SUFFIX l
31 # define DATA_TYPE uint32_t
32 # define BSWAP bswap32
33 #elif DATA_SIZE == 2
34 # define SUFFIX w
35 # define DATA_TYPE uint16_t
36 # define BSWAP bswap16
37 #elif DATA_SIZE == 1
38 # define SUFFIX b
39 # define DATA_TYPE uint8_t
40 # define BSWAP
41 #else
42 # error unsupported data size
43 #endif
44
45 #if DATA_SIZE >= 4
46 # define ABI_TYPE DATA_TYPE
47 #else
48 # define ABI_TYPE uint32_t
49 #endif
50
51 /* Define host-endian atomic operations. Note that END is used within
52 the ATOMIC_NAME macro, and redefined below. */
53 #if DATA_SIZE == 1
54 # define END
55 #elif defined(HOST_WORDS_BIGENDIAN)
56 # define END _be
57 #else
58 # define END _le
59 #endif
60
61 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
62 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
63 {
64 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
65 return atomic_cmpxchg__nocheck(haddr, cmpv, newv);
66 }
67
68 #if DATA_SIZE >= 16
69 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
70 {
71 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
72 __atomic_load(haddr, &val, __ATOMIC_RELAXED);
73 return val;
74 }
75
76 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
77 ABI_TYPE val EXTRA_ARGS)
78 {
79 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
80 __atomic_store(haddr, &val, __ATOMIC_RELAXED);
81 }
82 #else
83 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
84 ABI_TYPE val EXTRA_ARGS)
85 {
86 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
87 return atomic_xchg__nocheck(haddr, val);
88 }
89
90 #define GEN_ATOMIC_HELPER(X) \
91 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
92 ABI_TYPE val EXTRA_ARGS) \
93 { \
94 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
95 return atomic_##X(haddr, val); \
96 } \
97
98 GEN_ATOMIC_HELPER(fetch_add)
99 GEN_ATOMIC_HELPER(fetch_and)
100 GEN_ATOMIC_HELPER(fetch_or)
101 GEN_ATOMIC_HELPER(fetch_xor)
102 GEN_ATOMIC_HELPER(add_fetch)
103 GEN_ATOMIC_HELPER(and_fetch)
104 GEN_ATOMIC_HELPER(or_fetch)
105 GEN_ATOMIC_HELPER(xor_fetch)
106
107 #undef GEN_ATOMIC_HELPER
108 #endif /* DATA SIZE >= 16 */
109
110 #undef END
111
112 #if DATA_SIZE > 1
113
114 /* Define reverse-host-endian atomic operations. Note that END is used
115 within the ATOMIC_NAME macro. */
116 #ifdef HOST_WORDS_BIGENDIAN
117 # define END _le
118 #else
119 # define END _be
120 #endif
121
122 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
123 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
124 {
125 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
126 return BSWAP(atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv)));
127 }
128
129 #if DATA_SIZE >= 16
130 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
131 {
132 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
133 __atomic_load(haddr, &val, __ATOMIC_RELAXED);
134 return BSWAP(val);
135 }
136
137 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
138 ABI_TYPE val EXTRA_ARGS)
139 {
140 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
141 val = BSWAP(val);
142 __atomic_store(haddr, &val, __ATOMIC_RELAXED);
143 }
144 #else
145 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
146 ABI_TYPE val EXTRA_ARGS)
147 {
148 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
149 return BSWAP(atomic_xchg__nocheck(haddr, BSWAP(val)));
150 }
151
152 #define GEN_ATOMIC_HELPER(X) \
153 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
154 ABI_TYPE val EXTRA_ARGS) \
155 { \
156 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
157 return BSWAP(atomic_##X(haddr, BSWAP(val))); \
158 }
159
160 GEN_ATOMIC_HELPER(fetch_and)
161 GEN_ATOMIC_HELPER(fetch_or)
162 GEN_ATOMIC_HELPER(fetch_xor)
163 GEN_ATOMIC_HELPER(and_fetch)
164 GEN_ATOMIC_HELPER(or_fetch)
165 GEN_ATOMIC_HELPER(xor_fetch)
166
167 #undef GEN_ATOMIC_HELPER
168
169 /* Note that for addition, we need to use a separate cmpxchg loop instead
170 of bswaps for the reverse-host-endian helpers. */
171 ABI_TYPE ATOMIC_NAME(fetch_add)(CPUArchState *env, target_ulong addr,
172 ABI_TYPE val EXTRA_ARGS)
173 {
174 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
175 DATA_TYPE ldo, ldn, ret, sto;
176
177 ldo = atomic_read__nocheck(haddr);
178 while (1) {
179 ret = BSWAP(ldo);
180 sto = BSWAP(ret + val);
181 ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
182 if (ldn == ldo) {
183 return ret;
184 }
185 ldo = ldn;
186 }
187 }
188
189 ABI_TYPE ATOMIC_NAME(add_fetch)(CPUArchState *env, target_ulong addr,
190 ABI_TYPE val EXTRA_ARGS)
191 {
192 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
193 DATA_TYPE ldo, ldn, ret, sto;
194
195 ldo = atomic_read__nocheck(haddr);
196 while (1) {
197 ret = BSWAP(ldo) + val;
198 sto = BSWAP(ret);
199 ldn = atomic_cmpxchg__nocheck(haddr, ldo, sto);
200 if (ldn == ldo) {
201 return ret;
202 }
203 ldo = ldn;
204 }
205 }
206 #endif /* DATA_SIZE >= 16 */
207
208 #undef END
209 #endif /* DATA_SIZE > 1 */
210
211 #undef BSWAP
212 #undef ABI_TYPE
213 #undef DATA_TYPE
214 #undef SUFFIX
215 #undef DATA_SIZE