memory: drop find_ram_block()
[qemu.git] / include / exec / ram_addr.h
1 /*
2 * Declarations for cpu physical memory functions
3 *
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates
5 *
6 * Authors:
7 * Avi Kivity <avi@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or
10 * later. See the COPYING file in the top-level directory.
11 *
12 */
13
14 /*
15 * This header is for use by exec.c and memory.c ONLY. Do not include it.
16 * The functions declared here will be removed soon.
17 */
18
19 #ifndef RAM_ADDR_H
20 #define RAM_ADDR_H
21
22 #ifndef CONFIG_USER_ONLY
23 #include "hw/xen/xen.h"
24
25 struct RAMBlock {
26 struct rcu_head rcu;
27 struct MemoryRegion *mr;
28 uint8_t *host;
29 ram_addr_t offset;
30 ram_addr_t used_length;
31 ram_addr_t max_length;
32 void (*resized)(const char*, uint64_t length, void *host);
33 uint32_t flags;
34 /* Protected by iothread lock. */
35 char idstr[256];
36 /* RCU-enabled, writes protected by the ramlist lock */
37 QLIST_ENTRY(RAMBlock) next;
38 int fd;
39 };
40
41 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
42 {
43 return (b && b->host && offset < b->used_length) ? true : false;
44 }
45
46 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
47 {
48 assert(offset_in_ramblock(block, offset));
49 return (char *)block->host + offset;
50 }
51
52 /* The dirty memory bitmap is split into fixed-size blocks to allow growth
53 * under RCU. The bitmap for a block can be accessed as follows:
54 *
55 * rcu_read_lock();
56 *
57 * DirtyMemoryBlocks *blocks =
58 * atomic_rcu_read(&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]);
59 *
60 * ram_addr_t idx = (addr >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
61 * unsigned long *block = blocks.blocks[idx];
62 * ...access block bitmap...
63 *
64 * rcu_read_unlock();
65 *
66 * Remember to check for the end of the block when accessing a range of
67 * addresses. Move on to the next block if you reach the end.
68 *
69 * Organization into blocks allows dirty memory to grow (but not shrink) under
70 * RCU. When adding new RAMBlocks requires the dirty memory to grow, a new
71 * DirtyMemoryBlocks array is allocated with pointers to existing blocks kept
72 * the same. Other threads can safely access existing blocks while dirty
73 * memory is being grown. When no threads are using the old DirtyMemoryBlocks
74 * anymore it is freed by RCU (but the underlying blocks stay because they are
75 * pointed to from the new DirtyMemoryBlocks).
76 */
77 #define DIRTY_MEMORY_BLOCK_SIZE ((ram_addr_t)256 * 1024 * 8)
78 typedef struct {
79 struct rcu_head rcu;
80 unsigned long *blocks[];
81 } DirtyMemoryBlocks;
82
83 typedef struct RAMList {
84 QemuMutex mutex;
85 RAMBlock *mru_block;
86 /* RCU-enabled, writes protected by the ramlist lock. */
87 QLIST_HEAD(, RAMBlock) blocks;
88 DirtyMemoryBlocks *dirty_memory[DIRTY_MEMORY_NUM];
89 uint32_t version;
90 } RAMList;
91 extern RAMList ram_list;
92
93 ram_addr_t last_ram_offset(void);
94 void qemu_mutex_lock_ramlist(void);
95 void qemu_mutex_unlock_ramlist(void);
96
97 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
98 bool share, const char *mem_path,
99 Error **errp);
100 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
101 MemoryRegion *mr, Error **errp);
102 RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
103 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
104 void (*resized)(const char*,
105 uint64_t length,
106 void *host),
107 MemoryRegion *mr, Error **errp);
108 int qemu_get_ram_fd(ram_addr_t addr);
109 void qemu_set_ram_fd(ram_addr_t addr, int fd);
110 void *qemu_get_ram_block_host_ptr(ram_addr_t addr);
111 void qemu_ram_free(RAMBlock *block);
112
113 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
114
115 #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1)
116 #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
117
118 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
119 ram_addr_t length,
120 unsigned client)
121 {
122 DirtyMemoryBlocks *blocks;
123 unsigned long end, page;
124 unsigned long idx, offset, base;
125 bool dirty = false;
126
127 assert(client < DIRTY_MEMORY_NUM);
128
129 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
130 page = start >> TARGET_PAGE_BITS;
131
132 rcu_read_lock();
133
134 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
135
136 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
137 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
138 base = page - offset;
139 while (page < end) {
140 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
141 unsigned long num = next - base;
142 unsigned long found = find_next_bit(blocks->blocks[idx], num, offset);
143 if (found < num) {
144 dirty = true;
145 break;
146 }
147
148 page = next;
149 idx++;
150 offset = 0;
151 base += DIRTY_MEMORY_BLOCK_SIZE;
152 }
153
154 rcu_read_unlock();
155
156 return dirty;
157 }
158
159 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
160 ram_addr_t length,
161 unsigned client)
162 {
163 DirtyMemoryBlocks *blocks;
164 unsigned long end, page;
165 unsigned long idx, offset, base;
166 bool dirty = true;
167
168 assert(client < DIRTY_MEMORY_NUM);
169
170 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
171 page = start >> TARGET_PAGE_BITS;
172
173 rcu_read_lock();
174
175 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
176
177 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
178 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
179 base = page - offset;
180 while (page < end) {
181 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
182 unsigned long num = next - base;
183 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
184 if (found < num) {
185 dirty = false;
186 break;
187 }
188
189 page = next;
190 idx++;
191 offset = 0;
192 base += DIRTY_MEMORY_BLOCK_SIZE;
193 }
194
195 rcu_read_unlock();
196
197 return dirty;
198 }
199
200 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
201 unsigned client)
202 {
203 return cpu_physical_memory_get_dirty(addr, 1, client);
204 }
205
206 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
207 {
208 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
209 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
210 bool migration =
211 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
212 return !(vga && code && migration);
213 }
214
215 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
216 ram_addr_t length,
217 uint8_t mask)
218 {
219 uint8_t ret = 0;
220
221 if (mask & (1 << DIRTY_MEMORY_VGA) &&
222 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
223 ret |= (1 << DIRTY_MEMORY_VGA);
224 }
225 if (mask & (1 << DIRTY_MEMORY_CODE) &&
226 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
227 ret |= (1 << DIRTY_MEMORY_CODE);
228 }
229 if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
230 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
231 ret |= (1 << DIRTY_MEMORY_MIGRATION);
232 }
233 return ret;
234 }
235
236 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
237 unsigned client)
238 {
239 unsigned long page, idx, offset;
240 DirtyMemoryBlocks *blocks;
241
242 assert(client < DIRTY_MEMORY_NUM);
243
244 page = addr >> TARGET_PAGE_BITS;
245 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
246 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
247
248 rcu_read_lock();
249
250 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
251
252 set_bit_atomic(offset, blocks->blocks[idx]);
253
254 rcu_read_unlock();
255 }
256
257 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
258 ram_addr_t length,
259 uint8_t mask)
260 {
261 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
262 unsigned long end, page;
263 unsigned long idx, offset, base;
264 int i;
265
266 if (!mask && !xen_enabled()) {
267 return;
268 }
269
270 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
271 page = start >> TARGET_PAGE_BITS;
272
273 rcu_read_lock();
274
275 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
276 blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
277 }
278
279 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
280 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
281 base = page - offset;
282 while (page < end) {
283 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
284
285 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
286 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
287 offset, next - page);
288 }
289 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
290 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
291 offset, next - page);
292 }
293 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
294 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
295 offset, next - page);
296 }
297
298 page = next;
299 idx++;
300 offset = 0;
301 base += DIRTY_MEMORY_BLOCK_SIZE;
302 }
303
304 rcu_read_unlock();
305
306 xen_modified_memory(start, length);
307 }
308
309 #if !defined(_WIN32)
310 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
311 ram_addr_t start,
312 ram_addr_t pages)
313 {
314 unsigned long i, j;
315 unsigned long page_number, c;
316 hwaddr addr;
317 ram_addr_t ram_addr;
318 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
319 unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
320 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
321
322 /* start address is aligned at the start of a word? */
323 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
324 (hpratio == 1)) {
325 unsigned long **blocks[DIRTY_MEMORY_NUM];
326 unsigned long idx;
327 unsigned long offset;
328 long k;
329 long nr = BITS_TO_LONGS(pages);
330
331 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
332 offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
333 DIRTY_MEMORY_BLOCK_SIZE);
334
335 rcu_read_lock();
336
337 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
338 blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
339 }
340
341 for (k = 0; k < nr; k++) {
342 if (bitmap[k]) {
343 unsigned long temp = leul_to_cpu(bitmap[k]);
344
345 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
346 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
347 if (tcg_enabled()) {
348 atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
349 }
350 }
351
352 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
353 offset = 0;
354 idx++;
355 }
356 }
357
358 rcu_read_unlock();
359
360 xen_modified_memory(start, pages << TARGET_PAGE_BITS);
361 } else {
362 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
363 /*
364 * bitmap-traveling is faster than memory-traveling (for addr...)
365 * especially when most of the memory is not dirty.
366 */
367 for (i = 0; i < len; i++) {
368 if (bitmap[i] != 0) {
369 c = leul_to_cpu(bitmap[i]);
370 do {
371 j = ctzl(c);
372 c &= ~(1ul << j);
373 page_number = (i * HOST_LONG_BITS + j) * hpratio;
374 addr = page_number * TARGET_PAGE_SIZE;
375 ram_addr = start + addr;
376 cpu_physical_memory_set_dirty_range(ram_addr,
377 TARGET_PAGE_SIZE * hpratio, clients);
378 } while (c != 0);
379 }
380 }
381 }
382 }
383 #endif /* not _WIN32 */
384
385 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
386 ram_addr_t length,
387 unsigned client);
388
389 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
390 ram_addr_t length)
391 {
392 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
393 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
394 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
395 }
396
397
398 static inline
399 uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
400 ram_addr_t start,
401 ram_addr_t length)
402 {
403 ram_addr_t addr;
404 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
405 uint64_t num_dirty = 0;
406
407 /* start address is aligned at the start of a word? */
408 if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
409 int k;
410 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
411 unsigned long * const *src;
412 unsigned long idx = (page * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
413 unsigned long offset = BIT_WORD((page * BITS_PER_LONG) %
414 DIRTY_MEMORY_BLOCK_SIZE);
415
416 rcu_read_lock();
417
418 src = atomic_rcu_read(
419 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
420
421 for (k = page; k < page + nr; k++) {
422 if (src[idx][offset]) {
423 unsigned long bits = atomic_xchg(&src[idx][offset], 0);
424 unsigned long new_dirty;
425 new_dirty = ~dest[k];
426 dest[k] |= bits;
427 new_dirty &= bits;
428 num_dirty += ctpopl(new_dirty);
429 }
430
431 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
432 offset = 0;
433 idx++;
434 }
435 }
436
437 rcu_read_unlock();
438 } else {
439 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
440 if (cpu_physical_memory_test_and_clear_dirty(
441 start + addr,
442 TARGET_PAGE_SIZE,
443 DIRTY_MEMORY_MIGRATION)) {
444 long k = (start + addr) >> TARGET_PAGE_BITS;
445 if (!test_and_set_bit(k, dest)) {
446 num_dirty++;
447 }
448 }
449 }
450 }
451
452 return num_dirty;
453 }
454
455 void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
456 #endif
457 #endif