Merge tag 'block-pull-request' of https://gitlab.com/stefanha/qemu into staging
[qemu.git] / include / exec / ram_addr.h
1 /*
2 * Declarations for cpu physical memory functions
3 *
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates
5 *
6 * Authors:
7 * Avi Kivity <avi@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or
10 * later. See the COPYING file in the top-level directory.
11 *
12 */
13
14 /*
15 * This header is for use by exec.c and memory.c ONLY. Do not include it.
16 * The functions declared here will be removed soon.
17 */
18
19 #ifndef RAM_ADDR_H
20 #define RAM_ADDR_H
21
22 #ifndef CONFIG_USER_ONLY
23 #include "cpu.h"
24 #include "sysemu/xen.h"
25 #include "sysemu/tcg.h"
26 #include "exec/ramlist.h"
27 #include "exec/ramblock.h"
28
29 extern uint64_t total_dirty_pages;
30
31 /**
32 * clear_bmap_size: calculate clear bitmap size
33 *
34 * @pages: number of guest pages
35 * @shift: guest page number shift
36 *
37 * Returns: number of bits for the clear bitmap
38 */
39 static inline long clear_bmap_size(uint64_t pages, uint8_t shift)
40 {
41 return DIV_ROUND_UP(pages, 1UL << shift);
42 }
43
44 /**
45 * clear_bmap_set: set clear bitmap for the page range. Must be with
46 * bitmap_mutex held.
47 *
48 * @rb: the ramblock to operate on
49 * @start: the start page number
50 * @size: number of pages to set in the bitmap
51 *
52 * Returns: None
53 */
54 static inline void clear_bmap_set(RAMBlock *rb, uint64_t start,
55 uint64_t npages)
56 {
57 uint8_t shift = rb->clear_bmap_shift;
58
59 bitmap_set(rb->clear_bmap, start >> shift, clear_bmap_size(npages, shift));
60 }
61
62 /**
63 * clear_bmap_test_and_clear: test clear bitmap for the page, clear if set.
64 * Must be with bitmap_mutex held.
65 *
66 * @rb: the ramblock to operate on
67 * @page: the page number to check
68 *
69 * Returns: true if the bit was set, false otherwise
70 */
71 static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page)
72 {
73 uint8_t shift = rb->clear_bmap_shift;
74
75 return bitmap_test_and_clear(rb->clear_bmap, page >> shift, 1);
76 }
77
78 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
79 {
80 return (b && b->host && offset < b->used_length) ? true : false;
81 }
82
83 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
84 {
85 assert(offset_in_ramblock(block, offset));
86 return (char *)block->host + offset;
87 }
88
89 static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr,
90 RAMBlock *rb)
91 {
92 uint64_t host_addr_offset =
93 (uint64_t)(uintptr_t)(host_addr - (void *)rb->host);
94 return host_addr_offset >> TARGET_PAGE_BITS;
95 }
96
97 bool ramblock_is_pmem(RAMBlock *rb);
98
99 long qemu_minrampagesize(void);
100 long qemu_maxrampagesize(void);
101
102 /**
103 * qemu_ram_alloc_from_file,
104 * qemu_ram_alloc_from_fd: Allocate a ram block from the specified backing
105 * file or device
106 *
107 * Parameters:
108 * @size: the size in bytes of the ram block
109 * @mr: the memory region where the ram block is
110 * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM,
111 * RAM_NORESERVE.
112 * @mem_path or @fd: specify the backing file or device
113 * @readonly: true to open @path for reading, false for read/write.
114 * @errp: pointer to Error*, to store an error if it happens
115 *
116 * Return:
117 * On success, return a pointer to the ram block.
118 * On failure, return NULL.
119 */
120 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
121 uint32_t ram_flags, const char *mem_path,
122 bool readonly, Error **errp);
123 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
124 uint32_t ram_flags, int fd, off_t offset,
125 bool readonly, Error **errp);
126
127 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
128 MemoryRegion *mr, Error **errp);
129 RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags, MemoryRegion *mr,
130 Error **errp);
131 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
132 void (*resized)(const char*,
133 uint64_t length,
134 void *host),
135 MemoryRegion *mr, Error **errp);
136 void qemu_ram_free(RAMBlock *block);
137
138 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
139
140 void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length);
141
142 /* Clear whole block of mem */
143 static inline void qemu_ram_block_writeback(RAMBlock *block)
144 {
145 qemu_ram_msync(block, 0, block->used_length);
146 }
147
148 #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1)
149 #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
150
151 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
152 ram_addr_t length,
153 unsigned client)
154 {
155 DirtyMemoryBlocks *blocks;
156 unsigned long end, page;
157 unsigned long idx, offset, base;
158 bool dirty = false;
159
160 assert(client < DIRTY_MEMORY_NUM);
161
162 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
163 page = start >> TARGET_PAGE_BITS;
164
165 WITH_RCU_READ_LOCK_GUARD() {
166 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
167
168 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
169 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
170 base = page - offset;
171 while (page < end) {
172 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
173 unsigned long num = next - base;
174 unsigned long found = find_next_bit(blocks->blocks[idx],
175 num, offset);
176 if (found < num) {
177 dirty = true;
178 break;
179 }
180
181 page = next;
182 idx++;
183 offset = 0;
184 base += DIRTY_MEMORY_BLOCK_SIZE;
185 }
186 }
187
188 return dirty;
189 }
190
191 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
192 ram_addr_t length,
193 unsigned client)
194 {
195 DirtyMemoryBlocks *blocks;
196 unsigned long end, page;
197 unsigned long idx, offset, base;
198 bool dirty = true;
199
200 assert(client < DIRTY_MEMORY_NUM);
201
202 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
203 page = start >> TARGET_PAGE_BITS;
204
205 RCU_READ_LOCK_GUARD();
206
207 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
208
209 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
210 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
211 base = page - offset;
212 while (page < end) {
213 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
214 unsigned long num = next - base;
215 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
216 if (found < num) {
217 dirty = false;
218 break;
219 }
220
221 page = next;
222 idx++;
223 offset = 0;
224 base += DIRTY_MEMORY_BLOCK_SIZE;
225 }
226
227 return dirty;
228 }
229
230 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
231 unsigned client)
232 {
233 return cpu_physical_memory_get_dirty(addr, 1, client);
234 }
235
236 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
237 {
238 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
239 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
240 bool migration =
241 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
242 return !(vga && code && migration);
243 }
244
245 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
246 ram_addr_t length,
247 uint8_t mask)
248 {
249 uint8_t ret = 0;
250
251 if (mask & (1 << DIRTY_MEMORY_VGA) &&
252 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
253 ret |= (1 << DIRTY_MEMORY_VGA);
254 }
255 if (mask & (1 << DIRTY_MEMORY_CODE) &&
256 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
257 ret |= (1 << DIRTY_MEMORY_CODE);
258 }
259 if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
260 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
261 ret |= (1 << DIRTY_MEMORY_MIGRATION);
262 }
263 return ret;
264 }
265
266 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
267 unsigned client)
268 {
269 unsigned long page, idx, offset;
270 DirtyMemoryBlocks *blocks;
271
272 assert(client < DIRTY_MEMORY_NUM);
273
274 page = addr >> TARGET_PAGE_BITS;
275 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
276 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
277
278 RCU_READ_LOCK_GUARD();
279
280 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
281
282 set_bit_atomic(offset, blocks->blocks[idx]);
283 }
284
285 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
286 ram_addr_t length,
287 uint8_t mask)
288 {
289 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
290 unsigned long end, page;
291 unsigned long idx, offset, base;
292 int i;
293
294 if (!mask && !xen_enabled()) {
295 return;
296 }
297
298 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
299 page = start >> TARGET_PAGE_BITS;
300
301 WITH_RCU_READ_LOCK_GUARD() {
302 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
303 blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]);
304 }
305
306 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
307 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
308 base = page - offset;
309 while (page < end) {
310 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
311
312 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
313 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
314 offset, next - page);
315 }
316 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
317 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
318 offset, next - page);
319 }
320 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
321 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
322 offset, next - page);
323 }
324
325 page = next;
326 idx++;
327 offset = 0;
328 base += DIRTY_MEMORY_BLOCK_SIZE;
329 }
330 }
331
332 xen_hvm_modified_memory(start, length);
333 }
334
335 #if !defined(_WIN32)
336 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
337 ram_addr_t start,
338 ram_addr_t pages)
339 {
340 unsigned long i, j;
341 unsigned long page_number, c;
342 hwaddr addr;
343 ram_addr_t ram_addr;
344 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
345 unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE;
346 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
347
348 /* start address is aligned at the start of a word? */
349 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
350 (hpratio == 1)) {
351 unsigned long **blocks[DIRTY_MEMORY_NUM];
352 unsigned long idx;
353 unsigned long offset;
354 long k;
355 long nr = BITS_TO_LONGS(pages);
356
357 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
358 offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
359 DIRTY_MEMORY_BLOCK_SIZE);
360
361 WITH_RCU_READ_LOCK_GUARD() {
362 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
363 blocks[i] =
364 qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
365 }
366
367 for (k = 0; k < nr; k++) {
368 if (bitmap[k]) {
369 unsigned long temp = leul_to_cpu(bitmap[k]);
370
371 qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
372
373 if (global_dirty_tracking) {
374 qatomic_or(
375 &blocks[DIRTY_MEMORY_MIGRATION][idx][offset],
376 temp);
377 if (unlikely(
378 global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
379 total_dirty_pages += ctpopl(temp);
380 }
381 }
382
383 if (tcg_enabled()) {
384 qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset],
385 temp);
386 }
387 }
388
389 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
390 offset = 0;
391 idx++;
392 }
393 }
394 }
395
396 xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS);
397 } else {
398 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
399
400 if (!global_dirty_tracking) {
401 clients &= ~(1 << DIRTY_MEMORY_MIGRATION);
402 }
403
404 /*
405 * bitmap-traveling is faster than memory-traveling (for addr...)
406 * especially when most of the memory is not dirty.
407 */
408 for (i = 0; i < len; i++) {
409 if (bitmap[i] != 0) {
410 c = leul_to_cpu(bitmap[i]);
411 if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
412 total_dirty_pages += ctpopl(c);
413 }
414 do {
415 j = ctzl(c);
416 c &= ~(1ul << j);
417 page_number = (i * HOST_LONG_BITS + j) * hpratio;
418 addr = page_number * TARGET_PAGE_SIZE;
419 ram_addr = start + addr;
420 cpu_physical_memory_set_dirty_range(ram_addr,
421 TARGET_PAGE_SIZE * hpratio, clients);
422 } while (c != 0);
423 }
424 }
425 }
426 }
427 #endif /* not _WIN32 */
428
429 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
430 ram_addr_t length,
431 unsigned client);
432
433 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
434 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client);
435
436 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
437 ram_addr_t start,
438 ram_addr_t length);
439
440 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
441 ram_addr_t length)
442 {
443 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
444 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
445 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
446 }
447
448
449 /* Called with RCU critical section */
450 static inline
451 uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb,
452 ram_addr_t start,
453 ram_addr_t length)
454 {
455 ram_addr_t addr;
456 unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS);
457 uint64_t num_dirty = 0;
458 unsigned long *dest = rb->bmap;
459
460 /* start address and length is aligned at the start of a word? */
461 if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) ==
462 (start + rb->offset) &&
463 !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) {
464 int k;
465 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
466 unsigned long * const *src;
467 unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
468 unsigned long offset = BIT_WORD((word * BITS_PER_LONG) %
469 DIRTY_MEMORY_BLOCK_SIZE);
470 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
471
472 src = qatomic_rcu_read(
473 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
474
475 for (k = page; k < page + nr; k++) {
476 if (src[idx][offset]) {
477 unsigned long bits = qatomic_xchg(&src[idx][offset], 0);
478 unsigned long new_dirty;
479 new_dirty = ~dest[k];
480 dest[k] |= bits;
481 new_dirty &= bits;
482 num_dirty += ctpopl(new_dirty);
483 }
484
485 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
486 offset = 0;
487 idx++;
488 }
489 }
490
491 if (rb->clear_bmap) {
492 /*
493 * Postpone the dirty bitmap clear to the point before we
494 * really send the pages, also we will split the clear
495 * dirty procedure into smaller chunks.
496 */
497 clear_bmap_set(rb, start >> TARGET_PAGE_BITS,
498 length >> TARGET_PAGE_BITS);
499 } else {
500 /* Slow path - still do that in a huge chunk */
501 memory_region_clear_dirty_bitmap(rb->mr, start, length);
502 }
503 } else {
504 ram_addr_t offset = rb->offset;
505
506 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
507 if (cpu_physical_memory_test_and_clear_dirty(
508 start + addr + offset,
509 TARGET_PAGE_SIZE,
510 DIRTY_MEMORY_MIGRATION)) {
511 long k = (start + addr) >> TARGET_PAGE_BITS;
512 if (!test_and_set_bit(k, dest)) {
513 num_dirty++;
514 }
515 }
516 }
517 }
518
519 return num_dirty;
520 }
521 #endif
522 #endif