Merge tag 'for-upstream' of https://gitlab.com/bonzini/qemu into staging
[qemu.git] / include / exec / ram_addr.h
1 /*
2 * Declarations for cpu physical memory functions
3 *
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates
5 *
6 * Authors:
7 * Avi Kivity <avi@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or
10 * later. See the COPYING file in the top-level directory.
11 *
12 */
13
14 /*
15 * This header is for use by exec.c and memory.c ONLY. Do not include it.
16 * The functions declared here will be removed soon.
17 */
18
19 #ifndef RAM_ADDR_H
20 #define RAM_ADDR_H
21
22 #ifndef CONFIG_USER_ONLY
23 #include "cpu.h"
24 #include "sysemu/xen.h"
25 #include "sysemu/tcg.h"
26 #include "exec/ramlist.h"
27 #include "exec/ramblock.h"
28
29 extern uint64_t total_dirty_pages;
30
31 /**
32 * clear_bmap_size: calculate clear bitmap size
33 *
34 * @pages: number of guest pages
35 * @shift: guest page number shift
36 *
37 * Returns: number of bits for the clear bitmap
38 */
39 static inline long clear_bmap_size(uint64_t pages, uint8_t shift)
40 {
41 return DIV_ROUND_UP(pages, 1UL << shift);
42 }
43
44 /**
45 * clear_bmap_set: set clear bitmap for the page range
46 *
47 * @rb: the ramblock to operate on
48 * @start: the start page number
49 * @size: number of pages to set in the bitmap
50 *
51 * Returns: None
52 */
53 static inline void clear_bmap_set(RAMBlock *rb, uint64_t start,
54 uint64_t npages)
55 {
56 uint8_t shift = rb->clear_bmap_shift;
57
58 bitmap_set_atomic(rb->clear_bmap, start >> shift,
59 clear_bmap_size(npages, shift));
60 }
61
62 /**
63 * clear_bmap_test_and_clear: test clear bitmap for the page, clear if set
64 *
65 * @rb: the ramblock to operate on
66 * @page: the page number to check
67 *
68 * Returns: true if the bit was set, false otherwise
69 */
70 static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page)
71 {
72 uint8_t shift = rb->clear_bmap_shift;
73
74 return bitmap_test_and_clear_atomic(rb->clear_bmap, page >> shift, 1);
75 }
76
77 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
78 {
79 return (b && b->host && offset < b->used_length) ? true : false;
80 }
81
82 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
83 {
84 assert(offset_in_ramblock(block, offset));
85 return (char *)block->host + offset;
86 }
87
88 static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr,
89 RAMBlock *rb)
90 {
91 uint64_t host_addr_offset =
92 (uint64_t)(uintptr_t)(host_addr - (void *)rb->host);
93 return host_addr_offset >> TARGET_PAGE_BITS;
94 }
95
96 bool ramblock_is_pmem(RAMBlock *rb);
97
98 long qemu_minrampagesize(void);
99 long qemu_maxrampagesize(void);
100
101 /**
102 * qemu_ram_alloc_from_file,
103 * qemu_ram_alloc_from_fd: Allocate a ram block from the specified backing
104 * file or device
105 *
106 * Parameters:
107 * @size: the size in bytes of the ram block
108 * @mr: the memory region where the ram block is
109 * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM,
110 * RAM_NORESERVE.
111 * @mem_path or @fd: specify the backing file or device
112 * @readonly: true to open @path for reading, false for read/write.
113 * @errp: pointer to Error*, to store an error if it happens
114 *
115 * Return:
116 * On success, return a pointer to the ram block.
117 * On failure, return NULL.
118 */
119 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
120 uint32_t ram_flags, const char *mem_path,
121 bool readonly, Error **errp);
122 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
123 uint32_t ram_flags, int fd, off_t offset,
124 bool readonly, Error **errp);
125
126 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
127 MemoryRegion *mr, Error **errp);
128 RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags, MemoryRegion *mr,
129 Error **errp);
130 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
131 void (*resized)(const char*,
132 uint64_t length,
133 void *host),
134 MemoryRegion *mr, Error **errp);
135 void qemu_ram_free(RAMBlock *block);
136
137 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
138
139 void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length);
140
141 /* Clear whole block of mem */
142 static inline void qemu_ram_block_writeback(RAMBlock *block)
143 {
144 qemu_ram_msync(block, 0, block->used_length);
145 }
146
147 #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1)
148 #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
149
150 void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end);
151
152 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
153 ram_addr_t length,
154 unsigned client)
155 {
156 DirtyMemoryBlocks *blocks;
157 unsigned long end, page;
158 unsigned long idx, offset, base;
159 bool dirty = false;
160
161 assert(client < DIRTY_MEMORY_NUM);
162
163 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
164 page = start >> TARGET_PAGE_BITS;
165
166 WITH_RCU_READ_LOCK_GUARD() {
167 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
168
169 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
170 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
171 base = page - offset;
172 while (page < end) {
173 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
174 unsigned long num = next - base;
175 unsigned long found = find_next_bit(blocks->blocks[idx],
176 num, offset);
177 if (found < num) {
178 dirty = true;
179 break;
180 }
181
182 page = next;
183 idx++;
184 offset = 0;
185 base += DIRTY_MEMORY_BLOCK_SIZE;
186 }
187 }
188
189 return dirty;
190 }
191
192 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
193 ram_addr_t length,
194 unsigned client)
195 {
196 DirtyMemoryBlocks *blocks;
197 unsigned long end, page;
198 unsigned long idx, offset, base;
199 bool dirty = true;
200
201 assert(client < DIRTY_MEMORY_NUM);
202
203 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
204 page = start >> TARGET_PAGE_BITS;
205
206 RCU_READ_LOCK_GUARD();
207
208 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
209
210 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
211 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
212 base = page - offset;
213 while (page < end) {
214 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
215 unsigned long num = next - base;
216 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
217 if (found < num) {
218 dirty = false;
219 break;
220 }
221
222 page = next;
223 idx++;
224 offset = 0;
225 base += DIRTY_MEMORY_BLOCK_SIZE;
226 }
227
228 return dirty;
229 }
230
231 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
232 unsigned client)
233 {
234 return cpu_physical_memory_get_dirty(addr, 1, client);
235 }
236
237 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
238 {
239 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
240 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
241 bool migration =
242 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
243 return !(vga && code && migration);
244 }
245
246 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
247 ram_addr_t length,
248 uint8_t mask)
249 {
250 uint8_t ret = 0;
251
252 if (mask & (1 << DIRTY_MEMORY_VGA) &&
253 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
254 ret |= (1 << DIRTY_MEMORY_VGA);
255 }
256 if (mask & (1 << DIRTY_MEMORY_CODE) &&
257 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
258 ret |= (1 << DIRTY_MEMORY_CODE);
259 }
260 if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
261 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
262 ret |= (1 << DIRTY_MEMORY_MIGRATION);
263 }
264 return ret;
265 }
266
267 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
268 unsigned client)
269 {
270 unsigned long page, idx, offset;
271 DirtyMemoryBlocks *blocks;
272
273 assert(client < DIRTY_MEMORY_NUM);
274
275 page = addr >> TARGET_PAGE_BITS;
276 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
277 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
278
279 RCU_READ_LOCK_GUARD();
280
281 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
282
283 set_bit_atomic(offset, blocks->blocks[idx]);
284 }
285
286 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
287 ram_addr_t length,
288 uint8_t mask)
289 {
290 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
291 unsigned long end, page;
292 unsigned long idx, offset, base;
293 int i;
294
295 if (!mask && !xen_enabled()) {
296 return;
297 }
298
299 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
300 page = start >> TARGET_PAGE_BITS;
301
302 WITH_RCU_READ_LOCK_GUARD() {
303 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
304 blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]);
305 }
306
307 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
308 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
309 base = page - offset;
310 while (page < end) {
311 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
312
313 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
314 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
315 offset, next - page);
316 }
317 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
318 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
319 offset, next - page);
320 }
321 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
322 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
323 offset, next - page);
324 }
325
326 page = next;
327 idx++;
328 offset = 0;
329 base += DIRTY_MEMORY_BLOCK_SIZE;
330 }
331 }
332
333 xen_hvm_modified_memory(start, length);
334 }
335
336 #if !defined(_WIN32)
337 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
338 ram_addr_t start,
339 ram_addr_t pages)
340 {
341 unsigned long i, j;
342 unsigned long page_number, c;
343 hwaddr addr;
344 ram_addr_t ram_addr;
345 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
346 unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE;
347 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
348
349 /* start address is aligned at the start of a word? */
350 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
351 (hpratio == 1)) {
352 unsigned long **blocks[DIRTY_MEMORY_NUM];
353 unsigned long idx;
354 unsigned long offset;
355 long k;
356 long nr = BITS_TO_LONGS(pages);
357
358 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
359 offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
360 DIRTY_MEMORY_BLOCK_SIZE);
361
362 WITH_RCU_READ_LOCK_GUARD() {
363 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
364 blocks[i] =
365 qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
366 }
367
368 for (k = 0; k < nr; k++) {
369 if (bitmap[k]) {
370 unsigned long temp = leul_to_cpu(bitmap[k]);
371
372 qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
373
374 if (global_dirty_tracking) {
375 qatomic_or(
376 &blocks[DIRTY_MEMORY_MIGRATION][idx][offset],
377 temp);
378 if (unlikely(
379 global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
380 total_dirty_pages += ctpopl(temp);
381 }
382 }
383
384 if (tcg_enabled()) {
385 qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset],
386 temp);
387 }
388 }
389
390 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
391 offset = 0;
392 idx++;
393 }
394 }
395 }
396
397 xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS);
398 } else {
399 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
400
401 if (!global_dirty_tracking) {
402 clients &= ~(1 << DIRTY_MEMORY_MIGRATION);
403 }
404
405 /*
406 * bitmap-traveling is faster than memory-traveling (for addr...)
407 * especially when most of the memory is not dirty.
408 */
409 for (i = 0; i < len; i++) {
410 if (bitmap[i] != 0) {
411 c = leul_to_cpu(bitmap[i]);
412 if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) {
413 total_dirty_pages += ctpopl(c);
414 }
415 do {
416 j = ctzl(c);
417 c &= ~(1ul << j);
418 page_number = (i * HOST_LONG_BITS + j) * hpratio;
419 addr = page_number * TARGET_PAGE_SIZE;
420 ram_addr = start + addr;
421 cpu_physical_memory_set_dirty_range(ram_addr,
422 TARGET_PAGE_SIZE * hpratio, clients);
423 } while (c != 0);
424 }
425 }
426 }
427 }
428 #endif /* not _WIN32 */
429
430 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
431 ram_addr_t length,
432 unsigned client);
433
434 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
435 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client);
436
437 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
438 ram_addr_t start,
439 ram_addr_t length);
440
441 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
442 ram_addr_t length)
443 {
444 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
445 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
446 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
447 }
448
449
450 /* Called with RCU critical section */
451 static inline
452 uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb,
453 ram_addr_t start,
454 ram_addr_t length)
455 {
456 ram_addr_t addr;
457 unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS);
458 uint64_t num_dirty = 0;
459 unsigned long *dest = rb->bmap;
460
461 /* start address and length is aligned at the start of a word? */
462 if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) ==
463 (start + rb->offset) &&
464 !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) {
465 int k;
466 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
467 unsigned long * const *src;
468 unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
469 unsigned long offset = BIT_WORD((word * BITS_PER_LONG) %
470 DIRTY_MEMORY_BLOCK_SIZE);
471 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
472
473 src = qatomic_rcu_read(
474 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
475
476 for (k = page; k < page + nr; k++) {
477 if (src[idx][offset]) {
478 unsigned long bits = qatomic_xchg(&src[idx][offset], 0);
479 unsigned long new_dirty;
480 new_dirty = ~dest[k];
481 dest[k] |= bits;
482 new_dirty &= bits;
483 num_dirty += ctpopl(new_dirty);
484 }
485
486 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
487 offset = 0;
488 idx++;
489 }
490 }
491
492 if (rb->clear_bmap) {
493 /*
494 * Postpone the dirty bitmap clear to the point before we
495 * really send the pages, also we will split the clear
496 * dirty procedure into smaller chunks.
497 */
498 clear_bmap_set(rb, start >> TARGET_PAGE_BITS,
499 length >> TARGET_PAGE_BITS);
500 } else {
501 /* Slow path - still do that in a huge chunk */
502 memory_region_clear_dirty_bitmap(rb->mr, start, length);
503 }
504 } else {
505 ram_addr_t offset = rb->offset;
506
507 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
508 if (cpu_physical_memory_test_and_clear_dirty(
509 start + addr + offset,
510 TARGET_PAGE_SIZE,
511 DIRTY_MEMORY_MIGRATION)) {
512 long k = (start + addr) >> TARGET_PAGE_BITS;
513 if (!test_and_set_bit(k, dest)) {
514 num_dirty++;
515 }
516 }
517 }
518 }
519
520 return num_dirty;
521 }
522 #endif
523 #endif