accel: Move Xen accelerator code under accel/xen/
[qemu.git] / include / exec / ram_addr.h
1 /*
2 * Declarations for cpu physical memory functions
3 *
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates
5 *
6 * Authors:
7 * Avi Kivity <avi@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2 or
10 * later. See the COPYING file in the top-level directory.
11 *
12 */
13
14 /*
15 * This header is for use by exec.c and memory.c ONLY. Do not include it.
16 * The functions declared here will be removed soon.
17 */
18
19 #ifndef RAM_ADDR_H
20 #define RAM_ADDR_H
21
22 #ifndef CONFIG_USER_ONLY
23 #include "cpu.h"
24 #include "sysemu/xen.h"
25 #include "sysemu/tcg.h"
26 #include "exec/ramlist.h"
27 #include "exec/ramblock.h"
28
29 /**
30 * clear_bmap_size: calculate clear bitmap size
31 *
32 * @pages: number of guest pages
33 * @shift: guest page number shift
34 *
35 * Returns: number of bits for the clear bitmap
36 */
37 static inline long clear_bmap_size(uint64_t pages, uint8_t shift)
38 {
39 return DIV_ROUND_UP(pages, 1UL << shift);
40 }
41
42 /**
43 * clear_bmap_set: set clear bitmap for the page range
44 *
45 * @rb: the ramblock to operate on
46 * @start: the start page number
47 * @size: number of pages to set in the bitmap
48 *
49 * Returns: None
50 */
51 static inline void clear_bmap_set(RAMBlock *rb, uint64_t start,
52 uint64_t npages)
53 {
54 uint8_t shift = rb->clear_bmap_shift;
55
56 bitmap_set_atomic(rb->clear_bmap, start >> shift,
57 clear_bmap_size(npages, shift));
58 }
59
60 /**
61 * clear_bmap_test_and_clear: test clear bitmap for the page, clear if set
62 *
63 * @rb: the ramblock to operate on
64 * @page: the page number to check
65 *
66 * Returns: true if the bit was set, false otherwise
67 */
68 static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page)
69 {
70 uint8_t shift = rb->clear_bmap_shift;
71
72 return bitmap_test_and_clear_atomic(rb->clear_bmap, page >> shift, 1);
73 }
74
75 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
76 {
77 return (b && b->host && offset < b->used_length) ? true : false;
78 }
79
80 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
81 {
82 assert(offset_in_ramblock(block, offset));
83 return (char *)block->host + offset;
84 }
85
86 static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr,
87 RAMBlock *rb)
88 {
89 uint64_t host_addr_offset =
90 (uint64_t)(uintptr_t)(host_addr - (void *)rb->host);
91 return host_addr_offset >> TARGET_PAGE_BITS;
92 }
93
94 bool ramblock_is_pmem(RAMBlock *rb);
95
96 long qemu_minrampagesize(void);
97 long qemu_maxrampagesize(void);
98
99 /**
100 * qemu_ram_alloc_from_file,
101 * qemu_ram_alloc_from_fd: Allocate a ram block from the specified backing
102 * file or device
103 *
104 * Parameters:
105 * @size: the size in bytes of the ram block
106 * @mr: the memory region where the ram block is
107 * @ram_flags: specify the properties of the ram block, which can be one
108 * or bit-or of following values
109 * - RAM_SHARED: mmap the backing file or device with MAP_SHARED
110 * - RAM_PMEM: the backend @mem_path or @fd is persistent memory
111 * Other bits are ignored.
112 * @mem_path or @fd: specify the backing file or device
113 * @errp: pointer to Error*, to store an error if it happens
114 *
115 * Return:
116 * On success, return a pointer to the ram block.
117 * On failure, return NULL.
118 */
119 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
120 uint32_t ram_flags, const char *mem_path,
121 Error **errp);
122 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
123 uint32_t ram_flags, int fd,
124 Error **errp);
125
126 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
127 MemoryRegion *mr, Error **errp);
128 RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share, MemoryRegion *mr,
129 Error **errp);
130 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
131 void (*resized)(const char*,
132 uint64_t length,
133 void *host),
134 MemoryRegion *mr, Error **errp);
135 void qemu_ram_free(RAMBlock *block);
136
137 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
138
139 void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length);
140
141 /* Clear whole block of mem */
142 static inline void qemu_ram_block_writeback(RAMBlock *block)
143 {
144 qemu_ram_msync(block, 0, block->used_length);
145 }
146
147 #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1)
148 #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
149
150 void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end);
151
152 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
153 ram_addr_t length,
154 unsigned client)
155 {
156 DirtyMemoryBlocks *blocks;
157 unsigned long end, page;
158 unsigned long idx, offset, base;
159 bool dirty = false;
160
161 assert(client < DIRTY_MEMORY_NUM);
162
163 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
164 page = start >> TARGET_PAGE_BITS;
165
166 WITH_RCU_READ_LOCK_GUARD() {
167 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
168
169 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
170 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
171 base = page - offset;
172 while (page < end) {
173 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
174 unsigned long num = next - base;
175 unsigned long found = find_next_bit(blocks->blocks[idx],
176 num, offset);
177 if (found < num) {
178 dirty = true;
179 break;
180 }
181
182 page = next;
183 idx++;
184 offset = 0;
185 base += DIRTY_MEMORY_BLOCK_SIZE;
186 }
187 }
188
189 return dirty;
190 }
191
192 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
193 ram_addr_t length,
194 unsigned client)
195 {
196 DirtyMemoryBlocks *blocks;
197 unsigned long end, page;
198 unsigned long idx, offset, base;
199 bool dirty = true;
200
201 assert(client < DIRTY_MEMORY_NUM);
202
203 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
204 page = start >> TARGET_PAGE_BITS;
205
206 RCU_READ_LOCK_GUARD();
207
208 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
209
210 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
211 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
212 base = page - offset;
213 while (page < end) {
214 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
215 unsigned long num = next - base;
216 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
217 if (found < num) {
218 dirty = false;
219 break;
220 }
221
222 page = next;
223 idx++;
224 offset = 0;
225 base += DIRTY_MEMORY_BLOCK_SIZE;
226 }
227
228 return dirty;
229 }
230
231 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
232 unsigned client)
233 {
234 return cpu_physical_memory_get_dirty(addr, 1, client);
235 }
236
237 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
238 {
239 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
240 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
241 bool migration =
242 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
243 return !(vga && code && migration);
244 }
245
246 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
247 ram_addr_t length,
248 uint8_t mask)
249 {
250 uint8_t ret = 0;
251
252 if (mask & (1 << DIRTY_MEMORY_VGA) &&
253 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
254 ret |= (1 << DIRTY_MEMORY_VGA);
255 }
256 if (mask & (1 << DIRTY_MEMORY_CODE) &&
257 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
258 ret |= (1 << DIRTY_MEMORY_CODE);
259 }
260 if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
261 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
262 ret |= (1 << DIRTY_MEMORY_MIGRATION);
263 }
264 return ret;
265 }
266
267 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
268 unsigned client)
269 {
270 unsigned long page, idx, offset;
271 DirtyMemoryBlocks *blocks;
272
273 assert(client < DIRTY_MEMORY_NUM);
274
275 page = addr >> TARGET_PAGE_BITS;
276 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
277 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
278
279 RCU_READ_LOCK_GUARD();
280
281 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
282
283 set_bit_atomic(offset, blocks->blocks[idx]);
284 }
285
286 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
287 ram_addr_t length,
288 uint8_t mask)
289 {
290 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
291 unsigned long end, page;
292 unsigned long idx, offset, base;
293 int i;
294
295 if (!mask && !xen_enabled()) {
296 return;
297 }
298
299 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
300 page = start >> TARGET_PAGE_BITS;
301
302 WITH_RCU_READ_LOCK_GUARD() {
303 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
304 blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
305 }
306
307 idx = page / DIRTY_MEMORY_BLOCK_SIZE;
308 offset = page % DIRTY_MEMORY_BLOCK_SIZE;
309 base = page - offset;
310 while (page < end) {
311 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
312
313 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
314 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
315 offset, next - page);
316 }
317 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
318 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
319 offset, next - page);
320 }
321 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
322 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
323 offset, next - page);
324 }
325
326 page = next;
327 idx++;
328 offset = 0;
329 base += DIRTY_MEMORY_BLOCK_SIZE;
330 }
331 }
332
333 xen_hvm_modified_memory(start, length);
334 }
335
336 #if !defined(_WIN32)
337 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
338 ram_addr_t start,
339 ram_addr_t pages)
340 {
341 unsigned long i, j;
342 unsigned long page_number, c;
343 hwaddr addr;
344 ram_addr_t ram_addr;
345 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
346 unsigned long hpratio = qemu_real_host_page_size / TARGET_PAGE_SIZE;
347 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
348
349 /* start address is aligned at the start of a word? */
350 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
351 (hpratio == 1)) {
352 unsigned long **blocks[DIRTY_MEMORY_NUM];
353 unsigned long idx;
354 unsigned long offset;
355 long k;
356 long nr = BITS_TO_LONGS(pages);
357
358 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
359 offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
360 DIRTY_MEMORY_BLOCK_SIZE);
361
362 WITH_RCU_READ_LOCK_GUARD() {
363 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
364 blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
365 }
366
367 for (k = 0; k < nr; k++) {
368 if (bitmap[k]) {
369 unsigned long temp = leul_to_cpu(bitmap[k]);
370
371 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
372
373 if (global_dirty_log) {
374 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset],
375 temp);
376 }
377
378 if (tcg_enabled()) {
379 atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset],
380 temp);
381 }
382 }
383
384 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
385 offset = 0;
386 idx++;
387 }
388 }
389 }
390
391 xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS);
392 } else {
393 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
394
395 if (!global_dirty_log) {
396 clients &= ~(1 << DIRTY_MEMORY_MIGRATION);
397 }
398
399 /*
400 * bitmap-traveling is faster than memory-traveling (for addr...)
401 * especially when most of the memory is not dirty.
402 */
403 for (i = 0; i < len; i++) {
404 if (bitmap[i] != 0) {
405 c = leul_to_cpu(bitmap[i]);
406 do {
407 j = ctzl(c);
408 c &= ~(1ul << j);
409 page_number = (i * HOST_LONG_BITS + j) * hpratio;
410 addr = page_number * TARGET_PAGE_SIZE;
411 ram_addr = start + addr;
412 cpu_physical_memory_set_dirty_range(ram_addr,
413 TARGET_PAGE_SIZE * hpratio, clients);
414 } while (c != 0);
415 }
416 }
417 }
418 }
419 #endif /* not _WIN32 */
420
421 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
422 ram_addr_t length,
423 unsigned client);
424
425 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
426 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client);
427
428 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
429 ram_addr_t start,
430 ram_addr_t length);
431
432 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
433 ram_addr_t length)
434 {
435 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
436 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
437 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
438 }
439
440
441 /* Called with RCU critical section */
442 static inline
443 uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb,
444 ram_addr_t start,
445 ram_addr_t length,
446 uint64_t *real_dirty_pages)
447 {
448 ram_addr_t addr;
449 unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS);
450 uint64_t num_dirty = 0;
451 unsigned long *dest = rb->bmap;
452
453 /* start address and length is aligned at the start of a word? */
454 if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) ==
455 (start + rb->offset) &&
456 !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) {
457 int k;
458 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
459 unsigned long * const *src;
460 unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
461 unsigned long offset = BIT_WORD((word * BITS_PER_LONG) %
462 DIRTY_MEMORY_BLOCK_SIZE);
463 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
464
465 src = atomic_rcu_read(
466 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
467
468 for (k = page; k < page + nr; k++) {
469 if (src[idx][offset]) {
470 unsigned long bits = atomic_xchg(&src[idx][offset], 0);
471 unsigned long new_dirty;
472 *real_dirty_pages += ctpopl(bits);
473 new_dirty = ~dest[k];
474 dest[k] |= bits;
475 new_dirty &= bits;
476 num_dirty += ctpopl(new_dirty);
477 }
478
479 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
480 offset = 0;
481 idx++;
482 }
483 }
484
485 if (rb->clear_bmap) {
486 /*
487 * Postpone the dirty bitmap clear to the point before we
488 * really send the pages, also we will split the clear
489 * dirty procedure into smaller chunks.
490 */
491 clear_bmap_set(rb, start >> TARGET_PAGE_BITS,
492 length >> TARGET_PAGE_BITS);
493 } else {
494 /* Slow path - still do that in a huge chunk */
495 memory_region_clear_dirty_bitmap(rb->mr, start, length);
496 }
497 } else {
498 ram_addr_t offset = rb->offset;
499
500 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
501 if (cpu_physical_memory_test_and_clear_dirty(
502 start + addr + offset,
503 TARGET_PAGE_SIZE,
504 DIRTY_MEMORY_MIGRATION)) {
505 *real_dirty_pages += 1;
506 long k = (start + addr) >> TARGET_PAGE_BITS;
507 if (!test_and_set_bit(k, dest)) {
508 num_dirty++;
509 }
510 }
511 }
512 }
513
514 return num_dirty;
515 }
516 #endif
517 #endif