cpu: Move breakpoints field from CPU_COMMON to CPUState
[qemu.git] / exec.c
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifndef _WIN32
21 #include <sys/types.h>
22 #include <sys/mman.h>
23 #endif
24
25 #include "qemu-common.h"
26 #include "cpu.h"
27 #include "tcg.h"
28 #include "hw/hw.h"
29 #include "hw/qdev.h"
30 #include "qemu/osdep.h"
31 #include "sysemu/kvm.h"
32 #include "sysemu/sysemu.h"
33 #include "hw/xen/xen.h"
34 #include "qemu/timer.h"
35 #include "qemu/config-file.h"
36 #include "exec/memory.h"
37 #include "sysemu/dma.h"
38 #include "exec/address-spaces.h"
39 #if defined(CONFIG_USER_ONLY)
40 #include <qemu.h>
41 #else /* !CONFIG_USER_ONLY */
42 #include "sysemu/xen-mapcache.h"
43 #include "trace.h"
44 #endif
45 #include "exec/cpu-all.h"
46
47 #include "exec/cputlb.h"
48 #include "translate-all.h"
49
50 #include "exec/memory-internal.h"
51 #include "exec/ram_addr.h"
52 #include "qemu/cache-utils.h"
53
54 #include "qemu/range.h"
55
56 //#define DEBUG_SUBPAGE
57
58 #if !defined(CONFIG_USER_ONLY)
59 static bool in_migration;
60
61 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
62
63 static MemoryRegion *system_memory;
64 static MemoryRegion *system_io;
65
66 AddressSpace address_space_io;
67 AddressSpace address_space_memory;
68
69 MemoryRegion io_mem_rom, io_mem_notdirty;
70 static MemoryRegion io_mem_unassigned;
71
72 #endif
73
74 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
75 /* current CPU in the current thread. It is only valid inside
76 cpu_exec() */
77 DEFINE_TLS(CPUState *, current_cpu);
78 /* 0 = Do not count executed instructions.
79 1 = Precise instruction counting.
80 2 = Adaptive rate instruction counting. */
81 int use_icount;
82
83 #if !defined(CONFIG_USER_ONLY)
84
85 typedef struct PhysPageEntry PhysPageEntry;
86
87 struct PhysPageEntry {
88 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
89 uint32_t skip : 6;
90 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
91 uint32_t ptr : 26;
92 };
93
94 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
95
96 /* Size of the L2 (and L3, etc) page tables. */
97 #define ADDR_SPACE_BITS 64
98
99 #define P_L2_BITS 9
100 #define P_L2_SIZE (1 << P_L2_BITS)
101
102 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
103
104 typedef PhysPageEntry Node[P_L2_SIZE];
105
106 typedef struct PhysPageMap {
107 unsigned sections_nb;
108 unsigned sections_nb_alloc;
109 unsigned nodes_nb;
110 unsigned nodes_nb_alloc;
111 Node *nodes;
112 MemoryRegionSection *sections;
113 } PhysPageMap;
114
115 struct AddressSpaceDispatch {
116 /* This is a multi-level map on the physical address space.
117 * The bottom level has pointers to MemoryRegionSections.
118 */
119 PhysPageEntry phys_map;
120 PhysPageMap map;
121 AddressSpace *as;
122 };
123
124 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
125 typedef struct subpage_t {
126 MemoryRegion iomem;
127 AddressSpace *as;
128 hwaddr base;
129 uint16_t sub_section[TARGET_PAGE_SIZE];
130 } subpage_t;
131
132 #define PHYS_SECTION_UNASSIGNED 0
133 #define PHYS_SECTION_NOTDIRTY 1
134 #define PHYS_SECTION_ROM 2
135 #define PHYS_SECTION_WATCH 3
136
137 static void io_mem_init(void);
138 static void memory_map_init(void);
139 static void tcg_commit(MemoryListener *listener);
140
141 static MemoryRegion io_mem_watch;
142 #endif
143
144 #if !defined(CONFIG_USER_ONLY)
145
146 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
147 {
148 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
149 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
150 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
151 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
152 }
153 }
154
155 static uint32_t phys_map_node_alloc(PhysPageMap *map)
156 {
157 unsigned i;
158 uint32_t ret;
159
160 ret = map->nodes_nb++;
161 assert(ret != PHYS_MAP_NODE_NIL);
162 assert(ret != map->nodes_nb_alloc);
163 for (i = 0; i < P_L2_SIZE; ++i) {
164 map->nodes[ret][i].skip = 1;
165 map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
166 }
167 return ret;
168 }
169
170 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
171 hwaddr *index, hwaddr *nb, uint16_t leaf,
172 int level)
173 {
174 PhysPageEntry *p;
175 int i;
176 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
177
178 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
179 lp->ptr = phys_map_node_alloc(map);
180 p = map->nodes[lp->ptr];
181 if (level == 0) {
182 for (i = 0; i < P_L2_SIZE; i++) {
183 p[i].skip = 0;
184 p[i].ptr = PHYS_SECTION_UNASSIGNED;
185 }
186 }
187 } else {
188 p = map->nodes[lp->ptr];
189 }
190 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
191
192 while (*nb && lp < &p[P_L2_SIZE]) {
193 if ((*index & (step - 1)) == 0 && *nb >= step) {
194 lp->skip = 0;
195 lp->ptr = leaf;
196 *index += step;
197 *nb -= step;
198 } else {
199 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
200 }
201 ++lp;
202 }
203 }
204
205 static void phys_page_set(AddressSpaceDispatch *d,
206 hwaddr index, hwaddr nb,
207 uint16_t leaf)
208 {
209 /* Wildly overreserve - it doesn't matter much. */
210 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
211
212 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
213 }
214
215 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
216 * and update our entry so we can skip it and go directly to the destination.
217 */
218 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
219 {
220 unsigned valid_ptr = P_L2_SIZE;
221 int valid = 0;
222 PhysPageEntry *p;
223 int i;
224
225 if (lp->ptr == PHYS_MAP_NODE_NIL) {
226 return;
227 }
228
229 p = nodes[lp->ptr];
230 for (i = 0; i < P_L2_SIZE; i++) {
231 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
232 continue;
233 }
234
235 valid_ptr = i;
236 valid++;
237 if (p[i].skip) {
238 phys_page_compact(&p[i], nodes, compacted);
239 }
240 }
241
242 /* We can only compress if there's only one child. */
243 if (valid != 1) {
244 return;
245 }
246
247 assert(valid_ptr < P_L2_SIZE);
248
249 /* Don't compress if it won't fit in the # of bits we have. */
250 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
251 return;
252 }
253
254 lp->ptr = p[valid_ptr].ptr;
255 if (!p[valid_ptr].skip) {
256 /* If our only child is a leaf, make this a leaf. */
257 /* By design, we should have made this node a leaf to begin with so we
258 * should never reach here.
259 * But since it's so simple to handle this, let's do it just in case we
260 * change this rule.
261 */
262 lp->skip = 0;
263 } else {
264 lp->skip += p[valid_ptr].skip;
265 }
266 }
267
268 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
269 {
270 DECLARE_BITMAP(compacted, nodes_nb);
271
272 if (d->phys_map.skip) {
273 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
274 }
275 }
276
277 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
278 Node *nodes, MemoryRegionSection *sections)
279 {
280 PhysPageEntry *p;
281 hwaddr index = addr >> TARGET_PAGE_BITS;
282 int i;
283
284 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
285 if (lp.ptr == PHYS_MAP_NODE_NIL) {
286 return &sections[PHYS_SECTION_UNASSIGNED];
287 }
288 p = nodes[lp.ptr];
289 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
290 }
291
292 if (sections[lp.ptr].size.hi ||
293 range_covers_byte(sections[lp.ptr].offset_within_address_space,
294 sections[lp.ptr].size.lo, addr)) {
295 return &sections[lp.ptr];
296 } else {
297 return &sections[PHYS_SECTION_UNASSIGNED];
298 }
299 }
300
301 bool memory_region_is_unassigned(MemoryRegion *mr)
302 {
303 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
304 && mr != &io_mem_watch;
305 }
306
307 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
308 hwaddr addr,
309 bool resolve_subpage)
310 {
311 MemoryRegionSection *section;
312 subpage_t *subpage;
313
314 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
315 if (resolve_subpage && section->mr->subpage) {
316 subpage = container_of(section->mr, subpage_t, iomem);
317 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
318 }
319 return section;
320 }
321
322 static MemoryRegionSection *
323 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
324 hwaddr *plen, bool resolve_subpage)
325 {
326 MemoryRegionSection *section;
327 Int128 diff;
328
329 section = address_space_lookup_region(d, addr, resolve_subpage);
330 /* Compute offset within MemoryRegionSection */
331 addr -= section->offset_within_address_space;
332
333 /* Compute offset within MemoryRegion */
334 *xlat = addr + section->offset_within_region;
335
336 diff = int128_sub(section->mr->size, int128_make64(addr));
337 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
338 return section;
339 }
340
341 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
342 {
343 if (memory_region_is_ram(mr)) {
344 return !(is_write && mr->readonly);
345 }
346 if (memory_region_is_romd(mr)) {
347 return !is_write;
348 }
349
350 return false;
351 }
352
353 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
354 hwaddr *xlat, hwaddr *plen,
355 bool is_write)
356 {
357 IOMMUTLBEntry iotlb;
358 MemoryRegionSection *section;
359 MemoryRegion *mr;
360 hwaddr len = *plen;
361
362 for (;;) {
363 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
364 mr = section->mr;
365
366 if (!mr->iommu_ops) {
367 break;
368 }
369
370 iotlb = mr->iommu_ops->translate(mr, addr);
371 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
372 | (addr & iotlb.addr_mask));
373 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
374 if (!(iotlb.perm & (1 << is_write))) {
375 mr = &io_mem_unassigned;
376 break;
377 }
378
379 as = iotlb.target_as;
380 }
381
382 if (memory_access_is_direct(mr, is_write)) {
383 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
384 len = MIN(page, len);
385 }
386
387 *plen = len;
388 *xlat = addr;
389 return mr;
390 }
391
392 MemoryRegionSection *
393 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
394 hwaddr *plen)
395 {
396 MemoryRegionSection *section;
397 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
398
399 assert(!section->mr->iommu_ops);
400 return section;
401 }
402 #endif
403
404 void cpu_exec_init_all(void)
405 {
406 #if !defined(CONFIG_USER_ONLY)
407 qemu_mutex_init(&ram_list.mutex);
408 memory_map_init();
409 io_mem_init();
410 #endif
411 }
412
413 #if !defined(CONFIG_USER_ONLY)
414
415 static int cpu_common_post_load(void *opaque, int version_id)
416 {
417 CPUState *cpu = opaque;
418
419 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
420 version_id is increased. */
421 cpu->interrupt_request &= ~0x01;
422 tlb_flush(cpu->env_ptr, 1);
423
424 return 0;
425 }
426
427 const VMStateDescription vmstate_cpu_common = {
428 .name = "cpu_common",
429 .version_id = 1,
430 .minimum_version_id = 1,
431 .minimum_version_id_old = 1,
432 .post_load = cpu_common_post_load,
433 .fields = (VMStateField []) {
434 VMSTATE_UINT32(halted, CPUState),
435 VMSTATE_UINT32(interrupt_request, CPUState),
436 VMSTATE_END_OF_LIST()
437 }
438 };
439
440 #endif
441
442 CPUState *qemu_get_cpu(int index)
443 {
444 CPUState *cpu;
445
446 CPU_FOREACH(cpu) {
447 if (cpu->cpu_index == index) {
448 return cpu;
449 }
450 }
451
452 return NULL;
453 }
454
455 #if !defined(CONFIG_USER_ONLY)
456 void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
457 {
458 /* We only support one address space per cpu at the moment. */
459 assert(cpu->as == as);
460
461 if (cpu->tcg_as_listener) {
462 memory_listener_unregister(cpu->tcg_as_listener);
463 } else {
464 cpu->tcg_as_listener = g_new0(MemoryListener, 1);
465 }
466 cpu->tcg_as_listener->commit = tcg_commit;
467 memory_listener_register(cpu->tcg_as_listener, as);
468 }
469 #endif
470
471 void cpu_exec_init(CPUArchState *env)
472 {
473 CPUState *cpu = ENV_GET_CPU(env);
474 CPUClass *cc = CPU_GET_CLASS(cpu);
475 CPUState *some_cpu;
476 int cpu_index;
477
478 #if defined(CONFIG_USER_ONLY)
479 cpu_list_lock();
480 #endif
481 cpu_index = 0;
482 CPU_FOREACH(some_cpu) {
483 cpu_index++;
484 }
485 cpu->cpu_index = cpu_index;
486 cpu->numa_node = 0;
487 QTAILQ_INIT(&cpu->breakpoints);
488 QTAILQ_INIT(&cpu->watchpoints);
489 #ifndef CONFIG_USER_ONLY
490 cpu->as = &address_space_memory;
491 cpu->thread_id = qemu_get_thread_id();
492 #endif
493 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
494 #if defined(CONFIG_USER_ONLY)
495 cpu_list_unlock();
496 #endif
497 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
498 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
499 }
500 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
501 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
502 cpu_save, cpu_load, env);
503 assert(cc->vmsd == NULL);
504 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
505 #endif
506 if (cc->vmsd != NULL) {
507 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
508 }
509 }
510
511 #if defined(TARGET_HAS_ICE)
512 #if defined(CONFIG_USER_ONLY)
513 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
514 {
515 tb_invalidate_phys_page_range(pc, pc + 1, 0);
516 }
517 #else
518 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
519 {
520 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
521 if (phys != -1) {
522 tb_invalidate_phys_addr(cpu->as,
523 phys | (pc & ~TARGET_PAGE_MASK));
524 }
525 }
526 #endif
527 #endif /* TARGET_HAS_ICE */
528
529 #if defined(CONFIG_USER_ONLY)
530 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
531
532 {
533 }
534
535 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
536 int flags, CPUWatchpoint **watchpoint)
537 {
538 return -ENOSYS;
539 }
540 #else
541 /* Add a watchpoint. */
542 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
543 int flags, CPUWatchpoint **watchpoint)
544 {
545 CPUState *cpu = ENV_GET_CPU(env);
546 target_ulong len_mask = ~(len - 1);
547 CPUWatchpoint *wp;
548
549 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
550 if ((len & (len - 1)) || (addr & ~len_mask) ||
551 len == 0 || len > TARGET_PAGE_SIZE) {
552 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
553 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
554 return -EINVAL;
555 }
556 wp = g_malloc(sizeof(*wp));
557
558 wp->vaddr = addr;
559 wp->len_mask = len_mask;
560 wp->flags = flags;
561
562 /* keep all GDB-injected watchpoints in front */
563 if (flags & BP_GDB) {
564 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
565 } else {
566 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
567 }
568
569 tlb_flush_page(env, addr);
570
571 if (watchpoint)
572 *watchpoint = wp;
573 return 0;
574 }
575
576 /* Remove a specific watchpoint. */
577 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
578 int flags)
579 {
580 CPUState *cpu = ENV_GET_CPU(env);
581 target_ulong len_mask = ~(len - 1);
582 CPUWatchpoint *wp;
583
584 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
585 if (addr == wp->vaddr && len_mask == wp->len_mask
586 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
587 cpu_watchpoint_remove_by_ref(env, wp);
588 return 0;
589 }
590 }
591 return -ENOENT;
592 }
593
594 /* Remove a specific watchpoint by reference. */
595 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
596 {
597 CPUState *cpu = ENV_GET_CPU(env);
598
599 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
600
601 tlb_flush_page(env, watchpoint->vaddr);
602
603 g_free(watchpoint);
604 }
605
606 /* Remove all matching watchpoints. */
607 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
608 {
609 CPUState *cpu = ENV_GET_CPU(env);
610 CPUWatchpoint *wp, *next;
611
612 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
613 if (wp->flags & mask)
614 cpu_watchpoint_remove_by_ref(env, wp);
615 }
616 }
617 #endif
618
619 /* Add a breakpoint. */
620 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
621 CPUBreakpoint **breakpoint)
622 {
623 #if defined(TARGET_HAS_ICE)
624 CPUState *cpu = ENV_GET_CPU(env);
625 CPUBreakpoint *bp;
626
627 bp = g_malloc(sizeof(*bp));
628
629 bp->pc = pc;
630 bp->flags = flags;
631
632 /* keep all GDB-injected breakpoints in front */
633 if (flags & BP_GDB) {
634 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
635 } else {
636 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
637 }
638
639 breakpoint_invalidate(cpu, pc);
640
641 if (breakpoint) {
642 *breakpoint = bp;
643 }
644 return 0;
645 #else
646 return -ENOSYS;
647 #endif
648 }
649
650 /* Remove a specific breakpoint. */
651 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
652 {
653 #if defined(TARGET_HAS_ICE)
654 CPUState *cpu = ENV_GET_CPU(env);
655 CPUBreakpoint *bp;
656
657 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
658 if (bp->pc == pc && bp->flags == flags) {
659 cpu_breakpoint_remove_by_ref(env, bp);
660 return 0;
661 }
662 }
663 return -ENOENT;
664 #else
665 return -ENOSYS;
666 #endif
667 }
668
669 /* Remove a specific breakpoint by reference. */
670 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
671 {
672 #if defined(TARGET_HAS_ICE)
673 CPUState *cpu = ENV_GET_CPU(env);
674
675 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
676
677 breakpoint_invalidate(cpu, breakpoint->pc);
678
679 g_free(breakpoint);
680 #endif
681 }
682
683 /* Remove all matching breakpoints. */
684 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
685 {
686 #if defined(TARGET_HAS_ICE)
687 CPUState *cpu = ENV_GET_CPU(env);
688 CPUBreakpoint *bp, *next;
689
690 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
691 if (bp->flags & mask)
692 cpu_breakpoint_remove_by_ref(env, bp);
693 }
694 #endif
695 }
696
697 /* enable or disable single step mode. EXCP_DEBUG is returned by the
698 CPU loop after each instruction */
699 void cpu_single_step(CPUState *cpu, int enabled)
700 {
701 #if defined(TARGET_HAS_ICE)
702 if (cpu->singlestep_enabled != enabled) {
703 cpu->singlestep_enabled = enabled;
704 if (kvm_enabled()) {
705 kvm_update_guest_debug(cpu, 0);
706 } else {
707 /* must flush all the translated code to avoid inconsistencies */
708 /* XXX: only flush what is necessary */
709 CPUArchState *env = cpu->env_ptr;
710 tb_flush(env);
711 }
712 }
713 #endif
714 }
715
716 void cpu_abort(CPUArchState *env, const char *fmt, ...)
717 {
718 CPUState *cpu = ENV_GET_CPU(env);
719 va_list ap;
720 va_list ap2;
721
722 va_start(ap, fmt);
723 va_copy(ap2, ap);
724 fprintf(stderr, "qemu: fatal: ");
725 vfprintf(stderr, fmt, ap);
726 fprintf(stderr, "\n");
727 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
728 if (qemu_log_enabled()) {
729 qemu_log("qemu: fatal: ");
730 qemu_log_vprintf(fmt, ap2);
731 qemu_log("\n");
732 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
733 qemu_log_flush();
734 qemu_log_close();
735 }
736 va_end(ap2);
737 va_end(ap);
738 #if defined(CONFIG_USER_ONLY)
739 {
740 struct sigaction act;
741 sigfillset(&act.sa_mask);
742 act.sa_handler = SIG_DFL;
743 sigaction(SIGABRT, &act, NULL);
744 }
745 #endif
746 abort();
747 }
748
749 #if !defined(CONFIG_USER_ONLY)
750 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
751 {
752 RAMBlock *block;
753
754 /* The list is protected by the iothread lock here. */
755 block = ram_list.mru_block;
756 if (block && addr - block->offset < block->length) {
757 goto found;
758 }
759 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
760 if (addr - block->offset < block->length) {
761 goto found;
762 }
763 }
764
765 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
766 abort();
767
768 found:
769 ram_list.mru_block = block;
770 return block;
771 }
772
773 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
774 {
775 ram_addr_t start1;
776 RAMBlock *block;
777 ram_addr_t end;
778
779 end = TARGET_PAGE_ALIGN(start + length);
780 start &= TARGET_PAGE_MASK;
781
782 block = qemu_get_ram_block(start);
783 assert(block == qemu_get_ram_block(end - 1));
784 start1 = (uintptr_t)block->host + (start - block->offset);
785 cpu_tlb_reset_dirty_all(start1, length);
786 }
787
788 /* Note: start and end must be within the same ram block. */
789 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
790 unsigned client)
791 {
792 if (length == 0)
793 return;
794 cpu_physical_memory_clear_dirty_range(start, length, client);
795
796 if (tcg_enabled()) {
797 tlb_reset_dirty_range_all(start, length);
798 }
799 }
800
801 static void cpu_physical_memory_set_dirty_tracking(bool enable)
802 {
803 in_migration = enable;
804 }
805
806 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
807 MemoryRegionSection *section,
808 target_ulong vaddr,
809 hwaddr paddr, hwaddr xlat,
810 int prot,
811 target_ulong *address)
812 {
813 CPUState *cpu = ENV_GET_CPU(env);
814 hwaddr iotlb;
815 CPUWatchpoint *wp;
816
817 if (memory_region_is_ram(section->mr)) {
818 /* Normal RAM. */
819 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
820 + xlat;
821 if (!section->readonly) {
822 iotlb |= PHYS_SECTION_NOTDIRTY;
823 } else {
824 iotlb |= PHYS_SECTION_ROM;
825 }
826 } else {
827 iotlb = section - section->address_space->dispatch->map.sections;
828 iotlb += xlat;
829 }
830
831 /* Make accesses to pages with watchpoints go via the
832 watchpoint trap routines. */
833 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
834 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
835 /* Avoid trapping reads of pages with a write breakpoint. */
836 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
837 iotlb = PHYS_SECTION_WATCH + paddr;
838 *address |= TLB_MMIO;
839 break;
840 }
841 }
842 }
843
844 return iotlb;
845 }
846 #endif /* defined(CONFIG_USER_ONLY) */
847
848 #if !defined(CONFIG_USER_ONLY)
849
850 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
851 uint16_t section);
852 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
853
854 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
855
856 /*
857 * Set a custom physical guest memory alloator.
858 * Accelerators with unusual needs may need this. Hopefully, we can
859 * get rid of it eventually.
860 */
861 void phys_mem_set_alloc(void *(*alloc)(size_t))
862 {
863 phys_mem_alloc = alloc;
864 }
865
866 static uint16_t phys_section_add(PhysPageMap *map,
867 MemoryRegionSection *section)
868 {
869 /* The physical section number is ORed with a page-aligned
870 * pointer to produce the iotlb entries. Thus it should
871 * never overflow into the page-aligned value.
872 */
873 assert(map->sections_nb < TARGET_PAGE_SIZE);
874
875 if (map->sections_nb == map->sections_nb_alloc) {
876 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
877 map->sections = g_renew(MemoryRegionSection, map->sections,
878 map->sections_nb_alloc);
879 }
880 map->sections[map->sections_nb] = *section;
881 memory_region_ref(section->mr);
882 return map->sections_nb++;
883 }
884
885 static void phys_section_destroy(MemoryRegion *mr)
886 {
887 memory_region_unref(mr);
888
889 if (mr->subpage) {
890 subpage_t *subpage = container_of(mr, subpage_t, iomem);
891 memory_region_destroy(&subpage->iomem);
892 g_free(subpage);
893 }
894 }
895
896 static void phys_sections_free(PhysPageMap *map)
897 {
898 while (map->sections_nb > 0) {
899 MemoryRegionSection *section = &map->sections[--map->sections_nb];
900 phys_section_destroy(section->mr);
901 }
902 g_free(map->sections);
903 g_free(map->nodes);
904 }
905
906 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
907 {
908 subpage_t *subpage;
909 hwaddr base = section->offset_within_address_space
910 & TARGET_PAGE_MASK;
911 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
912 d->map.nodes, d->map.sections);
913 MemoryRegionSection subsection = {
914 .offset_within_address_space = base,
915 .size = int128_make64(TARGET_PAGE_SIZE),
916 };
917 hwaddr start, end;
918
919 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
920
921 if (!(existing->mr->subpage)) {
922 subpage = subpage_init(d->as, base);
923 subsection.address_space = d->as;
924 subsection.mr = &subpage->iomem;
925 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
926 phys_section_add(&d->map, &subsection));
927 } else {
928 subpage = container_of(existing->mr, subpage_t, iomem);
929 }
930 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
931 end = start + int128_get64(section->size) - 1;
932 subpage_register(subpage, start, end,
933 phys_section_add(&d->map, section));
934 }
935
936
937 static void register_multipage(AddressSpaceDispatch *d,
938 MemoryRegionSection *section)
939 {
940 hwaddr start_addr = section->offset_within_address_space;
941 uint16_t section_index = phys_section_add(&d->map, section);
942 uint64_t num_pages = int128_get64(int128_rshift(section->size,
943 TARGET_PAGE_BITS));
944
945 assert(num_pages);
946 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
947 }
948
949 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
950 {
951 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
952 AddressSpaceDispatch *d = as->next_dispatch;
953 MemoryRegionSection now = *section, remain = *section;
954 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
955
956 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
957 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
958 - now.offset_within_address_space;
959
960 now.size = int128_min(int128_make64(left), now.size);
961 register_subpage(d, &now);
962 } else {
963 now.size = int128_zero();
964 }
965 while (int128_ne(remain.size, now.size)) {
966 remain.size = int128_sub(remain.size, now.size);
967 remain.offset_within_address_space += int128_get64(now.size);
968 remain.offset_within_region += int128_get64(now.size);
969 now = remain;
970 if (int128_lt(remain.size, page_size)) {
971 register_subpage(d, &now);
972 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
973 now.size = page_size;
974 register_subpage(d, &now);
975 } else {
976 now.size = int128_and(now.size, int128_neg(page_size));
977 register_multipage(d, &now);
978 }
979 }
980 }
981
982 void qemu_flush_coalesced_mmio_buffer(void)
983 {
984 if (kvm_enabled())
985 kvm_flush_coalesced_mmio_buffer();
986 }
987
988 void qemu_mutex_lock_ramlist(void)
989 {
990 qemu_mutex_lock(&ram_list.mutex);
991 }
992
993 void qemu_mutex_unlock_ramlist(void)
994 {
995 qemu_mutex_unlock(&ram_list.mutex);
996 }
997
998 #ifdef __linux__
999
1000 #include <sys/vfs.h>
1001
1002 #define HUGETLBFS_MAGIC 0x958458f6
1003
1004 static long gethugepagesize(const char *path)
1005 {
1006 struct statfs fs;
1007 int ret;
1008
1009 do {
1010 ret = statfs(path, &fs);
1011 } while (ret != 0 && errno == EINTR);
1012
1013 if (ret != 0) {
1014 perror(path);
1015 return 0;
1016 }
1017
1018 if (fs.f_type != HUGETLBFS_MAGIC)
1019 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
1020
1021 return fs.f_bsize;
1022 }
1023
1024 static sigjmp_buf sigjump;
1025
1026 static void sigbus_handler(int signal)
1027 {
1028 siglongjmp(sigjump, 1);
1029 }
1030
1031 static void *file_ram_alloc(RAMBlock *block,
1032 ram_addr_t memory,
1033 const char *path)
1034 {
1035 char *filename;
1036 char *sanitized_name;
1037 char *c;
1038 void *area;
1039 int fd;
1040 unsigned long hpagesize;
1041
1042 hpagesize = gethugepagesize(path);
1043 if (!hpagesize) {
1044 goto error;
1045 }
1046
1047 if (memory < hpagesize) {
1048 return NULL;
1049 }
1050
1051 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1052 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
1053 goto error;
1054 }
1055
1056 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1057 sanitized_name = g_strdup(block->mr->name);
1058 for (c = sanitized_name; *c != '\0'; c++) {
1059 if (*c == '/')
1060 *c = '_';
1061 }
1062
1063 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1064 sanitized_name);
1065 g_free(sanitized_name);
1066
1067 fd = mkstemp(filename);
1068 if (fd < 0) {
1069 perror("unable to create backing store for hugepages");
1070 g_free(filename);
1071 goto error;
1072 }
1073 unlink(filename);
1074 g_free(filename);
1075
1076 memory = (memory+hpagesize-1) & ~(hpagesize-1);
1077
1078 /*
1079 * ftruncate is not supported by hugetlbfs in older
1080 * hosts, so don't bother bailing out on errors.
1081 * If anything goes wrong with it under other filesystems,
1082 * mmap will fail.
1083 */
1084 if (ftruncate(fd, memory))
1085 perror("ftruncate");
1086
1087 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1088 if (area == MAP_FAILED) {
1089 perror("file_ram_alloc: can't mmap RAM pages");
1090 close(fd);
1091 goto error;
1092 }
1093
1094 if (mem_prealloc) {
1095 int ret, i;
1096 struct sigaction act, oldact;
1097 sigset_t set, oldset;
1098
1099 memset(&act, 0, sizeof(act));
1100 act.sa_handler = &sigbus_handler;
1101 act.sa_flags = 0;
1102
1103 ret = sigaction(SIGBUS, &act, &oldact);
1104 if (ret) {
1105 perror("file_ram_alloc: failed to install signal handler");
1106 exit(1);
1107 }
1108
1109 /* unblock SIGBUS */
1110 sigemptyset(&set);
1111 sigaddset(&set, SIGBUS);
1112 pthread_sigmask(SIG_UNBLOCK, &set, &oldset);
1113
1114 if (sigsetjmp(sigjump, 1)) {
1115 fprintf(stderr, "file_ram_alloc: failed to preallocate pages\n");
1116 exit(1);
1117 }
1118
1119 /* MAP_POPULATE silently ignores failures */
1120 for (i = 0; i < (memory/hpagesize); i++) {
1121 memset(area + (hpagesize*i), 0, 1);
1122 }
1123
1124 ret = sigaction(SIGBUS, &oldact, NULL);
1125 if (ret) {
1126 perror("file_ram_alloc: failed to reinstall signal handler");
1127 exit(1);
1128 }
1129
1130 pthread_sigmask(SIG_SETMASK, &oldset, NULL);
1131 }
1132
1133 block->fd = fd;
1134 return area;
1135
1136 error:
1137 if (mem_prealloc) {
1138 exit(1);
1139 }
1140 return NULL;
1141 }
1142 #else
1143 static void *file_ram_alloc(RAMBlock *block,
1144 ram_addr_t memory,
1145 const char *path)
1146 {
1147 fprintf(stderr, "-mem-path not supported on this host\n");
1148 exit(1);
1149 }
1150 #endif
1151
1152 static ram_addr_t find_ram_offset(ram_addr_t size)
1153 {
1154 RAMBlock *block, *next_block;
1155 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1156
1157 assert(size != 0); /* it would hand out same offset multiple times */
1158
1159 if (QTAILQ_EMPTY(&ram_list.blocks))
1160 return 0;
1161
1162 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1163 ram_addr_t end, next = RAM_ADDR_MAX;
1164
1165 end = block->offset + block->length;
1166
1167 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1168 if (next_block->offset >= end) {
1169 next = MIN(next, next_block->offset);
1170 }
1171 }
1172 if (next - end >= size && next - end < mingap) {
1173 offset = end;
1174 mingap = next - end;
1175 }
1176 }
1177
1178 if (offset == RAM_ADDR_MAX) {
1179 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1180 (uint64_t)size);
1181 abort();
1182 }
1183
1184 return offset;
1185 }
1186
1187 ram_addr_t last_ram_offset(void)
1188 {
1189 RAMBlock *block;
1190 ram_addr_t last = 0;
1191
1192 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1193 last = MAX(last, block->offset + block->length);
1194
1195 return last;
1196 }
1197
1198 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1199 {
1200 int ret;
1201
1202 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1203 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1204 "dump-guest-core", true)) {
1205 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1206 if (ret) {
1207 perror("qemu_madvise");
1208 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1209 "but dump_guest_core=off specified\n");
1210 }
1211 }
1212 }
1213
1214 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1215 {
1216 RAMBlock *new_block, *block;
1217
1218 new_block = NULL;
1219 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1220 if (block->offset == addr) {
1221 new_block = block;
1222 break;
1223 }
1224 }
1225 assert(new_block);
1226 assert(!new_block->idstr[0]);
1227
1228 if (dev) {
1229 char *id = qdev_get_dev_path(dev);
1230 if (id) {
1231 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1232 g_free(id);
1233 }
1234 }
1235 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1236
1237 /* This assumes the iothread lock is taken here too. */
1238 qemu_mutex_lock_ramlist();
1239 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1240 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1241 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1242 new_block->idstr);
1243 abort();
1244 }
1245 }
1246 qemu_mutex_unlock_ramlist();
1247 }
1248
1249 static int memory_try_enable_merging(void *addr, size_t len)
1250 {
1251 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1252 /* disabled by the user */
1253 return 0;
1254 }
1255
1256 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1257 }
1258
1259 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1260 MemoryRegion *mr)
1261 {
1262 RAMBlock *block, *new_block;
1263 ram_addr_t old_ram_size, new_ram_size;
1264
1265 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1266
1267 size = TARGET_PAGE_ALIGN(size);
1268 new_block = g_malloc0(sizeof(*new_block));
1269 new_block->fd = -1;
1270
1271 /* This assumes the iothread lock is taken here too. */
1272 qemu_mutex_lock_ramlist();
1273 new_block->mr = mr;
1274 new_block->offset = find_ram_offset(size);
1275 if (host) {
1276 new_block->host = host;
1277 new_block->flags |= RAM_PREALLOC_MASK;
1278 } else if (xen_enabled()) {
1279 if (mem_path) {
1280 fprintf(stderr, "-mem-path not supported with Xen\n");
1281 exit(1);
1282 }
1283 xen_ram_alloc(new_block->offset, size, mr);
1284 } else {
1285 if (mem_path) {
1286 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1287 /*
1288 * file_ram_alloc() needs to allocate just like
1289 * phys_mem_alloc, but we haven't bothered to provide
1290 * a hook there.
1291 */
1292 fprintf(stderr,
1293 "-mem-path not supported with this accelerator\n");
1294 exit(1);
1295 }
1296 new_block->host = file_ram_alloc(new_block, size, mem_path);
1297 }
1298 if (!new_block->host) {
1299 new_block->host = phys_mem_alloc(size);
1300 if (!new_block->host) {
1301 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1302 new_block->mr->name, strerror(errno));
1303 exit(1);
1304 }
1305 memory_try_enable_merging(new_block->host, size);
1306 }
1307 }
1308 new_block->length = size;
1309
1310 /* Keep the list sorted from biggest to smallest block. */
1311 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1312 if (block->length < new_block->length) {
1313 break;
1314 }
1315 }
1316 if (block) {
1317 QTAILQ_INSERT_BEFORE(block, new_block, next);
1318 } else {
1319 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1320 }
1321 ram_list.mru_block = NULL;
1322
1323 ram_list.version++;
1324 qemu_mutex_unlock_ramlist();
1325
1326 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1327
1328 if (new_ram_size > old_ram_size) {
1329 int i;
1330 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1331 ram_list.dirty_memory[i] =
1332 bitmap_zero_extend(ram_list.dirty_memory[i],
1333 old_ram_size, new_ram_size);
1334 }
1335 }
1336 cpu_physical_memory_set_dirty_range(new_block->offset, size);
1337
1338 qemu_ram_setup_dump(new_block->host, size);
1339 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1340 qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1341
1342 if (kvm_enabled())
1343 kvm_setup_guest_memory(new_block->host, size);
1344
1345 return new_block->offset;
1346 }
1347
1348 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1349 {
1350 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1351 }
1352
1353 void qemu_ram_free_from_ptr(ram_addr_t addr)
1354 {
1355 RAMBlock *block;
1356
1357 /* This assumes the iothread lock is taken here too. */
1358 qemu_mutex_lock_ramlist();
1359 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1360 if (addr == block->offset) {
1361 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1362 ram_list.mru_block = NULL;
1363 ram_list.version++;
1364 g_free(block);
1365 break;
1366 }
1367 }
1368 qemu_mutex_unlock_ramlist();
1369 }
1370
1371 void qemu_ram_free(ram_addr_t addr)
1372 {
1373 RAMBlock *block;
1374
1375 /* This assumes the iothread lock is taken here too. */
1376 qemu_mutex_lock_ramlist();
1377 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1378 if (addr == block->offset) {
1379 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1380 ram_list.mru_block = NULL;
1381 ram_list.version++;
1382 if (block->flags & RAM_PREALLOC_MASK) {
1383 ;
1384 } else if (xen_enabled()) {
1385 xen_invalidate_map_cache_entry(block->host);
1386 #ifndef _WIN32
1387 } else if (block->fd >= 0) {
1388 munmap(block->host, block->length);
1389 close(block->fd);
1390 #endif
1391 } else {
1392 qemu_anon_ram_free(block->host, block->length);
1393 }
1394 g_free(block);
1395 break;
1396 }
1397 }
1398 qemu_mutex_unlock_ramlist();
1399
1400 }
1401
1402 #ifndef _WIN32
1403 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1404 {
1405 RAMBlock *block;
1406 ram_addr_t offset;
1407 int flags;
1408 void *area, *vaddr;
1409
1410 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1411 offset = addr - block->offset;
1412 if (offset < block->length) {
1413 vaddr = block->host + offset;
1414 if (block->flags & RAM_PREALLOC_MASK) {
1415 ;
1416 } else if (xen_enabled()) {
1417 abort();
1418 } else {
1419 flags = MAP_FIXED;
1420 munmap(vaddr, length);
1421 if (block->fd >= 0) {
1422 #ifdef MAP_POPULATE
1423 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1424 MAP_PRIVATE;
1425 #else
1426 flags |= MAP_PRIVATE;
1427 #endif
1428 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1429 flags, block->fd, offset);
1430 } else {
1431 /*
1432 * Remap needs to match alloc. Accelerators that
1433 * set phys_mem_alloc never remap. If they did,
1434 * we'd need a remap hook here.
1435 */
1436 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1437
1438 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1439 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1440 flags, -1, 0);
1441 }
1442 if (area != vaddr) {
1443 fprintf(stderr, "Could not remap addr: "
1444 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1445 length, addr);
1446 exit(1);
1447 }
1448 memory_try_enable_merging(vaddr, length);
1449 qemu_ram_setup_dump(vaddr, length);
1450 }
1451 return;
1452 }
1453 }
1454 }
1455 #endif /* !_WIN32 */
1456
1457 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1458 With the exception of the softmmu code in this file, this should
1459 only be used for local memory (e.g. video ram) that the device owns,
1460 and knows it isn't going to access beyond the end of the block.
1461
1462 It should not be used for general purpose DMA.
1463 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1464 */
1465 void *qemu_get_ram_ptr(ram_addr_t addr)
1466 {
1467 RAMBlock *block = qemu_get_ram_block(addr);
1468
1469 if (xen_enabled()) {
1470 /* We need to check if the requested address is in the RAM
1471 * because we don't want to map the entire memory in QEMU.
1472 * In that case just map until the end of the page.
1473 */
1474 if (block->offset == 0) {
1475 return xen_map_cache(addr, 0, 0);
1476 } else if (block->host == NULL) {
1477 block->host =
1478 xen_map_cache(block->offset, block->length, 1);
1479 }
1480 }
1481 return block->host + (addr - block->offset);
1482 }
1483
1484 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1485 * but takes a size argument */
1486 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1487 {
1488 if (*size == 0) {
1489 return NULL;
1490 }
1491 if (xen_enabled()) {
1492 return xen_map_cache(addr, *size, 1);
1493 } else {
1494 RAMBlock *block;
1495
1496 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1497 if (addr - block->offset < block->length) {
1498 if (addr - block->offset + *size > block->length)
1499 *size = block->length - addr + block->offset;
1500 return block->host + (addr - block->offset);
1501 }
1502 }
1503
1504 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1505 abort();
1506 }
1507 }
1508
1509 /* Some of the softmmu routines need to translate from a host pointer
1510 (typically a TLB entry) back to a ram offset. */
1511 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1512 {
1513 RAMBlock *block;
1514 uint8_t *host = ptr;
1515
1516 if (xen_enabled()) {
1517 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1518 return qemu_get_ram_block(*ram_addr)->mr;
1519 }
1520
1521 block = ram_list.mru_block;
1522 if (block && block->host && host - block->host < block->length) {
1523 goto found;
1524 }
1525
1526 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1527 /* This case append when the block is not mapped. */
1528 if (block->host == NULL) {
1529 continue;
1530 }
1531 if (host - block->host < block->length) {
1532 goto found;
1533 }
1534 }
1535
1536 return NULL;
1537
1538 found:
1539 *ram_addr = block->offset + (host - block->host);
1540 return block->mr;
1541 }
1542
1543 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1544 uint64_t val, unsigned size)
1545 {
1546 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1547 tb_invalidate_phys_page_fast(ram_addr, size);
1548 }
1549 switch (size) {
1550 case 1:
1551 stb_p(qemu_get_ram_ptr(ram_addr), val);
1552 break;
1553 case 2:
1554 stw_p(qemu_get_ram_ptr(ram_addr), val);
1555 break;
1556 case 4:
1557 stl_p(qemu_get_ram_ptr(ram_addr), val);
1558 break;
1559 default:
1560 abort();
1561 }
1562 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
1563 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
1564 /* we remove the notdirty callback only if the code has been
1565 flushed */
1566 if (!cpu_physical_memory_is_clean(ram_addr)) {
1567 CPUArchState *env = current_cpu->env_ptr;
1568 tlb_set_dirty(env, current_cpu->mem_io_vaddr);
1569 }
1570 }
1571
1572 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1573 unsigned size, bool is_write)
1574 {
1575 return is_write;
1576 }
1577
1578 static const MemoryRegionOps notdirty_mem_ops = {
1579 .write = notdirty_mem_write,
1580 .valid.accepts = notdirty_mem_accepts,
1581 .endianness = DEVICE_NATIVE_ENDIAN,
1582 };
1583
1584 /* Generate a debug exception if a watchpoint has been hit. */
1585 static void check_watchpoint(int offset, int len_mask, int flags)
1586 {
1587 CPUState *cpu = current_cpu;
1588 CPUArchState *env = cpu->env_ptr;
1589 target_ulong pc, cs_base;
1590 target_ulong vaddr;
1591 CPUWatchpoint *wp;
1592 int cpu_flags;
1593
1594 if (cpu->watchpoint_hit) {
1595 /* We re-entered the check after replacing the TB. Now raise
1596 * the debug interrupt so that is will trigger after the
1597 * current instruction. */
1598 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
1599 return;
1600 }
1601 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1602 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1603 if ((vaddr == (wp->vaddr & len_mask) ||
1604 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1605 wp->flags |= BP_WATCHPOINT_HIT;
1606 if (!cpu->watchpoint_hit) {
1607 cpu->watchpoint_hit = wp;
1608 tb_check_watchpoint(env);
1609 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1610 cpu->exception_index = EXCP_DEBUG;
1611 cpu_loop_exit(env);
1612 } else {
1613 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1614 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1615 cpu_resume_from_signal(env, NULL);
1616 }
1617 }
1618 } else {
1619 wp->flags &= ~BP_WATCHPOINT_HIT;
1620 }
1621 }
1622 }
1623
1624 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1625 so these check for a hit then pass through to the normal out-of-line
1626 phys routines. */
1627 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1628 unsigned size)
1629 {
1630 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1631 switch (size) {
1632 case 1: return ldub_phys(&address_space_memory, addr);
1633 case 2: return lduw_phys(&address_space_memory, addr);
1634 case 4: return ldl_phys(&address_space_memory, addr);
1635 default: abort();
1636 }
1637 }
1638
1639 static void watch_mem_write(void *opaque, hwaddr addr,
1640 uint64_t val, unsigned size)
1641 {
1642 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1643 switch (size) {
1644 case 1:
1645 stb_phys(&address_space_memory, addr, val);
1646 break;
1647 case 2:
1648 stw_phys(&address_space_memory, addr, val);
1649 break;
1650 case 4:
1651 stl_phys(&address_space_memory, addr, val);
1652 break;
1653 default: abort();
1654 }
1655 }
1656
1657 static const MemoryRegionOps watch_mem_ops = {
1658 .read = watch_mem_read,
1659 .write = watch_mem_write,
1660 .endianness = DEVICE_NATIVE_ENDIAN,
1661 };
1662
1663 static uint64_t subpage_read(void *opaque, hwaddr addr,
1664 unsigned len)
1665 {
1666 subpage_t *subpage = opaque;
1667 uint8_t buf[4];
1668
1669 #if defined(DEBUG_SUBPAGE)
1670 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
1671 subpage, len, addr);
1672 #endif
1673 address_space_read(subpage->as, addr + subpage->base, buf, len);
1674 switch (len) {
1675 case 1:
1676 return ldub_p(buf);
1677 case 2:
1678 return lduw_p(buf);
1679 case 4:
1680 return ldl_p(buf);
1681 default:
1682 abort();
1683 }
1684 }
1685
1686 static void subpage_write(void *opaque, hwaddr addr,
1687 uint64_t value, unsigned len)
1688 {
1689 subpage_t *subpage = opaque;
1690 uint8_t buf[4];
1691
1692 #if defined(DEBUG_SUBPAGE)
1693 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1694 " value %"PRIx64"\n",
1695 __func__, subpage, len, addr, value);
1696 #endif
1697 switch (len) {
1698 case 1:
1699 stb_p(buf, value);
1700 break;
1701 case 2:
1702 stw_p(buf, value);
1703 break;
1704 case 4:
1705 stl_p(buf, value);
1706 break;
1707 default:
1708 abort();
1709 }
1710 address_space_write(subpage->as, addr + subpage->base, buf, len);
1711 }
1712
1713 static bool subpage_accepts(void *opaque, hwaddr addr,
1714 unsigned len, bool is_write)
1715 {
1716 subpage_t *subpage = opaque;
1717 #if defined(DEBUG_SUBPAGE)
1718 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
1719 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1720 #endif
1721
1722 return address_space_access_valid(subpage->as, addr + subpage->base,
1723 len, is_write);
1724 }
1725
1726 static const MemoryRegionOps subpage_ops = {
1727 .read = subpage_read,
1728 .write = subpage_write,
1729 .valid.accepts = subpage_accepts,
1730 .endianness = DEVICE_NATIVE_ENDIAN,
1731 };
1732
1733 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1734 uint16_t section)
1735 {
1736 int idx, eidx;
1737
1738 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1739 return -1;
1740 idx = SUBPAGE_IDX(start);
1741 eidx = SUBPAGE_IDX(end);
1742 #if defined(DEBUG_SUBPAGE)
1743 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1744 __func__, mmio, start, end, idx, eidx, section);
1745 #endif
1746 for (; idx <= eidx; idx++) {
1747 mmio->sub_section[idx] = section;
1748 }
1749
1750 return 0;
1751 }
1752
1753 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1754 {
1755 subpage_t *mmio;
1756
1757 mmio = g_malloc0(sizeof(subpage_t));
1758
1759 mmio->as = as;
1760 mmio->base = base;
1761 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1762 "subpage", TARGET_PAGE_SIZE);
1763 mmio->iomem.subpage = true;
1764 #if defined(DEBUG_SUBPAGE)
1765 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
1766 mmio, base, TARGET_PAGE_SIZE);
1767 #endif
1768 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1769
1770 return mmio;
1771 }
1772
1773 static uint16_t dummy_section(PhysPageMap *map, MemoryRegion *mr)
1774 {
1775 MemoryRegionSection section = {
1776 .address_space = &address_space_memory,
1777 .mr = mr,
1778 .offset_within_address_space = 0,
1779 .offset_within_region = 0,
1780 .size = int128_2_64(),
1781 };
1782
1783 return phys_section_add(map, &section);
1784 }
1785
1786 MemoryRegion *iotlb_to_region(AddressSpace *as, hwaddr index)
1787 {
1788 return as->dispatch->map.sections[index & ~TARGET_PAGE_MASK].mr;
1789 }
1790
1791 static void io_mem_init(void)
1792 {
1793 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1794 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1795 "unassigned", UINT64_MAX);
1796 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1797 "notdirty", UINT64_MAX);
1798 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1799 "watch", UINT64_MAX);
1800 }
1801
1802 static void mem_begin(MemoryListener *listener)
1803 {
1804 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1805 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
1806 uint16_t n;
1807
1808 n = dummy_section(&d->map, &io_mem_unassigned);
1809 assert(n == PHYS_SECTION_UNASSIGNED);
1810 n = dummy_section(&d->map, &io_mem_notdirty);
1811 assert(n == PHYS_SECTION_NOTDIRTY);
1812 n = dummy_section(&d->map, &io_mem_rom);
1813 assert(n == PHYS_SECTION_ROM);
1814 n = dummy_section(&d->map, &io_mem_watch);
1815 assert(n == PHYS_SECTION_WATCH);
1816
1817 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
1818 d->as = as;
1819 as->next_dispatch = d;
1820 }
1821
1822 static void mem_commit(MemoryListener *listener)
1823 {
1824 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1825 AddressSpaceDispatch *cur = as->dispatch;
1826 AddressSpaceDispatch *next = as->next_dispatch;
1827
1828 phys_page_compact_all(next, next->map.nodes_nb);
1829
1830 as->dispatch = next;
1831
1832 if (cur) {
1833 phys_sections_free(&cur->map);
1834 g_free(cur);
1835 }
1836 }
1837
1838 static void tcg_commit(MemoryListener *listener)
1839 {
1840 CPUState *cpu;
1841
1842 /* since each CPU stores ram addresses in its TLB cache, we must
1843 reset the modified entries */
1844 /* XXX: slow ! */
1845 CPU_FOREACH(cpu) {
1846 CPUArchState *env = cpu->env_ptr;
1847
1848 /* FIXME: Disentangle the cpu.h circular files deps so we can
1849 directly get the right CPU from listener. */
1850 if (cpu->tcg_as_listener != listener) {
1851 continue;
1852 }
1853 tlb_flush(env, 1);
1854 }
1855 }
1856
1857 static void core_log_global_start(MemoryListener *listener)
1858 {
1859 cpu_physical_memory_set_dirty_tracking(true);
1860 }
1861
1862 static void core_log_global_stop(MemoryListener *listener)
1863 {
1864 cpu_physical_memory_set_dirty_tracking(false);
1865 }
1866
1867 static MemoryListener core_memory_listener = {
1868 .log_global_start = core_log_global_start,
1869 .log_global_stop = core_log_global_stop,
1870 .priority = 1,
1871 };
1872
1873 void address_space_init_dispatch(AddressSpace *as)
1874 {
1875 as->dispatch = NULL;
1876 as->dispatch_listener = (MemoryListener) {
1877 .begin = mem_begin,
1878 .commit = mem_commit,
1879 .region_add = mem_add,
1880 .region_nop = mem_add,
1881 .priority = 0,
1882 };
1883 memory_listener_register(&as->dispatch_listener, as);
1884 }
1885
1886 void address_space_destroy_dispatch(AddressSpace *as)
1887 {
1888 AddressSpaceDispatch *d = as->dispatch;
1889
1890 memory_listener_unregister(&as->dispatch_listener);
1891 g_free(d);
1892 as->dispatch = NULL;
1893 }
1894
1895 static void memory_map_init(void)
1896 {
1897 system_memory = g_malloc(sizeof(*system_memory));
1898
1899 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
1900 address_space_init(&address_space_memory, system_memory, "memory");
1901
1902 system_io = g_malloc(sizeof(*system_io));
1903 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1904 65536);
1905 address_space_init(&address_space_io, system_io, "I/O");
1906
1907 memory_listener_register(&core_memory_listener, &address_space_memory);
1908 }
1909
1910 MemoryRegion *get_system_memory(void)
1911 {
1912 return system_memory;
1913 }
1914
1915 MemoryRegion *get_system_io(void)
1916 {
1917 return system_io;
1918 }
1919
1920 #endif /* !defined(CONFIG_USER_ONLY) */
1921
1922 /* physical memory access (slow version, mainly for debug) */
1923 #if defined(CONFIG_USER_ONLY)
1924 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1925 uint8_t *buf, int len, int is_write)
1926 {
1927 int l, flags;
1928 target_ulong page;
1929 void * p;
1930
1931 while (len > 0) {
1932 page = addr & TARGET_PAGE_MASK;
1933 l = (page + TARGET_PAGE_SIZE) - addr;
1934 if (l > len)
1935 l = len;
1936 flags = page_get_flags(page);
1937 if (!(flags & PAGE_VALID))
1938 return -1;
1939 if (is_write) {
1940 if (!(flags & PAGE_WRITE))
1941 return -1;
1942 /* XXX: this code should not depend on lock_user */
1943 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1944 return -1;
1945 memcpy(p, buf, l);
1946 unlock_user(p, addr, l);
1947 } else {
1948 if (!(flags & PAGE_READ))
1949 return -1;
1950 /* XXX: this code should not depend on lock_user */
1951 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1952 return -1;
1953 memcpy(buf, p, l);
1954 unlock_user(p, addr, 0);
1955 }
1956 len -= l;
1957 buf += l;
1958 addr += l;
1959 }
1960 return 0;
1961 }
1962
1963 #else
1964
1965 static void invalidate_and_set_dirty(hwaddr addr,
1966 hwaddr length)
1967 {
1968 if (cpu_physical_memory_is_clean(addr)) {
1969 /* invalidate code */
1970 tb_invalidate_phys_page_range(addr, addr + length, 0);
1971 /* set dirty bit */
1972 cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_VGA);
1973 cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
1974 }
1975 xen_modified_memory(addr, length);
1976 }
1977
1978 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1979 {
1980 unsigned access_size_max = mr->ops->valid.max_access_size;
1981
1982 /* Regions are assumed to support 1-4 byte accesses unless
1983 otherwise specified. */
1984 if (access_size_max == 0) {
1985 access_size_max = 4;
1986 }
1987
1988 /* Bound the maximum access by the alignment of the address. */
1989 if (!mr->ops->impl.unaligned) {
1990 unsigned align_size_max = addr & -addr;
1991 if (align_size_max != 0 && align_size_max < access_size_max) {
1992 access_size_max = align_size_max;
1993 }
1994 }
1995
1996 /* Don't attempt accesses larger than the maximum. */
1997 if (l > access_size_max) {
1998 l = access_size_max;
1999 }
2000 if (l & (l - 1)) {
2001 l = 1 << (qemu_fls(l) - 1);
2002 }
2003
2004 return l;
2005 }
2006
2007 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
2008 int len, bool is_write)
2009 {
2010 hwaddr l;
2011 uint8_t *ptr;
2012 uint64_t val;
2013 hwaddr addr1;
2014 MemoryRegion *mr;
2015 bool error = false;
2016
2017 while (len > 0) {
2018 l = len;
2019 mr = address_space_translate(as, addr, &addr1, &l, is_write);
2020
2021 if (is_write) {
2022 if (!memory_access_is_direct(mr, is_write)) {
2023 l = memory_access_size(mr, l, addr1);
2024 /* XXX: could force current_cpu to NULL to avoid
2025 potential bugs */
2026 switch (l) {
2027 case 8:
2028 /* 64 bit write access */
2029 val = ldq_p(buf);
2030 error |= io_mem_write(mr, addr1, val, 8);
2031 break;
2032 case 4:
2033 /* 32 bit write access */
2034 val = ldl_p(buf);
2035 error |= io_mem_write(mr, addr1, val, 4);
2036 break;
2037 case 2:
2038 /* 16 bit write access */
2039 val = lduw_p(buf);
2040 error |= io_mem_write(mr, addr1, val, 2);
2041 break;
2042 case 1:
2043 /* 8 bit write access */
2044 val = ldub_p(buf);
2045 error |= io_mem_write(mr, addr1, val, 1);
2046 break;
2047 default:
2048 abort();
2049 }
2050 } else {
2051 addr1 += memory_region_get_ram_addr(mr);
2052 /* RAM case */
2053 ptr = qemu_get_ram_ptr(addr1);
2054 memcpy(ptr, buf, l);
2055 invalidate_and_set_dirty(addr1, l);
2056 }
2057 } else {
2058 if (!memory_access_is_direct(mr, is_write)) {
2059 /* I/O case */
2060 l = memory_access_size(mr, l, addr1);
2061 switch (l) {
2062 case 8:
2063 /* 64 bit read access */
2064 error |= io_mem_read(mr, addr1, &val, 8);
2065 stq_p(buf, val);
2066 break;
2067 case 4:
2068 /* 32 bit read access */
2069 error |= io_mem_read(mr, addr1, &val, 4);
2070 stl_p(buf, val);
2071 break;
2072 case 2:
2073 /* 16 bit read access */
2074 error |= io_mem_read(mr, addr1, &val, 2);
2075 stw_p(buf, val);
2076 break;
2077 case 1:
2078 /* 8 bit read access */
2079 error |= io_mem_read(mr, addr1, &val, 1);
2080 stb_p(buf, val);
2081 break;
2082 default:
2083 abort();
2084 }
2085 } else {
2086 /* RAM case */
2087 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2088 memcpy(buf, ptr, l);
2089 }
2090 }
2091 len -= l;
2092 buf += l;
2093 addr += l;
2094 }
2095
2096 return error;
2097 }
2098
2099 bool address_space_write(AddressSpace *as, hwaddr addr,
2100 const uint8_t *buf, int len)
2101 {
2102 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2103 }
2104
2105 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2106 {
2107 return address_space_rw(as, addr, buf, len, false);
2108 }
2109
2110
2111 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2112 int len, int is_write)
2113 {
2114 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2115 }
2116
2117 enum write_rom_type {
2118 WRITE_DATA,
2119 FLUSH_CACHE,
2120 };
2121
2122 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2123 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2124 {
2125 hwaddr l;
2126 uint8_t *ptr;
2127 hwaddr addr1;
2128 MemoryRegion *mr;
2129
2130 while (len > 0) {
2131 l = len;
2132 mr = address_space_translate(as, addr, &addr1, &l, true);
2133
2134 if (!(memory_region_is_ram(mr) ||
2135 memory_region_is_romd(mr))) {
2136 /* do nothing */
2137 } else {
2138 addr1 += memory_region_get_ram_addr(mr);
2139 /* ROM/RAM case */
2140 ptr = qemu_get_ram_ptr(addr1);
2141 switch (type) {
2142 case WRITE_DATA:
2143 memcpy(ptr, buf, l);
2144 invalidate_and_set_dirty(addr1, l);
2145 break;
2146 case FLUSH_CACHE:
2147 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2148 break;
2149 }
2150 }
2151 len -= l;
2152 buf += l;
2153 addr += l;
2154 }
2155 }
2156
2157 /* used for ROM loading : can write in RAM and ROM */
2158 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2159 const uint8_t *buf, int len)
2160 {
2161 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2162 }
2163
2164 void cpu_flush_icache_range(hwaddr start, int len)
2165 {
2166 /*
2167 * This function should do the same thing as an icache flush that was
2168 * triggered from within the guest. For TCG we are always cache coherent,
2169 * so there is no need to flush anything. For KVM / Xen we need to flush
2170 * the host's instruction cache at least.
2171 */
2172 if (tcg_enabled()) {
2173 return;
2174 }
2175
2176 cpu_physical_memory_write_rom_internal(&address_space_memory,
2177 start, NULL, len, FLUSH_CACHE);
2178 }
2179
2180 typedef struct {
2181 MemoryRegion *mr;
2182 void *buffer;
2183 hwaddr addr;
2184 hwaddr len;
2185 } BounceBuffer;
2186
2187 static BounceBuffer bounce;
2188
2189 typedef struct MapClient {
2190 void *opaque;
2191 void (*callback)(void *opaque);
2192 QLIST_ENTRY(MapClient) link;
2193 } MapClient;
2194
2195 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2196 = QLIST_HEAD_INITIALIZER(map_client_list);
2197
2198 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2199 {
2200 MapClient *client = g_malloc(sizeof(*client));
2201
2202 client->opaque = opaque;
2203 client->callback = callback;
2204 QLIST_INSERT_HEAD(&map_client_list, client, link);
2205 return client;
2206 }
2207
2208 static void cpu_unregister_map_client(void *_client)
2209 {
2210 MapClient *client = (MapClient *)_client;
2211
2212 QLIST_REMOVE(client, link);
2213 g_free(client);
2214 }
2215
2216 static void cpu_notify_map_clients(void)
2217 {
2218 MapClient *client;
2219
2220 while (!QLIST_EMPTY(&map_client_list)) {
2221 client = QLIST_FIRST(&map_client_list);
2222 client->callback(client->opaque);
2223 cpu_unregister_map_client(client);
2224 }
2225 }
2226
2227 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2228 {
2229 MemoryRegion *mr;
2230 hwaddr l, xlat;
2231
2232 while (len > 0) {
2233 l = len;
2234 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2235 if (!memory_access_is_direct(mr, is_write)) {
2236 l = memory_access_size(mr, l, addr);
2237 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2238 return false;
2239 }
2240 }
2241
2242 len -= l;
2243 addr += l;
2244 }
2245 return true;
2246 }
2247
2248 /* Map a physical memory region into a host virtual address.
2249 * May map a subset of the requested range, given by and returned in *plen.
2250 * May return NULL if resources needed to perform the mapping are exhausted.
2251 * Use only for reads OR writes - not for read-modify-write operations.
2252 * Use cpu_register_map_client() to know when retrying the map operation is
2253 * likely to succeed.
2254 */
2255 void *address_space_map(AddressSpace *as,
2256 hwaddr addr,
2257 hwaddr *plen,
2258 bool is_write)
2259 {
2260 hwaddr len = *plen;
2261 hwaddr done = 0;
2262 hwaddr l, xlat, base;
2263 MemoryRegion *mr, *this_mr;
2264 ram_addr_t raddr;
2265
2266 if (len == 0) {
2267 return NULL;
2268 }
2269
2270 l = len;
2271 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2272 if (!memory_access_is_direct(mr, is_write)) {
2273 if (bounce.buffer) {
2274 return NULL;
2275 }
2276 /* Avoid unbounded allocations */
2277 l = MIN(l, TARGET_PAGE_SIZE);
2278 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2279 bounce.addr = addr;
2280 bounce.len = l;
2281
2282 memory_region_ref(mr);
2283 bounce.mr = mr;
2284 if (!is_write) {
2285 address_space_read(as, addr, bounce.buffer, l);
2286 }
2287
2288 *plen = l;
2289 return bounce.buffer;
2290 }
2291
2292 base = xlat;
2293 raddr = memory_region_get_ram_addr(mr);
2294
2295 for (;;) {
2296 len -= l;
2297 addr += l;
2298 done += l;
2299 if (len == 0) {
2300 break;
2301 }
2302
2303 l = len;
2304 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2305 if (this_mr != mr || xlat != base + done) {
2306 break;
2307 }
2308 }
2309
2310 memory_region_ref(mr);
2311 *plen = done;
2312 return qemu_ram_ptr_length(raddr + base, plen);
2313 }
2314
2315 /* Unmaps a memory region previously mapped by address_space_map().
2316 * Will also mark the memory as dirty if is_write == 1. access_len gives
2317 * the amount of memory that was actually read or written by the caller.
2318 */
2319 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2320 int is_write, hwaddr access_len)
2321 {
2322 if (buffer != bounce.buffer) {
2323 MemoryRegion *mr;
2324 ram_addr_t addr1;
2325
2326 mr = qemu_ram_addr_from_host(buffer, &addr1);
2327 assert(mr != NULL);
2328 if (is_write) {
2329 while (access_len) {
2330 unsigned l;
2331 l = TARGET_PAGE_SIZE;
2332 if (l > access_len)
2333 l = access_len;
2334 invalidate_and_set_dirty(addr1, l);
2335 addr1 += l;
2336 access_len -= l;
2337 }
2338 }
2339 if (xen_enabled()) {
2340 xen_invalidate_map_cache_entry(buffer);
2341 }
2342 memory_region_unref(mr);
2343 return;
2344 }
2345 if (is_write) {
2346 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2347 }
2348 qemu_vfree(bounce.buffer);
2349 bounce.buffer = NULL;
2350 memory_region_unref(bounce.mr);
2351 cpu_notify_map_clients();
2352 }
2353
2354 void *cpu_physical_memory_map(hwaddr addr,
2355 hwaddr *plen,
2356 int is_write)
2357 {
2358 return address_space_map(&address_space_memory, addr, plen, is_write);
2359 }
2360
2361 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2362 int is_write, hwaddr access_len)
2363 {
2364 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2365 }
2366
2367 /* warning: addr must be aligned */
2368 static inline uint32_t ldl_phys_internal(AddressSpace *as, hwaddr addr,
2369 enum device_endian endian)
2370 {
2371 uint8_t *ptr;
2372 uint64_t val;
2373 MemoryRegion *mr;
2374 hwaddr l = 4;
2375 hwaddr addr1;
2376
2377 mr = address_space_translate(as, addr, &addr1, &l, false);
2378 if (l < 4 || !memory_access_is_direct(mr, false)) {
2379 /* I/O case */
2380 io_mem_read(mr, addr1, &val, 4);
2381 #if defined(TARGET_WORDS_BIGENDIAN)
2382 if (endian == DEVICE_LITTLE_ENDIAN) {
2383 val = bswap32(val);
2384 }
2385 #else
2386 if (endian == DEVICE_BIG_ENDIAN) {
2387 val = bswap32(val);
2388 }
2389 #endif
2390 } else {
2391 /* RAM case */
2392 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2393 & TARGET_PAGE_MASK)
2394 + addr1);
2395 switch (endian) {
2396 case DEVICE_LITTLE_ENDIAN:
2397 val = ldl_le_p(ptr);
2398 break;
2399 case DEVICE_BIG_ENDIAN:
2400 val = ldl_be_p(ptr);
2401 break;
2402 default:
2403 val = ldl_p(ptr);
2404 break;
2405 }
2406 }
2407 return val;
2408 }
2409
2410 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
2411 {
2412 return ldl_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
2413 }
2414
2415 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
2416 {
2417 return ldl_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
2418 }
2419
2420 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
2421 {
2422 return ldl_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
2423 }
2424
2425 /* warning: addr must be aligned */
2426 static inline uint64_t ldq_phys_internal(AddressSpace *as, hwaddr addr,
2427 enum device_endian endian)
2428 {
2429 uint8_t *ptr;
2430 uint64_t val;
2431 MemoryRegion *mr;
2432 hwaddr l = 8;
2433 hwaddr addr1;
2434
2435 mr = address_space_translate(as, addr, &addr1, &l,
2436 false);
2437 if (l < 8 || !memory_access_is_direct(mr, false)) {
2438 /* I/O case */
2439 io_mem_read(mr, addr1, &val, 8);
2440 #if defined(TARGET_WORDS_BIGENDIAN)
2441 if (endian == DEVICE_LITTLE_ENDIAN) {
2442 val = bswap64(val);
2443 }
2444 #else
2445 if (endian == DEVICE_BIG_ENDIAN) {
2446 val = bswap64(val);
2447 }
2448 #endif
2449 } else {
2450 /* RAM case */
2451 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2452 & TARGET_PAGE_MASK)
2453 + addr1);
2454 switch (endian) {
2455 case DEVICE_LITTLE_ENDIAN:
2456 val = ldq_le_p(ptr);
2457 break;
2458 case DEVICE_BIG_ENDIAN:
2459 val = ldq_be_p(ptr);
2460 break;
2461 default:
2462 val = ldq_p(ptr);
2463 break;
2464 }
2465 }
2466 return val;
2467 }
2468
2469 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
2470 {
2471 return ldq_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
2472 }
2473
2474 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
2475 {
2476 return ldq_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
2477 }
2478
2479 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
2480 {
2481 return ldq_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
2482 }
2483
2484 /* XXX: optimize */
2485 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
2486 {
2487 uint8_t val;
2488 address_space_rw(as, addr, &val, 1, 0);
2489 return val;
2490 }
2491
2492 /* warning: addr must be aligned */
2493 static inline uint32_t lduw_phys_internal(AddressSpace *as, hwaddr addr,
2494 enum device_endian endian)
2495 {
2496 uint8_t *ptr;
2497 uint64_t val;
2498 MemoryRegion *mr;
2499 hwaddr l = 2;
2500 hwaddr addr1;
2501
2502 mr = address_space_translate(as, addr, &addr1, &l,
2503 false);
2504 if (l < 2 || !memory_access_is_direct(mr, false)) {
2505 /* I/O case */
2506 io_mem_read(mr, addr1, &val, 2);
2507 #if defined(TARGET_WORDS_BIGENDIAN)
2508 if (endian == DEVICE_LITTLE_ENDIAN) {
2509 val = bswap16(val);
2510 }
2511 #else
2512 if (endian == DEVICE_BIG_ENDIAN) {
2513 val = bswap16(val);
2514 }
2515 #endif
2516 } else {
2517 /* RAM case */
2518 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2519 & TARGET_PAGE_MASK)
2520 + addr1);
2521 switch (endian) {
2522 case DEVICE_LITTLE_ENDIAN:
2523 val = lduw_le_p(ptr);
2524 break;
2525 case DEVICE_BIG_ENDIAN:
2526 val = lduw_be_p(ptr);
2527 break;
2528 default:
2529 val = lduw_p(ptr);
2530 break;
2531 }
2532 }
2533 return val;
2534 }
2535
2536 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
2537 {
2538 return lduw_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
2539 }
2540
2541 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
2542 {
2543 return lduw_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
2544 }
2545
2546 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
2547 {
2548 return lduw_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
2549 }
2550
2551 /* warning: addr must be aligned. The ram page is not masked as dirty
2552 and the code inside is not invalidated. It is useful if the dirty
2553 bits are used to track modified PTEs */
2554 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
2555 {
2556 uint8_t *ptr;
2557 MemoryRegion *mr;
2558 hwaddr l = 4;
2559 hwaddr addr1;
2560
2561 mr = address_space_translate(as, addr, &addr1, &l,
2562 true);
2563 if (l < 4 || !memory_access_is_direct(mr, true)) {
2564 io_mem_write(mr, addr1, val, 4);
2565 } else {
2566 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2567 ptr = qemu_get_ram_ptr(addr1);
2568 stl_p(ptr, val);
2569
2570 if (unlikely(in_migration)) {
2571 if (cpu_physical_memory_is_clean(addr1)) {
2572 /* invalidate code */
2573 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2574 /* set dirty bit */
2575 cpu_physical_memory_set_dirty_flag(addr1,
2576 DIRTY_MEMORY_MIGRATION);
2577 cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_VGA);
2578 }
2579 }
2580 }
2581 }
2582
2583 /* warning: addr must be aligned */
2584 static inline void stl_phys_internal(AddressSpace *as,
2585 hwaddr addr, uint32_t val,
2586 enum device_endian endian)
2587 {
2588 uint8_t *ptr;
2589 MemoryRegion *mr;
2590 hwaddr l = 4;
2591 hwaddr addr1;
2592
2593 mr = address_space_translate(as, addr, &addr1, &l,
2594 true);
2595 if (l < 4 || !memory_access_is_direct(mr, true)) {
2596 #if defined(TARGET_WORDS_BIGENDIAN)
2597 if (endian == DEVICE_LITTLE_ENDIAN) {
2598 val = bswap32(val);
2599 }
2600 #else
2601 if (endian == DEVICE_BIG_ENDIAN) {
2602 val = bswap32(val);
2603 }
2604 #endif
2605 io_mem_write(mr, addr1, val, 4);
2606 } else {
2607 /* RAM case */
2608 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2609 ptr = qemu_get_ram_ptr(addr1);
2610 switch (endian) {
2611 case DEVICE_LITTLE_ENDIAN:
2612 stl_le_p(ptr, val);
2613 break;
2614 case DEVICE_BIG_ENDIAN:
2615 stl_be_p(ptr, val);
2616 break;
2617 default:
2618 stl_p(ptr, val);
2619 break;
2620 }
2621 invalidate_and_set_dirty(addr1, 4);
2622 }
2623 }
2624
2625 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2626 {
2627 stl_phys_internal(as, addr, val, DEVICE_NATIVE_ENDIAN);
2628 }
2629
2630 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2631 {
2632 stl_phys_internal(as, addr, val, DEVICE_LITTLE_ENDIAN);
2633 }
2634
2635 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2636 {
2637 stl_phys_internal(as, addr, val, DEVICE_BIG_ENDIAN);
2638 }
2639
2640 /* XXX: optimize */
2641 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2642 {
2643 uint8_t v = val;
2644 address_space_rw(as, addr, &v, 1, 1);
2645 }
2646
2647 /* warning: addr must be aligned */
2648 static inline void stw_phys_internal(AddressSpace *as,
2649 hwaddr addr, uint32_t val,
2650 enum device_endian endian)
2651 {
2652 uint8_t *ptr;
2653 MemoryRegion *mr;
2654 hwaddr l = 2;
2655 hwaddr addr1;
2656
2657 mr = address_space_translate(as, addr, &addr1, &l, true);
2658 if (l < 2 || !memory_access_is_direct(mr, true)) {
2659 #if defined(TARGET_WORDS_BIGENDIAN)
2660 if (endian == DEVICE_LITTLE_ENDIAN) {
2661 val = bswap16(val);
2662 }
2663 #else
2664 if (endian == DEVICE_BIG_ENDIAN) {
2665 val = bswap16(val);
2666 }
2667 #endif
2668 io_mem_write(mr, addr1, val, 2);
2669 } else {
2670 /* RAM case */
2671 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2672 ptr = qemu_get_ram_ptr(addr1);
2673 switch (endian) {
2674 case DEVICE_LITTLE_ENDIAN:
2675 stw_le_p(ptr, val);
2676 break;
2677 case DEVICE_BIG_ENDIAN:
2678 stw_be_p(ptr, val);
2679 break;
2680 default:
2681 stw_p(ptr, val);
2682 break;
2683 }
2684 invalidate_and_set_dirty(addr1, 2);
2685 }
2686 }
2687
2688 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2689 {
2690 stw_phys_internal(as, addr, val, DEVICE_NATIVE_ENDIAN);
2691 }
2692
2693 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2694 {
2695 stw_phys_internal(as, addr, val, DEVICE_LITTLE_ENDIAN);
2696 }
2697
2698 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2699 {
2700 stw_phys_internal(as, addr, val, DEVICE_BIG_ENDIAN);
2701 }
2702
2703 /* XXX: optimize */
2704 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
2705 {
2706 val = tswap64(val);
2707 address_space_rw(as, addr, (void *) &val, 8, 1);
2708 }
2709
2710 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
2711 {
2712 val = cpu_to_le64(val);
2713 address_space_rw(as, addr, (void *) &val, 8, 1);
2714 }
2715
2716 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
2717 {
2718 val = cpu_to_be64(val);
2719 address_space_rw(as, addr, (void *) &val, 8, 1);
2720 }
2721
2722 /* virtual memory access for debug (includes writing to ROM) */
2723 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2724 uint8_t *buf, int len, int is_write)
2725 {
2726 int l;
2727 hwaddr phys_addr;
2728 target_ulong page;
2729
2730 while (len > 0) {
2731 page = addr & TARGET_PAGE_MASK;
2732 phys_addr = cpu_get_phys_page_debug(cpu, page);
2733 /* if no physical page mapped, return an error */
2734 if (phys_addr == -1)
2735 return -1;
2736 l = (page + TARGET_PAGE_SIZE) - addr;
2737 if (l > len)
2738 l = len;
2739 phys_addr += (addr & ~TARGET_PAGE_MASK);
2740 if (is_write) {
2741 cpu_physical_memory_write_rom(cpu->as, phys_addr, buf, l);
2742 } else {
2743 address_space_rw(cpu->as, phys_addr, buf, l, 0);
2744 }
2745 len -= l;
2746 buf += l;
2747 addr += l;
2748 }
2749 return 0;
2750 }
2751 #endif
2752
2753 #if !defined(CONFIG_USER_ONLY)
2754
2755 /*
2756 * A helper function for the _utterly broken_ virtio device model to find out if
2757 * it's running on a big endian machine. Don't do this at home kids!
2758 */
2759 bool virtio_is_big_endian(void);
2760 bool virtio_is_big_endian(void)
2761 {
2762 #if defined(TARGET_WORDS_BIGENDIAN)
2763 return true;
2764 #else
2765 return false;
2766 #endif
2767 }
2768
2769 #endif
2770
2771 #ifndef CONFIG_USER_ONLY
2772 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2773 {
2774 MemoryRegion*mr;
2775 hwaddr l = 1;
2776
2777 mr = address_space_translate(&address_space_memory,
2778 phys_addr, &phys_addr, &l, false);
2779
2780 return !(memory_region_is_ram(mr) ||
2781 memory_region_is_romd(mr));
2782 }
2783
2784 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2785 {
2786 RAMBlock *block;
2787
2788 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2789 func(block->host, block->offset, block->length, opaque);
2790 }
2791 }
2792 #endif