exec: remove ram_addr argument from qemu_ram_block_from_host
[qemu.git] / exec.c
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "qemu/osdep.h"
20 #include "qapi/error.h"
21 #ifndef _WIN32
22 #include <sys/mman.h>
23 #endif
24
25 #include "qemu/cutils.h"
26 #include "cpu.h"
27 #include "exec/exec-all.h"
28 #include "tcg.h"
29 #include "hw/qdev-core.h"
30 #if !defined(CONFIG_USER_ONLY)
31 #include "hw/boards.h"
32 #include "hw/xen/xen.h"
33 #endif
34 #include "sysemu/kvm.h"
35 #include "sysemu/sysemu.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "qemu/error-report.h"
39 #if defined(CONFIG_USER_ONLY)
40 #include <qemu.h>
41 #else /* !CONFIG_USER_ONLY */
42 #include "hw/hw.h"
43 #include "exec/memory.h"
44 #include "exec/ioport.h"
45 #include "sysemu/dma.h"
46 #include "exec/address-spaces.h"
47 #include "sysemu/xen-mapcache.h"
48 #include "trace.h"
49 #endif
50 #include "exec/cpu-all.h"
51 #include "qemu/rcu_queue.h"
52 #include "qemu/main-loop.h"
53 #include "translate-all.h"
54 #include "sysemu/replay.h"
55
56 #include "exec/memory-internal.h"
57 #include "exec/ram_addr.h"
58 #include "exec/log.h"
59
60 #include "qemu/range.h"
61 #ifndef _WIN32
62 #include "qemu/mmap-alloc.h"
63 #endif
64
65 //#define DEBUG_SUBPAGE
66
67 #if !defined(CONFIG_USER_ONLY)
68 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
69 * are protected by the ramlist lock.
70 */
71 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
72
73 static MemoryRegion *system_memory;
74 static MemoryRegion *system_io;
75
76 AddressSpace address_space_io;
77 AddressSpace address_space_memory;
78
79 MemoryRegion io_mem_rom, io_mem_notdirty;
80 static MemoryRegion io_mem_unassigned;
81
82 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
83 #define RAM_PREALLOC (1 << 0)
84
85 /* RAM is mmap-ed with MAP_SHARED */
86 #define RAM_SHARED (1 << 1)
87
88 /* Only a portion of RAM (used_length) is actually used, and migrated.
89 * This used_length size can change across reboots.
90 */
91 #define RAM_RESIZEABLE (1 << 2)
92
93 #endif
94
95 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
96 /* current CPU in the current thread. It is only valid inside
97 cpu_exec() */
98 __thread CPUState *current_cpu;
99 /* 0 = Do not count executed instructions.
100 1 = Precise instruction counting.
101 2 = Adaptive rate instruction counting. */
102 int use_icount;
103
104 #if !defined(CONFIG_USER_ONLY)
105
106 typedef struct PhysPageEntry PhysPageEntry;
107
108 struct PhysPageEntry {
109 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
110 uint32_t skip : 6;
111 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
112 uint32_t ptr : 26;
113 };
114
115 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
116
117 /* Size of the L2 (and L3, etc) page tables. */
118 #define ADDR_SPACE_BITS 64
119
120 #define P_L2_BITS 9
121 #define P_L2_SIZE (1 << P_L2_BITS)
122
123 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
124
125 typedef PhysPageEntry Node[P_L2_SIZE];
126
127 typedef struct PhysPageMap {
128 struct rcu_head rcu;
129
130 unsigned sections_nb;
131 unsigned sections_nb_alloc;
132 unsigned nodes_nb;
133 unsigned nodes_nb_alloc;
134 Node *nodes;
135 MemoryRegionSection *sections;
136 } PhysPageMap;
137
138 struct AddressSpaceDispatch {
139 struct rcu_head rcu;
140
141 MemoryRegionSection *mru_section;
142 /* This is a multi-level map on the physical address space.
143 * The bottom level has pointers to MemoryRegionSections.
144 */
145 PhysPageEntry phys_map;
146 PhysPageMap map;
147 AddressSpace *as;
148 };
149
150 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
151 typedef struct subpage_t {
152 MemoryRegion iomem;
153 AddressSpace *as;
154 hwaddr base;
155 uint16_t sub_section[TARGET_PAGE_SIZE];
156 } subpage_t;
157
158 #define PHYS_SECTION_UNASSIGNED 0
159 #define PHYS_SECTION_NOTDIRTY 1
160 #define PHYS_SECTION_ROM 2
161 #define PHYS_SECTION_WATCH 3
162
163 static void io_mem_init(void);
164 static void memory_map_init(void);
165 static void tcg_commit(MemoryListener *listener);
166
167 static MemoryRegion io_mem_watch;
168
169 /**
170 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
171 * @cpu: the CPU whose AddressSpace this is
172 * @as: the AddressSpace itself
173 * @memory_dispatch: its dispatch pointer (cached, RCU protected)
174 * @tcg_as_listener: listener for tracking changes to the AddressSpace
175 */
176 struct CPUAddressSpace {
177 CPUState *cpu;
178 AddressSpace *as;
179 struct AddressSpaceDispatch *memory_dispatch;
180 MemoryListener tcg_as_listener;
181 };
182
183 #endif
184
185 #if !defined(CONFIG_USER_ONLY)
186
187 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
188 {
189 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
190 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
191 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
192 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
193 }
194 }
195
196 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
197 {
198 unsigned i;
199 uint32_t ret;
200 PhysPageEntry e;
201 PhysPageEntry *p;
202
203 ret = map->nodes_nb++;
204 p = map->nodes[ret];
205 assert(ret != PHYS_MAP_NODE_NIL);
206 assert(ret != map->nodes_nb_alloc);
207
208 e.skip = leaf ? 0 : 1;
209 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
210 for (i = 0; i < P_L2_SIZE; ++i) {
211 memcpy(&p[i], &e, sizeof(e));
212 }
213 return ret;
214 }
215
216 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
217 hwaddr *index, hwaddr *nb, uint16_t leaf,
218 int level)
219 {
220 PhysPageEntry *p;
221 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
222
223 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
224 lp->ptr = phys_map_node_alloc(map, level == 0);
225 }
226 p = map->nodes[lp->ptr];
227 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
228
229 while (*nb && lp < &p[P_L2_SIZE]) {
230 if ((*index & (step - 1)) == 0 && *nb >= step) {
231 lp->skip = 0;
232 lp->ptr = leaf;
233 *index += step;
234 *nb -= step;
235 } else {
236 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
237 }
238 ++lp;
239 }
240 }
241
242 static void phys_page_set(AddressSpaceDispatch *d,
243 hwaddr index, hwaddr nb,
244 uint16_t leaf)
245 {
246 /* Wildly overreserve - it doesn't matter much. */
247 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
248
249 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
250 }
251
252 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
253 * and update our entry so we can skip it and go directly to the destination.
254 */
255 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
256 {
257 unsigned valid_ptr = P_L2_SIZE;
258 int valid = 0;
259 PhysPageEntry *p;
260 int i;
261
262 if (lp->ptr == PHYS_MAP_NODE_NIL) {
263 return;
264 }
265
266 p = nodes[lp->ptr];
267 for (i = 0; i < P_L2_SIZE; i++) {
268 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
269 continue;
270 }
271
272 valid_ptr = i;
273 valid++;
274 if (p[i].skip) {
275 phys_page_compact(&p[i], nodes, compacted);
276 }
277 }
278
279 /* We can only compress if there's only one child. */
280 if (valid != 1) {
281 return;
282 }
283
284 assert(valid_ptr < P_L2_SIZE);
285
286 /* Don't compress if it won't fit in the # of bits we have. */
287 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
288 return;
289 }
290
291 lp->ptr = p[valid_ptr].ptr;
292 if (!p[valid_ptr].skip) {
293 /* If our only child is a leaf, make this a leaf. */
294 /* By design, we should have made this node a leaf to begin with so we
295 * should never reach here.
296 * But since it's so simple to handle this, let's do it just in case we
297 * change this rule.
298 */
299 lp->skip = 0;
300 } else {
301 lp->skip += p[valid_ptr].skip;
302 }
303 }
304
305 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
306 {
307 DECLARE_BITMAP(compacted, nodes_nb);
308
309 if (d->phys_map.skip) {
310 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
311 }
312 }
313
314 static inline bool section_covers_addr(const MemoryRegionSection *section,
315 hwaddr addr)
316 {
317 /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
318 * the section must cover the entire address space.
319 */
320 return section->size.hi ||
321 range_covers_byte(section->offset_within_address_space,
322 section->size.lo, addr);
323 }
324
325 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
326 Node *nodes, MemoryRegionSection *sections)
327 {
328 PhysPageEntry *p;
329 hwaddr index = addr >> TARGET_PAGE_BITS;
330 int i;
331
332 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
333 if (lp.ptr == PHYS_MAP_NODE_NIL) {
334 return &sections[PHYS_SECTION_UNASSIGNED];
335 }
336 p = nodes[lp.ptr];
337 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
338 }
339
340 if (section_covers_addr(&sections[lp.ptr], addr)) {
341 return &sections[lp.ptr];
342 } else {
343 return &sections[PHYS_SECTION_UNASSIGNED];
344 }
345 }
346
347 bool memory_region_is_unassigned(MemoryRegion *mr)
348 {
349 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
350 && mr != &io_mem_watch;
351 }
352
353 /* Called from RCU critical section */
354 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
355 hwaddr addr,
356 bool resolve_subpage)
357 {
358 MemoryRegionSection *section = atomic_read(&d->mru_section);
359 subpage_t *subpage;
360 bool update;
361
362 if (section && section != &d->map.sections[PHYS_SECTION_UNASSIGNED] &&
363 section_covers_addr(section, addr)) {
364 update = false;
365 } else {
366 section = phys_page_find(d->phys_map, addr, d->map.nodes,
367 d->map.sections);
368 update = true;
369 }
370 if (resolve_subpage && section->mr->subpage) {
371 subpage = container_of(section->mr, subpage_t, iomem);
372 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
373 }
374 if (update) {
375 atomic_set(&d->mru_section, section);
376 }
377 return section;
378 }
379
380 /* Called from RCU critical section */
381 static MemoryRegionSection *
382 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
383 hwaddr *plen, bool resolve_subpage)
384 {
385 MemoryRegionSection *section;
386 MemoryRegion *mr;
387 Int128 diff;
388
389 section = address_space_lookup_region(d, addr, resolve_subpage);
390 /* Compute offset within MemoryRegionSection */
391 addr -= section->offset_within_address_space;
392
393 /* Compute offset within MemoryRegion */
394 *xlat = addr + section->offset_within_region;
395
396 mr = section->mr;
397
398 /* MMIO registers can be expected to perform full-width accesses based only
399 * on their address, without considering adjacent registers that could
400 * decode to completely different MemoryRegions. When such registers
401 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
402 * regions overlap wildly. For this reason we cannot clamp the accesses
403 * here.
404 *
405 * If the length is small (as is the case for address_space_ldl/stl),
406 * everything works fine. If the incoming length is large, however,
407 * the caller really has to do the clamping through memory_access_size.
408 */
409 if (memory_region_is_ram(mr)) {
410 diff = int128_sub(section->size, int128_make64(addr));
411 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
412 }
413 return section;
414 }
415
416 /* Called from RCU critical section */
417 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
418 hwaddr *xlat, hwaddr *plen,
419 bool is_write)
420 {
421 IOMMUTLBEntry iotlb;
422 MemoryRegionSection *section;
423 MemoryRegion *mr;
424
425 for (;;) {
426 AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
427 section = address_space_translate_internal(d, addr, &addr, plen, true);
428 mr = section->mr;
429
430 if (!mr->iommu_ops) {
431 break;
432 }
433
434 iotlb = mr->iommu_ops->translate(mr, addr, is_write);
435 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
436 | (addr & iotlb.addr_mask));
437 *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
438 if (!(iotlb.perm & (1 << is_write))) {
439 mr = &io_mem_unassigned;
440 break;
441 }
442
443 as = iotlb.target_as;
444 }
445
446 if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
447 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
448 *plen = MIN(page, *plen);
449 }
450
451 *xlat = addr;
452 return mr;
453 }
454
455 /* Called from RCU critical section */
456 MemoryRegionSection *
457 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
458 hwaddr *xlat, hwaddr *plen)
459 {
460 MemoryRegionSection *section;
461 AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
462
463 section = address_space_translate_internal(d, addr, xlat, plen, false);
464
465 assert(!section->mr->iommu_ops);
466 return section;
467 }
468 #endif
469
470 #if !defined(CONFIG_USER_ONLY)
471
472 static int cpu_common_post_load(void *opaque, int version_id)
473 {
474 CPUState *cpu = opaque;
475
476 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
477 version_id is increased. */
478 cpu->interrupt_request &= ~0x01;
479 tlb_flush(cpu, 1);
480
481 return 0;
482 }
483
484 static int cpu_common_pre_load(void *opaque)
485 {
486 CPUState *cpu = opaque;
487
488 cpu->exception_index = -1;
489
490 return 0;
491 }
492
493 static bool cpu_common_exception_index_needed(void *opaque)
494 {
495 CPUState *cpu = opaque;
496
497 return tcg_enabled() && cpu->exception_index != -1;
498 }
499
500 static const VMStateDescription vmstate_cpu_common_exception_index = {
501 .name = "cpu_common/exception_index",
502 .version_id = 1,
503 .minimum_version_id = 1,
504 .needed = cpu_common_exception_index_needed,
505 .fields = (VMStateField[]) {
506 VMSTATE_INT32(exception_index, CPUState),
507 VMSTATE_END_OF_LIST()
508 }
509 };
510
511 static bool cpu_common_crash_occurred_needed(void *opaque)
512 {
513 CPUState *cpu = opaque;
514
515 return cpu->crash_occurred;
516 }
517
518 static const VMStateDescription vmstate_cpu_common_crash_occurred = {
519 .name = "cpu_common/crash_occurred",
520 .version_id = 1,
521 .minimum_version_id = 1,
522 .needed = cpu_common_crash_occurred_needed,
523 .fields = (VMStateField[]) {
524 VMSTATE_BOOL(crash_occurred, CPUState),
525 VMSTATE_END_OF_LIST()
526 }
527 };
528
529 const VMStateDescription vmstate_cpu_common = {
530 .name = "cpu_common",
531 .version_id = 1,
532 .minimum_version_id = 1,
533 .pre_load = cpu_common_pre_load,
534 .post_load = cpu_common_post_load,
535 .fields = (VMStateField[]) {
536 VMSTATE_UINT32(halted, CPUState),
537 VMSTATE_UINT32(interrupt_request, CPUState),
538 VMSTATE_END_OF_LIST()
539 },
540 .subsections = (const VMStateDescription*[]) {
541 &vmstate_cpu_common_exception_index,
542 &vmstate_cpu_common_crash_occurred,
543 NULL
544 }
545 };
546
547 #endif
548
549 CPUState *qemu_get_cpu(int index)
550 {
551 CPUState *cpu;
552
553 CPU_FOREACH(cpu) {
554 if (cpu->cpu_index == index) {
555 return cpu;
556 }
557 }
558
559 return NULL;
560 }
561
562 #if !defined(CONFIG_USER_ONLY)
563 void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx)
564 {
565 CPUAddressSpace *newas;
566
567 /* Target code should have set num_ases before calling us */
568 assert(asidx < cpu->num_ases);
569
570 if (asidx == 0) {
571 /* address space 0 gets the convenience alias */
572 cpu->as = as;
573 }
574
575 /* KVM cannot currently support multiple address spaces. */
576 assert(asidx == 0 || !kvm_enabled());
577
578 if (!cpu->cpu_ases) {
579 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
580 }
581
582 newas = &cpu->cpu_ases[asidx];
583 newas->cpu = cpu;
584 newas->as = as;
585 if (tcg_enabled()) {
586 newas->tcg_as_listener.commit = tcg_commit;
587 memory_listener_register(&newas->tcg_as_listener, as);
588 }
589 }
590
591 AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
592 {
593 /* Return the AddressSpace corresponding to the specified index */
594 return cpu->cpu_ases[asidx].as;
595 }
596 #endif
597
598 #ifndef CONFIG_USER_ONLY
599 static DECLARE_BITMAP(cpu_index_map, MAX_CPUMASK_BITS);
600
601 static int cpu_get_free_index(Error **errp)
602 {
603 int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);
604
605 if (cpu >= MAX_CPUMASK_BITS) {
606 error_setg(errp, "Trying to use more CPUs than max of %d",
607 MAX_CPUMASK_BITS);
608 return -1;
609 }
610
611 bitmap_set(cpu_index_map, cpu, 1);
612 return cpu;
613 }
614
615 void cpu_exec_exit(CPUState *cpu)
616 {
617 if (cpu->cpu_index == -1) {
618 /* cpu_index was never allocated by this @cpu or was already freed. */
619 return;
620 }
621
622 bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
623 cpu->cpu_index = -1;
624 }
625 #else
626
627 static int cpu_get_free_index(Error **errp)
628 {
629 CPUState *some_cpu;
630 int cpu_index = 0;
631
632 CPU_FOREACH(some_cpu) {
633 cpu_index++;
634 }
635 return cpu_index;
636 }
637
638 void cpu_exec_exit(CPUState *cpu)
639 {
640 }
641 #endif
642
643 void cpu_exec_init(CPUState *cpu, Error **errp)
644 {
645 CPUClass *cc = CPU_GET_CLASS(cpu);
646 Error *local_err = NULL;
647
648 cpu->as = NULL;
649 cpu->num_ases = 0;
650
651 #ifndef CONFIG_USER_ONLY
652 cpu->thread_id = qemu_get_thread_id();
653
654 /* This is a softmmu CPU object, so create a property for it
655 * so users can wire up its memory. (This can't go in qom/cpu.c
656 * because that file is compiled only once for both user-mode
657 * and system builds.) The default if no link is set up is to use
658 * the system address space.
659 */
660 object_property_add_link(OBJECT(cpu), "memory", TYPE_MEMORY_REGION,
661 (Object **)&cpu->memory,
662 qdev_prop_allow_set_link_before_realize,
663 OBJ_PROP_LINK_UNREF_ON_RELEASE,
664 &error_abort);
665 cpu->memory = system_memory;
666 object_ref(OBJECT(cpu->memory));
667 #endif
668
669 #if defined(CONFIG_USER_ONLY)
670 cpu_list_lock();
671 #endif
672 cpu->cpu_index = cpu_get_free_index(&local_err);
673 if (local_err) {
674 error_propagate(errp, local_err);
675 #if defined(CONFIG_USER_ONLY)
676 cpu_list_unlock();
677 #endif
678 return;
679 }
680 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
681 #if defined(CONFIG_USER_ONLY)
682 (void) cc;
683 cpu_list_unlock();
684 #else
685 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
686 vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
687 }
688 if (cc->vmsd != NULL) {
689 vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
690 }
691 #endif
692 }
693
694 #if defined(CONFIG_USER_ONLY)
695 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
696 {
697 tb_invalidate_phys_page_range(pc, pc + 1, 0);
698 }
699 #else
700 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
701 {
702 MemTxAttrs attrs;
703 hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
704 int asidx = cpu_asidx_from_attrs(cpu, attrs);
705 if (phys != -1) {
706 tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
707 phys | (pc & ~TARGET_PAGE_MASK));
708 }
709 }
710 #endif
711
712 #if defined(CONFIG_USER_ONLY)
713 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
714
715 {
716 }
717
718 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
719 int flags)
720 {
721 return -ENOSYS;
722 }
723
724 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
725 {
726 }
727
728 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
729 int flags, CPUWatchpoint **watchpoint)
730 {
731 return -ENOSYS;
732 }
733 #else
734 /* Add a watchpoint. */
735 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
736 int flags, CPUWatchpoint **watchpoint)
737 {
738 CPUWatchpoint *wp;
739
740 /* forbid ranges which are empty or run off the end of the address space */
741 if (len == 0 || (addr + len - 1) < addr) {
742 error_report("tried to set invalid watchpoint at %"
743 VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
744 return -EINVAL;
745 }
746 wp = g_malloc(sizeof(*wp));
747
748 wp->vaddr = addr;
749 wp->len = len;
750 wp->flags = flags;
751
752 /* keep all GDB-injected watchpoints in front */
753 if (flags & BP_GDB) {
754 QTAILQ_INSERT_HEAD(&cpu->watchpoints, wp, entry);
755 } else {
756 QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
757 }
758
759 tlb_flush_page(cpu, addr);
760
761 if (watchpoint)
762 *watchpoint = wp;
763 return 0;
764 }
765
766 /* Remove a specific watchpoint. */
767 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
768 int flags)
769 {
770 CPUWatchpoint *wp;
771
772 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
773 if (addr == wp->vaddr && len == wp->len
774 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
775 cpu_watchpoint_remove_by_ref(cpu, wp);
776 return 0;
777 }
778 }
779 return -ENOENT;
780 }
781
782 /* Remove a specific watchpoint by reference. */
783 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
784 {
785 QTAILQ_REMOVE(&cpu->watchpoints, watchpoint, entry);
786
787 tlb_flush_page(cpu, watchpoint->vaddr);
788
789 g_free(watchpoint);
790 }
791
792 /* Remove all matching watchpoints. */
793 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
794 {
795 CPUWatchpoint *wp, *next;
796
797 QTAILQ_FOREACH_SAFE(wp, &cpu->watchpoints, entry, next) {
798 if (wp->flags & mask) {
799 cpu_watchpoint_remove_by_ref(cpu, wp);
800 }
801 }
802 }
803
804 /* Return true if this watchpoint address matches the specified
805 * access (ie the address range covered by the watchpoint overlaps
806 * partially or completely with the address range covered by the
807 * access).
808 */
809 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
810 vaddr addr,
811 vaddr len)
812 {
813 /* We know the lengths are non-zero, but a little caution is
814 * required to avoid errors in the case where the range ends
815 * exactly at the top of the address space and so addr + len
816 * wraps round to zero.
817 */
818 vaddr wpend = wp->vaddr + wp->len - 1;
819 vaddr addrend = addr + len - 1;
820
821 return !(addr > wpend || wp->vaddr > addrend);
822 }
823
824 #endif
825
826 /* Add a breakpoint. */
827 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
828 CPUBreakpoint **breakpoint)
829 {
830 CPUBreakpoint *bp;
831
832 bp = g_malloc(sizeof(*bp));
833
834 bp->pc = pc;
835 bp->flags = flags;
836
837 /* keep all GDB-injected breakpoints in front */
838 if (flags & BP_GDB) {
839 QTAILQ_INSERT_HEAD(&cpu->breakpoints, bp, entry);
840 } else {
841 QTAILQ_INSERT_TAIL(&cpu->breakpoints, bp, entry);
842 }
843
844 breakpoint_invalidate(cpu, pc);
845
846 if (breakpoint) {
847 *breakpoint = bp;
848 }
849 return 0;
850 }
851
852 /* Remove a specific breakpoint. */
853 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
854 {
855 CPUBreakpoint *bp;
856
857 QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
858 if (bp->pc == pc && bp->flags == flags) {
859 cpu_breakpoint_remove_by_ref(cpu, bp);
860 return 0;
861 }
862 }
863 return -ENOENT;
864 }
865
866 /* Remove a specific breakpoint by reference. */
867 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
868 {
869 QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
870
871 breakpoint_invalidate(cpu, breakpoint->pc);
872
873 g_free(breakpoint);
874 }
875
876 /* Remove all matching breakpoints. */
877 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
878 {
879 CPUBreakpoint *bp, *next;
880
881 QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
882 if (bp->flags & mask) {
883 cpu_breakpoint_remove_by_ref(cpu, bp);
884 }
885 }
886 }
887
888 /* enable or disable single step mode. EXCP_DEBUG is returned by the
889 CPU loop after each instruction */
890 void cpu_single_step(CPUState *cpu, int enabled)
891 {
892 if (cpu->singlestep_enabled != enabled) {
893 cpu->singlestep_enabled = enabled;
894 if (kvm_enabled()) {
895 kvm_update_guest_debug(cpu, 0);
896 } else {
897 /* must flush all the translated code to avoid inconsistencies */
898 /* XXX: only flush what is necessary */
899 tb_flush(cpu);
900 }
901 }
902 }
903
904 void cpu_abort(CPUState *cpu, const char *fmt, ...)
905 {
906 va_list ap;
907 va_list ap2;
908
909 va_start(ap, fmt);
910 va_copy(ap2, ap);
911 fprintf(stderr, "qemu: fatal: ");
912 vfprintf(stderr, fmt, ap);
913 fprintf(stderr, "\n");
914 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
915 if (qemu_log_separate()) {
916 qemu_log("qemu: fatal: ");
917 qemu_log_vprintf(fmt, ap2);
918 qemu_log("\n");
919 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
920 qemu_log_flush();
921 qemu_log_close();
922 }
923 va_end(ap2);
924 va_end(ap);
925 replay_finish();
926 #if defined(CONFIG_USER_ONLY)
927 {
928 struct sigaction act;
929 sigfillset(&act.sa_mask);
930 act.sa_handler = SIG_DFL;
931 sigaction(SIGABRT, &act, NULL);
932 }
933 #endif
934 abort();
935 }
936
937 #if !defined(CONFIG_USER_ONLY)
938 /* Called from RCU critical section */
939 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
940 {
941 RAMBlock *block;
942
943 block = atomic_rcu_read(&ram_list.mru_block);
944 if (block && addr - block->offset < block->max_length) {
945 return block;
946 }
947 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
948 if (addr - block->offset < block->max_length) {
949 goto found;
950 }
951 }
952
953 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
954 abort();
955
956 found:
957 /* It is safe to write mru_block outside the iothread lock. This
958 * is what happens:
959 *
960 * mru_block = xxx
961 * rcu_read_unlock()
962 * xxx removed from list
963 * rcu_read_lock()
964 * read mru_block
965 * mru_block = NULL;
966 * call_rcu(reclaim_ramblock, xxx);
967 * rcu_read_unlock()
968 *
969 * atomic_rcu_set is not needed here. The block was already published
970 * when it was placed into the list. Here we're just making an extra
971 * copy of the pointer.
972 */
973 ram_list.mru_block = block;
974 return block;
975 }
976
977 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
978 {
979 CPUState *cpu;
980 ram_addr_t start1;
981 RAMBlock *block;
982 ram_addr_t end;
983
984 end = TARGET_PAGE_ALIGN(start + length);
985 start &= TARGET_PAGE_MASK;
986
987 rcu_read_lock();
988 block = qemu_get_ram_block(start);
989 assert(block == qemu_get_ram_block(end - 1));
990 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
991 CPU_FOREACH(cpu) {
992 tlb_reset_dirty(cpu, start1, length);
993 }
994 rcu_read_unlock();
995 }
996
997 /* Note: start and end must be within the same ram block. */
998 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
999 ram_addr_t length,
1000 unsigned client)
1001 {
1002 DirtyMemoryBlocks *blocks;
1003 unsigned long end, page;
1004 bool dirty = false;
1005
1006 if (length == 0) {
1007 return false;
1008 }
1009
1010 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
1011 page = start >> TARGET_PAGE_BITS;
1012
1013 rcu_read_lock();
1014
1015 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
1016
1017 while (page < end) {
1018 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
1019 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
1020 unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
1021
1022 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
1023 offset, num);
1024 page += num;
1025 }
1026
1027 rcu_read_unlock();
1028
1029 if (dirty && tcg_enabled()) {
1030 tlb_reset_dirty_range_all(start, length);
1031 }
1032
1033 return dirty;
1034 }
1035
1036 /* Called from RCU critical section */
1037 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
1038 MemoryRegionSection *section,
1039 target_ulong vaddr,
1040 hwaddr paddr, hwaddr xlat,
1041 int prot,
1042 target_ulong *address)
1043 {
1044 hwaddr iotlb;
1045 CPUWatchpoint *wp;
1046
1047 if (memory_region_is_ram(section->mr)) {
1048 /* Normal RAM. */
1049 iotlb = memory_region_get_ram_addr(section->mr) + xlat;
1050 if (!section->readonly) {
1051 iotlb |= PHYS_SECTION_NOTDIRTY;
1052 } else {
1053 iotlb |= PHYS_SECTION_ROM;
1054 }
1055 } else {
1056 AddressSpaceDispatch *d;
1057
1058 d = atomic_rcu_read(&section->address_space->dispatch);
1059 iotlb = section - d->map.sections;
1060 iotlb += xlat;
1061 }
1062
1063 /* Make accesses to pages with watchpoints go via the
1064 watchpoint trap routines. */
1065 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
1066 if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {
1067 /* Avoid trapping reads of pages with a write breakpoint. */
1068 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
1069 iotlb = PHYS_SECTION_WATCH + paddr;
1070 *address |= TLB_MMIO;
1071 break;
1072 }
1073 }
1074 }
1075
1076 return iotlb;
1077 }
1078 #endif /* defined(CONFIG_USER_ONLY) */
1079
1080 #if !defined(CONFIG_USER_ONLY)
1081
1082 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1083 uint16_t section);
1084 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
1085
1086 static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
1087 qemu_anon_ram_alloc;
1088
1089 /*
1090 * Set a custom physical guest memory alloator.
1091 * Accelerators with unusual needs may need this. Hopefully, we can
1092 * get rid of it eventually.
1093 */
1094 void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))
1095 {
1096 phys_mem_alloc = alloc;
1097 }
1098
1099 static uint16_t phys_section_add(PhysPageMap *map,
1100 MemoryRegionSection *section)
1101 {
1102 /* The physical section number is ORed with a page-aligned
1103 * pointer to produce the iotlb entries. Thus it should
1104 * never overflow into the page-aligned value.
1105 */
1106 assert(map->sections_nb < TARGET_PAGE_SIZE);
1107
1108 if (map->sections_nb == map->sections_nb_alloc) {
1109 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1110 map->sections = g_renew(MemoryRegionSection, map->sections,
1111 map->sections_nb_alloc);
1112 }
1113 map->sections[map->sections_nb] = *section;
1114 memory_region_ref(section->mr);
1115 return map->sections_nb++;
1116 }
1117
1118 static void phys_section_destroy(MemoryRegion *mr)
1119 {
1120 bool have_sub_page = mr->subpage;
1121
1122 memory_region_unref(mr);
1123
1124 if (have_sub_page) {
1125 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1126 object_unref(OBJECT(&subpage->iomem));
1127 g_free(subpage);
1128 }
1129 }
1130
1131 static void phys_sections_free(PhysPageMap *map)
1132 {
1133 while (map->sections_nb > 0) {
1134 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1135 phys_section_destroy(section->mr);
1136 }
1137 g_free(map->sections);
1138 g_free(map->nodes);
1139 }
1140
1141 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
1142 {
1143 subpage_t *subpage;
1144 hwaddr base = section->offset_within_address_space
1145 & TARGET_PAGE_MASK;
1146 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
1147 d->map.nodes, d->map.sections);
1148 MemoryRegionSection subsection = {
1149 .offset_within_address_space = base,
1150 .size = int128_make64(TARGET_PAGE_SIZE),
1151 };
1152 hwaddr start, end;
1153
1154 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1155
1156 if (!(existing->mr->subpage)) {
1157 subpage = subpage_init(d->as, base);
1158 subsection.address_space = d->as;
1159 subsection.mr = &subpage->iomem;
1160 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1161 phys_section_add(&d->map, &subsection));
1162 } else {
1163 subpage = container_of(existing->mr, subpage_t, iomem);
1164 }
1165 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1166 end = start + int128_get64(section->size) - 1;
1167 subpage_register(subpage, start, end,
1168 phys_section_add(&d->map, section));
1169 }
1170
1171
1172 static void register_multipage(AddressSpaceDispatch *d,
1173 MemoryRegionSection *section)
1174 {
1175 hwaddr start_addr = section->offset_within_address_space;
1176 uint16_t section_index = phys_section_add(&d->map, section);
1177 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1178 TARGET_PAGE_BITS));
1179
1180 assert(num_pages);
1181 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1182 }
1183
1184 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
1185 {
1186 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1187 AddressSpaceDispatch *d = as->next_dispatch;
1188 MemoryRegionSection now = *section, remain = *section;
1189 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1190
1191 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
1192 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
1193 - now.offset_within_address_space;
1194
1195 now.size = int128_min(int128_make64(left), now.size);
1196 register_subpage(d, &now);
1197 } else {
1198 now.size = int128_zero();
1199 }
1200 while (int128_ne(remain.size, now.size)) {
1201 remain.size = int128_sub(remain.size, now.size);
1202 remain.offset_within_address_space += int128_get64(now.size);
1203 remain.offset_within_region += int128_get64(now.size);
1204 now = remain;
1205 if (int128_lt(remain.size, page_size)) {
1206 register_subpage(d, &now);
1207 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1208 now.size = page_size;
1209 register_subpage(d, &now);
1210 } else {
1211 now.size = int128_and(now.size, int128_neg(page_size));
1212 register_multipage(d, &now);
1213 }
1214 }
1215 }
1216
1217 void qemu_flush_coalesced_mmio_buffer(void)
1218 {
1219 if (kvm_enabled())
1220 kvm_flush_coalesced_mmio_buffer();
1221 }
1222
1223 void qemu_mutex_lock_ramlist(void)
1224 {
1225 qemu_mutex_lock(&ram_list.mutex);
1226 }
1227
1228 void qemu_mutex_unlock_ramlist(void)
1229 {
1230 qemu_mutex_unlock(&ram_list.mutex);
1231 }
1232
1233 #ifdef __linux__
1234 static void *file_ram_alloc(RAMBlock *block,
1235 ram_addr_t memory,
1236 const char *path,
1237 Error **errp)
1238 {
1239 bool unlink_on_error = false;
1240 char *filename;
1241 char *sanitized_name;
1242 char *c;
1243 void *area;
1244 int fd = -1;
1245 int64_t page_size;
1246
1247 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1248 error_setg(errp,
1249 "host lacks kvm mmu notifiers, -mem-path unsupported");
1250 return NULL;
1251 }
1252
1253 for (;;) {
1254 fd = open(path, O_RDWR);
1255 if (fd >= 0) {
1256 /* @path names an existing file, use it */
1257 break;
1258 }
1259 if (errno == ENOENT) {
1260 /* @path names a file that doesn't exist, create it */
1261 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1262 if (fd >= 0) {
1263 unlink_on_error = true;
1264 break;
1265 }
1266 } else if (errno == EISDIR) {
1267 /* @path names a directory, create a file there */
1268 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1269 sanitized_name = g_strdup(memory_region_name(block->mr));
1270 for (c = sanitized_name; *c != '\0'; c++) {
1271 if (*c == '/') {
1272 *c = '_';
1273 }
1274 }
1275
1276 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1277 sanitized_name);
1278 g_free(sanitized_name);
1279
1280 fd = mkstemp(filename);
1281 if (fd >= 0) {
1282 unlink(filename);
1283 g_free(filename);
1284 break;
1285 }
1286 g_free(filename);
1287 }
1288 if (errno != EEXIST && errno != EINTR) {
1289 error_setg_errno(errp, errno,
1290 "can't open backing store %s for guest RAM",
1291 path);
1292 goto error;
1293 }
1294 /*
1295 * Try again on EINTR and EEXIST. The latter happens when
1296 * something else creates the file between our two open().
1297 */
1298 }
1299
1300 page_size = qemu_fd_getpagesize(fd);
1301 block->mr->align = MAX(page_size, QEMU_VMALLOC_ALIGN);
1302
1303 if (memory < page_size) {
1304 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1305 "or larger than page size 0x%" PRIx64,
1306 memory, page_size);
1307 goto error;
1308 }
1309
1310 memory = ROUND_UP(memory, page_size);
1311
1312 /*
1313 * ftruncate is not supported by hugetlbfs in older
1314 * hosts, so don't bother bailing out on errors.
1315 * If anything goes wrong with it under other filesystems,
1316 * mmap will fail.
1317 */
1318 if (ftruncate(fd, memory)) {
1319 perror("ftruncate");
1320 }
1321
1322 area = qemu_ram_mmap(fd, memory, block->mr->align,
1323 block->flags & RAM_SHARED);
1324 if (area == MAP_FAILED) {
1325 error_setg_errno(errp, errno,
1326 "unable to map backing store for guest RAM");
1327 goto error;
1328 }
1329
1330 if (mem_prealloc) {
1331 os_mem_prealloc(fd, area, memory);
1332 }
1333
1334 block->fd = fd;
1335 return area;
1336
1337 error:
1338 if (unlink_on_error) {
1339 unlink(path);
1340 }
1341 if (fd != -1) {
1342 close(fd);
1343 }
1344 return NULL;
1345 }
1346 #endif
1347
1348 /* Called with the ramlist lock held. */
1349 static ram_addr_t find_ram_offset(ram_addr_t size)
1350 {
1351 RAMBlock *block, *next_block;
1352 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1353
1354 assert(size != 0); /* it would hand out same offset multiple times */
1355
1356 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1357 return 0;
1358 }
1359
1360 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1361 ram_addr_t end, next = RAM_ADDR_MAX;
1362
1363 end = block->offset + block->max_length;
1364
1365 QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
1366 if (next_block->offset >= end) {
1367 next = MIN(next, next_block->offset);
1368 }
1369 }
1370 if (next - end >= size && next - end < mingap) {
1371 offset = end;
1372 mingap = next - end;
1373 }
1374 }
1375
1376 if (offset == RAM_ADDR_MAX) {
1377 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1378 (uint64_t)size);
1379 abort();
1380 }
1381
1382 return offset;
1383 }
1384
1385 ram_addr_t last_ram_offset(void)
1386 {
1387 RAMBlock *block;
1388 ram_addr_t last = 0;
1389
1390 rcu_read_lock();
1391 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1392 last = MAX(last, block->offset + block->max_length);
1393 }
1394 rcu_read_unlock();
1395 return last;
1396 }
1397
1398 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1399 {
1400 int ret;
1401
1402 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1403 if (!machine_dump_guest_core(current_machine)) {
1404 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1405 if (ret) {
1406 perror("qemu_madvise");
1407 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1408 "but dump_guest_core=off specified\n");
1409 }
1410 }
1411 }
1412
1413 const char *qemu_ram_get_idstr(RAMBlock *rb)
1414 {
1415 return rb->idstr;
1416 }
1417
1418 /* Called with iothread lock held. */
1419 void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
1420 {
1421 RAMBlock *block;
1422
1423 assert(new_block);
1424 assert(!new_block->idstr[0]);
1425
1426 if (dev) {
1427 char *id = qdev_get_dev_path(dev);
1428 if (id) {
1429 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1430 g_free(id);
1431 }
1432 }
1433 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1434
1435 rcu_read_lock();
1436 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1437 if (block != new_block &&
1438 !strcmp(block->idstr, new_block->idstr)) {
1439 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1440 new_block->idstr);
1441 abort();
1442 }
1443 }
1444 rcu_read_unlock();
1445 }
1446
1447 /* Called with iothread lock held. */
1448 void qemu_ram_unset_idstr(RAMBlock *block)
1449 {
1450 /* FIXME: arch_init.c assumes that this is not called throughout
1451 * migration. Ignore the problem since hot-unplug during migration
1452 * does not work anyway.
1453 */
1454 if (block) {
1455 memset(block->idstr, 0, sizeof(block->idstr));
1456 }
1457 }
1458
1459 static int memory_try_enable_merging(void *addr, size_t len)
1460 {
1461 if (!machine_mem_merge(current_machine)) {
1462 /* disabled by the user */
1463 return 0;
1464 }
1465
1466 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1467 }
1468
1469 /* Only legal before guest might have detected the memory size: e.g. on
1470 * incoming migration, or right after reset.
1471 *
1472 * As memory core doesn't know how is memory accessed, it is up to
1473 * resize callback to update device state and/or add assertions to detect
1474 * misuse, if necessary.
1475 */
1476 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
1477 {
1478 assert(block);
1479
1480 newsize = HOST_PAGE_ALIGN(newsize);
1481
1482 if (block->used_length == newsize) {
1483 return 0;
1484 }
1485
1486 if (!(block->flags & RAM_RESIZEABLE)) {
1487 error_setg_errno(errp, EINVAL,
1488 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1489 " in != 0x" RAM_ADDR_FMT, block->idstr,
1490 newsize, block->used_length);
1491 return -EINVAL;
1492 }
1493
1494 if (block->max_length < newsize) {
1495 error_setg_errno(errp, EINVAL,
1496 "Length too large: %s: 0x" RAM_ADDR_FMT
1497 " > 0x" RAM_ADDR_FMT, block->idstr,
1498 newsize, block->max_length);
1499 return -EINVAL;
1500 }
1501
1502 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1503 block->used_length = newsize;
1504 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1505 DIRTY_CLIENTS_ALL);
1506 memory_region_set_size(block->mr, newsize);
1507 if (block->resized) {
1508 block->resized(block->idstr, newsize, block->host);
1509 }
1510 return 0;
1511 }
1512
1513 /* Called with ram_list.mutex held */
1514 static void dirty_memory_extend(ram_addr_t old_ram_size,
1515 ram_addr_t new_ram_size)
1516 {
1517 ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
1518 DIRTY_MEMORY_BLOCK_SIZE);
1519 ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
1520 DIRTY_MEMORY_BLOCK_SIZE);
1521 int i;
1522
1523 /* Only need to extend if block count increased */
1524 if (new_num_blocks <= old_num_blocks) {
1525 return;
1526 }
1527
1528 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1529 DirtyMemoryBlocks *old_blocks;
1530 DirtyMemoryBlocks *new_blocks;
1531 int j;
1532
1533 old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
1534 new_blocks = g_malloc(sizeof(*new_blocks) +
1535 sizeof(new_blocks->blocks[0]) * new_num_blocks);
1536
1537 if (old_num_blocks) {
1538 memcpy(new_blocks->blocks, old_blocks->blocks,
1539 old_num_blocks * sizeof(old_blocks->blocks[0]));
1540 }
1541
1542 for (j = old_num_blocks; j < new_num_blocks; j++) {
1543 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1544 }
1545
1546 atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
1547
1548 if (old_blocks) {
1549 g_free_rcu(old_blocks, rcu);
1550 }
1551 }
1552 }
1553
1554 static void ram_block_add(RAMBlock *new_block, Error **errp)
1555 {
1556 RAMBlock *block;
1557 RAMBlock *last_block = NULL;
1558 ram_addr_t old_ram_size, new_ram_size;
1559 Error *err = NULL;
1560
1561 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1562
1563 qemu_mutex_lock_ramlist();
1564 new_block->offset = find_ram_offset(new_block->max_length);
1565
1566 if (!new_block->host) {
1567 if (xen_enabled()) {
1568 xen_ram_alloc(new_block->offset, new_block->max_length,
1569 new_block->mr, &err);
1570 if (err) {
1571 error_propagate(errp, err);
1572 qemu_mutex_unlock_ramlist();
1573 return;
1574 }
1575 } else {
1576 new_block->host = phys_mem_alloc(new_block->max_length,
1577 &new_block->mr->align);
1578 if (!new_block->host) {
1579 error_setg_errno(errp, errno,
1580 "cannot set up guest memory '%s'",
1581 memory_region_name(new_block->mr));
1582 qemu_mutex_unlock_ramlist();
1583 return;
1584 }
1585 memory_try_enable_merging(new_block->host, new_block->max_length);
1586 }
1587 }
1588
1589 new_ram_size = MAX(old_ram_size,
1590 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
1591 if (new_ram_size > old_ram_size) {
1592 migration_bitmap_extend(old_ram_size, new_ram_size);
1593 dirty_memory_extend(old_ram_size, new_ram_size);
1594 }
1595 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1596 * QLIST (which has an RCU-friendly variant) does not have insertion at
1597 * tail, so save the last element in last_block.
1598 */
1599 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1600 last_block = block;
1601 if (block->max_length < new_block->max_length) {
1602 break;
1603 }
1604 }
1605 if (block) {
1606 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1607 } else if (last_block) {
1608 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1609 } else { /* list is empty */
1610 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1611 }
1612 ram_list.mru_block = NULL;
1613
1614 /* Write list before version */
1615 smp_wmb();
1616 ram_list.version++;
1617 qemu_mutex_unlock_ramlist();
1618
1619 cpu_physical_memory_set_dirty_range(new_block->offset,
1620 new_block->used_length,
1621 DIRTY_CLIENTS_ALL);
1622
1623 if (new_block->host) {
1624 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1625 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1626 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
1627 if (kvm_enabled()) {
1628 kvm_setup_guest_memory(new_block->host, new_block->max_length);
1629 }
1630 }
1631 }
1632
1633 #ifdef __linux__
1634 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
1635 bool share, const char *mem_path,
1636 Error **errp)
1637 {
1638 RAMBlock *new_block;
1639 Error *local_err = NULL;
1640
1641 if (xen_enabled()) {
1642 error_setg(errp, "-mem-path not supported with Xen");
1643 return NULL;
1644 }
1645
1646 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1647 /*
1648 * file_ram_alloc() needs to allocate just like
1649 * phys_mem_alloc, but we haven't bothered to provide
1650 * a hook there.
1651 */
1652 error_setg(errp,
1653 "-mem-path not supported with this accelerator");
1654 return NULL;
1655 }
1656
1657 size = HOST_PAGE_ALIGN(size);
1658 new_block = g_malloc0(sizeof(*new_block));
1659 new_block->mr = mr;
1660 new_block->used_length = size;
1661 new_block->max_length = size;
1662 new_block->flags = share ? RAM_SHARED : 0;
1663 new_block->host = file_ram_alloc(new_block, size,
1664 mem_path, errp);
1665 if (!new_block->host) {
1666 g_free(new_block);
1667 return NULL;
1668 }
1669
1670 ram_block_add(new_block, &local_err);
1671 if (local_err) {
1672 g_free(new_block);
1673 error_propagate(errp, local_err);
1674 return NULL;
1675 }
1676 return new_block;
1677 }
1678 #endif
1679
1680 static
1681 RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
1682 void (*resized)(const char*,
1683 uint64_t length,
1684 void *host),
1685 void *host, bool resizeable,
1686 MemoryRegion *mr, Error **errp)
1687 {
1688 RAMBlock *new_block;
1689 Error *local_err = NULL;
1690
1691 size = HOST_PAGE_ALIGN(size);
1692 max_size = HOST_PAGE_ALIGN(max_size);
1693 new_block = g_malloc0(sizeof(*new_block));
1694 new_block->mr = mr;
1695 new_block->resized = resized;
1696 new_block->used_length = size;
1697 new_block->max_length = max_size;
1698 assert(max_size >= size);
1699 new_block->fd = -1;
1700 new_block->host = host;
1701 if (host) {
1702 new_block->flags |= RAM_PREALLOC;
1703 }
1704 if (resizeable) {
1705 new_block->flags |= RAM_RESIZEABLE;
1706 }
1707 ram_block_add(new_block, &local_err);
1708 if (local_err) {
1709 g_free(new_block);
1710 error_propagate(errp, local_err);
1711 return NULL;
1712 }
1713 return new_block;
1714 }
1715
1716 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1717 MemoryRegion *mr, Error **errp)
1718 {
1719 return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
1720 }
1721
1722 RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
1723 {
1724 return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
1725 }
1726
1727 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
1728 void (*resized)(const char*,
1729 uint64_t length,
1730 void *host),
1731 MemoryRegion *mr, Error **errp)
1732 {
1733 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
1734 }
1735
1736 static void reclaim_ramblock(RAMBlock *block)
1737 {
1738 if (block->flags & RAM_PREALLOC) {
1739 ;
1740 } else if (xen_enabled()) {
1741 xen_invalidate_map_cache_entry(block->host);
1742 #ifndef _WIN32
1743 } else if (block->fd >= 0) {
1744 qemu_ram_munmap(block->host, block->max_length);
1745 close(block->fd);
1746 #endif
1747 } else {
1748 qemu_anon_ram_free(block->host, block->max_length);
1749 }
1750 g_free(block);
1751 }
1752
1753 void qemu_ram_free(RAMBlock *block)
1754 {
1755 if (!block) {
1756 return;
1757 }
1758
1759 qemu_mutex_lock_ramlist();
1760 QLIST_REMOVE_RCU(block, next);
1761 ram_list.mru_block = NULL;
1762 /* Write list before version */
1763 smp_wmb();
1764 ram_list.version++;
1765 call_rcu(block, reclaim_ramblock, rcu);
1766 qemu_mutex_unlock_ramlist();
1767 }
1768
1769 #ifndef _WIN32
1770 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1771 {
1772 RAMBlock *block;
1773 ram_addr_t offset;
1774 int flags;
1775 void *area, *vaddr;
1776
1777 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1778 offset = addr - block->offset;
1779 if (offset < block->max_length) {
1780 vaddr = ramblock_ptr(block, offset);
1781 if (block->flags & RAM_PREALLOC) {
1782 ;
1783 } else if (xen_enabled()) {
1784 abort();
1785 } else {
1786 flags = MAP_FIXED;
1787 if (block->fd >= 0) {
1788 flags |= (block->flags & RAM_SHARED ?
1789 MAP_SHARED : MAP_PRIVATE);
1790 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1791 flags, block->fd, offset);
1792 } else {
1793 /*
1794 * Remap needs to match alloc. Accelerators that
1795 * set phys_mem_alloc never remap. If they did,
1796 * we'd need a remap hook here.
1797 */
1798 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1799
1800 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1801 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1802 flags, -1, 0);
1803 }
1804 if (area != vaddr) {
1805 fprintf(stderr, "Could not remap addr: "
1806 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1807 length, addr);
1808 exit(1);
1809 }
1810 memory_try_enable_merging(vaddr, length);
1811 qemu_ram_setup_dump(vaddr, length);
1812 }
1813 }
1814 }
1815 }
1816 #endif /* !_WIN32 */
1817
1818 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1819 * This should not be used for general purpose DMA. Use address_space_map
1820 * or address_space_rw instead. For local memory (e.g. video ram) that the
1821 * device owns, use memory_region_get_ram_ptr.
1822 *
1823 * Called within RCU critical section.
1824 */
1825 void *qemu_get_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
1826 {
1827 RAMBlock *block = ram_block;
1828
1829 if (block == NULL) {
1830 block = qemu_get_ram_block(addr);
1831 }
1832
1833 if (xen_enabled() && block->host == NULL) {
1834 /* We need to check if the requested address is in the RAM
1835 * because we don't want to map the entire memory in QEMU.
1836 * In that case just map until the end of the page.
1837 */
1838 if (block->offset == 0) {
1839 return xen_map_cache(addr, 0, 0);
1840 }
1841
1842 block->host = xen_map_cache(block->offset, block->max_length, 1);
1843 }
1844 return ramblock_ptr(block, addr - block->offset);
1845 }
1846
1847 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1848 * but takes a size argument.
1849 *
1850 * Called within RCU critical section.
1851 */
1852 static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
1853 hwaddr *size)
1854 {
1855 RAMBlock *block = ram_block;
1856 ram_addr_t offset_inside_block;
1857 if (*size == 0) {
1858 return NULL;
1859 }
1860
1861 if (block == NULL) {
1862 block = qemu_get_ram_block(addr);
1863 }
1864 offset_inside_block = addr - block->offset;
1865 *size = MIN(*size, block->max_length - offset_inside_block);
1866
1867 if (xen_enabled() && block->host == NULL) {
1868 /* We need to check if the requested address is in the RAM
1869 * because we don't want to map the entire memory in QEMU.
1870 * In that case just map the requested area.
1871 */
1872 if (block->offset == 0) {
1873 return xen_map_cache(addr, *size, 1);
1874 }
1875
1876 block->host = xen_map_cache(block->offset, block->max_length, 1);
1877 }
1878
1879 return ramblock_ptr(block, offset_inside_block);
1880 }
1881
1882 /*
1883 * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
1884 * in that RAMBlock.
1885 *
1886 * ptr: Host pointer to look up
1887 * round_offset: If true round the result offset down to a page boundary
1888 * *ram_addr: set to result ram_addr
1889 * *offset: set to result offset within the RAMBlock
1890 *
1891 * Returns: RAMBlock (or NULL if not found)
1892 *
1893 * By the time this function returns, the returned pointer is not protected
1894 * by RCU anymore. If the caller is not within an RCU critical section and
1895 * does not hold the iothread lock, it must have other means of protecting the
1896 * pointer, such as a reference to the region that includes the incoming
1897 * ram_addr_t.
1898 */
1899 RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
1900 ram_addr_t *offset)
1901 {
1902 RAMBlock *block;
1903 uint8_t *host = ptr;
1904
1905 if (xen_enabled()) {
1906 ram_addr_t ram_addr;
1907 rcu_read_lock();
1908 ram_addr = xen_ram_addr_from_mapcache(ptr);
1909 block = qemu_get_ram_block(ram_addr);
1910 if (block) {
1911 *offset = (host - block->host);
1912 }
1913 rcu_read_unlock();
1914 return block;
1915 }
1916
1917 rcu_read_lock();
1918 block = atomic_rcu_read(&ram_list.mru_block);
1919 if (block && block->host && host - block->host < block->max_length) {
1920 goto found;
1921 }
1922
1923 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1924 /* This case append when the block is not mapped. */
1925 if (block->host == NULL) {
1926 continue;
1927 }
1928 if (host - block->host < block->max_length) {
1929 goto found;
1930 }
1931 }
1932
1933 rcu_read_unlock();
1934 return NULL;
1935
1936 found:
1937 *offset = (host - block->host);
1938 if (round_offset) {
1939 *offset &= TARGET_PAGE_MASK;
1940 }
1941 rcu_read_unlock();
1942 return block;
1943 }
1944
1945 /*
1946 * Finds the named RAMBlock
1947 *
1948 * name: The name of RAMBlock to find
1949 *
1950 * Returns: RAMBlock (or NULL if not found)
1951 */
1952 RAMBlock *qemu_ram_block_by_name(const char *name)
1953 {
1954 RAMBlock *block;
1955
1956 QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
1957 if (!strcmp(name, block->idstr)) {
1958 return block;
1959 }
1960 }
1961
1962 return NULL;
1963 }
1964
1965 /* Some of the softmmu routines need to translate from a host pointer
1966 (typically a TLB entry) back to a ram offset. */
1967 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1968 {
1969 RAMBlock *block;
1970 ram_addr_t offset;
1971
1972 block = qemu_ram_block_from_host(ptr, false, &offset);
1973 *ram_addr = block->offset + offset;
1974 if (!block) {
1975 return NULL;
1976 }
1977
1978 return block->mr;
1979 }
1980
1981 /* Called within RCU critical section. */
1982 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1983 uint64_t val, unsigned size)
1984 {
1985 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1986 tb_invalidate_phys_page_fast(ram_addr, size);
1987 }
1988 switch (size) {
1989 case 1:
1990 stb_p(qemu_get_ram_ptr(NULL, ram_addr), val);
1991 break;
1992 case 2:
1993 stw_p(qemu_get_ram_ptr(NULL, ram_addr), val);
1994 break;
1995 case 4:
1996 stl_p(qemu_get_ram_ptr(NULL, ram_addr), val);
1997 break;
1998 default:
1999 abort();
2000 }
2001 /* Set both VGA and migration bits for simplicity and to remove
2002 * the notdirty callback faster.
2003 */
2004 cpu_physical_memory_set_dirty_range(ram_addr, size,
2005 DIRTY_CLIENTS_NOCODE);
2006 /* we remove the notdirty callback only if the code has been
2007 flushed */
2008 if (!cpu_physical_memory_is_clean(ram_addr)) {
2009 tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
2010 }
2011 }
2012
2013 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
2014 unsigned size, bool is_write)
2015 {
2016 return is_write;
2017 }
2018
2019 static const MemoryRegionOps notdirty_mem_ops = {
2020 .write = notdirty_mem_write,
2021 .valid.accepts = notdirty_mem_accepts,
2022 .endianness = DEVICE_NATIVE_ENDIAN,
2023 };
2024
2025 /* Generate a debug exception if a watchpoint has been hit. */
2026 static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
2027 {
2028 CPUState *cpu = current_cpu;
2029 CPUClass *cc = CPU_GET_CLASS(cpu);
2030 CPUArchState *env = cpu->env_ptr;
2031 target_ulong pc, cs_base;
2032 target_ulong vaddr;
2033 CPUWatchpoint *wp;
2034 uint32_t cpu_flags;
2035
2036 if (cpu->watchpoint_hit) {
2037 /* We re-entered the check after replacing the TB. Now raise
2038 * the debug interrupt so that is will trigger after the
2039 * current instruction. */
2040 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
2041 return;
2042 }
2043 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2044 QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
2045 if (cpu_watchpoint_address_matches(wp, vaddr, len)
2046 && (wp->flags & flags)) {
2047 if (flags == BP_MEM_READ) {
2048 wp->flags |= BP_WATCHPOINT_HIT_READ;
2049 } else {
2050 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
2051 }
2052 wp->hitaddr = vaddr;
2053 wp->hitattrs = attrs;
2054 if (!cpu->watchpoint_hit) {
2055 if (wp->flags & BP_CPU &&
2056 !cc->debug_check_watchpoint(cpu, wp)) {
2057 wp->flags &= ~BP_WATCHPOINT_HIT;
2058 continue;
2059 }
2060 cpu->watchpoint_hit = wp;
2061 tb_check_watchpoint(cpu);
2062 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2063 cpu->exception_index = EXCP_DEBUG;
2064 cpu_loop_exit(cpu);
2065 } else {
2066 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2067 tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
2068 cpu_resume_from_signal(cpu, NULL);
2069 }
2070 }
2071 } else {
2072 wp->flags &= ~BP_WATCHPOINT_HIT;
2073 }
2074 }
2075 }
2076
2077 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2078 so these check for a hit then pass through to the normal out-of-line
2079 phys routines. */
2080 static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
2081 unsigned size, MemTxAttrs attrs)
2082 {
2083 MemTxResult res;
2084 uint64_t data;
2085 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2086 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2087
2088 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
2089 switch (size) {
2090 case 1:
2091 data = address_space_ldub(as, addr, attrs, &res);
2092 break;
2093 case 2:
2094 data = address_space_lduw(as, addr, attrs, &res);
2095 break;
2096 case 4:
2097 data = address_space_ldl(as, addr, attrs, &res);
2098 break;
2099 default: abort();
2100 }
2101 *pdata = data;
2102 return res;
2103 }
2104
2105 static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
2106 uint64_t val, unsigned size,
2107 MemTxAttrs attrs)
2108 {
2109 MemTxResult res;
2110 int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
2111 AddressSpace *as = current_cpu->cpu_ases[asidx].as;
2112
2113 check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
2114 switch (size) {
2115 case 1:
2116 address_space_stb(as, addr, val, attrs, &res);
2117 break;
2118 case 2:
2119 address_space_stw(as, addr, val, attrs, &res);
2120 break;
2121 case 4:
2122 address_space_stl(as, addr, val, attrs, &res);
2123 break;
2124 default: abort();
2125 }
2126 return res;
2127 }
2128
2129 static const MemoryRegionOps watch_mem_ops = {
2130 .read_with_attrs = watch_mem_read,
2131 .write_with_attrs = watch_mem_write,
2132 .endianness = DEVICE_NATIVE_ENDIAN,
2133 };
2134
2135 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2136 unsigned len, MemTxAttrs attrs)
2137 {
2138 subpage_t *subpage = opaque;
2139 uint8_t buf[8];
2140 MemTxResult res;
2141
2142 #if defined(DEBUG_SUBPAGE)
2143 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
2144 subpage, len, addr);
2145 #endif
2146 res = address_space_read(subpage->as, addr + subpage->base,
2147 attrs, buf, len);
2148 if (res) {
2149 return res;
2150 }
2151 switch (len) {
2152 case 1:
2153 *data = ldub_p(buf);
2154 return MEMTX_OK;
2155 case 2:
2156 *data = lduw_p(buf);
2157 return MEMTX_OK;
2158 case 4:
2159 *data = ldl_p(buf);
2160 return MEMTX_OK;
2161 case 8:
2162 *data = ldq_p(buf);
2163 return MEMTX_OK;
2164 default:
2165 abort();
2166 }
2167 }
2168
2169 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2170 uint64_t value, unsigned len, MemTxAttrs attrs)
2171 {
2172 subpage_t *subpage = opaque;
2173 uint8_t buf[8];
2174
2175 #if defined(DEBUG_SUBPAGE)
2176 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2177 " value %"PRIx64"\n",
2178 __func__, subpage, len, addr, value);
2179 #endif
2180 switch (len) {
2181 case 1:
2182 stb_p(buf, value);
2183 break;
2184 case 2:
2185 stw_p(buf, value);
2186 break;
2187 case 4:
2188 stl_p(buf, value);
2189 break;
2190 case 8:
2191 stq_p(buf, value);
2192 break;
2193 default:
2194 abort();
2195 }
2196 return address_space_write(subpage->as, addr + subpage->base,
2197 attrs, buf, len);
2198 }
2199
2200 static bool subpage_accepts(void *opaque, hwaddr addr,
2201 unsigned len, bool is_write)
2202 {
2203 subpage_t *subpage = opaque;
2204 #if defined(DEBUG_SUBPAGE)
2205 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
2206 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2207 #endif
2208
2209 return address_space_access_valid(subpage->as, addr + subpage->base,
2210 len, is_write);
2211 }
2212
2213 static const MemoryRegionOps subpage_ops = {
2214 .read_with_attrs = subpage_read,
2215 .write_with_attrs = subpage_write,
2216 .impl.min_access_size = 1,
2217 .impl.max_access_size = 8,
2218 .valid.min_access_size = 1,
2219 .valid.max_access_size = 8,
2220 .valid.accepts = subpage_accepts,
2221 .endianness = DEVICE_NATIVE_ENDIAN,
2222 };
2223
2224 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2225 uint16_t section)
2226 {
2227 int idx, eidx;
2228
2229 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2230 return -1;
2231 idx = SUBPAGE_IDX(start);
2232 eidx = SUBPAGE_IDX(end);
2233 #if defined(DEBUG_SUBPAGE)
2234 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2235 __func__, mmio, start, end, idx, eidx, section);
2236 #endif
2237 for (; idx <= eidx; idx++) {
2238 mmio->sub_section[idx] = section;
2239 }
2240
2241 return 0;
2242 }
2243
2244 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
2245 {
2246 subpage_t *mmio;
2247
2248 mmio = g_malloc0(sizeof(subpage_t));
2249
2250 mmio->as = as;
2251 mmio->base = base;
2252 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2253 NULL, TARGET_PAGE_SIZE);
2254 mmio->iomem.subpage = true;
2255 #if defined(DEBUG_SUBPAGE)
2256 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
2257 mmio, base, TARGET_PAGE_SIZE);
2258 #endif
2259 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
2260
2261 return mmio;
2262 }
2263
2264 static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
2265 MemoryRegion *mr)
2266 {
2267 assert(as);
2268 MemoryRegionSection section = {
2269 .address_space = as,
2270 .mr = mr,
2271 .offset_within_address_space = 0,
2272 .offset_within_region = 0,
2273 .size = int128_2_64(),
2274 };
2275
2276 return phys_section_add(map, &section);
2277 }
2278
2279 MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index, MemTxAttrs attrs)
2280 {
2281 int asidx = cpu_asidx_from_attrs(cpu, attrs);
2282 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
2283 AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
2284 MemoryRegionSection *sections = d->map.sections;
2285
2286 return sections[index & ~TARGET_PAGE_MASK].mr;
2287 }
2288
2289 static void io_mem_init(void)
2290 {
2291 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);
2292 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2293 NULL, UINT64_MAX);
2294 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
2295 NULL, UINT64_MAX);
2296 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
2297 NULL, UINT64_MAX);
2298 }
2299
2300 static void mem_begin(MemoryListener *listener)
2301 {
2302 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2303 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2304 uint16_t n;
2305
2306 n = dummy_section(&d->map, as, &io_mem_unassigned);
2307 assert(n == PHYS_SECTION_UNASSIGNED);
2308 n = dummy_section(&d->map, as, &io_mem_notdirty);
2309 assert(n == PHYS_SECTION_NOTDIRTY);
2310 n = dummy_section(&d->map, as, &io_mem_rom);
2311 assert(n == PHYS_SECTION_ROM);
2312 n = dummy_section(&d->map, as, &io_mem_watch);
2313 assert(n == PHYS_SECTION_WATCH);
2314
2315 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2316 d->as = as;
2317 as->next_dispatch = d;
2318 }
2319
2320 static void address_space_dispatch_free(AddressSpaceDispatch *d)
2321 {
2322 phys_sections_free(&d->map);
2323 g_free(d);
2324 }
2325
2326 static void mem_commit(MemoryListener *listener)
2327 {
2328 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
2329 AddressSpaceDispatch *cur = as->dispatch;
2330 AddressSpaceDispatch *next = as->next_dispatch;
2331
2332 phys_page_compact_all(next, next->map.nodes_nb);
2333
2334 atomic_rcu_set(&as->dispatch, next);
2335 if (cur) {
2336 call_rcu(cur, address_space_dispatch_free, rcu);
2337 }
2338 }
2339
2340 static void tcg_commit(MemoryListener *listener)
2341 {
2342 CPUAddressSpace *cpuas;
2343 AddressSpaceDispatch *d;
2344
2345 /* since each CPU stores ram addresses in its TLB cache, we must
2346 reset the modified entries */
2347 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2348 cpu_reloading_memory_map();
2349 /* The CPU and TLB are protected by the iothread lock.
2350 * We reload the dispatch pointer now because cpu_reloading_memory_map()
2351 * may have split the RCU critical section.
2352 */
2353 d = atomic_rcu_read(&cpuas->as->dispatch);
2354 cpuas->memory_dispatch = d;
2355 tlb_flush(cpuas->cpu, 1);
2356 }
2357
2358 void address_space_init_dispatch(AddressSpace *as)
2359 {
2360 as->dispatch = NULL;
2361 as->dispatch_listener = (MemoryListener) {
2362 .begin = mem_begin,
2363 .commit = mem_commit,
2364 .region_add = mem_add,
2365 .region_nop = mem_add,
2366 .priority = 0,
2367 };
2368 memory_listener_register(&as->dispatch_listener, as);
2369 }
2370
2371 void address_space_unregister(AddressSpace *as)
2372 {
2373 memory_listener_unregister(&as->dispatch_listener);
2374 }
2375
2376 void address_space_destroy_dispatch(AddressSpace *as)
2377 {
2378 AddressSpaceDispatch *d = as->dispatch;
2379
2380 atomic_rcu_set(&as->dispatch, NULL);
2381 if (d) {
2382 call_rcu(d, address_space_dispatch_free, rcu);
2383 }
2384 }
2385
2386 static void memory_map_init(void)
2387 {
2388 system_memory = g_malloc(sizeof(*system_memory));
2389
2390 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2391 address_space_init(&address_space_memory, system_memory, "memory");
2392
2393 system_io = g_malloc(sizeof(*system_io));
2394 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2395 65536);
2396 address_space_init(&address_space_io, system_io, "I/O");
2397 }
2398
2399 MemoryRegion *get_system_memory(void)
2400 {
2401 return system_memory;
2402 }
2403
2404 MemoryRegion *get_system_io(void)
2405 {
2406 return system_io;
2407 }
2408
2409 #endif /* !defined(CONFIG_USER_ONLY) */
2410
2411 /* physical memory access (slow version, mainly for debug) */
2412 #if defined(CONFIG_USER_ONLY)
2413 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2414 uint8_t *buf, int len, int is_write)
2415 {
2416 int l, flags;
2417 target_ulong page;
2418 void * p;
2419
2420 while (len > 0) {
2421 page = addr & TARGET_PAGE_MASK;
2422 l = (page + TARGET_PAGE_SIZE) - addr;
2423 if (l > len)
2424 l = len;
2425 flags = page_get_flags(page);
2426 if (!(flags & PAGE_VALID))
2427 return -1;
2428 if (is_write) {
2429 if (!(flags & PAGE_WRITE))
2430 return -1;
2431 /* XXX: this code should not depend on lock_user */
2432 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2433 return -1;
2434 memcpy(p, buf, l);
2435 unlock_user(p, addr, l);
2436 } else {
2437 if (!(flags & PAGE_READ))
2438 return -1;
2439 /* XXX: this code should not depend on lock_user */
2440 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2441 return -1;
2442 memcpy(buf, p, l);
2443 unlock_user(p, addr, 0);
2444 }
2445 len -= l;
2446 buf += l;
2447 addr += l;
2448 }
2449 return 0;
2450 }
2451
2452 #else
2453
2454 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2455 hwaddr length)
2456 {
2457 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2458 /* No early return if dirty_log_mask is or becomes 0, because
2459 * cpu_physical_memory_set_dirty_range will still call
2460 * xen_modified_memory.
2461 */
2462 if (dirty_log_mask) {
2463 dirty_log_mask =
2464 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2465 }
2466 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2467 tb_invalidate_phys_range(addr, addr + length);
2468 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2469 }
2470 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2471 }
2472
2473 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2474 {
2475 unsigned access_size_max = mr->ops->valid.max_access_size;
2476
2477 /* Regions are assumed to support 1-4 byte accesses unless
2478 otherwise specified. */
2479 if (access_size_max == 0) {
2480 access_size_max = 4;
2481 }
2482
2483 /* Bound the maximum access by the alignment of the address. */
2484 if (!mr->ops->impl.unaligned) {
2485 unsigned align_size_max = addr & -addr;
2486 if (align_size_max != 0 && align_size_max < access_size_max) {
2487 access_size_max = align_size_max;
2488 }
2489 }
2490
2491 /* Don't attempt accesses larger than the maximum. */
2492 if (l > access_size_max) {
2493 l = access_size_max;
2494 }
2495 l = pow2floor(l);
2496
2497 return l;
2498 }
2499
2500 static bool prepare_mmio_access(MemoryRegion *mr)
2501 {
2502 bool unlocked = !qemu_mutex_iothread_locked();
2503 bool release_lock = false;
2504
2505 if (unlocked && mr->global_locking) {
2506 qemu_mutex_lock_iothread();
2507 unlocked = false;
2508 release_lock = true;
2509 }
2510 if (mr->flush_coalesced_mmio) {
2511 if (unlocked) {
2512 qemu_mutex_lock_iothread();
2513 }
2514 qemu_flush_coalesced_mmio_buffer();
2515 if (unlocked) {
2516 qemu_mutex_unlock_iothread();
2517 }
2518 }
2519
2520 return release_lock;
2521 }
2522
2523 /* Called within RCU critical section. */
2524 static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr,
2525 MemTxAttrs attrs,
2526 const uint8_t *buf,
2527 int len, hwaddr addr1,
2528 hwaddr l, MemoryRegion *mr)
2529 {
2530 uint8_t *ptr;
2531 uint64_t val;
2532 MemTxResult result = MEMTX_OK;
2533 bool release_lock = false;
2534
2535 for (;;) {
2536 if (!memory_access_is_direct(mr, true)) {
2537 release_lock |= prepare_mmio_access(mr);
2538 l = memory_access_size(mr, l, addr1);
2539 /* XXX: could force current_cpu to NULL to avoid
2540 potential bugs */
2541 switch (l) {
2542 case 8:
2543 /* 64 bit write access */
2544 val = ldq_p(buf);
2545 result |= memory_region_dispatch_write(mr, addr1, val, 8,
2546 attrs);
2547 break;
2548 case 4:
2549 /* 32 bit write access */
2550 val = ldl_p(buf);
2551 result |= memory_region_dispatch_write(mr, addr1, val, 4,
2552 attrs);
2553 break;
2554 case 2:
2555 /* 16 bit write access */
2556 val = lduw_p(buf);
2557 result |= memory_region_dispatch_write(mr, addr1, val, 2,
2558 attrs);
2559 break;
2560 case 1:
2561 /* 8 bit write access */
2562 val = ldub_p(buf);
2563 result |= memory_region_dispatch_write(mr, addr1, val, 1,
2564 attrs);
2565 break;
2566 default:
2567 abort();
2568 }
2569 } else {
2570 addr1 += memory_region_get_ram_addr(mr);
2571 /* RAM case */
2572 ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
2573 memcpy(ptr, buf, l);
2574 invalidate_and_set_dirty(mr, addr1, l);
2575 }
2576
2577 if (release_lock) {
2578 qemu_mutex_unlock_iothread();
2579 release_lock = false;
2580 }
2581
2582 len -= l;
2583 buf += l;
2584 addr += l;
2585
2586 if (!len) {
2587 break;
2588 }
2589
2590 l = len;
2591 mr = address_space_translate(as, addr, &addr1, &l, true);
2592 }
2593
2594 return result;
2595 }
2596
2597 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2598 const uint8_t *buf, int len)
2599 {
2600 hwaddr l;
2601 hwaddr addr1;
2602 MemoryRegion *mr;
2603 MemTxResult result = MEMTX_OK;
2604
2605 if (len > 0) {
2606 rcu_read_lock();
2607 l = len;
2608 mr = address_space_translate(as, addr, &addr1, &l, true);
2609 result = address_space_write_continue(as, addr, attrs, buf, len,
2610 addr1, l, mr);
2611 rcu_read_unlock();
2612 }
2613
2614 return result;
2615 }
2616
2617 /* Called within RCU critical section. */
2618 MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
2619 MemTxAttrs attrs, uint8_t *buf,
2620 int len, hwaddr addr1, hwaddr l,
2621 MemoryRegion *mr)
2622 {
2623 uint8_t *ptr;
2624 uint64_t val;
2625 MemTxResult result = MEMTX_OK;
2626 bool release_lock = false;
2627
2628 for (;;) {
2629 if (!memory_access_is_direct(mr, false)) {
2630 /* I/O case */
2631 release_lock |= prepare_mmio_access(mr);
2632 l = memory_access_size(mr, l, addr1);
2633 switch (l) {
2634 case 8:
2635 /* 64 bit read access */
2636 result |= memory_region_dispatch_read(mr, addr1, &val, 8,
2637 attrs);
2638 stq_p(buf, val);
2639 break;
2640 case 4:
2641 /* 32 bit read access */
2642 result |= memory_region_dispatch_read(mr, addr1, &val, 4,
2643 attrs);
2644 stl_p(buf, val);
2645 break;
2646 case 2:
2647 /* 16 bit read access */
2648 result |= memory_region_dispatch_read(mr, addr1, &val, 2,
2649 attrs);
2650 stw_p(buf, val);
2651 break;
2652 case 1:
2653 /* 8 bit read access */
2654 result |= memory_region_dispatch_read(mr, addr1, &val, 1,
2655 attrs);
2656 stb_p(buf, val);
2657 break;
2658 default:
2659 abort();
2660 }
2661 } else {
2662 /* RAM case */
2663 ptr = qemu_get_ram_ptr(mr->ram_block,
2664 memory_region_get_ram_addr(mr) + addr1);
2665 memcpy(buf, ptr, l);
2666 }
2667
2668 if (release_lock) {
2669 qemu_mutex_unlock_iothread();
2670 release_lock = false;
2671 }
2672
2673 len -= l;
2674 buf += l;
2675 addr += l;
2676
2677 if (!len) {
2678 break;
2679 }
2680
2681 l = len;
2682 mr = address_space_translate(as, addr, &addr1, &l, false);
2683 }
2684
2685 return result;
2686 }
2687
2688 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
2689 MemTxAttrs attrs, uint8_t *buf, int len)
2690 {
2691 hwaddr l;
2692 hwaddr addr1;
2693 MemoryRegion *mr;
2694 MemTxResult result = MEMTX_OK;
2695
2696 if (len > 0) {
2697 rcu_read_lock();
2698 l = len;
2699 mr = address_space_translate(as, addr, &addr1, &l, false);
2700 result = address_space_read_continue(as, addr, attrs, buf, len,
2701 addr1, l, mr);
2702 rcu_read_unlock();
2703 }
2704
2705 return result;
2706 }
2707
2708 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2709 uint8_t *buf, int len, bool is_write)
2710 {
2711 if (is_write) {
2712 return address_space_write(as, addr, attrs, (uint8_t *)buf, len);
2713 } else {
2714 return address_space_read(as, addr, attrs, (uint8_t *)buf, len);
2715 }
2716 }
2717
2718 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2719 int len, int is_write)
2720 {
2721 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
2722 buf, len, is_write);
2723 }
2724
2725 enum write_rom_type {
2726 WRITE_DATA,
2727 FLUSH_CACHE,
2728 };
2729
2730 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2731 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2732 {
2733 hwaddr l;
2734 uint8_t *ptr;
2735 hwaddr addr1;
2736 MemoryRegion *mr;
2737
2738 rcu_read_lock();
2739 while (len > 0) {
2740 l = len;
2741 mr = address_space_translate(as, addr, &addr1, &l, true);
2742
2743 if (!(memory_region_is_ram(mr) ||
2744 memory_region_is_romd(mr))) {
2745 l = memory_access_size(mr, l, addr1);
2746 } else {
2747 addr1 += memory_region_get_ram_addr(mr);
2748 /* ROM/RAM case */
2749 ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
2750 switch (type) {
2751 case WRITE_DATA:
2752 memcpy(ptr, buf, l);
2753 invalidate_and_set_dirty(mr, addr1, l);
2754 break;
2755 case FLUSH_CACHE:
2756 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2757 break;
2758 }
2759 }
2760 len -= l;
2761 buf += l;
2762 addr += l;
2763 }
2764 rcu_read_unlock();
2765 }
2766
2767 /* used for ROM loading : can write in RAM and ROM */
2768 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2769 const uint8_t *buf, int len)
2770 {
2771 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2772 }
2773
2774 void cpu_flush_icache_range(hwaddr start, int len)
2775 {
2776 /*
2777 * This function should do the same thing as an icache flush that was
2778 * triggered from within the guest. For TCG we are always cache coherent,
2779 * so there is no need to flush anything. For KVM / Xen we need to flush
2780 * the host's instruction cache at least.
2781 */
2782 if (tcg_enabled()) {
2783 return;
2784 }
2785
2786 cpu_physical_memory_write_rom_internal(&address_space_memory,
2787 start, NULL, len, FLUSH_CACHE);
2788 }
2789
2790 typedef struct {
2791 MemoryRegion *mr;
2792 void *buffer;
2793 hwaddr addr;
2794 hwaddr len;
2795 bool in_use;
2796 } BounceBuffer;
2797
2798 static BounceBuffer bounce;
2799
2800 typedef struct MapClient {
2801 QEMUBH *bh;
2802 QLIST_ENTRY(MapClient) link;
2803 } MapClient;
2804
2805 QemuMutex map_client_list_lock;
2806 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2807 = QLIST_HEAD_INITIALIZER(map_client_list);
2808
2809 static void cpu_unregister_map_client_do(MapClient *client)
2810 {
2811 QLIST_REMOVE(client, link);
2812 g_free(client);
2813 }
2814
2815 static void cpu_notify_map_clients_locked(void)
2816 {
2817 MapClient *client;
2818
2819 while (!QLIST_EMPTY(&map_client_list)) {
2820 client = QLIST_FIRST(&map_client_list);
2821 qemu_bh_schedule(client->bh);
2822 cpu_unregister_map_client_do(client);
2823 }
2824 }
2825
2826 void cpu_register_map_client(QEMUBH *bh)
2827 {
2828 MapClient *client = g_malloc(sizeof(*client));
2829
2830 qemu_mutex_lock(&map_client_list_lock);
2831 client->bh = bh;
2832 QLIST_INSERT_HEAD(&map_client_list, client, link);
2833 if (!atomic_read(&bounce.in_use)) {
2834 cpu_notify_map_clients_locked();
2835 }
2836 qemu_mutex_unlock(&map_client_list_lock);
2837 }
2838
2839 void cpu_exec_init_all(void)
2840 {
2841 qemu_mutex_init(&ram_list.mutex);
2842 io_mem_init();
2843 memory_map_init();
2844 qemu_mutex_init(&map_client_list_lock);
2845 }
2846
2847 void cpu_unregister_map_client(QEMUBH *bh)
2848 {
2849 MapClient *client;
2850
2851 qemu_mutex_lock(&map_client_list_lock);
2852 QLIST_FOREACH(client, &map_client_list, link) {
2853 if (client->bh == bh) {
2854 cpu_unregister_map_client_do(client);
2855 break;
2856 }
2857 }
2858 qemu_mutex_unlock(&map_client_list_lock);
2859 }
2860
2861 static void cpu_notify_map_clients(void)
2862 {
2863 qemu_mutex_lock(&map_client_list_lock);
2864 cpu_notify_map_clients_locked();
2865 qemu_mutex_unlock(&map_client_list_lock);
2866 }
2867
2868 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2869 {
2870 MemoryRegion *mr;
2871 hwaddr l, xlat;
2872
2873 rcu_read_lock();
2874 while (len > 0) {
2875 l = len;
2876 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2877 if (!memory_access_is_direct(mr, is_write)) {
2878 l = memory_access_size(mr, l, addr);
2879 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2880 return false;
2881 }
2882 }
2883
2884 len -= l;
2885 addr += l;
2886 }
2887 rcu_read_unlock();
2888 return true;
2889 }
2890
2891 /* Map a physical memory region into a host virtual address.
2892 * May map a subset of the requested range, given by and returned in *plen.
2893 * May return NULL if resources needed to perform the mapping are exhausted.
2894 * Use only for reads OR writes - not for read-modify-write operations.
2895 * Use cpu_register_map_client() to know when retrying the map operation is
2896 * likely to succeed.
2897 */
2898 void *address_space_map(AddressSpace *as,
2899 hwaddr addr,
2900 hwaddr *plen,
2901 bool is_write)
2902 {
2903 hwaddr len = *plen;
2904 hwaddr done = 0;
2905 hwaddr l, xlat, base;
2906 MemoryRegion *mr, *this_mr;
2907 ram_addr_t raddr;
2908 void *ptr;
2909
2910 if (len == 0) {
2911 return NULL;
2912 }
2913
2914 l = len;
2915 rcu_read_lock();
2916 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2917
2918 if (!memory_access_is_direct(mr, is_write)) {
2919 if (atomic_xchg(&bounce.in_use, true)) {
2920 rcu_read_unlock();
2921 return NULL;
2922 }
2923 /* Avoid unbounded allocations */
2924 l = MIN(l, TARGET_PAGE_SIZE);
2925 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2926 bounce.addr = addr;
2927 bounce.len = l;
2928
2929 memory_region_ref(mr);
2930 bounce.mr = mr;
2931 if (!is_write) {
2932 address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
2933 bounce.buffer, l);
2934 }
2935
2936 rcu_read_unlock();
2937 *plen = l;
2938 return bounce.buffer;
2939 }
2940
2941 base = xlat;
2942 raddr = memory_region_get_ram_addr(mr);
2943
2944 for (;;) {
2945 len -= l;
2946 addr += l;
2947 done += l;
2948 if (len == 0) {
2949 break;
2950 }
2951
2952 l = len;
2953 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2954 if (this_mr != mr || xlat != base + done) {
2955 break;
2956 }
2957 }
2958
2959 memory_region_ref(mr);
2960 *plen = done;
2961 ptr = qemu_ram_ptr_length(mr->ram_block, raddr + base, plen);
2962 rcu_read_unlock();
2963
2964 return ptr;
2965 }
2966
2967 /* Unmaps a memory region previously mapped by address_space_map().
2968 * Will also mark the memory as dirty if is_write == 1. access_len gives
2969 * the amount of memory that was actually read or written by the caller.
2970 */
2971 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2972 int is_write, hwaddr access_len)
2973 {
2974 if (buffer != bounce.buffer) {
2975 MemoryRegion *mr;
2976 ram_addr_t addr1;
2977
2978 mr = qemu_ram_addr_from_host(buffer, &addr1);
2979 assert(mr != NULL);
2980 if (is_write) {
2981 invalidate_and_set_dirty(mr, addr1, access_len);
2982 }
2983 if (xen_enabled()) {
2984 xen_invalidate_map_cache_entry(buffer);
2985 }
2986 memory_region_unref(mr);
2987 return;
2988 }
2989 if (is_write) {
2990 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
2991 bounce.buffer, access_len);
2992 }
2993 qemu_vfree(bounce.buffer);
2994 bounce.buffer = NULL;
2995 memory_region_unref(bounce.mr);
2996 atomic_mb_set(&bounce.in_use, false);
2997 cpu_notify_map_clients();
2998 }
2999
3000 void *cpu_physical_memory_map(hwaddr addr,
3001 hwaddr *plen,
3002 int is_write)
3003 {
3004 return address_space_map(&address_space_memory, addr, plen, is_write);
3005 }
3006
3007 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3008 int is_write, hwaddr access_len)
3009 {
3010 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3011 }
3012
3013 /* warning: addr must be aligned */
3014 static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
3015 MemTxAttrs attrs,
3016 MemTxResult *result,
3017 enum device_endian endian)
3018 {
3019 uint8_t *ptr;
3020 uint64_t val;
3021 MemoryRegion *mr;
3022 hwaddr l = 4;
3023 hwaddr addr1;
3024 MemTxResult r;
3025 bool release_lock = false;
3026
3027 rcu_read_lock();
3028 mr = address_space_translate(as, addr, &addr1, &l, false);
3029 if (l < 4 || !memory_access_is_direct(mr, false)) {
3030 release_lock |= prepare_mmio_access(mr);
3031
3032 /* I/O case */
3033 r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
3034 #if defined(TARGET_WORDS_BIGENDIAN)
3035 if (endian == DEVICE_LITTLE_ENDIAN) {
3036 val = bswap32(val);
3037 }
3038 #else
3039 if (endian == DEVICE_BIG_ENDIAN) {
3040 val = bswap32(val);
3041 }
3042 #endif
3043 } else {
3044 /* RAM case */
3045 ptr = qemu_get_ram_ptr(mr->ram_block,
3046 memory_region_get_ram_addr(mr) + addr1);
3047 switch (endian) {
3048 case DEVICE_LITTLE_ENDIAN:
3049 val = ldl_le_p(ptr);
3050 break;
3051 case DEVICE_BIG_ENDIAN:
3052 val = ldl_be_p(ptr);
3053 break;
3054 default:
3055 val = ldl_p(ptr);
3056 break;
3057 }
3058 r = MEMTX_OK;
3059 }
3060 if (result) {
3061 *result = r;
3062 }
3063 if (release_lock) {
3064 qemu_mutex_unlock_iothread();
3065 }
3066 rcu_read_unlock();
3067 return val;
3068 }
3069
3070 uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
3071 MemTxAttrs attrs, MemTxResult *result)
3072 {
3073 return address_space_ldl_internal(as, addr, attrs, result,
3074 DEVICE_NATIVE_ENDIAN);
3075 }
3076
3077 uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
3078 MemTxAttrs attrs, MemTxResult *result)
3079 {
3080 return address_space_ldl_internal(as, addr, attrs, result,
3081 DEVICE_LITTLE_ENDIAN);
3082 }
3083
3084 uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
3085 MemTxAttrs attrs, MemTxResult *result)
3086 {
3087 return address_space_ldl_internal(as, addr, attrs, result,
3088 DEVICE_BIG_ENDIAN);
3089 }
3090
3091 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
3092 {
3093 return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3094 }
3095
3096 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
3097 {
3098 return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3099 }
3100
3101 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
3102 {
3103 return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3104 }
3105
3106 /* warning: addr must be aligned */
3107 static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
3108 MemTxAttrs attrs,
3109 MemTxResult *result,
3110 enum device_endian endian)
3111 {
3112 uint8_t *ptr;
3113 uint64_t val;
3114 MemoryRegion *mr;
3115 hwaddr l = 8;
3116 hwaddr addr1;
3117 MemTxResult r;
3118 bool release_lock = false;
3119
3120 rcu_read_lock();
3121 mr = address_space_translate(as, addr, &addr1, &l,
3122 false);
3123 if (l < 8 || !memory_access_is_direct(mr, false)) {
3124 release_lock |= prepare_mmio_access(mr);
3125
3126 /* I/O case */
3127 r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
3128 #if defined(TARGET_WORDS_BIGENDIAN)
3129 if (endian == DEVICE_LITTLE_ENDIAN) {
3130 val = bswap64(val);
3131 }
3132 #else
3133 if (endian == DEVICE_BIG_ENDIAN) {
3134 val = bswap64(val);
3135 }
3136 #endif
3137 } else {
3138 /* RAM case */
3139 ptr = qemu_get_ram_ptr(mr->ram_block,
3140 memory_region_get_ram_addr(mr) + addr1);
3141 switch (endian) {
3142 case DEVICE_LITTLE_ENDIAN:
3143 val = ldq_le_p(ptr);
3144 break;
3145 case DEVICE_BIG_ENDIAN:
3146 val = ldq_be_p(ptr);
3147 break;
3148 default:
3149 val = ldq_p(ptr);
3150 break;
3151 }
3152 r = MEMTX_OK;
3153 }
3154 if (result) {
3155 *result = r;
3156 }
3157 if (release_lock) {
3158 qemu_mutex_unlock_iothread();
3159 }
3160 rcu_read_unlock();
3161 return val;
3162 }
3163
3164 uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
3165 MemTxAttrs attrs, MemTxResult *result)
3166 {
3167 return address_space_ldq_internal(as, addr, attrs, result,
3168 DEVICE_NATIVE_ENDIAN);
3169 }
3170
3171 uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
3172 MemTxAttrs attrs, MemTxResult *result)
3173 {
3174 return address_space_ldq_internal(as, addr, attrs, result,
3175 DEVICE_LITTLE_ENDIAN);
3176 }
3177
3178 uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
3179 MemTxAttrs attrs, MemTxResult *result)
3180 {
3181 return address_space_ldq_internal(as, addr, attrs, result,
3182 DEVICE_BIG_ENDIAN);
3183 }
3184
3185 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
3186 {
3187 return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3188 }
3189
3190 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
3191 {
3192 return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3193 }
3194
3195 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
3196 {
3197 return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3198 }
3199
3200 /* XXX: optimize */
3201 uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
3202 MemTxAttrs attrs, MemTxResult *result)
3203 {
3204 uint8_t val;
3205 MemTxResult r;
3206
3207 r = address_space_rw(as, addr, attrs, &val, 1, 0);
3208 if (result) {
3209 *result = r;
3210 }
3211 return val;
3212 }
3213
3214 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
3215 {
3216 return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3217 }
3218
3219 /* warning: addr must be aligned */
3220 static inline uint32_t address_space_lduw_internal(AddressSpace *as,
3221 hwaddr addr,
3222 MemTxAttrs attrs,
3223 MemTxResult *result,
3224 enum device_endian endian)
3225 {
3226 uint8_t *ptr;
3227 uint64_t val;
3228 MemoryRegion *mr;
3229 hwaddr l = 2;
3230 hwaddr addr1;
3231 MemTxResult r;
3232 bool release_lock = false;
3233
3234 rcu_read_lock();
3235 mr = address_space_translate(as, addr, &addr1, &l,
3236 false);
3237 if (l < 2 || !memory_access_is_direct(mr, false)) {
3238 release_lock |= prepare_mmio_access(mr);
3239
3240 /* I/O case */
3241 r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
3242 #if defined(TARGET_WORDS_BIGENDIAN)
3243 if (endian == DEVICE_LITTLE_ENDIAN) {
3244 val = bswap16(val);
3245 }
3246 #else
3247 if (endian == DEVICE_BIG_ENDIAN) {
3248 val = bswap16(val);
3249 }
3250 #endif
3251 } else {
3252 /* RAM case */
3253 ptr = qemu_get_ram_ptr(mr->ram_block,
3254 memory_region_get_ram_addr(mr) + addr1);
3255 switch (endian) {
3256 case DEVICE_LITTLE_ENDIAN:
3257 val = lduw_le_p(ptr);
3258 break;
3259 case DEVICE_BIG_ENDIAN:
3260 val = lduw_be_p(ptr);
3261 break;
3262 default:
3263 val = lduw_p(ptr);
3264 break;
3265 }
3266 r = MEMTX_OK;
3267 }
3268 if (result) {
3269 *result = r;
3270 }
3271 if (release_lock) {
3272 qemu_mutex_unlock_iothread();
3273 }
3274 rcu_read_unlock();
3275 return val;
3276 }
3277
3278 uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
3279 MemTxAttrs attrs, MemTxResult *result)
3280 {
3281 return address_space_lduw_internal(as, addr, attrs, result,
3282 DEVICE_NATIVE_ENDIAN);
3283 }
3284
3285 uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
3286 MemTxAttrs attrs, MemTxResult *result)
3287 {
3288 return address_space_lduw_internal(as, addr, attrs, result,
3289 DEVICE_LITTLE_ENDIAN);
3290 }
3291
3292 uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
3293 MemTxAttrs attrs, MemTxResult *result)
3294 {
3295 return address_space_lduw_internal(as, addr, attrs, result,
3296 DEVICE_BIG_ENDIAN);
3297 }
3298
3299 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
3300 {
3301 return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3302 }
3303
3304 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
3305 {
3306 return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3307 }
3308
3309 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
3310 {
3311 return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
3312 }
3313
3314 /* warning: addr must be aligned. The ram page is not masked as dirty
3315 and the code inside is not invalidated. It is useful if the dirty
3316 bits are used to track modified PTEs */
3317 void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
3318 MemTxAttrs attrs, MemTxResult *result)
3319 {
3320 uint8_t *ptr;
3321 MemoryRegion *mr;
3322 hwaddr l = 4;
3323 hwaddr addr1;
3324 MemTxResult r;
3325 uint8_t dirty_log_mask;
3326 bool release_lock = false;
3327
3328 rcu_read_lock();
3329 mr = address_space_translate(as, addr, &addr1, &l,
3330 true);
3331 if (l < 4 || !memory_access_is_direct(mr, true)) {
3332 release_lock |= prepare_mmio_access(mr);
3333
3334 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3335 } else {
3336 addr1 += memory_region_get_ram_addr(mr);
3337 ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
3338 stl_p(ptr, val);
3339
3340 dirty_log_mask = memory_region_get_dirty_log_mask(mr);
3341 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
3342 cpu_physical_memory_set_dirty_range(addr1, 4, dirty_log_mask);
3343 r = MEMTX_OK;
3344 }
3345 if (result) {
3346 *result = r;
3347 }
3348 if (release_lock) {
3349 qemu_mutex_unlock_iothread();
3350 }
3351 rcu_read_unlock();
3352 }
3353
3354 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
3355 {
3356 address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3357 }
3358
3359 /* warning: addr must be aligned */
3360 static inline void address_space_stl_internal(AddressSpace *as,
3361 hwaddr addr, uint32_t val,
3362 MemTxAttrs attrs,
3363 MemTxResult *result,
3364 enum device_endian endian)
3365 {
3366 uint8_t *ptr;
3367 MemoryRegion *mr;
3368 hwaddr l = 4;
3369 hwaddr addr1;
3370 MemTxResult r;
3371 bool release_lock = false;
3372
3373 rcu_read_lock();
3374 mr = address_space_translate(as, addr, &addr1, &l,
3375 true);
3376 if (l < 4 || !memory_access_is_direct(mr, true)) {
3377 release_lock |= prepare_mmio_access(mr);
3378
3379 #if defined(TARGET_WORDS_BIGENDIAN)
3380 if (endian == DEVICE_LITTLE_ENDIAN) {
3381 val = bswap32(val);
3382 }
3383 #else
3384 if (endian == DEVICE_BIG_ENDIAN) {
3385 val = bswap32(val);
3386 }
3387 #endif
3388 r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
3389 } else {
3390 /* RAM case */
3391 addr1 += memory_region_get_ram_addr(mr);
3392 ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
3393 switch (endian) {
3394 case DEVICE_LITTLE_ENDIAN:
3395 stl_le_p(ptr, val);
3396 break;
3397 case DEVICE_BIG_ENDIAN:
3398 stl_be_p(ptr, val);
3399 break;
3400 default:
3401 stl_p(ptr, val);
3402 break;
3403 }
3404 invalidate_and_set_dirty(mr, addr1, 4);
3405 r = MEMTX_OK;
3406 }
3407 if (result) {
3408 *result = r;
3409 }
3410 if (release_lock) {
3411 qemu_mutex_unlock_iothread();
3412 }
3413 rcu_read_unlock();
3414 }
3415
3416 void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
3417 MemTxAttrs attrs, MemTxResult *result)
3418 {
3419 address_space_stl_internal(as, addr, val, attrs, result,
3420 DEVICE_NATIVE_ENDIAN);
3421 }
3422
3423 void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
3424 MemTxAttrs attrs, MemTxResult *result)
3425 {
3426 address_space_stl_internal(as, addr, val, attrs, result,
3427 DEVICE_LITTLE_ENDIAN);
3428 }
3429
3430 void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
3431 MemTxAttrs attrs, MemTxResult *result)
3432 {
3433 address_space_stl_internal(as, addr, val, attrs, result,
3434 DEVICE_BIG_ENDIAN);
3435 }
3436
3437 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3438 {
3439 address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3440 }
3441
3442 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3443 {
3444 address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3445 }
3446
3447 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3448 {
3449 address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3450 }
3451
3452 /* XXX: optimize */
3453 void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
3454 MemTxAttrs attrs, MemTxResult *result)
3455 {
3456 uint8_t v = val;
3457 MemTxResult r;
3458
3459 r = address_space_rw(as, addr, attrs, &v, 1, 1);
3460 if (result) {
3461 *result = r;
3462 }
3463 }
3464
3465 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3466 {
3467 address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3468 }
3469
3470 /* warning: addr must be aligned */
3471 static inline void address_space_stw_internal(AddressSpace *as,
3472 hwaddr addr, uint32_t val,
3473 MemTxAttrs attrs,
3474 MemTxResult *result,
3475 enum device_endian endian)
3476 {
3477 uint8_t *ptr;
3478 MemoryRegion *mr;
3479 hwaddr l = 2;
3480 hwaddr addr1;
3481 MemTxResult r;
3482 bool release_lock = false;
3483
3484 rcu_read_lock();
3485 mr = address_space_translate(as, addr, &addr1, &l, true);
3486 if (l < 2 || !memory_access_is_direct(mr, true)) {
3487 release_lock |= prepare_mmio_access(mr);
3488
3489 #if defined(TARGET_WORDS_BIGENDIAN)
3490 if (endian == DEVICE_LITTLE_ENDIAN) {
3491 val = bswap16(val);
3492 }
3493 #else
3494 if (endian == DEVICE_BIG_ENDIAN) {
3495 val = bswap16(val);
3496 }
3497 #endif
3498 r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
3499 } else {
3500 /* RAM case */
3501 addr1 += memory_region_get_ram_addr(mr);
3502 ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
3503 switch (endian) {
3504 case DEVICE_LITTLE_ENDIAN:
3505 stw_le_p(ptr, val);
3506 break;
3507 case DEVICE_BIG_ENDIAN:
3508 stw_be_p(ptr, val);
3509 break;
3510 default:
3511 stw_p(ptr, val);
3512 break;
3513 }
3514 invalidate_and_set_dirty(mr, addr1, 2);
3515 r = MEMTX_OK;
3516 }
3517 if (result) {
3518 *result = r;
3519 }
3520 if (release_lock) {
3521 qemu_mutex_unlock_iothread();
3522 }
3523 rcu_read_unlock();
3524 }
3525
3526 void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
3527 MemTxAttrs attrs, MemTxResult *result)
3528 {
3529 address_space_stw_internal(as, addr, val, attrs, result,
3530 DEVICE_NATIVE_ENDIAN);
3531 }
3532
3533 void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
3534 MemTxAttrs attrs, MemTxResult *result)
3535 {
3536 address_space_stw_internal(as, addr, val, attrs, result,
3537 DEVICE_LITTLE_ENDIAN);
3538 }
3539
3540 void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
3541 MemTxAttrs attrs, MemTxResult *result)
3542 {
3543 address_space_stw_internal(as, addr, val, attrs, result,
3544 DEVICE_BIG_ENDIAN);
3545 }
3546
3547 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3548 {
3549 address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3550 }
3551
3552 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3553 {
3554 address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3555 }
3556
3557 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
3558 {
3559 address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
3560 }