Merge tag 'ide-pull-request' of https://gitlab.com/jsnow/qemu into staging
[qemu.git] / hw / ppc / spapr_softmmu.c
1 #include "qemu/osdep.h"
2 #include "qemu/cutils.h"
3 #include "qapi/error.h"
4 #include "sysemu/hw_accel.h"
5 #include "sysemu/runstate.h"
6 #include "qemu/log.h"
7 #include "qemu/main-loop.h"
8 #include "qemu/module.h"
9 #include "qemu/error-report.h"
10 #include "cpu.h"
11 #include "exec/exec-all.h"
12 #include "helper_regs.h"
13 #include "hw/ppc/spapr.h"
14 #include "hw/ppc/spapr_cpu_core.h"
15 #include "mmu-hash64.h"
16 #include "cpu-models.h"
17 #include "trace.h"
18 #include "kvm_ppc.h"
19 #include "hw/ppc/fdt.h"
20 #include "hw/ppc/spapr_ovec.h"
21 #include "mmu-book3s-v3.h"
22 #include "hw/mem/memory-device.h"
23
24 static inline bool valid_ptex(PowerPCCPU *cpu, target_ulong ptex)
25 {
26 /*
27 * hash value/pteg group index is normalized by HPT mask
28 */
29 if (((ptex & ~7ULL) / HPTES_PER_GROUP) & ~ppc_hash64_hpt_mask(cpu)) {
30 return false;
31 }
32 return true;
33 }
34
35 static target_ulong h_enter(PowerPCCPU *cpu, SpaprMachineState *spapr,
36 target_ulong opcode, target_ulong *args)
37 {
38 target_ulong flags = args[0];
39 target_ulong ptex = args[1];
40 target_ulong pteh = args[2];
41 target_ulong ptel = args[3];
42 unsigned apshift;
43 target_ulong raddr;
44 target_ulong slot;
45 const ppc_hash_pte64_t *hptes;
46
47 apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel);
48 if (!apshift) {
49 /* Bad page size encoding */
50 return H_PARAMETER;
51 }
52
53 raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
54
55 if (is_ram_address(spapr, raddr)) {
56 /* Regular RAM - should have WIMG=0010 */
57 if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
58 return H_PARAMETER;
59 }
60 } else {
61 target_ulong wimg_flags;
62 /* Looks like an IO address */
63 /* FIXME: What WIMG combinations could be sensible for IO?
64 * For now we allow WIMG=010x, but are there others? */
65 /* FIXME: Should we check against registered IO addresses? */
66 wimg_flags = (ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M));
67
68 if (wimg_flags != HPTE64_R_I &&
69 wimg_flags != (HPTE64_R_I | HPTE64_R_M)) {
70 return H_PARAMETER;
71 }
72 }
73
74 pteh &= ~0x60ULL;
75
76 if (!valid_ptex(cpu, ptex)) {
77 return H_PARAMETER;
78 }
79
80 slot = ptex & 7ULL;
81 ptex = ptex & ~7ULL;
82
83 if (likely((flags & H_EXACT) == 0)) {
84 hptes = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
85 for (slot = 0; slot < 8; slot++) {
86 if (!(ppc_hash64_hpte0(cpu, hptes, slot) & HPTE64_V_VALID)) {
87 break;
88 }
89 }
90 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
91 if (slot == 8) {
92 return H_PTEG_FULL;
93 }
94 } else {
95 hptes = ppc_hash64_map_hptes(cpu, ptex + slot, 1);
96 if (ppc_hash64_hpte0(cpu, hptes, 0) & HPTE64_V_VALID) {
97 ppc_hash64_unmap_hptes(cpu, hptes, ptex + slot, 1);
98 return H_PTEG_FULL;
99 }
100 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
101 }
102
103 spapr_store_hpte(cpu, ptex + slot, pteh | HPTE64_V_HPTE_DIRTY, ptel);
104
105 args[0] = ptex + slot;
106 return H_SUCCESS;
107 }
108
109 typedef enum {
110 REMOVE_SUCCESS = 0,
111 REMOVE_NOT_FOUND = 1,
112 REMOVE_PARM = 2,
113 REMOVE_HW = 3,
114 } RemoveResult;
115
116 static RemoveResult remove_hpte(PowerPCCPU *cpu
117 , target_ulong ptex,
118 target_ulong avpn,
119 target_ulong flags,
120 target_ulong *vp, target_ulong *rp)
121 {
122 const ppc_hash_pte64_t *hptes;
123 target_ulong v, r;
124
125 if (!valid_ptex(cpu, ptex)) {
126 return REMOVE_PARM;
127 }
128
129 hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
130 v = ppc_hash64_hpte0(cpu, hptes, 0);
131 r = ppc_hash64_hpte1(cpu, hptes, 0);
132 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
133
134 if ((v & HPTE64_V_VALID) == 0 ||
135 ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
136 ((flags & H_ANDCOND) && (v & avpn) != 0)) {
137 return REMOVE_NOT_FOUND;
138 }
139 *vp = v;
140 *rp = r;
141 spapr_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
142 ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
143 return REMOVE_SUCCESS;
144 }
145
146 static target_ulong h_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
147 target_ulong opcode, target_ulong *args)
148 {
149 CPUPPCState *env = &cpu->env;
150 target_ulong flags = args[0];
151 target_ulong ptex = args[1];
152 target_ulong avpn = args[2];
153 RemoveResult ret;
154
155 ret = remove_hpte(cpu, ptex, avpn, flags,
156 &args[0], &args[1]);
157
158 switch (ret) {
159 case REMOVE_SUCCESS:
160 check_tlb_flush(env, true);
161 return H_SUCCESS;
162
163 case REMOVE_NOT_FOUND:
164 return H_NOT_FOUND;
165
166 case REMOVE_PARM:
167 return H_PARAMETER;
168
169 case REMOVE_HW:
170 return H_HARDWARE;
171 }
172
173 g_assert_not_reached();
174 }
175
176 #define H_BULK_REMOVE_TYPE 0xc000000000000000ULL
177 #define H_BULK_REMOVE_REQUEST 0x4000000000000000ULL
178 #define H_BULK_REMOVE_RESPONSE 0x8000000000000000ULL
179 #define H_BULK_REMOVE_END 0xc000000000000000ULL
180 #define H_BULK_REMOVE_CODE 0x3000000000000000ULL
181 #define H_BULK_REMOVE_SUCCESS 0x0000000000000000ULL
182 #define H_BULK_REMOVE_NOT_FOUND 0x1000000000000000ULL
183 #define H_BULK_REMOVE_PARM 0x2000000000000000ULL
184 #define H_BULK_REMOVE_HW 0x3000000000000000ULL
185 #define H_BULK_REMOVE_RC 0x0c00000000000000ULL
186 #define H_BULK_REMOVE_FLAGS 0x0300000000000000ULL
187 #define H_BULK_REMOVE_ABSOLUTE 0x0000000000000000ULL
188 #define H_BULK_REMOVE_ANDCOND 0x0100000000000000ULL
189 #define H_BULK_REMOVE_AVPN 0x0200000000000000ULL
190 #define H_BULK_REMOVE_PTEX 0x00ffffffffffffffULL
191
192 #define H_BULK_REMOVE_MAX_BATCH 4
193
194 static target_ulong h_bulk_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
195 target_ulong opcode, target_ulong *args)
196 {
197 CPUPPCState *env = &cpu->env;
198 int i;
199 target_ulong rc = H_SUCCESS;
200
201 for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
202 target_ulong *tsh = &args[i*2];
203 target_ulong tsl = args[i*2 + 1];
204 target_ulong v, r, ret;
205
206 if ((*tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) {
207 break;
208 } else if ((*tsh & H_BULK_REMOVE_TYPE) != H_BULK_REMOVE_REQUEST) {
209 return H_PARAMETER;
210 }
211
212 *tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS;
213 *tsh |= H_BULK_REMOVE_RESPONSE;
214
215 if ((*tsh & H_BULK_REMOVE_ANDCOND) && (*tsh & H_BULK_REMOVE_AVPN)) {
216 *tsh |= H_BULK_REMOVE_PARM;
217 return H_PARAMETER;
218 }
219
220 ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
221 (*tsh & H_BULK_REMOVE_FLAGS) >> 26,
222 &v, &r);
223
224 *tsh |= ret << 60;
225
226 switch (ret) {
227 case REMOVE_SUCCESS:
228 *tsh |= (r & (HPTE64_R_C | HPTE64_R_R)) << 43;
229 break;
230
231 case REMOVE_PARM:
232 rc = H_PARAMETER;
233 goto exit;
234
235 case REMOVE_HW:
236 rc = H_HARDWARE;
237 goto exit;
238 }
239 }
240 exit:
241 check_tlb_flush(env, true);
242
243 return rc;
244 }
245
246 static target_ulong h_protect(PowerPCCPU *cpu, SpaprMachineState *spapr,
247 target_ulong opcode, target_ulong *args)
248 {
249 CPUPPCState *env = &cpu->env;
250 target_ulong flags = args[0];
251 target_ulong ptex = args[1];
252 target_ulong avpn = args[2];
253 const ppc_hash_pte64_t *hptes;
254 target_ulong v, r;
255
256 if (!valid_ptex(cpu, ptex)) {
257 return H_PARAMETER;
258 }
259
260 hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
261 v = ppc_hash64_hpte0(cpu, hptes, 0);
262 r = ppc_hash64_hpte1(cpu, hptes, 0);
263 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
264
265 if ((v & HPTE64_V_VALID) == 0 ||
266 ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
267 return H_NOT_FOUND;
268 }
269
270 r &= ~(HPTE64_R_PP0 | HPTE64_R_PP | HPTE64_R_N |
271 HPTE64_R_KEY_HI | HPTE64_R_KEY_LO);
272 r |= (flags << 55) & HPTE64_R_PP0;
273 r |= (flags << 48) & HPTE64_R_KEY_HI;
274 r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
275 spapr_store_hpte(cpu, ptex,
276 (v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
277 ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
278 /* Flush the tlb */
279 check_tlb_flush(env, true);
280 /* Don't need a memory barrier, due to qemu's global lock */
281 spapr_store_hpte(cpu, ptex, v | HPTE64_V_HPTE_DIRTY, r);
282 return H_SUCCESS;
283 }
284
285 static target_ulong h_read(PowerPCCPU *cpu, SpaprMachineState *spapr,
286 target_ulong opcode, target_ulong *args)
287 {
288 target_ulong flags = args[0];
289 target_ulong ptex = args[1];
290 int i, ridx, n_entries = 1;
291 const ppc_hash_pte64_t *hptes;
292
293 if (!valid_ptex(cpu, ptex)) {
294 return H_PARAMETER;
295 }
296
297 if (flags & H_READ_4) {
298 /* Clear the two low order bits */
299 ptex &= ~(3ULL);
300 n_entries = 4;
301 }
302
303 hptes = ppc_hash64_map_hptes(cpu, ptex, n_entries);
304 for (i = 0, ridx = 0; i < n_entries; i++) {
305 args[ridx++] = ppc_hash64_hpte0(cpu, hptes, i);
306 args[ridx++] = ppc_hash64_hpte1(cpu, hptes, i);
307 }
308 ppc_hash64_unmap_hptes(cpu, hptes, ptex, n_entries);
309
310 return H_SUCCESS;
311 }
312
313 struct SpaprPendingHpt {
314 /* These fields are read-only after initialization */
315 int shift;
316 QemuThread thread;
317
318 /* These fields are protected by the BQL */
319 bool complete;
320
321 /* These fields are private to the preparation thread if
322 * !complete, otherwise protected by the BQL */
323 int ret;
324 void *hpt;
325 };
326
327 static void free_pending_hpt(SpaprPendingHpt *pending)
328 {
329 if (pending->hpt) {
330 qemu_vfree(pending->hpt);
331 }
332
333 g_free(pending);
334 }
335
336 static void *hpt_prepare_thread(void *opaque)
337 {
338 SpaprPendingHpt *pending = opaque;
339 size_t size = 1ULL << pending->shift;
340
341 pending->hpt = qemu_try_memalign(size, size);
342 if (pending->hpt) {
343 memset(pending->hpt, 0, size);
344 pending->ret = H_SUCCESS;
345 } else {
346 pending->ret = H_NO_MEM;
347 }
348
349 qemu_mutex_lock_iothread();
350
351 if (SPAPR_MACHINE(qdev_get_machine())->pending_hpt == pending) {
352 /* Ready to go */
353 pending->complete = true;
354 } else {
355 /* We've been cancelled, clean ourselves up */
356 free_pending_hpt(pending);
357 }
358
359 qemu_mutex_unlock_iothread();
360 return NULL;
361 }
362
363 /* Must be called with BQL held */
364 static void cancel_hpt_prepare(SpaprMachineState *spapr)
365 {
366 SpaprPendingHpt *pending = spapr->pending_hpt;
367
368 /* Let the thread know it's cancelled */
369 spapr->pending_hpt = NULL;
370
371 if (!pending) {
372 /* Nothing to do */
373 return;
374 }
375
376 if (!pending->complete) {
377 /* thread will clean itself up */
378 return;
379 }
380
381 free_pending_hpt(pending);
382 }
383
384 target_ulong softmmu_resize_hpt_prepare(PowerPCCPU *cpu,
385 SpaprMachineState *spapr,
386 target_ulong shift)
387 {
388 SpaprPendingHpt *pending = spapr->pending_hpt;
389
390 if (pending) {
391 /* something already in progress */
392 if (pending->shift == shift) {
393 /* and it's suitable */
394 if (pending->complete) {
395 return pending->ret;
396 } else {
397 return H_LONG_BUSY_ORDER_100_MSEC;
398 }
399 }
400
401 /* not suitable, cancel and replace */
402 cancel_hpt_prepare(spapr);
403 }
404
405 if (!shift) {
406 /* nothing to do */
407 return H_SUCCESS;
408 }
409
410 /* start new prepare */
411
412 pending = g_new0(SpaprPendingHpt, 1);
413 pending->shift = shift;
414 pending->ret = H_HARDWARE;
415
416 qemu_thread_create(&pending->thread, "sPAPR HPT prepare",
417 hpt_prepare_thread, pending, QEMU_THREAD_DETACHED);
418
419 spapr->pending_hpt = pending;
420
421 /* In theory we could estimate the time more accurately based on
422 * the new size, but there's not much point */
423 return H_LONG_BUSY_ORDER_100_MSEC;
424 }
425
426 static uint64_t new_hpte_load0(void *htab, uint64_t pteg, int slot)
427 {
428 uint8_t *addr = htab;
429
430 addr += pteg * HASH_PTEG_SIZE_64;
431 addr += slot * HASH_PTE_SIZE_64;
432 return ldq_p(addr);
433 }
434
435 static void new_hpte_store(void *htab, uint64_t pteg, int slot,
436 uint64_t pte0, uint64_t pte1)
437 {
438 uint8_t *addr = htab;
439
440 addr += pteg * HASH_PTEG_SIZE_64;
441 addr += slot * HASH_PTE_SIZE_64;
442
443 stq_p(addr, pte0);
444 stq_p(addr + HASH_PTE_SIZE_64 / 2, pte1);
445 }
446
447 static int rehash_hpte(PowerPCCPU *cpu,
448 const ppc_hash_pte64_t *hptes,
449 void *old_hpt, uint64_t oldsize,
450 void *new_hpt, uint64_t newsize,
451 uint64_t pteg, int slot)
452 {
453 uint64_t old_hash_mask = (oldsize >> 7) - 1;
454 uint64_t new_hash_mask = (newsize >> 7) - 1;
455 target_ulong pte0 = ppc_hash64_hpte0(cpu, hptes, slot);
456 target_ulong pte1;
457 uint64_t avpn;
458 unsigned base_pg_shift;
459 uint64_t hash, new_pteg, replace_pte0;
460
461 if (!(pte0 & HPTE64_V_VALID) || !(pte0 & HPTE64_V_BOLTED)) {
462 return H_SUCCESS;
463 }
464
465 pte1 = ppc_hash64_hpte1(cpu, hptes, slot);
466
467 base_pg_shift = ppc_hash64_hpte_page_shift_noslb(cpu, pte0, pte1);
468 assert(base_pg_shift); /* H_ENTER shouldn't allow a bad encoding */
469 avpn = HPTE64_V_AVPN_VAL(pte0) & ~(((1ULL << base_pg_shift) - 1) >> 23);
470
471 if (pte0 & HPTE64_V_SECONDARY) {
472 pteg = ~pteg;
473 }
474
475 if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_256M) {
476 uint64_t offset, vsid;
477
478 /* We only have 28 - 23 bits of offset in avpn */
479 offset = (avpn & 0x1f) << 23;
480 vsid = avpn >> 5;
481 /* We can find more bits from the pteg value */
482 if (base_pg_shift < 23) {
483 offset |= ((vsid ^ pteg) & old_hash_mask) << base_pg_shift;
484 }
485
486 hash = vsid ^ (offset >> base_pg_shift);
487 } else if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_1T) {
488 uint64_t offset, vsid;
489
490 /* We only have 40 - 23 bits of seg_off in avpn */
491 offset = (avpn & 0x1ffff) << 23;
492 vsid = avpn >> 17;
493 if (base_pg_shift < 23) {
494 offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask)
495 << base_pg_shift;
496 }
497
498 hash = vsid ^ (vsid << 25) ^ (offset >> base_pg_shift);
499 } else {
500 error_report("rehash_pte: Bad segment size in HPTE");
501 return H_HARDWARE;
502 }
503
504 new_pteg = hash & new_hash_mask;
505 if (pte0 & HPTE64_V_SECONDARY) {
506 assert(~pteg == (hash & old_hash_mask));
507 new_pteg = ~new_pteg;
508 } else {
509 assert(pteg == (hash & old_hash_mask));
510 }
511 assert((oldsize != newsize) || (pteg == new_pteg));
512 replace_pte0 = new_hpte_load0(new_hpt, new_pteg, slot);
513 /*
514 * Strictly speaking, we don't need all these tests, since we only
515 * ever rehash bolted HPTEs. We might in future handle non-bolted
516 * HPTEs, though so make the logic correct for those cases as
517 * well.
518 */
519 if (replace_pte0 & HPTE64_V_VALID) {
520 assert(newsize < oldsize);
521 if (replace_pte0 & HPTE64_V_BOLTED) {
522 if (pte0 & HPTE64_V_BOLTED) {
523 /* Bolted collision, nothing we can do */
524 return H_PTEG_FULL;
525 } else {
526 /* Discard this hpte */
527 return H_SUCCESS;
528 }
529 }
530 }
531
532 new_hpte_store(new_hpt, new_pteg, slot, pte0, pte1);
533 return H_SUCCESS;
534 }
535
536 static int rehash_hpt(PowerPCCPU *cpu,
537 void *old_hpt, uint64_t oldsize,
538 void *new_hpt, uint64_t newsize)
539 {
540 uint64_t n_ptegs = oldsize >> 7;
541 uint64_t pteg;
542 int slot;
543 int rc;
544
545 for (pteg = 0; pteg < n_ptegs; pteg++) {
546 hwaddr ptex = pteg * HPTES_PER_GROUP;
547 const ppc_hash_pte64_t *hptes
548 = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
549
550 if (!hptes) {
551 return H_HARDWARE;
552 }
553
554 for (slot = 0; slot < HPTES_PER_GROUP; slot++) {
555 rc = rehash_hpte(cpu, hptes, old_hpt, oldsize, new_hpt, newsize,
556 pteg, slot);
557 if (rc != H_SUCCESS) {
558 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
559 return rc;
560 }
561 }
562 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
563 }
564
565 return H_SUCCESS;
566 }
567
568 target_ulong softmmu_resize_hpt_commit(PowerPCCPU *cpu,
569 SpaprMachineState *spapr,
570 target_ulong flags,
571 target_ulong shift)
572 {
573 SpaprPendingHpt *pending = spapr->pending_hpt;
574 int rc;
575 size_t newsize;
576
577 if (flags != 0) {
578 return H_PARAMETER;
579 }
580
581 if (!pending || (pending->shift != shift)) {
582 /* no matching prepare */
583 return H_CLOSED;
584 }
585
586 if (!pending->complete) {
587 /* prepare has not completed */
588 return H_BUSY;
589 }
590
591 /* Shouldn't have got past PREPARE without an HPT */
592 g_assert(spapr->htab_shift);
593
594 newsize = 1ULL << pending->shift;
595 rc = rehash_hpt(cpu, spapr->htab, HTAB_SIZE(spapr),
596 pending->hpt, newsize);
597 if (rc == H_SUCCESS) {
598 qemu_vfree(spapr->htab);
599 spapr->htab = pending->hpt;
600 spapr->htab_shift = pending->shift;
601
602 push_sregs_to_kvm_pr(spapr);
603
604 pending->hpt = NULL; /* so it's not free()d */
605 }
606
607 /* Clean up */
608 spapr->pending_hpt = NULL;
609 free_pending_hpt(pending);
610
611 return rc;
612 }
613
614 static void hypercall_register_types(void)
615 {
616 /* hcall-pft */
617 spapr_register_hypercall(H_ENTER, h_enter);
618 spapr_register_hypercall(H_REMOVE, h_remove);
619 spapr_register_hypercall(H_PROTECT, h_protect);
620 spapr_register_hypercall(H_READ, h_read);
621
622 /* hcall-bulk */
623 spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
624
625 }
626
627 type_init(hypercall_register_types)