Merge tag 'pull-la-20220805' of https://gitlab.com/rth7680/qemu into staging
[qemu.git] / hw / ppc / spapr_softmmu.c
1 #include "qemu/osdep.h"
2 #include "qemu/cutils.h"
3 #include "qemu/memalign.h"
4 #include "cpu.h"
5 #include "helper_regs.h"
6 #include "hw/ppc/spapr.h"
7 #include "mmu-hash64.h"
8 #include "mmu-book3s-v3.h"
9
10 static inline bool valid_ptex(PowerPCCPU *cpu, target_ulong ptex)
11 {
12 /*
13 * hash value/pteg group index is normalized by HPT mask
14 */
15 if (((ptex & ~7ULL) / HPTES_PER_GROUP) & ~ppc_hash64_hpt_mask(cpu)) {
16 return false;
17 }
18 return true;
19 }
20
21 static target_ulong h_enter(PowerPCCPU *cpu, SpaprMachineState *spapr,
22 target_ulong opcode, target_ulong *args)
23 {
24 target_ulong flags = args[0];
25 target_ulong ptex = args[1];
26 target_ulong pteh = args[2];
27 target_ulong ptel = args[3];
28 unsigned apshift;
29 target_ulong raddr;
30 target_ulong slot;
31 const ppc_hash_pte64_t *hptes;
32
33 apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel);
34 if (!apshift) {
35 /* Bad page size encoding */
36 return H_PARAMETER;
37 }
38
39 raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
40
41 if (is_ram_address(spapr, raddr)) {
42 /* Regular RAM - should have WIMG=0010 */
43 if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
44 return H_PARAMETER;
45 }
46 } else {
47 target_ulong wimg_flags;
48 /* Looks like an IO address */
49 /* FIXME: What WIMG combinations could be sensible for IO?
50 * For now we allow WIMG=010x, but are there others? */
51 /* FIXME: Should we check against registered IO addresses? */
52 wimg_flags = (ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M));
53
54 if (wimg_flags != HPTE64_R_I &&
55 wimg_flags != (HPTE64_R_I | HPTE64_R_M)) {
56 return H_PARAMETER;
57 }
58 }
59
60 pteh &= ~0x60ULL;
61
62 if (!valid_ptex(cpu, ptex)) {
63 return H_PARAMETER;
64 }
65
66 slot = ptex & 7ULL;
67 ptex = ptex & ~7ULL;
68
69 if (likely((flags & H_EXACT) == 0)) {
70 hptes = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
71 for (slot = 0; slot < 8; slot++) {
72 if (!(ppc_hash64_hpte0(cpu, hptes, slot) & HPTE64_V_VALID)) {
73 break;
74 }
75 }
76 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
77 if (slot == 8) {
78 return H_PTEG_FULL;
79 }
80 } else {
81 hptes = ppc_hash64_map_hptes(cpu, ptex + slot, 1);
82 if (ppc_hash64_hpte0(cpu, hptes, 0) & HPTE64_V_VALID) {
83 ppc_hash64_unmap_hptes(cpu, hptes, ptex + slot, 1);
84 return H_PTEG_FULL;
85 }
86 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
87 }
88
89 spapr_store_hpte(cpu, ptex + slot, pteh | HPTE64_V_HPTE_DIRTY, ptel);
90
91 args[0] = ptex + slot;
92 return H_SUCCESS;
93 }
94
95 typedef enum {
96 REMOVE_SUCCESS = 0,
97 REMOVE_NOT_FOUND = 1,
98 REMOVE_PARM = 2,
99 REMOVE_HW = 3,
100 } RemoveResult;
101
102 static RemoveResult remove_hpte(PowerPCCPU *cpu
103 , target_ulong ptex,
104 target_ulong avpn,
105 target_ulong flags,
106 target_ulong *vp, target_ulong *rp)
107 {
108 const ppc_hash_pte64_t *hptes;
109 target_ulong v, r;
110
111 if (!valid_ptex(cpu, ptex)) {
112 return REMOVE_PARM;
113 }
114
115 hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
116 v = ppc_hash64_hpte0(cpu, hptes, 0);
117 r = ppc_hash64_hpte1(cpu, hptes, 0);
118 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
119
120 if ((v & HPTE64_V_VALID) == 0 ||
121 ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
122 ((flags & H_ANDCOND) && (v & avpn) != 0)) {
123 return REMOVE_NOT_FOUND;
124 }
125 *vp = v;
126 *rp = r;
127 spapr_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
128 ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
129 return REMOVE_SUCCESS;
130 }
131
132 static target_ulong h_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
133 target_ulong opcode, target_ulong *args)
134 {
135 CPUPPCState *env = &cpu->env;
136 target_ulong flags = args[0];
137 target_ulong ptex = args[1];
138 target_ulong avpn = args[2];
139 RemoveResult ret;
140
141 ret = remove_hpte(cpu, ptex, avpn, flags,
142 &args[0], &args[1]);
143
144 switch (ret) {
145 case REMOVE_SUCCESS:
146 check_tlb_flush(env, true);
147 return H_SUCCESS;
148
149 case REMOVE_NOT_FOUND:
150 return H_NOT_FOUND;
151
152 case REMOVE_PARM:
153 return H_PARAMETER;
154
155 case REMOVE_HW:
156 return H_HARDWARE;
157 }
158
159 g_assert_not_reached();
160 }
161
162 #define H_BULK_REMOVE_TYPE 0xc000000000000000ULL
163 #define H_BULK_REMOVE_REQUEST 0x4000000000000000ULL
164 #define H_BULK_REMOVE_RESPONSE 0x8000000000000000ULL
165 #define H_BULK_REMOVE_END 0xc000000000000000ULL
166 #define H_BULK_REMOVE_CODE 0x3000000000000000ULL
167 #define H_BULK_REMOVE_SUCCESS 0x0000000000000000ULL
168 #define H_BULK_REMOVE_NOT_FOUND 0x1000000000000000ULL
169 #define H_BULK_REMOVE_PARM 0x2000000000000000ULL
170 #define H_BULK_REMOVE_HW 0x3000000000000000ULL
171 #define H_BULK_REMOVE_RC 0x0c00000000000000ULL
172 #define H_BULK_REMOVE_FLAGS 0x0300000000000000ULL
173 #define H_BULK_REMOVE_ABSOLUTE 0x0000000000000000ULL
174 #define H_BULK_REMOVE_ANDCOND 0x0100000000000000ULL
175 #define H_BULK_REMOVE_AVPN 0x0200000000000000ULL
176 #define H_BULK_REMOVE_PTEX 0x00ffffffffffffffULL
177
178 #define H_BULK_REMOVE_MAX_BATCH 4
179
180 static target_ulong h_bulk_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
181 target_ulong opcode, target_ulong *args)
182 {
183 CPUPPCState *env = &cpu->env;
184 int i;
185 target_ulong rc = H_SUCCESS;
186
187 for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
188 target_ulong *tsh = &args[i*2];
189 target_ulong tsl = args[i*2 + 1];
190 target_ulong v, r, ret;
191
192 if ((*tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) {
193 break;
194 } else if ((*tsh & H_BULK_REMOVE_TYPE) != H_BULK_REMOVE_REQUEST) {
195 return H_PARAMETER;
196 }
197
198 *tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS;
199 *tsh |= H_BULK_REMOVE_RESPONSE;
200
201 if ((*tsh & H_BULK_REMOVE_ANDCOND) && (*tsh & H_BULK_REMOVE_AVPN)) {
202 *tsh |= H_BULK_REMOVE_PARM;
203 return H_PARAMETER;
204 }
205
206 ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
207 (*tsh & H_BULK_REMOVE_FLAGS) >> 26,
208 &v, &r);
209
210 *tsh |= ret << 60;
211
212 switch (ret) {
213 case REMOVE_SUCCESS:
214 *tsh |= (r & (HPTE64_R_C | HPTE64_R_R)) << 43;
215 break;
216
217 case REMOVE_PARM:
218 rc = H_PARAMETER;
219 goto exit;
220
221 case REMOVE_HW:
222 rc = H_HARDWARE;
223 goto exit;
224 }
225 }
226 exit:
227 check_tlb_flush(env, true);
228
229 return rc;
230 }
231
232 static target_ulong h_protect(PowerPCCPU *cpu, SpaprMachineState *spapr,
233 target_ulong opcode, target_ulong *args)
234 {
235 CPUPPCState *env = &cpu->env;
236 target_ulong flags = args[0];
237 target_ulong ptex = args[1];
238 target_ulong avpn = args[2];
239 const ppc_hash_pte64_t *hptes;
240 target_ulong v, r;
241
242 if (!valid_ptex(cpu, ptex)) {
243 return H_PARAMETER;
244 }
245
246 hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
247 v = ppc_hash64_hpte0(cpu, hptes, 0);
248 r = ppc_hash64_hpte1(cpu, hptes, 0);
249 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
250
251 if ((v & HPTE64_V_VALID) == 0 ||
252 ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
253 return H_NOT_FOUND;
254 }
255
256 r &= ~(HPTE64_R_PP0 | HPTE64_R_PP | HPTE64_R_N |
257 HPTE64_R_KEY_HI | HPTE64_R_KEY_LO);
258 r |= (flags << 55) & HPTE64_R_PP0;
259 r |= (flags << 48) & HPTE64_R_KEY_HI;
260 r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
261 spapr_store_hpte(cpu, ptex,
262 (v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
263 ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
264 /* Flush the tlb */
265 check_tlb_flush(env, true);
266 /* Don't need a memory barrier, due to qemu's global lock */
267 spapr_store_hpte(cpu, ptex, v | HPTE64_V_HPTE_DIRTY, r);
268 return H_SUCCESS;
269 }
270
271 static target_ulong h_read(PowerPCCPU *cpu, SpaprMachineState *spapr,
272 target_ulong opcode, target_ulong *args)
273 {
274 target_ulong flags = args[0];
275 target_ulong ptex = args[1];
276 int i, ridx, n_entries = 1;
277 const ppc_hash_pte64_t *hptes;
278
279 if (!valid_ptex(cpu, ptex)) {
280 return H_PARAMETER;
281 }
282
283 if (flags & H_READ_4) {
284 /* Clear the two low order bits */
285 ptex &= ~(3ULL);
286 n_entries = 4;
287 }
288
289 hptes = ppc_hash64_map_hptes(cpu, ptex, n_entries);
290 for (i = 0, ridx = 0; i < n_entries; i++) {
291 args[ridx++] = ppc_hash64_hpte0(cpu, hptes, i);
292 args[ridx++] = ppc_hash64_hpte1(cpu, hptes, i);
293 }
294 ppc_hash64_unmap_hptes(cpu, hptes, ptex, n_entries);
295
296 return H_SUCCESS;
297 }
298
299 struct SpaprPendingHpt {
300 /* These fields are read-only after initialization */
301 int shift;
302 QemuThread thread;
303
304 /* These fields are protected by the BQL */
305 bool complete;
306
307 /* These fields are private to the preparation thread if
308 * !complete, otherwise protected by the BQL */
309 int ret;
310 void *hpt;
311 };
312
313 static void free_pending_hpt(SpaprPendingHpt *pending)
314 {
315 if (pending->hpt) {
316 qemu_vfree(pending->hpt);
317 }
318
319 g_free(pending);
320 }
321
322 static void *hpt_prepare_thread(void *opaque)
323 {
324 SpaprPendingHpt *pending = opaque;
325 size_t size = 1ULL << pending->shift;
326
327 pending->hpt = qemu_try_memalign(size, size);
328 if (pending->hpt) {
329 memset(pending->hpt, 0, size);
330 pending->ret = H_SUCCESS;
331 } else {
332 pending->ret = H_NO_MEM;
333 }
334
335 qemu_mutex_lock_iothread();
336
337 if (SPAPR_MACHINE(qdev_get_machine())->pending_hpt == pending) {
338 /* Ready to go */
339 pending->complete = true;
340 } else {
341 /* We've been cancelled, clean ourselves up */
342 free_pending_hpt(pending);
343 }
344
345 qemu_mutex_unlock_iothread();
346 return NULL;
347 }
348
349 /* Must be called with BQL held */
350 static void cancel_hpt_prepare(SpaprMachineState *spapr)
351 {
352 SpaprPendingHpt *pending = spapr->pending_hpt;
353
354 /* Let the thread know it's cancelled */
355 spapr->pending_hpt = NULL;
356
357 if (!pending) {
358 /* Nothing to do */
359 return;
360 }
361
362 if (!pending->complete) {
363 /* thread will clean itself up */
364 return;
365 }
366
367 free_pending_hpt(pending);
368 }
369
370 target_ulong softmmu_resize_hpt_prepare(PowerPCCPU *cpu,
371 SpaprMachineState *spapr,
372 target_ulong shift)
373 {
374 SpaprPendingHpt *pending = spapr->pending_hpt;
375
376 if (pending) {
377 /* something already in progress */
378 if (pending->shift == shift) {
379 /* and it's suitable */
380 if (pending->complete) {
381 return pending->ret;
382 } else {
383 return H_LONG_BUSY_ORDER_100_MSEC;
384 }
385 }
386
387 /* not suitable, cancel and replace */
388 cancel_hpt_prepare(spapr);
389 }
390
391 if (!shift) {
392 /* nothing to do */
393 return H_SUCCESS;
394 }
395
396 /* start new prepare */
397
398 pending = g_new0(SpaprPendingHpt, 1);
399 pending->shift = shift;
400 pending->ret = H_HARDWARE;
401
402 qemu_thread_create(&pending->thread, "sPAPR HPT prepare",
403 hpt_prepare_thread, pending, QEMU_THREAD_DETACHED);
404
405 spapr->pending_hpt = pending;
406
407 /* In theory we could estimate the time more accurately based on
408 * the new size, but there's not much point */
409 return H_LONG_BUSY_ORDER_100_MSEC;
410 }
411
412 static uint64_t new_hpte_load0(void *htab, uint64_t pteg, int slot)
413 {
414 uint8_t *addr = htab;
415
416 addr += pteg * HASH_PTEG_SIZE_64;
417 addr += slot * HASH_PTE_SIZE_64;
418 return ldq_p(addr);
419 }
420
421 static void new_hpte_store(void *htab, uint64_t pteg, int slot,
422 uint64_t pte0, uint64_t pte1)
423 {
424 uint8_t *addr = htab;
425
426 addr += pteg * HASH_PTEG_SIZE_64;
427 addr += slot * HASH_PTE_SIZE_64;
428
429 stq_p(addr, pte0);
430 stq_p(addr + HPTE64_DW1, pte1);
431 }
432
433 static int rehash_hpte(PowerPCCPU *cpu,
434 const ppc_hash_pte64_t *hptes,
435 void *old_hpt, uint64_t oldsize,
436 void *new_hpt, uint64_t newsize,
437 uint64_t pteg, int slot)
438 {
439 uint64_t old_hash_mask = (oldsize >> 7) - 1;
440 uint64_t new_hash_mask = (newsize >> 7) - 1;
441 target_ulong pte0 = ppc_hash64_hpte0(cpu, hptes, slot);
442 target_ulong pte1;
443 uint64_t avpn;
444 unsigned base_pg_shift;
445 uint64_t hash, new_pteg, replace_pte0;
446
447 if (!(pte0 & HPTE64_V_VALID) || !(pte0 & HPTE64_V_BOLTED)) {
448 return H_SUCCESS;
449 }
450
451 pte1 = ppc_hash64_hpte1(cpu, hptes, slot);
452
453 base_pg_shift = ppc_hash64_hpte_page_shift_noslb(cpu, pte0, pte1);
454 assert(base_pg_shift); /* H_ENTER shouldn't allow a bad encoding */
455 avpn = HPTE64_V_AVPN_VAL(pte0) & ~(((1ULL << base_pg_shift) - 1) >> 23);
456
457 if (pte0 & HPTE64_V_SECONDARY) {
458 pteg = ~pteg;
459 }
460
461 if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_256M) {
462 uint64_t offset, vsid;
463
464 /* We only have 28 - 23 bits of offset in avpn */
465 offset = (avpn & 0x1f) << 23;
466 vsid = avpn >> 5;
467 /* We can find more bits from the pteg value */
468 if (base_pg_shift < 23) {
469 offset |= ((vsid ^ pteg) & old_hash_mask) << base_pg_shift;
470 }
471
472 hash = vsid ^ (offset >> base_pg_shift);
473 } else if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_1T) {
474 uint64_t offset, vsid;
475
476 /* We only have 40 - 23 bits of seg_off in avpn */
477 offset = (avpn & 0x1ffff) << 23;
478 vsid = avpn >> 17;
479 if (base_pg_shift < 23) {
480 offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask)
481 << base_pg_shift;
482 }
483
484 hash = vsid ^ (vsid << 25) ^ (offset >> base_pg_shift);
485 } else {
486 error_report("rehash_pte: Bad segment size in HPTE");
487 return H_HARDWARE;
488 }
489
490 new_pteg = hash & new_hash_mask;
491 if (pte0 & HPTE64_V_SECONDARY) {
492 assert(~pteg == (hash & old_hash_mask));
493 new_pteg = ~new_pteg;
494 } else {
495 assert(pteg == (hash & old_hash_mask));
496 }
497 assert((oldsize != newsize) || (pteg == new_pteg));
498 replace_pte0 = new_hpte_load0(new_hpt, new_pteg, slot);
499 /*
500 * Strictly speaking, we don't need all these tests, since we only
501 * ever rehash bolted HPTEs. We might in future handle non-bolted
502 * HPTEs, though so make the logic correct for those cases as
503 * well.
504 */
505 if (replace_pte0 & HPTE64_V_VALID) {
506 assert(newsize < oldsize);
507 if (replace_pte0 & HPTE64_V_BOLTED) {
508 if (pte0 & HPTE64_V_BOLTED) {
509 /* Bolted collision, nothing we can do */
510 return H_PTEG_FULL;
511 } else {
512 /* Discard this hpte */
513 return H_SUCCESS;
514 }
515 }
516 }
517
518 new_hpte_store(new_hpt, new_pteg, slot, pte0, pte1);
519 return H_SUCCESS;
520 }
521
522 static int rehash_hpt(PowerPCCPU *cpu,
523 void *old_hpt, uint64_t oldsize,
524 void *new_hpt, uint64_t newsize)
525 {
526 uint64_t n_ptegs = oldsize >> 7;
527 uint64_t pteg;
528 int slot;
529 int rc;
530
531 for (pteg = 0; pteg < n_ptegs; pteg++) {
532 hwaddr ptex = pteg * HPTES_PER_GROUP;
533 const ppc_hash_pte64_t *hptes
534 = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
535
536 if (!hptes) {
537 return H_HARDWARE;
538 }
539
540 for (slot = 0; slot < HPTES_PER_GROUP; slot++) {
541 rc = rehash_hpte(cpu, hptes, old_hpt, oldsize, new_hpt, newsize,
542 pteg, slot);
543 if (rc != H_SUCCESS) {
544 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
545 return rc;
546 }
547 }
548 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
549 }
550
551 return H_SUCCESS;
552 }
553
554 target_ulong softmmu_resize_hpt_commit(PowerPCCPU *cpu,
555 SpaprMachineState *spapr,
556 target_ulong flags,
557 target_ulong shift)
558 {
559 SpaprPendingHpt *pending = spapr->pending_hpt;
560 int rc;
561 size_t newsize;
562
563 if (flags != 0) {
564 return H_PARAMETER;
565 }
566
567 if (!pending || (pending->shift != shift)) {
568 /* no matching prepare */
569 return H_CLOSED;
570 }
571
572 if (!pending->complete) {
573 /* prepare has not completed */
574 return H_BUSY;
575 }
576
577 /* Shouldn't have got past PREPARE without an HPT */
578 g_assert(spapr->htab_shift);
579
580 newsize = 1ULL << pending->shift;
581 rc = rehash_hpt(cpu, spapr->htab, HTAB_SIZE(spapr),
582 pending->hpt, newsize);
583 if (rc == H_SUCCESS) {
584 qemu_vfree(spapr->htab);
585 spapr->htab = pending->hpt;
586 spapr->htab_shift = pending->shift;
587
588 push_sregs_to_kvm_pr(spapr);
589
590 pending->hpt = NULL; /* so it's not free()d */
591 }
592
593 /* Clean up */
594 spapr->pending_hpt = NULL;
595 free_pending_hpt(pending);
596
597 return rc;
598 }
599
600 static void hypercall_register_types(void)
601 {
602 /* hcall-pft */
603 spapr_register_hypercall(H_ENTER, h_enter);
604 spapr_register_hypercall(H_REMOVE, h_remove);
605 spapr_register_hypercall(H_PROTECT, h_protect);
606 spapr_register_hypercall(H_READ, h_read);
607
608 /* hcall-bulk */
609 spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
610
611 }
612
613 type_init(hypercall_register_types)