hw/arm/bcm2836: Only provide "enabled-cpus" property to multicore SoCs
[qemu.git] / hw / arm / smmuv3.c
1 /*
2 * Copyright (C) 2014-2016 Broadcom Corporation
3 * Copyright (c) 2017 Red Hat, Inc.
4 * Written by Prem Mallappa, Eric Auger
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, see <http://www.gnu.org/licenses/>.
17 */
18
19 #include "qemu/osdep.h"
20 #include "hw/irq.h"
21 #include "hw/sysbus.h"
22 #include "migration/vmstate.h"
23 #include "hw/qdev-core.h"
24 #include "hw/pci/pci.h"
25 #include "exec/address-spaces.h"
26 #include "cpu.h"
27 #include "trace.h"
28 #include "qemu/log.h"
29 #include "qemu/error-report.h"
30 #include "qapi/error.h"
31
32 #include "hw/arm/smmuv3.h"
33 #include "smmuv3-internal.h"
34
35 /**
36 * smmuv3_trigger_irq - pulse @irq if enabled and update
37 * GERROR register in case of GERROR interrupt
38 *
39 * @irq: irq type
40 * @gerror_mask: mask of gerrors to toggle (relevant if @irq is GERROR)
41 */
42 static void smmuv3_trigger_irq(SMMUv3State *s, SMMUIrq irq,
43 uint32_t gerror_mask)
44 {
45
46 bool pulse = false;
47
48 switch (irq) {
49 case SMMU_IRQ_EVTQ:
50 pulse = smmuv3_eventq_irq_enabled(s);
51 break;
52 case SMMU_IRQ_PRIQ:
53 qemu_log_mask(LOG_UNIMP, "PRI not yet supported\n");
54 break;
55 case SMMU_IRQ_CMD_SYNC:
56 pulse = true;
57 break;
58 case SMMU_IRQ_GERROR:
59 {
60 uint32_t pending = s->gerror ^ s->gerrorn;
61 uint32_t new_gerrors = ~pending & gerror_mask;
62
63 if (!new_gerrors) {
64 /* only toggle non pending errors */
65 return;
66 }
67 s->gerror ^= new_gerrors;
68 trace_smmuv3_write_gerror(new_gerrors, s->gerror);
69
70 pulse = smmuv3_gerror_irq_enabled(s);
71 break;
72 }
73 }
74 if (pulse) {
75 trace_smmuv3_trigger_irq(irq);
76 qemu_irq_pulse(s->irq[irq]);
77 }
78 }
79
80 static void smmuv3_write_gerrorn(SMMUv3State *s, uint32_t new_gerrorn)
81 {
82 uint32_t pending = s->gerror ^ s->gerrorn;
83 uint32_t toggled = s->gerrorn ^ new_gerrorn;
84
85 if (toggled & ~pending) {
86 qemu_log_mask(LOG_GUEST_ERROR,
87 "guest toggles non pending errors = 0x%x\n",
88 toggled & ~pending);
89 }
90
91 /*
92 * We do not raise any error in case guest toggles bits corresponding
93 * to not active IRQs (CONSTRAINED UNPREDICTABLE)
94 */
95 s->gerrorn = new_gerrorn;
96
97 trace_smmuv3_write_gerrorn(toggled & pending, s->gerrorn);
98 }
99
100 static inline MemTxResult queue_read(SMMUQueue *q, void *data)
101 {
102 dma_addr_t addr = Q_CONS_ENTRY(q);
103
104 return dma_memory_read(&address_space_memory, addr, data, q->entry_size);
105 }
106
107 static MemTxResult queue_write(SMMUQueue *q, void *data)
108 {
109 dma_addr_t addr = Q_PROD_ENTRY(q);
110 MemTxResult ret;
111
112 ret = dma_memory_write(&address_space_memory, addr, data, q->entry_size);
113 if (ret != MEMTX_OK) {
114 return ret;
115 }
116
117 queue_prod_incr(q);
118 return MEMTX_OK;
119 }
120
121 static MemTxResult smmuv3_write_eventq(SMMUv3State *s, Evt *evt)
122 {
123 SMMUQueue *q = &s->eventq;
124 MemTxResult r;
125
126 if (!smmuv3_eventq_enabled(s)) {
127 return MEMTX_ERROR;
128 }
129
130 if (smmuv3_q_full(q)) {
131 return MEMTX_ERROR;
132 }
133
134 r = queue_write(q, evt);
135 if (r != MEMTX_OK) {
136 return r;
137 }
138
139 if (!smmuv3_q_empty(q)) {
140 smmuv3_trigger_irq(s, SMMU_IRQ_EVTQ, 0);
141 }
142 return MEMTX_OK;
143 }
144
145 void smmuv3_record_event(SMMUv3State *s, SMMUEventInfo *info)
146 {
147 Evt evt = {};
148 MemTxResult r;
149
150 if (!smmuv3_eventq_enabled(s)) {
151 return;
152 }
153
154 EVT_SET_TYPE(&evt, info->type);
155 EVT_SET_SID(&evt, info->sid);
156
157 switch (info->type) {
158 case SMMU_EVT_NONE:
159 return;
160 case SMMU_EVT_F_UUT:
161 EVT_SET_SSID(&evt, info->u.f_uut.ssid);
162 EVT_SET_SSV(&evt, info->u.f_uut.ssv);
163 EVT_SET_ADDR(&evt, info->u.f_uut.addr);
164 EVT_SET_RNW(&evt, info->u.f_uut.rnw);
165 EVT_SET_PNU(&evt, info->u.f_uut.pnu);
166 EVT_SET_IND(&evt, info->u.f_uut.ind);
167 break;
168 case SMMU_EVT_C_BAD_STREAMID:
169 EVT_SET_SSID(&evt, info->u.c_bad_streamid.ssid);
170 EVT_SET_SSV(&evt, info->u.c_bad_streamid.ssv);
171 break;
172 case SMMU_EVT_F_STE_FETCH:
173 EVT_SET_SSID(&evt, info->u.f_ste_fetch.ssid);
174 EVT_SET_SSV(&evt, info->u.f_ste_fetch.ssv);
175 EVT_SET_ADDR2(&evt, info->u.f_ste_fetch.addr);
176 break;
177 case SMMU_EVT_C_BAD_STE:
178 EVT_SET_SSID(&evt, info->u.c_bad_ste.ssid);
179 EVT_SET_SSV(&evt, info->u.c_bad_ste.ssv);
180 break;
181 case SMMU_EVT_F_STREAM_DISABLED:
182 break;
183 case SMMU_EVT_F_TRANS_FORBIDDEN:
184 EVT_SET_ADDR(&evt, info->u.f_transl_forbidden.addr);
185 EVT_SET_RNW(&evt, info->u.f_transl_forbidden.rnw);
186 break;
187 case SMMU_EVT_C_BAD_SUBSTREAMID:
188 EVT_SET_SSID(&evt, info->u.c_bad_substream.ssid);
189 break;
190 case SMMU_EVT_F_CD_FETCH:
191 EVT_SET_SSID(&evt, info->u.f_cd_fetch.ssid);
192 EVT_SET_SSV(&evt, info->u.f_cd_fetch.ssv);
193 EVT_SET_ADDR(&evt, info->u.f_cd_fetch.addr);
194 break;
195 case SMMU_EVT_C_BAD_CD:
196 EVT_SET_SSID(&evt, info->u.c_bad_cd.ssid);
197 EVT_SET_SSV(&evt, info->u.c_bad_cd.ssv);
198 break;
199 case SMMU_EVT_F_WALK_EABT:
200 case SMMU_EVT_F_TRANSLATION:
201 case SMMU_EVT_F_ADDR_SIZE:
202 case SMMU_EVT_F_ACCESS:
203 case SMMU_EVT_F_PERMISSION:
204 EVT_SET_STALL(&evt, info->u.f_walk_eabt.stall);
205 EVT_SET_STAG(&evt, info->u.f_walk_eabt.stag);
206 EVT_SET_SSID(&evt, info->u.f_walk_eabt.ssid);
207 EVT_SET_SSV(&evt, info->u.f_walk_eabt.ssv);
208 EVT_SET_S2(&evt, info->u.f_walk_eabt.s2);
209 EVT_SET_ADDR(&evt, info->u.f_walk_eabt.addr);
210 EVT_SET_RNW(&evt, info->u.f_walk_eabt.rnw);
211 EVT_SET_PNU(&evt, info->u.f_walk_eabt.pnu);
212 EVT_SET_IND(&evt, info->u.f_walk_eabt.ind);
213 EVT_SET_CLASS(&evt, info->u.f_walk_eabt.class);
214 EVT_SET_ADDR2(&evt, info->u.f_walk_eabt.addr2);
215 break;
216 case SMMU_EVT_F_CFG_CONFLICT:
217 EVT_SET_SSID(&evt, info->u.f_cfg_conflict.ssid);
218 EVT_SET_SSV(&evt, info->u.f_cfg_conflict.ssv);
219 break;
220 /* rest is not implemented */
221 case SMMU_EVT_F_BAD_ATS_TREQ:
222 case SMMU_EVT_F_TLB_CONFLICT:
223 case SMMU_EVT_E_PAGE_REQ:
224 default:
225 g_assert_not_reached();
226 }
227
228 trace_smmuv3_record_event(smmu_event_string(info->type), info->sid);
229 r = smmuv3_write_eventq(s, &evt);
230 if (r != MEMTX_OK) {
231 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_EVENTQ_ABT_ERR_MASK);
232 }
233 info->recorded = true;
234 }
235
236 static void smmuv3_init_regs(SMMUv3State *s)
237 {
238 /**
239 * IDR0: stage1 only, AArch64 only, coherent access, 16b ASID,
240 * multi-level stream table
241 */
242 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, S1P, 1); /* stage 1 supported */
243 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTF, 2); /* AArch64 PTW only */
244 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, COHACC, 1); /* IO coherent */
245 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ASID16, 1); /* 16-bit ASID */
246 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TTENDIAN, 2); /* little endian */
247 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STALL_MODEL, 1); /* No stall */
248 /* terminated transaction will always be aborted/error returned */
249 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, TERM_MODEL, 1);
250 /* 2-level stream table supported */
251 s->idr[0] = FIELD_DP32(s->idr[0], IDR0, STLEVEL, 1);
252
253 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SIDSIZE, SMMU_IDR1_SIDSIZE);
254 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, EVENTQS, SMMU_EVENTQS);
255 s->idr[1] = FIELD_DP32(s->idr[1], IDR1, CMDQS, SMMU_CMDQS);
256
257 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 1);
258 s->idr[3] = FIELD_DP32(s->idr[3], IDR3, HAD, 1);
259
260 /* 4K and 64K granule support */
261 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN4K, 1);
262 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN64K, 1);
263 s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS); /* 44 bits */
264
265 s->cmdq.base = deposit64(s->cmdq.base, 0, 5, SMMU_CMDQS);
266 s->cmdq.prod = 0;
267 s->cmdq.cons = 0;
268 s->cmdq.entry_size = sizeof(struct Cmd);
269 s->eventq.base = deposit64(s->eventq.base, 0, 5, SMMU_EVENTQS);
270 s->eventq.prod = 0;
271 s->eventq.cons = 0;
272 s->eventq.entry_size = sizeof(struct Evt);
273
274 s->features = 0;
275 s->sid_split = 0;
276 s->aidr = 0x1;
277 }
278
279 static int smmu_get_ste(SMMUv3State *s, dma_addr_t addr, STE *buf,
280 SMMUEventInfo *event)
281 {
282 int ret;
283
284 trace_smmuv3_get_ste(addr);
285 /* TODO: guarantee 64-bit single-copy atomicity */
286 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf));
287 if (ret != MEMTX_OK) {
288 qemu_log_mask(LOG_GUEST_ERROR,
289 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
290 event->type = SMMU_EVT_F_STE_FETCH;
291 event->u.f_ste_fetch.addr = addr;
292 return -EINVAL;
293 }
294 return 0;
295
296 }
297
298 /* @ssid > 0 not supported yet */
299 static int smmu_get_cd(SMMUv3State *s, STE *ste, uint32_t ssid,
300 CD *buf, SMMUEventInfo *event)
301 {
302 dma_addr_t addr = STE_CTXPTR(ste);
303 int ret;
304
305 trace_smmuv3_get_cd(addr);
306 /* TODO: guarantee 64-bit single-copy atomicity */
307 ret = dma_memory_read(&address_space_memory, addr, buf, sizeof(*buf));
308 if (ret != MEMTX_OK) {
309 qemu_log_mask(LOG_GUEST_ERROR,
310 "Cannot fetch pte at address=0x%"PRIx64"\n", addr);
311 event->type = SMMU_EVT_F_CD_FETCH;
312 event->u.f_ste_fetch.addr = addr;
313 return -EINVAL;
314 }
315 return 0;
316 }
317
318 /* Returns < 0 in case of invalid STE, 0 otherwise */
319 static int decode_ste(SMMUv3State *s, SMMUTransCfg *cfg,
320 STE *ste, SMMUEventInfo *event)
321 {
322 uint32_t config;
323
324 if (!STE_VALID(ste)) {
325 if (!event->inval_ste_allowed) {
326 qemu_log_mask(LOG_GUEST_ERROR, "invalid STE\n");
327 }
328 goto bad_ste;
329 }
330
331 config = STE_CONFIG(ste);
332
333 if (STE_CFG_ABORT(config)) {
334 cfg->aborted = true;
335 return 0;
336 }
337
338 if (STE_CFG_BYPASS(config)) {
339 cfg->bypassed = true;
340 return 0;
341 }
342
343 if (STE_CFG_S2_ENABLED(config)) {
344 qemu_log_mask(LOG_UNIMP, "SMMUv3 does not support stage 2 yet\n");
345 goto bad_ste;
346 }
347
348 if (STE_S1CDMAX(ste) != 0) {
349 qemu_log_mask(LOG_UNIMP,
350 "SMMUv3 does not support multiple context descriptors yet\n");
351 goto bad_ste;
352 }
353
354 if (STE_S1STALLD(ste)) {
355 qemu_log_mask(LOG_UNIMP,
356 "SMMUv3 S1 stalling fault model not allowed yet\n");
357 goto bad_ste;
358 }
359 return 0;
360
361 bad_ste:
362 event->type = SMMU_EVT_C_BAD_STE;
363 return -EINVAL;
364 }
365
366 /**
367 * smmu_find_ste - Return the stream table entry associated
368 * to the sid
369 *
370 * @s: smmuv3 handle
371 * @sid: stream ID
372 * @ste: returned stream table entry
373 * @event: handle to an event info
374 *
375 * Supports linear and 2-level stream table
376 * Return 0 on success, -EINVAL otherwise
377 */
378 static int smmu_find_ste(SMMUv3State *s, uint32_t sid, STE *ste,
379 SMMUEventInfo *event)
380 {
381 dma_addr_t addr, strtab_base;
382 uint32_t log2size;
383 int strtab_size_shift;
384 int ret;
385
386 trace_smmuv3_find_ste(sid, s->features, s->sid_split);
387 log2size = FIELD_EX32(s->strtab_base_cfg, STRTAB_BASE_CFG, LOG2SIZE);
388 /*
389 * Check SID range against both guest-configured and implementation limits
390 */
391 if (sid >= (1 << MIN(log2size, SMMU_IDR1_SIDSIZE))) {
392 event->type = SMMU_EVT_C_BAD_STREAMID;
393 return -EINVAL;
394 }
395 if (s->features & SMMU_FEATURE_2LVL_STE) {
396 int l1_ste_offset, l2_ste_offset, max_l2_ste, span;
397 dma_addr_t l1ptr, l2ptr;
398 STEDesc l1std;
399
400 /*
401 * Align strtab base address to table size. For this purpose, assume it
402 * is not bounded by SMMU_IDR1_SIDSIZE.
403 */
404 strtab_size_shift = MAX(5, (int)log2size - s->sid_split - 1 + 3);
405 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
406 ~MAKE_64BIT_MASK(0, strtab_size_shift);
407 l1_ste_offset = sid >> s->sid_split;
408 l2_ste_offset = sid & ((1 << s->sid_split) - 1);
409 l1ptr = (dma_addr_t)(strtab_base + l1_ste_offset * sizeof(l1std));
410 /* TODO: guarantee 64-bit single-copy atomicity */
411 ret = dma_memory_read(&address_space_memory, l1ptr, &l1std,
412 sizeof(l1std));
413 if (ret != MEMTX_OK) {
414 qemu_log_mask(LOG_GUEST_ERROR,
415 "Could not read L1PTR at 0X%"PRIx64"\n", l1ptr);
416 event->type = SMMU_EVT_F_STE_FETCH;
417 event->u.f_ste_fetch.addr = l1ptr;
418 return -EINVAL;
419 }
420
421 span = L1STD_SPAN(&l1std);
422
423 if (!span) {
424 /* l2ptr is not valid */
425 if (!event->inval_ste_allowed) {
426 qemu_log_mask(LOG_GUEST_ERROR,
427 "invalid sid=%d (L1STD span=0)\n", sid);
428 }
429 event->type = SMMU_EVT_C_BAD_STREAMID;
430 return -EINVAL;
431 }
432 max_l2_ste = (1 << span) - 1;
433 l2ptr = l1std_l2ptr(&l1std);
434 trace_smmuv3_find_ste_2lvl(s->strtab_base, l1ptr, l1_ste_offset,
435 l2ptr, l2_ste_offset, max_l2_ste);
436 if (l2_ste_offset > max_l2_ste) {
437 qemu_log_mask(LOG_GUEST_ERROR,
438 "l2_ste_offset=%d > max_l2_ste=%d\n",
439 l2_ste_offset, max_l2_ste);
440 event->type = SMMU_EVT_C_BAD_STE;
441 return -EINVAL;
442 }
443 addr = l2ptr + l2_ste_offset * sizeof(*ste);
444 } else {
445 strtab_size_shift = log2size + 5;
446 strtab_base = s->strtab_base & SMMU_BASE_ADDR_MASK &
447 ~MAKE_64BIT_MASK(0, strtab_size_shift);
448 addr = strtab_base + sid * sizeof(*ste);
449 }
450
451 if (smmu_get_ste(s, addr, ste, event)) {
452 return -EINVAL;
453 }
454
455 return 0;
456 }
457
458 static int decode_cd(SMMUTransCfg *cfg, CD *cd, SMMUEventInfo *event)
459 {
460 int ret = -EINVAL;
461 int i;
462
463 if (!CD_VALID(cd) || !CD_AARCH64(cd)) {
464 goto bad_cd;
465 }
466 if (!CD_A(cd)) {
467 goto bad_cd; /* SMMU_IDR0.TERM_MODEL == 1 */
468 }
469 if (CD_S(cd)) {
470 goto bad_cd; /* !STE_SECURE && SMMU_IDR0.STALL_MODEL == 1 */
471 }
472 if (CD_HA(cd) || CD_HD(cd)) {
473 goto bad_cd; /* HTTU = 0 */
474 }
475
476 /* we support only those at the moment */
477 cfg->aa64 = true;
478 cfg->stage = 1;
479
480 cfg->oas = oas2bits(CD_IPS(cd));
481 cfg->oas = MIN(oas2bits(SMMU_IDR5_OAS), cfg->oas);
482 cfg->tbi = CD_TBI(cd);
483 cfg->asid = CD_ASID(cd);
484
485 trace_smmuv3_decode_cd(cfg->oas);
486
487 /* decode data dependent on TT */
488 for (i = 0; i <= 1; i++) {
489 int tg, tsz;
490 SMMUTransTableInfo *tt = &cfg->tt[i];
491
492 cfg->tt[i].disabled = CD_EPD(cd, i);
493 if (cfg->tt[i].disabled) {
494 continue;
495 }
496
497 tsz = CD_TSZ(cd, i);
498 if (tsz < 16 || tsz > 39) {
499 goto bad_cd;
500 }
501
502 tg = CD_TG(cd, i);
503 tt->granule_sz = tg2granule(tg, i);
504 if ((tt->granule_sz != 12 && tt->granule_sz != 16) || CD_ENDI(cd)) {
505 goto bad_cd;
506 }
507
508 tt->tsz = tsz;
509 tt->ttb = CD_TTB(cd, i);
510 if (tt->ttb & ~(MAKE_64BIT_MASK(0, cfg->oas))) {
511 goto bad_cd;
512 }
513 tt->had = CD_HAD(cd, i);
514 trace_smmuv3_decode_cd_tt(i, tt->tsz, tt->ttb, tt->granule_sz, tt->had);
515 }
516
517 event->record_trans_faults = CD_R(cd);
518
519 return 0;
520
521 bad_cd:
522 event->type = SMMU_EVT_C_BAD_CD;
523 return ret;
524 }
525
526 /**
527 * smmuv3_decode_config - Prepare the translation configuration
528 * for the @mr iommu region
529 * @mr: iommu memory region the translation config must be prepared for
530 * @cfg: output translation configuration which is populated through
531 * the different configuration decoding steps
532 * @event: must be zero'ed by the caller
533 *
534 * return < 0 in case of config decoding error (@event is filled
535 * accordingly). Return 0 otherwise.
536 */
537 static int smmuv3_decode_config(IOMMUMemoryRegion *mr, SMMUTransCfg *cfg,
538 SMMUEventInfo *event)
539 {
540 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
541 uint32_t sid = smmu_get_sid(sdev);
542 SMMUv3State *s = sdev->smmu;
543 int ret;
544 STE ste;
545 CD cd;
546
547 ret = smmu_find_ste(s, sid, &ste, event);
548 if (ret) {
549 return ret;
550 }
551
552 ret = decode_ste(s, cfg, &ste, event);
553 if (ret) {
554 return ret;
555 }
556
557 if (cfg->aborted || cfg->bypassed) {
558 return 0;
559 }
560
561 ret = smmu_get_cd(s, &ste, 0 /* ssid */, &cd, event);
562 if (ret) {
563 return ret;
564 }
565
566 return decode_cd(cfg, &cd, event);
567 }
568
569 /**
570 * smmuv3_get_config - Look up for a cached copy of configuration data for
571 * @sdev and on cache miss performs a configuration structure decoding from
572 * guest RAM.
573 *
574 * @sdev: SMMUDevice handle
575 * @event: output event info
576 *
577 * The configuration cache contains data resulting from both STE and CD
578 * decoding under the form of an SMMUTransCfg struct. The hash table is indexed
579 * by the SMMUDevice handle.
580 */
581 static SMMUTransCfg *smmuv3_get_config(SMMUDevice *sdev, SMMUEventInfo *event)
582 {
583 SMMUv3State *s = sdev->smmu;
584 SMMUState *bc = &s->smmu_state;
585 SMMUTransCfg *cfg;
586
587 cfg = g_hash_table_lookup(bc->configs, sdev);
588 if (cfg) {
589 sdev->cfg_cache_hits++;
590 trace_smmuv3_config_cache_hit(smmu_get_sid(sdev),
591 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
592 100 * sdev->cfg_cache_hits /
593 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
594 } else {
595 sdev->cfg_cache_misses++;
596 trace_smmuv3_config_cache_miss(smmu_get_sid(sdev),
597 sdev->cfg_cache_hits, sdev->cfg_cache_misses,
598 100 * sdev->cfg_cache_hits /
599 (sdev->cfg_cache_hits + sdev->cfg_cache_misses));
600 cfg = g_new0(SMMUTransCfg, 1);
601
602 if (!smmuv3_decode_config(&sdev->iommu, cfg, event)) {
603 g_hash_table_insert(bc->configs, sdev, cfg);
604 } else {
605 g_free(cfg);
606 cfg = NULL;
607 }
608 }
609 return cfg;
610 }
611
612 static void smmuv3_flush_config(SMMUDevice *sdev)
613 {
614 SMMUv3State *s = sdev->smmu;
615 SMMUState *bc = &s->smmu_state;
616
617 trace_smmuv3_config_cache_inv(smmu_get_sid(sdev));
618 g_hash_table_remove(bc->configs, sdev);
619 }
620
621 static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr,
622 IOMMUAccessFlags flag, int iommu_idx)
623 {
624 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
625 SMMUv3State *s = sdev->smmu;
626 uint32_t sid = smmu_get_sid(sdev);
627 SMMUEventInfo event = {.type = SMMU_EVT_NONE,
628 .sid = sid,
629 .inval_ste_allowed = false};
630 SMMUPTWEventInfo ptw_info = {};
631 SMMUTranslationStatus status;
632 SMMUState *bs = ARM_SMMU(s);
633 uint64_t page_mask, aligned_addr;
634 SMMUTLBEntry *cached_entry = NULL;
635 SMMUTransTableInfo *tt;
636 SMMUTransCfg *cfg = NULL;
637 IOMMUTLBEntry entry = {
638 .target_as = &address_space_memory,
639 .iova = addr,
640 .translated_addr = addr,
641 .addr_mask = ~(hwaddr)0,
642 .perm = IOMMU_NONE,
643 };
644
645 qemu_mutex_lock(&s->mutex);
646
647 if (!smmu_enabled(s)) {
648 status = SMMU_TRANS_DISABLE;
649 goto epilogue;
650 }
651
652 cfg = smmuv3_get_config(sdev, &event);
653 if (!cfg) {
654 status = SMMU_TRANS_ERROR;
655 goto epilogue;
656 }
657
658 if (cfg->aborted) {
659 status = SMMU_TRANS_ABORT;
660 goto epilogue;
661 }
662
663 if (cfg->bypassed) {
664 status = SMMU_TRANS_BYPASS;
665 goto epilogue;
666 }
667
668 tt = select_tt(cfg, addr);
669 if (!tt) {
670 if (event.record_trans_faults) {
671 event.type = SMMU_EVT_F_TRANSLATION;
672 event.u.f_translation.addr = addr;
673 event.u.f_translation.rnw = flag & 0x1;
674 }
675 status = SMMU_TRANS_ERROR;
676 goto epilogue;
677 }
678
679 page_mask = (1ULL << (tt->granule_sz)) - 1;
680 aligned_addr = addr & ~page_mask;
681
682 cached_entry = smmu_iotlb_lookup(bs, cfg, tt, aligned_addr);
683 if (cached_entry) {
684 if ((flag & IOMMU_WO) && !(cached_entry->entry.perm & IOMMU_WO)) {
685 status = SMMU_TRANS_ERROR;
686 if (event.record_trans_faults) {
687 event.type = SMMU_EVT_F_PERMISSION;
688 event.u.f_permission.addr = addr;
689 event.u.f_permission.rnw = flag & 0x1;
690 }
691 } else {
692 status = SMMU_TRANS_SUCCESS;
693 }
694 goto epilogue;
695 }
696
697 cached_entry = g_new0(SMMUTLBEntry, 1);
698
699 if (smmu_ptw(cfg, aligned_addr, flag, cached_entry, &ptw_info)) {
700 g_free(cached_entry);
701 switch (ptw_info.type) {
702 case SMMU_PTW_ERR_WALK_EABT:
703 event.type = SMMU_EVT_F_WALK_EABT;
704 event.u.f_walk_eabt.addr = addr;
705 event.u.f_walk_eabt.rnw = flag & 0x1;
706 event.u.f_walk_eabt.class = 0x1;
707 event.u.f_walk_eabt.addr2 = ptw_info.addr;
708 break;
709 case SMMU_PTW_ERR_TRANSLATION:
710 if (event.record_trans_faults) {
711 event.type = SMMU_EVT_F_TRANSLATION;
712 event.u.f_translation.addr = addr;
713 event.u.f_translation.rnw = flag & 0x1;
714 }
715 break;
716 case SMMU_PTW_ERR_ADDR_SIZE:
717 if (event.record_trans_faults) {
718 event.type = SMMU_EVT_F_ADDR_SIZE;
719 event.u.f_addr_size.addr = addr;
720 event.u.f_addr_size.rnw = flag & 0x1;
721 }
722 break;
723 case SMMU_PTW_ERR_ACCESS:
724 if (event.record_trans_faults) {
725 event.type = SMMU_EVT_F_ACCESS;
726 event.u.f_access.addr = addr;
727 event.u.f_access.rnw = flag & 0x1;
728 }
729 break;
730 case SMMU_PTW_ERR_PERMISSION:
731 if (event.record_trans_faults) {
732 event.type = SMMU_EVT_F_PERMISSION;
733 event.u.f_permission.addr = addr;
734 event.u.f_permission.rnw = flag & 0x1;
735 }
736 break;
737 default:
738 g_assert_not_reached();
739 }
740 status = SMMU_TRANS_ERROR;
741 } else {
742 smmu_iotlb_insert(bs, cfg, cached_entry);
743 status = SMMU_TRANS_SUCCESS;
744 }
745
746 epilogue:
747 qemu_mutex_unlock(&s->mutex);
748 switch (status) {
749 case SMMU_TRANS_SUCCESS:
750 entry.perm = flag;
751 entry.translated_addr = cached_entry->entry.translated_addr +
752 (addr & cached_entry->entry.addr_mask);
753 entry.addr_mask = cached_entry->entry.addr_mask;
754 trace_smmuv3_translate_success(mr->parent_obj.name, sid, addr,
755 entry.translated_addr, entry.perm);
756 break;
757 case SMMU_TRANS_DISABLE:
758 entry.perm = flag;
759 entry.addr_mask = ~TARGET_PAGE_MASK;
760 trace_smmuv3_translate_disable(mr->parent_obj.name, sid, addr,
761 entry.perm);
762 break;
763 case SMMU_TRANS_BYPASS:
764 entry.perm = flag;
765 entry.addr_mask = ~TARGET_PAGE_MASK;
766 trace_smmuv3_translate_bypass(mr->parent_obj.name, sid, addr,
767 entry.perm);
768 break;
769 case SMMU_TRANS_ABORT:
770 /* no event is recorded on abort */
771 trace_smmuv3_translate_abort(mr->parent_obj.name, sid, addr,
772 entry.perm);
773 break;
774 case SMMU_TRANS_ERROR:
775 qemu_log_mask(LOG_GUEST_ERROR,
776 "%s translation failed for iova=0x%"PRIx64"(%s)\n",
777 mr->parent_obj.name, addr, smmu_event_string(event.type));
778 smmuv3_record_event(s, &event);
779 break;
780 }
781
782 return entry;
783 }
784
785 /**
786 * smmuv3_notify_iova - call the notifier @n for a given
787 * @asid and @iova tuple.
788 *
789 * @mr: IOMMU mr region handle
790 * @n: notifier to be called
791 * @asid: address space ID or negative value if we don't care
792 * @iova: iova
793 * @tg: translation granule (if communicated through range invalidation)
794 * @num_pages: number of @granule sized pages (if tg != 0), otherwise 1
795 */
796 static void smmuv3_notify_iova(IOMMUMemoryRegion *mr,
797 IOMMUNotifier *n,
798 int asid, dma_addr_t iova,
799 uint8_t tg, uint64_t num_pages)
800 {
801 SMMUDevice *sdev = container_of(mr, SMMUDevice, iommu);
802 IOMMUTLBEntry entry;
803 uint8_t granule = tg;
804
805 if (!tg) {
806 SMMUEventInfo event = {.inval_ste_allowed = true};
807 SMMUTransCfg *cfg = smmuv3_get_config(sdev, &event);
808 SMMUTransTableInfo *tt;
809
810 if (!cfg) {
811 return;
812 }
813
814 if (asid >= 0 && cfg->asid != asid) {
815 return;
816 }
817
818 tt = select_tt(cfg, iova);
819 if (!tt) {
820 return;
821 }
822 granule = tt->granule_sz;
823 }
824
825 entry.target_as = &address_space_memory;
826 entry.iova = iova;
827 entry.addr_mask = num_pages * (1 << granule) - 1;
828 entry.perm = IOMMU_NONE;
829
830 memory_region_notify_one(n, &entry);
831 }
832
833 /* invalidate an asid/iova range tuple in all mr's */
834 static void smmuv3_inv_notifiers_iova(SMMUState *s, int asid, dma_addr_t iova,
835 uint8_t tg, uint64_t num_pages)
836 {
837 SMMUDevice *sdev;
838
839 QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) {
840 IOMMUMemoryRegion *mr = &sdev->iommu;
841 IOMMUNotifier *n;
842
843 trace_smmuv3_inv_notifiers_iova(mr->parent_obj.name, asid, iova,
844 tg, num_pages);
845
846 IOMMU_NOTIFIER_FOREACH(n, mr) {
847 smmuv3_notify_iova(mr, n, asid, iova, tg, num_pages);
848 }
849 }
850 }
851
852 static void smmuv3_s1_range_inval(SMMUState *s, Cmd *cmd)
853 {
854 uint8_t scale = 0, num = 0, ttl = 0;
855 dma_addr_t addr = CMD_ADDR(cmd);
856 uint8_t type = CMD_TYPE(cmd);
857 uint16_t vmid = CMD_VMID(cmd);
858 bool leaf = CMD_LEAF(cmd);
859 uint8_t tg = CMD_TG(cmd);
860 hwaddr num_pages = 1;
861 int asid = -1;
862
863 if (tg) {
864 scale = CMD_SCALE(cmd);
865 num = CMD_NUM(cmd);
866 ttl = CMD_TTL(cmd);
867 num_pages = (num + 1) * (1 << (scale));
868 }
869
870 if (type == SMMU_CMD_TLBI_NH_VA) {
871 asid = CMD_ASID(cmd);
872 }
873 trace_smmuv3_s1_range_inval(vmid, asid, addr, tg, num_pages, ttl, leaf);
874 smmuv3_inv_notifiers_iova(s, asid, addr, tg, num_pages);
875 smmu_iotlb_inv_iova(s, asid, addr, tg, num_pages, ttl);
876 }
877
878 static int smmuv3_cmdq_consume(SMMUv3State *s)
879 {
880 SMMUState *bs = ARM_SMMU(s);
881 SMMUCmdError cmd_error = SMMU_CERROR_NONE;
882 SMMUQueue *q = &s->cmdq;
883 SMMUCommandType type = 0;
884
885 if (!smmuv3_cmdq_enabled(s)) {
886 return 0;
887 }
888 /*
889 * some commands depend on register values, typically CR0. In case those
890 * register values change while handling the command, spec says it
891 * is UNPREDICTABLE whether the command is interpreted under the new
892 * or old value.
893 */
894
895 while (!smmuv3_q_empty(q)) {
896 uint32_t pending = s->gerror ^ s->gerrorn;
897 Cmd cmd;
898
899 trace_smmuv3_cmdq_consume(Q_PROD(q), Q_CONS(q),
900 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
901
902 if (FIELD_EX32(pending, GERROR, CMDQ_ERR)) {
903 break;
904 }
905
906 if (queue_read(q, &cmd) != MEMTX_OK) {
907 cmd_error = SMMU_CERROR_ABT;
908 break;
909 }
910
911 type = CMD_TYPE(&cmd);
912
913 trace_smmuv3_cmdq_opcode(smmu_cmd_string(type));
914
915 qemu_mutex_lock(&s->mutex);
916 switch (type) {
917 case SMMU_CMD_SYNC:
918 if (CMD_SYNC_CS(&cmd) & CMD_SYNC_SIG_IRQ) {
919 smmuv3_trigger_irq(s, SMMU_IRQ_CMD_SYNC, 0);
920 }
921 break;
922 case SMMU_CMD_PREFETCH_CONFIG:
923 case SMMU_CMD_PREFETCH_ADDR:
924 break;
925 case SMMU_CMD_CFGI_STE:
926 {
927 uint32_t sid = CMD_SID(&cmd);
928 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, sid);
929 SMMUDevice *sdev;
930
931 if (CMD_SSEC(&cmd)) {
932 cmd_error = SMMU_CERROR_ILL;
933 break;
934 }
935
936 if (!mr) {
937 break;
938 }
939
940 trace_smmuv3_cmdq_cfgi_ste(sid);
941 sdev = container_of(mr, SMMUDevice, iommu);
942 smmuv3_flush_config(sdev);
943
944 break;
945 }
946 case SMMU_CMD_CFGI_STE_RANGE: /* same as SMMU_CMD_CFGI_ALL */
947 {
948 uint32_t start = CMD_SID(&cmd), end, i;
949 uint8_t range = CMD_STE_RANGE(&cmd);
950
951 if (CMD_SSEC(&cmd)) {
952 cmd_error = SMMU_CERROR_ILL;
953 break;
954 }
955
956 end = start + (1 << (range + 1)) - 1;
957 trace_smmuv3_cmdq_cfgi_ste_range(start, end);
958
959 for (i = start; i <= end; i++) {
960 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, i);
961 SMMUDevice *sdev;
962
963 if (!mr) {
964 continue;
965 }
966 sdev = container_of(mr, SMMUDevice, iommu);
967 smmuv3_flush_config(sdev);
968 }
969 break;
970 }
971 case SMMU_CMD_CFGI_CD:
972 case SMMU_CMD_CFGI_CD_ALL:
973 {
974 uint32_t sid = CMD_SID(&cmd);
975 IOMMUMemoryRegion *mr = smmu_iommu_mr(bs, sid);
976 SMMUDevice *sdev;
977
978 if (CMD_SSEC(&cmd)) {
979 cmd_error = SMMU_CERROR_ILL;
980 break;
981 }
982
983 if (!mr) {
984 break;
985 }
986
987 trace_smmuv3_cmdq_cfgi_cd(sid);
988 sdev = container_of(mr, SMMUDevice, iommu);
989 smmuv3_flush_config(sdev);
990 break;
991 }
992 case SMMU_CMD_TLBI_NH_ASID:
993 {
994 uint16_t asid = CMD_ASID(&cmd);
995
996 trace_smmuv3_cmdq_tlbi_nh_asid(asid);
997 smmu_inv_notifiers_all(&s->smmu_state);
998 smmu_iotlb_inv_asid(bs, asid);
999 break;
1000 }
1001 case SMMU_CMD_TLBI_NH_ALL:
1002 case SMMU_CMD_TLBI_NSNH_ALL:
1003 trace_smmuv3_cmdq_tlbi_nh();
1004 smmu_inv_notifiers_all(&s->smmu_state);
1005 smmu_iotlb_inv_all(bs);
1006 break;
1007 case SMMU_CMD_TLBI_NH_VAA:
1008 case SMMU_CMD_TLBI_NH_VA:
1009 smmuv3_s1_range_inval(bs, &cmd);
1010 break;
1011 case SMMU_CMD_TLBI_EL3_ALL:
1012 case SMMU_CMD_TLBI_EL3_VA:
1013 case SMMU_CMD_TLBI_EL2_ALL:
1014 case SMMU_CMD_TLBI_EL2_ASID:
1015 case SMMU_CMD_TLBI_EL2_VA:
1016 case SMMU_CMD_TLBI_EL2_VAA:
1017 case SMMU_CMD_TLBI_S12_VMALL:
1018 case SMMU_CMD_TLBI_S2_IPA:
1019 case SMMU_CMD_ATC_INV:
1020 case SMMU_CMD_PRI_RESP:
1021 case SMMU_CMD_RESUME:
1022 case SMMU_CMD_STALL_TERM:
1023 trace_smmuv3_unhandled_cmd(type);
1024 break;
1025 default:
1026 cmd_error = SMMU_CERROR_ILL;
1027 qemu_log_mask(LOG_GUEST_ERROR,
1028 "Illegal command type: %d\n", CMD_TYPE(&cmd));
1029 break;
1030 }
1031 qemu_mutex_unlock(&s->mutex);
1032 if (cmd_error) {
1033 break;
1034 }
1035 /*
1036 * We only increment the cons index after the completion of
1037 * the command. We do that because the SYNC returns immediately
1038 * and does not check the completion of previous commands
1039 */
1040 queue_cons_incr(q);
1041 }
1042
1043 if (cmd_error) {
1044 trace_smmuv3_cmdq_consume_error(smmu_cmd_string(type), cmd_error);
1045 smmu_write_cmdq_err(s, cmd_error);
1046 smmuv3_trigger_irq(s, SMMU_IRQ_GERROR, R_GERROR_CMDQ_ERR_MASK);
1047 }
1048
1049 trace_smmuv3_cmdq_consume_out(Q_PROD(q), Q_CONS(q),
1050 Q_PROD_WRAP(q), Q_CONS_WRAP(q));
1051
1052 return 0;
1053 }
1054
1055 static MemTxResult smmu_writell(SMMUv3State *s, hwaddr offset,
1056 uint64_t data, MemTxAttrs attrs)
1057 {
1058 switch (offset) {
1059 case A_GERROR_IRQ_CFG0:
1060 s->gerror_irq_cfg0 = data;
1061 return MEMTX_OK;
1062 case A_STRTAB_BASE:
1063 s->strtab_base = data;
1064 return MEMTX_OK;
1065 case A_CMDQ_BASE:
1066 s->cmdq.base = data;
1067 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1068 if (s->cmdq.log2size > SMMU_CMDQS) {
1069 s->cmdq.log2size = SMMU_CMDQS;
1070 }
1071 return MEMTX_OK;
1072 case A_EVENTQ_BASE:
1073 s->eventq.base = data;
1074 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1075 if (s->eventq.log2size > SMMU_EVENTQS) {
1076 s->eventq.log2size = SMMU_EVENTQS;
1077 }
1078 return MEMTX_OK;
1079 case A_EVENTQ_IRQ_CFG0:
1080 s->eventq_irq_cfg0 = data;
1081 return MEMTX_OK;
1082 default:
1083 qemu_log_mask(LOG_UNIMP,
1084 "%s Unexpected 64-bit access to 0x%"PRIx64" (WI)\n",
1085 __func__, offset);
1086 return MEMTX_OK;
1087 }
1088 }
1089
1090 static MemTxResult smmu_writel(SMMUv3State *s, hwaddr offset,
1091 uint64_t data, MemTxAttrs attrs)
1092 {
1093 switch (offset) {
1094 case A_CR0:
1095 s->cr[0] = data;
1096 s->cr0ack = data & ~SMMU_CR0_RESERVED;
1097 /* in case the command queue has been enabled */
1098 smmuv3_cmdq_consume(s);
1099 return MEMTX_OK;
1100 case A_CR1:
1101 s->cr[1] = data;
1102 return MEMTX_OK;
1103 case A_CR2:
1104 s->cr[2] = data;
1105 return MEMTX_OK;
1106 case A_IRQ_CTRL:
1107 s->irq_ctrl = data;
1108 return MEMTX_OK;
1109 case A_GERRORN:
1110 smmuv3_write_gerrorn(s, data);
1111 /*
1112 * By acknowledging the CMDQ_ERR, SW may notify cmds can
1113 * be processed again
1114 */
1115 smmuv3_cmdq_consume(s);
1116 return MEMTX_OK;
1117 case A_GERROR_IRQ_CFG0: /* 64b */
1118 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 0, 32, data);
1119 return MEMTX_OK;
1120 case A_GERROR_IRQ_CFG0 + 4:
1121 s->gerror_irq_cfg0 = deposit64(s->gerror_irq_cfg0, 32, 32, data);
1122 return MEMTX_OK;
1123 case A_GERROR_IRQ_CFG1:
1124 s->gerror_irq_cfg1 = data;
1125 return MEMTX_OK;
1126 case A_GERROR_IRQ_CFG2:
1127 s->gerror_irq_cfg2 = data;
1128 return MEMTX_OK;
1129 case A_STRTAB_BASE: /* 64b */
1130 s->strtab_base = deposit64(s->strtab_base, 0, 32, data);
1131 return MEMTX_OK;
1132 case A_STRTAB_BASE + 4:
1133 s->strtab_base = deposit64(s->strtab_base, 32, 32, data);
1134 return MEMTX_OK;
1135 case A_STRTAB_BASE_CFG:
1136 s->strtab_base_cfg = data;
1137 if (FIELD_EX32(data, STRTAB_BASE_CFG, FMT) == 1) {
1138 s->sid_split = FIELD_EX32(data, STRTAB_BASE_CFG, SPLIT);
1139 s->features |= SMMU_FEATURE_2LVL_STE;
1140 }
1141 return MEMTX_OK;
1142 case A_CMDQ_BASE: /* 64b */
1143 s->cmdq.base = deposit64(s->cmdq.base, 0, 32, data);
1144 s->cmdq.log2size = extract64(s->cmdq.base, 0, 5);
1145 if (s->cmdq.log2size > SMMU_CMDQS) {
1146 s->cmdq.log2size = SMMU_CMDQS;
1147 }
1148 return MEMTX_OK;
1149 case A_CMDQ_BASE + 4: /* 64b */
1150 s->cmdq.base = deposit64(s->cmdq.base, 32, 32, data);
1151 return MEMTX_OK;
1152 case A_CMDQ_PROD:
1153 s->cmdq.prod = data;
1154 smmuv3_cmdq_consume(s);
1155 return MEMTX_OK;
1156 case A_CMDQ_CONS:
1157 s->cmdq.cons = data;
1158 return MEMTX_OK;
1159 case A_EVENTQ_BASE: /* 64b */
1160 s->eventq.base = deposit64(s->eventq.base, 0, 32, data);
1161 s->eventq.log2size = extract64(s->eventq.base, 0, 5);
1162 if (s->eventq.log2size > SMMU_EVENTQS) {
1163 s->eventq.log2size = SMMU_EVENTQS;
1164 }
1165 return MEMTX_OK;
1166 case A_EVENTQ_BASE + 4:
1167 s->eventq.base = deposit64(s->eventq.base, 32, 32, data);
1168 return MEMTX_OK;
1169 case A_EVENTQ_PROD:
1170 s->eventq.prod = data;
1171 return MEMTX_OK;
1172 case A_EVENTQ_CONS:
1173 s->eventq.cons = data;
1174 return MEMTX_OK;
1175 case A_EVENTQ_IRQ_CFG0: /* 64b */
1176 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 0, 32, data);
1177 return MEMTX_OK;
1178 case A_EVENTQ_IRQ_CFG0 + 4:
1179 s->eventq_irq_cfg0 = deposit64(s->eventq_irq_cfg0, 32, 32, data);
1180 return MEMTX_OK;
1181 case A_EVENTQ_IRQ_CFG1:
1182 s->eventq_irq_cfg1 = data;
1183 return MEMTX_OK;
1184 case A_EVENTQ_IRQ_CFG2:
1185 s->eventq_irq_cfg2 = data;
1186 return MEMTX_OK;
1187 default:
1188 qemu_log_mask(LOG_UNIMP,
1189 "%s Unexpected 32-bit access to 0x%"PRIx64" (WI)\n",
1190 __func__, offset);
1191 return MEMTX_OK;
1192 }
1193 }
1194
1195 static MemTxResult smmu_write_mmio(void *opaque, hwaddr offset, uint64_t data,
1196 unsigned size, MemTxAttrs attrs)
1197 {
1198 SMMUState *sys = opaque;
1199 SMMUv3State *s = ARM_SMMUV3(sys);
1200 MemTxResult r;
1201
1202 /* CONSTRAINED UNPREDICTABLE choice to have page0/1 be exact aliases */
1203 offset &= ~0x10000;
1204
1205 switch (size) {
1206 case 8:
1207 r = smmu_writell(s, offset, data, attrs);
1208 break;
1209 case 4:
1210 r = smmu_writel(s, offset, data, attrs);
1211 break;
1212 default:
1213 r = MEMTX_ERROR;
1214 break;
1215 }
1216
1217 trace_smmuv3_write_mmio(offset, data, size, r);
1218 return r;
1219 }
1220
1221 static MemTxResult smmu_readll(SMMUv3State *s, hwaddr offset,
1222 uint64_t *data, MemTxAttrs attrs)
1223 {
1224 switch (offset) {
1225 case A_GERROR_IRQ_CFG0:
1226 *data = s->gerror_irq_cfg0;
1227 return MEMTX_OK;
1228 case A_STRTAB_BASE:
1229 *data = s->strtab_base;
1230 return MEMTX_OK;
1231 case A_CMDQ_BASE:
1232 *data = s->cmdq.base;
1233 return MEMTX_OK;
1234 case A_EVENTQ_BASE:
1235 *data = s->eventq.base;
1236 return MEMTX_OK;
1237 default:
1238 *data = 0;
1239 qemu_log_mask(LOG_UNIMP,
1240 "%s Unexpected 64-bit access to 0x%"PRIx64" (RAZ)\n",
1241 __func__, offset);
1242 return MEMTX_OK;
1243 }
1244 }
1245
1246 static MemTxResult smmu_readl(SMMUv3State *s, hwaddr offset,
1247 uint64_t *data, MemTxAttrs attrs)
1248 {
1249 switch (offset) {
1250 case A_IDREGS ... A_IDREGS + 0x2f:
1251 *data = smmuv3_idreg(offset - A_IDREGS);
1252 return MEMTX_OK;
1253 case A_IDR0 ... A_IDR5:
1254 *data = s->idr[(offset - A_IDR0) / 4];
1255 return MEMTX_OK;
1256 case A_IIDR:
1257 *data = s->iidr;
1258 return MEMTX_OK;
1259 case A_AIDR:
1260 *data = s->aidr;
1261 return MEMTX_OK;
1262 case A_CR0:
1263 *data = s->cr[0];
1264 return MEMTX_OK;
1265 case A_CR0ACK:
1266 *data = s->cr0ack;
1267 return MEMTX_OK;
1268 case A_CR1:
1269 *data = s->cr[1];
1270 return MEMTX_OK;
1271 case A_CR2:
1272 *data = s->cr[2];
1273 return MEMTX_OK;
1274 case A_STATUSR:
1275 *data = s->statusr;
1276 return MEMTX_OK;
1277 case A_IRQ_CTRL:
1278 case A_IRQ_CTRL_ACK:
1279 *data = s->irq_ctrl;
1280 return MEMTX_OK;
1281 case A_GERROR:
1282 *data = s->gerror;
1283 return MEMTX_OK;
1284 case A_GERRORN:
1285 *data = s->gerrorn;
1286 return MEMTX_OK;
1287 case A_GERROR_IRQ_CFG0: /* 64b */
1288 *data = extract64(s->gerror_irq_cfg0, 0, 32);
1289 return MEMTX_OK;
1290 case A_GERROR_IRQ_CFG0 + 4:
1291 *data = extract64(s->gerror_irq_cfg0, 32, 32);
1292 return MEMTX_OK;
1293 case A_GERROR_IRQ_CFG1:
1294 *data = s->gerror_irq_cfg1;
1295 return MEMTX_OK;
1296 case A_GERROR_IRQ_CFG2:
1297 *data = s->gerror_irq_cfg2;
1298 return MEMTX_OK;
1299 case A_STRTAB_BASE: /* 64b */
1300 *data = extract64(s->strtab_base, 0, 32);
1301 return MEMTX_OK;
1302 case A_STRTAB_BASE + 4: /* 64b */
1303 *data = extract64(s->strtab_base, 32, 32);
1304 return MEMTX_OK;
1305 case A_STRTAB_BASE_CFG:
1306 *data = s->strtab_base_cfg;
1307 return MEMTX_OK;
1308 case A_CMDQ_BASE: /* 64b */
1309 *data = extract64(s->cmdq.base, 0, 32);
1310 return MEMTX_OK;
1311 case A_CMDQ_BASE + 4:
1312 *data = extract64(s->cmdq.base, 32, 32);
1313 return MEMTX_OK;
1314 case A_CMDQ_PROD:
1315 *data = s->cmdq.prod;
1316 return MEMTX_OK;
1317 case A_CMDQ_CONS:
1318 *data = s->cmdq.cons;
1319 return MEMTX_OK;
1320 case A_EVENTQ_BASE: /* 64b */
1321 *data = extract64(s->eventq.base, 0, 32);
1322 return MEMTX_OK;
1323 case A_EVENTQ_BASE + 4: /* 64b */
1324 *data = extract64(s->eventq.base, 32, 32);
1325 return MEMTX_OK;
1326 case A_EVENTQ_PROD:
1327 *data = s->eventq.prod;
1328 return MEMTX_OK;
1329 case A_EVENTQ_CONS:
1330 *data = s->eventq.cons;
1331 return MEMTX_OK;
1332 default:
1333 *data = 0;
1334 qemu_log_mask(LOG_UNIMP,
1335 "%s unhandled 32-bit access at 0x%"PRIx64" (RAZ)\n",
1336 __func__, offset);
1337 return MEMTX_OK;
1338 }
1339 }
1340
1341 static MemTxResult smmu_read_mmio(void *opaque, hwaddr offset, uint64_t *data,
1342 unsigned size, MemTxAttrs attrs)
1343 {
1344 SMMUState *sys = opaque;
1345 SMMUv3State *s = ARM_SMMUV3(sys);
1346 MemTxResult r;
1347
1348 /* CONSTRAINED UNPREDICTABLE choice to have page0/1 be exact aliases */
1349 offset &= ~0x10000;
1350
1351 switch (size) {
1352 case 8:
1353 r = smmu_readll(s, offset, data, attrs);
1354 break;
1355 case 4:
1356 r = smmu_readl(s, offset, data, attrs);
1357 break;
1358 default:
1359 r = MEMTX_ERROR;
1360 break;
1361 }
1362
1363 trace_smmuv3_read_mmio(offset, *data, size, r);
1364 return r;
1365 }
1366
1367 static const MemoryRegionOps smmu_mem_ops = {
1368 .read_with_attrs = smmu_read_mmio,
1369 .write_with_attrs = smmu_write_mmio,
1370 .endianness = DEVICE_LITTLE_ENDIAN,
1371 .valid = {
1372 .min_access_size = 4,
1373 .max_access_size = 8,
1374 },
1375 .impl = {
1376 .min_access_size = 4,
1377 .max_access_size = 8,
1378 },
1379 };
1380
1381 static void smmu_init_irq(SMMUv3State *s, SysBusDevice *dev)
1382 {
1383 int i;
1384
1385 for (i = 0; i < ARRAY_SIZE(s->irq); i++) {
1386 sysbus_init_irq(dev, &s->irq[i]);
1387 }
1388 }
1389
1390 static void smmu_reset(DeviceState *dev)
1391 {
1392 SMMUv3State *s = ARM_SMMUV3(dev);
1393 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1394
1395 c->parent_reset(dev);
1396
1397 smmuv3_init_regs(s);
1398 }
1399
1400 static void smmu_realize(DeviceState *d, Error **errp)
1401 {
1402 SMMUState *sys = ARM_SMMU(d);
1403 SMMUv3State *s = ARM_SMMUV3(sys);
1404 SMMUv3Class *c = ARM_SMMUV3_GET_CLASS(s);
1405 SysBusDevice *dev = SYS_BUS_DEVICE(d);
1406 Error *local_err = NULL;
1407
1408 c->parent_realize(d, &local_err);
1409 if (local_err) {
1410 error_propagate(errp, local_err);
1411 return;
1412 }
1413
1414 qemu_mutex_init(&s->mutex);
1415
1416 memory_region_init_io(&sys->iomem, OBJECT(s),
1417 &smmu_mem_ops, sys, TYPE_ARM_SMMUV3, 0x20000);
1418
1419 sys->mrtypename = TYPE_SMMUV3_IOMMU_MEMORY_REGION;
1420
1421 sysbus_init_mmio(dev, &sys->iomem);
1422
1423 smmu_init_irq(s, dev);
1424 }
1425
1426 static const VMStateDescription vmstate_smmuv3_queue = {
1427 .name = "smmuv3_queue",
1428 .version_id = 1,
1429 .minimum_version_id = 1,
1430 .fields = (VMStateField[]) {
1431 VMSTATE_UINT64(base, SMMUQueue),
1432 VMSTATE_UINT32(prod, SMMUQueue),
1433 VMSTATE_UINT32(cons, SMMUQueue),
1434 VMSTATE_UINT8(log2size, SMMUQueue),
1435 VMSTATE_END_OF_LIST(),
1436 },
1437 };
1438
1439 static const VMStateDescription vmstate_smmuv3 = {
1440 .name = "smmuv3",
1441 .version_id = 1,
1442 .minimum_version_id = 1,
1443 .priority = MIG_PRI_IOMMU,
1444 .fields = (VMStateField[]) {
1445 VMSTATE_UINT32(features, SMMUv3State),
1446 VMSTATE_UINT8(sid_size, SMMUv3State),
1447 VMSTATE_UINT8(sid_split, SMMUv3State),
1448
1449 VMSTATE_UINT32_ARRAY(cr, SMMUv3State, 3),
1450 VMSTATE_UINT32(cr0ack, SMMUv3State),
1451 VMSTATE_UINT32(statusr, SMMUv3State),
1452 VMSTATE_UINT32(irq_ctrl, SMMUv3State),
1453 VMSTATE_UINT32(gerror, SMMUv3State),
1454 VMSTATE_UINT32(gerrorn, SMMUv3State),
1455 VMSTATE_UINT64(gerror_irq_cfg0, SMMUv3State),
1456 VMSTATE_UINT32(gerror_irq_cfg1, SMMUv3State),
1457 VMSTATE_UINT32(gerror_irq_cfg2, SMMUv3State),
1458 VMSTATE_UINT64(strtab_base, SMMUv3State),
1459 VMSTATE_UINT32(strtab_base_cfg, SMMUv3State),
1460 VMSTATE_UINT64(eventq_irq_cfg0, SMMUv3State),
1461 VMSTATE_UINT32(eventq_irq_cfg1, SMMUv3State),
1462 VMSTATE_UINT32(eventq_irq_cfg2, SMMUv3State),
1463
1464 VMSTATE_STRUCT(cmdq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1465 VMSTATE_STRUCT(eventq, SMMUv3State, 0, vmstate_smmuv3_queue, SMMUQueue),
1466
1467 VMSTATE_END_OF_LIST(),
1468 },
1469 };
1470
1471 static void smmuv3_instance_init(Object *obj)
1472 {
1473 /* Nothing much to do here as of now */
1474 }
1475
1476 static void smmuv3_class_init(ObjectClass *klass, void *data)
1477 {
1478 DeviceClass *dc = DEVICE_CLASS(klass);
1479 SMMUv3Class *c = ARM_SMMUV3_CLASS(klass);
1480
1481 dc->vmsd = &vmstate_smmuv3;
1482 device_class_set_parent_reset(dc, smmu_reset, &c->parent_reset);
1483 c->parent_realize = dc->realize;
1484 dc->realize = smmu_realize;
1485 }
1486
1487 static int smmuv3_notify_flag_changed(IOMMUMemoryRegion *iommu,
1488 IOMMUNotifierFlag old,
1489 IOMMUNotifierFlag new,
1490 Error **errp)
1491 {
1492 SMMUDevice *sdev = container_of(iommu, SMMUDevice, iommu);
1493 SMMUv3State *s3 = sdev->smmu;
1494 SMMUState *s = &(s3->smmu_state);
1495
1496 if (new & IOMMU_NOTIFIER_MAP) {
1497 error_setg(errp,
1498 "device %02x.%02x.%x requires iommu MAP notifier which is "
1499 "not currently supported", pci_bus_num(sdev->bus),
1500 PCI_SLOT(sdev->devfn), PCI_FUNC(sdev->devfn));
1501 return -EINVAL;
1502 }
1503
1504 if (old == IOMMU_NOTIFIER_NONE) {
1505 trace_smmuv3_notify_flag_add(iommu->parent_obj.name);
1506 QLIST_INSERT_HEAD(&s->devices_with_notifiers, sdev, next);
1507 } else if (new == IOMMU_NOTIFIER_NONE) {
1508 trace_smmuv3_notify_flag_del(iommu->parent_obj.name);
1509 QLIST_REMOVE(sdev, next);
1510 }
1511 return 0;
1512 }
1513
1514 static void smmuv3_iommu_memory_region_class_init(ObjectClass *klass,
1515 void *data)
1516 {
1517 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
1518
1519 imrc->translate = smmuv3_translate;
1520 imrc->notify_flag_changed = smmuv3_notify_flag_changed;
1521 }
1522
1523 static const TypeInfo smmuv3_type_info = {
1524 .name = TYPE_ARM_SMMUV3,
1525 .parent = TYPE_ARM_SMMU,
1526 .instance_size = sizeof(SMMUv3State),
1527 .instance_init = smmuv3_instance_init,
1528 .class_size = sizeof(SMMUv3Class),
1529 .class_init = smmuv3_class_init,
1530 };
1531
1532 static const TypeInfo smmuv3_iommu_memory_region_info = {
1533 .parent = TYPE_IOMMU_MEMORY_REGION,
1534 .name = TYPE_SMMUV3_IOMMU_MEMORY_REGION,
1535 .class_init = smmuv3_iommu_memory_region_class_init,
1536 };
1537
1538 static void smmuv3_register_types(void)
1539 {
1540 type_register(&smmuv3_type_info);
1541 type_register(&smmuv3_iommu_memory_region_info);
1542 }
1543
1544 type_init(smmuv3_register_types)
1545