migration: increase max-bandwidth to 128 MiB/s (1 Gib/s)
[qemu.git] / target / arm / mte_helper.c
1 /*
2 * ARM v8.5-MemTag Operations
3 *
4 * Copyright (c) 2020 Linaro, Ltd.
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "cpu.h"
22 #include "internals.h"
23 #include "exec/exec-all.h"
24 #include "exec/ram_addr.h"
25 #include "exec/cpu_ldst.h"
26 #include "exec/helper-proto.h"
27 #include "qapi/error.h"
28 #include "qemu/guest-random.h"
29
30
31 static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude)
32 {
33 if (exclude == 0xffff) {
34 return 0;
35 }
36 if (offset == 0) {
37 while (exclude & (1 << tag)) {
38 tag = (tag + 1) & 15;
39 }
40 } else {
41 do {
42 do {
43 tag = (tag + 1) & 15;
44 } while (exclude & (1 << tag));
45 } while (--offset > 0);
46 }
47 return tag;
48 }
49
50 /**
51 * allocation_tag_mem:
52 * @env: the cpu environment
53 * @ptr_mmu_idx: the addressing regime to use for the virtual address
54 * @ptr: the virtual address for which to look up tag memory
55 * @ptr_access: the access to use for the virtual address
56 * @ptr_size: the number of bytes in the normal memory access
57 * @tag_access: the access to use for the tag memory
58 * @tag_size: the number of bytes in the tag memory access
59 * @ra: the return address for exception handling
60 *
61 * Our tag memory is formatted as a sequence of little-endian nibbles.
62 * That is, the byte at (addr >> (LOG2_TAG_GRANULE + 1)) contains two
63 * tags, with the tag at [3:0] for the lower addr and the tag at [7:4]
64 * for the higher addr.
65 *
66 * Here, resolve the physical address from the virtual address, and return
67 * a pointer to the corresponding tag byte. Exit with exception if the
68 * virtual address is not accessible for @ptr_access.
69 *
70 * The @ptr_size and @tag_size values may not have an obvious relation
71 * due to the alignment of @ptr, and the number of tag checks required.
72 *
73 * If there is no tag storage corresponding to @ptr, return NULL.
74 */
75 static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx,
76 uint64_t ptr, MMUAccessType ptr_access,
77 int ptr_size, MMUAccessType tag_access,
78 int tag_size, uintptr_t ra)
79 {
80 #ifdef CONFIG_USER_ONLY
81 /* Tag storage not implemented. */
82 return NULL;
83 #else
84 uintptr_t index;
85 CPUIOTLBEntry *iotlbentry;
86 int in_page, flags;
87 ram_addr_t ptr_ra;
88 hwaddr ptr_paddr, tag_paddr, xlat;
89 MemoryRegion *mr;
90 ARMASIdx tag_asi;
91 AddressSpace *tag_as;
92 void *host;
93
94 /*
95 * Probe the first byte of the virtual address. This raises an
96 * exception for inaccessible pages, and resolves the virtual address
97 * into the softmmu tlb.
98 *
99 * When RA == 0, this is for mte_probe1. The page is expected to be
100 * valid. Indicate to probe_access_flags no-fault, then assert that
101 * we received a valid page.
102 */
103 flags = probe_access_flags(env, ptr, ptr_access, ptr_mmu_idx,
104 ra == 0, &host, ra);
105 assert(!(flags & TLB_INVALID_MASK));
106
107 /*
108 * Find the iotlbentry for ptr. This *must* be present in the TLB
109 * because we just found the mapping.
110 * TODO: Perhaps there should be a cputlb helper that returns a
111 * matching tlb entry + iotlb entry.
112 */
113 index = tlb_index(env, ptr_mmu_idx, ptr);
114 # ifdef CONFIG_DEBUG_TCG
115 {
116 CPUTLBEntry *entry = tlb_entry(env, ptr_mmu_idx, ptr);
117 target_ulong comparator = (ptr_access == MMU_DATA_LOAD
118 ? entry->addr_read
119 : tlb_addr_write(entry));
120 g_assert(tlb_hit(comparator, ptr));
121 }
122 # endif
123 iotlbentry = &env_tlb(env)->d[ptr_mmu_idx].iotlb[index];
124
125 /* If the virtual page MemAttr != Tagged, access unchecked. */
126 if (!arm_tlb_mte_tagged(&iotlbentry->attrs)) {
127 return NULL;
128 }
129
130 /*
131 * If not backed by host ram, there is no tag storage: access unchecked.
132 * This is probably a guest os bug though, so log it.
133 */
134 if (unlikely(flags & TLB_MMIO)) {
135 qemu_log_mask(LOG_GUEST_ERROR,
136 "Page @ 0x%" PRIx64 " indicates Tagged Normal memory "
137 "but is not backed by host ram\n", ptr);
138 return NULL;
139 }
140
141 /*
142 * The Normal memory access can extend to the next page. E.g. a single
143 * 8-byte access to the last byte of a page will check only the last
144 * tag on the first page.
145 * Any page access exception has priority over tag check exception.
146 */
147 in_page = -(ptr | TARGET_PAGE_MASK);
148 if (unlikely(ptr_size > in_page)) {
149 void *ignore;
150 flags |= probe_access_flags(env, ptr + in_page, ptr_access,
151 ptr_mmu_idx, ra == 0, &ignore, ra);
152 assert(!(flags & TLB_INVALID_MASK));
153 }
154
155 /* Any debug exception has priority over a tag check exception. */
156 if (unlikely(flags & TLB_WATCHPOINT)) {
157 int wp = ptr_access == MMU_DATA_LOAD ? BP_MEM_READ : BP_MEM_WRITE;
158 assert(ra != 0);
159 cpu_check_watchpoint(env_cpu(env), ptr, ptr_size,
160 iotlbentry->attrs, wp, ra);
161 }
162
163 /*
164 * Find the physical address within the normal mem space.
165 * The memory region lookup must succeed because TLB_MMIO was
166 * not set in the cputlb lookup above.
167 */
168 mr = memory_region_from_host(host, &ptr_ra);
169 tcg_debug_assert(mr != NULL);
170 tcg_debug_assert(memory_region_is_ram(mr));
171 ptr_paddr = ptr_ra;
172 do {
173 ptr_paddr += mr->addr;
174 mr = mr->container;
175 } while (mr);
176
177 /* Convert to the physical address in tag space. */
178 tag_paddr = ptr_paddr >> (LOG2_TAG_GRANULE + 1);
179
180 /* Look up the address in tag space. */
181 tag_asi = iotlbentry->attrs.secure ? ARMASIdx_TagS : ARMASIdx_TagNS;
182 tag_as = cpu_get_address_space(env_cpu(env), tag_asi);
183 mr = address_space_translate(tag_as, tag_paddr, &xlat, NULL,
184 tag_access == MMU_DATA_STORE,
185 iotlbentry->attrs);
186
187 /*
188 * Note that @mr will never be NULL. If there is nothing in the address
189 * space at @tag_paddr, the translation will return the unallocated memory
190 * region. For our purposes, the result must be ram.
191 */
192 if (unlikely(!memory_region_is_ram(mr))) {
193 /* ??? Failure is a board configuration error. */
194 qemu_log_mask(LOG_UNIMP,
195 "Tag Memory @ 0x%" HWADDR_PRIx " not found for "
196 "Normal Memory @ 0x%" HWADDR_PRIx "\n",
197 tag_paddr, ptr_paddr);
198 return NULL;
199 }
200
201 /*
202 * Ensure the tag memory is dirty on write, for migration.
203 * Tag memory can never contain code or display memory (vga).
204 */
205 if (tag_access == MMU_DATA_STORE) {
206 ram_addr_t tag_ra = memory_region_get_ram_addr(mr) + xlat;
207 cpu_physical_memory_set_dirty_flag(tag_ra, DIRTY_MEMORY_MIGRATION);
208 }
209
210 return memory_region_get_ram_ptr(mr) + xlat;
211 #endif
212 }
213
214 uint64_t HELPER(irg)(CPUARMState *env, uint64_t rn, uint64_t rm)
215 {
216 uint16_t exclude = extract32(rm | env->cp15.gcr_el1, 0, 16);
217 int rrnd = extract32(env->cp15.gcr_el1, 16, 1);
218 int start = extract32(env->cp15.rgsr_el1, 0, 4);
219 int seed = extract32(env->cp15.rgsr_el1, 8, 16);
220 int offset, i, rtag;
221
222 /*
223 * Our IMPDEF choice for GCR_EL1.RRND==1 is to continue to use the
224 * deterministic algorithm. Except that with RRND==1 the kernel is
225 * not required to have set RGSR_EL1.SEED != 0, which is required for
226 * the deterministic algorithm to function. So we force a non-zero
227 * SEED for that case.
228 */
229 if (unlikely(seed == 0) && rrnd) {
230 do {
231 Error *err = NULL;
232 uint16_t two;
233
234 if (qemu_guest_getrandom(&two, sizeof(two), &err) < 0) {
235 /*
236 * Failed, for unknown reasons in the crypto subsystem.
237 * Best we can do is log the reason and use a constant seed.
238 */
239 qemu_log_mask(LOG_UNIMP, "IRG: Crypto failure: %s\n",
240 error_get_pretty(err));
241 error_free(err);
242 two = 1;
243 }
244 seed = two;
245 } while (seed == 0);
246 }
247
248 /* RandomTag */
249 for (i = offset = 0; i < 4; ++i) {
250 /* NextRandomTagBit */
251 int top = (extract32(seed, 5, 1) ^ extract32(seed, 3, 1) ^
252 extract32(seed, 2, 1) ^ extract32(seed, 0, 1));
253 seed = (top << 15) | (seed >> 1);
254 offset |= top << i;
255 }
256 rtag = choose_nonexcluded_tag(start, offset, exclude);
257 env->cp15.rgsr_el1 = rtag | (seed << 8);
258
259 return address_with_allocation_tag(rn, rtag);
260 }
261
262 uint64_t HELPER(addsubg)(CPUARMState *env, uint64_t ptr,
263 int32_t offset, uint32_t tag_offset)
264 {
265 int start_tag = allocation_tag_from_addr(ptr);
266 uint16_t exclude = extract32(env->cp15.gcr_el1, 0, 16);
267 int rtag = choose_nonexcluded_tag(start_tag, tag_offset, exclude);
268
269 return address_with_allocation_tag(ptr + offset, rtag);
270 }
271
272 static int load_tag1(uint64_t ptr, uint8_t *mem)
273 {
274 int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
275 return extract32(*mem, ofs, 4);
276 }
277
278 uint64_t HELPER(ldg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
279 {
280 int mmu_idx = cpu_mmu_index(env, false);
281 uint8_t *mem;
282 int rtag = 0;
283
284 /* Trap if accessing an invalid page. */
285 mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD, 1,
286 MMU_DATA_LOAD, 1, GETPC());
287
288 /* Load if page supports tags. */
289 if (mem) {
290 rtag = load_tag1(ptr, mem);
291 }
292
293 return address_with_allocation_tag(xt, rtag);
294 }
295
296 static void check_tag_aligned(CPUARMState *env, uint64_t ptr, uintptr_t ra)
297 {
298 if (unlikely(!QEMU_IS_ALIGNED(ptr, TAG_GRANULE))) {
299 arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE,
300 cpu_mmu_index(env, false), ra);
301 g_assert_not_reached();
302 }
303 }
304
305 /* For use in a non-parallel context, store to the given nibble. */
306 static void store_tag1(uint64_t ptr, uint8_t *mem, int tag)
307 {
308 int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
309 *mem = deposit32(*mem, ofs, 4, tag);
310 }
311
312 /* For use in a parallel context, atomically store to the given nibble. */
313 static void store_tag1_parallel(uint64_t ptr, uint8_t *mem, int tag)
314 {
315 int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
316 uint8_t old = qatomic_read(mem);
317
318 while (1) {
319 uint8_t new = deposit32(old, ofs, 4, tag);
320 uint8_t cmp = qatomic_cmpxchg(mem, old, new);
321 if (likely(cmp == old)) {
322 return;
323 }
324 old = cmp;
325 }
326 }
327
328 typedef void stg_store1(uint64_t, uint8_t *, int);
329
330 static inline void do_stg(CPUARMState *env, uint64_t ptr, uint64_t xt,
331 uintptr_t ra, stg_store1 store1)
332 {
333 int mmu_idx = cpu_mmu_index(env, false);
334 uint8_t *mem;
335
336 check_tag_aligned(env, ptr, ra);
337
338 /* Trap if accessing an invalid page. */
339 mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, TAG_GRANULE,
340 MMU_DATA_STORE, 1, ra);
341
342 /* Store if page supports tags. */
343 if (mem) {
344 store1(ptr, mem, allocation_tag_from_addr(xt));
345 }
346 }
347
348 void HELPER(stg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
349 {
350 do_stg(env, ptr, xt, GETPC(), store_tag1);
351 }
352
353 void HELPER(stg_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt)
354 {
355 do_stg(env, ptr, xt, GETPC(), store_tag1_parallel);
356 }
357
358 void HELPER(stg_stub)(CPUARMState *env, uint64_t ptr)
359 {
360 int mmu_idx = cpu_mmu_index(env, false);
361 uintptr_t ra = GETPC();
362
363 check_tag_aligned(env, ptr, ra);
364 probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra);
365 }
366
367 static inline void do_st2g(CPUARMState *env, uint64_t ptr, uint64_t xt,
368 uintptr_t ra, stg_store1 store1)
369 {
370 int mmu_idx = cpu_mmu_index(env, false);
371 int tag = allocation_tag_from_addr(xt);
372 uint8_t *mem1, *mem2;
373
374 check_tag_aligned(env, ptr, ra);
375
376 /*
377 * Trap if accessing an invalid page(s).
378 * This takes priority over !allocation_tag_access_enabled.
379 */
380 if (ptr & TAG_GRANULE) {
381 /* Two stores unaligned mod TAG_GRANULE*2 -- modify two bytes. */
382 mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
383 TAG_GRANULE, MMU_DATA_STORE, 1, ra);
384 mem2 = allocation_tag_mem(env, mmu_idx, ptr + TAG_GRANULE,
385 MMU_DATA_STORE, TAG_GRANULE,
386 MMU_DATA_STORE, 1, ra);
387
388 /* Store if page(s) support tags. */
389 if (mem1) {
390 store1(TAG_GRANULE, mem1, tag);
391 }
392 if (mem2) {
393 store1(0, mem2, tag);
394 }
395 } else {
396 /* Two stores aligned mod TAG_GRANULE*2 -- modify one byte. */
397 mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
398 2 * TAG_GRANULE, MMU_DATA_STORE, 1, ra);
399 if (mem1) {
400 tag |= tag << 4;
401 qatomic_set(mem1, tag);
402 }
403 }
404 }
405
406 void HELPER(st2g)(CPUARMState *env, uint64_t ptr, uint64_t xt)
407 {
408 do_st2g(env, ptr, xt, GETPC(), store_tag1);
409 }
410
411 void HELPER(st2g_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt)
412 {
413 do_st2g(env, ptr, xt, GETPC(), store_tag1_parallel);
414 }
415
416 void HELPER(st2g_stub)(CPUARMState *env, uint64_t ptr)
417 {
418 int mmu_idx = cpu_mmu_index(env, false);
419 uintptr_t ra = GETPC();
420 int in_page = -(ptr | TARGET_PAGE_MASK);
421
422 check_tag_aligned(env, ptr, ra);
423
424 if (likely(in_page >= 2 * TAG_GRANULE)) {
425 probe_write(env, ptr, 2 * TAG_GRANULE, mmu_idx, ra);
426 } else {
427 probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra);
428 probe_write(env, ptr + TAG_GRANULE, TAG_GRANULE, mmu_idx, ra);
429 }
430 }
431
432 #define LDGM_STGM_SIZE (4 << GMID_EL1_BS)
433
434 uint64_t HELPER(ldgm)(CPUARMState *env, uint64_t ptr)
435 {
436 int mmu_idx = cpu_mmu_index(env, false);
437 uintptr_t ra = GETPC();
438 void *tag_mem;
439
440 ptr = QEMU_ALIGN_DOWN(ptr, LDGM_STGM_SIZE);
441
442 /* Trap if accessing an invalid page. */
443 tag_mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD,
444 LDGM_STGM_SIZE, MMU_DATA_LOAD,
445 LDGM_STGM_SIZE / (2 * TAG_GRANULE), ra);
446
447 /* The tag is squashed to zero if the page does not support tags. */
448 if (!tag_mem) {
449 return 0;
450 }
451
452 QEMU_BUILD_BUG_ON(GMID_EL1_BS != 6);
453 /*
454 * We are loading 64-bits worth of tags. The ordering of elements
455 * within the word corresponds to a 64-bit little-endian operation.
456 */
457 return ldq_le_p(tag_mem);
458 }
459
460 void HELPER(stgm)(CPUARMState *env, uint64_t ptr, uint64_t val)
461 {
462 int mmu_idx = cpu_mmu_index(env, false);
463 uintptr_t ra = GETPC();
464 void *tag_mem;
465
466 ptr = QEMU_ALIGN_DOWN(ptr, LDGM_STGM_SIZE);
467
468 /* Trap if accessing an invalid page. */
469 tag_mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
470 LDGM_STGM_SIZE, MMU_DATA_LOAD,
471 LDGM_STGM_SIZE / (2 * TAG_GRANULE), ra);
472
473 /*
474 * Tag store only happens if the page support tags,
475 * and if the OS has enabled access to the tags.
476 */
477 if (!tag_mem) {
478 return;
479 }
480
481 QEMU_BUILD_BUG_ON(GMID_EL1_BS != 6);
482 /*
483 * We are storing 64-bits worth of tags. The ordering of elements
484 * within the word corresponds to a 64-bit little-endian operation.
485 */
486 stq_le_p(tag_mem, val);
487 }
488
489 void HELPER(stzgm_tags)(CPUARMState *env, uint64_t ptr, uint64_t val)
490 {
491 uintptr_t ra = GETPC();
492 int mmu_idx = cpu_mmu_index(env, false);
493 int log2_dcz_bytes, log2_tag_bytes;
494 intptr_t dcz_bytes, tag_bytes;
495 uint8_t *mem;
496
497 /*
498 * In arm_cpu_realizefn, we assert that dcz > LOG2_TAG_GRANULE+1,
499 * i.e. 32 bytes, which is an unreasonably small dcz anyway,
500 * to make sure that we can access one complete tag byte here.
501 */
502 log2_dcz_bytes = env_archcpu(env)->dcz_blocksize + 2;
503 log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1);
504 dcz_bytes = (intptr_t)1 << log2_dcz_bytes;
505 tag_bytes = (intptr_t)1 << log2_tag_bytes;
506 ptr &= -dcz_bytes;
507
508 mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, dcz_bytes,
509 MMU_DATA_STORE, tag_bytes, ra);
510 if (mem) {
511 int tag_pair = (val & 0xf) * 0x11;
512 memset(mem, tag_pair, tag_bytes);
513 }
514 }
515
516 /* Record a tag check failure. */
517 static void mte_check_fail(CPUARMState *env, uint32_t desc,
518 uint64_t dirty_ptr, uintptr_t ra)
519 {
520 int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
521 ARMMMUIdx arm_mmu_idx = core_to_aa64_mmu_idx(mmu_idx);
522 int el, reg_el, tcf, select, is_write, syn;
523 uint64_t sctlr;
524
525 reg_el = regime_el(env, arm_mmu_idx);
526 sctlr = env->cp15.sctlr_el[reg_el];
527
528 switch (arm_mmu_idx) {
529 case ARMMMUIdx_E10_0:
530 case ARMMMUIdx_E20_0:
531 el = 0;
532 tcf = extract64(sctlr, 38, 2);
533 break;
534 default:
535 el = reg_el;
536 tcf = extract64(sctlr, 40, 2);
537 }
538
539 switch (tcf) {
540 case 1:
541 /*
542 * Tag check fail causes a synchronous exception.
543 *
544 * In restore_state_to_opc, we set the exception syndrome
545 * for the load or store operation. Unwind first so we
546 * may overwrite that with the syndrome for the tag check.
547 */
548 cpu_restore_state(env_cpu(env), ra, true);
549 env->exception.vaddress = dirty_ptr;
550
551 is_write = FIELD_EX32(desc, MTEDESC, WRITE);
552 syn = syn_data_abort_no_iss(el != 0, 0, 0, 0, 0, is_write, 0x11);
553 raise_exception(env, EXCP_DATA_ABORT, syn, exception_target_el(env));
554 /* noreturn, but fall through to the assert anyway */
555
556 case 0:
557 /*
558 * Tag check fail does not affect the PE.
559 * We eliminate this case by not setting MTE_ACTIVE
560 * in tb_flags, so that we never make this runtime call.
561 */
562 g_assert_not_reached();
563
564 case 2:
565 /* Tag check fail causes asynchronous flag set. */
566 mmu_idx = arm_mmu_idx_el(env, el);
567 if (regime_has_2_ranges(mmu_idx)) {
568 select = extract64(dirty_ptr, 55, 1);
569 } else {
570 select = 0;
571 }
572 env->cp15.tfsr_el[el] |= 1 << select;
573 break;
574
575 default:
576 /* Case 3: Reserved. */
577 qemu_log_mask(LOG_GUEST_ERROR,
578 "Tag check failure with SCTLR_EL%d.TCF%s "
579 "set to reserved value %d\n",
580 reg_el, el ? "" : "0", tcf);
581 break;
582 }
583 }
584
585 /*
586 * Perform an MTE checked access for a single logical or atomic access.
587 */
588 static bool mte_probe1_int(CPUARMState *env, uint32_t desc, uint64_t ptr,
589 uintptr_t ra, int bit55)
590 {
591 int mem_tag, mmu_idx, ptr_tag, size;
592 MMUAccessType type;
593 uint8_t *mem;
594
595 ptr_tag = allocation_tag_from_addr(ptr);
596
597 if (tcma_check(desc, bit55, ptr_tag)) {
598 return true;
599 }
600
601 mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
602 type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
603 size = FIELD_EX32(desc, MTEDESC, ESIZE);
604
605 mem = allocation_tag_mem(env, mmu_idx, ptr, type, size,
606 MMU_DATA_LOAD, 1, ra);
607 if (!mem) {
608 return true;
609 }
610
611 mem_tag = load_tag1(ptr, mem);
612 return ptr_tag == mem_tag;
613 }
614
615 /*
616 * No-fault version of mte_check1, to be used by SVE for MemSingleNF.
617 * Returns false if the access is Checked and the check failed. This
618 * is only intended to probe the tag -- the validity of the page must
619 * be checked beforehand.
620 */
621 bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr)
622 {
623 int bit55 = extract64(ptr, 55, 1);
624
625 /* If TBI is disabled, the access is unchecked. */
626 if (unlikely(!tbi_check(desc, bit55))) {
627 return true;
628 }
629
630 return mte_probe1_int(env, desc, ptr, 0, bit55);
631 }
632
633 uint64_t mte_check1(CPUARMState *env, uint32_t desc,
634 uint64_t ptr, uintptr_t ra)
635 {
636 int bit55 = extract64(ptr, 55, 1);
637
638 /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
639 if (unlikely(!tbi_check(desc, bit55))) {
640 return ptr;
641 }
642
643 if (unlikely(!mte_probe1_int(env, desc, ptr, ra, bit55))) {
644 mte_check_fail(env, desc, ptr, ra);
645 }
646
647 return useronly_clean_ptr(ptr);
648 }
649
650 uint64_t HELPER(mte_check1)(CPUARMState *env, uint32_t desc, uint64_t ptr)
651 {
652 return mte_check1(env, desc, ptr, GETPC());
653 }
654
655 /*
656 * Perform an MTE checked access for multiple logical accesses.
657 */
658
659 /**
660 * checkN:
661 * @tag: tag memory to test
662 * @odd: true to begin testing at tags at odd nibble
663 * @cmp: the tag to compare against
664 * @count: number of tags to test
665 *
666 * Return the number of successful tests.
667 * Thus a return value < @count indicates a failure.
668 *
669 * A note about sizes: count is expected to be small.
670 *
671 * The most common use will be LDP/STP of two integer registers,
672 * which means 16 bytes of memory touching at most 2 tags, but
673 * often the access is aligned and thus just 1 tag.
674 *
675 * Using AdvSIMD LD/ST (multiple), one can access 64 bytes of memory,
676 * touching at most 5 tags. SVE LDR/STR (vector) with the default
677 * vector length is also 64 bytes; the maximum architectural length
678 * is 256 bytes touching at most 9 tags.
679 *
680 * The loop below uses 7 logical operations and 1 memory operation
681 * per tag pair. An implementation that loads an aligned word and
682 * uses masking to ignore adjacent tags requires 18 logical operations
683 * and thus does not begin to pay off until 6 tags.
684 * Which, according to the survey above, is unlikely to be common.
685 */
686 static int checkN(uint8_t *mem, int odd, int cmp, int count)
687 {
688 int n = 0, diff;
689
690 /* Replicate the test tag and compare. */
691 cmp *= 0x11;
692 diff = *mem++ ^ cmp;
693
694 if (odd) {
695 goto start_odd;
696 }
697
698 while (1) {
699 /* Test even tag. */
700 if (unlikely((diff) & 0x0f)) {
701 break;
702 }
703 if (++n == count) {
704 break;
705 }
706
707 start_odd:
708 /* Test odd tag. */
709 if (unlikely((diff) & 0xf0)) {
710 break;
711 }
712 if (++n == count) {
713 break;
714 }
715
716 diff = *mem++ ^ cmp;
717 }
718 return n;
719 }
720
721 uint64_t mte_checkN(CPUARMState *env, uint32_t desc,
722 uint64_t ptr, uintptr_t ra)
723 {
724 int mmu_idx, ptr_tag, bit55;
725 uint64_t ptr_last, ptr_end, prev_page, next_page;
726 uint64_t tag_first, tag_end;
727 uint64_t tag_byte_first, tag_byte_end;
728 uint32_t esize, total, tag_count, tag_size, n, c;
729 uint8_t *mem1, *mem2;
730 MMUAccessType type;
731
732 bit55 = extract64(ptr, 55, 1);
733
734 /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
735 if (unlikely(!tbi_check(desc, bit55))) {
736 return ptr;
737 }
738
739 ptr_tag = allocation_tag_from_addr(ptr);
740
741 if (tcma_check(desc, bit55, ptr_tag)) {
742 goto done;
743 }
744
745 mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
746 type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
747 esize = FIELD_EX32(desc, MTEDESC, ESIZE);
748 total = FIELD_EX32(desc, MTEDESC, TSIZE);
749
750 /* Find the addr of the end of the access, and of the last element. */
751 ptr_end = ptr + total;
752 ptr_last = ptr_end - esize;
753
754 /* Round the bounds to the tag granule, and compute the number of tags. */
755 tag_first = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE);
756 tag_end = QEMU_ALIGN_UP(ptr_last, TAG_GRANULE);
757 tag_count = (tag_end - tag_first) / TAG_GRANULE;
758
759 /* Round the bounds to twice the tag granule, and compute the bytes. */
760 tag_byte_first = QEMU_ALIGN_DOWN(ptr, 2 * TAG_GRANULE);
761 tag_byte_end = QEMU_ALIGN_UP(ptr_last, 2 * TAG_GRANULE);
762
763 /* Locate the page boundaries. */
764 prev_page = ptr & TARGET_PAGE_MASK;
765 next_page = prev_page + TARGET_PAGE_SIZE;
766
767 if (likely(tag_end - prev_page <= TARGET_PAGE_SIZE)) {
768 /* Memory access stays on one page. */
769 tag_size = (tag_byte_end - tag_byte_first) / (2 * TAG_GRANULE);
770 mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, total,
771 MMU_DATA_LOAD, tag_size, ra);
772 if (!mem1) {
773 goto done;
774 }
775 /* Perform all of the comparisons. */
776 n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, tag_count);
777 } else {
778 /* Memory access crosses to next page. */
779 tag_size = (next_page - tag_byte_first) / (2 * TAG_GRANULE);
780 mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, next_page - ptr,
781 MMU_DATA_LOAD, tag_size, ra);
782
783 tag_size = (tag_byte_end - next_page) / (2 * TAG_GRANULE);
784 mem2 = allocation_tag_mem(env, mmu_idx, next_page, type,
785 ptr_end - next_page,
786 MMU_DATA_LOAD, tag_size, ra);
787
788 /*
789 * Perform all of the comparisons.
790 * Note the possible but unlikely case of the operation spanning
791 * two pages that do not both have tagging enabled.
792 */
793 n = c = (next_page - tag_first) / TAG_GRANULE;
794 if (mem1) {
795 n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, c);
796 }
797 if (n == c) {
798 if (!mem2) {
799 goto done;
800 }
801 n += checkN(mem2, 0, ptr_tag, tag_count - c);
802 }
803 }
804
805 /*
806 * If we failed, we know which granule. Compute the element that
807 * is first in that granule, and signal failure on that element.
808 */
809 if (unlikely(n < tag_count)) {
810 uint64_t fail_ofs;
811
812 fail_ofs = tag_first + n * TAG_GRANULE - ptr;
813 fail_ofs = ROUND_UP(fail_ofs, esize);
814 mte_check_fail(env, desc, ptr + fail_ofs, ra);
815 }
816
817 done:
818 return useronly_clean_ptr(ptr);
819 }
820
821 uint64_t HELPER(mte_checkN)(CPUARMState *env, uint32_t desc, uint64_t ptr)
822 {
823 return mte_checkN(env, desc, ptr, GETPC());
824 }
825
826 /*
827 * Perform an MTE checked access for DC_ZVA.
828 */
829 uint64_t HELPER(mte_check_zva)(CPUARMState *env, uint32_t desc, uint64_t ptr)
830 {
831 uintptr_t ra = GETPC();
832 int log2_dcz_bytes, log2_tag_bytes;
833 int mmu_idx, bit55;
834 intptr_t dcz_bytes, tag_bytes, i;
835 void *mem;
836 uint64_t ptr_tag, mem_tag, align_ptr;
837
838 bit55 = extract64(ptr, 55, 1);
839
840 /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
841 if (unlikely(!tbi_check(desc, bit55))) {
842 return ptr;
843 }
844
845 ptr_tag = allocation_tag_from_addr(ptr);
846
847 if (tcma_check(desc, bit55, ptr_tag)) {
848 goto done;
849 }
850
851 /*
852 * In arm_cpu_realizefn, we asserted that dcz > LOG2_TAG_GRANULE+1,
853 * i.e. 32 bytes, which is an unreasonably small dcz anyway, to make
854 * sure that we can access one complete tag byte here.
855 */
856 log2_dcz_bytes = env_archcpu(env)->dcz_blocksize + 2;
857 log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1);
858 dcz_bytes = (intptr_t)1 << log2_dcz_bytes;
859 tag_bytes = (intptr_t)1 << log2_tag_bytes;
860 align_ptr = ptr & -dcz_bytes;
861
862 /*
863 * Trap if accessing an invalid page. DC_ZVA requires that we supply
864 * the original pointer for an invalid page. But watchpoints require
865 * that we probe the actual space. So do both.
866 */
867 mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
868 (void) probe_write(env, ptr, 1, mmu_idx, ra);
869 mem = allocation_tag_mem(env, mmu_idx, align_ptr, MMU_DATA_STORE,
870 dcz_bytes, MMU_DATA_LOAD, tag_bytes, ra);
871 if (!mem) {
872 goto done;
873 }
874
875 /*
876 * Unlike the reasoning for checkN, DC_ZVA is always aligned, and thus
877 * it is quite easy to perform all of the comparisons at once without
878 * any extra masking.
879 *
880 * The most common zva block size is 64; some of the thunderx cpus use
881 * a block size of 128. For user-only, aarch64_max_initfn will set the
882 * block size to 512. Fill out the other cases for future-proofing.
883 *
884 * In order to be able to find the first miscompare later, we want the
885 * tag bytes to be in little-endian order.
886 */
887 switch (log2_tag_bytes) {
888 case 0: /* zva_blocksize 32 */
889 mem_tag = *(uint8_t *)mem;
890 ptr_tag *= 0x11u;
891 break;
892 case 1: /* zva_blocksize 64 */
893 mem_tag = cpu_to_le16(*(uint16_t *)mem);
894 ptr_tag *= 0x1111u;
895 break;
896 case 2: /* zva_blocksize 128 */
897 mem_tag = cpu_to_le32(*(uint32_t *)mem);
898 ptr_tag *= 0x11111111u;
899 break;
900 case 3: /* zva_blocksize 256 */
901 mem_tag = cpu_to_le64(*(uint64_t *)mem);
902 ptr_tag *= 0x1111111111111111ull;
903 break;
904
905 default: /* zva_blocksize 512, 1024, 2048 */
906 ptr_tag *= 0x1111111111111111ull;
907 i = 0;
908 do {
909 mem_tag = cpu_to_le64(*(uint64_t *)(mem + i));
910 if (unlikely(mem_tag != ptr_tag)) {
911 goto fail;
912 }
913 i += 8;
914 align_ptr += 16 * TAG_GRANULE;
915 } while (i < tag_bytes);
916 goto done;
917 }
918
919 if (likely(mem_tag == ptr_tag)) {
920 goto done;
921 }
922
923 fail:
924 /* Locate the first nibble that differs. */
925 i = ctz64(mem_tag ^ ptr_tag) >> 4;
926 mte_check_fail(env, desc, align_ptr + i * TAG_GRANULE, ra);
927
928 done:
929 return useronly_clean_ptr(ptr);
930 }