target/arm: Implement helper_mte_checkN
[qemu.git] / target / arm / mte_helper.c
1 /*
2 * ARM v8.5-MemTag Operations
3 *
4 * Copyright (c) 2020 Linaro, Ltd.
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "cpu.h"
22 #include "internals.h"
23 #include "exec/exec-all.h"
24 #include "exec/cpu_ldst.h"
25 #include "exec/helper-proto.h"
26
27
28 static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude)
29 {
30 if (exclude == 0xffff) {
31 return 0;
32 }
33 if (offset == 0) {
34 while (exclude & (1 << tag)) {
35 tag = (tag + 1) & 15;
36 }
37 } else {
38 do {
39 do {
40 tag = (tag + 1) & 15;
41 } while (exclude & (1 << tag));
42 } while (--offset > 0);
43 }
44 return tag;
45 }
46
47 /**
48 * allocation_tag_mem:
49 * @env: the cpu environment
50 * @ptr_mmu_idx: the addressing regime to use for the virtual address
51 * @ptr: the virtual address for which to look up tag memory
52 * @ptr_access: the access to use for the virtual address
53 * @ptr_size: the number of bytes in the normal memory access
54 * @tag_access: the access to use for the tag memory
55 * @tag_size: the number of bytes in the tag memory access
56 * @ra: the return address for exception handling
57 *
58 * Our tag memory is formatted as a sequence of little-endian nibbles.
59 * That is, the byte at (addr >> (LOG2_TAG_GRANULE + 1)) contains two
60 * tags, with the tag at [3:0] for the lower addr and the tag at [7:4]
61 * for the higher addr.
62 *
63 * Here, resolve the physical address from the virtual address, and return
64 * a pointer to the corresponding tag byte. Exit with exception if the
65 * virtual address is not accessible for @ptr_access.
66 *
67 * The @ptr_size and @tag_size values may not have an obvious relation
68 * due to the alignment of @ptr, and the number of tag checks required.
69 *
70 * If there is no tag storage corresponding to @ptr, return NULL.
71 */
72 static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx,
73 uint64_t ptr, MMUAccessType ptr_access,
74 int ptr_size, MMUAccessType tag_access,
75 int tag_size, uintptr_t ra)
76 {
77 /* Tag storage not implemented. */
78 return NULL;
79 }
80
81 uint64_t HELPER(irg)(CPUARMState *env, uint64_t rn, uint64_t rm)
82 {
83 int rtag;
84
85 /*
86 * Our IMPDEF choice for GCR_EL1.RRND==1 is to behave as if
87 * GCR_EL1.RRND==0, always producing deterministic results.
88 */
89 uint16_t exclude = extract32(rm | env->cp15.gcr_el1, 0, 16);
90 int start = extract32(env->cp15.rgsr_el1, 0, 4);
91 int seed = extract32(env->cp15.rgsr_el1, 8, 16);
92 int offset, i;
93
94 /* RandomTag */
95 for (i = offset = 0; i < 4; ++i) {
96 /* NextRandomTagBit */
97 int top = (extract32(seed, 5, 1) ^ extract32(seed, 3, 1) ^
98 extract32(seed, 2, 1) ^ extract32(seed, 0, 1));
99 seed = (top << 15) | (seed >> 1);
100 offset |= top << i;
101 }
102 rtag = choose_nonexcluded_tag(start, offset, exclude);
103 env->cp15.rgsr_el1 = rtag | (seed << 8);
104
105 return address_with_allocation_tag(rn, rtag);
106 }
107
108 uint64_t HELPER(addsubg)(CPUARMState *env, uint64_t ptr,
109 int32_t offset, uint32_t tag_offset)
110 {
111 int start_tag = allocation_tag_from_addr(ptr);
112 uint16_t exclude = extract32(env->cp15.gcr_el1, 0, 16);
113 int rtag = choose_nonexcluded_tag(start_tag, tag_offset, exclude);
114
115 return address_with_allocation_tag(ptr + offset, rtag);
116 }
117
118 static int load_tag1(uint64_t ptr, uint8_t *mem)
119 {
120 int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
121 return extract32(*mem, ofs, 4);
122 }
123
124 uint64_t HELPER(ldg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
125 {
126 int mmu_idx = cpu_mmu_index(env, false);
127 uint8_t *mem;
128 int rtag = 0;
129
130 /* Trap if accessing an invalid page. */
131 mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD, 1,
132 MMU_DATA_LOAD, 1, GETPC());
133
134 /* Load if page supports tags. */
135 if (mem) {
136 rtag = load_tag1(ptr, mem);
137 }
138
139 return address_with_allocation_tag(xt, rtag);
140 }
141
142 static void check_tag_aligned(CPUARMState *env, uint64_t ptr, uintptr_t ra)
143 {
144 if (unlikely(!QEMU_IS_ALIGNED(ptr, TAG_GRANULE))) {
145 arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE,
146 cpu_mmu_index(env, false), ra);
147 g_assert_not_reached();
148 }
149 }
150
151 /* For use in a non-parallel context, store to the given nibble. */
152 static void store_tag1(uint64_t ptr, uint8_t *mem, int tag)
153 {
154 int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
155 *mem = deposit32(*mem, ofs, 4, tag);
156 }
157
158 /* For use in a parallel context, atomically store to the given nibble. */
159 static void store_tag1_parallel(uint64_t ptr, uint8_t *mem, int tag)
160 {
161 int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
162 uint8_t old = atomic_read(mem);
163
164 while (1) {
165 uint8_t new = deposit32(old, ofs, 4, tag);
166 uint8_t cmp = atomic_cmpxchg(mem, old, new);
167 if (likely(cmp == old)) {
168 return;
169 }
170 old = cmp;
171 }
172 }
173
174 typedef void stg_store1(uint64_t, uint8_t *, int);
175
176 static inline void do_stg(CPUARMState *env, uint64_t ptr, uint64_t xt,
177 uintptr_t ra, stg_store1 store1)
178 {
179 int mmu_idx = cpu_mmu_index(env, false);
180 uint8_t *mem;
181
182 check_tag_aligned(env, ptr, ra);
183
184 /* Trap if accessing an invalid page. */
185 mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, TAG_GRANULE,
186 MMU_DATA_STORE, 1, ra);
187
188 /* Store if page supports tags. */
189 if (mem) {
190 store1(ptr, mem, allocation_tag_from_addr(xt));
191 }
192 }
193
194 void HELPER(stg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
195 {
196 do_stg(env, ptr, xt, GETPC(), store_tag1);
197 }
198
199 void HELPER(stg_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt)
200 {
201 do_stg(env, ptr, xt, GETPC(), store_tag1_parallel);
202 }
203
204 void HELPER(stg_stub)(CPUARMState *env, uint64_t ptr)
205 {
206 int mmu_idx = cpu_mmu_index(env, false);
207 uintptr_t ra = GETPC();
208
209 check_tag_aligned(env, ptr, ra);
210 probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra);
211 }
212
213 static inline void do_st2g(CPUARMState *env, uint64_t ptr, uint64_t xt,
214 uintptr_t ra, stg_store1 store1)
215 {
216 int mmu_idx = cpu_mmu_index(env, false);
217 int tag = allocation_tag_from_addr(xt);
218 uint8_t *mem1, *mem2;
219
220 check_tag_aligned(env, ptr, ra);
221
222 /*
223 * Trap if accessing an invalid page(s).
224 * This takes priority over !allocation_tag_access_enabled.
225 */
226 if (ptr & TAG_GRANULE) {
227 /* Two stores unaligned mod TAG_GRANULE*2 -- modify two bytes. */
228 mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
229 TAG_GRANULE, MMU_DATA_STORE, 1, ra);
230 mem2 = allocation_tag_mem(env, mmu_idx, ptr + TAG_GRANULE,
231 MMU_DATA_STORE, TAG_GRANULE,
232 MMU_DATA_STORE, 1, ra);
233
234 /* Store if page(s) support tags. */
235 if (mem1) {
236 store1(TAG_GRANULE, mem1, tag);
237 }
238 if (mem2) {
239 store1(0, mem2, tag);
240 }
241 } else {
242 /* Two stores aligned mod TAG_GRANULE*2 -- modify one byte. */
243 mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
244 2 * TAG_GRANULE, MMU_DATA_STORE, 1, ra);
245 if (mem1) {
246 tag |= tag << 4;
247 atomic_set(mem1, tag);
248 }
249 }
250 }
251
252 void HELPER(st2g)(CPUARMState *env, uint64_t ptr, uint64_t xt)
253 {
254 do_st2g(env, ptr, xt, GETPC(), store_tag1);
255 }
256
257 void HELPER(st2g_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt)
258 {
259 do_st2g(env, ptr, xt, GETPC(), store_tag1_parallel);
260 }
261
262 void HELPER(st2g_stub)(CPUARMState *env, uint64_t ptr)
263 {
264 int mmu_idx = cpu_mmu_index(env, false);
265 uintptr_t ra = GETPC();
266 int in_page = -(ptr | TARGET_PAGE_MASK);
267
268 check_tag_aligned(env, ptr, ra);
269
270 if (likely(in_page >= 2 * TAG_GRANULE)) {
271 probe_write(env, ptr, 2 * TAG_GRANULE, mmu_idx, ra);
272 } else {
273 probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra);
274 probe_write(env, ptr + TAG_GRANULE, TAG_GRANULE, mmu_idx, ra);
275 }
276 }
277
278 #define LDGM_STGM_SIZE (4 << GMID_EL1_BS)
279
280 uint64_t HELPER(ldgm)(CPUARMState *env, uint64_t ptr)
281 {
282 int mmu_idx = cpu_mmu_index(env, false);
283 uintptr_t ra = GETPC();
284 void *tag_mem;
285
286 ptr = QEMU_ALIGN_DOWN(ptr, LDGM_STGM_SIZE);
287
288 /* Trap if accessing an invalid page. */
289 tag_mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD,
290 LDGM_STGM_SIZE, MMU_DATA_LOAD,
291 LDGM_STGM_SIZE / (2 * TAG_GRANULE), ra);
292
293 /* The tag is squashed to zero if the page does not support tags. */
294 if (!tag_mem) {
295 return 0;
296 }
297
298 QEMU_BUILD_BUG_ON(GMID_EL1_BS != 6);
299 /*
300 * We are loading 64-bits worth of tags. The ordering of elements
301 * within the word corresponds to a 64-bit little-endian operation.
302 */
303 return ldq_le_p(tag_mem);
304 }
305
306 void HELPER(stgm)(CPUARMState *env, uint64_t ptr, uint64_t val)
307 {
308 int mmu_idx = cpu_mmu_index(env, false);
309 uintptr_t ra = GETPC();
310 void *tag_mem;
311
312 ptr = QEMU_ALIGN_DOWN(ptr, LDGM_STGM_SIZE);
313
314 /* Trap if accessing an invalid page. */
315 tag_mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
316 LDGM_STGM_SIZE, MMU_DATA_LOAD,
317 LDGM_STGM_SIZE / (2 * TAG_GRANULE), ra);
318
319 /*
320 * Tag store only happens if the page support tags,
321 * and if the OS has enabled access to the tags.
322 */
323 if (!tag_mem) {
324 return;
325 }
326
327 QEMU_BUILD_BUG_ON(GMID_EL1_BS != 6);
328 /*
329 * We are storing 64-bits worth of tags. The ordering of elements
330 * within the word corresponds to a 64-bit little-endian operation.
331 */
332 stq_le_p(tag_mem, val);
333 }
334
335 void HELPER(stzgm_tags)(CPUARMState *env, uint64_t ptr, uint64_t val)
336 {
337 uintptr_t ra = GETPC();
338 int mmu_idx = cpu_mmu_index(env, false);
339 int log2_dcz_bytes, log2_tag_bytes;
340 intptr_t dcz_bytes, tag_bytes;
341 uint8_t *mem;
342
343 /*
344 * In arm_cpu_realizefn, we assert that dcz > LOG2_TAG_GRANULE+1,
345 * i.e. 32 bytes, which is an unreasonably small dcz anyway,
346 * to make sure that we can access one complete tag byte here.
347 */
348 log2_dcz_bytes = env_archcpu(env)->dcz_blocksize + 2;
349 log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1);
350 dcz_bytes = (intptr_t)1 << log2_dcz_bytes;
351 tag_bytes = (intptr_t)1 << log2_tag_bytes;
352 ptr &= -dcz_bytes;
353
354 mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, dcz_bytes,
355 MMU_DATA_STORE, tag_bytes, ra);
356 if (mem) {
357 int tag_pair = (val & 0xf) * 0x11;
358 memset(mem, tag_pair, tag_bytes);
359 }
360 }
361
362 /* Record a tag check failure. */
363 static void mte_check_fail(CPUARMState *env, int mmu_idx,
364 uint64_t dirty_ptr, uintptr_t ra)
365 {
366 ARMMMUIdx arm_mmu_idx = core_to_aa64_mmu_idx(mmu_idx);
367 int el, reg_el, tcf, select;
368 uint64_t sctlr;
369
370 reg_el = regime_el(env, arm_mmu_idx);
371 sctlr = env->cp15.sctlr_el[reg_el];
372
373 switch (arm_mmu_idx) {
374 case ARMMMUIdx_E10_0:
375 case ARMMMUIdx_E20_0:
376 el = 0;
377 tcf = extract64(sctlr, 38, 2);
378 break;
379 default:
380 el = reg_el;
381 tcf = extract64(sctlr, 40, 2);
382 }
383
384 switch (tcf) {
385 case 1:
386 /*
387 * Tag check fail causes a synchronous exception.
388 *
389 * In restore_state_to_opc, we set the exception syndrome
390 * for the load or store operation. Unwind first so we
391 * may overwrite that with the syndrome for the tag check.
392 */
393 cpu_restore_state(env_cpu(env), ra, true);
394 env->exception.vaddress = dirty_ptr;
395 raise_exception(env, EXCP_DATA_ABORT,
396 syn_data_abort_no_iss(el != 0, 0, 0, 0, 0, 0, 0x11),
397 exception_target_el(env));
398 /* noreturn, but fall through to the assert anyway */
399
400 case 0:
401 /*
402 * Tag check fail does not affect the PE.
403 * We eliminate this case by not setting MTE_ACTIVE
404 * in tb_flags, so that we never make this runtime call.
405 */
406 g_assert_not_reached();
407
408 case 2:
409 /* Tag check fail causes asynchronous flag set. */
410 mmu_idx = arm_mmu_idx_el(env, el);
411 if (regime_has_2_ranges(mmu_idx)) {
412 select = extract64(dirty_ptr, 55, 1);
413 } else {
414 select = 0;
415 }
416 env->cp15.tfsr_el[el] |= 1 << select;
417 break;
418
419 default:
420 /* Case 3: Reserved. */
421 qemu_log_mask(LOG_GUEST_ERROR,
422 "Tag check failure with SCTLR_EL%d.TCF%s "
423 "set to reserved value %d\n",
424 reg_el, el ? "" : "0", tcf);
425 break;
426 }
427 }
428
429 /*
430 * Perform an MTE checked access for a single logical or atomic access.
431 */
432 static bool mte_probe1_int(CPUARMState *env, uint32_t desc, uint64_t ptr,
433 uintptr_t ra, int bit55)
434 {
435 int mem_tag, mmu_idx, ptr_tag, size;
436 MMUAccessType type;
437 uint8_t *mem;
438
439 ptr_tag = allocation_tag_from_addr(ptr);
440
441 if (tcma_check(desc, bit55, ptr_tag)) {
442 return true;
443 }
444
445 mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
446 type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
447 size = FIELD_EX32(desc, MTEDESC, ESIZE);
448
449 mem = allocation_tag_mem(env, mmu_idx, ptr, type, size,
450 MMU_DATA_LOAD, 1, ra);
451 if (!mem) {
452 return true;
453 }
454
455 mem_tag = load_tag1(ptr, mem);
456 return ptr_tag == mem_tag;
457 }
458
459 /*
460 * No-fault version of mte_check1, to be used by SVE for MemSingleNF.
461 * Returns false if the access is Checked and the check failed. This
462 * is only intended to probe the tag -- the validity of the page must
463 * be checked beforehand.
464 */
465 bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr)
466 {
467 int bit55 = extract64(ptr, 55, 1);
468
469 /* If TBI is disabled, the access is unchecked. */
470 if (unlikely(!tbi_check(desc, bit55))) {
471 return true;
472 }
473
474 return mte_probe1_int(env, desc, ptr, 0, bit55);
475 }
476
477 uint64_t mte_check1(CPUARMState *env, uint32_t desc,
478 uint64_t ptr, uintptr_t ra)
479 {
480 int bit55 = extract64(ptr, 55, 1);
481
482 /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
483 if (unlikely(!tbi_check(desc, bit55))) {
484 return ptr;
485 }
486
487 if (unlikely(!mte_probe1_int(env, desc, ptr, ra, bit55))) {
488 int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
489 mte_check_fail(env, mmu_idx, ptr, ra);
490 }
491
492 return useronly_clean_ptr(ptr);
493 }
494
495 uint64_t HELPER(mte_check1)(CPUARMState *env, uint32_t desc, uint64_t ptr)
496 {
497 return mte_check1(env, desc, ptr, GETPC());
498 }
499
500 /*
501 * Perform an MTE checked access for multiple logical accesses.
502 */
503
504 /**
505 * checkN:
506 * @tag: tag memory to test
507 * @odd: true to begin testing at tags at odd nibble
508 * @cmp: the tag to compare against
509 * @count: number of tags to test
510 *
511 * Return the number of successful tests.
512 * Thus a return value < @count indicates a failure.
513 *
514 * A note about sizes: count is expected to be small.
515 *
516 * The most common use will be LDP/STP of two integer registers,
517 * which means 16 bytes of memory touching at most 2 tags, but
518 * often the access is aligned and thus just 1 tag.
519 *
520 * Using AdvSIMD LD/ST (multiple), one can access 64 bytes of memory,
521 * touching at most 5 tags. SVE LDR/STR (vector) with the default
522 * vector length is also 64 bytes; the maximum architectural length
523 * is 256 bytes touching at most 9 tags.
524 *
525 * The loop below uses 7 logical operations and 1 memory operation
526 * per tag pair. An implementation that loads an aligned word and
527 * uses masking to ignore adjacent tags requires 18 logical operations
528 * and thus does not begin to pay off until 6 tags.
529 * Which, according to the survey above, is unlikely to be common.
530 */
531 static int checkN(uint8_t *mem, int odd, int cmp, int count)
532 {
533 int n = 0, diff;
534
535 /* Replicate the test tag and compare. */
536 cmp *= 0x11;
537 diff = *mem++ ^ cmp;
538
539 if (odd) {
540 goto start_odd;
541 }
542
543 while (1) {
544 /* Test even tag. */
545 if (unlikely((diff) & 0x0f)) {
546 break;
547 }
548 if (++n == count) {
549 break;
550 }
551
552 start_odd:
553 /* Test odd tag. */
554 if (unlikely((diff) & 0xf0)) {
555 break;
556 }
557 if (++n == count) {
558 break;
559 }
560
561 diff = *mem++ ^ cmp;
562 }
563 return n;
564 }
565
566 uint64_t mte_checkN(CPUARMState *env, uint32_t desc,
567 uint64_t ptr, uintptr_t ra)
568 {
569 int mmu_idx, ptr_tag, bit55;
570 uint64_t ptr_last, ptr_end, prev_page, next_page;
571 uint64_t tag_first, tag_end;
572 uint64_t tag_byte_first, tag_byte_end;
573 uint32_t esize, total, tag_count, tag_size, n, c;
574 uint8_t *mem1, *mem2;
575 MMUAccessType type;
576
577 bit55 = extract64(ptr, 55, 1);
578
579 /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
580 if (unlikely(!tbi_check(desc, bit55))) {
581 return ptr;
582 }
583
584 ptr_tag = allocation_tag_from_addr(ptr);
585
586 if (tcma_check(desc, bit55, ptr_tag)) {
587 goto done;
588 }
589
590 mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
591 type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
592 esize = FIELD_EX32(desc, MTEDESC, ESIZE);
593 total = FIELD_EX32(desc, MTEDESC, TSIZE);
594
595 /* Find the addr of the end of the access, and of the last element. */
596 ptr_end = ptr + total;
597 ptr_last = ptr_end - esize;
598
599 /* Round the bounds to the tag granule, and compute the number of tags. */
600 tag_first = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE);
601 tag_end = QEMU_ALIGN_UP(ptr_last, TAG_GRANULE);
602 tag_count = (tag_end - tag_first) / TAG_GRANULE;
603
604 /* Round the bounds to twice the tag granule, and compute the bytes. */
605 tag_byte_first = QEMU_ALIGN_DOWN(ptr, 2 * TAG_GRANULE);
606 tag_byte_end = QEMU_ALIGN_UP(ptr_last, 2 * TAG_GRANULE);
607
608 /* Locate the page boundaries. */
609 prev_page = ptr & TARGET_PAGE_MASK;
610 next_page = prev_page + TARGET_PAGE_SIZE;
611
612 if (likely(tag_end - prev_page <= TARGET_PAGE_SIZE)) {
613 /* Memory access stays on one page. */
614 tag_size = (tag_byte_end - tag_byte_first) / (2 * TAG_GRANULE);
615 mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, total,
616 MMU_DATA_LOAD, tag_size, ra);
617 if (!mem1) {
618 goto done;
619 }
620 /* Perform all of the comparisons. */
621 n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, tag_count);
622 } else {
623 /* Memory access crosses to next page. */
624 tag_size = (next_page - tag_byte_first) / (2 * TAG_GRANULE);
625 mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, next_page - ptr,
626 MMU_DATA_LOAD, tag_size, ra);
627
628 tag_size = (tag_byte_end - next_page) / (2 * TAG_GRANULE);
629 mem2 = allocation_tag_mem(env, mmu_idx, next_page, type,
630 ptr_end - next_page,
631 MMU_DATA_LOAD, tag_size, ra);
632
633 /*
634 * Perform all of the comparisons.
635 * Note the possible but unlikely case of the operation spanning
636 * two pages that do not both have tagging enabled.
637 */
638 n = c = (next_page - tag_first) / TAG_GRANULE;
639 if (mem1) {
640 n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, c);
641 }
642 if (n == c) {
643 if (!mem2) {
644 goto done;
645 }
646 n += checkN(mem2, 0, ptr_tag, tag_count - c);
647 }
648 }
649
650 /*
651 * If we failed, we know which granule. Compute the element that
652 * is first in that granule, and signal failure on that element.
653 */
654 if (unlikely(n < tag_count)) {
655 uint64_t fail_ofs;
656
657 fail_ofs = tag_first + n * TAG_GRANULE - ptr;
658 fail_ofs = ROUND_UP(fail_ofs, esize);
659 mte_check_fail(env, mmu_idx, ptr + fail_ofs, ra);
660 }
661
662 done:
663 return useronly_clean_ptr(ptr);
664 }
665
666 uint64_t HELPER(mte_checkN)(CPUARMState *env, uint32_t desc, uint64_t ptr)
667 {
668 return mte_checkN(env, desc, ptr, GETPC());
669 }