1 /*
2 * Copyright (c) 2016-2017, Linaro Limited. All rights reserved.
3 * Copyright (c) 2014-2020, Arm Limited. All rights reserved.
4 * Copyright (c) 2014, STMicroelectronics International N.V.
5 * All rights reserved.
6 *
7 * SPDX-License-Identifier: BSD-3-Clause
8 */
9
10 #include <assert.h>
11 #include <stdio.h>
12 #include <string.h>
13
14 #include <platform_def.h>
15
16 #include <arch.h>
17 #include <arch_helpers.h>
18 #include <common/debug.h>
19 #include <lib/cassert.h>
20 #include <lib/utils.h>
21 #include <lib/xlat_tables/xlat_tables.h>
22
23 #include "../xlat_tables_private.h"
24
25 #ifdef ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING
26 #error "ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING flag is set. \
27 This module is to be used when LPAE is not supported"
28 #endif
29
30 CASSERT(PLAT_VIRT_ADDR_SPACE_SIZE == (1ULL << 32), invalid_vaddr_space_size);
31 CASSERT(PLAT_PHY_ADDR_SPACE_SIZE == (1ULL << 32), invalid_paddr_space_size);
32
33 #define MMU32B_UNSET_DESC ~0UL
34 #define MMU32B_INVALID_DESC 0UL
35
36 #define MT_UNKNOWN ~0U
37
38 /*
39 * MMU related values
40 */
41
42 /* Sharable */
43 #define MMU32B_TTB_S (1U << 1)
44
45 /* Not Outer Sharable */
46 #define MMU32B_TTB_NOS (1U << 5)
47
48 /* Normal memory, Inner Non-cacheable */
49 #define MMU32B_TTB_IRGN_NC 0U
50
51 /* Normal memory, Inner Write-Back Write-Allocate Cacheable */
52 #define MMU32B_TTB_IRGN_WBWA (1U << 6)
53
54 /* Normal memory, Inner Write-Through Cacheable */
55 #define MMU32B_TTB_IRGN_WT 1U
56
57 /* Normal memory, Inner Write-Back no Write-Allocate Cacheable */
58 #define MMU32B_TTB_IRGN_WB (1U | (1U << 6))
59
60 /* Normal memory, Outer Write-Back Write-Allocate Cacheable */
61 #define MMU32B_TTB_RNG_WBWA (1U << 3)
62
63 #define MMU32B_DEFAULT_ATTRS \
64 (MMU32B_TTB_S | MMU32B_TTB_NOS | \
65 MMU32B_TTB_IRGN_WBWA | MMU32B_TTB_RNG_WBWA)
66
67 /* armv7 memory mapping attributes: section mapping */
68 #define SECTION_SECURE (0U << 19)
69 #define SECTION_NOTSECURE (1U << 19)
70 #define SECTION_SHARED (1U << 16)
71 #define SECTION_NOTGLOBAL (1U << 17)
72 #define SECTION_ACCESS_FLAG (1U << 10)
73 #define SECTION_UNPRIV (1U << 11)
74 #define SECTION_RO (1U << 15)
75 #define SECTION_TEX(tex) ((((tex) >> 2) << 12) | \
76 ((((tex) >> 1) & 0x1) << 3) | \
77 (((tex) & 0x1) << 2))
78 #define SECTION_DEVICE SECTION_TEX(MMU32B_ATTR_DEVICE_INDEX)
79 #define SECTION_NORMAL SECTION_TEX(MMU32B_ATTR_DEVICE_INDEX)
80 #define SECTION_NORMAL_CACHED \
81 SECTION_TEX(MMU32B_ATTR_IWBWA_OWBWA_INDEX)
82
83 #define SECTION_XN (1U << 4)
84 #define SECTION_PXN (1U << 0)
85 #define SECTION_SECTION (2U << 0)
86
87 #define SECTION_PT_NOTSECURE (1U << 3)
88 #define SECTION_PT_PT (1U << 0)
89
90 #define SMALL_PAGE_SMALL_PAGE (1U << 1)
91 #define SMALL_PAGE_SHARED (1U << 10)
92 #define SMALL_PAGE_NOTGLOBAL (1U << 11)
93 #define SMALL_PAGE_TEX(tex) ((((tex) >> 2) << 6) | \
94 ((((tex) >> 1) & 0x1) << 3) | \
95 (((tex) & 0x1) << 2))
96 #define SMALL_PAGE_DEVICE \
97 SMALL_PAGE_TEX(MMU32B_ATTR_DEVICE_INDEX)
98 #define SMALL_PAGE_NORMAL \
99 SMALL_PAGE_TEX(MMU32B_ATTR_DEVICE_INDEX)
100 #define SMALL_PAGE_NORMAL_CACHED \
101 SMALL_PAGE_TEX(MMU32B_ATTR_IWBWA_OWBWA_INDEX)
102 #define SMALL_PAGE_ACCESS_FLAG (1U << 4)
103 #define SMALL_PAGE_UNPRIV (1U << 5)
104 #define SMALL_PAGE_RO (1U << 9)
105 #define SMALL_PAGE_XN (1U << 0)
106
107 /* The TEX, C and B bits concatenated */
108 #define MMU32B_ATTR_DEVICE_INDEX 0U
109 #define MMU32B_ATTR_IWBWA_OWBWA_INDEX 1U
110
111 #define MMU32B_PRRR_IDX(idx, tr, nos) (((tr) << (2 * (idx))) | \
112 ((uint32_t)(nos) << ((idx) + 24)))
113 #define MMU32B_NMRR_IDX(idx, ir, or) (((ir) << (2 * (idx))) | \
114 ((uint32_t)(or) << (2 * (idx) + 16)))
115 #define MMU32B_PRRR_DS0 (1U << 16)
116 #define MMU32B_PRRR_DS1 (1U << 17)
117 #define MMU32B_PRRR_NS0 (1U << 18)
118 #define MMU32B_PRRR_NS1 (1U << 19)
119
120 #define DACR_DOMAIN(num, perm) ((perm) << ((num) * 2))
121 #define DACR_DOMAIN_PERM_NO_ACCESS 0U
122 #define DACR_DOMAIN_PERM_CLIENT 1U
123 #define DACR_DOMAIN_PERM_MANAGER 3U
124
125 #define NUM_1MB_IN_4GB (1UL << 12)
126 #define NUM_4K_IN_1MB (1UL << 8)
127
128 #define ONE_MB_SHIFT 20
129
130 /* mmu 32b integration */
131 #define MMU32B_L1_TABLE_SIZE (NUM_1MB_IN_4GB * 4)
132 #define MMU32B_L2_TABLE_SIZE (NUM_4K_IN_1MB * 4)
133 #define MMU32B_L1_TABLE_ALIGN (1U << 14)
134 #define MMU32B_L2_TABLE_ALIGN (1U << 10)
135
136 static unsigned int next_xlat;
137 static unsigned long long xlat_max_pa;
138 static uintptr_t xlat_max_va;
139
140 static uint32_t mmu_l1_base[NUM_1MB_IN_4GB]
141 __aligned(MMU32B_L1_TABLE_ALIGN) __attribute__((section("xlat_table")));
142
143 static uint32_t mmu_l2_base[MAX_XLAT_TABLES][NUM_4K_IN_1MB]
144 __aligned(MMU32B_L2_TABLE_ALIGN) __attribute__((section("xlat_table")));
145
146 /*
147 * Array of all memory regions stored in order of ascending base address.
148 * The list is terminated by the first entry with size == 0.
149 */
150 static mmap_region_t mmap[MAX_MMAP_REGIONS + 1];
151
print_mmap(void)152 void print_mmap(void)
153 {
154 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
155 mmap_region_t *mm = mmap;
156
157 printf("init xlat - l1:%p l2:%p (%d)\n",
158 (void *)mmu_l1_base, (void *)mmu_l2_base, MAX_XLAT_TABLES);
159 printf("mmap:\n");
160 while (mm->size) {
161 printf(" VA:%p PA:0x%llx size:0x%zx attr:0x%x\n",
162 (void *)mm->base_va, mm->base_pa,
163 mm->size, mm->attr);
164 ++mm;
165 };
166 printf("\n");
167 #endif
168 }
169
mmap_add(const mmap_region_t * mm)170 void mmap_add(const mmap_region_t *mm)
171 {
172 const mmap_region_t *mm_cursor = mm;
173
174 while ((mm_cursor->size != 0U) || (mm_cursor->attr != 0U)) {
175 mmap_add_region(mm_cursor->base_pa, mm_cursor->base_va,
176 mm_cursor->size, mm_cursor->attr);
177 mm_cursor++;
178 }
179 }
180
mmap_add_region(unsigned long long base_pa,uintptr_t base_va,size_t size,unsigned int attr)181 void mmap_add_region(unsigned long long base_pa, uintptr_t base_va,
182 size_t size, unsigned int attr)
183 {
184 mmap_region_t *mm = mmap;
185 const mmap_region_t *mm_last = mm + ARRAY_SIZE(mmap) - 1U;
186 unsigned long long end_pa = base_pa + size - 1U;
187 uintptr_t end_va = base_va + size - 1U;
188
189 assert(IS_PAGE_ALIGNED(base_pa));
190 assert(IS_PAGE_ALIGNED(base_va));
191 assert(IS_PAGE_ALIGNED(size));
192
193 if (size == 0U) {
194 return;
195 }
196
197 assert(base_pa < end_pa); /* Check for overflows */
198 assert(base_va < end_va);
199
200 assert((base_va + (uintptr_t)size - (uintptr_t)1) <=
201 (PLAT_VIRT_ADDR_SPACE_SIZE - 1U));
202 assert((base_pa + (unsigned long long)size - 1ULL) <=
203 (PLAT_PHY_ADDR_SPACE_SIZE - 1U));
204
205 #if ENABLE_ASSERTIONS
206
207 /* Check for PAs and VAs overlaps with all other regions */
208 for (mm = mmap; mm->size; ++mm) {
209
210 uintptr_t mm_end_va = mm->base_va + mm->size - 1U;
211
212 /*
213 * Check if one of the regions is completely inside the other
214 * one.
215 */
216 bool fully_overlapped_va =
217 ((base_va >= mm->base_va) && (end_va <= mm_end_va)) ||
218 ((mm->base_va >= base_va) && (mm_end_va <= end_va));
219
220 /*
221 * Full VA overlaps are only allowed if both regions are
222 * identity mapped (zero offset) or have the same VA to PA
223 * offset. Also, make sure that it's not the exact same area.
224 */
225 if (fully_overlapped_va) {
226 assert((mm->base_va - mm->base_pa) ==
227 (base_va - base_pa));
228 assert((base_va != mm->base_va) || (size != mm->size));
229 } else {
230 /*
231 * If the regions do not have fully overlapping VAs,
232 * then they must have fully separated VAs and PAs.
233 * Partial overlaps are not allowed
234 */
235
236 unsigned long long mm_end_pa =
237 mm->base_pa + mm->size - 1;
238
239 bool separated_pa = (end_pa < mm->base_pa) ||
240 (base_pa > mm_end_pa);
241 bool separated_va = (end_va < mm->base_va) ||
242 (base_va > mm_end_va);
243
244 assert(separated_va && separated_pa);
245 }
246 }
247
248 mm = mmap; /* Restore pointer to the start of the array */
249
250 #endif /* ENABLE_ASSERTIONS */
251
252 /* Find correct place in mmap to insert new region */
253 while ((mm->base_va < base_va) && (mm->size != 0U)) {
254 ++mm;
255 }
256
257 /*
258 * If a section is contained inside another one with the same base
259 * address, it must be placed after the one it is contained in:
260 *
261 * 1st |-----------------------|
262 * 2nd |------------|
263 * 3rd |------|
264 *
265 * This is required for mmap_region_attr() to get the attributes of the
266 * small region correctly.
267 */
268 while ((mm->base_va == base_va) && (mm->size > size)) {
269 ++mm;
270 }
271
272 /* Make room for new region by moving other regions up by one place */
273 (void)memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm);
274
275 /* Check we haven't lost the empty sentinal from the end of the array */
276 assert(mm_last->size == 0U);
277
278 mm->base_pa = base_pa;
279 mm->base_va = base_va;
280 mm->size = size;
281 mm->attr = attr;
282
283 if (end_pa > xlat_max_pa) {
284 xlat_max_pa = end_pa;
285 }
286 if (end_va > xlat_max_va) {
287 xlat_max_va = end_va;
288 }
289 }
290
291 /* map all memory as shared/global/domain0/no-usr access */
mmap_desc(unsigned attr,unsigned int addr_pa,unsigned int level)292 static uint32_t mmap_desc(unsigned attr, unsigned int addr_pa,
293 unsigned int level)
294 {
295 uint32_t desc;
296
297 switch (level) {
298 case 1U:
299 assert((addr_pa & (MMU32B_L1_TABLE_ALIGN - 1)) == 0U);
300
301 desc = SECTION_SECTION | SECTION_SHARED;
302
303 desc |= (attr & MT_NS) != 0U ? SECTION_NOTSECURE : 0U;
304
305 desc |= SECTION_ACCESS_FLAG;
306 desc |= (attr & MT_RW) != 0U ? 0U : SECTION_RO;
307
308 desc |= (attr & MT_MEMORY) != 0U ?
309 SECTION_NORMAL_CACHED : SECTION_DEVICE;
310
311 if (((attr & MT_RW) != 0U) || ((attr & MT_MEMORY) == 0U)) {
312 desc |= SECTION_XN;
313 }
314 break;
315 case 2U:
316 assert((addr_pa & (MMU32B_L2_TABLE_ALIGN - 1)) == 0U);
317
318 desc = SMALL_PAGE_SMALL_PAGE | SMALL_PAGE_SHARED;
319
320 desc |= SMALL_PAGE_ACCESS_FLAG;
321 desc |= (attr & MT_RW) != 0U ? 0U : SMALL_PAGE_RO;
322
323 desc |= (attr & MT_MEMORY) != 0U ?
324 SMALL_PAGE_NORMAL_CACHED : SMALL_PAGE_DEVICE;
325
326 if (((attr & MT_RW) != 0U) || ((attr & MT_MEMORY) == 0U)) {
327 desc |= SMALL_PAGE_XN;
328 }
329 break;
330 default:
331 panic();
332 }
333 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
334 /* dump only the non-lpae level 2 tables */
335 if (level == 2U) {
336 printf(attr & MT_MEMORY ? "MEM" : "dev");
337 printf(attr & MT_RW ? "-rw" : "-RO");
338 printf(attr & MT_NS ? "-NS" : "-S");
339 }
340 #endif
341 return desc | addr_pa;
342 }
343
mmap_region_attr(const mmap_region_t * mm,uintptr_t base_va,size_t size,unsigned int * attr)344 static unsigned int mmap_region_attr(const mmap_region_t *mm, uintptr_t base_va,
345 size_t size, unsigned int *attr)
346 {
347 /* Don't assume that the area is contained in the first region */
348 unsigned int ret = MT_UNKNOWN;
349
350 /*
351 * Get attributes from last (innermost) region that contains the
352 * requested area. Don't stop as soon as one region doesn't contain it
353 * because there may be other internal regions that contain this area:
354 *
355 * |-----------------------------1-----------------------------|
356 * |----2----| |-------3-------| |----5----|
357 * |--4--|
358 *
359 * |---| <- Area we want the attributes of.
360 *
361 * In this example, the area is contained in regions 1, 3 and 4 but not
362 * in region 2. The loop shouldn't stop at region 2 as inner regions
363 * have priority over outer regions, it should stop at region 5.
364 */
365 for ( ; ; ++mm) {
366
367 if (mm->size == 0U) {
368 return ret; /* Reached end of list */
369 }
370
371 if (mm->base_va > (base_va + size - 1U)) {
372 return ret; /* Next region is after area so end */
373 }
374
375 if ((mm->base_va + mm->size - 1U) < base_va) {
376 continue; /* Next region has already been overtaken */
377 }
378
379 if ((ret == 0U) && (mm->attr == *attr)) {
380 continue; /* Region doesn't override attribs so skip */
381 }
382
383 if ((mm->base_va > base_va) ||
384 ((mm->base_va + mm->size - 1U) <
385 (base_va + size - 1U))) {
386 return MT_UNKNOWN; /* Region doesn't fully cover area */
387 }
388
389 *attr = mm->attr;
390 ret = 0U;
391 }
392 }
393
init_xlation_table_inner(mmap_region_t * mm,unsigned int base_va,uint32_t * table,unsigned int level)394 static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm,
395 unsigned int base_va,
396 uint32_t *table,
397 unsigned int level)
398 {
399 unsigned int level_size_shift = (level == 1U) ?
400 ONE_MB_SHIFT : FOUR_KB_SHIFT;
401 unsigned int level_size = 1U << level_size_shift;
402 unsigned int level_index_mask = (level == 1U) ?
403 (NUM_1MB_IN_4GB - 1) << ONE_MB_SHIFT :
404 (NUM_4K_IN_1MB - 1) << FOUR_KB_SHIFT;
405
406 assert((level == 1U) || (level == 2U));
407
408 VERBOSE("init xlat table at %p (level%1u)\n", (void *)table, level);
409
410 do {
411 uint32_t desc = MMU32B_UNSET_DESC;
412
413 if (mm->base_va + mm->size <= base_va) {
414 /* Area now after the region so skip it */
415 ++mm;
416 continue;
417 }
418 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
419 /* dump only non-lpae level 2 tables content */
420 if (level == 2U) {
421 printf(" 0x%lx %x " + 6 - 2 * level,
422 base_va, level_size);
423 }
424 #endif
425 if (mm->base_va >= base_va + level_size) {
426 /* Next region is after area so nothing to map yet */
427 desc = MMU32B_INVALID_DESC;
428 } else if ((mm->base_va <= base_va) &&
429 (mm->base_va + mm->size) >=
430 (base_va + level_size)) {
431 /* Next region covers all of area */
432 unsigned int attr = mm->attr;
433 unsigned int r = mmap_region_attr(mm, base_va,
434 level_size, &attr);
435
436 if (r == 0U) {
437 desc = mmap_desc(attr,
438 base_va - mm->base_va + mm->base_pa,
439 level);
440 }
441 }
442
443 if (desc == MMU32B_UNSET_DESC) {
444 uintptr_t xlat_table;
445
446 /*
447 * Area not covered by a region so need finer table
448 * Reuse next level table if any (assert attrib matching).
449 * Otherwise allocate a xlat table.
450 */
451 if (*table) {
452 assert((*table & 3) == SECTION_PT_PT);
453 assert(((*table & SECTION_PT_NOTSECURE) == 0U)
454 == ((mm->attr & MT_NS) == 0U));
455
456 xlat_table = (*table) &
457 ~(MMU32B_L1_TABLE_ALIGN - 1);
458 desc = *table;
459 } else {
460 xlat_table = (uintptr_t)mmu_l2_base +
461 next_xlat * MMU32B_L2_TABLE_SIZE;
462 next_xlat++;
463 assert(next_xlat <= MAX_XLAT_TABLES);
464 (void)memset((char *)xlat_table, 0,
465 MMU32B_L2_TABLE_SIZE);
466
467 desc = xlat_table | SECTION_PT_PT;
468 desc |= (mm->attr & MT_NS) != 0U ?
469 SECTION_PT_NOTSECURE : 0;
470 }
471 /* Recurse to fill in new table */
472 mm = init_xlation_table_inner(mm, base_va,
473 (uint32_t *)xlat_table,
474 level + 1);
475 }
476 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
477 /* dump only non-lpae level 2 tables content */
478 if (level == 2U) {
479 printf("\n");
480 }
481 #endif
482 *table++ = desc;
483 base_va += level_size;
484 } while ((mm->size != 0U) && ((base_va & level_index_mask) != 0U));
485
486 return mm;
487 }
488
init_xlat_tables(void)489 void init_xlat_tables(void)
490 {
491 print_mmap();
492
493 assert(((unsigned int)mmu_l1_base & (MMU32B_L1_TABLE_ALIGN - 1)) == 0U);
494 assert(((unsigned int)mmu_l2_base & (MMU32B_L2_TABLE_ALIGN - 1)) == 0U);
495
496 (void)memset(mmu_l1_base, 0, MMU32B_L1_TABLE_SIZE);
497
498 init_xlation_table_inner(mmap, 0, (uint32_t *)mmu_l1_base, 1);
499
500 VERBOSE("init xlat - max_va=%p, max_pa=%llx\n",
501 (void *)xlat_max_va, xlat_max_pa);
502 assert(xlat_max_pa <= (PLAT_VIRT_ADDR_SPACE_SIZE - 1));
503 }
504
505 /*******************************************************************************
506 * Function for enabling the MMU in Secure PL1, assuming that the
507 * page-tables have already been created.
508 ******************************************************************************/
enable_mmu_svc_mon(unsigned int flags)509 void enable_mmu_svc_mon(unsigned int flags)
510 {
511 unsigned int prrr;
512 unsigned int nmrr;
513 unsigned int sctlr;
514
515 assert(IS_IN_SECURE());
516 assert((read_sctlr() & SCTLR_M_BIT) == 0U);
517
518 /* Enable Access flag (simplified access permissions) and TEX remap */
519 write_sctlr(read_sctlr() | SCTLR_AFE_BIT | SCTLR_TRE_BIT);
520
521 prrr = MMU32B_PRRR_IDX(MMU32B_ATTR_DEVICE_INDEX, 1, 0) \
522 | MMU32B_PRRR_IDX(MMU32B_ATTR_IWBWA_OWBWA_INDEX, 2, 1);
523 nmrr = MMU32B_NMRR_IDX(MMU32B_ATTR_DEVICE_INDEX, 0, 0) \
524 | MMU32B_NMRR_IDX(MMU32B_ATTR_IWBWA_OWBWA_INDEX, 1, 1);
525
526 prrr |= MMU32B_PRRR_NS1 | MMU32B_PRRR_DS1;
527
528 write_prrr(prrr);
529 write_nmrr(nmrr);
530
531 /* Program Domain access control register: domain 0 only */
532 write_dacr(DACR_DOMAIN(0, DACR_DOMAIN_PERM_CLIENT));
533
534 /* Invalidate TLBs at the current exception level */
535 tlbiall();
536
537 /* set MMU base xlat table entry (use only TTBR0) */
538 write_ttbr0((uint32_t)mmu_l1_base | MMU32B_DEFAULT_ATTRS);
539 write_ttbr1(0U);
540
541 /*
542 * Ensure all translation table writes have drained
543 * into memory, the TLB invalidation is complete,
544 * and translation register writes are committed
545 * before enabling the MMU
546 */
547 dsb();
548 isb();
549
550 sctlr = read_sctlr();
551 sctlr |= SCTLR_M_BIT;
552 #ifdef ARMV7_SUPPORTS_VIRTUALIZATION
553 sctlr |= SCTLR_WXN_BIT;
554 #endif
555
556 if ((flags & DISABLE_DCACHE) != 0U) {
557 sctlr &= ~SCTLR_C_BIT;
558 } else {
559 sctlr |= SCTLR_C_BIT;
560 }
561
562 write_sctlr(sctlr);
563
564 /* Ensure the MMU enable takes effect immediately */
565 isb();
566 }
567