1 /*
2 * Copyright (c) 2016-2018, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <stdbool.h>
9 #include <stdint.h>
10 #include <string.h>
11
12 #include <platform_def.h>
13
14 #include <arch.h>
15 #include <arch_helpers.h>
16 #include <common/debug.h>
17 #include <lib/cassert.h>
18 #include <lib/utils.h>
19 #include <lib/xlat_tables/xlat_tables.h>
20 #include <plat/common/common_def.h>
21
22 #include "xlat_tables_private.h"
23
24 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
25 #define LVL0_SPACER ""
26 #define LVL1_SPACER " "
27 #define LVL2_SPACER " "
28 #define LVL3_SPACER " "
29 #define get_level_spacer(level) \
30 (((level) == U(0)) ? LVL0_SPACER : \
31 (((level) == U(1)) ? LVL1_SPACER : \
32 (((level) == U(2)) ? LVL2_SPACER : LVL3_SPACER)))
33 #define debug_print(...) printf(__VA_ARGS__)
34 #else
35 #define debug_print(...) ((void)0)
36 #endif
37
38 #define UNSET_DESC ~0ULL
39 #define MT_UNKNOWN ~0U
40
41 static uint64_t xlat_tables[MAX_XLAT_TABLES][XLAT_TABLE_ENTRIES]
42 __aligned(XLAT_TABLE_SIZE) __section("xlat_table");
43
44 static unsigned int next_xlat;
45 static unsigned long long xlat_max_pa;
46 static uintptr_t xlat_max_va;
47
48 static uint64_t execute_never_mask;
49 static uint64_t ap1_mask;
50
51 /*
52 * Array of all memory regions stored in order of ascending base address.
53 * The list is terminated by the first entry with size == 0.
54 */
55 static mmap_region_t mmap[MAX_MMAP_REGIONS + 1];
56
57
print_mmap(void)58 void print_mmap(void)
59 {
60 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE
61 debug_print("mmap:\n");
62 mmap_region_t *mm = mmap;
63 while (mm->size != 0U) {
64 debug_print(" VA:%p PA:0x%llx size:0x%zx attr:0x%x\n",
65 (void *)mm->base_va, mm->base_pa,
66 mm->size, mm->attr);
67 ++mm;
68 };
69 debug_print("\n");
70 #endif
71 }
72
mmap_add_region(unsigned long long base_pa,uintptr_t base_va,size_t size,unsigned int attr)73 void mmap_add_region(unsigned long long base_pa, uintptr_t base_va,
74 size_t size, unsigned int attr)
75 {
76 mmap_region_t *mm = mmap;
77 const mmap_region_t *mm_last = mm + ARRAY_SIZE(mmap) - 1U;
78 unsigned long long end_pa = base_pa + size - 1U;
79 uintptr_t end_va = base_va + size - 1U;
80
81 assert(IS_PAGE_ALIGNED(base_pa));
82 assert(IS_PAGE_ALIGNED(base_va));
83 assert(IS_PAGE_ALIGNED(size));
84
85 if (size == 0U)
86 return;
87
88 assert(base_pa < end_pa); /* Check for overflows */
89 assert(base_va < end_va);
90
91 assert((base_va + (uintptr_t)size - (uintptr_t)1) <=
92 (PLAT_VIRT_ADDR_SPACE_SIZE - 1U));
93 assert((base_pa + (unsigned long long)size - 1ULL) <=
94 (PLAT_PHY_ADDR_SPACE_SIZE - 1U));
95
96 #if ENABLE_ASSERTIONS
97
98 /* Check for PAs and VAs overlaps with all other regions */
99 for (mm = mmap; mm->size; ++mm) {
100
101 uintptr_t mm_end_va = mm->base_va + mm->size - 1U;
102
103 /*
104 * Check if one of the regions is completely inside the other
105 * one.
106 */
107 bool fully_overlapped_va =
108 ((base_va >= mm->base_va) && (end_va <= mm_end_va)) ||
109 ((mm->base_va >= base_va) && (mm_end_va <= end_va));
110
111 /*
112 * Full VA overlaps are only allowed if both regions are
113 * identity mapped (zero offset) or have the same VA to PA
114 * offset. Also, make sure that it's not the exact same area.
115 */
116 if (fully_overlapped_va) {
117 assert((mm->base_va - mm->base_pa) ==
118 (base_va - base_pa));
119 assert((base_va != mm->base_va) || (size != mm->size));
120 } else {
121 /*
122 * If the regions do not have fully overlapping VAs,
123 * then they must have fully separated VAs and PAs.
124 * Partial overlaps are not allowed
125 */
126
127 unsigned long long mm_end_pa =
128 mm->base_pa + mm->size - 1;
129
130 bool separated_pa = (end_pa < mm->base_pa) ||
131 (base_pa > mm_end_pa);
132 bool separated_va = (end_va < mm->base_va) ||
133 (base_va > mm_end_va);
134
135 assert(separated_va && separated_pa);
136 }
137 }
138
139 mm = mmap; /* Restore pointer to the start of the array */
140
141 #endif /* ENABLE_ASSERTIONS */
142
143 /* Find correct place in mmap to insert new region */
144 while ((mm->base_va < base_va) && (mm->size != 0U))
145 ++mm;
146
147 /*
148 * If a section is contained inside another one with the same base
149 * address, it must be placed after the one it is contained in:
150 *
151 * 1st |-----------------------|
152 * 2nd |------------|
153 * 3rd |------|
154 *
155 * This is required for mmap_region_attr() to get the attributes of the
156 * small region correctly.
157 */
158 while ((mm->base_va == base_va) && (mm->size > size))
159 ++mm;
160
161 /* Make room for new region by moving other regions up by one place */
162 (void)memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm);
163
164 /* Check we haven't lost the empty sentinal from the end of the array */
165 assert(mm_last->size == 0U);
166
167 mm->base_pa = base_pa;
168 mm->base_va = base_va;
169 mm->size = size;
170 mm->attr = attr;
171
172 if (end_pa > xlat_max_pa)
173 xlat_max_pa = end_pa;
174 if (end_va > xlat_max_va)
175 xlat_max_va = end_va;
176 }
177
mmap_add(const mmap_region_t * mm)178 void mmap_add(const mmap_region_t *mm)
179 {
180 const mmap_region_t *mm_cursor = mm;
181
182 while ((mm_cursor->size != 0U) || (mm_cursor->attr != 0U)) {
183 mmap_add_region(mm_cursor->base_pa, mm_cursor->base_va,
184 mm_cursor->size, mm_cursor->attr);
185 mm_cursor++;
186 }
187 }
188
mmap_desc(unsigned int attr,unsigned long long addr_pa,unsigned int level)189 static uint64_t mmap_desc(unsigned int attr, unsigned long long addr_pa,
190 unsigned int level)
191 {
192 uint64_t desc;
193 int mem_type;
194
195 /* Make sure that the granularity is fine enough to map this address. */
196 assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0U);
197
198 desc = addr_pa;
199 /*
200 * There are different translation table descriptors for level 3 and the
201 * rest.
202 */
203 desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC;
204 desc |= ((attr & MT_NS) != 0U) ? LOWER_ATTRS(NS) : 0U;
205 desc |= ((attr & MT_RW) != 0U) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO);
206 /*
207 * Always set the access flag, as this library assumes access flag
208 * faults aren't managed.
209 */
210 desc |= LOWER_ATTRS(ACCESS_FLAG);
211 desc |= ap1_mask;
212
213 /*
214 * Deduce shareability domain and executability of the memory region
215 * from the memory type.
216 *
217 * Data accesses to device memory and non-cacheable normal memory are
218 * coherent for all observers in the system, and correspondingly are
219 * always treated as being Outer Shareable. Therefore, for these 2 types
220 * of memory, it is not strictly needed to set the shareability field
221 * in the translation tables.
222 */
223 mem_type = MT_TYPE(attr);
224 if (mem_type == MT_DEVICE) {
225 desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH);
226 /*
227 * Always map device memory as execute-never.
228 * This is to avoid the possibility of a speculative instruction
229 * fetch, which could be an issue if this memory region
230 * corresponds to a read-sensitive peripheral.
231 */
232 desc |= execute_never_mask;
233
234 } else { /* Normal memory */
235 /*
236 * Always map read-write normal memory as execute-never.
237 * This library assumes that it is used by software that does
238 * not self-modify its code, therefore R/W memory is reserved
239 * for data storage, which must not be executable.
240 *
241 * Note that setting the XN bit here is for consistency only.
242 * The function that enables the MMU sets the SCTLR_ELx.WXN bit,
243 * which makes any writable memory region to be treated as
244 * execute-never, regardless of the value of the XN bit in the
245 * translation table.
246 *
247 * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER
248 * attribute to figure out the value of the XN bit.
249 */
250 if (((attr & MT_RW) != 0U) || ((attr & MT_EXECUTE_NEVER) != 0U)) {
251 desc |= execute_never_mask;
252 }
253
254 if (mem_type == MT_MEMORY) {
255 desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH);
256 } else {
257 assert(mem_type == MT_NON_CACHEABLE);
258 desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH);
259 }
260 }
261
262 debug_print((mem_type == MT_MEMORY) ? "MEM" :
263 ((mem_type == MT_NON_CACHEABLE) ? "NC" : "DEV"));
264 debug_print(((attr & MT_RW) != 0U) ? "-RW" : "-RO");
265 debug_print(((attr & MT_NS) != 0U) ? "-NS" : "-S");
266 debug_print(((attr & MT_EXECUTE_NEVER) != 0U) ? "-XN" : "-EXEC");
267 return desc;
268 }
269
270 /*
271 * Look for the innermost region that contains the area at `base_va` with size
272 * `size`. Populate *attr with the attributes of this region.
273 *
274 * On success, this function returns 0.
275 * If there are partial overlaps (meaning that a smaller size is needed) or if
276 * the region can't be found in the given area, it returns MT_UNKNOWN. In this
277 * case the value pointed by attr should be ignored by the caller.
278 */
mmap_region_attr(const mmap_region_t * mm,uintptr_t base_va,size_t size,unsigned int * attr)279 static unsigned int mmap_region_attr(const mmap_region_t *mm, uintptr_t base_va,
280 size_t size, unsigned int *attr)
281 {
282 /* Don't assume that the area is contained in the first region */
283 unsigned int ret = MT_UNKNOWN;
284
285 /*
286 * Get attributes from last (innermost) region that contains the
287 * requested area. Don't stop as soon as one region doesn't contain it
288 * because there may be other internal regions that contain this area:
289 *
290 * |-----------------------------1-----------------------------|
291 * |----2----| |-------3-------| |----5----|
292 * |--4--|
293 *
294 * |---| <- Area we want the attributes of.
295 *
296 * In this example, the area is contained in regions 1, 3 and 4 but not
297 * in region 2. The loop shouldn't stop at region 2 as inner regions
298 * have priority over outer regions, it should stop at region 5.
299 */
300 for ( ; ; ++mm) {
301
302 if (mm->size == 0U)
303 return ret; /* Reached end of list */
304
305 if (mm->base_va > (base_va + size - 1U))
306 return ret; /* Next region is after area so end */
307
308 if ((mm->base_va + mm->size - 1U) < base_va)
309 continue; /* Next region has already been overtaken */
310
311 if ((ret == 0U) && (mm->attr == *attr))
312 continue; /* Region doesn't override attribs so skip */
313
314 if ((mm->base_va > base_va) ||
315 ((mm->base_va + mm->size - 1U) < (base_va + size - 1U)))
316 return MT_UNKNOWN; /* Region doesn't fully cover area */
317
318 *attr = mm->attr;
319 ret = 0U;
320 }
321 return ret;
322 }
323
init_xlation_table_inner(mmap_region_t * mm,uintptr_t base_va,uint64_t * table,unsigned int level)324 static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm,
325 uintptr_t base_va,
326 uint64_t *table,
327 unsigned int level)
328 {
329 assert((level >= XLAT_TABLE_LEVEL_MIN) &&
330 (level <= XLAT_TABLE_LEVEL_MAX));
331
332 unsigned int level_size_shift =
333 L0_XLAT_ADDRESS_SHIFT - level * XLAT_TABLE_ENTRIES_SHIFT;
334 u_register_t level_size = (u_register_t)1 << level_size_shift;
335 u_register_t level_index_mask =
336 ((u_register_t)XLAT_TABLE_ENTRIES_MASK) << level_size_shift;
337
338 debug_print("New xlat table:\n");
339
340 do {
341 uint64_t desc = UNSET_DESC;
342
343 if (mm->size == 0U) {
344 /* Done mapping regions; finish zeroing the table */
345 desc = INVALID_DESC;
346 } else if ((mm->base_va + mm->size - 1U) < base_va) {
347 /* This area is after the region so get next region */
348 ++mm;
349 continue;
350 }
351
352 debug_print("%s VA:%p size:0x%llx ", get_level_spacer(level),
353 (void *)base_va, (unsigned long long)level_size);
354
355 if (mm->base_va > (base_va + level_size - 1U)) {
356 /* Next region is after this area. Nothing to map yet */
357 desc = INVALID_DESC;
358 /* Make sure that the current level allows block descriptors */
359 } else if (level >= XLAT_BLOCK_LEVEL_MIN) {
360 /*
361 * Try to get attributes of this area. It will fail if
362 * there are partially overlapping regions. On success,
363 * it will return the innermost region's attributes.
364 */
365 unsigned int attr;
366 unsigned int r = mmap_region_attr(mm, base_va,
367 level_size, &attr);
368
369 if (r == 0U) {
370 desc = mmap_desc(attr,
371 base_va - mm->base_va + mm->base_pa,
372 level);
373 }
374 }
375
376 if (desc == UNSET_DESC) {
377 /* Area not covered by a region so need finer table */
378 uint64_t *new_table = xlat_tables[next_xlat];
379
380 next_xlat++;
381 assert(next_xlat <= MAX_XLAT_TABLES);
382 desc = TABLE_DESC | (uintptr_t)new_table;
383
384 /* Recurse to fill in new table */
385 mm = init_xlation_table_inner(mm, base_va,
386 new_table, level + 1U);
387 }
388
389 debug_print("\n");
390
391 *table++ = desc;
392 base_va += level_size;
393 } while ((base_va & level_index_mask) &&
394 ((base_va - 1U) < (PLAT_VIRT_ADDR_SPACE_SIZE - 1U)));
395
396 return mm;
397 }
398
init_xlation_table(uintptr_t base_va,uint64_t * table,unsigned int level,uintptr_t * max_va,unsigned long long * max_pa)399 void init_xlation_table(uintptr_t base_va, uint64_t *table,
400 unsigned int level, uintptr_t *max_va,
401 unsigned long long *max_pa)
402 {
403 unsigned int el = xlat_arch_current_el();
404
405 execute_never_mask = xlat_arch_get_xn_desc(el);
406
407 if (el == 3U) {
408 ap1_mask = LOWER_ATTRS(AP_ONE_VA_RANGE_RES1);
409 } else {
410 assert(el == 1U);
411 ap1_mask = 0ULL;
412 }
413
414 init_xlation_table_inner(mmap, base_va, table, level);
415 *max_va = xlat_max_va;
416 *max_pa = xlat_max_pa;
417 }
418