Lines Matching refs:addr
92 static void __ref zero_pte_populate(pmd_t *pmd, unsigned long addr, in zero_pte_populate() argument
95 pte_t *pte = pte_offset_kernel(pmd, addr); in zero_pte_populate()
102 while (addr + PAGE_SIZE <= end) { in zero_pte_populate()
103 set_pte_at(&init_mm, addr, pte, zero_pte); in zero_pte_populate()
104 addr += PAGE_SIZE; in zero_pte_populate()
105 pte = pte_offset_kernel(pmd, addr); in zero_pte_populate()
109 static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr, in zero_pmd_populate() argument
112 pmd_t *pmd = pmd_offset(pud, addr); in zero_pmd_populate()
116 next = pmd_addr_end(addr, end); in zero_pmd_populate()
118 if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { in zero_pmd_populate()
136 zero_pte_populate(pmd, addr, next); in zero_pmd_populate()
137 } while (pmd++, addr = next, addr != end); in zero_pmd_populate()
142 static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr, in zero_pud_populate() argument
145 pud_t *pud = pud_offset(p4d, addr); in zero_pud_populate()
149 next = pud_addr_end(addr, end); in zero_pud_populate()
150 if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { in zero_pud_populate()
155 pmd = pmd_offset(pud, addr); in zero_pud_populate()
165 p = pmd_alloc(&init_mm, pud, addr); in zero_pud_populate()
173 zero_pmd_populate(pud, addr, next); in zero_pud_populate()
174 } while (pud++, addr = next, addr != end); in zero_pud_populate()
179 static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr, in zero_p4d_populate() argument
182 p4d_t *p4d = p4d_offset(pgd, addr); in zero_p4d_populate()
186 next = p4d_addr_end(addr, end); in zero_p4d_populate()
187 if (IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) { in zero_p4d_populate()
193 pud = pud_offset(p4d, addr); in zero_p4d_populate()
196 pmd = pmd_offset(pud, addr); in zero_p4d_populate()
206 p = pud_alloc(&init_mm, p4d, addr); in zero_p4d_populate()
214 zero_pud_populate(p4d, addr, next); in zero_p4d_populate()
215 } while (p4d++, addr = next, addr != end); in zero_p4d_populate()
229 unsigned long addr = (unsigned long)shadow_start; in kasan_populate_early_shadow() local
231 pgd_t *pgd = pgd_offset_k(addr); in kasan_populate_early_shadow()
235 next = pgd_addr_end(addr, end); in kasan_populate_early_shadow()
237 if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { in kasan_populate_early_shadow()
252 p4d = p4d_offset(pgd, addr); in kasan_populate_early_shadow()
255 pud = pud_offset(p4d, addr); in kasan_populate_early_shadow()
258 pmd = pmd_offset(pud, addr); in kasan_populate_early_shadow()
268 p = p4d_alloc(&init_mm, pgd, addr); in kasan_populate_early_shadow()
276 zero_p4d_populate(pgd, addr, next); in kasan_populate_early_shadow()
277 } while (pgd++, addr = next, addr != end); in kasan_populate_early_shadow()
342 static void kasan_remove_pte_table(pte_t *pte, unsigned long addr, in kasan_remove_pte_table() argument
347 for (; addr < end; addr = next, pte++) { in kasan_remove_pte_table()
348 next = (addr + PAGE_SIZE) & PAGE_MASK; in kasan_remove_pte_table()
357 pte_clear(&init_mm, addr, pte); in kasan_remove_pte_table()
361 static void kasan_remove_pmd_table(pmd_t *pmd, unsigned long addr, in kasan_remove_pmd_table() argument
366 for (; addr < end; addr = next, pmd++) { in kasan_remove_pmd_table()
369 next = pmd_addr_end(addr, end); in kasan_remove_pmd_table()
375 if (IS_ALIGNED(addr, PMD_SIZE) && in kasan_remove_pmd_table()
381 pte = pte_offset_kernel(pmd, addr); in kasan_remove_pmd_table()
382 kasan_remove_pte_table(pte, addr, next); in kasan_remove_pmd_table()
387 static void kasan_remove_pud_table(pud_t *pud, unsigned long addr, in kasan_remove_pud_table() argument
392 for (; addr < end; addr = next, pud++) { in kasan_remove_pud_table()
395 next = pud_addr_end(addr, end); in kasan_remove_pud_table()
401 if (IS_ALIGNED(addr, PUD_SIZE) && in kasan_remove_pud_table()
407 pmd = pmd_offset(pud, addr); in kasan_remove_pud_table()
409 kasan_remove_pmd_table(pmd, addr, next); in kasan_remove_pud_table()
414 static void kasan_remove_p4d_table(p4d_t *p4d, unsigned long addr, in kasan_remove_p4d_table() argument
419 for (; addr < end; addr = next, p4d++) { in kasan_remove_p4d_table()
422 next = p4d_addr_end(addr, end); in kasan_remove_p4d_table()
428 if (IS_ALIGNED(addr, P4D_SIZE) && in kasan_remove_p4d_table()
434 pud = pud_offset(p4d, addr); in kasan_remove_p4d_table()
435 kasan_remove_pud_table(pud, addr, next); in kasan_remove_p4d_table()
442 unsigned long addr, end, next; in kasan_remove_zero_shadow() local
445 addr = (unsigned long)kasan_mem_to_shadow(start); in kasan_remove_zero_shadow()
446 end = addr + (size >> KASAN_SHADOW_SCALE_SHIFT); in kasan_remove_zero_shadow()
452 for (; addr < end; addr = next) { in kasan_remove_zero_shadow()
455 next = pgd_addr_end(addr, end); in kasan_remove_zero_shadow()
457 pgd = pgd_offset_k(addr); in kasan_remove_zero_shadow()
462 if (IS_ALIGNED(addr, PGDIR_SIZE) && in kasan_remove_zero_shadow()
469 p4d = p4d_offset(pgd, addr); in kasan_remove_zero_shadow()
470 kasan_remove_p4d_table(p4d, addr, next); in kasan_remove_zero_shadow()