in mm/vmem.c [312:364]
static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end,
bool add, bool direct)
{
unsigned long next, prot, pages = 0;
int ret = -ENOMEM;
pud_t *pud;
pmd_t *pmd;
prot = pgprot_val(REGION3_KERNEL);
if (!MACHINE_HAS_NX)
prot &= ~_REGION_ENTRY_NOEXEC;
pud = pud_offset(p4d, addr);
for (; addr < end; addr = next, pud++) {
next = pud_addr_end(addr, end);
if (!add) {
if (pud_none(*pud))
continue;
if (pud_large(*pud)) {
if (IS_ALIGNED(addr, PUD_SIZE) &&
IS_ALIGNED(next, PUD_SIZE)) {
pud_clear(pud);
pages++;
}
continue;
}
} else if (pud_none(*pud)) {
if (IS_ALIGNED(addr, PUD_SIZE) &&
IS_ALIGNED(next, PUD_SIZE) &&
MACHINE_HAS_EDAT2 && addr && direct &&
!debug_pagealloc_enabled()) {
pud_val(*pud) = __pa(addr) | prot;
pages++;
continue;
}
pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY);
if (!pmd)
goto out;
pud_populate(&init_mm, pud, pmd);
} else if (pud_large(*pud)) {
continue;
}
ret = modify_pmd_table(pud, addr, next, add, direct);
if (ret)
goto out;
if (!add)
try_free_pmd_table(pud, addr & PUD_MASK);
}
ret = 0;
out:
if (direct)
update_page_count(PG_DIRECT_MAP_2G, add ? pages : -pages);
return ret;
}