in mm/book3s64/hash_utils.c [1397:1604]
int hash_page_mm(struct mm_struct *mm, unsigned long ea,
unsigned long access, unsigned long trap,
unsigned long flags)
{
bool is_thp;
pgd_t *pgdir;
unsigned long vsid;
pte_t *ptep;
unsigned hugeshift;
int rc, user_region = 0;
int psize, ssize;
DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n",
ea, access, trap);
trace_hash_fault(ea, access, trap);
/* Get region & vsid */
switch (get_region_id(ea)) {
case USER_REGION_ID:
user_region = 1;
if (! mm) {
DBG_LOW(" user region with no mm !\n");
rc = 1;
goto bail;
}
psize = get_slice_psize(mm, ea);
ssize = user_segment_size(ea);
vsid = get_user_vsid(&mm->context, ea, ssize);
break;
case VMALLOC_REGION_ID:
vsid = get_kernel_vsid(ea, mmu_kernel_ssize);
psize = mmu_vmalloc_psize;
ssize = mmu_kernel_ssize;
flags |= HPTE_USE_KERNEL_KEY;
break;
case IO_REGION_ID:
vsid = get_kernel_vsid(ea, mmu_kernel_ssize);
psize = mmu_io_psize;
ssize = mmu_kernel_ssize;
flags |= HPTE_USE_KERNEL_KEY;
break;
default:
/*
* Not a valid range
* Send the problem up to do_page_fault()
*/
rc = 1;
goto bail;
}
DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm, mm->pgd, vsid);
/* Bad address. */
if (!vsid) {
DBG_LOW("Bad address!\n");
rc = 1;
goto bail;
}
/* Get pgdir */
pgdir = mm->pgd;
if (pgdir == NULL) {
rc = 1;
goto bail;
}
/* Check CPU locality */
if (user_region && mm_is_thread_local(mm))
flags |= HPTE_LOCAL_UPDATE;
#ifndef CONFIG_PPC_64K_PAGES
/*
* If we use 4K pages and our psize is not 4K, then we might
* be hitting a special driver mapping, and need to align the
* address before we fetch the PTE.
*
* It could also be a hugepage mapping, in which case this is
* not necessary, but it's not harmful, either.
*/
if (psize != MMU_PAGE_4K)
ea &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
#endif /* CONFIG_PPC_64K_PAGES */
/* Get PTE and page size from page tables */
ptep = find_linux_pte(pgdir, ea, &is_thp, &hugeshift);
if (ptep == NULL || !pte_present(*ptep)) {
DBG_LOW(" no PTE !\n");
rc = 1;
goto bail;
}
/*
* Add _PAGE_PRESENT to the required access perm. If there are parallel
* updates to the pte that can possibly clear _PAGE_PTE, catch that too.
*
* We can safely use the return pte address in rest of the function
* because we do set H_PAGE_BUSY which prevents further updates to pte
* from generic code.
*/
access |= _PAGE_PRESENT | _PAGE_PTE;
/*
* Pre-check access permissions (will be re-checked atomically
* in __hash_page_XX but this pre-check is a fast path
*/
if (!check_pte_access(access, pte_val(*ptep))) {
DBG_LOW(" no access !\n");
rc = 1;
goto bail;
}
if (hugeshift) {
if (is_thp)
rc = __hash_page_thp(ea, access, vsid, (pmd_t *)ptep,
trap, flags, ssize, psize);
#ifdef CONFIG_HUGETLB_PAGE
else
rc = __hash_page_huge(ea, access, vsid, ptep, trap,
flags, ssize, hugeshift, psize);
#else
else {
/*
* if we have hugeshift, and is not transhuge with
* hugetlb disabled, something is really wrong.
*/
rc = 1;
WARN_ON(1);
}
#endif
if (current->mm == mm)
check_paca_psize(ea, mm, psize, user_region);
goto bail;
}
#ifndef CONFIG_PPC_64K_PAGES
DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep));
#else
DBG_LOW(" i-pte: %016lx %016lx\n", pte_val(*ptep),
pte_val(*(ptep + PTRS_PER_PTE)));
#endif
/* Do actual hashing */
#ifdef CONFIG_PPC_64K_PAGES
/* If H_PAGE_4K_PFN is set, make sure this is a 4k segment */
if ((pte_val(*ptep) & H_PAGE_4K_PFN) && psize == MMU_PAGE_64K) {
demote_segment_4k(mm, ea);
psize = MMU_PAGE_4K;
}
/*
* If this PTE is non-cacheable and we have restrictions on
* using non cacheable large pages, then we switch to 4k
*/
if (mmu_ci_restrictions && psize == MMU_PAGE_64K && pte_ci(*ptep)) {
if (user_region) {
demote_segment_4k(mm, ea);
psize = MMU_PAGE_4K;
} else if (ea < VMALLOC_END) {
/*
* some driver did a non-cacheable mapping
* in vmalloc space, so switch vmalloc
* to 4k pages
*/
printk(KERN_ALERT "Reducing vmalloc segment "
"to 4kB pages because of "
"non-cacheable mapping\n");
psize = mmu_vmalloc_psize = MMU_PAGE_4K;
copro_flush_all_slbs(mm);
}
}
#endif /* CONFIG_PPC_64K_PAGES */
if (current->mm == mm)
check_paca_psize(ea, mm, psize, user_region);
#ifdef CONFIG_PPC_64K_PAGES
if (psize == MMU_PAGE_64K)
rc = __hash_page_64K(ea, access, vsid, ptep, trap,
flags, ssize);
else
#endif /* CONFIG_PPC_64K_PAGES */
{
int spp = subpage_protection(mm, ea);
if (access & spp)
rc = -2;
else
rc = __hash_page_4K(ea, access, vsid, ptep, trap,
flags, ssize, spp);
}
/*
* Dump some info in case of hash insertion failure, they should
* never happen so it is really useful to know if/when they do
*/
if (rc == -1)
hash_failure_debug(ea, access, vsid, trap, ssize, psize,
psize, pte_val(*ptep));
#ifndef CONFIG_PPC_64K_PAGES
DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep));
#else
DBG_LOW(" o-pte: %016lx %016lx\n", pte_val(*ptep),
pte_val(*(ptep + PTRS_PER_PTE)));
#endif
DBG_LOW(" -> rc=%d\n", rc);
bail:
return rc;
}