in mm/tlbex.c [2108:2307]
static void build_r4000_tlb_load_handler(void)
{
u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbl);
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
struct work_registers wr;
memset(p, 0, handle_tlbl_end - (char *)p);
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
if (bcm1250_m3_war()) {
unsigned int segbits = 44;
uasm_i_dmfc0(&p, K0, C0_BADVADDR);
uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
uasm_i_xor(&p, K0, K0, K1);
uasm_i_dsrl_safe(&p, K1, K0, 62);
uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
uasm_i_or(&p, K0, K0, K1);
uasm_il_bnez(&p, &r, K0, label_leave);
/* No need for uasm_i_nop */
}
wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
if (m4kc_tlbp_war())
build_tlb_probe_entry(&p);
if (cpu_has_rixi && !cpu_has_rixiex) {
/*
* If the page is not _PAGE_VALID, RI or XI could not
* have triggered it. Skip the expensive test..
*/
if (use_bbit_insns()) {
uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
label_tlbl_goaround1);
} else {
uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1);
}
uasm_i_nop(&p);
/*
* Warn if something may race with us & replace the TLB entry
* before we read it here. Everything with such races should
* also have dedicated RiXi exception handlers, so this
* shouldn't be hit.
*/
WARN(cpu_has_tlbex_tlbp_race(), "Unhandled race in RiXi path");
uasm_i_tlbr(&p);
switch (current_cpu_type()) {
default:
if (cpu_has_mips_r2_exec_hazard) {
uasm_i_ehb(&p);
fallthrough;
case CPU_CAVIUM_OCTEON:
case CPU_CAVIUM_OCTEON_PLUS:
case CPU_CAVIUM_OCTEON2:
break;
}
}
/* Examine entrylo 0 or 1 based on ptr. */
if (use_bbit_insns()) {
uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
} else {
uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
uasm_i_beqz(&p, wr.r3, 8);
}
/* load it in the delay slot*/
UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
/* load it if ptr is odd */
UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
/*
* If the entryLo (now in wr.r3) is valid (bit 1), RI or
* XI must have triggered it.
*/
if (use_bbit_insns()) {
uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl);
uasm_i_nop(&p);
uasm_l_tlbl_goaround1(&l, p);
} else {
uasm_i_andi(&p, wr.r3, wr.r3, 2);
uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl);
uasm_i_nop(&p);
}
uasm_l_tlbl_goaround1(&l, p);
}
build_make_valid(&p, &r, wr.r1, wr.r2, wr.r3);
build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
/*
* This is the entry point when build_r4000_tlbchange_handler_head
* spots a huge page.
*/
uasm_l_tlb_huge_update(&l, p);
iPTE_LW(&p, wr.r1, wr.r2);
build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
build_tlb_probe_entry(&p);
if (cpu_has_rixi && !cpu_has_rixiex) {
/*
* If the page is not _PAGE_VALID, RI or XI could not
* have triggered it. Skip the expensive test..
*/
if (use_bbit_insns()) {
uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
label_tlbl_goaround2);
} else {
uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
}
uasm_i_nop(&p);
/*
* Warn if something may race with us & replace the TLB entry
* before we read it here. Everything with such races should
* also have dedicated RiXi exception handlers, so this
* shouldn't be hit.
*/
WARN(cpu_has_tlbex_tlbp_race(), "Unhandled race in RiXi path");
uasm_i_tlbr(&p);
switch (current_cpu_type()) {
default:
if (cpu_has_mips_r2_exec_hazard) {
uasm_i_ehb(&p);
case CPU_CAVIUM_OCTEON:
case CPU_CAVIUM_OCTEON_PLUS:
case CPU_CAVIUM_OCTEON2:
break;
}
}
/* Examine entrylo 0 or 1 based on ptr. */
if (use_bbit_insns()) {
uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
} else {
uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
uasm_i_beqz(&p, wr.r3, 8);
}
/* load it in the delay slot*/
UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
/* load it if ptr is odd */
UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
/*
* If the entryLo (now in wr.r3) is valid (bit 1), RI or
* XI must have triggered it.
*/
if (use_bbit_insns()) {
uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2);
} else {
uasm_i_andi(&p, wr.r3, wr.r3, 2);
uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
}
if (PM_DEFAULT_MASK == 0)
uasm_i_nop(&p);
/*
* We clobbered C0_PAGEMASK, restore it. On the other branch
* it is restored in build_huge_tlb_write_entry.
*/
build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0);
uasm_l_tlbl_goaround2(&l, p);
}
uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID));
build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1);
#endif
uasm_l_nopage_tlbl(&l, p);
if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS))
uasm_i_sync(&p, 0);
build_restore_work_registers(&p);
#ifdef CONFIG_CPU_MICROMIPS
if ((unsigned long)tlb_do_page_fault_0 & 1) {
uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0));
uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0));
uasm_i_jr(&p, K0);
} else
#endif
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
uasm_i_nop(&p);
if (p >= (u32 *)handle_tlbl_end)
panic("TLB load handler fastpath space exceeded");
uasm_resolve_relocs(relocs, labels);
pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
(unsigned int)(p - (u32 *)handle_tlbl));
dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_end);
}