in net/bpf_jit_comp64.c [624:1060]
int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
{
u8 dst = bpf2mips64[insn->dst_reg];
u8 src = bpf2mips64[insn->src_reg];
u8 res = bpf2mips64[BPF_REG_0];
u8 code = insn->code;
s16 off = insn->off;
s32 imm = insn->imm;
s32 val, rel;
u8 alu, jmp;
switch (code) {
/* ALU operations */
/* dst = imm */
case BPF_ALU | BPF_MOV | BPF_K:
emit_mov_i(ctx, dst, imm);
emit_zext_ver(ctx, dst);
break;
/* dst = src */
case BPF_ALU | BPF_MOV | BPF_X:
if (imm == 1) {
/* Special mov32 for zext */
emit_zext(ctx, dst);
} else {
emit_mov_r(ctx, dst, src);
emit_zext_ver(ctx, dst);
}
break;
/* dst = -dst */
case BPF_ALU | BPF_NEG:
emit_sext(ctx, dst, dst);
emit_alu_i(ctx, dst, 0, BPF_NEG);
emit_zext_ver(ctx, dst);
break;
/* dst = dst & imm */
/* dst = dst | imm */
/* dst = dst ^ imm */
/* dst = dst << imm */
case BPF_ALU | BPF_OR | BPF_K:
case BPF_ALU | BPF_AND | BPF_K:
case BPF_ALU | BPF_XOR | BPF_K:
case BPF_ALU | BPF_LSH | BPF_K:
if (!valid_alu_i(BPF_OP(code), imm)) {
emit_mov_i(ctx, MIPS_R_T4, imm);
emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
emit_alu_i(ctx, dst, val, alu);
}
emit_zext_ver(ctx, dst);
break;
/* dst = dst >> imm */
/* dst = dst >> imm (arithmetic) */
/* dst = dst + imm */
/* dst = dst - imm */
/* dst = dst * imm */
/* dst = dst / imm */
/* dst = dst % imm */
case BPF_ALU | BPF_RSH | BPF_K:
case BPF_ALU | BPF_ARSH | BPF_K:
case BPF_ALU | BPF_ADD | BPF_K:
case BPF_ALU | BPF_SUB | BPF_K:
case BPF_ALU | BPF_MUL | BPF_K:
case BPF_ALU | BPF_DIV | BPF_K:
case BPF_ALU | BPF_MOD | BPF_K:
if (!valid_alu_i(BPF_OP(code), imm)) {
emit_sext(ctx, dst, dst);
emit_mov_i(ctx, MIPS_R_T4, imm);
emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
emit_sext(ctx, dst, dst);
emit_alu_i(ctx, dst, val, alu);
}
emit_zext_ver(ctx, dst);
break;
/* dst = dst & src */
/* dst = dst | src */
/* dst = dst ^ src */
/* dst = dst << src */
case BPF_ALU | BPF_AND | BPF_X:
case BPF_ALU | BPF_OR | BPF_X:
case BPF_ALU | BPF_XOR | BPF_X:
case BPF_ALU | BPF_LSH | BPF_X:
emit_alu_r(ctx, dst, src, BPF_OP(code));
emit_zext_ver(ctx, dst);
break;
/* dst = dst >> src */
/* dst = dst >> src (arithmetic) */
/* dst = dst + src */
/* dst = dst - src */
/* dst = dst * src */
/* dst = dst / src */
/* dst = dst % src */
case BPF_ALU | BPF_RSH | BPF_X:
case BPF_ALU | BPF_ARSH | BPF_X:
case BPF_ALU | BPF_ADD | BPF_X:
case BPF_ALU | BPF_SUB | BPF_X:
case BPF_ALU | BPF_MUL | BPF_X:
case BPF_ALU | BPF_DIV | BPF_X:
case BPF_ALU | BPF_MOD | BPF_X:
emit_sext(ctx, dst, dst);
emit_sext(ctx, MIPS_R_T4, src);
emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
emit_zext_ver(ctx, dst);
break;
/* dst = imm (64-bit) */
case BPF_ALU64 | BPF_MOV | BPF_K:
emit_mov_i(ctx, dst, imm);
break;
/* dst = src (64-bit) */
case BPF_ALU64 | BPF_MOV | BPF_X:
emit_mov_r(ctx, dst, src);
break;
/* dst = -dst (64-bit) */
case BPF_ALU64 | BPF_NEG:
emit_alu_i64(ctx, dst, 0, BPF_NEG);
break;
/* dst = dst & imm (64-bit) */
/* dst = dst | imm (64-bit) */
/* dst = dst ^ imm (64-bit) */
/* dst = dst << imm (64-bit) */
/* dst = dst >> imm (64-bit) */
/* dst = dst >> imm ((64-bit, arithmetic) */
/* dst = dst + imm (64-bit) */
/* dst = dst - imm (64-bit) */
/* dst = dst * imm (64-bit) */
/* dst = dst / imm (64-bit) */
/* dst = dst % imm (64-bit) */
case BPF_ALU64 | BPF_AND | BPF_K:
case BPF_ALU64 | BPF_OR | BPF_K:
case BPF_ALU64 | BPF_XOR | BPF_K:
case BPF_ALU64 | BPF_LSH | BPF_K:
case BPF_ALU64 | BPF_RSH | BPF_K:
case BPF_ALU64 | BPF_ARSH | BPF_K:
case BPF_ALU64 | BPF_ADD | BPF_K:
case BPF_ALU64 | BPF_SUB | BPF_K:
case BPF_ALU64 | BPF_MUL | BPF_K:
case BPF_ALU64 | BPF_DIV | BPF_K:
case BPF_ALU64 | BPF_MOD | BPF_K:
if (!valid_alu_i(BPF_OP(code), imm)) {
emit_mov_i(ctx, MIPS_R_T4, imm);
emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code));
} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
emit_alu_i64(ctx, dst, val, alu);
}
break;
/* dst = dst & src (64-bit) */
/* dst = dst | src (64-bit) */
/* dst = dst ^ src (64-bit) */
/* dst = dst << src (64-bit) */
/* dst = dst >> src (64-bit) */
/* dst = dst >> src (64-bit, arithmetic) */
/* dst = dst + src (64-bit) */
/* dst = dst - src (64-bit) */
/* dst = dst * src (64-bit) */
/* dst = dst / src (64-bit) */
/* dst = dst % src (64-bit) */
case BPF_ALU64 | BPF_AND | BPF_X:
case BPF_ALU64 | BPF_OR | BPF_X:
case BPF_ALU64 | BPF_XOR | BPF_X:
case BPF_ALU64 | BPF_LSH | BPF_X:
case BPF_ALU64 | BPF_RSH | BPF_X:
case BPF_ALU64 | BPF_ARSH | BPF_X:
case BPF_ALU64 | BPF_ADD | BPF_X:
case BPF_ALU64 | BPF_SUB | BPF_X:
case BPF_ALU64 | BPF_MUL | BPF_X:
case BPF_ALU64 | BPF_DIV | BPF_X:
case BPF_ALU64 | BPF_MOD | BPF_X:
emit_alu_r64(ctx, dst, src, BPF_OP(code));
break;
/* dst = htole(dst) */
/* dst = htobe(dst) */
case BPF_ALU | BPF_END | BPF_FROM_LE:
case BPF_ALU | BPF_END | BPF_FROM_BE:
if (BPF_SRC(code) ==
#ifdef __BIG_ENDIAN
BPF_FROM_LE
#else
BPF_FROM_BE
#endif
)
emit_bswap_r64(ctx, dst, imm);
else
emit_trunc_r64(ctx, dst, imm);
break;
/* dst = imm64 */
case BPF_LD | BPF_IMM | BPF_DW:
emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32));
return 1;
/* LDX: dst = *(size *)(src + off) */
case BPF_LDX | BPF_MEM | BPF_W:
case BPF_LDX | BPF_MEM | BPF_H:
case BPF_LDX | BPF_MEM | BPF_B:
case BPF_LDX | BPF_MEM | BPF_DW:
emit_ldx(ctx, dst, src, off, BPF_SIZE(code));
break;
/* ST: *(size *)(dst + off) = imm */
case BPF_ST | BPF_MEM | BPF_W:
case BPF_ST | BPF_MEM | BPF_H:
case BPF_ST | BPF_MEM | BPF_B:
case BPF_ST | BPF_MEM | BPF_DW:
emit_mov_i(ctx, MIPS_R_T4, imm);
emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code));
break;
/* STX: *(size *)(dst + off) = src */
case BPF_STX | BPF_MEM | BPF_W:
case BPF_STX | BPF_MEM | BPF_H:
case BPF_STX | BPF_MEM | BPF_B:
case BPF_STX | BPF_MEM | BPF_DW:
emit_stx(ctx, dst, src, off, BPF_SIZE(code));
break;
/* Speculation barrier */
case BPF_ST | BPF_NOSPEC:
break;
/* Atomics */
case BPF_STX | BPF_ATOMIC | BPF_W:
case BPF_STX | BPF_ATOMIC | BPF_DW:
switch (imm) {
case BPF_ADD:
case BPF_ADD | BPF_FETCH:
case BPF_AND:
case BPF_AND | BPF_FETCH:
case BPF_OR:
case BPF_OR | BPF_FETCH:
case BPF_XOR:
case BPF_XOR | BPF_FETCH:
case BPF_XCHG:
if (BPF_SIZE(code) == BPF_DW) {
emit_atomic_r64(ctx, dst, src, off, imm);
} else if (imm & BPF_FETCH) {
u8 tmp = dst;
if (src == dst) { /* Don't overwrite dst */
emit_mov_r(ctx, MIPS_R_T4, dst);
tmp = MIPS_R_T4;
}
emit_sext(ctx, src, src);
emit_atomic_r(ctx, tmp, src, off, imm);
emit_zext_ver(ctx, src);
} else { /* 32-bit, no fetch */
emit_sext(ctx, MIPS_R_T4, src);
emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm);
}
break;
case BPF_CMPXCHG:
if (BPF_SIZE(code) == BPF_DW) {
emit_cmpxchg_r64(ctx, dst, src, off);
} else {
u8 tmp = res;
if (res == dst) /* Don't overwrite dst */
tmp = MIPS_R_T4;
emit_sext(ctx, tmp, res);
emit_sext(ctx, MIPS_R_T5, src);
emit_cmpxchg_r(ctx, dst, MIPS_R_T5, tmp, off);
if (res == dst) /* Restore result */
emit_mov_r(ctx, res, MIPS_R_T4);
/* Result zext inserted by verifier */
}
break;
default:
goto notyet;
}
break;
/* PC += off if dst == src */
/* PC += off if dst != src */
/* PC += off if dst & src */
/* PC += off if dst > src */
/* PC += off if dst >= src */
/* PC += off if dst < src */
/* PC += off if dst <= src */
/* PC += off if dst > src (signed) */
/* PC += off if dst >= src (signed) */
/* PC += off if dst < src (signed) */
/* PC += off if dst <= src (signed) */
case BPF_JMP32 | BPF_JEQ | BPF_X:
case BPF_JMP32 | BPF_JNE | BPF_X:
case BPF_JMP32 | BPF_JSET | BPF_X:
case BPF_JMP32 | BPF_JGT | BPF_X:
case BPF_JMP32 | BPF_JGE | BPF_X:
case BPF_JMP32 | BPF_JLT | BPF_X:
case BPF_JMP32 | BPF_JLE | BPF_X:
case BPF_JMP32 | BPF_JSGT | BPF_X:
case BPF_JMP32 | BPF_JSGE | BPF_X:
case BPF_JMP32 | BPF_JSLT | BPF_X:
case BPF_JMP32 | BPF_JSLE | BPF_X:
if (off == 0)
break;
setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */
emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */
emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp);
if (finish_jmp(ctx, jmp, off) < 0)
goto toofar;
break;
/* PC += off if dst == imm */
/* PC += off if dst != imm */
/* PC += off if dst & imm */
/* PC += off if dst > imm */
/* PC += off if dst >= imm */
/* PC += off if dst < imm */
/* PC += off if dst <= imm */
/* PC += off if dst > imm (signed) */
/* PC += off if dst >= imm (signed) */
/* PC += off if dst < imm (signed) */
/* PC += off if dst <= imm (signed) */
case BPF_JMP32 | BPF_JEQ | BPF_K:
case BPF_JMP32 | BPF_JNE | BPF_K:
case BPF_JMP32 | BPF_JSET | BPF_K:
case BPF_JMP32 | BPF_JGT | BPF_K:
case BPF_JMP32 | BPF_JGE | BPF_K:
case BPF_JMP32 | BPF_JLT | BPF_K:
case BPF_JMP32 | BPF_JLE | BPF_K:
case BPF_JMP32 | BPF_JSGT | BPF_K:
case BPF_JMP32 | BPF_JSGE | BPF_K:
case BPF_JMP32 | BPF_JSLT | BPF_K:
case BPF_JMP32 | BPF_JSLE | BPF_K:
if (off == 0)
break;
setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel);
emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */
if (valid_jmp_i(jmp, imm)) {
emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp);
} else {
/* Move large immediate to register, sign-extended */
emit_mov_i(ctx, MIPS_R_T5, imm);
emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp);
}
if (finish_jmp(ctx, jmp, off) < 0)
goto toofar;
break;
/* PC += off if dst == src */
/* PC += off if dst != src */
/* PC += off if dst & src */
/* PC += off if dst > src */
/* PC += off if dst >= src */
/* PC += off if dst < src */
/* PC += off if dst <= src */
/* PC += off if dst > src (signed) */
/* PC += off if dst >= src (signed) */
/* PC += off if dst < src (signed) */
/* PC += off if dst <= src (signed) */
case BPF_JMP | BPF_JEQ | BPF_X:
case BPF_JMP | BPF_JNE | BPF_X:
case BPF_JMP | BPF_JSET | BPF_X:
case BPF_JMP | BPF_JGT | BPF_X:
case BPF_JMP | BPF_JGE | BPF_X:
case BPF_JMP | BPF_JLT | BPF_X:
case BPF_JMP | BPF_JLE | BPF_X:
case BPF_JMP | BPF_JSGT | BPF_X:
case BPF_JMP | BPF_JSGE | BPF_X:
case BPF_JMP | BPF_JSLT | BPF_X:
case BPF_JMP | BPF_JSLE | BPF_X:
if (off == 0)
break;
setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
emit_jmp_r(ctx, dst, src, rel, jmp);
if (finish_jmp(ctx, jmp, off) < 0)
goto toofar;
break;
/* PC += off if dst == imm */
/* PC += off if dst != imm */
/* PC += off if dst & imm */
/* PC += off if dst > imm */
/* PC += off if dst >= imm */
/* PC += off if dst < imm */
/* PC += off if dst <= imm */
/* PC += off if dst > imm (signed) */
/* PC += off if dst >= imm (signed) */
/* PC += off if dst < imm (signed) */
/* PC += off if dst <= imm (signed) */
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP | BPF_JNE | BPF_K:
case BPF_JMP | BPF_JSET | BPF_K:
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP | BPF_JGE | BPF_K:
case BPF_JMP | BPF_JLT | BPF_K:
case BPF_JMP | BPF_JLE | BPF_K:
case BPF_JMP | BPF_JSGT | BPF_K:
case BPF_JMP | BPF_JSGE | BPF_K:
case BPF_JMP | BPF_JSLT | BPF_K:
case BPF_JMP | BPF_JSLE | BPF_K:
if (off == 0)
break;
setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel);
if (valid_jmp_i(jmp, imm)) {
emit_jmp_i(ctx, dst, imm, rel, jmp);
} else {
/* Move large immediate to register */
emit_mov_i(ctx, MIPS_R_T4, imm);
emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp);
}
if (finish_jmp(ctx, jmp, off) < 0)
goto toofar;
break;
/* PC += off */
case BPF_JMP | BPF_JA:
if (off == 0)
break;
if (emit_ja(ctx, off) < 0)
goto toofar;
break;
/* Tail call */
case BPF_JMP | BPF_TAIL_CALL:
if (emit_tail_call(ctx) < 0)
goto invalid;
break;
/* Function call */
case BPF_JMP | BPF_CALL:
if (emit_call(ctx, insn) < 0)
goto invalid;
break;
/* Function return */
case BPF_JMP | BPF_EXIT:
/*
* Optimization: when last instruction is EXIT
* simply continue to epilogue.
*/
if (ctx->bpf_index == ctx->program->len - 1)
break;
if (emit_exit(ctx) < 0)
goto toofar;
break;
default:
invalid:
pr_err_once("unknown opcode %02x\n", code);
return -EINVAL;
notyet:
pr_info_once("*** NOT YET: opcode %02x ***\n", code);
return -EFAULT;
toofar:
pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n",
ctx->bpf_index, code);
return -E2BIG;
}
return 0;
}