in net/bpf_jit_comp32.c [1455:1898]
int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
{
const u8 *dst = bpf2mips32[insn->dst_reg];
const u8 *src = bpf2mips32[insn->src_reg];
const u8 *res = bpf2mips32[BPF_REG_0];
const u8 *tmp = bpf2mips32[JIT_REG_TMP];
u8 code = insn->code;
s16 off = insn->off;
s32 imm = insn->imm;
s32 val, rel;
u8 alu, jmp;
switch (code) {
/* ALU operations */
/* dst = imm */
case BPF_ALU | BPF_MOV | BPF_K:
emit_mov_i(ctx, lo(dst), imm);
emit_zext_ver(ctx, dst);
break;
/* dst = src */
case BPF_ALU | BPF_MOV | BPF_X:
if (imm == 1) {
/* Special mov32 for zext */
emit_mov_i(ctx, hi(dst), 0);
} else {
emit_mov_r(ctx, lo(dst), lo(src));
emit_zext_ver(ctx, dst);
}
break;
/* dst = -dst */
case BPF_ALU | BPF_NEG:
emit_alu_i(ctx, lo(dst), 0, BPF_NEG);
emit_zext_ver(ctx, dst);
break;
/* dst = dst & imm */
/* dst = dst | imm */
/* dst = dst ^ imm */
/* dst = dst << imm */
/* dst = dst >> imm */
/* dst = dst >> imm (arithmetic) */
/* dst = dst + imm */
/* dst = dst - imm */
/* dst = dst * imm */
/* dst = dst / imm */
/* dst = dst % imm */
case BPF_ALU | BPF_OR | BPF_K:
case BPF_ALU | BPF_AND | BPF_K:
case BPF_ALU | BPF_XOR | BPF_K:
case BPF_ALU | BPF_LSH | BPF_K:
case BPF_ALU | BPF_RSH | BPF_K:
case BPF_ALU | BPF_ARSH | BPF_K:
case BPF_ALU | BPF_ADD | BPF_K:
case BPF_ALU | BPF_SUB | BPF_K:
case BPF_ALU | BPF_MUL | BPF_K:
case BPF_ALU | BPF_DIV | BPF_K:
case BPF_ALU | BPF_MOD | BPF_K:
if (!valid_alu_i(BPF_OP(code), imm)) {
emit_mov_i(ctx, MIPS_R_T6, imm);
emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code));
} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
emit_alu_i(ctx, lo(dst), val, alu);
}
emit_zext_ver(ctx, dst);
break;
/* dst = dst & src */
/* dst = dst | src */
/* dst = dst ^ src */
/* dst = dst << src */
/* dst = dst >> src */
/* dst = dst >> src (arithmetic) */
/* dst = dst + src */
/* dst = dst - src */
/* dst = dst * src */
/* dst = dst / src */
/* dst = dst % src */
case BPF_ALU | BPF_AND | BPF_X:
case BPF_ALU | BPF_OR | BPF_X:
case BPF_ALU | BPF_XOR | BPF_X:
case BPF_ALU | BPF_LSH | BPF_X:
case BPF_ALU | BPF_RSH | BPF_X:
case BPF_ALU | BPF_ARSH | BPF_X:
case BPF_ALU | BPF_ADD | BPF_X:
case BPF_ALU | BPF_SUB | BPF_X:
case BPF_ALU | BPF_MUL | BPF_X:
case BPF_ALU | BPF_DIV | BPF_X:
case BPF_ALU | BPF_MOD | BPF_X:
emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code));
emit_zext_ver(ctx, dst);
break;
/* dst = imm (64-bit) */
case BPF_ALU64 | BPF_MOV | BPF_K:
emit_mov_se_i64(ctx, dst, imm);
break;
/* dst = src (64-bit) */
case BPF_ALU64 | BPF_MOV | BPF_X:
emit_mov_r(ctx, lo(dst), lo(src));
emit_mov_r(ctx, hi(dst), hi(src));
break;
/* dst = -dst (64-bit) */
case BPF_ALU64 | BPF_NEG:
emit_neg_i64(ctx, dst);
break;
/* dst = dst & imm (64-bit) */
case BPF_ALU64 | BPF_AND | BPF_K:
emit_alu_i64(ctx, dst, imm, BPF_OP(code));
break;
/* dst = dst | imm (64-bit) */
/* dst = dst ^ imm (64-bit) */
/* dst = dst + imm (64-bit) */
/* dst = dst - imm (64-bit) */
case BPF_ALU64 | BPF_OR | BPF_K:
case BPF_ALU64 | BPF_XOR | BPF_K:
case BPF_ALU64 | BPF_ADD | BPF_K:
case BPF_ALU64 | BPF_SUB | BPF_K:
if (imm)
emit_alu_i64(ctx, dst, imm, BPF_OP(code));
break;
/* dst = dst << imm (64-bit) */
/* dst = dst >> imm (64-bit) */
/* dst = dst >> imm (64-bit, arithmetic) */
case BPF_ALU64 | BPF_LSH | BPF_K:
case BPF_ALU64 | BPF_RSH | BPF_K:
case BPF_ALU64 | BPF_ARSH | BPF_K:
if (imm)
emit_shift_i64(ctx, dst, imm, BPF_OP(code));
break;
/* dst = dst * imm (64-bit) */
case BPF_ALU64 | BPF_MUL | BPF_K:
emit_mul_i64(ctx, dst, imm);
break;
/* dst = dst / imm (64-bit) */
/* dst = dst % imm (64-bit) */
case BPF_ALU64 | BPF_DIV | BPF_K:
case BPF_ALU64 | BPF_MOD | BPF_K:
/*
* Sign-extend the immediate value into a temporary register,
* and then do the operation on this register.
*/
emit_mov_se_i64(ctx, tmp, imm);
emit_divmod_r64(ctx, dst, tmp, BPF_OP(code));
break;
/* dst = dst & src (64-bit) */
/* dst = dst | src (64-bit) */
/* dst = dst ^ src (64-bit) */
/* dst = dst + src (64-bit) */
/* dst = dst - src (64-bit) */
case BPF_ALU64 | BPF_AND | BPF_X:
case BPF_ALU64 | BPF_OR | BPF_X:
case BPF_ALU64 | BPF_XOR | BPF_X:
case BPF_ALU64 | BPF_ADD | BPF_X:
case BPF_ALU64 | BPF_SUB | BPF_X:
emit_alu_r64(ctx, dst, src, BPF_OP(code));
break;
/* dst = dst << src (64-bit) */
/* dst = dst >> src (64-bit) */
/* dst = dst >> src (64-bit, arithmetic) */
case BPF_ALU64 | BPF_LSH | BPF_X:
case BPF_ALU64 | BPF_RSH | BPF_X:
case BPF_ALU64 | BPF_ARSH | BPF_X:
emit_shift_r64(ctx, dst, lo(src), BPF_OP(code));
break;
/* dst = dst * src (64-bit) */
case BPF_ALU64 | BPF_MUL | BPF_X:
emit_mul_r64(ctx, dst, src);
break;
/* dst = dst / src (64-bit) */
/* dst = dst % src (64-bit) */
case BPF_ALU64 | BPF_DIV | BPF_X:
case BPF_ALU64 | BPF_MOD | BPF_X:
emit_divmod_r64(ctx, dst, src, BPF_OP(code));
break;
/* dst = htole(dst) */
/* dst = htobe(dst) */
case BPF_ALU | BPF_END | BPF_FROM_LE:
case BPF_ALU | BPF_END | BPF_FROM_BE:
if (BPF_SRC(code) ==
#ifdef __BIG_ENDIAN
BPF_FROM_LE
#else
BPF_FROM_BE
#endif
)
emit_bswap_r64(ctx, dst, imm);
else
emit_trunc_r64(ctx, dst, imm);
break;
/* dst = imm64 */
case BPF_LD | BPF_IMM | BPF_DW:
emit_mov_i(ctx, lo(dst), imm);
emit_mov_i(ctx, hi(dst), insn[1].imm);
return 1;
/* LDX: dst = *(size *)(src + off) */
case BPF_LDX | BPF_MEM | BPF_W:
case BPF_LDX | BPF_MEM | BPF_H:
case BPF_LDX | BPF_MEM | BPF_B:
case BPF_LDX | BPF_MEM | BPF_DW:
emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code));
break;
/* ST: *(size *)(dst + off) = imm */
case BPF_ST | BPF_MEM | BPF_W:
case BPF_ST | BPF_MEM | BPF_H:
case BPF_ST | BPF_MEM | BPF_B:
case BPF_ST | BPF_MEM | BPF_DW:
switch (BPF_SIZE(code)) {
case BPF_DW:
/* Sign-extend immediate value into temporary reg */
emit_mov_se_i64(ctx, tmp, imm);
break;
case BPF_W:
case BPF_H:
case BPF_B:
emit_mov_i(ctx, lo(tmp), imm);
break;
}
emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code));
break;
/* STX: *(size *)(dst + off) = src */
case BPF_STX | BPF_MEM | BPF_W:
case BPF_STX | BPF_MEM | BPF_H:
case BPF_STX | BPF_MEM | BPF_B:
case BPF_STX | BPF_MEM | BPF_DW:
emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code));
break;
/* Speculation barrier */
case BPF_ST | BPF_NOSPEC:
break;
/* Atomics */
case BPF_STX | BPF_ATOMIC | BPF_W:
switch (imm) {
case BPF_ADD:
case BPF_ADD | BPF_FETCH:
case BPF_AND:
case BPF_AND | BPF_FETCH:
case BPF_OR:
case BPF_OR | BPF_FETCH:
case BPF_XOR:
case BPF_XOR | BPF_FETCH:
case BPF_XCHG:
if (cpu_has_llsc)
emit_atomic_r(ctx, lo(dst), lo(src), off, imm);
else /* Non-ll/sc fallback */
emit_atomic_r32(ctx, lo(dst), lo(src),
off, imm);
if (imm & BPF_FETCH)
emit_zext_ver(ctx, src);
break;
case BPF_CMPXCHG:
if (cpu_has_llsc)
emit_cmpxchg_r(ctx, lo(dst), lo(src),
lo(res), off);
else /* Non-ll/sc fallback */
emit_cmpxchg_r32(ctx, lo(dst), lo(src), off);
/* Result zero-extension inserted by verifier */
break;
default:
goto notyet;
}
break;
/* Atomics (64-bit) */
case BPF_STX | BPF_ATOMIC | BPF_DW:
switch (imm) {
case BPF_ADD:
case BPF_ADD | BPF_FETCH:
case BPF_AND:
case BPF_AND | BPF_FETCH:
case BPF_OR:
case BPF_OR | BPF_FETCH:
case BPF_XOR:
case BPF_XOR | BPF_FETCH:
case BPF_XCHG:
emit_atomic_r64(ctx, lo(dst), src, off, imm);
break;
case BPF_CMPXCHG:
emit_cmpxchg_r64(ctx, lo(dst), src, off);
break;
default:
goto notyet;
}
break;
/* PC += off if dst == src */
/* PC += off if dst != src */
/* PC += off if dst & src */
/* PC += off if dst > src */
/* PC += off if dst >= src */
/* PC += off if dst < src */
/* PC += off if dst <= src */
/* PC += off if dst > src (signed) */
/* PC += off if dst >= src (signed) */
/* PC += off if dst < src (signed) */
/* PC += off if dst <= src (signed) */
case BPF_JMP32 | BPF_JEQ | BPF_X:
case BPF_JMP32 | BPF_JNE | BPF_X:
case BPF_JMP32 | BPF_JSET | BPF_X:
case BPF_JMP32 | BPF_JGT | BPF_X:
case BPF_JMP32 | BPF_JGE | BPF_X:
case BPF_JMP32 | BPF_JLT | BPF_X:
case BPF_JMP32 | BPF_JLE | BPF_X:
case BPF_JMP32 | BPF_JSGT | BPF_X:
case BPF_JMP32 | BPF_JSGE | BPF_X:
case BPF_JMP32 | BPF_JSLT | BPF_X:
case BPF_JMP32 | BPF_JSLE | BPF_X:
if (off == 0)
break;
setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp);
if (finish_jmp(ctx, jmp, off) < 0)
goto toofar;
break;
/* PC += off if dst == imm */
/* PC += off if dst != imm */
/* PC += off if dst & imm */
/* PC += off if dst > imm */
/* PC += off if dst >= imm */
/* PC += off if dst < imm */
/* PC += off if dst <= imm */
/* PC += off if dst > imm (signed) */
/* PC += off if dst >= imm (signed) */
/* PC += off if dst < imm (signed) */
/* PC += off if dst <= imm (signed) */
case BPF_JMP32 | BPF_JEQ | BPF_K:
case BPF_JMP32 | BPF_JNE | BPF_K:
case BPF_JMP32 | BPF_JSET | BPF_K:
case BPF_JMP32 | BPF_JGT | BPF_K:
case BPF_JMP32 | BPF_JGE | BPF_K:
case BPF_JMP32 | BPF_JLT | BPF_K:
case BPF_JMP32 | BPF_JLE | BPF_K:
case BPF_JMP32 | BPF_JSGT | BPF_K:
case BPF_JMP32 | BPF_JSGE | BPF_K:
case BPF_JMP32 | BPF_JSLT | BPF_K:
case BPF_JMP32 | BPF_JSLE | BPF_K:
if (off == 0)
break;
setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel);
if (valid_jmp_i(jmp, imm)) {
emit_jmp_i(ctx, lo(dst), imm, rel, jmp);
} else {
/* Move large immediate to register */
emit_mov_i(ctx, MIPS_R_T6, imm);
emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp);
}
if (finish_jmp(ctx, jmp, off) < 0)
goto toofar;
break;
/* PC += off if dst == src */
/* PC += off if dst != src */
/* PC += off if dst & src */
/* PC += off if dst > src */
/* PC += off if dst >= src */
/* PC += off if dst < src */
/* PC += off if dst <= src */
/* PC += off if dst > src (signed) */
/* PC += off if dst >= src (signed) */
/* PC += off if dst < src (signed) */
/* PC += off if dst <= src (signed) */
case BPF_JMP | BPF_JEQ | BPF_X:
case BPF_JMP | BPF_JNE | BPF_X:
case BPF_JMP | BPF_JSET | BPF_X:
case BPF_JMP | BPF_JGT | BPF_X:
case BPF_JMP | BPF_JGE | BPF_X:
case BPF_JMP | BPF_JLT | BPF_X:
case BPF_JMP | BPF_JLE | BPF_X:
case BPF_JMP | BPF_JSGT | BPF_X:
case BPF_JMP | BPF_JSGE | BPF_X:
case BPF_JMP | BPF_JSLT | BPF_X:
case BPF_JMP | BPF_JSLE | BPF_X:
if (off == 0)
break;
setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
emit_jmp_r64(ctx, dst, src, rel, jmp);
if (finish_jmp(ctx, jmp, off) < 0)
goto toofar;
break;
/* PC += off if dst == imm */
/* PC += off if dst != imm */
/* PC += off if dst & imm */
/* PC += off if dst > imm */
/* PC += off if dst >= imm */
/* PC += off if dst < imm */
/* PC += off if dst <= imm */
/* PC += off if dst > imm (signed) */
/* PC += off if dst >= imm (signed) */
/* PC += off if dst < imm (signed) */
/* PC += off if dst <= imm (signed) */
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP | BPF_JNE | BPF_K:
case BPF_JMP | BPF_JSET | BPF_K:
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP | BPF_JGE | BPF_K:
case BPF_JMP | BPF_JLT | BPF_K:
case BPF_JMP | BPF_JLE | BPF_K:
case BPF_JMP | BPF_JSGT | BPF_K:
case BPF_JMP | BPF_JSGE | BPF_K:
case BPF_JMP | BPF_JSLT | BPF_K:
case BPF_JMP | BPF_JSLE | BPF_K:
if (off == 0)
break;
setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel);
emit_jmp_i64(ctx, dst, imm, rel, jmp);
if (finish_jmp(ctx, jmp, off) < 0)
goto toofar;
break;
/* PC += off */
case BPF_JMP | BPF_JA:
if (off == 0)
break;
if (emit_ja(ctx, off) < 0)
goto toofar;
break;
/* Tail call */
case BPF_JMP | BPF_TAIL_CALL:
if (emit_tail_call(ctx) < 0)
goto invalid;
break;
/* Function call */
case BPF_JMP | BPF_CALL:
if (emit_call(ctx, insn) < 0)
goto invalid;
break;
/* Function return */
case BPF_JMP | BPF_EXIT:
/*
* Optimization: when last instruction is EXIT
* simply continue to epilogue.
*/
if (ctx->bpf_index == ctx->program->len - 1)
break;
if (emit_exit(ctx) < 0)
goto toofar;
break;
default:
invalid:
pr_err_once("unknown opcode %02x\n", code);
return -EINVAL;
notyet:
pr_info_once("*** NOT YET: opcode %02x ***\n", code);
return -EFAULT;
toofar:
pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n",
ctx->bpf_index, code);
return -E2BIG;
}
return 0;
}