int build_insn()

in net/bpf_jit_comp32.c [1455:1898]


int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
{
	const u8 *dst = bpf2mips32[insn->dst_reg];
	const u8 *src = bpf2mips32[insn->src_reg];
	const u8 *res = bpf2mips32[BPF_REG_0];
	const u8 *tmp = bpf2mips32[JIT_REG_TMP];
	u8 code = insn->code;
	s16 off = insn->off;
	s32 imm = insn->imm;
	s32 val, rel;
	u8 alu, jmp;

	switch (code) {
	/* ALU operations */
	/* dst = imm */
	case BPF_ALU | BPF_MOV | BPF_K:
		emit_mov_i(ctx, lo(dst), imm);
		emit_zext_ver(ctx, dst);
		break;
	/* dst = src */
	case BPF_ALU | BPF_MOV | BPF_X:
		if (imm == 1) {
			/* Special mov32 for zext */
			emit_mov_i(ctx, hi(dst), 0);
		} else {
			emit_mov_r(ctx, lo(dst), lo(src));
			emit_zext_ver(ctx, dst);
		}
		break;
	/* dst = -dst */
	case BPF_ALU | BPF_NEG:
		emit_alu_i(ctx, lo(dst), 0, BPF_NEG);
		emit_zext_ver(ctx, dst);
		break;
	/* dst = dst & imm */
	/* dst = dst | imm */
	/* dst = dst ^ imm */
	/* dst = dst << imm */
	/* dst = dst >> imm */
	/* dst = dst >> imm (arithmetic) */
	/* dst = dst + imm */
	/* dst = dst - imm */
	/* dst = dst * imm */
	/* dst = dst / imm */
	/* dst = dst % imm */
	case BPF_ALU | BPF_OR | BPF_K:
	case BPF_ALU | BPF_AND | BPF_K:
	case BPF_ALU | BPF_XOR | BPF_K:
	case BPF_ALU | BPF_LSH | BPF_K:
	case BPF_ALU | BPF_RSH | BPF_K:
	case BPF_ALU | BPF_ARSH | BPF_K:
	case BPF_ALU | BPF_ADD | BPF_K:
	case BPF_ALU | BPF_SUB | BPF_K:
	case BPF_ALU | BPF_MUL | BPF_K:
	case BPF_ALU | BPF_DIV | BPF_K:
	case BPF_ALU | BPF_MOD | BPF_K:
		if (!valid_alu_i(BPF_OP(code), imm)) {
			emit_mov_i(ctx, MIPS_R_T6, imm);
			emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code));
		} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
			emit_alu_i(ctx, lo(dst), val, alu);
		}
		emit_zext_ver(ctx, dst);
		break;
	/* dst = dst & src */
	/* dst = dst | src */
	/* dst = dst ^ src */
	/* dst = dst << src */
	/* dst = dst >> src */
	/* dst = dst >> src (arithmetic) */
	/* dst = dst + src */
	/* dst = dst - src */
	/* dst = dst * src */
	/* dst = dst / src */
	/* dst = dst % src */
	case BPF_ALU | BPF_AND | BPF_X:
	case BPF_ALU | BPF_OR | BPF_X:
	case BPF_ALU | BPF_XOR | BPF_X:
	case BPF_ALU | BPF_LSH | BPF_X:
	case BPF_ALU | BPF_RSH | BPF_X:
	case BPF_ALU | BPF_ARSH | BPF_X:
	case BPF_ALU | BPF_ADD | BPF_X:
	case BPF_ALU | BPF_SUB | BPF_X:
	case BPF_ALU | BPF_MUL | BPF_X:
	case BPF_ALU | BPF_DIV | BPF_X:
	case BPF_ALU | BPF_MOD | BPF_X:
		emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code));
		emit_zext_ver(ctx, dst);
		break;
	/* dst = imm (64-bit) */
	case BPF_ALU64 | BPF_MOV | BPF_K:
		emit_mov_se_i64(ctx, dst, imm);
		break;
	/* dst = src (64-bit) */
	case BPF_ALU64 | BPF_MOV | BPF_X:
		emit_mov_r(ctx, lo(dst), lo(src));
		emit_mov_r(ctx, hi(dst), hi(src));
		break;
	/* dst = -dst (64-bit) */
	case BPF_ALU64 | BPF_NEG:
		emit_neg_i64(ctx, dst);
		break;
	/* dst = dst & imm (64-bit) */
	case BPF_ALU64 | BPF_AND | BPF_K:
		emit_alu_i64(ctx, dst, imm, BPF_OP(code));
		break;
	/* dst = dst | imm (64-bit) */
	/* dst = dst ^ imm (64-bit) */
	/* dst = dst + imm (64-bit) */
	/* dst = dst - imm (64-bit) */
	case BPF_ALU64 | BPF_OR | BPF_K:
	case BPF_ALU64 | BPF_XOR | BPF_K:
	case BPF_ALU64 | BPF_ADD | BPF_K:
	case BPF_ALU64 | BPF_SUB | BPF_K:
		if (imm)
			emit_alu_i64(ctx, dst, imm, BPF_OP(code));
		break;
	/* dst = dst << imm (64-bit) */
	/* dst = dst >> imm (64-bit) */
	/* dst = dst >> imm (64-bit, arithmetic) */
	case BPF_ALU64 | BPF_LSH | BPF_K:
	case BPF_ALU64 | BPF_RSH | BPF_K:
	case BPF_ALU64 | BPF_ARSH | BPF_K:
		if (imm)
			emit_shift_i64(ctx, dst, imm, BPF_OP(code));
		break;
	/* dst = dst * imm (64-bit) */
	case BPF_ALU64 | BPF_MUL | BPF_K:
		emit_mul_i64(ctx, dst, imm);
		break;
	/* dst = dst / imm (64-bit) */
	/* dst = dst % imm (64-bit) */
	case BPF_ALU64 | BPF_DIV | BPF_K:
	case BPF_ALU64 | BPF_MOD | BPF_K:
		/*
		 * Sign-extend the immediate value into a temporary register,
		 * and then do the operation on this register.
		 */
		emit_mov_se_i64(ctx, tmp, imm);
		emit_divmod_r64(ctx, dst, tmp, BPF_OP(code));
		break;
	/* dst = dst & src (64-bit) */
	/* dst = dst | src (64-bit) */
	/* dst = dst ^ src (64-bit) */
	/* dst = dst + src (64-bit) */
	/* dst = dst - src (64-bit) */
	case BPF_ALU64 | BPF_AND | BPF_X:
	case BPF_ALU64 | BPF_OR | BPF_X:
	case BPF_ALU64 | BPF_XOR | BPF_X:
	case BPF_ALU64 | BPF_ADD | BPF_X:
	case BPF_ALU64 | BPF_SUB | BPF_X:
		emit_alu_r64(ctx, dst, src, BPF_OP(code));
		break;
	/* dst = dst << src (64-bit) */
	/* dst = dst >> src (64-bit) */
	/* dst = dst >> src (64-bit, arithmetic) */
	case BPF_ALU64 | BPF_LSH | BPF_X:
	case BPF_ALU64 | BPF_RSH | BPF_X:
	case BPF_ALU64 | BPF_ARSH | BPF_X:
		emit_shift_r64(ctx, dst, lo(src), BPF_OP(code));
		break;
	/* dst = dst * src (64-bit) */
	case BPF_ALU64 | BPF_MUL | BPF_X:
		emit_mul_r64(ctx, dst, src);
		break;
	/* dst = dst / src (64-bit) */
	/* dst = dst % src (64-bit) */
	case BPF_ALU64 | BPF_DIV | BPF_X:
	case BPF_ALU64 | BPF_MOD | BPF_X:
		emit_divmod_r64(ctx, dst, src, BPF_OP(code));
		break;
	/* dst = htole(dst) */
	/* dst = htobe(dst) */
	case BPF_ALU | BPF_END | BPF_FROM_LE:
	case BPF_ALU | BPF_END | BPF_FROM_BE:
		if (BPF_SRC(code) ==
#ifdef __BIG_ENDIAN
		    BPF_FROM_LE
#else
		    BPF_FROM_BE
#endif
		    )
			emit_bswap_r64(ctx, dst, imm);
		else
			emit_trunc_r64(ctx, dst, imm);
		break;
	/* dst = imm64 */
	case BPF_LD | BPF_IMM | BPF_DW:
		emit_mov_i(ctx, lo(dst), imm);
		emit_mov_i(ctx, hi(dst), insn[1].imm);
		return 1;
	/* LDX: dst = *(size *)(src + off) */
	case BPF_LDX | BPF_MEM | BPF_W:
	case BPF_LDX | BPF_MEM | BPF_H:
	case BPF_LDX | BPF_MEM | BPF_B:
	case BPF_LDX | BPF_MEM | BPF_DW:
		emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code));
		break;
	/* ST: *(size *)(dst + off) = imm */
	case BPF_ST | BPF_MEM | BPF_W:
	case BPF_ST | BPF_MEM | BPF_H:
	case BPF_ST | BPF_MEM | BPF_B:
	case BPF_ST | BPF_MEM | BPF_DW:
		switch (BPF_SIZE(code)) {
		case BPF_DW:
			/* Sign-extend immediate value into temporary reg */
			emit_mov_se_i64(ctx, tmp, imm);
			break;
		case BPF_W:
		case BPF_H:
		case BPF_B:
			emit_mov_i(ctx, lo(tmp), imm);
			break;
		}
		emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code));
		break;
	/* STX: *(size *)(dst + off) = src */
	case BPF_STX | BPF_MEM | BPF_W:
	case BPF_STX | BPF_MEM | BPF_H:
	case BPF_STX | BPF_MEM | BPF_B:
	case BPF_STX | BPF_MEM | BPF_DW:
		emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code));
		break;
	/* Speculation barrier */
	case BPF_ST | BPF_NOSPEC:
		break;
	/* Atomics */
	case BPF_STX | BPF_ATOMIC | BPF_W:
		switch (imm) {
		case BPF_ADD:
		case BPF_ADD | BPF_FETCH:
		case BPF_AND:
		case BPF_AND | BPF_FETCH:
		case BPF_OR:
		case BPF_OR | BPF_FETCH:
		case BPF_XOR:
		case BPF_XOR | BPF_FETCH:
		case BPF_XCHG:
			if (cpu_has_llsc)
				emit_atomic_r(ctx, lo(dst), lo(src), off, imm);
			else /* Non-ll/sc fallback */
				emit_atomic_r32(ctx, lo(dst), lo(src),
						off, imm);
			if (imm & BPF_FETCH)
				emit_zext_ver(ctx, src);
			break;
		case BPF_CMPXCHG:
			if (cpu_has_llsc)
				emit_cmpxchg_r(ctx, lo(dst), lo(src),
					       lo(res), off);
			else /* Non-ll/sc fallback */
				emit_cmpxchg_r32(ctx, lo(dst), lo(src), off);
			/* Result zero-extension inserted by verifier */
			break;
		default:
			goto notyet;
		}
		break;
	/* Atomics (64-bit) */
	case BPF_STX | BPF_ATOMIC | BPF_DW:
		switch (imm) {
		case BPF_ADD:
		case BPF_ADD | BPF_FETCH:
		case BPF_AND:
		case BPF_AND | BPF_FETCH:
		case BPF_OR:
		case BPF_OR | BPF_FETCH:
		case BPF_XOR:
		case BPF_XOR | BPF_FETCH:
		case BPF_XCHG:
			emit_atomic_r64(ctx, lo(dst), src, off, imm);
			break;
		case BPF_CMPXCHG:
			emit_cmpxchg_r64(ctx, lo(dst), src, off);
			break;
		default:
			goto notyet;
		}
		break;
	/* PC += off if dst == src */
	/* PC += off if dst != src */
	/* PC += off if dst & src */
	/* PC += off if dst > src */
	/* PC += off if dst >= src */
	/* PC += off if dst < src */
	/* PC += off if dst <= src */
	/* PC += off if dst > src (signed) */
	/* PC += off if dst >= src (signed) */
	/* PC += off if dst < src (signed) */
	/* PC += off if dst <= src (signed) */
	case BPF_JMP32 | BPF_JEQ | BPF_X:
	case BPF_JMP32 | BPF_JNE | BPF_X:
	case BPF_JMP32 | BPF_JSET | BPF_X:
	case BPF_JMP32 | BPF_JGT | BPF_X:
	case BPF_JMP32 | BPF_JGE | BPF_X:
	case BPF_JMP32 | BPF_JLT | BPF_X:
	case BPF_JMP32 | BPF_JLE | BPF_X:
	case BPF_JMP32 | BPF_JSGT | BPF_X:
	case BPF_JMP32 | BPF_JSGE | BPF_X:
	case BPF_JMP32 | BPF_JSLT | BPF_X:
	case BPF_JMP32 | BPF_JSLE | BPF_X:
		if (off == 0)
			break;
		setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
		emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp);
		if (finish_jmp(ctx, jmp, off) < 0)
			goto toofar;
		break;
	/* PC += off if dst == imm */
	/* PC += off if dst != imm */
	/* PC += off if dst & imm */
	/* PC += off if dst > imm */
	/* PC += off if dst >= imm */
	/* PC += off if dst < imm */
	/* PC += off if dst <= imm */
	/* PC += off if dst > imm (signed) */
	/* PC += off if dst >= imm (signed) */
	/* PC += off if dst < imm (signed) */
	/* PC += off if dst <= imm (signed) */
	case BPF_JMP32 | BPF_JEQ | BPF_K:
	case BPF_JMP32 | BPF_JNE | BPF_K:
	case BPF_JMP32 | BPF_JSET | BPF_K:
	case BPF_JMP32 | BPF_JGT | BPF_K:
	case BPF_JMP32 | BPF_JGE | BPF_K:
	case BPF_JMP32 | BPF_JLT | BPF_K:
	case BPF_JMP32 | BPF_JLE | BPF_K:
	case BPF_JMP32 | BPF_JSGT | BPF_K:
	case BPF_JMP32 | BPF_JSGE | BPF_K:
	case BPF_JMP32 | BPF_JSLT | BPF_K:
	case BPF_JMP32 | BPF_JSLE | BPF_K:
		if (off == 0)
			break;
		setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel);
		if (valid_jmp_i(jmp, imm)) {
			emit_jmp_i(ctx, lo(dst), imm, rel, jmp);
		} else {
			/* Move large immediate to register */
			emit_mov_i(ctx, MIPS_R_T6, imm);
			emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp);
		}
		if (finish_jmp(ctx, jmp, off) < 0)
			goto toofar;
		break;
	/* PC += off if dst == src */
	/* PC += off if dst != src */
	/* PC += off if dst & src */
	/* PC += off if dst > src */
	/* PC += off if dst >= src */
	/* PC += off if dst < src */
	/* PC += off if dst <= src */
	/* PC += off if dst > src (signed) */
	/* PC += off if dst >= src (signed) */
	/* PC += off if dst < src (signed) */
	/* PC += off if dst <= src (signed) */
	case BPF_JMP | BPF_JEQ | BPF_X:
	case BPF_JMP | BPF_JNE | BPF_X:
	case BPF_JMP | BPF_JSET | BPF_X:
	case BPF_JMP | BPF_JGT | BPF_X:
	case BPF_JMP | BPF_JGE | BPF_X:
	case BPF_JMP | BPF_JLT | BPF_X:
	case BPF_JMP | BPF_JLE | BPF_X:
	case BPF_JMP | BPF_JSGT | BPF_X:
	case BPF_JMP | BPF_JSGE | BPF_X:
	case BPF_JMP | BPF_JSLT | BPF_X:
	case BPF_JMP | BPF_JSLE | BPF_X:
		if (off == 0)
			break;
		setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
		emit_jmp_r64(ctx, dst, src, rel, jmp);
		if (finish_jmp(ctx, jmp, off) < 0)
			goto toofar;
		break;
	/* PC += off if dst == imm */
	/* PC += off if dst != imm */
	/* PC += off if dst & imm */
	/* PC += off if dst > imm */
	/* PC += off if dst >= imm */
	/* PC += off if dst < imm */
	/* PC += off if dst <= imm */
	/* PC += off if dst > imm (signed) */
	/* PC += off if dst >= imm (signed) */
	/* PC += off if dst < imm (signed) */
	/* PC += off if dst <= imm (signed) */
	case BPF_JMP | BPF_JEQ | BPF_K:
	case BPF_JMP | BPF_JNE | BPF_K:
	case BPF_JMP | BPF_JSET | BPF_K:
	case BPF_JMP | BPF_JGT | BPF_K:
	case BPF_JMP | BPF_JGE | BPF_K:
	case BPF_JMP | BPF_JLT | BPF_K:
	case BPF_JMP | BPF_JLE | BPF_K:
	case BPF_JMP | BPF_JSGT | BPF_K:
	case BPF_JMP | BPF_JSGE | BPF_K:
	case BPF_JMP | BPF_JSLT | BPF_K:
	case BPF_JMP | BPF_JSLE | BPF_K:
		if (off == 0)
			break;
		setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel);
		emit_jmp_i64(ctx, dst, imm, rel, jmp);
		if (finish_jmp(ctx, jmp, off) < 0)
			goto toofar;
		break;
	/* PC += off */
	case BPF_JMP | BPF_JA:
		if (off == 0)
			break;
		if (emit_ja(ctx, off) < 0)
			goto toofar;
		break;
	/* Tail call */
	case BPF_JMP | BPF_TAIL_CALL:
		if (emit_tail_call(ctx) < 0)
			goto invalid;
		break;
	/* Function call */
	case BPF_JMP | BPF_CALL:
		if (emit_call(ctx, insn) < 0)
			goto invalid;
		break;
	/* Function return */
	case BPF_JMP | BPF_EXIT:
		/*
		 * Optimization: when last instruction is EXIT
		 * simply continue to epilogue.
		 */
		if (ctx->bpf_index == ctx->program->len - 1)
			break;
		if (emit_exit(ctx) < 0)
			goto toofar;
		break;

	default:
invalid:
		pr_err_once("unknown opcode %02x\n", code);
		return -EINVAL;
notyet:
		pr_info_once("*** NOT YET: opcode %02x ***\n", code);
		return -EFAULT;
toofar:
		pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n",
			     ctx->bpf_index, code);
		return -E2BIG;
	}
	return 0;
}