int build_insn()

in net/bpf_jit_comp64.c [624:1060]


int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
{
	u8 dst = bpf2mips64[insn->dst_reg];
	u8 src = bpf2mips64[insn->src_reg];
	u8 res = bpf2mips64[BPF_REG_0];
	u8 code = insn->code;
	s16 off = insn->off;
	s32 imm = insn->imm;
	s32 val, rel;
	u8 alu, jmp;

	switch (code) {
	/* ALU operations */
	/* dst = imm */
	case BPF_ALU | BPF_MOV | BPF_K:
		emit_mov_i(ctx, dst, imm);
		emit_zext_ver(ctx, dst);
		break;
	/* dst = src */
	case BPF_ALU | BPF_MOV | BPF_X:
		if (imm == 1) {
			/* Special mov32 for zext */
			emit_zext(ctx, dst);
		} else {
			emit_mov_r(ctx, dst, src);
			emit_zext_ver(ctx, dst);
		}
		break;
	/* dst = -dst */
	case BPF_ALU | BPF_NEG:
		emit_sext(ctx, dst, dst);
		emit_alu_i(ctx, dst, 0, BPF_NEG);
		emit_zext_ver(ctx, dst);
		break;
	/* dst = dst & imm */
	/* dst = dst | imm */
	/* dst = dst ^ imm */
	/* dst = dst << imm */
	case BPF_ALU | BPF_OR | BPF_K:
	case BPF_ALU | BPF_AND | BPF_K:
	case BPF_ALU | BPF_XOR | BPF_K:
	case BPF_ALU | BPF_LSH | BPF_K:
		if (!valid_alu_i(BPF_OP(code), imm)) {
			emit_mov_i(ctx, MIPS_R_T4, imm);
			emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
		} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
			emit_alu_i(ctx, dst, val, alu);
		}
		emit_zext_ver(ctx, dst);
		break;
	/* dst = dst >> imm */
	/* dst = dst >> imm (arithmetic) */
	/* dst = dst + imm */
	/* dst = dst - imm */
	/* dst = dst * imm */
	/* dst = dst / imm */
	/* dst = dst % imm */
	case BPF_ALU | BPF_RSH | BPF_K:
	case BPF_ALU | BPF_ARSH | BPF_K:
	case BPF_ALU | BPF_ADD | BPF_K:
	case BPF_ALU | BPF_SUB | BPF_K:
	case BPF_ALU | BPF_MUL | BPF_K:
	case BPF_ALU | BPF_DIV | BPF_K:
	case BPF_ALU | BPF_MOD | BPF_K:
		if (!valid_alu_i(BPF_OP(code), imm)) {
			emit_sext(ctx, dst, dst);
			emit_mov_i(ctx, MIPS_R_T4, imm);
			emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
		} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
			emit_sext(ctx, dst, dst);
			emit_alu_i(ctx, dst, val, alu);
		}
		emit_zext_ver(ctx, dst);
		break;
	/* dst = dst & src */
	/* dst = dst | src */
	/* dst = dst ^ src */
	/* dst = dst << src */
	case BPF_ALU | BPF_AND | BPF_X:
	case BPF_ALU | BPF_OR | BPF_X:
	case BPF_ALU | BPF_XOR | BPF_X:
	case BPF_ALU | BPF_LSH | BPF_X:
		emit_alu_r(ctx, dst, src, BPF_OP(code));
		emit_zext_ver(ctx, dst);
		break;
	/* dst = dst >> src */
	/* dst = dst >> src (arithmetic) */
	/* dst = dst + src */
	/* dst = dst - src */
	/* dst = dst * src */
	/* dst = dst / src */
	/* dst = dst % src */
	case BPF_ALU | BPF_RSH | BPF_X:
	case BPF_ALU | BPF_ARSH | BPF_X:
	case BPF_ALU | BPF_ADD | BPF_X:
	case BPF_ALU | BPF_SUB | BPF_X:
	case BPF_ALU | BPF_MUL | BPF_X:
	case BPF_ALU | BPF_DIV | BPF_X:
	case BPF_ALU | BPF_MOD | BPF_X:
		emit_sext(ctx, dst, dst);
		emit_sext(ctx, MIPS_R_T4, src);
		emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
		emit_zext_ver(ctx, dst);
		break;
	/* dst = imm (64-bit) */
	case BPF_ALU64 | BPF_MOV | BPF_K:
		emit_mov_i(ctx, dst, imm);
		break;
	/* dst = src (64-bit) */
	case BPF_ALU64 | BPF_MOV | BPF_X:
		emit_mov_r(ctx, dst, src);
		break;
	/* dst = -dst (64-bit) */
	case BPF_ALU64 | BPF_NEG:
		emit_alu_i64(ctx, dst, 0, BPF_NEG);
		break;
	/* dst = dst & imm (64-bit) */
	/* dst = dst | imm (64-bit) */
	/* dst = dst ^ imm (64-bit) */
	/* dst = dst << imm (64-bit) */
	/* dst = dst >> imm (64-bit) */
	/* dst = dst >> imm ((64-bit, arithmetic) */
	/* dst = dst + imm (64-bit) */
	/* dst = dst - imm (64-bit) */
	/* dst = dst * imm (64-bit) */
	/* dst = dst / imm (64-bit) */
	/* dst = dst % imm (64-bit) */
	case BPF_ALU64 | BPF_AND | BPF_K:
	case BPF_ALU64 | BPF_OR | BPF_K:
	case BPF_ALU64 | BPF_XOR | BPF_K:
	case BPF_ALU64 | BPF_LSH | BPF_K:
	case BPF_ALU64 | BPF_RSH | BPF_K:
	case BPF_ALU64 | BPF_ARSH | BPF_K:
	case BPF_ALU64 | BPF_ADD | BPF_K:
	case BPF_ALU64 | BPF_SUB | BPF_K:
	case BPF_ALU64 | BPF_MUL | BPF_K:
	case BPF_ALU64 | BPF_DIV | BPF_K:
	case BPF_ALU64 | BPF_MOD | BPF_K:
		if (!valid_alu_i(BPF_OP(code), imm)) {
			emit_mov_i(ctx, MIPS_R_T4, imm);
			emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code));
		} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
			emit_alu_i64(ctx, dst, val, alu);
		}
		break;
	/* dst = dst & src (64-bit) */
	/* dst = dst | src (64-bit) */
	/* dst = dst ^ src (64-bit) */
	/* dst = dst << src (64-bit) */
	/* dst = dst >> src (64-bit) */
	/* dst = dst >> src (64-bit, arithmetic) */
	/* dst = dst + src (64-bit) */
	/* dst = dst - src (64-bit) */
	/* dst = dst * src (64-bit) */
	/* dst = dst / src (64-bit) */
	/* dst = dst % src (64-bit) */
	case BPF_ALU64 | BPF_AND | BPF_X:
	case BPF_ALU64 | BPF_OR | BPF_X:
	case BPF_ALU64 | BPF_XOR | BPF_X:
	case BPF_ALU64 | BPF_LSH | BPF_X:
	case BPF_ALU64 | BPF_RSH | BPF_X:
	case BPF_ALU64 | BPF_ARSH | BPF_X:
	case BPF_ALU64 | BPF_ADD | BPF_X:
	case BPF_ALU64 | BPF_SUB | BPF_X:
	case BPF_ALU64 | BPF_MUL | BPF_X:
	case BPF_ALU64 | BPF_DIV | BPF_X:
	case BPF_ALU64 | BPF_MOD | BPF_X:
		emit_alu_r64(ctx, dst, src, BPF_OP(code));
		break;
	/* dst = htole(dst) */
	/* dst = htobe(dst) */
	case BPF_ALU | BPF_END | BPF_FROM_LE:
	case BPF_ALU | BPF_END | BPF_FROM_BE:
		if (BPF_SRC(code) ==
#ifdef __BIG_ENDIAN
		    BPF_FROM_LE
#else
		    BPF_FROM_BE
#endif
		    )
			emit_bswap_r64(ctx, dst, imm);
		else
			emit_trunc_r64(ctx, dst, imm);
		break;
	/* dst = imm64 */
	case BPF_LD | BPF_IMM | BPF_DW:
		emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32));
		return 1;
	/* LDX: dst = *(size *)(src + off) */
	case BPF_LDX | BPF_MEM | BPF_W:
	case BPF_LDX | BPF_MEM | BPF_H:
	case BPF_LDX | BPF_MEM | BPF_B:
	case BPF_LDX | BPF_MEM | BPF_DW:
		emit_ldx(ctx, dst, src, off, BPF_SIZE(code));
		break;
	/* ST: *(size *)(dst + off) = imm */
	case BPF_ST | BPF_MEM | BPF_W:
	case BPF_ST | BPF_MEM | BPF_H:
	case BPF_ST | BPF_MEM | BPF_B:
	case BPF_ST | BPF_MEM | BPF_DW:
		emit_mov_i(ctx, MIPS_R_T4, imm);
		emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code));
		break;
	/* STX: *(size *)(dst + off) = src */
	case BPF_STX | BPF_MEM | BPF_W:
	case BPF_STX | BPF_MEM | BPF_H:
	case BPF_STX | BPF_MEM | BPF_B:
	case BPF_STX | BPF_MEM | BPF_DW:
		emit_stx(ctx, dst, src, off, BPF_SIZE(code));
		break;
	/* Speculation barrier */
	case BPF_ST | BPF_NOSPEC:
		break;
	/* Atomics */
	case BPF_STX | BPF_ATOMIC | BPF_W:
	case BPF_STX | BPF_ATOMIC | BPF_DW:
		switch (imm) {
		case BPF_ADD:
		case BPF_ADD | BPF_FETCH:
		case BPF_AND:
		case BPF_AND | BPF_FETCH:
		case BPF_OR:
		case BPF_OR | BPF_FETCH:
		case BPF_XOR:
		case BPF_XOR | BPF_FETCH:
		case BPF_XCHG:
			if (BPF_SIZE(code) == BPF_DW) {
				emit_atomic_r64(ctx, dst, src, off, imm);
			} else if (imm & BPF_FETCH) {
				u8 tmp = dst;

				if (src == dst) { /* Don't overwrite dst */
					emit_mov_r(ctx, MIPS_R_T4, dst);
					tmp = MIPS_R_T4;
				}
				emit_sext(ctx, src, src);
				emit_atomic_r(ctx, tmp, src, off, imm);
				emit_zext_ver(ctx, src);
			} else { /* 32-bit, no fetch */
				emit_sext(ctx, MIPS_R_T4, src);
				emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm);
			}
			break;
		case BPF_CMPXCHG:
			if (BPF_SIZE(code) == BPF_DW) {
				emit_cmpxchg_r64(ctx, dst, src, off);
			} else {
				u8 tmp = res;

				if (res == dst)   /* Don't overwrite dst */
					tmp = MIPS_R_T4;
				emit_sext(ctx, tmp, res);
				emit_sext(ctx, MIPS_R_T5, src);
				emit_cmpxchg_r(ctx, dst, MIPS_R_T5, tmp, off);
				if (res == dst)   /* Restore result */
					emit_mov_r(ctx, res, MIPS_R_T4);
				/* Result zext inserted by verifier */
			}
			break;
		default:
			goto notyet;
		}
		break;
	/* PC += off if dst == src */
	/* PC += off if dst != src */
	/* PC += off if dst & src */
	/* PC += off if dst > src */
	/* PC += off if dst >= src */
	/* PC += off if dst < src */
	/* PC += off if dst <= src */
	/* PC += off if dst > src (signed) */
	/* PC += off if dst >= src (signed) */
	/* PC += off if dst < src (signed) */
	/* PC += off if dst <= src (signed) */
	case BPF_JMP32 | BPF_JEQ | BPF_X:
	case BPF_JMP32 | BPF_JNE | BPF_X:
	case BPF_JMP32 | BPF_JSET | BPF_X:
	case BPF_JMP32 | BPF_JGT | BPF_X:
	case BPF_JMP32 | BPF_JGE | BPF_X:
	case BPF_JMP32 | BPF_JLT | BPF_X:
	case BPF_JMP32 | BPF_JLE | BPF_X:
	case BPF_JMP32 | BPF_JSGT | BPF_X:
	case BPF_JMP32 | BPF_JSGE | BPF_X:
	case BPF_JMP32 | BPF_JSLT | BPF_X:
	case BPF_JMP32 | BPF_JSLE | BPF_X:
		if (off == 0)
			break;
		setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
		emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */
		emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */
		emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp);
		if (finish_jmp(ctx, jmp, off) < 0)
			goto toofar;
		break;
	/* PC += off if dst == imm */
	/* PC += off if dst != imm */
	/* PC += off if dst & imm */
	/* PC += off if dst > imm */
	/* PC += off if dst >= imm */
	/* PC += off if dst < imm */
	/* PC += off if dst <= imm */
	/* PC += off if dst > imm (signed) */
	/* PC += off if dst >= imm (signed) */
	/* PC += off if dst < imm (signed) */
	/* PC += off if dst <= imm (signed) */
	case BPF_JMP32 | BPF_JEQ | BPF_K:
	case BPF_JMP32 | BPF_JNE | BPF_K:
	case BPF_JMP32 | BPF_JSET | BPF_K:
	case BPF_JMP32 | BPF_JGT | BPF_K:
	case BPF_JMP32 | BPF_JGE | BPF_K:
	case BPF_JMP32 | BPF_JLT | BPF_K:
	case BPF_JMP32 | BPF_JLE | BPF_K:
	case BPF_JMP32 | BPF_JSGT | BPF_K:
	case BPF_JMP32 | BPF_JSGE | BPF_K:
	case BPF_JMP32 | BPF_JSLT | BPF_K:
	case BPF_JMP32 | BPF_JSLE | BPF_K:
		if (off == 0)
			break;
		setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel);
		emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */
		if (valid_jmp_i(jmp, imm)) {
			emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp);
		} else {
			/* Move large immediate to register, sign-extended */
			emit_mov_i(ctx, MIPS_R_T5, imm);
			emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp);
		}
		if (finish_jmp(ctx, jmp, off) < 0)
			goto toofar;
		break;
	/* PC += off if dst == src */
	/* PC += off if dst != src */
	/* PC += off if dst & src */
	/* PC += off if dst > src */
	/* PC += off if dst >= src */
	/* PC += off if dst < src */
	/* PC += off if dst <= src */
	/* PC += off if dst > src (signed) */
	/* PC += off if dst >= src (signed) */
	/* PC += off if dst < src (signed) */
	/* PC += off if dst <= src (signed) */
	case BPF_JMP | BPF_JEQ | BPF_X:
	case BPF_JMP | BPF_JNE | BPF_X:
	case BPF_JMP | BPF_JSET | BPF_X:
	case BPF_JMP | BPF_JGT | BPF_X:
	case BPF_JMP | BPF_JGE | BPF_X:
	case BPF_JMP | BPF_JLT | BPF_X:
	case BPF_JMP | BPF_JLE | BPF_X:
	case BPF_JMP | BPF_JSGT | BPF_X:
	case BPF_JMP | BPF_JSGE | BPF_X:
	case BPF_JMP | BPF_JSLT | BPF_X:
	case BPF_JMP | BPF_JSLE | BPF_X:
		if (off == 0)
			break;
		setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
		emit_jmp_r(ctx, dst, src, rel, jmp);
		if (finish_jmp(ctx, jmp, off) < 0)
			goto toofar;
		break;
	/* PC += off if dst == imm */
	/* PC += off if dst != imm */
	/* PC += off if dst & imm */
	/* PC += off if dst > imm */
	/* PC += off if dst >= imm */
	/* PC += off if dst < imm */
	/* PC += off if dst <= imm */
	/* PC += off if dst > imm (signed) */
	/* PC += off if dst >= imm (signed) */
	/* PC += off if dst < imm (signed) */
	/* PC += off if dst <= imm (signed) */
	case BPF_JMP | BPF_JEQ | BPF_K:
	case BPF_JMP | BPF_JNE | BPF_K:
	case BPF_JMP | BPF_JSET | BPF_K:
	case BPF_JMP | BPF_JGT | BPF_K:
	case BPF_JMP | BPF_JGE | BPF_K:
	case BPF_JMP | BPF_JLT | BPF_K:
	case BPF_JMP | BPF_JLE | BPF_K:
	case BPF_JMP | BPF_JSGT | BPF_K:
	case BPF_JMP | BPF_JSGE | BPF_K:
	case BPF_JMP | BPF_JSLT | BPF_K:
	case BPF_JMP | BPF_JSLE | BPF_K:
		if (off == 0)
			break;
		setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel);
		if (valid_jmp_i(jmp, imm)) {
			emit_jmp_i(ctx, dst, imm, rel, jmp);
		} else {
			/* Move large immediate to register */
			emit_mov_i(ctx, MIPS_R_T4, imm);
			emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp);
		}
		if (finish_jmp(ctx, jmp, off) < 0)
			goto toofar;
		break;
	/* PC += off */
	case BPF_JMP | BPF_JA:
		if (off == 0)
			break;
		if (emit_ja(ctx, off) < 0)
			goto toofar;
		break;
	/* Tail call */
	case BPF_JMP | BPF_TAIL_CALL:
		if (emit_tail_call(ctx) < 0)
			goto invalid;
		break;
	/* Function call */
	case BPF_JMP | BPF_CALL:
		if (emit_call(ctx, insn) < 0)
			goto invalid;
		break;
	/* Function return */
	case BPF_JMP | BPF_EXIT:
		/*
		 * Optimization: when last instruction is EXIT
		 * simply continue to epilogue.
		 */
		if (ctx->bpf_index == ctx->program->len - 1)
			break;
		if (emit_exit(ctx) < 0)
			goto toofar;
		break;

	default:
invalid:
		pr_err_once("unknown opcode %02x\n", code);
		return -EINVAL;
notyet:
		pr_info_once("*** NOT YET: opcode %02x ***\n", code);
		return -EFAULT;
toofar:
		pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n",
			     ctx->bpf_index, code);
		return -E2BIG;
	}
	return 0;
}