in tcg/i386/tcg-target.c.inc [2178:2630]
static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg args[TCG_MAX_OP_ARGS],
const int const_args[TCG_MAX_OP_ARGS])
{
TCGArg a0, a1, a2;
int c, const_a2, vexop, rexw = 0;
#if TCG_TARGET_REG_BITS == 64
# define OP_32_64(x) \
case glue(glue(INDEX_op_, x), _i64): \
rexw = P_REXW; /* FALLTHRU */ \
case glue(glue(INDEX_op_, x), _i32)
#else
# define OP_32_64(x) \
case glue(glue(INDEX_op_, x), _i32)
#endif
/* Hoist the loads of the most common arguments. */
a0 = args[0];
a1 = args[1];
a2 = args[2];
const_a2 = const_args[2];
switch (opc) {
case INDEX_op_exit_tb:
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
tcg_out_jmp(s, tcg_code_gen_epilogue);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
tcg_out_jmp(s, tb_ret_addr);
}
break;
case INDEX_op_goto_tb:
if (s->tb_jmp_insn_offset) {
/* direct jump method */
int gap;
/* jump displacement must be aligned for atomic patching;
* see if we need to add extra nops before jump
*/
gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
if (gap != 1) {
tcg_out_nopn(s, gap - 1);
}
tcg_out8(s, OPC_JMP_long); /* jmp im */
s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
tcg_out32(s, 0);
} else {
/* indirect jump method */
tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
(intptr_t)(s->tb_jmp_target_addr + a0));
}
set_jmp_reset_offset(s, a0);
break;
case INDEX_op_goto_ptr:
/* jmp to the given host address (could be epilogue) */
tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0);
break;
case INDEX_op_br:
tcg_out_jxx(s, JCC_JMP, arg_label(a0), 0);
break;
OP_32_64(ld8u):
/* Note that we can ignore REXW for the zero-extend to 64-bit. */
tcg_out_modrm_offset(s, OPC_MOVZBL, a0, a1, a2);
break;
OP_32_64(ld8s):
tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, a0, a1, a2);
break;
OP_32_64(ld16u):
/* Note that we can ignore REXW for the zero-extend to 64-bit. */
tcg_out_modrm_offset(s, OPC_MOVZWL, a0, a1, a2);
break;
OP_32_64(ld16s):
tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, a0, a1, a2);
break;
#if TCG_TARGET_REG_BITS == 64
case INDEX_op_ld32u_i64:
#endif
case INDEX_op_ld_i32:
tcg_out_ld(s, TCG_TYPE_I32, a0, a1, a2);
break;
OP_32_64(st8):
if (const_args[0]) {
tcg_out_modrm_offset(s, OPC_MOVB_EvIz, 0, a1, a2);
tcg_out8(s, a0);
} else {
tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R, a0, a1, a2);
}
break;
OP_32_64(st16):
if (const_args[0]) {
tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16, 0, a1, a2);
tcg_out16(s, a0);
} else {
tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16, a0, a1, a2);
}
break;
#if TCG_TARGET_REG_BITS == 64
case INDEX_op_st32_i64:
#endif
case INDEX_op_st_i32:
if (const_args[0]) {
tcg_out_modrm_offset(s, OPC_MOVL_EvIz, 0, a1, a2);
tcg_out32(s, a0);
} else {
tcg_out_st(s, TCG_TYPE_I32, a0, a1, a2);
}
break;
OP_32_64(add):
/* For 3-operand addition, use LEA. */
if (a0 != a1) {
TCGArg c3 = 0;
if (const_a2) {
c3 = a2, a2 = -1;
} else if (a0 == a2) {
/* Watch out for dest = src + dest, since we've removed
the matching constraint on the add. */
tgen_arithr(s, ARITH_ADD + rexw, a0, a1);
break;
}
tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, c3);
break;
}
c = ARITH_ADD;
goto gen_arith;
OP_32_64(sub):
c = ARITH_SUB;
goto gen_arith;
OP_32_64(and):
c = ARITH_AND;
goto gen_arith;
OP_32_64(or):
c = ARITH_OR;
goto gen_arith;
OP_32_64(xor):
c = ARITH_XOR;
goto gen_arith;
gen_arith:
if (const_a2) {
tgen_arithi(s, c + rexw, a0, a2, 0);
} else {
tgen_arithr(s, c + rexw, a0, a2);
}
break;
OP_32_64(andc):
if (const_a2) {
tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1);
tgen_arithi(s, ARITH_AND + rexw, a0, ~a2, 0);
} else {
tcg_out_vex_modrm(s, OPC_ANDN + rexw, a0, a2, a1);
}
break;
OP_32_64(mul):
if (const_a2) {
int32_t val;
val = a2;
if (val == (int8_t)val) {
tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, a0, a0);
tcg_out8(s, val);
} else {
tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, a0, a0);
tcg_out32(s, val);
}
} else {
tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, a0, a2);
}
break;
OP_32_64(div2):
tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, args[4]);
break;
OP_32_64(divu2):
tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, args[4]);
break;
OP_32_64(shl):
/* For small constant 3-operand shift, use LEA. */
if (const_a2 && a0 != a1 && (a2 - 1) < 3) {
if (a2 - 1 == 0) {
/* shl $1,a1,a0 -> lea (a1,a1),a0 */
tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a1, 0, 0);
} else {
/* shl $n,a1,a0 -> lea 0(,a1,n),a0 */
tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, -1, a1, a2, 0);
}
break;
}
c = SHIFT_SHL;
vexop = OPC_SHLX;
goto gen_shift_maybe_vex;
OP_32_64(shr):
c = SHIFT_SHR;
vexop = OPC_SHRX;
goto gen_shift_maybe_vex;
OP_32_64(sar):
c = SHIFT_SAR;
vexop = OPC_SARX;
goto gen_shift_maybe_vex;
OP_32_64(rotl):
c = SHIFT_ROL;
goto gen_shift;
OP_32_64(rotr):
c = SHIFT_ROR;
goto gen_shift;
gen_shift_maybe_vex:
if (have_bmi2) {
if (!const_a2) {
tcg_out_vex_modrm(s, vexop + rexw, a0, a2, a1);
break;
}
tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1);
}
/* FALLTHRU */
gen_shift:
if (const_a2) {
tcg_out_shifti(s, c + rexw, a0, a2);
} else {
tcg_out_modrm(s, OPC_SHIFT_cl + rexw, c, a0);
}
break;
OP_32_64(ctz):
tcg_out_ctz(s, rexw, args[0], args[1], args[2], const_args[2]);
break;
OP_32_64(clz):
tcg_out_clz(s, rexw, args[0], args[1], args[2], const_args[2]);
break;
OP_32_64(ctpop):
tcg_out_modrm(s, OPC_POPCNT + rexw, a0, a1);
break;
case INDEX_op_brcond_i32:
tcg_out_brcond32(s, a2, a0, a1, const_args[1], arg_label(args[3]), 0);
break;
case INDEX_op_setcond_i32:
tcg_out_setcond32(s, args[3], a0, a1, a2, const_a2);
break;
case INDEX_op_movcond_i32:
tcg_out_movcond32(s, args[5], a0, a1, a2, const_a2, args[3]);
break;
OP_32_64(bswap16):
if (a2 & TCG_BSWAP_OS) {
/* Output must be sign-extended. */
if (rexw) {
tcg_out_bswap64(s, a0);
tcg_out_shifti(s, SHIFT_SAR + rexw, a0, 48);
} else {
tcg_out_bswap32(s, a0);
tcg_out_shifti(s, SHIFT_SAR, a0, 16);
}
} else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
/* Output must be zero-extended, but input isn't. */
tcg_out_bswap32(s, a0);
tcg_out_shifti(s, SHIFT_SHR, a0, 16);
} else {
tcg_out_rolw_8(s, a0);
}
break;
OP_32_64(bswap32):
tcg_out_bswap32(s, a0);
if (rexw && (a2 & TCG_BSWAP_OS)) {
tcg_out_ext32s(s, a0, a0);
}
break;
OP_32_64(neg):
tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, a0);
break;
OP_32_64(not):
tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0);
break;
OP_32_64(ext8s):
tcg_out_ext8s(s, a0, a1, rexw);
break;
OP_32_64(ext16s):
tcg_out_ext16s(s, a0, a1, rexw);
break;
OP_32_64(ext8u):
tcg_out_ext8u(s, a0, a1);
break;
OP_32_64(ext16u):
tcg_out_ext16u(s, a0, a1);
break;
case INDEX_op_qemu_ld_i32:
tcg_out_qemu_ld(s, args, 0);
break;
case INDEX_op_qemu_ld_i64:
tcg_out_qemu_ld(s, args, 1);
break;
case INDEX_op_qemu_st_i32:
case INDEX_op_qemu_st8_i32:
tcg_out_qemu_st(s, args, 0);
break;
case INDEX_op_qemu_st_i64:
tcg_out_qemu_st(s, args, 1);
break;
OP_32_64(mulu2):
tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, args[3]);
break;
OP_32_64(muls2):
tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IMUL, args[3]);
break;
OP_32_64(add2):
if (const_args[4]) {
tgen_arithi(s, ARITH_ADD + rexw, a0, args[4], 1);
} else {
tgen_arithr(s, ARITH_ADD + rexw, a0, args[4]);
}
if (const_args[5]) {
tgen_arithi(s, ARITH_ADC + rexw, a1, args[5], 1);
} else {
tgen_arithr(s, ARITH_ADC + rexw, a1, args[5]);
}
break;
OP_32_64(sub2):
if (const_args[4]) {
tgen_arithi(s, ARITH_SUB + rexw, a0, args[4], 1);
} else {
tgen_arithr(s, ARITH_SUB + rexw, a0, args[4]);
}
if (const_args[5]) {
tgen_arithi(s, ARITH_SBB + rexw, a1, args[5], 1);
} else {
tgen_arithr(s, ARITH_SBB + rexw, a1, args[5]);
}
break;
#if TCG_TARGET_REG_BITS == 32
case INDEX_op_brcond2_i32:
tcg_out_brcond2(s, args, const_args, 0);
break;
case INDEX_op_setcond2_i32:
tcg_out_setcond2(s, args, const_args);
break;
#else /* TCG_TARGET_REG_BITS == 64 */
case INDEX_op_ld32s_i64:
tcg_out_modrm_offset(s, OPC_MOVSLQ, a0, a1, a2);
break;
case INDEX_op_ld_i64:
tcg_out_ld(s, TCG_TYPE_I64, a0, a1, a2);
break;
case INDEX_op_st_i64:
if (const_args[0]) {
tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_REXW, 0, a1, a2);
tcg_out32(s, a0);
} else {
tcg_out_st(s, TCG_TYPE_I64, a0, a1, a2);
}
break;
case INDEX_op_brcond_i64:
tcg_out_brcond64(s, a2, a0, a1, const_args[1], arg_label(args[3]), 0);
break;
case INDEX_op_setcond_i64:
tcg_out_setcond64(s, args[3], a0, a1, a2, const_a2);
break;
case INDEX_op_movcond_i64:
tcg_out_movcond64(s, args[5], a0, a1, a2, const_a2, args[3]);
break;
case INDEX_op_bswap64_i64:
tcg_out_bswap64(s, a0);
break;
case INDEX_op_extu_i32_i64:
case INDEX_op_ext32u_i64:
case INDEX_op_extrl_i64_i32:
tcg_out_ext32u(s, a0, a1);
break;
case INDEX_op_ext_i32_i64:
case INDEX_op_ext32s_i64:
tcg_out_ext32s(s, a0, a1);
break;
case INDEX_op_extrh_i64_i32:
tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32);
break;
#endif
OP_32_64(deposit):
if (args[3] == 0 && args[4] == 8) {
/* load bits 0..7 */
tcg_out_modrm(s, OPC_MOVB_EvGv | P_REXB_R | P_REXB_RM, a2, a0);
} else if (args[3] == 8 && args[4] == 8) {
/* load bits 8..15 */
tcg_out_modrm(s, OPC_MOVB_EvGv, a2, a0 + 4);
} else if (args[3] == 0 && args[4] == 16) {
/* load bits 0..15 */
tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, a2, a0);
} else {
tcg_abort();
}
break;
case INDEX_op_extract_i64:
if (a2 + args[3] == 32) {
/* This is a 32-bit zero-extending right shift. */
tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
tcg_out_shifti(s, SHIFT_SHR, a0, a2);
break;
}
/* FALLTHRU */
case INDEX_op_extract_i32:
/* On the off-chance that we can use the high-byte registers.
Otherwise we emit the same ext16 + shift pattern that we
would have gotten from the normal tcg-op.c expansion. */
tcg_debug_assert(a2 == 8 && args[3] == 8);
if (a1 < 4 && a0 < 8) {
tcg_out_modrm(s, OPC_MOVZBL, a0, a1 + 4);
} else {
tcg_out_ext16u(s, a0, a1);
tcg_out_shifti(s, SHIFT_SHR, a0, 8);
}
break;
case INDEX_op_sextract_i32:
/* We don't implement sextract_i64, as we cannot sign-extend to
64-bits without using the REX prefix that explicitly excludes
access to the high-byte registers. */
tcg_debug_assert(a2 == 8 && args[3] == 8);
if (a1 < 4 && a0 < 8) {
tcg_out_modrm(s, OPC_MOVSBL, a0, a1 + 4);
} else {
tcg_out_ext16s(s, a0, a1, 0);
tcg_out_shifti(s, SHIFT_SAR, a0, 8);
}
break;
OP_32_64(extract2):
/* Note that SHRD outputs to the r/m operand. */
tcg_out_modrm(s, OPC_SHRD_Ib + rexw, a2, a0);
tcg_out8(s, args[3]);
break;
case INDEX_op_mb:
tcg_out_mb(s, a0);
break;
case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
case INDEX_op_mov_i64:
case INDEX_op_call: /* Always emitted via tcg_out_call. */
default:
tcg_abort();
}
#undef OP_32_64
}