in runtime/vm/compiler/backend/inliner.cc [3745:4259]
bool FlowGraphInliner::TryInlineRecognizedMethod(
FlowGraph* flow_graph,
intptr_t receiver_cid,
const Function& target,
Definition* call,
Definition* receiver,
const InstructionSource& source,
const ICData* ic_data,
GraphEntryInstr* graph_entry,
FunctionEntryInstr** entry,
Instruction** last,
Definition** result,
SpeculativeInliningPolicy* policy,
FlowGraphInliner::ExactnessInfo* exactness) {
COMPILER_TIMINGS_TIMER_SCOPE(flow_graph->thread(), InlineRecognizedMethod);
if (receiver_cid == kSentinelCid) {
// Receiver was defined in dead code and was replaced by the sentinel.
// Original receiver cid is lost, so don't try to inline recognized
// methods.
return false;
}
const bool can_speculate = policy->IsAllowedForInlining(call->deopt_id());
const bool is_dynamic_call = Function::IsDynamicInvocationForwarderName(
String::Handle(flow_graph->zone(), target.name()));
const MethodRecognizer::Kind kind = target.recognized_kind();
switch (kind) {
// Recognized [] operators.
case MethodRecognizer::kImmutableArrayGetIndexed:
case MethodRecognizer::kObjectArrayGetIndexed:
case MethodRecognizer::kGrowableArrayGetIndexed:
case MethodRecognizer::kInt8ArrayGetIndexed:
case MethodRecognizer::kUint8ArrayGetIndexed:
case MethodRecognizer::kUint8ClampedArrayGetIndexed:
case MethodRecognizer::kExternalUint8ArrayGetIndexed:
case MethodRecognizer::kExternalUint8ClampedArrayGetIndexed:
case MethodRecognizer::kInt16ArrayGetIndexed:
case MethodRecognizer::kUint16ArrayGetIndexed:
return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind,
call, receiver, graph_entry, entry, last, result);
case MethodRecognizer::kFloat32ArrayGetIndexed:
case MethodRecognizer::kFloat64ArrayGetIndexed:
if (!CanUnboxDouble()) {
return false;
}
return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind,
call, receiver, graph_entry, entry, last, result);
case MethodRecognizer::kFloat32x4ArrayGetIndexed:
case MethodRecognizer::kFloat64x2ArrayGetIndexed:
if (!ShouldInlineSimd()) {
return false;
}
return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind,
call, receiver, graph_entry, entry, last, result);
case MethodRecognizer::kInt32ArrayGetIndexed:
case MethodRecognizer::kUint32ArrayGetIndexed:
return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind,
call, receiver, graph_entry, entry, last, result);
case MethodRecognizer::kInt64ArrayGetIndexed:
case MethodRecognizer::kUint64ArrayGetIndexed:
return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind,
call, receiver, graph_entry, entry, last, result);
case MethodRecognizer::kClassIDgetID:
return InlineLoadClassId(flow_graph, call, graph_entry, entry, last,
result);
default:
break;
}
// The following ones need to speculate.
if (!can_speculate) {
return false;
}
switch (kind) {
// Recognized []= operators.
case MethodRecognizer::kObjectArraySetIndexed:
case MethodRecognizer::kGrowableArraySetIndexed:
case MethodRecognizer::kObjectArraySetIndexedUnchecked:
case MethodRecognizer::kGrowableArraySetIndexedUnchecked:
return InlineSetIndexed(flow_graph, kind, target, call, receiver, source,
/* value_check = */ NULL, exactness, graph_entry,
entry, last, result);
case MethodRecognizer::kInt8ArraySetIndexed:
case MethodRecognizer::kUint8ArraySetIndexed:
case MethodRecognizer::kUint8ClampedArraySetIndexed:
case MethodRecognizer::kExternalUint8ArraySetIndexed:
case MethodRecognizer::kExternalUint8ClampedArraySetIndexed:
case MethodRecognizer::kInt16ArraySetIndexed:
case MethodRecognizer::kUint16ArraySetIndexed: {
// Optimistically assume Smi.
if (ic_data != NULL && ic_data->HasDeoptReason(ICData::kDeoptCheckSmi)) {
// Optimistic assumption failed at least once.
return false;
}
Cids* value_check = Cids::CreateMonomorphic(Z, kSmiCid);
return InlineSetIndexed(flow_graph, kind, target, call, receiver, source,
value_check, exactness, graph_entry, entry, last,
result);
}
case MethodRecognizer::kInt32ArraySetIndexed:
case MethodRecognizer::kUint32ArraySetIndexed: {
// Value check not needed for Int32 and Uint32 arrays because they
// implicitly contain unboxing instructions which check for right type.
return InlineSetIndexed(flow_graph, kind, target, call, receiver, source,
/* value_check = */ NULL, exactness, graph_entry,
entry, last, result);
}
case MethodRecognizer::kInt64ArraySetIndexed:
case MethodRecognizer::kUint64ArraySetIndexed:
return InlineSetIndexed(flow_graph, kind, target, call, receiver, source,
/* value_check = */ NULL, exactness, graph_entry,
entry, last, result);
case MethodRecognizer::kFloat32ArraySetIndexed:
case MethodRecognizer::kFloat64ArraySetIndexed: {
if (!CanUnboxDouble()) {
return false;
}
Cids* value_check = Cids::CreateMonomorphic(Z, kDoubleCid);
return InlineSetIndexed(flow_graph, kind, target, call, receiver, source,
value_check, exactness, graph_entry, entry, last,
result);
}
case MethodRecognizer::kFloat32x4ArraySetIndexed: {
if (!ShouldInlineSimd()) {
return false;
}
Cids* value_check = Cids::CreateMonomorphic(Z, kFloat32x4Cid);
return InlineSetIndexed(flow_graph, kind, target, call, receiver, source,
value_check, exactness, graph_entry, entry, last,
result);
}
case MethodRecognizer::kFloat64x2ArraySetIndexed: {
if (!ShouldInlineSimd()) {
return false;
}
Cids* value_check = Cids::CreateMonomorphic(Z, kFloat64x2Cid);
return InlineSetIndexed(flow_graph, kind, target, call, receiver, source,
value_check, exactness, graph_entry, entry, last,
result);
}
case MethodRecognizer::kByteArrayBaseGetInt8:
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataInt8ArrayCid, graph_entry, entry,
last, result);
case MethodRecognizer::kByteArrayBaseGetUint8:
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataUint8ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseGetInt16:
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataInt16ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseGetUint16:
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataUint16ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseGetInt32:
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataInt32ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseGetUint32:
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataUint32ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseGetInt64:
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataInt64ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseGetUint64:
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataUint64ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseGetFloat32:
if (!CanUnboxDouble()) {
return false;
}
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataFloat32ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseGetFloat64:
if (!CanUnboxDouble()) {
return false;
}
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataFloat64ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseGetFloat32x4:
if (!ShouldInlineSimd()) {
return false;
}
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataFloat32x4ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseGetInt32x4:
if (!ShouldInlineSimd()) {
return false;
}
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid,
kTypedDataInt32x4ArrayCid, graph_entry,
entry, last, result);
case MethodRecognizer::kByteArrayBaseSetInt8:
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataInt8ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetUint8:
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataUint8ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetInt16:
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataInt16ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetUint16:
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataUint16ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetInt32:
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataInt32ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetUint32:
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataUint32ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetInt64:
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataInt64ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetUint64:
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataUint64ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetFloat32:
if (!CanUnboxDouble()) {
return false;
}
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataFloat32ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetFloat64:
if (!CanUnboxDouble()) {
return false;
}
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataFloat64ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetFloat32x4:
if (!ShouldInlineSimd()) {
return false;
}
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataFloat32x4ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kByteArrayBaseSetInt32x4:
if (!ShouldInlineSimd()) {
return false;
}
return InlineByteArrayBaseStore(flow_graph, target, call, receiver,
receiver_cid, kTypedDataInt32x4ArrayCid,
graph_entry, entry, last, result);
case MethodRecognizer::kOneByteStringCodeUnitAt:
case MethodRecognizer::kTwoByteStringCodeUnitAt:
case MethodRecognizer::kExternalOneByteStringCodeUnitAt:
case MethodRecognizer::kExternalTwoByteStringCodeUnitAt:
return InlineStringCodeUnitAt(flow_graph, call, receiver, receiver_cid,
graph_entry, entry, last, result);
case MethodRecognizer::kStringBaseCharAt:
return InlineStringBaseCharAt(flow_graph, call, receiver, receiver_cid,
graph_entry, entry, last, result);
case MethodRecognizer::kDoubleAdd:
return InlineDoubleOp(flow_graph, Token::kADD, call, receiver,
graph_entry, entry, last, result);
case MethodRecognizer::kDoubleSub:
return InlineDoubleOp(flow_graph, Token::kSUB, call, receiver,
graph_entry, entry, last, result);
case MethodRecognizer::kDoubleMul:
return InlineDoubleOp(flow_graph, Token::kMUL, call, receiver,
graph_entry, entry, last, result);
case MethodRecognizer::kDoubleDiv:
return InlineDoubleOp(flow_graph, Token::kDIV, call, receiver,
graph_entry, entry, last, result);
case MethodRecognizer::kDouble_getIsNaN:
case MethodRecognizer::kDouble_getIsInfinite:
return InlineDoubleTestOp(flow_graph, call, receiver, kind, graph_entry,
entry, last, result);
case MethodRecognizer::kGrowableArraySetData:
ASSERT((receiver_cid == kGrowableObjectArrayCid) ||
((receiver_cid == kDynamicCid) && call->IsStaticCall()));
return InlineGrowableArraySetter(
flow_graph, Slot::GrowableObjectArray_data(), kEmitStoreBarrier, call,
receiver, graph_entry, entry, last, result);
case MethodRecognizer::kGrowableArraySetLength:
ASSERT((receiver_cid == kGrowableObjectArrayCid) ||
((receiver_cid == kDynamicCid) && call->IsStaticCall()));
return InlineGrowableArraySetter(
flow_graph, Slot::GrowableObjectArray_length(), kNoStoreBarrier, call,
receiver, graph_entry, entry, last, result);
case MethodRecognizer::kFloat32x4Abs:
case MethodRecognizer::kFloat32x4Clamp:
case MethodRecognizer::kFloat32x4FromDoubles:
case MethodRecognizer::kFloat32x4Equal:
case MethodRecognizer::kFloat32x4GetSignMask:
case MethodRecognizer::kFloat32x4GreaterThan:
case MethodRecognizer::kFloat32x4GreaterThanOrEqual:
case MethodRecognizer::kFloat32x4LessThan:
case MethodRecognizer::kFloat32x4LessThanOrEqual:
case MethodRecognizer::kFloat32x4Max:
case MethodRecognizer::kFloat32x4Min:
case MethodRecognizer::kFloat32x4Negate:
case MethodRecognizer::kFloat32x4NotEqual:
case MethodRecognizer::kFloat32x4Reciprocal:
case MethodRecognizer::kFloat32x4ReciprocalSqrt:
case MethodRecognizer::kFloat32x4Scale:
case MethodRecognizer::kFloat32x4ShuffleW:
case MethodRecognizer::kFloat32x4ShuffleX:
case MethodRecognizer::kFloat32x4ShuffleY:
case MethodRecognizer::kFloat32x4ShuffleZ:
case MethodRecognizer::kFloat32x4Splat:
case MethodRecognizer::kFloat32x4Sqrt:
case MethodRecognizer::kFloat32x4ToFloat64x2:
case MethodRecognizer::kFloat32x4ToInt32x4:
case MethodRecognizer::kFloat32x4WithW:
case MethodRecognizer::kFloat32x4WithX:
case MethodRecognizer::kFloat32x4WithY:
case MethodRecognizer::kFloat32x4WithZ:
case MethodRecognizer::kFloat32x4Zero:
case MethodRecognizer::kFloat64x2Abs:
case MethodRecognizer::kFloat64x2FromDoubles:
case MethodRecognizer::kFloat64x2GetSignMask:
case MethodRecognizer::kFloat64x2GetX:
case MethodRecognizer::kFloat64x2GetY:
case MethodRecognizer::kFloat64x2Max:
case MethodRecognizer::kFloat64x2Min:
case MethodRecognizer::kFloat64x2Negate:
case MethodRecognizer::kFloat64x2Scale:
case MethodRecognizer::kFloat64x2Splat:
case MethodRecognizer::kFloat64x2Sqrt:
case MethodRecognizer::kFloat64x2ToFloat32x4:
case MethodRecognizer::kFloat64x2WithX:
case MethodRecognizer::kFloat64x2WithY:
case MethodRecognizer::kFloat64x2Zero:
case MethodRecognizer::kInt32x4FromBools:
case MethodRecognizer::kInt32x4FromInts:
case MethodRecognizer::kInt32x4GetFlagW:
case MethodRecognizer::kInt32x4GetFlagX:
case MethodRecognizer::kInt32x4GetFlagY:
case MethodRecognizer::kInt32x4GetFlagZ:
case MethodRecognizer::kInt32x4GetSignMask:
case MethodRecognizer::kInt32x4Select:
case MethodRecognizer::kInt32x4ToFloat32x4:
case MethodRecognizer::kInt32x4WithFlagW:
case MethodRecognizer::kInt32x4WithFlagX:
case MethodRecognizer::kInt32x4WithFlagY:
case MethodRecognizer::kInt32x4WithFlagZ:
case MethodRecognizer::kFloat32x4ShuffleMix:
case MethodRecognizer::kInt32x4ShuffleMix:
case MethodRecognizer::kFloat32x4Shuffle:
case MethodRecognizer::kInt32x4Shuffle:
case MethodRecognizer::kFloat32x4Mul:
case MethodRecognizer::kFloat32x4Div:
case MethodRecognizer::kFloat32x4Add:
case MethodRecognizer::kFloat32x4Sub:
case MethodRecognizer::kFloat64x2Mul:
case MethodRecognizer::kFloat64x2Div:
case MethodRecognizer::kFloat64x2Add:
case MethodRecognizer::kFloat64x2Sub:
return InlineSimdOp(flow_graph, is_dynamic_call, call, receiver, kind,
graph_entry, entry, last, result);
case MethodRecognizer::kMathIntPow:
return InlineMathIntPow(flow_graph, call, graph_entry, entry, last,
result);
case MethodRecognizer::kObjectConstructor: {
*entry = new (Z)
FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
call->GetBlock()->try_index(), DeoptId::kNone);
(*entry)->InheritDeoptTarget(Z, call);
ASSERT(!call->HasUses());
*last = NULL; // Empty body.
*result = NULL; // Since no uses of original call, result will be unused.
return true;
}
case MethodRecognizer::kListFactory: {
// We only want to inline new List(n) which decreases code size and
// improves performance. We don't want to inline new List().
if (call->ArgumentCount() != 2) {
return false;
}
const auto type = new (Z) Value(call->ArgumentAt(0));
const auto num_elements = new (Z) Value(call->ArgumentAt(1));
*entry = new (Z)
FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
call->GetBlock()->try_index(), DeoptId::kNone);
(*entry)->InheritDeoptTarget(Z, call);
*last = new (Z) CreateArrayInstr(call->source(), type, num_elements,
call->deopt_id());
flow_graph->AppendTo(
*entry, *last,
call->deopt_id() != DeoptId::kNone ? call->env() : NULL,
FlowGraph::kValue);
*result = (*last)->AsDefinition();
return true;
}
case MethodRecognizer::kObjectArrayAllocate: {
Value* num_elements = new (Z) Value(call->ArgumentAt(1));
intptr_t length = 0;
if (IsSmiValue(num_elements, &length)) {
if (Array::IsValidLength(length)) {
Value* type = new (Z) Value(call->ArgumentAt(0));
*entry = new (Z)
FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
call->GetBlock()->try_index(), DeoptId::kNone);
(*entry)->InheritDeoptTarget(Z, call);
*last = new (Z) CreateArrayInstr(call->source(), type, num_elements,
call->deopt_id());
flow_graph->AppendTo(
*entry, *last,
call->deopt_id() != DeoptId::kNone ? call->env() : NULL,
FlowGraph::kValue);
*result = (*last)->AsDefinition();
return true;
}
}
return false;
}
case MethodRecognizer::kObjectRuntimeType: {
Type& type = Type::ZoneHandle(Z);
if (receiver_cid == kDynamicCid) {
return false;
} else if (IsStringClassId(receiver_cid)) {
type = Type::StringType();
} else if (receiver_cid == kDoubleCid) {
type = Type::Double();
} else if (IsIntegerClassId(receiver_cid)) {
type = Type::IntType();
} else if (IsTypeClassId(receiver_cid)) {
type = Type::DartTypeType();
} else if (receiver_cid != kClosureCid) {
const Class& cls = Class::Handle(
Z, flow_graph->isolate_group()->class_table()->At(receiver_cid));
if (!cls.IsGeneric()) {
type = cls.DeclarationType();
}
}
if (!type.IsNull()) {
*entry = new (Z)
FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
call->GetBlock()->try_index(), DeoptId::kNone);
(*entry)->InheritDeoptTarget(Z, call);
ConstantInstr* ctype = flow_graph->GetConstant(type);
// Create a synthetic (re)definition for return to flag insertion.
// TODO(ajcbik): avoid this mechanism altogether
RedefinitionInstr* redef =
new (Z) RedefinitionInstr(new (Z) Value(ctype));
flow_graph->AppendTo(
*entry, redef,
call->deopt_id() != DeoptId::kNone ? call->env() : NULL,
FlowGraph::kValue);
*last = *result = redef;
return true;
}
return false;
}
case MethodRecognizer::kWriteIntoOneByteString:
case MethodRecognizer::kWriteIntoTwoByteString: {
// This is an internal method, no need to check argument types nor
// range.
*entry = new (Z)
FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(),
call->GetBlock()->try_index(), DeoptId::kNone);
(*entry)->InheritDeoptTarget(Z, call);
Definition* str = call->ArgumentAt(0);
Definition* index = call->ArgumentAt(1);
Definition* value = call->ArgumentAt(2);
auto env = call->deopt_id() != DeoptId::kNone ? call->env() : nullptr;
// Insert explicit unboxing instructions with truncation to avoid relying
// on [SelectRepresentations] which doesn't mark them as truncating.
value =
UnboxInstr::Create(kUnboxedIntPtr, new (Z) Value(value),
call->deopt_id(), Instruction::kNotSpeculative);
value->AsUnboxInteger()->mark_truncating();
flow_graph->AppendTo(*entry, value, env, FlowGraph::kValue);
const bool is_onebyte = kind == MethodRecognizer::kWriteIntoOneByteString;
const intptr_t index_scale = is_onebyte ? 1 : 2;
const intptr_t cid = is_onebyte ? kOneByteStringCid : kTwoByteStringCid;
*last = new (Z) StoreIndexedInstr(
new (Z) Value(str), new (Z) Value(index), new (Z) Value(value),
kNoStoreBarrier, /*index_unboxed=*/false, index_scale, cid,
kAlignedAccess, call->deopt_id(), call->source());
flow_graph->AppendTo(value, *last, env, FlowGraph::kEffect);
// We need a return value to replace uses of the original definition.
// The final instruction is a use of 'void operator[]=()', so we use null.
*result = flow_graph->constant_null();
return true;
}
default:
return false;
}
}