in compiler/optimizing/intrinsics_x86_64.cc [1132:1357]
void IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopy(HInvoke* invoke) {
X86_64Assembler* assembler = GetAssembler();
LocationSummary* locations = invoke->GetLocations();
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
CpuRegister src = locations->InAt(0).AsRegister<CpuRegister>();
Location src_pos = locations->InAt(1);
CpuRegister dest = locations->InAt(2).AsRegister<CpuRegister>();
Location dest_pos = locations->InAt(3);
Location length = locations->InAt(4);
CpuRegister temp1 = locations->GetTemp(0).AsRegister<CpuRegister>();
CpuRegister temp2 = locations->GetTemp(1).AsRegister<CpuRegister>();
CpuRegister temp3 = locations->GetTemp(2).AsRegister<CpuRegister>();
SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86_64(invoke);
codegen_->AddSlowPath(slow_path);
NearLabel conditions_on_positions_validated;
SystemArrayCopyOptimizations optimizations(invoke);
// If source and destination are the same, we go to slow path if we need to do
// forward copying.
if (src_pos.IsConstant()) {
int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
if (dest_pos.IsConstant()) {
int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
if (optimizations.GetDestinationIsSource()) {
// Checked when building locations.
DCHECK_GE(src_pos_constant, dest_pos_constant);
} else if (src_pos_constant < dest_pos_constant) {
__ cmpl(src, dest);
__ j(kEqual, slow_path->GetEntryLabel());
}
} else {
if (!optimizations.GetDestinationIsSource()) {
__ cmpl(src, dest);
__ j(kNotEqual, &conditions_on_positions_validated);
}
__ cmpl(dest_pos.AsRegister<CpuRegister>(), Immediate(src_pos_constant));
__ j(kGreater, slow_path->GetEntryLabel());
}
} else {
if (!optimizations.GetDestinationIsSource()) {
__ cmpl(src, dest);
__ j(kNotEqual, &conditions_on_positions_validated);
}
if (dest_pos.IsConstant()) {
int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
__ cmpl(src_pos.AsRegister<CpuRegister>(), Immediate(dest_pos_constant));
__ j(kLess, slow_path->GetEntryLabel());
} else {
__ cmpl(src_pos.AsRegister<CpuRegister>(), dest_pos.AsRegister<CpuRegister>());
__ j(kLess, slow_path->GetEntryLabel());
}
}
__ Bind(&conditions_on_positions_validated);
if (!optimizations.GetSourceIsNotNull()) {
// Bail out if the source is null.
__ testl(src, src);
__ j(kEqual, slow_path->GetEntryLabel());
}
if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) {
// Bail out if the destination is null.
__ testl(dest, dest);
__ j(kEqual, slow_path->GetEntryLabel());
}
// If the length is negative, bail out.
// We have already checked in the LocationsBuilder for the constant case.
if (!length.IsConstant() &&
!optimizations.GetCountIsSourceLength() &&
!optimizations.GetCountIsDestinationLength()) {
__ testl(length.AsRegister<CpuRegister>(), length.AsRegister<CpuRegister>());
__ j(kLess, slow_path->GetEntryLabel());
}
// Validity checks: source.
CheckPosition(assembler,
src_pos,
src,
length,
slow_path,
temp1,
temp2,
optimizations.GetCountIsSourceLength());
// Validity checks: dest.
CheckPosition(assembler,
dest_pos,
dest,
length,
slow_path,
temp1,
temp2,
optimizations.GetCountIsDestinationLength());
if (!optimizations.GetDoesNotNeedTypeCheck()) {
// Check whether all elements of the source array are assignable to the component
// type of the destination array. We do two checks: the classes are the same,
// or the destination is Object[]. If none of these checks succeed, we go to the
// slow path.
__ movl(temp1, Address(dest, class_offset));
__ movl(temp2, Address(src, class_offset));
bool did_unpoison = false;
if (!optimizations.GetDestinationIsNonPrimitiveArray() ||
!optimizations.GetSourceIsNonPrimitiveArray()) {
// One or two of the references need to be unpoisoned. Unpoison them
// both to make the identity check valid.
__ MaybeUnpoisonHeapReference(temp1);
__ MaybeUnpoisonHeapReference(temp2);
did_unpoison = true;
}
if (!optimizations.GetDestinationIsNonPrimitiveArray()) {
// Bail out if the destination is not a non primitive array.
// /* HeapReference<Class> */ TMP = temp1->component_type_
__ movl(CpuRegister(TMP), Address(temp1, component_offset));
__ testl(CpuRegister(TMP), CpuRegister(TMP));
__ j(kEqual, slow_path->GetEntryLabel());
__ MaybeUnpoisonHeapReference(CpuRegister(TMP));
__ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot));
__ j(kNotEqual, slow_path->GetEntryLabel());
}
if (!optimizations.GetSourceIsNonPrimitiveArray()) {
// Bail out if the source is not a non primitive array.
// /* HeapReference<Class> */ TMP = temp2->component_type_
__ movl(CpuRegister(TMP), Address(temp2, component_offset));
__ testl(CpuRegister(TMP), CpuRegister(TMP));
__ j(kEqual, slow_path->GetEntryLabel());
__ MaybeUnpoisonHeapReference(CpuRegister(TMP));
__ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot));
__ j(kNotEqual, slow_path->GetEntryLabel());
}
__ cmpl(temp1, temp2);
if (optimizations.GetDestinationIsTypedObjectArray()) {
NearLabel do_copy;
__ j(kEqual, &do_copy);
if (!did_unpoison) {
__ MaybeUnpoisonHeapReference(temp1);
}
// /* HeapReference<Class> */ temp1 = temp1->component_type_
__ movl(temp1, Address(temp1, component_offset));
__ MaybeUnpoisonHeapReference(temp1);
// /* HeapReference<Class> */ temp1 = temp1->super_class_
__ movl(temp1, Address(temp1, super_offset));
// No need to unpoison the result, we're comparing against null.
__ testl(temp1, temp1);
__ j(kNotEqual, slow_path->GetEntryLabel());
__ Bind(&do_copy);
} else {
__ j(kNotEqual, slow_path->GetEntryLabel());
}
} else if (!optimizations.GetSourceIsNonPrimitiveArray()) {
DCHECK(optimizations.GetDestinationIsNonPrimitiveArray());
// Bail out if the source is not a non primitive array.
// /* HeapReference<Class> */ temp1 = src->klass_
__ movl(temp1, Address(src, class_offset));
__ MaybeUnpoisonHeapReference(temp1);
// /* HeapReference<Class> */ TMP = temp1->component_type_
__ movl(CpuRegister(TMP), Address(temp1, component_offset));
__ testl(CpuRegister(TMP), CpuRegister(TMP));
__ j(kEqual, slow_path->GetEntryLabel());
__ MaybeUnpoisonHeapReference(CpuRegister(TMP));
__ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot));
__ j(kNotEqual, slow_path->GetEntryLabel());
}
// Compute base source address, base destination address, and end source address.
uint32_t element_size = sizeof(int32_t);
uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value();
if (src_pos.IsConstant()) {
int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
__ leal(temp1, Address(src, element_size * constant + offset));
} else {
__ leal(temp1, Address(src, src_pos.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, offset));
}
if (dest_pos.IsConstant()) {
int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
__ leal(temp2, Address(dest, element_size * constant + offset));
} else {
__ leal(temp2, Address(dest, dest_pos.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, offset));
}
if (length.IsConstant()) {
int32_t constant = length.GetConstant()->AsIntConstant()->GetValue();
__ leal(temp3, Address(temp1, element_size * constant));
} else {
__ leal(temp3, Address(temp1, length.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, 0));
}
// Iterate over the arrays and do a raw copy of the objects. We don't need to
// poison/unpoison, nor do any read barrier as the next uses of the destination
// array will do it.
NearLabel loop, done;
__ cmpl(temp1, temp3);
__ j(kEqual, &done);
__ Bind(&loop);
__ movl(CpuRegister(TMP), Address(temp1, 0));
__ movl(Address(temp2, 0), CpuRegister(TMP));
__ addl(temp1, Immediate(element_size));
__ addl(temp2, Immediate(element_size));
__ cmpl(temp1, temp3);
__ j(kNotEqual, &loop);
__ Bind(&done);
// We only need one card marking on the destination array.
codegen_->MarkGCCard(temp1,
temp2,
dest,
CpuRegister(kNoRegister),
/* value_can_be_null */ false);
__ Bind(slow_path->GetExitLabel());
}