diff options
Diffstat (limited to 'compiler/optimizing')
| -rw-r--r-- | compiler/optimizing/code_generator_arm.cc | 164 | ||||
| -rw-r--r-- | compiler/optimizing/code_generator_arm.h | 3 | ||||
| -rw-r--r-- | compiler/optimizing/code_generator_arm64.cc | 156 | ||||
| -rw-r--r-- | compiler/optimizing/code_generator_arm64.h | 3 | ||||
| -rw-r--r-- | compiler/optimizing/code_generator_x86.cc | 197 | ||||
| -rw-r--r-- | compiler/optimizing/code_generator_x86.h | 3 | ||||
| -rw-r--r-- | compiler/optimizing/code_generator_x86_64.cc | 63 | ||||
| -rw-r--r-- | compiler/optimizing/locations.h | 6 |
8 files changed, 441 insertions, 154 deletions
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index 18f2ff3e30..d709e861e1 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -6033,12 +6033,26 @@ void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); } -static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { - return kEmitCompilerReadBarrier && - (kUseBakerReadBarrier || - type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck); +// Temp is used for read barrier. +static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { + if (kEmitCompilerReadBarrier && + (kUseBakerReadBarrier || + type_check_kind == TypeCheckKind::kAbstractClassCheck || + type_check_kind == TypeCheckKind::kClassHierarchyCheck || + type_check_kind == TypeCheckKind::kArrayObjectCheck)) { + return 1; + } + return 0; +} + +// InteraceCheck has 3 temps, one for holding the number of interfaces, one for the current +// interface pointer, one for loading the current interface. +// The other checks have one temp for loading the object's class. +static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { + if (type_check_kind == TypeCheckKind::kInterfaceCheck) { + return 3; + } + return 1 + NumberOfInstanceOfTemps(type_check_kind); } void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { @@ -6070,11 +6084,7 @@ void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { // The "out" register is used as a temporary, so it overlaps with the inputs. // Note that TypeCheckSlowPathARM uses this register too. locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - // When read barriers are enabled, we need a temporary register for - // some cases. - if (TypeCheckNeedsATemporary(type_check_kind)) { - locations->AddTemp(Location::RequiresRegister()); - } + locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); } void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { @@ -6085,9 +6095,9 @@ void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { Register cls = locations->InAt(1).AsRegister<Register>(); Location out_loc = locations->Out(); Register out = out_loc.AsRegister<Register>(); - Location maybe_temp_loc = TypeCheckNeedsATemporary(type_check_kind) ? - locations->GetTemp(0) : - Location::NoLocation(); + const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); + DCHECK_LE(num_temps, 1u); + Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -6102,7 +6112,12 @@ void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { } // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset, maybe_temp_loc); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kEmitCompilerReadBarrier); switch (type_check_kind) { case TypeCheckKind::kExactCheck: { @@ -6254,13 +6269,7 @@ void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); - // Note that TypeCheckSlowPathARM uses this "temp" register too. - locations->AddTemp(Location::RequiresRegister()); - // When read barriers are enabled, we need an additional temporary - // register for some cases. - if (TypeCheckNeedsATemporary(type_check_kind)) { - locations->AddTemp(Location::RequiresRegister()); - } + locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); } void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { @@ -6271,13 +6280,18 @@ void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { Register cls = locations->InAt(1).AsRegister<Register>(); Location temp_loc = locations->GetTemp(0); Register temp = temp_loc.AsRegister<Register>(); - Location maybe_temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ? - locations->GetTemp(1) : - Location::NoLocation(); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); + DCHECK_LE(num_temps, 3u); + Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); + Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); + const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); + const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); + const uint32_t object_array_data_offset = + mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); bool is_type_check_slow_path_fatal = (type_check_kind == TypeCheckKind::kExactCheck || @@ -6296,12 +6310,17 @@ void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { __ CompareAndBranchIfZero(obj, &done); } - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); - switch (type_check_kind) { case TypeCheckKind::kExactCheck: case TypeCheckKind::kArrayCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kEmitCompilerReadBarrier); + __ cmp(temp, ShifterOperand(cls)); // Jump to slow path for throwing the exception or doing a // more involved array check. @@ -6310,6 +6329,14 @@ void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kAbstractClassCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kEmitCompilerReadBarrier); + // If the class is abstract, we eagerly fetch the super class of the // object to avoid doing a comparison we know will fail. Label loop; @@ -6328,6 +6355,14 @@ void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kClassHierarchyCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kEmitCompilerReadBarrier); + // Walk over the class hierarchy to find a match. Label loop; __ Bind(&loop); @@ -6346,6 +6381,14 @@ void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kArrayObjectCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kEmitCompilerReadBarrier); + // Do an exact check. __ cmp(temp, ShifterOperand(cls)); __ b(&done, EQ); @@ -6364,10 +6407,7 @@ void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: - // We always go into the type check slow path for the unresolved - // and interface check cases. - // + // We always go into the type check slow path for the unresolved check case. // We cannot directly call the CheckCast runtime entry point // without resorting to a type checking slow path here (i.e. by // calling InvokeRuntime directly), as it would require to @@ -6375,8 +6415,54 @@ void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { // instruction (following the runtime calling convention), which // might be cluttered by the potential first read barrier // emission at the beginning of this method. + __ b(type_check_slow_path->GetEntryLabel()); break; + + case TypeCheckKind::kInterfaceCheck: { + // Avoid read barriers to improve performance of the fast path. We can not get false + // positives by doing this. + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + /*emit_read_barrier*/ false); + + // /* HeapReference<Class> */ temp = temp->iftable_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + temp_loc, + iftable_offset, + maybe_temp2_loc, + /*emit_read_barrier*/ false); + Label is_null; + // Null iftable means it is empty and will always fail the check. + // Not cbz since the temp may not be a low register. + __ CompareAndBranchIfZero(temp, &is_null); + + // Loop through the iftable and check if any class matches. + __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); + + Label start_loop; + __ Bind(&start_loop); + __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); + __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); + __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); + __ b(&done, EQ); // Return if same class. + // Go to next interface. + __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); + __ sub(maybe_temp2_loc.AsRegister<Register>(), + maybe_temp2_loc.AsRegister<Register>(), + ShifterOperand(2)); + // Not cbnz since the temp may not be a low register. + __ CompareAndBranchIfNonZero(maybe_temp2_loc.AsRegister<Register>(), &start_loop); + __ Bind(&is_null); + + __ b(type_check_slow_path->GetEntryLabel()); + break; + } } __ Bind(&done); @@ -6684,10 +6770,12 @@ void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters(HInstruction Location out, Location obj, uint32_t offset, - Location maybe_temp) { + Location maybe_temp, + bool emit_read_barrier) { Register out_reg = out.AsRegister<Register>(); Register obj_reg = obj.AsRegister<Register>(); - if (kEmitCompilerReadBarrier) { + if (emit_read_barrier) { + DCHECK(kEmitCompilerReadBarrier); if (kUseBakerReadBarrier) { DCHECK(maybe_temp.IsRegister()) << maybe_temp; // Load with fast path based Baker's read barrier. diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h index d3ed9a261c..158b3959e5 100644 --- a/compiler/optimizing/code_generator_arm.h +++ b/compiler/optimizing/code_generator_arm.h @@ -278,7 +278,8 @@ class InstructionCodeGeneratorARM : public InstructionCodeGenerator { Location out, Location obj, uint32_t offset, - Location maybe_temp); + Location maybe_temp, + bool emit_read_barrier); // Generate a GC root reference load: // // root <- *(obj + offset) diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 4418474929..f92297bdd0 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -3323,12 +3323,26 @@ void InstructionCodeGeneratorARM64::VisitInstanceFieldSet(HInstanceFieldSet* ins HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); } -static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { - return kEmitCompilerReadBarrier && +// Temp is used for read barrier. +static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { + if (kEmitCompilerReadBarrier && (kUseBakerReadBarrier || - type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck); + type_check_kind == TypeCheckKind::kAbstractClassCheck || + type_check_kind == TypeCheckKind::kClassHierarchyCheck || + type_check_kind == TypeCheckKind::kArrayObjectCheck)) { + return 1; + } + return 0; +} + +// InteraceCheck has 3 temps, one for holding the number of interfaces, one for the current +// interface pointer, one for loading the current interface. +// The other checks have one temp for loading the object's class. +static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { + if (type_check_kind == TypeCheckKind::kInterfaceCheck) { + return 3; + } + return 1 + NumberOfInstanceOfTemps(type_check_kind); } void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) { @@ -3360,11 +3374,8 @@ void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) { // The "out" register is used as a temporary, so it overlaps with the inputs. // Note that TypeCheckSlowPathARM64 uses this register too. locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - // When read barriers are enabled, we need a temporary register for - // some cases. - if (TypeCheckNeedsATemporary(type_check_kind)) { - locations->AddTemp(Location::RequiresRegister()); - } + // Add temps if necessary for read barriers. + locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); } void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) { @@ -3375,9 +3386,9 @@ void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) { Register cls = InputRegisterAt(instruction, 1); Location out_loc = locations->Out(); Register out = OutputRegister(instruction); - Location maybe_temp_loc = TypeCheckNeedsATemporary(type_check_kind) ? - locations->GetTemp(0) : - Location::NoLocation(); + const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); + DCHECK_LE(num_temps, 1u); + Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -3393,7 +3404,12 @@ void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) { } // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset, maybe_temp_loc); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kEmitCompilerReadBarrier); switch (type_check_kind) { case TypeCheckKind::kExactCheck: { @@ -3545,13 +3561,8 @@ void LocationsBuilderARM64::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); - // Note that TypeCheckSlowPathARM64 uses this "temp" register too. - locations->AddTemp(Location::RequiresRegister()); - // When read barriers are enabled, we need an additional temporary - // register for some cases. - if (TypeCheckNeedsATemporary(type_check_kind)) { - locations->AddTemp(Location::RequiresRegister()); - } + // Add temps for read barriers and other uses. One is used by TypeCheckSlowPathARM64. + locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); } void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { @@ -3560,15 +3571,21 @@ void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { Location obj_loc = locations->InAt(0); Register obj = InputRegisterAt(instruction, 0); Register cls = InputRegisterAt(instruction, 1); + const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); + DCHECK_GE(num_temps, 1u); + DCHECK_LE(num_temps, 3u); Location temp_loc = locations->GetTemp(0); - Location maybe_temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ? - locations->GetTemp(1) : - Location::NoLocation(); + Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); + Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); Register temp = WRegisterFrom(temp_loc); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); + const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); + const uint32_t object_array_data_offset = + mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); bool is_type_check_slow_path_fatal = (type_check_kind == TypeCheckKind::kExactCheck || @@ -3587,12 +3604,17 @@ void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { __ Cbz(obj, &done); } - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); - switch (type_check_kind) { case TypeCheckKind::kExactCheck: case TypeCheckKind::kArrayCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kEmitCompilerReadBarrier); + __ Cmp(temp, cls); // Jump to slow path for throwing the exception or doing a // more involved array check. @@ -3601,6 +3623,14 @@ void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kAbstractClassCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kEmitCompilerReadBarrier); + // If the class is abstract, we eagerly fetch the super class of the // object to avoid doing a comparison we know will fail. vixl::aarch64::Label loop; @@ -3618,6 +3648,14 @@ void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kClassHierarchyCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kEmitCompilerReadBarrier); + // Walk over the class hierarchy to find a match. vixl::aarch64::Label loop; __ Bind(&loop); @@ -3636,6 +3674,14 @@ void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kArrayObjectCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kEmitCompilerReadBarrier); + // Do an exact check. __ Cmp(temp, cls); __ B(eq, &done); @@ -3655,9 +3701,7 @@ void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: - // We always go into the type check slow path for the unresolved - // and interface check cases. + // We always go into the type check slow path for the unresolved check cases. // // We cannot directly call the CheckCast runtime entry point // without resorting to a type checking slow path here (i.e. by @@ -3668,6 +3712,44 @@ void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { // emission at the beginning of this method. __ B(type_check_slow_path->GetEntryLabel()); break; + case TypeCheckKind::kInterfaceCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + /*emit_read_barrier*/ false); + + // /* HeapReference<Class> */ temp = temp->iftable_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + temp_loc, + iftable_offset, + maybe_temp2_loc, + /*emit_read_barrier*/ false); + vixl::aarch64::Label is_null; + // Null iftable means it is empty and will always fail the check. + __ Cbz(temp, &is_null); + + // Loop through the iftable and check if any class matches. + __ Ldr(WRegisterFrom(maybe_temp2_loc), HeapOperand(temp.W(), array_length_offset)); + + vixl::aarch64::Label start_loop; + __ Bind(&start_loop); + __ Ldr(WRegisterFrom(maybe_temp3_loc), HeapOperand(temp.W(), object_array_data_offset)); + GetAssembler()->MaybeUnpoisonHeapReference(WRegisterFrom(maybe_temp3_loc)); + __ Cmp(cls, WRegisterFrom(maybe_temp3_loc)); + __ B(eq, &done); // Return if same class. + // Go to next interface. + __ Add(temp, temp, 2 * kHeapReferenceSize); + __ Sub(WRegisterFrom(maybe_temp2_loc), WRegisterFrom(maybe_temp2_loc), 2); + __ Cbnz(WRegisterFrom(maybe_temp2_loc), &start_loop); + __ Bind(&is_null); + + __ B(type_check_slow_path->GetEntryLabel()); + break; + } } __ Bind(&done); @@ -5227,11 +5309,13 @@ void InstructionCodeGeneratorARM64::GenerateReferenceLoadTwoRegisters(HInstructi Location out, Location obj, uint32_t offset, - Location maybe_temp) { + Location maybe_temp, + bool emit_read_barrier) { Primitive::Type type = Primitive::kPrimNot; Register out_reg = RegisterFrom(out, type); Register obj_reg = RegisterFrom(obj, type); - if (kEmitCompilerReadBarrier) { + if (emit_read_barrier) { + DCHECK(kEmitCompilerReadBarrier); if (kUseBakerReadBarrier) { // Load with fast path based Baker's read barrier. Register temp_reg = RegisterFrom(maybe_temp, type); diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index f0b979d1a4..de36a87c93 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -284,7 +284,8 @@ class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator { Location out, Location obj, uint32_t offset, - Location maybe_temp); + Location maybe_temp, + bool emit_read_barrier); // Generate a GC root reference load: // // root <- *(obj + offset) diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index eb748f81ed..af46babcce 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -6357,12 +6357,26 @@ void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) { CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); } -static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { - return kEmitCompilerReadBarrier && +// Temp is used for read barrier. +static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { + if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier && (type_check_kind == TypeCheckKind::kAbstractClassCheck || type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck); + type_check_kind == TypeCheckKind::kArrayObjectCheck)) { + return 1; + } + return 0; +} + +// InteraceCheck has 3 temps, one for holding the number of interfaces, one for the current +// interface pointer, one for loading the current interface. +// The other checks have one temp for loading the object's class. +static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { + if (type_check_kind == TypeCheckKind::kInterfaceCheck && !kPoisonHeapReferences) { + return 2; + } + return 1 + NumberOfInstanceOfTemps(type_check_kind); } void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) { @@ -6393,11 +6407,8 @@ void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) { locations->SetInAt(1, Location::Any()); // Note that TypeCheckSlowPathX86 uses this "out" register too. locations->SetOut(Location::RequiresRegister()); - // When read barriers are enabled, we need a temporary register for - // some cases. - if (TypeCheckNeedsATemporary(type_check_kind)) { - locations->AddTemp(Location::RequiresRegister()); - } + // When read barriers are enabled, we need a temporary register for some cases. + locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); } void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) { @@ -6408,9 +6419,9 @@ void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) { Location cls = locations->InAt(1); Location out_loc = locations->Out(); Register out = out_loc.AsRegister<Register>(); - Location maybe_temp_loc = TypeCheckNeedsATemporary(type_check_kind) ? - locations->GetTemp(0) : - Location::NoLocation(); + const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); + DCHECK_LE(num_temps, 1u); + Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -6426,7 +6437,11 @@ void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) { } // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); switch (type_check_kind) { case TypeCheckKind::kExactCheck: { @@ -6591,26 +6606,46 @@ void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kAbstractClassCheck: case TypeCheckKind::kClassHierarchyCheck: case TypeCheckKind::kArrayObjectCheck: + case TypeCheckKind::kInterfaceCheck: call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. break; case TypeCheckKind::kArrayCheck: case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: call_kind = LocationSummary::kCallOnSlowPath; break; } LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::Any()); + if (type_check_kind == TypeCheckKind::kInterfaceCheck) { + // Require a register for the interface check since there is a loop that compares the class to + // a memory address. + locations->SetInAt(1, Location::RequiresRegister()); + } else { + locations->SetInAt(1, Location::Any()); + } // Note that TypeCheckSlowPathX86 uses this "temp" register too. locations->AddTemp(Location::RequiresRegister()); - // When read barriers are enabled, we need an additional temporary - // register for some cases. - if (TypeCheckNeedsATemporary(type_check_kind)) { - locations->AddTemp(Location::RequiresRegister()); + // When read barriers are enabled, we need an additional temporary register for some cases. + locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); +} + +static bool IsTypeCheckSlowPathFatal(TypeCheckKind type_check_kind, bool throws_into_catch) { + switch (type_check_kind) { + case TypeCheckKind::kExactCheck: + case TypeCheckKind::kAbstractClassCheck: + case TypeCheckKind::kClassHierarchyCheck: + case TypeCheckKind::kArrayObjectCheck: + return !throws_into_catch && !kEmitCompilerReadBarrier; + case TypeCheckKind::kInterfaceCheck: + return !throws_into_catch && !kEmitCompilerReadBarrier && !kPoisonHeapReferences; + case TypeCheckKind::kArrayCheck: + case TypeCheckKind::kUnresolvedCheck: + return false; } + LOG(FATAL) << "Unreachable"; + UNREACHABLE(); } void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { @@ -6621,20 +6656,22 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { Location cls = locations->InAt(1); Location temp_loc = locations->GetTemp(0); Register temp = temp_loc.AsRegister<Register>(); - Location maybe_temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ? - locations->GetTemp(1) : - Location::NoLocation(); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); + DCHECK_GE(num_temps, 1u); + DCHECK_LE(num_temps, 2u); + Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); + const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); + const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); + const uint32_t object_array_data_offset = + mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); bool is_type_check_slow_path_fatal = - (type_check_kind == TypeCheckKind::kExactCheck || - type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck) && - !instruction->CanThrowIntoCatchBlock(); + IsTypeCheckSlowPathFatal(type_check_kind, instruction->CanThrowIntoCatchBlock()); + SlowPathCode* type_check_slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction, is_type_check_slow_path_fatal); @@ -6647,12 +6684,16 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { __ j(kEqual, &done); } - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset); - switch (type_check_kind) { case TypeCheckKind::kExactCheck: case TypeCheckKind::kArrayCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); + if (cls.IsRegister()) { __ cmpl(temp, cls.AsRegister<Register>()); } else { @@ -6666,6 +6707,13 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kAbstractClassCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); + // If the class is abstract, we eagerly fetch the super class of the // object to avoid doing a comparison we know will fail. NearLabel loop; @@ -6690,6 +6738,13 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kClassHierarchyCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); + // Walk over the class hierarchy to find a match. NearLabel loop; __ Bind(&loop); @@ -6714,6 +6769,13 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kArrayObjectCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kEmitCompilerReadBarrier); + // Do an exact check. if (cls.IsRegister()) { __ cmpl(temp, cls.AsRegister<Register>()); @@ -6738,10 +6800,7 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { } case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: - // We always go into the type check slow path for the unresolved - // and interface check cases. - // + // We always go into the type check slow path for the unresolved check case. // We cannot directly call the CheckCast runtime entry point // without resorting to a type checking slow path here (i.e. by // calling InvokeRuntime directly), as it would require to @@ -6749,15 +6808,59 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { // instruction (following the runtime calling convention), which // might be cluttered by the potential first read barrier // emission at the beginning of this method. - // - // TODO: Introduce a new runtime entry point taking the object - // to test (instead of its class) as argument, and let it deal - // with the read barrier issues. This will let us refactor this - // case of the `switch` code as it was previously (with a direct - // call to the runtime not using a type checking slow path). - // This should also be beneficial for the other cases above. __ jmp(type_check_slow_path->GetEntryLabel()); break; + + case TypeCheckKind::kInterfaceCheck: { + // Fast path for the interface check. Since we compare with a memory location in the inner + // loop we would need to have cls poisoned. However unpoisoning cls would reset the + // conditional flags and cause the conditional jump to be incorrect. Therefore we just jump + // to the slow path if we are running under poisoning + if (!kPoisonHeapReferences) { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + /*emit_read_barrier*/ false); + + // Try to avoid read barriers to improve the fast path. We can not get false positives by + // doing this. + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + /*emit_read_barrier*/ false); + + // /* HeapReference<Class> */ temp = temp->iftable_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + temp_loc, + iftable_offset, + /*emit_read_barrier*/ false); + NearLabel is_null; + // Null iftable means it is empty. + __ testl(temp, temp); + __ j(kZero, &is_null); + + // Loop through the iftable and check if any class matches. + __ movl(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); + + NearLabel start_loop; + __ Bind(&start_loop); + __ cmpl(cls.AsRegister<Register>(), Address(temp, object_array_data_offset)); + __ j(kEqual, &done); // Return if same class. + // Go to next interface. + __ addl(temp, Immediate(2 * kHeapReferenceSize)); + __ subl(maybe_temp2_loc.AsRegister<Register>(), Immediate(2)); + __ j(kNotZero, &start_loop); + __ Bind(&is_null); + } + + __ jmp(type_check_slow_path->GetEntryLabel()); + break; + } } __ Bind(&done); @@ -6949,10 +7052,12 @@ void InstructionCodeGeneratorX86::GenerateReferenceLoadOneRegister(HInstruction* void InstructionCodeGeneratorX86::GenerateReferenceLoadTwoRegisters(HInstruction* instruction, Location out, Location obj, - uint32_t offset) { + uint32_t offset, + bool emit_read_barrier) { Register out_reg = out.AsRegister<Register>(); Register obj_reg = obj.AsRegister<Register>(); - if (kEmitCompilerReadBarrier) { + if (emit_read_barrier) { + DCHECK(kEmitCompilerReadBarrier); if (kUseBakerReadBarrier) { // Load with fast path based Baker's read barrier. // /* HeapReference<Object> */ out = *(obj + offset) diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index a2cb9bb178..7cf3bae79b 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -254,7 +254,8 @@ class InstructionCodeGeneratorX86 : public InstructionCodeGenerator { void GenerateReferenceLoadTwoRegisters(HInstruction* instruction, Location out, Location obj, - uint32_t offset); + uint32_t offset, + bool emit_read_barrier); // Generate a GC root reference load: // // root <- *address diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index df77362d4d..038b14c262 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -6007,7 +6007,7 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { } } -bool IsTypeCheckSlowPathFatal(TypeCheckKind type_check_kind, bool throws_into_catch) { +static bool IsTypeCheckSlowPathFatal(TypeCheckKind type_check_kind, bool throws_into_catch) { switch (type_check_kind) { case TypeCheckKind::kExactCheck: case TypeCheckKind::kAbstractClassCheck: @@ -6067,7 +6067,8 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); - const int object_array_data_offset = mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); + const uint32_t object_array_data_offset = + mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); bool is_type_check_slow_path_fatal = IsTypeCheckSlowPathFatal(type_check_kind, instruction->CanThrowIntoCatchBlock()); @@ -6225,17 +6226,7 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { break; } - case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: - NearLabel done; - - // Avoid null check if we know obj is not null. - if (instruction->MustDoNullCheck()) { - __ testl(obj, obj); - __ j(kEqual, &done); - } - - + case TypeCheckKind::kUnresolvedCheck: { // We always go into the type check slow path for the unresolved case. // // We cannot directly call the CheckCast runtime entry point @@ -6245,18 +6236,30 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // instruction (following the runtime calling convention), which // might be cluttered by the potential first read barrier // emission at the beginning of this method. - // - // TODO: Introduce a new runtime entry point taking the object - // to test (instead of its class) as argument, and let it deal - // with the read barrier issues. This will let us refactor this - // case of the `switch` code as it was previously (with a direct - // call to the runtime not using a type checking slow path). - // This should also be beneficial for the other cases above. - // Fast path for the interface check. Since we compare with a memory location in the inner - // loop we would need to have cls poisoned. However unpoisoning cls would reset the - // conditional flags and cause the conditional jump to be incorrect. - if (type_check_kind == TypeCheckKind::kInterfaceCheck && !kPoisonHeapReferences) { + NearLabel done; + // Avoid null check if we know obj is not null. + if (instruction->MustDoNullCheck()) { + __ testl(obj, obj); + __ j(kEqual, &done); + } + __ jmp(type_check_slow_path->GetEntryLabel()); + __ Bind(&done); + break; + } + + case TypeCheckKind::kInterfaceCheck: + NearLabel done; + + // Avoid null check if we know obj is not null. + if (instruction->MustDoNullCheck()) { + __ testl(obj, obj); + __ j(kEqual, &done); + } + + // Fast path for the interface check. We always go slow path for heap poisoning since + // unpoisoning cls would require an extra temp. + if (!kPoisonHeapReferences) { // Try to avoid read barriers to improve the fast path. We can not get false positives by // doing this. // /* HeapReference<Class> */ temp = obj->klass_ @@ -6274,21 +6277,19 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { /*emit_read_barrier*/ false); NearLabel is_null; // Null iftable means it is empty. - __ testl(temp_loc.AsRegister<CpuRegister>(), temp_loc.AsRegister<CpuRegister>()); + __ testl(temp, temp); __ j(kZero, &is_null); // Loop through the iftable and check if any class matches. - __ movl(maybe_temp2_loc.AsRegister<CpuRegister>(), - Address(temp_loc.AsRegister<CpuRegister>(), array_length_offset)); + __ movl(maybe_temp2_loc.AsRegister<CpuRegister>(), Address(temp, array_length_offset)); NearLabel start_loop; __ Bind(&start_loop); - __ cmpl(cls.AsRegister<CpuRegister>(), - Address(temp_loc.AsRegister<CpuRegister>(), object_array_data_offset)); + __ cmpl(cls.AsRegister<CpuRegister>(), Address(temp, object_array_data_offset)); __ j(kEqual, &done); // Return if same class. // Go to next interface. - __ addq(temp_loc.AsRegister<CpuRegister>(), Immediate(2 * kHeapReferenceSize)); - __ subq(maybe_temp2_loc.AsRegister<CpuRegister>(), Immediate(2)); + __ addl(temp, Immediate(2 * kHeapReferenceSize)); + __ subl(maybe_temp2_loc.AsRegister<CpuRegister>(), Immediate(2)); __ j(kNotZero, &start_loop); __ Bind(&is_null); } diff --git a/compiler/optimizing/locations.h b/compiler/optimizing/locations.h index 52747c0cc4..091b58a63d 100644 --- a/compiler/optimizing/locations.h +++ b/compiler/optimizing/locations.h @@ -525,6 +525,12 @@ class LocationSummary : public ArenaObject<kArenaAllocLocationSummary> { temps_.push_back(location); } + void AddRegisterTemps(size_t count) { + for (size_t i = 0; i < count; ++i) { + AddTemp(Location::RequiresRegister()); + } + } + Location GetTemp(uint32_t at) const { return temps_[at]; } |