diff options
Diffstat (limited to 'compiler/optimizing')
20 files changed, 998 insertions, 253 deletions
diff --git a/compiler/optimizing/bounds_check_elimination_test.cc b/compiler/optimizing/bounds_check_elimination_test.cc index 5d58207511..cb6e14b2bd 100644 --- a/compiler/optimizing/bounds_check_elimination_test.cc +++ b/compiler/optimizing/bounds_check_elimination_test.cc @@ -43,7 +43,7 @@ class BoundsCheckEliminationTest : public testing::Test { void RunBCE() { graph_->BuildDominatorTree(); - InstructionSimplifier(graph_).Run(); + InstructionSimplifier(graph_, /* codegen */ nullptr).Run(); SideEffectsAnalysis side_effects(graph_); side_effects.Run(); diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index 3fc7ebde6a..5f02a52417 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -391,7 +391,8 @@ class SuspendCheckSlowPathMIPS : public SlowPathCodeMIPS { class TypeCheckSlowPathMIPS : public SlowPathCodeMIPS { public: - explicit TypeCheckSlowPathMIPS(HInstruction* instruction) : SlowPathCodeMIPS(instruction) {} + explicit TypeCheckSlowPathMIPS(HInstruction* instruction, bool is_fatal) + : SlowPathCodeMIPS(instruction), is_fatal_(is_fatal) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); @@ -401,7 +402,9 @@ class TypeCheckSlowPathMIPS : public SlowPathCodeMIPS { CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); + if (!is_fatal_) { + SaveLiveRegisters(codegen, locations); + } // We're moving two locations to locations that could overlap, so we need a parallel // move resolver. @@ -424,13 +427,19 @@ class TypeCheckSlowPathMIPS : public SlowPathCodeMIPS { CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); } - RestoreLiveRegisters(codegen, locations); - __ B(GetExitLabel()); + if (!is_fatal_) { + RestoreLiveRegisters(codegen, locations); + __ B(GetExitLabel()); + } } const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathMIPS"; } + bool IsFatal() const OVERRIDE { return is_fatal_; } + private: + const bool is_fatal_; + DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathMIPS); }; @@ -1887,9 +1896,9 @@ void LocationsBuilderMIPS::VisitArrayGet(HArrayGet* instruction) { } } -auto InstructionCodeGeneratorMIPS::GetImplicitNullChecker(HInstruction* instruction) { - auto null_checker = [this, instruction]() { - this->codegen_->MaybeRecordImplicitNullCheck(instruction); +static auto GetImplicitNullChecker(HInstruction* instruction, CodeGeneratorMIPS* codegen) { + auto null_checker = [codegen, instruction]() { + codegen->MaybeRecordImplicitNullCheck(instruction); }; return null_checker; } @@ -1899,7 +1908,7 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { Register obj = locations->InAt(0).AsRegister<Register>(); Location index = locations->InAt(1); uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); - auto null_checker = GetImplicitNullChecker(instruction); + auto null_checker = GetImplicitNullChecker(instruction, codegen_); Primitive::Type type = instruction->GetType(); const bool maybe_compressed_char_at = mirror::kUseStringCompression && @@ -2136,7 +2145,7 @@ void InstructionCodeGeneratorMIPS::VisitArraySet(HArraySet* instruction) { bool needs_runtime_call = locations->WillCall(); bool needs_write_barrier = CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); - auto null_checker = GetImplicitNullChecker(instruction); + auto null_checker = GetImplicitNullChecker(instruction, codegen_); Register base_reg = index.IsConstant() ? obj : TMP; switch (value_type) { @@ -2319,30 +2328,178 @@ void InstructionCodeGeneratorMIPS::VisitBoundsCheck(HBoundsCheck* instruction) { } void LocationsBuilderMIPS::VisitCheckCast(HCheckCast* instruction) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( - instruction, - LocationSummary::kCallOnSlowPath); + LocationSummary::CallKind call_kind = LocationSummary::kNoCall; + bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); + + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); + switch (type_check_kind) { + case TypeCheckKind::kExactCheck: + case TypeCheckKind::kAbstractClassCheck: + case TypeCheckKind::kClassHierarchyCheck: + case TypeCheckKind::kArrayObjectCheck: + call_kind = throws_into_catch + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. + break; + case TypeCheckKind::kArrayCheck: + case TypeCheckKind::kUnresolvedCheck: + case TypeCheckKind::kInterfaceCheck: + call_kind = LocationSummary::kCallOnSlowPath; + break; + } + + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); - // Note that TypeCheckSlowPathMIPS uses this register too. locations->AddTemp(Location::RequiresRegister()); } void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); Register obj = locations->InAt(0).AsRegister<Register>(); Register cls = locations->InAt(1).AsRegister<Register>(); - Register obj_cls = locations->GetTemp(0).AsRegister<Register>(); + Register temp = locations->GetTemp(0).AsRegister<Register>(); + const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); + const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); + const uint32_t object_array_data_offset = + mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); + MipsLabel done; - SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction); + // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases + // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding + // read barriers is done for performance and code size reasons. + bool is_type_check_slow_path_fatal = false; + if (!kEmitCompilerReadBarrier) { + is_type_check_slow_path_fatal = + (type_check_kind == TypeCheckKind::kExactCheck || + type_check_kind == TypeCheckKind::kAbstractClassCheck || + type_check_kind == TypeCheckKind::kClassHierarchyCheck || + type_check_kind == TypeCheckKind::kArrayObjectCheck) && + !instruction->CanThrowIntoCatchBlock(); + } + SlowPathCodeMIPS* slow_path = + new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction, + is_type_check_slow_path_fatal); codegen_->AddSlowPath(slow_path); - // TODO: avoid this check if we know obj is not null. - __ Beqz(obj, slow_path->GetExitLabel()); - // Compare the class of `obj` with `cls`. - __ LoadFromOffset(kLoadWord, obj_cls, obj, mirror::Object::ClassOffset().Int32Value()); - __ MaybeUnpoisonHeapReference(obj_cls); - __ Bne(obj_cls, cls, slow_path->GetEntryLabel()); + // Avoid this check if we know `obj` is not null. + if (instruction->MustDoNullCheck()) { + __ Beqz(obj, &done); + } + + switch (type_check_kind) { + case TypeCheckKind::kExactCheck: + case TypeCheckKind::kArrayCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + __ LoadFromOffset(kLoadWord, temp, obj, class_offset); + __ MaybeUnpoisonHeapReference(temp); + // Jump to slow path for throwing the exception or doing a + // more involved array check. + __ Bne(temp, cls, slow_path->GetEntryLabel()); + break; + } + + case TypeCheckKind::kAbstractClassCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + __ LoadFromOffset(kLoadWord, temp, obj, class_offset); + __ MaybeUnpoisonHeapReference(temp); + // If the class is abstract, we eagerly fetch the super class of the + // object to avoid doing a comparison we know will fail. + MipsLabel loop; + __ Bind(&loop); + // /* HeapReference<Class> */ temp = temp->super_class_ + __ LoadFromOffset(kLoadWord, temp, temp, super_offset); + __ MaybeUnpoisonHeapReference(temp); + // If the class reference currently in `temp` is null, jump to the slow path to throw the + // exception. + __ Beqz(temp, slow_path->GetEntryLabel()); + // Otherwise, compare the classes. + __ Bne(temp, cls, &loop); + break; + } + + case TypeCheckKind::kClassHierarchyCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + __ LoadFromOffset(kLoadWord, temp, obj, class_offset); + __ MaybeUnpoisonHeapReference(temp); + // Walk over the class hierarchy to find a match. + MipsLabel loop; + __ Bind(&loop); + __ Beq(temp, cls, &done); + // /* HeapReference<Class> */ temp = temp->super_class_ + __ LoadFromOffset(kLoadWord, temp, temp, super_offset); + __ MaybeUnpoisonHeapReference(temp); + // If the class reference currently in `temp` is null, jump to the slow path to throw the + // exception. Otherwise, jump to the beginning of the loop. + __ Bnez(temp, &loop); + __ B(slow_path->GetEntryLabel()); + break; + } + + case TypeCheckKind::kArrayObjectCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + __ LoadFromOffset(kLoadWord, temp, obj, class_offset); + __ MaybeUnpoisonHeapReference(temp); + // Do an exact check. + __ Beq(temp, cls, &done); + // Otherwise, we need to check that the object's class is a non-primitive array. + // /* HeapReference<Class> */ temp = temp->component_type_ + __ LoadFromOffset(kLoadWord, temp, temp, component_offset); + __ MaybeUnpoisonHeapReference(temp); + // If the component type is null, jump to the slow path to throw the exception. + __ Beqz(temp, slow_path->GetEntryLabel()); + // Otherwise, the object is indeed an array, further check that this component + // type is not a primitive type. + __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); + static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); + __ Bnez(temp, slow_path->GetEntryLabel()); + break; + } + + case TypeCheckKind::kUnresolvedCheck: + // We always go into the type check slow path for the unresolved check case. + // We cannot directly call the CheckCast runtime entry point + // without resorting to a type checking slow path here (i.e. by + // calling InvokeRuntime directly), as it would require to + // assign fixed registers for the inputs of this HInstanceOf + // instruction (following the runtime calling convention), which + // might be cluttered by the potential first read barrier + // emission at the beginning of this method. + __ B(slow_path->GetEntryLabel()); + break; + + case TypeCheckKind::kInterfaceCheck: { + // Avoid read barriers to improve performance of the fast path. We can not get false + // positives by doing this. + // /* HeapReference<Class> */ temp = obj->klass_ + __ LoadFromOffset(kLoadWord, temp, obj, class_offset); + __ MaybeUnpoisonHeapReference(temp); + // /* HeapReference<Class> */ temp = temp->iftable_ + __ LoadFromOffset(kLoadWord, temp, temp, iftable_offset); + __ MaybeUnpoisonHeapReference(temp); + // Iftable is never null. + __ Lw(TMP, temp, array_length_offset); + // Loop through the iftable and check if any class matches. + MipsLabel loop; + __ Bind(&loop); + __ Addiu(temp, temp, 2 * kHeapReferenceSize); // Possibly in delay slot on R2. + __ Beqz(TMP, slow_path->GetEntryLabel()); + __ Lw(AT, temp, object_array_data_offset - 2 * kHeapReferenceSize); + __ MaybeUnpoisonHeapReference(AT); + // Go to next interface. + __ Addiu(TMP, TMP, -2); + // Compare the classes and continue the loop if they do not match. + __ Bne(AT, cls, &loop); + break; + } + } + + __ Bind(&done); __ Bind(slow_path->GetExitLabel()); } @@ -4911,7 +5068,7 @@ void InstructionCodeGeneratorMIPS::HandleFieldGet(HInstruction* instruction, LoadOperandType load_type = kLoadUnsignedByte; bool is_volatile = field_info.IsVolatile(); uint32_t offset = field_info.GetFieldOffset().Uint32Value(); - auto null_checker = GetImplicitNullChecker(instruction); + auto null_checker = GetImplicitNullChecker(instruction, codegen_); switch (type) { case Primitive::kPrimBoolean: @@ -5040,7 +5197,7 @@ void InstructionCodeGeneratorMIPS::HandleFieldSet(HInstruction* instruction, bool is_volatile = field_info.IsVolatile(); uint32_t offset = field_info.GetFieldOffset().Uint32Value(); bool needs_write_barrier = CodeGenerator::StoreNeedsWriteBarrier(type, instruction->InputAt(1)); - auto null_checker = GetImplicitNullChecker(instruction); + auto null_checker = GetImplicitNullChecker(instruction, codegen_); switch (type) { case Primitive::kPrimBoolean: @@ -5181,8 +5338,22 @@ void InstructionCodeGeneratorMIPS::GenerateGcRootFieldLoad( } void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) { - LocationSummary::CallKind call_kind = - instruction->IsExactCheck() ? LocationSummary::kNoCall : LocationSummary::kCallOnSlowPath; + LocationSummary::CallKind call_kind = LocationSummary::kNoCall; + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); + switch (type_check_kind) { + case TypeCheckKind::kExactCheck: + case TypeCheckKind::kAbstractClassCheck: + case TypeCheckKind::kClassHierarchyCheck: + case TypeCheckKind::kArrayObjectCheck: + call_kind = LocationSummary::kNoCall; + break; + case TypeCheckKind::kArrayCheck: + case TypeCheckKind::kUnresolvedCheck: + case TypeCheckKind::kInterfaceCheck: + call_kind = LocationSummary::kCallOnSlowPath; + break; + } + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); @@ -5192,36 +5363,143 @@ void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) { } void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); Register obj = locations->InAt(0).AsRegister<Register>(); Register cls = locations->InAt(1).AsRegister<Register>(); Register out = locations->Out().AsRegister<Register>(); - + uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); MipsLabel done; + SlowPathCodeMIPS* slow_path = nullptr; // Return 0 if `obj` is null. - // TODO: Avoid this check if we know `obj` is not null. - __ Move(out, ZERO); - __ Beqz(obj, &done); - - // Compare the class of `obj` with `cls`. - __ LoadFromOffset(kLoadWord, out, obj, mirror::Object::ClassOffset().Int32Value()); - __ MaybeUnpoisonHeapReference(out); - if (instruction->IsExactCheck()) { - // Classes must be equal for the instanceof to succeed. - __ Xor(out, out, cls); - __ Sltiu(out, out, 1); - } else { - // If the classes are not equal, we go into a slow path. - DCHECK(locations->OnlyCallsOnSlowPath()); - SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction); - codegen_->AddSlowPath(slow_path); - __ Bne(out, cls, slow_path->GetEntryLabel()); - __ LoadConst32(out, 1); - __ Bind(slow_path->GetExitLabel()); + // Avoid this check if we know `obj` is not null. + if (instruction->MustDoNullCheck()) { + __ Move(out, ZERO); + __ Beqz(obj, &done); + } + + switch (type_check_kind) { + case TypeCheckKind::kExactCheck: { + // /* HeapReference<Class> */ out = obj->klass_ + __ LoadFromOffset(kLoadWord, out, obj, class_offset); + __ MaybeUnpoisonHeapReference(out); + // Classes must be equal for the instanceof to succeed. + __ Xor(out, out, cls); + __ Sltiu(out, out, 1); + break; + } + + case TypeCheckKind::kAbstractClassCheck: { + // /* HeapReference<Class> */ out = obj->klass_ + __ LoadFromOffset(kLoadWord, out, obj, class_offset); + __ MaybeUnpoisonHeapReference(out); + // If the class is abstract, we eagerly fetch the super class of the + // object to avoid doing a comparison we know will fail. + MipsLabel loop; + __ Bind(&loop); + // /* HeapReference<Class> */ out = out->super_class_ + __ LoadFromOffset(kLoadWord, out, out, super_offset); + __ MaybeUnpoisonHeapReference(out); + // If `out` is null, we use it for the result, and jump to `done`. + __ Beqz(out, &done); + __ Bne(out, cls, &loop); + __ LoadConst32(out, 1); + break; + } + + case TypeCheckKind::kClassHierarchyCheck: { + // /* HeapReference<Class> */ out = obj->klass_ + __ LoadFromOffset(kLoadWord, out, obj, class_offset); + __ MaybeUnpoisonHeapReference(out); + // Walk over the class hierarchy to find a match. + MipsLabel loop, success; + __ Bind(&loop); + __ Beq(out, cls, &success); + // /* HeapReference<Class> */ out = out->super_class_ + __ LoadFromOffset(kLoadWord, out, out, super_offset); + __ MaybeUnpoisonHeapReference(out); + __ Bnez(out, &loop); + // If `out` is null, we use it for the result, and jump to `done`. + __ B(&done); + __ Bind(&success); + __ LoadConst32(out, 1); + break; + } + + case TypeCheckKind::kArrayObjectCheck: { + // /* HeapReference<Class> */ out = obj->klass_ + __ LoadFromOffset(kLoadWord, out, obj, class_offset); + __ MaybeUnpoisonHeapReference(out); + // Do an exact check. + MipsLabel success; + __ Beq(out, cls, &success); + // Otherwise, we need to check that the object's class is a non-primitive array. + // /* HeapReference<Class> */ out = out->component_type_ + __ LoadFromOffset(kLoadWord, out, out, component_offset); + __ MaybeUnpoisonHeapReference(out); + // If `out` is null, we use it for the result, and jump to `done`. + __ Beqz(out, &done); + __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); + static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); + __ Sltiu(out, out, 1); + __ B(&done); + __ Bind(&success); + __ LoadConst32(out, 1); + break; + } + + case TypeCheckKind::kArrayCheck: { + // No read barrier since the slow path will retry upon failure. + // /* HeapReference<Class> */ out = obj->klass_ + __ LoadFromOffset(kLoadWord, out, obj, class_offset); + __ MaybeUnpoisonHeapReference(out); + DCHECK(locations->OnlyCallsOnSlowPath()); + slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction, + /* is_fatal */ false); + codegen_->AddSlowPath(slow_path); + __ Bne(out, cls, slow_path->GetEntryLabel()); + __ LoadConst32(out, 1); + break; + } + + case TypeCheckKind::kUnresolvedCheck: + case TypeCheckKind::kInterfaceCheck: { + // Note that we indeed only call on slow path, but we always go + // into the slow path for the unresolved and interface check + // cases. + // + // We cannot directly call the InstanceofNonTrivial runtime + // entry point without resorting to a type checking slow path + // here (i.e. by calling InvokeRuntime directly), as it would + // require to assign fixed registers for the inputs of this + // HInstanceOf instruction (following the runtime calling + // convention), which might be cluttered by the potential first + // read barrier emission at the beginning of this method. + // + // TODO: Introduce a new runtime entry point taking the object + // to test (instead of its class) as argument, and let it deal + // with the read barrier issues. This will let us refactor this + // case of the `switch` code as it was previously (with a direct + // call to the runtime not using a type checking slow path). + // This should also be beneficial for the other cases above. + DCHECK(locations->OnlyCallsOnSlowPath()); + slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction, + /* is_fatal */ false); + codegen_->AddSlowPath(slow_path); + __ B(slow_path->GetEntryLabel()); + break; + } } __ Bind(&done); + + if (slow_path != nullptr) { + __ Bind(slow_path->GetExitLabel()); + } } void LocationsBuilderMIPS::VisitIntConstant(HIntConstant* constant) { diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index 76ca395a3e..98fee24a74 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -297,7 +297,6 @@ class InstructionCodeGeneratorMIPS : public InstructionCodeGenerator { void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction); void GenerateDivRemIntegral(HBinaryOperation* instruction); void HandleGoto(HInstruction* got, HBasicBlock* successor); - auto GetImplicitNullChecker(HInstruction* instruction); void GenPackedSwitchWithCompares(Register value_reg, int32_t lower_bound, uint32_t num_entries, diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index 580bc4737c..02c3ad6e39 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -336,7 +336,8 @@ class SuspendCheckSlowPathMIPS64 : public SlowPathCodeMIPS64 { class TypeCheckSlowPathMIPS64 : public SlowPathCodeMIPS64 { public: - explicit TypeCheckSlowPathMIPS64(HInstruction* instruction) : SlowPathCodeMIPS64(instruction) {} + explicit TypeCheckSlowPathMIPS64(HInstruction* instruction, bool is_fatal) + : SlowPathCodeMIPS64(instruction), is_fatal_(is_fatal) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); @@ -347,7 +348,9 @@ class TypeCheckSlowPathMIPS64 : public SlowPathCodeMIPS64 { CodeGeneratorMIPS64* mips64_codegen = down_cast<CodeGeneratorMIPS64*>(codegen); __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); + if (!is_fatal_) { + SaveLiveRegisters(codegen, locations); + } // We're moving two locations to locations that could overlap, so we need a parallel // move resolver. @@ -370,13 +373,19 @@ class TypeCheckSlowPathMIPS64 : public SlowPathCodeMIPS64 { CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); } - RestoreLiveRegisters(codegen, locations); - __ Bc(GetExitLabel()); + if (!is_fatal_) { + RestoreLiveRegisters(codegen, locations); + __ Bc(GetExitLabel()); + } } const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathMIPS64"; } + bool IsFatal() const OVERRIDE { return is_fatal_; } + private: + const bool is_fatal_; + DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathMIPS64); }; @@ -1471,11 +1480,19 @@ void LocationsBuilderMIPS64::VisitArrayGet(HArrayGet* instruction) { } } +static auto GetImplicitNullChecker(HInstruction* instruction, CodeGeneratorMIPS64* codegen) { + auto null_checker = [codegen, instruction]() { + codegen->MaybeRecordImplicitNullCheck(instruction); + }; + return null_checker; +} + void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { LocationSummary* locations = instruction->GetLocations(); GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>(); Location index = locations->InAt(1); uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); + auto null_checker = GetImplicitNullChecker(instruction, codegen_); Primitive::Type type = instruction->GetType(); const bool maybe_compressed_char_at = mirror::kUseStringCompression && @@ -1486,10 +1503,10 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset; - __ LoadFromOffset(kLoadUnsignedByte, out, obj, offset); + __ LoadFromOffset(kLoadUnsignedByte, out, obj, offset, null_checker); } else { __ Daddu(TMP, obj, index.AsRegister<GpuRegister>()); - __ LoadFromOffset(kLoadUnsignedByte, out, TMP, data_offset); + __ LoadFromOffset(kLoadUnsignedByte, out, TMP, data_offset, null_checker); } break; } @@ -1499,10 +1516,10 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset; - __ LoadFromOffset(kLoadSignedByte, out, obj, offset); + __ LoadFromOffset(kLoadSignedByte, out, obj, offset, null_checker); } else { __ Daddu(TMP, obj, index.AsRegister<GpuRegister>()); - __ LoadFromOffset(kLoadSignedByte, out, TMP, data_offset); + __ LoadFromOffset(kLoadSignedByte, out, TMP, data_offset, null_checker); } break; } @@ -1512,11 +1529,11 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset; - __ LoadFromOffset(kLoadSignedHalfword, out, obj, offset); + __ LoadFromOffset(kLoadSignedHalfword, out, obj, offset, null_checker); } else { __ Dsll(TMP, index.AsRegister<GpuRegister>(), TIMES_2); __ Daddu(TMP, obj, TMP); - __ LoadFromOffset(kLoadSignedHalfword, out, TMP, data_offset); + __ LoadFromOffset(kLoadSignedHalfword, out, TMP, data_offset, null_checker); } break; } @@ -1525,8 +1542,7 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { GpuRegister out = locations->Out().AsRegister<GpuRegister>(); if (maybe_compressed_char_at) { uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); - __ LoadFromOffset(kLoadWord, TMP, obj, count_offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); + __ LoadFromOffset(kLoadWord, TMP, obj, count_offset, null_checker); __ Dext(TMP, TMP, 0, 1); static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, "Expecting 0=compressed, 1=uncompressed"); @@ -1551,7 +1567,8 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { __ LoadFromOffset(kLoadUnsignedHalfword, out, obj, - data_offset + (const_index << TIMES_2)); + data_offset + (const_index << TIMES_2), + null_checker); } } else { GpuRegister index_reg = index.AsRegister<GpuRegister>(); @@ -1569,7 +1586,7 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { } else { __ Dsll(TMP, index_reg, TIMES_2); __ Daddu(TMP, obj, TMP); - __ LoadFromOffset(kLoadUnsignedHalfword, out, TMP, data_offset); + __ LoadFromOffset(kLoadUnsignedHalfword, out, TMP, data_offset, null_checker); } } break; @@ -1583,11 +1600,11 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ LoadFromOffset(load_type, out, obj, offset); + __ LoadFromOffset(load_type, out, obj, offset, null_checker); } else { __ Dsll(TMP, index.AsRegister<GpuRegister>(), TIMES_4); __ Daddu(TMP, obj, TMP); - __ LoadFromOffset(load_type, out, TMP, data_offset); + __ LoadFromOffset(load_type, out, TMP, data_offset, null_checker); } break; } @@ -1597,11 +1614,11 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ LoadFromOffset(kLoadDoubleword, out, obj, offset); + __ LoadFromOffset(kLoadDoubleword, out, obj, offset, null_checker); } else { __ Dsll(TMP, index.AsRegister<GpuRegister>(), TIMES_8); __ Daddu(TMP, obj, TMP); - __ LoadFromOffset(kLoadDoubleword, out, TMP, data_offset); + __ LoadFromOffset(kLoadDoubleword, out, TMP, data_offset, null_checker); } break; } @@ -1611,11 +1628,11 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ LoadFpuFromOffset(kLoadWord, out, obj, offset); + __ LoadFpuFromOffset(kLoadWord, out, obj, offset, null_checker); } else { __ Dsll(TMP, index.AsRegister<GpuRegister>(), TIMES_4); __ Daddu(TMP, obj, TMP); - __ LoadFpuFromOffset(kLoadWord, out, TMP, data_offset); + __ LoadFpuFromOffset(kLoadWord, out, TMP, data_offset, null_checker); } break; } @@ -1625,11 +1642,11 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ LoadFpuFromOffset(kLoadDoubleword, out, obj, offset); + __ LoadFpuFromOffset(kLoadDoubleword, out, obj, offset, null_checker); } else { __ Dsll(TMP, index.AsRegister<GpuRegister>(), TIMES_8); __ Daddu(TMP, obj, TMP); - __ LoadFpuFromOffset(kLoadDoubleword, out, TMP, data_offset); + __ LoadFpuFromOffset(kLoadDoubleword, out, TMP, data_offset, null_checker); } break; } @@ -1638,9 +1655,6 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { LOG(FATAL) << "Unreachable type " << instruction->GetType(); UNREACHABLE(); } - if (!maybe_compressed_char_at) { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } if (type == Primitive::kPrimNot) { GpuRegister out = locations->Out().AsRegister<GpuRegister>(); @@ -1696,6 +1710,7 @@ void InstructionCodeGeneratorMIPS64::VisitArraySet(HArraySet* instruction) { bool needs_runtime_call = locations->WillCall(); bool needs_write_barrier = CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); + auto null_checker = GetImplicitNullChecker(instruction, codegen_); switch (value_type) { case Primitive::kPrimBoolean: @@ -1705,10 +1720,10 @@ void InstructionCodeGeneratorMIPS64::VisitArraySet(HArraySet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset; - __ StoreToOffset(kStoreByte, value, obj, offset); + __ StoreToOffset(kStoreByte, value, obj, offset, null_checker); } else { __ Daddu(TMP, obj, index.AsRegister<GpuRegister>()); - __ StoreToOffset(kStoreByte, value, TMP, data_offset); + __ StoreToOffset(kStoreByte, value, TMP, data_offset, null_checker); } break; } @@ -1720,11 +1735,11 @@ void InstructionCodeGeneratorMIPS64::VisitArraySet(HArraySet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset; - __ StoreToOffset(kStoreHalfword, value, obj, offset); + __ StoreToOffset(kStoreHalfword, value, obj, offset, null_checker); } else { __ Dsll(TMP, index.AsRegister<GpuRegister>(), TIMES_2); __ Daddu(TMP, obj, TMP); - __ StoreToOffset(kStoreHalfword, value, TMP, data_offset); + __ StoreToOffset(kStoreHalfword, value, TMP, data_offset, null_checker); } break; } @@ -1774,10 +1789,10 @@ void InstructionCodeGeneratorMIPS64::VisitArraySet(HArraySet* instruction) { } __ PoisonHeapReference(AT, value); __ Sw(AT, base_reg, data_offset); + null_checker(); } else { - __ StoreToOffset(kStoreWord, value, base_reg, data_offset); + __ StoreToOffset(kStoreWord, value, base_reg, data_offset, null_checker); } - codegen_->MaybeRecordImplicitNullCheck(instruction); if (needs_write_barrier) { DCHECK_EQ(value_type, Primitive::kPrimNot); codegen_->MarkGCCard(obj, value, instruction->GetValueCanBeNull()); @@ -1798,11 +1813,11 @@ void InstructionCodeGeneratorMIPS64::VisitArraySet(HArraySet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ StoreToOffset(kStoreDoubleword, value, obj, offset); + __ StoreToOffset(kStoreDoubleword, value, obj, offset, null_checker); } else { __ Dsll(TMP, index.AsRegister<GpuRegister>(), TIMES_8); __ Daddu(TMP, obj, TMP); - __ StoreToOffset(kStoreDoubleword, value, TMP, data_offset); + __ StoreToOffset(kStoreDoubleword, value, TMP, data_offset, null_checker); } break; } @@ -1814,11 +1829,11 @@ void InstructionCodeGeneratorMIPS64::VisitArraySet(HArraySet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ StoreFpuToOffset(kStoreWord, value, obj, offset); + __ StoreFpuToOffset(kStoreWord, value, obj, offset, null_checker); } else { __ Dsll(TMP, index.AsRegister<GpuRegister>(), TIMES_4); __ Daddu(TMP, obj, TMP); - __ StoreFpuToOffset(kStoreWord, value, TMP, data_offset); + __ StoreFpuToOffset(kStoreWord, value, TMP, data_offset, null_checker); } break; } @@ -1830,11 +1845,11 @@ void InstructionCodeGeneratorMIPS64::VisitArraySet(HArraySet* instruction) { if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ StoreFpuToOffset(kStoreDoubleword, value, obj, offset); + __ StoreFpuToOffset(kStoreDoubleword, value, obj, offset, null_checker); } else { __ Dsll(TMP, index.AsRegister<GpuRegister>(), TIMES_8); __ Daddu(TMP, obj, TMP); - __ StoreFpuToOffset(kStoreDoubleword, value, TMP, data_offset); + __ StoreFpuToOffset(kStoreDoubleword, value, TMP, data_offset, null_checker); } break; } @@ -1843,11 +1858,6 @@ void InstructionCodeGeneratorMIPS64::VisitArraySet(HArraySet* instruction) { LOG(FATAL) << "Unreachable type " << instruction->GetType(); UNREACHABLE(); } - - // Ints and objects are handled in the switch. - if (value_type != Primitive::kPrimInt && value_type != Primitive::kPrimNot) { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } } void LocationsBuilderMIPS64::VisitBoundsCheck(HBoundsCheck* instruction) { @@ -1876,31 +1886,178 @@ void InstructionCodeGeneratorMIPS64::VisitBoundsCheck(HBoundsCheck* instruction) } void LocationsBuilderMIPS64::VisitCheckCast(HCheckCast* instruction) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( - instruction, - LocationSummary::kCallOnSlowPath); + LocationSummary::CallKind call_kind = LocationSummary::kNoCall; + bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); + + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); + switch (type_check_kind) { + case TypeCheckKind::kExactCheck: + case TypeCheckKind::kAbstractClassCheck: + case TypeCheckKind::kClassHierarchyCheck: + case TypeCheckKind::kArrayObjectCheck: + call_kind = throws_into_catch + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. + break; + case TypeCheckKind::kArrayCheck: + case TypeCheckKind::kUnresolvedCheck: + case TypeCheckKind::kInterfaceCheck: + call_kind = LocationSummary::kCallOnSlowPath; + break; + } + + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); - // Note that TypeCheckSlowPathMIPS64 uses this register too. locations->AddTemp(Location::RequiresRegister()); } void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>(); GpuRegister cls = locations->InAt(1).AsRegister<GpuRegister>(); - GpuRegister obj_cls = locations->GetTemp(0).AsRegister<GpuRegister>(); + GpuRegister temp = locations->GetTemp(0).AsRegister<GpuRegister>(); + const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); + const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); + const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); + const uint32_t object_array_data_offset = + mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); + Mips64Label done; + // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases + // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding + // read barriers is done for performance and code size reasons. + bool is_type_check_slow_path_fatal = false; + if (!kEmitCompilerReadBarrier) { + is_type_check_slow_path_fatal = + (type_check_kind == TypeCheckKind::kExactCheck || + type_check_kind == TypeCheckKind::kAbstractClassCheck || + type_check_kind == TypeCheckKind::kClassHierarchyCheck || + type_check_kind == TypeCheckKind::kArrayObjectCheck) && + !instruction->CanThrowIntoCatchBlock(); + } SlowPathCodeMIPS64* slow_path = - new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction); + new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction, + is_type_check_slow_path_fatal); codegen_->AddSlowPath(slow_path); - // TODO: avoid this check if we know obj is not null. - __ Beqzc(obj, slow_path->GetExitLabel()); - // Compare the class of `obj` with `cls`. - __ LoadFromOffset(kLoadUnsignedWord, obj_cls, obj, mirror::Object::ClassOffset().Int32Value()); - __ MaybeUnpoisonHeapReference(obj_cls); - __ Bnec(obj_cls, cls, slow_path->GetEntryLabel()); + // Avoid this check if we know `obj` is not null. + if (instruction->MustDoNullCheck()) { + __ Beqzc(obj, &done); + } + + switch (type_check_kind) { + case TypeCheckKind::kExactCheck: + case TypeCheckKind::kArrayCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset); + __ MaybeUnpoisonHeapReference(temp); + // Jump to slow path for throwing the exception or doing a + // more involved array check. + __ Bnec(temp, cls, slow_path->GetEntryLabel()); + break; + } + + case TypeCheckKind::kAbstractClassCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset); + __ MaybeUnpoisonHeapReference(temp); + // If the class is abstract, we eagerly fetch the super class of the + // object to avoid doing a comparison we know will fail. + Mips64Label loop; + __ Bind(&loop); + // /* HeapReference<Class> */ temp = temp->super_class_ + __ LoadFromOffset(kLoadUnsignedWord, temp, temp, super_offset); + __ MaybeUnpoisonHeapReference(temp); + // If the class reference currently in `temp` is null, jump to the slow path to throw the + // exception. + __ Beqzc(temp, slow_path->GetEntryLabel()); + // Otherwise, compare the classes. + __ Bnec(temp, cls, &loop); + break; + } + + case TypeCheckKind::kClassHierarchyCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset); + __ MaybeUnpoisonHeapReference(temp); + // Walk over the class hierarchy to find a match. + Mips64Label loop; + __ Bind(&loop); + __ Beqc(temp, cls, &done); + // /* HeapReference<Class> */ temp = temp->super_class_ + __ LoadFromOffset(kLoadUnsignedWord, temp, temp, super_offset); + __ MaybeUnpoisonHeapReference(temp); + // If the class reference currently in `temp` is null, jump to the slow path to throw the + // exception. Otherwise, jump to the beginning of the loop. + __ Bnezc(temp, &loop); + __ Bc(slow_path->GetEntryLabel()); + break; + } + + case TypeCheckKind::kArrayObjectCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset); + __ MaybeUnpoisonHeapReference(temp); + // Do an exact check. + __ Beqc(temp, cls, &done); + // Otherwise, we need to check that the object's class is a non-primitive array. + // /* HeapReference<Class> */ temp = temp->component_type_ + __ LoadFromOffset(kLoadUnsignedWord, temp, temp, component_offset); + __ MaybeUnpoisonHeapReference(temp); + // If the component type is null, jump to the slow path to throw the exception. + __ Beqzc(temp, slow_path->GetEntryLabel()); + // Otherwise, the object is indeed an array, further check that this component + // type is not a primitive type. + __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); + static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); + __ Bnezc(temp, slow_path->GetEntryLabel()); + break; + } + + case TypeCheckKind::kUnresolvedCheck: + // We always go into the type check slow path for the unresolved check case. + // We cannot directly call the CheckCast runtime entry point + // without resorting to a type checking slow path here (i.e. by + // calling InvokeRuntime directly), as it would require to + // assign fixed registers for the inputs of this HInstanceOf + // instruction (following the runtime calling convention), which + // might be cluttered by the potential first read barrier + // emission at the beginning of this method. + __ Bc(slow_path->GetEntryLabel()); + break; + + case TypeCheckKind::kInterfaceCheck: { + // Avoid read barriers to improve performance of the fast path. We can not get false + // positives by doing this. + // /* HeapReference<Class> */ temp = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset); + __ MaybeUnpoisonHeapReference(temp); + // /* HeapReference<Class> */ temp = temp->iftable_ + __ LoadFromOffset(kLoadUnsignedWord, temp, temp, iftable_offset); + __ MaybeUnpoisonHeapReference(temp); + // Iftable is never null. + __ Lw(TMP, temp, array_length_offset); + // Loop through the iftable and check if any class matches. + Mips64Label loop; + __ Bind(&loop); + __ Beqzc(TMP, slow_path->GetEntryLabel()); + __ Lwu(AT, temp, object_array_data_offset); + __ MaybeUnpoisonHeapReference(AT); + // Go to next interface. + __ Daddiu(temp, temp, 2 * kHeapReferenceSize); + __ Addiu(TMP, TMP, -2); + // Compare the classes and continue the loop if they do not match. + __ Bnec(AT, cls, &loop); + break; + } + } + + __ Bind(&done); __ Bind(slow_path->GetExitLabel()); } @@ -3116,6 +3273,8 @@ void InstructionCodeGeneratorMIPS64::HandleFieldGet(HInstruction* instruction, GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>(); LoadOperandType load_type = kLoadUnsignedByte; uint32_t offset = field_info.GetFieldOffset().Uint32Value(); + auto null_checker = GetImplicitNullChecker(instruction, codegen_); + switch (type) { case Primitive::kPrimBoolean: load_type = kLoadUnsignedByte; @@ -3147,14 +3306,12 @@ void InstructionCodeGeneratorMIPS64::HandleFieldGet(HInstruction* instruction, if (!Primitive::IsFloatingPointType(type)) { DCHECK(locations->Out().IsRegister()); GpuRegister dst = locations->Out().AsRegister<GpuRegister>(); - __ LoadFromOffset(load_type, dst, obj, offset); + __ LoadFromOffset(load_type, dst, obj, offset, null_checker); } else { DCHECK(locations->Out().IsFpuRegister()); FpuRegister dst = locations->Out().AsFpuRegister<FpuRegister>(); - __ LoadFpuFromOffset(load_type, dst, obj, offset); + __ LoadFpuFromOffset(load_type, dst, obj, offset, null_checker); } - - codegen_->MaybeRecordImplicitNullCheck(instruction); // TODO: memory barrier? if (type == Primitive::kPrimNot) { @@ -3184,6 +3341,8 @@ void InstructionCodeGeneratorMIPS64::HandleFieldSet(HInstruction* instruction, StoreOperandType store_type = kStoreByte; uint32_t offset = field_info.GetFieldOffset().Uint32Value(); bool needs_write_barrier = CodeGenerator::StoreNeedsWriteBarrier(type, instruction->InputAt(1)); + auto null_checker = GetImplicitNullChecker(instruction, codegen_); + switch (type) { case Primitive::kPrimBoolean: case Primitive::kPrimByte: @@ -3215,17 +3374,16 @@ void InstructionCodeGeneratorMIPS64::HandleFieldSet(HInstruction* instruction, // need poisoning. DCHECK_EQ(type, Primitive::kPrimNot); __ PoisonHeapReference(TMP, src); - __ StoreToOffset(store_type, TMP, obj, offset); + __ StoreToOffset(store_type, TMP, obj, offset, null_checker); } else { - __ StoreToOffset(store_type, src, obj, offset); + __ StoreToOffset(store_type, src, obj, offset, null_checker); } } else { DCHECK(locations->InAt(1).IsFpuRegister()); FpuRegister src = locations->InAt(1).AsFpuRegister<FpuRegister>(); - __ StoreFpuToOffset(store_type, src, obj, offset); + __ StoreFpuToOffset(store_type, src, obj, offset, null_checker); } - codegen_->MaybeRecordImplicitNullCheck(instruction); // TODO: memory barriers? if (needs_write_barrier) { DCHECK(locations->InAt(1).IsRegister()); @@ -3268,8 +3426,22 @@ void InstructionCodeGeneratorMIPS64::GenerateGcRootFieldLoad( } void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { - LocationSummary::CallKind call_kind = - instruction->IsExactCheck() ? LocationSummary::kNoCall : LocationSummary::kCallOnSlowPath; + LocationSummary::CallKind call_kind = LocationSummary::kNoCall; + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); + switch (type_check_kind) { + case TypeCheckKind::kExactCheck: + case TypeCheckKind::kAbstractClassCheck: + case TypeCheckKind::kClassHierarchyCheck: + case TypeCheckKind::kArrayObjectCheck: + call_kind = LocationSummary::kNoCall; + break; + case TypeCheckKind::kArrayCheck: + case TypeCheckKind::kUnresolvedCheck: + case TypeCheckKind::kInterfaceCheck: + call_kind = LocationSummary::kCallOnSlowPath; + break; + } + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); @@ -3279,37 +3451,143 @@ void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { } void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { + TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>(); GpuRegister cls = locations->InAt(1).AsRegister<GpuRegister>(); GpuRegister out = locations->Out().AsRegister<GpuRegister>(); - + uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); Mips64Label done; + SlowPathCodeMIPS64* slow_path = nullptr; // Return 0 if `obj` is null. - // TODO: Avoid this check if we know `obj` is not null. - __ Move(out, ZERO); - __ Beqzc(obj, &done); - - // Compare the class of `obj` with `cls`. - __ LoadFromOffset(kLoadUnsignedWord, out, obj, mirror::Object::ClassOffset().Int32Value()); - __ MaybeUnpoisonHeapReference(out); - if (instruction->IsExactCheck()) { - // Classes must be equal for the instanceof to succeed. - __ Xor(out, out, cls); - __ Sltiu(out, out, 1); - } else { - // If the classes are not equal, we go into a slow path. - DCHECK(locations->OnlyCallsOnSlowPath()); - SlowPathCodeMIPS64* slow_path = - new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction); - codegen_->AddSlowPath(slow_path); - __ Bnec(out, cls, slow_path->GetEntryLabel()); - __ LoadConst32(out, 1); - __ Bind(slow_path->GetExitLabel()); + // Avoid this check if we know `obj` is not null. + if (instruction->MustDoNullCheck()) { + __ Move(out, ZERO); + __ Beqzc(obj, &done); + } + + switch (type_check_kind) { + case TypeCheckKind::kExactCheck: { + // /* HeapReference<Class> */ out = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset); + __ MaybeUnpoisonHeapReference(out); + // Classes must be equal for the instanceof to succeed. + __ Xor(out, out, cls); + __ Sltiu(out, out, 1); + break; + } + + case TypeCheckKind::kAbstractClassCheck: { + // /* HeapReference<Class> */ out = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset); + __ MaybeUnpoisonHeapReference(out); + // If the class is abstract, we eagerly fetch the super class of the + // object to avoid doing a comparison we know will fail. + Mips64Label loop; + __ Bind(&loop); + // /* HeapReference<Class> */ out = out->super_class_ + __ LoadFromOffset(kLoadUnsignedWord, out, out, super_offset); + __ MaybeUnpoisonHeapReference(out); + // If `out` is null, we use it for the result, and jump to `done`. + __ Beqzc(out, &done); + __ Bnec(out, cls, &loop); + __ LoadConst32(out, 1); + break; + } + + case TypeCheckKind::kClassHierarchyCheck: { + // /* HeapReference<Class> */ out = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset); + __ MaybeUnpoisonHeapReference(out); + // Walk over the class hierarchy to find a match. + Mips64Label loop, success; + __ Bind(&loop); + __ Beqc(out, cls, &success); + // /* HeapReference<Class> */ out = out->super_class_ + __ LoadFromOffset(kLoadUnsignedWord, out, out, super_offset); + __ MaybeUnpoisonHeapReference(out); + __ Bnezc(out, &loop); + // If `out` is null, we use it for the result, and jump to `done`. + __ Bc(&done); + __ Bind(&success); + __ LoadConst32(out, 1); + break; + } + + case TypeCheckKind::kArrayObjectCheck: { + // /* HeapReference<Class> */ out = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset); + __ MaybeUnpoisonHeapReference(out); + // Do an exact check. + Mips64Label success; + __ Beqc(out, cls, &success); + // Otherwise, we need to check that the object's class is a non-primitive array. + // /* HeapReference<Class> */ out = out->component_type_ + __ LoadFromOffset(kLoadUnsignedWord, out, out, component_offset); + __ MaybeUnpoisonHeapReference(out); + // If `out` is null, we use it for the result, and jump to `done`. + __ Beqzc(out, &done); + __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); + static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); + __ Sltiu(out, out, 1); + __ Bc(&done); + __ Bind(&success); + __ LoadConst32(out, 1); + break; + } + + case TypeCheckKind::kArrayCheck: { + // No read barrier since the slow path will retry upon failure. + // /* HeapReference<Class> */ out = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset); + __ MaybeUnpoisonHeapReference(out); + DCHECK(locations->OnlyCallsOnSlowPath()); + slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction, + /* is_fatal */ false); + codegen_->AddSlowPath(slow_path); + __ Bnec(out, cls, slow_path->GetEntryLabel()); + __ LoadConst32(out, 1); + break; + } + + case TypeCheckKind::kUnresolvedCheck: + case TypeCheckKind::kInterfaceCheck: { + // Note that we indeed only call on slow path, but we always go + // into the slow path for the unresolved and interface check + // cases. + // + // We cannot directly call the InstanceofNonTrivial runtime + // entry point without resorting to a type checking slow path + // here (i.e. by calling InvokeRuntime directly), as it would + // require to assign fixed registers for the inputs of this + // HInstanceOf instruction (following the runtime calling + // convention), which might be cluttered by the potential first + // read barrier emission at the beginning of this method. + // + // TODO: Introduce a new runtime entry point taking the object + // to test (instead of its class) as argument, and let it deal + // with the read barrier issues. This will let us refactor this + // case of the `switch` code as it was previously (with a direct + // call to the runtime not using a type checking slow path). + // This should also be beneficial for the other cases above. + DCHECK(locations->OnlyCallsOnSlowPath()); + slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction, + /* is_fatal */ false); + codegen_->AddSlowPath(slow_path); + __ Bc(slow_path->GetEntryLabel()); + break; + } } __ Bind(&done); + + if (slow_path != nullptr) { + __ Bind(slow_path->GetExitLabel()); + } } void LocationsBuilderMIPS64::VisitIntConstant(HIntConstant* constant) { diff --git a/compiler/optimizing/induction_var_range.cc b/compiler/optimizing/induction_var_range.cc index 1cd65c1c66..d6513c8e34 100644 --- a/compiler/optimizing/induction_var_range.cc +++ b/compiler/optimizing/induction_var_range.cc @@ -57,21 +57,27 @@ static bool IsIntAndGet(HInstruction* instruction, int64_t* value) { return false; } -/** Returns b^e for b,e >= 1. Sets overflow if arithmetic wrap-around occurred. */ +/** Computes a * b for a,b > 0 (at least until first overflow happens). */ +static int64_t SafeMul(int64_t a, int64_t b, /*out*/ bool* overflow) { + if (a > 0 && b > 0 && a > (std::numeric_limits<int64_t>::max() / b)) { + *overflow = true; + } + return a * b; +} + +/** Returns b^e for b,e > 0. Sets overflow if arithmetic wrap-around occurred. */ static int64_t IntPow(int64_t b, int64_t e, /*out*/ bool* overflow) { - DCHECK_GE(b, 1); - DCHECK_GE(e, 1); + DCHECK_LT(0, b); + DCHECK_LT(0, e); int64_t pow = 1; while (e) { if (e & 1) { - int64_t oldpow = pow; - pow *= b; - if (pow < oldpow) { - *overflow = true; - } + pow = SafeMul(pow, b, overflow); } e >>= 1; - b *= b; + if (e) { + b = SafeMul(b, b, overflow); + } } return pow; } @@ -384,7 +390,8 @@ bool InductionVarRange::IsUnitStride(HInstruction* instruction, HInductionVarAnalysis::InductionInfo* trip = nullptr; if (HasInductionInfo(instruction, instruction, &loop, &info, &trip)) { if (info->induction_class == HInductionVarAnalysis::kLinear && - info->op_b->operation == HInductionVarAnalysis::kFetch) { + info->op_b->operation == HInductionVarAnalysis::kFetch && + !HInductionVarAnalysis::IsNarrowingLinear(info)) { int64_t stride_value = 0; if (IsConstant(info->op_a, kExact, &stride_value) && stride_value == 1) { int64_t off_value = 0; diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 3e340908bf..e3926c58c4 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -549,7 +549,7 @@ HInstruction* HInliner::AddTypeGuard(HInstruction* receiver, is_referrer, invoke_instruction->GetDexPc(), /* needs_access_check */ false); - HLoadClass::LoadKind kind = HSharpening::SharpenClass( + HLoadClass::LoadKind kind = HSharpening::ComputeLoadClassKind( load_class, codegen_, compiler_driver_, caller_compilation_unit_); DCHECK(kind != HLoadClass::LoadKind::kInvalid) << "We should always be able to reference a class for inline caches"; @@ -1491,7 +1491,7 @@ size_t HInliner::RunOptimizations(HGraph* callee_graph, HDeadCodeElimination dce(callee_graph, inline_stats_, "dead_code_elimination$inliner"); HConstantFolding fold(callee_graph, "constant_folding$inliner"); HSharpening sharpening(callee_graph, codegen_, dex_compilation_unit, compiler_driver_, handles_); - InstructionSimplifier simplify(callee_graph, inline_stats_); + InstructionSimplifier simplify(callee_graph, codegen_, inline_stats_); IntrinsicsRecognizer intrinsics(callee_graph, inline_stats_); HOptimization* optimizations[] = { diff --git a/compiler/optimizing/instruction_builder.cc b/compiler/optimizing/instruction_builder.cc index c60f6e5393..88f67fae04 100644 --- a/compiler/optimizing/instruction_builder.cc +++ b/compiler/optimizing/instruction_builder.cc @@ -37,37 +37,45 @@ HBasicBlock* HInstructionBuilder::FindBlockStartingAt(uint32_t dex_pc) const { return block_builder_->GetBlockAt(dex_pc); } -ArenaVector<HInstruction*>* HInstructionBuilder::GetLocalsFor(HBasicBlock* block) { +inline ArenaVector<HInstruction*>* HInstructionBuilder::GetLocalsFor(HBasicBlock* block) { ArenaVector<HInstruction*>* locals = &locals_for_[block->GetBlockId()]; const size_t vregs = graph_->GetNumberOfVRegs(); - if (locals->size() != vregs) { - locals->resize(vregs, nullptr); - - if (block->IsCatchBlock()) { - // We record incoming inputs of catch phis at throwing instructions and - // must therefore eagerly create the phis. Phis for undefined vregs will - // be deleted when the first throwing instruction with the vreg undefined - // is encountered. Unused phis will be removed by dead phi analysis. - for (size_t i = 0; i < vregs; ++i) { - // No point in creating the catch phi if it is already undefined at - // the first throwing instruction. - HInstruction* current_local_value = (*current_locals_)[i]; - if (current_local_value != nullptr) { - HPhi* phi = new (arena_) HPhi( - arena_, - i, - 0, - current_local_value->GetType()); - block->AddPhi(phi); - (*locals)[i] = phi; - } + if (locals->size() == vregs) { + return locals; + } + return GetLocalsForWithAllocation(block, locals, vregs); +} + +ArenaVector<HInstruction*>* HInstructionBuilder::GetLocalsForWithAllocation( + HBasicBlock* block, + ArenaVector<HInstruction*>* locals, + const size_t vregs) { + DCHECK_NE(locals->size(), vregs); + locals->resize(vregs, nullptr); + if (block->IsCatchBlock()) { + // We record incoming inputs of catch phis at throwing instructions and + // must therefore eagerly create the phis. Phis for undefined vregs will + // be deleted when the first throwing instruction with the vreg undefined + // is encountered. Unused phis will be removed by dead phi analysis. + for (size_t i = 0; i < vregs; ++i) { + // No point in creating the catch phi if it is already undefined at + // the first throwing instruction. + HInstruction* current_local_value = (*current_locals_)[i]; + if (current_local_value != nullptr) { + HPhi* phi = new (arena_) HPhi( + arena_, + i, + 0, + current_local_value->GetType()); + block->AddPhi(phi); + (*locals)[i] = phi; } } } return locals; } -HInstruction* HInstructionBuilder::ValueOfLocalAt(HBasicBlock* block, size_t local) { +inline HInstruction* HInstructionBuilder::ValueOfLocalAt(HBasicBlock* block, size_t local) { ArenaVector<HInstruction*>* locals = GetLocalsFor(block); return (*locals)[local]; } @@ -1676,10 +1684,10 @@ HLoadClass* HInstructionBuilder::BuildLoadClass(dex::TypeIndex type_index, dex_pc, needs_access_check); - HLoadClass::LoadKind load_kind = HSharpening::SharpenClass(load_class, - code_generator_, - compiler_driver_, - *dex_compilation_unit_); + HLoadClass::LoadKind load_kind = HSharpening::ComputeLoadClassKind(load_class, + code_generator_, + compiler_driver_, + *dex_compilation_unit_); if (load_kind == HLoadClass::LoadKind::kInvalid) { // We actually cannot reference this class, we're forced to bail. diff --git a/compiler/optimizing/instruction_builder.h b/compiler/optimizing/instruction_builder.h index e735a0c46d..7fdc1883ca 100644 --- a/compiler/optimizing/instruction_builder.h +++ b/compiler/optimizing/instruction_builder.h @@ -93,6 +93,10 @@ class HInstructionBuilder : public ValueObject { HBasicBlock* FindBlockStartingAt(uint32_t dex_pc) const; ArenaVector<HInstruction*>* GetLocalsFor(HBasicBlock* block); + // Out of line version of GetLocalsFor(), which has a fast path that is + // beneficial to get inlined by callers. + ArenaVector<HInstruction*>* GetLocalsForWithAllocation( + HBasicBlock* block, ArenaVector<HInstruction*>* locals, const size_t vregs); HInstruction* ValueOfLocalAt(HBasicBlock* block, size_t local); HInstruction* LoadLocal(uint32_t register_index, Primitive::Type type) const; HInstruction* LoadNullCheckedLocal(uint32_t register_index, uint32_t dex_pc); diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc index 35f59cb4a4..17421fc364 100644 --- a/compiler/optimizing/instruction_simplifier.cc +++ b/compiler/optimizing/instruction_simplifier.cc @@ -19,14 +19,18 @@ #include "escape.h" #include "intrinsics.h" #include "mirror/class-inl.h" +#include "sharpening.h" #include "scoped_thread_state_change-inl.h" namespace art { class InstructionSimplifierVisitor : public HGraphDelegateVisitor { public: - InstructionSimplifierVisitor(HGraph* graph, OptimizingCompilerStats* stats) + InstructionSimplifierVisitor(HGraph* graph, + CodeGenerator* codegen, + OptimizingCompilerStats* stats) : HGraphDelegateVisitor(graph), + codegen_(codegen), stats_(stats) {} void Run(); @@ -112,6 +116,7 @@ class InstructionSimplifierVisitor : public HGraphDelegateVisitor { void SimplifyAllocationIntrinsic(HInvoke* invoke); void SimplifyMemBarrier(HInvoke* invoke, MemBarrierKind barrier_kind); + CodeGenerator* codegen_; OptimizingCompilerStats* stats_; bool simplification_occurred_ = false; int simplifications_at_current_position_ = 0; @@ -123,7 +128,7 @@ class InstructionSimplifierVisitor : public HGraphDelegateVisitor { }; void InstructionSimplifier::Run() { - InstructionSimplifierVisitor visitor(graph_, stats_); + InstructionSimplifierVisitor visitor(graph_, codegen_, stats_); visitor.Run(); } @@ -1805,6 +1810,8 @@ void InstructionSimplifierVisitor::SimplifySystemArrayCopy(HInvoke* instruction) { ScopedObjectAccess soa(Thread::Current()); + Primitive::Type source_component_type = Primitive::kPrimVoid; + Primitive::Type destination_component_type = Primitive::kPrimVoid; ReferenceTypeInfo destination_rti = destination->GetReferenceTypeInfo(); if (destination_rti.IsValid()) { if (destination_rti.IsObjectArray()) { @@ -1814,6 +1821,8 @@ void InstructionSimplifierVisitor::SimplifySystemArrayCopy(HInvoke* instruction) optimizations.SetDestinationIsTypedObjectArray(); } if (destination_rti.IsPrimitiveArrayClass()) { + destination_component_type = + destination_rti.GetTypeHandle()->GetComponentType()->GetPrimitiveType(); optimizations.SetDestinationIsPrimitiveArray(); } else if (destination_rti.IsNonPrimitiveArrayClass()) { optimizations.SetDestinationIsNonPrimitiveArray(); @@ -1826,10 +1835,55 @@ void InstructionSimplifierVisitor::SimplifySystemArrayCopy(HInvoke* instruction) } if (source_rti.IsPrimitiveArrayClass()) { optimizations.SetSourceIsPrimitiveArray(); + source_component_type = source_rti.GetTypeHandle()->GetComponentType()->GetPrimitiveType(); } else if (source_rti.IsNonPrimitiveArrayClass()) { optimizations.SetSourceIsNonPrimitiveArray(); } } + // For primitive arrays, use their optimized ArtMethod implementations. + if ((source_component_type != Primitive::kPrimVoid) && + (source_component_type == destination_component_type)) { + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + PointerSize image_size = class_linker->GetImagePointerSize(); + HInvokeStaticOrDirect* invoke = instruction->AsInvokeStaticOrDirect(); + mirror::Class* system = invoke->GetResolvedMethod()->GetDeclaringClass(); + ArtMethod* method = nullptr; + switch (source_component_type) { + case Primitive::kPrimBoolean: + method = system->FindDeclaredDirectMethod("arraycopy", "([ZI[ZII)V", image_size); + break; + case Primitive::kPrimByte: + method = system->FindDeclaredDirectMethod("arraycopy", "([BI[BII)V", image_size); + break; + case Primitive::kPrimChar: + method = system->FindDeclaredDirectMethod("arraycopy", "([CI[CII)V", image_size); + break; + case Primitive::kPrimShort: + method = system->FindDeclaredDirectMethod("arraycopy", "([SI[SII)V", image_size); + break; + case Primitive::kPrimInt: + method = system->FindDeclaredDirectMethod("arraycopy", "([II[III)V", image_size); + break; + case Primitive::kPrimFloat: + method = system->FindDeclaredDirectMethod("arraycopy", "([FI[FII)V", image_size); + break; + case Primitive::kPrimLong: + method = system->FindDeclaredDirectMethod("arraycopy", "([JI[JII)V", image_size); + break; + case Primitive::kPrimDouble: + method = system->FindDeclaredDirectMethod("arraycopy", "([DI[DII)V", image_size); + break; + default: + LOG(FATAL) << "Unreachable"; + } + DCHECK(method != nullptr); + invoke->SetResolvedMethod(method); + // Sharpen the new invoke. Note that we do not update the dex method index of + // the invoke, as we would need to look it up in the current dex file, and it + // is unlikely that it exists. The most usual situation for such typed + // arraycopy methods is a direct pointer to the boot image. + HSharpening::SharpenInvokeStaticOrDirect(invoke, codegen_); + } } } diff --git a/compiler/optimizing/instruction_simplifier.h b/compiler/optimizing/instruction_simplifier.h index 7fe1067aa9..f7329a4a1f 100644 --- a/compiler/optimizing/instruction_simplifier.h +++ b/compiler/optimizing/instruction_simplifier.h @@ -23,6 +23,8 @@ namespace art { +class CodeGenerator; + /** * Implements optimizations specific to each instruction. * @@ -36,15 +38,19 @@ namespace art { class InstructionSimplifier : public HOptimization { public: explicit InstructionSimplifier(HGraph* graph, + CodeGenerator* codegen, OptimizingCompilerStats* stats = nullptr, const char* name = kInstructionSimplifierPassName) - : HOptimization(graph, name, stats) {} + : HOptimization(graph, name, stats), + codegen_(codegen) {} static constexpr const char* kInstructionSimplifierPassName = "instruction_simplifier"; void Run() OVERRIDE; private: + CodeGenerator* codegen_; + DISALLOW_COPY_AND_ASSIGN(InstructionSimplifier); }; diff --git a/compiler/optimizing/load_store_elimination.cc b/compiler/optimizing/load_store_elimination.cc index 2d3c00fb97..48699b33ae 100644 --- a/compiler/optimizing/load_store_elimination.cc +++ b/compiler/optimizing/load_store_elimination.cc @@ -38,7 +38,8 @@ class ReferenceInfo : public ArenaObject<kArenaAllocMisc> { position_(pos), is_singleton_(true), is_singleton_and_not_returned_(true), - is_singleton_and_not_deopt_visible_(true) { + is_singleton_and_not_deopt_visible_(true), + has_index_aliasing_(false) { CalculateEscape(reference_, nullptr, &is_singleton_, @@ -68,13 +69,36 @@ class ReferenceInfo : public ArenaObject<kArenaAllocMisc> { return is_singleton_and_not_returned_ && is_singleton_and_not_deopt_visible_; } + // Returns true if reference_ is a singleton and returned to the caller or + // used as an environment local of an HDeoptimize instruction. + bool IsSingletonAndNonRemovable() const { + return is_singleton_ && + (!is_singleton_and_not_returned_ || !is_singleton_and_not_deopt_visible_); + } + + bool HasIndexAliasing() { + return has_index_aliasing_; + } + + void SetHasIndexAliasing(bool has_index_aliasing) { + // Only allow setting to true. + DCHECK(has_index_aliasing); + has_index_aliasing_ = has_index_aliasing; + } + private: HInstruction* const reference_; const size_t position_; // position in HeapLocationCollector's ref_info_array_. - bool is_singleton_; // can only be referred to by a single name in the method, - bool is_singleton_and_not_returned_; // and not returned to caller, - bool is_singleton_and_not_deopt_visible_; // and not used as an environment local of HDeoptimize. + // Can only be referred to by a single name in the method. + bool is_singleton_; + // Is singleton and not returned to caller. + bool is_singleton_and_not_returned_; + // Is singleton and not used as an environment local of HDeoptimize. + bool is_singleton_and_not_deopt_visible_; + // Some heap locations with reference_ have array index aliasing, + // e.g. arr[i] and arr[j] may be the same location. + bool has_index_aliasing_; DISALLOW_COPY_AND_ASSIGN(ReferenceInfo); }; @@ -321,6 +345,8 @@ class HeapLocationCollector : public HGraphVisitor { // Different constant indices do not alias. return false; } + ReferenceInfo* ref_info = loc1->GetReferenceInfo(); + ref_info->SetHasIndexAliasing(true); } return true; } @@ -497,7 +523,8 @@ class LSEVisitor : public HGraphVisitor { removed_loads_(graph->GetArena()->Adapter(kArenaAllocLSE)), substitute_instructions_for_loads_(graph->GetArena()->Adapter(kArenaAllocLSE)), possibly_removed_stores_(graph->GetArena()->Adapter(kArenaAllocLSE)), - singleton_new_instances_(graph->GetArena()->Adapter(kArenaAllocLSE)) { + singleton_new_instances_(graph->GetArena()->Adapter(kArenaAllocLSE)), + singleton_new_arrays_(graph->GetArena()->Adapter(kArenaAllocLSE)) { } void VisitBasicBlock(HBasicBlock* block) OVERRIDE { @@ -534,20 +561,24 @@ class LSEVisitor : public HGraphVisitor { } // At this point, stores in possibly_removed_stores_ can be safely removed. - for (size_t i = 0, e = possibly_removed_stores_.size(); i < e; i++) { - HInstruction* store = possibly_removed_stores_[i]; + for (HInstruction* store : possibly_removed_stores_) { DCHECK(store->IsInstanceFieldSet() || store->IsStaticFieldSet() || store->IsArraySet()); store->GetBlock()->RemoveInstruction(store); } // Eliminate allocations that are not used. - for (size_t i = 0, e = singleton_new_instances_.size(); i < e; i++) { - HInstruction* new_instance = singleton_new_instances_[i]; + for (HInstruction* new_instance : singleton_new_instances_) { if (!new_instance->HasNonEnvironmentUses()) { new_instance->RemoveEnvironmentUsers(); new_instance->GetBlock()->RemoveInstruction(new_instance); } } + for (HInstruction* new_array : singleton_new_arrays_) { + if (!new_array->HasNonEnvironmentUses()) { + new_array->RemoveEnvironmentUsers(); + new_array->GetBlock()->RemoveInstruction(new_array); + } + } } private: @@ -558,7 +589,7 @@ class LSEVisitor : public HGraphVisitor { void KeepIfIsStore(HInstruction* heap_value) { if (heap_value == kDefaultHeapValue || heap_value == kUnknownHeapValue || - !heap_value->IsInstanceFieldSet()) { + !(heap_value->IsInstanceFieldSet() || heap_value->IsArraySet())) { return; } auto idx = std::find(possibly_removed_stores_.begin(), @@ -600,14 +631,17 @@ class LSEVisitor : public HGraphVisitor { for (size_t i = 0; i < heap_values.size(); i++) { HeapLocation* location = heap_location_collector_.GetHeapLocation(i); ReferenceInfo* ref_info = location->GetReferenceInfo(); - if (!ref_info->IsSingleton() || location->IsValueKilledByLoopSideEffects()) { - // heap value is killed by loop side effects (stored into directly, or due to - // aliasing). + if (ref_info->IsSingletonAndRemovable() && + !location->IsValueKilledByLoopSideEffects()) { + // A removable singleton's field that's not stored into inside a loop is + // invariant throughout the loop. Nothing to do. + DCHECK(ref_info->IsSingletonAndRemovable()); + } else { + // heap value is killed by loop side effects (stored into directly, or + // due to aliasing). Or the heap value may be needed after method return + // or deoptimization. KeepIfIsStore(pre_header_heap_values[i]); heap_values[i] = kUnknownHeapValue; - } else { - // A singleton's field that's not stored into inside a loop is invariant throughout - // the loop. } } } @@ -626,7 +660,7 @@ class LSEVisitor : public HGraphVisitor { bool from_all_predecessors = true; ReferenceInfo* ref_info = heap_location_collector_.GetHeapLocation(i)->GetReferenceInfo(); HInstruction* singleton_ref = nullptr; - if (ref_info->IsSingletonAndRemovable()) { + if (ref_info->IsSingleton()) { // We do more analysis of liveness when merging heap values for such // cases since stores into such references may potentially be eliminated. singleton_ref = ref_info->GetReference(); @@ -652,8 +686,9 @@ class LSEVisitor : public HGraphVisitor { } } - if (merged_value == kUnknownHeapValue) { - // There are conflicting heap values from different predecessors. + if (merged_value == kUnknownHeapValue || ref_info->IsSingletonAndNonRemovable()) { + // There are conflicting heap values from different predecessors, + // or the heap value may be needed after method return or deoptimization. // Keep the last store in each predecessor since future loads cannot be eliminated. for (HBasicBlock* predecessor : predecessors) { ArenaVector<HInstruction*>& pred_values = heap_values_for_[predecessor->GetBlockId()]; @@ -734,13 +769,16 @@ class LSEVisitor : public HGraphVisitor { heap_values[idx] = constant; return; } - if (heap_value != kUnknownHeapValue && heap_value->IsInstanceFieldSet()) { - HInstruction* store = heap_value; - // This load must be from a singleton since it's from the same field - // that a "removed" store puts the value. That store must be to a singleton's field. - DCHECK(ref_info->IsSingleton()); - // Get the real heap value of the store. - heap_value = store->InputAt(1); + if (heap_value != kUnknownHeapValue) { + if (heap_value->IsInstanceFieldSet() || heap_value->IsArraySet()) { + HInstruction* store = heap_value; + // This load must be from a singleton since it's from the same + // field/element that a "removed" store puts the value. That store + // must be to a singleton's field/element. + DCHECK(ref_info->IsSingleton()); + // Get the real heap value of the store. + heap_value = heap_value->IsInstanceFieldSet() ? store->InputAt(1) : store->InputAt(2); + } } if (heap_value == kUnknownHeapValue) { // Load isn't eliminated. Put the load as the value into the HeapLocation. @@ -796,19 +834,19 @@ class LSEVisitor : public HGraphVisitor { if (Equal(heap_value, value)) { // Store into the heap location with the same value. same_value = true; - } else if (index != nullptr) { - // For array element, don't eliminate stores since it can be easily aliased - // with non-constant index. - } else if (ref_info->IsSingletonAndRemovable()) { - // Store into a field of a singleton that's not returned. The value cannot be - // killed due to aliasing/invocation. It can be redundant since future loads can + } else if (index != nullptr && ref_info->HasIndexAliasing()) { + // For array element, don't eliminate stores if the index can be aliased. + } else if (ref_info->IsSingleton()) { + // Store into a field of a singleton. The value cannot be killed due to + // aliasing/invocation. It can be redundant since future loads can // directly get the value set by this instruction. The value can still be killed due to // merging or loop side effects. Stores whose values are killed due to merging/loop side // effects later will be removed from possibly_removed_stores_ when that is detected. + // Stores whose values may be needed after method return or deoptimization + // are also removed from possibly_removed_stores_ when that is detected. possibly_redundant = true; HNewInstance* new_instance = ref_info->GetReference()->AsNewInstance(); - DCHECK(new_instance != nullptr); - if (new_instance->IsFinalizable()) { + if (new_instance != nullptr && new_instance->IsFinalizable()) { // Finalizable objects escape globally. Need to keep the store. possibly_redundant = false; } else { @@ -834,7 +872,7 @@ class LSEVisitor : public HGraphVisitor { if (!same_value) { if (possibly_redundant) { - DCHECK(instruction->IsInstanceFieldSet()); + DCHECK(instruction->IsInstanceFieldSet() || instruction->IsArraySet()); // Put the store as the heap value. If the value is loaded from heap // by a load later, this store isn't really redundant. heap_values[idx] = instruction; @@ -914,6 +952,33 @@ class LSEVisitor : public HGraphVisitor { value); } + void VisitDeoptimize(HDeoptimize* instruction) { + const ArenaVector<HInstruction*>& heap_values = + heap_values_for_[instruction->GetBlock()->GetBlockId()]; + for (HInstruction* heap_value : heap_values) { + // Filter out fake instructions before checking instruction kind below. + if (heap_value == kUnknownHeapValue || heap_value == kDefaultHeapValue) { + continue; + } + // A store is kept as the heap value for possibly removed stores. + if (heap_value->IsInstanceFieldSet() || heap_value->IsArraySet()) { + // Check whether the reference for a store is used by an environment local of + // HDeoptimize. + HInstruction* reference = heap_value->InputAt(0); + DCHECK(heap_location_collector_.FindReferenceInfoOf(reference)->IsSingleton()); + for (const HUseListNode<HEnvironment*>& use : reference->GetEnvUses()) { + HEnvironment* user = use.GetUser(); + if (user->GetHolder() == instruction) { + // The singleton for the store is visible at this deoptimization + // point. Need to keep the store so that the heap value is + // seen by the interpreter. + KeepIfIsStore(heap_value); + } + } + } + } + } + void HandleInvoke(HInstruction* invoke) { ArenaVector<HInstruction*>& heap_values = heap_values_for_[invoke->GetBlock()->GetBlockId()]; @@ -995,6 +1060,27 @@ class LSEVisitor : public HGraphVisitor { } } + void VisitNewArray(HNewArray* new_array) OVERRIDE { + ReferenceInfo* ref_info = heap_location_collector_.FindReferenceInfoOf(new_array); + if (ref_info == nullptr) { + // new_array isn't used for array accesses. No need to process it. + return; + } + if (ref_info->IsSingletonAndRemovable()) { + singleton_new_arrays_.push_back(new_array); + } + ArenaVector<HInstruction*>& heap_values = + heap_values_for_[new_array->GetBlock()->GetBlockId()]; + for (size_t i = 0; i < heap_values.size(); i++) { + HeapLocation* location = heap_location_collector_.GetHeapLocation(i); + HInstruction* ref = location->GetReferenceInfo()->GetReference(); + if (ref == new_array && location->GetIndex() != nullptr) { + // Array elements are set to default heap values. + heap_values[i] = kDefaultHeapValue; + } + } + } + // Find an instruction's substitute if it should be removed. // Return the same instruction if it should not be removed. HInstruction* FindSubstitute(HInstruction* instruction) { @@ -1023,6 +1109,7 @@ class LSEVisitor : public HGraphVisitor { ArenaVector<HInstruction*> possibly_removed_stores_; ArenaVector<HInstruction*> singleton_new_instances_; + ArenaVector<HInstruction*> singleton_new_arrays_; DISALLOW_COPY_AND_ASSIGN(LSEVisitor); }; diff --git a/compiler/optimizing/loop_optimization.cc b/compiler/optimizing/loop_optimization.cc index 26c9ab83c2..8df513f410 100644 --- a/compiler/optimizing/loop_optimization.cc +++ b/compiler/optimizing/loop_optimization.cc @@ -16,6 +16,7 @@ #include "loop_optimization.h" +#include "driver/compiler_driver.h" #include "linear_order.h" namespace art { @@ -57,8 +58,10 @@ static bool IsEarlyExit(HLoopInformation* loop_info) { // HLoopOptimization::HLoopOptimization(HGraph* graph, + CompilerDriver* compiler_driver, HInductionVarAnalysis* induction_analysis) : HOptimization(graph, kLoopOptimizationPassName), + compiler_driver_(compiler_driver), induction_range_(induction_analysis), loop_allocator_(nullptr), top_loop_(nullptr), @@ -69,7 +72,7 @@ HLoopOptimization::HLoopOptimization(HGraph* graph, } void HLoopOptimization::Run() { - // Well-behaved loops only. + // Skip if there is no loop or the graph has try-catch/irreducible loops. // TODO: make this less of a sledgehammer. if (!graph_->HasLoops() || graph_->HasTryCatch() || graph_->HasIrreducibleLoops()) { return; @@ -85,6 +88,7 @@ void HLoopOptimization::Run() { LocalRun(); if (top_loop_ == nullptr) { + // All loops have been eliminated. graph_->SetHasLoops(false); } diff --git a/compiler/optimizing/loop_optimization.h b/compiler/optimizing/loop_optimization.h index 9ddab4150c..0b798fc7a9 100644 --- a/compiler/optimizing/loop_optimization.h +++ b/compiler/optimizing/loop_optimization.h @@ -23,13 +23,17 @@ namespace art { +class CompilerDriver; + /** * Loop optimizations. Builds a loop hierarchy and applies optimizations to * the detected nested loops, such as removal of dead induction and empty loops. */ class HLoopOptimization : public HOptimization { public: - HLoopOptimization(HGraph* graph, HInductionVarAnalysis* induction_analysis); + HLoopOptimization(HGraph* graph, + CompilerDriver* compiler_driver, + HInductionVarAnalysis* induction_analysis); void Run() OVERRIDE; @@ -76,6 +80,9 @@ class HLoopOptimization : public HOptimization { bool TryReplaceWithLastValue(HInstruction* instruction, HBasicBlock* block); void RemoveDeadInstructions(const HInstructionList& list); + // Compiler driver (to query ISA features). + const CompilerDriver* compiler_driver_; + // Range information based on prior induction variable analysis. InductionVarRange induction_range_; diff --git a/compiler/optimizing/loop_optimization_test.cc b/compiler/optimizing/loop_optimization_test.cc index 9a6b4935b2..5b9350689e 100644 --- a/compiler/optimizing/loop_optimization_test.cc +++ b/compiler/optimizing/loop_optimization_test.cc @@ -31,7 +31,7 @@ class LoopOptimizationTest : public CommonCompilerTest { allocator_(&pool_), graph_(CreateGraph(&allocator_)), iva_(new (&allocator_) HInductionVarAnalysis(graph_)), - loop_opt_(new (&allocator_) HLoopOptimization(graph_, iva_)) { + loop_opt_(new (&allocator_) HLoopOptimization(graph_, nullptr, iva_)) { BuildGraph(); } diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index 9f29692451..542b218cf8 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -1734,11 +1734,11 @@ class SideEffects : public ValueObject { // A HEnvironment object contains the values of virtual registers at a given location. class HEnvironment : public ArenaObject<kArenaAllocEnvironment> { public: - HEnvironment(ArenaAllocator* arena, - size_t number_of_vregs, - ArtMethod* method, - uint32_t dex_pc, - HInstruction* holder) + ALWAYS_INLINE HEnvironment(ArenaAllocator* arena, + size_t number_of_vregs, + ArtMethod* method, + uint32_t dex_pc, + HInstruction* holder) : vregs_(number_of_vregs, arena->Adapter(kArenaAllocEnvironmentVRegs)), locations_(number_of_vregs, arena->Adapter(kArenaAllocEnvironmentLocations)), parent_(nullptr), @@ -1747,7 +1747,7 @@ class HEnvironment : public ArenaObject<kArenaAllocEnvironment> { holder_(holder) { } - HEnvironment(ArenaAllocator* arena, const HEnvironment& to_copy, HInstruction* holder) + ALWAYS_INLINE HEnvironment(ArenaAllocator* arena, const HEnvironment& to_copy, HInstruction* holder) : HEnvironment(arena, to_copy.Size(), to_copy.GetMethod(), @@ -3915,6 +3915,7 @@ class HInvoke : public HVariableInputSizeInstruction { bool IsIntrinsic() const { return intrinsic_ != Intrinsics::kNone; } ArtMethod* GetResolvedMethod() const { return resolved_method_; } + void SetResolvedMethod(ArtMethod* method) { resolved_method_ = method; } DECLARE_ABSTRACT_INSTRUCTION(Invoke); @@ -3957,7 +3958,7 @@ class HInvoke : public HVariableInputSizeInstruction { } uint32_t number_of_arguments_; - ArtMethod* const resolved_method_; + ArtMethod* resolved_method_; const uint32_t dex_method_index_; Intrinsics intrinsic_; diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 3842ef98da..d6153b091c 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -507,7 +507,7 @@ static HOptimization* BuildOptimization( } else if (opt_name == HInductionVarAnalysis::kInductionPassName) { return new (arena) HInductionVarAnalysis(graph); } else if (opt_name == InstructionSimplifier::kInstructionSimplifierPassName) { - return new (arena) InstructionSimplifier(graph, stats, pass_name.c_str()); + return new (arena) InstructionSimplifier(graph, codegen, stats, pass_name.c_str()); } else if (opt_name == IntrinsicsRecognizer::kIntrinsicsRecognizerPassName) { return new (arena) IntrinsicsRecognizer(graph, stats); } else if (opt_name == LICM::kLoopInvariantCodeMotionPassName) { @@ -519,7 +519,7 @@ static HOptimization* BuildOptimization( } else if (opt_name == SideEffectsAnalysis::kSideEffectsAnalysisPassName) { return new (arena) SideEffectsAnalysis(graph); } else if (opt_name == HLoopOptimization::kLoopOptimizationPassName) { - return new (arena) HLoopOptimization(graph, most_recent_induction); + return new (arena) HLoopOptimization(graph, driver, most_recent_induction); } else if (opt_name == CHAGuardOptimization::kCHAGuardOptimizationPassName) { return new (arena) CHAGuardOptimization(graph); } else if (opt_name == CodeSinking::kCodeSinkingPassName) { @@ -768,26 +768,29 @@ void OptimizingCompiler::RunOptimizations(HGraph* graph, HDeadCodeElimination* dce3 = new (arena) HDeadCodeElimination( graph, stats, "dead_code_elimination$final"); HConstantFolding* fold1 = new (arena) HConstantFolding(graph, "constant_folding"); - InstructionSimplifier* simplify1 = new (arena) InstructionSimplifier(graph, stats); + InstructionSimplifier* simplify1 = new (arena) InstructionSimplifier(graph, codegen, stats); HSelectGenerator* select_generator = new (arena) HSelectGenerator(graph, stats); HConstantFolding* fold2 = new (arena) HConstantFolding( graph, "constant_folding$after_inlining"); HConstantFolding* fold3 = new (arena) HConstantFolding(graph, "constant_folding$after_bce"); - SideEffectsAnalysis* side_effects = new (arena) SideEffectsAnalysis(graph); - GVNOptimization* gvn = new (arena) GVNOptimization(graph, *side_effects); - LICM* licm = new (arena) LICM(graph, *side_effects, stats); - LoadStoreElimination* lse = new (arena) LoadStoreElimination(graph, *side_effects); + SideEffectsAnalysis* side_effects1 = new (arena) SideEffectsAnalysis( + graph, "side_effects$before_gvn"); + SideEffectsAnalysis* side_effects2 = new (arena) SideEffectsAnalysis( + graph, "side_effects$before_lse"); + GVNOptimization* gvn = new (arena) GVNOptimization(graph, *side_effects1); + LICM* licm = new (arena) LICM(graph, *side_effects1, stats); HInductionVarAnalysis* induction = new (arena) HInductionVarAnalysis(graph); - BoundsCheckElimination* bce = new (arena) BoundsCheckElimination(graph, *side_effects, induction); - HLoopOptimization* loop = new (arena) HLoopOptimization(graph, induction); + BoundsCheckElimination* bce = new (arena) BoundsCheckElimination(graph, *side_effects1, induction); + HLoopOptimization* loop = new (arena) HLoopOptimization(graph, driver, induction); + LoadStoreElimination* lse = new (arena) LoadStoreElimination(graph, *side_effects2); HSharpening* sharpening = new (arena) HSharpening( graph, codegen, dex_compilation_unit, driver, handles); InstructionSimplifier* simplify2 = new (arena) InstructionSimplifier( - graph, stats, "instruction_simplifier$after_inlining"); + graph, codegen, stats, "instruction_simplifier$after_inlining"); InstructionSimplifier* simplify3 = new (arena) InstructionSimplifier( - graph, stats, "instruction_simplifier$after_bce"); + graph, codegen, stats, "instruction_simplifier$after_bce"); InstructionSimplifier* simplify4 = new (arena) InstructionSimplifier( - graph, stats, "instruction_simplifier$before_codegen"); + graph, codegen, stats, "instruction_simplifier$before_codegen"); IntrinsicsRecognizer* intrinsics = new (arena) IntrinsicsRecognizer(graph, stats); CHAGuardOptimization* cha_guard = new (arena) CHAGuardOptimization(graph); CodeSinking* code_sinking = new (arena) CodeSinking(graph, stats); @@ -810,7 +813,7 @@ void OptimizingCompiler::RunOptimizations(HGraph* graph, fold2, // TODO: if we don't inline we can also skip fold2. simplify2, dce2, - side_effects, + side_effects1, gvn, licm, induction, @@ -818,6 +821,7 @@ void OptimizingCompiler::RunOptimizations(HGraph* graph, loop, fold3, // evaluates code generated by dynamic bce simplify3, + side_effects2, lse, cha_guard, dce3, diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index f003148d0a..7bd38c7a8c 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -41,7 +41,7 @@ void HSharpening::Run() { for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { HInstruction* instruction = it.Current(); if (instruction->IsInvokeStaticOrDirect()) { - ProcessInvokeStaticOrDirect(instruction->AsInvokeStaticOrDirect()); + SharpenInvokeStaticOrDirect(instruction->AsInvokeStaticOrDirect(), codegen_); } else if (instruction->IsLoadString()) { ProcessLoadString(instruction->AsLoadString()); } @@ -68,7 +68,9 @@ static bool AOTCanEmbedMethod(ArtMethod* method, const CompilerOptions& options) return IsInBootImage(method) && !options.GetCompilePic(); } -void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { + +void HSharpening::SharpenInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke, + CodeGenerator* codegen) { if (invoke->IsStringInit()) { // Not using the dex cache arrays. But we could still try to use a better dispatch... // TODO: Use direct_method and direct_code for the appropriate StringFactory method. @@ -95,12 +97,12 @@ void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { // We don't optimize for debuggable as it would prevent us from obsoleting the method in some // situations. - if (callee == codegen_->GetGraph()->GetArtMethod() && !codegen_->GetGraph()->IsDebuggable()) { + if (callee == codegen->GetGraph()->GetArtMethod() && !codegen->GetGraph()->IsDebuggable()) { // Recursive call. method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kRecursive; code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallSelf; } else if (Runtime::Current()->UseJitCompilation() || - AOTCanEmbedMethod(callee, codegen_->GetCompilerOptions())) { + AOTCanEmbedMethod(callee, codegen->GetCompilerOptions())) { // JIT or on-device AOT compilation referencing a boot image method. // Use the method address directly. method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress; @@ -109,13 +111,17 @@ void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { } else { // Use PC-relative access to the dex cache arrays. method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative; - DexCacheArraysLayout layout(GetInstructionSetPointerSize(codegen_->GetInstructionSet()), - &graph_->GetDexFile()); + // Note: we use the invoke's graph instead of the codegen graph, which are + // different when inlining (the codegen graph is the most outer graph). The + // invoke's dex method index is relative to the dex file where the invoke's graph + // was built from. + DexCacheArraysLayout layout(GetInstructionSetPointerSize(codegen->GetInstructionSet()), + &invoke->GetBlock()->GetGraph()->GetDexFile()); method_load_data = layout.MethodOffset(invoke->GetDexMethodIndex()); code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; } - if (graph_->IsDebuggable()) { + if (codegen->GetGraph()->IsDebuggable()) { // For debuggable apps always use the code pointer from ArtMethod // so that we don't circumvent instrumentation stubs if installed. code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; @@ -125,14 +131,14 @@ void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { method_load_kind, code_ptr_location, method_load_data }; HInvokeStaticOrDirect::DispatchInfo dispatch_info = - codegen_->GetSupportedInvokeStaticOrDirectDispatch(desired_dispatch_info, invoke); + codegen->GetSupportedInvokeStaticOrDirectDispatch(desired_dispatch_info, invoke); invoke->SetDispatchInfo(dispatch_info); } -HLoadClass::LoadKind HSharpening::SharpenClass(HLoadClass* load_class, - CodeGenerator* codegen, - CompilerDriver* compiler_driver, - const DexCompilationUnit& dex_compilation_unit) { +HLoadClass::LoadKind HSharpening::ComputeLoadClassKind(HLoadClass* load_class, + CodeGenerator* codegen, + CompilerDriver* compiler_driver, + const DexCompilationUnit& dex_compilation_unit) { Handle<mirror::Class> klass = load_class->GetClass(); DCHECK(load_class->GetLoadKind() == HLoadClass::LoadKind::kDexCacheViaMethod || load_class->GetLoadKind() == HLoadClass::LoadKind::kReferrersClass) diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h index 4240b2f339..10707c796f 100644 --- a/compiler/optimizing/sharpening.h +++ b/compiler/optimizing/sharpening.h @@ -48,14 +48,16 @@ class HSharpening : public HOptimization { static constexpr const char* kSharpeningPassName = "sharpening"; // Used by the builder and the inliner. - static HLoadClass::LoadKind SharpenClass(HLoadClass* load_class, - CodeGenerator* codegen, - CompilerDriver* compiler_driver, - const DexCompilationUnit& dex_compilation_unit) + static HLoadClass::LoadKind ComputeLoadClassKind(HLoadClass* load_class, + CodeGenerator* codegen, + CompilerDriver* compiler_driver, + const DexCompilationUnit& dex_compilation_unit) REQUIRES_SHARED(Locks::mutator_lock_); + // Used by Sharpening and InstructionSimplifier. + static void SharpenInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke, CodeGenerator* codegen); + private: - void ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke); void ProcessLoadString(HLoadString* load_string); CodeGenerator* codegen_; diff --git a/compiler/optimizing/side_effects_analysis.h b/compiler/optimizing/side_effects_analysis.h index bac6088bf7..fea47e66d9 100644 --- a/compiler/optimizing/side_effects_analysis.h +++ b/compiler/optimizing/side_effects_analysis.h @@ -25,8 +25,8 @@ namespace art { class SideEffectsAnalysis : public HOptimization { public: - explicit SideEffectsAnalysis(HGraph* graph) - : HOptimization(graph, kSideEffectsAnalysisPassName), + SideEffectsAnalysis(HGraph* graph, const char* pass_name = kSideEffectsAnalysisPassName) + : HOptimization(graph, pass_name), graph_(graph), block_effects_(graph->GetBlocks().size(), graph->GetArena()->Adapter(kArenaAllocSideEffectsAnalysis)), @@ -41,7 +41,7 @@ class SideEffectsAnalysis : public HOptimization { bool HasRun() const { return has_run_; } - static constexpr const char* kSideEffectsAnalysisPassName = "SideEffects"; + static constexpr const char* kSideEffectsAnalysisPassName = "side_effects"; private: void UpdateLoopEffects(HLoopInformation* info, SideEffects effects); diff --git a/compiler/optimizing/ssa_liveness_analysis.h b/compiler/optimizing/ssa_liveness_analysis.h index b62bf4e5f9..a239bd50c2 100644 --- a/compiler/optimizing/ssa_liveness_analysis.h +++ b/compiler/optimizing/ssa_liveness_analysis.h @@ -331,7 +331,7 @@ class LiveInterval : public ArenaObject<kArenaAllocSsaLiveness> { instruction, /* environment */ nullptr, input_index, block->GetLifetimeEnd(), first_use_); } - void AddRange(size_t start, size_t end) { + ALWAYS_INLINE void AddRange(size_t start, size_t end) { if (first_range_ == nullptr) { first_range_ = last_range_ = range_search_start_ = new (allocator_) LiveRange(start, end, first_range_); |