diff options
Diffstat (limited to 'compiler/optimizing/inliner.cc')
| -rw-r--r-- | compiler/optimizing/inliner.cc | 840 |
1 files changed, 601 insertions, 239 deletions
diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 3e340908bf..298ae5c847 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -46,29 +46,100 @@ namespace art { -static constexpr size_t kMaximumNumberOfHInstructions = 32; +// Instruction limit to control memory. +static constexpr size_t kMaximumNumberOfTotalInstructions = 1024; + +// Maximum number of instructions for considering a method small, +// which we will always try to inline if the other non-instruction limits +// are not reached. +static constexpr size_t kMaximumNumberOfInstructionsForSmallMethod = 3; // Limit the number of dex registers that we accumulate while inlining // to avoid creating large amount of nested environments. static constexpr size_t kMaximumNumberOfCumulatedDexRegisters = 64; -// Avoid inlining within a huge method due to memory pressure. -static constexpr size_t kMaximumCodeUnitSize = 4096; +// Limit recursive call inlining, which do not benefit from too +// much inlining compared to code locality. +static constexpr size_t kMaximumNumberOfRecursiveCalls = 4; + +// Controls the use of inline caches in AOT mode. +static constexpr bool kUseAOTInlineCaches = true; + +// We check for line numbers to make sure the DepthString implementation +// aligns the output nicely. +#define LOG_INTERNAL(msg) \ + static_assert(__LINE__ > 10, "Unhandled line number"); \ + static_assert(__LINE__ < 10000, "Unhandled line number"); \ + VLOG(compiler) << DepthString(__LINE__) << msg + +#define LOG_TRY() LOG_INTERNAL("Try inlinining call: ") +#define LOG_NOTE() LOG_INTERNAL("Note: ") +#define LOG_SUCCESS() LOG_INTERNAL("Success: ") +#define LOG_FAIL(stat) MaybeRecordStat(stat); LOG_INTERNAL("Fail: ") +#define LOG_FAIL_NO_STAT() LOG_INTERNAL("Fail: ") + +std::string HInliner::DepthString(int line) const { + std::string value; + // Indent according to the inlining depth. + size_t count = depth_; + // Line numbers get printed in the log, so add a space if the log's line is less + // than 1000, and two if less than 100. 10 cannot be reached as it's the copyright. + if (!kIsTargetBuild) { + if (line < 100) { + value += " "; + } + if (line < 1000) { + value += " "; + } + // Safeguard if this file reaches more than 10000 lines. + DCHECK_LT(line, 10000); + } + for (size_t i = 0; i < count; ++i) { + value += " "; + } + return value; +} -void HInliner::Run() { - const CompilerOptions& compiler_options = compiler_driver_->GetCompilerOptions(); - if ((compiler_options.GetInlineDepthLimit() == 0) - || (compiler_options.GetInlineMaxCodeUnits() == 0)) { - return; +static size_t CountNumberOfInstructions(HGraph* graph) { + size_t number_of_instructions = 0; + for (HBasicBlock* block : graph->GetReversePostOrderSkipEntryBlock()) { + for (HInstructionIterator instr_it(block->GetInstructions()); + !instr_it.Done(); + instr_it.Advance()) { + ++number_of_instructions; + } } - if (caller_compilation_unit_.GetCodeItem()->insns_size_in_code_units_ > kMaximumCodeUnitSize) { - return; + return number_of_instructions; +} + +void HInliner::UpdateInliningBudget() { + if (total_number_of_instructions_ >= kMaximumNumberOfTotalInstructions) { + // Always try to inline small methods. + inlining_budget_ = kMaximumNumberOfInstructionsForSmallMethod; + } else { + inlining_budget_ = std::max( + kMaximumNumberOfInstructionsForSmallMethod, + kMaximumNumberOfTotalInstructions - total_number_of_instructions_); } +} + +void HInliner::Run() { if (graph_->IsDebuggable()) { // For simplicity, we currently never inline when the graph is debuggable. This avoids // doing some logic in the runtime to discover if a method could have been inlined. return; } + + // Initialize the number of instructions for the method being compiled. Recursive calls + // to HInliner::Run have already updated the instruction count. + if (outermost_graph_ == graph_) { + total_number_of_instructions_ = CountNumberOfInstructions(graph_); + } + + UpdateInliningBudget(); + DCHECK_NE(total_number_of_instructions_, 0u); + DCHECK_NE(inlining_budget_, 0u); + // Keep a copy of all blocks when starting the visit. ArenaVector<HBasicBlock*> blocks = graph_->GetReversePostOrder(); DCHECK(!blocks.empty()); @@ -249,20 +320,25 @@ class ScopedProfilingInfoInlineUse { ProfilingInfo* const profiling_info_; }; -static bool IsMonomorphic(Handle<mirror::ObjectArray<mirror::Class>> classes) - REQUIRES_SHARED(Locks::mutator_lock_) { - DCHECK_GE(InlineCache::kIndividualCacheSize, 2); - return classes->Get(0) != nullptr && classes->Get(1) == nullptr; -} - -static bool IsMegamorphic(Handle<mirror::ObjectArray<mirror::Class>> classes) - REQUIRES_SHARED(Locks::mutator_lock_) { - for (size_t i = 0; i < InlineCache::kIndividualCacheSize; ++i) { - if (classes->Get(i) == nullptr) { - return false; +HInliner::InlineCacheType HInliner::GetInlineCacheType( + const Handle<mirror::ObjectArray<mirror::Class>>& classes) + REQUIRES_SHARED(Locks::mutator_lock_) { + uint8_t number_of_types = 0; + for (; number_of_types < InlineCache::kIndividualCacheSize; ++number_of_types) { + if (classes->Get(number_of_types) == nullptr) { + break; } } - return true; + + if (number_of_types == 0) { + return kInlineCacheUninitialized; + } else if (number_of_types == 1) { + return kInlineCacheMonomorphic; + } else if (number_of_types == InlineCache::kIndividualCacheSize) { + return kInlineCacheMegamorphic; + } else { + return kInlineCachePolymorphic; + } } static mirror::Class* GetMonomorphicType(Handle<mirror::ObjectArray<mirror::Class>> classes) @@ -271,18 +347,6 @@ static mirror::Class* GetMonomorphicType(Handle<mirror::ObjectArray<mirror::Clas return classes->Get(0); } -static bool IsUninitialized(Handle<mirror::ObjectArray<mirror::Class>> classes) - REQUIRES_SHARED(Locks::mutator_lock_) { - return classes->Get(0) == nullptr; -} - -static bool IsPolymorphic(Handle<mirror::ObjectArray<mirror::Class>> classes) - REQUIRES_SHARED(Locks::mutator_lock_) { - DCHECK_GE(InlineCache::kIndividualCacheSize, 3); - return classes->Get(1) != nullptr && - classes->Get(InlineCache::kIndividualCacheSize - 1) == nullptr; -} - ArtMethod* HInliner::TryCHADevirtualization(ArtMethod* resolved_method) { if (!resolved_method->HasSingleImplementation()) { return nullptr; @@ -296,7 +360,24 @@ ArtMethod* HInliner::TryCHADevirtualization(ArtMethod* resolved_method) { return nullptr; } PointerSize pointer_size = caller_compilation_unit_.GetClassLinker()->GetImagePointerSize(); - return resolved_method->GetSingleImplementation(pointer_size); + ArtMethod* single_impl = resolved_method->GetSingleImplementation(pointer_size); + if (single_impl == nullptr) { + return nullptr; + } + if (single_impl->IsProxyMethod()) { + // Proxy method is a generic invoker that's not worth + // devirtualizing/inlining. It also causes issues when the proxy + // method is in another dex file if we try to rewrite invoke-interface to + // invoke-virtual because a proxy method doesn't have a real dex file. + return nullptr; + } + if (!single_impl->GetDeclaringClass()->IsResolved()) { + // There's a race with the class loading, which updates the CHA info + // before setting the class to resolved. So we just bail for this + // rare occurence. + return nullptr; + } + return single_impl; } bool HInliner::TryInline(HInvoke* invoke_instruction) { @@ -309,17 +390,18 @@ bool HInliner::TryInline(HInvoke* invoke_instruction) { ScopedObjectAccess soa(Thread::Current()); uint32_t method_index = invoke_instruction->GetDexMethodIndex(); const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile(); - VLOG(compiler) << "Try inlining " << caller_dex_file.PrettyMethod(method_index); + LOG_TRY() << caller_dex_file.PrettyMethod(method_index); - // We can query the dex cache directly. The verifier has populated it already. ArtMethod* resolved_method = invoke_instruction->GetResolvedMethod(); - ArtMethod* actual_method = nullptr; if (resolved_method == nullptr) { DCHECK(invoke_instruction->IsInvokeStaticOrDirect()); DCHECK(invoke_instruction->AsInvokeStaticOrDirect()->IsStringInit()); - VLOG(compiler) << "Not inlining a String.<init> method"; + LOG_FAIL_NO_STAT() << "Not inlining a String.<init> method"; return false; - } else if (invoke_instruction->IsInvokeStaticOrDirect()) { + } + ArtMethod* actual_method = nullptr; + + if (invoke_instruction->IsInvokeStaticOrDirect()) { actual_method = resolved_method; } else { // Check if we can statically find the method. @@ -332,6 +414,7 @@ bool HInliner::TryInline(HInvoke* invoke_instruction) { if (method != nullptr) { cha_devirtualize = true; actual_method = method; + LOG_NOTE() << "Try CHA-based inlining of " << actual_method->PrettyMethod(); } } @@ -353,67 +436,226 @@ bool HInliner::TryInline(HInvoke* invoke_instruction) { } return result; } - DCHECK(!invoke_instruction->IsInvokeStaticOrDirect()); - // Check if we can use an inline cache. - ArtMethod* caller = graph_->GetArtMethod(); - if (Runtime::Current()->UseJitCompilation()) { - // Under JIT, we should always know the caller. - DCHECK(caller != nullptr); - ScopedProfilingInfoInlineUse spiis(caller, soa.Self()); - ProfilingInfo* profiling_info = spiis.GetProfilingInfo(); - if (profiling_info != nullptr) { - StackHandleScope<1> hs(soa.Self()); - ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); - Handle<mirror::ObjectArray<mirror::Class>> inline_cache = hs.NewHandle( - mirror::ObjectArray<mirror::Class>::Alloc( - soa.Self(), - class_linker->GetClassRoot(ClassLinker::kClassArrayClass), - InlineCache::kIndividualCacheSize)); - if (inline_cache == nullptr) { - // We got an OOME. Just clear the exception, and don't inline. - DCHECK(soa.Self()->IsExceptionPending()); - soa.Self()->ClearException(); - VLOG(compiler) << "Out of memory in the compiler when trying to inline"; - return false; + // Try using inline caches. + return TryInlineFromInlineCache(caller_dex_file, invoke_instruction, resolved_method); +} + +static Handle<mirror::ObjectArray<mirror::Class>> AllocateInlineCacheHolder( + const DexCompilationUnit& compilation_unit, + StackHandleScope<1>* hs) + REQUIRES_SHARED(Locks::mutator_lock_) { + Thread* self = Thread::Current(); + ClassLinker* class_linker = compilation_unit.GetClassLinker(); + Handle<mirror::ObjectArray<mirror::Class>> inline_cache = hs->NewHandle( + mirror::ObjectArray<mirror::Class>::Alloc( + self, + class_linker->GetClassRoot(ClassLinker::kClassArrayClass), + InlineCache::kIndividualCacheSize)); + if (inline_cache == nullptr) { + // We got an OOME. Just clear the exception, and don't inline. + DCHECK(self->IsExceptionPending()); + self->ClearException(); + VLOG(compiler) << "Out of memory in the compiler when trying to inline"; + } + return inline_cache; +} + +bool HInliner::TryInlineFromInlineCache(const DexFile& caller_dex_file, + HInvoke* invoke_instruction, + ArtMethod* resolved_method) + REQUIRES_SHARED(Locks::mutator_lock_) { + if (Runtime::Current()->IsAotCompiler() && !kUseAOTInlineCaches) { + return false; + } + + StackHandleScope<1> hs(Thread::Current()); + Handle<mirror::ObjectArray<mirror::Class>> inline_cache; + InlineCacheType inline_cache_type = Runtime::Current()->IsAotCompiler() + ? GetInlineCacheAOT(caller_dex_file, invoke_instruction, &hs, &inline_cache) + : GetInlineCacheJIT(invoke_instruction, &hs, &inline_cache); + + switch (inline_cache_type) { + case kInlineCacheNoData: { + LOG_FAIL_NO_STAT() + << "Interface or virtual call to " + << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex()) + << " could not be statically determined"; + return false; + } + + case kInlineCacheUninitialized: { + LOG_FAIL_NO_STAT() + << "Interface or virtual call to " + << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex()) + << " is not hit and not inlined"; + return false; + } + + case kInlineCacheMonomorphic: { + MaybeRecordStat(kMonomorphicCall); + if (outermost_graph_->IsCompilingOsr()) { + // If we are compiling OSR, we pretend this call is polymorphic, as we may come from the + // interpreter and it may have seen different receiver types. + return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache); } else { - Runtime::Current()->GetJit()->GetCodeCache()->CopyInlineCacheInto( - *profiling_info->GetInlineCache(invoke_instruction->GetDexPc()), - inline_cache); - if (IsUninitialized(inline_cache)) { - VLOG(compiler) << "Interface or virtual call to " - << caller_dex_file.PrettyMethod(method_index) - << " is not hit and not inlined"; - return false; - } else if (IsMonomorphic(inline_cache)) { - MaybeRecordStat(kMonomorphicCall); - if (outermost_graph_->IsCompilingOsr()) { - // If we are compiling OSR, we pretend this call is polymorphic, as we may come from the - // interpreter and it may have seen different receiver types. - return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache); - } else { - return TryInlineMonomorphicCall(invoke_instruction, resolved_method, inline_cache); - } - } else if (IsPolymorphic(inline_cache)) { - MaybeRecordStat(kPolymorphicCall); - return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache); - } else { - DCHECK(IsMegamorphic(inline_cache)); - VLOG(compiler) << "Interface or virtual call to " - << caller_dex_file.PrettyMethod(method_index) - << " is megamorphic and not inlined"; - MaybeRecordStat(kMegamorphicCall); - return false; - } + return TryInlineMonomorphicCall(invoke_instruction, resolved_method, inline_cache); } } + + case kInlineCachePolymorphic: { + MaybeRecordStat(kPolymorphicCall); + return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache); + } + + case kInlineCacheMegamorphic: { + LOG_FAIL_NO_STAT() + << "Interface or virtual call to " + << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex()) + << " is megamorphic and not inlined"; + MaybeRecordStat(kMegamorphicCall); + return false; + } + + case kInlineCacheMissingTypes: { + LOG_FAIL_NO_STAT() + << "Interface or virtual call to " + << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex()) + << " is missing types and not inlined"; + return false; + } } + UNREACHABLE(); +} - VLOG(compiler) << "Interface or virtual call to " - << caller_dex_file.PrettyMethod(method_index) - << " could not be statically determined"; - return false; +HInliner::InlineCacheType HInliner::GetInlineCacheJIT( + HInvoke* invoke_instruction, + StackHandleScope<1>* hs, + /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache) + REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK(Runtime::Current()->UseJitCompilation()); + + ArtMethod* caller = graph_->GetArtMethod(); + // Under JIT, we should always know the caller. + DCHECK(caller != nullptr); + ScopedProfilingInfoInlineUse spiis(caller, Thread::Current()); + ProfilingInfo* profiling_info = spiis.GetProfilingInfo(); + + if (profiling_info == nullptr) { + return kInlineCacheNoData; + } + + *inline_cache = AllocateInlineCacheHolder(caller_compilation_unit_, hs); + if (inline_cache->Get() == nullptr) { + // We can't extract any data if we failed to allocate; + return kInlineCacheNoData; + } else { + Runtime::Current()->GetJit()->GetCodeCache()->CopyInlineCacheInto( + *profiling_info->GetInlineCache(invoke_instruction->GetDexPc()), + *inline_cache); + return GetInlineCacheType(*inline_cache); + } +} + +HInliner::InlineCacheType HInliner::GetInlineCacheAOT( + const DexFile& caller_dex_file, + HInvoke* invoke_instruction, + StackHandleScope<1>* hs, + /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache) + REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK(Runtime::Current()->IsAotCompiler()); + const ProfileCompilationInfo* pci = compiler_driver_->GetProfileCompilationInfo(); + if (pci == nullptr) { + return kInlineCacheNoData; + } + + ProfileCompilationInfo::OfflineProfileMethodInfo offline_profile; + bool found = pci->GetMethod(caller_dex_file.GetLocation(), + caller_dex_file.GetLocationChecksum(), + caller_compilation_unit_.GetDexMethodIndex(), + &offline_profile); + if (!found) { + return kInlineCacheNoData; // no profile information for this invocation. + } + + *inline_cache = AllocateInlineCacheHolder(caller_compilation_unit_, hs); + if (inline_cache == nullptr) { + // We can't extract any data if we failed to allocate; + return kInlineCacheNoData; + } else { + return ExtractClassesFromOfflineProfile(invoke_instruction, + offline_profile, + *inline_cache); + } +} + +HInliner::InlineCacheType HInliner::ExtractClassesFromOfflineProfile( + const HInvoke* invoke_instruction, + const ProfileCompilationInfo::OfflineProfileMethodInfo& offline_profile, + /*out*/Handle<mirror::ObjectArray<mirror::Class>> inline_cache) + REQUIRES_SHARED(Locks::mutator_lock_) { + const auto it = offline_profile.inline_caches.find(invoke_instruction->GetDexPc()); + if (it == offline_profile.inline_caches.end()) { + return kInlineCacheUninitialized; + } + + const ProfileCompilationInfo::DexPcData& dex_pc_data = it->second; + + if (dex_pc_data.is_missing_types) { + return kInlineCacheMissingTypes; + } + if (dex_pc_data.is_megamorphic) { + return kInlineCacheMegamorphic; + } + + DCHECK_LE(dex_pc_data.classes.size(), InlineCache::kIndividualCacheSize); + Thread* self = Thread::Current(); + // We need to resolve the class relative to the containing dex file. + // So first, build a mapping from the index of dex file in the profile to + // its dex cache. This will avoid repeating the lookup when walking over + // the inline cache types. + std::vector<ObjPtr<mirror::DexCache>> dex_profile_index_to_dex_cache( + offline_profile.dex_references.size()); + for (size_t i = 0; i < offline_profile.dex_references.size(); i++) { + bool found = false; + for (const DexFile* dex_file : compiler_driver_->GetDexFilesForOatFile()) { + if (offline_profile.dex_references[i].MatchesDex(dex_file)) { + dex_profile_index_to_dex_cache[i] = + caller_compilation_unit_.GetClassLinker()->FindDexCache(self, *dex_file); + found = true; + } + } + if (!found) { + VLOG(compiler) << "Could not find profiled dex file: " + << offline_profile.dex_references[i].dex_location; + return kInlineCacheMissingTypes; + } + } + + // Walk over the classes and resolve them. If we cannot find a type we return + // kInlineCacheMissingTypes. + int ic_index = 0; + for (const ProfileCompilationInfo::ClassReference& class_ref : dex_pc_data.classes) { + ObjPtr<mirror::DexCache> dex_cache = + dex_profile_index_to_dex_cache[class_ref.dex_profile_index]; + DCHECK(dex_cache != nullptr); + ObjPtr<mirror::Class> clazz = ClassLinker::LookupResolvedType( + class_ref.type_index, + dex_cache, + caller_compilation_unit_.GetClassLoader().Get()); + if (clazz != nullptr) { + inline_cache->Set(ic_index++, clazz); + } else { + VLOG(compiler) << "Could not resolve class from inline cache in AOT mode " + << caller_compilation_unit_.GetDexFile()->PrettyMethod( + invoke_instruction->GetDexMethodIndex()) << " : " + << caller_compilation_unit_ + .GetDexFile()->StringByTypeIdx(class_ref.type_index); + return kInlineCacheMissingTypes; + } + } + return GetInlineCacheType(inline_cache); } HInstanceFieldGet* HInliner::BuildGetReceiverClass(ClassLinker* class_linker, @@ -436,6 +678,32 @@ HInstanceFieldGet* HInliner::BuildGetReceiverClass(ClassLinker* class_linker, return result; } +static ArtMethod* ResolveMethodFromInlineCache(Handle<mirror::Class> klass, + ArtMethod* resolved_method, + HInstruction* invoke_instruction, + PointerSize pointer_size) + REQUIRES_SHARED(Locks::mutator_lock_) { + if (Runtime::Current()->IsAotCompiler()) { + // We can get unrelated types when working with profiles (corruption, + // systme updates, or anyone can write to it). So first check if the class + // actually implements the declaring class of the method that is being + // called in bytecode. + // Note: the lookup methods used below require to have assignable types. + if (!resolved_method->GetDeclaringClass()->IsAssignableFrom(klass.Get())) { + return nullptr; + } + } + + if (invoke_instruction->IsInvokeInterface()) { + resolved_method = klass->FindVirtualMethodForInterface(resolved_method, pointer_size); + } else { + DCHECK(invoke_instruction->IsInvokeVirtual()); + resolved_method = klass->FindVirtualMethodForVirtual(resolved_method, pointer_size); + } + DCHECK(resolved_method != nullptr); + return resolved_method; +} + bool HInliner::TryInlineMonomorphicCall(HInvoke* invoke_instruction, ArtMethod* resolved_method, Handle<mirror::ObjectArray<mirror::Class>> classes) { @@ -445,27 +713,29 @@ bool HInliner::TryInlineMonomorphicCall(HInvoke* invoke_instruction, dex::TypeIndex class_index = FindClassIndexIn( GetMonomorphicType(classes), caller_compilation_unit_); if (!class_index.IsValid()) { - VLOG(compiler) << "Call to " << ArtMethod::PrettyMethod(resolved_method) - << " from inline cache is not inlined because its class is not" - << " accessible to the caller"; + LOG_FAIL(kNotInlinedDexCache) + << "Call to " << ArtMethod::PrettyMethod(resolved_method) + << " from inline cache is not inlined because its class is not" + << " accessible to the caller"; return false; } ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); PointerSize pointer_size = class_linker->GetImagePointerSize(); - if (invoke_instruction->IsInvokeInterface()) { - resolved_method = GetMonomorphicType(classes)->FindVirtualMethodForInterface( - resolved_method, pointer_size); - } else { - DCHECK(invoke_instruction->IsInvokeVirtual()); - resolved_method = GetMonomorphicType(classes)->FindVirtualMethodForVirtual( - resolved_method, pointer_size); + Handle<mirror::Class> monomorphic_type = handles_->NewHandle(GetMonomorphicType(classes)); + resolved_method = ResolveMethodFromInlineCache( + monomorphic_type, resolved_method, invoke_instruction, pointer_size); + + LOG_NOTE() << "Try inline monomorphic call to " << resolved_method->PrettyMethod(); + if (resolved_method == nullptr) { + // Bogus AOT profile, bail. + DCHECK(Runtime::Current()->IsAotCompiler()); + return false; } - DCHECK(resolved_method != nullptr); + HInstruction* receiver = invoke_instruction->InputAt(0); HInstruction* cursor = invoke_instruction->GetPrevious(); HBasicBlock* bb_cursor = invoke_instruction->GetBlock(); - Handle<mirror::Class> monomorphic_type = handles_->NewHandle(GetMonomorphicType(classes)); if (!TryInlineAndReplace(invoke_instruction, resolved_method, ReferenceTypeInfo::Create(monomorphic_type, /* is_exact */ true), @@ -504,7 +774,8 @@ void HInliner::AddCHAGuard(HInstruction* invoke_instruction, HShouldDeoptimizeFlag(graph_->GetArena(), dex_pc); HInstruction* compare = new (graph_->GetArena()) HNotEqual( deopt_flag, graph_->GetIntConstant(0, dex_pc)); - HInstruction* deopt = new (graph_->GetArena()) HDeoptimize(compare, dex_pc); + HInstruction* deopt = new (graph_->GetArena()) HDeoptimize( + graph_->GetArena(), compare, HDeoptimize::Kind::kInline, dex_pc); if (cursor != nullptr) { bb_cursor->InsertInstructionAfter(deopt_flag, cursor); @@ -549,21 +820,35 @@ HInstruction* HInliner::AddTypeGuard(HInstruction* receiver, is_referrer, invoke_instruction->GetDexPc(), /* needs_access_check */ false); - HLoadClass::LoadKind kind = HSharpening::SharpenClass( + HLoadClass::LoadKind kind = HSharpening::ComputeLoadClassKind( load_class, codegen_, compiler_driver_, caller_compilation_unit_); DCHECK(kind != HLoadClass::LoadKind::kInvalid) << "We should always be able to reference a class for inline caches"; // Insert before setting the kind, as setting the kind affects the inputs. bb_cursor->InsertInstructionAfter(load_class, receiver_class); load_class->SetLoadKind(kind); + // In AOT mode, we will most likely load the class from BSS, which will involve a call + // to the runtime. In this case, the load instruction will need an environment so copy + // it from the invoke instruction. + if (load_class->NeedsEnvironment()) { + DCHECK(Runtime::Current()->IsAotCompiler()); + load_class->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); + } HNotEqual* compare = new (graph_->GetArena()) HNotEqual(load_class, receiver_class); bb_cursor->InsertInstructionAfter(compare, load_class); if (with_deoptimization) { HDeoptimize* deoptimize = new (graph_->GetArena()) HDeoptimize( - compare, invoke_instruction->GetDexPc()); + graph_->GetArena(), + compare, + receiver, + HDeoptimize::Kind::kInline, + invoke_instruction->GetDexPc()); bb_cursor->InsertInstructionAfter(deoptimize, compare); deoptimize->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); + DCHECK_EQ(invoke_instruction->InputAt(0), receiver); + receiver->ReplaceUsesDominatedBy(deoptimize, deoptimize); + deoptimize->SetReferenceTypeInfo(receiver->GetReferenceTypeInfo()); } return compare; } @@ -590,11 +875,14 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, ArtMethod* method = nullptr; Handle<mirror::Class> handle = handles_->NewHandle(classes->Get(i)); - if (invoke_instruction->IsInvokeInterface()) { - method = handle->FindVirtualMethodForInterface(resolved_method, pointer_size); - } else { - DCHECK(invoke_instruction->IsInvokeVirtual()); - method = handle->FindVirtualMethodForVirtual(resolved_method, pointer_size); + method = ResolveMethodFromInlineCache( + handle, resolved_method, invoke_instruction, pointer_size); + if (method == nullptr) { + DCHECK(Runtime::Current()->IsAotCompiler()); + // AOT profile is bogus. This loop expects to iterate over all entries, + // so just just continue. + all_targets_inlined = false; + continue; } HInstruction* receiver = invoke_instruction->InputAt(0); @@ -603,6 +891,7 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, dex::TypeIndex class_index = FindClassIndexIn(handle.Get(), caller_compilation_unit_); HInstruction* return_replacement = nullptr; + LOG_NOTE() << "Try inline polymorphic call to " << method->PrettyMethod(); if (!class_index.IsValid() || !TryBuildAndInline(invoke_instruction, method, @@ -612,8 +901,8 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, } else { one_target_inlined = true; - VLOG(compiler) << "Polymorphic call to " << ArtMethod::PrettyMethod(resolved_method) - << " has inlined " << ArtMethod::PrettyMethod(method); + LOG_SUCCESS() << "Polymorphic call to " << ArtMethod::PrettyMethod(resolved_method) + << " has inlined " << ArtMethod::PrettyMethod(method); // If we have inlined all targets before, and this receiver is the last seen, // we deoptimize instead of keeping the original invoke instruction. @@ -638,7 +927,7 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, } invoke_instruction->GetBlock()->RemoveInstruction(invoke_instruction); // Because the inline cache data can be populated concurrently, we force the end of the - // iteration. Otherhwise, we could see a new receiver type. + // iteration. Otherwise, we could see a new receiver type. break; } else { CreateDiamondPatternForPolymorphicInline(compare, return_replacement, invoke_instruction); @@ -647,9 +936,10 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, } if (!one_target_inlined) { - VLOG(compiler) << "Call to " << ArtMethod::PrettyMethod(resolved_method) - << " from inline cache is not inlined because none" - << " of its targets could be inlined"; + LOG_FAIL_NO_STAT() + << "Call to " << ArtMethod::PrettyMethod(resolved_method) + << " from inline cache is not inlined because none" + << " of its targets could be inlined"; return false; } @@ -746,11 +1036,10 @@ bool HInliner::TryInlinePolymorphicCallToSameTarget( ArtMethod* resolved_method, Handle<mirror::ObjectArray<mirror::Class>> classes) { // This optimization only works under JIT for now. - DCHECK(Runtime::Current()->UseJitCompilation()); - if (graph_->GetInstructionSet() == kMips64) { - // TODO: Support HClassTableGet for mips64. + if (!Runtime::Current()->UseJitCompilation()) { return false; } + ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); PointerSize pointer_size = class_linker->GetImagePointerSize(); @@ -784,9 +1073,6 @@ bool HInliner::TryInlinePolymorphicCallToSameTarget( actual_method = new_method; } else if (actual_method != new_method) { // Different methods, bailout. - VLOG(compiler) << "Call to " << ArtMethod::PrettyMethod(resolved_method) - << " from inline cache is not inlined because it resolves" - << " to different methods"; return false; } } @@ -840,13 +1126,19 @@ bool HInliner::TryInlinePolymorphicCallToSameTarget( CreateDiamondPatternForPolymorphicInline(compare, return_replacement, invoke_instruction); } else { HDeoptimize* deoptimize = new (graph_->GetArena()) HDeoptimize( - compare, invoke_instruction->GetDexPc()); + graph_->GetArena(), + compare, + receiver, + HDeoptimize::Kind::kInline, + invoke_instruction->GetDexPc()); bb_cursor->InsertInstructionAfter(deoptimize, compare); deoptimize->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); if (return_replacement != nullptr) { invoke_instruction->ReplaceWith(return_replacement); } + receiver->ReplaceUsesDominatedBy(deoptimize, deoptimize); invoke_instruction->GetBlock()->RemoveInstruction(invoke_instruction); + deoptimize->SetReferenceTypeInfo(receiver->GetReferenceTypeInfo()); } // Run type propagation to get the guard typed. @@ -859,6 +1151,7 @@ bool HInliner::TryInlinePolymorphicCallToSameTarget( MaybeRecordStat(kInlinedPolymorphicCall); + LOG_SUCCESS() << "Inlined same polymorphic target " << actual_method->PrettyMethod(); return true; } @@ -873,11 +1166,23 @@ bool HInliner::TryInlineAndReplace(HInvoke* invoke_instruction, HBasicBlock* bb_cursor = invoke_instruction->GetBlock(); if (!TryBuildAndInline(invoke_instruction, method, receiver_type, &return_replacement)) { if (invoke_instruction->IsInvokeInterface()) { + DCHECK(!method->IsProxyMethod()); // Turn an invoke-interface into an invoke-virtual. An invoke-virtual is always // better than an invoke-interface because: // 1) In the best case, the interface call has one more indirection (to fetch the IMT). // 2) We will not go to the conflict trampoline with an invoke-virtual. // TODO: Consider sharpening once it is not dependent on the compiler driver. + + if (method->IsDefault() && !method->IsCopied()) { + // Changing to invoke-virtual cannot be done on an original default method + // since it's not in any vtable. Devirtualization by exact type/inline-cache + // always uses a method in the iftable which is never an original default + // method. + // On the other hand, inlining an original default method by CHA is fine. + DCHECK(cha_devirtualize); + return false; + } + const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile(); uint32_t dex_method_index = FindMethodIndexIn( method, caller_dex_file, invoke_instruction->GetDexMethodIndex()); @@ -928,13 +1233,34 @@ bool HInliner::TryInlineAndReplace(HInvoke* invoke_instruction, return true; } +size_t HInliner::CountRecursiveCallsOf(ArtMethod* method) const { + const HInliner* current = this; + size_t count = 0; + do { + if (current->graph_->GetArtMethod() == method) { + ++count; + } + current = current->parent_; + } while (current != nullptr); + return count; +} + bool HInliner::TryBuildAndInline(HInvoke* invoke_instruction, ArtMethod* method, ReferenceTypeInfo receiver_type, HInstruction** return_replacement) { if (method->IsProxyMethod()) { - VLOG(compiler) << "Method " << method->PrettyMethod() - << " is not inlined because of unimplemented inline support for proxy methods."; + LOG_FAIL(kNotInlinedProxy) + << "Method " << method->PrettyMethod() + << " is not inlined because of unimplemented inline support for proxy methods."; + return false; + } + + if (CountRecursiveCallsOf(method) > kMaximumNumberOfRecursiveCalls) { + LOG_FAIL(kNotInlinedRecursiveBudget) + << "Method " + << method->PrettyMethod() + << " is not inlined because it has reached its recursive call budget."; return false; } @@ -943,15 +1269,16 @@ bool HInliner::TryBuildAndInline(HInvoke* invoke_instruction, if (!compiler_driver_->MayInline(method->GetDexFile(), outer_compilation_unit_.GetDexFile())) { if (TryPatternSubstitution(invoke_instruction, method, return_replacement)) { - VLOG(compiler) << "Successfully replaced pattern of invoke " - << method->PrettyMethod(); + LOG_SUCCESS() << "Successfully replaced pattern of invoke " + << method->PrettyMethod(); MaybeRecordStat(kReplacedInvokeWithSimplePattern); return true; } - VLOG(compiler) << "Won't inline " << method->PrettyMethod() << " in " - << outer_compilation_unit_.GetDexFile()->GetLocation() << " (" - << caller_compilation_unit_.GetDexFile()->GetLocation() << ") from " - << method->GetDexFile()->GetLocation(); + LOG_FAIL(kNotInlinedWont) + << "Won't inline " << method->PrettyMethod() << " in " + << outer_compilation_unit_.GetDexFile()->GetLocation() << " (" + << caller_compilation_unit_.GetDexFile()->GetLocation() << ") from " + << method->GetDexFile()->GetLocation(); return false; } @@ -960,30 +1287,32 @@ bool HInliner::TryBuildAndInline(HInvoke* invoke_instruction, const DexFile::CodeItem* code_item = method->GetCodeItem(); if (code_item == nullptr) { - VLOG(compiler) << "Method " << method->PrettyMethod() - << " is not inlined because it is native"; + LOG_FAIL_NO_STAT() + << "Method " << method->PrettyMethod() << " is not inlined because it is native"; return false; } size_t inline_max_code_units = compiler_driver_->GetCompilerOptions().GetInlineMaxCodeUnits(); if (code_item->insns_size_in_code_units_ > inline_max_code_units) { - VLOG(compiler) << "Method " << method->PrettyMethod() - << " is too big to inline: " - << code_item->insns_size_in_code_units_ - << " > " - << inline_max_code_units; + LOG_FAIL(kNotInlinedCodeItem) + << "Method " << method->PrettyMethod() + << " is not inlined because its code item is too big: " + << code_item->insns_size_in_code_units_ + << " > " + << inline_max_code_units; return false; } if (code_item->tries_size_ != 0) { - VLOG(compiler) << "Method " << method->PrettyMethod() - << " is not inlined because of try block"; + LOG_FAIL(kNotInlinedTryCatch) + << "Method " << method->PrettyMethod() << " is not inlined because of try block"; return false; } if (!method->IsCompilable()) { - VLOG(compiler) << "Method " << method->PrettyMethod() - << " has soft failures un-handled by the compiler, so it cannot be inlined"; + LOG_FAIL(kNotInlinedNotVerified) + << "Method " << method->PrettyMethod() + << " has soft failures un-handled by the compiler, so it cannot be inlined"; } if (!method->GetDeclaringClass()->IsVerified()) { @@ -991,8 +1320,9 @@ bool HInliner::TryBuildAndInline(HInvoke* invoke_instruction, if (Runtime::Current()->UseJitCompilation() || !compiler_driver_->IsMethodVerifiedWithoutFailures( method->GetDexMethodIndex(), class_def_idx, *method->GetDexFile())) { - VLOG(compiler) << "Method " << method->PrettyMethod() - << " couldn't be verified, so it cannot be inlined"; + LOG_FAIL(kNotInlinedNotVerified) + << "Method " << method->PrettyMethod() + << " couldn't be verified, so it cannot be inlined"; return false; } } @@ -1001,9 +1331,9 @@ bool HInliner::TryBuildAndInline(HInvoke* invoke_instruction, invoke_instruction->AsInvokeStaticOrDirect()->IsStaticWithImplicitClinitCheck()) { // Case of a static method that cannot be inlined because it implicitly // requires an initialization check of its declaring class. - VLOG(compiler) << "Method " << method->PrettyMethod() - << " is not inlined because it is static and requires a clinit" - << " check that cannot be emitted due to Dex cache limitations"; + LOG_FAIL(kNotInlinedDexCache) << "Method " << method->PrettyMethod() + << " is not inlined because it is static and requires a clinit" + << " check that cannot be emitted due to Dex cache limitations"; return false; } @@ -1012,7 +1342,7 @@ bool HInliner::TryBuildAndInline(HInvoke* invoke_instruction, return false; } - VLOG(compiler) << "Successfully inlined " << method->PrettyMethod(); + LOG_SUCCESS() << method->PrettyMethod(); MaybeRecordStat(kInlinedInvoke); return true; } @@ -1064,9 +1394,8 @@ bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction, // TODO: Needs null check. return false; } - Handle<mirror::DexCache> dex_cache(handles_->NewHandle(resolved_method->GetDexCache())); HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg); - HInstanceFieldGet* iget = CreateInstanceFieldGet(dex_cache, data.field_idx, obj); + HInstanceFieldGet* iget = CreateInstanceFieldGet(data.field_idx, resolved_method, obj); DCHECK_EQ(iget->GetFieldOffset().Uint32Value(), data.field_offset); DCHECK_EQ(iget->IsVolatile() ? 1u : 0u, data.is_volatile); invoke_instruction->GetBlock()->InsertInstructionBefore(iget, invoke_instruction); @@ -1079,10 +1408,9 @@ bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction, // TODO: Needs null check. return false; } - Handle<mirror::DexCache> dex_cache(handles_->NewHandle(resolved_method->GetDexCache())); HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg); HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, data.src_arg); - HInstanceFieldSet* iput = CreateInstanceFieldSet(dex_cache, data.field_idx, obj, value); + HInstanceFieldSet* iput = CreateInstanceFieldSet(data.field_idx, resolved_method, obj, value); DCHECK_EQ(iput->GetFieldOffset().Uint32Value(), data.field_offset); DCHECK_EQ(iput->IsVolatile() ? 1u : 0u, data.is_volatile); invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction); @@ -1116,24 +1444,19 @@ bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction, [](uint16_t index) { return index != DexFile::kDexNoIndex16; })); // Create HInstanceFieldSet for each IPUT that stores non-zero data. - Handle<mirror::DexCache> dex_cache; HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, /* this */ 0u); bool needs_constructor_barrier = false; for (size_t i = 0; i != number_of_iputs; ++i) { HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, iput_args[i]); if (!value->IsConstant() || !value->AsConstant()->IsZeroBitPattern()) { - if (dex_cache.GetReference() == nullptr) { - dex_cache = handles_->NewHandle(resolved_method->GetDexCache()); - } uint16_t field_index = iput_field_indexes[i]; - HInstanceFieldSet* iput = CreateInstanceFieldSet(dex_cache, field_index, obj, value); + bool is_final; + HInstanceFieldSet* iput = + CreateInstanceFieldSet(field_index, resolved_method, obj, value, &is_final); invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction); // Check whether the field is final. If it is, we need to add a barrier. - PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet()); - ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size); - DCHECK(resolved_field != nullptr); - if (resolved_field->IsFinal()) { + if (is_final) { needs_constructor_barrier = true; } } @@ -1152,12 +1475,13 @@ bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction, return true; } -HInstanceFieldGet* HInliner::CreateInstanceFieldGet(Handle<mirror::DexCache> dex_cache, - uint32_t field_index, +HInstanceFieldGet* HInliner::CreateInstanceFieldGet(uint32_t field_index, + ArtMethod* referrer, HInstruction* obj) REQUIRES_SHARED(Locks::mutator_lock_) { - PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet()); - ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + ArtField* resolved_field = + class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false); DCHECK(resolved_field != nullptr); HInstanceFieldGet* iget = new (graph_->GetArena()) HInstanceFieldGet( obj, @@ -1167,12 +1491,13 @@ HInstanceFieldGet* HInliner::CreateInstanceFieldGet(Handle<mirror::DexCache> dex resolved_field->IsVolatile(), field_index, resolved_field->GetDeclaringClass()->GetDexClassDefIndex(), - *dex_cache->GetDexFile(), + *referrer->GetDexFile(), // Read barrier generates a runtime call in slow path and we need a valid // dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537. /* dex_pc */ 0); if (iget->GetType() == Primitive::kPrimNot) { // Use the same dex_cache that we used for field lookup as the hint_dex_cache. + Handle<mirror::DexCache> dex_cache = handles_->NewHandle(referrer->GetDexCache()); ReferenceTypePropagation rtp(graph_, outer_compilation_unit_.GetClassLoader(), dex_cache, @@ -1183,14 +1508,21 @@ HInstanceFieldGet* HInliner::CreateInstanceFieldGet(Handle<mirror::DexCache> dex return iget; } -HInstanceFieldSet* HInliner::CreateInstanceFieldSet(Handle<mirror::DexCache> dex_cache, - uint32_t field_index, +HInstanceFieldSet* HInliner::CreateInstanceFieldSet(uint32_t field_index, + ArtMethod* referrer, HInstruction* obj, - HInstruction* value) + HInstruction* value, + bool* is_final) REQUIRES_SHARED(Locks::mutator_lock_) { - PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet()); - ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + ArtField* resolved_field = + class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false); DCHECK(resolved_field != nullptr); + if (is_final != nullptr) { + // This information is needed only for constructors. + DCHECK(referrer->IsConstructor()); + *is_final = resolved_field->IsFinal(); + } HInstanceFieldSet* iput = new (graph_->GetArena()) HInstanceFieldSet( obj, value, @@ -1200,7 +1532,7 @@ HInstanceFieldSet* HInliner::CreateInstanceFieldSet(Handle<mirror::DexCache> dex resolved_field->IsVolatile(), field_index, resolved_field->GetDeclaringClass()->GetDexClassDefIndex(), - *dex_cache->GetDexFile(), + *referrer->GetDexFile(), // Read barrier generates a runtime call in slow path and we need a valid // dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537. /* dex_pc */ 0); @@ -1298,15 +1630,17 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, handles_); if (builder.BuildGraph() != kAnalysisSuccess) { - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " could not be built, so cannot be inlined"; + LOG_FAIL(kNotInlinedCannotBuild) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " could not be built, so cannot be inlined"; return false; } if (!RegisterAllocator::CanAllocateRegistersFor(*callee_graph, compiler_driver_->GetInstructionSet())) { - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " cannot be inlined because of the register allocator"; + LOG_FAIL(kNotInlinedRegisterAllocator) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " cannot be inlined because of the register allocator"; return false; } @@ -1353,15 +1687,13 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, /* is_first_run */ false).Run(); } - size_t number_of_instructions_budget = kMaximumNumberOfHInstructions; - size_t number_of_inlined_instructions = - RunOptimizations(callee_graph, code_item, dex_compilation_unit); - number_of_instructions_budget += number_of_inlined_instructions; + RunOptimizations(callee_graph, code_item, dex_compilation_unit); HBasicBlock* exit_block = callee_graph->GetExitBlock(); if (exit_block == nullptr) { - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " could not be inlined because it has an infinite loop"; + LOG_FAIL(kNotInlinedInfiniteLoop) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " could not be inlined because it has an infinite loop"; return false; } @@ -1370,15 +1702,24 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, if (predecessor->GetLastInstruction()->IsThrow()) { if (invoke_instruction->GetBlock()->IsTryBlock()) { // TODO(ngeoffray): Support adding HTryBoundary in Hgraph::InlineInto. - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " could not be inlined because one branch always throws and" - << " caller is in a try/catch block"; + LOG_FAIL(kNotInlinedTryCatch) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " could not be inlined because one branch always throws and" + << " caller is in a try/catch block"; return false; } else if (graph_->GetExitBlock() == nullptr) { // TODO(ngeoffray): Support adding HExit in the caller graph. + LOG_FAIL(kNotInlinedInfiniteLoop) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " could not be inlined because one branch always throws and" + << " caller does not have an exit block"; + return false; + } else if (graph_->HasIrreducibleLoops()) { + // TODO(ngeoffray): Support re-computing loop information to graphs with + // irreducible loops? VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) << " could not be inlined because one branch always throws and" - << " caller does not have an exit block"; + << " caller has irreducible loops"; return false; } } else { @@ -1387,32 +1728,31 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, } if (!has_one_return) { - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " could not be inlined because it always throws"; + LOG_FAIL(kNotInlinedAlwaysThrows) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " could not be inlined because it always throws"; return false; } size_t number_of_instructions = 0; - - bool can_inline_environment = - total_number_of_dex_registers_ < kMaximumNumberOfCumulatedDexRegisters; - // Skip the entry block, it does not contain instructions that prevent inlining. for (HBasicBlock* block : callee_graph->GetReversePostOrderSkipEntryBlock()) { if (block->IsLoopHeader()) { if (block->GetLoopInformation()->IsIrreducible()) { // Don't inline methods with irreducible loops, they could prevent some // optimizations to run. - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " could not be inlined because it contains an irreducible loop"; + LOG_FAIL(kNotInlinedIrreducibleLoop) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " could not be inlined because it contains an irreducible loop"; return false; } if (!block->GetLoopInformation()->HasExitEdge()) { // Don't inline methods with loops without exit, since they cause the // loop information to be computed incorrectly when updating after // inlining. - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " could not be inlined because it contains a loop with no exit"; + LOG_FAIL(kNotInlinedLoopWithoutExit) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " could not be inlined because it contains a loop with no exit"; return false; } } @@ -1420,34 +1760,39 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, for (HInstructionIterator instr_it(block->GetInstructions()); !instr_it.Done(); instr_it.Advance()) { - if (number_of_instructions++ == number_of_instructions_budget) { - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " is not inlined because its caller has reached" - << " its instruction budget limit."; + if (++number_of_instructions >= inlining_budget_) { + LOG_FAIL(kNotInlinedInstructionBudget) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " is not inlined because the outer method has reached" + << " its instruction budget limit."; return false; } HInstruction* current = instr_it.Current(); - if (!can_inline_environment && current->NeedsEnvironment()) { - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " is not inlined because its caller has reached" - << " its environment budget limit."; + if (current->NeedsEnvironment() && + (total_number_of_dex_registers_ >= kMaximumNumberOfCumulatedDexRegisters)) { + LOG_FAIL(kNotInlinedEnvironmentBudget) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " is not inlined because its caller has reached" + << " its environment budget limit."; return false; } if (current->NeedsEnvironment() && !CanEncodeInlinedMethodInStackMap(*caller_compilation_unit_.GetDexFile(), resolved_method)) { - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " could not be inlined because " << current->DebugName() - << " needs an environment, is in a different dex file" - << ", and cannot be encoded in the stack maps."; + LOG_FAIL(kNotInlinedStackMaps) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " could not be inlined because " << current->DebugName() + << " needs an environment, is in a different dex file" + << ", and cannot be encoded in the stack maps."; return false; } if (!same_dex_file && current->NeedsDexCacheOfDeclaringClass()) { - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " could not be inlined because " << current->DebugName() - << " it is in a different dex file and requires access to the dex cache"; + LOG_FAIL(kNotInlinedDexCache) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " could not be inlined because " << current->DebugName() + << " it is in a different dex file and requires access to the dex cache"; return false; } @@ -1456,21 +1801,24 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, current->IsUnresolvedStaticFieldSet() || current->IsUnresolvedInstanceFieldSet()) { // Entrypoint for unresolved fields does not handle inlined frames. - VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) - << " could not be inlined because it is using an unresolved" - << " entrypoint"; + LOG_FAIL(kNotInlinedUnresolvedEntrypoint) + << "Method " << callee_dex_file.PrettyMethod(method_index) + << " could not be inlined because it is using an unresolved" + << " entrypoint"; return false; } } } - number_of_inlined_instructions_ += number_of_instructions; - DCHECK_EQ(caller_instruction_counter, graph_->GetCurrentInstructionId()) << "No instructions can be added to the outer graph while inner graph is being built"; + // Inline the callee graph inside the caller graph. const int32_t callee_instruction_counter = callee_graph->GetCurrentInstructionId(); graph_->SetCurrentInstructionId(callee_instruction_counter); *return_replacement = callee_graph->InlineInto(graph_, invoke_instruction); + // Update our budget for other inlining attempts in `caller_graph`. + total_number_of_instructions_ += number_of_instructions; + UpdateInliningBudget(); DCHECK_EQ(callee_instruction_counter, callee_graph->GetCurrentInstructionId()) << "No instructions can be added to the inner graph during inlining into the outer graph"; @@ -1483,15 +1831,15 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, return true; } -size_t HInliner::RunOptimizations(HGraph* callee_graph, - const DexFile::CodeItem* code_item, - const DexCompilationUnit& dex_compilation_unit) { +void HInliner::RunOptimizations(HGraph* callee_graph, + const DexFile::CodeItem* code_item, + const DexCompilationUnit& dex_compilation_unit) { // Note: if the outermost_graph_ is being compiled OSR, we should not run any // optimization that could lead to a HDeoptimize. The following optimizations do not. HDeadCodeElimination dce(callee_graph, inline_stats_, "dead_code_elimination$inliner"); HConstantFolding fold(callee_graph, "constant_folding$inliner"); HSharpening sharpening(callee_graph, codegen_, dex_compilation_unit, compiler_driver_, handles_); - InstructionSimplifier simplify(callee_graph, inline_stats_); + InstructionSimplifier simplify(callee_graph, codegen_, inline_stats_); IntrinsicsRecognizer intrinsics(callee_graph, inline_stats_); HOptimization* optimizations[] = { @@ -1507,23 +1855,37 @@ size_t HInliner::RunOptimizations(HGraph* callee_graph, optimization->Run(); } - size_t number_of_inlined_instructions = 0u; - if (depth_ + 1 < compiler_driver_->GetCompilerOptions().GetInlineDepthLimit()) { - HInliner inliner(callee_graph, - outermost_graph_, - codegen_, - outer_compilation_unit_, - dex_compilation_unit, - compiler_driver_, - handles_, - inline_stats_, - total_number_of_dex_registers_ + code_item->registers_size_, - depth_ + 1); - inliner.Run(); - number_of_inlined_instructions += inliner.number_of_inlined_instructions_; + // Bail early for pathological cases on the environment (for example recursive calls, + // or too large environment). + if (total_number_of_dex_registers_ >= kMaximumNumberOfCumulatedDexRegisters) { + LOG_NOTE() << "Calls in " << callee_graph->GetArtMethod()->PrettyMethod() + << " will not be inlined because the outer method has reached" + << " its environment budget limit."; + return; + } + + // Bail early if we know we already are over the limit. + size_t number_of_instructions = CountNumberOfInstructions(callee_graph); + if (number_of_instructions > inlining_budget_) { + LOG_NOTE() << "Calls in " << callee_graph->GetArtMethod()->PrettyMethod() + << " will not be inlined because the outer method has reached" + << " its instruction budget limit. " << number_of_instructions; + return; } - return number_of_inlined_instructions; + HInliner inliner(callee_graph, + outermost_graph_, + codegen_, + outer_compilation_unit_, + dex_compilation_unit, + compiler_driver_, + handles_, + inline_stats_, + total_number_of_dex_registers_ + code_item->registers_size_, + total_number_of_instructions_ + number_of_instructions, + this, + depth_ + 1); + inliner.Run(); } static bool IsReferenceTypeRefinement(ReferenceTypeInfo declared_rti, |