Move dex register check upwards in the Inliner stack

We can check that before we try to build and run optimizations

Bug: 200671122
Test: ART tests
Change-Id: Ie3b60445a7ed90490ad6a97834ae74631e26ff93

1 files changed, 16 insertions, 27 deletions

diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc
index 6331a70dd5..2e06117b61 100644
--- a/compiler/optimizing/inliner.cc
+++ b/compiler/optimizing/inliner.cc
@@ -433,6 +433,12 @@ static bool AlwaysThrows(ArtMethod* method)
 bool HInliner::TryInline(HInvoke* invoke_instruction) {
   MaybeRecordStat(stats_, MethodCompilationStat::kTryInline);
 
+  // Don't bother to move further if the outer method has too many registers.
+  if (total_number_of_dex_registers_ > kMaximumNumberOfCumulatedDexRegisters) {
+    MaybeRecordStat(stats_, MethodCompilationStat::kNotInlinedEnvironmentBudget);
+    return false;
+  }
+
   // Don't bother to move further if we know the method is unresolved or the invocation is
   // polymorphic (invoke-{polymorphic,custom}).
   if (invoke_instruction->IsInvokeUnresolved()) {
@@ -1845,24 +1851,16 @@ bool HInliner::CanInlineBody(const HGraph* callee_graph,
         return false;
       }
       HInstruction* current = instr_it.Current();
-      if (current->NeedsEnvironment() &&
-          (total_number_of_dex_registers_ > kMaximumNumberOfCumulatedDexRegisters)) {
-        LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedEnvironmentBudget)
-            << "Method " << resolved_method->PrettyMethod()
-            << " is not inlined because its caller has reached"
-            << " its environment budget limit.";
-        return false;
-      }
-
-      if (current->NeedsEnvironment() &&
-          !CanEncodeInlinedMethodInStackMap(*caller_compilation_unit_.GetDexFile(),
-                                            resolved_method)) {
-        LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedStackMaps)
-            << "Method " << resolved_method->PrettyMethod()
-            << " could not be inlined because " << current->DebugName()
-            << " needs an environment, is in a different dex file"
-            << ", and cannot be encoded in the stack maps.";
-        return false;
+      if (current->NeedsEnvironment()) {
+        DCHECK_LE(total_number_of_dex_registers_, kMaximumNumberOfCumulatedDexRegisters);
+        if (!CanEncodeInlinedMethodInStackMap(*caller_compilation_unit_.GetDexFile(),
+                                              resolved_method)) {
+          LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedStackMaps)
+              << "Method " << resolved_method->PrettyMethod() << " could not be inlined because "
+              << current->DebugName() << " needs an environment, is in a different dex file"
+              << ", and cannot be encoded in the stack maps.";
+          return false;
+        }
       }
 
       if (current->IsUnresolvedStaticFieldGet() ||
@@ -2030,15 +2028,6 @@ void HInliner::RunOptimizations(HGraph* callee_graph,
     optimization->Run();
   }
 
-  // Bail early for pathological cases on the environment (for example recursive calls,
-  // or too large environment).
-  if (total_number_of_dex_registers_ > kMaximumNumberOfCumulatedDexRegisters) {
-    LOG_NOTE() << "Calls in " << callee_graph->GetArtMethod()->PrettyMethod()
-             << " will not be inlined because the outer method has reached"
-             << " its environment budget limit.";
-    return;
-  }
-
   // Bail early if we know we already are over the limit.
   size_t number_of_instructions = CountNumberOfInstructions(callee_graph);
   if (number_of_instructions > inlining_budget_) {