Revert "Move dex register check upwards in the Inliner stack"

This reverts commit 7a7a1e2c826dc5f73df00936f98c21d8add7022c.

Reason for revert: We only want to check the register count if we have to create an environment

Bug: 200671122
Change-Id: I26af9075248f5b82f7118c6af9c6e758b7c14280
diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc
index 2e06117..6331a70 100644
--- a/compiler/optimizing/inliner.cc
+++ b/compiler/optimizing/inliner.cc
@@ -433,12 +433,6 @@
 bool HInliner::TryInline(HInvoke* invoke_instruction) {
   MaybeRecordStat(stats_, MethodCompilationStat::kTryInline);
 
-  // Don't bother to move further if the outer method has too many registers.
-  if (total_number_of_dex_registers_ > kMaximumNumberOfCumulatedDexRegisters) {
-    MaybeRecordStat(stats_, MethodCompilationStat::kNotInlinedEnvironmentBudget);
-    return false;
-  }
-
   // Don't bother to move further if we know the method is unresolved or the invocation is
   // polymorphic (invoke-{polymorphic,custom}).
   if (invoke_instruction->IsInvokeUnresolved()) {
@@ -1851,16 +1845,24 @@
         return false;
       }
       HInstruction* current = instr_it.Current();
-      if (current->NeedsEnvironment()) {
-        DCHECK_LE(total_number_of_dex_registers_, kMaximumNumberOfCumulatedDexRegisters);
-        if (!CanEncodeInlinedMethodInStackMap(*caller_compilation_unit_.GetDexFile(),
-                                              resolved_method)) {
-          LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedStackMaps)
-              << "Method " << resolved_method->PrettyMethod() << " could not be inlined because "
-              << current->DebugName() << " needs an environment, is in a different dex file"
-              << ", and cannot be encoded in the stack maps.";
-          return false;
-        }
+      if (current->NeedsEnvironment() &&
+          (total_number_of_dex_registers_ > kMaximumNumberOfCumulatedDexRegisters)) {
+        LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedEnvironmentBudget)
+            << "Method " << resolved_method->PrettyMethod()
+            << " is not inlined because its caller has reached"
+            << " its environment budget limit.";
+        return false;
+      }
+
+      if (current->NeedsEnvironment() &&
+          !CanEncodeInlinedMethodInStackMap(*caller_compilation_unit_.GetDexFile(),
+                                            resolved_method)) {
+        LOG_FAIL(stats_, MethodCompilationStat::kNotInlinedStackMaps)
+            << "Method " << resolved_method->PrettyMethod()
+            << " could not be inlined because " << current->DebugName()
+            << " needs an environment, is in a different dex file"
+            << ", and cannot be encoded in the stack maps.";
+        return false;
       }
 
       if (current->IsUnresolvedStaticFieldGet() ||
@@ -2028,6 +2030,15 @@
     optimization->Run();
   }
 
+  // Bail early for pathological cases on the environment (for example recursive calls,
+  // or too large environment).
+  if (total_number_of_dex_registers_ > kMaximumNumberOfCumulatedDexRegisters) {
+    LOG_NOTE() << "Calls in " << callee_graph->GetArtMethod()->PrettyMethod()
+             << " will not be inlined because the outer method has reached"
+             << " its environment budget limit.";
+    return;
+  }
+
   // Bail early if we know we already are over the limit.
   size_t number_of_instructions = CountNumberOfInstructions(callee_graph);
   if (number_of_instructions > inlining_budget_) {