Make sure the frame size isn't unreasonably large when compiling

This CLs adds a check that we don't have an unreasonably large
frame size, and bails out from the compile if we detect it.
We had a similar check on RISC-V but not in other archs.

Bug: 323309447
Test: art/test/testrunner/testrunner.py --host --64 --optimizing -b
Change-Id: I4619dbbbc12c6e22ce335a17d15d90af8878808e

1 files changed, 9 insertions, 0 deletions

diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc
index 70d9013f7d..a1c4130bc1 100644
--- a/compiler/optimizing/optimizing_compiler.cc
+++ b/compiler/optimizing/optimizing_compiler.cc
@@ -937,6 +937,14 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* allocator,
                     regalloc_strategy,
                     compilation_stats_.get());
 
+  if (UNLIKELY(codegen->GetFrameSize() > codegen->GetMaximumFrameSize())) {
+    LOG(WARNING) << "Stack frame size is " << codegen->GetFrameSize()
+                 << " which is larger than the maximum of " << codegen->GetMaximumFrameSize()
+                 << " bytes. Method: " << graph->PrettyMethod();
+    MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kNotCompiledFrameTooBig);
+    return nullptr;
+  }
+
   codegen->Compile();
   pass_observer.DumpDisassembly();
 
@@ -1035,6 +1043,7 @@ CodeGenerator* OptimizingCompiler::TryCompileIntrinsic(
     return nullptr;
   }
 
+  CHECK_LE(codegen->GetFrameSize(), codegen->GetMaximumFrameSize());
   codegen->Compile();
   pass_observer.DumpDisassembly();