Make sure the frame size isn't unreasonably large when compiling

This CLs adds a check that we don't have an unreasonably large
frame size, and bails out from the compile if we detect it.
We had a similar check on RISC-V but not in other archs.

Bug: 323309447
Test: art/test/testrunner/testrunner.py --host --64 --optimizing -b
Change-Id: I4619dbbbc12c6e22ce335a17d15d90af8878808e

8 files changed, 27 insertions, 4 deletions

diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h
index fbb4f9e21e..c54c96c40f 100644
--- a/compiler/optimizing/code_generator.h
+++ b/compiler/optimizing/code_generator.h
@@ -251,6 +251,10 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> {
 
   uint32_t GetFrameSize() const { return frame_size_; }
   void SetFrameSize(uint32_t size) { frame_size_ = size; }
+  uint32_t GetMaximumFrameSize() const {
+    return GetStackOverflowReservedBytes(GetInstructionSet());
+  }
+
   uint32_t GetCoreSpillMask() const { return core_spill_mask_; }
   uint32_t GetFpuSpillMask() const { return fpu_spill_mask_; }
 
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 1e3835dab6..5ba26b4754 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -1395,6 +1395,9 @@ void CodeGeneratorARM64::GenerateFrameEntry() {
   }
 
   if (!HasEmptyFrame()) {
+    // Make sure the frame size isn't unreasonably large.
+    DCHECK_LE(GetFrameSize(), GetMaximumFrameSize());
+
     // Stack layout:
     //      sp[frame_size - 8]        : lr.
     //      ...                       : other preserved core registers.
diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc
index c5c5f1b5ff..85f61f5303 100644
--- a/compiler/optimizing/code_generator_arm_vixl.cc
+++ b/compiler/optimizing/code_generator_arm_vixl.cc
@@ -2389,6 +2389,9 @@ void CodeGeneratorARMVIXL::GenerateFrameEntry() {
     return;
   }
 
+  // Make sure the frame size isn't unreasonably large.
+  DCHECK_LE(GetFrameSize(), GetMaximumFrameSize());
+
   if (!skip_overflow_check) {
     // Using r4 instead of IP saves 2 bytes.
     UseScratchRegisterScope temps(GetVIXLAssembler());
diff --git a/compiler/optimizing/code_generator_riscv64.cc b/compiler/optimizing/code_generator_riscv64.cc
index 4c5234b0f7..ed57683e0a 100644
--- a/compiler/optimizing/code_generator_riscv64.cc
+++ b/compiler/optimizing/code_generator_riscv64.cc
@@ -5876,10 +5876,7 @@ void CodeGeneratorRISCV64::GenerateFrameEntry() {
 
   if (!HasEmptyFrame()) {
     // Make sure the frame size isn't unreasonably large.
-    if (GetFrameSize() > GetStackOverflowReservedBytes(InstructionSet::kRiscv64)) {
-      LOG(FATAL) << "Stack frame larger than "
-                 << GetStackOverflowReservedBytes(InstructionSet::kRiscv64) << " bytes";
-    }
+    DCHECK_LE(GetFrameSize(), GetMaximumFrameSize());
 
     // Spill callee-saved registers.
 
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index 4a0235e919..a61dca3022 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -1429,6 +1429,9 @@ void CodeGeneratorX86::GenerateFrameEntry() {
   }
 
   if (!HasEmptyFrame()) {
+    // Make sure the frame size isn't unreasonably large.
+    DCHECK_LE(GetFrameSize(), GetMaximumFrameSize());
+
     for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
       Register reg = kCoreCalleeSaves[i];
       if (allocated_registers_.ContainsCoreRegister(reg)) {
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index 8f56677f21..db4062b00d 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -1862,6 +1862,9 @@ void CodeGeneratorX86_64::GenerateFrameEntry() {
   }
 
   if (!HasEmptyFrame()) {
+    // Make sure the frame size isn't unreasonably large.
+    DCHECK_LE(GetFrameSize(), GetMaximumFrameSize());
+
     for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
       Register reg = kCoreCalleeSaves[i];
       if (allocated_registers_.ContainsCoreRegister(reg)) {
diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc
index 70d9013f7d..a1c4130bc1 100644
--- a/compiler/optimizing/optimizing_compiler.cc
+++ b/compiler/optimizing/optimizing_compiler.cc
@@ -937,6 +937,14 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* allocator,
                     regalloc_strategy,
                     compilation_stats_.get());
 
+  if (UNLIKELY(codegen->GetFrameSize() > codegen->GetMaximumFrameSize())) {
+    LOG(WARNING) << "Stack frame size is " << codegen->GetFrameSize()
+                 << " which is larger than the maximum of " << codegen->GetMaximumFrameSize()
+                 << " bytes. Method: " << graph->PrettyMethod();
+    MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kNotCompiledFrameTooBig);
+    return nullptr;
+  }
+
   codegen->Compile();
   pass_observer.DumpDisassembly();
 
@@ -1035,6 +1043,7 @@ CodeGenerator* OptimizingCompiler::TryCompileIntrinsic(
     return nullptr;
   }
 
+  CHECK_LE(codegen->GetFrameSize(), codegen->GetMaximumFrameSize());
   codegen->Compile();
   pass_observer.DumpDisassembly();
 
diff --git a/compiler/optimizing/optimizing_compiler_stats.h b/compiler/optimizing/optimizing_compiler_stats.h
index 60d18d2f24..22c43fc7ad 100644
--- a/compiler/optimizing/optimizing_compiler_stats.h
+++ b/compiler/optimizing/optimizing_compiler_stats.h
@@ -68,6 +68,7 @@ enum class MethodCompilationStat {
   kNotCompiledInliningIrreducibleLoop,
   kNotCompiledIrreducibleLoopAndStringInit,
   kNotCompiledPhiEquivalentInOsr,
+  kNotCompiledFrameTooBig,
   kInlinedMonomorphicCall,
   kInlinedPolymorphicCall,
   kMonomorphicCall,