Move HasShouldDeoptimizeFlag from method header to CodeInfo.

We don't have room for the extra bit if the code size is greater than 1GB.

Bug: 288215378
Bug: 288833213
Bug: 297093110
Test: ./art/test.py -b --host --64
Change-Id: I0070b270c7fe03f2f6fbb26223e78de41305c547

11 files changed, 59 insertions, 44 deletions

diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc
index ea668202ab..1fea30a359 100644
--- a/compiler/optimizing/code_generator.cc
+++ b/compiler/optimizing/code_generator.cc
@@ -279,7 +279,8 @@ void CodeGenerator::Compile() {
                                    fpu_spill_mask_,
                                    GetGraph()->GetNumberOfVRegs(),
                                    GetGraph()->IsCompilingBaseline(),
-                                   GetGraph()->IsDebuggable());
+                                   GetGraph()->IsDebuggable(),
+                                   GetGraph()->HasShouldDeoptimizeFlag());
 
   size_t frame_start = GetAssembler()->CodeSize();
   GenerateFrameEntry();
diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc
index f86ee4f70e..211c6417c2 100644
--- a/compiler/optimizing/optimizing_compiler.cc
+++ b/compiler/optimizing/optimizing_compiler.cc
@@ -1400,7 +1400,6 @@ bool OptimizingCompiler::JitCompile(Thread* self,
                             debug_info,
                             /* is_full_debug_info= */ compiler_options.GetGenerateDebugInfo(),
                             compilation_kind,
-                            /* has_should_deoptimize_flag= */ false,
                             cha_single_implementation_list)) {
       code_cache->Free(self, region, reserved_code.data(), reserved_data.data());
       return false;
@@ -1508,8 +1507,9 @@ bool OptimizingCompiler::JitCompile(Thread* self,
                           debug_info,
                           /* is_full_debug_info= */ compiler_options.GetGenerateDebugInfo(),
                           compilation_kind,
-                          codegen->GetGraph()->HasShouldDeoptimizeFlag(),
                           codegen->GetGraph()->GetCHASingleImplementationList())) {
+    CHECK_EQ(CodeInfo::HasShouldDeoptimizeFlag(stack_map.data()),
+             codegen->GetGraph()->HasShouldDeoptimizeFlag());
     code_cache->Free(self, region, reserved_code.data(), reserved_data.data());
     return false;
   }
diff --git a/compiler/optimizing/stack_map_stream.cc b/compiler/optimizing/stack_map_stream.cc
index 1a368ed347..13bdcfcdf8 100644
--- a/compiler/optimizing/stack_map_stream.cc
+++ b/compiler/optimizing/stack_map_stream.cc
@@ -51,7 +51,8 @@ void StackMapStream::BeginMethod(size_t frame_size_in_bytes,
                                  size_t fp_spill_mask,
                                  uint32_t num_dex_registers,
                                  bool baseline,
-                                 bool debuggable) {
+                                 bool debuggable,
+                                 bool has_should_deoptimize_flag) {
   DCHECK(!in_method_) << "Mismatched Begin/End calls";
   in_method_ = true;
   DCHECK_EQ(packed_frame_size_, 0u) << "BeginMethod was already called";
@@ -63,6 +64,7 @@ void StackMapStream::BeginMethod(size_t frame_size_in_bytes,
   num_dex_registers_ = num_dex_registers;
   baseline_ = baseline;
   debuggable_ = debuggable;
+  has_should_deoptimize_flag_ = has_should_deoptimize_flag;
 
   if (kVerifyStackMaps) {
     dchecks_.emplace_back([=](const CodeInfo& code_info) {
@@ -374,10 +376,12 @@ ScopedArenaVector<uint8_t> StackMapStream::Encode() {
   DCHECK(in_stack_map_ == false) << "Mismatched Begin/End calls";
   DCHECK(in_inline_info_ == false) << "Mismatched Begin/End calls";
 
-  uint32_t flags = (inline_infos_.size() > 0) ? CodeInfo::kHasInlineInfo : 0;
+  uint32_t flags = 0;
+  flags |= (inline_infos_.size() > 0) ? CodeInfo::kHasInlineInfo : 0;
   flags |= baseline_ ? CodeInfo::kIsBaseline : 0;
   flags |= debuggable_ ? CodeInfo::kIsDebuggable : 0;
-  DCHECK_LE(flags, kVarintMax);  // Ensure flags can be read directly as byte.
+  flags |= has_should_deoptimize_flag_ ? CodeInfo::kHasShouldDeoptimizeFlag : 0;
+
   uint32_t bit_table_flags = 0;
   ForEachBitTable([&bit_table_flags](size_t i, auto bit_table) {
     if (bit_table->size() != 0) {  // Record which bit-tables are stored.
@@ -409,6 +413,8 @@ ScopedArenaVector<uint8_t> StackMapStream::Encode() {
   CHECK_EQ(code_info.GetNumberOfStackMaps(), stack_maps_.size());
   CHECK_EQ(CodeInfo::HasInlineInfo(buffer.data()), inline_infos_.size() > 0);
   CHECK_EQ(CodeInfo::IsBaseline(buffer.data()), baseline_);
+  CHECK_EQ(CodeInfo::IsDebuggable(buffer.data()), debuggable_);
+  CHECK_EQ(CodeInfo::HasShouldDeoptimizeFlag(buffer.data()), has_should_deoptimize_flag_);
 
   // Verify all written data (usually only in debug builds).
   if (kVerifyStackMaps) {
diff --git a/compiler/optimizing/stack_map_stream.h b/compiler/optimizing/stack_map_stream.h
index 643af2da94..f027850ce6 100644
--- a/compiler/optimizing/stack_map_stream.h
+++ b/compiler/optimizing/stack_map_stream.h
@@ -66,7 +66,8 @@ class StackMapStream : public DeletableArenaObject<kArenaAllocStackMapStream> {
                    size_t fp_spill_mask,
                    uint32_t num_dex_registers,
                    bool baseline,
-                   bool debuggable);
+                   bool debuggable,
+                   bool has_should_deoptimize_flag = false);
   void EndMethod(size_t code_size);
 
   void BeginStackMapEntry(
@@ -129,8 +130,9 @@ class StackMapStream : public DeletableArenaObject<kArenaAllocStackMapStream> {
   uint32_t core_spill_mask_ = 0;
   uint32_t fp_spill_mask_ = 0;
   uint32_t num_dex_registers_ = 0;
-  bool baseline_;
-  bool debuggable_;
+  bool baseline_ = false;
+  bool debuggable_ = false;
+  bool has_should_deoptimize_flag_ = false;
   BitTableBuilder<StackMap> stack_maps_;
   BitTableBuilder<RegisterMask> register_masks_;
   BitmapTableBuilder stack_masks_;
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index aae923aa6f..0744aec412 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -663,7 +663,6 @@ bool JitCodeCache::Commit(Thread* self,
                           const std::vector<uint8_t>& debug_info,
                           bool is_full_debug_info,
                           CompilationKind compilation_kind,
-                          bool has_should_deoptimize_flag,
                           const ArenaSet<ArtMethod*>& cha_single_implementation_list) {
   DCHECK_IMPLIES(method->IsNative(), (compilation_kind != CompilationKind::kOsr));
 
@@ -681,8 +680,7 @@ bool JitCodeCache::Commit(Thread* self,
   // We need to make sure that there will be no jit-gcs going on and wait for any ongoing one to
   // finish.
   WaitForPotentialCollectionToCompleteRunnable(self);
-  const uint8_t* code_ptr = region->CommitCode(
-      reserved_code, code, stack_map_data, has_should_deoptimize_flag);
+  const uint8_t* code_ptr = region->CommitCode(reserved_code, code, stack_map_data);
   if (code_ptr == nullptr) {
     return false;
   }
diff --git a/runtime/jit/jit_code_cache.h b/runtime/jit/jit_code_cache.h
index a6b101be25..75b9c1ed04 100644
--- a/runtime/jit/jit_code_cache.h
+++ b/runtime/jit/jit_code_cache.h
@@ -264,11 +264,10 @@ class JitCodeCache {
               ArrayRef<const uint8_t> code,           // Compiler output (source).
               ArrayRef<const uint8_t> reserved_data,  // Uninitialized destination.
               const std::vector<Handle<mirror::Object>>& roots,
-              ArrayRef<const uint8_t> stack_map,      // Compiler output (source).
+              ArrayRef<const uint8_t> stack_map,  // Compiler output (source).
               const std::vector<uint8_t>& debug_info,
               bool is_full_debug_info,
               CompilationKind compilation_kind,
-              bool has_should_deoptimize_flag,
               const ArenaSet<ArtMethod*>& cha_single_implementation_list)
       REQUIRES_SHARED(Locks::mutator_lock_)
       REQUIRES(!Locks::jit_lock_);
diff --git a/runtime/jit/jit_memory_region.cc b/runtime/jit/jit_memory_region.cc
index cbfd39a24f..5cd1cbb3ea 100644
--- a/runtime/jit/jit_memory_region.cc
+++ b/runtime/jit/jit_memory_region.cc
@@ -355,8 +355,7 @@ void* JitMemoryRegion::MoreCore(const void* mspace, intptr_t increment) NO_THREA
 
 const uint8_t* JitMemoryRegion::CommitCode(ArrayRef<const uint8_t> reserved_code,
                                            ArrayRef<const uint8_t> code,
-                                           const uint8_t* stack_map,
-                                           bool has_should_deoptimize_flag) {
+                                           const uint8_t* stack_map) {
   DCHECK(IsInExecSpace(reserved_code.data()));
   ScopedCodeCacheWrite scc(*this);
 
@@ -382,9 +381,6 @@ const uint8_t* JitMemoryRegion::CommitCode(ArrayRef<const uint8_t> reserved_code
   OatQuickMethodHeader* method_header =
       OatQuickMethodHeader::FromCodePointer(w_memory + header_size);
   new (method_header) OatQuickMethodHeader((stack_map != nullptr) ? result - stack_map : 0u);
-  if (has_should_deoptimize_flag) {
-    method_header->SetHasShouldDeoptimizeFlag();
-  }
 
   // Both instruction and data caches need flushing to the point of unification where both share
   // a common view of memory. Flushing the data cache ensures the dirty cachelines from the
diff --git a/runtime/jit/jit_memory_region.h b/runtime/jit/jit_memory_region.h
index 6db931d187..8a3d6c3e55 100644
--- a/runtime/jit/jit_memory_region.h
+++ b/runtime/jit/jit_memory_region.h
@@ -90,8 +90,7 @@ class JitMemoryRegion {
   // Returns pointer to copied code (within reserved_code region; after OatQuickMethodHeader).
   const uint8_t* CommitCode(ArrayRef<const uint8_t> reserved_code,
                             ArrayRef<const uint8_t> code,
-                            const uint8_t* stack_map,
-                            bool has_should_deoptimize_flag)
+                            const uint8_t* stack_map)
       REQUIRES(Locks::jit_lock_);
 
   // Emit roots and stack map into the memory pointed by `roots_data` (despite it being const).
diff --git a/runtime/oat.h b/runtime/oat.h
index acd2923511..5ae612cdfa 100644
--- a/runtime/oat.h
+++ b/runtime/oat.h
@@ -44,8 +44,8 @@ std::ostream& operator<<(std::ostream& stream, StubType stub_type);
 class PACKED(4) OatHeader {
  public:
   static constexpr std::array<uint8_t, 4> kOatMagic { { 'o', 'a', 't', '\n' } };
-  // Last oat version changed reason: Revert thread suspension CL after another version bump.
-  static constexpr std::array<uint8_t, 4> kOatVersion{{'2', '3', '7', '\0'}};
+  // Last oat version changed reason: Move HasShouldDeoptimizeFlag from method header to CodeInfo.
+  static constexpr std::array<uint8_t, 4> kOatVersion{{'2', '3', '8', '\0'}};
 
   static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline";
   static constexpr const char* kDebuggableKey = "debuggable";
diff --git a/runtime/oat_quick_method_header.h b/runtime/oat_quick_method_header.h
index 2b7ec01830..837b564d77 100644
--- a/runtime/oat_quick_method_header.h
+++ b/runtime/oat_quick_method_header.h
@@ -109,7 +109,7 @@ class PACKED(4) OatQuickMethodHeader {
 
   void SetCodeInfoOffset(uint32_t offset) {
     data_ = kIsCodeInfoMask | offset;
-    DCHECK_EQ(GetCodeInfoOffset(), offset);
+    CHECK_EQ(GetCodeInfoOffset(), offset);
   }
 
   bool Contains(uintptr_t pc) const {
@@ -182,20 +182,14 @@ class PACKED(4) OatQuickMethodHeader {
                    bool abort_on_failure = true) const
       REQUIRES_SHARED(Locks::mutator_lock_);
 
-  void SetHasShouldDeoptimizeFlag() {
-    DCHECK(!HasShouldDeoptimizeFlag());
-    data_ |= kShouldDeoptimizeMask;
-  }
-
   bool HasShouldDeoptimizeFlag() const {
-    return (data_ & kShouldDeoptimizeMask) != 0;
+    return IsOptimized() && CodeInfo::HasShouldDeoptimizeFlag(GetOptimizedCodeInfoPtr());
   }
 
  private:
-  static constexpr uint32_t kShouldDeoptimizeMask = 0x80000000;
-  static constexpr uint32_t kIsCodeInfoMask       = 0x40000000;
-  static constexpr uint32_t kCodeInfoMask         = 0x3FFFFFFF;  // If kIsCodeInfoMask is set.
-  static constexpr uint32_t kCodeSizeMask         = 0x3FFFFFFF;  // If kIsCodeInfoMask is clear.
+  static constexpr uint32_t kIsCodeInfoMask = 0x80000000;
+  static constexpr uint32_t kCodeInfoMask = 0x7FFFFFFF;  // If kIsCodeInfoMask is set.
+  static constexpr uint32_t kCodeSizeMask = 0x7FFFFFFF;  // If kIsCodeInfoMask is clear.
 
   // In order to not confuse a stub with Java-generated code, we prefix each
   // stub with a 0xFFFFFFFF marker.
diff --git a/runtime/stack_map.h b/runtime/stack_map.h
index 514d30ed0c..0f25243228 100644
--- a/runtime/stack_map.h
+++ b/runtime/stack_map.h
@@ -479,16 +479,38 @@ class CodeInfo {
   // Accumulate code info size statistics into the given Stats tree.
   static void CollectSizeStats(const uint8_t* code_info, /*out*/ Stats& parent);
 
+  template <uint32_t kFlag>
+  ALWAYS_INLINE static bool HasFlag(const uint8_t* code_info_data) {
+    // Fast path - read just the one specific bit from the header.
+    bool result;
+    uint8_t varint = (*code_info_data) & MaxInt<uint8_t>(kVarintBits);
+    if (LIKELY(varint <= kVarintMax)) {
+      result = (varint & kFlag) != 0;
+    } else {
+      DCHECK_EQ(varint, kVarintMax + 1);  // Only up to 8 flags are supported for now.
+      constexpr uint32_t bit_offset = kNumHeaders * kVarintBits + WhichPowerOf2(kFlag);
+      result = (code_info_data[bit_offset / kBitsPerByte] & (1 << bit_offset % kBitsPerByte)) != 0;
+    }
+    // Slow path - dcheck that we got the correct result against the naive implementation.
+    BitMemoryReader reader(code_info_data);
+    DCHECK_EQ(result, (reader.ReadInterleavedVarints<kNumHeaders>()[0] & kFlag) != 0);
+    return result;
+  }
+
   ALWAYS_INLINE static bool HasInlineInfo(const uint8_t* code_info_data) {
-    return (*code_info_data & kHasInlineInfo) != 0;
+    return HasFlag<kHasInlineInfo>(code_info_data);
+  }
+
+  ALWAYS_INLINE static bool HasShouldDeoptimizeFlag(const uint8_t* code_info_data) {
+    return HasFlag<kHasShouldDeoptimizeFlag>(code_info_data);
   }
 
   ALWAYS_INLINE static bool IsBaseline(const uint8_t* code_info_data) {
-    return (*code_info_data & kIsBaseline) != 0;
+    return HasFlag<kIsBaseline>(code_info_data);
   }
 
   ALWAYS_INLINE static bool IsDebuggable(const uint8_t* code_info_data) {
-    return (*code_info_data & kIsDebuggable) != 0;
+    return HasFlag<kIsDebuggable>(code_info_data);
   }
 
   uint32_t GetNumberOfDexRegisters() {
@@ -538,20 +560,18 @@ class CodeInfo {
   void SetBitTableDeduped(size_t i) { bit_table_flags_ |= 1 << (kNumBitTables + i); }
   bool HasDedupedBitTables() { return (bit_table_flags_ >> kNumBitTables) != 0u; }
 
+  // NB: The first three flags should be the most common ones.
+  //     Maximum of 8 flags is supported right now (see the HasFlag method).
   enum Flags {
     kHasInlineInfo = 1 << 0,
-    kIsBaseline = 1 << 1,
-    kIsDebuggable = 1 << 2,
+    kHasShouldDeoptimizeFlag = 1 << 1,
+    kIsBaseline = 1 << 2,
+    kIsDebuggable = 1 << 3,
   };
 
   // The CodeInfo starts with sequence of variable-length bit-encoded integers.
   // (Please see kVarintMax for more details about encoding).
   static constexpr size_t kNumHeaders = 7;
-  // Note that the space for flags is limited to three bits. We use a custom encoding where we
-  // encode the value inline if it is less than kVarintMax. We want to access flags without
-  // decoding the entire CodeInfo header so the value of flags cannot be more than kVarintMax.
-  // See IsDebuggable / IsBaseline / HasInlineInfo on how we access flags_ without decoding the
-  // header.
   uint32_t flags_ = 0;
   uint32_t code_size_ = 0;  // The size of native PC range in bytes.
   uint32_t packed_frame_size_ = 0;  // Frame size in kStackAlignment units.