Implement ClassStatus::kVisiblyInitialized.

Previously all class initialization checks involved a memory
barrier to ensure appropriate memory visibility. We change
that by introducing the kVisiblyInitialized status which can
be checked without a memory barrier. Before we mark a class
as visibly initialized, we run a checkpoint on all threads
to ensure memory visibility. This is done in batches for up
to 32 classes to reduce the overhead.

Avoiding memory barriers in the compiled code reduces code
size and improves performance. This is also the first step
toward fixing a long-standing synchronization bug 18161648.

Prebuilt sizes for aosp_taimen-userdebug:
 - before:
   arm/boot*.oat: 19150696
   arm64/boot*.oat: 22574336
   oat/arm64/services.odex: 21929800
 - after:
   arm/boot*.oat: 19134508 (-16KiB)
   arm64/boot*.oat: 22553664 (-20KiB)
   oat/arm64/services.odex: 21888760 (-40KiB)

Test: m test-art-host-gtest
Test: testrunner.py --host --optimizing
Test: aosp_taimen-userdebug boots
Test: run-gtests.sh -j4
Test: testrunner.py --target --optimizing
Test: Manually diff `m dump-oat-boot` output from before
      with output after this CL without codegen changes,
      with `sed` replacements for class status. Check that
      only checksums and the oatdump runtime values of
      DexCache.dexFile differ.
Bug: 18161648
Bug: 36692143
Change-Id: Ida10439d347e680a0abf4674546923374ffaa957

17 files changed, 362 insertions, 65 deletions

diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index cf596c7bf1..43d466baf8 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -1756,17 +1756,14 @@ void InstructionCodeGeneratorARM64::GenerateClassInitializationCheck(SlowPathCod
   UseScratchRegisterScope temps(GetVIXLAssembler());
   Register temp = temps.AcquireW();
   constexpr size_t status_lsb_position = SubtypeCheckBits::BitStructSizeOf();
-  const size_t status_byte_offset =
-      mirror::Class::StatusOffset().SizeValue() + (status_lsb_position / kBitsPerByte);
-  constexpr uint32_t shifted_initialized_value =
-      enum_cast<uint32_t>(ClassStatus::kInitialized) << (status_lsb_position % kBitsPerByte);
-
-  // Even if the initialized flag is set, we need to ensure consistent memory ordering.
-  // TODO(vixl): Let the MacroAssembler handle MemOperand.
-  __ Add(temp, class_reg, status_byte_offset);
-  __ Ldarb(temp, HeapOperand(temp));
-  __ Cmp(temp, shifted_initialized_value);
-  __ B(lo, slow_path->GetEntryLabel());
+  constexpr uint32_t visibly_initialized = enum_cast<uint32_t>(ClassStatus::kVisiblyInitialized);
+  static_assert(visibly_initialized == MaxInt<uint32_t>(32u - status_lsb_position),
+                "kVisiblyInitialized must have all bits set");
+
+  const size_t status_offset = mirror::Class::StatusOffset().SizeValue();
+  __ Ldr(temp, HeapOperand(class_reg, status_offset));
+  __ Mvn(temp, Operand(temp, ASR, status_lsb_position));  // Were all the bits of the status set?
+  __ Cbnz(temp, slow_path->GetEntryLabel());              // If not, go to slow path.
   __ Bind(slow_path->GetExitLabel());
 }
 
diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc
index 49e7695e31..b72a1a04fc 100644
--- a/compiler/optimizing/code_generator_arm_vixl.cc
+++ b/compiler/optimizing/code_generator_arm_vixl.cc
@@ -7182,17 +7182,13 @@ void InstructionCodeGeneratorARMVIXL::GenerateClassInitializationCheck(
   UseScratchRegisterScope temps(GetVIXLAssembler());
   vixl32::Register temp = temps.Acquire();
   constexpr size_t status_lsb_position = SubtypeCheckBits::BitStructSizeOf();
-  const size_t status_byte_offset =
-      mirror::Class::StatusOffset().SizeValue() + (status_lsb_position / kBitsPerByte);
-  constexpr uint32_t shifted_initialized_value =
-      enum_cast<uint32_t>(ClassStatus::kInitialized) << (status_lsb_position % kBitsPerByte);
+  constexpr uint32_t shifted_visibly_initialized_value =
+      enum_cast<uint32_t>(ClassStatus::kVisiblyInitialized) << status_lsb_position;
 
-  GetAssembler()->LoadFromOffset(kLoadUnsignedByte, temp, class_reg, status_byte_offset);
-  __ Cmp(temp, shifted_initialized_value);
+  const size_t status_offset = mirror::Class::StatusOffset().SizeValue();
+  GetAssembler()->LoadFromOffset(kLoadWord, temp, class_reg, status_offset);
+  __ Cmp(temp, shifted_visibly_initialized_value);
   __ B(lo, slow_path->GetEntryLabel());
-  // Even if the initialized flag is set, we may be in a situation where caches are not synced
-  // properly. Therefore, we do a memory fence.
-  __ Dmb(ISH);
   __ Bind(slow_path->GetExitLabel());
 }
 
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index 7f7e3a5178..5159553bce 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -6709,13 +6709,12 @@ void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
   constexpr size_t status_lsb_position = SubtypeCheckBits::BitStructSizeOf();
   const size_t status_byte_offset =
       mirror::Class::StatusOffset().SizeValue() + (status_lsb_position / kBitsPerByte);
-  constexpr uint32_t shifted_initialized_value =
-      enum_cast<uint32_t>(ClassStatus::kInitialized) << (status_lsb_position % kBitsPerByte);
+  constexpr uint32_t shifted_visibly_initialized_value =
+      enum_cast<uint32_t>(ClassStatus::kVisiblyInitialized) << (status_lsb_position % kBitsPerByte);
 
-  __ cmpb(Address(class_reg,  status_byte_offset), Immediate(shifted_initialized_value));
+  __ cmpb(Address(class_reg,  status_byte_offset), Immediate(shifted_visibly_initialized_value));
   __ j(kBelow, slow_path->GetEntryLabel());
   __ Bind(slow_path->GetExitLabel());
-  // No need for memory fence, thanks to the X86 memory model.
 }
 
 void InstructionCodeGeneratorX86::GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check,
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index 8067b9c8a6..8c8b5e641d 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -5837,13 +5837,12 @@ void InstructionCodeGeneratorX86_64::GenerateClassInitializationCheck(
   constexpr size_t status_lsb_position = SubtypeCheckBits::BitStructSizeOf();
   const size_t status_byte_offset =
       mirror::Class::StatusOffset().SizeValue() + (status_lsb_position / kBitsPerByte);
-  constexpr uint32_t shifted_initialized_value =
-      enum_cast<uint32_t>(ClassStatus::kInitialized) << (status_lsb_position % kBitsPerByte);
+  constexpr uint32_t shifted_visibly_initialized_value =
+      enum_cast<uint32_t>(ClassStatus::kVisiblyInitialized) << (status_lsb_position % kBitsPerByte);
 
-  __ cmpb(Address(class_reg,  status_byte_offset), Immediate(shifted_initialized_value));
+  __ cmpb(Address(class_reg,  status_byte_offset), Immediate(shifted_visibly_initialized_value));
   __ j(kBelow, slow_path->GetEntryLabel());
   __ Bind(slow_path->GetExitLabel());
-  // No need for memory fence, thanks to the x86-64 memory model.
 }
 
 void InstructionCodeGeneratorX86_64::GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check,
diff --git a/dex2oat/driver/compiler_driver.cc b/dex2oat/driver/compiler_driver.cc
index c0b2f3e5c3..9eaf17d7d3 100644
--- a/dex2oat/driver/compiler_driver.cc
+++ b/dex2oat/driver/compiler_driver.cc
@@ -1961,7 +1961,12 @@ class VerifyClassVisitor : public CompilationVisitor {
 
       // Class has a meaningful status for the compiler now, record it.
       ClassReference ref(manager_->GetDexFile(), class_def_index);
-      manager_->GetCompiler()->RecordClassStatus(ref, klass->GetStatus());
+      ClassStatus status = klass->GetStatus();
+      if (status == ClassStatus::kInitialized) {
+        // Initialized classes shall be visibly initialized when loaded from the image.
+        status = ClassStatus::kVisiblyInitialized;
+      }
+      manager_->GetCompiler()->RecordClassStatus(ref, status);
 
       // It is *very* problematic if there are resolution errors in the boot classpath.
       //
@@ -2017,6 +2022,9 @@ void CompilerDriver::VerifyDexFile(jobject class_loader,
                               : verifier::HardFailLogMode::kLogWarning;
   VerifyClassVisitor visitor(&context, log_level);
   context.ForAll(0, dex_file.NumClassDefs(), &visitor, thread_count);
+
+  // Make initialized classes visibly initialized.
+  class_linker->MakeInitializedClassesVisiblyInitialized(Thread::Current(), /*wait=*/ true);
 }
 
 class SetVerifiedClassVisitor : public CompilationVisitor {
@@ -2285,6 +2293,10 @@ class InitializeClassVisitor : public CompilationVisitor {
         soa.Self()->AssertNoPendingException();
       }
     }
+    if (old_status == ClassStatus::kInitialized) {
+      // Initialized classes shall be visibly initialized when loaded from the image.
+      old_status = ClassStatus::kVisiblyInitialized;
+    }
     // Record the final class status if necessary.
     ClassReference ref(&dex_file, klass->GetDexClassDefIndex());
     // Back up the status before doing initialization for static encoded fields,
@@ -2496,6 +2508,9 @@ void CompilerDriver::InitializeClasses(jobject jni_class_loader,
   }
   InitializeClassVisitor visitor(&context);
   context.ForAll(0, dex_file.NumClassDefs(), &visitor, init_thread_count);
+
+  // Make initialized classes visibly initialized.
+  class_linker->MakeInitializedClassesVisiblyInitialized(Thread::Current(), /*wait=*/ true);
 }
 
 class InitializeArrayClassesAndCreateConflictTablesVisitor : public ClassVisitor {
@@ -2759,7 +2774,7 @@ void CompilerDriver::RecordClassStatus(const ClassReference& ref, ClassStatus st
     case ClassStatus::kRetryVerificationAtRuntime:
     case ClassStatus::kVerified:
     case ClassStatus::kSuperclassValidated:
-    case ClassStatus::kInitialized:
+    case ClassStatus::kVisiblyInitialized:
       break;  // Expected states.
     default:
       LOG(FATAL) << "Unexpected class status for class "
diff --git a/dex2oat/driver/compiler_driver_test.cc b/dex2oat/driver/compiler_driver_test.cc
index 81262d3938..8ec7003dd9 100644
--- a/dex2oat/driver/compiler_driver_test.cc
+++ b/dex2oat/driver/compiler_driver_test.cc
@@ -354,7 +354,8 @@ TEST_F(CompilerDriverVerifyTest, RetryVerifcationStatusCheckVerified) {
     const ClassStatus expected_status = enum_cast<ClassStatus>(i);
     // Skip unsupported status that are not supposed to be ever recorded.
     if (expected_status == ClassStatus::kVerifyingAtRuntime ||
-        expected_status == ClassStatus::kInitializing) {
+        expected_status == ClassStatus::kInitializing ||
+        expected_status == ClassStatus::kInitialized) {
       continue;
     }
     compiler_driver_->RecordClassStatus(ref, expected_status);
diff --git a/dex2oat/linker/image_writer.cc b/dex2oat/linker/image_writer.cc
index 8a8ef4dd42..1f1bf33786 100644
--- a/dex2oat/linker/image_writer.cc
+++ b/dex2oat/linker/image_writer.cc
@@ -822,7 +822,7 @@ ImageWriter::Bin ImageWriter::AssignImageBinSlot(mirror::Object* object, size_t
       if (dirty_image_objects_ != nullptr &&
           dirty_image_objects_->find(klass->PrettyDescriptor()) != dirty_image_objects_->end()) {
         bin = Bin::kKnownDirty;
-      } else if (klass->GetStatus() == ClassStatus::kInitialized) {
+      } else if (klass->GetStatus() == ClassStatus::kVisiblyInitialized) {
         bin = Bin::kClassInitialized;
 
         // If the class's static fields are all final, put it into a separate bin
@@ -888,7 +888,8 @@ bool ImageWriter::WillMethodBeDirty(ArtMethod* m) const {
   }
   ObjPtr<mirror::Class> declaring_class = m->GetDeclaringClass();
   // Initialized is highly unlikely to dirty since there's no entry points to mutate.
-  return declaring_class == nullptr || declaring_class->GetStatus() != ClassStatus::kInitialized;
+  return declaring_class == nullptr ||
+         declaring_class->GetStatus() != ClassStatus::kVisiblyInitialized;
 }
 
 bool ImageWriter::IsImageBinSlotAssigned(mirror::Object* object) const {
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index fca40c5d11..731dfe7eeb 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -20,6 +20,7 @@
 
 #include <algorithm>
 #include <deque>
+#include <forward_list>
 #include <iostream>
 #include <map>
 #include <memory>
@@ -35,11 +36,13 @@
 
 #include "art_field-inl.h"
 #include "art_method-inl.h"
+#include "barrier.h"
 #include "base/arena_allocator.h"
 #include "base/casts.h"
 #include "base/file_utils.h"
 #include "base/leb128.h"
 #include "base/logging.h"
+#include "base/mutex-inl.h"
 #include "base/os.h"
 #include "base/quasi_atomic.h"
 #include "base/scoped_arena_containers.h"
@@ -233,6 +236,189 @@ static void EnsureSkipAccessChecksMethods(Handle<mirror::Class> klass, PointerSi
   }
 }
 
+// Callback responsible for making a batch of classes visibly initialized
+// after all threads have called it from a checkpoint, ensuring visibility.
+class ClassLinker::VisiblyInitializedCallback final
+    : public Closure, public IntrusiveForwardListNode<VisiblyInitializedCallback> {
+ public:
+  explicit VisiblyInitializedCallback(ClassLinker* class_linker)
+      : class_linker_(class_linker),
+        num_classes_(0u),
+        thread_visibility_counter_(0),
+        barriers_() {
+    std::fill_n(classes_, kMaxClasses, nullptr);
+  }
+
+  bool IsEmpty() const {
+    DCHECK_LE(num_classes_, kMaxClasses);
+    return num_classes_ == 0u;
+  }
+
+  bool IsFull() const {
+    DCHECK_LE(num_classes_, kMaxClasses);
+    return num_classes_ == kMaxClasses;
+  }
+
+  void AddClass(Thread* self, ObjPtr<mirror::Class> klass) REQUIRES_SHARED(Locks::mutator_lock_) {
+    DCHECK_EQ(klass->GetStatus(), ClassStatus::kInitialized);
+    DCHECK(!IsFull());
+    classes_[num_classes_] = self->GetJniEnv()->GetVm()->AddWeakGlobalRef(self, klass);
+    ++num_classes_;
+  }
+
+  void AddBarrier(Barrier* barrier) {
+    barriers_.push_front(barrier);
+  }
+
+  std::forward_list<Barrier*> GetAndClearBarriers() {
+    std::forward_list<Barrier*> result;
+    result.swap(barriers_);
+    result.reverse();  // Return barriers in insertion order.
+    return result;
+  }
+
+  void MakeVisible(Thread* self) {
+    DCHECK_EQ(thread_visibility_counter_.load(std::memory_order_relaxed), 0);
+    size_t count = Runtime::Current()->GetThreadList()->RunCheckpoint(this);
+    AdjustThreadVisibilityCounter(self, count);
+  }
+
+  void Run(Thread* self) override {
+    self->ClearMakeVisiblyInitializedCounter();
+    AdjustThreadVisibilityCounter(self, -1);
+  }
+
+ private:
+  void AdjustThreadVisibilityCounter(Thread* self, ssize_t adjustment) {
+    ssize_t old = thread_visibility_counter_.fetch_add(adjustment, std::memory_order_relaxed);
+    if (old + adjustment == 0) {
+      // All threads passed the checkpoint. Mark classes as visibly initialized.
+      {
+        ScopedObjectAccess soa(self);
+        StackHandleScope<1u> hs(self);
+        MutableHandle<mirror::Class> klass = hs.NewHandle<mirror::Class>(nullptr);
+        JavaVMExt* vm = self->GetJniEnv()->GetVm();
+        for (size_t i = 0, num = num_classes_; i != num; ++i) {
+          klass.Assign(ObjPtr<mirror::Class>::DownCast(self->DecodeJObject(classes_[i])));
+          vm->DeleteWeakGlobalRef(self, classes_[i]);
+          if (klass != nullptr) {
+            ObjectLock<mirror::Class> lock(self, klass);
+            mirror::Class::SetStatus(klass, ClassStatus::kVisiblyInitialized, self);
+          }
+        }
+        num_classes_ = 0u;
+      }
+      class_linker_->VisiblyInitializedCallbackDone(self, this);
+    }
+  }
+
+  static constexpr size_t kMaxClasses = 32;
+
+  ClassLinker* const class_linker_;
+  size_t num_classes_;
+  jweak classes_[kMaxClasses];
+
+  // The thread visibility counter starts at 0 and it is incremented by the number of
+  // threads that need to run this callback (by the thread that request the callback
+  // to be run) and decremented once for each `Run()` execution. When it reaches 0,
+  // whether after the increment or after a decrement, we know that `Run()` was executed
+  // for all threads and therefore we can mark the classes as visibly initialized.
+  std::atomic<ssize_t> thread_visibility_counter_;
+
+  // List of barries to `Pass()` for threads that wait for the callback to complete.
+  std::forward_list<Barrier*> barriers_;
+};
+
+void ClassLinker::MakeInitializedClassesVisiblyInitialized(Thread* self, bool wait) {
+  if (kRuntimeISA == InstructionSet::kX86 || kRuntimeISA == InstructionSet::kX86_64) {
+    return;  // Nothing to do. Thanks to the x86 memory model classes skip the initialized status.
+  }
+  std::optional<Barrier> maybe_barrier;  // Avoid constructing the Barrier for `wait == false`.
+  if (wait) {
+    maybe_barrier.emplace(0);
+  }
+  int wait_count = 0;
+  VisiblyInitializedCallback* callback = nullptr;
+  {
+    MutexLock lock(self, visibly_initialized_callback_lock_);
+    if (visibly_initialized_callback_ != nullptr && !visibly_initialized_callback_->IsEmpty()) {
+      callback = visibly_initialized_callback_.release();
+      running_visibly_initialized_callbacks_.push_front(*callback);
+    }
+    if (wait) {
+      DCHECK(maybe_barrier.has_value());
+      Barrier* barrier = std::addressof(*maybe_barrier);
+      for (VisiblyInitializedCallback& cb : running_visibly_initialized_callbacks_) {
+        cb.AddBarrier(barrier);
+        ++wait_count;
+      }
+    }
+  }
+  if (callback != nullptr) {
+    callback->MakeVisible(self);
+  }
+  if (wait_count != 0) {
+    DCHECK(maybe_barrier.has_value());
+    maybe_barrier->Increment(self, wait_count);
+  }
+}
+
+void ClassLinker::VisiblyInitializedCallbackDone(Thread* self,
+                                                 VisiblyInitializedCallback* callback) {
+  MutexLock lock(self, visibly_initialized_callback_lock_);
+  // Pass the barriers if requested.
+  for (Barrier* barrier : callback->GetAndClearBarriers()) {
+    barrier->Pass(self);
+  }
+  // Remove the callback from the list of running callbacks.
+  auto before = running_visibly_initialized_callbacks_.before_begin();
+  auto it = running_visibly_initialized_callbacks_.begin();
+  DCHECK(it != running_visibly_initialized_callbacks_.end());
+  while (std::addressof(*it) != callback) {
+    before = it;
+    ++it;
+    DCHECK(it != running_visibly_initialized_callbacks_.end());
+  }
+  running_visibly_initialized_callbacks_.erase_after(before);
+  // Reuse or destroy the callback object.
+  if (visibly_initialized_callback_ == nullptr) {
+    visibly_initialized_callback_.reset(callback);
+  } else {
+    delete callback;
+  }
+}
+
+ClassLinker::VisiblyInitializedCallback* ClassLinker::MarkClassInitialized(
+    Thread* self, Handle<mirror::Class> klass) {
+  if (kRuntimeISA == InstructionSet::kX86 || kRuntimeISA == InstructionSet::kX86_64) {
+    // Thanks to the x86 memory model, we do not need any memory fences and
+    // we can immediately mark the class as visibly initialized.
+    mirror::Class::SetStatus(klass, ClassStatus::kVisiblyInitialized, self);
+    return nullptr;
+  }
+  if (Runtime::Current()->IsActiveTransaction()) {
+    // Transactions are single-threaded, so we can mark the class as visibly intialized.
+    // (Otherwise we'd need to track the callback's entry in the transaction for rollback.)
+    mirror::Class::SetStatus(klass, ClassStatus::kVisiblyInitialized, self);
+    return nullptr;
+  }
+  mirror::Class::SetStatus(klass, ClassStatus::kInitialized, self);
+  MutexLock lock(self, visibly_initialized_callback_lock_);
+  if (visibly_initialized_callback_ == nullptr) {
+    visibly_initialized_callback_.reset(new VisiblyInitializedCallback(this));
+  }
+  DCHECK(!visibly_initialized_callback_->IsFull());
+  visibly_initialized_callback_->AddClass(self, klass.Get());
+
+  if (visibly_initialized_callback_->IsFull()) {
+    VisiblyInitializedCallback* callback = visibly_initialized_callback_.release();
+    running_visibly_initialized_callbacks_.push_front(*callback);
+    return callback;
+  } else {
+    return nullptr;
+  }
+}
+
 void ClassLinker::ThrowEarlierClassFailure(ObjPtr<mirror::Class> c,
                                            bool wrap_in_no_class_def,
                                            bool log) {
@@ -418,6 +604,8 @@ ClassLinker::ClassLinker(InternTable* intern_table, bool fast_class_not_found_ex
       quick_generic_jni_trampoline_(nullptr),
       quick_to_interpreter_bridge_trampoline_(nullptr),
       image_pointer_size_(kRuntimePointerSize),
+      visibly_initialized_callback_lock_("visibly initialized callback lock"),
+      visibly_initialized_callback_(nullptr),
       cha_(Runtime::Current()->IsAotCompiler() ? nullptr : new ClassHierarchyAnalysis()) {
   // For CHA disabled during Aot, see b/34193647.
 
@@ -2479,6 +2667,11 @@ ClassLinker::~ClassLinker() {
     DeleteClassLoader(self, data, /*cleanup_cha=*/ false);
   }
   class_loaders_.clear();
+  while (!running_visibly_initialized_callbacks_.empty()) {
+    std::unique_ptr<VisiblyInitializedCallback> callback(
+        std::addressof(running_visibly_initialized_callbacks_.front()));
+    running_visibly_initialized_callbacks_.pop_front();
+  }
 }
 
 void ClassLinker::DeleteClassLoader(Thread* self, const ClassLoaderData& data, bool cleanup_cha) {
@@ -2646,7 +2839,9 @@ void ClassLinker::FinishArrayClassSetup(ObjPtr<mirror::Class> array_class) {
 
   array_class->SetAccessFlags(access_flags);
 
-  array_class->SetStatusForPrimitiveOrArray(ClassStatus::kInitialized);
+  // Array classes are fully initialized either during single threaded startup,
+  // or from a pre-fence visitor, so visibly initialized.
+  array_class->SetStatusForPrimitiveOrArray(ClassStatus::kVisiblyInitialized);
 }
 
 void ClassLinker::FinishCoreArrayClassSetup(ClassRoot array_root) {
@@ -4159,7 +4354,8 @@ void ClassLinker::CreatePrimitiveClass(Thread* self,
   // the kAccVerificationAttempted flag was added above, and there are no
   // methods that need the kAccSkipAccessChecks flag.
   DCHECK_EQ(primitive_class->NumMethods(), 0u);
-  primitive_class->SetStatusForPrimitiveOrArray(ClassStatus::kInitialized);
+  // Primitive classes are initialized during single threaded startup, so visibly initialized.
+  primitive_class->SetStatusForPrimitiveOrArray(ClassStatus::kVisiblyInitialized);
   const char* descriptor = Primitive::Descriptor(type);
   ObjPtr<mirror::Class> existing = InsertClass(descriptor,
                                                primitive_class,
@@ -5003,11 +5199,16 @@ ObjPtr<mirror::Class> ClassLinker::CreateProxyClass(ScopedObjectAccessAlreadyRun
     // TODO: Avoid taking subtype_check_lock_ if SubtypeCheck for j.l.r.Proxy is already assigned.
   }
 
+  VisiblyInitializedCallback* callback = nullptr;
   {
     // Lock on klass is released. Lock new class object.
     ObjectLock<mirror::Class> initialization_lock(self, klass);
     EnsureSkipAccessChecksMethods(klass, image_pointer_size_);
-    mirror::Class::SetStatus(klass, ClassStatus::kInitialized, self);
+    // Conservatively go through the ClassStatus::kInitialized state.
+    callback = MarkClassInitialized(self, klass);
+  }
+  if (callback != nullptr) {
+    callback->MakeVisible(self);
   }
 
   // sanity checks
@@ -5421,6 +5622,7 @@ bool ClassLinker::InitializeClass(Thread* self, Handle<mirror::Class> klass,
   self->AllowThreadSuspension();
   uint64_t t1 = NanoTime();
 
+  VisiblyInitializedCallback* callback = nullptr;
   bool success = true;
   {
     ObjectLock<mirror::Class> lock(self, klass);
@@ -5446,7 +5648,7 @@ bool ClassLinker::InitializeClass(Thread* self, Handle<mirror::Class> klass,
       global_stats->class_init_time_ns += (t1 - t0 - t_sub);
       thread_stats->class_init_time_ns += (t1 - t0 - t_sub);
       // Set the class as initialized except if failed to initialize static fields.
-      mirror::Class::SetStatus(klass, ClassStatus::kInitialized, self);
+      callback = MarkClassInitialized(self, klass);
       if (VLOG_IS_ON(class_linker)) {
         std::string temp;
         LOG(INFO) << "Initialized class " << klass->GetDescriptor(&temp) << " from " <<
@@ -5456,6 +5658,9 @@ bool ClassLinker::InitializeClass(Thread* self, Handle<mirror::Class> klass,
       FixupStaticTrampolines(klass.Get());
     }
   }
+  if (callback != nullptr) {
+    callback->MakeVisible(self);
+  }
   return success;
 }
 
diff --git a/runtime/class_linker.h b/runtime/class_linker.h
index a72468526e..dd9f56fd90 100644
--- a/runtime/class_linker.h
+++ b/runtime/class_linker.h
@@ -26,6 +26,8 @@
 #include <vector>
 
 #include "base/enums.h"
+#include "base/mutex.h"
+#include "base/intrusive_forward_list.h"
 #include "base/locks.h"
 #include "base/macros.h"
 #include "dex/class_accessor.h"
@@ -719,6 +721,8 @@ class ClassLinker {
     return cha_.get();
   }
 
+  void MakeInitializedClassesVisiblyInitialized(Thread* self, bool wait);
+
   struct DexCacheData {
     // Construct an invalid data object.
     DexCacheData()
@@ -764,6 +768,7 @@ class ClassLinker {
 
  private:
   class LinkInterfaceMethodsHelper;
+  class VisiblyInitializedCallback;
 
   struct ClassLoaderData {
     jweak weak_root;  // Weak root to enable class unloading.
@@ -771,6 +776,10 @@ class ClassLinker {
     LinearAlloc* allocator;
   };
 
+  void VisiblyInitializedCallbackDone(Thread* self, VisiblyInitializedCallback* callback);
+  VisiblyInitializedCallback* MarkClassInitialized(Thread* self, Handle<mirror::Class> klass)
+      REQUIRES_SHARED(Locks::mutator_lock_);
+
   // Ensures that the supertype of 'klass' ('supertype') is verified. Returns false and throws
   // appropriate exceptions if verification failed hard. Returns true for successful verification or
   // soft-failures.
@@ -1393,6 +1402,13 @@ class ClassLinker {
   // Image pointer size.
   PointerSize image_pointer_size_;
 
+  // Classes to transition from ClassStatus::kInitialized to ClassStatus::kVisiblyInitialized.
+  Mutex visibly_initialized_callback_lock_;
+  std::unique_ptr<VisiblyInitializedCallback> visibly_initialized_callback_
+      GUARDED_BY(visibly_initialized_callback_lock_);
+  IntrusiveForwardList<VisiblyInitializedCallback> running_visibly_initialized_callbacks_
+      GUARDED_BY(visibly_initialized_callback_lock_);
+
   std::unique_ptr<ClassHierarchyAnalysis> cha_;
 
   class FindVirtualMethodHolderVisitor;
diff --git a/runtime/class_linker_test.cc b/runtime/class_linker_test.cc
index 0ffbc4a56d..4377d7e03d 100644
--- a/runtime/class_linker_test.cc
+++ b/runtime/class_linker_test.cc
@@ -92,7 +92,7 @@ class ClassLinkerTest : public CommonRuntimeTest {
     EXPECT_TRUE(primitive->GetSuperClass() == nullptr);
     EXPECT_FALSE(primitive->HasSuperClass());
     EXPECT_TRUE(primitive->GetClassLoader() == nullptr);
-    EXPECT_EQ(ClassStatus::kInitialized, primitive->GetStatus());
+    EXPECT_EQ(ClassStatus::kVisiblyInitialized, primitive->GetStatus());
     EXPECT_FALSE(primitive->IsErroneous());
     EXPECT_TRUE(primitive->IsLoaded());
     EXPECT_TRUE(primitive->IsResolved());
@@ -131,7 +131,8 @@ class ClassLinkerTest : public CommonRuntimeTest {
     EXPECT_TRUE(JavaLangObject->GetSuperClass() == nullptr);
     EXPECT_FALSE(JavaLangObject->HasSuperClass());
     EXPECT_TRUE(JavaLangObject->GetClassLoader() == nullptr);
-    EXPECT_EQ(ClassStatus::kInitialized, JavaLangObject->GetStatus());
+    class_linker_->MakeInitializedClassesVisiblyInitialized(Thread::Current(), /*wait=*/ true);
+    EXPECT_EQ(ClassStatus::kVisiblyInitialized, JavaLangObject->GetStatus());
     EXPECT_FALSE(JavaLangObject->IsErroneous());
     EXPECT_TRUE(JavaLangObject->IsLoaded());
     EXPECT_TRUE(JavaLangObject->IsResolved());
@@ -207,7 +208,7 @@ class ClassLinkerTest : public CommonRuntimeTest {
     EXPECT_TRUE(array->HasSuperClass());
     ASSERT_TRUE(array->GetComponentType() != nullptr);
     ASSERT_GT(strlen(array->GetComponentType()->GetDescriptor(&temp)), 0U);
-    EXPECT_EQ(ClassStatus::kInitialized, array->GetStatus());
+    EXPECT_EQ(ClassStatus::kVisiblyInitialized, array->GetStatus());
     EXPECT_FALSE(array->IsErroneous());
     EXPECT_TRUE(array->IsLoaded());
     EXPECT_TRUE(array->IsResolved());
diff --git a/runtime/class_status.h b/runtime/class_status.h
index ada286338b..de64ab7617 100644
--- a/runtime/class_status.h
+++ b/runtime/class_status.h
@@ -87,7 +87,8 @@ enum class ClassStatus : uint8_t {
   kSuperclassValidated = 12,  // Superclass validation part of init done.
   kInitializing = 13,  // Class init in progress.
   kInitialized = 14,  // Ready to go.
-  kLast = kInitialized
+  kVisiblyInitialized = 15,  // Initialized and visible to all threads.
+  kLast = kVisiblyInitialized
 };
 
 std::ostream& operator<<(std::ostream& os, const ClassStatus& rhs);
diff --git a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc
index 838b5b5083..0b286e3a1c 100644
--- a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc
@@ -140,6 +140,12 @@ extern "C" mirror::Class* artInitializeStaticStorageFromCode(mirror::Class* klas
   ScopedQuickEntrypointChecks sqec(self);
   DCHECK(klass != nullptr);
   ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+  if (UNLIKELY(klass->IsInitialized())) {
+    if (self->IncrementMakeVisiblyInitializedCounter()) {
+      class_linker->MakeInitializedClassesVisiblyInitialized(self, /*wait=*/ false);
+    }
+    return klass;
+  }
   StackHandleScope<1> hs(self);
   Handle<mirror::Class> h_klass = hs.NewHandle(klass);
   bool success = class_linker->EnsureInitialized(
diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
index 20781867c7..79196212a2 100644
--- a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
@@ -769,22 +769,31 @@ extern "C" uint64_t artQuickToInterpreterBridge(ArtMethod* method, Thread* self,
     BuildQuickShadowFrameVisitor shadow_frame_builder(sp, method->IsStatic(), shorty, shorty_len,
                                                       shadow_frame, first_arg_reg);
     shadow_frame_builder.VisitArguments();
-    const bool needs_initialization =
-        method->IsStatic() && !method->GetDeclaringClass()->IsInitialized();
     // Push a transition back into managed code onto the linked list in thread.
     self->PushManagedStackFragment(&fragment);
     self->PushShadowFrame(shadow_frame);
     self->EndAssertNoThreadSuspension(old_cause);
 
-    if (needs_initialization) {
-      // Ensure static method's class is initialized.
-      StackHandleScope<1> hs(self);
-      Handle<mirror::Class> h_class(hs.NewHandle(shadow_frame->GetMethod()->GetDeclaringClass()));
-      if (!Runtime::Current()->GetClassLinker()->EnsureInitialized(self, h_class, true, true)) {
-        DCHECK(Thread::Current()->IsExceptionPending())
-            << shadow_frame->GetMethod()->PrettyMethod();
-        self->PopManagedStackFragment(fragment);
-        return 0;
+    if (method->IsStatic()) {
+      ObjPtr<mirror::Class> declaring_class = method->GetDeclaringClass();
+      if (LIKELY(declaring_class->IsVisiblyInitialized())) {
+        // Visibly initialized, nothing to do.
+      } else if (!declaring_class->IsInitialized()) {
+        // Ensure static method's class is initialized.
+        StackHandleScope<1> hs(self);
+        Handle<mirror::Class> h_class(hs.NewHandle(shadow_frame->GetMethod()->GetDeclaringClass()));
+        if (!Runtime::Current()->GetClassLinker()->EnsureInitialized(self, h_class, true, true)) {
+          DCHECK(Thread::Current()->IsExceptionPending())
+              << shadow_frame->GetMethod()->PrettyMethod();
+          self->PopManagedStackFragment(fragment);
+          return 0;
+        }
+      } else {
+        // Initialized but not visibly initialized.
+        if (self->IncrementMakeVisiblyInitializedCounter()) {
+          Runtime::Current()->GetClassLinker()->MakeInitializedClassesVisiblyInitialized(
+              self, /*wait=*/ false);
+        }
       }
     }
 
diff --git a/runtime/mirror/class.cc b/runtime/mirror/class.cc
index 422625dbb4..846517b524 100644
--- a/runtime/mirror/class.cc
+++ b/runtime/mirror/class.cc
@@ -201,7 +201,11 @@ void Class::SetStatus(Handle<Class> h_this, ClassStatus new_status, Thread* self
   // Setting the object size alloc fast path needs to be after the status write so that if the
   // alloc path sees a valid object size, we would know that it's initialized as long as it has a
   // load-acquire/fake dependency.
-  if (new_status == ClassStatus::kInitialized && !h_this->IsVariableSize()) {
+  // TODO: Update the object size alloc fast path only for ClassStatus::kVisiblyInitialized
+  // and take advantage of this in allocation entrypoints. b/36692143
+  if (new_status >= ClassStatus::kInitialized &&
+      old_status < ClassStatus::kInitialized &&
+      !h_this->IsVariableSize()) {
     DCHECK_EQ(h_this->GetObjectSizeAllocFastPath(), std::numeric_limits<uint32_t>::max());
     // Finalizable objects must always go slow path.
     if (!h_this->IsFinalizable()) {
diff --git a/runtime/mirror/class.h b/runtime/mirror/class.h
index 7c4c1d524b..960f49c5fc 100644
--- a/runtime/mirror/class.h
+++ b/runtime/mirror/class.h
@@ -89,12 +89,16 @@ class MANAGED Class final : public Object {
   static constexpr uint32_t kPrimitiveTypeSizeShiftShift = 16;
   static constexpr uint32_t kPrimitiveTypeMask = (1u << kPrimitiveTypeSizeShiftShift) - 1;
 
-  template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags>
+  template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags,
+           bool kWithSynchronizationBarrier = true>
   ClassStatus GetStatus() REQUIRES_SHARED(Locks::mutator_lock_) {
+    // Reading the field without barrier is used exclusively for IsVisiblyInitialized().
+    int32_t field_value = kWithSynchronizationBarrier
+        ? GetField32Volatile<kVerifyFlags>(StatusOffset())
+        : GetField32<kVerifyFlags>(StatusOffset());
     // Avoid including "subtype_check_bits_and_status.h" to get the field.
     // The ClassStatus is always in the 4 most-significant bits of status_.
-    return enum_cast<ClassStatus>(
-        static_cast<uint32_t>(GetField32Volatile<kVerifyFlags>(StatusOffset())) >> (32 - 4));
+    return enum_cast<ClassStatus>(static_cast<uint32_t>(field_value) >> (32 - 4));
   }
 
   // This is static because 'this' may be moved by GC.
@@ -176,7 +180,15 @@ class MANAGED Class final : public Object {
   // Returns true if the class is initialized.
   template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags>
   bool IsInitialized() REQUIRES_SHARED(Locks::mutator_lock_) {
-    return GetStatus<kVerifyFlags>() == ClassStatus::kInitialized;
+    return GetStatus<kVerifyFlags>() >= ClassStatus::kInitialized;
+  }
+
+  // Returns true if the class is visibly initialized.
+  template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags>
+  bool IsVisiblyInitialized() REQUIRES_SHARED(Locks::mutator_lock_) {
+    // Note: Avoiding the synchronization barrier for the visibly initialized check.
+    ClassStatus status = GetStatus<kVerifyFlags, /*kWithSynchronizationBarrier=*/ false>();
+    return status == ClassStatus::kVisiblyInitialized;
   }
 
   template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags>
diff --git a/runtime/runtime.cc b/runtime/runtime.cc
index f1b14dc2d8..91a3c45006 100644
--- a/runtime/runtime.cc
+++ b/runtime/runtime.cc
@@ -2384,6 +2384,10 @@ bool Runtime::IsActiveTransaction() const {
 void Runtime::EnterTransactionMode() {
   DCHECK(IsAotCompiler());
   DCHECK(!IsActiveTransaction());
+  // Make initialized classes visibly initialized now. If that happened during the transaction
+  // and then the transaction was aborted, we would roll back the status update but not the
+  // ClassLinker's bookkeeping structures, so these classes would never be visibly initialized.
+  GetClassLinker()->MakeInitializedClassesVisiblyInitialized(Thread::Current(), /*wait=*/ true);
   preinitialization_transactions_.push_back(std::make_unique<Transaction>());
 }
 
diff --git a/runtime/thread.h b/runtime/thread.h
index 98aaa926e5..dbb811b2ab 100644
--- a/runtime/thread.h
+++ b/runtime/thread.h
@@ -1229,6 +1229,15 @@ class Thread {
     return tls32_.force_interpreter_count != 0;
   }
 
+  bool IncrementMakeVisiblyInitializedCounter() {
+    tls32_.make_visibly_initialized_counter += 1u;
+    return tls32_.make_visibly_initialized_counter == kMakeVisiblyInitializedCounterTriggerCount;
+  }
+
+  void ClearMakeVisiblyInitializedCounter() {
+    tls32_.make_visibly_initialized_counter = 0u;
+  }
+
   void PushVerifier(verifier::MethodVerifier* verifier);
   void PopVerifier(verifier::MethodVerifier* verifier);
 
@@ -1473,6 +1482,8 @@ class Thread {
   // Stores the jit sensitive thread (which for now is the UI thread).
   static Thread* jit_sensitive_thread_;
 
+  static constexpr uint32_t kMakeVisiblyInitializedCounterTriggerCount = 1024;
+
   /***********************************************************************************************/
   // Thread local storage. Fields are grouped by size to enable 32 <-> 64 searching to account for
   // pointer size differences. To encourage shorter encoding, more frequently used values appear
@@ -1484,14 +1495,26 @@ class Thread {
     // to be 4-byte quantities.
     typedef uint32_t bool32_t;
 
-    explicit tls_32bit_sized_values(bool is_daemon) :
-      suspend_count(0), debug_suspend_count(0), thin_lock_thread_id(0), tid(0),
-      daemon(is_daemon), throwing_OutOfMemoryError(false), no_thread_suspension(0),
-      thread_exit_check_count(0), handling_signal_(false),
-      is_transitioning_to_runnable(false), ready_for_debug_invoke(false),
-      debug_method_entry_(false), is_gc_marking(false), weak_ref_access_enabled(true),
-      disable_thread_flip_count(0), user_code_suspend_count(0), force_interpreter_count(0) {
-    }
+    explicit tls_32bit_sized_values(bool is_daemon)
+        : suspend_count(0),
+          debug_suspend_count(0),
+          thin_lock_thread_id(0),
+          tid(0),
+          daemon(is_daemon),
+          throwing_OutOfMemoryError(false),
+          no_thread_suspension(0),
+          thread_exit_check_count(0),
+          handling_signal_(false),
+          is_transitioning_to_runnable(false),
+          ready_for_debug_invoke(false),
+          debug_method_entry_(false),
+          is_gc_marking(false),
+          weak_ref_access_enabled(true),
+          disable_thread_flip_count(0),
+          user_code_suspend_count(0),
+          force_interpreter_count(0),
+          use_mterp(0),
+          make_visibly_initialized_counter(0) {}
 
     union StateAndFlags state_and_flags;
     static_assert(sizeof(union StateAndFlags) == sizeof(int32_t),
@@ -1581,6 +1604,14 @@ class Thread {
     // True if everything is in the ideal state for fast interpretation.
     // False if we need to switch to the C++ interpreter to handle special cases.
     std::atomic<bool32_t> use_mterp;
+
+    // Counter for calls to initialize a class that's initialized but not visibly initialized.
+    // When this reaches kMakeVisiblyInitializedCounterTriggerCount, we call the runtime to
+    // make initialized classes visibly initialized. This is needed because we usually make
+    // classes visibly initialized in batches but we do not want to be stuck with a class
+    // initialized but not visibly initialized for a long time even if no more classes are
+    // being initialized anymore.
+    uint32_t make_visibly_initialized_counter;
   } tls32_;
 
   struct PACKED(8) tls_64bit_sized_values {