Always mark reference referents in transaction mode.

Fix a to-space invariant check failure in EnqueueFinalizerReferences.

Reference processing can be a problem and useless during transaction
because it's not easy to roll back what reference processing does and
there's no daemon threads running (in the unstarted runtime). To avoid
issues, always mark reference referents.

Add a do_atomic_update parameter to MarkHeapReference.

Bug: 35417063
Test: test-art-host with CC/CMS/SS.
Change-Id: If32eba8fca19ef86e5d13f7925d179c8aecb9e27

1 files changed, 17 insertions, 0 deletions

diff --git a/runtime/gc/reference_processor.cc b/runtime/gc/reference_processor.cc
index 86b152211c..65a550ee5c 100644
--- a/runtime/gc/reference_processor.cc
+++ b/runtime/gc/reference_processor.cc
@@ -139,6 +139,14 @@ void ReferenceProcessor::ProcessReferences(bool concurrent,
       CHECK_EQ(!self->GetWeakRefAccessEnabled(), concurrent);
     }
   }
+  if (kIsDebugBuild && collector->IsTransactionActive()) {
+    // In transaction mode, we shouldn't enqueue any Reference to the queues.
+    // See DelayReferenceReferent().
+    DCHECK(soft_reference_queue_.IsEmpty());
+    DCHECK(weak_reference_queue_.IsEmpty());
+    DCHECK(finalizer_reference_queue_.IsEmpty());
+    DCHECK(phantom_reference_queue_.IsEmpty());
+  }
   // Unless required to clear soft references with white references, preserve some white referents.
   if (!clear_soft_references) {
     TimingLogger::ScopedTiming split(concurrent ? "ForwardSoftReferences" :
@@ -206,6 +214,15 @@ void ReferenceProcessor::DelayReferenceReferent(ObjPtr<mirror::Class> klass,
   // do_atomic_update needs to be true because this happens outside of the reference processing
   // phase.
   if (!collector->IsNullOrMarkedHeapReference(referent, /*do_atomic_update*/true)) {
+    if (UNLIKELY(collector->IsTransactionActive())) {
+      // In transaction mode, keep the referent alive and avoid any reference processing to avoid the
+      // issue of rolling back reference processing.  do_atomic_update needs to be true because this
+      // happens outside of the reference processing phase.
+      if (!referent->IsNull()) {
+        collector->MarkHeapReference(referent, /*do_atomic_update*/ true);
+      }
+      return;
+    }
     Thread* self = Thread::Current();
     // TODO: Remove these locks, and use atomic stacks for storing references?
     // We need to check that the references haven't already been enqueued since we can end up