runtime/read_barrier-inl.h - LeafOS-Project/android_art - Gitiles

 /*
  * Copyright (C) 2014 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #ifndef ART_RUNTIME_READ_BARRIER_INL_H_
 #define ART_RUNTIME_READ_BARRIER_INL_H_

 #include "read_barrier.h"

 #include "gc/collector/concurrent_copying.h"
 #include "gc/heap.h"
 #include "mirror/object_reference.h"
 #include "mirror/reference.h"
 #include "runtime.h"
 #include "utils.h"

 namespace art {

 template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup>
 inline MirrorType* ReadBarrier::Barrier(
     mirror::Object* obj, MemberOffset offset, mirror::HeapReference<MirrorType>* ref_addr) {
   constexpr bool with_read_barrier = kReadBarrierOption == kWithReadBarrier;
   if (with_read_barrier && kUseBakerReadBarrier) {
     // The higher bits of the rb ptr, rb_ptr_high_bits (must be zero)
     // is used to create artificial data dependency from the is_gray
     // load to the ref field (ptr) load to avoid needing a load-load
     // barrier between the two.
     uintptr_t rb_ptr_high_bits;
     bool is_gray = HasGrayReadBarrierPointer(obj, &rb_ptr_high_bits);
     ref_addr = reinterpret_cast<mirror::HeapReference<MirrorType>*>(
         rb_ptr_high_bits | reinterpret_cast<uintptr_t>(ref_addr));
     MirrorType* ref = ref_addr->AsMirrorPtr();
     if (is_gray) {
       // Slow-path.
       ref = reinterpret_cast<MirrorType*>(Mark(ref));
     }
     if (kEnableReadBarrierInvariantChecks) {
       CHECK_EQ(rb_ptr_high_bits, 0U) << obj << " rb_ptr=" << obj->GetReadBarrierPointer();
     }
     AssertToSpaceInvariant(obj, offset, ref);
     return ref;
   } else if (with_read_barrier && kUseBrooksReadBarrier) {
     // To be implemented.
     return ref_addr->AsMirrorPtr();
   } else if (with_read_barrier && kUseTableLookupReadBarrier) {
     MirrorType* ref = ref_addr->AsMirrorPtr();
     MirrorType* old_ref = ref;
     // The heap or the collector can be null at startup. TODO: avoid the need for this null check.
     gc::Heap* heap = Runtime::Current()->GetHeap();
     if (heap != nullptr && heap->GetReadBarrierTable()->IsSet(old_ref)) {
       ref = reinterpret_cast<MirrorType*>(Mark(old_ref));
       // Update the field atomically. This may fail if mutator updates before us, but it's ok.
       if (ref != old_ref) {
         obj->CasFieldStrongRelaxedObjectWithoutWriteBarrier<false, false>(
             offset, old_ref, ref);
       }
     }
     AssertToSpaceInvariant(obj, offset, ref);
     return ref;
   } else {
     // No read barrier.
     return ref_addr->AsMirrorPtr();
   }
 }

 template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup>
 inline MirrorType* ReadBarrier::BarrierForRoot(MirrorType** root,
                                                GcRootSource* gc_root_source) {
   MirrorType* ref = *root;
   const bool with_read_barrier = kReadBarrierOption == kWithReadBarrier;
   if (with_read_barrier && kUseBakerReadBarrier) {
     // TODO: separate the read barrier code from the collector code more.
     Thread* self = Thread::Current();
     if (self != nullptr && self->GetIsGcMarking()) {
       ref = reinterpret_cast<MirrorType*>(Mark(ref));
     }
     AssertToSpaceInvariant(gc_root_source, ref);
     return ref;
   } else if (with_read_barrier && kUseBrooksReadBarrier) {
     // To be implemented.
     return ref;
   } else if (with_read_barrier && kUseTableLookupReadBarrier) {
     Thread* self = Thread::Current();
     if (self != nullptr &&
         self->GetIsGcMarking() &&
         Runtime::Current()->GetHeap()->GetReadBarrierTable()->IsSet(ref)) {
       MirrorType* old_ref = ref;
       ref = reinterpret_cast<MirrorType*>(Mark(old_ref));
       // Update the field atomically. This may fail if mutator updates before us, but it's ok.
       if (ref != old_ref) {
         Atomic<mirror::Object*>* atomic_root = reinterpret_cast<Atomic<mirror::Object*>*>(root);
         atomic_root->CompareExchangeStrongRelaxed(old_ref, ref);
       }
     }
     AssertToSpaceInvariant(gc_root_source, ref);
     return ref;
   } else {
     return ref;
   }
 }

 // TODO: Reduce copy paste
 template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup>
 inline MirrorType* ReadBarrier::BarrierForRoot(mirror::CompressedReference<MirrorType>* root,
                                                GcRootSource* gc_root_source) {
   MirrorType* ref = root->AsMirrorPtr();
   const bool with_read_barrier = kReadBarrierOption == kWithReadBarrier;
   if (with_read_barrier && kUseBakerReadBarrier) {
     // TODO: separate the read barrier code from the collector code more.
     Thread* self = Thread::Current();
     if (self != nullptr && self->GetIsGcMarking()) {
       ref = reinterpret_cast<MirrorType*>(Mark(ref));
     }
     AssertToSpaceInvariant(gc_root_source, ref);
     return ref;
   } else if (with_read_barrier && kUseBrooksReadBarrier) {
     // To be implemented.
     return ref;
   } else if (with_read_barrier && kUseTableLookupReadBarrier) {
     Thread* self = Thread::Current();
     if (self != nullptr &&
         self->GetIsGcMarking() &&
         Runtime::Current()->GetHeap()->GetReadBarrierTable()->IsSet(ref)) {
       auto old_ref = mirror::CompressedReference<MirrorType>::FromMirrorPtr(ref);
       ref = reinterpret_cast<MirrorType*>(Mark(ref));
       auto new_ref = mirror::CompressedReference<MirrorType>::FromMirrorPtr(ref);
       // Update the field atomically. This may fail if mutator updates before us, but it's ok.
       if (new_ref.AsMirrorPtr() != old_ref.AsMirrorPtr()) {
         auto* atomic_root =
             reinterpret_cast<Atomic<mirror::CompressedReference<MirrorType>>*>(root);
         atomic_root->CompareExchangeStrongRelaxed(old_ref, new_ref);
       }
     }
     AssertToSpaceInvariant(gc_root_source, ref);
     return ref;
   } else {
     return ref;
   }
 }

 inline bool ReadBarrier::IsDuringStartup() {
   gc::Heap* heap = Runtime::Current()->GetHeap();
   if (heap == nullptr) {
     // During startup, the heap can be null.
     return true;
   }
   if (heap->CurrentCollectorType() != gc::kCollectorTypeCC) {
     // CC isn't running.
     return true;
   }
   gc::collector::ConcurrentCopying* collector = heap->ConcurrentCopyingCollector();
   if (collector == nullptr) {
     // During startup, the collector can be null.
     return true;
   }
   return false;
 }

 inline void ReadBarrier::AssertToSpaceInvariant(mirror::Object* obj, MemberOffset offset,
                                                 mirror::Object* ref) {
   if (kEnableToSpaceInvariantChecks || kIsDebugBuild) {
     if (ref == nullptr || IsDuringStartup()) {
       return;
     }
     Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->
         AssertToSpaceInvariant(obj, offset, ref);
   }
 }

 inline void ReadBarrier::AssertToSpaceInvariant(GcRootSource* gc_root_source,
                                                 mirror::Object* ref) {
   if (kEnableToSpaceInvariantChecks || kIsDebugBuild) {
     if (ref == nullptr || IsDuringStartup()) {
       return;
     }
     Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->
         AssertToSpaceInvariant(gc_root_source, ref);
   }
 }

 inline mirror::Object* ReadBarrier::Mark(mirror::Object* obj) {
   return Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->Mark(obj);
 }

 inline bool ReadBarrier::HasGrayReadBarrierPointer(mirror::Object* obj,
                                                    uintptr_t* out_rb_ptr_high_bits) {
   mirror::Object* rb_ptr = obj->GetReadBarrierPointer();
   uintptr_t rb_ptr_bits = reinterpret_cast<uintptr_t>(rb_ptr);
   uintptr_t rb_ptr_low_bits = rb_ptr_bits & rb_ptr_mask_;
   if (kEnableReadBarrierInvariantChecks) {
     CHECK(rb_ptr_low_bits == white_ptr_ || rb_ptr_low_bits == gray_ptr_ ||
           rb_ptr_low_bits == black_ptr_)
         << "obj=" << obj << " rb_ptr=" << rb_ptr << " " << PrettyTypeOf(obj);
   }
   bool is_gray = rb_ptr_low_bits == gray_ptr_;
   // The high bits are supposed to be zero. We check this on the caller side.
   *out_rb_ptr_high_bits = rb_ptr_bits & ~rb_ptr_mask_;
   return is_gray;
 }

 }  // namespace art

 #endif  // ART_RUNTIME_READ_BARRIER_INL_H_
	/*
	* Copyright (C) 2014 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#ifndef ART_RUNTIME_READ_BARRIER_INL_H_
	#define ART_RUNTIME_READ_BARRIER_INL_H_

	#include "read_barrier.h"

	#include "gc/collector/concurrent_copying.h"
	#include "gc/heap.h"
	#include "mirror/object_reference.h"
	#include "mirror/reference.h"
	#include "runtime.h"
	#include "utils.h"

	namespace art {

	template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup>
	inline MirrorType* ReadBarrier::Barrier(
	mirror::Object* obj, MemberOffset offset, mirror::HeapReference<MirrorType>* ref_addr) {
	constexpr bool with_read_barrier = kReadBarrierOption == kWithReadBarrier;
	if (with_read_barrier && kUseBakerReadBarrier) {
	// The higher bits of the rb ptr, rb_ptr_high_bits (must be zero)
	// is used to create artificial data dependency from the is_gray
	// load to the ref field (ptr) load to avoid needing a load-load
	// barrier between the two.
	uintptr_t rb_ptr_high_bits;
	bool is_gray = HasGrayReadBarrierPointer(obj, &rb_ptr_high_bits);
	ref_addr = reinterpret_cast<mirror::HeapReference<MirrorType>*>(
	rb_ptr_high_bits \| reinterpret_cast<uintptr_t>(ref_addr));
	MirrorType* ref = ref_addr->AsMirrorPtr();
	if (is_gray) {
	// Slow-path.
	ref = reinterpret_cast<MirrorType*>(Mark(ref));
	}
	if (kEnableReadBarrierInvariantChecks) {
	CHECK_EQ(rb_ptr_high_bits, 0U) << obj << " rb_ptr=" << obj->GetReadBarrierPointer();
	}
	AssertToSpaceInvariant(obj, offset, ref);
	return ref;
	} else if (with_read_barrier && kUseBrooksReadBarrier) {
	// To be implemented.
	return ref_addr->AsMirrorPtr();
	} else if (with_read_barrier && kUseTableLookupReadBarrier) {
	MirrorType* ref = ref_addr->AsMirrorPtr();
	MirrorType* old_ref = ref;
	// The heap or the collector can be null at startup. TODO: avoid the need for this null check.
	gc::Heap* heap = Runtime::Current()->GetHeap();
	if (heap != nullptr && heap->GetReadBarrierTable()->IsSet(old_ref)) {
	ref = reinterpret_cast<MirrorType*>(Mark(old_ref));
	// Update the field atomically. This may fail if mutator updates before us, but it's ok.
	if (ref != old_ref) {
	obj->CasFieldStrongRelaxedObjectWithoutWriteBarrier<false, false>(
	offset, old_ref, ref);
	}
	}
	AssertToSpaceInvariant(obj, offset, ref);
	return ref;
	} else {
	// No read barrier.
	return ref_addr->AsMirrorPtr();
	}
	}

	template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup>
	inline MirrorType* ReadBarrier::BarrierForRoot(MirrorType** root,
	GcRootSource* gc_root_source) {
	MirrorType* ref = *root;
	const bool with_read_barrier = kReadBarrierOption == kWithReadBarrier;
	if (with_read_barrier && kUseBakerReadBarrier) {
	// TODO: separate the read barrier code from the collector code more.
	Thread* self = Thread::Current();
	if (self != nullptr && self->GetIsGcMarking()) {
	ref = reinterpret_cast<MirrorType*>(Mark(ref));
	}
	AssertToSpaceInvariant(gc_root_source, ref);
	return ref;
	} else if (with_read_barrier && kUseBrooksReadBarrier) {
	// To be implemented.
	return ref;
	} else if (with_read_barrier && kUseTableLookupReadBarrier) {
	Thread* self = Thread::Current();
	if (self != nullptr &&
	self->GetIsGcMarking() &&
	Runtime::Current()->GetHeap()->GetReadBarrierTable()->IsSet(ref)) {
	MirrorType* old_ref = ref;
	ref = reinterpret_cast<MirrorType*>(Mark(old_ref));
	// Update the field atomically. This may fail if mutator updates before us, but it's ok.
	if (ref != old_ref) {
	Atomic<mirror::Object> atomic_root = reinterpret_cast<Atomic<mirror::Object>>(root);
	atomic_root->CompareExchangeStrongRelaxed(old_ref, ref);
	}
	}
	AssertToSpaceInvariant(gc_root_source, ref);
	return ref;
	} else {
	return ref;
	}
	}

	// TODO: Reduce copy paste
	template <typename MirrorType, ReadBarrierOption kReadBarrierOption, bool kMaybeDuringStartup>
	inline MirrorType* ReadBarrier::BarrierForRoot(mirror::CompressedReference<MirrorType>* root,
	GcRootSource* gc_root_source) {
	MirrorType* ref = root->AsMirrorPtr();
	const bool with_read_barrier = kReadBarrierOption == kWithReadBarrier;
	if (with_read_barrier && kUseBakerReadBarrier) {
	// TODO: separate the read barrier code from the collector code more.
	Thread* self = Thread::Current();
	if (self != nullptr && self->GetIsGcMarking()) {
	ref = reinterpret_cast<MirrorType*>(Mark(ref));
	}
	AssertToSpaceInvariant(gc_root_source, ref);
	return ref;
	} else if (with_read_barrier && kUseBrooksReadBarrier) {
	// To be implemented.
	return ref;
	} else if (with_read_barrier && kUseTableLookupReadBarrier) {
	Thread* self = Thread::Current();
	if (self != nullptr &&
	self->GetIsGcMarking() &&
	Runtime::Current()->GetHeap()->GetReadBarrierTable()->IsSet(ref)) {
	auto old_ref = mirror::CompressedReference<MirrorType>::FromMirrorPtr(ref);
	ref = reinterpret_cast<MirrorType*>(Mark(ref));
	auto new_ref = mirror::CompressedReference<MirrorType>::FromMirrorPtr(ref);
	// Update the field atomically. This may fail if mutator updates before us, but it's ok.
	if (new_ref.AsMirrorPtr() != old_ref.AsMirrorPtr()) {
	auto* atomic_root =
	reinterpret_cast<Atomic<mirror::CompressedReference<MirrorType>>*>(root);
	atomic_root->CompareExchangeStrongRelaxed(old_ref, new_ref);
	}
	}
	AssertToSpaceInvariant(gc_root_source, ref);
	return ref;
	} else {
	return ref;
	}
	}

	inline bool ReadBarrier::IsDuringStartup() {
	gc::Heap* heap = Runtime::Current()->GetHeap();
	if (heap == nullptr) {
	// During startup, the heap can be null.
	return true;
	}
	if (heap->CurrentCollectorType() != gc::kCollectorTypeCC) {
	// CC isn't running.
	return true;
	}
	gc::collector::ConcurrentCopying* collector = heap->ConcurrentCopyingCollector();
	if (collector == nullptr) {
	// During startup, the collector can be null.
	return true;
	}
	return false;
	}

	inline void ReadBarrier::AssertToSpaceInvariant(mirror::Object* obj, MemberOffset offset,
	mirror::Object* ref) {
	if (kEnableToSpaceInvariantChecks \|\| kIsDebugBuild) {
	if (ref == nullptr \|\| IsDuringStartup()) {
	return;
	}
	Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->
	AssertToSpaceInvariant(obj, offset, ref);
	}
	}

	inline void ReadBarrier::AssertToSpaceInvariant(GcRootSource* gc_root_source,
	mirror::Object* ref) {
	if (kEnableToSpaceInvariantChecks \|\| kIsDebugBuild) {
	if (ref == nullptr \|\| IsDuringStartup()) {
	return;
	}
	Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->
	AssertToSpaceInvariant(gc_root_source, ref);
	}
	}

	inline mirror::Object* ReadBarrier::Mark(mirror::Object* obj) {
	return Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->Mark(obj);
	}

	inline bool ReadBarrier::HasGrayReadBarrierPointer(mirror::Object* obj,
	uintptr_t* out_rb_ptr_high_bits) {
	mirror::Object* rb_ptr = obj->GetReadBarrierPointer();
	uintptr_t rb_ptr_bits = reinterpret_cast<uintptr_t>(rb_ptr);
	uintptr_t rb_ptr_low_bits = rb_ptr_bits & rb_ptr_mask_;
	if (kEnableReadBarrierInvariantChecks) {
	CHECK(rb_ptr_low_bits == white_ptr_ \|\| rb_ptr_low_bits == gray_ptr_ \|\|
	rb_ptr_low_bits == black_ptr_)
	<< "obj=" << obj << " rb_ptr=" << rb_ptr << " " << PrettyTypeOf(obj);
	}
	bool is_gray = rb_ptr_low_bits == gray_ptr_;
	// The high bits are supposed to be zero. We check this on the caller side.
	*out_rb_ptr_high_bits = rb_ptr_bits & ~rb_ptr_mask_;
	return is_gray;
	}

	} // namespace art

	#endif // ART_RUNTIME_READ_BARRIER_INL_H_