Clean up the runtime read barrier and fix fake address dependency.
- Rename GetReadBarrierPointer to GetReadBarrierState.
- Change its return type to uint32_t.
- Fix the runtime fake address dependency for arm/arm64 using inline
asm.
- Drop ReadBarrier::black_ptr_ and some brooks code.
Bug: 12687968
Test: test-art with CC, Ritz EAAC, libartd boot on N9.
Change-Id: I595970db825db5be2e98ee1fcbd7696d5501af55
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index d1ac139..d0bfeb5 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -2067,13 +2067,8 @@
void ImageWriter::FixupObject(Object* orig, Object* copy) {
DCHECK(orig != nullptr);
DCHECK(copy != nullptr);
- if (kUseBakerOrBrooksReadBarrier) {
- orig->AssertReadBarrierPointer();
- if (kUseBrooksReadBarrier) {
- // Note the address 'copy' isn't the same as the image address of 'orig'.
- copy->SetReadBarrierPointer(GetImageAddress(orig));
- DCHECK_EQ(copy->GetReadBarrierPointer(), GetImageAddress(orig));
- }
+ if (kUseBakerReadBarrier) {
+ orig->AssertReadBarrierState();
}
auto* klass = orig->GetClass();
if (klass->IsIntArrayClass() || klass->IsLongArrayClass()) {
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc
index be65f89..57823c9 100644
--- a/compiler/optimizing/code_generator_arm.cc
+++ b/compiler/optimizing/code_generator_arm.cc
@@ -6843,7 +6843,7 @@
// uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
// lfence; // Load fence or artificial data dependency to prevent load-load reordering
// HeapReference<Object> ref = *src; // Original reference load.
- // bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
// if (is_gray) {
// ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
// }
@@ -6919,14 +6919,13 @@
}
AddSlowPath(slow_path);
- // if (rb_state == ReadBarrier::gray_ptr_)
+ // if (rb_state == ReadBarrier::GrayState())
// ref = ReadBarrier::Mark(ref);
// Given the numeric representation, it's enough to check the low bit of the
// rb_state. We do that by shifting the bit out of the lock word with LSRS
// which can be a 16-bit instruction unlike the TST immediate.
- static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1");
- static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2");
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
__ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1);
__ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS.
__ Bind(slow_path->GetExitLabel());
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index b537509..b411a43 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -5426,7 +5426,7 @@
// uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
// lfence; // Load fence or artificial data dependency to prevent load-load reordering
// HeapReference<Object> ref = *src; // Original reference load.
- // bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
// if (is_gray) {
// ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
// }
@@ -5517,12 +5517,11 @@
}
AddSlowPath(slow_path);
- // if (rb_state == ReadBarrier::gray_ptr_)
+ // if (rb_state == ReadBarrier::GrayState())
// ref = ReadBarrier::Mark(ref);
// Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1");
- static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2");
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
__ Tbnz(temp, LockWord::kReadBarrierStateShift, slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel());
}
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index efd33c7..2f946e4 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -7093,7 +7093,7 @@
// uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
// lfence; // Load fence or artificial data dependency to prevent load-load reordering
// HeapReference<Object> ref = *src; // Original reference load.
- // bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
// if (is_gray) {
// ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
// }
@@ -7111,14 +7111,13 @@
uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
// Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1");
- static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2");
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
constexpr uint32_t gray_byte_position = LockWord::kReadBarrierStateShift / kBitsPerByte;
constexpr uint32_t gray_bit_position = LockWord::kReadBarrierStateShift % kBitsPerByte;
constexpr int32_t test_value = static_cast<int8_t>(1 << gray_bit_position);
- // if (rb_state == ReadBarrier::gray_ptr_)
+ // if (rb_state == ReadBarrier::GrayState())
// ref = ReadBarrier::Mark(ref);
// At this point, just do the "if" and make sure that flags are preserved until the branch.
__ testb(Address(obj, monitor_offset + gray_byte_position), Immediate(test_value));
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index fcabeea..49f33d2 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -6535,7 +6535,7 @@
// uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
// lfence; // Load fence or artificial data dependency to prevent load-load reordering
// HeapReference<Object> ref = *src; // Original reference load.
- // bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
// if (is_gray) {
// ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
// }
@@ -6553,14 +6553,13 @@
uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
// Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1");
- static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2");
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
constexpr uint32_t gray_byte_position = LockWord::kReadBarrierStateShift / kBitsPerByte;
constexpr uint32_t gray_bit_position = LockWord::kReadBarrierStateShift % kBitsPerByte;
constexpr int32_t test_value = static_cast<int8_t>(1 << gray_bit_position);
- // if (rb_state == ReadBarrier::gray_ptr_)
+ // if (rb_state == ReadBarrier::GrayState())
// ref = ReadBarrier::Mark(ref);
// At this point, just do the "if" and make sure that flags are preserved until the branch.
__ testb(Address(obj, monitor_offset + gray_byte_position), Immediate(test_value));
diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc
index 8790c1e..93a2340 100644
--- a/compiler/optimizing/intrinsics_arm.cc
+++ b/compiler/optimizing/intrinsics_arm.cc
@@ -1945,7 +1945,7 @@
// if (src_ptr != end_ptr) {
// uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState();
// lfence; // Load fence or artificial data dependency to prevent load-load reordering
- // bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
// if (is_gray) {
// // Slow-path copy.
// do {
@@ -1986,9 +1986,8 @@
// Given the numeric representation, it's enough to check the low bit of the
// rb_state. We do that by shifting the bit out of the lock word with LSRS
// which can be a 16-bit instruction unlike the TST immediate.
- static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1");
- static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2");
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
__ Lsrs(temp2, temp2, LockWord::kReadBarrierStateShift + 1);
// Carry flag is the last bit shifted out by LSRS.
__ b(read_barrier_slow_path->GetEntryLabel(), CS);
diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc
index db1c022..47e6d96 100644
--- a/compiler/optimizing/intrinsics_arm64.cc
+++ b/compiler/optimizing/intrinsics_arm64.cc
@@ -2659,7 +2659,7 @@
// if (src_ptr != end_ptr) {
// uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState();
// lfence; // Load fence or artificial data dependency to prevent load-load reordering
- // bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
// if (is_gray) {
// // Slow-path copy.
// do {
@@ -2704,9 +2704,8 @@
codegen_->AddSlowPath(read_barrier_slow_path);
// Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1");
- static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2");
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
__ Tbnz(tmp, LockWord::kReadBarrierStateShift, read_barrier_slow_path->GetEntryLabel());
// Fast-path copy.
diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc
index aae3899..43682c5 100644
--- a/compiler/optimizing/intrinsics_x86.cc
+++ b/compiler/optimizing/intrinsics_x86.cc
@@ -3200,7 +3200,7 @@
// if (src_ptr != end_ptr) {
// uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState();
// lfence; // Load fence or artificial data dependency to prevent load-load reordering
- // bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
// if (is_gray) {
// // Slow-path copy.
// for (size_t i = 0; i != length; ++i) {
@@ -3222,14 +3222,13 @@
__ j(kEqual, &done);
// Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1");
- static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2");
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
constexpr uint32_t gray_byte_position = LockWord::kReadBarrierStateShift / kBitsPerByte;
constexpr uint32_t gray_bit_position = LockWord::kReadBarrierStateShift % kBitsPerByte;
constexpr int32_t test_value = static_cast<int8_t>(1 << gray_bit_position);
- // if (rb_state == ReadBarrier::gray_ptr_)
+ // if (rb_state == ReadBarrier::GrayState())
// goto slow_path;
// At this point, just do the "if" and make sure that flags are preserved until the branch.
__ testb(Address(src, monitor_offset + gray_byte_position), Immediate(test_value));
diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc
index cdef22f..de2606c 100644
--- a/compiler/optimizing/intrinsics_x86_64.cc
+++ b/compiler/optimizing/intrinsics_x86_64.cc
@@ -1399,7 +1399,7 @@
// if (src_ptr != end_ptr) {
// uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState();
// lfence; // Load fence or artificial data dependency to prevent load-load reordering
- // bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
// if (is_gray) {
// // Slow-path copy.
// do {
@@ -1420,14 +1420,13 @@
__ j(kEqual, &done);
// Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1");
- static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2");
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
constexpr uint32_t gray_byte_position = LockWord::kReadBarrierStateShift / kBitsPerByte;
constexpr uint32_t gray_bit_position = LockWord::kReadBarrierStateShift % kBitsPerByte;
constexpr int32_t test_value = static_cast<int8_t>(1 << gray_bit_position);
- // if (rb_state == ReadBarrier::gray_ptr_)
+ // if (rb_state == ReadBarrier::GrayState())
// goto slow_path;
// At this point, just do the "if" and make sure that flags are preserved until the branch.
__ testb(Address(src, monitor_offset + gray_byte_position), Immediate(test_value));