Use _PRIVATE versions of futex ops.
This flag allows some performance optimizations in the kernel for futex
words that are only used in one process.
Tested: art$ grep FUTEX_ **/*.cc **/*.h
Change-Id: I490b9592ca0f0ab5ab5431682e8b2104f5c917ca
diff --git a/runtime/base/mutex-inl.h b/runtime/base/mutex-inl.h
index 74db312..e775fe4 100644
--- a/runtime/base/mutex-inl.h
+++ b/runtime/base/mutex-inl.h
@@ -196,7 +196,7 @@
if (num_pending_writers_.load(std::memory_order_seq_cst) > 0 ||
num_pending_readers_.load(std::memory_order_seq_cst) > 0) {
// Wake any exclusive waiters as there are now no readers.
- futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0);
+ futex(state_.Address(), FUTEX_WAKE_PRIVATE, -1, nullptr, nullptr, 0);
}
}
} else {
diff --git a/runtime/base/mutex.cc b/runtime/base/mutex.cc
index 0adcb37..ca2ed80 100644
--- a/runtime/base/mutex.cc
+++ b/runtime/base/mutex.cc
@@ -447,7 +447,7 @@
if (UNLIKELY(should_respond_to_empty_checkpoint_request_)) {
self->CheckEmptyCheckpointFromMutex();
}
- if (futex(state_.Address(), FUTEX_WAIT, 1, nullptr, nullptr, 0) != 0) {
+ if (futex(state_.Address(), FUTEX_WAIT_PRIVATE, 1, nullptr, nullptr, 0) != 0) {
// EAGAIN and EINTR both indicate a spurious failure, try again from the beginning.
// We don't use TEMP_FAILURE_RETRY so we can intentionally retry to acquire the lock.
if ((errno != EAGAIN) && (errno != EINTR)) {
@@ -551,7 +551,7 @@
if (LIKELY(done)) { // Spurious fail?
// Wake a contender.
if (UNLIKELY(num_contenders_.load(std::memory_order_seq_cst) > 0)) {
- futex(state_.Address(), FUTEX_WAKE, 1, nullptr, nullptr, 0);
+ futex(state_.Address(), FUTEX_WAKE_PRIVATE, 1, nullptr, nullptr, 0);
}
}
} else {
@@ -594,7 +594,7 @@
// Wake up all the waiters so they will respond to the emtpy checkpoint.
DCHECK(should_respond_to_empty_checkpoint_request_);
if (UNLIKELY(num_contenders_.load(std::memory_order_relaxed) > 0)) {
- futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0);
+ futex(state_.Address(), FUTEX_WAKE_PRIVATE, -1, nullptr, nullptr, 0);
}
#else
LOG(FATAL) << "Non futex case isn't supported.";
@@ -648,7 +648,7 @@
if (UNLIKELY(should_respond_to_empty_checkpoint_request_)) {
self->CheckEmptyCheckpointFromMutex();
}
- if (futex(state_.Address(), FUTEX_WAIT, cur_state, nullptr, nullptr, 0) != 0) {
+ if (futex(state_.Address(), FUTEX_WAIT_PRIVATE, cur_state, nullptr, nullptr, 0) != 0) {
// EAGAIN and EINTR both indicate a spurious failure, try again from the beginning.
// We don't use TEMP_FAILURE_RETRY so we can intentionally retry to acquire the lock.
if ((errno != EAGAIN) && (errno != EINTR)) {
@@ -689,7 +689,7 @@
// Wake any waiters.
if (UNLIKELY(num_pending_readers_.load(std::memory_order_seq_cst) > 0 ||
num_pending_writers_.load(std::memory_order_seq_cst) > 0)) {
- futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0);
+ futex(state_.Address(), FUTEX_WAKE_PRIVATE, -1, nullptr, nullptr, 0);
}
}
} else {
@@ -727,7 +727,7 @@
if (UNLIKELY(should_respond_to_empty_checkpoint_request_)) {
self->CheckEmptyCheckpointFromMutex();
}
- if (futex(state_.Address(), FUTEX_WAIT, cur_state, &rel_ts, nullptr, 0) != 0) {
+ if (futex(state_.Address(), FUTEX_WAIT_PRIVATE, cur_state, &rel_ts, nullptr, 0) != 0) {
if (errno == ETIMEDOUT) {
--num_pending_writers_;
return false; // Timed out.
@@ -768,7 +768,7 @@
if (UNLIKELY(should_respond_to_empty_checkpoint_request_)) {
self->CheckEmptyCheckpointFromMutex();
}
- if (futex(state_.Address(), FUTEX_WAIT, cur_state, nullptr, nullptr, 0) != 0) {
+ if (futex(state_.Address(), FUTEX_WAIT_PRIVATE, cur_state, nullptr, nullptr, 0) != 0) {
if (errno != EAGAIN && errno != EINTR) {
PLOG(FATAL) << "futex wait failed for " << name_;
}
@@ -846,7 +846,7 @@
DCHECK(should_respond_to_empty_checkpoint_request_);
if (UNLIKELY(num_pending_readers_.load(std::memory_order_relaxed) > 0 ||
num_pending_writers_.load(std::memory_order_relaxed) > 0)) {
- futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0);
+ futex(state_.Address(), FUTEX_WAKE_PRIVATE, -1, nullptr, nullptr, 0);
}
#else
LOG(FATAL) << "Non futex case isn't supported.";
@@ -908,7 +908,7 @@
// Move waiters from the condition variable's futex to the guard's futex,
// so that they will be woken up when the mutex is released.
bool done = futex(sequence_.Address(),
- FUTEX_REQUEUE,
+ FUTEX_REQUEUE_PRIVATE,
/* Threads to wake */ 0,
/* Threads to requeue*/ reinterpret_cast<const timespec*>(count),
guard_.state_.Address(),
@@ -947,7 +947,7 @@
guard_.recursion_count_ = 1;
int32_t cur_sequence = sequence_.load(std::memory_order_relaxed);
guard_.ExclusiveUnlock(self);
- if (futex(sequence_.Address(), FUTEX_WAIT, cur_sequence, nullptr, nullptr, 0) != 0) {
+ if (futex(sequence_.Address(), FUTEX_WAIT_PRIVATE, cur_sequence, nullptr, nullptr, 0) != 0) {
// Futex failed, check it is an expected error.
// EAGAIN == EWOULDBLK, so we let the caller try again.
// EINTR implies a signal was sent to this thread.
@@ -998,7 +998,7 @@
guard_.recursion_count_ = 1;
int32_t cur_sequence = sequence_.load(std::memory_order_relaxed);
guard_.ExclusiveUnlock(self);
- if (futex(sequence_.Address(), FUTEX_WAIT, cur_sequence, &rel_ts, nullptr, 0) != 0) {
+ if (futex(sequence_.Address(), FUTEX_WAIT_PRIVATE, cur_sequence, &rel_ts, nullptr, 0) != 0) {
if (errno == ETIMEDOUT) {
// Timed out we're done.
timed_out = true;
diff --git a/runtime/thread.cc b/runtime/thread.cc
index b5d214d..dda3b82 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -1351,7 +1351,7 @@
done = pending_threads->CompareAndSetWeakRelaxed(cur_val, cur_val - 1);
#if ART_USE_FUTEXES
if (done && (cur_val - 1) == 0) { // Weak CAS may fail spuriously.
- futex(pending_threads->Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0);
+ futex(pending_threads->Address(), FUTEX_WAKE_PRIVATE, -1, nullptr, nullptr, 0);
}
#endif
} while (!done);
diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc
index d21b600..23e57a2 100644
--- a/runtime/thread_list.cc
+++ b/runtime/thread_list.cc
@@ -764,7 +764,8 @@
int32_t cur_val = pending_threads.load(std::memory_order_relaxed);
if (LIKELY(cur_val > 0)) {
#if ART_USE_FUTEXES
- if (futex(pending_threads.Address(), FUTEX_WAIT, cur_val, &wait_timeout, nullptr, 0) != 0) {
+ if (futex(pending_threads.Address(), FUTEX_WAIT_PRIVATE, cur_val, &wait_timeout, nullptr, 0)
+ != 0) {
// EAGAIN and EINTR both indicate a spurious failure, try again from the beginning.
if ((errno != EAGAIN) && (errno != EINTR)) {
if (errno == ETIMEDOUT) {