Inflate contended lock word by suspending owner.
Bug 6961405.
Don't inflate monitors for Notify and NotifyAll.
Tidy lock word, handle recursive lock case alongside unlocked case and move
assembly out of line (except for ARM quick). Also handle null in out-of-line
assembly as the test is quick and the enter/exit code is already a safepoint.
To gain ownership of a monitor on behalf of another thread, monitor contenders
must not hold the monitor_lock_, so they wait on a condition variable.
Reduce size of per mutex contention log.
Be consistent in calling thin lock thread ids just thread ids.
Fix potential thread death races caused by the use of FindThreadByThreadId,
make it invariant that returned threads are either self or suspended now.
Code size reduction on ARM boot.oat 0.2%.
Old nexus 7 speedup 0.25%, new nexus 7 speedup 1.4%, nexus 10 speedup 2.24%,
nexus 4 speedup 2.09% on DeltaBlue.
Change-Id: Id52558b914f160d9c8578fdd7fc8199a9598576a
diff --git a/compiler/llvm/llvm_compilation_unit.cc b/compiler/llvm/llvm_compilation_unit.cc
index aa439cc..feb495e 100644
--- a/compiler/llvm/llvm_compilation_unit.cc
+++ b/compiler/llvm/llvm_compilation_unit.cc
@@ -82,7 +82,6 @@
#include "ir_builder.h"
#include "os.h"
#include "runtime_support_builder_arm.h"
-#include "runtime_support_builder_thumb2.h"
#include "runtime_support_builder_x86.h"
#include "utils_llvm.h"
@@ -118,12 +117,10 @@
default:
runtime_support_.reset(new RuntimeSupportBuilder(*context_, *module_, *irb_));
break;
+ case kThumb2:
case kArm:
runtime_support_.reset(new RuntimeSupportBuilderARM(*context_, *module_, *irb_));
break;
- case kThumb2:
- runtime_support_.reset(new RuntimeSupportBuilderThumb2(*context_, *module_, *irb_));
- break;
case kX86:
runtime_support_.reset(new RuntimeSupportBuilderX86(*context_, *module_, *irb_));
break;
diff --git a/compiler/llvm/runtime_support_builder.cc b/compiler/llvm/runtime_support_builder.cc
index 24e283d..c825fbf 100644
--- a/compiler/llvm/runtime_support_builder.cc
+++ b/compiler/llvm/runtime_support_builder.cc
@@ -164,89 +164,13 @@
/* Monitor */
void RuntimeSupportBuilder::EmitLockObject(::llvm::Value* object) {
- Value* monitor =
- irb_.LoadFromObjectOffset(object,
- mirror::Object::MonitorOffset().Int32Value(),
- irb_.getJIntTy(),
- kTBAARuntimeInfo);
-
- Value* real_monitor =
- irb_.CreateAnd(monitor, ~(LW_HASH_STATE_MASK << LW_HASH_STATE_SHIFT));
-
- // Is thin lock, unheld and not recursively acquired.
- Value* unheld = irb_.CreateICmpEQ(real_monitor, irb_.getInt32(0));
-
- Function* parent_func = irb_.GetInsertBlock()->getParent();
- BasicBlock* bb_fast = BasicBlock::Create(context_, "lock_fast", parent_func);
- BasicBlock* bb_slow = BasicBlock::Create(context_, "lock_slow", parent_func);
- BasicBlock* bb_cont = BasicBlock::Create(context_, "lock_cont", parent_func);
- irb_.CreateCondBr(unheld, bb_fast, bb_slow, kLikely);
-
- irb_.SetInsertPoint(bb_fast);
-
- // Calculate new monitor: new = old | (lock_id << LW_LOCK_OWNER_SHIFT)
- Value* lock_id =
- EmitLoadFromThreadOffset(Thread::ThinLockIdOffset().Int32Value(),
- irb_.getInt32Ty(), kTBAARuntimeInfo);
-
- Value* owner = irb_.CreateShl(lock_id, LW_LOCK_OWNER_SHIFT);
- Value* new_monitor = irb_.CreateOr(monitor, owner);
-
- // Atomically update monitor.
- Value* old_monitor =
- irb_.CompareExchangeObjectOffset(object,
- mirror::Object::MonitorOffset().Int32Value(),
- monitor, new_monitor, kTBAARuntimeInfo);
-
- Value* retry_slow_path = irb_.CreateICmpEQ(old_monitor, monitor);
- irb_.CreateCondBr(retry_slow_path, bb_cont, bb_slow, kLikely);
-
- irb_.SetInsertPoint(bb_slow);
Function* slow_func = GetRuntimeSupportFunction(runtime_support::LockObject);
irb_.CreateCall2(slow_func, object, EmitGetCurrentThread());
- irb_.CreateBr(bb_cont);
-
- irb_.SetInsertPoint(bb_cont);
}
void RuntimeSupportBuilder::EmitUnlockObject(::llvm::Value* object) {
- Value* lock_id =
- EmitLoadFromThreadOffset(Thread::ThinLockIdOffset().Int32Value(),
- irb_.getJIntTy(),
- kTBAARuntimeInfo);
- Value* monitor =
- irb_.LoadFromObjectOffset(object,
- mirror::Object::MonitorOffset().Int32Value(),
- irb_.getJIntTy(),
- kTBAARuntimeInfo);
-
- Value* my_monitor = irb_.CreateShl(lock_id, LW_LOCK_OWNER_SHIFT);
- Value* hash_state = irb_.CreateAnd(monitor, (LW_HASH_STATE_MASK << LW_HASH_STATE_SHIFT));
- Value* real_monitor = irb_.CreateAnd(monitor, ~(LW_HASH_STATE_MASK << LW_HASH_STATE_SHIFT));
-
- // Is thin lock, held by us and not recursively acquired
- Value* is_fast_path = irb_.CreateICmpEQ(real_monitor, my_monitor);
-
- Function* parent_func = irb_.GetInsertBlock()->getParent();
- BasicBlock* bb_fast = BasicBlock::Create(context_, "unlock_fast", parent_func);
- BasicBlock* bb_slow = BasicBlock::Create(context_, "unlock_slow", parent_func);
- BasicBlock* bb_cont = BasicBlock::Create(context_, "unlock_cont", parent_func);
- irb_.CreateCondBr(is_fast_path, bb_fast, bb_slow, kLikely);
-
- irb_.SetInsertPoint(bb_fast);
- // Set all bits to zero (except hash state)
- irb_.StoreToObjectOffset(object,
- mirror::Object::MonitorOffset().Int32Value(),
- hash_state,
- kTBAARuntimeInfo);
- irb_.CreateBr(bb_cont);
-
- irb_.SetInsertPoint(bb_slow);
Function* slow_func = GetRuntimeSupportFunction(runtime_support::UnlockObject);
irb_.CreateCall2(slow_func, object, EmitGetCurrentThread());
- irb_.CreateBr(bb_cont);
-
- irb_.SetInsertPoint(bb_cont);
}
diff --git a/compiler/llvm/runtime_support_builder.h b/compiler/llvm/runtime_support_builder.h
index e92ac0a..898611a 100644
--- a/compiler/llvm/runtime_support_builder.h
+++ b/compiler/llvm/runtime_support_builder.h
@@ -64,8 +64,8 @@
virtual void EmitTestSuspend();
/* Monitor */
- virtual void EmitLockObject(::llvm::Value* object);
- virtual void EmitUnlockObject(::llvm::Value* object);
+ void EmitLockObject(::llvm::Value* object);
+ void EmitUnlockObject(::llvm::Value* object);
/* MarkGCCard */
virtual void EmitMarkGCCard(::llvm::Value* value, ::llvm::Value* target_addr);
diff --git a/compiler/llvm/runtime_support_builder_arm.cc b/compiler/llvm/runtime_support_builder_arm.cc
index 569d825..cad4624 100644
--- a/compiler/llvm/runtime_support_builder_arm.cc
+++ b/compiler/llvm/runtime_support_builder_arm.cc
@@ -116,24 +116,5 @@
return old_thread_register;
}
-
-/* Monitor */
-
-void RuntimeSupportBuilderARM::EmitLockObject(Value* object) {
- RuntimeSupportBuilder::EmitLockObject(object);
- FunctionType* func_ty = FunctionType::get(/*Result=*/Type::getVoidTy(context_),
- /*isVarArg=*/false);
- InlineAsm* func = InlineAsm::get(func_ty, "dmb sy", "", true);
- irb_.CreateCall(func);
-}
-
-void RuntimeSupportBuilderARM::EmitUnlockObject(Value* object) {
- RuntimeSupportBuilder::EmitUnlockObject(object);
- FunctionType* func_ty = FunctionType::get(/*Result=*/Type::getVoidTy(context_),
- /*isVarArg=*/false);
- InlineAsm* func = InlineAsm::get(func_ty, "dmb sy", "", true);
- irb_.CreateCall(func);
-}
-
} // namespace llvm
} // namespace art
diff --git a/compiler/llvm/runtime_support_builder_arm.h b/compiler/llvm/runtime_support_builder_arm.h
index 5a353d7..0d01509 100644
--- a/compiler/llvm/runtime_support_builder_arm.h
+++ b/compiler/llvm/runtime_support_builder_arm.h
@@ -34,10 +34,6 @@
virtual void EmitStoreToThreadOffset(int64_t offset, ::llvm::Value* value,
TBAASpecialType s_ty);
virtual ::llvm::Value* EmitSetCurrentThread(::llvm::Value* thread);
-
- /* Monitor */
- virtual void EmitLockObject(::llvm::Value* object);
- virtual void EmitUnlockObject(::llvm::Value* object);
};
} // namespace llvm
diff --git a/compiler/llvm/runtime_support_builder_thumb2.cc b/compiler/llvm/runtime_support_builder_thumb2.cc
deleted file mode 100644
index eff29c8..0000000
--- a/compiler/llvm/runtime_support_builder_thumb2.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "runtime_support_builder_thumb2.h"
-
-#include "ir_builder.h"
-#include "mirror/object.h"
-#include "monitor.h"
-#include "thread.h"
-#include "utils_llvm.h"
-
-#include <llvm/IR/DerivedTypes.h>
-#include <llvm/IR/Function.h>
-#include <llvm/IR/InlineAsm.h>
-#include <llvm/IR/Module.h>
-#include <llvm/IR/Type.h>
-
-#include <inttypes.h>
-#include <vector>
-
-using ::llvm::BasicBlock;
-using ::llvm::Function;
-using ::llvm::FunctionType;
-using ::llvm::InlineAsm;
-using ::llvm::Type;
-using ::llvm::Value;
-
-namespace art {
-namespace llvm {
-
-
-void RuntimeSupportBuilderThumb2::EmitLockObject(Value* object) {
- FunctionType* func_ty = FunctionType::get(/*Result=*/irb_.getInt32Ty(),
- /*Params=*/irb_.getJObjectTy(),
- /*isVarArg=*/false);
- // $0: result
- // $1: object
- // $2: temp
- // $3: temp
- std::string asms;
- StringAppendF(&asms, "add $3, $1, #%" PRId32 "\n", mirror::Object::MonitorOffset().Int32Value());
- StringAppendF(&asms, "ldr $2, [r9, #%" PRId32 "]\n", Thread::ThinLockIdOffset().Int32Value());
- StringAppendF(&asms, "ldrex $0, [$3]\n");
- StringAppendF(&asms, "lsl $2, $2, %d\n", LW_LOCK_OWNER_SHIFT);
- StringAppendF(&asms, "bfi $2, $0, #0, #%d\n", LW_LOCK_OWNER_SHIFT - 1);
- StringAppendF(&asms, "bfc $0, #%d, #%d\n", LW_HASH_STATE_SHIFT, LW_LOCK_OWNER_SHIFT - 1);
- StringAppendF(&asms, "cmp $0, #0\n");
- StringAppendF(&asms, "it eq\n");
- StringAppendF(&asms, "strexeq $0, $2, [$3]\n");
-
- InlineAsm* func = InlineAsm::get(func_ty, asms, "=&l,l,~l,~l", true);
-
- Value* retry_slow_path = irb_.CreateCall(func, object);
- retry_slow_path = irb_.CreateICmpNE(retry_slow_path, irb_.getJInt(0));
-
- Function* parent_func = irb_.GetInsertBlock()->getParent();
- BasicBlock* basic_block_lock = BasicBlock::Create(context_, "lock", parent_func);
- BasicBlock* basic_block_cont = BasicBlock::Create(context_, "lock_cont", parent_func);
- irb_.CreateCondBr(retry_slow_path, basic_block_lock, basic_block_cont, kUnlikely);
-
- irb_.SetInsertPoint(basic_block_lock);
- Function* slow_func = GetRuntimeSupportFunction(runtime_support::LockObject);
- irb_.CreateCall2(slow_func, object, EmitGetCurrentThread());
- irb_.CreateBr(basic_block_cont);
-
- irb_.SetInsertPoint(basic_block_cont);
- { // Memory barrier
- FunctionType* asm_ty = FunctionType::get(/*Result=*/Type::getVoidTy(context_),
- /*isVarArg=*/false);
- InlineAsm* func = InlineAsm::get(asm_ty, "dmb sy", "", true);
- irb_.CreateCall(func);
- }
-}
-
-
-} // namespace llvm
-} // namespace art
diff --git a/compiler/llvm/runtime_support_builder_thumb2.h b/compiler/llvm/runtime_support_builder_thumb2.h
deleted file mode 100644
index c47a274..0000000
--- a/compiler/llvm/runtime_support_builder_thumb2.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef ART_COMPILER_LLVM_RUNTIME_SUPPORT_BUILDER_THUMB2_H_
-#define ART_COMPILER_LLVM_RUNTIME_SUPPORT_BUILDER_THUMB2_H_
-
-#include "runtime_support_builder_arm.h"
-
-namespace art {
-namespace llvm {
-
-class RuntimeSupportBuilderThumb2 : public RuntimeSupportBuilderARM {
- public:
- RuntimeSupportBuilderThumb2(::llvm::LLVMContext& context, ::llvm::Module& module, IRBuilder& irb)
- : RuntimeSupportBuilderARM(context, module, irb) {}
-
- /* Monitor */
- virtual void EmitLockObject(::llvm::Value* object);
-};
-
-} // namespace llvm
-} // namespace art
-
-#endif // ART_COMPILER_LLVM_RUNTIME_SUPPORT_BUILDER_THUMB2_H_