diff options
author | 2019-06-06 09:04:07 +0100 | |
---|---|---|
committer | 2019-06-06 12:53:49 +0000 | |
commit | 2a905b2b50fb77723f6627e2d6a5df218d21081f (patch) | |
tree | 4fa5d59381ed16073bc5f57cf9c9522e6f81acdd | |
parent | 29bb803f2d41ea48e9f255dadd22b4c291e0a036 (diff) |
Move memory related regions of the JIT cache in its own class.
- Moves the JIT lock into the global locks list
- Jit cache has two regions: one inherited from zygote, and its own.
Bug: 119800099
Test: boot, test.py
Change-Id: I833ba2b5a3d3c10fa01286cdf52603fa174a2cc7
-rw-r--r-- | runtime/Android.bp | 1 | ||||
-rw-r--r-- | runtime/base/locks.cc | 6 | ||||
-rw-r--r-- | runtime/base/locks.h | 5 | ||||
-rw-r--r-- | runtime/gc/accounting/bitmap.cc | 2 | ||||
-rw-r--r-- | runtime/jit/jit_code_cache.cc | 568 | ||||
-rw-r--r-- | runtime/jit/jit_code_cache.h | 249 | ||||
-rw-r--r-- | runtime/jit/jit_memory_region.cc | 327 | ||||
-rw-r--r-- | runtime/jit/jit_memory_region.h | 175 | ||||
-rw-r--r-- | runtime/jit/jit_scoped_code_cache_write.h | 67 | ||||
-rw-r--r-- | test/667-jit-jni-stub/jit_jni_stub_test.cc | 2 |
10 files changed, 775 insertions, 627 deletions
diff --git a/runtime/Android.bp b/runtime/Android.bp index 4a4162437f..701d8c3f39 100644 --- a/runtime/Android.bp +++ b/runtime/Android.bp @@ -115,6 +115,7 @@ libart_cc_defaults { "jit/debugger_interface.cc", "jit/jit.cc", "jit/jit_code_cache.cc", + "jit/jit_memory_region.cc", "jit/profiling_info.cc", "jit/profile_saver.cc", "jni/check_jni.cc", diff --git a/runtime/base/locks.cc b/runtime/base/locks.cc index 4349be09b0..1cec44c27e 100644 --- a/runtime/base/locks.cc +++ b/runtime/base/locks.cc @@ -63,6 +63,7 @@ Mutex* Locks::reference_queue_weak_references_lock_ = nullptr; Mutex* Locks::runtime_shutdown_lock_ = nullptr; Mutex* Locks::runtime_thread_pool_lock_ = nullptr; Mutex* Locks::cha_lock_ = nullptr; +Mutex* Locks::jit_lock_ = nullptr; Mutex* Locks::subtype_check_lock_ = nullptr; Mutex* Locks::thread_list_lock_ = nullptr; ConditionVariable* Locks::thread_exit_cond_ = nullptr; @@ -147,6 +148,7 @@ void Locks::Init() { DCHECK(mutator_lock_ != nullptr); DCHECK(profiler_lock_ != nullptr); DCHECK(cha_lock_ != nullptr); + DCHECK(jit_lock_ != nullptr); DCHECK(subtype_check_lock_ != nullptr); DCHECK(thread_list_lock_ != nullptr); DCHECK(thread_suspend_count_lock_ != nullptr); @@ -303,6 +305,10 @@ void Locks::Init() { DCHECK(custom_tls_lock_ == nullptr); custom_tls_lock_ = new Mutex("Thread::custom_tls_ lock", current_lock_level); + UPDATE_CURRENT_LOCK_LEVEL(kJitCodeCacheLock); + DCHECK(jit_lock_ == nullptr); + jit_lock_ = new Mutex("Jit code cache", current_lock_level); + UPDATE_CURRENT_LOCK_LEVEL(kCHALock); DCHECK(cha_lock_ == nullptr); cha_lock_ = new Mutex("CHA lock", current_lock_level); diff --git a/runtime/base/locks.h b/runtime/base/locks.h index b15fd32f4d..77e304debd 100644 --- a/runtime/base/locks.h +++ b/runtime/base/locks.h @@ -328,8 +328,11 @@ class Locks { // GetThreadLocalStorage. static Mutex* custom_tls_lock_ ACQUIRED_AFTER(jni_function_table_lock_); + // Guard access to any JIT data structure. + static Mutex* jit_lock_ ACQUIRED_AFTER(custom_tls_lock_); + // Guards Class Hierarchy Analysis (CHA). - static Mutex* cha_lock_ ACQUIRED_AFTER(custom_tls_lock_); + static Mutex* cha_lock_ ACQUIRED_AFTER(jit_lock_); // When declaring any Mutex add BOTTOM_MUTEX_ACQUIRED_AFTER to use annotalysis to check the code // doesn't try to acquire a higher level Mutex. NB Due to the way the annotalysis works this diff --git a/runtime/gc/accounting/bitmap.cc b/runtime/gc/accounting/bitmap.cc index 8a15af2fbc..e7ef03ea3e 100644 --- a/runtime/gc/accounting/bitmap.cc +++ b/runtime/gc/accounting/bitmap.cc @@ -21,7 +21,7 @@ #include "base/bit_utils.h" #include "base/mem_map.h" #include "card_table.h" -#include "jit/jit_code_cache.h" +#include "jit/jit_memory_region.h" namespace art { namespace gc { diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc index 2505111058..f430d58c50 100644 --- a/runtime/jit/jit_code_cache.cc +++ b/runtime/jit/jit_code_cache.cc @@ -19,7 +19,6 @@ #include <sstream> #include <android-base/logging.h> -#include <android-base/unique_fd.h> #include "arch/context.h" #include "art_method-inl.h" @@ -47,6 +46,7 @@ #include "intern_table.h" #include "jit/jit.h" #include "jit/profiling_info.h" +#include "jit/jit_scoped_code_cache_write.h" #include "linear_alloc.h" #include "oat_file-inl.h" #include "oat_quick_method_header.h" @@ -57,37 +57,12 @@ #include "thread-current-inl.h" #include "thread_list.h" -using android::base::unique_fd; - namespace art { namespace jit { static constexpr size_t kCodeSizeLogThreshold = 50 * KB; static constexpr size_t kStackMapSizeLogThreshold = 50 * KB; -// Data cache will be half of the capacity -// Code cache will be the other half of the capacity. -// TODO: Make this variable? -static constexpr size_t kCodeAndDataCapacityDivider = 2; - -static constexpr int kProtR = PROT_READ; -static constexpr int kProtRW = PROT_READ | PROT_WRITE; -static constexpr int kProtRWX = PROT_READ | PROT_WRITE | PROT_EXEC; -static constexpr int kProtRX = PROT_READ | PROT_EXEC; - -namespace { - -// Translate an address belonging to one memory map into an address in a second. This is useful -// when there are two virtual memory ranges for the same physical memory range. -template <typename T> -T* TranslateAddress(T* src_ptr, const MemMap& src, const MemMap& dst) { - CHECK(src.HasAddress(src_ptr)); - uint8_t* const raw_src_ptr = reinterpret_cast<uint8_t*>(src_ptr); - return reinterpret_cast<T*>(raw_src_ptr - src.Begin() + dst.Begin()); -} - -} // namespace - class JitCodeCache::JniStubKey { public: explicit JniStubKey(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_) @@ -189,171 +164,6 @@ class JitCodeCache::JniStubData { std::vector<ArtMethod*> methods_; }; -bool JitCodeCache::InitializeMappings(bool rwx_memory_allowed, - bool is_zygote, - std::string* error_msg) { - ScopedTrace trace(__PRETTY_FUNCTION__); - - const size_t capacity = max_capacity_; - const size_t data_capacity = capacity / kCodeAndDataCapacityDivider; - const size_t exec_capacity = capacity - data_capacity; - - // File descriptor enabling dual-view mapping of code section. - unique_fd mem_fd; - - // Zygote shouldn't create a shared mapping for JIT, so we cannot use dual view - // for it. - if (!is_zygote) { - // Bionic supports memfd_create, but the call may fail on older kernels. - mem_fd = unique_fd(art::memfd_create("/jit-cache", /* flags= */ 0)); - if (mem_fd.get() < 0) { - std::ostringstream oss; - oss << "Failed to initialize dual view JIT. memfd_create() error: " << strerror(errno); - if (!rwx_memory_allowed) { - // Without using RWX page permissions, the JIT can not fallback to single mapping as it - // requires tranitioning the code pages to RWX for updates. - *error_msg = oss.str(); - return false; - } - VLOG(jit) << oss.str(); - } - } - - if (mem_fd.get() >= 0 && ftruncate(mem_fd, capacity) != 0) { - std::ostringstream oss; - oss << "Failed to initialize memory file: " << strerror(errno); - *error_msg = oss.str(); - return false; - } - - std::string data_cache_name = is_zygote ? "zygote-data-code-cache" : "data-code-cache"; - std::string exec_cache_name = is_zygote ? "zygote-jit-code-cache" : "jit-code-cache"; - - std::string error_str; - // Map name specific for android_os_Debug.cpp accounting. - // Map in low 4gb to simplify accessing root tables for x86_64. - // We could do PC-relative addressing to avoid this problem, but that - // would require reserving code and data area before submitting, which - // means more windows for the code memory to be RWX. - int base_flags; - MemMap data_pages; - if (mem_fd.get() >= 0) { - // Dual view of JIT code cache case. Create an initial mapping of data pages large enough - // for data and non-writable view of JIT code pages. We use the memory file descriptor to - // enable dual mapping - we'll create a second mapping using the descriptor below. The - // mappings will look like: - // - // VA PA - // - // +---------------+ - // | non exec code |\ - // +---------------+ \ - // : :\ \ - // +---------------+.\.+---------------+ - // | exec code | \| code | - // +---------------+...+---------------+ - // | data | | data | - // +---------------+...+---------------+ - // - // In this configuration code updates are written to the non-executable view of the code - // cache, and the executable view of the code cache has fixed RX memory protections. - // - // This memory needs to be mapped shared as the code portions will have two mappings. - base_flags = MAP_SHARED; - data_pages = MemMap::MapFile( - data_capacity + exec_capacity, - kProtRW, - base_flags, - mem_fd, - /* start= */ 0, - /* low_4gb= */ true, - data_cache_name.c_str(), - &error_str); - } else { - // Single view of JIT code cache case. Create an initial mapping of data pages large enough - // for data and JIT code pages. The mappings will look like: - // - // VA PA - // - // +---------------+...+---------------+ - // | exec code | | code | - // +---------------+...+---------------+ - // | data | | data | - // +---------------+...+---------------+ - // - // In this configuration code updates are written to the executable view of the code cache, - // and the executable view of the code cache transitions RX to RWX for the update and then - // back to RX after the update. - base_flags = MAP_PRIVATE | MAP_ANON; - data_pages = MemMap::MapAnonymous( - data_cache_name.c_str(), - data_capacity + exec_capacity, - kProtRW, - /* low_4gb= */ true, - &error_str); - } - - if (!data_pages.IsValid()) { - std::ostringstream oss; - oss << "Failed to create read write cache: " << error_str << " size=" << capacity; - *error_msg = oss.str(); - return false; - } - - MemMap exec_pages; - MemMap non_exec_pages; - if (exec_capacity > 0) { - uint8_t* const divider = data_pages.Begin() + data_capacity; - // Set initial permission for executable view to catch any SELinux permission problems early - // (for processes that cannot map WX pages). Otherwise, this region does not need to be - // executable as there is no code in the cache yet. - exec_pages = data_pages.RemapAtEnd(divider, - exec_cache_name.c_str(), - kProtRX, - base_flags | MAP_FIXED, - mem_fd.get(), - (mem_fd.get() >= 0) ? data_capacity : 0, - &error_str); - if (!exec_pages.IsValid()) { - std::ostringstream oss; - oss << "Failed to create read execute code cache: " << error_str << " size=" << capacity; - *error_msg = oss.str(); - return false; - } - - if (mem_fd.get() >= 0) { - // For dual view, create the secondary view of code memory used for updating code. This view - // is never executable. - std::string name = exec_cache_name + "-rw"; - non_exec_pages = MemMap::MapFile(exec_capacity, - kProtR, - base_flags, - mem_fd, - /* start= */ data_capacity, - /* low_4GB= */ false, - name.c_str(), - &error_str); - if (!non_exec_pages.IsValid()) { - static const char* kFailedNxView = "Failed to map non-executable view of JIT code cache"; - if (rwx_memory_allowed) { - // Log and continue as single view JIT (requires RWX memory). - VLOG(jit) << kFailedNxView; - } else { - *error_msg = kFailedNxView; - return false; - } - } - } - } else { - // Profiling only. No memory for code required. - } - - data_pages_ = std::move(data_pages); - exec_pages_ = std::move(exec_pages); - non_exec_pages_ = std::move(non_exec_pages); - return true; -} - JitCodeCache* JitCodeCache::Create(bool used_only_for_profile_data, bool rwx_memory_allowed, bool is_zygote, @@ -385,19 +195,20 @@ JitCodeCache* JitCodeCache::Create(bool used_only_for_profile_data, std::unique_ptr<JitCodeCache> jit_code_cache(new JitCodeCache()); - MutexLock mu(Thread::Current(), jit_code_cache->lock_); - jit_code_cache->InitializeState(initial_capacity, max_capacity); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); + jit_code_cache->private_region_.InitializeState(initial_capacity, max_capacity); // Zygote should never collect code to share the memory with the children. if (is_zygote) { jit_code_cache->garbage_collect_code_ = false; } - if (!jit_code_cache->InitializeMappings(rwx_memory_allowed, is_zygote, error_msg)) { + if (!jit_code_cache->private_region_.InitializeMappings( + rwx_memory_allowed, is_zygote, error_msg)) { return nullptr; } - jit_code_cache->InitializeSpaces(); + jit_code_cache->private_region_.InitializeSpaces(); VLOG(jit) << "Created jit code cache: initial capacity=" << PrettySize(initial_capacity) @@ -408,82 +219,24 @@ JitCodeCache* JitCodeCache::Create(bool used_only_for_profile_data, } JitCodeCache::JitCodeCache() - : lock_("Jit code cache", kJitCodeCacheLock), - lock_cond_("Jit code cache condition variable", lock_), + : is_weak_access_enabled_(true), + inline_cache_cond_("Jit inline cache condition variable", *Locks::jit_lock_), + lock_cond_("Jit code cache condition variable", *Locks::jit_lock_), collection_in_progress_(false), last_collection_increased_code_cache_(false), garbage_collect_code_(true), - used_memory_for_data_(0), - used_memory_for_code_(0), number_of_compilations_(0), number_of_osr_compilations_(0), number_of_collections_(0), histogram_stack_map_memory_use_("Memory used for stack maps", 16), histogram_code_memory_use_("Memory used for compiled code", 16), - histogram_profiling_info_memory_use_("Memory used for profiling info", 16), - is_weak_access_enabled_(true), - inline_cache_cond_("Jit inline cache condition variable", lock_), - zygote_data_pages_(), - zygote_exec_pages_(), - zygote_data_mspace_(nullptr), - zygote_exec_mspace_(nullptr) { -} - -void JitCodeCache::InitializeState(size_t initial_capacity, size_t max_capacity) { - CHECK_GE(max_capacity, initial_capacity); - CHECK(max_capacity <= 1 * GB) << "The max supported size for JIT code cache is 1GB"; - // Align both capacities to page size, as that's the unit mspaces use. - initial_capacity = RoundDown(initial_capacity, 2 * kPageSize); - max_capacity = RoundDown(max_capacity, 2 * kPageSize); - - used_memory_for_data_ = 0; - used_memory_for_code_ = 0; - number_of_compilations_ = 0; - number_of_osr_compilations_ = 0; - number_of_collections_ = 0; - - data_pages_ = MemMap(); - exec_pages_ = MemMap(); - non_exec_pages_ = MemMap(); - initial_capacity_ = initial_capacity; - max_capacity_ = max_capacity; - current_capacity_ = initial_capacity, - data_end_ = initial_capacity / kCodeAndDataCapacityDivider; - exec_end_ = initial_capacity - data_end_; -} - -void JitCodeCache::InitializeSpaces() { - // Initialize the data heap - data_mspace_ = create_mspace_with_base(data_pages_.Begin(), data_end_, false /*locked*/); - CHECK(data_mspace_ != nullptr) << "create_mspace_with_base (data) failed"; - - // Initialize the code heap - MemMap* code_heap = nullptr; - if (non_exec_pages_.IsValid()) { - code_heap = &non_exec_pages_; - } else if (exec_pages_.IsValid()) { - code_heap = &exec_pages_; - } - if (code_heap != nullptr) { - // Make all pages reserved for the code heap writable. The mspace allocator, that manages the - // heap, will take and initialize pages in create_mspace_with_base(). - CheckedCall(mprotect, "create code heap", code_heap->Begin(), code_heap->Size(), kProtRW); - exec_mspace_ = create_mspace_with_base(code_heap->Begin(), exec_end_, false /*locked*/); - CHECK(exec_mspace_ != nullptr) << "create_mspace_with_base (exec) failed"; - SetFootprintLimit(initial_capacity_); - // Protect pages containing heap metadata. Updates to the code heap toggle write permission to - // perform the update and there are no other times write access is required. - CheckedCall(mprotect, "protect code heap", code_heap->Begin(), code_heap->Size(), kProtR); - } else { - exec_mspace_ = nullptr; - SetFootprintLimit(initial_capacity_); - } + histogram_profiling_info_memory_use_("Memory used for profiling info", 16) { } JitCodeCache::~JitCodeCache() {} bool JitCodeCache::ContainsPc(const void* ptr) const { - return exec_pages_.HasAddress(ptr) || zygote_exec_pages_.HasAddress(ptr); + return private_region_.IsInExecSpace(ptr) || shared_region_.IsInExecSpace(ptr); } bool JitCodeCache::WillExecuteJitCode(ArtMethod* method) { @@ -498,7 +251,7 @@ bool JitCodeCache::WillExecuteJitCode(ArtMethod* method) { } bool JitCodeCache::ContainsMethod(ArtMethod* method) { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); if (UNLIKELY(method->IsNative())) { auto it = jni_stubs_map_.find(JniStubKey(method)); if (it != jni_stubs_map_.end() && @@ -518,7 +271,7 @@ bool JitCodeCache::ContainsMethod(ArtMethod* method) { const void* JitCodeCache::GetJniStubCode(ArtMethod* method) { DCHECK(method->IsNative()); - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); auto it = jni_stubs_map_.find(JniStubKey(method)); if (it != jni_stubs_map_.end()) { JniStubData& data = it->second; @@ -568,34 +321,6 @@ const void* JitCodeCache::GetZygoteSavedEntryPoint(ArtMethod* method) { return nullptr; } -class ScopedCodeCacheWrite : ScopedTrace { - public: - explicit ScopedCodeCacheWrite(const JitCodeCache* const code_cache) - : ScopedTrace("ScopedCodeCacheWrite"), - code_cache_(code_cache) { - ScopedTrace trace("mprotect all"); - const MemMap* const updatable_pages = code_cache_->GetUpdatableCodeMapping(); - if (updatable_pages != nullptr) { - int prot = code_cache_->HasDualCodeMapping() ? kProtRW : kProtRWX; - CheckedCall(mprotect, "Cache +W", updatable_pages->Begin(), updatable_pages->Size(), prot); - } - } - - ~ScopedCodeCacheWrite() { - ScopedTrace trace("mprotect code"); - const MemMap* const updatable_pages = code_cache_->GetUpdatableCodeMapping(); - if (updatable_pages != nullptr) { - int prot = code_cache_->HasDualCodeMapping() ? kProtR : kProtRX; - CheckedCall(mprotect, "Cache -W", updatable_pages->Begin(), updatable_pages->Size(), prot); - } - } - - private: - const JitCodeCache* const code_cache_; - - DISALLOW_COPY_AND_ASSIGN(ScopedCodeCacheWrite); -}; - uint8_t* JitCodeCache::CommitCode(Thread* self, ArtMethod* method, uint8_t* stack_map, @@ -741,7 +466,7 @@ static inline void ProcessWeakClass(GcRoot<mirror::Class>* root_ptr, } void JitCodeCache::SweepRootTables(IsMarkedVisitor* visitor) { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); for (const auto& entry : method_code_map_) { uint32_t number_of_roots = 0; uint8_t* roots_data = GetRootTable(entry.first, &number_of_roots); @@ -790,15 +515,10 @@ void JitCodeCache::FreeCodeAndData(const void* code_ptr) { // It does nothing if we are not using native debugger. RemoveNativeDebugInfoForJit(Thread::Current(), code_ptr); if (OatQuickMethodHeader::FromCodePointer(code_ptr)->IsOptimized()) { - FreeData(GetRootTable(code_ptr)); + private_region_.FreeData(GetRootTable(code_ptr)); } // else this is a JNI stub without any data. - uint8_t* code_allocation = reinterpret_cast<uint8_t*>(allocation); - if (HasDualCodeMapping()) { - code_allocation = TranslateAddress(code_allocation, exec_pages_, non_exec_pages_); - } - - FreeCode(code_allocation); + private_region_.FreeCode(reinterpret_cast<uint8_t*>(allocation)); } void JitCodeCache::FreeAllMethodHeaders( @@ -807,14 +527,14 @@ void JitCodeCache::FreeAllMethodHeaders( // first since once we do FreeCode() below, the memory can be reused // so it's possible for the same method_header to start representing // different compile code. - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); { MutexLock mu2(Thread::Current(), *Locks::cha_lock_); Runtime::Current()->GetClassLinker()->GetClassHierarchyAnalysis() ->RemoveDependentsWithMethodHeaders(method_headers); } - ScopedCodeCacheWrite scc(this); + ScopedCodeCacheWrite scc(private_region_); for (const OatQuickMethodHeader* method_header : method_headers) { FreeCodeAndData(method_header->GetCode()); } @@ -828,12 +548,12 @@ void JitCodeCache::RemoveMethodsIn(Thread* self, const LinearAlloc& alloc) { // the CHA dependency map just once with an unordered_set. std::unordered_set<OatQuickMethodHeader*> method_headers; { - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); // We do not check if a code cache GC is in progress, as this method comes // with the classlinker_classes_lock_ held, and suspending ourselves could // lead to a deadlock. { - ScopedCodeCacheWrite scc(this); + ScopedCodeCacheWrite scc(private_region_); for (auto it = jni_stubs_map_.begin(); it != jni_stubs_map_.end();) { it->second.RemoveMethodsIn(alloc); if (it->second.GetMethods().empty()) { @@ -866,7 +586,7 @@ void JitCodeCache::RemoveMethodsIn(Thread* self, const LinearAlloc& alloc) { ProfilingInfo* info = *it; if (alloc.ContainsUnsafe(info->GetMethod())) { info->GetMethod()->SetProfilingInfo(nullptr); - FreeData(reinterpret_cast<uint8_t*>(info)); + private_region_.FreeData(reinterpret_cast<uint8_t*>(info)); it = profiling_infos_.erase(it); } else { ++it; @@ -887,7 +607,7 @@ void JitCodeCache::WaitUntilInlineCacheAccessible(Thread* self) { return; } ScopedThreadSuspension sts(self, kWaitingWeakGcRootRead); - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); while (!IsWeakAccessEnabled(self)) { inline_cache_cond_.Wait(self); } @@ -895,7 +615,7 @@ void JitCodeCache::WaitUntilInlineCacheAccessible(Thread* self) { void JitCodeCache::BroadcastForInlineCacheAccess() { Thread* self = Thread::Current(); - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); inline_cache_cond_.Broadcast(self); } @@ -940,23 +660,13 @@ static void ClearMethodCounter(ArtMethod* method, bool was_warm) void JitCodeCache::WaitForPotentialCollectionToCompleteRunnable(Thread* self) { while (collection_in_progress_) { - lock_.Unlock(self); + Locks::jit_lock_->Unlock(self); { ScopedThreadSuspension sts(self, kSuspended); - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); WaitForPotentialCollectionToComplete(self); } - lock_.Lock(self); - } -} - -const MemMap* JitCodeCache::GetUpdatableCodeMapping() const { - if (HasDualCodeMapping()) { - return &non_exec_pages_; - } else if (HasCodeMapping()) { - return &exec_pages_; - } else { - return nullptr; + Locks::jit_lock_->Lock(self); } } @@ -983,12 +693,12 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, OatQuickMethodHeader* method_header = nullptr; uint8_t* code_ptr = nullptr; - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); // We need to make sure that there will be no jit-gcs going on and wait for any ongoing one to // finish. WaitForPotentialCollectionToCompleteRunnable(self); { - ScopedCodeCacheWrite scc(this); + ScopedCodeCacheWrite scc(private_region_); size_t alignment = GetInstructionSetAlignment(kRuntimeISA); // Ensure the header ends up at expected instruction alignment. @@ -997,7 +707,7 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, // AllocateCode allocates memory in non-executable region for alignment header and code. The // header size may include alignment padding. - uint8_t* nox_memory = AllocateCode(total_size); + uint8_t* nox_memory = private_region_.AllocateCode(total_size); if (nox_memory == nullptr) { return nullptr; } @@ -1008,9 +718,7 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, method_header = OatQuickMethodHeader::FromCodePointer(code_ptr); // From here code_ptr points to executable code. - if (HasDualCodeMapping()) { - code_ptr = TranslateAddress(code_ptr, non_exec_pages_, exec_pages_); - } + code_ptr = private_region_.GetExecutableAddress(code_ptr); new (method_header) OatQuickMethodHeader( (stack_map != nullptr) ? code_ptr - stack_map : 0u, @@ -1022,9 +730,7 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, } // Update method_header pointer to executable code region. - if (HasDualCodeMapping()) { - method_header = TranslateAddress(method_header, non_exec_pages_, exec_pages_); - } + method_header = private_region_.GetExecutableAddress(method_header); // Both instruction and data caches need flushing to the point of unification where both share // a common view of memory. Flushing the data cache ensures the dirty cachelines from the @@ -1041,7 +747,7 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, // For reference, this behavior is caused by this commit: // https://android.googlesource.com/kernel/msm/+/3fbe6bc28a6b9939d0650f2f17eb5216c719950c // - if (HasDualCodeMapping()) { + if (private_region_.HasDualCodeMapping()) { // Flush the data cache lines associated with the non-executable copy of the code just added. FlushDataCache(nox_memory, nox_memory + total_size); } @@ -1162,7 +868,7 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, } size_t JitCodeCache::CodeCacheSize() { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); return CodeCacheSizeLocked(); } @@ -1170,7 +876,7 @@ bool JitCodeCache::RemoveMethod(ArtMethod* method, bool release_memory) { // This function is used only for testing and only with non-native methods. CHECK(!method->IsNative()); - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); bool osr = osr_code_map_.find(method) != osr_code_map_.end(); bool in_cache = RemoveMethodLocked(method, release_memory); @@ -1200,7 +906,7 @@ bool JitCodeCache::RemoveMethodLocked(ArtMethod* method, bool release_memory) { } bool in_cache = false; - ScopedCodeCacheWrite ccw(this); + ScopedCodeCacheWrite ccw(private_region_); if (UNLIKELY(method->IsNative())) { auto it = jni_stubs_map_.find(JniStubKey(method)); if (it != jni_stubs_map_.end() && it->second.RemoveMethod(method)) { @@ -1240,7 +946,7 @@ bool JitCodeCache::RemoveMethodLocked(ArtMethod* method, bool release_memory) { // any cached information it has on the method. All threads must be suspended before calling this // method. The compiled code for the method (if there is any) must not be in any threads call stack. void JitCodeCache::NotifyMethodRedefined(ArtMethod* method) { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); RemoveMethodLocked(method, /* release_memory= */ true); } @@ -1251,7 +957,7 @@ void JitCodeCache::NotifyMethodRedefined(ArtMethod* method) { // shouldn't be used since it is no longer logically in the jit code cache. // TODO We should add DCHECKS that validate that the JIT is paused when this method is entered. void JitCodeCache::MoveObsoleteMethod(ArtMethod* old_method, ArtMethod* new_method) { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); if (old_method->IsNative()) { // Update methods in jni_stubs_map_. for (auto& entry : jni_stubs_map_) { @@ -1288,7 +994,7 @@ void JitCodeCache::MoveObsoleteMethod(ArtMethod* old_method, ArtMethod* new_meth } void JitCodeCache::ClearEntryPointsInZygoteExecSpace() { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); // Iterate over profiling infos to know which methods may have been JITted. Note that // to be JITted, a method must have a profiling info. for (ProfilingInfo* info : profiling_infos_) { @@ -1306,24 +1012,24 @@ void JitCodeCache::ClearEntryPointsInZygoteExecSpace() { } size_t JitCodeCache::CodeCacheSizeLocked() { - return used_memory_for_code_; + return private_region_.GetUsedMemoryForCode(); } size_t JitCodeCache::DataCacheSize() { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); return DataCacheSizeLocked(); } size_t JitCodeCache::DataCacheSizeLocked() { - return used_memory_for_data_; + return private_region_.GetUsedMemoryForData(); } void JitCodeCache::ClearData(Thread* self, uint8_t* stack_map_data, uint8_t* roots_data) { DCHECK_EQ(FromStackMapToRoots(stack_map_data), roots_data); - MutexLock mu(self, lock_); - FreeData(reinterpret_cast<uint8_t*>(roots_data)); + MutexLock mu(self, *Locks::jit_lock_); + private_region_.FreeData(reinterpret_cast<uint8_t*>(roots_data)); } size_t JitCodeCache::ReserveData(Thread* self, @@ -1338,21 +1044,21 @@ size_t JitCodeCache::ReserveData(Thread* self, { ScopedThreadSuspension sts(self, kSuspended); - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); WaitForPotentialCollectionToComplete(self); - result = AllocateData(size); + result = private_region_.AllocateData(size); } if (result == nullptr) { // Retry. GarbageCollectCache(self); ScopedThreadSuspension sts(self, kSuspended); - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); WaitForPotentialCollectionToComplete(self); - result = AllocateData(size); + result = private_region_.AllocateData(size); } - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); histogram_stack_map_memory_use_.AddValue(size); if (size > kStackMapSizeLogThreshold) { LOG(INFO) << "JIT allocated " @@ -1429,40 +1135,6 @@ void JitCodeCache::NotifyCollectionDone(Thread* self) { lock_cond_.Broadcast(self); } -void JitCodeCache::SetFootprintLimit(size_t new_footprint) { - size_t data_space_footprint = new_footprint / kCodeAndDataCapacityDivider; - DCHECK(IsAlignedParam(data_space_footprint, kPageSize)); - DCHECK_EQ(data_space_footprint * kCodeAndDataCapacityDivider, new_footprint); - mspace_set_footprint_limit(data_mspace_, data_space_footprint); - if (HasCodeMapping()) { - ScopedCodeCacheWrite scc(this); - mspace_set_footprint_limit(exec_mspace_, new_footprint - data_space_footprint); - } -} - -bool JitCodeCache::IncreaseCodeCacheCapacity() { - if (current_capacity_ == max_capacity_) { - return false; - } - - // Double the capacity if we're below 1MB, or increase it by 1MB if - // we're above. - if (current_capacity_ < 1 * MB) { - current_capacity_ *= 2; - } else { - current_capacity_ += 1 * MB; - } - if (current_capacity_ > max_capacity_) { - current_capacity_ = max_capacity_; - } - - VLOG(jit) << "Increasing code cache capacity to " << PrettySize(current_capacity_); - - SetFootprintLimit(current_capacity_); - - return true; -} - void JitCodeCache::MarkCompiledCodeOnThreadStacks(Thread* self) { Barrier barrier(0); size_t threads_running_checkpoint = 0; @@ -1477,10 +1149,10 @@ void JitCodeCache::MarkCompiledCodeOnThreadStacks(Thread* self) { } bool JitCodeCache::ShouldDoFullCollection() { - if (current_capacity_ == max_capacity_) { + if (private_region_.GetCurrentCapacity() == private_region_.GetMaxCapacity()) { // Always do a full collection when the code cache is full. return true; - } else if (current_capacity_ < kReservedCapacity) { + } else if (private_region_.GetCurrentCapacity() < kReservedCapacity) { // Always do partial collection when the code cache size is below the reserved // capacity. return false; @@ -1498,9 +1170,9 @@ void JitCodeCache::GarbageCollectCache(Thread* self) { // Wait for an existing collection, or let everyone know we are starting one. { ScopedThreadSuspension sts(self, kSuspended); - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); if (!garbage_collect_code_) { - IncreaseCodeCacheCapacity(); + private_region_.IncreaseCodeCacheCapacity(); return; } else if (WaitForPotentialCollectionToComplete(self)) { return; @@ -1508,8 +1180,9 @@ void JitCodeCache::GarbageCollectCache(Thread* self) { number_of_collections_++; live_bitmap_.reset(CodeCacheBitmap::Create( "code-cache-bitmap", - reinterpret_cast<uintptr_t>(exec_pages_.Begin()), - reinterpret_cast<uintptr_t>(exec_pages_.Begin() + current_capacity_ / 2))); + reinterpret_cast<uintptr_t>(private_region_.GetExecPages()->Begin()), + reinterpret_cast<uintptr_t>( + private_region_.GetExecPages()->Begin() + private_region_.GetCurrentCapacity() / 2))); collection_in_progress_ = true; } } @@ -1520,7 +1193,7 @@ void JitCodeCache::GarbageCollectCache(Thread* self) { bool do_full_collection = false; { - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); do_full_collection = ShouldDoFullCollection(); } @@ -1537,7 +1210,7 @@ void JitCodeCache::GarbageCollectCache(Thread* self) { << ", data=" << PrettySize(DataCacheSize()); { - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); // Increase the code cache only when we do partial collections. // TODO: base this strategy on how full the code cache is? @@ -1545,7 +1218,7 @@ void JitCodeCache::GarbageCollectCache(Thread* self) { last_collection_increased_code_cache_ = false; } else { last_collection_increased_code_cache_ = true; - IncreaseCodeCacheCapacity(); + private_region_.IncreaseCodeCacheCapacity(); } bool next_collection_will_be_full = ShouldDoFullCollection(); @@ -1597,8 +1270,8 @@ void JitCodeCache::RemoveUnmarkedCode(Thread* self) { ScopedTrace trace(__FUNCTION__); std::unordered_set<OatQuickMethodHeader*> method_headers; { - MutexLock mu(self, lock_); - ScopedCodeCacheWrite scc(this); + MutexLock mu(self, *Locks::jit_lock_); + ScopedCodeCacheWrite scc(private_region_); // Iterate over all compiled code and remove entries that are not marked. for (auto it = jni_stubs_map_.begin(); it != jni_stubs_map_.end();) { JniStubData* data = &it->second; @@ -1627,13 +1300,13 @@ void JitCodeCache::RemoveUnmarkedCode(Thread* self) { } bool JitCodeCache::GetGarbageCollectCode() { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); return garbage_collect_code_; } void JitCodeCache::SetGarbageCollectCode(bool value) { Thread* self = Thread::Current(); - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); if (garbage_collect_code_ != value) { if (garbage_collect_code_) { // When dynamically disabling the garbage collection, we neee @@ -1652,7 +1325,7 @@ void JitCodeCache::SetGarbageCollectCode(bool value) { void JitCodeCache::DoCollection(Thread* self, bool collect_profiling_info) { ScopedTrace trace(__FUNCTION__); { - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); if (collect_profiling_info) { // Clear the profiling info of methods that do not have compiled code as entrypoint. // Also remove the saved entry point from the ProfilingInfo objects. @@ -1722,7 +1395,7 @@ void JitCodeCache::DoCollection(Thread* self, bool collect_profiling_info) { RemoveUnmarkedCode(self); if (collect_profiling_info) { - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); // Free all profiling infos of methods not compiled nor being compiled. auto profiling_kept_end = std::remove_if(profiling_infos_.begin(), profiling_infos_.end(), [this] (ProfilingInfo* info) NO_THREAD_SAFETY_ANALYSIS { @@ -1738,7 +1411,7 @@ void JitCodeCache::DoCollection(Thread* self, bool collect_profiling_info) { info->GetMethod()->SetProfilingInfo(info); } else if (info->GetMethod()->GetProfilingInfo(kRuntimePointerSize) != info) { // No need for this ProfilingInfo object anymore. - FreeData(reinterpret_cast<uint8_t*>(info)); + private_region_.FreeData(reinterpret_cast<uint8_t*>(info)); return true; } return false; @@ -1762,7 +1435,7 @@ OatQuickMethodHeader* JitCodeCache::LookupMethodHeader(uintptr_t pc, ArtMethod* CHECK(method != nullptr); } - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); OatQuickMethodHeader* method_header = nullptr; ArtMethod* found_method = nullptr; // Only for DCHECK(), not for JNI stubs. if (method != nullptr && UNLIKELY(method->IsNative())) { @@ -1811,7 +1484,7 @@ OatQuickMethodHeader* JitCodeCache::LookupMethodHeader(uintptr_t pc, ArtMethod* } OatQuickMethodHeader* JitCodeCache::LookupOsrMethodHeader(ArtMethod* method) { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); auto it = osr_code_map_.find(method); if (it == osr_code_map_.end()) { return nullptr; @@ -1829,19 +1502,19 @@ ProfilingInfo* JitCodeCache::AddProfilingInfo(Thread* self, if (!retry_allocation) { // If we are allocating for the interpreter, just try to lock, to avoid // lock contention with the JIT. - if (lock_.ExclusiveTryLock(self)) { + if (Locks::jit_lock_->ExclusiveTryLock(self)) { info = AddProfilingInfoInternal(self, method, entries); - lock_.ExclusiveUnlock(self); + Locks::jit_lock_->ExclusiveUnlock(self); } } else { { - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); info = AddProfilingInfoInternal(self, method, entries); } if (info == nullptr) { GarbageCollectCache(self); - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); info = AddProfilingInfoInternal(self, method, entries); } } @@ -1861,7 +1534,7 @@ ProfilingInfo* JitCodeCache::AddProfilingInfoInternal(Thread* self ATTRIBUTE_UNU return info; } - uint8_t* data = AllocateData(profile_info_size); + uint8_t* data = private_region_.AllocateData(profile_info_size); if (data == nullptr) { return nullptr; } @@ -1877,28 +1550,15 @@ ProfilingInfo* JitCodeCache::AddProfilingInfoInternal(Thread* self ATTRIBUTE_UNU return info; } -// NO_THREAD_SAFETY_ANALYSIS as this is called from mspace code, at which point the lock -// is already held. -void* JitCodeCache::MoreCore(const void* mspace, intptr_t increment) NO_THREAD_SAFETY_ANALYSIS { - if (mspace == exec_mspace_) { - DCHECK(exec_mspace_ != nullptr); - const MemMap* const code_pages = GetUpdatableCodeMapping(); - void* result = code_pages->Begin() + exec_end_; - exec_end_ += increment; - return result; - } else { - DCHECK_EQ(data_mspace_, mspace); - void* result = data_pages_.Begin() + data_end_; - data_end_ += increment; - return result; - } +void* JitCodeCache::MoreCore(const void* mspace, intptr_t increment) { + return private_region_.MoreCore(mspace, increment); } void JitCodeCache::GetProfiledMethods(const std::set<std::string>& dex_base_locations, std::vector<ProfileMethodInfo>& methods) { Thread* self = Thread::Current(); WaitUntilInlineCacheAccessible(self); - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); ScopedTrace trace(__FUNCTION__); uint16_t jit_compile_threshold = Runtime::Current()->GetJITOptions()->GetCompileThreshold(); for (const ProfilingInfo* info : profiling_infos_) { @@ -1979,7 +1639,7 @@ void JitCodeCache::GetProfiledMethods(const std::set<std::string>& dex_base_loca } bool JitCodeCache::IsOsrCompiled(ArtMethod* method) { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); return osr_code_map_.find(method) != osr_code_map_.end(); } @@ -2002,7 +1662,7 @@ bool JitCodeCache::NotifyCompilationOf(ArtMethod* method, Thread* self, bool osr } } - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); if (osr && (osr_code_map_.find(method) != osr_code_map_.end())) { return false; } @@ -2063,7 +1723,7 @@ bool JitCodeCache::NotifyCompilationOf(ArtMethod* method, Thread* self, bool osr } ProfilingInfo* JitCodeCache::NotifyCompilerUse(ArtMethod* method, Thread* self) { - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); if (info != nullptr) { if (!info->IncrementInlineUse()) { @@ -2075,7 +1735,7 @@ ProfilingInfo* JitCodeCache::NotifyCompilerUse(ArtMethod* method, Thread* self) } void JitCodeCache::DoneCompilerUse(ArtMethod* method, Thread* self) { - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); ProfilingInfo* info = method->GetProfilingInfo(kRuntimePointerSize); DCHECK(info != nullptr); info->DecrementInlineUse(); @@ -2083,7 +1743,7 @@ void JitCodeCache::DoneCompilerUse(ArtMethod* method, Thread* self) { void JitCodeCache::DoneCompiling(ArtMethod* method, Thread* self, bool osr) { DCHECK_EQ(Thread::Current(), self); - MutexLock mu(self, lock_); + MutexLock mu(self, *Locks::jit_lock_); if (UNLIKELY(method->IsNative())) { auto it = jni_stubs_map_.find(JniStubKey(method)); DCHECK(it != jni_stubs_map_.end()); @@ -2124,7 +1784,7 @@ void JitCodeCache::InvalidateCompiledCodeFor(ArtMethod* method, method, GetQuickToInterpreterBridge()); ClearMethodCounter(method, /*was_warm=*/ profiling_info != nullptr); } else { - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); auto it = osr_code_map_.find(method); if (it != osr_code_map_.end() && OatQuickMethodHeader::FromCodePointer(it->second) == header) { // Remove the OSR method, to avoid using it again. @@ -2133,49 +1793,14 @@ void JitCodeCache::InvalidateCompiledCodeFor(ArtMethod* method, } } -uint8_t* JitCodeCache::AllocateCode(size_t allocation_size) { - // Each allocation should be on its own set of cache lines. The allocation must be large enough - // for header, code, and any padding. - uint8_t* result = reinterpret_cast<uint8_t*>( - mspace_memalign(exec_mspace_, kJitCodeAlignment, allocation_size)); - size_t alignment = GetInstructionSetAlignment(kRuntimeISA); - size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment); - // Ensure the header ends up at expected instruction alignment. - DCHECK_ALIGNED_PARAM(reinterpret_cast<uintptr_t>(result + header_size), alignment); - used_memory_for_code_ += mspace_usable_size(result); - return result; -} - -void JitCodeCache::FreeCode(uint8_t* code) { - if (IsInZygoteExecSpace(code)) { - // No need to free, this is shared memory. - return; - } - used_memory_for_code_ -= mspace_usable_size(code); - mspace_free(exec_mspace_, code); -} - -uint8_t* JitCodeCache::AllocateData(size_t data_size) { - void* result = mspace_malloc(data_mspace_, data_size); - used_memory_for_data_ += mspace_usable_size(result); - return reinterpret_cast<uint8_t*>(result); -} - -void JitCodeCache::FreeData(uint8_t* data) { - if (IsInZygoteDataSpace(data)) { - // No need to free, this is shared memory. - return; - } - used_memory_for_data_ -= mspace_usable_size(data); - mspace_free(data_mspace_, data); -} - void JitCodeCache::Dump(std::ostream& os) { - MutexLock mu(Thread::Current(), lock_); - os << "Current JIT code cache size: " << PrettySize(used_memory_for_code_) << "\n" - << "Current JIT data cache size: " << PrettySize(used_memory_for_data_) << "\n" + MutexLock mu(Thread::Current(), *Locks::jit_lock_); + os << "Current JIT code cache size: " << PrettySize(private_region_.GetUsedMemoryForCode()) + << "\n" + << "Current JIT data cache size: " << PrettySize(private_region_.GetUsedMemoryForData()) + << "\n" << "Current JIT mini-debug-info size: " << PrettySize(GetJitMiniDebugInfoMemUsage()) << "\n" - << "Current JIT capacity: " << PrettySize(current_capacity_) << "\n" + << "Current JIT capacity: " << PrettySize(private_region_.GetCurrentCapacity()) << "\n" << "Current number of JIT JNI stub entries: " << jni_stubs_map_.size() << "\n" << "Current number of JIT code cache entries: " << method_code_map_.size() << "\n" << "Total number of JIT compilations: " << number_of_compilations_ << "\n" @@ -2192,25 +1817,28 @@ void JitCodeCache::PostForkChildAction(bool is_system_server, bool is_zygote) { // Don't transition if this is for a child zygote. return; } - MutexLock mu(Thread::Current(), lock_); + MutexLock mu(Thread::Current(), *Locks::jit_lock_); + + shared_region_ = std::move(private_region_); - zygote_data_pages_ = std::move(data_pages_); - zygote_exec_pages_ = std::move(exec_pages_); - zygote_data_mspace_ = data_mspace_; - zygote_exec_mspace_ = exec_mspace_; + // Reset all statistics to be specific to this process. + number_of_compilations_ = 0; + number_of_osr_compilations_ = 0; + number_of_collections_ = 0; size_t initial_capacity = Runtime::Current()->GetJITOptions()->GetCodeCacheInitialCapacity(); size_t max_capacity = Runtime::Current()->GetJITOptions()->GetCodeCacheMaxCapacity(); - InitializeState(initial_capacity, max_capacity); + private_region_.InitializeState(initial_capacity, max_capacity); std::string error_msg; - if (!InitializeMappings(/* rwx_memory_allowed= */ !is_system_server, is_zygote, &error_msg)) { + if (!private_region_.InitializeMappings( + /* rwx_memory_allowed= */ !is_system_server, is_zygote, &error_msg)) { LOG(WARNING) << "Could not reset JIT state after zygote fork: " << error_msg; return; } - InitializeSpaces(); + private_region_.InitializeSpaces(); } } // namespace jit diff --git a/runtime/jit/jit_code_cache.h b/runtime/jit/jit_code_cache.h index 3078e2be8f..a56f6f05f8 100644 --- a/runtime/jit/jit_code_cache.h +++ b/runtime/jit/jit_code_cache.h @@ -31,6 +31,7 @@ #include "base/mem_map.h" #include "base/mutex.h" #include "base/safe_map.h" +#include "jit_memory_region.h" namespace art { @@ -72,14 +73,6 @@ template<class T> class ObjectArray; namespace jit { class MarkCodeClosure; -class ScopedCodeCacheWrite; - -// Alignment in bytes that will suit all architectures for JIT code cache allocations. The -// allocated block is used for method header followed by generated code. Allocations should be -// aligned to avoid sharing cache lines between different allocations. The alignment should be -// determined from the hardware, but this isn't readily exposed in userland plus some hardware -// misreports. -static constexpr int kJitCodeAlignment = 64; using CodeCacheBitmap = gc::accounting::MemoryRangeBitmap<kJitCodeAlignment>; @@ -103,11 +96,11 @@ class JitCodeCache { bool NotifyCompilationOf(ArtMethod* method, Thread* self, bool osr, bool prejit) REQUIRES_SHARED(Locks::mutator_lock_) - REQUIRES(!lock_); + REQUIRES(!Locks::jit_lock_); void NotifyMethodRedefined(ArtMethod* method) REQUIRES(Locks::mutator_lock_) - REQUIRES(!lock_); + REQUIRES(!Locks::jit_lock_); // Notify to the code cache that the compiler wants to use the // profiling info of `method` to drive optimizations, @@ -115,15 +108,15 @@ class JitCodeCache { // collected. ProfilingInfo* NotifyCompilerUse(ArtMethod* method, Thread* self) REQUIRES_SHARED(Locks::mutator_lock_) - REQUIRES(!lock_); + REQUIRES(!Locks::jit_lock_); void DoneCompiling(ArtMethod* method, Thread* self, bool osr) REQUIRES_SHARED(Locks::mutator_lock_) - REQUIRES(!lock_); + REQUIRES(!Locks::jit_lock_); void DoneCompilerUse(ArtMethod* method, Thread* self) REQUIRES_SHARED(Locks::mutator_lock_) - REQUIRES(!lock_); + REQUIRES(!Locks::jit_lock_); // Allocate and write code and its metadata to the code cache. // `cha_single_implementation_list` needs to be registered via CHA (if it's @@ -143,20 +136,20 @@ class JitCodeCache { bool has_should_deoptimize_flag, const ArenaSet<ArtMethod*>& cha_single_implementation_list) REQUIRES_SHARED(Locks::mutator_lock_) - REQUIRES(!lock_); + REQUIRES(!Locks::jit_lock_); // Return true if the code cache contains this pc. bool ContainsPc(const void* pc) const; // Returns true if either the method's entrypoint is JIT compiled code or it is the // instrumentation entrypoint and we can jump to jit code for this method. For testing use only. - bool WillExecuteJitCode(ArtMethod* method) REQUIRES(!lock_); + bool WillExecuteJitCode(ArtMethod* method) REQUIRES(!Locks::jit_lock_); // Return true if the code cache contains this method. - bool ContainsMethod(ArtMethod* method) REQUIRES(!lock_); + bool ContainsMethod(ArtMethod* method) REQUIRES(!Locks::jit_lock_); // Return the code pointer for a JNI-compiled stub if the method is in the cache, null otherwise. - const void* GetJniStubCode(ArtMethod* method) REQUIRES(!lock_); + const void* GetJniStubCode(ArtMethod* method) REQUIRES(!Locks::jit_lock_); // Allocate a region of data that contain `size` bytes, and potentially space // for storing `number_of_roots` roots. Returns null if there is no more room. @@ -168,43 +161,43 @@ class JitCodeCache { uint8_t** stack_map_data, uint8_t** roots_data) REQUIRES_SHARED(Locks::mutator_lock_) - REQUIRES(!lock_); + REQUIRES(!Locks::jit_lock_); // Clear data from the data portion of the code cache. void ClearData(Thread* self, uint8_t* stack_map_data, uint8_t* roots_data) REQUIRES_SHARED(Locks::mutator_lock_) - REQUIRES(!lock_); + REQUIRES(!Locks::jit_lock_); // Perform a collection on the code cache. void GarbageCollectCache(Thread* self) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); // Given the 'pc', try to find the JIT compiled code associated with it. // Return null if 'pc' is not in the code cache. 'method' is passed for // sanity check. OatQuickMethodHeader* LookupMethodHeader(uintptr_t pc, ArtMethod* method) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); OatQuickMethodHeader* LookupOsrMethodHeader(ArtMethod* method) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); // Removes method from the cache for testing purposes. The caller // must ensure that all threads are suspended and the method should // not be in any thread's stack. bool RemoveMethod(ArtMethod* method, bool release_memory) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES(Locks::mutator_lock_); // Remove all methods in our cache that were allocated by 'alloc'. void RemoveMethodsIn(Thread* self, const LinearAlloc& alloc) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); void CopyInlineCacheInto(const InlineCache& ic, Handle<mirror::ObjectArray<mirror::Class>> array) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); // Create a 'ProfileInfo' for 'method'. If 'retry_allocation' is true, @@ -213,11 +206,11 @@ class JitCodeCache { ArtMethod* method, const std::vector<uint32_t>& entries, bool retry_allocation) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); bool OwnsSpace(const void* mspace) const NO_THREAD_SAFETY_ANALYSIS { - return mspace == data_mspace_ || mspace == exec_mspace_; + return private_region_.OwnsSpace(mspace); } void* MoreCore(const void* mspace, intptr_t increment); @@ -225,36 +218,36 @@ class JitCodeCache { // Adds to `methods` all profiled methods which are part of any of the given dex locations. void GetProfiledMethods(const std::set<std::string>& dex_base_locations, std::vector<ProfileMethodInfo>& methods) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); void InvalidateCompiledCodeFor(ArtMethod* method, const OatQuickMethodHeader* code) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); - void Dump(std::ostream& os) REQUIRES(!lock_); + void Dump(std::ostream& os) REQUIRES(!Locks::jit_lock_); - bool IsOsrCompiled(ArtMethod* method) REQUIRES(!lock_); + bool IsOsrCompiled(ArtMethod* method) REQUIRES(!Locks::jit_lock_); void SweepRootTables(IsMarkedVisitor* visitor) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); // The GC needs to disallow the reading of inline caches when it processes them, // to avoid having a class being used while it is being deleted. - void AllowInlineCacheAccess() REQUIRES(!lock_); - void DisallowInlineCacheAccess() REQUIRES(!lock_); - void BroadcastForInlineCacheAccess() REQUIRES(!lock_); + void AllowInlineCacheAccess() REQUIRES(!Locks::jit_lock_); + void DisallowInlineCacheAccess() REQUIRES(!Locks::jit_lock_); + void BroadcastForInlineCacheAccess() REQUIRES(!Locks::jit_lock_); // Notify the code cache that the method at the pointer 'old_method' is being moved to the pointer // 'new_method' since it is being made obsolete. void MoveObsoleteMethod(ArtMethod* old_method, ArtMethod* new_method) - REQUIRES(!lock_) REQUIRES(Locks::mutator_lock_); + REQUIRES(!Locks::jit_lock_) REQUIRES(Locks::mutator_lock_); // Dynamically change whether we want to garbage collect code. - void SetGarbageCollectCode(bool value) REQUIRES(!lock_); + void SetGarbageCollectCode(bool value) REQUIRES(!Locks::jit_lock_); - bool GetGarbageCollectCode() REQUIRES(!lock_); + bool GetGarbageCollectCode() REQUIRES(!Locks::jit_lock_); // Unsafe variant for debug checks. bool GetGarbageCollectCodeUnsafe() const NO_THREAD_SAFETY_ANALYSIS { @@ -264,13 +257,13 @@ class JitCodeCache { // If Jit-gc has been disabled (and instrumentation has been enabled) this will return the // jit-compiled entrypoint for this method. Otherwise it will return null. const void* FindCompiledCodeForInstrumentation(ArtMethod* method) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); // Fetch the entrypoint that zygote may have saved for a method. The zygote saves an entrypoint // only for the case when the method's declaring class is not initialized. const void* GetZygoteSavedEntryPoint(ArtMethod* method) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); void PostForkChildAction(bool is_system_server, bool is_zygote); @@ -278,18 +271,11 @@ class JitCodeCache { // Clear the entrypoints of JIT compiled methods that belong in the zygote space. // This is used for removing non-debuggable JIT code at the point we realize the runtime // is debuggable. - void ClearEntryPointsInZygoteExecSpace() REQUIRES(!lock_) REQUIRES(Locks::mutator_lock_); + void ClearEntryPointsInZygoteExecSpace() REQUIRES(!Locks::jit_lock_) REQUIRES(Locks::mutator_lock_); private: JitCodeCache(); - void InitializeState(size_t initial_capacity, size_t max_capacity) REQUIRES(lock_); - - bool InitializeMappings(bool rwx_memory_allowed, bool is_zygote, std::string* error_msg) - REQUIRES(lock_); - - void InitializeSpaces() REQUIRES(lock_); - // Internal version of 'CommitCode' that will not retry if the // allocation fails. Return null if the allocation fails. uint8_t* CommitCodeInternal(Thread* self, @@ -303,208 +289,163 @@ class JitCodeCache { const std::vector<Handle<mirror::Object>>& roots, bool has_should_deoptimize_flag, const ArenaSet<ArtMethod*>& cha_single_implementation_list) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); // Adds the given roots to the roots_data. Only a member for annotalysis. void FillRootTable(uint8_t* roots_data, const std::vector<Handle<mirror::Object>>& roots) - REQUIRES(lock_) + REQUIRES(Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); ProfilingInfo* AddProfilingInfoInternal(Thread* self, ArtMethod* method, const std::vector<uint32_t>& entries) - REQUIRES(lock_) + REQUIRES(Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); // If a collection is in progress, wait for it to finish. Must be called with the mutator lock. // The non-mutator lock version should be used if possible. This method will release then // re-acquire the mutator lock. void WaitForPotentialCollectionToCompleteRunnable(Thread* self) - REQUIRES(lock_, !Roles::uninterruptible_) REQUIRES_SHARED(Locks::mutator_lock_); + REQUIRES(Locks::jit_lock_, !Roles::uninterruptible_) REQUIRES_SHARED(Locks::mutator_lock_); // If a collection is in progress, wait for it to finish. Return // whether the thread actually waited. bool WaitForPotentialCollectionToComplete(Thread* self) - REQUIRES(lock_) REQUIRES(!Locks::mutator_lock_); + REQUIRES(Locks::jit_lock_) REQUIRES(!Locks::mutator_lock_); // Remove CHA dependents and underlying allocations for entries in `method_headers`. void FreeAllMethodHeaders(const std::unordered_set<OatQuickMethodHeader*>& method_headers) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES(!Locks::cha_lock_); // Removes method from the cache. The caller must ensure that all threads // are suspended and the method should not be in any thread's stack. bool RemoveMethodLocked(ArtMethod* method, bool release_memory) - REQUIRES(lock_) + REQUIRES(Locks::jit_lock_) REQUIRES(Locks::mutator_lock_); // Free code and data allocations for `code_ptr`. - void FreeCodeAndData(const void* code_ptr) REQUIRES(lock_); + void FreeCodeAndData(const void* code_ptr) REQUIRES(Locks::jit_lock_); // Number of bytes allocated in the code cache. - size_t CodeCacheSize() REQUIRES(!lock_); + size_t CodeCacheSize() REQUIRES(!Locks::jit_lock_); // Number of bytes allocated in the data cache. - size_t DataCacheSize() REQUIRES(!lock_); + size_t DataCacheSize() REQUIRES(!Locks::jit_lock_); // Number of bytes allocated in the code cache. - size_t CodeCacheSizeLocked() REQUIRES(lock_); + size_t CodeCacheSizeLocked() REQUIRES(Locks::jit_lock_); // Number of bytes allocated in the data cache. - size_t DataCacheSizeLocked() REQUIRES(lock_); + size_t DataCacheSizeLocked() REQUIRES(Locks::jit_lock_); // Notify all waiting threads that a collection is done. - void NotifyCollectionDone(Thread* self) REQUIRES(lock_); - - // Try to increase the current capacity of the code cache. Return whether we - // succeeded at doing so. - bool IncreaseCodeCacheCapacity() REQUIRES(lock_); - - // Set the footprint limit of the code cache. - void SetFootprintLimit(size_t new_footprint) REQUIRES(lock_); + void NotifyCollectionDone(Thread* self) REQUIRES(Locks::jit_lock_); // Return whether we should do a full collection given the current state of the cache. bool ShouldDoFullCollection() - REQUIRES(lock_) + REQUIRES(Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); void DoCollection(Thread* self, bool collect_profiling_info) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); void RemoveUnmarkedCode(Thread* self) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); void MarkCompiledCodeOnThreadStacks(Thread* self) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); CodeCacheBitmap* GetLiveBitmap() const { return live_bitmap_.get(); } - uint8_t* AllocateCode(size_t code_size) REQUIRES(lock_); - void FreeCode(uint8_t* code) REQUIRES(lock_); - uint8_t* AllocateData(size_t data_size) REQUIRES(lock_); - void FreeData(uint8_t* data) REQUIRES(lock_); - - bool HasDualCodeMapping() const { - return non_exec_pages_.IsValid(); - } - - bool HasCodeMapping() const { - return exec_pages_.IsValid(); - } - - const MemMap* GetUpdatableCodeMapping() const; - bool IsInZygoteDataSpace(const void* ptr) const { - return zygote_data_pages_.HasAddress(ptr); + return shared_region_.IsInDataSpace(ptr); } bool IsInZygoteExecSpace(const void* ptr) const { - return zygote_exec_pages_.HasAddress(ptr); + return shared_region_.IsInExecSpace(ptr); } bool IsWeakAccessEnabled(Thread* self) const; void WaitUntilInlineCacheAccessible(Thread* self) - REQUIRES(!lock_) + REQUIRES(!Locks::jit_lock_) REQUIRES_SHARED(Locks::mutator_lock_); class JniStubKey; class JniStubData; - // Lock for guarding allocations, collections, and the method_code_map_. - Mutex lock_ BOTTOM_MUTEX_ACQUIRED_AFTER; - // Condition to wait on during collection. - ConditionVariable lock_cond_ GUARDED_BY(lock_); - // Whether there is a code cache collection in progress. - bool collection_in_progress_ GUARDED_BY(lock_); - // Mem map which holds data (stack maps and profiling info). - MemMap data_pages_; - // Mem map which holds code and has executable permission. - MemMap exec_pages_; - // Mem map which holds code with non executable permission. Only valid for dual view JIT when - // this is the non-executable view of code used to write updates. - MemMap non_exec_pages_; - // The opaque mspace for allocating data. - void* data_mspace_ GUARDED_BY(lock_); - // The opaque mspace for allocating code. - void* exec_mspace_ GUARDED_BY(lock_); - // Bitmap for collecting code and data. - std::unique_ptr<CodeCacheBitmap> live_bitmap_; + // Whether the GC allows accessing weaks in inline caches. Note that this + // is not used by the concurrent collector, which uses + // Thread::SetWeakRefAccessEnabled instead. + Atomic<bool> is_weak_access_enabled_; + + // Condition to wait on for accessing inline caches. + ConditionVariable inline_cache_cond_ GUARDED_BY(Locks::jit_lock_); + + // -------------- JIT memory regions ------------------------------------- // + + // Shared region, inherited from the zygote. + JitMemoryRegion shared_region_; + + // Process's own region. + JitMemoryRegion private_region_; + + // -------------- Global JIT maps --------------------------------------- // + // Holds compiled code associated with the shorty for a JNI stub. - SafeMap<JniStubKey, JniStubData> jni_stubs_map_ GUARDED_BY(lock_); + SafeMap<JniStubKey, JniStubData> jni_stubs_map_ GUARDED_BY(Locks::jit_lock_); + // Holds compiled code associated to the ArtMethod. - SafeMap<const void*, ArtMethod*> method_code_map_ GUARDED_BY(lock_); + SafeMap<const void*, ArtMethod*> method_code_map_ GUARDED_BY(Locks::jit_lock_); + // Holds osr compiled code associated to the ArtMethod. - SafeMap<ArtMethod*, const void*> osr_code_map_ GUARDED_BY(lock_); - // ProfilingInfo objects we have allocated. - std::vector<ProfilingInfo*> profiling_infos_ GUARDED_BY(lock_); + SafeMap<ArtMethod*, const void*> osr_code_map_ GUARDED_BY(Locks::jit_lock_); - // The initial capacity in bytes this code cache starts with. - size_t initial_capacity_ GUARDED_BY(lock_); + // ProfilingInfo objects we have allocated. + std::vector<ProfilingInfo*> profiling_infos_ GUARDED_BY(Locks::jit_lock_); - // The maximum capacity in bytes this code cache can go to. - size_t max_capacity_ GUARDED_BY(lock_); + // -------------- JIT GC related data structures ----------------------- // - // The current capacity in bytes of the code cache. - size_t current_capacity_ GUARDED_BY(lock_); + // Condition to wait on during collection. + ConditionVariable lock_cond_ GUARDED_BY(Locks::jit_lock_); - // The current footprint in bytes of the data portion of the code cache. - size_t data_end_ GUARDED_BY(lock_); + // Whether there is a code cache collection in progress. + bool collection_in_progress_ GUARDED_BY(Locks::jit_lock_); - // The current footprint in bytes of the code portion of the code cache. - size_t exec_end_ GUARDED_BY(lock_); + // Bitmap for collecting code and data. + std::unique_ptr<CodeCacheBitmap> live_bitmap_; // Whether the last collection round increased the code cache. - bool last_collection_increased_code_cache_ GUARDED_BY(lock_); + bool last_collection_increased_code_cache_ GUARDED_BY(Locks::jit_lock_); // Whether we can do garbage collection. Not 'const' as tests may override this. - bool garbage_collect_code_ GUARDED_BY(lock_); - - // The size in bytes of used memory for the data portion of the code cache. - size_t used_memory_for_data_ GUARDED_BY(lock_); + bool garbage_collect_code_ GUARDED_BY(Locks::jit_lock_); - // The size in bytes of used memory for the code portion of the code cache. - size_t used_memory_for_code_ GUARDED_BY(lock_); + // ---------------- JIT statistics -------------------------------------- // // Number of compilations done throughout the lifetime of the JIT. - size_t number_of_compilations_ GUARDED_BY(lock_); + size_t number_of_compilations_ GUARDED_BY(Locks::jit_lock_); // Number of compilations for on-stack-replacement done throughout the lifetime of the JIT. - size_t number_of_osr_compilations_ GUARDED_BY(lock_); + size_t number_of_osr_compilations_ GUARDED_BY(Locks::jit_lock_); // Number of code cache collections done throughout the lifetime of the JIT. - size_t number_of_collections_ GUARDED_BY(lock_); + size_t number_of_collections_ GUARDED_BY(Locks::jit_lock_); // Histograms for keeping track of stack map size statistics. - Histogram<uint64_t> histogram_stack_map_memory_use_ GUARDED_BY(lock_); + Histogram<uint64_t> histogram_stack_map_memory_use_ GUARDED_BY(Locks::jit_lock_); // Histograms for keeping track of code size statistics. - Histogram<uint64_t> histogram_code_memory_use_ GUARDED_BY(lock_); + Histogram<uint64_t> histogram_code_memory_use_ GUARDED_BY(Locks::jit_lock_); // Histograms for keeping track of profiling info statistics. - Histogram<uint64_t> histogram_profiling_info_memory_use_ GUARDED_BY(lock_); - - // Whether the GC allows accessing weaks in inline caches. Note that this - // is not used by the concurrent collector, which uses - // Thread::SetWeakRefAccessEnabled instead. - Atomic<bool> is_weak_access_enabled_; - - // Condition to wait on for accessing inline caches. - ConditionVariable inline_cache_cond_ GUARDED_BY(lock_); - - // Mem map which holds zygote data (stack maps and profiling info). - MemMap zygote_data_pages_; - // Mem map which holds zygote code and has executable permission. - MemMap zygote_exec_pages_; - // The opaque mspace for allocating zygote data. - void* zygote_data_mspace_ GUARDED_BY(lock_); - // The opaque mspace for allocating zygote code. - void* zygote_exec_mspace_ GUARDED_BY(lock_); + Histogram<uint64_t> histogram_profiling_info_memory_use_ GUARDED_BY(Locks::jit_lock_); friend class art::JitJniStubTestHelper; friend class ScopedCodeCacheWrite; diff --git a/runtime/jit/jit_memory_region.cc b/runtime/jit/jit_memory_region.cc new file mode 100644 index 0000000000..ac02bd8274 --- /dev/null +++ b/runtime/jit/jit_memory_region.cc @@ -0,0 +1,327 @@ +/* + * Copyright 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "jit_memory_region.h" + +#include <android-base/unique_fd.h> +#include "base/bit_utils.h" // For RoundDown, RoundUp +#include "base/globals.h" +#include "base/logging.h" // For VLOG. +#include "base/memfd.h" +#include "base/systrace.h" +#include "gc/allocator/dlmalloc.h" +#include "jit/jit_scoped_code_cache_write.h" +#include "oat_quick_method_header.h" + +using android::base::unique_fd; + +namespace art { +namespace jit { + +// Data cache will be half of the capacity +// Code cache will be the other half of the capacity. +// TODO: Make this variable? +static constexpr size_t kCodeAndDataCapacityDivider = 2; + +bool JitMemoryRegion::InitializeMappings(bool rwx_memory_allowed, + bool is_zygote, + std::string* error_msg) { + ScopedTrace trace(__PRETTY_FUNCTION__); + + const size_t capacity = max_capacity_; + const size_t data_capacity = capacity / kCodeAndDataCapacityDivider; + const size_t exec_capacity = capacity - data_capacity; + + // File descriptor enabling dual-view mapping of code section. + unique_fd mem_fd; + + // Zygote shouldn't create a shared mapping for JIT, so we cannot use dual view + // for it. + if (!is_zygote) { + // Bionic supports memfd_create, but the call may fail on older kernels. + mem_fd = unique_fd(art::memfd_create("/jit-cache", /* flags= */ 0)); + if (mem_fd.get() < 0) { + std::ostringstream oss; + oss << "Failed to initialize dual view JIT. memfd_create() error: " << strerror(errno); + if (!rwx_memory_allowed) { + // Without using RWX page permissions, the JIT can not fallback to single mapping as it + // requires tranitioning the code pages to RWX for updates. + *error_msg = oss.str(); + return false; + } + VLOG(jit) << oss.str(); + } + } + + if (mem_fd.get() >= 0 && ftruncate(mem_fd, capacity) != 0) { + std::ostringstream oss; + oss << "Failed to initialize memory file: " << strerror(errno); + *error_msg = oss.str(); + return false; + } + + std::string data_cache_name = is_zygote ? "zygote-data-code-cache" : "data-code-cache"; + std::string exec_cache_name = is_zygote ? "zygote-jit-code-cache" : "jit-code-cache"; + + std::string error_str; + // Map name specific for android_os_Debug.cpp accounting. + // Map in low 4gb to simplify accessing root tables for x86_64. + // We could do PC-relative addressing to avoid this problem, but that + // would require reserving code and data area before submitting, which + // means more windows for the code memory to be RWX. + int base_flags; + MemMap data_pages; + if (mem_fd.get() >= 0) { + // Dual view of JIT code cache case. Create an initial mapping of data pages large enough + // for data and non-writable view of JIT code pages. We use the memory file descriptor to + // enable dual mapping - we'll create a second mapping using the descriptor below. The + // mappings will look like: + // + // VA PA + // + // +---------------+ + // | non exec code |\ + // +---------------+ \ + // : :\ \ + // +---------------+.\.+---------------+ + // | exec code | \| code | + // +---------------+...+---------------+ + // | data | | data | + // +---------------+...+---------------+ + // + // In this configuration code updates are written to the non-executable view of the code + // cache, and the executable view of the code cache has fixed RX memory protections. + // + // This memory needs to be mapped shared as the code portions will have two mappings. + base_flags = MAP_SHARED; + data_pages = MemMap::MapFile( + data_capacity + exec_capacity, + kProtRW, + base_flags, + mem_fd, + /* start= */ 0, + /* low_4gb= */ true, + data_cache_name.c_str(), + &error_str); + } else { + // Single view of JIT code cache case. Create an initial mapping of data pages large enough + // for data and JIT code pages. The mappings will look like: + // + // VA PA + // + // +---------------+...+---------------+ + // | exec code | | code | + // +---------------+...+---------------+ + // | data | | data | + // +---------------+...+---------------+ + // + // In this configuration code updates are written to the executable view of the code cache, + // and the executable view of the code cache transitions RX to RWX for the update and then + // back to RX after the update. + base_flags = MAP_PRIVATE | MAP_ANON; + data_pages = MemMap::MapAnonymous( + data_cache_name.c_str(), + data_capacity + exec_capacity, + kProtRW, + /* low_4gb= */ true, + &error_str); + } + + if (!data_pages.IsValid()) { + std::ostringstream oss; + oss << "Failed to create read write cache: " << error_str << " size=" << capacity; + *error_msg = oss.str(); + return false; + } + + MemMap exec_pages; + MemMap non_exec_pages; + if (exec_capacity > 0) { + uint8_t* const divider = data_pages.Begin() + data_capacity; + // Set initial permission for executable view to catch any SELinux permission problems early + // (for processes that cannot map WX pages). Otherwise, this region does not need to be + // executable as there is no code in the cache yet. + exec_pages = data_pages.RemapAtEnd(divider, + exec_cache_name.c_str(), + kProtRX, + base_flags | MAP_FIXED, + mem_fd.get(), + (mem_fd.get() >= 0) ? data_capacity : 0, + &error_str); + if (!exec_pages.IsValid()) { + std::ostringstream oss; + oss << "Failed to create read execute code cache: " << error_str << " size=" << capacity; + *error_msg = oss.str(); + return false; + } + + if (mem_fd.get() >= 0) { + // For dual view, create the secondary view of code memory used for updating code. This view + // is never executable. + std::string name = exec_cache_name + "-rw"; + non_exec_pages = MemMap::MapFile(exec_capacity, + kProtR, + base_flags, + mem_fd, + /* start= */ data_capacity, + /* low_4GB= */ false, + name.c_str(), + &error_str); + if (!non_exec_pages.IsValid()) { + static const char* kFailedNxView = "Failed to map non-executable view of JIT code cache"; + if (rwx_memory_allowed) { + // Log and continue as single view JIT (requires RWX memory). + VLOG(jit) << kFailedNxView; + } else { + *error_msg = kFailedNxView; + return false; + } + } + } + } else { + // Profiling only. No memory for code required. + } + + data_pages_ = std::move(data_pages); + exec_pages_ = std::move(exec_pages); + non_exec_pages_ = std::move(non_exec_pages); + return true; +} + +void JitMemoryRegion::InitializeState(size_t initial_capacity, size_t max_capacity) { + CHECK_GE(max_capacity, initial_capacity); + CHECK(max_capacity <= 1 * GB) << "The max supported size for JIT code cache is 1GB"; + // Align both capacities to page size, as that's the unit mspaces use. + initial_capacity_ = RoundDown(initial_capacity, 2 * kPageSize); + max_capacity_ = RoundDown(max_capacity, 2 * kPageSize); + current_capacity_ = initial_capacity, + data_end_ = initial_capacity / kCodeAndDataCapacityDivider; + exec_end_ = initial_capacity - data_end_; +} + +void JitMemoryRegion::InitializeSpaces() { + // Initialize the data heap + data_mspace_ = create_mspace_with_base(data_pages_.Begin(), data_end_, false /*locked*/); + CHECK(data_mspace_ != nullptr) << "create_mspace_with_base (data) failed"; + + // Initialize the code heap + MemMap* code_heap = nullptr; + if (non_exec_pages_.IsValid()) { + code_heap = &non_exec_pages_; + } else if (exec_pages_.IsValid()) { + code_heap = &exec_pages_; + } + if (code_heap != nullptr) { + // Make all pages reserved for the code heap writable. The mspace allocator, that manages the + // heap, will take and initialize pages in create_mspace_with_base(). + CheckedCall(mprotect, "create code heap", code_heap->Begin(), code_heap->Size(), kProtRW); + exec_mspace_ = create_mspace_with_base(code_heap->Begin(), exec_end_, false /*locked*/); + CHECK(exec_mspace_ != nullptr) << "create_mspace_with_base (exec) failed"; + SetFootprintLimit(initial_capacity_); + // Protect pages containing heap metadata. Updates to the code heap toggle write permission to + // perform the update and there are no other times write access is required. + CheckedCall(mprotect, "protect code heap", code_heap->Begin(), code_heap->Size(), kProtR); + } else { + exec_mspace_ = nullptr; + SetFootprintLimit(initial_capacity_); + } +} + +void JitMemoryRegion::SetFootprintLimit(size_t new_footprint) { + size_t data_space_footprint = new_footprint / kCodeAndDataCapacityDivider; + DCHECK(IsAlignedParam(data_space_footprint, kPageSize)); + DCHECK_EQ(data_space_footprint * kCodeAndDataCapacityDivider, new_footprint); + mspace_set_footprint_limit(data_mspace_, data_space_footprint); + if (HasCodeMapping()) { + ScopedCodeCacheWrite scc(*this); + mspace_set_footprint_limit(exec_mspace_, new_footprint - data_space_footprint); + } +} + +bool JitMemoryRegion::IncreaseCodeCacheCapacity() { + if (current_capacity_ == max_capacity_) { + return false; + } + + // Double the capacity if we're below 1MB, or increase it by 1MB if + // we're above. + if (current_capacity_ < 1 * MB) { + current_capacity_ *= 2; + } else { + current_capacity_ += 1 * MB; + } + if (current_capacity_ > max_capacity_) { + current_capacity_ = max_capacity_; + } + + VLOG(jit) << "Increasing code cache capacity to " << PrettySize(current_capacity_); + + SetFootprintLimit(current_capacity_); + + return true; +} + +// NO_THREAD_SAFETY_ANALYSIS as this is called from mspace code, at which point the lock +// is already held. +void* JitMemoryRegion::MoreCore(const void* mspace, intptr_t increment) NO_THREAD_SAFETY_ANALYSIS { + if (mspace == exec_mspace_) { + DCHECK(exec_mspace_ != nullptr); + const MemMap* const code_pages = GetUpdatableCodeMapping(); + void* result = code_pages->Begin() + exec_end_; + exec_end_ += increment; + return result; + } else { + DCHECK_EQ(data_mspace_, mspace); + void* result = data_pages_.Begin() + data_end_; + data_end_ += increment; + return result; + } +} + +uint8_t* JitMemoryRegion::AllocateCode(size_t code_size) { + // Each allocation should be on its own set of cache lines. + // `code_size` covers the OatQuickMethodHeader, the JIT generated machine code, + // and any alignment padding. + size_t alignment = GetInstructionSetAlignment(kRuntimeISA); + size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment); + DCHECK_GT(code_size, header_size); + uint8_t* result = reinterpret_cast<uint8_t*>( + mspace_memalign(exec_mspace_, kJitCodeAlignment, code_size)); + // Ensure the header ends up at expected instruction alignment. + DCHECK_ALIGNED_PARAM(reinterpret_cast<uintptr_t>(result + header_size), alignment); + used_memory_for_code_ += mspace_usable_size(result); + return result; +} + +void JitMemoryRegion::FreeCode(uint8_t* code) { + code = GetNonExecutableAddress(code); + used_memory_for_code_ -= mspace_usable_size(code); + mspace_free(exec_mspace_, code); +} + +uint8_t* JitMemoryRegion::AllocateData(size_t data_size) { + void* result = mspace_malloc(data_mspace_, data_size); + used_memory_for_data_ += mspace_usable_size(result); + return reinterpret_cast<uint8_t*>(result); +} + +void JitMemoryRegion::FreeData(uint8_t* data) { + used_memory_for_data_ -= mspace_usable_size(data); + mspace_free(data_mspace_, data); +} + +} // namespace jit +} // namespace art diff --git a/runtime/jit/jit_memory_region.h b/runtime/jit/jit_memory_region.h new file mode 100644 index 0000000000..5886587659 --- /dev/null +++ b/runtime/jit/jit_memory_region.h @@ -0,0 +1,175 @@ +/* + * Copyright 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_JIT_JIT_MEMORY_REGION_H_ +#define ART_RUNTIME_JIT_JIT_MEMORY_REGION_H_ + +#include <string> + +#include "base/globals.h" +#include "base/locks.h" +#include "base/mem_map.h" + +namespace art { +namespace jit { + +// Alignment in bytes that will suit all architectures for JIT code cache allocations. The +// allocated block is used for method header followed by generated code. Allocations should be +// aligned to avoid sharing cache lines between different allocations. The alignment should be +// determined from the hardware, but this isn't readily exposed in userland plus some hardware +// misreports. +static constexpr int kJitCodeAlignment = 64; + +// Represents a memory region for the JIT, where code and data are stored. This class +// provides allocation and deallocation primitives. +class JitMemoryRegion { + public: + JitMemoryRegion() + : used_memory_for_code_(0), + used_memory_for_data_(0) {} + + void InitializeState(size_t initial_capacity, size_t max_capacity) + REQUIRES(Locks::jit_lock_); + + bool InitializeMappings(bool rwx_memory_allowed, bool is_zygote, std::string* error_msg) + REQUIRES(Locks::jit_lock_); + + void InitializeSpaces() REQUIRES(Locks::jit_lock_); + + // Try to increase the current capacity of the code cache. Return whether we + // succeeded at doing so. + bool IncreaseCodeCacheCapacity() REQUIRES(Locks::jit_lock_); + + // Set the footprint limit of the code cache. + void SetFootprintLimit(size_t new_footprint) REQUIRES(Locks::jit_lock_); + uint8_t* AllocateCode(size_t code_size) REQUIRES(Locks::jit_lock_); + void FreeCode(uint8_t* code) REQUIRES(Locks::jit_lock_); + uint8_t* AllocateData(size_t data_size) REQUIRES(Locks::jit_lock_); + void FreeData(uint8_t* data) REQUIRES(Locks::jit_lock_); + + bool HasDualCodeMapping() const { + return non_exec_pages_.IsValid(); + } + + bool HasCodeMapping() const { + return exec_pages_.IsValid(); + } + + bool IsInDataSpace(const void* ptr) const { + return data_pages_.HasAddress(ptr); + } + + bool IsInExecSpace(const void* ptr) const { + return exec_pages_.HasAddress(ptr); + } + + const MemMap* GetUpdatableCodeMapping() const { + if (HasDualCodeMapping()) { + return &non_exec_pages_; + } else if (HasCodeMapping()) { + return &exec_pages_; + } else { + return nullptr; + } + } + + const MemMap* GetExecPages() const { + return &exec_pages_; + } + + template <typename T> T* GetExecutableAddress(T* src_ptr) { + return TranslateAddress(src_ptr, non_exec_pages_, exec_pages_); + } + + template <typename T> T* GetNonExecutableAddress(T* src_ptr) { + return TranslateAddress(src_ptr, exec_pages_, non_exec_pages_); + } + + void* MoreCore(const void* mspace, intptr_t increment); + + bool OwnsSpace(const void* mspace) const NO_THREAD_SAFETY_ANALYSIS { + return mspace == data_mspace_ || mspace == exec_mspace_; + } + + size_t GetCurrentCapacity() const REQUIRES(Locks::jit_lock_) { + return current_capacity_; + } + + size_t GetMaxCapacity() const REQUIRES(Locks::jit_lock_) { + return max_capacity_; + } + + size_t GetUsedMemoryForCode() const REQUIRES(Locks::jit_lock_) { + return used_memory_for_code_; + } + + size_t GetUsedMemoryForData() const REQUIRES(Locks::jit_lock_) { + return used_memory_for_data_; + } + + private: + template <typename T> + T* TranslateAddress(T* src_ptr, const MemMap& src, const MemMap& dst) { + if (!HasDualCodeMapping()) { + return src_ptr; + } + CHECK(src.HasAddress(src_ptr)); + uint8_t* const raw_src_ptr = reinterpret_cast<uint8_t*>(src_ptr); + return reinterpret_cast<T*>(raw_src_ptr - src.Begin() + dst.Begin()); + } + + // The initial capacity in bytes this code region starts with. + size_t initial_capacity_ GUARDED_BY(Locks::jit_lock_); + + // The maximum capacity in bytes this region can go to. + size_t max_capacity_ GUARDED_BY(Locks::jit_lock_); + + // The current capacity in bytes of the region. + size_t current_capacity_ GUARDED_BY(Locks::jit_lock_); + + // The current footprint in bytes of the data portion of the region. + size_t data_end_ GUARDED_BY(Locks::jit_lock_); + + // The current footprint in bytes of the code portion of the region. + size_t exec_end_ GUARDED_BY(Locks::jit_lock_); + + // The size in bytes of used memory for the code portion of the region. + size_t used_memory_for_code_ GUARDED_BY(Locks::jit_lock_); + + // The size in bytes of used memory for the data portion of the region. + size_t used_memory_for_data_ GUARDED_BY(Locks::jit_lock_); + + // Mem map which holds data (stack maps and profiling info). + MemMap data_pages_; + + // Mem map which holds code and has executable permission. + MemMap exec_pages_; + + // Mem map which holds code with non executable permission. Only valid for dual view JIT when + // this is the non-executable view of code used to write updates. + MemMap non_exec_pages_; + + // The opaque mspace for allocating data. + void* data_mspace_ GUARDED_BY(Locks::jit_lock_); + + // The opaque mspace for allocating code. + void* exec_mspace_ GUARDED_BY(Locks::jit_lock_); +}; + +} // namespace jit +} // namespace art + +#endif // ART_RUNTIME_JIT_JIT_MEMORY_REGION_H_ diff --git a/runtime/jit/jit_scoped_code_cache_write.h b/runtime/jit/jit_scoped_code_cache_write.h new file mode 100644 index 0000000000..ea99bdf0d0 --- /dev/null +++ b/runtime/jit/jit_scoped_code_cache_write.h @@ -0,0 +1,67 @@ +/* + * Copyright 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_JIT_JIT_SCOPED_CODE_CACHE_WRITE_H_ +#define ART_RUNTIME_JIT_JIT_SCOPED_CODE_CACHE_WRITE_H_ + +#include <sys/mman.h> + +#include "base/systrace.h" +#include "base/utils.h" // For CheckedCall + +namespace art { +namespace jit { + +class JitMemoryRegion; + +static constexpr int kProtR = PROT_READ; +static constexpr int kProtRW = PROT_READ | PROT_WRITE; +static constexpr int kProtRWX = PROT_READ | PROT_WRITE | PROT_EXEC; +static constexpr int kProtRX = PROT_READ | PROT_EXEC; + +// Helper for toggling JIT memory R <-> RW. +class ScopedCodeCacheWrite : ScopedTrace { + public: + explicit ScopedCodeCacheWrite(const JitMemoryRegion& region) + : ScopedTrace("ScopedCodeCacheWrite"), + region_(region) { + ScopedTrace trace("mprotect all"); + const MemMap* const updatable_pages = region.GetUpdatableCodeMapping(); + if (updatable_pages != nullptr) { + int prot = region.HasDualCodeMapping() ? kProtRW : kProtRWX; + CheckedCall(mprotect, "Cache +W", updatable_pages->Begin(), updatable_pages->Size(), prot); + } + } + + ~ScopedCodeCacheWrite() { + ScopedTrace trace("mprotect code"); + const MemMap* const updatable_pages = region_.GetUpdatableCodeMapping(); + if (updatable_pages != nullptr) { + int prot = region_.HasDualCodeMapping() ? kProtR : kProtRX; + CheckedCall(mprotect, "Cache -W", updatable_pages->Begin(), updatable_pages->Size(), prot); + } + } + + private: + const JitMemoryRegion& region_; + + DISALLOW_COPY_AND_ASSIGN(ScopedCodeCacheWrite); +}; + +} // namespace jit +} // namespace art + +#endif // ART_RUNTIME_JIT_JIT_SCOPED_CODE_CACHE_WRITE_H_ diff --git a/test/667-jit-jni-stub/jit_jni_stub_test.cc b/test/667-jit-jni-stub/jit_jni_stub_test.cc index 82e06fc018..c21971fa8b 100644 --- a/test/667-jit-jni-stub/jit_jni_stub_test.cc +++ b/test/667-jit-jni-stub/jit_jni_stub_test.cc @@ -31,7 +31,7 @@ class JitJniStubTestHelper { static bool isNextJitGcFull(Thread* self) REQUIRES_SHARED(Locks::mutator_lock_) { CHECK(Runtime::Current()->GetJit() != nullptr); jit::JitCodeCache* cache = Runtime::Current()->GetJit()->GetCodeCache(); - MutexLock mu(self, cache->lock_); + MutexLock mu(self, *Locks::jit_lock_); return cache->ShouldDoFullCollection(); } }; |