Merge "ART: Modernize counting-down loops"
diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc
index e84896b..d440cf3 100644
--- a/compiler/optimizing/code_generator.cc
+++ b/compiler/optimizing/code_generator.cc
@@ -197,7 +197,7 @@
return GetNumberOfJitStringRoots() + GetNumberOfJitClassRoots();
}
- void EmitJitRoots(Handle<mirror::ObjectArray<mirror::Object>> roots)
+ void EmitJitRoots(/*out*/std::vector<Handle<mirror::Object>>* roots)
REQUIRES_SHARED(Locks::mutator_lock_);
private:
@@ -230,29 +230,31 @@
};
void CodeGenerator::CodeGenerationData::EmitJitRoots(
- Handle<mirror::ObjectArray<mirror::Object>> roots) {
- DCHECK_EQ(static_cast<size_t>(roots->GetLength()), GetNumberOfJitRoots());
+ /*out*/std::vector<Handle<mirror::Object>>* roots) {
+ DCHECK(roots->empty());
+ roots->reserve(GetNumberOfJitRoots());
ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
size_t index = 0;
for (auto& entry : jit_string_roots_) {
// Update the `roots` with the string, and replace the address temporarily
// stored to the index in the table.
uint64_t address = entry.second;
- roots->Set(index, reinterpret_cast<StackReference<mirror::String>*>(address)->AsMirrorPtr());
- DCHECK(roots->Get(index) != nullptr);
+ roots->emplace_back(reinterpret_cast<StackReference<mirror::Object>*>(address));
+ DCHECK(roots->back() != nullptr);
+ DCHECK(roots->back()->IsString());
entry.second = index;
// Ensure the string is strongly interned. This is a requirement on how the JIT
// handles strings. b/32995596
- class_linker->GetInternTable()->InternStrong(
- reinterpret_cast<mirror::String*>(roots->Get(index)));
+ class_linker->GetInternTable()->InternStrong(roots->back()->AsString());
++index;
}
for (auto& entry : jit_class_roots_) {
// Update the `roots` with the class, and replace the address temporarily
// stored to the index in the table.
uint64_t address = entry.second;
- roots->Set(index, reinterpret_cast<StackReference<mirror::Class>*>(address)->AsMirrorPtr());
- DCHECK(roots->Get(index) != nullptr);
+ roots->emplace_back(reinterpret_cast<StackReference<mirror::Object>*>(address));
+ DCHECK(roots->back() != nullptr);
+ DCHECK(roots->back()->IsClass());
entry.second = index;
++index;
}
@@ -1645,8 +1647,8 @@
}
void CodeGenerator::EmitJitRoots(uint8_t* code,
- Handle<mirror::ObjectArray<mirror::Object>> roots,
- const uint8_t* roots_data) {
+ const uint8_t* roots_data,
+ /*out*/std::vector<Handle<mirror::Object>>* roots) {
code_generation_data_->EmitJitRoots(roots);
EmitJitRootPatches(code, roots_data);
}
diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h
index 2ca15f7..4e73e0b 100644
--- a/compiler/optimizing/code_generator.h
+++ b/compiler/optimizing/code_generator.h
@@ -356,8 +356,8 @@
// Fills the `literals` array with literals collected during code generation.
// Also emits literal patches.
void EmitJitRoots(uint8_t* code,
- Handle<mirror::ObjectArray<mirror::Object>> roots,
- const uint8_t* roots_data)
+ const uint8_t* roots_data,
+ /*out*/std::vector<Handle<mirror::Object>>* roots)
REQUIRES_SHARED(Locks::mutator_lock_);
bool IsLeafMethod() const {
diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc
index 5f6f71d..9ae025b 100644
--- a/compiler/optimizing/optimizing_compiler.cc
+++ b/compiler/optimizing/optimizing_compiler.cc
@@ -1218,7 +1218,7 @@
const CompilerOptions& compiler_options = GetCompilerDriver()->GetCompilerOptions();
JniCompiledMethod jni_compiled_method = ArtQuickJniCompileMethod(
compiler_options, access_flags, method_idx, *dex_file);
- ScopedNullHandle<mirror::ObjectArray<mirror::Object>> roots;
+ std::vector<Handle<mirror::Object>> roots;
ArenaSet<ArtMethod*, std::less<ArtMethod*>> cha_single_implementation_list(
allocator.Adapter(kArenaAllocCHA));
ArenaStack arena_stack(runtime->GetJitArenaPool());
@@ -1320,19 +1320,6 @@
ScopedArenaVector<uint8_t> stack_map = codegen->BuildStackMaps(code_item);
size_t number_of_roots = codegen->GetNumberOfJitRoots();
- // We allocate an object array to ensure the JIT roots that we will collect in EmitJitRoots
- // will be visible by the GC between EmitLiterals and CommitCode. Once CommitCode is
- // executed, this array is not needed.
- Handle<mirror::ObjectArray<mirror::Object>> roots(
- hs.NewHandle(mirror::ObjectArray<mirror::Object>::Alloc(
- self, GetClassRoot<mirror::ObjectArray<mirror::Object>>(), number_of_roots)));
- if (roots == nullptr) {
- // Out of memory, just clear the exception to avoid any Java exception uncaught problems.
- MaybeRecordStat(compilation_stats_.get(), MethodCompilationStat::kJitOutOfMemoryForCommit);
- DCHECK(self->IsExceptionPending());
- self->ClearException();
- return false;
- }
uint8_t* stack_map_data = nullptr;
uint8_t* roots_data = nullptr;
uint32_t data_size = code_cache->ReserveData(self,
@@ -1346,7 +1333,14 @@
return false;
}
memcpy(stack_map_data, stack_map.data(), stack_map.size());
- codegen->EmitJitRoots(code_allocator.GetData(), roots, roots_data);
+ std::vector<Handle<mirror::Object>> roots;
+ codegen->EmitJitRoots(code_allocator.GetData(), roots_data, &roots);
+ // The root Handle<>s filled by the codegen reference entries in the VariableSizedHandleScope.
+ DCHECK(std::all_of(roots.begin(),
+ roots.end(),
+ [&handles](Handle<mirror::Object> root){
+ return handles.Contains(root.GetReference());
+ }));
const void* code = code_cache->CommitCode(
self,
diff --git a/libartbase/base/mem_map.cc b/libartbase/base/mem_map.cc
index 1bf553d..92551f1 100644
--- a/libartbase/base/mem_map.cc
+++ b/libartbase/base/mem_map.cc
@@ -692,6 +692,24 @@
int tail_prot,
std::string* error_msg,
bool use_debug_name) {
+ return RemapAtEnd(new_end,
+ tail_name,
+ tail_prot,
+ MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS,
+ /* fd */ -1,
+ /* offset */ 0,
+ error_msg,
+ use_debug_name);
+}
+
+MemMap MemMap::RemapAtEnd(uint8_t* new_end,
+ const char* tail_name,
+ int tail_prot,
+ int flags,
+ int fd,
+ off_t offset,
+ std::string* error_msg,
+ bool use_debug_name) {
DCHECK_GE(new_end, Begin());
DCHECK_LE(new_end, End());
DCHECK_LE(begin_ + size_, reinterpret_cast<uint8_t*>(base_begin_) + base_size_);
@@ -715,9 +733,6 @@
DCHECK_EQ(tail_base_begin + tail_base_size, old_base_end);
DCHECK_ALIGNED(tail_base_size, kPageSize);
- unique_fd fd;
- int flags = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS;
-
MEMORY_TOOL_MAKE_UNDEFINED(tail_base_begin, tail_base_size);
// Note: Do not explicitly unmap the tail region, mmap() with MAP_FIXED automatically
// removes old mappings for the overlapping region. This makes the operation atomic
@@ -726,13 +741,13 @@
tail_base_size,
tail_prot,
flags,
- fd.get(),
- 0));
+ fd,
+ offset));
if (actual == MAP_FAILED) {
PrintFileToLog("/proc/self/maps", LogSeverity::WARNING);
- *error_msg = StringPrintf("anonymous mmap(%p, %zd, 0x%x, 0x%x, %d, 0) failed. See process "
+ *error_msg = StringPrintf("map(%p, %zd, 0x%x, 0x%x, %d, 0) failed. See process "
"maps in the log.", tail_base_begin, tail_base_size, tail_prot, flags,
- fd.get());
+ fd);
return Invalid();
}
// Update *this.
diff --git a/libartbase/base/mem_map.h b/libartbase/base/mem_map.h
index 20eda32..309da27 100644
--- a/libartbase/base/mem_map.h
+++ b/libartbase/base/mem_map.h
@@ -261,6 +261,16 @@
std::string* error_msg,
bool use_debug_name = true);
+ // Unmap the pages of a file at end and remap them to create another memory map.
+ MemMap RemapAtEnd(uint8_t* new_end,
+ const char* tail_name,
+ int tail_prot,
+ int tail_flags,
+ int fd,
+ off_t offset,
+ std::string* error_msg,
+ bool use_debug_name = true);
+
// Take ownership of pages at the beginning of the mapping. The mapping must be an
// anonymous reservation mapping, owning entire pages. The `byte_count` must not
// exceed the size of this reservation.
diff --git a/libartbase/base/mem_map_test.cc b/libartbase/base/mem_map_test.cc
index ab3d18f..bf143d4 100644
--- a/libartbase/base/mem_map_test.cc
+++ b/libartbase/base/mem_map_test.cc
@@ -455,6 +455,53 @@
}
#endif
+TEST_F(MemMapTest, RemapFileViewAtEnd) {
+ CommonInit();
+ std::string error_msg;
+ ScratchFile scratch_file;
+
+ // Create a scratch file 3 pages large.
+ constexpr size_t kMapSize = 3 * kPageSize;
+ std::unique_ptr<uint8_t[]> data(new uint8_t[kMapSize]());
+ memset(data.get(), 1, kPageSize);
+ memset(&data[0], 0x55, kPageSize);
+ memset(&data[kPageSize], 0x5a, kPageSize);
+ memset(&data[2 * kPageSize], 0xaa, kPageSize);
+ ASSERT_TRUE(scratch_file.GetFile()->WriteFully(&data[0], kMapSize));
+
+ MemMap map = MemMap::MapFile(/*byte_count*/kMapSize,
+ PROT_READ,
+ MAP_PRIVATE,
+ scratch_file.GetFd(),
+ /*start*/0,
+ /*low_4gb*/true,
+ scratch_file.GetFilename().c_str(),
+ &error_msg);
+ ASSERT_TRUE(map.IsValid()) << error_msg;
+ ASSERT_TRUE(error_msg.empty());
+ ASSERT_EQ(map.Size(), kMapSize);
+ ASSERT_LT(reinterpret_cast<uintptr_t>(map.BaseBegin()), 1ULL << 32);
+ ASSERT_EQ(data[0], *map.Begin());
+ ASSERT_EQ(data[kPageSize], *(map.Begin() + kPageSize));
+ ASSERT_EQ(data[2 * kPageSize], *(map.Begin() + 2 * kPageSize));
+
+ for (size_t offset = 2 * kPageSize; offset > 0; offset -= kPageSize) {
+ MemMap tail = map.RemapAtEnd(map.Begin() + offset,
+ "bad_offset_map",
+ PROT_READ,
+ MAP_PRIVATE | MAP_FIXED,
+ scratch_file.GetFd(),
+ offset,
+ &error_msg);
+ ASSERT_TRUE(tail.IsValid()) << error_msg;
+ ASSERT_TRUE(error_msg.empty());
+ ASSERT_EQ(offset, map.Size());
+ ASSERT_EQ(static_cast<size_t>(kPageSize), tail.Size());
+ ASSERT_EQ(tail.Begin(), map.Begin() + map.Size());
+ ASSERT_EQ(data[offset], *tail.Begin());
+ }
+}
+
TEST_F(MemMapTest, MapAnonymousExactAddr32bitHighAddr) {
// Some MIPS32 hardware (namely the Creator Ci20 development board)
// cannot allocate in the 2GB-4GB region.
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index beaf5d8..3a14d67 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -18,12 +18,15 @@
#include <sstream>
+#include "android-base/unique_fd.h"
+
#include "arch/context.h"
#include "art_method-inl.h"
#include "base/enums.h"
#include "base/histogram-inl.h"
#include "base/logging.h" // For VLOG.
#include "base/membarrier.h"
+#include "base/memfd.h"
#include "base/mem_map.h"
#include "base/quasi_atomic.h"
#include "base/stl_util.h"
@@ -52,16 +55,32 @@
#include "thread-current-inl.h"
#include "thread_list.h"
+using android::base::unique_fd;
+
namespace art {
namespace jit {
-static constexpr int kProtCode = PROT_READ | PROT_EXEC;
-static constexpr int kProtData = PROT_READ | PROT_WRITE;
-static constexpr int kProtProfile = PROT_READ;
-
static constexpr size_t kCodeSizeLogThreshold = 50 * KB;
static constexpr size_t kStackMapSizeLogThreshold = 50 * KB;
+static constexpr int kProtR = PROT_READ;
+static constexpr int kProtRW = PROT_READ | PROT_WRITE;
+static constexpr int kProtRWX = PROT_READ | PROT_WRITE | PROT_EXEC;
+static constexpr int kProtRX = PROT_READ | PROT_EXEC;
+
+namespace {
+
+// Translate an address belonging to one memory map into an address in a second. This is useful
+// when there are two virtual memory ranges for the same physical memory range.
+template <typename T>
+T* TranslateAddress(T* src_ptr, const MemMap& src, const MemMap& dst) {
+ CHECK(src.HasAddress(src_ptr));
+ uint8_t* const raw_src_ptr = reinterpret_cast<uint8_t*>(src_ptr);
+ return reinterpret_cast<T*>(raw_src_ptr - src.Begin() + dst.Begin());
+}
+
+} // namespace
+
class JitCodeCache::JniStubKey {
public:
explicit JniStubKey(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_)
@@ -190,17 +209,41 @@
// Register for membarrier expedited sync core if JIT will be generating code.
if (!used_only_for_profile_data) {
- art::membarrier(art::MembarrierCommand::kRegisterPrivateExpeditedSyncCore);
+ if (art::membarrier(art::MembarrierCommand::kRegisterPrivateExpeditedSyncCore) != 0) {
+ // MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE ensures that CPU instruction pipelines are
+ // flushed and it's used when adding code to the JIT. The memory used by the new code may
+ // have just been released and, in theory, the old code could still be in a pipeline.
+ VLOG(jit) << "Kernel does not support membarrier sync-core";
+ }
}
- // Decide how we should map the code and data sections.
- // If we use the code cache just for profiling we do not need to map the code section as
- // executable.
- // NOTE 1: this is yet another workaround to bypass strict SElinux policies in order to be able
- // to profile system server.
- // NOTE 2: We could just not create the code section at all but we will need to
- // special case too many cases.
- int memmap_flags_prot_code = used_only_for_profile_data ? kProtProfile : kProtCode;
+ // File descriptor enabling dual-view mapping of code section.
+ unique_fd mem_fd;
+
+ // Bionic supports memfd_create, but the call may fail on older kernels.
+ mem_fd = unique_fd(art::memfd_create("/jit-cache", /* flags */ 0));
+ if (mem_fd.get() < 0) {
+ VLOG(jit) << "Failed to initialize dual view JIT. memfd_create() error: "
+ << strerror(errno);
+ }
+
+ if (mem_fd.get() >= 0 && ftruncate(mem_fd, max_capacity) != 0) {
+ std::ostringstream oss;
+ oss << "Failed to initialize memory file: " << strerror(errno);
+ *error_msg = oss.str();
+ return nullptr;
+ }
+
+ // Data cache will be half of the initial allocation.
+ // Code cache will be the other half of the initial allocation.
+ // TODO: Make this variable?
+
+ // Align both capacities to page size, as that's the unit mspaces use.
+ initial_capacity = RoundDown(initial_capacity, 2 * kPageSize);
+ max_capacity = RoundDown(max_capacity, 2 * kPageSize);
+ const size_t data_capacity = max_capacity / 2;
+ const size_t exec_capacity = used_only_for_profile_data ? 0 : max_capacity - data_capacity;
+ DCHECK_LE(data_capacity + exec_capacity, max_capacity);
std::string error_str;
// Map name specific for android_os_Debug.cpp accounting.
@@ -208,71 +251,149 @@
// We could do PC-relative addressing to avoid this problem, but that
// would require reserving code and data area before submitting, which
// means more windows for the code memory to be RWX.
- MemMap data_map = MemMap::MapAnonymous(
- "data-code-cache",
- /* addr */ nullptr,
- max_capacity,
- kProtData,
- /* low_4gb */ true,
- /* reuse */ false,
- /* reservation */ nullptr,
- &error_str);
- if (!data_map.IsValid()) {
+ int base_flags;
+ MemMap data_pages;
+ if (mem_fd.get() >= 0) {
+ // Dual view of JIT code cache case. Create an initial mapping of data pages large enough
+ // for data and non-writable view of JIT code pages. We use the memory file descriptor to
+ // enable dual mapping - we'll create a second mapping using the descriptor below. The
+ // mappings will look like:
+ //
+ // VA PA
+ //
+ // +---------------+
+ // | non exec code |\
+ // +---------------+ \
+ // : :\ \
+ // +---------------+.\.+---------------+
+ // | exec code | \| code |
+ // +---------------+...+---------------+
+ // | data | | data |
+ // +---------------+...+---------------+
+ //
+ // In this configuration code updates are written to the non-executable view of the code
+ // cache, and the executable view of the code cache has fixed RX memory protections.
+ //
+ // This memory needs to be mapped shared as the code portions will have two mappings.
+ base_flags = MAP_SHARED;
+ data_pages = MemMap::MapFile(
+ data_capacity + exec_capacity,
+ kProtRW,
+ base_flags,
+ mem_fd,
+ /* start */ 0,
+ /* low_4gb */ true,
+ "data-code-cache",
+ &error_str);
+ } else {
+ // Single view of JIT code cache case. Create an initial mapping of data pages large enough
+ // for data and JIT code pages. The mappings will look like:
+ //
+ // VA PA
+ //
+ // +---------------+...+---------------+
+ // | exec code | | code |
+ // +---------------+...+---------------+
+ // | data | | data |
+ // +---------------+...+---------------+
+ //
+ // In this configuration code updates are written to the executable view of the code cache,
+ // and the executable view of the code cache transitions RX to RWX for the update and then
+ // back to RX after the update.
+ base_flags = MAP_PRIVATE | MAP_ANON;
+ data_pages = MemMap::MapAnonymous(
+ "data-code-cache",
+ /* addr */ nullptr,
+ data_capacity + exec_capacity,
+ kProtRW,
+ /* low_4gb */ true,
+ /* reuse */ false,
+ /* reservation */ nullptr,
+ &error_str);
+ }
+
+ if (!data_pages.IsValid()) {
std::ostringstream oss;
oss << "Failed to create read write cache: " << error_str << " size=" << max_capacity;
*error_msg = oss.str();
return nullptr;
}
- // Align both capacities to page size, as that's the unit mspaces use.
- initial_capacity = RoundDown(initial_capacity, 2 * kPageSize);
- max_capacity = RoundDown(max_capacity, 2 * kPageSize);
+ MemMap exec_pages;
+ MemMap non_exec_pages;
+ if (exec_capacity > 0) {
+ uint8_t* const divider = data_pages.Begin() + data_capacity;
+ // Set initial permission for executable view to catch any SELinux permission problems early
+ // (for processes that cannot map WX pages). Otherwise, this region does not need to be
+ // executable as there is no code in the cache yet.
+ exec_pages = data_pages.RemapAtEnd(divider,
+ "jit-code-cache",
+ kProtRX,
+ base_flags | MAP_FIXED,
+ mem_fd.get(),
+ (mem_fd.get() >= 0) ? data_capacity : 0,
+ &error_str);
+ if (!exec_pages.IsValid()) {
+ std::ostringstream oss;
+ oss << "Failed to create read execute code cache: " << error_str << " size=" << max_capacity;
+ *error_msg = oss.str();
+ return nullptr;
+ }
- // Data cache is 1 / 2 of the map.
- // TODO: Make this variable?
- size_t data_size = max_capacity / 2;
- size_t code_size = max_capacity - data_size;
- DCHECK_EQ(code_size + data_size, max_capacity);
- uint8_t* divider = data_map.Begin() + data_size;
-
- MemMap code_map = data_map.RemapAtEnd(
- divider, "jit-code-cache", memmap_flags_prot_code | PROT_WRITE, &error_str);
- if (!code_map.IsValid()) {
- std::ostringstream oss;
- oss << "Failed to create read write execute cache: " << error_str << " size=" << max_capacity;
- *error_msg = oss.str();
- return nullptr;
+ if (mem_fd.get() >= 0) {
+ // For dual view, create the secondary view of code memory used for updating code. This view
+ // is never executable.
+ non_exec_pages = MemMap::MapFile(exec_capacity,
+ kProtR,
+ base_flags,
+ mem_fd,
+ /* start */ data_capacity,
+ /* low_4GB */ false,
+ "jit-code-cache-rw",
+ &error_str);
+ if (!exec_pages.IsValid()) {
+ std::ostringstream oss;
+ oss << "Failed to create read write code cache: " << error_str << " size=" << max_capacity;
+ *error_msg = oss.str();
+ return nullptr;
+ }
+ }
+ } else {
+ // Profiling only. No memory for code required.
+ DCHECK(used_only_for_profile_data);
}
- DCHECK_EQ(code_map.Begin(), divider);
- data_size = initial_capacity / 2;
- code_size = initial_capacity - data_size;
- DCHECK_EQ(code_size + data_size, initial_capacity);
+
+ const size_t initial_data_capacity = initial_capacity / 2;
+ const size_t initial_exec_capacity =
+ (exec_capacity == 0) ? 0 : (initial_capacity - initial_data_capacity);
+
return new JitCodeCache(
- std::move(code_map),
- std::move(data_map),
- code_size,
- data_size,
+ std::move(data_pages),
+ std::move(exec_pages),
+ std::move(non_exec_pages),
+ initial_data_capacity,
+ initial_exec_capacity,
max_capacity,
- garbage_collect_code,
- memmap_flags_prot_code);
+ garbage_collect_code);
}
-JitCodeCache::JitCodeCache(MemMap&& code_map,
- MemMap&& data_map,
- size_t initial_code_capacity,
+JitCodeCache::JitCodeCache(MemMap&& data_pages,
+ MemMap&& exec_pages,
+ MemMap&& non_exec_pages,
size_t initial_data_capacity,
+ size_t initial_exec_capacity,
size_t max_capacity,
- bool garbage_collect_code,
- int memmap_flags_prot_code)
+ bool garbage_collect_code)
: lock_("Jit code cache", kJitCodeCacheLock),
lock_cond_("Jit code cache condition variable", lock_),
collection_in_progress_(false),
- code_map_(std::move(code_map)),
- data_map_(std::move(data_map)),
+ data_pages_(std::move(data_pages)),
+ exec_pages_(std::move(exec_pages)),
+ non_exec_pages_(std::move(non_exec_pages)),
max_capacity_(max_capacity),
- current_capacity_(initial_code_capacity + initial_data_capacity),
- code_end_(initial_code_capacity),
+ current_capacity_(initial_exec_capacity + initial_data_capacity),
data_end_(initial_data_capacity),
+ exec_end_(initial_exec_capacity),
last_collection_increased_code_cache_(false),
garbage_collect_code_(garbage_collect_code),
used_memory_for_data_(0),
@@ -284,40 +405,46 @@
histogram_code_memory_use_("Memory used for compiled code", 16),
histogram_profiling_info_memory_use_("Memory used for profiling info", 16),
is_weak_access_enabled_(true),
- inline_cache_cond_("Jit inline cache condition variable", lock_),
- memmap_flags_prot_code_(memmap_flags_prot_code) {
+ inline_cache_cond_("Jit inline cache condition variable", lock_) {
- DCHECK_GE(max_capacity, initial_code_capacity + initial_data_capacity);
- code_mspace_ = create_mspace_with_base(code_map_.Begin(), code_end_, false /*locked*/);
- data_mspace_ = create_mspace_with_base(data_map_.Begin(), data_end_, false /*locked*/);
+ DCHECK_GE(max_capacity, initial_exec_capacity + initial_data_capacity);
- if (code_mspace_ == nullptr || data_mspace_ == nullptr) {
- PLOG(FATAL) << "create_mspace_with_base failed";
+ // Initialize the data heap
+ data_mspace_ = create_mspace_with_base(data_pages_.Begin(), data_end_, false /*locked*/);
+ CHECK(data_mspace_ != nullptr) << "create_mspace_with_base (data) failed";
+
+ // Initialize the code heap
+ MemMap* code_heap = nullptr;
+ if (non_exec_pages_.IsValid()) {
+ code_heap = &non_exec_pages_;
+ } else if (exec_pages_.IsValid()) {
+ code_heap = &exec_pages_;
}
-
- SetFootprintLimit(current_capacity_);
-
- CheckedCall(mprotect,
- "mprotect jit code cache",
- code_map_.Begin(),
- code_map_.Size(),
- memmap_flags_prot_code_);
- CheckedCall(mprotect,
- "mprotect jit data cache",
- data_map_.Begin(),
- data_map_.Size(),
- kProtData);
+ if (code_heap != nullptr) {
+ // Make all pages reserved for the code heap writable. The mspace allocator, that manages the
+ // heap, will take and initialize pages in create_mspace_with_base().
+ CheckedCall(mprotect, "create code heap", code_heap->Begin(), code_heap->Size(), kProtRW);
+ exec_mspace_ = create_mspace_with_base(code_heap->Begin(), exec_end_, false /*locked*/);
+ CHECK(exec_mspace_ != nullptr) << "create_mspace_with_base (exec) failed";
+ SetFootprintLimit(current_capacity_);
+ // Protect pages containing heap metadata. Updates to the code heap toggle write permission to
+ // perform the update and there are no other times write access is required.
+ CheckedCall(mprotect, "protect code heap", code_heap->Begin(), code_heap->Size(), kProtR);
+ } else {
+ SetFootprintLimit(current_capacity_);
+ exec_mspace_ = nullptr;
+ }
VLOG(jit) << "Created jit code cache: initial data size="
<< PrettySize(initial_data_capacity)
<< ", initial code size="
- << PrettySize(initial_code_capacity);
+ << PrettySize(initial_exec_capacity);
}
JitCodeCache::~JitCodeCache() {}
bool JitCodeCache::ContainsPc(const void* ptr) const {
- return code_map_.Begin() <= ptr && ptr < code_map_.End();
+ return exec_pages_.Begin() <= ptr && ptr < exec_pages_.End();
}
bool JitCodeCache::WillExecuteJitCode(ArtMethod* method) {
@@ -385,22 +512,20 @@
: ScopedTrace("ScopedCodeCacheWrite"),
code_cache_(code_cache) {
ScopedTrace trace("mprotect all");
- CheckedCall(
- mprotect,
- "make code writable",
- code_cache_->code_map_.Begin(),
- code_cache_->code_map_.Size(),
- code_cache_->memmap_flags_prot_code_ | PROT_WRITE);
+ const MemMap* const updatable_pages = code_cache_->GetUpdatableCodeMapping();
+ if (updatable_pages != nullptr) {
+ int prot = code_cache_->HasDualCodeMapping() ? kProtRW : kProtRWX;
+ CheckedCall(mprotect, "Cache +W", updatable_pages->Begin(), updatable_pages->Size(), prot);
+ }
}
~ScopedCodeCacheWrite() {
ScopedTrace trace("mprotect code");
- CheckedCall(
- mprotect,
- "make code protected",
- code_cache_->code_map_.Begin(),
- code_cache_->code_map_.Size(),
- code_cache_->memmap_flags_prot_code_);
+ const MemMap* const updatable_pages = code_cache_->GetUpdatableCodeMapping();
+ if (updatable_pages != nullptr) {
+ int prot = code_cache_->HasDualCodeMapping() ? kProtR : kProtRX;
+ CheckedCall(mprotect, "Cache -W", updatable_pages->Begin(), updatable_pages->Size(), prot);
+ }
}
private:
@@ -417,7 +542,7 @@
size_t code_size,
size_t data_size,
bool osr,
- Handle<mirror::ObjectArray<mirror::Object>> roots,
+ const std::vector<Handle<mirror::Object>>& roots,
bool has_should_deoptimize_flag,
const ArenaSet<ArtMethod*>& cha_single_implementation_list) {
uint8_t* result = CommitCodeInternal(self,
@@ -483,18 +608,16 @@
return stack_map_data - ComputeRootTableSize(GetNumberOfRoots(stack_map_data));
}
-static void DCheckRootsAreValid(Handle<mirror::ObjectArray<mirror::Object>> roots)
+static void DCheckRootsAreValid(const std::vector<Handle<mirror::Object>>& roots)
REQUIRES(!Locks::intern_table_lock_) REQUIRES_SHARED(Locks::mutator_lock_) {
if (!kIsDebugBuild) {
return;
}
- const uint32_t length = roots->GetLength();
// Put all roots in `roots_data`.
- for (uint32_t i = 0; i < length; ++i) {
- ObjPtr<mirror::Object> object = roots->Get(i);
+ for (Handle<mirror::Object> object : roots) {
// Ensure the string is strongly interned. b/32995596
if (object->IsString()) {
- ObjPtr<mirror::String> str = ObjPtr<mirror::String>::DownCast(object);
+ ObjPtr<mirror::String> str = object->AsString();
ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
CHECK(class_linker->GetInternTable()->LookupStrong(Thread::Current(), str) != nullptr);
}
@@ -502,12 +625,12 @@
}
void JitCodeCache::FillRootTable(uint8_t* roots_data,
- Handle<mirror::ObjectArray<mirror::Object>> roots) {
+ const std::vector<Handle<mirror::Object>>& roots) {
GcRoot<mirror::Object>* gc_roots = reinterpret_cast<GcRoot<mirror::Object>*>(roots_data);
- const uint32_t length = roots->GetLength();
+ const uint32_t length = roots.size();
// Put all roots in `roots_data`.
for (uint32_t i = 0; i < length; ++i) {
- ObjPtr<mirror::Object> object = roots->Get(i);
+ ObjPtr<mirror::Object> object = roots[i].Get();
gc_roots[i] = GcRoot<mirror::Object>(object);
}
}
@@ -604,7 +727,13 @@
if (OatQuickMethodHeader::FromCodePointer(code_ptr)->IsOptimized()) {
FreeData(GetRootTable(code_ptr));
} // else this is a JNI stub without any data.
- FreeCode(reinterpret_cast<uint8_t*>(allocation));
+
+ uint8_t* code_allocation = reinterpret_cast<uint8_t*>(allocation);
+ if (HasDualCodeMapping()) {
+ code_allocation = TranslateAddress(code_allocation, exec_pages_, non_exec_pages_);
+ }
+
+ FreeCode(code_allocation);
}
void JitCodeCache::FreeAllMethodHeaders(
@@ -755,6 +884,16 @@
}
}
+const MemMap* JitCodeCache::GetUpdatableCodeMapping() const {
+ if (HasDualCodeMapping()) {
+ return &non_exec_pages_;
+ } else if (HasCodeMapping()) {
+ return &exec_pages_;
+ } else {
+ return nullptr;
+ }
+}
+
uint8_t* JitCodeCache::CommitCodeInternal(Thread* self,
ArtMethod* method,
uint8_t* stack_map,
@@ -763,7 +902,7 @@
size_t code_size,
size_t data_size,
bool osr,
- Handle<mirror::ObjectArray<mirror::Object>> roots,
+ const std::vector<Handle<mirror::Object>>& roots,
bool has_should_deoptimize_flag,
const ArenaSet<ArtMethod*>&
cha_single_implementation_list) {
@@ -775,31 +914,52 @@
DCheckRootsAreValid(roots);
}
- size_t alignment = GetInstructionSetAlignment(kRuntimeISA);
- // Ensure the header ends up at expected instruction alignment.
- size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment);
- size_t total_size = header_size + code_size;
-
OatQuickMethodHeader* method_header = nullptr;
uint8_t* code_ptr = nullptr;
- uint8_t* memory = nullptr;
+
MutexLock mu(self, lock_);
// We need to make sure that there will be no jit-gcs going on and wait for any ongoing one to
// finish.
WaitForPotentialCollectionToCompleteRunnable(self);
{
ScopedCodeCacheWrite scc(this);
- memory = AllocateCode(total_size);
- if (memory == nullptr) {
+
+ size_t alignment = GetInstructionSetAlignment(kRuntimeISA);
+ // Ensure the header ends up at expected instruction alignment.
+ size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment);
+ size_t total_size = header_size + code_size;
+
+ // AllocateCode allocates memory in non-executable region for alignment header and code. The
+ // header size may include alignment padding.
+ uint8_t* nox_memory = AllocateCode(total_size);
+ if (nox_memory == nullptr) {
return nullptr;
}
- code_ptr = memory + header_size;
+ // code_ptr points to non-executable code.
+ code_ptr = nox_memory + header_size;
std::copy(code, code + code_size, code_ptr);
method_header = OatQuickMethodHeader::FromCodePointer(code_ptr);
+
+ // From here code_ptr points to executable code.
+ if (non_exec_pages_.IsValid()) {
+ code_ptr = TranslateAddress(code_ptr, non_exec_pages_, exec_pages_);
+ }
+
new (method_header) OatQuickMethodHeader(
(stack_map != nullptr) ? code_ptr - stack_map : 0u,
code_size);
+
+ DCHECK(!Runtime::Current()->IsAotCompiler());
+ if (has_should_deoptimize_flag) {
+ method_header->SetHasShouldDeoptimizeFlag();
+ }
+
+ // Update method_header pointer to executable code region.
+ if (non_exec_pages_.IsValid()) {
+ method_header = TranslateAddress(method_header, non_exec_pages_, exec_pages_);
+ }
+
// Flush caches before we remove write permission because some ARMv8 Qualcomm kernels may
// trigger a segfault if a page fault occurs when requesting a cache maintenance operation.
// This is a kernel bug that we need to work around until affected devices (e.g. Nexus 5X and
@@ -815,16 +975,14 @@
// shootdown (incidentally invalidating the CPU pipelines by sending an IPI to all cores to
// notify them of the TLB invalidation). Some architectures, notably ARM and ARM64, have
// hardware support that broadcasts TLB invalidations and so their kernels have no software
- // based TLB shootdown.
+ // based TLB shootdown. The sync-core flavor of membarrier was introduced in Linux 4.16 to
+ // address this (see mbarrier(2)). The membarrier here will fail on prior kernels and on
+ // platforms lacking the appropriate support.
art::membarrier(art::MembarrierCommand::kPrivateExpeditedSyncCore);
- DCHECK(!Runtime::Current()->IsAotCompiler());
- if (has_should_deoptimize_flag) {
- method_header->SetHasShouldDeoptimizeFlag();
- }
-
number_of_compilations_++;
}
+
// We need to update the entry point in the runnable state for the instrumentation.
{
// The following needs to be guarded by cha_lock_ also. Otherwise it's possible that the
@@ -1169,9 +1327,9 @@
DCHECK(IsAlignedParam(per_space_footprint, kPageSize));
DCHECK_EQ(per_space_footprint * 2, new_footprint);
mspace_set_footprint_limit(data_mspace_, per_space_footprint);
- {
+ if (exec_mspace_ != nullptr) {
ScopedCodeCacheWrite scc(this);
- mspace_set_footprint_limit(code_mspace_, per_space_footprint);
+ mspace_set_footprint_limit(exec_mspace_, per_space_footprint);
}
}
@@ -1246,8 +1404,8 @@
number_of_collections_++;
live_bitmap_.reset(CodeCacheBitmap::Create(
"code-cache-bitmap",
- reinterpret_cast<uintptr_t>(code_map_.Begin()),
- reinterpret_cast<uintptr_t>(code_map_.Begin() + current_capacity_ / 2)));
+ reinterpret_cast<uintptr_t>(exec_pages_.Begin()),
+ reinterpret_cast<uintptr_t>(exec_pages_.Begin() + current_capacity_ / 2)));
collection_in_progress_ = true;
}
}
@@ -1616,15 +1774,17 @@
// NO_THREAD_SAFETY_ANALYSIS as this is called from mspace code, at which point the lock
// is already held.
void* JitCodeCache::MoreCore(const void* mspace, intptr_t increment) NO_THREAD_SAFETY_ANALYSIS {
- if (code_mspace_ == mspace) {
- size_t result = code_end_;
- code_end_ += increment;
- return reinterpret_cast<void*>(result + code_map_.Begin());
+ if (mspace == exec_mspace_) {
+ DCHECK(exec_mspace_ != nullptr);
+ const MemMap* const code_pages = GetUpdatableCodeMapping();
+ void* result = code_pages->Begin() + exec_end_;
+ exec_end_ += increment;
+ return result;
} else {
DCHECK_EQ(data_mspace_, mspace);
- size_t result = data_end_;
+ void* result = data_pages_.Begin() + data_end_;
data_end_ += increment;
- return reinterpret_cast<void*>(result + data_map_.Begin());
+ return result;
}
}
@@ -1852,7 +2012,7 @@
uint8_t* JitCodeCache::AllocateCode(size_t code_size) {
size_t alignment = GetInstructionSetAlignment(kRuntimeISA);
uint8_t* result = reinterpret_cast<uint8_t*>(
- mspace_memalign(code_mspace_, alignment, code_size));
+ mspace_memalign(exec_mspace_, alignment, code_size));
size_t header_size = RoundUp(sizeof(OatQuickMethodHeader), alignment);
// Ensure the header ends up at expected instruction alignment.
DCHECK_ALIGNED_PARAM(reinterpret_cast<uintptr_t>(result + header_size), alignment);
@@ -1862,7 +2022,7 @@
void JitCodeCache::FreeCode(uint8_t* code) {
used_memory_for_code_ -= mspace_usable_size(code);
- mspace_free(code_mspace_, code);
+ mspace_free(exec_mspace_, code);
}
uint8_t* JitCodeCache::AllocateData(size_t data_size) {
diff --git a/runtime/jit/jit_code_cache.h b/runtime/jit/jit_code_cache.h
index a4a0f8f..76ad8db 100644
--- a/runtime/jit/jit_code_cache.h
+++ b/runtime/jit/jit_code_cache.h
@@ -141,7 +141,7 @@
size_t code_size,
size_t data_size,
bool osr,
- Handle<mirror::ObjectArray<mirror::Object>> roots,
+ const std::vector<Handle<mirror::Object>>& roots,
bool has_should_deoptimize_flag,
const ArenaSet<ArtMethod*>& cha_single_implementation_list)
REQUIRES_SHARED(Locks::mutator_lock_)
@@ -223,7 +223,7 @@
REQUIRES_SHARED(Locks::mutator_lock_);
bool OwnsSpace(const void* mspace) const NO_THREAD_SAFETY_ANALYSIS {
- return mspace == code_mspace_ || mspace == data_mspace_;
+ return mspace == data_mspace_ || mspace == exec_mspace_;
}
void* MoreCore(const void* mspace, intptr_t increment);
@@ -279,13 +279,13 @@
private:
// Take ownership of maps.
- JitCodeCache(MemMap&& code_map,
- MemMap&& data_map,
- size_t initial_code_capacity,
+ JitCodeCache(MemMap&& data_pages,
+ MemMap&& exec_pages,
+ MemMap&& non_exec_pages,
size_t initial_data_capacity,
+ size_t initial_exec_capacity,
size_t max_capacity,
- bool garbage_collect_code,
- int memmap_flags_prot_code);
+ bool garbage_collect_code);
// Internal version of 'CommitCode' that will not retry if the
// allocation fails. Return null if the allocation fails.
@@ -297,14 +297,14 @@
size_t code_size,
size_t data_size,
bool osr,
- Handle<mirror::ObjectArray<mirror::Object>> roots,
+ const std::vector<Handle<mirror::Object>>& roots,
bool has_should_deoptimize_flag,
const ArenaSet<ArtMethod*>& cha_single_implementation_list)
REQUIRES(!lock_)
REQUIRES_SHARED(Locks::mutator_lock_);
// Adds the given roots to the roots_data. Only a member for annotalysis.
- void FillRootTable(uint8_t* roots_data, Handle<mirror::ObjectArray<mirror::Object>> roots)
+ void FillRootTable(uint8_t* roots_data, const std::vector<Handle<mirror::Object>>& roots)
REQUIRES(lock_)
REQUIRES_SHARED(Locks::mutator_lock_);
@@ -381,6 +381,16 @@
uint8_t* AllocateData(size_t data_size) REQUIRES(lock_);
void FreeData(uint8_t* data) REQUIRES(lock_);
+ bool HasDualCodeMapping() const {
+ return non_exec_pages_.IsValid();
+ }
+
+ bool HasCodeMapping() const {
+ return exec_pages_.IsValid();
+ }
+
+ const MemMap* GetUpdatableCodeMapping() const;
+
bool IsWeakAccessEnabled(Thread* self) const;
void WaitUntilInlineCacheAccessible(Thread* self)
REQUIRES(!lock_)
@@ -395,14 +405,17 @@
ConditionVariable lock_cond_ GUARDED_BY(lock_);
// Whether there is a code cache collection in progress.
bool collection_in_progress_ GUARDED_BY(lock_);
- // Mem map which holds code.
- MemMap code_map_;
// Mem map which holds data (stack maps and profiling info).
- MemMap data_map_;
- // The opaque mspace for allocating code.
- void* code_mspace_ GUARDED_BY(lock_);
+ MemMap data_pages_;
+ // Mem map which holds code and has executable permission.
+ MemMap exec_pages_;
+ // Mem map which holds code with non executable permission. Only valid for dual view JIT when
+ // this is the non-executable view of code used to write updates.
+ MemMap non_exec_pages_;
// The opaque mspace for allocating data.
void* data_mspace_ GUARDED_BY(lock_);
+ // The opaque mspace for allocating code.
+ void* exec_mspace_ GUARDED_BY(lock_);
// Bitmap for collecting code and data.
std::unique_ptr<CodeCacheBitmap> live_bitmap_;
// Holds compiled code associated with the shorty for a JNI stub.
@@ -420,12 +433,12 @@
// The current capacity in bytes of the code cache.
size_t current_capacity_ GUARDED_BY(lock_);
- // The current footprint in bytes of the code portion of the code cache.
- size_t code_end_ GUARDED_BY(lock_);
-
// The current footprint in bytes of the data portion of the code cache.
size_t data_end_ GUARDED_BY(lock_);
+ // The current footprint in bytes of the code portion of the code cache.
+ size_t exec_end_ GUARDED_BY(lock_);
+
// Whether the last collection round increased the code cache.
bool last_collection_increased_code_cache_ GUARDED_BY(lock_);
@@ -464,9 +477,6 @@
// Condition to wait on for accessing inline caches.
ConditionVariable inline_cache_cond_ GUARDED_BY(lock_);
- // Mapping flags for the code section.
- const int memmap_flags_prot_code_;
-
friend class art::JitJniStubTestHelper;
friend class ScopedCodeCacheWrite;
diff --git a/test/Android.bp b/test/Android.bp
index d1b23b1..6a9ec2d 100644
--- a/test/Android.bp
+++ b/test/Android.bp
@@ -545,7 +545,11 @@
name: "badsuspend-defaults",
host_supported: true,
srcs: ["1952-bad-jit-suspend/bad_suspend.cc"],
- defaults: ["art_defaults"],
+ defaults: [
+ "art_test_defaults",
+ "art_debug_defaults",
+ "art_defaults",
+ ],
// Note that this tool needs to be built for both 32-bit and 64-bit since it requires
// to be same ISA as what it is attached to.
diff --git a/tools/libcore_failures.txt b/tools/libcore_failures.txt
index d83c522..4c9fd96 100644
--- a/tools/libcore_failures.txt
+++ b/tools/libcore_failures.txt
@@ -223,6 +223,8 @@
names: ["libcore.libcore.io.FdsanTest#testFileInputStream",
"libcore.libcore.io.FdsanTest#testFileOutputStream",
"libcore.libcore.io.FdsanTest#testRandomAccessFile",
- "libcore.libcore.io.FdsanTest#testParcelFileDescriptor"]
+ "libcore.libcore.io.FdsanTest#testParcelFileDescriptor",
+ "libcore.libcore.io.FdsanTest#testDatagramSocket",
+ "libcore.libcore.io.FdsanTest#testSocket"]
}
]