diff options
author | 2023-11-20 18:09:32 +0000 | |
---|---|---|
committer | 2023-12-27 18:13:06 +0000 | |
commit | b343f663e9b0d63ed4423f6e217d5de34e437440 (patch) | |
tree | ba8e73334b342c398f7cc201f278807452b70635 | |
parent | 225de25638f0e7bfa4d4beb2c82f4d60b971c5af (diff) |
Optimize access to gPageSize
This change refactors the global constants dependent on gPageSize in
a way that gPageSize and dependent constants are now internal to the
runtime. Also it removes static initialization dependency on gPageSize
and disallows accessing gPageSize during the static initialization
phase. Consequently, this removes a need in the condition check when
accessing gPageSize and allows compiler to recognise the value as
a constant that can be cached where used repeatedly.
From this point, gPageSize should only be used within libart. For most
of the other cases MemMap::GetPageSize() should be used. Where MemMap
is unavailable e.g. during static initialization or another stage when
MemMap isn't yet initialized, or in a component which might operate
without MemMap being initialized, the GetPageSizeSlow() would be
generally suitable. For performance-sensitive code, GetPageSizeSlow()
shouldn't be used without caching the value to remove repeated calls
of the function.
- The gPageSize constant is now local to libart. Access to the variable
is disallowed until Runtime is initialized, so removing the chance of
static initialization order issue related to this value.
The initializers for the other constants with initialization
that used to depend on the gPageSize value are now referring to
GetPageSizeSlow instead, so removing the static initialization
dependency, and consequently, possible order issues.
This removes the condition check upon each load of the gPageSize.
Also as the variable is defined with "hidden" visibility, it isn't
exported and so is unavailable to other DSOs.
This also removes one indirection level in accessing it as GOT lookup
is not required anymore.
- Remove gPMDSize and gPUDSize from libartbase
Move the constants to gc::Heap in form of functions computing their
values. Also move the BestPageTableAlignment function dependent on
the values, as it is only used within the GC.
Direct accesses to gPMDSize are only performed for assertions and so
replacing that with a function call shouldn't matter for performance.
gPUDSize isn't accessed directly.
The rest the values accesses are done via BestPageTableAlignment,
which is called from:
- MarkCompact constructor
- MarkCompact::AddLinearAllocSpaceData
- GcVisitedArenaPool::AddMap
In all cases, it's performed in conjunction with MapAnonymousAligned,
which involves an mmap system call. The potential slight slowdown
in BestPageTableAlignment - due to the dynamic computation of the
gPMDSize and gPUDSize values - is expected to be negligible in
comparison to the performance cost of mmap system call.
- Move gPageSize declaration and definition to the runtime / component
Replace the removed uses of gPageSize outside libart with
MemMap::GetPageSize() / GetPageSizeSlow().
There is one exception to the above statement in this change - in
FdFile, where the page size appears to be used arbitrarily to compute
the maximum buffer size: in that case replace the maximum buffer size
with a constant which would have matched the value when the page size
is 4K.
- Make gNumLrtSlots a member variable
The gNumLrtSlots global is only used within the SmallLrtAllocator
class. Move it inside the class as a member variable.
- Remove gStackOverflowProtectedSize global
Value of the gStackOverflowProtectedSize is computed as a page size
multiplied by a constant dependent on whether the address sanitizer
is enabled. If address sanitizer is disabled, the constant is 1,
in which case gStackOverflowProtectedSize is equal to the page size.
Replace the global with an always-inlined function which is expected
to expand into a simple reference to the gPageSize at each call site
if the address sanitizer is not enabled.
- Remove ART_PAGE_SIZE_AGNOSTIC_DECLARE_* macros
The macros aren't used anymore. Remove them and GlobalConst.
The tests were run for legacy 4K, page size agnostic 4K and 16K.
Co-authored-by: Ruben Ayrapetyan <ruben.ayrapetyan@arm.com>
Co-authored-by: Richard Neill <richard.neill@arm.com>
Test: art/tools/run-gtests.sh
Test: art/test/testrunner/testrunner.py --target --64
Test: art/tools/run-libcore-tests.sh --mode=device --variant=X64
Test: art/tools/run-libjdwp-tests.sh --mode=device --variant=X64
Test: cd art && mma
Change-Id: I07fd85cd35dd443dee5cff0f7435ab47ba727c6f
41 files changed, 528 insertions, 395 deletions
diff --git a/compiler/common_compiler_test.cc b/compiler/common_compiler_test.cc index eb8c21c0d1..f94d3048c2 100644 --- a/compiler/common_compiler_test.cc +++ b/compiler/common_compiler_test.cc @@ -51,15 +51,16 @@ class CommonCompilerTestImpl::CodeAndMetadata { CodeAndMetadata(ArrayRef<const uint8_t> code, ArrayRef<const uint8_t> vmap_table, InstructionSet instruction_set) { + const size_t page_size = MemMap::GetPageSize(); const uint32_t code_size = code.size(); CHECK_NE(code_size, 0u); const uint32_t vmap_table_offset = vmap_table.empty() ? 0u : sizeof(OatQuickMethodHeader) + vmap_table.size(); OatQuickMethodHeader method_header(vmap_table_offset); const size_t code_alignment = GetInstructionSetCodeAlignment(instruction_set); - DCHECK_ALIGNED_PARAM(static_cast<size_t>(gPageSize), code_alignment); + DCHECK_ALIGNED_PARAM(page_size, code_alignment); const uint32_t code_offset = RoundUp(vmap_table.size() + sizeof(method_header), code_alignment); - const uint32_t capacity = RoundUp(code_offset + code_size, gPageSize); + const uint32_t capacity = RoundUp(code_offset + code_size, page_size); // Create a memfd handle with sufficient capacity. android::base::unique_fd mem_fd(art::memfd_create_compat("test code", /*flags=*/ 0)); diff --git a/compiler/jit/jit_logger.cc b/compiler/jit/jit_logger.cc index c50103df9c..0ca5c38fb1 100644 --- a/compiler/jit/jit_logger.cc +++ b/compiler/jit/jit_logger.cc @@ -211,7 +211,7 @@ void JitLogger::OpenMarkerFile() { int fd = jit_dump_file_->Fd(); // The 'perf inject' tool requires that the jit-PID.dump file // must have a mmap(PROT_READ|PROT_EXEC) record in perf.data. - marker_address_ = mmap(nullptr, gPageSize, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); + marker_address_ = mmap(nullptr, MemMap::GetPageSize(), PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); if (marker_address_ == MAP_FAILED) { LOG(WARNING) << "Failed to create record in perf.data. JITed code profiling will not work."; return; @@ -220,7 +220,7 @@ void JitLogger::OpenMarkerFile() { void JitLogger::CloseMarkerFile() { if (marker_address_ != nullptr) { - munmap(marker_address_, gPageSize); + munmap(marker_address_, MemMap::GetPageSize()); } } diff --git a/dex2oat/common_compiler_driver_test.cc b/dex2oat/common_compiler_driver_test.cc index d47982f54a..91fa859c92 100644 --- a/dex2oat/common_compiler_driver_test.cc +++ b/dex2oat/common_compiler_driver_test.cc @@ -111,13 +111,14 @@ void CommonCompilerDriverTest::SetUp() { // Note: We cannot use MemMap because some tests tear down the Runtime and destroy // the gMaps, so when destroying the MemMap, the test would crash. - inaccessible_page_ = mmap(nullptr, gPageSize, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + const size_t page_size = MemMap::GetPageSize(); + inaccessible_page_ = mmap(nullptr, page_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); CHECK(inaccessible_page_ != MAP_FAILED) << strerror(errno); } void CommonCompilerDriverTest::TearDown() { if (inaccessible_page_ != nullptr) { - munmap(inaccessible_page_, gPageSize); + munmap(inaccessible_page_, MemMap::GetPageSize()); inaccessible_page_ = nullptr; } image_reservation_.Reset(); diff --git a/dex2oat/linker/elf_writer_test.cc b/dex2oat/linker/elf_writer_test.cc index b01ebf59e3..7800c28f68 100644 --- a/dex2oat/linker/elf_writer_test.cc +++ b/dex2oat/linker/elf_writer_test.cc @@ -99,7 +99,7 @@ TEST_F(ElfWriterTest, dlsym) { bool success = ef->GetLoadedSize(&size, &error_msg); CHECK(success) << error_msg; MemMap reservation = MemMap::MapAnonymous("ElfWriterTest#dlsym reservation", - RoundUp(size, gPageSize), + RoundUp(size, MemMap::GetPageSize()), PROT_NONE, /*low_4gb=*/ true, &error_msg); diff --git a/dex2oat/utils/swap_space.cc b/dex2oat/utils/swap_space.cc index 336aad8de5..b0686a47a1 100644 --- a/dex2oat/utils/swap_space.cc +++ b/dex2oat/utils/swap_space.cc @@ -22,6 +22,7 @@ #include <numeric> #include "base/bit_utils.h" +#include "base/mem_map.h" #include "base/macros.h" #include "base/mutex.h" #include "thread-current-inl.h" @@ -146,7 +147,8 @@ void* SwapSpace::Alloc(size_t size) { SwapSpace::SpaceChunk SwapSpace::NewFileChunk(size_t min_size) { #if !defined(__APPLE__) - size_t next_part = std::max(RoundUp(min_size, gPageSize), RoundUp(kMinimumMapSize, gPageSize)); + const size_t page_size = MemMap::GetPageSize(); + size_t next_part = std::max(RoundUp(min_size, page_size), RoundUp(kMinimumMapSize, page_size)); int result = TEMP_FAILURE_RETRY(ftruncate64(fd_, size_ + next_part)); if (result != 0) { PLOG(FATAL) << "Unable to increase swap file."; diff --git a/dexlayout/dex_visualize.cc b/dexlayout/dex_visualize.cc index f47f27c848..9b11ec7f39 100644 --- a/dexlayout/dex_visualize.cc +++ b/dexlayout/dex_visualize.cc @@ -31,6 +31,7 @@ #include <android-base/logging.h> +#include "base/mem_map.h" #include "dex_ir.h" #include "dexlayout.h" #include "profile/profile_compilation_info.h" @@ -49,7 +50,8 @@ class Dumper { explicit Dumper(dex_ir::Header* header) : out_file_(nullptr), sorted_sections_( - dex_ir::GetSortedDexFileSections(header, dex_ir::SortDirection::kSortDescending)) { } + dex_ir::GetSortedDexFileSections(header, dex_ir::SortDirection::kSortDescending)), + page_size_(MemMap::GetPageSize()) {} bool OpenAndPrintHeader(size_t dex_index) { // Open the file and emit the gnuplot prologue. @@ -70,7 +72,7 @@ class Dumper { if (printed_one) { fprintf(out_file_, ", "); } - fprintf(out_file_, "\"%s\" %" PRIuPTR, s.name.c_str(), s.offset / gPageSize); + fprintf(out_file_, "\"%s\" %" PRIuPTR, s.name.c_str(), s.offset / page_size_); printed_one = true; } } @@ -98,8 +100,8 @@ class Dumper { } void DumpAddressRange(uint32_t from, uint32_t size, int class_index) { - const uint32_t low_page = from / gPageSize; - const uint32_t high_page = (size > 0) ? (from + size - 1) / gPageSize : low_page; + const uint32_t low_page = from / page_size_; + const uint32_t high_page = (size > 0) ? (from + size - 1) / page_size_ : low_page; const uint32_t size_delta = high_page - low_page; fprintf(out_file_, "%d %d %d 0 %d\n", low_page, class_index, size_delta, GetColor(from)); } @@ -234,6 +236,7 @@ class Dumper { FILE* out_file_; std::vector<dex_ir::DexFileSection> sorted_sections_; + const size_t page_size_; DISALLOW_COPY_AND_ASSIGN(Dumper); }; @@ -316,7 +319,7 @@ static uint32_t FindNextByteAfterSection(dex_ir::Header* header, /* * Dumps the offset and size of sections within the file. */ -void ShowDexSectionStatistics(dex_ir::Header* header, size_t dex_file_index) { +void ShowDexSectionStatistics(dex_ir::Header* header, size_t dex_file_index, size_t page_size) { // Compute the (multidex) class file name). fprintf(stdout, "%s (%d bytes)\n", MultidexName("classes", dex_file_index, ".dex").c_str(), @@ -336,7 +339,7 @@ void ShowDexSectionStatistics(dex_ir::Header* header, size_t dex_file_index) { file_section.offset, file_section.size, bytes, - RoundUp(bytes, gPageSize) / gPageSize, + RoundUp(bytes, page_size) / page_size, 100 * bytes / header->FileSize()); } fprintf(stdout, "\n"); diff --git a/dexlayout/dex_visualize.h b/dexlayout/dex_visualize.h index a1aa2cd8bc..8406412858 100644 --- a/dexlayout/dex_visualize.h +++ b/dexlayout/dex_visualize.h @@ -38,7 +38,7 @@ void VisualizeDexLayout(dex_ir::Header* header, size_t dex_file_index, ProfileCompilationInfo* profile_info); -void ShowDexSectionStatistics(dex_ir::Header* header, size_t dex_file_index); +void ShowDexSectionStatistics(dex_ir::Header* header, size_t dex_file_index, size_t page_size); } // namespace art diff --git a/dexlayout/dexdiag.cc b/dexlayout/dexdiag.cc index 21bb1c47ee..2821374ef4 100644 --- a/dexlayout/dexdiag.cc +++ b/dexlayout/dexdiag.cc @@ -186,7 +186,7 @@ static char PageTypeChar(uint16_t type) { static uint16_t FindSectionTypeForPage(size_t page, const std::vector<dex_ir::DexFileSection>& sections) { for (const auto& section : sections) { - size_t first_page_of_section = section.offset / gPageSize; + size_t first_page_of_section = section.offset / MemMap::GetPageSize(); // Only consider non-empty sections. if (section.size == 0) { continue; @@ -287,14 +287,15 @@ static void ProcessOneDexMapping(const std::vector<uint64_t>& pagemap, << std::endl; return; } - uint64_t start_page = (dex_file_start - vdex_start) / gPageSize; - uint64_t start_address = start_page * gPageSize; - uint64_t end_page = RoundUp(start_address + dex_file_size, gPageSize) / gPageSize; + uint64_t start_page = (dex_file_start - vdex_start) / MemMap::GetPageSize(); + uint64_t start_address = start_page * MemMap::GetPageSize(); + uint64_t end_page = RoundUp(start_address + dex_file_size, + MemMap::GetPageSize()) / MemMap::GetPageSize(); std::cout << "DEX " << dex_file->GetLocation().c_str() << StringPrintf(": %" PRIx64 "-%" PRIx64, - map_start + start_page * gPageSize, - map_start + end_page * gPageSize) + map_start + start_page * MemMap::GetPageSize(), + map_start + end_page * MemMap::GetPageSize()) << std::endl; // Build a list of the dex file section types, sorted from highest offset to lowest. std::vector<dex_ir::DexFileSection> sections; diff --git a/dexlayout/dexlayout.cc b/dexlayout/dexlayout.cc index e8d0a07e23..d674fd2ebc 100644 --- a/dexlayout/dexlayout.cc +++ b/dexlayout/dexlayout.cc @@ -2218,7 +2218,7 @@ bool DexLayout::ProcessDexFile(const char* file_name, } if (options_.show_section_statistics_) { - ShowDexSectionStatistics(header_, dex_file_index); + ShowDexSectionStatistics(header_, dex_file_index, page_size_); return true; } diff --git a/dexlayout/dexlayout.h b/dexlayout/dexlayout.h index c6f76e5d9f..42383070f5 100644 --- a/dexlayout/dexlayout.h +++ b/dexlayout/dexlayout.h @@ -29,6 +29,7 @@ #include <set> #include <unordered_map> +#include "base/mem_map.h" #include "dex/compact_dex_level.h" #include "dex_container.h" #include "dex/dex_file_layout.h" @@ -111,7 +112,8 @@ class DexLayout { : options_(options), info_(info), out_file_(out_file), - header_(header) { } + header_(header), + page_size_(MemMap::GetPageSize()) { } int ProcessFile(const char* file_name); bool ProcessDexFile(const char* file_name, @@ -194,6 +196,7 @@ class DexLayout { ProfileCompilationInfo* info_; FILE* out_file_; dex_ir::Header* header_; + const size_t page_size_; DexLayoutSections dex_sections_; // Layout hotness information is only calculated when dexlayout is enabled. DexLayoutHotnessInfo layout_hotness_info_; diff --git a/imgdiag/imgdiag.cc b/imgdiag/imgdiag.cc index 1510a04e6f..91ee7d6f4b 100644 --- a/imgdiag/imgdiag.cc +++ b/imgdiag/imgdiag.cc @@ -229,9 +229,9 @@ size_t EntrySize(ArtMethod* art_method) REQUIRES_SHARED(Locks::mutator_lock_) { // Print all pages the entry belongs to void PrintEntryPages(uintptr_t entry_address, size_t entry_size, std::ostream& os) { const char* tabs = " "; - const uintptr_t first_page_idx = entry_address / gPageSize; + const uintptr_t first_page_idx = entry_address / MemMap::GetPageSize(); const uintptr_t last_page_idx = RoundUp(entry_address + entry_size, - kObjectAlignment) / gPageSize; + kObjectAlignment) / MemMap::GetPageSize(); for (uintptr_t page_idx = first_page_idx; page_idx <= last_page_idx; ++page_idx) { os << tabs << "page_idx=" << page_idx << "\n"; } @@ -298,13 +298,14 @@ struct RegionCommon { uintptr_t entry_address = reinterpret_cast<uintptr_t>(entry); // Iterate every page this entry belongs to do { - current_page_idx = entry_address / gPageSize + page_off; + current_page_idx = entry_address / MemMap::GetPageSize() + page_off; if (dirty_pages.find(current_page_idx) != dirty_pages.end()) { // This entry is on a dirty page return true; } page_off++; - } while ((current_page_idx * gPageSize) < RoundUp(entry_address + size, kObjectAlignment)); + } while ((current_page_idx * MemMap::GetPageSize()) < RoundUp(entry_address + size, + kObjectAlignment)); return false; } @@ -1155,7 +1156,8 @@ class RegionData : public RegionSpecializedBase<T> { // Looking at only dirty pages, figure out how many of those bytes belong to dirty entries. // TODO: fix this now that there are multiple regions in a mapping. float true_dirtied_percent = - RegionCommon<T>::GetDirtyEntryBytes() * 1.0f / (mapping_data.dirty_pages * gPageSize); + (RegionCommon<T>::GetDirtyEntryBytes() * 1.0f) / + (mapping_data.dirty_pages * MemMap::GetPageSize()); // Entry specific statistics. os_ << RegionCommon<T>::GetDifferentEntryCount() << " different entries, \n " @@ -1413,7 +1415,7 @@ class ImgDiagDumper { MappingData* mapping_data /*out*/, std::string* error_msg /*out*/) { // Iterate through one page at a time. Boot map begin/end already implicitly aligned. - for (uintptr_t begin = boot_map.start; begin != boot_map.end; begin += gPageSize) { + for (uintptr_t begin = boot_map.start; begin != boot_map.end; begin += MemMap::GetPageSize()) { const ptrdiff_t offset = begin - boot_map.start; // We treat the image header as part of the memory map for now @@ -1422,11 +1424,11 @@ class ImgDiagDumper { const uint8_t* zygote_ptr = &zygote_contents[offset]; const uint8_t* remote_ptr = &remote_contents[offset]; - if (memcmp(zygote_ptr, remote_ptr, gPageSize) != 0) { + if (memcmp(zygote_ptr, remote_ptr, MemMap::GetPageSize()) != 0) { mapping_data->different_pages++; // Count the number of 32-bit integers that are different. - for (size_t i = 0; i < gPageSize / sizeof(uint32_t); ++i) { + for (size_t i = 0; i < MemMap::GetPageSize() / sizeof(uint32_t); ++i) { const uint32_t* remote_ptr_int32 = reinterpret_cast<const uint32_t*>(remote_ptr); const uint32_t* zygote_ptr_int32 = reinterpret_cast<const uint32_t*>(zygote_ptr); @@ -1435,7 +1437,7 @@ class ImgDiagDumper { } } // Count the number of bytes that are different. - for (size_t i = 0; i < gPageSize; ++i) { + for (size_t i = 0; i < MemMap::GetPageSize(); ++i) { if (remote_ptr[i] != zygote_ptr[i]) { mapping_data->different_bytes++; } @@ -1443,11 +1445,11 @@ class ImgDiagDumper { } } - for (uintptr_t begin = boot_map.start; begin != boot_map.end; begin += gPageSize) { + for (uintptr_t begin = boot_map.start; begin != boot_map.end; begin += MemMap::GetPageSize()) { ptrdiff_t offset = begin - boot_map.start; // Virtual page number (for an absolute memory address) - size_t virtual_page_idx = begin / gPageSize; + size_t virtual_page_idx = begin / MemMap::GetPageSize(); uint64_t page_count = 0xC0FFEE; // TODO: virtual_page_idx needs to be from the same process @@ -1555,7 +1557,7 @@ class ImgDiagDumper { // Adjust the `end` of the mapping. Some other mappings may have been // inserted within the image. - boot_map.end = RoundUp(boot_map.start + image_header.GetImageSize(), gPageSize); + boot_map.end = RoundUp(boot_map.start + image_header.GetImageSize(), MemMap::GetPageSize()); // The size of the boot image mapping. size_t boot_map_size = boot_map.end - boot_map.start; @@ -1569,7 +1571,8 @@ class ImgDiagDumper { android::procinfo::MapInfo& zygote_boot_map = *maybe_zygote_boot_map; // Adjust the `end` of the mapping. Some other mappings may have been // inserted within the image. - zygote_boot_map.end = RoundUp(zygote_boot_map.start + image_header.GetImageSize(), gPageSize); + zygote_boot_map.end = RoundUp(zygote_boot_map.start + image_header.GetImageSize(), + MemMap::GetPageSize()); if (zygote_boot_map.start != boot_map.start) { os << "Zygote boot map does not match image boot map: " << "zygote begin " << reinterpret_cast<const void*>(zygote_boot_map.start) @@ -1589,8 +1592,8 @@ class ImgDiagDumper { const uint8_t* image_end_unaligned = image_begin_unaligned + image_header.GetImageSize(); // Adjust range to nearest page - const uint8_t* image_begin = AlignDown(image_begin_unaligned, gPageSize); - const uint8_t* image_end = AlignUp(image_end_unaligned, gPageSize); + const uint8_t* image_begin = AlignDown(image_begin_unaligned, MemMap::GetPageSize()); + const uint8_t* image_end = AlignUp(image_end_unaligned, MemMap::GetPageSize()); size_t image_size = image_end - image_begin; if (image_size != boot_map_size) { @@ -1603,8 +1606,8 @@ class ImgDiagDumper { auto read_contents = [&](File* mem_file, /*out*/ MemMap* map, /*out*/ ArrayRef<uint8_t>* contents) { - DCHECK_ALIGNED_PARAM(boot_map.start, gPageSize); - DCHECK_ALIGNED_PARAM(boot_map_size, gPageSize); + DCHECK_ALIGNED_PARAM(boot_map.start, MemMap::GetPageSize()); + DCHECK_ALIGNED_PARAM(boot_map_size, MemMap::GetPageSize()); std::string name = "Contents of " + mem_file->GetPath(); std::string local_error_msg; // We need to use low 4 GiB memory so that we can walk the objects using standard diff --git a/imgdiag/page_info.cc b/imgdiag/page_info.cc index 628cc60b26..e5b14d4c45 100644 --- a/imgdiag/page_info.cc +++ b/imgdiag/page_info.cc @@ -82,8 +82,9 @@ bool OpenProcFiles(pid_t pid, /*out*/ ProcFiles& files, /*out*/ std::string& err return true; } -void DumpPageInfo(uint64_t virtual_page_index, ProcFiles& proc_files, std::ostream& os) { - const uint64_t virtual_page_addr = virtual_page_index * gPageSize; +void DumpPageInfo(uint64_t virtual_page_index, ProcFiles& proc_files, std::ostream& os, + size_t page_size) { + const uint64_t virtual_page_addr = virtual_page_index * page_size; os << "Virtual page index: " << virtual_page_index << "\n"; os << "Virtual page addr: " << virtual_page_addr << "\n"; @@ -117,7 +118,7 @@ void DumpPageInfo(uint64_t virtual_page_index, ProcFiles& proc_files, std::ostre os << "kpageflags: " << page_flags << "\n"; if (page_count != 0) { - std::vector<uint8_t> page_contents(gPageSize); + std::vector<uint8_t> page_contents(page_size); if (!proc_files.mem.PreadFully(page_contents.data(), page_contents.size(), virtual_page_addr)) { os << "Failed to read page contents\n"; return; @@ -150,13 +151,14 @@ struct MapPageCounts { bool GetMapPageCounts(ProcFiles& proc_files, const android::procinfo::MapInfo& map_info, MapPageCounts& map_page_counts, - std::string& error_msg) { + std::string& error_msg, + size_t page_size) { map_page_counts.name = map_info.name; map_page_counts.start = map_info.start; map_page_counts.end = map_info.end; - std::vector<uint8_t> page_contents(gPageSize); - for (uint64_t begin = map_info.start; begin < map_info.end; begin += gPageSize) { - const size_t virtual_page_index = begin / gPageSize; + std::vector<uint8_t> page_contents(page_size); + for (uint64_t begin = map_info.start; begin < map_info.end; begin += page_size) { + const size_t virtual_page_index = begin / page_size; uint64_t page_frame_number = -1; if (!GetPageFrameNumber(proc_files.pagemap, virtual_page_index, page_frame_number, error_msg)) { return false; @@ -201,7 +203,7 @@ bool GetMapPageCounts(ProcFiles& proc_files, return true; } -void CountZeroPages(pid_t pid, ProcFiles& proc_files, std::ostream& os) { +void CountZeroPages(pid_t pid, ProcFiles& proc_files, std::ostream& os, size_t page_size) { std::vector<android::procinfo::MapInfo> proc_maps; if (!android::procinfo::ReadProcessMaps(pid, &proc_maps)) { os << "Could not read process maps for " << pid; @@ -213,7 +215,7 @@ void CountZeroPages(pid_t pid, ProcFiles& proc_files, std::ostream& os) { for (const android::procinfo::MapInfo& map_info : proc_maps) { MapPageCounts map_page_counts; std::string error_msg; - if (!GetMapPageCounts(proc_files, map_info, map_page_counts, error_msg)) { + if (!GetMapPageCounts(proc_files, map_info, map_page_counts, error_msg, page_size)) { os << "Error getting map page counts for: " << map_info.name << "\n" << error_msg << "\n\n"; continue; } @@ -304,7 +306,8 @@ void CountZeroPages(pid_t pid, ProcFiles& proc_files, std::ostream& os) { int PageInfo(std::ostream& os, pid_t pid, bool count_zero_pages, - std::optional<uint64_t> virtual_page_index) { + std::optional<uint64_t> virtual_page_index, + size_t page_size) { ProcFiles proc_files; std::string error_msg; if (!OpenProcFiles(pid, proc_files, error_msg)) { @@ -312,10 +315,10 @@ int PageInfo(std::ostream& os, return EXIT_FAILURE; } if (virtual_page_index != std::nullopt) { - DumpPageInfo(virtual_page_index.value(), proc_files, os); + DumpPageInfo(virtual_page_index.value(), proc_files, os, page_size); } if (count_zero_pages) { - CountZeroPages(pid, proc_files, os); + CountZeroPages(pid, proc_files, os, page_size); } return EXIT_SUCCESS; } @@ -416,7 +419,8 @@ struct PageInfoMain : public CmdlineMain<PageInfoArgs> { CHECK(args_->os_ != nullptr); return PageInfo( - *args_->os_, args_->pid_, args_->count_zero_pages_, args_->virtual_page_index_) == + *args_->os_, args_->pid_, args_->count_zero_pages_, args_->virtual_page_index_, + MemMap::GetPageSize()) == EXIT_SUCCESS; } diff --git a/libartbase/base/globals.h b/libartbase/base/globals.h index 858e1f6468..027681a1d7 100644 --- a/libartbase/base/globals.h +++ b/libartbase/base/globals.h @@ -124,91 +124,24 @@ static constexpr bool kHostStaticBuildEnabled = true; static constexpr bool kHostStaticBuildEnabled = false; #endif -// Helper class that acts as a global constant which can be initialized with -// a dynamically computed value while not being subject to static initialization -// order issues via gating access to the value through a function which ensures -// the value is initialized before being accessed. -// -// The Initialize function should return T type. It shouldn't have side effects -// and should always return the same value. -template<typename T, auto Initialize> -struct GlobalConst { - operator T() const { - static T data = Initialize(); - return data; - } -}; - -// Helper macros for declaring and defining page size agnostic global values -// which are constants in page size agnostic configuration and constexpr -// in non page size agnostic configuration. -// -// For the former case, this uses the GlobalConst class initializing it with given expression -// which might be the same as for the non page size agnostic configuration (then -// ART_PAGE_SIZE_AGNOSTIC_DECLARE is most suitable to avoid duplication) or might be different -// (in which case ART_PAGE_SIZE_AGNOSTIC_DECLARE_ALT should be used). -// -// The motivation behind these helpers is mainly to provide a way to declare / define / initialize -// the global constants protected from static initialization order issues. -// -// Adding a new value e.g. `const uint32_t gNewVal = function(gPageSize);` can be done, -// for example, via: -// - declaring it using ART_PAGE_SIZE_AGNOSTIC_DECLARE in this header; -// - and defining it with ART_PAGE_SIZE_AGNOSTIC_DEFINE in the globals_unix.cc -// or another suitable module. -// The statements might look as follows: -// ART_PAGE_SIZE_AGNOSTIC_DECLARE(uint32_t, gNewVal, function(gPageSize)); -// ART_PAGE_SIZE_AGNOSTIC_DEFINE(uint32_t, gNewVal); -// -// NOTE: -// The initializer expressions shouldn't have side effects -// and should always return the same value. - +// Within libart, gPageSize should be used to get the page size value once Runtime is initialized. +// For most other cases MemMap::GetPageSize() should be used instead. However, where MemMap is +// unavailable e.g. during static initialization or another stage when MemMap isn't yet initialized, +// or in a component which might operate without MemMap being initialized, the GetPageSizeSlow() +// would be generally suitable. For performance-sensitive code, GetPageSizeSlow() shouldn't be used +// without caching the value to remove repeated calls of the function. #ifdef ART_PAGE_SIZE_AGNOSTIC -// Declaration (page size agnostic version). -#define ART_PAGE_SIZE_AGNOSTIC_DECLARE_ALT(type, name, page_size_agnostic_expr, const_expr) \ - inline type __attribute__((visibility("default"))) \ - name ## _Initializer(void) { \ - return (page_size_agnostic_expr); \ - } \ - extern GlobalConst<type, name ## _Initializer> name -// Definition (page size agnostic version). -#define ART_PAGE_SIZE_AGNOSTIC_DEFINE(type, name) GlobalConst<type, name ## _Initializer> name +inline ALWAYS_INLINE size_t GetPageSizeSlow() { + static_assert(kPageSizeAgnostic, "The dynamic version is only for page size agnostic build"); + static const size_t page_size = sysconf(_SC_PAGE_SIZE); + return page_size; +} #else -// Declaration (non page size agnostic version). -#define ART_PAGE_SIZE_AGNOSTIC_DECLARE_ALT(type, name, page_size_agnostic_expr, const_expr) \ - static constexpr type name = (const_expr) -// Definition (non page size agnostic version). -#define ART_PAGE_SIZE_AGNOSTIC_DEFINE(type, name) -#endif // ART_PAGE_SIZE_AGNOSTIC - -// ART_PAGE_SIZE_AGNOSTIC_DECLARE is same as ART_PAGE_SIZE_AGNOSTIC_DECLARE_ALT -// for the case when the initializer expressions are the same. -#define ART_PAGE_SIZE_AGNOSTIC_DECLARE(type, name, expr) \ - ART_PAGE_SIZE_AGNOSTIC_DECLARE_ALT(type, name, expr, expr) - -// Declaration and definition combined. -#define ART_PAGE_SIZE_AGNOSTIC_DECLARE_AND_DEFINE(type, name, expr) \ - ART_PAGE_SIZE_AGNOSTIC_DECLARE(type, name, expr); \ - ART_PAGE_SIZE_AGNOSTIC_DEFINE(type, name) - -// System page size. We check this against sysconf(_SC_PAGE_SIZE) at runtime, -// but for non page size agnostic configuration we use a simple compile-time -// constant so the compiler can generate better code. -ART_PAGE_SIZE_AGNOSTIC_DECLARE_ALT(size_t, gPageSize, sysconf(_SC_PAGE_SIZE), 4096); - -// TODO: Kernels for arm and x86 in both, 32-bit and 64-bit modes use 512 entries per page-table -// page. Find a way to confirm that in userspace. -// Address range covered by 1 Page Middle Directory (PMD) entry in the page table -ART_PAGE_SIZE_AGNOSTIC_DECLARE(size_t, gPMDSize, (gPageSize / sizeof(uint64_t)) * gPageSize); -// Address range covered by 1 Page Upper Directory (PUD) entry in the page table -ART_PAGE_SIZE_AGNOSTIC_DECLARE(size_t, gPUDSize, (gPageSize / sizeof(uint64_t)) * gPMDSize); - -// Returns the ideal alignment corresponding to page-table levels for the -// given size. -static inline size_t BestPageTableAlignment(size_t size) { - return size < gPUDSize ? gPMDSize : gPUDSize; +constexpr size_t GetPageSizeSlow() { + static_assert(!kPageSizeAgnostic, "The constexpr version is only for page size agnostic build"); + return kMinPageSize; } +#endif } // namespace art diff --git a/libartbase/base/globals_unix.cc b/libartbase/base/globals_unix.cc index 07226241db..9a67dc49dd 100644 --- a/libartbase/base/globals_unix.cc +++ b/libartbase/base/globals_unix.cc @@ -22,10 +22,6 @@ namespace art { -ART_PAGE_SIZE_AGNOSTIC_DEFINE(size_t, gPageSize); -ART_PAGE_SIZE_AGNOSTIC_DEFINE(size_t, gPMDSize); -ART_PAGE_SIZE_AGNOSTIC_DEFINE(size_t, gPUDSize); - #ifndef ART_STATIC_LIBARTBASE #ifdef __APPLE__ diff --git a/libartbase/base/macros.h b/libartbase/base/macros.h index 2a02d6f9c4..ee2801825a 100644 --- a/libartbase/base/macros.h +++ b/libartbase/base/macros.h @@ -123,4 +123,8 @@ template<typename T> ART_FRIEND_TEST(test_set_name, individual_test) #define EXPORT #endif +// Some global variables shouldn't be visible outside libraries declaring them. +// The attribute allows hiding them, so preventing direct access. +#define ALWAYS_HIDDEN __attribute__((visibility("hidden"))) + #endif // ART_LIBARTBASE_BASE_MACROS_H_ diff --git a/libartbase/base/mem_map.cc b/libartbase/base/mem_map.cc index 0d4150c50b..0ab47f2384 100644 --- a/libartbase/base/mem_map.cc +++ b/libartbase/base/mem_map.cc @@ -89,6 +89,9 @@ std::ostream& operator<<(std::ostream& os, const Maps& mem_maps) { } std::mutex* MemMap::mem_maps_lock_ = nullptr; +#ifdef ART_PAGE_SIZE_AGNOSTIC +size_t MemMap::page_size_ = 0; +#endif #if USE_ART_LOW_4G_ALLOCATOR // Handling mem_map in 32b address range for 64b architectures that do not support MAP_32BIT. @@ -103,7 +106,7 @@ static constexpr uintptr_t LOW_MEM_START = 64 * KB; // ART_BASE_ADDR = 0001XXXXXXXXXXXXXXX // ---------------------------------------- // = 0000111111111111111 -// & ~(gPageSize - 1) =~0000000000000001111 +// & ~(page_size - 1) =~0000000000000001111 // ---------------------------------------- // mask = 0000111111111110000 // & random data = YYYYYYYYYYYYYYYYYYY @@ -118,7 +121,7 @@ static constexpr uintptr_t LOW_MEM_START = 64 * KB; // Function is standalone so it can be tested somewhat in mem_map_test.cc. #ifdef __BIONIC__ -uintptr_t CreateStartPos(uint64_t input) { +uintptr_t CreateStartPos(uint64_t input, size_t page_size) { CHECK_NE(0, ART_BASE_ADDRESS); // Start with all bits below highest bit in ART_BASE_ADDRESS. @@ -126,26 +129,26 @@ uintptr_t CreateStartPos(uint64_t input) { constexpr uintptr_t mask_ones = (1 << (31 - leading_zeros)) - 1; // Lowest (usually 12) bits are not used, as aligned by page size. - const uintptr_t mask = mask_ones & ~(gPageSize - 1); + const uintptr_t mask = mask_ones & ~(page_size - 1); // Mask input data. return (input & mask) + LOW_MEM_START; } #endif -static uintptr_t GenerateNextMemPos() { +static uintptr_t GenerateNextMemPos(size_t page_size) { #ifdef __BIONIC__ uint64_t random_data; arc4random_buf(&random_data, sizeof(random_data)); - return CreateStartPos(random_data); + return CreateStartPos(random_data, page_size); #else + UNUSED(page_size); // No arc4random on host, see above. return LOW_MEM_START; #endif } -// Initialize linear scan to random position. -uintptr_t MemMap::next_mem_pos_ = GenerateNextMemPos(); +uintptr_t MemMap::next_mem_pos_; #endif // Return true if the address range is contained in a single memory map by either reading @@ -236,7 +239,7 @@ bool MemMap::CheckReservation(uint8_t* expected_ptr, *error_msg = StringPrintf("Invalid reservation for %s", name); return false; } - DCHECK_ALIGNED_PARAM(reservation.Begin(), gPageSize); + DCHECK_ALIGNED_PARAM(reservation.Begin(), GetPageSize()); if (reservation.Begin() != expected_ptr) { *error_msg = StringPrintf("Bad image reservation start for %s: %p instead of %p", name, @@ -317,7 +320,7 @@ MemMap MemMap::MapAnonymous(const char* name, *error_msg = "Empty MemMap requested."; return Invalid(); } - size_t page_aligned_byte_count = RoundUp(byte_count, gPageSize); + size_t page_aligned_byte_count = RoundUp(byte_count, GetPageSize()); int flags = MAP_PRIVATE | MAP_ANONYMOUS; if (reuse) { @@ -393,15 +396,15 @@ MemMap MemMap::MapAnonymousAligned(const char* name, size_t alignment, /*out=*/std::string* error_msg) { DCHECK(IsPowerOfTwo(alignment)); - DCHECK_GT(alignment, gPageSize); + DCHECK_GT(alignment, GetPageSize()); - // Allocate extra 'alignment - gPageSize' bytes so that the mapping can be aligned. + // Allocate extra 'alignment - GetPageSize()' bytes so that the mapping can be aligned. MemMap ret = MapAnonymous(name, /*addr=*/nullptr, // AlignBy requires the size to be page-aligned, so // rounding it here. It is corrected afterwards with // SetSize after AlignBy. - RoundUp(byte_count, gPageSize) + alignment - gPageSize, + RoundUp(byte_count, GetPageSize()) + alignment - GetPageSize(), prot, low_4gb, /*reuse=*/false, @@ -420,7 +423,7 @@ MemMap MemMap::MapPlaceholder(const char* name, uint8_t* addr, size_t byte_count if (byte_count == 0) { return Invalid(); } - const size_t page_aligned_byte_count = RoundUp(byte_count, gPageSize); + const size_t page_aligned_byte_count = RoundUp(byte_count, GetPageSize()); return MemMap(name, addr, byte_count, addr, page_aligned_byte_count, 0, /* reuse= */ true); } @@ -547,10 +550,10 @@ MemMap MemMap::MapFileAtAddress(uint8_t* expected_ptr, return Invalid(); } // Adjust 'offset' to be page-aligned as required by mmap. - int page_offset = start % gPageSize; + int page_offset = start % GetPageSize(); off_t page_aligned_offset = start - page_offset; // Adjust 'byte_count' to be page-aligned as we will map this anyway. - size_t page_aligned_byte_count = RoundUp(byte_count + page_offset, gPageSize); + size_t page_aligned_byte_count = RoundUp(byte_count + page_offset, GetPageSize()); // The 'expected_ptr' is modified (if specified, ie non-null) to be page aligned to the file but // not necessarily to virtual memory. mmap will page align 'expected' for us. uint8_t* page_aligned_expected = @@ -558,7 +561,7 @@ MemMap MemMap::MapFileAtAddress(uint8_t* expected_ptr, size_t redzone_size = 0; if (kRunningOnMemoryTool && kMemoryToolAddsRedzones && expected_ptr == nullptr) { - redzone_size = gPageSize; + redzone_size = GetPageSize(); page_aligned_byte_count += redzone_size; } @@ -753,10 +756,10 @@ MemMap MemMap::RemapAtEnd(uint8_t* new_end, DCHECK_GE(new_end, Begin()); DCHECK_LE(new_end, End()); DCHECK_LE(begin_ + size_, reinterpret_cast<uint8_t*>(base_begin_) + base_size_); - DCHECK_ALIGNED_PARAM(begin_, gPageSize); - DCHECK_ALIGNED_PARAM(base_begin_, gPageSize); - DCHECK_ALIGNED_PARAM(reinterpret_cast<uint8_t*>(base_begin_) + base_size_, gPageSize); - DCHECK_ALIGNED_PARAM(new_end, gPageSize); + DCHECK_ALIGNED_PARAM(begin_, GetPageSize()); + DCHECK_ALIGNED_PARAM(base_begin_, GetPageSize()); + DCHECK_ALIGNED_PARAM(reinterpret_cast<uint8_t*>(base_begin_) + base_size_, GetPageSize()); + DCHECK_ALIGNED_PARAM(new_end, GetPageSize()); uint8_t* old_end = begin_ + size_; uint8_t* old_base_end = reinterpret_cast<uint8_t*>(base_begin_) + base_size_; uint8_t* new_base_end = new_end; @@ -771,7 +774,7 @@ MemMap MemMap::RemapAtEnd(uint8_t* new_end, uint8_t* tail_base_begin = new_base_end; size_t tail_base_size = old_base_end - new_base_end; DCHECK_EQ(tail_base_begin + tail_base_size, old_base_end); - DCHECK_ALIGNED_PARAM(tail_base_size, gPageSize); + DCHECK_ALIGNED_PARAM(tail_base_size, GetPageSize()); MEMORY_TOOL_MAKE_UNDEFINED(tail_base_begin, tail_base_size); // Note: Do not explicitly unmap the tail region, mmap() with MAP_FIXED automatically @@ -810,7 +813,7 @@ MemMap MemMap::RemapAtEnd(uint8_t* new_end, MemMap MemMap::TakeReservedMemory(size_t byte_count, bool reuse) { uint8_t* begin = Begin(); ReleaseReservedMemory(byte_count); // Performs necessary DCHECK()s on this reservation. - size_t base_size = RoundUp(byte_count, gPageSize); + size_t base_size = RoundUp(byte_count, GetPageSize()); return MemMap(name_, begin, byte_count, begin, base_size, prot_, reuse); } @@ -822,13 +825,13 @@ void MemMap::ReleaseReservedMemory(size_t byte_count) { DCHECK_EQ(redzone_size_, 0u); DCHECK_EQ(begin_, base_begin_); DCHECK_EQ(size_, base_size_); - DCHECK_ALIGNED_PARAM(begin_, gPageSize); - DCHECK_ALIGNED_PARAM(size_, gPageSize); + DCHECK_ALIGNED_PARAM(begin_, GetPageSize()); + DCHECK_ALIGNED_PARAM(size_, GetPageSize()); // Check and round up the `byte_count`. DCHECK_NE(byte_count, 0u); DCHECK_LE(byte_count, size_); - byte_count = RoundUp(byte_count, gPageSize); + byte_count = RoundUp(byte_count, GetPageSize()); if (byte_count == size_) { Invalidate(); @@ -943,7 +946,7 @@ void MemMap::DumpMapsLocked(std::ostream& os, bool terse) { size_t num_gaps = 0; size_t num = 1u; size_t size = map->BaseSize(); - CHECK_ALIGNED_PARAM(size, gPageSize); + CHECK_ALIGNED_PARAM(size, GetPageSize()); void* end = map->BaseEnd(); while (it != maps_end && it->second->GetProtect() == map->GetProtect() && @@ -951,24 +954,24 @@ void MemMap::DumpMapsLocked(std::ostream& os, bool terse) { (it->second->BaseBegin() == end || num_gaps < kMaxGaps)) { if (it->second->BaseBegin() != end) { ++num_gaps; - os << "+0x" << std::hex << (size / gPageSize) << "P"; + os << "+0x" << std::hex << (size / GetPageSize()) << "P"; if (num != 1u) { os << "(" << std::dec << num << ")"; } size_t gap = reinterpret_cast<uintptr_t>(it->second->BaseBegin()) - reinterpret_cast<uintptr_t>(end); - CHECK_ALIGNED_PARAM(gap, gPageSize); - os << "~0x" << std::hex << (gap / gPageSize) << "P"; + CHECK_ALIGNED_PARAM(gap, GetPageSize()); + os << "~0x" << std::hex << (gap / GetPageSize()) << "P"; num = 0u; size = 0u; } - CHECK_ALIGNED_PARAM(it->second->BaseSize(), gPageSize); + CHECK_ALIGNED_PARAM(it->second->BaseSize(), GetPageSize()); ++num; size += it->second->BaseSize(); end = it->second->BaseEnd(); ++it; } - os << "+0x" << std::hex << (size / gPageSize) << "P"; + os << "+0x" << std::hex << (size / GetPageSize()) << "P"; if (num != 1u) { os << "(" << std::dec << num << ")"; } @@ -1009,12 +1012,19 @@ void MemMap::Init() { return; } - CHECK_GE(gPageSize, kMinPageSize); - CHECK_LE(gPageSize, kMaxPageSize); - mem_maps_lock_ = new std::mutex(); // Not for thread safety, but for the annotation that gMaps is GUARDED_BY(mem_maps_lock_). std::lock_guard<std::mutex> mu(*mem_maps_lock_); +#ifdef ART_PAGE_SIZE_AGNOSTIC + page_size_ = GetPageSizeSlow(); +#endif + CHECK_GE(GetPageSize(), kMinPageSize); + CHECK_LE(GetPageSize(), kMaxPageSize); +#if USE_ART_LOW_4G_ALLOCATOR + // Initialize linear scan to random position. + CHECK_EQ(next_mem_pos_, 0u); + next_mem_pos_ = GenerateNextMemPos(GetPageSize()); +#endif DCHECK(gMaps == nullptr); gMaps = new Maps; @@ -1035,6 +1045,9 @@ void MemMap::Shutdown() { delete gMaps; gMaps = nullptr; } +#if USE_ART_LOW_4G_ALLOCATOR + next_mem_pos_ = 0u; +#endif delete mem_maps_lock_; mem_maps_lock_ = nullptr; } @@ -1042,7 +1055,7 @@ void MemMap::Shutdown() { void MemMap::SetSize(size_t new_size) { CHECK_LE(new_size, size_); size_t new_base_size = RoundUp(new_size + static_cast<size_t>(PointerDiff(Begin(), BaseBegin())), - gPageSize); + GetPageSize()); if (new_base_size == base_size_) { size_ = new_size; return; @@ -1071,7 +1084,7 @@ void* MemMap::MapInternalArtLow4GBAllocator(size_t length, bool first_run = true; std::lock_guard<std::mutex> mu(*mem_maps_lock_); - for (uintptr_t ptr = next_mem_pos_; ptr < 4 * GB; ptr += gPageSize) { + for (uintptr_t ptr = next_mem_pos_; ptr < 4 * GB; ptr += GetPageSize()) { // Use gMaps as an optimization to skip over large maps. // Find the first map which is address > ptr. auto it = gMaps->upper_bound(reinterpret_cast<void*>(ptr)); @@ -1080,7 +1093,7 @@ void* MemMap::MapInternalArtLow4GBAllocator(size_t length, --before_it; // Start at the end of the map before the upper bound. ptr = std::max(ptr, reinterpret_cast<uintptr_t>(before_it->second->BaseEnd())); - CHECK_ALIGNED_PARAM(ptr, gPageSize); + CHECK_ALIGNED_PARAM(ptr, GetPageSize()); } while (it != gMaps->end()) { // How much space do we have until the next map? @@ -1091,7 +1104,7 @@ void* MemMap::MapInternalArtLow4GBAllocator(size_t length, } // Otherwise, skip to the end of the map. ptr = reinterpret_cast<uintptr_t>(it->second->BaseEnd()); - CHECK_ALIGNED_PARAM(ptr, gPageSize); + CHECK_ALIGNED_PARAM(ptr, GetPageSize()); ++it; } @@ -1106,7 +1119,7 @@ void* MemMap::MapInternalArtLow4GBAllocator(size_t length, // Not enough memory until 4GB. if (first_run) { // Try another time from the bottom; - ptr = LOW_MEM_START - gPageSize; + ptr = LOW_MEM_START - GetPageSize(); first_run = false; continue; } else { @@ -1119,8 +1132,8 @@ void* MemMap::MapInternalArtLow4GBAllocator(size_t length, // Check pages are free. bool safe = true; - for (tail_ptr = ptr; tail_ptr < ptr + length; tail_ptr += gPageSize) { - if (msync(reinterpret_cast<void*>(tail_ptr), gPageSize, 0) == 0) { + for (tail_ptr = ptr; tail_ptr < ptr + length; tail_ptr += GetPageSize()) { + if (msync(reinterpret_cast<void*>(tail_ptr), GetPageSize(), 0) == 0) { safe = false; break; } else { @@ -1176,7 +1189,7 @@ void* MemMap::MapInternal(void* addr, #else UNUSED(low_4gb); #endif - DCHECK_ALIGNED_PARAM(length, gPageSize); + DCHECK_ALIGNED_PARAM(length, GetPageSize()); // TODO: // A page allocator would be a useful abstraction here, as // 1) It is doubtful that MAP_32BIT on x86_64 is doing the right job for us @@ -1235,11 +1248,11 @@ void MemMap::TryReadable() { CHECK_NE(prot_ & PROT_READ, 0); volatile uint8_t* begin = reinterpret_cast<volatile uint8_t*>(base_begin_); volatile uint8_t* end = begin + base_size_; - DCHECK(IsAlignedParam(begin, gPageSize)); - DCHECK(IsAlignedParam(end, gPageSize)); + DCHECK(IsAlignedParam(begin, GetPageSize())); + DCHECK(IsAlignedParam(end, GetPageSize())); // Read the first byte of each page. Use volatile to prevent the compiler from optimizing away the // reads. - for (volatile uint8_t* ptr = begin; ptr < end; ptr += gPageSize) { + for (volatile uint8_t* ptr = begin; ptr < end; ptr += GetPageSize()) { // This read could fault if protection wasn't set correctly. uint8_t value = *ptr; UNUSED(value); @@ -1251,9 +1264,9 @@ static void inline RawClearMemory(uint8_t* begin, uint8_t* end) { } #if defined(__linux__) -static inline void ClearMemory(uint8_t* page_begin, size_t size, bool resident) { - DCHECK(IsAlignedParam(page_begin, gPageSize)); - DCHECK(IsAlignedParam(page_begin + size, gPageSize)); +static inline void ClearMemory(uint8_t* page_begin, size_t size, bool resident, size_t page_size) { + DCHECK(IsAlignedParam(page_begin, page_size)); + DCHECK(IsAlignedParam(page_begin + size, page_size)); if (resident) { RawClearMemory(page_begin, page_begin + size); // Note we check madvise return value against -1, as it seems old kernels @@ -1275,8 +1288,8 @@ void ZeroMemory(void* address, size_t length, bool release_eagerly) { } uint8_t* const mem_begin = reinterpret_cast<uint8_t*>(address); uint8_t* const mem_end = mem_begin + length; - uint8_t* const page_begin = AlignUp(mem_begin, gPageSize); - uint8_t* const page_end = AlignDown(mem_end, gPageSize); + uint8_t* const page_begin = AlignUp(mem_begin, MemMap::GetPageSize()); + uint8_t* const page_end = AlignDown(mem_end, MemMap::GetPageSize()); if (!kMadviseZeroes || page_begin >= page_end) { // No possible area to madvise. RawClearMemory(mem_begin, mem_end); @@ -1296,24 +1309,24 @@ void ZeroMemory(void* address, size_t length, bool release_eagerly) { // mincore() is linux-specific syscall. #if defined(__linux__) if (!release_eagerly) { - size_t vec_len = (page_end - page_begin) / gPageSize; + size_t vec_len = (page_end - page_begin) / MemMap::GetPageSize(); std::unique_ptr<unsigned char[]> vec(new unsigned char[vec_len]); if (mincore(page_begin, page_end - page_begin, vec.get()) == 0) { uint8_t* current_page = page_begin; - size_t current_size = gPageSize; + size_t current_size = MemMap::GetPageSize(); uint32_t old_state = vec[0] & 0x1; for (size_t i = 1; i < vec_len; ++i) { uint32_t new_state = vec[i] & 0x1; if (old_state == new_state) { - current_size += gPageSize; + current_size += MemMap::GetPageSize(); } else { - ClearMemory(current_page, current_size, old_state); + ClearMemory(current_page, current_size, old_state, MemMap::GetPageSize()); current_page = current_page + current_size; - current_size = gPageSize; + current_size = MemMap::GetPageSize(); old_state = new_state; } } - ClearMemory(current_page, current_size, old_state); + ClearMemory(current_page, current_size, old_state, MemMap::GetPageSize()); return; } static bool logged_about_mincore = false; @@ -1334,8 +1347,8 @@ void ZeroMemory(void* address, size_t length, bool release_eagerly) { void MemMap::AlignBy(size_t alignment, bool align_both_ends) { CHECK_EQ(begin_, base_begin_) << "Unsupported"; CHECK_EQ(size_, base_size_) << "Unsupported"; - CHECK_GT(alignment, static_cast<size_t>(gPageSize)); - CHECK_ALIGNED_PARAM(alignment, gPageSize); + CHECK_GT(alignment, static_cast<size_t>(GetPageSize())); + CHECK_ALIGNED_PARAM(alignment, GetPageSize()); CHECK(!reuse_); if (IsAlignedParam(reinterpret_cast<uintptr_t>(base_begin_), alignment) && (!align_both_ends || IsAlignedParam(base_size_, alignment))) { diff --git a/libartbase/base/mem_map.h b/libartbase/base/mem_map.h index 4321b4304d..90e2031cf9 100644 --- a/libartbase/base/mem_map.h +++ b/libartbase/base/mem_map.h @@ -346,6 +346,17 @@ class MemMap { // 'redzone_size_ == 0' indicates that we are not using memory-tool on this mapping. size_t GetRedzoneSize() const { return redzone_size_; } +#ifdef ART_PAGE_SIZE_AGNOSTIC + static inline size_t GetPageSize() { + DCHECK_NE(page_size_, 0u); + return page_size_; + } +#else + static constexpr size_t GetPageSize() { + return GetPageSizeSlow(); + } +#endif + private: MemMap(const std::string& name, uint8_t* begin, @@ -435,6 +446,10 @@ class MemMap { static std::mutex* mem_maps_lock_; +#ifdef ART_PAGE_SIZE_AGNOSTIC + static size_t page_size_; +#endif + friend class MemMapTest; // To allow access to base_begin_ and base_size_. }; diff --git a/libartbase/base/mem_map_test.cc b/libartbase/base/mem_map_test.cc index 73a5be7d83..37f5b5a9fe 100644 --- a/libartbase/base/mem_map_test.cc +++ b/libartbase/base/mem_map_test.cc @@ -64,7 +64,7 @@ class MemMapTest : public CommonArtTest { static void RemapAtEndTest(bool low_4gb) { std::string error_msg; // Cast the page size to size_t. - const size_t page_size = static_cast<size_t>(gPageSize); + const size_t page_size = MemMap::GetPageSize(); // Map a two-page memory region. MemMap m0 = MemMap::MapAnonymous("MemMapTest_RemapAtEndTest_map0", 2 * page_size, @@ -134,25 +134,31 @@ class MemMapTest : public CommonArtTest { #if defined(__LP64__) && !defined(__x86_64__) #ifdef __BIONIC__ -extern uintptr_t CreateStartPos(uint64_t input); +extern uintptr_t CreateStartPos(uint64_t input, uint64_t page_size); #endif +TEST_F(MemMapTest, PageSize) { + const size_t page_size = MemMap::GetPageSize(); + EXPECT_EQ(page_size, GetPageSizeSlow()); +} + TEST_F(MemMapTest, Start) { CommonInit(); uintptr_t start = GetLinearScanPos(); EXPECT_LE(64 * KB, start); EXPECT_LT(start, static_cast<uintptr_t>(ART_BASE_ADDRESS)); #ifdef __BIONIC__ + const size_t page_size = MemMap::GetPageSize(); // Test a couple of values. Make sure they are different. uintptr_t last = 0; for (size_t i = 0; i < 100; ++i) { - uintptr_t random_start = CreateStartPos(i * gPageSize); + uintptr_t random_start = CreateStartPos(i * page_size, page_size); EXPECT_NE(last, random_start); last = random_start; } // Even on max, should be below ART_BASE_ADDRESS. - EXPECT_LT(CreateStartPos(~0), static_cast<uintptr_t>(ART_BASE_ADDRESS)); + EXPECT_LT(CreateStartPos(~0, page_size), static_cast<uintptr_t>(ART_BASE_ADDRESS)); #endif // End of test. } @@ -161,15 +167,16 @@ TEST_F(MemMapTest, Start) { // We need mremap to be able to test ReplaceMapping at all #if HAVE_MREMAP_SYSCALL TEST_F(MemMapTest, ReplaceMapping_SameSize) { + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; MemMap dest = MemMap::MapAnonymous("MapAnonymousEmpty-atomic-replace-dest", - gPageSize, + page_size, PROT_READ, /*low_4gb=*/ false, &error_msg); ASSERT_TRUE(dest.IsValid()); MemMap source = MemMap::MapAnonymous("MapAnonymous-atomic-replace-source", - gPageSize, + page_size, PROT_WRITE | PROT_READ, /*low_4gb=*/ false, &error_msg); @@ -179,7 +186,7 @@ TEST_F(MemMapTest, ReplaceMapping_SameSize) { ASSERT_TRUE(IsAddressMapped(source_addr)); ASSERT_TRUE(IsAddressMapped(dest_addr)); - std::vector<uint8_t> data = RandomData(gPageSize); + std::vector<uint8_t> data = RandomData(page_size); memcpy(source.Begin(), data.data(), data.size()); ASSERT_TRUE(dest.ReplaceWith(&source, &error_msg)) << error_msg; @@ -188,15 +195,16 @@ TEST_F(MemMapTest, ReplaceMapping_SameSize) { ASSERT_TRUE(IsAddressMapped(dest_addr)); ASSERT_FALSE(source.IsValid()); - ASSERT_EQ(dest.Size(), static_cast<size_t>(gPageSize)); + ASSERT_EQ(dest.Size(), static_cast<size_t>(page_size)); ASSERT_EQ(memcmp(dest.Begin(), data.data(), dest.Size()), 0); } TEST_F(MemMapTest, ReplaceMapping_MakeLarger) { + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; MemMap dest = MemMap::MapAnonymous("MapAnonymousEmpty-atomic-replace-dest", - 5 * gPageSize, // Need to make it larger + 5 * page_size, // Need to make it larger // initially so we know // there won't be mappings // in the way when we move @@ -206,7 +214,7 @@ TEST_F(MemMapTest, ReplaceMapping_MakeLarger) { &error_msg); ASSERT_TRUE(dest.IsValid()); MemMap source = MemMap::MapAnonymous("MapAnonymous-atomic-replace-source", - 3 * gPageSize, + 3 * page_size, PROT_WRITE | PROT_READ, /*low_4gb=*/ false, &error_msg); @@ -216,37 +224,38 @@ TEST_F(MemMapTest, ReplaceMapping_MakeLarger) { ASSERT_TRUE(IsAddressMapped(source_addr)); // Fill the source with random data. - std::vector<uint8_t> data = RandomData(3 * gPageSize); + std::vector<uint8_t> data = RandomData(3 * page_size); memcpy(source.Begin(), data.data(), data.size()); // Make the dest smaller so that we know we'll have space. - dest.SetSize(gPageSize); + dest.SetSize(page_size); ASSERT_TRUE(IsAddressMapped(dest_addr)); - ASSERT_FALSE(IsAddressMapped(dest_addr + 2 * gPageSize)); - ASSERT_EQ(dest.Size(), static_cast<size_t>(gPageSize)); + ASSERT_FALSE(IsAddressMapped(dest_addr + 2 * page_size)); + ASSERT_EQ(dest.Size(), static_cast<size_t>(page_size)); ASSERT_TRUE(dest.ReplaceWith(&source, &error_msg)) << error_msg; ASSERT_FALSE(IsAddressMapped(source_addr)); - ASSERT_EQ(dest.Size(), static_cast<size_t>(3 * gPageSize)); + ASSERT_EQ(dest.Size(), static_cast<size_t>(3 * page_size)); ASSERT_TRUE(IsAddressMapped(dest_addr)); - ASSERT_TRUE(IsAddressMapped(dest_addr + 2 * gPageSize)); + ASSERT_TRUE(IsAddressMapped(dest_addr + 2 * page_size)); ASSERT_FALSE(source.IsValid()); ASSERT_EQ(memcmp(dest.Begin(), data.data(), dest.Size()), 0); } TEST_F(MemMapTest, ReplaceMapping_MakeSmaller) { + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; MemMap dest = MemMap::MapAnonymous("MapAnonymousEmpty-atomic-replace-dest", - 3 * gPageSize, + 3 * page_size, PROT_READ, /*low_4gb=*/ false, &error_msg); ASSERT_TRUE(dest.IsValid()); MemMap source = MemMap::MapAnonymous("MapAnonymous-atomic-replace-source", - gPageSize, + page_size, PROT_WRITE | PROT_READ, /*low_4gb=*/ false, &error_msg); @@ -255,59 +264,60 @@ TEST_F(MemMapTest, ReplaceMapping_MakeSmaller) { uint8_t* dest_addr = dest.Begin(); ASSERT_TRUE(IsAddressMapped(source_addr)); ASSERT_TRUE(IsAddressMapped(dest_addr)); - ASSERT_TRUE(IsAddressMapped(dest_addr + 2 * gPageSize)); - ASSERT_EQ(dest.Size(), static_cast<size_t>(3 * gPageSize)); + ASSERT_TRUE(IsAddressMapped(dest_addr + 2 * page_size)); + ASSERT_EQ(dest.Size(), static_cast<size_t>(3 * page_size)); - std::vector<uint8_t> data = RandomData(gPageSize); - memcpy(source.Begin(), data.data(), gPageSize); + std::vector<uint8_t> data = RandomData(page_size); + memcpy(source.Begin(), data.data(), page_size); ASSERT_TRUE(dest.ReplaceWith(&source, &error_msg)) << error_msg; ASSERT_FALSE(IsAddressMapped(source_addr)); - ASSERT_EQ(dest.Size(), static_cast<size_t>(gPageSize)); + ASSERT_EQ(dest.Size(), static_cast<size_t>(page_size)); ASSERT_TRUE(IsAddressMapped(dest_addr)); - ASSERT_FALSE(IsAddressMapped(dest_addr + 2 * gPageSize)); + ASSERT_FALSE(IsAddressMapped(dest_addr + 2 * page_size)); ASSERT_FALSE(source.IsValid()); ASSERT_EQ(memcmp(dest.Begin(), data.data(), dest.Size()), 0); } TEST_F(MemMapTest, ReplaceMapping_FailureOverlap) { + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; MemMap dest = MemMap::MapAnonymous( "MapAnonymousEmpty-atomic-replace-dest", - 3 * gPageSize, // Need to make it larger initially so we know there won't be mappings in + 3 * page_size, // Need to make it larger initially so we know there won't be mappings in // the way when we move source. PROT_READ | PROT_WRITE, /*low_4gb=*/ false, &error_msg); ASSERT_TRUE(dest.IsValid()); // Resize down to 1 page so we can remap the rest. - dest.SetSize(gPageSize); + dest.SetSize(page_size); // Create source from the last 2 pages MemMap source = MemMap::MapAnonymous("MapAnonymous-atomic-replace-source", - dest.Begin() + gPageSize, - 2 * gPageSize, + dest.Begin() + page_size, + 2 * page_size, PROT_WRITE | PROT_READ, /*low_4gb=*/ false, /*reuse=*/ false, /*reservation=*/ nullptr, &error_msg); ASSERT_TRUE(source.IsValid()); - ASSERT_EQ(dest.Begin() + gPageSize, source.Begin()); + ASSERT_EQ(dest.Begin() + page_size, source.Begin()); uint8_t* source_addr = source.Begin(); uint8_t* dest_addr = dest.Begin(); ASSERT_TRUE(IsAddressMapped(source_addr)); // Fill the source and dest with random data. - std::vector<uint8_t> data = RandomData(2 * gPageSize); + std::vector<uint8_t> data = RandomData(2 * page_size); memcpy(source.Begin(), data.data(), data.size()); - std::vector<uint8_t> dest_data = RandomData(gPageSize); + std::vector<uint8_t> dest_data = RandomData(page_size); memcpy(dest.Begin(), dest_data.data(), dest_data.size()); ASSERT_TRUE(IsAddressMapped(dest_addr)); - ASSERT_EQ(dest.Size(), static_cast<size_t>(gPageSize)); + ASSERT_EQ(dest.Size(), static_cast<size_t>(page_size)); ASSERT_FALSE(dest.ReplaceWith(&source, &error_msg)) << error_msg; @@ -323,6 +333,7 @@ TEST_F(MemMapTest, ReplaceMapping_FailureOverlap) { TEST_F(MemMapTest, MapAnonymousEmpty) { CommonInit(); + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; MemMap map = MemMap::MapAnonymous("MapAnonymousEmpty", /*byte_count=*/ 0, @@ -334,7 +345,7 @@ TEST_F(MemMapTest, MapAnonymousEmpty) { error_msg.clear(); map = MemMap::MapAnonymous("MapAnonymousNonEmpty", - gPageSize, + page_size, PROT_READ | PROT_WRITE, /*low_4gb=*/ false, &error_msg); @@ -344,9 +355,10 @@ TEST_F(MemMapTest, MapAnonymousEmpty) { TEST_F(MemMapTest, MapAnonymousFailNullError) { CommonInit(); + const size_t page_size = MemMap::GetPageSize(); // Test that we don't crash with a null error_str when mapping at an invalid location. MemMap map = MemMap::MapAnonymous("MapAnonymousInvalid", - reinterpret_cast<uint8_t*>(static_cast<size_t>(gPageSize)), + reinterpret_cast<uint8_t*>(static_cast<size_t>(page_size)), 0x20000, PROT_READ | PROT_WRITE, /*low_4gb=*/ false, @@ -359,6 +371,7 @@ TEST_F(MemMapTest, MapAnonymousFailNullError) { #ifdef __LP64__ TEST_F(MemMapTest, MapAnonymousEmpty32bit) { CommonInit(); + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; MemMap map = MemMap::MapAnonymous("MapAnonymousEmpty", /*byte_count=*/ 0, @@ -370,7 +383,7 @@ TEST_F(MemMapTest, MapAnonymousEmpty32bit) { error_msg.clear(); map = MemMap::MapAnonymous("MapAnonymousNonEmpty", - gPageSize, + page_size, PROT_READ | PROT_WRITE, /*low_4gb=*/ true, &error_msg); @@ -382,7 +395,7 @@ TEST_F(MemMapTest, MapFile32Bit) { CommonInit(); std::string error_msg; ScratchFile scratch_file; - const size_t map_size = gPageSize; + const size_t map_size = MemMap::GetPageSize(); std::unique_ptr<uint8_t[]> data(new uint8_t[map_size]()); ASSERT_TRUE(scratch_file.GetFile()->WriteFully(&data[0], map_size)); MemMap map = MemMap::MapFile(/*byte_count=*/map_size, @@ -408,13 +421,14 @@ TEST_F(MemMapTest, MapAnonymousExactAddr) { TEST_DISABLED_FOR_MEMORY_TOOL(); CommonInit(); + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; // Find a valid address. - uint8_t* valid_address = GetValidMapAddress(gPageSize, /*low_4gb=*/false); + uint8_t* valid_address = GetValidMapAddress(page_size, /*low_4gb=*/false); // Map at an address that should work, which should succeed. MemMap map0 = MemMap::MapAnonymous("MapAnonymous0", valid_address, - gPageSize, + page_size, PROT_READ | PROT_WRITE, /*low_4gb=*/ false, /*reuse=*/ false, @@ -425,7 +439,7 @@ TEST_F(MemMapTest, MapAnonymousExactAddr) { ASSERT_TRUE(map0.BaseBegin() == valid_address); // Map at an unspecified address, which should succeed. MemMap map1 = MemMap::MapAnonymous("MapAnonymous1", - gPageSize, + page_size, PROT_READ | PROT_WRITE, /*low_4gb=*/ false, &error_msg); @@ -435,7 +449,7 @@ TEST_F(MemMapTest, MapAnonymousExactAddr) { // Attempt to map at the same address, which should fail. MemMap map2 = MemMap::MapAnonymous("MapAnonymous2", reinterpret_cast<uint8_t*>(map1.BaseBegin()), - gPageSize, + page_size, PROT_READ | PROT_WRITE, /*low_4gb=*/ false, /*reuse=*/ false, @@ -457,16 +471,17 @@ TEST_F(MemMapTest, RemapAtEnd32bit) { TEST_F(MemMapTest, RemapFileViewAtEnd) { CommonInit(); + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; ScratchFile scratch_file; // Create a scratch file 3 pages large. - const size_t map_size = 3 * gPageSize; + const size_t map_size = 3 * page_size; std::unique_ptr<uint8_t[]> data(new uint8_t[map_size]()); - memset(data.get(), 1, gPageSize); - memset(&data[0], 0x55, gPageSize); - memset(&data[gPageSize], 0x5a, gPageSize); - memset(&data[2 * gPageSize], 0xaa, gPageSize); + memset(data.get(), 1, page_size); + memset(&data[0], 0x55, page_size); + memset(&data[page_size], 0x5a, page_size); + memset(&data[2 * page_size], 0xaa, page_size); ASSERT_TRUE(scratch_file.GetFile()->WriteFully(&data[0], map_size)); MemMap map = MemMap::MapFile(/*byte_count=*/map_size, @@ -482,10 +497,10 @@ TEST_F(MemMapTest, RemapFileViewAtEnd) { ASSERT_EQ(map.Size(), map_size); ASSERT_LT(reinterpret_cast<uintptr_t>(map.BaseBegin()), 1ULL << 32); ASSERT_EQ(data[0], *map.Begin()); - ASSERT_EQ(data[gPageSize], *(map.Begin() + gPageSize)); - ASSERT_EQ(data[2 * gPageSize], *(map.Begin() + 2 * gPageSize)); + ASSERT_EQ(data[page_size], *(map.Begin() + page_size)); + ASSERT_EQ(data[2 * page_size], *(map.Begin() + 2 * page_size)); - for (size_t offset = 2 * gPageSize; offset > 0; offset -= gPageSize) { + for (size_t offset = 2 * page_size; offset > 0; offset -= page_size) { MemMap tail = map.RemapAtEnd(map.Begin() + offset, "bad_offset_map", PROT_READ, @@ -496,7 +511,7 @@ TEST_F(MemMapTest, RemapFileViewAtEnd) { ASSERT_TRUE(tail.IsValid()) << error_msg; ASSERT_TRUE(error_msg.empty()); ASSERT_EQ(offset, map.Size()); - ASSERT_EQ(static_cast<size_t>(gPageSize), tail.Size()); + ASSERT_EQ(static_cast<size_t>(page_size), tail.Size()); ASSERT_EQ(tail.Begin(), map.Begin() + map.Size()); ASSERT_EQ(data[offset], *tail.Begin()); } @@ -534,12 +549,13 @@ TEST_F(MemMapTest, MapAnonymousExactAddr32bitHighAddr) { TEST_F(MemMapTest, MapAnonymousOverflow) { CommonInit(); + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; uintptr_t ptr = 0; - ptr -= gPageSize; // Now it's close to the top. + ptr -= page_size; // Now it's close to the top. MemMap map = MemMap::MapAnonymous("MapAnonymousOverflow", reinterpret_cast<uint8_t*>(ptr), - 2 * gPageSize, // brings it over the top. + 2 * page_size, // brings it over the top. PROT_READ | PROT_WRITE, /*low_4gb=*/ false, /*reuse=*/ false, @@ -552,11 +568,12 @@ TEST_F(MemMapTest, MapAnonymousOverflow) { #ifdef __LP64__ TEST_F(MemMapTest, MapAnonymousLow4GBExpectedTooHigh) { CommonInit(); + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; MemMap map = MemMap::MapAnonymous("MapAnonymousLow4GBExpectedTooHigh", reinterpret_cast<uint8_t*>(UINT64_C(0x100000000)), - gPageSize, + page_size, PROT_READ | PROT_WRITE, /*low_4gb=*/ true, /*reuse=*/ false, @@ -606,11 +623,12 @@ TEST_F(MemMapTest, MapAnonymousReuse) { TEST_F(MemMapTest, CheckNoGaps) { CommonInit(); + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; constexpr size_t kNumPages = 3; // Map a 3-page mem map. MemMap reservation = MemMap::MapAnonymous("MapAnonymous0", - gPageSize * kNumPages, + page_size * kNumPages, PROT_READ | PROT_WRITE, /*low_4gb=*/ false, &error_msg); @@ -621,7 +639,7 @@ TEST_F(MemMapTest, CheckNoGaps) { // Map at the same address, taking from the `map` reservation. MemMap map0 = MemMap::MapAnonymous("MapAnonymous0", - gPageSize, + page_size, PROT_READ | PROT_WRITE, /*low_4gb=*/ false, &reservation, @@ -630,23 +648,23 @@ TEST_F(MemMapTest, CheckNoGaps) { ASSERT_TRUE(error_msg.empty()); ASSERT_EQ(map_base, map0.Begin()); MemMap map1 = MemMap::MapAnonymous("MapAnonymous1", - gPageSize, + page_size, PROT_READ | PROT_WRITE, /*low_4gb=*/ false, &reservation, &error_msg); ASSERT_TRUE(map1.IsValid()) << error_msg; ASSERT_TRUE(error_msg.empty()); - ASSERT_EQ(map_base + gPageSize, map1.Begin()); + ASSERT_EQ(map_base + page_size, map1.Begin()); MemMap map2 = MemMap::MapAnonymous("MapAnonymous2", - gPageSize, + page_size, PROT_READ | PROT_WRITE, /*low_4gb=*/ false, &reservation, &error_msg); ASSERT_TRUE(map2.IsValid()) << error_msg; ASSERT_TRUE(error_msg.empty()); - ASSERT_EQ(map_base + 2 * gPageSize, map2.Begin()); + ASSERT_EQ(map_base + 2 * page_size, map2.Begin()); ASSERT_FALSE(reservation.IsValid()); // The entire reservation was used. // One-map cases. @@ -668,9 +686,8 @@ TEST_F(MemMapTest, CheckNoGaps) { TEST_F(MemMapTest, AlignBy) { CommonInit(); + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; - // Cast the page size to size_t. - const size_t page_size = static_cast<size_t>(gPageSize); // Map a region. MemMap m0 = MemMap::MapAnonymous("MemMapTest_AlignByTest_map0", 14 * page_size, @@ -771,9 +788,10 @@ TEST_F(MemMapTest, AlignBy) { TEST_F(MemMapTest, Reservation) { CommonInit(); + const size_t page_size = MemMap::GetPageSize(); std::string error_msg; ScratchFile scratch_file; - const size_t map_size = 5 * gPageSize; + const size_t map_size = 5 * page_size; std::unique_ptr<uint8_t[]> data(new uint8_t[map_size]()); ASSERT_TRUE(scratch_file.GetFile()->WriteFully(&data[0], map_size)); @@ -786,7 +804,7 @@ TEST_F(MemMapTest, Reservation) { ASSERT_TRUE(error_msg.empty()); // Map first part of the reservation. - const size_t chunk1_size = gPageSize - 1u; + const size_t chunk1_size = page_size - 1u; ASSERT_LT(chunk1_size, map_size) << "We want to split the reservation."; uint8_t* addr1 = reservation.Begin(); MemMap map1 = MemMap::MapFileAtAddress(addr1, @@ -810,7 +828,7 @@ TEST_F(MemMapTest, Reservation) { ASSERT_EQ(map1.BaseEnd(), reservation.Begin()); // Map second part as an anonymous mapping. - const size_t chunk2_size = 2 * gPageSize; + const size_t chunk2_size = 2 * page_size; DCHECK_LT(chunk2_size, reservation.Size()); // We want to split the reservation. uint8_t* addr2 = reservation.Begin(); MemMap map2 = MemMap::MapAnonymous("MiddleReservation", @@ -850,7 +868,7 @@ TEST_F(MemMapTest, Reservation) { ASSERT_FALSE(reservation.IsValid()); // Now split the MiddleReservation. - const size_t chunk2a_size = gPageSize - 1u; + const size_t chunk2a_size = page_size - 1u; DCHECK_LT(chunk2a_size, map2.Size()); // We want to split the reservation. MemMap map2a = map2.TakeReservedMemory(chunk2a_size); ASSERT_TRUE(map2a.IsValid()) << error_msg; diff --git a/libartbase/base/unix_file/fd_file.cc b/libartbase/base/unix_file/fd_file.cc index 5dc491138b..c5307e249b 100644 --- a/libartbase/base/unix_file/fd_file.cc +++ b/libartbase/base/unix_file/fd_file.cc @@ -498,8 +498,8 @@ bool FdFile::Copy(FdFile* input_file, int64_t offset, int64_t size) { if (lseek(input_file->Fd(), off, SEEK_SET) != off) { return false; } - const size_t max_buffer_size = 4 * ::art::gPageSize; - const size_t buffer_size = std::min<uint64_t>(size, max_buffer_size); + constexpr size_t kMaxBufferSize = 16 * ::art::KB; + const size_t buffer_size = std::min<uint64_t>(size, kMaxBufferSize); art::UniqueCPtr<void> buffer(malloc(buffer_size)); if (buffer == nullptr) { errno = ENOMEM; diff --git a/libartbase/base/utils.cc b/libartbase/base/utils.cc index 75320005fd..bc359db173 100644 --- a/libartbase/base/utils.cc +++ b/libartbase/base/utils.cc @@ -31,6 +31,7 @@ #include "android-base/stringprintf.h" #include "android-base/strings.h" +#include "base/mem_map.h" #include "base/stl_util.h" #include "bit_utils.h" #include "os.h" @@ -89,9 +90,10 @@ int CacheFlush(uintptr_t start, uintptr_t limit) { return r; } -bool TouchAndFlushCacheLinesWithinPage(uintptr_t start, uintptr_t limit, size_t attempts) { +bool TouchAndFlushCacheLinesWithinPage(uintptr_t start, uintptr_t limit, size_t attempts, + size_t page_size) { CHECK_LT(start, limit); - CHECK_EQ(RoundDown(start, gPageSize), RoundDown(limit - 1, gPageSize)) << "range spans pages"; + CHECK_EQ(RoundDown(start, page_size), RoundDown(limit - 1, page_size)) << "range spans pages"; // Declare a volatile variable so the compiler does not elide reads from the page being touched. [[maybe_unused]] volatile uint8_t v = 0; for (size_t i = 0; i < attempts; ++i) { @@ -130,6 +132,8 @@ bool FlushCpuCaches(void* begin, void* end) { // (2) fault handling that allows flushing/invalidation to continue after // a missing page has been faulted in. + const size_t page_size = MemMap::GetPageSize(); + uintptr_t start = reinterpret_cast<uintptr_t>(begin); const uintptr_t limit = reinterpret_cast<uintptr_t>(end); if (LIKELY(CacheFlush(start, limit) == 0)) { @@ -139,14 +143,14 @@ bool FlushCpuCaches(void* begin, void* end) { // A rare failure has occurred implying that part of the range (begin, end] has been swapped // out. Retry flushing but this time grouping cache-line flushes on individual pages and // touching each page before flushing. - uintptr_t next_page = RoundUp(start + 1, gPageSize); + uintptr_t next_page = RoundUp(start + 1, page_size); while (start < limit) { uintptr_t boundary = std::min(next_page, limit); - if (!TouchAndFlushCacheLinesWithinPage(start, boundary, kMaxFlushAttempts)) { + if (!TouchAndFlushCacheLinesWithinPage(start, boundary, kMaxFlushAttempts, page_size)) { return false; } start = boundary; - next_page += gPageSize; + next_page += page_size; } return true; } @@ -366,8 +370,9 @@ bool IsAddressKnownBackedByFileOrShared(const void* addr) { // We use the Linux pagemap interface for knowing if an address is backed // by a file or is shared. See: // https://www.kernel.org/doc/Documentation/vm/pagemap.txt - uintptr_t vmstart = reinterpret_cast<uintptr_t>(AlignDown(addr, gPageSize)); - off_t index = (vmstart / gPageSize) * sizeof(uint64_t); + const size_t page_size = MemMap::GetPageSize(); + uintptr_t vmstart = reinterpret_cast<uintptr_t>(AlignDown(addr, page_size)); + off_t index = (vmstart / page_size) * sizeof(uint64_t); android::base::unique_fd pagemap(open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC)); if (pagemap == -1) { return false; diff --git a/libdexfile/dex/code_item_accessors_test.cc b/libdexfile/dex/code_item_accessors_test.cc index c545efb2a9..a815d06cb8 100644 --- a/libdexfile/dex/code_item_accessors_test.cc +++ b/libdexfile/dex/code_item_accessors_test.cc @@ -20,6 +20,7 @@ #include <memory> #include <vector> +#include "base/mem_map.h" #include "dex_file_loader.h" #include "gtest/gtest.h" @@ -28,7 +29,7 @@ namespace art { class CodeItemAccessorsTest : public testing::Test {}; std::unique_ptr<const DexFile> CreateFakeDex(bool compact_dex, std::vector<uint8_t>* data) { - data->resize(gPageSize); + data->resize(MemMap::GetPageSize()); if (compact_dex) { CompactDexFile::Header* header = const_cast<CompactDexFile::Header*>(CompactDexFile::Header::At(data->data())); diff --git a/libelffile/elf/xz_utils.cc b/libelffile/elf/xz_utils.cc index 0ea216274d..4ec184a9ef 100644 --- a/libelffile/elf/xz_utils.cc +++ b/libelffile/elf/xz_utils.cc @@ -22,6 +22,7 @@ #include "base/array_ref.h" #include "base/bit_utils.h" #include "base/leb128.h" +#include "base/mem_map.h" #include "dwarf/writer.h" // liblzma. @@ -98,6 +99,8 @@ void XzCompress(ArrayRef<const uint8_t> src, } void XzDecompress(ArrayRef<const uint8_t> src, std::vector<uint8_t>* dst) { + const size_t page_size = MemMap::GetPageSize(); + XzInitCrc(); std::unique_ptr<CXzUnpacker> state(new CXzUnpacker()); ISzAlloc alloc; @@ -109,7 +112,7 @@ void XzDecompress(ArrayRef<const uint8_t> src, std::vector<uint8_t>* dst) { size_t dst_offset = 0; ECoderStatus status; do { - dst->resize(RoundUp(dst_offset + gPageSize / 4, gPageSize)); + dst->resize(RoundUp(dst_offset + page_size / 4, page_size)); size_t src_remaining = src.size() - src_offset; size_t dst_remaining = dst->size() - dst_offset; int return_val = XzUnpacker_Code(state.get(), diff --git a/runtime/base/gc_visited_arena_pool.cc b/runtime/base/gc_visited_arena_pool.cc index 82131189dd..878b9d7770 100644 --- a/runtime/base/gc_visited_arena_pool.cc +++ b/runtime/base/gc_visited_arena_pool.cc @@ -105,8 +105,8 @@ uint8_t* GcVisitedArenaPool::AddMap(size_t min_size) { size = std::max(min_size, kLow4GBLinearAllocPoolSize); } #endif - size_t alignment = BestPageTableAlignment(size); - DCHECK_GE(size, gPMDSize); + size_t alignment = gc::Heap::BestPageTableAlignment(size); + DCHECK_GE(size, gc::Heap::GetPMDSize()); std::string err_msg; maps_.emplace_back(MemMap::MapAnonymousAligned( name_, size, PROT_READ | PROT_WRITE, low_4gb_, alignment, &err_msg)); diff --git a/runtime/base/mem_map_arena_pool.cc b/runtime/base/mem_map_arena_pool.cc index 647115fbf8..c291e22052 100644 --- a/runtime/base/mem_map_arena_pool.cc +++ b/runtime/base/mem_map_arena_pool.cc @@ -28,6 +28,7 @@ #include "base/arena_allocator-inl.h" #include "base/mem_map.h" #include "base/systrace.h" +#include "runtime_globals.h" namespace art { diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc index 6d755aaa26..5033e3f8dd 100644 --- a/runtime/gc/accounting/space_bitmap.cc +++ b/runtime/gc/accounting/space_bitmap.cc @@ -173,7 +173,7 @@ void SpaceBitmap<kAlignment>::ClearRange(const mirror::Object* begin, const mirr const uintptr_t start_index = OffsetToIndex(begin_offset); const uintptr_t end_index = OffsetToIndex(end_offset); ZeroAndReleaseMemory(reinterpret_cast<uint8_t*>(&bitmap_begin_[start_index]), - (end_index - start_index) * sizeof(*bitmap_begin_)); + (end_index - start_index) * sizeof(*bitmap_begin_)); } template<size_t kAlignment> diff --git a/runtime/gc/accounting/space_bitmap_test.cc b/runtime/gc/accounting/space_bitmap_test.cc index 3641fb72ad..0285ba52ec 100644 --- a/runtime/gc/accounting/space_bitmap_test.cc +++ b/runtime/gc/accounting/space_bitmap_test.cc @@ -19,8 +19,8 @@ #include <stdint.h> #include <memory> -#include "base/common_art_test.h" #include "base/mutex.h" +#include "common_runtime_test.h" #include "gc/space/large_object_space.h" #include "runtime_globals.h" #include "space_bitmap-inl.h" @@ -29,8 +29,10 @@ namespace art { namespace gc { namespace accounting { +// SpaceBitmapTest is a CommonRuntimeTest as the test requires runtime to be initialized to enable +// access to space::LargeObjectSpace::ObjectAlignment(). template <typename T> -class SpaceBitmapTest : public CommonArtTest {}; +class SpaceBitmapTest : public CommonRuntimeTest {}; // Main test parameters. For each test case, we pair together a SpaceBitmap // implementation with an object alignment. The object alignment may be larger @@ -38,21 +40,21 @@ class SpaceBitmapTest : public CommonArtTest {}; template <typename T, size_t kAlignment> struct SpaceBitmapTestType { using SpaceBitmap = T; - static const size_t gObjectAlignment = kAlignment; + static constexpr size_t GetObjectAlignment() { + return kAlignment; + } }; -// This is a special case where gObjectAlignment is set to large-object -// alignment at runtime. +// This is a special case where object alignment is chosen to be the large-object +// alignment determined at runtime. template <typename T> struct SpaceBitmapTestPageSizeType { using SpaceBitmap = T; - static const size_t gObjectAlignment; + static size_t GetObjectAlignment() { + return space::LargeObjectSpace::ObjectAlignment(); + } }; -template <typename T> -const size_t SpaceBitmapTestPageSizeType<T>::gObjectAlignment = - space::LargeObjectSpace::ObjectAlignment(); - using SpaceBitmapTestTypes = ::testing::Types<SpaceBitmapTestType<ContinuousSpaceBitmap, kObjectAlignment>, // Large objects are aligned to the OS page size, try @@ -94,7 +96,7 @@ class BitmapVerify { TYPED_TEST(SpaceBitmapTest, ScanRange) { uint8_t* heap_begin = reinterpret_cast<uint8_t*>(0x10000000); size_t heap_capacity = 16 * MB; - const size_t gObjectAlignment = TypeParam::gObjectAlignment; + const size_t gObjectAlignment = TypeParam::GetObjectAlignment(); auto space_bitmap(TypeParam::SpaceBitmap::Create("test bitmap", heap_begin, heap_capacity)); EXPECT_TRUE(space_bitmap.IsValid()); @@ -123,9 +125,10 @@ TYPED_TEST(SpaceBitmapTest, ScanRange) { } TYPED_TEST(SpaceBitmapTest, ClearRange) { + const size_t page_size = MemMap::GetPageSize(); uint8_t* heap_begin = reinterpret_cast<uint8_t*>(0x10000000); size_t heap_capacity = 16 * MB; - const size_t gObjectAlignment = TypeParam::gObjectAlignment; + const size_t gObjectAlignment = TypeParam::GetObjectAlignment(); auto bitmap(TypeParam::SpaceBitmap::Create("test bitmap", heap_begin, heap_capacity)); EXPECT_TRUE(bitmap.IsValid()); @@ -153,7 +156,7 @@ TYPED_TEST(SpaceBitmapTest, ClearRange) { for (uintptr_t i = 0; i < range.first; i += gObjectAlignment) { EXPECT_TRUE(bitmap.Test(reinterpret_cast<mirror::Object*>(heap_begin + i))); } - for (uintptr_t i = range.second; i < range.second + gPageSize; i += gObjectAlignment) { + for (uintptr_t i = range.second; i < range.second + page_size; i += gObjectAlignment) { EXPECT_TRUE(bitmap.Test(reinterpret_cast<mirror::Object*>(heap_begin + i))); } // Everything inside should be cleared. @@ -239,7 +242,7 @@ TYPED_TEST(SpaceBitmapTest, VisitorAlignment) { space_bitmap->VisitMarkedRange(range_begin, range_end, count_fn); EXPECT_EQ(count, manual_count); }; - RunTest<SpaceBitmap>(TypeParam::gObjectAlignment, count_test_fn); + RunTest<SpaceBitmap>(TypeParam::GetObjectAlignment(), count_test_fn); } TYPED_TEST(SpaceBitmapTest, OrderAlignment) { @@ -268,7 +271,7 @@ TYPED_TEST(SpaceBitmapTest, OrderAlignment) { EXPECT_NE(nullptr, last_ptr); } }; - RunTest<SpaceBitmap>(TypeParam::gObjectAlignment, order_test_fn); + RunTest<SpaceBitmap>(TypeParam::GetObjectAlignment(), order_test_fn); } } // namespace accounting diff --git a/runtime/gc/collector/immune_spaces_test.cc b/runtime/gc/collector/immune_spaces_test.cc index 0ea9888179..2337eadf7a 100644 --- a/runtime/gc/collector/immune_spaces_test.cc +++ b/runtime/gc/collector/immune_spaces_test.cc @@ -67,12 +67,14 @@ class ImmuneSpacesTest : public CommonArtTest { ImmuneSpacesTest() {} void ReserveBitmaps() { + const size_t page_size = MemMap::GetPageSize(); + // Create a bunch of fake bitmaps since these are required to create image spaces. The bitmaps // do not need to cover the image spaces though. for (size_t i = 0; i < kMaxBitmaps; ++i) { accounting::ContinuousSpaceBitmap bitmap( accounting::ContinuousSpaceBitmap::Create( - "bitmap", reinterpret_cast<uint8_t*>(static_cast<size_t>(gPageSize)), gPageSize)); + "bitmap", reinterpret_cast<uint8_t*>(static_cast<size_t>(page_size)), page_size)); CHECK(bitmap.IsValid()); live_bitmaps_.push_back(std::move(bitmap)); } @@ -82,8 +84,8 @@ class ImmuneSpacesTest : public CommonArtTest { // If the image is aligned to the current runtime page size, it will already // be naturally aligned. On the other hand, MayAnonymousAligned() requires // that the requested alignment is higher. - DCHECK_LE(gPageSize, kElfSegmentAlignment); - if (gPageSize == kElfSegmentAlignment) { + DCHECK_LE(MemMap::GetPageSize(), kElfSegmentAlignment); + if (MemMap::GetPageSize() == kElfSegmentAlignment) { return MemMap::MapAnonymous("reserve", image_size, PROT_READ | PROT_WRITE, diff --git a/runtime/gc/collector/mark_compact.cc b/runtime/gc/collector/mark_compact.cc index cf80840a1a..ce7c739c8d 100644 --- a/runtime/gc/collector/mark_compact.cc +++ b/runtime/gc/collector/mark_compact.cc @@ -96,12 +96,13 @@ using ::android::base::ParseBoolResult; namespace art { static bool HaveMremapDontunmap() { - void* old = mmap(nullptr, gPageSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED, -1, 0); + const size_t page_size = GetPageSizeSlow(); + void* old = mmap(nullptr, page_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED, -1, 0); CHECK_NE(old, MAP_FAILED); - void* addr = mremap(old, gPageSize, gPageSize, MREMAP_MAYMOVE | MREMAP_DONTUNMAP, nullptr); - CHECK_EQ(munmap(old, gPageSize), 0); + void* addr = mremap(old, page_size, page_size, MREMAP_MAYMOVE | MREMAP_DONTUNMAP, nullptr); + CHECK_EQ(munmap(old, page_size), 0); if (addr != MAP_FAILED) { - CHECK_EQ(munmap(addr, gPageSize), 0); + CHECK_EQ(munmap(addr, page_size), 0); return true; } else { return false; @@ -467,7 +468,7 @@ MarkCompact::MarkCompact(Heap* heap) DCHECK_EQ(total, info_map_.Size()); } - size_t moving_space_alignment = BestPageTableAlignment(moving_space_size); + size_t moving_space_alignment = Heap::BestPageTableAlignment(moving_space_size); // The moving space is created at a fixed address, which is expected to be // PMD-size aligned. if (!IsAlignedParam(bump_pointer_space_->Begin(), moving_space_alignment)) { @@ -550,8 +551,8 @@ MarkCompact::MarkCompact(Heap* heap) void MarkCompact::AddLinearAllocSpaceData(uint8_t* begin, size_t len) { DCHECK_ALIGNED_PARAM(begin, gPageSize); DCHECK_ALIGNED_PARAM(len, gPageSize); - DCHECK_GE(len, gPMDSize); - size_t alignment = BestPageTableAlignment(len); + DCHECK_GE(len, Heap::GetPMDSize()); + size_t alignment = Heap::BestPageTableAlignment(len); bool is_shared = false; // We use MAP_SHARED on non-zygote processes for leveraging userfaultfd's minor-fault feature. if (map_linear_alloc_shared_) { diff --git a/runtime/gc/heap.h b/runtime/gc/heap.h index 06cf842fae..ab7a95f55a 100644 --- a/runtime/gc/heap.h +++ b/runtime/gc/heap.h @@ -1012,6 +1012,25 @@ class Heap { bool AddHeapTask(gc::HeapTask* task); + // TODO: Kernels for arm and x86 in both, 32-bit and 64-bit modes use 512 entries per page-table + // page. Find a way to confirm that in userspace. + // Address range covered by 1 Page Middle Directory (PMD) entry in the page table + static inline ALWAYS_INLINE size_t GetPMDSize() { + return (gPageSize / sizeof(uint64_t)) * gPageSize; + } + // Address range covered by 1 Page Upper Directory (PUD) entry in the page table + static inline ALWAYS_INLINE size_t GetPUDSize() { + return (gPageSize / sizeof(uint64_t)) * GetPMDSize(); + } + + // Returns the ideal alignment corresponding to page-table levels for the + // given size. + static inline size_t BestPageTableAlignment(size_t size) { + const size_t pud_size = GetPUDSize(); + const size_t pmd_size = GetPMDSize(); + return size < pud_size ? pmd_size : pud_size; + } + private: class ConcurrentGCTask; class CollectorTransitionTask; diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc index 3240c66ba1..ef7a742c16 100644 --- a/runtime/jit/jit_code_cache.cc +++ b/runtime/jit/jit_code_cache.cc @@ -258,6 +258,7 @@ JitCodeCache* JitCodeCache::Create(bool used_only_for_profile_data, JitCodeCache::JitCodeCache() : is_weak_access_enabled_(true), inline_cache_cond_("Jit inline cache condition variable", *Locks::jit_lock_), + reserved_capacity_(GetInitialCapacity() * kReservedCapacityMultiplier), zygote_map_(&shared_region_), lock_cond_("Jit code cache condition variable", *Locks::jit_lock_), collection_in_progress_(false), diff --git a/runtime/jit/jit_code_cache.h b/runtime/jit/jit_code_cache.h index 956687af16..8a5c355ba9 100644 --- a/runtime/jit/jit_code_cache.h +++ b/runtime/jit/jit_code_cache.h @@ -185,17 +185,15 @@ class JitCodeCache { // Default initial capacity of the JIT code cache. static size_t GetInitialCapacity() { + // This function is called during static initialization + // when gPageSize might not be available yet. + const size_t page_size = GetPageSizeSlow(); + // Put the default to a very low amount for debug builds to stress the code cache // collection. It should be at least two pages, however, as the storage is split // into data and code sections with sizes that should be aligned to page size each // as that's the unit mspaces use. See also: JitMemoryRegion::Initialize. - return std::max(kIsDebugBuild ? 8 * KB : 64 * KB, 2 * gPageSize); - } - - // Reserved capacity of the JIT code cache. - // By default, do not GC until reaching four times the initial capacity. - static size_t GetReservedCapacity() { - return GetInitialCapacity() * 4; + return std::max(kIsDebugBuild ? 8 * KB : 64 * KB, 2 * page_size); } // Create the code cache with a code + data capacity equal to "capacity", error message is passed @@ -519,6 +517,10 @@ class JitCodeCache { return shared_region_.IsInDataSpace(ptr); } + size_t GetReservedCapacity() { + return reserved_capacity_; + } + bool IsWeakAccessEnabled(Thread* self) const; void WaitUntilInlineCacheAccessible(Thread* self) REQUIRES(!Locks::jit_lock_) @@ -538,6 +540,12 @@ class JitCodeCache { // Condition to wait on for accessing inline caches. ConditionVariable inline_cache_cond_ GUARDED_BY(Locks::jit_lock_); + // Reserved capacity of the JIT code cache. + const size_t reserved_capacity_; + + // By default, do not GC until reaching four times the initial capacity. + static constexpr size_t kReservedCapacityMultiplier = 4; + // -------------- JIT memory regions ------------------------------------- // // Shared region, inherited from the zygote. diff --git a/runtime/jit/jit_memory_region_test.cc b/runtime/jit/jit_memory_region_test.cc index 2f8ae2ca0f..bf943f9981 100644 --- a/runtime/jit/jit_memory_region_test.cc +++ b/runtime/jit/jit_memory_region_test.cc @@ -58,13 +58,13 @@ class TestZygoteMemory : public testing::Test { // Zygote JIT memory only works on kernels that don't segfault on flush. TEST_DISABLED_FOR_KERNELS_WITH_CACHE_SEGFAULT(); std::string error_msg; - size_t size = gPageSize; - android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(size, &error_msg)); + const size_t page_size = GetPageSizeSlow(); + android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(page_size, &error_msg)); CHECK_NE(fd.get(), -1); // Create a writable mapping. int32_t* addr = reinterpret_cast<int32_t*>( - mmap(nullptr, gPageSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0)); + mmap(nullptr, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0)); CHECK(addr != nullptr); CHECK_NE(addr, MAP_FAILED); @@ -82,24 +82,24 @@ class TestZygoteMemory : public testing::Test { // Test that we cannot create another writable mapping. int32_t* addr2 = reinterpret_cast<int32_t*>( - mmap(nullptr, gPageSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0)); + mmap(nullptr, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0)); CHECK_EQ(addr2, MAP_FAILED); // With the existing mapping, we can toggle read/write. - CHECK_EQ(mprotect(addr, size, PROT_READ), 0) << strerror(errno); - CHECK_EQ(mprotect(addr, size, PROT_READ | PROT_WRITE), 0) << strerror(errno); + CHECK_EQ(mprotect(addr, page_size, PROT_READ), 0) << strerror(errno); + CHECK_EQ(mprotect(addr, page_size, PROT_READ | PROT_WRITE), 0) << strerror(errno); // Test mremap with old_size = 0. From the man pages: // If the value of old_size is zero, and old_address refers to a shareable mapping // (see mmap(2) MAP_SHARED), then mremap() will create a new mapping of the same pages. - addr2 = reinterpret_cast<int32_t*>(mremap(addr, 0, gPageSize, MREMAP_MAYMOVE)); + addr2 = reinterpret_cast<int32_t*>(mremap(addr, 0, page_size, MREMAP_MAYMOVE)); CHECK_NE(addr2, MAP_FAILED); // Test that we can write into the remapped mapping. addr2[0] = 3; CHECK_EQ(addr2[0], 3); - addr2 = reinterpret_cast<int32_t*>(mremap(addr, gPageSize, 2 * gPageSize, MREMAP_MAYMOVE)); + addr2 = reinterpret_cast<int32_t*>(mremap(addr, page_size, 2 * page_size, MREMAP_MAYMOVE)); CHECK_NE(addr2, MAP_FAILED); // Test that we can write into the remapped mapping. @@ -111,16 +111,16 @@ class TestZygoteMemory : public testing::Test { // Zygote JIT memory only works on kernels that don't segfault on flush. TEST_DISABLED_FOR_KERNELS_WITH_CACHE_SEGFAULT(); std::string error_msg; - size_t size = gPageSize; + const size_t page_size = GetPageSizeSlow(); int32_t* addr = nullptr; int32_t* addr2 = nullptr; { - android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(size, &error_msg)); + android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(page_size, &error_msg)); CHECK_NE(fd.get(), -1); // Create a writable mapping. addr = reinterpret_cast<int32_t*>( - mmap(nullptr, gPageSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0)); + mmap(nullptr, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0)); CHECK(addr != nullptr); CHECK_NE(addr, MAP_FAILED); @@ -130,7 +130,7 @@ class TestZygoteMemory : public testing::Test { // Create a read-only mapping. addr2 = reinterpret_cast<int32_t*>( - mmap(nullptr, gPageSize, PROT_READ, MAP_SHARED, fd.get(), 0)); + mmap(nullptr, page_size, PROT_READ, MAP_SHARED, fd.get(), 0)); CHECK(addr2 != nullptr); // Protect the memory. @@ -141,10 +141,10 @@ class TestZygoteMemory : public testing::Test { // there. // Create a mapping of atomic ints to communicate between processes. - android::base::unique_fd fd2(JitMemoryRegion::CreateZygoteMemory(size, &error_msg)); + android::base::unique_fd fd2(JitMemoryRegion::CreateZygoteMemory(page_size, &error_msg)); CHECK_NE(fd2.get(), -1); std::atomic<int32_t>* shared = reinterpret_cast<std::atomic<int32_t>*>( - mmap(nullptr, gPageSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd2.get(), 0)); + mmap(nullptr, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd2.get(), 0)); // Values used for the tests below. const int32_t parent_value = 66; @@ -163,7 +163,7 @@ class TestZygoteMemory : public testing::Test { CHECK_EQ(addr2[0], child_value); // Unmap the writable mappping. - munmap(addr, gPageSize); + munmap(addr, page_size); CHECK_EQ(addr2[0], child_value); @@ -198,9 +198,9 @@ class TestZygoteMemory : public testing::Test { CHECK_EQ(WEXITSTATUS(status), kReturnFromFault); CHECK_EQ(addr[0], parent_value); CHECK_EQ(addr2[0], parent_value); - munmap(addr, gPageSize); - munmap(addr2, gPageSize); - munmap(shared, gPageSize); + munmap(addr, page_size); + munmap(addr2, page_size); + munmap(shared, page_size); } } @@ -208,19 +208,19 @@ class TestZygoteMemory : public testing::Test { // Zygote JIT memory only works on kernels that don't segfault on flush. TEST_DISABLED_FOR_KERNELS_WITH_CACHE_SEGFAULT(); std::string error_msg; - size_t size = gPageSize; + const size_t page_size = GetPageSizeSlow(); int32_t* addr = nullptr; int32_t* addr2 = nullptr; { - android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(size, &error_msg)); + android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(page_size, &error_msg)); CHECK_NE(fd.get(), -1); // Create a writable mapping. addr = reinterpret_cast<int32_t*>( - mmap(nullptr, gPageSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0)); + mmap(nullptr, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0)); CHECK(addr != nullptr); CHECK_NE(addr, MAP_FAILED); - CHECK_EQ(madvise(addr, gPageSize, MADV_DONTFORK), 0); + CHECK_EQ(madvise(addr, page_size, MADV_DONTFORK), 0); // Test that we can write into the mapping. addr[0] = 42; @@ -228,7 +228,7 @@ class TestZygoteMemory : public testing::Test { // Create a read-only mapping. addr2 = reinterpret_cast<int32_t*>( - mmap(nullptr, gPageSize, PROT_READ, MAP_SHARED, fd.get(), 0)); + mmap(nullptr, page_size, PROT_READ, MAP_SHARED, fd.get(), 0)); CHECK(addr2 != nullptr); // Protect the memory. @@ -239,10 +239,10 @@ class TestZygoteMemory : public testing::Test { // there. // Create a mapping of atomic ints to communicate between processes. - android::base::unique_fd fd2(JitMemoryRegion::CreateZygoteMemory(size, &error_msg)); + android::base::unique_fd fd2(JitMemoryRegion::CreateZygoteMemory(page_size, &error_msg)); CHECK_NE(fd2.get(), -1); std::atomic<int32_t>* shared = reinterpret_cast<std::atomic<int32_t>*>( - mmap(nullptr, gPageSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd2.get(), 0)); + mmap(nullptr, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd2.get(), 0)); // Values used for the tests below. const int32_t parent_value = 66; @@ -288,9 +288,9 @@ class TestZygoteMemory : public testing::Test { CHECK_EQ(addr[0], parent_value); CHECK_EQ(addr2[0], parent_value); - munmap(addr, gPageSize); - munmap(addr2, gPageSize); - munmap(shared, gPageSize); + munmap(addr, page_size); + munmap(addr2, page_size); + munmap(shared, page_size); } } @@ -307,14 +307,14 @@ class TestZygoteMemory : public testing::Test { return; } std::string error_msg; - size_t size = gPageSize; + const size_t page_size = GetPageSizeSlow(); int32_t* addr = nullptr; - android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(size, &error_msg)); + android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(page_size, &error_msg)); CHECK_NE(fd.get(), -1); // Create a writable mapping. addr = reinterpret_cast<int32_t*>( - mmap(nullptr, gPageSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0)); + mmap(nullptr, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0)); CHECK(addr != nullptr); CHECK_NE(addr, MAP_FAILED); @@ -323,10 +323,10 @@ class TestZygoteMemory : public testing::Test { CHECK_EQ(addr[0], 42); // Create another mapping of atomic ints to communicate between processes. - android::base::unique_fd fd2(JitMemoryRegion::CreateZygoteMemory(size, &error_msg)); + android::base::unique_fd fd2(JitMemoryRegion::CreateZygoteMemory(page_size, &error_msg)); CHECK_NE(fd2.get(), -1); std::atomic<int32_t>* shared = reinterpret_cast<std::atomic<int32_t>*>( - mmap(nullptr, gPageSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd2.get(), 0)); + mmap(nullptr, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd2.get(), 0)); // Protect the memory. CHECK(JitMemoryRegion::ProtectZygoteMemory(fd.get(), &error_msg)); @@ -342,7 +342,7 @@ class TestZygoteMemory : public testing::Test { shared[0] = 0; pid_t pid = fork(); if (pid == 0) { - CHECK_EQ(mmap(addr, gPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), + CHECK_EQ(mmap(addr, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), addr); addr[0] = child_value; exit(0); @@ -361,18 +361,18 @@ class TestZygoteMemory : public testing::Test { if (pid == 0) { // Map it private with write access. MAP_FIXED will replace the existing // mapping. - CHECK_EQ(mmap(addr, gPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), + CHECK_EQ(mmap(addr, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), addr); addr[0] = child_value; CHECK_EQ(addr[0], child_value); // Check that mapping shared with write access fails. - CHECK_EQ(mmap(addr, gPageSize, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd.get(), 0), + CHECK_EQ(mmap(addr, page_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd.get(), 0), MAP_FAILED); CHECK_EQ(errno, EPERM); // Map shared with read access. - CHECK_EQ(mmap(addr, gPageSize, PROT_READ, MAP_SHARED | MAP_FIXED, fd.get(), 0), addr); + CHECK_EQ(mmap(addr, page_size, PROT_READ, MAP_SHARED | MAP_FIXED, fd.get(), 0), addr); CHECK_NE(addr[0], child_value); // Wait for the parent to notify. @@ -385,13 +385,13 @@ class TestZygoteMemory : public testing::Test { shared[0] = 2; // Map it private again. - CHECK_EQ(mmap(addr, gPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), + CHECK_EQ(mmap(addr, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), addr); addr[0] = child_value + 1; CHECK_EQ(addr[0], child_value + 1); // And map it back shared. - CHECK_EQ(mmap(addr, gPageSize, PROT_READ, MAP_SHARED | MAP_FIXED, fd.get(), 0), addr); + CHECK_EQ(mmap(addr, page_size, PROT_READ, MAP_SHARED | MAP_FIXED, fd.get(), 0), addr); while (shared[0] != 3) { sched_yield(); } @@ -425,7 +425,7 @@ class TestZygoteMemory : public testing::Test { addr[0] = starting_value; pid = fork(); if (pid == 0) { - CHECK_EQ(mmap(addr, gPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), + CHECK_EQ(mmap(addr, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), addr); CHECK_EQ(addr[0], starting_value); addr[0] = child_value; @@ -442,7 +442,7 @@ class TestZygoteMemory : public testing::Test { CHECK_EQ(addr[0], child_value); // Test the buffer contains the parent data after a new mmap. - CHECK_EQ(mmap(addr, gPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), + CHECK_EQ(mmap(addr, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), addr); CHECK_EQ(addr[0], parent_value); exit(0); @@ -467,7 +467,7 @@ class TestZygoteMemory : public testing::Test { addr[0] = starting_value; pid = fork(); if (pid == 0) { - CHECK_EQ(mmap(addr, gPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), + CHECK_EQ(mmap(addr, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd.get(), 0), addr); CHECK_EQ(addr[0], starting_value); // Notify the parent for a new update of the buffer. @@ -489,8 +489,8 @@ class TestZygoteMemory : public testing::Test { CHECK(WIFEXITED(status)) << strerror(errno); CHECK_EQ(addr[0], parent_value); } - munmap(addr, gPageSize); - munmap(shared, gPageSize); + munmap(addr, page_size); + munmap(shared, page_size); } // Test that a readable mapping created befire sealing future writes, can be @@ -499,15 +499,15 @@ class TestZygoteMemory : public testing::Test { // Zygote JIT memory only works on kernels that don't segfault on flush. TEST_DISABLED_FOR_KERNELS_WITH_CACHE_SEGFAULT(); std::string error_msg; - size_t size = gPageSize; + const size_t page_size = GetPageSizeSlow(); int32_t* addr = nullptr; { - android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(size, &error_msg)); + android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(page_size, &error_msg)); CHECK_NE(fd.get(), -1); // Create a shared readable mapping. addr = reinterpret_cast<int32_t*>( - mmap(nullptr, gPageSize, PROT_READ, MAP_SHARED, fd.get(), 0)); + mmap(nullptr, page_size, PROT_READ, MAP_SHARED, fd.get(), 0)); CHECK(addr != nullptr); CHECK_NE(addr, MAP_FAILED); @@ -517,7 +517,7 @@ class TestZygoteMemory : public testing::Test { } // At this point, the fd has been dropped, but the memory mappings are still // there. - int res = mprotect(addr, gPageSize, PROT_WRITE); + int res = mprotect(addr, page_size, PROT_WRITE); CHECK_EQ(res, 0); } @@ -526,10 +526,10 @@ class TestZygoteMemory : public testing::Test { // Zygote JIT memory only works on kernels that don't segfault on flush. TEST_DISABLED_FOR_KERNELS_WITH_CACHE_SEGFAULT(); std::string error_msg; - size_t size = gPageSize; + const size_t page_size = GetPageSizeSlow(); int32_t* addr = nullptr; { - android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(size, &error_msg)); + android::base::unique_fd fd(JitMemoryRegion::CreateZygoteMemory(page_size, &error_msg)); CHECK_NE(fd.get(), -1); // Protect the memory. @@ -538,13 +538,13 @@ class TestZygoteMemory : public testing::Test { // Create a shared readable mapping. addr = reinterpret_cast<int32_t*>( - mmap(nullptr, gPageSize, PROT_READ, MAP_SHARED, fd.get(), 0)); + mmap(nullptr, page_size, PROT_READ, MAP_SHARED, fd.get(), 0)); CHECK(addr != nullptr); CHECK_NE(addr, MAP_FAILED); } // At this point, the fd has been dropped, but the memory mappings are still // there. - int res = mprotect(addr, gPageSize, PROT_WRITE); + int res = mprotect(addr, page_size, PROT_WRITE); CHECK_EQ(res, -1); CHECK_EQ(errno, EACCES); } diff --git a/runtime/jni/local_reference_table.cc b/runtime/jni/local_reference_table.cc index 93ad671655..bf55f0ec3e 100644 --- a/runtime/jni/local_reference_table.cc +++ b/runtime/jni/local_reference_table.cc @@ -40,10 +40,6 @@ namespace jni { static constexpr bool kDumpStackOnNonLocalReference = false; static constexpr bool kDebugLRT = false; -// Number of free lists in the allocator. -ART_PAGE_SIZE_AGNOSTIC_DECLARE_AND_DEFINE(size_t, gNumLrtSlots, - WhichPowerOf2(gPageSize / kInitialLrtBytes)); - // Mmap an "indirect ref table region. Table_bytes is a multiple of a page size. static inline MemMap NewLRTMap(size_t table_bytes, std::string* error_msg) { return MemMap::MapAnonymous("local ref table", @@ -54,7 +50,7 @@ static inline MemMap NewLRTMap(size_t table_bytes, std::string* error_msg) { } SmallLrtAllocator::SmallLrtAllocator() - : free_lists_(gNumLrtSlots, nullptr), + : free_lists_(num_lrt_slots_, nullptr), shared_lrt_maps_(), lock_("Small LRT allocator lock", LockLevel::kGenericBottomLock) { } @@ -64,7 +60,7 @@ inline size_t SmallLrtAllocator::GetIndex(size_t size) { DCHECK_LT(size, gPageSize / sizeof(LrtEntry)); DCHECK(IsPowerOfTwo(size)); size_t index = WhichPowerOf2(size / kSmallLrtEntries); - DCHECK_LT(index, gNumLrtSlots); + DCHECK_LT(index, num_lrt_slots_); return index; } @@ -72,11 +68,11 @@ LrtEntry* SmallLrtAllocator::Allocate(size_t size, std::string* error_msg) { size_t index = GetIndex(size); MutexLock lock(Thread::Current(), lock_); size_t fill_from = index; - while (fill_from != gNumLrtSlots && free_lists_[fill_from] == nullptr) { + while (fill_from != num_lrt_slots_ && free_lists_[fill_from] == nullptr) { ++fill_from; } void* result = nullptr; - if (fill_from != gNumLrtSlots) { + if (fill_from != num_lrt_slots_) { // We found a slot with enough memory. result = free_lists_[fill_from]; free_lists_[fill_from] = *reinterpret_cast<void**>(result); @@ -105,13 +101,13 @@ LrtEntry* SmallLrtAllocator::Allocate(size_t size, std::string* error_msg) { void SmallLrtAllocator::Deallocate(LrtEntry* unneeded, size_t size) { size_t index = GetIndex(size); MutexLock lock(Thread::Current(), lock_); - while (index < gNumLrtSlots) { + while (index < num_lrt_slots_) { // Check if we can merge this free block with another block with the same size. void** other = reinterpret_cast<void**>( reinterpret_cast<uintptr_t>(unneeded) ^ (kInitialLrtBytes << index)); void** before = &free_lists_[index]; - if (index + 1u == gNumLrtSlots && *before == other && *other == nullptr) { - // Do not unmap the page if we do not have other free blocks with index `gNumLrtSlots - 1`. + if (index + 1u == num_lrt_slots_ && *before == other && *other == nullptr) { + // Do not unmap the page if we do not have other free blocks with index `num_lrt_slots_ - 1`. // (Keep at least one free block to avoid a situation where creating and destroying a single // thread with no local references would map and unmap a page in the `SmallLrtAllocator`.) break; @@ -129,9 +125,9 @@ void SmallLrtAllocator::Deallocate(LrtEntry* unneeded, size_t size) { unneeded = reinterpret_cast<LrtEntry*>( reinterpret_cast<uintptr_t>(unneeded) & reinterpret_cast<uintptr_t>(other)); } - if (index == gNumLrtSlots) { + if (index == num_lrt_slots_) { // Free the entire page. - DCHECK(free_lists_[gNumLrtSlots - 1u] != nullptr); + DCHECK(free_lists_[num_lrt_slots_ - 1u] != nullptr); auto match = [=](MemMap& map) { return unneeded == reinterpret_cast<LrtEntry*>(map.Begin()); }; auto it = std::find_if(shared_lrt_maps_.begin(), shared_lrt_maps_.end(), match); DCHECK(it != shared_lrt_maps_.end()); diff --git a/runtime/jni/local_reference_table.h b/runtime/jni/local_reference_table.h index 97993da061..df2b5075a5 100644 --- a/runtime/jni/local_reference_table.h +++ b/runtime/jni/local_reference_table.h @@ -227,7 +227,14 @@ class SmallLrtAllocator { void Deallocate(LrtEntry* unneeded, size_t size) REQUIRES(!lock_); private: - static size_t GetIndex(size_t size); + // Number of free lists in the allocator. +#ifdef ART_PAGE_SIZE_AGNOSTIC + const size_t num_lrt_slots_ = (WhichPowerOf2(gPageSize / kInitialLrtBytes)); +#else + static constexpr size_t num_lrt_slots_ = (WhichPowerOf2(gPageSize / kInitialLrtBytes)); +#endif + + size_t GetIndex(size_t size); // Free lists of small chunks linked through the first word. dchecked_vector<void*> free_lists_; diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc index c4c6dca9f3..347ad7fb6e 100644 --- a/runtime/oat_file.cc +++ b/runtime/oat_file.cc @@ -626,7 +626,7 @@ bool OatFileBase::Setup(int zip_fd, // however not aligned to kElfSegmentAlignment. While technically this isn't // correct as per requirement in the ELF header, it has to be supported for // now. See also the comment at ImageHeader::RelocateImageReferences. - if (!IsAlignedParam(bss_begin_, gPageSize) || + if (!IsAlignedParam(bss_begin_, MemMap::GetPageSize()) || !IsAlignedParam(bss_methods_, static_cast<size_t>(pointer_size)) || !IsAlignedParam(bss_roots_, static_cast<size_t>(pointer_size)) || !IsAligned<alignof(GcRoot<mirror::Object>)>(bss_end_)) { diff --git a/runtime/runtime.cc b/runtime/runtime.cc index f79351ce93..6f0a040f66 100644 --- a/runtime/runtime.cc +++ b/runtime/runtime.cc @@ -206,6 +206,12 @@ static constexpr double kLowMemoryMaxLoadFactor = 0.8; static constexpr double kNormalMinLoadFactor = 0.4; static constexpr double kNormalMaxLoadFactor = 0.7; +#ifdef ART_PAGE_SIZE_AGNOSTIC +// Declare the constant as ALWAYS_HIDDEN to ensure it isn't visible from outside libart.so. +const size_t PageSize::value_ ALWAYS_HIDDEN = GetPageSizeSlow(); +PageSize gPageSize ALWAYS_HIDDEN; +#endif + Runtime* Runtime::instance_ = nullptr; struct TraceConfig { @@ -548,6 +554,12 @@ Runtime::~Runtime() { // instance. We rely on a small initialization order issue in Runtime::Start() that requires // elements of WellKnownClasses to be null, see b/65500943. WellKnownClasses::Clear(); + +#ifdef ART_PAGE_SIZE_AGNOSTIC + // This is added to ensure no test is able to access gPageSize prior to initializing Runtime just + // because a Runtime instance was created (and subsequently destroyed) by another test. + gPageSize.DisallowAccess(); +#endif } struct AbortState { @@ -1492,6 +1504,10 @@ bool Runtime::Init(RuntimeArgumentMap&& runtime_options_in) { // Take a snapshot of the environment at the time the runtime was created, for use by Exec, etc. env_snapshot_.TakeSnapshot(); +#ifdef ART_PAGE_SIZE_AGNOSTIC + gPageSize.AllowAccess(); +#endif + using Opt = RuntimeArgumentMap; Opt runtime_options(std::move(runtime_options_in)); ScopedTrace trace(__FUNCTION__); diff --git a/runtime/runtime_globals.h b/runtime/runtime_globals.h index 468c75cd8a..c128c19c36 100644 --- a/runtime/runtime_globals.h +++ b/runtime/runtime_globals.h @@ -17,6 +17,8 @@ #ifndef ART_RUNTIME_RUNTIME_GLOBALS_H_ #define ART_RUNTIME_RUNTIME_GLOBALS_H_ +#include <android-base/logging.h> + #include "base/globals.h" namespace art { @@ -24,6 +26,71 @@ namespace art { // Size of Dex virtual registers. static constexpr size_t kVRegSize = 4; +#ifdef ART_PAGE_SIZE_AGNOSTIC +// Accessor for the page size constant local to the libart. +// +// The value is only available after the Runtime initialization started - to ensure there is no +// static initialization order issues where initialization of other values is dependent on the page +// size. In those cases, GetPageSizeSlow() should be used. +struct PageSize { + PageSize() + : is_initialized_(true), is_access_allowed_(false) {} + + ALWAYS_INLINE operator size_t() const { + DCHECK(is_initialized_ && is_access_allowed_); + return value_; + } + + private: + friend class Runtime; + + void AllowAccess() { + SetAccessAllowed(true); + } + + void DisallowAccess() { + SetAccessAllowed(false); + } + + void SetAccessAllowed(bool is_allowed) { + // is_initialized_ is set to true when the page size value is initialized during the static + // initialization. This CHECK is added as an auxiliary way to help catching incorrect use of + // the method. + CHECK(is_initialized_); + is_access_allowed_ = is_allowed; + } + + // The page size value. + // + // It is declared as a static constant value to ensure compiler recognizes that it doesn't change + // once it is initialized. + // + // It is declared as "hidden" i.e. local to the libart, to ensure: + // - no other library can access it, so no static initialization dependency from other libraries + // is possible; + // - the variable can be addressed via offset from the program counter, instead of the global + // offset table which would've added another level of indirection. + static const size_t value_ ALWAYS_HIDDEN; + + // There are two flags in the accessor which help to ensure the value is accessed only after the + // static initialization is complete. + // + // is_initialized_ is used to assert the page size value is indeed initialized when the value + // access is allowed and when it is accessed. + // + // is_access_allowed_ is used to ensure the value is only accessed after Runtime initialization + // started. + const bool is_initialized_; + bool is_access_allowed_; +}; + +// gPageSize should only be used within libart. For most of the other cases MemMap::GetPageSize() +// or GetPageSizeSlow() should be used. See also the comment for GetPageSizeSlow(). +extern PageSize gPageSize ALWAYS_HIDDEN; +#else +static constexpr size_t gPageSize = kMinPageSize; +#endif + // Returns whether the given memory offset can be used for generating // an implicit null check. static inline bool CanDoImplicitNullCheckOn(uintptr_t offset) { diff --git a/runtime/thread-inl.h b/runtime/thread-inl.h index 3fd42a7d94..65049f1500 100644 --- a/runtime/thread-inl.h +++ b/runtime/thread-inl.h @@ -40,6 +40,13 @@ inline Thread* Thread::ForEnv(JNIEnv* env) { return full_env->GetSelf(); } +inline size_t Thread::GetStackOverflowProtectedSize() { + // The kMemoryToolStackGuardSizeScale is expected to be 1 when ASan is not enabled. + // As the function is always inlined, in those cases each function call should turn + // into a simple reference to gPageSize. + return kMemoryToolStackGuardSizeScale * gPageSize; +} + inline ObjPtr<mirror::Object> Thread::DecodeJObject(jobject obj) const { if (obj == nullptr) { return nullptr; diff --git a/runtime/thread.cc b/runtime/thread.cc index c26050c17d..818300aacc 100644 --- a/runtime/thread.cc +++ b/runtime/thread.cc @@ -147,13 +147,6 @@ thread_local Thread* Thread::self_tls_ = nullptr; static constexpr bool kVerifyImageObjectsMarked = kIsDebugBuild; -// For implicit overflow checks we reserve an extra piece of memory at the bottom -// of the stack (lowest memory). The higher portion of the memory -// is protected against reads and the lower is available for use while -// throwing the StackOverflow exception. -ART_PAGE_SIZE_AGNOSTIC_DECLARE_AND_DEFINE(size_t, gStackOverflowProtectedSize, - kMemoryToolStackGuardSizeScale * gPageSize); - static const char* kThreadNameDuringStartup = "<native thread without managed peer>"; void Thread::InitCardTable() { @@ -749,14 +742,14 @@ static uint8_t* FindStackTop() { // overflow is detected. It is located right below the stack_begin_. ATTRIBUTE_NO_SANITIZE_ADDRESS void Thread::InstallImplicitProtection() { - uint8_t* pregion = tlsPtr_.stack_begin - gStackOverflowProtectedSize; + uint8_t* pregion = tlsPtr_.stack_begin - GetStackOverflowProtectedSize(); // Page containing current top of stack. uint8_t* stack_top = FindStackTop(); // Try to directly protect the stack. VLOG(threads) << "installing stack protected region at " << std::hex << static_cast<void*>(pregion) << " to " << - static_cast<void*>(pregion + gStackOverflowProtectedSize - 1); + static_cast<void*>(pregion + GetStackOverflowProtectedSize() - 1); if (ProtectStack(/* fatal_on_error= */ false)) { // Tell the kernel that we won't be needing these pages any more. // NB. madvise will probably write zeroes into the memory (on linux it does). @@ -827,7 +820,7 @@ void Thread::InstallImplicitProtection() { VLOG(threads) << "(again) installing stack protected region at " << std::hex << static_cast<void*>(pregion) << " to " << - static_cast<void*>(pregion + gStackOverflowProtectedSize - 1); + static_cast<void*>(pregion + GetStackOverflowProtectedSize() - 1); // Protect the bottom of the stack to prevent read/write to it. ProtectStack(/* fatal_on_error= */ true); @@ -1362,9 +1355,9 @@ bool Thread::InitStackHwm() { // // On systems with 4K page size, typically the minimum stack size will be 4+8+4 = 16K. // The thread won't be able to do much with this stack: even the GC takes between 8K and 12K. - DCHECK_ALIGNED_PARAM(static_cast<size_t>(gStackOverflowProtectedSize), + DCHECK_ALIGNED_PARAM(static_cast<size_t>(GetStackOverflowProtectedSize()), static_cast<int32_t>(gPageSize)); - size_t min_stack = gStackOverflowProtectedSize + + size_t min_stack = GetStackOverflowProtectedSize() + RoundUp(GetStackOverflowReservedBytes(kRuntimeISA) + 4 * KB, gPageSize); if (read_stack_size <= min_stack) { // Note, as we know the stack is small, avoid operations that could use a lot of stack. @@ -1395,9 +1388,9 @@ bool Thread::InitStackHwm() { // to install our own region so we need to move the limits // of the stack to make room for it. - tlsPtr_.stack_begin += read_guard_size + gStackOverflowProtectedSize; - tlsPtr_.stack_end += read_guard_size + gStackOverflowProtectedSize; - tlsPtr_.stack_size -= read_guard_size + gStackOverflowProtectedSize; + tlsPtr_.stack_begin += read_guard_size + GetStackOverflowProtectedSize(); + tlsPtr_.stack_end += read_guard_size + GetStackOverflowProtectedSize(); + tlsPtr_.stack_size -= read_guard_size + GetStackOverflowProtectedSize(); InstallImplicitProtection(); } @@ -2492,6 +2485,7 @@ Thread::Thread(bool daemon) tlsPtr_.mutator_lock = Locks::mutator_lock_; DCHECK(tlsPtr_.mutator_lock != nullptr); tlsPtr_.name.store(kThreadNameDuringStartup, std::memory_order_relaxed); + CHECK_NE(GetStackOverflowProtectedSize(), 0u); static_assert((sizeof(Thread) % 4) == 0U, "art::Thread has a size which is not a multiple of 4."); @@ -4648,14 +4642,14 @@ std::ostream& operator<<(std::ostream& os, const Thread& thread) { } bool Thread::ProtectStack(bool fatal_on_error) { - void* pregion = tlsPtr_.stack_begin - gStackOverflowProtectedSize; + void* pregion = tlsPtr_.stack_begin - GetStackOverflowProtectedSize(); VLOG(threads) << "Protecting stack at " << pregion; - if (mprotect(pregion, gStackOverflowProtectedSize, PROT_NONE) == -1) { + if (mprotect(pregion, GetStackOverflowProtectedSize(), PROT_NONE) == -1) { if (fatal_on_error) { // b/249586057, LOG(FATAL) times out LOG(ERROR) << "Unable to create protected region in stack for implicit overflow check. " "Reason: " - << strerror(errno) << " size: " << gStackOverflowProtectedSize; + << strerror(errno) << " size: " << GetStackOverflowProtectedSize(); exit(1); } return false; @@ -4664,9 +4658,9 @@ bool Thread::ProtectStack(bool fatal_on_error) { } bool Thread::UnprotectStack() { - void* pregion = tlsPtr_.stack_begin - gStackOverflowProtectedSize; + void* pregion = tlsPtr_.stack_begin - GetStackOverflowProtectedSize(); VLOG(threads) << "Unprotecting stack at " << pregion; - return mprotect(pregion, gStackOverflowProtectedSize, PROT_READ|PROT_WRITE) == 0; + return mprotect(pregion, GetStackOverflowProtectedSize(), PROT_READ|PROT_WRITE) == 0; } size_t Thread::NumberOfHeldMutexes() const { diff --git a/runtime/thread.h b/runtime/thread.h index db1efdc6a4..6a006be985 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -285,6 +285,11 @@ class Thread { // Get the thread from the JNI environment. static Thread* ForEnv(JNIEnv* env); + // For implicit overflow checks we reserve an extra piece of memory at the bottom of the stack + // (lowest memory). The higher portion of the memory is protected against reads and the lower is + // available for use while throwing the StackOverflow exception. + ALWAYS_INLINE static size_t GetStackOverflowProtectedSize(); + // On a runnable thread, check for pending thread suspension request and handle if pending. void AllowThreadSuspension() REQUIRES_SHARED(Locks::mutator_lock_); diff --git a/test/305-other-fault-handler/fault_handler.cc b/test/305-other-fault-handler/fault_handler.cc index e459a75da4..c78f7d4336 100644 --- a/test/305-other-fault-handler/fault_handler.cc +++ b/test/305-other-fault-handler/fault_handler.cc @@ -36,7 +36,7 @@ class TestFaultHandler final : public FaultHandler { map_error_(), target_map_(MemMap::MapAnonymous("test-305-mmap", /* addr */ nullptr, - /* byte_count */ gPageSize, + /* byte_count */ MemMap::GetPageSize(), /* prot */ PROT_NONE, /* low_4gb */ false, /* reuse */ false, |