Gitiles
Code Review Sign In
LeafOS / LeafOS-Project / android_art / a9102c122084ced4a5369d2c2174ea1b83b4e1c9 / . / runtime / gc / heap.cc
blob: 329777c7d7a4dc41245b7069e0f86626bdbc5d81 [file] [log] [blame]
/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "heap.h"
#include <limits>
#include "android-base/thread_annotations.h"
#if defined(__BIONIC__) || defined(__GLIBC__) || defined(ANDROID_HOST_MUSL)
#include <malloc.h> // For mallinfo()
#endif
#include <memory>
#include <random>
#include <unistd.h>
#include <sys/types.h>
#include <vector>
#include "android-base/stringprintf.h"
#include "allocation_listener.h"
#include "art_field-inl.h"
#include "backtrace_helper.h"
#include "base/allocator.h"
#include "base/arena_allocator.h"
#include "base/dumpable.h"
#include "base/file_utils.h"
#include "base/histogram-inl.h"
#include "base/logging.h" // For VLOG.
#include "base/memory_tool.h"
#include "base/mutex.h"
#include "base/os.h"
#include "base/stl_util.h"
#include "base/systrace.h"
#include "base/time_utils.h"
#include "base/utils.h"
#include "class_root-inl.h"
#include "common_throws.h"
#include "debugger.h"
#include "dex/dex_file-inl.h"
#include "entrypoints/quick/quick_alloc_entrypoints.h"
#include "gc/accounting/card_table-inl.h"
#include "gc/accounting/heap_bitmap-inl.h"
#include "gc/accounting/mod_union_table-inl.h"
#include "gc/accounting/read_barrier_table.h"
#include "gc/accounting/remembered_set.h"
#include "gc/accounting/space_bitmap-inl.h"
#include "gc/collector/concurrent_copying.h"
#include "gc/collector/mark_compact.h"
#include "gc/collector/mark_sweep.h"
#include "gc/collector/partial_mark_sweep.h"
#include "gc/collector/semi_space.h"
#include "gc/collector/sticky_mark_sweep.h"
#include "gc/racing_check.h"
#include "gc/reference_processor.h"
#include "gc/scoped_gc_critical_section.h"
#include "gc/space/bump_pointer_space.h"
#include "gc/space/dlmalloc_space-inl.h"
#include "gc/space/image_space.h"
#include "gc/space/large_object_space.h"
#include "gc/space/region_space.h"
#include "gc/space/rosalloc_space-inl.h"
#include "gc/space/space-inl.h"
#include "gc/space/zygote_space.h"
#include "gc/task_processor.h"
#include "gc/verification.h"
#include "gc_pause_listener.h"
#include "gc_root.h"
#include "handle_scope-inl.h"
#include "heap-inl.h"
#include "heap-visit-objects-inl.h"
#include "image.h"
#include "intern_table.h"
#include "jit/jit.h"
#include "jit/jit_code_cache.h"
#include "jni/java_vm_ext.h"
#include "mirror/class-inl.h"
#include "mirror/executable-inl.h"
#include "mirror/field.h"
#include "mirror/method_handle_impl.h"
#include "mirror/object-inl.h"
#include "mirror/object-refvisitor-inl.h"
#include "mirror/object_array-inl.h"
#include "mirror/reference-inl.h"
#include "mirror/var_handle.h"
#include "nativehelper/scoped_local_ref.h"
#include "obj_ptr-inl.h"
#ifdef ART_TARGET_ANDROID
#include "perfetto/heap_profile.h"
#endif
#include "reflection.h"
#include "runtime.h"
#include "javaheapprof/javaheapsampler.h"
#include "scoped_thread_state_change-inl.h"
#include "thread-inl.h"
#include "thread_list.h"
#include "verify_object-inl.h"
#include "well_known_classes.h"
namespace art {
#ifdef ART_TARGET_ANDROID
namespace {
// Enable the heap sampler Callback function used by Perfetto.
void EnableHeapSamplerCallback(void* enable_ptr,
const AHeapProfileEnableCallbackInfo* enable_info_ptr) {
HeapSampler* sampler_self = reinterpret_cast<HeapSampler*>(enable_ptr);
// Set the ART profiler sampling interval to the value from Perfetto.
uint64_t interval = AHeapProfileEnableCallbackInfo_getSamplingInterval(enable_info_ptr);
if (interval > 0) {
sampler_self->SetSamplingInterval(interval);
}
// Else default is 4K sampling interval. However, default case shouldn't happen for Perfetto API.
// AHeapProfileEnableCallbackInfo_getSamplingInterval should always give the requested
// (non-negative) sampling interval. It is a uint64_t and gets checked for != 0
// Do not call heap as a temp here, it will build but test run will silently fail.
// Heap is not fully constructed yet in some cases.
sampler_self->EnableHeapSampler();
}
// Disable the heap sampler Callback function used by Perfetto.
void DisableHeapSamplerCallback(void* disable_ptr,
[[maybe_unused]] const AHeapProfileDisableCallbackInfo* info_ptr) {
HeapSampler* sampler_self = reinterpret_cast<HeapSampler*>(disable_ptr);
sampler_self->DisableHeapSampler();
}
} // namespace
#endif
namespace gc {
DEFINE_RUNTIME_DEBUG_FLAG(Heap, kStressCollectorTransition);
// Minimum amount of remaining bytes before a concurrent GC is triggered.
static constexpr size_t kMinConcurrentRemainingBytes = 128 * KB;
static constexpr size_t kMaxConcurrentRemainingBytes = 512 * KB;
// Sticky GC throughput adjustment, divided by 4. Increasing this causes sticky GC to occur more
// relative to partial/full GC. This may be desirable since sticky GCs interfere less with mutator
// threads (lower pauses, use less memory bandwidth).
static double GetStickyGcThroughputAdjustment(bool use_generational_cc) {
return use_generational_cc ? 0.5 : 1.0;
}
// Whether or not we compact the zygote in PreZygoteFork.
static constexpr bool kCompactZygote = kMovingCollector;
// How many reserve entries are at the end of the allocation stack, these are only needed if the
// allocation stack overflows.
static constexpr size_t kAllocationStackReserveSize = 1024;
// Default mark stack size in bytes.
static const size_t kDefaultMarkStackSize = 64 * KB;
// Define space name.
static const char* kDlMallocSpaceName[2] = {"main dlmalloc space", "main dlmalloc space 1"};
static const char* kRosAllocSpaceName[2] = {"main rosalloc space", "main rosalloc space 1"};
static const char* kMemMapSpaceName[2] = {"main space", "main space 1"};
static const char* kNonMovingSpaceName = "non moving space";
static const char* kZygoteSpaceName = "zygote space";
static constexpr bool kGCALotMode = false;
// GC alot mode uses a small allocation stack to stress test a lot of GC.
static constexpr size_t kGcAlotAllocationStackSize = 4 * KB /
sizeof(mirror::HeapReference<mirror::Object>);
// Verify objet has a small allocation stack size since searching the allocation stack is slow.
static constexpr size_t kVerifyObjectAllocationStackSize = 16 * KB /
sizeof(mirror::HeapReference<mirror::Object>);
static constexpr size_t kDefaultAllocationStackSize = 8 * MB /
sizeof(mirror::HeapReference<mirror::Object>);
// If we violate BOTH of the following constraints, we throw OOME.
// They differ due to concurrent allocation.
// After a GC (due to allocation failure) we should retrieve at least this
// fraction of the current max heap size.
static constexpr double kMinFreedHeapAfterGcForAlloc = 0.05;
// After a GC (due to allocation failure), at least this fraction of the
// heap should be available.
static constexpr double kMinFreeHeapAfterGcForAlloc = 0.01;
// For deterministic compilation, we need the heap to be at a well-known address.
static constexpr uint32_t kAllocSpaceBeginForDeterministicAoT = 0x40000000;
// Dump the rosalloc stats on SIGQUIT.
static constexpr bool kDumpRosAllocStatsOnSigQuit = false;
static const char* kRegionSpaceName = "main space (region space)";
// If true, we log all GCs in the both the foreground and background. Used for debugging.
static constexpr bool kLogAllGCs = false;
// Use Max heap for 2 seconds, this is smaller than the usual 5s window since we don't want to leave
// allocate with relaxed ergonomics for that long.
static constexpr size_t kPostForkMaxHeapDurationMS = 2000;
#if defined(__LP64__) || !defined(ADDRESS_SANITIZER)
// 300 MB (0x12c00000) - (default non-moving space capacity).
uint8_t* const Heap::kPreferredAllocSpaceBegin =
reinterpret_cast<uint8_t*>(300 * MB - kDefaultNonMovingSpaceCapacity);
#else
#ifdef __ANDROID__
// For 32-bit Android, use 0x20000000 because asan reserves 0x04000000 - 0x20000000.
uint8_t* const Heap::kPreferredAllocSpaceBegin = reinterpret_cast<uint8_t*>(0x20000000);
#else
// For 32-bit host, use 0x40000000 because asan uses most of the space below this.
uint8_t* const Heap::kPreferredAllocSpaceBegin = reinterpret_cast<uint8_t*>(0x40000000);
#endif
#endif
// Log GC on regular (but fairly large) intervals during GC stress mode.
// It is expected that the other runtime options will be used to reduce the usual logging.
// This allows us to make the logging much less verbose while still reporting some
// progress (biased towards expensive GCs), and while still reporting pathological cases.
static constexpr int64_t kGcStressModeGcLogSampleFrequencyNs = MsToNs(10000);
static inline bool CareAboutPauseTimes() {
return Runtime::Current()->InJankPerceptibleProcessState();
}
static void VerifyBootImagesContiguity(const std::vector<gc::space::ImageSpace*>& image_spaces) {
uint32_t boot_image_size = 0u;
for (size_t i = 0u, num_spaces = image_spaces.size(); i != num_spaces; ) {
const ImageHeader& image_header = image_spaces[i]->GetImageHeader();
uint32_t reservation_size = image_header.GetImageReservationSize();
uint32_t image_count = image_header.GetImageSpaceCount();
CHECK_NE(image_count, 0u);
CHECK_LE(image_count, num_spaces - i);
CHECK_NE(reservation_size, 0u);
for (size_t j = 1u; j != image_count; ++j) {
CHECK_EQ(image_spaces[i + j]->GetImageHeader().GetComponentCount(), 0u);
CHECK_EQ(image_spaces[i + j]->GetImageHeader().GetImageReservationSize(), 0u);
}
// Check the start of the heap.
CHECK_EQ(image_spaces[0]->Begin() + boot_image_size, image_spaces[i]->Begin());
// Check contiguous layout of images and oat files.
const uint8_t* current_heap = image_spaces[i]->Begin();
const uint8_t* current_oat = image_spaces[i]->GetImageHeader().GetOatFileBegin();
for (size_t j = 0u; j != image_count; ++j) {
const ImageHeader& current_header = image_spaces[i + j]->GetImageHeader();
CHECK_EQ(current_heap, image_spaces[i + j]->Begin());
CHECK_EQ(current_oat, current_header.GetOatFileBegin());
current_heap += RoundUp(current_header.GetImageSize(), kPageSize);
CHECK_GT(current_header.GetOatFileEnd(), current_header.GetOatFileBegin());
current_oat = current_header.GetOatFileEnd();
}
// Check that oat files start at the end of images.
CHECK_EQ(current_heap, image_spaces[i]->GetImageHeader().GetOatFileBegin());
// Check that the reservation size equals the size of images and oat files.
CHECK_EQ(reservation_size, static_cast<size_t>(current_oat - image_spaces[i]->Begin()));
boot_image_size += reservation_size;
i += image_count;
}
}
Heap::Heap(size_t initial_size,
size_t growth_limit,
size_t min_free,
size_t max_free,
double target_utilization,
double foreground_heap_growth_multiplier,
size_t stop_for_native_allocs,
size_t capacity,
size_t non_moving_space_capacity,
const std::vector<std::string>& boot_class_path,
const std::vector<std::string>& boot_class_path_locations,
ArrayRef<File> boot_class_path_files,
ArrayRef<File> boot_class_path_image_files,
ArrayRef<File> boot_class_path_vdex_files,
ArrayRef<File> boot_class_path_oat_files,
const std::vector<std::string>& image_file_names,
const InstructionSet image_instruction_set,
CollectorType foreground_collector_type,
CollectorType background_collector_type,
space::LargeObjectSpaceType large_object_space_type,
size_t large_object_threshold,
size_t parallel_gc_threads,
size_t conc_gc_threads,
bool low_memory_mode,
size_t long_pause_log_threshold,
size_t long_gc_log_threshold,
bool ignore_target_footprint,
bool always_log_explicit_gcs,
bool use_tlab,
bool verify_pre_gc_heap,
bool verify_pre_sweeping_heap,
bool verify_post_gc_heap,
bool verify_pre_gc_rosalloc,
bool verify_pre_sweeping_rosalloc,
bool verify_post_gc_rosalloc,
bool gc_stress_mode,
bool measure_gc_performance,
bool use_homogeneous_space_compaction_for_oom,
bool use_generational_cc,
uint64_t min_interval_homogeneous_space_compaction_by_oom,
bool dump_region_info_before_gc,
bool dump_region_info_after_gc)
: non_moving_space_(nullptr),
rosalloc_space_(nullptr),
dlmalloc_space_(nullptr),
main_space_(nullptr),
collector_type_(kCollectorTypeNone),
foreground_collector_type_(foreground_collector_type),
background_collector_type_(background_collector_type),
desired_collector_type_(foreground_collector_type_),
pending_task_lock_(nullptr),
parallel_gc_threads_(parallel_gc_threads),
conc_gc_threads_(conc_gc_threads),
low_memory_mode_(low_memory_mode),
long_pause_log_threshold_(long_pause_log_threshold),
long_gc_log_threshold_(long_gc_log_threshold),
process_cpu_start_time_ns_(ProcessCpuNanoTime()),
pre_gc_last_process_cpu_time_ns_(process_cpu_start_time_ns_),
post_gc_last_process_cpu_time_ns_(process_cpu_start_time_ns_),
pre_gc_weighted_allocated_bytes_(0.0),
post_gc_weighted_allocated_bytes_(0.0),
ignore_target_footprint_(ignore_target_footprint),
always_log_explicit_gcs_(always_log_explicit_gcs),
zygote_creation_lock_("zygote creation lock", kZygoteCreationLock),
zygote_space_(nullptr),
large_object_threshold_(large_object_threshold),
disable_thread_flip_count_(0),
thread_flip_running_(false),
collector_type_running_(kCollectorTypeNone),
last_gc_cause_(kGcCauseNone),
thread_running_gc_(nullptr),
last_gc_type_(collector::kGcTypeNone),
next_gc_type_(collector::kGcTypePartial),
capacity_(capacity),
growth_limit_(growth_limit),
initial_heap_size_(initial_size),
target_footprint_(initial_size),
// Using kPostMonitorLock as a lock at kDefaultMutexLevel is acquired after
// this one.
process_state_update_lock_("process state update lock", kPostMonitorLock),
min_foreground_target_footprint_(0),
min_foreground_concurrent_start_bytes_(0),
concurrent_start_bytes_(std::numeric_limits<size_t>::max()),
total_bytes_freed_ever_(0),
total_objects_freed_ever_(0),
num_bytes_allocated_(0),
native_bytes_registered_(0),
old_native_bytes_allocated_(0),
native_objects_notified_(0),
num_bytes_freed_revoke_(0),
num_bytes_alive_after_gc_(0),
verify_missing_card_marks_(false),
verify_system_weaks_(false),
verify_pre_gc_heap_(verify_pre_gc_heap),
verify_pre_sweeping_heap_(verify_pre_sweeping_heap),
verify_post_gc_heap_(verify_post_gc_heap),
verify_mod_union_table_(false),
verify_pre_gc_rosalloc_(verify_pre_gc_rosalloc),
verify_pre_sweeping_rosalloc_(verify_pre_sweeping_rosalloc),
verify_post_gc_rosalloc_(verify_post_gc_rosalloc),
gc_stress_mode_(gc_stress_mode),
/* For GC a lot mode, we limit the allocation stacks to be kGcAlotInterval allocations. This
* causes a lot of GC since we do a GC for alloc whenever the stack is full. When heap
* verification is enabled, we limit the size of allocation stacks to speed up their
* searching.
*/
max_allocation_stack_size_(kGCALotMode
? kGcAlotAllocationStackSize
: (kVerifyObjectSupport > kVerifyObjectModeFast)
? kVerifyObjectAllocationStackSize
: kDefaultAllocationStackSize),
current_allocator_(kAllocatorTypeDlMalloc),
current_non_moving_allocator_(kAllocatorTypeNonMoving),
bump_pointer_space_(nullptr),
temp_space_(nullptr),
region_space_(nullptr),
min_free_(min_free),
max_free_(max_free),
target_utilization_(target_utilization),
foreground_heap_growth_multiplier_(foreground_heap_growth_multiplier),
stop_for_native_allocs_(stop_for_native_allocs),
total_wait_time_(0),
verify_object_mode_(kVerifyObjectModeDisabled),
disable_moving_gc_count_(0),
semi_space_collector_(nullptr),
active_concurrent_copying_collector_(nullptr),
young_concurrent_copying_collector_(nullptr),
concurrent_copying_collector_(nullptr),
is_running_on_memory_tool_(Runtime::Current()->IsRunningOnMemoryTool()),
use_tlab_(use_tlab),
main_space_backup_(nullptr),
min_interval_homogeneous_space_compaction_by_oom_(
min_interval_homogeneous_space_compaction_by_oom),
last_time_homogeneous_space_compaction_by_oom_(NanoTime()),
gcs_completed_(0u),
max_gc_requested_(0u),
pending_collector_transition_(nullptr),
pending_heap_trim_(nullptr),
use_homogeneous_space_compaction_for_oom_(use_homogeneous_space_compaction_for_oom),
use_generational_cc_(use_generational_cc),
running_collection_is_blocking_(false),
blocking_gc_count_(0U),
blocking_gc_time_(0U),
last_update_time_gc_count_rate_histograms_( // Round down by the window duration.
(NanoTime() / kGcCountRateHistogramWindowDuration) * kGcCountRateHistogramWindowDuration),
gc_count_last_window_(0U),
blocking_gc_count_last_window_(0U),
gc_count_rate_histogram_("gc count rate histogram", 1U, kGcCountRateMaxBucketCount),
blocking_gc_count_rate_histogram_(
"blocking gc count rate histogram", 1U, kGcCountRateMaxBucketCount),
alloc_tracking_enabled_(false),
alloc_record_depth_(AllocRecordObjectMap::kDefaultAllocStackDepth),
backtrace_lock_(nullptr),
seen_backtrace_count_(0u),
unique_backtrace_count_(0u),
gc_disabled_for_shutdown_(false),
dump_region_info_before_gc_(dump_region_info_before_gc),
dump_region_info_after_gc_(dump_region_info_after_gc),
boot_image_spaces_(),
boot_images_start_address_(0u),
boot_images_size_(0u),
pre_oome_gc_count_(0u) {
if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
LOG(INFO) << "Heap() entering";
}
LOG(INFO) << "Using " << foreground_collector_type_ << " GC.";
if (gUseUserfaultfd) {
CHECK_EQ(foreground_collector_type_, kCollectorTypeCMC);
CHECK_EQ(background_collector_type_, kCollectorTypeCMCBackground);
} else {
// This ensures that userfaultfd syscall is done before any seccomp filter is installed.
// TODO(b/266731037): Remove this when we no longer need to collect metric on userfaultfd
// support.
auto [uffd_supported, minor_fault_supported] = collector::MarkCompact::GetUffdAndMinorFault();
// The check is just to ensure that compiler doesn't eliminate the function call above.
// Userfaultfd support is certain to be there if its minor-fault feature is supported.
CHECK_IMPLIES(minor_fault_supported, uffd_supported);
}
if (gUseReadBarrier) {
CHECK_EQ(foreground_collector_type_, kCollectorTypeCC);
CHECK_EQ(background_collector_type_, kCollectorTypeCCBackground);
} else if (background_collector_type_ != gc::kCollectorTypeHomogeneousSpaceCompact) {
CHECK_EQ(IsMovingGc(foreground_collector_type_), IsMovingGc(background_collector_type_))
<< "Changing from " << foreground_collector_type_ << " to "
<< background_collector_type_ << " (or visa versa) is not supported.";
}
verification_.reset(new Verification(this));
CHECK_GE(large_object_threshold, kMinLargeObjectThreshold);
ScopedTrace trace(__FUNCTION__);
Runtime* const runtime = Runtime::Current();
// If we aren't the zygote, switch to the default non zygote allocator. This may update the
// entrypoints.
const bool is_zygote = runtime->IsZygote();
if (!is_zygote) {
// Background compaction is currently not supported for command line runs.
if (background_collector_type_ != foreground_collector_type_) {
VLOG(heap) << "Disabling background compaction for non zygote";
background_collector_type_ = foreground_collector_type_;
}
}
ChangeCollector(desired_collector_type_);
live_bitmap_.reset(new accounting::HeapBitmap(this));
mark_bitmap_.reset(new accounting::HeapBitmap(this));
// We don't have hspace compaction enabled with CC.
if (foreground_collector_type_ == kCollectorTypeCC
|| foreground_collector_type_ == kCollectorTypeCMC) {
use_homogeneous_space_compaction_for_oom_ = false;
}
bool support_homogeneous_space_compaction =
background_collector_type_ == gc::kCollectorTypeHomogeneousSpaceCompact ||
use_homogeneous_space_compaction_for_oom_;
// We may use the same space the main space for the non moving space if we don't need to compact
// from the main space.
// This is not the case if we support homogeneous compaction or have a moving background
// collector type.
bool separate_non_moving_space = is_zygote ||
support_homogeneous_space_compaction || IsMovingGc(foreground_collector_type_) ||
IsMovingGc(background_collector_type_);
// Requested begin for the alloc space, to follow the mapped image and oat files
uint8_t* request_begin = nullptr;
// Calculate the extra space required after the boot image, see allocations below.
size_t heap_reservation_size = 0u;
if (separate_non_moving_space) {
heap_reservation_size = non_moving_space_capacity;
} else if (foreground_collector_type_ != kCollectorTypeCC && is_zygote) {
heap_reservation_size = capacity_;
}
heap_reservation_size = RoundUp(heap_reservation_size, kPageSize);
// Load image space(s).
std::vector<std::unique_ptr<space::ImageSpace>> boot_image_spaces;
MemMap heap_reservation;
if (space::ImageSpace::LoadBootImage(boot_class_path,
boot_class_path_locations,
boot_class_path_files,
boot_class_path_image_files,
boot_class_path_vdex_files,
boot_class_path_oat_files,
image_file_names,
image_instruction_set,
runtime->ShouldRelocate(),
/*executable=*/!runtime->IsAotCompiler(),
heap_reservation_size,
runtime->AllowInMemoryCompilation(),
runtime->GetApexVersions(),
&boot_image_spaces,
&heap_reservation)) {
DCHECK_EQ(heap_reservation_size, heap_reservation.IsValid() ? heap_reservation.Size() : 0u);
DCHECK(!boot_image_spaces.empty());
request_begin = boot_image_spaces.back()->GetImageHeader().GetOatFileEnd();
DCHECK_IMPLIES(heap_reservation.IsValid(), request_begin == heap_reservation.Begin())
<< "request_begin=" << static_cast<const void*>(request_begin)
<< " heap_reservation.Begin()=" << static_cast<const void*>(heap_reservation.Begin());
for (std::unique_ptr<space::ImageSpace>& space : boot_image_spaces) {
boot_image_spaces_.push_back(space.get());
AddSpace(space.release());
}
boot_images_start_address_ = PointerToLowMemUInt32(boot_image_spaces_.front()->Begin());
uint32_t boot_images_end =
PointerToLowMemUInt32(boot_image_spaces_.back()->GetImageHeader().GetOatFileEnd());
boot_images_size_ = boot_images_end - boot_images_start_address_;
if (kIsDebugBuild) {
VerifyBootImagesContiguity(boot_image_spaces_);
}
} else {
if (foreground_collector_type_ == kCollectorTypeCC) {
// Need to use a low address so that we can allocate a contiguous 2 * Xmx space
// when there's no image (dex2oat for target).
request_begin = kPreferredAllocSpaceBegin;
}
// Gross hack to make dex2oat deterministic.
if (foreground_collector_type_ == kCollectorTypeMS && Runtime::Current()->IsAotCompiler()) {
// Currently only enabled for MS collector since that is what the deterministic dex2oat uses.
// b/26849108
request_begin = reinterpret_cast<uint8_t*>(kAllocSpaceBeginForDeterministicAoT);
}
}
/*
requested_alloc_space_begin -> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
+- nonmoving space (non_moving_space_capacity)+-
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
+-????????????????????????????????????????????+-
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
+-main alloc space / bump space 1 (capacity_) +-
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
+-????????????????????????????????????????????+-
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
+-main alloc space2 / bump space 2 (capacity_)+-
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-
*/
MemMap main_mem_map_1;
MemMap main_mem_map_2;
std::string error_str;
MemMap non_moving_space_mem_map;
if (separate_non_moving_space) {
ScopedTrace trace2("Create separate non moving space");
// If we are the zygote, the non moving space becomes the zygote space when we run
// PreZygoteFork the first time. In this case, call the map "zygote space" since we can't
// rename the mem map later.
const char* space_name = is_zygote ? kZygoteSpaceName : kNonMovingSpaceName;
// Reserve the non moving mem map before the other two since it needs to be at a specific
// address.
DCHECK_EQ(heap_reservation.IsValid(), !boot_image_spaces_.empty());
if (heap_reservation.IsValid()) {
non_moving_space_mem_map = heap_reservation.RemapAtEnd(
heap_reservation.Begin(), space_name, PROT_READ | PROT_WRITE, &error_str);
} else {
non_moving_space_mem_map = MapAnonymousPreferredAddress(
space_name, request_begin, non_moving_space_capacity, &error_str);
}
CHECK(non_moving_space_mem_map.IsValid()) << error_str;
DCHECK(!heap_reservation.IsValid());
// Try to reserve virtual memory at a lower address if we have a separate non moving space.
request_begin = kPreferredAllocSpaceBegin + non_moving_space_capacity;
}
// Attempt to create 2 mem maps at or after the requested begin.
if (foreground_collector_type_ != kCollectorTypeCC) {
ScopedTrace trace2("Create main mem map");
if (separate_non_moving_space || !is_zygote) {
main_mem_map_1 = MapAnonymousPreferredAddress(
kMemMapSpaceName[0], request_begin, capacity_, &error_str);
} else {
// If no separate non-moving space and we are the zygote, the main space must come right after
// the image space to avoid a gap. This is required since we want the zygote space to be
// adjacent to the image space.
DCHECK_EQ(heap_reservation.IsValid(), !boot_image_spaces_.empty());
main_mem_map_1 = MemMap::MapAnonymous(
kMemMapSpaceName[0],
request_begin,
capacity_,
PROT_READ | PROT_WRITE,
/* low_4gb= */ true,
/* reuse= */ false,
heap_reservation.IsValid() ? &heap_reservation : nullptr,
&error_str);
}
CHECK(main_mem_map_1.IsValid()) << error_str;
DCHECK(!heap_reservation.IsValid());
}
if (support_homogeneous_space_compaction ||
background_collector_type_ == kCollectorTypeSS ||
foreground_collector_type_ == kCollectorTypeSS) {
ScopedTrace trace2("Create main mem map 2");
main_mem_map_2 = MapAnonymousPreferredAddress(
kMemMapSpaceName[1], main_mem_map_1.End(), capacity_, &error_str);
CHECK(main_mem_map_2.IsValid()) << error_str;
}
// Create the non moving space first so that bitmaps don't take up the address range.
if (separate_non_moving_space) {
ScopedTrace trace2("Add non moving space");
// Non moving space is always dlmalloc since we currently don't have support for multiple
// active rosalloc spaces.
const size_t size = non_moving_space_mem_map.Size();
const void* non_moving_space_mem_map_begin = non_moving_space_mem_map.Begin();
non_moving_space_ = space::DlMallocSpace::CreateFromMemMap(std::move(non_moving_space_mem_map),
"zygote / non moving space",
kDefaultStartingSize,
initial_size,
size,
size,
/* can_move_objects= */ false);
CHECK(non_moving_space_ != nullptr) << "Failed creating non moving space "
<< non_moving_space_mem_map_begin;
non_moving_space_->SetFootprintLimit(non_moving_space_->Capacity());
AddSpace(non_moving_space_);
}
// Create other spaces based on whether or not we have a moving GC.
if (foreground_collector_type_ == kCollectorTypeCC) {
CHECK(separate_non_moving_space);
// Reserve twice the capacity, to allow evacuating every region for explicit GCs.
MemMap region_space_mem_map =
space::RegionSpace::CreateMemMap(kRegionSpaceName, capacity_ * 2, request_begin);
CHECK(region_space_mem_map.IsValid()) << "No region space mem map";
region_space_ = space::RegionSpace::Create(
kRegionSpaceName, std::move(region_space_mem_map), use_generational_cc_);
AddSpace(region_space_);
} else if (IsMovingGc(foreground_collector_type_)) {
// Create bump pointer spaces.
// We only to create the bump pointer if the foreground collector is a compacting GC.
// TODO: Place bump-pointer spaces somewhere to minimize size of card table.
bump_pointer_space_ = space::BumpPointerSpace::CreateFromMemMap("Bump pointer space 1",
std::move(main_mem_map_1));
CHECK(bump_pointer_space_ != nullptr) << "Failed to create bump pointer space";
AddSpace(bump_pointer_space_);
// For Concurrent Mark-compact GC we don't need the temp space to be in
// lower 4GB. So its temp space will be created by the GC itself.
if (foreground_collector_type_ != kCollectorTypeCMC) {
temp_space_ = space::BumpPointerSpace::CreateFromMemMap("Bump pointer space 2",
std::move(main_mem_map_2));
CHECK(temp_space_ != nullptr) << "Failed to create bump pointer space";
AddSpace(temp_space_);
}
CHECK(separate_non_moving_space);
} else {
CreateMainMallocSpace(std::move(main_mem_map_1), initial_size, growth_limit_, capacity_);
CHECK(main_space_ != nullptr);
AddSpace(main_space_);
if (!separate_non_moving_space) {
non_moving_space_ = main_space_;
CHECK(!non_moving_space_->CanMoveObjects());
}
if (main_mem_map_2.IsValid()) {
const char* name = kUseRosAlloc ? kRosAllocSpaceName[1] : kDlMallocSpaceName[1];
main_space_backup_.reset(CreateMallocSpaceFromMemMap(std::move(main_mem_map_2),
initial_size,
growth_limit_,
capacity_,
name,
/* can_move_objects= */ true));
CHECK(main_space_backup_.get() != nullptr);
// Add the space so its accounted for in the heap_begin and heap_end.
AddSpace(main_space_backup_.get());
}
}
CHECK(non_moving_space_ != nullptr);
CHECK(!non_moving_space_->CanMoveObjects());
// Allocate the large object space.
if (large_object_space_type == space::LargeObjectSpaceType::kFreeList) {
large_object_space_ = space::FreeListSpace::Create("free list large object space", capacity_);
CHECK(large_object_space_ != nullptr) << "Failed to create large object space";
} else if (large_object_space_type == space::LargeObjectSpaceType::kMap) {
large_object_space_ = space::LargeObjectMapSpace::Create("mem map large object space");
CHECK(large_object_space_ != nullptr) << "Failed to create large object space";
} else {
// Disable the large object space by making the cutoff excessively large.
large_object_threshold_ = std::numeric_limits<size_t>::max();
large_object_space_ = nullptr;
}
if (large_object_space_ != nullptr) {
AddSpace(large_object_space_);
}
// Compute heap capacity. Continuous spaces are sorted in order of Begin().
CHECK(!continuous_spaces_.empty());
// Relies on the spaces being sorted.
uint8_t* heap_begin = continuous_spaces_.front()->Begin();
uint8_t* heap_end = continuous_spaces_.back()->Limit();
size_t heap_capacity = heap_end - heap_begin;
// Remove the main backup space since it slows down the GC to have unused extra spaces.
// TODO: Avoid needing to do this.
if (main_space_backup_.get() != nullptr) {
RemoveSpace(main_space_backup_.get());
}
// Allocate the card table.
// We currently don't support dynamically resizing the card table.
// Since we don't know where in the low_4gb the app image will be located, make the card table
// cover the whole low_4gb. TODO: Extend the card table in AddSpace.
UNUSED(heap_capacity);
// Start at 4 KB, we can be sure there are no spaces mapped this low since the address range is
// reserved by the kernel.
static constexpr size_t kMinHeapAddress = 4 * KB;
card_table_.reset(accounting::CardTable::Create(reinterpret_cast<uint8_t*>(kMinHeapAddress),
4 * GB - kMinHeapAddress));
CHECK(card_table_.get() != nullptr) << "Failed to create card table";
if (foreground_collector_type_ == kCollectorTypeCC && kUseTableLookupReadBarrier) {
rb_table_.reset(new accounting::ReadBarrierTable());
DCHECK(rb_table_->IsAllCleared());
}
if (HasBootImageSpace()) {
// Don't add the image mod union table if we are running without an image, this can crash if
// we use the CardCache implementation.
for (space::ImageSpace* image_space : GetBootImageSpaces()) {
accounting::ModUnionTable* mod_union_table = new accounting::ModUnionTableToZygoteAllocspace(
"Image mod-union table", this, image_space);
CHECK(mod_union_table != nullptr) << "Failed to create image mod-union table";
AddModUnionTable(mod_union_table);
}
}
if (collector::SemiSpace::kUseRememberedSet && non_moving_space_ != main_space_) {
accounting::RememberedSet* non_moving_space_rem_set =
new accounting::RememberedSet("Non-moving space remembered set", this, non_moving_space_);
CHECK(non_moving_space_rem_set != nullptr) << "Failed to create non-moving space remembered set";
AddRememberedSet(non_moving_space_rem_set);
}
// TODO: Count objects in the image space here?
num_bytes_allocated_.store(0, std::memory_order_relaxed);
mark_stack_.reset(accounting::ObjectStack::Create("mark stack", kDefaultMarkStackSize,
kDefaultMarkStackSize));
const size_t alloc_stack_capacity = max_allocation_stack_size_ + kAllocationStackReserveSize;
allocation_stack_.reset(accounting::ObjectStack::Create(
"allocation stack", max_allocation_stack_size_, alloc_stack_capacity));
live_stack_.reset(accounting::ObjectStack::Create(
"live stack", max_allocation_stack_size_, alloc_stack_capacity));
// It's still too early to take a lock because there are no threads yet, but we can create locks
// now. We don't create it earlier to make it clear that you can't use locks during heap
// initialization.
gc_complete_lock_ = new Mutex("GC complete lock");
gc_complete_cond_.reset(new ConditionVariable("GC complete condition variable",
*gc_complete_lock_));
thread_flip_lock_ = new Mutex("GC thread flip lock");
thread_flip_cond_.reset(new ConditionVariable("GC thread flip condition variable",
*thread_flip_lock_));
task_processor_.reset(new TaskProcessor());
reference_processor_.reset(new ReferenceProcessor());
pending_task_lock_ = new Mutex("Pending task lock");
if (ignore_target_footprint_) {
SetIdealFootprint(std::numeric_limits<size_t>::max());
concurrent_start_bytes_ = std::numeric_limits<size_t>::max();
}
CHECK_NE(target_footprint_.load(std::memory_order_relaxed), 0U);
// Create our garbage collectors.
for (size_t i = 0; i < 2; ++i) {
const bool concurrent = i != 0;
if ((MayUseCollector(kCollectorTypeCMS) && concurrent) ||
(MayUseCollector(kCollectorTypeMS) && !concurrent)) {
garbage_collectors_.push_back(new collector::MarkSweep(this, concurrent));
garbage_collectors_.push_back(new collector::PartialMarkSweep(this, concurrent));
garbage_collectors_.push_back(new collector::StickyMarkSweep(this, concurrent));
}
}
if (kMovingCollector) {
if (MayUseCollector(kCollectorTypeSS) ||
MayUseCollector(kCollectorTypeHomogeneousSpaceCompact) ||
use_homogeneous_space_compaction_for_oom_) {
semi_space_collector_ = new collector::SemiSpace(this);
garbage_collectors_.push_back(semi_space_collector_);
}
if (MayUseCollector(kCollectorTypeCMC)) {
mark_compact_ = new collector::MarkCompact(this);
garbage_collectors_.push_back(mark_compact_);
}
if (MayUseCollector(kCollectorTypeCC)) {
concurrent_copying_collector_ = new collector::ConcurrentCopying(this,
/*young_gen=*/false,
use_generational_cc_,
"",
measure_gc_performance);
if (use_generational_cc_) {
young_concurrent_copying_collector_ = new collector::ConcurrentCopying(
this,
/*young_gen=*/true,
use_generational_cc_,
"young",
measure_gc_performance);
}
active_concurrent_copying_collector_.store(concurrent_copying_collector_,
std::memory_order_relaxed);
DCHECK(region_space_ != nullptr);
concurrent_copying_collector_->SetRegionSpace(region_space_);
if (use_generational_cc_) {
young_concurrent_copying_collector_->SetRegionSpace(region_space_);
// At this point, non-moving space should be created.
DCHECK(non_moving_space_ != nullptr);
concurrent_copying_collector_->CreateInterRegionRefBitmaps();
}
garbage_collectors_.push_back(concurrent_copying_collector_);
if (use_generational_cc_) {
garbage_collectors_.push_back(young_concurrent_copying_collector_);
}
}
}
if (!GetBootImageSpaces().empty() && non_moving_space_ != nullptr &&
(is_zygote || separate_non_moving_space)) {
// Check that there's no gap between the image space and the non moving space so that the
// immune region won't break (eg. due to a large object allocated in the gap). This is only
// required when we're the zygote.
// Space with smallest Begin().
space::ImageSpace* first_space = nullptr;
for (space::ImageSpace* space : boot_image_spaces_) {
if (first_space == nullptr || space->Begin() < first_space->Begin()) {
first_space = space;
}
}
bool no_gap = MemMap::CheckNoGaps(*first_space->GetMemMap(), *non_moving_space_->GetMemMap());
if (!no_gap) {
PrintFileToLog("/proc/self/maps", LogSeverity::ERROR);
MemMap::DumpMaps(LOG_STREAM(ERROR), /* terse= */ true);
LOG(FATAL) << "There's a gap between the image space and the non-moving space";
}
}
// Perfetto Java Heap Profiler Support.
if (runtime->IsPerfettoJavaHeapStackProfEnabled()) {
// Perfetto Plugin is loaded and enabled, initialize the Java Heap Profiler.
InitPerfettoJavaHeapProf();
} else {
// Disable the Java Heap Profiler.
GetHeapSampler().DisableHeapSampler();
}
instrumentation::Instrumentation* const instrumentation = runtime->GetInstrumentation();
if (gc_stress_mode_) {
backtrace_lock_ = new Mutex("GC complete lock");
}
if (is_running_on_memory_tool_ || gc_stress_mode_) {
instrumentation->InstrumentQuickAllocEntryPoints();
}
if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
LOG(INFO) << "Heap() exiting";
}
}
MemMap Heap::MapAnonymousPreferredAddress(const char* name,
uint8_t* request_begin,
size_t capacity,
std::string* out_error_str) {
while (true) {
MemMap map = MemMap::MapAnonymous(name,
request_begin,
capacity,
PROT_READ | PROT_WRITE,
/*low_4gb=*/ true,
/*reuse=*/ false,
/*reservation=*/ nullptr,
out_error_str);
if (map.IsValid() || request_begin == nullptr) {
return map;
}
// Retry a second time with no specified request begin.
request_begin = nullptr;
}
}
bool Heap::MayUseCollector(CollectorType type) const {
return foreground_collector_type_ == type || background_collector_type_ == type;
}
space::MallocSpace* Heap::CreateMallocSpaceFromMemMap(MemMap&& mem_map,
size_t initial_size,
size_t growth_limit,
size_t capacity,
const char* name,
bool can_move_objects) {
space::MallocSpace* malloc_space = nullptr;
if (kUseRosAlloc) {
// Create rosalloc space.
malloc_space = space::RosAllocSpace::CreateFromMemMap(std::move(mem_map),
name,
kDefaultStartingSize,
initial_size,
growth_limit,
capacity,
low_memory_mode_,
can_move_objects);
} else {
malloc_space = space::DlMallocSpace::CreateFromMemMap(std::move(mem_map),
name,
kDefaultStartingSize,
initial_size,
growth_limit,
capacity,
can_move_objects);
}
if (collector::SemiSpace::kUseRememberedSet) {
accounting::RememberedSet* rem_set =
new accounting::RememberedSet(std::string(name) + " remembered set", this, malloc_space);
CHECK(rem_set != nullptr) << "Failed to create main space remembered set";
AddRememberedSet(rem_set);
}
CHECK(malloc_space != nullptr) << "Failed to create " << name;
malloc_space->SetFootprintLimit(malloc_space->Capacity());
return malloc_space;
}
void Heap::CreateMainMallocSpace(MemMap&& mem_map,
size_t initial_size,
size_t growth_limit,
size_t capacity) {
// Is background compaction is enabled?
bool can_move_objects = IsMovingGc(background_collector_type_) !=
IsMovingGc(foreground_collector_type_) || use_homogeneous_space_compaction_for_oom_;
// If we are the zygote and don't yet have a zygote space, it means that the zygote fork will
// happen in the future. If this happens and we have kCompactZygote enabled we wish to compact
// from the main space to the zygote space. If background compaction is enabled, always pass in
// that we can move objets.
if (kCompactZygote && Runtime::Current()->IsZygote() && !can_move_objects) {
// After the zygote we want this to be false if we don't have background compaction enabled so
// that getting primitive array elements is faster.
can_move_objects = !HasZygoteSpace();
}
if (collector::SemiSpace::kUseRememberedSet && main_space_ != nullptr) {
RemoveRememberedSet(main_space_);
}
const char* name = kUseRosAlloc ? kRosAllocSpaceName[0] : kDlMallocSpaceName[0];
main_space_ = CreateMallocSpaceFromMemMap(std::move(mem_map),
initial_size,
growth_limit,
capacity, name,
can_move_objects);
SetSpaceAsDefault(main_space_);
VLOG(heap) << "Created main space " << main_space_;
}
void Heap::ChangeAllocator(AllocatorType allocator) {
if (current_allocator_ != allocator) {
// These two allocators are only used internally and don't have any entrypoints.
CHECK_NE(allocator, kAllocatorTypeLOS);
CHECK_NE(allocator, kAllocatorTypeNonMoving);
current_allocator_ = allocator;
MutexLock mu(nullptr, *Locks::runtime_shutdown_lock_);
SetQuickAllocEntryPointsAllocator(current_allocator_);
Runtime::Current()->GetInstrumentation()->ResetQuickAllocEntryPoints();
}
}
bool Heap::IsCompilingBoot() const {
if (!Runtime::Current()->IsAotCompiler()) {
return false;
}
ScopedObjectAccess soa(Thread::Current());
for (const auto& space : continuous_spaces_) {
if (space->IsImageSpace() || space->IsZygoteSpace()) {
return false;
}
}
return true;
}
void Heap::IncrementDisableMovingGC(Thread* self) {
// Need to do this holding the lock to prevent races where the GC is about to run / running when
// we attempt to disable it.
ScopedThreadStateChange tsc(self, ThreadState::kWaitingForGcToComplete);
MutexLock mu(self, *gc_complete_lock_);
++disable_moving_gc_count_;
if (IsMovingGc(collector_type_running_)) {
WaitForGcToCompleteLocked(kGcCauseDisableMovingGc, self);
}
}
void Heap::DecrementDisableMovingGC(Thread* self) {
MutexLock mu(self, *gc_complete_lock_);
CHECK_GT(disable_moving_gc_count_, 0U);
--disable_moving_gc_count_;
}
void Heap::IncrementDisableThreadFlip(Thread* self) {
// Supposed to be called by mutators. If thread_flip_running_ is true, block. Otherwise, go ahead.
bool is_nested = self->GetDisableThreadFlipCount() > 0;
self->IncrementDisableThreadFlipCount();
if (is_nested) {
// If this is a nested JNI critical section enter, we don't need to wait or increment the global
// counter. The global counter is incremented only once for a thread for the outermost enter.
return;
}
ScopedThreadStateChange tsc(self, ThreadState::kWaitingForGcThreadFlip);
MutexLock mu(self, *thread_flip_lock_);
thread_flip_cond_->CheckSafeToWait(self);
bool has_waited = false;
uint64_t wait_start = 0;
if (thread_flip_running_) {
wait_start = NanoTime();
ScopedTrace trace("IncrementDisableThreadFlip");
while (thread_flip_running_) {
has_waited = true;
thread_flip_cond_->Wait(self);
}
}
++disable_thread_flip_count_;
if (has_waited) {
uint64_t wait_time = NanoTime() - wait_start;
total_wait_time_ += wait_time;
if (wait_time > long_pause_log_threshold_) {
LOG(INFO) << __FUNCTION__ << " blocked for " << PrettyDuration(wait_time);
}
}
}
void Heap::EnsureObjectUserfaulted(ObjPtr<mirror::Object> obj) {
if (gUseUserfaultfd) {
// Use volatile to ensure that compiler loads from memory to trigger userfaults, if required.
const uint8_t* start = reinterpret_cast<uint8_t*>(obj.Ptr());
const uint8_t* end = AlignUp(start + obj->SizeOf(), kPageSize);
// The first page is already touched by SizeOf().
start += kPageSize;
while (start < end) {
ForceRead(start);
start += kPageSize;
}
}
}
void Heap::DecrementDisableThreadFlip(Thread* self) {
// Supposed to be called by mutators. Decrement disable_thread_flip_count_ and potentially wake up
// the GC waiting before doing a thread flip.
self->DecrementDisableThreadFlipCount();
bool is_outermost = self->GetDisableThreadFlipCount() == 0;
if (!is_outermost) {
// If this is not an outermost JNI critical exit, we don't need to decrement the global counter.
// The global counter is decremented only once for a thread for the outermost exit.
return;
}
MutexLock mu(self, *thread_flip_lock_);
CHECK_GT(disable_thread_flip_count_, 0U);
--disable_thread_flip_count_;
if (disable_thread_flip_count_ == 0) {
// Potentially notify the GC thread blocking to begin a thread flip.
thread_flip_cond_->Broadcast(self);
}
}
void Heap::ThreadFlipBegin(Thread* self) {
// Supposed to be called by GC. Set thread_flip_running_ to be true. If disable_thread_flip_count_
// > 0, block. Otherwise, go ahead.
ScopedThreadStateChange tsc(self, ThreadState::kWaitingForGcThreadFlip);
MutexLock mu(self, *thread_flip_lock_);
thread_flip_cond_->CheckSafeToWait(self);
bool has_waited = false;
uint64_t wait_start = NanoTime();
CHECK(!thread_flip_running_);
// Set this to true before waiting so that frequent JNI critical enter/exits won't starve
// GC. This like a writer preference of a reader-writer lock.
thread_flip_running_ = true;
while (disable_thread_flip_count_ > 0) {
has_waited = true;
thread_flip_cond_->Wait(self);
}
if (has_waited) {
uint64_t wait_time = NanoTime() - wait_start;
total_wait_time_ += wait_time;
if (wait_time > long_pause_log_threshold_) {
LOG(INFO) << __FUNCTION__ << " blocked for " << PrettyDuration(wait_time);
}
}
}
void Heap::ThreadFlipEnd(Thread* self) {
// Supposed to be called by GC. Set thread_flip_running_ to false and potentially wake up mutators
// waiting before doing a JNI critical.
MutexLock mu(self, *thread_flip_lock_);
CHECK(thread_flip_running_);
thread_flip_running_ = false;
// Potentially notify mutator threads blocking to enter a JNI critical section.
thread_flip_cond_->Broadcast(self);
}
void Heap::GrowHeapOnJankPerceptibleSwitch() {
MutexLock mu(Thread::Current(), process_state_update_lock_);
size_t orig_target_footprint = target_footprint_.load(std::memory_order_relaxed);
if (orig_target_footprint < min_foreground_target_footprint_) {
target_footprint_.compare_exchange_strong(orig_target_footprint,
min_foreground_target_footprint_,
std::memory_order_relaxed);
}
if (IsGcConcurrent() && concurrent_start_bytes_ < min_foreground_concurrent_start_bytes_) {
concurrent_start_bytes_ = min_foreground_concurrent_start_bytes_;
}
}
void Heap::UpdateProcessState(ProcessState old_process_state, ProcessState new_process_state) {
if (old_process_state != new_process_state) {
const bool jank_perceptible = new_process_state == kProcessStateJankPerceptible;
if (jank_perceptible) {
// Transition back to foreground right away to prevent jank.
RequestCollectorTransition(foreground_collector_type_, 0);
GrowHeapOnJankPerceptibleSwitch();
} else {
// If background_collector_type_ is kCollectorTypeHomogeneousSpaceCompact then we have
// special handling which does a homogenous space compaction once but then doesn't transition
// the collector. Similarly, we invoke a full compaction for kCollectorTypeCC but don't
// transition the collector.
RequestCollectorTransition(background_collector_type_, 0);
}
}
}
void Heap::CreateThreadPool(size_t num_threads) {
if (num_threads == 0) {
num_threads = std::max(parallel_gc_threads_, conc_gc_threads_);
}
if (num_threads != 0) {
thread_pool_.reset(new ThreadPool("Heap thread pool", num_threads));
}
}
void Heap::WaitForWorkersToBeCreated() {
DCHECK(!Runtime::Current()->IsShuttingDown(Thread::Current()))
<< "Cannot create new threads during runtime shutdown";
if (thread_pool_ != nullptr) {
thread_pool_->WaitForWorkersToBeCreated();
}
}
void Heap::MarkAllocStackAsLive(accounting::ObjectStack* stack) {
space::ContinuousSpace* space1 = main_space_ != nullptr ? main_space_ : non_moving_space_;
space::ContinuousSpace* space2 = non_moving_space_;
// TODO: Generalize this to n bitmaps?
CHECK(space1 != nullptr);
CHECK(space2 != nullptr);
MarkAllocStack(space1->GetLiveBitmap(), space2->GetLiveBitmap(),
(large_object_space_ != nullptr ? large_object_space_->GetLiveBitmap() : nullptr),
stack);
}
void Heap::DeleteThreadPool() {
thread_pool_.reset(nullptr);
}
void Heap::AddSpace(space::Space* space) {
CHECK(space != nullptr);
WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
if (space->IsContinuousSpace()) {
DCHECK(!space->IsDiscontinuousSpace());
space::ContinuousSpace* continuous_space = space->AsContinuousSpace();
// Continuous spaces don't necessarily have bitmaps.
accounting::ContinuousSpaceBitmap* live_bitmap = continuous_space->GetLiveBitmap();
accounting::ContinuousSpaceBitmap* mark_bitmap = continuous_space->GetMarkBitmap();
// The region space bitmap is not added since VisitObjects visits the region space objects with
// special handling.
if (live_bitmap != nullptr && !space->IsRegionSpace()) {
CHECK(mark_bitmap != nullptr);
live_bitmap_->AddContinuousSpaceBitmap(live_bitmap);
mark_bitmap_->AddContinuousSpaceBitmap(mark_bitmap);
}
continuous_spaces_.push_back(continuous_space);
// Ensure that spaces remain sorted in increasing order of start address.
std::sort(continuous_spaces_.begin(), continuous_spaces_.end(),
[](const space::ContinuousSpace* a, const space::ContinuousSpace* b) {
return a->Begin() < b->Begin();
});
} else {
CHECK(space->IsDiscontinuousSpace());
space::DiscontinuousSpace* discontinuous_space = space->AsDiscontinuousSpace();
live_bitmap_->AddLargeObjectBitmap(discontinuous_space->GetLiveBitmap());
mark_bitmap_->AddLargeObjectBitmap(discontinuous_space->GetMarkBitmap());
discontinuous_spaces_.push_back(discontinuous_space);
}
if (space->IsAllocSpace()) {
alloc_spaces_.push_back(space->AsAllocSpace());
}
}
void Heap::SetSpaceAsDefault(space::ContinuousSpace* continuous_space) {
WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
if (continuous_space->IsDlMallocSpace()) {
dlmalloc_space_ = continuous_space->AsDlMallocSpace();
} else if (continuous_space->IsRosAllocSpace()) {
rosalloc_space_ = continuous_space->AsRosAllocSpace();
}
}
void Heap::RemoveSpace(space::Space* space) {
DCHECK(space != nullptr);
WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
if (space->IsContinuousSpace()) {
DCHECK(!space->IsDiscontinuousSpace());
space::ContinuousSpace* continuous_space = space->AsContinuousSpace();
// Continuous spaces don't necessarily have bitmaps.
accounting::ContinuousSpaceBitmap* live_bitmap = continuous_space->GetLiveBitmap();
accounting::ContinuousSpaceBitmap* mark_bitmap = continuous_space->GetMarkBitmap();
if (live_bitmap != nullptr && !space->IsRegionSpace()) {
DCHECK(mark_bitmap != nullptr);
live_bitmap_->RemoveContinuousSpaceBitmap(live_bitmap);
mark_bitmap_->RemoveContinuousSpaceBitmap(mark_bitmap);
}
auto it = std::find(continuous_spaces_.begin(), continuous_spaces_.end(), continuous_space);
DCHECK(it != continuous_spaces_.end());
continuous_spaces_.erase(it);
} else {
DCHECK(space->IsDiscontinuousSpace());
space::DiscontinuousSpace* discontinuous_space = space->AsDiscontinuousSpace();
live_bitmap_->RemoveLargeObjectBitmap(discontinuous_space->GetLiveBitmap());
mark_bitmap_->RemoveLargeObjectBitmap(discontinuous_space->GetMarkBitmap());
auto it = std::find(discontinuous_spaces_.begin(), discontinuous_spaces_.end(),
discontinuous_space);
DCHECK(it != discontinuous_spaces_.end());
discontinuous_spaces_.erase(it);
}
if (space->IsAllocSpace()) {
auto it = std::find(alloc_spaces_.begin(), alloc_spaces_.end(), space->AsAllocSpace());
DCHECK(it != alloc_spaces_.end());
alloc_spaces_.erase(it);
}
}
double Heap::CalculateGcWeightedAllocatedBytes(uint64_t gc_last_process_cpu_time_ns,
uint64_t current_process_cpu_time) const {
uint64_t bytes_allocated = GetBytesAllocated();
double weight = current_process_cpu_time - gc_last_process_cpu_time_ns;
return weight * bytes_allocated;
}
void Heap::CalculatePreGcWeightedAllocatedBytes() {
uint64_t current_process_cpu_time = ProcessCpuNanoTime();
pre_gc_weighted_allocated_bytes_ +=
CalculateGcWeightedAllocatedBytes(pre_gc_last_process_cpu_time_ns_, current_process_cpu_time);
pre_gc_last_process_cpu_time_ns_ = current_process_cpu_time;
}
void Heap::CalculatePostGcWeightedAllocatedBytes() {
uint64_t current_process_cpu_time = ProcessCpuNanoTime();
post_gc_weighted_allocated_bytes_ +=
CalculateGcWeightedAllocatedBytes(post_gc_last_process_cpu_time_ns_, current_process_cpu_time);
post_gc_last_process_cpu_time_ns_ = current_process_cpu_time;
}
uint64_t Heap::GetTotalGcCpuTime() {
uint64_t sum = 0;
for (auto* collector : garbage_collectors_) {
sum += collector->GetTotalCpuTime();
}
return sum;
}
void Heap::DumpGcPerformanceInfo(std::ostream& os) {
// Dump cumulative timings.
os << "Dumping cumulative Gc timings\n";
uint64_t total_duration = 0;
// Dump cumulative loggers for each GC type.
uint64_t total_paused_time = 0;
for (auto* collector : garbage_collectors_) {
total_duration += collector->GetCumulativeTimings().GetTotalNs();
total_paused_time += collector->GetTotalPausedTimeNs();
collector->DumpPerformanceInfo(os);
}
if (total_duration != 0) {
const double total_seconds = total_duration / 1.0e9;
const double total_cpu_seconds = GetTotalGcCpuTime() / 1.0e9;
os << "Total time spent in GC: " << PrettyDuration(total_duration) << "\n";
os << "Mean GC size throughput: "
<< PrettySize(GetBytesFreedEver() / total_seconds) << "/s"
<< " per cpu-time: "
<< PrettySize(GetBytesFreedEver() / total_cpu_seconds) << "/s\n";
}
os << "Total bytes allocated " << PrettySize(GetBytesAllocatedEver()) << "\n";
os << "Total bytes freed " << PrettySize(GetBytesFreedEver()) << "\n";
os << "Free memory " << PrettySize(GetFreeMemory()) << "\n";
os << "Free memory until GC " << PrettySize(GetFreeMemoryUntilGC()) << "\n";
os << "Free memory until OOME " << PrettySize(GetFreeMemoryUntilOOME()) << "\n";
os << "Total memory " << PrettySize(GetTotalMemory()) << "\n";
os << "Max memory " << PrettySize(GetMaxMemory()) << "\n";
if (HasZygoteSpace()) {
os << "Zygote space size " << PrettySize(zygote_space_->Size()) << "\n";
}
os << "Total mutator paused time: " << PrettyDuration(total_paused_time) << "\n";
os << "Total time waiting for GC to complete: " << PrettyDuration(total_wait_time_) << "\n";
os << "Total GC count: " << GetGcCount() << "\n";
os << "Total GC time: " << PrettyDuration(GetGcTime()) << "\n";
os << "Total blocking GC count: " << GetBlockingGcCount() << "\n";
os << "Total blocking GC time: " << PrettyDuration(GetBlockingGcTime()) << "\n";
os << "Total pre-OOME GC count: " << GetPreOomeGcCount() << "\n";
{
MutexLock mu(Thread::Current(), *gc_complete_lock_);
if (gc_count_rate_histogram_.SampleSize() > 0U) {
os << "Histogram of GC count per " << NsToMs(kGcCountRateHistogramWindowDuration) << " ms: ";
gc_count_rate_histogram_.DumpBins(os);
os << "\n";
}
if (blocking_gc_count_rate_histogram_.SampleSize() > 0U) {
os << "Histogram of blocking GC count per "
<< NsToMs(kGcCountRateHistogramWindowDuration) << " ms: ";
blocking_gc_count_rate_histogram_.DumpBins(os);
os << "\n";
}
}
if (kDumpRosAllocStatsOnSigQuit && rosalloc_space_ != nullptr) {
rosalloc_space_->DumpStats(os);
}
os << "Native bytes total: " << GetNativeBytes()
<< " registered: " << native_bytes_registered_.load(std::memory_order_relaxed) << "\n";
os << "Total native bytes at last GC: "
<< old_native_bytes_allocated_.load(std::memory_order_relaxed) << "\n";
BaseMutex::DumpAll(os);
}
void Heap::ResetGcPerformanceInfo() {
for (auto* collector : garbage_collectors_) {
collector->ResetMeasurements();
}
process_cpu_start_time_ns_ = ProcessCpuNanoTime();
pre_gc_last_process_cpu_time_ns_ = process_cpu_start_time_ns_;
pre_gc_weighted_allocated_bytes_ = 0u;
post_gc_last_process_cpu_time_ns_ = process_cpu_start_time_ns_;
post_gc_weighted_allocated_bytes_ = 0u;
total_bytes_freed_ever_.store(0);
total_objects_freed_ever_.store(0);
total_wait_time_ = 0;
blocking_gc_count_ = 0;
blocking_gc_time_ = 0;
pre_oome_gc_count_.store(0, std::memory_order_relaxed);
gc_count_last_window_ = 0;
blocking_gc_count_last_window_ = 0;
last_update_time_gc_count_rate_histograms_ = // Round down by the window duration.
(NanoTime() / kGcCountRateHistogramWindowDuration) * kGcCountRateHistogramWindowDuration;
{
MutexLock mu(Thread::Current(), *gc_complete_lock_);
gc_count_rate_histogram_.Reset();
blocking_gc_count_rate_histogram_.Reset();
}
}
uint64_t Heap::GetGcCount() const {
uint64_t gc_count = 0U;
for (auto* collector : garbage_collectors_) {
gc_count += collector->GetCumulativeTimings().GetIterations();
}
return gc_count;
}
uint64_t Heap::GetGcTime() const {
uint64_t gc_time = 0U;
for (auto* collector : garbage_collectors_) {
gc_time += collector->GetCumulativeTimings().GetTotalNs();
}
return gc_time;
}
uint64_t Heap::GetBlockingGcCount() const {
return blocking_gc_count_;
}
uint64_t Heap::GetBlockingGcTime() const {
return blocking_gc_time_;
}
void Heap::DumpGcCountRateHistogram(std::ostream& os) const {
MutexLock mu(Thread::Current(), *gc_complete_lock_);
if (gc_count_rate_histogram_.SampleSize() > 0U) {
gc_count_rate_histogram_.DumpBins(os);
}
}
void Heap::DumpBlockingGcCountRateHistogram(std::ostream& os) const {
MutexLock mu(Thread::Current(), *gc_complete_lock_);
if (blocking_gc_count_rate_histogram_.SampleSize() > 0U) {
blocking_gc_count_rate_histogram_.DumpBins(os);
}
}
uint64_t Heap::GetPreOomeGcCount() const {
return pre_oome_gc_count_.load(std::memory_order_relaxed);
}
ALWAYS_INLINE
static inline AllocationListener* GetAndOverwriteAllocationListener(
Atomic<AllocationListener*>* storage, AllocationListener* new_value) {
return storage->exchange(new_value);
}
Heap::~Heap() {
VLOG(heap) << "Starting ~Heap()";
STLDeleteElements(&garbage_collectors_);
// If we don't reset then the mark stack complains in its destructor.
allocation_stack_->Reset();
allocation_records_.reset();
live_stack_->Reset();
STLDeleteValues(&mod_union_tables_);
STLDeleteValues(&remembered_sets_);
STLDeleteElements(&continuous_spaces_);
STLDeleteElements(&discontinuous_spaces_);
delete gc_complete_lock_;
delete thread_flip_lock_;
delete pending_task_lock_;
delete backtrace_lock_;
uint64_t unique_count = unique_backtrace_count_.load();
uint64_t seen_count = seen_backtrace_count_.load();
if (unique_count != 0 || seen_count != 0) {
LOG(INFO) << "gc stress unique=" << unique_count << " total=" << (unique_count + seen_count);
}
VLOG(heap) << "Finished ~Heap()";
}
space::ContinuousSpace* Heap::FindContinuousSpaceFromAddress(const mirror::Object* addr) const {
for (const auto& space : continuous_spaces_) {
if (space->Contains(addr)) {
return space;
}
}
return nullptr;
}
space::ContinuousSpace* Heap::FindContinuousSpaceFromObject(ObjPtr<mirror::Object> obj,
bool fail_ok) const {
space::ContinuousSpace* space = FindContinuousSpaceFromAddress(obj.Ptr());
if (space != nullptr) {
return space;
}
if (!fail_ok) {
LOG(FATAL) << "object " << obj << " not inside any spaces!";
}
return nullptr;
}
space::DiscontinuousSpace* Heap::FindDiscontinuousSpaceFromObject(ObjPtr<mirror::Object> obj,
bool fail_ok) const {
for (const auto& space : discontinuous_spaces_) {
if (space->Contains(obj.Ptr())) {
return space;
}
}
if (!fail_ok) {
LOG(FATAL) << "object " << obj << " not inside any spaces!";
}
return nullptr;
}
space::Space* Heap::FindSpaceFromObject(ObjPtr<mirror::Object> obj, bool fail_ok) const {
space::Space* result = FindContinuousSpaceFromObject(obj, true);
if (result != nullptr) {
return result;
}
return FindDiscontinuousSpaceFromObject(obj, fail_ok);
}
space::Space* Heap::FindSpaceFromAddress(const void* addr) const {
for (const auto& space : continuous_spaces_) {
if (space->Contains(reinterpret_cast<const mirror::Object*>(addr))) {
return space;
}
}
for (const auto& space : discontinuous_spaces_) {
if (space->Contains(reinterpret_cast<const mirror::Object*>(addr))) {
return space;
}
}
return nullptr;
}
std::string Heap::DumpSpaceNameFromAddress(const void* addr) const {
space::Space* space = FindSpaceFromAddress(addr);
return (space != nullptr) ? space->GetName() : "no space";
}
void Heap::ThrowOutOfMemoryError(Thread* self, size_t byte_count, AllocatorType allocator_type) {
// If we're in a stack overflow, do not create a new exception. It would require running the
// constructor, which will of course still be in a stack overflow.
if (self->IsHandlingStackOverflow()) {
self->SetException(
Runtime::Current()->GetPreAllocatedOutOfMemoryErrorWhenHandlingStackOverflow());
return;
}
// Allow plugins to intercept out of memory errors.
Runtime::Current()->OutOfMemoryErrorHook();
std::ostringstream oss;
size_t total_bytes_free = GetFreeMemory();
oss << "Failed to allocate a " << byte_count << " byte allocation with " << total_bytes_free
<< " free bytes and " << PrettySize(GetFreeMemoryUntilOOME()) << " until OOM,"
<< " target footprint " << target_footprint_.load(std::memory_order_relaxed)
<< ", growth limit "
<< growth_limit_;
// If the allocation failed due to fragmentation, print out the largest continuous allocation.
if (total_bytes_free >= byte_count) {
space::AllocSpace* space = nullptr;
if (allocator_type == kAllocatorTypeNonMoving) {
space = non_moving_space_;
} else if (allocator_type == kAllocatorTypeRosAlloc ||
allocator_type == kAllocatorTypeDlMalloc) {
space = main_space_;
} else if (allocator_type == kAllocatorTypeBumpPointer ||
allocator_type == kAllocatorTypeTLAB) {
space = bump_pointer_space_;
} else if (allocator_type == kAllocatorTypeRegion ||
allocator_type == kAllocatorTypeRegionTLAB) {
space = region_space_;
}
// There is no fragmentation info to log for large-object space.
if (allocator_type != kAllocatorTypeLOS) {
CHECK(space != nullptr) << "allocator_type:" << allocator_type
<< " byte_count:" << byte_count
<< " total_bytes_free:" << total_bytes_free;
// LogFragmentationAllocFailure returns true if byte_count is greater than
// the largest free contiguous chunk in the space. Return value false
// means that we are throwing OOME because the amount of free heap after
// GC is less than kMinFreeHeapAfterGcForAlloc in proportion of the heap-size.
// Log an appropriate message in that case.
if (!space->LogFragmentationAllocFailure(oss, byte_count)) {
oss << "; giving up on allocation because <"
<< kMinFreeHeapAfterGcForAlloc * 100
<< "% of heap free after GC.";
}
}
}
self->ThrowOutOfMemoryError(oss.str().c_str());
}
void Heap::DoPendingCollectorTransition() {
CollectorType desired_collector_type = desired_collector_type_;
if (collector_type_ == kCollectorTypeCC || collector_type_ == kCollectorTypeCMC) {
// App's allocations (since last GC) more than the threshold then do TransitionGC
// when the app was in background. If not then don't do TransitionGC.
// num_bytes_allocated_since_gc should always be positive even if initially
// num_bytes_alive_after_gc_ is coming from Zygote. This gives positive or zero value.
size_t num_bytes_allocated_since_gc =
UnsignedDifference(GetBytesAllocated(), num_bytes_alive_after_gc_);
if (num_bytes_allocated_since_gc <
(UnsignedDifference(target_footprint_.load(std::memory_order_relaxed),
num_bytes_alive_after_gc_)/4)
&& !kStressCollectorTransition
&& !IsLowMemoryMode()) {
return;
}
}
// Launch homogeneous space compaction if it is desired.
if (desired_collector_type == kCollectorTypeHomogeneousSpaceCompact) {
if (!CareAboutPauseTimes()) {
PerformHomogeneousSpaceCompact();
} else {
VLOG(gc) << "Homogeneous compaction ignored due to jank perceptible process state";
}
} else if (desired_collector_type == kCollectorTypeCCBackground ||
desired_collector_type == kCollectorTypeCMCBackground) {
if (!CareAboutPauseTimes()) {
// Invoke full compaction.
CollectGarbageInternal(collector::kGcTypeFull,
kGcCauseCollectorTransition,
/*clear_soft_references=*/false, GetCurrentGcNum() + 1);
} else {
VLOG(gc) << "background compaction ignored due to jank perceptible process state";
}
} else {
CHECK_EQ(desired_collector_type, collector_type_) << "Unsupported collector transition";
}
}
void Heap::Trim(Thread* self) {
Runtime* const runtime = Runtime::Current();
if (!CareAboutPauseTimes()) {
// Deflate the monitors, this can cause a pause but shouldn't matter since we don't care
// about pauses.
ScopedTrace trace("Deflating monitors");
// Avoid race conditions on the lock word for CC.
ScopedGCCriticalSection gcs(self, kGcCauseTrim, kCollectorTypeHeapTrim);
ScopedSuspendAll ssa(__FUNCTION__);
uint64_t start_time = NanoTime();
size_t count = runtime->GetMonitorList()->DeflateMonitors();
VLOG(heap) << "Deflating " << count << " monitors took "
<< PrettyDuration(NanoTime() - start_time);
}
TrimIndirectReferenceTables(self);
TrimSpaces(self);
// Trim arenas that may have been used by JIT or verifier.
runtime->GetArenaPool()->TrimMaps();
}
class TrimIndirectReferenceTableClosure : public Closure {
public:
explicit TrimIndirectReferenceTableClosure(Barrier* barrier) : barrier_(barrier) {
}
void Run(Thread* thread) override NO_THREAD_SAFETY_ANALYSIS {
thread->GetJniEnv()->TrimLocals();
// If thread is a running mutator, then act on behalf of the trim thread.
// See the code in ThreadList::RunCheckpoint.
barrier_->Pass(Thread::Current());
}
private:
Barrier* const barrier_;
};
void Heap::TrimIndirectReferenceTables(Thread* self) {
ScopedObjectAccess soa(self);
ScopedTrace trace(__PRETTY_FUNCTION__);
JavaVMExt* vm = soa.Vm();
// Trim globals indirect reference table.
vm->TrimGlobals();
// Trim locals indirect reference tables.
// TODO: May also want to look for entirely empty pages maintained by SmallIrtAllocator.
Barrier barrier(0);
TrimIndirectReferenceTableClosure closure(&barrier);
ScopedThreadStateChange tsc(self, ThreadState::kWaitingForCheckPointsToRun);
size_t barrier_count = Runtime::Current()->GetThreadList()->RunCheckpoint(&closure);
if (barrier_count != 0) {
barrier.Increment(self, barrier_count);
}
}
void Heap::StartGC(Thread* self, GcCause cause, CollectorType collector_type) {
// Need to do this before acquiring the locks since we don't want to get suspended while
// holding any locks.
ScopedThreadStateChange tsc(self, ThreadState::kWaitingForGcToComplete);
MutexLock mu(self, *gc_complete_lock_);
// Ensure there is only one GC at a time.
WaitForGcToCompleteLocked(cause, self);
collector_type_running_ = collector_type;
last_gc_cause_ = cause;
thread_running_gc_ = self;
}
void Heap::TrimSpaces(Thread* self) {
// Pretend we are doing a GC to prevent background compaction from deleting the space we are
// trimming.
StartGC(self, kGcCauseTrim, kCollectorTypeHeapTrim);
ScopedTrace trace(__PRETTY_FUNCTION__);
const uint64_t start_ns = NanoTime();
// Trim the managed spaces.
uint64_t total_alloc_space_allocated = 0;
uint64_t total_alloc_space_size = 0;
uint64_t managed_reclaimed = 0;
{
ScopedObjectAccess soa(self);
for (const auto& space : continuous_spaces_) {
if (space->IsMallocSpace()) {
gc::space::MallocSpace* malloc_space = space->AsMallocSpace();
if (malloc_space->IsRosAllocSpace() || !CareAboutPauseTimes()) {
// Don't trim dlmalloc spaces if we care about pauses since this can hold the space lock
// for a long period of time.
managed_reclaimed += malloc_space->Trim();
}
total_alloc_space_size += malloc_space->Size();
}
}
}
total_alloc_space_allocated = GetBytesAllocated();
if (large_object_space_ != nullptr) {
total_alloc_space_allocated -= large_object_space_->GetBytesAllocated();
}
if (bump_pointer_space_ != nullptr) {
total_alloc_space_allocated -= bump_pointer_space_->Size();
}
if (region_space_ != nullptr) {
total_alloc_space_allocated -= region_space_->GetBytesAllocated();
}
const float managed_utilization = static_cast<float>(total_alloc_space_allocated) /
static_cast<float>(total_alloc_space_size);
uint64_t gc_heap_end_ns = NanoTime();
// We never move things in the native heap, so we can finish the GC at this point.
FinishGC(self, collector::kGcTypeNone);
VLOG(heap) << "Heap trim of managed (duration=" << PrettyDuration(gc_heap_end_ns - start_ns)
<< ", advised=" << PrettySize(managed_reclaimed) << ") heap. Managed heap utilization of "
<< static_cast<int>(100 * managed_utilization) << "%.";
}
bool Heap::IsValidObjectAddress(const void* addr) const {
if (addr == nullptr) {
return true;
}
return IsAligned<kObjectAlignment>(addr) && FindSpaceFromAddress(addr) != nullptr;
}
bool Heap::IsNonDiscontinuousSpaceHeapAddress(const void* addr) const {
return FindContinuousSpaceFromAddress(reinterpret_cast<const mirror::Object*>(addr)) != nullptr;
}
bool Heap::IsLiveObjectLocked(ObjPtr<mirror::Object> obj,
bool search_allocation_stack,
bool search_live_stack,
bool sorted) {
if (UNLIKELY(!IsAligned<kObjectAlignment>(obj.Ptr()))) {
return false;
}
if (bump_pointer_space_ != nullptr && bump_pointer_space_->HasAddress(obj.Ptr())) {
mirror::Class* klass = obj->GetClass<kVerifyNone>();
if (obj == klass) {
// This case happens for java.lang.Class.
return true;
}
return VerifyClassClass(klass) && IsLiveObjectLocked(klass);
} else if (temp_space_ != nullptr && temp_space_->HasAddress(obj.Ptr())) {
// If we are in the allocated region of the temp space, then we are probably live (e.g. during
// a GC). When a GC isn't running End() - Begin() is 0 which means no objects are contained.
return temp_space_->Contains(obj.Ptr());
}
if (region_space_ != nullptr && region_space_->HasAddress(obj.Ptr())) {
return true;
}
space::ContinuousSpace* c_space = FindContinuousSpaceFromObject(obj, true);
space::DiscontinuousSpace* d_space = nullptr;
if (c_space != nullptr) {
if (c_space->GetLiveBitmap()->Test(obj.Ptr())) {
return true;
}
} else {
d_space = FindDiscontinuousSpaceFromObject(obj, true);
if (d_space != nullptr) {
if (d_space->GetLiveBitmap()->Test(obj.Ptr())) {
return true;
}
}
}
// This is covering the allocation/live stack swapping that is done without mutators suspended.
for (size_t i = 0; i < (sorted ? 1 : 5); ++i) {
if (i > 0) {
NanoSleep(MsToNs(10));
}
if (search_allocation_stack) {
if (sorted) {
if (allocation_stack_->ContainsSorted(obj.Ptr())) {
return true;
}
} else if (allocation_stack_->Contains(obj.Ptr())) {
return true;
}
}
if (search_live_stack) {
if (sorted) {
if (live_stack_->ContainsSorted(obj.Ptr())) {
return true;
}
} else if (live_stack_->Contains(obj.Ptr())) {
return true;
}
}
}
// We need to check the bitmaps again since there is a race where we mark something as live and
// then clear the stack containing it.
if (c_space != nullptr) {
if (c_space->GetLiveBitmap()->Test(obj.Ptr())) {
return true;
}
} else {
d_space = FindDiscontinuousSpaceFromObject(obj, true);
if (d_space != nullptr && d_space->GetLiveBitmap()->Test(obj.Ptr())) {
return true;
}
}
return false;
}
std::string Heap::DumpSpaces() const {
std::ostringstream oss;
DumpSpaces(oss);
return oss.str();
}
void Heap::DumpSpaces(std::ostream& stream) const {
for (const auto& space : continuous_spaces_) {
accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap();
accounting::ContinuousSpaceBitmap* mark_bitmap = space->GetMarkBitmap();
stream << space << " " << *space << "\n";
if (live_bitmap != nullptr) {
stream << live_bitmap << " " << *live_bitmap << "\n";
}
if (mark_bitmap != nullptr) {
stream << mark_bitmap << " " << *mark_bitmap << "\n";
}
}
for (const auto& space : discontinuous_spaces_) {
stream << space << " " << *space << "\n";
}
}
void Heap::VerifyObjectBody(ObjPtr<mirror::Object> obj) {
if (verify_object_mode_ == kVerifyObjectModeDisabled) {
return;
}
// Ignore early dawn of the universe verifications.
if (UNLIKELY(num_bytes_allocated_.load(std::memory_order_relaxed) < 10 * KB)) {
return;
}
CHECK_ALIGNED(obj.Ptr(), kObjectAlignment) << "Object isn't aligned";
mirror::Class* c = obj->GetFieldObject<mirror::Class, kVerifyNone>(mirror::Object::ClassOffset());
CHECK(c != nullptr) << "Null class in object " << obj;
CHECK_ALIGNED(c, kObjectAlignment) << "Class " << c << " not aligned in object " << obj;
CHECK(VerifyClassClass(c));
if (verify_object_mode_ > kVerifyObjectModeFast) {
// Note: the bitmap tests below are racy since we don't hold the heap bitmap lock.
CHECK(IsLiveObjectLocked(obj)) << "Object is dead " << obj << "\n" << DumpSpaces();
}
}
void Heap::VerifyHeap() {
ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
auto visitor = [&](mirror::Object* obj) NO_THREAD_SAFETY_ANALYSIS {
VerifyObjectBody(obj);
};
// Technically we need the mutator lock here to call Visit. However, VerifyObjectBody is already
// NO_THREAD_SAFETY_ANALYSIS.
auto no_thread_safety_analysis = [&]() NO_THREAD_SAFETY_ANALYSIS {
GetLiveBitmap()->Visit(visitor);
};
no_thread_safety_analysis();
}
void Heap::RecordFree(uint64_t freed_objects, int64_t freed_bytes) {
// Use signed comparison since freed bytes can be negative when background compaction foreground
// transitions occurs. This is typically due to objects moving from a bump pointer space to a
// free list backed space, which may increase memory footprint due to padding and binning.
RACING_DCHECK_LE(freed_bytes,
static_cast<int64_t>(num_bytes_allocated_.load(std::memory_order_relaxed)));
// Note: This relies on 2s complement for handling negative freed_bytes.
num_bytes_allocated_.fetch_sub(static_cast<ssize_t>(freed_bytes), std::memory_order_relaxed);
if (Runtime::Current()->HasStatsEnabled()) {
RuntimeStats* thread_stats = Thread::Current()->GetStats();
thread_stats->freed_objects += freed_objects;
thread_stats->freed_bytes += freed_bytes;
// TODO: Do this concurrently.
RuntimeStats* global_stats = Runtime::Current()->GetStats();
global_stats->freed_objects += freed_objects;
global_stats->freed_bytes += freed_bytes;
}
}
void Heap::RecordFreeRevoke() {
// Subtract num_bytes_freed_revoke_ from num_bytes_allocated_ to cancel out the
// ahead-of-time, bulk counting of bytes allocated in rosalloc thread-local buffers.
// If there's a concurrent revoke, ok to not necessarily reset num_bytes_freed_revoke_
// all the way to zero exactly as the remainder will be subtracted at the next GC.
size_t bytes_freed = num_bytes_freed_revoke_.load(std::memory_order_relaxed);
CHECK_GE(num_bytes_freed_revoke_.fetch_sub(bytes_freed, std::memory_order_relaxed),
bytes_freed) << "num_bytes_freed_revoke_ underflow";
CHECK_GE(num_bytes_allocated_.fetch_sub(bytes_freed, std::memory_order_relaxed),
bytes_freed) << "num_bytes_allocated_ underflow";
GetCurrentGcIteration()->SetFreedRevoke(bytes_freed);
}
space::RosAllocSpace* Heap::GetRosAllocSpace(gc::allocator::RosAlloc* rosalloc) const {
if (rosalloc_space_ != nullptr && rosalloc_space_->GetRosAlloc() == rosalloc) {
return rosalloc_space_;
}
for (const auto& space : continuous_spaces_) {
if (space->AsContinuousSpace()->IsRosAllocSpace()) {
if (space->AsContinuousSpace()->AsRosAllocSpace()->GetRosAlloc() == rosalloc) {
return space->AsContinuousSpace()->AsRosAllocSpace();
}
}
}
return nullptr;
}
static inline bool EntrypointsInstrumented() REQUIRES_SHARED(Locks::mutator_lock_) {
instrumentation::Instrumentation* const instrumentation =
Runtime::Current()->GetInstrumentation();
return instrumentation != nullptr && instrumentation->AllocEntrypointsInstrumented();
}
mirror::Object* Heap::AllocateInternalWithGc(Thread* self,
AllocatorType allocator,
bool instrumented,
size_t alloc_size,
size_t* bytes_allocated,
size_t* usable_size,
size_t* bytes_tl_bulk_allocated,
ObjPtr<mirror::Class>* klass) {
bool was_default_allocator = allocator == GetCurrentAllocator();
// Make sure there is no pending exception since we may need to throw an OOME.
self->AssertNoPendingException();
DCHECK(klass != nullptr);
StackHandleScope<1> hs(self);
HandleWrapperObjPtr<mirror::Class> h_klass(hs.NewHandleWrapper(klass));
auto send_object_pre_alloc =
[&]() REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!Roles::uninterruptible_) {
if (UNLIKELY(instrumented)) {
AllocationListener* l = alloc_listener_.load(std::memory_order_seq_cst);
if (UNLIKELY(l != nullptr) && UNLIKELY(l->HasPreAlloc())) {
l->PreObjectAllocated(self, h_klass, &alloc_size);
}
}
};
#define PERFORM_SUSPENDING_OPERATION(op) \
[&]() REQUIRES(Roles::uninterruptible_) REQUIRES_SHARED(Locks::mutator_lock_) { \
ScopedAllowThreadSuspension ats; \
auto res = (op); \
send_object_pre_alloc(); \
return res; \
}()
// The allocation failed. If the GC is running, block until it completes, and then retry the
// allocation.
collector::GcType last_gc =
PERFORM_SUSPENDING_OPERATION(WaitForGcToComplete(kGcCauseForAlloc, self));
// If we were the default allocator but the allocator changed while we were suspended,
// abort the allocation.
if ((was_default_allocator && allocator != GetCurrentAllocator()) ||
(!instrumented && EntrypointsInstrumented())) {
return nullptr;
}
uint32_t starting_gc_num = GetCurrentGcNum();
if (last_gc != collector::kGcTypeNone) {
// A GC was in progress and we blocked, retry allocation now that memory has been freed.
mirror::Object* ptr = TryToAllocate<true, false>(self, allocator, alloc_size, bytes_allocated,
usable_size, bytes_tl_bulk_allocated);
if (ptr != nullptr) {
return ptr;
}
}
if (IsGCDisabledForShutdown()) {
// We're just shutting down and GCs don't work anymore. Try a different allocator.
mirror::Object* ptr = TryToAllocate<true, false>(self,
kAllocatorTypeNonMoving,
alloc_size,
bytes_allocated,
usable_size,
bytes_tl_bulk_allocated);
if (ptr != nullptr) {
return ptr;
}
}
int64_t bytes_freed_before = GetBytesFreedEver();
auto have_reclaimed_enough = [&]() {
size_t curr_bytes_allocated = GetBytesAllocated();
size_t free_heap = UnsignedDifference(growth_limit_, curr_bytes_allocated);
int64_t newly_freed = GetBytesFreedEver() - bytes_freed_before;
double free_heap_ratio = static_cast<double>(free_heap) / growth_limit_;
double newly_freed_ratio = static_cast<double>(newly_freed) / growth_limit_;
return free_heap_ratio >= kMinFreeHeapAfterGcForAlloc ||
newly_freed_ratio >= kMinFreedHeapAfterGcForAlloc;
};
// We perform one GC as per the next_gc_type_ (chosen in GrowForUtilization),
// if it's not already tried. If that doesn't succeed then go for the most
// exhaustive option. Perform a full-heap collection including clearing
// SoftReferences. In case of ConcurrentCopying, it will also ensure that
// all regions are evacuated. If allocation doesn't succeed even after that
// then there is no hope, so we throw OOME.
collector::GcType tried_type = next_gc_type_;
if (last_gc < tried_type) {
const bool gc_ran = PERFORM_SUSPENDING_OPERATION(
CollectGarbageInternal(tried_type, kGcCauseForAlloc, false, starting_gc_num + 1)
!= collector::kGcTypeNone);
if ((was_default_allocator && allocator != GetCurrentAllocator()) ||
(!instrumented && EntrypointsInstrumented())) {
return nullptr;
}
if (gc_ran && have_reclaimed_enough()) {
mirror::Object* ptr = TryToAllocate<true, false>(self, allocator,
alloc_size, bytes_allocated,
usable_size, bytes_tl_bulk_allocated);
if (ptr != nullptr) {
return ptr;
}
}
}
// Most allocations should have succeeded by now, so the heap is really full, really fragmented,
// or the requested size is really big. Do another GC, collecting SoftReferences this time. The
// VM spec requires that all SoftReferences have been collected and cleared before throwing
// OOME.
VLOG(gc) << "Forcing collection of SoftReferences for " << PrettySize(alloc_size)
<< " allocation";
// TODO: Run finalization, but this may cause more allocations to occur.
// We don't need a WaitForGcToComplete here either.
// TODO: Should check whether another thread already just ran a GC with soft
// references.
DCHECK(!gc_plan_.empty());
int64_t min_freed_to_continue =
static_cast<int64_t>(kMinFreedHeapAfterGcForAlloc * growth_limit_ + alloc_size);
// Repeatedly collect the entire heap until either
// (a) this was insufficiently productive at reclaiming memory and we should give upt to avoid
// "GC thrashing", or
// (b) GC was sufficiently productive (reclaimed min_freed_to_continue bytes) AND allowed us to
// satisfy the allocation request.
do {
bytes_freed_before = GetBytesFreedEver();
pre_oome_gc_count_.fetch_add(1, std::memory_order_relaxed);
PERFORM_SUSPENDING_OPERATION(
CollectGarbageInternal(gc_plan_.back(), kGcCauseForAlloc, true, GC_NUM_ANY));
if ((was_default_allocator && allocator != GetCurrentAllocator()) ||
(!instrumented && EntrypointsInstrumented())) {
return nullptr;
}
bool ran_homogeneous_space_compaction = false;
bool immediately_reclaimed_enough = have_reclaimed_enough();
if (!immediately_reclaimed_enough) {
const uint64_t current_time = NanoTime();
if (allocator == kAllocatorTypeRosAlloc || allocator == kAllocatorTypeDlMalloc) {
if (use_homogeneous_space_compaction_for_oom_ &&
current_time - last_time_homogeneous_space_compaction_by_oom_ >
min_interval_homogeneous_space_compaction_by_oom_) {
last_time_homogeneous_space_compaction_by_oom_ = current_time;
ran_homogeneous_space_compaction =
(PERFORM_SUSPENDING_OPERATION(PerformHomogeneousSpaceCompact()) ==
HomogeneousSpaceCompactResult::kSuccess);
// Thread suspension could have occurred.
if ((was_default_allocator && allocator != GetCurrentAllocator()) ||
(!instrumented && EntrypointsInstrumented())) {
return nullptr;
}
// Always print that we ran homogeneous space compation since this can cause jank.
VLOG(heap) << "Ran heap homogeneous space compaction, "
<< " requested defragmentation "
<< count_requested_homogeneous_space_compaction_.load()
<< " performed defragmentation "
<< count_performed_homogeneous_space_compaction_.load()
<< " ignored homogeneous space compaction "
<< count_ignored_homogeneous_space_compaction_.load()
<< " delayed count = "
<< count_delayed_oom_.load();
}
}
}
if (immediately_reclaimed_enough ||
(ran_homogeneous_space_compaction && have_reclaimed_enough())) {
mirror::Object* ptr = TryToAllocate<true, true>(
self, allocator, alloc_size, bytes_allocated, usable_size, bytes_tl_bulk_allocated);
if (ptr != nullptr) {
if (ran_homogeneous_space_compaction) {
count_delayed_oom_++;
}
return ptr;
}
}
// This loops only if we reclaimed plenty of memory, but presumably some other thread beat us
// to allocating it. In the very unlikely case that we're running into a serious fragmentation
// issue, and there is no other thread allocating, GCs will quickly become unsuccessful, and we
// will stop then. If another thread is allocating aggressively, this may go on for a while,
// but we are still making progress somewhere.
} while (GetBytesFreedEver() - bytes_freed_before > min_freed_to_continue);
#undef PERFORM_SUSPENDING_OPERATION
// Throw an OOM error.
{
ScopedAllowThreadSuspension ats;
ThrowOutOfMemoryError(self, alloc_size, allocator);
}
return nullptr;
}
void Heap::SetTargetHeapUtilization(float target) {
DCHECK_GT(target, 0.1f); // asserted in Java code
DCHECK_LT(target, 1.0f);
target_utilization_ = target;
}
size_t Heap::GetObjectsAllocated() const {
Thread* const self = Thread::Current();
ScopedThreadStateChange tsc(self, ThreadState::kWaitingForGetObjectsAllocated);
// Prevent GC running during GetObjectsAllocated since we may get a checkpoint request that tells
// us to suspend while we are doing SuspendAll. b/35232978
gc::ScopedGCCriticalSection gcs(Thread::Current(),
gc::kGcCauseGetObjectsAllocated,
gc::kCollectorTypeGetObjectsAllocated);
// Need SuspendAll here to prevent lock violation if RosAlloc does it during InspectAll.
ScopedSuspendAll ssa(__FUNCTION__);
ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_);
size_t total = 0;
for (space::AllocSpace* space : alloc_spaces_) {
total += space->GetObjectsAllocated();
}
return total;
}
uint64_t Heap::GetBytesAllocatedEver() const {
// Force the returned value to be monotonically increasing, in the sense that if this is called
// at A and B, such that A happens-before B, then the call at B returns a value no smaller than
// that at A. This is not otherwise guaranteed, since num_bytes_allocated_ is decremented first,
// and total_bytes_freed_ever_ is incremented later.
static std::atomic<uint64_t> max_bytes_so_far(0);
uint64_t so_far = max_bytes_so_far.load(std::memory_order_relaxed);
uint64_t current_bytes = GetBytesFreedEver(std::memory_order_acquire) + GetBytesAllocated();
DCHECK(current_bytes < (static_cast<uint64_t>(1) << 63)); // result is "positive".
do {
if (current_bytes <= so_far) {
return so_far;
}
} while (!max_bytes_so_far.compare_exchange_weak(so_far /* updated */,
current_bytes, std::memory_order_relaxed));
return current_bytes;
}
// Check whether the given object is an instance of the given class.
static bool MatchesClass(mirror::Object* obj,
Handle<mirror::Class> h_class,
bool use_is_assignable_from) REQUIRES_SHARED(Locks::mutator_lock_) {
mirror::Class* instance_class = obj->GetClass();
CHECK(instance_class != nullptr);
ObjPtr<mirror::Class> klass = h_class.Get();
if (use_is_assignable_from) {
return klass != nullptr && klass->IsAssignableFrom(instance_class);
}
return instance_class == klass;
}
void Heap::CountInstances(const std::vector<Handle<mirror::Class>>& classes,
bool use_is_assignable_from,
uint64_t* counts) {
auto instance_counter = [&](mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) {
for (size_t i = 0; i < classes.size(); ++i) {
if (MatchesClass(obj, classes[i], use_is_assignable_from)) {
++counts[i];
}
}
};
VisitObjects(instance_counter);
}
void Heap::CollectGarbage(bool clear_soft_references, GcCause cause) {
// Even if we waited for a GC we still need to do another GC since weaks allocated during the
// last GC will not have necessarily been cleared.
CollectGarbageInternal(gc_plan_.back(), cause, clear_soft_references, GC_NUM_ANY);
}
bool Heap::SupportHomogeneousSpaceCompactAndCollectorTransitions() const {
return main_space_backup_.get() != nullptr && main_space_ != nullptr &&
foreground_collector_type_ == kCollectorTypeCMS;
}
HomogeneousSpaceCompactResult Heap::PerformHomogeneousSpaceCompact() {
Thread* self = Thread::Current();
// Inc requested homogeneous space compaction.
count_requested_homogeneous_space_compaction_++;
// Store performed homogeneous space compaction at a new request arrival.
ScopedThreadStateChange tsc(self, ThreadState::kWaitingPerformingGc);
Locks::mutator_lock_->AssertNotHeld(self);
{
ScopedThreadStateChange tsc2(self, ThreadState::kWaitingForGcToComplete);
MutexLock mu(self, *gc_complete_lock_);
// Ensure there is only one GC at a time.
WaitForGcToCompleteLocked(kGcCauseHomogeneousSpaceCompact, self);
// Homogeneous space compaction is a copying transition, can't run it if the moving GC disable
// count is non zero.
// If the collector type changed to something which doesn't benefit from homogeneous space
// compaction, exit.
if (disable_moving_gc_count_ != 0 || IsMovingGc(collector_type_) ||
!main_space_->CanMoveObjects()) {
return kErrorReject;
}
if (!SupportHomogeneousSpaceCompactAndCollectorTransitions()) {
return kErrorUnsupported;
}
collector_type_running_ = kCollectorTypeHomogeneousSpaceCompact;
}
if (Runtime::Current()->IsShuttingDown(self)) {
// Don't allow heap transitions to happen if the runtime is shutting down since these can
// cause objects to get finalized.
FinishGC(self, collector::kGcTypeNone);
return HomogeneousSpaceCompactResult::kErrorVMShuttingDown;
}
collector::GarbageCollector* collector;
{
ScopedSuspendAll ssa(__FUNCTION__);
uint64_t start_time = NanoTime();
// Launch compaction.
space::MallocSpace* to_space = main_space_backup_.release();
space::MallocSpace* from_space = main_space_;
to_space->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
const uint64_t space_size_before_compaction = from_space->Size();
AddSpace(to_space);
// Make sure that we will have enough room to copy.
CHECK_GE(to_space->GetFootprintLimit(), from_space->GetFootprintLimit());
collector = Compact(to_space, from_space, kGcCauseHomogeneousSpaceCompact);
const uint64_t space_size_after_compaction = to_space->Size();
main_space_ = to_space;
main_space_backup_.reset(from_space);
RemoveSpace(from_space);
SetSpaceAsDefault(main_space_); // Set as default to reset the proper dlmalloc space.
// Update performed homogeneous space compaction count.
count_performed_homogeneous_space_compaction_++;
// Print statics log and resume all threads.
uint64_t duration = NanoTime() - start_time;
VLOG(heap) << "Heap homogeneous space compaction took " << PrettyDuration(duration) << " size: "
<< PrettySize(space_size_before_compaction) << " -> "
<< PrettySize(space_size_after_compaction) << " compact-ratio: "
<< std::fixed << static_cast<double>(space_size_after_compaction) /
static_cast<double>(space_size_before_compaction);
}
// Finish GC.
// Get the references we need to enqueue.
SelfDeletingTask* clear = reference_processor_->CollectClearedReferences(self);
GrowForUtilization(semi_space_collector_);
LogGC(kGcCauseHomogeneousSpaceCompact, collector);
FinishGC(self, collector::kGcTypeFull);
// Enqueue any references after losing the GC locks.
clear->Run(self);
clear->Finalize();
{
ScopedObjectAccess soa(self);
soa.Vm()->UnloadNativeLibraries();
}
return HomogeneousSpaceCompactResult::kSuccess;
}
void Heap::SetDefaultConcurrentStartBytes() {
MutexLock mu(Thread::Current(), *gc_complete_lock_);
if (collector_type_running_ != kCollectorTypeNone) {
// If a collector is already running, just let it set concurrent_start_bytes_ .
return;
}
SetDefaultConcurrentStartBytesLocked();
}
void Heap::SetDefaultConcurrentStartBytesLocked() {
if (IsGcConcurrent()) {
size_t target_footprint = target_footprint_.load(std::memory_order_relaxed);
size_t reserve_bytes = target_footprint / 4;
reserve_bytes = std::min(reserve_bytes, kMaxConcurrentRemainingBytes);
reserve_bytes = std::max(reserve_bytes, kMinConcurrentRemainingBytes);
concurrent_start_bytes_ = UnsignedDifference(target_footprint, reserve_bytes);
} else {
concurrent_start_bytes_ = std::numeric_limits<size_t>::max();
}
}
void Heap::ChangeCollector(CollectorType collector_type) {
// TODO: Only do this with all mutators suspended to avoid races.
if (collector_type != collector_type_) {
collector_type_ = collector_type;
gc_plan_.clear();
switch (collector_type_) {
case kCollectorTypeCC: {
if (use_generational_cc_) {
gc_plan_.push_back(collector::kGcTypeSticky);
}
gc_plan_.push_back(collector::kGcTypeFull);
if (use_tlab_) {
ChangeAllocator(kAllocatorTypeRegionTLAB);
} else {
ChangeAllocator(kAllocatorTypeRegion);
}
break;
}
case kCollectorTypeCMC: {
gc_plan_.push_back(collector::kGcTypeFull);
if (use_tlab_) {
ChangeAllocator(kAllocatorTypeTLAB);
} else {
ChangeAllocator(kAllocatorTypeBumpPointer);
}
break;
}
case kCollectorTypeSS: {
gc_plan_.push_back(collector::kGcTypeFull);
if (use_tlab_) {
ChangeAllocator(kAllocatorTypeTLAB);
} else {
ChangeAllocator(kAllocatorTypeBumpPointer);
}
break;
}
case kCollectorTypeMS: {
gc_plan_.push_back(collector::kGcTypeSticky);
gc_plan_.push_back(collector::kGcTypePartial);
gc_plan_.push_back(collector::kGcTypeFull);
ChangeAllocator(kUseRosAlloc ? kAllocatorTypeRosAlloc : kAllocatorTypeDlMalloc);
break;
}
case kCollectorTypeCMS: {
gc_plan_.push_back(collector::kGcTypeSticky);
gc_plan_.push_back(collector::kGcTypePartial);
gc_plan_.push_back(collector::kGcTypeFull);
ChangeAllocator(kUseRosAlloc ? kAllocatorTypeRosAlloc : kAllocatorTypeDlMalloc);
break;
}
default: {
UNIMPLEMENTED(FATAL);
UNREACHABLE();
}
}
SetDefaultConcurrentStartBytesLocked();
}
}
// Special compacting collector which uses sub-optimal bin packing to reduce zygote space size.
class ZygoteCompactingCollector final : public collector::SemiSpace {
public:
ZygoteCompactingCollector(gc::Heap* heap, bool is_running_on_memory_tool)
: SemiSpace(heap, "zygote collector"),
bin_live_bitmap_(nullptr),
bin_mark_bitmap_(nullptr),
is_running_on_memory_tool_(is_running_on_memory_tool) {}
void BuildBins(space::ContinuousSpace* space) REQUIRES_SHARED(Locks::mutator_lock_) {
bin_live_bitmap_ = space->GetLiveBitmap();
bin_mark_bitmap_ = space->GetMarkBitmap();
uintptr_t prev = reinterpret_cast<uintptr_t>(space->Begin());
WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
// Note: This requires traversing the space in increasing order of object addresses.
auto visitor = [&](mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) {
uintptr_t object_addr = reinterpret_cast<uintptr_t>(obj);
size_t bin_size = object_addr - prev;
// Add the bin consisting of the end of the previous object to the start of the current object.
AddBin(bin_size, prev);
prev = object_addr + RoundUp(obj->SizeOf<kDefaultVerifyFlags>(), kObjectAlignment);
};
bin_live_bitmap_->Walk(visitor);
// Add the last bin which spans after the last object to the end of the space.
AddBin(reinterpret_cast<uintptr_t>(space->End()) - prev, prev);
}
private:
// Maps from bin sizes to locations.
std::multimap<size_t, uintptr_t> bins_;
// Live bitmap of the space which contains the bins.
accounting::ContinuousSpaceBitmap* bin_live_bitmap_;
// Mark bitmap of the space which contains the bins.
accounting::ContinuousSpaceBitmap* bin_mark_bitmap_;
const bool is_running_on_memory_tool_;
void AddBin(size_t size, uintptr_t position) {
if (is_running_on_memory_tool_) {
MEMORY_TOOL_MAKE_DEFINED(reinterpret_cast<void*>(position), size);
}
if (size != 0) {
bins_.insert(std::make_pair(size, position));
}
}
bool ShouldSweepSpace([[maybe_unused]] space::ContinuousSpace* space) const override {
// Don't sweep any spaces since we probably blasted the internal accounting of the free list
// allocator.
return false;
}
mirror::Object* MarkNonForwardedObject(mirror::Object* obj) override
REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_) {
size_t obj_size = obj->SizeOf<kDefaultVerifyFlags>();
size_t alloc_size = RoundUp(obj_size, kObjectAlignment);
mirror::Object* forward_address;
// Find the smallest bin which we can move obj in.
auto it = bins_.lower_bound(alloc_size);
if (it == bins_.end()) {
// No available space in the bins, place it in the target space instead (grows the zygote
// space).
size_t bytes_allocated, unused_bytes_tl_bulk_allocated;
forward_address = to_space_->Alloc(
self_, alloc_size, &bytes_allocated, nullptr, &unused_bytes_tl_bulk_allocated);
if (to_space_live_bitmap_ != nullptr) {
to_space_live_bitmap_->Set(forward_address);
} else {
GetHeap()->GetNonMovingSpace()->GetLiveBitmap()->Set(forward_address);
GetHeap()->GetNonMovingSpace()->GetMarkBitmap()->Set(forward_address);
}
} else {
size_t size = it->first;
uintptr_t pos = it->second;
bins_.erase(it); // Erase the old bin which we replace with the new smaller bin.
forward_address = reinterpret_cast<mirror::Object*>(pos);
// Set the live and mark bits so that sweeping system weaks works properly.
bin_live_bitmap_->Set(forward_address);
bin_mark_bitmap_->Set(forward_address);
DCHECK_GE(size, alloc_size);
// Add a new bin with the remaining space.
AddBin(size - alloc_size, pos + alloc_size);
}
// Copy the object over to its new location.
// Historical note: We did not use `alloc_size` to avoid a Valgrind error.
memcpy(reinterpret_cast<void*>(forward_address), obj, obj_size);
if (kUseBakerReadBarrier) {
obj->AssertReadBarrierState();
forward_address->AssertReadBarrierState();
}
return forward_address;
}
};
void Heap::UnBindBitmaps() {
TimingLogger::ScopedTiming t("UnBindBitmaps", GetCurrentGcIteration()->GetTimings());
for (const auto& space : GetContinuousSpaces()) {
if (space->IsContinuousMemMapAllocSpace()) {
space::ContinuousMemMapAllocSpace* alloc_space = space->AsContinuousMemMapAllocSpace();
if (alloc_space->GetLiveBitmap() != nullptr && alloc_space->HasBoundBitmaps()) {
alloc_space->UnBindBitmaps();
}
}
}
}
void Heap::IncrementFreedEver() {
// Counters are updated only by us, but may be read concurrently.
// The updates should become visible after the corresponding live object info.
total_objects_freed_ever_.store(total_objects_freed_ever_.load(std::memory_order_relaxed)
+ GetCurrentGcIteration()->GetFreedObjects()
+ GetCurrentGcIteration()->GetFreedLargeObjects(),
std::memory_order_release);
total_bytes_freed_ever_.store(total_bytes_freed_ever_.load(std::memory_order_relaxed)
+ GetCurrentGcIteration()->GetFreedBytes()
+ GetCurrentGcIteration()->GetFreedLargeObjectBytes(),
std::memory_order_release);
}
#pragma clang diagnostic push
#if !ART_USE_FUTEXES
// Frame gets too large, perhaps due to Bionic pthread_mutex_lock size. We don't care.
# pragma clang diagnostic ignored "-Wframe-larger-than="
#endif
// This has a large frame, but shouldn't be run anywhere near the stack limit.
// FIXME: BUT it did exceed... http://b/197647048
# pragma clang diagnostic ignored "-Wframe-larger-than="
void Heap::PreZygoteFork() {
if (!HasZygoteSpace()) {
// We still want to GC in case there is some unreachable non moving objects that could cause a
// suboptimal bin packing when we compact the zygote space.
CollectGarbageInternal(collector::kGcTypeFull, kGcCauseBackground, false, GC_NUM_ANY);
// Trim the pages at the end of the non moving space. Trim while not holding zygote lock since
// the trim process may require locking the mutator lock.
non_moving_space_->Trim();
}
// We need to close userfaultfd fd for app/webview zygotes to avoid getattr
// (stat) on the fd during fork.
Thread* self = Thread::Current();
MutexLock mu(self, zygote_creation_lock_);
// Try to see if we have any Zygote spaces.
if (HasZygoteSpace()) {
return;
}
Runtime* runtime = Runtime::Current();
// Setup linear-alloc pool for post-zygote fork allocations before freezing
// snapshots of intern-table and class-table.
runtime->SetupLinearAllocForPostZygoteFork(self);
runtime->GetInternTable()->AddNewTable();
runtime->GetClassLinker()->MoveClassTableToPreZygote();
VLOG(heap) << "Starting PreZygoteFork";
// The end of the non-moving space may be protected, unprotect it so that we can copy the zygote
// there.
non_moving_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
const bool same_space = non_moving_space_ == main_space_;
if (kCompactZygote) {
// Temporarily disable rosalloc verification because the zygote
// compaction will mess up the rosalloc internal metadata.
ScopedDisableRosAllocVerification disable_rosalloc_verif(this);
ZygoteCompactingCollector zygote_collector(this, is_running_on_memory_tool_);
zygote_collector.BuildBins(non_moving_space_);
// Create a new bump pointer space which we will compact into.
space::BumpPointerSpace target_space("zygote bump space", non_moving_space_->End(),
non_moving_space_->Limit());
// Compact the bump pointer space to a new zygote bump pointer space.
bool reset_main_space = false;
if (IsMovingGc(collector_type_)) {
if (collector_type_ == kCollectorTypeCC) {
zygote_collector.SetFromSpace(region_space_);
} else {
zygote_collector.SetFromSpace(bump_pointer_space_);
}
} else {
CHECK(main_space_ != nullptr);
CHECK_NE(main_space_, non_moving_space_)
<< "Does not make sense to compact within the same space";
// Copy from the main space.
zygote_collector.SetFromSpace(main_space_);
reset_main_space = true;
}
zygote_collector.SetToSpace(&target_space);
zygote_collector.SetSwapSemiSpaces(false);
zygote_collector.Run(kGcCauseCollectorTransition, false);
if (reset_main_space) {
main_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
madvise(main_space_->Begin(), main_space_->Capacity(), MADV_DONTNEED);
MemMap mem_map = main_space_->ReleaseMemMap();
RemoveSpace(main_space_);
space::Space* old_main_space = main_space_;
CreateMainMallocSpace(std::move(mem_map),
kDefaultInitialSize,
std::min(mem_map.Size(), growth_limit_),
mem_map.Size());
delete old_main_space;
AddSpace(main_space_);
} else {
if (collector_type_ == kCollectorTypeCC) {
region_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
// Evacuated everything out of the region space, clear the mark bitmap.
region_space_->GetMarkBitmap()->Clear();
} else {
bump_pointer_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
}
}
if (temp_space_ != nullptr) {
CHECK(temp_space_->IsEmpty());
}
IncrementFreedEver();
// Update the end and write out image.
non_moving_space_->SetEnd(target_space.End());
non_moving_space_->SetLimit(target_space.Limit());
VLOG(heap) << "Create zygote space with size=" << non_moving_space_->Size() << " bytes";
}
// Change the collector to the post zygote one.
ChangeCollector(foreground_collector_type_);
// Save the old space so that we can remove it after we complete creating the zygote space.
space::MallocSpace* old_alloc_space = non_moving_space_;
// Turn the current alloc space into a zygote space and obtain the new alloc space composed of
// the remaining available space.
// Remove the old space before creating the zygote space since creating the zygote space sets
// the old alloc space's bitmaps to null.
RemoveSpace(old_alloc_space);
if (collector::SemiSpace::kUseRememberedSet) {
// Consistency bound check.
FindRememberedSetFromSpace(old_alloc_space)->AssertAllDirtyCardsAreWithinSpace();
// Remove the remembered set for the now zygote space (the old
// non-moving space). Note now that we have compacted objects into
// the zygote space, the data in the remembered set is no longer
// needed. The zygote space will instead have a mod-union table
// from this point on.
RemoveRememberedSet(old_alloc_space);
}
// Remaining space becomes the new non moving space.
zygote_space_ = old_alloc_space->CreateZygoteSpace(kNonMovingSpaceName, low_memory_mode_,
&non_moving_space_);
CHECK(!non_moving_space_->CanMoveObjects());
if (same_space) {
main_space_ = non_moving_space_;
SetSpaceAsDefault(main_space_);
}
delete old_alloc_space;
CHECK(HasZygoteSpace()) << "Failed creating zygote space";
AddSpace(zygote_space_);
non_moving_space_->SetFootprintLimit(non_moving_space_->Capacity());
AddSpace(non_moving_space_);
constexpr bool set_mark_bit = kUseBakerReadBarrier
&& gc::collector::ConcurrentCopying::kGrayDirtyImmuneObjects;
if (set_mark_bit) {
// Treat all of the objects in the zygote as marked to avoid unnecessary dirty pages. This is
// safe since we mark all of the objects that may reference non immune objects as gray.
zygote_space_->SetMarkBitInLiveObjects();
}
// Create the zygote space mod union table.
accounting::ModUnionTable* mod_union_table =
new accounting::ModUnionTableCardCache("zygote space mod-union table", this, zygote_space_);
CHECK(mod_union_table != nullptr) << "Failed to create zygote space mod-union table";
if (collector_type_ != kCollectorTypeCC && collector_type_ != kCollectorTypeCMC) {
// Set all the cards in the mod-union table since we don't know which objects contain references
// to large objects.
mod_union_table->SetCards();
} else {
// Make sure to clear the zygote space cards so that we don't dirty pages in the next GC. There
// may be dirty cards from the zygote compaction or reference processing. These cards are not
// necessary to have marked since the zygote space may not refer to any objects not in the
// zygote or image spaces at this point.
mod_union_table->ProcessCards();
mod_union_table->ClearTable();
// For CC and CMC we never collect zygote large objects. This means we do not need to set the
// cards for the zygote mod-union table and we can also clear all of the existing image
// mod-union tables. The existing mod-union tables are only for image spaces and may only
// reference zygote and image objects.
for (auto& pair : mod_union_tables_) {
CHECK(pair.first->IsImageSpace());
CHECK(!pair.first->AsImageSpace()->GetImageHeader().IsAppImage());
accounting::ModUnionTable* table = pair.second;
table->ClearTable();
}
}
AddModUnionTable(mod_union_table);
large_object_space_->SetAllLargeObjectsAsZygoteObjects(self, set_mark_bit);
if (collector::SemiSpace::kUseRememberedSet) {
// Add a new remembered set for the post-zygote non-moving space.
accounting::RememberedSet* post_zygote_non_moving_space_rem_set =
new accounting::RememberedSet("Post-zygote non-moving space remembered set", this,
non_moving_space_);
CHECK(post_zygote_non_moving_space_rem_set != nullptr)
<< "Failed to create post-zygote non-moving space remembered set";
AddRememberedSet(post_zygote_non_moving_space_rem_set);
}
}
#pragma clang diagnostic pop
void Heap::FlushAllocStack() {
MarkAllocStackAsLive(allocation_stack_.get());
allocation_stack_->Reset();
}
void Heap::MarkAllocStack(accounting::ContinuousSpaceBitmap* bitmap1,
accounting::ContinuousSpaceBitmap* bitmap2,
accounting::LargeObjectBitmap* large_objects,
accounting::ObjectStack* stack) {
DCHECK(bitmap1 != nullptr);
DCHECK(bitmap2 != nullptr);
const auto* limit = stack->End();
for (auto* it = stack->Begin(); it != limit; ++it) {
const mirror::Object* obj = it->AsMirrorPtr();
if (!kUseThreadLocalAllocationStack || obj != nullptr) {
if (bitmap1->HasAddress(obj)) {
bitmap1->Set(obj);
} else if (bitmap2->HasAddress(obj)) {
bitmap2->Set(obj);
} else {
DCHECK(large_objects != nullptr);
large_objects->Set(obj);
}
}
}
}
void Heap::SwapSemiSpaces() {
CHECK(bump_pointer_space_ != nullptr);
CHECK(temp_space_ != nullptr);
std::swap(bump_pointer_space_, temp_space_);
}
collector::GarbageCollector* Heap::Compact(space::ContinuousMemMapAllocSpace* target_space,
space::ContinuousMemMapAllocSpace* source_space,
GcCause gc_cause) {
CHECK(kMovingCollector);
if (target_space != source_space) {
// Don't swap spaces since this isn't a typical semi space collection.
semi_space_collector_->SetSwapSemiSpaces(false);
semi_space_collector_->SetFromSpace(source_space);
semi_space_collector_->SetToSpace(target_space);
semi_space_collector_->Run(gc_cause, false);
return semi_space_collector_;
}
LOG(FATAL) << "Unsupported";
UNREACHABLE();
}
void Heap::TraceHeapSize(size_t heap_size) {
ATraceIntegerValue("Heap size (KB)", heap_size / KB);
}
#if defined(__GLIBC__)
# define IF_GLIBC(x) x
#else
# define IF_GLIBC(x)
#endif
size_t Heap::GetNativeBytes() {
size_t malloc_bytes;
#if defined(__BIONIC__) || defined(__GLIBC__) || defined(ANDROID_HOST_MUSL)
IF_GLIBC(size_t mmapped_bytes;)
struct mallinfo mi = mallinfo();
// In spite of the documentation, the jemalloc version of this call seems to do what we want,
// and it is thread-safe.
if (sizeof(size_t) > sizeof(mi.uordblks) && sizeof(size_t) > sizeof(mi.hblkhd)) {
// Shouldn't happen, but glibc declares uordblks as int.
// Avoiding sign extension gets us correct behavior for another 2 GB.
malloc_bytes = (unsigned int)mi.uordblks;
IF_GLIBC(mmapped_bytes = (unsigned int)mi.hblkhd;)
} else {
malloc_bytes = mi.uordblks;
IF_GLIBC(mmapped_bytes = mi.hblkhd;)
}
// From the spec, it appeared mmapped_bytes <= malloc_bytes. Reality was sometimes
// dramatically different. (b/119580449 was an early bug.) If so, we try to fudge it.
// However, malloc implementations seem to interpret hblkhd differently, namely as
// mapped blocks backing the entire heap (e.g. jemalloc) vs. large objects directly
// allocated via mmap (e.g. glibc). Thus we now only do this for glibc, where it
// previously helped, and which appears to use a reading of the spec compatible
// with our adjustment.
#if defined(__GLIBC__)
if (mmapped_bytes > malloc_bytes) {
malloc_bytes = mmapped_bytes;
}
#endif // GLIBC
#else // Neither Bionic nor Glibc
// We should hit this case only in contexts in which GC triggering is not critical. Effectively
// disable GC triggering based on malloc().
malloc_bytes = 1000;
#endif
return malloc_bytes + native_bytes_registered_.load(std::memory_order_relaxed);
// An alternative would be to get RSS from /proc/self/statm. Empirically, that's no
// more expensive, and it would allow us to count memory allocated by means other than malloc.
// However it would change as pages are unmapped and remapped due to memory pressure, among
// other things. It seems risky to trigger GCs as a result of such changes.
}
static inline bool GCNumberLt(uint32_t gc_num1, uint32_t gc_num2) {
// unsigned comparison, assuming a non-huge difference, but dealing correctly with wrapping.
uint32_t difference = gc_num2 - gc_num1;
bool completed_more_than_requested = difference > 0x80000000;
return difference > 0 && !completed_more_than_requested;
}
collector::GcType Heap::CollectGarbageInternal(collector::GcType gc_type,
GcCause gc_cause,
bool clear_soft_references,
uint32_t requested_gc_num) {
Thread* self = Thread::Current();
Runtime* runtime = Runtime::Current();
// If the heap can't run the GC, silently fail and return that no GC was run.
switch (gc_type) {
case collector::kGcTypePartial: {
if (!HasZygoteSpace()) {
// Do not increment gcs_completed_ . We should retry with kGcTypeFull.
return collector::kGcTypeNone;
}
break;
}
default: {
// Other GC types don't have any special cases which makes them not runnable. The main case
// here is full GC.
}
}
ScopedThreadStateChange tsc(self, ThreadState::kWaitingPerformingGc);
Locks::mutator_lock_->AssertNotHeld(self);
if (self->IsHandlingStackOverflow()) {
// If we are throwing a stack overflow error we probably don't have enough remaining stack
// space to run the GC.
// Count this as a GC in case someone is waiting for it to complete.
gcs_completed_.fetch_add(1, std::memory_order_release);
return collector::kGcTypeNone;
}
bool compacting_gc;
{
gc_complete_lock_->AssertNotHeld(self);
ScopedThreadStateChange tsc2(self, ThreadState::kWaitingForGcToComplete);
MutexLock mu(self, *gc_complete_lock_);
// Ensure there is only one GC at a time.
WaitForGcToCompleteLocked(gc_cause, self);
if (requested_gc_num != GC_NUM_ANY && !GCNumberLt(GetCurrentGcNum(), requested_gc_num)) {
// The appropriate GC was already triggered elsewhere.
return collector::kGcTypeNone;
}
compacting_gc = IsMovingGc(collector_type_);
// GC can be disabled if someone has a used GetPrimitiveArrayCritical.
if (compacting_gc && disable_moving_gc_count_ != 0) {
LOG(WARNING) << "Skipping GC due to disable moving GC count " << disable_moving_gc_count_;
// Again count this as a GC.
gcs_completed_.fetch_add(1, std::memory_order_release);
return collector::kGcTypeNone;
}
if (gc_disabled_for_shutdown_) {
gcs_completed_.fetch_add(1, std::memory_order_release);
return collector::kGcTypeNone;
}
collector_type_running_ = collector_type_;
last_gc_cause_ = gc_cause;
}
if (gc_cause == kGcCauseForAlloc && runtime->HasStatsEnabled()) {
++runtime->GetStats()->gc_for_alloc_count;
++self->GetStats()->gc_for_alloc_count;
}
const size_t bytes_allocated_before_gc = GetBytesAllocated();
DCHECK_LT(gc_type, collector::kGcTypeMax);
DCHECK_NE(gc_type, collector::kGcTypeNone);
collector::GarbageCollector* collector = nullptr;
// TODO: Clean this up.
if (compacting_gc) {
DCHECK(current_allocator_ == kAllocatorTypeBumpPointer ||
current_allocator_ == kAllocatorTypeTLAB ||
current_allocator_ == kAllocatorTypeRegion ||
current_allocator_ == kAllocatorTypeRegionTLAB);
switch (collector_type_) {
case kCollectorTypeSS:
semi_space_collector_->SetFromSpace(bump_pointer_space_);
semi_space_collector_->SetToSpace(temp_space_);
semi_space_collector_->SetSwapSemiSpaces(true);
collector = semi_space_collector_;
break;
case kCollectorTypeCMC:
collector = mark_compact_;
break;
case kCollectorTypeCC:
collector::ConcurrentCopying* active_cc_collector;
if (use_generational_cc_) {
// TODO: Other threads must do the flip checkpoint before they start poking at
// active_concurrent_copying_collector_. So we should not concurrency here.
active_cc_collector = (gc_type == collector::kGcTypeSticky) ?
young_concurrent_copying_collector_ : concurrent_copying_collector_;
active_concurrent_copying_collector_.store(active_cc_collector,
std::memory_order_relaxed);
DCHECK(active_cc_collector->RegionSpace() == region_space_);
collector = active_cc_collector;
} else {
collector = active_concurrent_copying_collector_.load(std::memory_order_relaxed);
}
break;
default:
LOG(FATAL) << "Invalid collector type " << static_cast<size_t>(collector_type_);
}
// temp_space_ will be null for kCollectorTypeCMC.
if (temp_space_ != nullptr
&& collector != active_concurrent_copying_collector_.load(std::memory_order_relaxed)) {
temp_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
if (kIsDebugBuild) {
// Try to read each page of the memory map in case mprotect didn't work properly b/19894268.
temp_space_->GetMemMap()->TryReadable();
}
CHECK(temp_space_->IsEmpty());
}
} else if (current_allocator_ == kAllocatorTypeRosAlloc ||
current_allocator_ == kAllocatorTypeDlMalloc) {
collector = FindCollectorByGcType(gc_type);
} else {
LOG(FATAL) << "Invalid current allocator " << current_allocator_;
}
CHECK(collector != nullptr)
<< "Could not find garbage collector with collector_type="
<< static_cast<size_t>(collector_type_) << " and gc_type=" << gc_type;
collector->Run(gc_cause, clear_soft_references || runtime->IsZygote());
IncrementFreedEver();
RequestTrim(self);
// Collect cleared references.
SelfDeletingTask* clear = reference_processor_->CollectClearedReferences(self);
// Grow the heap so that we know when to perform the next GC.
GrowForUtilization(collector, bytes_allocated_before_gc);
old_native_bytes_allocated_.store(GetNativeBytes());
LogGC(gc_cause, collector);
FinishGC(self, gc_type);
// Actually enqueue all cleared references. Do this after the GC has officially finished since
// otherwise we can deadlock.
clear->Run(self);
clear->Finalize();
// Inform DDMS that a GC completed.
Dbg::GcDidFinish();
// Unload native libraries for class unloading. We do this after calling FinishGC to prevent
// deadlocks in case the JNI_OnUnload function does allocations.
{
ScopedObjectAccess soa(self);
soa.Vm()->UnloadNativeLibraries();
}
return gc_type;
}
void Heap::LogGC(GcCause gc_cause, collector::GarbageCollector* collector) {
const size_t duration = GetCurrentGcIteration()->GetDurationNs();
const std::vector<uint64_t>& pause_times = GetCurrentGcIteration()->GetPauseTimes();
// Print the GC if it is an explicit GC (e.g. Runtime.gc()) or a slow GC
// (mutator time blocked >= long_pause_log_threshold_).
bool log_gc = kLogAllGCs || (gc_cause == kGcCauseExplicit && always_log_explicit_gcs_);
if (!log_gc && CareAboutPauseTimes()) {
// GC for alloc pauses the allocating thread, so consider it as a pause.
log_gc = duration > long_gc_log_threshold_ ||
(gc_cause == kGcCauseForAlloc && duration > long_pause_log_threshold_);
for (uint64_t pause : pause_times) {
log_gc = log_gc || pause >= long_pause_log_threshold_;
}
}
bool is_sampled = false;
if (UNLIKELY(gc_stress_mode_)) {
static std::atomic_int64_t accumulated_duration_ns = 0;
accumulated_duration_ns += duration;
if (accumulated_duration_ns >= kGcStressModeGcLogSampleFrequencyNs) {
accumulated_duration_ns -= kGcStressModeGcLogSampleFrequencyNs;
log_gc = true;
is_sampled = true;
}
}
if (log_gc) {
const size_t percent_free = GetPercentFree();
const size_t current_heap_size = GetBytesAllocated();
const size_t total_memory = GetTotalMemory();
std::ostringstream pause_string;
for (size_t i = 0; i < pause_times.size(); ++i) {
pause_string << PrettyDuration((pause_times[i] / 1000) * 1000)
<< ((i != pause_times.size() - 1) ? "," : "");
}
LOG(INFO) << gc_cause << " " << collector->GetName()
<< (is_sampled ? " (sampled)" : "")
<< " GC freed "
<< PrettySize(current_gc_iteration_.GetFreedBytes()) << " AllocSpace bytes, "
<< current_gc_iteration_.GetFreedLargeObjects() << "("
<< PrettySize(current_gc_iteration_.GetFreedLargeObjectBytes()) << ") LOS objects, "
<< percent_free << "% free, " << PrettySize(current_heap_size) << "/"
<< PrettySize(total_memory) << ", " << "paused " << pause_string.str()
<< " total " << PrettyDuration((duration / 1000) * 1000);
VLOG(heap) << Dumpable<TimingLogger>(*current_gc_iteration_.GetTimings());
}
}
void Heap::FinishGC(Thread* self, collector::GcType gc_type) {
MutexLock mu(self, *gc_complete_lock_);
collector_type_running_ = kCollectorTypeNone;
if (gc_type != collector::kGcTypeNone) {
last_gc_type_ = gc_type;
// Update stats.
++gc_count_last_window_;
if (running_collection_is_blocking_) {
// If the currently running collection was a blocking one,
// increment the counters and reset the flag.
++blocking_gc_count_;
blocking_gc_time_ += GetCurrentGcIteration()->GetDurationNs();
++blocking_gc_count_last_window_;
}
// Update the gc count rate histograms if due.
UpdateGcCountRateHistograms();
}
// Reset.
running_collection_is_blocking_ = false;
thread_running_gc_ = nullptr;
if (gc_type != collector::kGcTypeNone) {
gcs_completed_.fetch_add(1, std::memory_order_release);
}
// Wake anyone who may have been waiting for the GC to complete.
gc_complete_cond_->Broadcast(self);
}
void Heap::UpdateGcCountRateHistograms() {
// Invariant: if the time since the last update includes more than
// one windows, all the GC runs (if > 0) must have happened in first
// window because otherwise the update must have already taken place
// at an earlier GC run. So, we report the non-first windows with
// zero counts to the histograms.
DCHECK_EQ(last_update_time_gc_count_rate_histograms_ % kGcCountRateHistogramWindowDuration, 0U);
uint64_t now = NanoTime();
DCHECK_GE(now, last_update_time_gc_count_rate_histograms_);
uint64_t time_since_last_update = now - last_update_time_gc_count_rate_histograms_;
uint64_t num_of_windows = time_since_last_update / kGcCountRateHistogramWindowDuration;
// The computed number of windows can be incoherently high if NanoTime() is not monotonic.
// Setting a limit on its maximum value reduces the impact on CPU time in such cases.
if (num_of_windows > kGcCountRateHistogramMaxNumMissedWindows) {
LOG(WARNING) << "Reducing the number of considered missed Gc histogram windows from "
<< num_of_windows << " to " << kGcCountRateHistogramMaxNumMissedWindows;
num_of_windows = kGcCountRateHistogramMaxNumMissedWindows;
}
if (time_since_last_update >= kGcCountRateHistogramWindowDuration) {
// Record the first window.
gc_count_rate_histogram_.AddValue(gc_count_last_window_ - 1); // Exclude the current run.
blocking_gc_count_rate_histogram_.AddValue(running_collection_is_blocking_ ?
blocking_gc_count_last_window_ - 1 : blocking_gc_count_last_window_);
// Record the other windows (with zero counts).
for (uint64_t i = 0; i < num_of_windows - 1; ++i) {
gc_count_rate_histogram_.AddValue(0);
blocking_gc_count_rate_histogram_.AddValue(0);
}
// Update the last update time and reset the counters.
last_update_time_gc_count_rate_histograms_ =
(now / kGcCountRateHistogramWindowDuration) * kGcCountRateHistogramWindowDuration;
gc_count_last_window_ = 1; // Include the current run.
blocking_gc_count_last_window_ = running_collection_is_blocking_ ? 1 : 0;
}
DCHECK_EQ(last_update_time_gc_count_rate_histograms_ % kGcCountRateHistogramWindowDuration, 0U);
}
class RootMatchesObjectVisitor : public SingleRootVisitor {
public:
explicit RootMatchesObjectVisitor(const mirror::Object* obj) : obj_(obj) { }
void VisitRoot(mirror::Object* root, const RootInfo& info)
override REQUIRES_SHARED(Locks::mutator_lock_) {
if (root == obj_) {
LOG(INFO) << "Object " << obj_ << " is a root " << info.ToString();
}
}
private:
const mirror::Object* const obj_;
};
class ScanVisitor {
public:
void operator()(const mirror::Object* obj) const {
LOG(ERROR) << "Would have rescanned object " << obj;
}
};
// Verify a reference from an object.
class VerifyReferenceVisitor : public SingleRootVisitor {
public:
VerifyReferenceVisitor(Thread* self, Heap* heap, size_t* fail_count, bool verify_referent)
REQUIRES_SHARED(Locks::mutator_lock_)
: self_(self), heap_(heap), fail_count_(fail_count), verify_referent_(verify_referent) {
CHECK_EQ(self_, Thread::Current());
}
void operator()([[maybe_unused]] ObjPtr<mirror::Class> klass, ObjPtr<mirror::Reference> ref) const
REQUIRES_SHARED(Locks::mutator_lock_) {
if (verify_referent_) {
VerifyReference(ref.Ptr(), ref->GetReferent(), mirror::Reference::ReferentOffset());
}
}
void operator()(ObjPtr<mirror::Object> obj,
MemberOffset offset,
[[maybe_unused]] bool is_static) const REQUIRES_SHARED(Locks::mutator_lock_) {
VerifyReference(obj.Ptr(), obj->GetFieldObject<mirror::Object>(offset), offset);
}
bool IsLive(ObjPtr<mirror::Object> obj) const NO_THREAD_SAFETY_ANALYSIS {
return heap_->IsLiveObjectLocked(obj, true, false, true);
}
void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const
REQUIRES_SHARED(Locks::mutator_lock_) {
if (!root->IsNull()) {
VisitRoot(root);
}
}
void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const
REQUIRES_SHARED(Locks::mutator_lock_) {
const_cast<VerifyReferenceVisitor*>(this)->VisitRoot(
root->AsMirrorPtr(), RootInfo(kRootVMInternal));
}
void VisitRoot(mirror::Object* root, const RootInfo& root_info) override
REQUIRES_SHARED(Locks::mutator_lock_) {
if (root == nullptr) {
LOG(ERROR) << "Root is null with info " << root_info.GetType();
} else if (!VerifyReference(nullptr, root, MemberOffset(0))) {
LOG(ERROR) << "Root " << root << " is dead with type " << mirror::Object::PrettyTypeOf(root)
<< " thread_id= " << root_info.GetThreadId() << " root_type= " << root_info.GetType();
}
}
private:
// TODO: Fix the no thread safety analysis.
// Returns false on failure.
bool VerifyReference(mirror::Object* obj, mirror::Object* ref, MemberOffset offset) const
NO_THREAD_SAFETY_ANALYSIS {
if (ref == nullptr || IsLive(ref)) {
// Verify that the reference is live.
return true;
}
CHECK_EQ(self_, Thread::Current()); // fail_count_ is private to the calling thread.
*fail_count_ += 1;
if (*fail_count_ == 1) {
// Only print message for the first failure to prevent spam.
LOG(ERROR) << "!!!!!!!!!!!!!!Heap corruption detected!!!!!!!!!!!!!!!!!!!";
}
if (obj != nullptr) {
// Only do this part for non roots.
accounting::CardTable* card_table = heap_->GetCardTable();
accounting::ObjectStack* alloc_stack = heap_->allocation_stack_.get();
accounting::ObjectStack* live_stack = heap_->live_stack_.get();
uint8_t* card_addr = card_table->CardFromAddr(obj);
LOG(ERROR) << "Object " << obj << " references dead object " << ref << " at offset "
<< offset << "\n card value = " << static_cast<int>(*card_addr);
if (heap_->IsValidObjectAddress(obj->GetClass())) {
LOG(ERROR) << "Obj type " << obj->PrettyTypeOf();
} else {
LOG(ERROR) << "Object " << obj << " class(" << obj->GetClass() << ") not a heap address";
}
// Attempt to find the class inside of the recently freed objects.
space::ContinuousSpace* ref_space = heap_->FindContinuousSpaceFromObject(ref, true);
if (ref_space != nullptr && ref_space->IsMallocSpace()) {
space::MallocSpace* space = ref_space->AsMallocSpace();
mirror::Class* ref_class = space->FindRecentFreedObject(ref);
if (ref_class != nullptr) {
LOG(ERROR) << "Reference " << ref << " found as a recently freed object with class "
<< ref_class->PrettyClass();
} else {
LOG(ERROR) << "Reference " << ref << " not found as a recently freed object";
}
}
if (ref->GetClass() != nullptr && heap_->IsValidObjectAddress(ref->GetClass()) &&
ref->GetClass()->IsClass()) {
LOG(ERROR) << "Ref type " << ref->PrettyTypeOf();
} else {
LOG(ERROR) << "Ref " << ref << " class(" << ref->GetClass()
<< ") is not a valid heap address";
}
card_table->CheckAddrIsInCardTable(reinterpret_cast<const uint8_t*>(obj));
void* cover_begin = card_table->AddrFromCard(card_addr);
void* cover_end = reinterpret_cast<void*>(reinterpret_cast<size_t>(cover_begin) +
accounting::CardTable::kCardSize);
LOG(ERROR) << "Card " << reinterpret_cast<void*>(card_addr) << " covers " << cover_begin
<< "-" << cover_end;
accounting::ContinuousSpaceBitmap* bitmap =
heap_->GetLiveBitmap()->GetContinuousSpaceBitmap(obj);
if (bitmap == nullptr) {
LOG(ERROR) << "Object " << obj << " has no bitmap";
if (!VerifyClassClass(obj->GetClass())) {
LOG(ERROR) << "Object " << obj << " failed class verification!";
}
} else {
// Print out how the object is live.
if (bitmap->Test(obj)) {
LOG(ERROR) << "Object " << obj << " found in live bitmap";
}
if (alloc_stack->Contains(const_cast<mirror::Object*>(obj))) {
LOG(ERROR) << "Object " << obj << " found in allocation stack";
}
if (live_stack->Contains(const_cast<mirror::Object*>(obj))) {
LOG(ERROR) << "Object " << obj << " found in live stack";
}
if (alloc_stack->Contains(const_cast<mirror::Object*>(ref))) {
LOG(ERROR) << "Ref " << ref << " found in allocation stack";
}
if (live_stack->Contains(const_cast<mirror::Object*>(ref))) {
LOG(ERROR) << "Ref " << ref << " found in live stack";
}
// Attempt to see if the card table missed the reference.
ScanVisitor scan_visitor;
uint8_t* byte_cover_begin = reinterpret_cast<uint8_t*>(card_table->AddrFromCard(card_addr));
card_table->Scan<false>(bitmap, byte_cover_begin,
byte_cover_begin + accounting::CardTable::kCardSize, scan_visitor);
}
// Search to see if any of the roots reference our object.
RootMatchesObjectVisitor visitor1(obj);
Runtime::Current()->VisitRoots(&visitor1);
// Search to see if any of the roots reference our reference.
RootMatchesObjectVisitor visitor2(ref);
Runtime::Current()->VisitRoots(&visitor2);
}
return false;
}
Thread* const self_;
Heap* const heap_;
size_t* const fail_count_;
const bool verify_referent_;
};
// Verify all references within an object, for use with HeapBitmap::Visit.
class VerifyObjectVisitor {
public:
VerifyObjectVisitor(Thread* self, Heap* heap, size_t* fail_count, bool verify_referent)
: self_(self), heap_(heap), fail_count_(fail_count), verify_referent_(verify_referent) {}
void operator()(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) {
// Note: we are verifying the references in obj but not obj itself, this is because obj must
// be live or else how did we find it in the live bitmap?
VerifyReferenceVisitor visitor(self_, heap_, fail_count_, verify_referent_);
// The class doesn't count as a reference but we should verify it anyways.
obj->VisitReferences(visitor, visitor);
}
void VerifyRoots() REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!Locks::heap_bitmap_lock_) {
ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
VerifyReferenceVisitor visitor(self_, heap_, fail_count_, verify_referent_);
Runtime::Current()->VisitRoots(&visitor);
}
uint32_t GetFailureCount() const REQUIRES(Locks::mutator_lock_) {
CHECK_EQ(self_, Thread::Current());
return *fail_count_;
}
private:
Thread* const self_;
Heap* const heap_;
size_t* const fail_count_;
const bool verify_referent_;
};
void Heap::PushOnAllocationStackWithInternalGC(Thread* self, ObjPtr<mirror::Object>* obj) {
// Slow path, the allocation stack push back must have already failed.
DCHECK(!allocation_stack_->AtomicPushBack(obj->Ptr()));
do {
// TODO: Add handle VerifyObject.
StackHandleScope<1> hs(self);
HandleWrapperObjPtr<mirror::Object> wrapper(hs.NewHandleWrapper(obj));
// Push our object into the reserve region of the allocation stack. This is only required due
// to heap verification requiring that roots are live (either in the live bitmap or in the
// allocation stack).
CHECK(allocation_stack_->AtomicPushBackIgnoreGrowthLimit(obj->Ptr()));
CollectGarbageInternal(collector::kGcTypeSticky,
kGcCauseForAlloc,
false,
GetCurrentGcNum() + 1);
} while (!allocation_stack_->AtomicPushBack(obj->Ptr()));
}
void Heap::PushOnThreadLocalAllocationStackWithInternalGC(Thread* self,
ObjPtr<mirror::Object>* obj) {
// Slow path, the allocation stack push back must have already failed.
DCHECK(!self->PushOnThreadLocalAllocationStack(obj->Ptr()));
StackReference<mirror::Object>* start_address;
StackReference<mirror::Object>* end_address;
while (!allocation_stack_->AtomicBumpBack(kThreadLocalAllocationStackSize, &start_address,
&end_address)) {
// TODO: Add handle VerifyObject.
StackHandleScope<1> hs(self);
HandleWrapperObjPtr<mirror::Object> wrapper(hs.NewHandleWrapper(obj));
// Push our object into the reserve region of the allocaiton stack. This is only required due
// to heap verification requiring that roots are live (either in the live bitmap or in the
// allocation stack).
CHECK(allocation_stack_->AtomicPushBackIgnoreGrowthLimit(obj->Ptr()));
// Push into the reserve allocation stack.
CollectGarbageInternal(collector::kGcTypeSticky,
kGcCauseForAlloc,
false,
GetCurrentGcNum() + 1);
}
self->SetThreadLocalAllocationStack(start_address, end_address);
// Retry on the new thread-local allocation stack.
CHECK(self->PushOnThreadLocalAllocationStack(obj->Ptr())); // Must succeed.
}
// Must do this with mutators suspended since we are directly accessing the allocation stacks.
size_t Heap::VerifyHeapReferences(bool verify_referents) {
Thread* self = Thread::Current();
Locks::mutator_lock_->AssertExclusiveHeld(self);
// Lets sort our allocation stacks so that we can efficiently binary search them.
allocation_stack_->Sort();
live_stack_->Sort();
// Since we sorted the allocation stack content, need to revoke all
// thread-local allocation stacks.
RevokeAllThreadLocalAllocationStacks(self);
size_t fail_count = 0;
VerifyObjectVisitor visitor(self, this, &fail_count, verify_referents);
// Verify objects in the allocation stack since these will be objects which were:
// 1. Allocated prior to the GC (pre GC verification).
// 2. Allocated during the GC (pre sweep GC verification).
// We don't want to verify the objects in the live stack since they themselves may be
// pointing to dead objects if they are not reachable.
VisitObjectsPaused(visitor);
// Verify the roots:
visitor.VerifyRoots();
if (visitor.GetFailureCount() > 0) {
// Dump mod-union tables.
for (const auto& table_pair : mod_union_tables_) {
accounting::ModUnionTable* mod_union_table = table_pair.second;
mod_union_table->Dump(LOG_STREAM(ERROR) << mod_union_table->GetName() << ": ");
}
// Dump remembered sets.
for (const auto& table_pair : remembered_sets_) {
accounting::RememberedSet* remembered_set = table_pair.second;
remembered_set->Dump(LOG_STREAM(ERROR) << remembered_set->GetName() << ": ");
}
DumpSpaces(LOG_STREAM(ERROR));
}
return visitor.GetFailureCount();
}
class VerifyReferenceCardVisitor {
public:
VerifyReferenceCardVisitor(Heap* heap, bool* failed)
REQUIRES_SHARED(Locks::mutator_lock_,
Locks::heap_bitmap_lock_)
: heap_(heap), failed_(failed) {
}
// There is no card marks for native roots on a class.
void VisitRootIfNonNull(
[[maybe_unused]] mirror::CompressedReference<mirror::Object>* root) const {}
void VisitRoot([[maybe_unused]] mirror::CompressedReference<mirror::Object>* root) const {}
// TODO: Fix lock analysis to not use NO_THREAD_SAFETY_ANALYSIS, requires support for
// annotalysis on visitors.
void operator()(mirror::Object* obj, MemberOffset offset, bool is_static) const
NO_THREAD_SAFETY_ANALYSIS {
mirror::Object* ref = obj->GetFieldObject<mirror::Object>(offset);
// Filter out class references since changing an object's class does not mark the card as dirty.
// Also handles large objects, since the only reference they hold is a class reference.
if (ref != nullptr && !ref->IsClass()) {
accounting::CardTable* card_table = heap_->GetCardTable();
// If the object is not dirty and it is referencing something in the live stack other than
// class, then it must be on a dirty card.
if (!card_table->AddrIsInCardTable(obj)) {
LOG(ERROR) << "Object " << obj << " is not in the address range of the card table";
*failed_ = true;
} else if (!card_table->IsDirty(obj)) {
// TODO: Check mod-union tables.
// Card should be either kCardDirty if it got re-dirtied after we aged it, or
// kCardDirty - 1 if it didnt get touched since we aged it.
accounting::ObjectStack* live_stack = heap_->live_stack_.get();
if (live_stack->ContainsSorted(ref)) {
if (live_stack->ContainsSorted(obj)) {
LOG(ERROR) << "Object " << obj << " found in live stack";
}
if (heap_->GetLiveBitmap()->Test(obj)) {
LOG(ERROR) << "Object " << obj << " found in live bitmap";
}
LOG(ERROR) << "Object " << obj << " " << mirror::Object::PrettyTypeOf(obj)
<< " references " << ref << " " << mirror::Object::PrettyTypeOf(ref)
<< " in live stack";
// Print which field of the object is dead.
if (!obj->IsObjectArray()) {
ObjPtr<mirror::Class> klass = is_static ? obj->AsClass() : obj->GetClass();
CHECK(klass != nullptr);
for (ArtField& field : (is_static ? klass->GetSFields() : klass->GetIFields())) {
if (field.GetOffset().Int32Value() == offset.Int32Value()) {
LOG(ERROR) << (is_static ? "Static " : "") << "field in the live stack is "
<< field.PrettyField();
break;
}
}
} else {
ObjPtr<mirror::ObjectArray<mirror::Object>> object_array =
obj->AsObjectArray<mirror::Object>();
for (int32_t i = 0; i < object_array->GetLength(); ++i) {
if (object_array->Get(i) == ref) {
LOG(ERROR) << (is_static ? "Static " : "") << "obj[" << i << "] = ref";
}
}
}
*failed_ = true;
}
}
}
}
private:
Heap* const heap_;
bool* const failed_;
};
class VerifyLiveStackReferences {
public:
explicit VerifyLiveStackReferences(Heap* heap)
: heap_(heap),
failed_(false) {}
void operator()(mirror::Object* obj) const
REQUIRES_SHARED(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
VerifyReferenceCardVisitor visitor(heap_, const_cast<bool*>(&failed_));
obj->VisitReferences(visitor, VoidFunctor());
}
bool Failed() const {
return failed_;
}
private:
Heap* const heap_;
bool failed_;
};
bool Heap::VerifyMissingCardMarks() {
Thread* self = Thread::Current();
Locks::mutator_lock_->AssertExclusiveHeld(self);
// We need to sort the live stack since we binary search it.
live_stack_->Sort();
// Since we sorted the allocation stack content, need to revoke all
// thread-local allocation stacks.
RevokeAllThreadLocalAllocationStacks(self);
VerifyLiveStackReferences visitor(this);
GetLiveBitmap()->Visit(visitor);
// We can verify objects in the live stack since none of these should reference dead objects.
for (auto* it = live_stack_->Begin(); it != live_stack_->End(); ++it) {
if (!kUseThreadLocalAllocationStack || it->AsMirrorPtr() != nullptr) {
visitor(it->AsMirrorPtr());
}
}
return !visitor.Failed();
}
void Heap::SwapStacks() {
if (kUseThreadLocalAllocationStack) {
live_stack_->AssertAllZero();
}
allocation_stack_.swap(live_stack_);
}
void Heap::RevokeAllThreadLocalAllocationStacks(Thread* self) {
// This must be called only during the pause.
DCHECK(Locks::mutator_lock_->IsExclusiveHeld(self));
MutexLock mu(self, *Locks::runtime_shutdown_lock_);
MutexLock mu2(self, *Locks::thread_list_lock_);
std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList();
for (Thread* t : thread_list) {
t->RevokeThreadLocalAllocationStack();
}
}
void Heap::AssertThreadLocalBuffersAreRevoked(Thread* thread) {
if (kIsDebugBuild) {
if (rosalloc_space_ != nullptr) {
rosalloc_space_->AssertThreadLocalBuffersAreRevoked(thread);
}
if (bump_pointer_space_ != nullptr) {
bump_pointer_space_->AssertThreadLocalBuffersAreRevoked(thread);
}
}
}
void Heap::AssertAllBumpPointerSpaceThreadLocalBuffersAreRevoked() {
if (kIsDebugBuild) {
if (bump_pointer_space_ != nullptr) {
bump_pointer_space_->AssertAllThreadLocalBuffersAreRevoked();
}
}
}
accounting::ModUnionTable* Heap::FindModUnionTableFromSpace(space::Space* space) {
auto it = mod_union_tables_.find(space);
if (it == mod_union_tables_.end()) {
return nullptr;
}
return it->second;
}
accounting::RememberedSet* Heap::FindRememberedSetFromSpace(space::Space* space) {
auto it = remembered_sets_.find(space);
if (it == remembered_sets_.end()) {
return nullptr;
}
return it->second;
}
void Heap::ProcessCards(TimingLogger* timings,
bool use_rem_sets,
bool process_alloc_space_cards,
bool clear_alloc_space_cards) {
TimingLogger::ScopedTiming t(__FUNCTION__, timings);
// Clear cards and keep track of cards cleared in the mod-union table.
for (const auto& space : continuous_spaces_) {
accounting::ModUnionTable* table = FindModUnionTableFromSpace(space);
accounting::RememberedSet* rem_set = FindRememberedSetFromSpace(space);
if (table != nullptr) {
const char* name = space->IsZygoteSpace() ? "ZygoteModUnionClearCards" :
"ImageModUnionClearCards";
TimingLogger::ScopedTiming t2(name, timings);
table->ProcessCards();
} else if (use_rem_sets && rem_set != nullptr) {
DCHECK(collector::SemiSpace::kUseRememberedSet) << static_cast<int>(collector_type_);
TimingLogger::ScopedTiming t2("AllocSpaceRemSetClearCards", timings);
rem_set->ClearCards();
} else if (process_alloc_space_cards) {
TimingLogger::ScopedTiming t2("AllocSpaceClearCards", timings);
if (clear_alloc_space_cards) {
uint8_t* end = space->End();
if (space->IsImageSpace()) {
// Image space end is the end of the mirror objects, it is not necessarily page or card
// aligned. Align up so that the check in ClearCardRange does not fail.
end = AlignUp(end, accounting::CardTable::kCardSize);
}
card_table_->ClearCardRange(space->Begin(), end);
} else {
// No mod union table for the AllocSpace. Age the cards so that the GC knows that these
// cards were dirty before the GC started.
// TODO: Need to use atomic for the case where aged(cleaning thread) -> dirty(other thread)
// -> clean(cleaning thread).
// The races are we either end up with: Aged card, unaged card. Since we have the
// checkpoint roots and then we scan / update mod union tables after. We will always
// scan either card. If we end up with the non aged card, we scan it it in the pause.
card_table_->ModifyCardsAtomic(space->Begin(), space->End(), AgeCardVisitor(),
VoidFunctor());
}
}
}
}
struct IdentityMarkHeapReferenceVisitor : public MarkObjectVisitor {
mirror::Object* MarkObject(mirror::Object* obj) override {
return obj;
}
void MarkHeapReference(mirror::HeapReference<mirror::Object>*, bool) override {
}
};
void Heap::PreGcVerificationPaused(collector::GarbageCollector* gc) {
Thread* const self = Thread::Current();
TimingLogger* const timings = current_gc_iteration_.GetTimings();
TimingLogger::ScopedTiming t(__FUNCTION__, timings);
if (verify_pre_gc_heap_) {
TimingLogger::ScopedTiming t2("(Paused)PreGcVerifyHeapReferences", timings);
size_t failures = VerifyHeapReferences();
if (failures > 0) {
LOG(FATAL) << "Pre " << gc->GetName() << " heap verification failed with " << failures
<< " failures";
}
}
// Check that all objects which reference things in the live stack are on dirty cards.
if (verify_missing_card_marks_) {
TimingLogger::ScopedTiming t2("(Paused)PreGcVerifyMissingCardMarks", timings);
ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_);
SwapStacks();
// Sort the live stack so that we can quickly binary search it later.
CHECK(VerifyMissingCardMarks()) << "Pre " << gc->GetName()
<< " missing card mark verification failed\n" << DumpSpaces();
SwapStacks();
}
if (verify_mod_union_table_) {
TimingLogger::ScopedTiming t2("(Paused)PreGcVerifyModUnionTables", timings);
ReaderMutexLock reader_lock(self, *Locks::heap_bitmap_lock_);
for (const auto& table_pair : mod_union_tables_) {
accounting::ModUnionTable* mod_union_table = table_pair.second;
IdentityMarkHeapReferenceVisitor visitor;
mod_union_table->UpdateAndMarkReferences(&visitor);
mod_union_table->Verify();
}
}
}
void Heap::PreGcVerification(collector::GarbageCollector* gc) {
if (verify_pre_gc_heap_ || verify_missing_card_marks_ || verify_mod_union_table_) {
collector::GarbageCollector::ScopedPause pause(gc, false);
PreGcVerificationPaused(gc);
}
}
void Heap::PrePauseRosAllocVerification([[maybe_unused]] collector::GarbageCollector* gc) {
// TODO: Add a new runtime option for this?
if (verify_pre_gc_rosalloc_) {
RosAllocVerification(current_gc_iteration_.GetTimings(), "PreGcRosAllocVerification");
}
}
void Heap::PreSweepingGcVerification(collector::GarbageCollector* gc) {
Thread* const self = Thread::Current();
TimingLogger* const timings = current_gc_iteration_.GetTimings();
TimingLogger::ScopedTiming t(__FUNCTION__, timings);
// Called before sweeping occurs since we want to make sure we are not going so reclaim any
// reachable objects.
if (verify_pre_sweeping_heap_) {
TimingLogger::ScopedTiming t2("(Paused)PostSweepingVerifyHeapReferences", timings);
CHECK_NE(self->GetState(), ThreadState::kRunnable);
{
WriterMutexLock mu(self, *Locks::heap_bitmap_lock_);
// Swapping bound bitmaps does nothing.
gc->SwapBitmaps();
}
// Pass in false since concurrent reference processing can mean that the reference referents
// may point to dead objects at the point which PreSweepingGcVerification is called.
size_t failures = VerifyHeapReferences(false);
if (failures > 0) {
LOG(FATAL) << "Pre sweeping " << gc->GetName() << " GC verification failed with " << failures
<< " failures";
}
{
WriterMutexLock mu(self, *Locks::heap_bitmap_lock_);
gc->SwapBitmaps();
}
}
if (verify_pre_sweeping_rosalloc_) {
RosAllocVerification(timings, "PreSweepingRosAllocVerification");
}
}
void Heap::PostGcVerificationPaused(collector::GarbageCollector* gc) {
// Only pause if we have to do some verification.
Thread* const self = Thread::Current();
TimingLogger* const timings = GetCurrentGcIteration()->GetTimings();
TimingLogger::ScopedTiming t(__FUNCTION__, timings);
if (verify_system_weaks_) {
ReaderMutexLock mu2(self, *Locks::heap_bitmap_lock_);
collector::MarkSweep* mark_sweep = down_cast<collector::MarkSweep*>(gc);
mark_sweep->VerifySystemWeaks();
}
if (verify_post_gc_rosalloc_) {
RosAllocVerification(timings, "(Paused)PostGcRosAllocVerification");
}
if (verify_post_gc_heap_) {
TimingLogger::ScopedTiming t2("(Paused)PostGcVerifyHeapReferences", timings);
size_t failures = VerifyHeapReferences();
if (failures > 0) {
LOG(FATAL) << "Pre " << gc->GetName() << " heap verification failed with " << failures
<< " failures";
}
}
}
void Heap::PostGcVerification(collector::GarbageCollector* gc) {
if (verify_system_weaks_ || verify_post_gc_rosalloc_ || verify_post_gc_heap_) {
collector::GarbageCollector::ScopedPause pause(gc, false);
PostGcVerificationPaused(gc);
}
}
void Heap::RosAllocVerification(TimingLogger* timings, const char* name) {
TimingLogger::ScopedTiming t(name, timings);
for (const auto& space : continuous_spaces_) {
if (space->IsRosAllocSpace()) {
VLOG(heap) << name << " : " << space->GetName();
space->AsRosAllocSpace()->Verify();
}
}
}
collector::GcType Heap::WaitForGcToComplete(GcCause cause, Thread* self) {
ScopedThreadStateChange tsc(self, ThreadState::kWaitingForGcToComplete);
MutexLock mu(self, *gc_complete_lock_);
return WaitForGcToCompleteLocked(cause, self);
}
collector::GcType Heap::WaitForGcToCompleteLocked(GcCause cause, Thread* self) {
gc_complete_cond_->CheckSafeToWait(self);
collector::GcType last_gc_type = collector::kGcTypeNone;
GcCause last_gc_cause = kGcCauseNone;
uint64_t wait_start = NanoTime();
while (collector_type_running_ != kCollectorTypeNone) {
if (self != task_processor_->GetRunningThread()) {
// The current thread is about to wait for a currently running
// collection to finish. If the waiting thread is not the heap
// task daemon thread, the currently running collection is
// considered as a blocking GC.
running_collection_is_blocking_ = true;
VLOG(gc) << "Waiting for a blocking GC " << cause;
}
SCOPED_TRACE << "GC: Wait For Completion " << cause;
// We must wait, change thread state then sleep on gc_complete_cond_;
gc_complete_cond_->Wait(self);
last_gc_type = last_gc_type_;
last_gc_cause = last_gc_cause_;
}
uint64_t wait_time = NanoTime() - wait_start;
total_wait_time_ += wait_time;
if (wait_time > long_pause_log_threshold_) {
LOG(INFO) << "WaitForGcToComplete blocked " << cause << " on " << last_gc_cause << " for "
<< PrettyDuration(wait_time);
}
if (self != task_processor_->GetRunningThread()) {
// The current thread is about to run a collection. If the thread
// is not the heap task daemon thread, it's considered as a
// blocking GC (i.e., blocking itself).
running_collection_is_blocking_ = true;
// Don't log fake "GC" types that are only used for debugger or hidden APIs. If we log these,
// it results in log spam. kGcCauseExplicit is already logged in LogGC, so avoid it here too.
if (cause == kGcCauseForAlloc ||
cause == kGcCauseDisableMovingGc) {
VLOG(gc) << "Starting a blocking GC " << cause;
}
}
return last_gc_type;
}
void Heap::DumpForSigQuit(std::ostream& os) {
os << "Heap: " << GetPercentFree() << "% free, " << PrettySize(GetBytesAllocated()) << "/"
<< PrettySize(GetTotalMemory());
{
os << "Image spaces:\n";
ScopedObjectAccess soa(Thread::Current());
for (const auto& space : continuous_spaces_) {
if (space->IsImageSpace()) {
os << space->GetName() << "\n";
}
}
}
DumpGcPerformanceInfo(os);
}
size_t Heap::GetPercentFree() {
return static_cast<size_t>(100.0f * static_cast<float>(
GetFreeMemory()) / target_footprint_.load(std::memory_order_relaxed));
}
void Heap::SetIdealFootprint(size_t target_footprint) {
if (target_footprint > GetMaxMemory()) {
VLOG(gc) << "Clamp target GC heap from " << PrettySize(target_footprint) << " to "
<< PrettySize(GetMaxMemory());
target_footprint = GetMaxMemory();
}
target_footprint_.store(target_footprint, std::memory_order_relaxed);
}
bool Heap::IsMovableObject(ObjPtr<mirror::Object> obj) const {
if (kMovingCollector) {
space::Space* space = FindContinuousSpaceFromObject(obj.Ptr(), true);
if (space != nullptr) {
// TODO: Check large object?
return space->CanMoveObjects();
}
}
return false;
}
collector::GarbageCollector* Heap::FindCollectorByGcType(collector::GcType gc_type) {
for (auto* collector : garbage_collectors_) {
if (collector->GetCollectorType() == collector_type_ &&
collector->GetGcType() == gc_type) {
return collector;
}
}
return nullptr;
}
double Heap::HeapGrowthMultiplier() const {
// If we don't care about pause times we are background, so return 1.0.
if (!CareAboutPauseTimes()) {
return 1.0;
}
return foreground_heap_growth_multiplier_;
}
void Heap::GrowForUtilization(collector::GarbageCollector* collector_ran,
size_t bytes_allocated_before_gc) {
// We're running in the thread that set collector_type_running_ to something other than none,
// thus ensuring that there is only one of us running. Thus
// collector_type_running_ != kCollectorTypeNone, but that's a little tricky to turn into a
// DCHECK.
// We know what our utilization is at this moment.
// This doesn't actually resize any memory. It just lets the heap grow more when necessary.
const size_t bytes_allocated = GetBytesAllocated();
// Trace the new heap size after the GC is finished.
TraceHeapSize(bytes_allocated);
uint64_t target_size, grow_bytes;
collector::GcType gc_type = collector_ran->GetGcType();
MutexLock mu(Thread::Current(), process_state_update_lock_);
// Use the multiplier to grow more for foreground.
const double multiplier = HeapGrowthMultiplier();
if (gc_type != collector::kGcTypeSticky) {
// Grow the heap for non sticky GC.
uint64_t delta = bytes_allocated * (1.0 / GetTargetHeapUtilization() - 1.0);
DCHECK_LE(delta, std::numeric_limits<size_t>::max()) << "bytes_allocated=" << bytes_allocated
<< " target_utilization_=" << target_utilization_;
grow_bytes = std::min(delta, static_cast<uint64_t>(max_free_));
grow_bytes = std::max(grow_bytes, static_cast<uint64_t>(min_free_));
target_size = bytes_allocated + static_cast<uint64_t>(grow_bytes * multiplier);
next_gc_type_ = collector::kGcTypeSticky;
} else {
collector::GcType non_sticky_gc_type = NonStickyGcType();
// Find what the next non sticky collector will be.
collector::GarbageCollector* non_sticky_collector = FindCollectorByGcType(non_sticky_gc_type);
if (use_generational_cc_) {
if (non_sticky_collector == nullptr) {
non_sticky_collector = FindCollectorByGcType(collector::kGcTypePartial);
}
CHECK(non_sticky_collector != nullptr);
}
double sticky_gc_throughput_adjustment = GetStickyGcThroughputAdjustment(use_generational_cc_);
// If the throughput of the current sticky GC >= throughput of the non sticky collector, then
// do another sticky collection next.
// We also check that the bytes allocated aren't over the target_footprint, or
// concurrent_start_bytes in case of concurrent GCs, in order to prevent a
// pathological case where dead objects which aren't reclaimed by sticky could get accumulated
// if the sticky GC throughput always remained >= the full/partial throughput.
size_t target_footprint = target_footprint_.load(std::memory_order_relaxed);
if (current_gc_iteration_.GetEstimatedThroughput() * sticky_gc_throughput_adjustment >=
non_sticky_collector->GetEstimatedMeanThroughput() &&
non_sticky_collector->NumberOfIterations() > 0 &&
bytes_allocated <= (IsGcConcurrent() ? concurrent_start_bytes_ : target_footprint)) {
next_gc_type_ = collector::kGcTypeSticky;
} else {
next_gc_type_ = non_sticky_gc_type;
}
// If we have freed enough memory, shrink the heap back down.
const size_t adjusted_max_free = static_cast<size_t>(max_free_ * multiplier);
if (bytes_allocated + adjusted_max_free < target_footprint) {
target_size = bytes_allocated + adjusted_max_free;
grow_bytes = max_free_;
} else {
target_size = std::max(bytes_allocated, target_footprint);
// The same whether jank perceptible or not; just avoid the adjustment.
grow_bytes = 0;
}
}
CHECK_LE(target_size, std::numeric_limits<size_t>::max());
if (!ignore_target_footprint_) {
SetIdealFootprint(target_size);
// Store target size (computed with foreground heap growth multiplier) for updating
// target_footprint_ when process state switches to foreground.
// target_size = 0 ensures that target_footprint_ is not updated on
// process-state switch.
min_foreground_target_footprint_ =
(multiplier <= 1.0 && grow_bytes > 0)
? std::min(
bytes_allocated + static_cast<size_t>(grow_bytes * foreground_heap_growth_multiplier_),
GetMaxMemory())
: 0;
if (IsGcConcurrent()) {
const uint64_t freed_bytes = current_gc_iteration_.GetFreedBytes() +
current_gc_iteration_.GetFreedLargeObjectBytes() +
current_gc_iteration_.GetFreedRevokeBytes();
// Records the number of bytes allocated at the time of GC finish,excluding the number of
// bytes allocated during GC.
num_bytes_alive_after_gc_ = UnsignedDifference(bytes_allocated_before_gc, freed_bytes);
// Bytes allocated will shrink by freed_bytes after the GC runs, so if we want to figure out
// how many bytes were allocated during the GC we need to add freed_bytes back on.
// Almost always bytes_allocated + freed_bytes >= bytes_allocated_before_gc.
const size_t bytes_allocated_during_gc =
UnsignedDifference(bytes_allocated + freed_bytes, bytes_allocated_before_gc);
// Calculate when to perform the next ConcurrentGC.
// Estimate how many remaining bytes we will have when we need to start the next GC.
size_t remaining_bytes = bytes_allocated_during_gc;
remaining_bytes = std::min(remaining_bytes, kMaxConcurrentRemainingBytes);
remaining_bytes = std::max(remaining_bytes, kMinConcurrentRemainingBytes);
size_t target_footprint = target_footprint_.load(std::memory_order_relaxed);
if (UNLIKELY(remaining_bytes > target_footprint)) {
// A never going to happen situation that from the estimated allocation rate we will exceed
// the applications entire footprint with the given estimated allocation rate. Schedule
// another GC nearly straight away.
remaining_bytes = std::min(kMinConcurrentRemainingBytes, target_footprint);
}
DCHECK_LE(target_footprint_.load(std::memory_order_relaxed), GetMaxMemory());
// Start a concurrent GC when we get close to the estimated remaining bytes. When the
// allocation rate is very high, remaining_bytes could tell us that we should start a GC
// right away.
concurrent_start_bytes_ = std::max(target_footprint - remaining_bytes, bytes_allocated);
// Store concurrent_start_bytes_ (computed with foreground heap growth multiplier) for update
// itself when process state switches to foreground.
min_foreground_concurrent_start_bytes_ =
min_foreground_target_footprint_ != 0
? std::max(min_foreground_target_footprint_ - remaining_bytes, bytes_allocated)
: 0;
}
}
}
void Heap::ClampGrowthLimit() {
// Use heap bitmap lock to guard against races with BindLiveToMarkBitmap.
ScopedObjectAccess soa(Thread::Current());
WriterMutexLock mu(soa.Self(), *Locks::heap_bitmap_lock_);
capacity_ = growth_limit_;
for (const auto& space : continuous_spaces_) {
if (space->IsMallocSpace()) {
gc::space::MallocSpace* malloc_space = space->AsMallocSpace();
malloc_space->ClampGrowthLimit();
}
}
if (large_object_space_ != nullptr) {
large_object_space_->ClampGrowthLimit(capacity_);
}
if (collector_type_ == kCollectorTypeCC) {
DCHECK(region_space_ != nullptr);
// Twice the capacity as CC needs extra space for evacuating objects.
region_space_->ClampGrowthLimit(2 * capacity_);
} else if (collector_type_ == kCollectorTypeCMC) {
DCHECK(gUseUserfaultfd);
DCHECK_NE(mark_compact_, nullptr);
DCHECK_NE(bump_pointer_space_, nullptr);
mark_compact_->ClampGrowthLimit(capacity_);
}
// This space isn't added for performance reasons.
if (main_space_backup_.get() != nullptr) {
main_space_backup_->ClampGrowthLimit();
}
}
void Heap::ClearGrowthLimit() {
if (target_footprint_.load(std::memory_order_relaxed) == growth_limit_
&& growth_limit_ < capacity_) {
target_footprint_.store(capacity_, std::memory_order_relaxed);
SetDefaultConcurrentStartBytes();
}
growth_limit_ = capacity_;
ScopedObjectAccess soa(Thread::Current());
for (const auto& space : continuous_spaces_) {
if (space->IsMallocSpace()) {
gc::space::MallocSpace* malloc_space = space->AsMallocSpace();
malloc_space->ClearGrowthLimit();
malloc_space->SetFootprintLimit(malloc_space->Capacity());
}
}
// This space isn't added for performance reasons.
if (main_space_backup_.get() != nullptr) {
main_space_backup_->ClearGrowthLimit();
main_space_backup_->SetFootprintLimit(main_space_backup_->Capacity());
}
}
void Heap::AddFinalizerReference(Thread* self, ObjPtr<mirror::Object>* object) {
ScopedObjectAccess soa(self);
StackHandleScope<1u> hs(self);
// Use handle wrapper to update the `*object` if the object gets moved.
HandleWrapperObjPtr<mirror::Object> h_object = hs.NewHandleWrapper(object);
WellKnownClasses::java_lang_ref_FinalizerReference_add->InvokeStatic<'V', 'L'>(
self, h_object.Get());
}
void Heap::RequestConcurrentGCAndSaveObject(Thread* self,
bool force_full,
uint32_t observed_gc_num,
ObjPtr<mirror::Object>* obj) {
StackHandleScope<1> hs(self);
HandleWrapperObjPtr<mirror::Object> wrapper(hs.NewHandleWrapper(obj));
RequestConcurrentGC(self, kGcCauseBackground, force_full, observed_gc_num);
}
class Heap::ConcurrentGCTask : public HeapTask {
public:
ConcurrentGCTask(uint64_t target_time, GcCause cause, bool force_full, uint32_t gc_num)
: HeapTask(target_time), cause_(cause), force_full_(force_full), my_gc_num_(gc_num) {}
void Run(Thread* self) override {
Runtime* runtime = Runtime::Current();
gc::Heap* heap = runtime->GetHeap();
DCHECK(GCNumberLt(my_gc_num_, heap->GetCurrentGcNum() + 2)); // <= current_gc_num + 1
heap->ConcurrentGC(self, cause_, force_full_, my_gc_num_);
CHECK_IMPLIES(GCNumberLt(heap->GetCurrentGcNum(), my_gc_num_), runtime->IsShuttingDown(self));
}
private:
const GcCause cause_;
const bool force_full_; // If true, force full (or partial) collection.
const uint32_t my_gc_num_; // Sequence number of requested GC.
};
static bool CanAddHeapTask(Thread* self) REQUIRES(!Locks::runtime_shutdown_lock_) {
Runtime* runtime = Runtime::Current();
return runtime != nullptr && runtime->IsFinishedStarting() && !runtime->IsShuttingDown(self) &&
!self->IsHandlingStackOverflow();
}
bool Heap::RequestConcurrentGC(Thread* self,
GcCause cause,
bool force_full,
uint32_t observed_gc_num) {
uint32_t max_gc_requested = max_gc_requested_.load(std::memory_order_relaxed);
if (!GCNumberLt(observed_gc_num, max_gc_requested)) {
// observed_gc_num >= max_gc_requested: Nobody beat us to requesting the next gc.
if (CanAddHeapTask(self)) {
// Since observed_gc_num >= max_gc_requested, this increases max_gc_requested_, if successful.
if (max_gc_requested_.CompareAndSetStrongRelaxed(max_gc_requested, observed_gc_num + 1)) {
task_processor_->AddTask(self, new ConcurrentGCTask(NanoTime(), // Start straight away.
cause,
force_full,
observed_gc_num + 1));
}
DCHECK(GCNumberLt(observed_gc_num, max_gc_requested_.load(std::memory_order_relaxed)));
// If we increased max_gc_requested_, then we added a task that will eventually cause
// gcs_completed_ to be incremented (to at least observed_gc_num + 1).
// If the CAS failed, somebody else did.
return true;
}
return false;
}
return true; // Vacuously.
}
void Heap::ConcurrentGC(Thread* self, GcCause cause, bool force_full, uint32_t requested_gc_num) {
if (!Runtime::Current()->IsShuttingDown(self)) {
// Wait for any GCs currently running to finish. If this incremented GC number, we're done.
WaitForGcToComplete(cause, self);
if (GCNumberLt(GetCurrentGcNum(), requested_gc_num)) {
collector::GcType next_gc_type = next_gc_type_;
// If forcing full and next gc type is sticky, override with a non-sticky type.
if (force_full && next_gc_type == collector::kGcTypeSticky) {
next_gc_type = NonStickyGcType();
}
// If we can't run the GC type we wanted to run, find the next appropriate one and try
// that instead. E.g. can't do partial, so do full instead.
// We must ensure that we run something that ends up incrementing gcs_completed_.
// In the kGcTypePartial case, the initial CollectGarbageInternal call may not have that
// effect, but the subsequent KGcTypeFull call will.
if (CollectGarbageInternal(next_gc_type, cause, false, requested_gc_num)
== collector::kGcTypeNone) {
for (collector::GcType gc_type : gc_plan_) {
if (!GCNumberLt(GetCurrentGcNum(), requested_gc_num)) {
// Somebody did it for us.
break;
}
// Attempt to run the collector, if we succeed, we are done.
if (gc_type > next_gc_type &&
CollectGarbageInternal(gc_type, cause, false, requested_gc_num)
!= collector::kGcTypeNone) {
break;
}
}
}
}
}
}
class Heap::CollectorTransitionTask : public HeapTask {
public:
explicit CollectorTransitionTask(uint64_t target_time) : HeapTask(target_time) {}
void Run(Thread* self) override {
gc::Heap* heap = Runtime::Current()->GetHeap();
heap->DoPendingCollectorTransition();
heap->ClearPendingCollectorTransition(self);
}
};
void Heap::ClearPendingCollectorTransition(Thread* self) {
MutexLock mu(self, *pending_task_lock_);
pending_collector_transition_ = nullptr;
}
void Heap::RequestCollectorTransition(CollectorType desired_collector_type, uint64_t delta_time) {
Thread* self = Thread::Current();
desired_collector_type_ = desired_collector_type;
if (desired_collector_type_ == collector_type_ || !CanAddHeapTask(self)) {
return;
}
if (collector_type_ == kCollectorTypeCC) {
// For CC, we invoke a full compaction when going to the background, but the collector type
// doesn't change.
DCHECK_EQ(desired_collector_type_, kCollectorTypeCCBackground);
}
if (collector_type_ == kCollectorTypeCMC) {
// For CMC collector type doesn't change.
DCHECK_EQ(desired_collector_type_, kCollectorTypeCMCBackground);
}
DCHECK_NE(collector_type_, kCollectorTypeCCBackground);
DCHECK_NE(collector_type_, kCollectorTypeCMCBackground);
CollectorTransitionTask* added_task = nullptr;
const uint64_t target_time = NanoTime() + delta_time;
{
MutexLock mu(self, *pending_task_lock_);
// If we have an existing collector transition, update the target time to be the new target.
if (pending_collector_transition_ != nullptr) {
task_processor_->UpdateTargetRunTime(self, pending_collector_transition_, target_time);
return;
}
added_task = new CollectorTransitionTask(target_time);
pending_collector_transition_ = added_task;
}
task_processor_->AddTask(self, added_task);
}
class Heap::HeapTrimTask : public HeapTask {
public:
explicit HeapTrimTask(uint64_t delta_time) : HeapTask(NanoTime() + delta_time) { }
void Run(Thread* self) override {
gc::Heap* heap = Runtime::Current()->GetHeap();
heap->Trim(self);
heap->ClearPendingTrim(self);
}
};
void Heap::ClearPendingTrim(Thread* self) {
MutexLock mu(self, *pending_task_lock_);
pending_heap_trim_ = nullptr;
}
void Heap::RequestTrim(Thread* self) {
if (!CanAddHeapTask(self)) {
return;
}
// GC completed and now we must decide whether to request a heap trim (advising pages back to the
// kernel) or not. Issuing a request will also cause trimming of the libc heap. As a trim scans
// a space it will hold its lock and can become a cause of jank.
// Note, the large object space self trims and the Zygote space was trimmed and unchanging since
// forking.
// We don't have a good measure of how worthwhile a trim might be. We can't use the live bitmap
// because that only marks object heads, so a large array looks like lots of empty space. We
// don't just call dlmalloc all the time, because the cost of an _attempted_ trim is proportional
// to utilization (which is probably inversely proportional to how much benefit we can expect).
// We could try mincore(2) but that's only a measure of how many pages we haven't given away,
// not how much use we're making of those pages.
HeapTrimTask* added_task = nullptr;
{
MutexLock mu(self, *pending_task_lock_);
if (pending_heap_trim_ != nullptr) {
// Already have a heap trim request in task processor, ignore this request.
return;
}
added_task = new HeapTrimTask(kHeapTrimWait);
pending_heap_trim_ = added_task;
}
task_processor_->AddTask(self, added_task);
}
void Heap::IncrementNumberOfBytesFreedRevoke(size_t freed_bytes_revoke) {
size_t previous_num_bytes_freed_revoke =
num_bytes_freed_revoke_.fetch_add(freed_bytes_revoke, std::memory_order_relaxed);
// Check the updated value is less than the number of bytes allocated. There is a risk of
// execution being suspended between the increment above and the CHECK below, leading to
// the use of previous_num_bytes_freed_revoke in the comparison.
CHECK_GE(num_bytes_allocated_.load(std::memory_order_relaxed),
previous_num_bytes_freed_revoke + freed_bytes_revoke);
}
void Heap::RevokeThreadLocalBuffers(Thread* thread) {
if (rosalloc_space_ != nullptr) {
size_t freed_bytes_revoke = rosalloc_space_->RevokeThreadLocalBuffers(thread);
if (freed_bytes_revoke > 0U) {
IncrementNumberOfBytesFreedRevoke(freed_bytes_revoke);
}
}
if (bump_pointer_space_ != nullptr) {
CHECK_EQ(bump_pointer_space_->RevokeThreadLocalBuffers(thread), 0U);
}
if (region_space_ != nullptr) {
CHECK_EQ(region_space_->RevokeThreadLocalBuffers(thread), 0U);
}
}
void Heap::RevokeRosAllocThreadLocalBuffers(Thread* thread) {
if (rosalloc_space_ != nullptr) {
size_t freed_bytes_revoke = rosalloc_space_->RevokeThreadLocalBuffers(thread);
if (freed_bytes_revoke > 0U) {
IncrementNumberOfBytesFreedRevoke(freed_bytes_revoke);
}
}
}
void Heap::RevokeAllThreadLocalBuffers() {
if (rosalloc_space_ != nullptr) {
size_t freed_bytes_revoke = rosalloc_space_->RevokeAllThreadLocalBuffers();
if (freed_bytes_revoke > 0U) {
IncrementNumberOfBytesFreedRevoke(freed_bytes_revoke);
}
}
if (bump_pointer_space_ != nullptr) {
CHECK_EQ(bump_pointer_space_->RevokeAllThreadLocalBuffers(), 0U);
}
if (region_space_ != nullptr) {
CHECK_EQ(region_space_->RevokeAllThreadLocalBuffers(), 0U);
}
}
// For GC triggering purposes, we count old (pre-last-GC) and new native allocations as
// different fractions of Java allocations.
// For now, we essentially do not count old native allocations at all, so that we can preserve the
// existing behavior of not limiting native heap size. If we seriously considered it, we would
// have to adjust collection thresholds when we encounter large amounts of old native memory,
// and handle native out-of-memory situations.
static constexpr size_t kOldNativeDiscountFactor = 65536; // Approximately infinite for now.
static constexpr size_t kNewNativeDiscountFactor = 2;
// If weighted java + native memory use exceeds our target by kStopForNativeFactor, and
// newly allocated memory exceeds stop_for_native_allocs_, we wait for GC to complete to avoid
// running out of memory.
static constexpr float kStopForNativeFactor = 4.0;
// Return the ratio of the weighted native + java allocated bytes to its target value.
// A return value > 1.0 means we should collect. Significantly larger values mean we're falling
// behind.
inline float Heap::NativeMemoryOverTarget(size_t current_native_bytes, bool is_gc_concurrent) {
// Collection check for native allocation. Does not enforce Java heap bounds.
// With adj_start_bytes defined below, effectively checks
// <java bytes allocd> + c1*<old native allocd> + c2*<new native allocd) >= adj_start_bytes,
// where c3 > 1, and currently c1 and c2 are 1 divided by the values defined above.
size_t old_native_bytes = old_native_bytes_allocated_.load(std::memory_order_relaxed);
if (old_native_bytes > current_native_bytes) {
// Net decrease; skip the check, but update old value.
// It's OK to lose an update if two stores race.
old_native_bytes_allocated_.store(current_native_bytes, std::memory_order_relaxed);
return 0.0;
} else {
size_t new_native_bytes = UnsignedDifference(current_native_bytes, old_native_bytes);
size_t weighted_native_bytes = new_native_bytes / kNewNativeDiscountFactor
+ old_native_bytes / kOldNativeDiscountFactor;
size_t add_bytes_allowed = static_cast<size_t>(
NativeAllocationGcWatermark() * HeapGrowthMultiplier());
size_t java_gc_start_bytes = is_gc_concurrent
? concurrent_start_bytes_
: target_footprint_.load(std::memory_order_relaxed);
size_t adj_start_bytes = UnsignedSum(java_gc_start_bytes,
add_bytes_allowed / kNewNativeDiscountFactor);
return static_cast<float>(GetBytesAllocated() + weighted_native_bytes)
/ static_cast<float>(adj_start_bytes);
}
}
inline void Heap::CheckGCForNative(Thread* self) {
bool is_gc_concurrent = IsGcConcurrent();
uint32_t starting_gc_num = GetCurrentGcNum();
size_t current_native_bytes = GetNativeBytes();
float gc_urgency = NativeMemoryOverTarget(current_native_bytes, is_gc_concurrent);
if (UNLIKELY(gc_urgency >= 1.0)) {
if (is_gc_concurrent) {
bool requested =
RequestConcurrentGC(self, kGcCauseForNativeAlloc, /*force_full=*/true, starting_gc_num);
if (requested && gc_urgency > kStopForNativeFactor
&& current_native_bytes > stop_for_native_allocs_) {
// We're in danger of running out of memory due to rampant native allocation.
if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
LOG(INFO) << "Stopping for native allocation, urgency: " << gc_urgency;
}
// Count how many times we do this, so we can warn if this becomes excessive.
// Stop after a while, out of excessive caution.
static constexpr int kGcWaitIters = 20;
for (int i = 1; i <= kGcWaitIters; ++i) {
if (!GCNumberLt(GetCurrentGcNum(), max_gc_requested_.load(std::memory_order_relaxed))
|| WaitForGcToComplete(kGcCauseForNativeAlloc, self) != collector::kGcTypeNone) {
break;
}
CHECK(GCNumberLt(starting_gc_num, max_gc_requested_.load(std::memory_order_relaxed)));
if (i % 10 == 0) {
LOG(WARNING) << "Slept " << i << " times in native allocation, waiting for GC";
}
static constexpr int kGcWaitSleepMicros = 2000;
usleep(kGcWaitSleepMicros); // Encourage our requested GC to start.
}
}
} else {
CollectGarbageInternal(NonStickyGcType(), kGcCauseForNativeAlloc, false, starting_gc_num + 1);
}
}
}
// About kNotifyNativeInterval allocations have occurred. Check whether we should garbage collect.
void Heap::NotifyNativeAllocations(JNIEnv* env) {
native_objects_notified_.fetch_add(kNotifyNativeInterval, std::memory_order_relaxed);
CheckGCForNative(Thread::ForEnv(env));
}
// Register a native allocation with an explicit size.
// This should only be done for large allocations of non-malloc memory, which we wouldn't
// otherwise see.
void Heap::RegisterNativeAllocation(JNIEnv* env, size_t bytes) {
// Cautiously check for a wrapped negative bytes argument.
DCHECK(sizeof(size_t) < 8 || bytes < (std::numeric_limits<size_t>::max() / 2));
native_bytes_registered_.fetch_add(bytes, std::memory_order_relaxed);
uint32_t objects_notified =
native_objects_notified_.fetch_add(1, std::memory_order_relaxed);
if (objects_notified % kNotifyNativeInterval == kNotifyNativeInterval - 1
|| bytes > kCheckImmediatelyThreshold) {
CheckGCForNative(Thread::ForEnv(env));
}
// Heap profiler treats this as a Java allocation with a null object.
JHPCheckNonTlabSampleAllocation(Thread::Current(), nullptr, bytes);
}
void Heap::RegisterNativeFree(JNIEnv*, size_t bytes) {
size_t allocated;
size_t new_freed_bytes;
do {
allocated = native_bytes_registered_.load(std::memory_order_relaxed);
new_freed_bytes = std::min(allocated, bytes);
// We should not be registering more free than allocated bytes.
// But correctly keep going in non-debug builds.
DCHECK_EQ(new_freed_bytes, bytes);
} while (!native_bytes_registered_.CompareAndSetWeakRelaxed(allocated,
allocated - new_freed_bytes));
}
size_t Heap::GetTotalMemory() const {
return std::max(target_footprint_.load(std::memory_order_relaxed), GetBytesAllocated());
}
void Heap::AddModUnionTable(accounting::ModUnionTable* mod_union_table) {
DCHECK(mod_union_table != nullptr);
mod_union_tables_.Put(mod_union_table->GetSpace(), mod_union_table);
}
void Heap::CheckPreconditionsForAllocObject(ObjPtr<mirror::Class> c, size_t byte_count) {
// Compare rounded sizes since the allocation may have been retried after rounding the size.
// See b/37885600
CHECK(c == nullptr || (c->IsClassClass() && byte_count >= sizeof(mirror::Class)) ||
(c->IsVariableSize() ||
RoundUp(c->GetObjectSize(), kObjectAlignment) ==
RoundUp(byte_count, kObjectAlignment)))
<< "ClassFlags=" << c->GetClassFlags()
<< " IsClassClass=" << c->IsClassClass()
<< " byte_count=" << byte_count
<< " IsVariableSize=" << c->IsVariableSize()
<< " ObjectSize=" << c->GetObjectSize()
<< " sizeof(Class)=" << sizeof(mirror::Class)
<< " " << verification_->DumpObjectInfo(c.Ptr(), /*tag=*/ "klass");
CHECK_GE(byte_count, sizeof(mirror::Object));
}
void Heap::AddRememberedSet(accounting::RememberedSet* remembered_set) {
CHECK(remembered_set != nullptr);
space::Space* space = remembered_set->GetSpace();
CHECK(space != nullptr);
CHECK(remembered_sets_.find(space) == remembered_sets_.end()) << space;
remembered_sets_.Put(space, remembered_set);
CHECK(remembered_sets_.find(space) != remembered_sets_.end()) << space;
}
void Heap::RemoveRememberedSet(space::Space* space) {
CHECK(space != nullptr);
auto it = remembered_sets_.find(space);
CHECK(it != remembered_sets_.end());
delete it->second;
remembered_sets_.erase(it);
CHECK(remembered_sets_.find(space) == remembered_sets_.end());
}
void Heap::ClearMarkedObjects(bool release_eagerly) {
// Clear all of the spaces' mark bitmaps.
for (const auto& space : GetContinuousSpaces()) {
if (space->GetLiveBitmap() != nullptr && !space->HasBoundBitmaps()) {
space->GetMarkBitmap()->Clear(release_eagerly);
}
}
// Clear the marked objects in the discontinous space object sets.
for (const auto& space : GetDiscontinuousSpaces()) {
space->GetMarkBitmap()->Clear(release_eagerly);
}
}
void Heap::SetAllocationRecords(AllocRecordObjectMap* records) {
allocation_records_.reset(records);
}
void Heap::VisitAllocationRecords(RootVisitor* visitor) const {
if (IsAllocTrackingEnabled()) {
MutexLock mu(Thread::Current(), *Locks::alloc_tracker_lock_);
if (IsAllocTrackingEnabled()) {
GetAllocationRecords()->VisitRoots(visitor);
}
}
}
void Heap::SweepAllocationRecords(IsMarkedVisitor* visitor) const {
if (IsAllocTrackingEnabled()) {
MutexLock mu(Thread::Current(), *Locks::alloc_tracker_lock_);
if (IsAllocTrackingEnabled()) {
GetAllocationRecords()->SweepAllocationRecords(visitor);
}
}
}
void Heap::AllowNewAllocationRecords() const {
CHECK(!gUseReadBarrier);
MutexLock mu(Thread::Current(), *Locks::alloc_tracker_lock_);
AllocRecordObjectMap* allocation_records = GetAllocationRecords();
if (allocation_records != nullptr) {
allocation_records->AllowNewAllocationRecords();
}
}
void Heap::DisallowNewAllocationRecords() const {
CHECK(!gUseReadBarrier);
MutexLock mu(Thread::Current(), *Locks::alloc_tracker_lock_);
AllocRecordObjectMap* allocation_records = GetAllocationRecords();
if (allocation_records != nullptr) {
allocation_records->DisallowNewAllocationRecords();
}
}
void Heap::BroadcastForNewAllocationRecords() const {
// Always broadcast without checking IsAllocTrackingEnabled() because IsAllocTrackingEnabled() may
// be set to false while some threads are waiting for system weak access in
// AllocRecordObjectMap::RecordAllocation() and we may fail to wake them up. b/27467554.
MutexLock mu(Thread::Current(), *Locks::alloc_tracker_lock_);
AllocRecordObjectMap* allocation_records = GetAllocationRecords();
if (allocation_records != nullptr) {
allocation_records->BroadcastForNewAllocationRecords();
}
}
// Perfetto Java Heap Profiler Support.
// Perfetto initialization.
void Heap::InitPerfettoJavaHeapProf() {
// Initialize Perfetto Heap info and Heap id.
uint32_t heap_id = 1; // Initialize to 1, to be overwritten by Perfetto heap id.
#ifdef ART_TARGET_ANDROID
// Register the heap and create the heapid.
// Use a Perfetto heap name = "com.android.art" for the Java Heap Profiler.
AHeapInfo* info = AHeapInfo_create("com.android.art");
// Set the Enable Callback, there is no callback data ("nullptr").
AHeapInfo_setEnabledCallback(info, &EnableHeapSamplerCallback, &heap_sampler_);
// Set the Disable Callback.
AHeapInfo_setDisabledCallback(info, &DisableHeapSamplerCallback, &heap_sampler_);
heap_id = AHeapProfile_registerHeap(info);
// Do not enable the Java Heap Profiler in this case, wait for Perfetto to enable it through
// the callback function.
#else
// This is the host case, enable the Java Heap Profiler for host testing.
// Perfetto API is currently not available on host.
heap_sampler_.EnableHeapSampler();
#endif
heap_sampler_.SetHeapID(heap_id);
VLOG(heap) << "Java Heap Profiler Initialized";
}
// Check if the Java Heap Profiler is enabled and initialized.
int Heap::CheckPerfettoJHPEnabled() {
return GetHeapSampler().IsEnabled();
}
void Heap::JHPCheckNonTlabSampleAllocation(Thread* self, mirror::Object* obj, size_t alloc_size) {
bool take_sample = false;
size_t bytes_until_sample = 0;
HeapSampler& prof_heap_sampler = GetHeapSampler();
if (prof_heap_sampler.IsEnabled()) {
// An allocation occurred, sample it, even if non-Tlab.
// In case take_sample is already set from the previous GetSampleOffset
// because we tried the Tlab allocation first, we will not use this value.
// A new value is generated below. Also bytes_until_sample will be updated.
// Note that we are not using the return value from the GetSampleOffset in
// the NonTlab case here.
prof_heap_sampler.GetSampleOffset(alloc_size,
self->GetTlabPosOffset(),
&take_sample,
&bytes_until_sample);
prof_heap_sampler.SetBytesUntilSample(bytes_until_sample);
if (take_sample) {
prof_heap_sampler.ReportSample(obj, alloc_size);
}
VLOG(heap) << "JHP:NonTlab Non-moving or Large Allocation or RegisterNativeAllocation";
}
}
size_t Heap::JHPCalculateNextTlabSize(Thread* self,
size_t jhp_def_tlab_size,
size_t alloc_size,
bool* take_sample,
size_t* bytes_until_sample) {
size_t next_tlab_size = jhp_def_tlab_size;
if (CheckPerfettoJHPEnabled()) {
size_t next_sample_point =
GetHeapSampler().GetSampleOffset(alloc_size,
self->GetTlabPosOffset(),
take_sample,
bytes_until_sample);
next_tlab_size = std::min(next_sample_point, jhp_def_tlab_size);
}
return next_tlab_size;
}
void Heap::AdjustSampleOffset(size_t adjustment) {
GetHeapSampler().AdjustSampleOffset(adjustment);
}
void Heap::CheckGcStressMode(Thread* self, ObjPtr<mirror::Object>* obj) {
DCHECK(gc_stress_mode_);
auto* const runtime = Runtime::Current();
if (runtime->GetClassLinker()->IsInitialized() && !runtime->IsActiveTransaction()) {
// Check if we should GC.
bool new_backtrace = false;
{
static constexpr size_t kMaxFrames = 16u;
MutexLock mu(self, *backtrace_lock_);
FixedSizeBacktrace<kMaxFrames> backtrace;
backtrace.Collect(/* skip_count= */ 2);
uint64_t hash = backtrace.Hash();
new_backtrace = seen_backtraces_.find(hash) == seen_backtraces_.end();
if (new_backtrace) {
seen_backtraces_.insert(hash);
}
}
if (new_backtrace) {
StackHandleScope<1> hs(self);
auto h = hs.NewHandleWrapper(obj);
CollectGarbage(/* clear_soft_references= */ false);
unique_backtrace_count_.fetch_add(1);
} else {
seen_backtrace_count_.fetch_add(1);
}
}
}
void Heap::DisableGCForShutdown() {
MutexLock mu(Thread::Current(), *gc_complete_lock_);
gc_disabled_for_shutdown_ = true;
}
bool Heap::IsGCDisabledForShutdown() const {
MutexLock mu(Thread::Current(), *gc_complete_lock_);
return gc_disabled_for_shutdown_;
}
bool Heap::ObjectIsInBootImageSpace(ObjPtr<mirror::Object> obj) const {
DCHECK_EQ(IsBootImageAddress(obj.Ptr()),
any_of(boot_image_spaces_.begin(),
boot_image_spaces_.end(),
[obj](gc::space::ImageSpace* space) REQUIRES_SHARED(Locks::mutator_lock_) {
return space->HasAddress(obj.Ptr());
}));
return IsBootImageAddress(obj.Ptr());
}
bool Heap::IsInBootImageOatFile(const void* p) const {
DCHECK_EQ(IsBootImageAddress(p),
any_of(boot_image_spaces_.begin(),
boot_image_spaces_.end(),
[p](gc::space::ImageSpace* space) REQUIRES_SHARED(Locks::mutator_lock_) {
return space->GetOatFile()->Contains(p);
}));
return IsBootImageAddress(p);
}
void Heap::SetAllocationListener(AllocationListener* l) {
AllocationListener* old = GetAndOverwriteAllocationListener(&alloc_listener_, l);
if (old == nullptr) {
Runtime::Current()->GetInstrumentation()->InstrumentQuickAllocEntryPoints();
}
}
void Heap::RemoveAllocationListener() {
AllocationListener* old = GetAndOverwriteAllocationListener(&alloc_listener_, nullptr);
if (old != nullptr) {
Runtime::Current()->GetInstrumentation()->UninstrumentQuickAllocEntryPoints();
}
}
void Heap::SetGcPauseListener(GcPauseListener* l) {
gc_pause_listener_.store(l, std::memory_order_relaxed);
}
void Heap::RemoveGcPauseListener() {
gc_pause_listener_.store(nullptr, std::memory_order_relaxed);
}
mirror::Object* Heap::AllocWithNewTLAB(Thread* self,
AllocatorType allocator_type,
size_t alloc_size,
bool grow,
size_t* bytes_allocated,
size_t* usable_size,
size_t* bytes_tl_bulk_allocated) {
mirror::Object* ret = nullptr;
bool take_sample = false;
size_t bytes_until_sample = 0;
if (kUsePartialTlabs && alloc_size <= self->TlabRemainingCapacity()) {
DCHECK_GT(alloc_size, self->TlabSize());
// There is enough space if we grow the TLAB. Lets do that. This increases the
// TLAB bytes.
const size_t min_expand_size = alloc_size - self->TlabSize();
size_t next_tlab_size = JHPCalculateNextTlabSize(self,
kPartialTlabSize,
alloc_size,
&take_sample,
&bytes_until_sample);
const size_t expand_bytes = std::max(
min_expand_size,
std::min(self->TlabRemainingCapacity() - self->TlabSize(), next_tlab_size));
if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type, expand_bytes, grow))) {
return nullptr;
}
*bytes_tl_bulk_allocated = expand_bytes;
self->ExpandTlab(expand_bytes);
DCHECK_LE(alloc_size, self->TlabSize());
} else if (allocator_type == kAllocatorTypeTLAB) {
DCHECK(bump_pointer_space_ != nullptr);
// Try to allocate a page-aligned TLAB (not necessary though).
// TODO: for large allocations, which are rare, maybe we should allocate
// that object and return. There is no need to revoke the current TLAB,
// particularly if it's mostly unutilized.
size_t def_pr_tlab_size = RoundDown(alloc_size + kDefaultTLABSize, kPageSize) - alloc_size;
size_t next_tlab_size = JHPCalculateNextTlabSize(self,
def_pr_tlab_size,
alloc_size,
&take_sample,
&bytes_until_sample);
const size_t new_tlab_size = alloc_size + next_tlab_size;
if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type, new_tlab_size, grow))) {
return nullptr;
}
// Try allocating a new thread local buffer, if the allocation fails the space must be
// full so return null.
if (!bump_pointer_space_->AllocNewTlab(self, new_tlab_size)) {
return nullptr;
}
*bytes_tl_bulk_allocated = new_tlab_size;
if (CheckPerfettoJHPEnabled()) {
VLOG(heap) << "JHP:kAllocatorTypeTLAB, New Tlab bytes allocated= " << new_tlab_size;
}
} else {
DCHECK(allocator_type == kAllocatorTypeRegionTLAB);
DCHECK(region_space_ != nullptr);
if (space::RegionSpace::kRegionSize >= alloc_size) {
// Non-large. Check OOME for a tlab.
if (LIKELY(!IsOutOfMemoryOnAllocation(allocator_type,
space::RegionSpace::kRegionSize,
grow))) {
size_t def_pr_tlab_size = kUsePartialTlabs
? kPartialTlabSize
: gc::space::RegionSpace::kRegionSize;
size_t next_pr_tlab_size = JHPCalculateNextTlabSize(self,
def_pr_tlab_size,
alloc_size,
&take_sample,
&bytes_until_sample);
const size_t new_tlab_size = kUsePartialTlabs
? std::max(alloc_size, next_pr_tlab_size)
: next_pr_tlab_size;
// Try to allocate a tlab.
if (!region_space_->AllocNewTlab(self, new_tlab_size, bytes_tl_bulk_allocated)) {
// Failed to allocate a tlab. Try non-tlab.
ret = region_space_->AllocNonvirtual<false>(alloc_size,
bytes_allocated,
usable_size,
bytes_tl_bulk_allocated);
JHPCheckNonTlabSampleAllocation(self, ret, alloc_size);
return ret;
}
// Fall-through to using the TLAB below.
} else {
// Check OOME for a non-tlab allocation.
if (!IsOutOfMemoryOnAllocation(allocator_type, alloc_size, grow)) {
ret = region_space_->AllocNonvirtual<false>(alloc_size,
bytes_allocated,
usable_size,
bytes_tl_bulk_allocated);
JHPCheckNonTlabSampleAllocation(self, ret, alloc_size);
return ret;
}
// Neither tlab or non-tlab works. Give up.
return nullptr;
}
} else {
// Large. Check OOME.
if (LIKELY(!IsOutOfMemoryOnAllocation(allocator_type, alloc_size, grow))) {
ret = region_space_->AllocNonvirtual<false>(alloc_size,
bytes_allocated,
usable_size,
bytes_tl_bulk_allocated);
JHPCheckNonTlabSampleAllocation(self, ret, alloc_size);
return ret;
}
return nullptr;
}
}
// Refilled TLAB, return.
ret = self->AllocTlab(alloc_size);
DCHECK(ret != nullptr);
*bytes_allocated = alloc_size;
*usable_size = alloc_size;
// JavaHeapProfiler: Send the thread information about this allocation in case a sample is
// requested.
// This is the fallthrough from both the if and else if above cases => Cases that use TLAB.
if (CheckPerfettoJHPEnabled()) {
if (take_sample) {
GetHeapSampler().ReportSample(ret, alloc_size);
// Update the bytes_until_sample now that the allocation is already done.
GetHeapSampler().SetBytesUntilSample(bytes_until_sample);
}
VLOG(heap) << "JHP:Fallthrough Tlab allocation";
}
return ret;
}
const Verification* Heap::GetVerification() const {
return verification_.get();
}
void Heap::VlogHeapGrowth(size_t old_footprint, size_t new_footprint, size_t alloc_size) {
VLOG(heap) << "Growing heap from " << PrettySize(old_footprint) << " to "
<< PrettySize(new_footprint) << " for a " << PrettySize(alloc_size) << " allocation";
}
// Run a gc if we haven't run one since initial_gc_num. This forces processes to
// reclaim memory allocated during startup, even if they don't do much
// allocation post startup. If the process is actively allocating and triggering
// GCs, or has moved to the background and hence forced a GC, this does nothing.
class Heap::TriggerPostForkCCGcTask : public HeapTask {
public:
explicit TriggerPostForkCCGcTask(uint64_t target_time, uint32_t initial_gc_num) :
HeapTask(target_time), initial_gc_num_(initial_gc_num) {}
void Run(Thread* self) override {
gc::Heap* heap = Runtime::Current()->GetHeap();
if (heap->GetCurrentGcNum() == initial_gc_num_) {
if (kLogAllGCs) {
LOG(INFO) << "Forcing GC for allocation-inactive process";
}
heap->RequestConcurrentGC(self, kGcCauseBackground, false, initial_gc_num_);
}
}
private:
uint32_t initial_gc_num_;
};
// Reduce target footprint, if no GC has occurred since initial_gc_num.
// If a GC already occurred, it will have done this for us.
class Heap::ReduceTargetFootprintTask : public HeapTask {
public:
explicit ReduceTargetFootprintTask(uint64_t target_time, size_t new_target_sz,
uint32_t initial_gc_num) :
HeapTask(target_time), new_target_sz_(new_target_sz), initial_gc_num_(initial_gc_num) {}
void Run(Thread* self) override {
gc::Heap* heap = Runtime::Current()->GetHeap();
MutexLock mu(self, *(heap->gc_complete_lock_));
if (heap->GetCurrentGcNum() == initial_gc_num_
&& heap->collector_type_running_ == kCollectorTypeNone) {
size_t target_footprint = heap->target_footprint_.load(std::memory_order_relaxed);
if (target_footprint > new_target_sz_) {
if (heap->target_footprint_.CompareAndSetStrongRelaxed(target_footprint, new_target_sz_)) {
heap->SetDefaultConcurrentStartBytesLocked();
}
}
}
}
private:
size_t new_target_sz_;
uint32_t initial_gc_num_;
};
// Return a pseudo-random integer between 0 and 19999, using the uid as a seed. We want this to
// be deterministic for a given process, but to vary randomly across processes. Empirically, the
// uids for processes for which this matters are distinct.
static uint32_t GetPseudoRandomFromUid() {
std::default_random_engine rng(getuid());
std::uniform_int_distribution<int> dist(0, 19999);
return dist(rng);
}
void Heap::PostForkChildAction(Thread* self) {
uint32_t starting_gc_num = GetCurrentGcNum();
uint64_t last_adj_time = NanoTime();
next_gc_type_ = NonStickyGcType(); // Always start with a full gc.
LOG(INFO) << "Using " << foreground_collector_type_ << " GC.";
if (gUseUserfaultfd) {
DCHECK_NE(mark_compact_, nullptr);
mark_compact_->CreateUserfaultfd(/*post_fork*/true);
}
// Temporarily increase target_footprint_ and concurrent_start_bytes_ to
// max values to avoid GC during app launch.
// Set target_footprint_ to the largest allowed value.
SetIdealFootprint(growth_limit_);
SetDefaultConcurrentStartBytes();
// Shrink heap after kPostForkMaxHeapDurationMS, to force a memory hog process to GC.
// This remains high enough that many processes will continue without a GC.
if (initial_heap_size_ < growth_limit_) {
size_t first_shrink_size = std::max(growth_limit_ / 4, initial_heap_size_);
last_adj_time += MsToNs(kPostForkMaxHeapDurationMS);
GetTaskProcessor()->AddTask(
self, new ReduceTargetFootprintTask(last_adj_time, first_shrink_size, starting_gc_num));
// Shrink to a small value after a substantial time period. This will typically force a
// GC if none has occurred yet. Has no effect if there was a GC before this anyway, which
// is commonly the case, e.g. because of a process transition.
if (initial_heap_size_ < first_shrink_size) {
last_adj_time += MsToNs(4 * kPostForkMaxHeapDurationMS);
GetTaskProcessor()->AddTask(
self,
new ReduceTargetFootprintTask(last_adj_time, initial_heap_size_, starting_gc_num));
}
}
// Schedule a GC after a substantial period of time. This will become a no-op if another GC is
// scheduled in the interim. If not, we want to avoid holding onto start-up garbage.
uint64_t post_fork_gc_time = last_adj_time
+ MsToNs(4 * kPostForkMaxHeapDurationMS + GetPseudoRandomFromUid());
GetTaskProcessor()->AddTask(self,
new TriggerPostForkCCGcTask(post_fork_gc_time, starting_gc_num));
}
void Heap::VisitReflectiveTargets(ReflectiveValueVisitor *visit) {
VisitObjectsPaused([&visit](mirror::Object* ref) NO_THREAD_SAFETY_ANALYSIS {
art::ObjPtr<mirror::Class> klass(ref->GetClass());
// All these classes are in the BootstrapClassLoader.
if (!klass->IsBootStrapClassLoaded()) {
return;
}
if (GetClassRoot<mirror::Method>()->IsAssignableFrom(klass) ||
GetClassRoot<mirror::Constructor>()->IsAssignableFrom(klass)) {
down_cast<mirror::Executable*>(ref)->VisitTarget(visit);
} else if (art::GetClassRoot<art::mirror::Field>() == klass) {
down_cast<mirror::Field*>(ref)->VisitTarget(visit);
} else if (art::GetClassRoot<art::mirror::MethodHandle>()->IsAssignableFrom(klass)) {
down_cast<mirror::MethodHandle*>(ref)->VisitTarget(visit);
} else if (art::GetClassRoot<art::mirror::StaticFieldVarHandle>()->IsAssignableFrom(klass)) {
down_cast<mirror::StaticFieldVarHandle*>(ref)->VisitTarget(visit);
} else if (art::GetClassRoot<art::mirror::FieldVarHandle>()->IsAssignableFrom(klass)) {
down_cast<mirror::FieldVarHandle*>(ref)->VisitTarget(visit);
} else if (art::GetClassRoot<art::mirror::DexCache>()->IsAssignableFrom(klass)) {
down_cast<mirror::DexCache*>(ref)->VisitReflectiveTargets(visit);
}
});
}
bool Heap::AddHeapTask(gc::HeapTask* task) {
Thread* const self = Thread::Current();
if (!CanAddHeapTask(self)) {
return false;
}
GetTaskProcessor()->AddTask(self, task);
return true;
}
} // namespace gc
} // namespace art