diff options
Diffstat (limited to 'runtime')
164 files changed, 1395 insertions, 631 deletions
diff --git a/runtime/Android.bp b/runtime/Android.bp index aa7dc65871..7f27e33e6a 100644 --- a/runtime/Android.bp +++ b/runtime/Android.bp @@ -123,6 +123,7 @@ cc_defaults { "jni_internal.cc", "jobject_comparator.cc", "linear_alloc.cc", + "managed_stack.cc", "mem_map.cc", "memory_region.cc", "method_handles.cc", diff --git a/runtime/arch/arm/context_arm.cc b/runtime/arch/arm/context_arm.cc index 9cbec1e5bc..817dcf5783 100644 --- a/runtime/arch/arm/context_arm.cc +++ b/runtime/arch/arm/context_arm.cc @@ -18,7 +18,7 @@ #include "base/bit_utils.h" #include "quick/quick_method_frame_info.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace arm { diff --git a/runtime/arch/arm/fault_handler_arm.cc b/runtime/arch/arm/fault_handler_arm.cc index 4c15450ff7..b4bca014f4 100644 --- a/runtime/arch/arm/fault_handler_arm.cc +++ b/runtime/arch/arm/fault_handler_arm.cc @@ -25,7 +25,7 @@ #include "base/logging.h" #include "base/macros.h" #include "globals.h" -#include "thread-inl.h" +#include "thread-current-inl.h" // // ARM specific fault handler functions. diff --git a/runtime/arch/arm64/context_arm64.cc b/runtime/arch/arm64/context_arm64.cc index d5d1ec7f07..a8f034eaf4 100644 --- a/runtime/arch/arm64/context_arm64.cc +++ b/runtime/arch/arm64/context_arm64.cc @@ -20,7 +20,7 @@ #include "base/bit_utils.h" #include "quick/quick_method_frame_info.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace arm64 { diff --git a/runtime/arch/arm64/fault_handler_arm64.cc b/runtime/arch/arm64/fault_handler_arm64.cc index dc4e8f389e..0ead732cdd 100644 --- a/runtime/arch/arm64/fault_handler_arm64.cc +++ b/runtime/arch/arm64/fault_handler_arm64.cc @@ -26,7 +26,7 @@ #include "base/macros.h" #include "globals.h" #include "registers_arm64.h" -#include "thread-inl.h" +#include "thread-current-inl.h" extern "C" void art_quick_throw_stack_overflow(); extern "C" void art_quick_throw_null_pointer_exception_from_signal(); diff --git a/runtime/arch/mips/fault_handler_mips.cc b/runtime/arch/mips/fault_handler_mips.cc index 7072a8a613..25e442c3e6 100644 --- a/runtime/arch/mips/fault_handler_mips.cc +++ b/runtime/arch/mips/fault_handler_mips.cc @@ -24,7 +24,7 @@ #include "globals.h" #include "quick_method_frame_info_mips.h" #include "registers_mips.h" -#include "thread-inl.h" +#include "thread-current-inl.h" extern "C" void art_quick_throw_stack_overflow(); extern "C" void art_quick_throw_null_pointer_exception_from_signal(); diff --git a/runtime/arch/mips64/fault_handler_mips64.cc b/runtime/arch/mips64/fault_handler_mips64.cc index f9a92c834e..69d73b09c2 100644 --- a/runtime/arch/mips64/fault_handler_mips64.cc +++ b/runtime/arch/mips64/fault_handler_mips64.cc @@ -25,7 +25,7 @@ #include "globals.h" #include "quick_method_frame_info_mips64.h" #include "registers_mips64.h" -#include "thread-inl.h" +#include "thread-current-inl.h" extern "C" void art_quick_throw_stack_overflow(); extern "C" void art_quick_throw_null_pointer_exception_from_signal(); diff --git a/runtime/arch/x86/fault_handler_x86.cc b/runtime/arch/x86/fault_handler_x86.cc index 7d8abb8cc5..798c500f18 100644 --- a/runtime/arch/x86/fault_handler_x86.cc +++ b/runtime/arch/x86/fault_handler_x86.cc @@ -26,7 +26,7 @@ #include "base/macros.h" #include "base/safe_copy.h" #include "globals.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #if defined(__APPLE__) #define ucontext __darwin_ucontext diff --git a/runtime/arch/x86/thread_x86.cc b/runtime/arch/x86/thread_x86.cc index 241650eaf4..cc8f1fa00e 100644 --- a/runtime/arch/x86/thread_x86.cc +++ b/runtime/arch/x86/thread_x86.cc @@ -22,7 +22,7 @@ #include "asm_support_x86.h" #include "base/enums.h" #include "base/macros.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #if defined(__APPLE__) diff --git a/runtime/arch/x86_64/thread_x86_64.cc b/runtime/arch/x86_64/thread_x86_64.cc index 553b6569c8..19d25f6990 100644 --- a/runtime/arch/x86_64/thread_x86_64.cc +++ b/runtime/arch/x86_64/thread_x86_64.cc @@ -18,7 +18,7 @@ #include "asm_support_x86_64.h" #include "base/macros.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #if defined(__linux__) diff --git a/runtime/art_field-inl.h b/runtime/art_field-inl.h index 0de0f02f9b..a8a58e135e 100644 --- a/runtime/art_field-inl.h +++ b/runtime/art_field-inl.h @@ -28,7 +28,7 @@ #include "mirror/dex_cache-inl.h" #include "mirror/object-inl.h" #include "primitive.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "scoped_thread_state_change-inl.h" #include "well_known_classes.h" @@ -352,11 +352,6 @@ inline ObjPtr<mirror::String> ArtField::GetStringName(Thread* self, bool resolve return name; } -template<typename RootVisitorType> -inline void ArtField::VisitRoots(RootVisitorType& visitor) { - visitor.VisitRoot(declaring_class_.AddressWithoutBarrier()); -} - template <typename Visitor> inline void ArtField::UpdateObjects(const Visitor& visitor) { ObjPtr<mirror::Class> old_class = DeclaringClassRoot().Read<kWithoutReadBarrier>(); diff --git a/runtime/art_field.h b/runtime/art_field.h index 3789b0ce2f..5114578933 100644 --- a/runtime/art_field.h +++ b/runtime/art_field.h @@ -171,7 +171,9 @@ class ArtField FINAL { // NO_THREAD_SAFETY_ANALYSIS since we don't know what the callback requires. template<typename RootVisitorType> - void VisitRoots(RootVisitorType& visitor) NO_THREAD_SAFETY_ANALYSIS; + ALWAYS_INLINE inline void VisitRoots(RootVisitorType& visitor) NO_THREAD_SAFETY_ANALYSIS { + visitor.VisitRoot(declaring_class_.AddressWithoutBarrier()); + } bool IsVolatile() REQUIRES_SHARED(Locks::mutator_lock_) { return (GetAccessFlags() & kAccVolatile) != 0; diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h index 59cd978a66..8567c004fa 100644 --- a/runtime/art_method-inl.h +++ b/runtime/art_method-inl.h @@ -36,10 +36,9 @@ #include "oat.h" #include "obj_ptr-inl.h" #include "quick/quick_method_frame_info.h" -#include "read_barrier-inl.h" #include "runtime-inl.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "utils.h" namespace art { diff --git a/runtime/asm_support.h b/runtime/asm_support.h index 6d271ed380..1ce7fd33bb 100644 --- a/runtime/asm_support.h +++ b/runtime/asm_support.h @@ -20,6 +20,7 @@ #if defined(__cplusplus) #include "art_method.h" #include "base/bit_utils.h" +#include "gc/accounting/card_table.h" #include "gc/allocator/rosalloc.h" #include "gc/heap.h" #include "jit/jit.h" @@ -29,6 +30,7 @@ #include "mirror/string.h" #include "utils/dex_cache_arrays_layout.h" #include "runtime.h" +#include "stack.h" #include "thread.h" #endif diff --git a/runtime/atomic.cc b/runtime/atomic.cc index d5ae570c30..07aceb7cfc 100644 --- a/runtime/atomic.cc +++ b/runtime/atomic.cc @@ -17,7 +17,7 @@ #include "atomic.h" #include "base/mutex.h" #include "base/stl_util.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/atomic.h b/runtime/atomic.h index 45c3165b18..25dd1a3a5e 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -257,6 +257,13 @@ class PACKED(sizeof(T)) Atomic : public std::atomic<T> { return this->compare_exchange_strong(expected_value, desired_value, std::memory_order_relaxed); } + // Atomically replace the value with desired value if it matches the expected value. Prior writes + // to other memory locations become visible to the threads that do a consume or an acquire on the + // same location. + bool CompareExchangeStrongRelease(T expected_value, T desired_value) { + return this->compare_exchange_strong(expected_value, desired_value, std::memory_order_release); + } + // The same, except it may fail spuriously. bool CompareExchangeWeakRelaxed(T expected_value, T desired_value) { return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_relaxed); diff --git a/runtime/barrier_test.cc b/runtime/barrier_test.cc index f68a5d42e4..25b6925fd8 100644 --- a/runtime/barrier_test.cc +++ b/runtime/barrier_test.cc @@ -22,7 +22,7 @@ #include "common_runtime_test.h" #include "mirror/object_array-inl.h" #include "thread_pool.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { class CheckWaitTask : public Task { diff --git a/runtime/base/arena_allocator.cc b/runtime/base/arena_allocator.cc index f672882254..54b40f28cf 100644 --- a/runtime/base/arena_allocator.cc +++ b/runtime/base/arena_allocator.cc @@ -14,16 +14,19 @@ * limitations under the License. */ +#include "arena_allocator-inl.h" + +#include <sys/mman.h> + #include <algorithm> #include <cstddef> #include <iomanip> #include <numeric> -#include "arena_allocator-inl.h" #include "logging.h" #include "mem_map.h" #include "mutex.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "systrace.h" namespace art { diff --git a/runtime/base/dumpable-inl.h b/runtime/base/dumpable-inl.h index 2cdf083f01..9d7fc39093 100644 --- a/runtime/base/dumpable-inl.h +++ b/runtime/base/dumpable-inl.h @@ -19,7 +19,7 @@ #include "base/dumpable.h" #include "base/mutex.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/base/logging.cc b/runtime/base/logging.cc index 553928d20a..adfd7d323c 100644 --- a/runtime/base/logging.cc +++ b/runtime/base/logging.cc @@ -21,7 +21,7 @@ #include <sstream> #include "base/mutex.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "utils.h" // Headers for LogMessage::LogLine. diff --git a/runtime/base/mutex-inl.h b/runtime/base/mutex-inl.h index 08b370ec4e..0ac2399a5d 100644 --- a/runtime/base/mutex-inl.h +++ b/runtime/base/mutex-inl.h @@ -194,6 +194,16 @@ inline uint64_t Mutex::GetExclusiveOwnerTid() const { return exclusive_owner_; } +inline void Mutex::AssertExclusiveHeld(const Thread* self) const { + if (kDebugLocking && (gAborting == 0)) { + CHECK(IsExclusiveHeld(self)) << *this; + } +} + +inline void Mutex::AssertHeld(const Thread* self) const { + AssertExclusiveHeld(self); +} + inline bool ReaderWriterMutex::IsExclusiveHeld(const Thread* self) const { DCHECK(self == nullptr || self == Thread::Current()); bool result = (GetExclusiveOwnerTid() == SafeGetTid(self)); @@ -221,6 +231,16 @@ inline uint64_t ReaderWriterMutex::GetExclusiveOwnerTid() const { #endif } +inline void ReaderWriterMutex::AssertExclusiveHeld(const Thread* self) const { + if (kDebugLocking && (gAborting == 0)) { + CHECK(IsExclusiveHeld(self)) << *this; + } +} + +inline void ReaderWriterMutex::AssertWriterHeld(const Thread* self) const { + AssertExclusiveHeld(self); +} + inline void MutatorMutex::TransitionFromRunnableToSuspended(Thread* self) { AssertSharedHeld(self); RegisterAsUnlocked(self); @@ -231,6 +251,19 @@ inline void MutatorMutex::TransitionFromSuspendedToRunnable(Thread* self) { AssertSharedHeld(self); } +inline ReaderMutexLock::ReaderMutexLock(Thread* self, ReaderWriterMutex& mu) + : self_(self), mu_(mu) { + mu_.SharedLock(self_); +} + +inline ReaderMutexLock::~ReaderMutexLock() { + mu_.SharedUnlock(self_); +} + +// Catch bug where variable name is omitted. "ReaderMutexLock (lock);" instead of +// "ReaderMutexLock mu(lock)". +#define ReaderMutexLock(x) static_assert(0, "ReaderMutexLock declaration missing variable name") + } // namespace art #endif // ART_RUNTIME_BASE_MUTEX_INL_H_ diff --git a/runtime/base/mutex.h b/runtime/base/mutex.h index 03ae63a068..e77d8d749d 100644 --- a/runtime/base/mutex.h +++ b/runtime/base/mutex.h @@ -244,15 +244,11 @@ class LOCKABLE Mutex : public BaseMutex { void Unlock(Thread* self) RELEASE() { ExclusiveUnlock(self); } // Is the current thread the exclusive holder of the Mutex. - bool IsExclusiveHeld(const Thread* self) const; + ALWAYS_INLINE bool IsExclusiveHeld(const Thread* self) const; // Assert that the Mutex is exclusively held by the current thread. - void AssertExclusiveHeld(const Thread* self) ASSERT_CAPABILITY(this) { - if (kDebugLocking && (gAborting == 0)) { - CHECK(IsExclusiveHeld(self)) << *this; - } - } - void AssertHeld(const Thread* self) ASSERT_CAPABILITY(this) { AssertExclusiveHeld(self); } + ALWAYS_INLINE void AssertExclusiveHeld(const Thread* self) const ASSERT_CAPABILITY(this); + ALWAYS_INLINE void AssertHeld(const Thread* self) const ASSERT_CAPABILITY(this); // Assert that the Mutex is not held by the current thread. void AssertNotHeldExclusive(const Thread* self) ASSERT_CAPABILITY(!*this) { @@ -349,15 +345,11 @@ class SHARED_LOCKABLE ReaderWriterMutex : public BaseMutex { void ReaderUnlock(Thread* self) RELEASE_SHARED() { SharedUnlock(self); } // Is the current thread the exclusive holder of the ReaderWriterMutex. - bool IsExclusiveHeld(const Thread* self) const; + ALWAYS_INLINE bool IsExclusiveHeld(const Thread* self) const; // Assert the current thread has exclusive access to the ReaderWriterMutex. - void AssertExclusiveHeld(const Thread* self) ASSERT_CAPABILITY(this) { - if (kDebugLocking && (gAborting == 0)) { - CHECK(IsExclusiveHeld(self)) << *this; - } - } - void AssertWriterHeld(const Thread* self) ASSERT_CAPABILITY(this) { AssertExclusiveHeld(self); } + ALWAYS_INLINE void AssertExclusiveHeld(const Thread* self) const ASSERT_CAPABILITY(this); + ALWAYS_INLINE void AssertWriterHeld(const Thread* self) const ASSERT_CAPABILITY(this); // Assert the current thread doesn't have exclusive access to the ReaderWriterMutex. void AssertNotExclusiveHeld(const Thread* self) ASSERT_CAPABILITY(!this) { @@ -517,23 +509,15 @@ class SCOPED_CAPABILITY MutexLock { // construction and releases it upon destruction. class SCOPED_CAPABILITY ReaderMutexLock { public: - ReaderMutexLock(Thread* self, ReaderWriterMutex& mu) ACQUIRE(mu) ALWAYS_INLINE : - self_(self), mu_(mu) { - mu_.SharedLock(self_); - } + ALWAYS_INLINE ReaderMutexLock(Thread* self, ReaderWriterMutex& mu) ACQUIRE(mu); - ~ReaderMutexLock() RELEASE() ALWAYS_INLINE { - mu_.SharedUnlock(self_); - } + ALWAYS_INLINE ~ReaderMutexLock() RELEASE(); private: Thread* const self_; ReaderWriterMutex& mu_; DISALLOW_COPY_AND_ASSIGN(ReaderMutexLock); }; -// Catch bug where variable name is omitted. "ReaderMutexLock (lock);" instead of -// "ReaderMutexLock mu(lock)". -#define ReaderMutexLock(x) static_assert(0, "ReaderMutexLock declaration missing variable name") // Scoped locker/unlocker for a ReaderWriterMutex that acquires write access to mu upon // construction and releases it upon destruction. diff --git a/runtime/base/mutex_test.cc b/runtime/base/mutex_test.cc index 340550f02e..752e77a7c0 100644 --- a/runtime/base/mutex_test.cc +++ b/runtime/base/mutex_test.cc @@ -14,10 +14,10 @@ * limitations under the License. */ -#include "mutex.h" +#include "mutex-inl.h" #include "common_runtime_test.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/base/timing_logger.cc b/runtime/base/timing_logger.cc index 9a0e0d02a6..aaa24317bb 100644 --- a/runtime/base/timing_logger.cc +++ b/runtime/base/timing_logger.cc @@ -24,7 +24,9 @@ #include "base/histogram-inl.h" #include "base/systrace.h" #include "base/time_utils.h" -#include "thread-inl.h" +#include "gc/heap.h" +#include "runtime.h" +#include "thread-current-inl.h" #include <cmath> #include <iomanip> diff --git a/runtime/cha.h b/runtime/cha.h index d9692a684e..81458db601 100644 --- a/runtime/cha.h +++ b/runtime/cha.h @@ -17,7 +17,6 @@ #ifndef ART_RUNTIME_CHA_H_ #define ART_RUNTIME_CHA_H_ -#include "art_method.h" #include "base/enums.h" #include "base/mutex.h" #include "handle.h" @@ -28,6 +27,8 @@ namespace art { +class ArtMethod; + /** * Class Hierarchy Analysis (CHA) tries to devirtualize virtual calls into * direct calls based on the info generated by analyzing class hierarchies. diff --git a/runtime/check_reference_map_visitor.h b/runtime/check_reference_map_visitor.h index a955cb5acb..f6c8fa9659 100644 --- a/runtime/check_reference_map_visitor.h +++ b/runtime/check_reference_map_visitor.h @@ -20,6 +20,7 @@ #include "art_method-inl.h" #include "oat_quick_method_header.h" #include "scoped_thread_state_change-inl.h" +#include "stack.h" #include "stack_map.h" namespace art { diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index 81ca764523..c3a8fc5383 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -8949,7 +8949,8 @@ class GetResolvedClassesVisitor : public ClassVisitor { last_dex_file_ = &dex_file; DexCacheResolvedClasses resolved_classes(dex_file.GetLocation(), dex_file.GetBaseLocation(), - dex_file.GetLocationChecksum()); + dex_file.GetLocationChecksum(), + dex_file.NumMethodIds()); last_resolved_classes_ = result_->find(resolved_classes); if (last_resolved_classes_ == result_->end()) { last_resolved_classes_ = result_->insert(resolved_classes).first; @@ -9046,6 +9047,12 @@ std::unordered_set<std::string> ClassLinker::GetClassDescriptorsForResolvedClass << info.GetClasses().size() << " classes"; DCHECK_EQ(dex_file->GetLocationChecksum(), info.GetLocationChecksum()); for (dex::TypeIndex type_idx : info.GetClasses()) { + if (!dex_file->IsTypeIndexValid(type_idx)) { + // Something went bad. The profile is probably corrupted. Abort and return an emtpy set. + LOG(WARNING) << "Corrupted profile: invalid type index " + << type_idx.index_ << " in dex " << location; + return std::unordered_set<std::string>(); + } const DexFile::TypeId& type_id = dex_file->GetTypeId(type_idx); const char* descriptor = dex_file->GetTypeDescriptor(type_id); ret.insert(descriptor); diff --git a/runtime/class_linker_test.cc b/runtime/class_linker_test.cc index b421810113..684a261cca 100644 --- a/runtime/class_linker_test.cc +++ b/runtime/class_linker_test.cc @@ -50,7 +50,7 @@ #include "mirror/string-inl.h" #include "handle_scope-inl.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/class_table-inl.h b/runtime/class_table-inl.h index dfe8949134..35fce4063b 100644 --- a/runtime/class_table-inl.h +++ b/runtime/class_table-inl.h @@ -93,7 +93,7 @@ inline mirror::Class* ClassTable::TableSlot::Read() const { if (kReadBarrierOption != kWithoutReadBarrier && before_ptr != after_ptr) { // If another thread raced and updated the reference, do not store the read barrier updated // one. - data_.CompareExchangeStrongRelaxed(before, Encode(after_ptr, MaskHash(before))); + data_.CompareExchangeStrongRelease(before, Encode(after_ptr, MaskHash(before))); } return after_ptr.Ptr(); } @@ -108,7 +108,7 @@ inline void ClassTable::TableSlot::VisitRoot(const Visitor& visitor) const { if (before_ptr != after_ptr) { // If another thread raced and updated the reference, do not store the read barrier updated // one. - data_.CompareExchangeStrongRelaxed(before, Encode(after_ptr, MaskHash(before))); + data_.CompareExchangeStrongRelease(before, Encode(after_ptr, MaskHash(before))); } } diff --git a/runtime/debugger.cc b/runtime/debugger.cc index d0b50fe820..cfa56a5769 100644 --- a/runtime/debugger.cc +++ b/runtime/debugger.cc @@ -38,7 +38,7 @@ #include "gc/scoped_gc_critical_section.h" #include "gc/space/large_object_space.h" #include "gc/space/space-inl.h" -#include "handle_scope.h" +#include "handle_scope-inl.h" #include "jdwp/jdwp_priv.h" #include "jdwp/object_registry.h" #include "jni_internal.h" @@ -56,7 +56,7 @@ #include "scoped_thread_state_change-inl.h" #include "ScopedLocalRef.h" #include "ScopedPrimitiveArray.h" -#include "handle_scope-inl.h" +#include "stack.h" #include "thread_list.h" #include "utf.h" #include "well_known_classes.h" diff --git a/runtime/dex_cache_resolved_classes.h b/runtime/dex_cache_resolved_classes.h index bebdf0dbfe..2278b052ed 100644 --- a/runtime/dex_cache_resolved_classes.h +++ b/runtime/dex_cache_resolved_classes.h @@ -30,10 +30,12 @@ class DexCacheResolvedClasses { public: DexCacheResolvedClasses(const std::string& dex_location, const std::string& base_location, - uint32_t location_checksum) + uint32_t location_checksum, + uint32_t num_method_ids) : dex_location_(dex_location), base_location_(base_location), - location_checksum_(location_checksum) {} + location_checksum_(location_checksum), + num_method_ids_(num_method_ids) {} // Only compare the key elements, ignore the resolved classes. int Compare(const DexCacheResolvedClasses& other) const { @@ -69,10 +71,15 @@ class DexCacheResolvedClasses { return classes_; } + size_t NumMethodIds() const { + return num_method_ids_; + } + private: const std::string dex_location_; const std::string base_location_; const uint32_t location_checksum_; + const uint32_t num_method_ids_; // Array of resolved class def indexes. mutable std::unordered_set<dex::TypeIndex> classes_; }; diff --git a/runtime/dex_file.h b/runtime/dex_file.h index 36c734197a..591ba42003 100644 --- a/runtime/dex_file.h +++ b/runtime/dex_file.h @@ -582,6 +582,10 @@ class DexFile { return header_->type_ids_size_; } + bool IsTypeIndexValid(dex::TypeIndex idx) const { + return idx.IsValid() && idx.index_ < NumTypeIds(); + } + // Returns the TypeId at the specified index. const TypeId& GetTypeId(dex::TypeIndex idx) const { DCHECK_LT(idx.index_, NumTypeIds()) << GetLocation(); diff --git a/runtime/dex_file_test.cc b/runtime/dex_file_test.cc index 6627550574..78d5c5f4ba 100644 --- a/runtime/dex_file_test.cc +++ b/runtime/dex_file_test.cc @@ -16,6 +16,8 @@ #include "dex_file.h" +#include <sys/mman.h> + #include <memory> #include "base/stl_util.h" @@ -25,7 +27,7 @@ #include "mem_map.h" #include "os.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "utils.h" namespace art { diff --git a/runtime/dex_file_verifier_test.cc b/runtime/dex_file_verifier_test.cc index 068e1223e5..0e58e6d564 100644 --- a/runtime/dex_file_verifier_test.cc +++ b/runtime/dex_file_verifier_test.cc @@ -29,7 +29,7 @@ #include "dex_file_types.h" #include "leb128.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "utils.h" namespace art { diff --git a/runtime/dex_method_iterator_test.cc b/runtime/dex_method_iterator_test.cc index cd8c39096d..e83829bb46 100644 --- a/runtime/dex_method_iterator_test.cc +++ b/runtime/dex_method_iterator_test.cc @@ -20,7 +20,7 @@ #include "common_runtime_test.h" #include "oat_file.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/entrypoints/quick/callee_save_frame.h b/runtime/entrypoints/quick/callee_save_frame.h index df37f9586f..c94bf4a318 100644 --- a/runtime/entrypoints/quick/callee_save_frame.h +++ b/runtime/entrypoints/quick/callee_save_frame.h @@ -21,7 +21,7 @@ #include "base/enums.h" #include "base/mutex.h" #include "runtime.h" -#include "thread-inl.h" +#include "thread.h" // Specific frame size code is in architecture-specific files. We include this to compile-time // specialize the code. @@ -46,13 +46,6 @@ class ScopedQuickEntrypointChecks { } } - ScopedQuickEntrypointChecks() REQUIRES_SHARED(Locks::mutator_lock_) - : self_(kIsDebugBuild ? Thread::Current() : nullptr), exit_check_(kIsDebugBuild) { - if (kIsDebugBuild) { - TestsOnEntry(); - } - } - ~ScopedQuickEntrypointChecks() REQUIRES_SHARED(Locks::mutator_lock_) { if (exit_check_) { TestsOnExit(); diff --git a/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc b/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc index 81560ccbaf..aa1ebb7ee7 100644 --- a/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc @@ -21,7 +21,7 @@ #include "instrumentation.h" #include "mirror/object-inl.h" #include "runtime.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc index 2b349e39a0..90231e289e 100644 --- a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc @@ -18,6 +18,7 @@ #include "base/enums.h" #include "callee_save_frame.h" #include "common_throws.h" +#include "debugger.h" #include "dex_file-inl.h" #include "dex_instruction-inl.h" #include "entrypoints/entrypoint_utils-inl.h" @@ -40,7 +41,7 @@ #include "runtime.h" #include "scoped_thread_state_change-inl.h" #include "stack.h" -#include "debugger.h" +#include "thread-inl.h" #include "well_known_classes.h" namespace art { diff --git a/runtime/fault_handler.cc b/runtime/fault_handler.cc index 5594f4dfc7..fd0cd5f0b2 100644 --- a/runtime/fault_handler.cc +++ b/runtime/fault_handler.cc @@ -27,7 +27,7 @@ #include "mirror/object_reference.h" #include "oat_quick_method_header.h" #include "sigchain.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "verify_object-inl.h" namespace art { diff --git a/runtime/gc/accounting/card_table.cc b/runtime/gc/accounting/card_table.cc index 450659791d..01b5896650 100644 --- a/runtime/gc/accounting/card_table.cc +++ b/runtime/gc/accounting/card_table.cc @@ -16,6 +16,8 @@ #include "card_table.h" +#include <sys/mman.h> + #include "base/logging.h" #include "base/systrace.h" #include "card_table-inl.h" diff --git a/runtime/gc/accounting/card_table.h b/runtime/gc/accounting/card_table.h index c3dd21f113..17acc763d1 100644 --- a/runtime/gc/accounting/card_table.h +++ b/runtime/gc/accounting/card_table.h @@ -155,6 +155,14 @@ class CardTable { }; } // namespace accounting + +class AgeCardVisitor { + public: + uint8_t operator()(uint8_t card) const { + return (card == accounting::CardTable::kCardDirty) ? card - 1 : 0; + } +}; + } // namespace gc } // namespace art diff --git a/runtime/gc/accounting/mod_union_table.cc b/runtime/gc/accounting/mod_union_table.cc index c416b9cc3d..57c290ea94 100644 --- a/runtime/gc/accounting/mod_union_table.cc +++ b/runtime/gc/accounting/mod_union_table.cc @@ -28,7 +28,7 @@ #include "mirror/object-inl.h" #include "mirror/object-refvisitor-inl.h" #include "space_bitmap-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace gc { diff --git a/runtime/gc/accounting/mod_union_table_test.cc b/runtime/gc/accounting/mod_union_table_test.cc index 48a8742cc8..e5b8ea5609 100644 --- a/runtime/gc/accounting/mod_union_table_test.cc +++ b/runtime/gc/accounting/mod_union_table_test.cc @@ -21,7 +21,7 @@ #include "gc/space/space-inl.h" #include "mirror/array-inl.h" #include "space_bitmap-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" namespace art { diff --git a/runtime/gc/allocator/rosalloc.cc b/runtime/gc/allocator/rosalloc.cc index 35a251fda8..d5d3540b1f 100644 --- a/runtime/gc/allocator/rosalloc.cc +++ b/runtime/gc/allocator/rosalloc.cc @@ -30,7 +30,7 @@ #include "mirror/class-inl.h" #include "mirror/object.h" #include "mirror/object-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" namespace art { diff --git a/runtime/gc/collector/concurrent_copying-inl.h b/runtime/gc/collector/concurrent_copying-inl.h index 3503973321..85a656ec51 100644 --- a/runtime/gc/collector/concurrent_copying-inl.h +++ b/runtime/gc/collector/concurrent_copying-inl.h @@ -19,11 +19,12 @@ #include "concurrent_copying.h" +#include "gc/accounting/atomic_stack.h" #include "gc/accounting/space_bitmap-inl.h" #include "gc/heap.h" #include "gc/space/region_space.h" -#include "mirror/object-readbarrier-inl.h" #include "lock_word.h" +#include "mirror/object-readbarrier-inl.h" namespace art { namespace gc { diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc index ef843c6650..c0d648117c 100644 --- a/runtime/gc/collector/concurrent_copying.cc +++ b/runtime/gc/collector/concurrent_copying.cc @@ -359,7 +359,7 @@ class ConcurrentCopying::ThreadFlipVisitor : public Closure, public RootVisitor ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); // We can use the non-CAS VisitRoots functions below because we update thread-local GC roots // only. - thread->VisitRoots(this); + thread->VisitRoots(this, kVisitRootFlagAllRoots); concurrent_copying_->GetBarrier().Pass(self); } @@ -2086,8 +2086,11 @@ inline void ConcurrentCopying::Process(mirror::Object* obj, MemberOffset offset) // It was updated by the mutator. break; } - } while (!obj->CasFieldWeakRelaxedObjectWithoutWriteBarrier< - false, false, kVerifyNone>(offset, expected_ref, new_ref)); + // Use release cas to make sure threads reading the reference see contents of copied objects. + } while (!obj->CasFieldWeakReleaseObjectWithoutWriteBarrier<false, false, kVerifyNone>( + offset, + expected_ref, + new_ref)); } // Process some roots. diff --git a/runtime/gc/collector/concurrent_copying.h b/runtime/gc/collector/concurrent_copying.h index 377f4d30ba..f8ca8dba42 100644 --- a/runtime/gc/collector/concurrent_copying.h +++ b/runtime/gc/collector/concurrent_copying.h @@ -23,7 +23,6 @@ #include "jni.h" #include "object_callbacks.h" #include "offsets.h" -#include "gc/accounting/space_bitmap.h" #include "mirror/object.h" #include "mirror/object_reference.h" #include "safe_map.h" @@ -40,6 +39,7 @@ namespace gc { namespace accounting { template<typename T> class AtomicStack; typedef AtomicStack<mirror::Object> ObjectStack; + template <size_t kAlignment> class SpaceBitmap; typedef SpaceBitmap<kObjectAlignment> ContinuousSpaceBitmap; class HeapBitmap; class ReadBarrierTable; @@ -284,7 +284,7 @@ class ConcurrentCopying : public GarbageCollector { bool is_active_; // True while the collection is ongoing. bool is_asserting_to_space_invariant_; // True while asserting the to-space invariant. ImmuneSpaces immune_spaces_; - accounting::SpaceBitmap<kObjectAlignment>* region_space_bitmap_; + accounting::ContinuousSpaceBitmap* region_space_bitmap_; // A cache of Heap::GetMarkBitmap(). accounting::HeapBitmap* heap_mark_bitmap_; size_t live_stack_freeze_size_; diff --git a/runtime/gc/collector/garbage_collector.cc b/runtime/gc/collector/garbage_collector.cc index 1e4196b1ac..c5a341fc80 100644 --- a/runtime/gc/collector/garbage_collector.cc +++ b/runtime/gc/collector/garbage_collector.cc @@ -31,7 +31,8 @@ #include "gc/heap.h" #include "gc/space/large_object_space.h" #include "gc/space/space-inl.h" -#include "thread-inl.h" +#include "runtime.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "utils.h" diff --git a/runtime/gc/collector/garbage_collector.h b/runtime/gc/collector/garbage_collector.h index 14d049971f..dec206be30 100644 --- a/runtime/gc/collector/garbage_collector.h +++ b/runtime/gc/collector/garbage_collector.h @@ -27,6 +27,8 @@ #include "gc/gc_cause.h" #include "gc_root.h" #include "gc_type.h" +#include "iteration.h" +#include "object_byte_pair.h" #include "object_callbacks.h" namespace art { @@ -43,85 +45,6 @@ class Heap; namespace collector { -struct ObjectBytePair { - explicit ObjectBytePair(uint64_t num_objects = 0, int64_t num_bytes = 0) - : objects(num_objects), bytes(num_bytes) {} - void Add(const ObjectBytePair& other) { - objects += other.objects; - bytes += other.bytes; - } - // Number of objects which were freed. - uint64_t objects; - // Freed bytes are signed since the GC can free negative bytes if it promotes objects to a space - // which has a larger allocation size. - int64_t bytes; -}; - -// A information related single garbage collector iteration. Since we only ever have one GC running -// at any given time, we can have a single iteration info. -class Iteration { - public: - Iteration(); - // Returns how long the mutators were paused in nanoseconds. - const std::vector<uint64_t>& GetPauseTimes() const { - return pause_times_; - } - TimingLogger* GetTimings() { - return &timings_; - } - // Returns how long the GC took to complete in nanoseconds. - uint64_t GetDurationNs() const { - return duration_ns_; - } - int64_t GetFreedBytes() const { - return freed_.bytes; - } - int64_t GetFreedLargeObjectBytes() const { - return freed_los_.bytes; - } - uint64_t GetFreedObjects() const { - return freed_.objects; - } - uint64_t GetFreedLargeObjects() const { - return freed_los_.objects; - } - uint64_t GetFreedRevokeBytes() const { - return freed_bytes_revoke_; - } - void SetFreedRevoke(uint64_t freed) { - freed_bytes_revoke_ = freed; - } - void Reset(GcCause gc_cause, bool clear_soft_references); - // Returns the estimated throughput of the iteration. - uint64_t GetEstimatedThroughput() const; - bool GetClearSoftReferences() const { - return clear_soft_references_; - } - void SetClearSoftReferences(bool clear_soft_references) { - clear_soft_references_ = clear_soft_references; - } - GcCause GetGcCause() const { - return gc_cause_; - } - - private: - void SetDurationNs(uint64_t duration) { - duration_ns_ = duration; - } - - GcCause gc_cause_; - bool clear_soft_references_; - uint64_t duration_ns_; - TimingLogger timings_; - ObjectBytePair freed_; - ObjectBytePair freed_los_; - uint64_t freed_bytes_revoke_; // see Heap::num_bytes_freed_revoke_. - std::vector<uint64_t> pause_times_; - - friend class GarbageCollector; - DISALLOW_COPY_AND_ASSIGN(Iteration); -}; - class GarbageCollector : public RootVisitor, public IsMarkedVisitor, public MarkObjectVisitor { public: class SCOPED_LOCKABLE ScopedPause { diff --git a/runtime/gc/collector/immune_spaces_test.cc b/runtime/gc/collector/immune_spaces_test.cc index cf93ec614d..9823708606 100644 --- a/runtime/gc/collector/immune_spaces_test.cc +++ b/runtime/gc/collector/immune_spaces_test.cc @@ -14,12 +14,14 @@ * limitations under the License. */ +#include <sys/mman.h> + #include "common_runtime_test.h" #include "gc/collector/immune_spaces.h" #include "gc/space/image_space.h" #include "gc/space/space-inl.h" #include "oat_file.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace mirror { diff --git a/runtime/gc/collector/iteration.h b/runtime/gc/collector/iteration.h new file mode 100644 index 0000000000..fbe41664f7 --- /dev/null +++ b/runtime/gc/collector/iteration.h @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_GC_COLLECTOR_ITERATION_H_ +#define ART_RUNTIME_GC_COLLECTOR_ITERATION_H_ + +#include <inttypes.h> +#include <vector> + +#include "android-base/macros.h" +#include "base/timing_logger.h" +#include "object_byte_pair.h" + +namespace art { +namespace gc { +namespace collector { + +// A information related single garbage collector iteration. Since we only ever have one GC running +// at any given time, we can have a single iteration info. +class Iteration { + public: + Iteration(); + // Returns how long the mutators were paused in nanoseconds. + const std::vector<uint64_t>& GetPauseTimes() const { + return pause_times_; + } + TimingLogger* GetTimings() { + return &timings_; + } + // Returns how long the GC took to complete in nanoseconds. + uint64_t GetDurationNs() const { + return duration_ns_; + } + int64_t GetFreedBytes() const { + return freed_.bytes; + } + int64_t GetFreedLargeObjectBytes() const { + return freed_los_.bytes; + } + uint64_t GetFreedObjects() const { + return freed_.objects; + } + uint64_t GetFreedLargeObjects() const { + return freed_los_.objects; + } + uint64_t GetFreedRevokeBytes() const { + return freed_bytes_revoke_; + } + void SetFreedRevoke(uint64_t freed) { + freed_bytes_revoke_ = freed; + } + void Reset(GcCause gc_cause, bool clear_soft_references); + // Returns the estimated throughput of the iteration. + uint64_t GetEstimatedThroughput() const; + bool GetClearSoftReferences() const { + return clear_soft_references_; + } + void SetClearSoftReferences(bool clear_soft_references) { + clear_soft_references_ = clear_soft_references; + } + GcCause GetGcCause() const { + return gc_cause_; + } + + private: + void SetDurationNs(uint64_t duration) { + duration_ns_ = duration; + } + + GcCause gc_cause_; + bool clear_soft_references_; + uint64_t duration_ns_; + TimingLogger timings_; + ObjectBytePair freed_; + ObjectBytePair freed_los_; + uint64_t freed_bytes_revoke_; // see Heap::num_bytes_freed_revoke_. + std::vector<uint64_t> pause_times_; + + friend class GarbageCollector; + DISALLOW_COPY_AND_ASSIGN(Iteration); +}; + +} // namespace collector +} // namespace gc +} // namespace art + +#endif // ART_RUNTIME_GC_COLLECTOR_ITERATION_H_ diff --git a/runtime/gc/collector/mark_compact.cc b/runtime/gc/collector/mark_compact.cc index 9d3d950a0f..aef98dee58 100644 --- a/runtime/gc/collector/mark_compact.cc +++ b/runtime/gc/collector/mark_compact.cc @@ -32,7 +32,7 @@ #include "mirror/object-refvisitor-inl.h" #include "runtime.h" #include "stack.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" namespace art { diff --git a/runtime/gc/collector/mark_sweep.cc b/runtime/gc/collector/mark_sweep.cc index f591cf09ca..fb82b4d270 100644 --- a/runtime/gc/collector/mark_sweep.cc +++ b/runtime/gc/collector/mark_sweep.cc @@ -42,7 +42,7 @@ #include "mirror/object-inl.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" namespace art { @@ -1141,7 +1141,7 @@ class MarkSweep::CheckpointMarkThreadRoots : public Closure, public RootVisitor Thread* const self = Thread::Current(); CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) << thread->GetState() << " thread " << thread << " self " << self; - thread->VisitRoots(this); + thread->VisitRoots(this, kVisitRootFlagAllRoots); if (revoke_ros_alloc_thread_local_buffers_at_checkpoint_) { ScopedTrace trace2("RevokeRosAllocThreadLocalBuffers"); mark_sweep_->GetHeap()->RevokeRosAllocThreadLocalBuffers(thread); diff --git a/runtime/gc/collector/object_byte_pair.h b/runtime/gc/collector/object_byte_pair.h new file mode 100644 index 0000000000..16ef06b6dd --- /dev/null +++ b/runtime/gc/collector/object_byte_pair.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_GC_COLLECTOR_OBJECT_BYTE_PAIR_H_ +#define ART_RUNTIME_GC_COLLECTOR_OBJECT_BYTE_PAIR_H_ + +#include <inttypes.h> + +namespace art { +namespace gc { +namespace collector { + +struct ObjectBytePair { + explicit ObjectBytePair(uint64_t num_objects = 0, int64_t num_bytes = 0) + : objects(num_objects), bytes(num_bytes) {} + void Add(const ObjectBytePair& other) { + objects += other.objects; + bytes += other.bytes; + } + // Number of objects which were freed. + uint64_t objects; + // Freed bytes are signed since the GC can free negative bytes if it promotes objects to a space + // which has a larger allocation size. + int64_t bytes; +}; + +} // namespace collector +} // namespace gc +} // namespace art + +#endif // ART_RUNTIME_GC_COLLECTOR_OBJECT_BYTE_PAIR_H_ diff --git a/runtime/gc/collector/partial_mark_sweep.cc b/runtime/gc/collector/partial_mark_sweep.cc index 984779484e..f6ca867e69 100644 --- a/runtime/gc/collector/partial_mark_sweep.cc +++ b/runtime/gc/collector/partial_mark_sweep.cc @@ -19,7 +19,7 @@ #include "gc/heap.h" #include "gc/space/space.h" #include "partial_mark_sweep.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace gc { diff --git a/runtime/gc/collector/sticky_mark_sweep.cc b/runtime/gc/collector/sticky_mark_sweep.cc index a2dbe3f7a0..98fdfac17b 100644 --- a/runtime/gc/collector/sticky_mark_sweep.cc +++ b/runtime/gc/collector/sticky_mark_sweep.cc @@ -14,11 +14,15 @@ * limitations under the License. */ +#include "sticky_mark_sweep.h" + +#include "gc/accounting/atomic_stack.h" +#include "gc/accounting/card_table.h" #include "gc/heap.h" #include "gc/space/large_object_space.h" #include "gc/space/space-inl.h" -#include "sticky_mark_sweep.h" -#include "thread-inl.h" +#include "runtime.h" +#include "thread-current-inl.h" namespace art { namespace gc { diff --git a/runtime/gc/heap-inl.h b/runtime/gc/heap-inl.h index 79086da703..060f12db33 100644 --- a/runtime/gc/heap-inl.h +++ b/runtime/gc/heap-inl.h @@ -21,6 +21,7 @@ #include "allocation_listener.h" #include "base/time_utils.h" +#include "gc/accounting/atomic_stack.h" #include "gc/accounting/card_table-inl.h" #include "gc/allocation_record.h" #include "gc/collector/semi_space.h" diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc index df097a0e60..1af3b57830 100644 --- a/runtime/gc/heap.cc +++ b/runtime/gc/heap.cc @@ -37,10 +37,10 @@ #include "cutils/sched_policy.h" #include "debugger.h" #include "dex_file-inl.h" -#include "gc/accounting/atomic_stack.h" #include "gc/accounting/card_table-inl.h" #include "gc/accounting/heap_bitmap-inl.h" #include "gc/accounting/mod_union_table-inl.h" +#include "gc/accounting/read_barrier_table.h" #include "gc/accounting/remembered_set.h" #include "gc/accounting/space_bitmap-inl.h" #include "gc/collector/concurrent_copying.h" @@ -63,6 +63,7 @@ #include "gc/verification.h" #include "entrypoints/quick/quick_alloc_entrypoints.h" #include "gc_pause_listener.h" +#include "gc_root.h" #include "heap-inl.h" #include "image.h" #include "intern_table.h" diff --git a/runtime/gc/heap.h b/runtime/gc/heap.h index 72871785e5..24f4ce29e2 100644 --- a/runtime/gc/heap.h +++ b/runtime/gc/heap.h @@ -26,11 +26,9 @@ #include "arch/instruction_set.h" #include "atomic.h" #include "base/time_utils.h" -#include "gc/accounting/atomic_stack.h" -#include "gc/accounting/card_table.h" -#include "gc/accounting/read_barrier_table.h" #include "gc/gc_cause.h" #include "gc/collector/gc_type.h" +#include "gc/collector/iteration.h" #include "gc/collector_type.h" #include "gc/space/large_object_space.h" #include "globals.h" @@ -46,6 +44,7 @@ namespace art { class ConditionVariable; class Mutex; +class RootVisitor; class StackVisitor; class Thread; class ThreadPool; @@ -67,8 +66,12 @@ class TaskProcessor; class Verification; namespace accounting { + template <typename T> class AtomicStack; + typedef AtomicStack<mirror::Object> ObjectStack; + class CardTable; class HeapBitmap; class ModUnionTable; + class ReadBarrierTable; class RememberedSet; } // namespace accounting @@ -99,13 +102,6 @@ namespace space { class ZygoteSpace; } // namespace space -class AgeCardVisitor { - public: - uint8_t operator()(uint8_t card) const { - return (card == accounting::CardTable::kCardDirty) ? card - 1 : 0; - } -}; - enum HomogeneousSpaceCompactResult { // Success. kSuccess, diff --git a/runtime/gc/scoped_gc_critical_section.cc b/runtime/gc/scoped_gc_critical_section.cc index f937d2c778..2976dd0252 100644 --- a/runtime/gc/scoped_gc_critical_section.cc +++ b/runtime/gc/scoped_gc_critical_section.cc @@ -19,7 +19,7 @@ #include "gc/collector_type.h" #include "gc/heap.h" #include "runtime.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace gc { diff --git a/runtime/gc/space/bump_pointer_space-inl.h b/runtime/gc/space/bump_pointer_space-inl.h index 45cea5a48c..1509bb027d 100644 --- a/runtime/gc/space/bump_pointer_space-inl.h +++ b/runtime/gc/space/bump_pointer_space-inl.h @@ -17,9 +17,10 @@ #ifndef ART_RUNTIME_GC_SPACE_BUMP_POINTER_SPACE_INL_H_ #define ART_RUNTIME_GC_SPACE_BUMP_POINTER_SPACE_INL_H_ -#include "base/bit_utils.h" #include "bump_pointer_space.h" +#include "base/bit_utils.h" + namespace art { namespace gc { namespace space { @@ -86,15 +87,6 @@ inline mirror::Object* BumpPointerSpace::AllocNonvirtual(size_t num_bytes) { return ret; } -inline size_t BumpPointerSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) - REQUIRES_SHARED(Locks::mutator_lock_) { - size_t num_bytes = obj->SizeOf(); - if (usable_size != nullptr) { - *usable_size = RoundUp(num_bytes, kAlignment); - } - return num_bytes; -} - } // namespace space } // namespace gc } // namespace art diff --git a/runtime/gc/space/bump_pointer_space.cc b/runtime/gc/space/bump_pointer_space.cc index 426b33218c..bb1ede15f2 100644 --- a/runtime/gc/space/bump_pointer_space.cc +++ b/runtime/gc/space/bump_pointer_space.cc @@ -271,6 +271,14 @@ void BumpPointerSpace::LogFragmentationAllocFailure(std::ostream& os, // Caller's job to print failed_alloc_bytes. } +size_t BumpPointerSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) { + size_t num_bytes = obj->SizeOf(); + if (usable_size != nullptr) { + *usable_size = RoundUp(num_bytes, kAlignment); + } + return num_bytes; +} + } // namespace space } // namespace gc } // namespace art diff --git a/runtime/gc/space/image_space.h b/runtime/gc/space/image_space.h index a4065bf6d6..3383d6b383 100644 --- a/runtime/gc/space/image_space.h +++ b/runtime/gc/space/image_space.h @@ -19,6 +19,7 @@ #include "arch/instruction_set.h" #include "gc/accounting/space_bitmap.h" +#include "image.h" #include "space.h" namespace art { diff --git a/runtime/gc/space/large_object_space.cc b/runtime/gc/space/large_object_space.cc index 3988073de8..4597a96ce2 100644 --- a/runtime/gc/space/large_object_space.cc +++ b/runtime/gc/space/large_object_space.cc @@ -16,19 +16,22 @@ #include "large_object_space.h" +#include <sys/mman.h> + #include <memory> -#include "gc/accounting/heap_bitmap-inl.h" -#include "gc/accounting/space_bitmap-inl.h" #include "base/logging.h" #include "base/memory_tool.h" #include "base/mutex-inl.h" #include "base/stl_util.h" +#include "gc/accounting/heap_bitmap-inl.h" +#include "gc/accounting/space_bitmap-inl.h" +#include "gc/heap.h" #include "image.h" #include "os.h" #include "scoped_thread_state_change-inl.h" #include "space-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace gc { diff --git a/runtime/gc/space/region_space-inl.h b/runtime/gc/space/region_space-inl.h index 3910a03342..fc24fc2974 100644 --- a/runtime/gc/space/region_space-inl.h +++ b/runtime/gc/space/region_space-inl.h @@ -18,7 +18,7 @@ #define ART_RUNTIME_GC_SPACE_REGION_SPACE_INL_H_ #include "region_space.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace gc { @@ -138,20 +138,6 @@ inline mirror::Object* RegionSpace::Region::Alloc(size_t num_bytes, size_t* byte return reinterpret_cast<mirror::Object*>(old_top); } -inline size_t RegionSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) { - size_t num_bytes = obj->SizeOf(); - if (usable_size != nullptr) { - if (LIKELY(num_bytes <= kRegionSize)) { - DCHECK(RefToRegion(obj)->IsAllocated()); - *usable_size = RoundUp(num_bytes, kAlignment); - } else { - DCHECK(RefToRegion(obj)->IsLarge()); - *usable_size = RoundUp(num_bytes, kRegionSize); - } - } - return num_bytes; -} - template<RegionSpace::RegionType kRegionType> uint64_t RegionSpace::GetBytesAllocatedInternal() { uint64_t bytes = 0; diff --git a/runtime/gc/space/region_space.cc b/runtime/gc/space/region_space.cc index 09b4a3a183..27f30e0719 100644 --- a/runtime/gc/space/region_space.cc +++ b/runtime/gc/space/region_space.cc @@ -16,6 +16,7 @@ #include "bump_pointer_space.h" #include "bump_pointer_space-inl.h" +#include "gc/accounting/read_barrier_table.h" #include "mirror/object-inl.h" #include "mirror/class-inl.h" #include "thread_list.h" @@ -511,6 +512,20 @@ void RegionSpace::Region::Dump(std::ostream& os) const { << " is_newly_allocated=" << is_newly_allocated_ << " is_a_tlab=" << is_a_tlab_ << " thread=" << thread_ << "\n"; } +size_t RegionSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) { + size_t num_bytes = obj->SizeOf(); + if (usable_size != nullptr) { + if (LIKELY(num_bytes <= kRegionSize)) { + DCHECK(RefToRegion(obj)->IsAllocated()); + *usable_size = RoundUp(num_bytes, kAlignment); + } else { + DCHECK(RefToRegion(obj)->IsLarge()); + *usable_size = RoundUp(num_bytes, kRegionSize); + } + } + return num_bytes; +} + } // namespace space } // namespace gc } // namespace art diff --git a/runtime/gc/space/region_space.h b/runtime/gc/space/region_space.h index 80eeccaf42..1d1d27e0f4 100644 --- a/runtime/gc/space/region_space.h +++ b/runtime/gc/space/region_space.h @@ -17,13 +17,17 @@ #ifndef ART_RUNTIME_GC_SPACE_REGION_SPACE_H_ #define ART_RUNTIME_GC_SPACE_REGION_SPACE_H_ -#include "gc/accounting/read_barrier_table.h" #include "object_callbacks.h" #include "space.h" #include "thread.h" namespace art { namespace gc { + +namespace accounting { +class ReadBarrierTable; +} // namespace accounting + namespace space { // A space that consists of equal-sized regions. diff --git a/runtime/gc/space/rosalloc_space-inl.h b/runtime/gc/space/rosalloc_space-inl.h index 8bff2b4c0f..09aa7cf8a3 100644 --- a/runtime/gc/space/rosalloc_space-inl.h +++ b/runtime/gc/space/rosalloc_space-inl.h @@ -17,49 +17,17 @@ #ifndef ART_RUNTIME_GC_SPACE_ROSALLOC_SPACE_INL_H_ #define ART_RUNTIME_GC_SPACE_ROSALLOC_SPACE_INL_H_ +#include "rosalloc_space.h" + #include "base/memory_tool.h" #include "gc/allocator/rosalloc-inl.h" #include "gc/space/memory_tool_settings.h" -#include "rosalloc_space.h" #include "thread.h" namespace art { namespace gc { namespace space { -template<bool kMaybeIsRunningOnMemoryTool> -inline size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) { - // obj is a valid object. Use its class in the header to get the size. - // Don't use verification since the object may be dead if we are sweeping. - size_t size = obj->SizeOf<kVerifyNone>(); - bool add_redzones = false; - if (kMaybeIsRunningOnMemoryTool) { - add_redzones = RUNNING_ON_MEMORY_TOOL ? kMemoryToolAddsRedzones : 0; - if (add_redzones) { - size += 2 * kDefaultMemoryToolRedZoneBytes; - } - } else { - DCHECK_EQ(RUNNING_ON_MEMORY_TOOL, 0U); - } - size_t size_by_size = rosalloc_->UsableSize(size); - if (kIsDebugBuild) { - // On memory tool, the red zone has an impact... - const uint8_t* obj_ptr = reinterpret_cast<const uint8_t*>(obj); - size_t size_by_ptr = rosalloc_->UsableSize( - obj_ptr - (add_redzones ? kDefaultMemoryToolRedZoneBytes : 0)); - if (size_by_size != size_by_ptr) { - LOG(INFO) << "Found a bad sized obj of size " << size - << " at " << std::hex << reinterpret_cast<intptr_t>(obj_ptr) << std::dec - << " size_by_size=" << size_by_size << " size_by_ptr=" << size_by_ptr; - } - DCHECK_EQ(size_by_size, size_by_ptr); - } - if (usable_size != nullptr) { - *usable_size = size_by_size; - } - return size_by_size; -} - template<bool kThreadSafe> inline mirror::Object* RosAllocSpace::AllocCommon(Thread* self, size_t num_bytes, size_t* bytes_allocated, size_t* usable_size, diff --git a/runtime/gc/space/rosalloc_space.cc b/runtime/gc/space/rosalloc_space.cc index 8ccbfaa7a3..8d8b745b71 100644 --- a/runtime/gc/space/rosalloc_space.cc +++ b/runtime/gc/space/rosalloc_space.cc @@ -373,6 +373,39 @@ void RosAllocSpace::DumpStats(std::ostream& os) { rosalloc_->DumpStats(os); } +template<bool kMaybeIsRunningOnMemoryTool> +size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) { + // obj is a valid object. Use its class in the header to get the size. + // Don't use verification since the object may be dead if we are sweeping. + size_t size = obj->SizeOf<kVerifyNone>(); + bool add_redzones = false; + if (kMaybeIsRunningOnMemoryTool) { + add_redzones = RUNNING_ON_MEMORY_TOOL ? kMemoryToolAddsRedzones : 0; + if (add_redzones) { + size += 2 * kDefaultMemoryToolRedZoneBytes; + } + } else { + DCHECK_EQ(RUNNING_ON_MEMORY_TOOL, 0U); + } + size_t size_by_size = rosalloc_->UsableSize(size); + if (kIsDebugBuild) { + // On memory tool, the red zone has an impact... + const uint8_t* obj_ptr = reinterpret_cast<const uint8_t*>(obj); + size_t size_by_ptr = rosalloc_->UsableSize( + obj_ptr - (add_redzones ? kDefaultMemoryToolRedZoneBytes : 0)); + if (size_by_size != size_by_ptr) { + LOG(INFO) << "Found a bad sized obj of size " << size + << " at " << std::hex << reinterpret_cast<intptr_t>(obj_ptr) << std::dec + << " size_by_size=" << size_by_size << " size_by_ptr=" << size_by_ptr; + } + DCHECK_EQ(size_by_size, size_by_ptr); + } + if (usable_size != nullptr) { + *usable_size = size_by_size; + } + return size_by_size; +} + } // namespace space namespace allocator { diff --git a/runtime/gc/space/space.cc b/runtime/gc/space/space.cc index a2e2c1c7fb..74ce273abf 100644 --- a/runtime/gc/space/space.cc +++ b/runtime/gc/space/space.cc @@ -19,8 +19,9 @@ #include "base/logging.h" #include "gc/accounting/heap_bitmap.h" #include "gc/accounting/space_bitmap-inl.h" +#include "gc/heap.h" #include "runtime.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace gc { diff --git a/runtime/gc/space/space.h b/runtime/gc/space/space.h index fc558cf8e4..2a4f830843 100644 --- a/runtime/gc/space/space.h +++ b/runtime/gc/space/space.h @@ -24,9 +24,8 @@ #include "base/macros.h" #include "base/mutex.h" #include "gc/accounting/space_bitmap.h" -#include "gc/collector/garbage_collector.h" +#include "gc/collector/object_byte_pair.h" #include "globals.h" -#include "image.h" #include "mem_map.h" namespace art { diff --git a/runtime/gc/space/zygote_space.cc b/runtime/gc/space/zygote_space.cc index bbfcb31ab1..fddb3f2dd2 100644 --- a/runtime/gc/space/zygote_space.cc +++ b/runtime/gc/space/zygote_space.cc @@ -16,10 +16,12 @@ #include "zygote_space.h" +#include "base/mutex-inl.h" #include "gc/accounting/card_table-inl.h" #include "gc/accounting/space_bitmap-inl.h" #include "gc/heap.h" -#include "thread-inl.h" +#include "runtime.h" +#include "thread-current-inl.h" #include "utils.h" namespace art { diff --git a/runtime/gc/task_processor_test.cc b/runtime/gc/task_processor_test.cc index f1d26d9a41..5a75b37b67 100644 --- a/runtime/gc/task_processor_test.cc +++ b/runtime/gc/task_processor_test.cc @@ -18,7 +18,7 @@ #include "common_runtime_test.h" #include "task_processor.h" #include "thread_pool.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace gc { diff --git a/runtime/handle_scope-inl.h b/runtime/handle_scope-inl.h index 492d4b4bd9..d091e7f371 100644 --- a/runtime/handle_scope-inl.h +++ b/runtime/handle_scope-inl.h @@ -22,7 +22,7 @@ #include "base/mutex.h" #include "handle.h" #include "obj_ptr-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "verify_object.h" namespace art { diff --git a/runtime/imtable_test.cc b/runtime/imtable_test.cc index 17149dfe44..d482183d86 100644 --- a/runtime/imtable_test.cc +++ b/runtime/imtable_test.cc @@ -29,7 +29,7 @@ #include "mirror/class_loader.h" #include "handle_scope-inl.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/indirect_reference_table-inl.h b/runtime/indirect_reference_table-inl.h index 2128f8cde8..9673bd9728 100644 --- a/runtime/indirect_reference_table-inl.h +++ b/runtime/indirect_reference_table-inl.h @@ -111,12 +111,12 @@ inline void IrtEntry::Add(ObjPtr<mirror::Object> obj) { if (serial_ == kIRTPrevCount) { serial_ = 0; } - references_[serial_] = GcRoot<mirror::Object>(obj); + references_[serial_] = GcRoot<mirror::Object>(obj.Ptr()); } inline void IrtEntry::SetReference(ObjPtr<mirror::Object> obj) { DCHECK_LT(serial_, kIRTPrevCount); - references_[serial_] = GcRoot<mirror::Object>(obj); + references_[serial_] = GcRoot<mirror::Object>(obj.Ptr()); } } // namespace art diff --git a/runtime/indirect_reference_table.cc b/runtime/indirect_reference_table.cc index c852d5af3a..cff3ea7ecd 100644 --- a/runtime/indirect_reference_table.cc +++ b/runtime/indirect_reference_table.cc @@ -34,6 +34,9 @@ namespace art { static constexpr bool kDumpStackOnNonLocalReference = false; static constexpr bool kDebugIRT = false; +// Maximum table size we allow. +static constexpr size_t kMaxTableSizeInBytes = 128 * MB; + const char* GetIndirectRefKindString(const IndirectRefKind& kind) { switch (kind) { case kHandleScopeOrInvalid: @@ -71,6 +74,9 @@ IndirectReferenceTable::IndirectReferenceTable(size_t max_count, CHECK(error_msg != nullptr); CHECK_NE(desired_kind, kHandleScopeOrInvalid); + // Overflow and maximum check. + CHECK_LE(max_count, kMaxTableSizeInBytes / sizeof(IrtEntry)); + const size_t table_bytes = max_count * sizeof(IrtEntry); table_mem_map_.reset(MemMap::MapAnonymous("indirect ref table", nullptr, table_bytes, PROT_READ | PROT_WRITE, false, false, error_msg)); @@ -203,6 +209,13 @@ static inline void CheckHoleCount(IrtEntry* table, bool IndirectReferenceTable::Resize(size_t new_size, std::string* error_msg) { CHECK_GT(new_size, max_entries_); + constexpr size_t kMaxEntries = kMaxTableSizeInBytes / sizeof(IrtEntry); + if (new_size > kMaxEntries) { + *error_msg = android::base::StringPrintf("Requested size exceeds maximum: %zu", new_size); + return false; + } + // Note: the above check also ensures that there is no overflow below. + const size_t table_bytes = new_size * sizeof(IrtEntry); std::unique_ptr<MemMap> new_map(MemMap::MapAnonymous("indirect ref table", nullptr, @@ -247,6 +260,14 @@ IndirectRef IndirectReferenceTable::Add(IRTSegmentState previous_state, } // Try to double space. + if (std::numeric_limits<size_t>::max() / 2 < max_entries_) { + LOG(FATAL) << "JNI ERROR (app bug): " << kind_ << " table overflow " + << "(max=" << max_entries_ << ")" << std::endl + << MutatorLockedDumpable<IndirectReferenceTable>(*this) + << " Resizing failed: exceeds size_t"; + UNREACHABLE(); + } + std::string error_msg; if (!Resize(max_entries_ * 2, &error_msg)) { LOG(FATAL) << "JNI ERROR (app bug): " << kind_ << " table overflow " @@ -453,4 +474,38 @@ void IndirectReferenceTable::SetSegmentState(IRTSegmentState new_state) { segment_state_ = new_state; } +bool IndirectReferenceTable::EnsureFreeCapacity(size_t free_capacity, std::string* error_msg) { + size_t top_index = segment_state_.top_index; + if (top_index < max_entries_ && top_index + free_capacity <= max_entries_) { + return true; + } + + // We're only gonna do a simple best-effort here, ensuring the asked-for capacity at the end. + if (resizable_ == ResizableCapacity::kNo) { + *error_msg = "Table is not resizable"; + return false; + } + + // Try to increase the table size. + + // Would this overflow? + if (std::numeric_limits<size_t>::max() - free_capacity < top_index) { + *error_msg = "Cannot resize table, overflow."; + return false; + } + + if (!Resize(top_index + free_capacity, error_msg)) { + LOG(WARNING) << "JNI ERROR: Unable to reserve space in EnsureFreeCapacity (" << free_capacity + << "): " << std::endl + << MutatorLockedDumpable<IndirectReferenceTable>(*this) + << " Resizing failed: " << *error_msg; + return false; + } + return true; +} + +size_t IndirectReferenceTable::FreeCapacity() { + return max_entries_ - segment_state_.top_index; +} + } // namespace art diff --git a/runtime/indirect_reference_table.h b/runtime/indirect_reference_table.h index 7e452a270a..79d620126c 100644 --- a/runtime/indirect_reference_table.h +++ b/runtime/indirect_reference_table.h @@ -285,6 +285,13 @@ class IndirectReferenceTable { return segment_state_.top_index; } + // Ensure that at least free_capacity elements are available, or return false. + bool EnsureFreeCapacity(size_t free_capacity, std::string* error_msg) + REQUIRES_SHARED(Locks::mutator_lock_); + // See implementation of EnsureFreeCapacity. We'll only state here how much is trivially free, + // without recovering holes. Thus this is a conservative estimate. + size_t FreeCapacity() REQUIRES_SHARED(Locks::mutator_lock_); + // Note IrtIterator does not have a read barrier as it's used to visit roots. IrtIterator begin() { return IrtIterator(table_, 0, Capacity()); diff --git a/runtime/instrumentation_test.cc b/runtime/instrumentation_test.cc index 7f9f04f435..9926ee7386 100644 --- a/runtime/instrumentation_test.cc +++ b/runtime/instrumentation_test.cc @@ -27,7 +27,7 @@ #include "runtime.h" #include "scoped_thread_state_change-inl.h" #include "thread_list.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace instrumentation { diff --git a/runtime/interpreter/interpreter.cc b/runtime/interpreter/interpreter.cc index d2f5232de1..4bc0f2fa12 100644 --- a/runtime/interpreter/interpreter.cc +++ b/runtime/interpreter/interpreter.cc @@ -22,15 +22,16 @@ #include "interpreter_common.h" #include "interpreter_mterp_impl.h" #include "interpreter_switch_impl.h" +#include "jit/jit.h" +#include "jit/jit_code_cache.h" #include "jvalue-inl.h" #include "mirror/string-inl.h" +#include "mterp/mterp.h" #include "scoped_thread_state_change-inl.h" #include "ScopedLocalRef.h" #include "stack.h" +#include "thread-inl.h" #include "unstarted_runtime.h" -#include "mterp/mterp.h" -#include "jit/jit.h" -#include "jit/jit_code_cache.h" namespace art { namespace interpreter { diff --git a/runtime/interpreter/interpreter_common.cc b/runtime/interpreter/interpreter_common.cc index 084cb4218f..d06ac23d3c 100644 --- a/runtime/interpreter/interpreter_common.cc +++ b/runtime/interpreter/interpreter_common.cc @@ -32,6 +32,7 @@ #include "reflection.h" #include "reflection-inl.h" #include "stack.h" +#include "thread-inl.h" #include "well_known_classes.h" namespace art { diff --git a/runtime/interpreter/unstarted_runtime.cc b/runtime/interpreter/unstarted_runtime.cc index 96934bc0ca..152cce4c60 100644 --- a/runtime/interpreter/unstarted_runtime.cc +++ b/runtime/interpreter/unstarted_runtime.cc @@ -50,7 +50,7 @@ #include "mirror/string-inl.h" #include "nth_caller_visitor.h" #include "reflection.h" -#include "thread.h" +#include "thread-inl.h" #include "transaction.h" #include "well_known_classes.h" #include "zip_archive.h" diff --git a/runtime/java_vm_ext.cc b/runtime/java_vm_ext.cc index 6d3118e4aa..753684917c 100644 --- a/runtime/java_vm_ext.cc +++ b/runtime/java_vm_ext.cc @@ -22,12 +22,13 @@ #include "art_method-inl.h" #include "base/dumpable.h" -#include "base/mutex.h" +#include "base/mutex-inl.h" #include "base/stl_util.h" #include "base/systrace.h" #include "check_jni.h" #include "dex_file-inl.h" #include "fault_handler.h" +#include "gc_root-inl.h" #include "indirect_reference_table-inl.h" #include "mirror/class-inl.h" #include "mirror/class_loader.h" diff --git a/runtime/jdwp/jdwp_adb.cc b/runtime/jdwp/jdwp_adb.cc index 0aa04c10ca..ede4f9edb7 100644 --- a/runtime/jdwp/jdwp_adb.cc +++ b/runtime/jdwp/jdwp_adb.cc @@ -24,7 +24,7 @@ #include "base/logging.h" #include "jdwp/jdwp_priv.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #ifdef ART_TARGET_ANDROID #include "cutils/sockets.h" diff --git a/runtime/jdwp/jdwp_handler.cc b/runtime/jdwp/jdwp_handler.cc index e8a9904dc6..618332b7ef 100644 --- a/runtime/jdwp/jdwp_handler.cc +++ b/runtime/jdwp/jdwp_handler.cc @@ -33,7 +33,7 @@ #include "jdwp/jdwp_priv.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "utils.h" namespace art { diff --git a/runtime/jit/debugger_interface.cc b/runtime/jit/debugger_interface.cc index ae0004426d..135d9b1f51 100644 --- a/runtime/jit/debugger_interface.cc +++ b/runtime/jit/debugger_interface.cc @@ -18,7 +18,7 @@ #include "base/logging.h" #include "base/mutex.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread.h" #include <unordered_map> diff --git a/runtime/jit/jit.cc b/runtime/jit/jit.cc index ae474da7c0..969a5708c4 100644 --- a/runtime/jit/jit.cc +++ b/runtime/jit/jit.cc @@ -32,7 +32,9 @@ #include "profile_saver.h" #include "runtime.h" #include "runtime_options.h" +#include "stack.h" #include "stack_map.h" +#include "thread-inl.h" #include "thread_list.h" #include "utils.h" diff --git a/runtime/jit/jit.h b/runtime/jit/jit.h index 4f5bebfbf9..75f9b0ac76 100644 --- a/runtime/jit/jit.h +++ b/runtime/jit/jit.h @@ -17,7 +17,6 @@ #ifndef ART_RUNTIME_JIT_JIT_H_ #define ART_RUNTIME_JIT_JIT_H_ -#include "base/arena_allocator.h" #include "base/histogram-inl.h" #include "base/macros.h" #include "base/mutex.h" diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc index 42d7653b9b..388a51751e 100644 --- a/runtime/jit/jit_code_cache.cc +++ b/runtime/jit/jit_code_cache.cc @@ -34,7 +34,9 @@ #include "linear_alloc.h" #include "mem_map.h" #include "oat_file-inl.h" +#include "oat_quick_method_header.h" #include "scoped_thread_state_change-inl.h" +#include "stack.h" #include "thread_list.h" namespace art { @@ -526,6 +528,15 @@ void JitCodeCache::CopyInlineCacheInto(const InlineCache& ic, } } +static void ClearMethodCounter(ArtMethod* method, bool was_warm) { + if (was_warm) { + method->AddAccessFlags(kAccPreviouslyWarm); + } + // We reset the counter to 1 so that the profile knows that the method was executed at least once. + // This is required for layout purposes. + method->SetCounter(1); +} + uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, ArtMethod* method, uint8_t* stack_map, @@ -600,7 +611,7 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, // Simply discard the compiled code. Clear the counter so that it may be recompiled later. // Hopefully the class hierarchy will be more stable when compilation is retried. single_impl_still_valid = false; - method->SetCounter(1); + ClearMethodCounter(method, /*was_warm*/ false); break; } } @@ -1068,9 +1079,8 @@ void JitCodeCache::DoCollection(Thread* self, bool collect_profiling_info) { if (info->GetSavedEntryPoint() != nullptr) { info->SetSavedEntryPoint(nullptr); // We are going to move this method back to interpreter. Clear the counter now to - // give it a chance to be hot again, but set it to 1 so that this method can still be - // considered a startup method in case it's not executed again. - info->GetMethod()->SetCounter(1); + // give it a chance to be hot again. + ClearMethodCounter(info->GetMethod(), /*was_warm*/ true); } } } else if (kIsDebugBuild) { @@ -1379,7 +1389,7 @@ bool JitCodeCache::NotifyCompilationOf(ArtMethod* method, Thread* self, bool osr VLOG(jit) << method->PrettyMethod() << " needs a ProfilingInfo to be compiled"; // Because the counter is not atomic, there are some rare cases where we may not hit the // threshold for creating the ProfilingInfo. Reset the counter now to "correct" this. - method->SetCounter(1); + ClearMethodCounter(method, /*was_warm*/ false); return false; } @@ -1432,11 +1442,10 @@ void JitCodeCache::InvalidateCompiledCodeFor(ArtMethod* method, if (method->GetEntryPointFromQuickCompiledCode() == header->GetEntryPoint()) { // The entrypoint is the one to invalidate, so we just update it to the interpreter entry point - // and clear the counter to get the method Jitted again. We reset the counter to 1 to preserve - // it as a potential startup method. + // and clear the counter to get the method Jitted again. Runtime::Current()->GetInstrumentation()->UpdateMethodsCode( method, GetQuickToInterpreterBridge()); - method->SetCounter(1); + ClearMethodCounter(method, /*was_warm*/ profiling_info != nullptr); } else { MutexLock mu(Thread::Current(), lock_); auto it = osr_code_map_.find(method); diff --git a/runtime/jit/jit_code_cache.h b/runtime/jit/jit_code_cache.h index 612d06ba1c..eea2771500 100644 --- a/runtime/jit/jit_code_cache.h +++ b/runtime/jit/jit_code_cache.h @@ -39,6 +39,7 @@ namespace art { class ArtMethod; class LinearAlloc; class InlineCache; +class OatQuickMethodHeader; class ProfilingInfo; namespace jit { diff --git a/runtime/jit/profile_compilation_info.cc b/runtime/jit/profile_compilation_info.cc index 1e720c0cf4..9c039e2270 100644 --- a/runtime/jit/profile_compilation_info.cc +++ b/runtime/jit/profile_compilation_info.cc @@ -49,7 +49,7 @@ namespace art { const uint8_t ProfileCompilationInfo::kProfileMagic[] = { 'p', 'r', 'o', '\0' }; // Last profile version: Instead of method index, put the difference with the last // method's index. -const uint8_t ProfileCompilationInfo::kProfileVersion[] = { '0', '0', '7', '\0' }; +const uint8_t ProfileCompilationInfo::kProfileVersion[] = { '0', '0', '8', '\0' }; static constexpr uint16_t kMaxDexFileKeyLength = PATH_MAX; @@ -132,6 +132,33 @@ std::string ProfileCompilationInfo::GetProfileDexFileKey(const std::string& dex_ } } +bool ProfileCompilationInfo::AddSampledMethod(bool startup, + const std::string& dex_location, + uint32_t checksum, + uint16_t method_idx, + uint32_t num_method_ids) { + DexFileData* data = GetOrAddDexFileData(GetProfileDexFileKey(dex_location), + checksum, + num_method_ids); + if (data == nullptr) { + return false; + } + data->AddSampledMethod(startup, method_idx); + return true; +} + +bool ProfileCompilationInfo::AddSampledMethods(bool startup, + std::vector<MethodReference>& methods) { + for (const MethodReference& ref : methods) { + DexFileData* data = GetOrAddDexFileData(ref.dex_file); + if (data == nullptr) { + return false; + } + data->AddSampledMethod(startup, ref.dex_method_index); + } + return true; +} + bool ProfileCompilationInfo::AddMethodsAndClasses( const std::vector<ProfileMethodInfo>& methods, const std::set<DexCacheResolvedClasses>& resolved_classes) { @@ -252,7 +279,7 @@ static void AddUintToBuffer(std::vector<uint8_t>* buffer, T value) { static constexpr size_t kLineHeaderSize = 2 * sizeof(uint16_t) + // class_set.size + dex_location.size - 2 * sizeof(uint32_t); // method_map.size + checksum + 3 * sizeof(uint32_t); // method_map.size + checksum + num_method_ids /** * Serialization format: @@ -297,7 +324,8 @@ bool ProfileCompilationInfo::Save(int fd) { required_capacity += kLineHeaderSize + dex_data.profile_key.size() + sizeof(uint16_t) * dex_data.class_set.size() + - methods_region_size; + methods_region_size + + dex_data.bitmap_storage.size(); } if (required_capacity > kProfileSizeErrorThresholdInBytes) { LOG(ERROR) << "Profile data size exceeds " @@ -335,10 +363,12 @@ bool ProfileCompilationInfo::Save(int fd) { DCHECK_LE(dex_data.profile_key.size(), std::numeric_limits<uint16_t>::max()); DCHECK_LE(dex_data.class_set.size(), std::numeric_limits<uint16_t>::max()); + // Write profile line header. AddUintToBuffer(&buffer, static_cast<uint16_t>(dex_data.profile_key.size())); AddUintToBuffer(&buffer, static_cast<uint16_t>(dex_data.class_set.size())); AddUintToBuffer(&buffer, methods_region_size); // uint32_t AddUintToBuffer(&buffer, dex_data.checksum); // uint32_t + AddUintToBuffer(&buffer, dex_data.num_method_ids); // uint32_t AddStringToBuffer(&buffer, dex_data.profile_key); @@ -362,6 +392,10 @@ bool ProfileCompilationInfo::Save(int fd) { last_class_index = class_id.index_; AddUintToBuffer(&buffer, diff_with_last_class_index); } + + buffer.insert(buffer.end(), + dex_data.bitmap_storage.begin(), + dex_data.bitmap_storage.end()); } uint32_t output_size = 0; @@ -476,7 +510,8 @@ void ProfileCompilationInfo::GroupClassesByDex( ProfileCompilationInfo::DexFileData* ProfileCompilationInfo::GetOrAddDexFileData( const std::string& profile_key, - uint32_t checksum) { + uint32_t checksum, + uint32_t num_method_ids) { const auto profile_index_it = profile_key_map_.FindOrAdd(profile_key, profile_key_map_.size()); if (profile_key_map_.size() > std::numeric_limits<uint8_t>::max()) { // Allow only 255 dex files to be profiled. This allows us to save bytes @@ -492,7 +527,11 @@ ProfileCompilationInfo::DexFileData* ProfileCompilationInfo::GetOrAddDexFileData if (info_.size() <= profile_index) { // This is a new addition. Add it to the info_ array. DexFileData* dex_file_data = new (&arena_) DexFileData( - &arena_, profile_key, checksum, profile_index); + &arena_, + profile_key, + checksum, + profile_index, + num_method_ids); info_.push_back(dex_file_data); } DexFileData* result = info_[profile_index]; @@ -500,6 +539,7 @@ ProfileCompilationInfo::DexFileData* ProfileCompilationInfo::GetOrAddDexFileData // This should always be the case since since the cache map is managed by ProfileCompilationInfo. DCHECK_EQ(profile_key, result->profile_key); DCHECK_EQ(profile_index, result->profile_index); + DCHECK_EQ(num_method_ids, result->num_method_ids); // Check that the checksum matches. // This may different if for example the dex file was updated and @@ -528,7 +568,7 @@ const ProfileCompilationInfo::DexFileData* ProfileCompilationInfo::FindDexData( bool ProfileCompilationInfo::AddResolvedClasses(const DexCacheResolvedClasses& classes) { const std::string dex_location = GetProfileDexFileKey(classes.GetDexLocation()); const uint32_t checksum = classes.GetLocationChecksum(); - DexFileData* const data = GetOrAddDexFileData(dex_location, checksum); + DexFileData* const data = GetOrAddDexFileData(dex_location, checksum, classes.NumMethodIds()); if (data == nullptr) { return false; } @@ -538,15 +578,23 @@ bool ProfileCompilationInfo::AddResolvedClasses(const DexCacheResolvedClasses& c bool ProfileCompilationInfo::AddMethodIndex(const std::string& dex_location, uint32_t dex_checksum, - uint16_t method_index) { - return AddMethod(dex_location, dex_checksum, method_index, OfflineProfileMethodInfo(nullptr)); + uint16_t method_index, + uint32_t num_method_ids) { + return AddMethod(dex_location, + dex_checksum, + method_index, + num_method_ids, + OfflineProfileMethodInfo(nullptr)); } bool ProfileCompilationInfo::AddMethod(const std::string& dex_location, uint32_t dex_checksum, uint16_t method_index, + uint32_t num_method_ids, const OfflineProfileMethodInfo& pmi) { - DexFileData* const data = GetOrAddDexFileData(GetProfileDexFileKey(dex_location), dex_checksum); + DexFileData* const data = GetOrAddDexFileData(GetProfileDexFileKey(dex_location), + dex_checksum, + num_method_ids); if (data == nullptr) { // checksum mismatch return false; } @@ -579,7 +627,8 @@ bool ProfileCompilationInfo::AddMethod(const std::string& dex_location, const DexReference& dex_ref = pmi.dex_references[class_ref.dex_profile_index]; DexFileData* class_dex_data = GetOrAddDexFileData( GetProfileDexFileKey(dex_ref.dex_location), - dex_ref.dex_checksum); + dex_ref.dex_checksum, + dex_ref.num_method_ids); if (class_dex_data == nullptr) { // checksum mismatch return false; } @@ -590,9 +639,7 @@ bool ProfileCompilationInfo::AddMethod(const std::string& dex_location, } bool ProfileCompilationInfo::AddMethod(const ProfileMethodInfo& pmi) { - DexFileData* const data = GetOrAddDexFileData( - GetProfileDexFileKey(pmi.dex_file->GetLocation()), - pmi.dex_file->GetLocationChecksum()); + DexFileData* const data = GetOrAddDexFileData(pmi.dex_file); if (data == nullptr) { // checksum mismatch return false; } @@ -604,9 +651,7 @@ bool ProfileCompilationInfo::AddMethod(const ProfileMethodInfo& pmi) { continue; } for (const TypeReference& class_ref : cache.classes) { - DexFileData* class_dex_data = GetOrAddDexFileData( - GetProfileDexFileKey(class_ref.dex_file->GetLocation()), - class_ref.dex_file->GetLocationChecksum()); + DexFileData* class_dex_data = GetOrAddDexFileData(class_ref.dex_file); if (class_dex_data == nullptr) { // checksum mismatch return false; } @@ -623,8 +668,9 @@ bool ProfileCompilationInfo::AddMethod(const ProfileMethodInfo& pmi) { bool ProfileCompilationInfo::AddClassIndex(const std::string& dex_location, uint32_t checksum, - dex::TypeIndex type_idx) { - DexFileData* const data = GetOrAddDexFileData(dex_location, checksum); + dex::TypeIndex type_idx, + uint32_t num_method_ids) { + DexFileData* const data = GetOrAddDexFileData(dex_location, checksum, num_method_ids); if (data == nullptr) { return false; } @@ -694,7 +740,9 @@ bool ProfileCompilationInfo::ReadMethods(SafeBuffer& buffer, - line_header.method_region_size_bytes; uint16_t last_method_index = 0; while (buffer.CountUnreadBytes() > expected_unread_bytes_after_operation) { - DexFileData* const data = GetOrAddDexFileData(line_header.dex_location, line_header.checksum); + DexFileData* const data = GetOrAddDexFileData(line_header.dex_location, + line_header.checksum, + line_header.num_method_ids); uint16_t diff_with_last_method_index; READ_UINT(uint16_t, buffer, diff_with_last_method_index, error); uint16_t method_index = last_method_index + diff_with_last_method_index; @@ -729,7 +777,8 @@ bool ProfileCompilationInfo::ReadClasses(SafeBuffer& buffer, last_class_index = type_index; if (!AddClassIndex(line_header.dex_location, line_header.checksum, - dex::TypeIndex(type_index))) { + dex::TypeIndex(type_index), + line_header.num_method_ids)) { return false; } } @@ -863,6 +912,7 @@ bool ProfileCompilationInfo::ReadProfileLineHeaderElements(SafeBuffer& buffer, READ_UINT(uint16_t, buffer, line_header->class_set_size, error); READ_UINT(uint32_t, buffer, line_header->method_region_size_bytes, error); READ_UINT(uint32_t, buffer, line_header->checksum, error); + READ_UINT(uint32_t, buffer, line_header->num_method_ids, error); return true; } @@ -902,7 +952,10 @@ ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::ReadProfileLine uint8_t number_of_dex_files, const ProfileLineHeader& line_header, /*out*/std::string* error) { - if (GetOrAddDexFileData(line_header.dex_location, line_header.checksum) == nullptr) { + DexFileData* data = GetOrAddDexFileData(line_header.dex_location, + line_header.checksum, + line_header.num_method_ids); + if (data == nullptr) { *error = "Error when reading profile file line header: checksum mismatch for " + line_header.dex_location; return kProfileLoadBadData; @@ -915,6 +968,16 @@ ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::ReadProfileLine if (!ReadClasses(buffer, line_header, error)) { return kProfileLoadBadData; } + + const size_t bytes = data->bitmap_storage.size(); + if (buffer.CountUnreadBytes() < bytes) { + *error += "Profile EOF reached prematurely for ReadProfileHeaderDexLocation"; + return kProfileLoadBadData; + } + const uint8_t* base_ptr = buffer.GetCurrentPtr(); + std::copy_n(base_ptr, bytes, &data->bitmap_storage[0]); + buffer.Advance(bytes); + // Read method bitmap. return kProfileLoadSuccess; } @@ -932,6 +995,15 @@ bool ProfileCompilationInfo::Load(int fd) { } } +void ProfileCompilationInfo::DexFileData::CreateBitmap() { + const size_t num_bits = num_method_ids * kMethodBitCount; + bitmap_storage.resize(RoundUp(num_bits, kBitsPerByte) / kBitsPerByte); + if (!bitmap_storage.empty()) { + method_bitmap = + BitMemoryRegion(MemoryRegion(&bitmap_storage[0], bitmap_storage.size()), 0, num_bits); + } +} + // TODO(calin): fail fast if the dex checksums don't match. ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::LoadInternal( int fd, std::string* error) { @@ -1110,7 +1182,8 @@ bool ProfileCompilationInfo::MergeWith(const ProfileCompilationInfo& other) { SafeMap<uint8_t, uint8_t> dex_profile_index_remap; for (const DexFileData* other_dex_data : other.info_) { const DexFileData* dex_data = GetOrAddDexFileData(other_dex_data->profile_key, - other_dex_data->checksum); + other_dex_data->checksum, + other_dex_data->num_method_ids); if (dex_data == nullptr) { return false; // Could happen if we exceed the number of allowed dex files. } @@ -1147,6 +1220,9 @@ bool ProfileCompilationInfo::MergeWith(const ProfileCompilationInfo& other) { } } } + + // Merge the bitmaps. + dex_data->MergeBitmap(*other_dex_data); } return true; } @@ -1159,6 +1235,27 @@ static bool ChecksumMatch(const DexFile& dex_file, uint32_t checksum) { return ChecksumMatch(dex_file.GetLocationChecksum(), checksum); } +bool ProfileCompilationInfo::IsStartupOrHotMethod(const MethodReference& method_ref) const { + return IsStartupOrHotMethod(method_ref.dex_file->GetLocation(), + method_ref.dex_file->GetLocationChecksum(), + method_ref.dex_method_index); +} + +bool ProfileCompilationInfo::IsStartupOrHotMethod(const std::string& dex_location, + uint32_t dex_checksum, + uint16_t dex_method_index) const { + const DexFileData* dex_data = FindDexData(GetProfileDexFileKey(dex_location)); + if (dex_data == nullptr || !ChecksumMatch(dex_checksum, dex_data->checksum)) { + return false; + } + if (dex_data->HasSampledMethod(/*startup*/ true, dex_method_index)) { + return true; + } + const MethodMap& methods = dex_data->method_map; + const auto method_it = methods.find(dex_method_index); + return method_it != methods.end(); +} + bool ProfileCompilationInfo::ContainsMethod(const MethodReference& method_ref) const { return FindMethod(method_ref.dex_file->GetLocation(), method_ref.dex_file->GetLocationChecksum(), @@ -1196,6 +1293,7 @@ std::unique_ptr<ProfileCompilationInfo::OfflineProfileMethodInfo> ProfileCompila for (const DexFileData* dex_data : info_) { pmi->dex_references[dex_data->profile_index].dex_location = dex_data->profile_key; pmi->dex_references[dex_data->profile_index].dex_checksum = dex_data->checksum; + pmi->dex_references[dex_data->profile_index].num_method_ids = dex_data->num_method_ids; } return pmi; @@ -1314,9 +1412,12 @@ std::string ProfileCompilationInfo::DumpInfo(const std::vector<const DexFile*>* return os.str(); } -bool ProfileCompilationInfo::GetClassesAndMethods(const DexFile& dex_file, - std::set<dex::TypeIndex>* class_set, - std::set<uint16_t>* method_set) const { +bool ProfileCompilationInfo::GetClassesAndMethods( + const DexFile& dex_file, + /*out*/std::set<dex::TypeIndex>* class_set, + /*out*/std::set<uint16_t>* hot_method_set, + /*out*/std::set<uint16_t>* startup_method_set, + /*out*/std::set<uint16_t>* post_startup_method_method_set) const { std::set<std::string> ret; std::string profile_key = GetProfileDexFileKey(dex_file.GetLocation()); const DexFileData* dex_data = FindDexData(profile_key); @@ -1324,7 +1425,15 @@ bool ProfileCompilationInfo::GetClassesAndMethods(const DexFile& dex_file, return false; } for (const auto& it : dex_data->method_map) { - method_set->insert(it.first); + hot_method_set->insert(it.first); + } + for (uint32_t method_idx = 0; method_idx < dex_data->num_method_ids; ++method_idx) { + if (dex_data->HasSampledMethod(/*startup*/ true, method_idx)) { + startup_method_set->insert(method_idx); + } + if (dex_data->HasSampledMethod(/*startup*/ false, method_idx)) { + post_startup_method_method_set->insert(method_idx); + } } for (const dex::TypeIndex& type_index : dex_data->class_set) { class_set->insert(type_index); @@ -1349,16 +1458,27 @@ bool ProfileCompilationInfo::Equals(const ProfileCompilationInfo& other) { } std::set<DexCacheResolvedClasses> ProfileCompilationInfo::GetResolvedClasses( - const std::unordered_set<std::string>& dex_files_locations) const { - std::unordered_map<std::string, std::string> key_to_location_map; - for (const std::string& location : dex_files_locations) { - key_to_location_map.emplace(GetProfileDexFileKey(location), location); + const std::vector<const DexFile*>& dex_files) const { + std::unordered_map<std::string, const DexFile* > key_to_dex_file; + for (const DexFile* dex_file : dex_files) { + key_to_dex_file.emplace(GetProfileDexFileKey(dex_file->GetLocation()), dex_file); } std::set<DexCacheResolvedClasses> ret; for (const DexFileData* dex_data : info_) { - const auto it = key_to_location_map.find(dex_data->profile_key); - if (it != key_to_location_map.end()) { - DexCacheResolvedClasses classes(it->second, it->second, dex_data->checksum); + const auto it = key_to_dex_file.find(dex_data->profile_key); + if (it != key_to_dex_file.end()) { + const DexFile* dex_file = it->second; + const std::string& dex_location = dex_file->GetLocation(); + if (dex_data->checksum != it->second->GetLocationChecksum()) { + LOG(ERROR) << "Dex checksum mismatch when getting resolved classes from profile for " + << "location " << dex_location << " (checksum=" << dex_file->GetLocationChecksum() + << ", profile checksum=" << dex_data->checksum; + return std::set<DexCacheResolvedClasses>(); + } + DexCacheResolvedClasses classes(dex_location, + dex_location, + dex_data->checksum, + dex_data->num_method_ids); classes.AddClasses(dex_data->class_set.begin(), dex_data->class_set.end()); ret.insert(classes); } @@ -1375,8 +1495,8 @@ bool ProfileCompilationInfo::GenerateTestProfile(int fd, const std::string base_dex_location = "base.apk"; ProfileCompilationInfo info; // The limits are defined by the dex specification. - uint16_t max_method = std::numeric_limits<uint16_t>::max(); - uint16_t max_classes = std::numeric_limits<uint16_t>::max(); + const uint16_t max_method = std::numeric_limits<uint16_t>::max(); + const uint16_t max_classes = std::numeric_limits<uint16_t>::max(); uint16_t number_of_methods = max_method * method_ratio / 100; uint16_t number_of_classes = max_classes * class_ratio / 100; @@ -1396,7 +1516,7 @@ bool ProfileCompilationInfo::GenerateTestProfile(int fd, if (m < (number_of_methods / kFavorSplit)) { method_idx %= kFavorFirstN; } - info.AddMethodIndex(profile_key, 0, method_idx); + info.AddMethodIndex(profile_key, 0, method_idx, max_method); } for (uint16_t c = 0; c < number_of_classes; c++) { @@ -1404,7 +1524,7 @@ bool ProfileCompilationInfo::GenerateTestProfile(int fd, if (c < (number_of_classes / kFavorSplit)) { type_idx %= kFavorFirstN; } - info.AddClassIndex(profile_key, 0, dex::TypeIndex(type_idx)); + info.AddClassIndex(profile_key, 0, dex::TypeIndex(type_idx), max_method); } } return info.Save(fd); @@ -1423,13 +1543,16 @@ bool ProfileCompilationInfo::GenerateTestProfile( for (uint32_t i = 0; i < dex_file->NumClassDefs(); ++i) { // Randomly add a class from the dex file (with 50% chance). if (std::rand() % 2 != 0) { - info.AddClassIndex(location, checksum, dex::TypeIndex(dex_file->GetClassDef(i).class_idx_)); + info.AddClassIndex(location, + checksum, + dex::TypeIndex(dex_file->GetClassDef(i).class_idx_), + dex_file->NumMethodIds()); } } for (uint32_t i = 0; i < dex_file->NumMethodIds(); ++i) { // Randomly add a method from the dex file (with 50% chance). if (std::rand() % 2 != 0) { - info.AddMethodIndex(location, checksum, i); + info.AddMethodIndex(location, checksum, i, dex_file->NumMethodIds()); } } } diff --git a/runtime/jit/profile_compilation_info.h b/runtime/jit/profile_compilation_info.h index a9a5b13b75..2b89a41dd8 100644 --- a/runtime/jit/profile_compilation_info.h +++ b/runtime/jit/profile_compilation_info.h @@ -23,6 +23,7 @@ #include "atomic.h" #include "base/arena_object.h" #include "base/arena_containers.h" +#include "bit_memory_region.h" #include "dex_cache_resolved_classes.h" #include "dex_file.h" #include "dex_file_types.h" @@ -54,7 +55,9 @@ struct ProfileMethodInfo { ProfileMethodInfo(const DexFile* dex, uint32_t method_index, const std::vector<ProfileInlineCache>& caches) - : dex_file(dex), dex_method_index(method_index), inline_caches(caches) {} + : dex_file(dex), + dex_method_index(method_index), + inline_caches(caches) {} const DexFile* dex_file; const uint32_t dex_method_index; @@ -79,13 +82,15 @@ class ProfileCompilationInfo { // A dex location together with its checksum. struct DexReference { - DexReference() : dex_checksum(0) {} + DexReference() : dex_checksum(0), num_method_ids(0) {} - DexReference(const std::string& location, uint32_t checksum) - : dex_location(location), dex_checksum(checksum) {} + DexReference(const std::string& location, uint32_t checksum, uint32_t num_methods) + : dex_location(location), dex_checksum(checksum), num_method_ids(num_methods) {} bool operator==(const DexReference& other) const { - return dex_checksum == other.dex_checksum && dex_location == other.dex_location; + return dex_checksum == other.dex_checksum && + dex_location == other.dex_location && + num_method_ids == other.num_method_ids; } bool MatchesDex(const DexFile* dex_file) const { @@ -95,6 +100,7 @@ class ProfileCompilationInfo { std::string dex_location; uint32_t dex_checksum; + uint32_t num_method_ids; }; // Encodes a class reference in the profile. @@ -191,6 +197,24 @@ class ProfileCompilationInfo { bool AddMethodsAndClasses(const std::vector<ProfileMethodInfo>& methods, const std::set<DexCacheResolvedClasses>& resolved_classes); + // Add a method index to the profile (without inline caches). + bool AddMethodIndex(const std::string& dex_location, + uint32_t checksum, + uint16_t method_idx, + uint32_t num_method_ids); + + // Add a method to the profile using its online representation (containing runtime structures). + bool AddMethod(const ProfileMethodInfo& pmi); + + // Add methods that have samples but are are not necessarily hot. These are partitioned into two + // possibly interesecting sets startup and post startup. + bool AddSampledMethods(bool startup, std::vector<MethodReference>& methods); + bool AddSampledMethod(bool startup, + const std::string& dex_location, + uint32_t checksum, + uint16_t method_idx, + uint32_t num_method_ids); + // Load profile information from the given file descriptor. // If the current profile is non-empty the load will fail. bool Load(int fd); @@ -216,6 +240,12 @@ class ProfileCompilationInfo { // Return the number of resolved classes that were profiled. uint32_t GetNumberOfResolvedClasses() const; + // Return true if the method reference is a hot or startup method in the profiling info. + bool IsStartupOrHotMethod(const MethodReference& method_ref) const; + bool IsStartupOrHotMethod(const std::string& dex_location, + uint32_t dex_checksum, + uint16_t dex_method_index) const; + // Return true if the method reference is present in the profiling info. bool ContainsMethod(const MethodReference& method_ref) const; @@ -244,14 +274,16 @@ class ProfileCompilationInfo { // file is register and has a matching checksum, false otherwise. bool GetClassesAndMethods(const DexFile& dex_file, /*out*/std::set<dex::TypeIndex>* class_set, - /*out*/std::set<uint16_t>* method_set) const; + /*out*/std::set<uint16_t>* hot_method_set, + /*out*/std::set<uint16_t>* startup_method_set, + /*out*/std::set<uint16_t>* post_startup_method_method_set) const; // Perform an equality test with the `other` profile information. bool Equals(const ProfileCompilationInfo& other); // Return the class descriptors for all of the classes in the profiles' class sets. std::set<DexCacheResolvedClasses> GetResolvedClasses( - const std::unordered_set<std::string>& dex_files_locations) const; + const std::vector<const DexFile*>& dex_files_) const; // Return the profile key associated with the given dex location. static std::string GetProfileDexFileKey(const std::string& dex_location); @@ -301,13 +333,31 @@ class ProfileCompilationInfo { DexFileData(ArenaAllocator* arena, const std::string& key, uint32_t location_checksum, - uint16_t index) + uint16_t index, + uint32_t num_methods) : arena_(arena), profile_key(key), profile_index(index), checksum(location_checksum), method_map(std::less<uint16_t>(), arena->Adapter(kArenaAllocProfile)), - class_set(std::less<dex::TypeIndex>(), arena->Adapter(kArenaAllocProfile)) {} + class_set(std::less<dex::TypeIndex>(), arena->Adapter(kArenaAllocProfile)), + num_method_ids(num_methods), + bitmap_storage(arena->Adapter(kArenaAllocProfile)) { + CreateBitmap(); + } + + bool operator==(const DexFileData& other) const { + return checksum == other.checksum && method_map == other.method_map; + } + + // Mark a method as executed at least once. + void AddSampledMethod(bool startup, size_t index) { + method_bitmap.StoreBit(MethodBitIndex(startup, index), true); + } + + bool HasSampledMethod(bool startup, size_t index) const { + return method_bitmap.LoadBit(MethodBitIndex(startup, index)); + } // The arena used to allocate new inline cache maps. ArenaAllocator* arena_; @@ -322,32 +372,64 @@ class ProfileCompilationInfo { // The classes which have been profiled. Note that these don't necessarily include // all the classes that can be found in the inline caches reference. ArenaSet<dex::TypeIndex> class_set; - - bool operator==(const DexFileData& other) const { - return checksum == other.checksum && method_map == other.method_map; - } - // Find the inline caches of the the given method index. Add an empty entry if // no previous data is found. InlineCacheMap* FindOrAddMethod(uint16_t method_index); + // Num method ids. + uint32_t num_method_ids; + ArenaVector<uint8_t> bitmap_storage; + BitMemoryRegion method_bitmap; + + void CreateBitmap(); + + void MergeBitmap(const DexFileData& other) { + DCHECK_EQ(bitmap_storage.size(), other.bitmap_storage.size()); + for (size_t i = 0; i < bitmap_storage.size(); ++i) { + bitmap_storage[i] |= other.bitmap_storage[i]; + } + } + + private: + enum Bits { + kMethodBitStartup, + kMethodBitAfterStartup, + kMethodBitCount, + }; + + size_t MethodBitIndex(bool startup, size_t index) const { + DCHECK_LT(index, num_method_ids); + if (!startup) { + index += num_method_ids; + } + return index; + } }; // Return the profile data for the given profile key or null if the dex location // already exists but has a different checksum - DexFileData* GetOrAddDexFileData(const std::string& profile_key, uint32_t checksum); + DexFileData* GetOrAddDexFileData(const std::string& profile_key, + uint32_t checksum, + uint32_t num_method_ids); - // Add a method to the profile using its online representation (containing runtime structures). - bool AddMethod(const ProfileMethodInfo& pmi); + DexFileData* GetOrAddDexFileData(const DexFile* dex_file) { + return GetOrAddDexFileData(GetProfileDexFileKey(dex_file->GetLocation()), + dex_file->GetLocationChecksum(), + dex_file->NumMethodIds()); + } // Add a method to the profile using its offline representation. // This is mostly used to facilitate testing. bool AddMethod(const std::string& dex_location, uint32_t dex_checksum, uint16_t method_index, + uint32_t num_method_ids, const OfflineProfileMethodInfo& pmi); // Add a class index to the profile. - bool AddClassIndex(const std::string& dex_location, uint32_t checksum, dex::TypeIndex type_idx); + bool AddClassIndex(const std::string& dex_location, + uint32_t checksum, + dex::TypeIndex type_idx, + uint32_t num_method_ids); // Add all classes from the given dex cache to the the profile. bool AddResolvedClasses(const DexCacheResolvedClasses& classes); @@ -392,6 +474,7 @@ class ProfileCompilationInfo { uint16_t class_set_size; uint32_t method_region_size_bytes; uint32_t checksum; + uint32_t num_method_ids; }; // A helper structure to make sure we don't read past our buffers in the loops. diff --git a/runtime/jit/profile_compilation_info_test.cc b/runtime/jit/profile_compilation_info_test.cc index 1cfa3552b9..615149feb3 100644 --- a/runtime/jit/profile_compilation_info_test.cc +++ b/runtime/jit/profile_compilation_info_test.cc @@ -32,6 +32,8 @@ namespace art { +static constexpr size_t kMaxMethodIds = 65535; + class ProfileCompilationInfoTest : public CommonRuntimeTest { public: void PostRuntimeCreate() OVERRIDE { @@ -61,7 +63,7 @@ class ProfileCompilationInfoTest : public CommonRuntimeTest { uint32_t checksum, uint16_t method_index, ProfileCompilationInfo* info) { - return info->AddMethodIndex(dex_location, checksum, method_index); + return info->AddMethodIndex(dex_location, checksum, method_index, kMaxMethodIds); } bool AddMethod(const std::string& dex_location, @@ -69,14 +71,14 @@ class ProfileCompilationInfoTest : public CommonRuntimeTest { uint16_t method_index, const ProfileCompilationInfo::OfflineProfileMethodInfo& pmi, ProfileCompilationInfo* info) { - return info->AddMethod(dex_location, checksum, method_index, pmi); + return info->AddMethod(dex_location, checksum, method_index, kMaxMethodIds, pmi); } bool AddClass(const std::string& dex_location, uint32_t checksum, uint16_t class_index, ProfileCompilationInfo* info) { - return info->AddMethodIndex(dex_location, checksum, class_index); + return info->AddMethodIndex(dex_location, checksum, class_index, kMaxMethodIds); } uint32_t GetFd(const ScratchFile& file) { @@ -149,7 +151,9 @@ class ProfileCompilationInfoTest : public CommonRuntimeTest { std::vector<TypeReference> classes; caches.emplace_back(dex_pc, /*is_missing_types*/true, classes); } - ProfileMethodInfo pmi(method->GetDexFile(), method->GetDexMethodIndex(), caches); + ProfileMethodInfo pmi(method->GetDexFile(), + method->GetDexMethodIndex(), + caches); profile_methods.push_back(pmi); profile_methods_map->Put(method, pmi); } @@ -191,7 +195,8 @@ class ProfileCompilationInfoTest : public CommonRuntimeTest { const std::string& dex_key = ProfileCompilationInfo::GetProfileDexFileKey( class_ref.dex_file->GetLocation()); offline_pmi.dex_references.emplace_back(dex_key, - class_ref.dex_file->GetLocationChecksum()); + class_ref.dex_file->GetLocationChecksum(), + class_ref.dex_file->NumMethodIds()); } } } @@ -201,6 +206,7 @@ class ProfileCompilationInfoTest : public CommonRuntimeTest { // Creates an offline profile used for testing inline caches. ProfileCompilationInfo::OfflineProfileMethodInfo GetOfflineProfileMethodInfo() { ProfileCompilationInfo::InlineCacheMap* ic_map = CreateInlineCacheMap(); + // Monomorphic for (uint16_t dex_pc = 0; dex_pc < 11; dex_pc++) { ProfileCompilationInfo::DexPcData dex_pc_data(arena_.get()); @@ -231,9 +237,9 @@ class ProfileCompilationInfoTest : public CommonRuntimeTest { ProfileCompilationInfo::OfflineProfileMethodInfo pmi(ic_map); - pmi.dex_references.emplace_back("dex_location1", /* checksum */1); - pmi.dex_references.emplace_back("dex_location2", /* checksum */2); - pmi.dex_references.emplace_back("dex_location3", /* checksum */3); + pmi.dex_references.emplace_back("dex_location1", /* checksum */1, kMaxMethodIds); + pmi.dex_references.emplace_back("dex_location2", /* checksum */2, kMaxMethodIds); + pmi.dex_references.emplace_back("dex_location3", /* checksum */3, kMaxMethodIds); return pmi; } @@ -694,8 +700,8 @@ TEST_F(ProfileCompilationInfoTest, MergeInlineCacheTriggerReindex) { ProfileCompilationInfo::InlineCacheMap* ic_map = CreateInlineCacheMap(); ProfileCompilationInfo::OfflineProfileMethodInfo pmi(ic_map); - pmi.dex_references.emplace_back("dex_location1", /* checksum */ 1); - pmi.dex_references.emplace_back("dex_location2", /* checksum */ 2); + pmi.dex_references.emplace_back("dex_location1", /* checksum */ 1, kMaxMethodIds); + pmi.dex_references.emplace_back("dex_location2", /* checksum */ 2, kMaxMethodIds); for (uint16_t dex_pc = 1; dex_pc < 5; dex_pc++) { ProfileCompilationInfo::DexPcData dex_pc_data(arena_.get()); dex_pc_data.AddClass(0, dex::TypeIndex(0)); @@ -705,8 +711,8 @@ TEST_F(ProfileCompilationInfoTest, MergeInlineCacheTriggerReindex) { ProfileCompilationInfo::InlineCacheMap* ic_map_reindexed = CreateInlineCacheMap(); ProfileCompilationInfo::OfflineProfileMethodInfo pmi_reindexed(ic_map_reindexed); - pmi_reindexed.dex_references.emplace_back("dex_location2", /* checksum */ 2); - pmi_reindexed.dex_references.emplace_back("dex_location1", /* checksum */ 1); + pmi_reindexed.dex_references.emplace_back("dex_location2", /* checksum */ 2, kMaxMethodIds); + pmi_reindexed.dex_references.emplace_back("dex_location1", /* checksum */ 1, kMaxMethodIds); for (uint16_t dex_pc = 1; dex_pc < 5; dex_pc++) { ProfileCompilationInfo::DexPcData dex_pc_data(arena_.get()); dex_pc_data.AddClass(1, dex::TypeIndex(0)); @@ -761,7 +767,7 @@ TEST_F(ProfileCompilationInfoTest, MegamorphicInlineCachesMerge) { // Create a megamorphic inline cache. ProfileCompilationInfo::InlineCacheMap* ic_map = CreateInlineCacheMap(); ProfileCompilationInfo::OfflineProfileMethodInfo pmi(ic_map); - pmi.dex_references.emplace_back("dex_location1", /* checksum */ 1); + pmi.dex_references.emplace_back("dex_location1", /* checksum */ 1, kMaxMethodIds); ProfileCompilationInfo::DexPcData dex_pc_data(arena_.get()); dex_pc_data.SetIsMegamorphic(); ic_map->Put(/*dex_pc*/ 0, dex_pc_data); @@ -791,7 +797,7 @@ TEST_F(ProfileCompilationInfoTest, MissingTypesInlineCachesMerge) { // Create an inline cache with missing types ProfileCompilationInfo::InlineCacheMap* ic_map = CreateInlineCacheMap(); ProfileCompilationInfo::OfflineProfileMethodInfo pmi(ic_map); - pmi.dex_references.emplace_back("dex_location1", /* checksum */ 1); + pmi.dex_references.emplace_back("dex_location1", /* checksum */ 1, kMaxMethodIds); ProfileCompilationInfo::DexPcData dex_pc_data(arena_.get()); dex_pc_data.SetIsMissingTypes(); ic_map->Put(/*dex_pc*/ 0, dex_pc_data); @@ -839,4 +845,48 @@ TEST_F(ProfileCompilationInfoTest, LoadShouldClearExistingDataFromProfiles) { // This should fail since the test_info already contains data and the load would overwrite it. ASSERT_FALSE(test_info.Load(GetFd(profile))); } + +TEST_F(ProfileCompilationInfoTest, SampledMethodsTest) { + ProfileCompilationInfo test_info; + static constexpr size_t kNumMethods = 1000; + static constexpr size_t kChecksum1 = 1234; + static constexpr size_t kChecksum2 = 4321; + static const std::string kDex1 = "dex1"; + static const std::string kDex2 = "dex2"; + test_info.AddSampledMethod(true, kDex1, kChecksum1, 1, kNumMethods); + test_info.AddSampledMethod(true, kDex1, kChecksum1, 5, kNumMethods); + test_info.AddSampledMethod(false, kDex2, kChecksum2, 1, kNumMethods); + test_info.AddSampledMethod(false, kDex2, kChecksum2, 5, kNumMethods); + auto run_test = [](const ProfileCompilationInfo& info) { + EXPECT_FALSE(info.IsStartupOrHotMethod(kDex1, kChecksum1, 0)); + EXPECT_TRUE(info.IsStartupOrHotMethod(kDex1, kChecksum1, 1)); + EXPECT_FALSE(info.IsStartupOrHotMethod(kDex1, kChecksum1, 3)); + EXPECT_TRUE(info.IsStartupOrHotMethod(kDex1, kChecksum1, 5)); + EXPECT_FALSE(info.IsStartupOrHotMethod(kDex1, kChecksum1, 6)); + EXPECT_FALSE(info.IsStartupOrHotMethod(kDex2, kChecksum2, 5)); + EXPECT_FALSE(info.IsStartupOrHotMethod(kDex2, kChecksum2, 5)); + }; + run_test(test_info); + + // Save the profile. + ScratchFile profile; + ASSERT_TRUE(test_info.Save(GetFd(profile))); + ASSERT_EQ(0, profile.GetFile()->Flush()); + ASSERT_TRUE(profile.GetFile()->ResetOffset()); + + // Load the profile and make sure we can read the data and it matches what we expect. + ProfileCompilationInfo loaded_info; + ASSERT_TRUE(loaded_info.Load(GetFd(profile))); + run_test(loaded_info); + + // Test that the bitmap gets merged properly. + EXPECT_FALSE(test_info.IsStartupOrHotMethod(kDex1, kChecksum1, 11)); + { + ProfileCompilationInfo merge_info; + merge_info.AddSampledMethod(true, kDex1, kChecksum1, 11, kNumMethods); + test_info.MergeWith(merge_info); + } + EXPECT_TRUE(test_info.IsStartupOrHotMethod(kDex1, kChecksum1, 11)); +} + } // namespace art diff --git a/runtime/jit/profile_saver.cc b/runtime/jit/profile_saver.cc index 166b6f4ba1..c96ca88874 100644 --- a/runtime/jit/profile_saver.cc +++ b/runtime/jit/profile_saver.cc @@ -183,8 +183,11 @@ void ProfileSaver::NotifyJitActivityInternal() { // Excludes native methods and classes in the boot image. class GetMethodsVisitor : public ClassVisitor { public: - GetMethodsVisitor(std::vector<MethodReference>* methods, uint32_t startup_method_samples) - : methods_(methods), + GetMethodsVisitor(std::vector<MethodReference>* hot_methods, + std::vector<MethodReference>* startup_methods, + uint32_t startup_method_samples) + : hot_methods_(hot_methods), + startup_methods_(startup_methods), startup_method_samples_(startup_method_samples) {} virtual bool operator()(ObjPtr<mirror::Class> klass) REQUIRES_SHARED(Locks::mutator_lock_) { @@ -192,21 +195,26 @@ class GetMethodsVisitor : public ClassVisitor { return true; } for (ArtMethod& method : klass->GetMethods(kRuntimePointerSize)) { - if (!method.IsNative()) { - if (method.GetCounter() >= startup_method_samples_ || - method.GetProfilingInfo(kRuntimePointerSize) != nullptr) { - // Have samples, add to profile. - const DexFile* dex_file = - method.GetInterfaceMethodIfProxy(kRuntimePointerSize)->GetDexFile(); - methods_->push_back(MethodReference(dex_file, method.GetDexMethodIndex())); + if (!method.IsNative() && !method.IsProxyMethod()) { + const uint16_t counter = method.GetCounter(); + MethodReference ref(method.GetDexFile(), method.GetDexMethodIndex()); + if (method.GetProfilingInfo(kRuntimePointerSize) != nullptr || + (method.GetAccessFlags() & kAccPreviouslyWarm) != 0) { + hot_methods_->push_back(ref); + startup_methods_->push_back(ref); + } else if (counter >= startup_method_samples_) { + startup_methods_->push_back(ref); } + } else { + CHECK_EQ(method.GetCounter(), 0u); } } return true; } private: - std::vector<MethodReference>* const methods_; + std::vector<MethodReference>* const hot_methods_; + std::vector<MethodReference>* const startup_methods_; uint32_t startup_method_samples_; }; @@ -217,7 +225,8 @@ void ProfileSaver::FetchAndCacheResolvedClassesAndMethods() { ResolveTrackedLocations(); Thread* const self = Thread::Current(); - std::vector<MethodReference> methods; + std::vector<MethodReference> hot_methods; + std::vector<MethodReference> startup_methods; std::set<DexCacheResolvedClasses> resolved_classes; { ScopedObjectAccess soa(self); @@ -230,10 +239,13 @@ void ProfileSaver::FetchAndCacheResolvedClassesAndMethods() { { ScopedTrace trace2("Get hot methods"); - GetMethodsVisitor visitor(&methods, options_.GetStartupMethodSamples()); + GetMethodsVisitor visitor(&hot_methods, + &startup_methods, + options_.GetStartupMethodSamples()); class_linker->VisitClasses(&visitor); - VLOG(profiler) << "Methods with samples greater than " - << options_.GetStartupMethodSamples() << " = " << methods.size(); + VLOG(profiler) << "Profile saver recorded " << hot_methods.size() << " hot methods and " + << startup_methods.size() << " startup methods with threshold " + << options_.GetStartupMethodSamples(); } } MutexLock mu(self, *Locks::profiler_lock_); @@ -244,11 +256,18 @@ void ProfileSaver::FetchAndCacheResolvedClassesAndMethods() { const std::string& filename = it.first; const std::set<std::string>& locations = it.second; std::vector<ProfileMethodInfo> profile_methods_for_location; - for (const MethodReference& ref : methods) { + for (const MethodReference& ref : hot_methods) { if (locations.find(ref.dex_file->GetBaseLocation()) != locations.end()) { profile_methods_for_location.emplace_back(ref.dex_file, ref.dex_method_index); } } + std::vector<MethodReference> startup_methods_for_locations; + for (const MethodReference& ref : startup_methods) { + if (locations.find(ref.dex_file->GetBaseLocation()) != locations.end()) { + startup_methods_for_locations.push_back(ref); + } + } + for (const DexCacheResolvedClasses& classes : resolved_classes) { if (locations.find(classes.GetBaseLocation()) != locations.end()) { VLOG(profiler) << "Added " << classes.GetClasses().size() << " classes for location " @@ -264,8 +283,8 @@ void ProfileSaver::FetchAndCacheResolvedClassesAndMethods() { new ProfileCompilationInfo(Runtime::Current()->GetArenaPool())); ProfileCompilationInfo* cached_info = info_it->second; - cached_info->AddMethodsAndClasses(profile_methods_for_location, - resolved_classes_for_location); + cached_info->AddMethodsAndClasses(profile_methods_for_location, resolved_classes_for_location); + cached_info->AddSampledMethods(/*startup*/ true, startup_methods_for_locations); total_number_of_profile_entries_cached += resolved_classes_for_location.size(); } max_number_of_profile_entries_cached_ = std::max( @@ -316,8 +335,7 @@ bool ProfileSaver::ProcessProfilingInfo(bool force_save, /*out*/uint16_t* number uint64_t last_save_number_of_methods = info.GetNumberOfMethods(); uint64_t last_save_number_of_classes = info.GetNumberOfResolvedClasses(); - info.AddMethodsAndClasses(profile_methods, - std::set<DexCacheResolvedClasses>()); + info.AddMethodsAndClasses(profile_methods, std::set<DexCacheResolvedClasses>()); auto profile_cache_it = profile_cache_.find(filename); if (profile_cache_it != profile_cache_.end()) { info.MergeWith(*(profile_cache_it->second)); diff --git a/runtime/jni_env_ext.cc b/runtime/jni_env_ext.cc index 0148a1c3b0..3ff94f995d 100644 --- a/runtime/jni_env_ext.cc +++ b/runtime/jni_env_ext.cc @@ -28,7 +28,7 @@ #include "lock_word.h" #include "mirror/object-inl.h" #include "nth_caller_visitor.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" namespace art { @@ -123,8 +123,8 @@ void JNIEnvExt::DumpReferenceTables(std::ostream& os) { monitors.Dump(os); } -void JNIEnvExt::PushFrame(int capacity ATTRIBUTE_UNUSED) { - // TODO: take 'capacity' into account. +void JNIEnvExt::PushFrame(int capacity) { + DCHECK_GE(locals.FreeCapacity(), static_cast<size_t>(capacity)); stacked_local_ref_cookies.push_back(local_ref_cookie); local_ref_cookie = locals.GetSegmentState(); } diff --git a/runtime/jni_internal.cc b/runtime/jni_internal.cc index 2d0ea0cc62..6be0953727 100644 --- a/runtime/jni_internal.cc +++ b/runtime/jni_internal.cc @@ -2399,18 +2399,18 @@ class JNI { static jint EnsureLocalCapacityInternal(ScopedObjectAccess& soa, jint desired_capacity, const char* caller) REQUIRES_SHARED(Locks::mutator_lock_) { - // TODO: we should try to expand the table if necessary. - if (desired_capacity < 0 || desired_capacity > static_cast<jint>(kLocalsInitial)) { + if (desired_capacity < 0) { LOG(ERROR) << "Invalid capacity given to " << caller << ": " << desired_capacity; return JNI_ERR; } - // TODO: this isn't quite right, since "capacity" includes holes. - const size_t capacity = soa.Env()->locals.Capacity(); - bool okay = (static_cast<jint>(kLocalsInitial - capacity) >= desired_capacity); - if (!okay) { - soa.Self()->ThrowOutOfMemoryError(caller); + + std::string error_msg; + if (!soa.Env()->locals.EnsureFreeCapacity(static_cast<size_t>(desired_capacity), &error_msg)) { + std::string caller_error = android::base::StringPrintf("%s: %s", caller, error_msg.c_str()); + soa.Self()->ThrowOutOfMemoryError(caller_error.c_str()); + return JNI_ERR; } - return okay ? JNI_OK : JNI_ERR; + return JNI_OK; } template<typename JniT, typename ArtT> diff --git a/runtime/jni_internal_test.cc b/runtime/jni_internal_test.cc index 08d1eeb95d..e1e4f9c7d6 100644 --- a/runtime/jni_internal_test.cc +++ b/runtime/jni_internal_test.cc @@ -1908,9 +1908,6 @@ TEST_F(JniInternalTest, PushLocalFrame_10395422) { // Negative capacities are not allowed. ASSERT_EQ(JNI_ERR, env_->PushLocalFrame(-1)); - - // And it's okay to have an upper limit. Ours is currently 512. - ASSERT_EQ(JNI_ERR, env_->PushLocalFrame(8192)); } TEST_F(JniInternalTest, PushLocalFrame_PopLocalFrame) { @@ -1962,6 +1959,28 @@ TEST_F(JniInternalTest, PushLocalFrame_PopLocalFrame) { check_jni_abort_catcher.Check("use of deleted local reference"); } +TEST_F(JniInternalTest, PushLocalFrame_LimitAndOverflow) { + // Try a very large value that should fail. + ASSERT_NE(JNI_OK, env_->PushLocalFrame(std::numeric_limits<jint>::max())); + ASSERT_TRUE(env_->ExceptionCheck()); + env_->ExceptionClear(); + + // On 32-bit, also check for some overflow conditions. +#ifndef __LP64__ + ASSERT_EQ(JNI_OK, env_->PushLocalFrame(10)); + ASSERT_NE(JNI_OK, env_->PushLocalFrame(std::numeric_limits<jint>::max() - 10)); + ASSERT_TRUE(env_->ExceptionCheck()); + env_->ExceptionClear(); + EXPECT_EQ(env_->PopLocalFrame(nullptr), nullptr); +#endif +} + +TEST_F(JniInternalTest, PushLocalFrame_b62223672) { + // The 512 entry limit has been lifted, try a larger value. + ASSERT_EQ(JNI_OK, env_->PushLocalFrame(1024)); + EXPECT_EQ(env_->PopLocalFrame(nullptr), nullptr); +} + TEST_F(JniInternalTest, NewGlobalRef_nullptr) { EXPECT_EQ(env_->NewGlobalRef(nullptr), nullptr); } diff --git a/runtime/linear_alloc.cc b/runtime/linear_alloc.cc index e9db9b8b4c..3f01fc329a 100644 --- a/runtime/linear_alloc.cc +++ b/runtime/linear_alloc.cc @@ -16,7 +16,7 @@ #include "linear_alloc.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/managed_stack-inl.h b/runtime/managed_stack-inl.h new file mode 100644 index 0000000000..f3f31cf8e8 --- /dev/null +++ b/runtime/managed_stack-inl.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_MANAGED_STACK_INL_H_ +#define ART_RUNTIME_MANAGED_STACK_INL_H_ + +#include "managed_stack.h" + +#include <cstring> +#include <stdint.h> +#include <string> + +#include "stack.h" + +namespace art { + +inline ShadowFrame* ManagedStack::PushShadowFrame(ShadowFrame* new_top_frame) { + DCHECK(top_quick_frame_ == nullptr); + ShadowFrame* old_frame = top_shadow_frame_; + top_shadow_frame_ = new_top_frame; + new_top_frame->SetLink(old_frame); + return old_frame; +} + +inline ShadowFrame* ManagedStack::PopShadowFrame() { + DCHECK(top_quick_frame_ == nullptr); + CHECK(top_shadow_frame_ != nullptr); + ShadowFrame* frame = top_shadow_frame_; + top_shadow_frame_ = frame->GetLink(); + return frame; +} + +} // namespace art + +#endif // ART_RUNTIME_MANAGED_STACK_INL_H_ diff --git a/runtime/managed_stack.cc b/runtime/managed_stack.cc new file mode 100644 index 0000000000..be609c325d --- /dev/null +++ b/runtime/managed_stack.cc @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "managed_stack-inl.h" + +#include "android-base/stringprintf.h" + +#include "art_method.h" +#include "mirror/object.h" +#include "stack_reference.h" + +namespace art { + +size_t ManagedStack::NumJniShadowFrameReferences() const { + size_t count = 0; + for (const ManagedStack* current_fragment = this; current_fragment != nullptr; + current_fragment = current_fragment->GetLink()) { + for (ShadowFrame* current_frame = current_fragment->top_shadow_frame_; + current_frame != nullptr; + current_frame = current_frame->GetLink()) { + if (current_frame->GetMethod()->IsNative()) { + // The JNI ShadowFrame only contains references. (For indirect reference.) + count += current_frame->NumberOfVRegs(); + } + } + } + return count; +} + +bool ManagedStack::ShadowFramesContain(StackReference<mirror::Object>* shadow_frame_entry) const { + for (const ManagedStack* current_fragment = this; current_fragment != nullptr; + current_fragment = current_fragment->GetLink()) { + for (ShadowFrame* current_frame = current_fragment->top_shadow_frame_; + current_frame != nullptr; + current_frame = current_frame->GetLink()) { + if (current_frame->Contains(shadow_frame_entry)) { + return true; + } + } + } + return false; +} + +} // namespace art diff --git a/runtime/managed_stack.h b/runtime/managed_stack.h new file mode 100644 index 0000000000..8337f968ac --- /dev/null +++ b/runtime/managed_stack.h @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_MANAGED_STACK_H_ +#define ART_RUNTIME_MANAGED_STACK_H_ + +#include <cstring> +#include <stdint.h> +#include <string> + +#include "base/logging.h" +#include "base/macros.h" +#include "base/mutex.h" + +namespace art { + +namespace mirror { +class Object; +} // namespace mirror + +class ArtMethod; +class ShadowFrame; +template <typename T> class StackReference; + +// The managed stack is used to record fragments of managed code stacks. Managed code stacks +// may either be shadow frames or lists of frames using fixed frame sizes. Transition records are +// necessary for transitions between code using different frame layouts and transitions into native +// code. +class PACKED(4) ManagedStack { + public: + ManagedStack() + : top_quick_frame_(nullptr), link_(nullptr), top_shadow_frame_(nullptr) {} + + void PushManagedStackFragment(ManagedStack* fragment) { + // Copy this top fragment into given fragment. + memcpy(fragment, this, sizeof(ManagedStack)); + // Clear this fragment, which has become the top. + memset(this, 0, sizeof(ManagedStack)); + // Link our top fragment onto the given fragment. + link_ = fragment; + } + + void PopManagedStackFragment(const ManagedStack& fragment) { + DCHECK(&fragment == link_); + // Copy this given fragment back to the top. + memcpy(this, &fragment, sizeof(ManagedStack)); + } + + ManagedStack* GetLink() const { + return link_; + } + + ArtMethod** GetTopQuickFrame() const { + return top_quick_frame_; + } + + void SetTopQuickFrame(ArtMethod** top) { + DCHECK(top_shadow_frame_ == nullptr); + top_quick_frame_ = top; + } + + static size_t TopQuickFrameOffset() { + return OFFSETOF_MEMBER(ManagedStack, top_quick_frame_); + } + + ALWAYS_INLINE ShadowFrame* PushShadowFrame(ShadowFrame* new_top_frame); + ALWAYS_INLINE ShadowFrame* PopShadowFrame(); + + ShadowFrame* GetTopShadowFrame() const { + return top_shadow_frame_; + } + + void SetTopShadowFrame(ShadowFrame* top) { + DCHECK(top_quick_frame_ == nullptr); + top_shadow_frame_ = top; + } + + static size_t TopShadowFrameOffset() { + return OFFSETOF_MEMBER(ManagedStack, top_shadow_frame_); + } + + size_t NumJniShadowFrameReferences() const REQUIRES_SHARED(Locks::mutator_lock_); + + bool ShadowFramesContain(StackReference<mirror::Object>* shadow_frame_entry) const; + + private: + ArtMethod** top_quick_frame_; + ManagedStack* link_; + ShadowFrame* top_shadow_frame_; +}; + +} // namespace art + +#endif // ART_RUNTIME_MANAGED_STACK_H_ diff --git a/runtime/mem_map_test.cc b/runtime/mem_map_test.cc index aa306ac337..5f027b1105 100644 --- a/runtime/mem_map_test.cc +++ b/runtime/mem_map_test.cc @@ -16,6 +16,8 @@ #include "mem_map.h" +#include <sys/mman.h> + #include <memory> #include "common_runtime_test.h" diff --git a/runtime/method_handles.h b/runtime/method_handles.h index e8a2dce42e..e02e62052c 100644 --- a/runtime/method_handles.h +++ b/runtime/method_handles.h @@ -23,6 +23,7 @@ #include "handle.h" #include "jvalue.h" #include "mirror/class.h" +#include "stack.h" namespace art { diff --git a/runtime/mirror/object-inl.h b/runtime/mirror/object-inl.h index baed5f167c..d3fc95ff2d 100644 --- a/runtime/mirror/object-inl.h +++ b/runtime/mirror/object-inl.h @@ -899,6 +899,36 @@ inline bool Object::CasFieldWeakRelaxedObjectWithoutWriteBarrier( return success; } +template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags> +inline bool Object::CasFieldWeakReleaseObjectWithoutWriteBarrier( + MemberOffset field_offset, + ObjPtr<Object> old_value, + ObjPtr<Object> new_value) { + if (kCheckTransaction) { + DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction()); + } + if (kVerifyFlags & kVerifyThis) { + VerifyObject(this); + } + if (kVerifyFlags & kVerifyWrites) { + VerifyObject(new_value); + } + if (kVerifyFlags & kVerifyReads) { + VerifyObject(old_value); + } + if (kTransactionActive) { + Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); + } + HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value)); + HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value)); + uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); + Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); + + bool success = atomic_addr->CompareExchangeWeakRelease(old_ref.reference_, + new_ref.reference_); + return success; +} + template<bool kIsStatic, VerifyObjectFlags kVerifyFlags, ReadBarrierOption kReadBarrierOption, diff --git a/runtime/mirror/object-readbarrier-inl.h b/runtime/mirror/object-readbarrier-inl.h index 58e7c20667..69365af7fd 100644 --- a/runtime/mirror/object-readbarrier-inl.h +++ b/runtime/mirror/object-readbarrier-inl.h @@ -221,6 +221,36 @@ inline bool Object::CasFieldStrongRelaxedObjectWithoutWriteBarrier( return success; } +template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags> +inline bool Object::CasFieldStrongReleaseObjectWithoutWriteBarrier( + MemberOffset field_offset, + ObjPtr<Object> old_value, + ObjPtr<Object> new_value) { + if (kCheckTransaction) { + DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction()); + } + if (kVerifyFlags & kVerifyThis) { + VerifyObject(this); + } + if (kVerifyFlags & kVerifyWrites) { + VerifyObject(new_value); + } + if (kVerifyFlags & kVerifyReads) { + VerifyObject(old_value); + } + if (kTransactionActive) { + Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true); + } + HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value)); + HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value)); + uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); + Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); + + bool success = atomic_addr->CompareExchangeStrongRelease(old_ref.reference_, + new_ref.reference_); + return success; +} + } // namespace mirror } // namespace art diff --git a/runtime/mirror/object.h b/runtime/mirror/object.h index 35a1b733e1..9cf42522d1 100644 --- a/runtime/mirror/object.h +++ b/runtime/mirror/object.h @@ -350,10 +350,25 @@ class MANAGED LOCKABLE Object { template<bool kTransactionActive, bool kCheckTransaction = true, VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags> + bool CasFieldWeakReleaseObjectWithoutWriteBarrier(MemberOffset field_offset, + ObjPtr<Object> old_value, + ObjPtr<Object> new_value) + REQUIRES_SHARED(Locks::mutator_lock_); + + template<bool kTransactionActive, + bool kCheckTransaction = true, + VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags> bool CasFieldStrongRelaxedObjectWithoutWriteBarrier(MemberOffset field_offset, ObjPtr<Object> old_value, ObjPtr<Object> new_value) REQUIRES_SHARED(Locks::mutator_lock_); + template<bool kTransactionActive, + bool kCheckTransaction = true, + VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags> + bool CasFieldStrongReleaseObjectWithoutWriteBarrier(MemberOffset field_offset, + ObjPtr<Object> old_value, + ObjPtr<Object> new_value) + REQUIRES_SHARED(Locks::mutator_lock_); template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags> HeapReference<Object>* GetFieldObjectReferenceAddr(MemberOffset field_offset); diff --git a/runtime/mirror/reference-inl.h b/runtime/mirror/reference-inl.h index a449b41087..f8de6e6d90 100644 --- a/runtime/mirror/reference-inl.h +++ b/runtime/mirror/reference-inl.h @@ -20,6 +20,7 @@ #include "reference.h" #include "obj_ptr-inl.h" +#include "runtime.h" namespace art { namespace mirror { diff --git a/runtime/modifiers.h b/runtime/modifiers.h index 461f870b1c..68ab4a4035 100644 --- a/runtime/modifiers.h +++ b/runtime/modifiers.h @@ -60,16 +60,20 @@ static constexpr uint32_t kAccFastNative = 0x00080000; // method (de static constexpr uint32_t kAccCopied = 0x00100000; // method (runtime) static constexpr uint32_t kAccMiranda = 0x00200000; // method (dex only) static constexpr uint32_t kAccDefault = 0x00400000; // method (runtime) + +// Set by the JIT when clearing profiling infos to denote that a method was previously warm. +static constexpr uint32_t kAccPreviouslyWarm = 0x00800000; // method (runtime) + // This is set by the class linker during LinkInterfaceMethods. Prior to that point we do not know // if any particular method needs to be a default conflict. Used to figure out at runtime if // invoking this method will throw an exception. -static constexpr uint32_t kAccDefaultConflict = 0x00800000; // method (runtime) +static constexpr uint32_t kAccDefaultConflict = 0x01000000; // method (runtime) // Set by the verifier for a method we do not want the compiler to compile. -static constexpr uint32_t kAccCompileDontBother = 0x01000000; // method (runtime) +static constexpr uint32_t kAccCompileDontBother = 0x02000000; // method (runtime) // Set by the verifier for a method that could not be verified to follow structured locking. -static constexpr uint32_t kAccMustCountLocks = 0x02000000; // method (runtime) +static constexpr uint32_t kAccMustCountLocks = 0x04000000; // method (runtime) // Set by the class linker for a method that has only one implementation for a // virtual call. @@ -85,8 +89,8 @@ static constexpr uint32_t kAccHasDefaultMethod = 0x40000000; // class/ancestor overrides finalize() static constexpr uint32_t kAccClassIsFinalizable = 0x80000000; -static constexpr uint32_t kAccFlagsNotUsedByIntrinsic = 0x007FFFFF; -static constexpr uint32_t kAccMaxIntrinsic = 0xFF; +static constexpr uint32_t kAccFlagsNotUsedByIntrinsic = 0x00FFFFFF; +static constexpr uint32_t kAccMaxIntrinsic = 0x7F; // Valid (meaningful) bits for a field. static constexpr uint32_t kAccValidFieldFlags = kAccPublic | kAccPrivate | kAccProtected | @@ -96,7 +100,7 @@ static constexpr uint32_t kAccValidFieldFlags = kAccPublic | kAccPrivate | kAccP static constexpr uint32_t kAccValidMethodFlags = kAccPublic | kAccPrivate | kAccProtected | kAccStatic | kAccFinal | kAccSynchronized | kAccBridge | kAccVarargs | kAccNative | kAccAbstract | kAccStrict | kAccSynthetic | kAccMiranda | kAccConstructor | - kAccDeclaredSynchronized; + kAccDeclaredSynchronized | kAccPreviouslyWarm; // Valid (meaningful) bits for a class (not interface). // Note 1. These are positive bits. Other bits may have to be zero. diff --git a/runtime/monitor.cc b/runtime/monitor.cc index bb33047895..f94edcde94 100644 --- a/runtime/monitor.cc +++ b/runtime/monitor.cc @@ -32,6 +32,7 @@ #include "mirror/class-inl.h" #include "mirror/object-inl.h" #include "scoped_thread_state_change-inl.h" +#include "stack.h" #include "thread.h" #include "thread_list.h" #include "verifier/method_verifier.h" @@ -437,17 +438,11 @@ void Monitor::Lock(Thread* self) { << " in " << ArtMethod::PrettyMethod(m) << " for " << PrettyDuration(MsToNs(wait_ms)); } - const char* owners_filename; - int32_t owners_line_number; - TranslateLocation(owners_method, - owners_dex_pc, - &owners_filename, - &owners_line_number); LogContentionEvent(self, wait_ms, sample_percent, - owners_filename, - owners_line_number); + owners_method, + owners_dex_pc); } } } diff --git a/runtime/monitor.h b/runtime/monitor.h index e80d31cdd5..6dc706f8b8 100644 --- a/runtime/monitor.h +++ b/runtime/monitor.h @@ -181,8 +181,11 @@ class Monitor { REQUIRES_SHARED(Locks::mutator_lock_) NO_THREAD_SAFETY_ANALYSIS; // For m->Install(self) - void LogContentionEvent(Thread* self, uint32_t wait_ms, uint32_t sample_percent, - const char* owner_filename, int32_t owner_line_number) + void LogContentionEvent(Thread* self, + uint32_t wait_ms, + uint32_t sample_percent, + ArtMethod* owner_method, + uint32_t owner_dex_pc) REQUIRES_SHARED(Locks::mutator_lock_); static void FailedUnlock(mirror::Object* obj, diff --git a/runtime/monitor_android.cc b/runtime/monitor_android.cc index 1dd60f8d78..a5978523e9 100644 --- a/runtime/monitor_android.cc +++ b/runtime/monitor_android.cc @@ -49,8 +49,15 @@ static char* EventLogWriteString(char* dst, const char* value, size_t len) { return dst + len; } -void Monitor::LogContentionEvent(Thread* self, uint32_t wait_ms, uint32_t sample_percent, - const char* owner_filename, int32_t owner_line_number) { +void Monitor::LogContentionEvent(Thread* self, + uint32_t wait_ms, + uint32_t sample_percent, + ArtMethod* owner_method, + uint32_t owner_dex_pc) { + const char* owner_filename; + int32_t owner_line_number; + TranslateLocation(owner_method, owner_dex_pc, &owner_filename, &owner_line_number); + // Emit the event list length, 1 byte. char eventBuffer[174]; char* cp = eventBuffer; diff --git a/runtime/monitor_linux.cc b/runtime/monitor_linux.cc index 1c77ac0eb3..667866149b 100644 --- a/runtime/monitor_linux.cc +++ b/runtime/monitor_linux.cc @@ -18,7 +18,7 @@ namespace art { -void Monitor::LogContentionEvent(Thread*, uint32_t, uint32_t, const char*, int32_t) { +void Monitor::LogContentionEvent(Thread*, uint32_t, uint32_t, ArtMethod*, uint32_t) { } } // namespace art diff --git a/runtime/monitor_pool.cc b/runtime/monitor_pool.cc index 0f4e2387cc..48e9a6b47d 100644 --- a/runtime/monitor_pool.cc +++ b/runtime/monitor_pool.cc @@ -18,7 +18,7 @@ #include "base/logging.h" #include "base/mutex-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "monitor.h" namespace art { diff --git a/runtime/monitor_pool_test.cc b/runtime/monitor_pool_test.cc index a111c6c16a..5463877b83 100644 --- a/runtime/monitor_pool_test.cc +++ b/runtime/monitor_pool_test.cc @@ -18,7 +18,7 @@ #include "common_runtime_test.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/native/dalvik_system_ZygoteHooks.cc b/runtime/native/dalvik_system_ZygoteHooks.cc index cbf4f323a1..31aeba06f9 100644 --- a/runtime/native/dalvik_system_ZygoteHooks.cc +++ b/runtime/native/dalvik_system_ZygoteHooks.cc @@ -32,7 +32,8 @@ #include "non_debuggable_classes.h" #include "scoped_thread_state_change-inl.h" #include "ScopedUtfChars.h" -#include "thread-inl.h" +#include "stack.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "trace.h" diff --git a/runtime/native/org_apache_harmony_dalvik_ddmc_DdmVmInternal.cc b/runtime/native/org_apache_harmony_dalvik_ddmc_DdmVmInternal.cc index 18c6a9417a..0a254aca54 100644 --- a/runtime/native/org_apache_harmony_dalvik_ddmc_DdmVmInternal.cc +++ b/runtime/native/org_apache_harmony_dalvik_ddmc_DdmVmInternal.cc @@ -21,6 +21,7 @@ #include "base/logging.h" #include "base/mutex.h" #include "debugger.h" +#include "gc/heap.h" #include "jni_internal.h" #include "native_util.h" #include "scoped_fast_native_object_access-inl.h" diff --git a/runtime/native_stack_dump.cc b/runtime/native_stack_dump.cc index cbc502487f..cbff0bb2f2 100644 --- a/runtime/native_stack_dump.cc +++ b/runtime/native_stack_dump.cc @@ -45,7 +45,7 @@ #include "base/unix_file/fd_file.h" #include "oat_quick_method_header.h" #include "os.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "utils.h" #endif diff --git a/runtime/non_debuggable_classes.cc b/runtime/non_debuggable_classes.cc index 829ea65876..9cc7e60fa8 100644 --- a/runtime/non_debuggable_classes.cc +++ b/runtime/non_debuggable_classes.cc @@ -21,7 +21,7 @@ #include "mirror/class-inl.h" #include "obj_ptr-inl.h" #include "ScopedLocalRef.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/oat.h b/runtime/oat.h index b7c715cc03..57c2f9f6e6 100644 --- a/runtime/oat.h +++ b/runtime/oat.h @@ -32,7 +32,7 @@ class InstructionSetFeatures; class PACKED(4) OatHeader { public: static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' }; - static constexpr uint8_t kOatVersion[] = { '1', '2', '5', '\0' }; // ARM Baker narrow thunks. + static constexpr uint8_t kOatVersion[] = { '1', '2', '6', '\0' }; // Shuffle access flags. static constexpr const char* kImageLocationKey = "image-location"; static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline"; diff --git a/runtime/oat_file.h b/runtime/oat_file.h index 06c76b5464..a6d2eba354 100644 --- a/runtime/oat_file.h +++ b/runtime/oat_file.h @@ -39,9 +39,10 @@ class BitVector; class ElfFile; template <class MirrorType> class GcRoot; class MemMap; -class OatMethodOffsets; -class OatHeader; class OatDexFile; +class OatHeader; +class OatMethodOffsets; +class OatQuickMethodHeader; class VdexFile; namespace gc { diff --git a/runtime/oat_file_assistant_test.cc b/runtime/oat_file_assistant_test.cc index b2b86ee289..c2029165ad 100644 --- a/runtime/oat_file_assistant_test.cc +++ b/runtime/oat_file_assistant_test.cc @@ -28,7 +28,7 @@ #include "oat_file_manager.h" #include "os.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "utils.h" namespace art { diff --git a/runtime/oat_file_manager.cc b/runtime/oat_file_manager.cc index c1cf800e5d..6fb4e5eecb 100644 --- a/runtime/oat_file_manager.cc +++ b/runtime/oat_file_manager.cc @@ -38,7 +38,7 @@ #include "oat_file_assistant.h" #include "obj_ptr-inl.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "well_known_classes.h" diff --git a/runtime/obj_ptr-inl.h b/runtime/obj_ptr-inl.h index f2921daeaa..3d9b3c6cf7 100644 --- a/runtime/obj_ptr-inl.h +++ b/runtime/obj_ptr-inl.h @@ -18,7 +18,7 @@ #define ART_RUNTIME_OBJ_PTR_INL_H_ #include "obj_ptr.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/openjdkjvmti/OpenjdkJvmTi.cc b/runtime/openjdkjvmti/OpenjdkJvmTi.cc index 9be486e269..45773fdfbf 100644 --- a/runtime/openjdkjvmti/OpenjdkJvmTi.cc +++ b/runtime/openjdkjvmti/OpenjdkJvmTi.cc @@ -46,7 +46,7 @@ #include "object_tagging.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "ti_class.h" #include "ti_dump.h" diff --git a/runtime/openjdkjvmti/events.cc b/runtime/openjdkjvmti/events.cc index 0ec92b7c60..320c59c810 100644 --- a/runtime/openjdkjvmti/events.cc +++ b/runtime/openjdkjvmti/events.cc @@ -44,7 +44,7 @@ #include "runtime.h" #include "ScopedLocalRef.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace openjdkjvmti { diff --git a/runtime/openjdkjvmti/jvmti_weak_table.h b/runtime/openjdkjvmti/jvmti_weak_table.h index be6edefae2..01c24b1917 100644 --- a/runtime/openjdkjvmti/jvmti_weak_table.h +++ b/runtime/openjdkjvmti/jvmti_weak_table.h @@ -41,7 +41,7 @@ #include "globals.h" #include "jvmti.h" #include "mirror/object.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace openjdkjvmti { diff --git a/runtime/openjdkjvmti/ti_class.cc b/runtime/openjdkjvmti/ti_class.cc index dd90a71240..0aa93dfb57 100644 --- a/runtime/openjdkjvmti/ti_class.cc +++ b/runtime/openjdkjvmti/ti_class.cc @@ -63,7 +63,7 @@ #include "runtime_callbacks.h" #include "ScopedLocalRef.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "ti_class_loader.h" #include "ti_phase.h" diff --git a/runtime/openjdkjvmti/ti_dump.cc b/runtime/openjdkjvmti/ti_dump.cc index d9e3ef1bcf..7a1e53f6e5 100644 --- a/runtime/openjdkjvmti/ti_dump.cc +++ b/runtime/openjdkjvmti/ti_dump.cc @@ -39,7 +39,7 @@ #include "events-inl.h" #include "runtime_callbacks.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" namespace openjdkjvmti { diff --git a/runtime/openjdkjvmti/ti_field.cc b/runtime/openjdkjvmti/ti_field.cc index 1e5fbda35b..342d8be2b0 100644 --- a/runtime/openjdkjvmti/ti_field.cc +++ b/runtime/openjdkjvmti/ti_field.cc @@ -39,7 +39,7 @@ #include "mirror/object_array-inl.h" #include "modifiers.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace openjdkjvmti { diff --git a/runtime/openjdkjvmti/ti_heap.cc b/runtime/openjdkjvmti/ti_heap.cc index 99774c67b5..319b1c2a9c 100644 --- a/runtime/openjdkjvmti/ti_heap.cc +++ b/runtime/openjdkjvmti/ti_heap.cc @@ -35,6 +35,7 @@ #include "primitive.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" +#include "stack.h" #include "thread-inl.h" #include "thread_list.h" diff --git a/runtime/openjdkjvmti/ti_jni.cc b/runtime/openjdkjvmti/ti_jni.cc index 88f0395ba5..dd2dda118a 100644 --- a/runtime/openjdkjvmti/ti_jni.cc +++ b/runtime/openjdkjvmti/ti_jni.cc @@ -38,7 +38,7 @@ #include "java_vm_ext.h" #include "jni_env_ext.h" #include "runtime.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace openjdkjvmti { diff --git a/runtime/openjdkjvmti/ti_method.cc b/runtime/openjdkjvmti/ti_method.cc index f7e53474aa..beb639e208 100644 --- a/runtime/openjdkjvmti/ti_method.cc +++ b/runtime/openjdkjvmti/ti_method.cc @@ -42,7 +42,7 @@ #include "runtime_callbacks.h" #include "scoped_thread_state_change-inl.h" #include "ScopedLocalRef.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "ti_phase.h" diff --git a/runtime/openjdkjvmti/ti_monitor.cc b/runtime/openjdkjvmti/ti_monitor.cc index 645faea41b..61bf533eb7 100644 --- a/runtime/openjdkjvmti/ti_monitor.cc +++ b/runtime/openjdkjvmti/ti_monitor.cc @@ -39,7 +39,7 @@ #include "art_jvmti.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace openjdkjvmti { diff --git a/runtime/openjdkjvmti/ti_object.cc b/runtime/openjdkjvmti/ti_object.cc index bf84499035..2506acac3a 100644 --- a/runtime/openjdkjvmti/ti_object.cc +++ b/runtime/openjdkjvmti/ti_object.cc @@ -34,7 +34,7 @@ #include "art_jvmti.h" #include "mirror/object-inl.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace openjdkjvmti { diff --git a/runtime/openjdkjvmti/ti_phase.cc b/runtime/openjdkjvmti/ti_phase.cc index 941cf7b73b..3c8bdc61d0 100644 --- a/runtime/openjdkjvmti/ti_phase.cc +++ b/runtime/openjdkjvmti/ti_phase.cc @@ -38,7 +38,7 @@ #include "runtime_callbacks.h" #include "ScopedLocalRef.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "ti_thread.h" diff --git a/runtime/openjdkjvmti/ti_properties.cc b/runtime/openjdkjvmti/ti_properties.cc index 8ee5366140..e399b484ec 100644 --- a/runtime/openjdkjvmti/ti_properties.cc +++ b/runtime/openjdkjvmti/ti_properties.cc @@ -40,7 +40,7 @@ #include "art_jvmti.h" #include "runtime.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "ti_phase.h" #include "well_known_classes.h" diff --git a/runtime/openjdkjvmti/ti_redefine.cc b/runtime/openjdkjvmti/ti_redefine.cc index cca1486975..ca3a0e631a 100644 --- a/runtime/openjdkjvmti/ti_redefine.cc +++ b/runtime/openjdkjvmti/ti_redefine.cc @@ -602,8 +602,8 @@ bool Redefiner::ClassRedefinition::CheckSameMethods() { // Since direct methods have different flags than virtual ones (specifically direct methods must // have kAccPrivate or kAccStatic or kAccConstructor flags) we can tell if a method changes from // virtual to direct. - uint32_t new_flags = new_iter.GetMethodAccessFlags(); - if (new_flags != (old_method->GetAccessFlags() & art::kAccValidMethodFlags)) { + uint32_t new_flags = new_iter.GetMethodAccessFlags() & ~art::kAccPreviouslyWarm; + if (new_flags != (old_method->GetAccessFlags() & (art::kAccValidMethodFlags ^ art::kAccPreviouslyWarm))) { RecordFailure(ERR(UNSUPPORTED_REDEFINITION_METHOD_MODIFIERS_CHANGED), StringPrintf("method '%s' (sig: %s) had different access flags", new_method_name, diff --git a/runtime/openjdkjvmti/ti_search.cc b/runtime/openjdkjvmti/ti_search.cc index ec139f2004..6e0196edc3 100644 --- a/runtime/openjdkjvmti/ti_search.cc +++ b/runtime/openjdkjvmti/ti_search.cc @@ -49,7 +49,7 @@ #include "scoped_thread_state_change-inl.h" #include "ScopedLocalRef.h" #include "ti_phase.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "well_known_classes.h" diff --git a/runtime/openjdkjvmti/ti_stack.cc b/runtime/openjdkjvmti/ti_stack.cc index 1ddf04feb4..22da2d2f65 100644 --- a/runtime/openjdkjvmti/ti_stack.cc +++ b/runtime/openjdkjvmti/ti_stack.cc @@ -52,7 +52,7 @@ #include "scoped_thread_state_change-inl.h" #include "ScopedLocalRef.h" #include "stack.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "thread_pool.h" #include "well_known_classes.h" diff --git a/runtime/openjdkjvmti/ti_thread.cc b/runtime/openjdkjvmti/ti_thread.cc index 3dfa63313d..2cc2a26c3b 100644 --- a/runtime/openjdkjvmti/ti_thread.cc +++ b/runtime/openjdkjvmti/ti_thread.cc @@ -49,7 +49,7 @@ #include "runtime_callbacks.h" #include "ScopedLocalRef.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "well_known_classes.h" diff --git a/runtime/openjdkjvmti/ti_threadgroup.cc b/runtime/openjdkjvmti/ti_threadgroup.cc index dd7be113d6..c0597ad0cc 100644 --- a/runtime/openjdkjvmti/ti_threadgroup.cc +++ b/runtime/openjdkjvmti/ti_threadgroup.cc @@ -45,7 +45,7 @@ #include "object_lock.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" #include "thread_list.h" #include "well_known_classes.h" diff --git a/runtime/read_barrier-inl.h b/runtime/read_barrier-inl.h index dbe7f5c957..2d06e54f78 100644 --- a/runtime/read_barrier-inl.h +++ b/runtime/read_barrier-inl.h @@ -19,6 +19,7 @@ #include "read_barrier.h" +#include "gc/accounting/read_barrier_table.h" #include "gc/collector/concurrent_copying-inl.h" #include "gc/heap.h" #include "mirror/object_reference.h" @@ -62,7 +63,7 @@ inline MirrorType* ReadBarrier::Barrier( // If kAlwaysUpdateField is true, update the field atomically. This may fail if mutator // updates before us, but it's OK. if (kAlwaysUpdateField && ref != old_ref) { - obj->CasFieldStrongRelaxedObjectWithoutWriteBarrier<false, false>( + obj->CasFieldStrongReleaseObjectWithoutWriteBarrier<false, false>( offset, old_ref, ref); } } @@ -80,7 +81,7 @@ inline MirrorType* ReadBarrier::Barrier( ref = reinterpret_cast<MirrorType*>(Mark(old_ref)); // Update the field atomically. This may fail if mutator updates before us, but it's ok. if (ref != old_ref) { - obj->CasFieldStrongRelaxedObjectWithoutWriteBarrier<false, false>( + obj->CasFieldStrongReleaseObjectWithoutWriteBarrier<false, false>( offset, old_ref, ref); } } diff --git a/runtime/reference_table_test.cc b/runtime/reference_table_test.cc index e809ecf1f6..260be8f41f 100644 --- a/runtime/reference_table_test.cc +++ b/runtime/reference_table_test.cc @@ -29,7 +29,7 @@ #include "primitive.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/runtime_android.cc b/runtime/runtime_android.cc index 495296cf7d..4bd3b3ae3a 100644 --- a/runtime/runtime_android.cc +++ b/runtime/runtime_android.cc @@ -27,7 +27,11 @@ namespace art { struct sigaction old_action; void HandleUnexpectedSignalAndroid(int signal_number, siginfo_t* info, void* raw_context) { - HandleUnexpectedSignalCommon(signal_number, info, raw_context, /* running_on_linux */ false); + HandleUnexpectedSignalCommon(signal_number, + info, + raw_context, + /* handle_timeout_signal */ false, + /* dump_on_stderr */ false); // Run the old signal handler. old_action.sa_sigaction(signal_number, info, raw_context); diff --git a/runtime/runtime_common.cc b/runtime/runtime_common.cc index 36901293bb..940e4611f6 100644 --- a/runtime/runtime_common.cc +++ b/runtime/runtime_common.cc @@ -29,7 +29,8 @@ #include "base/macros.h" #include "base/mutex.h" #include "native_stack_dump.h" -#include "thread-inl.h" +#include "runtime.h" +#include "thread-current-inl.h" #include "thread_list.h" namespace art { @@ -370,10 +371,8 @@ static bool IsTimeoutSignal(int signal_number) { void HandleUnexpectedSignalCommon(int signal_number, siginfo_t* info, void* raw_context, - bool running_on_linux) { - bool handle_timeout_signal = running_on_linux; - bool dump_on_stderr = running_on_linux; - + bool handle_timeout_signal, + bool dump_on_stderr) { static bool handling_unexpected_signal = false; if (handling_unexpected_signal) { LogHelper::LogLineLowStack(__FILE__, @@ -393,39 +392,41 @@ void HandleUnexpectedSignalCommon(int signal_number, gAborting++; // set before taking any locks MutexLock mu(Thread::Current(), *Locks::unexpected_signal_lock_); - bool has_address = (signal_number == SIGILL || signal_number == SIGBUS || - signal_number == SIGFPE || signal_number == SIGSEGV); + auto logger = [&](auto& stream) { + bool has_address = (signal_number == SIGILL || signal_number == SIGBUS || + signal_number == SIGFPE || signal_number == SIGSEGV); + OsInfo os_info; + const char* cmd_line = GetCmdLine(); + if (cmd_line == nullptr) { + cmd_line = "<unset>"; // Because no-one called InitLogging. + } + pid_t tid = GetTid(); + std::string thread_name(GetThreadName(tid)); + UContext thread_context(raw_context); + Backtrace thread_backtrace(raw_context); + + stream << "*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***" << std::endl + << StringPrintf("Fatal signal %d (%s), code %d (%s)", + signal_number, + GetSignalName(signal_number), + info->si_code, + GetSignalCodeName(signal_number, info->si_code)) + << (has_address ? StringPrintf(" fault addr %p", info->si_addr) : "") << std::endl + << "OS: " << Dumpable<OsInfo>(os_info) << std::endl + << "Cmdline: " << cmd_line << std::endl + << "Thread: " << tid << " \"" << thread_name << "\"" << std::endl + << "Registers:\n" << Dumpable<UContext>(thread_context) << std::endl + << "Backtrace:\n" << Dumpable<Backtrace>(thread_backtrace) << std::endl; + stream << std::flush; + }; - OsInfo os_info; - const char* cmd_line = GetCmdLine(); - if (cmd_line == nullptr) { - cmd_line = "<unset>"; // Because no-one called InitLogging. - } - pid_t tid = GetTid(); - std::string thread_name(GetThreadName(tid)); - UContext thread_context(raw_context); - Backtrace thread_backtrace(raw_context); - - std::ostringstream stream; - stream << "*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***\n" - << StringPrintf("Fatal signal %d (%s), code %d (%s)", - signal_number, - GetSignalName(signal_number), - info->si_code, - GetSignalCodeName(signal_number, info->si_code)) - << (has_address ? StringPrintf(" fault addr %p", info->si_addr) : "") << '\n' - << "OS: " << Dumpable<OsInfo>(os_info) << '\n' - << "Cmdline: " << cmd_line << '\n' - << "Thread: " << tid << " \"" << thread_name << "\"" << '\n' - << "Registers:\n" << Dumpable<UContext>(thread_context) << '\n' - << "Backtrace:\n" << Dumpable<Backtrace>(thread_backtrace) << '\n'; if (dump_on_stderr) { // Note: We are using cerr directly instead of LOG macros to ensure even just partial output // makes it out. That means we lose the "dalvikvm..." prefix, but that is acceptable // considering this is an abort situation. - std::cerr << stream.str() << std::flush; + logger(std::cerr); } else { - LOG(FATAL_WITHOUT_ABORT) << stream.str() << std::flush; + logger(LOG_STREAM(FATAL_WITHOUT_ABORT)); } if (kIsDebugBuild && signal_number == SIGSEGV) { PrintFileToLog("/proc/self/maps", LogSeverity::FATAL_WITHOUT_ABORT); diff --git a/runtime/runtime_common.h b/runtime/runtime_common.h index 832b6bbf3e..06d66270af 100644 --- a/runtime/runtime_common.h +++ b/runtime/runtime_common.h @@ -68,7 +68,8 @@ int GetTimeoutSignal(); void HandleUnexpectedSignalCommon(int signal_number, siginfo_t* info, void* raw_context, - bool running_on_linux); + bool handle_timeout_signal, + bool dump_on_stderr); void InitPlatformSignalHandlersCommon(void (*newact)(int, siginfo_t*, void*), struct sigaction* oldact, diff --git a/runtime/runtime_linux.cc b/runtime/runtime_linux.cc index ad61cf373b..424dcf85cf 100644 --- a/runtime/runtime_linux.cc +++ b/runtime/runtime_linux.cc @@ -25,7 +25,13 @@ namespace art { void HandleUnexpectedSignalLinux(int signal_number, siginfo_t* info, void* raw_context) { - HandleUnexpectedSignalCommon(signal_number, info, raw_context, /* running_on_linux */ true); + // Linux is mainly used for host testing. Under those conditions, react to the timeout signal, + // and dump to stderr to avoid missing output on double-faults. + HandleUnexpectedSignalCommon(signal_number, + info, + raw_context, + /* handle_timeout_signal */ true, + /* dump_on_stderr */ true); if (getenv("debug_db_uid") != nullptr || getenv("art_wait_for_gdb_on_crash") != nullptr) { pid_t tid = GetTid(); diff --git a/runtime/safe_map.h b/runtime/safe_map.h index e638fdb504..b54f587715 100644 --- a/runtime/safe_map.h +++ b/runtime/safe_map.h @@ -46,6 +46,7 @@ class SafeMap { SafeMap() = default; SafeMap(const SafeMap&) = default; + SafeMap(SafeMap&&) = default; explicit SafeMap(const key_compare& cmp, const allocator_type& allocator = allocator_type()) : map_(cmp, allocator) { } @@ -151,6 +152,11 @@ class SafeMap { return map_ == rhs.map_; } + template <class... Args> + std::pair<iterator, bool> emplace(Args&&... args) { + return map_.emplace(std::forward<Args>(args)...); + } + private: ::std::map<K, V, Comparator, Allocator> map_; }; diff --git a/runtime/scoped_thread_state_change-inl.h b/runtime/scoped_thread_state_change-inl.h index ed6e349de4..aa96871145 100644 --- a/runtime/scoped_thread_state_change-inl.h +++ b/runtime/scoped_thread_state_change-inl.h @@ -22,6 +22,7 @@ #include "base/casts.h" #include "jni_env_ext-inl.h" #include "obj_ptr-inl.h" +#include "runtime.h" #include "thread-inl.h" namespace art { diff --git a/runtime/stack.cc b/runtime/stack.cc index 5c6eead34b..8fcac1ea7f 100644 --- a/runtime/stack.cc +++ b/runtime/stack.cc @@ -29,6 +29,7 @@ #include "jit/jit.h" #include "jit/jit_code_cache.h" #include "linear_alloc.h" +#include "managed_stack.h" #include "mirror/class-inl.h" #include "mirror/object-inl.h" #include "mirror/object_array-inl.h" @@ -68,34 +69,6 @@ mirror::Object* ShadowFrame::GetThisObject(uint16_t num_ins) const { } } -size_t ManagedStack::NumJniShadowFrameReferences() const { - size_t count = 0; - for (const ManagedStack* current_fragment = this; current_fragment != nullptr; - current_fragment = current_fragment->GetLink()) { - for (ShadowFrame* current_frame = current_fragment->top_shadow_frame_; current_frame != nullptr; - current_frame = current_frame->GetLink()) { - if (current_frame->GetMethod()->IsNative()) { - // The JNI ShadowFrame only contains references. (For indirect reference.) - count += current_frame->NumberOfVRegs(); - } - } - } - return count; -} - -bool ManagedStack::ShadowFramesContain(StackReference<mirror::Object>* shadow_frame_entry) const { - for (const ManagedStack* current_fragment = this; current_fragment != nullptr; - current_fragment = current_fragment->GetLink()) { - for (ShadowFrame* current_frame = current_fragment->top_shadow_frame_; current_frame != nullptr; - current_frame = current_frame->GetLink()) { - if (current_frame->Contains(shadow_frame_entry)) { - return true; - } - } - } - return false; -} - StackVisitor::StackVisitor(Thread* thread, Context* context, StackWalkKind walk_kind, diff --git a/runtime/stack.h b/runtime/stack.h index bdaa4c3ca2..8c74a8c405 100644 --- a/runtime/stack.h +++ b/runtime/stack.h @@ -512,86 +512,6 @@ class JavaFrameRootInfo FINAL : public RootInfo { const size_t vreg_; }; -// The managed stack is used to record fragments of managed code stacks. Managed code stacks -// may either be shadow frames or lists of frames using fixed frame sizes. Transition records are -// necessary for transitions between code using different frame layouts and transitions into native -// code. -class PACKED(4) ManagedStack { - public: - ManagedStack() - : top_quick_frame_(nullptr), link_(nullptr), top_shadow_frame_(nullptr) {} - - void PushManagedStackFragment(ManagedStack* fragment) { - // Copy this top fragment into given fragment. - memcpy(fragment, this, sizeof(ManagedStack)); - // Clear this fragment, which has become the top. - memset(this, 0, sizeof(ManagedStack)); - // Link our top fragment onto the given fragment. - link_ = fragment; - } - - void PopManagedStackFragment(const ManagedStack& fragment) { - DCHECK(&fragment == link_); - // Copy this given fragment back to the top. - memcpy(this, &fragment, sizeof(ManagedStack)); - } - - ManagedStack* GetLink() const { - return link_; - } - - ArtMethod** GetTopQuickFrame() const { - return top_quick_frame_; - } - - void SetTopQuickFrame(ArtMethod** top) { - DCHECK(top_shadow_frame_ == nullptr); - top_quick_frame_ = top; - } - - static size_t TopQuickFrameOffset() { - return OFFSETOF_MEMBER(ManagedStack, top_quick_frame_); - } - - ShadowFrame* PushShadowFrame(ShadowFrame* new_top_frame) { - DCHECK(top_quick_frame_ == nullptr); - ShadowFrame* old_frame = top_shadow_frame_; - top_shadow_frame_ = new_top_frame; - new_top_frame->SetLink(old_frame); - return old_frame; - } - - ShadowFrame* PopShadowFrame() { - DCHECK(top_quick_frame_ == nullptr); - CHECK(top_shadow_frame_ != nullptr); - ShadowFrame* frame = top_shadow_frame_; - top_shadow_frame_ = frame->GetLink(); - return frame; - } - - ShadowFrame* GetTopShadowFrame() const { - return top_shadow_frame_; - } - - void SetTopShadowFrame(ShadowFrame* top) { - DCHECK(top_quick_frame_ == nullptr); - top_shadow_frame_ = top; - } - - static size_t TopShadowFrameOffset() { - return OFFSETOF_MEMBER(ManagedStack, top_shadow_frame_); - } - - size_t NumJniShadowFrameReferences() const REQUIRES_SHARED(Locks::mutator_lock_); - - bool ShadowFramesContain(StackReference<mirror::Object>* shadow_frame_entry) const; - - private: - ArtMethod** top_quick_frame_; - ManagedStack* link_; - ShadowFrame* top_shadow_frame_; -}; - class StackVisitor { public: // This enum defines a flag to control whether inlined frames are included diff --git a/runtime/thread-current-inl.h b/runtime/thread-current-inl.h new file mode 100644 index 0000000000..9241b1f875 --- /dev/null +++ b/runtime/thread-current-inl.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_THREAD_CURRENT_INL_H_ +#define ART_RUNTIME_THREAD_CURRENT_INL_H_ + +#include "thread.h" + +#ifdef ART_TARGET_ANDROID +#include <bionic_tls.h> // Access to our own TLS slot. +#endif + +#include <pthread.h> + +namespace art { + +inline Thread* Thread::Current() { + // We rely on Thread::Current returning null for a detached thread, so it's not obvious + // that we can replace this with a direct %fs access on x86. + if (!is_started_) { + return nullptr; + } else { +#ifdef ART_TARGET_ANDROID + void* thread = __get_tls()[TLS_SLOT_ART_THREAD_SELF]; +#else + void* thread = pthread_getspecific(Thread::pthread_key_self_); +#endif + return reinterpret_cast<Thread*>(thread); + } +} + +} // namespace art + +#endif // ART_RUNTIME_THREAD_CURRENT_INL_H_ diff --git a/runtime/thread-inl.h b/runtime/thread-inl.h index aa769fa23d..7da15d9f4c 100644 --- a/runtime/thread-inl.h +++ b/runtime/thread-inl.h @@ -19,18 +19,13 @@ #include "thread.h" -#ifdef ART_TARGET_ANDROID -#include <bionic_tls.h> // Access to our own TLS slot. -#endif - -#include <pthread.h> - #include "base/casts.h" #include "base/mutex-inl.h" -#include "gc/heap.h" +#include "base/time_utils.h" #include "jni_env_ext.h" +#include "managed_stack-inl.h" #include "obj_ptr.h" -#include "runtime.h" +#include "thread-current-inl.h" #include "thread_pool.h" namespace art { @@ -41,21 +36,6 @@ static inline Thread* ThreadForEnv(JNIEnv* env) { return full_env->self; } -inline Thread* Thread::Current() { - // We rely on Thread::Current returning null for a detached thread, so it's not obvious - // that we can replace this with a direct %fs access on x86. - if (!is_started_) { - return nullptr; - } else { -#ifdef ART_TARGET_ANDROID - void* thread = __get_tls()[TLS_SLOT_ART_THREAD_SELF]; -#else - void* thread = pthread_getspecific(Thread::pthread_key_self_); -#endif - return reinterpret_cast<Thread*>(thread); - } -} - inline void Thread::AllowThreadSuspension() { DCHECK_EQ(Thread::Current(), this); if (UNLIKELY(TestAllFlags())) { @@ -295,14 +275,6 @@ inline ThreadState Thread::TransitionFromSuspendedToRunnable() { return static_cast<ThreadState>(old_state); } -inline void Thread::VerifyStack() { - if (kVerifyStack) { - if (Runtime::Current()->GetHeap()->IsObjectValidationEnabled()) { - VerifyStackImpl(); - } - } -} - inline mirror::Object* Thread::AllocTlab(size_t bytes) { DCHECK_GE(TlabSize(), bytes); ++tlsPtr_.thread_local_objects; @@ -386,6 +358,14 @@ inline bool Thread::ModifySuspendCount(Thread* self, } } +inline ShadowFrame* Thread::PushShadowFrame(ShadowFrame* new_top_frame) { + return tlsPtr_.managed_stack.PushShadowFrame(new_top_frame); +} + +inline ShadowFrame* Thread::PopShadowFrame() { + return tlsPtr_.managed_stack.PopShadowFrame(); +} + } // namespace art #endif // ART_RUNTIME_THREAD_INL_H_ diff --git a/runtime/thread.cc b/runtime/thread.cc index 6848686e5d..789f571253 100644 --- a/runtime/thread.cc +++ b/runtime/thread.cc @@ -55,6 +55,7 @@ #include "gc/allocator/rosalloc.h" #include "gc/heap.h" #include "gc/space/space-inl.h" +#include "gc_root.h" #include "handle_scope-inl.h" #include "indirect_reference_table-inl.h" #include "java_vm_ext.h" @@ -2160,7 +2161,7 @@ Thread::~Thread() { TearDownAlternateSignalStack(); } -void Thread::HandleUncaughtExceptions(ScopedObjectAccess& soa) { +void Thread::HandleUncaughtExceptions(ScopedObjectAccessAlreadyRunnable& soa) { if (!IsExceptionPending()) { return; } @@ -2180,7 +2181,7 @@ void Thread::HandleUncaughtExceptions(ScopedObjectAccess& soa) { tlsPtr_.jni_env->ExceptionClear(); } -void Thread::RemoveFromThreadGroup(ScopedObjectAccess& soa) { +void Thread::RemoveFromThreadGroup(ScopedObjectAccessAlreadyRunnable& soa) { // this.group.removeThread(this); // group can be null if we're in the compiler or a test. ObjPtr<mirror::Object> ogroup = jni::DecodeArtField(WellKnownClasses::java_lang_Thread_group) @@ -3442,11 +3443,13 @@ class VerifyRootVisitor : public SingleRootVisitor { }; void Thread::VerifyStackImpl() { - VerifyRootVisitor visitor; - std::unique_ptr<Context> context(Context::Create()); - RootCallbackVisitor visitor_to_callback(&visitor, GetThreadId()); - ReferenceMapVisitor<RootCallbackVisitor> mapper(this, context.get(), visitor_to_callback); - mapper.WalkStack(); + if (Runtime::Current()->GetHeap()->IsObjectValidationEnabled()) { + VerifyRootVisitor visitor; + std::unique_ptr<Context> context(Context::Create()); + RootCallbackVisitor visitor_to_callback(&visitor, GetThreadId()); + ReferenceMapVisitor<RootCallbackVisitor> mapper(this, context.get(), visitor_to_callback); + mapper.WalkStack(); + } } // Set the stack end to that to be used during a stack overflow diff --git a/runtime/thread.h b/runtime/thread.h index a60fd58ca0..e85ee0d2f3 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -33,15 +33,13 @@ #include "base/mutex.h" #include "entrypoints/jni/jni_entrypoints.h" #include "entrypoints/quick/quick_entrypoints.h" -#include "gc_root.h" #include "globals.h" #include "handle_scope.h" #include "instrumentation.h" #include "jvalue.h" -#include "object_callbacks.h" +#include "managed_stack.h" #include "offsets.h" #include "runtime_stats.h" -#include "stack.h" #include "thread_state.h" class BacktraceMap; @@ -87,12 +85,14 @@ class FrameIdToShadowFrame; class JavaVMExt; struct JNIEnvExt; class Monitor; +class RootVisitor; class ScopedObjectAccessAlreadyRunnable; class ShadowFrame; class SingleStepControl; class StackedShadowFrameRecord; class Thread; class ThreadList; +enum VisitRootFlags : uint8_t; // Thread priorities. These must match the Thread.MIN_PRIORITY, // Thread.NORM_PRIORITY, and Thread.MAX_PRIORITY constants. @@ -149,6 +149,7 @@ static constexpr size_t kNumRosAllocThreadLocalSizeBracketsInThread = 16; class Thread { public: static const size_t kStackOverflowImplicitCheckSize; + static constexpr bool kVerifyStack = kIsDebugBuild; // Creates a new native thread corresponding to the given managed peer. // Used to implement Thread.start. @@ -560,10 +561,14 @@ class Thread { return tlsPtr_.frame_id_to_shadow_frame != nullptr; } - void VisitRoots(RootVisitor* visitor, VisitRootFlags flags = kVisitRootFlagAllRoots) + void VisitRoots(RootVisitor* visitor, VisitRootFlags flags) REQUIRES_SHARED(Locks::mutator_lock_); - ALWAYS_INLINE void VerifyStack() REQUIRES_SHARED(Locks::mutator_lock_); + void VerifyStack() REQUIRES_SHARED(Locks::mutator_lock_) { + if (kVerifyStack) { + VerifyStackImpl(); + } + } // // Offsets of various members of native Thread class, used by compiled code. @@ -793,13 +798,8 @@ class Thread { tlsPtr_.managed_stack.PopManagedStackFragment(fragment); } - ShadowFrame* PushShadowFrame(ShadowFrame* new_top_frame) { - return tlsPtr_.managed_stack.PushShadowFrame(new_top_frame); - } - - ShadowFrame* PopShadowFrame() { - return tlsPtr_.managed_stack.PopShadowFrame(); - } + ALWAYS_INLINE ShadowFrame* PushShadowFrame(ShadowFrame* new_top_frame); + ALWAYS_INLINE ShadowFrame* PopShadowFrame(); template<PointerSize pointer_size> static ThreadOffset<pointer_size> TopShadowFrameOffset() { @@ -1250,9 +1250,10 @@ class Thread { static void* CreateCallback(void* arg); - void HandleUncaughtExceptions(ScopedObjectAccess& soa) + void HandleUncaughtExceptions(ScopedObjectAccessAlreadyRunnable& soa) + REQUIRES_SHARED(Locks::mutator_lock_); + void RemoveFromThreadGroup(ScopedObjectAccessAlreadyRunnable& soa) REQUIRES_SHARED(Locks::mutator_lock_); - void RemoveFromThreadGroup(ScopedObjectAccess& soa) REQUIRES_SHARED(Locks::mutator_lock_); // Initialize a thread. // diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc index dc2af2ae34..5094189dfc 100644 --- a/runtime/thread_list.cc +++ b/runtime/thread_list.cc @@ -35,7 +35,9 @@ #include "debugger.h" #include "gc/collector/concurrent_copying.h" #include "gc/gc_pause_listener.h" +#include "gc/heap.h" #include "gc/reference_processor.h" +#include "gc_root.h" #include "jni_internal.h" #include "lock_word.h" #include "monitor.h" @@ -756,7 +758,7 @@ void ThreadList::SuspendAllInternal(Thread* self, // EAGAIN and EINTR both indicate a spurious failure, try again from the beginning. if ((errno != EAGAIN) && (errno != EINTR)) { if (errno == ETIMEDOUT) { - LOG(::android::base::FATAL) + LOG(kIsDebugBuild ? ::android::base::FATAL : ::android::base::ERROR) << "Timed out waiting for threads to suspend, waited for " << PrettyDuration(NanoTime() - start_time); } else { @@ -1508,7 +1510,7 @@ void ThreadList::VisitRootsForSuspendedThreads(RootVisitor* visitor) { // Visit roots without holding thread_list_lock_ and thread_suspend_count_lock_ to prevent lock // order violations. for (Thread* thread : threads_to_visit) { - thread->VisitRoots(visitor); + thread->VisitRoots(visitor, kVisitRootFlagAllRoots); } // Restore suspend counts. diff --git a/runtime/thread_list.h b/runtime/thread_list.h index 0ce1d78382..92702c6498 100644 --- a/runtime/thread_list.h +++ b/runtime/thread_list.h @@ -22,9 +22,7 @@ #include "base/mutex.h" #include "base/time_utils.h" #include "base/value_object.h" -#include "gc_root.h" #include "jni.h" -#include "object_callbacks.h" #include <bitset> #include <list> @@ -38,8 +36,10 @@ namespace gc { class GcPauseListener; } // namespace gc class Closure; +class RootVisitor; class Thread; class TimingLogger; +enum VisitRootFlags : uint8_t; class ThreadList { public: diff --git a/runtime/thread_pool.cc b/runtime/thread_pool.cc index d24a5e5c4a..8349f33028 100644 --- a/runtime/thread_pool.cc +++ b/runtime/thread_pool.cc @@ -18,6 +18,7 @@ #include <pthread.h> +#include <sys/mman.h> #include <sys/time.h> #include <sys/resource.h> @@ -29,7 +30,7 @@ #include "base/stl_util.h" #include "base/time_utils.h" #include "runtime.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { diff --git a/runtime/trace.cc b/runtime/trace.cc index 3a9975a4e2..3550d56bd8 100644 --- a/runtime/trace.cc +++ b/runtime/trace.cc @@ -41,6 +41,7 @@ #include "os.h" #include "scoped_thread_state_change-inl.h" #include "ScopedLocalRef.h" +#include "stack.h" #include "thread.h" #include "thread_list.h" #include "utils.h" diff --git a/runtime/verifier/method_verifier.cc b/runtime/verifier/method_verifier.cc index 7490611cb6..8872173079 100644 --- a/runtime/verifier/method_verifier.cc +++ b/runtime/verifier/method_verifier.cc @@ -50,6 +50,7 @@ #include "register_line-inl.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" +#include "stack.h" #include "utils.h" #include "verifier_deps.h" #include "verifier_compiler_binding.h" diff --git a/runtime/verifier/reg_type_test.cc b/runtime/verifier/reg_type_test.cc index 49dac26bb4..b0ea6c857c 100644 --- a/runtime/verifier/reg_type_test.cc +++ b/runtime/verifier/reg_type_test.cc @@ -25,7 +25,7 @@ #include "reg_type_cache-inl.h" #include "reg_type-inl.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { namespace verifier { diff --git a/runtime/verifier/verifier_deps.h b/runtime/verifier/verifier_deps.h index 70ce0c4a29..43eb948c64 100644 --- a/runtime/verifier/verifier_deps.h +++ b/runtime/verifier/verifier_deps.h @@ -23,6 +23,7 @@ #include "base/array_ref.h" #include "base/mutex.h" +#include "dex_file_types.h" #include "handle.h" #include "method_resolution_kind.h" #include "obj_ptr.h" diff --git a/runtime/verify_object.h b/runtime/verify_object.h index 519f7f5f5a..e4c01d0f78 100644 --- a/runtime/verify_object.h +++ b/runtime/verify_object.h @@ -48,7 +48,6 @@ enum VerifyObjectFlags { kVerifyAll = kVerifyThis | kVerifyReads | kVerifyWrites, }; -static constexpr bool kVerifyStack = kIsDebugBuild; static constexpr VerifyObjectFlags kDefaultVerifyFlags = kVerifyNone; static constexpr VerifyObjectMode kVerifyObjectSupport = kDefaultVerifyFlags != 0 ? kVerifyObjectModeFast : kVerifyObjectModeDisabled; diff --git a/runtime/well_known_classes.cc b/runtime/well_known_classes.cc index 5aef062728..24f194b5ee 100644 --- a/runtime/well_known_classes.cc +++ b/runtime/well_known_classes.cc @@ -30,7 +30,7 @@ #include "obj_ptr-inl.h" #include "ScopedLocalRef.h" #include "scoped_thread_state_change-inl.h" -#include "thread-inl.h" +#include "thread-current-inl.h" namespace art { |