Move mirror::ArtMethod to native
Optimizing + quick tests are passing, devices boot.
TODO: Test and fix bugs in mips64.
Saves 16 bytes per most ArtMethod, 7.5MB reduction in system PSS.
Some of the savings are from removal of virtual methods and direct
methods object arrays.
Bug: 19264997
Change-Id: I622469a0cfa0e7082a2119f3d6a9491eb61e3f3d
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index 02a2588..11743f3 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -23,6 +23,7 @@
#include <vector>
#include "art_field-inl.h"
+#include "art_method-inl.h"
#include "base/logging.h"
#include "base/unix_file/fd_file.h"
#include "class_linker-inl.h"
@@ -43,11 +44,12 @@
#include "intern_table.h"
#include "linear_alloc.h"
#include "lock_word.h"
-#include "mirror/art_method-inl.h"
+#include "mirror/abstract_method.h"
#include "mirror/array-inl.h"
#include "mirror/class-inl.h"
#include "mirror/class_loader.h"
#include "mirror/dex_cache-inl.h"
+#include "mirror/method.h"
#include "mirror/object-inl.h"
#include "mirror/object_array-inl.h"
#include "mirror/string-inl.h"
@@ -58,10 +60,8 @@
#include "handle_scope-inl.h"
#include "utils/dex_cache_arrays_layout-inl.h"
-using ::art::mirror::ArtMethod;
using ::art::mirror::Class;
using ::art::mirror::DexCache;
-using ::art::mirror::EntryPointFromInterpreter;
using ::art::mirror::Object;
using ::art::mirror::ObjectArray;
using ::art::mirror::String;
@@ -169,10 +169,11 @@
ElfWriter::GetOatElfInformation(oat_file.get(), &oat_loaded_size, &oat_data_offset);
Thread::Current()->TransitionFromSuspendedToRunnable();
+
CreateHeader(oat_loaded_size, oat_data_offset);
+ CopyAndFixupNativeData();
// TODO: heap validation can't handle these fix up passes.
Runtime::Current()->GetHeap()->DisableObjectValidation();
- CopyAndFixupNativeData();
CopyAndFixupObjects();
Thread::Current()->TransitionFromRunnableToSuspended(kNative);
@@ -195,9 +196,8 @@
return EXIT_FAILURE;
}
- // Write out the image + fields.
- const auto write_count = image_header->GetImageSize() + image_header->GetArtFieldsSize();
- CHECK_EQ(image_end_, image_header->GetImageSize());
+ // Write out the image + fields + methods.
+ const auto write_count = image_header->GetImageSize();
if (!image_file->WriteFully(image_->Begin(), write_count)) {
PLOG(ERROR) << "Failed to write image file " << image_filename;
image_file->Erase();
@@ -205,17 +205,16 @@
}
// Write out the image bitmap at the page aligned start of the image end.
- CHECK_ALIGNED(image_header->GetImageBitmapOffset(), kPageSize);
+ const auto& bitmap_section = image_header->GetImageSection(ImageHeader::kSectionImageBitmap);
+ CHECK_ALIGNED(bitmap_section.Offset(), kPageSize);
if (!image_file->Write(reinterpret_cast<char*>(image_bitmap_->Begin()),
- image_header->GetImageBitmapSize(),
- image_header->GetImageBitmapOffset())) {
+ bitmap_section.Size(), bitmap_section.Offset())) {
PLOG(ERROR) << "Failed to write image file " << image_filename;
image_file->Erase();
return false;
}
- CHECK_EQ(image_header->GetImageBitmapOffset() + image_header->GetImageBitmapSize(),
- static_cast<size_t>(image_file->GetLength()));
+ CHECK_EQ(bitmap_section.End(), static_cast<size_t>(image_file->GetLength()));
if (image_file->FlushCloseOrErase() != 0) {
PLOG(ERROR) << "Failed to flush and close image file " << image_filename;
return false;
@@ -245,9 +244,16 @@
}
// The object is already deflated from when we set the bin slot. Just overwrite the lock word.
object->SetLockWord(LockWord::FromForwardingAddress(offset), false);
+ DCHECK_EQ(object->GetLockWord(false).ReadBarrierState(), 0u);
DCHECK(IsImageOffsetAssigned(object));
}
+void ImageWriter::UpdateImageOffset(mirror::Object* obj, uintptr_t offset) {
+ DCHECK(IsImageOffsetAssigned(obj)) << obj << " " << offset;
+ obj->SetLockWord(LockWord::FromForwardingAddress(offset), false);
+ DCHECK_EQ(obj->GetLockWord(false).ReadBarrierState(), 0u);
+}
+
void ImageWriter::AssignImageOffset(mirror::Object* object, ImageWriter::BinSlot bin_slot) {
DCHECK(object != nullptr);
DCHECK_NE(image_objects_offset_begin_, 0u);
@@ -304,6 +310,7 @@
}
object->SetLockWord(LockWord::FromForwardingAddress(static_cast<uint32_t>(bin_slot)),
false);
+ DCHECK_EQ(object->GetLockWord(false).ReadBarrierState(), 0u);
DCHECK(IsImageBinSlotAssigned(object));
}
@@ -324,16 +331,18 @@
auto strings_size = layout.StringsSize(dex_file->NumStringIds());
dex_cache_array_indexes_.Put(
dex_cache->GetResolvedTypes(),
- DexCacheArrayLocation {size + layout.TypesOffset(), types_size});
+ DexCacheArrayLocation {size + layout.TypesOffset(), types_size, kBinRegular});
dex_cache_array_indexes_.Put(
dex_cache->GetResolvedMethods(),
- DexCacheArrayLocation {size + layout.MethodsOffset(), methods_size});
+ DexCacheArrayLocation {size + layout.MethodsOffset(), methods_size, kBinArtMethodClean});
+ AddMethodPointerArray(dex_cache->GetResolvedMethods());
dex_cache_array_indexes_.Put(
dex_cache->GetResolvedFields(),
- DexCacheArrayLocation {size + layout.FieldsOffset(), fields_size});
+ DexCacheArrayLocation {size + layout.FieldsOffset(), fields_size, kBinArtField});
+ pointer_arrays_.emplace(dex_cache->GetResolvedFields(), kBinArtField);
dex_cache_array_indexes_.Put(
dex_cache->GetStrings(),
- DexCacheArrayLocation {size + layout.StringsOffset(), strings_size});
+ DexCacheArrayLocation {size + layout.StringsOffset(), strings_size, kBinRegular});
size += layout.Size();
CHECK_EQ(layout.Size(), types_size + methods_size + fields_size + strings_size);
}
@@ -342,6 +351,23 @@
bin_slot_sizes_[kBinDexCacheArray] = size;
}
+void ImageWriter::AddMethodPointerArray(mirror::PointerArray* arr) {
+ DCHECK(arr != nullptr);
+ if (kIsDebugBuild) {
+ for (size_t i = 0, len = arr->GetLength(); i < len; i++) {
+ auto* method = arr->GetElementPtrSize<ArtMethod*>(i, target_ptr_size_);
+ if (method != nullptr && !method->IsRuntimeMethod()) {
+ auto* klass = method->GetDeclaringClass();
+ CHECK(klass == nullptr || IsImageClass(klass)) << PrettyClass(klass)
+ << " should be an image class";
+ }
+ }
+ }
+ // kBinArtMethodClean picked arbitrarily, just required to differentiate between ArtFields and
+ // ArtMethods.
+ pointer_arrays_.emplace(arr, kBinArtMethodClean);
+}
+
void ImageWriter::AssignImageBinSlot(mirror::Object* object) {
DCHECK(object != nullptr);
size_t object_size = object->SizeOf();
@@ -393,6 +419,20 @@
bin = kBinClassVerified;
mirror::Class* klass = object->AsClass();
+ // Add non-embedded vtable to the pointer array table if there is one.
+ auto* vtable = klass->GetVTable();
+ if (vtable != nullptr) {
+ AddMethodPointerArray(vtable);
+ }
+ auto* iftable = klass->GetIfTable();
+ if (iftable != nullptr) {
+ for (int32_t i = 0; i < klass->GetIfTableCount(); ++i) {
+ if (iftable->GetMethodArrayCount(i) > 0) {
+ AddMethodPointerArray(iftable->GetMethodArray(i));
+ }
+ }
+ }
+
if (klass->GetStatus() == Class::kStatusInitialized) {
bin = kBinClassInitialized;
@@ -417,26 +457,11 @@
}
}
}
- } else if (object->IsArtMethod<kVerifyNone>()) {
- mirror::ArtMethod* art_method = down_cast<ArtMethod*>(object);
- if (art_method->IsNative()) {
- bin = kBinArtMethodNative;
- } else {
- mirror::Class* declaring_class = art_method->GetDeclaringClass();
- if (declaring_class->GetStatus() != Class::kStatusInitialized) {
- bin = kBinArtMethodNotInitialized;
- } else {
- // This is highly unlikely to dirty since there's no entry points to mutate.
- bin = kBinArtMethodsManagedInitialized;
- }
- }
} else if (object->GetClass<kVerifyNone>()->IsStringClass()) {
bin = kBinString; // Strings are almost always immutable (except for object header).
} else if (object->IsArrayInstance()) {
mirror::Class* klass = object->GetClass<kVerifyNone>();
- auto* component_type = klass->GetComponentType();
- if (!component_type->IsPrimitive() || component_type->IsPrimitiveInt() ||
- component_type->IsPrimitiveLong()) {
+ if (klass->IsObjectArrayClass() || klass->IsIntArrayClass() || klass->IsLongArrayClass()) {
auto it = dex_cache_array_indexes_.find(object);
if (it != dex_cache_array_indexes_.end()) {
bin = kBinDexCacheArray;
@@ -451,6 +476,7 @@
size_t offset_delta = RoundUp(object_size, kObjectAlignment); // 64-bit alignment
if (bin != kBinDexCacheArray) {
+ DCHECK(dex_cache_array_indexes_.find(object) == dex_cache_array_indexes_.end()) << object;
current_offset = bin_slot_sizes_[bin]; // How many bytes the current bin is at (aligned).
// Move the current bin size up to accomodate the object we just assigned a bin slot.
bin_slot_sizes_[bin] += offset_delta;
@@ -468,6 +494,15 @@
DCHECK_LT(image_end_, image_->Size());
}
+bool ImageWriter::WillMethodBeDirty(ArtMethod* m) const {
+ if (m->IsNative()) {
+ return true;
+ }
+ mirror::Class* declaring_class = m->GetDeclaringClass();
+ // Initialized is highly unlikely to dirty since there's no entry points to mutate.
+ return declaring_class == nullptr || declaring_class->GetStatus() != Class::kStatusInitialized;
+}
+
bool ImageWriter::IsImageBinSlotAssigned(mirror::Object* object) const {
DCHECK(object != nullptr);
@@ -604,6 +639,9 @@
}
bool ImageWriter::IsImageClass(Class* klass) {
+ if (klass == nullptr) {
+ return false;
+ }
std::string temp;
return compiler_driver_.IsImageClass(klass->GetDescriptor(&temp));
}
@@ -619,6 +657,7 @@
}
Runtime* runtime = Runtime::Current();
ClassLinker* class_linker = runtime->GetClassLinker();
+ Thread* self = Thread::Current();
// Make a list of classes we would like to prune.
std::set<std::string> non_image_classes;
@@ -634,27 +673,45 @@
}
// Clear references to removed classes from the DexCaches.
- ArtMethod* resolution_method = runtime->GetResolutionMethod();
- ReaderMutexLock mu(Thread::Current(), *class_linker->DexLock());
- size_t dex_cache_count = class_linker->GetDexCacheCount();
+ const ArtMethod* resolution_method = runtime->GetResolutionMethod();
+ size_t dex_cache_count;
+ {
+ ReaderMutexLock mu(self, *class_linker->DexLock());
+ dex_cache_count = class_linker->GetDexCacheCount();
+ }
for (size_t idx = 0; idx < dex_cache_count; ++idx) {
- DexCache* dex_cache = class_linker->GetDexCache(idx);
+ DexCache* dex_cache;
+ {
+ ReaderMutexLock mu(self, *class_linker->DexLock());
+ dex_cache = class_linker->GetDexCache(idx);
+ }
for (size_t i = 0; i < dex_cache->NumResolvedTypes(); i++) {
Class* klass = dex_cache->GetResolvedType(i);
if (klass != nullptr && !IsImageClass(klass)) {
dex_cache->SetResolvedType(i, nullptr);
}
}
- for (size_t i = 0; i < dex_cache->NumResolvedMethods(); i++) {
- ArtMethod* method = dex_cache->GetResolvedMethod(i);
- if (method != nullptr && !IsImageClass(method->GetDeclaringClass())) {
- dex_cache->SetResolvedMethod(i, resolution_method);
+ auto* resolved_methods = down_cast<mirror::PointerArray*>(dex_cache->GetResolvedMethods());
+ for (size_t i = 0, len = resolved_methods->GetLength(); i < len; i++) {
+ auto* method = resolved_methods->GetElementPtrSize<ArtMethod*>(i, target_ptr_size_);
+ if (method != nullptr) {
+ auto* declaring_class = method->GetDeclaringClass();
+ // Miranda methods may be held live by a class which was not an image class but have a
+ // declaring class which is an image class. Set it to the resolution method to be safe and
+ // prevent dangling pointers.
+ if (method->IsMiranda() || !IsImageClass(declaring_class)) {
+ resolved_methods->SetElementPtrSize(i, resolution_method, target_ptr_size_);
+ } else {
+ // Check that the class is still in the classes table.
+ DCHECK(class_linker->ClassInClassTable(declaring_class)) << "Class "
+ << PrettyClass(declaring_class) << " not in class linker table";
+ }
}
}
for (size_t i = 0; i < dex_cache->NumResolvedFields(); i++) {
- ArtField* field = dex_cache->GetResolvedField(i, sizeof(void*));
+ ArtField* field = dex_cache->GetResolvedField(i, target_ptr_size_);
if (field != nullptr && !IsImageClass(field->GetDeclaringClass())) {
- dex_cache->SetResolvedField(i, nullptr, sizeof(void*));
+ dex_cache->SetResolvedField(i, nullptr, target_ptr_size_);
}
}
// Clean the dex field. It might have been populated during the initialization phase, but
@@ -757,19 +814,8 @@
}
// build an Object[] of the roots needed to restore the runtime
- Handle<ObjectArray<Object>> image_roots(hs.NewHandle(
+ auto image_roots(hs.NewHandle(
ObjectArray<Object>::Alloc(self, object_array_class.Get(), ImageHeader::kImageRootsMax)));
- image_roots->Set<false>(ImageHeader::kResolutionMethod, runtime->GetResolutionMethod());
- image_roots->Set<false>(ImageHeader::kImtConflictMethod, runtime->GetImtConflictMethod());
- image_roots->Set<false>(ImageHeader::kImtUnimplementedMethod,
- runtime->GetImtUnimplementedMethod());
- image_roots->Set<false>(ImageHeader::kDefaultImt, runtime->GetDefaultImt());
- image_roots->Set<false>(ImageHeader::kCalleeSaveMethod,
- runtime->GetCalleeSaveMethod(Runtime::kSaveAll));
- image_roots->Set<false>(ImageHeader::kRefsOnlySaveMethod,
- runtime->GetCalleeSaveMethod(Runtime::kRefsOnly));
- image_roots->Set<false>(ImageHeader::kRefsAndArgsSaveMethod,
- runtime->GetCalleeSaveMethod(Runtime::kRefsAndArgs));
image_roots->Set<false>(ImageHeader::kDexCaches, dex_caches.Get());
image_roots->Set<false>(ImageHeader::kClassRoots, class_linker->GetClassRoots());
for (int i = 0; i < ImageHeader::kImageRootsMax; i++) {
@@ -816,7 +862,7 @@
// Walk static fields of a Class.
if (h_obj->IsClass()) {
size_t num_reference_static_fields = klass->NumReferenceStaticFields();
- MemberOffset field_offset = klass->GetFirstReferenceStaticFieldOffset();
+ MemberOffset field_offset = klass->GetFirstReferenceStaticFieldOffset(target_ptr_size_);
for (size_t i = 0; i < num_reference_static_fields; ++i) {
mirror::Object* value = h_obj->GetFieldObject<mirror::Object>(field_offset);
if (value != nullptr) {
@@ -825,21 +871,38 @@
field_offset = MemberOffset(field_offset.Uint32Value() +
sizeof(mirror::HeapReference<mirror::Object>));
}
-
// Visit and assign offsets for fields.
- ArtField* fields[2] = { h_obj->AsClass()->GetSFields(), h_obj->AsClass()->GetIFields() };
- size_t num_fields[2] = { h_obj->AsClass()->NumStaticFields(),
- h_obj->AsClass()->NumInstanceFields() };
+ auto* as_klass = h_obj->AsClass();
+ ArtField* fields[] = { as_klass->GetSFields(), as_klass->GetIFields() };
+ size_t num_fields[] = { as_klass->NumStaticFields(), as_klass->NumInstanceFields() };
for (size_t i = 0; i < 2; ++i) {
for (size_t j = 0; j < num_fields[i]; ++j) {
auto* field = fields[i] + j;
- auto it = art_field_reloc_.find(field);
- CHECK(it == art_field_reloc_.end()) << "Field at index " << i << ":" << j
+ auto it = native_object_reloc_.find(field);
+ CHECK(it == native_object_reloc_.end()) << "Field at index " << i << ":" << j
<< " already assigned " << PrettyField(field);
- art_field_reloc_.emplace(field, bin_slot_sizes_[kBinArtField]);
+ native_object_reloc_.emplace(
+ field, NativeObjectReloc { bin_slot_sizes_[kBinArtField], kBinArtField });
bin_slot_sizes_[kBinArtField] += sizeof(ArtField);
}
}
+ // Visit and assign offsets for methods.
+ IterationRange<StrideIterator<ArtMethod>> method_arrays[] = {
+ as_klass->GetDirectMethods(target_ptr_size_),
+ as_klass->GetVirtualMethods(target_ptr_size_)
+ };
+ for (auto& array : method_arrays) {
+ bool any_dirty = false;
+ size_t count = 0;
+ for (auto& m : array) {
+ any_dirty = any_dirty || WillMethodBeDirty(&m);
+ ++count;
+ }
+ for (auto& m : array) {
+ AssignMethodOffset(&m, any_dirty ? kBinArtMethodDirty : kBinArtMethodClean);
+ }
+ (any_dirty ? dirty_methods_ : clean_methods_) += count;
+ }
} else if (h_obj->IsObjectArray()) {
// Walk elements of an object array.
int32_t length = h_obj->AsObjectArray<mirror::Object>()->GetLength();
@@ -854,6 +917,14 @@
}
}
+void ImageWriter::AssignMethodOffset(ArtMethod* method, Bin bin) {
+ auto it = native_object_reloc_.find(method);
+ CHECK(it == native_object_reloc_.end()) << "Method " << method << " already assigned "
+ << PrettyMethod(method);
+ native_object_reloc_.emplace(method, NativeObjectReloc { bin_slot_sizes_[bin], bin });
+ bin_slot_sizes_[bin] += ArtMethod::ObjectSize(target_ptr_size_);
+}
+
void ImageWriter::WalkFieldsCallback(mirror::Object* obj, void* arg) {
ImageWriter* writer = reinterpret_cast<ImageWriter*>(arg);
DCHECK(writer != nullptr);
@@ -879,11 +950,12 @@
}
void ImageWriter::CalculateNewObjectOffsets() {
- Thread* self = Thread::Current();
+ Thread* const self = Thread::Current();
StackHandleScope<1> hs(self);
Handle<ObjectArray<Object>> image_roots(hs.NewHandle(CreateImageRoots()));
- gc::Heap* heap = Runtime::Current()->GetHeap();
+ auto* runtime = Runtime::Current();
+ auto* heap = runtime->GetHeap();
DCHECK_EQ(0U, image_end_);
// Leave space for the header, but do not write it yet, we need to
@@ -896,6 +968,21 @@
PrepareDexCacheArraySlots();
// Clear any pre-existing monitors which may have been in the monitor words, assign bin slots.
heap->VisitObjects(WalkFieldsCallback, this);
+ // Write the image runtime methods.
+ image_methods_[ImageHeader::kResolutionMethod] = runtime->GetResolutionMethod();
+ image_methods_[ImageHeader::kImtConflictMethod] = runtime->GetImtConflictMethod();
+ image_methods_[ImageHeader::kImtUnimplementedMethod] = runtime->GetImtUnimplementedMethod();
+ image_methods_[ImageHeader::kCalleeSaveMethod] = runtime->GetCalleeSaveMethod(Runtime::kSaveAll);
+ image_methods_[ImageHeader::kRefsOnlySaveMethod] =
+ runtime->GetCalleeSaveMethod(Runtime::kRefsOnly);
+ image_methods_[ImageHeader::kRefsAndArgsSaveMethod] =
+ runtime->GetCalleeSaveMethod(Runtime::kRefsAndArgs);
+ for (auto* m : image_methods_) {
+ CHECK(m != nullptr);
+ CHECK(m->IsRuntimeMethod());
+ AssignMethodOffset(m, kBinArtMethodDirty);
+ }
+
// Calculate cumulative bin slot sizes.
size_t previous_sizes = 0u;
for (size_t i = 0; i != kBinSize; ++i) {
@@ -913,7 +1000,14 @@
image_roots_address_ = PointerToLowMemUInt32(GetImageAddress(image_roots.Get()));
- // Note that image_end_ is left at end of used mirror space
+ // Update the native relocations by adding their bin sums.
+ for (auto& pair : native_object_reloc_) {
+ auto& native_reloc = pair.second;
+ native_reloc.offset += image_objects_offset_begin_ +
+ bin_slot_previous_sizes_[native_reloc.bin_type];
+ }
+
+ // Note that image_end_ is left at end of used mirror object section.
}
void ImageWriter::CreateHeader(size_t oat_loaded_size, size_t oat_data_offset) {
@@ -922,47 +1016,87 @@
const uint8_t* oat_file_end = oat_file_begin + oat_loaded_size;
oat_data_begin_ = oat_file_begin + oat_data_offset;
const uint8_t* oat_data_end = oat_data_begin_ + oat_file_->Size();
- // Write out sections.
- size_t cur_pos = image_end_;
- // Add fields.
- auto fields_offset = cur_pos;
- CHECK_EQ(image_objects_offset_begin_ + GetBinSizeSum(kBinArtField), fields_offset);
- auto fields_size = bin_slot_sizes_[kBinArtField];
- cur_pos += fields_size;
- // Return to write header at start of image with future location of image_roots. At this point,
- // image_end_ is the size of the image (excluding bitmaps, ArtFields).
- /*
- const size_t heap_bytes_per_bitmap_byte = kBitsPerByte * kObjectAlignment;
- const size_t bitmap_bytes = RoundUp(image_end_, heap_bytes_per_bitmap_byte) /
- heap_bytes_per_bitmap_byte;
- */
+
+ // Create the image sections.
+ ImageSection sections[ImageHeader::kSectionCount];
+ // Objects section
+ auto* objects_section = §ions[ImageHeader::kSectionObjects];
+ *objects_section = ImageSection(0u, image_end_);
+ size_t cur_pos = objects_section->End();
+ // Add field section.
+ auto* field_section = §ions[ImageHeader::kSectionArtFields];
+ *field_section = ImageSection(cur_pos, bin_slot_sizes_[kBinArtField]);
+ CHECK_EQ(image_objects_offset_begin_ + bin_slot_previous_sizes_[kBinArtField],
+ field_section->Offset());
+ cur_pos = field_section->End();
+ // Add method section.
+ auto* methods_section = §ions[ImageHeader::kSectionArtMethods];
+ *methods_section = ImageSection(cur_pos, bin_slot_sizes_[kBinArtMethodClean] +
+ bin_slot_sizes_[kBinArtMethodDirty]);
+ CHECK_EQ(image_objects_offset_begin_ + bin_slot_previous_sizes_[kBinArtMethodClean],
+ methods_section->Offset());
+ cur_pos = methods_section->End();
+ // Finally bitmap section.
const size_t bitmap_bytes = image_bitmap_->Size();
- auto bitmap_offset = RoundUp(cur_pos, kPageSize);
- auto bitmap_size = RoundUp(bitmap_bytes, kPageSize);
- cur_pos += bitmap_size;
- new (image_->Begin()) ImageHeader(PointerToLowMemUInt32(image_begin_),
- static_cast<uint32_t>(image_end_),
- fields_offset, fields_size,
- bitmap_offset, bitmap_size,
- image_roots_address_,
- oat_file_->GetOatHeader().GetChecksum(),
- PointerToLowMemUInt32(oat_file_begin),
- PointerToLowMemUInt32(oat_data_begin_),
- PointerToLowMemUInt32(oat_data_end),
- PointerToLowMemUInt32(oat_file_end),
- compile_pic_);
+ auto* bitmap_section = §ions[ImageHeader::kSectionImageBitmap];
+ *bitmap_section = ImageSection(RoundUp(cur_pos, kPageSize), RoundUp(bitmap_bytes, kPageSize));
+ cur_pos = bitmap_section->End();
+ if (kIsDebugBuild) {
+ size_t idx = 0;
+ for (auto& section : sections) {
+ LOG(INFO) << static_cast<ImageHeader::ImageSections>(idx) << " " << section;
+ ++idx;
+ }
+ LOG(INFO) << "Methods: clean=" << clean_methods_ << " dirty=" << dirty_methods_;
+ }
+ // Create the header.
+ new (image_->Begin()) ImageHeader(
+ PointerToLowMemUInt32(image_begin_), static_cast<uint32_t>(methods_section->End()), sections,
+ image_roots_address_, oat_file_->GetOatHeader().GetChecksum(),
+ PointerToLowMemUInt32(oat_file_begin), PointerToLowMemUInt32(oat_data_begin_),
+ PointerToLowMemUInt32(oat_data_end), PointerToLowMemUInt32(oat_file_end), target_ptr_size_,
+ compile_pic_);
+}
+
+ArtMethod* ImageWriter::GetImageMethodAddress(ArtMethod* method) {
+ auto it = native_object_reloc_.find(method);
+ CHECK(it != native_object_reloc_.end()) << PrettyMethod(method) << " @ " << method;
+ CHECK_GE(it->second.offset, image_end_) << "ArtMethods should be after Objects";
+ return reinterpret_cast<ArtMethod*>(image_begin_ + it->second.offset);
}
void ImageWriter::CopyAndFixupNativeData() {
- // Copy ArtFields to their locations and update the array for convenience.
- auto fields_offset = image_objects_offset_begin_ + GetBinSizeSum(kBinArtField);
- for (auto& pair : art_field_reloc_) {
- pair.second += fields_offset;
- auto* dest = image_->Begin() + pair.second;
- DCHECK_GE(dest, image_->Begin() + image_end_);
- memcpy(dest, pair.first, sizeof(ArtField));
- reinterpret_cast<ArtField*>(dest)->SetDeclaringClass(
- down_cast<Class*>(GetImageAddress(pair.first->GetDeclaringClass())));
+ // Copy ArtFields and methods to their locations and update the array for convenience.
+ for (auto& pair : native_object_reloc_) {
+ auto& native_reloc = pair.second;
+ if (native_reloc.bin_type == kBinArtField) {
+ auto* dest = image_->Begin() + native_reloc.offset;
+ DCHECK_GE(dest, image_->Begin() + image_end_);
+ memcpy(dest, pair.first, sizeof(ArtField));
+ reinterpret_cast<ArtField*>(dest)->SetDeclaringClass(
+ GetImageAddress(reinterpret_cast<ArtField*>(pair.first)->GetDeclaringClass()));
+ } else {
+ CHECK(IsArtMethodBin(native_reloc.bin_type)) << native_reloc.bin_type;
+ auto* dest = image_->Begin() + native_reloc.offset;
+ DCHECK_GE(dest, image_->Begin() + image_end_);
+ CopyAndFixupMethod(reinterpret_cast<ArtMethod*>(pair.first),
+ reinterpret_cast<ArtMethod*>(dest));
+ }
+ }
+ // Fixup the image method roots.
+ auto* image_header = reinterpret_cast<ImageHeader*>(image_->Begin());
+ const auto& methods_section = image_header->GetMethodsSection();
+ for (size_t i = 0; i < ImageHeader::kImageMethodsCount; ++i) {
+ auto* m = image_methods_[i];
+ CHECK(m != nullptr);
+ auto it = native_object_reloc_.find(m);
+ CHECK(it != native_object_reloc_.end()) << "No fowarding for " << PrettyMethod(m);
+ auto& native_reloc = it->second;
+ CHECK(methods_section.Contains(native_reloc.offset)) << native_reloc.offset << " not in "
+ << methods_section;
+ CHECK(IsArtMethodBin(native_reloc.bin_type)) << native_reloc.bin_type;
+ auto* dest = reinterpret_cast<ArtMethod*>(image_begin_ + it->second.offset);
+ image_header->SetImageMethod(static_cast<ImageHeader::ImageMethod>(i), dest);
}
}
@@ -984,59 +1118,37 @@
reinterpret_cast<ImageWriter*>(arg)->CopyAndFixupObject(obj);
}
-bool ImageWriter::CopyAndFixupIfDexCacheFieldArray(mirror::Object* dst, mirror::Object* obj,
- mirror::Class* klass) {
- if (!klass->IsArrayClass()) {
- return false;
- }
- auto* component_type = klass->GetComponentType();
- bool is_int_arr = component_type->IsPrimitiveInt();
- bool is_long_arr = component_type->IsPrimitiveLong();
- if (!is_int_arr && !is_long_arr) {
- return false;
- }
- auto it = dex_cache_array_indexes_.find(obj); // Is this a dex cache array?
- if (it == dex_cache_array_indexes_.end()) {
- return false;
- }
- mirror::Array* arr = obj->AsArray();
- CHECK_EQ(reinterpret_cast<Object*>(
- image_->Begin() + it->second.offset_ + image_objects_offset_begin_), dst);
- dex_cache_array_indexes_.erase(it);
- // Fixup int pointers for the field array.
- CHECK(!arr->IsObjectArray());
+void ImageWriter::FixupPointerArray(mirror::Object* dst, mirror::PointerArray* arr,
+ mirror::Class* klass, Bin array_type) {
+ CHECK(klass->IsArrayClass());
+ CHECK(arr->IsIntArray() || arr->IsLongArray()) << PrettyClass(klass) << " " << arr;
+ // Fixup int and long pointers for the ArtMethod or ArtField arrays.
const size_t num_elements = arr->GetLength();
- if (target_ptr_size_ == 4u) {
- // Will get fixed up by fixup object.
- dst->SetClass<kVerifyNone>(down_cast<mirror::Class*>(
- GetImageAddress(mirror::IntArray::GetArrayClass())));
- } else {
- DCHECK_EQ(target_ptr_size_, 8u);
- dst->SetClass<kVerifyNone>(down_cast<mirror::Class*>(
- GetImageAddress(mirror::LongArray::GetArrayClass())));
- }
- mirror::Array* dest_array = down_cast<mirror::Array*>(dst);
- dest_array->SetLength(num_elements);
+ dst->SetClass(GetImageAddress(arr->GetClass()));
+ auto* dest_array = down_cast<mirror::PointerArray*>(dst);
for (size_t i = 0, count = num_elements; i < count; ++i) {
- ArtField* field = reinterpret_cast<ArtField*>(is_int_arr ?
- arr->AsIntArray()->GetWithoutChecks(i) : arr->AsLongArray()->GetWithoutChecks(i));
- uint8_t* fixup_location = nullptr;
- if (field != nullptr) {
- auto it2 = art_field_reloc_.find(field);
- CHECK(it2 != art_field_reloc_.end()) << "No relocation for field " << PrettyField(field);
- fixup_location = image_begin_ + it2->second;
+ auto* elem = arr->GetElementPtrSize<void*>(i, target_ptr_size_);
+ if (elem != nullptr) {
+ auto it = native_object_reloc_.find(elem);
+ if (it == native_object_reloc_.end()) {
+ if (IsArtMethodBin(array_type)) {
+ auto* method = reinterpret_cast<ArtMethod*>(elem);
+ LOG(FATAL) << "No relocation entry for ArtMethod " << PrettyMethod(method) << " @ "
+ << method << " idx=" << i << "/" << num_elements << " with declaring class "
+ << PrettyClass(method->GetDeclaringClass());
+ } else {
+ CHECK_EQ(array_type, kBinArtField);
+ auto* field = reinterpret_cast<ArtField*>(elem);
+ LOG(FATAL) << "No relocation entry for ArtField " << PrettyField(field) << " @ "
+ << field << " idx=" << i << "/" << num_elements << " with declaring class "
+ << PrettyClass(field->GetDeclaringClass());
+ }
+ } else {
+ elem = image_begin_ + it->second.offset;
+ }
}
- if (target_ptr_size_ == 4u) {
- down_cast<mirror::IntArray*>(dest_array)->SetWithoutChecks<false, false, kVerifyNone>(
- i, static_cast<uint32_t>(reinterpret_cast<uint64_t>(fixup_location)));
- } else {
- DCHECK_EQ(target_ptr_size_, 8u);
- down_cast<mirror::LongArray*>(dest_array)->SetWithoutChecks<false, false, kVerifyNone>(
- i, reinterpret_cast<uint64_t>(fixup_location));
- }
+ dest_array->SetElementPtrSize<false, true>(i, elem, target_ptr_size_);
}
- dst->SetLockWord<kVerifyNone>(LockWord::Default(), false);
- return true;
}
void ImageWriter::CopyAndFixupObject(Object* obj) {
@@ -1044,19 +1156,8 @@
size_t offset = GetImageOffset(obj);
auto* dst = reinterpret_cast<Object*>(image_->Begin() + offset);
const uint8_t* src = reinterpret_cast<const uint8_t*>(obj);
- size_t n;
- mirror::Class* klass = obj->GetClass();
- if (CopyAndFixupIfDexCacheFieldArray(dst, obj, klass)) {
- return;
- }
- if (klass->IsArtMethodClass()) {
- // Size without pointer fields since we don't want to overrun the buffer if target art method
- // is 32 bits but source is 64 bits.
- n = mirror::ArtMethod::SizeWithoutPointerFields(target_ptr_size_);
- } else {
- n = obj->SizeOf();
- }
+ size_t n = obj->SizeOf();
DCHECK_LE(offset + n, image_->Size());
memcpy(dst, src, n);
@@ -1115,23 +1216,51 @@
void ImageWriter::FixupClass(mirror::Class* orig, mirror::Class* copy) {
// Copy and fix up ArtFields in the class.
- ArtField* fields[2] = { orig->AsClass()->GetSFields(), orig->AsClass()->GetIFields() };
- size_t num_fields[2] = { orig->AsClass()->NumStaticFields(),
- orig->AsClass()->NumInstanceFields() };
- // Update the arrays.
+ ArtField* fields[2] = { orig->GetSFields(), orig->GetIFields() };
+ size_t num_fields[2] = { orig->NumStaticFields(), orig->NumInstanceFields() };
+ // Update the field arrays.
for (size_t i = 0; i < 2; ++i) {
if (num_fields[i] == 0) {
CHECK(fields[i] == nullptr);
continue;
}
- auto it = art_field_reloc_.find(fields[i]);
- CHECK(it != art_field_reloc_.end()) << PrettyClass(orig->AsClass()) << " : "
- << PrettyField(fields[i]);
- auto* image_fields = reinterpret_cast<ArtField*>(image_begin_ + it->second);
+ auto it = native_object_reloc_.find(fields[i]);
+ CHECK(it != native_object_reloc_.end()) << PrettyClass(orig) << " : " << PrettyField(fields[i]);
+ auto* image_fields = reinterpret_cast<ArtField*>(image_begin_ + it->second.offset);
if (i == 0) {
- down_cast<Class*>(copy)->SetSFieldsUnchecked(image_fields);
+ copy->SetSFieldsUnchecked(image_fields);
} else {
- down_cast<Class*>(copy)->SetIFieldsUnchecked(image_fields);
+ copy->SetIFieldsUnchecked(image_fields);
+ }
+ }
+ // Update direct / virtual method arrays.
+ auto* direct_methods = orig->GetDirectMethodsPtr();
+ if (direct_methods != nullptr) {
+ auto it = native_object_reloc_.find(direct_methods);
+ CHECK(it != native_object_reloc_.end()) << PrettyClass(orig);
+ copy->SetDirectMethodsPtrUnchecked(
+ reinterpret_cast<ArtMethod*>(image_begin_ + it->second.offset));
+ }
+ auto* virtual_methods = orig->GetVirtualMethodsPtr();
+ if (virtual_methods != nullptr) {
+ auto it = native_object_reloc_.find(virtual_methods);
+ CHECK(it != native_object_reloc_.end()) << PrettyClass(orig);
+ copy->SetVirtualMethodsPtr(
+ reinterpret_cast<ArtMethod*>(image_begin_ + it->second.offset));
+ }
+ // Fix up embedded tables.
+ if (orig->ShouldHaveEmbeddedImtAndVTable()) {
+ for (int32_t i = 0; i < orig->GetEmbeddedVTableLength(); ++i) {
+ auto it = native_object_reloc_.find(orig->GetEmbeddedVTableEntry(i, target_ptr_size_));
+ CHECK(it != native_object_reloc_.end()) << PrettyClass(orig);
+ copy->SetEmbeddedVTableEntryUnchecked(
+ i, reinterpret_cast<ArtMethod*>(image_begin_ + it->second.offset), target_ptr_size_);
+ }
+ for (size_t i = 0; i < mirror::Class::kImtSize; ++i) {
+ auto it = native_object_reloc_.find(orig->GetEmbeddedImTableEntry(i, target_ptr_size_));
+ CHECK(it != native_object_reloc_.end()) << PrettyClass(orig);
+ copy->SetEmbeddedImTableEntry(
+ i, reinterpret_cast<ArtMethod*>(image_begin_ + it->second.offset), target_ptr_size_);
}
}
FixupClassVisitor visitor(this, copy);
@@ -1149,18 +1278,39 @@
DCHECK_EQ(copy->GetReadBarrierPointer(), GetImageAddress(orig));
}
}
+ auto* klass = orig->GetClass();
+ if (klass->IsIntArrayClass() || klass->IsLongArrayClass()) {
+ // Is this a native dex cache array?
+ auto it = pointer_arrays_.find(down_cast<mirror::PointerArray*>(orig));
+ if (it != pointer_arrays_.end()) {
+ // Should only need to fixup every pointer array exactly once.
+ FixupPointerArray(copy, down_cast<mirror::PointerArray*>(orig), klass, it->second);
+ pointer_arrays_.erase(it);
+ return;
+ }
+ CHECK(dex_cache_array_indexes_.find(orig) == dex_cache_array_indexes_.end())
+ << "Should have been pointer array.";
+ }
if (orig->IsClass()) {
FixupClass(orig->AsClass<kVerifyNone>(), down_cast<mirror::Class*>(copy));
} else {
+ if (klass == mirror::Method::StaticClass() || klass == mirror::Constructor::StaticClass()) {
+ // Need to go update the ArtMethod.
+ auto* dest = down_cast<mirror::AbstractMethod*>(copy);
+ auto* src = down_cast<mirror::AbstractMethod*>(orig);
+ ArtMethod* src_method = src->GetArtMethod();
+ auto it = native_object_reloc_.find(src_method);
+ CHECK(it != native_object_reloc_.end()) << "Missing relocation for AbstractMethod.artMethod "
+ << PrettyMethod(src_method);
+ dest->SetArtMethod(
+ reinterpret_cast<ArtMethod*>(image_begin_ + it->second.offset));
+ }
FixupVisitor visitor(this, copy);
orig->VisitReferences<true /*visit class*/>(visitor, visitor);
}
- if (orig->IsArtMethod<kVerifyNone>()) {
- FixupMethod(orig->AsArtMethod<kVerifyNone>(), down_cast<ArtMethod*>(copy));
- }
}
-const uint8_t* ImageWriter::GetQuickCode(mirror::ArtMethod* method, bool* quick_is_interpreted) {
+const uint8_t* ImageWriter::GetQuickCode(ArtMethod* method, bool* quick_is_interpreted) {
DCHECK(!method->IsResolutionMethod() && !method->IsImtConflictMethod() &&
!method->IsImtUnimplementedMethod() && !method->IsAbstract()) << PrettyMethod(method);
@@ -1172,27 +1322,31 @@
method->GetEntryPointFromQuickCompiledCodePtrSize(target_ptr_size_));
const uint8_t* quick_code = GetOatAddress(quick_oat_code_offset);
*quick_is_interpreted = false;
- if (quick_code != nullptr &&
- (!method->IsStatic() || method->IsConstructor() || method->GetDeclaringClass()->IsInitialized())) {
+ if (quick_code != nullptr && (!method->IsStatic() || method->IsConstructor() ||
+ method->GetDeclaringClass()->IsInitialized())) {
// We have code for a non-static or initialized method, just use the code.
+ DCHECK_GE(quick_code, oat_data_begin_);
} else if (quick_code == nullptr && method->IsNative() &&
(!method->IsStatic() || method->GetDeclaringClass()->IsInitialized())) {
// Non-static or initialized native method missing compiled code, use generic JNI version.
quick_code = GetOatAddress(quick_generic_jni_trampoline_offset_);
+ DCHECK_GE(quick_code, oat_data_begin_);
} else if (quick_code == nullptr && !method->IsNative()) {
// We don't have code at all for a non-native method, use the interpreter.
quick_code = GetOatAddress(quick_to_interpreter_bridge_offset_);
*quick_is_interpreted = true;
+ DCHECK_GE(quick_code, oat_data_begin_);
} else {
CHECK(!method->GetDeclaringClass()->IsInitialized());
// We have code for a static method, but need to go through the resolution stub for class
// initialization.
quick_code = GetOatAddress(quick_resolution_trampoline_offset_);
+ DCHECK_GE(quick_code, oat_data_begin_);
}
return quick_code;
}
-const uint8_t* ImageWriter::GetQuickEntryPoint(mirror::ArtMethod* method) {
+const uint8_t* ImageWriter::GetQuickEntryPoint(ArtMethod* method) {
// Calculate the quick entry point following the same logic as FixupMethod() below.
// The resolution method has a special trampoline to call.
Runtime* runtime = Runtime::Current();
@@ -1214,50 +1368,57 @@
}
}
-void ImageWriter::FixupMethod(ArtMethod* orig, ArtMethod* copy) {
+void ImageWriter::CopyAndFixupMethod(ArtMethod* orig, ArtMethod* copy) {
+ memcpy(copy, orig, ArtMethod::ObjectSize(target_ptr_size_));
+
+ copy->SetDeclaringClass(GetImageAddress(orig->GetDeclaringClassUnchecked()));
+ copy->SetDexCacheResolvedMethods(GetImageAddress(orig->GetDexCacheResolvedMethods()));
+ copy->SetDexCacheResolvedTypes(GetImageAddress(orig->GetDexCacheResolvedTypes()));
+
// OatWriter replaces the code_ with an offset value. Here we re-adjust to a pointer relative to
// oat_begin_
- // For 64 bit targets we need to repack the current runtime pointer sized fields to the right
- // locations.
- // Copy all of the fields from the runtime methods to the target methods first since we did a
- // bytewise copy earlier.
- copy->SetEntryPointFromInterpreterPtrSize<kVerifyNone>(
- orig->GetEntryPointFromInterpreterPtrSize(target_ptr_size_), target_ptr_size_);
- copy->SetEntryPointFromJniPtrSize<kVerifyNone>(
- orig->GetEntryPointFromJniPtrSize(target_ptr_size_), target_ptr_size_);
- copy->SetEntryPointFromQuickCompiledCodePtrSize<kVerifyNone>(
- orig->GetEntryPointFromQuickCompiledCodePtrSize(target_ptr_size_), target_ptr_size_);
// The resolution method has a special trampoline to call.
Runtime* runtime = Runtime::Current();
if (UNLIKELY(orig == runtime->GetResolutionMethod())) {
- copy->SetEntryPointFromQuickCompiledCodePtrSize<kVerifyNone>(
+ copy->SetEntryPointFromQuickCompiledCodePtrSize(
GetOatAddress(quick_resolution_trampoline_offset_), target_ptr_size_);
} else if (UNLIKELY(orig == runtime->GetImtConflictMethod() ||
orig == runtime->GetImtUnimplementedMethod())) {
- copy->SetEntryPointFromQuickCompiledCodePtrSize<kVerifyNone>(
+ copy->SetEntryPointFromQuickCompiledCodePtrSize(
GetOatAddress(quick_imt_conflict_trampoline_offset_), target_ptr_size_);
+ } else if (UNLIKELY(orig->IsRuntimeMethod())) {
+ bool found_one = false;
+ for (size_t i = 0; i < static_cast<size_t>(Runtime::kLastCalleeSaveType); ++i) {
+ auto idx = static_cast<Runtime::CalleeSaveType>(i);
+ if (runtime->HasCalleeSaveMethod(idx) && runtime->GetCalleeSaveMethod(idx) == orig) {
+ found_one = true;
+ break;
+ }
+ }
+ CHECK(found_one) << "Expected to find callee save method but got " << PrettyMethod(orig);
+ CHECK(copy->IsRuntimeMethod());
} else {
// We assume all methods have code. If they don't currently then we set them to the use the
// resolution trampoline. Abstract methods never have code and so we need to make sure their
// use results in an AbstractMethodError. We use the interpreter to achieve this.
if (UNLIKELY(orig->IsAbstract())) {
- copy->SetEntryPointFromQuickCompiledCodePtrSize<kVerifyNone>(
+ copy->SetEntryPointFromQuickCompiledCodePtrSize(
GetOatAddress(quick_to_interpreter_bridge_offset_), target_ptr_size_);
- copy->SetEntryPointFromInterpreterPtrSize<kVerifyNone>(
+ copy->SetEntryPointFromInterpreterPtrSize(
reinterpret_cast<EntryPointFromInterpreter*>(const_cast<uint8_t*>(
GetOatAddress(interpreter_to_interpreter_bridge_offset_))), target_ptr_size_);
} else {
bool quick_is_interpreted;
const uint8_t* quick_code = GetQuickCode(orig, &quick_is_interpreted);
- copy->SetEntryPointFromQuickCompiledCodePtrSize<kVerifyNone>(quick_code, target_ptr_size_);
+ copy->SetEntryPointFromQuickCompiledCodePtrSize(quick_code, target_ptr_size_);
// JNI entrypoint:
if (orig->IsNative()) {
// The native method's pointer is set to a stub to lookup via dlsym.
// Note this is not the code_ pointer, that is handled above.
- copy->SetEntryPointFromJniPtrSize<kVerifyNone>(GetOatAddress(jni_dlsym_lookup_offset_),
- target_ptr_size_);
+ copy->SetEntryPointFromJniPtrSize(
+ GetOatAddress(jni_dlsym_lookup_offset_), target_ptr_size_);
}
// Interpreter entrypoint:
@@ -1268,8 +1429,7 @@
EntryPointFromInterpreter* interpreter_entrypoint =
reinterpret_cast<EntryPointFromInterpreter*>(
const_cast<uint8_t*>(GetOatAddress(interpreter_code)));
- copy->SetEntryPointFromInterpreterPtrSize<kVerifyNone>(
- interpreter_entrypoint, target_ptr_size_);
+ copy->SetEntryPointFromInterpreterPtrSize(interpreter_entrypoint, target_ptr_size_);
}
}
}
@@ -1306,8 +1466,8 @@
ImageWriter::BinSlot::BinSlot(uint32_t lockword) : lockword_(lockword) {
// These values may need to get updated if more bins are added to the enum Bin
- static_assert(kBinBits == 4, "wrong number of bin bits");
- static_assert(kBinShift == 28, "wrong number of shift");
+ static_assert(kBinBits == 3, "wrong number of bin bits");
+ static_assert(kBinShift == 27, "wrong number of shift");
static_assert(sizeof(BinSlot) == sizeof(LockWord), "BinSlot/LockWord must have equal sizes");
DCHECK_LT(GetBin(), kBinSize);
@@ -1327,13 +1487,4 @@
return lockword_ & ~kBinMask;
}
-void ImageWriter::FreeStringDataArray() {
- if (string_data_array_ != nullptr) {
- gc::space::LargeObjectSpace* los = Runtime::Current()->GetHeap()->GetLargeObjectsSpace();
- if (los != nullptr) {
- los->Free(Thread::Current(), reinterpret_cast<mirror::Object*>(string_data_array_));
- }
- }
-}
-
} // namespace art