diff options
author | 2025-02-07 19:05:44 +0000 | |
---|---|---|
committer | 2025-02-21 09:51:24 -0800 | |
commit | a7d85a831f0a330ce9914c57ae146fc03193451c (patch) | |
tree | 0efa8d066859fa765c8f517358bdfc9da87356a5 | |
parent | 191dd4948709c2bf6272f503e642d248740327cd (diff) |
Support loading an ART file from a zip file.
Bug: 377474232
Test: art/test.py --host -g
Change-Id: I1f8acd1d6eeff96cf83af8fcd5111865e114bcef
-rw-r--r-- | runtime/class_linker.cc | 63 | ||||
-rw-r--r-- | runtime/gc/space/image_space.cc | 91 | ||||
-rw-r--r-- | runtime/mirror/dex_cache-inl.h | 18 | ||||
-rw-r--r-- | runtime/mirror/dex_cache.h | 7 | ||||
-rw-r--r-- | runtime/oat/oat_file.cc | 11 | ||||
-rw-r--r-- | runtime/oat/oat_file_assistant_test.cc | 50 |
6 files changed, 177 insertions, 63 deletions
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index 6418c1a885..b4720c2dbc 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -92,12 +92,14 @@ #include "gc/space/image_space.h" #include "gc/space/space-inl.h" #include "gc_root-inl.h" +#include "handle.h" #include "handle_scope-inl.h" #include "hidden_api.h" #include "imt_conflict_table.h" #include "imtable-inl.h" #include "instrumentation-inl.h" #include "intern_table-inl.h" +#include "intern_table.h" #include "interpreter/interpreter.h" #include "interpreter/mterp/nterp.h" #include "jit/debugger_interface.h" @@ -1697,16 +1699,50 @@ static void VerifyInternedStringReferences(gc::space::ImageSpace* space) CHECK_EQ(num_recorded_refs, num_found_refs); } +static bool PatchDexCacheLocations(Handle<mirror::ObjectArray<mirror::DexCache>> dex_caches, + InternTable* intern_table, + std::string* error_msg) REQUIRES_SHARED(Locks::mutator_lock_) { + // Replace the location in the dex cache in the app image (the `--dex-location` passed to + // dex2oat) with the actual location if needed. + // The actual location is computed by the logic in `OatFileBase::Setup`. + // This is needed when the location on device is unknown at compile-time, typically during + // Cloud Compilation because the compilation is done on the server and the apk is later + // installed on device into `/data/app/<random_string>`. + // This is not needed during dexpreopt because the location on device is known to be a certain + // location in /system, /product, etc. + Thread* self = Thread::Current(); + StackHandleScope<1> hs(self); + MutableHandle<mirror::DexCache> dex_cache = hs.NewHandle<mirror::DexCache>(nullptr); + for (auto dex_cache_ptr : dex_caches.Iterate<mirror::DexCache>()) { + dex_cache.Assign(dex_cache_ptr); + std::string dex_file_location = + dex_cache->GetLocation(/*allow_location_mismatch=*/true)->ToModifiedUtf8(); + const DexFile* dex_file = dex_cache->GetDexFile(); + if (dex_file_location != dex_file->GetLocation()) { + ObjPtr<mirror::String> location = intern_table->InternWeak(dex_file->GetLocation().c_str()); + if (location == nullptr) { + self->AssertPendingOOMException(); + *error_msg = "Failed to intern string for dex cache location"; + return false; + } + dex_cache->SetLocation(location); + } + } + return true; +} + // new_class_set is the set of classes that were read from the class table section in the image. // If there was no class table section, it is null. // Note: using a class here to avoid having to make ClassLinker internals public. class AppImageLoadingHelper { public: - static void Update( + static bool Update( ClassLinker* class_linker, gc::space::ImageSpace* space, Handle<mirror::ClassLoader> class_loader, - Handle<mirror::ObjectArray<mirror::DexCache>> dex_caches) + Handle<mirror::ObjectArray<mirror::DexCache>> dex_caches, + InternTable* intern_table, + std::string* error_msg) REQUIRES(!Locks::dex_lock_) REQUIRES_SHARED(Locks::mutator_lock_); @@ -1714,11 +1750,13 @@ class AppImageLoadingHelper { REQUIRES_SHARED(Locks::mutator_lock_); }; -void AppImageLoadingHelper::Update( +bool AppImageLoadingHelper::Update( ClassLinker* class_linker, gc::space::ImageSpace* space, Handle<mirror::ClassLoader> class_loader, - Handle<mirror::ObjectArray<mirror::DexCache>> dex_caches) + Handle<mirror::ObjectArray<mirror::DexCache>> dex_caches, + InternTable* intern_table, + std::string* error_msg) REQUIRES(!Locks::dex_lock_) REQUIRES_SHARED(Locks::mutator_lock_) { ScopedTrace app_image_timing("AppImage:Updating"); @@ -1728,6 +1766,9 @@ void AppImageLoadingHelper::Update( // the Runtime::LoadAppImageStartupCache() option. VerifyInternedStringReferences(space); } + if (!PatchDexCacheLocations(dex_caches, intern_table, error_msg)) { + return false; + } DCHECK(class_loader.Get() != nullptr); Thread* const self = Thread::Current(); Runtime* const runtime = Runtime::Current(); @@ -1778,6 +1819,8 @@ void AppImageLoadingHelper::Update( } }, space->Begin(), kRuntimePointerSize); } + + return true; } void AppImageLoadingHelper::HandleAppImageStrings(gc::space::ImageSpace* space) { @@ -1931,6 +1974,13 @@ bool ClassLinker::OpenAndInitImageDexFiles( for (auto dex_cache : dex_caches.Iterate<mirror::DexCache>()) { std::string dex_file_location = dex_cache->GetLocation()->ToModifiedUtf8(); + // At this point, the location in the dex cache (from `--dex-location` passed to dex2oat) is not + // necessarily the actual dex location on device. `OpenOatDexFile` uses the table + // `OatFile::oat_dex_files_` to find the dex file. For each dex file, the table contains two + // keys corresponding to it, one from the oat header (from `--dex-location` passed to dex2oat) + // and the other being the actual dex location on device, unless they are the same. The lookup + // is based on the former key. Later, `PatchDexCacheLocations` will replace the location in the + // dex cache with the actual dex location, which is the latter key in the table. std::unique_ptr<const DexFile> dex_file = OpenOatDexFile(oat_file, dex_file_location.c_str(), error_msg); if (dex_file == nullptr) { @@ -2338,7 +2388,10 @@ bool ClassLinker::AddImageSpace(gc::space::ImageSpace* space, VLOG(image) << "Adding class table classes took " << PrettyDuration(NanoTime() - start_time2); } if (app_image) { - AppImageLoadingHelper::Update(this, space, class_loader, dex_caches); + if (!AppImageLoadingHelper::Update( + this, space, class_loader, dex_caches, intern_table_, error_msg)) { + return false; + } { ScopedTrace trace("AppImage:UpdateClassLoaders"); diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index f236d2a5d7..3d55c04828 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -21,6 +21,7 @@ #include <unistd.h> #include <array> +#include <cstddef> #include <memory> #include <optional> #include <random> @@ -664,33 +665,39 @@ class ImageSpace::Loader { CHECK(image_filename != nullptr); CHECK(image_location != nullptr); - std::unique_ptr<File> file; + FileWithRange file_with_range; { TimingLogger::ScopedTiming timing("OpenImageFile", logger); - file.reset(OS::OpenFileForReading(image_filename)); - if (file == nullptr) { - *error_msg = StringPrintf("Failed to open '%s'", image_filename); + // Most likely, the image is compressed and doesn't really need alignment. We enforce page + // size alignment just in case the image is uncompressed. + file_with_range = OS::OpenFileDirectlyOrFromZip( + image_filename, OatFile::kZipSeparator, /*alignment=*/MemMap::GetPageSize(), error_msg); + if (file_with_range.file == nullptr) { return nullptr; } } - return Init(file.get(), + return Init(file_with_range.file.get(), + file_with_range.start, + file_with_range.length, image_filename, image_location, - /*profile_files=*/ {}, - /*allow_direct_mapping=*/ true, + /*profile_files=*/{}, + /*allow_direct_mapping=*/true, logger, image_reservation, error_msg); } static std::unique_ptr<ImageSpace> Init(File* file, + off_t start, + size_t image_file_size, const char* image_filename, const char* image_location, const std::vector<std::string>& profile_files, bool allow_direct_mapping, TimingLogger* logger, - /*inout*/MemMap* image_reservation, - /*out*/std::string* error_msg) { + /*inout*/ MemMap* image_reservation, + /*out*/ std::string* error_msg) { CHECK(image_filename != nullptr); CHECK(image_location != nullptr); @@ -699,19 +706,17 @@ class ImageSpace::Loader { ImageHeader image_header; { TimingLogger::ScopedTiming timing("ReadImageHeader", logger); - bool success = file->PreadFully(&image_header, sizeof(image_header), /*offset=*/ 0u); + bool success = file->PreadFully(&image_header, sizeof(image_header), start); if (!success || !image_header.IsValid()) { *error_msg = StringPrintf("Invalid image header in '%s'", image_filename); return nullptr; } } // Check that the file is larger or equal to the header size + data size. - const uint64_t image_file_size = static_cast<uint64_t>(file->GetLength()); if (image_file_size < sizeof(ImageHeader) + image_header.GetDataSize()) { - *error_msg = StringPrintf( - "Image file truncated: %" PRIu64 " vs. %" PRIu64 ".", - image_file_size, - static_cast<uint64_t>(sizeof(ImageHeader) + image_header.GetDataSize())); + *error_msg = StringPrintf("Image file truncated: %zu vs. %" PRIu64 ".", + image_file_size, + sizeof(ImageHeader) + image_header.GetDataSize()); return nullptr; } @@ -733,10 +738,9 @@ class ImageSpace::Loader { RoundUp(sizeof(ImageHeader) + image_header.GetDataSize(), kElfSegmentAlignment); const size_t end_of_bitmap = image_bitmap_offset + bitmap_section.Size(); if (end_of_bitmap != image_file_size) { - *error_msg = StringPrintf( - "Image file size does not equal end of bitmap: size=%" PRIu64 " vs. %zu.", - image_file_size, - end_of_bitmap); + *error_msg = StringPrintf("Image file size does not equal end of bitmap: size=%zu vs. %zu.", + image_file_size, + end_of_bitmap); return nullptr; } @@ -746,15 +750,15 @@ class ImageSpace::Loader { // avoid reading proc maps for a mapping failure and slowing everything down. // For the boot image, we have already reserved the memory and we load the image // into the `image_reservation`. - MemMap map = LoadImageFile( - image_filename, - image_location, - image_header, - file->Fd(), - allow_direct_mapping, - logger, - image_reservation, - error_msg); + MemMap map = LoadImageFile(image_filename, + image_location, + image_header, + file->Fd(), + start, + allow_direct_mapping, + logger, + image_reservation, + error_msg); if (!map.IsValid()) { DCHECK(!error_msg->empty()); return nullptr; @@ -765,8 +769,8 @@ class ImageSpace::Loader { PROT_READ, MAP_PRIVATE, file->Fd(), - image_bitmap_offset, - /*low_4gb=*/ false, + start + image_bitmap_offset, + /*low_4gb=*/false, image_filename, error_msg); if (!image_bitmap_map.IsValid()) { @@ -986,10 +990,11 @@ class ImageSpace::Loader { const char* image_location, const ImageHeader& image_header, int fd, + off_t start, bool allow_direct_mapping, TimingLogger* logger, - /*inout*/MemMap* image_reservation, - /*out*/std::string* error_msg) { + /*inout*/ MemMap* image_reservation, + /*out*/ std::string* error_msg) { TimingLogger::ScopedTiming timing("MapImageFile", logger); // The runtime might not be available at this point if we're running dex2oat or oatdump, in @@ -1008,7 +1013,7 @@ class ImageSpace::Loader { PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, - /*start=*/0, + start, /*low_4gb=*/true, image_filename, /*reuse=*/false, @@ -1037,8 +1042,8 @@ class ImageSpace::Loader { PROT_READ, MAP_PRIVATE, fd, - /*start=*/ 0, - /*low_4gb=*/ false, + start, + /*low_4gb=*/false, image_filename, error_msg); if (!temp_map.IsValid()) { @@ -1054,7 +1059,7 @@ class ImageSpace::Loader { Runtime::ScopedThreadPoolUsage stpu; ThreadPool* const pool = stpu.GetThreadPool(); - const uint64_t start = NanoTime(); + const uint64_t start_time = NanoTime(); Thread* const self = Thread::Current(); static constexpr size_t kMinBlocks = 2u; const bool use_parallel = pool != nullptr && image_header.GetBlockCount() >= kMinBlocks; @@ -1085,7 +1090,7 @@ class ImageSpace::Loader { ScopedTrace trace("Waiting for workers"); pool->Wait(self, true, false); } - const uint64_t time = NanoTime() - start; + const uint64_t time = NanoTime() - start_time; // Add one 1 ns to prevent possible divide by 0. VLOG(image) << "Decompressing image took " << PrettyDuration(time) << " (" << PrettySize(static_cast<uint64_t>(map.Size()) * MsToNs(1000) / (time + 1)) @@ -2821,12 +2826,20 @@ class ImageSpace::BootImageLoader { VLOG(startup) << "Using image file " << image_filename.c_str() << " for image location " << image_location << " for compiled extension"; - File image_file(art_fd.release(), image_filename, /*check_usage=*/ false); + File image_file(art_fd.release(), image_filename, /*check_usage=*/false); + int64_t file_length = image_file.GetLength(); + if (file_length < 0) { + *error_msg = + ART_FORMAT("Failed to get file length of '{}': {}", image_filename, strerror(errno)); + return nullptr; + } std::unique_ptr<ImageSpace> result = Loader::Init(&image_file, + /*start=*/0, + file_length, image_filename.c_str(), image_location.c_str(), profile_files, - /*allow_direct_mapping=*/ false, + /*allow_direct_mapping=*/false, logger, image_reservation, error_msg); diff --git a/runtime/mirror/dex_cache-inl.h b/runtime/mirror/dex_cache-inl.h index 4ac5131958..71ada5db9e 100644 --- a/runtime/mirror/dex_cache-inl.h +++ b/runtime/mirror/dex_cache-inl.h @@ -19,12 +19,14 @@ #include "dex_cache.h" -#include <android-base/logging.h> +#include <atomic> +#include "android-base/logging.h" #include "art_field.h" #include "art_method.h" #include "base/atomic_pair.h" #include "base/casts.h" +#include "base/globals.h" #include "base/pointer_size.h" #include "class_linker.h" #include "dex/dex_file.h" @@ -38,8 +40,6 @@ #include "runtime.h" #include "write_barrier-inl.h" -#include <atomic> - namespace art HIDDEN { namespace mirror { @@ -346,9 +346,17 @@ inline void DexCache::VisitNativeRoots(const Visitor& visitor) { } template <VerifyObjectFlags kVerifyFlags, ReadBarrierOption kReadBarrierOption> -inline ObjPtr<String> DexCache::GetLocation() { - return GetFieldObject<String, kVerifyFlags, kReadBarrierOption>( +inline ObjPtr<String> DexCache::GetLocation(bool allow_location_mismatch) { + ObjPtr<String> location = GetFieldObject<String, kVerifyFlags, kReadBarrierOption>( OFFSET_OF_OBJECT_MEMBER(DexCache, location_)); + // At runtime, if the DexCache is from an app image or dynamically created, then its location must + // match the DexFile location. + // TODO(jiakaiz): Remove the AOT compiler and boot classpath checks? + if (kIsDebugBuild && !allow_location_mismatch && !Runtime::Current()->IsAotCompiler() && + GetDexFile() != nullptr && !ClassLinker::IsBootClassLoader(GetClassLoader())) { + DCHECK_EQ(location->ToModifiedUtf8(), GetDexFile()->GetLocation()); + } + return location; } } // namespace mirror diff --git a/runtime/mirror/dex_cache.h b/runtime/mirror/dex_cache.h index c1d8cd8335..5cfb37a5b3 100644 --- a/runtime/mirror/dex_cache.h +++ b/runtime/mirror/dex_cache.h @@ -308,9 +308,10 @@ class MANAGED DexCache final : public Object { // WARNING: This does not free the memory since it is in LinearAlloc. EXPORT void ResetNativeArrays() REQUIRES_SHARED(Locks::mutator_lock_); - template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags, - ReadBarrierOption kReadBarrierOption = kWithReadBarrier> - ObjPtr<String> GetLocation() REQUIRES_SHARED(Locks::mutator_lock_); + template <VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags, + ReadBarrierOption kReadBarrierOption = kWithReadBarrier> + ObjPtr<String> GetLocation(bool allow_location_mismatch = false) + REQUIRES_SHARED(Locks::mutator_lock_); String* GetResolvedString(dex::StringIndex string_idx) ALWAYS_INLINE REQUIRES_SHARED(Locks::mutator_lock_); diff --git a/runtime/oat/oat_file.cc b/runtime/oat/oat_file.cc index 772d126651..1ecad4abf9 100644 --- a/runtime/oat/oat_file.cc +++ b/runtime/oat/oat_file.cc @@ -785,9 +785,14 @@ bool OatFileBase::Setup(int zip_fd, std::string dex_file_name = dex_file_location; if (!dex_filenames.empty()) { dex_file_name.replace(/*pos*/ 0u, primary_location.size(), primary_location_replacement); - // If the location does not contain path and matches the file name component, - // use the provided file name also as the location. - // TODO: Do we need this for anything other than tests? + // If the location (the `--dex-location` passed to dex2oat) only contains the basename and + // matches the basename in the provided file name, use the provided file name also as the + // location. + // This is needed when the location on device is unknown at compile-time, typically during + // Cloud Compilation because the compilation is done on the server and the apk is later + // installed on device into `/data/app/<random_string>`. + // This is not needed during dexpreopt because the location on device is known to be a certain + // location in /system, /product, etc. if (dex_file_location.find('/') == std::string::npos && dex_file_name.size() > dex_file_location.size() && dex_file_name[dex_file_name.size() - dex_file_location.size() - 1u] == '/' && diff --git a/runtime/oat/oat_file_assistant_test.cc b/runtime/oat/oat_file_assistant_test.cc index 580191b355..b1f196a23a 100644 --- a/runtime/oat/oat_file_assistant_test.cc +++ b/runtime/oat/oat_file_assistant_test.cc @@ -26,6 +26,7 @@ #include <optional> #include <string> #include <type_traits> +#include <unordered_map> #include <vector> #include "android-base/scopeguard.h" @@ -42,6 +43,7 @@ #include "oat_file.h" #include "oat_file_assistant_context.h" #include "oat_file_manager.h" +#include "obj_ptr.h" #include "scoped_thread_state_change.h" #include "thread.h" @@ -2181,10 +2183,26 @@ TEST_P(OatFileAssistantTest, ShouldRecompileForImageFromSpeedProfile) { oat_file_assistant.GetDexOptNeeded(CompilerFilter::kVerify)); } -// Test that GetLocation of a dex file is the same whether the dex -// filed is backed by an oat file or not. +class CollectDexCacheVisitor : public DexCacheVisitor { + public: + explicit CollectDexCacheVisitor( + std::unordered_map<std::string, ObjPtr<mirror::DexCache>>& dex_caches) + : dex_caches_(dex_caches) {} + + void Visit(ObjPtr<mirror::DexCache> dex_cache) + REQUIRES_SHARED(Locks::dex_lock_, Locks::mutator_lock_) override { + dex_caches_[dex_cache->GetDexFile()->GetLocation()] = dex_cache; + } + + private: + std::unordered_map<std::string, ObjPtr<mirror::DexCache>>& dex_caches_; +}; + +// Test that, no matter the dex file is backed by an oat file or not, the location fields in +// DexFile, OatDexFile, and DexCache are the same as the actual dex location. TEST_F(OatFileAssistantBaseTest, GetDexLocation) { std::string dex_location = GetScratchDir() + "/TestDex.jar"; + std::string dex_location_multidex = dex_location + "!classes2.dex"; std::string oat_location = GetOdexDir() + "/TestDex.odex"; std::string art_location = GetOdexDir() + "/TestDex.art"; @@ -2192,7 +2210,7 @@ TEST_F(OatFileAssistantBaseTest, GetDexLocation) { Thread::Current()->TransitionFromSuspendedToRunnable(); runtime_->Start(); - Copy(GetDexSrc1(), dex_location); + Copy(GetMultiDexSrc1(), dex_location); std::vector<std::unique_ptr<const DexFile>> dex_files; std::vector<std::string> error_msgs; @@ -2204,9 +2222,11 @@ TEST_F(OatFileAssistantBaseTest, GetDexLocation) { /*dex_elements=*/nullptr, &oat_file, &error_msgs); - ASSERT_EQ(dex_files.size(), 1u) << android::base::Join(error_msgs, "\n"); + ASSERT_EQ(dex_files.size(), 2u) << android::base::Join(error_msgs, "\n"); EXPECT_EQ(oat_file, nullptr); - std::string stored_dex_location = dex_files[0]->GetLocation(); + EXPECT_EQ(dex_files[0]->GetLocation(), dex_location); + EXPECT_EQ(dex_files[1]->GetLocation(), dex_location_multidex); + { // Create the oat file. std::vector<std::string> args; @@ -2223,10 +2243,24 @@ TEST_F(OatFileAssistantBaseTest, GetDexLocation) { /*dex_elements=*/nullptr, &oat_file, &error_msgs); - ASSERT_EQ(dex_files.size(), 1u) << android::base::Join(error_msgs, "\n"); + ASSERT_EQ(dex_files.size(), 2u) << android::base::Join(error_msgs, "\n"); ASSERT_NE(oat_file, nullptr); - std::string oat_stored_dex_location = dex_files[0]->GetLocation(); - EXPECT_EQ(oat_stored_dex_location, stored_dex_location); + EXPECT_EQ(dex_files[0]->GetLocation(), dex_location); + EXPECT_EQ(dex_files[1]->GetLocation(), dex_location_multidex); + EXPECT_EQ(oat_file->GetOatDexFiles()[0]->GetLocation(), dex_location); + EXPECT_EQ(oat_file->GetOatDexFiles()[1]->GetLocation(), dex_location_multidex); + + std::unordered_map<std::string, ObjPtr<mirror::DexCache>> dex_caches; + CollectDexCacheVisitor visitor(dex_caches); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + { + ScopedObjectAccess soa(Thread::Current()); + ReaderMutexLock mu(Thread::Current(), *Locks::dex_lock_); + class_linker->VisitDexCaches(&visitor); + EXPECT_EQ(dex_caches[dex_location]->GetLocation()->ToModifiedUtf8(), dex_location); + EXPECT_EQ(dex_caches[dex_location_multidex]->GetLocation()->ToModifiedUtf8(), + dex_location_multidex); + } } // Test that a dex file on the platform location gets the right hiddenapi domain, |