Special case dex files that have hidden API data.

Revert of 5462920633c948669dc6e28c61ef92cfeb0cce2b.

Dexlayout can increase the size of a dex file, so overestimate the
size needed when we know we're going to run dexlayout on it.

Bug: 123474797

Test: CtsJvmtiRunTest983HostTestCases
Test: art/test/run-test --always-clean  --host --prebuild --compact-dex-level fast --optimizing --no-relocate --runtime-option -Xcheck:jni --debuggable --jvmti-redefine-stress --64  980-redefine-object
Change-Id: Iea3d38a7f3a55d9a694b95687fb333f680f8106b

5 files changed, 20 insertions, 18 deletions

diff --git a/libdexfile/dex/dex_file.h b/libdexfile/dex/dex_file.h
index c7fbe784fc..8ea3c090e5 100644
--- a/libdexfile/dex/dex_file.h
+++ b/libdexfile/dex/dex_file.h
@@ -618,6 +618,10 @@ class DexFile {
     return hiddenapi_class_data_;
   }
 
+  ALWAYS_INLINE bool HasHiddenapiClassData() const {
+    return hiddenapi_class_data_ != nullptr;
+  }
+
   const dex::AnnotationItem* GetAnnotationItem(const dex::AnnotationSetItem* set_item,
                                                uint32_t index) const {
     DCHECK_LE(index, set_item->size_);
diff --git a/libdexfile/dex/standard_dex_file.h b/libdexfile/dex/standard_dex_file.h
index 838d4e3e78..48671c946d 100644
--- a/libdexfile/dex/standard_dex_file.h
+++ b/libdexfile/dex/standard_dex_file.h
@@ -84,7 +84,10 @@ class StandardDexFile : public DexFile {
   uint32_t GetCodeItemSize(const dex::CodeItem& item) const override;
 
   size_t GetDequickenedSize() const override {
-    return Size();
+    // JVMTI will run dex layout on standard dex files that have hidden API data,
+    // in order to remove that data. As dexlayout may increase the size of the dex file,
+    // be (very) conservative and add one MB to the size.
+    return Size() + (HasHiddenapiClassData() ? 1 * MB : 0);
   }
 
  private:
diff --git a/openjdkjvmti/fixed_up_dex_file.cc b/openjdkjvmti/fixed_up_dex_file.cc
index 079cd98915..da7eef963d 100644
--- a/openjdkjvmti/fixed_up_dex_file.cc
+++ b/openjdkjvmti/fixed_up_dex_file.cc
@@ -87,8 +87,7 @@ static void DCheckVerifyDexFile(const art::DexFile& dex) {
   }
 }
 
-std::unique_ptr<FixedUpDexFile> FixedUpDexFile::Create(jobject class_loader,
-                                                       const art::DexFile& original,
+std::unique_ptr<FixedUpDexFile> FixedUpDexFile::Create(const art::DexFile& original,
                                                        const char* descriptor) {
   // Copy the data into mutable memory.
   std::vector<unsigned char> data;
@@ -101,11 +100,11 @@ std::unique_ptr<FixedUpDexFile> FixedUpDexFile::Create(jobject class_loader,
   // property from `original` to `new_dex_file`.
   const art::DexFileLoader dex_file_loader;
 
-  if (original.IsCompactDexFile() || class_loader == nullptr) {
+  if (original.IsCompactDexFile() || original.HasHiddenapiClassData()) {
     // Since we are supposed to return a standard dex, convert back using dexlayout. It's OK to do
     // this before unquickening.
-    // We also do dex layout for boot classpath dex files, as they contain hidden API flags which
-    // we want to remove.
+    // We also do dex layout for dex files that have hidden API data, as we want to remove that
+    // data.
     art::Options options;
     options.compact_dex_level_ = art::CompactDexLevel::kCompactDexLevelNone;
     // Add a filter to only include the class that has the matching descriptor.
diff --git a/openjdkjvmti/fixed_up_dex_file.h b/openjdkjvmti/fixed_up_dex_file.h
index e09d70baf7..594e8a7358 100644
--- a/openjdkjvmti/fixed_up_dex_file.h
+++ b/openjdkjvmti/fixed_up_dex_file.h
@@ -49,8 +49,7 @@ namespace openjdkjvmti {
 // are running on.
 class FixedUpDexFile {
  public:
-  static std::unique_ptr<FixedUpDexFile> Create(jobject class_loader,
-                                                const art::DexFile& original,
+  static std::unique_ptr<FixedUpDexFile> Create(const art::DexFile& original,
                                                 const char* descriptor);
 
   const art::DexFile& GetDexFile() {
diff --git a/openjdkjvmti/ti_class_definition.cc b/openjdkjvmti/ti_class_definition.cc
index 2345a0ad83..20feb784bd 100644
--- a/openjdkjvmti/ti_class_definition.cc
+++ b/openjdkjvmti/ti_class_definition.cc
@@ -57,7 +57,7 @@ void ArtClassDefinition::InitializeMemory() const {
 
   std::string desc = std::string("L") + name_ + ";";
   std::unique_ptr<FixedUpDexFile>
-      fixed_dex_file(FixedUpDexFile::Create(loader_, *initial_dex_file_unquickened_, desc.c_str()));
+      fixed_dex_file(FixedUpDexFile::Create(*initial_dex_file_unquickened_, desc.c_str()));
   CHECK(fixed_dex_file.get() != nullptr);
   CHECK_LE(fixed_dex_file->Size(), temp_mmap_.Size());
   CHECK_EQ(temp_mmap_.Size(), dex_data_mmap_.Size());
@@ -132,20 +132,18 @@ jvmtiError ArtClassDefinition::InitCommon(art::Thread* self, jclass klass) {
   return OK;
 }
 
-static void DequickenDexFile(jobject class_loader,
-                             const art::DexFile* dex_file,
+static void DequickenDexFile(const art::DexFile* dex_file,
                              const char* descriptor,
                              /*out*/std::vector<unsigned char>* dex_data)
     REQUIRES_SHARED(art::Locks::mutator_lock_) {
   std::unique_ptr<FixedUpDexFile> fixed_dex_file(
-      FixedUpDexFile::Create(class_loader, *dex_file, descriptor));
+      FixedUpDexFile::Create(*dex_file, descriptor));
   dex_data->resize(fixed_dex_file->Size());
   memcpy(dex_data->data(), fixed_dex_file->Begin(), fixed_dex_file->Size());
 }
 
 // Gets the data surrounding the given class.
-static void GetDexDataForRetransformation(art::ScopedObjectAccess& soa,
-                                          art::Handle<art::mirror::Class> klass,
+static void GetDexDataForRetransformation(art::Handle<art::mirror::Class> klass,
                                           /*out*/std::vector<unsigned char>* dex_data)
     REQUIRES_SHARED(art::Locks::mutator_lock_) {
   art::StackHandleScope<3> hs(art::Thread::Current());
@@ -182,8 +180,7 @@ static void GetDexDataForRetransformation(art::ScopedObjectAccess& soa,
     dex_file = &klass->GetDexFile();
   }
   std::string storage;
-  jobject loader = soa.AddLocalReference<jobject>(klass->GetClassLoader());
-  DequickenDexFile(loader, dex_file, klass->GetDescriptor(&storage), dex_data);
+  DequickenDexFile(dex_file, klass->GetDescriptor(&storage), dex_data);
 }
 
 static bool DexNeedsDequickening(art::Handle<art::mirror::Class> klass,
@@ -336,7 +333,7 @@ jvmtiError ArtClassDefinition::Init(art::Thread* self, jclass klass) {
   const art::DexFile* quick_dex = GetQuickenedDexFile(m_klass);
   auto get_original = [&](/*out*/std::vector<unsigned char>* dex_data)
       REQUIRES_SHARED(art::Locks::mutator_lock_) {
-    GetDexDataForRetransformation(soa, m_klass, dex_data);
+    GetDexDataForRetransformation(m_klass, dex_data);
   };
   InitWithDex(get_original, quick_dex);
   return OK;
@@ -369,7 +366,7 @@ void ArtClassDefinition::InitFirstLoad(const char* descriptor,
   protection_domain_ = nullptr;
   auto get_original = [&](/*out*/std::vector<unsigned char>* dex_data)
       REQUIRES_SHARED(art::Locks::mutator_lock_) {
-    DequickenDexFile(loader_, &dex_file, descriptor, dex_data);
+    DequickenDexFile(&dex_file, descriptor, dex_data);
   };
   InitWithDex(get_original, &dex_file);
 }