Special case boot dex files in FixedUpDexFile::Create.

Doing a raw copy doesn't work on boot classpath dex files as
they contain hidden API flags. So instead run dexlayout on them,
which as a side effect will remove the hidden API information.

Bug: 123474797
Test: CtsJvmtiRunTest983HostTestCases
Change-Id: I327662ad96fc8bce1cf68ce26980b9a1887760e6

3 files changed, 18 insertions, 10 deletions

diff --git a/openjdkjvmti/fixed_up_dex_file.cc b/openjdkjvmti/fixed_up_dex_file.cc
index a3e06e6d98..079cd98915 100644
--- a/openjdkjvmti/fixed_up_dex_file.cc
+++ b/openjdkjvmti/fixed_up_dex_file.cc
@@ -87,7 +87,8 @@ static void DCheckVerifyDexFile(const art::DexFile& dex) {
   }
 }
 
-std::unique_ptr<FixedUpDexFile> FixedUpDexFile::Create(const art::DexFile& original,
+std::unique_ptr<FixedUpDexFile> FixedUpDexFile::Create(jobject class_loader,
+                                                       const art::DexFile& original,
                                                        const char* descriptor) {
   // Copy the data into mutable memory.
   std::vector<unsigned char> data;
@@ -100,9 +101,11 @@ std::unique_ptr<FixedUpDexFile> FixedUpDexFile::Create(const art::DexFile& origi
   // property from `original` to `new_dex_file`.
   const art::DexFileLoader dex_file_loader;
 
-  if (original.IsCompactDexFile()) {
+  if (original.IsCompactDexFile() || class_loader == nullptr) {
     // Since we are supposed to return a standard dex, convert back using dexlayout. It's OK to do
     // this before unquickening.
+    // We also do dex layout for boot classpath dex files, as they contain hidden API flags which
+    // we want to remove.
     art::Options options;
     options.compact_dex_level_ = art::CompactDexLevel::kCompactDexLevelNone;
     // Add a filter to only include the class that has the matching descriptor.
diff --git a/openjdkjvmti/fixed_up_dex_file.h b/openjdkjvmti/fixed_up_dex_file.h
index 594e8a7358..e09d70baf7 100644
--- a/openjdkjvmti/fixed_up_dex_file.h
+++ b/openjdkjvmti/fixed_up_dex_file.h
@@ -49,7 +49,8 @@ namespace openjdkjvmti {
 // are running on.
 class FixedUpDexFile {
  public:
-  static std::unique_ptr<FixedUpDexFile> Create(const art::DexFile& original,
+  static std::unique_ptr<FixedUpDexFile> Create(jobject class_loader,
+                                                const art::DexFile& original,
                                                 const char* descriptor);
 
   const art::DexFile& GetDexFile() {
diff --git a/openjdkjvmti/ti_class_definition.cc b/openjdkjvmti/ti_class_definition.cc
index 795a68a189..2345a0ad83 100644
--- a/openjdkjvmti/ti_class_definition.cc
+++ b/openjdkjvmti/ti_class_definition.cc
@@ -57,7 +57,7 @@ void ArtClassDefinition::InitializeMemory() const {
 
   std::string desc = std::string("L") + name_ + ";";
   std::unique_ptr<FixedUpDexFile>
-      fixed_dex_file(FixedUpDexFile::Create(*initial_dex_file_unquickened_, desc.c_str()));
+      fixed_dex_file(FixedUpDexFile::Create(loader_, *initial_dex_file_unquickened_, desc.c_str()));
   CHECK(fixed_dex_file.get() != nullptr);
   CHECK_LE(fixed_dex_file->Size(), temp_mmap_.Size());
   CHECK_EQ(temp_mmap_.Size(), dex_data_mmap_.Size());
@@ -132,17 +132,20 @@ jvmtiError ArtClassDefinition::InitCommon(art::Thread* self, jclass klass) {
   return OK;
 }
 
-static void DequickenDexFile(const art::DexFile* dex_file,
+static void DequickenDexFile(jobject class_loader,
+                             const art::DexFile* dex_file,
                              const char* descriptor,
                              /*out*/std::vector<unsigned char>* dex_data)
     REQUIRES_SHARED(art::Locks::mutator_lock_) {
-  std::unique_ptr<FixedUpDexFile> fixed_dex_file(FixedUpDexFile::Create(*dex_file, descriptor));
+  std::unique_ptr<FixedUpDexFile> fixed_dex_file(
+      FixedUpDexFile::Create(class_loader, *dex_file, descriptor));
   dex_data->resize(fixed_dex_file->Size());
   memcpy(dex_data->data(), fixed_dex_file->Begin(), fixed_dex_file->Size());
 }
 
 // Gets the data surrounding the given class.
-static void GetDexDataForRetransformation(art::Handle<art::mirror::Class> klass,
+static void GetDexDataForRetransformation(art::ScopedObjectAccess& soa,
+                                          art::Handle<art::mirror::Class> klass,
                                           /*out*/std::vector<unsigned char>* dex_data)
     REQUIRES_SHARED(art::Locks::mutator_lock_) {
   art::StackHandleScope<3> hs(art::Thread::Current());
@@ -179,7 +182,8 @@ static void GetDexDataForRetransformation(art::Handle<art::mirror::Class> klass,
     dex_file = &klass->GetDexFile();
   }
   std::string storage;
-  DequickenDexFile(dex_file, klass->GetDescriptor(&storage), dex_data);
+  jobject loader = soa.AddLocalReference<jobject>(klass->GetClassLoader());
+  DequickenDexFile(loader, dex_file, klass->GetDescriptor(&storage), dex_data);
 }
 
 static bool DexNeedsDequickening(art::Handle<art::mirror::Class> klass,
@@ -332,7 +336,7 @@ jvmtiError ArtClassDefinition::Init(art::Thread* self, jclass klass) {
   const art::DexFile* quick_dex = GetQuickenedDexFile(m_klass);
   auto get_original = [&](/*out*/std::vector<unsigned char>* dex_data)
       REQUIRES_SHARED(art::Locks::mutator_lock_) {
-    GetDexDataForRetransformation(m_klass, dex_data);
+    GetDexDataForRetransformation(soa, m_klass, dex_data);
   };
   InitWithDex(get_original, quick_dex);
   return OK;
@@ -365,7 +369,7 @@ void ArtClassDefinition::InitFirstLoad(const char* descriptor,
   protection_domain_ = nullptr;
   auto get_original = [&](/*out*/std::vector<unsigned char>* dex_data)
       REQUIRES_SHARED(art::Locks::mutator_lock_) {
-    DequickenDexFile(&dex_file, descriptor, dex_data);
+    DequickenDexFile(loader_, &dex_file, descriptor, dex_data);
   };
   InitWithDex(get_original, &dex_file);
 }