diff options
| -rw-r--r-- | PREUPLOAD.cfg | 1 | ||||
| -rw-r--r-- | build/Android.gtest.mk | 1 | ||||
| -rw-r--r-- | build/art.go | 6 | ||||
| -rw-r--r-- | compiler/oat_writer.cc | 4 | ||||
| -rw-r--r-- | compiler/oat_writer.h | 2 | ||||
| -rw-r--r-- | dex2oat/dex2oat.cc | 155 | ||||
| -rw-r--r-- | dex2oat/dex2oat_test.cc | 123 | ||||
| -rw-r--r-- | runtime/Android.bp | 2 | ||||
| -rw-r--r-- | runtime/base/arena_allocator.h | 3 | ||||
| -rw-r--r-- | runtime/class_loader_context.cc | 285 | ||||
| -rw-r--r-- | runtime/class_loader_context.h | 150 | ||||
| -rw-r--r-- | runtime/class_loader_context_test.cc | 265 | ||||
| -rw-r--r-- | runtime/interpreter/interpreter_common.h | 2 | ||||
| -rw-r--r-- | runtime/interpreter/unstarted_runtime_test.cc | 19 | ||||
| -rw-r--r-- | runtime/oat_file.cc | 2 | ||||
| -rw-r--r-- | runtime/oat_file.h | 2 | ||||
| -rw-r--r-- | runtime/oat_file_assistant.cc | 30 | ||||
| -rw-r--r-- | runtime/oat_file_assistant.h | 7 | ||||
| -rw-r--r-- | runtime/thread.cc | 9 | ||||
| -rw-r--r-- | test/912-classes/src-art/art/Test912.java | 3 | ||||
| -rw-r--r-- | tools/libcore_failures.txt | 5 | ||||
| -rwxr-xr-x | tools/test_presubmit.py | 159 |
22 files changed, 1128 insertions, 107 deletions
diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg index 0ed230c408..8a8df36101 100644 --- a/PREUPLOAD.cfg +++ b/PREUPLOAD.cfg @@ -1,3 +1,4 @@ [Hook Scripts] check_generated_files_up_to_date = tools/cpp-define-generator/presubmit-check-files-up-to-date +check_generated_tests_up_to_date = tools/test_presubmit.py check_cpplint_on_changed_files = tools/cpplint_presubmit.py diff --git a/build/Android.gtest.mk b/build/Android.gtest.mk index b6ffcc54f7..ff4f9016c0 100644 --- a/build/Android.gtest.mk +++ b/build/Android.gtest.mk @@ -104,6 +104,7 @@ ART_GTEST_dex2oat_environment_tests_DEX_DEPS := Main MainStripped MultiDex Multi ART_GTEST_atomic_method_ref_map_test_DEX_DEPS := Interfaces ART_GTEST_class_linker_test_DEX_DEPS := AllFields ErroneousA ErroneousB ErroneousInit ForClassLoaderA ForClassLoaderB ForClassLoaderC ForClassLoaderD Interfaces MethodTypes MultiDex MyClass Nested Statics StaticsFromCode +ART_GTEST_class_loader_context_test_DEX_DEPS := Main MultiDex MyClass ART_GTEST_class_table_test_DEX_DEPS := XandY ART_GTEST_compiler_driver_test_DEX_DEPS := AbstractMethod StaticLeafMethods ProfileTestMultiDex ART_GTEST_dex_cache_test_DEX_DEPS := Main Packages MethodTypes diff --git a/build/art.go b/build/art.go index 1b9c646433..c06be0abed 100644 --- a/build/art.go +++ b/build/art.go @@ -97,6 +97,12 @@ func globalFlags(ctx android.BaseContext) ([]string, []string) { "-DART_STACK_OVERFLOW_GAP_x86_64=8192") } + if envTrue(ctx, "ART_ENABLE_ADDRESS_SANITIZER") { + // Used to enable full sanitization, i.e., user poisoning, under ASAN. + cflags = append(cflags, "-DART_ENABLE_ADDRESS_SANITIZER=1") + asflags = append(asflags, "-DART_ENABLE_ADDRESS_SANITIZER=1") + } + return cflags, asflags } diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc index f7465c0d5f..7cb3166cde 100644 --- a/compiler/oat_writer.cc +++ b/compiler/oat_writer.cc @@ -473,8 +473,8 @@ bool OatWriter::AddRawDexFileSource(const ArrayRef<const uint8_t>& data, return true; } -dchecked_vector<const char*> OatWriter::GetSourceLocations() const { - dchecked_vector<const char*> locations; +dchecked_vector<std::string> OatWriter::GetSourceLocations() const { + dchecked_vector<std::string> locations; locations.reserve(oat_dex_files_.size()); for (const OatDexFile& oat_dex_file : oat_dex_files_) { locations.push_back(oat_dex_file.GetLocation()); diff --git a/compiler/oat_writer.h b/compiler/oat_writer.h index 9217701bc5..024a3e80ca 100644 --- a/compiler/oat_writer.h +++ b/compiler/oat_writer.h @@ -153,7 +153,7 @@ class OatWriter { const VdexFile& vdex_file, const char* location, CreateTypeLookupTable create_type_lookup_table = CreateTypeLookupTable::kDefault); - dchecked_vector<const char*> GetSourceLocations() const; + dchecked_vector<std::string> GetSourceLocations() const; // Write raw dex files to the vdex file, mmap the file and open the dex files from it. // Supporting data structures are written into the .rodata section of the oat file. diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index ca0bae1c8e..6257c7c1c3 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -49,6 +49,7 @@ #include "base/timing_logger.h" #include "base/unix_file/fd_file.h" #include "class_linker.h" +#include "class_loader_context.h" #include "compiler.h" #include "compiler_callbacks.h" #include "debug/elf_debug_writer.h" @@ -400,6 +401,27 @@ NO_RETURN static void Usage(const char* fmt, ...) { UsageError(""); UsageError(" --classpath-dir=<directory-path>: directory used to resolve relative class paths."); UsageError(""); + UsageError(" --class-loader-context=<string spec>: a string specifying the intended"); + UsageError(" runtime loading context for the compiled dex files."); + UsageError(" "); + UsageError(" It describes how the class loader chain should be built in order to ensure"); + UsageError(" classes are resolved during dex2aot as they would be resolved at runtime."); + UsageError(" This spec will be encoded in the oat file. If at runtime the dex file is"); + UsageError(" loaded in a different context, the oat file will be rejected."); + UsageError(" "); + UsageError(" The chain is interpreted in the natural 'parent order', meaning that class"); + UsageError(" loader 'i+1' will be the parent of class loader 'i'."); + UsageError(" The compilation sources will be added to the classpath of the last class"); + UsageError(" loader. This allows the compiled dex files to be loaded at runtime in"); + UsageError(" a class loader that contains other dex files as well (e.g. shared libraries)."); + UsageError(" "); + UsageError(" Note that the compiler will be tolerant if the source dex files specified"); + UsageError(" with --dex-file are found in the classpath. The source dex files will be"); + UsageError(" removed from any class loader's classpath possibly resulting in empty"); + UsageError(" class loaders."); + UsageError(" "); + UsageError(" Example: --classloader-spec=PCL[lib1.dex:lib2.dex];DLC[lib3.dex]"); + UsageError(""); std::cerr << "See log for usage error information\n"; exit(EXIT_FAILURE); } @@ -1271,6 +1293,12 @@ class Dex2Oat FINAL { force_determinism_ = true; } else if (option.starts_with("--classpath-dir=")) { classpath_dir_ = option.substr(strlen("--classpath-dir=")).data(); + } else if (option.starts_with("--class-loader-context=")) { + class_loader_context_ = ClassLoaderContext::Create( + option.substr(strlen("--class-loader-context=")).data()); + if (class_loader_context_== nullptr) { + Usage("Option --class-loader-context has an incorrect format: %s", option.data()); + } } else if (!compiler_options_->ParseCompilerOption(option, Usage)) { Usage("Unknown argument %s", option.data()); } @@ -1542,25 +1570,45 @@ class Dex2Oat FINAL { } // Open dex files for class path. - std::vector<std::string> class_path_locations = - GetClassPathLocations(runtime_->GetClassPathString()); - OpenClassPathFiles(class_path_locations, - &class_path_files_, - &opened_oat_files_, - runtime_->GetInstructionSet(), - classpath_dir_); - - // Store the classpath we have right now. - std::vector<const DexFile*> class_path_files = MakeNonOwningPointerVector(class_path_files_); - std::string encoded_class_path; - if (class_path_locations.size() == 1 && - class_path_locations[0] == OatFile::kSpecialSharedLibrary) { - // When passing the special shared library as the classpath, it is the only path. - encoded_class_path = OatFile::kSpecialSharedLibrary; - } else { - encoded_class_path = OatFile::EncodeDexFileDependencies(class_path_files, classpath_dir_); + if (class_loader_context_ == nullptr) { + // TODO(calin): Temporary workaround while we transition to use + // --class-loader-context instead of --runtime-arg -cp + if (runtime_->GetClassPathString().empty()) { + class_loader_context_ = std::unique_ptr<ClassLoaderContext>( + new ClassLoaderContext()); + } else { + std::string spec = runtime_->GetClassPathString() == OatFile::kSpecialSharedLibrary + ? OatFile::kSpecialSharedLibrary + : "PCL[" + runtime_->GetClassPathString() + "]"; + class_loader_context_ = ClassLoaderContext::Create(spec); + } + } + CHECK(class_loader_context_ != nullptr); + DCHECK_EQ(oat_writers_.size(), 1u); + + // Note: Ideally we would reject context where the source dex files are also + // specified in the classpath (as it doesn't make sense). However this is currently + // needed for non-prebuild tests and benchmarks which expects on the fly compilation. + // Also, for secondary dex files we do not have control on the actual classpath. + // Instead of aborting, remove all the source location from the context classpaths. + if (class_loader_context_->RemoveLocationsFromClassPaths( + oat_writers_[0]->GetSourceLocations())) { + LOG(WARNING) << "The source files to be compiled are also in the classpath."; + } + + // We need to open the dex files before encoding the context in the oat file. + // (because the encoding adds the dex checksum...) + // TODO(calin): consider redesigning this so we don't have to open the dex files before + // creating the actual class loader. + if (!class_loader_context_->OpenDexFiles(runtime_->GetInstructionSet(), classpath_dir_)) { + // Do not abort if we couldn't open files from the classpath. They might be + // apks without dex files and right now are opening flow will fail them. + LOG(WARNING) << "Failed to open classpath dex files"; } - key_value_store_->Put(OatHeader::kClassPathKey, encoded_class_path); + + // Store the class loader context in the oat header. + key_value_store_->Put(OatHeader::kClassPathKey, + class_loader_context_->EncodeContextForOatFile(classpath_dir_)); } // Now that we have finalized key_value_store_, start writing the oat file. @@ -1656,17 +1704,7 @@ class Dex2Oat FINAL { if (kSaveDexInput) { SaveDexInput(); } - - // Handle and ClassLoader creation needs to come after Runtime::Create. - ScopedObjectAccess soa(self); - - // Classpath: first the class-path given. - std::vector<const DexFile*> class_path_files = MakeNonOwningPointerVector(class_path_files_); - - // Then the dex files we'll compile. Thus we'll resolve the class-path first. - class_path_files.insert(class_path_files.end(), dex_files_.begin(), dex_files_.end()); - - class_loader_ = class_linker->CreatePathClassLoader(self, class_path_files); + class_loader_ = class_loader_context_->CreateClassLoader(dex_files_); } // Ensure opened dex files are writable for dex-to-dex transformations. @@ -1721,7 +1759,12 @@ class Dex2Oat FINAL { if (!no_inline_filters.empty()) { ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); - std::vector<const DexFile*> class_path_files = MakeNonOwningPointerVector(class_path_files_); + std::vector<const DexFile*> class_path_files; + if (!IsBootImage()) { + // The class loader context is used only for apps. + class_path_files = class_loader_context_->FlattenOpenedDexFiles(); + } + std::vector<const std::vector<const DexFile*>*> dex_file_vectors = { &class_linker->GetBootClassPath(), &class_path_files, @@ -2224,8 +2267,8 @@ class Dex2Oat FINAL { DCHECK(!IsBootImage()); DCHECK_EQ(oat_writers_.size(), 1u); std::vector<std::string> dex_files_canonical_locations; - for (const char* location : oat_writers_[0]->GetSourceLocations()) { - dex_files_canonical_locations.push_back(DexFile::GetDexCanonicalLocation(location)); + for (const std::string& location : oat_writers_[0]->GetSourceLocations()) { + dex_files_canonical_locations.push_back(DexFile::GetDexCanonicalLocation(location.c_str())); } std::vector<std::string> parsed; @@ -2240,48 +2283,6 @@ class Dex2Oat FINAL { return parsed; } - // Opens requested class path files and appends them to opened_dex_files. If the dex files have - // been stripped, this opens them from their oat files and appends them to opened_oat_files. - static void OpenClassPathFiles(std::vector<std::string>& class_path_locations, - std::vector<std::unique_ptr<const DexFile>>* opened_dex_files, - std::vector<std::unique_ptr<OatFile>>* opened_oat_files, - InstructionSet isa, - std::string& classpath_dir) { - DCHECK(opened_dex_files != nullptr) << "OpenClassPathFiles dex out-param is nullptr"; - DCHECK(opened_oat_files != nullptr) << "OpenClassPathFiles oat out-param is nullptr"; - for (std::string& location : class_path_locations) { - // Stop early if we detect the special shared library, which may be passed as the classpath - // for dex2oat when we want to skip the shared libraries check. - if (location == OatFile::kSpecialSharedLibrary) { - break; - } - // If path is relative, append it to the provided base directory. - if (!classpath_dir.empty() && location[0] != '/') { - location = classpath_dir + '/' + location; - } - static constexpr bool kVerifyChecksum = true; - std::string error_msg; - if (!DexFile::Open( - location.c_str(), location.c_str(), kVerifyChecksum, &error_msg, opened_dex_files)) { - // If we fail to open the dex file because it's been stripped, try to open the dex file - // from its corresponding oat file. - OatFileAssistant oat_file_assistant(location.c_str(), isa, false); - std::unique_ptr<OatFile> oat_file(oat_file_assistant.GetBestOatFile()); - if (oat_file == nullptr) { - LOG(WARNING) << "Failed to open dex file and associated oat file for '" << location - << "': " << error_msg; - } else { - std::vector<std::unique_ptr<const DexFile>> oat_dex_files = - oat_file_assistant.LoadDexFiles(*oat_file, location.c_str()); - opened_oat_files->push_back(std::move(oat_file)); - opened_dex_files->insert(opened_dex_files->end(), - std::make_move_iterator(oat_dex_files.begin()), - std::make_move_iterator(oat_dex_files.end())); - } - } - } - } - bool PrepareImageClasses() { // If --image-classes was specified, calculate the full list of classes to include in the image. if (image_classes_filename_ != nullptr) { @@ -2737,8 +2738,8 @@ class Dex2Oat FINAL { std::unique_ptr<Runtime> runtime_; - // Ownership for the class path files. - std::vector<std::unique_ptr<const DexFile>> class_path_files_; + // The spec describing how the class loader should be setup for compilation. + std::unique_ptr<ClassLoaderContext> class_loader_context_; size_t thread_count_; uint64_t start_ns_; @@ -2792,9 +2793,9 @@ class Dex2Oat FINAL { std::unique_ptr<CompilerDriver> driver_; std::vector<std::unique_ptr<MemMap>> opened_dex_files_maps_; - std::vector<std::unique_ptr<OatFile>> opened_oat_files_; std::vector<std::unique_ptr<const DexFile>> opened_dex_files_; + // Note that this might contain pointers owned by class_loader_context_. std::vector<const DexFile*> no_inline_from_dex_files_; bool dump_stats_; diff --git a/dex2oat/dex2oat_test.cc b/dex2oat/dex2oat_test.cc index b604e8b5f1..1505eb56c8 100644 --- a/dex2oat/dex2oat_test.cc +++ b/dex2oat/dex2oat_test.cc @@ -89,7 +89,8 @@ class Dex2oatTest : public Dex2oatEnvironmentTest { CompilerFilter::Filter filter, const std::vector<std::string>& extra_args = {}, bool expect_success = true, - bool use_fd = false) { + bool use_fd = false, + std::function<void(const OatFile&)> check_oat = [](const OatFile&) {}) { std::string error_msg; int status = GenerateOdexForTestWithStatus(dex_location, odex_location, @@ -113,6 +114,7 @@ class Dex2oatTest : public Dex2oatEnvironmentTest { ASSERT_TRUE(odex_file.get() != nullptr) << error_msg; CheckFilter(filter, odex_file->GetCompilerFilter()); + check_oat(*(odex_file.get())); } else { ASSERT_FALSE(success) << output_; @@ -895,4 +897,123 @@ TEST_F(Dex2oatReturnCodeTest, TestCreateRuntime) { EXPECT_EQ(static_cast<int>(dex2oat::ReturnCode::kCreateRuntime), WEXITSTATUS(status)) << output_; } +class Dex2oatClassLoaderContextTest : public Dex2oatTest { + protected: + void RunTest(const char* class_loader_context, + const char* expected_classpath_key, + bool expected_success, + bool use_second_source = false) { + std::string dex_location = GetUsedDexLocation(); + std::string odex_location = GetUsedOatLocation(); + + Copy(use_second_source ? GetDexSrc2() : GetDexSrc1(), dex_location); + + std::string error_msg; + std::vector<std::string> extra_args; + if (class_loader_context != nullptr) { + extra_args.push_back(std::string("--class-loader-context=") + class_loader_context); + } + auto check_oat = [expected_classpath_key](const OatFile& oat_file) { + ASSERT_TRUE(expected_classpath_key != nullptr); + const char* classpath = oat_file.GetOatHeader().GetStoreValueByKey(OatHeader::kClassPathKey); + ASSERT_TRUE(classpath != nullptr); + ASSERT_STREQ(expected_classpath_key, classpath); + }; + + GenerateOdexForTest(dex_location, + odex_location, + CompilerFilter::kQuicken, + extra_args, + expected_success, + /*use_fd*/ false, + check_oat); + } + + std::string GetUsedDexLocation() { + return GetScratchDir() + "/Context.jar"; + } + + std::string GetUsedOatLocation() { + return GetOdexDir() + "/Context.odex"; + } + + const char* kEmptyClassPathKey = ""; +}; + +TEST_F(Dex2oatClassLoaderContextTest, InvalidContext) { + RunTest("Invalid[]", /*expected_classpath_key*/ nullptr, /*expected_success*/ false); +} + +TEST_F(Dex2oatClassLoaderContextTest, EmptyContext) { + RunTest("PCL[]", kEmptyClassPathKey, /*expected_success*/ true); +} + +TEST_F(Dex2oatClassLoaderContextTest, SpecialContext) { + RunTest(OatFile::kSpecialSharedLibrary, + OatFile::kSpecialSharedLibrary, + /*expected_success*/ true); +} + +TEST_F(Dex2oatClassLoaderContextTest, ContextWithTheSourceDexFiles) { + std::string context = "PCL[" + GetUsedDexLocation() + "]"; + RunTest(context.c_str(), kEmptyClassPathKey, /*expected_success*/ true); +} + +TEST_F(Dex2oatClassLoaderContextTest, ContextWithOtherDexFiles) { + std::vector<std::unique_ptr<const DexFile>> dex_files = OpenTestDexFiles("Nested"); + std::string expected_classpath_key = + OatFile::EncodeDexFileDependencies(MakeNonOwningPointerVector(dex_files), ""); + + std::string context = "PCL[" + dex_files[0]->GetLocation() + "]"; + RunTest(context.c_str(), expected_classpath_key.c_str(), true); +} + +TEST_F(Dex2oatClassLoaderContextTest, ContextWithStrippedDexFiles) { + std::string stripped_classpath = GetScratchDir() + "/stripped_classpath.jar"; + Copy(GetStrippedDexSrc1(), stripped_classpath); + + std::string context = "PCL[" + stripped_classpath + "]"; + // Expect an empty context because stripped dex files cannot be open. + RunTest(context.c_str(), /*expected_classpath_key*/ "" , /*expected_success*/ true); +} + +TEST_F(Dex2oatClassLoaderContextTest, ContextWithStrippedDexFilesBackedByOdex) { + std::string stripped_classpath = GetScratchDir() + "/stripped_classpath.jar"; + std::string odex_for_classpath = GetOdexDir() + "/stripped_classpath.odex"; + + Copy(GetDexSrc1(), stripped_classpath); + + GenerateOdexForTest(stripped_classpath, + odex_for_classpath, + CompilerFilter::kQuicken, + {}, + true); + + // Strip the dex file + Copy(GetStrippedDexSrc1(), stripped_classpath); + + std::string context = "PCL[" + stripped_classpath + "]"; + std::string expected_classpath; + { + // Open the oat file to get the expected classpath. + OatFileAssistant oat_file_assistant(stripped_classpath.c_str(), kRuntimeISA, false); + std::unique_ptr<OatFile> oat_file(oat_file_assistant.GetBestOatFile()); + std::vector<std::unique_ptr<const DexFile>> oat_dex_files = + OatFileAssistant::LoadDexFiles(*oat_file, stripped_classpath.c_str()); + expected_classpath = OatFile::EncodeDexFileDependencies( + MakeNonOwningPointerVector(oat_dex_files), ""); + } + + RunTest(context.c_str(), + expected_classpath.c_str(), + /*expected_success*/ true, + /*use_second_source*/ true); +} + +TEST_F(Dex2oatClassLoaderContextTest, ContextWithNotExistentDexFiles) { + std::string context = "PCL[does_not_exists.dex]"; + // Expect an empty context because stripped dex files cannot be open. + RunTest(context.c_str(), kEmptyClassPathKey, /*expected_success*/ true); +} + } // namespace art diff --git a/runtime/Android.bp b/runtime/Android.bp index 0dfc60d88a..8d15c349dd 100644 --- a/runtime/Android.bp +++ b/runtime/Android.bp @@ -48,6 +48,7 @@ cc_defaults { "cha.cc", "check_jni.cc", "class_linker.cc", + "class_loader_context.cc", "class_table.cc", "code_simulator_container.cc", "common_throws.cc", @@ -542,6 +543,7 @@ art_cc_test { "base/unix_file/fd_file_test.cc", "cha_test.cc", "class_linker_test.cc", + "class_loader_context_test.cc", "class_table_test.cc", "compiler_filter_test.cc", "dex_file_test.cc", diff --git a/runtime/base/arena_allocator.h b/runtime/base/arena_allocator.h index ebde82db55..a484c5c105 100644 --- a/runtime/base/arena_allocator.h +++ b/runtime/base/arena_allocator.h @@ -336,7 +336,8 @@ class ArenaAllocator auto* end = reinterpret_cast<uint8_t*>(ptr) + aligned_ptr_size; // If we haven't allocated anything else, we can safely extend. if (end == ptr_) { - DCHECK(!IsRunningOnMemoryTool()); // Red zone prevents end == ptr_. + // Red zone prevents end == ptr_ (unless input = allocator state = null). + DCHECK(!IsRunningOnMemoryTool() || ptr_ == nullptr); const size_t aligned_new_size = RoundUp(new_size, kAlignment); const size_t size_delta = aligned_new_size - aligned_ptr_size; // Check remain space. diff --git a/runtime/class_loader_context.cc b/runtime/class_loader_context.cc new file mode 100644 index 0000000000..5cbcd8f1ff --- /dev/null +++ b/runtime/class_loader_context.cc @@ -0,0 +1,285 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "class_loader_context.h" + +#include "base/dchecked_vector.h" +#include "base/stl_util.h" +#include "class_linker.h" +#include "dex_file.h" +#include "oat_file_assistant.h" +#include "runtime.h" +#include "scoped_thread_state_change-inl.h" +#include "thread.h" + +namespace art { + +static constexpr char kPathClassLoaderString[] = "PCL"; +static constexpr char kDelegateLastClassLoaderString[] = "DLC"; +static constexpr char kClassLoaderOpeningMark = '['; +static constexpr char kClassLoaderClosingMark = ']'; +static constexpr char kClassLoaderSep = ';'; +static constexpr char kClasspathSep = ':'; + +ClassLoaderContext::ClassLoaderContext() + : special_shared_library_(false), + dex_files_open_attempted_(false), + dex_files_open_result_(false) {} + +std::unique_ptr<ClassLoaderContext> ClassLoaderContext::Create(const std::string& spec) { + std::unique_ptr<ClassLoaderContext> result(new ClassLoaderContext()); + if (result->Parse(spec)) { + return result; + } else { + return nullptr; + } +} + +// The expected format is: "ClassLoaderType1[ClasspathElem1:ClasspathElem2...]". +bool ClassLoaderContext::ParseClassLoaderSpec(const std::string& class_loader_spec, + ClassLoaderType class_loader_type) { + const char* class_loader_type_str = GetClassLoaderTypeName(class_loader_type); + size_t type_str_size = strlen(class_loader_type_str); + + CHECK_EQ(0, class_loader_spec.compare(0, type_str_size, class_loader_type_str)); + + // Check the opening and closing markers. + if (class_loader_spec[type_str_size] != kClassLoaderOpeningMark) { + return false; + } + if (class_loader_spec[class_loader_spec.length() - 1] != kClassLoaderClosingMark) { + return false; + } + + // At this point we know the format is ok; continue and extract the classpath. + // Note that class loaders with an empty class path are allowed. + std::string classpath = class_loader_spec.substr(type_str_size + 1, + class_loader_spec.length() - type_str_size - 2); + + class_loader_chain_.push_back(ClassLoaderInfo(class_loader_type)); + Split(classpath, kClasspathSep, &class_loader_chain_.back().classpath); + + return true; +} + +// Extracts the class loader type from the given spec. +// Return ClassLoaderContext::kInvalidClassLoader if the class loader type is not +// recognized. +ClassLoaderContext::ClassLoaderType +ClassLoaderContext::ExtractClassLoaderType(const std::string& class_loader_spec) { + const ClassLoaderType kValidTypes[] = {kPathClassLoader, kDelegateLastClassLoader}; + for (const ClassLoaderType& type : kValidTypes) { + const char* type_str = GetClassLoaderTypeName(type); + if (class_loader_spec.compare(0, strlen(type_str), type_str) == 0) { + return type; + } + } + return kInvalidClassLoader; +} + +// The format: ClassLoaderType1[ClasspathElem1:ClasspathElem2...];ClassLoaderType2[...]... +// ClassLoaderType is either "PCL" (PathClassLoader) or "DLC" (DelegateLastClassLoader). +// ClasspathElem is the path of dex/jar/apk file. +bool ClassLoaderContext::Parse(const std::string& spec) { + if (spec.empty()) { + LOG(ERROR) << "Empty string passed to Parse"; + return false; + } + // Stop early if we detect the special shared library, which may be passed as the classpath + // for dex2oat when we want to skip the shared libraries check. + if (spec == OatFile::kSpecialSharedLibrary) { + LOG(INFO) << "The ClassLoaderContext is a special shared library."; + special_shared_library_ = true; + return true; + } + + std::vector<std::string> class_loaders; + Split(spec, kClassLoaderSep, &class_loaders); + + for (const std::string& class_loader : class_loaders) { + ClassLoaderType type = ExtractClassLoaderType(class_loader); + if (type == kInvalidClassLoader) { + LOG(ERROR) << "Invalid class loader type: " << class_loader; + return false; + } + if (!ParseClassLoaderSpec(class_loader, type)) { + LOG(ERROR) << "Invalid class loader spec: " << class_loader; + return false; + } + } + return true; +} + +// Opens requested class path files and appends them to opened_dex_files. If the dex files have +// been stripped, this opens them from their oat files (which get added to opened_oat_files). +bool ClassLoaderContext::OpenDexFiles(InstructionSet isa, const std::string& classpath_dir) { + CHECK(!dex_files_open_attempted_) << "OpenDexFiles should not be called twice"; + + dex_files_open_attempted_ = true; + // Assume we can open all dex files. If not, we will set this to false as we go. + dex_files_open_result_ = true; + + if (special_shared_library_) { + // Nothing to open if the context is a special shared library. + return true; + } + + // Note that we try to open all dex files even if some fail. + // We may get resource-only apks which we cannot load. + // TODO(calin): Refine the dex opening interface to be able to tell if an archive contains + // no dex files. So that we can distinguish the real failures... + for (ClassLoaderInfo& info : class_loader_chain_) { + for (const std::string& cp_elem : info.classpath) { + // If path is relative, append it to the provided base directory. + std::string location = cp_elem; + if (location[0] != '/') { + location = classpath_dir + '/' + location; + } + std::string error_msg; + // When opening the dex files from the context we expect their checksum to match their + // contents. So pass true to verify_checksum. + if (!DexFile::Open(location.c_str(), + location.c_str(), + /*verify_checksum*/ true, + &error_msg, + &info.opened_dex_files)) { + // If we fail to open the dex file because it's been stripped, try to open the dex file + // from its corresponding oat file. + // This could happen when we need to recompile a pre-build whose dex code has been stripped. + // (for example, if the pre-build is only quicken and we want to re-compile it + // speed-profile). + // TODO(calin): Use the vdex directly instead of going through the oat file. + OatFileAssistant oat_file_assistant(location.c_str(), isa, false); + std::unique_ptr<OatFile> oat_file(oat_file_assistant.GetBestOatFile()); + std::vector<std::unique_ptr<const DexFile>> oat_dex_files; + if (oat_file != nullptr && + OatFileAssistant::LoadDexFiles(*oat_file, location, &oat_dex_files)) { + info.opened_oat_files.push_back(std::move(oat_file)); + info.opened_dex_files.insert(info.opened_dex_files.end(), + std::make_move_iterator(oat_dex_files.begin()), + std::make_move_iterator(oat_dex_files.end())); + } else { + LOG(WARNING) << "Could not open dex files from location: " << location; + dex_files_open_result_ = false; + } + } + } + } + + return dex_files_open_result_; +} + +bool ClassLoaderContext::RemoveLocationsFromClassPaths( + const dchecked_vector<std::string>& locations) { + CHECK(!dex_files_open_attempted_) + << "RemoveLocationsFromClasspaths cannot be call after OpenDexFiles"; + + std::set<std::string> canonical_locations; + for (const std::string& location : locations) { + canonical_locations.insert(DexFile::GetDexCanonicalLocation(location.c_str())); + } + bool removed_locations = false; + for (ClassLoaderInfo& info : class_loader_chain_) { + size_t initial_size = info.classpath.size(); + auto kept_it = std::remove_if( + info.classpath.begin(), + info.classpath.end(), + [canonical_locations](const std::string& location) { + return ContainsElement(canonical_locations, + DexFile::GetDexCanonicalLocation(location.c_str())); + }); + info.classpath.erase(kept_it, info.classpath.end()); + if (initial_size != info.classpath.size()) { + removed_locations = true; + } + } + return removed_locations; +} + +std::string ClassLoaderContext::EncodeContextForOatFile(const std::string& base_dir) const { + CheckDexFilesOpened("EncodeContextForOatFile"); + if (special_shared_library_) { + return OatFile::kSpecialSharedLibrary; + } + + if (class_loader_chain_.empty()) { + return ""; + } + + // TODO(calin): Transition period: assume we only have a classloader until + // the oat file assistant implements the full class loader check. + CHECK_EQ(1u, class_loader_chain_.size()); + + return OatFile::EncodeDexFileDependencies(MakeNonOwningPointerVector( + class_loader_chain_[0].opened_dex_files), base_dir); +} + +jobject ClassLoaderContext::CreateClassLoader( + const std::vector<const DexFile*>& compilation_sources) const { + CheckDexFilesOpened("CreateClassLoader"); + + Thread* self = Thread::Current(); + ScopedObjectAccess soa(self); + + std::vector<const DexFile*> class_path_files; + + // TODO(calin): Transition period: assume we only have a classloader until + // the oat file assistant implements the full class loader check. + if (!class_loader_chain_.empty()) { + CHECK_EQ(1u, class_loader_chain_.size()); + CHECK_EQ(kPathClassLoader, class_loader_chain_[0].type); + class_path_files = MakeNonOwningPointerVector(class_loader_chain_[0].opened_dex_files); + } + + // Classpath: first the class-path given; then the dex files we'll compile. + // Thus we'll resolve the class-path first. + class_path_files.insert(class_path_files.end(), + compilation_sources.begin(), + compilation_sources.end()); + + ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); + return class_linker->CreatePathClassLoader(self, class_path_files); +} + +std::vector<const DexFile*> ClassLoaderContext::FlattenOpenedDexFiles() const { + CheckDexFilesOpened("FlattenOpenedDexFiles"); + + std::vector<const DexFile*> result; + for (const ClassLoaderInfo& info : class_loader_chain_) { + for (const std::unique_ptr<const DexFile>& dex_file : info.opened_dex_files) { + result.push_back(dex_file.get()); + } + } + return result; +} + +const char* ClassLoaderContext::GetClassLoaderTypeName(ClassLoaderType type) { + switch (type) { + case kPathClassLoader: return kPathClassLoaderString; + case kDelegateLastClassLoader: return kDelegateLastClassLoaderString; + default: + LOG(FATAL) << "Invalid class loader type " << type; + UNREACHABLE(); + } +} + +void ClassLoaderContext::CheckDexFilesOpened(const std::string& calling_method) const { + CHECK(dex_files_open_attempted_) + << "Dex files were not successfully opened before the call to " << calling_method + << "attempt=" << dex_files_open_attempted_ << ", result=" << dex_files_open_result_; +} +} // namespace art + diff --git a/runtime/class_loader_context.h b/runtime/class_loader_context.h new file mode 100644 index 0000000000..4af5017ef2 --- /dev/null +++ b/runtime/class_loader_context.h @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_CLASS_LOADER_CONTEXT_H_ +#define ART_RUNTIME_CLASS_LOADER_CONTEXT_H_ + +#include <string> +#include <vector> + +#include "arch/instruction_set.h" +#include "base/dchecked_vector.h" +#include "jni.h" + +namespace art { + +class DexFile; +class OatFile; + +// Utility class which holds the class loader context used during compilation/verification. +class ClassLoaderContext { + public: + // Creates an empty context (with no class loaders). + ClassLoaderContext(); + + // Opens requested class path files and appends them to ClassLoaderInfo::opened_dex_files. + // If the dex files have been stripped, the method opens them from their oat files which are added + // to ClassLoaderInfo::opened_oat_files. The 'classpath_dir' argument specifies the directory to + // use for the relative class paths. + // Returns true if all dex files where successfully opened. + // It may be called only once per ClassLoaderContext. The second call will abort. + // + // Note that a "false" return could mean that either an apk/jar contained no dex files or + // that we hit a I/O or checksum mismatch error. + // TODO(calin): Currently there's no easy way to tell the difference. + // + // TODO(calin): we're forced to complicate the flow in this class with a different + // OpenDexFiles step because the current dex2oat flow requires the dex files be opened before + // the class loader is created. Consider reworking the dex2oat part. + bool OpenDexFiles(InstructionSet isa, const std::string& classpath_dir); + + // Remove the specified compilation sources from all classpaths present in this context. + // Should only be called before the first call to OpenDexFiles(). + bool RemoveLocationsFromClassPaths(const dchecked_vector<std::string>& compilation_sources); + + // Creates the entire class loader hierarchy according to the current context. + // The compilation sources are appended to the classpath of the top class loader + // (i.e the class loader whose parent is the BootClassLoader). + // Should only be called if OpenDexFiles() returned true. + jobject CreateClassLoader(const std::vector<const DexFile*>& compilation_sources) const; + + // Encodes the context as a string suitable to be added in oat files. + // (so that it can be read and verified at runtime against the actual class + // loader hierarchy). + // Should only be called if OpenDexFiles() returned true. + // E.g. if the context is PCL[a.dex:b.dex] this will return "a.dex*a_checksum*b.dex*a_checksum". + std::string EncodeContextForOatFile(const std::string& base_dir) const; + + // Flattens the opened dex files into the given vector. + // Should only be called if OpenDexFiles() returned true. + std::vector<const DexFile*> FlattenOpenedDexFiles() const; + + // Creates the class loader context from the given string. + // The format: ClassLoaderType1[ClasspathElem1:ClasspathElem2...];ClassLoaderType2[...]... + // ClassLoaderType is either "PCL" (PathClassLoader) or "DLC" (DelegateLastClassLoader). + // ClasspathElem is the path of dex/jar/apk file. + // Note that we allowed class loaders with an empty class path in order to support a custom + // class loader for the source dex files. + static std::unique_ptr<ClassLoaderContext> Create(const std::string& spec); + + private: + enum ClassLoaderType { + kInvalidClassLoader = 0, + kPathClassLoader = 1, + kDelegateLastClassLoader = 2 + }; + + struct ClassLoaderInfo { + // The type of this class loader. + ClassLoaderType type; + // The list of class path elements that this loader loads. + // Note that this list may contain relative paths. + std::vector<std::string> classpath; + // After OpenDexFiles is called this holds the opened dex files. + std::vector<std::unique_ptr<const DexFile>> opened_dex_files; + // After OpenDexFiles, in case some of the dex files were opened from their oat files + // this holds the list of opened oat files. + std::vector<std::unique_ptr<OatFile>> opened_oat_files; + + explicit ClassLoaderInfo(ClassLoaderType cl_type) : type(cl_type) {} + }; + + // Reads the class loader spec in place and returns true if the spec is valid and the + // compilation context was constructed. + bool Parse(const std::string& spec); + + // Attempts to parse a single class loader spec for the given class_loader_type. + // If successful the class loader spec will be added to the chain. + // Returns whether or not the operation was successful. + bool ParseClassLoaderSpec(const std::string& class_loader_spec, + ClassLoaderType class_loader_type); + + // Extracts the class loader type from the given spec. + // Return ClassLoaderContext::kInvalidClassLoader if the class loader type is not + // recognized. + static ClassLoaderType ExtractClassLoaderType(const std::string& class_loader_spec); + + // Returns the string representation of the class loader type. + // The returned format can be used when parsing a context spec. + static const char* GetClassLoaderTypeName(ClassLoaderType type); + + // CHECKs that the dex files were opened (OpenDexFiles was called). Aborts if not. + void CheckDexFilesOpened(const std::string& calling_method) const; + + // The class loader chain represented as a vector. + // The parent of class_loader_chain_[i] is class_loader_chain_[i++]. + // The parent of the last element is assumed to be the boot class loader. + std::vector<ClassLoaderInfo> class_loader_chain_; + + // Whether or not the class loader context should be ignored at runtime when loading the oat + // files. When true, dex2oat will use OatFile::kSpecialSharedLibrary as the classpath key in + // the oat file. + // TODO(calin): Can we get rid of this and cover all relevant use cases? + // (e.g. packages using prebuild system packages as shared libraries b/36480683) + bool special_shared_library_; + + // Whether or not OpenDexFiles() was called. + bool dex_files_open_attempted_; + // The result of the last OpenDexFiles() operation. + bool dex_files_open_result_; + + friend class ClassLoaderContextTest; + + DISALLOW_COPY_AND_ASSIGN(ClassLoaderContext); +}; + +} // namespace art +#endif // ART_RUNTIME_CLASS_LOADER_CONTEXT_H_ diff --git a/runtime/class_loader_context_test.cc b/runtime/class_loader_context_test.cc new file mode 100644 index 0000000000..4643e78226 --- /dev/null +++ b/runtime/class_loader_context_test.cc @@ -0,0 +1,265 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + + +#include "class_loader_context.h" +#include "common_runtime_test.h" + +#include "base/dchecked_vector.h" +#include "base/stl_util.h" +#include "class_linker.h" +#include "dex_file.h" +#include "handle_scope-inl.h" +#include "mirror/class.h" +#include "mirror/class_loader.h" +#include "mirror/object-inl.h" +#include "oat_file_assistant.h" +#include "runtime.h" +#include "scoped_thread_state_change-inl.h" +#include "thread.h" +#include "well_known_classes.h" + +namespace art { + +class ClassLoaderContextTest : public CommonRuntimeTest { + public: + void VerifyContextSize(ClassLoaderContext* context, size_t expected_size) { + ASSERT_TRUE(context != nullptr); + ASSERT_EQ(expected_size, context->class_loader_chain_.size()); + } + + void VerifyClassLoaderPCL(ClassLoaderContext* context, + size_t index, + std::string classpath) { + VerifyClassLoaderInfo( + context, index, ClassLoaderContext::kPathClassLoader, classpath); + } + + void VerifyClassLoaderDLC(ClassLoaderContext* context, + size_t index, + std::string classpath) { + VerifyClassLoaderInfo( + context, index, ClassLoaderContext::kDelegateLastClassLoader, classpath); + } + + void VerifyOpenDexFiles( + ClassLoaderContext* context, + size_t index, + std::vector<std::vector<std::unique_ptr<const DexFile>>*>& all_dex_files) { + ASSERT_TRUE(context != nullptr); + ASSERT_TRUE(context->dex_files_open_attempted_); + ASSERT_TRUE(context->dex_files_open_result_); + ClassLoaderContext::ClassLoaderInfo& info = context->class_loader_chain_[index]; + ASSERT_EQ(all_dex_files.size(), info.classpath.size()); + size_t cur_open_dex_index = 0; + for (size_t k = 0; k < all_dex_files.size(); k++) { + std::vector<std::unique_ptr<const DexFile>>& dex_files_for_cp_elem = *(all_dex_files[k]); + for (size_t i = 0; i < dex_files_for_cp_elem.size(); i++) { + ASSERT_LT(cur_open_dex_index, info.opened_dex_files.size()); + + std::unique_ptr<const DexFile>& opened_dex_file = + info.opened_dex_files[cur_open_dex_index++]; + std::unique_ptr<const DexFile>& expected_dex_file = dex_files_for_cp_elem[i]; + + ASSERT_EQ(expected_dex_file->GetLocation(), opened_dex_file->GetLocation()); + ASSERT_EQ(expected_dex_file->GetLocationChecksum(), opened_dex_file->GetLocationChecksum()); + ASSERT_EQ(info.classpath[k], opened_dex_file->GetBaseLocation()); + } + } + } + + private: + void VerifyClassLoaderInfo(ClassLoaderContext* context, + size_t index, + ClassLoaderContext::ClassLoaderType type, + std::string classpath) { + ASSERT_TRUE(context != nullptr); + ASSERT_GT(context->class_loader_chain_.size(), index); + ClassLoaderContext::ClassLoaderInfo& info = context->class_loader_chain_[index]; + ASSERT_EQ(type, info.type); + std::vector<std::string> expected_classpath; + Split(classpath, ':', &expected_classpath); + ASSERT_EQ(expected_classpath, info.classpath); + } +}; + +TEST_F(ClassLoaderContextTest, ParseValidContextPCL) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[a.dex]"); + VerifyContextSize(context.get(), 1); + VerifyClassLoaderPCL(context.get(), 0, "a.dex"); +} + +TEST_F(ClassLoaderContextTest, ParseValidContextDLC) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("DLC[a.dex]"); + VerifyContextSize(context.get(), 1); + VerifyClassLoaderDLC(context.get(), 0, "a.dex"); +} + +TEST_F(ClassLoaderContextTest, ParseValidContextChain) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[a.dex:b.dex];DLC[c.dex:d.dex];PCL[e.dex]"); + VerifyContextSize(context.get(), 3); + VerifyClassLoaderPCL(context.get(), 0, "a.dex:b.dex"); + VerifyClassLoaderDLC(context.get(), 1, "c.dex:d.dex"); + VerifyClassLoaderPCL(context.get(), 2, "e.dex"); +} + +TEST_F(ClassLoaderContextTest, ParseValidEmptyContextDLC) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("DLC[]"); + VerifyContextSize(context.get(), 1); + VerifyClassLoaderDLC(context.get(), 0, ""); +} + +TEST_F(ClassLoaderContextTest, ParseValidContextSpecialSymbol) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create(OatFile::kSpecialSharedLibrary); + VerifyContextSize(context.get(), 0); +} + +TEST_F(ClassLoaderContextTest, ParseInvalidValidContexts) { + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("ABC[a.dex]")); + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("PCL")); + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("PCL[a.dex")); + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("PCLa.dex]")); + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("PCL{a.dex}")); + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("PCL[a.dex];DLC[b.dex")); +} + +TEST_F(ClassLoaderContextTest, OpenInvalidDexFiles) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[does_not_exist.dex]"); + VerifyContextSize(context.get(), 1); + ASSERT_FALSE(context->OpenDexFiles(InstructionSet::kArm, ".")); +} + +TEST_F(ClassLoaderContextTest, OpenValidDexFiles) { + std::string multidex_name = GetTestDexFileName("MultiDex"); + std::vector<std::unique_ptr<const DexFile>> multidex_files = OpenTestDexFiles("MultiDex"); + std::string myclass_dex_name = GetTestDexFileName("MyClass"); + std::vector<std::unique_ptr<const DexFile>> myclass_dex_files = OpenTestDexFiles("MyClass"); + std::string dex_name = GetTestDexFileName("Main"); + std::vector<std::unique_ptr<const DexFile>> dex_files = OpenTestDexFiles("Main"); + + + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create( + "PCL[" + multidex_name + ":" + myclass_dex_name + "];" + + "DLC[" + dex_name + "]"); + + ASSERT_TRUE(context->OpenDexFiles(InstructionSet::kArm, /*classpath_dir*/ "")); + + VerifyContextSize(context.get(), 2); + std::vector<std::vector<std::unique_ptr<const DexFile>>*> all_dex_files0; + all_dex_files0.push_back(&multidex_files); + all_dex_files0.push_back(&myclass_dex_files); + std::vector<std::vector<std::unique_ptr<const DexFile>>*> all_dex_files1; + all_dex_files1.push_back(&dex_files); + + VerifyOpenDexFiles(context.get(), 0, all_dex_files0); + VerifyOpenDexFiles(context.get(), 1, all_dex_files1); +} + +TEST_F(ClassLoaderContextTest, OpenInvalidDexFilesMix) { + std::string dex_name = GetTestDexFileName("Main"); + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[does_not_exist.dex];DLC[" + dex_name + "]"); + ASSERT_FALSE(context->OpenDexFiles(InstructionSet::kArm, "")); +} + +TEST_F(ClassLoaderContextTest, CreateClassLoader) { + std::string dex_name = GetTestDexFileName("Main"); + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[" + dex_name + "]"); + ASSERT_TRUE(context->OpenDexFiles(InstructionSet::kArm, "")); + + std::vector<std::unique_ptr<const DexFile>> classpath_dex = OpenTestDexFiles("Main"); + std::vector<std::unique_ptr<const DexFile>> compilation_sources = OpenTestDexFiles("MultiDex"); + + std::vector<const DexFile*> compilation_sources_raw = + MakeNonOwningPointerVector(compilation_sources); + jobject jclass_loader = context->CreateClassLoader(compilation_sources_raw); + ASSERT_TRUE(jclass_loader != nullptr); + + ScopedObjectAccess soa(Thread::Current()); + + StackHandleScope<2> hs(soa.Self()); + Handle<mirror::ClassLoader> class_loader = hs.NewHandle( + soa.Decode<mirror::ClassLoader>(jclass_loader)); + + ASSERT_TRUE(class_loader->GetClass() == + soa.Decode<mirror::Class>(WellKnownClasses::dalvik_system_PathClassLoader)); + ASSERT_TRUE(class_loader->GetParent()->GetClass() == + soa.Decode<mirror::Class>(WellKnownClasses::java_lang_BootClassLoader)); + + + std::vector<const DexFile*> class_loader_dex_files = GetDexFiles(jclass_loader); + ASSERT_EQ(classpath_dex.size() + compilation_sources.size(), class_loader_dex_files.size()); + + // The classpath dex files must come first. + for (size_t i = 0; i < classpath_dex.size(); i++) { + ASSERT_EQ(classpath_dex[i]->GetLocation(), + class_loader_dex_files[i]->GetLocation()); + ASSERT_EQ(classpath_dex[i]->GetLocationChecksum(), + class_loader_dex_files[i]->GetLocationChecksum()); + } + + // The compilation dex files must come second. + for (size_t i = 0, k = classpath_dex.size(); i < compilation_sources.size(); i++, k++) { + ASSERT_EQ(compilation_sources[i]->GetLocation(), + class_loader_dex_files[k]->GetLocation()); + ASSERT_EQ(compilation_sources[i]->GetLocationChecksum(), + class_loader_dex_files[k]->GetLocationChecksum()); + } +} + +TEST_F(ClassLoaderContextTest, RemoveSourceLocations) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[a.dex]"); + dchecked_vector<std::string> classpath_dex; + classpath_dex.push_back("a.dex"); + dchecked_vector<std::string> compilation_sources; + compilation_sources.push_back("src.dex"); + + // Nothing should be removed. + ASSERT_FALSE(context->RemoveLocationsFromClassPaths(compilation_sources)); + VerifyClassLoaderPCL(context.get(), 0, "a.dex"); + // Classes should be removed. + ASSERT_TRUE(context->RemoveLocationsFromClassPaths(classpath_dex)); + VerifyClassLoaderPCL(context.get(), 0, ""); +} + +TEST_F(ClassLoaderContextTest, EncodeInOatFile) { + std::string dex1_name = GetTestDexFileName("Main"); + std::string dex2_name = GetTestDexFileName("MyClass"); + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[" + dex1_name + ":" + dex2_name + "]"); + ASSERT_TRUE(context->OpenDexFiles(InstructionSet::kArm, "")); + + std::vector<std::unique_ptr<const DexFile>> dex1 = OpenTestDexFiles("Main"); + std::vector<std::unique_ptr<const DexFile>> dex2 = OpenTestDexFiles("MyClass"); + std::string encoding = context->EncodeContextForOatFile(""); + std::string expected_encoding = + dex1[0]->GetLocation() + "*" + std::to_string(dex1[0]->GetLocationChecksum()) + "*" + + dex2[0]->GetLocation() + "*" + std::to_string(dex2[0]->GetLocationChecksum()) + "*"; + ASSERT_EQ(expected_encoding, context->EncodeContextForOatFile("")); +} + +} // namespace art diff --git a/runtime/interpreter/interpreter_common.h b/runtime/interpreter/interpreter_common.h index 38edc7a9e7..74fec48342 100644 --- a/runtime/interpreter/interpreter_common.h +++ b/runtime/interpreter/interpreter_common.h @@ -241,7 +241,7 @@ static inline bool DoInvokeVirtualQuick(Thread* self, ShadowFrame& shadow_frame, } CHECK(receiver->GetClass()->ShouldHaveEmbeddedVTable()); ArtMethod* const called_method = receiver->GetClass()->GetEmbeddedVTableEntry( - vtable_idx, kRuntimePointerSize); + vtable_idx, Runtime::Current()->GetClassLinker()->GetImagePointerSize()); if (UNLIKELY(called_method == nullptr)) { CHECK(self->IsExceptionPending()); result->SetJ(0); diff --git a/runtime/interpreter/unstarted_runtime_test.cc b/runtime/interpreter/unstarted_runtime_test.cc index c314f3c35e..c2ef72460d 100644 --- a/runtime/interpreter/unstarted_runtime_test.cc +++ b/runtime/interpreter/unstarted_runtime_test.cc @@ -386,8 +386,9 @@ TEST_F(UnstartedRuntimeTest, StringInit) { Thread* self = Thread::Current(); ScopedObjectAccess soa(self); mirror::Class* klass = mirror::String::GetJavaLangString(); - ArtMethod* method = klass->FindDeclaredDirectMethod("<init>", "(Ljava/lang/String;)V", - kRuntimePointerSize); + ArtMethod* method = + klass->FindDeclaredDirectMethod("<init>", "(Ljava/lang/String;)V", + Runtime::Current()->GetClassLinker()->GetImagePointerSize()); // create instruction data for invoke-direct {v0, v1} of method with fake index uint16_t inst_data[3] = { 0x2070, 0x0000, 0x0010 }; @@ -1335,10 +1336,16 @@ TEST_F(UnstartedRuntimeTest, ConstructorNewInstance0) { ArtMethod* throw_cons = throw_class->FindDeclaredDirectMethod( "<init>", "(Ljava/lang/String;)V", class_linker->GetImagePointerSize()); ASSERT_TRUE(throw_cons != nullptr); - - Handle<mirror::Constructor> cons = hs.NewHandle( - mirror::Constructor::CreateFromArtMethod<kRuntimePointerSize, false>(self, throw_cons)); - ASSERT_TRUE(cons != nullptr); + Handle<mirror::Constructor> cons; + if (class_linker->GetImagePointerSize() == PointerSize::k64) { + cons = hs.NewHandle( + mirror::Constructor::CreateFromArtMethod<PointerSize::k64, false>(self, throw_cons)); + ASSERT_TRUE(cons != nullptr); + } else { + cons = hs.NewHandle( + mirror::Constructor::CreateFromArtMethod<PointerSize::k32, false>(self, throw_cons)); + ASSERT_TRUE(cons != nullptr); + } Handle<mirror::ObjectArray<mirror::Object>> args = hs.NewHandle( mirror::ObjectArray<mirror::Object>::Alloc( diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc index 888de457dc..2ed30df372 100644 --- a/runtime/oat_file.cc +++ b/runtime/oat_file.cc @@ -1577,7 +1577,7 @@ CompilerFilter::Filter OatFile::GetCompilerFilter() const { static constexpr char kDexClassPathEncodingSeparator = '*'; std::string OatFile::EncodeDexFileDependencies(const std::vector<const DexFile*>& dex_files, - std::string& base_dir) { + const std::string& base_dir) { std::ostringstream out; for (const DexFile* dex_file : dex_files) { diff --git a/runtime/oat_file.h b/runtime/oat_file.h index 66ed44f1b9..6393e091db 100644 --- a/runtime/oat_file.h +++ b/runtime/oat_file.h @@ -299,7 +299,7 @@ class OatFile { // Create a dependency list (dex locations and checksums) for the given dex files. // Removes dex file paths prefixed with base_dir to convert them back to relative paths. static std::string EncodeDexFileDependencies(const std::vector<const DexFile*>& dex_files, - std::string& base_dir); + const std::string& base_dir); // Finds the associated oat class for a dex_file and descriptor. Returns an invalid OatClass on // error and sets found to false. diff --git a/runtime/oat_file_assistant.cc b/runtime/oat_file_assistant.cc index 4820feb56c..c8766578c4 100644 --- a/runtime/oat_file_assistant.cc +++ b/runtime/oat_file_assistant.cc @@ -298,28 +298,38 @@ std::string OatFileAssistant::GetStatusDump() { } std::vector<std::unique_ptr<const DexFile>> OatFileAssistant::LoadDexFiles( - const OatFile& oat_file, const char* dex_location) { + const OatFile &oat_file, const char *dex_location) { std::vector<std::unique_ptr<const DexFile>> dex_files; + if (LoadDexFiles(oat_file, dex_location, &dex_files)) { + return dex_files; + } else { + return std::vector<std::unique_ptr<const DexFile>>(); + } +} +bool OatFileAssistant::LoadDexFiles( + const OatFile &oat_file, + const std::string& dex_location, + std::vector<std::unique_ptr<const DexFile>>* out_dex_files) { // Load the main dex file. std::string error_msg; const OatFile::OatDexFile* oat_dex_file = oat_file.GetOatDexFile( - dex_location, nullptr, &error_msg); + dex_location.c_str(), nullptr, &error_msg); if (oat_dex_file == nullptr) { LOG(WARNING) << error_msg; - return std::vector<std::unique_ptr<const DexFile>>(); + return false; } std::unique_ptr<const DexFile> dex_file = oat_dex_file->OpenDexFile(&error_msg); if (dex_file.get() == nullptr) { LOG(WARNING) << "Failed to open dex file from oat dex file: " << error_msg; - return std::vector<std::unique_ptr<const DexFile>>(); + return false; } - dex_files.push_back(std::move(dex_file)); + out_dex_files->push_back(std::move(dex_file)); // Load the rest of the multidex entries - for (size_t i = 1; ; i++) { - std::string multidex_dex_location = DexFile::GetMultiDexLocation(i, dex_location); + for (size_t i = 1;; i++) { + std::string multidex_dex_location = DexFile::GetMultiDexLocation(i, dex_location.c_str()); oat_dex_file = oat_file.GetOatDexFile(multidex_dex_location.c_str(), nullptr); if (oat_dex_file == nullptr) { // There are no more multidex entries to load. @@ -329,11 +339,11 @@ std::vector<std::unique_ptr<const DexFile>> OatFileAssistant::LoadDexFiles( dex_file = oat_dex_file->OpenDexFile(&error_msg); if (dex_file.get() == nullptr) { LOG(WARNING) << "Failed to open dex file from oat dex file: " << error_msg; - return std::vector<std::unique_ptr<const DexFile>>(); + return false; } - dex_files.push_back(std::move(dex_file)); + out_dex_files->push_back(std::move(dex_file)); } - return dex_files; + return true; } bool OatFileAssistant::HasOriginalDexFiles() { diff --git a/runtime/oat_file_assistant.h b/runtime/oat_file_assistant.h index 03d9ca38a8..92d87eaeae 100644 --- a/runtime/oat_file_assistant.h +++ b/runtime/oat_file_assistant.h @@ -207,6 +207,13 @@ class OatFileAssistant { static std::vector<std::unique_ptr<const DexFile>> LoadDexFiles( const OatFile& oat_file, const char* dex_location); + // Same as `std::vector<std::unique_ptr<const DexFile>> LoadDexFiles(...)` with the difference: + // - puts the dex files in the given vector + // - returns whether or not all dex files were successfully opened + static bool LoadDexFiles(const OatFile& oat_file, + const std::string& dex_location, + std::vector<std::unique_ptr<const DexFile>>* out_dex_files); + // Returns true if there are dex files in the original dex location that can // be compiled with dex2oat for this dex location. // Returns false if there is no original dex file, or if the original dex diff --git a/runtime/thread.cc b/runtime/thread.cc index 36ecd3398c..fc30832189 100644 --- a/runtime/thread.cc +++ b/runtime/thread.cc @@ -2134,6 +2134,10 @@ void Thread::Destroy() { ScopedObjectAccess soa(self); // We may need to call user-supplied managed code, do this before final clean-up. HandleUncaughtExceptions(soa); + Runtime* runtime = Runtime::Current(); + if (runtime != nullptr) { + runtime->GetRuntimeCallbacks()->ThreadDeath(self); + } RemoveFromThreadGroup(soa); // this.nativePeer = 0; @@ -2144,11 +2148,6 @@ void Thread::Destroy() { jni::DecodeArtField(WellKnownClasses::java_lang_Thread_nativePeer) ->SetLong<false>(tlsPtr_.opeer, 0); } - Runtime* runtime = Runtime::Current(); - if (runtime != nullptr) { - runtime->GetRuntimeCallbacks()->ThreadDeath(self); - } - // Thread.join() is implemented as an Object.wait() on the Thread.lock object. Signal anyone // who is waiting. diff --git a/test/912-classes/src-art/art/Test912.java b/test/912-classes/src-art/art/Test912.java index 9896eacfeb..fbf8794c50 100644 --- a/test/912-classes/src-art/art/Test912.java +++ b/test/912-classes/src-art/art/Test912.java @@ -228,7 +228,8 @@ public class Test912 { // The JIT may deeply inline and load some classes. Preload these for test determinism. final String PRELOAD_FOR_JIT[] = { "java.nio.charset.CoderMalfunctionError", - "java.util.NoSuchElementException" + "java.util.NoSuchElementException", + "java.io.FileNotFoundException", // b/63581208 }; for (String s : PRELOAD_FOR_JIT) { Class.forName(s); diff --git a/tools/libcore_failures.txt b/tools/libcore_failures.txt index 8a4c2df2c1..c6553f89fa 100644 --- a/tools/libcore_failures.txt +++ b/tools/libcore_failures.txt @@ -223,5 +223,10 @@ result: EXEC_FAILED, bug: 62408076, names: ["libcore.java.lang.reflect.annotations.AnnotatedElementParameterTest#testImplicitConstructorParameters_singleAnnotation"] +}, +{ + description: "java.io.IOException: Error writing ASN.1 encoding", + result: EXEC_FAILED, + names: ["libcore.javax.crypto.spec.AlgorithmParametersTestGCM#testEncoding"] } ] diff --git a/tools/test_presubmit.py b/tools/test_presubmit.py new file mode 100755 index 0000000000..f6e6df9a68 --- /dev/null +++ b/tools/test_presubmit.py @@ -0,0 +1,159 @@ +#!/usr/bin/python3 +# +# Copyright 2017, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# There are many run-tests which generate their sources automatically. +# It is desirable to keep the checked-in source code, as we re-run generators very rarely. +# +# This script will re-run the generators only if their dependent files have changed and then +# complain if the outputs no longer matched what's in the source tree. +# + +import os +import pathlib +import subprocess +import sys +import tempfile + +THIS_PATH = os.path.dirname(os.path.realpath(__file__)) + +TOOLS_GEN_SRCS = [ + # tool -> path to a script to generate a file + # reference_files -> list of files that the script can generate + # args -> lambda(path) that generates arguments the 'tool' in order to output to 'path' + # interesting_files -> which files much change in order to re-run the tool. + # interesting_to_reference_files: lambda(x,reference_files) + # given the interesting file 'x' and a list of reference_files, + # return exactly one reference file that corresponds to it. + { 'tool' : 'test/988-method-trace/gen_srcs.py', + 'reference_files' : ['test/988-method-trace/src/art/Test988Intrinsics.java'], + 'args' : lambda output_path: [output_path], + 'interesting_files' : ['compiler/intrinsics_list.h'], + 'interesting_to_reference_file' : lambda interesting, references: references[0], + }, +] + +DEBUG = False + +def debug_print(msg): + if DEBUG: + print("[DEBUG]: " + msg, file=sys.stderr) + +def is_interesting(f, tool_dict): + """ + Returns true if this is a file we want to run this tool before uploading. False otherwise. + """ + path = pathlib.Path(f) + return str(path) in tool_dict['interesting_files'] + +def get_changed_files(commit): + """ + Gets the files changed in the given commit. + """ + return subprocess.check_output( + ["git", 'diff-tree', '--no-commit-id', '--name-only', '-r', commit], + stderr=subprocess.STDOUT, + universal_newlines=True).split() + +def command_line_for_tool(tool_dict, output): + """ + Calculate the command line for this tool when ran against the output file 'output'. + """ + proc_args = [tool_dict['tool']] + tool_dict['args'](output) + return proc_args + +def run_tool(tool_dict, output): + """ + Execute this tool by passing the tool args to the tool. + """ + proc_args = command_line_for_tool(tool_dict, output) + debug_print("PROC_ARGS: %s" %(proc_args)) + succ = subprocess.call(proc_args) + return succ + +def get_reference_file(changed_file, tool_dict): + """ + Lookup the file that the tool is generating in response to changing an interesting file + """ + return tool_dict['interesting_to_reference_file'](changed_file, tool_dict['reference_files']) + +def run_diff(changed_file, tool_dict, original_file): + ref_file = get_reference_file(changed_file, tool_dict) + + return subprocess.call(["diff", ref_file, original_file]) != 0 + +def run_gen_srcs(files): + """ + Runs test tools only for interesting files that were changed in this commit. + """ + if len(files) == 0: + return + + success = 0 # exit code 0 = success, >0 error. + had_diffs = False + + for tool_dict in TOOLS_GEN_SRCS: + tool_ran_at_least_once = False + for f in files: + if is_interesting(f, tool_dict): + tmp_file = tempfile.mktemp() + reference_file = get_reference_file(f, tool_dict) + + # Generate the source code with a temporary file as the output. + success = run_tool(tool_dict, tmp_file) + if success != 0: + # Immediately abort if the tool fails with a non-0 exit code, do not go any further. + print("[FATAL] Error when running tool (return code %s)" %(success), file=sys.stderr) + print("$> %s" %(" ".join(command_line_for_tool(tool_dict, tmp_file))), file=sys.stderr) + sys.exit(success) + if run_diff(f, tool_dict, tmp_file): + # If the tool succeeded, but there was a diff, then the generated code has diverged. + # Output the diff information and continue to the next files/tools. + had_diffs = True + print("-----------------------------------------------------------", file=sys.stderr) + print("File '%s' diverged from generated file; please re-run tools:" %(reference_file), file=sys.stderr) + print("$> %s" %(" ".join(command_line_for_tool(tool_dict, reference_file))), file=sys.stderr) + else: + debug_print("File %s is consistent with tool %s" %(reference_file, tool_dict['tool'])) + + tool_ran_at_least_once = True + + if not tool_ran_at_least_once: + debug_print("Interesting files %s unchanged, skipping tool '%s'" %(tool_dict['interesting_files'], tool_dict['tool'])) + + if had_diffs: + success = 1 + # Always return non-0 exit code when there were diffs so that the presubmit hooks are FAILED. + + return success + + +def main(): + if 'PREUPLOAD_COMMIT' in os.environ: + commit = os.environ['PREUPLOAD_COMMIT'] + else: + print("WARNING: Not running as a pre-upload hook. Assuming commit to check = 'HEAD'", file=sys.stderr) + commit = "HEAD" + + os.chdir(os.path.join(THIS_PATH, '..')) # run tool relative to 'art' directory + debug_print("CWD: %s" %(os.getcwd())) + + changed_files = get_changed_files(commit) + debug_print("Changed files: %s" %(changed_files)) + return run_gen_srcs(changed_files) + +if __name__ == '__main__': + sys.exit(main()) |