Create vdex file for dex loaded with InMemoryDexClassLoader
Previous CL introduced a background verification thread for dex bytecode
loaded with InMemoryDexClassLoader. Extend the logic to collect the
results of class verification into an instance of VerifierDeps and dump
it into a vdex file in the app's data folder.
The background thread does not collect full VerifierDeps (e.g.
assignability dependencies, etc), just a bit vector of whether a class
was successfully verified or not.
The vdex format is extended to include boot classpath checksums and the
class loader context it was created for. These are optional and
currently left empty for regular vdex files.
The generated vdex files are treated as a cache with a limited capacity,
currently capped at 8 files. The least recently used file (in terms of
atime reported by stat()) is unlinked if the cache is full and a new
vdex is about to be generated.
Bug: 72131483
Test: art/tools/run-libcore-tests.sh
Test: art/test.py -b -r -t 692 -t 693
Change-Id: I26080d894d34d8f35f00c7925db569f22f008d2c
diff --git a/test/692-vdex-inmem-loader/expected.txt b/test/692-vdex-inmem-loader/expected.txt
index 0990d72..a127604 100644
--- a/test/692-vdex-inmem-loader/expected.txt
+++ b/test/692-vdex-inmem-loader/expected.txt
@@ -1,3 +1,4 @@
JNI_OnLoad called
Hello
Hello
+Hello
diff --git a/test/692-vdex-inmem-loader/src/Main.java b/test/692-vdex-inmem-loader/src/Main.java
index b3a5e58..75aef8b 100644
--- a/test/692-vdex-inmem-loader/src/Main.java
+++ b/test/692-vdex-inmem-loader/src/Main.java
@@ -40,9 +40,15 @@
return new ClassLoader[] { clA, clB };
}
- private static void test(ClassLoader loader, boolean invokeMethod) throws Exception {
+ private static void test(ClassLoader loader,
+ boolean expectedHasVdexFile,
+ boolean invokeMethod) throws Exception {
+ // If ART created a vdex file, it must have verified all the classes.
+ boolean expectedClassesVerified = expectedHasVdexFile;
+
waitForVerifier();
- check(!isDebuggable(), areClassesVerified(loader), "areClassesVerified");
+ check(expectedClassesVerified, areClassesVerified(loader), "areClassesVerified");
+ check(expectedHasVdexFile, hasVdexFile(loader), "areClassesVerified");
if (invokeMethod) {
loader.loadClass("art.ClassB").getDeclaredMethod("printHello").invoke(null);
@@ -52,21 +58,31 @@
public static void main(String[] args) throws Exception {
System.loadLibrary(args[0]);
ClassLoader[] loaders = null;
+ boolean featureEnabled = !isDebuggable();
- // Test loading both dex files in a single class loader.
- // Background verification task should verify all their classes.
- test(singleLoader(), /*invokeMethod*/true);
+ // Data directory not set. Background verification job should not have run
+ // and vdex should not have been created.
+ test(singleLoader(), /*hasVdex*/ false, /*invokeMethod*/ true);
+
+ // Set data directory for this process.
+ setProcessDataDir(DEX_LOCATION);
+
+ // Data directory is now set. Background verification job should have run,
+ // should have verified classes and written results to a vdex.
+ test(singleLoader(), /*hasVdex*/ featureEnabled, /*invokeMethod*/ true);
// Test loading the two dex files with separate class loaders.
// Background verification task should still verify all classes.
loaders = multiLoader();
- test(loaders[0], /*invokeMethod*/false);
- test(loaders[1], /*invokeMethod*/true);
+ test(loaders[0], /*hasVdex*/ featureEnabled, /*invokeMethod*/ false);
+ test(loaders[1], /*hasVdex*/ featureEnabled, /*invokeMethod*/ true);
}
private static native boolean isDebuggable();
+ private static native void setProcessDataDir(String path);
private static native void waitForVerifier();
private static native boolean areClassesVerified(ClassLoader loader);
+ private static native boolean hasVdexFile(ClassLoader loader);
// Defined in 674-hiddenapi.
private static native void appendToBootClassLoader(String dexPath, boolean isCorePlatform);
diff --git a/test/692-vdex-inmem-loader/vdex_inmem_loader.cc b/test/692-vdex-inmem-loader/vdex_inmem_loader.cc
index f064953..a5d09e9 100644
--- a/test/692-vdex-inmem-loader/vdex_inmem_loader.cc
+++ b/test/692-vdex-inmem-loader/vdex_inmem_loader.cc
@@ -29,6 +29,12 @@
Runtime::Current()->GetOatFileManager().WaitForBackgroundVerificationTasks();
}
+extern "C" JNIEXPORT void JNICALL Java_Main_setProcessDataDir(JNIEnv* env, jclass, jstring jpath) {
+ const char* path = env->GetStringUTFChars(jpath, nullptr);
+ Runtime::Current()->SetProcessDataDirectory(path);
+ env->ReleaseStringUTFChars(jpath, path);
+}
+
extern "C" JNIEXPORT jboolean JNICALL Java_Main_areClassesVerified(JNIEnv*,
jclass,
jobject loader) {
@@ -68,5 +74,47 @@
return all_verified ? JNI_TRUE : JNI_FALSE;
}
+extern "C" JNIEXPORT bool JNICALL Java_Main_hasVdexFile(JNIEnv*,
+ jclass,
+ jobject loader) {
+ ScopedObjectAccess soa(Thread::Current());
+ StackHandleScope<1> hs(soa.Self());
+ Handle<mirror::ClassLoader> h_loader = hs.NewHandle(soa.Decode<mirror::ClassLoader>(loader));
+
+ std::vector<const DexFile::Header*> dex_headers;
+ VisitClassLoaderDexFiles(
+ soa,
+ h_loader,
+ [&](const DexFile* dex_file) {
+ dex_headers.push_back(&dex_file->GetHeader());
+ return true;
+ });
+
+ uint32_t location_checksum;
+ std::string dex_location;
+ std::string vdex_filename;
+ std::string error_msg;
+ return OatFileAssistant::AnonymousDexVdexLocation(dex_headers,
+ kRuntimeISA,
+ &location_checksum,
+ &dex_location,
+ &vdex_filename) &&
+ OS::FileExists(vdex_filename.c_str());
+}
+
+extern "C" JNIEXPORT jint JNICALL Java_Main_getVdexCacheSize(JNIEnv*, jclass) {
+ return static_cast<jint>(OatFileManager::kAnonymousVdexCacheSize);
+}
+
+extern "C" JNIEXPORT jboolean JNICALL Java_Main_isAnonymousVdexBasename(JNIEnv* env,
+ jclass,
+ jstring basename) {
+ if (basename == nullptr) {
+ return JNI_FALSE;
+ }
+ ScopedUtfChars basename_utf(env, basename);
+ return OatFileAssistant::IsAnonymousVdexBasename(basename_utf.c_str()) ? JNI_TRUE : JNI_FALSE;
+}
+
} // namespace Test692VdexInmemLoader
} // namespace art
diff --git a/test/693-vdex-inmem-loader-evict/expected.txt b/test/693-vdex-inmem-loader-evict/expected.txt
new file mode 100644
index 0000000..6a5618e
--- /dev/null
+++ b/test/693-vdex-inmem-loader-evict/expected.txt
@@ -0,0 +1 @@
+JNI_OnLoad called
diff --git a/test/693-vdex-inmem-loader-evict/info.txt b/test/693-vdex-inmem-loader-evict/info.txt
new file mode 100644
index 0000000..29a48d1
--- /dev/null
+++ b/test/693-vdex-inmem-loader-evict/info.txt
@@ -0,0 +1,2 @@
+Check that caching of verification results for InMemoryDexClassLoader obeys a cap on the number
+of vdex files in the data folder. Least recently used vdex files should be unlinked.
\ No newline at end of file
diff --git a/test/693-vdex-inmem-loader-evict/src-secondary/gen.sh b/test/693-vdex-inmem-loader-evict/src-secondary/gen.sh
new file mode 100755
index 0000000..96b426d
--- /dev/null
+++ b/test/693-vdex-inmem-loader-evict/src-secondary/gen.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Copyright 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script generates dex files which all contain an empty class called
+# MyClassXX, where XX is the ID of the dex file. It prints the first file
+# (ID=01) and a list of checksum/signature bytes for all of the dex files
+# (ID 01 through NUM_FILES).
+# The idea is that the associated test will be able to efficiently generate
+# up to NUM_FILES dex files from the first dex file by substituting its
+# checksum/signature bytes with that of the dex file of a given ID. Note that
+# it is also necessary to replace the ID in the class descriptor, but this
+# script does not help with that.
+
+set -e
+TMP=`mktemp -d`
+NUM_FILES=30
+
+echo ' private static final byte[][] DEX_CHECKSUMS = new byte[][] {'
+for i in $(seq 1 ${NUM_FILES}); do
+ if [ ${i} -lt 10 ]; then
+ suffix=0${i}
+ else
+ suffix=${i}
+ fi
+ (cd "$TMP" && \
+ echo "public class MyClass${suffix} { }" > "$TMP/MyClass${suffix}.java" && \
+ javac -d "${TMP}" "$TMP/MyClass${suffix}.java" && \
+ d8 --output "$TMP" "$TMP/MyClass${suffix}.class" && \
+ mv "$TMP/classes.dex" "$TMP/file${suffix}.dex")
+
+ # Dump bytes 8-32 (checksum + signature) that need to change for other files.
+ checksum=`head -c 32 -z "$TMP/file${suffix}.dex" | tail -c 24 -z | base64`
+ echo ' Base64.getDecoder().decode("'${checksum}'"),'
+done
+echo ' };'
+
+# Dump first dex file as base.
+echo ' private static final byte[] DEX_BYTES_01 = Base64.getDecoder().decode('
+base64 "${TMP}/file01.dex" | sed -E 's/^/ "/' | sed ':a;N;$!ba;s/\n/" +\n/g' | sed -E '$ s/$/");/'
+
+rm -rf "$TMP"
diff --git a/test/693-vdex-inmem-loader-evict/src/Main.java b/test/693-vdex-inmem-loader-evict/src/Main.java
new file mode 100644
index 0000000..7d9da1b
--- /dev/null
+++ b/test/693-vdex-inmem-loader-evict/src/Main.java
@@ -0,0 +1,182 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+import dalvik.system.InMemoryDexClassLoader;
+import java.lang.reflect.Method;
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.Base64;
+
+public class Main {
+ public static void main(String[] args) throws Exception {
+ System.loadLibrary(args[0]);
+
+ if (isDebuggable()) {
+ // Background verification is disabled in debuggable mode. This test makes
+ // no sense then.
+ return;
+ }
+
+ setProcessDataDir(DEX_LOCATION);
+
+ final int maxCacheSize = getVdexCacheSize();
+ final int numDexFiles = DEX_BYTES_CHECKSUMS.length;
+ if (numDexFiles <= maxCacheSize) {
+ throw new IllegalStateException("Not enough dex files to test cache eviction");
+ }
+
+ // Simply load each dex file one by one.
+ check(0, getCurrentCacheSize(), "There should be no vdex files in the beginning");
+ for (int i = 0; i < numDexFiles; ++i) {
+ ClassLoader loader = loadDex(i);
+ waitForVerifier();
+ check(true, hasVdexFile(loader), "Loading dex file should have produced a vdex");
+ check(Math.min(i + 1, maxCacheSize), getCurrentCacheSize(),
+ "Unexpected number of cache entries");
+ }
+
+ // More complicated pattern where some dex files get reused.
+ for (int s = 1; s < numDexFiles; ++s) {
+ for (int i = 0; i < maxCacheSize; ++i) {
+ ClassLoader loader = loadDex(i);
+ waitForVerifier();
+ check(true, hasVdexFile(loader), "Loading dex file should have produced a vdex");
+ check(maxCacheSize, getCurrentCacheSize(), "Unexpected number of cache entries");
+ }
+ }
+ }
+
+ private static native boolean isDebuggable();
+ private static native void setProcessDataDir(String path);
+ private static native void waitForVerifier();
+ private static native boolean hasVdexFile(ClassLoader loader);
+ private static native int getVdexCacheSize();
+ private static native boolean isAnonymousVdexBasename(String basename);
+
+ private static <T> void check(T expected, T actual, String message) {
+ if (!expected.equals(actual)) {
+ System.err.println("ERROR: " + message + " (expected=" + expected.toString() +
+ ", actual=" + actual.toString() + ")");
+ }
+ }
+
+ private static int getCurrentCacheSize() {
+ int count = 0;
+ File folder = new File(DEX_LOCATION, "oat");
+ File[] subfolders = folder.listFiles();
+ if (subfolders.length != 1) {
+ throw new IllegalStateException("Expect only one subfolder - isa");
+ }
+ folder = subfolders[0];
+ for (File f : folder.listFiles()) {
+ if (f.isFile() && isAnonymousVdexBasename(f.getName())) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ private static byte[] createDex(int index) {
+ if (index >= 100) {
+ throw new IllegalArgumentException("Not more than two decimals");
+ }
+
+ // Clone the base dex file. This is the dex file for index 0 (class ID "01").
+ byte[] dex = DEX_BYTES_BASE.clone();
+
+ // Overwrite the checksum and sha1 signature.
+ System.arraycopy(DEX_BYTES_CHECKSUMS[index], 0, dex, DEX_BYTES_CHECKSUM_OFFSET,
+ DEX_BYTES_CHECKSUM_SIZE);
+
+ // Check that the class ID offsets match expectations - they should contains "01".
+ if (dex[DEX_BYTES_CLASS_ID_OFFSET1 + 0] != 0x30 ||
+ dex[DEX_BYTES_CLASS_ID_OFFSET1 + 1] != 0x31 ||
+ dex[DEX_BYTES_CLASS_ID_OFFSET2 + 0] != 0x30 ||
+ dex[DEX_BYTES_CLASS_ID_OFFSET2 + 1] != 0x31) {
+ throw new IllegalStateException("Wrong class name values");
+ }
+
+ // Overwrite class ID.
+ byte str_id1 = (byte) (0x30 + ((index + 1) / 10));
+ byte str_id2 = (byte) (0x30 + ((index + 1) % 10));
+ dex[DEX_BYTES_CLASS_ID_OFFSET1 + 0] = str_id1;
+ dex[DEX_BYTES_CLASS_ID_OFFSET1 + 1] = str_id2;
+ dex[DEX_BYTES_CLASS_ID_OFFSET2 + 0] = str_id1;
+ dex[DEX_BYTES_CLASS_ID_OFFSET2 + 1] = str_id2;
+
+ return dex;
+ }
+
+ private static ClassLoader loadDex(int index) {
+ return new InMemoryDexClassLoader(ByteBuffer.wrap(createDex(index)), /*parent*/ null);
+ }
+
+ private static final String DEX_LOCATION = System.getenv("DEX_LOCATION");
+
+ private static final int DEX_BYTES_CLASS_ID_OFFSET1 = 0xfd;
+ private static final int DEX_BYTES_CLASS_ID_OFFSET2 = 0x11d;
+ private static final int DEX_BYTES_CHECKSUM_OFFSET = 8;
+ private static final int DEX_BYTES_CHECKSUM_SIZE = 24;
+
+ // Dex file for: "public class MyClass01 {}".
+ private static final byte[] DEX_BYTES_BASE = Base64.getDecoder().decode(
+ "ZGV4CjAzNQBHVjDjQ9WQ2TSezZ0exFH00hvlJrenqvNEAgAAcAAAAHhWNBIAAAAAAAAAALABAAAG" +
+ "AAAAcAAAAAMAAACIAAAAAQAAAJQAAAAAAAAAAAAAAAIAAACgAAAAAQAAALAAAAB0AQAA0AAAAOwA" +
+ "AAD0AAAAAQEAABUBAAAlAQAAKAEAAAEAAAACAAAABAAAAAQAAAACAAAAAAAAAAAAAAAAAAAAAQAA" +
+ "AAAAAAAAAAAAAQAAAAEAAAAAAAAAAwAAAAAAAACfAQAAAAAAAAEAAQABAAAA6AAAAAQAAABwEAEA" +
+ "AAAOAAEADgAGPGluaXQ+AAtMTXlDbGFzczAxOwASTGphdmEvbGFuZy9PYmplY3Q7AA5NeUNsYXNz" +
+ "MDEuamF2YQABVgB1fn5EOHsiY29tcGlsYXRpb24tbW9kZSI6ImRlYnVnIiwibWluLWFwaSI6MSwi" +
+ "c2hhLTEiOiI4ZjI5NTlkMDExNmMyYjdmZTZlMDUxNWQ3MTQxZTRmMGY0ZTczYzBiIiwidmVyc2lv" +
+ "biI6IjEuNS41LWRldiJ9AAAAAQAAgYAE0AEAAAAAAAAADAAAAAAAAAABAAAAAAAAAAEAAAAGAAAA" +
+ "cAAAAAIAAAADAAAAiAAAAAMAAAABAAAAlAAAAAUAAAACAAAAoAAAAAYAAAABAAAAsAAAAAEgAAAB" +
+ "AAAA0AAAAAMgAAABAAAA6AAAAAIgAAAGAAAA7AAAAAAgAAABAAAAnwEAAAMQAAABAAAArAEAAAAQ" +
+ "AAABAAAAsAEAAA==");
+
+ // Checksum/SHA1 signature diff for classes MyClass01 - MyClassXX.
+ // This is just a convenient way of storing many similar dex files.
+ private static final byte[][] DEX_BYTES_CHECKSUMS = new byte[][] {
+ Base64.getDecoder().decode("R1Yw40PVkNk0ns2dHsRR9NIb5Sa3p6rz"),
+ Base64.getDecoder().decode("i1V1U3C8nexVk4uw185lXZd9kzd82iaA"),
+ Base64.getDecoder().decode("tFPbVPdpzuoDWqH71Ak5HpltBHg0frMU"),
+ Base64.getDecoder().decode("eFSc7dENiK8nxviKBmd/O2s7h/NAj+l/"),
+ Base64.getDecoder().decode("DlUfNQ3cuVrCHRyw/cOFhqEe+0r6wlUP"),
+ Base64.getDecoder().decode("KVaBmdG8Y8kx8ltEPXWyi9OCdL14yeiW"),
+ Base64.getDecoder().decode("K1bioDTHtPwmrPXkvZ0XYCiripH6KsC2"),
+ Base64.getDecoder().decode("oVHctdpHG3YTNeQlVCshTkFKVra9TG4k"),
+ Base64.getDecoder().decode("eVWMFHRY+w4lpn9Uo9jn+eNAmaRK4HEw"),
+ Base64.getDecoder().decode("/lW3Q3U4ot5A2qkhiv4Aj+s8zv7984MA"),
+ Base64.getDecoder().decode("BFRB+4HwRbuD164DB3sVy28dc+Ea5YVQ"),
+ Base64.getDecoder().decode("klQBLEXyr0cviHDHlqFyWPGKaQQnqMiD"),
+ Base64.getDecoder().decode("jlTcJAkpnbDI/E4msuvMyWqKxNMTN0YU"),
+ Base64.getDecoder().decode("vlUOrp4aN0PxcaqQrQmm597P+Ymu5Adt"),
+ Base64.getDecoder().decode("HlXyT1GoJk1m33O8OMaYxqy3K1Byyf1S"),
+ Base64.getDecoder().decode("d1O5toIKjTXNZkgP3p9RiiafhuKw4gUH"),
+ Base64.getDecoder().decode("11RsuG9UrFHPipOj9zjuGU9obctMJbq6"),
+ Base64.getDecoder().decode("dlSW5egObqheoHSRthlR2c2jVKLGQ3QL"),
+ Base64.getDecoder().decode("ulMgQEhC0XMhmKxHtgdURY6B6JEqNb3E"),
+ Base64.getDecoder().decode("YFV08vrcs49xYr1OBhrza5H8Ha86FODz"),
+ Base64.getDecoder().decode("jFKPxTFd3kn6K0p6n8YEPgm0hiozXW1p"),
+ Base64.getDecoder().decode("LlUZdlCXwAn4qksYL6Urw+bZC/fYuJ1T"),
+ Base64.getDecoder().decode("K1SuRt9xZX5lAVtbpMauOWLVXs2KooUA"),
+ Base64.getDecoder().decode("2FJAWIk0JS9EdvkgHjquLL9qdcLeHaRJ"),
+ Base64.getDecoder().decode("YVResABr9IvZLV8eeIhM3TXfGC+Y6/x1"),
+ Base64.getDecoder().decode("UVTrkVGIh8u7FBHgcbS9flI0CY5g2E3m"),
+ Base64.getDecoder().decode("oVIu6RsrT6HgnbPzNGiYZSpKS0cqNi+a"),
+ Base64.getDecoder().decode("2FR/slWq9YC6kJRDEw21RVGmJhr3/uKZ"),
+ Base64.getDecoder().decode("CFbaSi70ZVaumL7zsXWlD/ernHxCZPx6"),
+ Base64.getDecoder().decode("7FTY+T1/qevWQM6Yoe+OwNcUdgcCUomJ"),
+ };
+}
diff --git a/test/knownfailures.json b/test/knownfailures.json
index 34de414..b76bd5c 100644
--- a/test/knownfailures.json
+++ b/test/knownfailures.json
@@ -560,6 +560,7 @@
"690-hiddenapi-same-name-methods",
"691-hiddenapi-proxy",
"692-vdex-inmem-loader",
+ "693-vdex-inmem-loader-evict",
"944-transform-classloaders",
"999-redefine-hiddenapi"
],
@@ -1096,6 +1097,7 @@
"690-hiddenapi-same-name-methods",
"691-hiddenapi-proxy",
"692-vdex-inmem-loader",
+ "693-vdex-inmem-loader-evict",
"999-redefine-hiddenapi",
"1000-non-moving-space-stress",
"1001-app-image-regions",