diff options
| author | 2024-08-16 16:59:46 -0700 | |
|---|---|---|
| committer | 2024-08-19 09:22:06 -0700 | |
| commit | efbec067ee742ff6e57458b6d6155040a3333aef (patch) | |
| tree | bdb673c4eb591780908fd899643f802de19164fc | |
| parent | c906b847dcc0b487be636033d73d4dfb15824918 (diff) | |
Ravenizer tool skelton
- It still doesn't do any conversion, but the outer structure is done.
- Also make the "load class structure" step a bit faster by loading
classes in parallel.
Flag: EXEMPT host test change only
Bug: 360390999
Test: $ANDROID_BUILD_TOP/frameworks/base/ravenwood/scripts/run-ravenwood-tests.sh
Change-Id: I0f28ccd7388c310f0f733900fea9ad709e16f1cb
15 files changed, 796 insertions, 217 deletions
diff --git a/ravenwood/Android.bp b/ravenwood/Android.bp index 615034338c6b..58cd2e4cee6c 100644 --- a/ravenwood/Android.bp +++ b/ravenwood/Android.bp @@ -166,6 +166,14 @@ java_library { jarjar_rules: ":ravenwood-services-jarjar-rules", } +java_device_for_host { + name: "ravenwood-junit-impl-for-ravenizer", + libs: [ + "ravenwood-junit-impl", + ], + visibility: [":__subpackages__"], +} + // Separated out from ravenwood-junit-impl since it needs to compile // against `module_current` java_library { diff --git a/ravenwood/tools/ravenizer-fake/ravenizer b/ravenwood/tools/ravenizer-fake/ravenizer deleted file mode 100755 index 84b3c8ee365e..000000000000 --- a/ravenwood/tools/ravenizer-fake/ravenizer +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# "Fake" ravenizer, which just copies the file. -# We need it to add ravenizer support to Soong on AOSP, -# when the actual ravenizer is not in AOSP yet. - -invalid_arg() { - echo "Ravenizer(fake): invalid args" 1>&2 - exit 1 -} - -(( $# >= 4 )) || invalid_arg -[[ "$1" == "--in-jar" ]] || invalid_arg -[[ "$3" == "--out-jar" ]] || invalid_arg - -echo "Ravenizer(fake): copiyng $2 to $4" - -cp "$2" "$4" diff --git a/ravenwood/tools/ravenizer-fake/Android.bp b/ravenwood/tools/ravenizer/Android.bp index 7e2c407f2116..2892d0778ec6 100644 --- a/ravenwood/tools/ravenizer-fake/Android.bp +++ b/ravenwood/tools/ravenizer/Android.bp @@ -7,8 +7,19 @@ package { default_applicable_licenses: ["frameworks_base_license"], } -sh_binary_host { +java_binary_host { name: "ravenizer", - src: "ravenizer", + main_class: "com.android.platform.test.ravenwood.ravenizer.RavenizerMain", + srcs: ["src/**/*.kt"], + static_libs: [ + "hoststubgen-lib", + "ow2-asm", + "ow2-asm-analysis", + "ow2-asm-commons", + "ow2-asm-tree", + "ow2-asm-util", + "junit", + "ravenwood-junit-impl-for-ravenizer", + ], visibility: ["//visibility:public"], } diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Ravenizer.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Ravenizer.kt new file mode 100644 index 000000000000..da9c7d97dac5 --- /dev/null +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Ravenizer.kt @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.platform.test.ravenwood.ravenizer + +import com.android.hoststubgen.GeneralUserErrorException +import com.android.hoststubgen.asm.ClassNodes +import com.android.hoststubgen.asm.zipEntryNameToClassName +import com.android.hoststubgen.executableName +import com.android.hoststubgen.log +import com.android.platform.test.ravenwood.ravenizer.adapter.TestRunnerRewritingAdapter +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassVisitor +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.util.CheckClassAdapter +import java.io.BufferedInputStream +import java.io.BufferedOutputStream +import java.io.FileOutputStream +import java.io.InputStream +import java.io.OutputStream +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream + +/** + * Various stats on Ravenizer. + */ +data class RavenizerStats( + /** Total end-to-end time. */ + var totalTime: Double = .0, + + /** Time took to build [ClasNodes] */ + var loadStructureTime: Double = .0, + + /** Total real time spent for converting the jar file */ + var totalProcessTime: Double = .0, + + /** Total real time spent for converting class files (except for I/O time). */ + var totalConversionTime: Double = .0, + + /** Total real time spent for copying class files without modification. */ + var totalCopyTime: Double = .0, + + /** # of entries in the input jar file */ + var totalEntiries: Int = 0, + + /** # of *.class files in the input jar file */ + var totalClasses: Int = 0, + + /** # of *.class files that have been processed. */ + var processedClasses: Int = 0, +) { + override fun toString(): String { + return """ + RavenizerStats{ + totalTime=$totalTime, + loadStructureTime=$loadStructureTime, + totalProcessTime=$totalProcessTime, + totalConversionTime=$totalConversionTime, + totalCopyTime=$totalCopyTime, + totalEntiries=$totalEntiries, + totalClasses=$totalClasses, + processedClasses=$processedClasses, + } + """.trimIndent() + } +} + +/** + * Main class. + */ +class Ravenizer(val options: RavenizerOptions) { + fun run() { + val stats = RavenizerStats() + stats.totalTime = log.nTime { + process(options.inJar.get, options.outJar.get, stats) + } + log.i(stats.toString()) + } + + private fun process(inJar: String, outJar: String, stats: RavenizerStats) { + var allClasses = ClassNodes.loadClassStructures(inJar) { + time -> stats.loadStructureTime = time + } + + stats.totalProcessTime = log.iTime("$executableName processing $inJar") { + ZipFile(inJar).use { inZip -> + val inEntries = inZip.entries() + + stats.totalEntiries = inZip.size() + + ZipOutputStream(BufferedOutputStream(FileOutputStream(outJar))).use { outZip -> + while (inEntries.hasMoreElements()) { + val entry = inEntries.nextElement() + + if (entry.name.endsWith(".dex")) { + // Seems like it's an ART jar file. We can't process it. + // It's a fatal error. + throw GeneralUserErrorException( + "$inJar is not a desktop jar file. It contains a *.dex file." + ) + } + + val className = zipEntryNameToClassName(entry.name) + + if (className != null) { + stats.totalClasses += 1 + } + + if (className != null && shouldProcessClass(allClasses, className)) { + stats.processedClasses += 1 + processSingleClass(inZip, entry, outZip, allClasses, stats) + } else { + // Too slow, let's use merge_zips to bring back the original classes. + copyZipEntry(inZip, entry, outZip, stats) + } + } + } + } + } + } + + /** + * Copy a single ZIP entry to the output. + */ + private fun copyZipEntry( + inZip: ZipFile, + entry: ZipEntry, + out: ZipOutputStream, + stats: RavenizerStats, + ) { + stats.totalCopyTime += log.nTime { + inZip.getInputStream(entry).use { ins -> + // Copy unknown entries as is to the impl out. (but not to the stub out.) + val outEntry = ZipEntry(entry.name) + outEntry.method = 0 + outEntry.size = entry.size + outEntry.crc = entry.crc + out.putNextEntry(outEntry) + + ins.transferTo(out) + + out.closeEntry() + } + } + } + + private fun processSingleClass( + inZip: ZipFile, + entry: ZipEntry, + outZip: ZipOutputStream, + allClasses: ClassNodes, + stats: RavenizerStats, + ) { + val newEntry = ZipEntry(entry.name) + outZip.putNextEntry(newEntry) + + BufferedInputStream(inZip.getInputStream(entry)).use { bis -> + processSingleClass(entry, bis, outZip, allClasses, stats) + } + outZip.closeEntry() + } + + /** + * Whether a class needs to be processed. This must be kept in sync with [processSingleClass]. + */ + private fun shouldProcessClass(classes: ClassNodes, classInternalName: String): Boolean { + return TestRunnerRewritingAdapter.shouldProcess(classes, classInternalName) + } + + private fun processSingleClass( + entry: ZipEntry, + input: InputStream, + output: OutputStream, + allClasses: ClassNodes, + stats: RavenizerStats, + ) { + val cr = ClassReader(input) + + lateinit var data: ByteArray + stats.totalConversionTime += log.vTime("Modify ${entry.name}") { + val flags = ClassWriter.COMPUTE_MAXS + val cw = ClassWriter(flags) + var outVisitor: ClassVisitor = cw + + val enableChecker = false + if (enableChecker) { + outVisitor = CheckClassAdapter(outVisitor) + } + + // This must be kept in sync with shouldProcessClass. + outVisitor = TestRunnerRewritingAdapter(allClasses, outVisitor) + + cr.accept(outVisitor, ClassReader.EXPAND_FRAMES) + + data = cw.toByteArray() + } + output.write(data) + } +} diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerMain.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerMain.kt new file mode 100644 index 000000000000..ff41818cd370 --- /dev/null +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerMain.kt @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:JvmName("RavenizerMain") + +package com.android.platform.test.ravenwood.ravenizer + +import com.android.hoststubgen.LogLevel +import com.android.hoststubgen.executableName +import com.android.hoststubgen.log +import com.android.hoststubgen.runMainWithBoilerplate + +/** + * Entry point. + */ +fun main(args: Array<String>) { + executableName = "Ravenizer" + log.setConsoleLogLevel(LogLevel.Info) + + runMainWithBoilerplate { + val options = RavenizerOptions.parseArgs(args) + + log.i("$executableName started") + log.v("Options: $options") + + // Run. + Ravenizer(options).run() + } +} diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerOptions.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerOptions.kt new file mode 100644 index 000000000000..e85e3be31b77 --- /dev/null +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerOptions.kt @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.platform.test.ravenwood.ravenizer + +import com.android.hoststubgen.ArgIterator +import com.android.hoststubgen.ArgumentsException +import com.android.hoststubgen.SetOnce +import com.android.hoststubgen.ensureFileExists +import com.android.hoststubgen.log + +class RavenizerOptions( + /** Input jar file*/ + var inJar: SetOnce<String> = SetOnce(""), + + /** Output jar file */ + var outJar: SetOnce<String> = SetOnce(""), +) { + companion object { + fun parseArgs(args: Array<String>): RavenizerOptions { + val ret = RavenizerOptions() + val ai = ArgIterator.withAtFiles(args) + + while (true) { + val arg = ai.nextArgOptional() + if (arg == null) { + break + } + + fun nextArg(): String = ai.nextArgRequired(arg) + + if (log.maybeHandleCommandLineArg(arg) { nextArg() }) { + continue + } + try { + when (arg) { + // TODO: Write help + "-h", "--help" -> TODO("Help is not implemented yet") + + "--in-jar" -> ret.inJar.set(nextArg()).ensureFileExists() + "--out-jar" -> ret.outJar.set(nextArg()) + + else -> throw ArgumentsException("Unknown option: $arg") + } + } catch (e: SetOnce.SetMoreThanOnceException) { + throw ArgumentsException("Duplicate or conflicting argument found: $arg") + } + } + + if (!ret.inJar.isSet) { + throw ArgumentsException("Required option missing: --in-jar") + } + if (!ret.outJar.isSet) { + throw ArgumentsException("Required option missing: --out-jar") + } + return ret + } + } + + override fun toString(): String { + return """ + RavenizerOptions{ + inJar=$inJar, + outJar=$outJar, + } + """.trimIndent() + } +} diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Utils.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Utils.kt new file mode 100644 index 000000000000..0018648998dc --- /dev/null +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Utils.kt @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.platform.test.ravenwood.ravenizer + +import com.android.hoststubgen.asm.ClassNodes +import com.android.hoststubgen.asm.findAnyAnnotation +import org.objectweb.asm.Type + +val junitTestMethodType = Type.getType(org.junit.Test::class.java) +val junitRunWithType = Type.getType(org.junit.runner.RunWith::class.java) + +val junitTestMethodDescriptor = junitTestMethodType.descriptor +val junitRunWithDescriptor = junitRunWithType.descriptor + +val junitTestMethodDescriptors = setOf<String>(junitTestMethodDescriptor) +val junitRunWithDescriptors = setOf<String>(junitRunWithDescriptor) + +/** + * Returns true, if a test looks like it's a test class which needs to be processed. + */ +fun isTestLookingClass(classes: ClassNodes, className: String): Boolean { + // Similar to com.android.tradefed.lite.HostUtils.testLoadClass(), except it's more lenient, + // and accept non-public and/or abstract classes. + // HostUtils also checks "Suppress" or "SuiteClasses" but this one doesn't. + // TODO: SuiteClasses may need to be supported. + + val cn = classes.findClass(className) ?: return false + + if (cn.findAnyAnnotation(junitRunWithDescriptors) != null) { + return true + } + cn.methods?.forEach { method -> + if (method.findAnyAnnotation(junitTestMethodDescriptors) != null) { + return true + } + } + if (cn.superName == null) { + return false + } + return isTestLookingClass(classes, cn.superName) +} diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/TestRunnerRewritingAdapter.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/TestRunnerRewritingAdapter.kt new file mode 100644 index 000000000000..c5399084fb33 --- /dev/null +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/TestRunnerRewritingAdapter.kt @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.platform.test.ravenwood.ravenizer.adapter + +import com.android.hoststubgen.asm.ClassNodes +import com.android.hoststubgen.visitors.OPCODE_VERSION +import com.android.platform.test.ravenwood.ravenizer.isTestLookingClass +import org.objectweb.asm.ClassVisitor + +/** + * Class visitor to rewrite the test runner for Ravenwood + * + * TODO: Implement it. + */ +class TestRunnerRewritingAdapter( + protected val classes: ClassNodes, + nextVisitor: ClassVisitor, +) : ClassVisitor(OPCODE_VERSION, nextVisitor) { + companion object { + /** + * Returns true if a target class is interesting to this adapter. + */ + fun shouldProcess(classes: ClassNodes, className: String): Boolean { + return isTestLookingClass(classes, className) + } + } +} diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/Exceptions.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/Exceptions.kt index 910bf59b3d8d..f59e143c1e4e 100644 --- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/Exceptions.kt +++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/Exceptions.kt @@ -15,6 +15,8 @@ */ package com.android.hoststubgen +import java.io.File + /** * We will not print the stack trace for exceptions implementing it. */ @@ -49,4 +51,22 @@ class InvalidAnnotationException(message: String) : Exception(message), UserErro /** * We use this for general "user" errors. */ -class HostStubGenUserErrorException(message: String) : Exception(message), UserErrorException +class GeneralUserErrorException(message: String) : Exception(message), UserErrorException + +/** Base exception class for invalid command line arguments. */ +open class ArgumentsException(message: String?) : Exception(message), UserErrorException + +/** Thrown when the same annotation is used with different annotation arguments. */ +class DuplicateAnnotationException(annotationName: String?) : + ArgumentsException("Duplicate annotation specified: '$annotationName'") + +/** Thrown when an input file does not exist. */ +class InputFileNotFoundException(filename: String) : + ArgumentsException("File '$filename' not found") + +fun String.ensureFileExists(): String { + if (!File(this).exists()) { + throw InputFileNotFoundException(this) + } + return this +} diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenErrors.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenErrors.kt index 6b01d48b52b1..a218c5599553 100644 --- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenErrors.kt +++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenErrors.kt @@ -19,6 +19,6 @@ open class HostStubGenErrors { open fun onErrorFound(message: String) { // TODO: For now, we just throw as soon as any error is found, but eventually we should keep // all errors and print them at the end. - throw HostStubGenUserErrorException(message) + throw GeneralUserErrorException(message) } }
\ No newline at end of file diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenLogger.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenLogger.kt index fcdf8247e7d0..4bcee409aaec 100644 --- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenLogger.kt +++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenLogger.kt @@ -89,6 +89,8 @@ class HostStubGenLogger { addPrinter(StreamPrinter(level, PrintWriter(BufferedOutputStream( FileOutputStream(logFilename))))) + log.i("Log file set: $logFilename for $level") + return this } @@ -122,6 +124,9 @@ class HostStubGenLogger { } fun println(level: LogLevel, message: String) { + if (message.isEmpty()) { + return // Don't print an empty message. + } printers.forEach { if (it.logLevel.ordinal >= level.ordinal) { it.println(level, indent, message) @@ -185,31 +190,45 @@ class HostStubGenLogger { println(LogLevel.Debug, format, *args) } - inline fun <T> logTime(level: LogLevel, message: String, block: () -> T): T { + inline fun <T> logTime(level: LogLevel, message: String, block: () -> T): Double { + var ret: Double = -1.0 val start = System.currentTimeMillis() try { - return block() + block() } finally { val end = System.currentTimeMillis() + ret = (end - start) / 1000.0 if (isEnabled(level)) { println(level, String.format("%s: took %.1f second(s).", message, (end - start) / 1000.0)) } } + return ret } - inline fun <T> iTime(message: String, block: () -> T): T { + /** Do an "i" log with how long it took. */ + inline fun <T> iTime(message: String, block: () -> T): Double { return logTime(LogLevel.Info, message, block) } - inline fun <T> vTime(message: String, block: () -> T): T { + /** Do a "v" log with how long it took. */ + inline fun <T> vTime(message: String, block: () -> T): Double { return logTime(LogLevel.Verbose, message, block) } - inline fun <T> dTime(message: String, block: () -> T): T { + /** Do a "d" log with how long it took. */ + inline fun <T> dTime(message: String, block: () -> T): Double { return logTime(LogLevel.Debug, message, block) } + /** + * Similar to the other "xTime" methods, but the message is not supposed to be printed. + * It's only used to measure the duration with the same interface as other log methods. + */ + inline fun <T> nTime(block: () -> T): Double { + return logTime(LogLevel.Debug, "", block) + } + inline fun forVerbose(block: () -> Unit) { if (isEnabled(LogLevel.Verbose)) { block() @@ -253,6 +272,21 @@ class HostStubGenLogger { } } } + + /** + * Handle log-related command line arguments. + */ + fun maybeHandleCommandLineArg(currentArg: String, nextArgProvider: () -> String): Boolean { + when (currentArg) { + "-v", "--verbose" -> setConsoleLogLevel(LogLevel.Verbose) + "-d", "--debug" -> setConsoleLogLevel(LogLevel.Debug) + "-q", "--quiet" -> setConsoleLogLevel(LogLevel.None) + "--verbose-log" -> addFilePrinter(LogLevel.Verbose, nextArgProvider()) + "--debug-log" -> addFilePrinter(LogLevel.Debug, nextArgProvider()) + else -> return false + } + return true + } } private interface LogPrinter { diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenMain.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenMain.kt index 45e7e301c0d1..85064661cd2b 100644 --- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenMain.kt +++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenMain.kt @@ -24,20 +24,32 @@ import java.io.PrintWriter */ fun main(args: Array<String>) { executableName = "HostStubGen" + runMainWithBoilerplate { + // Parse the command line arguments. + var clanupOnError = false + try { + val options = HostStubGenOptions.parseArgs(args) + clanupOnError = options.cleanUpOnError.get - var success = false - var clanupOnError = false + log.v("$executableName started") + log.v("Options: $options") - try { - // Parse the command line arguments. - val options = HostStubGenOptions.parseArgs(args) - clanupOnError = options.cleanUpOnError.get + // Run. + HostStubGen(options).run() + } catch (e: Throwable) { + if (clanupOnError) { + TODO("Remove output jars here") + } + throw e + } + } +} - log.v("$executableName started") - log.v("Options: $options") +inline fun runMainWithBoilerplate(realMain: () -> Unit) { + var success = false - // Run. - HostStubGen(options).run() + try { + realMain() success = true } catch (e: Throwable) { @@ -45,9 +57,6 @@ fun main(args: Array<String>) { if (e !is UserErrorException) { e.printStackTrace(PrintWriter(log.getWriter(LogLevel.Error))) } - if (clanupOnError) { - TODO("Remove output jars here") - } } finally { log.i("$executableName finished") log.flush() diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt index 2f833a873133..f88b10728dfa 100644 --- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt +++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt @@ -17,20 +17,17 @@ package com.android.hoststubgen import com.android.hoststubgen.filters.FilterPolicy import java.io.BufferedReader -import java.io.File import java.io.FileReader /** * A single value that can only set once. */ -class SetOnce<T>( - private var value: T, -) { +open class SetOnce<T>(private var value: T) { class SetMoreThanOnceException : Exception() private var set = false - fun set(v: T) { + fun set(v: T): T { if (set) { throw SetMoreThanOnceException() } @@ -39,6 +36,7 @@ class SetOnce<T>( } set = true value = v + return v } val get: T @@ -59,6 +57,16 @@ class SetOnce<T>( } } +class IntSetOnce(value: Int) : SetOnce<Int>(value) { + fun set(v: String): Int { + try { + return this.set(v.toInt()) + } catch (e: NumberFormatException) { + throw ArgumentsException("Invalid integer $v") + } + } +} + /** * Options that can be set from command line arguments. */ @@ -113,18 +121,11 @@ class HostStubGenOptions( var apiListFile: SetOnce<String?> = SetOnce(null), - var numShards: SetOnce<Int> = SetOnce(1), - var shard: SetOnce<Int> = SetOnce(0), + var numShards: IntSetOnce = IntSetOnce(1), + var shard: IntSetOnce = IntSetOnce(0), ) { companion object { - private fun String.ensureFileExists(): String { - if (!File(this).exists()) { - throw InputFileNotFoundException(this) - } - return this - } - private fun parsePackageRedirect(fromColonTo: String): Pair<String, String> { val colon = fromColonTo.indexOf(':') if ((colon < 1) || (colon + 1 >= fromColonTo.length)) { @@ -137,7 +138,7 @@ class HostStubGenOptions( fun parseArgs(args: Array<String>): HostStubGenOptions { val ret = HostStubGenOptions() - val ai = ArgIterator(expandAtFiles(args)) + val ai = ArgIterator.withAtFiles(args) var allAnnotations = mutableSetOf<String>() @@ -148,11 +149,6 @@ class HostStubGenOptions( return name } - fun setLogFile(level: LogLevel, filename: String) { - log.addFilePrinter(level, filename) - log.i("$level log file: $filename") - } - while (true) { val arg = ai.nextArgOptional() if (arg == null) { @@ -161,33 +157,23 @@ class HostStubGenOptions( // Define some shorthands... fun nextArg(): String = ai.nextArgRequired(arg) - fun SetOnce<String>.setNextStringArg(): String = nextArg().also { this.set(it) } - fun SetOnce<String?>.setNextStringArg(): String = nextArg().also { this.set(it) } fun MutableSet<String>.addUniqueAnnotationArg(): String = nextArg().also { this += ensureUniqueAnnotation(it) } - fun SetOnce<Int>.setNextIntArg(): String = nextArg().also { - try { - this.set(it.toInt()) - } catch (e: NumberFormatException) { - throw ArgumentsException("Invalid integer for $arg: $it") - } - } + if (log.maybeHandleCommandLineArg(arg) { nextArg() }) { + continue + } try { when (arg) { // TODO: Write help "-h", "--help" -> TODO("Help is not implemented yet") - "-v", "--verbose" -> log.setConsoleLogLevel(LogLevel.Verbose) - "-d", "--debug" -> log.setConsoleLogLevel(LogLevel.Debug) - "-q", "--quiet" -> log.setConsoleLogLevel(LogLevel.None) - - "--in-jar" -> ret.inJar.setNextStringArg().ensureFileExists() - "--out-stub-jar" -> ret.outStubJar.setNextStringArg() - "--out-impl-jar" -> ret.outImplJar.setNextStringArg() + "--in-jar" -> ret.inJar.set(nextArg()).ensureFileExists() + "--out-stub-jar" -> ret.outStubJar.set(nextArg()) + "--out-impl-jar" -> ret.outImplJar.set(nextArg()) "--policy-override-file" -> - ret.policyOverrideFile.setNextStringArg().ensureFileExists() + ret.policyOverrideFile.set(nextArg())!!.ensureFileExists() "--clean-up-on-error" -> ret.cleanUpOnError.set(true) "--no-clean-up-on-error" -> ret.cleanUpOnError.set(false) @@ -231,19 +217,19 @@ class HostStubGenOptions( ret.packageRedirects += parsePackageRedirect(nextArg()) "--annotation-allowed-classes-file" -> - ret.annotationAllowedClassesFile.setNextStringArg() + ret.annotationAllowedClassesFile.set(nextArg()) "--default-class-load-hook" -> - ret.defaultClassLoadHook.setNextStringArg() + ret.defaultClassLoadHook.set(nextArg()) "--default-method-call-hook" -> - ret.defaultMethodCallHook.setNextStringArg() + ret.defaultMethodCallHook.set(nextArg()) "--intersect-stub-jar" -> ret.intersectStubJars += nextArg().ensureFileExists() "--gen-keep-all-file" -> - ret.inputJarAsKeepAllFile.setNextStringArg() + ret.inputJarAsKeepAllFile.set(nextArg()) // Following options are for debugging. "--enable-class-checker" -> ret.enableClassChecker.set(true) @@ -261,16 +247,21 @@ class HostStubGenOptions( "--no-non-stub-method-check" -> ret.enableNonStubMethodCallDetection.set(false) - "--gen-input-dump-file" -> ret.inputJarDumpFile.setNextStringArg() - - "--verbose-log" -> setLogFile(LogLevel.Verbose, nextArg()) - "--debug-log" -> setLogFile(LogLevel.Debug, nextArg()) + "--gen-input-dump-file" -> ret.inputJarDumpFile.set(nextArg()) - "--stats-file" -> ret.statsFile.setNextStringArg() - "--supported-api-list-file" -> ret.apiListFile.setNextStringArg() + "--stats-file" -> ret.statsFile.set(nextArg()) + "--supported-api-list-file" -> ret.apiListFile.set(nextArg()) - "--num-shards" -> ret.numShards.setNextIntArg() - "--shard-index" -> ret.shard.setNextIntArg() + "--num-shards" -> ret.numShards.set(nextArg()).also { + if (it < 1) { + throw ArgumentsException("$arg must be positive integer") + } + } + "--shard-index" -> ret.shard.set(nextArg()).also { + if (it < 0) { + throw ArgumentsException("$arg must be positive integer or zero") + } + } else -> throw ArgumentsException("Unknown option: $arg") } @@ -286,94 +277,22 @@ class HostStubGenOptions( log.w("Neither --out-stub-jar nor --out-impl-jar is set." + " $executableName will not generate jar files.") } - - if (ret.enableNonStubMethodCallDetection.get) { - log.w("--enable-non-stub-method-check is not fully implemented yet." + - " See the todo in doesMethodNeedNonStubCallCheck().") + if (ret.numShards.isSet != ret.shard.isSet) { + throw ArgumentsException("--num-shards and --shard-index must be used together") } - return ret - } - - /** - * Scan the arguments, and if any of them starts with an `@`, then load from the file - * and use its content as arguments. - * - * In this file, each line is treated as a single argument. - * - * The file can contain '#' as comments. - */ - private fun expandAtFiles(args: Array<String>): List<String> { - val ret = mutableListOf<String>() - - args.forEach { arg -> - if (!arg.startsWith('@')) { - ret += arg - return@forEach - } - // Read from the file, and add each line to the result. - val filename = arg.substring(1).ensureFileExists() - - log.v("Expanding options file $filename") - - BufferedReader(FileReader(filename)).use { reader -> - while (true) { - var line = reader.readLine() - if (line == null) { - break // EOF - } - - line = normalizeTextLine(line) - if (line.isNotEmpty()) { - ret += line - } - } + if (ret.numShards.isSet) { + if (ret.shard.get >= ret.numShards.get) { + throw ArgumentsException("--shard-index must be smaller than --num-shards") } } - return ret - } - } - open class ArgumentsException(message: String?) : Exception(message), UserErrorException - - /** Thrown when the same annotation is used with different annotation arguments. */ - class DuplicateAnnotationException(annotationName: String?) : - ArgumentsException("Duplicate annotation specified: '$annotationName'") - - /** Thrown when an input file does not exist. */ - class InputFileNotFoundException(filename: String) : - ArgumentsException("File '$filename' not found") - - private class ArgIterator( - private val args: List<String>, - private var currentIndex: Int = -1 - ) { - val current: String - get() = args.get(currentIndex) - - /** - * Get the next argument, or [null] if there's no more arguments. - */ - fun nextArgOptional(): String? { - if ((currentIndex + 1) >= args.size) { - return null + if (ret.enableNonStubMethodCallDetection.get) { + log.w("--enable-non-stub-method-check is not fully implemented yet." + + " See the todo in doesMethodNeedNonStubCallCheck().") } - return args.get(++currentIndex) - } - /** - * Get the next argument, or throw if - */ - fun nextArgRequired(argName: String): String { - nextArgOptional().let { - if (it == null) { - throw ArgumentsException("Missing parameter for option $argName") - } - if (it.isEmpty()) { - throw ArgumentsException("Parameter can't be empty for option $argName") - } - return it - } + return ret } } @@ -415,3 +334,80 @@ class HostStubGenOptions( """.trimIndent() } } + +class ArgIterator( + private val args: List<String>, + private var currentIndex: Int = -1 +) { + val current: String + get() = args.get(currentIndex) + + /** + * Get the next argument, or [null] if there's no more arguments. + */ + fun nextArgOptional(): String? { + if ((currentIndex + 1) >= args.size) { + return null + } + return args.get(++currentIndex) + } + + /** + * Get the next argument, or throw if + */ + fun nextArgRequired(argName: String): String { + nextArgOptional().let { + if (it == null) { + throw ArgumentsException("Missing parameter for option $argName") + } + if (it.isEmpty()) { + throw ArgumentsException("Parameter can't be empty for option $argName") + } + return it + } + } + + companion object { + fun withAtFiles(args: Array<String>): ArgIterator { + return ArgIterator(expandAtFiles(args)) + } + } +} + +/** + * Scan the arguments, and if any of them starts with an `@`, then load from the file + * and use its content as arguments. + * + * In this file, each line is treated as a single argument. + * + * The file can contain '#' as comments. + */ +private fun expandAtFiles(args: Array<String>): List<String> { + val ret = mutableListOf<String>() + + args.forEach { arg -> + if (!arg.startsWith('@')) { + ret += arg + return@forEach + } + // Read from the file, and add each line to the result. + val filename = arg.substring(1).ensureFileExists() + + log.v("Expanding options file $filename") + + BufferedReader(FileReader(filename)).use { reader -> + while (true) { + var line = reader.readLine() + if (line == null) { + break // EOF + } + + line = normalizeTextLine(line) + if (line.isNotEmpty()) { + ret += line + } + } + } + } + return ret +} diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/AsmUtils.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/AsmUtils.kt index f219dac7f0a5..6cf214300b43 100644 --- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/AsmUtils.kt +++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/AsmUtils.kt @@ -58,7 +58,24 @@ fun findAnyAnnotation( return null } -fun findAnnotationValueAsString(an: AnnotationNode, propertyName: String): String? { +fun ClassNode.findAnyAnnotation(set: Set<String>): AnnotationNode? { + return findAnyAnnotation(set, this.visibleAnnotations, this.invisibleAnnotations) +} + +fun MethodNode.findAnyAnnotation(set: Set<String>): AnnotationNode? { + return findAnyAnnotation(set, this.visibleAnnotations, this.invisibleAnnotations) +} + +fun FieldNode.findAnyAnnotation(set: Set<String>): AnnotationNode? { + return findAnyAnnotation(set, this.visibleAnnotations, this.invisibleAnnotations) +} + +fun <T> findAnnotationValueAsObject( + an: AnnotationNode, + propertyName: String, + expectedTypeHumanReadableName: String, + converter: (Any?) -> T?, +): T? { for (i in 0..(an.values?.size ?: 0) - 2 step 2) { val name = an.values[i] @@ -66,16 +83,30 @@ fun findAnnotationValueAsString(an: AnnotationNode, propertyName: String): Strin continue } val value = an.values[i + 1] - if (value is String) { - return value + if (value == null) { + return null + } + + try { + return converter(value) + } catch (e: ClassCastException) { + throw ClassParseException( + "The type of '$propertyName' in annotation @${an.desc} must be " + + "$expectedTypeHumanReadableName, but is ${value?.javaClass?.canonicalName}") } - throw ClassParseException( - "The type of '$name' in annotation \"${an.desc}\" must be String" + - ", but is ${value?.javaClass?.canonicalName}") } return null } +fun findAnnotationValueAsString(an: AnnotationNode, propertyName: String): String? { + return findAnnotationValueAsObject(an, propertyName, "String", {it as String}) +} + +fun findAnnotationValueAsType(an: AnnotationNode, propertyName: String): Type? { + return findAnnotationValueAsObject(an, propertyName, "Class", {it as Type}) +} + + val periodOrSlash = charArrayOf('.', '/') fun getPackageNameFromFullClassName(fullClassName: String): String { @@ -125,6 +156,24 @@ fun splitWithLastPeriod(name: String): Pair<String, String>? { return Pair(name.substring(0, pos), name.substring(pos + 1)) } +fun String.startsWithAny(vararg prefixes: String): Boolean { + prefixes.forEach { + if (this.startsWith(it)) { + return true + } + } + return false +} + +fun String.endsWithAny(vararg suffixes: String): Boolean { + suffixes.forEach { + if (this.endsWith(it)) { + return true + } + } + return false +} + fun String.toJvmClassName(): String { return this.replace('.', '/') } @@ -137,6 +186,14 @@ fun String.toHumanReadableMethodName(): String { return this.replace('/', '.') } +fun zipEntryNameToClassName(entryFilename: String): String? { + val suffix = ".class" + if (!entryFilename.endsWith(suffix)) { + return null + } + return entryFilename.substring(0, entryFilename.length - suffix.length) +} + private val numericalInnerClassName = """.*\$\d+$""".toRegex() fun isAnonymousInnerClass(cn: ClassNode): Boolean { @@ -278,6 +335,14 @@ fun MethodNode.isStatic(): Boolean { return (this.access and Opcodes.ACC_STATIC) != 0 } +fun MethodNode.isPublic(): Boolean { + return (this.access and Opcodes.ACC_PUBLIC) != 0 +} + +fun MethodNode.isSpecial(): Boolean { + return CTOR_NAME == this.name || CLASS_INITIALIZER_NAME == this.name +} + fun FieldNode.isEnum(): Boolean { return (this.access and Opcodes.ACC_ENUM) != 0 } diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/ClassNodes.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/ClassNodes.kt index 2607df63f146..e2647eb13ed3 100644 --- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/ClassNodes.kt +++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/ClassNodes.kt @@ -27,6 +27,11 @@ import org.objectweb.asm.tree.TypeAnnotationNode import java.io.BufferedInputStream import java.io.PrintWriter import java.util.Arrays +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference +import java.util.function.Consumer +import java.util.zip.ZipEntry import java.util.zip.ZipFile /** @@ -183,10 +188,43 @@ class ClassNodes { /** * Load all the classes, without code. */ - fun loadClassStructures(inJar: String): ClassNodes { - log.iTime("Reading class structure from $inJar") { - val allClasses = ClassNodes() + fun loadClassStructures( + inJar: String, + timeCollector: Consumer<Double>? = null, + ): ClassNodes { + val allClasses = ClassNodes() + // Load classes in parallel. + val executor = Executors.newFixedThreadPool(4) + + // First exception defected. + val exception = AtomicReference<Throwable>() + + // Called on a BG thread. Read a single jar entry and add it to [allClasses]. + fun parseClass(inZip: ZipFile, entry: ZipEntry) { + try { + inZip.getInputStream(entry).use { ins -> + val cr = ClassReader(BufferedInputStream(ins)) + val cn = ClassNode() + cr.accept( + cn, ClassReader.SKIP_CODE + or ClassReader.SKIP_DEBUG + or ClassReader.SKIP_FRAMES + ) + synchronized(allClasses) { + if (!allClasses.addClass(cn)) { + log.w("Duplicate class found: ${cn.name}") + } + } + } + } catch (e: Throwable) { + log.e("Failed to load class: $e") + exception.compareAndSet(null, e) + } + } + + // Actually open the jar and read it on worker threads. + val time = log.iTime("Reading class structure from $inJar") { log.withIndent { ZipFile(inJar).use { inZip -> val inEntries = inZip.entries() @@ -194,40 +232,42 @@ class ClassNodes { while (inEntries.hasMoreElements()) { val entry = inEntries.nextElement() - BufferedInputStream(inZip.getInputStream(entry)).use { bis -> - if (entry.name.endsWith(".class")) { - val cr = ClassReader(bis) - val cn = ClassNode() - cr.accept( - cn, ClassReader.SKIP_CODE - or ClassReader.SKIP_DEBUG - or ClassReader.SKIP_FRAMES - ) - if (!allClasses.addClass(cn)) { - log.w("Duplicate class found: ${cn.name}") - } - } else if (entry.name.endsWith(".dex")) { - // Seems like it's an ART jar file. We can't process it. - // It's a fatal error. - throw InvalidJarFileException( - "$inJar is not a desktop jar file." - + " It contains a *.dex file." - ) - } else { - // Unknown file type. Skip. - while (bis.available() > 0) { - bis.skip((1024 * 1024).toLong()) - } + if (entry.name.endsWith(".class")) { + executor.submit { + parseClass(inZip, entry) } + } else if (entry.name.endsWith(".dex")) { + // Seems like it's an ART jar file. We can't process it. + // It's a fatal error. + throw InvalidJarFileException( + "$inJar is not a desktop jar file." + + " It contains a *.dex file." + ) + } else { + // Unknown file type. Skip. } } + // Wait for all the work to complete. (must do it before closing the zip) + log.i("Waiting for all loaders to finish...") + executor.shutdown() + executor.awaitTermination(5, TimeUnit.MINUTES) + log.i("All loaders to finished.") } } + + // If any exception is detected, throw it. + exception.get()?.let { + throw it + } + if (allClasses.size == 0) { log.w("$inJar contains no *.class files.") + } else { + log.i("Loaded ${allClasses.size} classes from $inJar.") } - return allClasses } + timeCollector?.accept(time) + return allClasses } } }
\ No newline at end of file |