diff options
author | 2025-03-06 23:49:15 +0000 | |
---|---|---|
committer | 2025-03-06 23:58:17 +0000 | |
commit | 24b00ea7b8fecf17deb290d5e7987c34d48c3104 (patch) | |
tree | e263b85be0fdfa7a56d4b04d6e321b7a93740365 | |
parent | 9e244748a5185dd8cbae2fd6c383501b2beccef4 (diff) |
[Ravenizer] Use HostStubGen to process test and target code
Bug: 397498134
Flag: EXEMPT host side change only
Test: f/b/r/scripts/run-ravenwood-tests.sh
Change-Id: Ib298d31adccb1e5ba2b65f4329971adad69926d9
23 files changed, 337 insertions, 179 deletions
diff --git a/ravenwood/Android.bp b/ravenwood/Android.bp index ccbc46fdb03b..5424ac3bd897 100644 --- a/ravenwood/Android.bp +++ b/ravenwood/Android.bp @@ -16,7 +16,7 @@ filegroup { srcs: [ "texts/ravenwood-common-policies.txt", ], - visibility: ["//visibility:private"], + visibility: [":__subpackages__"], } filegroup { @@ -44,6 +44,22 @@ filegroup { } filegroup { + name: "ravenwood-standard-annotations", + srcs: [ + "texts/ravenwood-standard-annotations.txt", + ], + visibility: [":__subpackages__"], +} + +filegroup { + name: "ravenizer-standard-options", + srcs: [ + "texts/ravenizer-standard-options.txt", + ], + visibility: [":__subpackages__"], +} + +filegroup { name: "ravenwood-annotation-allowed-classes", srcs: [ "texts/ravenwood-annotation-allowed-classes.txt", diff --git a/ravenwood/Framework.bp b/ravenwood/Framework.bp index e36677189e02..f5b075b17fd4 100644 --- a/ravenwood/Framework.bp +++ b/ravenwood/Framework.bp @@ -33,6 +33,7 @@ genrule_defaults { ":ravenwood-common-policies", ":ravenwood-framework-policies", ":ravenwood-standard-options", + ":ravenwood-standard-annotations", ":ravenwood-annotation-allowed-classes", ], out: [ @@ -44,6 +45,7 @@ genrule_defaults { framework_minus_apex_cmd = "$(location hoststubgen) " + "@$(location :ravenwood-standard-options) " + + "@$(location :ravenwood-standard-annotations) " + "--debug-log $(location hoststubgen_framework-minus-apex.log) " + "--out-jar $(location ravenwood.jar) " + "--in-jar $(location :framework-minus-apex-for-host) " + @@ -178,6 +180,7 @@ java_genrule { tools: ["hoststubgen"], cmd: "$(location hoststubgen) " + "@$(location :ravenwood-standard-options) " + + "@$(location :ravenwood-standard-annotations) " + "--debug-log $(location hoststubgen_services.core.log) " + "--stats-file $(location hoststubgen_services.core_stats.csv) " + @@ -196,6 +199,7 @@ java_genrule { ":ravenwood-common-policies", ":ravenwood-services-policies", ":ravenwood-standard-options", + ":ravenwood-standard-annotations", ":ravenwood-annotation-allowed-classes", ], out: [ @@ -247,6 +251,7 @@ java_genrule { tools: ["hoststubgen"], cmd: "$(location hoststubgen) " + "@$(location :ravenwood-standard-options) " + + "@$(location :ravenwood-standard-annotations) " + "--debug-log $(location hoststubgen_core-icu4j-for-host.log) " + "--stats-file $(location hoststubgen_core-icu4j-for-host_stats.csv) " + @@ -265,6 +270,7 @@ java_genrule { ":ravenwood-common-policies", ":icu-ravenwood-policies", ":ravenwood-standard-options", + ":ravenwood-standard-annotations", ], out: [ "ravenwood.jar", @@ -301,6 +307,7 @@ java_genrule { tools: ["hoststubgen"], cmd: "$(location hoststubgen) " + "@$(location :ravenwood-standard-options) " + + "@$(location :ravenwood-standard-annotations) " + "--debug-log $(location framework-configinfrastructure.log) " + "--stats-file $(location framework-configinfrastructure_stats.csv) " + @@ -319,6 +326,7 @@ java_genrule { ":ravenwood-common-policies", ":framework-configinfrastructure-ravenwood-policies", ":ravenwood-standard-options", + ":ravenwood-standard-annotations", ], out: [ "ravenwood.jar", @@ -355,6 +363,7 @@ java_genrule { tools: ["hoststubgen"], cmd: "$(location hoststubgen) " + "@$(location :ravenwood-standard-options) " + + "@$(location :ravenwood-standard-annotations) " + "--debug-log $(location framework-statsd.log) " + "--stats-file $(location framework-statsd_stats.csv) " + @@ -373,6 +382,7 @@ java_genrule { ":ravenwood-common-policies", ":framework-statsd-ravenwood-policies", ":ravenwood-standard-options", + ":ravenwood-standard-annotations", ], out: [ "ravenwood.jar", @@ -409,6 +419,7 @@ java_genrule { tools: ["hoststubgen"], cmd: "$(location hoststubgen) " + "@$(location :ravenwood-standard-options) " + + "@$(location :ravenwood-standard-annotations) " + "--debug-log $(location framework-graphics.log) " + "--stats-file $(location framework-graphics_stats.csv) " + @@ -427,6 +438,7 @@ java_genrule { ":ravenwood-common-policies", ":framework-graphics-ravenwood-policies", ":ravenwood-standard-options", + ":ravenwood-standard-annotations", ], out: [ "ravenwood.jar", diff --git a/ravenwood/tests/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodJdkPatchTest.java b/ravenwood/tests/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodJdkPatchTest.java new file mode 100644 index 000000000000..cdfd4a877f43 --- /dev/null +++ b/ravenwood/tests/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodJdkPatchTest.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ravenwoodtest.bivalenttest.ravenizer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import android.system.ErrnoException; +import android.system.Os; +import android.system.OsConstants; + +import org.junit.Test; + +import java.io.FileDescriptor; +import java.util.LinkedHashMap; +import java.util.regex.Pattern; + +public class RavenwoodJdkPatchTest { + + @Test + public void testUnicodeRegex() { + var pattern = Pattern.compile("\\w+"); + assertTrue(pattern.matcher("über").matches()); + } + + @Test + public void testLinkedHashMapEldest() { + var map = new LinkedHashMap<String, String>(); + map.put("a", "b"); + map.put("x", "y"); + assertEquals(map.entrySet().iterator().next(), map.eldest()); + } + + @Test + public void testFileDescriptorGetSetInt() throws ErrnoException { + FileDescriptor fd = Os.open("/dev/zero", OsConstants.O_RDONLY, 0); + try { + int fdRaw = fd.getInt$(); + assertNotEquals(-1, fdRaw); + fd.setInt$(-1); + assertEquals(-1, fd.getInt$()); + fd.setInt$(fdRaw); + Os.close(fd); + assertEquals(-1, fd.getInt$()); + } finally { + Os.close(fd); + } + } +} diff --git a/ravenwood/texts/ravenizer-standard-options.txt b/ravenwood/texts/ravenizer-standard-options.txt new file mode 100644 index 000000000000..cef736f87e72 --- /dev/null +++ b/ravenwood/texts/ravenizer-standard-options.txt @@ -0,0 +1,13 @@ +# File containing standard options to Ravenizer for Ravenwood + +# Keep all classes / methods / fields in tests and its target +--default-keep + +--delete-finals + +# Include standard annotations +@jar:texts/ravenwood-standard-annotations.txt + +# Apply common policies +--policy-override-file + jar:texts/ravenwood-common-policies.txt diff --git a/ravenwood/texts/ravenwood-standard-annotations.txt b/ravenwood/texts/ravenwood-standard-annotations.txt new file mode 100644 index 000000000000..75ec5cadb6fc --- /dev/null +++ b/ravenwood/texts/ravenwood-standard-annotations.txt @@ -0,0 +1,37 @@ +# Standard annotations. +# Note, each line is a single argument, so we need newlines after each `--xxx-annotation`. +--keep-annotation + android.ravenwood.annotation.RavenwoodKeep + +--keep-annotation + android.ravenwood.annotation.RavenwoodKeepPartialClass + +--keep-class-annotation + android.ravenwood.annotation.RavenwoodKeepWholeClass + +--throw-annotation + android.ravenwood.annotation.RavenwoodThrow + +--remove-annotation + android.ravenwood.annotation.RavenwoodRemove + +--ignore-annotation + android.ravenwood.annotation.RavenwoodIgnore + +--partially-allowed-annotation + android.ravenwood.annotation.RavenwoodPartiallyAllowlisted + +--substitute-annotation + android.ravenwood.annotation.RavenwoodReplace + +--redirect-annotation + android.ravenwood.annotation.RavenwoodRedirect + +--redirection-class-annotation + android.ravenwood.annotation.RavenwoodRedirectionClass + +--class-load-hook-annotation + android.ravenwood.annotation.RavenwoodClassLoadHook + +--keep-static-initializer-annotation + android.ravenwood.annotation.RavenwoodKeepStaticInitializer diff --git a/ravenwood/texts/ravenwood-standard-options.txt b/ravenwood/texts/ravenwood-standard-options.txt index 233657557747..0a650254a71f 100644 --- a/ravenwood/texts/ravenwood-standard-options.txt +++ b/ravenwood/texts/ravenwood-standard-options.txt @@ -15,41 +15,3 @@ #--default-class-load-hook # com.android.hoststubgen.hosthelper.HostTestUtils.logClassLoaded - -# Standard annotations. -# Note, each line is a single argument, so we need newlines after each `--xxx-annotation`. ---keep-annotation - android.ravenwood.annotation.RavenwoodKeep - ---keep-annotation - android.ravenwood.annotation.RavenwoodKeepPartialClass - ---keep-class-annotation - android.ravenwood.annotation.RavenwoodKeepWholeClass - ---throw-annotation - android.ravenwood.annotation.RavenwoodThrow - ---remove-annotation - android.ravenwood.annotation.RavenwoodRemove - ---ignore-annotation - android.ravenwood.annotation.RavenwoodIgnore - ---partially-allowed-annotation - android.ravenwood.annotation.RavenwoodPartiallyAllowlisted - ---substitute-annotation - android.ravenwood.annotation.RavenwoodReplace - ---redirect-annotation - android.ravenwood.annotation.RavenwoodRedirect - ---redirection-class-annotation - android.ravenwood.annotation.RavenwoodRedirectionClass - ---class-load-hook-annotation - android.ravenwood.annotation.RavenwoodClassLoadHook - ---keep-static-initializer-annotation - android.ravenwood.annotation.RavenwoodKeepStaticInitializer diff --git a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/Exceptions.kt b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/Exceptions.kt index f59e143c1e4e..ae0a00855650 100644 --- a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/Exceptions.kt +++ b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/Exceptions.kt @@ -15,8 +15,6 @@ */ package com.android.hoststubgen -import java.io.File - /** * We will not print the stack trace for exceptions implementing it. */ @@ -64,9 +62,6 @@ class DuplicateAnnotationException(annotationName: String?) : class InputFileNotFoundException(filename: String) : ArgumentsException("File '$filename' not found") -fun String.ensureFileExists(): String { - if (!File(this).exists()) { - throw InputFileNotFoundException(this) - } - return this -} +/** Thrown when a JAR resource does not exist. */ +class JarResourceNotFoundException(path: String) : + ArgumentsException("JAR resource '$path' not found") diff --git a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenClassProcessor.kt b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenClassProcessor.kt index 4fe21eac6972..530429865040 100644 --- a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenClassProcessor.kt +++ b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenClassProcessor.kt @@ -16,6 +16,7 @@ package com.android.hoststubgen import com.android.hoststubgen.asm.ClassNodes +import com.android.hoststubgen.asm.findAnyAnnotation import com.android.hoststubgen.filters.AnnotationBasedFilter import com.android.hoststubgen.filters.ClassWidePolicyPropagatingFilter import com.android.hoststubgen.filters.ConstantFilter @@ -26,6 +27,7 @@ import com.android.hoststubgen.filters.KeepNativeFilter import com.android.hoststubgen.filters.OutputFilter import com.android.hoststubgen.filters.SanitizationFilter import com.android.hoststubgen.filters.TextFileFilterPolicyBuilder +import com.android.hoststubgen.hosthelper.HostStubGenProcessedAsKeep import com.android.hoststubgen.utils.ClassPredicate import com.android.hoststubgen.visitors.BaseAdapter import com.android.hoststubgen.visitors.PackageRedirectRemapper @@ -40,7 +42,7 @@ import org.objectweb.asm.util.CheckClassAdapter */ class HostStubGenClassProcessor( private val options: HostStubGenClassProcessorOptions, - private val allClasses: ClassNodes, + val allClasses: ClassNodes, private val errors: HostStubGenErrors = HostStubGenErrors(), private val stats: HostStubGenStats? = null, ) { @@ -48,6 +50,7 @@ class HostStubGenClassProcessor( val remapper = FilterRemapper(filter) private val packageRedirector = PackageRedirectRemapper(options.packageRedirects) + private val processedAnnotation = setOf(HostStubGenProcessedAsKeep.CLASS_DESCRIPTOR) /** * Build the filter, which decides what classes/methods/fields should be put in stub or impl @@ -133,6 +136,12 @@ class HostStubGenClassProcessor( fun processClassBytecode(bytecode: ByteArray): ByteArray { val cr = ClassReader(bytecode) + // If the class was already processed previously, skip + val clz = allClasses.getClass(cr.className) + if (clz.findAnyAnnotation(processedAnnotation) != null) { + return bytecode + } + // COMPUTE_FRAMES wouldn't be happy if code uses val flags = ClassWriter.COMPUTE_MAXS // or ClassWriter.COMPUTE_FRAMES val cw = ClassWriter(flags) diff --git a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenClassProcessorOptions.kt b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenClassProcessorOptions.kt index c7c45e65a8b6..e7166f11f597 100644 --- a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenClassProcessorOptions.kt +++ b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenClassProcessorOptions.kt @@ -18,6 +18,7 @@ package com.android.hoststubgen import com.android.hoststubgen.filters.FilterPolicy import com.android.hoststubgen.utils.ArgIterator import com.android.hoststubgen.utils.BaseOptions +import com.android.hoststubgen.utils.FileOrResource import com.android.hoststubgen.utils.SetOnce private fun parsePackageRedirect(fromColonTo: String): Pair<String, String> { @@ -53,7 +54,7 @@ open class HostStubGenClassProcessorOptions( var defaultClassLoadHook: SetOnce<String?> = SetOnce(null), var defaultMethodCallHook: SetOnce<String?> = SetOnce(null), - var policyOverrideFiles: MutableList<String> = mutableListOf(), + var policyOverrideFiles: MutableList<FileOrResource> = mutableListOf(), var defaultPolicy: SetOnce<FilterPolicy> = SetOnce(FilterPolicy.Remove), @@ -73,15 +74,14 @@ open class HostStubGenClassProcessorOptions( return name } - override fun parseOption(option: String, ai: ArgIterator): Boolean { + override fun parseOption(option: String, args: ArgIterator): Boolean { // Define some shorthands... - fun nextArg(): String = ai.nextArgRequired(option) + fun nextArg(): String = args.nextArgRequired(option) fun MutableSet<String>.addUniqueAnnotationArg(): String = nextArg().also { this += ensureUniqueAnnotation(it) } when (option) { - "--policy-override-file" -> - policyOverrideFiles.add(nextArg().ensureFileExists()) + "--policy-override-file" -> policyOverrideFiles.add(FileOrResource(nextArg())) "--default-remove" -> defaultPolicy.set(FilterPolicy.Remove) "--default-throw" -> defaultPolicy.set(FilterPolicy.Throw) diff --git a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/asm/AsmUtils.kt b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/asm/AsmUtils.kt index b41ce0f65017..112ef01e20cb 100644 --- a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/asm/AsmUtils.kt +++ b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/asm/AsmUtils.kt @@ -217,7 +217,7 @@ private val numericalInnerClassName = """.*\$\d+$""".toRegex() fun isAnonymousInnerClass(cn: ClassNode): Boolean { // TODO: Is there a better way? - return cn.name.matches(numericalInnerClassName) + return cn.outerClass != null && cn.name.matches(numericalInnerClassName) } /** diff --git a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/filters/ImplicitOutputFilter.kt b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/filters/ImplicitOutputFilter.kt index 474da6dfa1b9..d44d016f7c5b 100644 --- a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/filters/ImplicitOutputFilter.kt +++ b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/filters/ImplicitOutputFilter.kt @@ -16,7 +16,6 @@ package com.android.hoststubgen.filters import com.android.hoststubgen.HostStubGenErrors -import com.android.hoststubgen.HostStubGenInternalException import com.android.hoststubgen.asm.CLASS_INITIALIZER_DESC import com.android.hoststubgen.asm.CLASS_INITIALIZER_NAME import com.android.hoststubgen.asm.ClassNodes @@ -37,19 +36,15 @@ import org.objectweb.asm.tree.ClassNode * TODO: Do we need a way to make anonymous class methods and lambdas "throw"? */ class ImplicitOutputFilter( - private val errors: HostStubGenErrors, - private val classes: ClassNodes, - fallback: OutputFilter + private val errors: HostStubGenErrors, + private val classes: ClassNodes, + fallback: OutputFilter ) : DelegatingFilter(fallback) { - private fun getClassImplicitPolicy(className: String, cn: ClassNode): FilterPolicyWithReason? { + private fun getClassImplicitPolicy(cn: ClassNode): FilterPolicyWithReason? { if (isAnonymousInnerClass(cn)) { log.forDebug { // log.d(" anon-inner class: ${className} outer: ${cn.outerClass} ") } - if (cn.outerClass == null) { - throw HostStubGenInternalException( - "outerClass is null for anonymous inner class") - } // If the outer class needs to be in impl, it should be in impl too. val outerPolicy = outermostFilter.getPolicyForClass(cn.outerClass) if (outerPolicy.policy.needsInOutput) { @@ -65,15 +60,15 @@ class ImplicitOutputFilter( val cn = classes.getClass(className) // Use the implicit policy, if any. - getClassImplicitPolicy(className, cn)?.let { return it } + getClassImplicitPolicy(cn)?.let { return it } return fallback } override fun getPolicyForMethod( - className: String, - methodName: String, - descriptor: String + className: String, + methodName: String, + descriptor: String ): FilterPolicyWithReason { val fallback = super.getPolicyForMethod(className, methodName, descriptor) val classPolicy = outermostFilter.getPolicyForClass(className) @@ -84,12 +79,14 @@ class ImplicitOutputFilter( // "keep" instead. // Unless it's an enum -- in that case, the below code would handle it. if (!cn.isEnum() && - fallback.policy == FilterPolicy.Throw && - methodName == CLASS_INITIALIZER_NAME && descriptor == CLASS_INITIALIZER_DESC) { + fallback.policy == FilterPolicy.Throw && + methodName == CLASS_INITIALIZER_NAME && descriptor == CLASS_INITIALIZER_DESC + ) { // TODO Maybe show a warning?? But that'd be too noisy with --default-throw. return FilterPolicy.Ignore.withReason( "'throw' on static initializer is handled as 'ignore'" + - " [original throw reason: ${fallback.reason}]") + " [original throw reason: ${fallback.reason}]" + ) } log.d("Class ${cn.name} Class policy: $classPolicy") @@ -120,7 +117,8 @@ class ImplicitOutputFilter( // For synthetic methods (such as lambdas), let's just inherit the class's // policy. return memberPolicy.withReason(classPolicy.reason).wrapReason( - "is-synthetic-method") + "is-synthetic-method" + ) } } } @@ -129,8 +127,8 @@ class ImplicitOutputFilter( } override fun getPolicyForField( - className: String, - fieldName: String + className: String, + fieldName: String ): FilterPolicyWithReason { val fallback = super.getPolicyForField(className, fieldName) @@ -161,4 +159,4 @@ class ImplicitOutputFilter( return fallback } -}
\ No newline at end of file +} diff --git a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/filters/TextFileFilterPolicyParser.kt b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/filters/TextFileFilterPolicyParser.kt index dd353e9caeff..2c75e68306cd 100644 --- a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/filters/TextFileFilterPolicyParser.kt +++ b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/filters/TextFileFilterPolicyParser.kt @@ -22,13 +22,13 @@ import com.android.hoststubgen.asm.toHumanReadableClassName import com.android.hoststubgen.asm.toJvmClassName import com.android.hoststubgen.log import com.android.hoststubgen.normalizeTextLine +import com.android.hoststubgen.utils.FileOrResource import com.android.hoststubgen.whitespaceRegex -import org.objectweb.asm.tree.ClassNode import java.io.BufferedReader -import java.io.FileReader import java.io.PrintWriter import java.io.Reader import java.util.regex.Pattern +import org.objectweb.asm.tree.ClassNode /** * Print a class node as a "keep" policy. @@ -143,10 +143,10 @@ class TextFileFilterPolicyBuilder( * Parse a given policy file. This method can be called multiple times to read from * multiple files. To get the resulting filter, use [createOutputFilter] */ - fun parse(file: String) { + fun parse(file: FileOrResource) { // We may parse multiple files, but we reuse the same parser, because the parser // will make sure there'll be no dupplicating "special class" policies. - parser.parse(FileReader(file), file, Processor()) + parser.parse(file.open(), file.path, Processor()) } /** diff --git a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/utils/OptionUtils.kt b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/utils/OptionUtils.kt index 0b17879b862c..d0869929edfb 100644 --- a/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/utils/OptionUtils.kt +++ b/ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/utils/OptionUtils.kt @@ -16,11 +16,16 @@ package com.android.hoststubgen.utils import com.android.hoststubgen.ArgumentsException -import com.android.hoststubgen.ensureFileExists +import com.android.hoststubgen.InputFileNotFoundException +import com.android.hoststubgen.JarResourceNotFoundException import com.android.hoststubgen.log import com.android.hoststubgen.normalizeTextLine -import java.io.BufferedReader +import java.io.File import java.io.FileReader +import java.io.InputStreamReader +import java.io.Reader + +const val JAR_RESOURCE_PREFIX = "jar:" /** * Base class for parsing arguments from commandline. @@ -73,7 +78,7 @@ abstract class BaseOptions { * * Subclasses override/extend this method to support more options. */ - abstract fun parseOption(option: String, ai: ArgIterator): Boolean + abstract fun parseOption(option: String, args: ArgIterator): Boolean abstract fun dumpFields(): String } @@ -112,7 +117,9 @@ class ArgIterator( companion object { fun withAtFiles(args: List<String>): ArgIterator { - return ArgIterator(expandAtFiles(args)) + val expanded = mutableListOf<String>() + expandAtFiles(args.asSequence(), expanded) + return ArgIterator(expanded) } /** @@ -125,34 +132,30 @@ class ArgIterator( * * The file can contain '#' as comments. */ - private fun expandAtFiles(args: List<String>): List<String> { - val ret = mutableListOf<String>() - + private fun expandAtFiles(args: Sequence<String>, out: MutableList<String>) { args.forEach { arg -> if (arg.startsWith("@@")) { - ret += arg.substring(1) + out.add(arg.substring(1)) return@forEach } else if (!arg.startsWith('@')) { - ret += arg + out.add(arg) return@forEach } + // Read from the file, and add each line to the result. - val filename = arg.substring(1).ensureFileExists() + val file = FileOrResource(arg.substring(1)) - log.v("Expanding options file $filename") + log.v("Expanding options file ${file.path}") - BufferedReader(FileReader(filename)).use { reader -> - while (true) { - var line = reader.readLine() ?: break // EOF + val fileArgs = file + .open() + .buffered() + .lineSequence() + .map(::normalizeTextLine) + .filter(CharSequence::isNotEmpty) - line = normalizeTextLine(line) - if (line.isNotEmpty()) { - ret += line - } - } - } + expandAtFiles(fileArgs, out) } - return ret } } } @@ -204,3 +207,37 @@ class IntSetOnce(value: Int) : SetOnce<Int>(value) { } } } + +/** + * A path either points to a file in filesystem, or an entry in the JAR. + */ +class FileOrResource(val path: String) { + init { + path.ensureFileExists() + } + + /** + * Either read from filesystem, or read from JAR resources. + */ + fun open(): Reader { + return if (path.startsWith(JAR_RESOURCE_PREFIX)) { + val path = path.removePrefix(JAR_RESOURCE_PREFIX) + InputStreamReader(this::class.java.classLoader.getResourceAsStream(path)!!) + } else { + FileReader(path) + } + } +} + +fun String.ensureFileExists(): String { + if (this.startsWith(JAR_RESOURCE_PREFIX)) { + val cl = FileOrResource::class.java.classLoader + val path = this.removePrefix(JAR_RESOURCE_PREFIX) + if (cl.getResource(path) == null) { + throw JarResourceNotFoundException(path) + } + } else if (!File(this).exists()) { + throw InputFileNotFoundException(this) + } + return this +} diff --git a/ravenwood/tools/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt b/ravenwood/tools/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt index 8bb454fa12e7..d9cc54aebf51 100644 --- a/ravenwood/tools/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt +++ b/ravenwood/tools/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt @@ -18,6 +18,7 @@ package com.android.hoststubgen import com.android.hoststubgen.utils.ArgIterator import com.android.hoststubgen.utils.IntSetOnce import com.android.hoststubgen.utils.SetOnce +import com.android.hoststubgen.utils.ensureFileExists /** * Options that can be set from command line arguments. @@ -61,9 +62,9 @@ class HostStubGenOptions( } } - override fun parseOption(option: String, ai: ArgIterator): Boolean { + override fun parseOption(option: String, args: ArgIterator): Boolean { // Define some shorthands... - fun nextArg(): String = ai.nextArgRequired(option) + fun nextArg(): String = args.nextArgRequired(option) when (option) { // TODO: Write help @@ -94,7 +95,7 @@ class HostStubGenOptions( } } - else -> return super.parseOption(option, ai) + else -> return super.parseOption(option, args) } return true diff --git a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/PtaOptions.kt b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/PtaOptions.kt index f7fd0804c151..58bd9e987fd1 100644 --- a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/PtaOptions.kt +++ b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/PtaOptions.kt @@ -16,10 +16,10 @@ package com.android.platform.test.ravenwood.ravenhelper.policytoannot import com.android.hoststubgen.ArgumentsException -import com.android.hoststubgen.ensureFileExists import com.android.hoststubgen.utils.ArgIterator import com.android.hoststubgen.utils.BaseOptions import com.android.hoststubgen.utils.SetOnce +import com.android.hoststubgen.utils.ensureFileExists /** * Options for the "ravenhelper pta" subcommand. @@ -41,8 +41,8 @@ class PtaOptions( var dumpOperations: SetOnce<Boolean> = SetOnce(false), ) : BaseOptions() { - override fun parseOption(option: String, ai: ArgIterator): Boolean { - fun nextArg(): String = ai.nextArgRequired(option) + override fun parseOption(option: String, args: ArgIterator): Boolean { + fun nextArg(): String = args.nextArgRequired(option) when (option) { // TODO: Write help diff --git a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/sourcemap/MapOptions.kt b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/sourcemap/MapOptions.kt index 8b95843f08a6..6e0b7b89cf13 100644 --- a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/sourcemap/MapOptions.kt +++ b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/sourcemap/MapOptions.kt @@ -16,10 +16,10 @@ package com.android.platform.test.ravenwood.ravenhelper.sourcemap import com.android.hoststubgen.ArgumentsException -import com.android.hoststubgen.ensureFileExists import com.android.hoststubgen.utils.ArgIterator import com.android.hoststubgen.utils.BaseOptions import com.android.hoststubgen.utils.SetOnce +import com.android.hoststubgen.utils.ensureFileExists /** * Options for the "ravenhelper map" subcommand. @@ -38,8 +38,8 @@ class MapOptions( var text: SetOnce<String?> = SetOnce(null), ) : BaseOptions() { - override fun parseOption(option: String, ai: ArgIterator): Boolean { - fun nextArg(): String = ai.nextArgRequired(option) + override fun parseOption(option: String, args: ArgIterator): Boolean { + fun nextArg(): String = args.nextArgRequired(option) when (option) { // TODO: Write help diff --git a/ravenwood/tools/ravenizer/Android.bp b/ravenwood/tools/ravenizer/Android.bp index 957e20647d44..93cda4e3c4c9 100644 --- a/ravenwood/tools/ravenizer/Android.bp +++ b/ravenwood/tools/ravenizer/Android.bp @@ -15,5 +15,10 @@ java_binary_host { "hoststubgen-lib", "ravenwood-junit-for-ravenizer", ], + java_resources: [ + ":ravenizer-standard-options", + ":ravenwood-standard-annotations", + ":ravenwood-common-policies", + ], visibility: ["//visibility:public"], } diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Ravenizer.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Ravenizer.kt index e67c730df069..04e3bda2ba27 100644 --- a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Ravenizer.kt +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Ravenizer.kt @@ -16,23 +16,22 @@ package com.android.platform.test.ravenwood.ravenizer import com.android.hoststubgen.GeneralUserErrorException +import com.android.hoststubgen.HostStubGenClassProcessor import com.android.hoststubgen.asm.ClassNodes import com.android.hoststubgen.asm.zipEntryNameToClassName import com.android.hoststubgen.executableName import com.android.hoststubgen.log import com.android.platform.test.ravenwood.ravenizer.adapter.RunnerRewritingAdapter -import org.objectweb.asm.ClassReader -import org.objectweb.asm.ClassVisitor -import org.objectweb.asm.ClassWriter -import org.objectweb.asm.util.CheckClassAdapter import java.io.BufferedInputStream import java.io.BufferedOutputStream import java.io.FileOutputStream -import java.io.InputStream -import java.io.OutputStream import java.util.zip.ZipEntry import java.util.zip.ZipFile import java.util.zip.ZipOutputStream +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassVisitor +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.util.CheckClassAdapter /** * Various stats on Ravenizer. @@ -41,7 +40,7 @@ data class RavenizerStats( /** Total end-to-end time. */ var totalTime: Double = .0, - /** Time took to build [ClasNodes] */ + /** Time took to build [ClassNodes] */ var loadStructureTime: Double = .0, /** Time took to validate the classes */ @@ -50,14 +49,17 @@ data class RavenizerStats( /** Total real time spent for converting the jar file */ var totalProcessTime: Double = .0, - /** Total real time spent for converting class files (except for I/O time). */ - var totalConversionTime: Double = .0, + /** Total real time spent for ravenizing class files (excluding I/O time). */ + var totalRavenizeTime: Double = .0, + + /** Total real time spent for processing class files HSG style (excluding I/O time). */ + var totalHostStubGenTime: Double = .0, /** Total real time spent for copying class files without modification. */ var totalCopyTime: Double = .0, /** # of entries in the input jar file */ - var totalEntiries: Int = 0, + var totalEntries: Int = 0, /** # of *.class files in the input jar file */ var totalClasses: Int = 0, @@ -67,14 +69,15 @@ data class RavenizerStats( ) { override fun toString(): String { return """ - RavenizerStats{ + RavenizerStats { totalTime=$totalTime, loadStructureTime=$loadStructureTime, validationTime=$validationTime, totalProcessTime=$totalProcessTime, - totalConversionTime=$totalConversionTime, + totalRavenizeTime=$totalRavenizeTime, + totalHostStubGenTime=$totalHostStubGenTime, totalCopyTime=$totalCopyTime, - totalEntiries=$totalEntiries, + totalEntries=$totalEntries, totalClasses=$totalClasses, processedClasses=$processedClasses, } @@ -90,12 +93,18 @@ class Ravenizer { val stats = RavenizerStats() stats.totalTime = log.nTime { + val allClasses = ClassNodes.loadClassStructures(options.inJar.get) { + stats.loadStructureTime = it + } + val processor = HostStubGenClassProcessor(options, allClasses) + process( options.inJar.get, options.outJar.get, options.enableValidation.get, options.fatalValidation.get, options.stripMockito.get, + processor, stats, ) } @@ -108,15 +117,13 @@ class Ravenizer { enableValidation: Boolean, fatalValidation: Boolean, stripMockito: Boolean, + processor: HostStubGenClassProcessor, stats: RavenizerStats, ) { - var allClasses = ClassNodes.loadClassStructures(inJar) { - time -> stats.loadStructureTime = time - } if (enableValidation) { stats.validationTime = log.iTime("Validating classes") { - if (!validateClasses(allClasses)) { - var message = "Invalid test class(es) detected." + + if (!validateClasses(processor.allClasses)) { + val message = "Invalid test class(es) detected." + " See error log for details." if (fatalValidation) { throw RavenizerInvalidTestException(message) @@ -126,7 +133,7 @@ class Ravenizer { } } } - if (includeUnsupportedMockito(allClasses)) { + if (includeUnsupportedMockito(processor.allClasses)) { log.w("Unsupported Mockito detected in $inJar!") } @@ -134,7 +141,7 @@ class Ravenizer { ZipFile(inJar).use { inZip -> val inEntries = inZip.entries() - stats.totalEntiries = inZip.size() + stats.totalEntries = inZip.size() ZipOutputStream(BufferedOutputStream(FileOutputStream(outJar))).use { outZip -> while (inEntries.hasMoreElements()) { @@ -159,9 +166,9 @@ class Ravenizer { stats.totalClasses += 1 } - if (className != null && shouldProcessClass(allClasses, className)) { - stats.processedClasses += 1 - processSingleClass(inZip, entry, outZip, allClasses, stats) + if (className != null && + shouldProcessClass(processor.allClasses, className)) { + processSingleClass(inZip, entry, outZip, processor, stats) } else { // Too slow, let's use merge_zips to bring back the original classes. copyZipEntry(inZip, entry, outZip, stats) @@ -201,14 +208,22 @@ class Ravenizer { inZip: ZipFile, entry: ZipEntry, outZip: ZipOutputStream, - allClasses: ClassNodes, + processor: HostStubGenClassProcessor, stats: RavenizerStats, ) { + stats.processedClasses += 1 val newEntry = ZipEntry(entry.name) outZip.putNextEntry(newEntry) BufferedInputStream(inZip.getInputStream(entry)).use { bis -> - processSingleClass(entry, bis, outZip, allClasses, stats) + var classBytes = bis.readBytes() + stats.totalRavenizeTime += log.vTime("Ravenize ${entry.name}") { + classBytes = ravenizeSingleClass(entry, classBytes, processor.allClasses) + } + stats.totalHostStubGenTime += log.vTime("HostStubGen ${entry.name}") { + classBytes = processor.processClassBytecode(classBytes) + } + outZip.write(classBytes) } outZip.closeEntry() } @@ -217,41 +232,34 @@ class Ravenizer { * Whether a class needs to be processed. This must be kept in sync with [processSingleClass]. */ private fun shouldProcessClass(classes: ClassNodes, classInternalName: String): Boolean { - return !classInternalName.shouldByBypassed() + return !classInternalName.shouldBypass() && RunnerRewritingAdapter.shouldProcess(classes, classInternalName) } - private fun processSingleClass( + private fun ravenizeSingleClass( entry: ZipEntry, - input: InputStream, - output: OutputStream, + input: ByteArray, allClasses: ClassNodes, - stats: RavenizerStats, - ) { - val cr = ClassReader(input) - - lateinit var data: ByteArray - stats.totalConversionTime += log.vTime("Modify ${entry.name}") { + ): ByteArray { + val classInternalName = zipEntryNameToClassName(entry.name) + ?: throw RavenizerInternalException("Unexpected zip entry name: ${entry.name}") - val classInternalName = zipEntryNameToClassName(entry.name) - ?: throw RavenizerInternalException("Unexpected zip entry name: ${entry.name}") - val flags = ClassWriter.COMPUTE_MAXS - val cw = ClassWriter(flags) - var outVisitor: ClassVisitor = cw + val flags = ClassWriter.COMPUTE_MAXS + val cw = ClassWriter(flags) + var outVisitor: ClassVisitor = cw - val enableChecker = false - if (enableChecker) { - outVisitor = CheckClassAdapter(outVisitor) - } + val enableChecker = false + if (enableChecker) { + outVisitor = CheckClassAdapter(outVisitor) + } - // This must be kept in sync with shouldProcessClass. - outVisitor = RunnerRewritingAdapter.maybeApply( - classInternalName, allClasses, outVisitor) + // This must be kept in sync with shouldProcessClass. + outVisitor = RunnerRewritingAdapter.maybeApply( + classInternalName, allClasses, outVisitor) - cr.accept(outVisitor, ClassReader.EXPAND_FRAMES) + val cr = ClassReader(input) + cr.accept(outVisitor, ClassReader.EXPAND_FRAMES) - data = cw.toByteArray() - } - output.write(data) + return cw.toByteArray() } } diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerMain.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerMain.kt index 7f4829ec6127..8a09e6d533b8 100644 --- a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerMain.kt +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerMain.kt @@ -21,6 +21,7 @@ import com.android.hoststubgen.LogLevel import com.android.hoststubgen.executableName import com.android.hoststubgen.log import com.android.hoststubgen.runMainWithBoilerplate +import com.android.hoststubgen.utils.JAR_RESOURCE_PREFIX import java.nio.file.Paths import kotlin.io.path.exists @@ -36,6 +37,10 @@ import kotlin.io.path.exists */ private val RAVENIZER_DOTFILE = System.getenv("HOME") + "/.ravenizer-unsafe" +/** + * This is the name of the standard option text file embedded inside ravenizer.jar. + */ +private const val RAVENIZER_STANDARD_OPTIONS = "texts/ravenizer-standard-options.txt" /** * Entry point. @@ -45,12 +50,12 @@ fun main(args: Array<String>) { log.setConsoleLogLevel(LogLevel.Info) runMainWithBoilerplate { - var newArgs = args.asList() + val newArgs = args.toMutableList() + newArgs.add(0, "@$JAR_RESOURCE_PREFIX$RAVENIZER_STANDARD_OPTIONS") + if (Paths.get(RAVENIZER_DOTFILE).exists()) { log.i("Reading options from $RAVENIZER_DOTFILE") - newArgs = args.toMutableList().apply { - add(0, "@$RAVENIZER_DOTFILE") - } + newArgs.add(0, "@$RAVENIZER_DOTFILE") } val options = RavenizerOptions().apply { parseArgs(newArgs) } diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerOptions.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerOptions.kt index 2c0365404ab6..5d278bb046ae 100644 --- a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerOptions.kt +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerOptions.kt @@ -16,10 +16,10 @@ package com.android.platform.test.ravenwood.ravenizer import com.android.hoststubgen.ArgumentsException -import com.android.hoststubgen.ensureFileExists +import com.android.hoststubgen.HostStubGenClassProcessorOptions import com.android.hoststubgen.utils.ArgIterator -import com.android.hoststubgen.utils.BaseOptions import com.android.hoststubgen.utils.SetOnce +import com.android.hoststubgen.utils.ensureFileExists class RavenizerOptions( /** Input jar file*/ @@ -36,10 +36,10 @@ class RavenizerOptions( /** Whether to remove mockito and dexmaker classes. */ var stripMockito: SetOnce<Boolean> = SetOnce(false), -) : BaseOptions() { +) : HostStubGenClassProcessorOptions() { - override fun parseOption(option: String, ai: ArgIterator): Boolean { - fun nextArg(): String = ai.nextArgRequired(option) + override fun parseOption(option: String, args: ArgIterator): Boolean { + fun nextArg(): String = args.nextArgRequired(option) when (option) { // TODO: Write help @@ -57,7 +57,7 @@ class RavenizerOptions( "--strip-mockito" -> stripMockito.set(true) "--no-strip-mockito" -> stripMockito.set(false) - else -> return false + else -> return super.parseOption(option, args) } return true @@ -79,6 +79,6 @@ class RavenizerOptions( enableValidation=$enableValidation, fatalValidation=$fatalValidation, stripMockito=$stripMockito, - """.trimIndent() + """.trimIndent() + '\n' + super.dumpFields() } } diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Utils.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Utils.kt index 6092fcc9402d..b394a761c7ae 100644 --- a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Utils.kt +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Utils.kt @@ -15,8 +15,8 @@ */ package com.android.platform.test.ravenwood.ravenizer -import android.platform.test.annotations.internal.InnerRunner import android.platform.test.annotations.NoRavenizer +import android.platform.test.annotations.internal.InnerRunner import android.platform.test.ravenwood.RavenwoodAwareTestRunner import com.android.hoststubgen.asm.ClassNodes import com.android.hoststubgen.asm.findAnyAnnotation @@ -85,7 +85,7 @@ fun String.isRavenwoodClass(): Boolean { /** * Classes that should never be modified. */ -fun String.shouldByBypassed(): Boolean { +fun String.shouldBypass(): Boolean { if (this.isRavenwoodClass()) { return true } diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Validator.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Validator.kt index 61e254b225c3..d252b4dc8ab6 100644 --- a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Validator.kt +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Validator.kt @@ -20,8 +20,8 @@ import com.android.hoststubgen.asm.isAbstract import com.android.hoststubgen.asm.startsWithAny import com.android.hoststubgen.asm.toHumanReadableClassName import com.android.hoststubgen.log -import org.objectweb.asm.tree.ClassNode import java.util.regex.Pattern +import org.objectweb.asm.tree.ClassNode fun validateClasses(classes: ClassNodes): Boolean { var allOk = true @@ -37,7 +37,7 @@ fun validateClasses(classes: ClassNodes): Boolean { * */ fun checkClass(cn: ClassNode, classes: ClassNodes): Boolean { - if (cn.name.shouldByBypassed()) { + if (cn.name.shouldBypass()) { // Class doesn't need to be checked. return true } @@ -145,4 +145,4 @@ com.android.server.power.stats.BatteryStatsTimerTest private fun isAllowListedLegacyTest(targetClass: ClassNode): Boolean { return allowListedLegacyTests.contains(targetClass.name) -}
\ No newline at end of file +} diff --git a/services/tests/InputMethodSystemServerTests/Android.bp b/services/tests/InputMethodSystemServerTests/Android.bp index ae9a34efc222..c1d8382fcd0e 100644 --- a/services/tests/InputMethodSystemServerTests/Android.bp +++ b/services/tests/InputMethodSystemServerTests/Android.bp @@ -73,10 +73,7 @@ android_ravenwood_test { static_libs: [ "androidx.annotation_annotation", "androidx.test.rules", - "framework", - "ravenwood-runtime", - "ravenwood-utils", - "services", + "services.core", ], libs: [ "android.test.base.stubs.system", |