diff options
| -rw-r--r-- | Ravenwood.bp | 232 | ||||
| -rw-r--r-- | ravenwood/Android.bp | 8 | ||||
| -rw-r--r-- | tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGen.kt | 22 | ||||
| -rw-r--r-- | tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt | 15 |
4 files changed, 249 insertions, 28 deletions
diff --git a/Ravenwood.bp b/Ravenwood.bp index cebeb18b84bb..7faa33f8834e 100644 --- a/Ravenwood.bp +++ b/Ravenwood.bp @@ -26,10 +26,105 @@ java_library { } // Generate the stub/impl from framework-all, with hidden APIs. -java_genrule { - name: "framework-minus-apex.ravenwood-base", +// This step takes several tens of seconds, so we manually shard it to multiple modules. +// All the copies have to be kept in sync. +// TODO: Do the sharding better. + +genrule_defaults { + name: "framework-minus-apex.ravenwood-base_defaults", tools: ["hoststubgen"], + srcs: [ + ":framework-minus-apex-for-hoststubgen", + ":ravenwood-framework-policies", + ":ravenwood-standard-options", + ":ravenwood-annotation-allowed-classes", + ], + out: [ + "ravenwood.jar", + + // Following files are created just as FYI. + "hoststubgen_framework-minus-apex_keep_all.txt", + "hoststubgen_framework-minus-apex_dump.txt", + + "hoststubgen_framework-minus-apex.log", + "hoststubgen_framework-minus-apex_stats.csv", + "hoststubgen_framework-minus-apex_apis.csv", + ], + visibility: ["//visibility:private"], +} + +java_genrule { + name: "framework-minus-apex.ravenwood-base_X0", + defaults: ["framework-minus-apex.ravenwood-base_defaults"], + cmd: "$(location hoststubgen) " + + "--num-shards 6 --shard-index 0 " + // Only this line differs + + "@$(location :ravenwood-standard-options) " + + + "--debug-log $(location hoststubgen_framework-minus-apex.log) " + + "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " + + "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " + + + "--out-impl-jar $(location ravenwood.jar) " + + + "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " + + "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " + + + "--in-jar $(location :framework-minus-apex-for-hoststubgen) " + + "--policy-override-file $(location :ravenwood-framework-policies) " + + "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ", +} + +java_genrule { + name: "framework-minus-apex.ravenwood-base_X1", + defaults: ["framework-minus-apex.ravenwood-base_defaults"], + cmd: "$(location hoststubgen) " + + "--num-shards 6 --shard-index 1 " + // Only this line differs + + "@$(location :ravenwood-standard-options) " + + + "--debug-log $(location hoststubgen_framework-minus-apex.log) " + + "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " + + "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " + + + "--out-impl-jar $(location ravenwood.jar) " + + + "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " + + "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " + + + "--in-jar $(location :framework-minus-apex-for-hoststubgen) " + + "--policy-override-file $(location :ravenwood-framework-policies) " + + "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ", +} + +java_genrule { + name: "framework-minus-apex.ravenwood-base_X2", + defaults: ["framework-minus-apex.ravenwood-base_defaults"], + cmd: "$(location hoststubgen) " + + "--num-shards 6 --shard-index 2 " + // Only this line differs + + "@$(location :ravenwood-standard-options) " + + + "--debug-log $(location hoststubgen_framework-minus-apex.log) " + + "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " + + "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " + + + "--out-impl-jar $(location ravenwood.jar) " + + + "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " + + "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " + + + "--in-jar $(location :framework-minus-apex-for-hoststubgen) " + + "--policy-override-file $(location :ravenwood-framework-policies) " + + "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ", +} + +java_genrule { + name: "framework-minus-apex.ravenwood-base_X3", + defaults: ["framework-minus-apex.ravenwood-base_defaults"], cmd: "$(location hoststubgen) " + + "--num-shards 6 --shard-index 3 " + // Only this line differs + "@$(location :ravenwood-standard-options) " + "--debug-log $(location hoststubgen_framework-minus-apex.log) " + @@ -44,39 +139,130 @@ java_genrule { "--in-jar $(location :framework-minus-apex-for-hoststubgen) " + "--policy-override-file $(location :ravenwood-framework-policies) " + "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ", +} + +java_genrule { + name: "framework-minus-apex.ravenwood-base_X4", + defaults: ["framework-minus-apex.ravenwood-base_defaults"], + cmd: "$(location hoststubgen) " + + "--num-shards 6 --shard-index 4 " + // Only this line differs + + "@$(location :ravenwood-standard-options) " + + + "--debug-log $(location hoststubgen_framework-minus-apex.log) " + + "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " + + "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " + + + "--out-impl-jar $(location ravenwood.jar) " + + + "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " + + "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " + + + "--in-jar $(location :framework-minus-apex-for-hoststubgen) " + + "--policy-override-file $(location :ravenwood-framework-policies) " + + "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ", +} + +java_genrule { + name: "framework-minus-apex.ravenwood-base_X5", + defaults: ["framework-minus-apex.ravenwood-base_defaults"], + cmd: "$(location hoststubgen) " + + "--num-shards 6 --shard-index 5 " + // Only this line differs + + "@$(location :ravenwood-standard-options) " + + + "--debug-log $(location hoststubgen_framework-minus-apex.log) " + + "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " + + "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " + + + "--out-impl-jar $(location ravenwood.jar) " + + + "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " + + "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " + + + "--in-jar $(location :framework-minus-apex-for-hoststubgen) " + + "--policy-override-file $(location :ravenwood-framework-policies) " + + "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ", +} + +// Marge all the sharded jars +java_genrule { + name: "framework-minus-apex.ravenwood", + defaults: ["ravenwood-internal-only-visibility-java"], + cmd: "$(location merge_zips) $(out) $(in)", + tools: ["merge_zips"], srcs: [ - ":framework-minus-apex-for-hoststubgen", - ":ravenwood-framework-policies", - ":ravenwood-standard-options", - ":ravenwood-annotation-allowed-classes", + ":framework-minus-apex.ravenwood-base_X0{ravenwood.jar}", + ":framework-minus-apex.ravenwood-base_X1{ravenwood.jar}", + ":framework-minus-apex.ravenwood-base_X2{ravenwood.jar}", + ":framework-minus-apex.ravenwood-base_X3{ravenwood.jar}", + ":framework-minus-apex.ravenwood-base_X4{ravenwood.jar}", + ":framework-minus-apex.ravenwood-base_X5{ravenwood.jar}", ], out: [ - "ravenwood.jar", - - // Following files are created just as FYI. - "hoststubgen_framework-minus-apex_keep_all.txt", - "hoststubgen_framework-minus-apex_dump.txt", + "framework-minus-apex.ravenwood.jar", + ], +} - "hoststubgen_framework-minus-apex.log", - "hoststubgen_framework-minus-apex_stats.csv", - "hoststubgen_framework-minus-apex_apis.csv", +// Merge the sharded text files +genrule { + name: "hoststubgen_framework-minus-apex_stats.csv", + defaults: ["ravenwood-internal-only-visibility-genrule"], + cmd: "cat $(in) > $(out)", + srcs: [ + ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_stats.csv}", + ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_stats.csv}", + ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_stats.csv}", + ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_stats.csv}", + ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_stats.csv}", + ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_stats.csv}", ], + out: ["hoststubgen_framework-minus-apex_stats.csv"], +} + +genrule { + name: "hoststubgen_framework-minus-apex_apis.csv", defaults: ["ravenwood-internal-only-visibility-genrule"], + cmd: "cat $(in) > $(out)", + srcs: [ + ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_apis.csv}", + ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_apis.csv}", + ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_apis.csv}", + ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_apis.csv}", + ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_apis.csv}", + ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_apis.csv}", + ], + out: ["hoststubgen_framework-minus-apex_apis.csv"], } -// Extract the impl jar from "framework-minus-apex.ravenwood-base" for subsequent build rules. -// Note this emits a "device side" output, so that ravenwood tests can (implicitly) -// depend on it. -java_genrule { - name: "framework-minus-apex.ravenwood", +genrule { + name: "hoststubgen_framework-minus-apex_keep_all.txt", defaults: ["ravenwood-internal-only-visibility-genrule"], - cmd: "cp $(in) $(out)", + cmd: "cat $(in) > $(out)", srcs: [ - ":framework-minus-apex.ravenwood-base{ravenwood.jar}", + ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_keep_all.txt}", + ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_keep_all.txt}", + ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_keep_all.txt}", + ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_keep_all.txt}", + ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_keep_all.txt}", + ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_keep_all.txt}", ], - out: [ - "framework-minus-apex.ravenwood.jar", + out: ["hoststubgen_framework-minus-apex_keep_all.txt"], +} + +genrule { + name: "hoststubgen_framework-minus-apex_dump.txt", + defaults: ["ravenwood-internal-only-visibility-genrule"], + cmd: "cat $(in) > $(out)", + srcs: [ + ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_dump.txt}", + ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_dump.txt}", + ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_dump.txt}", + ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_dump.txt}", + ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_dump.txt}", + ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_dump.txt}", ], + out: ["hoststubgen_framework-minus-apex_dump.txt"], } java_library { diff --git a/ravenwood/Android.bp b/ravenwood/Android.bp index 2de3c5ef1967..4faf03c395f3 100644 --- a/ravenwood/Android.bp +++ b/ravenwood/Android.bp @@ -280,10 +280,10 @@ sh_test_host { src: "scripts/ravenwood-stats-checker.sh", test_suites: ["general-tests"], data: [ - ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_stats.csv}", - ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_apis.csv}", - ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_keep_all.txt}", - ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_dump.txt}", + ":hoststubgen_framework-minus-apex_stats.csv", + ":hoststubgen_framework-minus-apex_apis.csv", + ":hoststubgen_framework-minus-apex_keep_all.txt", + ":hoststubgen_framework-minus-apex_dump.txt", ":services.core.ravenwood-base{hoststubgen_services.core_stats.csv}", ":services.core.ravenwood-base{hoststubgen_services.core_apis.csv}", ":services.core.ravenwood-base{hoststubgen_services.core_keep_all.txt}", diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGen.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGen.kt index 73cfd3f7ceec..5dde265c79fb 100644 --- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGen.kt +++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGen.kt @@ -85,6 +85,8 @@ class HostStubGen(val options: HostStubGenOptions) { errors, stats, policyFileRemapper, + options.numShards.get, + options.shard.get, ) // Dump statistics, if specified. @@ -213,6 +215,8 @@ class HostStubGen(val options: HostStubGenOptions) { errors: HostStubGenErrors, stats: HostStubGenStats, remapper: Remapper?, + numShards: Int, + shard: Int, ) { log.i("Converting %s into [stub: %s, impl: %s] ...", inJar, outStubJar, outImplJar) log.i("ASM CheckClassAdapter is %s", if (enableChecker) "enabled" else "disabled") @@ -221,17 +225,32 @@ class HostStubGen(val options: HostStubGenOptions) { val packageRedirector = PackageRedirectRemapper(options.packageRedirects) + var itemIndex = 0 + var numItemsProcessed = 0 + var numItems = -1 // == Unknown + log.withIndent { // Open the input jar file and process each entry. ZipFile(inJar).use { inZip -> + + numItems = inZip.size() + val shardStart = numItems * shard / numShards + val shardNextStart = numItems * (shard + 1) / numShards + maybeWithZipOutputStream(outStubJar) { stubOutStream -> maybeWithZipOutputStream(outImplJar) { implOutStream -> val inEntries = inZip.entries() while (inEntries.hasMoreElements()) { val entry = inEntries.nextElement() + val inShard = (shardStart <= itemIndex) && (itemIndex < shardNextStart) + itemIndex++ + if (!inShard) { + continue + } convertSingleEntry(inZip, entry, stubOutStream, implOutStream, filter, packageRedirector, remapper, enableChecker, classes, errors, stats) + numItemsProcessed++ } log.i("Converted all entries.") } @@ -241,7 +260,8 @@ class HostStubGen(val options: HostStubGenOptions) { } } val end = System.currentTimeMillis() - log.i("Done transforming the jar in %.1f second(s).", (end - start) / 1000.0) + log.i("Done transforming the jar in %.1f second(s). %d / %d item(s) processed.", + (end - start) / 1000.0, numItemsProcessed, numItems) } private fun <T> maybeWithZipOutputStream(filename: String?, block: (ZipOutputStream?) -> T): T { diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt index e192516d334d..2f833a873133 100644 --- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt +++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt @@ -112,6 +112,9 @@ class HostStubGenOptions( var statsFile: SetOnce<String?> = SetOnce(null), var apiListFile: SetOnce<String?> = SetOnce(null), + + var numShards: SetOnce<Int> = SetOnce(1), + var shard: SetOnce<Int> = SetOnce(0), ) { companion object { @@ -162,6 +165,13 @@ class HostStubGenOptions( fun SetOnce<String?>.setNextStringArg(): String = nextArg().also { this.set(it) } fun MutableSet<String>.addUniqueAnnotationArg(): String = nextArg().also { this += ensureUniqueAnnotation(it) } + fun SetOnce<Int>.setNextIntArg(): String = nextArg().also { + try { + this.set(it.toInt()) + } catch (e: NumberFormatException) { + throw ArgumentsException("Invalid integer for $arg: $it") + } + } try { when (arg) { @@ -259,6 +269,9 @@ class HostStubGenOptions( "--stats-file" -> ret.statsFile.setNextStringArg() "--supported-api-list-file" -> ret.apiListFile.setNextStringArg() + "--num-shards" -> ret.numShards.setNextIntArg() + "--shard-index" -> ret.shard.setNextIntArg() + else -> throw ArgumentsException("Unknown option: $arg") } } catch (e: SetOnce.SetMoreThanOnceException) { @@ -396,6 +409,8 @@ class HostStubGenOptions( enableNonStubMethodCallDetection=$enableNonStubMethodCallDetection, statsFile=$statsFile, apiListFile=$apiListFile, + numShards=$numShards, + shard=$shard, } """.trimIndent() } |