From 384642657654170b0cd37a35d25b70d188584a41 Mon Sep 17 00:00:00 2001 From: Artur Satayev Date: Wed, 20 Nov 2019 10:44:09 +0000 Subject: Pass hiddenapi-greylist-max-q.txt to greylist generator. Bug: 137350495 Test: m, manual inspection of out/soong/hiddenapi/hiddenapi-flags.csv Change-Id: I67227307ec08ab56a385980eceacaab1b985eb49 --- java/hiddenapi_singleton.go | 2 ++ 1 file changed, 2 insertions(+) (limited to 'java/hiddenapi_singleton.go') diff --git a/java/hiddenapi_singleton.go b/java/hiddenapi_singleton.go index 8379f5397..c0ef444fc 100644 --- a/java/hiddenapi_singleton.go +++ b/java/hiddenapi_singleton.go @@ -241,6 +241,8 @@ func flagsRule(ctx android.SingletonContext) android.Path { android.PathForSource(ctx, "frameworks/base/config/hiddenapi-greylist.txt")). FlagWithInput("--greylist-ignore-conflicts ", greylistIgnoreConflicts). + FlagWithInput("--greylist-max-q ", + android.PathForSource(ctx, "frameworks/base/config/hiddenapi-greylist-max-q.txt")). FlagWithInput("--greylist-max-p ", android.PathForSource(ctx, "frameworks/base/config/hiddenapi-greylist-max-p.txt")). FlagWithInput("--greylist-max-o-ignore-conflicts ", -- cgit v1.2.3-59-g8ed1b From 8a950790ee7c2aa3d2eb45d7da54978a3cc64169 Mon Sep 17 00:00:00 2001 From: Artur Satayev Date: Wed, 19 Feb 2020 16:39:59 +0000 Subject: Merge CSV files generated by UnsupportedAppUsageProcessor. Flow: 1. Annotation processor generates a CSV file per class as a CLASS_OUTPUT resource. 2. hiddenapi.go extracts individual .csv files and merges them into an index.csv file per module. 3. hiddenapi_singleton.go merges individual index.csv files into a combined .csv file. In a follow up hiddenapi-index.csv would replace unsupportedappusage_index.csv Bug: 145132366 Change-Id: I87d92f9c8d4b1cc1df526fc576ee3c2101116b58 Merged-In: I87d92f9c8d4b1cc1df526fc576ee3c2101116b58 Test: diff unsupportedappusage_index.csv hiddenapi-index.csv Exempt-From-Owner-Approval: cp from r.android.com/1239709 --- java/hiddenapi.go | 51 +++++++++++++++++++++++++++------------------ java/hiddenapi_singleton.go | 49 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 22 deletions(-) (limited to 'java/hiddenapi_singleton.go') diff --git a/java/hiddenapi.go b/java/hiddenapi.go index 8f34714d3..884a757e2 100644 --- a/java/hiddenapi.go +++ b/java/hiddenapi.go @@ -28,9 +28,10 @@ var hiddenAPIGenerateCSVRule = pctx.AndroidStaticRule("hiddenAPIGenerateCSV", bl }, "outFlag", "stubAPIFlags") type hiddenAPI struct { + bootDexJarPath android.Path flagsCSVPath android.Path + indexCSVPath android.Path metadataCSVPath android.Path - bootDexJarPath android.Path } func (h *hiddenAPI) flagsCSV() android.Path { @@ -45,17 +46,21 @@ func (h *hiddenAPI) bootDexJar() android.Path { return h.bootDexJarPath } +func (h *hiddenAPI) indexCSV() android.Path { + return h.indexCSVPath +} + type hiddenAPIIntf interface { + bootDexJar() android.Path flagsCSV() android.Path + indexCSV() android.Path metadataCSV() android.Path - bootDexJar() android.Path } var _ hiddenAPIIntf = (*hiddenAPI)(nil) -func (h *hiddenAPI) hiddenAPI(ctx android.ModuleContext, dexJar android.ModuleOutPath, implementationJar android.Path, - uncompressDex bool) android.ModuleOutPath { - +func (h *hiddenAPI) hiddenAPI(ctx android.ModuleContext, dexJar android.ModuleOutPath, + implementationJar android.Path, uncompressDex bool) android.ModuleOutPath { if !ctx.Config().IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") { name := ctx.ModuleName() @@ -77,9 +82,8 @@ func (h *hiddenAPI) hiddenAPI(ctx android.ModuleContext, dexJar android.ModuleOu // Derive the greylist from classes jar. flagsCSV := android.PathForModuleOut(ctx, "hiddenapi", "flags.csv") metadataCSV := android.PathForModuleOut(ctx, "hiddenapi", "metadata.csv") - hiddenAPIGenerateCSV(ctx, flagsCSV, metadataCSV, implementationJar) - h.flagsCSVPath = flagsCSV - h.metadataCSVPath = metadataCSV + indexCSV := android.PathForModuleOut(ctx, "hiddenapi", "index.csv") + h.hiddenAPIGenerateCSV(ctx, flagsCSV, metadataCSV, indexCSV, implementationJar) // If this module is actually on the boot jars list and not providing // hiddenapi information for a module on the boot jars list then encode @@ -96,9 +100,7 @@ func (h *hiddenAPI) hiddenAPI(ctx android.ModuleContext, dexJar android.ModuleOu return dexJar } -func hiddenAPIGenerateCSV(ctx android.ModuleContext, flagsCSV, metadataCSV android.WritablePath, - classesJar android.Path) { - +func (h *hiddenAPI) hiddenAPIGenerateCSV(ctx android.ModuleContext, flagsCSV, metadataCSV, indexCSV android.WritablePath, classesJar android.Path) { stubFlagsCSV := hiddenAPISingletonPaths(ctx).stubFlags ctx.Build(pctx, android.BuildParams{ @@ -112,6 +114,7 @@ func hiddenAPIGenerateCSV(ctx android.ModuleContext, flagsCSV, metadataCSV andro "stubAPIFlags": stubFlagsCSV.String(), }, }) + h.flagsCSVPath = flagsCSV ctx.Build(pctx, android.BuildParams{ Rule: hiddenAPIGenerateCSVRule, @@ -124,18 +127,26 @@ func hiddenAPIGenerateCSV(ctx android.ModuleContext, flagsCSV, metadataCSV andro "stubAPIFlags": stubFlagsCSV.String(), }, }) - + h.metadataCSVPath = metadataCSV + + rule := android.NewRuleBuilder() + rule.Command(). + BuiltTool(ctx, "merge_csv"). + FlagWithInput("--zip_input=", classesJar). + FlagWithOutput("--output=", indexCSV) + rule.Build(pctx, ctx, "merged-hiddenapi-index", "Merged Hidden API index") + h.indexCSVPath = indexCSV } var hiddenAPIEncodeDexRule = pctx.AndroidStaticRule("hiddenAPIEncodeDex", blueprint.RuleParams{ - Command: `rm -rf $tmpDir && mkdir -p $tmpDir && mkdir $tmpDir/dex-input && mkdir $tmpDir/dex-output && ` + - `unzip -o -q $in 'classes*.dex' -d $tmpDir/dex-input && ` + - `for INPUT_DEX in $$(find $tmpDir/dex-input -maxdepth 1 -name 'classes*.dex' | sort); do ` + - ` echo "--input-dex=$${INPUT_DEX}"; ` + - ` echo "--output-dex=$tmpDir/dex-output/$$(basename $${INPUT_DEX})"; ` + - `done | xargs ${config.HiddenAPI} encode --api-flags=$flagsCsv $hiddenapiFlags && ` + - `${config.SoongZipCmd} $soongZipFlags -o $tmpDir/dex.jar -C $tmpDir/dex-output -f "$tmpDir/dex-output/classes*.dex" && ` + - `${config.MergeZipsCmd} -D -zipToNotStrip $tmpDir/dex.jar -stripFile "classes*.dex" $out $tmpDir/dex.jar $in`, + Command: `rm -rf $tmpDir && mkdir -p $tmpDir && mkdir $tmpDir/dex-input && mkdir $tmpDir/dex-output && + unzip -o -q $in 'classes*.dex' -d $tmpDir/dex-input && + for INPUT_DEX in $$(find $tmpDir/dex-input -maxdepth 1 -name 'classes*.dex' | sort); do + echo "--input-dex=$${INPUT_DEX}"; + echo "--output-dex=$tmpDir/dex-output/$$(basename $${INPUT_DEX})"; + done | xargs ${config.HiddenAPI} encode --api-flags=$flagsCsv $hiddenapiFlags && + ${config.SoongZipCmd} $soongZipFlags -o $tmpDir/dex.jar -C $tmpDir/dex-output -f "$tmpDir/dex-output/classes*.dex" && + ${config.MergeZipsCmd} -D -zipToNotStrip $tmpDir/dex.jar -stripFile "classes*.dex" -stripFile "**/*.uau" $out $tmpDir/dex.jar $in`, CommandDeps: []string{ "${config.HiddenAPI}", "${config.SoongZipCmd}", diff --git a/java/hiddenapi_singleton.go b/java/hiddenapi_singleton.go index 785019324..7e7e955f5 100644 --- a/java/hiddenapi_singleton.go +++ b/java/hiddenapi_singleton.go @@ -22,13 +22,15 @@ import ( func init() { android.RegisterSingletonType("hiddenapi", hiddenAPISingletonFactory) + android.RegisterSingletonType("hiddenapi_index", hiddenAPIIndexSingletonFactory) android.RegisterModuleType("hiddenapi_flags", hiddenAPIFlagsFactory) } type hiddenAPISingletonPathsStruct struct { - stubFlags android.OutputPath flags android.OutputPath + index android.OutputPath metadata android.OutputPath + stubFlags android.OutputPath } var hiddenAPISingletonPathsKey = android.NewOnceKey("hiddenAPISingletonPathsKey") @@ -39,9 +41,10 @@ var hiddenAPISingletonPathsKey = android.NewOnceKey("hiddenAPISingletonPathsKey" func hiddenAPISingletonPaths(ctx android.PathContext) hiddenAPISingletonPathsStruct { return ctx.Config().Once(hiddenAPISingletonPathsKey, func() interface{} { return hiddenAPISingletonPathsStruct{ - stubFlags: android.PathForOutput(ctx, "hiddenapi", "hiddenapi-stub-flags.txt"), flags: android.PathForOutput(ctx, "hiddenapi", "hiddenapi-flags.csv"), + index: android.PathForOutput(ctx, "hiddenapi", "hiddenapi-index.csv"), metadata: android.PathForOutput(ctx, "hiddenapi", "hiddenapi-greylist.csv"), + stubFlags: android.PathForOutput(ctx, "hiddenapi", "hiddenapi-stub-flags.txt"), } }).(hiddenAPISingletonPathsStruct) } @@ -364,3 +367,45 @@ func hiddenAPIFlagsFactory() android.Module { android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon) return module } + +func hiddenAPIIndexSingletonFactory() android.Singleton { + return &hiddenAPIIndexSingleton{} +} + +type hiddenAPIIndexSingleton struct { + index android.Path +} + +func (h *hiddenAPIIndexSingleton) GenerateBuildActions(ctx android.SingletonContext) { + // Don't run any hiddenapi rules if UNSAFE_DISABLE_HIDDENAPI_FLAGS=true + if ctx.Config().IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") { + return + } + + indexes := android.Paths{} + ctx.VisitAllModules(func(module android.Module) { + if h, ok := module.(hiddenAPIIntf); ok { + if h.indexCSV() != nil { + indexes = append(indexes, h.indexCSV()) + } + } + }) + + rule := android.NewRuleBuilder() + rule.Command(). + BuiltTool(ctx, "merge_csv"). + FlagWithArg("--header=", "signature,file,startline,startcol,endline,endcol,properties"). + FlagWithOutput("--output=", hiddenAPISingletonPaths(ctx).index). + Inputs(indexes) + rule.Build(pctx, ctx, "singleton-merged-hiddenapi-index", "Singleton merged Hidden API index") + + h.index = hiddenAPISingletonPaths(ctx).index +} + +func (h *hiddenAPIIndexSingleton) MakeVars(ctx android.MakeVarsContext) { + if ctx.Config().IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") { + return + } + + ctx.Strict("INTERNAL_PLATFORM_HIDDENAPI_INDEX", h.index.String()) +} -- cgit v1.2.3-59-g8ed1b From a8ec55990f6ac6f0f5a601df5942763fe1d01dc2 Mon Sep 17 00:00:00 2001 From: Artur Satayev Date: Wed, 25 Mar 2020 16:48:49 +0000 Subject: Use modular removed-dex.txt files for greylisting. Use droidstubs for public and system stubs to provide a list of @removed APIs. As these APIs are not present in the stubs, they are not whitelisted / greylised automatically. Keep them on greylist manually. Bug: 143864733 Test: diff out/soong/hiddenapi/hiddenapi-flags.csv Change-Id: I4c8e6899fadfdfd1da82f6f453cc92e71aa9b78c Merged-In: I4c8e6899fadfdfd1da82f6f453cc92e71aa9b78c Exempt-From-Owner-Approval: clean cherry-pick (cherry picked from commit c7fb5c99644d2434c9e6c957f7366d0467284211) --- java/hiddenapi_singleton.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'java/hiddenapi_singleton.go') diff --git a/java/hiddenapi_singleton.go b/java/hiddenapi_singleton.go index 7e7e955f5..c7f7cbdfe 100644 --- a/java/hiddenapi_singleton.go +++ b/java/hiddenapi_singleton.go @@ -211,23 +211,30 @@ func stubFlagsRule(ctx android.SingletonContext) { // the greylists. func flagsRule(ctx android.SingletonContext) android.Path { var flagsCSV android.Paths - - var greylistIgnoreConflicts android.Path + var greylistRemovedApis android.Paths ctx.VisitAllModules(func(module android.Module) { if h, ok := module.(hiddenAPIIntf); ok { if csv := h.flagsCSV(); csv != nil { flagsCSV = append(flagsCSV, csv) } - } else if ds, ok := module.(*Droidstubs); ok && ctx.ModuleName(module) == "hiddenapi-lists-docs" { - greylistIgnoreConflicts = ds.removedDexApiFile + } else if ds, ok := module.(*Droidstubs); ok { + // Track @removed public and system APIs via corresponding droidstubs targets. + // These APIs are not present in the stubs, however, we have to keep allowing access + // to them at runtime. + if m := ctx.ModuleName(module); m == "api-stubs-docs" || m == "system-api-stubs-docs" { + greylistRemovedApis = append(greylistRemovedApis, ds.removedDexApiFile) + } } }) - if greylistIgnoreConflicts == nil { - ctx.Errorf("failed to find removed_dex_api_filename from hiddenapi-lists-docs module") - return nil - } + combinedRemovedApis := android.PathForOutput(ctx, "hiddenapi", "combined-removed-dex.txt") + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cat, + Inputs: greylistRemovedApis, + Output: combinedRemovedApis, + Description: "Combine removed apis for " + combinedRemovedApis.String(), + }) rule := android.NewRuleBuilder() @@ -242,8 +249,7 @@ func flagsRule(ctx android.SingletonContext) android.Path { Inputs(flagsCSV). FlagWithInput("--greylist ", android.PathForSource(ctx, "frameworks/base/config/hiddenapi-greylist.txt")). - FlagWithInput("--greylist-ignore-conflicts ", - greylistIgnoreConflicts). + FlagWithInput("--greylist-ignore-conflicts ", combinedRemovedApis). FlagWithInput("--greylist-max-q ", android.PathForSource(ctx, "frameworks/base/config/hiddenapi-greylist-max-q.txt")). FlagWithInput("--greylist-max-p ", -- cgit v1.2.3-59-g8ed1b From 9791efa3e8e88836201e16f1e57401a0c215d0de Mon Sep 17 00:00:00 2001 From: Eric Jeong Date: Thu, 4 Jun 2020 17:56:18 -0700 Subject: Include car project in grey list removed apis - Car framework has removed system APIs. - These APIs should be tracked at run-time. Bug: 154832144 Test: 1) android.signature.cts.api.AnnotationTest#testAnnotation 2) diff $OUT/soong/hiddenapi/hiddenapi-flags.csv before and after this CL Change-Id: I9813df4b514da41e82d45e6c0bc92643916fe0d8 --- java/hiddenapi_singleton.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'java/hiddenapi_singleton.go') diff --git a/java/hiddenapi_singleton.go b/java/hiddenapi_singleton.go index c7f7cbdfe..95dd0bb09 100644 --- a/java/hiddenapi_singleton.go +++ b/java/hiddenapi_singleton.go @@ -207,6 +207,15 @@ func stubFlagsRule(ctx android.SingletonContext) { rule.Build(pctx, ctx, "hiddenAPIStubFlagsFile", "hiddenapi stub flags") } +func moduleForGreyListRemovedApis(ctx android.SingletonContext, module android.Module) bool { + switch ctx.ModuleName(module) { + case "api-stubs-docs", "system-api-stubs-docs", "android.car-stubs-docs", "android.car-system-stubs-docs": + return true + default: + return false + } +} + // flagsRule creates a rule to build hiddenapi-flags.csv out of flags.csv files generated for boot image modules and // the greylists. func flagsRule(ctx android.SingletonContext) android.Path { @@ -222,7 +231,7 @@ func flagsRule(ctx android.SingletonContext) android.Path { // Track @removed public and system APIs via corresponding droidstubs targets. // These APIs are not present in the stubs, however, we have to keep allowing access // to them at runtime. - if m := ctx.ModuleName(module); m == "api-stubs-docs" || m == "system-api-stubs-docs" { + if moduleForGreyListRemovedApis(ctx, module) { greylistRemovedApis = append(greylistRemovedApis, ds.removedDexApiFile) } } -- cgit v1.2.3-59-g8ed1b