diff options
37 files changed, 1384 insertions, 164 deletions
diff --git a/android/filegroup.go b/android/filegroup.go index 7a6cc4ffc..674a196cb 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -17,8 +17,6 @@ package android import ( "android/soong/bazel" "strings" - - "github.com/google/blueprint/proptools" ) func init() { @@ -28,7 +26,6 @@ func init() { // https://docs.bazel.build/versions/master/be/general.html#filegroup type bazelFilegroupAttributes struct { - Name *string Srcs bazel.LabelList } @@ -50,20 +47,19 @@ func (bfg *bazelFilegroup) Name() string { func (bfg *bazelFilegroup) GenerateAndroidBuildActions(ctx ModuleContext) {} -// TODO: Create helper functions to avoid this boilerplate. func FilegroupBp2Build(ctx TopDownMutatorContext) { fg, ok := ctx.Module().(*fileGroup) - if !ok { + if !ok || !fg.properties.Bazel_module.Bp2build_available { return } - name := "__bp2build__" + fg.base().BaseModuleName() - ctx.CreateModule(BazelFileGroupFactory, &bazelFilegroupAttributes{ - Name: proptools.StringPtr(name), + attrs := &bazelFilegroupAttributes{ Srcs: BazelLabelForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs), - }, &bazel.BazelTargetModuleProperties{ - Rule_class: "filegroup", - }) + } + + props := bazel.NewBazelTargetModuleProperties(fg.Name(), "filegroup", "") + + ctx.CreateBazelTargetModule(BazelFileGroupFactory, props, attrs) } type fileGroupProperties struct { diff --git a/android/mutator.go b/android/mutator.go index 15be65f9a..c38719334 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -15,7 +15,10 @@ package android import ( + "android/soong/bazel" + "fmt" "reflect" + "strings" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -275,6 +278,12 @@ type TopDownMutatorContext interface { // CreateModule creates a new module by calling the factory method for the specified moduleType, and applies // the specified property structs to it as if the properties were set in a blueprint file. CreateModule(ModuleFactory, ...interface{}) Module + + // CreateBazelTargetModule creates a BazelTargetModule by calling the + // factory method, just like in CreateModule, but also requires + // BazelTargetModuleProperties containing additional metadata for the + // bp2build codegenerator. + CreateBazelTargetModule(ModuleFactory, bazel.BazelTargetModuleProperties, interface{}) BazelTargetModule } type topDownMutatorContext struct { @@ -502,6 +511,21 @@ func registerDepsMutatorBp2Build(ctx RegisterMutatorsContext) { ctx.BottomUp("deps", depsMutator).Parallel() } +func (t *topDownMutatorContext) CreateBazelTargetModule( + factory ModuleFactory, + bazelProps bazel.BazelTargetModuleProperties, + attrs interface{}) BazelTargetModule { + if !strings.HasPrefix(*bazelProps.Name, bazel.BazelTargetModuleNamePrefix) { + panic(fmt.Errorf( + "bp2build error: the bazel target module name must start with '%s': %s", + bazel.BazelTargetModuleNamePrefix, + *bazelProps.Name, + )) + } + + return t.CreateModule(factory, &bazelProps, attrs).(BazelTargetModule) +} + func (t *topDownMutatorContext) AppendProperties(props ...interface{}) { for _, p := range props { err := proptools.AppendMatchingProperties(t.Module().base().customizableProperties, diff --git a/android/variable.go b/android/variable.go index 9b3ed1765..799369d33 100644 --- a/android/variable.go +++ b/android/variable.go @@ -121,10 +121,6 @@ type variableProperties struct { Cppflags []string } - Use_lmkd_stats_log struct { - Cflags []string - } - Arc struct { Cflags []string `android:"arch_variant"` Exclude_srcs []string `android:"arch_variant"` @@ -240,7 +236,6 @@ type productVariables struct { Treble_linker_namespaces *bool `json:",omitempty"` Enforce_vintf_manifest *bool `json:",omitempty"` Uml *bool `json:",omitempty"` - Use_lmkd_stats_log *bool `json:",omitempty"` Arc *bool `json:",omitempty"` MinimizeJavaDebugInfo *bool `json:",omitempty"` diff --git a/apex/apex_test.go b/apex/apex_test.go index 85d62590e..83eb56a41 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -4362,7 +4362,7 @@ func TestPrebuiltExportDexImplementationJars(t *testing.T) { // Empty transformation. } - checkDexJarBuildPath := func(ctx *android.TestContext, name string) { + checkDexJarBuildPath := func(t *testing.T, ctx *android.TestContext, name string) { // Make sure the import has been given the correct path to the dex jar. p := ctx.ModuleForTests(name, "android_common_myapex").Module().(java.Dependency) dexJarBuildPath := p.DexJarBuildPath() @@ -4371,7 +4371,7 @@ func TestPrebuiltExportDexImplementationJars(t *testing.T) { } } - ensureNoSourceVariant := func(ctx *android.TestContext) { + ensureNoSourceVariant := func(t *testing.T, ctx *android.TestContext) { // Make sure that an apex variant is not created for the source module. if expected, actual := []string{"android_common"}, ctx.ModuleVariantsForTests("libfoo"); !reflect.DeepEqual(expected, actual) { t.Errorf("invalid set of variants for %q: expected %q, found %q", "libfoo", expected, actual) @@ -4402,7 +4402,7 @@ func TestPrebuiltExportDexImplementationJars(t *testing.T) { // Make sure that dexpreopt can access dex implementation files from the prebuilt. ctx := testDexpreoptWithApexes(t, bp, "", transform) - checkDexJarBuildPath(ctx, "libfoo") + checkDexJarBuildPath(t, ctx, "libfoo") }) t.Run("prebuilt with source preferred", func(t *testing.T) { @@ -4434,8 +4434,8 @@ func TestPrebuiltExportDexImplementationJars(t *testing.T) { // Make sure that dexpreopt can access dex implementation files from the prebuilt. ctx := testDexpreoptWithApexes(t, bp, "", transform) - checkDexJarBuildPath(ctx, "prebuilt_libfoo") - ensureNoSourceVariant(ctx) + checkDexJarBuildPath(t, ctx, "prebuilt_libfoo") + ensureNoSourceVariant(t, ctx) }) t.Run("prebuilt preferred with source", func(t *testing.T) { @@ -4467,8 +4467,8 @@ func TestPrebuiltExportDexImplementationJars(t *testing.T) { // Make sure that dexpreopt can access dex implementation files from the prebuilt. ctx := testDexpreoptWithApexes(t, bp, "", transform) - checkDexJarBuildPath(ctx, "prebuilt_libfoo") - ensureNoSourceVariant(ctx) + checkDexJarBuildPath(t, ctx, "prebuilt_libfoo") + ensureNoSourceVariant(t, ctx) }) } @@ -4477,7 +4477,7 @@ func TestBootDexJarsFromSourcesAndPrebuilts(t *testing.T) { config.BootJars = android.CreateTestConfiguredJarList([]string{"myapex:libfoo"}) } - checkBootDexJarPath := func(ctx *android.TestContext, bootDexJarPath string) { + checkBootDexJarPath := func(t *testing.T, ctx *android.TestContext, bootDexJarPath string) { s := ctx.SingletonForTests("dex_bootjars") foundLibfooJar := false for _, output := range s.AllOutputs() { @@ -4495,6 +4495,12 @@ func TestBootDexJarsFromSourcesAndPrebuilts(t *testing.T) { } } + checkHiddenAPIIndexInputs := func(t *testing.T, ctx *android.TestContext, expectedInputs string) { + hiddenAPIIndex := ctx.SingletonForTests("hiddenapi_index") + indexRule := hiddenAPIIndex.Rule("singleton-merged-hiddenapi-index") + java.CheckHiddenAPIRuleInputs(t, expectedInputs, indexRule) + } + t.Run("prebuilt only", func(t *testing.T) { bp := ` prebuilt_apex { @@ -4518,7 +4524,12 @@ func TestBootDexJarsFromSourcesAndPrebuilts(t *testing.T) { ` ctx := testDexpreoptWithApexes(t, bp, "", transform) - checkBootDexJarPath(ctx, ".intermediates/myapex.deapexer/android_common/deapexer/javalib/libfoo.jar") + checkBootDexJarPath(t, ctx, ".intermediates/myapex.deapexer/android_common/deapexer/javalib/libfoo.jar") + + // Make sure that the dex file from the prebuilt_apex contributes to the hiddenapi index file. + checkHiddenAPIIndexInputs(t, ctx, ` +.intermediates/libfoo/android_common_myapex/hiddenapi/index.csv +`) }) t.Run("prebuilt with source library preferred", func(t *testing.T) { @@ -4587,7 +4598,12 @@ func TestBootDexJarsFromSourcesAndPrebuilts(t *testing.T) { ` ctx := testDexpreoptWithApexes(t, bp, "", transform) - checkBootDexJarPath(ctx, ".intermediates/myapex.deapexer/android_common/deapexer/javalib/libfoo.jar") + checkBootDexJarPath(t, ctx, ".intermediates/myapex.deapexer/android_common/deapexer/javalib/libfoo.jar") + + // Make sure that the dex file from the prebuilt_apex contributes to the hiddenapi index file. + checkHiddenAPIIndexInputs(t, ctx, ` +.intermediates/prebuilt_libfoo/android_common_myapex/hiddenapi/index.csv +`) }) t.Run("prebuilt with source apex preferred", func(t *testing.T) { @@ -4631,7 +4647,12 @@ func TestBootDexJarsFromSourcesAndPrebuilts(t *testing.T) { ` ctx := testDexpreoptWithApexes(t, bp, "", transform) - checkBootDexJarPath(ctx, ".intermediates/libfoo/android_common_apex10000/aligned/libfoo.jar") + checkBootDexJarPath(t, ctx, ".intermediates/libfoo/android_common_apex10000/hiddenapi/libfoo.jar") + + // Make sure that the dex file from the prebuilt_apex contributes to the hiddenapi index file. + checkHiddenAPIIndexInputs(t, ctx, ` +.intermediates/libfoo/android_common_apex10000/hiddenapi/index.csv +`) }) t.Run("prebuilt preferred with source apex disabled", func(t *testing.T) { @@ -4677,7 +4698,12 @@ func TestBootDexJarsFromSourcesAndPrebuilts(t *testing.T) { ` ctx := testDexpreoptWithApexes(t, bp, "", transform) - checkBootDexJarPath(ctx, ".intermediates/myapex.deapexer/android_common/deapexer/javalib/libfoo.jar") + checkBootDexJarPath(t, ctx, ".intermediates/myapex.deapexer/android_common/deapexer/javalib/libfoo.jar") + + // Make sure that the dex file from the prebuilt_apex contributes to the hiddenapi index file. + checkHiddenAPIIndexInputs(t, ctx, ` +.intermediates/prebuilt_libfoo/android_common_prebuilt_myapex/hiddenapi/index.csv +`) }) } @@ -6304,6 +6330,7 @@ func testDexpreoptWithApexes(t *testing.T, bp, errmsg string, transformDexpreopt android.RegisterPrebuiltMutators(ctx) cc.RegisterRequiredBuildComponentsForTest(ctx) java.RegisterRequiredBuildComponentsForTest(ctx) + java.RegisterHiddenApiSingletonComponents(ctx) ctx.PostDepsMutators(android.RegisterOverridePostDepsMutators) ctx.PreDepsMutators(RegisterPreDepsMutators) ctx.PostDepsMutators(RegisterPostDepsMutators) @@ -6315,6 +6342,11 @@ func testDexpreoptWithApexes(t *testing.T, bp, errmsg string, transformDexpreopt transformDexpreoptConfig(dexpreoptConfig) dexpreopt.SetTestGlobalConfig(config, dexpreoptConfig) + // Make sure that any changes to these dexpreopt properties are mirrored in the corresponding + // product variables. + config.TestProductVariables.BootJars = dexpreoptConfig.BootJars + config.TestProductVariables.UpdatableBootJars = dexpreoptConfig.UpdatableBootJars + _, errs := ctx.ParseBlueprintsFiles("Android.bp") android.FailIfErrored(t, errs) diff --git a/apex/builder.go b/apex/builder.go index 16ca74cf2..2663a679f 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -700,15 +700,20 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { }) a.apisUsedByModuleFile = apisUsedbyOutputFile + var libNames []string + for _, f := range a.filesInfo { + if f.class == nativeSharedLib { + libNames = append(libNames, f.stem()) + } + } apisBackedbyOutputFile := android.PathForModuleOut(ctx, a.Name()+"_backing.txt") ndkLibraryList := android.PathForSource(ctx, "system/core/rootdir/etc/public.libraries.android.txt") rule := android.NewRuleBuilder(pctx, ctx) rule.Command(). Tool(android.PathForSource(ctx, "build/soong/scripts/gen_ndk_backedby_apex.sh")). - Text(imageDir.String()). - Implicits(implicitInputs). Output(apisBackedbyOutputFile). - Input(ndkLibraryList) + Input(ndkLibraryList). + Flags(libNames) rule.Build("ndk_backedby_list", "Generate API libraries backed by Apex") a.apisBackedByModuleFile = apisBackedbyOutputFile diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 314995205..041afb33a 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -260,12 +260,36 @@ func prebuiltSelectSourceMutator(ctx android.BottomUpMutatorContext) { } } +type exportedDependencyTag struct { + blueprint.BaseDependencyTag + name string +} + +// Mark this tag so dependencies that use it are excluded from visibility enforcement. +// +// This does allow any prebuilt_apex to reference any module which does open up a small window for +// restricted visibility modules to be referenced from the wrong prebuilt_apex. However, doing so +// avoids opening up a much bigger window by widening the visibility of modules that need files +// provided by the prebuilt_apex to include all the possible locations they may be defined, which +// could include everything below vendor/. +// +// A prebuilt_apex that references a module via this tag will have to contain the appropriate files +// corresponding to that module, otherwise it will fail when attempting to retrieve the files from +// the .apex file. It will also have to be included in the module's apex_available property too. +// That makes it highly unlikely that a prebuilt_apex would reference a restricted module +// incorrectly. +func (t exportedDependencyTag) ExcludeFromVisibilityEnforcement() {} + +var ( + exportedJavaLibTag = exportedDependencyTag{name: "exported_java_lib"} +) + func (p *Prebuilt) DepsMutator(ctx android.BottomUpMutatorContext) { // Add dependencies onto the java modules that represent the java libraries that are provided by // and exported from this prebuilt apex. for _, lib := range p.properties.Exported_java_libs { dep := prebuiltApexExportedModuleName(ctx, lib) - ctx.AddFarVariationDependencies(ctx.Config().AndroidCommonTarget.Variations(), javaLibTag, dep) + ctx.AddFarVariationDependencies(ctx.Config().AndroidCommonTarget.Variations(), exportedJavaLibTag, dep) } } @@ -305,7 +329,7 @@ func (p *Prebuilt) ApexInfoMutator(mctx android.TopDownMutatorContext) { var dependencies []android.ApexModule mctx.VisitDirectDeps(func(m android.Module) { tag := mctx.OtherModuleDependencyTag(m) - if tag == javaLibTag { + if tag == exportedJavaLibTag { depName := mctx.OtherModuleName(m) // It is an error if the other module is not a prebuilt. diff --git a/bazel/properties.go b/bazel/properties.go index 5b98d159d..8055306b2 100644 --- a/bazel/properties.go +++ b/bazel/properties.go @@ -14,9 +14,17 @@ package bazel +import ( + "fmt" + "strings" +) + type bazelModuleProperties struct { // The label of the Bazel target replacing this Soong module. Label string + + // If true, bp2build will generate the converted Bazel target for this module. + Bp2build_available bool } // Properties contains common module properties for Bazel migration purposes. @@ -29,6 +37,8 @@ type Properties struct { // BazelTargetModuleProperties contain properties and metadata used for // Blueprint to BUILD file conversion. type BazelTargetModuleProperties struct { + Name *string + // The Bazel rule class for this target. Rule_class string @@ -36,6 +46,23 @@ type BazelTargetModuleProperties struct { Bzl_load_location string } +const BazelTargetModuleNamePrefix = "__bp2build__" + +func NewBazelTargetModuleProperties(name string, ruleClass string, bzlLoadLocation string) BazelTargetModuleProperties { + if strings.HasPrefix(name, BazelTargetModuleNamePrefix) { + panic(fmt.Errorf( + "The %s name prefix is added automatically, do not set it manually: %s", + BazelTargetModuleNamePrefix, + name)) + } + name = BazelTargetModuleNamePrefix + name + return BazelTargetModuleProperties{ + Name: &name, + Rule_class: ruleClass, + Bzl_load_location: bzlLoadLocation, + } +} + // Label is used to represent a Bazel compatible Label. Also stores the original bp text to support // string replacement. type Label struct { diff --git a/bp2build/Android.bp b/bp2build/Android.bp index 2bbe4b53c..54fc93f3b 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -12,6 +12,7 @@ bootstrap_go_package { "soong-android", "soong-bazel", "soong-genrule", + "soong-sh", ], testSrcs: [ "build_conversion_test.go", diff --git a/bp2build/build_conversion.go b/bp2build/build_conversion.go index 2c293eae7..7ffcfa4a3 100644 --- a/bp2build/build_conversion.go +++ b/bp2build/build_conversion.go @@ -173,6 +173,12 @@ func GenerateBazelTargets(ctx bpToBuildContext, codegenMode CodegenMode) map[str } t = generateBazelTarget(ctx, m) case QueryView: + // Blocklist certain module types from being generated. + if canonicalizeModuleType(ctx.ModuleType(m)) == "package" { + // package module name contain slashes, and thus cannot + // be mapped cleanly to a bazel label. + return + } t = generateSoongModuleTarget(ctx, m) default: panic(fmt.Errorf("Unknown code-generation mode: %s", codegenMode)) @@ -213,6 +219,8 @@ func generateBazelTarget(ctx bpToBuildContext, m blueprint.Module) BazelTarget { // Delete it from being generated in the BUILD file. delete(props.Attrs, "bzl_load_location") + delete(props.Attrs, "bp2build_available") + // Return the Bazel target with rule class and attributes, ready to be // code-generated. attributes := propsToAttributes(props.Attrs) @@ -461,7 +469,7 @@ func makeIndent(indent int) string { } func targetNameForBp2Build(c bpToBuildContext, logicModule blueprint.Module) string { - return strings.Replace(c.ModuleName(logicModule), "__bp2build__", "", 1) + return strings.Replace(c.ModuleName(logicModule), bazel.BazelTargetModuleNamePrefix, "", 1) } func targetNameWithVariant(c bpToBuildContext, logicModule blueprint.Module) string { diff --git a/bp2build/build_conversion_test.go b/bp2build/build_conversion_test.go index 081b0e564..df554a09a 100644 --- a/bp2build/build_conversion_test.go +++ b/bp2build/build_conversion_test.go @@ -17,6 +17,7 @@ package bp2build import ( "android/soong/android" "android/soong/genrule" + "android/soong/sh" "strings" "testing" ) @@ -228,6 +229,7 @@ func TestGenerateBazelTargetModules(t *testing.T) { name: "foo", string_list_prop: ["a", "b"], string_prop: "a", + bazel_module: { bp2build_available: true }, }`, expectedBazelTarget: `custom( name = "foo", @@ -356,6 +358,12 @@ load("//build/bazel/rules:java.bzl", "java_binary")`, ruleClass: "genrule", // Note: no bzlLoadLocation for native rules }, + BazelTarget{ + name: "sh_binary_target", + ruleClass: "sh_binary", + // Note: no bzlLoadLocation for native rules + // TODO(ruperts): Could open source the existing, experimental Starlark sh_ rules? + }, }, expectedLoadStatements: `load("//build/bazel/rules:cc.bzl", "cc_binary") load("//build/bazel/rules:java.bzl", "java_binary")`, @@ -382,6 +390,7 @@ func TestGenerateBazelTargetModules_OneToMany_LoadedFromStarlark(t *testing.T) { { bp: `custom { name: "bar", + bazel_module: { bp2build_available: true }, }`, expectedBazelTarget: `my_library( name = "bar", @@ -474,6 +483,7 @@ genrule { bp: `filegroup { name: "fg_foo", srcs: [], + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{ `filegroup( @@ -491,6 +501,7 @@ genrule { bp: `filegroup { name: "fg_foo", srcs: ["a", "b"], + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{`filegroup( name = "fg_foo", @@ -510,6 +521,7 @@ genrule { name: "fg_foo", srcs: ["a", "b"], exclude_srcs: ["a"], + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{`filegroup( name = "fg_foo", @@ -527,6 +539,7 @@ genrule { bp: `filegroup { name: "foo", srcs: ["**/*.txt"], + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{`filegroup( name = "foo", @@ -552,6 +565,7 @@ genrule { bp: `filegroup { name: "foo", srcs: ["a.txt"], + bazel_module: { bp2build_available: true }, }`, dir: "other", expectedBazelTargets: []string{`filegroup( @@ -567,6 +581,7 @@ genrule { "other/Android.bp": `filegroup { name: "fg_foo", srcs: ["**/*.txt"], + bazel_module: { bp2build_available: true }, }`, "other/a.txt": "", "other/b.txt": "", @@ -585,6 +600,7 @@ genrule { ":foo", "c", ], + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{`filegroup( name = "foobar", @@ -612,6 +628,7 @@ genrule { out: ["foo_tool.out"], srcs: ["foo_tool.in"], cmd: "cp $(in) $(out)", + bazel_module: { bp2build_available: true }, } genrule { @@ -620,6 +637,7 @@ genrule { srcs: ["foo.in"], tools: [":foo.tool"], cmd: "$(location :foo.tool) --genDir=$(genDir) arg $(in) $(out)", + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{ `genrule( @@ -658,7 +676,8 @@ genrule { out: ["foo_tool.out", "foo_tool2.out"], srcs: ["foo_tool.in"], cmd: "cp $(in) $(out)", - } + bazel_module: { bp2build_available: true }, +} genrule { name: "foo", @@ -666,6 +685,7 @@ genrule { srcs: ["foo.in"], tools: [":foo.tools"], cmd: "$(locations :foo.tools) -s $(out) $(in)", + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{`genrule( name = "foo", @@ -705,6 +725,7 @@ genrule { srcs: ["foo.in"], tool_files: [":foo.tool"], cmd: "$(locations :foo.tool) -s $(out) $(in)", + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{`genrule( name = "foo", @@ -734,6 +755,7 @@ genrule { srcs: [":other.tool"], tool_files: [":foo.tool"], cmd: "$(locations :foo.tool) -s $(out) $(location :other.tool)", + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{`genrule( name = "foo", @@ -763,6 +785,7 @@ genrule { srcs: ["foo.in"], tool_files: [":foo.tool", ":other.tool"], cmd: "$(location) -s $(out) $(in)", + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{`genrule( name = "foo", @@ -793,6 +816,7 @@ genrule { srcs: ["foo.in"], tools: [":foo.tool", ":other.tool"], cmd: "$(locations) -s $(out) $(in)", + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{`genrule( name = "foo", @@ -822,6 +846,7 @@ genrule { out: ["foo.out"], srcs: ["foo.in"], cmd: "cp $(in) $(out)", + bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{`genrule( name = "foo", @@ -835,6 +860,23 @@ genrule { )`, }, }, + { + description: "sh_binary test", + moduleTypeUnderTest: "sh_binary", + moduleTypeUnderTestFactory: sh.ShBinaryFactory, + moduleTypeUnderTestBp2BuildMutator: sh.ShBinaryBp2Build, + bp: `sh_binary { + name: "foo", + src: "foo.sh", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{`sh_binary( + name = "foo", + srcs = [ + "foo.sh", + ], +)`}, + }, } dir := "." @@ -927,6 +969,7 @@ genrule { out: ["out"], srcs: ["in1"], defaults: ["gen_defaults"], + bazel_module: { bp2build_available: true }, } `, expectedBazelTarget: `genrule( @@ -961,6 +1004,7 @@ genrule { srcs: ["in1"], defaults: ["gen_defaults"], cmd: "do-something $(in) $(out)", + bazel_module: { bp2build_available: true }, } `, expectedBazelTarget: `genrule( @@ -999,6 +1043,7 @@ genrule { name: "gen", out: ["out"], defaults: ["gen_defaults1", "gen_defaults2"], + bazel_module: { bp2build_available: true }, } `, expectedBazelTarget: `genrule( @@ -1045,6 +1090,7 @@ genrule { name: "gen", out: ["out"], defaults: ["gen_defaults1"], + bazel_module: { bp2build_available: true }, } `, expectedBazelTarget: `genrule( @@ -1097,3 +1143,80 @@ genrule { } } } + +func TestAllowlistingBp2buildTargets(t *testing.T) { + testCases := []struct { + moduleTypeUnderTest string + moduleTypeUnderTestFactory android.ModuleFactory + moduleTypeUnderTestBp2BuildMutator bp2buildMutator + bp string + expectedCount int + description string + }{ + { + description: "explicitly unavailable", + moduleTypeUnderTest: "filegroup", + moduleTypeUnderTestFactory: android.FileGroupFactory, + moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, + bp: `filegroup { + name: "foo", + srcs: ["a", "b"], + bazel_module: { bp2build_available: false }, +}`, + expectedCount: 0, + }, + { + description: "implicitly unavailable", + moduleTypeUnderTest: "filegroup", + moduleTypeUnderTestFactory: android.FileGroupFactory, + moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, + bp: `filegroup { + name: "foo", + srcs: ["a", "b"], +}`, + expectedCount: 0, + }, + { + description: "explicitly available", + moduleTypeUnderTest: "filegroup", + moduleTypeUnderTestFactory: android.FileGroupFactory, + moduleTypeUnderTestBp2BuildMutator: android.FilegroupBp2Build, + bp: `filegroup { + name: "foo", + srcs: ["a", "b"], + bazel_module: { bp2build_available: true }, +}`, + expectedCount: 1, + }, + { + description: "generates more than 1 target if needed", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactory, + moduleTypeUnderTestBp2BuildMutator: customBp2BuildMutatorFromStarlark, + bp: `custom { + name: "foo", + bazel_module: { bp2build_available: true }, +}`, + expectedCount: 3, + }, + } + + dir := "." + for _, testCase := range testCases { + config := android.TestConfig(buildDir, nil, testCase.bp, nil) + ctx := android.NewTestContext(config) + ctx.RegisterModuleType(testCase.moduleTypeUnderTest, testCase.moduleTypeUnderTestFactory) + ctx.RegisterBp2BuildMutator(testCase.moduleTypeUnderTest, testCase.moduleTypeUnderTestBp2BuildMutator) + ctx.RegisterForBazelConversion() + + _, errs := ctx.ParseFileList(dir, []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.ResolveDependencies(config) + android.FailIfErrored(t, errs) + + bazelTargets := GenerateBazelTargets(ctx.Context.Context, Bp2Build)[dir] + if actualCount := len(bazelTargets); actualCount != testCase.expectedCount { + t.Fatalf("%s: Expected %d bazel target, got %d", testCase.description, testCase.expectedCount, actualCount) + } + } +} diff --git a/bp2build/bzl_conversion_test.go b/bp2build/bzl_conversion_test.go index f2a405854..30c1a5b6a 100644 --- a/bp2build/bzl_conversion_test.go +++ b/bp2build/bzl_conversion_test.go @@ -86,6 +86,10 @@ custom = rule( "soong_module_name": attr.string(mandatory = True), "soong_module_variant": attr.string(), "soong_module_deps": attr.label_list(providers = [SoongModuleInfo]), + # bazel_module start +# "label": attr.string(), +# "bp2build_available": attr.bool(), + # bazel_module end "bool_prop": attr.bool(), "bool_ptr_prop": attr.bool(), "int64_ptr_prop": attr.int(), diff --git a/bp2build/conversion.go b/bp2build/conversion.go index 62cd8d4b6..081e08243 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -50,14 +50,19 @@ func createBuildFiles(buildToTargets map[string]BazelTargets, mode CodegenMode) sort.Slice(targets, func(i, j int) bool { return targets[i].name < targets[j].name }) content := soongModuleLoad if mode == Bp2Build { - content = targets.LoadStatements() + content = `# This file was automatically generated by bp2build for the Bazel migration project. +# Feel free to edit or test it, but do *not* check it into your version control system.` + content += "\n\n" + content += "package(default_visibility = [\"//visibility:public\"])" + content += "\n\n" + content += targets.LoadStatements() } if content != "" { // If there are load statements, add a couple of newlines. content += "\n\n" } content += targets.String() - files = append(files, newFile(dir, "BUILD.bazel", content)) + files = append(files, newFile(dir, "BUILD", content)) } return files } diff --git a/bp2build/testing.go b/bp2build/testing.go index 5e6481b32..2e59999c2 100644 --- a/bp2build/testing.go +++ b/bp2build/testing.go @@ -3,8 +3,6 @@ package bp2build import ( "android/soong/android" "android/soong/bazel" - - "github.com/google/blueprint/proptools" ) type nestedProps struct { @@ -29,6 +27,8 @@ type customModule struct { android.ModuleBase props customProps + + bazelProps bazel.Properties } // OutputFiles is needed because some instances of this module use dist with a @@ -44,6 +44,7 @@ func (m *customModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { func customModuleFactoryBase() android.Module { module := &customModule{} module.AddProperties(&module.props) + module.AddProperties(&module.bazelProps) return module } @@ -105,7 +106,6 @@ func customDefaultsModuleFactory() android.Module { } type customBazelModuleAttributes struct { - Name *string String_prop string String_list_prop []string } @@ -127,14 +127,18 @@ func (m *customBazelModule) GenerateAndroidBuildActions(ctx android.ModuleContex func customBp2BuildMutator(ctx android.TopDownMutatorContext) { if m, ok := ctx.Module().(*customModule); ok { - name := "__bp2build__" + m.Name() - ctx.CreateModule(customBazelModuleFactory, &customBazelModuleAttributes{ - Name: proptools.StringPtr(name), + if !m.bazelProps.Bazel_module.Bp2build_available { + return + } + + attrs := &customBazelModuleAttributes{ String_prop: m.props.String_prop, String_list_prop: m.props.String_list_prop, - }, &bazel.BazelTargetModuleProperties{ - Rule_class: "custom", - }) + } + + props := bazel.NewBazelTargetModuleProperties(m.Name(), "custom", "") + + ctx.CreateBazelTargetModule(customBazelModuleFactory, props, attrs) } } @@ -142,24 +146,32 @@ func customBp2BuildMutator(ctx android.TopDownMutatorContext) { // module to target. func customBp2BuildMutatorFromStarlark(ctx android.TopDownMutatorContext) { if m, ok := ctx.Module().(*customModule); ok { - baseName := "__bp2build__" + m.Name() - ctx.CreateModule(customBazelModuleFactory, &customBazelModuleAttributes{ - Name: proptools.StringPtr(baseName), - }, &bazel.BazelTargetModuleProperties{ - Rule_class: "my_library", - Bzl_load_location: "//build/bazel/rules:rules.bzl", - }) - ctx.CreateModule(customBazelModuleFactory, &customBazelModuleAttributes{ - Name: proptools.StringPtr(baseName + "_proto_library_deps"), - }, &bazel.BazelTargetModuleProperties{ - Rule_class: "proto_library", - Bzl_load_location: "//build/bazel/rules:proto.bzl", - }) - ctx.CreateModule(customBazelModuleFactory, &customBazelModuleAttributes{ - Name: proptools.StringPtr(baseName + "_my_proto_library_deps"), - }, &bazel.BazelTargetModuleProperties{ - Rule_class: "my_proto_library", - Bzl_load_location: "//build/bazel/rules:proto.bzl", - }) + if !m.bazelProps.Bazel_module.Bp2build_available { + return + } + + baseName := m.Name() + attrs := &customBazelModuleAttributes{} + + myLibraryProps := bazel.NewBazelTargetModuleProperties( + baseName, + "my_library", + "//build/bazel/rules:rules.bzl", + ) + ctx.CreateBazelTargetModule(customBazelModuleFactory, myLibraryProps, attrs) + + protoLibraryProps := bazel.NewBazelTargetModuleProperties( + baseName+"_proto_library_deps", + "proto_library", + "//build/bazel/rules:proto.bzl", + ) + ctx.CreateBazelTargetModule(customBazelModuleFactory, protoLibraryProps, attrs) + + myProtoLibraryProps := bazel.NewBazelTargetModuleProperties( + baseName+"_my_proto_library_deps", + "my_proto_library", + "//build/bazel/rules:proto.bzl", + ) + ctx.CreateBazelTargetModule(customBazelModuleFactory, myProtoLibraryProps, attrs) } } diff --git a/cc/androidmk.go b/cc/androidmk.go index ddb81d99b..8652c1042 100644 --- a/cc/androidmk.go +++ b/cc/androidmk.go @@ -26,9 +26,9 @@ import ( var ( nativeBridgeSuffix = ".native_bridge" productSuffix = ".product" - vendorSuffix = ".vendor" + VendorSuffix = ".vendor" ramdiskSuffix = ".ramdisk" - vendorRamdiskSuffix = ".vendor_ramdisk" + VendorRamdiskSuffix = ".vendor_ramdisk" recoverySuffix = ".recovery" sdkSuffix = ".sdk" ) @@ -1544,7 +1544,7 @@ func (c *Module) getNameSuffixWithVndkVersion(ctx android.ModuleContext) string nameSuffix = productSuffix } else { vndkVersion = ctx.DeviceConfig().VndkVersion() - nameSuffix = vendorSuffix + nameSuffix = VendorSuffix } if vndkVersion == "current" { vndkVersion = ctx.DeviceConfig().PlatformVndkVersion() @@ -1591,11 +1591,11 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { } else if _, ok := c.linker.(*vndkPrebuiltLibraryDecorator); ok { // .vendor suffix is added for backward compatibility with VNDK snapshot whose names with // such suffixes are already hard-coded in prebuilts/vndk/.../Android.bp. - c.Properties.SubName += vendorSuffix + c.Properties.SubName += VendorSuffix } else if c.InRamdisk() && !c.OnlyInRamdisk() { c.Properties.SubName += ramdiskSuffix } else if c.InVendorRamdisk() && !c.OnlyInVendorRamdisk() { - c.Properties.SubName += vendorRamdiskSuffix + c.Properties.SubName += VendorRamdiskSuffix } else if c.InRecovery() && !c.OnlyInRecovery() { c.Properties.SubName += recoverySuffix } else if c.IsSdkVariant() && (c.Properties.SdkAndPlatformVariantVisibleToMake || c.SplitPerApiLevel()) { @@ -2927,7 +2927,7 @@ func (c *Module) makeLibName(ctx android.ModuleContext, ccDep LinkableInterface, } else if ccDep.InRamdisk() && !ccDep.OnlyInRamdisk() { return libName + ramdiskSuffix } else if ccDep.InVendorRamdisk() && !ccDep.OnlyInVendorRamdisk() { - return libName + vendorRamdiskSuffix + return libName + VendorRamdiskSuffix } else if ccDep.InRecovery() && !ccDep.OnlyInRecovery() { return libName + recoverySuffix } else if ccDep.Target().NativeBridge == android.NativeBridgeEnabled { diff --git a/cc/snapshot_prebuilt.go b/cc/snapshot_prebuilt.go index ffaed8e55..b82628ccb 100644 --- a/cc/snapshot_prebuilt.go +++ b/cc/snapshot_prebuilt.go @@ -144,7 +144,7 @@ func (vendorSnapshotImage) imageVariantName(cfg android.DeviceConfig) string { } func (vendorSnapshotImage) moduleNameSuffix() string { - return vendorSuffix + return VendorSuffix } func (recoverySnapshotImage) init(ctx android.RegistrationContext) { diff --git a/cc/testing.go b/cc/testing.go index 3a5bd1762..45e531208 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -43,6 +43,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { name: "libatomic", defaults: ["linux_bionic_supported"], vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, native_bridge_supported: true, @@ -52,6 +53,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { toolchain_library { name: "libcompiler_rt-extras", vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, src: "", @@ -60,6 +62,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { toolchain_library { name: "libclang_rt.builtins-arm-android", vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, native_bridge_supported: true, @@ -69,6 +72,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { toolchain_library { name: "libclang_rt.builtins-aarch64-android", vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, native_bridge_supported: true, @@ -93,6 +97,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { toolchain_library { name: "libclang_rt.builtins-i686-android", vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, native_bridge_supported: true, @@ -103,6 +108,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { name: "libclang_rt.builtins-x86_64-android", defaults: ["linux_bionic_supported"], vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, native_bridge_supported: true, @@ -113,6 +119,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { name: "libunwind", defaults: ["linux_bionic_supported"], vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, native_bridge_supported: true, @@ -238,6 +245,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { cc_library { name: "libprofile-extras", vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, native_coverage: false, @@ -248,6 +256,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { cc_library { name: "libprofile-clang-extras", vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, native_coverage: false, @@ -319,6 +328,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { system_shared_libs: [], stl: "none", vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, host_supported: true, @@ -356,6 +366,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { stl: "none", host_supported: false, vendor_available: true, + vendor_ramdisk_available: true, product_available: true, recovery_available: true, min_sdk_version: "29", @@ -380,6 +391,7 @@ func GatherRequiredDepsForTest(oses ...android.OsType) string { defaults: ["linux_bionic_supported"], recovery_available: true, vendor_available: true, + vendor_ramdisk_available: true, product_available: true, native_bridge_supported: true, stl: "none", diff --git a/genrule/genrule.go b/genrule/genrule.go index 62aa7f834..9fa6c484f 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -780,7 +780,6 @@ type genRuleProperties struct { } type bazelGenruleAttributes struct { - Name *string Srcs bazel.LabelList Outs []string Tools bazel.LabelList @@ -801,10 +800,10 @@ func BazelGenruleFactory() android.Module { func GenruleBp2Build(ctx android.TopDownMutatorContext) { m, ok := ctx.Module().(*Module) - if !ok { + if !ok || !m.properties.Bazel_module.Bp2build_available { return } - name := "__bp2build__" + m.Name() + // Bazel only has the "tools" attribute. tools := android.BazelLabelForModuleDeps(ctx, m.properties.Tools) tool_files := android.BazelLabelForModuleSrc(ctx, m.properties.Tool_files) @@ -847,16 +846,17 @@ func GenruleBp2Build(ctx android.TopDownMutatorContext) { } } - // Create the BazelTargetModule. - ctx.CreateModule(BazelGenruleFactory, &bazelGenruleAttributes{ - Name: proptools.StringPtr(name), + attrs := &bazelGenruleAttributes{ Srcs: srcs, Outs: outs, Cmd: cmd, Tools: tools, - }, &bazel.BazelTargetModuleProperties{ - Rule_class: "genrule", - }) + } + + props := bazel.NewBazelTargetModuleProperties(m.Name(), "genrule", "") + + // Create the BazelTargetModule. + ctx.CreateBazelTargetModule(BazelGenruleFactory, props, attrs) } func (m *bazelGenrule) Name() string { diff --git a/java/androidmk.go b/java/androidmk.go index 21f3012a4..6e7c437ab 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -646,6 +646,9 @@ func (a *AndroidAppImport) AndroidMkEntries() []android.AndroidMkEntries { entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", a.dexpreopter.builtInstalled) } entries.AddStrings("LOCAL_INSTALLED_MODULE_STEM", a.installPath.Rel()) + if Bool(a.properties.Export_package_resources) { + entries.SetPath("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE", a.outputFile) + } }, }, }} diff --git a/java/app_import.go b/java/app_import.go index 6f21bfbbf..59eb10a9b 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -92,6 +92,10 @@ type AndroidAppImportProperties struct { // Optional name for the installed app. If unspecified, it is derived from the module name. Filename *string + + // If set, create package-export.apk, which other packages can + // use to get PRODUCT-agnostic resource data like IDs and type definitions. + Export_package_resources *bool } func (a *AndroidAppImport) IsInstallable() bool { @@ -142,13 +146,17 @@ func MergePropertiesFromVariant(ctx android.EarlyModuleContext, } } +func (a *AndroidAppImport) isPrebuiltFrameworkRes() bool { + return a.Name() == "prebuilt_framework-res" +} + func (a *AndroidAppImport) DepsMutator(ctx android.BottomUpMutatorContext) { cert := android.SrcIsModule(String(a.properties.Certificate)) if cert != "" { ctx.AddDependency(ctx.Module(), certificateTag, cert) } - a.usesLibrary.deps(ctx, true) + a.usesLibrary.deps(ctx, !a.isPrebuiltFrameworkRes()) } func (a *AndroidAppImport) uncompressEmbeddedJniLibs( @@ -247,7 +255,12 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext a.uncompressEmbeddedJniLibs(ctx, srcApk, jnisUncompressed.OutputPath) var installDir android.InstallPath - if Bool(a.properties.Privileged) { + + if a.isPrebuiltFrameworkRes() { + // framework-res.apk is installed as system/framework/framework-res.apk + installDir = android.PathForModuleInstall(ctx, "framework") + a.preprocessed = true + } else if Bool(a.properties.Privileged) { installDir = android.PathForModuleInstall(ctx, "priv-app", a.BaseModuleName()) } else if ctx.InstallInTestcases() { installDir = android.PathForModuleInstall(ctx, a.BaseModuleName(), ctx.DeviceConfig().DeviceArch()) @@ -275,7 +288,15 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext // TODO: Handle EXTERNAL // Sign or align the package if package has not been preprocessed - if a.preprocessed { + + if a.isPrebuiltFrameworkRes() { + a.outputFile = srcApk + certificates = processMainCert(a.ModuleBase, String(a.properties.Certificate), certificates, ctx) + if len(certificates) != 1 { + ctx.ModuleErrorf("Unexpected number of certificates were extracted: %q", certificates) + } + a.certificate = certificates[0] + } else if a.preprocessed { a.outputFile = srcApk a.certificate = PresignedCertificate } else if !Bool(a.properties.Presigned) { diff --git a/java/app_import_test.go b/java/app_import_test.go index 344d23b55..d7f69eb36 100644 --- a/java/app_import_test.go +++ b/java/app_import_test.go @@ -393,6 +393,69 @@ func TestAndroidAppImport_overridesDisabledAndroidApp(t *testing.T) { } } +func TestAndroidAppImport_frameworkRes(t *testing.T) { + ctx, config := testJava(t, ` + android_app_import { + name: "framework-res", + certificate: "platform", + apk: "package-res.apk", + prefer: true, + export_package_resources: true, + // Disable dexpreopt and verify_uses_libraries check as the app + // contains no Java code to be dexpreopted. + enforce_uses_libs: false, + dex_preopt: { + enabled: false, + }, + } + `) + + mod := ctx.ModuleForTests("prebuilt_framework-res", "android_common").Module() + a := mod.(*AndroidAppImport) + + if !a.preprocessed { + t.Errorf("prebuilt framework-res is not preprocessed") + } + + expectedInstallPath := buildDir + "/target/product/test_device/system/framework/framework-res.apk" + + if a.dexpreopter.installPath.String() != expectedInstallPath { + t.Errorf("prebuilt framework-res installed to incorrect location, actual: %s, expected: %s", a.dexpreopter.installPath, expectedInstallPath) + + } + + entries := android.AndroidMkEntriesForTest(t, config, "", mod)[0] + + expectedPath := "." + // From apk property above, in the root of the source tree. + expectedPrebuiltModuleFile := "package-res.apk" + // Verify that the apk is preprocessed: The export package is the same + // as the prebuilt. + expectedSoongResourceExportPackage := expectedPrebuiltModuleFile + + actualPath := entries.EntryMap["LOCAL_PATH"] + actualPrebuiltModuleFile := entries.EntryMap["LOCAL_PREBUILT_MODULE_FILE"] + actualSoongResourceExportPackage := entries.EntryMap["LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE"] + + if len(actualPath) != 1 { + t.Errorf("LOCAL_PATH incorrect len %d", len(actualPath)) + } else if actualPath[0] != expectedPath { + t.Errorf("LOCAL_PATH mismatch, actual: %s, expected: %s", actualPath[0], expectedPath) + } + + if len(actualPrebuiltModuleFile) != 1 { + t.Errorf("LOCAL_PREBUILT_MODULE_FILE incorrect len %d", len(actualPrebuiltModuleFile)) + } else if actualPrebuiltModuleFile[0] != expectedPrebuiltModuleFile { + t.Errorf("LOCAL_PREBUILT_MODULE_FILE mismatch, actual: %s, expected: %s", actualPrebuiltModuleFile[0], expectedPrebuiltModuleFile) + } + + if len(actualSoongResourceExportPackage) != 1 { + t.Errorf("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE incorrect len %d", len(actualSoongResourceExportPackage)) + } else if actualSoongResourceExportPackage[0] != expectedSoongResourceExportPackage { + t.Errorf("LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE mismatch, actual: %s, expected: %s", actualSoongResourceExportPackage[0], expectedSoongResourceExportPackage) + } +} + func TestAndroidTestImport(t *testing.T) { ctx, config := testJava(t, ` android_test_import { diff --git a/java/builder.go b/java/builder.go index 995160d0e..22a891ae1 100644 --- a/java/builder.go +++ b/java/builder.go @@ -193,12 +193,19 @@ var ( jarjar = pctx.AndroidStaticRule("jarjar", blueprint.RuleParams{ - Command: "${config.JavaCmd} ${config.JavaVmFlags}" + + Command: "" + + // Jarjar doesn't exit with an error when the rules file contains a syntax error, + // leading to stale or missing files later in the build. Remove the output file + // before running jarjar. + "rm -f ${out} && " + + "${config.JavaCmd} ${config.JavaVmFlags}" + // b/146418363 Enable Android specific jarjar transformer to drop compat annotations // for newly repackaged classes. Dropping @UnsupportedAppUsage on repackaged classes // avoids adding new hiddenapis after jarjar'ing. " -DremoveAndroidCompatAnnotations=true" + - " -jar ${config.JarjarCmd} process $rulesFile $in $out", + " -jar ${config.JarjarCmd} process $rulesFile $in $out && " + + // Turn a missing output file into a ninja error + `[ -e ${out} ] || (echo "Missing output file"; exit 1)`, CommandDeps: []string{"${config.JavaCmd}", "${config.JarjarCmd}", "$rulesFile"}, }, "rulesFile") diff --git a/java/hiddenapi.go b/java/hiddenapi.go index eafbf5df0..069595eb9 100644 --- a/java/hiddenapi.go +++ b/java/hiddenapi.go @@ -28,10 +28,21 @@ var hiddenAPIGenerateCSVRule = pctx.AndroidStaticRule("hiddenAPIGenerateCSV", bl }, "outFlag", "stubAPIFlags") type hiddenAPI struct { + // True if the module containing this structure contributes to the hiddenapi information. + active bool + + // True if the module only contains additional annotations and so does not require hiddenapi + // information to be encoded in its dex file and should not be used to generate the + // hiddenAPISingletonPathsStruct.stubFlags file. + annotationsOnly bool + // The path to the dex jar that is in the boot class path. If this is nil then the associated // module is not a boot jar, but could be one of the <x>-hiddenapi modules that provide additional // annotations for the <x> boot dex jar but which do not actually provide a boot dex jar // themselves. + // + // This must be the path to the unencoded dex jar as the encoded dex jar indirectly depends on + // this file so using the encoded dex jar here would result in a cycle in the ninja rules. bootDexJarPath android.Path // The path to the CSV file that contains mappings from Java signature to various flags derived @@ -89,52 +100,82 @@ type hiddenAPIIntf interface { var _ hiddenAPIIntf = (*hiddenAPI)(nil) -func (h *hiddenAPI) hiddenAPI(ctx android.ModuleContext, name string, primary bool, dexJar android.OutputPath, +// Initialize the hiddenapi structure +func (h *hiddenAPI) initHiddenAPI(ctx android.BaseModuleContext, name string) { + // If hiddenapi processing is disabled treat this as inactive. + if ctx.Config().IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") { + return + } + + // Modules whose names are of the format <x>-hiddenapi provide hiddenapi information for the boot + // jar module <x>. Otherwise, the module provides information for itself. Either way extract the + // name of the boot jar module. + bootJarName := strings.TrimSuffix(name, "-hiddenapi") + + // It is important that hiddenapi information is only gathered for/from modules that are actually + // on the boot jars list because the runtime only enforces access to the hidden API for the + // bootclassloader. If information is gathered for modules not on the list then that will cause + // failures in the CtsHiddenApiBlocklist... tests. + h.active = inList(bootJarName, ctx.Config().BootJars()) + + // If this module has a suffix of -hiddenapi then it only provides additional annotation + // information for a module on the boot jars list. + h.annotationsOnly = strings.HasSuffix(name, "-hiddenapi") +} + +// hiddenAPIExtractAndEncode is called by any module that could contribute to the hiddenapi +// processing. +// +// It ignores any module that has not had initHiddenApi() called on it and which is not in the boot +// jar list. +// +// Otherwise, it generates ninja rules to do the following: +// 1. Extract information needed for hiddenapi processing from the module and output it into CSV +// files. +// 2. Conditionally adds the supplied dex file to the list of files used to generate the +// hiddenAPISingletonPathsStruct.stubsFlag file. +// 3. Conditionally creates a copy of the supplied dex file into which it has encoded the hiddenapi +// flags and returns this instead of the supplied dex jar, otherwise simply returns the supplied +// dex jar. +func (h *hiddenAPI) hiddenAPIExtractAndEncode(ctx android.ModuleContext, name string, primary bool, dexJar android.OutputPath, implementationJar android.Path, uncompressDex bool) android.OutputPath { - if !ctx.Config().IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") { - - // Modules whose names are of the format <x>-hiddenapi provide hiddenapi information - // for the boot jar module <x>. Otherwise, the module provides information for itself. - // Either way extract the name of the boot jar module. - bootJarName := strings.TrimSuffix(name, "-hiddenapi") - - // If this module is on the boot jars list (or providing information for a module - // on the list) then extract the hiddenapi information from it, and if necessary - // encode that information in the generated dex file. - // - // It is important that hiddenapi information is only gathered for/from modules on - // that are actually on the boot jars list because the runtime only enforces access - // to the hidden API for the bootclassloader. If information is gathered for modules - // not on the list then that will cause failures in the CtsHiddenApiBlacklist... - // tests. - if inList(bootJarName, ctx.Config().BootJars()) { - // Create ninja rules to generate various CSV files needed by hiddenapi and store the paths - // in the hiddenAPI structure. - h.hiddenAPIGenerateCSV(ctx, implementationJar) - - // If this module is actually on the boot jars list and not providing - // hiddenapi information for a module on the boot jars list then encode - // the gathered information in the generated dex file. - if name == bootJarName { - hiddenAPIJar := android.PathForModuleOut(ctx, "hiddenapi", name+".jar").OutputPath - - // More than one library with the same classes can be encoded but only one can - // be added to the global set of flags, otherwise it will result in duplicate - // classes which is an error. Therefore, only add the dex jar of one of them - // to the global set of flags. - if primary { - h.bootDexJarPath = dexJar - } - hiddenAPIEncodeDex(ctx, hiddenAPIJar, dexJar, uncompressDex) - dexJar = hiddenAPIJar - } - } + + if !h.active { + return dexJar + } + + h.hiddenAPIExtractInformation(ctx, dexJar, implementationJar, primary) + + if !h.annotationsOnly { + hiddenAPIJar := android.PathForModuleOut(ctx, "hiddenapi", name+".jar").OutputPath + + // Create a copy of the dex jar which has been encoded with hiddenapi flags. + hiddenAPIEncodeDex(ctx, hiddenAPIJar, dexJar, uncompressDex) + + // Use the encoded dex jar from here onwards. + dexJar = hiddenAPIJar } return dexJar } -func (h *hiddenAPI) hiddenAPIGenerateCSV(ctx android.ModuleContext, classesJar android.Path) { +// hiddenAPIExtractInformation generates ninja rules to extract the information from the classes +// jar, and outputs it to the appropriate module specific CSV file. +// +// It also makes the dex jar available for use when generating the +// hiddenAPISingletonPathsStruct.stubFlags. +func (h *hiddenAPI) hiddenAPIExtractInformation(ctx android.ModuleContext, dexJar, classesJar android.Path, primary bool) { + if !h.active { + return + } + + // More than one library with the same classes may need to be encoded but only one should be + // used as a source of information for hidden API processing otherwise it will result in + // duplicate entries in the files. + if !primary { + return + } + stubFlagsCSV := hiddenAPISingletonPaths(ctx).stubFlags flagsCSV := android.PathForModuleOut(ctx, "hiddenapi", "flags.csv") @@ -173,6 +214,10 @@ func (h *hiddenAPI) hiddenAPIGenerateCSV(ctx android.ModuleContext, classesJar a FlagWithOutput("--output=", indexCSV) rule.Build("merged-hiddenapi-index", "Merged Hidden API index") h.indexCSVPath = indexCSV + + // Save the unencoded dex jar so it can be used when generating the + // hiddenAPISingletonPathsStruct.stubFlags file. + h.bootDexJarPath = dexJar } var hiddenAPIEncodeDexRule = pctx.AndroidStaticRule("hiddenAPIEncodeDex", blueprint.RuleParams{ diff --git a/java/hiddenapi_singleton.go b/java/hiddenapi_singleton.go index 32d1e3faa..568d15fce 100644 --- a/java/hiddenapi_singleton.go +++ b/java/hiddenapi_singleton.go @@ -22,9 +22,13 @@ import ( ) func init() { - android.RegisterSingletonType("hiddenapi", hiddenAPISingletonFactory) - android.RegisterSingletonType("hiddenapi_index", hiddenAPIIndexSingletonFactory) - android.RegisterModuleType("hiddenapi_flags", hiddenAPIFlagsFactory) + RegisterHiddenApiSingletonComponents(android.InitRegistrationContext) +} + +func RegisterHiddenApiSingletonComponents(ctx android.RegistrationContext) { + ctx.RegisterSingletonType("hiddenapi", hiddenAPISingletonFactory) + ctx.RegisterSingletonType("hiddenapi_index", hiddenAPIIndexSingletonFactory) + ctx.RegisterModuleType("hiddenapi_flags", hiddenAPIFlagsFactory) } type hiddenAPISingletonPathsStruct struct { @@ -368,7 +372,7 @@ func flagsRule(ctx android.SingletonContext) android.Path { stubFlags := hiddenAPISingletonPaths(ctx).stubFlags rule.Command(). - Tool(android.PathForSource(ctx, "frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py")). + BuiltTool("generate_hiddenapi_lists"). FlagWithInput("--csv ", stubFlags). Inputs(flagsCSV). FlagWithInput("--unsupported ", diff --git a/java/hiddenapi_singleton_test.go b/java/hiddenapi_singleton_test.go index 27f363e0c..77cfff45c 100644 --- a/java/hiddenapi_singleton_test.go +++ b/java/hiddenapi_singleton_test.go @@ -15,11 +15,12 @@ package java import ( - "android/soong/android" "fmt" "strings" "testing" + "android/soong/android" + "github.com/google/blueprint/proptools" ) @@ -32,7 +33,7 @@ func testConfigWithBootJars(bp string, bootJars []string, prebuiltHiddenApiDir * func testContextWithHiddenAPI(config android.Config) *android.TestContext { ctx := testContext(config) - ctx.RegisterSingletonType("hiddenapi", hiddenAPISingletonFactory) + RegisterHiddenApiSingletonComponents(ctx) return ctx } @@ -64,7 +65,7 @@ func TestHiddenAPISingleton(t *testing.T) { name: "foo", srcs: ["a.java"], compile_dex: true, - } + } `, []string{"platform:foo"}, nil) hiddenAPI := ctx.SingletonForTests("hiddenapi") @@ -75,6 +76,37 @@ func TestHiddenAPISingleton(t *testing.T) { } } +func TestHiddenAPIIndexSingleton(t *testing.T) { + ctx, _ := testHiddenAPIBootJars(t, ` + java_library { + name: "foo", + srcs: ["a.java"], + compile_dex: true, + } + + java_import { + name: "foo", + jars: ["a.jar"], + compile_dex: true, + prefer: false, + } + + java_sdk_library { + name: "bar", + srcs: ["a.java"], + compile_dex: true, + } + `, []string{"platform:foo", "platform:bar"}, nil) + + hiddenAPIIndex := ctx.SingletonForTests("hiddenapi_index") + indexRule := hiddenAPIIndex.Rule("singleton-merged-hiddenapi-index") + CheckHiddenAPIRuleInputs(t, ` +.intermediates/bar/android_common/hiddenapi/index.csv +.intermediates/foo/android_common/hiddenapi/index.csv +`, + indexRule) +} + func TestHiddenAPISingletonWithPrebuilt(t *testing.T) { ctx, _ := testHiddenAPIBootJars(t, ` java_import { @@ -98,14 +130,14 @@ func TestHiddenAPISingletonWithPrebuiltUseSource(t *testing.T) { name: "foo", srcs: ["a.java"], compile_dex: true, - } + } java_import { name: "foo", jars: ["a.jar"], compile_dex: true, prefer: false, - } + } `, []string{"platform:foo"}, nil) hiddenAPI := ctx.SingletonForTests("hiddenapi") @@ -127,14 +159,14 @@ func TestHiddenAPISingletonWithPrebuiltOverrideSource(t *testing.T) { name: "foo", srcs: ["a.java"], compile_dex: true, - } + } java_import { name: "foo", jars: ["a.jar"], compile_dex: true, prefer: true, - } + } `, []string{"platform:foo"}, nil) hiddenAPI := ctx.SingletonForTests("hiddenapi") diff --git a/java/java.go b/java/java.go index bed232b4e..d194ff7d9 100644 --- a/java/java.go +++ b/java/java.go @@ -1805,7 +1805,7 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { primary = primary && !j.IsReplacedByPrebuilt() // Hidden API CSV generation and dex encoding - dexOutputFile = j.hiddenAPI.hiddenAPI(ctx, configurationName, primary, dexOutputFile, j.implementationJarFile, + dexOutputFile = j.hiddenAPIExtractAndEncode(ctx, configurationName, primary, dexOutputFile, j.implementationJarFile, proptools.Bool(j.dexProperties.Uncompress_dex)) // merge dex jar with resources if necessary @@ -2090,6 +2090,11 @@ func (j *Module) Stem() string { return proptools.StringDefault(j.deviceProperties.Stem, j.Name()) } +// ConfigurationName returns the name of the module as used in build configuration. +// +// This is usually the same as BaseModuleName() except for the <x>.impl libraries created by +// java_sdk_library in which case this is the BaseModuleName() without the ".impl" suffix, +// i.e. just <x>. func (j *Module) ConfigurationName() string { return proptools.StringDefault(j.deviceProperties.ConfigurationName, j.BaseModuleName()) } @@ -2149,6 +2154,11 @@ func shouldUncompressDex(ctx android.ModuleContext, dexpreopter *dexpreopter) bo } func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Initialize the hiddenapi structure. Pass in the configuration name rather than the module name + // so the hidden api will encode the <x>.impl java_ library created by java_sdk_library just as it + // would the <x> library if <x> was configured as a boot jar. + j.initHiddenAPI(ctx, j.ConfigurationName()) + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) if !apexInfo.IsForPlatform() { j.hideApexVariantFromMake = true @@ -2849,6 +2859,9 @@ func (j *Import) DepsMutator(ctx android.BottomUpMutatorContext) { } func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Initialize the hiddenapi structure. + j.initHiddenAPI(ctx, j.BaseModuleName()) + if !ctx.Provider(android.ApexInfoProvider).(android.ApexInfo).IsForPlatform() { j.hideApexVariantFromMake = true } @@ -2904,6 +2917,9 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.exportAidlIncludeDirs = android.PathsForModuleSrc(ctx, j.properties.Aidl.Export_include_dirs) if ctx.Device() { + configurationName := j.BaseModuleName() + primary := j.Prebuilt().UsePrebuilt() + // If this is a variant created for a prebuilt_apex then use the dex implementation jar // obtained from the associated deapexer module. ai := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) @@ -2917,8 +2933,10 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Get the path of the dex implementation jar from the `deapexer` module. di := ctx.OtherModuleProvider(deapexerModule, android.DeapexerProvider).(android.DeapexerInfo) - j.dexJarFile = di.PrebuiltExportPath(j.BaseModuleName(), ".dexjar") - if j.dexJarFile == nil { + if dexOutputPath := di.PrebuiltExportPath(j.BaseModuleName(), ".dexjar"); dexOutputPath != nil { + j.dexJarFile = dexOutputPath + j.hiddenAPI.hiddenAPIExtractInformation(ctx, dexOutputPath, outputFile, primary) + } else { // This should never happen as a variant for a prebuilt_apex is only created if the // prebuilt_apex has been configured to export the java library dex file. ctx.ModuleErrorf("internal error: no dex implementation jar available from prebuilt_apex %q", deapexerModule.Name()) @@ -2948,11 +2966,8 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { return } - configurationName := j.BaseModuleName() - primary := j.Prebuilt().UsePrebuilt() - // Hidden API CSV generation and dex encoding - dexOutputFile = j.hiddenAPI.hiddenAPI(ctx, configurationName, primary, dexOutputFile, outputFile, + dexOutputFile = j.hiddenAPIExtractAndEncode(ctx, configurationName, primary, dexOutputFile, outputFile, proptools.Bool(j.dexProperties.Uncompress_dex)) j.dexJarFile = dexOutputFile diff --git a/java/testing.go b/java/testing.go index 5fcf84c6b..781106ff2 100644 --- a/java/testing.go +++ b/java/testing.go @@ -18,6 +18,7 @@ import ( "fmt" "reflect" "sort" + "strings" "testing" "android/soong/android" @@ -237,3 +238,11 @@ func CheckModuleDependencies(t *testing.T, ctx *android.TestContext, name, varia t.Errorf("expected %#q, found %#q", expected, actual) } } + +func CheckHiddenAPIRuleInputs(t *testing.T, expected string, hiddenAPIRule android.TestingBuildParams) { + actual := strings.TrimSpace(strings.Join(android.NormalizePathsForTesting(hiddenAPIRule.Implicits), "\n")) + expected = strings.TrimSpace(expected) + if actual != expected { + t.Errorf("Expected hiddenapi rule inputs:\n%s\nactual inputs:\n%s", expected, actual) + } +} diff --git a/rust/image.go b/rust/image.go index ac8c1b32c..628aca3e4 100644 --- a/rust/image.go +++ b/rust/image.go @@ -24,7 +24,7 @@ import ( var _ android.ImageInterface = (*Module)(nil) func (mod *Module) VendorRamdiskVariantNeeded(ctx android.BaseModuleContext) bool { - return false + return mod.Properties.VendorRamdiskVariantNeeded } func (mod *Module) CoreVariantNeeded(ctx android.BaseModuleContext) bool { @@ -52,6 +52,10 @@ func (mod *Module) InRecovery() bool { return false } +func (mod *Module) InVendorRamdisk() bool { + return mod.ModuleBase.InVendorRamdisk() || mod.ModuleBase.InstallInVendorRamdisk() +} + func (mod *Module) OnlyInRamdisk() bool { // TODO(b/165791368) return false @@ -86,7 +90,9 @@ func (c *Module) InProduct() bool { func (mod *Module) SetImageVariation(ctx android.BaseModuleContext, variant string, module android.Module) { m := module.(*Module) - if strings.HasPrefix(variant, cc.VendorVariationPrefix) { + if variant == android.VendorRamdiskVariation { + m.MakeAsPlatform() + } else if strings.HasPrefix(variant, cc.VendorVariationPrefix) { m.Properties.ImageVariationPrefix = cc.VendorVariationPrefix m.Properties.VndkVersion = strings.TrimPrefix(variant, cc.VendorVariationPrefix) @@ -117,6 +123,8 @@ func (mod *Module) ImageMutatorBegin(mctx android.BaseModuleContext) { } coreVariantNeeded := true + vendorRamdiskVariantNeeded := false + var vendorVariants []string if mod.HasVendorVariant() { @@ -138,15 +146,23 @@ func (mod *Module) ImageMutatorBegin(mctx android.BaseModuleContext) { // We can't check shared() here because image mutator is called before the library mutator, so we need to // check buildShared() if lib.buildShared() { - mctx.PropertyErrorf(prop, "can only be set for rust_ffi_static modules.") + mctx.PropertyErrorf(prop, "cannot be set for rust_ffi or rust_ffi_shared modules.") } else { vendorVariants = append(vendorVariants, platformVndkVersion) } } } + if Bool(mod.Properties.Vendor_ramdisk_available) { + if lib, ok := mod.compiler.(libraryInterface); !ok || (ok && lib.buildShared()) { + mctx.PropertyErrorf("vendor_ramdisk_available", "cannot be set for rust_ffi or rust_ffi_shared modules.") + } else { + vendorRamdiskVariantNeeded = true + } + } + if vendorSpecific { - if lib, ok := mod.compiler.(libraryInterface); !ok || (ok && !lib.static()) { + if lib, ok := mod.compiler.(libraryInterface); !ok || (ok && (lib.buildShared() || lib.buildDylib() || lib.buildRlib())) { mctx.ModuleErrorf("Rust vendor specific modules are currently only supported for rust_ffi_static modules.") } else { coreVariantNeeded = false @@ -155,6 +171,8 @@ func (mod *Module) ImageMutatorBegin(mctx android.BaseModuleContext) { } mod.Properties.CoreVariantNeeded = coreVariantNeeded + mod.Properties.VendorRamdiskVariantNeeded = vendorRamdiskVariantNeeded + for _, variant := range android.FirstUniqueStrings(vendorVariants) { mod.Properties.ExtraVariants = append(mod.Properties.ExtraVariants, cc.VendorVariationPrefix+variant) } diff --git a/rust/image_test.go b/rust/image_test.go index fd719628b..1515aa264 100644 --- a/rust/image_test.go +++ b/rust/image_test.go @@ -21,7 +21,7 @@ import ( "android/soong/cc" ) -// Test that cc_binaries can link against rust_ffi_static libraries. +// Test that cc modules can link against vendor_available rust_ffi_static libraries. func TestVendorLinkage(t *testing.T) { ctx := testRust(t, ` cc_binary { @@ -44,9 +44,33 @@ func TestVendorLinkage(t *testing.T) { } } +// Test that cc modules can link against vendor_ramdisk_available rust_ffi_static libraries. +func TestVendorRamdiskLinkage(t *testing.T) { + ctx := testRust(t, ` + cc_library_static { + name: "libcc_vendor_ramdisk", + static_libs: ["libfoo_vendor_ramdisk"], + system_shared_libs: [], + vendor_ramdisk_available: true, + } + rust_ffi_static { + name: "libfoo_vendor_ramdisk", + crate_name: "foo", + srcs: ["foo.rs"], + vendor_ramdisk_available: true, + } + `) + + vendorRamdiskLibrary := ctx.ModuleForTests("libcc_vendor_ramdisk", "android_vendor_ramdisk_arm64_armv8-a_static").Module().(*cc.Module) + + if !android.InList("libfoo_vendor_ramdisk.vendor_ramdisk", vendorRamdiskLibrary.Properties.AndroidMkStaticLibs) { + t.Errorf("libcc_vendor_ramdisk should have a dependency on libfoo_vendor_ramdisk") + } +} + // Test that shared libraries cannot be made vendor available until proper support is added. func TestForbiddenVendorLinkage(t *testing.T) { - testRustError(t, "can only be set for rust_ffi_static modules", ` + testRustError(t, "cannot be set for rust_ffi or rust_ffi_shared modules.", ` rust_ffi_shared { name: "libfoo_vendor", crate_name: "foo", @@ -54,6 +78,14 @@ func TestForbiddenVendorLinkage(t *testing.T) { vendor_available: true, } `) + testRustError(t, "cannot be set for rust_ffi or rust_ffi_shared modules.", ` + rust_ffi_shared { + name: "libfoo_vendor", + crate_name: "foo", + srcs: ["foo.rs"], + vendor_ramdisk_available: true, + } + `) testRustError(t, "Rust vendor specific modules are currently only supported for rust_ffi_static modules.", ` rust_ffi { name: "libfoo_vendor", @@ -70,4 +102,13 @@ func TestForbiddenVendorLinkage(t *testing.T) { vendor: true, } `) + testRustError(t, "Rust vendor specific modules are currently only supported for rust_ffi_static modules.", ` + rust_binary { + name: "foo_vendor", + crate_name: "foo", + srcs: ["foo.rs"], + vendor: true, + } + `) + } diff --git a/rust/rust.go b/rust/rust.go index 0b733cc35..dc23abb8d 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -74,8 +74,16 @@ type BaseProperties struct { SubName string `blueprint:"mutated"` // Set by imageMutator - CoreVariantNeeded bool `blueprint:"mutated"` - ExtraVariants []string `blueprint:"mutated"` + CoreVariantNeeded bool `blueprint:"mutated"` + VendorRamdiskVariantNeeded bool `blueprint:"mutated"` + ExtraVariants []string `blueprint:"mutated"` + + // Make this module available when building for vendor ramdisk. + // On device without a dedicated recovery partition, the module is only + // available after switching root into + // /first_stage_ramdisk. To expose the module before switching root, install + // the recovery variant instead (TODO(b/165791368) recovery not yet supported) + Vendor_ramdisk_available *bool // Minimum sdk version that the artifact should support when it runs as part of mainline modules(APEX). Min_sdk_version *string @@ -658,7 +666,9 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { // Differentiate static libraries that are vendor available if mod.UseVndk() { - mod.Properties.SubName += ".vendor" + mod.Properties.SubName += cc.VendorSuffix + } else if mod.InVendorRamdisk() && !mod.OnlyInVendorRamdisk() { + mod.Properties.SubName += cc.VendorRamdiskSuffix } if !toolchain.Supported() { diff --git a/rust/testing.go b/rust/testing.go index bb511b648..4c4df4a6d 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -101,6 +101,7 @@ func GatherRequiredDepsForTest() string { no_stdlibs: true, host_supported: true, vendor_available: true, + vendor_ramdisk_available: true, native_coverage: false, sysroot: true, apex_available: ["//apex_available:platform", "//apex_available:anyapex"], @@ -113,6 +114,7 @@ func GatherRequiredDepsForTest() string { no_stdlibs: true, host_supported: true, vendor_available: true, + vendor_ramdisk_available: true, native_coverage: false, sysroot: true, apex_available: ["//apex_available:platform", "//apex_available:anyapex"], diff --git a/scripts/gen_ndk_backedby_apex.sh b/scripts/gen_ndk_backedby_apex.sh index e0da60236..4abaaba41 100755 --- a/scripts/gen_ndk_backedby_apex.sh +++ b/scripts/gen_ndk_backedby_apex.sh @@ -23,33 +23,50 @@ printHelp() { echo "**************************** Usage Instructions ****************************" echo "This script is used to generate the Mainline modules backed-by NDK symbols." echo "" - echo "To run this script use: ./ndk_backedby_module.sh \$BINARY_IMAGE_DIRECTORY \$OUTPUT_FILE_PATH \$NDK_LIB_NAME_LIST" - echo "For example: If all the module image files that you would like to run is under directory '/myModule' and output write to /backedby.txt then the command would be:" - echo "./ndk_usedby_module.sh /myModule /backedby.txt /ndkLibList.txt" + echo "To run this script use: ./gen_ndk_backed_by_apex.sh \$OUTPUT_FILE_PATH \$NDK_LIB_NAME_LIST \$MODULE_LIB1 \$MODULE_LIB2..." + echo "For example: If output write to /backedby.txt then the command would be:" + echo "./gen_ndk_backed_by_apex.sh /backedby.txt /ndkLibList.txt lib1.so lib2.so" echo "If the module1 is backing lib1 then the backedby.txt would contains: " echo "lib1" } +contains() { + val="$1" + shift + for x in "$@"; do + if [ "$x" = "$val" ]; then + return 0 + fi + done + return 1 +} + + genBackedByList() { - dir="$1" - [[ ! -e "$2" ]] && echo "" >> "$2" + out="$1" + shift + ndk_list="$1" + shift + rm -f "$out" + touch "$out" while IFS= read -r line do soFileName=$(echo "$line" | sed 's/\(.*so\).*/\1/') if [[ ! -z "$soFileName" && "$soFileName" != *"#"* ]] then - find "$dir" -type f -name "$soFileName" -exec echo "$soFileName" >> "$2" \; + if contains "$soFileName" "$@"; then + echo "$soFileName" >> "$out" + fi fi - done < "$3" + done < "$ndk_list" } if [[ "$1" == "help" ]] then printHelp -elif [[ "$#" -ne 3 ]] +elif [[ "$#" -lt 2 ]] then - echo "Wrong argument length. Expecting 3 argument representing image file directory, output path, path to ndk library list." + echo "Wrong argument length. Expecting at least 2 argument representing output path, path to ndk library list, followed by a list of libraries in the Mainline module." else - [[ -e "$2" ]] && rm "$2" - genBackedByList "$1" "$2" "$3" + genBackedByList "$@" fi diff --git a/scripts/hiddenapi/Android.bp b/scripts/hiddenapi/Android.bp new file mode 100644 index 000000000..a669cadd4 --- /dev/null +++ b/scripts/hiddenapi/Android.bp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +python_binary_host { + name: "merge_csv", + main: "merge_csv.py", + srcs: ["merge_csv.py"], + version: { + py2: { + enabled: false, + }, + py3: { + enabled: true, + embedded_launcher: true, + }, + }, +} + +python_binary_host { + name: "generate_hiddenapi_lists", + main: "generate_hiddenapi_lists.py", + srcs: ["generate_hiddenapi_lists.py"], + version: { + py2: { + enabled: false, + }, + py3: { + enabled: true, + embedded_launcher: true, + }, + }, +} diff --git a/scripts/hiddenapi/generate_hiddenapi_lists.py b/scripts/hiddenapi/generate_hiddenapi_lists.py new file mode 100755 index 000000000..681647548 --- /dev/null +++ b/scripts/hiddenapi/generate_hiddenapi_lists.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Generate API lists for non-SDK API enforcement.""" +import argparse +from collections import defaultdict, namedtuple +import functools +import os +import re +import sys + +# Names of flags recognized by the `hiddenapi` tool. +FLAG_SDK = 'sdk' +FLAG_UNSUPPORTED = 'unsupported' +FLAG_BLOCKED = 'blocked' +FLAG_MAX_TARGET_O = 'max-target-o' +FLAG_MAX_TARGET_P = 'max-target-p' +FLAG_MAX_TARGET_Q = 'max-target-q' +FLAG_MAX_TARGET_R = 'max-target-r' +FLAG_CORE_PLATFORM_API = 'core-platform-api' +FLAG_PUBLIC_API = 'public-api' +FLAG_SYSTEM_API = 'system-api' +FLAG_TEST_API = 'test-api' + +# List of all known flags. +FLAGS_API_LIST = [ + FLAG_SDK, + FLAG_UNSUPPORTED, + FLAG_BLOCKED, + FLAG_MAX_TARGET_O, + FLAG_MAX_TARGET_P, + FLAG_MAX_TARGET_Q, + FLAG_MAX_TARGET_R, +] +ALL_FLAGS = FLAGS_API_LIST + [ + FLAG_CORE_PLATFORM_API, + FLAG_PUBLIC_API, + FLAG_SYSTEM_API, + FLAG_TEST_API, +] + +FLAGS_API_LIST_SET = set(FLAGS_API_LIST) +ALL_FLAGS_SET = set(ALL_FLAGS) + +# Option specified after one of FLAGS_API_LIST to indicate that +# only known and otherwise unassigned entries should be assign the +# given flag. +# For example, the max-target-P list is checked in as it was in P, +# but signatures have changes since then. The flag instructs this +# script to skip any entries which do not exist any more. +FLAG_IGNORE_CONFLICTS = "ignore-conflicts" + +# Option specified after one of FLAGS_API_LIST to express that all +# apis within a given set of packages should be assign the given flag. +FLAG_PACKAGES = "packages" + +# Option specified after one of FLAGS_API_LIST to indicate an extra +# tag that should be added to the matching APIs. +FLAG_TAG = "tag" + +# Regex patterns of fields/methods used in serialization. These are +# considered public API despite being hidden. +SERIALIZATION_PATTERNS = [ + r'readObject\(Ljava/io/ObjectInputStream;\)V', + r'readObjectNoData\(\)V', + r'readResolve\(\)Ljava/lang/Object;', + r'serialVersionUID:J', + r'serialPersistentFields:\[Ljava/io/ObjectStreamField;', + r'writeObject\(Ljava/io/ObjectOutputStream;\)V', + r'writeReplace\(\)Ljava/lang/Object;', +] + +# Single regex used to match serialization API. It combines all the +# SERIALIZATION_PATTERNS into a single regular expression. +SERIALIZATION_REGEX = re.compile(r'.*->(' + '|'.join(SERIALIZATION_PATTERNS) + r')$') + +# Predicates to be used with filter_apis. +HAS_NO_API_LIST_ASSIGNED = lambda api, flags: not FLAGS_API_LIST_SET.intersection(flags) +IS_SERIALIZATION = lambda api, flags: SERIALIZATION_REGEX.match(api) + + +class StoreOrderedOptions(argparse.Action): + """An argparse action that stores a number of option arguments in the order that + they were specified. + """ + def __call__(self, parser, args, values, option_string = None): + items = getattr(args, self.dest, None) + if items is None: + items = [] + items.append([option_string.lstrip('-'), values]) + setattr(args, self.dest, items) + +def get_args(): + """Parses command line arguments. + + Returns: + Namespace: dictionary of parsed arguments + """ + parser = argparse.ArgumentParser() + parser.add_argument('--output', required=True) + parser.add_argument('--csv', nargs='*', default=[], metavar='CSV_FILE', + help='CSV files to be merged into output') + + for flag in ALL_FLAGS: + parser.add_argument('--' + flag, dest='ordered_flags', metavar='TXT_FILE', + action=StoreOrderedOptions, help='lists of entries with flag "' + flag + '"') + parser.add_argument('--' + FLAG_IGNORE_CONFLICTS, dest='ordered_flags', nargs=0, + action=StoreOrderedOptions, help='Indicates that only known and otherwise unassigned ' + 'entries should be assign the given flag. Must follow a list of entries and applies ' + 'to the preceding such list.') + parser.add_argument('--' + FLAG_PACKAGES, dest='ordered_flags', nargs=0, + action=StoreOrderedOptions, help='Indicates that the previous list of entries ' + 'is a list of packages. All members in those packages will be given the flag. ' + 'Must follow a list of entries and applies to the preceding such list.') + parser.add_argument('--' + FLAG_TAG, dest='ordered_flags', nargs=1, + action=StoreOrderedOptions, help='Adds an extra tag to the previous list of entries. ' + 'Must follow a list of entries and applies to the preceding such list.') + + return parser.parse_args() + + +def read_lines(filename): + """Reads entire file and return it as a list of lines. + + Lines which begin with a hash are ignored. + + Args: + filename (string): Path to the file to read from. + + Returns: + Lines of the file as a list of string. + """ + with open(filename, 'r') as f: + lines = f.readlines(); + lines = filter(lambda line: not line.startswith('#'), lines) + lines = map(lambda line: line.strip(), lines) + return set(lines) + + +def write_lines(filename, lines): + """Writes list of lines into a file, overwriting the file if it exists. + + Args: + filename (string): Path to the file to be writing into. + lines (list): List of strings to write into the file. + """ + lines = map(lambda line: line + '\n', lines) + with open(filename, 'w') as f: + f.writelines(lines) + + +def extract_package(signature): + """Extracts the package from a signature. + + Args: + signature (string): JNI signature of a method or field. + + Returns: + The package name of the class containing the field/method. + """ + full_class_name = signature.split(";->")[0] + # Example: Landroid/hardware/radio/V1_2/IRadio$Proxy + if (full_class_name[0] != "L"): + raise ValueError("Expected to start with 'L': %s" % full_class_name) + full_class_name = full_class_name[1:] + # If full_class_name doesn't contain '/', then package_name will be ''. + package_name = full_class_name.rpartition("/")[0] + return package_name.replace('/', '.') + + +class FlagsDict: + def __init__(self): + self._dict_keyset = set() + self._dict = defaultdict(set) + + def _check_entries_set(self, keys_subset, source): + assert isinstance(keys_subset, set) + assert keys_subset.issubset(self._dict_keyset), ( + "Error: {} specifies signatures not present in code:\n" + "{}" + "Please visit go/hiddenapi for more information.").format( + source, "".join(map(lambda x: " " + str(x) + "\n", keys_subset - self._dict_keyset))) + + def _check_flags_set(self, flags_subset, source): + assert isinstance(flags_subset, set) + assert flags_subset.issubset(ALL_FLAGS_SET), ( + "Error processing: {}\n" + "The following flags were not recognized: \n" + "{}\n" + "Please visit go/hiddenapi for more information.").format( + source, "\n".join(flags_subset - ALL_FLAGS_SET)) + + def filter_apis(self, filter_fn): + """Returns APIs which match a given predicate. + + This is a helper function which allows to filter on both signatures (keys) and + flags (values). The built-in filter() invokes the lambda only with dict's keys. + + Args: + filter_fn : Function which takes two arguments (signature/flags) and returns a boolean. + + Returns: + A set of APIs which match the predicate. + """ + return set(filter(lambda x: filter_fn(x, self._dict[x]), self._dict_keyset)) + + def get_valid_subset_of_unassigned_apis(self, api_subset): + """Sanitizes a key set input to only include keys which exist in the dictionary + and have not been assigned any API list flags. + + Args: + entries_subset (set/list): Key set to be sanitized. + + Returns: + Sanitized key set. + """ + assert isinstance(api_subset, set) + return api_subset.intersection(self.filter_apis(HAS_NO_API_LIST_ASSIGNED)) + + def generate_csv(self): + """Constructs CSV entries from a dictionary. + + Old versions of flags are used to generate the file. + + Returns: + List of lines comprising a CSV file. See "parse_and_merge_csv" for format description. + """ + lines = [] + for api in self._dict: + flags = sorted(self._dict[api]) + lines.append(",".join([api] + flags)) + return sorted(lines) + + def parse_and_merge_csv(self, csv_lines, source = "<unknown>"): + """Parses CSV entries and merges them into a given dictionary. + + The expected CSV format is: + <api signature>,<flag1>,<flag2>,...,<flagN> + + Args: + csv_lines (list of strings): Lines read from a CSV file. + source (string): Origin of `csv_lines`. Will be printed in error messages. + + Throws: + AssertionError if parsed flags are invalid. + """ + # Split CSV lines into arrays of values. + csv_values = [ line.split(',') for line in csv_lines ] + + # Update the full set of API signatures. + self._dict_keyset.update([ csv[0] for csv in csv_values ]) + + # Check that all flags are known. + csv_flags = set() + for csv in csv_values: + csv_flags.update(csv[1:]) + self._check_flags_set(csv_flags, source) + + # Iterate over all CSV lines, find entry in dict and append flags to it. + for csv in csv_values: + flags = csv[1:] + if (FLAG_PUBLIC_API in flags) or (FLAG_SYSTEM_API in flags): + flags.append(FLAG_SDK) + self._dict[csv[0]].update(flags) + + def assign_flag(self, flag, apis, source="<unknown>", tag = None): + """Assigns a flag to given subset of entries. + + Args: + flag (string): One of ALL_FLAGS. + apis (set): Subset of APIs to receive the flag. + source (string): Origin of `entries_subset`. Will be printed in error messages. + + Throws: + AssertionError if parsed API signatures of flags are invalid. + """ + # Check that all APIs exist in the dict. + self._check_entries_set(apis, source) + + # Check that the flag is known. + self._check_flags_set(set([ flag ]), source) + + # Iterate over the API subset, find each entry in dict and assign the flag to it. + for api in apis: + self._dict[api].add(flag) + if tag: + self._dict[api].add(tag) + + +FlagFile = namedtuple('FlagFile', ('flag', 'file', 'ignore_conflicts', 'packages', 'tag')) + +def parse_ordered_flags(ordered_flags): + r = [] + currentflag, file, ignore_conflicts, packages, tag = None, None, False, False, None + for flag_value in ordered_flags: + flag, value = flag_value[0], flag_value[1] + if flag in ALL_FLAGS_SET: + if currentflag: + r.append(FlagFile(currentflag, file, ignore_conflicts, packages, tag)) + ignore_conflicts, packages, tag = False, False, None + currentflag = flag + file = value + else: + if currentflag is None: + raise argparse.ArgumentError('--%s is only allowed after one of %s' % ( + flag, ' '.join(['--%s' % f for f in ALL_FLAGS_SET]))) + if flag == FLAG_IGNORE_CONFLICTS: + ignore_conflicts = True + elif flag == FLAG_PACKAGES: + packages = True + elif flag == FLAG_TAG: + tag = value[0] + + + if currentflag: + r.append(FlagFile(currentflag, file, ignore_conflicts, packages, tag)) + return r + + +def main(argv): + # Parse arguments. + args = vars(get_args()) + flagfiles = parse_ordered_flags(args['ordered_flags']) + + # Initialize API->flags dictionary. + flags = FlagsDict() + + # Merge input CSV files into the dictionary. + # Do this first because CSV files produced by parsing API stubs will + # contain the full set of APIs. Subsequent additions from text files + # will be able to detect invalid entries, and/or filter all as-yet + # unassigned entries. + for filename in args["csv"]: + flags.parse_and_merge_csv(read_lines(filename), filename) + + # Combine inputs which do not require any particular order. + # (1) Assign serialization API to SDK. + flags.assign_flag(FLAG_SDK, flags.filter_apis(IS_SERIALIZATION)) + + # (2) Merge text files with a known flag into the dictionary. + for info in flagfiles: + if (not info.ignore_conflicts) and (not info.packages): + flags.assign_flag(info.flag, read_lines(info.file), info.file, info.tag) + + # Merge text files where conflicts should be ignored. + # This will only assign the given flag if: + # (a) the entry exists, and + # (b) it has not been assigned any other flag. + # Because of (b), this must run after all strict assignments have been performed. + for info in flagfiles: + if info.ignore_conflicts: + valid_entries = flags.get_valid_subset_of_unassigned_apis(read_lines(info.file)) + flags.assign_flag(info.flag, valid_entries, filename, info.tag) + + # All members in the specified packages will be assigned the appropriate flag. + for info in flagfiles: + if info.packages: + packages_needing_list = set(read_lines(info.file)) + should_add_signature_to_list = lambda sig,lists: extract_package( + sig) in packages_needing_list and not lists + valid_entries = flags.filter_apis(should_add_signature_to_list) + flags.assign_flag(info.flag, valid_entries, info.file, info.tag) + + # Mark all remaining entries as blocked. + flags.assign_flag(FLAG_BLOCKED, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED)) + + # Write output. + write_lines(args["output"], flags.generate_csv()) + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/hiddenapi/generate_hiddenapi_lists_test.py b/scripts/hiddenapi/generate_hiddenapi_lists_test.py new file mode 100755 index 000000000..ff3d70881 --- /dev/null +++ b/scripts/hiddenapi/generate_hiddenapi_lists_test.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for Hidden API list generation.""" +import unittest +from generate_hiddenapi_lists import * + +class TestHiddenapiListGeneration(unittest.TestCase): + + def test_filter_apis(self): + # Initialize flags so that A and B are put on the allow list and + # C, D, E are left unassigned. Try filtering for the unassigned ones. + flags = FlagsDict() + flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B,' + FLAG_SDK, + 'C', 'D', 'E']) + filter_set = flags.filter_apis(lambda api, flags: not flags) + self.assertTrue(isinstance(filter_set, set)) + self.assertEqual(filter_set, set([ 'C', 'D', 'E' ])) + + def test_get_valid_subset_of_unassigned_keys(self): + # Create flags where only A is unassigned. + flags = FlagsDict() + flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B', 'C']) + flags.assign_flag(FLAG_UNSUPPORTED, set(['C'])) + self.assertEqual(flags.generate_csv(), + [ 'A,' + FLAG_SDK, 'B', 'C,' + FLAG_UNSUPPORTED ]) + + # Check three things: + # (1) B is selected as valid unassigned + # (2) A is not selected because it is assigned to the allow list + # (3) D is not selected because it is not a valid key + self.assertEqual( + flags.get_valid_subset_of_unassigned_apis(set(['A', 'B', 'D'])), set([ 'B' ])) + + def test_parse_and_merge_csv(self): + flags = FlagsDict() + + # Test empty CSV entry. + self.assertEqual(flags.generate_csv(), []) + + # Test new additions. + flags.parse_and_merge_csv([ + 'A,' + FLAG_UNSUPPORTED, + 'B,' + FLAG_BLOCKED + ',' + FLAG_MAX_TARGET_O, + 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, + 'D,' + FLAG_UNSUPPORTED + ',' + FLAG_TEST_API, + 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, + ]) + self.assertEqual(flags.generate_csv(), [ + 'A,' + FLAG_UNSUPPORTED, + 'B,' + FLAG_BLOCKED + "," + FLAG_MAX_TARGET_O, + 'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API, + 'D,' + FLAG_TEST_API + ',' + FLAG_UNSUPPORTED, + 'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API, + ]) + + # Test unknown flag. + with self.assertRaises(AssertionError): + flags.parse_and_merge_csv([ 'Z,foo' ]) + + def test_assign_flag(self): + flags = FlagsDict() + flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B']) + + # Test new additions. + flags.assign_flag(FLAG_UNSUPPORTED, set([ 'A', 'B' ])) + self.assertEqual(flags.generate_csv(), + [ 'A,' + FLAG_SDK + "," + FLAG_UNSUPPORTED, 'B,' + FLAG_UNSUPPORTED ]) + + # Test invalid API signature. + with self.assertRaises(AssertionError): + flags.assign_flag(FLAG_SDK, set([ 'C' ])) + + # Test invalid flag. + with self.assertRaises(AssertionError): + flags.assign_flag('foo', set([ 'A' ])) + + def test_extract_package(self): + signature = 'Lcom/foo/bar/Baz;->method1()Lcom/bar/Baz;' + expected_package = 'com.foo.bar' + self.assertEqual(extract_package(signature), expected_package) + + signature = 'Lcom/foo1/bar/MyClass;->method2()V' + expected_package = 'com.foo1.bar' + self.assertEqual(extract_package(signature), expected_package) + + signature = 'Lcom/foo_bar/baz/MyClass;->method3()V' + expected_package = 'com.foo_bar.baz' + self.assertEqual(extract_package(signature), expected_package) + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/hiddenapi/merge_csv.py b/scripts/hiddenapi/merge_csv.py new file mode 100755 index 000000000..6a5b0e134 --- /dev/null +++ b/scripts/hiddenapi/merge_csv.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Merge multiple CSV files, possibly with different columns. +""" + +import argparse +import csv +import io + +from zipfile import ZipFile + +args_parser = argparse.ArgumentParser(description='Merge given CSV files into a single one.') +args_parser.add_argument('--header', help='Comma separated field names; ' + 'if missing determines the header from input files.') +args_parser.add_argument('--zip_input', help='ZIP archive with all CSV files to merge.') +args_parser.add_argument('--output', help='Output file for merged CSV.', + default='-', type=argparse.FileType('w')) +args_parser.add_argument('files', nargs=argparse.REMAINDER) +args = args_parser.parse_args() + + +def dict_reader(input): + return csv.DictReader(input, delimiter=',', quotechar='|') + + +if args.zip_input and len(args.files) > 0: + raise ValueError('Expecting either a single ZIP with CSV files' + ' or a list of CSV files as input; not both.') + +csv_readers = [] +if len(args.files) > 0: + for file in args.files: + csv_readers.append(dict_reader(open(file, 'r'))) +elif args.zip_input: + with ZipFile(args.zip_input) as zip: + for entry in zip.namelist(): + if entry.endswith('.uau'): + csv_readers.append(dict_reader(io.TextIOWrapper(zip.open(entry, 'r')))) + +headers = set() +if args.header: + fieldnames = args.header.split(',') +else: + # Build union of all columns from source files: + for reader in csv_readers: + headers = headers.union(reader.fieldnames) + fieldnames = sorted(headers) + +# Concatenate all files to output: +writer = csv.DictWriter(args.output, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL, + dialect='unix', fieldnames=fieldnames) +writer.writeheader() +for reader in csv_readers: + for row in reader: + writer.writerow(row) diff --git a/sh/sh_binary.go b/sh/sh_binary.go index 66e493bf1..54dfc2467 100644 --- a/sh/sh_binary.go +++ b/sh/sh_binary.go @@ -24,6 +24,7 @@ import ( "github.com/google/blueprint/proptools" "android/soong/android" + "android/soong/bazel" "android/soong/cc" "android/soong/tradefed" ) @@ -43,6 +44,8 @@ func init() { android.RegisterModuleType("sh_binary_host", ShBinaryHostFactory) android.RegisterModuleType("sh_test", ShTestFactory) android.RegisterModuleType("sh_test_host", ShTestHostFactory) + + android.RegisterBp2BuildMutator("sh_binary", ShBinaryBp2Build) } type shBinaryProperties struct { @@ -81,6 +84,9 @@ type shBinaryProperties struct { // Make this module available when building for recovery. Recovery_available *bool + + // Properties for Bazel migration purposes. + bazel.Properties } type TestProperties struct { @@ -461,4 +467,62 @@ func ShTestHostFactory() android.Module { return module } +type bazelShBinaryAttributes struct { + Srcs bazel.LabelList + // Bazel also supports the attributes below, but (so far) these are not required for Bionic + // deps + // data + // args + // compatible_with + // deprecation + // distribs + // env + // exec_compatible_with + // exec_properties + // features + // licenses + // output_licenses + // restricted_to + // tags + // target_compatible_with + // testonly + // toolchains + // visibility +} + +type bazelShBinary struct { + android.BazelTargetModuleBase + bazelShBinaryAttributes +} + +func BazelShBinaryFactory() android.Module { + module := &bazelShBinary{} + module.AddProperties(&module.bazelShBinaryAttributes) + android.InitBazelTargetModule(module) + return module +} + +func ShBinaryBp2Build(ctx android.TopDownMutatorContext) { + m, ok := ctx.Module().(*ShBinary) + if !ok || !m.properties.Bazel_module.Bp2build_available { + return + } + + srcs := android.BazelLabelForModuleSrc(ctx, []string{*m.properties.Src}) + + attrs := &bazelShBinaryAttributes{ + Srcs: srcs, + } + + props := bazel.NewBazelTargetModuleProperties(m.Name(), "sh_binary", "") + + ctx.CreateBazelTargetModule(BazelShBinaryFactory, props, attrs) +} + +func (m *bazelShBinary) Name() string { + return m.BaseModuleName() +} + +func (m *bazelShBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) {} + var Bool = proptools.Bool |