diff options
41 files changed, 1144 insertions, 839 deletions
diff --git a/android/bazel_paths.go b/android/bazel_paths.go index e151521b1..bad7baf06 100644 --- a/android/bazel_paths.go +++ b/android/bazel_paths.go @@ -461,11 +461,6 @@ func bp2buildModuleLabel(ctx BazelConversionContext, module blueprint.Module) st return fmt.Sprintf("//%s:%s", moduleDir, moduleName) } -// ModuleFromBazelLabel reverses the logic in bp2buildModuleLabel -func ModuleFromBazelLabel(label string) string { - return strings.Split(label, ":")[1] -} - // BazelOutPath is a Bazel output path compatible to be used for mixed builds within Soong/Ninja. type BazelOutPath struct { OutputPath diff --git a/android/config.go b/android/config.go index a8b0a40b4..087bae117 100644 --- a/android/config.go +++ b/android/config.go @@ -1494,6 +1494,10 @@ func (c *config) ApexCompressionEnabled() bool { return Bool(c.productVariables.CompressedApex) && !c.UnbundledBuildApps() } +func (c *config) ApexTrimEnabled() bool { + return Bool(c.productVariables.TrimmedApex) +} + func (c *config) EnforceSystemCertificate() bool { return Bool(c.productVariables.EnforceSystemCertificate) } diff --git a/android/variable.go b/android/variable.go index e838b7cbd..e714fc49a 100644 --- a/android/variable.go +++ b/android/variable.go @@ -380,6 +380,7 @@ type productVariables struct { Ndk_abis *bool `json:",omitempty"` + TrimmedApex *bool `json:",omitempty"` Flatten_apex *bool `json:",omitempty"` ForceApexSymlinkOptimization *bool `json:",omitempty"` CompressedApex *bool `json:",omitempty"` @@ -502,6 +503,7 @@ func (v *productVariables) SetDefaultConfig() { Malloc_zero_contents: boolPtr(true), Malloc_pattern_fill_contents: boolPtr(false), Safestack: boolPtr(false), + TrimmedApex: boolPtr(false), BootJars: ConfiguredJarList{apexes: []string{}, jars: []string{}}, ApexBootJars: ConfiguredJarList{apexes: []string{}, jars: []string{}}, diff --git a/apex/androidmk.go b/apex/androidmk.go index aadccb724..7babd45ba 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -309,7 +309,7 @@ func (a *apexBundle) writeRequiredModules(w io.Writer, moduleNames []string) { targetRequired = append(targetRequired, fi.targetRequiredModuleNames...) hostRequired = append(hostRequired, fi.hostRequiredModuleNames...) } - android.AndroidMkEmitAssignList(w, "LOCAL_REQUIRED_MODULES", moduleNames, a.requiredDeps, required) + android.AndroidMkEmitAssignList(w, "LOCAL_REQUIRED_MODULES", moduleNames, a.makeModulesToInstall, required) android.AndroidMkEmitAssignList(w, "LOCAL_TARGET_REQUIRED_MODULES", targetRequired) android.AndroidMkEmitAssignList(w, "LOCAL_HOST_REQUIRED_MODULES", hostRequired) } @@ -317,14 +317,14 @@ func (a *apexBundle) writeRequiredModules(w io.Writer, moduleNames []string) { func (a *apexBundle) androidMkForType() android.AndroidMkData { return android.AndroidMkData{ Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) { + moduleNames := []string{} apexType := a.properties.ApexType + if a.installable() { + apexName := proptools.StringDefault(a.properties.Apex_name, name) + moduleNames = a.androidMkForFiles(w, name, apexName, moduleDir, data) + } if apexType == flattenedApex { - var moduleNames []string = nil - if a.installable() { - apexName := proptools.StringDefault(a.properties.Apex_name, name) - moduleNames = a.androidMkForFiles(w, name, apexName, moduleDir, data) - } // Only image APEXes can be flattened. fmt.Fprintln(w, "\ninclude $(CLEAR_VARS) # apex.apexBundle.flat") fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) @@ -366,7 +366,7 @@ func (a *apexBundle) androidMkForType() android.AndroidMkData { } android.AndroidMkEmitAssignList(w, "LOCAL_OVERRIDES_MODULES", a.overridableProperties.Overrides) - a.writeRequiredModules(w, nil) + a.writeRequiredModules(w, moduleNames) fmt.Fprintln(w, "include $(BUILD_PREBUILT)") diff --git a/apex/apex.go b/apex/apex.go index 9485a4b0b..4ade6eb49 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -80,6 +80,7 @@ func RegisterPostDepsMutators(ctx android.RegisterMutatorsContext) { ctx.BottomUp("apex", apexMutator).Parallel() ctx.BottomUp("apex_directly_in_any", apexDirectlyInAnyMutator).Parallel() ctx.BottomUp("apex_flattened", apexFlattenedMutator).Parallel() + ctx.BottomUp("apex_dcla_deps", apexDCLADepsMutator).Parallel() // Register after apex_info mutator so that it can use ApexVariationName ctx.TopDown("apex_strict_updatability_lint", apexStrictUpdatibilityLintMutator).Parallel() } @@ -389,6 +390,9 @@ type overridableProperties struct { // conditions, e.g., target device needs to support APEX compression, are also fulfilled. // Default: false. Compressible *bool + + // Trim against a specific Dynamic Common Lib APEX + Trim_against *string } type apexBundle struct { @@ -439,8 +443,8 @@ type apexBundle struct { // GenerateAndroidBuildActions. filesInfo []apexFile - // List of other module names that should be installed when this APEX gets installed. - requiredDeps []string + // List of other module names that should be installed when this APEX gets installed (LOCAL_REQUIRED_MODULES). + makeModulesToInstall []string /////////////////////////////////////////////////////////////////////////////////////////// // Outputs (final and intermediates) @@ -675,6 +679,7 @@ var ( androidAppTag = &dependencyTag{name: "androidApp", payload: true} bpfTag = &dependencyTag{name: "bpf", payload: true} certificateTag = &dependencyTag{name: "certificate"} + dclaTag = &dependencyTag{name: "dcla"} executableTag = &dependencyTag{name: "executable", payload: true} fsTag = &dependencyTag{name: "filesystem", payload: true} bcpfTag = &dependencyTag{name: "bootclasspathFragment", payload: true, sourceOnly: true, memberType: java.BootclasspathFragmentSdkMemberType} @@ -908,6 +913,33 @@ func (a *apexBundle) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato } } +func apexDCLADepsMutator(mctx android.BottomUpMutatorContext) { + if !mctx.Config().ApexTrimEnabled() { + return + } + if a, ok := mctx.Module().(*apexBundle); ok && a.overridableProperties.Trim_against != nil { + commonVariation := mctx.Config().AndroidCommonTarget.Variations() + mctx.AddFarVariationDependencies(commonVariation, dclaTag, String(a.overridableProperties.Trim_against)) + } else if o, ok := mctx.Module().(*OverrideApex); ok { + for _, p := range o.GetProperties() { + properties, ok := p.(*overridableProperties) + if !ok { + continue + } + if properties.Trim_against != nil { + commonVariation := mctx.Config().AndroidCommonTarget.Variations() + mctx.AddFarVariationDependencies(commonVariation, dclaTag, String(properties.Trim_against)) + } + } + } +} + +type DCLAInfo struct { + ProvidedLibs []string +} + +var DCLAInfoProvider = blueprint.NewMutatorProvider(DCLAInfo{}, "apex_info") + type ApexBundleInfo struct { Contents *android.ApexContents } @@ -1035,6 +1067,12 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { child.(android.ApexModule).BuildForApex(apexInfo) // leave a mark! return true }) + + if a.dynamic_common_lib_apex() { + mctx.SetProvider(DCLAInfoProvider, DCLAInfo{ + ProvidedLibs: a.properties.Native_shared_libs, + }) + } } type ApexInfoMutator interface { @@ -1531,6 +1569,19 @@ func (a *apexBundle) dynamic_common_lib_apex() bool { return proptools.BoolDefault(a.properties.Dynamic_common_lib_apex, false) } +// See the list of libs to trim +func (a *apexBundle) libs_to_trim(ctx android.ModuleContext) []string { + dclaModules := ctx.GetDirectDepsWithTag(dclaTag) + if len(dclaModules) > 1 { + panic(fmt.Errorf("expected exactly at most one dcla dependency, got %d", len(dclaModules))) + } + if len(dclaModules) > 0 { + DCLAInfo := ctx.OtherModuleProvider(dclaModules[0], DCLAInfoProvider).(DCLAInfo) + return DCLAInfo.ProvidedLibs + } + return []string{} +} + // These functions are interfacing with cc/sanitizer.go. The entire APEX (along with all of its // members) can be sanitized, either forcibly, or by the global configuration. For some of the // sanitizers, extra dependencies can be forcibly added as well. @@ -1657,7 +1708,7 @@ func apexFileForRustLibrary(ctx android.BaseModuleContext, rustm *rust.Module) a return newApexFile(ctx, fileToCopy, androidMkModuleName, dirInApex, nativeSharedLib, rustm) } -func apexFileForPyBinary(ctx android.BaseModuleContext, py *python.Module) apexFile { +func apexFileForPyBinary(ctx android.BaseModuleContext, py *python.PythonBinaryModule) apexFile { dirInApex := "bin" fileToCopy := py.HostToolPath().Path() return newApexFile(ctx, fileToCopy, py.BaseModuleName(), dirInApex, pyBinary, py) @@ -1922,11 +1973,9 @@ func (a *apexBundle) ProcessBazelQueryResponse(ctx android.ModuleContext) { a.containerCertificateFile = android.PathForBazelOut(ctx, outputs.ContainerKeyInfo[0]) a.containerPrivateKeyFile = android.PathForBazelOut(ctx, outputs.ContainerKeyInfo[1]) - // Ensure ApexInfo.RequiresLibs are installed as part of a bundle build - for _, bazelLabel := range outputs.RequiresLibs { - // convert Bazel label back to Soong module name - a.requiredDeps = append(a.requiredDeps, android.ModuleFromBazelLabel(bazelLabel)) - } + // Ensure ApexMkInfo.install_to_system make module names are installed as + // part of a bundled build. + a.makeModulesToInstall = append(a.makeModulesToInstall, outputs.MakeModulesToInstall...) apexType := a.properties.ApexType switch apexType { @@ -2025,7 +2074,7 @@ func (a *apexBundle) setApexTypeAndSuffix(ctx android.ModuleContext) { a.primaryApexType = true if ctx.Config().InstallExtraFlattenedApexes() { - a.requiredDeps = append(a.requiredDeps, a.Name()+flattenedSuffix) + a.makeModulesToInstall = append(a.makeModulesToInstall, a.Name()+flattenedSuffix) } } case zipApex: @@ -2147,7 +2196,7 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, case *cc.Module: vctx.filesInfo = append(vctx.filesInfo, apexFileForExecutable(ctx, ch)) return true // track transitive dependencies - case *python.Module: + case *python.PythonBinaryModule: if ch.HostToolPath().Valid() { vctx.filesInfo = append(vctx.filesInfo, apexFileForPyBinary(ctx, ch)) } @@ -2177,7 +2226,7 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, vctx.filesInfo = append(vctx.filesInfo, apexBootclasspathFragmentFiles(ctx, child)...) for _, makeModuleName := range bcpfModule.BootImageDeviceInstallMakeModules() { - a.requiredDeps = append(a.requiredDeps, makeModuleName) + a.makeModulesToInstall = append(a.makeModulesToInstall, makeModuleName) } return true case sscpfTag: @@ -2340,11 +2389,14 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, // // Always include if we are a host-apex however since those won't have any // system libraries. - if ch.IsStubsImplementationRequired() && !am.DirectlyInAnyApex() { + // + // Skip the dependency in unbundled builds where the device image is not + // being built. + if ch.IsStubsImplementationRequired() && !am.DirectlyInAnyApex() && !ctx.Config().UnbundledBuild() { // we need a module name for Make name := ch.ImplementationModuleNameForMake(ctx) + ch.Properties.SubName - if !android.InList(name, a.requiredDeps) { - a.requiredDeps = append(a.requiredDeps, name) + if !android.InList(name, a.makeModulesToInstall) { + a.makeModulesToInstall = append(a.makeModulesToInstall, name) } } vctx.requireNativeLibs = append(vctx.requireNativeLibs, af.stem()) @@ -2479,7 +2531,6 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { } //////////////////////////////////////////////////////////////////////////////////////////// // 2) traverse the dependency tree to collect apexFile structs from them. - // Collect the module directory for IDE info in java/jdeps.go. a.modulePaths = append(a.modulePaths, ctx.ModuleDir()) diff --git a/apex/apex_test.go b/apex/apex_test.go index 395da952f..ea068eeba 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -526,6 +526,7 @@ func TestBasicApex(t *testing.T) { data.Custom(&builder, ab.BaseModuleName(), "TARGET_", "", data) androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := mylib.myapex\n") ensureNotContains(t, androidMk, "LOCAL_MODULE := mylib.com.android.myapex\n") optFlags := apexRule.Args["opt_flags"] @@ -2995,7 +2996,7 @@ func TestAndroidMk_VendorApexRequired(t *testing.T) { var builder strings.Builder data.Custom(&builder, name, prefix, "", data) androidMk := builder.String() - ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := libc.vendor libm.vendor libdl.vendor\n") + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := libc++.vendor.myapex:64 mylib.vendor.myapex:64 apex_manifest.pb.myapex apex_pubkey.myapex libc.vendor libm.vendor libdl.vendor\n") } func TestAndroidMkWritesCommonProperties(t *testing.T) { @@ -4147,6 +4148,7 @@ func TestApexName(t *testing.T) { var builder strings.Builder data.Custom(&builder, name, prefix, "", data) androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := mylib.myapex\n") ensureNotContains(t, androidMk, "LOCAL_MODULE := mylib.com.android.myapex\n") } @@ -5680,6 +5682,12 @@ func TestApexWithTests(t *testing.T) { var builder strings.Builder data.Custom(&builder, name, prefix, "", data) androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := mytest.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := mytest1.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := mytest2.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := mytest3.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := apex_manifest.pb.myapex\n") + ensureContains(t, androidMk, "LOCAL_MODULE := apex_pubkey.myapex\n") ensureContains(t, androidMk, "LOCAL_MODULE := myapex\n") flatBundle := ctx.ModuleForTests("myapex", "android_common_myapex_flattened").Module().(*apexBundle) @@ -5708,12 +5716,12 @@ func TestInstallExtraFlattenedApexes(t *testing.T) { }), ) ab := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) - ensureListContains(t, ab.requiredDeps, "myapex.flattened") + ensureListContains(t, ab.makeModulesToInstall, "myapex.flattened") mk := android.AndroidMkDataForTest(t, ctx, ab) var builder strings.Builder mk.Custom(&builder, ab.Name(), "TARGET_", "", mk) androidMk := builder.String() - ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := myapex.flattened\n") + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := apex_manifest.pb.myapex apex_pubkey.myapex myapex.flattened\n") } func TestErrorsIfDepsAreNotEnabled(t *testing.T) { @@ -6506,6 +6514,12 @@ func TestOverrideApex(t *testing.T) { var builder strings.Builder data.Custom(&builder, name, "TARGET_", "", data) androidMk := builder.String() + ensureContains(t, androidMk, "LOCAL_MODULE := override_app.override_myapex") + ensureContains(t, androidMk, "LOCAL_MODULE := overrideBpf.o.override_myapex") + ensureContains(t, androidMk, "LOCAL_MODULE := apex_manifest.pb.override_myapex") + ensureContains(t, androidMk, "LOCAL_MODULE := override_bcplib.override_myapex") + ensureContains(t, androidMk, "LOCAL_MODULE := override_systemserverlib.override_myapex") + ensureContains(t, androidMk, "LOCAL_MODULE := override_java_library.override_myapex") ensureContains(t, androidMk, "LOCAL_MODULE_STEM := override_myapex.apex") ensureContains(t, androidMk, "LOCAL_OVERRIDES_MODULES := unknownapex myapex") ensureNotContains(t, androidMk, "LOCAL_MODULE := app.myapex") @@ -7097,7 +7111,7 @@ func TestCarryRequiredModuleNames(t *testing.T) { var builder strings.Builder data.Custom(&builder, name, prefix, "", data) androidMk := builder.String() - ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := a b\n") + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := mylib.myapex:64 apex_manifest.pb.myapex apex_pubkey.myapex a b\n") ensureContains(t, androidMk, "LOCAL_HOST_REQUIRED_MODULES := c d\n") ensureContains(t, androidMk, "LOCAL_TARGET_REQUIRED_MODULES := e f\n") } @@ -7268,9 +7282,6 @@ func TestSymlinksFromApexToSystemRequiredModuleNames(t *testing.T) { "myapex", "//apex_available:platform", ], - stubs: { - versions: ["current"], - }, } `) @@ -7280,10 +7291,11 @@ func TestSymlinksFromApexToSystemRequiredModuleNames(t *testing.T) { data.Custom(&builder, apexBundle.BaseModuleName(), "TARGET_", "", data) androidMk := builder.String() // `myotherlib` is added to `myapex` as symlink + ensureContains(t, androidMk, "LOCAL_MODULE := mylib.myapex\n") ensureNotContains(t, androidMk, "LOCAL_MODULE := prebuilt_myotherlib.myapex\n") ensureNotContains(t, androidMk, "LOCAL_MODULE := myotherlib.myapex\n") // `myapex` should have `myotherlib` in its required line, not `prebuilt_myotherlib` - ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := myotherlib\n") + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := mylib.myapex:64 myotherlib:64 apex_manifest.pb.myapex apex_pubkey.myapex\n") } func TestApexWithJniLibs(t *testing.T) { @@ -8796,7 +8808,7 @@ func TestPreferredPrebuiltSharedLibDep(t *testing.T) { // The make level dependency needs to be on otherlib - prebuilt_otherlib isn't // a thing there. - ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := otherlib\n") + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := libc++:64 mylib.myapex:64 apex_manifest.pb.myapex apex_pubkey.myapex otherlib\n") } func TestExcludeDependency(t *testing.T) { @@ -9190,7 +9202,7 @@ func TestAndroidMk_DexpreoptBuiltInstalledForApex(t *testing.T) { var builder strings.Builder data.Custom(&builder, apexBundle.BaseModuleName(), "TARGET_", "", data) androidMk := builder.String() - ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := foo-dexpreopt-arm64-apex@myapex@javalib@foo.jar@classes.odex foo-dexpreopt-arm64-apex@myapex@javalib@foo.jar@classes.vdex\n") + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := foo.myapex apex_manifest.pb.myapex apex_pubkey.myapex foo-dexpreopt-arm64-apex@myapex@javalib@foo.jar@classes.odex foo-dexpreopt-arm64-apex@myapex@javalib@foo.jar@classes.vdex\n") } func TestAndroidMk_DexpreoptBuiltInstalledForApex_Prebuilt(t *testing.T) { @@ -9266,7 +9278,7 @@ func TestAndroidMk_RequiredModules(t *testing.T) { var builder strings.Builder data.Custom(&builder, apexBundle.BaseModuleName(), "TARGET_", "", data) androidMk := builder.String() - ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := otherapex") + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := foo.myapex apex_manifest.pb.myapex apex_pubkey.myapex otherapex") } func TestAndroidMk_RequiredDeps(t *testing.T) { @@ -9285,15 +9297,15 @@ func TestAndroidMk_RequiredDeps(t *testing.T) { `) bundle := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle) - bundle.requiredDeps = append(bundle.requiredDeps, "foo") + bundle.makeModulesToInstall = append(bundle.makeModulesToInstall, "foo") data := android.AndroidMkDataForTest(t, ctx, bundle) var builder strings.Builder data.Custom(&builder, bundle.BaseModuleName(), "TARGET_", "", data) androidMk := builder.String() - ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := foo\n") + ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := apex_manifest.pb.myapex apex_pubkey.myapex foo\n") flattenedBundle := ctx.ModuleForTests("myapex", "android_common_myapex_flattened").Module().(*apexBundle) - flattenedBundle.requiredDeps = append(flattenedBundle.requiredDeps, "foo") + flattenedBundle.makeModulesToInstall = append(flattenedBundle.makeModulesToInstall, "foo") flattenedData := android.AndroidMkDataForTest(t, ctx, flattenedBundle) var flattenedBuilder strings.Builder flattenedData.Custom(&flattenedBuilder, flattenedBundle.BaseModuleName(), "TARGET_", "", flattenedData) @@ -9537,7 +9549,7 @@ func TestSdkLibraryCanHaveHigherMinSdkVersion(t *testing.T) { func ensureContainsRequiredDeps(t *testing.T, ctx *android.TestContext, moduleName, variant string, deps []string) { a := ctx.ModuleForTests(moduleName, variant).Module().(*apexBundle) for _, dep := range deps { - android.AssertStringListContains(t, "", a.requiredDeps, dep) + android.AssertStringListContains(t, "", a.makeModulesToInstall, dep) } } @@ -9545,7 +9557,7 @@ func ensureContainsRequiredDeps(t *testing.T, ctx *android.TestContext, moduleNa func ensureDoesNotContainRequiredDeps(t *testing.T, ctx *android.TestContext, moduleName, variant string, deps []string) { a := ctx.ModuleForTests(moduleName, variant).Module().(*apexBundle) for _, dep := range deps { - android.AssertStringListDoesNotContain(t, "", a.requiredDeps, dep) + android.AssertStringListDoesNotContain(t, "", a.makeModulesToInstall, dep) } } diff --git a/apex/bp2build_test.go b/apex/bp2build_test.go index 01afa521b..2f2b61e64 100644 --- a/apex/bp2build_test.go +++ b/apex/bp2build_test.go @@ -42,6 +42,7 @@ apex { OutputBaseDir: outputBaseDir, LabelToApexInfo: map[string]cquery.ApexInfo{ "//:foo": cquery.ApexInfo{ + // ApexInfo Starlark provider. SignedOutput: "signed_out.apex", SignedCompressedOutput: "signed_out.capex", UnsignedOutput: "unsigned_out.apex", @@ -56,6 +57,9 @@ apex { // unused PackageName: "pkg_name", ProvidesLibs: []string{"a", "b"}, + + // ApexMkInfo Starlark provider + MakeModulesToInstall: []string{"c"}, // d deliberately omitted }, }, } @@ -111,7 +115,12 @@ apex { if w := "$(call dist-for-goals,checkbuild,out/bazel/execroot/__main__/installed-files.txt:foo-installed-files.txt)"; !strings.Contains(data, w) { t.Errorf("Expected %q in androidmk data, but did not find %q", w, data) } - if w := "LOCAL_REQUIRED_MODULES := c d"; !strings.Contains(data, w) { + + // make modules to be installed to system + if len(ab.makeModulesToInstall) != 1 && ab.makeModulesToInstall[0] != "c" { + t.Errorf("Expected makeModulesToInstall slice to only contain 'c', got %q", ab.makeModulesToInstall) + } + if w := "LOCAL_REQUIRED_MODULES := c"; !strings.Contains(data, w) { t.Errorf("Expected %q in androidmk data, but did not find it in %q", w, data) } } @@ -212,6 +221,7 @@ override_apex { OutputBaseDir: outputBaseDir, LabelToApexInfo: map[string]cquery.ApexInfo{ "//:foo": cquery.ApexInfo{ + // ApexInfo Starlark provider SignedOutput: "signed_out.apex", UnsignedOutput: "unsigned_out.apex", BundleKeyInfo: []string{"public_key", "private_key"}, @@ -225,8 +235,12 @@ override_apex { // unused PackageName: "pkg_name", ProvidesLibs: []string{"a", "b"}, + + // ApexMkInfo Starlark provider + MakeModulesToInstall: []string{"c"}, // d deliberately omitted }, "//:override_foo": cquery.ApexInfo{ + // ApexInfo Starlark provider SignedOutput: "override_signed_out.apex", UnsignedOutput: "override_unsigned_out.apex", BundleKeyInfo: []string{"override_public_key", "override_private_key"}, @@ -240,6 +254,9 @@ override_apex { // unused PackageName: "override_pkg_name", ProvidesLibs: []string{"a", "b"}, + + // ApexMkInfo Starlark provider + MakeModulesToInstall: []string{"c"}, // d deliberately omitted }, }, } @@ -295,7 +312,12 @@ override_apex { if w := "$(call dist-for-goals,checkbuild,out/bazel/execroot/__main__/override_installed-files.txt:override_foo-installed-files.txt)"; !strings.Contains(data, w) { t.Errorf("Expected %q in androidmk data, but did not find %q", w, data) } - if w := "LOCAL_REQUIRED_MODULES := c d"; !strings.Contains(data, w) { + + // make modules to be installed to system + if len(ab.makeModulesToInstall) != 1 && ab.makeModulesToInstall[0] != "c" { + t.Errorf("Expected makeModulestoInstall slice to only contain 'c', got %q", ab.makeModulesToInstall) + } + if w := "LOCAL_REQUIRED_MODULES := c"; !strings.Contains(data, w) { t.Errorf("Expected %q in androidmk data, but did not find it in %q", w, data) } } diff --git a/apex/builder.go b/apex/builder.go index 18d0836b6..4331d3ec2 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -40,6 +40,8 @@ func init() { pctx.Import("android/soong/java") pctx.HostBinToolVariable("apexer", "apexer") pctx.HostBinToolVariable("apexer_with_DCLA_preprocessing", "apexer_with_DCLA_preprocessing") + pctx.HostBinToolVariable("apexer_with_trim_preprocessing", "apexer_with_trim_preprocessing") + // ART minimal builds (using the master-art manifest) do not have the "frameworks/base" // projects, and hence cannot build 'aapt2'. Use the SDK prebuilt instead. hostBinToolVariableWithPrebuilt := func(name, prebuiltDir, tool string) { @@ -146,6 +148,34 @@ var ( }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest", "is_DCLA") + TrimmedApexRule = pctx.StaticRule("TrimmedApexRule", blueprint.RuleParams{ + Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` + + `(. ${out}.copy_commands) && ` + + `APEXER_TOOL_PATH=${tool_path} ` + + `${apexer_with_trim_preprocessing} ` + + `--apexer ${apexer} ` + + `--canned_fs_config ${canned_fs_config} ` + + `--manifest ${manifest} ` + + `--libs_to_trim ${libs_to_trim} ` + + `${image_dir} ` + + `${out} ` + + `-- ` + + `--include_build_info ` + + `--force ` + + `--payload_type image ` + + `--key ${key} ` + + `--file_contexts ${file_contexts} ` + + `${opt_flags} `, + CommandDeps: []string{"${apexer_with_trim_preprocessing}", "${apexer}", "${avbtool}", "${e2fsdroid}", + "${merge_zips}", "${mke2fs}", "${resize2fs}", "${sefcontext_compile}", "${make_f2fs}", + "${sload_f2fs}", "${make_erofs}", "${soong_zip}", "${zipalign}", "${aapt2}", + "prebuilts/sdk/current/public/android.jar"}, + Rspfile: "${out}.copy_commands", + RspfileContent: "${copy_commands}", + Description: "APEX ${image_dir} => ${out}", + }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", + "opt_flags", "manifest", "libs_to_trim") + zipApexRule = pctx.StaticRule("zipApexRule", blueprint.RuleParams{ Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` + `(. ${out}.copy_commands) && ` + @@ -706,6 +736,24 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { "opt_flags": strings.Join(optFlags, " "), }, }) + } else if ctx.Config().ApexTrimEnabled() && len(a.libs_to_trim(ctx)) > 0 { + ctx.Build(pctx, android.BuildParams{ + Rule: TrimmedApexRule, + Implicits: implicitInputs, + Output: unsignedOutputFile, + Description: "apex (" + apexType.name() + ")", + Args: map[string]string{ + "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir, + "image_dir": imageDir.String(), + "copy_commands": strings.Join(copyCommands, " && "), + "manifest": a.manifestPbOut.String(), + "file_contexts": fileContexts.String(), + "canned_fs_config": cannedFsConfig.String(), + "key": a.privateKeyFile.String(), + "opt_flags": strings.Join(optFlags, " "), + "libs_to_trim": strings.Join(a.libs_to_trim(ctx), ","), + }, + }) } else { ctx.Build(pctx, android.BuildParams{ Rule: apexRule, diff --git a/apex/vndk.go b/apex/vndk.go index ef3e5e1c5..80560cf0c 100644 --- a/apex/vndk.go +++ b/apex/vndk.go @@ -65,8 +65,23 @@ func apexVndkMutator(mctx android.TopDownMutatorContext) { } vndkVersion := ab.vndkVersion(mctx.DeviceConfig()) + // Ensure VNDK APEX mount point is formatted as com.android.vndk.v### ab.properties.Apex_name = proptools.StringPtr(vndkApexNamePrefix + vndkVersion) + + apiLevel, err := android.ApiLevelFromUser(mctx, vndkVersion) + if err != nil { + mctx.PropertyErrorf("vndk_version", "%s", err.Error()) + return + } + + targets := mctx.MultiTargets() + if len(targets) > 0 && apiLevel.LessThan(cc.MinApiForArch(mctx, targets[0].Arch.ArchType)) { + // Disable VNDK apexes for VNDK versions less than the minimum supported API level for the primary + // architecture. + ab.Disable() + } + } } diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go index 950ad982d..665419100 100644 --- a/bazel/cquery/request_type.go +++ b/bazel/cquery/request_type.go @@ -257,6 +257,10 @@ signed_compressed_output = "" # no .capex if the apex is not compressible, canno if info.signed_compressed_output: signed_compressed_output = info.signed_compressed_output.path +mk_info = providers(target).get("//build/bazel/rules/apex:apex_info.bzl%ApexMkInfo") +if not mk_info: + fail("%s did not provide ApexMkInfo" % id_string) + return json_encode({ "signed_output": info.signed_output.path, "signed_compressed_output": signed_compressed_output, @@ -271,10 +275,12 @@ return json_encode({ "backing_libs": info.backing_libs.path, "bundle_file": info.base_with_config_zip.path, "installed_files": info.installed_files.path, + "make_modules_to_install": mk_info.make_modules_to_install, })` } type ApexInfo struct { + // From the ApexInfo provider SignedOutput string `json:"signed_output"` SignedCompressedOutput string `json:"signed_compressed_output"` UnsignedOutput string `json:"unsigned_output"` @@ -288,6 +294,9 @@ type ApexInfo struct { BackingLibs string `json:"backing_libs"` BundleFile string `json:"bundle_file"` InstalledFiles string `json:"installed_files"` + + // From the ApexMkInfo provider + MakeModulesToInstall []string `json:"make_modules_to_install"` } // ParseResult returns a value obtained by parsing the result of the request's Starlark function. diff --git a/bazel/cquery/request_type_test.go b/bazel/cquery/request_type_test.go index 1d30535f0..7003ce193 100644 --- a/bazel/cquery/request_type_test.go +++ b/bazel/cquery/request_type_test.go @@ -177,9 +177,11 @@ func TestGetApexInfoParseResults(t *testing.T) { "backing_libs":"path/to/backing.txt", "bundle_file": "dir/bundlefile.zip", "installed_files":"path/to/installed-files.txt", - "provides_native_libs":[] + "provides_native_libs":[], + "make_modules_to_install": ["foo","bar"] }`, expectedOutput: ApexInfo{ + // ApexInfo SignedOutput: "my.apex", UnsignedOutput: "my.apex.unsigned", RequiresLibs: []string{"//bionic/libc:libc", "//bionic/libdl:libdl"}, @@ -191,6 +193,9 @@ func TestGetApexInfoParseResults(t *testing.T) { BackingLibs: "path/to/backing.txt", BundleFile: "dir/bundlefile.zip", InstalledFiles: "path/to/installed-files.txt", + + // ApexMkInfo + MakeModulesToInstall: []string{"foo", "bar"}, }, }, } diff --git a/bazel/properties.go b/bazel/properties.go index 0fca60b34..9be21ebc7 100644 --- a/bazel/properties.go +++ b/bazel/properties.go @@ -152,7 +152,7 @@ func (ll *LabelList) Append(other LabelList) { ll.Includes = append(ll.Includes, other.Includes...) } if len(ll.Excludes) > 0 || len(other.Excludes) > 0 { - ll.Excludes = append(other.Excludes, other.Excludes...) + ll.Excludes = append(ll.Excludes, other.Excludes...) } } @@ -888,7 +888,7 @@ func (lla *LabelListAttribute) ResolveExcludes() { // then remove all config-specific excludes allLabels := baseLabels.deepCopy() allLabels.Append(val) - lla.ConfigurableValues[axis][config] = SubtractBazelLabelList(allLabels, LabelList{Includes: val.Excludes}) + lla.ConfigurableValues[axis][config] = SubtractBazelLabelList(allLabels, LabelList{Includes: allLabels.Excludes}) } // After going through all configs, delete the duplicates in the config diff --git a/bazel/properties_test.go b/bazel/properties_test.go index 8729381b5..cf03eb5f6 100644 --- a/bazel/properties_test.go +++ b/bazel/properties_test.go @@ -231,6 +231,7 @@ func TestResolveExcludes(t *testing.T) { "all_include", "arm_exclude", "android_exclude", + "product_config_exclude", }, []string{"all_exclude"}, ), @@ -251,10 +252,10 @@ func TestResolveExcludes(t *testing.T) { "a": makeLabelList([]string{}, []string{"not_in_value"}), "b": makeLabelList([]string{"b_val"}, []string{}), "c": makeLabelList([]string{"c_val"}, []string{}), - ConditionsDefaultConfigKey: makeLabelList([]string{"c_val", "default", "default2"}, []string{}), + ConditionsDefaultConfigKey: makeLabelList([]string{"c_val", "default", "default2", "all_exclude"}, []string{}), }, ProductVariableConfigurationAxis("product_only_with_excludes", NoConfigAxis): labelListSelectValues{ - "a": makeLabelList([]string{}, []string{"not_in_value"}), + "a": makeLabelList([]string{}, []string{"product_config_exclude"}), }, }, } @@ -287,6 +288,10 @@ func TestResolveExcludes(t *testing.T) { "c": makeLabels("c_val"), ConditionsDefaultConfigKey: makeLabels("c_val", "default", "default2"), }, + ProductVariableConfigurationAxis("product_only_with_excludes", NoConfigAxis): { + "a": nilLabels, + ConditionsDefaultConfigKey: makeLabels("product_config_exclude"), + }, } for _, axis := range attr.SortedConfigurationAxes() { if _, ok := expectedConfiguredIncludes[axis]; !ok { diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 052bc324c..244ca9c2c 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -3437,23 +3437,23 @@ cc_library { ExpectedBazelTargets: []string{ MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{ "implementation_deps": `[":baz__BP2BUILD__MISSING__DEP"] + select({ - "//build/bazel/rules/apex:non_apex": [":buh__BP2BUILD__MISSING__DEP"], - "//conditions:default": [], + "//build/bazel/rules/apex:in_apex": [], + "//conditions:default": [":buh__BP2BUILD__MISSING__DEP"], })`, "implementation_dynamic_deps": `[":baz__BP2BUILD__MISSING__DEP"] + select({ - "//build/bazel/rules/apex:non_apex": [":bar__BP2BUILD__MISSING__DEP"], - "//conditions:default": [], + "//build/bazel/rules/apex:in_apex": [], + "//conditions:default": [":bar__BP2BUILD__MISSING__DEP"], })`, "local_includes": `["."]`, }), MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{ "implementation_deps": `[":baz__BP2BUILD__MISSING__DEP"] + select({ - "//build/bazel/rules/apex:non_apex": [":buh__BP2BUILD__MISSING__DEP"], - "//conditions:default": [], + "//build/bazel/rules/apex:in_apex": [], + "//conditions:default": [":buh__BP2BUILD__MISSING__DEP"], })`, "implementation_dynamic_deps": `[":baz__BP2BUILD__MISSING__DEP"] + select({ - "//build/bazel/rules/apex:non_apex": [":bar__BP2BUILD__MISSING__DEP"], - "//conditions:default": [], + "//build/bazel/rules/apex:in_apex": [], + "//conditions:default": [":bar__BP2BUILD__MISSING__DEP"], })`, "local_includes": `["."]`, }), @@ -3483,16 +3483,16 @@ cc_library_static { ExpectedBazelTargets: []string{ MakeBazelTarget("cc_library_static", "foo", AttrNameToString{ "implementation_dynamic_deps": `select({ - "//build/bazel/rules/apex:non_apex": [":bar__BP2BUILD__MISSING__DEP"], - "//conditions:default": [], + "//build/bazel/rules/apex:in_apex": [], + "//conditions:default": [":bar__BP2BUILD__MISSING__DEP"], })`, "dynamic_deps": `select({ - "//build/bazel/rules/apex:non_apex": [":baz__BP2BUILD__MISSING__DEP"], - "//conditions:default": [], + "//build/bazel/rules/apex:in_apex": [], + "//conditions:default": [":baz__BP2BUILD__MISSING__DEP"], })`, "deps": `select({ - "//build/bazel/rules/apex:non_apex": [":abc__BP2BUILD__MISSING__DEP"], - "//conditions:default": [], + "//build/bazel/rules/apex:in_apex": [], + "//conditions:default": [":abc__BP2BUILD__MISSING__DEP"], })`, "local_includes": `["."]`, }), diff --git a/bp2build/java_library_conversion_test.go b/bp2build/java_library_conversion_test.go index 93a617401..0784f4b97 100644 --- a/bp2build/java_library_conversion_test.go +++ b/bp2build/java_library_conversion_test.go @@ -715,3 +715,43 @@ func TestJavaLibraryKotlinCommonSrcs(t *testing.T) { }, }) } + +func TestJavaLibraryArchVariantLibs(t *testing.T) { + runJavaLibraryTestCase(t, Bp2buildTestCase{ + Description: "java_library with arch variant libs", + Blueprint: `java_library { + name: "java-lib-1", + srcs: ["a.java"], + libs: ["java-lib-2"], + target: { + android: { + libs: ["java-lib-3"], + }, + }, + bazel_module: { bp2build_available: true }, +} + + java_library{ + name: "java-lib-2", +} + + java_library{ + name: "java-lib-3", +} +`, + ExpectedBazelTargets: []string{ + MakeBazelTarget("java_library", "java-lib-1", AttrNameToString{ + "srcs": `["a.java"]`, + "deps": `[":java-lib-2-neverlink"] + select({ + "//build/bazel/platforms/os:android": [":java-lib-3-neverlink"], + "//conditions:default": [], + })`, + }), + MakeNeverlinkDuplicateTarget("java_library", "java-lib-1"), + MakeBazelTarget("java_library", "java-lib-2", AttrNameToString{}), + MakeNeverlinkDuplicateTarget("java_library", "java-lib-2"), + MakeBazelTarget("java_library", "java-lib-3", AttrNameToString{}), + MakeNeverlinkDuplicateTarget("java_library", "java-lib-3"), + }, + }) +} diff --git a/bp2build/metrics.go b/bp2build/metrics.go index d6e5cf3e8..7e29fac58 100644 --- a/bp2build/metrics.go +++ b/bp2build/metrics.go @@ -188,6 +188,10 @@ const ( ) func (metrics *CodegenMetrics) AddConvertedModule(m blueprint.Module, moduleType string, dir string, conversionType ConversionType) { + //a package module has empty name + if moduleType == "package" { + return + } // Undo prebuilt_ module name prefix modifications moduleName := android.RemoveOptionalPrebuiltPrefix(m.Name()) metrics.serialized.ConvertedModules = append(metrics.serialized.ConvertedModules, moduleName) diff --git a/bp2build/soong_config_module_type_conversion_test.go b/bp2build/soong_config_module_type_conversion_test.go index 89be767dc..ba42f34df 100644 --- a/bp2build/soong_config_module_type_conversion_test.go +++ b/bp2build/soong_config_module_type_conversion_test.go @@ -742,6 +742,101 @@ cc_library { name: "lib_default", bazel_module: { bp2build_available: false } } )`}}) } +func TestSoongConfigModuleType_UnsetConditionsExcludeLibs(t *testing.T) { + bp := ` +soong_config_string_variable { + name: "library_linking_strategy", + values: [ + "prefer_static", + ], +} + +soong_config_module_type { + name: "library_linking_strategy_cc_defaults", + module_type: "cc_defaults", + config_namespace: "ANDROID", + variables: ["library_linking_strategy"], + properties: ["shared_libs"], +} + +library_linking_strategy_cc_defaults { + name: "library_linking_strategy_lib_a_defaults", + soong_config_variables: { + library_linking_strategy: { + prefer_static: {}, + conditions_default: { + shared_libs: [ + "lib_a", + ], + }, + }, + }, +} + +library_linking_strategy_cc_defaults { + name: "library_linking_strategy_merged_defaults", + defaults: ["library_linking_strategy_lib_a_defaults"], + host_supported: true, + soong_config_variables: { + library_linking_strategy: { + prefer_static: {}, + conditions_default: { + shared_libs: [ + "lib_b", + "lib_c", + ], + }, + }, + }, + exclude_shared_libs: ["lib_a"], +} + +cc_binary { + name: "library_linking_strategy_sample_binary", + defaults: ["library_linking_strategy_merged_defaults"], + include_build_directory: false, +} + +cc_binary { + name: "library_linking_strategy_sample_binary_with_excludes", + defaults: ["library_linking_strategy_merged_defaults"], + exclude_shared_libs: ["lib_c"], + include_build_directory: false, +}` + + otherDeps := ` +cc_library { name: "lib_a", bazel_module: { bp2build_available: false } } +cc_library { name: "lib_b", bazel_module: { bp2build_available: false } } +cc_library { name: "lib_c", bazel_module: { bp2build_available: false } } +` + + runSoongConfigModuleTypeTest(t, Bp2buildTestCase{ + Description: "soong config variables - generates selects for library_linking_strategy", + ModuleTypeUnderTest: "cc_binary", + ModuleTypeUnderTestFactory: cc.BinaryFactory, + Blueprint: bp, + Filesystem: map[string]string{ + "foo/bar/Android.bp": otherDeps, + }, + ExpectedBazelTargets: []string{ + MakeBazelTargetNoRestrictions("cc_binary", "library_linking_strategy_sample_binary", AttrNameToString{ + "dynamic_deps": `select({ + "//build/bazel/product_variables:android__library_linking_strategy__prefer_static": [], + "//conditions:default": [ + "//foo/bar:lib_b", + "//foo/bar:lib_c", + ], + })`, + }), + MakeBazelTargetNoRestrictions("cc_binary", "library_linking_strategy_sample_binary_with_excludes", AttrNameToString{ + "dynamic_deps": `select({ + "//build/bazel/product_variables:android__library_linking_strategy__prefer_static": [], + "//conditions:default": ["//foo/bar:lib_b"], + })`, + }), + }}) +} + func TestSoongConfigModuleType_Defaults(t *testing.T) { bp := ` soong_config_string_variable { diff --git a/bp2build/testing.go b/bp2build/testing.go index c340a8f72..92a9bf1b5 100644 --- a/bp2build/testing.go +++ b/bp2build/testing.go @@ -258,6 +258,7 @@ func (b BazelTestResult) CompareAllBazelTargets(t *testing.T, description string } func (b BazelTestResult) CompareBazelTargets(t *testing.T, description string, expectedContents []string, actualTargets BazelTargets) { + t.Helper() if actualCount, expectedCount := len(actualTargets), len(expectedContents); actualCount != expectedCount { t.Errorf("%s: Expected %d bazel target (%s), got %d (%s)", description, expectedCount, expectedContents, actualCount, actualTargets) diff --git a/cc/api_level.go b/cc/api_level.go index fdff5cbcf..a5571f31f 100644 --- a/cc/api_level.go +++ b/cc/api_level.go @@ -20,7 +20,9 @@ import ( "android/soong/android" ) -func minApiForArch(ctx android.EarlyModuleContext, +// MinApiLevelForArch returns the ApiLevel for the Android version that +// first supported the architecture. +func MinApiForArch(ctx android.EarlyModuleContext, arch android.ArchType) android.ApiLevel { switch arch { @@ -38,7 +40,7 @@ func minApiForArch(ctx android.EarlyModuleContext, func nativeApiLevelFromUser(ctx android.BaseModuleContext, raw string) (android.ApiLevel, error) { - min := minApiForArch(ctx, ctx.Arch().ArchType) + min := MinApiForArch(ctx, ctx.Arch().ArchType) if raw == "minimum" { return min, nil } diff --git a/cc/bp2build.go b/cc/bp2build.go index aea1fa188..35419af7a 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -979,34 +979,22 @@ var ( // resolveTargetApex re-adds the shared and static libs in target.apex.exclude_shared|static_libs props to non-apex variant // since all libs are already excluded by default -func (la *linkerAttributes) resolveTargetApexProp(ctx android.BazelConversionPathContext, isBinary bool, props *BaseLinkerProperties) { - sharedLibsForNonApex := maybePartitionExportedAndImplementationsDeps( - ctx, - true, - props.Target.Apex.Exclude_shared_libs, - props.Export_shared_lib_headers, - bazelLabelForSharedDeps, - ) - dynamicDeps := la.dynamicDeps.SelectValue(bazel.InApexAxis, bazel.NonApex) - implDynamicDeps := la.implementationDynamicDeps.SelectValue(bazel.InApexAxis, bazel.NonApex) - (&dynamicDeps).Append(sharedLibsForNonApex.export) - (&implDynamicDeps).Append(sharedLibsForNonApex.implementation) - la.dynamicDeps.SetSelectValue(bazel.InApexAxis, bazel.NonApex, dynamicDeps) - la.implementationDynamicDeps.SetSelectValue(bazel.InApexAxis, bazel.NonApex, implDynamicDeps) - - staticLibsForNonApex := maybePartitionExportedAndImplementationsDeps( - ctx, - !isBinary, - props.Target.Apex.Exclude_static_libs, - props.Export_static_lib_headers, - bazelLabelForSharedDeps, - ) - deps := la.deps.SelectValue(bazel.InApexAxis, bazel.NonApex) - implDeps := la.implementationDeps.SelectValue(bazel.InApexAxis, bazel.NonApex) - (&deps).Append(staticLibsForNonApex.export) - (&implDeps).Append(staticLibsForNonApex.implementation) - la.deps.SetSelectValue(bazel.InApexAxis, bazel.NonApex, deps) - la.implementationDeps.SetSelectValue(bazel.InApexAxis, bazel.NonApex, implDeps) +func (la *linkerAttributes) resolveTargetApexProp(ctx android.BazelConversionPathContext, props *BaseLinkerProperties) { + excludeSharedLibs := bazelLabelForSharedDeps(ctx, props.Target.Apex.Exclude_shared_libs) + sharedExcludes := bazel.LabelList{Excludes: excludeSharedLibs.Includes} + sharedExcludesLabelList := bazel.LabelListAttribute{} + sharedExcludesLabelList.SetSelectValue(bazel.InApexAxis, bazel.InApex, sharedExcludes) + + la.dynamicDeps.Append(sharedExcludesLabelList) + la.implementationDynamicDeps.Append(sharedExcludesLabelList) + + excludeStaticLibs := bazelLabelForStaticDeps(ctx, props.Target.Apex.Exclude_static_libs) + staticExcludes := bazel.LabelList{Excludes: excludeStaticLibs.Includes} + staticExcludesLabelList := bazel.LabelListAttribute{} + staticExcludesLabelList.SetSelectValue(bazel.InApexAxis, bazel.InApex, staticExcludes) + + la.deps.Append(staticExcludesLabelList) + la.implementationDeps.Append(staticExcludesLabelList) } func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversionPathContext, isBinary bool, axis bazel.ConfigurationAxis, config string, props *BaseLinkerProperties) { @@ -1040,8 +1028,7 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion ctx, !isBinary, staticLibs, - // Exclude static libs in Exclude_static_libs and Target.Apex.Exclude_static_libs props - append(props.Exclude_static_libs, props.Target.Apex.Exclude_static_libs...), + props.Exclude_static_libs, props.Export_static_lib_headers, bazelLabelForStaticDepsExcludes, ) @@ -1080,14 +1067,13 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion ctx, !isBinary, sharedLibs, - // Exclude shared libs in Exclude_shared_libs and Target.Apex.Exclude_shared_libs props - append(props.Exclude_shared_libs, props.Target.Apex.Exclude_shared_libs...), + props.Exclude_shared_libs, props.Export_shared_lib_headers, bazelLabelForSharedDepsExcludes, ) la.dynamicDeps.SetSelectValue(axis, config, sharedDeps.export) la.implementationDynamicDeps.SetSelectValue(axis, config, sharedDeps.implementation) - la.resolveTargetApexProp(ctx, isBinary, props) + la.resolveTargetApexProp(ctx, props) if axis == bazel.NoConfigAxis || (axis == bazel.OsConfigurationAxis && config == bazel.OsAndroid) { // If a dependency in la.implementationDynamicDeps has stubs, its stub variant should be @@ -1445,6 +1445,8 @@ func isBionic(name string) bool { } func InstallToBootstrap(name string, config android.Config) bool { + // NOTE: also update //build/bazel/rules/apex/cc.bzl#_installed_to_bootstrap + // if this list is updated. if name == "libclang_rt.hwasan" { return true } @@ -2225,6 +2227,13 @@ func GetCrtVariations(ctx android.BottomUpMutatorContext, if err != nil { ctx.PropertyErrorf("min_sdk_version", err.Error()) } + + // Raise the minSdkVersion to the minimum supported for the architecture. + minApiForArch := MinApiForArch(ctx, m.Target().Arch.ArchType) + if apiLevel.LessThan(minApiForArch) { + apiLevel = minApiForArch + } + return []blueprint.Variation{ {Mutator: "sdk", Variation: "sdk"}, {Mutator: "version", Variation: apiLevel.String()}, @@ -3706,7 +3715,7 @@ func (c *Module) ShouldSupportSdkVersion(ctx android.BaseModuleContext, // This allows introducing new architectures in the platform that // need to be included in apexes that normally require an older // min_sdk_version. - minApiForArch := minApiForArch(ctx, c.Target().Arch.ArchType) + minApiForArch := MinApiForArch(ctx, c.Target().Arch.ArchType) if sdkVersion.LessThan(minApiForArch) { sdkVersion = minApiForArch } diff --git a/cc/config/global.go b/cc/config/global.go index 2205c9e48..d557c0b01 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -386,30 +386,17 @@ func init() { return strings.Join(deviceGlobalCflags, " ") }) - // Export the static default NoOverrideGlobalCflags and NoOverride64GlobalCflags to Bazel. + // Export the static default NoOverrideGlobalCflags to Bazel. exportedVars.ExportStringList("NoOverrideGlobalCflags", noOverrideGlobalCflags) - exportedVars.ExportStringList("NoOverride64GlobalCflags", noOverride64GlobalCflags) pctx.VariableFunc("NoOverrideGlobalCflags", func(ctx android.PackageVarContext) string { flags := noOverrideGlobalCflags if ctx.Config().IsEnvTrue("LLVM_NEXT") { flags = append(noOverrideGlobalCflags, llvmNextExtraCommonGlobalCflags...) - if ctx.Config().Android64() { - flags = append(noOverride64GlobalCflags) - } - } - return strings.Join(flags, " ") - }) - - // Export the static default NoOverride64GlobalCflags to Bazel. - exportedVars.ExportStringList("NoOverride64GlobalCflags", noOverride64GlobalCflags) - pctx.VariableFunc("NoOverride64GlobalCflags", func(ctx android.PackageVarContext) string { - flags := noOverride64GlobalCflags - if ctx.Config().IsEnvTrue("LLVM_NEXT") && ctx.Config().Android64() { - flags = append(noOverride64GlobalCflags, llvmNextExtraCommonGlobalCflags...) } return strings.Join(flags, " ") }) + exportedVars.ExportStringListStaticVariable("NoOverride64GlobalCflags", noOverride64GlobalCflags) exportedVars.ExportStringListStaticVariable("HostGlobalCflags", hostGlobalCflags) exportedVars.ExportStringListStaticVariable("NoOverrideExternalGlobalCflags", noOverrideExternalGlobalCflags) exportedVars.ExportStringListStaticVariable("CommonGlobalCppflags", commonGlobalCppflags) diff --git a/java/dex.go b/java/dex.go index 971da925e..b6fe10982 100644 --- a/java/dex.go +++ b/java/dex.go @@ -90,6 +90,7 @@ type dexer struct { // list of extra proguard flag files extraProguardFlagFiles android.Paths proguardDictionary android.OptionalPath + proguardConfiguration android.OptionalPath proguardUsageZip android.OptionalPath providesTransitiveHeaderJars @@ -133,17 +134,18 @@ var d8, d8RE = pctx.MultiCommandRemoteStaticRules("d8", var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + - `rm -f "$outDict" && rm -rf "${outUsageDir}" && ` + + `rm -f "$outDict" && rm -f "$outConfig" && rm -rf "${outUsageDir}" && ` + `mkdir -p $$(dirname ${outUsage}) && ` + `mkdir -p $$(dirname $tmpJar) && ` + `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` + `$r8Template${config.R8Cmd} ${config.R8Flags} -injars $tmpJar --output $outDir ` + `--no-data-resources ` + `-printmapping ${outDict} ` + + `--pg-conf-output ${outConfig} ` + `-printusage ${outUsage} ` + `--deps-file ${out}.d ` + `$r8Flags && ` + - `touch "${outDict}" "${outUsage}" && ` + + `touch "${outDict}" "${outConfig}" "${outUsage}" && ` + `${config.SoongZipCmd} -o ${outUsageZip} -C ${outUsageDir} -f ${outUsage} && ` + `rm -rf ${outUsageDir} && ` + `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + @@ -179,7 +181,7 @@ var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", ExecStrategy: "${config.RER8ExecStrategy}", Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, }, - }, []string{"outDir", "outDict", "outUsage", "outUsageZip", "outUsageDir", + }, []string{"outDir", "outDict", "outConfig", "outUsage", "outUsageZip", "outUsageDir", "r8Flags", "zipFlags", "tmpJar", "mergeZipsFlags"}, []string{"implicits"}) func (d *dexer) dexCommonFlags(ctx android.ModuleContext, @@ -370,6 +372,8 @@ func (d *dexer) compileDex(ctx android.ModuleContext, flags javaBuilderFlags, mi if useR8 { proguardDictionary := android.PathForModuleOut(ctx, "proguard_dictionary") d.proguardDictionary = android.OptionalPathForPath(proguardDictionary) + proguardConfiguration := android.PathForModuleOut(ctx, "proguard_configuration") + d.proguardConfiguration = android.OptionalPathForPath(proguardConfiguration) proguardUsageDir := android.PathForModuleOut(ctx, "proguard_usage") proguardUsage := proguardUsageDir.Join(ctx, ctx.Namespace().Path, android.ModuleNameWithPossibleOverride(ctx), "unused.txt") @@ -382,6 +386,7 @@ func (d *dexer) compileDex(ctx android.ModuleContext, flags javaBuilderFlags, mi "r8Flags": strings.Join(append(commonFlags, r8Flags...), " "), "zipFlags": zipFlags, "outDict": proguardDictionary.String(), + "outConfig": proguardConfiguration.String(), "outUsageDir": proguardUsageDir.String(), "outUsage": proguardUsage.String(), "outUsageZip": proguardUsageZip.String(), diff --git a/java/droidstubs.go b/java/droidstubs.go index 4bbe70ac4..7ea8d305a 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -148,6 +148,10 @@ type DroidstubsProperties struct { // path or filegroup to file defining extension an SDK name <-> numerical ID mapping and // what APIs exist in which SDKs; passed to metalava via --sdk-extensions-info Extensions_info_file *string `android:"path"` + + // API surface of this module. If set, the module contributes to an API surface. + // For the full list of available API surfaces, refer to soong/android/sdk_version.go + Api_surface *string } // Used by xsd_config @@ -178,6 +182,10 @@ func DroidstubsFactory() android.Module { &module.Javadoc.properties) InitDroiddocModule(module, android.HostAndDeviceSupported) + + module.SetDefaultableHook(func(ctx android.DefaultableHookContext) { + module.createApiContribution(ctx) + }) return module } @@ -862,6 +870,23 @@ func (d *Droidstubs) ConvertWithApiBp2build(ctx android.TopDownMutatorContext) { }, attrs) } +func (d *Droidstubs) createApiContribution(ctx android.DefaultableHookContext) { + api_file := d.properties.Check_api.Current.Api_file + api_surface := d.properties.Api_surface + + props := struct { + Name *string + Api_surface *string + Api_file *string + }{} + + props.Name = proptools.StringPtr(d.Name() + ".api.contribution") + props.Api_surface = api_surface + props.Api_file = api_file + + ctx.CreateModule(ApiContributionFactory, &props) +} + // TODO (b/262014796): Export the API contributions of CorePlatformApi // A map to populate the api surface of a droidstub from a substring appearing in its name // This map assumes that droidstubs (either checked-in or created by java_sdk_library) diff --git a/java/droidstubs_test.go b/java/droidstubs_test.go index ef2e6dc8a..6c2293746 100644 --- a/java/droidstubs_test.go +++ b/java/droidstubs_test.go @@ -346,3 +346,27 @@ func TestApiSurfaceFromDroidStubsName(t *testing.T) { android.AssertStringEquals(t, tc.desc, tc.expectedApiSurface, bazelApiSurfaceName(tc.name)) } } + +func TestDroidStubsApiContributionGeneration(t *testing.T) { + ctx, _ := testJavaWithFS(t, ` + droidstubs { + name: "foo", + srcs: ["A/a.java"], + api_surface: "public", + check_api: { + current: { + api_file: "A/current.txt", + removed_api_file: "A/removed.txt", + } + } + } + `, + map[string][]byte{ + "A/a.java": nil, + "A/current.txt": nil, + "A/removed.txt": nil, + }, + ) + + ctx.ModuleForTests("foo.api.contribution", "") +} diff --git a/java/java.go b/java/java.go index 7078cc38f..659f98a7c 100644 --- a/java/java.go +++ b/java/java.go @@ -1594,7 +1594,11 @@ type JavaApiImportInfo struct { var JavaApiImportProvider = blueprint.NewProvider(JavaApiImportInfo{}) func (ap *JavaApiContribution) GenerateAndroidBuildActions(ctx android.ModuleContext) { - apiFile := android.PathForModuleSrc(ctx, String(ap.properties.Api_file)) + var apiFile android.Path = nil + if apiFileString := ap.properties.Api_file; apiFileString != nil { + apiFile = android.PathForModuleSrc(ctx, String(apiFileString)) + } + ctx.SetProvider(JavaApiImportProvider, JavaApiImportInfo{ ApiFile: apiFile, }) @@ -1725,7 +1729,11 @@ func (al *ApiLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { switch tag { case javaApiContributionTag: provider := ctx.OtherModuleProvider(dep, JavaApiImportProvider).(JavaApiImportInfo) - srcFiles = append(srcFiles, android.PathForSource(ctx, provider.ApiFile.String())) + providerApiFile := provider.ApiFile + if providerApiFile == nil { + ctx.ModuleErrorf("Error: %s has an empty api file.", dep.Name()) + } + srcFiles = append(srcFiles, android.PathForSource(ctx, providerApiFile.String())) case libTag: provider := ctx.OtherModuleProvider(dep, JavaInfoProvider).(JavaInfo) classPaths = append(classPaths, provider.HeaderJars...) @@ -2595,7 +2603,7 @@ type bp2BuildJavaInfo struct { // to be returned to the calling function. func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) (*javaCommonAttributes, *bp2BuildJavaInfo) { var srcs bazel.LabelListAttribute - var deps bazel.LabelList + var deps bazel.LabelListAttribute var staticDeps bazel.LabelList archVariantProps := m.GetArchVariantProperties(ctx, &CommonProperties{}) @@ -2701,11 +2709,17 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) Javacopts: bazel.MakeStringListAttribute(javacopts), } - if m.properties.Libs != nil { - for _, d := range m.properties.Libs { - neverlinkLabel := android.BazelLabelForModuleDepSingle(ctx, d) - neverlinkLabel.Label = neverlinkLabel.Label + "-neverlink" - deps.Add(&neverlinkLabel) + for axis, configToProps := range archVariantProps { + for config, _props := range configToProps { + if archProps, ok := _props.(*CommonProperties); ok { + var libLabels []bazel.Label + for _, d := range archProps.Libs { + neverlinkLabel := android.BazelLabelForModuleDepSingle(ctx, d) + neverlinkLabel.Label = neverlinkLabel.Label + "-neverlink" + libLabels = append(libLabels, neverlinkLabel) + } + deps.SetSelectValue(axis, config, bazel.MakeLabelList(libLabels)) + } } } @@ -2723,7 +2737,7 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) staticDeps.Add(protoDepLabel) depLabels := &javaDependencyLabels{} - depLabels.Deps = bazel.MakeLabelListAttribute(deps) + depLabels.Deps = deps depLabels.StaticDeps = bazel.MakeLabelListAttribute(staticDeps) bp2BuildInfo := &bp2BuildJavaInfo{ diff --git a/java/java_test.go b/java/java_test.go index ae77842a5..04a112c0a 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -1840,6 +1840,20 @@ func TestDeviceBinaryWrapperGeneration(t *testing.T) { }`) } +func TestJavaApiContributionEmptyApiFile(t *testing.T) { + testJavaError(t, + "Error: foo has an empty api file.", + `java_api_contribution { + name: "foo", + } + java_api_library { + name: "bar", + api_surface: "public", + api_contributions: ["foo"], + } + `) +} + func TestJavaApiLibraryAndProviderLink(t *testing.T) { provider_bp_a := ` java_api_contribution { diff --git a/java/sdk_library.go b/java/sdk_library.go index 3b64bf733..b87236596 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -1599,6 +1599,7 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC Srcs []string Installable *bool Sdk_version *string + Api_surface *string System_modules *string Libs []string Output_javadoc_comments *bool @@ -1638,6 +1639,7 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC props.Srcs = append(props.Srcs, module.properties.Srcs...) props.Srcs = append(props.Srcs, module.sdkLibraryProperties.Api_srcs...) props.Sdk_version = module.deviceProperties.Sdk_version + props.Api_surface = &apiScope.name props.System_modules = module.deviceProperties.System_modules props.Installable = proptools.BoolPtr(false) // A droiddoc module has only one Libs property and doesn't distinguish between diff --git a/python/Android.bp b/python/Android.bp index e49fa6a3c..4584f1e83 100644 --- a/python/Android.bp +++ b/python/Android.bp @@ -11,11 +11,10 @@ bootstrap_go_package { "soong-tradefed", ], srcs: [ - "androidmk.go", "binary.go", + "bp2build.go", "builder.go", "defaults.go", - "installer.go", "library.go", "proto.go", "python.go", diff --git a/python/androidmk.go b/python/androidmk.go deleted file mode 100644 index 7dc471397..000000000 --- a/python/androidmk.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package python - -import ( - "path/filepath" - "strings" - - "android/soong/android" -) - -type subAndroidMkProvider interface { - AndroidMk(*Module, *android.AndroidMkEntries) -} - -func (p *Module) subAndroidMk(entries *android.AndroidMkEntries, obj interface{}) { - if p.subAndroidMkOnce == nil { - p.subAndroidMkOnce = make(map[subAndroidMkProvider]bool) - } - if androidmk, ok := obj.(subAndroidMkProvider); ok { - if !p.subAndroidMkOnce[androidmk] { - p.subAndroidMkOnce[androidmk] = true - androidmk.AndroidMk(p, entries) - } - } -} - -func (p *Module) AndroidMkEntries() []android.AndroidMkEntries { - entries := android.AndroidMkEntries{OutputFile: p.installSource} - - p.subAndroidMk(&entries, p.installer) - - return []android.AndroidMkEntries{entries} -} - -func (p *binaryDecorator) AndroidMk(base *Module, entries *android.AndroidMkEntries) { - entries.Class = "EXECUTABLES" - - entries.ExtraEntries = append(entries.ExtraEntries, - func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { - entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...) - }) - base.subAndroidMk(entries, p.pythonInstaller) -} - -func (p *testDecorator) AndroidMk(base *Module, entries *android.AndroidMkEntries) { - entries.Class = "NATIVE_TESTS" - - entries.ExtraEntries = append(entries.ExtraEntries, - func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { - entries.AddCompatibilityTestSuites(p.binaryDecorator.binaryProperties.Test_suites...) - if p.testConfig != nil { - entries.SetString("LOCAL_FULL_TEST_CONFIG", p.testConfig.String()) - } - - entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true)) - - entries.AddStrings("LOCAL_TEST_DATA", android.AndroidMkDataPaths(p.data)...) - - p.testProperties.Test_options.SetAndroidMkEntries(entries) - }) - base.subAndroidMk(entries, p.binaryDecorator.pythonInstaller) -} - -func (installer *pythonInstaller) AndroidMk(base *Module, entries *android.AndroidMkEntries) { - entries.Required = append(entries.Required, "libc++") - entries.ExtraEntries = append(entries.ExtraEntries, - func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { - path, file := filepath.Split(installer.path.String()) - stem := strings.TrimSuffix(file, filepath.Ext(file)) - - entries.SetString("LOCAL_MODULE_SUFFIX", filepath.Ext(file)) - entries.SetString("LOCAL_MODULE_PATH", path) - entries.SetString("LOCAL_MODULE_STEM", stem) - entries.AddStrings("LOCAL_SHARED_LIBRARIES", installer.androidMkSharedLibs...) - entries.SetBool("LOCAL_CHECK_ELF_FILES", false) - }) -} diff --git a/python/binary.go b/python/binary.go index 670e0d313..95eb2c66c 100644 --- a/python/binary.go +++ b/python/binary.go @@ -18,11 +18,12 @@ package python import ( "fmt" + "path/filepath" + "strings" - "android/soong/android" - "android/soong/bazel" + "github.com/google/blueprint" - "github.com/google/blueprint/proptools" + "android/soong/android" ) func init() { @@ -33,63 +34,6 @@ func registerPythonBinaryComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("python_binary_host", PythonBinaryHostFactory) } -type bazelPythonBinaryAttributes struct { - Main *bazel.Label - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - Python_version *string - Imports bazel.StringListAttribute -} - -func pythonBinaryBp2Build(ctx android.TopDownMutatorContext, m *Module) { - // TODO(b/182306917): this doesn't fully handle all nested props versioned - // by the python version, which would have been handled by the version split - // mutator. This is sufficient for very simple python_binary_host modules - // under Bionic. - py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, false) - py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false) - var python_version *string - if py3Enabled && py2Enabled { - panic(fmt.Errorf( - "error for '%s' module: bp2build's python_binary_host converter does not support "+ - "converting a module that is enabled for both Python 2 and 3 at the same time.", m.Name())) - } else if py2Enabled { - python_version = &pyVersion2 - } else { - // do nothing, since python_version defaults to PY3. - } - - baseAttrs := m.makeArchVariantBaseAttributes(ctx) - attrs := &bazelPythonBinaryAttributes{ - Main: nil, - Srcs: baseAttrs.Srcs, - Deps: baseAttrs.Deps, - Python_version: python_version, - Imports: baseAttrs.Imports, - } - - for _, propIntf := range m.GetProperties() { - if props, ok := propIntf.(*BinaryProperties); ok { - // main is optional. - if props.Main != nil { - main := android.BazelLabelForModuleSrcSingle(ctx, *props.Main) - attrs.Main = &main - break - } - } - } - - props := bazel.BazelTargetModuleProperties{ - // Use the native py_binary rule. - Rule_class: "py_binary", - } - - ctx.CreateBazelTargetModule(props, android.CommonAttributes{ - Name: m.Name(), - Data: baseAttrs.Data, - }, attrs) -} - type BinaryProperties struct { // the name of the source file that is the main entry point of the program. // this file must also be listed in srcs. @@ -118,52 +62,61 @@ type BinaryProperties struct { Auto_gen_config *bool } -type binaryDecorator struct { +type PythonBinaryModule struct { + PythonLibraryModule binaryProperties BinaryProperties - *pythonInstaller + // (.intermediate) module output path as installation source. + installSource android.Path + + // Final installation path. + installedDest android.Path + + androidMkSharedLibs []string } +var _ android.AndroidMkEntriesProvider = (*PythonBinaryModule)(nil) +var _ android.Module = (*PythonBinaryModule)(nil) + type IntermPathProvider interface { IntermPathForModuleOut() android.OptionalPath } -func NewBinary(hod android.HostOrDeviceSupported) (*Module, *binaryDecorator) { - module := newModule(hod, android.MultilibFirst) - decorator := &binaryDecorator{pythonInstaller: NewPythonInstaller("bin", "")} - - module.bootstrapper = decorator - module.installer = decorator - - return module, decorator +func NewBinary(hod android.HostOrDeviceSupported) *PythonBinaryModule { + return &PythonBinaryModule{ + PythonLibraryModule: *newModule(hod, android.MultilibFirst), + } } func PythonBinaryHostFactory() android.Module { - module, _ := NewBinary(android.HostSupported) - - android.InitBazelModule(module) - - return module.init() + return NewBinary(android.HostSupported).init() } -func (binary *binaryDecorator) autorun() bool { - return BoolDefault(binary.binaryProperties.Autorun, true) +func (p *PythonBinaryModule) init() android.Module { + p.AddProperties(&p.properties, &p.protoProperties) + p.AddProperties(&p.binaryProperties) + android.InitAndroidArchModule(p, p.hod, p.multilib) + android.InitDefaultableModule(p) + android.InitBazelModule(p) + return p } -func (binary *binaryDecorator) bootstrapperProps() []interface{} { - return []interface{}{&binary.binaryProperties} +func (p *PythonBinaryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + p.PythonLibraryModule.GenerateAndroidBuildActions(ctx) + p.buildBinary(ctx) + p.installedDest = ctx.InstallFile(installDir(ctx, "bin", "", ""), + p.installSource.Base(), p.installSource) } -func (binary *binaryDecorator) bootstrap(ctx android.ModuleContext, actualVersion string, - embeddedLauncher bool, srcsPathMappings []pathMapping, srcsZip android.Path, - depsSrcsZips android.Paths) android.OptionalPath { - +func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) { + depsSrcsZips := p.collectPathsFromTransitiveDeps(ctx) main := "" - if binary.autorun() { - main = binary.getPyMainFile(ctx, srcsPathMappings) + if p.autorun() { + main = p.getPyMainFile(ctx, p.srcsPathMappings) } var launcherPath android.OptionalPath + embeddedLauncher := p.isEmbeddedLauncherEnabled() if embeddedLauncher { ctx.VisitDirectDepsWithTag(launcherTag, func(m android.Module) { if provider, ok := m.(IntermPathProvider); ok { @@ -175,15 +128,137 @@ func (binary *binaryDecorator) bootstrap(ctx android.ModuleContext, actualVersio } }) } - binFile := registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath, - binary.getHostInterpreterName(ctx, actualVersion), - main, binary.getStem(ctx), append(android.Paths{srcsZip}, depsSrcsZips...)) + p.installSource = registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath, + p.getHostInterpreterName(ctx, p.properties.Actual_version), + main, p.getStem(ctx), append(android.Paths{p.srcsZip}, depsSrcsZips...)) + + var sharedLibs []string + // if embedded launcher is enabled, we need to collect the shared library dependencies of the + // launcher + for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) { + sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep)) + } + p.androidMkSharedLibs = sharedLibs +} + +func (p *PythonBinaryModule) AndroidMkEntries() []android.AndroidMkEntries { + entries := android.AndroidMkEntries{OutputFile: android.OptionalPathForPath(p.installSource)} + + entries.Class = "EXECUTABLES" + + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...) + }) + + entries.Required = append(entries.Required, "libc++") + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + path, file := filepath.Split(p.installedDest.String()) + stem := strings.TrimSuffix(file, filepath.Ext(file)) + + entries.SetString("LOCAL_MODULE_SUFFIX", filepath.Ext(file)) + entries.SetString("LOCAL_MODULE_PATH", path) + entries.SetString("LOCAL_MODULE_STEM", stem) + entries.AddStrings("LOCAL_SHARED_LIBRARIES", p.androidMkSharedLibs...) + entries.SetBool("LOCAL_CHECK_ELF_FILES", false) + }) + + return []android.AndroidMkEntries{entries} +} + +func (p *PythonBinaryModule) DepsMutator(ctx android.BottomUpMutatorContext) { + p.PythonLibraryModule.DepsMutator(ctx) + + versionVariation := []blueprint.Variation{ + {"python_version", p.properties.Actual_version}, + } + + // If this module will be installed and has an embedded launcher, we need to add dependencies for: + // * standard library + // * launcher + // * shared dependencies of the launcher + if p.isEmbeddedLauncherEnabled() { + var stdLib string + var launcherModule string + // Add launcher shared lib dependencies. Ideally, these should be + // derived from the `shared_libs` property of the launcher. However, we + // cannot read the property at this stage and it will be too late to add + // dependencies later. + launcherSharedLibDeps := []string{ + "libsqlite", + } + // Add launcher-specific dependencies for bionic + if ctx.Target().Os.Bionic() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm") + } + if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl") + } + + switch p.properties.Actual_version { + case pyVersion2: + stdLib = "py2-stdlib" - return android.OptionalPathForPath(binFile) + launcherModule = "py2-launcher" + if p.autorun() { + launcherModule = "py2-launcher-autorun" + } + + launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") + + case pyVersion3: + stdLib = "py3-stdlib" + + launcherModule = "py3-launcher" + if p.autorun() { + launcherModule = "py3-launcher-autorun" + } + if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl { + launcherModule += "-static" + } + + if ctx.Device() { + launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog") + } + default: + panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", + p.properties.Actual_version, ctx.ModuleName())) + } + ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib) + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule) + ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...) + } +} + +// HostToolPath returns a path if appropriate such that this module can be used as a host tool, +// fulfilling the android.HostToolProvider interface. +func (p *PythonBinaryModule) HostToolPath() android.OptionalPath { + // TODO: This should only be set when building host binaries -- tests built for device would be + // setting this incorrectly. + return android.OptionalPathForPath(p.installedDest) +} + +// OutputFiles returns output files based on given tag, returns an error if tag is unsupported. +func (p *PythonBinaryModule) OutputFiles(tag string) (android.Paths, error) { + switch tag { + case "": + return android.Paths{p.installSource}, nil + default: + return nil, fmt.Errorf("unsupported module reference tag %q", tag) + } +} + +func (p *PythonBinaryModule) isEmbeddedLauncherEnabled() bool { + return Bool(p.properties.Embedded_launcher) +} + +func (b *PythonBinaryModule) autorun() bool { + return BoolDefault(b.binaryProperties.Autorun, true) } // get host interpreter name. -func (binary *binaryDecorator) getHostInterpreterName(ctx android.ModuleContext, +func (p *PythonBinaryModule) getHostInterpreterName(ctx android.ModuleContext, actualVersion string) string { var interp string switch actualVersion { @@ -200,13 +275,13 @@ func (binary *binaryDecorator) getHostInterpreterName(ctx android.ModuleContext, } // find main program path within runfiles tree. -func (binary *binaryDecorator) getPyMainFile(ctx android.ModuleContext, +func (p *PythonBinaryModule) getPyMainFile(ctx android.ModuleContext, srcsPathMappings []pathMapping) string { var main string - if String(binary.binaryProperties.Main) == "" { + if String(p.binaryProperties.Main) == "" { main = ctx.ModuleName() + pyExt } else { - main = String(binary.binaryProperties.Main) + main = String(p.binaryProperties.Main) } for _, path := range srcsPathMappings { @@ -219,11 +294,21 @@ func (binary *binaryDecorator) getPyMainFile(ctx android.ModuleContext, return "" } -func (binary *binaryDecorator) getStem(ctx android.ModuleContext) string { +func (p *PythonBinaryModule) getStem(ctx android.ModuleContext) string { stem := ctx.ModuleName() - if String(binary.binaryProperties.Stem) != "" { - stem = String(binary.binaryProperties.Stem) + if String(p.binaryProperties.Stem) != "" { + stem = String(p.binaryProperties.Stem) } - return stem + String(binary.binaryProperties.Suffix) + return stem + String(p.binaryProperties.Suffix) +} + +func installDir(ctx android.ModuleContext, dir, dir64, relative string) android.InstallPath { + if ctx.Arch().ArchType.Multilib == "lib64" && dir64 != "" { + dir = dir64 + } + if !ctx.Host() && ctx.Config().HasMultilibConflict(ctx.Arch().ArchType) { + dir = filepath.Join(dir, ctx.Arch().ArchType.String()) + } + return android.PathForModuleInstall(ctx, dir, relative) } diff --git a/python/bp2build.go b/python/bp2build.go new file mode 100644 index 000000000..bdac2dc38 --- /dev/null +++ b/python/bp2build.go @@ -0,0 +1,226 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/google/blueprint/proptools" + + "android/soong/android" + "android/soong/bazel" +) + +type bazelPythonLibraryAttributes struct { + Srcs bazel.LabelListAttribute + Deps bazel.LabelListAttribute + Imports bazel.StringListAttribute + Srcs_version *string +} + +type bazelPythonProtoLibraryAttributes struct { + Deps bazel.LabelListAttribute +} + +type baseAttributes struct { + // TODO(b/200311466): Probably not translate b/c Bazel has no good equiv + //Pkg_path bazel.StringAttribute + // TODO: Related to Pkg_bath and similarLy gated + //Is_internal bazel.BoolAttribute + // Combines Srcs and Exclude_srcs + Srcs bazel.LabelListAttribute + Deps bazel.LabelListAttribute + // Combines Data and Java_data (invariant) + Data bazel.LabelListAttribute + Imports bazel.StringListAttribute +} + +func (m *PythonLibraryModule) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext) baseAttributes { + var attrs baseAttributes + archVariantBaseProps := m.GetArchVariantProperties(ctx, &BaseProperties{}) + for axis, configToProps := range archVariantBaseProps { + for config, props := range configToProps { + if baseProps, ok := props.(*BaseProperties); ok { + attrs.Srcs.SetSelectValue(axis, config, + android.BazelLabelForModuleSrcExcludes(ctx, baseProps.Srcs, baseProps.Exclude_srcs)) + attrs.Deps.SetSelectValue(axis, config, + android.BazelLabelForModuleDeps(ctx, baseProps.Libs)) + data := android.BazelLabelForModuleSrc(ctx, baseProps.Data) + data.Append(android.BazelLabelForModuleSrc(ctx, baseProps.Java_data)) + attrs.Data.SetSelectValue(axis, config, data) + } + } + } + + partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{ + "proto": android.ProtoSrcLabelPartition, + "py": bazel.LabelPartition{Keep_remainder: true}, + }) + attrs.Srcs = partitionedSrcs["py"] + + if !partitionedSrcs["proto"].IsEmpty() { + protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"]) + protoLabel := bazel.Label{Label: ":" + protoInfo.Name} + + pyProtoLibraryName := m.Name() + "_py_proto" + ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ + Rule_class: "py_proto_library", + Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl", + }, android.CommonAttributes{ + Name: pyProtoLibraryName, + }, &bazelPythonProtoLibraryAttributes{ + Deps: bazel.MakeSingleLabelListAttribute(protoLabel), + }) + + attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName)) + } + + // Bazel normally requires `import path.from.top.of.tree` statements in + // python code, but with soong you can directly import modules from libraries. + // Add "imports" attributes to the bazel library so it matches soong's behavior. + imports := "." + if m.properties.Pkg_path != nil { + // TODO(b/215119317) This is a hack to handle the fact that we don't convert + // pkg_path properly right now. If the folder structure that contains this + // Android.bp file matches pkg_path, we can set imports to an appropriate + // number of ../..s to emulate moving the files under a pkg_path folder. + pkg_path := filepath.Clean(*m.properties.Pkg_path) + if strings.HasPrefix(pkg_path, "/") { + ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path) + } + + if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path { + ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir()) + } + numFolders := strings.Count(pkg_path, "/") + 1 + dots := make([]string, numFolders) + for i := 0; i < numFolders; i++ { + dots[i] = ".." + } + imports = strings.Join(dots, "/") + } + attrs.Imports = bazel.MakeStringListAttribute([]string{imports}) + + return attrs +} + +func pythonLibBp2Build(ctx android.TopDownMutatorContext, m *PythonLibraryModule) { + // TODO(b/182306917): this doesn't fully handle all nested props versioned + // by the python version, which would have been handled by the version split + // mutator. This is sufficient for very simple python_library modules under + // Bionic. + py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, true) + py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false) + var python_version *string + if py2Enabled && !py3Enabled { + python_version = &pyVersion2 + } else if !py2Enabled && py3Enabled { + python_version = &pyVersion3 + } else if !py2Enabled && !py3Enabled { + ctx.ModuleErrorf("bp2build converter doesn't understand having neither py2 nor py3 enabled") + } else { + // do nothing, since python_version defaults to PY2ANDPY3 + } + + baseAttrs := m.makeArchVariantBaseAttributes(ctx) + + attrs := &bazelPythonLibraryAttributes{ + Srcs: baseAttrs.Srcs, + Deps: baseAttrs.Deps, + Srcs_version: python_version, + Imports: baseAttrs.Imports, + } + + props := bazel.BazelTargetModuleProperties{ + // Use the native py_library rule. + Rule_class: "py_library", + } + + ctx.CreateBazelTargetModule(props, android.CommonAttributes{ + Name: m.Name(), + Data: baseAttrs.Data, + }, attrs) +} + +type bazelPythonBinaryAttributes struct { + Main *bazel.Label + Srcs bazel.LabelListAttribute + Deps bazel.LabelListAttribute + Python_version *string + Imports bazel.StringListAttribute +} + +func pythonBinaryBp2Build(ctx android.TopDownMutatorContext, m *PythonBinaryModule) { + // TODO(b/182306917): this doesn't fully handle all nested props versioned + // by the python version, which would have been handled by the version split + // mutator. This is sufficient for very simple python_binary_host modules + // under Bionic. + py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, false) + py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false) + var python_version *string + if py3Enabled && py2Enabled { + panic(fmt.Errorf( + "error for '%s' module: bp2build's python_binary_host converter does not support "+ + "converting a module that is enabled for both Python 2 and 3 at the same time.", m.Name())) + } else if py2Enabled { + python_version = &pyVersion2 + } else { + // do nothing, since python_version defaults to PY3. + } + + baseAttrs := m.makeArchVariantBaseAttributes(ctx) + attrs := &bazelPythonBinaryAttributes{ + Main: nil, + Srcs: baseAttrs.Srcs, + Deps: baseAttrs.Deps, + Python_version: python_version, + Imports: baseAttrs.Imports, + } + + for _, propIntf := range m.GetProperties() { + if props, ok := propIntf.(*BinaryProperties); ok { + // main is optional. + if props.Main != nil { + main := android.BazelLabelForModuleSrcSingle(ctx, *props.Main) + attrs.Main = &main + break + } + } + } + + props := bazel.BazelTargetModuleProperties{ + // Use the native py_binary rule. + Rule_class: "py_binary", + } + + ctx.CreateBazelTargetModule(props, android.CommonAttributes{ + Name: m.Name(), + Data: baseAttrs.Data, + }, attrs) +} + +func (p *PythonLibraryModule) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + pythonLibBp2Build(ctx, p) +} + +func (p *PythonBinaryModule) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + pythonBinaryBp2Build(ctx, p) +} + +func (p *PythonTestModule) ConvertWithBp2build(_ android.TopDownMutatorContext) { + // Tests are currently unsupported +} diff --git a/python/installer.go b/python/installer.go deleted file mode 100644 index 396f03667..000000000 --- a/python/installer.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package python - -import ( - "path/filepath" - - "android/soong/android" -) - -// This file handles installing python executables into their final location - -type installLocation int - -const ( - InstallInData installLocation = iota -) - -type pythonInstaller struct { - dir string - dir64 string - relative string - - path android.InstallPath - - androidMkSharedLibs []string -} - -func NewPythonInstaller(dir, dir64 string) *pythonInstaller { - return &pythonInstaller{ - dir: dir, - dir64: dir64, - } -} - -var _ installer = (*pythonInstaller)(nil) - -func (installer *pythonInstaller) installDir(ctx android.ModuleContext) android.InstallPath { - dir := installer.dir - if ctx.Arch().ArchType.Multilib == "lib64" && installer.dir64 != "" { - dir = installer.dir64 - } - if !ctx.Host() && ctx.Config().HasMultilibConflict(ctx.Arch().ArchType) { - dir = filepath.Join(dir, ctx.Arch().ArchType.String()) - } - return android.PathForModuleInstall(ctx, dir, installer.relative) -} - -func (installer *pythonInstaller) install(ctx android.ModuleContext, file android.Path) { - installer.path = ctx.InstallFile(installer.installDir(ctx), file.Base(), file) -} - -func (installer *pythonInstaller) setAndroidMkSharedLibs(sharedLibs []string) { - installer.androidMkSharedLibs = sharedLibs -} diff --git a/python/library.go b/python/library.go index df92df42b..7cdb80b87 100644 --- a/python/library.go +++ b/python/library.go @@ -18,9 +18,6 @@ package python import ( "android/soong/android" - "android/soong/bazel" - - "github.com/google/blueprint/proptools" ) func init() { @@ -33,66 +30,9 @@ func registerPythonLibraryComponents(ctx android.RegistrationContext) { } func PythonLibraryHostFactory() android.Module { - module := newModule(android.HostSupported, android.MultilibFirst) - - android.InitBazelModule(module) - - return module.init() -} - -type bazelPythonLibraryAttributes struct { - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - Imports bazel.StringListAttribute - Srcs_version *string -} - -type bazelPythonProtoLibraryAttributes struct { - Deps bazel.LabelListAttribute -} - -func pythonLibBp2Build(ctx android.TopDownMutatorContext, m *Module) { - // TODO(b/182306917): this doesn't fully handle all nested props versioned - // by the python version, which would have been handled by the version split - // mutator. This is sufficient for very simple python_library modules under - // Bionic. - py3Enabled := proptools.BoolDefault(m.properties.Version.Py3.Enabled, true) - py2Enabled := proptools.BoolDefault(m.properties.Version.Py2.Enabled, false) - var python_version *string - if py2Enabled && !py3Enabled { - python_version = &pyVersion2 - } else if !py2Enabled && py3Enabled { - python_version = &pyVersion3 - } else if !py2Enabled && !py3Enabled { - ctx.ModuleErrorf("bp2build converter doesn't understand having neither py2 nor py3 enabled") - } else { - // do nothing, since python_version defaults to PY2ANDPY3 - } - - baseAttrs := m.makeArchVariantBaseAttributes(ctx) - - attrs := &bazelPythonLibraryAttributes{ - Srcs: baseAttrs.Srcs, - Deps: baseAttrs.Deps, - Srcs_version: python_version, - Imports: baseAttrs.Imports, - } - - props := bazel.BazelTargetModuleProperties{ - // Use the native py_library rule. - Rule_class: "py_library", - } - - ctx.CreateBazelTargetModule(props, android.CommonAttributes{ - Name: m.Name(), - Data: baseAttrs.Data, - }, attrs) + return newModule(android.HostSupported, android.MultilibFirst).init() } func PythonLibraryFactory() android.Module { - module := newModule(android.HostAndDeviceSupported, android.MultilibBoth) - - android.InitBazelModule(module) - - return module.init() + return newModule(android.HostAndDeviceSupported, android.MultilibBoth).init() } diff --git a/python/python.go b/python/python.go index 24e1bb2ec..2b71e83fe 100644 --- a/python/python.go +++ b/python/python.go @@ -22,8 +22,6 @@ import ( "regexp" "strings" - "android/soong/bazel" - "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -122,26 +120,13 @@ type BaseProperties struct { Embedded_launcher *bool `blueprint:"mutated"` } -type baseAttributes struct { - // TODO(b/200311466): Probably not translate b/c Bazel has no good equiv - //Pkg_path bazel.StringAttribute - // TODO: Related to Pkg_bath and similarLy gated - //Is_internal bazel.BoolAttribute - // Combines Srcs and Exclude_srcs - Srcs bazel.LabelListAttribute - Deps bazel.LabelListAttribute - // Combines Data and Java_data (invariant) - Data bazel.LabelListAttribute - Imports bazel.StringListAttribute -} - // Used to store files of current module after expanding dependencies type pathMapping struct { dest string src android.Path } -type Module struct { +type PythonLibraryModule struct { android.ModuleBase android.DefaultableModuleBase android.BazelModuleBase @@ -153,16 +138,6 @@ type Module struct { hod android.HostOrDeviceSupported multilib android.Multilib - // interface used to bootstrap .par executable when embedded_launcher is true - // this should be set by Python modules which are runnable, e.g. binaries and tests - // bootstrapper might be nil (e.g. Python library module). - bootstrapper bootstrapper - - // interface that implements functions required for installation - // this should be set by Python modules which are runnable, e.g. binaries and tests - // installer might be nil (e.g. Python library module). - installer installer - // the Python files of current module after expanding source dependencies. // pathMapping: <dest: runfile_path, src: source_path> srcsPathMappings []pathMapping @@ -173,110 +148,16 @@ type Module struct { // the zip filepath for zipping current module source/data files. srcsZip android.Path - - // dependency modules' zip filepath for zipping current module source/data files. - depsSrcsZips android.Paths - - // (.intermediate) module output path as installation source. - installSource android.OptionalPath - - // Map to ensure sub-part of the AndroidMk for this module is only added once - subAndroidMkOnce map[subAndroidMkProvider]bool } // newModule generates new Python base module -func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module { - return &Module{ +func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *PythonLibraryModule { + return &PythonLibraryModule{ hod: hod, multilib: multilib, } } -func (m *Module) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext) baseAttributes { - var attrs baseAttributes - archVariantBaseProps := m.GetArchVariantProperties(ctx, &BaseProperties{}) - for axis, configToProps := range archVariantBaseProps { - for config, props := range configToProps { - if baseProps, ok := props.(*BaseProperties); ok { - attrs.Srcs.SetSelectValue(axis, config, - android.BazelLabelForModuleSrcExcludes(ctx, baseProps.Srcs, baseProps.Exclude_srcs)) - attrs.Deps.SetSelectValue(axis, config, - android.BazelLabelForModuleDeps(ctx, baseProps.Libs)) - data := android.BazelLabelForModuleSrc(ctx, baseProps.Data) - data.Append(android.BazelLabelForModuleSrc(ctx, baseProps.Java_data)) - attrs.Data.SetSelectValue(axis, config, data) - } - } - } - - partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{ - "proto": android.ProtoSrcLabelPartition, - "py": bazel.LabelPartition{Keep_remainder: true}, - }) - attrs.Srcs = partitionedSrcs["py"] - - if !partitionedSrcs["proto"].IsEmpty() { - protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"]) - protoLabel := bazel.Label{Label: ":" + protoInfo.Name} - - pyProtoLibraryName := m.Name() + "_py_proto" - ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ - Rule_class: "py_proto_library", - Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl", - }, android.CommonAttributes{ - Name: pyProtoLibraryName, - }, &bazelPythonProtoLibraryAttributes{ - Deps: bazel.MakeSingleLabelListAttribute(protoLabel), - }) - - attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName)) - } - - // Bazel normally requires `import path.from.top.of.tree` statements in - // python code, but with soong you can directly import modules from libraries. - // Add "imports" attributes to the bazel library so it matches soong's behavior. - imports := "." - if m.properties.Pkg_path != nil { - // TODO(b/215119317) This is a hack to handle the fact that we don't convert - // pkg_path properly right now. If the folder structure that contains this - // Android.bp file matches pkg_path, we can set imports to an appropriate - // number of ../..s to emulate moving the files under a pkg_path folder. - pkg_path := filepath.Clean(*m.properties.Pkg_path) - if strings.HasPrefix(pkg_path, "/") { - ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path) - } - - if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path { - ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir()) - } - numFolders := strings.Count(pkg_path, "/") + 1 - dots := make([]string, numFolders) - for i := 0; i < numFolders; i++ { - dots[i] = ".." - } - imports = strings.Join(dots, "/") - } - attrs.Imports = bazel.MakeStringListAttribute([]string{imports}) - - return attrs -} - -// bootstrapper interface should be implemented for runnable modules, e.g. binary and test -type bootstrapper interface { - bootstrapperProps() []interface{} - bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool, - srcsPathMappings []pathMapping, srcsZip android.Path, - depsSrcsZips android.Paths) android.OptionalPath - - autorun() bool -} - -// installer interface should be implemented for installable modules, e.g. binary and test -type installer interface { - install(ctx android.ModuleContext, path android.Path) - setAndroidMkSharedLibs(sharedLibs []string) -} - // interface implemented by Python modules to provide source and data mappings and zip to python // modules that depend on it type pythonDependency interface { @@ -286,37 +167,31 @@ type pythonDependency interface { } // getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination -func (p *Module) getSrcsPathMappings() []pathMapping { +func (p *PythonLibraryModule) getSrcsPathMappings() []pathMapping { return p.srcsPathMappings } // getSrcsPathMappings gets this module's path mapping of data source path : runfiles destination -func (p *Module) getDataPathMappings() []pathMapping { +func (p *PythonLibraryModule) getDataPathMappings() []pathMapping { return p.dataPathMappings } // getSrcsZip returns the filepath where the current module's source/data files are zipped. -func (p *Module) getSrcsZip() android.Path { +func (p *PythonLibraryModule) getSrcsZip() android.Path { return p.srcsZip } -var _ pythonDependency = (*Module)(nil) +func (p *PythonLibraryModule) getBaseProperties() *BaseProperties { + return &p.properties +} -var _ android.AndroidMkEntriesProvider = (*Module)(nil) +var _ pythonDependency = (*PythonLibraryModule)(nil) -func (p *Module) init(additionalProps ...interface{}) android.Module { +func (p *PythonLibraryModule) init() android.Module { p.AddProperties(&p.properties, &p.protoProperties) - - // Add additional properties for bootstrapping/installation - // This is currently tied to the bootstrapper interface; - // however, these are a combination of properties for the installation and bootstrapping of a module - if p.bootstrapper != nil { - p.AddProperties(p.bootstrapper.bootstrapperProps()...) - } - android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitDefaultableModule(p) - + android.InitBazelModule(p) return p } @@ -350,24 +225,29 @@ var ( internalPath = "internal" ) +type basePropertiesProvider interface { + getBaseProperties() *BaseProperties +} + // versionSplitMutator creates version variants for modules and appends the version-specific // properties for a given variant to the properties in the variant module func versionSplitMutator() func(android.BottomUpMutatorContext) { return func(mctx android.BottomUpMutatorContext) { - if base, ok := mctx.Module().(*Module); ok { - versionNames := []string{} + if base, ok := mctx.Module().(basePropertiesProvider); ok { + props := base.getBaseProperties() + var versionNames []string // collect version specific properties, so that we can merge version-specific properties // into the module's overall properties - versionProps := []VersionProperties{} + var versionProps []VersionProperties // PY3 is first so that we alias the PY3 variant rather than PY2 if both // are available - if proptools.BoolDefault(base.properties.Version.Py3.Enabled, true) { + if proptools.BoolDefault(props.Version.Py3.Enabled, true) { versionNames = append(versionNames, pyVersion3) - versionProps = append(versionProps, base.properties.Version.Py3) + versionProps = append(versionProps, props.Version.Py3) } - if proptools.BoolDefault(base.properties.Version.Py2.Enabled, false) { + if proptools.BoolDefault(props.Version.Py2.Enabled, false) { versionNames = append(versionNames, pyVersion2) - versionProps = append(versionProps, base.properties.Version.Py2) + versionProps = append(versionProps, props.Version.Py2) } modules := mctx.CreateLocalVariations(versionNames...) // Alias module to the first variant @@ -376,9 +256,10 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) { } for i, v := range versionNames { // set the actual version for Python module. - modules[i].(*Module).properties.Actual_version = v + newProps := modules[i].(basePropertiesProvider).getBaseProperties() + newProps.Actual_version = v // append versioned properties for the Python module to the overall properties - err := proptools.AppendMatchingProperties([]interface{}{&modules[i].(*Module).properties}, &versionProps[i], nil) + err := proptools.AppendMatchingProperties([]interface{}{newProps}, &versionProps[i], nil) if err != nil { panic(err) } @@ -387,38 +268,6 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) { } } -// HostToolPath returns a path if appropriate such that this module can be used as a host tool, -// fulfilling HostToolProvider interface. -func (p *Module) HostToolPath() android.OptionalPath { - if p.installer != nil { - if bin, ok := p.installer.(*binaryDecorator); ok { - // TODO: This should only be set when building host binaries -- tests built for device would be - // setting this incorrectly. - return android.OptionalPathForPath(bin.path) - } - } - - return android.OptionalPath{} - -} - -// OutputFiles returns output files based on given tag, returns an error if tag is unsupported. -func (p *Module) OutputFiles(tag string) (android.Paths, error) { - switch tag { - case "": - if outputFile := p.installSource; outputFile.Valid() { - return android.Paths{outputFile.Path()}, nil - } - return android.Paths{}, nil - default: - return nil, fmt.Errorf("unsupported module reference tag %q", tag) - } -} - -func (p *Module) isEmbeddedLauncherEnabled() bool { - return p.installer != nil && Bool(p.properties.Embedded_launcher) -} - func anyHasExt(paths []string, ext string) bool { for _, p := range paths { if filepath.Ext(p) == ext { @@ -429,7 +278,7 @@ func anyHasExt(paths []string, ext string) bool { return false } -func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool { +func (p *PythonLibraryModule) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bool { return anyHasExt(p.properties.Srcs, ext) } @@ -437,7 +286,7 @@ func (p *Module) anySrcHasExt(ctx android.BottomUpMutatorContext, ext string) bo // - handles proto dependencies, // - if required, specifies launcher and adds launcher dependencies, // - applies python version mutations to Python dependencies -func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { +func (p *PythonLibraryModule) DepsMutator(ctx android.BottomUpMutatorContext) { android.ProtoDeps(ctx, &p.protoProperties) versionVariation := []blueprint.Variation{ @@ -452,111 +301,15 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) { // Add python library dependencies for this python version variation ctx.AddVariationDependencies(versionVariation, pythonLibTag, android.LastUniqueStrings(p.properties.Libs)...) - // If this module will be installed and has an embedded launcher, we need to add dependencies for: - // * standard library - // * launcher - // * shared dependencies of the launcher - if p.installer != nil && p.isEmbeddedLauncherEnabled() { - var stdLib string - var launcherModule string - // Add launcher shared lib dependencies. Ideally, these should be - // derived from the `shared_libs` property of the launcher. However, we - // cannot read the property at this stage and it will be too late to add - // dependencies later. - launcherSharedLibDeps := []string{ - "libsqlite", - } - // Add launcher-specific dependencies for bionic - if ctx.Target().Os.Bionic() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm") - } - if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl") - } - - switch p.properties.Actual_version { - case pyVersion2: - stdLib = "py2-stdlib" - - launcherModule = "py2-launcher" - if p.bootstrapper.autorun() { - launcherModule = "py2-launcher-autorun" - } - - launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++") - - case pyVersion3: - stdLib = "py3-stdlib" - - launcherModule = "py3-launcher" - if p.bootstrapper.autorun() { - launcherModule = "py3-launcher-autorun" - } - if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl { - launcherModule += "-static" - } - - if ctx.Device() { - launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog") - } - default: - panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.", - p.properties.Actual_version, ctx.ModuleName())) - } - ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib) - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule) - ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...) - } - // Emulate the data property for java_data but with the arch variation overridden to "common" // so that it can point to java modules. javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}} ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...) } -func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { - p.generatePythonBuildActions(ctx) - - // Only Python binary and test modules have non-empty bootstrapper. - if p.bootstrapper != nil { - // if the module is being installed, we need to collect all transitive dependencies to embed in - // the final par - p.collectPathsFromTransitiveDeps(ctx) - // bootstrap the module, including resolving main file, getting launcher path, and - // registering actions to build the par file - // bootstrap returns the binary output path - p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version, - p.isEmbeddedLauncherEnabled(), p.srcsPathMappings, p.srcsZip, p.depsSrcsZips) - } - - // Only Python binary and test modules have non-empty installer. - if p.installer != nil { - var sharedLibs []string - // if embedded launcher is enabled, we need to collect the shared library depenendencies of the - // launcher - for _, dep := range ctx.GetDirectDepsWithTag(launcherSharedLibTag) { - sharedLibs = append(sharedLibs, ctx.OtherModuleName(dep)) - } - - p.installer.setAndroidMkSharedLibs(sharedLibs) - - // Install the par file from installSource - if p.installSource.Valid() { - p.installer.install(ctx, p.installSource.Path()) - } - } -} - -// generatePythonBuildActions performs build actions common to all Python modules -func (p *Module) generatePythonBuildActions(ctx android.ModuleContext) { +// GenerateAndroidBuildActions performs build actions common to all Python modules +func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs) - requiresSrcs := true - if p.bootstrapper != nil && !p.bootstrapper.autorun() { - requiresSrcs = false - } - if len(expandedSrcs) == 0 && requiresSrcs { - ctx.ModuleErrorf("doesn't have any source files!") - } // expand data files from "data" property. expandedData := android.PathsForModuleSrc(ctx, p.properties.Data) @@ -607,7 +360,7 @@ func isValidPythonPath(path string) error { // For this module, generate unique pathMappings: <dest: runfiles_path, src: source_path> // for python/data files expanded from properties. -func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string, +func (p *PythonLibraryModule) genModulePathMappings(ctx android.ModuleContext, pkgPath string, expandedSrcs, expandedData android.Paths) { // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to // check current module duplicates. @@ -642,7 +395,7 @@ func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkgPath string } // createSrcsZip registers build actions to zip current module's sources and data. -func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path { +func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path { relativeRootMap := make(map[string]android.Paths) pathMappings := append(p.srcsPathMappings, p.dataPathMappings...) @@ -654,13 +407,8 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi if path.src.Ext() == protoExt { protoSrcs = append(protoSrcs, path.src) } else { - var relativeRoot string - relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel()) - if v, found := relativeRootMap[relativeRoot]; found { - relativeRootMap[relativeRoot] = append(v, path.src) - } else { - relativeRootMap[relativeRoot] = android.Paths{path.src} - } + relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel()) + relativeRootMap[relativeRoot] = append(relativeRootMap[relativeRoot], path.src) } } var zips android.Paths @@ -736,30 +484,20 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi } } -// isPythonLibModule returns whether the given module is a Python library Module or not +// isPythonLibModule returns whether the given module is a Python library PythonLibraryModule or not func isPythonLibModule(module blueprint.Module) bool { - if m, ok := module.(*Module); ok { - return m.isLibrary() + if _, ok := module.(*PythonLibraryModule); ok { + if _, ok := module.(*PythonBinaryModule); !ok { + return true + } } return false } -// This is distinguished by the fact that Python libraries are not installable, while other Python -// modules are. -func (p *Module) isLibrary() bool { - // Python library has no bootstrapper or installer - return p.bootstrapper == nil && p.installer == nil -} - -func (p *Module) isBinary() bool { - _, ok := p.bootstrapper.(*binaryDecorator) - return ok -} - // collectPathsFromTransitiveDeps checks for source/data files for duplicate paths // for module and its transitive dependencies and collects list of data/source file // zips for transitive dependencies. -func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { +func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext) android.Paths { // fetch <runfiles_path, source_path> pairs from "src" and "data" properties to // check duplicates. destToPySrcs := make(map[string]string) @@ -773,6 +511,8 @@ func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { seen := make(map[android.Module]bool) + var result android.Paths + // visit all its dependencies in depth first. ctx.WalkDeps(func(child, parent android.Module) bool { // we only collect dependencies tagged as python library deps @@ -801,10 +541,11 @@ func (p *Module) collectPathsFromTransitiveDeps(ctx android.ModuleContext) { checkForDuplicateOutputPath(ctx, destToPyData, path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child)) } - p.depsSrcsZips = append(p.depsSrcsZips, dep.getSrcsZip()) + result = append(result, dep.getSrcsZip()) } return true }) + return result } // chckForDuplicateOutputPath checks whether outputPath has already been included in map m, which @@ -825,18 +566,10 @@ func checkForDuplicateOutputPath(ctx android.ModuleContext, m map[string]string, } // InstallInData returns true as Python is not supported in the system partition -func (p *Module) InstallInData() bool { +func (p *PythonLibraryModule) InstallInData() bool { return true } -func (p *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { - if p.isLibrary() { - pythonLibBp2Build(ctx, p) - } else if p.isBinary() { - pythonBinaryBp2Build(ctx, p) - } -} - var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var String = proptools.String diff --git a/python/python_test.go b/python/python_test.go index 42a1ffb2c..6f4223a7c 100644 --- a/python/python_test.go +++ b/python/python_test.go @@ -312,10 +312,6 @@ var ( "e/file4.py", }, srcsZip: "out/soong/.intermediates/dir/bin/PY3/bin.py.srcszip", - depsSrcsZips: []string{ - "out/soong/.intermediates/dir/lib5/PY3/lib5.py.srcszip", - "out/soong/.intermediates/dir/lib6/PY3/lib6.py.srcszip", - }, }, }, }, @@ -346,17 +342,17 @@ func TestPythonModule(t *testing.T) { for _, e := range d.expectedBinaries { t.Run(e.name, func(t *testing.T) { - expectModule(t, result.TestContext, e.name, e.actualVersion, e.srcsZip, e.pyRunfiles, e.depsSrcsZips) + expectModule(t, result.TestContext, e.name, e.actualVersion, e.srcsZip, e.pyRunfiles) }) } }) } } -func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expectedSrcsZip string, expectedPyRunfiles, expectedDepsSrcsZips []string) { +func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expectedSrcsZip string, expectedPyRunfiles []string) { module := ctx.ModuleForTests(name, variant) - base, baseOk := module.Module().(*Module) + base, baseOk := module.Module().(*PythonLibraryModule) if !baseOk { t.Fatalf("%s is not Python module!", name) } @@ -369,8 +365,6 @@ func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expecte android.AssertDeepEquals(t, "pyRunfiles", expectedPyRunfiles, actualPyRunfiles) android.AssertPathRelativeToTopEquals(t, "srcsZip", expectedSrcsZip, base.srcsZip) - - android.AssertPathsRelativeToTopEquals(t, "depsSrcsZips", expectedDepsSrcsZips, base.depsSrcsZips) } func TestMain(m *testing.M) { diff --git a/python/test.go b/python/test.go index fc5c2112f..fb8e91806 100644 --- a/python/test.go +++ b/python/test.go @@ -32,6 +32,20 @@ func registerPythonTestComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("python_test", PythonTestFactory) } +func NewTest(hod android.HostOrDeviceSupported) *PythonTestModule { + return &PythonTestModule{PythonBinaryModule: *NewBinary(hod)} +} + +func PythonTestHostFactory() android.Module { + return NewTest(android.HostSupportedNoCross).init() +} + +func PythonTestFactory() android.Module { + module := NewTest(android.HostAndDeviceSupported) + module.multilib = android.MultilibBoth + return module.init() +} + type TestProperties struct { // the name of the test configuration (for example "AndroidTest.xml") that should be // installed with the module. @@ -52,76 +66,79 @@ type TestProperties struct { Test_options android.CommonTestOptions } -type testDecorator struct { - *binaryDecorator +type PythonTestModule struct { + PythonBinaryModule testProperties TestProperties - - testConfig android.Path - - data []android.DataPath + testConfig android.Path + data []android.DataPath } -func (test *testDecorator) bootstrapperProps() []interface{} { - return append(test.binaryDecorator.bootstrapperProps(), &test.testProperties) +func (p *PythonTestModule) init() android.Module { + p.AddProperties(&p.properties, &p.protoProperties) + p.AddProperties(&p.binaryProperties) + p.AddProperties(&p.testProperties) + android.InitAndroidArchModule(p, p.hod, p.multilib) + android.InitDefaultableModule(p) + android.InitBazelModule(p) + if p.hod == android.HostSupportedNoCross && p.testProperties.Test_options.Unit_test == nil { + p.testProperties.Test_options.Unit_test = proptools.BoolPtr(true) + } + return p } -func (test *testDecorator) install(ctx android.ModuleContext, file android.Path) { - test.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ - TestConfigProp: test.testProperties.Test_config, - TestConfigTemplateProp: test.testProperties.Test_config_template, - TestSuites: test.binaryDecorator.binaryProperties.Test_suites, - AutoGenConfig: test.binaryDecorator.binaryProperties.Auto_gen_config, +func (p *PythonTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // We inherit from only the library's GenerateAndroidBuildActions, and then + // just use buildBinary() so that the binary is not installed into the location + // it would be for regular binaries. + p.PythonLibraryModule.GenerateAndroidBuildActions(ctx) + p.buildBinary(ctx) + + p.testConfig = tradefed.AutoGenTestConfig(ctx, tradefed.AutoGenTestConfigOptions{ + TestConfigProp: p.testProperties.Test_config, + TestConfigTemplateProp: p.testProperties.Test_config_template, + TestSuites: p.binaryProperties.Test_suites, + AutoGenConfig: p.binaryProperties.Auto_gen_config, DeviceTemplate: "${PythonBinaryHostTestConfigTemplate}", HostTemplate: "${PythonBinaryHostTestConfigTemplate}", }) - test.binaryDecorator.pythonInstaller.dir = "nativetest" - test.binaryDecorator.pythonInstaller.dir64 = "nativetest64" + p.installedDest = ctx.InstallFile(installDir(ctx, "nativetest", "nativetest64", ctx.ModuleName()), p.installSource.Base(), p.installSource) - test.binaryDecorator.pythonInstaller.relative = ctx.ModuleName() - - test.binaryDecorator.pythonInstaller.install(ctx, file) - - dataSrcPaths := android.PathsForModuleSrc(ctx, test.testProperties.Data) - - for _, dataSrcPath := range dataSrcPaths { - test.data = append(test.data, android.DataPath{SrcPath: dataSrcPath}) + for _, dataSrcPath := range android.PathsForModuleSrc(ctx, p.testProperties.Data) { + p.data = append(p.data, android.DataPath{SrcPath: dataSrcPath}) } // Emulate the data property for java_data dependencies. for _, javaData := range ctx.GetDirectDepsWithTag(javaDataTag) { for _, javaDataSrcPath := range android.OutputFilesForModule(ctx, javaData, "") { - test.data = append(test.data, android.DataPath{SrcPath: javaDataSrcPath}) + p.data = append(p.data, android.DataPath{SrcPath: javaDataSrcPath}) } } } -func NewTest(hod android.HostOrDeviceSupported) *Module { - module, binary := NewBinary(hod) - - binary.pythonInstaller = NewPythonInstaller("nativetest", "nativetest64") - - test := &testDecorator{binaryDecorator: binary} - if hod == android.HostSupportedNoCross && test.testProperties.Test_options.Unit_test == nil { - test.testProperties.Test_options.Unit_test = proptools.BoolPtr(true) +func (p *PythonTestModule) AndroidMkEntries() []android.AndroidMkEntries { + entriesList := p.PythonBinaryModule.AndroidMkEntries() + if len(entriesList) != 1 { + panic("Expected 1 entry") } + entries := &entriesList[0] - module.bootstrapper = test - module.installer = test + entries.Class = "NATIVE_TESTS" - return module -} + entries.ExtraEntries = append(entries.ExtraEntries, + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + //entries.AddCompatibilityTestSuites(p.binaryProperties.Test_suites...) + if p.testConfig != nil { + entries.SetString("LOCAL_FULL_TEST_CONFIG", p.testConfig.String()) + } -func PythonTestHostFactory() android.Module { - module := NewTest(android.HostSupportedNoCross) + entries.SetBoolIfTrue("LOCAL_DISABLE_AUTO_GENERATE_TEST_CONFIG", !BoolDefault(p.binaryProperties.Auto_gen_config, true)) - return module.init() -} + entries.AddStrings("LOCAL_TEST_DATA", android.AndroidMkDataPaths(p.data)...) -func PythonTestFactory() android.Module { - module := NewTest(android.HostAndDeviceSupported) - module.multilib = android.MultilibBoth + p.testProperties.Test_options.SetAndroidMkEntries(entries) + }) - return module.init() + return entriesList } diff --git a/rust/config/global.go b/rust/config/global.go index 26e2d06ff..7549969b0 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.65.0" + RustDefaultVersion = "1.65.0.p1" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ diff --git a/zip/cmd/main.go b/zip/cmd/main.go index cbc73eda6..def76aa62 100644 --- a/zip/cmd/main.go +++ b/zip/cmd/main.go @@ -163,6 +163,7 @@ func main() { parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use") cpuProfile := flags.String("cpuprofile", "", "write cpu profile to file") traceFile := flags.String("trace", "", "write trace to file") + sha256Checksum := flags.Bool("sha256", false, "add a zip header to each file containing its SHA256 digest") flags.Var(&rootPrefix{}, "P", "path prefix within the zip at which to place files") flags.Var(&listFiles{}, "l", "file containing list of files to zip") @@ -224,6 +225,7 @@ func main() { WriteIfChanged: *writeIfChanged, StoreSymlinks: *symlinks, IgnoreMissingFiles: *ignoreMissingFiles, + Sha256Checksum: *sha256Checksum, }) if err != nil { fmt.Fprintln(os.Stderr, "error:", err.Error()) diff --git a/zip/zip.go b/zip/zip.go index 955fe68d0..6f1a8adaf 100644 --- a/zip/zip.go +++ b/zip/zip.go @@ -17,8 +17,11 @@ package zip import ( "bytes" "compress/flate" + "crypto/sha256" + "encoding/binary" "errors" "fmt" + "hash" "hash/crc32" "io" "io/ioutil" @@ -38,6 +41,14 @@ import ( "android/soong/third_party/zip" ) +// Sha256HeaderID is a custom Header ID for the `extra` field in +// the file header to store the SHA checksum. +const Sha256HeaderID = 0x4967 + +// Sha256HeaderSignature is the signature to verify that the extra +// data block is used to store the SHA checksum. +const Sha256HeaderSignature = 0x9514 + // Block size used during parallel compression of a single file. const parallelBlockSize = 1 * 1024 * 1024 // 1MB @@ -231,6 +242,8 @@ type ZipWriter struct { stderr io.Writer fs pathtools.FileSystem + + sha256Checksum bool } type zipEntry struct { @@ -257,6 +270,7 @@ type ZipArgs struct { WriteIfChanged bool StoreSymlinks bool IgnoreMissingFiles bool + Sha256Checksum bool Stderr io.Writer Filesystem pathtools.FileSystem @@ -280,6 +294,7 @@ func zipTo(args ZipArgs, w io.Writer) error { ignoreMissingFiles: args.IgnoreMissingFiles, stderr: args.Stderr, fs: args.Filesystem, + sha256Checksum: args.Sha256Checksum, } if z.fs == nil { @@ -782,15 +797,17 @@ func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.Reader // this based on actual buffer sizes in RateLimit. ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1) - // Calculate the CRC in the background, since reading the entire - // file could take a while. + // Calculate the CRC and SHA256 in the background, since reading + // the entire file could take a while. // // We could split this up into chunks as well, but it's faster // than the compression. Due to the Go Zip API, we also need to // know the result before we can begin writing the compressed // data out to the zipfile. + // + // We calculate SHA256 only if `-sha256` is set. wg.Add(1) - go z.crcFile(r, ze, compressChan, wg) + go z.checksumFileAsync(r, ze, compressChan, wg) for start := int64(0); start < fileSize; start += parallelBlockSize { sr := io.NewSectionReader(r, start, parallelBlockSize) @@ -829,20 +846,53 @@ func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.Reader return nil } -func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) { +func (z *ZipWriter) checksumFileAsync(r io.ReadSeeker, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) { defer wg.Done() defer z.cpuRateLimiter.Finish() + z.checksumFile(r, ze) + + resultChan <- ze + close(resultChan) +} + +func (z *ZipWriter) checksumFile(r io.ReadSeeker, ze *zipEntry) { crc := crc32.NewIEEE() - _, err := io.Copy(crc, r) + writers := []io.Writer{crc} + + var shaHasher hash.Hash + if z.sha256Checksum && !ze.fh.Mode().IsDir() { + shaHasher = sha256.New() + writers = append(writers, shaHasher) + } + + w := io.MultiWriter(writers...) + + _, err := io.Copy(w, r) if err != nil { z.errors <- err return } ze.fh.CRC32 = crc.Sum32() - resultChan <- ze - close(resultChan) + if shaHasher != nil { + z.appendSHAToExtra(ze, shaHasher.Sum(nil)) + } +} + +func (z *ZipWriter) appendSHAToExtra(ze *zipEntry, checksum []byte) { + // The block of SHA256 checksum consist of: + // - Header ID, equals to Sha256HeaderID (2 bytes) + // - Data size (2 bytes) + // - Data block: + // - Signature, equals to Sha256HeaderSignature (2 bytes) + // - Data, SHA checksum value + var buf []byte + buf = binary.LittleEndian.AppendUint16(buf, Sha256HeaderID) + buf = binary.LittleEndian.AppendUint16(buf, uint16(len(checksum)+2)) + buf = binary.LittleEndian.AppendUint16(buf, Sha256HeaderSignature) + buf = append(buf, checksum...) + ze.fh.Extra = append(ze.fh.Extra, buf...) } func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) { @@ -894,17 +944,9 @@ func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.B } func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) { + z.checksumFile(r, ze) - crc := crc32.NewIEEE() - _, err := io.Copy(crc, r) - if err != nil { - z.errors <- err - return - } - - ze.fh.CRC32 = crc.Sum32() - - _, err = r.Seek(0, 0) + _, err := r.Seek(0, 0) if err != nil { z.errors <- err return diff --git a/zip/zip_test.go b/zip/zip_test.go index c4832dc9a..e7fdea839 100644 --- a/zip/zip_test.go +++ b/zip/zip_test.go @@ -16,6 +16,7 @@ package zip import ( "bytes" + "encoding/hex" "hash/crc32" "io" "os" @@ -35,6 +36,10 @@ var ( fileEmpty = []byte("") fileManifest = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\n\n") + sha256FileA = "d53eda7a637c99cc7fb566d96e9fa109bf15c478410a3f5eb4d4c4e26cd081f6" + sha256FileB = "430c56c5818e62bcb6d478901ef86284e97714c138f3c86aa14fd6a84b7ce5d3" + sha256FileC = "31c5ab6111f1d6aa13c2c4e92bb3c0f7c76b61b42d141af1e846eb7f6586a51c" + fileCustomManifest = []byte("Custom manifest: true\n") customManifestAfter = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\nCustom manifest: true\n\n") ) @@ -67,6 +72,20 @@ func fh(name string, contents []byte, method uint16) zip.FileHeader { } } +func fhWithSHA256(name string, contents []byte, method uint16, sha256 string) zip.FileHeader { + h := fh(name, contents, method) + // The extra field contains 38 bytes, including 2 bytes of header ID, 2 bytes + // of size, 2 bytes of signature, and 32 bytes of checksum data block. + var extra [38]byte + // The first 6 bytes contains Sha256HeaderID (0x4967), size (unit(34)) and + // Sha256HeaderSignature (0x9514) + copy(extra[0:], []byte{103, 73, 34, 0, 20, 149}) + sha256Bytes, _ := hex.DecodeString(sha256) + copy(extra[6:], sha256Bytes) + h.Extra = append(h.Extra, extra[:]...) + return h +} + func fhManifest(contents []byte) zip.FileHeader { return zip.FileHeader{ Name: "META-INF/MANIFEST.MF", @@ -87,13 +106,18 @@ func fhLink(name string, to string) zip.FileHeader { } } -func fhDir(name string) zip.FileHeader { +type fhDirOptions struct { + extra []byte +} + +func fhDir(name string, opts fhDirOptions) zip.FileHeader { return zip.FileHeader{ Name: name, Method: zip.Store, CRC32: crc32.ChecksumIEEE(nil), UncompressedSize64: 0, ExternalAttrs: (syscall.S_IFDIR|0755)<<16 | 0x10, + Extra: opts.extra, } } @@ -114,6 +138,7 @@ func TestZip(t *testing.T) { manifest string storeSymlinks bool ignoreMissingFiles bool + sha256Checksum bool files []zip.FileHeader err error @@ -320,10 +345,10 @@ func TestZip(t *testing.T) { emulateJar: true, files: []zip.FileHeader{ - fhDir("META-INF/"), + fhDir("META-INF/", fhDirOptions{extra: []byte{254, 202, 0, 0}}), fhManifest(fileManifest), - fhDir("a/"), - fhDir("a/a/"), + fhDir("a/", fhDirOptions{}), + fhDir("a/a/", fhDirOptions{}), fh("a/a/a", fileA, zip.Deflate), fh("a/a/b", fileB, zip.Deflate), }, @@ -338,10 +363,10 @@ func TestZip(t *testing.T) { manifest: "manifest.txt", files: []zip.FileHeader{ - fhDir("META-INF/"), + fhDir("META-INF/", fhDirOptions{extra: []byte{254, 202, 0, 0}}), fhManifest(customManifestAfter), - fhDir("a/"), - fhDir("a/a/"), + fhDir("a/", fhDirOptions{}), + fhDir("a/a/", fhDirOptions{}), fh("a/a/a", fileA, zip.Deflate), fh("a/a/b", fileB, zip.Deflate), }, @@ -355,8 +380,8 @@ func TestZip(t *testing.T) { dirEntries: true, files: []zip.FileHeader{ - fhDir("a/"), - fhDir("a/a/"), + fhDir("a/", fhDirOptions{}), + fhDir("a/a/", fhDirOptions{}), fh("a/a/a", fileA, zip.Deflate), fh("a/a/b", fileB, zip.Deflate), }, @@ -412,6 +437,23 @@ func TestZip(t *testing.T) { fh("a/a/a", fileA, zip.Deflate), }, }, + { + name: "generate SHA256 checksum", + args: fileArgsBuilder(). + File("a/a/a"). + File("a/a/b"). + File("a/a/c"). + File("c"), + compressionLevel: 9, + sha256Checksum: true, + + files: []zip.FileHeader{ + fhWithSHA256("a/a/a", fileA, zip.Deflate, sha256FileA), + fhWithSHA256("a/a/b", fileB, zip.Deflate, sha256FileB), + fhWithSHA256("a/a/c", fileC, zip.Deflate, sha256FileC), + fhWithSHA256("c", fileC, zip.Deflate, sha256FileC), + }, + }, // errors { @@ -465,6 +507,7 @@ func TestZip(t *testing.T) { args.ManifestSourcePath = test.manifest args.StoreSymlinks = test.storeSymlinks args.IgnoreMissingFiles = test.ignoreMissingFiles + args.Sha256Checksum = test.sha256Checksum args.Filesystem = mockFs args.Stderr = &bytes.Buffer{} @@ -555,6 +598,11 @@ func TestZip(t *testing.T) { t.Errorf("incorrect file %s method want %v got %v", want.Name, want.Method, got.Method) } + + if !bytes.Equal(want.Extra, got.Extra) { + t.Errorf("incorrect file %s extra want %v got %v", want.Name, + want.Extra, got.Extra) + } } }) } |