diff options
88 files changed, 3365 insertions, 1923 deletions
diff --git a/Android.bp b/Android.bp index 63de01589..b1db8e937 100644 --- a/Android.bp +++ b/Android.bp @@ -130,3 +130,8 @@ buildinfo_prop { // Currently, only microdroid can refer to buildinfo.prop visibility: ["//packages/modules/Virtualization/microdroid"], } + +// container for apex_contributions selected using build flags +all_apex_contributions { + name: "all_apex_contributions", +} diff --git a/aconfig/cc_aconfig_library.go b/aconfig/cc_aconfig_library.go index 5b0fb4a82..210a58105 100644 --- a/aconfig/cc_aconfig_library.go +++ b/aconfig/cc_aconfig_library.go @@ -38,8 +38,11 @@ type CcAconfigLibraryProperties struct { // name of the aconfig_declarations module to generate a library for Aconfig_declarations string - // whether to generate test mode version of the library - Test *bool + // default mode is "production", the other accepted modes are: + // "test": to generate test mode version of the library + // "exported": to generate exported mode version of the library + // an error will be thrown if the mode is not supported + Mode *string } type CcAconfigLibraryCallbacks struct { @@ -121,12 +124,11 @@ func (this *CcAconfigLibraryCallbacks) GeneratorBuildActions(ctx cc.ModuleContex } declarations := ctx.OtherModuleProvider(declarationsModules[0], declarationsProviderKey).(declarationsProviderData) - var mode string - if proptools.Bool(this.properties.Test) { - mode = "test" - } else { - mode = "production" + mode := proptools.StringDefault(this.properties.Mode, "production") + if !isModeSupported(mode) { + ctx.PropertyErrorf("mode", "%q is not a supported mode", mode) } + ctx.Build(pctx, android.BuildParams{ Rule: cppRule, Input: declarations.IntermediatePath, diff --git a/aconfig/cc_aconfig_library_test.go b/aconfig/cc_aconfig_library_test.go index 6f17c7594..ba2725059 100644 --- a/aconfig/cc_aconfig_library_test.go +++ b/aconfig/cc_aconfig_library_test.go @@ -22,16 +22,17 @@ import ( "android/soong/cc" ) -var codegenModeTestData = []struct { +var ccCodegenModeTestData = []struct { setting, expected string }{ {"", "production"}, - {"test: false,", "production"}, - {"test: true,", "test"}, + {"mode: `production`,", "production"}, + {"mode: `test`,", "test"}, + {"mode: `exported`,", "exported"}, } func TestCCCodegenMode(t *testing.T) { - for _, testData := range codegenModeTestData { + for _, testData := range ccCodegenModeTestData { testCCCodegenModeHelper(t, testData.setting, testData.expected) } } @@ -65,3 +66,41 @@ func testCCCodegenModeHelper(t *testing.T, bpMode string, ruleMode string) { rule := module.Rule("cc_aconfig_library") android.AssertStringEquals(t, "rule must contain test mode", rule.Args["mode"], ruleMode) } + +var incorrectCCCodegenModeTestData = []struct { + setting, expectedErr string +}{ + {"mode: `unsupported`,", "mode: \"unsupported\" is not a supported mode"}, +} + +func TestIncorrectCCCodegenMode(t *testing.T) { + for _, testData := range incorrectCCCodegenModeTestData { + testIncorrectCCCodegenModeHelper(t, testData.setting, testData.expectedErr) + } +} + +func testIncorrectCCCodegenModeHelper(t *testing.T, bpMode string, err string) { + t.Helper() + android.GroupFixturePreparers( + PrepareForTestWithAconfigBuildComponents, + cc.PrepareForTestWithCcDefaultModules). + ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern(err)). + RunTestWithBp(t, fmt.Sprintf(` + aconfig_declarations { + name: "my_aconfig_declarations", + package: "com.example.package", + srcs: ["foo.aconfig"], + } + + cc_library { + name: "server_configurable_flags", + srcs: ["server_configurable_flags.cc"], + } + + cc_aconfig_library { + name: "my_cc_aconfig_library", + aconfig_declarations: "my_aconfig_declarations", + %s + } + `, bpMode)) +} diff --git a/aconfig/init.go b/aconfig/init.go index 3d62714bc..626e66d0f 100644 --- a/aconfig/init.go +++ b/aconfig/init.go @@ -40,7 +40,7 @@ var ( Restat: true, }, "release_version", "package", "declarations", "values", "default-permission") - // For java_aconfig_library: Generate java file + // For java_aconfig_library: Generate java library javaRule = pctx.AndroidStaticRule("java_aconfig_library", blueprint.RuleParams{ Command: `rm -rf ${out}.tmp` + @@ -58,7 +58,7 @@ var ( Restat: true, }, "mode") - // For java_aconfig_library: Generate java file + // For cc_aconfig_library: Generate C++ library cppRule = pctx.AndroidStaticRule("cc_aconfig_library", blueprint.RuleParams{ Command: `rm -rf ${gendir}` + @@ -69,10 +69,10 @@ var ( ` --out ${gendir}`, CommandDeps: []string{ "$aconfig", - "$soong_zip", }, }, "gendir", "mode") + // For rust_aconfig_library: Generate Rust library rustRule = pctx.AndroidStaticRule("rust_aconfig_library", blueprint.RuleParams{ Command: `rm -rf ${gendir}` + @@ -83,11 +83,10 @@ var ( ` --out ${gendir}`, CommandDeps: []string{ "$aconfig", - "$soong_zip", }, }, "gendir", "mode") - // For all_aconfig_declarations + // For all_aconfig_declarations: Combine all parsed_flags proto files allDeclarationsRule = pctx.AndroidStaticRule("all_aconfig_declarations_dump", blueprint.RuleParams{ Command: `${aconfig} dump --format protobuf --out ${out} ${cache_files}`, diff --git a/aconfig/java_aconfig_library.go b/aconfig/java_aconfig_library.go index f7f8db85d..eedb3c358 100644 --- a/aconfig/java_aconfig_library.go +++ b/aconfig/java_aconfig_library.go @@ -30,12 +30,17 @@ type declarationsTagType struct { var declarationsTag = declarationsTagType{} +var aconfigSupportedModes = []string{"production", "test", "exported"} + type JavaAconfigDeclarationsLibraryProperties struct { // name of the aconfig_declarations module to generate a library for Aconfig_declarations string - // whether to generate test mode version of the library - Test *bool + // default mode is "production", the other accepted modes are: + // "test": to generate test mode version of the library + // "exported": to generate exported mode version of the library + // an error will be thrown if the mode is not supported + Mode *string } type JavaAconfigDeclarationsLibraryCallbacks struct { @@ -72,12 +77,12 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) GenerateSourceJarBuild // Generate the action to build the srcjar srcJarPath := android.PathForModuleGen(ctx, ctx.ModuleName()+".srcjar") - var mode string - if proptools.Bool(callbacks.properties.Test) { - mode = "test" - } else { - mode = "production" + + mode := proptools.StringDefault(callbacks.properties.Mode, "production") + if !isModeSupported(mode) { + ctx.PropertyErrorf("mode", "%q is not a supported mode", mode) } + ctx.Build(pctx, android.BuildParams{ Rule: javaRule, Input: declarations.IntermediatePath, @@ -95,9 +100,12 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) GenerateSourceJarBuild return srcJarPath } +func isModeSupported(mode string) bool { + return android.InList(mode, aconfigSupportedModes) +} + type bazelJavaAconfigLibraryAttributes struct { Aconfig_declarations bazel.LabelAttribute - Test *bool Sdk_version *string Libs bazel.LabelListAttribute } @@ -138,7 +146,6 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) Bp2build(ctx android.B attrs := bazelJavaAconfigLibraryAttributes{ Aconfig_declarations: *bazel.MakeLabelAttribute(android.BazelLabelForModuleDepSingle(ctx, callbacks.properties.Aconfig_declarations).Label), - Test: callbacks.properties.Test, Sdk_version: &sdkVersion, Libs: libs, } diff --git a/aconfig/java_aconfig_library_test.go b/aconfig/java_aconfig_library_test.go index af50848f1..a803672db 100644 --- a/aconfig/java_aconfig_library_test.go +++ b/aconfig/java_aconfig_library_test.go @@ -178,14 +178,42 @@ func testCodegenMode(t *testing.T, bpMode string, ruleMode string) { android.AssertStringEquals(t, "rule must contain test mode", rule.Args["mode"], ruleMode) } +func testCodegenModeWithError(t *testing.T, bpMode string, err string) { + android.GroupFixturePreparers( + PrepareForTestWithAconfigBuildComponents, + java.PrepareForTestWithJavaDefaultModules). + ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern(err)). + RunTestWithBp(t, fmt.Sprintf(` + aconfig_declarations { + name: "my_aconfig_declarations", + package: "com.example.package", + srcs: ["foo.aconfig"], + } + + java_aconfig_library { + name: "my_java_aconfig_library", + aconfig_declarations: "my_aconfig_declarations", + %s + } + `, bpMode)) +} + func TestDefaultProdMode(t *testing.T) { testCodegenMode(t, "", "production") } func TestProdMode(t *testing.T) { - testCodegenMode(t, "test: false,", "production") + testCodegenMode(t, "mode: `production`,", "production") } func TestTestMode(t *testing.T) { - testCodegenMode(t, "test: true,", "test") + testCodegenMode(t, "mode: `test`,", "test") +} + +func TestExportedMode(t *testing.T) { + testCodegenMode(t, "mode: `exported`,", "exported") +} + +func TestUnsupportedMode(t *testing.T) { + testCodegenModeWithError(t, "mode: `unsupported`,", "mode: \"unsupported\" is not a supported mode") } diff --git a/aconfig/rust_aconfig_library.go b/aconfig/rust_aconfig_library.go index de41776a3..265685e6f 100644 --- a/aconfig/rust_aconfig_library.go +++ b/aconfig/rust_aconfig_library.go @@ -1,9 +1,10 @@ package aconfig import ( + "fmt" + "android/soong/android" "android/soong/rust" - "fmt" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -18,7 +19,12 @@ var rustDeclarationsTag = rustDeclarationsTagType{} type RustAconfigLibraryProperties struct { // name of the aconfig_declarations module to generate a library for Aconfig_declarations string - Test *bool + + // default mode is "production", the other accepted modes are: + // "test": to generate test mode version of the library + // "exported": to generate exported mode version of the library + // an error will be thrown if the mode is not supported + Mode *string } type aconfigDecorator struct { @@ -60,9 +66,9 @@ func (a *aconfigDecorator) GenerateSource(ctx rust.ModuleContext, deps rust.Path } declarations := ctx.OtherModuleProvider(declarationsModules[0], declarationsProviderKey).(declarationsProviderData) - mode := "production" - if proptools.Bool(a.Properties.Test) { - mode = "test" + mode := proptools.StringDefault(a.Properties.Mode, "production") + if !isModeSupported(mode) { + ctx.PropertyErrorf("mode", "%q is not a supported mode", mode) } ctx.Build(pctx, android.BuildParams{ @@ -84,6 +90,7 @@ func (a *aconfigDecorator) GenerateSource(ctx rust.ModuleContext, deps rust.Path func (a *aconfigDecorator) SourceProviderDeps(ctx rust.DepsContext, deps rust.Deps) rust.Deps { deps = a.BaseSourceProvider.SourceProviderDeps(ctx, deps) deps.Rustlibs = append(deps.Rustlibs, "libflags_rust") + deps.Rustlibs = append(deps.Rustlibs, "liblazy_static") ctx.AddDependency(ctx.Module(), rustDeclarationsTag, a.Properties.Aconfig_declarations) return deps } diff --git a/aconfig/rust_aconfig_library_test.go b/aconfig/rust_aconfig_library_test.go index 17385c3d1..3aeab7622 100644 --- a/aconfig/rust_aconfig_library_test.go +++ b/aconfig/rust_aconfig_library_test.go @@ -1,10 +1,11 @@ package aconfig import ( - "android/soong/android" - "android/soong/rust" "fmt" "testing" + + "android/soong/android" + "android/soong/rust" ) func TestRustAconfigLibrary(t *testing.T) { @@ -22,6 +23,11 @@ func TestRustAconfigLibrary(t *testing.T) { crate_name: "flags_rust", srcs: ["lib.rs"], } + rust_library { + name: "liblazy_static", // test mock + crate_name: "lazy_static", + srcs: ["src/lib.rs"], + } aconfig_declarations { name: "my_aconfig_declarations", package: "com.example.package", @@ -58,3 +64,96 @@ func TestRustAconfigLibrary(t *testing.T) { ) } } + +var rustCodegenModeTestData = []struct { + setting, expected string +}{ + {"", "production"}, + {"mode: `production`,", "production"}, + {"mode: `test`,", "test"}, + {"mode: `exported`,", "exported"}, +} + +func TestRustCodegenMode(t *testing.T) { + for _, testData := range rustCodegenModeTestData { + testRustCodegenModeHelper(t, testData.setting, testData.expected) + } +} + +func testRustCodegenModeHelper(t *testing.T, bpMode string, ruleMode string) { + t.Helper() + result := android.GroupFixturePreparers( + PrepareForTestWithAconfigBuildComponents, + rust.PrepareForTestWithRustIncludeVndk). + ExtendWithErrorHandler(android.FixtureExpectsNoErrors). + RunTestWithBp(t, fmt.Sprintf(` + rust_library { + name: "libflags_rust", // test mock + crate_name: "flags_rust", + srcs: ["lib.rs"], + } + rust_library { + name: "liblazy_static", // test mock + crate_name: "lazy_static", + srcs: ["src/lib.rs"], + } + aconfig_declarations { + name: "my_aconfig_declarations", + package: "com.example.package", + srcs: ["foo.aconfig"], + } + rust_aconfig_library { + name: "libmy_rust_aconfig_library", + crate_name: "my_rust_aconfig_library", + aconfig_declarations: "my_aconfig_declarations", + %s + } + `, bpMode)) + + module := result.ModuleForTests("libmy_rust_aconfig_library", "android_arm64_armv8-a_source") + rule := module.Rule("rust_aconfig_library") + android.AssertStringEquals(t, "rule must contain test mode", rule.Args["mode"], ruleMode) +} + +var incorrectRustCodegenModeTestData = []struct { + setting, expectedErr string +}{ + {"mode: `unsupported`,", "mode: \"unsupported\" is not a supported mode"}, +} + +func TestIncorrectRustCodegenMode(t *testing.T) { + for _, testData := range incorrectRustCodegenModeTestData { + testIncorrectRustCodegenModeHelper(t, testData.setting, testData.expectedErr) + } +} + +func testIncorrectRustCodegenModeHelper(t *testing.T, bpMode string, err string) { + t.Helper() + android.GroupFixturePreparers( + PrepareForTestWithAconfigBuildComponents, + rust.PrepareForTestWithRustIncludeVndk). + ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern(err)). + RunTestWithBp(t, fmt.Sprintf(` + rust_library { + name: "libflags_rust", // test mock + crate_name: "flags_rust", + srcs: ["lib.rs"], + } + rust_library { + name: "liblazy_static", // test mock + crate_name: "lazy_static", + srcs: ["src/lib.rs"], + } + aconfig_declarations { + name: "my_aconfig_declarations", + package: "com.example.package", + srcs: ["foo.aconfig"], + } + rust_aconfig_library { + name: "libmy_rust_aconfig_library", + crate_name: "my_rust_aconfig_library", + aconfig_declarations: "my_aconfig_declarations", + %s + } + `, bpMode)) +} diff --git a/android/Android.bp b/android/Android.bp index 7fbba4362..62f534c53 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -37,6 +37,7 @@ bootstrap_go_package { "api_levels.go", "arch.go", "arch_list.go", + "base_module_context.go", "bazel.go", "bazel_handler.go", "bazel_paths.go", @@ -51,6 +52,7 @@ bootstrap_go_package { "defs.go", "depset_generic.go", "deptag.go", + "early_module_context.go", "expand.go", "filegroup.go", "fixture.go", @@ -65,6 +67,7 @@ bootstrap_go_package { "makevars.go", "metrics.go", "module.go", + "module_context.go", "mutator.go", "namespace.go", "neverallow.go", diff --git a/android/apex_contributions.go b/android/apex_contributions.go index a28ac31bf..9b188deed 100644 --- a/android/apex_contributions.go +++ b/android/apex_contributions.go @@ -15,6 +15,7 @@ package android import ( + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -24,6 +25,7 @@ func init() { func RegisterApexContributionsBuildComponents(ctx RegistrationContext) { ctx.RegisterModuleType("apex_contributions", apexContributionsFactory) + ctx.RegisterSingletonModuleType("all_apex_contributions", allApexContributionsFactory) } type apexContributions struct { @@ -65,3 +67,109 @@ func apexContributionsFactory() Module { // prebuilts selection. func (m *apexContributions) GenerateAndroidBuildActions(ctx ModuleContext) { } + +// A container for apex_contributions. +// Based on product_config, it will create a dependency on the selected +// apex_contributions per mainline module +type allApexContributions struct { + SingletonModuleBase +} + +func allApexContributionsFactory() SingletonModule { + module := &allApexContributions{} + InitAndroidModule(module) + return module +} + +type apexContributionsDepTag struct { + blueprint.BaseDependencyTag +} + +var ( + acDepTag = apexContributionsDepTag{} +) + +// Creates a dep to each selected apex_contributions +func (a *allApexContributions) DepsMutator(ctx BottomUpMutatorContext) { + ctx.AddDependency(ctx.Module(), acDepTag, ctx.Config().AllApexContributions()...) +} + +// Set PrebuiltSelectionInfoProvider in post deps phase +func (a *allApexContributions) SetPrebuiltSelectionInfoProvider(ctx BaseModuleContext) { + addContentsToProvider := func(p *PrebuiltSelectionInfoMap, m *apexContributions) { + for _, content := range m.Contents() { + if !ctx.OtherModuleExists(content) && !ctx.Config().AllowMissingDependencies() { + ctx.ModuleErrorf("%s listed in apex_contributions %s does not exist\n", content, m.Name()) + } + pi := &PrebuiltSelectionInfo{ + baseModuleName: RemoveOptionalPrebuiltPrefix(content), + selectedModuleName: content, + metadataModuleName: m.Name(), + apiDomain: m.ApiDomain(), + } + p.Add(ctx, pi) + } + } + + if ctx.Config().Bp2buildMode() { // Skip bp2build + return + } + p := PrebuiltSelectionInfoMap{} + ctx.VisitDirectDepsWithTag(acDepTag, func(child Module) { + if m, ok := child.(*apexContributions); ok { + addContentsToProvider(&p, m) + } else { + ctx.ModuleErrorf("%s is not an apex_contributions module\n", child.Name()) + } + }) + ctx.SetProvider(PrebuiltSelectionInfoProvider, p) +} + +// A provider containing metadata about whether source or prebuilt should be used +// This provider will be used in prebuilt_select mutator to redirect deps +var PrebuiltSelectionInfoProvider = blueprint.NewMutatorProvider(PrebuiltSelectionInfoMap{}, "prebuilt_select") + +// Map of baseModuleName to the selected source or prebuilt +type PrebuiltSelectionInfoMap map[string]PrebuiltSelectionInfo + +// Add a new entry to the map with some validations +func (pm *PrebuiltSelectionInfoMap) Add(ctx BaseModuleContext, p *PrebuiltSelectionInfo) { + if p == nil { + return + } + // Do not allow dups. If the base module (without the prebuilt_) has been added before, raise an exception. + if old, exists := (*pm)[p.baseModuleName]; exists { + ctx.ModuleErrorf("Cannot use Soong module: %s from apex_contributions: %s because it has been added previously as: %s from apex_contributions: %s\n", + p.selectedModuleName, p.metadataModuleName, old.selectedModuleName, old.metadataModuleName, + ) + } + (*pm)[p.baseModuleName] = *p +} + +type PrebuiltSelectionInfo struct { + // e.g. libc + baseModuleName string + // e.g. (libc|prebuilt_libc) + selectedModuleName string + // Name of the apex_contributions module + metadataModuleName string + // e.g. com.android.runtime + apiDomain string +} + +// Returns true if `name` is explicitly requested using one of the selected +// apex_contributions metadata modules. +func (p *PrebuiltSelectionInfoMap) IsSelected(baseModuleName, name string) bool { + if i, exists := (*p)[baseModuleName]; exists { + return i.selectedModuleName == name + } else { + return false + } +} + +// This module type does not have any build actions. +func (a *allApexContributions) GenerateAndroidBuildActions(ctx ModuleContext) { +} + +func (a *allApexContributions) GenerateSingletonBuildActions(ctx SingletonContext) { +} diff --git a/android/base_module_context.go b/android/base_module_context.go new file mode 100644 index 000000000..ec9c888be --- /dev/null +++ b/android/base_module_context.go @@ -0,0 +1,656 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "github.com/google/blueprint" + "regexp" + "strings" +) + +// BaseModuleContext is the same as blueprint.BaseModuleContext except that Config() returns +// a Config instead of an interface{}, and some methods have been wrapped to use an android.Module +// instead of a blueprint.Module, plus some extra methods that return Android-specific information +// about the current module. +type BaseModuleContext interface { + EarlyModuleContext + + blueprintBaseModuleContext() blueprint.BaseModuleContext + + // OtherModuleName returns the name of another Module. See BaseModuleContext.ModuleName for more information. + // It is intended for use inside the visit functions of Visit* and WalkDeps. + OtherModuleName(m blueprint.Module) string + + // OtherModuleDir returns the directory of another Module. See BaseModuleContext.ModuleDir for more information. + // It is intended for use inside the visit functions of Visit* and WalkDeps. + OtherModuleDir(m blueprint.Module) string + + // OtherModuleErrorf reports an error on another Module. See BaseModuleContext.ModuleErrorf for more information. + // It is intended for use inside the visit functions of Visit* and WalkDeps. + OtherModuleErrorf(m blueprint.Module, fmt string, args ...interface{}) + + // OtherModuleDependencyTag returns the dependency tag used to depend on a module, or nil if there is no dependency + // on the module. When called inside a Visit* method with current module being visited, and there are multiple + // dependencies on the module being visited, it returns the dependency tag used for the current dependency. + OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag + + // OtherModuleExists returns true if a module with the specified name exists, as determined by the NameInterface + // passed to Context.SetNameInterface, or SimpleNameInterface if it was not called. + OtherModuleExists(name string) bool + + // OtherModuleDependencyVariantExists returns true if a module with the + // specified name and variant exists. The variant must match the given + // variations. It must also match all the non-local variations of the current + // module. In other words, it checks for the module that AddVariationDependencies + // would add a dependency on with the same arguments. + OtherModuleDependencyVariantExists(variations []blueprint.Variation, name string) bool + + // OtherModuleFarDependencyVariantExists returns true if a module with the + // specified name and variant exists. The variant must match the given + // variations, but not the non-local variations of the current module. In + // other words, it checks for the module that AddFarVariationDependencies + // would add a dependency on with the same arguments. + OtherModuleFarDependencyVariantExists(variations []blueprint.Variation, name string) bool + + // OtherModuleReverseDependencyVariantExists returns true if a module with the + // specified name exists with the same variations as the current module. In + // other words, it checks for the module that AddReverseDependency would add a + // dependency on with the same argument. + OtherModuleReverseDependencyVariantExists(name string) bool + + // OtherModuleType returns the type of another Module. See BaseModuleContext.ModuleType for more information. + // It is intended for use inside the visit functions of Visit* and WalkDeps. + OtherModuleType(m blueprint.Module) string + + // OtherModuleProvider returns the value for a provider for the given module. If the value is + // not set it returns the zero value of the type of the provider, so the return value can always + // be type asserted to the type of the provider. The value returned may be a deep copy of the + // value originally passed to SetProvider. + OtherModuleProvider(m blueprint.Module, provider blueprint.ProviderKey) interface{} + + // OtherModuleHasProvider returns true if the provider for the given module has been set. + OtherModuleHasProvider(m blueprint.Module, provider blueprint.ProviderKey) bool + + // Provider returns the value for a provider for the current module. If the value is + // not set it returns the zero value of the type of the provider, so the return value can always + // be type asserted to the type of the provider. It panics if called before the appropriate + // mutator or GenerateBuildActions pass for the provider. The value returned may be a deep + // copy of the value originally passed to SetProvider. + Provider(provider blueprint.ProviderKey) interface{} + + // HasProvider returns true if the provider for the current module has been set. + HasProvider(provider blueprint.ProviderKey) bool + + // SetProvider sets the value for a provider for the current module. It panics if not called + // during the appropriate mutator or GenerateBuildActions pass for the provider, if the value + // is not of the appropriate type, or if the value has already been set. The value should not + // be modified after being passed to SetProvider. + SetProvider(provider blueprint.ProviderKey, value interface{}) + + GetDirectDepsWithTag(tag blueprint.DependencyTag) []Module + + // GetDirectDepWithTag returns the Module the direct dependency with the specified name, or nil if + // none exists. It panics if the dependency does not have the specified tag. It skips any + // dependencies that are not an android.Module. + GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module + + // GetDirectDep returns the Module and DependencyTag for the direct dependency with the specified + // name, or nil if none exists. If there are multiple dependencies on the same module it returns + // the first DependencyTag. + GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) + + ModuleFromName(name string) (blueprint.Module, bool) + + // VisitDirectDepsBlueprint calls visit for each direct dependency. If there are multiple + // direct dependencies on the same module visit will be called multiple times on that module + // and OtherModuleDependencyTag will return a different tag for each. + // + // The Module passed to the visit function should not be retained outside of the visit + // function, it may be invalidated by future mutators. + VisitDirectDepsBlueprint(visit func(blueprint.Module)) + + // VisitDirectDeps calls visit for each direct dependency. If there are multiple + // direct dependencies on the same module visit will be called multiple times on that module + // and OtherModuleDependencyTag will return a different tag for each. It raises an error if any of the + // dependencies are not an android.Module. + // + // The Module passed to the visit function should not be retained outside of the visit + // function, it may be invalidated by future mutators. + VisitDirectDeps(visit func(Module)) + + VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) + + // VisitDirectDepsIf calls pred for each direct dependency, and if pred returns true calls visit. If there are + // multiple direct dependencies on the same module pred and visit will be called multiple times on that module and + // OtherModuleDependencyTag will return a different tag for each. It skips any + // dependencies that are not an android.Module. + // + // The Module passed to the visit function should not be retained outside of the visit function, it may be + // invalidated by future mutators. + VisitDirectDepsIf(pred func(Module) bool, visit func(Module)) + // Deprecated: use WalkDeps instead to support multiple dependency tags on the same module + VisitDepsDepthFirst(visit func(Module)) + // Deprecated: use WalkDeps instead to support multiple dependency tags on the same module + VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) + + // WalkDeps calls visit for each transitive dependency, traversing the dependency tree in top down order. visit may + // be called multiple times for the same (child, parent) pair if there are multiple direct dependencies between the + // child and parent with different tags. OtherModuleDependencyTag will return the tag for the currently visited + // (child, parent) pair. If visit returns false WalkDeps will not continue recursing down to child. It skips + // any dependencies that are not an android.Module. + // + // The Modules passed to the visit function should not be retained outside of the visit function, they may be + // invalidated by future mutators. + WalkDeps(visit func(child, parent Module) bool) + + // WalkDepsBlueprint calls visit for each transitive dependency, traversing the dependency + // tree in top down order. visit may be called multiple times for the same (child, parent) + // pair if there are multiple direct dependencies between the child and parent with different + // tags. OtherModuleDependencyTag will return the tag for the currently visited + // (child, parent) pair. If visit returns false WalkDeps will not continue recursing down + // to child. + // + // The Modules passed to the visit function should not be retained outside of the visit function, they may be + // invalidated by future mutators. + WalkDepsBlueprint(visit func(blueprint.Module, blueprint.Module) bool) + + // GetWalkPath is supposed to be called in visit function passed in WalkDeps() + // and returns a top-down dependency path from a start module to current child module. + GetWalkPath() []Module + + // PrimaryModule returns the first variant of the current module. Variants of a module are always visited in + // order by mutators and GenerateBuildActions, so the data created by the current mutator can be read from the + // Module returned by PrimaryModule without data races. This can be used to perform singleton actions that are + // only done once for all variants of a module. + PrimaryModule() Module + + // FinalModule returns the last variant of the current module. Variants of a module are always visited in + // order by mutators and GenerateBuildActions, so the data created by the current mutator can be read from all + // variants using VisitAllModuleVariants if the current module == FinalModule(). This can be used to perform + // singleton actions that are only done once for all variants of a module. + FinalModule() Module + + // VisitAllModuleVariants calls visit for each variant of the current module. Variants of a module are always + // visited in order by mutators and GenerateBuildActions, so the data created by the current mutator can be read + // from all variants if the current module == FinalModule(). Otherwise, care must be taken to not access any + // data modified by the current mutator. + VisitAllModuleVariants(visit func(Module)) + + // GetTagPath is supposed to be called in visit function passed in WalkDeps() + // and returns a top-down dependency tags path from a start module to current child module. + // It has one less entry than GetWalkPath() as it contains the dependency tags that + // exist between each adjacent pair of modules in the GetWalkPath(). + // GetTagPath()[i] is the tag between GetWalkPath()[i] and GetWalkPath()[i+1] + GetTagPath() []blueprint.DependencyTag + + // GetPathString is supposed to be called in visit function passed in WalkDeps() + // and returns a multi-line string showing the modules and dependency tags + // among them along the top-down dependency path from a start module to current child module. + // skipFirst when set to true, the output doesn't include the start module, + // which is already printed when this function is used along with ModuleErrorf(). + GetPathString(skipFirst bool) string + + AddMissingDependencies(missingDeps []string) + + // getMissingDependencies returns the list of missing dependencies. + // Calling this function prevents adding new dependencies. + getMissingDependencies() []string + + // AddUnconvertedBp2buildDep stores module name of a direct dependency that was not converted via bp2build + AddUnconvertedBp2buildDep(dep string) + + // AddMissingBp2buildDep stores the module name of a direct dependency that was not found. + AddMissingBp2buildDep(dep string) + + Target() Target + TargetPrimary() bool + + // The additional arch specific targets (e.g. 32/64 bit) that this module variant is + // responsible for creating. + MultiTargets() []Target + Arch() Arch + Os() OsType + Host() bool + Device() bool + Darwin() bool + Windows() bool + PrimaryArch() bool +} + +type baseModuleContext struct { + bp blueprint.BaseModuleContext + earlyModuleContext + os OsType + target Target + multiTargets []Target + targetPrimary bool + + walkPath []Module + tagPath []blueprint.DependencyTag + + strictVisitDeps bool // If true, enforce that all dependencies are enabled + + bazelConversionMode bool +} + +func (b *baseModuleContext) isBazelConversionMode() bool { + return b.bazelConversionMode +} +func (b *baseModuleContext) OtherModuleName(m blueprint.Module) string { + return b.bp.OtherModuleName(m) +} +func (b *baseModuleContext) OtherModuleDir(m blueprint.Module) string { return b.bp.OtherModuleDir(m) } +func (b *baseModuleContext) OtherModuleErrorf(m blueprint.Module, fmt string, args ...interface{}) { + b.bp.OtherModuleErrorf(m, fmt, args...) +} +func (b *baseModuleContext) OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag { + return b.bp.OtherModuleDependencyTag(m) +} +func (b *baseModuleContext) OtherModuleExists(name string) bool { return b.bp.OtherModuleExists(name) } +func (b *baseModuleContext) OtherModuleDependencyVariantExists(variations []blueprint.Variation, name string) bool { + return b.bp.OtherModuleDependencyVariantExists(variations, name) +} +func (b *baseModuleContext) OtherModuleFarDependencyVariantExists(variations []blueprint.Variation, name string) bool { + return b.bp.OtherModuleFarDependencyVariantExists(variations, name) +} +func (b *baseModuleContext) OtherModuleReverseDependencyVariantExists(name string) bool { + return b.bp.OtherModuleReverseDependencyVariantExists(name) +} +func (b *baseModuleContext) OtherModuleType(m blueprint.Module) string { + return b.bp.OtherModuleType(m) +} +func (b *baseModuleContext) OtherModuleProvider(m blueprint.Module, provider blueprint.ProviderKey) interface{} { + return b.bp.OtherModuleProvider(m, provider) +} +func (b *baseModuleContext) OtherModuleHasProvider(m blueprint.Module, provider blueprint.ProviderKey) bool { + return b.bp.OtherModuleHasProvider(m, provider) +} +func (b *baseModuleContext) Provider(provider blueprint.ProviderKey) interface{} { + return b.bp.Provider(provider) +} +func (b *baseModuleContext) HasProvider(provider blueprint.ProviderKey) bool { + return b.bp.HasProvider(provider) +} +func (b *baseModuleContext) SetProvider(provider blueprint.ProviderKey, value interface{}) { + b.bp.SetProvider(provider, value) +} + +func (b *baseModuleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module { + return b.bp.GetDirectDepWithTag(name, tag) +} + +func (b *baseModuleContext) blueprintBaseModuleContext() blueprint.BaseModuleContext { + return b.bp +} + +// AddUnconvertedBp2buildDep stores module name of a dependency that was not converted to Bazel. +func (b *baseModuleContext) AddUnconvertedBp2buildDep(dep string) { + unconvertedDeps := &b.Module().base().commonProperties.BazelConversionStatus.UnconvertedDeps + *unconvertedDeps = append(*unconvertedDeps, dep) +} + +// AddMissingBp2buildDep stores module name of a dependency that was not found in a Android.bp file. +func (b *baseModuleContext) AddMissingBp2buildDep(dep string) { + missingDeps := &b.Module().base().commonProperties.BazelConversionStatus.MissingDeps + *missingDeps = append(*missingDeps, dep) +} + +func (b *baseModuleContext) AddMissingDependencies(deps []string) { + if deps != nil { + missingDeps := &b.Module().base().commonProperties.MissingDeps + *missingDeps = append(*missingDeps, deps...) + *missingDeps = FirstUniqueStrings(*missingDeps) + } +} + +func (b *baseModuleContext) checkedMissingDeps() bool { + return b.Module().base().commonProperties.CheckedMissingDeps +} + +func (b *baseModuleContext) getMissingDependencies() []string { + checked := &b.Module().base().commonProperties.CheckedMissingDeps + *checked = true + var missingDeps []string + missingDeps = append(missingDeps, b.Module().base().commonProperties.MissingDeps...) + missingDeps = append(missingDeps, b.bp.EarlyGetMissingDependencies()...) + missingDeps = FirstUniqueStrings(missingDeps) + return missingDeps +} + +type AllowDisabledModuleDependency interface { + blueprint.DependencyTag + AllowDisabledModuleDependency(target Module) bool +} + +func (b *baseModuleContext) validateAndroidModule(module blueprint.Module, tag blueprint.DependencyTag, strict bool) Module { + aModule, _ := module.(Module) + + if !strict { + return aModule + } + + if aModule == nil { + b.ModuleErrorf("module %q (%#v) not an android module", b.OtherModuleName(module), tag) + return nil + } + + if !aModule.Enabled() { + if t, ok := tag.(AllowDisabledModuleDependency); !ok || !t.AllowDisabledModuleDependency(aModule) { + if b.Config().AllowMissingDependencies() { + b.AddMissingDependencies([]string{b.OtherModuleName(aModule)}) + } else { + b.ModuleErrorf("depends on disabled module %q", b.OtherModuleName(aModule)) + } + } + return nil + } + return aModule +} + +type dep struct { + mod blueprint.Module + tag blueprint.DependencyTag +} + +func (b *baseModuleContext) getDirectDepsInternal(name string, tag blueprint.DependencyTag) []dep { + var deps []dep + b.VisitDirectDepsBlueprint(func(module blueprint.Module) { + if aModule, _ := module.(Module); aModule != nil { + if aModule.base().BaseModuleName() == name { + returnedTag := b.bp.OtherModuleDependencyTag(aModule) + if tag == nil || returnedTag == tag { + deps = append(deps, dep{aModule, returnedTag}) + } + } + } else if b.bp.OtherModuleName(module) == name { + returnedTag := b.bp.OtherModuleDependencyTag(module) + if tag == nil || returnedTag == tag { + deps = append(deps, dep{module, returnedTag}) + } + } + }) + return deps +} + +func (b *baseModuleContext) getDirectDepInternal(name string, tag blueprint.DependencyTag) (blueprint.Module, blueprint.DependencyTag) { + deps := b.getDirectDepsInternal(name, tag) + if len(deps) == 1 { + return deps[0].mod, deps[0].tag + } else if len(deps) >= 2 { + panic(fmt.Errorf("Multiple dependencies having same BaseModuleName() %q found from %q", + name, b.ModuleName())) + } else { + return nil, nil + } +} + +func (b *baseModuleContext) getDirectDepFirstTag(name string) (blueprint.Module, blueprint.DependencyTag) { + foundDeps := b.getDirectDepsInternal(name, nil) + deps := map[blueprint.Module]bool{} + for _, dep := range foundDeps { + deps[dep.mod] = true + } + if len(deps) == 1 { + return foundDeps[0].mod, foundDeps[0].tag + } else if len(deps) >= 2 { + // this could happen if two dependencies have the same name in different namespaces + // TODO(b/186554727): this should not occur if namespaces are handled within + // getDirectDepsInternal. + panic(fmt.Errorf("Multiple dependencies having same BaseModuleName() %q found from %q", + name, b.ModuleName())) + } else { + return nil, nil + } +} + +func (b *baseModuleContext) GetDirectDepsWithTag(tag blueprint.DependencyTag) []Module { + var deps []Module + b.VisitDirectDepsBlueprint(func(module blueprint.Module) { + if aModule, _ := module.(Module); aModule != nil { + if b.bp.OtherModuleDependencyTag(aModule) == tag { + deps = append(deps, aModule) + } + } + }) + return deps +} + +// GetDirectDep returns the Module and DependencyTag for the direct dependency with the specified +// name, or nil if none exists. If there are multiple dependencies on the same module it returns the +// first DependencyTag. +func (b *baseModuleContext) GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) { + return b.getDirectDepFirstTag(name) +} + +func (b *baseModuleContext) ModuleFromName(name string) (blueprint.Module, bool) { + if !b.isBazelConversionMode() { + panic("cannot call ModuleFromName if not in bazel conversion mode") + } + var m blueprint.Module + var ok bool + if moduleName, _ := SrcIsModuleWithTag(name); moduleName != "" { + m, ok = b.bp.ModuleFromName(moduleName) + } else { + m, ok = b.bp.ModuleFromName(name) + } + if !ok { + return m, ok + } + // If this module is not preferred, tried to get the prebuilt version instead + if a, aOk := m.(Module); aOk && !IsModulePrebuilt(a) && !IsModulePreferred(a) { + return b.ModuleFromName("prebuilt_" + name) + } + return m, ok +} + +func (b *baseModuleContext) VisitDirectDepsBlueprint(visit func(blueprint.Module)) { + b.bp.VisitDirectDeps(visit) +} + +func (b *baseModuleContext) VisitDirectDeps(visit func(Module)) { + b.bp.VisitDirectDeps(func(module blueprint.Module) { + if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { + visit(aModule) + } + }) +} + +func (b *baseModuleContext) VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) { + b.bp.VisitDirectDeps(func(module blueprint.Module) { + if b.bp.OtherModuleDependencyTag(module) == tag { + if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { + visit(aModule) + } + } + }) +} + +func (b *baseModuleContext) VisitDirectDepsIf(pred func(Module) bool, visit func(Module)) { + b.bp.VisitDirectDepsIf( + // pred + func(module blueprint.Module) bool { + if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { + return pred(aModule) + } else { + return false + } + }, + // visit + func(module blueprint.Module) { + visit(module.(Module)) + }) +} + +func (b *baseModuleContext) VisitDepsDepthFirst(visit func(Module)) { + b.bp.VisitDepsDepthFirst(func(module blueprint.Module) { + if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { + visit(aModule) + } + }) +} + +func (b *baseModuleContext) VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) { + b.bp.VisitDepsDepthFirstIf( + // pred + func(module blueprint.Module) bool { + if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { + return pred(aModule) + } else { + return false + } + }, + // visit + func(module blueprint.Module) { + visit(module.(Module)) + }) +} + +func (b *baseModuleContext) WalkDepsBlueprint(visit func(blueprint.Module, blueprint.Module) bool) { + b.bp.WalkDeps(visit) +} + +func (b *baseModuleContext) WalkDeps(visit func(Module, Module) bool) { + b.walkPath = []Module{b.Module()} + b.tagPath = []blueprint.DependencyTag{} + b.bp.WalkDeps(func(child, parent blueprint.Module) bool { + childAndroidModule, _ := child.(Module) + parentAndroidModule, _ := parent.(Module) + if childAndroidModule != nil && parentAndroidModule != nil { + // record walkPath before visit + for b.walkPath[len(b.walkPath)-1] != parentAndroidModule { + b.walkPath = b.walkPath[0 : len(b.walkPath)-1] + b.tagPath = b.tagPath[0 : len(b.tagPath)-1] + } + b.walkPath = append(b.walkPath, childAndroidModule) + b.tagPath = append(b.tagPath, b.OtherModuleDependencyTag(childAndroidModule)) + return visit(childAndroidModule, parentAndroidModule) + } else { + return false + } + }) +} + +func (b *baseModuleContext) GetWalkPath() []Module { + return b.walkPath +} + +func (b *baseModuleContext) GetTagPath() []blueprint.DependencyTag { + return b.tagPath +} + +func (b *baseModuleContext) VisitAllModuleVariants(visit func(Module)) { + b.bp.VisitAllModuleVariants(func(module blueprint.Module) { + visit(module.(Module)) + }) +} + +func (b *baseModuleContext) PrimaryModule() Module { + return b.bp.PrimaryModule().(Module) +} + +func (b *baseModuleContext) FinalModule() Module { + return b.bp.FinalModule().(Module) +} + +// IsMetaDependencyTag returns true for cross-cutting metadata dependencies. +func IsMetaDependencyTag(tag blueprint.DependencyTag) bool { + if tag == licenseKindTag { + return true + } else if tag == licensesTag { + return true + } else if tag == acDepTag { + return true + } + return false +} + +// A regexp for removing boilerplate from BaseDependencyTag from the string representation of +// a dependency tag. +var tagCleaner = regexp.MustCompile(`\QBaseDependencyTag:{}\E(, )?`) + +// PrettyPrintTag returns string representation of the tag, but prefers +// custom String() method if available. +func PrettyPrintTag(tag blueprint.DependencyTag) string { + // Use tag's custom String() method if available. + if stringer, ok := tag.(fmt.Stringer); ok { + return stringer.String() + } + + // Otherwise, get a default string representation of the tag's struct. + tagString := fmt.Sprintf("%T: %+v", tag, tag) + + // Remove the boilerplate from BaseDependencyTag as it adds no value. + tagString = tagCleaner.ReplaceAllString(tagString, "") + return tagString +} + +func (b *baseModuleContext) GetPathString(skipFirst bool) string { + sb := strings.Builder{} + tagPath := b.GetTagPath() + walkPath := b.GetWalkPath() + if !skipFirst { + sb.WriteString(walkPath[0].String()) + } + for i, m := range walkPath[1:] { + sb.WriteString("\n") + sb.WriteString(fmt.Sprintf(" via tag %s\n", PrettyPrintTag(tagPath[i]))) + sb.WriteString(fmt.Sprintf(" -> %s", m.String())) + } + return sb.String() +} + +func (b *baseModuleContext) Target() Target { + return b.target +} + +func (b *baseModuleContext) TargetPrimary() bool { + return b.targetPrimary +} + +func (b *baseModuleContext) MultiTargets() []Target { + return b.multiTargets +} + +func (b *baseModuleContext) Arch() Arch { + return b.target.Arch +} + +func (b *baseModuleContext) Os() OsType { + return b.os +} + +func (b *baseModuleContext) Host() bool { + return b.os.Class == Host +} + +func (b *baseModuleContext) Device() bool { + return b.os.Class == Device +} + +func (b *baseModuleContext) Darwin() bool { + return b.os == Darwin +} + +func (b *baseModuleContext) Windows() bool { + return b.os == Windows +} + +func (b *baseModuleContext) PrimaryArch() bool { + if len(b.config.Targets[b.target.Os]) <= 1 { + return true + } + return b.target.Arch.ArchType == b.config.Targets[b.target.Os][0].Arch.ArchType +} diff --git a/android/config.go b/android/config.go index 4c31bb052..a69adc354 100644 --- a/android/config.go +++ b/android/config.go @@ -99,7 +99,7 @@ type CmdArgs struct { UseBazelProxy bool - BuildFromTextStub bool + BuildFromSourceStub bool EnsureAllowlistIntegrity bool } @@ -344,9 +344,9 @@ type config struct { // unix sockets, instead of spawning Bazel as a subprocess. UseBazelProxy bool - // If buildFromTextStub is true then the Java API stubs are - // built from the signature text files, not the source Java files. - buildFromTextStub bool + // If buildFromSourceStub is true then the Java API stubs are + // built from the source Java files, not the signature text files. + buildFromSourceStub bool // If ensureAllowlistIntegrity is true, then the presence of any allowlisted // modules that aren't mixed-built for at least one variant will cause a build @@ -563,15 +563,13 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) MultitreeBuild: cmdArgs.MultitreeBuild, UseBazelProxy: cmdArgs.UseBazelProxy, - buildFromTextStub: cmdArgs.BuildFromTextStub, + buildFromSourceStub: cmdArgs.BuildFromSourceStub, } config.deviceConfig = &deviceConfig{ config: config, } - config.productVariables.Build_from_text_stub = boolPtr(config.BuildFromTextStub()) - // Soundness check of the build and source directories. This won't catch strange // configurations with symlinks, but at least checks the obvious case. absBuildDir, err := filepath.Abs(cmdArgs.SoongOutDir) @@ -694,6 +692,7 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) "framework-media": {}, "framework-mediaprovider": {}, "framework-ondevicepersonalization": {}, + "framework-pdf": {}, "framework-permission": {}, "framework-permission-s": {}, "framework-scheduling": {}, @@ -707,6 +706,8 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) "i18n.module.public.api": {}, } + config.productVariables.Build_from_text_stub = boolPtr(config.BuildFromTextStub()) + return Config{config}, err } @@ -2075,15 +2076,21 @@ func (c *config) JavaCoverageEnabled() bool { return c.IsEnvTrue("EMMA_INSTRUMENT") || c.IsEnvTrue("EMMA_INSTRUMENT_STATIC") || c.IsEnvTrue("EMMA_INSTRUMENT_FRAMEWORK") } +func (c *deviceConfig) BuildFromSourceStub() bool { + return Bool(c.config.productVariables.BuildFromSourceStub) +} + func (c *config) BuildFromTextStub() bool { // TODO: b/302320354 - Remove the coverage build specific logic once the // robust solution for handling native properties in from-text stub build // is implemented. - return c.buildFromTextStub && !c.JavaCoverageEnabled() + return !c.buildFromSourceStub && + !c.JavaCoverageEnabled() && + !c.deviceConfig.BuildFromSourceStub() } func (c *config) SetBuildFromTextStub(b bool) { - c.buildFromTextStub = b + c.buildFromSourceStub = !b c.productVariables.Build_from_text_stub = boolPtr(b) } @@ -2129,3 +2136,41 @@ func (c *config) GetBuildFlag(name string) (string, bool) { val, ok := c.productVariables.BuildFlags[name] return val, ok } + +var ( + mainlineApexContributionBuildFlags = []string{ + "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES", + "RELEASE_APEX_CONTRIBUTIONS_APPSEARCH", + "RELEASE_APEX_CONTRIBUTIONS_ART", + "RELEASE_APEX_CONTRIBUTIONS_BLUETOOTH", + "RELEASE_APEX_CONTRIBUTIONS_CONFIGINFRASTRUCTURE", + "RELEASE_APEX_CONTRIBUTIONS_CONNECTIVITY", + "RELEASE_APEX_CONTRIBUTIONS_CONSCRYPT", + "RELEASE_APEX_CONTRIBUTIONS_CRASHRECOVERY", + "RELEASE_APEX_CONTRIBUTIONS_DEVICELOCK", + "RELEASE_APEX_CONTRIBUTIONS_HEALTHFITNESS", + "RELEASE_APEX_CONTRIBUTIONS_IPSEC", + "RELEASE_APEX_CONTRIBUTIONS_MEDIA", + "RELEASE_APEX_CONTRIBUTIONS_MEDIAPROVIDER", + "RELEASE_APEX_CONTRIBUTIONS_ONDEVICEPERSONALIZATION", + "RELEASE_APEX_CONTRIBUTIONS_PERMISSION", + "RELEASE_APEX_CONTRIBUTIONS_REMOTEKEYPROVISIONING", + "RELEASE_APEX_CONTRIBUTIONS_SCHEDULING", + "RELEASE_APEX_CONTRIBUTIONS_SDKEXTENSIONS", + "RELEASE_APEX_CONTRIBUTIONS_STATSD", + "RELEASE_APEX_CONTRIBUTIONS_UWB", + "RELEASE_APEX_CONTRIBUTIONS_WIFI", + } +) + +// Returns the list of _selected_ apex_contributions +// Each mainline module will have one entry in the list +func (c *config) AllApexContributions() []string { + ret := []string{} + for _, f := range mainlineApexContributionBuildFlags { + if val, exists := c.GetBuildFlag(f); exists && val != "" { + ret = append(ret, val) + } + } + return ret +} diff --git a/android/early_module_context.go b/android/early_module_context.go new file mode 100644 index 000000000..8f7577358 --- /dev/null +++ b/android/early_module_context.go @@ -0,0 +1,169 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "github.com/google/blueprint" + "os" + "text/scanner" +) + +// EarlyModuleContext provides methods that can be called early, as soon as the properties have +// been parsed into the module and before any mutators have run. +type EarlyModuleContext interface { + // Module returns the current module as a Module. It should rarely be necessary, as the module already has a + // reference to itself. + Module() Module + + // ModuleName returns the name of the module. This is generally the value that was returned by Module.Name() when + // the module was created, but may have been modified by calls to BaseMutatorContext.Rename. + ModuleName() string + + // ModuleDir returns the path to the directory that contains the definition of the module. + ModuleDir() string + + // ModuleType returns the name of the module type that was used to create the module, as specified in + // RegisterModuleType. + ModuleType() string + + // BlueprintFile returns the name of the blueprint file that contains the definition of this + // module. + BlueprintsFile() string + + // ContainsProperty returns true if the specified property name was set in the module definition. + ContainsProperty(name string) bool + + // Errorf reports an error at the specified position of the module definition file. + Errorf(pos scanner.Position, fmt string, args ...interface{}) + + // ModuleErrorf reports an error at the line number of the module type in the module definition. + ModuleErrorf(fmt string, args ...interface{}) + + // PropertyErrorf reports an error at the line number of a property in the module definition. + PropertyErrorf(property, fmt string, args ...interface{}) + + // Failed returns true if any errors have been reported. In most cases the module can continue with generating + // build rules after an error, allowing it to report additional errors in a single run, but in cases where the error + // has prevented the module from creating necessary data it can return early when Failed returns true. + Failed() bool + + // AddNinjaFileDeps adds dependencies on the specified files to the rule that creates the ninja manifest. The + // primary builder will be rerun whenever the specified files are modified. + AddNinjaFileDeps(deps ...string) + + DeviceSpecific() bool + SocSpecific() bool + ProductSpecific() bool + SystemExtSpecific() bool + Platform() bool + + Config() Config + DeviceConfig() DeviceConfig + + // Deprecated: use Config() + AConfig() Config + + // GlobWithDeps returns a list of files that match the specified pattern but do not match any + // of the patterns in excludes. It also adds efficient dependencies to rerun the primary + // builder whenever a file matching the pattern as added or removed, without rerunning if a + // file that does not match the pattern is added to a searched directory. + GlobWithDeps(pattern string, excludes []string) ([]string, error) + + Glob(globPattern string, excludes []string) Paths + GlobFiles(globPattern string, excludes []string) Paths + IsSymlink(path Path) bool + Readlink(path Path) string + + // Namespace returns the Namespace object provided by the NameInterface set by Context.SetNameInterface, or the + // default SimpleNameInterface if Context.SetNameInterface was not called. + Namespace() *Namespace +} + +// Deprecated: use EarlyModuleContext instead +type BaseContext interface { + EarlyModuleContext +} + +type earlyModuleContext struct { + blueprint.EarlyModuleContext + + kind moduleKind + config Config +} + +func (e *earlyModuleContext) Glob(globPattern string, excludes []string) Paths { + return Glob(e, globPattern, excludes) +} + +func (e *earlyModuleContext) GlobFiles(globPattern string, excludes []string) Paths { + return GlobFiles(e, globPattern, excludes) +} + +func (e *earlyModuleContext) IsSymlink(path Path) bool { + fileInfo, err := e.config.fs.Lstat(path.String()) + if err != nil { + e.ModuleErrorf("os.Lstat(%q) failed: %s", path.String(), err) + } + return fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink +} + +func (e *earlyModuleContext) Readlink(path Path) string { + dest, err := e.config.fs.Readlink(path.String()) + if err != nil { + e.ModuleErrorf("os.Readlink(%q) failed: %s", path.String(), err) + } + return dest +} + +func (e *earlyModuleContext) Module() Module { + module, _ := e.EarlyModuleContext.Module().(Module) + return module +} + +func (e *earlyModuleContext) Config() Config { + return e.EarlyModuleContext.Config().(Config) +} + +func (e *earlyModuleContext) AConfig() Config { + return e.config +} + +func (e *earlyModuleContext) DeviceConfig() DeviceConfig { + return DeviceConfig{e.config.deviceConfig} +} + +func (e *earlyModuleContext) Platform() bool { + return e.kind == platformModule +} + +func (e *earlyModuleContext) DeviceSpecific() bool { + return e.kind == deviceSpecificModule +} + +func (e *earlyModuleContext) SocSpecific() bool { + return e.kind == socSpecificModule +} + +func (e *earlyModuleContext) ProductSpecific() bool { + return e.kind == productSpecificModule +} + +func (e *earlyModuleContext) SystemExtSpecific() bool { + return e.kind == systemExtSpecificModule +} + +func (e *earlyModuleContext) Namespace() *Namespace { + return e.EarlyModuleContext.Namespace().(*Namespace) +} diff --git a/android/module.go b/android/module.go index 250161f46..af69a1b1c 100644 --- a/android/module.go +++ b/android/module.go @@ -15,22 +15,17 @@ package android import ( + "android/soong/bazel" + "android/soong/ui/metrics/bp2build_metrics_proto" "crypto/md5" "encoding/hex" "encoding/json" "fmt" "net/url" - "os" - "path" "path/filepath" "reflect" - "regexp" "sort" "strings" - "text/scanner" - - "android/soong/bazel" - "android/soong/ui/metrics/bp2build_metrics_proto" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -41,469 +36,6 @@ var ( DeviceStaticLibrary = "static_library" ) -// BuildParameters describes the set of potential parameters to build a Ninja rule. -// In general, these correspond to a Ninja concept. -type BuildParams struct { - // A Ninja Rule that will be written to the Ninja file. This allows factoring out common code - // among multiple modules to reduce repetition in the Ninja file of action requirements. A rule - // can contain variables that should be provided in Args. - Rule blueprint.Rule - // Deps represents the depfile format. When using RuleBuilder, this defaults to GCC when depfiles - // are used. - Deps blueprint.Deps - // Depfile is a writeable path that allows correct incremental builds when the inputs have not - // been fully specified by the Ninja rule. Ninja supports a subset of the Makefile depfile syntax. - Depfile WritablePath - // A description of the build action. - Description string - // Output is an output file of the action. When using this field, references to $out in the Ninja - // command will refer to this file. - Output WritablePath - // Outputs is a slice of output file of the action. When using this field, references to $out in - // the Ninja command will refer to these files. - Outputs WritablePaths - // SymlinkOutput is an output file specifically that is a symlink. - SymlinkOutput WritablePath - // SymlinkOutputs is a slice of output files specifically that is a symlink. - SymlinkOutputs WritablePaths - // ImplicitOutput is an output file generated by the action. Note: references to `$out` in the - // Ninja command will NOT include references to this file. - ImplicitOutput WritablePath - // ImplicitOutputs is a slice of output files generated by the action. Note: references to `$out` - // in the Ninja command will NOT include references to these files. - ImplicitOutputs WritablePaths - // Input is an input file to the Ninja action. When using this field, references to $in in the - // Ninja command will refer to this file. - Input Path - // Inputs is a slice of input files to the Ninja action. When using this field, references to $in - // in the Ninja command will refer to these files. - Inputs Paths - // Implicit is an input file to the Ninja action. Note: references to `$in` in the Ninja command - // will NOT include references to this file. - Implicit Path - // Implicits is a slice of input files to the Ninja action. Note: references to `$in` in the Ninja - // command will NOT include references to these files. - Implicits Paths - // OrderOnly are Ninja order-only inputs to the action. When these are out of date, the output is - // not rebuilt until they are built, but changes in order-only dependencies alone do not cause the - // output to be rebuilt. - OrderOnly Paths - // Validation is an output path for a validation action. Validation outputs imply lower - // non-blocking priority to building non-validation outputs. - Validation Path - // Validations is a slice of output path for a validation action. Validation outputs imply lower - // non-blocking priority to building non-validation outputs. - Validations Paths - // Whether to skip outputting a default target statement which will be built by Ninja when no - // targets are specified on Ninja's command line. - Default bool - // Args is a key value mapping for replacements of variables within the Rule - Args map[string]string -} - -type ModuleBuildParams BuildParams - -// EarlyModuleContext provides methods that can be called early, as soon as the properties have -// been parsed into the module and before any mutators have run. -type EarlyModuleContext interface { - // Module returns the current module as a Module. It should rarely be necessary, as the module already has a - // reference to itself. - Module() Module - - // ModuleName returns the name of the module. This is generally the value that was returned by Module.Name() when - // the module was created, but may have been modified by calls to BaseMutatorContext.Rename. - ModuleName() string - - // ModuleDir returns the path to the directory that contains the definition of the module. - ModuleDir() string - - // ModuleType returns the name of the module type that was used to create the module, as specified in - // RegisterModuleType. - ModuleType() string - - // BlueprintFile returns the name of the blueprint file that contains the definition of this - // module. - BlueprintsFile() string - - // ContainsProperty returns true if the specified property name was set in the module definition. - ContainsProperty(name string) bool - - // Errorf reports an error at the specified position of the module definition file. - Errorf(pos scanner.Position, fmt string, args ...interface{}) - - // ModuleErrorf reports an error at the line number of the module type in the module definition. - ModuleErrorf(fmt string, args ...interface{}) - - // PropertyErrorf reports an error at the line number of a property in the module definition. - PropertyErrorf(property, fmt string, args ...interface{}) - - // Failed returns true if any errors have been reported. In most cases the module can continue with generating - // build rules after an error, allowing it to report additional errors in a single run, but in cases where the error - // has prevented the module from creating necessary data it can return early when Failed returns true. - Failed() bool - - // AddNinjaFileDeps adds dependencies on the specified files to the rule that creates the ninja manifest. The - // primary builder will be rerun whenever the specified files are modified. - AddNinjaFileDeps(deps ...string) - - DeviceSpecific() bool - SocSpecific() bool - ProductSpecific() bool - SystemExtSpecific() bool - Platform() bool - - Config() Config - DeviceConfig() DeviceConfig - - // Deprecated: use Config() - AConfig() Config - - // GlobWithDeps returns a list of files that match the specified pattern but do not match any - // of the patterns in excludes. It also adds efficient dependencies to rerun the primary - // builder whenever a file matching the pattern as added or removed, without rerunning if a - // file that does not match the pattern is added to a searched directory. - GlobWithDeps(pattern string, excludes []string) ([]string, error) - - Glob(globPattern string, excludes []string) Paths - GlobFiles(globPattern string, excludes []string) Paths - IsSymlink(path Path) bool - Readlink(path Path) string - - // Namespace returns the Namespace object provided by the NameInterface set by Context.SetNameInterface, or the - // default SimpleNameInterface if Context.SetNameInterface was not called. - Namespace() *Namespace -} - -// BaseModuleContext is the same as blueprint.BaseModuleContext except that Config() returns -// a Config instead of an interface{}, and some methods have been wrapped to use an android.Module -// instead of a blueprint.Module, plus some extra methods that return Android-specific information -// about the current module. -type BaseModuleContext interface { - EarlyModuleContext - - blueprintBaseModuleContext() blueprint.BaseModuleContext - - // OtherModuleName returns the name of another Module. See BaseModuleContext.ModuleName for more information. - // It is intended for use inside the visit functions of Visit* and WalkDeps. - OtherModuleName(m blueprint.Module) string - - // OtherModuleDir returns the directory of another Module. See BaseModuleContext.ModuleDir for more information. - // It is intended for use inside the visit functions of Visit* and WalkDeps. - OtherModuleDir(m blueprint.Module) string - - // OtherModuleErrorf reports an error on another Module. See BaseModuleContext.ModuleErrorf for more information. - // It is intended for use inside the visit functions of Visit* and WalkDeps. - OtherModuleErrorf(m blueprint.Module, fmt string, args ...interface{}) - - // OtherModuleDependencyTag returns the dependency tag used to depend on a module, or nil if there is no dependency - // on the module. When called inside a Visit* method with current module being visited, and there are multiple - // dependencies on the module being visited, it returns the dependency tag used for the current dependency. - OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag - - // OtherModuleExists returns true if a module with the specified name exists, as determined by the NameInterface - // passed to Context.SetNameInterface, or SimpleNameInterface if it was not called. - OtherModuleExists(name string) bool - - // OtherModuleDependencyVariantExists returns true if a module with the - // specified name and variant exists. The variant must match the given - // variations. It must also match all the non-local variations of the current - // module. In other words, it checks for the module that AddVariationDependencies - // would add a dependency on with the same arguments. - OtherModuleDependencyVariantExists(variations []blueprint.Variation, name string) bool - - // OtherModuleFarDependencyVariantExists returns true if a module with the - // specified name and variant exists. The variant must match the given - // variations, but not the non-local variations of the current module. In - // other words, it checks for the module that AddFarVariationDependencies - // would add a dependency on with the same arguments. - OtherModuleFarDependencyVariantExists(variations []blueprint.Variation, name string) bool - - // OtherModuleReverseDependencyVariantExists returns true if a module with the - // specified name exists with the same variations as the current module. In - // other words, it checks for the module that AddReverseDependency would add a - // dependency on with the same argument. - OtherModuleReverseDependencyVariantExists(name string) bool - - // OtherModuleType returns the type of another Module. See BaseModuleContext.ModuleType for more information. - // It is intended for use inside the visit functions of Visit* and WalkDeps. - OtherModuleType(m blueprint.Module) string - - // OtherModuleProvider returns the value for a provider for the given module. If the value is - // not set it returns the zero value of the type of the provider, so the return value can always - // be type asserted to the type of the provider. The value returned may be a deep copy of the - // value originally passed to SetProvider. - OtherModuleProvider(m blueprint.Module, provider blueprint.ProviderKey) interface{} - - // OtherModuleHasProvider returns true if the provider for the given module has been set. - OtherModuleHasProvider(m blueprint.Module, provider blueprint.ProviderKey) bool - - // Provider returns the value for a provider for the current module. If the value is - // not set it returns the zero value of the type of the provider, so the return value can always - // be type asserted to the type of the provider. It panics if called before the appropriate - // mutator or GenerateBuildActions pass for the provider. The value returned may be a deep - // copy of the value originally passed to SetProvider. - Provider(provider blueprint.ProviderKey) interface{} - - // HasProvider returns true if the provider for the current module has been set. - HasProvider(provider blueprint.ProviderKey) bool - - // SetProvider sets the value for a provider for the current module. It panics if not called - // during the appropriate mutator or GenerateBuildActions pass for the provider, if the value - // is not of the appropriate type, or if the value has already been set. The value should not - // be modified after being passed to SetProvider. - SetProvider(provider blueprint.ProviderKey, value interface{}) - - GetDirectDepsWithTag(tag blueprint.DependencyTag) []Module - - // GetDirectDepWithTag returns the Module the direct dependency with the specified name, or nil if - // none exists. It panics if the dependency does not have the specified tag. It skips any - // dependencies that are not an android.Module. - GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module - - // GetDirectDep returns the Module and DependencyTag for the direct dependency with the specified - // name, or nil if none exists. If there are multiple dependencies on the same module it returns - // the first DependencyTag. - GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) - - ModuleFromName(name string) (blueprint.Module, bool) - - // VisitDirectDepsBlueprint calls visit for each direct dependency. If there are multiple - // direct dependencies on the same module visit will be called multiple times on that module - // and OtherModuleDependencyTag will return a different tag for each. - // - // The Module passed to the visit function should not be retained outside of the visit - // function, it may be invalidated by future mutators. - VisitDirectDepsBlueprint(visit func(blueprint.Module)) - - // VisitDirectDeps calls visit for each direct dependency. If there are multiple - // direct dependencies on the same module visit will be called multiple times on that module - // and OtherModuleDependencyTag will return a different tag for each. It raises an error if any of the - // dependencies are not an android.Module. - // - // The Module passed to the visit function should not be retained outside of the visit - // function, it may be invalidated by future mutators. - VisitDirectDeps(visit func(Module)) - - VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) - - // VisitDirectDepsIf calls pred for each direct dependency, and if pred returns true calls visit. If there are - // multiple direct dependencies on the same module pred and visit will be called multiple times on that module and - // OtherModuleDependencyTag will return a different tag for each. It skips any - // dependencies that are not an android.Module. - // - // The Module passed to the visit function should not be retained outside of the visit function, it may be - // invalidated by future mutators. - VisitDirectDepsIf(pred func(Module) bool, visit func(Module)) - // Deprecated: use WalkDeps instead to support multiple dependency tags on the same module - VisitDepsDepthFirst(visit func(Module)) - // Deprecated: use WalkDeps instead to support multiple dependency tags on the same module - VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) - - // WalkDeps calls visit for each transitive dependency, traversing the dependency tree in top down order. visit may - // be called multiple times for the same (child, parent) pair if there are multiple direct dependencies between the - // child and parent with different tags. OtherModuleDependencyTag will return the tag for the currently visited - // (child, parent) pair. If visit returns false WalkDeps will not continue recursing down to child. It skips - // any dependencies that are not an android.Module. - // - // The Modules passed to the visit function should not be retained outside of the visit function, they may be - // invalidated by future mutators. - WalkDeps(visit func(child, parent Module) bool) - - // WalkDepsBlueprint calls visit for each transitive dependency, traversing the dependency - // tree in top down order. visit may be called multiple times for the same (child, parent) - // pair if there are multiple direct dependencies between the child and parent with different - // tags. OtherModuleDependencyTag will return the tag for the currently visited - // (child, parent) pair. If visit returns false WalkDeps will not continue recursing down - // to child. - // - // The Modules passed to the visit function should not be retained outside of the visit function, they may be - // invalidated by future mutators. - WalkDepsBlueprint(visit func(blueprint.Module, blueprint.Module) bool) - - // GetWalkPath is supposed to be called in visit function passed in WalkDeps() - // and returns a top-down dependency path from a start module to current child module. - GetWalkPath() []Module - - // PrimaryModule returns the first variant of the current module. Variants of a module are always visited in - // order by mutators and GenerateBuildActions, so the data created by the current mutator can be read from the - // Module returned by PrimaryModule without data races. This can be used to perform singleton actions that are - // only done once for all variants of a module. - PrimaryModule() Module - - // FinalModule returns the last variant of the current module. Variants of a module are always visited in - // order by mutators and GenerateBuildActions, so the data created by the current mutator can be read from all - // variants using VisitAllModuleVariants if the current module == FinalModule(). This can be used to perform - // singleton actions that are only done once for all variants of a module. - FinalModule() Module - - // VisitAllModuleVariants calls visit for each variant of the current module. Variants of a module are always - // visited in order by mutators and GenerateBuildActions, so the data created by the current mutator can be read - // from all variants if the current module == FinalModule(). Otherwise, care must be taken to not access any - // data modified by the current mutator. - VisitAllModuleVariants(visit func(Module)) - - // GetTagPath is supposed to be called in visit function passed in WalkDeps() - // and returns a top-down dependency tags path from a start module to current child module. - // It has one less entry than GetWalkPath() as it contains the dependency tags that - // exist between each adjacent pair of modules in the GetWalkPath(). - // GetTagPath()[i] is the tag between GetWalkPath()[i] and GetWalkPath()[i+1] - GetTagPath() []blueprint.DependencyTag - - // GetPathString is supposed to be called in visit function passed in WalkDeps() - // and returns a multi-line string showing the modules and dependency tags - // among them along the top-down dependency path from a start module to current child module. - // skipFirst when set to true, the output doesn't include the start module, - // which is already printed when this function is used along with ModuleErrorf(). - GetPathString(skipFirst bool) string - - AddMissingDependencies(missingDeps []string) - - // getMissingDependencies returns the list of missing dependencies. - // Calling this function prevents adding new dependencies. - getMissingDependencies() []string - - // AddUnconvertedBp2buildDep stores module name of a direct dependency that was not converted via bp2build - AddUnconvertedBp2buildDep(dep string) - - // AddMissingBp2buildDep stores the module name of a direct dependency that was not found. - AddMissingBp2buildDep(dep string) - - Target() Target - TargetPrimary() bool - - // The additional arch specific targets (e.g. 32/64 bit) that this module variant is - // responsible for creating. - MultiTargets() []Target - Arch() Arch - Os() OsType - Host() bool - Device() bool - Darwin() bool - Windows() bool - PrimaryArch() bool -} - -// Deprecated: use EarlyModuleContext instead -type BaseContext interface { - EarlyModuleContext -} - -type ModuleContext interface { - BaseModuleContext - - blueprintModuleContext() blueprint.ModuleContext - - // Deprecated: use ModuleContext.Build instead. - ModuleBuild(pctx PackageContext, params ModuleBuildParams) - - // Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must - // be tagged with `android:"path" to support automatic source module dependency resolution. - // - // Deprecated: use PathsForModuleSrc or PathsForModuleSrcExcludes instead. - ExpandSources(srcFiles, excludes []string) Paths - - // Returns a single path expanded from globs and modules referenced using ":module" syntax. The property must - // be tagged with `android:"path" to support automatic source module dependency resolution. - // - // Deprecated: use PathForModuleSrc instead. - ExpandSource(srcFile, prop string) Path - - ExpandOptionalSource(srcFile *string, prop string) OptionalPath - - // InstallExecutable creates a rule to copy srcPath to name in the installPath directory, - // with the given additional dependencies. The file is marked executable after copying. - // - // The installed file will be returned by FilesToInstall(), and the PackagingSpec for the - // installed file will be returned by PackagingSpecs() on this module or by - // TransitivePackagingSpecs() on modules that depend on this module through dependency tags - // for which IsInstallDepNeeded returns true. - InstallExecutable(installPath InstallPath, name string, srcPath Path, deps ...Path) InstallPath - - // InstallFile creates a rule to copy srcPath to name in the installPath directory, - // with the given additional dependencies. - // - // The installed file will be returned by FilesToInstall(), and the PackagingSpec for the - // installed file will be returned by PackagingSpecs() on this module or by - // TransitivePackagingSpecs() on modules that depend on this module through dependency tags - // for which IsInstallDepNeeded returns true. - InstallFile(installPath InstallPath, name string, srcPath Path, deps ...Path) InstallPath - - // InstallFileWithExtraFilesZip creates a rule to copy srcPath to name in the installPath - // directory, and also unzip a zip file containing extra files to install into the same - // directory. - // - // The installed file will be returned by FilesToInstall(), and the PackagingSpec for the - // installed file will be returned by PackagingSpecs() on this module or by - // TransitivePackagingSpecs() on modules that depend on this module through dependency tags - // for which IsInstallDepNeeded returns true. - InstallFileWithExtraFilesZip(installPath InstallPath, name string, srcPath Path, extraZip Path, deps ...Path) InstallPath - - // InstallSymlink creates a rule to create a symlink from src srcPath to name in the installPath - // directory. - // - // The installed symlink will be returned by FilesToInstall(), and the PackagingSpec for the - // installed file will be returned by PackagingSpecs() on this module or by - // TransitivePackagingSpecs() on modules that depend on this module through dependency tags - // for which IsInstallDepNeeded returns true. - InstallSymlink(installPath InstallPath, name string, srcPath InstallPath) InstallPath - - // InstallAbsoluteSymlink creates a rule to create an absolute symlink from src srcPath to name - // in the installPath directory. - // - // The installed symlink will be returned by FilesToInstall(), and the PackagingSpec for the - // installed file will be returned by PackagingSpecs() on this module or by - // TransitivePackagingSpecs() on modules that depend on this module through dependency tags - // for which IsInstallDepNeeded returns true. - InstallAbsoluteSymlink(installPath InstallPath, name string, absPath string) InstallPath - - // PackageFile creates a PackagingSpec as if InstallFile was called, but without creating - // the rule to copy the file. This is useful to define how a module would be packaged - // without installing it into the global installation directories. - // - // The created PackagingSpec for the will be returned by PackagingSpecs() on this module or by - // TransitivePackagingSpecs() on modules that depend on this module through dependency tags - // for which IsInstallDepNeeded returns true. - PackageFile(installPath InstallPath, name string, srcPath Path) PackagingSpec - - CheckbuildFile(srcPath Path) - - InstallInData() bool - InstallInTestcases() bool - InstallInSanitizerDir() bool - InstallInRamdisk() bool - InstallInVendorRamdisk() bool - InstallInDebugRamdisk() bool - InstallInRecovery() bool - InstallInRoot() bool - InstallInVendor() bool - InstallForceOS() (*OsType, *ArchType) - - RequiredModuleNames() []string - HostRequiredModuleNames() []string - TargetRequiredModuleNames() []string - - ModuleSubDir() string - SoongConfigTraceHash() string - - Variable(pctx PackageContext, name, value string) - Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule - // Similar to blueprint.ModuleContext.Build, but takes Paths instead of []string, - // and performs more verification. - Build(pctx PackageContext, params BuildParams) - // Phony creates a Make-style phony rule, a rule with no commands that can depend on other - // phony rules or real files. Phony can be called on the same name multiple times to add - // additional dependencies. - Phony(phony string, deps ...Path) - - // GetMissingDependencies returns the list of dependencies that were passed to AddDependencies or related methods, - // but do not exist. - GetMissingDependencies() []string - - // LicenseMetadataFile returns the path where the license metadata for this module will be - // generated. - LicenseMetadataFile() Path -} - type Module interface { blueprint.Module @@ -1677,18 +1209,6 @@ func (m *ModuleBase) GetPartitionForBp2build() string { return m.commonProperties.BazelConversionStatus.Partition } -// AddUnconvertedBp2buildDep stores module name of a dependency that was not converted to Bazel. -func (b *baseModuleContext) AddUnconvertedBp2buildDep(dep string) { - unconvertedDeps := &b.Module().base().commonProperties.BazelConversionStatus.UnconvertedDeps - *unconvertedDeps = append(*unconvertedDeps, dep) -} - -// AddMissingBp2buildDep stores module name of a dependency that was not found in a Android.bp file. -func (b *baseModuleContext) AddMissingBp2buildDep(dep string) { - missingDeps := &b.Module().base().commonProperties.BazelConversionStatus.MissingDeps - *missingDeps = append(*missingDeps, dep) -} - // GetUnconvertedBp2buildDeps returns the list of module names of this module's direct dependencies that // were not converted to Bazel. func (m *ModuleBase) GetUnconvertedBp2buildDeps() []string { @@ -2605,162 +2125,6 @@ func checkDistProperties(ctx *moduleContext, property string, dist *Dist) { } -type earlyModuleContext struct { - blueprint.EarlyModuleContext - - kind moduleKind - config Config -} - -func (e *earlyModuleContext) Glob(globPattern string, excludes []string) Paths { - return Glob(e, globPattern, excludes) -} - -func (e *earlyModuleContext) GlobFiles(globPattern string, excludes []string) Paths { - return GlobFiles(e, globPattern, excludes) -} - -func (e *earlyModuleContext) IsSymlink(path Path) bool { - fileInfo, err := e.config.fs.Lstat(path.String()) - if err != nil { - e.ModuleErrorf("os.Lstat(%q) failed: %s", path.String(), err) - } - return fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink -} - -func (e *earlyModuleContext) Readlink(path Path) string { - dest, err := e.config.fs.Readlink(path.String()) - if err != nil { - e.ModuleErrorf("os.Readlink(%q) failed: %s", path.String(), err) - } - return dest -} - -func (e *earlyModuleContext) Module() Module { - module, _ := e.EarlyModuleContext.Module().(Module) - return module -} - -func (e *earlyModuleContext) Config() Config { - return e.EarlyModuleContext.Config().(Config) -} - -func (e *earlyModuleContext) AConfig() Config { - return e.config -} - -func (e *earlyModuleContext) DeviceConfig() DeviceConfig { - return DeviceConfig{e.config.deviceConfig} -} - -func (e *earlyModuleContext) Platform() bool { - return e.kind == platformModule -} - -func (e *earlyModuleContext) DeviceSpecific() bool { - return e.kind == deviceSpecificModule -} - -func (e *earlyModuleContext) SocSpecific() bool { - return e.kind == socSpecificModule -} - -func (e *earlyModuleContext) ProductSpecific() bool { - return e.kind == productSpecificModule -} - -func (e *earlyModuleContext) SystemExtSpecific() bool { - return e.kind == systemExtSpecificModule -} - -func (e *earlyModuleContext) Namespace() *Namespace { - return e.EarlyModuleContext.Namespace().(*Namespace) -} - -type baseModuleContext struct { - bp blueprint.BaseModuleContext - earlyModuleContext - os OsType - target Target - multiTargets []Target - targetPrimary bool - - walkPath []Module - tagPath []blueprint.DependencyTag - - strictVisitDeps bool // If true, enforce that all dependencies are enabled - - bazelConversionMode bool -} - -func (b *baseModuleContext) isBazelConversionMode() bool { - return b.bazelConversionMode -} -func (b *baseModuleContext) OtherModuleName(m blueprint.Module) string { - return b.bp.OtherModuleName(m) -} -func (b *baseModuleContext) OtherModuleDir(m blueprint.Module) string { return b.bp.OtherModuleDir(m) } -func (b *baseModuleContext) OtherModuleErrorf(m blueprint.Module, fmt string, args ...interface{}) { - b.bp.OtherModuleErrorf(m, fmt, args...) -} -func (b *baseModuleContext) OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag { - return b.bp.OtherModuleDependencyTag(m) -} -func (b *baseModuleContext) OtherModuleExists(name string) bool { return b.bp.OtherModuleExists(name) } -func (b *baseModuleContext) OtherModuleDependencyVariantExists(variations []blueprint.Variation, name string) bool { - return b.bp.OtherModuleDependencyVariantExists(variations, name) -} -func (b *baseModuleContext) OtherModuleFarDependencyVariantExists(variations []blueprint.Variation, name string) bool { - return b.bp.OtherModuleFarDependencyVariantExists(variations, name) -} -func (b *baseModuleContext) OtherModuleReverseDependencyVariantExists(name string) bool { - return b.bp.OtherModuleReverseDependencyVariantExists(name) -} -func (b *baseModuleContext) OtherModuleType(m blueprint.Module) string { - return b.bp.OtherModuleType(m) -} -func (b *baseModuleContext) OtherModuleProvider(m blueprint.Module, provider blueprint.ProviderKey) interface{} { - return b.bp.OtherModuleProvider(m, provider) -} -func (b *baseModuleContext) OtherModuleHasProvider(m blueprint.Module, provider blueprint.ProviderKey) bool { - return b.bp.OtherModuleHasProvider(m, provider) -} -func (b *baseModuleContext) Provider(provider blueprint.ProviderKey) interface{} { - return b.bp.Provider(provider) -} -func (b *baseModuleContext) HasProvider(provider blueprint.ProviderKey) bool { - return b.bp.HasProvider(provider) -} -func (b *baseModuleContext) SetProvider(provider blueprint.ProviderKey, value interface{}) { - b.bp.SetProvider(provider, value) -} - -func (b *baseModuleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module { - return b.bp.GetDirectDepWithTag(name, tag) -} - -func (b *baseModuleContext) blueprintBaseModuleContext() blueprint.BaseModuleContext { - return b.bp -} - -type moduleContext struct { - bp blueprint.ModuleContext - baseModuleContext - packagingSpecs []PackagingSpec - installFiles InstallPaths - checkbuildFiles Paths - module Module - phonies map[string]Paths - - katiInstalls []katiInstall - katiSymlinks []katiInstall - - // For tests - buildParams []BuildParams - ruleParams map[blueprint.Rule]blueprint.RuleParams - variables map[string]string -} - // katiInstall stores a request from Soong to Make to create an install rule. type katiInstall struct { from Path @@ -2804,523 +2168,6 @@ func (installs katiInstalls) InstallPaths() InstallPaths { return paths } -func (m *moduleContext) ninjaError(params BuildParams, err error) (PackageContext, BuildParams) { - return pctx, BuildParams{ - Rule: ErrorRule, - Description: params.Description, - Output: params.Output, - Outputs: params.Outputs, - ImplicitOutput: params.ImplicitOutput, - ImplicitOutputs: params.ImplicitOutputs, - Args: map[string]string{ - "error": err.Error(), - }, - } -} - -func (m *moduleContext) ModuleBuild(pctx PackageContext, params ModuleBuildParams) { - m.Build(pctx, BuildParams(params)) -} - -func validateBuildParams(params blueprint.BuildParams) error { - // Validate that the symlink outputs are declared outputs or implicit outputs - allOutputs := map[string]bool{} - for _, output := range params.Outputs { - allOutputs[output] = true - } - for _, output := range params.ImplicitOutputs { - allOutputs[output] = true - } - for _, symlinkOutput := range params.SymlinkOutputs { - if !allOutputs[symlinkOutput] { - return fmt.Errorf( - "Symlink output %s is not a declared output or implicit output", - symlinkOutput) - } - } - return nil -} - -// Convert build parameters from their concrete Android types into their string representations, -// and combine the singular and plural fields of the same type (e.g. Output and Outputs). -func convertBuildParams(params BuildParams) blueprint.BuildParams { - bparams := blueprint.BuildParams{ - Rule: params.Rule, - Description: params.Description, - Deps: params.Deps, - Outputs: params.Outputs.Strings(), - ImplicitOutputs: params.ImplicitOutputs.Strings(), - SymlinkOutputs: params.SymlinkOutputs.Strings(), - Inputs: params.Inputs.Strings(), - Implicits: params.Implicits.Strings(), - OrderOnly: params.OrderOnly.Strings(), - Validations: params.Validations.Strings(), - Args: params.Args, - Optional: !params.Default, - } - - if params.Depfile != nil { - bparams.Depfile = params.Depfile.String() - } - if params.Output != nil { - bparams.Outputs = append(bparams.Outputs, params.Output.String()) - } - if params.SymlinkOutput != nil { - bparams.SymlinkOutputs = append(bparams.SymlinkOutputs, params.SymlinkOutput.String()) - } - if params.ImplicitOutput != nil { - bparams.ImplicitOutputs = append(bparams.ImplicitOutputs, params.ImplicitOutput.String()) - } - if params.Input != nil { - bparams.Inputs = append(bparams.Inputs, params.Input.String()) - } - if params.Implicit != nil { - bparams.Implicits = append(bparams.Implicits, params.Implicit.String()) - } - if params.Validation != nil { - bparams.Validations = append(bparams.Validations, params.Validation.String()) - } - - bparams.Outputs = proptools.NinjaEscapeList(bparams.Outputs) - bparams.ImplicitOutputs = proptools.NinjaEscapeList(bparams.ImplicitOutputs) - bparams.SymlinkOutputs = proptools.NinjaEscapeList(bparams.SymlinkOutputs) - bparams.Inputs = proptools.NinjaEscapeList(bparams.Inputs) - bparams.Implicits = proptools.NinjaEscapeList(bparams.Implicits) - bparams.OrderOnly = proptools.NinjaEscapeList(bparams.OrderOnly) - bparams.Validations = proptools.NinjaEscapeList(bparams.Validations) - bparams.Depfile = proptools.NinjaEscape(bparams.Depfile) - - return bparams -} - -func (m *moduleContext) Variable(pctx PackageContext, name, value string) { - if m.config.captureBuild { - m.variables[name] = value - } - - m.bp.Variable(pctx.PackageContext, name, value) -} - -func (m *moduleContext) Rule(pctx PackageContext, name string, params blueprint.RuleParams, - argNames ...string) blueprint.Rule { - - if m.config.UseRemoteBuild() { - if params.Pool == nil { - // When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by goma/RBE, restrict - // jobs to the local parallelism value - params.Pool = localPool - } else if params.Pool == remotePool { - // remotePool is a fake pool used to identify rule that are supported for remoting. If the rule's - // pool is the remotePool, replace with nil so that ninja runs it at NINJA_REMOTE_NUM_JOBS - // parallelism. - params.Pool = nil - } - } - - rule := m.bp.Rule(pctx.PackageContext, name, params, argNames...) - - if m.config.captureBuild { - m.ruleParams[rule] = params - } - - return rule -} - -func (m *moduleContext) Build(pctx PackageContext, params BuildParams) { - if params.Description != "" { - params.Description = "${moduleDesc}" + params.Description + "${moduleDescSuffix}" - } - - if missingDeps := m.GetMissingDependencies(); len(missingDeps) > 0 { - pctx, params = m.ninjaError(params, fmt.Errorf("module %s missing dependencies: %s\n", - m.ModuleName(), strings.Join(missingDeps, ", "))) - } - - if m.config.captureBuild { - m.buildParams = append(m.buildParams, params) - } - - bparams := convertBuildParams(params) - err := validateBuildParams(bparams) - if err != nil { - m.ModuleErrorf( - "%s: build parameter validation failed: %s", - m.ModuleName(), - err.Error()) - } - m.bp.Build(pctx.PackageContext, bparams) -} - -func (m *moduleContext) Phony(name string, deps ...Path) { - addPhony(m.config, name, deps...) -} - -func (m *moduleContext) GetMissingDependencies() []string { - var missingDeps []string - missingDeps = append(missingDeps, m.Module().base().commonProperties.MissingDeps...) - missingDeps = append(missingDeps, m.bp.GetMissingDependencies()...) - missingDeps = FirstUniqueStrings(missingDeps) - return missingDeps -} - -func (b *baseModuleContext) AddMissingDependencies(deps []string) { - if deps != nil { - missingDeps := &b.Module().base().commonProperties.MissingDeps - *missingDeps = append(*missingDeps, deps...) - *missingDeps = FirstUniqueStrings(*missingDeps) - } -} - -func (b *baseModuleContext) checkedMissingDeps() bool { - return b.Module().base().commonProperties.CheckedMissingDeps -} - -func (b *baseModuleContext) getMissingDependencies() []string { - checked := &b.Module().base().commonProperties.CheckedMissingDeps - *checked = true - var missingDeps []string - missingDeps = append(missingDeps, b.Module().base().commonProperties.MissingDeps...) - missingDeps = append(missingDeps, b.bp.EarlyGetMissingDependencies()...) - missingDeps = FirstUniqueStrings(missingDeps) - return missingDeps -} - -type AllowDisabledModuleDependency interface { - blueprint.DependencyTag - AllowDisabledModuleDependency(target Module) bool -} - -func (b *baseModuleContext) validateAndroidModule(module blueprint.Module, tag blueprint.DependencyTag, strict bool) Module { - aModule, _ := module.(Module) - - if !strict { - return aModule - } - - if aModule == nil { - b.ModuleErrorf("module %q (%#v) not an android module", b.OtherModuleName(module), tag) - return nil - } - - if !aModule.Enabled() { - if t, ok := tag.(AllowDisabledModuleDependency); !ok || !t.AllowDisabledModuleDependency(aModule) { - if b.Config().AllowMissingDependencies() { - b.AddMissingDependencies([]string{b.OtherModuleName(aModule)}) - } else { - b.ModuleErrorf("depends on disabled module %q", b.OtherModuleName(aModule)) - } - } - return nil - } - return aModule -} - -type dep struct { - mod blueprint.Module - tag blueprint.DependencyTag -} - -func (b *baseModuleContext) getDirectDepsInternal(name string, tag blueprint.DependencyTag) []dep { - var deps []dep - b.VisitDirectDepsBlueprint(func(module blueprint.Module) { - if aModule, _ := module.(Module); aModule != nil { - if aModule.base().BaseModuleName() == name { - returnedTag := b.bp.OtherModuleDependencyTag(aModule) - if tag == nil || returnedTag == tag { - deps = append(deps, dep{aModule, returnedTag}) - } - } - } else if b.bp.OtherModuleName(module) == name { - returnedTag := b.bp.OtherModuleDependencyTag(module) - if tag == nil || returnedTag == tag { - deps = append(deps, dep{module, returnedTag}) - } - } - }) - return deps -} - -func (b *baseModuleContext) getDirectDepInternal(name string, tag blueprint.DependencyTag) (blueprint.Module, blueprint.DependencyTag) { - deps := b.getDirectDepsInternal(name, tag) - if len(deps) == 1 { - return deps[0].mod, deps[0].tag - } else if len(deps) >= 2 { - panic(fmt.Errorf("Multiple dependencies having same BaseModuleName() %q found from %q", - name, b.ModuleName())) - } else { - return nil, nil - } -} - -func (b *baseModuleContext) getDirectDepFirstTag(name string) (blueprint.Module, blueprint.DependencyTag) { - foundDeps := b.getDirectDepsInternal(name, nil) - deps := map[blueprint.Module]bool{} - for _, dep := range foundDeps { - deps[dep.mod] = true - } - if len(deps) == 1 { - return foundDeps[0].mod, foundDeps[0].tag - } else if len(deps) >= 2 { - // this could happen if two dependencies have the same name in different namespaces - // TODO(b/186554727): this should not occur if namespaces are handled within - // getDirectDepsInternal. - panic(fmt.Errorf("Multiple dependencies having same BaseModuleName() %q found from %q", - name, b.ModuleName())) - } else { - return nil, nil - } -} - -func (b *baseModuleContext) GetDirectDepsWithTag(tag blueprint.DependencyTag) []Module { - var deps []Module - b.VisitDirectDepsBlueprint(func(module blueprint.Module) { - if aModule, _ := module.(Module); aModule != nil { - if b.bp.OtherModuleDependencyTag(aModule) == tag { - deps = append(deps, aModule) - } - } - }) - return deps -} - -func (m *moduleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module { - module, _ := m.getDirectDepInternal(name, tag) - return module -} - -// GetDirectDep returns the Module and DependencyTag for the direct dependency with the specified -// name, or nil if none exists. If there are multiple dependencies on the same module it returns the -// first DependencyTag. -func (b *baseModuleContext) GetDirectDep(name string) (blueprint.Module, blueprint.DependencyTag) { - return b.getDirectDepFirstTag(name) -} - -func (b *baseModuleContext) ModuleFromName(name string) (blueprint.Module, bool) { - if !b.isBazelConversionMode() { - panic("cannot call ModuleFromName if not in bazel conversion mode") - } - var m blueprint.Module - var ok bool - if moduleName, _ := SrcIsModuleWithTag(name); moduleName != "" { - m, ok = b.bp.ModuleFromName(moduleName) - } else { - m, ok = b.bp.ModuleFromName(name) - } - if !ok { - return m, ok - } - // If this module is not preferred, tried to get the prebuilt version instead - if a, aOk := m.(Module); aOk && !IsModulePrebuilt(a) && !IsModulePreferred(a) { - return b.ModuleFromName("prebuilt_" + name) - } - return m, ok -} - -func (b *baseModuleContext) VisitDirectDepsBlueprint(visit func(blueprint.Module)) { - b.bp.VisitDirectDeps(visit) -} - -func (b *baseModuleContext) VisitDirectDeps(visit func(Module)) { - b.bp.VisitDirectDeps(func(module blueprint.Module) { - if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { - visit(aModule) - } - }) -} - -func (b *baseModuleContext) VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) { - b.bp.VisitDirectDeps(func(module blueprint.Module) { - if b.bp.OtherModuleDependencyTag(module) == tag { - if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { - visit(aModule) - } - } - }) -} - -func (b *baseModuleContext) VisitDirectDepsIf(pred func(Module) bool, visit func(Module)) { - b.bp.VisitDirectDepsIf( - // pred - func(module blueprint.Module) bool { - if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { - return pred(aModule) - } else { - return false - } - }, - // visit - func(module blueprint.Module) { - visit(module.(Module)) - }) -} - -func (b *baseModuleContext) VisitDepsDepthFirst(visit func(Module)) { - b.bp.VisitDepsDepthFirst(func(module blueprint.Module) { - if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { - visit(aModule) - } - }) -} - -func (b *baseModuleContext) VisitDepsDepthFirstIf(pred func(Module) bool, visit func(Module)) { - b.bp.VisitDepsDepthFirstIf( - // pred - func(module blueprint.Module) bool { - if aModule := b.validateAndroidModule(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { - return pred(aModule) - } else { - return false - } - }, - // visit - func(module blueprint.Module) { - visit(module.(Module)) - }) -} - -func (b *baseModuleContext) WalkDepsBlueprint(visit func(blueprint.Module, blueprint.Module) bool) { - b.bp.WalkDeps(visit) -} - -func (b *baseModuleContext) WalkDeps(visit func(Module, Module) bool) { - b.walkPath = []Module{b.Module()} - b.tagPath = []blueprint.DependencyTag{} - b.bp.WalkDeps(func(child, parent blueprint.Module) bool { - childAndroidModule, _ := child.(Module) - parentAndroidModule, _ := parent.(Module) - if childAndroidModule != nil && parentAndroidModule != nil { - // record walkPath before visit - for b.walkPath[len(b.walkPath)-1] != parentAndroidModule { - b.walkPath = b.walkPath[0 : len(b.walkPath)-1] - b.tagPath = b.tagPath[0 : len(b.tagPath)-1] - } - b.walkPath = append(b.walkPath, childAndroidModule) - b.tagPath = append(b.tagPath, b.OtherModuleDependencyTag(childAndroidModule)) - return visit(childAndroidModule, parentAndroidModule) - } else { - return false - } - }) -} - -func (b *baseModuleContext) GetWalkPath() []Module { - return b.walkPath -} - -func (b *baseModuleContext) GetTagPath() []blueprint.DependencyTag { - return b.tagPath -} - -func (b *baseModuleContext) VisitAllModuleVariants(visit func(Module)) { - b.bp.VisitAllModuleVariants(func(module blueprint.Module) { - visit(module.(Module)) - }) -} - -func (b *baseModuleContext) PrimaryModule() Module { - return b.bp.PrimaryModule().(Module) -} - -func (b *baseModuleContext) FinalModule() Module { - return b.bp.FinalModule().(Module) -} - -// IsMetaDependencyTag returns true for cross-cutting metadata dependencies. -func IsMetaDependencyTag(tag blueprint.DependencyTag) bool { - if tag == licenseKindTag { - return true - } else if tag == licensesTag { - return true - } - return false -} - -// A regexp for removing boilerplate from BaseDependencyTag from the string representation of -// a dependency tag. -var tagCleaner = regexp.MustCompile(`\QBaseDependencyTag:{}\E(, )?`) - -// PrettyPrintTag returns string representation of the tag, but prefers -// custom String() method if available. -func PrettyPrintTag(tag blueprint.DependencyTag) string { - // Use tag's custom String() method if available. - if stringer, ok := tag.(fmt.Stringer); ok { - return stringer.String() - } - - // Otherwise, get a default string representation of the tag's struct. - tagString := fmt.Sprintf("%T: %+v", tag, tag) - - // Remove the boilerplate from BaseDependencyTag as it adds no value. - tagString = tagCleaner.ReplaceAllString(tagString, "") - return tagString -} - -func (b *baseModuleContext) GetPathString(skipFirst bool) string { - sb := strings.Builder{} - tagPath := b.GetTagPath() - walkPath := b.GetWalkPath() - if !skipFirst { - sb.WriteString(walkPath[0].String()) - } - for i, m := range walkPath[1:] { - sb.WriteString("\n") - sb.WriteString(fmt.Sprintf(" via tag %s\n", PrettyPrintTag(tagPath[i]))) - sb.WriteString(fmt.Sprintf(" -> %s", m.String())) - } - return sb.String() -} - -func (m *moduleContext) ModuleSubDir() string { - return m.bp.ModuleSubDir() -} - -func (m *moduleContext) SoongConfigTraceHash() string { - return m.module.base().commonProperties.SoongConfigTraceHash -} - -func (b *baseModuleContext) Target() Target { - return b.target -} - -func (b *baseModuleContext) TargetPrimary() bool { - return b.targetPrimary -} - -func (b *baseModuleContext) MultiTargets() []Target { - return b.multiTargets -} - -func (b *baseModuleContext) Arch() Arch { - return b.target.Arch -} - -func (b *baseModuleContext) Os() OsType { - return b.os -} - -func (b *baseModuleContext) Host() bool { - return b.os.Class == Host -} - -func (b *baseModuleContext) Device() bool { - return b.os.Class == Device -} - -func (b *baseModuleContext) Darwin() bool { - return b.os == Darwin -} - -func (b *baseModuleContext) Windows() bool { - return b.os == Windows -} - -func (b *baseModuleContext) PrimaryArch() bool { - if len(b.config.Targets[b.target.Os]) <= 1 { - return true - } - return b.target.Arch.ArchType == b.config.Targets[b.target.Os][0].Arch.ArchType -} - // Makes this module a platform module, i.e. not specific to soc, device, // product, or system_ext. func (m *ModuleBase) MakeAsPlatform() { @@ -3344,274 +2191,6 @@ func (m *ModuleBase) IsNativeBridgeSupported() bool { return proptools.Bool(m.commonProperties.Native_bridge_supported) } -func (m *moduleContext) InstallInData() bool { - return m.module.InstallInData() -} - -func (m *moduleContext) InstallInTestcases() bool { - return m.module.InstallInTestcases() -} - -func (m *moduleContext) InstallInSanitizerDir() bool { - return m.module.InstallInSanitizerDir() -} - -func (m *moduleContext) InstallInRamdisk() bool { - return m.module.InstallInRamdisk() -} - -func (m *moduleContext) InstallInVendorRamdisk() bool { - return m.module.InstallInVendorRamdisk() -} - -func (m *moduleContext) InstallInDebugRamdisk() bool { - return m.module.InstallInDebugRamdisk() -} - -func (m *moduleContext) InstallInRecovery() bool { - return m.module.InstallInRecovery() -} - -func (m *moduleContext) InstallInRoot() bool { - return m.module.InstallInRoot() -} - -func (m *moduleContext) InstallForceOS() (*OsType, *ArchType) { - return m.module.InstallForceOS() -} - -func (m *moduleContext) InstallInVendor() bool { - return m.module.InstallInVendor() -} - -func (m *moduleContext) skipInstall() bool { - if m.module.base().commonProperties.SkipInstall { - return true - } - - if m.module.base().commonProperties.HideFromMake { - return true - } - - // We'll need a solution for choosing which of modules with the same name in different - // namespaces to install. For now, reuse the list of namespaces exported to Make as the - // list of namespaces to install in a Soong-only build. - if !m.module.base().commonProperties.NamespaceExportedToMake { - return true - } - - return false -} - -func (m *moduleContext) InstallFile(installPath InstallPath, name string, srcPath Path, - deps ...Path) InstallPath { - return m.installFile(installPath, name, srcPath, deps, false, nil) -} - -func (m *moduleContext) InstallExecutable(installPath InstallPath, name string, srcPath Path, - deps ...Path) InstallPath { - return m.installFile(installPath, name, srcPath, deps, true, nil) -} - -func (m *moduleContext) InstallFileWithExtraFilesZip(installPath InstallPath, name string, srcPath Path, - extraZip Path, deps ...Path) InstallPath { - return m.installFile(installPath, name, srcPath, deps, false, &extraFilesZip{ - zip: extraZip, - dir: installPath, - }) -} - -func (m *moduleContext) PackageFile(installPath InstallPath, name string, srcPath Path) PackagingSpec { - fullInstallPath := installPath.Join(m, name) - return m.packageFile(fullInstallPath, srcPath, false) -} - -func (m *moduleContext) packageFile(fullInstallPath InstallPath, srcPath Path, executable bool) PackagingSpec { - licenseFiles := m.Module().EffectiveLicenseFiles() - spec := PackagingSpec{ - relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), - srcPath: srcPath, - symlinkTarget: "", - executable: executable, - effectiveLicenseFiles: &licenseFiles, - partition: fullInstallPath.partition, - } - m.packagingSpecs = append(m.packagingSpecs, spec) - return spec -} - -func (m *moduleContext) installFile(installPath InstallPath, name string, srcPath Path, deps []Path, - executable bool, extraZip *extraFilesZip) InstallPath { - - fullInstallPath := installPath.Join(m, name) - m.module.base().hooks.runInstallHooks(m, srcPath, fullInstallPath, false) - - if !m.skipInstall() { - deps = append(deps, InstallPaths(m.module.base().installFilesDepSet.ToList()).Paths()...) - - var implicitDeps, orderOnlyDeps Paths - - if m.Host() { - // Installed host modules might be used during the build, depend directly on their - // dependencies so their timestamp is updated whenever their dependency is updated - implicitDeps = deps - } else { - orderOnlyDeps = deps - } - - if m.Config().KatiEnabled() { - // When creating the install rule in Soong but embedding in Make, write the rule to a - // makefile instead of directly to the ninja file so that main.mk can add the - // dependencies from the `required` property that are hard to resolve in Soong. - m.katiInstalls = append(m.katiInstalls, katiInstall{ - from: srcPath, - to: fullInstallPath, - implicitDeps: implicitDeps, - orderOnlyDeps: orderOnlyDeps, - executable: executable, - extraFiles: extraZip, - }) - } else { - rule := Cp - if executable { - rule = CpExecutable - } - - extraCmds := "" - if extraZip != nil { - extraCmds += fmt.Sprintf(" && ( unzip -qDD -d '%s' '%s' 2>&1 | grep -v \"zipfile is empty\"; exit $${PIPESTATUS[0]} )", - extraZip.dir.String(), extraZip.zip.String()) - extraCmds += " || ( code=$$?; if [ $$code -ne 0 -a $$code -ne 1 ]; then exit $$code; fi )" - implicitDeps = append(implicitDeps, extraZip.zip) - } - - m.Build(pctx, BuildParams{ - Rule: rule, - Description: "install " + fullInstallPath.Base(), - Output: fullInstallPath, - Input: srcPath, - Implicits: implicitDeps, - OrderOnly: orderOnlyDeps, - Default: !m.Config().KatiEnabled(), - Args: map[string]string{ - "extraCmds": extraCmds, - }, - }) - } - - m.installFiles = append(m.installFiles, fullInstallPath) - } - - m.packageFile(fullInstallPath, srcPath, executable) - - m.checkbuildFiles = append(m.checkbuildFiles, srcPath) - - return fullInstallPath -} - -func (m *moduleContext) InstallSymlink(installPath InstallPath, name string, srcPath InstallPath) InstallPath { - fullInstallPath := installPath.Join(m, name) - m.module.base().hooks.runInstallHooks(m, srcPath, fullInstallPath, true) - - relPath, err := filepath.Rel(path.Dir(fullInstallPath.String()), srcPath.String()) - if err != nil { - panic(fmt.Sprintf("Unable to generate symlink between %q and %q: %s", fullInstallPath.Base(), srcPath.Base(), err)) - } - if !m.skipInstall() { - - if m.Config().KatiEnabled() { - // When creating the symlink rule in Soong but embedding in Make, write the rule to a - // makefile instead of directly to the ninja file so that main.mk can add the - // dependencies from the `required` property that are hard to resolve in Soong. - m.katiSymlinks = append(m.katiSymlinks, katiInstall{ - from: srcPath, - to: fullInstallPath, - }) - } else { - // The symlink doesn't need updating when the target is modified, but we sometimes - // have a dependency on a symlink to a binary instead of to the binary directly, and - // the mtime of the symlink must be updated when the binary is modified, so use a - // normal dependency here instead of an order-only dependency. - m.Build(pctx, BuildParams{ - Rule: Symlink, - Description: "install symlink " + fullInstallPath.Base(), - Output: fullInstallPath, - Input: srcPath, - Default: !m.Config().KatiEnabled(), - Args: map[string]string{ - "fromPath": relPath, - }, - }) - } - - m.installFiles = append(m.installFiles, fullInstallPath) - m.checkbuildFiles = append(m.checkbuildFiles, srcPath) - } - - m.packagingSpecs = append(m.packagingSpecs, PackagingSpec{ - relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), - srcPath: nil, - symlinkTarget: relPath, - executable: false, - partition: fullInstallPath.partition, - }) - - return fullInstallPath -} - -// installPath/name -> absPath where absPath might be a path that is available only at runtime -// (e.g. /apex/...) -func (m *moduleContext) InstallAbsoluteSymlink(installPath InstallPath, name string, absPath string) InstallPath { - fullInstallPath := installPath.Join(m, name) - m.module.base().hooks.runInstallHooks(m, nil, fullInstallPath, true) - - if !m.skipInstall() { - if m.Config().KatiEnabled() { - // When creating the symlink rule in Soong but embedding in Make, write the rule to a - // makefile instead of directly to the ninja file so that main.mk can add the - // dependencies from the `required` property that are hard to resolve in Soong. - m.katiSymlinks = append(m.katiSymlinks, katiInstall{ - absFrom: absPath, - to: fullInstallPath, - }) - } else { - m.Build(pctx, BuildParams{ - Rule: Symlink, - Description: "install symlink " + fullInstallPath.Base() + " -> " + absPath, - Output: fullInstallPath, - Default: !m.Config().KatiEnabled(), - Args: map[string]string{ - "fromPath": absPath, - }, - }) - } - - m.installFiles = append(m.installFiles, fullInstallPath) - } - - m.packagingSpecs = append(m.packagingSpecs, PackagingSpec{ - relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), - srcPath: nil, - symlinkTarget: absPath, - executable: false, - partition: fullInstallPath.partition, - }) - - return fullInstallPath -} - -func (m *moduleContext) CheckbuildFile(srcPath Path) { - m.checkbuildFiles = append(m.checkbuildFiles, srcPath) -} - -func (m *moduleContext) blueprintModuleContext() blueprint.ModuleContext { - return m.bp -} - -func (m *moduleContext) LicenseMetadataFile() Path { - return m.module.base().licenseMetadataFile -} - // SrcIsModule decodes module references in the format ":unqualified-name" or "//namespace:name" // into the module name, or empty string if the input was not a module reference. func SrcIsModule(s string) (module string) { @@ -3818,44 +2397,6 @@ type HostToolProvider interface { HostToolPath() OptionalPath } -// Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must -// be tagged with `android:"path" to support automatic source module dependency resolution. -// -// Deprecated: use PathsForModuleSrc or PathsForModuleSrcExcludes instead. -func (m *moduleContext) ExpandSources(srcFiles, excludes []string) Paths { - return PathsForModuleSrcExcludes(m, srcFiles, excludes) -} - -// Returns a single path expanded from globs and modules referenced using ":module" syntax. The property must -// be tagged with `android:"path" to support automatic source module dependency resolution. -// -// Deprecated: use PathForModuleSrc instead. -func (m *moduleContext) ExpandSource(srcFile, _ string) Path { - return PathForModuleSrc(m, srcFile) -} - -// Returns an optional single path expanded from globs and modules referenced using ":module" syntax if -// the srcFile is non-nil. The property must be tagged with `android:"path" to support automatic source module -// dependency resolution. -func (m *moduleContext) ExpandOptionalSource(srcFile *string, _ string) OptionalPath { - if srcFile != nil { - return OptionalPathForPath(PathForModuleSrc(m, *srcFile)) - } - return OptionalPath{} -} - -func (m *moduleContext) RequiredModuleNames() []string { - return m.module.RequiredModuleNames() -} - -func (m *moduleContext) HostRequiredModuleNames() []string { - return m.module.HostRequiredModuleNames() -} - -func (m *moduleContext) TargetRequiredModuleNames() []string { - return m.module.TargetRequiredModuleNames() -} - func init() { RegisterParallelSingletonType("buildtarget", BuildTargetSingleton) RegisterParallelSingletonType("soongconfigtrace", soongConfigTraceSingletonFunc) diff --git a/android/module_context.go b/android/module_context.go new file mode 100644 index 000000000..a0a410428 --- /dev/null +++ b/android/module_context.go @@ -0,0 +1,698 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + "path" + "path/filepath" + "strings" +) + +// BuildParameters describes the set of potential parameters to build a Ninja rule. +// In general, these correspond to a Ninja concept. +type BuildParams struct { + // A Ninja Rule that will be written to the Ninja file. This allows factoring out common code + // among multiple modules to reduce repetition in the Ninja file of action requirements. A rule + // can contain variables that should be provided in Args. + Rule blueprint.Rule + // Deps represents the depfile format. When using RuleBuilder, this defaults to GCC when depfiles + // are used. + Deps blueprint.Deps + // Depfile is a writeable path that allows correct incremental builds when the inputs have not + // been fully specified by the Ninja rule. Ninja supports a subset of the Makefile depfile syntax. + Depfile WritablePath + // A description of the build action. + Description string + // Output is an output file of the action. When using this field, references to $out in the Ninja + // command will refer to this file. + Output WritablePath + // Outputs is a slice of output file of the action. When using this field, references to $out in + // the Ninja command will refer to these files. + Outputs WritablePaths + // SymlinkOutput is an output file specifically that is a symlink. + SymlinkOutput WritablePath + // SymlinkOutputs is a slice of output files specifically that is a symlink. + SymlinkOutputs WritablePaths + // ImplicitOutput is an output file generated by the action. Note: references to `$out` in the + // Ninja command will NOT include references to this file. + ImplicitOutput WritablePath + // ImplicitOutputs is a slice of output files generated by the action. Note: references to `$out` + // in the Ninja command will NOT include references to these files. + ImplicitOutputs WritablePaths + // Input is an input file to the Ninja action. When using this field, references to $in in the + // Ninja command will refer to this file. + Input Path + // Inputs is a slice of input files to the Ninja action. When using this field, references to $in + // in the Ninja command will refer to these files. + Inputs Paths + // Implicit is an input file to the Ninja action. Note: references to `$in` in the Ninja command + // will NOT include references to this file. + Implicit Path + // Implicits is a slice of input files to the Ninja action. Note: references to `$in` in the Ninja + // command will NOT include references to these files. + Implicits Paths + // OrderOnly are Ninja order-only inputs to the action. When these are out of date, the output is + // not rebuilt until they are built, but changes in order-only dependencies alone do not cause the + // output to be rebuilt. + OrderOnly Paths + // Validation is an output path for a validation action. Validation outputs imply lower + // non-blocking priority to building non-validation outputs. + Validation Path + // Validations is a slice of output path for a validation action. Validation outputs imply lower + // non-blocking priority to building non-validation outputs. + Validations Paths + // Whether to skip outputting a default target statement which will be built by Ninja when no + // targets are specified on Ninja's command line. + Default bool + // Args is a key value mapping for replacements of variables within the Rule + Args map[string]string +} + +type ModuleBuildParams BuildParams + +type ModuleContext interface { + BaseModuleContext + + blueprintModuleContext() blueprint.ModuleContext + + // Deprecated: use ModuleContext.Build instead. + ModuleBuild(pctx PackageContext, params ModuleBuildParams) + + // Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must + // be tagged with `android:"path" to support automatic source module dependency resolution. + // + // Deprecated: use PathsForModuleSrc or PathsForModuleSrcExcludes instead. + ExpandSources(srcFiles, excludes []string) Paths + + // Returns a single path expanded from globs and modules referenced using ":module" syntax. The property must + // be tagged with `android:"path" to support automatic source module dependency resolution. + // + // Deprecated: use PathForModuleSrc instead. + ExpandSource(srcFile, prop string) Path + + ExpandOptionalSource(srcFile *string, prop string) OptionalPath + + // InstallExecutable creates a rule to copy srcPath to name in the installPath directory, + // with the given additional dependencies. The file is marked executable after copying. + // + // The installed file will be returned by FilesToInstall(), and the PackagingSpec for the + // installed file will be returned by PackagingSpecs() on this module or by + // TransitivePackagingSpecs() on modules that depend on this module through dependency tags + // for which IsInstallDepNeeded returns true. + InstallExecutable(installPath InstallPath, name string, srcPath Path, deps ...InstallPath) InstallPath + + // InstallFile creates a rule to copy srcPath to name in the installPath directory, + // with the given additional dependencies. + // + // The installed file will be returned by FilesToInstall(), and the PackagingSpec for the + // installed file will be returned by PackagingSpecs() on this module or by + // TransitivePackagingSpecs() on modules that depend on this module through dependency tags + // for which IsInstallDepNeeded returns true. + InstallFile(installPath InstallPath, name string, srcPath Path, deps ...InstallPath) InstallPath + + // InstallFileWithExtraFilesZip creates a rule to copy srcPath to name in the installPath + // directory, and also unzip a zip file containing extra files to install into the same + // directory. + // + // The installed file will be returned by FilesToInstall(), and the PackagingSpec for the + // installed file will be returned by PackagingSpecs() on this module or by + // TransitivePackagingSpecs() on modules that depend on this module through dependency tags + // for which IsInstallDepNeeded returns true. + InstallFileWithExtraFilesZip(installPath InstallPath, name string, srcPath Path, extraZip Path, deps ...InstallPath) InstallPath + + // InstallSymlink creates a rule to create a symlink from src srcPath to name in the installPath + // directory. + // + // The installed symlink will be returned by FilesToInstall(), and the PackagingSpec for the + // installed file will be returned by PackagingSpecs() on this module or by + // TransitivePackagingSpecs() on modules that depend on this module through dependency tags + // for which IsInstallDepNeeded returns true. + InstallSymlink(installPath InstallPath, name string, srcPath InstallPath) InstallPath + + // InstallAbsoluteSymlink creates a rule to create an absolute symlink from src srcPath to name + // in the installPath directory. + // + // The installed symlink will be returned by FilesToInstall(), and the PackagingSpec for the + // installed file will be returned by PackagingSpecs() on this module or by + // TransitivePackagingSpecs() on modules that depend on this module through dependency tags + // for which IsInstallDepNeeded returns true. + InstallAbsoluteSymlink(installPath InstallPath, name string, absPath string) InstallPath + + // PackageFile creates a PackagingSpec as if InstallFile was called, but without creating + // the rule to copy the file. This is useful to define how a module would be packaged + // without installing it into the global installation directories. + // + // The created PackagingSpec for the will be returned by PackagingSpecs() on this module or by + // TransitivePackagingSpecs() on modules that depend on this module through dependency tags + // for which IsInstallDepNeeded returns true. + PackageFile(installPath InstallPath, name string, srcPath Path) PackagingSpec + + CheckbuildFile(srcPath Path) + + InstallInData() bool + InstallInTestcases() bool + InstallInSanitizerDir() bool + InstallInRamdisk() bool + InstallInVendorRamdisk() bool + InstallInDebugRamdisk() bool + InstallInRecovery() bool + InstallInRoot() bool + InstallInVendor() bool + InstallForceOS() (*OsType, *ArchType) + + RequiredModuleNames() []string + HostRequiredModuleNames() []string + TargetRequiredModuleNames() []string + + ModuleSubDir() string + SoongConfigTraceHash() string + + Variable(pctx PackageContext, name, value string) + Rule(pctx PackageContext, name string, params blueprint.RuleParams, argNames ...string) blueprint.Rule + // Similar to blueprint.ModuleContext.Build, but takes Paths instead of []string, + // and performs more verification. + Build(pctx PackageContext, params BuildParams) + // Phony creates a Make-style phony rule, a rule with no commands that can depend on other + // phony rules or real files. Phony can be called on the same name multiple times to add + // additional dependencies. + Phony(phony string, deps ...Path) + + // GetMissingDependencies returns the list of dependencies that were passed to AddDependencies or related methods, + // but do not exist. + GetMissingDependencies() []string + + // LicenseMetadataFile returns the path where the license metadata for this module will be + // generated. + LicenseMetadataFile() Path +} + +type moduleContext struct { + bp blueprint.ModuleContext + baseModuleContext + packagingSpecs []PackagingSpec + installFiles InstallPaths + checkbuildFiles Paths + module Module + phonies map[string]Paths + + katiInstalls []katiInstall + katiSymlinks []katiInstall + + // For tests + buildParams []BuildParams + ruleParams map[blueprint.Rule]blueprint.RuleParams + variables map[string]string +} + +func (m *moduleContext) ninjaError(params BuildParams, err error) (PackageContext, BuildParams) { + return pctx, BuildParams{ + Rule: ErrorRule, + Description: params.Description, + Output: params.Output, + Outputs: params.Outputs, + ImplicitOutput: params.ImplicitOutput, + ImplicitOutputs: params.ImplicitOutputs, + Args: map[string]string{ + "error": err.Error(), + }, + } +} + +func (m *moduleContext) ModuleBuild(pctx PackageContext, params ModuleBuildParams) { + m.Build(pctx, BuildParams(params)) +} + +func validateBuildParams(params blueprint.BuildParams) error { + // Validate that the symlink outputs are declared outputs or implicit outputs + allOutputs := map[string]bool{} + for _, output := range params.Outputs { + allOutputs[output] = true + } + for _, output := range params.ImplicitOutputs { + allOutputs[output] = true + } + for _, symlinkOutput := range params.SymlinkOutputs { + if !allOutputs[symlinkOutput] { + return fmt.Errorf( + "Symlink output %s is not a declared output or implicit output", + symlinkOutput) + } + } + return nil +} + +// Convert build parameters from their concrete Android types into their string representations, +// and combine the singular and plural fields of the same type (e.g. Output and Outputs). +func convertBuildParams(params BuildParams) blueprint.BuildParams { + bparams := blueprint.BuildParams{ + Rule: params.Rule, + Description: params.Description, + Deps: params.Deps, + Outputs: params.Outputs.Strings(), + ImplicitOutputs: params.ImplicitOutputs.Strings(), + SymlinkOutputs: params.SymlinkOutputs.Strings(), + Inputs: params.Inputs.Strings(), + Implicits: params.Implicits.Strings(), + OrderOnly: params.OrderOnly.Strings(), + Validations: params.Validations.Strings(), + Args: params.Args, + Optional: !params.Default, + } + + if params.Depfile != nil { + bparams.Depfile = params.Depfile.String() + } + if params.Output != nil { + bparams.Outputs = append(bparams.Outputs, params.Output.String()) + } + if params.SymlinkOutput != nil { + bparams.SymlinkOutputs = append(bparams.SymlinkOutputs, params.SymlinkOutput.String()) + } + if params.ImplicitOutput != nil { + bparams.ImplicitOutputs = append(bparams.ImplicitOutputs, params.ImplicitOutput.String()) + } + if params.Input != nil { + bparams.Inputs = append(bparams.Inputs, params.Input.String()) + } + if params.Implicit != nil { + bparams.Implicits = append(bparams.Implicits, params.Implicit.String()) + } + if params.Validation != nil { + bparams.Validations = append(bparams.Validations, params.Validation.String()) + } + + bparams.Outputs = proptools.NinjaEscapeList(bparams.Outputs) + bparams.ImplicitOutputs = proptools.NinjaEscapeList(bparams.ImplicitOutputs) + bparams.SymlinkOutputs = proptools.NinjaEscapeList(bparams.SymlinkOutputs) + bparams.Inputs = proptools.NinjaEscapeList(bparams.Inputs) + bparams.Implicits = proptools.NinjaEscapeList(bparams.Implicits) + bparams.OrderOnly = proptools.NinjaEscapeList(bparams.OrderOnly) + bparams.Validations = proptools.NinjaEscapeList(bparams.Validations) + bparams.Depfile = proptools.NinjaEscape(bparams.Depfile) + + return bparams +} + +func (m *moduleContext) Variable(pctx PackageContext, name, value string) { + if m.config.captureBuild { + m.variables[name] = value + } + + m.bp.Variable(pctx.PackageContext, name, value) +} + +func (m *moduleContext) Rule(pctx PackageContext, name string, params blueprint.RuleParams, + argNames ...string) blueprint.Rule { + + if m.config.UseRemoteBuild() { + if params.Pool == nil { + // When USE_GOMA=true or USE_RBE=true are set and the rule is not supported by goma/RBE, restrict + // jobs to the local parallelism value + params.Pool = localPool + } else if params.Pool == remotePool { + // remotePool is a fake pool used to identify rule that are supported for remoting. If the rule's + // pool is the remotePool, replace with nil so that ninja runs it at NINJA_REMOTE_NUM_JOBS + // parallelism. + params.Pool = nil + } + } + + rule := m.bp.Rule(pctx.PackageContext, name, params, argNames...) + + if m.config.captureBuild { + m.ruleParams[rule] = params + } + + return rule +} + +func (m *moduleContext) Build(pctx PackageContext, params BuildParams) { + if params.Description != "" { + params.Description = "${moduleDesc}" + params.Description + "${moduleDescSuffix}" + } + + if missingDeps := m.GetMissingDependencies(); len(missingDeps) > 0 { + pctx, params = m.ninjaError(params, fmt.Errorf("module %s missing dependencies: %s\n", + m.ModuleName(), strings.Join(missingDeps, ", "))) + } + + if m.config.captureBuild { + m.buildParams = append(m.buildParams, params) + } + + bparams := convertBuildParams(params) + err := validateBuildParams(bparams) + if err != nil { + m.ModuleErrorf( + "%s: build parameter validation failed: %s", + m.ModuleName(), + err.Error()) + } + m.bp.Build(pctx.PackageContext, bparams) +} + +func (m *moduleContext) Phony(name string, deps ...Path) { + addPhony(m.config, name, deps...) +} + +func (m *moduleContext) GetMissingDependencies() []string { + var missingDeps []string + missingDeps = append(missingDeps, m.Module().base().commonProperties.MissingDeps...) + missingDeps = append(missingDeps, m.bp.GetMissingDependencies()...) + missingDeps = FirstUniqueStrings(missingDeps) + return missingDeps +} + +func (m *moduleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module { + module, _ := m.getDirectDepInternal(name, tag) + return module +} + +func (m *moduleContext) ModuleSubDir() string { + return m.bp.ModuleSubDir() +} + +func (m *moduleContext) SoongConfigTraceHash() string { + return m.module.base().commonProperties.SoongConfigTraceHash +} + +func (m *moduleContext) InstallInData() bool { + return m.module.InstallInData() +} + +func (m *moduleContext) InstallInTestcases() bool { + return m.module.InstallInTestcases() +} + +func (m *moduleContext) InstallInSanitizerDir() bool { + return m.module.InstallInSanitizerDir() +} + +func (m *moduleContext) InstallInRamdisk() bool { + return m.module.InstallInRamdisk() +} + +func (m *moduleContext) InstallInVendorRamdisk() bool { + return m.module.InstallInVendorRamdisk() +} + +func (m *moduleContext) InstallInDebugRamdisk() bool { + return m.module.InstallInDebugRamdisk() +} + +func (m *moduleContext) InstallInRecovery() bool { + return m.module.InstallInRecovery() +} + +func (m *moduleContext) InstallInRoot() bool { + return m.module.InstallInRoot() +} + +func (m *moduleContext) InstallForceOS() (*OsType, *ArchType) { + return m.module.InstallForceOS() +} + +func (m *moduleContext) InstallInVendor() bool { + return m.module.InstallInVendor() +} + +func (m *moduleContext) skipInstall() bool { + if m.module.base().commonProperties.SkipInstall { + return true + } + + if m.module.base().commonProperties.HideFromMake { + return true + } + + // We'll need a solution for choosing which of modules with the same name in different + // namespaces to install. For now, reuse the list of namespaces exported to Make as the + // list of namespaces to install in a Soong-only build. + if !m.module.base().commonProperties.NamespaceExportedToMake { + return true + } + + return false +} + +func (m *moduleContext) InstallFile(installPath InstallPath, name string, srcPath Path, + deps ...InstallPath) InstallPath { + return m.installFile(installPath, name, srcPath, deps, false, nil) +} + +func (m *moduleContext) InstallExecutable(installPath InstallPath, name string, srcPath Path, + deps ...InstallPath) InstallPath { + return m.installFile(installPath, name, srcPath, deps, true, nil) +} + +func (m *moduleContext) InstallFileWithExtraFilesZip(installPath InstallPath, name string, srcPath Path, + extraZip Path, deps ...InstallPath) InstallPath { + return m.installFile(installPath, name, srcPath, deps, false, &extraFilesZip{ + zip: extraZip, + dir: installPath, + }) +} + +func (m *moduleContext) PackageFile(installPath InstallPath, name string, srcPath Path) PackagingSpec { + fullInstallPath := installPath.Join(m, name) + return m.packageFile(fullInstallPath, srcPath, false) +} + +func (m *moduleContext) packageFile(fullInstallPath InstallPath, srcPath Path, executable bool) PackagingSpec { + licenseFiles := m.Module().EffectiveLicenseFiles() + spec := PackagingSpec{ + relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), + srcPath: srcPath, + symlinkTarget: "", + executable: executable, + effectiveLicenseFiles: &licenseFiles, + partition: fullInstallPath.partition, + } + m.packagingSpecs = append(m.packagingSpecs, spec) + return spec +} + +func (m *moduleContext) installFile(installPath InstallPath, name string, srcPath Path, deps []InstallPath, + executable bool, extraZip *extraFilesZip) InstallPath { + + fullInstallPath := installPath.Join(m, name) + m.module.base().hooks.runInstallHooks(m, srcPath, fullInstallPath, false) + + if !m.skipInstall() { + deps = append(deps, InstallPaths(m.module.base().installFilesDepSet.ToList())...) + + var implicitDeps, orderOnlyDeps Paths + + if m.Host() { + // Installed host modules might be used during the build, depend directly on their + // dependencies so their timestamp is updated whenever their dependency is updated + implicitDeps = InstallPaths(deps).Paths() + } else { + orderOnlyDeps = InstallPaths(deps).Paths() + } + + if m.Config().KatiEnabled() { + // When creating the install rule in Soong but embedding in Make, write the rule to a + // makefile instead of directly to the ninja file so that main.mk can add the + // dependencies from the `required` property that are hard to resolve in Soong. + m.katiInstalls = append(m.katiInstalls, katiInstall{ + from: srcPath, + to: fullInstallPath, + implicitDeps: implicitDeps, + orderOnlyDeps: orderOnlyDeps, + executable: executable, + extraFiles: extraZip, + }) + } else { + rule := Cp + if executable { + rule = CpExecutable + } + + extraCmds := "" + if extraZip != nil { + extraCmds += fmt.Sprintf(" && ( unzip -qDD -d '%s' '%s' 2>&1 | grep -v \"zipfile is empty\"; exit $${PIPESTATUS[0]} )", + extraZip.dir.String(), extraZip.zip.String()) + extraCmds += " || ( code=$$?; if [ $$code -ne 0 -a $$code -ne 1 ]; then exit $$code; fi )" + implicitDeps = append(implicitDeps, extraZip.zip) + } + + m.Build(pctx, BuildParams{ + Rule: rule, + Description: "install " + fullInstallPath.Base(), + Output: fullInstallPath, + Input: srcPath, + Implicits: implicitDeps, + OrderOnly: orderOnlyDeps, + Default: !m.Config().KatiEnabled(), + Args: map[string]string{ + "extraCmds": extraCmds, + }, + }) + } + + m.installFiles = append(m.installFiles, fullInstallPath) + } + + m.packageFile(fullInstallPath, srcPath, executable) + + m.checkbuildFiles = append(m.checkbuildFiles, srcPath) + + return fullInstallPath +} + +func (m *moduleContext) InstallSymlink(installPath InstallPath, name string, srcPath InstallPath) InstallPath { + fullInstallPath := installPath.Join(m, name) + m.module.base().hooks.runInstallHooks(m, srcPath, fullInstallPath, true) + + relPath, err := filepath.Rel(path.Dir(fullInstallPath.String()), srcPath.String()) + if err != nil { + panic(fmt.Sprintf("Unable to generate symlink between %q and %q: %s", fullInstallPath.Base(), srcPath.Base(), err)) + } + if !m.skipInstall() { + + if m.Config().KatiEnabled() { + // When creating the symlink rule in Soong but embedding in Make, write the rule to a + // makefile instead of directly to the ninja file so that main.mk can add the + // dependencies from the `required` property that are hard to resolve in Soong. + m.katiSymlinks = append(m.katiSymlinks, katiInstall{ + from: srcPath, + to: fullInstallPath, + }) + } else { + // The symlink doesn't need updating when the target is modified, but we sometimes + // have a dependency on a symlink to a binary instead of to the binary directly, and + // the mtime of the symlink must be updated when the binary is modified, so use a + // normal dependency here instead of an order-only dependency. + m.Build(pctx, BuildParams{ + Rule: Symlink, + Description: "install symlink " + fullInstallPath.Base(), + Output: fullInstallPath, + Input: srcPath, + Default: !m.Config().KatiEnabled(), + Args: map[string]string{ + "fromPath": relPath, + }, + }) + } + + m.installFiles = append(m.installFiles, fullInstallPath) + m.checkbuildFiles = append(m.checkbuildFiles, srcPath) + } + + m.packagingSpecs = append(m.packagingSpecs, PackagingSpec{ + relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), + srcPath: nil, + symlinkTarget: relPath, + executable: false, + partition: fullInstallPath.partition, + }) + + return fullInstallPath +} + +// installPath/name -> absPath where absPath might be a path that is available only at runtime +// (e.g. /apex/...) +func (m *moduleContext) InstallAbsoluteSymlink(installPath InstallPath, name string, absPath string) InstallPath { + fullInstallPath := installPath.Join(m, name) + m.module.base().hooks.runInstallHooks(m, nil, fullInstallPath, true) + + if !m.skipInstall() { + if m.Config().KatiEnabled() { + // When creating the symlink rule in Soong but embedding in Make, write the rule to a + // makefile instead of directly to the ninja file so that main.mk can add the + // dependencies from the `required` property that are hard to resolve in Soong. + m.katiSymlinks = append(m.katiSymlinks, katiInstall{ + absFrom: absPath, + to: fullInstallPath, + }) + } else { + m.Build(pctx, BuildParams{ + Rule: Symlink, + Description: "install symlink " + fullInstallPath.Base() + " -> " + absPath, + Output: fullInstallPath, + Default: !m.Config().KatiEnabled(), + Args: map[string]string{ + "fromPath": absPath, + }, + }) + } + + m.installFiles = append(m.installFiles, fullInstallPath) + } + + m.packagingSpecs = append(m.packagingSpecs, PackagingSpec{ + relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), + srcPath: nil, + symlinkTarget: absPath, + executable: false, + partition: fullInstallPath.partition, + }) + + return fullInstallPath +} + +func (m *moduleContext) CheckbuildFile(srcPath Path) { + m.checkbuildFiles = append(m.checkbuildFiles, srcPath) +} + +func (m *moduleContext) blueprintModuleContext() blueprint.ModuleContext { + return m.bp +} + +func (m *moduleContext) LicenseMetadataFile() Path { + return m.module.base().licenseMetadataFile +} + +// Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must +// be tagged with `android:"path" to support automatic source module dependency resolution. +// +// Deprecated: use PathsForModuleSrc or PathsForModuleSrcExcludes instead. +func (m *moduleContext) ExpandSources(srcFiles, excludes []string) Paths { + return PathsForModuleSrcExcludes(m, srcFiles, excludes) +} + +// Returns a single path expanded from globs and modules referenced using ":module" syntax. The property must +// be tagged with `android:"path" to support automatic source module dependency resolution. +// +// Deprecated: use PathForModuleSrc instead. +func (m *moduleContext) ExpandSource(srcFile, _ string) Path { + return PathForModuleSrc(m, srcFile) +} + +// Returns an optional single path expanded from globs and modules referenced using ":module" syntax if +// the srcFile is non-nil. The property must be tagged with `android:"path" to support automatic source module +// dependency resolution. +func (m *moduleContext) ExpandOptionalSource(srcFile *string, _ string) OptionalPath { + if srcFile != nil { + return OptionalPathForPath(PathForModuleSrc(m, *srcFile)) + } + return OptionalPath{} +} + +func (m *moduleContext) RequiredModuleNames() []string { + return m.module.RequiredModuleNames() +} + +func (m *moduleContext) HostRequiredModuleNames() []string { + return m.module.HostRequiredModuleNames() +} + +func (m *moduleContext) TargetRequiredModuleNames() []string { + return m.module.TargetRequiredModuleNames() +} diff --git a/android/neverallow.go b/android/neverallow.go index 2be6a74f5..f721b945c 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -169,6 +169,8 @@ func createJavaDeviceForHostRules() []Rule { "external/kotlinx.coroutines", "external/robolectric-shadows", "external/robolectric", + "frameworks/base/ravenwood", + "frameworks/base/tools/hoststubgen", "frameworks/layoutlib", } diff --git a/android/paths.go b/android/paths.go index a6cda38f0..37504b62f 100644 --- a/android/paths.go +++ b/android/paths.go @@ -2210,6 +2210,14 @@ type DataPath struct { RelativeInstallPath string } +func (d *DataPath) ToRelativeInstallPath() string { + relPath := d.SrcPath.Rel() + if d.RelativeInstallPath != "" { + relPath = filepath.Join(d.RelativeInstallPath, relPath) + } + return relPath +} + // PathsIfNonNil returns a Paths containing only the non-nil input arguments. func PathsIfNonNil(paths ...Path) Paths { if len(paths) == 0 { diff --git a/android/prebuilt.go b/android/prebuilt.go index e7b79796e..91c0aa182 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -387,7 +387,7 @@ func RegisterPrebuiltsPreArchMutators(ctx RegisterMutatorsContext) { func RegisterPrebuiltsPostDepsMutators(ctx RegisterMutatorsContext) { ctx.BottomUp("prebuilt_source", PrebuiltSourceDepsMutator).Parallel() - ctx.TopDown("prebuilt_select", PrebuiltSelectModuleMutator).Parallel() + ctx.BottomUp("prebuilt_select", PrebuiltSelectModuleMutator).Parallel() ctx.BottomUp("prebuilt_postdeps", PrebuiltPostDepsMutator).Parallel() } @@ -406,6 +406,8 @@ func PrebuiltRenameMutator(ctx BottomUpMutatorContext) { // PrebuiltSourceDepsMutator adds dependencies to the prebuilt module from the // corresponding source module, if one exists for the same variant. +// Add a dependency from the prebuilt to `all_apex_contributions` +// The metadata will be used for source vs prebuilts selection func PrebuiltSourceDepsMutator(ctx BottomUpMutatorContext) { m := ctx.Module() // If this module is a prebuilt, is enabled and has not been renamed to source then add a @@ -416,6 +418,14 @@ func PrebuiltSourceDepsMutator(ctx BottomUpMutatorContext) { ctx.AddReverseDependency(ctx.Module(), PrebuiltDepTag, name) p.properties.SourceExists = true } + // Add a dependency from the prebuilt to the `all_apex_contributions` + // metadata module + // TODO: When all branches contain this singleton module, make this strict + // TODO: Add this dependency only for mainline prebuilts and not every prebuilt module + if ctx.OtherModuleExists("all_apex_contributions") { + ctx.AddDependency(m, acDepTag, "all_apex_contributions") + } + } } @@ -435,7 +445,11 @@ func checkInvariantsForSourceAndPrebuilt(ctx BaseModuleContext, s, p Module) { // PrebuiltSelectModuleMutator marks prebuilts that are used, either overriding source modules or // because the source module doesn't exist. It also disables installing overridden source modules. -func PrebuiltSelectModuleMutator(ctx TopDownMutatorContext) { +// +// If the visited module is the metadata module `all_apex_contributions`, it sets a +// provider containing metadata about whether source or prebuilt of mainline modules should be used. +// This logic was added here to prevent the overhead of creating a new mutator. +func PrebuiltSelectModuleMutator(ctx BottomUpMutatorContext) { m := ctx.Module() if p := GetEmbeddedPrebuilt(m); p != nil { if p.srcsSupplier == nil && p.srcsPropertyName == "" { @@ -444,6 +458,17 @@ func PrebuiltSelectModuleMutator(ctx TopDownMutatorContext) { if !p.properties.SourceExists { p.properties.UsePrebuilt = p.usePrebuilt(ctx, nil, m) } + // Propagate the provider received from `all_apex_contributions` + // to the source module + ctx.VisitDirectDepsWithTag(acDepTag, func(am Module) { + if ctx.Config().Bp2buildMode() { + // This provider key is not applicable in bp2build + return + } + psi := ctx.OtherModuleProvider(am, PrebuiltSelectionInfoProvider).(PrebuiltSelectionInfoMap) + ctx.SetProvider(PrebuiltSelectionInfoProvider, psi) + }) + } else if s, ok := ctx.Module().(Module); ok { ctx.VisitDirectDepsWithTag(PrebuiltDepTag, func(prebuiltModule Module) { p := GetEmbeddedPrebuilt(prebuiltModule) @@ -455,6 +480,11 @@ func PrebuiltSelectModuleMutator(ctx TopDownMutatorContext) { } }) } + // If this is `all_apex_contributions`, set a provider containing + // metadata about source vs prebuilts selection + if am, ok := m.(*allApexContributions); ok { + am.SetPrebuiltSelectionInfoProvider(ctx) + } } // PrebuiltPostDepsMutator replaces dependencies on the source module with dependencies on the @@ -480,9 +510,66 @@ func PrebuiltPostDepsMutator(ctx BottomUpMutatorContext) { } } +// A wrapper around PrebuiltSelectionInfoMap.IsSelected with special handling for java_sdk_library +// java_sdk_library is a macro that creates +// 1. top-level impl library +// 2. stub libraries (suffixed with .stubs...) +// +// java_sdk_library_import is a macro that creates +// 1. top-level "impl" library +// 2. stub libraries (suffixed with .stubs...) +// +// the impl of java_sdk_library_import is a "hook" for hiddenapi and dexpreopt processing. It does not have an impl jar, but acts as a shim +// to provide the jar deapxed from the prebuilt apex +// +// isSelected uses `all_apex_contributions` to supersede source vs prebuilts selection of the stub libraries. It does not supersede the +// selection of the top-level "impl" library so that this hook can work +// +// TODO (b/308174306) - Fix this when we need to support multiple prebuilts in main +func isSelected(psi PrebuiltSelectionInfoMap, m Module) bool { + if sdkLibrary, ok := m.(interface{ SdkLibraryName() *string }); ok && sdkLibrary.SdkLibraryName() != nil { + sln := proptools.String(sdkLibrary.SdkLibraryName()) + // This is the top-level library + // Do not supersede the existing prebuilts vs source selection mechanisms + if sln == m.base().BaseModuleName() { + return false + } + + // Stub library created by java_sdk_library_import + if p := GetEmbeddedPrebuilt(m); p != nil { + return psi.IsSelected(sln, PrebuiltNameFromSource(sln)) + } + + // Stub library created by java_sdk_library + return psi.IsSelected(sln, sln) + } + return psi.IsSelected(m.base().BaseModuleName(), m.Name()) +} + // usePrebuilt returns true if a prebuilt should be used instead of the source module. The prebuilt // will be used if it is marked "prefer" or if the source module is disabled. -func (p *Prebuilt) usePrebuilt(ctx TopDownMutatorContext, source Module, prebuilt Module) bool { +func (p *Prebuilt) usePrebuilt(ctx BaseMutatorContext, source Module, prebuilt Module) bool { + // Use `all_apex_contributions` for source vs prebuilt selection. + psi := PrebuiltSelectionInfoMap{} + ctx.VisitDirectDepsWithTag(PrebuiltDepTag, func(am Module) { + if ctx.OtherModuleHasProvider(am, PrebuiltSelectionInfoProvider) { + psi = ctx.OtherModuleProvider(am, PrebuiltSelectionInfoProvider).(PrebuiltSelectionInfoMap) + } + }) + + // If the source module is explicitly listed in the metadata module, use that + if source != nil && isSelected(psi, source) { + return false + } + // If the prebuilt module is explicitly listed in the metadata module, use that + if isSelected(psi, prebuilt) { + return true + } + + // If the baseModuleName could not be found in the metadata module, + // fall back to the existing source vs prebuilt selection. + // TODO: Drop the fallback mechanisms + if !ctx.Config().Bp2buildMode() { if p.srcsSupplier != nil && len(p.srcsSupplier(ctx, prebuilt)) == 0 { return false diff --git a/android/prebuilt_test.go b/android/prebuilt_test.go index fc47cfd97..953258edd 100644 --- a/android/prebuilt_test.go +++ b/android/prebuilt_test.go @@ -335,6 +335,78 @@ func TestPrebuilts(t *testing.T) { prebuilt: []OsType{Android, buildOS}, }, { + name: "apex_contributions supersedes any source preferred via use_source_config_var", + modules: ` + source { + name: "bar", + } + + prebuilt { + name: "bar", + use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, + srcs: ["prebuilt_file"], + } + apex_contributions { + name: "my_mainline_module_contribution", + api_domain: "apexfoo", + // this metadata module contains prebuilt + contents: ["prebuilt_bar"], + } + all_apex_contributions { + name: "all_apex_contributions", + } + `, + preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { + variables.VendorVars = map[string]map[string]string{ + "acme": { + "use_source": "true", + }, + } + variables.BuildFlags = map[string]string{ + "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": "my_mainline_module_contribution", + } + }), + // use_source_config_var indicates that source should be used + // but this is superseded by `my_mainline_module_contribution` + prebuilt: []OsType{Android, buildOS}, + }, + { + name: "apex_contributions supersedes any prebuilt preferred via use_source_config_var", + modules: ` + source { + name: "bar", + } + + prebuilt { + name: "bar", + use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, + srcs: ["prebuilt_file"], + } + apex_contributions { + name: "my_mainline_module_contribution", + api_domain: "apexfoo", + // this metadata module contains source + contents: ["bar"], + } + all_apex_contributions { + name: "all_apex_contributions", + } + `, + preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { + variables.VendorVars = map[string]map[string]string{ + "acme": { + "use_source": "false", + }, + } + variables.BuildFlags = map[string]string{ + "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": "my_mainline_module_contribution", + } + }), + // use_source_config_var indicates that prebuilt should be used + // but this is superseded by `my_mainline_module_contribution` + prebuilt: nil, + }, + { name: "prebuilt use_source_config_var={acme, use_source} - acme_use_source=true", modules: ` source { @@ -497,7 +569,7 @@ func TestPrebuilts(t *testing.T) { } } -func testPrebuiltError(t *testing.T, expectedError, bp string) { +func testPrebuiltErrorWithFixture(t *testing.T, expectedError, bp string, fixture FixturePreparer) { t.Helper() fs := MockFS{ "prebuilt_file": nil, @@ -508,9 +580,15 @@ func testPrebuiltError(t *testing.T, expectedError, bp string) { PrepareForTestWithOverrides, fs.AddToFixture(), FixtureRegisterWithContext(registerTestPrebuiltModules), + OptionalFixturePreparer(fixture), ). ExtendWithErrorHandler(FixtureExpectsAtLeastOneErrorMatchingPattern(expectedError)). RunTestWithBp(t, bp) + +} + +func testPrebuiltError(t *testing.T, expectedError, bp string) { + testPrebuiltErrorWithFixture(t, expectedError, bp, nil) } func TestPrebuiltShouldNotChangePartition(t *testing.T) { @@ -559,6 +637,7 @@ func registerTestPrebuiltModules(ctx RegistrationContext) { ctx.RegisterModuleType("soong_config_module_type", SoongConfigModuleTypeFactory) ctx.RegisterModuleType("soong_config_string_variable", SoongConfigStringVariableDummyFactory) ctx.RegisterModuleType("soong_config_bool_variable", SoongConfigBoolVariableDummyFactory) + RegisterApexContributionsBuildComponents(ctx) } type prebuiltModule struct { @@ -653,3 +732,33 @@ func newOverrideSourceModule() Module { InitOverrideModule(m) return m } + +func TestPrebuiltErrorCannotListBothSourceAndPrebuiltInContributions(t *testing.T) { + selectMainlineModuleContritbutions := GroupFixturePreparers( + FixtureModifyProductVariables(func(variables FixtureProductVariables) { + variables.BuildFlags = map[string]string{ + "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": "my_apex_contributions", + } + }), + ) + testPrebuiltErrorWithFixture(t, `Cannot use Soong module: prebuilt_foo from apex_contributions: my_apex_contributions because it has been added previously as: foo from apex_contributions: my_apex_contributions`, ` + source { + name: "foo", + } + prebuilt { + name: "foo", + srcs: ["prebuilt_file"], + } + apex_contributions { + name: "my_apex_contributions", + api_domain: "my_mainline_module", + contents: [ + "foo", + "prebuilt_foo", + ], + } + all_apex_contributions { + name: "all_apex_contributions", + } + `, selectMainlineModuleContritbutions) +} diff --git a/android/util_test.go b/android/util_test.go index 20161e52d..699135bfc 100644 --- a/android/util_test.go +++ b/android/util_test.go @@ -811,7 +811,7 @@ func TestReverseSlice(t *testing.T) { if !reflect.DeepEqual(slice, testCase.expected) { t.Errorf("expected %#v, got %#v", testCase.expected, slice) } - if slice != nil && unsafe.SliceData(testCase.in) == unsafe.SliceData(slice) { + if cap(slice) > 0 && unsafe.SliceData(testCase.in) == unsafe.SliceData(slice) { t.Errorf("expected slices to have different backing arrays") } }) diff --git a/android/variable.go b/android/variable.go index 648e4cf18..fe3a6d790 100644 --- a/android/variable.go +++ b/android/variable.go @@ -494,6 +494,8 @@ type ProductVariables struct { Release_expose_flagged_api *bool `json:",omitempty"` BuildFlags map[string]string `json:",omitempty"` + + BuildFromSourceStub *bool `json:",omitempty"` } type PartitionQualifiedVariablesType struct { diff --git a/apex/apex.go b/apex/apex.go index f2e8a0623..ecc794b5b 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -482,9 +482,6 @@ type apexBundle struct { nativeApisUsedByModuleFile android.ModuleOutPath nativeApisBackedByModuleFile android.ModuleOutPath javaApisUsedByModuleFile android.ModuleOutPath - - // Collect the module directory for IDE info in java/jdeps.go. - modulePaths []string } // apexFileClass represents a type of file that can be included in APEX. @@ -1893,7 +1890,7 @@ func (a *apexBundle) ProcessBazelQueryResponse(ctx android.ModuleContext) { installSuffix = imageCapexSuffix } a.installedFile = ctx.InstallFile(a.installDir, a.Name()+installSuffix, a.outputFile, - a.compatSymlinks.Paths()...) + a.compatSymlinks...) // filesInfo in mixed mode must retrieve all information about the apex's // contents completely from the Starlark providers. It should never rely on @@ -2406,8 +2403,6 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { } //////////////////////////////////////////////////////////////////////////////////////////// // 2) traverse the dependency tree to collect apexFile structs from them. - // Collect the module directory for IDE info in java/jdeps.go. - a.modulePaths = append(a.modulePaths, ctx.ModuleDir()) // TODO(jiyong): do this using WalkPayloadDeps // TODO(jiyong): make this clean!!! @@ -2972,7 +2967,6 @@ func (a *apexBundle) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Deps = append(dpInfo.Deps, a.properties.Java_libs...) dpInfo.Deps = append(dpInfo.Deps, a.properties.Bootclasspath_fragments...) dpInfo.Deps = append(dpInfo.Deps, a.properties.Systemserverclasspath_fragments...) - dpInfo.Paths = append(dpInfo.Paths, a.modulePaths...) } var ( @@ -3037,59 +3031,6 @@ func makeApexAvailableBaseline() map[string][]string { // // Module separator // - m["com.android.appsearch"] = []string{ - "icing-java-proto-lite", - } - // - // Module separator - // - m["com.android.btservices"] = []string{ - // empty - } - // - // Module separator - // - m["com.android.cellbroadcast"] = []string{} - // - // Module separator - // - m["com.android.extservices"] = []string{ - "ExtServices-core", - "libtextclassifier-java", - "textclassifier-statsd", - "TextClassifierNotificationLibNoManifest", - "TextClassifierServiceLibNoManifest", - } - // - // Module separator - // - m["com.android.neuralnetworks"] = []string{ - "android.hardware.neuralnetworks@1.0", - "android.hardware.neuralnetworks@1.1", - "android.hardware.neuralnetworks@1.2", - "android.hardware.neuralnetworks@1.3", - "android.hidl.allocator@1.0", - "android.hidl.memory.token@1.0", - "android.hidl.memory@1.0", - "android.hidl.safe_union@1.0", - "libarect", - "libprocpartition", - } - // - // Module separator - // - m["com.android.media"] = []string{ - // empty - } - // - // Module separator - // - m["com.android.media.swcodec"] = []string{ - // empty - } - // - // Module separator - // m["com.android.mediaprovider"] = []string{ "MediaProvider", "MediaProviderGoogle", diff --git a/apex/builder.go b/apex/builder.go index afbfa1cf9..3f358acd4 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -344,10 +344,12 @@ func (a *apexBundle) buildManifest(ctx android.ModuleContext, provideNativeLibs, func (a *apexBundle) buildFileContexts(ctx android.ModuleContext) android.OutputPath { var fileContexts android.Path var fileContextsDir string + isFileContextsModule := false if a.properties.File_contexts == nil { fileContexts = android.PathForSource(ctx, "system/sepolicy/apex", ctx.ModuleName()+"-file_contexts") } else { if m, t := android.SrcIsModuleWithTag(*a.properties.File_contexts); m != "" { + isFileContextsModule = true otherModule := android.GetModuleFromPathDep(ctx, m, t) fileContextsDir = ctx.OtherModuleDir(otherModule) } @@ -363,7 +365,7 @@ func (a *apexBundle) buildFileContexts(ctx android.ModuleContext) android.Output ctx.PropertyErrorf("file_contexts", "should be under system/sepolicy, but found in %q", fileContextsDir) } } - if !android.ExistentPathForSource(ctx, fileContexts.String()).Valid() { + if !isFileContextsModule && !android.ExistentPathForSource(ctx, fileContexts.String()).Valid() { ctx.PropertyErrorf("file_contexts", "cannot find file_contexts file: %q", fileContexts.String()) } @@ -372,12 +374,12 @@ func (a *apexBundle) buildFileContexts(ctx android.ModuleContext) android.Output output := android.PathForModuleOut(ctx, "file_contexts") rule := android.NewRuleBuilder(pctx, ctx) - forceLabel := "u:object_r:system_file:s0" + labelForRoot := "u:object_r:system_file:s0" + labelForManifest := "u:object_r:system_file:s0" if a.SocSpecific() && !a.vndkApex { - // APEX on /vendor should label ./ and ./apex_manifest.pb as vendor_apex_metadata_file. - // The reason why we skip VNDK APEX is that aosp_{pixel device} targets install VNDK APEX on /vendor - // even though VNDK APEX is supposed to be installed on /system. (See com.android.vndk.current.on_vendor) - forceLabel = "u:object_r:vendor_apex_metadata_file:s0" + // APEX on /vendor should label ./ and ./apex_manifest.pb as vendor file. + labelForRoot = "u:object_r:vendor_file:s0" + labelForManifest = "u:object_r:vendor_apex_metadata_file:s0" } // remove old file rule.Command().Text("rm").FlagWithOutput("-f ", output) @@ -387,8 +389,8 @@ func (a *apexBundle) buildFileContexts(ctx android.ModuleContext) android.Output rule.Command().Text("echo").Text(">>").Output(output) if !useFileContextsAsIs { // force-label /apex_manifest.pb and / - rule.Command().Text("echo").Text("/apex_manifest\\\\.pb").Text(forceLabel).Text(">>").Output(output) - rule.Command().Text("echo").Text("/").Text(forceLabel).Text(">>").Output(output) + rule.Command().Text("echo").Text("/apex_manifest\\\\.pb").Text(labelForManifest).Text(">>").Output(output) + rule.Command().Text("echo").Text("/").Text(labelForRoot).Text(">>").Output(output) } rule.Build("file_contexts."+a.Name(), "Generate file_contexts") @@ -563,13 +565,8 @@ func (a *apexBundle) buildApex(ctx android.ModuleContext) { // Copy the test files (if any) for _, d := range fi.dataPaths { // TODO(eakammer): This is now the third repetition of ~this logic for test paths, refactoring should be possible - relPath := d.SrcPath.Rel() - dataPath := d.SrcPath.String() - if !strings.HasSuffix(dataPath, relPath) { - panic(fmt.Errorf("path %q does not end with %q", dataPath, relPath)) - } - - dataDest := imageDir.Join(ctx, fi.apexRelativePath(relPath), d.RelativeInstallPath).String() + relPath := d.ToRelativeInstallPath() + dataDest := imageDir.Join(ctx, fi.apexRelativePath(relPath)).String() copyCommands = append(copyCommands, "cp -f "+d.SrcPath.String()+" "+dataDest) implicitInputs = append(implicitInputs, d.SrcPath) @@ -956,7 +953,7 @@ func (a *apexBundle) buildApex(ctx android.ModuleContext) { // Install to $OUT/soong/{target,host}/.../apex. a.installedFile = ctx.InstallFile(a.installDir, a.Name()+installSuffix, a.outputFile, - a.compatSymlinks.Paths()...) + a.compatSymlinks...) // installed-files.txt is dist'ed a.installedFilesFile = a.buildInstalledFilesFile(ctx, a.outputFile, imageDir) @@ -1095,7 +1092,8 @@ func (a *apexBundle) buildCannedFsConfig(ctx android.ModuleContext) android.Outp if f.installDir == "bin" || strings.HasPrefix(f.installDir, "bin/") { executablePaths = append(executablePaths, pathInApex) for _, d := range f.dataPaths { - readOnlyPaths = append(readOnlyPaths, filepath.Join(f.installDir, d.RelativeInstallPath, d.SrcPath.Rel())) + rel := d.ToRelativeInstallPath() + readOnlyPaths = append(readOnlyPaths, filepath.Join(f.installDir, rel)) } for _, s := range f.symlinks { executablePaths = append(executablePaths, filepath.Join(f.installDir, s)) diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 7a9d23e54..7d339d54e 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -794,7 +794,7 @@ func (p *Prebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) { } if p.installable() { - p.installedFile = ctx.InstallFile(p.installDir, p.installFilename, p.inputApex, p.compatSymlinks.Paths()...) + p.installedFile = ctx.InstallFile(p.installDir, p.installFilename, p.inputApex, p.compatSymlinks...) p.provenanceMetaDataFile = provenance.GenerateArtifactProvenanceMetaData(ctx, p.inputApex, p.installedFile) } } diff --git a/bp2build/Android.bp b/bp2build/Android.bp index 14e32edd9..64ee01f88 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -6,6 +6,7 @@ bootstrap_go_package { name: "soong-bp2build", pkgPath: "android/soong/bp2build", srcs: [ + "aconfig_conversion_test.go", "androidbp_to_build_templates.go", "bp2build.go", "bp2build_product_config.go", @@ -21,6 +22,7 @@ bootstrap_go_package { deps: [ "blueprint-bootstrap", "soong-aidl-library", + "soong-aconfig", "soong-android", "soong-android-allowlists", "soong-android-soongconfig", diff --git a/bp2build/aconfig_conversion_test.go b/bp2build/aconfig_conversion_test.go index ca41680a6..d6e20df12 100644 --- a/bp2build/aconfig_conversion_test.go +++ b/bp2build/aconfig_conversion_test.go @@ -156,7 +156,7 @@ func TestJavaAconfigLibrary(t *testing.T) { name: "foo", aconfig_declarations: "foo_aconfig_declarations", libs: ["foo_java_library"], - test: true, + mode: "test", } ` expectedBazelTargets := []string{ @@ -184,7 +184,6 @@ func TestJavaAconfigLibrary(t *testing.T) { AttrNameToString{ "aconfig_declarations": `":foo_aconfig_declarations"`, "libs": `[":foo_java_library-neverlink"]`, - "test": `True`, "sdk_version": `"system_current"`, "target_compatible_with": `["//build/bazel_common_rules/platforms/os:android"]`, }, @@ -213,7 +212,7 @@ func TestJavaAconfigLibraryAsTaggedOutput(t *testing.T) { java_aconfig_library { name: "foo_aconfig_library", aconfig_declarations: "foo_aconfig_declarations", - test: true, + mode: "test", } ` expectedBazelTargets := []string{ @@ -230,7 +229,6 @@ func TestJavaAconfigLibraryAsTaggedOutput(t *testing.T) { "foo_aconfig_library", AttrNameToString{ "aconfig_declarations": `":foo_aconfig_declarations"`, - "test": `True`, "sdk_version": `"system_current"`, "target_compatible_with": `["//build/bazel_common_rules/platforms/os:android"]`, }, diff --git a/cc/Android.bp b/cc/Android.bp index 8fa0fbeb0..77e96db35 100644 --- a/cc/Android.bp +++ b/cc/Android.bp @@ -19,6 +19,7 @@ bootstrap_go_package { "soong-multitree", "soong-snapshot", "soong-sysprop-bp2build", + "soong-testing", "soong-tradefed", ], srcs: [ diff --git a/cc/afdo.go b/cc/afdo.go index ac210d44a..91cf0b8a2 100644 --- a/cc/afdo.go +++ b/cc/afdo.go @@ -35,7 +35,7 @@ var ( var afdoProfileProjectsConfigKey = android.NewOnceKey("AfdoProfileProjects") // This flag needs to be in both CFlags and LdFlags to ensure correct symbol ordering -const afdoFlagsFormat = "-fprofile-sample-use=%s" +const afdoFlagsFormat = "-fprofile-sample-use=%s -fprofile-sample-accurate" func recordMissingAfdoProfileFile(ctx android.BaseModuleContext, missing string) { getNamedMapForConfig(ctx.Config(), modulesMissingProfileFileKey).Store(missing, true) diff --git a/cc/builder.go b/cc/builder.go index 3f582fa85..69cf75bdd 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -681,16 +681,11 @@ func transformSourceToObj(ctx ModuleContext, subdir string, srcFiles, noTidySrcs tidyCmd := "${config.ClangBin}/clang-tidy" rule := clangTidy - reducedCFlags := moduleFlags if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_CLANG_TIDY") { rule = clangTidyRE - // b/248371171, work around RBE input processor problem - // some cflags rejected by input processor, but usually - // do not affect included files or clang-tidy - reducedCFlags = config.TidyReduceCFlags(reducedCFlags) } - sharedCFlags := shareFlags("cFlags", reducedCFlags) + sharedCFlags := shareFlags("cFlags", moduleFlags) srcRelPath := srcFile.Rel() // Add the .tidy rule @@ -24,6 +24,7 @@ import ( "strconv" "strings" + "android/soong/testing" "android/soong/ui/metrics/bp2build_metrics_proto" "github.com/google/blueprint" @@ -862,9 +863,10 @@ type Module struct { Properties BaseProperties // initialize before calling Init - hod android.HostOrDeviceSupported - multilib android.Multilib - bazelable bool + hod android.HostOrDeviceSupported + multilib android.Multilib + bazelable bool + testModule bool // Allowable SdkMemberTypes of this module type. sdkMemberTypes []android.SdkMemberType @@ -2329,6 +2331,9 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { } } } + if c.testModule { + ctx.SetProvider(testing.TestModuleProviderKey, testing.TestModuleProviderData{}) + } c.maybeInstall(ctx, apexInfo) } diff --git a/cc/cc_test.go b/cc/cc_test.go index 794c5ee51..e2dba9043 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -720,7 +720,7 @@ func TestDataLibs(t *testing.T) { return } if len(testBinary.dataPaths()) != 1 { - t.Errorf("expected exactly one test data file. test data files: [%s]", testBinary.dataPaths()) + t.Errorf("expected exactly one test data file. test data files: [%v]", testBinary.dataPaths()) return } @@ -777,7 +777,7 @@ func TestDataLibsRelativeInstallPath(t *testing.T) { t.Fatalf("expected exactly one output file. output files: [%s]", outputFiles) } if len(testBinary.dataPaths()) != 2 { - t.Fatalf("expected exactly one test data file. test data files: [%s]", testBinary.dataPaths()) + t.Fatalf("expected exactly one test data file. test data files: [%v]", testBinary.dataPaths()) } outputPath := outputFiles[0].String() @@ -3332,7 +3332,7 @@ func TestDataLibsPrebuiltSharedTestLibrary(t *testing.T) { t.Errorf("expected exactly one output file. output files: [%s]", outputFiles) } if len(testBinary.dataPaths()) != 1 { - t.Errorf("expected exactly one test data file. test data files: [%s]", testBinary.dataPaths()) + t.Errorf("expected exactly one test data file. test data files: [%v]", testBinary.dataPaths()) } outputPath := outputFiles[0].String() diff --git a/cc/config/tidy.go b/cc/config/tidy.go index efa45495b..b40557a29 100644 --- a/cc/config/tidy.go +++ b/cc/config/tidy.go @@ -16,7 +16,6 @@ package config import ( "android/soong/android" - "regexp" "strings" ) @@ -281,11 +280,3 @@ func TidyFlagsForSrcFile(srcFile android.Path, flags string) string { } return flags } - -var ( - removedCFlags = regexp.MustCompile(" -fsanitize=[^ ]*memtag-[^ ]* ") -) - -func TidyReduceCFlags(flags string) string { - return removedCFlags.ReplaceAllString(flags, " ") -} diff --git a/cc/config/x86_64_device.go b/cc/config/x86_64_device.go index 9f093bb90..00a395f78 100644 --- a/cc/config/x86_64_device.go +++ b/cc/config/x86_64_device.go @@ -33,6 +33,8 @@ var ( "-Wl,--hash-style=gnu", } + X86_64Lldflags = x86_64Ldflags + x86_64ArchVariantCflags = map[string][]string{ "": []string{ "-march=x86-64", @@ -94,10 +96,23 @@ func init() { exportedVars.ExportStringListStaticVariable("X86_64ToolchainLdflags", []string{"-m64"}) exportedVars.ExportStringListStaticVariable("X86_64Ldflags", x86_64Ldflags) - exportedVars.ExportStringListStaticVariable("X86_64Lldflags", x86_64Ldflags) + exportedVars.ExportStringList("X86_64Lldflags", X86_64Lldflags) + pctx.VariableFunc("X86_64Lldflags", func(ctx android.PackageVarContext) string { + maxPageSizeFlag := "-Wl,-z,max-page-size=" + ctx.Config().MaxPageSizeSupported() + flags := append(X86_64Lldflags, maxPageSizeFlag) + return strings.Join(flags, " ") + }) // Clang cflags - exportedVars.ExportStringListStaticVariable("X86_64Cflags", x86_64Cflags) + exportedVars.ExportStringList("X86_64Cflags", x86_64Cflags) + pctx.VariableFunc("X86_64Cflags", func(ctx android.PackageVarContext) string { + flags := x86_64Cflags + if ctx.Config().PageSizeAgnostic() { + flags = append(flags, "-D__BIONIC_NO_PAGE_SIZE_MACRO") + } + return strings.Join(flags, " ") + }) + exportedVars.ExportStringListStaticVariable("X86_64Cppflags", x86_64Cppflags) // Yasm flags diff --git a/cc/fuzz.go b/cc/fuzz.go index df9f21ad2..8fc48983e 100644 --- a/cc/fuzz.go +++ b/cc/fuzz.go @@ -96,6 +96,7 @@ func fuzzMutatorDeps(mctx android.TopDownMutatorContext) { // your device, or $ANDROID_PRODUCT_OUT/data/fuzz in your build tree. func LibFuzzFactory() android.Module { module := NewFuzzer(android.HostAndDeviceSupported) + module.testModule = true return module.Init() } @@ -147,10 +147,11 @@ func (lto *lto) flags(ctx BaseModuleContext, flags Flags) Flags { } } - // Register allocation MLGO flags for ARM64. - if ctx.Arch().ArchType == android.Arm64 { - ltoCFlags = append(ltoCFlags, "-mllvm -regalloc-enable-advisor=release") - ltoLdFlags = append(ltoLdFlags, "-Wl,-mllvm,-regalloc-enable-advisor=release") + if !ctx.Config().IsEnvFalse("THINLTO_USE_MLGO") { + // Register allocation MLGO flags for ARM64. + if ctx.Arch().ArchType == android.Arm64 { + ltoLdFlags = append(ltoLdFlags, "-Wl,-mllvm,-regalloc-enable-advisor=release") + } // Flags for training MLGO model. if ctx.Config().IsEnvTrue("THINLTO_EMIT_INDEXES_AND_IMPORTS") { ltoLdFlags = append(ltoLdFlags, "-Wl,--save-temps=import") diff --git a/cc/test.go b/cc/test.go index 5b778dc87..a4224c34a 100644 --- a/cc/test.go +++ b/cc/test.go @@ -158,6 +158,7 @@ func TestLibraryFactory() android.Module { // binary. func BenchmarkFactory() android.Module { module := NewBenchmark(android.HostAndDeviceSupported) + module.testModule = true return module.Init() } @@ -489,6 +490,7 @@ func NewTest(hod android.HostOrDeviceSupported, bazelable bool) *Module { module, binary := newBinary(hod, bazelable) module.bazelable = bazelable module.multilib = android.MultilibBoth + module.testModule = true binary.baseInstaller = NewTestInstaller() test := &testBinary{ diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 568a6f822..5abbdb7d5 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -84,7 +84,7 @@ func init() { flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules") flag.BoolVar(&cmdlineArgs.BazelModeStaging, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules") flag.BoolVar(&cmdlineArgs.UseBazelProxy, "use-bazel-proxy", false, "communicate with bazel using unix socket proxy instead of spawning subprocesses") - flag.BoolVar(&cmdlineArgs.BuildFromTextStub, "build-from-text-stub", false, "build Java stubs from API text files instead of source files") + flag.BoolVar(&cmdlineArgs.BuildFromSourceStub, "build-from-source-stub", false, "build Java stubs from source files instead of API text files") flag.BoolVar(&cmdlineArgs.EnsureAllowlistIntegrity, "ensure-allowlist-integrity", false, "verify that allowlisted modules are mixed-built") // Flags that probably shouldn't be flags of soong_build, but we haven't found // the time to remove them yet diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index 34b158963..c94ff0721 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -692,9 +692,11 @@ func setMaxFiles(ctx build.Context) { } ctx.Verbosef("Current file limits: %d soft, %d hard", limits.Cur, limits.Max) - if limits.Cur == limits.Max { - return - } + + // Go 1.21 modifies the file limit but restores the original when + // execing subprocesses if it hasn't be overridden. Call Setrlimit + // here even if it doesn't appear to be necessary so that the + // syscall package considers it set. limits.Cur = limits.Max err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits) diff --git a/dexpreopt/config.go b/dexpreopt/config.go index ba41f4a66..c871e85bf 100644 --- a/dexpreopt/config.go +++ b/dexpreopt/config.go @@ -32,7 +32,7 @@ type GlobalConfig struct { DisablePreoptBootImages bool // disable prepot for boot images DisablePreoptModules []string // modules with preopt disabled by product-specific config - OnlyPreoptBootImageAndSystemServer bool // only preopt jars in the boot image or system server + OnlyPreoptArtBootImage bool // only preopt jars in the ART boot image PreoptWithUpdatableBcp bool // If updatable boot jars are included in dexpreopt or not. @@ -691,45 +691,45 @@ func (s *globalSoongConfigSingleton) MakeVars(ctx android.MakeVarsContext) { func GlobalConfigForTests(ctx android.PathContext) *GlobalConfig { return &GlobalConfig{ - DisablePreopt: false, - DisablePreoptModules: nil, - OnlyPreoptBootImageAndSystemServer: false, - HasSystemOther: false, - PatternsOnSystemOther: nil, - DisableGenerateProfile: false, - ProfileDir: "", - BootJars: android.EmptyConfiguredJarList(), - ApexBootJars: android.EmptyConfiguredJarList(), - ArtApexJars: android.EmptyConfiguredJarList(), - TestOnlyArtBootImageJars: android.EmptyConfiguredJarList(), - SystemServerJars: android.EmptyConfiguredJarList(), - SystemServerApps: nil, - ApexSystemServerJars: android.EmptyConfiguredJarList(), - StandaloneSystemServerJars: android.EmptyConfiguredJarList(), - ApexStandaloneSystemServerJars: android.EmptyConfiguredJarList(), - SpeedApps: nil, - PreoptFlags: nil, - DefaultCompilerFilter: "", - SystemServerCompilerFilter: "", - GenerateDMFiles: false, - NoDebugInfo: false, - DontResolveStartupStrings: false, - AlwaysSystemServerDebugInfo: false, - NeverSystemServerDebugInfo: false, - AlwaysOtherDebugInfo: false, - NeverOtherDebugInfo: false, - IsEng: false, - SanitizeLite: false, - DefaultAppImages: false, - Dex2oatXmx: "", - Dex2oatXms: "", - EmptyDirectory: "empty_dir", - CpuVariant: nil, - InstructionSetFeatures: nil, - BootImageProfiles: nil, - BootFlags: "", - Dex2oatImageXmx: "", - Dex2oatImageXms: "", + DisablePreopt: false, + DisablePreoptModules: nil, + OnlyPreoptArtBootImage: false, + HasSystemOther: false, + PatternsOnSystemOther: nil, + DisableGenerateProfile: false, + ProfileDir: "", + BootJars: android.EmptyConfiguredJarList(), + ApexBootJars: android.EmptyConfiguredJarList(), + ArtApexJars: android.EmptyConfiguredJarList(), + TestOnlyArtBootImageJars: android.EmptyConfiguredJarList(), + SystemServerJars: android.EmptyConfiguredJarList(), + SystemServerApps: nil, + ApexSystemServerJars: android.EmptyConfiguredJarList(), + StandaloneSystemServerJars: android.EmptyConfiguredJarList(), + ApexStandaloneSystemServerJars: android.EmptyConfiguredJarList(), + SpeedApps: nil, + PreoptFlags: nil, + DefaultCompilerFilter: "", + SystemServerCompilerFilter: "", + GenerateDMFiles: false, + NoDebugInfo: false, + DontResolveStartupStrings: false, + AlwaysSystemServerDebugInfo: false, + NeverSystemServerDebugInfo: false, + AlwaysOtherDebugInfo: false, + NeverOtherDebugInfo: false, + IsEng: false, + SanitizeLite: false, + DefaultAppImages: false, + Dex2oatXmx: "", + Dex2oatXms: "", + EmptyDirectory: "empty_dir", + CpuVariant: nil, + InstructionSetFeatures: nil, + BootImageProfiles: nil, + BootFlags: "", + Dex2oatImageXmx: "", + Dex2oatImageXms: "", } } diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index 29ae188cd..c13e14ad2 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -124,12 +124,7 @@ func dexpreoptDisabled(ctx android.PathContext, global *GlobalConfig, module *Mo return true } - // If OnlyPreoptBootImageAndSystemServer=true and module is not in boot class path skip - // Also preopt system server jars since selinux prevents system server from loading anything from - // /data. If we don't do this they will need to be extracted which is not favorable for RAM usage - // or performance. If PreoptExtractedApk is true, we ignore the only preopt boot image options. - if global.OnlyPreoptBootImageAndSystemServer && !global.BootJars.ContainsJar(module.Name) && - !global.AllSystemServerJars(ctx).ContainsJar(module.Name) && !module.PreoptExtractedApk { + if global.OnlyPreoptArtBootImage && !module.PreoptExtractedApk { return true } diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index 275d9c3ff..b7d29710d 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -150,8 +150,9 @@ type PrebuiltEtc struct { sourceFilePath android.Path outputFilePath android.OutputPath // The base install location, e.g. "etc" for prebuilt_etc, "usr/share" for prebuilt_usr_share. - installDirBase string - installDirBase64 string + installDirBase string + installDirBase64 string + installAvoidMultilibConflict bool // The base install location when soc_specific property is set to true, e.g. "firmware" for // prebuilt_firmware. socInstallDirBase string @@ -355,6 +356,10 @@ func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) { if p.SocSpecific() && p.socInstallDirBase != "" { installBaseDir = p.socInstallDirBase } + if p.installAvoidMultilibConflict && !ctx.Host() && ctx.Config().HasMultilibConflict(ctx.Arch().ArchType) { + installBaseDir = filepath.Join(installBaseDir, ctx.Arch().ArchType.String()) + } + p.installDirPath = android.PathForModuleInstall(ctx, installBaseDir, p.SubDir()) // Call InstallFile even when uninstallable to make the module included in the package @@ -590,6 +595,7 @@ func PrebuiltRenderScriptBitcodeFactory() android.Module { module := &PrebuiltEtc{} module.makeClass = "RENDERSCRIPT_BITCODE" module.installDirBase64 = "lib64" + module.installAvoidMultilibConflict = true InitPrebuiltEtcModule(module, "lib") // This module is device-only android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibBoth) diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index f2efd463f..3d491145b 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -350,13 +350,16 @@ func (f *filesystem) buildPropFile(ctx android.ModuleContext) (propFile android. addStr("avb_algorithm", algorithm) key := android.PathForModuleSrc(ctx, proptools.String(f.properties.Avb_private_key)) addPath("avb_key_path", key) + partitionName := proptools.StringDefault(f.properties.Partition_name, f.Name()) + addStr("partition_name", partitionName) avb_add_hashtree_footer_args := "--do_not_generate_fec" if hashAlgorithm := proptools.String(f.properties.Avb_hash_algorithm); hashAlgorithm != "" { avb_add_hashtree_footer_args += " --hash_algorithm " + hashAlgorithm } + securityPatchKey := "com.android.build." + partitionName + ".security_patch" + securityPatchValue := ctx.Config().PlatformSecurityPatch() + avb_add_hashtree_footer_args += " --prop " + securityPatchKey + ":" + securityPatchValue addStr("avb_add_hashtree_footer_args", avb_add_hashtree_footer_args) - partitionName := proptools.StringDefault(f.properties.Partition_name, f.Name()) - addStr("partition_name", partitionName) addStr("avb_salt", f.salt()) } diff --git a/genrule/allowlists.go b/genrule/allowlists.go index 8617de8a5..01b5666cf 100644 --- a/genrule/allowlists.go +++ b/genrule/allowlists.go @@ -18,31 +18,6 @@ var ( DepfileAllowList = []string{ // go/keep-sorted start "depfile_allowed_for_test", - "gen_uwb_core_proto", - "libtextclassifier_fbgen_actions_actions-entity-data", - "libtextclassifier_fbgen_actions_actions_model", - "libtextclassifier_fbgen_annotator_datetime_datetime", - "libtextclassifier_fbgen_annotator_entity-data", - "libtextclassifier_fbgen_annotator_experimental_experimental", - "libtextclassifier_fbgen_annotator_model", - "libtextclassifier_fbgen_annotator_person_name_person_name_model", - "libtextclassifier_fbgen_lang_id_common_flatbuffers_embedding-network", - "libtextclassifier_fbgen_lang_id_common_flatbuffers_model", - "libtextclassifier_fbgen_utils_codepoint-range", - "libtextclassifier_fbgen_utils_container_bit-vector", - "libtextclassifier_fbgen_utils_flatbuffers_flatbuffers", - "libtextclassifier_fbgen_utils_flatbuffers_flatbuffers_test", - "libtextclassifier_fbgen_utils_grammar_rules", - "libtextclassifier_fbgen_utils_grammar_semantics_expression", - "libtextclassifier_fbgen_utils_grammar_testing_value", - "libtextclassifier_fbgen_utils_i18n_language-tag", - "libtextclassifier_fbgen_utils_intents_intent-config", - "libtextclassifier_fbgen_utils_lua_utils_tests", - "libtextclassifier_fbgen_utils_normalization", - "libtextclassifier_fbgen_utils_resources", - "libtextclassifier_fbgen_utils_tflite_text_encoder_config", - "libtextclassifier_fbgen_utils_tokenizer", - "libtextclassifier_fbgen_utils_zlib_buffer", "tflite_support_metadata_schema", "tflite_support_spm_config", "tflite_support_spm_encoder_config", @@ -51,26 +26,7 @@ var ( SandboxingDenyModuleList = []string{ // go/keep-sorted start - "CompilationTestCases_package-dex-usage", - "ControlEnvProxyServerProto_cc", - "ControlEnvProxyServerProto_h", "CtsApkVerityTestDebugFiles", - "FrontendStub_cc", - "FrontendStub_h", - "HeadlessBuildTimestamp", - "ImageProcessing-rscript", - "ImageProcessing2-rscript", - "ImageProcessingJB-rscript", - "MultiDexLegacyTestApp_genrule", - "PackageManagerServiceServerTests_apks_as_resources", - "PacketStreamerStub_cc", - "PacketStreamerStub_h", - "RSTest-rscript", - "RSTest_v11-rscript", - "RSTest_v14-rscript", - "RSTest_v16-rscript", - "Refocus-rscript", - "RsBalls-rscript", "ScriptGroupTest-rscript", "TracingVMProtoStub_cc", "TracingVMProtoStub_h", @@ -82,110 +38,26 @@ var ( "VehicleServerProtoStub_h@default-grpc", "aidl-golden-test-build-hook-gen", "aidl_camera_build_version", - "android-cts-verifier", - "android-support-multidex-instrumentation-version", - "android-support-multidex-version", - "angle_commit_id", - "apexer_test_host_tools", - "atest_integration_fake_src", - "authfs_test_apk_assets", - "awkgram.tab.h", - "bluetooth_core_rust_packets", - "c2hal_test_genc++", - "c2hal_test_genc++_headers", "camera-its", "checkIn-service-stub-lite", "chre_atoms_log.h", - "common-profile-text-protos", - "core-tests-smali-dex", "cronet_aml_base_android_runtime_jni_headers", "cronet_aml_base_android_runtime_jni_headers__testing", "cronet_aml_base_android_runtime_unchecked_jni_headers", "cronet_aml_base_android_runtime_unchecked_jni_headers__testing", "deqp_spvtools_update_build_version", - "egl_extensions_functions_hdr", - "egl_functions_hdr", - "emp_ematch.yacc.c", - "emp_ematch.yacc.h", - "fdt_test_tree_empty_memory_range_dtb", - "fdt_test_tree_multiple_memory_ranges_dtb", - "fdt_test_tree_one_memory_range_dtb", - "futility_cmds", - "gd_hci_packets_python3_gen", - "gd_smp_packets_python3_gen", "gen_corrupt_rebootless_apex", - "gen_corrupt_superblock_apex", "gen_key_mismatch_capex", - "gen_manifest_mismatch_apex_no_hashtree", - "generate_hash_v1", - "gles1_core_functions_hdr", - "gles1_extensions_functions_hdr", - "gles2_core_functions_hdr", - "gles2_extensions_functions_hdr", - "gles31_only_functions_hdr", - "gles3_only_functions_hdr", - "hci_packets_python3_gen", - "hidl2aidl_test_gen_aidl", - "hidl2aidl_translate_cpp_test_gen_headers", - "hidl2aidl_translate_cpp_test_gen_src", - "hidl2aidl_translate_java_test_gen_src", - "hidl2aidl_translate_ndk_test_gen_headers", - "hidl2aidl_translate_ndk_test_gen_src", - "hidl_cpp_impl_test_gen-headers", - "hidl_cpp_impl_test_gen-sources", - "hidl_error_test_gen", - "hidl_export_test_gen-headers", - "hidl_format_test_diff", - "hidl_hash_test_gen", - "hidl_hash_version_gen", - "hidl_java_impl_test_gen", - "lib-test-profile-text-protos", "libbssl_sys_src_nostd", "libc_musl_sysroot_bits", - "libchrome-crypto-include", - "libchrome-include", "libcore-non-cts-tests-txt", - "libmojo_jni_headers", - "libxml2_schema_fuzz_corpus", - "libxml2_xml_fuzz_corpus", - "link_layer_packets_python3_gen", - "llcp_packets_python3_gen", - "measure_io_as_jar", - "openwrt_rootfs_combined_aarch64", - "openwrt_rootfs_combined_x86_64", - "openwrt_rootfs_customization_aarch64", - "openwrt_rootfs_customization_x86_64", - "pandora-python-gen-src", - "pdl_cxx_canonical_be_src_gen", - "pdl_cxx_canonical_be_test_gen", - "pdl_cxx_canonical_le_src_gen", - "pdl_cxx_canonical_le_test_gen", - "pdl_python_generator_be_test_gen", - "pdl_python_generator_le_test_gen", - "pdl_rust_noalloc_le_test_backend_srcs", - "pdl_rust_noalloc_le_test_gen_harness", - "pixelatoms_defs.h", - "pixelstatsatoms.cpp", - "pixelstatsatoms.h", "pvmfw_fdt_template_rs", "r8retrace-dexdump-sample-app", "r8retrace-run-retrace", - "rootcanal_bredr_bb_packets_cxx_gen", - "rootcanal_hci_packets_cxx_gen", - "rootcanal_link_layer_packets_cxx_gen", - "sample-profile-text-protos", "seller-frontend-service-stub-lite", - "services.core.protologsrc", - "statsd-config-protos", "swiftshader_spvtools_update_build_version", - "temp_layoutlib", "ue_unittest_erofs_imgs", - "uwb_core_artifacts", "vm-tests-tf-lib", - "vndk_abi_dump_zip", - "vts_vndk_abi_dump_zip", - "wm_shell_protolog_src", - "wmtests.protologsrc", // go/keep-sorted end } diff --git a/genrule/genrule.go b/genrule/genrule.go index 01cac5b1f..8f2c0475b 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -189,9 +189,6 @@ type Module struct { subName string subDir string - - // Collect the module directory for IDE info in java/jdeps.go. - modulePaths []string } var _ android.MixedBuildBuildable = (*Module)(nil) @@ -289,9 +286,6 @@ func (g *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) { func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) { g.subName = ctx.ModuleSubDir() - // Collect the module directory for IDE info in java/jdeps.go. - g.modulePaths = append(g.modulePaths, ctx.ModuleDir()) - if len(g.properties.Export_include_dirs) > 0 { for _, dir := range g.properties.Export_include_dirs { g.exportedIncludeDirs = append(g.exportedIncludeDirs, @@ -668,7 +662,6 @@ func (g *Module) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Deps = append(dpInfo.Deps, src) } } - dpInfo.Paths = append(dpInfo.Paths, g.modulePaths...) } func (g *Module) AndroidMk() android.AndroidMkData { diff --git a/java/Android.bp b/java/Android.bp index 4450c4275..cf968713c 100644 --- a/java/Android.bp +++ b/java/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "soong-dexpreopt", "soong-genrule", "soong-java-config", + "soong-testing", "soong-provenance", "soong-python", "soong-remoteexec", @@ -112,6 +113,7 @@ bootstrap_go_package { "sdk_library_test.go", "system_modules_test.go", "systemserver_classpath_fragment_test.go", + "test_spec_test.go", ], pluginFor: ["soong_build"], } diff --git a/java/aapt2.go b/java/aapt2.go index 3bb70b53f..17ee6ee42 100644 --- a/java/aapt2.go +++ b/java/aapt2.go @@ -25,17 +25,23 @@ import ( "android/soong/android" ) +func isPathValueResource(res android.Path) bool { + subDir := filepath.Dir(res.String()) + subDir, lastDir := filepath.Split(subDir) + return strings.HasPrefix(lastDir, "values") +} + // Convert input resource file path to output file path. // values-[config]/<file>.xml -> values-[config]_<file>.arsc.flat; // For other resource file, just replace the last "/" with "_" and add .flat extension. func pathToAapt2Path(ctx android.ModuleContext, res android.Path) android.WritablePath { name := res.Base() - subDir := filepath.Dir(res.String()) - subDir, lastDir := filepath.Split(subDir) - if strings.HasPrefix(lastDir, "values") { + if isPathValueResource(res) { name = strings.TrimSuffix(name, ".xml") + ".arsc" } + subDir := filepath.Dir(res.String()) + subDir, lastDir := filepath.Split(subDir) name = lastDir + "_" + name + ".flat" return android.PathForModuleOut(ctx, "aapt2", subDir, name) } @@ -63,7 +69,21 @@ var aapt2CompileRule = pctx.AndroidStaticRule("aapt2Compile", // aapt2Compile compiles resources and puts the results in the requested directory. func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Paths, - flags []string) android.WritablePaths { + flags []string, productToFilter string) android.WritablePaths { + if productToFilter != "" && productToFilter != "default" { + // --filter-product leaves only product-specific resources. Product-specific resources only exist + // in value resources (values/*.xml), so filter value resource files only. Ignore other types of + // resources as they don't need to be in product characteristics RRO (and they will cause aapt2 + // compile errors) + filteredPaths := android.Paths{} + for _, path := range paths { + if isPathValueResource(path) { + filteredPaths = append(filteredPaths, path) + } + } + paths = filteredPaths + flags = append([]string{"--filter-product " + productToFilter}, flags...) + } // Shard the input paths so that they can be processed in parallel. If we shard them into too // small chunks, the additional cost of spinning up aapt2 outweighs the performance gain. The diff --git a/java/aar.go b/java/aar.go index 6b8912912..e579008e1 100644 --- a/java/aar.go +++ b/java/aar.go @@ -102,6 +102,9 @@ type aaptProperties struct { // true if RRO is enforced for any of the dependent modules RROEnforcedForDependent bool `blueprint:"mutated"` + + // Filter only specified product and ignore other products + Filter_product *string `blueprint:"mutated"` } type aapt struct { @@ -162,6 +165,10 @@ func (a *aapt) useResourceProcessorBusyBox() bool { return BoolDefault(a.aaptProperties.Use_resource_processor, false) } +func (a *aapt) filterProduct() string { + return String(a.aaptProperties.Filter_product) +} + func (a *aapt) ExportPackage() android.Path { return a.exportPackage } @@ -432,7 +439,7 @@ func (a *aapt) buildActions(ctx android.ModuleContext, opts aaptBuildActionOptio var compiledResDirs []android.Paths for _, dir := range resDirs { a.resourceFiles = append(a.resourceFiles, dir.files...) - compiledResDirs = append(compiledResDirs, aapt2Compile(ctx, dir.dir, dir.files, compileFlags).Paths()) + compiledResDirs = append(compiledResDirs, aapt2Compile(ctx, dir.dir, dir.files, compileFlags, a.filterProduct()).Paths()) } for i, zip := range resZips { @@ -491,7 +498,7 @@ func (a *aapt) buildActions(ctx android.ModuleContext, opts aaptBuildActionOptio } for _, dir := range overlayDirs { - compiledOverlay = append(compiledOverlay, aapt2Compile(ctx, dir.dir, dir.files, compileFlags).Paths()...) + compiledOverlay = append(compiledOverlay, aapt2Compile(ctx, dir.dir, dir.files, compileFlags, a.filterProduct()).Paths()...) } var splitPackages android.WritablePaths diff --git a/java/androidmk.go b/java/androidmk.go index 97b303dfb..84f78c89b 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -343,10 +343,15 @@ func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries { Disabled: true, }} } + var required []string + if proptools.Bool(app.appProperties.Generate_product_characteristics_rro) { + required = []string{app.productCharacteristicsRROPackageName()} + } return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "APPS", OutputFile: android.OptionalPathForPath(app.outputFile), Include: "$(BUILD_SYSTEM)/soong_app_prebuilt.mk", + Required: required, ExtraEntries: []android.AndroidMkExtraEntriesFunc{ func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { // App module names can be overridden. diff --git a/java/app.go b/java/app.go index 7b9e6bb14..6d7411d47 100755 --- a/java/app.go +++ b/java/app.go @@ -22,6 +22,7 @@ import ( "path/filepath" "strings" + "android/soong/testing" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -130,6 +131,16 @@ type appProperties struct { // Specifies the file that contains the allowlist for this app. Privapp_allowlist *string `android:"path"` + + // If set, create an RRO package which contains only resources having PRODUCT_CHARACTERISTICS + // and install the RRO package to /product partition, instead of passing --product argument + // to aapt2. Default is false. + // Setting this will make this APK identical to all targets, regardless of + // PRODUCT_CHARACTERISTICS. + Generate_product_characteristics_rro *bool + + ProductCharacteristicsRROPackageName *string `blueprint:"mutated"` + ProductCharacteristicsRROManifestModuleName *string `blueprint:"mutated"` } // android_app properties that can be overridden by override_android_app @@ -454,8 +465,9 @@ func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { aaptLinkFlags := []string{} // Add TARGET_AAPT_CHARACTERISTICS values to AAPT link flags if they exist and --product flags were not provided. + autogenerateRRO := proptools.Bool(a.appProperties.Generate_product_characteristics_rro) hasProduct := android.PrefixInList(a.aaptProperties.Aaptflags, "--product") - if !hasProduct && len(ctx.Config().ProductAAPTCharacteristics()) > 0 { + if !autogenerateRRO && !hasProduct && len(ctx.Config().ProductAAPTCharacteristics()) > 0 { aaptLinkFlags = append(aaptLinkFlags, "--product", ctx.Config().ProductAAPTCharacteristics()) } @@ -857,7 +869,7 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { ctx.InstallFile(allowlistInstallPath, allowlistInstallFilename, a.privAppAllowlist.Path()) } - var extraInstalledPaths android.Paths + var extraInstalledPaths android.InstallPaths for _, extra := range a.extraOutputFiles { installed := ctx.InstallFile(a.installDir, extra.Base(), extra) extraInstalledPaths = append(extraInstalledPaths, installed) @@ -1056,6 +1068,8 @@ func (a *AndroidApp) OutputFiles(tag string) (android.Paths, error) { } case ".export-package.apk": return []android.Path{a.exportPackage}, nil + case ".manifest.xml": + return []android.Path{a.aapt.manifestPath}, nil } return a.Library.OutputFiles(tag) } @@ -1085,6 +1099,14 @@ func (a *AndroidApp) IDEInfo(dpInfo *android.IdeInfo) { a.aapt.IDEInfo(dpInfo) } +func (a *AndroidApp) productCharacteristicsRROPackageName() string { + return proptools.String(a.appProperties.ProductCharacteristicsRROPackageName) +} + +func (a *AndroidApp) productCharacteristicsRROManifestModuleName() string { + return proptools.String(a.appProperties.ProductCharacteristicsRROManifestModuleName) +} + // android_app compiles sources and Android resources into an Android application package `.apk` file. func AndroidAppFactory() android.Module { module := &AndroidApp{} @@ -1111,6 +1133,57 @@ func AndroidAppFactory() android.Module { android.InitApexModule(module) android.InitBazelModule(module) + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + a := ctx.Module().(*AndroidApp) + + characteristics := ctx.Config().ProductAAPTCharacteristics() + if characteristics == "default" || characteristics == "" { + module.appProperties.Generate_product_characteristics_rro = nil + // no need to create RRO + return + } + + if !proptools.Bool(module.appProperties.Generate_product_characteristics_rro) { + return + } + + rroPackageName := a.Name() + "__" + strings.ReplaceAll(characteristics, ",", "_") + "__auto_generated_characteristics_rro" + rroManifestName := rroPackageName + "_manifest" + + a.appProperties.ProductCharacteristicsRROPackageName = proptools.StringPtr(rroPackageName) + a.appProperties.ProductCharacteristicsRROManifestModuleName = proptools.StringPtr(rroManifestName) + + rroManifestProperties := struct { + Name *string + Tools []string + Out []string + Srcs []string + Cmd *string + }{ + Name: proptools.StringPtr(rroManifestName), + Tools: []string{"characteristics_rro_generator"}, + Out: []string{"AndroidManifest.xml"}, + Srcs: []string{":" + a.Name() + "{.manifest.xml}"}, + Cmd: proptools.StringPtr("$(location characteristics_rro_generator) $(in) $(out)"), + } + ctx.CreateModule(genrule.GenRuleFactory, &rroManifestProperties) + + rroProperties := struct { + Name *string + Filter_product *string + Aaptflags []string + Manifest *string + Resource_dirs []string + }{ + Name: proptools.StringPtr(rroPackageName), + Filter_product: proptools.StringPtr(characteristics), + Aaptflags: []string{"--auto-add-overlay"}, + Manifest: proptools.StringPtr(":" + rroManifestName), + Resource_dirs: a.aaptProperties.Resource_dirs, + } + ctx.CreateModule(RuntimeResourceOverlayFactory, &rroProperties) + }) + return module } @@ -1193,6 +1266,7 @@ func (a *AndroidTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.testConfig = a.FixTestConfig(ctx, testConfig) a.extraTestConfigs = android.PathsForModuleSrc(ctx, a.testProperties.Test_options.Extra_test_configs) a.data = android.PathsForModuleSrc(ctx, a.testProperties.Data) + ctx.SetProvider(testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } func (a *AndroidTest) FixTestConfig(ctx android.ModuleContext, testConfig android.Path) android.Path { @@ -1584,7 +1658,7 @@ func (u *usesLibrary) verifyUsesLibraries(ctx android.ModuleContext, inputFile a // non-linux build platforms where dexpreopt is generally disabled (the check may fail due to // various unrelated reasons, such as a failure to get manifest from an APK). global := dexpreopt.GetGlobalConfig(ctx) - if global.DisablePreopt || global.OnlyPreoptBootImageAndSystemServer { + if global.DisablePreopt || global.OnlyPreoptArtBootImage { return inputFile } diff --git a/java/base.go b/java/base.go index 3d7d3de01..fdc164e11 100644 --- a/java/base.go +++ b/java/base.go @@ -497,9 +497,6 @@ type Module struct { // list of the xref extraction files kytheFiles android.Paths - // Collect the module directory for IDE info in java/jdeps.go. - modulePaths []string - hideApexVariantFromMake bool sdkVersion android.SdkSpec @@ -2015,7 +2012,6 @@ func (j *Module) IDEInfo(dpInfo *android.IdeInfo) { if j.expandJarjarRules != nil { dpInfo.Jarjar_rules = append(dpInfo.Jarjar_rules, j.expandJarjarRules.String()) } - dpInfo.Paths = append(dpInfo.Paths, j.modulePaths...) dpInfo.Static_libs = append(dpInfo.Static_libs, j.properties.Static_libs...) dpInfo.Libs = append(dpInfo.Libs, j.properties.Libs...) dpInfo.SrcJars = append(dpInfo.SrcJars, j.annoSrcJars.Strings()...) diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index 7d8a9f7bb..191a65ec1 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -23,6 +23,7 @@ import ( "android/soong/android" "android/soong/dexpreopt" + "android/soong/testing" "github.com/google/blueprint/proptools" @@ -238,9 +239,6 @@ type BootclasspathFragmentModule struct { sourceOnlyProperties SourceOnlyBootclasspathProperties - // Collect the module directory for IDE info in java/jdeps.go. - modulePaths []string - // Path to the boot image profile. profilePath android.WritablePath } @@ -471,9 +469,6 @@ func (b *BootclasspathFragmentModule) GenerateAndroidBuildActions(ctx android.Mo // Generate classpaths.proto config b.generateClasspathProtoBuildActions(ctx) - // Collect the module directory for IDE info in java/jdeps.go. - b.modulePaths = append(b.modulePaths, ctx.ModuleDir()) - // Gather the bootclasspath fragment's contents. var contents []android.Module ctx.VisitDirectDeps(func(module android.Module) { @@ -505,6 +500,7 @@ func (b *BootclasspathFragmentModule) GenerateAndroidBuildActions(ctx android.Mo if ctx.Module() != ctx.FinalModule() { b.HideFromMake() } + ctx.SetProvider(testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } // getProfileProviderApex returns the name of the apex that provides a boot image profile, or an @@ -582,7 +578,7 @@ func (b *BootclasspathFragmentModule) configuredJars(ctx android.ModuleContext) // So ignore it even if it is not in PRODUCT_APEX_BOOT_JARS. // TODO(b/202896428): Add better way to handle this. _, unknown = android.RemoveFromList("android.car-module", unknown) - if len(unknown) > 0 { + if isActiveModule(ctx.Module()) && len(unknown) > 0 { ctx.ModuleErrorf("%s in contents must also be declared in PRODUCT_APEX_BOOT_JARS", unknown) } } @@ -801,7 +797,6 @@ func (b *BootclasspathFragmentModule) getProfilePath() android.Path { // Collect information for opening IDE project files in java/jdeps.go. func (b *BootclasspathFragmentModule) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Deps = append(dpInfo.Deps, b.properties.Contents...) - dpInfo.Paths = append(dpInfo.Paths, b.modulePaths...) } type bootclasspathFragmentMemberType struct { diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index c0f73afc5..5fb36df6d 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -610,7 +610,8 @@ func generateBootImage(ctx android.ModuleContext, imageConfig *bootImageConfig) profile := bootImageProfileRule(ctx, imageConfig) // If dexpreopt of boot image jars should be skipped, stop after generating a profile. - if SkipDexpreoptBootJars(ctx) { + global := dexpreopt.GetGlobalConfig(ctx) + if SkipDexpreoptBootJars(ctx) || (global.OnlyPreoptArtBootImage && imageConfig.name != "art") { return } diff --git a/java/dexpreopt_check.go b/java/dexpreopt_check.go index 7499481de..33be60352 100644 --- a/java/dexpreopt_check.go +++ b/java/dexpreopt_check.go @@ -68,7 +68,7 @@ func (m *dexpreoptSystemserverCheck) GenerateAndroidBuildActions(ctx android.Mod // The check should be skipped on unbundled builds because system server jars are not preopted on // unbundled builds since the artifacts are installed into the system image, not the APEXes. - if global.DisablePreopt || len(targets) == 0 || ctx.Config().UnbundledBuild() { + if global.DisablePreopt || global.OnlyPreoptArtBootImage || len(targets) == 0 || ctx.Config().UnbundledBuild() { return } diff --git a/java/java.go b/java/java.go index dd0418894..bb9357cc7 100644 --- a/java/java.go +++ b/java/java.go @@ -27,6 +27,7 @@ import ( "android/soong/bazel" "android/soong/bazel/cquery" "android/soong/remoteexec" + "android/soong/testing" "android/soong/ui/metrics/bp2build_metrics_proto" "github.com/google/blueprint" @@ -640,7 +641,7 @@ type Library struct { exportedProguardFlagFiles android.Paths - InstallMixin func(ctx android.ModuleContext, installPath android.Path) (extraInstallDeps android.Paths) + InstallMixin func(ctx android.ModuleContext, installPath android.Path) (extraInstallDeps android.InstallPaths) } var _ android.ApexModule = (*Library)(nil) @@ -719,12 +720,9 @@ func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { } j.compile(ctx, nil, nil, nil) - // Collect the module directory for IDE info in java/jdeps.go. - j.modulePaths = append(j.modulePaths, ctx.ModuleDir()) - exclusivelyForApex := !apexInfo.IsForPlatform() if (Bool(j.properties.Installable) || ctx.Host()) && !exclusivelyForApex { - var extraInstallDeps android.Paths + var extraInstallDeps android.InstallPaths if j.InstallMixin != nil { extraInstallDeps = j.InstallMixin(ctx, j.outputFile) } @@ -1228,10 +1226,12 @@ func (j *TestHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { } j.Test.generateAndroidBuildActionsWithConfig(ctx, configs) + ctx.SetProvider(testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } func (j *Test) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.generateAndroidBuildActionsWithConfig(ctx, nil) + ctx.SetProvider(testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } func (j *Test) generateAndroidBuildActionsWithConfig(ctx android.ModuleContext, configs []tradefed.Config) { diff --git a/java/jdeps.go b/java/jdeps.go index 4c8c11c5d..7e3a14f55 100644 --- a/java/jdeps.go +++ b/java/jdeps.go @@ -75,7 +75,7 @@ func (j *jdepsGeneratorSingleton) GenerateBuildActions(ctx android.SingletonCont dpInfo.Jarjar_rules = android.FirstUniqueStrings(dpInfo.Jarjar_rules) dpInfo.Jars = android.FirstUniqueStrings(dpInfo.Jars) dpInfo.SrcJars = android.FirstUniqueStrings(dpInfo.SrcJars) - dpInfo.Paths = android.FirstUniqueStrings(dpInfo.Paths) + dpInfo.Paths = []string{ctx.ModuleDir(module)} dpInfo.Static_libs = android.FirstUniqueStrings(dpInfo.Static_libs) dpInfo.Libs = android.FirstUniqueStrings(dpInfo.Libs) moduleInfos[name] = dpInfo diff --git a/java/resourceshrinker.go b/java/resourceshrinker.go index bf1b04d94..af13aa3cb 100644 --- a/java/resourceshrinker.go +++ b/java/resourceshrinker.go @@ -23,7 +23,8 @@ import ( var shrinkResources = pctx.AndroidStaticRule("shrinkResources", blueprint.RuleParams{ // Note that we suppress stdout to avoid successful log confirmations. - Command: `${config.ResourceShrinkerCmd} --output $out --input $in --raw_resources $raw_resources >/dev/null`, + Command: `RESOURCESHRINKER_OPTS=-Dcom.android.tools.r8.dexContainerExperiment ` + + `${config.ResourceShrinkerCmd} --output $out --input $in --raw_resources $raw_resources >/dev/null`, CommandDeps: []string{"${config.ResourceShrinkerCmd}"}, }, "raw_resources") diff --git a/java/robolectric.go b/java/robolectric.go index af5633934..a66b31007 100644 --- a/java/robolectric.go +++ b/java/robolectric.go @@ -22,6 +22,7 @@ import ( "android/soong/android" "android/soong/java/config" + "android/soong/testing" "android/soong/tradefed" "github.com/google/blueprint/proptools" @@ -225,7 +226,7 @@ func (r *robolectricTest) GenerateAndroidBuildActions(ctx android.ModuleContext) } installPath := android.PathForModuleInstall(ctx, r.BaseModuleName()) - var installDeps android.Paths + var installDeps android.InstallPaths if r.manifest != nil { r.data = append(r.data, r.manifest) @@ -253,6 +254,7 @@ func (r *robolectricTest) GenerateAndroidBuildActions(ctx android.ModuleContext) } r.installFile = ctx.InstallFile(installPath, ctx.ModuleName()+".jar", r.combinedJar, installDeps...) + ctx.SetProvider(testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } func generateRoboTestConfig(ctx android.ModuleContext, outputFile android.WritablePath, diff --git a/java/sdk_library.go b/java/sdk_library.go index ea451743b..fb2781213 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -624,6 +624,13 @@ type sdkLibraryProperties struct { Legacy_errors_allowed *bool } + // Determines if the module contributes to any api surfaces. + // This property should be set to true only if the module is listed under + // frameworks-base-api.bootclasspath in frameworks/base/api/Android.bp. + // Otherwise, this property should be set to false. + // Defaults to false. + Contribute_to_android_api *bool + // TODO: determines whether to create HTML doc or not // Html_doc *bool } @@ -1966,6 +1973,10 @@ func (module *SdkLibrary) UniqueApexVariations() bool { return module.uniqueApexVariations() } +func (module *SdkLibrary) ContributeToApi() bool { + return proptools.BoolDefault(module.sdkLibraryProperties.Contribute_to_android_api, false) +} + // Creates the xml file that publicizes the runtime library func (module *SdkLibrary) createXmlFile(mctx android.DefaultableHookContext) { moduleMinApiLevel := module.Library.MinSdkVersion(mctx) diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index 21f0bab37..82f8a4d50 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -1068,6 +1068,137 @@ func TestJavaSdkLibraryImport_Preferred(t *testing.T) { }) } +// If a module is listed in `mainline_module_contributions, it should be used +// It will supersede any other source vs prebuilt selection mechanism like `prefer` attribute +func TestSdkLibraryImport_MetadataModuleSupersedesPreferred(t *testing.T) { + bp := ` + apex_contributions { + name: "my_mainline_module_contributions", + api_domain: "my_mainline_module", + contents: [ + // legacy mechanism prefers the prebuilt + // mainline_module_contributions supersedes this since source is listed explicitly + "sdklib.prebuilt_preferred_using_legacy_flags", + + // legacy mechanism prefers the source + // mainline_module_contributions supersedes this since prebuilt is listed explicitly + "prebuilt_sdklib.source_preferred_using_legacy_flags", + ], + } + all_apex_contributions { + name: "all_apex_contributions", + } + java_sdk_library { + name: "sdklib.prebuilt_preferred_using_legacy_flags", + srcs: ["a.java"], + sdk_version: "none", + system_modules: "none", + public: { + enabled: true, + }, + system: { + enabled: true, + } + } + java_sdk_library_import { + name: "sdklib.prebuilt_preferred_using_legacy_flags", + prefer: true, // prebuilt is preferred using legacy mechanism + public: { + jars: ["a.jar"], + stub_srcs: ["a.java"], + current_api: "current.txt", + removed_api: "removed.txt", + annotations: "annotations.zip", + }, + system: { + jars: ["a.jar"], + stub_srcs: ["a.java"], + current_api: "current.txt", + removed_api: "removed.txt", + annotations: "annotations.zip", + }, + } + java_sdk_library { + name: "sdklib.source_preferred_using_legacy_flags", + srcs: ["a.java"], + sdk_version: "none", + system_modules: "none", + public: { + enabled: true, + }, + system: { + enabled: true, + } + } + java_sdk_library_import { + name: "sdklib.source_preferred_using_legacy_flags", + prefer: false, // source is preferred using legacy mechanism + public: { + jars: ["a.jar"], + stub_srcs: ["a.java"], + current_api: "current.txt", + removed_api: "removed.txt", + annotations: "annotations.zip", + }, + system: { + jars: ["a.jar"], + stub_srcs: ["a.java"], + current_api: "current.txt", + removed_api: "removed.txt", + annotations: "annotations.zip", + }, + } + + // rdeps + java_library { + name: "public", + srcs: ["a.java"], + libs: [ + // this should get source since source is listed in my_mainline_module_contributions + "sdklib.prebuilt_preferred_using_legacy_flags.stubs", + "sdklib.prebuilt_preferred_using_legacy_flags.stubs.system", + + // this should get prebuilt since source is listed in my_mainline_module_contributions + "sdklib.source_preferred_using_legacy_flags.stubs", + "sdklib.source_preferred_using_legacy_flags.stubs.system", + + ], + } + ` + result := android.GroupFixturePreparers( + prepareForJavaTest, + PrepareForTestWithJavaSdkLibraryFiles, + FixtureWithLastReleaseApis("sdklib.source_preferred_using_legacy_flags", "sdklib.prebuilt_preferred_using_legacy_flags"), + android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + android.RegisterApexContributionsBuildComponents(ctx) + }), + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.BuildFlags = map[string]string{ + "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": "my_mainline_module_contributions", + } + }), + ).RunTestWithBp(t, bp) + + // Make sure that rdeps get the correct source vs prebuilt based on mainline_module_contributions + public := result.ModuleForTests("public", "android_common") + rule := public.Output("javac/public.jar") + inputs := rule.Implicits.Strings() + expectedInputs := []string{ + // source + "out/soong/.intermediates/sdklib.prebuilt_preferred_using_legacy_flags.stubs/android_common/turbine-combined/sdklib.prebuilt_preferred_using_legacy_flags.stubs.jar", + "out/soong/.intermediates/sdklib.prebuilt_preferred_using_legacy_flags.stubs.system/android_common/turbine-combined/sdklib.prebuilt_preferred_using_legacy_flags.stubs.system.jar", + + // prebuilt + "out/soong/.intermediates/prebuilt_sdklib.source_preferred_using_legacy_flags.stubs/android_common/combined/sdklib.source_preferred_using_legacy_flags.stubs.jar", + "out/soong/.intermediates/prebuilt_sdklib.source_preferred_using_legacy_flags.stubs.system/android_common/combined/sdklib.source_preferred_using_legacy_flags.stubs.system.jar", + } + for _, expected := range expectedInputs { + if !android.InList(expected, inputs) { + t.Errorf("expected %q to contain %q", inputs, expected) + } + } +} + func TestJavaSdkLibraryEnforce(t *testing.T) { partitionToBpOption := func(partition string) string { switch partition { diff --git a/java/systemserver_classpath_fragment.go b/java/systemserver_classpath_fragment.go index 17d301b70..30dd55fa9 100644 --- a/java/systemserver_classpath_fragment.go +++ b/java/systemserver_classpath_fragment.go @@ -87,9 +87,6 @@ type SystemServerClasspathModule struct { ClasspathFragmentBase properties systemServerClasspathFragmentProperties - - // Collect the module directory for IDE info in java/jdeps.go. - modulePaths []string } func (s *SystemServerClasspathModule) ShouldSupportSdkVersion(ctx android.BaseModuleContext, sdkVersion android.ApiLevel) error { @@ -129,9 +126,6 @@ func (s *SystemServerClasspathModule) GenerateAndroidBuildActions(ctx android.Mo configuredJars = configuredJars.AppendList(&standaloneConfiguredJars) classpathJars = append(classpathJars, standaloneClasspathJars...) s.classpathFragmentBase().generateClasspathProtoBuildActions(ctx, configuredJars, classpathJars) - - // Collect the module directory for IDE info in java/jdeps.go. - s.modulePaths = append(s.modulePaths, ctx.ModuleDir()) } func (s *SystemServerClasspathModule) configuredJars(ctx android.ModuleContext) android.ConfiguredJarList { @@ -242,7 +236,6 @@ func (s *SystemServerClasspathModule) ComponentDepsMutator(ctx android.BottomUpM func (s *SystemServerClasspathModule) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Deps = append(dpInfo.Deps, s.properties.Contents...) dpInfo.Deps = append(dpInfo.Deps, s.properties.Standalone_contents...) - dpInfo.Paths = append(dpInfo.Paths, s.modulePaths...) } type systemServerClasspathFragmentMemberType struct { diff --git a/java/test_spec_test.go b/java/test_spec_test.go new file mode 100644 index 000000000..39aff4cef --- /dev/null +++ b/java/test_spec_test.go @@ -0,0 +1,133 @@ +package java + +import ( + "strings" + "testing" + + "android/soong/android" + soongTesting "android/soong/testing" + "android/soong/testing/test_spec_proto" + "google.golang.org/protobuf/proto" +) + +func TestTestSpec(t *testing.T) { + bp := `test_spec { + name: "module-name", + teamId: "12345", + tests: [ + "java-test-module-name-one", + "java-test-module-name-two" + ] + } + + java_test { + name: "java-test-module-name-one", + } + + java_test { + name: "java-test-module-name-two", + }` + result := runTest(t, android.FixtureExpectsNoErrors, bp) + + module := result.ModuleForTests( + "module-name", "", + ).Module().(*soongTesting.TestSpecModule) + + // Check that the provider has the right contents + data := result.ModuleProvider( + module, soongTesting.TestSpecProviderKey, + ).(soongTesting.TestSpecProviderData) + if !strings.HasSuffix( + data.IntermediatePath.String(), "/intermediateTestSpecMetadata.pb", + ) { + t.Errorf( + "Missing intermediates path in provider: %s", + data.IntermediatePath.String(), + ) + } + + buildParamsSlice := module.BuildParamsForTests() + var metadata = "" + for _, params := range buildParamsSlice { + if params.Rule.String() == "android/soong/android.writeFile" { + metadata = params.Args["content"] + } + } + + metadataList := make([]*test_spec_proto.TestSpec_OwnershipMetadata, 0, 2) + teamId := "12345" + bpFilePath := "Android.bp" + targetNames := []string{ + "java-test-module-name-one", "java-test-module-name-two", + } + + for _, test := range targetNames { + targetName := test + metadata := test_spec_proto.TestSpec_OwnershipMetadata{ + TrendyTeamId: &teamId, + TargetName: &targetName, + Path: &bpFilePath, + } + metadataList = append(metadataList, &metadata) + } + testSpecMetadata := test_spec_proto.TestSpec{OwnershipMetadataList: metadataList} + protoData, _ := proto.Marshal(&testSpecMetadata) + rawData := string(protoData) + formattedData := strings.ReplaceAll(rawData, "\n", "\\n") + expectedMetadata := "'" + formattedData + "\\n'" + + if metadata != expectedMetadata { + t.Errorf( + "Retrieved metadata: %s is not equal to expectedMetadata: %s", metadata, + expectedMetadata, + ) + } + + // Tests for all_test_spec singleton. + singleton := result.SingletonForTests("all_test_specs") + rule := singleton.Rule("all_test_specs_rule") + prebuiltOs := result.Config.PrebuiltOS() + expectedCmd := "out/soong/host/" + prebuiltOs + "/bin/metadata -rule test_spec -inputFile out/soong/all_test_spec_paths.rsp -outputFile out/soong/ownership/all_test_specs.pb" + expectedOutputFile := "out/soong/ownership/all_test_specs.pb" + expectedInputFile := "out/soong/.intermediates/module-name/intermediateTestSpecMetadata.pb" + if !strings.Contains( + strings.TrimSpace(rule.Output.String()), + expectedOutputFile, + ) { + t.Errorf( + "Retrieved singletonOutputFile: %s is not equal to expectedSingletonOutputFile: %s", + rule.Output.String(), expectedOutputFile, + ) + } + + if !strings.Contains( + strings.TrimSpace(rule.Inputs[0].String()), + expectedInputFile, + ) { + t.Errorf( + "Retrieved singletonInputFile: %s is not equal to expectedSingletonInputFile: %s", + rule.Inputs[0].String(), expectedInputFile, + ) + } + + if !strings.Contains( + strings.TrimSpace(rule.RuleParams.Command), + expectedCmd, + ) { + t.Errorf( + "Retrieved cmd: %s is not equal to expectedCmd: %s", + rule.RuleParams.Command, expectedCmd, + ) + } +} + +func runTest( + t *testing.T, errorHandler android.FixtureErrorHandler, bp string, +) *android.TestResult { + return android.GroupFixturePreparers( + soongTesting.PrepareForTestWithTestSpecBuildComponents, + PrepareForIntegrationTestWithJava, + ). + ExtendWithErrorHandler(errorHandler). + RunTestWithBp(t, bp) +} diff --git a/java/tradefed.go b/java/tradefed.go index ebbdec13d..349b327e8 100644 --- a/java/tradefed.go +++ b/java/tradefed.go @@ -30,8 +30,8 @@ func tradefedJavaLibraryFactory() android.Module { return module } -func tradefedJavaLibraryInstall(ctx android.ModuleContext, path android.Path) android.Paths { +func tradefedJavaLibraryInstall(ctx android.ModuleContext, path android.Path) android.InstallPaths { installedPath := ctx.InstallFile(android.PathForModuleInstall(ctx, "tradefed"), ctx.ModuleName()+".jar", path) - return android.Paths{installedPath} + return android.InstallPaths{installedPath} } diff --git a/kernel/prebuilt_kernel_modules.go b/kernel/prebuilt_kernel_modules.go index 5bcca047e..e200ee2c1 100644 --- a/kernel/prebuilt_kernel_modules.go +++ b/kernel/prebuilt_kernel_modules.go @@ -50,6 +50,9 @@ type prebuiltKernelModulesProperties struct { // Kernel version that these modules are for. Kernel modules are installed to // /lib/modules/<kernel_version> directory in the corresponding partition. Default is "". Kernel_version *string + + // Whether this module is directly installable to one of the partitions. Default is true + Installable *bool } // prebuilt_kernel_modules installs a set of prebuilt kernel module files to the correct directory. @@ -62,6 +65,10 @@ func prebuiltKernelModulesFactory() android.Module { return module } +func (pkm *prebuiltKernelModules) installable() bool { + return proptools.BoolDefault(pkm.properties.Installable, true) +} + func (pkm *prebuiltKernelModules) KernelVersion() string { return proptools.StringDefault(pkm.properties.Kernel_version, "") } @@ -71,6 +78,9 @@ func (pkm *prebuiltKernelModules) DepsMutator(ctx android.BottomUpMutatorContext } func (pkm *prebuiltKernelModules) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if !pkm.installable() { + pkm.SkipInstall() + } modules := android.PathsForModuleSrc(ctx, pkm.properties.Srcs) depmodOut := runDepmod(ctx, modules) diff --git a/python/Android.bp b/python/Android.bp index 75786733a..87810c9ed 100644 --- a/python/Android.bp +++ b/python/Android.bp @@ -10,6 +10,7 @@ bootstrap_go_package { "soong-android", "soong-tradefed", "soong-cc", + "soong-testing", ], srcs: [ "binary.go", diff --git a/python/test.go b/python/test.go index 6e23a447f..cd7c73b5a 100644 --- a/python/test.go +++ b/python/test.go @@ -17,6 +17,7 @@ package python import ( "fmt" + "android/soong/testing" "github.com/google/blueprint/proptools" "android/soong/android" @@ -205,6 +206,7 @@ func (p *PythonTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext p.data = append(p.data, android.DataPath{SrcPath: javaDataSrcPath}) } } + ctx.SetProvider(testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } func (p *PythonTestModule) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/remoteexec/remoteexec.go b/remoteexec/remoteexec.go index 690b47bbf..1e181fb17 100644 --- a/remoteexec/remoteexec.go +++ b/remoteexec/remoteexec.go @@ -30,7 +30,7 @@ const ( // DefaultImage is the default container image used for Android remote execution. The // image was built with the Dockerfile at // https://android.googlesource.com/platform/prebuilts/remoteexecution-client/+/refs/heads/master/docker/Dockerfile - DefaultImage = "docker://gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:953fed4a6b2501256a0d17f055dc17884ff71b024e50ade773e0b348a6c303e6" + DefaultImage = "docker://gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:1eb7f64b9e17102b970bd7a1af7daaebdb01c3fb777715899ef462d6c6d01a45" // DefaultWrapperPath is the default path to the remote execution wrapper. DefaultWrapperPath = "prebuilts/remoteexecution-client/live/rewrapper" diff --git a/rust/Android.bp b/rust/Android.bp index b01a94ad0..c5b200019 100644 --- a/rust/Android.bp +++ b/rust/Android.bp @@ -12,6 +12,7 @@ bootstrap_go_package { "soong-cc", "soong-rust-config", "soong-snapshot", + "soong-testing", ], srcs: [ "afdo.go", diff --git a/rust/compiler.go b/rust/compiler.go index 9afaec587..d453a5d6f 100644 --- a/rust/compiler.go +++ b/rust/compiler.go @@ -203,7 +203,7 @@ type BaseCompilerProperties struct { Relative_install_path *string `android:"arch_variant"` // whether to suppress inclusion of standard crates - defaults to false - No_stdlibs *bool + No_stdlibs *bool `android:"arch_variant"` // Change the rustlibs linkage to select rlib linkage by default for device targets. // Also link libstd as an rlib as well on device targets. diff --git a/rust/rust.go b/rust/rust.go index d315019ac..d4d33c71d 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -20,6 +20,7 @@ import ( "android/soong/bazel" "android/soong/bloaty" + "android/soong/testing" "android/soong/ui/metrics/bp2build_metrics_proto" "github.com/google/blueprint" @@ -144,8 +145,9 @@ type Module struct { Properties BaseProperties - hod android.HostOrDeviceSupported - multilib android.Multilib + hod android.HostOrDeviceSupported + multilib android.Multilib + testModule bool makeLinkType string @@ -1000,6 +1002,9 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { ctx.Phony("rust", ctx.RustModule().OutputFile().Path()) } + if mod.testModule { + ctx.SetProvider(testing.TestModuleProviderKey, testing.TestModuleProviderData{}) + } } func (mod *Module) deps(ctx DepsContext) Deps { diff --git a/rust/test.go b/rust/test.go index 4b5296e54..7ffc36767 100644 --- a/rust/test.go +++ b/rust/test.go @@ -222,11 +222,13 @@ func RustTestFactory() android.Module { // rustTestHostMultilib load hook to set MultilibFirst for the // host target. android.AddLoadHook(module, rustTestHostMultilib) + module.testModule = true return module.Init() } func RustTestHostFactory() android.Module { module, _ := NewRustTest(android.HostSupported) + module.testModule = true return module.Init() } diff --git a/rust/test_test.go b/rust/test_test.go index 8906f1cb0..6d0ebcf28 100644 --- a/rust/test_test.go +++ b/rust/test_test.go @@ -38,7 +38,7 @@ func TestRustTest(t *testing.T) { dataPaths := testingModule.Module().(*Module).compiler.(*testDecorator).dataPaths() if len(dataPaths) != 1 { - t.Errorf("expected exactly one test data file. test data files: [%s]", dataPaths) + t.Errorf("expected exactly one test data file. test data files: [%v]", dataPaths) return } } @@ -116,7 +116,7 @@ func TestDataLibs(t *testing.T) { t.Fatalf("expected exactly one output file. output files: [%s]", outputFiles) } if len(testBinary.dataPaths()) != 2 { - t.Fatalf("expected exactly two test data files. test data files: [%s]", testBinary.dataPaths()) + t.Fatalf("expected exactly two test data files. test data files: [%v]", testBinary.dataPaths()) } outputPath := outputFiles[0].String() @@ -178,7 +178,7 @@ func TestDataLibsRelativeInstallPath(t *testing.T) { t.Fatalf("expected exactly one output file. output files: [%s]", outputFiles) } if len(testBinary.dataPaths()) != 3 { - t.Fatalf("expected exactly two test data files. test data files: [%s]", testBinary.dataPaths()) + t.Fatalf("expected exactly two test data files. test data files: [%v]", testBinary.dataPaths()) } outputPath := outputFiles[0].String() diff --git a/scripts/build-ndk-prebuilts.sh b/scripts/build-ndk-prebuilts.sh index 0d14019a5..ef0f44a7c 100755 --- a/scripts/build-ndk-prebuilts.sh +++ b/scripts/build-ndk-prebuilts.sh @@ -19,9 +19,11 @@ if [ -z "${OUT_DIR}" ]; then exit 1 fi +# Note: NDK doesn't support flagging APIs, so we hardcode it to trunk_staging. # TODO: remove ALLOW_MISSING_DEPENDENCIES=true when all the riscv64 # dependencies exist (currently blocked by http://b/273792258). # TODO: remove BUILD_BROKEN_DISABLE_BAZEL=1 when bazel supports riscv64 (http://b/262192655). +TARGET_RELEASE=trunk_staging \ ALLOW_MISSING_DEPENDENCIES=true \ BUILD_BROKEN_DISABLE_BAZEL=1 \ TARGET_PRODUCT=ndk build/soong/soong_ui.bash --make-mode --soong-only ${OUT_DIR}/soong/ndk.timestamp diff --git a/scripts/conv_linker_config.py b/scripts/conv_linker_config.py index c6e6e306f..ed3fbb79f 100644 --- a/scripts/conv_linker_config.py +++ b/scripts/conv_linker_config.py @@ -149,6 +149,11 @@ def Validate(args): else: sys.exit(f'Unknown type: {args.type}') + # Reject contributions field at build time while keeping the runtime behavior for GRF. + if getattr(pb, 'contributions'): + sys.exit(f"{args.input}: 'contributions' is set. " + "It's deprecated. Instead, make the APEX 'visible' and use android_dlopen_ext().") + def ValidateAndWriteAsPbFile(pb, output_path): ValidateConfiguration(pb) diff --git a/sh/Android.bp b/sh/Android.bp index 1deedc731..930fcf54d 100644 --- a/sh/Android.bp +++ b/sh/Android.bp @@ -11,6 +11,7 @@ bootstrap_go_package { "soong-android", "soong-cc", "soong-java", + "soong-testing", "soong-tradefed", ], srcs: [ diff --git a/sh/sh_binary.go b/sh/sh_binary.go index 2e869f447..1bebc60be 100644 --- a/sh/sh_binary.go +++ b/sh/sh_binary.go @@ -20,6 +20,7 @@ import ( "sort" "strings" + "android/soong/testing" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -452,6 +453,7 @@ func (s *ShTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.PropertyErrorf(property, "%q of type %q is not supported", dep.Name(), ctx.OtherModuleType(dep)) } }) + ctx.SetProvider(testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } func (s *ShTest) InstallInData() bool { diff --git a/testing/Android.bp b/testing/Android.bp new file mode 100644 index 000000000..18dccb305 --- /dev/null +++ b/testing/Android.bp @@ -0,0 +1,21 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-testing", + pkgPath: "android/soong/testing", + deps: [ + "blueprint", + "soong-android", + "soong-testing-test_spec_proto", + + ], + srcs: [ + "all_test_specs.go", + "test_spec.go", + "init.go", + "test.go", + ], + pluginFor: ["soong_build"], +} diff --git a/testing/all_test_specs.go b/testing/all_test_specs.go new file mode 100644 index 000000000..9d4645b37 --- /dev/null +++ b/testing/all_test_specs.go @@ -0,0 +1,45 @@ +package testing + +import ( + "android/soong/android" +) + +const ownershipDirectory = "ownership" +const fileContainingFilePaths = "all_test_spec_paths.rsp" +const allTestSpecsFile = "all_test_specs.pb" + +func AllTestSpecsFactory() android.Singleton { + return &allTestSpecsSingleton{} +} + +type allTestSpecsSingleton struct { + // Path where the collected metadata is stored after successful validation. + outputPath android.OutputPath +} + +func (this *allTestSpecsSingleton) GenerateBuildActions(ctx android.SingletonContext) { + var intermediateMetadataPaths android.Paths + + ctx.VisitAllModules(func(module android.Module) { + if !ctx.ModuleHasProvider(module, TestSpecProviderKey) { + return + } + intermediateMetadataPaths = append(intermediateMetadataPaths, ctx.ModuleProvider(module, TestSpecProviderKey).(TestSpecProviderData).IntermediatePath) + }) + + rspFile := android.PathForOutput(ctx, fileContainingFilePaths) + this.outputPath = android.PathForOutput(ctx, ownershipDirectory, allTestSpecsFile) + + rule := android.NewRuleBuilder(pctx, ctx) + cmd := rule.Command(). + BuiltTool("metadata"). + FlagWithArg("-rule ", "test_spec"). + FlagWithRspFileInputList("-inputFile ", rspFile, intermediateMetadataPaths) + cmd.FlagWithOutput("-outputFile ", this.outputPath) + rule.Build("all_test_specs_rule", "Generate all test specifications") + ctx.Phony("all_test_specs", this.outputPath) +} + +func (this *allTestSpecsSingleton) MakeVars(ctx android.MakeVarsContext) { + ctx.DistForGoal("test_specs", this.outputPath) +} diff --git a/testing/init.go b/testing/init.go new file mode 100644 index 000000000..206b4306f --- /dev/null +++ b/testing/init.go @@ -0,0 +1,33 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testing + +import ( + "android/soong/android" +) + +var ( + pctx = android.NewPackageContext("android/soong/testing") +) + +func init() { + RegisterBuildComponents(android.InitRegistrationContext) + pctx.HostBinToolVariable("metadata", "metadata") +} + +func RegisterBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("test_spec", TestSpecFactory) + ctx.RegisterParallelSingletonType("all_test_specs", AllTestSpecsFactory) +} diff --git a/testing/test.go b/testing/test.go new file mode 100644 index 000000000..44824e4db --- /dev/null +++ b/testing/test.go @@ -0,0 +1,21 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testing + +import ( + "android/soong/android" +) + +var PrepareForTestWithTestSpecBuildComponents = android.FixtureRegisterWithContext(RegisterBuildComponents) diff --git a/testing/test_spec.go b/testing/test_spec.go new file mode 100644 index 000000000..c370f7174 --- /dev/null +++ b/testing/test_spec.go @@ -0,0 +1,127 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testing + +import ( + "path/filepath" + "strconv" + + "android/soong/android" + "android/soong/testing/test_spec_proto" + "github.com/google/blueprint" + "google.golang.org/protobuf/proto" +) + +// ErrTestModuleDataNotFound is the error message for missing test module provider data. +const ErrTestModuleDataNotFound = "The module '%s' does not provide test specification data. Hint: This issue could arise if either the module is not a valid testing module or if it lacks the required 'TestModuleProviderKey' provider.\n" + +func TestSpecFactory() android.Module { + module := &TestSpecModule{} + + android.InitAndroidModule(module) + android.InitDefaultableModule(module) + module.AddProperties(&module.properties) + + return module +} + +type TestSpecModule struct { + android.ModuleBase + android.DefaultableModuleBase + android.BazelModuleBase + + // Properties for "test_spec" + properties struct { + // Specifies the name of the test config. + Name string + // Specifies the team ID. + TeamId string + // Specifies the list of tests covered under this module. + Tests []string + } +} + +type testsDepTagType struct { + blueprint.BaseDependencyTag +} + +var testsDepTag = testsDepTagType{} + +func (module *TestSpecModule) DepsMutator(ctx android.BottomUpMutatorContext) { + // Validate Properties + if len(module.properties.TeamId) == 0 { + ctx.PropertyErrorf("TeamId", "Team Id not found in the test_spec module. Hint: Maybe the TeamId property hasn't been properly specified.") + } + if !isInt(module.properties.TeamId) { + ctx.PropertyErrorf("TeamId", "Invalid value for Team ID. The Team ID must be an integer.") + } + if len(module.properties.Tests) == 0 { + ctx.PropertyErrorf("Tests", "Expected to attribute some test but none found. Hint: Maybe the test property hasn't been properly specified.") + } + ctx.AddDependency(ctx.Module(), testsDepTag, module.properties.Tests...) +} +func isInt(s string) bool { + _, err := strconv.Atoi(s) + return err == nil +} + +// Provider published by TestSpec +type TestSpecProviderData struct { + IntermediatePath android.WritablePath +} + +var TestSpecProviderKey = blueprint.NewProvider(TestSpecProviderData{}) + +type TestModuleProviderData struct { +} + +var TestModuleProviderKey = blueprint.NewProvider(TestModuleProviderData{}) + +func (module *TestSpecModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + for _, m := range ctx.GetDirectDepsWithTag(testsDepTag) { + if !ctx.OtherModuleHasProvider(m, TestModuleProviderKey) { + ctx.ModuleErrorf(ErrTestModuleDataNotFound, m.Name()) + } + } + bpFilePath := filepath.Join(ctx.ModuleDir(), ctx.BlueprintsFile()) + metadataList := make( + []*test_spec_proto.TestSpec_OwnershipMetadata, 0, + len(module.properties.Tests), + ) + for _, test := range module.properties.Tests { + targetName := test + metadata := test_spec_proto.TestSpec_OwnershipMetadata{ + TrendyTeamId: &module.properties.TeamId, + TargetName: &targetName, + Path: &bpFilePath, + } + metadataList = append(metadataList, &metadata) + } + intermediatePath := android.PathForModuleOut( + ctx, "intermediateTestSpecMetadata.pb", + ) + testSpecMetadata := test_spec_proto.TestSpec{OwnershipMetadataList: metadataList} + protoData, err := proto.Marshal(&testSpecMetadata) + if err != nil { + ctx.ModuleErrorf("Error: %s", err.Error()) + } + android.WriteFileRule(ctx, intermediatePath, string(protoData)) + + ctx.SetProvider( + TestSpecProviderKey, TestSpecProviderData{ + IntermediatePath: intermediatePath, + }, + ) +} diff --git a/testing/test_spec_proto/Android.bp b/testing/test_spec_proto/Android.bp new file mode 100644 index 000000000..1cac492f1 --- /dev/null +++ b/testing/test_spec_proto/Android.bp @@ -0,0 +1,29 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-testing-test_spec_proto", + pkgPath: "android/soong/testing/test_spec_proto", + deps: [ + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + ], + srcs: [ + "test_spec.pb.go", + ], +} diff --git a/testing/test_spec_proto/OWNERS b/testing/test_spec_proto/OWNERS new file mode 100644 index 000000000..03bcdf1c4 --- /dev/null +++ b/testing/test_spec_proto/OWNERS @@ -0,0 +1,4 @@ +dariofreni@google.com +joeo@google.com +ronish@google.com +caditya@google.com diff --git a/testing/test_spec_proto/regen.sh b/testing/test_spec_proto/regen.sh new file mode 100644 index 000000000..2cf820375 --- /dev/null +++ b/testing/test_spec_proto/regen.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +aprotoc --go_out=paths=source_relative:. test_spec.proto diff --git a/testing/test_spec_proto/test_spec.pb.go b/testing/test_spec_proto/test_spec.pb.go new file mode 100644 index 000000000..5cce60029 --- /dev/null +++ b/testing/test_spec_proto/test_spec.pb.go @@ -0,0 +1,244 @@ +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.30.0 +// protoc v3.21.12 +// source: test_spec.proto + +package test_spec_proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type TestSpec struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // List of all test targets and their metadata. + OwnershipMetadataList []*TestSpec_OwnershipMetadata `protobuf:"bytes,1,rep,name=ownership_metadata_list,json=ownershipMetadataList" json:"ownership_metadata_list,omitempty"` +} + +func (x *TestSpec) Reset() { + *x = TestSpec{} + if protoimpl.UnsafeEnabled { + mi := &file_test_spec_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TestSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TestSpec) ProtoMessage() {} + +func (x *TestSpec) ProtoReflect() protoreflect.Message { + mi := &file_test_spec_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TestSpec.ProtoReflect.Descriptor instead. +func (*TestSpec) Descriptor() ([]byte, []int) { + return file_test_spec_proto_rawDescGZIP(), []int{0} +} + +func (x *TestSpec) GetOwnershipMetadataList() []*TestSpec_OwnershipMetadata { + if x != nil { + return x.OwnershipMetadataList + } + return nil +} + +type TestSpec_OwnershipMetadata struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TargetName *string `protobuf:"bytes,1,opt,name=target_name,json=targetName" json:"target_name,omitempty"` + Path *string `protobuf:"bytes,2,opt,name=path" json:"path,omitempty"` + TrendyTeamId *string `protobuf:"bytes,3,opt,name=trendy_team_id,json=trendyTeamId" json:"trendy_team_id,omitempty"` +} + +func (x *TestSpec_OwnershipMetadata) Reset() { + *x = TestSpec_OwnershipMetadata{} + if protoimpl.UnsafeEnabled { + mi := &file_test_spec_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TestSpec_OwnershipMetadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TestSpec_OwnershipMetadata) ProtoMessage() {} + +func (x *TestSpec_OwnershipMetadata) ProtoReflect() protoreflect.Message { + mi := &file_test_spec_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TestSpec_OwnershipMetadata.ProtoReflect.Descriptor instead. +func (*TestSpec_OwnershipMetadata) Descriptor() ([]byte, []int) { + return file_test_spec_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *TestSpec_OwnershipMetadata) GetTargetName() string { + if x != nil && x.TargetName != nil { + return *x.TargetName + } + return "" +} + +func (x *TestSpec_OwnershipMetadata) GetPath() string { + if x != nil && x.Path != nil { + return *x.Path + } + return "" +} + +func (x *TestSpec_OwnershipMetadata) GetTrendyTeamId() string { + if x != nil && x.TrendyTeamId != nil { + return *x.TrendyTeamId + } + return "" +} + +var File_test_spec_proto protoreflect.FileDescriptor + +var file_test_spec_proto_rawDesc = []byte{ + 0x0a, 0x0f, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x0f, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x22, 0xdf, 0x01, 0x0a, 0x08, 0x54, 0x65, 0x73, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, + 0x63, 0x0a, 0x17, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x73, 0x68, 0x69, 0x70, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x2b, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x4f, 0x77, 0x6e, 0x65, + 0x72, 0x73, 0x68, 0x69, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x15, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x73, 0x68, 0x69, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x4c, 0x69, 0x73, 0x74, 0x1a, 0x6e, 0x0a, 0x11, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x73, 0x68, 0x69, + 0x70, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x24, + 0x0a, 0x0e, 0x74, 0x72, 0x65, 0x6e, 0x64, 0x79, 0x5f, 0x74, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x74, 0x72, 0x65, 0x6e, 0x64, 0x79, 0x54, 0x65, + 0x61, 0x6d, 0x49, 0x64, 0x42, 0x27, 0x5a, 0x25, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, + 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, + 0x73, 0x74, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, +} + +var ( + file_test_spec_proto_rawDescOnce sync.Once + file_test_spec_proto_rawDescData = file_test_spec_proto_rawDesc +) + +func file_test_spec_proto_rawDescGZIP() []byte { + file_test_spec_proto_rawDescOnce.Do(func() { + file_test_spec_proto_rawDescData = protoimpl.X.CompressGZIP(file_test_spec_proto_rawDescData) + }) + return file_test_spec_proto_rawDescData +} + +var file_test_spec_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_test_spec_proto_goTypes = []interface{}{ + (*TestSpec)(nil), // 0: test_spec_proto.TestSpec + (*TestSpec_OwnershipMetadata)(nil), // 1: test_spec_proto.TestSpec.OwnershipMetadata +} +var file_test_spec_proto_depIdxs = []int32{ + 1, // 0: test_spec_proto.TestSpec.ownership_metadata_list:type_name -> test_spec_proto.TestSpec.OwnershipMetadata + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_test_spec_proto_init() } +func file_test_spec_proto_init() { + if File_test_spec_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_test_spec_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TestSpec); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_test_spec_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TestSpec_OwnershipMetadata); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_test_spec_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_test_spec_proto_goTypes, + DependencyIndexes: file_test_spec_proto_depIdxs, + MessageInfos: file_test_spec_proto_msgTypes, + }.Build() + File_test_spec_proto = out.File + file_test_spec_proto_rawDesc = nil + file_test_spec_proto_goTypes = nil + file_test_spec_proto_depIdxs = nil +} diff --git a/testing/test_spec_proto/test_spec.proto b/testing/test_spec_proto/test_spec.proto new file mode 100644 index 000000000..86bc78954 --- /dev/null +++ b/testing/test_spec_proto/test_spec.proto @@ -0,0 +1,33 @@ +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; +package test_spec_proto; +option go_package = "android/soong/testing/test_spec_proto"; + +message TestSpec { + + message OwnershipMetadata { + // REQUIRED: Name of the build target + optional string target_name = 1; + + // REQUIRED: Code location of the target. + // To be used to support legacy/backup systems that use OWNERS file and is + // also required for our dashboard to support per code location basis UI + optional string path = 2; + + // REQUIRED: Team ID of the team that owns this target. + optional string trendy_team_id = 3; + } + + // List of all test targets and their metadata. + repeated OwnershipMetadata ownership_metadata_list = 1; +} diff --git a/tests/genrule_sandbox_test.py b/tests/genrule_sandbox_test.py index 874859ad9..3799e92f8 100755 --- a/tests/genrule_sandbox_test.py +++ b/tests/genrule_sandbox_test.py @@ -15,12 +15,14 @@ # limitations under the License. import argparse +import asyncio import collections import json import os +import socket import subprocess import sys -import tempfile +import textwrap def get_top() -> str: path = '.' @@ -30,39 +32,65 @@ def get_top() -> str: path = os.path.join(path, '..') return os.path.abspath(path) -def _build_with_soong(targets, target_product, *, keep_going = False, extra_env={}): - env = { - **os.environ, - "TARGET_PRODUCT": target_product, - "TARGET_BUILD_VARIANT": "userdebug", - } - env.update(extra_env) +async def _build_with_soong(out_dir, targets, *, extra_env={}): + env = os.environ | extra_env + + # Use nsjail to remap the out_dir to out/, because some genrules write the path to the out + # dir into their artifacts, so if the out directories were different it would cause a diff + # that doesn't really matter. args = [ + 'prebuilts/build-tools/linux-x86/bin/nsjail', + '-q', + '--cwd', + os.getcwd(), + '-e', + '-B', + '/', + '-B', + f'{os.path.abspath(out_dir)}:{os.path.abspath("out")}', + '--time_limit', + '0', + '--skip_setsid', + '--keep_caps', + '--disable_clone_newcgroup', + '--disable_clone_newnet', + '--rlimit_as', + 'soft', + '--rlimit_core', + 'soft', + '--rlimit_cpu', + 'soft', + '--rlimit_fsize', + 'soft', + '--rlimit_nofile', + 'soft', + '--proc_rw', + '--hostname', + socket.gethostname(), + '--', "build/soong/soong_ui.bash", "--make-mode", "--skip-soong-tests", ] - if keep_going: - args.append("-k") args.extend(targets) - try: - subprocess.check_output( - args, - env=env, - ) - except subprocess.CalledProcessError as e: - print(e) - print(e.stdout) - print(e.stderr) - exit(1) + process = await asyncio.create_subprocess_exec( + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + stdout, stderr = await process.communicate() + if process.returncode != 0: + print(stdout) + print(stderr) + sys.exit(process.returncode) -def _find_outputs_for_modules(modules, out_dir, target_product): - module_path = os.path.join(out_dir, "soong", "module-actions.json") +async def _find_outputs_for_modules(modules): + module_path = "out/soong/module-actions.json" if not os.path.exists(module_path): - # Use GENRULE_SANDBOXING=false so that we don't cause re-analysis later when we do the no-sandboxing build - _build_with_soong(["json-module-graph"], target_product, extra_env={"GENRULE_SANDBOXING": "false"}) + await _build_with_soong('out', ["json-module-graph"]) with open(module_path) as f: action_graph = json.load(f) @@ -71,7 +99,7 @@ def _find_outputs_for_modules(modules, out_dir, target_product): for mod in action_graph: name = mod["Name"] if name in modules: - for act in mod["Module"]["Actions"]: + for act in (mod["Module"]["Actions"] or []): if "}generate" in act["Desc"]: module_to_outs[name].update(act["Outputs"]) return module_to_outs @@ -89,20 +117,19 @@ def _compare_outputs(module_to_outs, tempdir) -> dict[str, list[str]]: return different_modules -def main(): +async def main(): parser = argparse.ArgumentParser() parser.add_argument( - "--target_product", - "-t", - default="aosp_cf_arm64_phone", - help="optional, target product, always runs as eng", - ) - parser.add_argument( "modules", nargs="+", help="modules to compare builds with genrule sandboxing enabled/not", ) parser.add_argument( + "--check-determinism", + action="store_true", + help="Don't check for working sandboxing. Instead, run two default builds, and compare their outputs. This is used to check for nondeterminsim, which would also affect the sandboxed test.", + ) + parser.add_argument( "--show-diff", "-d", action="store_true", @@ -117,10 +144,13 @@ def main(): args = parser.parse_args() os.chdir(get_top()) - out_dir = os.environ.get("OUT_DIR", "out") + if "TARGET_PRODUCT" not in os.environ: + sys.exit("Please run lunch first") + if os.environ.get("OUT_DIR", "out") != "out": + sys.exit(f"This script expects OUT_DIR to be 'out', got: '{os.environ.get('OUT_DIR')}'") print("finding output files for the modules...") - module_to_outs = _find_outputs_for_modules(set(args.modules), out_dir, args.target_product) + module_to_outs = await _find_outputs_for_modules(set(args.modules)) if not module_to_outs: sys.exit("No outputs found") @@ -130,33 +160,48 @@ def main(): sys.exit(0) all_outs = list(set.union(*module_to_outs.values())) + for i, out in enumerate(all_outs): + if not out.startswith("out/"): + sys.exit("Expected output file to start with out/, found: " + out) + + other_out_dir = "out_check_determinism" if args.check_determinism else "out_not_sandboxed" + other_env = {"GENRULE_SANDBOXING": "false"} + if args.check_determinism: + other_env = {} + + # nsjail will complain if the out dir doesn't exist + os.makedirs("out", exist_ok=True) + os.makedirs(other_out_dir, exist_ok=True) + + print("building...") + await asyncio.gather( + _build_with_soong("out", all_outs), + _build_with_soong(other_out_dir, all_outs, extra_env=other_env) + ) + + diffs = collections.defaultdict(dict) + for module, outs in module_to_outs.items(): + for out in outs: + try: + subprocess.check_output(["diff", os.path.join(other_out_dir, out.removeprefix("out/")), out]) + except subprocess.CalledProcessError as e: + diffs[module][out] = e.stdout + + if len(diffs) == 0: + print("All modules are correct") + elif args.show_diff: + for m, files in diffs.items(): + print(f"Module {m} has diffs:") + for f, d in files.items(): + print(" "+f+":") + print(textwrap.indent(d, " ")) + else: + print(f"Modules {list(diffs.keys())} have diffs in these files:") + all_diff_files = [f for m in diffs.values() for f in m] + for f in all_diff_files: + print(f) - print("building without sandboxing...") - _build_with_soong(all_outs, args.target_product, extra_env={"GENRULE_SANDBOXING": "false"}) - with tempfile.TemporaryDirectory() as tempdir: - for f in all_outs: - subprocess.check_call(["cp", "--parents", f, tempdir]) - - print("building with sandboxing...") - _build_with_soong( - all_outs, - args.target_product, - # We've verified these build without sandboxing already, so do the sandboxing build - # with keep_going = True so that we can find all the genrules that fail to build with - # sandboxing. - keep_going = True, - extra_env={"GENRULE_SANDBOXING": "true"}, - ) - - diffs = _compare_outputs(module_to_outs, tempdir) - if len(diffs) == 0: - print("All modules are correct") - elif args.show_diff: - for m, d in diffs.items(): - print(f"Module {m} has diffs {d}") - else: - print(f"Modules {list(diffs.keys())} have diffs") if __name__ == "__main__": - main() + asyncio.run(main()) diff --git a/ui/build/config.go b/ui/build/config.go index 20d920485..d345415b5 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -86,7 +86,7 @@ type configImpl struct { searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces skipMetricsUpload bool buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time - buildFromTextStub bool + buildFromSourceStub bool ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built bazelExitCode int32 // For b runs - necessary for updating NonZeroExit besId string // For b runs, to identify the BuildEventService logs @@ -820,8 +820,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) { } else { ctx.Fatalf("unknown option for ninja_weight_source: %s", source) } - } else if arg == "--build-from-text-stub" { - c.buildFromTextStub = true + } else if arg == "--build-from-source-stub" { + c.buildFromSourceStub = true } else if strings.HasPrefix(arg, "--build-command=") { buildCmd := strings.TrimPrefix(arg, "--build-command=") // remove quotations @@ -1156,7 +1156,7 @@ func (c *configImpl) SkipConfig() bool { } func (c *configImpl) BuildFromTextStub() bool { - return c.buildFromTextStub + return !c.buildFromSourceStub } func (c *configImpl) TargetProduct() string { diff --git a/ui/build/soong.go b/ui/build/soong.go index 90f1798f3..667f0c90b 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -193,8 +193,8 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuil if pb.config.multitreeBuild { commonArgs = append(commonArgs, "--multitree-build") } - if pb.config.buildFromTextStub { - commonArgs = append(commonArgs, "--build-from-text-stub") + if pb.config.buildFromSourceStub { + commonArgs = append(commonArgs, "--build-from-source-stub") } commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list")) @@ -310,8 +310,8 @@ func bootstrapBlueprint(ctx Context, config Config) { if config.MultitreeBuild() { mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--multitree-build") } - if config.buildFromTextStub { - mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--build-from-text-stub") + if config.buildFromSourceStub { + mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--build-from-source-stub") } if config.ensureAllowlistIntegrity { mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--ensure-allowlist-integrity") |