diff options
87 files changed, 3960 insertions, 2785 deletions
diff --git a/.gitignore b/.gitignore index 5d2bc0d05..89de74ee1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.iml *.ipr *.iws - +*.swp +/.vscode @@ -449,6 +449,7 @@ soong_config_module_type { config_namespace: "acme", variables: ["board"], bool_variables: ["feature"], + list_variables: ["impl"], value_variables: ["width"], properties: ["cflags", "srcs"], } @@ -460,24 +461,40 @@ soong_config_string_variable { ``` This example describes a new `acme_cc_defaults` module type that extends the -`cc_defaults` module type, with three additional conditionals based on -variables `board`, `feature` and `width`, which can affect properties `cflags` -and `srcs`. Additionally, each conditional will contain a `conditions_default` -property can affect `cflags` and `srcs` in the following conditions: - -* bool variable (e.g. `feature`): the variable is unspecified or not set to a true value +`cc_defaults` module type, with four additional conditionals based on variables +`board`, `feature`, `impl` and `width` which can affect properties `cflags` and +`srcs`. The four types of soong variables control properties in the following +ways. + +* bool variable (e.g. `feature`): Properties are applied if set to `true`. +* list variable (e.g. `impl`): (lists of strings properties only) Properties are + applied for each value in the list, using `%s` substitution. For example, if + the property is `["%s.cpp", "%s.h"]` and the list value is `foo bar`, + the result is `["foo.cpp", "foo.h", "bar.cpp", "bar.h"]`. +* value variable (e.g. `width`): (strings or lists of strings) The value are + directly substituted into properties using `%s`. +* string variable (e.g. `board`): Properties are applied only if they match the + variable's value. + +Additionally, each conditional containing a `conditions_default` property can +affect `cflags` and `srcs` in the following conditions: + +* bool variable (e.g. `feature`): the variable is unspecified or not set to + `true` +* list variable (e.g. `impl`): the variable is unspecified * value variable (e.g. `width`): the variable is unspecified -* string variable (e.g. `board`): the variable is unspecified or the variable is set to a string unused in the -given module. For example, with `board`, if the `board` -conditional contains the properties `soc_a` and `conditions_default`, when -board=soc_b, the `cflags` and `srcs` values under `conditions_default` will be -used. To specify that no properties should be amended for `soc_b`, you can set -`soc_b: {},`. +* string variable (e.g. `board`): the variable is unspecified or the variable is + set to a string unused in the given module. For example, with `board`, if the + `board` conditional contains the properties `soc_a` and `conditions_default`, + when `board` is `soc_b`, the `cflags` and `srcs` values under + `conditions_default` is used. To specify that no properties should be amended + for `soc_b`, you can set `soc_b: {},`. The values of the variables can be set from a product's `BoardConfig.mk` file: ``` $(call soong_config_set,acme,board,soc_a) $(call soong_config_set,acme,feature,true) +$(call soong_config_set,acme,impl,foo.cpp bar.cpp) $(call soong_config_set,acme,width,200) ``` @@ -519,6 +536,12 @@ acme_cc_defaults { cflags: ["-DWIDTH=DEFAULT"], }, }, + impl: { + srcs: ["impl/%s"], + conditions_default: { + srcs: ["impl/default.cpp"], + }, + }, }, } @@ -530,7 +553,8 @@ cc_library { ``` With the `BoardConfig.mk` snippet above, `libacme_foo` would build with -`cflags: "-DGENERIC -DSOC_A -DFEATURE -DWIDTH=200"`. +`cflags: "-DGENERIC -DSOC_A -DFEATURE -DWIDTH=200"` and +`srcs: ["*.cpp", "impl/foo.cpp", "impl/bar.cpp"]`. Alternatively, with `DefaultBoardConfig.mk`: @@ -539,12 +563,14 @@ SOONG_CONFIG_NAMESPACES += acme SOONG_CONFIG_acme += \ board \ feature \ + impl \ width \ SOONG_CONFIG_acme_feature := false ``` -then `libacme_foo` would build with `cflags: "-DGENERIC -DSOC_DEFAULT -DFEATURE_DEFAULT -DSIZE=DEFAULT"`. +then `libacme_foo` would build with `cflags: "-DGENERIC -DSOC_DEFAULT -DFEATURE_DEFAULT -DSIZE=DEFAULT"` +and `srcs: ["*.cpp", "impl/default.cpp"]`. Alternatively, with `DefaultBoardConfig.mk`: @@ -553,13 +579,15 @@ SOONG_CONFIG_NAMESPACES += acme SOONG_CONFIG_acme += \ board \ feature \ + impl \ width \ SOONG_CONFIG_acme_board := soc_c +SOONG_CONFIG_acme_impl := baz ``` then `libacme_foo` would build with `cflags: "-DGENERIC -DSOC_DEFAULT --DFEATURE_DEFAULT -DSIZE=DEFAULT"`. +-DFEATURE_DEFAULT -DSIZE=DEFAULT"` and `srcs: ["*.cpp", "impl/baz.cpp"]`. `soong_config_module_type` modules will work best when used to wrap defaults modules (`cc_defaults`, `java_defaults`, etc.), which can then be referenced diff --git a/aconfig/codegen/cc_aconfig_library.go b/aconfig/codegen/cc_aconfig_library.go index 80e492620..2d3ee39f9 100644 --- a/aconfig/codegen/cc_aconfig_library.go +++ b/aconfig/codegen/cc_aconfig_library.go @@ -33,6 +33,11 @@ var ccDeclarationsTag = ccDeclarationsTagType{} const baseLibDep = "server_configurable_flags" +const libBaseDep = "libbase" +const libLogDep = "liblog" +const libAconfigStorageReadApiCcDep = "libaconfig_storage_read_api_cc" +const libAconfigStorageProtosCcDep = "libaconfig_storage_protos_cc" + type CcAconfigLibraryProperties struct { // name of the aconfig_declarations module to generate a library for Aconfig_declarations string @@ -82,6 +87,11 @@ func (this *CcAconfigLibraryCallbacks) GeneratorDeps(ctx cc.DepsContext, deps cc // Add a dependency for the aconfig flags base library if it is not forced read only if mode != "force-read-only" { deps.SharedLibs = append(deps.SharedLibs, baseLibDep) + + deps.SharedLibs = append(deps.SharedLibs, libBaseDep) + deps.SharedLibs = append(deps.SharedLibs, libLogDep) + deps.SharedLibs = append(deps.SharedLibs, libAconfigStorageReadApiCcDep) + deps.SharedLibs = append(deps.SharedLibs, libAconfigStorageProtosCcDep) } // TODO: It'd be really nice if we could reexport this library and not make everyone do it. diff --git a/aconfig/codegen/cc_aconfig_library_test.go b/aconfig/codegen/cc_aconfig_library_test.go index 05449bc6c..2e7fdc2c1 100644 --- a/aconfig/codegen/cc_aconfig_library_test.go +++ b/aconfig/codegen/cc_aconfig_library_test.go @@ -58,6 +58,26 @@ func testCCCodegenModeHelper(t *testing.T, bpMode string, ruleMode string) { srcs: ["server_configurable_flags.cc"], } + cc_library { + name: "libbase", + srcs: ["libbase.cc"], + } + + cc_library { + name: "liblog", + srcs: ["liblog.cc"], + } + + cc_library { + name: "libaconfig_storage_read_api_cc", + srcs: ["libaconfig_storage_read_api_cc.cc"], + } + + cc_library { + name: "libaconfig_storage_protos_cc", + srcs: ["libaconfig_storage_protos_cc.cc"], + } + cc_aconfig_library { name: "my_cc_aconfig_library", aconfig_declarations: "my_aconfig_declarations", @@ -100,6 +120,27 @@ func testIncorrectCCCodegenModeHelper(t *testing.T, bpMode string, err string) { srcs: ["server_configurable_flags.cc"], } + cc_library { + name: "libbase", + srcs: ["libbase.cc"], + } + + cc_library { + name: "liblog", + srcs: ["liblog.cc"], + } + + cc_library { + name: "libaconfig_storage_read_api_cc", + srcs: ["libaconfig_storage_read_api_cc.cc"], + } + + cc_library { + name: "libaconfig_storage_protos_cc", + srcs: ["libaconfig_storage_protos_cc.cc"], + } + + cc_aconfig_library { name: "my_cc_aconfig_library", aconfig_declarations: "my_aconfig_declarations", @@ -152,6 +193,30 @@ func TestAndroidMkCcLibrary(t *testing.T) { srcs: ["server_configurable_flags.cc"], vendor_available: true, } + + cc_library { + name: "libbase", + srcs: ["libbase.cc"], + vendor_available: true, + } + + cc_library { + name: "liblog", + srcs: ["liblog.cc"], + vendor_available: true, + } + + cc_library { + name: "libaconfig_storage_read_api_cc", + srcs: ["libaconfig_storage_read_api_cc.cc"], + vendor_available: true, + } + + cc_library { + name: "libaconfig_storage_protos_cc", + srcs: ["libaconfig_storage_protos_cc.cc"], + vendor_available: true, + } ` result := android.GroupFixturePreparers( PrepareForTestWithAconfigBuildComponents, diff --git a/android/Android.bp b/android/Android.bp index 4c59592c8..f130d3aee 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -138,6 +138,7 @@ bootstrap_go_package { "selects_test.go", "singleton_module_test.go", "soong_config_modules_test.go", + "test_suites_test.go", "util_test.go", "variable_test.go", "visibility_test.go", diff --git a/android/all_teams.go b/android/all_teams.go index 0c433a6cb..d4bf7d0c4 100644 --- a/android/all_teams.go +++ b/android/all_teams.go @@ -79,11 +79,6 @@ func (t *allTeamsSingleton) GenerateBuildActions(ctx SingletonContext) { ctx.VisitAllModules(func(module Module) { bpFile := ctx.BlueprintFile(module) - testModInfo := TestModuleInformation{} - if tmi, ok := SingletonModuleProvider(ctx, module, TestOnlyProviderKey); ok { - testModInfo = tmi - } - // Package Modules and Team Modules are stored in a map so we can look them up by name for // modules without a team. if pack, ok := module.(*packageModule); ok { @@ -97,6 +92,23 @@ func (t *allTeamsSingleton) GenerateBuildActions(ctx SingletonContext) { return } + testModInfo := TestModuleInformation{} + if tmi, ok := SingletonModuleProvider(ctx, module, TestOnlyProviderKey); ok { + testModInfo = tmi + } + + // Some modules, like java_test_host don't set the provider when the module isn't enabled: + // test_only, top_level + // AVFHostTestCases{os:linux_glibc,arch:common} {true true} + // AVFHostTestCases{os:windows,arch:common} {false false} + // Generally variant information of true override false or unset. + if testModInfo.TestOnly == false { + if prevValue, exists := t.teams_for_mods[module.Name()]; exists { + if prevValue.testOnly == true { + return + } + } + } entry := moduleTeamAndTestInfo{ bpFile: bpFile, testOnly: testModInfo.TestOnly, diff --git a/android/all_teams_test.go b/android/all_teams_test.go index 9c2b38e42..96ed92fc5 100644 --- a/android/all_teams_test.go +++ b/android/all_teams_test.go @@ -25,6 +25,8 @@ func TestAllTeams(t *testing.T) { t.Parallel() ctx := GroupFixturePreparers( prepareForTestWithTeamAndFakes, + // This adds two variants, one armv7-a-neon, one armv8-a + PrepareForTestWithArchMutator, FixtureRegisterWithContext(func(ctx RegistrationContext) { ctx.RegisterParallelSingletonType("all_teams", AllTeamsFactory) }), @@ -52,10 +54,35 @@ func TestAllTeams(t *testing.T) { name: "noteam", test_only: true, } + // write the test-only provider value once fake { - name: "test-and-team-and-top", + name: "test-and-team-and-top1", test_only: true, team: "team2", + arch: {arm: { skip: false}, + arm64: { skip: true}}, + } + // write the test-only provider once, but on the other arch + fake { + name: "test-and-team-and-top2", + test_only: true, + team: "team2", + arch: {arm: { skip: true}, + arm64: { skip: false}}, + } + // write the test-only provider value twice + fake { + name: "test-and-team-and-top3", + test_only: true, + team: "team2", + } + // Don't write the test-only provider value + fake { + name: "test-and-team-and-top4", + test_only: true, + team: "team2", + arch: {arm: { skip: true}, + arm64: { skip: true}}, } `) @@ -63,12 +90,16 @@ func TestAllTeams(t *testing.T) { teams = getTeamProtoOutput(t, ctx) // map of module name -> trendy team name. - actualTeams := make(map[string]*string) + actualTeams := make(map[string]string) actualTests := []string{} actualTopLevelTests := []string{} for _, teamProto := range teams.Teams { - actualTeams[teamProto.GetTargetName()] = teamProto.TrendyTeamId + if teamProto.TrendyTeamId != nil { + actualTeams[teamProto.GetTargetName()] = *teamProto.TrendyTeamId + } else { + actualTeams[teamProto.GetTargetName()] = "" + } if teamProto.GetTestOnly() { actualTests = append(actualTests, teamProto.GetTargetName()) } @@ -76,16 +107,23 @@ func TestAllTeams(t *testing.T) { actualTopLevelTests = append(actualTopLevelTests, teamProto.GetTargetName()) } } - expectedTeams := map[string]*string{ - "main_test": proto.String("cool_team"), - "tool": proto.String("22222"), - "test-and-team-and-top": proto.String("22222"), - "noteam": nil, + expectedTeams := map[string]string{ + "main_test": "cool_team", + "tool": "22222", + "test-and-team-and-top1": "22222", + "test-and-team-and-top2": "22222", + "test-and-team-and-top3": "22222", + "test-and-team-and-top4": "22222", + "noteam": "", } expectedTests := []string{ "noteam", - "test-and-team-and-top", + "test-and-team-and-top1", + "test-and-team-and-top2", + "test-and-team-and-top3", + // There should be no test-and-team-top4 as we skip writing all variants + // test-only for all variants } AssertDeepEquals(t, "compare maps", expectedTeams, actualTeams) AssertDeepEquals(t, "test matchup", expectedTests, actualTests) @@ -230,12 +268,16 @@ type fakeForTests struct { ModuleBase sourceProperties SourceProperties + props struct { + // If true, don't write test-only value in provider + Skip bool `android:"arch_variant"` + } } func fakeFactory() Module { module := &fakeForTests{} - module.AddProperties(&module.sourceProperties) - InitAndroidModule(module) + module.AddProperties(&module.sourceProperties, &module.props) + InitAndroidArchModule(module, HostAndDeviceSupported, MultilibBoth) return module } @@ -250,7 +292,7 @@ var prepareForTestWithTeamAndFakes = GroupFixturePreparers( func (f *fakeForTests) GenerateAndroidBuildActions(ctx ModuleContext) { if Bool(f.sourceProperties.Test_only) { SetProvider(ctx, TestOnlyProviderKey, TestModuleInformation{ - TestOnly: Bool(f.sourceProperties.Test_only), + TestOnly: Bool(f.sourceProperties.Test_only) && !f.props.Skip, TopLevelTarget: false, }) } diff --git a/android/apex.go b/android/apex.go index 87599051a..dc0aeed17 100644 --- a/android/apex.go +++ b/android/apex.go @@ -16,6 +16,7 @@ package android import ( "fmt" + "slices" "sort" "strconv" "strings" @@ -89,7 +90,13 @@ type ApexInfo struct { TestApexes []string } -var ApexInfoProvider = blueprint.NewMutatorProvider[ApexInfo]("apex") +// AllApexInfo holds the ApexInfo of all apexes that include this module. +type AllApexInfo struct { + ApexInfos []ApexInfo +} + +var ApexInfoProvider = blueprint.NewMutatorProvider[ApexInfo]("apex_mutate") +var AllApexInfoProvider = blueprint.NewMutatorProvider[*AllApexInfo]("apex_info") func (i ApexInfo) AddJSONData(d *map[string]interface{}) { (*d)["Apex"] = map[string]interface{}{ @@ -108,7 +115,7 @@ func (i ApexInfo) AddJSONData(d *map[string]interface{}) { // are configured to have the same alias variation named apex29. Whether platform APIs is allowed // or not also matters; if two APEXes don't have the same allowance, they get different names and // thus wouldn't be merged. -func (i ApexInfo) mergedName(ctx PathContext) string { +func (i ApexInfo) mergedName() string { name := "apex" + strconv.Itoa(i.MinSdkVersion.FinalOrFutureInt()) return name } @@ -543,17 +550,10 @@ func AvailableToSameApexes(mod1, mod2 ApexModule) bool { return true } -type byApexName []ApexInfo - -func (a byApexName) Len() int { return len(a) } -func (a byApexName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a byApexName) Less(i, j int) bool { return a[i].ApexVariationName < a[j].ApexVariationName } - // mergeApexVariations deduplicates apex variations that would build identically into a common // variation. It returns the reduced list of variations and a list of aliases from the original // variation names to the new variation names. -func mergeApexVariations(ctx PathContext, apexInfos []ApexInfo) (merged []ApexInfo, aliases [][2]string) { - sort.Sort(byApexName(apexInfos)) +func mergeApexVariations(apexInfos []ApexInfo) (merged []ApexInfo, aliases [][2]string) { seen := make(map[string]int) for _, apexInfo := range apexInfos { // If this is for a prebuilt apex then use the actual name of the apex variation to prevent this @@ -567,7 +567,7 @@ func mergeApexVariations(ctx PathContext, apexInfos []ApexInfo) (merged []ApexIn // this one into it, otherwise create a new merged ApexInfo from this one and save it away so // other ApexInfo instances can be merged into it. variantName := apexInfo.ApexVariationName - mergedName := apexInfo.mergedName(ctx) + mergedName := apexInfo.mergedName() if index, exists := seen[mergedName]; exists { // Variants having the same mergedName are deduped merged[index].InApexVariants = append(merged[index].InApexVariants, variantName) @@ -592,73 +592,131 @@ func mergeApexVariations(ctx PathContext, apexInfos []ApexInfo) (merged []ApexIn return merged, aliases } -// CreateApexVariations mutates a given module into multiple apex variants each of which is for an -// apexBundle (and/or the platform) where the module is part of. -func CreateApexVariations(mctx BottomUpMutatorContext, module ApexModule) []Module { +// IncomingApexTransition is called by apexTransitionMutator.IncomingTransition on modules that can be in apexes. +// The incomingVariation can be either the name of an apex if the dependency is coming directly from an apex +// module, or it can be the name of an apex variation (e.g. apex10000) if it is coming from another module that +// is in the apex. +func IncomingApexTransition(ctx IncomingTransitionContext, incomingVariation string) string { + module := ctx.Module().(ApexModule) base := module.apexModuleBase() + var apexInfos []ApexInfo + if allApexInfos, ok := ModuleProvider(ctx, AllApexInfoProvider); ok { + apexInfos = allApexInfos.ApexInfos + } + + // Dependencies from platform variations go to the platform variation. + if incomingVariation == "" { + return "" + } + + // If this module has no apex variations the use the platform variation. + if len(apexInfos) == 0 { + return "" + } + + // Convert the list of apex infos into from the AllApexInfoProvider into the merged list + // of apex variations and the aliases from apex names to apex variations. + var aliases [][2]string + if !module.UniqueApexVariations() && !base.ApexProperties.UniqueApexVariationsForDeps { + apexInfos, aliases = mergeApexVariations(apexInfos) + } + + // Check if the incoming variation matches an apex name, and if so use the corresponding + // apex variation. + aliasIndex := slices.IndexFunc(aliases, func(alias [2]string) bool { + return alias[0] == incomingVariation + }) + if aliasIndex >= 0 { + return aliases[aliasIndex][1] + } + + // Check if the incoming variation matches an apex variation. + apexIndex := slices.IndexFunc(apexInfos, func(info ApexInfo) bool { + return info.ApexVariationName == incomingVariation + }) + if apexIndex >= 0 { + return incomingVariation + } + + return "" +} + +func MutateApexTransition(ctx BaseModuleContext, variation string) { + module := ctx.Module().(ApexModule) + base := module.apexModuleBase() + platformVariation := variation == "" + + var apexInfos []ApexInfo + if allApexInfos, ok := ModuleProvider(ctx, AllApexInfoProvider); ok { + apexInfos = allApexInfos.ApexInfos + } + // Shortcut - if len(base.apexInfos) == 0 { - return nil + if len(apexInfos) == 0 { + return } // Do some validity checks. // TODO(jiyong): is this the right place? - base.checkApexAvailableProperty(mctx) + base.checkApexAvailableProperty(ctx) - var apexInfos []ApexInfo - var aliases [][2]string - if !mctx.Module().(ApexModule).UniqueApexVariations() && !base.ApexProperties.UniqueApexVariationsForDeps { - apexInfos, aliases = mergeApexVariations(mctx, base.apexInfos) - } else { - apexInfos = base.apexInfos + if !module.UniqueApexVariations() && !base.ApexProperties.UniqueApexVariationsForDeps { + apexInfos, _ = mergeApexVariations(apexInfos) } - // base.apexInfos is only needed to propagate the list of apexes from apexInfoMutator to - // apexMutator. It is no longer accurate after mergeApexVariations, and won't be copied to - // all but the first created variant. Clear it so it doesn't accidentally get used later. - base.apexInfos = nil - sort.Sort(byApexName(apexInfos)) var inApex ApexMembership for _, a := range apexInfos { for _, apexContents := range a.ApexContents { - inApex = inApex.merge(apexContents.contents[mctx.ModuleName()]) + inApex = inApex.merge(apexContents.contents[ctx.ModuleName()]) } } base.ApexProperties.InAnyApex = true base.ApexProperties.DirectlyInAnyApex = inApex == directlyInApex - defaultVariation := "" - mctx.SetDefaultDependencyVariation(&defaultVariation) - - variations := []string{defaultVariation} - testApexes := []string{} - for _, a := range apexInfos { - variations = append(variations, a.ApexVariationName) - testApexes = append(testApexes, a.TestApexes...) + if platformVariation && !ctx.Host() && !module.AvailableFor(AvailableToPlatform) { + // Do not install the module for platform, but still allow it to output + // uninstallable AndroidMk entries in certain cases when they have side + // effects. TODO(jiyong): move this routine to somewhere else + module.MakeUninstallable() } - modules := mctx.CreateVariations(variations...) - for i, mod := range modules { - platformVariation := i == 0 - if platformVariation && !mctx.Host() && !mod.(ApexModule).AvailableFor(AvailableToPlatform) { - // Do not install the module for platform, but still allow it to output - // uninstallable AndroidMk entries in certain cases when they have side - // effects. TODO(jiyong): move this routine to somewhere else - mod.MakeUninstallable() - } - if !platformVariation { - mctx.SetVariationProvider(mod, ApexInfoProvider, apexInfos[i-1]) + if !platformVariation { + var thisApexInfo ApexInfo + + apexIndex := slices.IndexFunc(apexInfos, func(info ApexInfo) bool { + return info.ApexVariationName == variation + }) + if apexIndex >= 0 { + thisApexInfo = apexInfos[apexIndex] + } else { + panic(fmt.Errorf("failed to find apexInfo for incoming variation %q", variation)) } - // Set the value of TestApexes in every single apex variant. - // This allows each apex variant to be aware of the test apexes in the user provided apex_available. - mod.(ApexModule).apexModuleBase().ApexProperties.TestApexes = testApexes + + SetProvider(ctx, ApexInfoProvider, thisApexInfo) } - for _, alias := range aliases { - mctx.CreateAliasVariation(alias[0], alias[1]) + // Set the value of TestApexes in every single apex variant. + // This allows each apex variant to be aware of the test apexes in the user provided apex_available. + var testApexes []string + for _, a := range apexInfos { + testApexes = append(testApexes, a.TestApexes...) } + base.ApexProperties.TestApexes = testApexes - return modules +} + +func ApexInfoMutator(ctx TopDownMutatorContext, module ApexModule) { + base := module.apexModuleBase() + if len(base.apexInfos) > 0 { + apexInfos := slices.Clone(base.apexInfos) + slices.SortFunc(apexInfos, func(a, b ApexInfo) int { + return strings.Compare(a.ApexVariationName, b.ApexVariationName) + }) + SetProvider(ctx, AllApexInfoProvider, &AllApexInfo{apexInfos}) + // base.apexInfos is only needed to propagate the list of apexes from the apex module to its + // contents within apexInfoMutator. Clear it so it doesn't accidentally get used later. + base.apexInfos = nil + } } // UpdateUniqueApexVariationsForDeps sets UniqueApexVariationsForDeps if any dependencies that are @@ -669,13 +727,16 @@ func UpdateUniqueApexVariationsForDeps(mctx BottomUpMutatorContext, am ApexModul // InApexVariants list in common. It is used instead of DepIsInSameApex because it needs to // determine if the dep is in the same APEX due to being directly included, not only if it // is included _because_ it is a dependency. - anyInSameApex := func(a, b []ApexInfo) bool { - collectApexes := func(infos []ApexInfo) []string { - var ret []string - for _, info := range infos { - ret = append(ret, info.InApexVariants...) + anyInSameApex := func(a, b ApexModule) bool { + collectApexes := func(m ApexModule) []string { + if allApexInfo, ok := OtherModuleProvider(mctx, m, AllApexInfoProvider); ok { + var ret []string + for _, info := range allApexInfo.ApexInfos { + ret = append(ret, info.InApexVariants...) + } + return ret } - return ret + return nil } aApexes := collectApexes(a) @@ -693,7 +754,7 @@ func UpdateUniqueApexVariationsForDeps(mctx BottomUpMutatorContext, am ApexModul // If any of the dependencies requires unique apex variations, so does this module. mctx.VisitDirectDeps(func(dep Module) { if depApexModule, ok := dep.(ApexModule); ok { - if anyInSameApex(depApexModule.apexModuleBase().apexInfos, am.apexModuleBase().apexInfos) && + if anyInSameApex(depApexModule, am) && (depApexModule.UniqueApexVariations() || depApexModule.apexModuleBase().ApexProperties.UniqueApexVariationsForDeps) { am.apexModuleBase().ApexProperties.UniqueApexVariationsForDeps = true diff --git a/android/apex_test.go b/android/apex_test.go index ddc730d05..347bf7d98 100644 --- a/android/apex_test.go +++ b/android/apex_test.go @@ -33,10 +33,22 @@ func Test_mergeApexVariations(t *testing.T) { { name: "single", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantAliases: [][2]string{ {"foo", "apex10000"}, @@ -45,98 +57,231 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", FutureApiLevel, false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, false, nil}}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo", "bar"}, + InApexModules: []string{"foo", "bar"}, + }}, wantAliases: [][2]string{ - {"bar", "apex10000"}, {"foo", "apex10000"}, + {"bar", "apex10000"}, }, }, { name: "don't merge version", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: uncheckedFinalApiLevel(30), + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex30", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, - {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "apex30", + MinSdkVersion: uncheckedFinalApiLevel(30), + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantAliases: [][2]string{ - {"bar", "apex30"}, {"foo", "apex10000"}, + {"bar", "apex30"}, }, }, { name: "merge updatable", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"foo", "bar"}, + InApexModules: []string{"foo", "bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantAliases: [][2]string{ - {"bar", "apex10000"}, {"foo", "apex10000"}, + {"bar", "apex10000"}, }, }, { name: "don't merge when for prebuilt_apex", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, // This one should not be merged in with the others because it is for // a prebuilt_apex. - {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex, nil}, + { + ApexVariationName: "baz", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"baz"}, + InApexModules: []string{"baz"}, + ForPrebuiltApex: ForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, - {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"foo", "bar"}, + InApexModules: []string{"foo", "bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "baz", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"baz"}, + InApexModules: []string{"baz"}, + ForPrebuiltApex: ForPrebuiltApex, + }, }, wantAliases: [][2]string{ - {"bar", "apex10000"}, {"foo", "apex10000"}, + {"bar", "apex10000"}, }, }, { name: "merge different UsePlatformApis but don't allow using platform api", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: FutureApiLevel, + UsePlatformApis: true, + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo", "bar"}, + InApexModules: []string{"foo", "bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantAliases: [][2]string{ - {"bar", "apex10000"}, {"foo", "apex10000"}, + {"bar", "apex10000"}, }, }, { name: "merge same UsePlatformApis and allow using platform api", in: []ApexInfo{ - {"foo", FutureApiLevel, false, true, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + UsePlatformApis: true, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: FutureApiLevel, + UsePlatformApis: true, + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, true, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + UsePlatformApis: true, + InApexVariants: []string{"foo", "bar"}, + InApexModules: []string{"foo", "bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantAliases: [][2]string{ - {"bar", "apex10000"}, {"foo", "apex10000"}, + {"bar", "apex10000"}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - config := TestConfig(t.TempDir(), nil, "", nil) - ctx := &configErrorWrapper{config: config} - gotMerged, gotAliases := mergeApexVariations(ctx, tt.in) + gotMerged, gotAliases := mergeApexVariations(tt.in) if !reflect.DeepEqual(gotMerged, tt.wantMerged) { t.Errorf("mergeApexVariations() gotMerged = %v, want %v", gotMerged, tt.wantMerged) } diff --git a/android/arch.go b/android/arch.go index 27ce4d464..cd8882bbd 100644 --- a/android/arch.go +++ b/android/arch.go @@ -16,16 +16,11 @@ package android import ( "encoding" - "encoding/json" "fmt" "reflect" "runtime" - "sort" "strings" - "android/soong/bazel" - "android/soong/starlark_fmt" - "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/proptools" @@ -980,12 +975,18 @@ func filterArchStruct(field reflect.StructField, prefix string) (bool, reflect.S panic(fmt.Errorf("unexpected tag format %q", field.Tag)) } // these tags don't need to be present in the runtime generated struct type. - values = RemoveListFromList(values, []string{"arch_variant", "variant_prepend", "path", "replace_instead_of_append"}) + // However replace_instead_of_append does, because it's read by the blueprint + // property extending util functions, which can operate on these generated arch + // property structs. + values = RemoveListFromList(values, []string{"arch_variant", "variant_prepend", "path"}) if len(values) > 0 { - panic(fmt.Errorf("unknown tags %q in field %q", values, prefix+field.Name)) + if values[0] != "replace_instead_of_append" || len(values) > 1 { + panic(fmt.Errorf("unknown tags %q in field %q", values, prefix+field.Name)) + } + field.Tag = `android:"replace_instead_of_append"` + } else { + field.Tag = `` } - - field.Tag = `` return true, field } return false, field @@ -1899,428 +1900,8 @@ func decodeMultilibTargets(multilib string, targets []Target, prefer32 bool) ([] return buildTargets, nil } -func (m *ModuleBase) getArchPropertySet(propertySet interface{}, archType ArchType) interface{} { - archString := archType.Field - for i := range m.archProperties { - if m.archProperties[i] == nil { - // Skip over nil properties - continue - } - - // Not archProperties are usable; this function looks for properties of a very specific - // form, and ignores the rest. - for _, archProperty := range m.archProperties[i] { - // archPropValue is a property struct, we are looking for the form: - // `arch: { arm: { key: value, ... }}` - archPropValue := reflect.ValueOf(archProperty).Elem() - - // Unwrap src so that it should looks like a pointer to `arm: { key: value, ... }` - src := archPropValue.FieldByName("Arch").Elem() - - // Step into non-nil pointers to structs in the src value. - if src.Kind() == reflect.Ptr { - if src.IsNil() { - continue - } - src = src.Elem() - } - - // Find the requested field (e.g. arm, x86) in the src struct. - src = src.FieldByName(archString) - - // We only care about structs. - if !src.IsValid() || src.Kind() != reflect.Struct { - continue - } - - // If the value of the field is a struct then step into the - // BlueprintEmbed field. The special "BlueprintEmbed" name is - // used by createArchPropTypeDesc to embed the arch properties - // in the parent struct, so the src arch prop should be in this - // field. - // - // See createArchPropTypeDesc for more details on how Arch-specific - // module properties are processed from the nested props and written - // into the module's archProperties. - src = src.FieldByName("BlueprintEmbed") - - // Clone the destination prop, since we want a unique prop struct per arch. - propertySetClone := reflect.New(reflect.ValueOf(propertySet).Elem().Type()).Interface() - - // Copy the located property struct into the cloned destination property struct. - err := proptools.ExtendMatchingProperties([]interface{}{propertySetClone}, src.Interface(), nil, proptools.OrderReplace) - if err != nil { - // This is fine, it just means the src struct doesn't match the type of propertySet. - continue - } - - return propertySetClone - } - } - // No property set was found specific to the given arch, so return an empty - // property set. - return reflect.New(reflect.ValueOf(propertySet).Elem().Type()).Interface() -} - -// getMultilibPropertySet returns a property set struct matching the type of -// `propertySet`, containing multilib-specific module properties for the given architecture. -// If no multilib-specific properties exist for the given architecture, returns an empty property -// set matching `propertySet`'s type. -func (m *ModuleBase) getMultilibPropertySet(propertySet interface{}, archType ArchType) interface{} { - // archType.Multilib is lowercase (for example, lib32) but property struct field is - // capitalized, such as Lib32, so use strings.Title to capitalize it. - multiLibString := strings.Title(archType.Multilib) - - for i := range m.archProperties { - if m.archProperties[i] == nil { - // Skip over nil properties - continue - } - - // Not archProperties are usable; this function looks for properties of a very specific - // form, and ignores the rest. - for _, archProperties := range m.archProperties[i] { - // archPropValue is a property struct, we are looking for the form: - // `multilib: { lib32: { key: value, ... }}` - archPropValue := reflect.ValueOf(archProperties).Elem() - - // Unwrap src so that it should looks like a pointer to `lib32: { key: value, ... }` - src := archPropValue.FieldByName("Multilib").Elem() - - // Step into non-nil pointers to structs in the src value. - if src.Kind() == reflect.Ptr { - if src.IsNil() { - // Ignore nil pointers. - continue - } - src = src.Elem() - } - - // Find the requested field (e.g. lib32) in the src struct. - src = src.FieldByName(multiLibString) - - // We only care about valid struct pointers. - if !src.IsValid() || src.Kind() != reflect.Ptr || src.Elem().Kind() != reflect.Struct { - continue - } - - // Get the zero value for the requested property set. - propertySetClone := reflect.New(reflect.ValueOf(propertySet).Elem().Type()).Interface() - - // Copy the located property struct into the "zero" property set struct. - err := proptools.ExtendMatchingProperties([]interface{}{propertySetClone}, src.Interface(), nil, proptools.OrderReplace) - - if err != nil { - // This is fine, it just means the src struct doesn't match. - continue - } - - return propertySetClone - } - } - - // There were no multilib properties specifically matching the given archtype. - // Return zeroed value. - return reflect.New(reflect.ValueOf(propertySet).Elem().Type()).Interface() -} - // ArchVariantContext defines the limited context necessary to retrieve arch_variant properties. type ArchVariantContext interface { ModuleErrorf(fmt string, args ...interface{}) PropertyErrorf(property, fmt string, args ...interface{}) } - -// ArchVariantProperties represents a map of arch-variant config strings to a property interface{}. -type ArchVariantProperties map[string]interface{} - -// ConfigurationAxisToArchVariantProperties represents a map of bazel.ConfigurationAxis to -// ArchVariantProperties, such that each independent arch-variant axis maps to the -// configs/properties for that axis. -type ConfigurationAxisToArchVariantProperties map[bazel.ConfigurationAxis]ArchVariantProperties - -// GetArchVariantProperties returns a ConfigurationAxisToArchVariantProperties where the -// arch-variant properties correspond to the values of the properties of the 'propertySet' struct -// that are specific to that axis/configuration. Each axis is independent, containing -// non-overlapping configs that correspond to the various "arch-variant" support, at this time: -// -// arches (including multilib) -// oses -// arch+os combinations -// -// For example, passing a struct { Foo bool, Bar string } will return an interface{} that can be -// type asserted back into the same struct, containing the config-specific property value specified -// by the module if defined. -// -// Arch-specific properties may come from an arch stanza or a multilib stanza; properties -// in these stanzas are combined. -// For example: `arch: { x86: { Foo: ["bar"] } }, multilib: { lib32: {` Foo: ["baz"] } }` -// will result in `Foo: ["bar", "baz"]` being returned for architecture x86, if the given -// propertyset contains `Foo []string`. -func (m *ModuleBase) GetArchVariantProperties(ctx ArchVariantContext, propertySet interface{}) ConfigurationAxisToArchVariantProperties { - // Return value of the arch types to the prop values for that arch. - axisToProps := ConfigurationAxisToArchVariantProperties{} - - // Nothing to do for non-arch-specific modules. - if !m.ArchSpecific() { - return axisToProps - } - - dstType := reflect.ValueOf(propertySet).Type() - var archProperties []interface{} - - // First find the property set in the module that corresponds to the requested - // one. m.archProperties[i] corresponds to m.GetProperties()[i]. - for i, generalProp := range m.GetProperties() { - srcType := reflect.ValueOf(generalProp).Type() - if srcType == dstType { - archProperties = m.archProperties[i] - axisToProps[bazel.NoConfigAxis] = ArchVariantProperties{"": generalProp} - break - } - } - - if archProperties == nil { - // This module does not have the property set requested - return axisToProps - } - - archToProp := ArchVariantProperties{} - // For each arch type (x86, arm64, etc.) - for _, arch := range ArchTypeList() { - // Arch properties are sometimes sharded (see createArchPropTypeDesc() ). - // Iterate over every shard and extract a struct with the same type as the - // input one that contains the data specific to that arch. - propertyStructs := make([]reflect.Value, 0) - archFeaturePropertyStructs := make(map[string][]reflect.Value, 0) - for _, archProperty := range archProperties { - archTypeStruct, ok := getArchTypeStruct(ctx, archProperty, arch) - if ok { - propertyStructs = append(propertyStructs, archTypeStruct) - - // For each feature this arch supports (arm: neon, x86: ssse3, sse4, ...) - for _, feature := range archFeatures[arch] { - prefix := "arch." + arch.Name + "." + feature - if featureProperties, ok := getChildPropertyStruct(ctx, archTypeStruct, feature, prefix); ok { - archFeaturePropertyStructs[feature] = append(archFeaturePropertyStructs[feature], featureProperties) - } - } - } - multilibStruct, ok := getMultilibStruct(ctx, archProperty, arch) - if ok { - propertyStructs = append(propertyStructs, multilibStruct) - } - } - - archToProp[arch.Name] = mergeStructs(ctx, propertyStructs, propertySet) - - // In soong, if multiple features match the current configuration, they're - // all used. In bazel, we have to have unambiguous select() statements, so - // we can't have two features that are both active in the same select(). - // One alternative is to split out each feature into a separate select(), - // but then it's difficult to support exclude_srcs, which may need to - // exclude things from the regular arch select() statement if a certain - // feature is active. Instead, keep the features in the same select - // statement as the arches, but emit the power set of all possible - // combinations of features, so that bazel can match the most precise one. - allFeatures := make([]string, 0, len(archFeaturePropertyStructs)) - for feature := range archFeaturePropertyStructs { - allFeatures = append(allFeatures, feature) - } - for _, features := range bazel.PowerSetWithoutEmptySet(allFeatures) { - sort.Strings(features) - propsForCurrentFeatureSet := make([]reflect.Value, 0) - propsForCurrentFeatureSet = append(propsForCurrentFeatureSet, propertyStructs...) - for _, feature := range features { - propsForCurrentFeatureSet = append(propsForCurrentFeatureSet, archFeaturePropertyStructs[feature]...) - } - archToProp[arch.Name+"-"+strings.Join(features, "-")] = - mergeStructs(ctx, propsForCurrentFeatureSet, propertySet) - } - } - axisToProps[bazel.ArchConfigurationAxis] = archToProp - - osToProp := ArchVariantProperties{} - archOsToProp := ArchVariantProperties{} - - linuxStructs := getTargetStructs(ctx, archProperties, "Linux") - bionicStructs := getTargetStructs(ctx, archProperties, "Bionic") - hostStructs := getTargetStructs(ctx, archProperties, "Host") - hostLinuxStructs := getTargetStructs(ctx, archProperties, "Host_linux") - hostNotWindowsStructs := getTargetStructs(ctx, archProperties, "Not_windows") - - // For android, linux, ... - for _, os := range osTypeList { - if os == CommonOS { - // It looks like this OS value is not used in Blueprint files - continue - } - osStructs := make([]reflect.Value, 0) - - osSpecificStructs := getTargetStructs(ctx, archProperties, os.Field) - if os.Class == Host { - osStructs = append(osStructs, hostStructs...) - } - if os.Linux() { - osStructs = append(osStructs, linuxStructs...) - } - if os.Bionic() { - osStructs = append(osStructs, bionicStructs...) - } - if os.Linux() && os.Class == Host { - osStructs = append(osStructs, hostLinuxStructs...) - } - - if os == LinuxMusl { - osStructs = append(osStructs, getTargetStructs(ctx, archProperties, "Musl")...) - } - if os == Linux { - osStructs = append(osStructs, getTargetStructs(ctx, archProperties, "Glibc")...) - } - - osStructs = append(osStructs, osSpecificStructs...) - - if os.Class == Host && os != Windows { - osStructs = append(osStructs, hostNotWindowsStructs...) - } - osToProp[os.Name] = mergeStructs(ctx, osStructs, propertySet) - - // For arm, x86, ... - for _, arch := range osArchTypeMap[os] { - osArchStructs := make([]reflect.Value, 0) - - // Auto-combine with Linux_ and Bionic_ targets. This potentially results in - // repetition and select() bloat, but use of Linux_* and Bionic_* targets is rare. - // TODO(b/201423152): Look into cleanup. - if os.Linux() { - targetField := "Linux_" + arch.Name - targetStructs := getTargetStructs(ctx, archProperties, targetField) - osArchStructs = append(osArchStructs, targetStructs...) - } - if os.Bionic() { - targetField := "Bionic_" + arch.Name - targetStructs := getTargetStructs(ctx, archProperties, targetField) - osArchStructs = append(osArchStructs, targetStructs...) - } - if os == LinuxMusl { - targetField := "Musl_" + arch.Name - targetStructs := getTargetStructs(ctx, archProperties, targetField) - osArchStructs = append(osArchStructs, targetStructs...) - } - if os == Linux { - targetField := "Glibc_" + arch.Name - targetStructs := getTargetStructs(ctx, archProperties, targetField) - osArchStructs = append(osArchStructs, targetStructs...) - } - - targetField := GetCompoundTargetField(os, arch) - targetName := fmt.Sprintf("%s_%s", os.Name, arch.Name) - targetStructs := getTargetStructs(ctx, archProperties, targetField) - osArchStructs = append(osArchStructs, targetStructs...) - - archOsToProp[targetName] = mergeStructs(ctx, osArchStructs, propertySet) - } - } - - axisToProps[bazel.OsConfigurationAxis] = osToProp - axisToProps[bazel.OsArchConfigurationAxis] = archOsToProp - return axisToProps -} - -// Returns a struct matching the propertySet interface, containing properties specific to the targetName -// For example, given these arguments: -// -// propertySet = BaseCompilerProperties -// targetName = "android_arm" -// -// And given this Android.bp fragment: -// -// target: -// android_arm: { -// srcs: ["foo.c"], -// } -// android_arm64: { -// srcs: ["bar.c"], -// } -// } -// -// This would return a BaseCompilerProperties with BaseCompilerProperties.Srcs = ["foo.c"] -func getTargetStructs(ctx ArchVariantContext, archProperties []interface{}, targetName string) []reflect.Value { - var propertyStructs []reflect.Value - for _, archProperty := range archProperties { - archPropValues := reflect.ValueOf(archProperty).Elem() - targetProp := archPropValues.FieldByName("Target").Elem() - targetStruct, ok := getChildPropertyStruct(ctx, targetProp, targetName, targetName) - if ok { - propertyStructs = append(propertyStructs, targetStruct) - } else { - return []reflect.Value{} - } - } - - return propertyStructs -} - -func mergeStructs(ctx ArchVariantContext, propertyStructs []reflect.Value, propertySet interface{}) interface{} { - // Create a new instance of the requested property set - value := reflect.New(reflect.ValueOf(propertySet).Elem().Type()).Interface() - - // Merge all the structs together - for _, propertyStruct := range propertyStructs { - mergePropertyStruct(ctx, value, propertyStruct) - } - - return value -} - -func printArchTypeStarlarkDict(dict map[ArchType][]string) string { - valDict := make(map[string]string, len(dict)) - for k, v := range dict { - valDict[k.String()] = starlark_fmt.PrintStringList(v, 1) - } - return starlark_fmt.PrintDict(valDict, 0) -} - -func printArchTypeNestedStarlarkDict(dict map[ArchType]map[string][]string) string { - valDict := make(map[string]string, len(dict)) - for k, v := range dict { - valDict[k.String()] = starlark_fmt.PrintStringListDict(v, 1) - } - return starlark_fmt.PrintDict(valDict, 0) -} - -func printArchConfigList(arches []archConfig) string { - jsonOut, err := json.MarshalIndent(arches, "", starlark_fmt.Indention(1)) - if err != nil { - panic(fmt.Errorf("Error converting arch configs %#v to json: %q", arches, err)) - } - return fmt.Sprintf("json.decode('''%s''')", string(jsonOut)) -} - -func StarlarkArchConfigurations() string { - return fmt.Sprintf(` -_arch_to_variants = %s - -_arch_to_cpu_variants = %s - -_arch_to_features = %s - -_android_arch_feature_for_arch_variant = %s - -_aml_arches = %s - -_ndk_arches = %s - -arch_to_variants = _arch_to_variants -arch_to_cpu_variants = _arch_to_cpu_variants -arch_to_features = _arch_to_features -android_arch_feature_for_arch_variants = _android_arch_feature_for_arch_variant -aml_arches = _aml_arches -ndk_arches = _ndk_arches -`, printArchTypeStarlarkDict(archVariants), - printArchTypeStarlarkDict(cpuVariants), - printArchTypeStarlarkDict(archFeatures), - printArchTypeNestedStarlarkDict(androidArchFeatureMap), - printArchConfigList(getAmlAbisConfig()), - printArchConfigList(getNdkAbisConfig()), - ) -} diff --git a/android/base_module_context.go b/android/base_module_context.go index c4922f4bc..c5fe58578 100644 --- a/android/base_module_context.go +++ b/android/base_module_context.go @@ -20,7 +20,7 @@ import ( "strings" "github.com/google/blueprint" - "github.com/google/blueprint/parser" + "github.com/google/blueprint/proptools" ) // BaseModuleContext is the same as blueprint.BaseModuleContext except that Config() returns @@ -219,7 +219,7 @@ type BaseModuleContext interface { // EvaluateConfiguration makes ModuleContext a valid proptools.ConfigurableEvaluator, so this context // can be used to evaluate the final value of Configurable properties. - EvaluateConfiguration(parser.SelectType, string, string) (string, bool) + EvaluateConfiguration(condition proptools.ConfigurableCondition, property string) proptools.ConfigurableValue } type baseModuleContext struct { @@ -577,6 +577,6 @@ func (b *baseModuleContext) GetPathString(skipFirst bool) string { return sb.String() } -func (m *baseModuleContext) EvaluateConfiguration(ty parser.SelectType, property, condition string) (string, bool) { - return m.Module().ConfigurableEvaluator(m).EvaluateConfiguration(ty, property, condition) +func (m *baseModuleContext) EvaluateConfiguration(condition proptools.ConfigurableCondition, property string) proptools.ConfigurableValue { + return m.Module().ConfigurableEvaluator(m).EvaluateConfiguration(condition, property) } diff --git a/android/config.go b/android/config.go index 11bd81b89..75d135fc0 100644 --- a/android/config.go +++ b/android/config.go @@ -949,7 +949,11 @@ func (c *config) PlatformPreviewSdkVersion() string { } func (c *config) PlatformMinSupportedTargetSdkVersion() string { - return String(c.productVariables.Platform_min_supported_target_sdk_version) + var val, ok = c.productVariables.BuildFlags["RELEASE_PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION"] + if !ok { + return "" + } + return val } func (c *config) PlatformBaseOS() string { diff --git a/android/module.go b/android/module.go index 89c4ddde9..5fe379c8a 100644 --- a/android/module.go +++ b/android/module.go @@ -29,7 +29,6 @@ import ( "android/soong/bazel" "github.com/google/blueprint" - "github.com/google/blueprint/parser" "github.com/google/blueprint/proptools" ) @@ -1043,12 +1042,12 @@ func (m *ModuleBase) baseDepsMutator(ctx BottomUpMutatorContext) { pv := ctx.Config().productVariables fullManifest := pv.DeviceArch != nil && pv.DeviceName != nil if fullManifest { - m.addRequiredDeps(ctx) + addRequiredDeps(ctx) } } // addRequiredDeps adds required, target_required, and host_required as dependencies. -func (m *ModuleBase) addRequiredDeps(ctx BottomUpMutatorContext) { +func addRequiredDeps(ctx BottomUpMutatorContext) { addDep := func(target Target, depName string) { if !ctx.OtherModuleExists(depName) { if ctx.Config().AllowMissingDependencies() { @@ -1061,9 +1060,9 @@ func (m *ModuleBase) addRequiredDeps(ctx BottomUpMutatorContext) { // in build/make/core/main.mk. // TODO(jiyong): the Make-side does this only when the required module is a shared // library or a native test. - bothInAndroid := m.Device() && target.Os.Class == Device - nativeArch := InList(m.Arch().ArchType.Multilib, []string{"lib32", "lib64"}) - sameBitness := m.Arch().ArchType.Multilib == target.Arch.ArchType.Multilib + bothInAndroid := ctx.Device() && target.Os.Class == Device + nativeArch := InList(ctx.Arch().ArchType.Multilib, []string{"lib32", "lib64"}) + sameBitness := ctx.Arch().ArchType.Multilib == target.Arch.ArchType.Multilib if bothInAndroid && nativeArch && !sameBitness { return } @@ -1082,31 +1081,31 @@ func (m *ModuleBase) addRequiredDeps(ctx BottomUpMutatorContext) { hostTargets = append(hostTargets, ctx.Config().Targets[ctx.Config().BuildOS]...) hostTargets = append(hostTargets, ctx.Config().BuildOSCommonTarget) - if m.Device() { - for _, depName := range m.RequiredModuleNames() { + if ctx.Device() { + for _, depName := range ctx.Module().RequiredModuleNames() { for _, target := range deviceTargets { addDep(target, depName) } } - for _, depName := range m.HostRequiredModuleNames() { + for _, depName := range ctx.Module().HostRequiredModuleNames() { for _, target := range hostTargets { addDep(target, depName) } } } - if m.Host() { - for _, depName := range m.RequiredModuleNames() { + if ctx.Host() { + for _, depName := range ctx.Module().RequiredModuleNames() { for _, target := range hostTargets { // When a host module requires another host module, don't make a // dependency if they have different OSes (i.e. hostcross). - if m.Target().HostCross != target.HostCross { + if ctx.Target().HostCross != target.HostCross { continue } addDep(target, depName) } } - for _, depName := range m.TargetRequiredModuleNames() { + for _, depName := range ctx.Module().TargetRequiredModuleNames() { for _, target := range deviceTargets { addDep(target, depName) } @@ -2140,41 +2139,82 @@ func (e configurationEvalutor) PropertyErrorf(property string, fmt string, args e.ctx.OtherModulePropertyErrorf(e.m, property, fmt, args) } -func (e configurationEvalutor) EvaluateConfiguration(ty parser.SelectType, property, condition string) (string, bool) { +func (e configurationEvalutor) EvaluateConfiguration(condition proptools.ConfigurableCondition, property string) proptools.ConfigurableValue { ctx := e.ctx m := e.m - switch ty { - case parser.SelectTypeReleaseVariable: - if v, ok := ctx.Config().productVariables.BuildFlags[condition]; ok { - return v, true + switch condition.FunctionName { + case "release_variable": + if len(condition.Args) != 1 { + ctx.OtherModulePropertyErrorf(m, property, "release_variable requires 1 argument, found %d", len(condition.Args)) + return proptools.ConfigurableValueUndefined() + } + if v, ok := ctx.Config().productVariables.BuildFlags[condition.Args[0]]; ok { + return proptools.ConfigurableValueString(v) } - return "", false - case parser.SelectTypeProductVariable: + return proptools.ConfigurableValueUndefined() + case "product_variable": // TODO(b/323382414): Might add these on a case-by-case basis ctx.OtherModulePropertyErrorf(m, property, "TODO(b/323382414): Product variables are not yet supported in selects") - return "", false - case parser.SelectTypeSoongConfigVariable: - parts := strings.Split(condition, ":") - namespace := parts[0] - variable := parts[1] + return proptools.ConfigurableValueUndefined() + case "soong_config_variable": + if len(condition.Args) != 2 { + ctx.OtherModulePropertyErrorf(m, property, "soong_config_variable requires 2 arguments, found %d", len(condition.Args)) + return proptools.ConfigurableValueUndefined() + } + namespace := condition.Args[0] + variable := condition.Args[1] if n, ok := ctx.Config().productVariables.VendorVars[namespace]; ok { if v, ok := n[variable]; ok { - return v, true + return proptools.ConfigurableValueString(v) } } - return "", false - case parser.SelectTypeVariant: - if condition == "arch" { - if !m.base().ArchReady() { - ctx.OtherModulePropertyErrorf(m, property, "A select on arch was attempted before the arch mutator ran") - return "", false + return proptools.ConfigurableValueUndefined() + case "arch": + if len(condition.Args) != 0 { + ctx.OtherModulePropertyErrorf(m, property, "arch requires no arguments, found %d", len(condition.Args)) + return proptools.ConfigurableValueUndefined() + } + if !m.base().ArchReady() { + ctx.OtherModulePropertyErrorf(m, property, "A select on arch was attempted before the arch mutator ran") + return proptools.ConfigurableValueUndefined() + } + return proptools.ConfigurableValueString(m.base().Arch().ArchType.Name) + case "os": + if len(condition.Args) != 0 { + ctx.OtherModulePropertyErrorf(m, property, "os requires no arguments, found %d", len(condition.Args)) + return proptools.ConfigurableValueUndefined() + } + // the arch mutator runs after the os mutator, we can just use this to enforce that os is ready. + if !m.base().ArchReady() { + ctx.OtherModulePropertyErrorf(m, property, "A select on os was attempted before the arch mutator ran (arch runs after os, we use it to lazily detect that os is ready)") + return proptools.ConfigurableValueUndefined() + } + return proptools.ConfigurableValueString(m.base().Os().Name) + case "boolean_var_for_testing": + // We currently don't have any other boolean variables (we should add support for typing + // the soong config variables), so add this fake one for testing the boolean select + // functionality. + if len(condition.Args) != 0 { + ctx.OtherModulePropertyErrorf(m, property, "boolean_var_for_testing requires 0 arguments, found %d", len(condition.Args)) + return proptools.ConfigurableValueUndefined() + } + + if n, ok := ctx.Config().productVariables.VendorVars["boolean_var"]; ok { + if v, ok := n["for_testing"]; ok { + switch v { + case "true": + return proptools.ConfigurableValueBool(true) + case "false": + return proptools.ConfigurableValueBool(false) + default: + ctx.OtherModulePropertyErrorf(m, property, "testing:my_boolean_var can only be true or false, found %q", v) + } } - return m.base().Arch().ArchType.Name, true } - ctx.OtherModulePropertyErrorf(m, property, "Unknown variant %s", condition) - return "", false + return proptools.ConfigurableValueUndefined() default: - panic("Should be unreachable") + ctx.OtherModulePropertyErrorf(m, property, "Unknown select condition %s", condition.FunctionName) + return proptools.ConfigurableValueUndefined() } } diff --git a/android/mutator.go b/android/mutator.go index 0ff4f48ea..75ba65048 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -391,6 +391,7 @@ func (x *registerMutatorsContext) BottomUpBlueprint(name string, m blueprint.Bot type IncomingTransitionContext interface { ArchModuleContext + ModuleProviderContext // Module returns the target of the dependency edge for which the transition // is being computed @@ -404,6 +405,7 @@ type IncomingTransitionContext interface { type OutgoingTransitionContext interface { ArchModuleContext + ModuleProviderContext // Module returns the target of the dependency edge for which the transition // is being computed @@ -505,6 +507,7 @@ type TransitionMutator interface { type androidTransitionMutator struct { finalPhase bool mutator TransitionMutator + name string } func (a *androidTransitionMutator) Split(ctx blueprint.BaseModuleContext) []string { @@ -537,6 +540,10 @@ func (c *outgoingTransitionContextImpl) DeviceConfig() DeviceConfig { return DeviceConfig{c.bp.Config().(Config).deviceConfig} } +func (c *outgoingTransitionContextImpl) provider(provider blueprint.AnyProviderKey) (any, bool) { + return c.bp.Provider(provider) +} + func (a *androidTransitionMutator) OutgoingTransition(bpctx blueprint.OutgoingTransitionContext, sourceVariation string) string { if m, ok := bpctx.Module().(Module); ok { ctx := outgoingTransitionContextPool.Get().(*outgoingTransitionContextImpl) @@ -568,6 +575,10 @@ func (c *incomingTransitionContextImpl) DeviceConfig() DeviceConfig { return DeviceConfig{c.bp.Config().(Config).deviceConfig} } +func (c *incomingTransitionContextImpl) provider(provider blueprint.AnyProviderKey) (any, bool) { + return c.bp.Provider(provider) +} + func (a *androidTransitionMutator) IncomingTransition(bpctx blueprint.IncomingTransitionContext, incomingVariation string) string { if m, ok := bpctx.Module().(Module); ok { ctx := incomingTransitionContextPool.Get().(*incomingTransitionContextImpl) @@ -586,6 +597,9 @@ func (a *androidTransitionMutator) Mutate(ctx blueprint.BottomUpMutatorContext, if am, ok := ctx.Module().(Module); ok { mctx := bottomUpMutatorContextFactory(ctx, am, a.finalPhase) defer bottomUpMutatorContextPool.Put(mctx) + base := am.base() + base.commonProperties.DebugMutators = append(base.commonProperties.DebugMutators, a.name) + base.commonProperties.DebugVariations = append(base.commonProperties.DebugVariations, variation) a.mutator.Mutate(mctx, variation) } } @@ -594,6 +608,7 @@ func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) { atm := &androidTransitionMutator{ finalPhase: x.finalPhase, mutator: m, + name: name, } mutator := &mutator{ name: name, diff --git a/android/prebuilt.go b/android/prebuilt.go index 91ba05b6b..2b7b55b5d 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -706,11 +706,6 @@ func (p *Prebuilt) usePrebuilt(ctx BaseMutatorContext, source Module, prebuilt M return true } - // If the use_source_config_var property is set then it overrides the prefer property setting. - if configVar := p.properties.Use_source_config_var; configVar != nil { - return !ctx.Config().VendorConfig(proptools.String(configVar.Config_namespace)).Bool(proptools.String(configVar.Var_name)) - } - // TODO: use p.Properties.Name and ctx.ModuleDir to override preference return Bool(p.properties.Prefer) } diff --git a/android/prebuilt_test.go b/android/prebuilt_test.go index 2241b0815..2574ed487 100644 --- a/android/prebuilt_test.go +++ b/android/prebuilt_test.go @@ -295,158 +295,6 @@ func TestPrebuilts(t *testing.T) { }`, prebuilt: []OsType{Android, buildOS}, }, - { - name: "prebuilt use_source_config_var={acme, use_source} - no var specified", - modules: ` - source { - name: "bar", - } - - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - }`, - // When use_source_env is specified then it will use the prebuilt by default if the environment - // variable is not set. - prebuilt: []OsType{Android, buildOS}, - }, - { - name: "prebuilt use_source_config_var={acme, use_source} - acme_use_source=false", - modules: ` - source { - name: "bar", - } - - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - }`, - preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "false", - }, - } - }), - // Setting the environment variable named in use_source_env to false will cause the prebuilt to - // be used. - prebuilt: []OsType{Android, buildOS}, - }, - { - name: "apex_contributions supersedes any source preferred via use_source_config_var", - modules: ` - source { - name: "bar", - } - - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - } - apex_contributions { - name: "my_mainline_module_contribution", - api_domain: "apexfoo", - // this metadata module contains prebuilt - contents: ["prebuilt_bar"], - } - all_apex_contributions { - name: "all_apex_contributions", - } - `, - preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "true", - }, - } - variables.BuildFlags = map[string]string{ - "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": "my_mainline_module_contribution", - } - }), - // use_source_config_var indicates that source should be used - // but this is superseded by `my_mainline_module_contribution` - prebuilt: []OsType{Android, buildOS}, - }, - { - name: "apex_contributions supersedes any prebuilt preferred via use_source_config_var", - modules: ` - source { - name: "bar", - } - - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - } - apex_contributions { - name: "my_mainline_module_contribution", - api_domain: "apexfoo", - // this metadata module contains source - contents: ["bar"], - } - all_apex_contributions { - name: "all_apex_contributions", - } - `, - preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "false", - }, - } - variables.BuildFlags = map[string]string{ - "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": "my_mainline_module_contribution", - } - }), - // use_source_config_var indicates that prebuilt should be used - // but this is superseded by `my_mainline_module_contribution` - prebuilt: nil, - }, - { - name: "prebuilt use_source_config_var={acme, use_source} - acme_use_source=true", - modules: ` - source { - name: "bar", - } - - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - }`, - preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "true", - }, - } - }), - // Setting the environment variable named in use_source_env to true will cause the source to be - // used. - prebuilt: nil, - }, - { - name: "prebuilt use_source_config_var={acme, use_source} - acme_use_source=true, no source", - modules: ` - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - }`, - preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "true", - }, - } - }), - // Although the environment variable says to use source there is no source available. - prebuilt: []OsType{Android, buildOS}, - }, } fs := MockFS{ diff --git a/android/selects_test.go b/android/selects_test.go index e59b3e65d..f912ce626 100644 --- a/android/selects_test.go +++ b/android/selects_test.go @@ -269,11 +269,11 @@ func TestSelects(t *testing.T) { }, }, { - name: "Select on variant", + name: "Select on arch", bp: ` my_module_type { name: "foo", - my_string: select(variant("arch"), { + my_string: select(arch(), { "x86": "my_x86", "x86_64": "my_x86_64", "arm": "my_arm", @@ -287,6 +287,22 @@ func TestSelects(t *testing.T) { }, }, { + name: "Select on os", + bp: ` + my_module_type { + name: "foo", + my_string: select(os(), { + "android": "my_android", + "linux": "my_linux", + default: "my_default", + }), + } + `, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("my_android"), + }, + }, + { name: "Unset value", bp: ` my_module_type { @@ -327,8 +343,10 @@ func TestSelects(t *testing.T) { my_module_type { name: "foo", my_string: select(soong_config_variable("my_namespace", "my_variable"), { + "foo": "bar", default: unset, }) + select(soong_config_variable("my_namespace", "my_variable2"), { + "baz": "qux", default: unset, }) } @@ -341,6 +359,7 @@ func TestSelects(t *testing.T) { my_module_type { name: "foo", my_string: select(soong_config_variable("my_namespace", "my_variable"), { + "foo": "bar", default: unset, }) + select(soong_config_variable("my_namespace", "my_variable2"), { default: "a", @@ -414,6 +433,169 @@ func TestSelects(t *testing.T) { replacing_string_list: &[]string{"b1"}, }, }, + { + name: "Multi-condition string 1", + bp: ` + my_module_type { + name: "foo", + my_string: select(( + soong_config_variable("my_namespace", "my_variable"), + soong_config_variable("my_namespace", "my_variable2"), + ), { + ("a", "b"): "a+b", + ("a", default): "a+default", + (default, default): "default", + }), + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "a", + "my_variable2": "b", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("a+b"), + }, + }, + { + name: "Multi-condition string 2", + bp: ` + my_module_type { + name: "foo", + my_string: select(( + soong_config_variable("my_namespace", "my_variable"), + soong_config_variable("my_namespace", "my_variable2"), + ), { + ("a", "b"): "a+b", + ("a", default): "a+default", + (default, default): "default", + }), + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "a", + "my_variable2": "c", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("a+default"), + }, + }, + { + name: "Multi-condition string 3", + bp: ` + my_module_type { + name: "foo", + my_string: select(( + soong_config_variable("my_namespace", "my_variable"), + soong_config_variable("my_namespace", "my_variable2"), + ), { + ("a", "b"): "a+b", + ("a", default): "a+default", + (default, default): "default", + }), + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "c", + "my_variable2": "b", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("default"), + }, + }, + { + name: "Select on boolean", + bp: ` + my_module_type { + name: "foo", + my_string: select(boolean_var_for_testing(), { + true: "t", + false: "f", + }), + } + `, + vendorVars: map[string]map[string]string{ + "boolean_var": { + "for_testing": "true", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("t"), + }, + }, + { + name: "Select on boolean false", + bp: ` + my_module_type { + name: "foo", + my_string: select(boolean_var_for_testing(), { + true: "t", + false: "f", + }), + } + `, + vendorVars: map[string]map[string]string{ + "boolean_var": { + "for_testing": "false", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("f"), + }, + }, + { + name: "Select on boolean undefined", + bp: ` + my_module_type { + name: "foo", + my_string: select(boolean_var_for_testing(), { + true: "t", + false: "f", + }), + } + `, + expectedError: "foo", + }, + { + name: "Select on boolean undefined with default", + bp: ` + my_module_type { + name: "foo", + my_string: select(boolean_var_for_testing(), { + true: "t", + false: "f", + default: "default", + }), + } + `, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("default"), + }, + }, + { + name: "Mismatched condition types", + bp: ` + my_module_type { + name: "foo", + my_string: select(boolean_var_for_testing(), { + "true": "t", + "false": "f", + default: "default", + }), + } + `, + vendorVars: map[string]map[string]string{ + "boolean_var": { + "for_testing": "false", + }, + }, + expectedError: "Expected all branches of a select on condition boolean_var_for_testing\\(\\) to have type bool, found string", + }, } for _, tc := range testCases { diff --git a/android/soong_config_modules.go b/android/soong_config_modules.go index 90b49eb19..38db92995 100644 --- a/android/soong_config_modules.go +++ b/android/soong_config_modules.go @@ -64,6 +64,7 @@ type soongConfigModuleTypeImportProperties struct { // specified in `conditions_default` will only be used under the following conditions: // bool variable: the variable is unspecified or not set to a true value // value variable: the variable is unspecified +// list variable: the variable is unspecified // string variable: the variable is unspecified or the variable is set to a string unused in the // given module. For example, string variable `test` takes values: "a" and "b", // if the module contains a property `a` and `conditions_default`, when test=b, @@ -104,6 +105,12 @@ type soongConfigModuleTypeImportProperties struct { // cflags: ["-DWIDTH=DEFAULT"], // }, // }, +// impl: { +// srcs: ["impl/%s"], +// conditions_default: { +// srcs: ["impl/default.cpp"], +// }, +// }, // }, // } // @@ -122,6 +129,7 @@ type soongConfigModuleTypeImportProperties struct { // variables: ["board"], // bool_variables: ["feature"], // value_variables: ["width"], +// list_variables: ["impl"], // properties: ["cflags", "srcs"], // } // @@ -135,8 +143,10 @@ type soongConfigModuleTypeImportProperties struct { // $(call add_soong_config_var_value, acme, board, soc_a) // $(call add_soong_config_var_value, acme, feature, true) // $(call add_soong_config_var_value, acme, width, 200) +// $(call add_soong_config_var_value, acme, impl, foo.cpp bar.cpp) // -// Then libacme_foo would build with cflags "-DGENERIC -DSOC_A -DFEATURE -DWIDTH=200". +// Then libacme_foo would build with cflags "-DGENERIC -DSOC_A -DFEATURE -DWIDTH=200" and srcs +// ["*.cpp", "impl/foo.cpp", "impl/bar.cpp"]. // // Alternatively, if acme BoardConfig.mk file contained: // @@ -148,7 +158,9 @@ type soongConfigModuleTypeImportProperties struct { // SOONG_CONFIG_acme_feature := false // // Then libacme_foo would build with cflags: -// "-DGENERIC -DSOC_DEFAULT -DFEATURE_DEFAULT -DSIZE=DEFAULT". +// "-DGENERIC -DSOC_DEFAULT -DFEATURE_DEFAULT -DSIZE=DEFAULT" +// and with srcs: +// ["*.cpp", "impl/default.cpp"]. // // Similarly, if acme BoardConfig.mk file contained: // @@ -158,9 +170,13 @@ type soongConfigModuleTypeImportProperties struct { // feature \ // // SOONG_CONFIG_acme_board := soc_c +// SOONG_CONFIG_acme_impl := foo.cpp bar.cpp // // Then libacme_foo would build with cflags: -// "-DGENERIC -DSOC_DEFAULT -DFEATURE_DEFAULT -DSIZE=DEFAULT". +// "-DGENERIC -DSOC_DEFAULT -DFEATURE_DEFAULT -DSIZE=DEFAULT" +// and with srcs: +// ["*.cpp", "impl/foo.cpp", "impl/bar.cpp"]. +// func SoongConfigModuleTypeImportFactory() Module { module := &soongConfigModuleTypeImport{} @@ -201,6 +217,7 @@ type soongConfigModuleTypeModule struct { // // bool variable: the variable is unspecified or not set to a true value // value variable: the variable is unspecified +// list variable: the variable is unspecified // string variable: the variable is unspecified or the variable is set to a string unused in the // given module. For example, string variable `test` takes values: "a" and "b", // if the module contains a property `a` and `conditions_default`, when test=b, @@ -209,56 +226,63 @@ type soongConfigModuleTypeModule struct { // // For example, an Android.bp file could have: // -// soong_config_module_type { -// name: "acme_cc_defaults", -// module_type: "cc_defaults", -// config_namespace: "acme", -// variables: ["board"], -// bool_variables: ["feature"], -// value_variables: ["width"], -// properties: ["cflags", "srcs"], -// } -// -// soong_config_string_variable { -// name: "board", -// values: ["soc_a", "soc_b"], -// } -// -// acme_cc_defaults { -// name: "acme_defaults", -// cflags: ["-DGENERIC"], -// soong_config_variables: { -// board: { -// soc_a: { -// cflags: ["-DSOC_A"], -// }, -// soc_b: { -// cflags: ["-DSOC_B"], -// }, -// conditions_default: { -// cflags: ["-DSOC_DEFAULT"], -// }, +// soong_config_module_type { +// name: "acme_cc_defaults", +// module_type: "cc_defaults", +// config_namespace: "acme", +// variables: ["board"], +// bool_variables: ["feature"], +// value_variables: ["width"], +// list_variables: ["impl"], +// properties: ["cflags", "srcs"], +// } +// +// soong_config_string_variable { +// name: "board", +// values: ["soc_a", "soc_b"], +// } +// +// acme_cc_defaults { +// name: "acme_defaults", +// cflags: ["-DGENERIC"], +// soong_config_variables: { +// board: { +// soc_a: { +// cflags: ["-DSOC_A"], +// }, +// soc_b: { +// cflags: ["-DSOC_B"], // }, -// feature: { -// cflags: ["-DFEATURE"], -// conditions_default: { -// cflags: ["-DFEATURE_DEFAULT"], -// }, +// conditions_default: { +// cflags: ["-DSOC_DEFAULT"], // }, -// width: { -// cflags: ["-DWIDTH=%s"], -// conditions_default: { -// cflags: ["-DWIDTH=DEFAULT"], -// }, +// }, +// feature: { +// cflags: ["-DFEATURE"], +// conditions_default: { +// cflags: ["-DFEATURE_DEFAULT"], +// }, +// }, +// width: { +// cflags: ["-DWIDTH=%s"], +// conditions_default: { +// cflags: ["-DWIDTH=DEFAULT"], +// }, +// }, +// impl: { +// srcs: ["impl/%s"], +// conditions_default: { +// srcs: ["impl/default.cpp"], // }, // }, -// } +// }, +// } // -// cc_library { -// name: "libacme_foo", -// defaults: ["acme_defaults"], -// srcs: ["*.cpp"], -// } +// cc_library { +// name: "libacme_foo", +// defaults: ["acme_defaults"], +// srcs: ["*.cpp"], +// } // // If an acme BoardConfig.mk file contained: // @@ -270,8 +294,10 @@ type soongConfigModuleTypeModule struct { // SOONG_CONFIG_acme_board := soc_a // SOONG_CONFIG_acme_feature := true // SOONG_CONFIG_acme_width := 200 +// SOONG_CONFIG_acme_impl := foo.cpp bar.cpp // -// Then libacme_foo would build with cflags "-DGENERIC -DSOC_A -DFEATURE". +// Then libacme_foo would build with cflags "-DGENERIC -DSOC_A -DFEATURE" and srcs +// ["*.cpp", "impl/foo.cpp", "impl/bar.cpp"]. func SoongConfigModuleTypeFactory() Module { module := &soongConfigModuleTypeModule{} diff --git a/android/soongconfig/modules.go b/android/soongconfig/modules.go index c78b72669..c910974f6 100644 --- a/android/soongconfig/modules.go +++ b/android/soongconfig/modules.go @@ -117,6 +117,10 @@ type ModuleTypeProperties struct { // inserted into the properties with %s substitution. Value_variables []string + // the list of SOONG_CONFIG list variables that this module type will read. Each value will be + // inserted into the properties with %s substitution. + List_variables []string + // the list of properties that this module type will extend. Properties []string } @@ -468,6 +472,18 @@ func newModuleType(props *ModuleTypeProperties) (*ModuleType, []error) { }) } + for _, name := range props.List_variables { + if err := checkVariableName(name); err != nil { + return nil, []error{fmt.Errorf("list_variables %s", err)} + } + + mt.Variables = append(mt.Variables, &listVariable{ + baseVariable: baseVariable{ + variable: name, + }, + }) + } + return mt, nil } @@ -730,6 +746,90 @@ func (s *valueVariable) printfIntoPropertyRecursive(fieldName []string, propStru return nil } +// Struct to allow conditions set based on a list variable, supporting string substitution. +type listVariable struct { + baseVariable +} + +func (s *listVariable) variableValuesType() reflect.Type { + return emptyInterfaceType +} + +// initializeProperties initializes a property to zero value of typ with an additional conditions +// default field. +func (s *listVariable) initializeProperties(v reflect.Value, typ reflect.Type) { + initializePropertiesWithDefault(v, typ) +} + +// PropertiesToApply returns an interface{} value based on initializeProperties to be applied to +// the module. If the variable was not set, conditions_default interface will be returned; +// otherwise, the interface in values, without conditions_default will be returned with all +// appropriate string substitutions based on variable being set. +func (s *listVariable) PropertiesToApply(config SoongConfig, values reflect.Value) (interface{}, error) { + // If this variable was not referenced in the module, there are no properties to apply. + if !values.IsValid() || values.Elem().IsZero() { + return nil, nil + } + if !config.IsSet(s.variable) { + return conditionsDefaultField(values.Elem().Elem()).Interface(), nil + } + configValues := strings.Split(config.String(s.variable), " ") + + values = removeDefault(values) + propStruct := values.Elem() + if !propStruct.IsValid() { + return nil, nil + } + if err := s.printfIntoPropertyRecursive(nil, propStruct, configValues); err != nil { + return nil, err + } + + return values.Interface(), nil +} + +func (s *listVariable) printfIntoPropertyRecursive(fieldName []string, propStruct reflect.Value, configValues []string) error { + for i := 0; i < propStruct.NumField(); i++ { + field := propStruct.Field(i) + kind := field.Kind() + if kind == reflect.Ptr { + if field.IsNil() { + continue + } + field = field.Elem() + kind = field.Kind() + } + switch kind { + case reflect.Slice: + elemType := field.Type().Elem() + newLen := field.Len() * len(configValues) + newField := reflect.MakeSlice(field.Type(), 0, newLen) + for j := 0; j < field.Len(); j++ { + for _, configValue := range configValues { + res := reflect.Indirect(reflect.New(elemType)) + res.Set(field.Index(j)) + err := printfIntoProperty(res, configValue) + if err != nil { + fieldName = append(fieldName, propStruct.Type().Field(i).Name) + return fmt.Errorf("soong_config_variables.%s.%s: %s", s.variable, strings.Join(fieldName, "."), err) + } + newField = reflect.Append(newField, res) + } + } + field.Set(newField) + case reflect.Struct: + fieldName = append(fieldName, propStruct.Type().Field(i).Name) + if err := s.printfIntoPropertyRecursive(fieldName, field, configValues); err != nil { + return err + } + fieldName = fieldName[:len(fieldName)-1] + default: + fieldName = append(fieldName, propStruct.Type().Field(i).Name) + return fmt.Errorf("soong_config_variables.%s.%s: unsupported property type %q", s.variable, strings.Join(fieldName, "."), kind) + } + } + return nil +} + func printfIntoProperty(propertyValue reflect.Value, configValue string) error { s := propertyValue.String() @@ -739,7 +839,7 @@ func printfIntoProperty(propertyValue reflect.Value, configValue string) error { } if count > 1 { - return fmt.Errorf("value variable properties only support a single '%%'") + return fmt.Errorf("list/value variable properties only support a single '%%'") } if !strings.Contains(s, "%s") { diff --git a/android/soongconfig/modules_test.go b/android/soongconfig/modules_test.go index 1da0b49ad..d76794ca5 100644 --- a/android/soongconfig/modules_test.go +++ b/android/soongconfig/modules_test.go @@ -291,11 +291,13 @@ func Test_createAffectablePropertiesType(t *testing.T) { type properties struct { A *string B bool + C []string } -type boolVarProps struct { +type varProps struct { A *string B bool + C []string Conditions_default *properties } @@ -311,6 +313,19 @@ type valueSoongConfigVars struct { My_value_var interface{} } +type listProperties struct { + C []string +} + +type listVarProps struct { + C []string + Conditions_default *listProperties +} + +type listSoongConfigVars struct { + List_var interface{} +} + func Test_PropertiesToApply_Bool(t *testing.T) { mt, _ := newModuleType(&ModuleTypeProperties{ Module_type: "foo", @@ -330,7 +345,7 @@ func Test_PropertiesToApply_Bool(t *testing.T) { Soong_config_variables boolSoongConfigVars }{ Soong_config_variables: boolSoongConfigVars{ - Bool_var: &boolVarProps{ + Bool_var: &varProps{ A: boolVarPositive.A, B: boolVarPositive.B, Conditions_default: conditionsDefault, @@ -373,6 +388,59 @@ func Test_PropertiesToApply_Bool(t *testing.T) { } } +func Test_PropertiesToApply_List(t *testing.T) { + mt, _ := newModuleType(&ModuleTypeProperties{ + Module_type: "foo", + Config_namespace: "bar", + List_variables: []string{"my_list_var"}, + Properties: []string{"c"}, + }) + conditionsDefault := &listProperties{ + C: []string{"default"}, + } + actualProps := &struct { + Soong_config_variables listSoongConfigVars + }{ + Soong_config_variables: listSoongConfigVars{ + List_var: &listVarProps{ + C: []string{"A=%s", "B=%s"}, + Conditions_default: conditionsDefault, + }, + }, + } + props := reflect.ValueOf(actualProps) + + testCases := []struct { + name string + config SoongConfig + wantProps []interface{} + }{ + { + name: "no_vendor_config", + config: Config(map[string]string{}), + wantProps: []interface{}{conditionsDefault}, + }, + { + name: "value_var_set", + config: Config(map[string]string{"my_list_var": "hello there"}), + wantProps: []interface{}{&listProperties{ + C: []string{"A=hello", "A=there", "B=hello", "B=there"}, + }}, + }, + } + + for _, tc := range testCases { + gotProps, err := PropertiesToApply(mt, props, tc.config) + if err != nil { + t.Errorf("%s: Unexpected error in PropertiesToApply: %s", tc.name, err) + } + + if !reflect.DeepEqual(gotProps, tc.wantProps) { + t.Errorf("%s: Expected %s, got %s", tc.name, tc.wantProps, gotProps) + } + } +} + func Test_PropertiesToApply_Value(t *testing.T) { mt, _ := newModuleType(&ModuleTypeProperties{ Module_type: "foo", @@ -388,7 +456,7 @@ func Test_PropertiesToApply_Value(t *testing.T) { Soong_config_variables valueSoongConfigVars }{ Soong_config_variables: valueSoongConfigVars{ - My_value_var: &boolVarProps{ + My_value_var: &varProps{ A: proptools.StringPtr("A=%s"), B: true, Conditions_default: conditionsDefault, @@ -524,7 +592,7 @@ func Test_PropertiesToApply_String_Error(t *testing.T) { Soong_config_variables stringSoongConfigVars }{ Soong_config_variables: stringSoongConfigVars{ - String_var: &boolVarProps{ + String_var: &varProps{ A: stringVarPositive.A, B: stringVarPositive.B, Conditions_default: conditionsDefault, diff --git a/android/test_suites.go b/android/test_suites.go index adcc15a6e..ff75f26bb 100644 --- a/android/test_suites.go +++ b/android/test_suites.go @@ -14,6 +14,11 @@ package android +import ( + "path/filepath" + "strings" +) + func init() { RegisterParallelSingletonType("testsuites", testSuiteFilesFactory) } @@ -23,8 +28,8 @@ func testSuiteFilesFactory() Singleton { } type testSuiteFiles struct { - robolectric WritablePath - ravenwood WritablePath + robolectric []Path + ravenwood []Path } type TestSuiteModule interface { @@ -48,53 +53,107 @@ func (t *testSuiteFiles) GenerateBuildActions(ctx SingletonContext) { }) t.robolectric = robolectricTestSuite(ctx, files["robolectric-tests"]) - ctx.Phony("robolectric-tests", t.robolectric) + ctx.Phony("robolectric-tests", t.robolectric...) t.ravenwood = ravenwoodTestSuite(ctx, files["ravenwood-tests"]) - ctx.Phony("ravenwood-tests", t.ravenwood) + ctx.Phony("ravenwood-tests", t.ravenwood...) } func (t *testSuiteFiles) MakeVars(ctx MakeVarsContext) { - ctx.DistForGoal("robolectric-tests", t.robolectric) - ctx.DistForGoal("ravenwood-tests", t.ravenwood) + ctx.DistForGoal("robolectric-tests", t.robolectric...) + ctx.DistForGoal("ravenwood-tests", t.ravenwood...) } -func robolectricTestSuite(ctx SingletonContext, files map[string]InstallPaths) WritablePath { +func robolectricTestSuite(ctx SingletonContext, files map[string]InstallPaths) []Path { var installedPaths InstallPaths for _, module := range SortedKeys(files) { installedPaths = append(installedPaths, files[module]...) } - testCasesDir := pathForInstall(ctx, ctx.Config().BuildOS, X86, "testcases") - outputFile := PathForOutput(ctx, "packaging", "robolectric-tests.zip") + outputFile := pathForPackaging(ctx, "robolectric-tests.zip") rule := NewRuleBuilder(pctx, ctx) rule.Command().BuiltTool("soong_zip"). FlagWithOutput("-o ", outputFile). FlagWithArg("-P ", "host/testcases"). - FlagWithArg("-C ", testCasesDir.String()). + FlagWithArg("-C ", pathForTestCases(ctx).String()). FlagWithRspFileInputList("-r ", outputFile.ReplaceExtension(ctx, "rsp"), installedPaths.Paths()). + Flag("-sha256") // necessary to save cas_uploader's time + + testList := buildTestList(ctx, "robolectric-tests_list", installedPaths) + testListZipOutputFile := pathForPackaging(ctx, "robolectric-tests_list.zip") + + rule.Command().BuiltTool("soong_zip"). + FlagWithOutput("-o ", testListZipOutputFile). + FlagWithArg("-C ", pathForPackaging(ctx).String()). + FlagWithInput("-f ", testList). Flag("-sha256") + rule.Build("robolectric_tests_zip", "robolectric-tests.zip") - return outputFile + return []Path{outputFile, testListZipOutputFile} } -func ravenwoodTestSuite(ctx SingletonContext, files map[string]InstallPaths) WritablePath { +func ravenwoodTestSuite(ctx SingletonContext, files map[string]InstallPaths) []Path { var installedPaths InstallPaths for _, module := range SortedKeys(files) { installedPaths = append(installedPaths, files[module]...) } - testCasesDir := pathForInstall(ctx, ctx.Config().BuildOS, X86, "testcases") - outputFile := PathForOutput(ctx, "packaging", "ravenwood-tests.zip") + outputFile := pathForPackaging(ctx, "ravenwood-tests.zip") rule := NewRuleBuilder(pctx, ctx) rule.Command().BuiltTool("soong_zip"). FlagWithOutput("-o ", outputFile). FlagWithArg("-P ", "host/testcases"). - FlagWithArg("-C ", testCasesDir.String()). + FlagWithArg("-C ", pathForTestCases(ctx).String()). FlagWithRspFileInputList("-r ", outputFile.ReplaceExtension(ctx, "rsp"), installedPaths.Paths()). + Flag("-sha256") // necessary to save cas_uploader's time + + testList := buildTestList(ctx, "ravenwood-tests_list", installedPaths) + testListZipOutputFile := pathForPackaging(ctx, "ravenwood-tests_list.zip") + + rule.Command().BuiltTool("soong_zip"). + FlagWithOutput("-o ", testListZipOutputFile). + FlagWithArg("-C ", pathForPackaging(ctx).String()). + FlagWithInput("-f ", testList). Flag("-sha256") + rule.Build("ravenwood_tests_zip", "ravenwood-tests.zip") + return []Path{outputFile, testListZipOutputFile} +} + +func buildTestList(ctx SingletonContext, listFile string, installedPaths InstallPaths) Path { + buf := &strings.Builder{} + for _, p := range installedPaths { + if p.Ext() != ".config" { + continue + } + pc, err := toTestListPath(p.String(), pathForTestCases(ctx).String(), "host/testcases") + if err != nil { + ctx.Errorf("Failed to convert path: %s, %v", p.String(), err) + continue + } + buf.WriteString(pc) + buf.WriteString("\n") + } + outputFile := pathForPackaging(ctx, listFile) + WriteFileRuleVerbatim(ctx, outputFile, buf.String()) return outputFile } + +func toTestListPath(path, relativeRoot, prefix string) (string, error) { + dest, err := filepath.Rel(relativeRoot, path) + if err != nil { + return "", err + } + return filepath.Join(prefix, dest), nil +} + +func pathForPackaging(ctx PathContext, pathComponents ...string) OutputPath { + pathComponents = append([]string{"packaging"}, pathComponents...) + return PathForOutput(ctx, pathComponents...) +} + +func pathForTestCases(ctx PathContext) InstallPath { + return pathForInstall(ctx, ctx.Config().BuildOS, X86, "testcases") +} diff --git a/android/test_suites_test.go b/android/test_suites_test.go new file mode 100644 index 000000000..db9a34d11 --- /dev/null +++ b/android/test_suites_test.go @@ -0,0 +1,117 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "path/filepath" + "testing" +) + +func TestBuildTestList(t *testing.T) { + t.Parallel() + ctx := GroupFixturePreparers( + prepareForFakeTestSuite, + FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.RegisterParallelSingletonType("testsuites", testSuiteFilesFactory) + }), + ).RunTestWithBp(t, ` + fake_module { + name: "module1", + outputs: [ + "Test1/Test1.config", + "Test1/Test1.apk", + ], + test_suites: ["ravenwood-tests"], + } + fake_module { + name: "module2", + outputs: [ + "Test2/Test21/Test21.config", + "Test2/Test21/Test21.apk", + ], + test_suites: ["ravenwood-tests", "robolectric-tests"], + } + fake_module { + name: "module_without_config", + outputs: [ + "BadTest/BadTest.jar", + ], + test_suites: ["robolectric-tests"], + } + `) + + config := ctx.SingletonForTests("testsuites") + allOutputs := config.AllOutputs() + + wantContents := map[string]string{ + "robolectric-tests.zip": "", + "robolectric-tests_list.zip": "", + "robolectric-tests_list": `host/testcases/Test2/Test21/Test21.config +`, + "ravenwood-tests.zip": "", + "ravenwood-tests_list.zip": "", + "ravenwood-tests_list": `host/testcases/Test1/Test1.config +host/testcases/Test2/Test21/Test21.config +`, + } + for _, output := range allOutputs { + want, ok := wantContents[filepath.Base(output)] + if !ok { + t.Errorf("unexpected output: %q", output) + continue + } + + got := "" + if want != "" { + got = ContentFromFileRuleForTests(t, ctx.TestContext, config.MaybeOutput(output)) + } + + if want != got { + t.Errorf("want %q, got %q", want, got) + } + } +} + +type fake_module struct { + ModuleBase + props struct { + Outputs []string + Test_suites []string + } +} + +func fakeTestSuiteFactory() Module { + module := &fake_module{} + base := module.base() + module.AddProperties(&base.nameProperties, &module.props) + InitAndroidModule(module) + return module +} + +var prepareForFakeTestSuite = GroupFixturePreparers( + FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.RegisterModuleType("fake_module", fakeTestSuiteFactory) + }), +) + +func (f *fake_module) GenerateAndroidBuildActions(ctx ModuleContext) { + for _, output := range f.props.Outputs { + ctx.InstallFile(pathForTestCases(ctx), output, nil) + } +} + +func (f *fake_module) TestSuites() []string { + return f.props.Test_suites +} diff --git a/android/util.go b/android/util.go index 363b31ce5..698a85650 100644 --- a/android/util.go +++ b/android/util.go @@ -524,22 +524,27 @@ func SplitFileExt(name string) (string, string, string) { return root, suffix, ext } -// ShardPaths takes a Paths, and returns a slice of Paths where each one has at most shardSize paths. -func ShardPaths(paths Paths, shardSize int) []Paths { - if len(paths) == 0 { +func shard[T ~[]E, E any](toShard T, shardSize int) []T { + if len(toShard) == 0 { return nil } - ret := make([]Paths, 0, (len(paths)+shardSize-1)/shardSize) - for len(paths) > shardSize { - ret = append(ret, paths[0:shardSize]) - paths = paths[shardSize:] + + ret := make([]T, 0, (len(toShard)+shardSize-1)/shardSize) + for len(toShard) > shardSize { + ret = append(ret, toShard[0:shardSize]) + toShard = toShard[shardSize:] } - if len(paths) > 0 { - ret = append(ret, paths) + if len(toShard) > 0 { + ret = append(ret, toShard) } return ret } +// ShardPaths takes a Paths, and returns a slice of Paths where each one has at most shardSize paths. +func ShardPaths(paths Paths, shardSize int) []Paths { + return shard(paths, shardSize) +} + // ShardString takes a string and returns a slice of strings where the length of each one is // at most shardSize. func ShardString(s string, shardSize int) []string { @@ -560,18 +565,7 @@ func ShardString(s string, shardSize int) []string { // ShardStrings takes a slice of strings, and returns a slice of slices of strings where each one has at most shardSize // elements. func ShardStrings(s []string, shardSize int) [][]string { - if len(s) == 0 { - return nil - } - ret := make([][]string, 0, (len(s)+shardSize-1)/shardSize) - for len(s) > shardSize { - ret = append(ret, s[0:shardSize]) - s = s[shardSize:] - } - if len(s) > 0 { - ret = append(ret, s) - } - return ret + return shard(s, shardSize) } // CheckDuplicate checks if there are duplicates in given string list. diff --git a/android/variable.go b/android/variable.go index 5a079db50..599f88e3f 100644 --- a/android/variable.go +++ b/android/variable.go @@ -20,8 +20,6 @@ import ( "runtime" "strings" - "android/soong/bazel" - "github.com/google/blueprint/proptools" ) @@ -201,23 +199,22 @@ type ProductVariables struct { BuildThumbprintFile *string `json:",omitempty"` DisplayBuildNumber *bool `json:",omitempty"` - Platform_display_version_name *string `json:",omitempty"` - Platform_version_name *string `json:",omitempty"` - Platform_sdk_version *int `json:",omitempty"` - Platform_sdk_codename *string `json:",omitempty"` - Platform_sdk_version_or_codename *string `json:",omitempty"` - Platform_sdk_final *bool `json:",omitempty"` - Platform_sdk_extension_version *int `json:",omitempty"` - Platform_base_sdk_extension_version *int `json:",omitempty"` - Platform_version_active_codenames []string `json:",omitempty"` - Platform_version_all_preview_codenames []string `json:",omitempty"` - Platform_systemsdk_versions []string `json:",omitempty"` - Platform_security_patch *string `json:",omitempty"` - Platform_preview_sdk_version *string `json:",omitempty"` - Platform_min_supported_target_sdk_version *string `json:",omitempty"` - Platform_base_os *string `json:",omitempty"` - Platform_version_last_stable *string `json:",omitempty"` - Platform_version_known_codenames *string `json:",omitempty"` + Platform_display_version_name *string `json:",omitempty"` + Platform_version_name *string `json:",omitempty"` + Platform_sdk_version *int `json:",omitempty"` + Platform_sdk_codename *string `json:",omitempty"` + Platform_sdk_version_or_codename *string `json:",omitempty"` + Platform_sdk_final *bool `json:",omitempty"` + Platform_sdk_extension_version *int `json:",omitempty"` + Platform_base_sdk_extension_version *int `json:",omitempty"` + Platform_version_active_codenames []string `json:",omitempty"` + Platform_version_all_preview_codenames []string `json:",omitempty"` + Platform_systemsdk_versions []string `json:",omitempty"` + Platform_security_patch *string `json:",omitempty"` + Platform_preview_sdk_version *string `json:",omitempty"` + Platform_base_os *string `json:",omitempty"` + Platform_version_last_stable *string `json:",omitempty"` + Platform_version_known_codenames *string `json:",omitempty"` DeviceName *string `json:",omitempty"` DeviceProduct *string `json:",omitempty"` @@ -493,10 +490,6 @@ type ProductVariables struct { CheckVendorSeappViolations *bool `json:",omitempty"` - // PartitionVarsForBazelMigrationOnlyDoNotUse are extra variables that are used to define the - // partition images. They should not be read from soong modules. - PartitionVarsForBazelMigrationOnlyDoNotUse PartitionVariables `json:",omitempty"` - BuildFlags map[string]string `json:",omitempty"` BuildFromSourceStub *bool `json:",omitempty"` @@ -645,387 +638,6 @@ func (this *ProductVariables) GetBuildFlagBool(flag string) bool { return val == "true" } -// ProductConfigContext requires the access to the Module to get product config properties. -type ProductConfigContext interface { - Module() Module -} - -// ProductConfigOrSoongConfigProperty represents either a soong config variable + its value -// or a product config variable. You can get both a ConfigurationAxis and a SelectKey from it -// for use in bazel attributes. ProductVariableProperties() will return a map from properties -> -// this interface -> property structs for use in bp2build converters -type ProductConfigOrSoongConfigProperty interface { - // Name of the product variable or soong config variable - Name() string - // AlwaysEmit returns true for soong config variables but false for product variables. This - // is intended to indicate if we need to always emit empty lists in the select statements. - AlwaysEmit() bool - // ConfigurationAxis returns the bazel.ConfigurationAxis that represents this variable. The - // configuration axis will change depending on the variable and whether it's arch/os variant - // as well. - ConfigurationAxis() bazel.ConfigurationAxis - // SelectKey returns a string that represents the key of a select branch, however, it is not - // actually the real label written out to the build file. - // this.ConfigurationAxis().SelectKey(this.SelectKey()) will give the actual label. - SelectKey() string -} - -// ProductConfigProperty represents a product config variable, and if it is arch-variant or not. -type ProductConfigProperty struct { - // The name of the product variable, e.g. "safestack", "malloc_not_svelte", - // "board" - name string - - arch string -} - -func (p ProductConfigProperty) Name() string { - return p.name -} - -func (p ProductConfigProperty) AlwaysEmit() bool { - return false -} - -func (p ProductConfigProperty) ConfigurationAxis() bazel.ConfigurationAxis { - return bazel.ProductVariableConfigurationAxis(p.arch != "", p.name+"__"+p.arch) -} - -func (p ProductConfigProperty) SelectKey() string { - if p.arch == "" { - return strings.ToLower(p.name) - } else { - return strings.ToLower(p.name + "-" + p.arch) - } -} - -// SoongConfigProperty represents a soong config variable, its value if it's a string variable, -// and if it's dependent on the OS or not -type SoongConfigProperty struct { - name string - namespace string - // Can be an empty string for bool/value soong config variables - value string - // If there is a target: field inside a soong config property struct, the os that it selects - // on will be represented here. - os string -} - -func (p SoongConfigProperty) Name() string { - return p.name -} - -func (p SoongConfigProperty) AlwaysEmit() bool { - return true -} - -func (p SoongConfigProperty) ConfigurationAxis() bazel.ConfigurationAxis { - return bazel.ProductVariableConfigurationAxis(false, p.namespace+"__"+p.name+"__"+p.os) -} - -// SelectKey returns the literal string that represents this variable in a BUILD -// select statement. -func (p SoongConfigProperty) SelectKey() string { - // p.value being conditions_default can happen with or without a desired os. When not using - // an os, we want to emit literally just //conditions:default in the select statement, but - // when using an os, we want to emit namespace__name__conditions_default__os, so that - // the branch is only taken if the variable is not set, and we're on the desired os. - // ConfigurationAxis#SelectKey will map the conditions_default result of this function to - // //conditions:default. - if p.value == bazel.ConditionsDefaultConfigKey && p.os == "" { - return bazel.ConditionsDefaultConfigKey - } - - parts := []string{p.namespace, p.name} - if p.value != "" && p.value != bazel.ConditionsDefaultSelectKey { - parts = append(parts, p.value) - } - if p.os != "" { - parts = append(parts, p.os) - } - - // e.g. acme__feature1, android__board__soc_a, my_namespace__my_variables__my_value__my_os - return strings.ToLower(strings.Join(parts, "__")) -} - -// ProductConfigProperties is a map of maps to group property values according -// their property name and the product config variable they're set under. -// -// The outer map key is the name of the property, like "cflags". -// -// The inner map key is a ProductConfigProperty, which is a struct of product -// variable name, namespace, and the "full configuration" of the product -// variable. -// -// e.g. product variable name: board, namespace: acme, full config: vendor_chip_foo -// -// The value of the map is the interface{} representing the value of the -// property, like ["-DDEFINES"] for cflags. -type ProductConfigProperties map[string]map[ProductConfigOrSoongConfigProperty]interface{} - -func (p *ProductConfigProperties) AddProductConfigProperty( - propertyName, productVariableName, arch string, propertyValue interface{}) { - - productConfigProp := ProductConfigProperty{ - name: productVariableName, // e.g. size, feature1, feature2, FEATURE3, board - arch: arch, // e.g. "", x86, arm64 - } - - p.AddEitherProperty(propertyName, productConfigProp, propertyValue) -} - -func (p *ProductConfigProperties) AddSoongConfigProperty( - propertyName, namespace, variableName, value, os string, propertyValue interface{}) { - - soongConfigProp := SoongConfigProperty{ - namespace: namespace, - name: variableName, // e.g. size, feature1, feature2, FEATURE3, board - value: value, - os: os, // e.g. android, linux_x86 - } - - p.AddEitherProperty(propertyName, soongConfigProp, propertyValue) -} - -func (p *ProductConfigProperties) AddEitherProperty( - propertyName string, key ProductConfigOrSoongConfigProperty, propertyValue interface{}) { - if (*p)[propertyName] == nil { - (*p)[propertyName] = make(map[ProductConfigOrSoongConfigProperty]interface{}) - } - - if existing, ok := (*p)[propertyName][key]; ok { - switch dst := existing.(type) { - case []string: - src, ok := propertyValue.([]string) - if !ok { - panic("Conflicting types") - } - dst = append(dst, src...) - (*p)[propertyName][key] = dst - default: - if existing != propertyValue { - panic(fmt.Errorf("TODO: handle merging value %#v", existing)) - } - } - } else { - (*p)[propertyName][key] = propertyValue - } -} - -// maybeExtractConfigVarProp attempts to read this value as a config var struct -// wrapped by interfaces and ptrs. If it's not the right type, the second return -// value is false. -func maybeExtractConfigVarProp(v reflect.Value) (reflect.Value, bool) { - if v.Kind() == reflect.Interface { - // The conditions_default value can be either - // 1) an ptr to an interface of a struct (bool config variables and product variables) - // 2) an interface of 1) (config variables with nested structs, like string vars) - v = v.Elem() - } - if v.Kind() != reflect.Ptr { - return v, false - } - v = reflect.Indirect(v) - if v.Kind() == reflect.Interface { - // Extract the struct from the interface - v = v.Elem() - } - - if !v.IsValid() { - return v, false - } - - if v.Kind() != reflect.Struct { - return v, false - } - return v, true -} - -func (productConfigProperties *ProductConfigProperties) AddProductConfigProperties(variableValues reflect.Value, arch string) { - // Example of product_variables: - // - // product_variables: { - // malloc_not_svelte: { - // shared_libs: ["malloc_not_svelte_shared_lib"], - // whole_static_libs: ["malloc_not_svelte_whole_static_lib"], - // exclude_static_libs: [ - // "malloc_not_svelte_static_lib_excludes", - // "malloc_not_svelte_whole_static_lib_excludes", - // ], - // }, - // }, - - for i := 0; i < variableValues.NumField(); i++ { - // e.g. Platform_sdk_version, Unbundled_build, Malloc_not_svelte, etc. - productVariableName := variableValues.Type().Field(i).Name - - variableValue := variableValues.Field(i) - // Check if any properties were set for the module - if variableValue.IsZero() { - // e.g. feature1: {}, malloc_not_svelte: {} - continue - } - - for j := 0; j < variableValue.NumField(); j++ { - property := variableValue.Field(j) - // e.g. Asflags, Cflags, Enabled, etc. - propertyName := variableValue.Type().Field(j).Name - if property.Kind() != reflect.Interface { - productConfigProperties.AddProductConfigProperty(propertyName, productVariableName, arch, property.Interface()) - } - } - } - -} - -func (productConfigProperties *ProductConfigProperties) AddSoongConfigProperties(namespace string, soongConfigVariablesStruct reflect.Value) error { - // - // Example of soong_config_variables: - // - // soong_config_variables: { - // feature1: { - // conditions_default: { - // ... - // }, - // cflags: ... - // }, - // feature2: { - // cflags: ... - // conditions_default: { - // ... - // }, - // }, - // board: { - // soc_a: { - // ... - // }, - // soc_b: { - // ... - // }, - // soc_c: {}, - // conditions_default: { - // ... - // }, - // }, - // } - for i := 0; i < soongConfigVariablesStruct.NumField(); i++ { - // e.g. feature1, feature2, board - variableName := soongConfigVariablesStruct.Type().Field(i).Name - variableStruct := soongConfigVariablesStruct.Field(i) - // Check if any properties were set for the module - if variableStruct.IsZero() { - // e.g. feature1: {} - continue - } - - // Unlike product variables, config variables require a few more - // indirections to extract the struct from the reflect.Value. - if v, ok := maybeExtractConfigVarProp(variableStruct); ok { - variableStruct = v - } else if !v.IsValid() { - // Skip invalid variables which may not used, else leads to panic - continue - } - - for j := 0; j < variableStruct.NumField(); j++ { - propertyOrStruct := variableStruct.Field(j) - // propertyOrValueName can either be: - // - A property, like: Asflags, Cflags, Enabled, etc. - // - A soong config string variable's value, like soc_a, soc_b, soc_c in the example above - // - "conditions_default" - propertyOrValueName := variableStruct.Type().Field(j).Name - - // If the property wasn't set, no need to pass it along - if propertyOrStruct.IsZero() { - continue - } - - if v, ok := maybeExtractConfigVarProp(propertyOrStruct); ok { - // The field is a struct, which is used by: - // 1) soong_config_string_variables - // - // soc_a: { - // cflags: ..., - // } - // - // soc_b: { - // cflags: ..., - // } - // - // 2) conditions_default structs for all soong config variable types. - // - // conditions_default: { - // cflags: ..., - // static_libs: ... - // } - // - // This means that propertyOrValueName is either conditions_default, or a soong - // config string variable's value. - field := v - // Iterate over fields of this struct prop. - for k := 0; k < field.NumField(); k++ { - // For product variables, zero values are irrelevant; however, for soong config variables, - // empty values are relevant because there can also be a conditions default which is not - // applied for empty variables. - if field.Field(k).IsZero() && namespace == "" { - continue - } - - propertyName := field.Type().Field(k).Name - if propertyName == "Target" { - productConfigProperties.AddSoongConfigPropertiesFromTargetStruct(namespace, variableName, proptools.PropertyNameForField(propertyOrValueName), field.Field(k)) - } else if propertyName == "Arch" || propertyName == "Multilib" { - return fmt.Errorf("Arch/Multilib are not currently supported in soong config variable structs") - } else { - productConfigProperties.AddSoongConfigProperty(propertyName, namespace, variableName, proptools.PropertyNameForField(propertyOrValueName), "", field.Field(k).Interface()) - } - } - } else if propertyOrStruct.Kind() != reflect.Interface { - // If not an interface, then this is not a conditions_default or - // a struct prop. That is, this is a bool/value config variable. - if propertyOrValueName == "Target" { - productConfigProperties.AddSoongConfigPropertiesFromTargetStruct(namespace, variableName, "", propertyOrStruct) - } else if propertyOrValueName == "Arch" || propertyOrValueName == "Multilib" { - return fmt.Errorf("Arch/Multilib are not currently supported in soong config variable structs") - } else { - productConfigProperties.AddSoongConfigProperty(propertyOrValueName, namespace, variableName, "", "", propertyOrStruct.Interface()) - } - } - } - } - return nil -} - -func (productConfigProperties *ProductConfigProperties) AddSoongConfigPropertiesFromTargetStruct(namespace, soongConfigVariableName string, soongConfigVariableValue string, targetStruct reflect.Value) { - // targetStruct will be a struct with fields like "android", "host", "arm", "x86", - // "android_arm", etc. The values of each of those fields will be a regular property struct. - for i := 0; i < targetStruct.NumField(); i++ { - targetFieldName := targetStruct.Type().Field(i).Name - archOrOsSpecificStruct := targetStruct.Field(i) - for j := 0; j < archOrOsSpecificStruct.NumField(); j++ { - property := archOrOsSpecificStruct.Field(j) - // e.g. Asflags, Cflags, Enabled, etc. - propertyName := archOrOsSpecificStruct.Type().Field(j).Name - - if targetFieldName == "Android" { - productConfigProperties.AddSoongConfigProperty(propertyName, namespace, soongConfigVariableName, soongConfigVariableValue, "android", property.Interface()) - } else if targetFieldName == "Host" { - for _, os := range osTypeList { - if os.Class == Host { - productConfigProperties.AddSoongConfigProperty(propertyName, namespace, soongConfigVariableName, soongConfigVariableValue, os.Name, property.Interface()) - } - } - } else if !archOrOsSpecificStruct.IsZero() { - // One problem with supporting additional fields is that if multiple branches of - // "target" overlap, we don't want them to be in the same select statement (aka - // configuration axis). "android" and "host" are disjoint, so it's ok that we only - // have 2 axes right now. (soongConfigVariables and soongConfigVariablesPlusOs) - panic("TODO: support other target types in soong config variable structs: " + targetFieldName) - } - } - } -} - func VariableMutator(mctx BottomUpMutatorContext) { var module Module var ok bool diff --git a/apex/aconfig_test.go b/apex/aconfig_test.go index 3e44f0b1b..3de928633 100644 --- a/apex/aconfig_test.go +++ b/apex/aconfig_test.go @@ -162,6 +162,18 @@ func TestValidationAcrossContainersExportedPass(t *testing.T) { name: "server_configurable_flags", srcs: ["server_configurable_flags.cc"], } + cc_library { + name: "libbase", + srcs: ["libbase.cc"], + } + cc_library { + name: "libaconfig_storage_read_api_cc", + srcs: ["libaconfig_storage_read_api_cc.cc"], + } + cc_library { + name: "libaconfig_storage_protos_cc", + srcs: ["libaconfig_storage_protos_cc.cc"], + } aconfig_declarations { name: "my_aconfig_declarations_bar", package: "com.example.package", @@ -410,6 +422,18 @@ func TestValidationAcrossContainersNotExportedFail(t *testing.T) { name: "server_configurable_flags", srcs: ["server_configurable_flags.cc"], } + cc_library { + name: "libbase", + srcs: ["libbase.cc"], + } + cc_library { + name: "libaconfig_storage_read_api_cc", + srcs: ["libaconfig_storage_read_api_cc.cc"], + } + cc_library { + name: "libaconfig_storage_protos_cc", + srcs: ["libaconfig_storage_protos_cc.cc"], + } aconfig_declarations { name: "my_aconfig_declarations_foo", package: "com.example.package", @@ -460,6 +484,18 @@ func TestValidationAcrossContainersNotExportedFail(t *testing.T) { name: "server_configurable_flags", srcs: ["server_configurable_flags.cc"], } + cc_library { + name: "libbase", + srcs: ["libbase.cc"], + } + cc_library { + name: "libaconfig_storage_read_api_cc", + srcs: ["libaconfig_storage_read_api_cc.cc"], + } + cc_library { + name: "libaconfig_storage_protos_cc", + srcs: ["libaconfig_storage_protos_cc.cc"], + } aconfig_declarations { name: "my_aconfig_declarations_foo", package: "com.example.package", diff --git a/apex/apex.go b/apex/apex.go index cb8449c5a..ef57d7efc 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -73,7 +73,7 @@ func RegisterPostDepsMutators(ctx android.RegisterMutatorsContext) { // Run mark_platform_availability before the apexMutator as the apexMutator needs to know whether // it should create a platform variant. ctx.BottomUp("mark_platform_availability", markPlatformAvailability).Parallel() - ctx.BottomUp("apex", apexMutator).Parallel() + ctx.Transition("apex", &apexTransitionMutator{}) ctx.BottomUp("apex_directly_in_any", apexDirectlyInAnyMutator).Parallel() ctx.BottomUp("apex_dcla_deps", apexDCLADepsMutator).Parallel() // Register after apex_info mutator so that it can use ApexVariationName @@ -1084,6 +1084,10 @@ func apexInfoMutator(mctx android.TopDownMutatorContext) { if a, ok := mctx.Module().(ApexInfoMutator); ok { a.ApexInfoMutator(mctx) } + + if am, ok := mctx.Module().(android.ApexModule); ok { + android.ApexInfoMutator(mctx, am) + } enforceAppUpdatability(mctx) } @@ -1284,40 +1288,41 @@ func markPlatformAvailability(mctx android.BottomUpMutatorContext) { } } -// apexMutator visits each module and creates apex variations if the module was marked in the -// previous run of apexInfoMutator. -func apexMutator(mctx android.BottomUpMutatorContext) { - if !mctx.Module().Enabled() { - return - } - - // This is the usual path. - if am, ok := mctx.Module().(android.ApexModule); ok && am.CanHaveApexVariants() { - android.CreateApexVariations(mctx, am) - return - } +type apexTransitionMutator struct{} +func (a *apexTransitionMutator) Split(ctx android.BaseModuleContext) []string { // apexBundle itself is mutated so that it and its dependencies have the same apex variant. - // Note that a default variation "" is also created as an alias, and the default dependency - // variation is set to the default variation. This is to allow an apex to depend on another - // module which is outside of the apex. This is because the dependent module is not mutated - // for this apex variant. - createApexVariation := func(apexBundleName string) { - defaultVariation := "" - mctx.SetDefaultDependencyVariation(&defaultVariation) - mctx.CreateVariations(apexBundleName) - mctx.CreateAliasVariation(defaultVariation, apexBundleName) - } - - if ai, ok := mctx.Module().(ApexInfoMutator); ok && apexModuleTypeRequiresVariant(ai) { - createApexVariation(ai.ApexVariationName()) - } else if o, ok := mctx.Module().(*OverrideApex); ok { + if ai, ok := ctx.Module().(ApexInfoMutator); ok && apexModuleTypeRequiresVariant(ai) { + return []string{ai.ApexVariationName()} + } else if o, ok := ctx.Module().(*OverrideApex); ok { apexBundleName := o.GetOverriddenModuleName() if apexBundleName == "" { - mctx.ModuleErrorf("base property is not set") - return + ctx.ModuleErrorf("base property is not set") } - createApexVariation(apexBundleName) + return []string{apexBundleName} + } + return []string{""} +} + +func (a *apexTransitionMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + return sourceVariation +} + +func (a *apexTransitionMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + if am, ok := ctx.Module().(android.ApexModule); ok && am.CanHaveApexVariants() { + return android.IncomingApexTransition(ctx, incomingVariation) + } else if ai, ok := ctx.Module().(ApexInfoMutator); ok { + return ai.ApexVariationName() + } else if o, ok := ctx.Module().(*OverrideApex); ok { + return o.GetOverriddenModuleName() + } + + return "" +} + +func (a *apexTransitionMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) { + if am, ok := ctx.Module().(android.ApexModule); ok && am.CanHaveApexVariants() { + android.MutateApexTransition(ctx, variation) } } diff --git a/apex/apex_test.go b/apex/apex_test.go index 2441b023b..8a3735cbe 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -10691,6 +10691,18 @@ func TestAconfigFilesJavaAndCcDeps(t *testing.T) { name: "server_configurable_flags", srcs: ["server_configurable_flags.cc"], } + cc_library { + name: "libbase", + srcs: ["libbase.cc"], + } + cc_library { + name: "libaconfig_storage_read_api_cc", + srcs: ["libaconfig_storage_read_api_cc.cc"], + } + cc_library { + name: "libaconfig_storage_protos_cc", + srcs: ["libaconfig_storage_protos_cc.cc"], + } `) mod := ctx.ModuleForTests("myapex", "android_common_myapex") diff --git a/bpfix/bpfix/bpfix_test.go b/bpfix/bpfix/bpfix_test.go index 672e852d4..b5b49b1ab 100644 --- a/bpfix/bpfix/bpfix_test.go +++ b/bpfix/bpfix/bpfix_test.go @@ -19,6 +19,7 @@ package bpfix import ( "bytes" "fmt" + "os" "reflect" "strings" "testing" @@ -2215,3 +2216,9 @@ func TestRemoveResourceAndAssetsIfDefault(t *testing.T) { }) } } + +func TestMain(m *testing.M) { + // Skip checking Android.mk path with cleaning "ANDROID_BUILD_TOP" + os.Setenv("ANDROID_BUILD_TOP", "") + os.Exit(m.Run()) +} diff --git a/cc/Android.bp b/cc/Android.bp index 5ba94270b..9ce89330e 100644 --- a/cc/Android.bp +++ b/cc/Android.bp @@ -96,7 +96,6 @@ bootstrap_go_package { "gen_test.go", "genrule_test.go", "library_headers_test.go", - "library_stub_test.go", "library_test.go", "lto_test.go", "ndk_test.go", @@ -1995,6 +1995,20 @@ func (d *Defaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { } func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { + ctx := moduleContextFromAndroidModuleContext(actx, c) + + // If Test_only is set on a module in bp file, respect the setting, otherwise + // see if is a known test module type. + testOnly := c.testModule || c.testLibrary() + if c.sourceProperties.Test_only != nil { + testOnly = Bool(c.sourceProperties.Test_only) + } + // Keep before any early returns. + android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{ + TestOnly: testOnly, + TopLevelTarget: c.testModule, + }) + // Handle the case of a test module split by `test_per_src` mutator. // // The `test_per_src` mutator adds an extra variation named "", depending on all the other @@ -2013,8 +2027,6 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { c.makeLinkType = GetMakeLinkType(actx, c) - ctx := moduleContextFromAndroidModuleContext(actx, c) - deps := c.depsToPaths(ctx) if ctx.Failed() { return @@ -2141,17 +2153,6 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { android.SetProvider(ctx, testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } - // If Test_only is set on a module in bp file, respect the setting, otherwise - // see if is a known test module type. - testOnly := c.testModule || c.testLibrary() - if c.sourceProperties.Test_only != nil { - testOnly = Bool(c.sourceProperties.Test_only) - } - android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{ - TestOnly: testOnly, - TopLevelTarget: c.testModule, - }) - android.SetProvider(ctx, blueprint.SrcsFileProviderKey, blueprint.SrcsFileProviderData{SrcPaths: deps.GeneratedSources.Strings()}) android.CollectDependencyAconfigFiles(ctx, &c.mergedAconfigFiles) diff --git a/cc/cc_test_only_property_test.go b/cc/cc_test_only_property_test.go index 972e86bc5..c14f34ecb 100644 --- a/cc/cc_test_only_property_test.go +++ b/cc/cc_test_only_property_test.go @@ -78,6 +78,38 @@ func TestTestOnlyProvider(t *testing.T) { } } +func TestTestOnlyValueWithTestPerSrcProp(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + prepareForCcTest, + ).RunTestWithBp(t, ` + // These should be test-only + cc_test { name: "cc-test", + gtest: false, + test_per_src: true, + srcs: ["foo_test.cpp"], + test_options: { unit_test: false, }, + } + `) + + // Ensure all variation of test-per-src tests are marked test-only. + ctx.VisitAllModules(func(m blueprint.Module) { + testOnly := false + if provider, ok := android.OtherModuleProvider(ctx.TestContext.OtherModuleProviderAdaptor(), m, android.TestOnlyProviderKey); ok { + if provider.TestOnly { + testOnly = true + } + } + if module, ok := m.(*Module); ok { + if testModule, ok := module.installer.(*testBinary); ok { + if !testOnly && *testModule.Properties.Test_per_src { + t.Errorf("%v is not test-only but should be", m) + } + } + } + }) +} + func TestTestOnlyInTeamsProto(t *testing.T) { t.Parallel() ctx := android.GroupFixturePreparers( diff --git a/cc/library_stub_test.go b/cc/library_stub_test.go deleted file mode 100644 index 4df0a4186..000000000 --- a/cc/library_stub_test.go +++ /dev/null @@ -1,459 +0,0 @@ -// Copyright 2021 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cc - -import ( - _ "fmt" - _ "sort" - - "testing" - - "android/soong/android" - - "github.com/google/blueprint" -) - -func hasDirectDependency(t *testing.T, ctx *android.TestResult, from android.Module, to android.Module) bool { - t.Helper() - var found bool - ctx.VisitDirectDeps(from, func(dep blueprint.Module) { - if dep == to { - found = true - } - }) - return found -} - -func TestApiLibraryReplacesExistingModule(t *testing.T) { - bp := ` - cc_library { - name: "libfoo", - shared_libs: ["libbar"], - vendor_available: true, - } - - cc_library { - name: "libbar", - } - - cc_api_library { - name: "libbar", - vendor_available: true, - src: "libbar.so", - } - - api_imports { - name: "api_imports", - shared_libs: [ - "libbar", - ], - } - ` - - ctx := prepareForCcTest.RunTestWithBp(t, bp) - - libfoo := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_shared").Module() - libbar := ctx.ModuleForTests("libbar", "android_arm64_armv8-a_shared").Module() - libbarApiImport := ctx.ModuleForTests("libbar.apiimport", "android_arm64_armv8-a_shared").Module() - - android.AssertBoolEquals(t, "original library should be linked with non-stub variant", true, hasDirectDependency(t, ctx, libfoo, libbar)) - android.AssertBoolEquals(t, "Stub library from API surface should be not linked with non-stub variant", false, hasDirectDependency(t, ctx, libfoo, libbarApiImport)) - - libfooVendor := ctx.ModuleForTests("libfoo", "android_vendor_arm64_armv8-a_shared").Module() - libbarApiImportVendor := ctx.ModuleForTests("libbar.apiimport", "android_vendor_arm64_armv8-a_shared").Module() - - android.AssertBoolEquals(t, "original library should not be linked", false, hasDirectDependency(t, ctx, libfooVendor, libbar)) - android.AssertBoolEquals(t, "Stub library from API surface should be linked", true, hasDirectDependency(t, ctx, libfooVendor, libbarApiImportVendor)) -} - -func TestApiLibraryDoNotRequireOriginalModule(t *testing.T) { - bp := ` - cc_library { - name: "libfoo", - shared_libs: ["libbar"], - vendor: true, - } - - cc_api_library { - name: "libbar", - src: "libbar.so", - vendor_available: true, - } - - api_imports { - name: "api_imports", - shared_libs: [ - "libbar", - ], - } - ` - - ctx := prepareForCcTest.RunTestWithBp(t, bp) - - libfoo := ctx.ModuleForTests("libfoo", "android_vendor_arm64_armv8-a_shared").Module() - libbarApiImport := ctx.ModuleForTests("libbar.apiimport", "android_vendor_arm64_armv8-a_shared").Module() - - android.AssertBoolEquals(t, "Stub library from API surface should be linked", true, hasDirectDependency(t, ctx, libfoo, libbarApiImport)) -} - -func TestApiLibraryShouldNotReplaceWithoutApiImport(t *testing.T) { - bp := ` - cc_library { - name: "libfoo", - shared_libs: ["libbar"], - vendor_available: true, - } - - cc_library { - name: "libbar", - vendor_available: true, - } - - cc_api_library { - name: "libbar", - src: "libbar.so", - vendor_available: true, - } - - api_imports { - name: "api_imports", - shared_libs: [], - } - ` - - ctx := prepareForCcTest.RunTestWithBp(t, bp) - - libfoo := ctx.ModuleForTests("libfoo", "android_vendor_arm64_armv8-a_shared").Module() - libbar := ctx.ModuleForTests("libbar", "android_vendor_arm64_armv8-a_shared").Module() - libbarApiImport := ctx.ModuleForTests("libbar.apiimport", "android_vendor_arm64_armv8-a_shared").Module() - - android.AssertBoolEquals(t, "original library should be linked", true, hasDirectDependency(t, ctx, libfoo, libbar)) - android.AssertBoolEquals(t, "Stub library from API surface should not be linked", false, hasDirectDependency(t, ctx, libfoo, libbarApiImport)) -} - -func TestExportDirFromStubLibrary(t *testing.T) { - bp := ` - cc_library { - name: "libfoo", - export_include_dirs: ["source_include_dir"], - export_system_include_dirs: ["source_system_include_dir"], - vendor_available: true, - } - cc_api_library { - name: "libfoo", - export_include_dirs: ["stub_include_dir"], - export_system_include_dirs: ["stub_system_include_dir"], - vendor_available: true, - src: "libfoo.so", - } - api_imports { - name: "api_imports", - shared_libs: [ - "libfoo", - ], - header_libs: [], - } - // vendor binary - cc_binary { - name: "vendorbin", - vendor: true, - srcs: ["vendor.cc"], - shared_libs: ["libfoo"], - } - ` - ctx := prepareForCcTest.RunTestWithBp(t, bp) - vendorCFlags := ctx.ModuleForTests("vendorbin", "android_vendor_arm64_armv8-a").Rule("cc").Args["cFlags"] - android.AssertStringDoesContain(t, "Vendor binary should compile using headers provided by stub", vendorCFlags, "-Istub_include_dir") - android.AssertStringDoesNotContain(t, "Vendor binary should not compile using headers of source", vendorCFlags, "-Isource_include_dir") - android.AssertStringDoesContain(t, "Vendor binary should compile using system headers provided by stub", vendorCFlags, "-isystem stub_system_include_dir") - android.AssertStringDoesNotContain(t, "Vendor binary should not compile using system headers of source", vendorCFlags, "-isystem source_system_include_dir") - - vendorImplicits := ctx.ModuleForTests("vendorbin", "android_vendor_arm64_armv8-a").Rule("cc").OrderOnly.Strings() - // Building the stub.so file first assembles its .h files in multi-tree out. - // These header files are required for compiling the other API domain (vendor in this case) - android.AssertStringListContains(t, "Vendor binary compilation should have an implicit dep on the stub .so file", vendorImplicits, "libfoo.so") -} - -func TestApiLibraryWithLlndkVariant(t *testing.T) { - bp := ` - cc_binary { - name: "binfoo", - vendor: true, - srcs: ["binfoo.cc"], - shared_libs: ["libbar"], - } - - cc_api_library { - name: "libbar", - // TODO(b/244244438) Remove src property once all variants are implemented. - src: "libbar.so", - vendor_available: true, - variants: [ - "llndk", - ], - } - - cc_api_variant { - name: "libbar", - variant: "llndk", - src: "libbar_llndk.so", - export_include_dirs: ["libbar_llndk_include"] - } - - api_imports { - name: "api_imports", - shared_libs: [ - "libbar", - ], - header_libs: [], - } - ` - - ctx := prepareForCcTest.RunTestWithBp(t, bp) - - binfoo := ctx.ModuleForTests("binfoo", "android_vendor_arm64_armv8-a").Module() - libbarApiImport := ctx.ModuleForTests("libbar.apiimport", "android_vendor_arm64_armv8-a_shared").Module() - libbarApiVariant := ctx.ModuleForTests("libbar.llndk.apiimport", "android_vendor_arm64_armv8-a").Module() - - android.AssertBoolEquals(t, "Stub library from API surface should be linked", true, hasDirectDependency(t, ctx, binfoo, libbarApiImport)) - android.AssertBoolEquals(t, "Stub library variant from API surface should be linked", true, hasDirectDependency(t, ctx, libbarApiImport, libbarApiVariant)) - - binFooLibFlags := ctx.ModuleForTests("binfoo", "android_vendor_arm64_armv8-a").Rule("ld").Args["libFlags"] - android.AssertStringDoesContain(t, "Vendor binary should be linked with LLNDK variant source", binFooLibFlags, "libbar.llndk.apiimport.so") - - binFooCFlags := ctx.ModuleForTests("binfoo", "android_vendor_arm64_armv8-a").Rule("cc").Args["cFlags"] - android.AssertStringDoesContain(t, "Vendor binary should include headers from the LLNDK variant source", binFooCFlags, "-Ilibbar_llndk_include") -} - -func TestApiLibraryWithNdkVariant(t *testing.T) { - bp := ` - cc_binary { - name: "binfoo", - sdk_version: "29", - srcs: ["binfoo.cc"], - shared_libs: ["libbar"], - stl: "c++_shared", - } - - cc_binary { - name: "binbaz", - sdk_version: "30", - srcs: ["binbaz.cc"], - shared_libs: ["libbar"], - stl: "c++_shared", - } - - cc_binary { - name: "binqux", - srcs: ["binfoo.cc"], - shared_libs: ["libbar"], - } - - cc_library { - name: "libbar", - srcs: ["libbar.cc"], - } - - cc_api_library { - name: "libbar", - // TODO(b/244244438) Remove src property once all variants are implemented. - src: "libbar.so", - variants: [ - "ndk.29", - "ndk.30", - "ndk.current", - ], - } - - cc_api_variant { - name: "libbar", - variant: "ndk", - version: "29", - src: "libbar_ndk_29.so", - export_include_dirs: ["libbar_ndk_29_include"] - } - - cc_api_variant { - name: "libbar", - variant: "ndk", - version: "30", - src: "libbar_ndk_30.so", - export_include_dirs: ["libbar_ndk_30_include"] - } - - cc_api_variant { - name: "libbar", - variant: "ndk", - version: "current", - src: "libbar_ndk_current.so", - export_include_dirs: ["libbar_ndk_current_include"] - } - - api_imports { - name: "api_imports", - shared_libs: [ - "libbar", - ], - header_libs: [], - } - ` - - ctx := prepareForCcTest.RunTestWithBp(t, bp) - - binfoo := ctx.ModuleForTests("binfoo", "android_arm64_armv8-a_sdk").Module() - libbarApiImportv29 := ctx.ModuleForTests("libbar.apiimport", "android_arm64_armv8-a_sdk_shared_29").Module() - libbarApiVariantv29 := ctx.ModuleForTests("libbar.ndk.29.apiimport", "android_arm64_armv8-a_sdk").Module() - libbarApiImportv30 := ctx.ModuleForTests("libbar.apiimport", "android_arm64_armv8-a_sdk_shared_30").Module() - libbarApiVariantv30 := ctx.ModuleForTests("libbar.ndk.30.apiimport", "android_arm64_armv8-a_sdk").Module() - - android.AssertBoolEquals(t, "Stub library from API surface should be linked with target version", true, hasDirectDependency(t, ctx, binfoo, libbarApiImportv29)) - android.AssertBoolEquals(t, "Stub library variant from API surface should be linked with target version", true, hasDirectDependency(t, ctx, libbarApiImportv29, libbarApiVariantv29)) - android.AssertBoolEquals(t, "Stub library from API surface should not be linked with different version", false, hasDirectDependency(t, ctx, binfoo, libbarApiImportv30)) - android.AssertBoolEquals(t, "Stub library variant from API surface should not be linked with different version", false, hasDirectDependency(t, ctx, libbarApiImportv29, libbarApiVariantv30)) - - binbaz := ctx.ModuleForTests("binbaz", "android_arm64_armv8-a_sdk").Module() - - android.AssertBoolEquals(t, "Stub library from API surface should be linked with target version", true, hasDirectDependency(t, ctx, binbaz, libbarApiImportv30)) - android.AssertBoolEquals(t, "Stub library from API surface should not be linked with different version", false, hasDirectDependency(t, ctx, binbaz, libbarApiImportv29)) - - binFooLibFlags := ctx.ModuleForTests("binfoo", "android_arm64_armv8-a_sdk").Rule("ld").Args["libFlags"] - android.AssertStringDoesContain(t, "Binary using sdk should be linked with NDK variant source", binFooLibFlags, "libbar.ndk.29.apiimport.so") - - binFooCFlags := ctx.ModuleForTests("binfoo", "android_arm64_armv8-a_sdk").Rule("cc").Args["cFlags"] - android.AssertStringDoesContain(t, "Binary using sdk should include headers from the NDK variant source", binFooCFlags, "-Ilibbar_ndk_29_include") - - binQux := ctx.ModuleForTests("binqux", "android_arm64_armv8-a").Module() - android.AssertBoolEquals(t, "NDK Stub library from API surface should not be linked with nonSdk binary", false, - (hasDirectDependency(t, ctx, binQux, libbarApiImportv30) || hasDirectDependency(t, ctx, binQux, libbarApiImportv29))) -} - -func TestApiLibraryWithMultipleVariants(t *testing.T) { - bp := ` - cc_binary { - name: "binfoo", - sdk_version: "29", - srcs: ["binfoo.cc"], - shared_libs: ["libbar"], - stl: "c++_shared", - } - - cc_binary { - name: "binbaz", - vendor: true, - srcs: ["binbaz.cc"], - shared_libs: ["libbar"], - } - - cc_library { - name: "libbar", - srcs: ["libbar.cc"], - } - - cc_api_library { - name: "libbar", - // TODO(b/244244438) Remove src property once all variants are implemented. - src: "libbar.so", - vendor_available: true, - variants: [ - "llndk", - "ndk.29", - "ndk.30", - "ndk.current", - "apex.29", - "apex.30", - "apex.current", - ], - } - - cc_api_variant { - name: "libbar", - variant: "ndk", - version: "29", - src: "libbar_ndk_29.so", - export_include_dirs: ["libbar_ndk_29_include"] - } - - cc_api_variant { - name: "libbar", - variant: "ndk", - version: "30", - src: "libbar_ndk_30.so", - export_include_dirs: ["libbar_ndk_30_include"] - } - - cc_api_variant { - name: "libbar", - variant: "ndk", - version: "current", - src: "libbar_ndk_current.so", - export_include_dirs: ["libbar_ndk_current_include"] - } - - cc_api_variant { - name: "libbar", - variant: "apex", - version: "29", - src: "libbar_apex_29.so", - export_include_dirs: ["libbar_apex_29_include"] - } - - cc_api_variant { - name: "libbar", - variant: "apex", - version: "30", - src: "libbar_apex_30.so", - export_include_dirs: ["libbar_apex_30_include"] - } - - cc_api_variant { - name: "libbar", - variant: "apex", - version: "current", - src: "libbar_apex_current.so", - export_include_dirs: ["libbar_apex_current_include"] - } - - cc_api_variant { - name: "libbar", - variant: "llndk", - src: "libbar_llndk.so", - export_include_dirs: ["libbar_llndk_include"] - } - - api_imports { - name: "api_imports", - shared_libs: [ - "libbar", - ], - apex_shared_libs: [ - "libbar", - ], - } - ` - ctx := prepareForCcTest.RunTestWithBp(t, bp) - - binfoo := ctx.ModuleForTests("binfoo", "android_arm64_armv8-a_sdk").Module() - libbarApiImportv29 := ctx.ModuleForTests("libbar.apiimport", "android_arm64_armv8-a_sdk_shared_29").Module() - libbarApiImportLlndk := ctx.ModuleForTests("libbar.apiimport", "android_vendor_arm64_armv8-a_shared").Module() - - android.AssertBoolEquals(t, "Binary using SDK should be linked with API library from NDK variant", true, hasDirectDependency(t, ctx, binfoo, libbarApiImportv29)) - android.AssertBoolEquals(t, "Binary using SDK should not be linked with API library from LLNDK variant", false, hasDirectDependency(t, ctx, binfoo, libbarApiImportLlndk)) - - binbaz := ctx.ModuleForTests("binbaz", "android_vendor_arm64_armv8-a").Module() - - android.AssertBoolEquals(t, "Vendor binary should be linked with API library from LLNDK variant", true, hasDirectDependency(t, ctx, binbaz, libbarApiImportLlndk)) - android.AssertBoolEquals(t, "Vendor binary should not be linked with API library from NDK variant", false, hasDirectDependency(t, ctx, binbaz, libbarApiImportv29)) - -} @@ -59,35 +59,6 @@ func sdkMutator(ctx android.BottomUpMutatorContext) { modules[1].(*Module).Properties.PreventInstall = true } ctx.AliasVariation("") - } else if isCcModule && ccModule.isImportedApiLibrary() { - apiLibrary, _ := ccModule.linker.(*apiLibraryDecorator) - if apiLibrary.hasNDKStubs() && ccModule.canUseSdk() { - variations := []string{"sdk"} - if apiLibrary.hasApexStubs() { - variations = append(variations, "") - } - // Handle cc_api_library module with NDK stubs and variants only which can use SDK - modules := ctx.CreateVariations(variations...) - // Mark the SDK variant. - modules[0].(*Module).Properties.IsSdkVariant = true - if ctx.Config().UnbundledBuildApps() { - if apiLibrary.hasApexStubs() { - // For an unbundled apps build, hide the platform variant from Make. - modules[1].(*Module).Properties.HideFromMake = true - } - modules[1].(*Module).Properties.PreventInstall = true - } else { - // For a platform build, mark the SDK variant so that it gets a ".sdk" suffix when - // exposed to Make. - modules[0].(*Module).Properties.SdkAndPlatformVariantVisibleToMake = true - // SDK variant is not supposed to be installed - modules[0].(*Module).Properties.PreventInstall = true - } - } else { - ccModule.Properties.Sdk_version = nil - ctx.CreateVariations("") - ctx.AliasVariation("") - } } else { if isCcModule { // Clear the sdk_version property for modules that don't have an SDK variant so diff --git a/cmd/release_config/Android.bp b/cmd/release_config/build_flag/Android.bp index 7f627ffb7..0f10c91cb 100644 --- a/cmd/release_config/Android.bp +++ b/cmd/release_config/build_flag/Android.bp @@ -2,14 +2,14 @@ package { default_applicable_licenses: ["Android-Apache-2.0"], } -bootstrap_go_package { - name: "release-config", - pkgPath: "android/soong/cmd/release_config", +blueprint_go_binary { + name: "build-flag", deps: [ "golang-protobuf-encoding-prototext", "golang-protobuf-reflect-protoreflect", "golang-protobuf-runtime-protoimpl", - "soong-cmd-release-config-proto", + "soong-cmd-release_config-proto", + "soong-cmd-release_config-lib", ], srcs: [ "main.go", @@ -17,14 +17,16 @@ bootstrap_go_package { } bootstrap_go_package { - name: "soong-cmd-release-config-proto", - pkgPath: "android/soong/cmd/release_config/release_config_proto", + name: "soong-cmd-release_config-build_flag", + pkgPath: "android/soong/cmd/release_config/build_flag", deps: [ + "golang-protobuf-encoding-prototext", "golang-protobuf-reflect-protoreflect", "golang-protobuf-runtime-protoimpl", + "soong-cmd-release_config-proto", + "soong-cmd-release_config-lib", ], srcs: [ - "release_config_proto/build_flags_out.pb.go", - "release_config_proto/build_flags_src.pb.go", + "main.go", ], } diff --git a/cmd/release_config/build_flag/main.go b/cmd/release_config/build_flag/main.go new file mode 100644 index 000000000..6f909af86 --- /dev/null +++ b/cmd/release_config/build_flag/main.go @@ -0,0 +1,229 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "strings" + + rc_lib "android/soong/cmd/release_config/release_config_lib" + rc_proto "android/soong/cmd/release_config/release_config_proto" + + "google.golang.org/protobuf/proto" +) + +type Flags struct { + // The path to the top of the workspace. Default: ".". + top string + + // Pathlist of release config map textproto files. + // If not specified, then the value is (if present): + // - build/release/release_config_map.textproto + // - vendor/google_shared/build/release/release_config_map.textproto + // - vendor/google/release/release_config_map.textproto + // + // Additionally, any maps specified in the environment variable + // `PRODUCT_RELEASE_CONFIG_MAPS` are used. + maps rc_lib.StringList + + // Output directory (relative to `top`). + outDir string + + // Which $TARGET_RELEASE(s) should we use. Some commands will only + // accept one value, others also accept `--release --all`. + targetReleases rc_lib.StringList + + // Disable warning messages + quiet bool +} + +type CommandFunc func(*rc_lib.ReleaseConfigs, Flags, string, []string) error + +var commandMap map[string]CommandFunc = map[string]CommandFunc{ + "get": GetCommand, + "set": SetCommand, + "trace": GetCommand, // Also handled by GetCommand +} + +// Find the top of the release config contribution directory. +// Returns the parent of the flag_declarations and flag_values directories. +func GetMapDir(path string) (string, error) { + for p := path; p != "."; p = filepath.Dir(p) { + switch filepath.Base(p) { + case "flag_declarations": + return filepath.Dir(p), nil + case "flag_values": + return filepath.Dir(p), nil + } + } + return "", fmt.Errorf("Could not determine directory from %s", path) +} + +func MarshalFlagValue(config *rc_lib.ReleaseConfig, name string) (ret string, err error) { + fa, ok := config.FlagArtifacts[name] + if !ok { + return "", fmt.Errorf("%s not found in %s", name, config.Name) + } + return rc_lib.MarshalValue(fa.Value), nil +} + +func GetReleaseArgs(configs *rc_lib.ReleaseConfigs, commonFlags Flags) ([]*rc_lib.ReleaseConfig, error) { + var all bool + relFlags := flag.NewFlagSet("set", flag.ExitOnError) + relFlags.BoolVar(&all, "all", false, "Display all flags") + relFlags.Parse(commonFlags.targetReleases) + var ret []*rc_lib.ReleaseConfig + if all { + for _, config := range configs.ReleaseConfigs { + ret = append(ret, config) + } + return ret, nil + } + for _, arg := range relFlags.Args() { + config, err := configs.GetReleaseConfig(arg) + if err != nil { + return nil, err + } + ret = append(ret, config) + } + return ret, nil +} + +func GetCommand(configs *rc_lib.ReleaseConfigs, commonFlags Flags, cmd string, args []string) error { + isTrace := cmd == "trace" + var all bool + getFlags := flag.NewFlagSet("set", flag.ExitOnError) + getFlags.BoolVar(&all, "all", false, "Display all flags") + getFlags.Parse(args) + args = getFlags.Args() + + releaseConfigList, err := GetReleaseArgs(configs, commonFlags) + if err != nil { + return err + } + if isTrace && len(releaseConfigList) > 1 { + return fmt.Errorf("trace command only allows one --release argument. Got: %s", strings.Join(commonFlags.targetReleases, " ")) + } + + if all { + args = []string{} + for _, fa := range configs.FlagArtifacts { + args = append(args, *fa.FlagDeclaration.Name) + } + } + + showName := len(releaseConfigList) > 1 || len(args) > 1 + for _, config := range releaseConfigList { + var configName string + if len(releaseConfigList) > 1 { + configName = fmt.Sprintf("%s.", config.Name) + } + for _, arg := range args { + val, err := MarshalFlagValue(config, arg) + if err != nil { + return err + } + if showName { + fmt.Printf("%s%s=%s\n", configName, arg, val) + } else { + fmt.Printf("%s\n", val) + } + if isTrace { + for _, trace := range config.FlagArtifacts[arg].Traces { + fmt.Printf(" => \"%s\" in %s\n", rc_lib.MarshalValue(trace.Value), *trace.Source) + } + } + } + } + return nil +} + +func SetCommand(configs *rc_lib.ReleaseConfigs, commonFlags Flags, cmd string, args []string) error { + var valueDir string + if len(commonFlags.targetReleases) > 1 { + return fmt.Errorf("set command only allows one --release argument. Got: %s", strings.Join(commonFlags.targetReleases, " ")) + } + targetRelease := commonFlags.targetReleases[0] + + setFlags := flag.NewFlagSet("set", flag.ExitOnError) + setFlags.StringVar(&valueDir, "dir", "", "Directory in which to place the value") + setFlags.Parse(args) + setArgs := setFlags.Args() + if len(setArgs) != 2 { + return fmt.Errorf("set command expected flag and value, got: %s", strings.Join(setArgs, " ")) + } + name := setArgs[0] + value := setArgs[1] + release, err := configs.GetReleaseConfig(targetRelease) + targetRelease = release.Name + if err != nil { + return err + } + flagArtifact, ok := release.FlagArtifacts[name] + if !ok { + return fmt.Errorf("Unknown build flag %s", name) + } + if valueDir == "" { + mapDir, err := GetMapDir(*flagArtifact.Traces[len(flagArtifact.Traces)-1].Source) + if err != nil { + return err + } + valueDir = mapDir + } + + flagValue := &rc_proto.FlagValue{ + Name: proto.String(name), + Value: rc_lib.UnmarshalValue(value), + } + flagPath := filepath.Join(valueDir, "flag_values", targetRelease, fmt.Sprintf("%s.textproto", name)) + return rc_lib.WriteMessage(flagPath, flagValue) +} + +func main() { + var err error + var commonFlags Flags + var configs *rc_lib.ReleaseConfigs + + outEnv := os.Getenv("OUT_DIR") + if outEnv == "" { + outEnv = "out" + } + // Handle the common arguments + flag.StringVar(&commonFlags.top, "top", ".", "path to top of workspace") + flag.BoolVar(&commonFlags.quiet, "quiet", false, "disable warning messages") + flag.Var(&commonFlags.maps, "map", "path to a release_config_map.textproto. may be repeated") + flag.StringVar(&commonFlags.outDir, "out_dir", rc_lib.GetDefaultOutDir(), "basepath for the output. Multiple formats are created") + flag.Var(&commonFlags.targetReleases, "release", "TARGET_RELEASE for this build") + flag.Parse() + + if commonFlags.quiet { + rc_lib.DisableWarnings() + } + + if len(commonFlags.targetReleases) == 0 { + commonFlags.targetReleases = rc_lib.StringList{"trunk_staging"} + } + + if err = os.Chdir(commonFlags.top); err != nil { + panic(err) + } + + // Get the current state of flagging. + relName := commonFlags.targetReleases[0] + if relName == "--all" || relName == "-all" { + // If the users said `--release --all`, grab trunk staging for simplicity. + relName = "trunk_staging" + } + configs, err = rc_lib.ReadReleaseConfigMaps(commonFlags.maps, relName) + if err != nil { + panic(err) + } + + if cmd, ok := commandMap[flag.Arg(0)]; ok { + args := flag.Args() + if err = cmd(configs, commonFlags, args[0], args[1:]); err != nil { + panic(err) + } + } +} diff --git a/cmd/release_config/crunch_flags/Android.bp b/cmd/release_config/crunch_flags/Android.bp new file mode 100644 index 000000000..89c95913d --- /dev/null +++ b/cmd/release_config/crunch_flags/Android.bp @@ -0,0 +1,32 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +blueprint_go_binary { + name: "crunch-flags", + deps: [ + "golang-protobuf-encoding-prototext", + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-release_config-lib", + "soong-cmd-release_config-proto", + ], + srcs: [ + "main.go", + ], +} + +bootstrap_go_package { + name: "soong-cmd-release_config-crunch_flags", + pkgPath: "android/soong/cmd/release_config/crunch_flags", + deps: [ + "golang-protobuf-encoding-prototext", + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-release_config-lib", + "soong-cmd-release_config-proto", + ], + srcs: [ + "main.go", + ], +} diff --git a/cmd/release_config/crunch_flags/main.go b/cmd/release_config/crunch_flags/main.go new file mode 100644 index 000000000..69abba2a5 --- /dev/null +++ b/cmd/release_config/crunch_flags/main.go @@ -0,0 +1,362 @@ +package main + +import ( + "flag" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + + rc_lib "android/soong/cmd/release_config/release_config_lib" + rc_proto "android/soong/cmd/release_config/release_config_proto" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/proto" +) + +// When a flag declaration has an initial value that is a string, the default workflow is PREBUILT. +// If the flag name starts with any of prefixes in manualFlagNamePrefixes, it is MANUAL. +var manualFlagNamePrefixes []string = []string{ + "RELEASE_ACONFIG_", + "RELEASE_PLATFORM_", +} + +var defaultFlagNamespace string = "android_UNKNOWN" + +func RenameNext(name string) string { + if name == "next" { + return "ap3a" + } + return name +} + +func WriteFile(path string, message proto.Message) error { + data, err := prototext.MarshalOptions{Multiline: true}.Marshal(message) + if err != nil { + return err + } + + err = os.MkdirAll(filepath.Dir(path), 0775) + if err != nil { + return err + } + return os.WriteFile(path, data, 0644) +} + +func WalkValueFiles(dir string, Func fs.WalkDirFunc) error { + valPath := filepath.Join(dir, "build_config") + if _, err := os.Stat(valPath); err != nil { + fmt.Printf("%s not found, ignoring.\n", valPath) + return nil + } + + return filepath.WalkDir(valPath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if strings.HasSuffix(d.Name(), ".scl") && d.Type().IsRegular() { + return Func(path, d, err) + } + return nil + }) +} + +func ProcessBuildFlags(dir string, namespaceMap map[string]string) error { + var rootAconfigModule string + + path := filepath.Join(dir, "build_flags.scl") + if _, err := os.Stat(path); err != nil { + fmt.Printf("%s not found, ignoring.\n", path) + return nil + } else { + fmt.Printf("Processing %s\n", path) + } + commentRegexp, err := regexp.Compile("^[[:space:]]*#(?<comment>.+)") + if err != nil { + return err + } + declRegexp, err := regexp.Compile("^[[:space:]]*flag.\"(?<name>[A-Z_0-9]+)\",[[:space:]]*(?<container>[_A-Z]*),[[:space:]]*(?<value>(\"[^\"]*\"|[^\",)]*))") + if err != nil { + return err + } + declIn, err := os.ReadFile(path) + if err != nil { + return err + } + lines := strings.Split(string(declIn), "\n") + var description string + for _, line := range lines { + if comment := commentRegexp.FindStringSubmatch(commentRegexp.FindString(line)); comment != nil { + // Description is the text from any contiguous series of lines before a `flag()` call. + descLine := strings.TrimSpace(comment[commentRegexp.SubexpIndex("comment")]) + if !strings.HasPrefix(descLine, "keep-sorted") { + description += fmt.Sprintf(" %s", descLine) + } + continue + } + matches := declRegexp.FindStringSubmatch(declRegexp.FindString(line)) + if matches == nil { + // The line is neither a comment nor a `flag()` call. + // Discard any description we have gathered and process the next line. + description = "" + continue + } + declValue := matches[declRegexp.SubexpIndex("value")] + declName := matches[declRegexp.SubexpIndex("name")] + container := rc_proto.Container(rc_proto.Container_value[matches[declRegexp.SubexpIndex("container")]]) + description = strings.TrimSpace(description) + var namespace string + var ok bool + if namespace, ok = namespaceMap[declName]; !ok { + namespace = defaultFlagNamespace + } + flagDeclaration := &rc_proto.FlagDeclaration{ + Name: proto.String(declName), + Namespace: proto.String(namespace), + Description: proto.String(description), + Container: &container, + } + description = "" + // Most build flags are `workflow: PREBUILT`. + workflow := rc_proto.Workflow(rc_proto.Workflow_PREBUILT) + switch { + case declName == "RELEASE_ACONFIG_VALUE_SETS": + rootAconfigModule = declValue[1 : len(declValue)-1] + continue + case strings.HasPrefix(declValue, "\""): + // String values mean that the flag workflow is (most likely) either MANUAL or PREBUILT. + declValue = declValue[1 : len(declValue)-1] + flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{declValue}} + for _, prefix := range manualFlagNamePrefixes { + if strings.HasPrefix(declName, prefix) { + workflow = rc_proto.Workflow(rc_proto.Workflow_MANUAL) + break + } + } + case declValue == "False" || declValue == "True": + // Boolean values are LAUNCH flags. + flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{declValue == "True"}} + workflow = rc_proto.Workflow(rc_proto.Workflow_LAUNCH) + case declValue == "None": + // Use PREBUILT workflow with no initial value. + default: + fmt.Printf("%s: Unexpected value %s=%s\n", path, declName, declValue) + } + flagDeclaration.Workflow = &workflow + if flagDeclaration != nil { + declPath := filepath.Join(dir, "flag_declarations", fmt.Sprintf("%s.textproto", declName)) + err := WriteFile(declPath, flagDeclaration) + if err != nil { + return err + } + } + } + if rootAconfigModule != "" { + rootProto := &rc_proto.ReleaseConfig{ + Name: proto.String("root"), + AconfigValueSets: []string{rootAconfigModule}, + } + return WriteFile(filepath.Join(dir, "release_configs", "root.textproto"), rootProto) + } + return nil +} + +func ProcessBuildConfigs(dir, name string, paths []string, releaseProto *rc_proto.ReleaseConfig) error { + valRegexp, err := regexp.Compile("[[:space:]]+value.\"(?<name>[A-Z_0-9]+)\",[[:space:]]*(?<value>[^,)]*)") + if err != nil { + return err + } + for _, path := range paths { + fmt.Printf("Processing %s\n", path) + valIn, err := os.ReadFile(path) + if err != nil { + fmt.Printf("%s: error: %v\n", path, err) + return err + } + vals := valRegexp.FindAllString(string(valIn), -1) + for _, val := range vals { + matches := valRegexp.FindStringSubmatch(val) + valValue := matches[valRegexp.SubexpIndex("value")] + valName := matches[valRegexp.SubexpIndex("name")] + flagValue := &rc_proto.FlagValue{ + Name: proto.String(valName), + } + switch { + case valName == "RELEASE_ACONFIG_VALUE_SETS": + flagValue = nil + if releaseProto.AconfigValueSets == nil { + releaseProto.AconfigValueSets = []string{} + } + releaseProto.AconfigValueSets = append(releaseProto.AconfigValueSets, valValue[1:len(valValue)-1]) + case strings.HasPrefix(valValue, "\""): + valValue = valValue[1 : len(valValue)-1] + flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{valValue}} + case valValue == "None": + // nothing to do here. + case valValue == "True": + flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{true}} + case valValue == "False": + flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{false}} + default: + fmt.Printf("%s: Unexpected value %s=%s\n", path, valName, valValue) + } + if flagValue != nil { + valPath := filepath.Join(dir, "flag_values", RenameNext(name), fmt.Sprintf("%s.textproto", valName)) + err := WriteFile(valPath, flagValue) + if err != nil { + return err + } + } + } + } + return err +} + +func ProcessReleaseConfigMap(dir string, descriptionMap map[string]string) error { + path := filepath.Join(dir, "release_config_map.mk") + if _, err := os.Stat(path); err != nil { + fmt.Printf("%s not found, ignoring.\n", path) + return nil + } else { + fmt.Printf("Processing %s\n", path) + } + configRegexp, err := regexp.Compile("^..call[[:space:]]+declare-release-config,[[:space:]]+(?<name>[_a-z0-9A-Z]+),[[:space:]]+(?<files>[^,]*)(,[[:space:]]*(?<inherits>.*)|[[:space:]]*)[)]$") + if err != nil { + return err + } + aliasRegexp, err := regexp.Compile("^..call[[:space:]]+alias-release-config,[[:space:]]+(?<name>[_a-z0-9A-Z]+),[[:space:]]+(?<target>[_a-z0-9A-Z]+)") + if err != nil { + return err + } + + mapIn, err := os.ReadFile(path) + if err != nil { + return err + } + cleanDir := strings.TrimLeft(dir, "../") + var defaultContainer rc_proto.Container + switch { + case strings.HasPrefix(cleanDir, "build/") || cleanDir == "vendor/google_shared/build": + defaultContainer = rc_proto.Container(rc_proto.Container_ALL) + case cleanDir == "vendor/google/release": + defaultContainer = rc_proto.Container(rc_proto.Container_ALL) + default: + defaultContainer = rc_proto.Container(rc_proto.Container_VENDOR) + } + releaseConfigMap := &rc_proto.ReleaseConfigMap{DefaultContainer: &defaultContainer} + // If we find a description for the directory, include it. + if description, ok := descriptionMap[cleanDir]; ok { + releaseConfigMap.Description = proto.String(description) + } + lines := strings.Split(string(mapIn), "\n") + for _, line := range lines { + alias := aliasRegexp.FindStringSubmatch(aliasRegexp.FindString(line)) + if alias != nil { + fmt.Printf("processing alias %s\n", line) + name := alias[aliasRegexp.SubexpIndex("name")] + target := alias[aliasRegexp.SubexpIndex("target")] + if target == "next" { + if RenameNext(target) != name { + return fmt.Errorf("Unexpected name for next (%s)", RenameNext(target)) + } + target, name = name, target + } + releaseConfigMap.Aliases = append(releaseConfigMap.Aliases, + &rc_proto.ReleaseAlias{ + Name: proto.String(name), + Target: proto.String(target), + }) + } + config := configRegexp.FindStringSubmatch(configRegexp.FindString(line)) + if config == nil { + continue + } + name := config[configRegexp.SubexpIndex("name")] + releaseConfig := &rc_proto.ReleaseConfig{ + Name: proto.String(RenameNext(name)), + } + configFiles := config[configRegexp.SubexpIndex("files")] + files := strings.Split(strings.ReplaceAll(configFiles, "$(local_dir)", dir+"/"), " ") + configInherits := config[configRegexp.SubexpIndex("inherits")] + if len(configInherits) > 0 { + releaseConfig.Inherits = strings.Split(configInherits, " ") + } + err := ProcessBuildConfigs(dir, name, files, releaseConfig) + if err != nil { + return err + } + + releasePath := filepath.Join(dir, "release_configs", fmt.Sprintf("%s.textproto", RenameNext(name))) + err = WriteFile(releasePath, releaseConfig) + if err != nil { + return err + } + } + return WriteFile(filepath.Join(dir, "release_config_map.textproto"), releaseConfigMap) +} + +func main() { + var err error + var top string + var dirs rc_lib.StringList + var namespacesFile string + var descriptionsFile string + + flag.StringVar(&top, "top", ".", "path to top of workspace") + flag.Var(&dirs, "dir", "directory to process, relative to the top of the workspace") + flag.StringVar(&namespacesFile, "namespaces", "", "location of file with 'flag_name namespace' information") + flag.StringVar(&descriptionsFile, "descriptions", "", "location of file with 'directory description' information") + flag.Parse() + + if err = os.Chdir(top); err != nil { + panic(err) + } + if len(dirs) == 0 { + dirs = rc_lib.StringList{"build/release", "vendor/google_shared/build/release", "vendor/google/release"} + } + + namespaceMap := make(map[string]string) + if namespacesFile != "" { + data, err := os.ReadFile(namespacesFile) + if err != nil { + panic(err) + } + for idx, line := range strings.Split(string(data), "\n") { + fields := strings.Split(line, " ") + if len(fields) > 2 { + panic(fmt.Errorf("line %d: too many fields: %s", idx, line)) + } + namespaceMap[fields[0]] = fields[1] + } + + } + + descriptionMap := make(map[string]string) + descriptionMap["build/release"] = "Published open-source flags and declarations" + if descriptionsFile != "" { + data, err := os.ReadFile(descriptionsFile) + if err != nil { + panic(err) + } + for _, line := range strings.Split(string(data), "\n") { + if strings.TrimSpace(line) != "" { + fields := strings.SplitN(line, " ", 2) + descriptionMap[fields[0]] = fields[1] + } + } + + } + + for _, dir := range dirs { + err = ProcessBuildFlags(dir, namespaceMap) + if err != nil { + panic(err) + } + + err = ProcessReleaseConfigMap(dir, descriptionMap) + if err != nil { + panic(err) + } + } +} diff --git a/cmd/release_config/main.go b/cmd/release_config/main.go deleted file mode 100644 index 3bb6b3dfe..000000000 --- a/cmd/release_config/main.go +++ /dev/null @@ -1,691 +0,0 @@ -// Copyright 2024 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "cmp" - "encoding/json" - "flag" - "fmt" - "io/fs" - "os" - "path/filepath" - "slices" - "strings" - - "android/soong/cmd/release_config/release_config_proto" - - "google.golang.org/protobuf/encoding/prototext" - "google.golang.org/protobuf/proto" -) - -var verboseFlag bool - -type StringList []string - -func (l *StringList) Set(v string) error { - *l = append(*l, v) - return nil -} - -func (l *StringList) String() string { - return fmt.Sprintf("%v", *l) -} - -var releaseConfigMapPaths StringList - -func DumpProtos(outDir string, message proto.Message) error { - basePath := filepath.Join(outDir, "all_release_configs") - writer := func(suffix string, marshal func() ([]byte, error)) error { - data, err := marshal() - if err != nil { - return err - } - return os.WriteFile(fmt.Sprintf("%s.%s", basePath, suffix), data, 0644) - } - err := writer("textproto", func() ([]byte, error) { return prototext.MarshalOptions{Multiline: true}.Marshal(message) }) - if err != nil { - return err - } - - err = writer("pb", func() ([]byte, error) { return proto.Marshal(message) }) - if err != nil { - return err - } - - return writer("json", func() ([]byte, error) { return json.MarshalIndent(message, "", " ") }) -} - -func LoadTextproto(path string, message proto.Message) error { - data, err := os.ReadFile(path) - if err != nil { - return err - } - ret := prototext.Unmarshal(data, message) - if verboseFlag { - debug, _ := prototext.Marshal(message) - fmt.Printf("%s: %s\n", path, debug) - } - return ret -} - -func WalkTextprotoFiles(root string, subdir string, Func fs.WalkDirFunc) error { - path := filepath.Join(root, subdir) - if _, err := os.Stat(path); err != nil { - // Missing subdirs are not an error. - return nil - } - return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if strings.HasSuffix(d.Name(), ".textproto") && d.Type().IsRegular() { - return Func(path, d, err) - } - return nil - }) -} - -type FlagValue struct { - // The path providing this value. - path string - - // Protobuf - proto release_config_proto.FlagValue -} - -func FlagValueFactory(protoPath string) (fv *FlagValue) { - fv = &FlagValue{path: protoPath} - if protoPath != "" { - LoadTextproto(protoPath, &fv.proto) - } - return fv -} - -// One directory's contribution to the a release config. -type ReleaseConfigContribution struct { - // Paths to files providing this config. - path string - - // The index of the config directory where this release config - // contribution was declared. - // Flag values cannot be set in a location with a lower index. - DeclarationIndex int - - // Protobufs relevant to the config. - proto release_config_proto.ReleaseConfig - - FlagValues []*FlagValue -} - -// A single release_config_map.textproto and its associated data. -// Used primarily for debugging. -type ReleaseConfigMap struct { - // The path to this release_config_map file. - path string - - // Data received - proto release_config_proto.ReleaseConfigMap - - ReleaseConfigContributions map[string]*ReleaseConfigContribution - FlagDeclarations []release_config_proto.FlagDeclaration -} - -// A generated release config. -type ReleaseConfig struct { - // the Name of the release config - Name string - - // The index of the config directory where this release config was - // first declared. - // Flag values cannot be set in a location with a lower index. - DeclarationIndex int - - // What contributes to this config. - Contributions []*ReleaseConfigContribution - - // Aliases for this release - OtherNames []string - - // The names of release configs that we inherit - InheritNames []string - - // Unmarshalled flag artifacts - FlagArtifacts FlagArtifacts - - // Generated release config - ReleaseConfigArtifact *release_config_proto.ReleaseConfigArtifact - - // We have begun compiling this release config. - compileInProgress bool -} - -type FlagArtifact struct { - FlagDeclaration *release_config_proto.FlagDeclaration - - // The index of the config directory where this flag was declared. - // Flag values cannot be set in a location with a lower index. - DeclarationIndex int - - Traces []*release_config_proto.Tracepoint - - // Assigned value - Value *release_config_proto.Value -} - -// Key is flag name. -type FlagArtifacts map[string]*FlagArtifact - -type ReleaseConfigDirMap map[string]int - -// The generated release configs. -type ReleaseConfigs struct { - // Ordered list of release config maps processed. - ReleaseConfigMaps []*ReleaseConfigMap - - // Aliases - Aliases map[string]*string - - // Dictionary of flag_name:FlagDeclaration, with no overrides applied. - FlagArtifacts FlagArtifacts - - // Dictionary of name:ReleaseConfig - ReleaseConfigs map[string]*ReleaseConfig - - // Generated release configs - Artifact release_config_proto.ReleaseConfigsArtifact - - // The list of config directories used. - ConfigDirs []string - - // A map from the config directory to its order in the list of config - // directories. - ConfigDirIndexes ReleaseConfigDirMap -} - -func (src *FlagArtifact) Clone() *FlagArtifact { - value := &release_config_proto.Value{} - proto.Merge(value, src.Value) - return &FlagArtifact{ - FlagDeclaration: src.FlagDeclaration, - Traces: src.Traces, - Value: value, - } -} - -func (src FlagArtifacts) Clone() (dst FlagArtifacts) { - if dst == nil { - dst = make(FlagArtifacts) - } - for k, v := range src { - dst[k] = v.Clone() - } - return -} - -func ReleaseConfigFactory(name string, index int) (c *ReleaseConfig) { - return &ReleaseConfig{Name: name, DeclarationIndex: index} -} - -func ReleaseConfigsFactory() (c *ReleaseConfigs) { - return &ReleaseConfigs{ - Aliases: make(map[string]*string), - FlagArtifacts: make(map[string]*FlagArtifact), - ReleaseConfigs: make(map[string]*ReleaseConfig), - ConfigDirs: []string{}, - ConfigDirIndexes: make(ReleaseConfigDirMap), - } -} - -func ReleaseConfigMapFactory(protoPath string) (m *ReleaseConfigMap) { - m = &ReleaseConfigMap{ - path: protoPath, - ReleaseConfigContributions: make(map[string]*ReleaseConfigContribution), - } - if protoPath != "" { - LoadTextproto(protoPath, &m.proto) - } - return m -} - -func FlagDeclarationFactory(protoPath string) (fd *release_config_proto.FlagDeclaration) { - fd = &release_config_proto.FlagDeclaration{} - if protoPath != "" { - LoadTextproto(protoPath, fd) - } - return fd -} - -func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex int) error { - m := ReleaseConfigMapFactory(path) - if m.proto.Origin == nil || *m.proto.Origin == "" { - return fmt.Errorf("Release config map %s lacks origin", path) - } - if m.proto.DefaultContainer == nil { - return fmt.Errorf("Release config map %s lacks default_container", path) - } - dir := filepath.Dir(path) - // Record any aliases, checking for duplicates. - for _, alias := range m.proto.Aliases { - name := *alias.Name - oldTarget, ok := configs.Aliases[name] - if ok { - if *oldTarget != *alias.Target { - return fmt.Errorf("Conflicting alias declarations: %s vs %s", - *oldTarget, *alias.Target) - } - } - configs.Aliases[name] = alias.Target - } - var err error - err = WalkTextprotoFiles(dir, "flag_declarations", func(path string, d fs.DirEntry, err error) error { - flagDeclaration := FlagDeclarationFactory(path) - // Container must be specified. - if flagDeclaration.Container == nil { - flagDeclaration.Container = m.proto.DefaultContainer - } - // TODO: drop flag_declaration.origin from the proto. - if flagDeclaration.Origin == nil { - flagDeclaration.Origin = m.proto.Origin - } - // There is always a default value. - if flagDeclaration.Value == nil { - flagDeclaration.Value = &release_config_proto.Value{Val: &release_config_proto.Value_UnspecifiedValue{true}} - } - m.FlagDeclarations = append(m.FlagDeclarations, *flagDeclaration) - name := *flagDeclaration.Name - if def, ok := configs.FlagArtifacts[name]; !ok { - configs.FlagArtifacts[name] = &FlagArtifact{FlagDeclaration: flagDeclaration, DeclarationIndex: ConfigDirIndex} - } else if !proto.Equal(def.FlagDeclaration, flagDeclaration) { - return fmt.Errorf("Duplicate definition of %s", *flagDeclaration.Name) - } - // Set the initial value in the flag artifact. - configs.FlagArtifacts[name].UpdateValue( - FlagValue{path: path, proto: release_config_proto.FlagValue{ - Name: proto.String(name), Value: flagDeclaration.Value}}) - return nil - }) - if err != nil { - return err - } - - err = WalkTextprotoFiles(dir, "release_configs", func(path string, d fs.DirEntry, err error) error { - releaseConfigContribution := &ReleaseConfigContribution{path: path, DeclarationIndex: ConfigDirIndex} - LoadTextproto(path, &releaseConfigContribution.proto) - name := *releaseConfigContribution.proto.Name - if fmt.Sprintf("%s.textproto", name) != filepath.Base(path) { - return fmt.Errorf("%s incorrectly declares release config %s", path, name) - } - if _, ok := configs.ReleaseConfigs[name]; !ok { - configs.ReleaseConfigs[name] = ReleaseConfigFactory(name, ConfigDirIndex) - } - config := configs.ReleaseConfigs[name] - config.InheritNames = append(config.InheritNames, releaseConfigContribution.proto.Inherits...) - - // Only walk flag_values/{RELEASE} for defined releases. - err2 := WalkTextprotoFiles(dir, filepath.Join("flag_values", name), func(path string, d fs.DirEntry, err error) error { - flagValue := FlagValueFactory(path) - if fmt.Sprintf("%s.textproto", *flagValue.proto.Name) != filepath.Base(path) { - return fmt.Errorf("%s incorrectly sets value for flag %s", path, *flagValue.proto.Name) - } - releaseConfigContribution.FlagValues = append(releaseConfigContribution.FlagValues, flagValue) - return nil - }) - if err2 != nil { - return err2 - } - m.ReleaseConfigContributions[name] = releaseConfigContribution - config.Contributions = append(config.Contributions, releaseConfigContribution) - return nil - }) - if err != nil { - return err - } - configs.ReleaseConfigMaps = append(configs.ReleaseConfigMaps, m) - return nil -} - -func (configs *ReleaseConfigs) GetReleaseConfig(name string) (*ReleaseConfig, error) { - trace := []string{name} - for target, ok := configs.Aliases[name]; ok; target, ok = configs.Aliases[name] { - name = *target - trace = append(trace, name) - } - if config, ok := configs.ReleaseConfigs[name]; ok { - return config, nil - } - return nil, fmt.Errorf("Missing config %s. Trace=%v", name, trace) -} - -func (configs *ReleaseConfigs) DumpMakefile(outDir, targetRelease string) error { - outFile := filepath.Join(outDir, "release_config.mk") - makeVars := make(map[string]string) - config, err := configs.GetReleaseConfig(targetRelease) - if err != nil { - return err - } - // Sort the flags by name first. - names := []string{} - for k, _ := range config.FlagArtifacts { - names = append(names, k) - } - slices.SortFunc(names, func(a, b string) int { - return cmp.Compare(a, b) - }) - partitions := make(map[string][]string) - - vNames := []string{} - addVar := func(name, suffix, value string) { - fullName := fmt.Sprintf("_ALL_RELEASE_FLAGS.%s.%s", name, suffix) - vNames = append(vNames, fullName) - makeVars[fullName] = value - } - - for _, name := range names { - flag := config.FlagArtifacts[name] - decl := flag.FlagDeclaration - - // cName := strings.ToLower(release_config_proto.Container_name[decl.GetContainer()]) - cName := strings.ToLower(decl.Container.String()) - if cName == strings.ToLower(release_config_proto.Container_ALL.String()) { - partitions["product"] = append(partitions["product"], name) - partitions["system"] = append(partitions["system"], name) - partitions["system_ext"] = append(partitions["system_ext"], name) - partitions["vendor"] = append(partitions["vendor"], name) - } else { - partitions[cName] = append(partitions[cName], name) - } - value := MarshalValue(flag.Value) - makeVars[name] = value - addVar(name, "PARTITIONS", cName) - addVar(name, "DEFAULT", MarshalValue(decl.Value)) - addVar(name, "VALUE", value) - addVar(name, "DECLARED_IN", *flag.Traces[0].Source) - addVar(name, "SET_IN", *flag.Traces[len(flag.Traces)-1].Source) - addVar(name, "ORIGIN", *decl.Origin) - } - pNames := []string{} - for k, _ := range partitions { - pNames = append(pNames, k) - } - slices.SortFunc(pNames, func(a, b string) int { - return cmp.Compare(a, b) - }) - - // Now sort the make variables, and output them. - slices.SortFunc(vNames, func(a, b string) int { - return cmp.Compare(a, b) - }) - - // Write the flags as: - // _ALL_RELELASE_FLAGS - // _ALL_RELEASE_FLAGS.PARTITIONS.* - // all _ALL_RELEASE_FLAGS.*, sorted by name - // Final flag values, sorted by name. - data := fmt.Sprintf("_ALL_RELEASE_FLAGS :=$= %s\n", strings.Join(names, " ")) - for _, pName := range pNames { - data += fmt.Sprintf("_ALL_RELEASE_FLAGS.PARTITIONS.%s :=$= %s\n", pName, strings.Join(partitions[pName], " ")) - } - for _, vName := range vNames { - data += fmt.Sprintf("%s :=$= %s\n", vName, makeVars[vName]) - } - data += "\n\n# Values for all build flags\n" - data += fmt.Sprintf("RELEASE_ACONFIG_VALUE_SETS :=$= %s\n", - strings.Join(config.ReleaseConfigArtifact.AconfigValueSets, " ")) - for _, name := range names { - data += fmt.Sprintf("%s :=$= %s\n", name, makeVars[name]) - } - return os.WriteFile(outFile, []byte(data), 0644) -} - -func (configs *ReleaseConfigs) GenerateReleaseConfigs(targetRelease string) error { - otherNames := make(map[string][]string) - for aliasName, aliasTarget := range configs.Aliases { - if _, ok := configs.ReleaseConfigs[aliasName]; ok { - return fmt.Errorf("Alias %s is a declared release config", aliasName) - } - if _, ok := configs.ReleaseConfigs[*aliasTarget]; !ok { - if _, ok2 := configs.Aliases[*aliasTarget]; !ok2 { - return fmt.Errorf("Alias %s points to non-existing config %s", aliasName, *aliasTarget) - } - } - otherNames[*aliasTarget] = append(otherNames[*aliasTarget], aliasName) - } - for name, aliases := range otherNames { - configs.ReleaseConfigs[name].OtherNames = aliases - } - - for _, config := range configs.ReleaseConfigs { - err := config.GenerateReleaseConfig(configs) - if err != nil { - return err - } - } - - releaseConfig, err := configs.GetReleaseConfig(targetRelease) - if err != nil { - return err - } - configs.Artifact = release_config_proto.ReleaseConfigsArtifact{ - ReleaseConfig: releaseConfig.ReleaseConfigArtifact, - OtherReleaseConfigs: func() []*release_config_proto.ReleaseConfigArtifact { - orc := []*release_config_proto.ReleaseConfigArtifact{} - for name, config := range configs.ReleaseConfigs { - if name != releaseConfig.Name { - orc = append(orc, config.ReleaseConfigArtifact) - } - } - return orc - }(), - } - return nil -} - -func MarshalValue(value *release_config_proto.Value) string { - switch val := value.Val.(type) { - case *release_config_proto.Value_UnspecifiedValue: - // Value was never set. - return "" - case *release_config_proto.Value_StringValue: - return val.StringValue - case *release_config_proto.Value_BoolValue: - if val.BoolValue { - return "true" - } - // False ==> empty string - return "" - case *release_config_proto.Value_Obsolete: - return " #OBSOLETE" - default: - // Flagged as error elsewhere, so return empty string here. - return "" - } -} - -func (fa *FlagArtifact) UpdateValue(flagValue FlagValue) error { - name := *flagValue.proto.Name - fa.Traces = append(fa.Traces, &release_config_proto.Tracepoint{Source: proto.String(flagValue.path), Value: flagValue.proto.Value}) - if fa.Value.GetObsolete() { - return fmt.Errorf("Attempting to set obsolete flag %s. Trace=%v", name, fa.Traces) - } - switch val := flagValue.proto.Value.Val.(type) { - case *release_config_proto.Value_StringValue: - fa.Value = &release_config_proto.Value{Val: &release_config_proto.Value_StringValue{val.StringValue}} - case *release_config_proto.Value_BoolValue: - fa.Value = &release_config_proto.Value{Val: &release_config_proto.Value_BoolValue{val.BoolValue}} - case *release_config_proto.Value_Obsolete: - if !val.Obsolete { - return fmt.Errorf("%s: Cannot set obsolete=false. Trace=%v", name, fa.Traces) - } - fa.Value = &release_config_proto.Value{Val: &release_config_proto.Value_Obsolete{true}} - default: - return fmt.Errorf("Invalid type for flag_value: %T. Trace=%v", val, fa.Traces) - } - return nil -} - -func (fa *FlagArtifact) Marshal() (*release_config_proto.FlagArtifact, error) { - return &release_config_proto.FlagArtifact{ - FlagDeclaration: fa.FlagDeclaration, - Value: fa.Value, - Traces: fa.Traces, - }, nil -} - -func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) error { - if config.ReleaseConfigArtifact != nil { - return nil - } - if config.compileInProgress { - return fmt.Errorf("Loop detected for release config %s", config.Name) - } - config.compileInProgress = true - - // Generate any configs we need to inherit. This will detect loops in - // the config. - contributionsToApply := []*ReleaseConfigContribution{} - myInherits := []string{} - myInheritsSet := make(map[string]bool) - for _, inherit := range config.InheritNames { - if _, ok := myInheritsSet[inherit]; ok { - continue - } - myInherits = append(myInherits, inherit) - myInheritsSet[inherit] = true - iConfig, err := configs.GetReleaseConfig(inherit) - if err != nil { - return err - } - iConfig.GenerateReleaseConfig(configs) - contributionsToApply = append(contributionsToApply, iConfig.Contributions...) - } - contributionsToApply = append(contributionsToApply, config.Contributions...) - - myAconfigValueSets := []string{} - myFlags := configs.FlagArtifacts.Clone() - myDirsMap := make(map[int]bool) - for _, contrib := range contributionsToApply { - myAconfigValueSets = append(myAconfigValueSets, contrib.proto.AconfigValueSets...) - myDirsMap[contrib.DeclarationIndex] = true - for _, value := range contrib.FlagValues { - fa, ok := myFlags[*value.proto.Name] - if !ok { - return fmt.Errorf("Setting value for undefined flag %s in %s\n", *value.proto.Name, value.path) - } - myDirsMap[fa.DeclarationIndex] = true - if fa.DeclarationIndex > contrib.DeclarationIndex { - // Setting location is to the left of declaration. - return fmt.Errorf("Setting value for flag %s not allowed in %s\n", *value.proto.Name, value.path) - } - if err := fa.UpdateValue(*value); err != nil { - return err - } - } - } - - directories := []string{} - for idx, confDir := range configs.ConfigDirs { - if _, ok := myDirsMap[idx]; ok { - directories = append(directories, confDir) - } - } - - config.FlagArtifacts = myFlags - config.ReleaseConfigArtifact = &release_config_proto.ReleaseConfigArtifact{ - Name: proto.String(config.Name), - OtherNames: config.OtherNames, - FlagArtifacts: func() []*release_config_proto.FlagArtifact { - ret := []*release_config_proto.FlagArtifact{} - for _, flag := range myFlags { - ret = append(ret, &release_config_proto.FlagArtifact{ - FlagDeclaration: flag.FlagDeclaration, - Traces: flag.Traces, - Value: flag.Value, - }) - } - return ret - }(), - AconfigValueSets: myAconfigValueSets, - Inherits: myInherits, - Directories: directories, - } - - config.compileInProgress = false - return nil -} - -func main() { - var targetRelease string - var outputDir string - - outEnv := os.Getenv("OUT_DIR") - if outEnv == "" { - outEnv = "out" - } - defaultOutputDir := filepath.Join(outEnv, "soong", "release-config") - var defaultMapPaths StringList - defaultLocations := StringList{ - "build/release/release_config_map.textproto", - "vendor/google_shared/build/release/release_config_map.textproto", - "vendor/google/release/release_config_map.textproto", - } - for _, path := range defaultLocations { - if _, err := os.Stat(path); err == nil { - defaultMapPaths = append(defaultMapPaths, path) - } - } - prodMaps := os.Getenv("PRODUCT_RELEASE_CONFIG_MAPS") - if prodMaps != "" { - defaultMapPaths = append(defaultMapPaths, strings.Split(prodMaps, " ")...) - } - - flag.BoolVar(&verboseFlag, "debug", false, "print debugging information") - flag.Var(&releaseConfigMapPaths, "map", "path to a release_config_map.textproto. may be repeated") - flag.StringVar(&targetRelease, "release", "trunk_staging", "TARGET_RELEASE for this build") - flag.StringVar(&outputDir, "out_dir", defaultOutputDir, "basepath for the output. Multiple formats are created") - flag.Parse() - - if len(releaseConfigMapPaths) == 0 { - releaseConfigMapPaths = defaultMapPaths - fmt.Printf("No --map argument provided. Using: --map %s\n", strings.Join(releaseConfigMapPaths, " --map ")) - } - - configs := ReleaseConfigsFactory() - for idx, releaseConfigMapPath := range releaseConfigMapPaths { - // Maintain an ordered list of release config directories. - configDir := filepath.Dir(releaseConfigMapPath) - configs.ConfigDirIndexes[configDir] = idx - configs.ConfigDirs = append(configs.ConfigDirs, configDir) - err := configs.LoadReleaseConfigMap(releaseConfigMapPath, idx) - if err != nil { - panic(err) - } - } - - // Now that we have all of the release config maps, can meld them and generate the artifacts. - err := configs.GenerateReleaseConfigs(targetRelease) - if err != nil { - panic(err) - } - err = os.MkdirAll(outputDir, 0775) - if err != nil { - panic(err) - } - err = configs.DumpMakefile(outputDir, targetRelease) - if err != nil { - panic(err) - } - DumpProtos(outputDir, &configs.Artifact) -} diff --git a/cmd/release_config/release_config/Android.bp b/cmd/release_config/release_config/Android.bp new file mode 100644 index 000000000..3c7382637 --- /dev/null +++ b/cmd/release_config/release_config/Android.bp @@ -0,0 +1,18 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-cmd-release_config-release_config", + pkgPath: "android/soong/cmd/release_config/release_config", + deps: [ + "golang-protobuf-encoding-prototext", + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-release_config-proto", + "soong-cmd-release_config-lib", + ], + srcs: [ + "main.go", + ], +} diff --git a/cmd/release_config/release_config/main.go b/cmd/release_config/release_config/main.go new file mode 100644 index 000000000..a43fdccbe --- /dev/null +++ b/cmd/release_config/release_config/main.go @@ -0,0 +1,96 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + rc_lib "android/soong/cmd/release_config/release_config_lib" +) + +func main() { + var top string + var quiet bool + var releaseConfigMapPaths rc_lib.StringList + var targetRelease string + var outputDir string + var err error + var configs *rc_lib.ReleaseConfigs + var json, pb, textproto bool + var product string + + defaultRelease := os.Getenv("TARGET_RELEASE") + if defaultRelease == "" { + defaultRelease = "trunk_staging" + } + + flag.StringVar(&top, "top", ".", "path to top of workspace") + flag.StringVar(&product, "product", os.Getenv("TARGET_PRODUCT"), "TARGET_PRODUCT for the build") + flag.BoolVar(&quiet, "quiet", false, "disable warning messages") + flag.Var(&releaseConfigMapPaths, "map", "path to a release_config_map.textproto. may be repeated") + flag.StringVar(&targetRelease, "release", defaultRelease, "TARGET_RELEASE for this build") + flag.StringVar(&outputDir, "out_dir", rc_lib.GetDefaultOutDir(), "basepath for the output. Multiple formats are created") + flag.BoolVar(&textproto, "textproto", true, "write artifacts as text protobuf") + flag.BoolVar(&json, "json", true, "write artifacts as json") + flag.BoolVar(&pb, "pb", true, "write artifacts as binary protobuf") + flag.Parse() + + if quiet { + rc_lib.DisableWarnings() + } + + if err = os.Chdir(top); err != nil { + panic(err) + } + configs, err = rc_lib.ReadReleaseConfigMaps(releaseConfigMapPaths, targetRelease) + if err != nil { + panic(err) + } + config, err := configs.GetReleaseConfig(targetRelease) + if err != nil { + panic(err) + } + releaseName := config.Name + err = os.MkdirAll(outputDir, 0775) + if err != nil { + panic(err) + } + makefilePath := filepath.Join(outputDir, fmt.Sprintf("release_config-%s-%s.mk", product, releaseName)) + err = configs.WriteMakefile(makefilePath, targetRelease) + if err != nil { + panic(err) + } + if json { + err = configs.WriteArtifact(outputDir, product, "json") + if err != nil { + panic(err) + } + } + if pb { + err = configs.WriteArtifact(outputDir, product, "pb") + if err != nil { + panic(err) + } + } + if textproto { + err = configs.WriteArtifact(outputDir, product, "textproto") + if err != nil { + panic(err) + } + } +} diff --git a/cmd/release_config/release_config_lib/Android.bp b/cmd/release_config/release_config_lib/Android.bp new file mode 100644 index 000000000..0c67e1106 --- /dev/null +++ b/cmd/release_config/release_config_lib/Android.bp @@ -0,0 +1,36 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-cmd-release_config-lib", + pkgPath: "android/soong/cmd/release_config/release_config_lib", + deps: [ + "golang-protobuf-encoding-prototext", + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-release_config-proto", + ], + srcs: [ + "flag_artifact.go", + "flag_declaration.go", + "flag_value.go", + "release_config.go", + "release_configs.go", + "util.go", + ], +} diff --git a/cmd/release_config/release_config_lib/flag_artifact.go b/cmd/release_config/release_config_lib/flag_artifact.go new file mode 100644 index 000000000..d6a629b10 --- /dev/null +++ b/cmd/release_config/release_config_lib/flag_artifact.go @@ -0,0 +1,131 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "fmt" + + rc_proto "android/soong/cmd/release_config/release_config_proto" + + "google.golang.org/protobuf/proto" +) + +// A flag artifact, with its final value and declaration/override history. +type FlagArtifact struct { + // The flag_declaration message. + FlagDeclaration *rc_proto.FlagDeclaration + + // The index of the config directory where this flag was declared. + // Flag values cannot be set in a location with a lower index. + DeclarationIndex int + + // A history of value assignments and overrides. + Traces []*rc_proto.Tracepoint + + // The value of the flag. + Value *rc_proto.Value + + // This flag is redacted. Set by UpdateValue when the FlagValue proto + // says to redact it. + Redacted bool +} + +// Key is flag name. +type FlagArtifacts map[string]*FlagArtifact + +// Create a clone of the flag artifact. +// +// Returns: +// +// *FlagArtifact: the copy of the artifact. +func (src *FlagArtifact) Clone() *FlagArtifact { + value := &rc_proto.Value{} + proto.Merge(value, src.Value) + return &FlagArtifact{ + FlagDeclaration: src.FlagDeclaration, + Traces: src.Traces, + Value: value, + } +} + +// Clone FlagArtifacts. +// +// Returns: +// +// FlagArtifacts: a copy of the source FlagArtifacts. +func (src FlagArtifacts) Clone() (dst FlagArtifacts) { + if dst == nil { + dst = make(FlagArtifacts) + } + for k, v := range src { + dst[k] = v.Clone() + } + return +} + +// Update the value of a flag. +// +// This appends to flagArtifact.Traces, and updates flagArtifact.Value. +// +// Args: +// +// flagValue FlagValue: the value to assign +// +// Returns: +// +// error: any error encountered +func (fa *FlagArtifact) UpdateValue(flagValue FlagValue) error { + name := *flagValue.proto.Name + fa.Traces = append(fa.Traces, &rc_proto.Tracepoint{Source: proto.String(flagValue.path), Value: flagValue.proto.Value}) + if flagValue.proto.GetRedacted() { + fa.Redacted = true + fmt.Printf("Redacting flag %s in %s\n", name, flagValue.path) + return nil + } + if fa.Value.GetObsolete() { + return fmt.Errorf("Attempting to set obsolete flag %s. Trace=%v", name, fa.Traces) + } + var newValue *rc_proto.Value + switch val := flagValue.proto.Value.Val.(type) { + case *rc_proto.Value_StringValue: + newValue = &rc_proto.Value{Val: &rc_proto.Value_StringValue{val.StringValue}} + case *rc_proto.Value_BoolValue: + newValue = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{val.BoolValue}} + case *rc_proto.Value_Obsolete: + if !val.Obsolete { + return fmt.Errorf("%s: Cannot set obsolete=false. Trace=%v", name, fa.Traces) + } + newValue = &rc_proto.Value{Val: &rc_proto.Value_Obsolete{true}} + default: + return fmt.Errorf("Invalid type for flag_value: %T. Trace=%v", val, fa.Traces) + } + if proto.Equal(newValue, fa.Value) { + warnf("%s: redundant override (set in %s)\n", flagValue.path, *fa.Traces[len(fa.Traces)-2].Source) + } + fa.Value = newValue + return nil +} + +// Marshal the FlagArtifact into a flag_artifact message. +func (fa *FlagArtifact) Marshal() (*rc_proto.FlagArtifact, error) { + if fa.Redacted { + return nil, nil + } + return &rc_proto.FlagArtifact{ + FlagDeclaration: fa.FlagDeclaration, + Value: fa.Value, + Traces: fa.Traces, + }, nil +} diff --git a/cmd/release_config/release_config_lib/flag_declaration.go b/cmd/release_config/release_config_lib/flag_declaration.go new file mode 100644 index 000000000..97d4d4c76 --- /dev/null +++ b/cmd/release_config/release_config_lib/flag_declaration.go @@ -0,0 +1,27 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + rc_proto "android/soong/cmd/release_config/release_config_proto" +) + +func FlagDeclarationFactory(protoPath string) (fd *rc_proto.FlagDeclaration) { + fd = &rc_proto.FlagDeclaration{} + if protoPath != "" { + LoadMessage(protoPath, fd) + } + return fd +} diff --git a/cmd/release_config/release_config_lib/flag_value.go b/cmd/release_config/release_config_lib/flag_value.go new file mode 100644 index 000000000..e155e7782 --- /dev/null +++ b/cmd/release_config/release_config_lib/flag_value.go @@ -0,0 +1,73 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "strings" + + rc_proto "android/soong/cmd/release_config/release_config_proto" +) + +type FlagValue struct { + // The path providing this value. + path string + + // Protobuf + proto rc_proto.FlagValue +} + +func FlagValueFactory(protoPath string) (fv *FlagValue) { + fv = &FlagValue{path: protoPath} + if protoPath != "" { + LoadMessage(protoPath, &fv.proto) + } + return fv +} + +func UnmarshalValue(str string) *rc_proto.Value { + ret := &rc_proto.Value{} + switch v := strings.ToLower(str); v { + case "true": + ret = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{true}} + case "false": + ret = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{false}} + case "##obsolete": + ret = &rc_proto.Value{Val: &rc_proto.Value_Obsolete{true}} + default: + ret = &rc_proto.Value{Val: &rc_proto.Value_StringValue{str}} + } + return ret +} + +func MarshalValue(value *rc_proto.Value) string { + switch val := value.Val.(type) { + case *rc_proto.Value_UnspecifiedValue: + // Value was never set. + return "" + case *rc_proto.Value_StringValue: + return val.StringValue + case *rc_proto.Value_BoolValue: + if val.BoolValue { + return "true" + } + // False ==> empty string + return "" + case *rc_proto.Value_Obsolete: + return " #OBSOLETE" + default: + // Flagged as error elsewhere, so return empty string here. + return "" + } +} diff --git a/cmd/release_config/release_config_lib/flag_value_test.go b/cmd/release_config/release_config_lib/flag_value_test.go new file mode 100644 index 000000000..aaa4cafeb --- /dev/null +++ b/cmd/release_config/release_config_lib/flag_value_test.go @@ -0,0 +1,67 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "os" + "path/filepath" + "testing" + + rc_proto "android/soong/cmd/release_config/release_config_proto" + + "google.golang.org/protobuf/proto" +) + +type testCaseFlagValue struct { + protoPath string + name string + data []byte + expected rc_proto.FlagValue + err error +} + +func (tc testCaseFlagValue) assertProtoEqual(t *testing.T, expected, actual proto.Message) { + if !proto.Equal(expected, actual) { + t.Errorf("Expected %q found %q", expected, actual) + } +} + +func TestFlagValue(t *testing.T) { + testCases := []testCaseFlagValue{ + { + name: "stringVal", + protoPath: "build/release/flag_values/test/RELEASE_FOO.textproto", + data: []byte(`name: "RELEASE_FOO" value {string_value: "BAR"}`), + expected: rc_proto.FlagValue{ + Name: proto.String("RELEASE_FOO"), + Value: &rc_proto.Value{Val: &rc_proto.Value_StringValue{"BAR"}}, + }, + err: nil, + }, + } + for _, tc := range testCases { + var err error + tempdir := t.TempDir() + path := filepath.Join(tempdir, tc.protoPath) + if err = os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(path, tc.data, 0644); err != nil { + t.Fatal(err) + } + actual := FlagValueFactory(path) + tc.assertProtoEqual(t, &tc.expected, &actual.proto) + } +} diff --git a/cmd/release_config/release_config_lib/release_config.go b/cmd/release_config/release_config_lib/release_config.go new file mode 100644 index 000000000..b08b6a339 --- /dev/null +++ b/cmd/release_config/release_config_lib/release_config.go @@ -0,0 +1,206 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "fmt" + "strings" + + rc_proto "android/soong/cmd/release_config/release_config_proto" + + "google.golang.org/protobuf/proto" +) + +// One directory's contribution to the a release config. +type ReleaseConfigContribution struct { + // Paths to files providing this config. + path string + + // The index of the config directory where this release config + // contribution was declared. + // Flag values cannot be set in a location with a lower index. + DeclarationIndex int + + // Protobufs relevant to the config. + proto rc_proto.ReleaseConfig + + FlagValues []*FlagValue +} + +// A generated release config. +type ReleaseConfig struct { + // the Name of the release config + Name string + + // The index of the config directory where this release config was + // first declared. + // Flag values cannot be set in a location with a lower index. + DeclarationIndex int + + // What contributes to this config. + Contributions []*ReleaseConfigContribution + + // Aliases for this release + OtherNames []string + + // The names of release configs that we inherit + InheritNames []string + + // Unmarshalled flag artifacts + FlagArtifacts FlagArtifacts + + // Generated release config + ReleaseConfigArtifact *rc_proto.ReleaseConfigArtifact + + // We have begun compiling this release config. + compileInProgress bool +} + +func ReleaseConfigFactory(name string, index int) (c *ReleaseConfig) { + return &ReleaseConfig{Name: name, DeclarationIndex: index} +} + +func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) error { + if config.ReleaseConfigArtifact != nil { + return nil + } + if config.compileInProgress { + return fmt.Errorf("Loop detected for release config %s", config.Name) + } + config.compileInProgress = true + isRoot := config.Name == "root" + + // Generate any configs we need to inherit. This will detect loops in + // the config. + contributionsToApply := []*ReleaseConfigContribution{} + myInherits := []string{} + myInheritsSet := make(map[string]bool) + // If there is a "root" release config, it is the start of every inheritance chain. + _, err := configs.GetReleaseConfig("root") + if err == nil && !isRoot { + config.InheritNames = append([]string{"root"}, config.InheritNames...) + } + for _, inherit := range config.InheritNames { + if _, ok := myInheritsSet[inherit]; ok { + continue + } + myInherits = append(myInherits, inherit) + myInheritsSet[inherit] = true + iConfig, err := configs.GetReleaseConfig(inherit) + if err != nil { + return err + } + iConfig.GenerateReleaseConfig(configs) + contributionsToApply = append(contributionsToApply, iConfig.Contributions...) + } + contributionsToApply = append(contributionsToApply, config.Contributions...) + + myAconfigValueSets := []string{} + myAconfigValueSetsMap := map[string]bool{} + myFlags := configs.FlagArtifacts.Clone() + workflowManual := rc_proto.Workflow(rc_proto.Workflow_MANUAL) + container := rc_proto.Container(rc_proto.Container_ALL) + releaseAconfigValueSets := FlagArtifact{ + FlagDeclaration: &rc_proto.FlagDeclaration{ + Name: proto.String("RELEASE_ACONFIG_VALUE_SETS"), + Namespace: proto.String("android_UNKNOWN"), + Description: proto.String("Aconfig value sets assembled by release-config"), + Workflow: &workflowManual, + Container: &container, + Value: &rc_proto.Value{Val: &rc_proto.Value_StringValue{""}}, + }, + DeclarationIndex: -1, + Traces: []*rc_proto.Tracepoint{ + &rc_proto.Tracepoint{ + Source: proto.String("$release-config"), + Value: &rc_proto.Value{Val: &rc_proto.Value_StringValue{""}}, + }, + }, + } + myFlags["RELEASE_ACONFIG_VALUE_SETS"] = &releaseAconfigValueSets + myDirsMap := make(map[int]bool) + for _, contrib := range contributionsToApply { + if len(contrib.proto.AconfigValueSets) > 0 { + contribAconfigValueSets := []string{} + for _, v := range contrib.proto.AconfigValueSets { + if _, ok := myAconfigValueSetsMap[v]; !ok { + contribAconfigValueSets = append(contribAconfigValueSets, v) + myAconfigValueSetsMap[v] = true + } + } + myAconfigValueSets = append(myAconfigValueSets, contribAconfigValueSets...) + releaseAconfigValueSets.Traces = append( + releaseAconfigValueSets.Traces, + &rc_proto.Tracepoint{ + Source: proto.String(contrib.path), + Value: &rc_proto.Value{Val: &rc_proto.Value_StringValue{strings.Join(contribAconfigValueSets, " ")}}, + }) + } + myDirsMap[contrib.DeclarationIndex] = true + for _, value := range contrib.FlagValues { + name := *value.proto.Name + fa, ok := myFlags[name] + if !ok { + return fmt.Errorf("Setting value for undefined flag %s in %s\n", name, value.path) + } + myDirsMap[fa.DeclarationIndex] = true + if fa.DeclarationIndex > contrib.DeclarationIndex { + // Setting location is to the left of declaration. + return fmt.Errorf("Setting value for flag %s not allowed in %s\n", name, value.path) + } + if isRoot && *fa.FlagDeclaration.Workflow != workflowManual { + // The "root" release config can only contain workflow: MANUAL flags. + return fmt.Errorf("Setting value for non-MANUAL flag %s is not allowed in %s", name, value.path) + } + if err := fa.UpdateValue(*value); err != nil { + return err + } + if fa.Redacted { + delete(myFlags, name) + } + } + } + releaseAconfigValueSets.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{strings.Join(myAconfigValueSets, " ")}} + + directories := []string{} + for idx, confDir := range configs.configDirs { + if _, ok := myDirsMap[idx]; ok { + directories = append(directories, confDir) + } + } + + config.FlagArtifacts = myFlags + config.ReleaseConfigArtifact = &rc_proto.ReleaseConfigArtifact{ + Name: proto.String(config.Name), + OtherNames: config.OtherNames, + FlagArtifacts: func() []*rc_proto.FlagArtifact { + ret := []*rc_proto.FlagArtifact{} + for _, flag := range myFlags { + ret = append(ret, &rc_proto.FlagArtifact{ + FlagDeclaration: flag.FlagDeclaration, + Traces: flag.Traces, + Value: flag.Value, + }) + } + return ret + }(), + AconfigValueSets: myAconfigValueSets, + Inherits: myInherits, + Directories: directories, + } + + config.compileInProgress = false + return nil +} diff --git a/cmd/release_config/release_config_lib/release_configs.go b/cmd/release_config/release_config_lib/release_configs.go new file mode 100644 index 000000000..aba8cd2c2 --- /dev/null +++ b/cmd/release_config/release_config_lib/release_configs.go @@ -0,0 +1,389 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "cmp" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + + rc_proto "android/soong/cmd/release_config/release_config_proto" + + "google.golang.org/protobuf/proto" +) + +// A single release_config_map.textproto and its associated data. +// Used primarily for debugging. +type ReleaseConfigMap struct { + // The path to this release_config_map file. + path string + + // Data received + proto rc_proto.ReleaseConfigMap + + // Map of name:contribution for release config contributions. + ReleaseConfigContributions map[string]*ReleaseConfigContribution + + // Flags declared this directory's flag_declarations/*.textproto + FlagDeclarations []rc_proto.FlagDeclaration +} + +type ReleaseConfigDirMap map[string]int + +// The generated release configs. +type ReleaseConfigs struct { + // Ordered list of release config maps processed. + ReleaseConfigMaps []*ReleaseConfigMap + + // Aliases + Aliases map[string]*string + + // Dictionary of flag_name:FlagDeclaration, with no overrides applied. + FlagArtifacts FlagArtifacts + + // Generated release configs artifact + Artifact rc_proto.ReleaseConfigsArtifact + + // Dictionary of name:ReleaseConfig + // Use `GetReleaseConfigs(name)` to get a release config. + ReleaseConfigs map[string]*ReleaseConfig + + // Map of directory to *ReleaseConfigMap + releaseConfigMapsMap map[string]*ReleaseConfigMap + + // The list of config directories used. + configDirs []string + + // A map from the config directory to its order in the list of config + // directories. + configDirIndexes ReleaseConfigDirMap +} + +// Write the "all_release_configs" artifact. +// +// The file will be in "{outDir}/all_release_configs-{product}.{format}" +// +// Args: +// +// outDir string: directory path. Will be created if not present. +// product string: TARGET_PRODUCT for the release_configs. +// format string: one of "json", "pb", or "textproto" +// +// Returns: +// +// error: Any error encountered. +func (configs *ReleaseConfigs) WriteArtifact(outDir, product, format string) error { + return WriteMessage( + filepath.Join(outDir, fmt.Sprintf("all_release_configs-%s.%s", product, format)), + &configs.Artifact) +} + +func ReleaseConfigsFactory() (c *ReleaseConfigs) { + return &ReleaseConfigs{ + Aliases: make(map[string]*string), + FlagArtifacts: make(map[string]*FlagArtifact), + ReleaseConfigs: make(map[string]*ReleaseConfig), + releaseConfigMapsMap: make(map[string]*ReleaseConfigMap), + configDirs: []string{}, + configDirIndexes: make(ReleaseConfigDirMap), + } +} + +func ReleaseConfigMapFactory(protoPath string) (m *ReleaseConfigMap) { + m = &ReleaseConfigMap{ + path: protoPath, + ReleaseConfigContributions: make(map[string]*ReleaseConfigContribution), + } + if protoPath != "" { + LoadMessage(protoPath, &m.proto) + } + return m +} + +func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex int) error { + m := ReleaseConfigMapFactory(path) + if m.proto.DefaultContainer == nil { + return fmt.Errorf("Release config map %s lacks default_container", path) + } + dir := filepath.Dir(path) + // Record any aliases, checking for duplicates. + for _, alias := range m.proto.Aliases { + name := *alias.Name + oldTarget, ok := configs.Aliases[name] + if ok { + if *oldTarget != *alias.Target { + return fmt.Errorf("Conflicting alias declarations: %s vs %s", + *oldTarget, *alias.Target) + } + } + configs.Aliases[name] = alias.Target + } + var err error + err = WalkTextprotoFiles(dir, "flag_declarations", func(path string, d fs.DirEntry, err error) error { + flagDeclaration := FlagDeclarationFactory(path) + // Container must be specified. + if flagDeclaration.Container == nil { + flagDeclaration.Container = m.proto.DefaultContainer + } + // TODO: once we have namespaces initialized, we can throw an error here. + if flagDeclaration.Namespace == nil { + flagDeclaration.Namespace = proto.String("android_UNKNOWN") + } + // If the input didn't specify a value, create one (== UnspecifiedValue). + if flagDeclaration.Value == nil { + flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_UnspecifiedValue{false}} + } + m.FlagDeclarations = append(m.FlagDeclarations, *flagDeclaration) + name := *flagDeclaration.Name + if def, ok := configs.FlagArtifacts[name]; !ok { + configs.FlagArtifacts[name] = &FlagArtifact{FlagDeclaration: flagDeclaration, DeclarationIndex: ConfigDirIndex} + } else if !proto.Equal(def.FlagDeclaration, flagDeclaration) { + return fmt.Errorf("Duplicate definition of %s", *flagDeclaration.Name) + } + // Set the initial value in the flag artifact. + configs.FlagArtifacts[name].UpdateValue( + FlagValue{path: path, proto: rc_proto.FlagValue{ + Name: proto.String(name), Value: flagDeclaration.Value}}) + if configs.FlagArtifacts[name].Redacted { + return fmt.Errorf("%s may not be redacted by default.", *flagDeclaration.Name) + } + return nil + }) + if err != nil { + return err + } + + err = WalkTextprotoFiles(dir, "release_configs", func(path string, d fs.DirEntry, err error) error { + releaseConfigContribution := &ReleaseConfigContribution{path: path, DeclarationIndex: ConfigDirIndex} + LoadMessage(path, &releaseConfigContribution.proto) + name := *releaseConfigContribution.proto.Name + if fmt.Sprintf("%s.textproto", name) != filepath.Base(path) { + return fmt.Errorf("%s incorrectly declares release config %s", path, name) + } + if _, ok := configs.ReleaseConfigs[name]; !ok { + configs.ReleaseConfigs[name] = ReleaseConfigFactory(name, ConfigDirIndex) + } + config := configs.ReleaseConfigs[name] + config.InheritNames = append(config.InheritNames, releaseConfigContribution.proto.Inherits...) + + // Only walk flag_values/{RELEASE} for defined releases. + err2 := WalkTextprotoFiles(dir, filepath.Join("flag_values", name), func(path string, d fs.DirEntry, err error) error { + flagValue := FlagValueFactory(path) + if fmt.Sprintf("%s.textproto", *flagValue.proto.Name) != filepath.Base(path) { + return fmt.Errorf("%s incorrectly sets value for flag %s", path, *flagValue.proto.Name) + } + releaseConfigContribution.FlagValues = append(releaseConfigContribution.FlagValues, flagValue) + return nil + }) + if err2 != nil { + return err2 + } + m.ReleaseConfigContributions[name] = releaseConfigContribution + config.Contributions = append(config.Contributions, releaseConfigContribution) + return nil + }) + if err != nil { + return err + } + configs.ReleaseConfigMaps = append(configs.ReleaseConfigMaps, m) + configs.releaseConfigMapsMap[dir] = m + return nil +} + +func (configs *ReleaseConfigs) GetReleaseConfig(name string) (*ReleaseConfig, error) { + trace := []string{name} + for target, ok := configs.Aliases[name]; ok; target, ok = configs.Aliases[name] { + name = *target + trace = append(trace, name) + } + if config, ok := configs.ReleaseConfigs[name]; ok { + return config, nil + } + return nil, fmt.Errorf("Missing config %s. Trace=%v", name, trace) +} + +// Write the makefile for this targetRelease. +func (configs *ReleaseConfigs) WriteMakefile(outFile, targetRelease string) error { + makeVars := make(map[string]string) + var allReleaseNames []string + for _, v := range configs.ReleaseConfigs { + allReleaseNames = append(allReleaseNames, v.Name) + allReleaseNames = append(allReleaseNames, v.OtherNames...) + } + config, err := configs.GetReleaseConfig(targetRelease) + if err != nil { + return err + } + + myFlagArtifacts := config.FlagArtifacts.Clone() + // Sort the flags by name first. + names := []string{} + for k, _ := range myFlagArtifacts { + names = append(names, k) + } + slices.SortFunc(names, func(a, b string) int { + return cmp.Compare(a, b) + }) + partitions := make(map[string][]string) + + vNames := []string{} + addVar := func(name, suffix, value string) { + fullName := fmt.Sprintf("_ALL_RELEASE_FLAGS.%s.%s", name, suffix) + vNames = append(vNames, fullName) + makeVars[fullName] = value + } + + for _, name := range names { + flag := myFlagArtifacts[name] + decl := flag.FlagDeclaration + + // cName := strings.ToLower(rc_proto.Container_name[decl.GetContainer()]) + cName := strings.ToLower(decl.Container.String()) + if cName == strings.ToLower(rc_proto.Container_ALL.String()) { + partitions["product"] = append(partitions["product"], name) + partitions["system"] = append(partitions["system"], name) + partitions["system_ext"] = append(partitions["system_ext"], name) + partitions["vendor"] = append(partitions["vendor"], name) + } else { + partitions[cName] = append(partitions[cName], name) + } + value := MarshalValue(flag.Value) + makeVars[name] = value + addVar(name, "PARTITIONS", cName) + addVar(name, "DEFAULT", MarshalValue(decl.Value)) + addVar(name, "VALUE", value) + addVar(name, "DECLARED_IN", *flag.Traces[0].Source) + addVar(name, "SET_IN", *flag.Traces[len(flag.Traces)-1].Source) + addVar(name, "NAMESPACE", *decl.Namespace) + } + pNames := []string{} + for k, _ := range partitions { + pNames = append(pNames, k) + } + slices.SortFunc(pNames, func(a, b string) int { + return cmp.Compare(a, b) + }) + + // Now sort the make variables, and output them. + slices.SortFunc(vNames, func(a, b string) int { + return cmp.Compare(a, b) + }) + + // Write the flags as: + // _ALL_RELELASE_FLAGS + // _ALL_RELEASE_FLAGS.PARTITIONS.* + // all _ALL_RELEASE_FLAGS.*, sorted by name + // Final flag values, sorted by name. + data := fmt.Sprintf("# TARGET_RELEASE=%s\n", config.Name) + if targetRelease != config.Name { + data += fmt.Sprintf("# User specified TARGET_RELEASE=%s\n", targetRelease) + } + // The variable _all_release_configs will get deleted during processing, so do not mark it read-only. + data += fmt.Sprintf("_all_release_configs := %s\n", strings.Join(allReleaseNames, " ")) + data += fmt.Sprintf("_ALL_RELEASE_FLAGS :=$= %s\n", strings.Join(names, " ")) + for _, pName := range pNames { + data += fmt.Sprintf("_ALL_RELEASE_FLAGS.PARTITIONS.%s :=$= %s\n", pName, strings.Join(partitions[pName], " ")) + } + for _, vName := range vNames { + data += fmt.Sprintf("%s :=$= %s\n", vName, makeVars[vName]) + } + data += "\n\n# Values for all build flags\n" + for _, name := range names { + data += fmt.Sprintf("%s :=$= %s\n", name, makeVars[name]) + } + return os.WriteFile(outFile, []byte(data), 0644) +} + +func (configs *ReleaseConfigs) GenerateReleaseConfigs(targetRelease string) error { + otherNames := make(map[string][]string) + for aliasName, aliasTarget := range configs.Aliases { + if _, ok := configs.ReleaseConfigs[aliasName]; ok { + return fmt.Errorf("Alias %s is a declared release config", aliasName) + } + if _, ok := configs.ReleaseConfigs[*aliasTarget]; !ok { + if _, ok2 := configs.Aliases[*aliasTarget]; !ok2 { + return fmt.Errorf("Alias %s points to non-existing config %s", aliasName, *aliasTarget) + } + } + otherNames[*aliasTarget] = append(otherNames[*aliasTarget], aliasName) + } + for name, aliases := range otherNames { + configs.ReleaseConfigs[name].OtherNames = aliases + } + + for _, config := range configs.ReleaseConfigs { + err := config.GenerateReleaseConfig(configs) + if err != nil { + return err + } + } + + releaseConfig, err := configs.GetReleaseConfig(targetRelease) + if err != nil { + return err + } + configs.Artifact = rc_proto.ReleaseConfigsArtifact{ + ReleaseConfig: releaseConfig.ReleaseConfigArtifact, + OtherReleaseConfigs: func() []*rc_proto.ReleaseConfigArtifact { + orc := []*rc_proto.ReleaseConfigArtifact{} + for name, config := range configs.ReleaseConfigs { + if name != releaseConfig.Name { + orc = append(orc, config.ReleaseConfigArtifact) + } + } + return orc + }(), + ReleaseConfigMapsMap: func() map[string]*rc_proto.ReleaseConfigMap { + ret := make(map[string]*rc_proto.ReleaseConfigMap) + for k, v := range configs.releaseConfigMapsMap { + ret[k] = &v.proto + } + return ret + }(), + } + return nil +} + +func ReadReleaseConfigMaps(releaseConfigMapPaths StringList, targetRelease string) (*ReleaseConfigs, error) { + var err error + + if len(releaseConfigMapPaths) == 0 { + releaseConfigMapPaths = GetDefaultMapPaths() + if len(releaseConfigMapPaths) == 0 { + return nil, fmt.Errorf("No maps found") + } + fmt.Printf("No --map argument provided. Using: --map %s\n", strings.Join(releaseConfigMapPaths, " --map ")) + } + + configs := ReleaseConfigsFactory() + for idx, releaseConfigMapPath := range releaseConfigMapPaths { + // Maintain an ordered list of release config directories. + configDir := filepath.Dir(releaseConfigMapPath) + configs.configDirIndexes[configDir] = idx + configs.configDirs = append(configs.configDirs, configDir) + err = configs.LoadReleaseConfigMap(releaseConfigMapPath, idx) + if err != nil { + return nil, err + } + } + + // Now that we have all of the release config maps, can meld them and generate the artifacts. + err = configs.GenerateReleaseConfigs(targetRelease) + return configs, err +} diff --git a/cmd/release_config/release_config_lib/util.go b/cmd/release_config/release_config_lib/util.go new file mode 100644 index 000000000..86940da68 --- /dev/null +++ b/cmd/release_config/release_config_lib/util.go @@ -0,0 +1,158 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/proto" +) + +var disableWarnings bool + +type StringList []string + +func (l *StringList) Set(v string) error { + *l = append(*l, v) + return nil +} + +func (l *StringList) String() string { + return fmt.Sprintf("%v", *l) +} + +// Write a marshalled message to a file. +// +// Marshal the message based on the extension of the path we are writing it to. +// +// Args: +// +// path string: the path of the file to write to. Directories are not created. +// Supported extensions are: ".json", ".pb", and ".textproto". +// message proto.Message: the message to write. +// +// Returns: +// +// error: any error encountered. +func WriteMessage(path string, message proto.Message) (err error) { + var data []byte + switch filepath.Ext(path) { + case ".json": + data, err = json.MarshalIndent(message, "", " ") + case ".pb": + data, err = proto.Marshal(message) + case ".textproto": + data, err = prototext.MarshalOptions{Multiline: true}.Marshal(message) + default: + return fmt.Errorf("Unknown message format for %s", path) + } + if err != nil { + return err + } + return os.WriteFile(path, data, 0644) +} + +// Read a message from a file. +// +// The message is unmarshalled based on the extension of the file read. +// +// Args: +// +// path string: the path of the file to read. +// message proto.Message: the message to unmarshal the message into. +// +// Returns: +// +// error: any error encountered. +func LoadMessage(path string, message proto.Message) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + switch filepath.Ext(path) { + case ".json": + return json.Unmarshal(data, message) + case ".pb": + return proto.Unmarshal(data, message) + case ".textproto": + return prototext.Unmarshal(data, message) + } + return fmt.Errorf("Unknown message format for %s", path) +} + +// Call Func for any textproto files found in {root}/{subdir}. +func WalkTextprotoFiles(root string, subdir string, Func fs.WalkDirFunc) error { + path := filepath.Join(root, subdir) + if _, err := os.Stat(path); err != nil { + // Missing subdirs are not an error. + return nil + } + return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if strings.HasSuffix(d.Name(), ".textproto") && d.Type().IsRegular() { + return Func(path, d, err) + } + return nil + }) +} + +// Turn off all warning output +func DisableWarnings() { + disableWarnings = true +} + +func warnf(format string, args ...any) (n int, err error) { + if !disableWarnings { + return fmt.Printf(format, args...) + } + return 0, nil +} + +// Returns the default value for release config artifacts. +func GetDefaultOutDir() string { + outEnv := os.Getenv("OUT_DIR") + if outEnv == "" { + outEnv = "out" + } + return filepath.Join(outEnv, "soong", "release-config") +} + +// Return the default list of map files to use. +func GetDefaultMapPaths() StringList { + var defaultMapPaths StringList + defaultLocations := StringList{ + "build/release/release_config_map.textproto", + "vendor/google_shared/build/release/release_config_map.textproto", + "vendor/google/release/release_config_map.textproto", + } + for _, path := range defaultLocations { + if _, err := os.Stat(path); err == nil { + defaultMapPaths = append(defaultMapPaths, path) + } + } + prodMaps := os.Getenv("PRODUCT_RELEASE_CONFIG_MAPS") + if prodMaps != "" { + defaultMapPaths = append(defaultMapPaths, strings.Split(prodMaps, " ")...) + } + return defaultMapPaths +} diff --git a/cmd/release_config/release_config_proto/Android.bp b/cmd/release_config/release_config_proto/Android.bp index a8660c753..8c47f2ac0 100644 --- a/cmd/release_config/release_config_proto/Android.bp +++ b/cmd/release_config/release_config_proto/Android.bp @@ -17,8 +17,8 @@ package { } bootstrap_go_package { - name: "soong-release_config_proto", - pkgPath: "android/soong/release_config/release_config_proto", + name: "soong-cmd-release_config-proto", + pkgPath: "android/soong/cmd/release_config/release_config_proto", deps: [ "golang-protobuf-reflect-protoreflect", "golang-protobuf-runtime-protoimpl", diff --git a/cmd/release_config/release_config_proto/build_flags_out.pb.go b/cmd/release_config/release_config_proto/build_flags_out.pb.go index adc1ea4bd..77e20698e 100644 --- a/cmd/release_config/release_config_proto/build_flags_out.pb.go +++ b/cmd/release_config/release_config_proto/build_flags_out.pb.go @@ -11,7 +11,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.30.0 // protoc v3.21.12 // source: build_flags_out.proto @@ -259,6 +259,8 @@ type ReleaseConfigsArtifact struct { ReleaseConfig *ReleaseConfigArtifact `protobuf:"bytes,1,opt,name=release_config,json=releaseConfig" json:"release_config,omitempty"` // All other release configs defined for this TARGET_PRODUCT. OtherReleaseConfigs []*ReleaseConfigArtifact `protobuf:"bytes,2,rep,name=other_release_configs,json=otherReleaseConfigs" json:"other_release_configs,omitempty"` + // Map of release_config_artifact.directories to release_config_map message. + ReleaseConfigMapsMap map[string]*ReleaseConfigMap `protobuf:"bytes,3,rep,name=release_config_maps_map,json=releaseConfigMapsMap" json:"release_config_maps_map,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` } func (x *ReleaseConfigsArtifact) Reset() { @@ -307,6 +309,13 @@ func (x *ReleaseConfigsArtifact) GetOtherReleaseConfigs() []*ReleaseConfigArtifa return nil } +func (x *ReleaseConfigsArtifact) GetReleaseConfigMapsMap() map[string]*ReleaseConfigMap { + if x != nil { + return x.ReleaseConfigMapsMap + } + return nil +} + var File_build_flags_out_proto protoreflect.FileDescriptor var file_build_flags_out_proto_rawDesc = []byte{ @@ -352,7 +361,7 @@ var file_build_flags_out_proto_rawDesc = []byte{ 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x64, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x22, 0xe3, 0x01, 0x0a, 0x18, 0x72, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x22, 0xe8, 0x03, 0x0a, 0x18, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x5c, 0x0a, 0x0e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, @@ -367,10 +376,26 @@ var file_build_flags_out_proto_rawDesc = []byte{ 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x13, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, - 0x42, 0x33, 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, - 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x87, 0x01, 0x0a, 0x17, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, 0x70, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x73, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x73, 0x4d, 0x61, 0x70, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x73, 0x4d, 0x61, 0x70, 0x1a, 0x79, 0x0a, 0x19, 0x52, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x73, 0x4d, + 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x46, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, + 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x33, 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, + 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -385,28 +410,32 @@ func file_build_flags_out_proto_rawDescGZIP() []byte { return file_build_flags_out_proto_rawDescData } -var file_build_flags_out_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_build_flags_out_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_build_flags_out_proto_goTypes = []interface{}{ (*Tracepoint)(nil), // 0: android.release_config_proto.tracepoint (*FlagArtifact)(nil), // 1: android.release_config_proto.flag_artifact (*ReleaseConfigArtifact)(nil), // 2: android.release_config_proto.release_config_artifact (*ReleaseConfigsArtifact)(nil), // 3: android.release_config_proto.release_configs_artifact - (*Value)(nil), // 4: android.release_config_proto.value - (*FlagDeclaration)(nil), // 5: android.release_config_proto.flag_declaration + nil, // 4: android.release_config_proto.release_configs_artifact.ReleaseConfigMapsMapEntry + (*Value)(nil), // 5: android.release_config_proto.value + (*FlagDeclaration)(nil), // 6: android.release_config_proto.flag_declaration + (*ReleaseConfigMap)(nil), // 7: android.release_config_proto.release_config_map } var file_build_flags_out_proto_depIdxs = []int32{ - 4, // 0: android.release_config_proto.tracepoint.value:type_name -> android.release_config_proto.value - 5, // 1: android.release_config_proto.flag_artifact.flag_declaration:type_name -> android.release_config_proto.flag_declaration - 4, // 2: android.release_config_proto.flag_artifact.value:type_name -> android.release_config_proto.value + 5, // 0: android.release_config_proto.tracepoint.value:type_name -> android.release_config_proto.value + 6, // 1: android.release_config_proto.flag_artifact.flag_declaration:type_name -> android.release_config_proto.flag_declaration + 5, // 2: android.release_config_proto.flag_artifact.value:type_name -> android.release_config_proto.value 0, // 3: android.release_config_proto.flag_artifact.traces:type_name -> android.release_config_proto.tracepoint 1, // 4: android.release_config_proto.release_config_artifact.flag_artifacts:type_name -> android.release_config_proto.flag_artifact 2, // 5: android.release_config_proto.release_configs_artifact.release_config:type_name -> android.release_config_proto.release_config_artifact 2, // 6: android.release_config_proto.release_configs_artifact.other_release_configs:type_name -> android.release_config_proto.release_config_artifact - 7, // [7:7] is the sub-list for method output_type - 7, // [7:7] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 4, // 7: android.release_config_proto.release_configs_artifact.release_config_maps_map:type_name -> android.release_config_proto.release_configs_artifact.ReleaseConfigMapsMapEntry + 7, // 8: android.release_config_proto.release_configs_artifact.ReleaseConfigMapsMapEntry.value:type_name -> android.release_config_proto.release_config_map + 9, // [9:9] is the sub-list for method output_type + 9, // [9:9] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_build_flags_out_proto_init() } @@ -471,7 +500,7 @@ func file_build_flags_out_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_build_flags_out_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/cmd/release_config/release_config_proto/build_flags_out.proto b/cmd/release_config/release_config_proto/build_flags_out.proto index fd8487bd4..05e770f3e 100644 --- a/cmd/release_config/release_config_proto/build_flags_out.proto +++ b/cmd/release_config/release_config_proto/build_flags_out.proto @@ -82,5 +82,8 @@ message release_configs_artifact { // All other release configs defined for this TARGET_PRODUCT. repeated release_config_artifact other_release_configs = 2; + + // Map of release_config_artifact.directories to release_config_map message. + map<string, release_config_map> release_config_maps_map = 3; } diff --git a/cmd/release_config/release_config_proto/build_flags_src.pb.go b/cmd/release_config/release_config_proto/build_flags_src.pb.go index 0f2c30b76..ca2005c4b 100644 --- a/cmd/release_config/release_config_proto/build_flags_src.pb.go +++ b/cmd/release_config/release_config_proto/build_flags_src.pb.go @@ -11,7 +11,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.30.0 // protoc v3.21.12 // source: build_flags_src.proto @@ -287,6 +287,9 @@ type FlagDeclaration struct { // The name of the flag. // See # name for format detail Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // Namespace the flag belongs to (required) + // See # namespace for format detail + Namespace *string `protobuf:"bytes,2,opt,name=namespace" json:"namespace,omitempty"` // Text description of the flag's purpose. Description *string `protobuf:"bytes,3,opt,name=description" json:"description,omitempty"` // Value for the flag @@ -296,9 +299,6 @@ type FlagDeclaration struct { // The container for this flag. This overrides any default container given // in the release_config_map message. Container *Container `protobuf:"varint,206,opt,name=container,enum=android.release_config_proto.Container" json:"container,omitempty"` - // Temporarily allow origin at the flag declaration level while we - // move flags to their own locations. - Origin *string `protobuf:"bytes,208,opt,name=origin" json:"origin,omitempty"` } func (x *FlagDeclaration) Reset() { @@ -340,6 +340,13 @@ func (x *FlagDeclaration) GetName() string { return "" } +func (x *FlagDeclaration) GetNamespace() string { + if x != nil && x.Namespace != nil { + return *x.Namespace + } + return "" +} + func (x *FlagDeclaration) GetDescription() string { if x != nil && x.Description != nil { return *x.Description @@ -368,13 +375,6 @@ func (x *FlagDeclaration) GetContainer() Container { return Container_UNSPECIFIED_container } -func (x *FlagDeclaration) GetOrigin() string { - if x != nil && x.Origin != nil { - return *x.Origin - } - return "" -} - type FlagValue struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -385,6 +385,9 @@ type FlagValue struct { Name *string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` // Value for the flag Value *Value `protobuf:"bytes,201,opt,name=value" json:"value,omitempty"` + // If true, the flag is completely removed from the release config as if + // never declared. + Redacted *bool `protobuf:"varint,202,opt,name=redacted" json:"redacted,omitempty"` } func (x *FlagValue) Reset() { @@ -433,6 +436,13 @@ func (x *FlagValue) GetValue() *Value { return nil } +func (x *FlagValue) GetRedacted() bool { + if x != nil && x.Redacted != nil { + return *x.Redacted + } + return false +} + // This replaces $(call declare-release-config). type ReleaseConfig struct { state protoimpl.MessageState @@ -568,8 +578,8 @@ type ReleaseConfigMap struct { // Any aliases. Aliases []*ReleaseAlias `protobuf:"bytes,1,rep,name=aliases" json:"aliases,omitempty"` - // The origin for flags declared here. - Origin *string `protobuf:"bytes,2,opt,name=origin" json:"origin,omitempty"` + // Description of this map and its intended use. + Description *string `protobuf:"bytes,2,opt,name=description" json:"description,omitempty"` // The default container for flags declared here. DefaultContainer *Container `protobuf:"varint,3,opt,name=default_container,json=defaultContainer,enum=android.release_config_proto.Container" json:"default_container,omitempty"` } @@ -613,9 +623,9 @@ func (x *ReleaseConfigMap) GetAliases() []*ReleaseAlias { return nil } -func (x *ReleaseConfigMap) GetOrigin() string { - if x != nil && x.Origin != nil { - return *x.Origin +func (x *ReleaseConfigMap) GetDescription() string { + if x != nil && x.Description != nil { + return *x.Description } return "" } @@ -643,71 +653,74 @@ var file_build_flags_src_proto_rawDesc = []byte{ 0x6c, 0x75, 0x65, 0x18, 0xca, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x08, 0x6f, 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x18, 0xcb, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x08, 0x6f, 0x62, - 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x05, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x22, 0xb8, 0x02, + 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x05, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x22, 0xbd, 0x02, 0x0a, 0x10, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x64, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, - 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x18, 0xcd, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, - 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, - 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x46, 0x0a, 0x09, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0xce, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, - 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x12, 0x17, 0x0a, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x18, 0xd0, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, - 0x4a, 0x06, 0x08, 0xcf, 0x01, 0x10, 0xd0, 0x01, 0x22, 0x5c, 0x0a, 0x0a, 0x66, 0x6c, 0x61, 0x67, - 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3a, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x6e, 0x64, - 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x6e, 0x0a, 0x0e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, - 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, - 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, - 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x53, 0x65, 0x74, 0x73, 0x22, 0x3b, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, - 0x65, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x22, 0xc9, 0x01, 0x0a, 0x12, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, 0x70, 0x12, 0x45, 0x0a, 0x07, 0x61, 0x6c, - 0x69, 0x61, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x61, 0x6e, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, + 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x18, 0xcd, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x08, 0x77, + 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x46, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x18, 0xce, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x07, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x12, 0x54, 0x0a, 0x11, 0x64, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, - 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x10, 0x64, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x2a, - 0x4a, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x18, 0x0a, 0x14, 0x55, - 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x55, 0x4e, 0x43, 0x48, 0x10, - 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x52, 0x45, 0x42, 0x55, 0x49, 0x4c, 0x54, 0x10, 0x02, 0x12, - 0x0a, 0x0a, 0x06, 0x4d, 0x41, 0x4e, 0x55, 0x41, 0x4c, 0x10, 0x03, 0x2a, 0x64, 0x0a, 0x09, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x15, 0x55, 0x4e, 0x53, 0x50, - 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, - 0x50, 0x52, 0x4f, 0x44, 0x55, 0x43, 0x54, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x59, 0x53, - 0x54, 0x45, 0x4d, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, - 0x45, 0x58, 0x54, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x56, 0x45, 0x4e, 0x44, 0x4f, 0x52, 0x10, - 0x05, 0x42, 0x33, 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, - 0x6e, 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4a, + 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x06, 0x08, 0xcf, 0x01, 0x10, 0xd0, 0x01, 0x22, 0x79, 0x0a, + 0x0a, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1b, 0x0a, 0x08, 0x72, + 0x65, 0x64, 0x61, 0x63, 0x74, 0x65, 0x64, 0x18, 0xca, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, + 0x72, 0x65, 0x64, 0x61, 0x63, 0x74, 0x65, 0x64, 0x22, 0x6e, 0x0a, 0x0e, 0x72, 0x65, 0x6c, 0x65, + 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, + 0x0a, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x53, 0x65, 0x74, 0x73, 0x22, 0x3b, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x65, + 0x61, 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xd3, 0x01, 0x0a, 0x12, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, 0x70, 0x12, 0x45, 0x0a, 0x07, + 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, + 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x07, 0x61, 0x6c, 0x69, 0x61, + 0x73, 0x65, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x54, 0x0a, 0x11, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x27, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x10, 0x64, 0x65, 0x66, 0x61, 0x75, + 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x2a, 0x4a, 0x0a, 0x08, 0x77, + 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x18, 0x0a, 0x14, 0x55, 0x4e, 0x53, 0x50, 0x45, + 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x10, + 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x55, 0x4e, 0x43, 0x48, 0x10, 0x01, 0x12, 0x0c, 0x0a, + 0x08, 0x50, 0x52, 0x45, 0x42, 0x55, 0x49, 0x4c, 0x54, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x4d, + 0x41, 0x4e, 0x55, 0x41, 0x4c, 0x10, 0x03, 0x2a, 0x64, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x15, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x10, 0x00, 0x12, + 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x4f, 0x44, + 0x55, 0x43, 0x54, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10, + 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, 0x45, 0x58, 0x54, 0x10, + 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x56, 0x45, 0x4e, 0x44, 0x4f, 0x52, 0x10, 0x05, 0x42, 0x33, 0x5a, + 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x72, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x72, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, } var ( diff --git a/cmd/release_config/release_config_proto/build_flags_src.proto b/cmd/release_config/release_config_proto/build_flags_src.proto index 0662716d5..92edc2a66 100644 --- a/cmd/release_config/release_config_proto/build_flags_src.proto +++ b/cmd/release_config/release_config_proto/build_flags_src.proto @@ -26,6 +26,11 @@ option go_package = "android/soong/release_config/release_config_proto"; // RELEASE_MY_PACKAGE_FLAG is a valid name, while MY_PACKAGE_FLAG, and // RELEASE_MY_PACKAGE__FLAG are invalid. // +// # namespace: namespace the flag belongs to +// +// format: a lowercase string in snake_case format, no consecutive underscores, and no leading +// digit. For example android_bar_system +// // # package: package to which the flag belongs // // format: lowercase strings in snake_case format, delimited by dots, no @@ -77,6 +82,10 @@ message flag_declaration { // See # name for format detail optional string name = 1; + // Namespace the flag belongs to (required) + // See # namespace for format detail + optional string namespace = 2; + // Text description of the flag's purpose. optional string description = 3; @@ -96,12 +105,6 @@ message flag_declaration { // The package associated with this flag. // (when Gantry is ready for it) optional string package = 207; reserved 207; - - // Temporarily allow origin at the flag declaration level while we - // move flags to their own locations. - optional string origin = 208; - - // TODO: do we want to include "first used in" (= ap2a)? } message flag_value { @@ -111,6 +114,10 @@ message flag_value { // Value for the flag optional value value = 201; + + // If true, the flag is completely removed from the release config as if + // never declared. + optional bool redacted = 202; } // This replaces $(call declare-release-config). @@ -141,8 +148,8 @@ message release_config_map { // Any aliases. repeated release_alias aliases = 1; - // The origin for flags declared here. - optional string origin = 2; + // Description of this map and its intended use. + optional string description = 2; // The default container for flags declared here. optional container default_container = 3; diff --git a/dexpreopt/class_loader_context.go b/dexpreopt/class_loader_context.go index 57c7ae851..af1d33da6 100644 --- a/dexpreopt/class_loader_context.go +++ b/dexpreopt/class_loader_context.go @@ -323,6 +323,7 @@ func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathCont } else if clc.Host == hostPath && clc.Device == devicePath { // Ok, the same library with the same paths. Don't re-add it, but don't raise an error // either, as the same library may be reachable via different transitional dependencies. + clc.Optional = clc.Optional && optional return nil } else { // Fail, as someone is trying to add the same library with different paths. This likely diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index 04bc61dad..93351f1fc 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -52,7 +52,7 @@ var DexpreoptRunningInSoong = false // GenerateDexpreoptRule generates a set of commands that will preopt a module based on a GlobalConfig and a // ModuleConfig. The produced files and their install locations will be available through rule.Installs(). func GenerateDexpreoptRule(ctx android.BuilderContext, globalSoong *GlobalSoongConfig, - global *GlobalConfig, module *ModuleConfig, productPackages android.Path) ( + global *GlobalConfig, module *ModuleConfig, productPackages android.Path, copyApexSystemServerJarDex bool) ( rule *android.RuleBuilder, err error) { defer func() { @@ -94,7 +94,7 @@ func GenerateDexpreoptRule(ctx android.BuilderContext, globalSoong *GlobalSoongC for archIdx, _ := range module.Archs { dexpreoptCommand(ctx, globalSoong, global, module, rule, archIdx, profile, appImage, - generateDM, productPackages) + generateDM, productPackages, copyApexSystemServerJarDex) } } } @@ -231,7 +231,7 @@ func ToOdexPath(path string, arch android.ArchType) string { func dexpreoptCommand(ctx android.BuilderContext, globalSoong *GlobalSoongConfig, global *GlobalConfig, module *ModuleConfig, rule *android.RuleBuilder, archIdx int, - profile android.WritablePath, appImage bool, generateDM bool, productPackages android.Path) { + profile android.WritablePath, appImage bool, generateDM bool, productPackages android.Path, copyApexSystemServerJarDex bool) { arch := module.Archs[archIdx] @@ -277,7 +277,7 @@ func dexpreoptCommand(ctx android.BuilderContext, globalSoong *GlobalSoongConfig clcTarget = append(clcTarget, GetSystemServerDexLocation(ctx, global, lib)) } - if DexpreoptRunningInSoong { + if DexpreoptRunningInSoong && copyApexSystemServerJarDex { // Copy the system server jar to a predefined location where dex2oat will find it. dexPathHost := SystemServerDexJarHostPath(ctx, module.Name) rule.Command().Text("mkdir -p").Flag(filepath.Dir(dexPathHost.String())) diff --git a/dexpreopt/dexpreopt_gen/dexpreopt_gen.go b/dexpreopt/dexpreopt_gen/dexpreopt_gen.go index 8033b48e5..75120052e 100644 --- a/dexpreopt/dexpreopt_gen/dexpreopt_gen.go +++ b/dexpreopt/dexpreopt_gen/dexpreopt_gen.go @@ -205,8 +205,9 @@ func writeScripts(ctx android.BuilderContext, globalSoong *dexpreopt.GlobalSoong panic(err) } } + cpApexSscpServerJar := false // dexpreopt_gen operates on make modules, and since sscp libraries are in soong, this should be a noop dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule( - ctx, globalSoong, global, module, android.PathForTesting(productPackagesPath)) + ctx, globalSoong, global, module, android.PathForTesting(productPackagesPath), cpApexSscpServerJar) if err != nil { panic(err) } diff --git a/dexpreopt/dexpreopt_test.go b/dexpreopt/dexpreopt_test.go index 7071f3e9e..eff2416e5 100644 --- a/dexpreopt/dexpreopt_test.go +++ b/dexpreopt/dexpreopt_test.go @@ -101,7 +101,7 @@ func TestDexPreopt(t *testing.T) { module := testSystemModuleConfig(ctx, "test") productPackages := android.PathForTesting("product_packages.txt") - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } @@ -161,7 +161,7 @@ func TestDexPreoptSystemOther(t *testing.T) { for _, test := range tests { global.PatternsOnSystemOther = test.patterns for _, mt := range test.moduleTests { - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, mt.module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, mt.module, productPackages, true) if err != nil { t.Fatal(err) } @@ -181,6 +181,11 @@ func TestDexPreoptSystemOther(t *testing.T) { } func TestDexPreoptApexSystemServerJars(t *testing.T) { + // modify the global variable for test + var oldDexpreoptRunningInSoong = DexpreoptRunningInSoong + DexpreoptRunningInSoong = true + + // test begin config := android.TestConfig("out", nil, "", nil) ctx := android.BuilderContextForTesting(config) globalSoong := globalSoongConfigForTests(ctx) @@ -191,7 +196,7 @@ func TestDexPreoptApexSystemServerJars(t *testing.T) { global.ApexSystemServerJars = android.CreateTestConfiguredJarList( []string{"com.android.apex1:service-A"}) - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } @@ -202,6 +207,18 @@ func TestDexPreoptApexSystemServerJars(t *testing.T) { } android.AssertStringEquals(t, "installs", wantInstalls.String(), rule.Installs().String()) + + android.AssertStringListContains(t, "apex sscp jar copy", rule.Outputs().Strings(), "out/soong/system_server_dexjars/service-A.jar") + + // rule with apex sscp cp as false + rule, err = GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, false) + if err != nil { + t.Fatal(err) + } + android.AssertStringListDoesNotContain(t, "apex sscp jar copy", rule.Outputs().Strings(), "out/soong/system_server_dexjars/service-A.jar") + + // cleanup the global variable for test + DexpreoptRunningInSoong = oldDexpreoptRunningInSoong } func TestDexPreoptStandaloneSystemServerJars(t *testing.T) { @@ -215,7 +232,7 @@ func TestDexPreoptStandaloneSystemServerJars(t *testing.T) { global.StandaloneSystemServerJars = android.CreateTestConfiguredJarList( []string{"platform:service-A"}) - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } @@ -239,7 +256,7 @@ func TestDexPreoptSystemExtSystemServerJars(t *testing.T) { global.StandaloneSystemServerJars = android.CreateTestConfiguredJarList( []string{"system_ext:service-A"}) - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } @@ -263,7 +280,7 @@ func TestDexPreoptApexStandaloneSystemServerJars(t *testing.T) { global.ApexStandaloneSystemServerJars = android.CreateTestConfiguredJarList( []string{"com.android.apex1:service-A"}) - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } @@ -286,7 +303,7 @@ func TestDexPreoptProfile(t *testing.T) { module.ProfileClassListing = android.OptionalPathForPath(android.PathForTesting("profile")) - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } diff --git a/filesystem/filesystem_test.go b/filesystem/filesystem_test.go index f4ecad46c..3a5071d34 100644 --- a/filesystem/filesystem_test.go +++ b/filesystem/filesystem_test.go @@ -84,12 +84,21 @@ func TestFileSystemDeps(t *testing.T) { cc_library { name: "libbar", required: ["libbaz"], + target: { + platform: { + required: ["lib_platform_only"], + }, + }, } cc_library { name: "libbaz", } + cc_library { + name: "lib_platform_only", + } + phony { name: "phony", required: [ @@ -120,6 +129,7 @@ func TestFileSystemDeps(t *testing.T) { "lib64/libbar.so", "lib64/libbaz.so", "lib64/libquz.so", + "lib64/lib_platform_only.so", "etc/bpf/bpf.o", } for _, e := range expected { diff --git a/java/aar.go b/java/aar.go index a36626732..0a91ff408 100644 --- a/java/aar.go +++ b/java/aar.go @@ -356,12 +356,13 @@ type aaptBuildActionOptions struct { forceNonFinalResourceIDs bool extraLinkFlags []string aconfigTextFiles android.Paths + usesLibrary *usesLibrary } func (a *aapt) buildActions(ctx android.ModuleContext, opts aaptBuildActionOptions) { staticResourcesNodesDepSet, sharedResourcesNodesDepSet, staticRRODirsDepSet, staticManifestsDepSet, sharedExportPackages, libFlags := - aaptLibs(ctx, opts.sdkContext, opts.classLoaderContexts) + aaptLibs(ctx, opts.sdkContext, opts.classLoaderContexts, opts.usesLibrary) // Exclude any libraries from the supplied list. opts.classLoaderContexts = opts.classLoaderContexts.ExcludeLibs(opts.excludedLibs) @@ -417,6 +418,9 @@ func (a *aapt) buildActions(ctx android.ModuleContext, opts aaptBuildActionOptio if a.isLibrary { linkFlags = append(linkFlags, "--static-lib") } + if opts.forceNonFinalResourceIDs { + linkFlags = append(linkFlags, "--non-final-ids") + } linkFlags = append(linkFlags, "--no-static-lib-packages") if a.isLibrary && a.useResourceProcessorBusyBox(ctx) { @@ -703,7 +707,8 @@ func (t transitiveAarDeps) assets() android.Paths { } // aaptLibs collects libraries from dependencies and sdk_version and converts them into paths -func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoaderContexts dexpreopt.ClassLoaderContextMap) ( +func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, + classLoaderContexts dexpreopt.ClassLoaderContextMap, usesLibrary *usesLibrary) ( staticResourcesNodes, sharedResourcesNodes *android.DepSet[*resourcesNode], staticRRODirs *android.DepSet[rroDir], staticManifests *android.DepSet[android.Path], sharedLibs android.Paths, flags []string) { @@ -753,6 +758,9 @@ func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoa } addCLCFromDep(ctx, module, classLoaderContexts) + if usesLibrary != nil { + addMissingOptionalUsesLibsFromDep(ctx, module, usesLibrary) + } }) // AAPT2 overlays are in lowest to highest priority order, the topological order will be reversed later. @@ -805,12 +813,12 @@ func (a *AndroidLibrary) OutputFiles(tag string) (android.Paths, error) { var _ AndroidLibraryDependency = (*AndroidLibrary)(nil) func (a *AndroidLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { + a.usesLibrary.deps(ctx, false) a.Module.deps(ctx) sdkDep := decodeSdkDep(ctx, android.SdkContext(a)) if sdkDep.hasFrameworkLibs() { a.aapt.deps(ctx, sdkDep) } - a.usesLibrary.deps(ctx, false) for _, aconfig_declaration := range a.aaptProperties.Flags_packages { ctx.AddDependency(ctx.Module(), aconfigDeclarationTag, aconfig_declaration) @@ -829,6 +837,7 @@ func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) classLoaderContexts: a.classLoaderContexts, enforceDefaultTargetSdkVersion: false, aconfigTextFiles: getAconfigFilePaths(ctx), + usesLibrary: &a.usesLibrary, }, ) @@ -1215,7 +1224,7 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { linkDeps = append(linkDeps, a.manifest) staticResourcesNodesDepSet, sharedResourcesNodesDepSet, staticRRODirsDepSet, staticManifestsDepSet, sharedLibs, libFlags := - aaptLibs(ctx, android.SdkContext(a), nil) + aaptLibs(ctx, android.SdkContext(a), nil, nil) _ = sharedResourcesNodesDepSet _ = staticRRODirsDepSet @@ -1287,6 +1296,7 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { } } addCLCFromDep(ctx, module, a.classLoaderContexts) + addMissingOptionalUsesLibsFromDep(ctx, module, &a.usesLibrary) }) var implementationJarFile android.OutputPath @@ -1405,6 +1415,12 @@ func (a *AARImport) ShouldSupportSdkVersion(ctx android.BaseModuleContext, var _ android.PrebuiltInterface = (*AARImport)(nil) +func (a *AARImport) UsesLibrary() *usesLibrary { + return &a.usesLibrary +} + +var _ ModuleWithUsesLibrary = (*AARImport)(nil) + // android_library_import imports an `.aar` file into the build graph as if it was built with android_library. // // This module is not suitable for installing on a device, but can be used as a `static_libs` dependency of diff --git a/java/app.go b/java/app.go index 1aa3afe8e..50d1a2f43 100644 --- a/java/app.go +++ b/java/app.go @@ -249,13 +249,13 @@ func (c Certificate) AndroidMkString() string { } func (a *AndroidApp) DepsMutator(ctx android.BottomUpMutatorContext) { - a.Module.deps(ctx) - if String(a.appProperties.Stl) == "c++_shared" && !a.SdkVersion(ctx).Specified() { ctx.PropertyErrorf("stl", "sdk_version must be set in order to use c++_shared") } sdkDep := decodeSdkDep(ctx, android.SdkContext(a)) + a.usesLibrary.deps(ctx, sdkDep.hasFrameworkLibs()) + a.Module.deps(ctx) if sdkDep.hasFrameworkLibs() { a.aapt.deps(ctx, sdkDep) } @@ -285,9 +285,6 @@ func (a *AndroidApp) DepsMutator(ctx android.BottomUpMutatorContext) { } ctx.AddFarVariationDependencies(variation, jniLibTag, a.appProperties.Jni_libs...) } - - a.usesLibrary.deps(ctx, sdkDep.hasFrameworkLibs()) - for _, aconfig_declaration := range a.aaptProperties.Flags_packages { ctx.AddDependency(ctx.Module(), aconfigDeclarationTag, aconfig_declaration) } @@ -534,6 +531,7 @@ func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { forceNonFinalResourceIDs: nonFinalIds, extraLinkFlags: aaptLinkFlags, aconfigTextFiles: getAconfigFilePaths(ctx), + usesLibrary: &a.usesLibrary, }, ) @@ -815,18 +813,10 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { // The decision to enforce <uses-library> checks is made before adding implicit SDK libraries. a.usesLibrary.freezeEnforceUsesLibraries() - // Add implicit SDK libraries to <uses-library> list. - requiredUsesLibs, optionalUsesLibs := a.classLoaderContexts.UsesLibs() - for _, usesLib := range requiredUsesLibs { - a.usesLibrary.addLib(usesLib, false) - } - for _, usesLib := range optionalUsesLibs { - a.usesLibrary.addLib(usesLib, true) - } - // Check that the <uses-library> list is coherent with the manifest. if a.usesLibrary.enforceUsesLibraries() { - manifestCheckFile := a.usesLibrary.verifyUsesLibrariesManifest(ctx, a.mergedManifestFile) + manifestCheckFile := a.usesLibrary.verifyUsesLibrariesManifest( + ctx, a.mergedManifestFile, &a.classLoaderContexts) apkDeps = append(apkDeps, manifestCheckFile) } @@ -1596,6 +1586,9 @@ type UsesLibraryProperties struct { // provide the android.test.base statically and use jarjar to rename them so they do not collide // with the classes provided by the android.test.base library. Exclude_uses_libs []string + + // The module names of optional uses-library libraries that are missing from the source tree. + Missing_optional_uses_libs []string `blueprint:"mutated"` } // usesLibrary provides properties and helper functions for AndroidApp and AndroidAppImport to verify that the @@ -1612,20 +1605,11 @@ type usesLibrary struct { shouldDisableDexpreopt bool } -func (u *usesLibrary) addLib(lib string, optional bool) { - if !android.InList(lib, u.usesLibraryProperties.Uses_libs) && !android.InList(lib, u.usesLibraryProperties.Optional_uses_libs) { - if optional { - u.usesLibraryProperties.Optional_uses_libs = append(u.usesLibraryProperties.Optional_uses_libs, lib) - } else { - u.usesLibraryProperties.Uses_libs = append(u.usesLibraryProperties.Uses_libs, lib) - } - } -} - func (u *usesLibrary) deps(ctx android.BottomUpMutatorContext, addCompatDeps bool) { if !ctx.Config().UnbundledBuild() || ctx.Config().UnbundledBuildImage() { ctx.AddVariationDependencies(nil, usesLibReqTag, u.usesLibraryProperties.Uses_libs...) - ctx.AddVariationDependencies(nil, usesLibOptTag, u.presentOptionalUsesLibs(ctx)...) + presentOptionalUsesLibs := u.presentOptionalUsesLibs(ctx) + ctx.AddVariationDependencies(nil, usesLibOptTag, presentOptionalUsesLibs...) // Only add these extra dependencies if the module is an app that depends on framework // libs. This avoids creating a cyclic dependency: // e.g. framework-res -> org.apache.http.legacy -> ... -> framework-res. @@ -1636,6 +1620,8 @@ func (u *usesLibrary) deps(ctx android.BottomUpMutatorContext, addCompatDeps boo ctx.AddVariationDependencies(nil, usesLibCompat28OptTag, dexpreopt.OptionalCompatUsesLibs28...) ctx.AddVariationDependencies(nil, usesLibCompat30OptTag, dexpreopt.OptionalCompatUsesLibs30...) } + _, diff, _ := android.ListSetDifference(u.usesLibraryProperties.Optional_uses_libs, presentOptionalUsesLibs) + u.usesLibraryProperties.Missing_optional_uses_libs = diff } else { ctx.AddVariationDependencies(nil, r8LibraryJarTag, u.usesLibraryProperties.Uses_libs...) ctx.AddVariationDependencies(nil, r8LibraryJarTag, u.presentOptionalUsesLibs(ctx)...) @@ -1654,15 +1640,6 @@ func (u *usesLibrary) presentOptionalUsesLibs(ctx android.BaseModuleContext) []s return optionalUsesLibs } -// Helper function to replace string in a list. -func replaceInList(list []string, oldstr, newstr string) { - for i, str := range list { - if str == oldstr { - list[i] = newstr - } - } -} - // Returns a map of module names of shared library dependencies to the paths to their dex jars on // host and on device. func (u *usesLibrary) classLoaderContextForUsesLibDeps(ctx android.ModuleContext) dexpreopt.ClassLoaderContextMap { @@ -1704,11 +1681,6 @@ func (u *usesLibrary) classLoaderContextForUsesLibDeps(ctx android.ModuleContext libName := dep if ulib, ok := m.(ProvidesUsesLib); ok && ulib.ProvidesUsesLib() != nil { libName = *ulib.ProvidesUsesLib() - // Replace module name with library name in `uses_libs`/`optional_uses_libs` in - // order to pass verify_uses_libraries check (which compares these properties - // against library names written in the manifest). - replaceInList(u.usesLibraryProperties.Uses_libs, dep, libName) - replaceInList(u.usesLibraryProperties.Optional_uses_libs, dep, libName) } clcMap.AddContext(ctx, tag.sdkVersion, libName, tag.optional, lib.DexJarBuildPath(ctx).PathOrNil(), lib.DexJarInstallPath(), @@ -1742,7 +1714,7 @@ func (u *usesLibrary) freezeEnforceUsesLibraries() { // an APK with the manifest embedded in it (manifest_check will know which one it is by the file // extension: APKs are supposed to end with '.apk'). func (u *usesLibrary) verifyUsesLibraries(ctx android.ModuleContext, inputFile android.Path, - outputFile android.WritablePath) android.Path { + outputFile android.WritablePath, classLoaderContexts *dexpreopt.ClassLoaderContextMap) android.Path { statusFile := dexpreopt.UsesLibrariesStatusFile(ctx) @@ -1770,27 +1742,37 @@ func (u *usesLibrary) verifyUsesLibraries(ctx android.ModuleContext, inputFile a cmd.Flag("--enforce-uses-libraries-relax") } - for _, lib := range u.usesLibraryProperties.Uses_libs { + requiredUsesLibs, optionalUsesLibs := classLoaderContexts.UsesLibs() + for _, lib := range requiredUsesLibs { cmd.FlagWithArg("--uses-library ", lib) } - - for _, lib := range u.usesLibraryProperties.Optional_uses_libs { + for _, lib := range optionalUsesLibs { cmd.FlagWithArg("--optional-uses-library ", lib) } + // Also add missing optional uses libs, as the manifest check expects them. + // Note that what we add here are the module names of those missing libs, not library names, while + // the manifest check actually expects library names. However, the case where a library is missing + // and the module name != the library name is too rare for us to handle. + for _, lib := range u.usesLibraryProperties.Missing_optional_uses_libs { + cmd.FlagWithArg("--missing-optional-uses-library ", lib) + } + rule.Build("verify_uses_libraries", "verify <uses-library>") return outputFile } // verifyUsesLibrariesManifest checks the <uses-library> tags in an AndroidManifest.xml against // the build system and returns the path to a copy of the manifest. -func (u *usesLibrary) verifyUsesLibrariesManifest(ctx android.ModuleContext, manifest android.Path) android.Path { +func (u *usesLibrary) verifyUsesLibrariesManifest(ctx android.ModuleContext, manifest android.Path, + classLoaderContexts *dexpreopt.ClassLoaderContextMap) android.Path { outputFile := android.PathForModuleOut(ctx, "manifest_check", "AndroidManifest.xml") - return u.verifyUsesLibraries(ctx, manifest, outputFile) + return u.verifyUsesLibraries(ctx, manifest, outputFile, classLoaderContexts) } // verifyUsesLibrariesAPK checks the <uses-library> tags in the manifest of an APK against the build // system and returns the path to a copy of the APK. -func (u *usesLibrary) verifyUsesLibrariesAPK(ctx android.ModuleContext, apk android.Path) { - u.verifyUsesLibraries(ctx, apk, nil) // for APKs manifest_check does not write output file +func (u *usesLibrary) verifyUsesLibrariesAPK(ctx android.ModuleContext, apk android.Path, + classLoaderContexts *dexpreopt.ClassLoaderContextMap) { + u.verifyUsesLibraries(ctx, apk, nil, classLoaderContexts) // for APKs manifest_check does not write output file } diff --git a/java/app_import.go b/java/app_import.go index 7387e168c..bb07c423a 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -355,7 +355,7 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext } if a.usesLibrary.enforceUsesLibraries() { - a.usesLibrary.verifyUsesLibrariesAPK(ctx, srcApk) + a.usesLibrary.verifyUsesLibrariesAPK(ctx, srcApk, &a.dexpreopter.classLoaderContexts) } a.dexpreopter.dexpreopt(ctx, android.RemoveOptionalPrebuiltPrefix(ctx.ModuleName()), jnisUncompressed) @@ -611,6 +611,12 @@ func createArchDpiVariantGroupType(archNames []string, dpiNames []string) reflec return return_struct } +func (a *AndroidAppImport) UsesLibrary() *usesLibrary { + return &a.usesLibrary +} + +var _ ModuleWithUsesLibrary = (*AndroidAppImport)(nil) + // android_app_import imports a prebuilt apk with additional processing specified in the module. // DPI-specific apk source files can be specified using dpi_variants. Example: // diff --git a/java/app_test.go b/java/app_test.go index 0c2800041..eab40e7da 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3244,7 +3244,10 @@ func TestUsesLibraries(t *testing.T) { name: "static-y", srcs: ["a.java"], uses_libs: ["runtime-required-y"], - optional_uses_libs: ["runtime-optional-y"], + optional_uses_libs: [ + "runtime-optional-y", + "missing-lib-a", + ], sdk_version: "current", } @@ -3280,7 +3283,7 @@ func TestUsesLibraries(t *testing.T) { sdk_version: "current", optional_uses_libs: [ "bar", - "baz", + "missing-lib-b", ], } @@ -3295,7 +3298,7 @@ func TestUsesLibraries(t *testing.T) { ], optional_uses_libs: [ "bar", - "baz", + "missing-lib-b", ], } ` @@ -3317,10 +3320,10 @@ func TestUsesLibraries(t *testing.T) { // propagated from dependencies. actualManifestFixerArgs := app.Output("manifest_fixer/AndroidManifest.xml").Args["args"] expectManifestFixerArgs := `--extract-native-libs=true ` + - `--uses-library qux ` + - `--uses-library quuz ` + `--uses-library foo ` + `--uses-library com.non.sdk.lib ` + + `--uses-library qux ` + + `--uses-library quuz ` + `--uses-library runtime-library ` + `--uses-library runtime-required-x ` + `--uses-library runtime-required-y ` + @@ -3339,9 +3342,10 @@ func TestUsesLibraries(t *testing.T) { `--uses-library runtime-required-x ` + `--uses-library runtime-required-y ` + `--optional-uses-library bar ` + - `--optional-uses-library baz ` + `--optional-uses-library runtime-optional-x ` + - `--optional-uses-library runtime-optional-y ` + `--optional-uses-library runtime-optional-y ` + + `--missing-optional-uses-library missing-lib-b ` + + `--missing-optional-uses-library missing-lib-a` android.AssertStringDoesContain(t, "verify cmd args", verifyCmd, verifyArgs) // Test that all libraries are verified for an APK (library order matters). @@ -3350,7 +3354,7 @@ func TestUsesLibraries(t *testing.T) { `--uses-library com.non.sdk.lib ` + `--uses-library android.test.runner ` + `--optional-uses-library bar ` + - `--optional-uses-library baz ` + `--missing-optional-uses-library missing-lib-b ` android.AssertStringDoesContain(t, "verify apk cmd args", verifyApkCmd, verifyApkArgs) // Test that necessary args are passed for constructing CLC in Ninja phase. diff --git a/java/base.go b/java/base.go index ef61f1cc2..938ac5e82 100644 --- a/java/base.go +++ b/java/base.go @@ -843,9 +843,11 @@ func (j *Module) deps(ctx android.BottomUpMutatorContext) { if dep != nil { if component, ok := dep.(SdkLibraryComponentDependency); ok { if lib := component.OptionalSdkLibraryImplementation(); lib != nil { - // Add library as optional if it's one of the optional compatibility libs. + // Add library as optional if it's one of the optional compatibility libs or it's + // explicitly listed in the optional_uses_libs property. tag := usesLibReqTag - if android.InList(*lib, dexpreopt.OptionalCompatUsesLibs) { + if android.InList(*lib, dexpreopt.OptionalCompatUsesLibs) || + android.InList(*lib, j.usesLibrary.usesLibraryProperties.Optional_uses_libs) { tag = usesLibOptTag } ctx.AddVariationDependencies(nil, tag, *lib) @@ -2387,6 +2389,7 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { } addCLCFromDep(ctx, module, j.classLoaderContexts) + addMissingOptionalUsesLibsFromDep(ctx, module, &j.usesLibrary) }) return deps @@ -2720,3 +2723,13 @@ type ModuleWithStem interface { } var _ ModuleWithStem = (*Module)(nil) + +type ModuleWithUsesLibrary interface { + UsesLibrary() *usesLibrary +} + +func (j *Module) UsesLibrary() *usesLibrary { + return &j.usesLibrary +} + +var _ ModuleWithUsesLibrary = (*Module)(nil) diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 38ed856ee..25e95db14 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -243,10 +243,6 @@ func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext, libName s return true } - if disableSourceApexVariant(ctx) { - return true - } - if _, isApex := android.ModuleProvider(ctx, android.ApexBundleInfoProvider); isApex { // dexpreopt rules for system server jars can be generated in the ModuleCtx of prebuilt apexes return false @@ -501,8 +497,12 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, libName string, dexJa Output(appProductPackages) productPackagesRule.Restat().Build("product_packages."+dexJarStem, "dexpreopt product_packages") + // Prebuilts are active, do not copy the dexpreopt'd source javalib to out/soong/system_server_dexjars + // The javalib from the deapexed prebuilt will be copied to this location. + // TODO (b/331665856): Implement a principled solution for this. + copyApexSystemServerJarDex := !disableSourceApexVariant(ctx) dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule( - ctx, globalSoong, global, dexpreoptConfig, appProductPackages) + ctx, globalSoong, global, dexpreoptConfig, appProductPackages, copyApexSystemServerJarDex) if err != nil { ctx.ModuleErrorf("error generating dexpreopt rule: %s", err.Error()) return diff --git a/java/droidstubs.go b/java/droidstubs.go index 02b81a4fe..ffd3caf99 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -532,8 +532,8 @@ func (d *Droidstubs) annotationsFlags(ctx android.ModuleContext, cmd *android.Ru cmd.Flag(config.MetalavaAnnotationsFlags) if params.migratingNullability { - previousApi := android.PathForModuleSrc(ctx, String(d.properties.Previous_api)) - cmd.FlagWithInput("--migrate-nullness ", previousApi) + previousApiFiles := android.PathsForModuleSrc(ctx, []string{String(d.properties.Previous_api)}) + cmd.FlagForEachInput("--migrate-nullness ", previousApiFiles) } if s := String(d.properties.Validate_nullability_from_list); s != "" { @@ -692,11 +692,11 @@ func (d *Droidstubs) apiCompatibilityFlags(ctx android.ModuleContext, cmd *andro ctx.PropertyErrorf("out", "out property may not be combined with check_api") } - apiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Api_file)) - removedApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Removed_api_file)) + apiFiles := android.PathsForModuleSrc(ctx, []string{String(d.properties.Check_api.Last_released.Api_file)}) + removedApiFiles := android.PathsForModuleSrc(ctx, []string{String(d.properties.Check_api.Last_released.Removed_api_file)}) - cmd.FlagWithInput("--check-compatibility:api:released ", apiFile) - cmd.FlagWithInput("--check-compatibility:removed:released ", removedApiFile) + cmd.FlagForEachInput("--check-compatibility:api:released ", apiFiles) + cmd.FlagForEachInput("--check-compatibility:removed:released ", removedApiFiles) baselineFile := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Last_released.Baseline_file) if baselineFile.Valid() { @@ -708,8 +708,8 @@ func metalavaUseRbe(ctx android.ModuleContext) bool { return ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_METALAVA") } -func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersion javaVersion, srcs android.Paths, - srcJarList android.Path, bootclasspath, classpath classpath, homeDir android.WritablePath) *android.RuleBuilderCommand { +func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, srcs android.Paths, + srcJarList android.Path, homeDir android.WritablePath, params stubsCommandConfigParams) *android.RuleBuilderCommand { rule.Command().Text("rm -rf").Flag(homeDir.String()) rule.Command().Text("mkdir -p").Flag(homeDir.String()) @@ -739,14 +739,14 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi cmd.BuiltTool("metalava").ImplicitTool(ctx.Config().HostJavaToolPath(ctx, "metalava.jar")). Flag(config.JavacVmFlags). Flag(config.MetalavaAddOpens). - FlagWithArg("--java-source ", javaVersion.String()). - FlagWithRspFileInputList("@", android.PathForModuleOut(ctx, "metalava.rsp"), srcs). + FlagWithArg("--java-source ", params.javaVersion.String()). + FlagWithRspFileInputList("@", android.PathForModuleOut(ctx, fmt.Sprintf("%s.metalava.rsp", params.stubsType.String())), srcs). FlagWithInput("@", srcJarList) // Metalava does not differentiate between bootclasspath and classpath and has not done so for // years, so it is unlikely to change any time soon. - combinedPaths := append(([]android.Path)(nil), bootclasspath.Paths()...) - combinedPaths = append(combinedPaths, classpath.Paths()...) + combinedPaths := append(([]android.Path)(nil), params.deps.bootClasspath.Paths()...) + combinedPaths = append(combinedPaths, params.deps.classpath.Paths()...) if len(combinedPaths) > 0 { cmd.FlagWithInputList("--classpath ", combinedPaths, ":") } @@ -827,8 +827,7 @@ func (d *Droidstubs) commonMetalavaStubCmd(ctx android.ModuleContext, rule *andr srcJarList := zipSyncCmd(ctx, rule, params.srcJarDir, d.Javadoc.srcJars) homeDir := android.PathForModuleOut(ctx, params.stubConfig.stubsType.String(), "home") - cmd := metalavaCmd(ctx, rule, params.stubConfig.javaVersion, d.Javadoc.srcFiles, srcJarList, - params.stubConfig.deps.bootClasspath, params.stubConfig.deps.classpath, homeDir) + cmd := metalavaCmd(ctx, rule, d.Javadoc.srcFiles, srcJarList, homeDir, params.stubConfig) cmd.Implicits(d.Javadoc.implicits) d.stubsFlags(ctx, cmd, params.stubsDir, params.stubConfig.stubsType, params.stubConfig.checkApi) @@ -950,12 +949,12 @@ func (d *Droidstubs) everythingOptionalCmd(ctx android.ModuleContext, cmd *andro // Add API lint options. if doApiLint { - newSince := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Api_lint.New_since) - if newSince.Valid() { - cmd.FlagWithInput("--api-lint ", newSince.Path()) - } else { - cmd.Flag("--api-lint") + var newSince android.Paths + if d.properties.Check_api.Api_lint.New_since != nil { + newSince = android.PathsForModuleSrc(ctx, []string{proptools.String(d.properties.Check_api.Api_lint.New_since)}) } + cmd.Flag("--api-lint") + cmd.FlagForEachInput("--api-lint-previous-api ", newSince) d.apiLintReport = android.PathForModuleOut(ctx, Everything.String(), "api_lint_report.txt") cmd.FlagWithOutput("--report-even-if-suppressed ", d.apiLintReport) // TODO: Change to ":api-lint" diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go index e4beb5e55..ae587eac3 100644 --- a/java/hiddenapi_modular.go +++ b/java/hiddenapi_modular.go @@ -1478,13 +1478,3 @@ func retrieveEncodedBootDexJarFromModule(ctx android.ModuleContext, module andro } return bootDexJar.Path() } - -// extractEncodedDexJarsFromModules extracts the encoded dex jars from the supplied modules. -func extractEncodedDexJarsFromModules(ctx android.ModuleContext, contents []android.Module) bootDexJarByModule { - encodedDexJarsByModuleName := bootDexJarByModule{} - for _, module := range contents { - path := retrieveEncodedBootDexJarFromModule(ctx, module) - encodedDexJarsByModuleName.addPath(module, path) - } - return encodedDexJarsByModuleName -} diff --git a/java/java.go b/java/java.go index ca99e6991..725e25abe 100644 --- a/java/java.go +++ b/java/java.go @@ -587,6 +587,7 @@ const ( JAVA_VERSION_9 = 9 JAVA_VERSION_11 = 11 JAVA_VERSION_17 = 17 + JAVA_VERSION_21 = 21 ) func (v javaVersion) String() string { @@ -605,6 +606,8 @@ func (v javaVersion) String() string { return "11" case JAVA_VERSION_17: return "17" + case JAVA_VERSION_21: + return "21" default: return "unsupported" } @@ -647,6 +650,8 @@ func normalizeJavaVersion(ctx android.BaseModuleContext, javaVersion string) jav return JAVA_VERSION_11 case "17": return JAVA_VERSION_17 + case "21": + return JAVA_VERSION_21 case "10", "12", "13", "14", "15", "16": ctx.PropertyErrorf("java_version", "Java language level %s is not supported", javaVersion) return JAVA_VERSION_UNSUPPORTED @@ -886,6 +891,12 @@ func init() { } func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if disableSourceApexVariant(ctx) { + // Prebuilts are active, do not create the installation rules for the source javalib. + // Even though the source javalib is not used, we need to hide it to prevent duplicate installation rules. + // TODO (b/331665856): Implement a principled solution for this. + j.HideFromMake() + } j.provideHiddenAPIPropertyInfo(ctx) j.sdkVersion = j.SdkVersion(ctx) @@ -966,8 +977,8 @@ func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { } func (j *Library) DepsMutator(ctx android.BottomUpMutatorContext) { - j.deps(ctx) j.usesLibrary.deps(ctx, false) + j.deps(ctx) } const ( @@ -3162,13 +3173,35 @@ func addCLCFromDep(ctx android.ModuleContext, depModule android.Module, // <uses_library> and should not be added to CLC, but the transitive <uses-library> dependencies // from its CLC should be added to the current CLC. if sdkLib != nil { - clcMap.AddContext(ctx, dexpreopt.AnySdkVersion, *sdkLib, false, + optional := false + if module, ok := ctx.Module().(ModuleWithUsesLibrary); ok { + if android.InList(*sdkLib, module.UsesLibrary().usesLibraryProperties.Optional_uses_libs) { + optional = true + } + } + clcMap.AddContext(ctx, dexpreopt.AnySdkVersion, *sdkLib, optional, dep.DexJarBuildPath(ctx).PathOrNil(), dep.DexJarInstallPath(), dep.ClassLoaderContexts()) } else { clcMap.AddContextMap(dep.ClassLoaderContexts(), depName) } } +func addMissingOptionalUsesLibsFromDep(ctx android.ModuleContext, depModule android.Module, + usesLibrary *usesLibrary) { + + dep, ok := depModule.(ModuleWithUsesLibrary) + if !ok { + return + } + + for _, lib := range dep.UsesLibrary().usesLibraryProperties.Missing_optional_uses_libs { + if !android.InList(lib, usesLibrary.usesLibraryProperties.Missing_optional_uses_libs) { + usesLibrary.usesLibraryProperties.Missing_optional_uses_libs = + append(usesLibrary.usesLibraryProperties.Missing_optional_uses_libs, lib) + } + } +} + type JavaApiContributionImport struct { JavaApiContribution diff --git a/java/prebuilt_apis.go b/java/prebuilt_apis.go index 6a79e5883..00613eee3 100644 --- a/java/prebuilt_apis.go +++ b/java/prebuilt_apis.go @@ -367,10 +367,23 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { info := latest[k] name := PrebuiltApiCombinedModuleName(info.module, info.scope, "latest") + // Iterate until the currentApiScope does not extend any other api scopes + // i.e. is not a superset of any other api scopes + // the relationship between the api scopes is defined in java/sdk_library.go var srcs []string currentApiScope := scopeByName[info.scope] - srcs = append(srcs, PrebuiltApiModuleName(info.module, currentApiScope.name, "latest")) + for currentApiScope != nil { + if _, ok := latest[fmt.Sprintf("%s.%s", info.module, currentApiScope.name)]; ok { + srcs = append(srcs, PrebuiltApiModuleName(info.module, currentApiScope.name, "latest")) + } + currentApiScope = currentApiScope.extends + } + // srcs is currently listed in the order from the widest api scope to the narrowest api scopes + // e.g. module lib -> system -> public + // In order to pass the files in metalava from the narrowest api scope to the widest api scope, + // the list has to be reversed. + android.ReverseSliceInPlace(srcs) createCombinedApiFilegroupModule(mctx, name, srcs) } } diff --git a/java/robolectric.go b/java/robolectric.go index 9e8850ce9..18386c90c 100644 --- a/java/robolectric.go +++ b/java/robolectric.go @@ -46,6 +46,7 @@ const robolectricPrebuiltLibPattern = "platform-robolectric-%s-prebuilt" var ( roboCoverageLibsTag = dependencyTag{name: "roboCoverageLibs"} roboRuntimesTag = dependencyTag{name: "roboRuntimes"} + roboRuntimeOnlyTag = dependencyTag{name: "roboRuntimeOnlyTag"} ) type robolectricProperties struct { @@ -70,6 +71,9 @@ type robolectricProperties struct { // Use /external/robolectric rather than /external/robolectric-shadows as the version of robolectric // to use. /external/robolectric closely tracks github's master, and will fully replace /external/robolectric-shadows Upstream *bool + + // Use strict mode to limit access of Robolectric API directly. See go/roboStrictMode + Strict_mode *bool } type robolectricTest struct { @@ -112,7 +116,7 @@ func (r *robolectricTest) DepsMutator(ctx android.BottomUpMutatorContext) { if v := String(r.robolectricProperties.Robolectric_prebuilt_version); v != "" { ctx.AddVariationDependencies(nil, libTag, fmt.Sprintf(robolectricPrebuiltLibPattern, v)) - } else { + } else if !proptools.Bool(r.robolectricProperties.Strict_mode) { if proptools.Bool(r.robolectricProperties.Upstream) { ctx.AddVariationDependencies(nil, libTag, robolectricCurrentLib+"_upstream") } else { @@ -120,6 +124,10 @@ func (r *robolectricTest) DepsMutator(ctx android.BottomUpMutatorContext) { } } + if proptools.Bool(r.robolectricProperties.Strict_mode) { + ctx.AddVariationDependencies(nil, roboRuntimeOnlyTag, robolectricCurrentLib+"_upstream") + } + ctx.AddVariationDependencies(nil, libTag, robolectricDefaultLibs...) ctx.AddVariationDependencies(nil, roboCoverageLibsTag, r.robolectricProperties.Coverage_libs...) @@ -192,19 +200,25 @@ func (r *robolectricTest) GenerateAndroidBuildActions(ctx android.ModuleContext) combinedJarJars = append(combinedJarJars, instrumentedApp.implementationAndResourcesJar) } - handleLibDeps := func(dep android.Module) { + handleLibDeps := func(dep android.Module, runtimeOnly bool) { m, _ := android.OtherModuleProvider(ctx, dep, JavaInfoProvider) - r.libs = append(r.libs, ctx.OtherModuleName(dep)) + if !runtimeOnly { + r.libs = append(r.libs, ctx.OtherModuleName(dep)) + } if !android.InList(ctx.OtherModuleName(dep), config.FrameworkLibraries) { combinedJarJars = append(combinedJarJars, m.ImplementationAndResourcesJars...) } } for _, dep := range ctx.GetDirectDepsWithTag(libTag) { - handleLibDeps(dep) + handleLibDeps(dep, false) } for _, dep := range ctx.GetDirectDepsWithTag(sdkLibTag) { - handleLibDeps(dep) + handleLibDeps(dep, false) + } + // handle the runtimeOnly tag for strict_mode + for _, dep := range ctx.GetDirectDepsWithTag(roboRuntimeOnlyTag) { + handleLibDeps(dep, true) } r.combinedJar = android.PathForModuleOut(ctx, "robolectric_combined", r.outputFile.Base()) diff --git a/java/sdk_library.go b/java/sdk_library.go index e7e53a2a8..113071fbb 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -705,10 +705,10 @@ type scopePaths struct { annotationsZip android.OptionalPath // The path to the latest API file. - latestApiPath android.OptionalPath + latestApiPaths android.Paths // The path to the latest removed API file. - latestRemovedApiPath android.OptionalPath + latestRemovedApiPaths android.Paths } func (paths *scopePaths) extractStubsLibraryInfoFromDependency(ctx android.ModuleContext, dep android.Module) error { @@ -829,28 +829,25 @@ func (paths *scopePaths) extractStubsSourceAndApiInfoFromApiStubsProvider(ctx an }) } -func extractSingleOptionalOutputPath(dep android.Module) (android.OptionalPath, error) { +func extractOutputPaths(dep android.Module) (android.Paths, error) { var paths android.Paths if sourceFileProducer, ok := dep.(android.SourceFileProducer); ok { paths = sourceFileProducer.Srcs() + return paths, nil } else { - return android.OptionalPath{}, fmt.Errorf("module %q does not produce source files", dep) + return nil, fmt.Errorf("module %q does not produce source files", dep) } - if len(paths) != 1 { - return android.OptionalPath{}, fmt.Errorf("expected one path from %q, got %q", dep, paths) - } - return android.OptionalPathForPath(paths[0]), nil } func (paths *scopePaths) extractLatestApiPath(ctx android.ModuleContext, dep android.Module) error { - outputPath, err := extractSingleOptionalOutputPath(dep) - paths.latestApiPath = outputPath + outputPaths, err := extractOutputPaths(dep) + paths.latestApiPaths = outputPaths return err } func (paths *scopePaths) extractLatestRemovedApiPath(ctx android.ModuleContext, dep android.Module) error { - outputPath, err := extractSingleOptionalOutputPath(dep) - paths.latestRemovedApiPath = outputPath + outputPaths, err := extractOutputPaths(dep) + paths.latestRemovedApiPaths = outputPaths return err } @@ -1562,6 +1559,12 @@ func (module *SdkLibrary) OutputFiles(tag string) (android.Paths, error) { } func (module *SdkLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if disableSourceApexVariant(ctx) { + // Prebuilts are active, do not create the installation rules for the source javalib. + // Even though the source javalib is not used, we need to hide it to prevent duplicate installation rules. + // TODO (b/331665856): Implement a principled solution for this. + module.HideFromMake() + } if proptools.String(module.deviceProperties.Min_sdk_version) != "" { module.CheckMinSdkVersion(ctx) } @@ -1619,11 +1622,15 @@ func (module *SdkLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) scopes[scope.name] = scopeInfo scopeInfo["current_api"] = scope.snapshotRelativeCurrentApiTxtPath(baseModuleName) scopeInfo["removed_api"] = scope.snapshotRelativeRemovedApiTxtPath(baseModuleName) - if p := scopePaths.latestApiPath; p.Valid() { - scopeInfo["latest_api"] = p.Path().String() + if p := scopePaths.latestApiPaths; len(p) > 0 { + // The last path in the list is the one that applies to this scope, the + // preceding ones, if any, are for the scope(s) that it extends. + scopeInfo["latest_api"] = p[len(p)-1].String() } - if p := scopePaths.latestRemovedApiPath; p.Valid() { - scopeInfo["latest_removed_api"] = p.Path().String() + if p := scopePaths.latestRemovedApiPaths; len(p) > 0 { + // The last path in the list is the one that applies to this scope, the + // preceding ones, if any, are for the scope(s) that it extends. + scopeInfo["latest_removed_api"] = p[len(p)-1].String() } } android.SetProvider(ctx, android.AdditionalSdkInfoProvider, android.AdditionalSdkInfo{additionalSdkInfo}) @@ -3309,6 +3316,7 @@ func (module *sdkLibraryXml) GenerateAndroidBuildActions(ctx android.ModuleConte android.WriteFileRuleVerbatim(ctx, module.outputFilePath, xmlContent) module.installDirPath = android.PathForModuleInstall(ctx, "etc", module.SubDir()) + ctx.PackageFile(module.installDirPath, libName+".xml", module.outputFilePath) } func (module *sdkLibraryXml) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index 5fac2556d..0f163e6e0 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -1085,18 +1085,6 @@ func TestJavaSdkLibraryImport_Preferred(t *testing.T) { t.Run("prefer", func(t *testing.T) { testJavaSdkLibraryImport_Preferred(t, "prefer: true,", android.NullFixturePreparer) }) - - t.Run("use_source_config_var", func(t *testing.T) { - testJavaSdkLibraryImport_Preferred(t, - "use_source_config_var: {config_namespace: \"acme\", var_name: \"use_source\"},", - android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "false", - }, - } - })) - }) } // If a module is listed in `mainline_module_contributions, it should be used diff --git a/java/testing.go b/java/testing.go index 631d51662..5ae326d93 100644 --- a/java/testing.go +++ b/java/testing.go @@ -711,7 +711,7 @@ var PrepareForTestWithFakeApexMutator = android.GroupFixturePreparers( func registerFakeApexMutator(ctx android.RegistrationContext) { ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("apex", fakeApexMutator).Parallel() + ctx.Transition("apex", &fakeApexMutator{}) }) } @@ -726,16 +726,30 @@ var _ apexModuleBase = (*SdkLibrary)(nil) // `apex_available`. It helps us avoid a dependency on the real mutator defined in "soong-apex", // which will cause a cyclic dependency, and it provides an easy way to create an APEX variant for // testing without dealing with all the complexities in the real mutator. -func fakeApexMutator(mctx android.BottomUpMutatorContext) { - switch mctx.Module().(type) { +type fakeApexMutator struct{} + +func (f *fakeApexMutator) Split(ctx android.BaseModuleContext) []string { + switch ctx.Module().(type) { case *Library, *SdkLibrary: - if len(mctx.Module().(apexModuleBase).ApexAvailable()) > 0 { - modules := mctx.CreateVariations("", "apex1000") - apexInfo := android.ApexInfo{ - ApexVariationName: "apex1000", - } - mctx.SetVariationProvider(modules[1], android.ApexInfoProvider, apexInfo) + return []string{"", "apex1000"} + } + return []string{""} +} + +func (f *fakeApexMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + return sourceVariation +} + +func (f *fakeApexMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + return incomingVariation +} + +func (f *fakeApexMutator) Mutate(ctx android.BottomUpMutatorContext, variation string) { + if variation != "" { + apexInfo := android.ApexInfo{ + ApexVariationName: "apex1000", } + android.SetProvider(ctx, android.ApexInfoProvider, apexInfo) } } diff --git a/partner/androidmk/androidmk_test.go b/partner/androidmk/androidmk_test.go index 6bae836d9..3ace7502d 100644 --- a/partner/androidmk/androidmk_test.go +++ b/partner/androidmk/androidmk_test.go @@ -54,6 +54,9 @@ cc_library_shared { } func TestEndToEnd(t *testing.T) { + // Skip checking Android.mk path with cleaning "ANDROID_BUILD_TOP" + t.Setenv("ANDROID_BUILD_TOP", "") + for i, test := range testCases { expected, err := bpfix.Reformat(test.expected) if err != nil { diff --git a/python/binary.go b/python/binary.go index d6750c655..c84eeeedb 100644 --- a/python/binary.go +++ b/python/binary.go @@ -203,7 +203,7 @@ func (p *PythonBinaryModule) OutputFiles(tag string) (android.Paths, error) { } func (p *PythonBinaryModule) isEmbeddedLauncherEnabled() bool { - return Bool(p.properties.Embedded_launcher) + return BoolDefault(p.properties.Embedded_launcher, true) } func (b *PythonBinaryModule) autorun() bool { diff --git a/python/python.go b/python/python.go index 2b1974eb8..e14fdf333 100644 --- a/python/python.go +++ b/python/python.go @@ -59,7 +59,7 @@ type VersionProperties struct { // list of the Python libraries used only for this Python version. Libs []string `android:"arch_variant"` - // whether the binary is required to be built with embedded launcher for this version, defaults to false. + // whether the binary is required to be built with embedded launcher for this version, defaults to true. Embedded_launcher *bool // TODO(b/174041232): Remove this property } @@ -151,6 +151,8 @@ type PythonLibraryModule struct { // The zip file containing the current module's source/data files, with the // source files precompiled. precompiledSrcsZip android.Path + + sourceProperties android.SourceProperties } // newModule generates new Python base module @@ -203,7 +205,7 @@ func (p *PythonLibraryModule) getBaseProperties() *BaseProperties { var _ pythonDependency = (*PythonLibraryModule)(nil) func (p *PythonLibraryModule) init() android.Module { - p.AddProperties(&p.properties, &p.protoProperties) + p.AddProperties(&p.properties, &p.protoProperties, &p.sourceProperties) android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitDefaultableModule(p) return p @@ -421,6 +423,11 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs) android.SetProvider(ctx, blueprint.SrcsFileProviderKey, blueprint.SrcsFileProviderData{SrcPaths: expandedSrcs.Strings()}) + // Keep before any early returns. + android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{ + TestOnly: Bool(p.sourceProperties.Test_only), + TopLevelTarget: p.sourceProperties.Top_level_test_target, + }) // expand data files from "data" property. expandedData := android.PathsForModuleSrc(ctx, p.properties.Data) diff --git a/python/python_test.go b/python/python_test.go index 75a6a899b..c0b7295f9 100644 --- a/python/python_test.go +++ b/python/python_test.go @@ -18,10 +18,13 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "android/soong/android" "android/soong/cc" + + "github.com/google/blueprint" ) type pyModule struct { @@ -360,6 +363,76 @@ cc_binary { } } +func TestTestOnlyProvider(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + PrepareForTestWithPythonBuildComponents, + android.PrepareForTestWithAllowMissingDependencies, + ).RunTestWithBp(t, ` + // These should be test-only + python_library { name: "py-lib-test", test_only: true } + python_library { name: "py-lib-test-host", test_only: true, host_supported: true } + python_test { name: "py-test", srcs: ["py-test.py"] } + python_test_host { name: "py-test-host", srcs: ["py-test-host.py"] } + python_binary_host { name: "py-bin-test", srcs: ["py-bin-test.py"] } + + // These should not be. + python_library { name: "py-lib" } + python_binary_host { name: "py-bin", srcs: ["py-bin.py"] } + `) + + // Visit all modules and ensure only the ones that should + // marked as test-only are marked as test-only. + + actualTestOnly := []string{} + ctx.VisitAllModules(func(m blueprint.Module) { + if provider, ok := android.OtherModuleProvider(ctx.TestContext.OtherModuleProviderAdaptor(), m, android.TestOnlyProviderKey); ok { + if provider.TestOnly { + actualTestOnly = append(actualTestOnly, m.Name()) + } + } + }) + expectedTestOnlyModules := []string{ + "py-lib-test", + "py-lib-test-host", + "py-test", + "py-test-host", + } + + notEqual, left, right := android.ListSetDifference(expectedTestOnlyModules, actualTestOnly) + if notEqual { + t.Errorf("test-only: Expected but not found: %v, Found but not expected: %v", left, right) + } +} + +// Don't allow setting test-only on things that are always tests or never tests. +func TestInvalidTestOnlyTargets(t *testing.T) { + testCases := []string{ + ` python_test { name: "py-test", test_only: true, srcs: ["py-test.py"] } `, + ` python_test_host { name: "py-test-host", test_only: true, srcs: ["py-test-host.py"] } `, + ` python_defaults { name: "py-defaults", test_only: true, srcs: ["foo.py"] } `, + } + + for i, bp := range testCases { + ctx := android.GroupFixturePreparers( + PrepareForTestWithPythonBuildComponents, + android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + + ctx.RegisterModuleType("python_defaults", DefaultsFactory) + }), + android.PrepareForTestWithAllowMissingDependencies). + ExtendWithErrorHandler(android.FixtureIgnoreErrors). + RunTestWithBp(t, bp) + if len(ctx.Errs) != 1 { + t.Errorf("Expected err setting test_only in testcase #%d: %d errs", i, len(ctx.Errs)) + continue + } + if !strings.Contains(ctx.Errs[0].Error(), "unrecognized property \"test_only\"") { + t.Errorf("ERR: %s bad bp: %s", ctx.Errs[0], bp) + } + } +} + func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expectedSrcsZip string, expectedPyRunfiles []string) { module := ctx.ModuleForTests(name, variant) diff --git a/python/test.go b/python/test.go index 826f35358..2b939e7e4 100644 --- a/python/test.go +++ b/python/test.go @@ -36,7 +36,9 @@ func registerPythonTestComponents(ctx android.RegistrationContext) { } func NewTest(hod android.HostOrDeviceSupported) *PythonTestModule { - return &PythonTestModule{PythonBinaryModule: *NewBinary(hod)} + p := &PythonTestModule{PythonBinaryModule: *NewBinary(hod)} + p.sourceProperties = android.SourceProperties{Test_only: proptools.BoolPtr(true), Top_level_test_target: true} + return p } func PythonTestHostFactory() android.Module { diff --git a/rust/bindgen.go b/rust/bindgen.go index 11ba74d45..eaed1b9d4 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -101,6 +101,9 @@ type BindgenProperties struct { // // "my_bindgen [flags] wrapper_header.h -o [output_path] -- [clang flags]" Custom_bindgen string + + // flag to indicate if bindgen should handle `static inline` functions (default is false) + Handle_static_inline bool } type bindgenDecorator struct { @@ -232,6 +235,9 @@ func (b *bindgenDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) andr bindgenFlags := defaultBindgenFlags bindgenFlags = append(bindgenFlags, esc(b.Properties.Bindgen_flags)...) + if b.Properties.Handle_static_inline { + bindgenFlags = append(bindgenFlags, "--experimental --wrap-static-fns") + } // cat reads from stdin if its command line is empty, // so we pass in /dev/null if there are no other flag files diff --git a/rust/bindgen_test.go b/rust/bindgen_test.go index 0c0a6dad0..11cfe4e88 100644 --- a/rust/bindgen_test.go +++ b/rust/bindgen_test.go @@ -227,3 +227,22 @@ func TestBindgenFlagFile(t *testing.T) { // TODO: The best we can do right now is check $flagfiles. Once bindgen.go switches to RuleBuilder, // we may be able to check libbinder.RuleParams.Command to see if it contains $(cat /dev/null flag_file.txt) } + + +func TestBindgenHandleStaticInlining(t *testing.T) { + ctx := testRust(t, ` + rust_bindgen { + name: "libbindgen", + wrapper_src: "src/any.h", + crate_name: "bindgen", + stem: "libbindgen", + source_stem: "bindings", + handle_static_inline: true + } + `) + libbindgen := ctx.ModuleForTests("libbindgen", "android_arm64_armv8-a_source").Output("bindings.rs") + // Make sure the flag to support `static inline` functions is present + if !strings.Contains(libbindgen.Args["flags"], "--wrap-static-fns") { + t.Errorf("missing flag to handle static inlining in rust_bindgen rule: flags %#v", libbindgen.Args["flags"]) + } +} diff --git a/rust/builder.go b/rust/builder.go index 2f5e12aa5..4f45e33c1 100644 --- a/rust/builder.go +++ b/rust/builder.go @@ -61,7 +61,7 @@ var ( // Use the metadata output as it has the smallest footprint. "--emit metadata -o $out --emit dep-info=$out.d.raw $in ${libFlags} " + "$rustcFlags $clippyFlags" + - " && grep \"^$out:\" $out.d.raw > $out.d", + " && grep ^$out: $out.d.raw > $out.d", CommandDeps: []string{"$clippyCmd"}, Deps: blueprint.DepsGCC, Depfile: "$out.d", diff --git a/rust/protobuf.go b/rust/protobuf.go index 0b26b80fa..fab5259a5 100644 --- a/rust/protobuf.go +++ b/rust/protobuf.go @@ -16,6 +16,7 @@ package rust import ( "fmt" + "strconv" "strings" "android/soong/android" @@ -122,41 +123,65 @@ func (proto *protobufDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) // stemFile must be first here as the first path in BaseSourceProvider.OutputFiles is the library entry-point. var outputs android.WritablePaths - rule := android.NewRuleBuilder(pctx, ctx) + for i, shard := range android.ShardPaths(protoFiles, 50) { + rule := android.NewRuleBuilder(pctx, ctx) - for _, protoFile := range protoFiles { - // Since we're iterating over the protoFiles already, make sure they're not redeclared in grpcFiles - if android.InList(protoFile.String(), grpcFiles.Strings()) { - ctx.PropertyErrorf("protos", - "A proto can only be added once to either grpc_protos or protos. %q is declared in both properties", - protoFile.String()) - } + for _, protoFile := range shard { + // Since we're iterating over the protoFiles already, make sure they're not redeclared in grpcFiles + if android.InList(protoFile.String(), grpcFiles.Strings()) { + ctx.PropertyErrorf("protos", + "A proto can only be added once to either grpc_protos or protos. %q is declared in both properties", + protoFile.String()) + } + + protoName := strings.TrimSuffix(protoFile.Base(), ".proto") + proto.protoNames = append(proto.protoNames, protoName) + + protoOut := android.PathForModuleOut(ctx, protoName+".rs") + depFile := android.PathForModuleOut(ctx, protoName+".d") - protoName := strings.TrimSuffix(protoFile.Base(), ".proto") - proto.protoNames = append(proto.protoNames, protoName) + ruleOutputs := android.WritablePaths{protoOut, depFile} - protoOut := android.PathForModuleOut(ctx, protoName+".rs") - depFile := android.PathForModuleOut(ctx, protoName+".d") + android.ProtoRule(rule, protoFile, protoFlags, protoFlags.Deps, outDir, depFile, ruleOutputs) + outputs = append(outputs, ruleOutputs...) + } - ruleOutputs := android.WritablePaths{protoOut, depFile} + ruleName := "protoc" + ruleDesc := "protoc" + if i > 0 { + ruleName += "_" + strconv.Itoa(i+1) + ruleDesc += " " + strconv.Itoa(i+1) + } - android.ProtoRule(rule, protoFile, protoFlags, protoFlags.Deps, outDir, depFile, ruleOutputs) - outputs = append(outputs, ruleOutputs...) + rule.Build(ruleName, ruleDesc) } - for _, grpcFile := range grpcFiles { - grpcName := strings.TrimSuffix(grpcFile.Base(), ".proto") - proto.grpcNames = append(proto.grpcNames, grpcName) + for i, shard := range android.ShardPaths(grpcFiles, 50) { + rule := android.NewRuleBuilder(pctx, ctx) + + for _, grpcFile := range shard { + grpcName := strings.TrimSuffix(grpcFile.Base(), ".proto") + proto.grpcNames = append(proto.grpcNames, grpcName) - // GRPC protos produce two files, a proto.rs and a proto_grpc.rs - protoOut := android.WritablePath(android.PathForModuleOut(ctx, grpcName+".rs")) - grpcOut := android.WritablePath(android.PathForModuleOut(ctx, grpcName+grpcSuffix+".rs")) - depFile := android.PathForModuleOut(ctx, grpcName+".d") + // GRPC protos produce two files, a proto.rs and a proto_grpc.rs + protoOut := android.WritablePath(android.PathForModuleOut(ctx, grpcName+".rs")) + grpcOut := android.WritablePath(android.PathForModuleOut(ctx, grpcName+grpcSuffix+".rs")) + depFile := android.PathForModuleOut(ctx, grpcName+".d") - ruleOutputs := android.WritablePaths{protoOut, grpcOut, depFile} + ruleOutputs := android.WritablePaths{protoOut, grpcOut, depFile} - android.ProtoRule(rule, grpcFile, grpcProtoFlags, grpcProtoFlags.Deps, outDir, depFile, ruleOutputs) - outputs = append(outputs, ruleOutputs...) + android.ProtoRule(rule, grpcFile, grpcProtoFlags, grpcProtoFlags.Deps, outDir, depFile, ruleOutputs) + outputs = append(outputs, ruleOutputs...) + } + + ruleName := "protoc_grpc" + ruleDesc := "protoc grpc" + if i > 0 { + ruleName += "_" + strconv.Itoa(i+1) + ruleDesc += " " + strconv.Itoa(i+1) + } + + rule.Build(ruleName, ruleDesc) } // Check that all proto base filenames are unique as outputs are written to the same directory. @@ -168,8 +193,6 @@ func (proto *protobufDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) android.WriteFileRule(ctx, stemFile, proto.genModFileContents()) - rule.Build("protoc_"+ctx.ModuleName(), "protoc "+ctx.ModuleName()) - // stemFile must be first here as the first path in BaseSourceProvider.OutputFiles is the library entry-point. proto.BaseSourceProvider.OutputFiles = append(android.Paths{stemFile}, outputs.Paths()...) diff --git a/scripts/manifest_check.py b/scripts/manifest_check.py index c33b104bd..b10125994 100755 --- a/scripts/manifest_check.py +++ b/scripts/manifest_check.py @@ -52,6 +52,14 @@ def parse_args(): 'required:false' ) parser.add_argument( + '--missing-optional-uses-library', + dest='missing_optional_uses_libraries', + action='append', + help='specify uses-library entries missing from the build system with ' + 'required:false', + default=[] + ) + parser.add_argument( '--enforce-uses-libraries', dest='enforce_uses_libraries', action='store_true', @@ -91,7 +99,7 @@ C_OFF = "\033[0m" C_BOLD = "\033[1m" -def enforce_uses_libraries(manifest, required, optional, relax, is_apk, path): +def enforce_uses_libraries(manifest, required, optional, missing_optional, relax, is_apk, path): """Verify that the <uses-library> tags in the manifest match those provided by the build system. @@ -119,7 +127,12 @@ def enforce_uses_libraries(manifest, required, optional, relax, is_apk, path): required = trim_namespace_parts(required) optional = trim_namespace_parts(optional) - if manifest_required == required and manifest_optional == optional: + existing_manifest_optional = [ + lib for lib in manifest_optional if lib not in missing_optional] + + # The order of the existing libraries matter, while the order of the missing + # ones doesn't. + if manifest_required == required and existing_manifest_optional == optional: return None #pylint: disable=line-too-long @@ -129,6 +142,7 @@ def enforce_uses_libraries(manifest, required, optional, relax, is_apk, path): '\t- required libraries in build system: %s[%s]%s\n' % (C_RED, ', '.join(required), C_OFF), '\t vs. in the manifest: %s[%s]%s\n' % (C_RED, ', '.join(manifest_required), C_OFF), '\t- optional libraries in build system: %s[%s]%s\n' % (C_RED, ', '.join(optional), C_OFF), + '\t and missing ones in build system: %s[%s]%s\n' % (C_RED, ', '.join(missing_optional), C_OFF), '\t vs. in the manifest: %s[%s]%s\n' % (C_RED, ', '.join(manifest_optional), C_OFF), '\t- tags in the manifest (%s):\n' % path, '\t\t%s\n' % '\t\t'.join(tags), @@ -340,11 +354,14 @@ def main(): if args.enforce_uses_libraries: # Load dexpreopt.config files and build a mapping from module - # names to library names. This is necessary because build system - # addresses libraries by their module name (`uses_libs`, - # `optional_uses_libs`, `LOCAL_USES_LIBRARIES`, - # `LOCAL_OPTIONAL_LIBRARY_NAMES` all contain module names), while - # the manifest addresses libraries by their name. + # names to library names. This is for Make only and it's necessary + # because Make passes module names from `LOCAL_USES_LIBRARIES`, + # `LOCAL_OPTIONAL_LIBRARY_NAMES`, while the manifest addresses + # libraries by their name. Soong doesn't use it and doesn't need it + # because it converts the module names to the library names and + # passes the library names. There is no need to translate missing + # optional libs because they are missing and therefore there is no + # mapping for them. mod_to_lib = load_dexpreopt_configs(args.dexpreopt_configs) required = translate_libnames(args.uses_libraries, mod_to_lib) optional = translate_libnames(args.optional_uses_libraries, @@ -354,8 +371,8 @@ def main(): # those in the manifest. Raise an exception on mismatch, unless the # script was passed a special parameter to suppress exceptions. errmsg = enforce_uses_libraries(manifest, required, optional, - args.enforce_uses_libraries_relax, - is_apk, args.input) + args.missing_optional_uses_libraries, + args.enforce_uses_libraries_relax, is_apk, args.input) # Create a status file that is empty on success, or contains an # error message on failure. When exceptions are suppressed, diff --git a/scripts/manifest_check_test.py b/scripts/manifest_check_test.py index 3be7a30bf..8003b3e19 100755 --- a/scripts/manifest_check_test.py +++ b/scripts/manifest_check_test.py @@ -44,15 +44,17 @@ def required_apk(value): class EnforceUsesLibrariesTest(unittest.TestCase): """Unit tests for add_extract_native_libs function.""" - def run_test(self, xml, apk, uses_libraries=[], optional_uses_libraries=[]): #pylint: disable=dangerous-default-value + def run_test(self, xml, apk, uses_libraries=[], optional_uses_libraries=[], + missing_optional_uses_libraries=[]): #pylint: disable=dangerous-default-value doc = minidom.parseString(xml) try: relax = False manifest_check.enforce_uses_libraries( - doc, uses_libraries, optional_uses_libraries, relax, False, - 'path/to/X/AndroidManifest.xml') + doc, uses_libraries, optional_uses_libraries, missing_optional_uses_libraries, + relax, False, 'path/to/X/AndroidManifest.xml') manifest_check.enforce_uses_libraries(apk, uses_libraries, optional_uses_libraries, + missing_optional_uses_libraries, relax, True, 'path/to/X/X.apk') return True @@ -102,6 +104,15 @@ class EnforceUsesLibrariesTest(unittest.TestCase): matches = self.run_test(xml, apk, optional_uses_libraries=['foo']) self.assertFalse(matches) + def test_expected_missing_optional_uses_library(self): + xml = self.xml_tmpl % ( + uses_library_xml('foo') + uses_library_xml('missing') + uses_library_xml('bar')) + apk = self.apk_tmpl % ( + uses_library_apk('foo') + uses_library_apk('missing') + uses_library_apk('bar')) + matches = self.run_test(xml, apk, optional_uses_libraries=['foo', 'bar'], + missing_optional_uses_libraries=['missing']) + self.assertFalse(matches) + def test_missing_uses_library(self): xml = self.xml_tmpl % ('') apk = self.apk_tmpl % ('') diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index 275860f32..0a5483b07 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -688,6 +688,12 @@ func TestSnapshotWithJavaSystemModules(t *testing.T) { public: { enabled: true, }, + system: { + enabled: true, + }, + module_lib: { + enabled: true, + }, } java_system_modules { @@ -752,6 +758,20 @@ java_sdk_library_import { removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", }, + system: { + jars: ["sdk_library/system/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], + current_api: "sdk_library/system/myjavalib.txt", + removed_api: "sdk_library/system/myjavalib-removed.txt", + sdk_version: "system_current", + }, + module_lib: { + jars: ["sdk_library/module-lib/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/module-lib/myjavalib_stub_sources"], + current_api: "sdk_library/module-lib/myjavalib.txt", + removed_api: "sdk_library/module-lib/myjavalib-removed.txt", + sdk_version: "module_current", + }, } java_system_modules_import { @@ -771,6 +791,12 @@ java_system_modules_import { .intermediates/myjavalib.stubs.exportable/android_common/combined/myjavalib.stubs.exportable.jar -> sdk_library/public/myjavalib-stubs.jar .intermediates/myjavalib.stubs.source/android_common/exportable/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt .intermediates/myjavalib.stubs.source/android_common/exportable/myjavalib.stubs.source_removed.txt -> sdk_library/public/myjavalib-removed.txt +.intermediates/myjavalib.stubs.exportable.system/android_common/combined/myjavalib.stubs.exportable.system.jar -> sdk_library/system/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source.system/android_common/exportable/myjavalib.stubs.source.system_api.txt -> sdk_library/system/myjavalib.txt +.intermediates/myjavalib.stubs.source.system/android_common/exportable/myjavalib.stubs.source.system_removed.txt -> sdk_library/system/myjavalib-removed.txt +.intermediates/myjavalib.stubs.exportable.module_lib/android_common/combined/myjavalib.stubs.exportable.module_lib.jar -> sdk_library/module-lib/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source.module_lib/android_common/exportable/myjavalib.stubs.source.module_lib_api.txt -> sdk_library/module-lib/myjavalib.txt +.intermediates/myjavalib.stubs.source.module_lib/android_common/exportable/myjavalib.stubs.source.module_lib_removed.txt -> sdk_library/module-lib/myjavalib-removed.txt `), checkInfoContents(result.Config, ` [ @@ -805,11 +831,23 @@ java_system_modules_import { "@name": "myjavalib", "dist_stem": "myjavalib", "scopes": { + "module-lib": { + "current_api": "sdk_library/module-lib/myjavalib.txt", + "latest_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib.api.module-lib.latest/gen/myjavalib.api.module-lib.latest", + "latest_removed_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib-removed.api.module-lib.latest/gen/myjavalib-removed.api.module-lib.latest", + "removed_api": "sdk_library/module-lib/myjavalib-removed.txt" + }, "public": { "current_api": "sdk_library/public/myjavalib.txt", "latest_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib.api.public.latest/gen/myjavalib.api.public.latest", "latest_removed_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib-removed.api.public.latest/gen/myjavalib-removed.api.public.latest", "removed_api": "sdk_library/public/myjavalib-removed.txt" + }, + "system": { + "current_api": "sdk_library/system/myjavalib.txt", + "latest_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib.api.system.latest/gen/myjavalib.api.system.latest", + "latest_removed_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib-removed.api.system.latest/gen/myjavalib-removed.api.system.latest", + "removed_api": "sdk_library/system/myjavalib-removed.txt" } } }, diff --git a/soong_ui.bash b/soong_ui.bash index 8e7cd195d..77378807f 100755 --- a/soong_ui.bash +++ b/soong_ui.bash @@ -35,6 +35,7 @@ source ${TOP}/build/soong/scripts/microfactory.bash soong_build_go soong_ui android/soong/cmd/soong_ui soong_build_go mk2rbc android/soong/mk2rbc/mk2rbc soong_build_go rbcrun rbcrun/rbcrun +soong_build_go release-config android/soong/cmd/release_config/release_config cd ${TOP} exec "$(getoutdir)/soong_ui" "$@" diff --git a/tradefed_modules/test_module_config.go b/tradefed_modules/test_module_config.go index 686753713..9127f67a9 100644 --- a/tradefed_modules/test_module_config.go +++ b/tradefed_modules/test_module_config.go @@ -297,10 +297,16 @@ func (m *testModuleConfigModule) validateBase(ctx android.ModuleContext, depTag // 1. manifest file to testcases dir // 2. New Module.config / AndroidTest.xml file with our options. func (m *testModuleConfigModule) generateManifestAndConfig(ctx android.ModuleContext) { + // Keep before early returns. + android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{ + TestOnly: true, + TopLevelTarget: true, + }) + if !m.validateTestSuites(ctx) { return } - // Ensure the provider is accurate + // Ensure the base provider is accurate if m.provider.TestConfig == nil { return } diff --git a/tradefed_modules/test_module_config_test.go b/tradefed_modules/test_module_config_test.go index 41dd3d479..6997228ce 100644 --- a/tradefed_modules/test_module_config_test.go +++ b/tradefed_modules/test_module_config_test.go @@ -19,6 +19,8 @@ import ( "strconv" "strings" "testing" + + "github.com/google/blueprint" ) const bp = ` @@ -347,6 +349,67 @@ func TestModuleConfigHostDuplicateTestSuitesGiveErrors(t *testing.T) { RunTestWithBp(t, badBp) } +func TestTestOnlyProvider(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + java.PrepareForTestWithJavaDefaultModules, + android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), + ).RunTestWithBp(t, ` + // These should be test-only + test_module_config_host { + name: "host-derived-test", + base: "host-base", + exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], + include_annotations: ["android.platform.test.annotations.LargeTest"], + test_suites: ["general-tests"], + } + + test_module_config { + name: "derived-test", + base: "base", + exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], + include_annotations: ["android.platform.test.annotations.LargeTest"], + test_suites: ["general-tests"], + } + + android_test { + name: "base", + sdk_version: "current", + data: ["data/testfile"], + } + + java_test_host { + name: "host-base", + srcs: ["a.java"], + test_suites: ["general-tests"], + }`, + ) + + // Visit all modules and ensure only the ones that should + // marked as test-only are marked as test-only. + + actualTestOnly := []string{} + ctx.VisitAllModules(func(m blueprint.Module) { + if provider, ok := android.OtherModuleProvider(ctx.TestContext.OtherModuleProviderAdaptor(), m, android.TestOnlyProviderKey); ok { + if provider.TestOnly { + actualTestOnly = append(actualTestOnly, m.Name()) + } + } + }) + expectedTestOnlyModules := []string{ + "host-derived-test", + "derived-test", + // android_test and java_test_host are tests too. + "host-base", + "base", + } + + notEqual, left, right := android.ListSetDifference(expectedTestOnlyModules, actualTestOnly) + if notEqual { + t.Errorf("test-only: Expected but not found: %v, Found but not expected: %v", left, right) + } +} + // Use for situations where the entries map contains pairs: [srcPath:installedPath1, srcPath2:installedPath2] // and we want to compare the RHS of the pairs, i.e. installedPath1, installedPath2 func assertEntryPairValues(t *testing.T, actual []string, expected []string) { |