diff options
64 files changed, 3024 insertions, 1266 deletions
diff --git a/android/all_teams.go b/android/all_teams.go index 0c433a6cb..d4bf7d0c4 100644 --- a/android/all_teams.go +++ b/android/all_teams.go @@ -79,11 +79,6 @@ func (t *allTeamsSingleton) GenerateBuildActions(ctx SingletonContext) { ctx.VisitAllModules(func(module Module) { bpFile := ctx.BlueprintFile(module) - testModInfo := TestModuleInformation{} - if tmi, ok := SingletonModuleProvider(ctx, module, TestOnlyProviderKey); ok { - testModInfo = tmi - } - // Package Modules and Team Modules are stored in a map so we can look them up by name for // modules without a team. if pack, ok := module.(*packageModule); ok { @@ -97,6 +92,23 @@ func (t *allTeamsSingleton) GenerateBuildActions(ctx SingletonContext) { return } + testModInfo := TestModuleInformation{} + if tmi, ok := SingletonModuleProvider(ctx, module, TestOnlyProviderKey); ok { + testModInfo = tmi + } + + // Some modules, like java_test_host don't set the provider when the module isn't enabled: + // test_only, top_level + // AVFHostTestCases{os:linux_glibc,arch:common} {true true} + // AVFHostTestCases{os:windows,arch:common} {false false} + // Generally variant information of true override false or unset. + if testModInfo.TestOnly == false { + if prevValue, exists := t.teams_for_mods[module.Name()]; exists { + if prevValue.testOnly == true { + return + } + } + } entry := moduleTeamAndTestInfo{ bpFile: bpFile, testOnly: testModInfo.TestOnly, diff --git a/android/all_teams_test.go b/android/all_teams_test.go index 9c2b38e42..96ed92fc5 100644 --- a/android/all_teams_test.go +++ b/android/all_teams_test.go @@ -25,6 +25,8 @@ func TestAllTeams(t *testing.T) { t.Parallel() ctx := GroupFixturePreparers( prepareForTestWithTeamAndFakes, + // This adds two variants, one armv7-a-neon, one armv8-a + PrepareForTestWithArchMutator, FixtureRegisterWithContext(func(ctx RegistrationContext) { ctx.RegisterParallelSingletonType("all_teams", AllTeamsFactory) }), @@ -52,10 +54,35 @@ func TestAllTeams(t *testing.T) { name: "noteam", test_only: true, } + // write the test-only provider value once fake { - name: "test-and-team-and-top", + name: "test-and-team-and-top1", test_only: true, team: "team2", + arch: {arm: { skip: false}, + arm64: { skip: true}}, + } + // write the test-only provider once, but on the other arch + fake { + name: "test-and-team-and-top2", + test_only: true, + team: "team2", + arch: {arm: { skip: true}, + arm64: { skip: false}}, + } + // write the test-only provider value twice + fake { + name: "test-and-team-and-top3", + test_only: true, + team: "team2", + } + // Don't write the test-only provider value + fake { + name: "test-and-team-and-top4", + test_only: true, + team: "team2", + arch: {arm: { skip: true}, + arm64: { skip: true}}, } `) @@ -63,12 +90,16 @@ func TestAllTeams(t *testing.T) { teams = getTeamProtoOutput(t, ctx) // map of module name -> trendy team name. - actualTeams := make(map[string]*string) + actualTeams := make(map[string]string) actualTests := []string{} actualTopLevelTests := []string{} for _, teamProto := range teams.Teams { - actualTeams[teamProto.GetTargetName()] = teamProto.TrendyTeamId + if teamProto.TrendyTeamId != nil { + actualTeams[teamProto.GetTargetName()] = *teamProto.TrendyTeamId + } else { + actualTeams[teamProto.GetTargetName()] = "" + } if teamProto.GetTestOnly() { actualTests = append(actualTests, teamProto.GetTargetName()) } @@ -76,16 +107,23 @@ func TestAllTeams(t *testing.T) { actualTopLevelTests = append(actualTopLevelTests, teamProto.GetTargetName()) } } - expectedTeams := map[string]*string{ - "main_test": proto.String("cool_team"), - "tool": proto.String("22222"), - "test-and-team-and-top": proto.String("22222"), - "noteam": nil, + expectedTeams := map[string]string{ + "main_test": "cool_team", + "tool": "22222", + "test-and-team-and-top1": "22222", + "test-and-team-and-top2": "22222", + "test-and-team-and-top3": "22222", + "test-and-team-and-top4": "22222", + "noteam": "", } expectedTests := []string{ "noteam", - "test-and-team-and-top", + "test-and-team-and-top1", + "test-and-team-and-top2", + "test-and-team-and-top3", + // There should be no test-and-team-top4 as we skip writing all variants + // test-only for all variants } AssertDeepEquals(t, "compare maps", expectedTeams, actualTeams) AssertDeepEquals(t, "test matchup", expectedTests, actualTests) @@ -230,12 +268,16 @@ type fakeForTests struct { ModuleBase sourceProperties SourceProperties + props struct { + // If true, don't write test-only value in provider + Skip bool `android:"arch_variant"` + } } func fakeFactory() Module { module := &fakeForTests{} - module.AddProperties(&module.sourceProperties) - InitAndroidModule(module) + module.AddProperties(&module.sourceProperties, &module.props) + InitAndroidArchModule(module, HostAndDeviceSupported, MultilibBoth) return module } @@ -250,7 +292,7 @@ var prepareForTestWithTeamAndFakes = GroupFixturePreparers( func (f *fakeForTests) GenerateAndroidBuildActions(ctx ModuleContext) { if Bool(f.sourceProperties.Test_only) { SetProvider(ctx, TestOnlyProviderKey, TestModuleInformation{ - TestOnly: Bool(f.sourceProperties.Test_only), + TestOnly: Bool(f.sourceProperties.Test_only) && !f.props.Skip, TopLevelTarget: false, }) } diff --git a/android/apex.go b/android/apex.go index 87599051a..c0acada13 100644 --- a/android/apex.go +++ b/android/apex.go @@ -16,6 +16,7 @@ package android import ( "fmt" + "slices" "sort" "strconv" "strings" @@ -108,7 +109,7 @@ func (i ApexInfo) AddJSONData(d *map[string]interface{}) { // are configured to have the same alias variation named apex29. Whether platform APIs is allowed // or not also matters; if two APEXes don't have the same allowance, they get different names and // thus wouldn't be merged. -func (i ApexInfo) mergedName(ctx PathContext) string { +func (i ApexInfo) mergedName() string { name := "apex" + strconv.Itoa(i.MinSdkVersion.FinalOrFutureInt()) return name } @@ -543,17 +544,10 @@ func AvailableToSameApexes(mod1, mod2 ApexModule) bool { return true } -type byApexName []ApexInfo - -func (a byApexName) Len() int { return len(a) } -func (a byApexName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a byApexName) Less(i, j int) bool { return a[i].ApexVariationName < a[j].ApexVariationName } - // mergeApexVariations deduplicates apex variations that would build identically into a common // variation. It returns the reduced list of variations and a list of aliases from the original // variation names to the new variation names. -func mergeApexVariations(ctx PathContext, apexInfos []ApexInfo) (merged []ApexInfo, aliases [][2]string) { - sort.Sort(byApexName(apexInfos)) +func mergeApexVariations(apexInfos []ApexInfo) (merged []ApexInfo, aliases [][2]string) { seen := make(map[string]int) for _, apexInfo := range apexInfos { // If this is for a prebuilt apex then use the actual name of the apex variation to prevent this @@ -567,7 +561,7 @@ func mergeApexVariations(ctx PathContext, apexInfos []ApexInfo) (merged []ApexIn // this one into it, otherwise create a new merged ApexInfo from this one and save it away so // other ApexInfo instances can be merged into it. variantName := apexInfo.ApexVariationName - mergedName := apexInfo.mergedName(ctx) + mergedName := apexInfo.mergedName() if index, exists := seen[mergedName]; exists { // Variants having the same mergedName are deduped merged[index].InApexVariants = append(merged[index].InApexVariants, variantName) @@ -606,18 +600,20 @@ func CreateApexVariations(mctx BottomUpMutatorContext, module ApexModule) []Modu // TODO(jiyong): is this the right place? base.checkApexAvailableProperty(mctx) - var apexInfos []ApexInfo - var aliases [][2]string - if !mctx.Module().(ApexModule).UniqueApexVariations() && !base.ApexProperties.UniqueApexVariationsForDeps { - apexInfos, aliases = mergeApexVariations(mctx, base.apexInfos) - } else { - apexInfos = base.apexInfos - } + apexInfos := base.apexInfos // base.apexInfos is only needed to propagate the list of apexes from apexInfoMutator to // apexMutator. It is no longer accurate after mergeApexVariations, and won't be copied to // all but the first created variant. Clear it so it doesn't accidentally get used later. base.apexInfos = nil - sort.Sort(byApexName(apexInfos)) + + slices.SortFunc(apexInfos, func(a, b ApexInfo) int { + return strings.Compare(a.ApexVariationName, b.ApexVariationName) + }) + + var aliases [][2]string + if !mctx.Module().(ApexModule).UniqueApexVariations() && !base.ApexProperties.UniqueApexVariationsForDeps { + apexInfos, aliases = mergeApexVariations(apexInfos) + } var inApex ApexMembership for _, a := range apexInfos { diff --git a/android/apex_test.go b/android/apex_test.go index ddc730d05..347bf7d98 100644 --- a/android/apex_test.go +++ b/android/apex_test.go @@ -33,10 +33,22 @@ func Test_mergeApexVariations(t *testing.T) { { name: "single", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantAliases: [][2]string{ {"foo", "apex10000"}, @@ -45,98 +57,231 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", FutureApiLevel, false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, false, nil}}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo", "bar"}, + InApexModules: []string{"foo", "bar"}, + }}, wantAliases: [][2]string{ - {"bar", "apex10000"}, {"foo", "apex10000"}, + {"bar", "apex10000"}, }, }, { name: "don't merge version", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: uncheckedFinalApiLevel(30), + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex30", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, - {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "apex30", + MinSdkVersion: uncheckedFinalApiLevel(30), + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantAliases: [][2]string{ - {"bar", "apex30"}, {"foo", "apex10000"}, + {"bar", "apex30"}, }, }, { name: "merge updatable", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"foo", "bar"}, + InApexModules: []string{"foo", "bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantAliases: [][2]string{ - {"bar", "apex10000"}, {"foo", "apex10000"}, + {"bar", "apex10000"}, }, }, { name: "don't merge when for prebuilt_apex", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, // This one should not be merged in with the others because it is for // a prebuilt_apex. - {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex, nil}, + { + ApexVariationName: "baz", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"baz"}, + InApexModules: []string{"baz"}, + ForPrebuiltApex: ForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, - {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"foo", "bar"}, + InApexModules: []string{"foo", "bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "baz", + MinSdkVersion: FutureApiLevel, + Updatable: true, + InApexVariants: []string{"baz"}, + InApexModules: []string{"baz"}, + ForPrebuiltApex: ForPrebuiltApex, + }, }, wantAliases: [][2]string{ - {"bar", "apex10000"}, {"foo", "apex10000"}, + {"bar", "apex10000"}, }, }, { name: "merge different UsePlatformApis but don't allow using platform api", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: FutureApiLevel, + UsePlatformApis: true, + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + InApexVariants: []string{"foo", "bar"}, + InApexModules: []string{"foo", "bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantAliases: [][2]string{ - {"bar", "apex10000"}, {"foo", "apex10000"}, + {"bar", "apex10000"}, }, }, { name: "merge same UsePlatformApis and allow using platform api", in: []ApexInfo{ - {"foo", FutureApiLevel, false, true, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex, nil}, - {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "foo", + MinSdkVersion: FutureApiLevel, + UsePlatformApis: true, + InApexVariants: []string{"foo"}, + InApexModules: []string{"foo"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, + { + ApexVariationName: "bar", + MinSdkVersion: FutureApiLevel, + UsePlatformApis: true, + InApexVariants: []string{"bar"}, + InApexModules: []string{"bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, true, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex, nil}, + { + ApexVariationName: "apex10000", + MinSdkVersion: FutureApiLevel, + UsePlatformApis: true, + InApexVariants: []string{"foo", "bar"}, + InApexModules: []string{"foo", "bar"}, + ForPrebuiltApex: NotForPrebuiltApex, + }, }, wantAliases: [][2]string{ - {"bar", "apex10000"}, {"foo", "apex10000"}, + {"bar", "apex10000"}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - config := TestConfig(t.TempDir(), nil, "", nil) - ctx := &configErrorWrapper{config: config} - gotMerged, gotAliases := mergeApexVariations(ctx, tt.in) + gotMerged, gotAliases := mergeApexVariations(tt.in) if !reflect.DeepEqual(gotMerged, tt.wantMerged) { t.Errorf("mergeApexVariations() gotMerged = %v, want %v", gotMerged, tt.wantMerged) } diff --git a/android/base_module_context.go b/android/base_module_context.go index c4922f4bc..c5fe58578 100644 --- a/android/base_module_context.go +++ b/android/base_module_context.go @@ -20,7 +20,7 @@ import ( "strings" "github.com/google/blueprint" - "github.com/google/blueprint/parser" + "github.com/google/blueprint/proptools" ) // BaseModuleContext is the same as blueprint.BaseModuleContext except that Config() returns @@ -219,7 +219,7 @@ type BaseModuleContext interface { // EvaluateConfiguration makes ModuleContext a valid proptools.ConfigurableEvaluator, so this context // can be used to evaluate the final value of Configurable properties. - EvaluateConfiguration(parser.SelectType, string, string) (string, bool) + EvaluateConfiguration(condition proptools.ConfigurableCondition, property string) proptools.ConfigurableValue } type baseModuleContext struct { @@ -577,6 +577,6 @@ func (b *baseModuleContext) GetPathString(skipFirst bool) string { return sb.String() } -func (m *baseModuleContext) EvaluateConfiguration(ty parser.SelectType, property, condition string) (string, bool) { - return m.Module().ConfigurableEvaluator(m).EvaluateConfiguration(ty, property, condition) +func (m *baseModuleContext) EvaluateConfiguration(condition proptools.ConfigurableCondition, property string) proptools.ConfigurableValue { + return m.Module().ConfigurableEvaluator(m).EvaluateConfiguration(condition, property) } diff --git a/android/config.go b/android/config.go index 11bd81b89..75d135fc0 100644 --- a/android/config.go +++ b/android/config.go @@ -949,7 +949,11 @@ func (c *config) PlatformPreviewSdkVersion() string { } func (c *config) PlatformMinSupportedTargetSdkVersion() string { - return String(c.productVariables.Platform_min_supported_target_sdk_version) + var val, ok = c.productVariables.BuildFlags["RELEASE_PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION"] + if !ok { + return "" + } + return val } func (c *config) PlatformBaseOS() string { diff --git a/android/module.go b/android/module.go index 89c4ddde9..5fe379c8a 100644 --- a/android/module.go +++ b/android/module.go @@ -29,7 +29,6 @@ import ( "android/soong/bazel" "github.com/google/blueprint" - "github.com/google/blueprint/parser" "github.com/google/blueprint/proptools" ) @@ -1043,12 +1042,12 @@ func (m *ModuleBase) baseDepsMutator(ctx BottomUpMutatorContext) { pv := ctx.Config().productVariables fullManifest := pv.DeviceArch != nil && pv.DeviceName != nil if fullManifest { - m.addRequiredDeps(ctx) + addRequiredDeps(ctx) } } // addRequiredDeps adds required, target_required, and host_required as dependencies. -func (m *ModuleBase) addRequiredDeps(ctx BottomUpMutatorContext) { +func addRequiredDeps(ctx BottomUpMutatorContext) { addDep := func(target Target, depName string) { if !ctx.OtherModuleExists(depName) { if ctx.Config().AllowMissingDependencies() { @@ -1061,9 +1060,9 @@ func (m *ModuleBase) addRequiredDeps(ctx BottomUpMutatorContext) { // in build/make/core/main.mk. // TODO(jiyong): the Make-side does this only when the required module is a shared // library or a native test. - bothInAndroid := m.Device() && target.Os.Class == Device - nativeArch := InList(m.Arch().ArchType.Multilib, []string{"lib32", "lib64"}) - sameBitness := m.Arch().ArchType.Multilib == target.Arch.ArchType.Multilib + bothInAndroid := ctx.Device() && target.Os.Class == Device + nativeArch := InList(ctx.Arch().ArchType.Multilib, []string{"lib32", "lib64"}) + sameBitness := ctx.Arch().ArchType.Multilib == target.Arch.ArchType.Multilib if bothInAndroid && nativeArch && !sameBitness { return } @@ -1082,31 +1081,31 @@ func (m *ModuleBase) addRequiredDeps(ctx BottomUpMutatorContext) { hostTargets = append(hostTargets, ctx.Config().Targets[ctx.Config().BuildOS]...) hostTargets = append(hostTargets, ctx.Config().BuildOSCommonTarget) - if m.Device() { - for _, depName := range m.RequiredModuleNames() { + if ctx.Device() { + for _, depName := range ctx.Module().RequiredModuleNames() { for _, target := range deviceTargets { addDep(target, depName) } } - for _, depName := range m.HostRequiredModuleNames() { + for _, depName := range ctx.Module().HostRequiredModuleNames() { for _, target := range hostTargets { addDep(target, depName) } } } - if m.Host() { - for _, depName := range m.RequiredModuleNames() { + if ctx.Host() { + for _, depName := range ctx.Module().RequiredModuleNames() { for _, target := range hostTargets { // When a host module requires another host module, don't make a // dependency if they have different OSes (i.e. hostcross). - if m.Target().HostCross != target.HostCross { + if ctx.Target().HostCross != target.HostCross { continue } addDep(target, depName) } } - for _, depName := range m.TargetRequiredModuleNames() { + for _, depName := range ctx.Module().TargetRequiredModuleNames() { for _, target := range deviceTargets { addDep(target, depName) } @@ -2140,41 +2139,82 @@ func (e configurationEvalutor) PropertyErrorf(property string, fmt string, args e.ctx.OtherModulePropertyErrorf(e.m, property, fmt, args) } -func (e configurationEvalutor) EvaluateConfiguration(ty parser.SelectType, property, condition string) (string, bool) { +func (e configurationEvalutor) EvaluateConfiguration(condition proptools.ConfigurableCondition, property string) proptools.ConfigurableValue { ctx := e.ctx m := e.m - switch ty { - case parser.SelectTypeReleaseVariable: - if v, ok := ctx.Config().productVariables.BuildFlags[condition]; ok { - return v, true + switch condition.FunctionName { + case "release_variable": + if len(condition.Args) != 1 { + ctx.OtherModulePropertyErrorf(m, property, "release_variable requires 1 argument, found %d", len(condition.Args)) + return proptools.ConfigurableValueUndefined() + } + if v, ok := ctx.Config().productVariables.BuildFlags[condition.Args[0]]; ok { + return proptools.ConfigurableValueString(v) } - return "", false - case parser.SelectTypeProductVariable: + return proptools.ConfigurableValueUndefined() + case "product_variable": // TODO(b/323382414): Might add these on a case-by-case basis ctx.OtherModulePropertyErrorf(m, property, "TODO(b/323382414): Product variables are not yet supported in selects") - return "", false - case parser.SelectTypeSoongConfigVariable: - parts := strings.Split(condition, ":") - namespace := parts[0] - variable := parts[1] + return proptools.ConfigurableValueUndefined() + case "soong_config_variable": + if len(condition.Args) != 2 { + ctx.OtherModulePropertyErrorf(m, property, "soong_config_variable requires 2 arguments, found %d", len(condition.Args)) + return proptools.ConfigurableValueUndefined() + } + namespace := condition.Args[0] + variable := condition.Args[1] if n, ok := ctx.Config().productVariables.VendorVars[namespace]; ok { if v, ok := n[variable]; ok { - return v, true + return proptools.ConfigurableValueString(v) } } - return "", false - case parser.SelectTypeVariant: - if condition == "arch" { - if !m.base().ArchReady() { - ctx.OtherModulePropertyErrorf(m, property, "A select on arch was attempted before the arch mutator ran") - return "", false + return proptools.ConfigurableValueUndefined() + case "arch": + if len(condition.Args) != 0 { + ctx.OtherModulePropertyErrorf(m, property, "arch requires no arguments, found %d", len(condition.Args)) + return proptools.ConfigurableValueUndefined() + } + if !m.base().ArchReady() { + ctx.OtherModulePropertyErrorf(m, property, "A select on arch was attempted before the arch mutator ran") + return proptools.ConfigurableValueUndefined() + } + return proptools.ConfigurableValueString(m.base().Arch().ArchType.Name) + case "os": + if len(condition.Args) != 0 { + ctx.OtherModulePropertyErrorf(m, property, "os requires no arguments, found %d", len(condition.Args)) + return proptools.ConfigurableValueUndefined() + } + // the arch mutator runs after the os mutator, we can just use this to enforce that os is ready. + if !m.base().ArchReady() { + ctx.OtherModulePropertyErrorf(m, property, "A select on os was attempted before the arch mutator ran (arch runs after os, we use it to lazily detect that os is ready)") + return proptools.ConfigurableValueUndefined() + } + return proptools.ConfigurableValueString(m.base().Os().Name) + case "boolean_var_for_testing": + // We currently don't have any other boolean variables (we should add support for typing + // the soong config variables), so add this fake one for testing the boolean select + // functionality. + if len(condition.Args) != 0 { + ctx.OtherModulePropertyErrorf(m, property, "boolean_var_for_testing requires 0 arguments, found %d", len(condition.Args)) + return proptools.ConfigurableValueUndefined() + } + + if n, ok := ctx.Config().productVariables.VendorVars["boolean_var"]; ok { + if v, ok := n["for_testing"]; ok { + switch v { + case "true": + return proptools.ConfigurableValueBool(true) + case "false": + return proptools.ConfigurableValueBool(false) + default: + ctx.OtherModulePropertyErrorf(m, property, "testing:my_boolean_var can only be true or false, found %q", v) + } } - return m.base().Arch().ArchType.Name, true } - ctx.OtherModulePropertyErrorf(m, property, "Unknown variant %s", condition) - return "", false + return proptools.ConfigurableValueUndefined() default: - panic("Should be unreachable") + ctx.OtherModulePropertyErrorf(m, property, "Unknown select condition %s", condition.FunctionName) + return proptools.ConfigurableValueUndefined() } } diff --git a/android/mutator.go b/android/mutator.go index 0ff4f48ea..75ba65048 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -391,6 +391,7 @@ func (x *registerMutatorsContext) BottomUpBlueprint(name string, m blueprint.Bot type IncomingTransitionContext interface { ArchModuleContext + ModuleProviderContext // Module returns the target of the dependency edge for which the transition // is being computed @@ -404,6 +405,7 @@ type IncomingTransitionContext interface { type OutgoingTransitionContext interface { ArchModuleContext + ModuleProviderContext // Module returns the target of the dependency edge for which the transition // is being computed @@ -505,6 +507,7 @@ type TransitionMutator interface { type androidTransitionMutator struct { finalPhase bool mutator TransitionMutator + name string } func (a *androidTransitionMutator) Split(ctx blueprint.BaseModuleContext) []string { @@ -537,6 +540,10 @@ func (c *outgoingTransitionContextImpl) DeviceConfig() DeviceConfig { return DeviceConfig{c.bp.Config().(Config).deviceConfig} } +func (c *outgoingTransitionContextImpl) provider(provider blueprint.AnyProviderKey) (any, bool) { + return c.bp.Provider(provider) +} + func (a *androidTransitionMutator) OutgoingTransition(bpctx blueprint.OutgoingTransitionContext, sourceVariation string) string { if m, ok := bpctx.Module().(Module); ok { ctx := outgoingTransitionContextPool.Get().(*outgoingTransitionContextImpl) @@ -568,6 +575,10 @@ func (c *incomingTransitionContextImpl) DeviceConfig() DeviceConfig { return DeviceConfig{c.bp.Config().(Config).deviceConfig} } +func (c *incomingTransitionContextImpl) provider(provider blueprint.AnyProviderKey) (any, bool) { + return c.bp.Provider(provider) +} + func (a *androidTransitionMutator) IncomingTransition(bpctx blueprint.IncomingTransitionContext, incomingVariation string) string { if m, ok := bpctx.Module().(Module); ok { ctx := incomingTransitionContextPool.Get().(*incomingTransitionContextImpl) @@ -586,6 +597,9 @@ func (a *androidTransitionMutator) Mutate(ctx blueprint.BottomUpMutatorContext, if am, ok := ctx.Module().(Module); ok { mctx := bottomUpMutatorContextFactory(ctx, am, a.finalPhase) defer bottomUpMutatorContextPool.Put(mctx) + base := am.base() + base.commonProperties.DebugMutators = append(base.commonProperties.DebugMutators, a.name) + base.commonProperties.DebugVariations = append(base.commonProperties.DebugVariations, variation) a.mutator.Mutate(mctx, variation) } } @@ -594,6 +608,7 @@ func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) { atm := &androidTransitionMutator{ finalPhase: x.finalPhase, mutator: m, + name: name, } mutator := &mutator{ name: name, diff --git a/android/prebuilt.go b/android/prebuilt.go index 91ba05b6b..2b7b55b5d 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -706,11 +706,6 @@ func (p *Prebuilt) usePrebuilt(ctx BaseMutatorContext, source Module, prebuilt M return true } - // If the use_source_config_var property is set then it overrides the prefer property setting. - if configVar := p.properties.Use_source_config_var; configVar != nil { - return !ctx.Config().VendorConfig(proptools.String(configVar.Config_namespace)).Bool(proptools.String(configVar.Var_name)) - } - // TODO: use p.Properties.Name and ctx.ModuleDir to override preference return Bool(p.properties.Prefer) } diff --git a/android/prebuilt_test.go b/android/prebuilt_test.go index 2241b0815..2574ed487 100644 --- a/android/prebuilt_test.go +++ b/android/prebuilt_test.go @@ -295,158 +295,6 @@ func TestPrebuilts(t *testing.T) { }`, prebuilt: []OsType{Android, buildOS}, }, - { - name: "prebuilt use_source_config_var={acme, use_source} - no var specified", - modules: ` - source { - name: "bar", - } - - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - }`, - // When use_source_env is specified then it will use the prebuilt by default if the environment - // variable is not set. - prebuilt: []OsType{Android, buildOS}, - }, - { - name: "prebuilt use_source_config_var={acme, use_source} - acme_use_source=false", - modules: ` - source { - name: "bar", - } - - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - }`, - preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "false", - }, - } - }), - // Setting the environment variable named in use_source_env to false will cause the prebuilt to - // be used. - prebuilt: []OsType{Android, buildOS}, - }, - { - name: "apex_contributions supersedes any source preferred via use_source_config_var", - modules: ` - source { - name: "bar", - } - - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - } - apex_contributions { - name: "my_mainline_module_contribution", - api_domain: "apexfoo", - // this metadata module contains prebuilt - contents: ["prebuilt_bar"], - } - all_apex_contributions { - name: "all_apex_contributions", - } - `, - preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "true", - }, - } - variables.BuildFlags = map[string]string{ - "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": "my_mainline_module_contribution", - } - }), - // use_source_config_var indicates that source should be used - // but this is superseded by `my_mainline_module_contribution` - prebuilt: []OsType{Android, buildOS}, - }, - { - name: "apex_contributions supersedes any prebuilt preferred via use_source_config_var", - modules: ` - source { - name: "bar", - } - - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - } - apex_contributions { - name: "my_mainline_module_contribution", - api_domain: "apexfoo", - // this metadata module contains source - contents: ["bar"], - } - all_apex_contributions { - name: "all_apex_contributions", - } - `, - preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "false", - }, - } - variables.BuildFlags = map[string]string{ - "RELEASE_APEX_CONTRIBUTIONS_ADSERVICES": "my_mainline_module_contribution", - } - }), - // use_source_config_var indicates that prebuilt should be used - // but this is superseded by `my_mainline_module_contribution` - prebuilt: nil, - }, - { - name: "prebuilt use_source_config_var={acme, use_source} - acme_use_source=true", - modules: ` - source { - name: "bar", - } - - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - }`, - preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "true", - }, - } - }), - // Setting the environment variable named in use_source_env to true will cause the source to be - // used. - prebuilt: nil, - }, - { - name: "prebuilt use_source_config_var={acme, use_source} - acme_use_source=true, no source", - modules: ` - prebuilt { - name: "bar", - use_source_config_var: {config_namespace: "acme", var_name: "use_source"}, - srcs: ["prebuilt_file"], - }`, - preparer: FixtureModifyProductVariables(func(variables FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "true", - }, - } - }), - // Although the environment variable says to use source there is no source available. - prebuilt: []OsType{Android, buildOS}, - }, } fs := MockFS{ diff --git a/android/selects_test.go b/android/selects_test.go index e59b3e65d..f912ce626 100644 --- a/android/selects_test.go +++ b/android/selects_test.go @@ -269,11 +269,11 @@ func TestSelects(t *testing.T) { }, }, { - name: "Select on variant", + name: "Select on arch", bp: ` my_module_type { name: "foo", - my_string: select(variant("arch"), { + my_string: select(arch(), { "x86": "my_x86", "x86_64": "my_x86_64", "arm": "my_arm", @@ -287,6 +287,22 @@ func TestSelects(t *testing.T) { }, }, { + name: "Select on os", + bp: ` + my_module_type { + name: "foo", + my_string: select(os(), { + "android": "my_android", + "linux": "my_linux", + default: "my_default", + }), + } + `, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("my_android"), + }, + }, + { name: "Unset value", bp: ` my_module_type { @@ -327,8 +343,10 @@ func TestSelects(t *testing.T) { my_module_type { name: "foo", my_string: select(soong_config_variable("my_namespace", "my_variable"), { + "foo": "bar", default: unset, }) + select(soong_config_variable("my_namespace", "my_variable2"), { + "baz": "qux", default: unset, }) } @@ -341,6 +359,7 @@ func TestSelects(t *testing.T) { my_module_type { name: "foo", my_string: select(soong_config_variable("my_namespace", "my_variable"), { + "foo": "bar", default: unset, }) + select(soong_config_variable("my_namespace", "my_variable2"), { default: "a", @@ -414,6 +433,169 @@ func TestSelects(t *testing.T) { replacing_string_list: &[]string{"b1"}, }, }, + { + name: "Multi-condition string 1", + bp: ` + my_module_type { + name: "foo", + my_string: select(( + soong_config_variable("my_namespace", "my_variable"), + soong_config_variable("my_namespace", "my_variable2"), + ), { + ("a", "b"): "a+b", + ("a", default): "a+default", + (default, default): "default", + }), + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "a", + "my_variable2": "b", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("a+b"), + }, + }, + { + name: "Multi-condition string 2", + bp: ` + my_module_type { + name: "foo", + my_string: select(( + soong_config_variable("my_namespace", "my_variable"), + soong_config_variable("my_namespace", "my_variable2"), + ), { + ("a", "b"): "a+b", + ("a", default): "a+default", + (default, default): "default", + }), + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "a", + "my_variable2": "c", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("a+default"), + }, + }, + { + name: "Multi-condition string 3", + bp: ` + my_module_type { + name: "foo", + my_string: select(( + soong_config_variable("my_namespace", "my_variable"), + soong_config_variable("my_namespace", "my_variable2"), + ), { + ("a", "b"): "a+b", + ("a", default): "a+default", + (default, default): "default", + }), + } + `, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_variable": "c", + "my_variable2": "b", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("default"), + }, + }, + { + name: "Select on boolean", + bp: ` + my_module_type { + name: "foo", + my_string: select(boolean_var_for_testing(), { + true: "t", + false: "f", + }), + } + `, + vendorVars: map[string]map[string]string{ + "boolean_var": { + "for_testing": "true", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("t"), + }, + }, + { + name: "Select on boolean false", + bp: ` + my_module_type { + name: "foo", + my_string: select(boolean_var_for_testing(), { + true: "t", + false: "f", + }), + } + `, + vendorVars: map[string]map[string]string{ + "boolean_var": { + "for_testing": "false", + }, + }, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("f"), + }, + }, + { + name: "Select on boolean undefined", + bp: ` + my_module_type { + name: "foo", + my_string: select(boolean_var_for_testing(), { + true: "t", + false: "f", + }), + } + `, + expectedError: "foo", + }, + { + name: "Select on boolean undefined with default", + bp: ` + my_module_type { + name: "foo", + my_string: select(boolean_var_for_testing(), { + true: "t", + false: "f", + default: "default", + }), + } + `, + provider: selectsTestProvider{ + my_string: proptools.StringPtr("default"), + }, + }, + { + name: "Mismatched condition types", + bp: ` + my_module_type { + name: "foo", + my_string: select(boolean_var_for_testing(), { + "true": "t", + "false": "f", + default: "default", + }), + } + `, + vendorVars: map[string]map[string]string{ + "boolean_var": { + "for_testing": "false", + }, + }, + expectedError: "Expected all branches of a select on condition boolean_var_for_testing\\(\\) to have type bool, found string", + }, } for _, tc := range testCases { diff --git a/android/util.go b/android/util.go index 363b31ce5..698a85650 100644 --- a/android/util.go +++ b/android/util.go @@ -524,22 +524,27 @@ func SplitFileExt(name string) (string, string, string) { return root, suffix, ext } -// ShardPaths takes a Paths, and returns a slice of Paths where each one has at most shardSize paths. -func ShardPaths(paths Paths, shardSize int) []Paths { - if len(paths) == 0 { +func shard[T ~[]E, E any](toShard T, shardSize int) []T { + if len(toShard) == 0 { return nil } - ret := make([]Paths, 0, (len(paths)+shardSize-1)/shardSize) - for len(paths) > shardSize { - ret = append(ret, paths[0:shardSize]) - paths = paths[shardSize:] + + ret := make([]T, 0, (len(toShard)+shardSize-1)/shardSize) + for len(toShard) > shardSize { + ret = append(ret, toShard[0:shardSize]) + toShard = toShard[shardSize:] } - if len(paths) > 0 { - ret = append(ret, paths) + if len(toShard) > 0 { + ret = append(ret, toShard) } return ret } +// ShardPaths takes a Paths, and returns a slice of Paths where each one has at most shardSize paths. +func ShardPaths(paths Paths, shardSize int) []Paths { + return shard(paths, shardSize) +} + // ShardString takes a string and returns a slice of strings where the length of each one is // at most shardSize. func ShardString(s string, shardSize int) []string { @@ -560,18 +565,7 @@ func ShardString(s string, shardSize int) []string { // ShardStrings takes a slice of strings, and returns a slice of slices of strings where each one has at most shardSize // elements. func ShardStrings(s []string, shardSize int) [][]string { - if len(s) == 0 { - return nil - } - ret := make([][]string, 0, (len(s)+shardSize-1)/shardSize) - for len(s) > shardSize { - ret = append(ret, s[0:shardSize]) - s = s[shardSize:] - } - if len(s) > 0 { - ret = append(ret, s) - } - return ret + return shard(s, shardSize) } // CheckDuplicate checks if there are duplicates in given string list. diff --git a/android/variable.go b/android/variable.go index 5a079db50..0040d836d 100644 --- a/android/variable.go +++ b/android/variable.go @@ -201,23 +201,22 @@ type ProductVariables struct { BuildThumbprintFile *string `json:",omitempty"` DisplayBuildNumber *bool `json:",omitempty"` - Platform_display_version_name *string `json:",omitempty"` - Platform_version_name *string `json:",omitempty"` - Platform_sdk_version *int `json:",omitempty"` - Platform_sdk_codename *string `json:",omitempty"` - Platform_sdk_version_or_codename *string `json:",omitempty"` - Platform_sdk_final *bool `json:",omitempty"` - Platform_sdk_extension_version *int `json:",omitempty"` - Platform_base_sdk_extension_version *int `json:",omitempty"` - Platform_version_active_codenames []string `json:",omitempty"` - Platform_version_all_preview_codenames []string `json:",omitempty"` - Platform_systemsdk_versions []string `json:",omitempty"` - Platform_security_patch *string `json:",omitempty"` - Platform_preview_sdk_version *string `json:",omitempty"` - Platform_min_supported_target_sdk_version *string `json:",omitempty"` - Platform_base_os *string `json:",omitempty"` - Platform_version_last_stable *string `json:",omitempty"` - Platform_version_known_codenames *string `json:",omitempty"` + Platform_display_version_name *string `json:",omitempty"` + Platform_version_name *string `json:",omitempty"` + Platform_sdk_version *int `json:",omitempty"` + Platform_sdk_codename *string `json:",omitempty"` + Platform_sdk_version_or_codename *string `json:",omitempty"` + Platform_sdk_final *bool `json:",omitempty"` + Platform_sdk_extension_version *int `json:",omitempty"` + Platform_base_sdk_extension_version *int `json:",omitempty"` + Platform_version_active_codenames []string `json:",omitempty"` + Platform_version_all_preview_codenames []string `json:",omitempty"` + Platform_systemsdk_versions []string `json:",omitempty"` + Platform_security_patch *string `json:",omitempty"` + Platform_preview_sdk_version *string `json:",omitempty"` + Platform_base_os *string `json:",omitempty"` + Platform_version_last_stable *string `json:",omitempty"` + Platform_version_known_codenames *string `json:",omitempty"` DeviceName *string `json:",omitempty"` DeviceProduct *string `json:",omitempty"` diff --git a/bpfix/bpfix/bpfix_test.go b/bpfix/bpfix/bpfix_test.go index 672e852d4..b5b49b1ab 100644 --- a/bpfix/bpfix/bpfix_test.go +++ b/bpfix/bpfix/bpfix_test.go @@ -19,6 +19,7 @@ package bpfix import ( "bytes" "fmt" + "os" "reflect" "strings" "testing" @@ -2215,3 +2216,9 @@ func TestRemoveResourceAndAssetsIfDefault(t *testing.T) { }) } } + +func TestMain(m *testing.M) { + // Skip checking Android.mk path with cleaning "ANDROID_BUILD_TOP" + os.Setenv("ANDROID_BUILD_TOP", "") + os.Exit(m.Run()) +} @@ -1995,6 +1995,20 @@ func (d *Defaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { } func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { + ctx := moduleContextFromAndroidModuleContext(actx, c) + + // If Test_only is set on a module in bp file, respect the setting, otherwise + // see if is a known test module type. + testOnly := c.testModule || c.testLibrary() + if c.sourceProperties.Test_only != nil { + testOnly = Bool(c.sourceProperties.Test_only) + } + // Keep before any early returns. + android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{ + TestOnly: testOnly, + TopLevelTarget: c.testModule, + }) + // Handle the case of a test module split by `test_per_src` mutator. // // The `test_per_src` mutator adds an extra variation named "", depending on all the other @@ -2013,8 +2027,6 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { c.makeLinkType = GetMakeLinkType(actx, c) - ctx := moduleContextFromAndroidModuleContext(actx, c) - deps := c.depsToPaths(ctx) if ctx.Failed() { return @@ -2141,17 +2153,6 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { android.SetProvider(ctx, testing.TestModuleProviderKey, testing.TestModuleProviderData{}) } - // If Test_only is set on a module in bp file, respect the setting, otherwise - // see if is a known test module type. - testOnly := c.testModule || c.testLibrary() - if c.sourceProperties.Test_only != nil { - testOnly = Bool(c.sourceProperties.Test_only) - } - android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{ - TestOnly: testOnly, - TopLevelTarget: c.testModule, - }) - android.SetProvider(ctx, blueprint.SrcsFileProviderKey, blueprint.SrcsFileProviderData{SrcPaths: deps.GeneratedSources.Strings()}) android.CollectDependencyAconfigFiles(ctx, &c.mergedAconfigFiles) diff --git a/cc/cc_test_only_property_test.go b/cc/cc_test_only_property_test.go index 972e86bc5..c14f34ecb 100644 --- a/cc/cc_test_only_property_test.go +++ b/cc/cc_test_only_property_test.go @@ -78,6 +78,38 @@ func TestTestOnlyProvider(t *testing.T) { } } +func TestTestOnlyValueWithTestPerSrcProp(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + prepareForCcTest, + ).RunTestWithBp(t, ` + // These should be test-only + cc_test { name: "cc-test", + gtest: false, + test_per_src: true, + srcs: ["foo_test.cpp"], + test_options: { unit_test: false, }, + } + `) + + // Ensure all variation of test-per-src tests are marked test-only. + ctx.VisitAllModules(func(m blueprint.Module) { + testOnly := false + if provider, ok := android.OtherModuleProvider(ctx.TestContext.OtherModuleProviderAdaptor(), m, android.TestOnlyProviderKey); ok { + if provider.TestOnly { + testOnly = true + } + } + if module, ok := m.(*Module); ok { + if testModule, ok := module.installer.(*testBinary); ok { + if !testOnly && *testModule.Properties.Test_per_src { + t.Errorf("%v is not test-only but should be", m) + } + } + } + }) +} + func TestTestOnlyInTeamsProto(t *testing.T) { t.Parallel() ctx := android.GroupFixturePreparers( diff --git a/cmd/release_config/Android.bp b/cmd/release_config/build_flag/Android.bp index 7f627ffb7..0f10c91cb 100644 --- a/cmd/release_config/Android.bp +++ b/cmd/release_config/build_flag/Android.bp @@ -2,14 +2,14 @@ package { default_applicable_licenses: ["Android-Apache-2.0"], } -bootstrap_go_package { - name: "release-config", - pkgPath: "android/soong/cmd/release_config", +blueprint_go_binary { + name: "build-flag", deps: [ "golang-protobuf-encoding-prototext", "golang-protobuf-reflect-protoreflect", "golang-protobuf-runtime-protoimpl", - "soong-cmd-release-config-proto", + "soong-cmd-release_config-proto", + "soong-cmd-release_config-lib", ], srcs: [ "main.go", @@ -17,14 +17,16 @@ bootstrap_go_package { } bootstrap_go_package { - name: "soong-cmd-release-config-proto", - pkgPath: "android/soong/cmd/release_config/release_config_proto", + name: "soong-cmd-release_config-build_flag", + pkgPath: "android/soong/cmd/release_config/build_flag", deps: [ + "golang-protobuf-encoding-prototext", "golang-protobuf-reflect-protoreflect", "golang-protobuf-runtime-protoimpl", + "soong-cmd-release_config-proto", + "soong-cmd-release_config-lib", ], srcs: [ - "release_config_proto/build_flags_out.pb.go", - "release_config_proto/build_flags_src.pb.go", + "main.go", ], } diff --git a/cmd/release_config/build_flag/main.go b/cmd/release_config/build_flag/main.go new file mode 100644 index 000000000..6f909af86 --- /dev/null +++ b/cmd/release_config/build_flag/main.go @@ -0,0 +1,229 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "strings" + + rc_lib "android/soong/cmd/release_config/release_config_lib" + rc_proto "android/soong/cmd/release_config/release_config_proto" + + "google.golang.org/protobuf/proto" +) + +type Flags struct { + // The path to the top of the workspace. Default: ".". + top string + + // Pathlist of release config map textproto files. + // If not specified, then the value is (if present): + // - build/release/release_config_map.textproto + // - vendor/google_shared/build/release/release_config_map.textproto + // - vendor/google/release/release_config_map.textproto + // + // Additionally, any maps specified in the environment variable + // `PRODUCT_RELEASE_CONFIG_MAPS` are used. + maps rc_lib.StringList + + // Output directory (relative to `top`). + outDir string + + // Which $TARGET_RELEASE(s) should we use. Some commands will only + // accept one value, others also accept `--release --all`. + targetReleases rc_lib.StringList + + // Disable warning messages + quiet bool +} + +type CommandFunc func(*rc_lib.ReleaseConfigs, Flags, string, []string) error + +var commandMap map[string]CommandFunc = map[string]CommandFunc{ + "get": GetCommand, + "set": SetCommand, + "trace": GetCommand, // Also handled by GetCommand +} + +// Find the top of the release config contribution directory. +// Returns the parent of the flag_declarations and flag_values directories. +func GetMapDir(path string) (string, error) { + for p := path; p != "."; p = filepath.Dir(p) { + switch filepath.Base(p) { + case "flag_declarations": + return filepath.Dir(p), nil + case "flag_values": + return filepath.Dir(p), nil + } + } + return "", fmt.Errorf("Could not determine directory from %s", path) +} + +func MarshalFlagValue(config *rc_lib.ReleaseConfig, name string) (ret string, err error) { + fa, ok := config.FlagArtifacts[name] + if !ok { + return "", fmt.Errorf("%s not found in %s", name, config.Name) + } + return rc_lib.MarshalValue(fa.Value), nil +} + +func GetReleaseArgs(configs *rc_lib.ReleaseConfigs, commonFlags Flags) ([]*rc_lib.ReleaseConfig, error) { + var all bool + relFlags := flag.NewFlagSet("set", flag.ExitOnError) + relFlags.BoolVar(&all, "all", false, "Display all flags") + relFlags.Parse(commonFlags.targetReleases) + var ret []*rc_lib.ReleaseConfig + if all { + for _, config := range configs.ReleaseConfigs { + ret = append(ret, config) + } + return ret, nil + } + for _, arg := range relFlags.Args() { + config, err := configs.GetReleaseConfig(arg) + if err != nil { + return nil, err + } + ret = append(ret, config) + } + return ret, nil +} + +func GetCommand(configs *rc_lib.ReleaseConfigs, commonFlags Flags, cmd string, args []string) error { + isTrace := cmd == "trace" + var all bool + getFlags := flag.NewFlagSet("set", flag.ExitOnError) + getFlags.BoolVar(&all, "all", false, "Display all flags") + getFlags.Parse(args) + args = getFlags.Args() + + releaseConfigList, err := GetReleaseArgs(configs, commonFlags) + if err != nil { + return err + } + if isTrace && len(releaseConfigList) > 1 { + return fmt.Errorf("trace command only allows one --release argument. Got: %s", strings.Join(commonFlags.targetReleases, " ")) + } + + if all { + args = []string{} + for _, fa := range configs.FlagArtifacts { + args = append(args, *fa.FlagDeclaration.Name) + } + } + + showName := len(releaseConfigList) > 1 || len(args) > 1 + for _, config := range releaseConfigList { + var configName string + if len(releaseConfigList) > 1 { + configName = fmt.Sprintf("%s.", config.Name) + } + for _, arg := range args { + val, err := MarshalFlagValue(config, arg) + if err != nil { + return err + } + if showName { + fmt.Printf("%s%s=%s\n", configName, arg, val) + } else { + fmt.Printf("%s\n", val) + } + if isTrace { + for _, trace := range config.FlagArtifacts[arg].Traces { + fmt.Printf(" => \"%s\" in %s\n", rc_lib.MarshalValue(trace.Value), *trace.Source) + } + } + } + } + return nil +} + +func SetCommand(configs *rc_lib.ReleaseConfigs, commonFlags Flags, cmd string, args []string) error { + var valueDir string + if len(commonFlags.targetReleases) > 1 { + return fmt.Errorf("set command only allows one --release argument. Got: %s", strings.Join(commonFlags.targetReleases, " ")) + } + targetRelease := commonFlags.targetReleases[0] + + setFlags := flag.NewFlagSet("set", flag.ExitOnError) + setFlags.StringVar(&valueDir, "dir", "", "Directory in which to place the value") + setFlags.Parse(args) + setArgs := setFlags.Args() + if len(setArgs) != 2 { + return fmt.Errorf("set command expected flag and value, got: %s", strings.Join(setArgs, " ")) + } + name := setArgs[0] + value := setArgs[1] + release, err := configs.GetReleaseConfig(targetRelease) + targetRelease = release.Name + if err != nil { + return err + } + flagArtifact, ok := release.FlagArtifacts[name] + if !ok { + return fmt.Errorf("Unknown build flag %s", name) + } + if valueDir == "" { + mapDir, err := GetMapDir(*flagArtifact.Traces[len(flagArtifact.Traces)-1].Source) + if err != nil { + return err + } + valueDir = mapDir + } + + flagValue := &rc_proto.FlagValue{ + Name: proto.String(name), + Value: rc_lib.UnmarshalValue(value), + } + flagPath := filepath.Join(valueDir, "flag_values", targetRelease, fmt.Sprintf("%s.textproto", name)) + return rc_lib.WriteMessage(flagPath, flagValue) +} + +func main() { + var err error + var commonFlags Flags + var configs *rc_lib.ReleaseConfigs + + outEnv := os.Getenv("OUT_DIR") + if outEnv == "" { + outEnv = "out" + } + // Handle the common arguments + flag.StringVar(&commonFlags.top, "top", ".", "path to top of workspace") + flag.BoolVar(&commonFlags.quiet, "quiet", false, "disable warning messages") + flag.Var(&commonFlags.maps, "map", "path to a release_config_map.textproto. may be repeated") + flag.StringVar(&commonFlags.outDir, "out_dir", rc_lib.GetDefaultOutDir(), "basepath for the output. Multiple formats are created") + flag.Var(&commonFlags.targetReleases, "release", "TARGET_RELEASE for this build") + flag.Parse() + + if commonFlags.quiet { + rc_lib.DisableWarnings() + } + + if len(commonFlags.targetReleases) == 0 { + commonFlags.targetReleases = rc_lib.StringList{"trunk_staging"} + } + + if err = os.Chdir(commonFlags.top); err != nil { + panic(err) + } + + // Get the current state of flagging. + relName := commonFlags.targetReleases[0] + if relName == "--all" || relName == "-all" { + // If the users said `--release --all`, grab trunk staging for simplicity. + relName = "trunk_staging" + } + configs, err = rc_lib.ReadReleaseConfigMaps(commonFlags.maps, relName) + if err != nil { + panic(err) + } + + if cmd, ok := commandMap[flag.Arg(0)]; ok { + args := flag.Args() + if err = cmd(configs, commonFlags, args[0], args[1:]); err != nil { + panic(err) + } + } +} diff --git a/cmd/release_config/crunch_flags/Android.bp b/cmd/release_config/crunch_flags/Android.bp new file mode 100644 index 000000000..89c95913d --- /dev/null +++ b/cmd/release_config/crunch_flags/Android.bp @@ -0,0 +1,32 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +blueprint_go_binary { + name: "crunch-flags", + deps: [ + "golang-protobuf-encoding-prototext", + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-release_config-lib", + "soong-cmd-release_config-proto", + ], + srcs: [ + "main.go", + ], +} + +bootstrap_go_package { + name: "soong-cmd-release_config-crunch_flags", + pkgPath: "android/soong/cmd/release_config/crunch_flags", + deps: [ + "golang-protobuf-encoding-prototext", + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-release_config-lib", + "soong-cmd-release_config-proto", + ], + srcs: [ + "main.go", + ], +} diff --git a/cmd/release_config/crunch_flags/main.go b/cmd/release_config/crunch_flags/main.go new file mode 100644 index 000000000..616674bc5 --- /dev/null +++ b/cmd/release_config/crunch_flags/main.go @@ -0,0 +1,359 @@ +package main + +import ( + "flag" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + + rc_lib "android/soong/cmd/release_config/release_config_lib" + rc_proto "android/soong/cmd/release_config/release_config_proto" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/proto" +) + +// When a flag declaration has an initial value that is a string, the default workflow is PREBUILT. +// If the flag name starts with any of prefixes in manualFlagNamePrefixes, it is MANUAL. +var manualFlagNamePrefixes []string = []string{ + "RELEASE_ACONFIG_", + "RELEASE_PLATFORM_", +} + +var defaultFlagNamespace string = "android_UNKNOWN" + +func RenameNext(name string) string { + if name == "next" { + return "ap3a" + } + return name +} + +func WriteFile(path string, message proto.Message) error { + data, err := prototext.MarshalOptions{Multiline: true}.Marshal(message) + if err != nil { + return err + } + + err = os.MkdirAll(filepath.Dir(path), 0775) + if err != nil { + return err + } + return os.WriteFile(path, data, 0644) +} + +func WalkValueFiles(dir string, Func fs.WalkDirFunc) error { + valPath := filepath.Join(dir, "build_config") + if _, err := os.Stat(valPath); err != nil { + fmt.Printf("%s not found, ignoring.\n", valPath) + return nil + } + + return filepath.WalkDir(valPath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if strings.HasSuffix(d.Name(), ".scl") && d.Type().IsRegular() { + return Func(path, d, err) + } + return nil + }) +} + +func ProcessBuildFlags(dir string, namespaceMap map[string]string) error { + var rootAconfigModule string + + path := filepath.Join(dir, "build_flags.scl") + if _, err := os.Stat(path); err != nil { + fmt.Printf("%s not found, ignoring.\n", path) + return nil + } else { + fmt.Printf("Processing %s\n", path) + } + commentRegexp, err := regexp.Compile("^[[:space:]]*#(?<comment>.+)") + if err != nil { + return err + } + declRegexp, err := regexp.Compile("^[[:space:]]*flag.\"(?<name>[A-Z_0-9]+)\",[[:space:]]*(?<container>[_A-Z]*),[[:space:]]*(?<value>(\"[^\"]*\"|[^\",)]*))") + if err != nil { + return err + } + declIn, err := os.ReadFile(path) + if err != nil { + return err + } + lines := strings.Split(string(declIn), "\n") + var description string + for _, line := range lines { + if comment := commentRegexp.FindStringSubmatch(commentRegexp.FindString(line)); comment != nil { + // Description is the text from any contiguous series of lines before a `flag()` call. + description += fmt.Sprintf(" %s", strings.TrimSpace(comment[commentRegexp.SubexpIndex("comment")])) + continue + } + matches := declRegexp.FindStringSubmatch(declRegexp.FindString(line)) + if matches == nil { + // The line is neither a comment nor a `flag()` call. + // Discard any description we have gathered and process the next line. + description = "" + continue + } + declValue := matches[declRegexp.SubexpIndex("value")] + declName := matches[declRegexp.SubexpIndex("name")] + container := rc_proto.Container(rc_proto.Container_value[matches[declRegexp.SubexpIndex("container")]]) + description = strings.TrimSpace(description) + var namespace string + var ok bool + if namespace, ok = namespaceMap[declName]; !ok { + namespace = defaultFlagNamespace + } + flagDeclaration := &rc_proto.FlagDeclaration{ + Name: proto.String(declName), + Namespace: proto.String(namespace), + Description: proto.String(description), + Container: &container, + } + description = "" + // Most build flags are `workflow: PREBUILT`. + workflow := rc_proto.Workflow(rc_proto.Workflow_PREBUILT) + switch { + case declName == "RELEASE_ACONFIG_VALUE_SETS": + rootAconfigModule = declValue[1 : len(declValue)-1] + continue + case strings.HasPrefix(declValue, "\""): + // String values mean that the flag workflow is (most likely) either MANUAL or PREBUILT. + declValue = declValue[1 : len(declValue)-1] + flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{declValue}} + for _, prefix := range manualFlagNamePrefixes { + if strings.HasPrefix(declName, prefix) { + workflow = rc_proto.Workflow(rc_proto.Workflow_MANUAL) + break + } + } + case declValue == "False" || declValue == "True": + // Boolean values are LAUNCH flags. + flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{declValue == "True"}} + workflow = rc_proto.Workflow(rc_proto.Workflow_LAUNCH) + case declValue == "None": + // Use PREBUILT workflow with no initial value. + default: + fmt.Printf("%s: Unexpected value %s=%s\n", path, declName, declValue) + } + flagDeclaration.Workflow = &workflow + if flagDeclaration != nil { + declPath := filepath.Join(dir, "flag_declarations", fmt.Sprintf("%s.textproto", declName)) + err := WriteFile(declPath, flagDeclaration) + if err != nil { + return err + } + } + } + if rootAconfigModule != "" { + rootProto := &rc_proto.ReleaseConfig{ + Name: proto.String("root"), + AconfigValueSets: []string{rootAconfigModule}, + } + return WriteFile(filepath.Join(dir, "release_configs", "root.textproto"), rootProto) + } + return nil +} + +func ProcessBuildConfigs(dir, name string, paths []string, releaseProto *rc_proto.ReleaseConfig) error { + valRegexp, err := regexp.Compile("[[:space:]]+value.\"(?<name>[A-Z_0-9]+)\",[[:space:]]*(?<value>[^,)]*)") + if err != nil { + return err + } + for _, path := range paths { + fmt.Printf("Processing %s\n", path) + valIn, err := os.ReadFile(path) + if err != nil { + fmt.Printf("%s: error: %v\n", path, err) + return err + } + vals := valRegexp.FindAllString(string(valIn), -1) + for _, val := range vals { + matches := valRegexp.FindStringSubmatch(val) + valValue := matches[valRegexp.SubexpIndex("value")] + valName := matches[valRegexp.SubexpIndex("name")] + flagValue := &rc_proto.FlagValue{ + Name: proto.String(valName), + } + switch { + case valName == "RELEASE_ACONFIG_VALUE_SETS": + flagValue = nil + if releaseProto.AconfigValueSets == nil { + releaseProto.AconfigValueSets = []string{} + } + releaseProto.AconfigValueSets = append(releaseProto.AconfigValueSets, valValue[1:len(valValue)-1]) + case strings.HasPrefix(valValue, "\""): + valValue = valValue[1 : len(valValue)-1] + flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{valValue}} + case valValue == "None": + // nothing to do here. + case valValue == "True": + flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{true}} + case valValue == "False": + flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{false}} + default: + fmt.Printf("%s: Unexpected value %s=%s\n", path, valName, valValue) + } + if flagValue != nil { + valPath := filepath.Join(dir, "flag_values", RenameNext(name), fmt.Sprintf("%s.textproto", valName)) + err := WriteFile(valPath, flagValue) + if err != nil { + return err + } + } + } + } + return err +} + +func ProcessReleaseConfigMap(dir string, descriptionMap map[string]string) error { + path := filepath.Join(dir, "release_config_map.mk") + if _, err := os.Stat(path); err != nil { + fmt.Printf("%s not found, ignoring.\n", path) + return nil + } else { + fmt.Printf("Processing %s\n", path) + } + configRegexp, err := regexp.Compile("^..call[[:space:]]+declare-release-config,[[:space:]]+(?<name>[_a-z0-0A-Z]+),[[:space:]]+(?<files>[^,]*)(,[[:space:]]*(?<inherits>.*)|[[:space:]]*)[)]$") + if err != nil { + return err + } + aliasRegexp, err := regexp.Compile("^..call[[:space:]]+alias-release-config,[[:space:]]+(?<name>[_a-z0-9A-Z]+),[[:space:]]+(?<target>[_a-z0-9A-Z]+)") + if err != nil { + return err + } + + mapIn, err := os.ReadFile(path) + if err != nil { + return err + } + cleanDir := strings.TrimLeft(dir, "../") + var defaultContainer rc_proto.Container + switch { + case strings.HasPrefix(cleanDir, "build/") || cleanDir == "vendor/google_shared/build": + defaultContainer = rc_proto.Container(rc_proto.Container_ALL) + case cleanDir == "vendor/google/release": + defaultContainer = rc_proto.Container(rc_proto.Container_ALL) + default: + defaultContainer = rc_proto.Container(rc_proto.Container_VENDOR) + } + releaseConfigMap := &rc_proto.ReleaseConfigMap{DefaultContainer: &defaultContainer} + // If we find a description for the directory, include it. + if description, ok := descriptionMap[cleanDir]; ok { + releaseConfigMap.Description = proto.String(description) + } + lines := strings.Split(string(mapIn), "\n") + for _, line := range lines { + alias := aliasRegexp.FindStringSubmatch(aliasRegexp.FindString(line)) + if alias != nil { + fmt.Printf("processing alias %s\n", line) + name := alias[aliasRegexp.SubexpIndex("name")] + target := alias[aliasRegexp.SubexpIndex("target")] + if target == "next" { + if RenameNext(target) != name { + return fmt.Errorf("Unexpected name for next (%s)", RenameNext(target)) + } + target, name = name, target + } + releaseConfigMap.Aliases = append(releaseConfigMap.Aliases, + &rc_proto.ReleaseAlias{ + Name: proto.String(name), + Target: proto.String(target), + }) + } + config := configRegexp.FindStringSubmatch(configRegexp.FindString(line)) + if config == nil { + continue + } + name := config[configRegexp.SubexpIndex("name")] + releaseConfig := &rc_proto.ReleaseConfig{ + Name: proto.String(RenameNext(name)), + } + configFiles := config[configRegexp.SubexpIndex("files")] + files := strings.Split(strings.ReplaceAll(configFiles, "$(local_dir)", dir+"/"), " ") + configInherits := config[configRegexp.SubexpIndex("inherits")] + if len(configInherits) > 0 { + releaseConfig.Inherits = strings.Split(configInherits, " ") + } + err := ProcessBuildConfigs(dir, name, files, releaseConfig) + if err != nil { + return err + } + + releasePath := filepath.Join(dir, "release_configs", fmt.Sprintf("%s.textproto", RenameNext(name))) + err = WriteFile(releasePath, releaseConfig) + if err != nil { + return err + } + } + return WriteFile(filepath.Join(dir, "release_config_map.textproto"), releaseConfigMap) +} + +func main() { + var err error + var top string + var dirs rc_lib.StringList + var namespacesFile string + var descriptionsFile string + + flag.StringVar(&top, "top", ".", "path to top of workspace") + flag.Var(&dirs, "dir", "directory to process, relative to the top of the workspace") + flag.StringVar(&namespacesFile, "namespaces", "", "location of file with 'flag_name namespace' information") + flag.StringVar(&descriptionsFile, "descriptions", "", "location of file with 'directory description' information") + flag.Parse() + + if err = os.Chdir(top); err != nil { + panic(err) + } + if len(dirs) == 0 { + dirs = rc_lib.StringList{"build/release", "vendor/google_shared/build/release", "vendor/google/release"} + } + + namespaceMap := make(map[string]string) + if namespacesFile != "" { + data, err := os.ReadFile(namespacesFile) + if err != nil { + panic(err) + } + for idx, line := range strings.Split(string(data), "\n") { + fields := strings.Split(line, " ") + if len(fields) > 2 { + panic(fmt.Errorf("line %d: too many fields: %s", idx, line)) + } + namespaceMap[fields[0]] = fields[1] + } + + } + + descriptionMap := make(map[string]string) + descriptionMap["build/release"] = "Published open-source flags and declarations" + if descriptionsFile != "" { + data, err := os.ReadFile(descriptionsFile) + if err != nil { + panic(err) + } + for _, line := range strings.Split(string(data), "\n") { + if strings.TrimSpace(line) != "" { + fields := strings.SplitN(line, " ", 2) + descriptionMap[fields[0]] = fields[1] + } + } + + } + + for _, dir := range dirs { + err = ProcessBuildFlags(dir, namespaceMap) + if err != nil { + panic(err) + } + + err = ProcessReleaseConfigMap(dir, descriptionMap) + if err != nil { + panic(err) + } + } +} diff --git a/cmd/release_config/main.go b/cmd/release_config/main.go deleted file mode 100644 index 3bb6b3dfe..000000000 --- a/cmd/release_config/main.go +++ /dev/null @@ -1,691 +0,0 @@ -// Copyright 2024 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "cmp" - "encoding/json" - "flag" - "fmt" - "io/fs" - "os" - "path/filepath" - "slices" - "strings" - - "android/soong/cmd/release_config/release_config_proto" - - "google.golang.org/protobuf/encoding/prototext" - "google.golang.org/protobuf/proto" -) - -var verboseFlag bool - -type StringList []string - -func (l *StringList) Set(v string) error { - *l = append(*l, v) - return nil -} - -func (l *StringList) String() string { - return fmt.Sprintf("%v", *l) -} - -var releaseConfigMapPaths StringList - -func DumpProtos(outDir string, message proto.Message) error { - basePath := filepath.Join(outDir, "all_release_configs") - writer := func(suffix string, marshal func() ([]byte, error)) error { - data, err := marshal() - if err != nil { - return err - } - return os.WriteFile(fmt.Sprintf("%s.%s", basePath, suffix), data, 0644) - } - err := writer("textproto", func() ([]byte, error) { return prototext.MarshalOptions{Multiline: true}.Marshal(message) }) - if err != nil { - return err - } - - err = writer("pb", func() ([]byte, error) { return proto.Marshal(message) }) - if err != nil { - return err - } - - return writer("json", func() ([]byte, error) { return json.MarshalIndent(message, "", " ") }) -} - -func LoadTextproto(path string, message proto.Message) error { - data, err := os.ReadFile(path) - if err != nil { - return err - } - ret := prototext.Unmarshal(data, message) - if verboseFlag { - debug, _ := prototext.Marshal(message) - fmt.Printf("%s: %s\n", path, debug) - } - return ret -} - -func WalkTextprotoFiles(root string, subdir string, Func fs.WalkDirFunc) error { - path := filepath.Join(root, subdir) - if _, err := os.Stat(path); err != nil { - // Missing subdirs are not an error. - return nil - } - return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if strings.HasSuffix(d.Name(), ".textproto") && d.Type().IsRegular() { - return Func(path, d, err) - } - return nil - }) -} - -type FlagValue struct { - // The path providing this value. - path string - - // Protobuf - proto release_config_proto.FlagValue -} - -func FlagValueFactory(protoPath string) (fv *FlagValue) { - fv = &FlagValue{path: protoPath} - if protoPath != "" { - LoadTextproto(protoPath, &fv.proto) - } - return fv -} - -// One directory's contribution to the a release config. -type ReleaseConfigContribution struct { - // Paths to files providing this config. - path string - - // The index of the config directory where this release config - // contribution was declared. - // Flag values cannot be set in a location with a lower index. - DeclarationIndex int - - // Protobufs relevant to the config. - proto release_config_proto.ReleaseConfig - - FlagValues []*FlagValue -} - -// A single release_config_map.textproto and its associated data. -// Used primarily for debugging. -type ReleaseConfigMap struct { - // The path to this release_config_map file. - path string - - // Data received - proto release_config_proto.ReleaseConfigMap - - ReleaseConfigContributions map[string]*ReleaseConfigContribution - FlagDeclarations []release_config_proto.FlagDeclaration -} - -// A generated release config. -type ReleaseConfig struct { - // the Name of the release config - Name string - - // The index of the config directory where this release config was - // first declared. - // Flag values cannot be set in a location with a lower index. - DeclarationIndex int - - // What contributes to this config. - Contributions []*ReleaseConfigContribution - - // Aliases for this release - OtherNames []string - - // The names of release configs that we inherit - InheritNames []string - - // Unmarshalled flag artifacts - FlagArtifacts FlagArtifacts - - // Generated release config - ReleaseConfigArtifact *release_config_proto.ReleaseConfigArtifact - - // We have begun compiling this release config. - compileInProgress bool -} - -type FlagArtifact struct { - FlagDeclaration *release_config_proto.FlagDeclaration - - // The index of the config directory where this flag was declared. - // Flag values cannot be set in a location with a lower index. - DeclarationIndex int - - Traces []*release_config_proto.Tracepoint - - // Assigned value - Value *release_config_proto.Value -} - -// Key is flag name. -type FlagArtifacts map[string]*FlagArtifact - -type ReleaseConfigDirMap map[string]int - -// The generated release configs. -type ReleaseConfigs struct { - // Ordered list of release config maps processed. - ReleaseConfigMaps []*ReleaseConfigMap - - // Aliases - Aliases map[string]*string - - // Dictionary of flag_name:FlagDeclaration, with no overrides applied. - FlagArtifacts FlagArtifacts - - // Dictionary of name:ReleaseConfig - ReleaseConfigs map[string]*ReleaseConfig - - // Generated release configs - Artifact release_config_proto.ReleaseConfigsArtifact - - // The list of config directories used. - ConfigDirs []string - - // A map from the config directory to its order in the list of config - // directories. - ConfigDirIndexes ReleaseConfigDirMap -} - -func (src *FlagArtifact) Clone() *FlagArtifact { - value := &release_config_proto.Value{} - proto.Merge(value, src.Value) - return &FlagArtifact{ - FlagDeclaration: src.FlagDeclaration, - Traces: src.Traces, - Value: value, - } -} - -func (src FlagArtifacts) Clone() (dst FlagArtifacts) { - if dst == nil { - dst = make(FlagArtifacts) - } - for k, v := range src { - dst[k] = v.Clone() - } - return -} - -func ReleaseConfigFactory(name string, index int) (c *ReleaseConfig) { - return &ReleaseConfig{Name: name, DeclarationIndex: index} -} - -func ReleaseConfigsFactory() (c *ReleaseConfigs) { - return &ReleaseConfigs{ - Aliases: make(map[string]*string), - FlagArtifacts: make(map[string]*FlagArtifact), - ReleaseConfigs: make(map[string]*ReleaseConfig), - ConfigDirs: []string{}, - ConfigDirIndexes: make(ReleaseConfigDirMap), - } -} - -func ReleaseConfigMapFactory(protoPath string) (m *ReleaseConfigMap) { - m = &ReleaseConfigMap{ - path: protoPath, - ReleaseConfigContributions: make(map[string]*ReleaseConfigContribution), - } - if protoPath != "" { - LoadTextproto(protoPath, &m.proto) - } - return m -} - -func FlagDeclarationFactory(protoPath string) (fd *release_config_proto.FlagDeclaration) { - fd = &release_config_proto.FlagDeclaration{} - if protoPath != "" { - LoadTextproto(protoPath, fd) - } - return fd -} - -func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex int) error { - m := ReleaseConfigMapFactory(path) - if m.proto.Origin == nil || *m.proto.Origin == "" { - return fmt.Errorf("Release config map %s lacks origin", path) - } - if m.proto.DefaultContainer == nil { - return fmt.Errorf("Release config map %s lacks default_container", path) - } - dir := filepath.Dir(path) - // Record any aliases, checking for duplicates. - for _, alias := range m.proto.Aliases { - name := *alias.Name - oldTarget, ok := configs.Aliases[name] - if ok { - if *oldTarget != *alias.Target { - return fmt.Errorf("Conflicting alias declarations: %s vs %s", - *oldTarget, *alias.Target) - } - } - configs.Aliases[name] = alias.Target - } - var err error - err = WalkTextprotoFiles(dir, "flag_declarations", func(path string, d fs.DirEntry, err error) error { - flagDeclaration := FlagDeclarationFactory(path) - // Container must be specified. - if flagDeclaration.Container == nil { - flagDeclaration.Container = m.proto.DefaultContainer - } - // TODO: drop flag_declaration.origin from the proto. - if flagDeclaration.Origin == nil { - flagDeclaration.Origin = m.proto.Origin - } - // There is always a default value. - if flagDeclaration.Value == nil { - flagDeclaration.Value = &release_config_proto.Value{Val: &release_config_proto.Value_UnspecifiedValue{true}} - } - m.FlagDeclarations = append(m.FlagDeclarations, *flagDeclaration) - name := *flagDeclaration.Name - if def, ok := configs.FlagArtifacts[name]; !ok { - configs.FlagArtifacts[name] = &FlagArtifact{FlagDeclaration: flagDeclaration, DeclarationIndex: ConfigDirIndex} - } else if !proto.Equal(def.FlagDeclaration, flagDeclaration) { - return fmt.Errorf("Duplicate definition of %s", *flagDeclaration.Name) - } - // Set the initial value in the flag artifact. - configs.FlagArtifacts[name].UpdateValue( - FlagValue{path: path, proto: release_config_proto.FlagValue{ - Name: proto.String(name), Value: flagDeclaration.Value}}) - return nil - }) - if err != nil { - return err - } - - err = WalkTextprotoFiles(dir, "release_configs", func(path string, d fs.DirEntry, err error) error { - releaseConfigContribution := &ReleaseConfigContribution{path: path, DeclarationIndex: ConfigDirIndex} - LoadTextproto(path, &releaseConfigContribution.proto) - name := *releaseConfigContribution.proto.Name - if fmt.Sprintf("%s.textproto", name) != filepath.Base(path) { - return fmt.Errorf("%s incorrectly declares release config %s", path, name) - } - if _, ok := configs.ReleaseConfigs[name]; !ok { - configs.ReleaseConfigs[name] = ReleaseConfigFactory(name, ConfigDirIndex) - } - config := configs.ReleaseConfigs[name] - config.InheritNames = append(config.InheritNames, releaseConfigContribution.proto.Inherits...) - - // Only walk flag_values/{RELEASE} for defined releases. - err2 := WalkTextprotoFiles(dir, filepath.Join("flag_values", name), func(path string, d fs.DirEntry, err error) error { - flagValue := FlagValueFactory(path) - if fmt.Sprintf("%s.textproto", *flagValue.proto.Name) != filepath.Base(path) { - return fmt.Errorf("%s incorrectly sets value for flag %s", path, *flagValue.proto.Name) - } - releaseConfigContribution.FlagValues = append(releaseConfigContribution.FlagValues, flagValue) - return nil - }) - if err2 != nil { - return err2 - } - m.ReleaseConfigContributions[name] = releaseConfigContribution - config.Contributions = append(config.Contributions, releaseConfigContribution) - return nil - }) - if err != nil { - return err - } - configs.ReleaseConfigMaps = append(configs.ReleaseConfigMaps, m) - return nil -} - -func (configs *ReleaseConfigs) GetReleaseConfig(name string) (*ReleaseConfig, error) { - trace := []string{name} - for target, ok := configs.Aliases[name]; ok; target, ok = configs.Aliases[name] { - name = *target - trace = append(trace, name) - } - if config, ok := configs.ReleaseConfigs[name]; ok { - return config, nil - } - return nil, fmt.Errorf("Missing config %s. Trace=%v", name, trace) -} - -func (configs *ReleaseConfigs) DumpMakefile(outDir, targetRelease string) error { - outFile := filepath.Join(outDir, "release_config.mk") - makeVars := make(map[string]string) - config, err := configs.GetReleaseConfig(targetRelease) - if err != nil { - return err - } - // Sort the flags by name first. - names := []string{} - for k, _ := range config.FlagArtifacts { - names = append(names, k) - } - slices.SortFunc(names, func(a, b string) int { - return cmp.Compare(a, b) - }) - partitions := make(map[string][]string) - - vNames := []string{} - addVar := func(name, suffix, value string) { - fullName := fmt.Sprintf("_ALL_RELEASE_FLAGS.%s.%s", name, suffix) - vNames = append(vNames, fullName) - makeVars[fullName] = value - } - - for _, name := range names { - flag := config.FlagArtifacts[name] - decl := flag.FlagDeclaration - - // cName := strings.ToLower(release_config_proto.Container_name[decl.GetContainer()]) - cName := strings.ToLower(decl.Container.String()) - if cName == strings.ToLower(release_config_proto.Container_ALL.String()) { - partitions["product"] = append(partitions["product"], name) - partitions["system"] = append(partitions["system"], name) - partitions["system_ext"] = append(partitions["system_ext"], name) - partitions["vendor"] = append(partitions["vendor"], name) - } else { - partitions[cName] = append(partitions[cName], name) - } - value := MarshalValue(flag.Value) - makeVars[name] = value - addVar(name, "PARTITIONS", cName) - addVar(name, "DEFAULT", MarshalValue(decl.Value)) - addVar(name, "VALUE", value) - addVar(name, "DECLARED_IN", *flag.Traces[0].Source) - addVar(name, "SET_IN", *flag.Traces[len(flag.Traces)-1].Source) - addVar(name, "ORIGIN", *decl.Origin) - } - pNames := []string{} - for k, _ := range partitions { - pNames = append(pNames, k) - } - slices.SortFunc(pNames, func(a, b string) int { - return cmp.Compare(a, b) - }) - - // Now sort the make variables, and output them. - slices.SortFunc(vNames, func(a, b string) int { - return cmp.Compare(a, b) - }) - - // Write the flags as: - // _ALL_RELELASE_FLAGS - // _ALL_RELEASE_FLAGS.PARTITIONS.* - // all _ALL_RELEASE_FLAGS.*, sorted by name - // Final flag values, sorted by name. - data := fmt.Sprintf("_ALL_RELEASE_FLAGS :=$= %s\n", strings.Join(names, " ")) - for _, pName := range pNames { - data += fmt.Sprintf("_ALL_RELEASE_FLAGS.PARTITIONS.%s :=$= %s\n", pName, strings.Join(partitions[pName], " ")) - } - for _, vName := range vNames { - data += fmt.Sprintf("%s :=$= %s\n", vName, makeVars[vName]) - } - data += "\n\n# Values for all build flags\n" - data += fmt.Sprintf("RELEASE_ACONFIG_VALUE_SETS :=$= %s\n", - strings.Join(config.ReleaseConfigArtifact.AconfigValueSets, " ")) - for _, name := range names { - data += fmt.Sprintf("%s :=$= %s\n", name, makeVars[name]) - } - return os.WriteFile(outFile, []byte(data), 0644) -} - -func (configs *ReleaseConfigs) GenerateReleaseConfigs(targetRelease string) error { - otherNames := make(map[string][]string) - for aliasName, aliasTarget := range configs.Aliases { - if _, ok := configs.ReleaseConfigs[aliasName]; ok { - return fmt.Errorf("Alias %s is a declared release config", aliasName) - } - if _, ok := configs.ReleaseConfigs[*aliasTarget]; !ok { - if _, ok2 := configs.Aliases[*aliasTarget]; !ok2 { - return fmt.Errorf("Alias %s points to non-existing config %s", aliasName, *aliasTarget) - } - } - otherNames[*aliasTarget] = append(otherNames[*aliasTarget], aliasName) - } - for name, aliases := range otherNames { - configs.ReleaseConfigs[name].OtherNames = aliases - } - - for _, config := range configs.ReleaseConfigs { - err := config.GenerateReleaseConfig(configs) - if err != nil { - return err - } - } - - releaseConfig, err := configs.GetReleaseConfig(targetRelease) - if err != nil { - return err - } - configs.Artifact = release_config_proto.ReleaseConfigsArtifact{ - ReleaseConfig: releaseConfig.ReleaseConfigArtifact, - OtherReleaseConfigs: func() []*release_config_proto.ReleaseConfigArtifact { - orc := []*release_config_proto.ReleaseConfigArtifact{} - for name, config := range configs.ReleaseConfigs { - if name != releaseConfig.Name { - orc = append(orc, config.ReleaseConfigArtifact) - } - } - return orc - }(), - } - return nil -} - -func MarshalValue(value *release_config_proto.Value) string { - switch val := value.Val.(type) { - case *release_config_proto.Value_UnspecifiedValue: - // Value was never set. - return "" - case *release_config_proto.Value_StringValue: - return val.StringValue - case *release_config_proto.Value_BoolValue: - if val.BoolValue { - return "true" - } - // False ==> empty string - return "" - case *release_config_proto.Value_Obsolete: - return " #OBSOLETE" - default: - // Flagged as error elsewhere, so return empty string here. - return "" - } -} - -func (fa *FlagArtifact) UpdateValue(flagValue FlagValue) error { - name := *flagValue.proto.Name - fa.Traces = append(fa.Traces, &release_config_proto.Tracepoint{Source: proto.String(flagValue.path), Value: flagValue.proto.Value}) - if fa.Value.GetObsolete() { - return fmt.Errorf("Attempting to set obsolete flag %s. Trace=%v", name, fa.Traces) - } - switch val := flagValue.proto.Value.Val.(type) { - case *release_config_proto.Value_StringValue: - fa.Value = &release_config_proto.Value{Val: &release_config_proto.Value_StringValue{val.StringValue}} - case *release_config_proto.Value_BoolValue: - fa.Value = &release_config_proto.Value{Val: &release_config_proto.Value_BoolValue{val.BoolValue}} - case *release_config_proto.Value_Obsolete: - if !val.Obsolete { - return fmt.Errorf("%s: Cannot set obsolete=false. Trace=%v", name, fa.Traces) - } - fa.Value = &release_config_proto.Value{Val: &release_config_proto.Value_Obsolete{true}} - default: - return fmt.Errorf("Invalid type for flag_value: %T. Trace=%v", val, fa.Traces) - } - return nil -} - -func (fa *FlagArtifact) Marshal() (*release_config_proto.FlagArtifact, error) { - return &release_config_proto.FlagArtifact{ - FlagDeclaration: fa.FlagDeclaration, - Value: fa.Value, - Traces: fa.Traces, - }, nil -} - -func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) error { - if config.ReleaseConfigArtifact != nil { - return nil - } - if config.compileInProgress { - return fmt.Errorf("Loop detected for release config %s", config.Name) - } - config.compileInProgress = true - - // Generate any configs we need to inherit. This will detect loops in - // the config. - contributionsToApply := []*ReleaseConfigContribution{} - myInherits := []string{} - myInheritsSet := make(map[string]bool) - for _, inherit := range config.InheritNames { - if _, ok := myInheritsSet[inherit]; ok { - continue - } - myInherits = append(myInherits, inherit) - myInheritsSet[inherit] = true - iConfig, err := configs.GetReleaseConfig(inherit) - if err != nil { - return err - } - iConfig.GenerateReleaseConfig(configs) - contributionsToApply = append(contributionsToApply, iConfig.Contributions...) - } - contributionsToApply = append(contributionsToApply, config.Contributions...) - - myAconfigValueSets := []string{} - myFlags := configs.FlagArtifacts.Clone() - myDirsMap := make(map[int]bool) - for _, contrib := range contributionsToApply { - myAconfigValueSets = append(myAconfigValueSets, contrib.proto.AconfigValueSets...) - myDirsMap[contrib.DeclarationIndex] = true - for _, value := range contrib.FlagValues { - fa, ok := myFlags[*value.proto.Name] - if !ok { - return fmt.Errorf("Setting value for undefined flag %s in %s\n", *value.proto.Name, value.path) - } - myDirsMap[fa.DeclarationIndex] = true - if fa.DeclarationIndex > contrib.DeclarationIndex { - // Setting location is to the left of declaration. - return fmt.Errorf("Setting value for flag %s not allowed in %s\n", *value.proto.Name, value.path) - } - if err := fa.UpdateValue(*value); err != nil { - return err - } - } - } - - directories := []string{} - for idx, confDir := range configs.ConfigDirs { - if _, ok := myDirsMap[idx]; ok { - directories = append(directories, confDir) - } - } - - config.FlagArtifacts = myFlags - config.ReleaseConfigArtifact = &release_config_proto.ReleaseConfigArtifact{ - Name: proto.String(config.Name), - OtherNames: config.OtherNames, - FlagArtifacts: func() []*release_config_proto.FlagArtifact { - ret := []*release_config_proto.FlagArtifact{} - for _, flag := range myFlags { - ret = append(ret, &release_config_proto.FlagArtifact{ - FlagDeclaration: flag.FlagDeclaration, - Traces: flag.Traces, - Value: flag.Value, - }) - } - return ret - }(), - AconfigValueSets: myAconfigValueSets, - Inherits: myInherits, - Directories: directories, - } - - config.compileInProgress = false - return nil -} - -func main() { - var targetRelease string - var outputDir string - - outEnv := os.Getenv("OUT_DIR") - if outEnv == "" { - outEnv = "out" - } - defaultOutputDir := filepath.Join(outEnv, "soong", "release-config") - var defaultMapPaths StringList - defaultLocations := StringList{ - "build/release/release_config_map.textproto", - "vendor/google_shared/build/release/release_config_map.textproto", - "vendor/google/release/release_config_map.textproto", - } - for _, path := range defaultLocations { - if _, err := os.Stat(path); err == nil { - defaultMapPaths = append(defaultMapPaths, path) - } - } - prodMaps := os.Getenv("PRODUCT_RELEASE_CONFIG_MAPS") - if prodMaps != "" { - defaultMapPaths = append(defaultMapPaths, strings.Split(prodMaps, " ")...) - } - - flag.BoolVar(&verboseFlag, "debug", false, "print debugging information") - flag.Var(&releaseConfigMapPaths, "map", "path to a release_config_map.textproto. may be repeated") - flag.StringVar(&targetRelease, "release", "trunk_staging", "TARGET_RELEASE for this build") - flag.StringVar(&outputDir, "out_dir", defaultOutputDir, "basepath for the output. Multiple formats are created") - flag.Parse() - - if len(releaseConfigMapPaths) == 0 { - releaseConfigMapPaths = defaultMapPaths - fmt.Printf("No --map argument provided. Using: --map %s\n", strings.Join(releaseConfigMapPaths, " --map ")) - } - - configs := ReleaseConfigsFactory() - for idx, releaseConfigMapPath := range releaseConfigMapPaths { - // Maintain an ordered list of release config directories. - configDir := filepath.Dir(releaseConfigMapPath) - configs.ConfigDirIndexes[configDir] = idx - configs.ConfigDirs = append(configs.ConfigDirs, configDir) - err := configs.LoadReleaseConfigMap(releaseConfigMapPath, idx) - if err != nil { - panic(err) - } - } - - // Now that we have all of the release config maps, can meld them and generate the artifacts. - err := configs.GenerateReleaseConfigs(targetRelease) - if err != nil { - panic(err) - } - err = os.MkdirAll(outputDir, 0775) - if err != nil { - panic(err) - } - err = configs.DumpMakefile(outputDir, targetRelease) - if err != nil { - panic(err) - } - DumpProtos(outputDir, &configs.Artifact) -} diff --git a/cmd/release_config/release_config/Android.bp b/cmd/release_config/release_config/Android.bp new file mode 100644 index 000000000..3c7382637 --- /dev/null +++ b/cmd/release_config/release_config/Android.bp @@ -0,0 +1,18 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-cmd-release_config-release_config", + pkgPath: "android/soong/cmd/release_config/release_config", + deps: [ + "golang-protobuf-encoding-prototext", + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-release_config-proto", + "soong-cmd-release_config-lib", + ], + srcs: [ + "main.go", + ], +} diff --git a/cmd/release_config/release_config/main.go b/cmd/release_config/release_config/main.go new file mode 100644 index 000000000..a43fdccbe --- /dev/null +++ b/cmd/release_config/release_config/main.go @@ -0,0 +1,96 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + rc_lib "android/soong/cmd/release_config/release_config_lib" +) + +func main() { + var top string + var quiet bool + var releaseConfigMapPaths rc_lib.StringList + var targetRelease string + var outputDir string + var err error + var configs *rc_lib.ReleaseConfigs + var json, pb, textproto bool + var product string + + defaultRelease := os.Getenv("TARGET_RELEASE") + if defaultRelease == "" { + defaultRelease = "trunk_staging" + } + + flag.StringVar(&top, "top", ".", "path to top of workspace") + flag.StringVar(&product, "product", os.Getenv("TARGET_PRODUCT"), "TARGET_PRODUCT for the build") + flag.BoolVar(&quiet, "quiet", false, "disable warning messages") + flag.Var(&releaseConfigMapPaths, "map", "path to a release_config_map.textproto. may be repeated") + flag.StringVar(&targetRelease, "release", defaultRelease, "TARGET_RELEASE for this build") + flag.StringVar(&outputDir, "out_dir", rc_lib.GetDefaultOutDir(), "basepath for the output. Multiple formats are created") + flag.BoolVar(&textproto, "textproto", true, "write artifacts as text protobuf") + flag.BoolVar(&json, "json", true, "write artifacts as json") + flag.BoolVar(&pb, "pb", true, "write artifacts as binary protobuf") + flag.Parse() + + if quiet { + rc_lib.DisableWarnings() + } + + if err = os.Chdir(top); err != nil { + panic(err) + } + configs, err = rc_lib.ReadReleaseConfigMaps(releaseConfigMapPaths, targetRelease) + if err != nil { + panic(err) + } + config, err := configs.GetReleaseConfig(targetRelease) + if err != nil { + panic(err) + } + releaseName := config.Name + err = os.MkdirAll(outputDir, 0775) + if err != nil { + panic(err) + } + makefilePath := filepath.Join(outputDir, fmt.Sprintf("release_config-%s-%s.mk", product, releaseName)) + err = configs.WriteMakefile(makefilePath, targetRelease) + if err != nil { + panic(err) + } + if json { + err = configs.WriteArtifact(outputDir, product, "json") + if err != nil { + panic(err) + } + } + if pb { + err = configs.WriteArtifact(outputDir, product, "pb") + if err != nil { + panic(err) + } + } + if textproto { + err = configs.WriteArtifact(outputDir, product, "textproto") + if err != nil { + panic(err) + } + } +} diff --git a/cmd/release_config/release_config_lib/Android.bp b/cmd/release_config/release_config_lib/Android.bp new file mode 100644 index 000000000..0c67e1106 --- /dev/null +++ b/cmd/release_config/release_config_lib/Android.bp @@ -0,0 +1,36 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-cmd-release_config-lib", + pkgPath: "android/soong/cmd/release_config/release_config_lib", + deps: [ + "golang-protobuf-encoding-prototext", + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-release_config-proto", + ], + srcs: [ + "flag_artifact.go", + "flag_declaration.go", + "flag_value.go", + "release_config.go", + "release_configs.go", + "util.go", + ], +} diff --git a/cmd/release_config/release_config_lib/flag_artifact.go b/cmd/release_config/release_config_lib/flag_artifact.go new file mode 100644 index 000000000..4446655f7 --- /dev/null +++ b/cmd/release_config/release_config_lib/flag_artifact.go @@ -0,0 +1,119 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "fmt" + + rc_proto "android/soong/cmd/release_config/release_config_proto" + + "google.golang.org/protobuf/proto" +) + +// A flag artifact, with its final value and declaration/override history. +type FlagArtifact struct { + // The flag_declaration message. + FlagDeclaration *rc_proto.FlagDeclaration + + // The index of the config directory where this flag was declared. + // Flag values cannot be set in a location with a lower index. + DeclarationIndex int + + // A history of value assignments and overrides. + Traces []*rc_proto.Tracepoint + + // The value of the flag. + Value *rc_proto.Value +} + +// Key is flag name. +type FlagArtifacts map[string]*FlagArtifact + +// Create a clone of the flag artifact. +// +// Returns: +// +// *FlagArtifact: the copy of the artifact. +func (src *FlagArtifact) Clone() *FlagArtifact { + value := &rc_proto.Value{} + proto.Merge(value, src.Value) + return &FlagArtifact{ + FlagDeclaration: src.FlagDeclaration, + Traces: src.Traces, + Value: value, + } +} + +// Clone FlagArtifacts. +// +// Returns: +// +// FlagArtifacts: a copy of the source FlagArtifacts. +func (src FlagArtifacts) Clone() (dst FlagArtifacts) { + if dst == nil { + dst = make(FlagArtifacts) + } + for k, v := range src { + dst[k] = v.Clone() + } + return +} + +// Update the value of a flag. +// +// This appends to flagArtifact.Traces, and updates flagArtifact.Value. +// +// Args: +// +// flagValue FlagValue: the value to assign +// +// Returns: +// +// error: any error encountered +func (fa *FlagArtifact) UpdateValue(flagValue FlagValue) error { + name := *flagValue.proto.Name + fa.Traces = append(fa.Traces, &rc_proto.Tracepoint{Source: proto.String(flagValue.path), Value: flagValue.proto.Value}) + if fa.Value.GetObsolete() { + return fmt.Errorf("Attempting to set obsolete flag %s. Trace=%v", name, fa.Traces) + } + var newValue *rc_proto.Value + switch val := flagValue.proto.Value.Val.(type) { + case *rc_proto.Value_StringValue: + newValue = &rc_proto.Value{Val: &rc_proto.Value_StringValue{val.StringValue}} + case *rc_proto.Value_BoolValue: + newValue = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{val.BoolValue}} + case *rc_proto.Value_Obsolete: + if !val.Obsolete { + return fmt.Errorf("%s: Cannot set obsolete=false. Trace=%v", name, fa.Traces) + } + newValue = &rc_proto.Value{Val: &rc_proto.Value_Obsolete{true}} + default: + return fmt.Errorf("Invalid type for flag_value: %T. Trace=%v", val, fa.Traces) + } + if proto.Equal(newValue, fa.Value) { + warnf("%s: redundant override (set in %s)\n", flagValue.path, *fa.Traces[len(fa.Traces)-2].Source) + } + fa.Value = newValue + return nil +} + +// Marshal the FlagArtifact into a flag_artifact message. +func (fa *FlagArtifact) Marshal() (*rc_proto.FlagArtifact, error) { + return &rc_proto.FlagArtifact{ + FlagDeclaration: fa.FlagDeclaration, + Value: fa.Value, + Traces: fa.Traces, + }, nil +} diff --git a/cmd/release_config/release_config_lib/flag_declaration.go b/cmd/release_config/release_config_lib/flag_declaration.go new file mode 100644 index 000000000..97d4d4c76 --- /dev/null +++ b/cmd/release_config/release_config_lib/flag_declaration.go @@ -0,0 +1,27 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + rc_proto "android/soong/cmd/release_config/release_config_proto" +) + +func FlagDeclarationFactory(protoPath string) (fd *rc_proto.FlagDeclaration) { + fd = &rc_proto.FlagDeclaration{} + if protoPath != "" { + LoadMessage(protoPath, fd) + } + return fd +} diff --git a/cmd/release_config/release_config_lib/flag_value.go b/cmd/release_config/release_config_lib/flag_value.go new file mode 100644 index 000000000..e155e7782 --- /dev/null +++ b/cmd/release_config/release_config_lib/flag_value.go @@ -0,0 +1,73 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "strings" + + rc_proto "android/soong/cmd/release_config/release_config_proto" +) + +type FlagValue struct { + // The path providing this value. + path string + + // Protobuf + proto rc_proto.FlagValue +} + +func FlagValueFactory(protoPath string) (fv *FlagValue) { + fv = &FlagValue{path: protoPath} + if protoPath != "" { + LoadMessage(protoPath, &fv.proto) + } + return fv +} + +func UnmarshalValue(str string) *rc_proto.Value { + ret := &rc_proto.Value{} + switch v := strings.ToLower(str); v { + case "true": + ret = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{true}} + case "false": + ret = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{false}} + case "##obsolete": + ret = &rc_proto.Value{Val: &rc_proto.Value_Obsolete{true}} + default: + ret = &rc_proto.Value{Val: &rc_proto.Value_StringValue{str}} + } + return ret +} + +func MarshalValue(value *rc_proto.Value) string { + switch val := value.Val.(type) { + case *rc_proto.Value_UnspecifiedValue: + // Value was never set. + return "" + case *rc_proto.Value_StringValue: + return val.StringValue + case *rc_proto.Value_BoolValue: + if val.BoolValue { + return "true" + } + // False ==> empty string + return "" + case *rc_proto.Value_Obsolete: + return " #OBSOLETE" + default: + // Flagged as error elsewhere, so return empty string here. + return "" + } +} diff --git a/cmd/release_config/release_config_lib/flag_value_test.go b/cmd/release_config/release_config_lib/flag_value_test.go new file mode 100644 index 000000000..aaa4cafeb --- /dev/null +++ b/cmd/release_config/release_config_lib/flag_value_test.go @@ -0,0 +1,67 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "os" + "path/filepath" + "testing" + + rc_proto "android/soong/cmd/release_config/release_config_proto" + + "google.golang.org/protobuf/proto" +) + +type testCaseFlagValue struct { + protoPath string + name string + data []byte + expected rc_proto.FlagValue + err error +} + +func (tc testCaseFlagValue) assertProtoEqual(t *testing.T, expected, actual proto.Message) { + if !proto.Equal(expected, actual) { + t.Errorf("Expected %q found %q", expected, actual) + } +} + +func TestFlagValue(t *testing.T) { + testCases := []testCaseFlagValue{ + { + name: "stringVal", + protoPath: "build/release/flag_values/test/RELEASE_FOO.textproto", + data: []byte(`name: "RELEASE_FOO" value {string_value: "BAR"}`), + expected: rc_proto.FlagValue{ + Name: proto.String("RELEASE_FOO"), + Value: &rc_proto.Value{Val: &rc_proto.Value_StringValue{"BAR"}}, + }, + err: nil, + }, + } + for _, tc := range testCases { + var err error + tempdir := t.TempDir() + path := filepath.Join(tempdir, tc.protoPath) + if err = os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(path, tc.data, 0644); err != nil { + t.Fatal(err) + } + actual := FlagValueFactory(path) + tc.assertProtoEqual(t, &tc.expected, &actual.proto) + } +} diff --git a/cmd/release_config/release_config_lib/release_config.go b/cmd/release_config/release_config_lib/release_config.go new file mode 100644 index 000000000..c67cee54a --- /dev/null +++ b/cmd/release_config/release_config_lib/release_config.go @@ -0,0 +1,202 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "fmt" + "strings" + + rc_proto "android/soong/cmd/release_config/release_config_proto" + + "google.golang.org/protobuf/proto" +) + +// One directory's contribution to the a release config. +type ReleaseConfigContribution struct { + // Paths to files providing this config. + path string + + // The index of the config directory where this release config + // contribution was declared. + // Flag values cannot be set in a location with a lower index. + DeclarationIndex int + + // Protobufs relevant to the config. + proto rc_proto.ReleaseConfig + + FlagValues []*FlagValue +} + +// A generated release config. +type ReleaseConfig struct { + // the Name of the release config + Name string + + // The index of the config directory where this release config was + // first declared. + // Flag values cannot be set in a location with a lower index. + DeclarationIndex int + + // What contributes to this config. + Contributions []*ReleaseConfigContribution + + // Aliases for this release + OtherNames []string + + // The names of release configs that we inherit + InheritNames []string + + // Unmarshalled flag artifacts + FlagArtifacts FlagArtifacts + + // Generated release config + ReleaseConfigArtifact *rc_proto.ReleaseConfigArtifact + + // We have begun compiling this release config. + compileInProgress bool +} + +func ReleaseConfigFactory(name string, index int) (c *ReleaseConfig) { + return &ReleaseConfig{Name: name, DeclarationIndex: index} +} + +func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) error { + if config.ReleaseConfigArtifact != nil { + return nil + } + if config.compileInProgress { + return fmt.Errorf("Loop detected for release config %s", config.Name) + } + config.compileInProgress = true + isRoot := config.Name == "root" + + // Generate any configs we need to inherit. This will detect loops in + // the config. + contributionsToApply := []*ReleaseConfigContribution{} + myInherits := []string{} + myInheritsSet := make(map[string]bool) + // If there is a "root" release config, it is the start of every inheritance chain. + _, err := configs.GetReleaseConfig("root") + if err == nil && !isRoot { + config.InheritNames = append([]string{"root"}, config.InheritNames...) + } + for _, inherit := range config.InheritNames { + if _, ok := myInheritsSet[inherit]; ok { + continue + } + myInherits = append(myInherits, inherit) + myInheritsSet[inherit] = true + iConfig, err := configs.GetReleaseConfig(inherit) + if err != nil { + return err + } + iConfig.GenerateReleaseConfig(configs) + contributionsToApply = append(contributionsToApply, iConfig.Contributions...) + } + contributionsToApply = append(contributionsToApply, config.Contributions...) + + myAconfigValueSets := []string{} + myAconfigValueSetsMap := map[string]bool{} + myFlags := configs.FlagArtifacts.Clone() + workflowManual := rc_proto.Workflow(rc_proto.Workflow_MANUAL) + container := rc_proto.Container(rc_proto.Container_ALL) + releaseAconfigValueSets := FlagArtifact{ + FlagDeclaration: &rc_proto.FlagDeclaration{ + Name: proto.String("RELEASE_ACONFIG_VALUE_SETS"), + Namespace: proto.String("android_UNKNOWN"), + Description: proto.String("Aconfig value sets assembled by release-config"), + Workflow: &workflowManual, + Container: &container, + Value: &rc_proto.Value{Val: &rc_proto.Value_StringValue{""}}, + }, + DeclarationIndex: -1, + Traces: []*rc_proto.Tracepoint{ + &rc_proto.Tracepoint{ + Source: proto.String("$release-config"), + Value: &rc_proto.Value{Val: &rc_proto.Value_StringValue{""}}, + }, + }, + } + myFlags["RELEASE_ACONFIG_VALUE_SETS"] = &releaseAconfigValueSets + myDirsMap := make(map[int]bool) + for _, contrib := range contributionsToApply { + if len(contrib.proto.AconfigValueSets) > 0 { + contribAconfigValueSets := []string{} + for _, v := range contrib.proto.AconfigValueSets { + if _, ok := myAconfigValueSetsMap[v]; !ok { + contribAconfigValueSets = append(contribAconfigValueSets, v) + myAconfigValueSetsMap[v] = true + } + } + myAconfigValueSets = append(myAconfigValueSets, contribAconfigValueSets...) + releaseAconfigValueSets.Traces = append( + releaseAconfigValueSets.Traces, + &rc_proto.Tracepoint{ + Source: proto.String(contrib.path), + Value: &rc_proto.Value{Val: &rc_proto.Value_StringValue{strings.Join(contribAconfigValueSets, " ")}}, + }) + } + myDirsMap[contrib.DeclarationIndex] = true + for _, value := range contrib.FlagValues { + fa, ok := myFlags[*value.proto.Name] + if !ok { + return fmt.Errorf("Setting value for undefined flag %s in %s\n", *value.proto.Name, value.path) + } + myDirsMap[fa.DeclarationIndex] = true + if fa.DeclarationIndex > contrib.DeclarationIndex { + // Setting location is to the left of declaration. + return fmt.Errorf("Setting value for flag %s not allowed in %s\n", *value.proto.Name, value.path) + } + if isRoot && *fa.FlagDeclaration.Workflow != workflowManual { + // The "root" release config can only contain workflow: MANUAL flags. + return fmt.Errorf("Setting value for non-MANUAL flag %s is not allowed in %s", *value.proto.Name, value.path) + } + if err := fa.UpdateValue(*value); err != nil { + return err + } + } + } + releaseAconfigValueSets.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{strings.Join(myAconfigValueSets, " ")}} + + directories := []string{} + for idx, confDir := range configs.configDirs { + if _, ok := myDirsMap[idx]; ok { + directories = append(directories, confDir) + } + } + + config.FlagArtifacts = myFlags + config.ReleaseConfigArtifact = &rc_proto.ReleaseConfigArtifact{ + Name: proto.String(config.Name), + OtherNames: config.OtherNames, + FlagArtifacts: func() []*rc_proto.FlagArtifact { + ret := []*rc_proto.FlagArtifact{} + for _, flag := range myFlags { + ret = append(ret, &rc_proto.FlagArtifact{ + FlagDeclaration: flag.FlagDeclaration, + Traces: flag.Traces, + Value: flag.Value, + }) + } + return ret + }(), + AconfigValueSets: myAconfigValueSets, + Inherits: myInherits, + Directories: directories, + } + + config.compileInProgress = false + return nil +} diff --git a/cmd/release_config/release_config_lib/release_configs.go b/cmd/release_config/release_config_lib/release_configs.go new file mode 100644 index 000000000..6efdb2f08 --- /dev/null +++ b/cmd/release_config/release_config_lib/release_configs.go @@ -0,0 +1,386 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "cmp" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + + rc_proto "android/soong/cmd/release_config/release_config_proto" + + "google.golang.org/protobuf/proto" +) + +// A single release_config_map.textproto and its associated data. +// Used primarily for debugging. +type ReleaseConfigMap struct { + // The path to this release_config_map file. + path string + + // Data received + proto rc_proto.ReleaseConfigMap + + // Map of name:contribution for release config contributions. + ReleaseConfigContributions map[string]*ReleaseConfigContribution + + // Flags declared this directory's flag_declarations/*.textproto + FlagDeclarations []rc_proto.FlagDeclaration +} + +type ReleaseConfigDirMap map[string]int + +// The generated release configs. +type ReleaseConfigs struct { + // Ordered list of release config maps processed. + ReleaseConfigMaps []*ReleaseConfigMap + + // Aliases + Aliases map[string]*string + + // Dictionary of flag_name:FlagDeclaration, with no overrides applied. + FlagArtifacts FlagArtifacts + + // Generated release configs artifact + Artifact rc_proto.ReleaseConfigsArtifact + + // Dictionary of name:ReleaseConfig + // Use `GetReleaseConfigs(name)` to get a release config. + ReleaseConfigs map[string]*ReleaseConfig + + // Map of directory to *ReleaseConfigMap + releaseConfigMapsMap map[string]*ReleaseConfigMap + + // The list of config directories used. + configDirs []string + + // A map from the config directory to its order in the list of config + // directories. + configDirIndexes ReleaseConfigDirMap +} + +// Write the "all_release_configs" artifact. +// +// The file will be in "{outDir}/all_release_configs-{product}.{format}" +// +// Args: +// +// outDir string: directory path. Will be created if not present. +// product string: TARGET_PRODUCT for the release_configs. +// format string: one of "json", "pb", or "textproto" +// +// Returns: +// +// error: Any error encountered. +func (configs *ReleaseConfigs) WriteArtifact(outDir, product, format string) error { + return WriteMessage( + filepath.Join(outDir, fmt.Sprintf("all_release_configs-%s.%s", product, format)), + &configs.Artifact) +} + +func ReleaseConfigsFactory() (c *ReleaseConfigs) { + return &ReleaseConfigs{ + Aliases: make(map[string]*string), + FlagArtifacts: make(map[string]*FlagArtifact), + ReleaseConfigs: make(map[string]*ReleaseConfig), + releaseConfigMapsMap: make(map[string]*ReleaseConfigMap), + configDirs: []string{}, + configDirIndexes: make(ReleaseConfigDirMap), + } +} + +func ReleaseConfigMapFactory(protoPath string) (m *ReleaseConfigMap) { + m = &ReleaseConfigMap{ + path: protoPath, + ReleaseConfigContributions: make(map[string]*ReleaseConfigContribution), + } + if protoPath != "" { + LoadMessage(protoPath, &m.proto) + } + return m +} + +func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex int) error { + m := ReleaseConfigMapFactory(path) + if m.proto.DefaultContainer == nil { + return fmt.Errorf("Release config map %s lacks default_container", path) + } + dir := filepath.Dir(path) + // Record any aliases, checking for duplicates. + for _, alias := range m.proto.Aliases { + name := *alias.Name + oldTarget, ok := configs.Aliases[name] + if ok { + if *oldTarget != *alias.Target { + return fmt.Errorf("Conflicting alias declarations: %s vs %s", + *oldTarget, *alias.Target) + } + } + configs.Aliases[name] = alias.Target + } + var err error + err = WalkTextprotoFiles(dir, "flag_declarations", func(path string, d fs.DirEntry, err error) error { + flagDeclaration := FlagDeclarationFactory(path) + // Container must be specified. + if flagDeclaration.Container == nil { + flagDeclaration.Container = m.proto.DefaultContainer + } + // TODO: once we have namespaces initialized, we can throw an error here. + if flagDeclaration.Namespace == nil { + flagDeclaration.Namespace = proto.String("android_UNKNOWN") + } + // If the input didn't specify a value, create one (== UnspecifiedValue). + if flagDeclaration.Value == nil { + flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_UnspecifiedValue{false}} + } + m.FlagDeclarations = append(m.FlagDeclarations, *flagDeclaration) + name := *flagDeclaration.Name + if def, ok := configs.FlagArtifacts[name]; !ok { + configs.FlagArtifacts[name] = &FlagArtifact{FlagDeclaration: flagDeclaration, DeclarationIndex: ConfigDirIndex} + } else if !proto.Equal(def.FlagDeclaration, flagDeclaration) { + return fmt.Errorf("Duplicate definition of %s", *flagDeclaration.Name) + } + // Set the initial value in the flag artifact. + configs.FlagArtifacts[name].UpdateValue( + FlagValue{path: path, proto: rc_proto.FlagValue{ + Name: proto.String(name), Value: flagDeclaration.Value}}) + return nil + }) + if err != nil { + return err + } + + err = WalkTextprotoFiles(dir, "release_configs", func(path string, d fs.DirEntry, err error) error { + releaseConfigContribution := &ReleaseConfigContribution{path: path, DeclarationIndex: ConfigDirIndex} + LoadMessage(path, &releaseConfigContribution.proto) + name := *releaseConfigContribution.proto.Name + if fmt.Sprintf("%s.textproto", name) != filepath.Base(path) { + return fmt.Errorf("%s incorrectly declares release config %s", path, name) + } + if _, ok := configs.ReleaseConfigs[name]; !ok { + configs.ReleaseConfigs[name] = ReleaseConfigFactory(name, ConfigDirIndex) + } + config := configs.ReleaseConfigs[name] + config.InheritNames = append(config.InheritNames, releaseConfigContribution.proto.Inherits...) + + // Only walk flag_values/{RELEASE} for defined releases. + err2 := WalkTextprotoFiles(dir, filepath.Join("flag_values", name), func(path string, d fs.DirEntry, err error) error { + flagValue := FlagValueFactory(path) + if fmt.Sprintf("%s.textproto", *flagValue.proto.Name) != filepath.Base(path) { + return fmt.Errorf("%s incorrectly sets value for flag %s", path, *flagValue.proto.Name) + } + releaseConfigContribution.FlagValues = append(releaseConfigContribution.FlagValues, flagValue) + return nil + }) + if err2 != nil { + return err2 + } + m.ReleaseConfigContributions[name] = releaseConfigContribution + config.Contributions = append(config.Contributions, releaseConfigContribution) + return nil + }) + if err != nil { + return err + } + configs.ReleaseConfigMaps = append(configs.ReleaseConfigMaps, m) + configs.releaseConfigMapsMap[dir] = m + return nil +} + +func (configs *ReleaseConfigs) GetReleaseConfig(name string) (*ReleaseConfig, error) { + trace := []string{name} + for target, ok := configs.Aliases[name]; ok; target, ok = configs.Aliases[name] { + name = *target + trace = append(trace, name) + } + if config, ok := configs.ReleaseConfigs[name]; ok { + return config, nil + } + return nil, fmt.Errorf("Missing config %s. Trace=%v", name, trace) +} + +// Write the makefile for this targetRelease. +func (configs *ReleaseConfigs) WriteMakefile(outFile, targetRelease string) error { + makeVars := make(map[string]string) + var allReleaseNames []string + for _, v := range configs.ReleaseConfigs { + allReleaseNames = append(allReleaseNames, v.Name) + allReleaseNames = append(allReleaseNames, v.OtherNames...) + } + config, err := configs.GetReleaseConfig(targetRelease) + if err != nil { + return err + } + + myFlagArtifacts := config.FlagArtifacts.Clone() + // Sort the flags by name first. + names := []string{} + for k, _ := range myFlagArtifacts { + names = append(names, k) + } + slices.SortFunc(names, func(a, b string) int { + return cmp.Compare(a, b) + }) + partitions := make(map[string][]string) + + vNames := []string{} + addVar := func(name, suffix, value string) { + fullName := fmt.Sprintf("_ALL_RELEASE_FLAGS.%s.%s", name, suffix) + vNames = append(vNames, fullName) + makeVars[fullName] = value + } + + for _, name := range names { + flag := myFlagArtifacts[name] + decl := flag.FlagDeclaration + + // cName := strings.ToLower(rc_proto.Container_name[decl.GetContainer()]) + cName := strings.ToLower(decl.Container.String()) + if cName == strings.ToLower(rc_proto.Container_ALL.String()) { + partitions["product"] = append(partitions["product"], name) + partitions["system"] = append(partitions["system"], name) + partitions["system_ext"] = append(partitions["system_ext"], name) + partitions["vendor"] = append(partitions["vendor"], name) + } else { + partitions[cName] = append(partitions[cName], name) + } + value := MarshalValue(flag.Value) + makeVars[name] = value + addVar(name, "PARTITIONS", cName) + addVar(name, "DEFAULT", MarshalValue(decl.Value)) + addVar(name, "VALUE", value) + addVar(name, "DECLARED_IN", *flag.Traces[0].Source) + addVar(name, "SET_IN", *flag.Traces[len(flag.Traces)-1].Source) + addVar(name, "NAMESPACE", *decl.Namespace) + } + pNames := []string{} + for k, _ := range partitions { + pNames = append(pNames, k) + } + slices.SortFunc(pNames, func(a, b string) int { + return cmp.Compare(a, b) + }) + + // Now sort the make variables, and output them. + slices.SortFunc(vNames, func(a, b string) int { + return cmp.Compare(a, b) + }) + + // Write the flags as: + // _ALL_RELELASE_FLAGS + // _ALL_RELEASE_FLAGS.PARTITIONS.* + // all _ALL_RELEASE_FLAGS.*, sorted by name + // Final flag values, sorted by name. + data := fmt.Sprintf("# TARGET_RELEASE=%s\n", config.Name) + if targetRelease != config.Name { + data += fmt.Sprintf("# User specified TARGET_RELEASE=%s\n", targetRelease) + } + // The variable _all_release_configs will get deleted during processing, so do not mark it read-only. + data += fmt.Sprintf("_all_release_configs := %s\n", strings.Join(allReleaseNames, " ")) + data += fmt.Sprintf("_ALL_RELEASE_FLAGS :=$= %s\n", strings.Join(names, " ")) + for _, pName := range pNames { + data += fmt.Sprintf("_ALL_RELEASE_FLAGS.PARTITIONS.%s :=$= %s\n", pName, strings.Join(partitions[pName], " ")) + } + for _, vName := range vNames { + data += fmt.Sprintf("%s :=$= %s\n", vName, makeVars[vName]) + } + data += "\n\n# Values for all build flags\n" + for _, name := range names { + data += fmt.Sprintf("%s :=$= %s\n", name, makeVars[name]) + } + return os.WriteFile(outFile, []byte(data), 0644) +} + +func (configs *ReleaseConfigs) GenerateReleaseConfigs(targetRelease string) error { + otherNames := make(map[string][]string) + for aliasName, aliasTarget := range configs.Aliases { + if _, ok := configs.ReleaseConfigs[aliasName]; ok { + return fmt.Errorf("Alias %s is a declared release config", aliasName) + } + if _, ok := configs.ReleaseConfigs[*aliasTarget]; !ok { + if _, ok2 := configs.Aliases[*aliasTarget]; !ok2 { + return fmt.Errorf("Alias %s points to non-existing config %s", aliasName, *aliasTarget) + } + } + otherNames[*aliasTarget] = append(otherNames[*aliasTarget], aliasName) + } + for name, aliases := range otherNames { + configs.ReleaseConfigs[name].OtherNames = aliases + } + + for _, config := range configs.ReleaseConfigs { + err := config.GenerateReleaseConfig(configs) + if err != nil { + return err + } + } + + releaseConfig, err := configs.GetReleaseConfig(targetRelease) + if err != nil { + return err + } + configs.Artifact = rc_proto.ReleaseConfigsArtifact{ + ReleaseConfig: releaseConfig.ReleaseConfigArtifact, + OtherReleaseConfigs: func() []*rc_proto.ReleaseConfigArtifact { + orc := []*rc_proto.ReleaseConfigArtifact{} + for name, config := range configs.ReleaseConfigs { + if name != releaseConfig.Name { + orc = append(orc, config.ReleaseConfigArtifact) + } + } + return orc + }(), + ReleaseConfigMapsMap: func() map[string]*rc_proto.ReleaseConfigMap { + ret := make(map[string]*rc_proto.ReleaseConfigMap) + for k, v := range configs.releaseConfigMapsMap { + ret[k] = &v.proto + } + return ret + }(), + } + return nil +} + +func ReadReleaseConfigMaps(releaseConfigMapPaths StringList, targetRelease string) (*ReleaseConfigs, error) { + var err error + + if len(releaseConfigMapPaths) == 0 { + releaseConfigMapPaths = GetDefaultMapPaths() + if len(releaseConfigMapPaths) == 0 { + return nil, fmt.Errorf("No maps found") + } + fmt.Printf("No --map argument provided. Using: --map %s\n", strings.Join(releaseConfigMapPaths, " --map ")) + } + + configs := ReleaseConfigsFactory() + for idx, releaseConfigMapPath := range releaseConfigMapPaths { + // Maintain an ordered list of release config directories. + configDir := filepath.Dir(releaseConfigMapPath) + configs.configDirIndexes[configDir] = idx + configs.configDirs = append(configs.configDirs, configDir) + err = configs.LoadReleaseConfigMap(releaseConfigMapPath, idx) + if err != nil { + return nil, err + } + } + + // Now that we have all of the release config maps, can meld them and generate the artifacts. + err = configs.GenerateReleaseConfigs(targetRelease) + return configs, err +} diff --git a/cmd/release_config/release_config_lib/util.go b/cmd/release_config/release_config_lib/util.go new file mode 100644 index 000000000..86940da68 --- /dev/null +++ b/cmd/release_config/release_config_lib/util.go @@ -0,0 +1,158 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package release_config_lib + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/proto" +) + +var disableWarnings bool + +type StringList []string + +func (l *StringList) Set(v string) error { + *l = append(*l, v) + return nil +} + +func (l *StringList) String() string { + return fmt.Sprintf("%v", *l) +} + +// Write a marshalled message to a file. +// +// Marshal the message based on the extension of the path we are writing it to. +// +// Args: +// +// path string: the path of the file to write to. Directories are not created. +// Supported extensions are: ".json", ".pb", and ".textproto". +// message proto.Message: the message to write. +// +// Returns: +// +// error: any error encountered. +func WriteMessage(path string, message proto.Message) (err error) { + var data []byte + switch filepath.Ext(path) { + case ".json": + data, err = json.MarshalIndent(message, "", " ") + case ".pb": + data, err = proto.Marshal(message) + case ".textproto": + data, err = prototext.MarshalOptions{Multiline: true}.Marshal(message) + default: + return fmt.Errorf("Unknown message format for %s", path) + } + if err != nil { + return err + } + return os.WriteFile(path, data, 0644) +} + +// Read a message from a file. +// +// The message is unmarshalled based on the extension of the file read. +// +// Args: +// +// path string: the path of the file to read. +// message proto.Message: the message to unmarshal the message into. +// +// Returns: +// +// error: any error encountered. +func LoadMessage(path string, message proto.Message) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + switch filepath.Ext(path) { + case ".json": + return json.Unmarshal(data, message) + case ".pb": + return proto.Unmarshal(data, message) + case ".textproto": + return prototext.Unmarshal(data, message) + } + return fmt.Errorf("Unknown message format for %s", path) +} + +// Call Func for any textproto files found in {root}/{subdir}. +func WalkTextprotoFiles(root string, subdir string, Func fs.WalkDirFunc) error { + path := filepath.Join(root, subdir) + if _, err := os.Stat(path); err != nil { + // Missing subdirs are not an error. + return nil + } + return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if strings.HasSuffix(d.Name(), ".textproto") && d.Type().IsRegular() { + return Func(path, d, err) + } + return nil + }) +} + +// Turn off all warning output +func DisableWarnings() { + disableWarnings = true +} + +func warnf(format string, args ...any) (n int, err error) { + if !disableWarnings { + return fmt.Printf(format, args...) + } + return 0, nil +} + +// Returns the default value for release config artifacts. +func GetDefaultOutDir() string { + outEnv := os.Getenv("OUT_DIR") + if outEnv == "" { + outEnv = "out" + } + return filepath.Join(outEnv, "soong", "release-config") +} + +// Return the default list of map files to use. +func GetDefaultMapPaths() StringList { + var defaultMapPaths StringList + defaultLocations := StringList{ + "build/release/release_config_map.textproto", + "vendor/google_shared/build/release/release_config_map.textproto", + "vendor/google/release/release_config_map.textproto", + } + for _, path := range defaultLocations { + if _, err := os.Stat(path); err == nil { + defaultMapPaths = append(defaultMapPaths, path) + } + } + prodMaps := os.Getenv("PRODUCT_RELEASE_CONFIG_MAPS") + if prodMaps != "" { + defaultMapPaths = append(defaultMapPaths, strings.Split(prodMaps, " ")...) + } + return defaultMapPaths +} diff --git a/cmd/release_config/release_config_proto/Android.bp b/cmd/release_config/release_config_proto/Android.bp index a8660c753..8c47f2ac0 100644 --- a/cmd/release_config/release_config_proto/Android.bp +++ b/cmd/release_config/release_config_proto/Android.bp @@ -17,8 +17,8 @@ package { } bootstrap_go_package { - name: "soong-release_config_proto", - pkgPath: "android/soong/release_config/release_config_proto", + name: "soong-cmd-release_config-proto", + pkgPath: "android/soong/cmd/release_config/release_config_proto", deps: [ "golang-protobuf-reflect-protoreflect", "golang-protobuf-runtime-protoimpl", diff --git a/cmd/release_config/release_config_proto/build_flags_out.pb.go b/cmd/release_config/release_config_proto/build_flags_out.pb.go index adc1ea4bd..0372d633b 100644 --- a/cmd/release_config/release_config_proto/build_flags_out.pb.go +++ b/cmd/release_config/release_config_proto/build_flags_out.pb.go @@ -259,6 +259,8 @@ type ReleaseConfigsArtifact struct { ReleaseConfig *ReleaseConfigArtifact `protobuf:"bytes,1,opt,name=release_config,json=releaseConfig" json:"release_config,omitempty"` // All other release configs defined for this TARGET_PRODUCT. OtherReleaseConfigs []*ReleaseConfigArtifact `protobuf:"bytes,2,rep,name=other_release_configs,json=otherReleaseConfigs" json:"other_release_configs,omitempty"` + // Map of release_config_artifact.directories to release_config_map message. + ReleaseConfigMapsMap map[string]*ReleaseConfigMap `protobuf:"bytes,3,rep,name=release_config_maps_map,json=releaseConfigMapsMap" json:"release_config_maps_map,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` } func (x *ReleaseConfigsArtifact) Reset() { @@ -307,6 +309,13 @@ func (x *ReleaseConfigsArtifact) GetOtherReleaseConfigs() []*ReleaseConfigArtifa return nil } +func (x *ReleaseConfigsArtifact) GetReleaseConfigMapsMap() map[string]*ReleaseConfigMap { + if x != nil { + return x.ReleaseConfigMapsMap + } + return nil +} + var File_build_flags_out_proto protoreflect.FileDescriptor var file_build_flags_out_proto_rawDesc = []byte{ @@ -352,7 +361,7 @@ var file_build_flags_out_proto_rawDesc = []byte{ 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x64, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x22, 0xe3, 0x01, 0x0a, 0x18, 0x72, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x22, 0xe8, 0x03, 0x0a, 0x18, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x5c, 0x0a, 0x0e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, @@ -367,10 +376,26 @@ var file_build_flags_out_proto_rawDesc = []byte{ 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x13, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, - 0x42, 0x33, 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, - 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x87, 0x01, 0x0a, 0x17, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, 0x70, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x73, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x73, 0x4d, 0x61, 0x70, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x73, 0x4d, 0x61, 0x70, 0x1a, 0x79, 0x0a, 0x19, 0x52, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x73, 0x4d, + 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x46, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, + 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, 0x70, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x33, 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, + 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -385,28 +410,32 @@ func file_build_flags_out_proto_rawDescGZIP() []byte { return file_build_flags_out_proto_rawDescData } -var file_build_flags_out_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_build_flags_out_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_build_flags_out_proto_goTypes = []interface{}{ (*Tracepoint)(nil), // 0: android.release_config_proto.tracepoint (*FlagArtifact)(nil), // 1: android.release_config_proto.flag_artifact (*ReleaseConfigArtifact)(nil), // 2: android.release_config_proto.release_config_artifact (*ReleaseConfigsArtifact)(nil), // 3: android.release_config_proto.release_configs_artifact - (*Value)(nil), // 4: android.release_config_proto.value - (*FlagDeclaration)(nil), // 5: android.release_config_proto.flag_declaration + nil, // 4: android.release_config_proto.release_configs_artifact.ReleaseConfigMapsMapEntry + (*Value)(nil), // 5: android.release_config_proto.value + (*FlagDeclaration)(nil), // 6: android.release_config_proto.flag_declaration + (*ReleaseConfigMap)(nil), // 7: android.release_config_proto.release_config_map } var file_build_flags_out_proto_depIdxs = []int32{ - 4, // 0: android.release_config_proto.tracepoint.value:type_name -> android.release_config_proto.value - 5, // 1: android.release_config_proto.flag_artifact.flag_declaration:type_name -> android.release_config_proto.flag_declaration - 4, // 2: android.release_config_proto.flag_artifact.value:type_name -> android.release_config_proto.value + 5, // 0: android.release_config_proto.tracepoint.value:type_name -> android.release_config_proto.value + 6, // 1: android.release_config_proto.flag_artifact.flag_declaration:type_name -> android.release_config_proto.flag_declaration + 5, // 2: android.release_config_proto.flag_artifact.value:type_name -> android.release_config_proto.value 0, // 3: android.release_config_proto.flag_artifact.traces:type_name -> android.release_config_proto.tracepoint 1, // 4: android.release_config_proto.release_config_artifact.flag_artifacts:type_name -> android.release_config_proto.flag_artifact 2, // 5: android.release_config_proto.release_configs_artifact.release_config:type_name -> android.release_config_proto.release_config_artifact 2, // 6: android.release_config_proto.release_configs_artifact.other_release_configs:type_name -> android.release_config_proto.release_config_artifact - 7, // [7:7] is the sub-list for method output_type - 7, // [7:7] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 4, // 7: android.release_config_proto.release_configs_artifact.release_config_maps_map:type_name -> android.release_config_proto.release_configs_artifact.ReleaseConfigMapsMapEntry + 7, // 8: android.release_config_proto.release_configs_artifact.ReleaseConfigMapsMapEntry.value:type_name -> android.release_config_proto.release_config_map + 9, // [9:9] is the sub-list for method output_type + 9, // [9:9] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_build_flags_out_proto_init() } @@ -471,7 +500,7 @@ func file_build_flags_out_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_build_flags_out_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/cmd/release_config/release_config_proto/build_flags_out.proto b/cmd/release_config/release_config_proto/build_flags_out.proto index fd8487bd4..05e770f3e 100644 --- a/cmd/release_config/release_config_proto/build_flags_out.proto +++ b/cmd/release_config/release_config_proto/build_flags_out.proto @@ -82,5 +82,8 @@ message release_configs_artifact { // All other release configs defined for this TARGET_PRODUCT. repeated release_config_artifact other_release_configs = 2; + + // Map of release_config_artifact.directories to release_config_map message. + map<string, release_config_map> release_config_maps_map = 3; } diff --git a/cmd/release_config/release_config_proto/build_flags_src.pb.go b/cmd/release_config/release_config_proto/build_flags_src.pb.go index 0f2c30b76..d0c924dae 100644 --- a/cmd/release_config/release_config_proto/build_flags_src.pb.go +++ b/cmd/release_config/release_config_proto/build_flags_src.pb.go @@ -287,6 +287,9 @@ type FlagDeclaration struct { // The name of the flag. // See # name for format detail Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // Namespace the flag belongs to (required) + // See # namespace for format detail + Namespace *string `protobuf:"bytes,2,opt,name=namespace" json:"namespace,omitempty"` // Text description of the flag's purpose. Description *string `protobuf:"bytes,3,opt,name=description" json:"description,omitempty"` // Value for the flag @@ -296,9 +299,6 @@ type FlagDeclaration struct { // The container for this flag. This overrides any default container given // in the release_config_map message. Container *Container `protobuf:"varint,206,opt,name=container,enum=android.release_config_proto.Container" json:"container,omitempty"` - // Temporarily allow origin at the flag declaration level while we - // move flags to their own locations. - Origin *string `protobuf:"bytes,208,opt,name=origin" json:"origin,omitempty"` } func (x *FlagDeclaration) Reset() { @@ -340,6 +340,13 @@ func (x *FlagDeclaration) GetName() string { return "" } +func (x *FlagDeclaration) GetNamespace() string { + if x != nil && x.Namespace != nil { + return *x.Namespace + } + return "" +} + func (x *FlagDeclaration) GetDescription() string { if x != nil && x.Description != nil { return *x.Description @@ -368,13 +375,6 @@ func (x *FlagDeclaration) GetContainer() Container { return Container_UNSPECIFIED_container } -func (x *FlagDeclaration) GetOrigin() string { - if x != nil && x.Origin != nil { - return *x.Origin - } - return "" -} - type FlagValue struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -568,8 +568,8 @@ type ReleaseConfigMap struct { // Any aliases. Aliases []*ReleaseAlias `protobuf:"bytes,1,rep,name=aliases" json:"aliases,omitempty"` - // The origin for flags declared here. - Origin *string `protobuf:"bytes,2,opt,name=origin" json:"origin,omitempty"` + // Description of this map and its intended use. + Description *string `protobuf:"bytes,2,opt,name=description" json:"description,omitempty"` // The default container for flags declared here. DefaultContainer *Container `protobuf:"varint,3,opt,name=default_container,json=defaultContainer,enum=android.release_config_proto.Container" json:"default_container,omitempty"` } @@ -613,9 +613,9 @@ func (x *ReleaseConfigMap) GetAliases() []*ReleaseAlias { return nil } -func (x *ReleaseConfigMap) GetOrigin() string { - if x != nil && x.Origin != nil { - return *x.Origin +func (x *ReleaseConfigMap) GetDescription() string { + if x != nil && x.Description != nil { + return *x.Description } return "" } @@ -643,71 +643,72 @@ var file_build_flags_src_proto_rawDesc = []byte{ 0x6c, 0x75, 0x65, 0x18, 0xca, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x08, 0x6f, 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x18, 0xcb, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x08, 0x6f, 0x62, - 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x05, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x22, 0xb8, 0x02, + 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x05, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x22, 0xbd, 0x02, 0x0a, 0x10, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x64, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, - 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x18, 0xcd, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, - 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, - 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x46, 0x0a, 0x09, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0xce, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, - 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x12, 0x17, 0x0a, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x18, 0xd0, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, - 0x4a, 0x06, 0x08, 0xcf, 0x01, 0x10, 0xd0, 0x01, 0x22, 0x5c, 0x0a, 0x0a, 0x66, 0x6c, 0x61, 0x67, - 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3a, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x6e, 0x64, - 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x6e, 0x0a, 0x0e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, - 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, - 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, - 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x53, 0x65, 0x74, 0x73, 0x22, 0x3b, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, - 0x65, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x22, 0xc9, 0x01, 0x0a, 0x12, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, 0x70, 0x12, 0x45, 0x0a, 0x07, 0x61, 0x6c, - 0x69, 0x61, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x61, 0x6e, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, + 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x18, 0xcd, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x08, 0x77, + 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x46, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x18, 0xce, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x07, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x12, 0x54, 0x0a, 0x11, 0x64, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, - 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x10, 0x64, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x2a, - 0x4a, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x18, 0x0a, 0x14, 0x55, - 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x55, 0x4e, 0x43, 0x48, 0x10, - 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x52, 0x45, 0x42, 0x55, 0x49, 0x4c, 0x54, 0x10, 0x02, 0x12, - 0x0a, 0x0a, 0x06, 0x4d, 0x41, 0x4e, 0x55, 0x41, 0x4c, 0x10, 0x03, 0x2a, 0x64, 0x0a, 0x09, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x15, 0x55, 0x4e, 0x53, 0x50, - 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, - 0x50, 0x52, 0x4f, 0x44, 0x55, 0x43, 0x54, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x59, 0x53, - 0x54, 0x45, 0x4d, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, - 0x45, 0x58, 0x54, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x56, 0x45, 0x4e, 0x44, 0x4f, 0x52, 0x10, - 0x05, 0x42, 0x33, 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, - 0x6e, 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4a, + 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x06, 0x08, 0xcf, 0x01, 0x10, 0xd0, 0x01, 0x22, 0x5c, 0x0a, + 0x0a, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x6e, 0x0a, 0x0e, 0x72, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x12, 0x2c, 0x0a, + 0x12, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x73, + 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x53, 0x65, 0x74, 0x73, 0x22, 0x3b, 0x0a, 0x0d, 0x72, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xd3, 0x01, 0x0a, 0x12, 0x72, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6d, 0x61, 0x70, 0x12, + 0x45, 0x0a, 0x07, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x2b, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x07, 0x61, + 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x54, 0x0a, 0x11, 0x64, 0x65, 0x66, 0x61, + 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x10, 0x64, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x2a, 0x4a, + 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x18, 0x0a, 0x14, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x55, 0x4e, 0x43, 0x48, 0x10, 0x01, + 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x52, 0x45, 0x42, 0x55, 0x49, 0x4c, 0x54, 0x10, 0x02, 0x12, 0x0a, + 0x0a, 0x06, 0x4d, 0x41, 0x4e, 0x55, 0x41, 0x4c, 0x10, 0x03, 0x2a, 0x64, 0x0a, 0x09, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x15, 0x55, 0x4e, 0x53, 0x50, 0x45, + 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x50, + 0x52, 0x4f, 0x44, 0x55, 0x43, 0x54, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x59, 0x53, 0x54, + 0x45, 0x4d, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, 0x45, + 0x58, 0x54, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x56, 0x45, 0x4e, 0x44, 0x4f, 0x52, 0x10, 0x05, + 0x42, 0x33, 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( diff --git a/cmd/release_config/release_config_proto/build_flags_src.proto b/cmd/release_config/release_config_proto/build_flags_src.proto index 0662716d5..c077f5cd8 100644 --- a/cmd/release_config/release_config_proto/build_flags_src.proto +++ b/cmd/release_config/release_config_proto/build_flags_src.proto @@ -26,6 +26,11 @@ option go_package = "android/soong/release_config/release_config_proto"; // RELEASE_MY_PACKAGE_FLAG is a valid name, while MY_PACKAGE_FLAG, and // RELEASE_MY_PACKAGE__FLAG are invalid. // +// # namespace: namespace the flag belongs to +// +// format: a lowercase string in snake_case format, no consecutive underscores, and no leading +// digit. For example android_bar_system +// // # package: package to which the flag belongs // // format: lowercase strings in snake_case format, delimited by dots, no @@ -77,6 +82,10 @@ message flag_declaration { // See # name for format detail optional string name = 1; + // Namespace the flag belongs to (required) + // See # namespace for format detail + optional string namespace = 2; + // Text description of the flag's purpose. optional string description = 3; @@ -96,12 +105,6 @@ message flag_declaration { // The package associated with this flag. // (when Gantry is ready for it) optional string package = 207; reserved 207; - - // Temporarily allow origin at the flag declaration level while we - // move flags to their own locations. - optional string origin = 208; - - // TODO: do we want to include "first used in" (= ap2a)? } message flag_value { @@ -141,8 +144,8 @@ message release_config_map { // Any aliases. repeated release_alias aliases = 1; - // The origin for flags declared here. - optional string origin = 2; + // Description of this map and its intended use. + optional string description = 2; // The default container for flags declared here. optional container default_container = 3; diff --git a/dexpreopt/class_loader_context.go b/dexpreopt/class_loader_context.go index 57c7ae851..af1d33da6 100644 --- a/dexpreopt/class_loader_context.go +++ b/dexpreopt/class_loader_context.go @@ -323,6 +323,7 @@ func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathCont } else if clc.Host == hostPath && clc.Device == devicePath { // Ok, the same library with the same paths. Don't re-add it, but don't raise an error // either, as the same library may be reachable via different transitional dependencies. + clc.Optional = clc.Optional && optional return nil } else { // Fail, as someone is trying to add the same library with different paths. This likely diff --git a/dexpreopt/dexpreopt.go b/dexpreopt/dexpreopt.go index 04bc61dad..93351f1fc 100644 --- a/dexpreopt/dexpreopt.go +++ b/dexpreopt/dexpreopt.go @@ -52,7 +52,7 @@ var DexpreoptRunningInSoong = false // GenerateDexpreoptRule generates a set of commands that will preopt a module based on a GlobalConfig and a // ModuleConfig. The produced files and their install locations will be available through rule.Installs(). func GenerateDexpreoptRule(ctx android.BuilderContext, globalSoong *GlobalSoongConfig, - global *GlobalConfig, module *ModuleConfig, productPackages android.Path) ( + global *GlobalConfig, module *ModuleConfig, productPackages android.Path, copyApexSystemServerJarDex bool) ( rule *android.RuleBuilder, err error) { defer func() { @@ -94,7 +94,7 @@ func GenerateDexpreoptRule(ctx android.BuilderContext, globalSoong *GlobalSoongC for archIdx, _ := range module.Archs { dexpreoptCommand(ctx, globalSoong, global, module, rule, archIdx, profile, appImage, - generateDM, productPackages) + generateDM, productPackages, copyApexSystemServerJarDex) } } } @@ -231,7 +231,7 @@ func ToOdexPath(path string, arch android.ArchType) string { func dexpreoptCommand(ctx android.BuilderContext, globalSoong *GlobalSoongConfig, global *GlobalConfig, module *ModuleConfig, rule *android.RuleBuilder, archIdx int, - profile android.WritablePath, appImage bool, generateDM bool, productPackages android.Path) { + profile android.WritablePath, appImage bool, generateDM bool, productPackages android.Path, copyApexSystemServerJarDex bool) { arch := module.Archs[archIdx] @@ -277,7 +277,7 @@ func dexpreoptCommand(ctx android.BuilderContext, globalSoong *GlobalSoongConfig clcTarget = append(clcTarget, GetSystemServerDexLocation(ctx, global, lib)) } - if DexpreoptRunningInSoong { + if DexpreoptRunningInSoong && copyApexSystemServerJarDex { // Copy the system server jar to a predefined location where dex2oat will find it. dexPathHost := SystemServerDexJarHostPath(ctx, module.Name) rule.Command().Text("mkdir -p").Flag(filepath.Dir(dexPathHost.String())) diff --git a/dexpreopt/dexpreopt_gen/dexpreopt_gen.go b/dexpreopt/dexpreopt_gen/dexpreopt_gen.go index 8033b48e5..75120052e 100644 --- a/dexpreopt/dexpreopt_gen/dexpreopt_gen.go +++ b/dexpreopt/dexpreopt_gen/dexpreopt_gen.go @@ -205,8 +205,9 @@ func writeScripts(ctx android.BuilderContext, globalSoong *dexpreopt.GlobalSoong panic(err) } } + cpApexSscpServerJar := false // dexpreopt_gen operates on make modules, and since sscp libraries are in soong, this should be a noop dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule( - ctx, globalSoong, global, module, android.PathForTesting(productPackagesPath)) + ctx, globalSoong, global, module, android.PathForTesting(productPackagesPath), cpApexSscpServerJar) if err != nil { panic(err) } diff --git a/dexpreopt/dexpreopt_test.go b/dexpreopt/dexpreopt_test.go index 7071f3e9e..eff2416e5 100644 --- a/dexpreopt/dexpreopt_test.go +++ b/dexpreopt/dexpreopt_test.go @@ -101,7 +101,7 @@ func TestDexPreopt(t *testing.T) { module := testSystemModuleConfig(ctx, "test") productPackages := android.PathForTesting("product_packages.txt") - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } @@ -161,7 +161,7 @@ func TestDexPreoptSystemOther(t *testing.T) { for _, test := range tests { global.PatternsOnSystemOther = test.patterns for _, mt := range test.moduleTests { - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, mt.module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, mt.module, productPackages, true) if err != nil { t.Fatal(err) } @@ -181,6 +181,11 @@ func TestDexPreoptSystemOther(t *testing.T) { } func TestDexPreoptApexSystemServerJars(t *testing.T) { + // modify the global variable for test + var oldDexpreoptRunningInSoong = DexpreoptRunningInSoong + DexpreoptRunningInSoong = true + + // test begin config := android.TestConfig("out", nil, "", nil) ctx := android.BuilderContextForTesting(config) globalSoong := globalSoongConfigForTests(ctx) @@ -191,7 +196,7 @@ func TestDexPreoptApexSystemServerJars(t *testing.T) { global.ApexSystemServerJars = android.CreateTestConfiguredJarList( []string{"com.android.apex1:service-A"}) - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } @@ -202,6 +207,18 @@ func TestDexPreoptApexSystemServerJars(t *testing.T) { } android.AssertStringEquals(t, "installs", wantInstalls.String(), rule.Installs().String()) + + android.AssertStringListContains(t, "apex sscp jar copy", rule.Outputs().Strings(), "out/soong/system_server_dexjars/service-A.jar") + + // rule with apex sscp cp as false + rule, err = GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, false) + if err != nil { + t.Fatal(err) + } + android.AssertStringListDoesNotContain(t, "apex sscp jar copy", rule.Outputs().Strings(), "out/soong/system_server_dexjars/service-A.jar") + + // cleanup the global variable for test + DexpreoptRunningInSoong = oldDexpreoptRunningInSoong } func TestDexPreoptStandaloneSystemServerJars(t *testing.T) { @@ -215,7 +232,7 @@ func TestDexPreoptStandaloneSystemServerJars(t *testing.T) { global.StandaloneSystemServerJars = android.CreateTestConfiguredJarList( []string{"platform:service-A"}) - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } @@ -239,7 +256,7 @@ func TestDexPreoptSystemExtSystemServerJars(t *testing.T) { global.StandaloneSystemServerJars = android.CreateTestConfiguredJarList( []string{"system_ext:service-A"}) - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } @@ -263,7 +280,7 @@ func TestDexPreoptApexStandaloneSystemServerJars(t *testing.T) { global.ApexStandaloneSystemServerJars = android.CreateTestConfiguredJarList( []string{"com.android.apex1:service-A"}) - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } @@ -286,7 +303,7 @@ func TestDexPreoptProfile(t *testing.T) { module.ProfileClassListing = android.OptionalPathForPath(android.PathForTesting("profile")) - rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages) + rule, err := GenerateDexpreoptRule(ctx, globalSoong, global, module, productPackages, true) if err != nil { t.Fatal(err) } diff --git a/filesystem/filesystem_test.go b/filesystem/filesystem_test.go index f4ecad46c..3a5071d34 100644 --- a/filesystem/filesystem_test.go +++ b/filesystem/filesystem_test.go @@ -84,12 +84,21 @@ func TestFileSystemDeps(t *testing.T) { cc_library { name: "libbar", required: ["libbaz"], + target: { + platform: { + required: ["lib_platform_only"], + }, + }, } cc_library { name: "libbaz", } + cc_library { + name: "lib_platform_only", + } + phony { name: "phony", required: [ @@ -120,6 +129,7 @@ func TestFileSystemDeps(t *testing.T) { "lib64/libbar.so", "lib64/libbaz.so", "lib64/libquz.so", + "lib64/lib_platform_only.so", "etc/bpf/bpf.o", } for _, e := range expected { diff --git a/java/aar.go b/java/aar.go index a36626732..f8955ce90 100644 --- a/java/aar.go +++ b/java/aar.go @@ -356,12 +356,13 @@ type aaptBuildActionOptions struct { forceNonFinalResourceIDs bool extraLinkFlags []string aconfigTextFiles android.Paths + usesLibrary *usesLibrary } func (a *aapt) buildActions(ctx android.ModuleContext, opts aaptBuildActionOptions) { staticResourcesNodesDepSet, sharedResourcesNodesDepSet, staticRRODirsDepSet, staticManifestsDepSet, sharedExportPackages, libFlags := - aaptLibs(ctx, opts.sdkContext, opts.classLoaderContexts) + aaptLibs(ctx, opts.sdkContext, opts.classLoaderContexts, opts.usesLibrary) // Exclude any libraries from the supplied list. opts.classLoaderContexts = opts.classLoaderContexts.ExcludeLibs(opts.excludedLibs) @@ -703,7 +704,8 @@ func (t transitiveAarDeps) assets() android.Paths { } // aaptLibs collects libraries from dependencies and sdk_version and converts them into paths -func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoaderContexts dexpreopt.ClassLoaderContextMap) ( +func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, + classLoaderContexts dexpreopt.ClassLoaderContextMap, usesLibrary *usesLibrary) ( staticResourcesNodes, sharedResourcesNodes *android.DepSet[*resourcesNode], staticRRODirs *android.DepSet[rroDir], staticManifests *android.DepSet[android.Path], sharedLibs android.Paths, flags []string) { @@ -753,6 +755,9 @@ func aaptLibs(ctx android.ModuleContext, sdkContext android.SdkContext, classLoa } addCLCFromDep(ctx, module, classLoaderContexts) + if usesLibrary != nil { + addMissingOptionalUsesLibsFromDep(ctx, module, usesLibrary) + } }) // AAPT2 overlays are in lowest to highest priority order, the topological order will be reversed later. @@ -805,12 +810,12 @@ func (a *AndroidLibrary) OutputFiles(tag string) (android.Paths, error) { var _ AndroidLibraryDependency = (*AndroidLibrary)(nil) func (a *AndroidLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { + a.usesLibrary.deps(ctx, false) a.Module.deps(ctx) sdkDep := decodeSdkDep(ctx, android.SdkContext(a)) if sdkDep.hasFrameworkLibs() { a.aapt.deps(ctx, sdkDep) } - a.usesLibrary.deps(ctx, false) for _, aconfig_declaration := range a.aaptProperties.Flags_packages { ctx.AddDependency(ctx.Module(), aconfigDeclarationTag, aconfig_declaration) @@ -829,6 +834,7 @@ func (a *AndroidLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) classLoaderContexts: a.classLoaderContexts, enforceDefaultTargetSdkVersion: false, aconfigTextFiles: getAconfigFilePaths(ctx), + usesLibrary: &a.usesLibrary, }, ) @@ -1215,7 +1221,7 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { linkDeps = append(linkDeps, a.manifest) staticResourcesNodesDepSet, sharedResourcesNodesDepSet, staticRRODirsDepSet, staticManifestsDepSet, sharedLibs, libFlags := - aaptLibs(ctx, android.SdkContext(a), nil) + aaptLibs(ctx, android.SdkContext(a), nil, nil) _ = sharedResourcesNodesDepSet _ = staticRRODirsDepSet @@ -1287,6 +1293,7 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { } } addCLCFromDep(ctx, module, a.classLoaderContexts) + addMissingOptionalUsesLibsFromDep(ctx, module, &a.usesLibrary) }) var implementationJarFile android.OutputPath @@ -1405,6 +1412,12 @@ func (a *AARImport) ShouldSupportSdkVersion(ctx android.BaseModuleContext, var _ android.PrebuiltInterface = (*AARImport)(nil) +func (a *AARImport) UsesLibrary() *usesLibrary { + return &a.usesLibrary +} + +var _ ModuleWithUsesLibrary = (*AARImport)(nil) + // android_library_import imports an `.aar` file into the build graph as if it was built with android_library. // // This module is not suitable for installing on a device, but can be used as a `static_libs` dependency of diff --git a/java/app.go b/java/app.go index 1aa3afe8e..50d1a2f43 100644 --- a/java/app.go +++ b/java/app.go @@ -249,13 +249,13 @@ func (c Certificate) AndroidMkString() string { } func (a *AndroidApp) DepsMutator(ctx android.BottomUpMutatorContext) { - a.Module.deps(ctx) - if String(a.appProperties.Stl) == "c++_shared" && !a.SdkVersion(ctx).Specified() { ctx.PropertyErrorf("stl", "sdk_version must be set in order to use c++_shared") } sdkDep := decodeSdkDep(ctx, android.SdkContext(a)) + a.usesLibrary.deps(ctx, sdkDep.hasFrameworkLibs()) + a.Module.deps(ctx) if sdkDep.hasFrameworkLibs() { a.aapt.deps(ctx, sdkDep) } @@ -285,9 +285,6 @@ func (a *AndroidApp) DepsMutator(ctx android.BottomUpMutatorContext) { } ctx.AddFarVariationDependencies(variation, jniLibTag, a.appProperties.Jni_libs...) } - - a.usesLibrary.deps(ctx, sdkDep.hasFrameworkLibs()) - for _, aconfig_declaration := range a.aaptProperties.Flags_packages { ctx.AddDependency(ctx.Module(), aconfigDeclarationTag, aconfig_declaration) } @@ -534,6 +531,7 @@ func (a *AndroidApp) aaptBuildActions(ctx android.ModuleContext) { forceNonFinalResourceIDs: nonFinalIds, extraLinkFlags: aaptLinkFlags, aconfigTextFiles: getAconfigFilePaths(ctx), + usesLibrary: &a.usesLibrary, }, ) @@ -815,18 +813,10 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { // The decision to enforce <uses-library> checks is made before adding implicit SDK libraries. a.usesLibrary.freezeEnforceUsesLibraries() - // Add implicit SDK libraries to <uses-library> list. - requiredUsesLibs, optionalUsesLibs := a.classLoaderContexts.UsesLibs() - for _, usesLib := range requiredUsesLibs { - a.usesLibrary.addLib(usesLib, false) - } - for _, usesLib := range optionalUsesLibs { - a.usesLibrary.addLib(usesLib, true) - } - // Check that the <uses-library> list is coherent with the manifest. if a.usesLibrary.enforceUsesLibraries() { - manifestCheckFile := a.usesLibrary.verifyUsesLibrariesManifest(ctx, a.mergedManifestFile) + manifestCheckFile := a.usesLibrary.verifyUsesLibrariesManifest( + ctx, a.mergedManifestFile, &a.classLoaderContexts) apkDeps = append(apkDeps, manifestCheckFile) } @@ -1596,6 +1586,9 @@ type UsesLibraryProperties struct { // provide the android.test.base statically and use jarjar to rename them so they do not collide // with the classes provided by the android.test.base library. Exclude_uses_libs []string + + // The module names of optional uses-library libraries that are missing from the source tree. + Missing_optional_uses_libs []string `blueprint:"mutated"` } // usesLibrary provides properties and helper functions for AndroidApp and AndroidAppImport to verify that the @@ -1612,20 +1605,11 @@ type usesLibrary struct { shouldDisableDexpreopt bool } -func (u *usesLibrary) addLib(lib string, optional bool) { - if !android.InList(lib, u.usesLibraryProperties.Uses_libs) && !android.InList(lib, u.usesLibraryProperties.Optional_uses_libs) { - if optional { - u.usesLibraryProperties.Optional_uses_libs = append(u.usesLibraryProperties.Optional_uses_libs, lib) - } else { - u.usesLibraryProperties.Uses_libs = append(u.usesLibraryProperties.Uses_libs, lib) - } - } -} - func (u *usesLibrary) deps(ctx android.BottomUpMutatorContext, addCompatDeps bool) { if !ctx.Config().UnbundledBuild() || ctx.Config().UnbundledBuildImage() { ctx.AddVariationDependencies(nil, usesLibReqTag, u.usesLibraryProperties.Uses_libs...) - ctx.AddVariationDependencies(nil, usesLibOptTag, u.presentOptionalUsesLibs(ctx)...) + presentOptionalUsesLibs := u.presentOptionalUsesLibs(ctx) + ctx.AddVariationDependencies(nil, usesLibOptTag, presentOptionalUsesLibs...) // Only add these extra dependencies if the module is an app that depends on framework // libs. This avoids creating a cyclic dependency: // e.g. framework-res -> org.apache.http.legacy -> ... -> framework-res. @@ -1636,6 +1620,8 @@ func (u *usesLibrary) deps(ctx android.BottomUpMutatorContext, addCompatDeps boo ctx.AddVariationDependencies(nil, usesLibCompat28OptTag, dexpreopt.OptionalCompatUsesLibs28...) ctx.AddVariationDependencies(nil, usesLibCompat30OptTag, dexpreopt.OptionalCompatUsesLibs30...) } + _, diff, _ := android.ListSetDifference(u.usesLibraryProperties.Optional_uses_libs, presentOptionalUsesLibs) + u.usesLibraryProperties.Missing_optional_uses_libs = diff } else { ctx.AddVariationDependencies(nil, r8LibraryJarTag, u.usesLibraryProperties.Uses_libs...) ctx.AddVariationDependencies(nil, r8LibraryJarTag, u.presentOptionalUsesLibs(ctx)...) @@ -1654,15 +1640,6 @@ func (u *usesLibrary) presentOptionalUsesLibs(ctx android.BaseModuleContext) []s return optionalUsesLibs } -// Helper function to replace string in a list. -func replaceInList(list []string, oldstr, newstr string) { - for i, str := range list { - if str == oldstr { - list[i] = newstr - } - } -} - // Returns a map of module names of shared library dependencies to the paths to their dex jars on // host and on device. func (u *usesLibrary) classLoaderContextForUsesLibDeps(ctx android.ModuleContext) dexpreopt.ClassLoaderContextMap { @@ -1704,11 +1681,6 @@ func (u *usesLibrary) classLoaderContextForUsesLibDeps(ctx android.ModuleContext libName := dep if ulib, ok := m.(ProvidesUsesLib); ok && ulib.ProvidesUsesLib() != nil { libName = *ulib.ProvidesUsesLib() - // Replace module name with library name in `uses_libs`/`optional_uses_libs` in - // order to pass verify_uses_libraries check (which compares these properties - // against library names written in the manifest). - replaceInList(u.usesLibraryProperties.Uses_libs, dep, libName) - replaceInList(u.usesLibraryProperties.Optional_uses_libs, dep, libName) } clcMap.AddContext(ctx, tag.sdkVersion, libName, tag.optional, lib.DexJarBuildPath(ctx).PathOrNil(), lib.DexJarInstallPath(), @@ -1742,7 +1714,7 @@ func (u *usesLibrary) freezeEnforceUsesLibraries() { // an APK with the manifest embedded in it (manifest_check will know which one it is by the file // extension: APKs are supposed to end with '.apk'). func (u *usesLibrary) verifyUsesLibraries(ctx android.ModuleContext, inputFile android.Path, - outputFile android.WritablePath) android.Path { + outputFile android.WritablePath, classLoaderContexts *dexpreopt.ClassLoaderContextMap) android.Path { statusFile := dexpreopt.UsesLibrariesStatusFile(ctx) @@ -1770,27 +1742,37 @@ func (u *usesLibrary) verifyUsesLibraries(ctx android.ModuleContext, inputFile a cmd.Flag("--enforce-uses-libraries-relax") } - for _, lib := range u.usesLibraryProperties.Uses_libs { + requiredUsesLibs, optionalUsesLibs := classLoaderContexts.UsesLibs() + for _, lib := range requiredUsesLibs { cmd.FlagWithArg("--uses-library ", lib) } - - for _, lib := range u.usesLibraryProperties.Optional_uses_libs { + for _, lib := range optionalUsesLibs { cmd.FlagWithArg("--optional-uses-library ", lib) } + // Also add missing optional uses libs, as the manifest check expects them. + // Note that what we add here are the module names of those missing libs, not library names, while + // the manifest check actually expects library names. However, the case where a library is missing + // and the module name != the library name is too rare for us to handle. + for _, lib := range u.usesLibraryProperties.Missing_optional_uses_libs { + cmd.FlagWithArg("--missing-optional-uses-library ", lib) + } + rule.Build("verify_uses_libraries", "verify <uses-library>") return outputFile } // verifyUsesLibrariesManifest checks the <uses-library> tags in an AndroidManifest.xml against // the build system and returns the path to a copy of the manifest. -func (u *usesLibrary) verifyUsesLibrariesManifest(ctx android.ModuleContext, manifest android.Path) android.Path { +func (u *usesLibrary) verifyUsesLibrariesManifest(ctx android.ModuleContext, manifest android.Path, + classLoaderContexts *dexpreopt.ClassLoaderContextMap) android.Path { outputFile := android.PathForModuleOut(ctx, "manifest_check", "AndroidManifest.xml") - return u.verifyUsesLibraries(ctx, manifest, outputFile) + return u.verifyUsesLibraries(ctx, manifest, outputFile, classLoaderContexts) } // verifyUsesLibrariesAPK checks the <uses-library> tags in the manifest of an APK against the build // system and returns the path to a copy of the APK. -func (u *usesLibrary) verifyUsesLibrariesAPK(ctx android.ModuleContext, apk android.Path) { - u.verifyUsesLibraries(ctx, apk, nil) // for APKs manifest_check does not write output file +func (u *usesLibrary) verifyUsesLibrariesAPK(ctx android.ModuleContext, apk android.Path, + classLoaderContexts *dexpreopt.ClassLoaderContextMap) { + u.verifyUsesLibraries(ctx, apk, nil, classLoaderContexts) // for APKs manifest_check does not write output file } diff --git a/java/app_import.go b/java/app_import.go index 7387e168c..bb07c423a 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -355,7 +355,7 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext } if a.usesLibrary.enforceUsesLibraries() { - a.usesLibrary.verifyUsesLibrariesAPK(ctx, srcApk) + a.usesLibrary.verifyUsesLibrariesAPK(ctx, srcApk, &a.dexpreopter.classLoaderContexts) } a.dexpreopter.dexpreopt(ctx, android.RemoveOptionalPrebuiltPrefix(ctx.ModuleName()), jnisUncompressed) @@ -611,6 +611,12 @@ func createArchDpiVariantGroupType(archNames []string, dpiNames []string) reflec return return_struct } +func (a *AndroidAppImport) UsesLibrary() *usesLibrary { + return &a.usesLibrary +} + +var _ ModuleWithUsesLibrary = (*AndroidAppImport)(nil) + // android_app_import imports a prebuilt apk with additional processing specified in the module. // DPI-specific apk source files can be specified using dpi_variants. Example: // diff --git a/java/app_test.go b/java/app_test.go index 0c2800041..eab40e7da 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -3244,7 +3244,10 @@ func TestUsesLibraries(t *testing.T) { name: "static-y", srcs: ["a.java"], uses_libs: ["runtime-required-y"], - optional_uses_libs: ["runtime-optional-y"], + optional_uses_libs: [ + "runtime-optional-y", + "missing-lib-a", + ], sdk_version: "current", } @@ -3280,7 +3283,7 @@ func TestUsesLibraries(t *testing.T) { sdk_version: "current", optional_uses_libs: [ "bar", - "baz", + "missing-lib-b", ], } @@ -3295,7 +3298,7 @@ func TestUsesLibraries(t *testing.T) { ], optional_uses_libs: [ "bar", - "baz", + "missing-lib-b", ], } ` @@ -3317,10 +3320,10 @@ func TestUsesLibraries(t *testing.T) { // propagated from dependencies. actualManifestFixerArgs := app.Output("manifest_fixer/AndroidManifest.xml").Args["args"] expectManifestFixerArgs := `--extract-native-libs=true ` + - `--uses-library qux ` + - `--uses-library quuz ` + `--uses-library foo ` + `--uses-library com.non.sdk.lib ` + + `--uses-library qux ` + + `--uses-library quuz ` + `--uses-library runtime-library ` + `--uses-library runtime-required-x ` + `--uses-library runtime-required-y ` + @@ -3339,9 +3342,10 @@ func TestUsesLibraries(t *testing.T) { `--uses-library runtime-required-x ` + `--uses-library runtime-required-y ` + `--optional-uses-library bar ` + - `--optional-uses-library baz ` + `--optional-uses-library runtime-optional-x ` + - `--optional-uses-library runtime-optional-y ` + `--optional-uses-library runtime-optional-y ` + + `--missing-optional-uses-library missing-lib-b ` + + `--missing-optional-uses-library missing-lib-a` android.AssertStringDoesContain(t, "verify cmd args", verifyCmd, verifyArgs) // Test that all libraries are verified for an APK (library order matters). @@ -3350,7 +3354,7 @@ func TestUsesLibraries(t *testing.T) { `--uses-library com.non.sdk.lib ` + `--uses-library android.test.runner ` + `--optional-uses-library bar ` + - `--optional-uses-library baz ` + `--missing-optional-uses-library missing-lib-b ` android.AssertStringDoesContain(t, "verify apk cmd args", verifyApkCmd, verifyApkArgs) // Test that necessary args are passed for constructing CLC in Ninja phase. diff --git a/java/base.go b/java/base.go index ef61f1cc2..938ac5e82 100644 --- a/java/base.go +++ b/java/base.go @@ -843,9 +843,11 @@ func (j *Module) deps(ctx android.BottomUpMutatorContext) { if dep != nil { if component, ok := dep.(SdkLibraryComponentDependency); ok { if lib := component.OptionalSdkLibraryImplementation(); lib != nil { - // Add library as optional if it's one of the optional compatibility libs. + // Add library as optional if it's one of the optional compatibility libs or it's + // explicitly listed in the optional_uses_libs property. tag := usesLibReqTag - if android.InList(*lib, dexpreopt.OptionalCompatUsesLibs) { + if android.InList(*lib, dexpreopt.OptionalCompatUsesLibs) || + android.InList(*lib, j.usesLibrary.usesLibraryProperties.Optional_uses_libs) { tag = usesLibOptTag } ctx.AddVariationDependencies(nil, tag, *lib) @@ -2387,6 +2389,7 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { } addCLCFromDep(ctx, module, j.classLoaderContexts) + addMissingOptionalUsesLibsFromDep(ctx, module, &j.usesLibrary) }) return deps @@ -2720,3 +2723,13 @@ type ModuleWithStem interface { } var _ ModuleWithStem = (*Module)(nil) + +type ModuleWithUsesLibrary interface { + UsesLibrary() *usesLibrary +} + +func (j *Module) UsesLibrary() *usesLibrary { + return &j.usesLibrary +} + +var _ ModuleWithUsesLibrary = (*Module)(nil) diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 38ed856ee..25e95db14 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -243,10 +243,6 @@ func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext, libName s return true } - if disableSourceApexVariant(ctx) { - return true - } - if _, isApex := android.ModuleProvider(ctx, android.ApexBundleInfoProvider); isApex { // dexpreopt rules for system server jars can be generated in the ModuleCtx of prebuilt apexes return false @@ -501,8 +497,12 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, libName string, dexJa Output(appProductPackages) productPackagesRule.Restat().Build("product_packages."+dexJarStem, "dexpreopt product_packages") + // Prebuilts are active, do not copy the dexpreopt'd source javalib to out/soong/system_server_dexjars + // The javalib from the deapexed prebuilt will be copied to this location. + // TODO (b/331665856): Implement a principled solution for this. + copyApexSystemServerJarDex := !disableSourceApexVariant(ctx) dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule( - ctx, globalSoong, global, dexpreoptConfig, appProductPackages) + ctx, globalSoong, global, dexpreoptConfig, appProductPackages, copyApexSystemServerJarDex) if err != nil { ctx.ModuleErrorf("error generating dexpreopt rule: %s", err.Error()) return diff --git a/java/droidstubs.go b/java/droidstubs.go index 02b81a4fe..870c6438e 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -532,8 +532,8 @@ func (d *Droidstubs) annotationsFlags(ctx android.ModuleContext, cmd *android.Ru cmd.Flag(config.MetalavaAnnotationsFlags) if params.migratingNullability { - previousApi := android.PathForModuleSrc(ctx, String(d.properties.Previous_api)) - cmd.FlagWithInput("--migrate-nullness ", previousApi) + previousApiFiles := android.PathsForModuleSrc(ctx, []string{String(d.properties.Previous_api)}) + cmd.FlagForEachInput("--migrate-nullness ", previousApiFiles) } if s := String(d.properties.Validate_nullability_from_list); s != "" { @@ -692,11 +692,11 @@ func (d *Droidstubs) apiCompatibilityFlags(ctx android.ModuleContext, cmd *andro ctx.PropertyErrorf("out", "out property may not be combined with check_api") } - apiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Api_file)) - removedApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Last_released.Removed_api_file)) + apiFiles := android.PathsForModuleSrc(ctx, []string{String(d.properties.Check_api.Last_released.Api_file)}) + removedApiFiles := android.PathsForModuleSrc(ctx, []string{String(d.properties.Check_api.Last_released.Removed_api_file)}) - cmd.FlagWithInput("--check-compatibility:api:released ", apiFile) - cmd.FlagWithInput("--check-compatibility:removed:released ", removedApiFile) + cmd.FlagForEachInput("--check-compatibility:api:released ", apiFiles) + cmd.FlagForEachInput("--check-compatibility:removed:released ", removedApiFiles) baselineFile := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Last_released.Baseline_file) if baselineFile.Valid() { @@ -950,9 +950,12 @@ func (d *Droidstubs) everythingOptionalCmd(ctx android.ModuleContext, cmd *andro // Add API lint options. if doApiLint { - newSince := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Api_lint.New_since) - if newSince.Valid() { - cmd.FlagWithInput("--api-lint ", newSince.Path()) + var newSince android.Paths + if d.properties.Check_api.Api_lint.New_since != nil { + newSince = android.PathsForModuleSrc(ctx, []string{proptools.String(d.properties.Check_api.Api_lint.New_since)}) + } + if len(newSince) > 0 { + cmd.FlagForEachInput("--api-lint ", newSince) } else { cmd.Flag("--api-lint") } diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go index e4beb5e55..ae587eac3 100644 --- a/java/hiddenapi_modular.go +++ b/java/hiddenapi_modular.go @@ -1478,13 +1478,3 @@ func retrieveEncodedBootDexJarFromModule(ctx android.ModuleContext, module andro } return bootDexJar.Path() } - -// extractEncodedDexJarsFromModules extracts the encoded dex jars from the supplied modules. -func extractEncodedDexJarsFromModules(ctx android.ModuleContext, contents []android.Module) bootDexJarByModule { - encodedDexJarsByModuleName := bootDexJarByModule{} - for _, module := range contents { - path := retrieveEncodedBootDexJarFromModule(ctx, module) - encodedDexJarsByModuleName.addPath(module, path) - } - return encodedDexJarsByModuleName -} diff --git a/java/java.go b/java/java.go index a48ac0a13..725e25abe 100644 --- a/java/java.go +++ b/java/java.go @@ -891,6 +891,12 @@ func init() { } func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if disableSourceApexVariant(ctx) { + // Prebuilts are active, do not create the installation rules for the source javalib. + // Even though the source javalib is not used, we need to hide it to prevent duplicate installation rules. + // TODO (b/331665856): Implement a principled solution for this. + j.HideFromMake() + } j.provideHiddenAPIPropertyInfo(ctx) j.sdkVersion = j.SdkVersion(ctx) @@ -971,8 +977,8 @@ func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { } func (j *Library) DepsMutator(ctx android.BottomUpMutatorContext) { - j.deps(ctx) j.usesLibrary.deps(ctx, false) + j.deps(ctx) } const ( @@ -3167,13 +3173,35 @@ func addCLCFromDep(ctx android.ModuleContext, depModule android.Module, // <uses_library> and should not be added to CLC, but the transitive <uses-library> dependencies // from its CLC should be added to the current CLC. if sdkLib != nil { - clcMap.AddContext(ctx, dexpreopt.AnySdkVersion, *sdkLib, false, + optional := false + if module, ok := ctx.Module().(ModuleWithUsesLibrary); ok { + if android.InList(*sdkLib, module.UsesLibrary().usesLibraryProperties.Optional_uses_libs) { + optional = true + } + } + clcMap.AddContext(ctx, dexpreopt.AnySdkVersion, *sdkLib, optional, dep.DexJarBuildPath(ctx).PathOrNil(), dep.DexJarInstallPath(), dep.ClassLoaderContexts()) } else { clcMap.AddContextMap(dep.ClassLoaderContexts(), depName) } } +func addMissingOptionalUsesLibsFromDep(ctx android.ModuleContext, depModule android.Module, + usesLibrary *usesLibrary) { + + dep, ok := depModule.(ModuleWithUsesLibrary) + if !ok { + return + } + + for _, lib := range dep.UsesLibrary().usesLibraryProperties.Missing_optional_uses_libs { + if !android.InList(lib, usesLibrary.usesLibraryProperties.Missing_optional_uses_libs) { + usesLibrary.usesLibraryProperties.Missing_optional_uses_libs = + append(usesLibrary.usesLibraryProperties.Missing_optional_uses_libs, lib) + } + } +} + type JavaApiContributionImport struct { JavaApiContribution diff --git a/java/prebuilt_apis.go b/java/prebuilt_apis.go index 6a79e5883..00613eee3 100644 --- a/java/prebuilt_apis.go +++ b/java/prebuilt_apis.go @@ -367,10 +367,23 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { info := latest[k] name := PrebuiltApiCombinedModuleName(info.module, info.scope, "latest") + // Iterate until the currentApiScope does not extend any other api scopes + // i.e. is not a superset of any other api scopes + // the relationship between the api scopes is defined in java/sdk_library.go var srcs []string currentApiScope := scopeByName[info.scope] - srcs = append(srcs, PrebuiltApiModuleName(info.module, currentApiScope.name, "latest")) + for currentApiScope != nil { + if _, ok := latest[fmt.Sprintf("%s.%s", info.module, currentApiScope.name)]; ok { + srcs = append(srcs, PrebuiltApiModuleName(info.module, currentApiScope.name, "latest")) + } + currentApiScope = currentApiScope.extends + } + // srcs is currently listed in the order from the widest api scope to the narrowest api scopes + // e.g. module lib -> system -> public + // In order to pass the files in metalava from the narrowest api scope to the widest api scope, + // the list has to be reversed. + android.ReverseSliceInPlace(srcs) createCombinedApiFilegroupModule(mctx, name, srcs) } } diff --git a/java/sdk_library.go b/java/sdk_library.go index e7e53a2a8..bb5730db1 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -705,10 +705,10 @@ type scopePaths struct { annotationsZip android.OptionalPath // The path to the latest API file. - latestApiPath android.OptionalPath + latestApiPaths android.Paths // The path to the latest removed API file. - latestRemovedApiPath android.OptionalPath + latestRemovedApiPaths android.Paths } func (paths *scopePaths) extractStubsLibraryInfoFromDependency(ctx android.ModuleContext, dep android.Module) error { @@ -829,28 +829,25 @@ func (paths *scopePaths) extractStubsSourceAndApiInfoFromApiStubsProvider(ctx an }) } -func extractSingleOptionalOutputPath(dep android.Module) (android.OptionalPath, error) { +func extractOutputPaths(dep android.Module) (android.Paths, error) { var paths android.Paths if sourceFileProducer, ok := dep.(android.SourceFileProducer); ok { paths = sourceFileProducer.Srcs() + return paths, nil } else { - return android.OptionalPath{}, fmt.Errorf("module %q does not produce source files", dep) + return nil, fmt.Errorf("module %q does not produce source files", dep) } - if len(paths) != 1 { - return android.OptionalPath{}, fmt.Errorf("expected one path from %q, got %q", dep, paths) - } - return android.OptionalPathForPath(paths[0]), nil } func (paths *scopePaths) extractLatestApiPath(ctx android.ModuleContext, dep android.Module) error { - outputPath, err := extractSingleOptionalOutputPath(dep) - paths.latestApiPath = outputPath + outputPaths, err := extractOutputPaths(dep) + paths.latestApiPaths = outputPaths return err } func (paths *scopePaths) extractLatestRemovedApiPath(ctx android.ModuleContext, dep android.Module) error { - outputPath, err := extractSingleOptionalOutputPath(dep) - paths.latestRemovedApiPath = outputPath + outputPaths, err := extractOutputPaths(dep) + paths.latestRemovedApiPaths = outputPaths return err } @@ -1562,6 +1559,12 @@ func (module *SdkLibrary) OutputFiles(tag string) (android.Paths, error) { } func (module *SdkLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if disableSourceApexVariant(ctx) { + // Prebuilts are active, do not create the installation rules for the source javalib. + // Even though the source javalib is not used, we need to hide it to prevent duplicate installation rules. + // TODO (b/331665856): Implement a principled solution for this. + module.HideFromMake() + } if proptools.String(module.deviceProperties.Min_sdk_version) != "" { module.CheckMinSdkVersion(ctx) } @@ -1619,11 +1622,15 @@ func (module *SdkLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) scopes[scope.name] = scopeInfo scopeInfo["current_api"] = scope.snapshotRelativeCurrentApiTxtPath(baseModuleName) scopeInfo["removed_api"] = scope.snapshotRelativeRemovedApiTxtPath(baseModuleName) - if p := scopePaths.latestApiPath; p.Valid() { - scopeInfo["latest_api"] = p.Path().String() + if p := scopePaths.latestApiPaths; len(p) > 0 { + // The last path in the list is the one that applies to this scope, the + // preceding ones, if any, are for the scope(s) that it extends. + scopeInfo["latest_api"] = p[len(p)-1].String() } - if p := scopePaths.latestRemovedApiPath; p.Valid() { - scopeInfo["latest_removed_api"] = p.Path().String() + if p := scopePaths.latestRemovedApiPaths; len(p) > 0 { + // The last path in the list is the one that applies to this scope, the + // preceding ones, if any, are for the scope(s) that it extends. + scopeInfo["latest_removed_api"] = p[len(p)-1].String() } } android.SetProvider(ctx, android.AdditionalSdkInfoProvider, android.AdditionalSdkInfo{additionalSdkInfo}) diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index 5fac2556d..0f163e6e0 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -1085,18 +1085,6 @@ func TestJavaSdkLibraryImport_Preferred(t *testing.T) { t.Run("prefer", func(t *testing.T) { testJavaSdkLibraryImport_Preferred(t, "prefer: true,", android.NullFixturePreparer) }) - - t.Run("use_source_config_var", func(t *testing.T) { - testJavaSdkLibraryImport_Preferred(t, - "use_source_config_var: {config_namespace: \"acme\", var_name: \"use_source\"},", - android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { - variables.VendorVars = map[string]map[string]string{ - "acme": { - "use_source": "false", - }, - } - })) - }) } // If a module is listed in `mainline_module_contributions, it should be used diff --git a/partner/androidmk/androidmk_test.go b/partner/androidmk/androidmk_test.go index 6bae836d9..3ace7502d 100644 --- a/partner/androidmk/androidmk_test.go +++ b/partner/androidmk/androidmk_test.go @@ -54,6 +54,9 @@ cc_library_shared { } func TestEndToEnd(t *testing.T) { + // Skip checking Android.mk path with cleaning "ANDROID_BUILD_TOP" + t.Setenv("ANDROID_BUILD_TOP", "") + for i, test := range testCases { expected, err := bpfix.Reformat(test.expected) if err != nil { diff --git a/python/python.go b/python/python.go index 2b1974eb8..621e429b9 100644 --- a/python/python.go +++ b/python/python.go @@ -151,6 +151,8 @@ type PythonLibraryModule struct { // The zip file containing the current module's source/data files, with the // source files precompiled. precompiledSrcsZip android.Path + + sourceProperties android.SourceProperties } // newModule generates new Python base module @@ -203,7 +205,7 @@ func (p *PythonLibraryModule) getBaseProperties() *BaseProperties { var _ pythonDependency = (*PythonLibraryModule)(nil) func (p *PythonLibraryModule) init() android.Module { - p.AddProperties(&p.properties, &p.protoProperties) + p.AddProperties(&p.properties, &p.protoProperties, &p.sourceProperties) android.InitAndroidArchModule(p, p.hod, p.multilib) android.InitDefaultableModule(p) return p @@ -421,6 +423,11 @@ func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.Botto func (p *PythonLibraryModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { expandedSrcs := android.PathsForModuleSrcExcludes(ctx, p.properties.Srcs, p.properties.Exclude_srcs) android.SetProvider(ctx, blueprint.SrcsFileProviderKey, blueprint.SrcsFileProviderData{SrcPaths: expandedSrcs.Strings()}) + // Keep before any early returns. + android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{ + TestOnly: Bool(p.sourceProperties.Test_only), + TopLevelTarget: p.sourceProperties.Top_level_test_target, + }) // expand data files from "data" property. expandedData := android.PathsForModuleSrc(ctx, p.properties.Data) diff --git a/python/python_test.go b/python/python_test.go index 75a6a899b..c0b7295f9 100644 --- a/python/python_test.go +++ b/python/python_test.go @@ -18,10 +18,13 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "android/soong/android" "android/soong/cc" + + "github.com/google/blueprint" ) type pyModule struct { @@ -360,6 +363,76 @@ cc_binary { } } +func TestTestOnlyProvider(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + PrepareForTestWithPythonBuildComponents, + android.PrepareForTestWithAllowMissingDependencies, + ).RunTestWithBp(t, ` + // These should be test-only + python_library { name: "py-lib-test", test_only: true } + python_library { name: "py-lib-test-host", test_only: true, host_supported: true } + python_test { name: "py-test", srcs: ["py-test.py"] } + python_test_host { name: "py-test-host", srcs: ["py-test-host.py"] } + python_binary_host { name: "py-bin-test", srcs: ["py-bin-test.py"] } + + // These should not be. + python_library { name: "py-lib" } + python_binary_host { name: "py-bin", srcs: ["py-bin.py"] } + `) + + // Visit all modules and ensure only the ones that should + // marked as test-only are marked as test-only. + + actualTestOnly := []string{} + ctx.VisitAllModules(func(m blueprint.Module) { + if provider, ok := android.OtherModuleProvider(ctx.TestContext.OtherModuleProviderAdaptor(), m, android.TestOnlyProviderKey); ok { + if provider.TestOnly { + actualTestOnly = append(actualTestOnly, m.Name()) + } + } + }) + expectedTestOnlyModules := []string{ + "py-lib-test", + "py-lib-test-host", + "py-test", + "py-test-host", + } + + notEqual, left, right := android.ListSetDifference(expectedTestOnlyModules, actualTestOnly) + if notEqual { + t.Errorf("test-only: Expected but not found: %v, Found but not expected: %v", left, right) + } +} + +// Don't allow setting test-only on things that are always tests or never tests. +func TestInvalidTestOnlyTargets(t *testing.T) { + testCases := []string{ + ` python_test { name: "py-test", test_only: true, srcs: ["py-test.py"] } `, + ` python_test_host { name: "py-test-host", test_only: true, srcs: ["py-test-host.py"] } `, + ` python_defaults { name: "py-defaults", test_only: true, srcs: ["foo.py"] } `, + } + + for i, bp := range testCases { + ctx := android.GroupFixturePreparers( + PrepareForTestWithPythonBuildComponents, + android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) { + + ctx.RegisterModuleType("python_defaults", DefaultsFactory) + }), + android.PrepareForTestWithAllowMissingDependencies). + ExtendWithErrorHandler(android.FixtureIgnoreErrors). + RunTestWithBp(t, bp) + if len(ctx.Errs) != 1 { + t.Errorf("Expected err setting test_only in testcase #%d: %d errs", i, len(ctx.Errs)) + continue + } + if !strings.Contains(ctx.Errs[0].Error(), "unrecognized property \"test_only\"") { + t.Errorf("ERR: %s bad bp: %s", ctx.Errs[0], bp) + } + } +} + func expectModule(t *testing.T, ctx *android.TestContext, name, variant, expectedSrcsZip string, expectedPyRunfiles []string) { module := ctx.ModuleForTests(name, variant) diff --git a/python/test.go b/python/test.go index 826f35358..2b939e7e4 100644 --- a/python/test.go +++ b/python/test.go @@ -36,7 +36,9 @@ func registerPythonTestComponents(ctx android.RegistrationContext) { } func NewTest(hod android.HostOrDeviceSupported) *PythonTestModule { - return &PythonTestModule{PythonBinaryModule: *NewBinary(hod)} + p := &PythonTestModule{PythonBinaryModule: *NewBinary(hod)} + p.sourceProperties = android.SourceProperties{Test_only: proptools.BoolPtr(true), Top_level_test_target: true} + return p } func PythonTestHostFactory() android.Module { diff --git a/rust/builder.go b/rust/builder.go index 2f5e12aa5..4f45e33c1 100644 --- a/rust/builder.go +++ b/rust/builder.go @@ -61,7 +61,7 @@ var ( // Use the metadata output as it has the smallest footprint. "--emit metadata -o $out --emit dep-info=$out.d.raw $in ${libFlags} " + "$rustcFlags $clippyFlags" + - " && grep \"^$out:\" $out.d.raw > $out.d", + " && grep ^$out: $out.d.raw > $out.d", CommandDeps: []string{"$clippyCmd"}, Deps: blueprint.DepsGCC, Depfile: "$out.d", diff --git a/scripts/manifest_check.py b/scripts/manifest_check.py index c33b104bd..b10125994 100755 --- a/scripts/manifest_check.py +++ b/scripts/manifest_check.py @@ -52,6 +52,14 @@ def parse_args(): 'required:false' ) parser.add_argument( + '--missing-optional-uses-library', + dest='missing_optional_uses_libraries', + action='append', + help='specify uses-library entries missing from the build system with ' + 'required:false', + default=[] + ) + parser.add_argument( '--enforce-uses-libraries', dest='enforce_uses_libraries', action='store_true', @@ -91,7 +99,7 @@ C_OFF = "\033[0m" C_BOLD = "\033[1m" -def enforce_uses_libraries(manifest, required, optional, relax, is_apk, path): +def enforce_uses_libraries(manifest, required, optional, missing_optional, relax, is_apk, path): """Verify that the <uses-library> tags in the manifest match those provided by the build system. @@ -119,7 +127,12 @@ def enforce_uses_libraries(manifest, required, optional, relax, is_apk, path): required = trim_namespace_parts(required) optional = trim_namespace_parts(optional) - if manifest_required == required and manifest_optional == optional: + existing_manifest_optional = [ + lib for lib in manifest_optional if lib not in missing_optional] + + # The order of the existing libraries matter, while the order of the missing + # ones doesn't. + if manifest_required == required and existing_manifest_optional == optional: return None #pylint: disable=line-too-long @@ -129,6 +142,7 @@ def enforce_uses_libraries(manifest, required, optional, relax, is_apk, path): '\t- required libraries in build system: %s[%s]%s\n' % (C_RED, ', '.join(required), C_OFF), '\t vs. in the manifest: %s[%s]%s\n' % (C_RED, ', '.join(manifest_required), C_OFF), '\t- optional libraries in build system: %s[%s]%s\n' % (C_RED, ', '.join(optional), C_OFF), + '\t and missing ones in build system: %s[%s]%s\n' % (C_RED, ', '.join(missing_optional), C_OFF), '\t vs. in the manifest: %s[%s]%s\n' % (C_RED, ', '.join(manifest_optional), C_OFF), '\t- tags in the manifest (%s):\n' % path, '\t\t%s\n' % '\t\t'.join(tags), @@ -340,11 +354,14 @@ def main(): if args.enforce_uses_libraries: # Load dexpreopt.config files and build a mapping from module - # names to library names. This is necessary because build system - # addresses libraries by their module name (`uses_libs`, - # `optional_uses_libs`, `LOCAL_USES_LIBRARIES`, - # `LOCAL_OPTIONAL_LIBRARY_NAMES` all contain module names), while - # the manifest addresses libraries by their name. + # names to library names. This is for Make only and it's necessary + # because Make passes module names from `LOCAL_USES_LIBRARIES`, + # `LOCAL_OPTIONAL_LIBRARY_NAMES`, while the manifest addresses + # libraries by their name. Soong doesn't use it and doesn't need it + # because it converts the module names to the library names and + # passes the library names. There is no need to translate missing + # optional libs because they are missing and therefore there is no + # mapping for them. mod_to_lib = load_dexpreopt_configs(args.dexpreopt_configs) required = translate_libnames(args.uses_libraries, mod_to_lib) optional = translate_libnames(args.optional_uses_libraries, @@ -354,8 +371,8 @@ def main(): # those in the manifest. Raise an exception on mismatch, unless the # script was passed a special parameter to suppress exceptions. errmsg = enforce_uses_libraries(manifest, required, optional, - args.enforce_uses_libraries_relax, - is_apk, args.input) + args.missing_optional_uses_libraries, + args.enforce_uses_libraries_relax, is_apk, args.input) # Create a status file that is empty on success, or contains an # error message on failure. When exceptions are suppressed, diff --git a/scripts/manifest_check_test.py b/scripts/manifest_check_test.py index 3be7a30bf..8003b3e19 100755 --- a/scripts/manifest_check_test.py +++ b/scripts/manifest_check_test.py @@ -44,15 +44,17 @@ def required_apk(value): class EnforceUsesLibrariesTest(unittest.TestCase): """Unit tests for add_extract_native_libs function.""" - def run_test(self, xml, apk, uses_libraries=[], optional_uses_libraries=[]): #pylint: disable=dangerous-default-value + def run_test(self, xml, apk, uses_libraries=[], optional_uses_libraries=[], + missing_optional_uses_libraries=[]): #pylint: disable=dangerous-default-value doc = minidom.parseString(xml) try: relax = False manifest_check.enforce_uses_libraries( - doc, uses_libraries, optional_uses_libraries, relax, False, - 'path/to/X/AndroidManifest.xml') + doc, uses_libraries, optional_uses_libraries, missing_optional_uses_libraries, + relax, False, 'path/to/X/AndroidManifest.xml') manifest_check.enforce_uses_libraries(apk, uses_libraries, optional_uses_libraries, + missing_optional_uses_libraries, relax, True, 'path/to/X/X.apk') return True @@ -102,6 +104,15 @@ class EnforceUsesLibrariesTest(unittest.TestCase): matches = self.run_test(xml, apk, optional_uses_libraries=['foo']) self.assertFalse(matches) + def test_expected_missing_optional_uses_library(self): + xml = self.xml_tmpl % ( + uses_library_xml('foo') + uses_library_xml('missing') + uses_library_xml('bar')) + apk = self.apk_tmpl % ( + uses_library_apk('foo') + uses_library_apk('missing') + uses_library_apk('bar')) + matches = self.run_test(xml, apk, optional_uses_libraries=['foo', 'bar'], + missing_optional_uses_libraries=['missing']) + self.assertFalse(matches) + def test_missing_uses_library(self): xml = self.xml_tmpl % ('') apk = self.apk_tmpl % ('') diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index 275860f32..0a5483b07 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -688,6 +688,12 @@ func TestSnapshotWithJavaSystemModules(t *testing.T) { public: { enabled: true, }, + system: { + enabled: true, + }, + module_lib: { + enabled: true, + }, } java_system_modules { @@ -752,6 +758,20 @@ java_sdk_library_import { removed_api: "sdk_library/public/myjavalib-removed.txt", sdk_version: "current", }, + system: { + jars: ["sdk_library/system/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], + current_api: "sdk_library/system/myjavalib.txt", + removed_api: "sdk_library/system/myjavalib-removed.txt", + sdk_version: "system_current", + }, + module_lib: { + jars: ["sdk_library/module-lib/myjavalib-stubs.jar"], + stub_srcs: ["sdk_library/module-lib/myjavalib_stub_sources"], + current_api: "sdk_library/module-lib/myjavalib.txt", + removed_api: "sdk_library/module-lib/myjavalib-removed.txt", + sdk_version: "module_current", + }, } java_system_modules_import { @@ -771,6 +791,12 @@ java_system_modules_import { .intermediates/myjavalib.stubs.exportable/android_common/combined/myjavalib.stubs.exportable.jar -> sdk_library/public/myjavalib-stubs.jar .intermediates/myjavalib.stubs.source/android_common/exportable/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt .intermediates/myjavalib.stubs.source/android_common/exportable/myjavalib.stubs.source_removed.txt -> sdk_library/public/myjavalib-removed.txt +.intermediates/myjavalib.stubs.exportable.system/android_common/combined/myjavalib.stubs.exportable.system.jar -> sdk_library/system/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source.system/android_common/exportable/myjavalib.stubs.source.system_api.txt -> sdk_library/system/myjavalib.txt +.intermediates/myjavalib.stubs.source.system/android_common/exportable/myjavalib.stubs.source.system_removed.txt -> sdk_library/system/myjavalib-removed.txt +.intermediates/myjavalib.stubs.exportable.module_lib/android_common/combined/myjavalib.stubs.exportable.module_lib.jar -> sdk_library/module-lib/myjavalib-stubs.jar +.intermediates/myjavalib.stubs.source.module_lib/android_common/exportable/myjavalib.stubs.source.module_lib_api.txt -> sdk_library/module-lib/myjavalib.txt +.intermediates/myjavalib.stubs.source.module_lib/android_common/exportable/myjavalib.stubs.source.module_lib_removed.txt -> sdk_library/module-lib/myjavalib-removed.txt `), checkInfoContents(result.Config, ` [ @@ -805,11 +831,23 @@ java_system_modules_import { "@name": "myjavalib", "dist_stem": "myjavalib", "scopes": { + "module-lib": { + "current_api": "sdk_library/module-lib/myjavalib.txt", + "latest_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib.api.module-lib.latest/gen/myjavalib.api.module-lib.latest", + "latest_removed_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib-removed.api.module-lib.latest/gen/myjavalib-removed.api.module-lib.latest", + "removed_api": "sdk_library/module-lib/myjavalib-removed.txt" + }, "public": { "current_api": "sdk_library/public/myjavalib.txt", "latest_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib.api.public.latest/gen/myjavalib.api.public.latest", "latest_removed_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib-removed.api.public.latest/gen/myjavalib-removed.api.public.latest", "removed_api": "sdk_library/public/myjavalib-removed.txt" + }, + "system": { + "current_api": "sdk_library/system/myjavalib.txt", + "latest_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib.api.system.latest/gen/myjavalib.api.system.latest", + "latest_removed_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib-removed.api.system.latest/gen/myjavalib-removed.api.system.latest", + "removed_api": "sdk_library/system/myjavalib-removed.txt" } } }, diff --git a/soong_ui.bash b/soong_ui.bash index 8e7cd195d..77378807f 100755 --- a/soong_ui.bash +++ b/soong_ui.bash @@ -35,6 +35,7 @@ source ${TOP}/build/soong/scripts/microfactory.bash soong_build_go soong_ui android/soong/cmd/soong_ui soong_build_go mk2rbc android/soong/mk2rbc/mk2rbc soong_build_go rbcrun rbcrun/rbcrun +soong_build_go release-config android/soong/cmd/release_config/release_config cd ${TOP} exec "$(getoutdir)/soong_ui" "$@" diff --git a/tradefed_modules/test_module_config.go b/tradefed_modules/test_module_config.go index 686753713..9127f67a9 100644 --- a/tradefed_modules/test_module_config.go +++ b/tradefed_modules/test_module_config.go @@ -297,10 +297,16 @@ func (m *testModuleConfigModule) validateBase(ctx android.ModuleContext, depTag // 1. manifest file to testcases dir // 2. New Module.config / AndroidTest.xml file with our options. func (m *testModuleConfigModule) generateManifestAndConfig(ctx android.ModuleContext) { + // Keep before early returns. + android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{ + TestOnly: true, + TopLevelTarget: true, + }) + if !m.validateTestSuites(ctx) { return } - // Ensure the provider is accurate + // Ensure the base provider is accurate if m.provider.TestConfig == nil { return } diff --git a/tradefed_modules/test_module_config_test.go b/tradefed_modules/test_module_config_test.go index 41dd3d479..6997228ce 100644 --- a/tradefed_modules/test_module_config_test.go +++ b/tradefed_modules/test_module_config_test.go @@ -19,6 +19,8 @@ import ( "strconv" "strings" "testing" + + "github.com/google/blueprint" ) const bp = ` @@ -347,6 +349,67 @@ func TestModuleConfigHostDuplicateTestSuitesGiveErrors(t *testing.T) { RunTestWithBp(t, badBp) } +func TestTestOnlyProvider(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + java.PrepareForTestWithJavaDefaultModules, + android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), + ).RunTestWithBp(t, ` + // These should be test-only + test_module_config_host { + name: "host-derived-test", + base: "host-base", + exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], + include_annotations: ["android.platform.test.annotations.LargeTest"], + test_suites: ["general-tests"], + } + + test_module_config { + name: "derived-test", + base: "base", + exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], + include_annotations: ["android.platform.test.annotations.LargeTest"], + test_suites: ["general-tests"], + } + + android_test { + name: "base", + sdk_version: "current", + data: ["data/testfile"], + } + + java_test_host { + name: "host-base", + srcs: ["a.java"], + test_suites: ["general-tests"], + }`, + ) + + // Visit all modules and ensure only the ones that should + // marked as test-only are marked as test-only. + + actualTestOnly := []string{} + ctx.VisitAllModules(func(m blueprint.Module) { + if provider, ok := android.OtherModuleProvider(ctx.TestContext.OtherModuleProviderAdaptor(), m, android.TestOnlyProviderKey); ok { + if provider.TestOnly { + actualTestOnly = append(actualTestOnly, m.Name()) + } + } + }) + expectedTestOnlyModules := []string{ + "host-derived-test", + "derived-test", + // android_test and java_test_host are tests too. + "host-base", + "base", + } + + notEqual, left, right := android.ListSetDifference(expectedTestOnlyModules, actualTestOnly) + if notEqual { + t.Errorf("test-only: Expected but not found: %v, Found but not expected: %v", left, right) + } +} + // Use for situations where the entries map contains pairs: [srcPath:installedPath1, srcPath2:installedPath2] // and we want to compare the RHS of the pairs, i.e. installedPath1, installedPath2 func assertEntryPairValues(t *testing.T, actual []string, expected []string) { |