diff options
99 files changed, 4709 insertions, 959 deletions
diff --git a/Android.bp b/Android.bp index cbe1c7b3f..42b7d83f7 100644 --- a/Android.bp +++ b/Android.bp @@ -206,3 +206,30 @@ build_prop { relative_install_path: "etc", // odm/etc/build.prop visibility: ["//visibility:private"], } + +build_prop { + name: "system_dlkm-build.prop", + stem: "build.prop", + system_dlkm_specific: true, + product_config: ":product_config", + relative_install_path: "etc", // system_dlkm/etc/build.prop + visibility: ["//visibility:private"], +} + +build_prop { + name: "vendor_dlkm-build.prop", + stem: "build.prop", + vendor_dlkm_specific: true, + product_config: ":product_config", + relative_install_path: "etc", // vendor_dlkm/etc/build.prop + visibility: ["//visibility:private"], +} + +build_prop { + name: "odm_dlkm-build.prop", + stem: "build.prop", + odm_dlkm_specific: true, + product_config: ":product_config", + relative_install_path: "etc", // odm_dlkm/etc/build.prop + visibility: ["//visibility:private"], +} diff --git a/android/Android.bp b/android/Android.bp index 1ed2dbaa1..a9a3564ab 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -31,6 +31,7 @@ bootstrap_go_package { srcs: [ "aconfig_providers.go", "all_teams.go", + "android_info.go", "androidmk.go", "apex.go", "apex_contributions.go", @@ -111,6 +112,7 @@ bootstrap_go_package { "util.go", "variable.go", "vintf_fragment.go", + "vintf_data.go", "visibility.go", ], testSrcs: [ @@ -120,6 +122,7 @@ bootstrap_go_package { "apex_test.go", "arch_test.go", "blueprint_e2e_test.go", + "build_prop_test.go", "config_test.go", "configured_jars_test.go", "csuite_config_test.go", diff --git a/android/android_info.go b/android/android_info.go new file mode 100644 index 000000000..a8d3d4e2c --- /dev/null +++ b/android/android_info.go @@ -0,0 +1,91 @@ +// Copyright 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" +) + +var ( + mergeAndRemoveComments = pctx.AndroidStaticRule("merge_and_remove_comments", + blueprint.RuleParams{ + Command: "cat $in | grep -v '#' > $out", + }, + ) + androidInfoTxtToProp = pctx.AndroidStaticRule("android_info_txt_to_prop", + blueprint.RuleParams{ + Command: "grep 'require version-' $in | sed -e 's/require version-/ro.build.expect./g' > $out", + }, + ) +) + +type androidInfoProperties struct { + // Name of output file. Defaults to module name + Stem *string + + // Paths of board-info.txt files. + Board_info_files []string `android:"path"` + + // Name of bootloader board. If board_info_files is empty, `board={bootloader_board_name}` will + // be printed to output. Ignored if board_info_files is not empty. + Bootloader_board_name *string +} + +type androidInfoModule struct { + ModuleBase + + properties androidInfoProperties +} + +func (p *androidInfoModule) GenerateAndroidBuildActions(ctx ModuleContext) { + if len(p.properties.Board_info_files) > 0 && p.properties.Bootloader_board_name != nil { + ctx.ModuleErrorf("Either Board_info_files or Bootloader_board_name should be set. Please remove one of them\n") + return + } + androidInfoTxtName := proptools.StringDefault(p.properties.Stem, ctx.ModuleName()+".txt") + androidInfoTxt := PathForModuleOut(ctx, androidInfoTxtName) + androidInfoProp := androidInfoTxt.ReplaceExtension(ctx, "prop") + + if boardInfoFiles := PathsForModuleSrc(ctx, p.properties.Board_info_files); len(boardInfoFiles) > 0 { + ctx.Build(pctx, BuildParams{ + Rule: mergeAndRemoveComments, + Inputs: boardInfoFiles, + Output: androidInfoTxt, + }) + } else if bootloaderBoardName := proptools.String(p.properties.Bootloader_board_name); bootloaderBoardName != "" { + WriteFileRule(ctx, androidInfoTxt, "board="+bootloaderBoardName) + } else { + WriteFileRule(ctx, androidInfoTxt, "") + } + + // Create android_info.prop + ctx.Build(pctx, BuildParams{ + Rule: androidInfoTxtToProp, + Input: androidInfoTxt, + Output: androidInfoProp, + }) + + ctx.SetOutputFiles(Paths{androidInfoProp}, "") +} + +// android_info module generate a file named android-info.txt that contains various information +// about the device we're building for. This file is typically packaged up with everything else. +func AndroidInfoFactory() Module { + module := &androidInfoModule{} + module.AddProperties(&module.properties) + InitAndroidModule(module) + return module +} diff --git a/android/apex.go b/android/apex.go index e90248e79..3486350d5 100644 --- a/android/apex.go +++ b/android/apex.go @@ -812,7 +812,7 @@ func UpdateDirectlyInAnyApex(mctx BottomUpMutatorContext, am ApexModule) { // If this is the FinalModule (last visited module) copy // AnyVariantDirectlyInAnyApex to all the other variants - if am == mctx.FinalModule().(ApexModule) { + if mctx.IsFinalModule(am) { mctx.VisitAllModuleVariants(func(variant Module) { variant.(ApexModule).apexModuleBase().ApexProperties.AnyVariantDirectlyInAnyApex = base.ApexProperties.AnyVariantDirectlyInAnyApex diff --git a/android/apex_contributions.go b/android/apex_contributions.go index 4cd8dda93..ce3427840 100644 --- a/android/apex_contributions.go +++ b/android/apex_contributions.go @@ -26,7 +26,7 @@ func init() { func RegisterApexContributionsBuildComponents(ctx RegistrationContext) { ctx.RegisterModuleType("apex_contributions", apexContributionsFactory) ctx.RegisterModuleType("apex_contributions_defaults", apexContributionsDefaultsFactory) - ctx.RegisterSingletonModuleType("all_apex_contributions", allApexContributionsFactory) + ctx.RegisterModuleType("all_apex_contributions", allApexContributionsFactory) } type apexContributions struct { @@ -87,10 +87,10 @@ func apexContributionsDefaultsFactory() Module { // Based on product_config, it will create a dependency on the selected // apex_contributions per mainline module type allApexContributions struct { - SingletonModuleBase + ModuleBase } -func allApexContributionsFactory() SingletonModule { +func allApexContributionsFactory() Module { module := &allApexContributions{} InitAndroidModule(module) return module @@ -191,7 +191,7 @@ func (p *PrebuiltSelectionInfoMap) GetSelectedModulesForApiDomain(apiDomain stri // This module type does not have any build actions. func (a *allApexContributions) GenerateAndroidBuildActions(ctx ModuleContext) { -} - -func (a *allApexContributions) GenerateSingletonBuildActions(ctx SingletonContext) { + if ctx.ModuleName() != "all_apex_contributions" { + ctx.ModuleErrorf("There can only be 1 all_apex_contributions module in build/soong") + } } diff --git a/android/base_module_context.go b/android/base_module_context.go index 7fa58766f..060fae5bc 100644 --- a/android/base_module_context.go +++ b/android/base_module_context.go @@ -88,6 +88,10 @@ type BaseModuleContext interface { // This method shouldn't be used directly, prefer the type-safe android.OtherModuleProvider instead. otherModuleProvider(m blueprint.Module, provider blueprint.AnyProviderKey) (any, bool) + // OtherModuleIsAutoGenerated returns true if the module is auto generated by another module + // instead of being defined in Android.bp file. + OtherModuleIsAutoGenerated(m blueprint.Module) bool + // Provider returns the value for a provider for the current module. If the value is // not set it returns nil and false. It panics if called before the appropriate // mutator or GenerateBuildActions pass for the provider. The value returned may be a deep @@ -194,9 +198,15 @@ type BaseModuleContext interface { // singleton actions that are only done once for all variants of a module. FinalModule() Module + // IsFinalModule returns if the current module is the last variant. Variants of a module are always visited in + // order by mutators and GenerateBuildActions, so the data created by the current mutator can be read from all + // variants using VisitAllModuleVariants if the current module is the last one. This can be used to perform + // singleton actions that are only done once for all variants of a module. + IsFinalModule(module Module) bool + // VisitAllModuleVariants calls visit for each variant of the current module. Variants of a module are always // visited in order by mutators and GenerateBuildActions, so the data created by the current mutator can be read - // from all variants if the current module == FinalModule(). Otherwise, care must be taken to not access any + // from all variants if the current module is the last one. Otherwise, care must be taken to not access any // data modified by the current mutator. VisitAllModuleVariants(visit func(Module)) @@ -276,6 +286,10 @@ func (b *baseModuleContext) otherModuleProvider(m blueprint.Module, provider blu return b.bp.OtherModuleProvider(m, provider) } +func (b *baseModuleContext) OtherModuleIsAutoGenerated(m blueprint.Module) bool { + return b.bp.OtherModuleIsAutoGenerated(m) +} + func (b *baseModuleContext) provider(provider blueprint.AnyProviderKey) (any, bool) { return b.bp.Provider(provider) } @@ -584,6 +598,10 @@ func (b *baseModuleContext) FinalModule() Module { return b.bp.FinalModule().(Module) } +func (b *baseModuleContext) IsFinalModule(module Module) bool { + return b.bp.IsFinalModule(module) +} + // IsMetaDependencyTag returns true for cross-cutting metadata dependencies. func IsMetaDependencyTag(tag blueprint.DependencyTag) bool { if tag == licenseKindTag { diff --git a/android/build_prop.go b/android/build_prop.go index 7c3c50645..838947045 100644 --- a/android/build_prop.go +++ b/android/build_prop.go @@ -19,8 +19,12 @@ import ( ) func init() { - ctx := InitRegistrationContext + registerBuildPropComponents(InitRegistrationContext) +} + +func registerBuildPropComponents(ctx RegistrationContext) { ctx.RegisterModuleType("build_prop", BuildPropFactory) + ctx.RegisterModuleType("android_info", AndroidInfoFactory) } type buildPropProperties struct { @@ -38,6 +42,10 @@ type buildPropProperties struct { // Path to a JSON file containing product configs. Product_config *string `android:"path"` + // Path to android-info.txt file containing board specific info. + // This is empty for build.prop of all partitions except vendor. + Android_info *string `android:"path"` + // Optional subdirectory under which this file is installed into Relative_install_path *string } @@ -66,7 +74,10 @@ func (p *buildPropModule) propFiles(ctx ModuleContext) Paths { } else if partition == "odm" { return ctx.Config().OdmPropFiles(ctx) } else if partition == "vendor" { - // TODO (b/375500423): Add android-info.txt to prop files + if p.properties.Android_info != nil { + androidInfo := PathForModuleSrc(ctx, proptools.String(p.properties.Android_info)) + return append(ctx.Config().VendorPropFiles(ctx), androidInfo) + } return ctx.Config().VendorPropFiles(ctx) } return nil @@ -98,6 +109,12 @@ func (p *buildPropModule) partition(config DeviceConfig) string { return "product" } else if p.SystemExtSpecific() { return "system_ext" + } else if p.InstallInSystemDlkm() { + return "system_dlkm" + } else if p.InstallInVendorDlkm() { + return "vendor_dlkm" + } else if p.InstallInOdmDlkm() { + return "odm_dlkm" } return "system" } @@ -108,16 +125,18 @@ var validPartitions = []string{ "product", "odm", "vendor", + "system_dlkm", + "vendor_dlkm", + "odm_dlkm", } func (p *buildPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { - p.outputFilePath = PathForModuleOut(ctx, "build.prop").OutputPath - if !ctx.Config().KatiEnabled() { - WriteFileRule(ctx, p.outputFilePath, "# no build.prop if kati is disabled") - ctx.SetOutputFiles(Paths{p.outputFilePath}, "") - return + if !p.SocSpecific() && p.properties.Android_info != nil { + ctx.ModuleErrorf("Android_info cannot be set if build.prop is not installed in vendor partition") } + p.outputFilePath = PathForModuleOut(ctx, "build.prop").OutputPath + partition := p.partition(ctx.DeviceConfig()) if !InList(partition, validPartitions) { ctx.PropertyErrorf("partition", "unsupported partition %q: only %q are supported", partition, validPartitions) diff --git a/android/build_prop_test.go b/android/build_prop_test.go new file mode 100644 index 000000000..e75975a0a --- /dev/null +++ b/android/build_prop_test.go @@ -0,0 +1,41 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "testing" +) + +func TestPropFileInputs(t *testing.T) { + bp := ` +build_prop { + name: "vendor-build.prop", + stem: "build.prop", + vendor: true, + android_info: ":board-info", + //product_config: ":product_config", +} +android_info { + name: "board-info", + stem: "android-info.txt", +} +` + + res := GroupFixturePreparers( + FixtureRegisterWithContext(registerBuildPropComponents), + ).RunTestWithBp(t, bp) + buildPropCmd := res.ModuleForTests("vendor-build.prop", "").Rule("vendor-build.prop_.vendor-build.prop").RuleParams.Command + AssertStringDoesContain(t, "Could not find android-info in prop files of vendor build.prop", buildPropCmd, "--prop-files=out/soong/.intermediates/board-info/android-info.prop") +} diff --git a/android/config.go b/android/config.go index 380cd1c60..27d3b87ba 100644 --- a/android/config.go +++ b/android/config.go @@ -413,18 +413,21 @@ type jsonConfigurable interface { // To add a new feature to the list, add the field in the struct // `partialCompileFlags` above, and then add the name of the field in the // switch statement below. -func (c *config) parsePartialCompileFlags() (partialCompileFlags, error) { - defaultFlags := partialCompileFlags{ - // Set any opt-out flags here. Opt-in flags are off by default. - enabled: false, +var defaultPartialCompileFlags = partialCompileFlags{ + // Set any opt-out flags here. Opt-in flags are off by default. + enabled: false, +} + +func (c *config) parsePartialCompileFlags(isEngBuild bool) (partialCompileFlags, error) { + if !isEngBuild { + return partialCompileFlags{}, nil } value := c.Getenv("SOONG_PARTIAL_COMPILE") - if value == "" { - return defaultFlags, nil + return defaultPartialCompileFlags, nil } - ret := defaultFlags + ret := defaultPartialCompileFlags tokens := strings.Split(strings.ToLower(value), ",") makeVal := func(state string, defaultValue bool) bool { switch state { @@ -455,17 +458,17 @@ func (c *config) parsePartialCompileFlags() (partialCompileFlags, error) { } switch tok { case "true": - ret = defaultFlags + ret = defaultPartialCompileFlags ret.enabled = true case "false": // Set everything to false. ret = partialCompileFlags{} case "enabled": - ret.enabled = makeVal(state, defaultFlags.enabled) + ret.enabled = makeVal(state, defaultPartialCompileFlags.enabled) case "use_d8": - ret.use_d8 = makeVal(state, defaultFlags.use_d8) + ret.use_d8 = makeVal(state, defaultPartialCompileFlags.use_d8) default: - return partialCompileFlags{}, fmt.Errorf("Unknown SOONG_PARTIAL_COMPILE value: %v", value) + return partialCompileFlags{}, fmt.Errorf("Unknown SOONG_PARTIAL_COMPILE value: %v", tok) } } return ret, nil @@ -616,6 +619,8 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) buildFromSourceStub: cmdArgs.BuildFromSourceStub, } + variant, ok := os.LookupEnv("TARGET_BUILD_VARIANT") + isEngBuild := !ok || variant == "eng" config.deviceConfig = &deviceConfig{ config: config, @@ -657,7 +662,7 @@ func NewConfig(cmdArgs CmdArgs, availableEnv map[string]string) (Config, error) return Config{}, err } - config.partialCompileFlags, err = config.parsePartialCompileFlags() + config.partialCompileFlags, err = config.parsePartialCompileFlags(isEngBuild) if err != nil { return Config{}, err } @@ -862,7 +867,7 @@ func (c *config) IsEnvFalse(key string) bool { } func (c *config) TargetsJava21() bool { - return c.IsEnvTrue("EXPERIMENTAL_TARGET_JAVA_VERSION_21") + return c.productVariables.GetBuildFlagBool("RELEASE_TARGET_JAVA_21") } // EnvDeps returns the environment variables this build depends on. The first @@ -1517,6 +1522,13 @@ func (c *deviceConfig) VendorPath() string { return "vendor" } +func (c *deviceConfig) VendorDlkmPath() string { + if c.config.productVariables.VendorDlkmPath != nil { + return *c.config.productVariables.VendorDlkmPath + } + return "vendor_dlkm" +} + func (c *deviceConfig) BuildingVendorImage() bool { return proptools.Bool(c.config.productVariables.BuildingVendorImage) } @@ -1544,6 +1556,17 @@ func (c *deviceConfig) OdmPath() string { return "odm" } +func (c *deviceConfig) BuildingOdmImage() bool { + return proptools.Bool(c.config.productVariables.BuildingOdmImage) +} + +func (c *deviceConfig) OdmDlkmPath() string { + if c.config.productVariables.OdmDlkmPath != nil { + return *c.config.productVariables.OdmDlkmPath + } + return "odm_dlkm" +} + func (c *deviceConfig) ProductPath() string { if c.config.productVariables.ProductPath != nil { return *c.config.productVariables.ProductPath @@ -1562,6 +1585,31 @@ func (c *deviceConfig) SystemExtPath() string { return "system_ext" } +func (c *deviceConfig) SystemDlkmPath() string { + if c.config.productVariables.SystemDlkmPath != nil { + return *c.config.productVariables.SystemDlkmPath + } + return "system_dlkm" +} + +func (c *deviceConfig) OemPath() string { + if c.config.productVariables.OemPath != nil { + return *c.config.productVariables.OemPath + } + return "oem" +} + +func (c *deviceConfig) UserdataPath() string { + if c.config.productVariables.UserdataPath != nil { + return *c.config.productVariables.UserdataPath + } + return "data" +} + +func (c *deviceConfig) BuildingUserdataImage() bool { + return proptools.Bool(c.config.productVariables.BuildingUserdataImage) +} + func (c *deviceConfig) BtConfigIncludeDir() string { return String(c.config.productVariables.BtConfigIncludeDir) } @@ -2225,3 +2273,27 @@ func (c *config) BoardAvbSystemAddHashtreeFooterArgs() []string { func (c Config) InstallApexSystemServerDexpreoptSamePartition() bool { return c.config.productVariables.GetBuildFlagBool("RELEASE_INSTALL_APEX_SYSTEMSERVER_DEXPREOPT_SAME_PARTITION") } + +func (c *config) DeviceMatrixFile() []string { + return c.productVariables.DeviceMatrixFile +} + +func (c *config) ProductManifestFiles() []string { + return c.productVariables.ProductManifestFiles +} + +func (c *config) SystemManifestFile() []string { + return c.productVariables.SystemManifestFile +} + +func (c *config) SystemExtManifestFiles() []string { + return c.productVariables.SystemExtManifestFiles +} + +func (c *config) DeviceManifestFiles() []string { + return c.productVariables.DeviceManifestFiles +} + +func (c *config) OdmManifestFiles() []string { + return c.productVariables.OdmManifestFiles +} diff --git a/android/config_test.go b/android/config_test.go index 773216844..adb5ffac5 100644 --- a/android/config_test.go +++ b/android/config_test.go @@ -212,3 +212,48 @@ func TestConfiguredJarList(t *testing.T) { assertStringEquals(t, "apex1:jarA", list5.String()) }) } + +func (p partialCompileFlags) updateEnabled(value bool) partialCompileFlags { + p.enabled = value + return p +} + +func (p partialCompileFlags) updateUseD8(value bool) partialCompileFlags { + p.use_d8 = value + return p +} + +func TestPartialCompile(t *testing.T) { + mockConfig := func(value string) *config { + c := &config{ + env: map[string]string{ + "SOONG_PARTIAL_COMPILE": value, + }, + } + return c + } + tests := []struct { + value string + isEngBuild bool + expected partialCompileFlags + }{ + {"", true, defaultPartialCompileFlags}, + {"false", true, partialCompileFlags{}}, + {"true", true, defaultPartialCompileFlags.updateEnabled(true)}, + {"true", false, partialCompileFlags{}}, + {"true,use_d8", true, defaultPartialCompileFlags.updateEnabled(true).updateUseD8(true)}, + {"true,-use_d8", true, defaultPartialCompileFlags.updateEnabled(true).updateUseD8(false)}, + {"use_d8,false", true, partialCompileFlags{}}, + {"false,+use_d8", true, partialCompileFlags{}.updateUseD8(true)}, + } + + for _, test := range tests { + t.Run(test.value, func(t *testing.T) { + config := mockConfig(test.value) + flags, _ := config.parsePartialCompileFlags(test.isEngBuild) + if flags != test.expected { + t.Errorf("expected %v found %v", test.expected, flags) + } + }) + } +} diff --git a/android/dirgroup.go b/android/dirgroup.go index 20c4d13a6..62fbaa5c8 100644 --- a/android/dirgroup.go +++ b/android/dirgroup.go @@ -39,8 +39,7 @@ type dirGroup struct { } type DirInfo struct { - // TODO(b/358302178): Use DirectoryPaths instead of Paths - Dirs Paths + Dirs DirectoryPaths } var DirProvider = blueprint.NewProvider[DirInfo]() diff --git a/android/filegroup_test.go b/android/filegroup_test.go index 14e9368ca..670037d56 100644 --- a/android/filegroup_test.go +++ b/android/filegroup_test.go @@ -1,55 +1,9 @@ package android import ( - "path/filepath" "testing" ) -func TestFileGroupWithPathProp(t *testing.T) { - // TODO(b/247782695), TODO(b/242847534) Fix mixed builds for filegroups - t.Skip("Re-enable once filegroups are corrected for mixed builds") - outBaseDir := "outputbase" - pathPrefix := outBaseDir + "/execroot/__main__" - expectedOutputfile := filepath.Join(pathPrefix, "a/b/c/d/test.aidl") - - testCases := []struct { - bp string - rel string - }{ - { - bp: ` - filegroup { - name: "baz", - srcs: ["a/b/c/d/test.aidl"], - path: "a/b", - bazel_module: { label: "//:baz" }, - } -`, - rel: "c/d/test.aidl", - }, - { - bp: ` - filegroup { - name: "baz", - srcs: ["a/b/c/d/test.aidl"], - bazel_module: { label: "//:baz" }, - } -`, - rel: "a/b/c/d/test.aidl", - }, - } - - for _, testCase := range testCases { - result := GroupFixturePreparers( - PrepareForTestWithFilegroup, - ).RunTestWithBp(t, testCase.bp) - - fg := result.Module("baz", "").(*fileGroup) - AssertStringEquals(t, "src relativeRoot", testCase.rel, fg.srcs[0].Rel()) - AssertStringEquals(t, "src full path", expectedOutputfile, fg.srcs[0].String()) - } -} - func TestFilegroupDefaults(t *testing.T) { bp := FixtureAddTextFile("p/Android.bp", ` filegroup_defaults { diff --git a/android/init.go b/android/init.go index 1ace34494..d3a13d0ed 100644 --- a/android/init.go +++ b/android/init.go @@ -20,6 +20,7 @@ func init() { gob.Register(extraFilesZip{}) gob.Register(InstallPath{}) gob.Register(ModuleGenPath{}) + gob.Register(ModuleObjPath{}) gob.Register(ModuleOutPath{}) gob.Register(OutputPath{}) gob.Register(PhonyPath{}) diff --git a/android/module.go b/android/module.go index 114843056..58ae885c8 100644 --- a/android/module.go +++ b/android/module.go @@ -81,10 +81,14 @@ type Module interface { InstallInProduct() bool InstallInVendor() bool InstallInSystemExt() bool + InstallInSystemDlkm() bool + InstallInVendorDlkm() bool + InstallInOdmDlkm() bool InstallForceOS() (*OsType, *ArchType) PartitionTag(DeviceConfig) string HideFromMake() IsHideFromMake() bool + SkipInstall() IsSkipInstall() bool MakeUninstallable() ReplacedByPrebuilt() @@ -385,6 +389,15 @@ type commonProperties struct { // Whether this module is installed to debug ramdisk Debug_ramdisk *bool + // Install to partition system_dlkm when set to true. + System_dlkm_specific *bool + + // Install to partition vendor_dlkm when set to true. + Vendor_dlkm_specific *bool + + // Install to partition odm_dlkm when set to true. + Odm_dlkm_specific *bool + // Whether this module is built for non-native architectures (also known as native bridge binary) Native_bridge_supported *bool `android:"arch_variant"` @@ -1421,6 +1434,7 @@ func (m *ModuleBase) IsSkipInstall() bool { func (m *ModuleBase) MakeUninstallable() { m.commonProperties.UninstallableApexPlatformVariant = true m.HideFromMake() + m.SkipInstall() } func (m *ModuleBase) ReplacedByPrebuilt() { @@ -1533,6 +1547,18 @@ func (m *ModuleBase) InstallInRoot() bool { return false } +func (m *ModuleBase) InstallInSystemDlkm() bool { + return Bool(m.commonProperties.System_dlkm_specific) +} + +func (m *ModuleBase) InstallInVendorDlkm() bool { + return Bool(m.commonProperties.Vendor_dlkm_specific) +} + +func (m *ModuleBase) InstallInOdmDlkm() bool { + return Bool(m.commonProperties.Odm_dlkm_specific) +} + func (m *ModuleBase) InstallForceOS() (*OsType, *ArchType) { return nil, nil } @@ -1820,6 +1846,8 @@ type CommonPropertiesProviderData struct { Enabled bool // Whether the module has been replaced by a prebuilt ReplacedByPrebuilt bool + // The Target of artifacts that this module variant is responsible for creating. + CompileTarget Target } var CommonPropertiesProviderKey = blueprint.NewProvider[CommonPropertiesProviderData]() @@ -2008,7 +2036,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) ctx.GetMissingDependencies() } - if m == ctx.FinalModule().(Module).base() { + if ctx.IsFinalModule(m.module) { m.generateModuleTarget(ctx) if ctx.Failed() { return @@ -2084,6 +2112,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) commonData := CommonPropertiesProviderData{ ReplacedByPrebuilt: m.commonProperties.ReplacedByPrebuilt, + CompileTarget: m.commonProperties.CompileTarget, } if m.commonProperties.ForcedDisabled { commonData.Enabled = false diff --git a/android/module_context.go b/android/module_context.go index 9fa3a6211..41cb0ccb2 100644 --- a/android/module_context.go +++ b/android/module_context.go @@ -19,6 +19,7 @@ import ( "github.com/google/blueprint/depset" "path" "path/filepath" + "slices" "strings" "github.com/google/blueprint" @@ -195,6 +196,9 @@ type ModuleContext interface { InstallInOdm() bool InstallInProduct() bool InstallInVendor() bool + InstallInSystemDlkm() bool + InstallInVendorDlkm() bool + InstallInOdmDlkm() bool InstallForceOS() (*OsType, *ArchType) RequiredModuleNames(ctx ConfigurableEvaluatorContext) []string @@ -492,15 +496,23 @@ func (m *moduleContext) InstallInVendor() bool { return m.module.InstallInVendor() } +func (m *moduleContext) InstallInSystemDlkm() bool { + return m.module.InstallInSystemDlkm() +} + +func (m *moduleContext) InstallInVendorDlkm() bool { + return m.module.InstallInVendorDlkm() +} + +func (m *moduleContext) InstallInOdmDlkm() bool { + return m.module.InstallInOdmDlkm() +} + func (m *moduleContext) skipInstall() bool { if m.module.base().commonProperties.SkipInstall { return true } - if m.module.base().commonProperties.HideFromMake { - return true - } - // We'll need a solution for choosing which of modules with the same name in different // namespaces to install. For now, reuse the list of namespaces exported to Make as the // list of namespaces to install in a Soong-only build. @@ -519,6 +531,10 @@ func (m *moduleContext) requiresFullInstall() bool { return false } + if m.module.base().commonProperties.HideFromMake { + return false + } + if proptools.Bool(m.module.base().commonProperties.No_full_install) { return false } @@ -562,9 +578,22 @@ func (m *moduleContext) setAconfigPaths(paths Paths) { m.aconfigFilePaths = paths } +func (m *moduleContext) getOwnerAndOverrides() (string, []string) { + owner := m.ModuleName() + overrides := slices.Clone(m.Module().base().commonProperties.Overrides) + if b, ok := m.Module().(OverridableModule); ok { + if b.GetOverriddenBy() != "" { + // overriding variant of base module + overrides = append(overrides, m.ModuleName()) // com.android.foo + owner = m.Module().Name() // com.company.android.foo + } + } + return owner, overrides +} + func (m *moduleContext) packageFile(fullInstallPath InstallPath, srcPath Path, executable bool) PackagingSpec { licenseFiles := m.Module().EffectiveLicenseFiles() - overrides := CopyOf(m.Module().base().commonProperties.Overrides) + owner, overrides := m.getOwnerAndOverrides() spec := PackagingSpec{ relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), srcPath: srcPath, @@ -576,7 +605,7 @@ func (m *moduleContext) packageFile(fullInstallPath InstallPath, srcPath Path, e aconfigPaths: m.getAconfigPaths(), archType: m.target.Arch.ArchType, overrides: &overrides, - owner: m.ModuleName(), + owner: owner, } m.packagingSpecs = append(m.packagingSpecs, spec) return spec @@ -695,7 +724,7 @@ func (m *moduleContext) InstallSymlink(installPath InstallPath, name string, src m.installFiles = append(m.installFiles, fullInstallPath) } - overrides := CopyOf(m.Module().base().commonProperties.Overrides) + owner, overrides := m.getOwnerAndOverrides() m.packagingSpecs = append(m.packagingSpecs, PackagingSpec{ relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), srcPath: nil, @@ -706,7 +735,7 @@ func (m *moduleContext) InstallSymlink(installPath InstallPath, name string, src aconfigPaths: m.getAconfigPaths(), archType: m.target.Arch.ArchType, overrides: &overrides, - owner: m.ModuleName(), + owner: owner, }) return fullInstallPath @@ -742,7 +771,7 @@ func (m *moduleContext) InstallAbsoluteSymlink(installPath InstallPath, name str m.installFiles = append(m.installFiles, fullInstallPath) } - overrides := CopyOf(m.Module().base().commonProperties.Overrides) + owner, overrides := m.getOwnerAndOverrides() m.packagingSpecs = append(m.packagingSpecs, PackagingSpec{ relPathInPackage: Rel(m, fullInstallPath.PartitionDir(), fullInstallPath.String()), srcPath: nil, @@ -753,7 +782,7 @@ func (m *moduleContext) InstallAbsoluteSymlink(installPath InstallPath, name str aconfigPaths: m.getAconfigPaths(), archType: m.target.Arch.ArchType, overrides: &overrides, - owner: m.ModuleName(), + owner: owner, }) return fullInstallPath diff --git a/android/module_proxy.go b/android/module_proxy.go index 2a650729f..1f9679926 100644 --- a/android/module_proxy.go +++ b/android/module_proxy.go @@ -106,6 +106,18 @@ func (m ModuleProxy) InstallInSystemExt() bool { panic("method is not implemented on ModuleProxy") } +func (m ModuleProxy) InstallInSystemDlkm() bool { + panic("method is not implemented on ModuleProxy") +} + +func (m ModuleProxy) InstallInVendorDlkm() bool { + panic("method is not implemented on ModuleProxy") +} + +func (m ModuleProxy) InstallInOdmDlkm() bool { + panic("method is not implemented on ModuleProxy") +} + func (m ModuleProxy) InstallForceOS() (*OsType, *ArchType) { panic("method is not implemented on ModuleProxy") } @@ -122,6 +134,10 @@ func (m ModuleProxy) IsHideFromMake() bool { panic("method is not implemented on ModuleProxy") } +func (m ModuleProxy) SkipInstall() { + panic("method is not implemented on ModuleProxy") +} + func (m ModuleProxy) IsSkipInstall() bool { panic("method is not implemented on ModuleProxy") } diff --git a/android/mutator.go b/android/mutator.go index 4ddc606b1..fdd16a889 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -70,7 +70,7 @@ type RegisterMutatorsContext interface { TopDown(name string, m TopDownMutator) MutatorHandle BottomUp(name string, m BottomUpMutator) MutatorHandle BottomUpBlueprint(name string, m blueprint.BottomUpMutator) MutatorHandle - Transition(name string, m TransitionMutator) + Transition(name string, m TransitionMutator) TransitionMutatorHandle } type RegisterMutatorFunc func(RegisterMutatorsContext) @@ -579,7 +579,7 @@ func (a *androidTransitionMutator) Mutate(ctx blueprint.BottomUpMutatorContext, } } -func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) { +func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) TransitionMutatorHandle { atm := &androidTransitionMutator{ finalPhase: x.finalPhase, mutator: m, @@ -587,8 +587,10 @@ func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) { } mutator := &mutator{ name: name, - transitionMutator: atm} + transitionMutator: atm, + } x.mutators = append(x.mutators, mutator) + return mutator } func (x *registerMutatorsContext) mutatorName(name string) string { @@ -625,7 +627,10 @@ func (mutator *mutator) register(ctx *Context) { } else if mutator.topDownMutator != nil { handle = blueprintCtx.RegisterTopDownMutator(mutator.name, mutator.topDownMutator) } else if mutator.transitionMutator != nil { - blueprintCtx.RegisterTransitionMutator(mutator.name, mutator.transitionMutator) + handle := blueprintCtx.RegisterTransitionMutator(mutator.name, mutator.transitionMutator) + if mutator.neverFar { + handle.NeverFar() + } } // Forward booleans set on the MutatorHandle to the blueprint.MutatorHandle. @@ -681,6 +686,14 @@ type MutatorHandle interface { MutatesGlobalState() MutatorHandle } +type TransitionMutatorHandle interface { + // NeverFar causes the variations created by this mutator to never be ignored when adding + // far variation dependencies. Normally, far variation dependencies ignore all the variants + // of the source module, and only use the variants explicitly requested by the + // AddFarVariationDependencies call. + NeverFar() MutatorHandle +} + func (mutator *mutator) Parallel() MutatorHandle { return mutator } @@ -715,6 +728,11 @@ func (mutator *mutator) MutatesGlobalState() MutatorHandle { return mutator } +func (mutator *mutator) NeverFar() MutatorHandle { + mutator.neverFar = true + return mutator +} + func RegisterComponentsMutator(ctx RegisterMutatorsContext) { ctx.BottomUp("component-deps", componentDepsMutator) } diff --git a/android/neverallow.go b/android/neverallow.go index 44ac2cd6c..6176a996a 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -63,6 +63,7 @@ func init() { AddNeverAllowRules(createLimitDirgroupRule()...) AddNeverAllowRules(createFilesystemIsAutoGeneratedRule()) AddNeverAllowRules(createKotlinPluginRule()...) + AddNeverAllowRules(createPrebuiltEtcBpDefineRule()) } // Add a NeverAllow rule to the set of rules to apply. @@ -321,6 +322,27 @@ func createKotlinPluginRule() []Rule { } } +// These module types are introduced to convert PRODUCT_COPY_FILES to Soong, +// and is only intended to be used by filesystem_creator. +func createPrebuiltEtcBpDefineRule() Rule { + return NeverAllow(). + ModuleType( + "prebuilt_usr_srec", + "prebuilt_priv_app", + "prebuilt_rfs", + "prebuilt_framework", + "prebuilt_res", + "prebuilt_wlc_upt", + "prebuilt_odm", + "prebuilt_vendor_dlkm", + "prebuilt_bt_firmware", + "prebuilt_tvservice", + "prebuilt_optee", + ). + DefinedInBpFile(). + Because("module type not allowed to be defined in bp file") +} + func neverallowMutator(ctx BottomUpMutatorContext) { m, ok := ctx.Module().(Module) if !ok { @@ -354,6 +376,10 @@ func neverallowMutator(ctx BottomUpMutatorContext) { continue } + if !n.appliesToBpDefinedModule(ctx) { + continue + } + ctx.ModuleErrorf("violates " + n.String()) } } @@ -477,6 +503,8 @@ type Rule interface { WithoutMatcher(properties string, matcher ValueMatcher) Rule + DefinedInBpFile() Rule + Because(reason string) Rule } @@ -498,6 +526,8 @@ type rule struct { unlessProps ruleProperties onlyBootclasspathJar bool + + definedInBp bool } // Create a new NeverAllow rule. @@ -571,6 +601,13 @@ func (r *rule) WithoutMatcher(properties string, matcher ValueMatcher) Rule { return r } +// DefinedInBpFile specifies that this rule applies to modules that are defined +// in bp files, and does not apply to modules that are auto generated by other modules. +func (r *rule) DefinedInBpFile() Rule { + r.definedInBp = true + return r +} + func selectMatcher(expected string) ValueMatcher { if expected == "*" { return anyMatcherInstance @@ -665,6 +702,13 @@ func (r *rule) appliesToProperties(ctx BottomUpMutatorContext, properties []inte return includeProps && !excludeProps } +func (r *rule) appliesToBpDefinedModule(ctx BottomUpMutatorContext) bool { + if !r.definedInBp { + return true + } + return !ctx.OtherModuleIsAutoGenerated(ctx.Module()) == r.definedInBp +} + func StartsWith(prefix string) ValueMatcher { return &startsWithMatcher{prefix} } diff --git a/android/neverallow_test.go b/android/neverallow_test.go index caec8c7d9..c74d5ff58 100644 --- a/android/neverallow_test.go +++ b/android/neverallow_test.go @@ -374,6 +374,20 @@ var neverallowTests = []struct { `is_auto_generated property is only allowed for filesystem modules in build/soong/fsgen directory`, }, }, + // Test for the rule restricting use of prebuilt_* module + { + name: `"prebuilt_usr_srec" defined in Android.bp file`, + fs: map[string][]byte{ + "a/b/Android.bp": []byte(` + prebuilt_usr_srec { + name: "foo", + } + `), + }, + expectedErrors: []string{ + `module type not allowed to be defined in bp file`, + }, + }, } var prepareForNeverAllowTest = GroupFixturePreparers( @@ -383,6 +397,7 @@ var prepareForNeverAllowTest = GroupFixturePreparers( ctx.RegisterModuleType("java_library_host", newMockJavaLibraryModule) ctx.RegisterModuleType("java_device_for_host", newMockJavaLibraryModule) ctx.RegisterModuleType("filesystem", newMockFilesystemModule) + ctx.RegisterModuleType("prebuilt_usr_srec", newMockPrebuiltUsrSrecModule) }), ) @@ -482,3 +497,16 @@ func newMockJavaLibraryModule() Module { func (p *mockJavaLibraryModule) GenerateAndroidBuildActions(ModuleContext) { } + +type mockPrebuiltUsrSrecModule struct { + ModuleBase +} + +func (p *mockPrebuiltUsrSrecModule) GenerateAndroidBuildActions(ModuleContext) { +} + +func newMockPrebuiltUsrSrecModule() Module { + m := &mockPrebuiltUsrSrecModule{} + InitAndroidModule(m) + return m +} diff --git a/android/packaging.go b/android/packaging.go index acafcd4f2..98c85fa0c 100644 --- a/android/packaging.go +++ b/android/packaging.go @@ -207,38 +207,49 @@ type PackagingBase struct { // If this is set to true by a module type inheriting PackagingBase, the deps property // collects the first target only even with compile_multilib: true. DepsCollectFirstTargetOnly bool + + // If this is set to try by a module type inheriting PackagingBase, the module type is + // allowed to utilize High_priority_deps. + AllowHighPriorityDeps bool } -type depsProperty struct { +type DepsProperty struct { + // Deps that have higher priority in packaging when there is a packaging conflict. + // For example, if multiple files are being installed to same filepath, the install file + // of the module listed in this property will have a higher priority over those in other + // deps properties. + High_priority_deps []string `android:"arch_variant"` + // Modules to include in this package Deps proptools.Configurable[[]string] `android:"arch_variant"` } type packagingMultilibProperties struct { - First depsProperty `android:"arch_variant"` - Common depsProperty `android:"arch_variant"` - Lib32 depsProperty `android:"arch_variant"` - Lib64 depsProperty `android:"arch_variant"` - Both depsProperty `android:"arch_variant"` - Prefer32 depsProperty `android:"arch_variant"` + First DepsProperty `android:"arch_variant"` + Common DepsProperty `android:"arch_variant"` + Lib32 DepsProperty `android:"arch_variant"` + Lib64 DepsProperty `android:"arch_variant"` + Both DepsProperty `android:"arch_variant"` + Prefer32 DepsProperty `android:"arch_variant"` } type packagingArchProperties struct { - Arm64 depsProperty - Arm depsProperty - X86_64 depsProperty - X86 depsProperty + Arm64 DepsProperty + Arm DepsProperty + X86_64 DepsProperty + X86 DepsProperty } type PackagingProperties struct { - Deps proptools.Configurable[[]string] `android:"arch_variant"` - Multilib packagingMultilibProperties `android:"arch_variant"` + DepsProperty + + Multilib packagingMultilibProperties `android:"arch_variant"` Arch packagingArchProperties } func InitPackageModule(p PackageModule) { base := p.packagingBase() - p.AddProperties(&base.properties) + p.AddProperties(&base.properties, &base.properties.DepsProperty) } func (p *PackagingBase) packagingBase() *PackagingBase { @@ -249,55 +260,72 @@ func (p *PackagingBase) packagingBase() *PackagingBase { // the current archicture when this module is not configured for multi target. When configured for // multi target, deps is selected for each of the targets and is NOT selected for the current // architecture which would be Common. -func (p *PackagingBase) getDepsForArch(ctx BaseModuleContext, arch ArchType) []string { - get := func(prop proptools.Configurable[[]string]) []string { - return prop.GetOrDefault(ctx, nil) +// It returns two lists, the normal and high priority deps, respectively. +func (p *PackagingBase) getDepsForArch(ctx BaseModuleContext, arch ArchType) ([]string, []string) { + var normalDeps []string + var highPriorityDeps []string + + get := func(prop DepsProperty) { + normalDeps = append(normalDeps, prop.Deps.GetOrDefault(ctx, nil)...) + highPriorityDeps = append(highPriorityDeps, prop.High_priority_deps...) + } + has := func(prop DepsProperty) bool { + return len(prop.Deps.GetOrDefault(ctx, nil)) > 0 || len(prop.High_priority_deps) > 0 } - var ret []string if arch == ctx.Target().Arch.ArchType && len(ctx.MultiTargets()) == 0 { - ret = append(ret, get(p.properties.Deps)...) + get(p.properties.DepsProperty) } else if arch.Multilib == "lib32" { - ret = append(ret, get(p.properties.Multilib.Lib32.Deps)...) + get(p.properties.Multilib.Lib32) // multilib.prefer32.deps are added for lib32 only when they support 32-bit arch - for _, dep := range get(p.properties.Multilib.Prefer32.Deps) { + for _, dep := range p.properties.Multilib.Prefer32.Deps.GetOrDefault(ctx, nil) { + if checkIfOtherModuleSupportsLib32(ctx, dep) { + normalDeps = append(normalDeps, dep) + } + } + for _, dep := range p.properties.Multilib.Prefer32.High_priority_deps { if checkIfOtherModuleSupportsLib32(ctx, dep) { - ret = append(ret, dep) + highPriorityDeps = append(highPriorityDeps, dep) } } } else if arch.Multilib == "lib64" { - ret = append(ret, get(p.properties.Multilib.Lib64.Deps)...) + get(p.properties.Multilib.Lib64) // multilib.prefer32.deps are added for lib64 only when they don't support 32-bit arch - for _, dep := range get(p.properties.Multilib.Prefer32.Deps) { + for _, dep := range p.properties.Multilib.Prefer32.Deps.GetOrDefault(ctx, nil) { if !checkIfOtherModuleSupportsLib32(ctx, dep) { - ret = append(ret, dep) + normalDeps = append(normalDeps, dep) + } + } + for _, dep := range p.properties.Multilib.Prefer32.High_priority_deps { + if !checkIfOtherModuleSupportsLib32(ctx, dep) { + highPriorityDeps = append(highPriorityDeps, dep) } } } else if arch == Common { - ret = append(ret, get(p.properties.Multilib.Common.Deps)...) + get(p.properties.Multilib.Common) } if p.DepsCollectFirstTargetOnly { - if len(get(p.properties.Multilib.First.Deps)) > 0 { + if has(p.properties.Multilib.First) { ctx.PropertyErrorf("multilib.first.deps", "not supported. use \"deps\" instead") } for i, t := range ctx.MultiTargets() { if t.Arch.ArchType == arch { - ret = append(ret, get(p.properties.Multilib.Both.Deps)...) + get(p.properties.Multilib.Both) if i == 0 { - ret = append(ret, get(p.properties.Deps)...) + get(p.properties.DepsProperty) } } } } else { - if len(get(p.properties.Multilib.Both.Deps)) > 0 { + if has(p.properties.Multilib.Both) { ctx.PropertyErrorf("multilib.both.deps", "not supported. use \"deps\" instead") } for i, t := range ctx.MultiTargets() { if t.Arch.ArchType == arch { - ret = append(ret, get(p.properties.Deps)...) + get(p.properties.DepsProperty) if i == 0 { - ret = append(ret, get(p.properties.Multilib.First.Deps)...) + get(p.properties.Multilib.First) } } } @@ -306,17 +334,21 @@ func (p *PackagingBase) getDepsForArch(ctx BaseModuleContext, arch ArchType) []s if ctx.Arch().ArchType == Common { switch arch { case Arm64: - ret = append(ret, get(p.properties.Arch.Arm64.Deps)...) + get(p.properties.Arch.Arm64) case Arm: - ret = append(ret, get(p.properties.Arch.Arm.Deps)...) + get(p.properties.Arch.Arm) case X86_64: - ret = append(ret, get(p.properties.Arch.X86_64.Deps)...) + get(p.properties.Arch.X86_64) case X86: - ret = append(ret, get(p.properties.Arch.X86.Deps)...) + get(p.properties.Arch.X86) } } - return FirstUniqueStrings(ret) + if len(highPriorityDeps) > 0 && !p.AllowHighPriorityDeps { + ctx.ModuleErrorf("Usage of high_priority_deps is not allowed for %s module type", ctx.ModuleType()) + } + + return FirstUniqueStrings(normalDeps), FirstUniqueStrings(highPriorityDeps) } func getSupportedTargets(ctx BaseModuleContext) []Target { @@ -360,6 +392,8 @@ type PackagingItem interface { IsPackagingItem() bool } +var _ PackagingItem = (*PackagingItemAlwaysDepTag)(nil) + // DepTag provides default implementation of PackagingItem interface. // PackagingBase-derived modules can define their own dependency tag by embedding this, which // can be passed to AddDeps() or AddDependencies(). @@ -371,31 +405,52 @@ func (PackagingItemAlwaysDepTag) IsPackagingItem() bool { return true } +// highPriorityDepTag provides default implementation of HighPriorityPackagingItem interface. +type highPriorityDepTag struct { + blueprint.DependencyTag +} + // See PackageModule.AddDeps func (p *PackagingBase) AddDeps(ctx BottomUpMutatorContext, depTag blueprint.DependencyTag) { + addDep := func(t Target, dep string, highPriority bool) { + if p.IgnoreMissingDependencies && !ctx.OtherModuleExists(dep) { + return + } + targetVariation := t.Variations() + sharedVariation := blueprint.Variation{ + Mutator: "link", + Variation: "shared", + } + // If a shared variation exists, use that. Static variants do not provide any standalone files + // for packaging. + if ctx.OtherModuleFarDependencyVariantExists([]blueprint.Variation{sharedVariation}, dep) { + targetVariation = append(targetVariation, sharedVariation) + } + depTagToUse := depTag + if highPriority { + depTagToUse = highPriorityDepTag{depTag} + } + + ctx.AddFarVariationDependencies(targetVariation, depTagToUse, dep) + } for _, t := range getSupportedTargets(ctx) { - for _, dep := range p.getDepsForArch(ctx, t.Arch.ArchType) { - if p.IgnoreMissingDependencies && !ctx.OtherModuleExists(dep) { - continue - } - targetVariation := t.Variations() - sharedVariation := blueprint.Variation{ - Mutator: "link", - Variation: "shared", - } - // If a shared variation exists, use that. Static variants do not provide any standalone files - // for packaging. - if ctx.OtherModuleFarDependencyVariantExists([]blueprint.Variation{sharedVariation}, dep) { - targetVariation = append(targetVariation, sharedVariation) - } - ctx.AddFarVariationDependencies(targetVariation, depTag, dep) + normalDeps, highPriorityDeps := p.getDepsForArch(ctx, t.Arch.ArchType) + for _, dep := range normalDeps { + addDep(t, dep, false) + } + for _, dep := range highPriorityDeps { + addDep(t, dep, true) } } } func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter func(PackagingSpec) bool) map[string]PackagingSpec { - // all packaging specs gathered from the dep. - var all []PackagingSpec + // packaging specs gathered from the dep that are not high priorities. + var regularPriorities []PackagingSpec + + // all packaging specs gathered from the high priority deps. + var highPriorities []PackagingSpec + // Name of the dependency which requested the packaging spec. // If this dep is overridden, the packaging spec will not be installed via this dependency chain. // (the packaging spec might still be installed if there are some other deps which depend on it). @@ -420,7 +475,8 @@ func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter } ctx.VisitDirectDeps(func(child Module) { - if pi, ok := ctx.OtherModuleDependencyTag(child).(PackagingItem); !ok || !pi.IsPackagingItem() { + depTag := ctx.OtherModuleDependencyTag(child) + if pi, ok := depTag.(PackagingItem); !ok || !pi.IsPackagingItem() { return } for _, ps := range OtherModuleProviderOrDefault( @@ -434,7 +490,13 @@ func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter continue } } - all = append(all, ps) + + if _, ok := depTag.(highPriorityDepTag); ok { + highPriorities = append(highPriorities, ps) + } else { + regularPriorities = append(regularPriorities, ps) + } + depNames = append(depNames, child.Name()) if ps.overrides != nil { overridden = append(overridden, *ps.overrides...) @@ -442,21 +504,26 @@ func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter } }) - // all minus packaging specs that are overridden - var filtered []PackagingSpec - for index, ps := range all { - if ps.owner != "" && InList(ps.owner, overridden) { - continue - } - // The dependency which requested this packaging spec has been overridden. - if InList(depNames[index], overridden) { - continue + filterOverridden := func(input []PackagingSpec) []PackagingSpec { + // input minus packaging specs that are overridden + var filtered []PackagingSpec + for index, ps := range input { + if ps.owner != "" && InList(ps.owner, overridden) { + continue + } + // The dependency which requested this packaging spec has been overridden. + if InList(depNames[index], overridden) { + continue + } + filtered = append(filtered, ps) } - filtered = append(filtered, ps) + return filtered } + filteredRegularPriority := filterOverridden(regularPriorities) + m := make(map[string]PackagingSpec) - for _, ps := range filtered { + for _, ps := range filteredRegularPriority { dstPath := ps.relPathInPackage if existingPs, ok := m[dstPath]; ok { if !existingPs.Equals(&ps) { @@ -466,6 +533,21 @@ func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter } m[dstPath] = ps } + + filteredHighPriority := filterOverridden(highPriorities) + highPriorityPs := make(map[string]PackagingSpec) + for _, ps := range filteredHighPriority { + dstPath := ps.relPathInPackage + if existingPs, ok := highPriorityPs[dstPath]; ok { + if !existingPs.Equals(&ps) { + ctx.ModuleErrorf("packaging conflict at %v:\n%v\n%v", dstPath, existingPs, ps) + } + continue + } + highPriorityPs[dstPath] = ps + m[dstPath] = ps + } + return m } diff --git a/android/path_properties.go b/android/path_properties.go index f3c62ea43..55a4dc066 100644 --- a/android/path_properties.go +++ b/android/path_properties.go @@ -52,16 +52,12 @@ func addPathDepsForProps(ctx BottomUpMutatorContext, props []interface{}) { var pathProperties []string var pathDeviceFirstProperties []string var pathDeviceFirstPrefer32Properties []string - var pathDeviceFirstVendorProperties []string - var pathDeviceFirstVendorSharedProperties []string var pathDeviceCommonProperties []string var pathCommonOsProperties []string for _, ps := range props { pathProperties = append(pathProperties, taggedPropertiesForPropertyStruct(ctx, ps, "path")...) pathDeviceFirstProperties = append(pathDeviceFirstProperties, taggedPropertiesForPropertyStruct(ctx, ps, "path_device_first")...) pathDeviceFirstPrefer32Properties = append(pathDeviceFirstPrefer32Properties, taggedPropertiesForPropertyStruct(ctx, ps, "path_device_first_prefer32")...) - pathDeviceFirstVendorProperties = append(pathDeviceFirstVendorProperties, taggedPropertiesForPropertyStruct(ctx, ps, "path_device_first_vendor")...) - pathDeviceFirstVendorSharedProperties = append(pathDeviceFirstVendorSharedProperties, taggedPropertiesForPropertyStruct(ctx, ps, "path_device_first_vendor_shared")...) pathDeviceCommonProperties = append(pathDeviceCommonProperties, taggedPropertiesForPropertyStruct(ctx, ps, "path_device_common")...) pathCommonOsProperties = append(pathCommonOsProperties, taggedPropertiesForPropertyStruct(ctx, ps, "path_common_os")...) } @@ -70,8 +66,6 @@ func addPathDepsForProps(ctx BottomUpMutatorContext, props []interface{}) { pathProperties = FirstUniqueStrings(pathProperties) pathDeviceFirstProperties = FirstUniqueStrings(pathDeviceFirstProperties) pathDeviceFirstPrefer32Properties = FirstUniqueStrings(pathDeviceFirstPrefer32Properties) - pathDeviceFirstVendorProperties = FirstUniqueStrings(pathDeviceFirstVendorProperties) - pathDeviceFirstVendorSharedProperties = FirstUniqueStrings(pathDeviceFirstVendorSharedProperties) pathDeviceCommonProperties = FirstUniqueStrings(pathDeviceCommonProperties) pathCommonOsProperties = FirstUniqueStrings(pathCommonOsProperties) @@ -108,23 +102,6 @@ func addPathDepsForProps(ctx BottomUpMutatorContext, props []interface{}) { } } } - // path_device_first_vendor is path_device_first + vendor variation - deviceFirstVendorVariations := ctx.Config().AndroidFirstDeviceTarget.Variations() - deviceFirstVendorVariations = append(deviceFirstVendorVariations, - blueprint.Variation{Mutator: "image", Variation: "vendor"}) - for _, s := range pathDeviceFirstVendorProperties { - if m, t := SrcIsModuleWithTag(s); m != "" { - ctx.AddVariationDependencies(deviceFirstVendorVariations, sourceOrOutputDepTag(m, t), m) - } - } - // path_device_first_vendor_shared is path_device_first_vendor + shared linkage variation - deviceFirstVendorSharedVariations := append(deviceFirstVendorVariations, - blueprint.Variation{Mutator: "link", Variation: "shared"}) - for _, s := range pathDeviceFirstVendorSharedProperties { - if m, t := SrcIsModuleWithTag(s); m != "" { - ctx.AddVariationDependencies(deviceFirstVendorSharedVariations, sourceOrOutputDepTag(m, t), m) - } - } // properties tagged "path_device_common" get the device common variant for _, s := range pathDeviceCommonProperties { if m, t := SrcIsModuleWithTag(s); m != "" { diff --git a/android/paths.go b/android/paths.go index bf2c3a097..9cb872d6f 100644 --- a/android/paths.go +++ b/android/paths.go @@ -117,6 +117,9 @@ type ModuleInstallPathContext interface { InstallInOdm() bool InstallInProduct() bool InstallInVendor() bool + InstallInSystemDlkm() bool + InstallInVendorDlkm() bool + InstallInOdmDlkm() bool InstallForceOS() (*OsType, *ArchType) } @@ -170,6 +173,18 @@ func (ctx *baseModuleContextToModuleInstallPathContext) InstallInVendor() bool { return ctx.Module().InstallInVendor() } +func (ctx *baseModuleContextToModuleInstallPathContext) InstallInSystemDlkm() bool { + return ctx.Module().InstallInSystemDlkm() +} + +func (ctx *baseModuleContextToModuleInstallPathContext) InstallInVendorDlkm() bool { + return ctx.Module().InstallInVendorDlkm() +} + +func (ctx *baseModuleContextToModuleInstallPathContext) InstallInOdmDlkm() bool { + return ctx.Module().InstallInOdmDlkm() +} + func (ctx *baseModuleContextToModuleInstallPathContext) InstallForceOS() (*OsType, *ArchType) { return ctx.Module().InstallForceOS() } @@ -551,15 +566,35 @@ func PathsRelativeToModuleSourceDir(input SourceInput) Paths { return ret } +type directoryPath struct { + basePath +} + +func (d *directoryPath) String() string { + return d.basePath.String() +} + +func (d *directoryPath) base() basePath { + return d.basePath +} + +// DirectoryPath represents a source path for directories. Incompatible with Path by design. +type DirectoryPath interface { + String() string + base() basePath +} + +var _ DirectoryPath = (*directoryPath)(nil) + +type DirectoryPaths []DirectoryPath + // DirectoryPathsForModuleSrcExcludes returns a Paths{} containing the resolved references in // directory paths. Elements of paths are resolved as: // - filepath, relative to local module directory, resolves as a filepath relative to the local // source directory // - other modules using the ":name" syntax. These modules must implement DirProvider. -// -// TODO(b/358302178): Implement DirectoryPath and change the return type. -func DirectoryPathsForModuleSrc(ctx ModuleMissingDepsPathContext, paths []string) Paths { - var ret Paths +func DirectoryPathsForModuleSrc(ctx ModuleMissingDepsPathContext, paths []string) DirectoryPaths { + var ret DirectoryPaths for _, path := range paths { if m, t := SrcIsModuleWithTag(path); m != "" { @@ -588,12 +623,12 @@ func DirectoryPathsForModuleSrc(ctx ModuleMissingDepsPathContext, paths []string } else if !isDir { ReportPathErrorf(ctx, "module directory path %q is not a directory", p) } else { - ret = append(ret, p) + ret = append(ret, &directoryPath{basePath{path: p.path, rel: p.rel}}) } } } - seen := make(map[Path]bool, len(ret)) + seen := make(map[DirectoryPath]bool, len(ret)) for _, path := range ret { if seen[path] { ReportPathErrorf(ctx, "duplicated path %q", path) @@ -2057,6 +2092,10 @@ func PathForMainlineSdksInstall(ctx PathContext, paths ...string) InstallPath { return base.Join(ctx, paths...) } +func PathForSuiteInstall(ctx PathContext, suite string, pathComponents ...string) InstallPath { + return pathForPartitionInstallDir(ctx, "test_suites", "test_suites", false).Join(ctx, suite).Join(ctx, pathComponents...) +} + func InstallPathToOnDevicePath(ctx PathContext, path InstallPath) string { rel := Rel(ctx, strings.TrimSuffix(path.PartitionDir(), path.partition), path.String()) return "/" + rel @@ -2111,6 +2150,12 @@ func modulePartition(ctx ModuleInstallPathContext, device bool) string { partition = ctx.DeviceConfig().SystemExtPath() } else if ctx.InstallInRoot() { partition = "root" + } else if ctx.InstallInSystemDlkm() { + partition = ctx.DeviceConfig().SystemDlkmPath() + } else if ctx.InstallInVendorDlkm() { + partition = ctx.DeviceConfig().VendorDlkmPath() + } else if ctx.InstallInOdmDlkm() { + partition = ctx.DeviceConfig().OdmDlkmPath() } else { partition = "system" } @@ -2314,6 +2359,9 @@ type testModuleInstallPathContext struct { inOdm bool inProduct bool inVendor bool + inSystemDlkm bool + inVendorDlkm bool + inOdmDlkm bool forceOS *OsType forceArch *ArchType } @@ -2368,6 +2416,18 @@ func (m testModuleInstallPathContext) InstallInVendor() bool { return m.inVendor } +func (m testModuleInstallPathContext) InstallInSystemDlkm() bool { + return m.inSystemDlkm +} + +func (m testModuleInstallPathContext) InstallInVendorDlkm() bool { + return m.inVendorDlkm +} + +func (m testModuleInstallPathContext) InstallInOdmDlkm() bool { + return m.inOdmDlkm +} + func (m testModuleInstallPathContext) InstallForceOS() (*OsType, *ArchType) { return m.forceOS, m.forceArch } diff --git a/android/paths_test.go b/android/paths_test.go index 941f0ca78..5e618f914 100644 --- a/android/paths_test.go +++ b/android/paths_test.go @@ -1592,6 +1592,12 @@ func TestPathRelativeToTop(t *testing.T) { }) } +func TestDirectoryPathIsIncompatibleWithPath(t *testing.T) { + d := (DirectoryPath)(&directoryPath{}) + _, ok := d.(Path) + AssertBoolEquals(t, "directoryPath shouldn't implement Path", ok, false) +} + func ExampleOutputPath_ReplaceExtension() { ctx := &configErrorWrapper{ config: TestConfig("out", nil, "", nil), diff --git a/android/prebuilt.go b/android/prebuilt.go index 5d75b62fe..51e72af9a 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -569,6 +569,7 @@ func hideUnflaggedModules(ctx BottomUpMutatorContext, psi PrebuiltSelectionInfoM for _, moduleInFamily := range allModulesInFamily { if moduleInFamily.Name() != selectedModuleInFamily.Name() { moduleInFamily.HideFromMake() + moduleInFamily.SkipInstall() // If this is a prebuilt module, unset properties.UsePrebuilt // properties.UsePrebuilt might evaluate to true via soong config var fallback mechanism // Set it to false explicitly so that the following mutator does not replace rdeps to this unselected prebuilt @@ -639,6 +640,7 @@ func PrebuiltPostDepsMutator(ctx BottomUpMutatorContext) { } } else { m.HideFromMake() + m.SkipInstall() } } } diff --git a/android/register.go b/android/register.go index bb1ead73c..8d2f19e73 100644 --- a/android/register.go +++ b/android/register.go @@ -98,6 +98,7 @@ type mutator struct { usesCreateModule bool mutatesDependencies bool mutatesGlobalState bool + neverFar bool } var _ sortableComponent = &mutator{} diff --git a/android/rule_builder.go b/android/rule_builder.go index 56de9cd00..403c18418 100644 --- a/android/rule_builder.go +++ b/android/rule_builder.go @@ -575,25 +575,28 @@ func (r *RuleBuilder) build(name string, desc string, ninjaEscapeCommandString b nsjailCmd.WriteString(r.outDir.String()) nsjailCmd.WriteString(":nsjail_build_sandbox/out") - for _, input := range inputs { + addBindMount := func(src, dst string) { nsjailCmd.WriteString(" -R $PWD/") - nsjailCmd.WriteString(input.String()) + nsjailCmd.WriteString(src) nsjailCmd.WriteString(":nsjail_build_sandbox/") - nsjailCmd.WriteString(r.nsjailPathForInputRel(input)) + nsjailCmd.WriteString(dst) + } + + for _, input := range inputs { + addBindMount(input.String(), r.nsjailPathForInputRel(input)) } for _, tool := range tools { - nsjailCmd.WriteString(" -R $PWD/") - nsjailCmd.WriteString(tool.String()) - nsjailCmd.WriteString(":nsjail_build_sandbox/") - nsjailCmd.WriteString(nsjailPathForToolRel(r.ctx, tool)) + addBindMount(tool.String(), nsjailPathForToolRel(r.ctx, tool)) } inputs = append(inputs, tools...) for _, c := range r.commands { + for _, directory := range c.implicitDirectories { + addBindMount(directory.String(), directory.String()) + // TODO(b/375551969): Add implicitDirectories to BuildParams, rather than relying on implicits + inputs = append(inputs, SourcePath{basePath: directory.base()}) + } for _, tool := range c.packagedTools { - nsjailCmd.WriteString(" -R $PWD/") - nsjailCmd.WriteString(tool.srcPath.String()) - nsjailCmd.WriteString(":nsjail_build_sandbox/") - nsjailCmd.WriteString(nsjailPathForPackagedToolRel(tool)) + addBindMount(tool.srcPath.String(), nsjailPathForPackagedToolRel(tool)) inputs = append(inputs, tool.srcPath) } } @@ -917,16 +920,17 @@ func (r *RuleBuilder) build(name string, desc string, ninjaEscapeCommandString b type RuleBuilderCommand struct { rule *RuleBuilder - buf strings.Builder - inputs Paths - implicits Paths - orderOnlys Paths - validations Paths - outputs WritablePaths - depFiles WritablePaths - tools Paths - packagedTools []PackagingSpec - rspFiles []rspFileAndPaths + buf strings.Builder + inputs Paths + implicits Paths + orderOnlys Paths + validations Paths + outputs WritablePaths + depFiles WritablePaths + tools Paths + packagedTools []PackagingSpec + rspFiles []rspFileAndPaths + implicitDirectories DirectoryPaths } type rspFileAndPaths struct { @@ -951,6 +955,10 @@ func (c *RuleBuilderCommand) addImplicit(path Path) { c.implicits = append(c.implicits, path) } +func (c *RuleBuilderCommand) addImplicitDirectory(path DirectoryPath) { + c.implicitDirectories = append(c.implicitDirectories, path) +} + func (c *RuleBuilderCommand) addOrderOnly(path Path) { checkPathNotNil(path) c.orderOnlys = append(c.orderOnlys, path) @@ -1313,6 +1321,16 @@ func (c *RuleBuilderCommand) Implicits(paths Paths) *RuleBuilderCommand { return c } +// ImplicitDirectory adds the specified input directory to the dependencies without modifying the +// command line. Added directories will be bind-mounted for the nsjail. +func (c *RuleBuilderCommand) ImplicitDirectory(path DirectoryPath) *RuleBuilderCommand { + if !c.rule.nsjail { + panic("ImplicitDirectory() must be called after Nsjail()") + } + c.addImplicitDirectory(path) + return c +} + // GetImplicits returns the command's implicit inputs. func (c *RuleBuilderCommand) GetImplicits() Paths { return c.implicits diff --git a/android/singleton.go b/android/singleton.go index 913bf6a56..0754b0ccb 100644 --- a/android/singleton.go +++ b/android/singleton.go @@ -64,6 +64,7 @@ type SingletonContext interface { VisitAllModulesBlueprint(visit func(blueprint.Module)) VisitAllModules(visit func(Module)) + VisitAllModuleProxies(visit func(proxy ModuleProxy)) VisitAllModulesIf(pred func(Module) bool, visit func(Module)) VisitDirectDeps(module Module, visit func(Module)) @@ -77,8 +78,10 @@ type SingletonContext interface { VisitAllModuleVariants(module Module, visit func(Module)) + VisitAllModuleVariantProxies(module Module, visit func(proxy ModuleProxy)) + PrimaryModule(module Module) Module - FinalModule(module Module) Module + IsFinalModule(module Module) bool AddNinjaFileDeps(deps ...string) @@ -193,7 +196,7 @@ func (s *singletonContextAdaptor) Eval(pctx PackageContext, ninjaStr string) (st } // visitAdaptor wraps a visit function that takes an android.Module parameter into -// a function that takes an blueprint.Module parameter and only calls the visit function if the +// a function that takes a blueprint.Module parameter and only calls the visit function if the // blueprint.Module is an android.Module. func visitAdaptor(visit func(Module)) func(blueprint.Module) { return func(module blueprint.Module) { @@ -203,6 +206,16 @@ func visitAdaptor(visit func(Module)) func(blueprint.Module) { } } +// visitProxyAdaptor wraps a visit function that takes an android.ModuleProxy parameter into +// a function that takes a blueprint.ModuleProxy parameter. +func visitProxyAdaptor(visit func(proxy ModuleProxy)) func(proxy blueprint.ModuleProxy) { + return func(module blueprint.ModuleProxy) { + visit(ModuleProxy{ + module: module, + }) + } +} + // predAdaptor wraps a pred function that takes an android.Module parameter // into a function that takes an blueprint.Module parameter and only calls the visit function if the // blueprint.Module is an android.Module, otherwise returns false. @@ -224,6 +237,10 @@ func (s *singletonContextAdaptor) VisitAllModules(visit func(Module)) { s.SingletonContext.VisitAllModules(visitAdaptor(visit)) } +func (s *singletonContextAdaptor) VisitAllModuleProxies(visit func(proxy ModuleProxy)) { + s.SingletonContext.VisitAllModuleProxies(visitProxyAdaptor(visit)) +} + func (s *singletonContextAdaptor) VisitAllModulesIf(pred func(Module) bool, visit func(Module)) { s.SingletonContext.VisitAllModulesIf(predAdaptor(pred), visitAdaptor(visit)) } @@ -248,12 +265,16 @@ func (s *singletonContextAdaptor) VisitAllModuleVariants(module Module, visit fu s.SingletonContext.VisitAllModuleVariants(module, visitAdaptor(visit)) } +func (s *singletonContextAdaptor) VisitAllModuleVariantProxies(module Module, visit func(proxy ModuleProxy)) { + s.SingletonContext.VisitAllModuleVariantProxies(module, visitProxyAdaptor(visit)) +} + func (s *singletonContextAdaptor) PrimaryModule(module Module) Module { return s.SingletonContext.PrimaryModule(module).(Module) } -func (s *singletonContextAdaptor) FinalModule(module Module) Module { - return s.SingletonContext.FinalModule(module).(Module) +func (s *singletonContextAdaptor) IsFinalModule(module Module) bool { + return s.SingletonContext.IsFinalModule(module) } func (s *singletonContextAdaptor) ModuleVariantsFromName(referer Module, name string) []Module { diff --git a/android/util.go b/android/util.go index 2d269b724..3fc4608e0 100644 --- a/android/util.go +++ b/android/util.go @@ -660,3 +660,13 @@ func (m *SyncMap[K, V]) LoadOrStore(key K, value V) (actual V, loaded bool) { v, loaded := m.Map.LoadOrStore(key, value) return v.(V), loaded } + +// AppendIfNotZero append the given value to the slice if it is not the zero value +// for its type. +func AppendIfNotZero[T comparable](slice []T, value T) []T { + var zeroValue T // Get the zero value of the type T + if value != zeroValue { + return append(slice, value) + } + return slice +} diff --git a/android/variable.go b/android/variable.go index 2969f97c0..6e4a4f709 100644 --- a/android/variable.go +++ b/android/variable.go @@ -338,12 +338,19 @@ type ProductVariables struct { HWASanIncludePaths []string `json:",omitempty"` HWASanExcludePaths []string `json:",omitempty"` - VendorPath *string `json:",omitempty"` - BuildingVendorImage *bool `json:",omitempty"` - OdmPath *string `json:",omitempty"` - ProductPath *string `json:",omitempty"` - BuildingProductImage *bool `json:",omitempty"` - SystemExtPath *string `json:",omitempty"` + VendorPath *string `json:",omitempty"` + VendorDlkmPath *string `json:",omitempty"` + BuildingVendorImage *bool `json:",omitempty"` + OdmPath *string `json:",omitempty"` + BuildingOdmImage *bool `json:",omitempty"` + OdmDlkmPath *string `json:",omitempty"` + ProductPath *string `json:",omitempty"` + BuildingProductImage *bool `json:",omitempty"` + SystemExtPath *string `json:",omitempty"` + SystemDlkmPath *string `json:",omitempty"` + OemPath *string `json:",omitempty"` + UserdataPath *string `json:",omitempty"` + BuildingUserdataImage *bool `json:",omitempty"` ClangTidy *bool `json:",omitempty"` TidyChecks *string `json:",omitempty"` @@ -534,6 +541,13 @@ type ProductVariables struct { ExtraAllowedDepsTxt *string `json:",omitempty"` AdbKeys *string `json:",omitempty"` + + DeviceMatrixFile []string `json:",omitempty"` + ProductManifestFiles []string `json:",omitempty"` + SystemManifestFile []string `json:",omitempty"` + SystemExtManifestFiles []string `json:",omitempty"` + DeviceManifestFiles []string `json:",omitempty"` + OdmManifestFiles []string `json:",omitempty"` } type PartitionQualifiedVariablesType struct { @@ -564,6 +578,14 @@ type PartitionQualifiedVariablesType struct { BoardAvbRollbackIndexLocation string `json:",omitempty"` } +type ChainedAvbPartitionProps struct { + Partitions []string `json:",omitempty"` + Key string `json:",omitempty"` + Algorithm string `json:",omitempty"` + RollbackIndex string `json:",omitempty"` + RollbackIndexLocation string `json:",omitempty"` +} + type PartitionVariables struct { ProductDirectory string `json:",omitempty"` PartitionQualifiedVariables map[string]PartitionQualifiedVariablesType @@ -588,12 +610,25 @@ type PartitionVariables struct { ProductUseDynamicPartitionSize bool `json:",omitempty"` CopyImagesForTargetFilesZip bool `json:",omitempty"` - BoardAvbEnable bool `json:",omitempty"` + BoardAvbEnable bool `json:",omitempty"` + BoardAvbAlgorithm string `json:",omitempty"` + BoardAvbKeyPath string `json:",omitempty"` + BoardAvbRollbackIndex string `json:",omitempty"` + BuildingVbmetaImage bool `json:",omitempty"` + ChainedVbmetaPartitions map[string]ChainedAvbPartitionProps `json:",omitempty"` - ProductPackages []string `json:",omitempty"` - ProductPackagesDebug []string `json:",omitempty"` + ProductPackages []string `json:",omitempty"` + ProductPackagesDebug []string `json:",omitempty"` + VendorLinkerConfigSrcs []string `json:",omitempty"` + ProductLinkerConfigSrcs []string `json:",omitempty"` + + BoardInfoFiles []string `json:",omitempty"` + BootLoaderBoardName string `json:",omitempty"` ProductCopyFiles map[string]string `json:",omitempty"` + + BuildingSystemDlkmImage bool `json:",omitempty"` + SystemKernelModules []string `json:",omitempty"` } func boolPtr(v bool) *bool { diff --git a/android/vintf_data.go b/android/vintf_data.go new file mode 100644 index 000000000..7823397e0 --- /dev/null +++ b/android/vintf_data.go @@ -0,0 +1,171 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "strings" + + "github.com/google/blueprint/proptools" +) + +const ( + deviceCmType = "device_cm" + systemManifestType = "system_manifest" + productManifestType = "product_manifest" + systemExtManifestType = "system_ext_manifest" + vendorManifestType = "vendor_manifest" + odmManifestType = "odm_manifest" + + defaultDcm = "system/libhidl/vintfdata/device_compatibility_matrix.default.xml" + defaultSystemManifest = "system/libhidl/vintfdata/manifest.xml" + defaultSystemExtManifest = "system/libhidl/vintfdata/system_ext_manifest.default.xml" +) + +type vintfDataProperties struct { + // Optional name for the installed file. If unspecified it will be manifest.xml by default. + Filename *string + + // Type of the vintf data type, the allowed type are device_compatibility_matrix, system_manifest, + // product_manifest, and system_ext_manifest. + Type *string +} + +type vintfDataRule struct { + ModuleBase + + properties vintfDataProperties + + installDirPath InstallPath + outputFilePath OutputPath + noAction bool +} + +func init() { + registerVintfDataComponents(InitRegistrationContext) +} + +func registerVintfDataComponents(ctx RegistrationContext) { + ctx.RegisterModuleType("vintf_data", vintfDataFactory) +} + +// vintf_fragment module processes vintf fragment file and installs under etc/vintf/manifest. +func vintfDataFactory() Module { + m := &vintfDataRule{} + m.AddProperties( + &m.properties, + ) + InitAndroidArchModule(m, DeviceSupported, MultilibFirst) + + return m +} + +func (m *vintfDataRule) GenerateAndroidBuildActions(ctx ModuleContext) { + builder := NewRuleBuilder(pctx, ctx) + gensrc := PathForModuleOut(ctx, "manifest.xml") + assembleVintfEnvs := []string{} + inputPaths := make(Paths, 0) + + switch proptools.String(m.properties.Type) { + case deviceCmType: + assembleVintfEnvs = append(assembleVintfEnvs, fmt.Sprintf("BOARD_SYSTEMSDK_VERSIONS=\"%s\"", strings.Join(ctx.DeviceConfig().SystemSdkVersions(), " "))) + + deviceMatrixs := PathsForSource(ctx, ctx.Config().DeviceMatrixFile()) + if len(deviceMatrixs) > 0 { + inputPaths = append(inputPaths, deviceMatrixs...) + } else { + inputPaths = append(inputPaths, PathForSource(ctx, defaultDcm)) + } + case systemManifestType: + assembleVintfEnvs = append(assembleVintfEnvs, fmt.Sprintf("PLATFORM_SYSTEMSDK_VERSIONS=\"%s\"", strings.Join(ctx.DeviceConfig().PlatformSystemSdkVersions(), " "))) + + inputPaths = append(inputPaths, PathForSource(ctx, defaultSystemManifest)) + systemManifestFiles := PathsForSource(ctx, ctx.Config().SystemManifestFile()) + if len(systemManifestFiles) > 0 { + inputPaths = append(inputPaths, systemManifestFiles...) + } + case productManifestType: + productManifestFiles := PathsForSource(ctx, ctx.Config().ProductManifestFiles()) + // Only need to generate the manifest if PRODUCT_MANIFEST_FILES not defined. + if len(productManifestFiles) == 0 { + m.noAction = true + return + } + + inputPaths = append(inputPaths, productManifestFiles...) + case systemExtManifestType: + assembleVintfEnvs = append(assembleVintfEnvs, fmt.Sprintf("PROVIDED_VNDK_VERSIONS=\"%s\"", strings.Join(ctx.DeviceConfig().ExtraVndkVersions(), " "))) + + inputPaths = append(inputPaths, PathForSource(ctx, defaultSystemExtManifest)) + systemExtManifestFiles := PathsForSource(ctx, ctx.Config().SystemExtManifestFiles()) + if len(systemExtManifestFiles) > 0 { + inputPaths = append(inputPaths, systemExtManifestFiles...) + } + case vendorManifestType: + assembleVintfEnvs = append(assembleVintfEnvs, fmt.Sprintf("BOARD_SEPOLICY_VERS=\"%s\"", ctx.DeviceConfig().BoardSepolicyVers())) + assembleVintfEnvs = append(assembleVintfEnvs, fmt.Sprintf("PRODUCT_ENFORCE_VINTF_MANIFEST=%t", *ctx.Config().productVariables.Enforce_vintf_manifest)) + deviceManifestFiles := PathsForSource(ctx, ctx.Config().DeviceManifestFiles()) + // Only need to generate the manifest if DEVICE_MANIFEST_FILE is defined. + if len(deviceManifestFiles) == 0 { + m.noAction = true + return + } + + inputPaths = append(inputPaths, deviceManifestFiles...) + case odmManifestType: + assembleVintfEnvs = append(assembleVintfEnvs, "VINTF_IGNORE_TARGET_FCM_VERSION=true") + odmManifestFiles := PathsForSource(ctx, ctx.Config().OdmManifestFiles()) + // Only need to generate the manifest if ODM_MANIFEST_FILES is defined. + if len(odmManifestFiles) == 0 { + m.noAction = true + return + } + + inputPaths = append(inputPaths, odmManifestFiles...) + default: + panic(fmt.Errorf("For %s: The attribute 'type' value only allowed device_cm, system_manifest, product_manifest, system_ext_manifest!", ctx.Module().Name())) + } + + // Process vintf fragment source file with assemble_vintf tool + builder.Command(). + Flags(assembleVintfEnvs). + BuiltTool("assemble_vintf"). + FlagWithArg("-i ", strings.Join(inputPaths.Strings(), ":")). + FlagWithOutput("-o ", gensrc) + + builder.Build("assemble_vintf", "Process vintf data "+gensrc.String()) + + m.installDirPath = PathForModuleInstall(ctx, "etc", "vintf") + m.outputFilePath = gensrc.OutputPath + + installFileName := "manifest.xml" + if filename := proptools.String(m.properties.Filename); filename != "" { + installFileName = filename + } + + ctx.InstallFile(m.installDirPath, installFileName, gensrc) +} + +// Make this module visible to AndroidMK so it can be referenced from modules defined from Android.mk files +func (m *vintfDataRule) AndroidMkEntries() []AndroidMkEntries { + if m.noAction { + return []AndroidMkEntries{} + } + + return []AndroidMkEntries{{ + Class: "ETC", + OutputFile: OptionalPathForPath(m.outputFilePath), + }} +} diff --git a/cc/Android.bp b/cc/Android.bp index a7b6d8161..3b29ae8cf 100644 --- a/cc/Android.bp +++ b/cc/Android.bp @@ -122,3 +122,31 @@ bootstrap_go_package { // Used by plugins visibility: ["//visibility:public"], } + +phony { + name: "llndk_libs", + required: [ + "libEGL", + "libGLESv1_CM", + "libGLESv2", + "libGLESv3", + "libRS", + "libandroid_net", + "libapexsupport", + "libbinder_ndk", + "libc", + "libcgrouprc", + "libclang_rt.asan", + "libdl", + "libft2", + "liblog", + "libm", + "libmediandk", + "libnativewindow", + "libselinux", + "libsync", + "libvendorsupport", + "libvndksupport", + "libvulkan", + ], +} diff --git a/cc/builder.go b/cc/builder.go index cd535c1e8..2948ca316 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -821,7 +821,7 @@ func generateRustStaticlib(ctx android.ModuleContext, rlibDeps []RustRlibDep) an return nil } - output := android.PathForModuleOut(ctx, "generated_rust_staticlib", "lib"+ctx.ModuleName()+"_rust_staticlib.a") + output := android.PathForModuleOut(ctx, "generated_rust_staticlib", "librustlibs.a") stemFile := output.ReplaceExtension(ctx, "rs") crateNames := []string{} @@ -37,6 +37,22 @@ import ( "android/soong/genrule" ) +type CcMakeVarsInfo struct { + WarningsAllowed string + UsingWnoError string + MissingProfile string +} + +var CcMakeVarsInfoProvider = blueprint.NewProvider[*CcMakeVarsInfo]() + +type CcObjectInfo struct { + objFiles android.Paths + tidyFiles android.Paths + kytheFiles android.Paths +} + +var CcObjectInfoProvider = blueprint.NewProvider[CcObjectInfo]() + func init() { RegisterCCBuildComponents(android.InitRegistrationContext) @@ -531,6 +547,7 @@ type ModuleContextIntf interface { getSharedFlags() *SharedFlags notInPlatform() bool optimizeForSize() bool + getOrCreateMakeVarsInfo() *CcMakeVarsInfo } type SharedFlags struct { @@ -639,10 +656,6 @@ type installer interface { installInRoot() bool } -type xref interface { - XrefCcFiles() android.Paths -} - type overridable interface { overriddenModules() []string } @@ -845,9 +858,10 @@ type Module struct { sourceProperties android.SourceProperties // initialize before calling Init - hod android.HostOrDeviceSupported - multilib android.Multilib - testModule bool + hod android.HostOrDeviceSupported + multilib android.Multilib + testModule bool + incremental bool // Allowable SdkMemberTypes of this module type. sdkMemberTypes []android.SdkMemberType @@ -890,12 +904,6 @@ type Module struct { staticAnalogue *StaticLibraryInfo makeLinkType string - // Kythe (source file indexer) paths for this compilation module - kytheFiles android.Paths - // Object .o file output paths for this compilation module - objFiles android.Paths - // Tidy .tidy file output paths for this compilation module - tidyFiles android.Paths // For apex variants, this is set as apex.min_sdk_version apexSdkVersion android.ApiLevel @@ -913,8 +921,16 @@ type Module struct { hasSysprop bool hasWinMsg bool hasYacc bool + + makeVarsInfo *CcMakeVarsInfo } +func (c *Module) IncrementalSupported() bool { + return c.incremental +} + +var _ blueprint.Incremental = (*Module)(nil) + func (c *Module) AddJSONData(d *map[string]interface{}) { c.AndroidModuleBase().AddJSONData(d) (*d)["Cc"] = map[string]interface{}{ @@ -1454,10 +1470,6 @@ func InstallToBootstrap(name string, config android.Config) bool { return isBionic(name) } -func (c *Module) XrefCcFiles() android.Paths { - return c.kytheFiles -} - func (c *Module) isCfiAssemblySupportEnabled() bool { return c.sanitize != nil && Bool(c.sanitize.Properties.Sanitize.Config.Cfi_assembly_support) @@ -1699,6 +1711,13 @@ func (ctx *moduleContextImpl) notInPlatform() bool { return ctx.mod.NotInPlatform() } +func (ctx *moduleContextImpl) getOrCreateMakeVarsInfo() *CcMakeVarsInfo { + if ctx.mod.makeVarsInfo == nil { + ctx.mod.makeVarsInfo = &CcMakeVarsInfo{} + } + return ctx.mod.makeVarsInfo +} + func newBaseModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module { return &Module{ hod: hod, @@ -2022,9 +2041,6 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if ctx.Failed() { return } - c.kytheFiles = objs.kytheFiles - c.objFiles = objs.objFiles - c.tidyFiles = objs.tidyFiles } if c.linker != nil { @@ -2093,7 +2109,28 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { c.hasYacc = b.hasSrcExt(ctx, ".y") || b.hasSrcExt(ctx, ".yy") } + ccObjectInfo := CcObjectInfo{ + kytheFiles: objs.kytheFiles, + } + if !ctx.Config().KatiEnabled() || !android.ShouldSkipAndroidMkProcessing(ctx, c) { + ccObjectInfo.objFiles = objs.objFiles + ccObjectInfo.tidyFiles = objs.tidyFiles + } + if len(ccObjectInfo.kytheFiles)+len(ccObjectInfo.objFiles)+len(ccObjectInfo.tidyFiles) > 0 { + android.SetProvider(ctx, CcObjectInfoProvider, ccObjectInfo) + } + c.setOutputFiles(ctx) + + if c.makeVarsInfo != nil { + android.SetProvider(ctx, CcMakeVarsInfoProvider, c.makeVarsInfo) + } +} + +func setOutputFilesIfNotEmpty(ctx ModuleContext, files android.Paths, tag string) { + if len(files) > 0 { + ctx.SetOutputFiles(files, tag) + } } func (c *Module) setOutputFiles(ctx ModuleContext) { @@ -2169,6 +2206,7 @@ func (c *Module) maybeInstall(ctx ModuleContext, apexInfo android.ApexInfo) { // modules can be hidden from make as some are needed for resolving make side // dependencies. c.HideFromMake() + c.SkipInstall() } else if !installable(c, apexInfo) { c.SkipInstall() } @@ -3929,9 +3967,10 @@ type kytheExtractAllSingleton struct { func (ks *kytheExtractAllSingleton) GenerateBuildActions(ctx android.SingletonContext) { var xrefTargets android.Paths - ctx.VisitAllModules(func(module android.Module) { - if ccModule, ok := module.(xref); ok { - xrefTargets = append(xrefTargets, ccModule.XrefCcFiles()...) + ctx.VisitAllModuleProxies(func(module android.ModuleProxy) { + files := android.OtherModuleProviderOrDefault(ctx, module, CcObjectInfoProvider).kytheFiles + if len(files) > 0 { + xrefTargets = append(xrefTargets, files...) } }) // TODO(asmundak): Perhaps emit a rule to output a warning if there were no xrefTargets diff --git a/cc/compiler.go b/cc/compiler.go index f06287cf2..91f107c29 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -78,11 +78,11 @@ type BaseCompilerProperties struct { // If possible, don't use this. If adding paths from the current directory use // local_include_dirs, if adding paths from other modules use export_include_dirs in // that module. - Include_dirs []string `android:"arch_variant,variant_prepend"` + Include_dirs proptools.Configurable[[]string] `android:"arch_variant,variant_prepend"` // list of directories relative to the Blueprints file that will // be added to the include path using -I - Local_include_dirs []string `android:"arch_variant,variant_prepend"` + Local_include_dirs proptools.Configurable[[]string] `android:"arch_variant,variant_prepend"` // Add the directory containing the Android.bp file to the list of include // directories. Defaults to true. @@ -411,13 +411,13 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps } // Include dir cflags - localIncludeDirs := android.PathsForModuleSrc(ctx, compiler.Properties.Local_include_dirs) + localIncludeDirs := android.PathsForModuleSrc(ctx, compiler.Properties.Local_include_dirs.GetOrDefault(ctx, nil)) if len(localIncludeDirs) > 0 { f := includeDirsToFlags(localIncludeDirs) flags.Local.CommonFlags = append(flags.Local.CommonFlags, f) flags.Local.YasmFlags = append(flags.Local.YasmFlags, f) } - rootIncludeDirs := android.PathsForSource(ctx, compiler.Properties.Include_dirs) + rootIncludeDirs := android.PathsForSource(ctx, compiler.Properties.Include_dirs.GetOrDefault(ctx, nil)) if len(rootIncludeDirs) > 0 { f := includeDirsToFlags(rootIncludeDirs) flags.Local.CommonFlags = append(flags.Local.CommonFlags, f) @@ -685,10 +685,10 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps if len(srcs) > 0 { module := ctx.ModuleDir() + "/Android.bp:" + ctx.ModuleName() if inList("-Wno-error", flags.Local.CFlags) || inList("-Wno-error", flags.Local.CppFlags) { - addToModuleList(ctx, modulesUsingWnoErrorKey, module) + ctx.getOrCreateMakeVarsInfo().UsingWnoError = module } else if !inList("-Werror", flags.Local.CFlags) && !inList("-Werror", flags.Local.CppFlags) { if warningsAreAllowed(ctx.ModuleDir()) { - addToModuleList(ctx, modulesWarningsAllowedKey, module) + ctx.getOrCreateMakeVarsInfo().WarningsAllowed = module } else { flags.Local.CFlags = append([]string{"-Werror"}, flags.Local.CFlags...) } @@ -807,7 +807,7 @@ func compileObjs(ctx ModuleContext, flags builderFlags, subdir string, type RustBindgenClangProperties struct { // list of directories relative to the Blueprints file that will // be added to the include path using -I - Local_include_dirs []string `android:"arch_variant,variant_prepend"` + Local_include_dirs proptools.Configurable[[]string] `android:"arch_variant,variant_prepend"` // list of static libraries that provide headers for this binding. Static_libs proptools.Configurable[[]string] `android:"arch_variant,variant_prepend"` diff --git a/cc/config/global.go b/cc/config/global.go index d63f86d66..36690d6be 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -72,6 +72,9 @@ var ( // Warnings disabled by default. + // We should encourage use of C23 features even when the whole project + // isn't C23-ready. + "-Wno-c23-extensions", // Designated initializer syntax is recommended by the Google C++ style // and is OK to use even if not formally supported by the chosen C++ // version. diff --git a/cc/makevars.go b/cc/makevars.go index f82e0e90e..4cb98e70a 100644 --- a/cc/makevars.go +++ b/cc/makevars.go @@ -25,10 +25,7 @@ import ( ) var ( - modulesWarningsAllowedKey = android.NewOnceKey("ModulesWarningsAllowed") - modulesUsingWnoErrorKey = android.NewOnceKey("ModulesUsingWnoError") - modulesMissingProfileFileKey = android.NewOnceKey("ModulesMissingProfileFile") - sanitizerVariables = map[string]string{ + sanitizerVariables = map[string]string{ "ADDRESS_SANITIZER_RUNTIME_LIBRARY": config.AddressSanitizerRuntimeLibrary(), "HWADDRESS_SANITIZER_RUNTIME_LIBRARY": config.HWAddressSanitizerRuntimeLibrary(), "HWADDRESS_SANITIZER_STATIC_LIBRARY": config.HWAddressSanitizerStaticLibrary(), @@ -50,15 +47,9 @@ func getNamedMapForConfig(config android.Config, key android.OnceKey) *sync.Map }).(*sync.Map) } -func makeStringOfKeys(ctx android.MakeVarsContext, key android.OnceKey) string { - set := getNamedMapForConfig(ctx.Config(), key) - keys := []string{} - set.Range(func(key interface{}, value interface{}) bool { - keys = append(keys, key.(string)) - return true - }) - sort.Strings(keys) - return strings.Join(keys, " ") +func makeVarsString(items []string) string { + items = android.SortedUniqueStrings(items) + return strings.Join(items, " ") } func makeStringOfWarningAllowedProjects() string { @@ -108,28 +99,33 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("GLOBAL_CLANG_EXTERNAL_CFLAGS_NO_OVERRIDE", "${config.NoOverrideExternalGlobalCflags}") // Filter vendor_public_library that are exported to make - exportedVendorPublicLibraries := []string{} + var exportedVendorPublicLibraries []string + var warningsAllowed []string + var usingWnoErrors []string + var missingProfiles []string ctx.VisitAllModules(func(module android.Module) { + if v, ok := android.OtherModuleProvider(ctx, module, CcMakeVarsInfoProvider); ok { + warningsAllowed = android.AppendIfNotZero(warningsAllowed, v.WarningsAllowed) + usingWnoErrors = android.AppendIfNotZero(usingWnoErrors, v.UsingWnoError) + missingProfiles = android.AppendIfNotZero(missingProfiles, v.MissingProfile) + } if ccModule, ok := module.(*Module); ok { baseName := ccModule.BaseModuleName() if ccModule.IsVendorPublicLibrary() && module.ExportedToMake() { - if !inList(baseName, exportedVendorPublicLibraries) { - exportedVendorPublicLibraries = append(exportedVendorPublicLibraries, baseName) - } + exportedVendorPublicLibraries = append(exportedVendorPublicLibraries, baseName) } } }) - sort.Strings(exportedVendorPublicLibraries) - ctx.Strict("VENDOR_PUBLIC_LIBRARIES", strings.Join(exportedVendorPublicLibraries, " ")) + ctx.Strict("VENDOR_PUBLIC_LIBRARIES", makeVarsString(exportedVendorPublicLibraries)) lsdumpPaths := *lsdumpPaths(ctx.Config()) sort.Strings(lsdumpPaths) ctx.Strict("LSDUMP_PATHS", strings.Join(lsdumpPaths, " ")) ctx.Strict("ANDROID_WARNING_ALLOWED_PROJECTS", makeStringOfWarningAllowedProjects()) - ctx.Strict("SOONG_MODULES_WARNINGS_ALLOWED", makeStringOfKeys(ctx, modulesWarningsAllowedKey)) - ctx.Strict("SOONG_MODULES_USING_WNO_ERROR", makeStringOfKeys(ctx, modulesUsingWnoErrorKey)) - ctx.Strict("SOONG_MODULES_MISSING_PGO_PROFILE_FILE", makeStringOfKeys(ctx, modulesMissingProfileFileKey)) + ctx.Strict("SOONG_MODULES_WARNINGS_ALLOWED", makeVarsString(warningsAllowed)) + ctx.Strict("SOONG_MODULES_USING_WNO_ERROR", makeVarsString(usingWnoErrors)) + ctx.Strict("SOONG_MODULES_MISSING_PGO_PROFILE_FILE", makeVarsString(missingProfiles)) ctx.Strict("CLANG_COVERAGE_CONFIG_CFLAGS", strings.Join(clangCoverageCFlags, " ")) ctx.Strict("CLANG_COVERAGE_CONFIG_COMMFLAGS", strings.Join(clangCoverageCommonFlags, " ")) diff --git a/cc/ndk_library.go b/cc/ndk_library.go index 01551ab49..2411614a4 100644 --- a/cc/ndk_library.go +++ b/cc/ndk_library.go @@ -144,11 +144,9 @@ func (stub *stubDecorator) implementationModuleName(name string) string { } func ndkLibraryVersions(ctx android.BaseModuleContext, from android.ApiLevel) []string { - var versions []android.ApiLevel versionStrs := []string{} - for _, version := range ctx.Config().AllSupportedApiLevels() { + for _, version := range ctx.Config().FinalApiLevels() { if version.GreaterThanOrEqualTo(from) { - versions = append(versions, version) versionStrs = append(versionStrs, version.String()) } } @@ -330,6 +328,12 @@ func (this *stubDecorator) findPrebuiltAbiDump(ctx ModuleContext, return android.ExistentPathForSource(ctx, subpath) } +func (this *stubDecorator) builtAbiDumpLocation(ctx ModuleContext, apiLevel android.ApiLevel) android.OutputPath { + return getNdkAbiDumpInstallBase(ctx).Join(ctx, + apiLevel.String(), ctx.Arch().ArchType.String(), + this.libraryName(ctx), "abi.stg") +} + // Feature flag. func (this *stubDecorator) canDumpAbi(ctx ModuleContext) bool { if runtime.GOOS == "darwin" { @@ -345,25 +349,24 @@ func (this *stubDecorator) canDumpAbi(ctx ModuleContext) bool { return false } - if this.apiLevel.IsCurrent() { - // "current" (AKA 10000) is not tracked. - return false - } - // http://b/156513478 return ctx.Config().ReleaseNdkAbiMonitored() } // Feature flag to disable diffing against prebuilts. -func canDiffAbi(config android.Config) bool { +func (this *stubDecorator) canDiffAbi(config android.Config) bool { + if this.apiLevel.IsCurrent() { + // Diffs are performed from this to next, and there's nothing after + // current. + return false + } + return config.ReleaseNdkAbiMonitored() } func (this *stubDecorator) dumpAbi(ctx ModuleContext, symbolList android.Path) { implementationLibrary := this.findImplementationLibrary(ctx) - this.abiDumpPath = getNdkAbiDumpInstallBase(ctx).Join(ctx, - this.apiLevel.String(), ctx.Arch().ArchType.String(), - this.libraryName(ctx), "abi.stg") + this.abiDumpPath = this.builtAbiDumpLocation(ctx, this.apiLevel) this.hasAbiDump = true headersList := getNdkABIHeadersFile(ctx) ctx.Build(pctx, android.BuildParams{ @@ -383,7 +386,7 @@ func (this *stubDecorator) dumpAbi(ctx ModuleContext, symbolList android.Path) { } func findNextApiLevel(ctx ModuleContext, apiLevel android.ApiLevel) *android.ApiLevel { - apiLevels := append(ctx.Config().AllSupportedApiLevels(), + apiLevels := append(ctx.Config().FinalApiLevels(), android.FutureApiLevel) for _, api := range apiLevels { if api.GreaterThan(apiLevel) { @@ -436,38 +439,49 @@ func (this *stubDecorator) diffAbi(ctx ModuleContext) { "non-current API level %s", this.apiLevel)) } - // "current" ABI is not tracked. - if !nextApiLevel.IsCurrent() { - nextAbiDiffPath := android.PathForModuleOut(ctx, - "abidiff_next.timestamp") - nextAbiDump := this.findPrebuiltAbiDump(ctx, *nextApiLevel) + // Preview ABI levels are not recorded in prebuilts. ABI compatibility + // for preview APIs is still monitored via "current" so we have early + // warning rather than learning about an ABI break during finalization, + // but is only checked against the "current" API dumps in the out + // directory. + nextAbiDiffPath := android.PathForModuleOut(ctx, + "abidiff_next.timestamp") + + var nextAbiDump android.OptionalPath + if nextApiLevel.IsCurrent() { + nextAbiDump = android.OptionalPathForPath( + this.builtAbiDumpLocation(ctx, *nextApiLevel), + ) + } else { + nextAbiDump = this.findPrebuiltAbiDump(ctx, *nextApiLevel) + } + + if !nextAbiDump.Valid() { missingNextPrebuiltError := fmt.Sprintf( missingPrebuiltErrorTemplate, this.libraryName(ctx), nextAbiDump.InvalidReason()) - if !nextAbiDump.Valid() { - ctx.Build(pctx, android.BuildParams{ - Rule: android.ErrorRule, - Output: nextAbiDiffPath, - Args: map[string]string{ - "error": missingNextPrebuiltError, - }, - }) - } else { - ctx.Build(pctx, android.BuildParams{ - Rule: stgdiff, - Description: fmt.Sprintf( - "Comparing ABI to the next API level %s %s", - prebuiltAbiDump, nextAbiDump), - Output: nextAbiDiffPath, - Inputs: android.Paths{ - prebuiltAbiDump.Path(), nextAbiDump.Path()}, - Args: map[string]string{ - "args": "--format=small --ignore=interface_addition", - }, - }) - } - this.abiDiffPaths = append(this.abiDiffPaths, nextAbiDiffPath) + ctx.Build(pctx, android.BuildParams{ + Rule: android.ErrorRule, + Output: nextAbiDiffPath, + Args: map[string]string{ + "error": missingNextPrebuiltError, + }, + }) + } else { + ctx.Build(pctx, android.BuildParams{ + Rule: stgdiff, + Description: fmt.Sprintf( + "Comparing ABI to the next API level %s %s", + prebuiltAbiDump, nextAbiDump), + Output: nextAbiDiffPath, + Inputs: android.Paths{ + prebuiltAbiDump.Path(), nextAbiDump.Path()}, + Args: map[string]string{ + "args": "--format=small --ignore=interface_addition", + }, + }) } + this.abiDiffPaths = append(this.abiDiffPaths, nextAbiDiffPath) } } @@ -492,7 +506,7 @@ func (c *stubDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) O c.versionScriptPath = nativeAbiResult.versionScript if c.canDumpAbi(ctx) { c.dumpAbi(ctx, nativeAbiResult.symbolList) - if canDiffAbi(ctx.Config()) { + if c.canDiffAbi(ctx.Config()) { c.diffAbi(ctx) } } diff --git a/cc/orderfile.go b/cc/orderfile.go index 38b89059b..6e08da7a0 100644 --- a/cc/orderfile.go +++ b/cc/orderfile.go @@ -54,10 +54,6 @@ func getOrderfileProjects(config android.DeviceConfig) []string { }) } -func recordMissingOrderfile(ctx BaseModuleContext, missing string) { - getNamedMapForConfig(ctx.Config(), modulesMissingProfileFileKey).Store(missing, true) -} - type OrderfileProperties struct { Orderfile struct { Instrumentation *bool @@ -117,7 +113,7 @@ func (props *OrderfileProperties) getOrderfile(ctx BaseModuleContext) android.Op // Record that this module's order file is absent missing := *props.Orderfile.Order_file_path + ":" + ctx.ModuleDir() + "/Android.bp:" + ctx.ModuleName() - recordMissingOrderfile(ctx, missing) + ctx.getOrCreateMakeVarsInfo().MissingProfile = missing return android.OptionalPath{} } diff --git a/cc/tidy.go b/cc/tidy.go index ec1e8a206..5cbf8f076 100644 --- a/cc/tidy.go +++ b/cc/tidy.go @@ -219,15 +219,11 @@ func collectTidyObjModuleTargets(ctx android.SingletonContext, module android.Mo subsetTidyFileGroups := make(map[string]android.Paths) // subset group name => tidy file Paths // (1) Collect all obj/tidy files into OS-specific groups. - ctx.VisitAllModuleVariants(module, func(variant android.Module) { - if ctx.Config().KatiEnabled() && android.ShouldSkipAndroidMkProcessing(ctx, variant) { - return - } - if m, ok := variant.(*Module); ok { - osName := variant.Target().Os.Name - addToOSGroup(osName, m.objFiles, allObjFileGroups, subsetObjFileGroups) - addToOSGroup(osName, m.tidyFiles, allTidyFileGroups, subsetTidyFileGroups) - } + ctx.VisitAllModuleVariantProxies(module, func(variant android.ModuleProxy) { + osName := android.OtherModuleProviderOrDefault(ctx, variant, android.CommonPropertiesProviderKey).CompileTarget.Os.Name + info := android.OtherModuleProviderOrDefault(ctx, variant, CcObjectInfoProvider) + addToOSGroup(osName, info.objFiles, allObjFileGroups, subsetObjFileGroups) + addToOSGroup(osName, info.tidyFiles, allTidyFileGroups, subsetTidyFileGroups) }) // (2) Add an all-OS group, with "" or "subset" name, to include all os-specific phony targets. @@ -258,7 +254,7 @@ func (m *tidyPhonySingleton) GenerateBuildActions(ctx android.SingletonContext) // Collect tidy/obj targets from the 'final' modules. ctx.VisitAllModules(func(module android.Module) { - if module == ctx.FinalModule(module) { + if ctx.IsFinalModule(module) { collectTidyObjModuleTargets(ctx, module, tidyModulesInDirGroup, objModulesInDirGroup) } }) diff --git a/cmd/find_input_delta/find_input_delta/Android.bp b/cmd/find_input_delta/find_input_delta/Android.bp new file mode 100644 index 000000000..6b2dbc764 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta/Android.bp @@ -0,0 +1,18 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-cmd-find_input_delta-find_input_delta", + pkgPath: "android/soong/cmd/find_input_delta/find_input_delta", + deps: [ + "golang-protobuf-encoding-prototext", + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-find_input_delta-proto", + "soong-cmd-find_input_delta-lib", + ], + srcs: [ + "main.go", + ], +} diff --git a/cmd/find_input_delta/find_input_delta/main.go b/cmd/find_input_delta/find_input_delta/main.go new file mode 100644 index 000000000..6b657ea57 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta/main.go @@ -0,0 +1,88 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os" + "strings" + + fid_lib "android/soong/cmd/find_input_delta/find_input_delta_lib" +) + +func main() { + var top string + var prior_state_file string + var new_state_file string + var target string + var inputs_file string + var template string + var inputs []string + var inspect bool + var err error + + flag.StringVar(&top, "top", ".", "path to top of workspace") + flag.StringVar(&prior_state_file, "prior_state", "", "prior internal state file") + flag.StringVar(&new_state_file, "new_state", "", "new internal state file") + flag.StringVar(&target, "target", "", "name of ninja output file for build action") + flag.StringVar(&inputs_file, "inputs_file", "", "file containing list of input files") + flag.StringVar(&template, "template", fid_lib.DefaultTemplate, "output template for FileList") + flag.BoolVar(&inspect, "inspect", false, "whether to inspect file contents") + + flag.Parse() + + if target == "" { + panic("must specify --target") + } + if prior_state_file == "" { + prior_state_file = target + ".pc_state" + } + if new_state_file == "" { + new_state_file = prior_state_file + ".new" + } + + if err = os.Chdir(top); err != nil { + panic(err) + } + + inputs = flag.Args() + if inputs_file != "" { + data, err := os.ReadFile(inputs_file) + if err != nil { + panic(err) + } + inputs = append(inputs, strings.Split(string(data), "\n")...) + } + + // Read the prior state + prior_state, err := fid_lib.LoadState(prior_state_file, fid_lib.OsFs) + if err != nil { + panic(err) + } + // Create the new state + new_state, err := fid_lib.CreateState(inputs, inspect, fid_lib.OsFs) + if err != nil { + panic(err) + } + if err = fid_lib.WriteState(new_state, new_state_file); err != nil { + panic(err) + } + + file_list := *fid_lib.CompareInternalState(prior_state, new_state, target) + + if err = file_list.Format(os.Stdout, template); err != nil { + panic(err) + } +} diff --git a/cmd/find_input_delta/find_input_delta_lib/Android.bp b/cmd/find_input_delta/find_input_delta_lib/Android.bp new file mode 100644 index 000000000..795b14038 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_lib/Android.bp @@ -0,0 +1,34 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-cmd-find_input_delta-lib", + pkgPath: "android/soong/cmd/find_input_delta/find_input_delta_lib", + deps: [ + "golang-protobuf-encoding-prototext", + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-find_input_delta-proto", + "blueprint-pathtools", + ], + srcs: [ + "fs.go", + "file_list.go", + "internal_state.go", + ], +} diff --git a/cmd/find_input_delta/find_input_delta_lib/file_list.go b/cmd/find_input_delta/find_input_delta_lib/file_list.go new file mode 100644 index 000000000..23337adc3 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_lib/file_list.go @@ -0,0 +1,78 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package find_input_delta_lib + +import ( + "io" + "text/template" + + fid_exp "android/soong/cmd/find_input_delta/find_input_delta_proto" + "google.golang.org/protobuf/proto" +) + +var DefaultTemplate = ` + {{- define "contents"}} + {{- range .Deletions}}-{{.}} {{end}} + {{- range .Additions}}+{{.}} {{end}} + {{- range .Changes}}+{{- .Name}} {{end}} + {{- range .Changes}} + {{- if or .Additions .Deletions .Changes}}--file {{.Name}} {{template "contents" .}}--endfile {{end}} + {{- end}} + {{- end}} + {{- template "contents" .}}` + +type FileList struct { + // The name of the parent for the list of file differences. + // For the outermost FileList, this is the name of the ninja target. + // Under `Changes`, it is the name of the changed file. + Name string + + // The added files + Additions []string + + // The deleted files + Deletions []string + + // The modified files + Changes []FileList +} + +func (fl FileList) Marshal() (*fid_exp.FileList, error) { + ret := &fid_exp.FileList{ + Name: proto.String(fl.Name), + } + if len(fl.Additions) > 0 { + ret.Additions = fl.Additions + } + for _, ch := range fl.Changes { + change, err := ch.Marshal() + if err != nil { + return nil, err + } + ret.Changes = append(ret.Changes, change) + } + if len(fl.Deletions) > 0 { + ret.Deletions = fl.Deletions + } + return ret, nil +} + +func (fl FileList) Format(wr io.Writer, format string) error { + tmpl, err := template.New("filelist").Parse(format) + if err != nil { + return err + } + return tmpl.Execute(wr, fl) +} diff --git a/cmd/find_input_delta/find_input_delta_lib/file_list_test.go b/cmd/find_input_delta/find_input_delta_lib/file_list_test.go new file mode 100644 index 000000000..2459f1ead --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_lib/file_list_test.go @@ -0,0 +1,131 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package find_input_delta_lib + +import ( + "bytes" + "slices" + "testing" + + // For Assert*. + "android/soong/android" +) + +func (fl *FileList) Equal(other *FileList) bool { + if fl.Name != other.Name { + return false + } + if !slices.Equal(fl.Additions, other.Additions) { + return false + } + if !slices.Equal(fl.Deletions, other.Deletions) { + return false + } + if len(fl.Changes) != len(other.Changes) { + return false + } + for idx, ch := range fl.Changes { + if !ch.Equal(&other.Changes[idx]) { + return false + } + } + return true +} + +func TestFormat(t *testing.T) { + testCases := []struct { + Name string + Template string + Input FileList + Expected string + Err error + }{ + { + Name: "no contents", + Template: DefaultTemplate, + Input: FileList{ + Name: "target", + Additions: []string{"add1", "add2"}, + Deletions: []string{"del1", "del2"}, + Changes: []FileList{ + FileList{Name: "mod1"}, + FileList{Name: "mod2"}, + }, + }, + Expected: "-del1 -del2 +add1 +add2 +mod1 +mod2 ", + Err: nil, + }, + { + Name: "adds", + Template: DefaultTemplate, + Input: FileList{ + Name: "target", + Additions: []string{"add1", "add2"}, + }, + Expected: "+add1 +add2 ", + Err: nil, + }, + { + Name: "deletes", + Template: DefaultTemplate, + Input: FileList{ + Name: "target", + Deletions: []string{"del1", "del2"}, + }, + Expected: "-del1 -del2 ", + Err: nil, + }, + { + Name: "changes", + Template: DefaultTemplate, + Input: FileList{ + Name: "target", + Changes: []FileList{ + FileList{Name: "mod1"}, + FileList{Name: "mod2"}, + }, + }, + Expected: "+mod1 +mod2 ", + Err: nil, + }, + { + Name: "with contents", + Template: DefaultTemplate, + Input: FileList{ + Name: "target", + Additions: []string{"add1", "add2"}, + Deletions: []string{"del1", "del2"}, + Changes: []FileList{ + FileList{ + Name: "mod1", + }, + FileList{ + Name: "mod2", + Additions: []string{"a1"}, + Deletions: []string{"d1"}, + }, + }, + }, + Expected: "-del1 -del2 +add1 +add2 +mod1 +mod2 --file mod2 -d1 +a1 --endfile ", + Err: nil, + }, + } + for _, tc := range testCases { + buf := bytes.NewBuffer([]byte{}) + err := tc.Input.Format(buf, tc.Template) + android.AssertSame(t, tc.Name, tc.Err, err) + android.AssertSame(t, tc.Name, tc.Expected, buf.String()) + } +} diff --git a/cmd/find_input_delta/find_input_delta_lib/fs.go b/cmd/find_input_delta/find_input_delta_lib/fs.go new file mode 100644 index 000000000..4a83ed7ff --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_lib/fs.go @@ -0,0 +1,46 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package find_input_delta_lib + +import ( + "io" + "io/fs" + "os" +) + +// OsFs provides a minimal implementation so that we can use testing/fstest for +// unit tests. +var OsFs fileSystem = osFS{} + +type fileSystem interface { + Open(path string) (fs.File, error) + Stat(path string) (os.FileInfo, error) + ReadFile(path string) ([]byte, error) +} + +type file interface { + io.Closer + io.Reader + io.ReaderAt + io.Seeker + Stat() (os.FileInfo, error) +} + +// osFS implements fileSystem using the local disk. +type osFS struct{} + +func (osFS) Open(path string) (fs.File, error) { return os.Open(path) } +func (osFS) Stat(path string) (os.FileInfo, error) { return os.Stat(path) } +func (osFS) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } diff --git a/cmd/find_input_delta/find_input_delta_lib/internal_state.go b/cmd/find_input_delta/find_input_delta_lib/internal_state.go new file mode 100644 index 000000000..b2ff8c704 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_lib/internal_state.go @@ -0,0 +1,122 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package find_input_delta_lib + +import ( + "errors" + "fmt" + "io/fs" + "slices" + + fid_proto "android/soong/cmd/find_input_delta/find_input_delta_proto_internal" + "github.com/google/blueprint/pathtools" + "google.golang.org/protobuf/proto" +) + +// Load the internal state from a file. +// If the file does not exist, an empty state is returned. +func LoadState(filename string, fsys fs.ReadFileFS) (*fid_proto.PartialCompileInputs, error) { + var message = &fid_proto.PartialCompileInputs{} + data, err := fsys.ReadFile(filename) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return message, err + } + proto.Unmarshal(data, message) + return message, nil +} + +type StatReadFileFS interface { + fs.StatFS + fs.ReadFileFS +} + +// Create the internal state by examining the inputs. +func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (*fid_proto.PartialCompileInputs, error) { + ret := &fid_proto.PartialCompileInputs{} + slices.Sort(inputs) + for _, input := range inputs { + stat, err := fs.Stat(fsys, input) + if err != nil { + return ret, err + } + pci := &fid_proto.PartialCompileInput{ + Name: proto.String(input), + MtimeNsec: proto.Int64(stat.ModTime().UnixNano()), + // If we ever have an easy hash, assign it here. + } + if inspect_contents { + contents, err := InspectFileContents(input) + if err != nil { + return ret, err + } + if contents != nil { + pci.Contents = contents + } + } + ret.InputFiles = append(ret.InputFiles, pci) + } + return ret, nil +} + +// Inspect the file and extract the state of the elements in the archive. +// If this is not an archive of some sort, nil is returned. +func InspectFileContents(name string) ([]*fid_proto.PartialCompileInput, error) { + // TODO: Actually inspect the contents. + fmt.Printf("inspecting contents for %s\n", name) + return nil, nil +} + +func WriteState(s *fid_proto.PartialCompileInputs, path string) error { + data, err := proto.Marshal(s) + if err != nil { + return err + } + return pathtools.WriteFileIfChanged(path, data, 0644) +} + +func CompareInternalState(prior, other *fid_proto.PartialCompileInputs, target string) *FileList { + return CompareInputFiles(prior.GetInputFiles(), other.GetInputFiles(), target) +} + +func CompareInputFiles(prior, other []*fid_proto.PartialCompileInput, name string) *FileList { + fl := &FileList{ + Name: name, + } + PriorMap := make(map[string]*fid_proto.PartialCompileInput, len(prior)) + // We know that the lists are properly sorted, so we can simply compare them. + for _, v := range prior { + PriorMap[v.GetName()] = v + } + otherMap := make(map[string]*fid_proto.PartialCompileInput, len(other)) + for _, v := range other { + name = v.GetName() + otherMap[name] = v + if _, ok := PriorMap[name]; !ok { + // Added file + fl.Additions = append(fl.Additions, name) + } else if !proto.Equal(PriorMap[name], v) { + // Changed file + fl.Changes = append(fl.Changes, *CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name)) + } + } + for _, v := range prior { + name := v.GetName() + if _, ok := otherMap[name]; !ok { + // Deleted file + fl.Deletions = append(fl.Deletions, name) + } + } + return fl +} diff --git a/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go b/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go new file mode 100644 index 000000000..20b8efaa8 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go @@ -0,0 +1,232 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package find_input_delta_lib + +import ( + "errors" + "io/fs" + "testing" + "testing/fstest" + "time" + + // For Assert*. + "android/soong/android" + + fid_proto "android/soong/cmd/find_input_delta/find_input_delta_proto_internal" + "google.golang.org/protobuf/proto" +) + +// Various state files + +func marshalProto(t *testing.T, message proto.Message) []byte { + data, err := proto.Marshal(message) + if err != nil { + t.Errorf("%v", err) + } + return data +} + +func protoFile(name string, mtime_nsec int64, hash string, contents []*fid_proto.PartialCompileInput) (pci *fid_proto.PartialCompileInput) { + pci = &fid_proto.PartialCompileInput{ + Name: proto.String(name), + } + if mtime_nsec != 0 { + pci.MtimeNsec = proto.Int64(mtime_nsec) + } + if len(hash) > 0 { + pci.Hash = proto.String(hash) + } + if contents != nil { + pci.Contents = contents + } + return +} + +func TestLoadState(t *testing.T) { + testCases := []struct { + Name string + Filename string + Mapfs fs.ReadFileFS + Expected *fid_proto.PartialCompileInputs + Err error + }{ + { + Name: "missing file", + Filename: "missing", + Mapfs: fstest.MapFS{}, + Expected: &fid_proto.PartialCompileInputs{}, + Err: nil, + }, + { + Name: "bad file", + Filename: ".", + Mapfs: OsFs, + Expected: &fid_proto.PartialCompileInputs{}, + Err: errors.New("read failed"), + }, + { + Name: "file with mtime", + Filename: "state.old", + Mapfs: fstest.MapFS{ + "state.old": &fstest.MapFile{ + Data: marshalProto(t, &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("input1", 100, "", nil), + }, + }), + }, + }, + Expected: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("input1", 100, "", nil), + }, + }, + Err: nil, + }, + { + Name: "file with mtime and hash", + Filename: "state.old", + Mapfs: fstest.MapFS{ + "state.old": &fstest.MapFile{ + Data: marshalProto(t, &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("input1", 100, "crc:crc_value", nil), + }, + }), + }, + }, + Expected: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("input1", 100, "crc:crc_value", nil), + }, + }, + Err: nil, + }, + } + for _, tc := range testCases { + actual, err := LoadState(tc.Filename, tc.Mapfs) + if tc.Err == nil { + android.AssertSame(t, tc.Name, tc.Err, err) + } else if err == nil { + t.Errorf("%s: expected error, did not get one", tc.Name) + } + if !proto.Equal(tc.Expected, actual) { + t.Errorf("%s: expected %v, actual %v", tc.Name, tc.Expected, actual) + } + } +} + +func TestCreateState(t *testing.T) { + testCases := []struct { + Name string + Inputs []string + Inspect bool + Mapfs StatReadFileFS + Expected *fid_proto.PartialCompileInputs + Err error + }{ + { + Name: "no inputs", + Inputs: []string{}, + Mapfs: fstest.MapFS{}, + Expected: &fid_proto.PartialCompileInputs{}, + Err: nil, + }, + { + Name: "files found", + Inputs: []string{"baz", "foo", "bar"}, + Mapfs: fstest.MapFS{ + "foo": &fstest.MapFile{ModTime: time.Unix(0, 100).UTC()}, + "baz": &fstest.MapFile{ModTime: time.Unix(0, 300).UTC()}, + "bar": &fstest.MapFile{ModTime: time.Unix(0, 200).UTC()}, + }, + Expected: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + // Files are always sorted. + protoFile("bar", 200, "", nil), + protoFile("baz", 300, "", nil), + protoFile("foo", 100, "", nil), + }, + }, + Err: nil, + }, + } + for _, tc := range testCases { + actual, err := CreateState(tc.Inputs, tc.Inspect, tc.Mapfs) + if tc.Err == nil { + android.AssertSame(t, tc.Name, tc.Err, err) + } else if err == nil { + t.Errorf("%s: expected error, did not get one", tc.Name) + } + if !proto.Equal(tc.Expected, actual) { + t.Errorf("%s: expected %v, actual %v", tc.Name, tc.Expected, actual) + } + } +} + +func TestCompareInternalState(t *testing.T) { + testCases := []struct { + Name string + Target string + Prior *fid_proto.PartialCompileInputs + New *fid_proto.PartialCompileInputs + Expected *FileList + }{ + { + Name: "prior is empty", + Target: "foo", + Prior: &fid_proto.PartialCompileInputs{}, + New: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file1", 100, "", nil), + }, + }, + Expected: &FileList{ + Name: "foo", + Additions: []string{"file1"}, + }, + }, + { + Name: "one of each", + Target: "foo", + Prior: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file0", 100, "", nil), + protoFile("file1", 100, "", nil), + protoFile("file2", 200, "", nil), + }, + }, + New: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file0", 100, "", nil), + protoFile("file1", 200, "", nil), + protoFile("file3", 300, "", nil), + }, + }, + Expected: &FileList{ + Name: "foo", + Additions: []string{"file3"}, + Changes: []FileList{FileList{Name: "file1"}}, + Deletions: []string{"file2"}, + }, + }, + } + for _, tc := range testCases { + actual := CompareInternalState(tc.Prior, tc.New, tc.Target) + if !tc.Expected.Equal(actual) { + t.Errorf("%s: expected %q, actual %q", tc.Name, tc.Expected, actual) + } + } +} diff --git a/cmd/find_input_delta/find_input_delta_proto/Android.bp b/cmd/find_input_delta/find_input_delta_proto/Android.bp new file mode 100644 index 000000000..18eba6bb8 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_proto/Android.bp @@ -0,0 +1,29 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-cmd-find_input_delta-proto", + pkgPath: "android/soong/cmd/find_input_delta/find_input_delta_proto", + deps: [ + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + ], + srcs: [ + "target_delta_files.pb.go", + ], +} diff --git a/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go b/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go new file mode 100644 index 000000000..648ef224a --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go @@ -0,0 +1,198 @@ +// +// Copyright (C) 2024 The Android Open-Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.21.12 +// source: file_list.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FileList struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The name of the file. + // In the outermost message, this is the name of the Ninja target. + // When used in `changes`, this is the name of the changed file. + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The added files. + Additions []string `protobuf:"bytes,2,rep,name=additions" json:"additions,omitempty"` + // The deleted files. + Deletions []string `protobuf:"bytes,3,rep,name=deletions" json:"deletions,omitempty"` + // The changed files. + Changes []*FileList `protobuf:"bytes,4,rep,name=changes" json:"changes,omitempty"` +} + +func (x *FileList) Reset() { + *x = FileList{} + if protoimpl.UnsafeEnabled { + mi := &file_file_list_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileList) ProtoMessage() {} + +func (x *FileList) ProtoReflect() protoreflect.Message { + mi := &file_file_list_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileList.ProtoReflect.Descriptor instead. +func (*FileList) Descriptor() ([]byte, []int) { + return file_file_list_proto_rawDescGZIP(), []int{0} +} + +func (x *FileList) GetName() string { + if x != nil && x.Name != nil { + return *x.Name + } + return "" +} + +func (x *FileList) GetAdditions() []string { + if x != nil { + return x.Additions + } + return nil +} + +func (x *FileList) GetDeletions() []string { + if x != nil { + return x.Deletions + } + return nil +} + +func (x *FileList) GetChanges() []*FileList { + if x != nil { + return x.Changes + } + return nil +} + +var File_file_list_proto protoreflect.FileDescriptor + +var file_file_list_proto_rawDesc = []byte{ + 0x0a, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x1e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0x9e, 0x01, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x42, + 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, + 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, + 0x65, 0x73, 0x42, 0x26, 0x5a, 0x24, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, + 0x6f, 0x6e, 0x67, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, + 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, +} + +var ( + file_file_list_proto_rawDescOnce sync.Once + file_file_list_proto_rawDescData = file_file_list_proto_rawDesc +) + +func file_file_list_proto_rawDescGZIP() []byte { + file_file_list_proto_rawDescOnce.Do(func() { + file_file_list_proto_rawDescData = protoimpl.X.CompressGZIP(file_file_list_proto_rawDescData) + }) + return file_file_list_proto_rawDescData +} + +var file_file_list_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_file_list_proto_goTypes = []interface{}{ + (*FileList)(nil), // 0: android.find_input_delta_proto.FileList +} +var file_file_list_proto_depIdxs = []int32{ + 0, // 0: android.find_input_delta_proto.FileList.changes:type_name -> android.find_input_delta_proto.FileList + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_file_list_proto_init() } +func file_file_list_proto_init() { + if File_file_list_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_file_list_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileList); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_file_list_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_file_list_proto_goTypes, + DependencyIndexes: file_file_list_proto_depIdxs, + MessageInfos: file_file_list_proto_msgTypes, + }.Build() + File_file_list_proto = out.File + file_file_list_proto_rawDesc = nil + file_file_list_proto_goTypes = nil + file_file_list_proto_depIdxs = nil +} diff --git a/cmd/find_input_delta/find_input_delta_proto/file_list.proto b/cmd/find_input_delta/find_input_delta_proto/file_list.proto new file mode 100644 index 000000000..d7faca9c8 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_proto/file_list.proto @@ -0,0 +1,34 @@ +// +// Copyright (C) 2024 The Android Open-Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; +package android.find_input_delta_proto; +option go_package = "android/soong/find_input_delta/proto"; + +message FileList { + // The name of the file. + // In the outermost message, this is the name of the Ninja target. + // When used in `changes`, this is the name of the changed file. + optional string name = 1; + + // The added files. + repeated string additions = 2; + + // The deleted files. + repeated string deletions = 3; + + // The changed files. + repeated FileList changes = 4; +} diff --git a/cmd/find_input_delta/find_input_delta_proto/regen.sh b/cmd/find_input_delta/find_input_delta_proto/regen.sh new file mode 100644 index 000000000..d77365941 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_proto/regen.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +aprotoc --go_out=paths=source_relative:. file_list.proto diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/Android.bp b/cmd/find_input_delta/find_input_delta_proto_internal/Android.bp new file mode 100644 index 000000000..00ba9ff03 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_proto_internal/Android.bp @@ -0,0 +1,29 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-cmd-find_input_delta-proto_internal", + pkgPath: "android/soong/cmd/find_input_delta/find_input_delta_proto_internal", + deps: [ + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + ], + srcs: [ + "internal_state.pb.go", + ], +} diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go new file mode 100644 index 000000000..2229a324d --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go @@ -0,0 +1,268 @@ +// +// Copyright (C) 2024 The Android Open-Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.21.12 +// source: internal_state.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// The state of all inputs. +type PartialCompileInputs struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The status of each file. + InputFiles []*PartialCompileInput `protobuf:"bytes,1,rep,name=input_files,json=inputFiles" json:"input_files,omitempty"` +} + +func (x *PartialCompileInputs) Reset() { + *x = PartialCompileInputs{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_state_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PartialCompileInputs) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PartialCompileInputs) ProtoMessage() {} + +func (x *PartialCompileInputs) ProtoReflect() protoreflect.Message { + mi := &file_internal_state_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PartialCompileInputs.ProtoReflect.Descriptor instead. +func (*PartialCompileInputs) Descriptor() ([]byte, []int) { + return file_internal_state_proto_rawDescGZIP(), []int{0} +} + +func (x *PartialCompileInputs) GetInputFiles() []*PartialCompileInput { + if x != nil { + return x.InputFiles + } + return nil +} + +// The state of one input. +type PartialCompileInput struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The name of the file. + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The timestamp of the file in (Unix) nanoseconds. + MtimeNsec *int64 `protobuf:"varint,2,opt,name=mtime_nsec,json=mtimeNsec" json:"mtime_nsec,omitempty"` + // The hash of the file, in the form ‘{HASHNAME}:{VALUE}’ + Hash *string `protobuf:"bytes,3,opt,name=hash" json:"hash,omitempty"` + // Contents of the file, if the file was inspected (such as jar files, etc). + Contents []*PartialCompileInput `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty"` +} + +func (x *PartialCompileInput) Reset() { + *x = PartialCompileInput{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_state_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PartialCompileInput) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PartialCompileInput) ProtoMessage() {} + +func (x *PartialCompileInput) ProtoReflect() protoreflect.Message { + mi := &file_internal_state_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PartialCompileInput.ProtoReflect.Descriptor instead. +func (*PartialCompileInput) Descriptor() ([]byte, []int) { + return file_internal_state_proto_rawDescGZIP(), []int{1} +} + +func (x *PartialCompileInput) GetName() string { + if x != nil && x.Name != nil { + return *x.Name + } + return "" +} + +func (x *PartialCompileInput) GetMtimeNsec() int64 { + if x != nil && x.MtimeNsec != nil { + return *x.MtimeNsec + } + return 0 +} + +func (x *PartialCompileInput) GetHash() string { + if x != nil && x.Hash != nil { + return *x.Hash + } + return "" +} + +func (x *PartialCompileInput) GetContents() []*PartialCompileInput { + if x != nil { + return x.Contents + } + return nil +} + +var File_internal_state_proto protoreflect.FileDescriptor + +var file_internal_state_proto_rawDesc = []byte{ + 0x0a, 0x14, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6c, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, + 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x54, + 0x0a, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, + 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, + 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x46, + 0x69, 0x6c, 0x65, 0x73, 0x22, 0xad, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, + 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, 0x63, 0x12, + 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, + 0x61, 0x73, 0x68, 0x12, 0x4f, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, + 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x73, 0x42, 0x26, 0x5a, 0x24, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, + 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, +} + +var ( + file_internal_state_proto_rawDescOnce sync.Once + file_internal_state_proto_rawDescData = file_internal_state_proto_rawDesc +) + +func file_internal_state_proto_rawDescGZIP() []byte { + file_internal_state_proto_rawDescOnce.Do(func() { + file_internal_state_proto_rawDescData = protoimpl.X.CompressGZIP(file_internal_state_proto_rawDescData) + }) + return file_internal_state_proto_rawDescData +} + +var file_internal_state_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_internal_state_proto_goTypes = []interface{}{ + (*PartialCompileInputs)(nil), // 0: android.find_input_delta_proto.PartialCompileInputs + (*PartialCompileInput)(nil), // 1: android.find_input_delta_proto.PartialCompileInput +} +var file_internal_state_proto_depIdxs = []int32{ + 1, // 0: android.find_input_delta_proto.PartialCompileInputs.input_files:type_name -> android.find_input_delta_proto.PartialCompileInput + 1, // 1: android.find_input_delta_proto.PartialCompileInput.contents:type_name -> android.find_input_delta_proto.PartialCompileInput + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_internal_state_proto_init() } +func file_internal_state_proto_init() { + if File_internal_state_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_internal_state_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PartialCompileInputs); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_state_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PartialCompileInput); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_internal_state_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_internal_state_proto_goTypes, + DependencyIndexes: file_internal_state_proto_depIdxs, + MessageInfos: file_internal_state_proto_msgTypes, + }.Build() + File_internal_state_proto = out.File + file_internal_state_proto_rawDesc = nil + file_internal_state_proto_goTypes = nil + file_internal_state_proto_depIdxs = nil +} diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto new file mode 100644 index 000000000..113fc6434 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto @@ -0,0 +1,39 @@ +// +// Copyright (C) 2024 The Android Open-Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; +package android.find_input_delta_proto; +option go_package = "android/soong/find_input_delta/proto"; + +// The state of all inputs. +message PartialCompileInputs { + // The status of each file. + repeated PartialCompileInput input_files = 1; +} + +// The state of one input. +message PartialCompileInput { + // The name of the file. + optional string name = 1; + + // The timestamp of the file in (Unix) nanoseconds. + optional int64 mtime_nsec = 2; + + // The hash of the file, in the form ‘{HASHNAME}:{VALUE}’ + optional string hash = 3; + + // Contents of the file, if the file was inspected (such as jar files, etc). + repeated PartialCompileInput contents = 4; +} diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/regen.sh b/cmd/find_input_delta/find_input_delta_proto_internal/regen.sh new file mode 100644 index 000000000..cbaf7d056 --- /dev/null +++ b/cmd/find_input_delta/find_input_delta_proto_internal/regen.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +aprotoc --go_out=paths=source_relative:. internal_state.proto diff --git a/compliance/notice.go b/compliance/notice.go index 4fc83ab70..edd1b3435 100644 --- a/compliance/notice.go +++ b/compliance/notice.go @@ -18,6 +18,7 @@ import ( "path/filepath" "android/soong/android" + "github.com/google/blueprint" ) @@ -62,8 +63,7 @@ type NoticeXmlModule struct { props noticeXmlProperties - outputFile android.OutputPath - installPath android.InstallPath + outputFile android.OutputPath } type noticeXmlProperties struct { @@ -86,10 +86,8 @@ func (nx *NoticeXmlModule) GenerateAndroidBuildActions(ctx android.ModuleContext nx.outputFile = output.OutputPath - if android.Bool(ctx.Config().ProductVariables().UseSoongSystemImage) { - nx.installPath = android.PathForModuleInPartitionInstall(ctx, nx.props.Partition_name, "etc") - ctx.InstallFile(nx.installPath, "NOTICE.xml.gz", nx.outputFile) - } + installPath := android.PathForModuleInPartitionInstall(ctx, nx.props.Partition_name, "etc") + ctx.PackageFile(installPath, "NOTICE.xml.gz", nx.outputFile) } func (nx *NoticeXmlModule) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/docs/selects.md b/docs/selects.md new file mode 100644 index 000000000..4f436ac4e --- /dev/null +++ b/docs/selects.md @@ -0,0 +1,170 @@ +# Select statements + +## Introduction + +Soong currently has the arch, target, product_variables, and soong config variable properties that all support changing the values of soong properties based on some condition. These are confusing for users, and particularly the soong config variable properties require a lot of boilerplate that is annoying to write. + +In addition, these properties are all implemented using reflection on property structs, and can combine in ways that the original authors did not expect. For example, soong config variables can be combined with arch/target by saying that they operate on "arch.arm.enabled" instead of just "enabled". But product variables do not have the same abilities. + +The goal here is to combine all these different configuration mechanisms into one, to reduce complexity and boilerplate both in Android.bp files and in soong code. + +## Usage + +The soong select statements take their name and inspiration from [bazel select statements](https://bazel.build/docs/configurable-attributes). + +### Syntax + +#### Basic + +The basic syntax for select statements looks like: + +``` +my_module_type { + name: "my_module", + some_string_property: select(arch(), { + "arm": "foo", + "x86": "bar", + default: "baz", + }), +} +``` + +That is, `select(` followed by a variable function, then a map of values of the variable to values to set the property to. + +Arguments can be passed to the "functions" that look up axes: + +``` +select(soong_config_variable("my_namespace", "my_variable"), { + "value1": "foo", + default: "bar", +}) +``` + + +The list of functions that can currently be selected on: + - `arch()` + - `os()` + - `soong_config_variable(namespace, variable)` + - `release_flag(flag)` + +The functions are [defined here](https://cs.android.com/android/platform/superproject/main/+/main:build/soong/android/module.go;l=2144;drc=3f01580c04bfe37c920e247015cce93cff2451c0), and it should be easy to add more. + +#### Multivariable + +To handle multivariable selects, multiple axes can be specified within parenthesis, to look like tuple destructuring. All of the variables being selected must match the corresponding value from the branch in order for the branch to be chosen. + +``` +select((arch(), os()), { + ("arm", "linux"): "foo", + (default, "windows"): "bar", + (default, default): "baz", +}) +``` + +#### Unset + +You can have unset branches of selects using the "unset" keyword, which will act as if the property was not assigned to. This is only really useful if you’re using defaults modules. + +``` +cc_binary { + name: "my_binary", + enabled: select(os(), { + "darwin": false, + default: unset, + }), +} +``` + +#### Appending + +You can append select statements to both scalar values and other select statements: + +``` +my_module_type { + name: "my_module", + // string_property will be set to something like penguin-four, apple-two, etc. + string_property: select(os(), { + "linux_glibc": "penguin", + "darwin": "apple", + default: "unknown", + }) + "-" + select(soong_config_variable("ANDROID", "favorite_vehicle"), { + "car": "four", + "tricycle": "three", + "bike": "two", + default: "unknown", + }) +} +``` + + +You can also append a select with a value with another select that may not have a value, because some of its branches are "unset". If an unset branch was selected it will not append anything to the other select. + +#### Binding the selected value to a Blueprint variable and the "any" keyword + +In case you want to allow a selected value to have an unbounded number of possible values, you can bind its value to a blueprint variable and use it within the expression for that select branch. + +``` +my_module_type { + name: "my_module", + my_string_property: select(soong_config_variable("my_namespace", "my_variable"), { + "some_value": "baz", + any @ my_var: "foo" + my_var, + default: "bar", + }), +} +``` + +The syntax is `any @ my_variable_name: <expression using my_variable_name>`. `any` is currently the only pattern that can be bound to a variable, but we may add more in the future. `any` is equivalent to `default` except it will not match undefined select conditions. + +#### Errors + +If a select statement does not have a "default" branch, and none of the other branches match the variable being selected on, it’s a compile-time error. This may be useful for enforcing a variable is 1 of only a few values. + +``` +# in product config: +$(call soong_config_set,ANDROID,my_variable,foo) + +// in an Android.bp: +my_module_type { + name: "my_module", + // Will error out with: soong_config_variable("ANDROID", "my_variable") had value "foo", which was not handled by the select + enabled: select(soong_config_variable("ANDROID", "my_variable"), { + "bar": true, + "baz": false, + }), +} +``` + +### Changes to property structs to support selects + +Currently, the way configurable properties work is that there is secretly another property struct that has the `target`, `arch`, etc. properties, and then when the arch mutator (or other relevant mutator) runs, it copies the values of these properties onto the regular property structs. There’s nothing stopping you from accessing your properties from a mutator that runs before the one that updates the properties based on configurable values. This is a potential source of bugs, and we want to make sure that select statements don’t have the same pitfall. For that reason, you have to read property’s values through a getter which can do this check. This requires changing the code on a property-by-property basis to support selects. + +To make a property support selects, it must be of type [proptools.Configurable[T]](https://cs.android.com/android/platform/superproject/main/+/main:build/blueprint/proptools/configurable.go;l=341;drc=a52b058cccd2caa778d0f97077adcd4ef7ffb68a). T is the old type of the property. Currently we support bool, string, and []string. Configurable has a `Get(evaluator)` method to get the value of the property. The evaluator can be a ModuleContext, or if you’re in a situation where you only have a very limited context and a module, (such as in a singleton) you can use [ModuleBase.ConfigurableEvaluator](https://cs.android.com/android/platform/superproject/main/+/main:build/soong/android/module.go;l=2133;drc=e19f741052cce097da940d9083d3f29e668de5cb). + +`proptools.Configurable[T]` will handle unset properties for you, so you don’t need to make it a pointer type. However, there is a not-widely-known feature of property structs, where normally, properties are appended when squashing defaults. But if the property was a pointer property, later defaults replace earlier values instead of appending. With selects, to maintain this behavior, add the `android:"replace_instead_of_append"` struct tag. The "append" behavior for boolean values is to boolean OR them together, which is rarely what you want, so most boolean properties are pointers today. + +Old: +``` +type commonProperties struct { + Enabled *bool `android:"arch_variant"` +} + +func (m *ModuleBase) Enabled() *bool { + return m.commonProperties.Enabled +} +``` + +New: +``` +type commonProperties struct { + Enabled proptools.Configurable[bool] `android:"arch_variant,replace_instead_of_append"` +} + +func (m *ModuleBase) Enabled(ctx ConfigAndErrorContext) *bool { + return m.commonProperties.Enabled.Get(m.ConfigurableEvaluator(ctx)) +} +``` + +The `android:"arch_variant"` tag is kept to support the old `target:` and `arch:` properties with this property, but if all their usages in bp files were replaced by selects, then that tag could be removed. + +The enabled property underwent this migration in https://r.android.com/3066188 diff --git a/etc/adb_keys.go b/etc/adb_keys.go index a2df41c8e..73bc3478f 100644 --- a/etc/adb_keys.go +++ b/etc/adb_keys.go @@ -36,6 +36,11 @@ func AdbKeysModuleFactory() android.Module { func (m *AdbKeysModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { productVariables := ctx.Config().ProductVariables() + + if !m.ProductSpecific() { + ctx.ModuleErrorf("adb_keys module type must set product_specific to true") + } + if !(android.Bool(productVariables.Debuggable) && len(android.String(productVariables.AdbKeys)) > 0) { m.SkipInstall() return @@ -48,7 +53,7 @@ func (m *AdbKeysModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { Output: m.outputPath, Input: input.Path(), }) - m.installPath = android.PathForModuleInPartitionInstall(ctx, ctx.DeviceConfig().ProductPath(), "etc/security") + m.installPath = android.PathForModuleInstall(ctx, "etc/security") ctx.InstallFile(m.installPath, "adb_keys", m.outputPath) } diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index 743bfd426..be943a3ca 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -59,16 +59,27 @@ func RegisterPrebuiltEtcBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("prebuilt_usr_keylayout", PrebuiltUserKeyLayoutFactory) ctx.RegisterModuleType("prebuilt_usr_keychars", PrebuiltUserKeyCharsFactory) ctx.RegisterModuleType("prebuilt_usr_idc", PrebuiltUserIdcFactory) + ctx.RegisterModuleType("prebuilt_usr_srec", PrebuiltUserSrecFactory) ctx.RegisterModuleType("prebuilt_font", PrebuiltFontFactory) ctx.RegisterModuleType("prebuilt_overlay", PrebuiltOverlayFactory) ctx.RegisterModuleType("prebuilt_firmware", PrebuiltFirmwareFactory) ctx.RegisterModuleType("prebuilt_dsp", PrebuiltDSPFactory) ctx.RegisterModuleType("prebuilt_rfsa", PrebuiltRFSAFactory) ctx.RegisterModuleType("prebuilt_renderscript_bitcode", PrebuiltRenderScriptBitcodeFactory) - ctx.RegisterModuleType("prebuilt_media_audio", PrebuiltMediaAudioFactory) + ctx.RegisterModuleType("prebuilt_media", PrebuiltMediaFactory) ctx.RegisterModuleType("prebuilt_voicepack", PrebuiltVoicepackFactory) ctx.RegisterModuleType("prebuilt_bin", PrebuiltBinaryFactory) ctx.RegisterModuleType("prebuilt_wallpaper", PrebuiltWallpaperFactory) + ctx.RegisterModuleType("prebuilt_priv_app", PrebuiltPrivAppFactory) + ctx.RegisterModuleType("prebuilt_rfs", PrebuiltRfsFactory) + ctx.RegisterModuleType("prebuilt_framework", PrebuiltFrameworkFactory) + ctx.RegisterModuleType("prebuilt_res", PrebuiltResFactory) + ctx.RegisterModuleType("prebuilt_wlc_upt", PrebuiltWlcUptFactory) + ctx.RegisterModuleType("prebuilt_odm", PrebuiltOdmFactory) + ctx.RegisterModuleType("prebuilt_vendor_dlkm", PrebuiltVendorDlkmFactory) + ctx.RegisterModuleType("prebuilt_bt_firmware", PrebuiltBtFirmwareFactory) + ctx.RegisterModuleType("prebuilt_tvservice", PrebuiltTvServiceFactory) + ctx.RegisterModuleType("prebuilt_optee", PrebuiltOpteeFactory) ctx.RegisterModuleType("prebuilt_defaults", defaultsFactory) @@ -125,6 +136,9 @@ type prebuiltEtcProperties struct { // Install symlinks to the installed file. Symlinks []string `android:"arch_variant"` + + // Install to partition oem when set to true. + Oem_specific *bool `android:"arch_variant"` } type prebuiltSubdirProperties struct { @@ -362,6 +376,10 @@ func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.PropertyErrorf("sub_dir", "relative_install_path is set. Cannot set sub_dir") } baseInstallDirPath := android.PathForModuleInstall(ctx, p.installBaseDir(ctx), p.SubDir()) + // TODO(b/377304441) + if android.Bool(p.properties.Oem_specific) { + baseInstallDirPath = android.PathForModuleInPartitionInstall(ctx, ctx.DeviceConfig().OemPath(), p.installBaseDir(ctx), p.SubDir()) + } filename := proptools.String(p.properties.Filename) filenameFromSrc := proptools.Bool(p.properties.Filename_from_src) @@ -720,6 +738,17 @@ func PrebuiltUserIdcFactory() android.Module { return module } +// prebuilt_usr_srec is for a prebuilt artifact that is installed in +// <partition>/usr/srec/<sub_dir> directory. +func PrebuiltUserSrecFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "usr/srec") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) + android.InitDefaultableModule(module) + return module +} + // prebuilt_font installs a font in <partition>/fonts directory. func PrebuiltFontFactory() android.Module { module := &PrebuiltEtc{} @@ -793,10 +822,10 @@ func PrebuiltRFSAFactory() android.Module { return module } -// prebuilt_media_audio installs audio files in <partition>/media/audio directory. -func PrebuiltMediaAudioFactory() android.Module { +// prebuilt_media installs media files in <partition>/media directory. +func PrebuiltMediaFactory() android.Module { module := &PrebuiltEtc{} - InitPrebuiltEtcModule(module, "media/audio") + InitPrebuiltEtcModule(module, "media") // This module is device-only android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) android.InitDefaultableModule(module) @@ -832,3 +861,103 @@ func PrebuiltWallpaperFactory() android.Module { android.InitDefaultableModule(module) return module } + +// prebuilt_priv_app installs files in <partition>/priv-app directory. +func PrebuiltPrivAppFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "priv-app") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} + +// prebuilt_rfs installs files in <partition>/rfs directory. +func PrebuiltRfsFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "rfs") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} + +// prebuilt_framework installs files in <partition>/framework directory. +func PrebuiltFrameworkFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "framework") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} + +// prebuilt_res installs files in <partition>/res directory. +func PrebuiltResFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "res") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} + +// prebuilt_wlc_upt installs files in <partition>/wlc_upt directory. +func PrebuiltWlcUptFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "wlc_upt") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} + +// prebuilt_odm installs files in <partition>/odm directory. +func PrebuiltOdmFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "odm") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} + +// prebuilt_vendor_dlkm installs files in <partition>/vendor_dlkm directory. +func PrebuiltVendorDlkmFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "vendor_dlkm") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} + +// prebuilt_bt_firmware installs files in <partition>/bt_firmware directory. +func PrebuiltBtFirmwareFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "bt_firmware") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} + +// prebuilt_tvservice installs files in <partition>/tvservice directory. +func PrebuiltTvServiceFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "tvservice") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} + +// prebuilt_optee installs files in <partition>/optee directory. +func PrebuiltOpteeFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "optee") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} diff --git a/etc/prebuilt_etc_test.go b/etc/prebuilt_etc_test.go index 4a0312f64..0fd04d85f 100644 --- a/etc/prebuilt_etc_test.go +++ b/etc/prebuilt_etc_test.go @@ -591,7 +591,7 @@ func TestPrebuiltRFSADirPath(t *testing.T) { func TestPrebuiltMediaAutoDirPath(t *testing.T) { result := prepareForPrebuiltEtcTest.RunTestWithBp(t, ` - prebuilt_media_audio { + prebuilt_media { name: "foo", src: "Alarm_Beep_01.ogg", product_specific: true, @@ -600,6 +600,6 @@ func TestPrebuiltMediaAutoDirPath(t *testing.T) { `) p := result.Module("foo", "android_common").(*PrebuiltEtc) - expected := "out/soong/target/product/test_device/product/media/audio/alarms" + expected := "out/soong/target/product/test_device/product/media/alarms" android.AssertPathRelativeToTopEquals(t, "install dir", expected, p.installDirPaths[0]) } diff --git a/filesystem/android_device.go b/filesystem/android_device.go index 68e60532e..ab1b96ed0 100644 --- a/filesystem/android_device.go +++ b/filesystem/android_device.go @@ -32,6 +32,12 @@ type PartitionNameProperties struct { Product_partition_name *string // Name of the Vendor partition filesystem module Vendor_partition_name *string + // Name of the Odm partition filesystem module + Odm_partition_name *string + // The vbmeta partition and its "chained" partitions + Vbmeta_partitions []string + // Name of the Userdata partition filesystem module + Userdata_partition_name *string } type androidDevice struct { @@ -44,7 +50,6 @@ func AndroidDeviceFactory() android.Module { module := &androidDevice{} module.AddProperties(&module.partitionProps) android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) - return module } @@ -66,6 +71,11 @@ func (a *androidDevice) DepsMutator(ctx android.BottomUpMutatorContext) { addDependencyIfDefined(a.partitionProps.System_ext_partition_name) addDependencyIfDefined(a.partitionProps.Product_partition_name) addDependencyIfDefined(a.partitionProps.Vendor_partition_name) + addDependencyIfDefined(a.partitionProps.Odm_partition_name) + addDependencyIfDefined(a.partitionProps.Userdata_partition_name) + for _, vbmetaPartition := range a.partitionProps.Vbmeta_partitions { + ctx.AddDependency(ctx.Module(), filesystemDepTag, vbmetaPartition) + } } func (a *androidDevice) GenerateAndroidBuildActions(ctx android.ModuleContext) { diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index 97421c8a3..e84139b9a 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -25,6 +25,7 @@ import ( "android/soong/android" "android/soong/cc" + "android/soong/linkerconfig" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -146,6 +147,10 @@ type FilesystemProperties struct { Erofs ErofsProperties + F2fs F2fsProperties + + Linkerconfig LinkerConfigProperties + // Determines if the module is auto-generated from Soong or not. If the module is // auto-generated, its deps are exempted from visibility enforcement. Is_auto_generated *bool @@ -163,6 +168,21 @@ type ErofsProperties struct { Sparse *bool } +// Additional properties required to generate f2fs FS partitions. +type F2fsProperties struct { + Sparse *bool +} + +type LinkerConfigProperties struct { + + // Build a linker.config.pb file + Gen_linker_config *bool + + // List of files (in .json format) that will be converted to a linker config file (in .pb format). + // The linker config file be installed in the filesystem at /etc/linker.config.pb + Linker_config_srcs []string `android:"path"` +} + // android_filesystem packages a set of modules and their transitive dependencies into a filesystem // image. The filesystem images are expected to be mounted in the target device, which means the // modules in the filesystem image are built for the target device (i.e. Android, not Linux host). @@ -179,6 +199,7 @@ func initFilesystemModule(module android.DefaultableModule, filesystemModule *fi module.AddProperties(&filesystemModule.properties) android.InitPackageModule(filesystemModule) filesystemModule.PackagingBase.DepsCollectFirstTargetOnly = true + filesystemModule.PackagingBase.AllowHighPriorityDeps = true android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) android.InitDefaultableModule(module) } @@ -213,6 +234,7 @@ type fsType int const ( ext4Type fsType = iota erofsType + f2fsType compressedCpioType cpioType // uncompressed unknown @@ -235,6 +257,8 @@ func GetFsTypeFromString(ctx android.EarlyModuleContext, typeStr string) fsType return ext4Type case "erofs": return erofsType + case "f2fs": + return f2fsType case "compressed_cpio": return compressedCpioType case "cpio": @@ -275,7 +299,7 @@ var pctx = android.NewPackageContext("android/soong/filesystem") func (f *filesystem) GenerateAndroidBuildActions(ctx android.ModuleContext) { validatePartitionType(ctx, f) switch f.fsType(ctx) { - case ext4Type, erofsType: + case ext4Type, erofsType, f2fsType: f.output = f.buildImageUsingBuildImage(ctx) case compressedCpioType: f.output = f.buildCpioImage(ctx, true) @@ -428,6 +452,7 @@ func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) androi f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir) f.buildEventLogtagsFile(ctx, builder, rebasedDir) f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir) + f.buildLinkerConfigFile(ctx, builder, rebasedDir) f.copyFilesToProductOut(ctx, builder, rebasedDir) // run host_init_verifier @@ -490,6 +515,8 @@ func (f *filesystem) buildPropFile(ctx android.ModuleContext) (propFile android. return "ext4" case erofsType: return "erofs" + case f2fsType: + return "f2fs" } panic(fmt.Errorf("unsupported fs type %v", t)) } @@ -539,8 +566,11 @@ func (f *filesystem) buildPropFile(ctx android.ModuleContext) (propFile android. addStr("uuid", uuid) addStr("hash_seed", uuid) } - // Add erofs properties - if f.fsType(ctx) == erofsType { + + fst := f.fsType(ctx) + switch fst { + case erofsType: + // Add erofs properties if compressor := f.properties.Erofs.Compressor; compressor != nil { addStr("erofs_default_compressor", proptools.String(compressor)) } @@ -551,17 +581,39 @@ func (f *filesystem) buildPropFile(ctx android.ModuleContext) (propFile android. // https://source.corp.google.com/h/googleplex-android/platform/build/+/88b1c67239ca545b11580237242774b411f2fed9:core/Makefile;l=2292;bpv=1;bpt=0;drc=ea8f34bc1d6e63656b4ec32f2391e9d54b3ebb6b addStr("erofs_sparse_flag", "-s") } - } else if f.properties.Erofs.Compressor != nil || f.properties.Erofs.Compress_hints != nil || f.properties.Erofs.Sparse != nil { - // Raise an exception if the propfile contains erofs properties, but the fstype is not erofs - fs := fsTypeStr(f.fsType(ctx)) - ctx.PropertyErrorf("erofs", "erofs is non-empty, but FS type is %s\n. Please delete erofs properties if this partition should use %s\n", fs, fs) + case f2fsType: + if proptools.BoolDefault(f.properties.F2fs.Sparse, true) { + // https://source.corp.google.com/h/googleplex-android/platform/build/+/88b1c67239ca545b11580237242774b411f2fed9:core/Makefile;l=2294;drc=ea8f34bc1d6e63656b4ec32f2391e9d54b3ebb6b;bpv=1;bpt=0 + addStr("f2fs_sparse_flag", "-S") + } } + f.checkFsTypePropertyError(ctx, fst, fsTypeStr(fst)) propFile = android.PathForModuleOut(ctx, "prop").OutputPath android.WriteFileRuleVerbatim(ctx, propFile, propFileString.String()) return propFile, deps } +// This method checks if there is any property set for the fstype(s) other than +// the current fstype. +func (f *filesystem) checkFsTypePropertyError(ctx android.ModuleContext, t fsType, fs string) { + raiseError := func(otherFsType, currentFsType string) { + errMsg := fmt.Sprintf("%s is non-empty, but FS type is %s\n. Please delete %s properties if this partition should use %s\n", otherFsType, currentFsType, otherFsType, currentFsType) + ctx.PropertyErrorf(otherFsType, errMsg) + } + + if t != erofsType { + if f.properties.Erofs.Compressor != nil || f.properties.Erofs.Compress_hints != nil || f.properties.Erofs.Sparse != nil { + raiseError("erofs", fs) + } + } + if t != f2fsType { + if f.properties.F2fs.Sparse != nil { + raiseError("f2fs", fs) + } + } +} + func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) android.OutputPath { if proptools.Bool(f.properties.Use_avb) { ctx.PropertyErrorf("use_avb", "signing compresed cpio image using avbtool is not supported."+ @@ -591,6 +643,7 @@ func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir) f.buildEventLogtagsFile(ctx, builder, rebasedDir) f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir) + f.buildLinkerConfigFile(ctx, builder, rebasedDir) f.copyFilesToProductOut(ctx, builder, rebasedDir) output := android.PathForModuleOut(ctx, f.installFileName()).OutputPath @@ -682,6 +735,18 @@ func (f *filesystem) buildEventLogtagsFile(ctx android.ModuleContext, builder *a f.appendToEntry(ctx, eventLogtagsPath) } +func (f *filesystem) buildLinkerConfigFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) { + if !proptools.Bool(f.properties.Linkerconfig.Gen_linker_config) { + return + } + + provideModules, _ := f.getLibsForLinkerConfig(ctx) + output := rebasedDir.Join(ctx, "etc", "linker.config.pb") + linkerconfig.BuildLinkerConfig(ctx, builder, android.PathsForModuleSrc(ctx, f.properties.Linkerconfig.Linker_config_srcs), provideModules, nil, output) + + f.appendToEntry(ctx, output) +} + type partition interface { PartitionType() string } @@ -790,3 +855,48 @@ var _ partition = (*filesystemDefaults)(nil) func (f *filesystemDefaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { validatePartitionType(ctx, f) } + +// getLibsForLinkerConfig returns +// 1. A list of libraries installed in this filesystem +// 2. A list of dep libraries _not_ installed in this filesystem +// +// `linkerconfig.BuildLinkerConfig` will convert these two to a linker.config.pb for the filesystem +// (1) will be added to --provideLibs if they are C libraries with a stable interface (has stubs) +// (2) will be added to --requireLibs if they are C libraries with a stable interface (has stubs) +func (f *filesystem) getLibsForLinkerConfig(ctx android.ModuleContext) ([]android.Module, []android.Module) { + // we need "Module"s for packaging items + modulesInPackageByModule := make(map[android.Module]bool) + modulesInPackageByName := make(map[string]bool) + + deps := f.gatherFilteredPackagingSpecs(ctx) + ctx.WalkDeps(func(child, parent android.Module) bool { + for _, ps := range android.OtherModuleProviderOrDefault( + ctx, child, android.InstallFilesProvider).PackagingSpecs { + if _, ok := deps[ps.RelPathInPackage()]; ok { + modulesInPackageByModule[child] = true + modulesInPackageByName[child.Name()] = true + return true + } + } + return true + }) + + provideModules := make([]android.Module, 0, len(modulesInPackageByModule)) + for mod := range modulesInPackageByModule { + provideModules = append(provideModules, mod) + } + + var requireModules []android.Module + ctx.WalkDeps(func(child, parent android.Module) bool { + _, parentInPackage := modulesInPackageByModule[parent] + _, childInPackageName := modulesInPackageByName[child.Name()] + + // When parent is in the package, and child (or its variant) is not, this can be from an interface. + if parentInPackage && !childInPackageName { + requireModules = append(requireModules, child) + } + return true + }) + + return provideModules, requireModules +} diff --git a/filesystem/filesystem_test.go b/filesystem/filesystem_test.go index 730006116..801a17545 100644 --- a/filesystem/filesystem_test.go +++ b/filesystem/filesystem_test.go @@ -585,6 +585,35 @@ func TestErofsPartition(t *testing.T) { android.AssertStringDoesContain(t, "erofs fs type sparse", buildImageConfig, "erofs_sparse_flag=-s") } +func TestF2fsPartition(t *testing.T) { + result := fixture.RunTestWithBp(t, ` + android_filesystem { + name: "f2fs_partition", + type: "f2fs", + } + `) + + partition := result.ModuleForTests("f2fs_partition", "android_common") + buildImageConfig := android.ContentFromFileRuleForTests(t, result.TestContext, partition.Output("prop")) + android.AssertStringDoesContain(t, "f2fs fs type", buildImageConfig, "fs_type=f2fs") + android.AssertStringDoesContain(t, "f2fs fs type sparse", buildImageConfig, "f2fs_sparse_flag=-S") +} + +func TestFsTypesPropertyError(t *testing.T) { + fixture.ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern( + "erofs: erofs is non-empty, but FS type is f2fs\n. Please delete erofs properties if this partition should use f2fs\n")). + RunTestWithBp(t, ` + android_filesystem { + name: "f2fs_partition", + type: "f2fs", + erofs: { + compressor: "lz4hc,9", + compress_hints: "compress_hints.txt", + }, + } + `) +} + // If a system_ext/ module depends on system/ module, the dependency should *not* // be installed in system_ext/ func TestDoNotPackageCrossPartitionDependencies(t *testing.T) { @@ -664,3 +693,54 @@ cc_library { fileList := android.ContentFromFileRuleForTests(t, result.TestContext, partition.Output("fileList")) android.AssertDeepEquals(t, "Shared library dep of overridden binary should not be installed", fileList, "bin/binfoo1\nlib64/libc++.so\nlib64/libc.so\nlib64/libdl.so\nlib64/libfoo2.so\nlib64/libm.so\n") } + +func TestInstallLinkerConfigFile(t *testing.T) { + result := fixture.RunTestWithBp(t, ` +android_filesystem { + name: "myfilesystem", + deps: ["libfoo_has_no_stubs", "libfoo_has_stubs"], + linkerconfig: { + gen_linker_config: true, + linker_config_srcs: ["linker.config.json"], + }, + partition_type: "vendor", +} +cc_library { + name: "libfoo_has_no_stubs", + vendor: true, +} +cc_library { + name: "libfoo_has_stubs", + stubs: {symbol_file: "libfoo.map.txt"}, + vendor: true, +} + `) + + linkerConfigCmd := result.ModuleForTests("myfilesystem", "android_common").Rule("build_filesystem_image").RuleParams.Command + android.AssertStringDoesContain(t, "Could not find linker.config.json file in cmd", linkerConfigCmd, "conv_linker_config proto --force -s linker.config.json") + android.AssertStringDoesContain(t, "Could not find stub in `provideLibs`", linkerConfigCmd, "--key provideLibs --value libfoo_has_stubs.so") +} + +// override_android_* modules implicitly override their base module. +// If both of these are listed in `deps`, the base module should not be installed. +func TestOverrideModulesInDeps(t *testing.T) { + result := fixture.RunTestWithBp(t, ` + android_filesystem { + name: "myfilesystem", + deps: ["myapp", "myoverrideapp"], + } + + android_app { + name: "myapp", + platform_apis: true, + } + override_android_app { + name: "myoverrideapp", + base: "myapp", + } + `) + + partition := result.ModuleForTests("myfilesystem", "android_common") + fileList := android.ContentFromFileRuleForTests(t, result.TestContext, partition.Output("fileList")) + android.AssertStringEquals(t, "filesystem with override app", "app/myoverrideapp/myoverrideapp.apk\n", fileList) +} diff --git a/filesystem/system_image.go b/filesystem/system_image.go index 7dbf98644..898987dbc 100644 --- a/filesystem/system_image.go +++ b/filesystem/system_image.go @@ -59,42 +59,9 @@ func (s *systemImage) buildLinkerConfigFile(ctx android.ModuleContext, root andr input := android.PathForModuleSrc(ctx, android.String(s.properties.Linker_config_src)) output := root.Join(ctx, "system", "etc", "linker.config.pb") - // we need "Module"s for packaging items - modulesInPackageByModule := make(map[android.Module]bool) - modulesInPackageByName := make(map[string]bool) - - deps := s.gatherFilteredPackagingSpecs(ctx) - ctx.WalkDeps(func(child, parent android.Module) bool { - for _, ps := range android.OtherModuleProviderOrDefault( - ctx, child, android.InstallFilesProvider).PackagingSpecs { - if _, ok := deps[ps.RelPathInPackage()]; ok { - modulesInPackageByModule[child] = true - modulesInPackageByName[child.Name()] = true - return true - } - } - return true - }) - - provideModules := make([]android.Module, 0, len(modulesInPackageByModule)) - for mod := range modulesInPackageByModule { - provideModules = append(provideModules, mod) - } - - var requireModules []android.Module - ctx.WalkDeps(func(child, parent android.Module) bool { - _, parentInPackage := modulesInPackageByModule[parent] - _, childInPackageName := modulesInPackageByName[child.Name()] - - // When parent is in the package, and child (or its variant) is not, this can be from an interface. - if parentInPackage && !childInPackageName { - requireModules = append(requireModules, child) - } - return true - }) - + provideModules, requireModules := s.getLibsForLinkerConfig(ctx) builder := android.NewRuleBuilder(pctx, ctx) - linkerconfig.BuildLinkerConfig(ctx, builder, input, provideModules, requireModules, output) + linkerconfig.BuildLinkerConfig(ctx, builder, android.Paths{input}, provideModules, requireModules, output) builder.Build("conv_linker_config", "Generate linker config protobuf "+output.String()) return output } diff --git a/filesystem/vbmeta.go b/filesystem/vbmeta.go index 0bae479cf..6a3fc1f18 100644 --- a/filesystem/vbmeta.go +++ b/filesystem/vbmeta.go @@ -25,19 +25,19 @@ import ( ) func init() { - android.RegisterModuleType("vbmeta", vbmetaFactory) + android.RegisterModuleType("vbmeta", VbmetaFactory) } type vbmeta struct { android.ModuleBase - properties vbmetaProperties + properties VbmetaProperties output android.OutputPath installDir android.InstallPath } -type vbmetaProperties struct { +type VbmetaProperties struct { // Name of the partition stored in vbmeta desc. Defaults to the name of this module. Partition_name *string @@ -50,9 +50,8 @@ type vbmetaProperties struct { // Algorithm that avbtool will use to sign this vbmeta image. Default is SHA256_RSA4096. Algorithm *string - // File whose content will provide the rollback index. If unspecified, the rollback index - // is from PLATFORM_SECURITY_PATCH - Rollback_index_file *string `android:"path"` + // The rollback index. If unspecified, the rollback index is from PLATFORM_SECURITY_PATCH + Rollback_index *int64 // Rollback index location of this vbmeta image. Must be 0, 1, 2, etc. Default is 0. Rollback_index_location *int64 @@ -62,7 +61,7 @@ type vbmetaProperties struct { Partitions proptools.Configurable[[]string] // List of chained partitions that this vbmeta deletages the verification. - Chained_partitions []chainedPartitionProperties + Chained_partitions []ChainedPartitionProperties // List of key-value pair of avb properties Avb_properties []avbProperty @@ -76,7 +75,7 @@ type avbProperty struct { Value *string } -type chainedPartitionProperties struct { +type ChainedPartitionProperties struct { // Name of the chained partition Name *string @@ -95,7 +94,7 @@ type chainedPartitionProperties struct { } // vbmeta is the partition image that has the verification information for other partitions. -func vbmetaFactory() android.Module { +func VbmetaFactory() android.Module { module := &vbmeta{} module.AddProperties(&module.properties) android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) @@ -217,15 +216,12 @@ func (v *vbmeta) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Returns the embedded shell command that prints the rollback index func (v *vbmeta) rollbackIndexCommand(ctx android.ModuleContext) string { - var cmd string - if v.properties.Rollback_index_file != nil { - f := android.PathForModuleSrc(ctx, proptools.String(v.properties.Rollback_index_file)) - cmd = "cat " + f.String() + if v.properties.Rollback_index != nil { + return fmt.Sprintf("%d", *v.properties.Rollback_index) } else { - cmd = "date -d 'TZ=\"GMT\" " + ctx.Config().PlatformSecurityPatch() + "' +%s" + // Take the first line and remove the newline char + return "$(date -d 'TZ=\"GMT\" " + ctx.Config().PlatformSecurityPatch() + "' +%s | head -1 | tr -d '\n'" + ")" } - // Take the first line and remove the newline char - return "$(" + cmd + " | head -1 | tr -d '\n'" + ")" } // Extract public keys from chained_partitions.private_key. The keys are indexed with the partition diff --git a/fsgen/Android.bp b/fsgen/Android.bp index 9fa955779..8cd7518cb 100644 --- a/fsgen/Android.bp +++ b/fsgen/Android.bp @@ -10,14 +10,19 @@ bootstrap_go_package { "soong", "soong-android", "soong-filesystem", + "soong-kernel", ], srcs: [ "filesystem_creator.go", + "fsgen_mutators.go", + "prebuilt_etc_modules_gen.go", + "vbmeta_partitions.go", ], testSrcs: [ "filesystem_creator_test.go", ], pluginFor: ["soong_build"], + visibility: ["//visibility:public"], } soong_filesystem_creator { diff --git a/fsgen/filesystem_creator.go b/fsgen/filesystem_creator.go index 766176d5a..bcf236160 100644 --- a/fsgen/filesystem_creator.go +++ b/fsgen/filesystem_creator.go @@ -17,13 +17,13 @@ package fsgen import ( "crypto/sha256" "fmt" - "slices" + "path/filepath" "strconv" "strings" - "sync" "android/soong/android" "android/soong/filesystem" + "android/soong/kernel" "github.com/google/blueprint" "github.com/google/blueprint/parser" @@ -41,315 +41,12 @@ func registerBuildComponents(ctx android.RegistrationContext) { ctx.PreDepsMutators(RegisterCollectFileSystemDepsMutators) } -func RegisterCollectFileSystemDepsMutators(ctx android.RegisterMutatorsContext) { - ctx.BottomUp("fs_collect_deps", collectDepsMutator).MutatesGlobalState() - ctx.BottomUp("fs_set_deps", setDepsMutator) -} - -var fsGenStateOnceKey = android.NewOnceKey("FsGenState") -var fsGenRemoveOverridesOnceKey = android.NewOnceKey("FsGenRemoveOverrides") - -// Map of partition module name to its partition that may be generated by Soong. -// Note that it is not guaranteed that all modules returned by this function are successfully -// created. -func getAllSoongGeneratedPartitionNames(config android.Config, partitions []string) map[string]string { - ret := map[string]string{} - for _, partition := range partitions { - ret[generatedModuleNameForPartition(config, partition)] = partition - } - return ret -} - -type depCandidateProps struct { - Namespace string - Multilib string - Arch []android.ArchType -} - -// Map of module name to depCandidateProps -type multilibDeps *map[string]*depCandidateProps - -// Information necessary to generate the filesystem modules, including details about their -// dependencies -type FsGenState struct { - // List of modules in `PRODUCT_PACKAGES` and `PRODUCT_PACKAGES_DEBUG` - depCandidates []string - // Map of names of partition to the information of modules to be added as deps - fsDeps map[string]multilibDeps - // List of name of partitions to be generated by the filesystem_creator module - soongGeneratedPartitions []string - // Mutex to protect the fsDeps - fsDepsMutex sync.Mutex - // Map of _all_ soong module names to their corresponding installation properties - moduleToInstallationProps map[string]installationProperties -} - -type installationProperties struct { - Required []string - Overrides []string -} - -func newMultilibDeps() multilibDeps { - return &map[string]*depCandidateProps{} -} - -func defaultDepCandidateProps(config android.Config) *depCandidateProps { - return &depCandidateProps{ - Namespace: ".", - Arch: []android.ArchType{config.BuildArch}, - } -} - -func createFsGenState(ctx android.LoadHookContext) *FsGenState { - return ctx.Config().Once(fsGenStateOnceKey, func() interface{} { - partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse - candidates := android.FirstUniqueStrings(android.Concat(partitionVars.ProductPackages, partitionVars.ProductPackagesDebug)) - - generatedPartitions := []string{"system"} - if ctx.DeviceConfig().SystemExtPath() == "system_ext" { - generatedPartitions = append(generatedPartitions, "system_ext") - } - if ctx.DeviceConfig().BuildingVendorImage() && ctx.DeviceConfig().VendorPath() == "vendor" { - generatedPartitions = append(generatedPartitions, "vendor") - } - if ctx.DeviceConfig().BuildingProductImage() && ctx.DeviceConfig().ProductPath() == "product" { - generatedPartitions = append(generatedPartitions, "product") - } - - return &FsGenState{ - depCandidates: candidates, - fsDeps: map[string]multilibDeps{ - // These additional deps are added according to the cuttlefish system image bp. - "system": &map[string]*depCandidateProps{ - "com.android.apex.cts.shim.v1_prebuilt": defaultDepCandidateProps(ctx.Config()), - "dex_bootjars": defaultDepCandidateProps(ctx.Config()), - "framework_compatibility_matrix.device.xml": defaultDepCandidateProps(ctx.Config()), - "idc_data": defaultDepCandidateProps(ctx.Config()), - "init.environ.rc-soong": defaultDepCandidateProps(ctx.Config()), - "keychars_data": defaultDepCandidateProps(ctx.Config()), - "keylayout_data": defaultDepCandidateProps(ctx.Config()), - "libclang_rt.asan": defaultDepCandidateProps(ctx.Config()), - "libcompiler_rt": defaultDepCandidateProps(ctx.Config()), - "libdmabufheap": defaultDepCandidateProps(ctx.Config()), - "libgsi": defaultDepCandidateProps(ctx.Config()), - "llndk.libraries.txt": defaultDepCandidateProps(ctx.Config()), - "logpersist.start": defaultDepCandidateProps(ctx.Config()), - "preloaded-classes": defaultDepCandidateProps(ctx.Config()), - "public.libraries.android.txt": defaultDepCandidateProps(ctx.Config()), - "update_engine_sideload": defaultDepCandidateProps(ctx.Config()), - }, - "vendor": &map[string]*depCandidateProps{ - "fs_config_files_vendor": defaultDepCandidateProps(ctx.Config()), - "fs_config_dirs_vendor": defaultDepCandidateProps(ctx.Config()), - generatedModuleName(ctx.Config(), "vendor-build.prop"): defaultDepCandidateProps(ctx.Config()), - }, - "odm": newMultilibDeps(), - "product": newMultilibDeps(), - "system_ext": &map[string]*depCandidateProps{ - // VNDK apexes are automatically included. - // This hardcoded list will need to be updated if `PRODUCT_EXTRA_VNDK_VERSIONS` is updated. - // https://cs.android.com/android/_/android/platform/build/+/adba533072b00c53ac0f198c550a3cbd7a00e4cd:core/main.mk;l=984;bpv=1;bpt=0;drc=174db7b179592cf07cbfd2adb0119486fda911e7 - "com.android.vndk.v30": defaultDepCandidateProps(ctx.Config()), - "com.android.vndk.v31": defaultDepCandidateProps(ctx.Config()), - "com.android.vndk.v32": defaultDepCandidateProps(ctx.Config()), - "com.android.vndk.v33": defaultDepCandidateProps(ctx.Config()), - "com.android.vndk.v34": defaultDepCandidateProps(ctx.Config()), - }, - }, - soongGeneratedPartitions: generatedPartitions, - fsDepsMutex: sync.Mutex{}, - moduleToInstallationProps: map[string]installationProperties{}, - } - }).(*FsGenState) -} - -func checkDepModuleInMultipleNamespaces(mctx android.BottomUpMutatorContext, foundDeps map[string]*depCandidateProps, module string, partitionName string) { - otherNamespace := mctx.Namespace().Path - if val, found := foundDeps[module]; found && otherNamespace != "." && !android.InList(val.Namespace, []string{".", otherNamespace}) { - mctx.ModuleErrorf("found in multiple namespaces(%s and %s) when including in %s partition", val.Namespace, otherNamespace, partitionName) - } -} - -func appendDepIfAppropriate(mctx android.BottomUpMutatorContext, deps *map[string]*depCandidateProps, installPartition string) { - checkDepModuleInMultipleNamespaces(mctx, *deps, mctx.Module().Name(), installPartition) - if _, ok := (*deps)[mctx.Module().Name()]; ok { - // Prefer the namespace-specific module over the platform module - if mctx.Namespace().Path != "." { - (*deps)[mctx.Module().Name()].Namespace = mctx.Namespace().Path - } - (*deps)[mctx.Module().Name()].Arch = append((*deps)[mctx.Module().Name()].Arch, mctx.Module().Target().Arch.ArchType) - } else { - multilib, _ := mctx.Module().DecodeMultilib(mctx) - (*deps)[mctx.Module().Name()] = &depCandidateProps{ - Namespace: mctx.Namespace().Path, - Multilib: multilib, - Arch: []android.ArchType{mctx.Module().Target().Arch.ArchType}, - } - } -} - -func collectDepsMutator(mctx android.BottomUpMutatorContext) { - fsGenState := mctx.Config().Get(fsGenStateOnceKey).(*FsGenState) - - m := mctx.Module() - if m.Target().Os.Class == android.Device && slices.Contains(fsGenState.depCandidates, m.Name()) { - installPartition := m.PartitionTag(mctx.DeviceConfig()) - fsGenState.fsDepsMutex.Lock() - // Only add the module as dependency when: - // - its enabled - // - its namespace is included in PRODUCT_SOONG_NAMESPACES - if m.Enabled(mctx) && m.ExportedToMake() { - appendDepIfAppropriate(mctx, fsGenState.fsDeps[installPartition], installPartition) - } - fsGenState.fsDepsMutex.Unlock() - } - // store the map of module to (required,overrides) even if the module is not in PRODUCT_PACKAGES. - // the module might be installed transitively. - if m.Target().Os.Class == android.Device && m.Enabled(mctx) && m.ExportedToMake() { - fsGenState.fsDepsMutex.Lock() - fsGenState.moduleToInstallationProps[m.Name()] = installationProperties{ - Required: m.RequiredModuleNames(mctx), - Overrides: m.Overrides(), - } - fsGenState.fsDepsMutex.Unlock() - } -} - -type depsStruct struct { - Deps []string -} - -type multilibDepsStruct struct { - Common depsStruct - Lib32 depsStruct - Lib64 depsStruct - Both depsStruct - Prefer32 depsStruct -} - -type packagingPropsStruct struct { - Deps []string - Multilib multilibDepsStruct -} - -func fullyQualifiedModuleName(moduleName, namespace string) string { - if namespace == "." { - return moduleName - } - return fmt.Sprintf("//%s:%s", namespace, moduleName) -} - -func getBitness(archTypes []android.ArchType) (ret []string) { - for _, archType := range archTypes { - if archType.Multilib == "" { - ret = append(ret, android.COMMON_VARIANT) - } else { - ret = append(ret, archType.Bitness()) - } - } - return ret -} - -func setDepsMutator(mctx android.BottomUpMutatorContext) { - removeOverriddenDeps(mctx) - fsGenState := mctx.Config().Get(fsGenStateOnceKey).(*FsGenState) - fsDeps := fsGenState.fsDeps - soongGeneratedPartitionMap := getAllSoongGeneratedPartitionNames(mctx.Config(), fsGenState.soongGeneratedPartitions) - m := mctx.Module() - if partition, ok := soongGeneratedPartitionMap[m.Name()]; ok { - depsStruct := generateDepStruct(*fsDeps[partition]) - if err := proptools.AppendMatchingProperties(m.GetProperties(), depsStruct, nil); err != nil { - mctx.ModuleErrorf(err.Error()) - } - } -} - -// removeOverriddenDeps collects PRODUCT_PACKAGES and (transitive) required deps. -// it then removes any modules which appear in `overrides` of the above list. -func removeOverriddenDeps(mctx android.BottomUpMutatorContext) { - mctx.Config().Once(fsGenRemoveOverridesOnceKey, func() interface{} { - fsGenState := mctx.Config().Get(fsGenStateOnceKey).(*FsGenState) - fsDeps := fsGenState.fsDeps - overridden := map[string]bool{} - allDeps := []string{} - - // Step 1: Initialization: Append PRODUCT_PACKAGES to the queue - for _, fsDep := range fsDeps { - for depName, _ := range *fsDep { - allDeps = append(allDeps, depName) - } - } - - // Step 2: Process the queue, and add required modules to the queue. - i := 0 - for { - if i == len(allDeps) { - break - } - depName := allDeps[i] - for _, overrides := range fsGenState.moduleToInstallationProps[depName].Overrides { - overridden[overrides] = true - } - // add required dep to the queue. - allDeps = append(allDeps, fsGenState.moduleToInstallationProps[depName].Required...) - i += 1 - } - - // Step 3: Delete all the overridden modules. - for overridden, _ := range overridden { - for partition, _ := range fsDeps { - delete(*fsDeps[partition], overridden) - } - } - return nil - }) -} - -func generateDepStruct(deps map[string]*depCandidateProps) *packagingPropsStruct { - depsStruct := packagingPropsStruct{} - for depName, depProps := range deps { - bitness := getBitness(depProps.Arch) - fullyQualifiedDepName := fullyQualifiedModuleName(depName, depProps.Namespace) - if android.InList("32", bitness) && android.InList("64", bitness) { - // If both 32 and 64 bit variants are enabled for this module - switch depProps.Multilib { - case string(android.MultilibBoth): - depsStruct.Multilib.Both.Deps = append(depsStruct.Multilib.Both.Deps, fullyQualifiedDepName) - case string(android.MultilibCommon), string(android.MultilibFirst): - depsStruct.Deps = append(depsStruct.Deps, fullyQualifiedDepName) - case "32": - depsStruct.Multilib.Lib32.Deps = append(depsStruct.Multilib.Lib32.Deps, fullyQualifiedDepName) - case "64", "darwin_universal": - depsStruct.Multilib.Lib64.Deps = append(depsStruct.Multilib.Lib64.Deps, fullyQualifiedDepName) - case "prefer32", "first_prefer32": - depsStruct.Multilib.Prefer32.Deps = append(depsStruct.Multilib.Prefer32.Deps, fullyQualifiedDepName) - default: - depsStruct.Multilib.Both.Deps = append(depsStruct.Multilib.Both.Deps, fullyQualifiedDepName) - } - } else if android.InList("64", bitness) { - // If only 64 bit variant is enabled - depsStruct.Multilib.Lib64.Deps = append(depsStruct.Multilib.Lib64.Deps, fullyQualifiedDepName) - } else if android.InList("32", bitness) { - // If only 32 bit variant is enabled - depsStruct.Multilib.Lib32.Deps = append(depsStruct.Multilib.Lib32.Deps, fullyQualifiedDepName) - } else { - // If only common variant is enabled - depsStruct.Multilib.Common.Deps = append(depsStruct.Multilib.Common.Deps, fullyQualifiedDepName) - } - } - depsStruct.Deps = android.SortedUniqueStrings(depsStruct.Deps) - depsStruct.Multilib.Lib32.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Lib32.Deps) - depsStruct.Multilib.Lib64.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Lib64.Deps) - depsStruct.Multilib.Prefer32.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Prefer32.Deps) - depsStruct.Multilib.Both.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Both.Deps) - depsStruct.Multilib.Common.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Common.Deps) - - return &depsStruct -} - type filesystemCreatorProps struct { Generated_partition_types []string `blueprint:"mutated"` Unsupported_partition_types []string `blueprint:"mutated"` + + Vbmeta_module_names []string `blueprint:"mutated"` + Vbmeta_partition_names []string `blueprint:"mutated"` } type filesystemCreator struct { @@ -364,7 +61,8 @@ func filesystemCreatorFactory() android.Module { android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) module.AddProperties(&module.properties) android.AddLoadHook(module, func(ctx android.LoadHookContext) { - createFsGenState(ctx) + generatedPrebuiltEtcModuleNames := createPrebuiltEtcModules(ctx) + createFsGenState(ctx, generatedPrebuiltEtcModuleNames) module.createInternalModules(ctx) }) @@ -372,16 +70,24 @@ func filesystemCreatorFactory() android.Module { } func (f *filesystemCreator) createInternalModules(ctx android.LoadHookContext) { - soongGeneratedPartitions := &ctx.Config().Get(fsGenStateOnceKey).(*FsGenState).soongGeneratedPartitions - for _, partitionType := range *soongGeneratedPartitions { + soongGeneratedPartitions := generatedPartitions(ctx) + finalSoongGeneratedPartitions := make([]string, 0, len(soongGeneratedPartitions)) + for _, partitionType := range soongGeneratedPartitions { if f.createPartition(ctx, partitionType) { f.properties.Generated_partition_types = append(f.properties.Generated_partition_types, partitionType) + finalSoongGeneratedPartitions = append(finalSoongGeneratedPartitions, partitionType) } else { f.properties.Unsupported_partition_types = append(f.properties.Unsupported_partition_types, partitionType) - _, *soongGeneratedPartitions = android.RemoveFromList(partitionType, *soongGeneratedPartitions) } } - f.createDeviceModule(ctx) + + for _, x := range createVbmetaPartitions(ctx, finalSoongGeneratedPartitions) { + f.properties.Vbmeta_module_names = append(f.properties.Vbmeta_module_names, x.moduleName) + f.properties.Vbmeta_partition_names = append(f.properties.Vbmeta_partition_names, x.partitionName) + } + + ctx.Config().Get(fsGenStateOnceKey).(*FsGenState).soongGeneratedPartitions = finalSoongGeneratedPartitions + f.createDeviceModule(ctx, finalSoongGeneratedPartitions, f.properties.Vbmeta_module_names) } func generatedModuleName(cfg android.Config, suffix string) string { @@ -396,7 +102,11 @@ func generatedModuleNameForPartition(cfg android.Config, partitionType string) s return generatedModuleName(cfg, fmt.Sprintf("%s_image", partitionType)) } -func (f *filesystemCreator) createDeviceModule(ctx android.LoadHookContext) { +func (f *filesystemCreator) createDeviceModule( + ctx android.LoadHookContext, + generatedPartitionTypes []string, + vbmetaPartitions []string, +) { baseProps := &struct { Name *string }{ @@ -405,18 +115,25 @@ func (f *filesystemCreator) createDeviceModule(ctx android.LoadHookContext) { // Currently, only the system and system_ext partition module is created. partitionProps := &filesystem.PartitionNameProperties{} - if android.InList("system", f.properties.Generated_partition_types) { + if android.InList("system", generatedPartitionTypes) { partitionProps.System_partition_name = proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), "system")) } - if android.InList("system_ext", f.properties.Generated_partition_types) { + if android.InList("system_ext", generatedPartitionTypes) { partitionProps.System_ext_partition_name = proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), "system_ext")) } - if android.InList("vendor", f.properties.Generated_partition_types) { + if android.InList("vendor", generatedPartitionTypes) { partitionProps.Vendor_partition_name = proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), "vendor")) } - if android.InList("product", f.properties.Generated_partition_types) { + if android.InList("product", generatedPartitionTypes) { partitionProps.Product_partition_name = proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), "product")) } + if android.InList("odm", generatedPartitionTypes) { + partitionProps.Odm_partition_name = proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), "odm")) + } + if android.InList("userdata", f.properties.Generated_partition_types) { + partitionProps.Userdata_partition_name = proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), "userdata")) + } + partitionProps.Vbmeta_partitions = vbmetaPartitions ctx.CreateModule(filesystem.AndroidDeviceFactory, baseProps, partitionProps) } @@ -438,6 +155,26 @@ func partitionSpecificFsProps(fsProps *filesystem.FilesystemProperties, partitio "framework/oat/*/*", // framework/oat/{arch} } fsProps.Fsverity.Libs = []string{":framework-res{.export-package.apk}"} + // TODO(b/377734331): only generate the symlinks if the relevant partitions exist + fsProps.Symlinks = []filesystem.SymlinkDefinition{ + filesystem.SymlinkDefinition{ + Target: proptools.StringPtr("/product"), + Name: proptools.StringPtr("system/product"), + }, + filesystem.SymlinkDefinition{ + Target: proptools.StringPtr("/system_ext"), + Name: proptools.StringPtr("system/system_ext"), + }, + filesystem.SymlinkDefinition{ + Target: proptools.StringPtr("/vendor"), + Name: proptools.StringPtr("system/vendor"), + }, + filesystem.SymlinkDefinition{ + Target: proptools.StringPtr("/system_dlkm/lib/modules"), + Name: proptools.StringPtr("system/lib/modules"), + }, + } + fsProps.Base_dir = proptools.StringPtr("system") case "system_ext": fsProps.Fsverity.Inputs = []string{ "framework/*", @@ -460,6 +197,17 @@ func partitionSpecificFsProps(fsProps *filesystem.FilesystemProperties, partitio }, } fsProps.Base_dir = proptools.StringPtr("vendor") + case "odm": + fsProps.Symlinks = []filesystem.SymlinkDefinition{ + filesystem.SymlinkDefinition{ + Target: proptools.StringPtr("/odm_dlkm/lib/modules"), + Name: proptools.StringPtr("odm/lib/modules"), + }, + } + fsProps.Base_dir = proptools.StringPtr("odm") + case "userdata": + fsProps.Base_dir = proptools.StringPtr("data") + } } @@ -473,9 +221,24 @@ func (f *filesystemCreator) createPartition(ctx android.LoadHookContext, partiti return false } + if partitionType == "vendor" || partitionType == "product" { + fsProps.Linkerconfig.Gen_linker_config = proptools.BoolPtr(true) + fsProps.Linkerconfig.Linker_config_srcs = f.createLinkerConfigSourceFilegroups(ctx, partitionType) + } + + if partitionType == "system_dlkm" { + kernelModules := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.SystemKernelModules + f.createPrebuiltKernelModules(ctx, partitionType, kernelModules) + } + var module android.Module if partitionType == "system" { module = ctx.CreateModule(filesystem.SystemImageFactory, baseProps, fsProps) + } else if partitionType == "system_dlkm" { + // Do not set partition_type. build/soong/android/paths#modulePartition currently does not support dlkm + // partitions. Since `android_filesystem` uses a partition based filter, setting the partition here + // would result in missing in entries. + module = ctx.CreateModule(filesystem.FilesystemFactory, baseProps, fsProps) } else { // Explicitly set the partition. fsProps.Partition_type = proptools.StringPtr(partitionType) @@ -483,36 +246,134 @@ func (f *filesystemCreator) createPartition(ctx android.LoadHookContext, partiti } module.HideFromMake() if partitionType == "vendor" { - // Create a build prop for vendor - vendorBuildProps := &struct { - Name *string - Vendor *bool - Stem *string - Product_config *string - }{ - Name: proptools.StringPtr(generatedModuleName(ctx.Config(), "vendor-build.prop")), - Vendor: proptools.BoolPtr(true), - Stem: proptools.StringPtr("build.prop"), - Product_config: proptools.StringPtr(":product_config"), - } - vendorBuildProp := ctx.CreateModule( - android.BuildPropFactory, - vendorBuildProps, - ) - vendorBuildProp.HideFromMake() + f.createVendorBuildProp(ctx) } return true } +// createPrebuiltKernelModules creates `prebuilt_kernel_modules`. These modules will be added to deps of the +// autogenerated *_dlkm filsystem modules. Each _dlkm partition should have a single prebuilt_kernel_modules dependency. +// This ensures that the depmod artifacts (modules.* installed in /lib/modules/) are generated with a complete view. + +// The input `kernelModules` is a space separated list of .ko files in the workspace. +func (f *filesystemCreator) createPrebuiltKernelModules(ctx android.LoadHookContext, partitionType string, kernelModules []string) { + fsGenState := ctx.Config().Get(fsGenStateOnceKey).(*FsGenState) + name := generatedModuleName(ctx.Config(), fmt.Sprintf("%s-kernel-modules", partitionType)) + props := &struct { + Name *string + Srcs []string + }{ + Name: proptools.StringPtr(name), + Srcs: kernelModules, + } + kernelModule := ctx.CreateModuleInDirectory( + kernel.PrebuiltKernelModulesFactory, + ".", // create in root directory for now + props, + ) + kernelModule.HideFromMake() + // Add to deps + (*fsGenState.fsDeps[partitionType])[name] = defaultDepCandidateProps(ctx.Config()) +} + +// Create a build_prop and android_info module. This will be used to create /vendor/build.prop +func (f *filesystemCreator) createVendorBuildProp(ctx android.LoadHookContext) { + // Create a android_info for vendor + // The board info files might be in a directory outside the root soong namespace, so create + // the module in "." + partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + androidInfoProps := &struct { + Name *string + Board_info_files []string + Bootloader_board_name *string + }{ + Name: proptools.StringPtr(generatedModuleName(ctx.Config(), "android-info.prop")), + Board_info_files: partitionVars.BoardInfoFiles, + } + if len(androidInfoProps.Board_info_files) == 0 { + androidInfoProps.Bootloader_board_name = proptools.StringPtr(partitionVars.BootLoaderBoardName) + } + androidInfoProp := ctx.CreateModuleInDirectory( + android.AndroidInfoFactory, + ".", + androidInfoProps, + ) + androidInfoProp.HideFromMake() + // Create a build prop for vendor + vendorBuildProps := &struct { + Name *string + Vendor *bool + Stem *string + Product_config *string + Android_info *string + }{ + Name: proptools.StringPtr(generatedModuleName(ctx.Config(), "vendor-build.prop")), + Vendor: proptools.BoolPtr(true), + Stem: proptools.StringPtr("build.prop"), + Product_config: proptools.StringPtr(":product_config"), + Android_info: proptools.StringPtr(":" + androidInfoProp.Name()), + } + vendorBuildProp := ctx.CreateModule( + android.BuildPropFactory, + vendorBuildProps, + ) + vendorBuildProp.HideFromMake() +} + +// createLinkerConfigSourceFilegroups creates filegroup modules to generate linker.config.pb for the following partitions +// 1. vendor: Using PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS (space separated file list) +// 1. product: Using PRODUCT_PRODUCT_LINKER_CONFIG_FRAGMENTS (space separated file list) +// It creates a filegroup for each file in the fragment list +// The filegroup modules are then added to `linker_config_srcs` of the autogenerated vendor `android_filesystem`. +func (f *filesystemCreator) createLinkerConfigSourceFilegroups(ctx android.LoadHookContext, partitionType string) []string { + ret := []string{} + partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + var linkerConfigSrcs []string + if partitionType == "vendor" { + linkerConfigSrcs = android.FirstUniqueStrings(partitionVars.VendorLinkerConfigSrcs) + } else if partitionType == "product" { + linkerConfigSrcs = android.FirstUniqueStrings(partitionVars.ProductLinkerConfigSrcs) + } else { + ctx.ModuleErrorf("linker.config.pb is only supported for vendor and product partitions. For system partition, use `android_system_image`") + } + + if len(linkerConfigSrcs) > 0 { + // Create a filegroup, and add `:<filegroup_name>` to ret. + for index, linkerConfigSrc := range linkerConfigSrcs { + dir := filepath.Dir(linkerConfigSrc) + base := filepath.Base(linkerConfigSrc) + fgName := generatedModuleName(ctx.Config(), fmt.Sprintf("%s-linker-config-src%s", partitionType, strconv.Itoa(index))) + srcs := []string{base} + fgProps := &struct { + Name *string + Srcs proptools.Configurable[[]string] + }{ + Name: proptools.StringPtr(fgName), + Srcs: proptools.NewSimpleConfigurable(srcs), + } + ctx.CreateModuleInDirectory( + android.FileGroupFactory, + dir, + fgProps, + ) + ret = append(ret, ":"+fgName) + } + } + return ret +} + type filesystemBaseProperty struct { Name *string Compile_multilib *string + Visibility []string } func generateBaseProps(namePtr *string) *filesystemBaseProperty { return &filesystemBaseProperty{ Name: namePtr, Compile_multilib: proptools.StringPtr("both"), + // The vbmeta modules are currently in the root directory and depend on the partitions + Visibility: []string{"//.", "//build/soong:__subpackages__"}, } } @@ -579,18 +440,13 @@ func (f *filesystemCreator) createDiffTest(ctx android.ModuleContext, partitionT ctx.ModuleErrorf("Expected module %s to provide FileysystemInfo", partitionModuleName) } makeFileList := android.PathForArbitraryOutput(ctx, fmt.Sprintf("target/product/%s/obj/PACKAGING/%s_intermediates/file_list.txt", ctx.Config().DeviceName(), partitionType)) - // For now, don't allowlist anything. The test will fail, but that's fine in the current - // early stages where we're just figuring out what we need - emptyAllowlistFile := android.PathForModuleOut(ctx, fmt.Sprintf("allowlist_%s.txt", partitionModuleName)) - android.WriteFileRule(ctx, emptyAllowlistFile, "") diffTestResultFile := android.PathForModuleOut(ctx, fmt.Sprintf("diff_test_%s.txt", partitionModuleName)) builder := android.NewRuleBuilder(pctx, ctx) builder.Command().BuiltTool("file_list_diff"). Input(makeFileList). Input(filesystemInfo.FileListFile). - Text(partitionModuleName). - FlagWithInput("--allowlists ", emptyAllowlistFile) + Text(partitionModuleName) builder.Command().Text("touch").Output(diffTestResultFile) builder.Build(partitionModuleName+" diff test", partitionModuleName+" diff test") return diffTestResultFile @@ -608,16 +464,42 @@ func createFailingCommand(ctx android.ModuleContext, message string) android.Pat return file } +func createVbmetaDiff(ctx android.ModuleContext, vbmetaModuleName string, vbmetaPartitionName string) android.Path { + vbmetaModule := ctx.GetDirectDepWithTag(vbmetaModuleName, generatedVbmetaPartitionDepTag) + outputFilesProvider, ok := android.OtherModuleProvider(ctx, vbmetaModule, android.OutputFilesProvider) + if !ok { + ctx.ModuleErrorf("Expected module %s to provide OutputFiles", vbmetaModule) + } + if len(outputFilesProvider.DefaultOutputFiles) != 1 { + ctx.ModuleErrorf("Expected 1 output file from module %s", vbmetaModule) + } + soongVbMetaFile := outputFilesProvider.DefaultOutputFiles[0] + makeVbmetaFile := android.PathForArbitraryOutput(ctx, fmt.Sprintf("target/product/%s/%s.img", ctx.Config().DeviceName(), vbmetaPartitionName)) + + diffTestResultFile := android.PathForModuleOut(ctx, fmt.Sprintf("diff_test_%s.txt", vbmetaModuleName)) + builder := android.NewRuleBuilder(pctx, ctx) + builder.Command().Text("diff"). + Input(soongVbMetaFile). + Input(makeVbmetaFile) + builder.Command().Text("touch").Output(diffTestResultFile) + builder.Build(vbmetaModuleName+" diff test", vbmetaModuleName+" diff test") + return diffTestResultFile +} + type systemImageDepTagType struct { blueprint.BaseDependencyTag } var generatedFilesystemDepTag systemImageDepTagType +var generatedVbmetaPartitionDepTag systemImageDepTagType func (f *filesystemCreator) DepsMutator(ctx android.BottomUpMutatorContext) { for _, partitionType := range f.properties.Generated_partition_types { ctx.AddDependency(ctx.Module(), generatedFilesystemDepTag, generatedModuleNameForPartition(ctx.Config(), partitionType)) } + for _, vbmetaModule := range f.properties.Vbmeta_module_names { + ctx.AddDependency(ctx.Module(), generatedVbmetaPartitionDepTag, vbmetaModule) + } } func (f *filesystemCreator) GenerateAndroidBuildActions(ctx android.ModuleContext) { @@ -647,6 +529,11 @@ func (f *filesystemCreator) GenerateAndroidBuildActions(ctx android.ModuleContex diffTestFiles = append(diffTestFiles, diffTestFile) ctx.Phony(fmt.Sprintf("soong_generated_%s_filesystem_test", partitionType), diffTestFile) } + for i, vbmetaModule := range f.properties.Vbmeta_module_names { + diffTestFile := createVbmetaDiff(ctx, vbmetaModule, f.properties.Vbmeta_partition_names[i]) + diffTestFiles = append(diffTestFiles, diffTestFile) + ctx.Phony(fmt.Sprintf("soong_generated_%s_filesystem_test", f.properties.Vbmeta_partition_names[i]), diffTestFile) + } ctx.Phony("soong_generated_filesystem_tests", diffTestFiles...) } @@ -655,9 +542,6 @@ func generateBpContent(ctx android.EarlyModuleContext, partitionType string) str if !fsTypeSupported { return "" } - if partitionType == "vendor" { - return "" // TODO: Handle struct props - } baseProps := generateBaseProps(proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), partitionType))) deps := ctx.Config().Get(fsGenStateOnceKey).(*FsGenState).fsDeps[partitionType] @@ -665,7 +549,8 @@ func generateBpContent(ctx android.EarlyModuleContext, partitionType string) str result, err := proptools.RepackProperties([]interface{}{baseProps, fsProps, depProps}) if err != nil { - ctx.ModuleErrorf(err.Error()) + ctx.ModuleErrorf("%s", err.Error()) + return "" } moduleType := "android_filesystem" diff --git a/fsgen/fsgen_mutators.go b/fsgen/fsgen_mutators.go new file mode 100644 index 000000000..e530a5b0a --- /dev/null +++ b/fsgen/fsgen_mutators.go @@ -0,0 +1,346 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fsgen + +import ( + "fmt" + "slices" + "sync" + + "android/soong/android" + + "github.com/google/blueprint/proptools" +) + +func RegisterCollectFileSystemDepsMutators(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("fs_collect_deps", collectDepsMutator).MutatesGlobalState() + ctx.BottomUp("fs_set_deps", setDepsMutator) +} + +var fsGenStateOnceKey = android.NewOnceKey("FsGenState") +var fsGenRemoveOverridesOnceKey = android.NewOnceKey("FsGenRemoveOverrides") + +// Map of partition module name to its partition that may be generated by Soong. +// Note that it is not guaranteed that all modules returned by this function are successfully +// created. +func getAllSoongGeneratedPartitionNames(config android.Config, partitions []string) map[string]string { + ret := map[string]string{} + for _, partition := range partitions { + ret[generatedModuleNameForPartition(config, partition)] = partition + } + return ret +} + +type depCandidateProps struct { + Namespace string + Multilib string + Arch []android.ArchType +} + +// Map of module name to depCandidateProps +type multilibDeps map[string]*depCandidateProps + +// Information necessary to generate the filesystem modules, including details about their +// dependencies +type FsGenState struct { + // List of modules in `PRODUCT_PACKAGES` and `PRODUCT_PACKAGES_DEBUG` + depCandidates []string + // Map of names of partition to the information of modules to be added as deps + fsDeps map[string]*multilibDeps + // List of name of partitions to be generated by the filesystem_creator module + soongGeneratedPartitions []string + // Mutex to protect the fsDeps + fsDepsMutex sync.Mutex + // Map of _all_ soong module names to their corresponding installation properties + moduleToInstallationProps map[string]installationProperties +} + +type installationProperties struct { + Required []string + Overrides []string +} + +func defaultDepCandidateProps(config android.Config) *depCandidateProps { + return &depCandidateProps{ + Namespace: ".", + Arch: []android.ArchType{config.BuildArch}, + } +} + +func generatedPartitions(ctx android.LoadHookContext) []string { + generatedPartitions := []string{"system"} + if ctx.DeviceConfig().SystemExtPath() == "system_ext" { + generatedPartitions = append(generatedPartitions, "system_ext") + } + if ctx.DeviceConfig().BuildingVendorImage() && ctx.DeviceConfig().VendorPath() == "vendor" { + generatedPartitions = append(generatedPartitions, "vendor") + } + if ctx.DeviceConfig().BuildingProductImage() && ctx.DeviceConfig().ProductPath() == "product" { + generatedPartitions = append(generatedPartitions, "product") + } + if ctx.DeviceConfig().BuildingOdmImage() && ctx.DeviceConfig().OdmPath() == "odm" { + generatedPartitions = append(generatedPartitions, "odm") + } + if ctx.DeviceConfig().BuildingUserdataImage() && ctx.DeviceConfig().UserdataPath() == "data" { + generatedPartitions = append(generatedPartitions, "userdata") + } + if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingSystemDlkmImage { + generatedPartitions = append(generatedPartitions, "system_dlkm") + } + return generatedPartitions +} + +func createFsGenState(ctx android.LoadHookContext, generatedPrebuiltEtcModuleNames []string) *FsGenState { + return ctx.Config().Once(fsGenStateOnceKey, func() interface{} { + partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + candidates := android.FirstUniqueStrings(android.Concat(partitionVars.ProductPackages, partitionVars.ProductPackagesDebug)) + candidates = android.Concat(candidates, generatedPrebuiltEtcModuleNames) + + return &FsGenState{ + depCandidates: candidates, + fsDeps: map[string]*multilibDeps{ + // These additional deps are added according to the cuttlefish system image bp. + "system": { + "com.android.apex.cts.shim.v1_prebuilt": defaultDepCandidateProps(ctx.Config()), + "dex_bootjars": defaultDepCandidateProps(ctx.Config()), + "framework_compatibility_matrix.device.xml": defaultDepCandidateProps(ctx.Config()), + "init.environ.rc-soong": defaultDepCandidateProps(ctx.Config()), + "libcompiler_rt": defaultDepCandidateProps(ctx.Config()), + "libdmabufheap": defaultDepCandidateProps(ctx.Config()), + "libgsi": defaultDepCandidateProps(ctx.Config()), + "llndk.libraries.txt": defaultDepCandidateProps(ctx.Config()), + "logpersist.start": defaultDepCandidateProps(ctx.Config()), + "update_engine_sideload": defaultDepCandidateProps(ctx.Config()), + }, + "vendor": { + "fs_config_files_vendor": defaultDepCandidateProps(ctx.Config()), + "fs_config_dirs_vendor": defaultDepCandidateProps(ctx.Config()), + generatedModuleName(ctx.Config(), "vendor-build.prop"): defaultDepCandidateProps(ctx.Config()), + }, + "odm": { + // fs_config_* files are automatically installed for all products with odm partitions. + // https://cs.android.com/android/_/android/platform/build/+/e4849e87ab660b59a6501b3928693db065ee873b:tools/fs_config/Android.mk;l=34;drc=8d6481b92c4b4e9b9f31a61545b6862090fcc14b;bpv=1;bpt=0 + "fs_config_files_odm": defaultDepCandidateProps(ctx.Config()), + "fs_config_dirs_odm": defaultDepCandidateProps(ctx.Config()), + }, + "product": {}, + "system_ext": { + // VNDK apexes are automatically included. + // This hardcoded list will need to be updated if `PRODUCT_EXTRA_VNDK_VERSIONS` is updated. + // https://cs.android.com/android/_/android/platform/build/+/adba533072b00c53ac0f198c550a3cbd7a00e4cd:core/main.mk;l=984;bpv=1;bpt=0;drc=174db7b179592cf07cbfd2adb0119486fda911e7 + "com.android.vndk.v30": defaultDepCandidateProps(ctx.Config()), + "com.android.vndk.v31": defaultDepCandidateProps(ctx.Config()), + "com.android.vndk.v32": defaultDepCandidateProps(ctx.Config()), + "com.android.vndk.v33": defaultDepCandidateProps(ctx.Config()), + "com.android.vndk.v34": defaultDepCandidateProps(ctx.Config()), + }, + "system_dlkm": {}, + "userdata": {}, + }, + fsDepsMutex: sync.Mutex{}, + moduleToInstallationProps: map[string]installationProperties{}, + } + }).(*FsGenState) +} + +func checkDepModuleInMultipleNamespaces(mctx android.BottomUpMutatorContext, foundDeps multilibDeps, module string, partitionName string) { + otherNamespace := mctx.Namespace().Path + if val, found := foundDeps[module]; found && otherNamespace != "." && !android.InList(val.Namespace, []string{".", otherNamespace}) { + mctx.ModuleErrorf("found in multiple namespaces(%s and %s) when including in %s partition", val.Namespace, otherNamespace, partitionName) + } +} + +func appendDepIfAppropriate(mctx android.BottomUpMutatorContext, deps *multilibDeps, installPartition string) { + moduleName := mctx.ModuleName() + checkDepModuleInMultipleNamespaces(mctx, *deps, moduleName, installPartition) + if _, ok := (*deps)[moduleName]; ok { + // Prefer the namespace-specific module over the platform module + if mctx.Namespace().Path != "." { + (*deps)[moduleName].Namespace = mctx.Namespace().Path + } + (*deps)[moduleName].Arch = append((*deps)[moduleName].Arch, mctx.Module().Target().Arch.ArchType) + } else { + multilib, _ := mctx.Module().DecodeMultilib(mctx) + (*deps)[moduleName] = &depCandidateProps{ + Namespace: mctx.Namespace().Path, + Multilib: multilib, + Arch: []android.ArchType{mctx.Module().Target().Arch.ArchType}, + } + } +} + +func collectDepsMutator(mctx android.BottomUpMutatorContext) { + fsGenState := mctx.Config().Get(fsGenStateOnceKey).(*FsGenState) + + m := mctx.Module() + if m.Target().Os.Class == android.Device && slices.Contains(fsGenState.depCandidates, mctx.ModuleName()) { + installPartition := m.PartitionTag(mctx.DeviceConfig()) + fsGenState.fsDepsMutex.Lock() + // Only add the module as dependency when: + // - its enabled + // - its namespace is included in PRODUCT_SOONG_NAMESPACES + if m.Enabled(mctx) && m.ExportedToMake() { + appendDepIfAppropriate(mctx, fsGenState.fsDeps[installPartition], installPartition) + } + fsGenState.fsDepsMutex.Unlock() + } + // store the map of module to (required,overrides) even if the module is not in PRODUCT_PACKAGES. + // the module might be installed transitively. + if m.Target().Os.Class == android.Device && m.Enabled(mctx) && m.ExportedToMake() { + fsGenState.fsDepsMutex.Lock() + fsGenState.moduleToInstallationProps[m.Name()] = installationProperties{ + Required: m.RequiredModuleNames(mctx), + Overrides: m.Overrides(), + } + fsGenState.fsDepsMutex.Unlock() + } +} + +type depsStruct struct { + Deps []string +} + +type multilibDepsStruct struct { + Common depsStruct + Lib32 depsStruct + Lib64 depsStruct + Both depsStruct + Prefer32 depsStruct +} + +type packagingPropsStruct struct { + High_priority_deps []string + Deps []string + Multilib multilibDepsStruct +} + +func fullyQualifiedModuleName(moduleName, namespace string) string { + if namespace == "." { + return moduleName + } + return fmt.Sprintf("//%s:%s", namespace, moduleName) +} + +func getBitness(archTypes []android.ArchType) (ret []string) { + for _, archType := range archTypes { + if archType.Multilib == "" { + ret = append(ret, android.COMMON_VARIANT) + } else { + ret = append(ret, archType.Bitness()) + } + } + return ret +} + +func setDepsMutator(mctx android.BottomUpMutatorContext) { + removeOverriddenDeps(mctx) + fsGenState := mctx.Config().Get(fsGenStateOnceKey).(*FsGenState) + fsDeps := fsGenState.fsDeps + soongGeneratedPartitionMap := getAllSoongGeneratedPartitionNames(mctx.Config(), fsGenState.soongGeneratedPartitions) + m := mctx.Module() + if partition, ok := soongGeneratedPartitionMap[m.Name()]; ok { + depsStruct := generateDepStruct(*fsDeps[partition]) + if err := proptools.AppendMatchingProperties(m.GetProperties(), depsStruct, nil); err != nil { + mctx.ModuleErrorf(err.Error()) + } + } +} + +// removeOverriddenDeps collects PRODUCT_PACKAGES and (transitive) required deps. +// it then removes any modules which appear in `overrides` of the above list. +func removeOverriddenDeps(mctx android.BottomUpMutatorContext) { + mctx.Config().Once(fsGenRemoveOverridesOnceKey, func() interface{} { + fsGenState := mctx.Config().Get(fsGenStateOnceKey).(*FsGenState) + fsDeps := fsGenState.fsDeps + overridden := map[string]bool{} + allDeps := []string{} + + // Step 1: Initialization: Append PRODUCT_PACKAGES to the queue + for _, fsDep := range fsDeps { + for depName, _ := range *fsDep { + allDeps = append(allDeps, depName) + } + } + + // Step 2: Process the queue, and add required modules to the queue. + i := 0 + for { + if i == len(allDeps) { + break + } + depName := allDeps[i] + for _, overrides := range fsGenState.moduleToInstallationProps[depName].Overrides { + overridden[overrides] = true + } + // add required dep to the queue. + allDeps = append(allDeps, fsGenState.moduleToInstallationProps[depName].Required...) + i += 1 + } + + // Step 3: Delete all the overridden modules. + for overridden, _ := range overridden { + for partition, _ := range fsDeps { + delete(*fsDeps[partition], overridden) + } + } + return nil + }) +} + +var HighPriorityDeps = []string{} + +func generateDepStruct(deps map[string]*depCandidateProps) *packagingPropsStruct { + depsStruct := packagingPropsStruct{} + for depName, depProps := range deps { + bitness := getBitness(depProps.Arch) + fullyQualifiedDepName := fullyQualifiedModuleName(depName, depProps.Namespace) + if android.InList(depName, HighPriorityDeps) { + depsStruct.High_priority_deps = append(depsStruct.High_priority_deps, fullyQualifiedDepName) + } else if android.InList("32", bitness) && android.InList("64", bitness) { + // If both 32 and 64 bit variants are enabled for this module + switch depProps.Multilib { + case string(android.MultilibBoth): + depsStruct.Multilib.Both.Deps = append(depsStruct.Multilib.Both.Deps, fullyQualifiedDepName) + case string(android.MultilibCommon), string(android.MultilibFirst): + depsStruct.Deps = append(depsStruct.Deps, fullyQualifiedDepName) + case "32": + depsStruct.Multilib.Lib32.Deps = append(depsStruct.Multilib.Lib32.Deps, fullyQualifiedDepName) + case "64", "darwin_universal": + depsStruct.Multilib.Lib64.Deps = append(depsStruct.Multilib.Lib64.Deps, fullyQualifiedDepName) + case "prefer32", "first_prefer32": + depsStruct.Multilib.Prefer32.Deps = append(depsStruct.Multilib.Prefer32.Deps, fullyQualifiedDepName) + default: + depsStruct.Multilib.Both.Deps = append(depsStruct.Multilib.Both.Deps, fullyQualifiedDepName) + } + } else if android.InList("64", bitness) { + // If only 64 bit variant is enabled + depsStruct.Multilib.Lib64.Deps = append(depsStruct.Multilib.Lib64.Deps, fullyQualifiedDepName) + } else if android.InList("32", bitness) { + // If only 32 bit variant is enabled + depsStruct.Multilib.Lib32.Deps = append(depsStruct.Multilib.Lib32.Deps, fullyQualifiedDepName) + } else { + // If only common variant is enabled + depsStruct.Multilib.Common.Deps = append(depsStruct.Multilib.Common.Deps, fullyQualifiedDepName) + } + } + depsStruct.Deps = android.SortedUniqueStrings(depsStruct.Deps) + depsStruct.Multilib.Lib32.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Lib32.Deps) + depsStruct.Multilib.Lib64.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Lib64.Deps) + depsStruct.Multilib.Prefer32.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Prefer32.Deps) + depsStruct.Multilib.Both.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Both.Deps) + depsStruct.Multilib.Common.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Common.Deps) + + return &depsStruct +} diff --git a/fsgen/prebuilt_etc_modules_gen.go b/fsgen/prebuilt_etc_modules_gen.go new file mode 100644 index 000000000..362ac31b8 --- /dev/null +++ b/fsgen/prebuilt_etc_modules_gen.go @@ -0,0 +1,300 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fsgen + +import ( + "android/soong/android" + "android/soong/etc" + "fmt" + "path/filepath" + "strings" + + "github.com/google/blueprint/proptools" +) + +type srcBaseFileInstallBaseFileTuple struct { + srcBaseFile string + installBaseFile string +} + +// prebuilt src files grouped by the install partitions. +// Each groups are a mapping of the relative install path to the name of the files +type prebuiltSrcGroupByInstallPartition struct { + system map[string][]srcBaseFileInstallBaseFileTuple + system_ext map[string][]srcBaseFileInstallBaseFileTuple + product map[string][]srcBaseFileInstallBaseFileTuple + vendor map[string][]srcBaseFileInstallBaseFileTuple +} + +func newPrebuiltSrcGroupByInstallPartition() *prebuiltSrcGroupByInstallPartition { + return &prebuiltSrcGroupByInstallPartition{ + system: map[string][]srcBaseFileInstallBaseFileTuple{}, + system_ext: map[string][]srcBaseFileInstallBaseFileTuple{}, + product: map[string][]srcBaseFileInstallBaseFileTuple{}, + vendor: map[string][]srcBaseFileInstallBaseFileTuple{}, + } +} + +func isSubdirectory(parent, child string) bool { + rel, err := filepath.Rel(parent, child) + if err != nil { + return false + } + return !strings.HasPrefix(rel, "..") +} + +func appendIfCorrectInstallPartition(partitionToInstallPathList []partitionToInstallPath, destPath, srcPath string, srcGroup *prebuiltSrcGroupByInstallPartition) { + for _, part := range partitionToInstallPathList { + partition := part.name + installPath := part.installPath + + if isSubdirectory(installPath, destPath) { + relativeInstallPath, _ := filepath.Rel(installPath, destPath) + relativeInstallDir := filepath.Dir(relativeInstallPath) + var srcMap map[string][]srcBaseFileInstallBaseFileTuple + switch partition { + case "system": + srcMap = srcGroup.system + case "system_ext": + srcMap = srcGroup.system_ext + case "product": + srcMap = srcGroup.product + case "vendor": + srcMap = srcGroup.vendor + } + if srcMap != nil { + srcMap[relativeInstallDir] = append(srcMap[relativeInstallDir], srcBaseFileInstallBaseFileTuple{ + srcBaseFile: filepath.Base(srcPath), + installBaseFile: filepath.Base(destPath), + }) + } + return + } + } +} + +func uniqueExistingProductCopyFileMap(ctx android.LoadHookContext) map[string]string { + seen := make(map[string]bool) + filtered := make(map[string]string) + + for src, dest := range ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.ProductCopyFiles { + if _, ok := seen[dest]; !ok { + if optionalPath := android.ExistentPathForSource(ctx, src); optionalPath.Valid() { + seen[dest] = true + filtered[src] = dest + } + } + } + + return filtered +} + +type partitionToInstallPath struct { + name string + installPath string +} + +func processProductCopyFiles(ctx android.LoadHookContext) map[string]*prebuiltSrcGroupByInstallPartition { + // Filter out duplicate dest entries and non existing src entries + productCopyFileMap := uniqueExistingProductCopyFileMap(ctx) + + // System is intentionally added at the last to consider the scenarios where + // non-system partitions are installed as part of the system partition + partitionToInstallPathList := []partitionToInstallPath{ + {name: "vendor", installPath: ctx.DeviceConfig().VendorPath()}, + {name: "product", installPath: ctx.DeviceConfig().ProductPath()}, + {name: "system_ext", installPath: ctx.DeviceConfig().SystemExtPath()}, + {name: "system", installPath: "system"}, + } + + groupedSources := map[string]*prebuiltSrcGroupByInstallPartition{} + for _, src := range android.SortedKeys(productCopyFileMap) { + dest := productCopyFileMap[src] + srcFileDir := filepath.Dir(src) + if _, ok := groupedSources[srcFileDir]; !ok { + groupedSources[srcFileDir] = newPrebuiltSrcGroupByInstallPartition() + } + appendIfCorrectInstallPartition(partitionToInstallPathList, dest, filepath.Base(src), groupedSources[srcFileDir]) + } + + return groupedSources +} + +type prebuiltModuleProperties struct { + Name *string + + Soc_specific *bool + Product_specific *bool + System_ext_specific *bool + + Srcs []string + Dsts []string + + No_full_install *bool + + NamespaceExportedToMake bool + + Visibility []string +} + +// Split relative_install_path to a separate struct, because it is not supported for every +// modules listed in [etcInstallPathToFactoryMap] +type prebuiltSubdirProperties struct { + // If the base file name of the src and dst all match, dsts property does not need to be + // set, and only relative_install_path can be set. + Relative_install_path *string +} + +var ( + etcInstallPathToFactoryList = map[string]android.ModuleFactory{ + "": etc.PrebuiltRootFactory, + "avb": etc.PrebuiltAvbFactory, + "bin": etc.PrebuiltBinaryFactory, + "bt_firmware": etc.PrebuiltBtFirmwareFactory, + "cacerts": etc.PrebuiltEtcCaCertsFactory, + "dsp": etc.PrebuiltDSPFactory, + "etc": etc.PrebuiltEtcFactory, + "etc/dsp": etc.PrebuiltDSPFactory, + "etc/firmware": etc.PrebuiltFirmwareFactory, + "firmware": etc.PrebuiltFirmwareFactory, + "fonts": etc.PrebuiltFontFactory, + "framework": etc.PrebuiltFrameworkFactory, + "lib": etc.PrebuiltRenderScriptBitcodeFactory, + "lib64": etc.PrebuiltRenderScriptBitcodeFactory, + "lib/rfsa": etc.PrebuiltRFSAFactory, + "media": etc.PrebuiltMediaFactory, + "odm": etc.PrebuiltOdmFactory, + "optee": etc.PrebuiltOpteeFactory, + "overlay": etc.PrebuiltOverlayFactory, + "priv-app": etc.PrebuiltPrivAppFactory, + "res": etc.PrebuiltResFactory, + "rfs": etc.PrebuiltRfsFactory, + "tts": etc.PrebuiltVoicepackFactory, + "tvservice": etc.PrebuiltTvServiceFactory, + "usr/share": etc.PrebuiltUserShareFactory, + "usr/hyphen-data": etc.PrebuiltUserHyphenDataFactory, + "usr/keylayout": etc.PrebuiltUserKeyLayoutFactory, + "usr/keychars": etc.PrebuiltUserKeyCharsFactory, + "usr/srec": etc.PrebuiltUserSrecFactory, + "usr/idc": etc.PrebuiltUserIdcFactory, + "vendor_dlkm": etc.PrebuiltVendorDlkmFactory, + "wallpaper": etc.PrebuiltWallpaperFactory, + "wlc_upt": etc.PrebuiltWlcUptFactory, + } +) + +func createPrebuiltEtcModule(ctx android.LoadHookContext, partition, srcDir, destDir string, destFiles []srcBaseFileInstallBaseFileTuple) string { + moduleProps := &prebuiltModuleProperties{} + propsList := []interface{}{moduleProps} + + // generated module name follows the pattern: + // <install partition>-<src file path>-<relative install path from partition root>-<install file extension> + // Note that all path separators are replaced with "_" in the name + moduleName := partition + if !android.InList(srcDir, []string{"", "."}) { + moduleName += fmt.Sprintf("-%s", strings.ReplaceAll(srcDir, string(filepath.Separator), "_")) + } + if !android.InList(destDir, []string{"", "."}) { + moduleName += fmt.Sprintf("-%s", strings.ReplaceAll(destDir, string(filepath.Separator), "_")) + } + if len(destFiles) > 0 { + if ext := filepath.Ext(destFiles[0].srcBaseFile); ext != "" { + moduleName += fmt.Sprintf("-%s", strings.TrimPrefix(ext, ".")) + } + } + moduleProps.Name = proptools.StringPtr(moduleName) + + allCopyFileNamesUnchanged := true + var srcBaseFiles, installBaseFiles []string + for _, tuple := range destFiles { + if tuple.srcBaseFile != tuple.installBaseFile { + allCopyFileNamesUnchanged = false + } + srcBaseFiles = append(srcBaseFiles, tuple.srcBaseFile) + installBaseFiles = append(installBaseFiles, tuple.installBaseFile) + } + + // Find out the most appropriate module type to generate + var etcInstallPathKey string + for _, etcInstallPath := range android.SortedKeys(etcInstallPathToFactoryList) { + // Do not break when found but iterate until the end to find a module with more + // specific install path + if strings.HasPrefix(destDir, etcInstallPath) { + etcInstallPathKey = etcInstallPath + } + } + destDir, _ = filepath.Rel(etcInstallPathKey, destDir) + + // Set partition specific properties + switch partition { + case "system_ext": + moduleProps.System_ext_specific = proptools.BoolPtr(true) + case "product": + moduleProps.Product_specific = proptools.BoolPtr(true) + case "vendor": + moduleProps.Soc_specific = proptools.BoolPtr(true) + } + + // Set appropriate srcs, dsts, and releative_install_path based on + // the source and install file names + if allCopyFileNamesUnchanged { + moduleProps.Srcs = srcBaseFiles + + // Specify relative_install_path if it is not installed in the root directory of the + // partition + if !android.InList(destDir, []string{"", "."}) { + propsList = append(propsList, &prebuiltSubdirProperties{ + Relative_install_path: proptools.StringPtr(destDir), + }) + } + } else { + moduleProps.Srcs = srcBaseFiles + dsts := []string{} + for _, installBaseFile := range installBaseFiles { + dsts = append(dsts, filepath.Join(destDir, installBaseFile)) + } + moduleProps.Dsts = dsts + } + + moduleProps.No_full_install = proptools.BoolPtr(true) + moduleProps.NamespaceExportedToMake = true + moduleProps.Visibility = []string{"//visibility:public"} + + ctx.CreateModuleInDirectory(etcInstallPathToFactoryList[etcInstallPathKey], srcDir, propsList...) + + return moduleName +} + +func createPrebuiltEtcModulesForPartition(ctx android.LoadHookContext, partition, srcDir string, destDirFilesMap map[string][]srcBaseFileInstallBaseFileTuple) (ret []string) { + for _, destDir := range android.SortedKeys(destDirFilesMap) { + ret = append(ret, createPrebuiltEtcModule(ctx, partition, srcDir, destDir, destDirFilesMap[destDir])) + } + return ret +} + +// Creates prebuilt_* modules based on the install paths and returns the list of generated +// module names +func createPrebuiltEtcModules(ctx android.LoadHookContext) (ret []string) { + groupedSources := processProductCopyFiles(ctx) + for _, srcDir := range android.SortedKeys(groupedSources) { + groupedSource := groupedSources[srcDir] + ret = append(ret, createPrebuiltEtcModulesForPartition(ctx, "system", srcDir, groupedSource.system)...) + ret = append(ret, createPrebuiltEtcModulesForPartition(ctx, "system_ext", srcDir, groupedSource.system_ext)...) + ret = append(ret, createPrebuiltEtcModulesForPartition(ctx, "product", srcDir, groupedSource.product)...) + ret = append(ret, createPrebuiltEtcModulesForPartition(ctx, "vendor", srcDir, groupedSource.vendor)...) + } + + return ret +} diff --git a/fsgen/vbmeta_partitions.go b/fsgen/vbmeta_partitions.go new file mode 100644 index 000000000..280e4052d --- /dev/null +++ b/fsgen/vbmeta_partitions.go @@ -0,0 +1,184 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fsgen + +import ( + "android/soong/android" + "android/soong/filesystem" + "slices" + "strconv" + + "github.com/google/blueprint/proptools" +) + +type vbmetaModuleInfo struct { + // The name of the generated vbmeta module + moduleName string + // The name of the module that avb understands. This is the name passed to --chain_partition, + // and also the basename of the output file. (the output file is called partitionName + ".img") + partitionName string +} + +// Creates the vbmeta partition and the chained vbmeta partitions. Returns the list of module names +// that the function created. May return nil if the product isn't using avb. +// +// AVB is Android Verified Boot: https://source.android.com/docs/security/features/verifiedboot +// It works by signing all the partitions, but then also including an extra metadata paritition +// called vbmeta that depends on all the other signed partitions. This creates a requirement +// that you update all those partitions and the vbmeta partition together, so in order to relax +// that requirement products can set up "chained" vbmeta partitions, where a chained partition +// like vbmeta_system might contain the avb metadata for just a few products. In cuttlefish +// vbmeta_system contains metadata about product, system, and system_ext. Using chained partitions, +// that group of partitions can be updated independently from the other signed partitions. +func createVbmetaPartitions(ctx android.LoadHookContext, generatedPartitionTypes []string) []vbmetaModuleInfo { + partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + // Some products seem to have BuildingVbmetaImage as true even when BoardAvbEnable is false + if !partitionVars.BuildingVbmetaImage || !partitionVars.BoardAvbEnable { + return nil + } + + var result []vbmetaModuleInfo + + var chainedPartitions []filesystem.ChainedPartitionProperties + var partitionTypesHandledByChainedPartitions []string + for chainedName, props := range partitionVars.ChainedVbmetaPartitions { + chainedName = "vbmeta_" + chainedName + if len(props.Partitions) == 0 { + continue + } + if len(props.Key) == 0 { + ctx.ModuleErrorf("No key found for chained avb partition %q", chainedName) + continue + } + if len(props.Algorithm) == 0 { + ctx.ModuleErrorf("No algorithm found for chained avb partition %q", chainedName) + continue + } + if len(props.RollbackIndex) == 0 { + ctx.ModuleErrorf("No rollback index found for chained avb partition %q", chainedName) + continue + } + ril, err := strconv.ParseInt(props.RollbackIndexLocation, 10, 32) + if err != nil { + ctx.ModuleErrorf("Rollback index location must be an int, got %q", props.RollbackIndexLocation) + continue + } + // The default is to use the PlatformSecurityPatch, and a lot of product config files + // just set it to the platform security patch, so detect that and don't set the property + // in soong. + var rollbackIndex *int64 + if props.RollbackIndex != ctx.Config().PlatformSecurityPatch() { + i, err := strconv.ParseInt(props.RollbackIndex, 10, 32) + if err != nil { + ctx.ModuleErrorf("Rollback index must be an int, got %q", props.RollbackIndex) + continue + } + rollbackIndex = &i + } + + var partitionModules []string + for _, partition := range props.Partitions { + partitionTypesHandledByChainedPartitions = append(partitionTypesHandledByChainedPartitions, partition) + if !slices.Contains(generatedPartitionTypes, partition) { + // The partition is probably unsupported. + continue + } + partitionModules = append(partitionModules, generatedModuleNameForPartition(ctx.Config(), partition)) + } + + name := generatedModuleName(ctx.Config(), chainedName) + ctx.CreateModuleInDirectory( + filesystem.VbmetaFactory, + ".", // Create in the root directory for now so its easy to get the key + &filesystem.VbmetaProperties{ + Partition_name: proptools.StringPtr(chainedName), + Stem: proptools.StringPtr(chainedName + ".img"), + Private_key: proptools.StringPtr(props.Key), + Algorithm: &props.Algorithm, + Rollback_index: rollbackIndex, + Rollback_index_location: &ril, + Partitions: proptools.NewSimpleConfigurable(partitionModules), + }, &struct { + Name *string + }{ + Name: &name, + }, + ).HideFromMake() + + chainedPartitions = append(chainedPartitions, filesystem.ChainedPartitionProperties{ + Name: &chainedName, + Rollback_index_location: &ril, + Private_key: &props.Key, + }) + + result = append(result, vbmetaModuleInfo{ + moduleName: name, + partitionName: chainedName, + }) + } + + vbmetaModuleName := generatedModuleName(ctx.Config(), "vbmeta") + + var algorithm *string + var ri *int64 + var key *string + if len(partitionVars.BoardAvbKeyPath) == 0 { + // Match make's defaults: https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/Makefile;l=4568;drc=5b55f926830963c02ab1d2d91e46442f04ba3af0 + key = proptools.StringPtr("external/avb/test/data/testkey_rsa4096.pem") + algorithm = proptools.StringPtr("SHA256_RSA4096") + } else { + key = proptools.StringPtr(partitionVars.BoardAvbKeyPath) + algorithm = proptools.StringPtr(partitionVars.BoardAvbAlgorithm) + } + if len(partitionVars.BoardAvbRollbackIndex) > 0 { + parsedRi, err := strconv.ParseInt(partitionVars.BoardAvbRollbackIndex, 10, 32) + if err != nil { + ctx.ModuleErrorf("Rollback index location must be an int, got %q", partitionVars.BoardAvbRollbackIndex) + } + ri = &parsedRi + } + + var partitionModules []string + for _, partitionType := range generatedPartitionTypes { + if slices.Contains(partitionTypesHandledByChainedPartitions, partitionType) { + // Already handled by a chained vbmeta partition + continue + } + partitionModules = append(partitionModules, generatedModuleNameForPartition(ctx.Config(), partitionType)) + } + + ctx.CreateModuleInDirectory( + filesystem.VbmetaFactory, + ".", // Create in the root directory for now so its easy to get the key + &filesystem.VbmetaProperties{ + Stem: proptools.StringPtr("vbmeta.img"), + Algorithm: algorithm, + Private_key: key, + Rollback_index: ri, + Chained_partitions: chainedPartitions, + Partitions: proptools.NewSimpleConfigurable(partitionModules), + }, &struct { + Name *string + }{ + Name: &vbmetaModuleName, + }, + ).HideFromMake() + + result = append(result, vbmetaModuleInfo{ + moduleName: vbmetaModuleName, + partitionName: "vbmeta", + }) + return result +} diff --git a/genrule/genrule.go b/genrule/genrule.go index 1ab137837..1282bfbae 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -231,7 +231,7 @@ type generateTask struct { // For nsjail tasks useNsjail bool - dirSrcs android.Paths + dirSrcs android.DirectoryPaths } func (g *Module) GeneratedSourceFiles() android.Paths { @@ -604,7 +604,8 @@ func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) { if task.useNsjail { for _, input := range task.dirSrcs { - cmd.Implicit(input) + cmd.ImplicitDirectory(input) + // TODO(b/375551969): remove glob if paths, err := ctx.GlobWithDeps(filepath.Join(input.String(), "**/*"), nil); err == nil { rule.NsjailImplicits(android.PathsForSource(ctx, paths)) } else { diff --git a/java/aar.go b/java/aar.go index 66ca00af8..b5cdde36a 100644 --- a/java/aar.go +++ b/java/aar.go @@ -77,7 +77,7 @@ type aaptProperties struct { // list of directories relative to the Blueprints file containing // Android resources. Defaults to ["res"] if a directory called res exists. // Set to [] to disable the default. - Resource_dirs []string `android:"path"` + Resource_dirs proptools.Configurable[[]string] `android:"path"` // list of zip files containing Android resources. Resource_zips []string `android:"path"` @@ -275,7 +275,7 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext android.SdkConte IncludeDirs: false, }) assetDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.aaptProperties.Asset_dirs, "assets") - resourceDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.aaptProperties.Resource_dirs, "res") + resourceDirs := android.PathsWithOptionalDefaultForModuleSrc(ctx, a.aaptProperties.Resource_dirs.GetOrDefault(ctx, nil), "res") resourceZips := android.PathsForModuleSrc(ctx, a.aaptProperties.Resource_zips) // Glob directories into lists of paths diff --git a/java/app.go b/java/app.go index 15b1114a7..94c9e5b82 100644 --- a/java/app.go +++ b/java/app.go @@ -1361,7 +1361,7 @@ func AndroidAppFactory() android.Module { Filter_product *string Aaptflags []string Manifest *string - Resource_dirs []string + Resource_dirs proptools.Configurable[[]string] Flags_packages []string }{ Name: proptools.StringPtr(rroPackageName), @@ -1465,8 +1465,6 @@ func (a *AndroidTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.data = append(a.data, android.PathsForModuleSrc(ctx, a.testProperties.Device_common_data)...) a.data = append(a.data, android.PathsForModuleSrc(ctx, a.testProperties.Device_first_data)...) a.data = append(a.data, android.PathsForModuleSrc(ctx, a.testProperties.Device_first_prefer32_data)...) - a.data = append(a.data, android.PathsForModuleSrc(ctx, a.testProperties.Device_first_vendor_data)...) - a.data = append(a.data, android.PathsForModuleSrc(ctx, a.testProperties.Device_first_vendor_shared_data)...) android.SetProvider(ctx, tradefed.BaseTestProviderKey, tradefed.BaseTestProviderData{ InstalledFiles: a.data, OutputFile: a.OutputFile(), diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index 1a3368057..7c0f54431 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -520,7 +520,7 @@ func (b *BootclasspathFragmentModule) GenerateAndroidBuildActions(ctx android.Mo // be output to Make but it does not really matter which variant is output. The default/platform // variant is the first (ctx.PrimaryModule()) and is usually hidden from make so this just picks // the last variant (ctx.FinalModule()). - if ctx.Module() != ctx.FinalModule() { + if !ctx.IsFinalModule(ctx.Module()) { b.HideFromMake() } } diff --git a/java/core-libraries/Android.bp b/java/core-libraries/Android.bp index da865404b..8c808e4b9 100644 --- a/java/core-libraries/Android.bp +++ b/java/core-libraries/Android.bp @@ -132,10 +132,10 @@ java_library { // prebuilts/sdk/update_prebuilts.py script to update the prebuilts/sdk // directory. java_library { - name: "core-current-stubs-for-system-modules", + name: "core-current-stubs-for-system-modules-exportable", visibility: ["//development/sdk"], static_libs: [ - "core.current.stubs", + "core.current.stubs.exportable", // This one is not on device but it's needed when javac compiles code // containing lambdas. "core-lambda-stubs-for-system-modules", @@ -155,6 +155,19 @@ java_library { ], } +java_library { + name: "core-current-stubs-for-system-modules", + visibility: ["//development/sdk"], + static_libs: [ + "core.current.stubs", + // This one is not on device but it's needed when javac compiles code + // containing lambdas. + "core-lambda-stubs-for-system-modules", + ], + sdk_version: "none", + system_modules: "none", +} + // Defaults module to strip out android annotations java_defaults { name: "system-modules-no-annotations", diff --git a/java/java.go b/java/java.go index ac4f49619..1d572faef 100644 --- a/java/java.go +++ b/java/java.go @@ -606,10 +606,8 @@ func getJavaVersion(ctx android.ModuleContext, javaVersion string, sdkContext an } else if ctx.Device() { return defaultJavaLanguageVersion(ctx, sdkContext.SdkVersion(ctx)) } else if ctx.Config().TargetsJava21() { - // Temporary experimental flag to be able to try and build with - // java version 21 options. The flag, if used, just sets Java - // 21 as the default version, leaving any components that - // target an older version intact. + // Build flag that controls whether Java 21 is used as the default + // target version, or Java 17. return JAVA_VERSION_21 } else { return JAVA_VERSION_17 @@ -944,6 +942,7 @@ func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Even though the source javalib is not used, we need to hide it to prevent duplicate installation rules. // TODO (b/331665856): Implement a principled solution for this. j.HideFromMake() + j.SkipInstall() } j.provideHiddenAPIPropertyInfo(ctx) @@ -1301,16 +1300,6 @@ type testProperties struct { // host test. Device_first_data []string `android:"path_device_first"` - // same as data, but adds dependencies using the device's os variation, the device's first - // architecture's variation, and the vendor image variation. Can be used to add a module built - // for device to the data of a host test. - Device_first_vendor_data []string `android:"path_device_first_vendor"` - - // same as data, but adds dependencies using the device's os variation, the device's first - // architecture's variation, the vendor image variation, and the shared linkage variation. Can - // be used to add a module built for device to the data of a host test. - Device_first_vendor_shared_data []string `android:"path_device_first_vendor_shared"` - // same as data, but adds dependencies using the device's os variation and the device's first // 32-bit architecture's variation. If a 32-bit arch doesn't exist for this device, it will use // a 64 bit arch instead. Can be used to add a module built for device to the data of a @@ -1608,8 +1597,6 @@ func (j *Test) generateAndroidBuildActionsWithConfig(ctx android.ModuleContext, j.data = append(j.data, android.PathsForModuleSrc(ctx, j.testProperties.Device_common_data)...) j.data = append(j.data, android.PathsForModuleSrc(ctx, j.testProperties.Device_first_data)...) j.data = append(j.data, android.PathsForModuleSrc(ctx, j.testProperties.Device_first_prefer32_data)...) - j.data = append(j.data, android.PathsForModuleSrc(ctx, j.testProperties.Device_first_vendor_data)...) - j.data = append(j.data, android.PathsForModuleSrc(ctx, j.testProperties.Device_first_vendor_shared_data)...) j.extraTestConfigs = android.PathsForModuleSrc(ctx, j.testProperties.Test_options.Extra_test_configs) diff --git a/java/platform_bootclasspath.go b/java/platform_bootclasspath.go index acfc7745e..8b0ca97a1 100644 --- a/java/platform_bootclasspath.go +++ b/java/platform_bootclasspath.go @@ -24,7 +24,7 @@ func init() { } func registerPlatformBootclasspathBuildComponents(ctx android.RegistrationContext) { - ctx.RegisterParallelSingletonModuleType("platform_bootclasspath", platformBootclasspathFactory) + ctx.RegisterModuleType("platform_bootclasspath", platformBootclasspathFactory) } // The tags used for the dependencies between the platform bootclasspath and any configured boot @@ -37,7 +37,7 @@ var ( ) type platformBootclasspathModule struct { - android.SingletonModuleBase + android.ModuleBase ClasspathFragmentBase properties platformBootclasspathProperties @@ -64,7 +64,7 @@ type platformBootclasspathProperties struct { HiddenAPIFlagFileProperties } -func platformBootclasspathFactory() android.SingletonModule { +func platformBootclasspathFactory() android.Module { m := &platformBootclasspathModule{} m.AddProperties(&m.properties) initClasspathFragment(m, BOOTCLASSPATH) @@ -156,18 +156,6 @@ func addDependenciesOntoBootImageModules(ctx android.BottomUpMutatorContext, mod } } -// GenerateSingletonBuildActions does nothing and must never do anything. -// -// This module only implements android.SingletonModule so that it can implement -// android.SingletonMakeVarsProvider. -func (b *platformBootclasspathModule) GenerateSingletonBuildActions(android.SingletonContext) { - // Keep empty -} - -func (d *platformBootclasspathModule) MakeVars(ctx android.MakeVarsContext) { - d.generateHiddenApiMakeVars(ctx) -} - func (b *platformBootclasspathModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Gather all the dependencies from the art, platform, and apex boot jars. artModules := gatherApexModulePairDepsWithTag(ctx, platformBootclasspathArtBootJarDepTag) @@ -428,13 +416,3 @@ func (b *platformBootclasspathModule) buildRuleMergeCSV(ctx android.ModuleContex rule.Build(desc, desc) } - -// generateHiddenApiMakeVars generates make variables needed by hidden API related make rules, e.g. -// veridex and run-appcompat. -func (b *platformBootclasspathModule) generateHiddenApiMakeVars(ctx android.MakeVarsContext) { - if ctx.Config().IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") { - return - } - // INTERNAL_PLATFORM_HIDDENAPI_FLAGS is used by Make rules in art/ and cts/. - ctx.Strict("INTERNAL_PLATFORM_HIDDENAPI_FLAGS", b.hiddenAPIFlagsCSV.String()) -} diff --git a/java/robolectric.go b/java/robolectric.go index 37cac2c74..5f46267f9 100644 --- a/java/robolectric.go +++ b/java/robolectric.go @@ -151,8 +151,6 @@ func (r *robolectricTest) GenerateAndroidBuildActions(ctx android.ModuleContext) r.data = append(r.data, android.PathsForModuleSrc(ctx, r.testProperties.Device_common_data)...) r.data = append(r.data, android.PathsForModuleSrc(ctx, r.testProperties.Device_first_data)...) r.data = append(r.data, android.PathsForModuleSrc(ctx, r.testProperties.Device_first_prefer32_data)...) - r.data = append(r.data, android.PathsForModuleSrc(ctx, r.testProperties.Device_first_vendor_data)...) - r.data = append(r.data, android.PathsForModuleSrc(ctx, r.testProperties.Device_first_vendor_shared_data)...) var ok bool var instrumentedApp *AndroidApp diff --git a/java/sdk.go b/java/sdk.go index 4537f1913..bb2aa8d7e 100644 --- a/java/sdk.go +++ b/java/sdk.go @@ -66,10 +66,8 @@ func defaultJavaLanguageVersion(ctx android.EarlyModuleContext, s android.SdkSpe } else if sdk.FinalOrFutureInt() <= 33 { return JAVA_VERSION_11 } else if ctx.Config().TargetsJava21() { - // Temporary experimental flag to be able to try and build with - // java version 21 options. The flag, if used, just sets Java - // 21 as the default version, leaving any components that - // target an older version intact. + // Build flag that controls whether Java 21 is used as the + // default target version, or Java 17. return JAVA_VERSION_21 } else { return JAVA_VERSION_17 diff --git a/java/sdk_library.go b/java/sdk_library.go index dfbde0e54..f6dfcdd19 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -1403,6 +1403,7 @@ func (module *SdkLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) // Even though the source javalib is not used, we need to hide it to prevent duplicate installation rules. // TODO (b/331665856): Implement a principled solution for this. module.HideFromMake() + module.SkipInstall() } module.stem = proptools.StringDefault(module.overridableProperties.Stem, ctx.ModuleName()) diff --git a/kernel/prebuilt_kernel_modules.go b/kernel/prebuilt_kernel_modules.go index e200ee2c1..4c0a9112b 100644 --- a/kernel/prebuilt_kernel_modules.go +++ b/kernel/prebuilt_kernel_modules.go @@ -32,7 +32,7 @@ func init() { } func registerKernelBuildComponents(ctx android.RegistrationContext) { - ctx.RegisterModuleType("prebuilt_kernel_modules", prebuiltKernelModulesFactory) + ctx.RegisterModuleType("prebuilt_kernel_modules", PrebuiltKernelModulesFactory) } type prebuiltKernelModules struct { @@ -58,7 +58,7 @@ type prebuiltKernelModulesProperties struct { // prebuilt_kernel_modules installs a set of prebuilt kernel module files to the correct directory. // In addition, this module builds modules.load, modules.dep, modules.softdep and modules.alias // using depmod and installs them as well. -func prebuiltKernelModulesFactory() android.Module { +func PrebuiltKernelModulesFactory() android.Module { module := &prebuiltKernelModules{} module.AddProperties(&module.properties) android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) diff --git a/linkerconfig/linkerconfig.go b/linkerconfig/linkerconfig.go index 05b99fd6d..d42287113 100644 --- a/linkerconfig/linkerconfig.go +++ b/linkerconfig/linkerconfig.go @@ -77,7 +77,7 @@ func (l *linkerConfig) GenerateAndroidBuildActions(ctx android.ModuleContext) { output := android.PathForModuleOut(ctx, "linker.config.pb").OutputPath builder := android.NewRuleBuilder(pctx, ctx) - BuildLinkerConfig(ctx, builder, input, nil, nil, output) + BuildLinkerConfig(ctx, builder, android.Paths{input}, nil, nil, output) builder.Build("conv_linker_config", "Generate linker config protobuf "+output.String()) l.outputFilePath = output @@ -91,16 +91,18 @@ func (l *linkerConfig) GenerateAndroidBuildActions(ctx android.ModuleContext) { } func BuildLinkerConfig(ctx android.ModuleContext, builder *android.RuleBuilder, - input android.Path, provideModules []android.Module, requireModules []android.Module, output android.OutputPath) { + inputs android.Paths, provideModules []android.Module, requireModules []android.Module, output android.OutputPath) { // First, convert the input json to protobuf format interimOutput := android.PathForModuleOut(ctx, "temp.pb") - builder.Command(). + cmd := builder.Command(). BuiltTool("conv_linker_config"). Flag("proto"). - Flag("--force"). - FlagWithInput("-s ", input). - FlagWithOutput("-o ", interimOutput) + Flag("--force") + for _, input := range inputs { + cmd.FlagWithInput("-s ", input) + } + cmd.FlagWithOutput("-o ", interimOutput) // Secondly, if there's provideLibs gathered from provideModules, append them var provideLibs []string diff --git a/rust/bindgen.go b/rust/bindgen.go index 394449574..d59057994 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -252,7 +252,7 @@ func (b *bindgenDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) andr // Module defined clang flags and include paths cflags = append(cflags, esc(cflagsProp)...) - for _, include := range b.ClangProperties.Local_include_dirs { + for _, include := range b.ClangProperties.Local_include_dirs.GetOrDefault(ctx, nil) { cflags = append(cflags, "-I"+android.PathForModuleSrc(ctx, include).String()) implicits = append(implicits, android.PathForModuleSrc(ctx, include)) } diff --git a/rust/builder.go b/rust/builder.go index f469f56f8..a1e17fc3f 100644 --- a/rust/builder.go +++ b/rust/builder.go @@ -170,6 +170,9 @@ func TransformSrctoRlib(ctx ModuleContext, mainSrc android.Path, deps PathDeps, return transformSrctoCrate(ctx, mainSrc, deps, flags, outputFile, getTransformProperties(ctx, "rlib")) } +// TransformRlibstoStaticlib is assumed to be called from the cc module, and +// thus needs to reconstruct the common set of flags which need to be passed +// to the rustc compiler. func TransformRlibstoStaticlib(ctx android.ModuleContext, mainSrc android.Path, deps []cc.RustRlibDep, outputFile android.WritablePath) android.Path { diff --git a/rust/config/global.go b/rust/config/global.go index 68a74c204..7b79fca85 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -15,6 +15,7 @@ package config import ( + "fmt" "strings" "android/soong/android" @@ -93,6 +94,16 @@ var ( } ) +func RustPath(ctx android.PathContext) string { + // I can't see any way to flatten the static variable inside Soong, so this + // reproduces the init logic. + var RustBase string = RustDefaultBase + if override := ctx.Config().Getenv("RUST_PREBUILTS_BASE"); override != "" { + RustBase = override + } + return fmt.Sprintf("%s/%s/%s", RustBase, HostPrebuiltTag(ctx.Config()), GetRustVersion(ctx)) +} + func init() { pctx.SourcePathVariable("RustDefaultBase", RustDefaultBase) pctx.VariableConfigMethod("HostPrebuiltTag", HostPrebuiltTag) diff --git a/rust/project_json.go b/rust/project_json.go index 6c1e32047..6e8cebeed 100644 --- a/rust/project_json.go +++ b/rust/project_json.go @@ -19,6 +19,7 @@ import ( "fmt" "android/soong/android" + "android/soong/rust/config" ) // This singleton collects Rust crate definitions and generates a JSON file @@ -44,17 +45,19 @@ type rustProjectDep struct { } type rustProjectCrate struct { - DisplayName string `json:"display_name"` - RootModule string `json:"root_module"` - Edition string `json:"edition,omitempty"` - Deps []rustProjectDep `json:"deps"` - Cfg []string `json:"cfg"` - Env map[string]string `json:"env"` - ProcMacro bool `json:"is_proc_macro"` + DisplayName string `json:"display_name"` + RootModule string `json:"root_module"` + Edition string `json:"edition,omitempty"` + Deps []rustProjectDep `json:"deps"` + Cfg []string `json:"cfg"` + Env map[string]string `json:"env"` + ProcMacro bool `json:"is_proc_macro"` + ProcMacroDylib *string `json:"proc_macro_dylib_path"` } type rustProjectJson struct { - Crates []rustProjectCrate `json:"crates"` + Sysroot string `json:"sysroot"` + Crates []rustProjectCrate `json:"crates"` } // crateInfo is used during the processing to keep track of the known crates. @@ -135,16 +138,21 @@ func (singleton *projectGeneratorSingleton) addCrate(ctx android.SingletonContex return 0, false } - _, procMacro := rModule.compiler.(*procMacroDecorator) + var procMacroDylib *string = nil + if procDec, procMacro := rModule.compiler.(*procMacroDecorator); procMacro { + procMacroDylib = new(string) + *procMacroDylib = procDec.baseCompiler.unstrippedOutputFilePath().String() + } crate := rustProjectCrate{ - DisplayName: rModule.Name(), - RootModule: rootModule.String(), - Edition: rModule.compiler.edition(), - Deps: make([]rustProjectDep, 0), - Cfg: make([]string, 0), - Env: make(map[string]string), - ProcMacro: procMacro, + DisplayName: rModule.Name(), + RootModule: rootModule.String(), + Edition: rModule.compiler.edition(), + Deps: make([]rustProjectDep, 0), + Cfg: make([]string, 0), + Env: make(map[string]string), + ProcMacro: procMacroDylib != nil, + ProcMacroDylib: procMacroDylib, } if rModule.compiler.cargoOutDir().Valid() { @@ -197,6 +205,8 @@ func (singleton *projectGeneratorSingleton) GenerateBuildActions(ctx android.Sin return } + singleton.project.Sysroot = config.RustPath(ctx) + singleton.knownCrates = make(map[string]crateInfo) ctx.VisitAllModules(func(module android.Module) { singleton.appendCrateAndDependencies(ctx, module) diff --git a/rust/rust.go b/rust/rust.go index 0b26f92a4..ed38ad7a7 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -977,6 +977,7 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { // side dependencies. In particular, proc-macros need to be captured in the // host snapshot. mod.HideFromMake() + mod.SkipInstall() } else if !mod.installable(apexInfo) { mod.SkipInstall() } diff --git a/rust/rust_test.go b/rust/rust_test.go index eeedf3f57..767508d5f 100644 --- a/rust/rust_test.go +++ b/rust/rust_test.go @@ -494,7 +494,7 @@ func TestRustRlibs(t *testing.T) { } // Make sure the static lib is included in the ld command - if !strings.Contains(libcc_shared_ld.Args["libFlags"], "generated_rust_staticlib/liblibcc_shared_rust_staticlib.a") { + if !strings.Contains(libcc_shared_ld.Args["libFlags"], "generated_rust_staticlib/librustlibs.a") { t.Errorf("missing generated static library in linker step libFlags %#v, libFlags: %#v", "libcc_shared.generated_rust_staticlib.a", libcc_shared_ld.Args["libFlags"]) } @@ -511,7 +511,7 @@ func TestRustRlibs(t *testing.T) { } // Make sure the static lib is included in the cc command - if !strings.Contains(ccbin_ld.Args["libFlags"], "generated_rust_staticlib/libccBin_rust_staticlib.a") { + if !strings.Contains(ccbin_ld.Args["libFlags"], "generated_rust_staticlib/librustlibs.a") { t.Errorf("missing generated static library in linker step libFlags, expecting %#v, libFlags: %#v", "ccBin.generated_rust_staticlib.a", ccbin_ld.Args["libFlags"]) } diff --git a/scripts/gen_build_prop.py b/scripts/gen_build_prop.py index e0686ed19..0b7780e96 100644 --- a/scripts/gen_build_prop.py +++ b/scripts/gen_build_prop.py @@ -608,6 +608,8 @@ def main(): build_product_prop(args) case "vendor": build_vendor_prop(args) + case "system_dlkm" | "vendor_dlkm" | "odm_dlkm": + build_prop(args, gen_build_info=False, gen_common_build_props=True, variables=[]) case _: sys.exit(f"not supported partition {args.partition}") diff --git a/sdk/Android.bp b/sdk/Android.bp index f436320df..f42b4787d 100644 --- a/sdk/Android.bp +++ b/sdk/Android.bp @@ -18,7 +18,6 @@ bootstrap_go_package { "bp.go", "build_release.go", "exports.go", - "genrule.go", "member_trait.go", "member_type.go", "sdk.go", @@ -31,7 +30,6 @@ bootstrap_go_package { "cc_sdk_test.go", "compat_config_sdk_test.go", "exports_test.go", - "genrule_test.go", "java_sdk_test.go", "license_sdk_test.go", "member_trait_test.go", diff --git a/sdk/genrule.go b/sdk/genrule.go deleted file mode 100644 index 347ab0556..000000000 --- a/sdk/genrule.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2023 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package sdk - -import ( - "android/soong/android" - "android/soong/genrule" -) - -func init() { - registerGenRuleBuildComponents(android.InitRegistrationContext) -} - -func registerGenRuleBuildComponents(ctx android.RegistrationContext) { - ctx.RegisterModuleType("sdk_genrule", SdkGenruleFactory) -} - -// sdk_genrule_host is a genrule that can depend on sdk and sdk_snapshot module types -// -// What this means is that it's a genrule with only the "common_os" variant. -// sdk modules have 3 variants: host, android, and common_os. The common_os one depends -// on the host/device ones and packages their result into a final snapshot zip. -// Genrules probably want access to this snapshot zip when they depend on an sdk module, -// which means they want to depend on the common_os variant and not the host/android -// variants. -func SdkGenruleFactory() android.Module { - module := genrule.NewGenRule() - - android.InitCommonOSAndroidMultiTargetsArchModule(module, android.NeitherHostNorDeviceSupported, android.MultilibCommon) - android.InitDefaultableModule(module) - - return module -} diff --git a/sdk/genrule_test.go b/sdk/genrule_test.go deleted file mode 100644 index bf67795a7..000000000 --- a/sdk/genrule_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2018 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sdk - -import ( - "testing" - - "android/soong/android" - "android/soong/genrule" - "android/soong/java" -) - -func TestSdkGenrule(t *testing.T) { - // Test that a genrule can depend on an sdk if using common_os_srcs - bp := ` - sdk { - name: "my_sdk", - } - genrule { - name: "my_regular_genrule", - common_os_srcs: [":my_sdk"], - out: ["out"], - cmd: "cp $(in) $(out)", - } - ` - android.GroupFixturePreparers( - // if java components aren't registered, the sdk module doesn't create a snapshot for some reason. - java.PrepareForTestWithJavaBuildComponents, - genrule.PrepareForTestWithGenRuleBuildComponents, - PrepareForTestWithSdkBuildComponents, - android.FixtureRegisterWithContext(registerGenRuleBuildComponents), - ).RunTestWithBp(t, bp) -} diff --git a/sh/sh_binary.go b/sh/sh_binary.go index 9c0db73ba..853f3d368 100644 --- a/sh/sh_binary.go +++ b/sh/sh_binary.go @@ -45,6 +45,7 @@ func registerShBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("sh_binary_host", ShBinaryHostFactory) ctx.RegisterModuleType("sh_test", ShTestFactory) ctx.RegisterModuleType("sh_test_host", ShTestHostFactory) + ctx.RegisterModuleType("sh_defaults", ShDefaultsFactory) } // Test fixture preparer that will register most sh build components. @@ -167,6 +168,7 @@ type TestProperties struct { type ShBinary struct { android.ModuleBase + android.DefaultableModuleBase properties shBinaryProperties @@ -548,6 +550,7 @@ func ShBinaryFactory() android.Module { module := &ShBinary{} initShBinaryModule(module) android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibFirst) + android.InitDefaultableModule(module) return module } @@ -557,6 +560,7 @@ func ShBinaryHostFactory() android.Module { module := &ShBinary{} initShBinaryModule(module) android.InitAndroidArchModule(module, android.HostSupported, android.MultilibFirst) + android.InitDefaultableModule(module) return module } @@ -567,6 +571,7 @@ func ShTestFactory() android.Module { module.AddProperties(&module.testProperties) android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibFirst) + android.InitDefaultableModule(module) return module } @@ -581,6 +586,21 @@ func ShTestHostFactory() android.Module { } android.InitAndroidArchModule(module, android.HostSupported, android.MultilibFirst) + android.InitDefaultableModule(module) + return module +} + +type ShDefaults struct { + android.ModuleBase + android.DefaultsModuleBase +} + +func ShDefaultsFactory() android.Module { + module := &ShDefaults{} + + module.AddProperties(&shBinaryProperties{}, &TestProperties{}) + android.InitDefaultsModule(module) + return module } diff --git a/sh/sh_binary_test.go b/sh/sh_binary_test.go index 37450b0e2..5a5043946 100644 --- a/sh/sh_binary_test.go +++ b/sh/sh_binary_test.go @@ -294,3 +294,92 @@ func TestShTestHost_javaData(t *testing.T) { actualData := entries.EntryMap["LOCAL_TEST_DATA"] android.AssertStringPathsRelativeToTopEquals(t, "LOCAL_TEST_DATA", config, expectedData, actualData) } + +func TestDefaultsForTests(t *testing.T) { + ctx, config := testShBinary(t, ` + sh_defaults { + name: "defaults", + src: "test.sh", + filename: "test.sh", + data: [ + "testdata/data1", + "testdata/sub/data2", + ], + } + sh_test_host { + name: "foo", + defaults: ["defaults"], + data: [ + "testdata/more_data", + ], + java_data: [ + "javalib", + ], + } + + java_library_host { + name: "javalib", + srcs: [], + } + + sh_test { + name: "sh-test", + defaults: ["defaults"], + } + + `) + buildOS := ctx.Config().BuildOS.String() + mod := ctx.ModuleForTests("foo", buildOS+"_x86_64").Module().(*ShTest) + if !mod.Host() { + t.Errorf("host bit is not set for a sh_test_host module.") + } + expectedData := []string{ + ":testdata/data1", + ":testdata/sub/data2", + ":testdata/more_data", + "out/soong/.intermediates/javalib/" + buildOS + "_common/combined/:javalib.jar", + } + + entries := android.AndroidMkEntriesForTest(t, ctx, mod)[0] + actualData := entries.EntryMap["LOCAL_TEST_DATA"] + android.AssertStringPathsRelativeToTopEquals(t, "LOCAL_TEST_DATA", config, expectedData, actualData) + + // Just the defaults + expectedData = []string{ + ":testdata/data1", + ":testdata/sub/data2", + } + mod = ctx.ModuleForTests("sh-test", "android_arm64_armv8-a").Module().(*ShTest) + entries = android.AndroidMkEntriesForTest(t, ctx, mod)[0] + actualData = entries.EntryMap["LOCAL_TEST_DATA"] + android.AssertStringPathsRelativeToTopEquals(t, "LOCAL_TEST_DATA", config, expectedData, actualData) +} + +func TestDefaultsForBinaries(t *testing.T) { + ctx, _ := testShBinary(t, ` + sh_defaults { + name: "defaults", + src: "test.sh", + filename: "test.sh", + } + sh_binary_host { + name: "the-host-binary", + defaults: ["defaults"], + } + sh_binary{ + name: "the-binary", + defaults: ["defaults"], + } + `) + buildOS := ctx.Config().BuildOS.String() + mod := ctx.ModuleForTests("the-host-binary", buildOS+"_x86_64").Module().(*ShBinary) + if !mod.Host() { + t.Errorf("host bit is not set for a sh_binary_host module.") + } + + expectedFilename := "test.sh" + android.AssertStringEquals(t, "Filename", expectedFilename, *mod.properties.Filename) + + mod = ctx.ModuleForTests("the-binary", "android_arm64_armv8-a").Module().(*ShBinary) + android.AssertStringEquals(t, "Filename", expectedFilename, *mod.properties.Filename) +} diff --git a/tests/lib.sh b/tests/lib.sh index 4c320d08b..0e26de55f 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -91,7 +91,6 @@ function symlink_directory { } function create_mock_soong { - create_mock_bazel copy_directory build/blueprint copy_directory build/soong copy_directory build/make @@ -143,41 +142,6 @@ function run_soong { USE_RBE=false TARGET_PRODUCT=aosp_arm TARGET_RELEASE=trunk_staging TARGET_BUILD_VARIANT=userdebug build/soong/soong_ui.bash --make-mode --skip-ninja --skip-config --soong-only --skip-soong-tests "$@" } -function create_mock_bazel { - copy_directory build/bazel - copy_directory build/bazel_common_rules - - # This requires pulling more tools into the mock top to build partitions - delete_directory build/bazel/examples/partitions - - symlink_directory packages/modules/common/build - symlink_directory prebuilts/bazel - symlink_directory prebuilts/clang - symlink_directory prebuilts/jdk - symlink_directory external/bazel-skylib - symlink_directory external/bazelbuild-rules_android - symlink_directory external/bazelbuild-rules_go - symlink_directory external/bazelbuild-rules_license - symlink_directory external/bazelbuild-kotlin-rules - symlink_directory external/bazelbuild-rules_cc - symlink_directory external/bazelbuild-rules_python - symlink_directory external/bazelbuild-rules_java - symlink_directory external/bazelbuild-rules_rust - symlink_directory external/bazelbuild-rules_testing - symlink_directory external/rust/crates/tinyjson - - symlink_file WORKSPACE - symlink_file BUILD -} - -function run_bazel { - # Remove the ninja_build output marker file to communicate to buildbot that this is not a regular Ninja build, and its - # output should not be parsed as such. - rm -rf out/ninja_build - - build/bazel/bin/bazel "$@" -} - function run_ninja { build/soong/soong_ui.bash --make-mode --skip-config --soong-only --skip-soong-tests "$@" } diff --git a/tradefed_modules/Android.bp b/tradefed_modules/Android.bp index 9969ae280..67d91b27c 100644 --- a/tradefed_modules/Android.bp +++ b/tradefed_modules/Android.bp @@ -13,9 +13,11 @@ bootstrap_go_package { ], srcs: [ "test_module_config.go", + "test_suite.go", ], testSrcs: [ "test_module_config_test.go", + "test_suite_test.go", ], pluginFor: ["soong_build"], } diff --git a/tradefed_modules/test_module_config.go b/tradefed_modules/test_module_config.go index 7a04c1994..5c13d64e8 100644 --- a/tradefed_modules/test_module_config.go +++ b/tradefed_modules/test_module_config.go @@ -383,6 +383,19 @@ func (m *testModuleConfigModule) generateManifestAndConfig(ctx android.ModuleCon // 4) Module.config / AndroidTest.xml m.testConfig = m.fixTestConfig(ctx, m.provider.TestConfig) + + // 5) We provide so we can be listed in test_suites. + android.SetProvider(ctx, tradefed.BaseTestProviderKey, tradefed.BaseTestProviderData{ + InstalledFiles: m.supportFiles.Paths(), + OutputFile: baseApk, + TestConfig: m.testConfig, + HostRequiredModuleNames: m.provider.HostRequiredModuleNames, + RequiredModuleNames: m.provider.RequiredModuleNames, + TestSuites: m.tradefedProperties.Test_suites, + IsHost: m.provider.IsHost, + LocalCertificate: m.provider.LocalCertificate, + IsUnitTest: m.provider.IsUnitTest, + }) } var _ android.AndroidMkEntriesProvider = (*testModuleConfigHostModule)(nil) diff --git a/tradefed_modules/test_suite.go b/tradefed_modules/test_suite.go new file mode 100644 index 000000000..00585f5f6 --- /dev/null +++ b/tradefed_modules/test_suite.go @@ -0,0 +1,173 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tradefed_modules + +import ( + "encoding/json" + "path" + "path/filepath" + + "android/soong/android" + "android/soong/tradefed" + "github.com/google/blueprint" +) + +const testSuiteModuleType = "test_suite" + +type testSuiteTag struct{ + blueprint.BaseDependencyTag +} + +type testSuiteManifest struct { + Name string `json:"name"` + Files []string `json:"files"` +} + +func init() { + RegisterTestSuiteBuildComponents(android.InitRegistrationContext) +} + +func RegisterTestSuiteBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType(testSuiteModuleType, TestSuiteFactory) +} + +var PrepareForTestWithTestSuiteBuildComponents = android.GroupFixturePreparers( + android.FixtureRegisterWithContext(RegisterTestSuiteBuildComponents), +) + +type testSuiteProperties struct { + Description string + Tests []string `android:"path,arch_variant"` +} + +type testSuiteModule struct { + android.ModuleBase + android.DefaultableModuleBase + testSuiteProperties +} + +func (t *testSuiteModule) DepsMutator(ctx android.BottomUpMutatorContext) { + for _, test := range t.Tests { + if ctx.OtherModuleDependencyVariantExists(ctx.Config().BuildOSCommonTarget.Variations(), test) { + // Host tests. + ctx.AddVariationDependencies(ctx.Config().BuildOSCommonTarget.Variations(), testSuiteTag{}, test) + } else { + // Target tests. + ctx.AddDependency(ctx.Module(), testSuiteTag{}, test) + } + } +} + +func (t *testSuiteModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + suiteName := ctx.ModuleName() + modulesByName := make(map[string]android.Module) + ctx.WalkDeps(func(child, parent android.Module) bool { + // Recurse into test_suite dependencies. + if ctx.OtherModuleType(child) == testSuiteModuleType { + ctx.Phony(suiteName, android.PathForPhony(ctx, child.Name())) + return true + } + + // Only write out top level test suite dependencies here. + if _, ok := ctx.OtherModuleDependencyTag(child).(testSuiteTag); !ok { + return false + } + + if !child.InstallInTestcases() { + ctx.ModuleErrorf("test_suite only supports modules installed in testcases. %q is not installed in testcases.", child.Name()) + return false + } + + modulesByName[child.Name()] = child + return false + }) + + var files []string + for name, module := range modulesByName { + // Get the test provider data from the child. + tp, ok := android.OtherModuleProvider(ctx, module, tradefed.BaseTestProviderKey) + if !ok { + // TODO: Consider printing out a list of all module types. + ctx.ModuleErrorf("%q is not a test module.", name) + continue + } + + files = append(files, packageModuleFiles(ctx, suiteName, module, tp)...) + ctx.Phony(suiteName, android.PathForPhony(ctx, name)) + } + + manifestPath := android.PathForSuiteInstall(ctx, suiteName, suiteName+".json") + b, err := json.Marshal(testSuiteManifest{Name: suiteName, Files: files}) + if err != nil { + ctx.ModuleErrorf("Failed to marshal manifest: %v", err) + return + } + android.WriteFileRule(ctx, manifestPath, string(b)) + + ctx.Phony(suiteName, manifestPath) +} + +func TestSuiteFactory() android.Module { + module := &testSuiteModule{} + module.AddProperties(&module.testSuiteProperties) + + android.InitAndroidArchModule(module, android.HostAndDeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + + return module +} + +func packageModuleFiles(ctx android.ModuleContext, suiteName string, module android.Module, tp tradefed.BaseTestProviderData) []string { + + hostOrTarget := "target" + if tp.IsHost { + hostOrTarget = "host" + } + + // suiteRoot at out/soong/packaging/<suiteName>. + suiteRoot := android.PathForSuiteInstall(ctx, suiteName) + + var installed android.InstallPaths + // Install links to installed files from the module. + if installFilesInfo, ok := android.OtherModuleProvider(ctx, module, android.InstallFilesProvider); ok { + for _, f := range installFilesInfo.InstallFiles { + // rel is anything under .../<partition>, normally under .../testcases. + rel := android.Rel(ctx, f.PartitionDir(), f.String()) + + // Install the file under <suiteRoot>/<host|target>/<partition>. + installDir := suiteRoot.Join(ctx, hostOrTarget, f.Partition(), path.Dir(rel)) + linkTo, err := filepath.Rel(installDir.String(), f.String()) + if err != nil { + ctx.ModuleErrorf("Failed to get relative path from %s to %s: %v", installDir.String(), f.String(), err) + continue + } + installed = append(installed, ctx.InstallAbsoluteSymlink(installDir, path.Base(rel), linkTo)) + } + } + + // Install config file. + if tp.TestConfig != nil { + moduleRoot := suiteRoot.Join(ctx, hostOrTarget, "testcases", module.Name()) + installed = append(installed, ctx.InstallFile(moduleRoot, module.Name() + ".config", tp.TestConfig)) + } + + // Add to phony and manifest, manifestpaths are relative to suiteRoot. + var manifestEntries []string + for _, f := range installed { + manifestEntries = append(manifestEntries, android.Rel(ctx, suiteRoot.String(), f.String())) + ctx.Phony(suiteName, f) + } + return manifestEntries +} diff --git a/tradefed_modules/test_suite_test.go b/tradefed_modules/test_suite_test.go new file mode 100644 index 000000000..3c0a9eb2c --- /dev/null +++ b/tradefed_modules/test_suite_test.go @@ -0,0 +1,151 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package tradefed_modules + +import ( + "android/soong/android" + "android/soong/java" + "encoding/json" + "slices" + "testing" +) + +func TestTestSuites(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + java.PrepareForTestWithJavaDefaultModules, + android.FixtureRegisterWithContext(RegisterTestSuiteBuildComponents), + ).RunTestWithBp(t, ` + android_test { + name: "TestModule1", + sdk_version: "current", + } + + android_test { + name: "TestModule2", + sdk_version: "current", + } + + test_suite { + name: "my-suite", + description: "a test suite", + tests: [ + "TestModule1", + "TestModule2", + ] + } + `) + manifestPath := ctx.ModuleForTests("my-suite", "android_common").Output("out/soong/test_suites/my-suite/my-suite.json") + var actual testSuiteManifest + if err := json.Unmarshal([]byte(android.ContentFromFileRuleForTests(t, ctx.TestContext, manifestPath)), &actual); err != nil { + t.Errorf("failed to unmarshal manifest: %v", err) + } + slices.Sort(actual.Files) + + expected := testSuiteManifest{ + Name: "my-suite", + Files: []string{ + "target/testcases/TestModule1/TestModule1.config", + "target/testcases/TestModule1/arm64/TestModule1.apk", + "target/testcases/TestModule2/TestModule2.config", + "target/testcases/TestModule2/arm64/TestModule2.apk", + }, + } + + android.AssertDeepEquals(t, "manifests differ", expected, actual) +} + +func TestTestSuitesWithNested(t *testing.T) { + t.Parallel() + ctx := android.GroupFixturePreparers( + java.PrepareForTestWithJavaDefaultModules, + android.FixtureRegisterWithContext(RegisterTestSuiteBuildComponents), + ).RunTestWithBp(t, ` + android_test { + name: "TestModule1", + sdk_version: "current", + } + + android_test { + name: "TestModule2", + sdk_version: "current", + } + + android_test { + name: "TestModule3", + sdk_version: "current", + } + + test_suite { + name: "my-child-suite", + description: "a child test suite", + tests: [ + "TestModule1", + "TestModule2", + ] + } + + test_suite { + name: "my-all-tests-suite", + description: "a parent test suite", + tests: [ + "TestModule1", + "TestModule3", + "my-child-suite", + ] + } + `) + manifestPath := ctx.ModuleForTests("my-all-tests-suite", "android_common").Output("out/soong/test_suites/my-all-tests-suite/my-all-tests-suite.json") + var actual testSuiteManifest + if err := json.Unmarshal([]byte(android.ContentFromFileRuleForTests(t, ctx.TestContext, manifestPath)), &actual); err != nil { + t.Errorf("failed to unmarshal manifest: %v", err) + } + slices.Sort(actual.Files) + + expected := testSuiteManifest{ + Name: "my-all-tests-suite", + Files: []string{ + "target/testcases/TestModule1/TestModule1.config", + "target/testcases/TestModule1/arm64/TestModule1.apk", + "target/testcases/TestModule2/TestModule2.config", + "target/testcases/TestModule2/arm64/TestModule2.apk", + "target/testcases/TestModule3/TestModule3.config", + "target/testcases/TestModule3/arm64/TestModule3.apk", + }, + } + + android.AssertDeepEquals(t, "manifests differ", expected, actual) +} + +func TestTestSuitesNotInstalledInTestcases(t *testing.T) { + t.Parallel() + android.GroupFixturePreparers( + java.PrepareForTestWithJavaDefaultModules, + android.FixtureRegisterWithContext(RegisterTestSuiteBuildComponents), + ).ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern([]string{ + `"SomeHostTest" is not installed in testcases`, + })).RunTestWithBp(t, ` + java_test_host { + name: "SomeHostTest", + srcs: ["a.java"], + } + test_suite { + name: "my-suite", + description: "a test suite", + tests: [ + "SomeHostTest", + ] + } + `) +} |