diff options
110 files changed, 2883 insertions, 1109 deletions
diff --git a/Android.bp b/Android.bp index 42b7d83f7..434ee9f96 100644 --- a/Android.bp +++ b/Android.bp @@ -172,9 +172,13 @@ build_prop { name: "system-build.prop", stem: "build.prop", product_config: ":product_config", + footer_files: [ + ":applied_backported_fixes", + ], // Currently, only microdroid, Ravenwood, and cf system image can refer to system-build.prop visibility: [ "//build/make/target/product/generic", + "//build/make/target/product/gsi", "//packages/modules/Virtualization/build/microdroid", "//frameworks/base/ravenwood", ], @@ -186,7 +190,7 @@ build_prop { system_ext_specific: true, product_config: ":product_config", relative_install_path: "etc", // system_ext/etc/build.prop - visibility: ["//visibility:private"], + visibility: ["//build/make/target/product/gsi"], } build_prop { @@ -195,7 +199,7 @@ build_prop { product_specific: true, product_config: ":product_config", relative_install_path: "etc", // product/etc/build.prop - visibility: ["//visibility:private"], + visibility: ["//build/make/target/product/gsi"], } build_prop { @@ -233,3 +237,12 @@ build_prop { relative_install_path: "etc", // odm_dlkm/etc/build.prop visibility: ["//visibility:private"], } + +build_prop { + name: "ramdisk-build.prop", + stem: "build.prop", + ramdisk: true, + product_config: ":product_config", + relative_install_path: "etc/ramdisk", // ramdisk/system/etc/ramdisk/build.prop + visibility: ["//visibility:private"], +} diff --git a/aconfig/build_flags/build_flags.go b/aconfig/build_flags/build_flags.go index e878b5aa6..94e1eb193 100644 --- a/aconfig/build_flags/build_flags.go +++ b/aconfig/build_flags/build_flags.go @@ -35,7 +35,7 @@ func registerBuildFlagsModuleType(ctx android.RegistrationContext) { type buildFlags struct { android.ModuleBase - outputPath android.OutputPath + outputPath android.Path } func buildFlagsFactory() android.Module { @@ -48,7 +48,7 @@ func (m *buildFlags) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Read the build_flags_<partition>.json file generated by soong // 'release-config' command. srcPath := android.PathForOutput(ctx, "release-config", fmt.Sprintf("build_flags_%s.json", m.PartitionTag(ctx.DeviceConfig()))) - m.outputPath = android.PathForModuleOut(ctx, outJsonFileName).OutputPath + outputPath := android.PathForModuleOut(ctx, outJsonFileName) // The 'release-config' command is called for every build, and generates the // build_flags_<partition>.json file. @@ -56,11 +56,12 @@ func (m *buildFlags) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.Build(pctx, android.BuildParams{ Rule: android.CpIfChanged, Input: srcPath, - Output: m.outputPath, + Output: outputPath, }) installPath := android.PathForModuleInstall(ctx, "etc") - ctx.InstallFile(installPath, outJsonFileName, m.outputPath) + ctx.InstallFile(installPath, outJsonFileName, outputPath) + m.outputPath = outputPath } func (m *buildFlags) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/android/aconfig_providers.go b/android/aconfig_providers.go index b902f8be5..210a65638 100644 --- a/android/aconfig_providers.go +++ b/android/aconfig_providers.go @@ -107,7 +107,7 @@ func aconfigUpdateAndroidBuildActions(ctx ModuleContext) { mergedAconfigFiles := make(map[string]Paths) mergedModeInfos := make(map[string]ModeInfo) - ctx.VisitDirectDeps(func(module Module) { + ctx.VisitDirectDepsProxy(func(module ModuleProxy) { if aconfig_dep, ok := OtherModuleProvider(ctx, module, CodegenInfoProvider); ok && len(aconfig_dep.ModeInfos) > 0 { maps.Copy(mergedModeInfos, aconfig_dep.ModeInfos) } diff --git a/android/apex.go b/android/apex.go index e73b3e662..9277ff31b 100644 --- a/android/apex.go +++ b/android/apex.go @@ -160,14 +160,6 @@ func (i ApexInfo) Equal(other any) bool { reflect.DeepEqual(i.InApexModules, otherApexInfo.InApexModules) } -// ApexTestForInfo stores the contents of APEXes for which this module is a test - although this -// module is not part of the APEX - and thus has access to APEX internals. -type ApexTestForInfo struct { - ApexContents []*ApexContents -} - -var ApexTestForInfoProvider = blueprint.NewMutatorProvider[ApexTestForInfo]("apex_test_for") - // ApexBundleInfo contains information about the dependencies of an apex type ApexBundleInfo struct { Contents *ApexContents @@ -269,12 +261,6 @@ type ApexModule interface { // check-platform-availability mutator in the apex package. SetNotAvailableForPlatform() - // Returns the list of APEXes that this module is a test for. The module has access to the - // private part of the listed APEXes even when it is not included in the APEXes. This by - // default returns nil. A module type should override the default implementation. For - // example, cc_test module type returns the value of test_for here. - TestFor() []string - // Returns nil (success) if this module should support the given sdk version. Returns an // error if not. No default implementation is provided for this method. A module type // implementing this interface should provide an implementation. A module supports an sdk @@ -457,13 +443,6 @@ func (m *ApexModuleBase) IsInstallableToApex() bool { return false } -// Implements ApexModule -func (m *ApexModuleBase) TestFor() []string { - // If needed, this will be overridden by concrete types inheriting - // ApexModuleBase - return nil -} - // Returns the test apexes that this module is included in. func (m *ApexModuleBase) TestApexes() []string { return m.ApexProperties.TestApexes @@ -891,28 +870,6 @@ func (i ApexMembership) merge(other ApexMembership) ApexMembership { return notInApex } -// Tests whether a module named moduleName is directly included in the apexBundle where this -// ApexContents is tagged. -func (ac *ApexContents) DirectlyInApex(moduleName string) bool { - return ac.contents[moduleName] == directlyInApex -} - -// Tests whether a module named moduleName is included in the apexBundle where this ApexContent is -// tagged. -func (ac *ApexContents) InApex(moduleName string) bool { - return ac.contents[moduleName] != notInApex -} - -// Tests whether a module named moduleName is directly depended on by all APEXes in an ApexInfo. -func DirectlyInAllApexes(apexInfo ApexInfo, moduleName string) bool { - for _, contents := range apexInfo.ApexContents { - if !contents.DirectlyInApex(moduleName) { - return false - } - } - return true -} - //////////////////////////////////////////////////////////////////////////////////////////////////// //Below are routines for extra safety checks. // @@ -939,8 +896,8 @@ type ApexModuleDepInfo struct { type DepNameToDepInfoMap map[string]ApexModuleDepInfo type ApexBundleDepsInfo struct { - flatListPath OutputPath - fullListPath OutputPath + flatListPath Path + fullListPath Path } type ApexBundleDepsInfoIntf interface { @@ -977,13 +934,15 @@ func (d *ApexBundleDepsInfo) BuildDepsInfoLists(ctx ModuleContext, minSdkVersion fmt.Fprintf(&flatContent, "%s\n", toName) } - d.fullListPath = PathForModuleOut(ctx, "depsinfo", "fulllist.txt").OutputPath - WriteFileRule(ctx, d.fullListPath, fullContent.String()) + fullListPath := PathForModuleOut(ctx, "depsinfo", "fulllist.txt") + WriteFileRule(ctx, fullListPath, fullContent.String()) + d.fullListPath = fullListPath - d.flatListPath = PathForModuleOut(ctx, "depsinfo", "flatlist.txt").OutputPath - WriteFileRule(ctx, d.flatListPath, flatContent.String()) + flatListPath := PathForModuleOut(ctx, "depsinfo", "flatlist.txt") + WriteFileRule(ctx, flatListPath, flatContent.String()) + d.flatListPath = flatListPath - ctx.Phony(fmt.Sprintf("%s-depsinfo", ctx.ModuleName()), d.fullListPath, d.flatListPath) + ctx.Phony(fmt.Sprintf("%s-depsinfo", ctx.ModuleName()), fullListPath, flatListPath) } // Function called while walking an APEX's payload dependencies. @@ -1062,12 +1021,6 @@ func MinSdkVersionFromValue(ctx EarlyModuleContext, value string) ApiLevel { return apiLevel } -// Implemented by apexBundle. -type ApexTestInterface interface { - // Return true if the apex bundle is an apex_test - IsTestApex() bool -} - var ApexExportsInfoProvider = blueprint.NewProvider[ApexExportsInfo]() // ApexExportsInfo contains information about the artifacts provided by apexes to dexpreopt and hiddenapi diff --git a/android/base_module_context.go b/android/base_module_context.go index 060fae5bc..1f89deaed 100644 --- a/android/base_module_context.go +++ b/android/base_module_context.go @@ -113,7 +113,7 @@ type BaseModuleContext interface { // GetDirectDepWithTag returns the Module the direct dependency with the specified name, or nil if // none exists. It panics if the dependency does not have the specified tag. It skips any // dependencies that are not an android.Module. - GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module + GetDirectDepWithTag(name string, tag blueprint.DependencyTag) Module // GetDirectDep returns the Module and DependencyTag for the direct dependency with the specified // name, or nil if none exists. If there are multiple dependencies on the same module it returns @@ -129,13 +129,14 @@ type BaseModuleContext interface { // function, it may be invalidated by future mutators. VisitDirectDeps(visit func(Module)) - // VisitDirectDeps calls visit for each direct dependency. If there are multiple + // VisitDirectDepsProxy calls visit for each direct dependency. If there are multiple // direct dependencies on the same module visit will be called multiple times on that module - // and OtherModuleDependencyTag will return a different tag for each. + // and OtherModuleDependencyTag will return a different tag for each. It raises an error if any of the + // dependencies are disabled. // - // The Module passed to the visit function should not be retained outside of the visit + // The ModuleProxy passed to the visit function should not be retained outside of the visit // function, it may be invalidated by future mutators. - VisitDirectDepsAllowDisabled(visit func(Module)) + VisitDirectDepsProxy(visit func(proxy ModuleProxy)) // VisitDirectDepsProxyAllowDisabled calls visit for each direct dependency. If there are // multiple direct dependencies on the same module visit will be called multiple times on @@ -210,6 +211,12 @@ type BaseModuleContext interface { // data modified by the current mutator. VisitAllModuleVariants(visit func(Module)) + // VisitAllModuleVariantProxies calls visit for each variant of the current module. Variants of a module are always + // visited in order by mutators and GenerateBuildActions, so the data created by the current mutator can be read + // from all variants if the current module is the last one. Otherwise, care must be taken to not access any + // data modified by the current mutator. + VisitAllModuleVariantProxies(visit func(proxy ModuleProxy)) + // GetTagPath is supposed to be called in visit function passed in WalkDeps() // and returns a top-down dependency tags path from a start module to current child module. // It has one less entry than GetWalkPath() as it contains the dependency tags that @@ -261,9 +268,11 @@ func (b *baseModuleContext) EqualModules(m1, m2 Module) bool { func (b *baseModuleContext) OtherModuleName(m blueprint.Module) string { return b.bp.OtherModuleName(getWrappedModule(m)) } -func (b *baseModuleContext) OtherModuleDir(m blueprint.Module) string { return b.bp.OtherModuleDir(m) } +func (b *baseModuleContext) OtherModuleDir(m blueprint.Module) string { + return b.bp.OtherModuleDir(getWrappedModule(m)) +} func (b *baseModuleContext) OtherModuleErrorf(m blueprint.Module, fmt string, args ...interface{}) { - b.bp.OtherModuleErrorf(m, fmt, args...) + b.bp.OtherModuleErrorf(getWrappedModule(m), fmt, args...) } func (b *baseModuleContext) OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag { return b.bp.OtherModuleDependencyTag(getWrappedModule(m)) @@ -298,8 +307,11 @@ func (b *baseModuleContext) setProvider(provider blueprint.AnyProviderKey, value b.bp.SetProvider(provider, value) } -func (b *baseModuleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module { - return b.bp.GetDirectDepWithTag(name, tag) +func (b *baseModuleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) Module { + if module := b.bp.GetDirectDepWithTag(name, tag); module != nil { + return module.(Module) + } + return nil } func (b *baseModuleContext) blueprintBaseModuleContext() blueprint.BaseModuleContext { @@ -376,7 +388,7 @@ func (b *baseModuleContext) validateAndroidModuleProxy( return &aModule } - if !OtherModuleProviderOrDefault(b, module, CommonPropertiesProviderKey).Enabled { + if !OtherModuleProviderOrDefault(b, module, CommonModuleInfoKey).Enabled { if t, ok := tag.(AllowDisabledModuleDependency); !ok || !t.AllowDisabledModuleDependencyProxy(b, aModule) { if b.Config().AllowMissingDependencies() { b.AddMissingDependencies([]string{b.OtherModuleName(aModule)}) @@ -464,18 +476,16 @@ func (b *baseModuleContext) VisitDirectDeps(visit func(Module)) { }) } -func (b *baseModuleContext) VisitDirectDepsAllowDisabled(visit func(Module)) { - b.bp.VisitDirectDeps(func(module blueprint.Module) { - visit(module.(Module)) +func (b *baseModuleContext) VisitDirectDepsProxy(visit func(ModuleProxy)) { + b.bp.VisitDirectDepsProxy(func(module blueprint.ModuleProxy) { + if aModule := b.validateAndroidModuleProxy(module, b.bp.OtherModuleDependencyTag(module), b.strictVisitDeps); aModule != nil { + visit(*aModule) + } }) } func (b *baseModuleContext) VisitDirectDepsProxyAllowDisabled(visit func(proxy ModuleProxy)) { - b.bp.VisitDirectDepsProxy(func(module blueprint.ModuleProxy) { - visit(ModuleProxy{ - module: module, - }) - }) + b.bp.VisitDirectDepsProxy(visitProxyAdaptor(visit)) } func (b *baseModuleContext) VisitDirectDepsWithTag(tag blueprint.DependencyTag, visit func(Module)) { @@ -590,6 +600,10 @@ func (b *baseModuleContext) VisitAllModuleVariants(visit func(Module)) { }) } +func (b *baseModuleContext) VisitAllModuleVariantProxies(visit func(ModuleProxy)) { + b.bp.VisitAllModuleVariantProxies(visitProxyAdaptor(visit)) +} + func (b *baseModuleContext) PrimaryModule() Module { return b.bp.PrimaryModule().(Module) } diff --git a/android/build_prop.go b/android/build_prop.go index 838947045..2f71bc03f 100644 --- a/android/build_prop.go +++ b/android/build_prop.go @@ -15,6 +15,8 @@ package android import ( + "fmt" + "github.com/google/blueprint/proptools" ) @@ -55,7 +57,7 @@ type buildPropModule struct { properties buildPropProperties - outputFilePath OutputPath + outputFilePath Path installPath InstallPath } @@ -115,33 +117,22 @@ func (p *buildPropModule) partition(config DeviceConfig) string { return "vendor_dlkm" } else if p.InstallInOdmDlkm() { return "odm_dlkm" + } else if p.InstallInRamdisk() { + // From this hardcoding in make: + // https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/sysprop.mk;l=311;drc=274435657e4682e5cee3fffd11fb301ab32a828d + return "bootimage" } return "system" } -var validPartitions = []string{ - "system", - "system_ext", - "product", - "odm", - "vendor", - "system_dlkm", - "vendor_dlkm", - "odm_dlkm", -} - func (p *buildPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { if !p.SocSpecific() && p.properties.Android_info != nil { ctx.ModuleErrorf("Android_info cannot be set if build.prop is not installed in vendor partition") } - p.outputFilePath = PathForModuleOut(ctx, "build.prop").OutputPath + outputFilePath := PathForModuleOut(ctx, "build.prop") partition := p.partition(ctx.DeviceConfig()) - if !InList(partition, validPartitions) { - ctx.PropertyErrorf("partition", "unsupported partition %q: only %q are supported", partition, validPartitions) - return - } rule := NewRuleBuilder(pctx, ctx) @@ -168,7 +159,7 @@ func (p *buildPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { cmd.FlagWithInput("--product-config=", PathForModuleSrc(ctx, proptools.String(p.properties.Product_config))) cmd.FlagWithArg("--partition=", partition) cmd.FlagForEachInput("--prop-files=", p.propFiles(ctx)) - cmd.FlagWithOutput("--out=", p.outputFilePath) + cmd.FlagWithOutput("--out=", outputFilePath) postProcessCmd := rule.Command().BuiltTool("post_process_props") if ctx.DeviceConfig().BuildBrokenDupSysprop() { @@ -181,17 +172,35 @@ func (p *buildPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { // still need to pass an empty string to kernel-version-file-for-uffd-gc postProcessCmd.FlagWithArg("--kernel-version-file-for-uffd-gc ", `""`) } - postProcessCmd.Text(p.outputFilePath.String()) + postProcessCmd.Text(outputFilePath.String()) postProcessCmd.Flags(p.properties.Block_list) - rule.Command().Text("echo").Text(proptools.NinjaAndShellEscape("# end of file")).FlagWithArg(">> ", p.outputFilePath.String()) + for _, footer := range p.properties.Footer_files { + path := PathForModuleSrc(ctx, footer) + rule.appendText(outputFilePath, "####################################") + rule.appendTextf(outputFilePath, "# Adding footer from %v", footer) + rule.appendTextf(outputFilePath, "# with path %v", path) + rule.appendText(outputFilePath, "####################################") + rule.Command().Text("cat").FlagWithInput("", path).FlagWithArg(">> ", outputFilePath.String()) + } + + rule.appendText(outputFilePath, "# end of file") rule.Build(ctx.ModuleName(), "generating build.prop") p.installPath = PathForModuleInstall(ctx, proptools.String(p.properties.Relative_install_path)) - ctx.InstallFile(p.installPath, p.stem(), p.outputFilePath) + ctx.InstallFile(p.installPath, p.stem(), outputFilePath) + + ctx.SetOutputFiles(Paths{outputFilePath}, "") + p.outputFilePath = outputFilePath +} + +func (r *RuleBuilder) appendText(path ModuleOutPath, text string) { + r.Command().Text("echo").Text(proptools.NinjaAndShellEscape(text)).FlagWithArg(">> ", path.String()) +} - ctx.SetOutputFiles(Paths{p.outputFilePath}, "") +func (r *RuleBuilder) appendTextf(path ModuleOutPath, format string, a ...any) { + r.appendText(path, fmt.Sprintf(format, a...)) } func (p *buildPropModule) AndroidMkEntries() []AndroidMkEntries { diff --git a/android/config.go b/android/config.go index 10bddf7cc..27d3b87ba 100644 --- a/android/config.go +++ b/android/config.go @@ -1834,10 +1834,6 @@ func (c *config) ApexCompressionEnabled() bool { return Bool(c.productVariables.CompressedApex) && !c.UnbundledBuildApps() } -func (c *config) ApexTrimEnabled() bool { - return Bool(c.productVariables.TrimmedApex) -} - func (c *config) UseSoongSystemImage() bool { return Bool(c.productVariables.UseSoongSystemImage) } diff --git a/android/csuite_config.go b/android/csuite_config.go index 20bd03563..26ad6e180 100644 --- a/android/csuite_config.go +++ b/android/csuite_config.go @@ -30,11 +30,11 @@ type csuiteConfigProperties struct { type CSuiteConfig struct { ModuleBase properties csuiteConfigProperties - OutputFilePath OutputPath + OutputFilePath Path } func (me *CSuiteConfig) GenerateAndroidBuildActions(ctx ModuleContext) { - me.OutputFilePath = PathForModuleOut(ctx, me.BaseModuleName()).OutputPath + me.OutputFilePath = PathForModuleOut(ctx, me.BaseModuleName()) } func (me *CSuiteConfig) AndroidMkEntries() []AndroidMkEntries { diff --git a/android/module.go b/android/module.go index 58ae885c8..ce995ad77 100644 --- a/android/module.go +++ b/android/module.go @@ -1382,6 +1382,8 @@ func (m *ModuleBase) PartitionTag(config DeviceConfig) string { if config.SystemExtPath() == "system_ext" { partition = "system_ext" } + } else if m.InstallInRamdisk() { + partition = "ramdisk" } return partition } @@ -1641,25 +1643,27 @@ func (m *ModuleBase) generateVariantTarget(ctx *moduleContext) { func (m *ModuleBase) generateModuleTarget(ctx *moduleContext) { var allInstalledFiles InstallPaths var allCheckbuildTargets Paths - ctx.VisitAllModuleVariants(func(module Module) { - a := module.base() + ctx.VisitAllModuleVariantProxies(func(module ModuleProxy) { var checkbuildTarget Path var uncheckedModule bool - if a == m { + var skipAndroidMkProcessing bool + if ctx.EqualModules(m.module, module) { allInstalledFiles = append(allInstalledFiles, ctx.installFiles...) checkbuildTarget = ctx.checkbuildTarget uncheckedModule = ctx.uncheckedModule + skipAndroidMkProcessing = shouldSkipAndroidMkProcessing(ctx, m) } else { info := OtherModuleProviderOrDefault(ctx, module, InstallFilesProvider) allInstalledFiles = append(allInstalledFiles, info.InstallFiles...) checkbuildTarget = info.CheckbuildTarget uncheckedModule = info.UncheckedModule + skipAndroidMkProcessing = OtherModuleProviderOrDefault(ctx, module, CommonModuleInfoKey).SkipAndroidMkProcessing } // A module's -checkbuild phony targets should // not be created if the module is not exported to make. // Those could depend on the build target and fail to compile // for the current build target. - if (!ctx.Config().KatiEnabled() || !shouldSkipAndroidMkProcessing(ctx, a)) && !uncheckedModule && checkbuildTarget != nil { + if (!ctx.Config().KatiEnabled() || !skipAndroidMkProcessing) && !uncheckedModule && checkbuildTarget != nil { allCheckbuildTargets = append(allCheckbuildTargets, checkbuildTarget) } }) @@ -1832,6 +1836,12 @@ type InstallFilesInfo struct { var InstallFilesProvider = blueprint.NewProvider[InstallFilesInfo]() +type SourceFilesInfo struct { + Srcs Paths +} + +var SourceFilesInfoKey = blueprint.NewProvider[SourceFilesInfo]() + type FinalModuleBuildTargetsInfo struct { // Used by buildTargetSingleton to create checkbuild and per-directory build targets // Only set on the final variant of each module @@ -1842,15 +1852,16 @@ type FinalModuleBuildTargetsInfo struct { var FinalModuleBuildTargetsProvider = blueprint.NewProvider[FinalModuleBuildTargetsInfo]() -type CommonPropertiesProviderData struct { +type CommonModuleInfo struct { Enabled bool // Whether the module has been replaced by a prebuilt ReplacedByPrebuilt bool // The Target of artifacts that this module variant is responsible for creating. - CompileTarget Target + CompileTarget Target + SkipAndroidMkProcessing bool } -var CommonPropertiesProviderKey = blueprint.NewProvider[CommonPropertiesProviderData]() +var CommonModuleInfoKey = blueprint.NewProvider[CommonModuleInfo]() type PrebuiltModuleProviderData struct { // Empty for now @@ -1932,9 +1943,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) if m.Enabled(ctx) { // ensure all direct android.Module deps are enabled - ctx.VisitDirectDeps(func(m Module) { - ctx.validateAndroidModule(m, ctx.OtherModuleDependencyTag(m), ctx.baseModuleContext.strictVisitDeps) - }) + ctx.VisitDirectDepsProxy(func(m ModuleProxy) {}) if m.Device() { // Handle any init.rc and vintf fragment files requested by the module. All files installed by this @@ -2036,6 +2045,10 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) ctx.GetMissingDependencies() } + if sourceFileProducer, ok := m.module.(SourceFileProducer); ok { + SetProvider(ctx, SourceFilesInfoKey, SourceFilesInfo{Srcs: sourceFileProducer.Srcs()}) + } + if ctx.IsFinalModule(m.module) { m.generateModuleTarget(ctx) if ctx.Failed() { @@ -2110,16 +2123,17 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) } buildComplianceMetadataProvider(ctx, m) - commonData := CommonPropertiesProviderData{ - ReplacedByPrebuilt: m.commonProperties.ReplacedByPrebuilt, - CompileTarget: m.commonProperties.CompileTarget, + commonData := CommonModuleInfo{ + ReplacedByPrebuilt: m.commonProperties.ReplacedByPrebuilt, + CompileTarget: m.commonProperties.CompileTarget, + SkipAndroidMkProcessing: shouldSkipAndroidMkProcessing(ctx, m), } if m.commonProperties.ForcedDisabled { commonData.Enabled = false } else { commonData.Enabled = m.commonProperties.Enabled.GetOrDefault(m.ConfigurableEvaluator(ctx), !m.Os().DefaultDisabled) } - SetProvider(ctx, CommonPropertiesProviderKey, commonData) + SetProvider(ctx, CommonModuleInfoKey, commonData) if p, ok := m.module.(PrebuiltInterface); ok && p.Prebuilt() != nil { SetProvider(ctx, PrebuiltModuleProviderKey, PrebuiltModuleProviderData{}) } @@ -2128,7 +2142,7 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) HostToolPath: h.HostToolPath()}) } - if p, ok := m.module.(AndroidMkProviderInfoProducer); ok && !shouldSkipAndroidMkProcessing(ctx, m) { + if p, ok := m.module.(AndroidMkProviderInfoProducer); ok && !commonData.SkipAndroidMkProcessing { SetProvider(ctx, AndroidMkInfoProvider, p.PrepareAndroidMKProviderInfo(ctx.Config())) } } @@ -2632,7 +2646,7 @@ type SourceFileProducer interface { // OutputFilesForModule returns the output file paths with the given tag. On error, including if the // module produced zero paths, it reports errors to the ctx and returns nil. -func OutputFilesForModule(ctx PathContext, module blueprint.Module, tag string) Paths { +func OutputFilesForModule(ctx PathContext, module Module, tag string) Paths { paths, err := outputFilesForModule(ctx, module, tag) if err != nil { reportPathError(ctx, err) @@ -2643,7 +2657,7 @@ func OutputFilesForModule(ctx PathContext, module blueprint.Module, tag string) // OutputFileForModule returns the output file paths with the given tag. On error, including if the // module produced zero or multiple paths, it reports errors to the ctx and returns nil. -func OutputFileForModule(ctx PathContext, module blueprint.Module, tag string) Path { +func OutputFileForModule(ctx PathContext, module Module, tag string) Path { paths, err := outputFilesForModule(ctx, module, tag) if err != nil { reportPathError(ctx, err) @@ -2676,20 +2690,34 @@ func OutputFileForModule(ctx PathContext, module blueprint.Module, tag string) P return paths[0] } -func outputFilesForModule(ctx PathContext, module blueprint.Module, tag string) (Paths, error) { +type OutputFilesProviderModuleContext interface { + OtherModuleProviderContext + Module() Module + GetOutputFiles() OutputFilesInfo + EqualModules(m1, m2 Module) bool +} + +func outputFilesForModule(ctx PathContext, module Module, tag string) (Paths, error) { outputFilesFromProvider, err := outputFilesForModuleFromProvider(ctx, module, tag) if outputFilesFromProvider != nil || err != OutputFilesProviderNotSet { return outputFilesFromProvider, err } - if sourceFileProducer, ok := module.(SourceFileProducer); ok { - if tag != "" { - return nil, fmt.Errorf("module %q is a SourceFileProducer, which does not support tag %q", pathContextName(ctx, module), tag) + + if octx, ok := ctx.(OutputFilesProviderModuleContext); ok { + if octx.EqualModules(octx.Module(), module) { + if sourceFileProducer, ok := module.(SourceFileProducer); ok { + return sourceFileProducer.Srcs(), nil + } + } else if sourceFiles, ok := OtherModuleProvider(octx, module, SourceFilesInfoKey); ok { + if tag != "" { + return nil, fmt.Errorf("module %q is a SourceFileProducer, which does not support tag %q", pathContextName(ctx, module), tag) + } + paths := sourceFiles.Srcs + return paths, nil } - paths := sourceFileProducer.Srcs() - return paths, nil - } else { - return nil, fmt.Errorf("module %q is not a SourceFileProducer or having valid output file for tag %q", pathContextName(ctx, module), tag) } + + return nil, fmt.Errorf("module %q is not a SourceFileProducer or having valid output file for tag %q", pathContextName(ctx, module), tag) } // This method uses OutputFilesProvider for output files @@ -2698,26 +2726,19 @@ func outputFilesForModule(ctx PathContext, module blueprint.Module, tag string) // from outputFiles property of module base, to avoid both setting and // reading OutputFilesProvider before GenerateBuildActions is finished. // If a module doesn't have the OutputFilesProvider, nil is returned. -func outputFilesForModuleFromProvider(ctx PathContext, module blueprint.Module, tag string) (Paths, error) { +func outputFilesForModuleFromProvider(ctx PathContext, module Module, tag string) (Paths, error) { var outputFiles OutputFilesInfo fromProperty := false - type OutputFilesProviderModuleContext interface { - OtherModuleProviderContext - Module() Module - GetOutputFiles() OutputFilesInfo - } - if mctx, isMctx := ctx.(OutputFilesProviderModuleContext); isMctx { - if mctx.Module() != module { + if !mctx.EqualModules(mctx.Module(), module) { outputFiles, _ = OtherModuleProvider(mctx, module, OutputFilesProvider) } else { outputFiles = mctx.GetOutputFiles() fromProperty = true } } else if cta, isCta := ctx.(*singletonContextAdaptor); isCta { - providerData, _ := cta.otherModuleProvider(module, OutputFilesProvider) - outputFiles, _ = providerData.(OutputFilesInfo) + outputFiles, _ = OtherModuleProvider(cta, module, OutputFilesProvider) } else { return nil, fmt.Errorf("unsupported context %q in method outputFilesForModuleFromProvider", reflect.TypeOf(ctx)) } diff --git a/android/module_context.go b/android/module_context.go index 41cb0ccb2..20149074e 100644 --- a/android/module_context.go +++ b/android/module_context.go @@ -16,13 +16,13 @@ package android import ( "fmt" - "github.com/google/blueprint/depset" "path" "path/filepath" "slices" "strings" "github.com/google/blueprint" + "github.com/google/blueprint/depset" "github.com/google/blueprint/proptools" ) @@ -439,9 +439,11 @@ func (m *moduleContext) GetMissingDependencies() []string { return missingDeps } -func (m *moduleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) blueprint.Module { - module, _ := m.getDirectDepInternal(name, tag) - return module +func (m *moduleContext) GetDirectDepWithTag(name string, tag blueprint.DependencyTag) Module { + if module, _ := m.getDirectDepInternal(name, tag); module != nil { + return module.(Module) + } + return nil } func (m *moduleContext) ModuleSubDir() string { diff --git a/android/module_test.go b/android/module_test.go index d76d9b329..d5bf94137 100644 --- a/android/module_test.go +++ b/android/module_test.go @@ -998,6 +998,10 @@ func (p *pathContextAddMissingDependenciesWrapper) GetOutputFiles() OutputFilesI return OutputFilesInfo{} } +func (p *pathContextAddMissingDependenciesWrapper) EqualModules(m1, m2 Module) bool { + return m1 == m2 +} + func TestOutputFileForModule(t *testing.T) { testcases := []struct { name string diff --git a/android/neverallow.go b/android/neverallow.go index 6176a996a..326150be5 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -338,6 +338,7 @@ func createPrebuiltEtcBpDefineRule() Rule { "prebuilt_bt_firmware", "prebuilt_tvservice", "prebuilt_optee", + "prebuilt_tvconfig", ). DefinedInBpFile(). Because("module type not allowed to be defined in bp file") diff --git a/android/packaging.go b/android/packaging.go index 98c85fa0c..dcd8844f0 100644 --- a/android/packaging.go +++ b/android/packaging.go @@ -165,6 +165,10 @@ func (p *PackagingSpec) Partition() string { return p.partition } +func (p *PackagingSpec) SetPartition(partition string) { + p.partition = partition +} + func (p *PackagingSpec) SkipInstall() bool { return p.skipInstall } @@ -186,6 +190,7 @@ type PackageModule interface { // GatherPackagingSpecs gathers PackagingSpecs of transitive dependencies. GatherPackagingSpecs(ctx ModuleContext) map[string]PackagingSpec GatherPackagingSpecsWithFilter(ctx ModuleContext, filter func(PackagingSpec) bool) map[string]PackagingSpec + GatherPackagingSpecsWithFilterAndModifier(ctx ModuleContext, filter func(PackagingSpec) bool, modifier func(*PackagingSpec)) map[string]PackagingSpec // CopyDepsToZip zips the built artifacts of the dependencies into the given zip file and // returns zip entries in it. This is expected to be called in GenerateAndroidBuildActions, @@ -405,9 +410,9 @@ func (PackagingItemAlwaysDepTag) IsPackagingItem() bool { return true } -// highPriorityDepTag provides default implementation of HighPriorityPackagingItem interface. type highPriorityDepTag struct { - blueprint.DependencyTag + blueprint.BaseDependencyTag + PackagingItemAlwaysDepTag } // See PackageModule.AddDeps @@ -428,7 +433,7 @@ func (p *PackagingBase) AddDeps(ctx BottomUpMutatorContext, depTag blueprint.Dep } depTagToUse := depTag if highPriority { - depTagToUse = highPriorityDepTag{depTag} + depTagToUse = highPriorityDepTag{} } ctx.AddFarVariationDependencies(targetVariation, depTagToUse, dep) @@ -444,7 +449,8 @@ func (p *PackagingBase) AddDeps(ctx BottomUpMutatorContext, depTag blueprint.Dep } } -func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter func(PackagingSpec) bool) map[string]PackagingSpec { +// See PackageModule.GatherPackagingSpecs +func (p *PackagingBase) GatherPackagingSpecsWithFilterAndModifier(ctx ModuleContext, filter func(PackagingSpec) bool, modifier func(*PackagingSpec)) map[string]PackagingSpec { // packaging specs gathered from the dep that are not high priorities. var regularPriorities []PackagingSpec @@ -474,7 +480,7 @@ func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter return false } - ctx.VisitDirectDeps(func(child Module) { + ctx.VisitDirectDepsProxy(func(child ModuleProxy) { depTag := ctx.OtherModuleDependencyTag(child) if pi, ok := depTag.(PackagingItem); !ok || !pi.IsPackagingItem() { return @@ -491,6 +497,10 @@ func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter } } + if modifier != nil { + modifier(&ps) + } + if _, ok := depTag.(highPriorityDepTag); ok { highPriorities = append(highPriorities, ps) } else { @@ -552,6 +562,11 @@ func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter } // See PackageModule.GatherPackagingSpecs +func (p *PackagingBase) GatherPackagingSpecsWithFilter(ctx ModuleContext, filter func(PackagingSpec) bool) map[string]PackagingSpec { + return p.GatherPackagingSpecsWithFilterAndModifier(ctx, filter, nil) +} + +// See PackageModule.GatherPackagingSpecs func (p *PackagingBase) GatherPackagingSpecs(ctx ModuleContext) map[string]PackagingSpec { return p.GatherPackagingSpecsWithFilter(ctx, nil) } diff --git a/android/path_properties_test.go b/android/path_properties_test.go index 07b48696c..6f44f2872 100644 --- a/android/path_properties_test.go +++ b/android/path_properties_test.go @@ -64,7 +64,7 @@ func (p *pathDepsMutatorTestModule) GenerateAndroidBuildActions(ctx ModuleContex if p.props.Foo != "" { // Make sure there is only one dependency on a module listed in a property present in multiple property structs m := SrcIsModule(p.props.Foo) - if GetModuleFromPathDep(ctx, m, "") == nil { + if GetModuleProxyFromPathDep(ctx, m, "") == nil { ctx.ModuleErrorf("GetDirectDepWithTag failed") } } diff --git a/android/paths.go b/android/paths.go index 9cb872d6f..7ab1f226c 100644 --- a/android/paths.go +++ b/android/paths.go @@ -91,6 +91,8 @@ type ModuleWithDepsPathContext interface { EarlyModulePathContext OtherModuleProviderContext VisitDirectDeps(visit func(Module)) + VisitDirectDepsProxy(visit func(ModuleProxy)) + VisitDirectDepsProxyWithTag(tag blueprint.DependencyTag, visit func(ModuleProxy)) OtherModuleDependencyTag(m blueprint.Module) blueprint.DependencyTag HasMutatorFinished(mutatorName string) bool } @@ -598,7 +600,7 @@ func DirectoryPathsForModuleSrc(ctx ModuleMissingDepsPathContext, paths []string for _, path := range paths { if m, t := SrcIsModuleWithTag(path); m != "" { - module := GetModuleFromPathDep(ctx, m, t) + module := GetModuleProxyFromPathDep(ctx, m, t) if module == nil { ctx.ModuleErrorf(`missing dependency on %q, is the property annotated with android:"path"?`, m) continue @@ -611,7 +613,7 @@ func DirectoryPathsForModuleSrc(ctx ModuleMissingDepsPathContext, paths []string if !ok { panic(fmt.Errorf("%s is not an OtherModuleProviderContext", ctx)) } - if dirProvider, ok := OtherModuleProvider(mctx, module, DirProvider); ok { + if dirProvider, ok := OtherModuleProvider(mctx, *module, DirProvider); ok { ret = append(ret, dirProvider.Dirs...) } else { ReportPathErrorf(ctx, "module %q does not implement DirProvider", module) @@ -669,14 +671,15 @@ func (p OutputPaths) Strings() []string { // If the dependency is not found, a missingErrorDependency is returned. // If the module dependency is not a SourceFileProducer or OutputFileProducer, appropriate errors will be returned. func getPathsFromModuleDep(ctx ModuleWithDepsPathContext, path, moduleName, tag string) (Paths, error) { - module := GetModuleFromPathDep(ctx, moduleName, tag) + module := GetModuleProxyFromPathDep(ctx, moduleName, tag) if module == nil { return nil, missingDependencyError{[]string{moduleName}} } - if aModule, ok := module.(Module); ok && !aModule.Enabled(ctx) { + if !OtherModuleProviderOrDefault(ctx, *module, CommonModuleInfoKey).Enabled { return nil, missingDependencyError{[]string{moduleName}} } - outputFiles, err := outputFilesForModule(ctx, module, tag) + + outputFiles, err := outputFilesForModule(ctx, *module, tag) if outputFiles != nil && err == nil { return outputFiles, nil } else { @@ -684,7 +687,7 @@ func getPathsFromModuleDep(ctx ModuleWithDepsPathContext, path, moduleName, tag } } -// GetModuleFromPathDep will return the module that was added as a dependency automatically for +// GetModuleProxyFromPathDep will return the module that was added as a dependency automatically for // properties tagged with `android:"path"` or manually using ExtractSourceDeps or // ExtractSourcesDeps. // @@ -694,6 +697,27 @@ func getPathsFromModuleDep(ctx ModuleWithDepsPathContext, path, moduleName, tag // // If tag is "" then the returned module will be the dependency that was added for ":moduleName". // Otherwise, it is the dependency that was added for ":moduleName{tag}". +func GetModuleProxyFromPathDep(ctx ModuleWithDepsPathContext, moduleName, tag string) *ModuleProxy { + var found *ModuleProxy + // The sourceOrOutputDepTag uniquely identifies the module dependency as it contains both the + // module name and the tag. Dependencies added automatically for properties tagged with + // `android:"path"` are deduped so are guaranteed to be unique. It is possible for duplicate + // dependencies to be added manually using ExtractSourcesDeps or ExtractSourceDeps but even then + // it will always be the case that the dependencies will be identical, i.e. the same tag and same + // moduleName referring to the same dependency module. + // + // It does not matter whether the moduleName is a fully qualified name or if the module + // dependency is a prebuilt module. All that matters is the same information is supplied to + // create the tag here as was supplied to create the tag when the dependency was added so that + // this finds the matching dependency module. + expectedTag := sourceOrOutputDepTag(moduleName, tag) + ctx.VisitDirectDepsProxyWithTag(expectedTag, func(module ModuleProxy) { + found = &module + }) + return found +} + +// Deprecated: use GetModuleProxyFromPathDep func GetModuleFromPathDep(ctx ModuleWithDepsPathContext, moduleName, tag string) blueprint.Module { var found blueprint.Module // The sourceOrOutputDepTag uniquely identifies the module dependency as it contains both the @@ -2589,3 +2613,19 @@ func IsThirdPartyPath(path string) bool { } return false } + +// ToRelativeSourcePath converts absolute source path to the path relative to the source root. +// This throws an error if the input path is outside of the source root and cannot be converted +// to the relative path. +// This should be rarely used given that the source path is relative in Soong. +func ToRelativeSourcePath(ctx PathContext, path string) string { + ret := path + if filepath.IsAbs(path) { + relPath, err := filepath.Rel(absSrcDir, path) + if err != nil || strings.HasPrefix(relPath, "..") { + ReportPathErrorf(ctx, "%s is outside of the source root", path) + } + ret = relPath + } + return ret +} diff --git a/android/prebuilt.go b/android/prebuilt.go index 51e72af9a..0ac67b3f4 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -381,7 +381,7 @@ func GetEmbeddedPrebuilt(module Module) *Prebuilt { // the right module. This function is only safe to call after all TransitionMutators // have run, e.g. in GenerateAndroidBuildActions. func PrebuiltGetPreferred(ctx BaseModuleContext, module Module) Module { - if !OtherModuleProviderOrDefault(ctx, module, CommonPropertiesProviderKey).ReplacedByPrebuilt { + if !OtherModuleProviderOrDefault(ctx, module, CommonModuleInfoKey).ReplacedByPrebuilt { return module } if _, ok := OtherModuleProvider(ctx, module, PrebuiltModuleProviderKey); ok { diff --git a/android/product_config.go b/android/product_config.go index ce3acc9f2..850f00334 100644 --- a/android/product_config.go +++ b/android/product_config.go @@ -32,7 +32,7 @@ func (p *productConfigModule) GenerateAndroidBuildActions(ctx ModuleContext) { ctx.ModuleErrorf("There can only be one product_config module in build/soong") return } - outputFilePath := PathForModuleOut(ctx, p.Name()+".json").OutputPath + outputFilePath := PathForModuleOut(ctx, p.Name()+".json") // DeviceProduct can be null so calling ctx.Config().DeviceProduct() may cause null dereference targetProduct := proptools.String(ctx.Config().config.productVariables.DeviceProduct) diff --git a/android/testing.go b/android/testing.go index 23aadda53..f243e81dd 100644 --- a/android/testing.go +++ b/android/testing.go @@ -1132,11 +1132,6 @@ func SetKatiEnabledForTests(config Config) { config.katiEnabled = true } -func SetTrimmedApexEnabledForTests(config Config) { - config.productVariables.TrimmedApex = new(bool) - *config.productVariables.TrimmedApex = true -} - func AndroidMkEntriesForTest(t *testing.T, ctx *TestContext, mod blueprint.Module) []AndroidMkEntries { t.Helper() var p AndroidMkEntriesProvider diff --git a/android/variable.go b/android/variable.go index 037037d6a..2d43c6da4 100644 --- a/android/variable.go +++ b/android/variable.go @@ -408,7 +408,6 @@ type ProductVariables struct { Ndk_abis *bool `json:",omitempty"` - TrimmedApex *bool `json:",omitempty"` ForceApexSymlinkOptimization *bool `json:",omitempty"` CompressedApex *bool `json:",omitempty"` Aml_abis *bool `json:",omitempty"` @@ -607,10 +606,20 @@ type PartitionVariables struct { BoardExt4ShareDupBlocks string `json:",omitempty"` BoardFlashLogicalBlockSize string `json:",omitempty"` BoardFlashEraseBlockSize string `json:",omitempty"` - BoardUsesRecoveryAsBoot bool `json:",omitempty"` ProductUseDynamicPartitionSize bool `json:",omitempty"` CopyImagesForTargetFilesZip bool `json:",omitempty"` + // Boot image stuff + ProductBuildBootImage bool `json:",omitempty"` + ProductBuildInitBootImage bool `json:",omitempty"` + BoardUsesRecoveryAsBoot bool `json:",omitempty"` + BoardPrebuiltBootimage string `json:",omitempty"` + BoardPrebuiltInitBootimage string `json:",omitempty"` + BoardBootimagePartitionSize string `json:",omitempty"` + BoardInitBootimagePartitionSize string `json:",omitempty"` + BoardBootHeaderVersion string `json:",omitempty"` + + // Avb (android verified boot) stuff BoardAvbEnable bool `json:",omitempty"` BoardAvbAlgorithm string `json:",omitempty"` BoardAvbKeyPath string `json:",omitempty"` @@ -626,10 +635,18 @@ type PartitionVariables struct { BoardInfoFiles []string `json:",omitempty"` BootLoaderBoardName string `json:",omitempty"` - ProductCopyFiles map[string]string `json:",omitempty"` - - BuildingSystemDlkmImage bool `json:",omitempty"` - SystemKernelModules []string `json:",omitempty"` + ProductCopyFiles []string `json:",omitempty"` + + BuildingSystemDlkmImage bool `json:",omitempty"` + SystemKernelModules []string `json:",omitempty"` + SystemKernelBlocklistFile string `json:",omitempty"` + SystemKernelLoadModules []string `json:",omitempty"` + BuildingVendorDlkmImage bool `json:",omitempty"` + VendorKernelModules []string `json:",omitempty"` + VendorKernelBlocklistFile string `json:",omitempty"` + BuildingOdmDlkmImage bool `json:",omitempty"` + OdmKernelModules []string `json:",omitempty"` + OdmKernelBlocklistFile string `json:",omitempty"` } func boolPtr(v bool) *bool { @@ -680,7 +697,6 @@ func (v *ProductVariables) SetDefaultConfig() { Malloc_zero_contents: boolPtr(true), Malloc_pattern_fill_contents: boolPtr(false), Safestack: boolPtr(false), - TrimmedApex: boolPtr(false), Build_from_text_stub: boolPtr(false), BootJars: ConfiguredJarList{apexes: []string{}, jars: []string{}}, diff --git a/android/vintf_data.go b/android/vintf_data.go index 7823397e0..401f4d2e5 100644 --- a/android/vintf_data.go +++ b/android/vintf_data.go @@ -49,7 +49,7 @@ type vintfDataRule struct { properties vintfDataProperties installDirPath InstallPath - outputFilePath OutputPath + outputFilePath Path noAction bool } @@ -148,7 +148,7 @@ func (m *vintfDataRule) GenerateAndroidBuildActions(ctx ModuleContext) { builder.Build("assemble_vintf", "Process vintf data "+gensrc.String()) m.installDirPath = PathForModuleInstall(ctx, "etc", "vintf") - m.outputFilePath = gensrc.OutputPath + m.outputFilePath = gensrc installFileName := "manifest.xml" if filename := proptools.String(m.properties.Filename); filename != "" { diff --git a/android/vintf_fragment.go b/android/vintf_fragment.go index 42eaaf01a..a3343fd5a 100644 --- a/android/vintf_fragment.go +++ b/android/vintf_fragment.go @@ -25,7 +25,7 @@ type vintfFragmentModule struct { properties vintfFragmentProperties installDirPath InstallPath - outputFilePath OutputPath + outputFilePath Path } func init() { @@ -64,7 +64,7 @@ func (m *vintfFragmentModule) GenerateAndroidBuildActions(ctx ModuleContext) { builder.Build("assemble_vintf", "Process vintf fragment "+processedVintfFragment.String()) m.installDirPath = PathForModuleInstall(ctx, "etc", "vintf", "manifest") - m.outputFilePath = processedVintfFragment.OutputPath + m.outputFilePath = processedVintfFragment ctx.InstallFile(m.installDirPath, processedVintfFragment.Base(), processedVintfFragment) } diff --git a/apex/apex.go b/apex/apex.go index 587f63fe1..dc24df3d1 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -63,14 +63,11 @@ func RegisterPreDepsMutators(ctx android.RegisterMutatorsContext) { func RegisterPostDepsMutators(ctx android.RegisterMutatorsContext) { ctx.TopDown("apex_info", apexInfoMutator) ctx.BottomUp("apex_unique", apexUniqueVariationsMutator) - ctx.BottomUp("apex_test_for_deps", apexTestForDepsMutator) - ctx.BottomUp("apex_test_for", apexTestForMutator) // Run mark_platform_availability before the apexMutator as the apexMutator needs to know whether // it should create a platform variant. ctx.BottomUp("mark_platform_availability", markPlatformAvailability) ctx.Transition("apex", &apexTransitionMutator{}) ctx.BottomUp("apex_directly_in_any", apexDirectlyInAnyMutator).MutatesDependencies() - ctx.BottomUp("apex_dcla_deps", apexDCLADepsMutator) } type apexBundleProperties struct { @@ -108,11 +105,10 @@ type apexBundleProperties struct { Rros []string // List of bootclasspath fragments that are embedded inside this APEX bundle. - Bootclasspath_fragments []string + Bootclasspath_fragments proptools.Configurable[[]string] // List of systemserverclasspath fragments that are embedded inside this APEX bundle. - Systemserverclasspath_fragments proptools.Configurable[[]string] - ResolvedSystemserverclasspathFragments []string `blueprint:"mutated"` + Systemserverclasspath_fragments proptools.Configurable[[]string] // List of java libraries that are embedded inside this APEX bundle. Java_libs []string @@ -515,7 +511,7 @@ type apexBundle struct { // Text file having the list of individual files that are included in this APEX. Used for // debugging purpose. - installedFilesFile android.WritablePath + installedFilesFile android.Path // List of module names that this APEX is including (to be shown via *-deps-info target). // Used for debugging purpose. @@ -734,7 +730,6 @@ var ( androidAppTag = &dependencyTag{name: "androidApp", payload: true} bpfTag = &dependencyTag{name: "bpf", payload: true} certificateTag = &dependencyTag{name: "certificate"} - dclaTag = &dependencyTag{name: "dcla"} executableTag = &dependencyTag{name: "executable", payload: true} fsTag = &dependencyTag{name: "filesystem", payload: true} bcpfTag = &dependencyTag{name: "bootclasspathFragment", payload: true, sourceOnly: true, memberType: java.BootclasspathFragmentSdkMemberType} @@ -747,7 +742,6 @@ var ( prebuiltTag = &dependencyTag{name: "prebuilt", payload: true} rroTag = &dependencyTag{name: "rro", payload: true} sharedLibTag = &dependencyTag{name: "sharedLib", payload: true} - testForTag = &dependencyTag{name: "test for"} testTag = &dependencyTag{name: "test", payload: true} shBinaryTag = &dependencyTag{name: "shBinary", payload: true} ) @@ -896,13 +890,11 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { } } - a.properties.ResolvedSystemserverclasspathFragments = a.properties.Systemserverclasspath_fragments.GetOrDefault(ctx, nil) - // Common-arch dependencies come next commonVariation := ctx.Config().AndroidCommonTarget.Variations() ctx.AddFarVariationDependencies(commonVariation, rroTag, a.properties.Rros...) - ctx.AddFarVariationDependencies(commonVariation, bcpfTag, a.properties.Bootclasspath_fragments...) - ctx.AddFarVariationDependencies(commonVariation, sscpfTag, a.properties.ResolvedSystemserverclasspathFragments...) + ctx.AddFarVariationDependencies(commonVariation, bcpfTag, a.properties.Bootclasspath_fragments.GetOrDefault(ctx, nil)...) + ctx.AddFarVariationDependencies(commonVariation, sscpfTag, a.properties.Systemserverclasspath_fragments.GetOrDefault(ctx, nil)...) ctx.AddFarVariationDependencies(commonVariation, javaLibTag, a.properties.Java_libs...) ctx.AddFarVariationDependencies(commonVariation, fsTag, a.properties.Filesystems...) ctx.AddFarVariationDependencies(commonVariation, compatConfigTag, a.properties.Compat_configs...) @@ -952,33 +944,6 @@ func (a *apexBundle) OverridablePropertiesDepsMutator(ctx android.BottomUpMutato } } -func apexDCLADepsMutator(mctx android.BottomUpMutatorContext) { - if !mctx.Config().ApexTrimEnabled() { - return - } - if a, ok := mctx.Module().(*apexBundle); ok && a.overridableProperties.Trim_against != nil { - commonVariation := mctx.Config().AndroidCommonTarget.Variations() - mctx.AddFarVariationDependencies(commonVariation, dclaTag, String(a.overridableProperties.Trim_against)) - } else if o, ok := mctx.Module().(*OverrideApex); ok { - for _, p := range o.GetProperties() { - properties, ok := p.(*overridableProperties) - if !ok { - continue - } - if properties.Trim_against != nil { - commonVariation := mctx.Config().AndroidCommonTarget.Variations() - mctx.AddFarVariationDependencies(commonVariation, dclaTag, String(properties.Trim_against)) - } - } - } -} - -type DCLAInfo struct { - ProvidedLibs []string -} - -var DCLAInfoProvider = blueprint.NewMutatorProvider[DCLAInfo]("apex_info") - var _ ApexInfoMutator = (*apexBundle)(nil) func (a *apexBundle) ApexVariationName() string { @@ -1087,12 +1052,6 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { child.(android.ApexModule).BuildForApex(apexInfo) // leave a mark! return true }) - - if a.dynamic_common_lib_apex() { - android.SetProvider(mctx, DCLAInfoProvider, DCLAInfo{ - ProvidedLibs: a.properties.Native_shared_libs.GetOrDefault(mctx, nil), - }) - } } type ApexInfoMutator interface { @@ -1185,40 +1144,6 @@ func apexUniqueVariationsMutator(mctx android.BottomUpMutatorContext) { } } -// apexTestForDepsMutator checks if this module is a test for an apex. If so, add a dependency on -// the apex in order to retrieve its contents later. -// TODO(jiyong): move this to android/apex.go? -func apexTestForDepsMutator(mctx android.BottomUpMutatorContext) { - if !mctx.Module().Enabled(mctx) { - return - } - if am, ok := mctx.Module().(android.ApexModule); ok { - if testFor := am.TestFor(); len(testFor) > 0 { - mctx.AddFarVariationDependencies([]blueprint.Variation{ - {Mutator: "os", Variation: am.Target().OsVariation()}, - {"arch", "common"}, - }, testForTag, testFor...) - } - } -} - -// TODO(jiyong): move this to android/apex.go? -func apexTestForMutator(mctx android.BottomUpMutatorContext) { - if !mctx.Module().Enabled(mctx) { - return - } - if _, ok := mctx.Module().(android.ApexModule); ok { - var contents []*android.ApexContents - for _, testFor := range mctx.GetDirectDepsWithTag(testForTag) { - abInfo, _ := android.OtherModuleProvider(mctx, testFor, android.ApexBundleInfoProvider) - contents = append(contents, abInfo.Contents) - } - android.SetProvider(mctx, android.ApexTestForInfoProvider, android.ApexTestForInfo{ - ApexContents: contents, - }) - } -} - // markPlatformAvailability marks whether or not a module can be available to platform. A module // cannot be available to platform if 1) it is explicitly marked as not available (i.e. // "//apex_available:platform" is absent) or 2) it depends on another module that isn't (or can't @@ -1442,19 +1367,6 @@ func (a *apexBundle) dynamic_common_lib_apex() bool { return proptools.BoolDefault(a.properties.Dynamic_common_lib_apex, false) } -// See the list of libs to trim -func (a *apexBundle) libs_to_trim(ctx android.ModuleContext) []string { - dclaModules := ctx.GetDirectDepsWithTag(dclaTag) - if len(dclaModules) > 1 { - panic(fmt.Errorf("expected exactly at most one dcla dependency, got %d", len(dclaModules))) - } - if len(dclaModules) > 0 { - DCLAInfo, _ := android.OtherModuleProvider(ctx, dclaModules[0], DCLAInfoProvider) - return DCLAInfo.ProvidedLibs - } - return []string{} -} - // These functions are interfacing with cc/sanitizer.go. The entire APEX (along with all of its // members) can be sanitized, either forcibly, or by the global configuration. For some of the // sanitizers, extra dependencies can be forcibly added as well. @@ -2159,8 +2071,7 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, af := apexFileForNativeLibrary(ctx, ch, vctx.handleSpecialLibs) af.transitiveDep = true - abInfo, _ := android.ModuleProvider(ctx, android.ApexBundleInfoProvider) - if !abInfo.Contents.DirectlyInApex(depName) && (ch.IsStubs() || ch.HasStubsVariants()) { + if ch.IsStubs() || ch.HasStubsVariants() { // If the dependency is a stubs lib, don't include it in this APEX, // but make sure that the lib is installed on the device. // In case no APEX is having the lib, the lib is installed to the system @@ -2664,7 +2575,10 @@ func (a *apexBundle) checkStaticLinkingToStubLibraries(ctx android.ModuleContext return } - abInfo, _ := android.ModuleProvider(ctx, android.ApexBundleInfoProvider) + librariesDirectlyInApex := make(map[string]bool) + ctx.VisitDirectDepsProxyWithTag(sharedLibTag, func(dep android.ModuleProxy) { + librariesDirectlyInApex[ctx.OtherModuleName(dep)] = true + }) a.WalkPayloadDeps(ctx, func(ctx android.BaseModuleContext, from blueprint.Module, to android.ApexModule, externalDep bool) bool { if ccm, ok := to.(*cc.Module); ok { @@ -2690,7 +2604,7 @@ func (a *apexBundle) checkStaticLinkingToStubLibraries(ctx android.ModuleContext return false } - isStubLibraryFromOtherApex := ccm.HasStubsVariants() && !abInfo.Contents.DirectlyInApex(toName) + isStubLibraryFromOtherApex := ccm.HasStubsVariants() && !librariesDirectlyInApex[toName] if isStubLibraryFromOtherApex && !externalDep { ctx.ModuleErrorf("%q required by %q is a native library providing stub. "+ "It shouldn't be included in this APEX via static linking. Dependency path: %s", to.String(), fromName, ctx.GetPathString(false)) @@ -2723,7 +2637,7 @@ func (a *apexBundle) checkUpdatable(ctx android.ModuleContext) { // checkClasspathFragments enforces that all classpath fragments in deps generate classpaths.proto config. func (a *apexBundle) checkClasspathFragments(ctx android.ModuleContext) { - ctx.VisitDirectDeps(func(module android.Module) { + ctx.VisitDirectDepsProxy(func(module android.ModuleProxy) { if tag := ctx.OtherModuleDependencyTag(module); tag == bcpfTag || tag == sscpfTag { info, _ := android.OtherModuleProvider(ctx, module, java.ClasspathFragmentProtoContentInfoProvider) if !info.ClasspathFragmentProtoGenerated { @@ -2830,12 +2744,12 @@ func (a *apexBundle) checkApexAvailability(ctx android.ModuleContext) { // checkStaticExecutable ensures that executables in an APEX are not static. func (a *apexBundle) checkStaticExecutables(ctx android.ModuleContext) { - ctx.VisitDirectDeps(func(module android.Module) { + ctx.VisitDirectDepsProxy(func(module android.ModuleProxy) { if ctx.OtherModuleDependencyTag(module) != executableTag { return } - if l, ok := module.(cc.LinkableInterface); ok && l.StaticExecutable() { + if android.OtherModuleProviderOrDefault(ctx, module, cc.LinkableInfoKey).StaticExecutable { apex := a.ApexVariationName() exec := ctx.OtherModuleName(module) if isStaticExecutableAllowed(apex, exec) { @@ -2861,8 +2775,8 @@ func isStaticExecutableAllowed(apex string, exec string) bool { // Collect information for opening IDE project files in java/jdeps.go. func (a *apexBundle) IDEInfo(ctx android.BaseModuleContext, dpInfo *android.IdeInfo) { dpInfo.Deps = append(dpInfo.Deps, a.properties.Java_libs...) - dpInfo.Deps = append(dpInfo.Deps, a.properties.Bootclasspath_fragments...) - dpInfo.Deps = append(dpInfo.Deps, a.properties.ResolvedSystemserverclasspathFragments...) + dpInfo.Deps = append(dpInfo.Deps, a.properties.Bootclasspath_fragments.GetOrDefault(ctx, nil)...) + dpInfo.Deps = append(dpInfo.Deps, a.properties.Systemserverclasspath_fragments.GetOrDefault(ctx, nil)...) } func init() { @@ -2941,10 +2855,6 @@ func rBcpPackages() map[string][]string { } } -func (a *apexBundle) IsTestApex() bool { - return a.testApex -} - // verifyNativeImplementationLibs compares the list of transitive implementation libraries used to link native // libraries in the apex against the list of implementation libraries in the apex, ensuring that none of the // libraries in the apex have references to private APIs from outside the apex. diff --git a/apex/apex_singleton.go b/apex/apex_singleton.go index 00dd44648..d46104e40 100644 --- a/apex/apex_singleton.go +++ b/apex/apex_singleton.go @@ -64,12 +64,14 @@ var ( if grep -v -h '^#' ${allowed_deps_list} | sort -u -f| diff -B -u - ${new_allowed_deps}; then touch ${out}; else - echo -e "\n******************************"; + echo; + echo "******************************"; echo "ERROR: go/apex-allowed-deps-error contains more information"; echo "******************************"; echo "Detected changes to allowed dependencies in updatable modules."; echo "To fix and update packages/modules/common/build/allowed_deps.txt, please run:"; - echo -e "$$ (croot && packages/modules/common/build/update-apex-allowed-deps.sh)\n"; + echo "$$ (croot && packages/modules/common/build/update-apex-allowed-deps.sh)"; + echo; echo "When submitting the generated CL, you must include the following information"; echo "in the commit message if you are adding a new dependency:"; echo "Apex-Size-Increase: Expected binary size increase for affected APEXes (or the size of the .jar / .so file of the new library)"; @@ -78,7 +80,8 @@ var ( echo "Test-Info: What’s the testing strategy for the new dependency? Does it have its own tests, and are you adding integration tests? How/when are the tests run?"; echo "You do not need OWNERS approval to submit the change, but mainline-modularization@"; echo "will periodically review additions and may require changes."; - echo -e "******************************\n"; + echo "******************************"; + echo; exit 1; fi; `, diff --git a/apex/apex_test.go b/apex/apex_test.go index 4e6aa1353..d0494d67d 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -929,7 +929,7 @@ func TestApexWithStubs(t *testing.T) { cc_library { name: "mylib", srcs: ["mylib.cpp"], - shared_libs: ["mylib2", "mylib3", "my_prebuilt_platform_lib", "my_prebuilt_platform_stub_only_lib"], + shared_libs: ["mylib2", "mylib3#impl", "my_prebuilt_platform_lib", "my_prebuilt_platform_stub_only_lib"], system_shared_libs: [], stl: "none", apex_available: [ "myapex" ], @@ -1025,7 +1025,7 @@ func TestApexWithStubs(t *testing.T) { // ... and not linking to the non-stub (impl) variant of mylib2 ensureNotContains(t, mylibLdFlags, "mylib2/android_arm64_armv8-a_shared/mylib2.so") - // Ensure that mylib is linking with the non-stub (impl) of mylib3 (because mylib3 is in the same apex) + // Ensure that mylib is linking with the non-stub (impl) of mylib3 (because the dependency is added with mylib3#impl) ensureContains(t, mylibLdFlags, "mylib3/android_arm64_armv8-a_shared_apex10000/mylib3.so") // .. and not linking to the stubs variant of mylib3 ensureNotContains(t, mylibLdFlags, "mylib3/android_arm64_armv8-a_shared_12/mylib3.so") @@ -1201,7 +1201,7 @@ func TestApexWithStubsWithMinSdkVersion(t *testing.T) { cc_library { name: "mylib", srcs: ["mylib.cpp"], - shared_libs: ["mylib2", "mylib3"], + shared_libs: ["mylib2", "mylib3#impl"], system_shared_libs: [], stl: "none", apex_available: [ "myapex" ], @@ -1264,7 +1264,7 @@ func TestApexWithStubsWithMinSdkVersion(t *testing.T) { // ... and not linking to the non-stub (impl) variant of mylib2 ensureNotContains(t, mylibLdFlags, "mylib2/android_arm64_armv8-a_shared/mylib2.so") - // Ensure that mylib is linking with the non-stub (impl) of mylib3 (because mylib3 is in the same apex) + // Ensure that mylib is linking with the non-stub (impl) of mylib3 (because the dependency is added with mylib3#impl) ensureContains(t, mylibLdFlags, "mylib3/android_arm64_armv8-a_shared_apex29/mylib3.so") // .. and not linking to the stubs variant of mylib3 ensureNotContains(t, mylibLdFlags, "mylib3/android_arm64_armv8-a_shared_29/mylib3.so") @@ -1797,8 +1797,8 @@ func TestApexWithSystemLibsStubs(t *testing.T) { cc_library { name: "mylib", srcs: ["mylib.cpp"], - system_shared_libs: ["libc", "libm"], - shared_libs: ["libdl#27"], + system_shared_libs: ["libc"], + shared_libs: ["libdl#27", "libm#impl"], stl: "none", apex_available: [ "myapex" ], } @@ -2962,8 +2962,7 @@ func TestApexMinSdkVersion_OkayEvenWhenDepIsNewer_IfItSatisfiesApexMinSdkVersion private_key: "testkey.pem", } - // mylib in myapex will link to mylib2#current - // mylib in otherapex will link to mylib2(non-stub) in otherapex as well + // mylib will link to mylib2#current cc_library { name: "mylib", srcs: ["mylib.cpp"], @@ -2997,7 +2996,7 @@ func TestApexMinSdkVersion_OkayEvenWhenDepIsNewer_IfItSatisfiesApexMinSdkVersion ensureContains(t, libFlags, "android_arm64_armv8-a_"+to_variant+"/"+to+".so") } expectLink("mylib", "shared_apex29", "mylib2", "shared_current") - expectLink("mylib", "shared_apex30", "mylib2", "shared_apex30") + expectLink("mylib", "shared_apex30", "mylib2", "shared_current") } func TestApexMinSdkVersion_WorksWithSdkCodename(t *testing.T) { @@ -10053,61 +10052,6 @@ func TestUpdatableApexEnforcesAppUpdatability(t *testing.T) { RunTestWithBp(t, bp) } -func TestTrimmedApex(t *testing.T) { - t.Parallel() - bp := ` - apex { - name: "myapex", - key: "myapex.key", - native_shared_libs: ["libfoo","libbaz"], - min_sdk_version: "29", - trim_against: "mydcla", - } - apex { - name: "mydcla", - key: "myapex.key", - native_shared_libs: ["libfoo","libbar"], - min_sdk_version: "29", - file_contexts: ":myapex-file_contexts", - dynamic_common_lib_apex: true, - } - apex_key { - name: "myapex.key", - } - cc_library { - name: "libfoo", - shared_libs: ["libc"], - apex_available: ["myapex","mydcla"], - min_sdk_version: "29", - } - cc_library { - name: "libbar", - shared_libs: ["libc"], - apex_available: ["myapex","mydcla"], - min_sdk_version: "29", - } - cc_library { - name: "libbaz", - shared_libs: ["libc"], - apex_available: ["myapex","mydcla"], - min_sdk_version: "29", - } - ` - ctx := testApex(t, bp) - module := ctx.ModuleForTests("myapex", "android_common_myapex") - apexRule := module.MaybeRule("apexRule") - if apexRule.Rule == nil { - t.Errorf("Expecting regular apex rule but a non regular apex rule found") - } - - ctx = testApex(t, bp, android.FixtureModifyConfig(android.SetTrimmedApexEnabledForTests)) - trimmedApexRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("TrimmedApexRule") - libs_to_trim := trimmedApexRule.Args["libs_to_trim"] - android.AssertStringDoesContain(t, "missing lib to trim", libs_to_trim, "libfoo") - android.AssertStringDoesContain(t, "missing lib to trim", libs_to_trim, "libbar") - android.AssertStringDoesNotContain(t, "unexpected libs in the libs to trim", libs_to_trim, "libbaz") -} - func TestCannedFsConfig(t *testing.T) { t.Parallel() ctx := testApex(t, ` @@ -10242,7 +10186,10 @@ func TestFileSystemShouldSkipApexLibraries(t *testing.T) { deps: [ "libfoo", ], - linker_config_src: "linker.config.json", + linker_config: { + gen_linker_config: true, + linker_config_srcs: ["linker.config.json"], + }, } cc_library { diff --git a/apex/builder.go b/apex/builder.go index 20b4dbeeb..d0acc8d6c 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -43,7 +43,6 @@ func init() { pctx.Import("android/soong/java") pctx.HostBinToolVariable("apexer", "apexer") pctx.HostBinToolVariable("apexer_with_DCLA_preprocessing", "apexer_with_DCLA_preprocessing") - pctx.HostBinToolVariable("apexer_with_trim_preprocessing", "apexer_with_trim_preprocessing") // ART minimal builds (using the master-art manifest) do not have the "frameworks/base" // projects, and hence cannot build 'aapt2'. Use the SDK prebuilt instead. @@ -173,34 +172,6 @@ var ( }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest", "is_DCLA") - TrimmedApexRule = pctx.StaticRule("TrimmedApexRule", blueprint.RuleParams{ - Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` + - `(. ${out}.copy_commands) && ` + - `APEXER_TOOL_PATH=${tool_path} ` + - `${apexer_with_trim_preprocessing} ` + - `--apexer ${apexer} ` + - `--canned_fs_config ${canned_fs_config} ` + - `--manifest ${manifest} ` + - `--libs_to_trim ${libs_to_trim} ` + - `${image_dir} ` + - `${out} ` + - `-- ` + - `--include_build_info ` + - `--force ` + - `--payload_type image ` + - `--key ${key} ` + - `--file_contexts ${file_contexts} ` + - `${opt_flags} `, - CommandDeps: []string{"${apexer_with_trim_preprocessing}", "${apexer}", "${avbtool}", "${e2fsdroid}", - "${merge_zips}", "${mke2fs}", "${resize2fs}", "${sefcontext_compile}", "${make_f2fs}", - "${sload_f2fs}", "${make_erofs}", "${soong_zip}", "${zipalign}", "${aapt2}", - "prebuilts/sdk/current/public/android.jar"}, - Rspfile: "${out}.copy_commands", - RspfileContent: "${copy_commands}", - Description: "APEX ${image_dir} => ${out}", - }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", - "opt_flags", "manifest", "libs_to_trim") - apexProtoConvertRule = pctx.AndroidStaticRule("apexProtoConvertRule", blueprint.RuleParams{ Command: `${aapt2} convert --output-format proto $in -o $out`, @@ -225,7 +196,7 @@ var ( Command: `diff --unchanged-group-format='' \` + `--changed-group-format='%<' \` + `${image_content_file} ${allowed_files_file} || (` + - `echo -e "New unexpected files were added to ${apex_module_name}." ` + + `echo "New unexpected files were added to ${apex_module_name}." ` + ` "To fix the build run following command:" && ` + `echo "system/apex/tools/update_allowed_list.sh ${allowed_files_file} ${image_content_file}" && ` + `exit 1); touch ${out}`, @@ -417,7 +388,7 @@ func (a *apexBundle) buildManifest(ctx android.ModuleContext, provideNativeLibs, // file for this APEX which is either from /systme/sepolicy/apex/<apexname>-file_contexts or from // the file_contexts property of this APEX. This is to make sure that the manifest file is correctly // labeled as system_file or vendor_apex_metadata_file. -func (a *apexBundle) buildFileContexts(ctx android.ModuleContext) android.OutputPath { +func (a *apexBundle) buildFileContexts(ctx android.ModuleContext) android.Path { var fileContexts android.Path var fileContextsDir string isFileContextsModule := false @@ -426,8 +397,10 @@ func (a *apexBundle) buildFileContexts(ctx android.ModuleContext) android.Output } else { if m, t := android.SrcIsModuleWithTag(*a.properties.File_contexts); m != "" { isFileContextsModule = true - otherModule := android.GetModuleFromPathDep(ctx, m, t) - fileContextsDir = ctx.OtherModuleDir(otherModule) + otherModule := android.GetModuleProxyFromPathDep(ctx, m, t) + if otherModule != nil { + fileContextsDir = ctx.OtherModuleDir(*otherModule) + } } fileContexts = android.PathForModuleSrc(ctx, *a.properties.File_contexts) } @@ -470,13 +443,13 @@ func (a *apexBundle) buildFileContexts(ctx android.ModuleContext) android.Output } rule.Build("file_contexts."+a.Name(), "Generate file_contexts") - return output.OutputPath + return output } // buildInstalledFilesFile creates a build rule for the installed-files.txt file where the list of // files included in this APEX is shown. The text file is dist'ed so that people can see what's // included in the APEX without actually downloading and extracting it. -func (a *apexBundle) buildInstalledFilesFile(ctx android.ModuleContext, builtApex android.Path, imageDir android.Path) android.OutputPath { +func (a *apexBundle) buildInstalledFilesFile(ctx android.ModuleContext, builtApex android.Path, imageDir android.Path) android.Path { output := android.PathForModuleOut(ctx, "installed-files.txt") rule := android.NewRuleBuilder(pctx, ctx) rule.Command(). @@ -486,12 +459,12 @@ func (a *apexBundle) buildInstalledFilesFile(ctx android.ModuleContext, builtApe Text(" | sort -nr > "). Output(output) rule.Build("installed-files."+a.Name(), "Installed files") - return output.OutputPath + return output } // buildBundleConfig creates a build rule for the bundle config file that will control the bundle // creation process. -func (a *apexBundle) buildBundleConfig(ctx android.ModuleContext) android.OutputPath { +func (a *apexBundle) buildBundleConfig(ctx android.ModuleContext) android.Path { output := android.PathForModuleOut(ctx, "bundle_config.json") type ApkConfig struct { @@ -536,7 +509,7 @@ func (a *apexBundle) buildBundleConfig(ctx android.ModuleContext) android.Output android.WriteFileRule(ctx, output, string(j)) - return output.OutputPath + return output } func markManifestTestOnly(ctx android.ModuleContext, androidManifestFile android.Path) android.Path { @@ -831,24 +804,6 @@ func (a *apexBundle) buildApex(ctx android.ModuleContext) { "opt_flags": strings.Join(optFlags, " "), }, }) - } else if ctx.Config().ApexTrimEnabled() && len(a.libs_to_trim(ctx)) > 0 { - ctx.Build(pctx, android.BuildParams{ - Rule: TrimmedApexRule, - Implicits: implicitInputs, - Output: unsignedOutputFile, - Description: "apex", - Args: map[string]string{ - "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir, - "image_dir": imageDir.String(), - "copy_commands": strings.Join(copyCommands, " && "), - "manifest": a.manifestPbOut.String(), - "file_contexts": fileContexts.String(), - "canned_fs_config": cannedFsConfig.String(), - "key": a.privateKeyFile.String(), - "opt_flags": strings.Join(optFlags, " "), - "libs_to_trim": strings.Join(a.libs_to_trim(ctx), ","), - }, - }) } else { ctx.Build(pctx, android.BuildParams{ Rule: apexRule, @@ -967,17 +922,17 @@ func (a *apexBundle) buildApex(ctx android.ModuleContext) { args["outCommaList"] = signedOutputFile.String() } var validations android.Paths - validations = append(validations, runApexLinkerconfigValidation(ctx, unsignedOutputFile.OutputPath, imageDir.OutputPath)) + validations = append(validations, runApexLinkerconfigValidation(ctx, unsignedOutputFile, imageDir)) // TODO(b/279688635) deapexer supports [ext4] if !a.testApex && suffix == imageApexSuffix && ext4 == a.payloadFsType { - validations = append(validations, runApexSepolicyTests(ctx, unsignedOutputFile.OutputPath)) + validations = append(validations, runApexSepolicyTests(ctx, unsignedOutputFile)) } if !a.testApex && len(a.properties.Unwanted_transitive_deps) > 0 { validations = append(validations, - runApexElfCheckerUnwanted(ctx, unsignedOutputFile.OutputPath, a.properties.Unwanted_transitive_deps)) + runApexElfCheckerUnwanted(ctx, unsignedOutputFile, a.properties.Unwanted_transitive_deps)) } if !a.testApex && android.InList(a.payloadFsType, []fsType{ext4, erofs}) { - validations = append(validations, runApexHostVerifier(ctx, a, unsignedOutputFile.OutputPath)) + validations = append(validations, runApexHostVerifier(ctx, a, unsignedOutputFile)) } ctx.Build(pctx, android.BuildParams{ Rule: rule, @@ -1180,7 +1135,7 @@ func (a *apexBundle) buildLintReports(ctx android.ModuleContext) { a.lintReports = java.BuildModuleLintReportZips(ctx, depSets, validations) } -func (a *apexBundle) buildCannedFsConfig(ctx android.ModuleContext) android.OutputPath { +func (a *apexBundle) buildCannedFsConfig(ctx android.ModuleContext) android.Path { var readOnlyPaths = []string{"apex_manifest.json", "apex_manifest.pb"} var executablePaths []string // this also includes dirs var appSetDirs []string @@ -1244,10 +1199,10 @@ func (a *apexBundle) buildCannedFsConfig(ctx android.ModuleContext) android.Outp cmd.Text(")").FlagWithOutput("> ", cannedFsConfig) builder.Build("generateFsConfig", fmt.Sprintf("Generating canned fs config for %s", a.BaseModuleName())) - return cannedFsConfig.OutputPath + return cannedFsConfig } -func runApexLinkerconfigValidation(ctx android.ModuleContext, apexFile android.OutputPath, imageDir android.OutputPath) android.Path { +func runApexLinkerconfigValidation(ctx android.ModuleContext, apexFile android.Path, imageDir android.Path) android.Path { timestamp := android.PathForModuleOut(ctx, "apex_linkerconfig_validation.timestamp") ctx.Build(pctx, android.BuildParams{ Rule: apexLinkerconfigValidationRule, @@ -1264,7 +1219,7 @@ func runApexLinkerconfigValidation(ctx android.ModuleContext, apexFile android.O // // $ deapexer list -Z {apex_file} > {file_contexts} // $ apex_sepolicy_tests -f {file_contexts} -func runApexSepolicyTests(ctx android.ModuleContext, apexFile android.OutputPath) android.Path { +func runApexSepolicyTests(ctx android.ModuleContext, apexFile android.Path) android.Path { timestamp := android.PathForModuleOut(ctx, "sepolicy_tests.timestamp") ctx.Build(pctx, android.BuildParams{ Rule: apexSepolicyTestsRule, @@ -1274,7 +1229,7 @@ func runApexSepolicyTests(ctx android.ModuleContext, apexFile android.OutputPath return timestamp } -func runApexElfCheckerUnwanted(ctx android.ModuleContext, apexFile android.OutputPath, unwanted []string) android.Path { +func runApexElfCheckerUnwanted(ctx android.ModuleContext, apexFile android.Path, unwanted []string) android.Path { timestamp := android.PathForModuleOut(ctx, "apex_elf_unwanted.timestamp") ctx.Build(pctx, android.BuildParams{ Rule: apexElfCheckerUnwantedRule, @@ -1288,7 +1243,7 @@ func runApexElfCheckerUnwanted(ctx android.ModuleContext, apexFile android.Outpu return timestamp } -func runApexHostVerifier(ctx android.ModuleContext, a *apexBundle, apexFile android.OutputPath) android.Path { +func runApexHostVerifier(ctx android.ModuleContext, a *apexBundle, apexFile android.Path) android.Path { timestamp := android.PathForModuleOut(ctx, "host_apex_verifier.timestamp") ctx.Build(pctx, android.BuildParams{ Rule: apexHostVerifierRule, diff --git a/apex/prebuilt.go b/apex/prebuilt.go index 9cd5688ba..acf3b91fe 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -386,7 +386,7 @@ type Prebuilt struct { inputApex android.Path - provenanceMetaDataFile android.OutputPath + provenanceMetaDataFile android.Path } type ApexFileProperties struct { @@ -697,7 +697,7 @@ func (p *Prebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.SetOutputFiles(android.Paths{p.outputApex}, "") } -func (p *Prebuilt) ProvenanceMetaDataFile() android.OutputPath { +func (p *Prebuilt) ProvenanceMetaDataFile() android.Path { return p.provenanceMetaDataFile } @@ -53,6 +53,13 @@ type CcObjectInfo struct { var CcObjectInfoProvider = blueprint.NewProvider[CcObjectInfo]() +type LinkableInfo struct { + // StaticExecutable returns true if this is a binary module with "static_executable: true". + StaticExecutable bool +} + +var LinkableInfoKey = blueprint.NewProvider[LinkableInfo]() + func init() { RegisterCCBuildComponents(android.InitRegistrationContext) @@ -410,11 +417,6 @@ type BaseProperties struct { // variant to have a ".sdk" suffix. SdkAndPlatformVariantVisibleToMake bool `blueprint:"mutated"` - // List of APEXes that this module has private access to for testing purpose. The module - // can depend on libraries that are not exported by the APEXes and use private symbols - // from the exported libraries. - Test_for []string `android:"arch_variant"` - Target struct { Platform struct { // List of modules required by the core variant. @@ -965,7 +967,6 @@ func (c *Module) AddJSONData(d *map[string]interface{}) { "IsLlndk": c.IsLlndk(), "IsVendorPublicLibrary": c.IsVendorPublicLibrary(), "ApexSdkVersion": c.apexSdkVersion, - "TestFor": c.TestFor(), "AidlSrcs": c.hasAidl, "LexSrcs": c.hasLex, "ProtoSrcs": c.hasProto, @@ -1566,12 +1567,11 @@ func (ctx *moduleContextImpl) minSdkVersion() string { } if ctx.ctx.Device() { - config := ctx.ctx.Config() - if ctx.inVendor() { - // If building for vendor with final API, then use the latest _stable_ API as "current". - if config.VendorApiLevelFrozen() && (ver == "" || ver == "current") { - ver = config.PlatformSdkVersion().String() - } + // When building for vendor/product, use the latest _stable_ API as "current". + // This is passed to clang/aidl compilers so that compiled/generated code works + // with the system. + if (ctx.inVendor() || ctx.inProduct()) && (ver == "" || ver == "current") { + ver = ctx.ctx.Config().PlatformSdkVersion().String() } } @@ -2126,6 +2126,10 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { android.SetProvider(ctx, CcObjectInfoProvider, ccObjectInfo) } + android.SetProvider(ctx, LinkableInfoKey, LinkableInfo{ + StaticExecutable: c.StaticExecutable(), + }) + c.setOutputFiles(ctx) if c.makeVarsInfo != nil { @@ -3360,8 +3364,6 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { } func ShouldUseStubForApex(ctx android.ModuleContext, dep android.Module) bool { - depName := ctx.OtherModuleName(dep) - inVendorOrProduct := false bootstrap := false if linkable, ok := ctx.Module().(LinkableInterface); !ok { @@ -3391,9 +3393,8 @@ func ShouldUseStubForApex(ctx android.ModuleContext, dep android.Module) bool { useStubs = isNotInPlatform && !bootstrap } else { - // If building for APEX, use stubs when the parent is in any APEX that - // the child is not in. - useStubs = !android.DirectlyInAllApexes(apexInfo, depName) + // If building for APEX, always use stubs (can be bypassed by depending on <dep>#impl) + useStubs = true } return useStubs @@ -3690,10 +3691,6 @@ func (c *Module) AvailableFor(what string) bool { } } -func (c *Module) TestFor() []string { - return c.Properties.Test_for -} - func (c *Module) EverInstallable() bool { return c.installer != nil && // Check to see whether the module is actually ever installable. diff --git a/cc/cc_test.go b/cc/cc_test.go index 989b043be..144b90b94 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -1008,7 +1008,7 @@ func TestLlndkLibrary(t *testing.T) { android.AssertArrayString(t, "variants for llndk stubs", expected, actual) params := result.ModuleForTests("libllndk", "android_vendor_arm_armv7-a-neon_shared").Description("generate stub") - android.AssertSame(t, "use Vendor API level for default stubs", "202404", params.Args["apiLevel"]) + android.AssertSame(t, "use Vendor API level for default stubs", "999999", params.Args["apiLevel"]) checkExportedIncludeDirs := func(module, variant string, expectedSystemDirs []string, expectedDirs ...string) { t.Helper() @@ -3153,7 +3153,7 @@ func TestImageVariants(t *testing.T) { testDepWithVariant("product") } -func TestVendorSdkVersion(t *testing.T) { +func TestVendorOrProductVariantUsesPlatformSdkVersionAsDefault(t *testing.T) { t.Parallel() bp := ` @@ -3161,31 +3161,29 @@ func TestVendorSdkVersion(t *testing.T) { name: "libfoo", srcs: ["libfoo.cc"], vendor_available: true, + product_available: true, } cc_library { name: "libbar", srcs: ["libbar.cc"], vendor_available: true, + product_available: true, min_sdk_version: "29", } ` ctx := prepareForCcTest.RunTestWithBp(t, bp) - testSdkVersionFlag := func(module, version string) { - flags := ctx.ModuleForTests(module, "android_vendor_arm64_armv8-a_static").Rule("cc").Args["cFlags"] - android.AssertStringDoesContain(t, "min sdk version", flags, "-target aarch64-linux-android"+version) + testSdkVersionFlag := func(module, variant, version string) { + flags := ctx.ModuleForTests(module, "android_"+variant+"_arm64_armv8-a_static").Rule("cc").Args["cFlags"] + android.AssertStringDoesContain(t, "target SDK version", flags, "-target aarch64-linux-android"+version) } - testSdkVersionFlag("libfoo", "10000") - testSdkVersionFlag("libbar", "29") - - ctx = android.GroupFixturePreparers( - prepareForCcTest, - android.PrepareForTestWithBuildFlag("RELEASE_BOARD_API_LEVEL_FROZEN", "true"), - ).RunTestWithBp(t, bp) - testSdkVersionFlag("libfoo", "30") - testSdkVersionFlag("libbar", "29") + testSdkVersionFlag("libfoo", "vendor", "30") + testSdkVersionFlag("libfoo", "product", "30") + // target SDK version can be set explicitly with min_sdk_version + testSdkVersionFlag("libbar", "vendor", "29") + testSdkVersionFlag("libbar", "product", "29") } func TestClangVerify(t *testing.T) { diff --git a/cc/cmake_snapshot.go b/cc/cmake_snapshot.go index 790a865d7..f553f27be 100644 --- a/cc/cmake_snapshot.go +++ b/cc/cmake_snapshot.go @@ -484,7 +484,7 @@ func (m *CmakeSnapshot) GenerateAndroidBuildActions(ctx android.ModuleContext) { // Packaging all make files into the zip file makefilesRspFile := android.PathForModuleObj(ctx, ctx.ModuleName()+"_makefiles.rsp") zipCmd. - FlagWithArg("-C ", android.PathForModuleGen(ctx).OutputPath.String()). + FlagWithArg("-C ", android.PathForModuleGen(ctx).String()). FlagWithRspFileInputList("-r ", makefilesRspFile, makefilesList) // Packaging all prebuilts into the zip file diff --git a/cc/compdb.go b/cc/compdb.go index b33f490f7..4132e090b 100644 --- a/cc/compdb.go +++ b/cc/compdb.go @@ -146,6 +146,8 @@ func getArguments(src android.Path, ctx android.SingletonContext, ccModule *Modu isAsm = false isCpp = true clangPath = cxxPath + case ".o": + return nil default: log.Print("Unknown file extension " + src.Ext() + " on file " + src.String()) isAsm = true @@ -185,6 +187,10 @@ func generateCompdbProject(compiledModule CompiledInterface, ctx android.Singlet } for _, src := range srcs { if _, ok := builds[src.String()]; !ok { + args := getArguments(src, ctx, ccModule, ccPath, cxxPath) + if args == nil { + continue + } builds[src.String()] = compDbEntry{ Directory: android.AbsSrcDirForExistingUseCases(), Arguments: getArguments(src, ctx, ccModule, ccPath, cxxPath), diff --git a/cc/config/global.go b/cc/config/global.go index 36690d6be..6984ea412 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -290,8 +290,6 @@ var ( "-Wno-error=deprecated", // in external/googletest/googletest // Disabling until the warning is fixed in libc++abi header files b/366180429 "-Wno-deprecated-dynamic-exception-spec", - // New warnings to be fixed after clang-r475365 - "-Wno-error=enum-constexpr-conversion", // http://b/243964282 // New warnings to be fixed after clang-r522817 "-Wno-error=invalid-offsetof", "-Wno-error=thread-safety-reference-return", diff --git a/cc/library.go b/cc/library.go index 7dffa7266..4ce506e90 100644 --- a/cc/library.go +++ b/cc/library.go @@ -566,16 +566,10 @@ func (library *libraryDecorator) getHeaderAbiCheckerProperties(m *Module) header func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects { if ctx.IsLlndk() { - vendorApiLevel := ctx.Config().VendorApiLevel() - if vendorApiLevel == "" { - // TODO(b/321892570): Some tests relying on old fixtures which - // doesn't set vendorApiLevel. Needs to fix them. - vendorApiLevel = ctx.Config().PlatformSdkVersion().String() - } - // This is the vendor variant of an LLNDK library, build the LLNDK stubs. + futureVendorApiLevel := android.ApiLevelOrPanic(ctx, "999999") nativeAbiResult := parseNativeAbiDefinition(ctx, String(library.Properties.Llndk.Symbol_file), - android.ApiLevelOrPanic(ctx, vendorApiLevel), "--llndk") + futureVendorApiLevel, "--llndk") objs := compileStubLibrary(ctx, flags, nativeAbiResult.stubSrc) if !Bool(library.Properties.Llndk.Unversioned) { library.versionScriptPath = android.OptionalPathForPath( diff --git a/cc/sanitize.go b/cc/sanitize.go index 124dda4ab..f0d734376 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -1813,7 +1813,7 @@ func memtagStackMakeVarsProvider(ctx android.MakeVarsContext) { type sanitizerLibrariesTxtModule struct { android.ModuleBase - outputFile android.OutputPath + outputFile android.Path } var _ etc.PrebuiltEtcModule = (*sanitizerLibrariesTxtModule)(nil) @@ -1896,13 +1896,14 @@ func (txt *sanitizerLibrariesTxtModule) getSanitizerLibs(ctx android.ModuleConte func (txt *sanitizerLibrariesTxtModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { filename := txt.Name() - txt.outputFile = android.PathForModuleOut(ctx, filename).OutputPath - android.WriteFileRule(ctx, txt.outputFile, txt.getSanitizerLibs(ctx)) + outputFile := android.PathForModuleOut(ctx, filename) + android.WriteFileRule(ctx, outputFile, txt.getSanitizerLibs(ctx)) installPath := android.PathForModuleInstall(ctx, "etc") - ctx.InstallFile(installPath, filename, txt.outputFile) + ctx.InstallFile(installPath, filename, outputFile) - ctx.SetOutputFiles(android.Paths{txt.outputFile}, "") + ctx.SetOutputFiles(android.Paths{outputFile}, "") + txt.outputFile = outputFile } func (txt *sanitizerLibrariesTxtModule) PrepareAndroidMKProviderInfo(config android.Config) *android.AndroidMkProviderInfo { diff --git a/cc/tidy.go b/cc/tidy.go index 5cbf8f076..6481b9537 100644 --- a/cc/tidy.go +++ b/cc/tidy.go @@ -220,7 +220,7 @@ func collectTidyObjModuleTargets(ctx android.SingletonContext, module android.Mo // (1) Collect all obj/tidy files into OS-specific groups. ctx.VisitAllModuleVariantProxies(module, func(variant android.ModuleProxy) { - osName := android.OtherModuleProviderOrDefault(ctx, variant, android.CommonPropertiesProviderKey).CompileTarget.Os.Name + osName := android.OtherModuleProviderOrDefault(ctx, variant, android.CommonModuleInfoKey).CompileTarget.Os.Name info := android.OtherModuleProviderOrDefault(ctx, variant, CcObjectInfoProvider) addToOSGroup(osName, info.objFiles, allObjFileGroups, subsetObjFileGroups) addToOSGroup(osName, info.tidyFiles, allTidyFileGroups, subsetTidyFileGroups) diff --git a/cmd/find_input_delta/find_input_delta/Android.bp b/cmd/find_input_delta/find_input_delta/Android.bp index 6b2dbc764..93a77083f 100644 --- a/cmd/find_input_delta/find_input_delta/Android.bp +++ b/cmd/find_input_delta/find_input_delta/Android.bp @@ -2,15 +2,15 @@ package { default_applicable_licenses: ["Android-Apache-2.0"], } -bootstrap_go_package { - name: "soong-cmd-find_input_delta-find_input_delta", - pkgPath: "android/soong/cmd/find_input_delta/find_input_delta", +blueprint_go_binary { + name: "find_input_delta", deps: [ "golang-protobuf-encoding-prototext", "golang-protobuf-reflect-protoreflect", "golang-protobuf-runtime-protoimpl", - "soong-cmd-find_input_delta-proto", "soong-cmd-find_input_delta-lib", + "soong-cmd-find_input_delta-proto", + "soong-cmd-find_input_delta-proto_internal", ], srcs: [ "main.go", diff --git a/cmd/find_input_delta/find_input_delta/main.go b/cmd/find_input_delta/find_input_delta/main.go index 6b657ea57..a8645843c 100644 --- a/cmd/find_input_delta/find_input_delta/main.go +++ b/cmd/find_input_delta/find_input_delta/main.go @@ -85,4 +85,11 @@ func main() { if err = file_list.Format(os.Stdout, template); err != nil { panic(err) } + + metrics_file := os.Getenv("SOONG_METRICS_AGGREGATION_FILE") + if metrics_file != "" { + if err = file_list.SendMetrics(metrics_file); err != nil { + panic(err) + } + } } diff --git a/cmd/find_input_delta/find_input_delta_lib/Android.bp b/cmd/find_input_delta/find_input_delta_lib/Android.bp index 795b14038..ef9c65b1d 100644 --- a/cmd/find_input_delta/find_input_delta_lib/Android.bp +++ b/cmd/find_input_delta/find_input_delta_lib/Android.bp @@ -24,6 +24,8 @@ bootstrap_go_package { "golang-protobuf-reflect-protoreflect", "golang-protobuf-runtime-protoimpl", "soong-cmd-find_input_delta-proto", + "soong-cmd-find_input_delta-proto_internal", + "android-archive-zip", "blueprint-pathtools", ], srcs: [ diff --git a/cmd/find_input_delta/find_input_delta_lib/file_list.go b/cmd/find_input_delta/find_input_delta_lib/file_list.go index 23337adc3..01242a0e7 100644 --- a/cmd/find_input_delta/find_input_delta_lib/file_list.go +++ b/cmd/find_input_delta/find_input_delta_lib/file_list.go @@ -15,10 +15,15 @@ package find_input_delta_lib import ( + "fmt" "io" + "os" + "path/filepath" + "slices" "text/template" fid_exp "android/soong/cmd/find_input_delta/find_input_delta_proto" + "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" ) @@ -47,28 +52,148 @@ type FileList struct { // The modified files Changes []FileList + + // Map of file_extension:counts + ExtCountMap map[string]*FileCounts + + // Total number of added/changed/deleted files. + TotalDelta uint32 +} + +// The maximum number of files that will be recorded by name. +var MaxFilesRecorded uint32 = 50 + +type FileCounts struct { + Additions uint32 + Deletions uint32 + Changes uint32 +} + +func FileListFactory(name string) *FileList { + return &FileList{ + Name: name, + ExtCountMap: make(map[string]*FileCounts), + } +} + +func (fl *FileList) addFile(name string) { + fl.Additions = append(fl.Additions, name) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Additions += 1 +} + +func (fl *FileList) deleteFile(name string) { + fl.Deletions = append(fl.Deletions, name) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Deletions += 1 +} + +func (fl *FileList) changeFile(name string, ch *FileList) { + fl.Changes = append(fl.Changes, *ch) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Changes += 1 +} + +func (fl FileList) ToProto() (*fid_exp.FileList, error) { + var count uint32 + return fl.toProto(&count) } -func (fl FileList) Marshal() (*fid_exp.FileList, error) { +func (fl FileList) toProto(count *uint32) (*fid_exp.FileList, error) { ret := &fid_exp.FileList{ Name: proto.String(fl.Name), } - if len(fl.Additions) > 0 { - ret.Additions = fl.Additions + for _, a := range fl.Additions { + if *count >= MaxFilesRecorded { + break + } + ret.Additions = append(ret.Additions, a) + *count += 1 } for _, ch := range fl.Changes { - change, err := ch.Marshal() - if err != nil { - return nil, err + if *count >= MaxFilesRecorded { + break + } else { + // Pre-increment to limit what the call adds. + *count += 1 + change, err := ch.toProto(count) + if err != nil { + return nil, err + } + ret.Changes = append(ret.Changes, change) + } + } + for _, d := range fl.Deletions { + if *count >= MaxFilesRecorded { + break } - ret.Changes = append(ret.Changes, change) + ret.Deletions = append(ret.Deletions, d) } - if len(fl.Deletions) > 0 { - ret.Deletions = fl.Deletions + ret.TotalDelta = proto.Uint32(*count) + exts := []string{} + for k := range fl.ExtCountMap { + exts = append(exts, k) + } + slices.Sort(exts) + for _, k := range exts { + v := fl.ExtCountMap[k] + ret.Counts = append(ret.Counts, &fid_exp.FileCount{ + Extension: proto.String(k), + Additions: proto.Uint32(v.Additions), + Deletions: proto.Uint32(v.Deletions), + Modifications: proto.Uint32(v.Changes), + }) } return ret, nil } +func (fl FileList) SendMetrics(path string) error { + if path == "" { + return fmt.Errorf("No path given") + } + message, err := fl.ToProto() + if err != nil { + return err + } + + // Marshal the message wrapped in SoongCombinedMetrics. + data := protowire.AppendVarint( + []byte{}, + protowire.EncodeTag( + protowire.Number(fid_exp.FieldNumbers_FIELD_NUMBERS_FILE_LIST), + protowire.BytesType)) + size := uint64(proto.Size(message)) + data = protowire.AppendVarint(data, size) + data, err = proto.MarshalOptions{UseCachedSize: true}.MarshalAppend(data, message) + if err != nil { + return err + } + + out, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if err := out.Close(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to close %s: %v\n", path, err) + } + }() + _, err = out.Write(data) + return err +} + func (fl FileList) Format(wr io.Writer, format string) error { tmpl, err := template.New("filelist").Parse(format) if err != nil { diff --git a/cmd/find_input_delta/find_input_delta_lib/fs.go b/cmd/find_input_delta/find_input_delta_lib/fs.go index 4a83ed7ff..09a8aa6cb 100644 --- a/cmd/find_input_delta/find_input_delta_lib/fs.go +++ b/cmd/find_input_delta/find_input_delta_lib/fs.go @@ -15,7 +15,6 @@ package find_input_delta_lib import ( - "io" "io/fs" "os" ) @@ -30,14 +29,6 @@ type fileSystem interface { ReadFile(path string) ([]byte, error) } -type file interface { - io.Closer - io.Reader - io.ReaderAt - io.Seeker - Stat() (os.FileInfo, error) -} - // osFS implements fileSystem using the local disk. type osFS struct{} diff --git a/cmd/find_input_delta/find_input_delta_lib/internal_state.go b/cmd/find_input_delta/find_input_delta_lib/internal_state.go index b2ff8c704..f0242b724 100644 --- a/cmd/find_input_delta/find_input_delta_lib/internal_state.go +++ b/cmd/find_input_delta/find_input_delta_lib/internal_state.go @@ -18,9 +18,11 @@ import ( "errors" "fmt" "io/fs" + "path/filepath" "slices" fid_proto "android/soong/cmd/find_input_delta/find_input_delta_proto_internal" + "android/soong/third_party/zip" "github.com/google/blueprint/pathtools" "google.golang.org/protobuf/proto" ) @@ -57,6 +59,7 @@ func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (* // If we ever have an easy hash, assign it here. } if inspect_contents { + // NOTE: When we find it useful, we can parallelize the file inspection for speed. contents, err := InspectFileContents(input) if err != nil { return ret, err @@ -73,9 +76,30 @@ func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (* // Inspect the file and extract the state of the elements in the archive. // If this is not an archive of some sort, nil is returned. func InspectFileContents(name string) ([]*fid_proto.PartialCompileInput, error) { - // TODO: Actually inspect the contents. - fmt.Printf("inspecting contents for %s\n", name) - return nil, nil + switch filepath.Ext(name) { + case ".jar", ".apex", ".apk": + return inspectZipFileContents(name) + default: + return nil, nil + } +} + +func inspectZipFileContents(name string) ([]*fid_proto.PartialCompileInput, error) { + rc, err := zip.OpenReader(name) + if err != nil { + return nil, err + } + ret := []*fid_proto.PartialCompileInput{} + for _, v := range rc.File { + pci := &fid_proto.PartialCompileInput{ + Name: proto.String(v.Name), + MtimeNsec: proto.Int64(v.ModTime().UnixNano()), + Hash: proto.String(fmt.Sprintf("%08x", v.CRC32)), + } + ret = append(ret, pci) + // We do not support nested inspection. + } + return ret, nil } func WriteState(s *fid_proto.PartialCompileInputs, path string) error { @@ -91,9 +115,7 @@ func CompareInternalState(prior, other *fid_proto.PartialCompileInputs, target s } func CompareInputFiles(prior, other []*fid_proto.PartialCompileInput, name string) *FileList { - fl := &FileList{ - Name: name, - } + fl := FileListFactory(name) PriorMap := make(map[string]*fid_proto.PartialCompileInput, len(prior)) // We know that the lists are properly sorted, so we can simply compare them. for _, v := range prior { @@ -105,17 +127,17 @@ func CompareInputFiles(prior, other []*fid_proto.PartialCompileInput, name strin otherMap[name] = v if _, ok := PriorMap[name]; !ok { // Added file - fl.Additions = append(fl.Additions, name) + fl.addFile(name) } else if !proto.Equal(PriorMap[name], v) { // Changed file - fl.Changes = append(fl.Changes, *CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name)) + fl.changeFile(name, CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name)) } } for _, v := range prior { name := v.GetName() if _, ok := otherMap[name]; !ok { // Deleted file - fl.Deletions = append(fl.Deletions, name) + fl.deleteFile(name) } } return fl diff --git a/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go b/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go index 20b8efaa8..e69424c54 100644 --- a/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go +++ b/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go @@ -199,7 +199,7 @@ func TestCompareInternalState(t *testing.T) { }, }, { - Name: "one of each", + Name: "one each add modify delete", Target: "foo", Prior: &fid_proto.PartialCompileInputs{ InputFiles: []*fid_proto.PartialCompileInput{ @@ -222,11 +222,40 @@ func TestCompareInternalState(t *testing.T) { Deletions: []string{"file2"}, }, }, + { + Name: "interior one each add modify delete", + Target: "bar", + Prior: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file1", 405, "", []*fid_proto.PartialCompileInput{ + protoFile("innerC", 400, "crc32:11111111", nil), + protoFile("innerD", 400, "crc32:44444444", nil), + }), + }, + }, + New: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file1", 505, "", []*fid_proto.PartialCompileInput{ + protoFile("innerA", 400, "crc32:55555555", nil), + protoFile("innerC", 500, "crc32:66666666", nil), + }), + }, + }, + Expected: &FileList{ + Name: "bar", + Changes: []FileList{FileList{ + Name: "file1", + Additions: []string{"innerA"}, + Changes: []FileList{FileList{Name: "innerC"}}, + Deletions: []string{"innerD"}, + }}, + }, + }, } for _, tc := range testCases { actual := CompareInternalState(tc.Prior, tc.New, tc.Target) if !tc.Expected.Equal(actual) { - t.Errorf("%s: expected %q, actual %q", tc.Name, tc.Expected, actual) + t.Errorf("%s: expected %v, actual %v", tc.Name, tc.Expected, actual) } } } diff --git a/cmd/find_input_delta/find_input_delta_proto/Android.bp b/cmd/find_input_delta/find_input_delta_proto/Android.bp index 18eba6bb8..1a05b9e29 100644 --- a/cmd/find_input_delta/find_input_delta_proto/Android.bp +++ b/cmd/find_input_delta/find_input_delta_proto/Android.bp @@ -24,6 +24,6 @@ bootstrap_go_package { "golang-protobuf-runtime-protoimpl", ], srcs: [ - "target_delta_files.pb.go", + "file_list.pb.go", ], } diff --git a/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go b/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go index 648ef224a..745de2db2 100644 --- a/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go +++ b/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go @@ -19,7 +19,7 @@ // protoc v3.21.12 // source: file_list.proto -package proto +package find_input_delta_proto import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -35,6 +35,62 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type FieldNumbers int32 + +const ( + FieldNumbers_FIELD_NUMBERS_UNSPECIFIED FieldNumbers = 0 + FieldNumbers_FIELD_NUMBERS_FILE_LIST FieldNumbers = 1 +) + +// Enum value maps for FieldNumbers. +var ( + FieldNumbers_name = map[int32]string{ + 0: "FIELD_NUMBERS_UNSPECIFIED", + 1: "FIELD_NUMBERS_FILE_LIST", + } + FieldNumbers_value = map[string]int32{ + "FIELD_NUMBERS_UNSPECIFIED": 0, + "FIELD_NUMBERS_FILE_LIST": 1, + } +) + +func (x FieldNumbers) Enum() *FieldNumbers { + p := new(FieldNumbers) + *p = x + return p +} + +func (x FieldNumbers) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FieldNumbers) Descriptor() protoreflect.EnumDescriptor { + return file_file_list_proto_enumTypes[0].Descriptor() +} + +func (FieldNumbers) Type() protoreflect.EnumType { + return &file_file_list_proto_enumTypes[0] +} + +func (x FieldNumbers) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *FieldNumbers) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = FieldNumbers(num) + return nil +} + +// Deprecated: Use FieldNumbers.Descriptor instead. +func (FieldNumbers) EnumDescriptor() ([]byte, []int) { + return file_file_list_proto_rawDescGZIP(), []int{0} +} + type FileList struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -46,10 +102,14 @@ type FileList struct { Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` // The added files. Additions []string `protobuf:"bytes,2,rep,name=additions" json:"additions,omitempty"` - // The deleted files. - Deletions []string `protobuf:"bytes,3,rep,name=deletions" json:"deletions,omitempty"` // The changed files. - Changes []*FileList `protobuf:"bytes,4,rep,name=changes" json:"changes,omitempty"` + Changes []*FileList `protobuf:"bytes,3,rep,name=changes" json:"changes,omitempty"` + // The deleted files. + Deletions []string `protobuf:"bytes,4,rep,name=deletions" json:"deletions,omitempty"` + // Count of files added/changed/deleted. + TotalDelta *uint32 `protobuf:"varint,5,opt,name=total_delta,json=totalDelta" json:"total_delta,omitempty"` + // Counts by extension. + Counts []*FileCount `protobuf:"bytes,6,rep,name=counts" json:"counts,omitempty"` } func (x *FileList) Reset() { @@ -98,6 +158,13 @@ func (x *FileList) GetAdditions() []string { return nil } +func (x *FileList) GetChanges() []*FileList { + if x != nil { + return x.Changes + } + return nil +} + func (x *FileList) GetDeletions() []string { if x != nil { return x.Deletions @@ -105,32 +172,135 @@ func (x *FileList) GetDeletions() []string { return nil } -func (x *FileList) GetChanges() []*FileList { +func (x *FileList) GetTotalDelta() uint32 { + if x != nil && x.TotalDelta != nil { + return *x.TotalDelta + } + return 0 +} + +func (x *FileList) GetCounts() []*FileCount { if x != nil { - return x.Changes + return x.Counts } return nil } +type FileCount struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The file extension + Extension *string `protobuf:"bytes,1,opt,name=extension" json:"extension,omitempty"` + // Number of added files with this extension. + Additions *uint32 `protobuf:"varint,2,opt,name=additions" json:"additions,omitempty"` + // Number of modified files with this extension. + Modifications *uint32 `protobuf:"varint,3,opt,name=modifications" json:"modifications,omitempty"` + // Number of deleted files with this extension. + Deletions *uint32 `protobuf:"varint,4,opt,name=deletions" json:"deletions,omitempty"` +} + +func (x *FileCount) Reset() { + *x = FileCount{} + if protoimpl.UnsafeEnabled { + mi := &file_file_list_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileCount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileCount) ProtoMessage() {} + +func (x *FileCount) ProtoReflect() protoreflect.Message { + mi := &file_file_list_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileCount.ProtoReflect.Descriptor instead. +func (*FileCount) Descriptor() ([]byte, []int) { + return file_file_list_proto_rawDescGZIP(), []int{1} +} + +func (x *FileCount) GetExtension() string { + if x != nil && x.Extension != nil { + return *x.Extension + } + return "" +} + +func (x *FileCount) GetAdditions() uint32 { + if x != nil && x.Additions != nil { + return *x.Additions + } + return 0 +} + +func (x *FileCount) GetModifications() uint32 { + if x != nil && x.Modifications != nil { + return *x.Modifications + } + return 0 +} + +func (x *FileCount) GetDeletions() uint32 { + if x != nil && x.Deletions != nil { + return *x.Deletions + } + return 0 +} + var File_file_list_proto protoreflect.FileDescriptor var file_file_list_proto_rawDesc = []byte{ 0x0a, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0x9e, 0x01, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x12, + 0x6f, 0x22, 0x82, 0x02, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x42, - 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, - 0x65, 0x73, 0x42, 0x26, 0x5a, 0x24, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, - 0x6f, 0x6e, 0x67, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, - 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x42, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, + 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x6e, 0x67, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x64, 0x65, 0x6c, 0x74, + 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x65, + 0x6c, 0x74, 0x61, 0x12, 0x41, 0x0a, 0x06, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, + 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x06, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x2a, 0x4a, 0x0a, 0x0c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x75, 0x6d, + 0x62, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x4e, 0x55, + 0x4d, 0x42, 0x45, 0x52, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x4e, 0x55, 0x4d, + 0x42, 0x45, 0x52, 0x53, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x01, + 0x42, 0x3b, 0x5a, 0x39, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, + 0x67, 0x2f, 0x63, 0x6d, 0x64, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -145,17 +315,21 @@ func file_file_list_proto_rawDescGZIP() []byte { return file_file_list_proto_rawDescData } -var file_file_list_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_file_list_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_file_list_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_file_list_proto_goTypes = []interface{}{ - (*FileList)(nil), // 0: android.find_input_delta_proto.FileList + (FieldNumbers)(0), // 0: android.find_input_delta_proto.FieldNumbers + (*FileList)(nil), // 1: android.find_input_delta_proto.FileList + (*FileCount)(nil), // 2: android.find_input_delta_proto.FileCount } var file_file_list_proto_depIdxs = []int32{ - 0, // 0: android.find_input_delta_proto.FileList.changes:type_name -> android.find_input_delta_proto.FileList - 1, // [1:1] is the sub-list for method output_type - 1, // [1:1] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 1, // 0: android.find_input_delta_proto.FileList.changes:type_name -> android.find_input_delta_proto.FileList + 2, // 1: android.find_input_delta_proto.FileList.counts:type_name -> android.find_input_delta_proto.FileCount + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_file_list_proto_init() } @@ -176,19 +350,32 @@ func file_file_list_proto_init() { return nil } } + file_file_list_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileCount); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_file_list_proto_rawDesc, - NumEnums: 0, - NumMessages: 1, + NumEnums: 1, + NumMessages: 2, NumExtensions: 0, NumServices: 0, }, GoTypes: file_file_list_proto_goTypes, DependencyIndexes: file_file_list_proto_depIdxs, + EnumInfos: file_file_list_proto_enumTypes, MessageInfos: file_file_list_proto_msgTypes, }.Build() File_file_list_proto = out.File diff --git a/cmd/find_input_delta/find_input_delta_proto/file_list.proto b/cmd/find_input_delta/find_input_delta_proto/file_list.proto index d7faca9c8..7180358c8 100644 --- a/cmd/find_input_delta/find_input_delta_proto/file_list.proto +++ b/cmd/find_input_delta/find_input_delta_proto/file_list.proto @@ -15,7 +15,12 @@ syntax = "proto2"; package android.find_input_delta_proto; -option go_package = "android/soong/find_input_delta/proto"; +option go_package = "android/soong/cmd/find_input_delta/find_input_delta_proto"; + +enum FieldNumbers { + FIELD_NUMBERS_UNSPECIFIED = 0; + FIELD_NUMBERS_FILE_LIST = 1; +} message FileList { // The name of the file. @@ -26,9 +31,29 @@ message FileList { // The added files. repeated string additions = 2; + // The changed files. + repeated FileList changes = 3; + // The deleted files. - repeated string deletions = 3; + repeated string deletions = 4; - // The changed files. - repeated FileList changes = 4; + // Count of files added/changed/deleted. + optional uint32 total_delta = 5; + + // Counts by extension. + repeated FileCount counts = 6; +} + +message FileCount { + // The file extension + optional string extension = 1; + + // Number of added files with this extension. + optional uint32 additions = 2; + + // Number of modified files with this extension. + optional uint32 modifications = 3; + + // Number of deleted files with this extension. + optional uint32 deletions = 4; } diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go index 2229a324d..c5b048bb8 100644 --- a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go +++ b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go @@ -19,7 +19,7 @@ // protoc v3.21.12 // source: internal_state.proto -package proto +package find_input_delta_proto_internal import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -94,7 +94,7 @@ type PartialCompileInput struct { Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` // The timestamp of the file in (Unix) nanoseconds. MtimeNsec *int64 `protobuf:"varint,2,opt,name=mtime_nsec,json=mtimeNsec" json:"mtime_nsec,omitempty"` - // The hash of the file, in the form ‘{HASHNAME}:{VALUE}’ + // The hash of the file. For crc32 hashes, this will be 8 hex digits. Hash *string `protobuf:"bytes,3,opt,name=hash" json:"hash,omitempty"` // Contents of the file, if the file was inspected (such as jar files, etc). Contents []*PartialCompileInput `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty"` @@ -164,29 +164,33 @@ var File_internal_state_proto protoreflect.FileDescriptor var file_internal_state_proto_rawDesc = []byte{ 0x0a, 0x14, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x27, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6c, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, - 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x54, - 0x0a, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, - 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, - 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x46, - 0x69, 0x6c, 0x65, 0x73, 0x22, 0xad, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, - 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, 0x63, 0x12, - 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, - 0x61, 0x73, 0x68, 0x12, 0x4f, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, - 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, - 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x73, 0x42, 0x26, 0x5a, 0x24, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, - 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, - 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x22, + 0x75, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, + 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x5d, 0x0a, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x61, + 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, + 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x22, 0xb6, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69, + 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, + 0x63, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x58, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, + 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, + 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, + 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x42, + 0x40, 0x5a, 0x3e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, + 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, + 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, + 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, } var ( @@ -203,12 +207,12 @@ func file_internal_state_proto_rawDescGZIP() []byte { var file_internal_state_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_internal_state_proto_goTypes = []interface{}{ - (*PartialCompileInputs)(nil), // 0: android.find_input_delta_proto.PartialCompileInputs - (*PartialCompileInput)(nil), // 1: android.find_input_delta_proto.PartialCompileInput + (*PartialCompileInputs)(nil), // 0: android.find_input_delta_proto_internal.PartialCompileInputs + (*PartialCompileInput)(nil), // 1: android.find_input_delta_proto_internal.PartialCompileInput } var file_internal_state_proto_depIdxs = []int32{ - 1, // 0: android.find_input_delta_proto.PartialCompileInputs.input_files:type_name -> android.find_input_delta_proto.PartialCompileInput - 1, // 1: android.find_input_delta_proto.PartialCompileInput.contents:type_name -> android.find_input_delta_proto.PartialCompileInput + 1, // 0: android.find_input_delta_proto_internal.PartialCompileInputs.input_files:type_name -> android.find_input_delta_proto_internal.PartialCompileInput + 1, // 1: android.find_input_delta_proto_internal.PartialCompileInput.contents:type_name -> android.find_input_delta_proto_internal.PartialCompileInput 2, // [2:2] is the sub-list for method output_type 2, // [2:2] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto index 113fc6434..54c90cc72 100644 --- a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto +++ b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto @@ -14,8 +14,8 @@ // limitations under the License. syntax = "proto2"; -package android.find_input_delta_proto; -option go_package = "android/soong/find_input_delta/proto"; +package android.find_input_delta_proto_internal; +option go_package = "android/soong/find_input_delta/find_input_delta_proto_internal"; // The state of all inputs. message PartialCompileInputs { @@ -31,7 +31,7 @@ message PartialCompileInput { // The timestamp of the file in (Unix) nanoseconds. optional int64 mtime_nsec = 2; - // The hash of the file, in the form ‘{HASHNAME}:{VALUE}’ + // The hash of the file. For crc32 hashes, this will be 8 hex digits. optional string hash = 3; // Contents of the file, if the file was inspected (such as jar files, etc). diff --git a/cmd/release_config/build_flag/main.go b/cmd/release_config/build_flag/main.go index 46efce7f7..5d103cc25 100644 --- a/cmd/release_config/build_flag/main.go +++ b/cmd/release_config/build_flag/main.go @@ -90,6 +90,9 @@ func MarshalFlagValue(config *rc_lib.ReleaseConfig, name string) (ret string, er if !ok { return "", fmt.Errorf("%s not found in %s", name, config.Name) } + if fa.Redacted { + return "==REDACTED==", nil + } return rc_lib.MarshalValue(fa.Value), nil } diff --git a/cmd/release_config/release_config_lib/flag_artifact.go b/cmd/release_config/release_config_lib/flag_artifact.go index 51c02d2dc..cb13fdce4 100644 --- a/cmd/release_config/release_config_lib/flag_artifact.go +++ b/cmd/release_config/release_config_lib/flag_artifact.go @@ -84,8 +84,10 @@ func FlagArtifactsFactory(artifactsPath string) *FlagArtifacts { func (fas *FlagArtifacts) SortedFlagNames() []string { var names []string - for k, _ := range *fas { - names = append(names, k) + for k, v := range *fas { + if !v.Redacted { + names = append(names, k) + } } slices.Sort(names) return names @@ -99,6 +101,9 @@ func (fa *FlagArtifact) GenerateFlagDeclarationArtifact() *rc_proto.FlagDeclarat if namespace := fa.FlagDeclaration.GetNamespace(); namespace != "" { ret.Namespace = proto.String(namespace) } + if bugs := fa.FlagDeclaration.GetBugs(); bugs != nil { + ret.Bugs = bugs + } if description := fa.FlagDeclaration.GetDescription(); description != "" { ret.Description = proto.String(description) } @@ -180,15 +185,20 @@ func (src FlagArtifacts) Clone() (dst FlagArtifacts) { // error: any error encountered func (fa *FlagArtifact) UpdateValue(flagValue FlagValue) error { name := *flagValue.proto.Name - fa.Traces = append(fa.Traces, &rc_proto.Tracepoint{Source: proto.String(flagValue.path), Value: flagValue.proto.Value}) - if flagValue.proto.GetRedacted() { - fa.Redacted = true + redacted := flagValue.proto.GetRedacted() + if redacted { + fa.Redact() + flagValue.proto.Value = fa.Value fmt.Printf("Redacting flag %s in %s\n", name, flagValue.path) - return nil + } else { + // If we are assigning a value, then the flag is no longer redacted. + fa.Redacted = false } + fa.Traces = append(fa.Traces, &rc_proto.Tracepoint{Source: proto.String(flagValue.path), Value: flagValue.proto.Value}) if fa.Value.GetObsolete() { return fmt.Errorf("Attempting to set obsolete flag %s. Trace=%v", name, fa.Traces) } + var newValue *rc_proto.Value switch val := flagValue.proto.Value.Val.(type) { case *rc_proto.Value_StringValue: @@ -210,6 +220,11 @@ func (fa *FlagArtifact) UpdateValue(flagValue FlagValue) error { return nil } +func (fa *FlagArtifact) Redact() { + fa.Redacted = true + fa.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{StringValue: "*REDACTED*"}} +} + // Marshal the FlagArtifact into a flag_artifact message. func (fa *FlagArtifact) Marshal() (*rc_proto.FlagArtifact, error) { if fa.Redacted { diff --git a/cmd/release_config/release_config_lib/release_config.go b/cmd/release_config/release_config_lib/release_config.go index ee71336c5..3c834cc1d 100644 --- a/cmd/release_config/release_config_lib/release_config.go +++ b/cmd/release_config/release_config_lib/release_config.go @@ -21,7 +21,6 @@ import ( "path/filepath" "regexp" "slices" - "sort" "strings" rc_proto "android/soong/cmd/release_config/release_config_proto" @@ -107,6 +106,9 @@ func (config *ReleaseConfig) InheritConfig(iConfig *ReleaseConfig) error { if !ok { return fmt.Errorf("Could not inherit flag %s from %s", name, iConfig.Name) } + if fa.Redacted { + myFa.Redact() + } if name == "RELEASE_ACONFIG_VALUE_SETS" { // If there is a value assigned, add the trace. if len(fa.Value.GetStringValue()) > 0 { @@ -166,6 +168,13 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro } myInherits = append(myInherits, inherit) myInheritsSet[inherit] = true + // TODO: there are some configs that rely on vgsbr being + // present on branches where it isn't. Once the broken configs + // are fixed, we can be more strict. In the meantime, they + // will wind up inheriting `trunk_stable` instead of the + // non-existent (alias) that they reference today. Once fixed, + // this becomes: + // iConfig, err := configs.GetReleaseConfigStrict(inherit) iConfig, err := configs.GetReleaseConfig(inherit) if err != nil { return err @@ -261,9 +270,6 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro if err := fa.UpdateValue(*value); err != nil { return err } - if fa.Redacted { - delete(config.FlagArtifacts, name) - } } } // Now remove any duplicates from the actual value of RELEASE_ACONFIG_VALUE_SETS @@ -313,6 +319,10 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro if err != nil { return err } + // Redacted flags return nil when rendered. + if artifact == nil { + continue + } for _, container := range v.FlagDeclaration.Containers { if _, ok := config.PartitionBuildFlags[container]; !ok { config.PartitionBuildFlags[container] = &rc_proto.FlagArtifacts{} @@ -325,12 +335,7 @@ func (config *ReleaseConfig) GenerateReleaseConfig(configs *ReleaseConfigs) erro OtherNames: config.OtherNames, Flags: func() []*rc_proto.FlagArtifact { ret := []*rc_proto.FlagArtifact{} - flagNames := []string{} - for k := range config.FlagArtifacts { - flagNames = append(flagNames, k) - } - sort.Strings(flagNames) - for _, flagName := range flagNames { + for _, flagName := range config.FlagArtifacts.SortedFlagNames() { flag := config.FlagArtifacts[flagName] ret = append(ret, &rc_proto.FlagArtifact{ FlagDeclaration: flag.FlagDeclaration, @@ -365,7 +370,7 @@ func (config *ReleaseConfig) WriteMakefile(outFile, targetRelease string, config } } for _, rcName := range extraAconfigReleaseConfigs { - rc, err := configs.GetReleaseConfig(rcName) + rc, err := configs.GetReleaseConfigStrict(rcName) if err != nil { return err } diff --git a/cmd/release_config/release_config_lib/release_configs.go b/cmd/release_config/release_config_lib/release_configs.go index 831ec02f2..f947b7f1f 100644 --- a/cmd/release_config/release_config_lib/release_configs.go +++ b/cmd/release_config/release_config_lib/release_configs.go @@ -410,6 +410,14 @@ func (configs *ReleaseConfigs) LoadReleaseConfigMap(path string, ConfigDirIndex } func (configs *ReleaseConfigs) GetReleaseConfig(name string) (*ReleaseConfig, error) { + return configs.getReleaseConfig(name, configs.allowMissing) +} + +func (configs *ReleaseConfigs) GetReleaseConfigStrict(name string) (*ReleaseConfig, error) { + return configs.getReleaseConfig(name, false) +} + +func (configs *ReleaseConfigs) getReleaseConfig(name string, allow_missing bool) (*ReleaseConfig, error) { trace := []string{name} for target, ok := configs.Aliases[name]; ok; target, ok = configs.Aliases[name] { name = *target @@ -418,7 +426,7 @@ func (configs *ReleaseConfigs) GetReleaseConfig(name string) (*ReleaseConfig, er if config, ok := configs.ReleaseConfigs[name]; ok { return config, nil } - if configs.allowMissing { + if allow_missing { if config, ok := configs.ReleaseConfigs["trunk_staging"]; ok { return config, nil } @@ -467,7 +475,7 @@ func (configs *ReleaseConfigs) GenerateReleaseConfigs(targetRelease string) erro dirName := filepath.Dir(contrib.path) for k, names := range contrib.FlagValueDirs { for _, rcName := range names { - if config, err := configs.GetReleaseConfig(rcName); err == nil { + if config, err := configs.getReleaseConfig(rcName, false); err == nil { rcPath := filepath.Join(dirName, "release_configs", fmt.Sprintf("%s.textproto", config.Name)) if _, err := os.Stat(rcPath); err != nil { errors = append(errors, fmt.Sprintf("%s exists but %s does not contribute to %s", diff --git a/cmd/release_config/release_config_proto/build_flags_declarations.pb.go b/cmd/release_config/release_config_proto/build_flags_declarations.pb.go index 7db945a64..1df246ce0 100644 --- a/cmd/release_config/release_config_proto/build_flags_declarations.pb.go +++ b/cmd/release_config/release_config_proto/build_flags_declarations.pb.go @@ -48,6 +48,8 @@ type FlagDeclarationArtifact struct { Namespace *string `protobuf:"bytes,2,opt,name=namespace" json:"namespace,omitempty"` // Text description of the flag's purpose. Description *string `protobuf:"bytes,3,opt,name=description" json:"description,omitempty"` + // The bug number associated with the flag. + Bugs []string `protobuf:"bytes,4,rep,name=bugs" json:"bugs,omitempty"` // Where the flag was declared. DeclarationPath *string `protobuf:"bytes,5,opt,name=declaration_path,json=declarationPath" json:"declaration_path,omitempty"` // Workflow for this flag. @@ -110,6 +112,13 @@ func (x *FlagDeclarationArtifact) GetDescription() string { return "" } +func (x *FlagDeclarationArtifact) GetBugs() []string { + if x != nil { + return x.Bugs + } + return nil +} + func (x *FlagDeclarationArtifact) GetDeclarationPath() string { if x != nil && x.DeclarationPath != nil { return *x.DeclarationPath @@ -187,37 +196,38 @@ var file_build_flags_declarations_proto_rawDesc = []byte{ 0x12, 0x1c, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x18, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, - 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8c, 0x02, 0x0a, 0x17, 0x46, 0x6c, 0x61, + 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x9a, 0x02, 0x0a, 0x17, 0x46, 0x6c, 0x61, 0x67, 0x44, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x10, 0x64, 0x65, 0x63, 0x6c, - 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0f, 0x64, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, - 0x61, 0x74, 0x68, 0x12, 0x43, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x18, - 0xcd, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, - 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x08, - 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x1f, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0xce, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, - 0x06, 0x08, 0xcf, 0x01, 0x10, 0xd0, 0x01, 0x22, 0x96, 0x01, 0x0a, 0x18, 0x46, 0x6c, 0x61, 0x67, - 0x44, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x73, 0x12, 0x7a, 0x0a, 0x1e, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x64, 0x65, 0x63, - 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x61, - 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x6c, 0x61, 0x67, - 0x44, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x52, 0x1b, 0x66, 0x6c, 0x61, 0x67, 0x44, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4c, 0x69, 0x73, 0x74, - 0x42, 0x33, 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, - 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x75, 0x67, 0x73, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x62, 0x75, 0x67, 0x73, 0x12, 0x29, 0x0a, 0x10, + 0x64, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x64, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x50, 0x61, 0x74, 0x68, 0x12, 0x43, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, + 0x6c, 0x6f, 0x77, 0x18, 0xcd, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x61, 0x6e, 0x64, + 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x1f, 0x0a, 0x0a, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0xce, 0x01, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x4a, 0x06, 0x08, + 0xcf, 0x01, 0x10, 0xd0, 0x01, 0x22, 0x96, 0x01, 0x0a, 0x18, 0x46, 0x6c, 0x61, 0x67, 0x44, 0x65, + 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x73, 0x12, 0x7a, 0x0a, 0x1e, 0x66, 0x6c, 0x61, 0x67, 0x5f, 0x64, 0x65, 0x63, 0x6c, 0x61, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, + 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x61, 0x6e, 0x64, + 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x6c, 0x61, 0x67, 0x44, 0x65, + 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x52, 0x1b, 0x66, 0x6c, 0x61, 0x67, 0x44, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x33, + 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, + 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x72, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, } var ( diff --git a/cmd/release_config/release_config_proto/build_flags_declarations.proto b/cmd/release_config/release_config_proto/build_flags_declarations.proto index d755e02ab..ccdccfbaf 100644 --- a/cmd/release_config/release_config_proto/build_flags_declarations.proto +++ b/cmd/release_config/release_config_proto/build_flags_declarations.proto @@ -51,8 +51,8 @@ message FlagDeclarationArtifact { // Text description of the flag's purpose. optional string description = 3; - // reserve this for bug, if needed. - reserved 4; + // The bug number associated with the flag. + repeated string bugs = 4; // Where the flag was declared. optional string declaration_path = 5; diff --git a/cmd/release_config/release_config_proto/build_flags_src.pb.go b/cmd/release_config/release_config_proto/build_flags_src.pb.go index d784dee2d..123c6d3cf 100644 --- a/cmd/release_config/release_config_proto/build_flags_src.pb.go +++ b/cmd/release_config/release_config_proto/build_flags_src.pb.go @@ -159,6 +159,8 @@ type FlagDeclaration struct { Namespace *string `protobuf:"bytes,2,opt,name=namespace" json:"namespace,omitempty"` // Text description of the flag's purpose. Description *string `protobuf:"bytes,3,opt,name=description" json:"description,omitempty"` + // The bug number associated with the flag. + Bugs []string `protobuf:"bytes,4,rep,name=bugs" json:"bugs,omitempty"` // Value for the flag Value *Value `protobuf:"bytes,201,opt,name=value" json:"value,omitempty"` // Workflow for this flag. @@ -221,6 +223,13 @@ func (x *FlagDeclaration) GetDescription() string { return "" } +func (x *FlagDeclaration) GetBugs() []string { + if x != nil { + return x.Bugs + } + return nil +} + func (x *FlagDeclaration) GetValue() *Value { if x != nil { return x.Value @@ -541,62 +550,63 @@ var file_build_flags_src_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x08, 0x6f, 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x18, 0xcb, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x08, 0x6f, 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, - 0x42, 0x05, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x95, 0x02, 0x0a, 0x0f, 0x46, 0x6c, 0x61, 0x67, + 0x42, 0x05, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x22, 0xa3, 0x02, 0x0a, 0x0f, 0x46, 0x6c, 0x61, 0x67, 0x44, 0x65, 0x63, 0x6c, 0x61, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x23, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, - 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x43, 0x0a, 0x08, 0x77, - 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x18, 0xcd, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, + 0x12, 0x0a, 0x04, 0x62, 0x75, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x62, + 0x75, 0x67, 0x73, 0x12, 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0xc9, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, + 0x43, 0x0a, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x18, 0xcd, 0x01, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, + 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x1f, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x73, 0x18, 0xce, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x73, 0x4a, 0x06, 0x08, 0xcf, 0x01, 0x10, 0xd0, 0x01, 0x22, 0x78, 0x0a, + 0x09, 0x46, 0x6c, 0x61, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3a, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, - 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x12, 0x1f, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0xce, - 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x73, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x06, 0x08, 0xcf, 0x01, 0x10, 0xd0, 0x01, 0x22, - 0x78, 0x0a, 0x09, 0x46, 0x6c, 0x61, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x3a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0xc9, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x23, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1b, 0x0a, 0x08, - 0x72, 0x65, 0x64, 0x61, 0x63, 0x74, 0x65, 0x64, 0x18, 0xca, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x08, 0x72, 0x65, 0x64, 0x61, 0x63, 0x74, 0x65, 0x64, 0x22, 0xbe, 0x01, 0x0a, 0x0d, 0x52, 0x65, - 0x6c, 0x65, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x61, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x73, 0x65, 0x74, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x53, 0x65, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6c, - 0x61, 0x67, 0x73, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x69, 0x6f, 0x72, - 0x5f, 0x73, 0x74, 0x61, 0x67, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x70, - 0x72, 0x69, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x67, 0x65, 0x73, 0x22, 0x3a, 0x0a, 0x0c, 0x52, 0x65, - 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, - 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xa9, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x6c, 0x65, 0x61, - 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x12, 0x44, 0x0a, 0x07, 0x61, - 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x61, - 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x6c, 0x65, - 0x61, 0x73, 0x65, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x07, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, - 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x2d, 0x0a, 0x12, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x11, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x73, 0x42, 0x33, 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, - 0x6f, 0x6e, 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1b, 0x0a, 0x08, 0x72, 0x65, + 0x64, 0x61, 0x63, 0x74, 0x65, 0x64, 0x18, 0xca, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, + 0x65, 0x64, 0x61, 0x63, 0x74, 0x65, 0x64, 0x22, 0xbe, 0x01, 0x0a, 0x0d, 0x52, 0x65, 0x6c, 0x65, + 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, + 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x08, 0x69, 0x6e, 0x68, 0x65, 0x72, 0x69, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x53, 0x65, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x10, 0x61, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x6c, 0x61, 0x67, + 0x73, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x5f, 0x73, + 0x74, 0x61, 0x67, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, 0x69, + 0x6f, 0x72, 0x53, 0x74, 0x61, 0x67, 0x65, 0x73, 0x22, 0x3a, 0x0a, 0x0c, 0x52, 0x65, 0x6c, 0x65, + 0x61, 0x73, 0x65, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x22, 0xa9, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x61, 0x70, 0x12, 0x44, 0x0a, 0x07, 0x61, 0x6c, 0x69, + 0x61, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x61, 0x6e, 0x64, + 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x07, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x12, + 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x2d, 0x0a, 0x12, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, + 0x42, 0x33, 0x5a, 0x31, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, + 0x67, 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x2f, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( diff --git a/cmd/release_config/release_config_proto/build_flags_src.proto b/cmd/release_config/release_config_proto/build_flags_src.proto index e1925bc75..b28b5e20d 100644 --- a/cmd/release_config/release_config_proto/build_flags_src.proto +++ b/cmd/release_config/release_config_proto/build_flags_src.proto @@ -67,8 +67,8 @@ message FlagDeclaration { // Text description of the flag's purpose. optional string description = 3; - // reserve this for bug, if needed. - reserved 4; + // The bug number associated with the flag. + repeated string bugs = 4; // Value for the flag optional Value value = 201; diff --git a/cmd/symbols_map/symbols_map_proto/symbols_map.proto b/cmd/symbols_map/symbols_map_proto/symbols_map.proto index 693fe3ecf..a76d17147 100644 --- a/cmd/symbols_map/symbols_map_proto/symbols_map.proto +++ b/cmd/symbols_map/symbols_map_proto/symbols_map.proto @@ -37,6 +37,21 @@ message Mapping { // type is the type of the mapping, either ELF or R8. optional Type type = 3; + + // LocationType is the place where to look for the file with the given + // identifier. + Enum LocationType { + // ZIP denotes the file with the given identifier is in the distribuited + // symbols.zip or proguard_dict.zip files, or the local disc. + ZIP = 0; + // AB denotes the file with the given identifier is in the AB artifacts but + // not in a symbols.zip or proguard_dict.zip. + AB = 1; + } + + // location_type is the Location Type that dictates where to search for the + // file with the given identifier. Defaults to ZIP if not present. + optional LocationType location_type = 4; } message Mappings { diff --git a/compliance/Android.bp b/compliance/Android.bp index 72c2f27c1..6662970d8 100644 --- a/compliance/Android.bp +++ b/compliance/Android.bp @@ -35,5 +35,6 @@ notice_xml { partition_name: "system", visibility: [ "//build/make/target/product/generic", + "//build/make/target/product/gsi", ], } diff --git a/dexpreopt/config.go b/dexpreopt/config.go index 3dd6f9a4f..e3804e57f 100644 --- a/dexpreopt/config.go +++ b/dexpreopt/config.go @@ -465,7 +465,7 @@ func (d dex2oatDependencyTag) AllowDisabledModuleDependency(target android.Modul func (d dex2oatDependencyTag) AllowDisabledModuleDependencyProxy( ctx android.OtherModuleProviderContext, target android.ModuleProxy) bool { return android.OtherModuleProviderOrDefault( - ctx, target, android.CommonPropertiesProviderKey).ReplacedByPrebuilt + ctx, target, android.CommonModuleInfoKey).ReplacedByPrebuilt } // Dex2oatDepTag represents the dependency onto the dex2oatd module. It is added to any module that diff --git a/etc/Android.bp b/etc/Android.bp index 8e043b86e..e92437e79 100644 --- a/etc/Android.bp +++ b/etc/Android.bp @@ -12,6 +12,7 @@ bootstrap_go_package { ], srcs: [ "adb_keys.go", + "avbpubkey.go", "install_symlink.go", "otacerts_zip.go", "prebuilt_etc.go", diff --git a/etc/adb_keys.go b/etc/adb_keys.go index 73bc3478f..cfcb1d502 100644 --- a/etc/adb_keys.go +++ b/etc/adb_keys.go @@ -24,7 +24,7 @@ func init() { type AdbKeysModule struct { android.ModuleBase - outputPath android.OutputPath + outputPath android.Path installPath android.InstallPath } @@ -46,15 +46,16 @@ func (m *AdbKeysModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { return } - m.outputPath = android.PathForModuleOut(ctx, "adb_keys").OutputPath + outputPath := android.PathForModuleOut(ctx, "adb_keys") input := android.ExistentPathForSource(ctx, android.String(productVariables.AdbKeys)) ctx.Build(pctx, android.BuildParams{ Rule: android.Cp, - Output: m.outputPath, + Output: outputPath, Input: input.Path(), }) m.installPath = android.PathForModuleInstall(ctx, "etc/security") - ctx.InstallFile(m.installPath, "adb_keys", m.outputPath) + ctx.InstallFile(m.installPath, "adb_keys", outputPath) + m.outputPath = outputPath } func (m *AdbKeysModule) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/etc/avbpubkey.go b/etc/avbpubkey.go new file mode 100644 index 000000000..3f998d487 --- /dev/null +++ b/etc/avbpubkey.go @@ -0,0 +1,84 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etc + +import ( + "android/soong/android" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" +) + +func init() { + android.RegisterModuleType("avbpubkey", AvbpubkeyModuleFactory) + pctx.HostBinToolVariable("avbtool", "avbtool") +} + +type avbpubkeyProperty struct { + Private_key *string `android:"path"` +} + +type AvbpubkeyModule struct { + android.ModuleBase + + properties avbpubkeyProperty + + outputPath android.WritablePath + installPath android.InstallPath +} + +func AvbpubkeyModuleFactory() android.Module { + module := &AvbpubkeyModule{} + module.AddProperties(&module.properties) + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) + return module +} + +var avbPubKeyRule = pctx.AndroidStaticRule("avbpubkey", + blueprint.RuleParams{ + Command: `${avbtool} extract_public_key --key ${in} --output ${out}.tmp` + + ` && ( if cmp -s ${out}.tmp ${out} ; then rm ${out}.tmp ; else mv ${out}.tmp ${out} ; fi )`, + CommandDeps: []string{"${avbtool}"}, + Description: "Extracting system_other avb key", + }) + +func (m *AvbpubkeyModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if !m.ProductSpecific() { + ctx.ModuleErrorf("avbpubkey module type must set product_specific to true") + } + + m.outputPath = android.PathForModuleOut(ctx, ctx.ModuleName(), "system_other.avbpubkey") + + ctx.Build(pctx, android.BuildParams{ + Rule: avbPubKeyRule, + Input: android.PathForModuleSrc(ctx, proptools.String(m.properties.Private_key)), + Output: m.outputPath, + }) + + m.installPath = android.PathForModuleInstall(ctx, "etc/security/avb") + ctx.InstallFile(m.installPath, "system_other.avbpubkey", m.outputPath) +} + +func (m *AvbpubkeyModule) AndroidMkEntries() []android.AndroidMkEntries { + if m.IsSkipInstall() { + return []android.AndroidMkEntries{} + } + + return []android.AndroidMkEntries{ + { + Class: "ETC", + OutputFile: android.OptionalPathForPath(m.outputPath), + }} +} diff --git a/etc/otacerts_zip.go b/etc/otacerts_zip.go index d12bdac7b..8220cea4c 100644 --- a/etc/otacerts_zip.go +++ b/etc/otacerts_zip.go @@ -44,7 +44,7 @@ type otacertsZipModule struct { android.ModuleBase properties otacertsZipProperties - outputPath android.OutputPath + outputPath android.Path } // otacerts_zip collects key files defined in PRODUCT_DEFAULT_DEV_CERTIFICATE @@ -117,11 +117,11 @@ func (m *otacertsZipModule) GenerateAndroidBuildActions(ctx android.ModuleContex // Read .x509.pem files listed in PRODUCT_EXTRA_OTA_KEYS or PRODUCT_EXTRA_RECOVERY_KEYS. extras := ctx.Config().ExtraOtaKeys(ctx, m.InRecovery()) srcPaths := append([]android.SourcePath{pem}, extras...) - m.outputPath = android.PathForModuleOut(ctx, m.outputFileName()).OutputPath + outputPath := android.PathForModuleOut(ctx, m.outputFileName()) rule := android.NewRuleBuilder(pctx, ctx) cmd := rule.Command().BuiltTool("soong_zip"). - FlagWithOutput("-o ", m.outputPath). + FlagWithOutput("-o ", outputPath). Flag("-j "). Flag("-symlinks=false ") for _, src := range srcPaths { @@ -130,7 +130,8 @@ func (m *otacertsZipModule) GenerateAndroidBuildActions(ctx android.ModuleContex rule.Build(ctx.ModuleName(), "Generating the otacerts zip file") installPath := android.PathForModuleInstall(ctx, "etc", proptools.String(m.properties.Relative_install_path)) - ctx.InstallFile(installPath, m.outputFileName(), m.outputPath) + ctx.InstallFile(installPath, m.outputFileName(), outputPath) + m.outputPath = outputPath } func (m *otacertsZipModule) AndroidMkEntries() []android.AndroidMkEntries { diff --git a/etc/prebuilt_etc.go b/etc/prebuilt_etc.go index be943a3ca..b0b5da946 100644 --- a/etc/prebuilt_etc.go +++ b/etc/prebuilt_etc.go @@ -80,6 +80,7 @@ func RegisterPrebuiltEtcBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("prebuilt_bt_firmware", PrebuiltBtFirmwareFactory) ctx.RegisterModuleType("prebuilt_tvservice", PrebuiltTvServiceFactory) ctx.RegisterModuleType("prebuilt_optee", PrebuiltOpteeFactory) + ctx.RegisterModuleType("prebuilt_tvconfig", PrebuiltTvConfigFactory) ctx.RegisterModuleType("prebuilt_defaults", defaultsFactory) @@ -87,7 +88,7 @@ func RegisterPrebuiltEtcBuildComponents(ctx android.RegistrationContext) { var PrepareForTestWithPrebuiltEtc = android.FixtureRegisterWithContext(RegisterPrebuiltEtcBuildComponents) -type prebuiltEtcProperties struct { +type PrebuiltEtcProperties struct { // Source file of this prebuilt. Can reference a genrule type module with the ":module" syntax. // Mutually exclusive with srcs. Src proptools.Configurable[string] `android:"path,arch_variant,replace_instead_of_append"` @@ -174,7 +175,7 @@ type PrebuiltEtc struct { android.ModuleBase android.DefaultableModuleBase - properties prebuiltEtcProperties + properties PrebuiltEtcProperties // rootProperties is used to return the value of the InstallInRoot() method. Currently, only // prebuilt_avb and prebuilt_root modules use this. @@ -183,7 +184,7 @@ type PrebuiltEtc struct { subdirProperties prebuiltSubdirProperties sourceFilePaths android.Paths - outputFilePaths android.OutputPaths + outputFilePaths android.WritablePaths // The base install location, e.g. "etc" for prebuilt_etc, "usr/share" for prebuilt_usr_share. installDirBase string installDirBase64 string @@ -316,7 +317,7 @@ func (p *PrebuiltEtc) SetAdditionalDependencies(paths android.Paths) { p.additionalDependencies = &paths } -func (p *PrebuiltEtc) OutputFile() android.OutputPath { +func (p *PrebuiltEtc) OutputFile() android.Path { if p.usedSrcsProperty { panic(fmt.Errorf("OutputFile not available on multi-source prebuilt %q", p.Name())) } @@ -409,7 +410,7 @@ func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.PropertyErrorf("filename", "filename cannot contain separator '/'") return } - p.outputFilePaths = android.OutputPaths{android.PathForModuleOut(ctx, filename).OutputPath} + p.outputFilePaths = android.WritablePaths{android.PathForModuleOut(ctx, filename)} ip := installProperties{ filename: filename, @@ -452,7 +453,7 @@ func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) { filename = src.Base() installDirPath = baseInstallDirPath } - output := android.PathForModuleOut(ctx, filename).OutputPath + output := android.PathForModuleOut(ctx, filename) ip := installProperties{ filename: filename, sourceFilePath: src, @@ -472,7 +473,7 @@ func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) { if filename == "" { filename = ctx.ModuleName() } - p.outputFilePaths = android.OutputPaths{android.PathForModuleOut(ctx, filename).OutputPath} + p.outputFilePaths = android.WritablePaths{android.PathForModuleOut(ctx, filename)} ip := installProperties{ filename: filename, sourceFilePath: p.sourceFilePaths[0], @@ -500,7 +501,7 @@ func (p *PrebuiltEtc) GenerateAndroidBuildActions(ctx android.ModuleContext) { type installProperties struct { filename string sourceFilePath android.Path - outputFilePath android.OutputPath + outputFilePath android.WritablePath installDirPath android.InstallPath symlinks []string } @@ -606,7 +607,7 @@ func DefaultsFactory(props ...interface{}) android.Module { module.AddProperties(props...) module.AddProperties( - &prebuiltEtcProperties{}, + &PrebuiltEtcProperties{}, &prebuiltSubdirProperties{}, ) @@ -961,3 +962,13 @@ func PrebuiltOpteeFactory() android.Module { android.InitDefaultableModule(module) return module } + +// prebuilt_tvconfig installs files in <partition>/tvconfig directory. +func PrebuiltTvConfigFactory() android.Module { + module := &PrebuiltEtc{} + InitPrebuiltEtcModule(module, "tvconfig") + // This module is device-only + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitDefaultableModule(module) + return module +} diff --git a/filesystem/avb_add_hash_footer.go b/filesystem/avb_add_hash_footer.go index 469f1fb0a..9d4ba3e95 100644 --- a/filesystem/avb_add_hash_footer.go +++ b/filesystem/avb_add_hash_footer.go @@ -29,7 +29,7 @@ type avbAddHashFooter struct { properties avbAddHashFooterProperties - output android.OutputPath + output android.Path installDir android.InstallPath } @@ -97,8 +97,8 @@ func (a *avbAddHashFooter) GenerateAndroidBuildActions(ctx android.ModuleContext return } input := android.PathForModuleSrc(ctx, proptools.String(a.properties.Src)) - a.output = android.PathForModuleOut(ctx, a.installFileName()).OutputPath - builder.Command().Text("cp").Input(input).Output(a.output) + output := android.PathForModuleOut(ctx, a.installFileName()) + builder.Command().Text("cp").Input(input).Output(output) cmd := builder.Command().BuiltTool("avbtool").Text("add_hash_footer") @@ -141,12 +141,13 @@ func (a *avbAddHashFooter) GenerateAndroidBuildActions(ctx android.ModuleContext cmd.Flag(fmt.Sprintf(" --rollback_index %d", rollbackIndex)) } - cmd.FlagWithOutput("--image ", a.output) + cmd.FlagWithOutput("--image ", output) builder.Build("avbAddHashFooter", fmt.Sprintf("avbAddHashFooter %s", ctx.ModuleName())) a.installDir = android.PathForModuleInstall(ctx, "etc") - ctx.InstallFile(a.installDir, a.installFileName(), a.output) + ctx.InstallFile(a.installDir, a.installFileName(), output) + a.output = output } func addAvbProp(ctx android.ModuleContext, cmd *android.RuleBuilderCommand, prop avbProp) { diff --git a/filesystem/avb_gen_vbmeta_image.go b/filesystem/avb_gen_vbmeta_image.go index a7fd7829e..0669a2c20 100644 --- a/filesystem/avb_gen_vbmeta_image.go +++ b/filesystem/avb_gen_vbmeta_image.go @@ -28,7 +28,7 @@ type avbGenVbmetaImage struct { properties avbGenVbmetaImageProperties - output android.OutputPath + output android.Path installDir android.InstallPath } @@ -78,11 +78,12 @@ func (a *avbGenVbmetaImage) GenerateAndroidBuildActions(ctx android.ModuleContex } cmd.FlagWithArg("--salt ", proptools.String(a.properties.Salt)) - a.output = android.PathForModuleOut(ctx, a.installFileName()).OutputPath - cmd.FlagWithOutput("--output_vbmeta_image ", a.output) + output := android.PathForModuleOut(ctx, a.installFileName()) + cmd.FlagWithOutput("--output_vbmeta_image ", output) builder.Build("avbGenVbmetaImage", fmt.Sprintf("avbGenVbmetaImage %s", ctx.ModuleName())) - ctx.SetOutputFiles([]android.Path{a.output}, "") + ctx.SetOutputFiles([]android.Path{output}, "") + a.output = output } var _ android.AndroidMkEntriesProvider = (*avbGenVbmetaImage)(nil) diff --git a/filesystem/bootimg.go b/filesystem/bootimg.go index 9d9392596..c9bd61788 100644 --- a/filesystem/bootimg.go +++ b/filesystem/bootimg.go @@ -34,7 +34,7 @@ type bootimg struct { properties bootimgProperties - output android.OutputPath + output android.Path installDir android.InstallPath } @@ -115,20 +115,20 @@ func (b *bootimg) GenerateAndroidBuildActions(ctx android.ModuleContext) { vendor := proptools.Bool(b.properties.Vendor_boot) unsignedOutput := b.buildBootImage(ctx, vendor) + output := unsignedOutput if proptools.Bool(b.properties.Use_avb) { - b.output = b.signImage(ctx, unsignedOutput) - } else { - b.output = unsignedOutput + output = b.signImage(ctx, unsignedOutput) } b.installDir = android.PathForModuleInstall(ctx, "etc") - ctx.InstallFile(b.installDir, b.installFileName(), b.output) + ctx.InstallFile(b.installDir, b.installFileName(), output) - ctx.SetOutputFiles([]android.Path{b.output}, "") + ctx.SetOutputFiles([]android.Path{output}, "") + b.output = output } -func (b *bootimg) buildBootImage(ctx android.ModuleContext, vendor bool) android.OutputPath { - output := android.PathForModuleOut(ctx, "unsigned", b.installFileName()).OutputPath +func (b *bootimg) buildBootImage(ctx android.ModuleContext, vendor bool) android.Path { + output := android.PathForModuleOut(ctx, "unsigned", b.installFileName()) builder := android.NewRuleBuilder(pctx, ctx) cmd := builder.Command().BuiltTool("mkbootimg") @@ -215,10 +215,10 @@ func (b *bootimg) buildBootImage(ctx android.ModuleContext, vendor bool) android return output } -func (b *bootimg) signImage(ctx android.ModuleContext, unsignedImage android.OutputPath) android.OutputPath { +func (b *bootimg) signImage(ctx android.ModuleContext, unsignedImage android.Path) android.Path { propFile, toolDeps := b.buildPropFile(ctx) - output := android.PathForModuleOut(ctx, b.installFileName()).OutputPath + output := android.PathForModuleOut(ctx, b.installFileName()) builder := android.NewRuleBuilder(pctx, ctx) builder.Command().Text("cp").Input(unsignedImage).Output(output) builder.Command().BuiltTool("verity_utils"). @@ -239,7 +239,7 @@ func (b *bootimg) salt() string { return sha1sum(input) } -func (b *bootimg) buildPropFile(ctx android.ModuleContext) (propFile android.OutputPath, toolDeps android.Paths) { +func (b *bootimg) buildPropFile(ctx android.ModuleContext) (android.Path, android.Paths) { var sb strings.Builder var deps android.Paths addStr := func(name string, value string) { @@ -261,7 +261,7 @@ func (b *bootimg) buildPropFile(ctx android.ModuleContext) (propFile android.Out addStr("partition_name", partitionName) addStr("avb_salt", b.salt()) - propFile = android.PathForModuleOut(ctx, "prop").OutputPath + propFile := android.PathForModuleOut(ctx, "prop") android.WriteFileRule(ctx, propFile, sb.String()) return propFile, deps } diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index e84139b9a..c34677060 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -52,21 +52,29 @@ type filesystem struct { properties FilesystemProperties - // Function that builds extra files under the root directory and returns the files - buildExtraFiles func(ctx android.ModuleContext, root android.OutputPath) android.OutputPaths - - // Function that filters PackagingSpec in PackagingBase.GatherPackagingSpecs() - filterPackagingSpec func(spec android.PackagingSpec) bool - - output android.OutputPath + output android.Path installDir android.InstallPath - fileListFile android.OutputPath + fileListFile android.Path // Keeps the entries installed from this filesystem entries []string + + filesystemBuilder filesystemBuilder +} + +type filesystemBuilder interface { + BuildLinkerConfigFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) + // Function that filters PackagingSpec in PackagingBase.GatherPackagingSpecs() + FilterPackagingSpec(spec android.PackagingSpec) bool + // Function that modifies PackagingSpec in PackagingBase.GatherPackagingSpecs() to customize. + // For example, GSI system.img contains system_ext and product artifacts and their + // relPathInPackage need to be rebased to system/system_ext and system/system_product. + ModifyPackagingSpec(spec *android.PackagingSpec) } +var _ filesystemBuilder = (*filesystem)(nil) + type SymlinkDefinition struct { Target *string Name *string @@ -149,7 +157,7 @@ type FilesystemProperties struct { F2fs F2fsProperties - Linkerconfig LinkerConfigProperties + Linker_config LinkerConfigProperties // Determines if the module is auto-generated from Soong or not. If the module is // auto-generated, its deps are exempted from visibility enforcement. @@ -190,7 +198,7 @@ type LinkerConfigProperties struct { // partitions like system.img. For example, cc_library modules are placed under ./lib[64] directory. func FilesystemFactory() android.Module { module := &filesystem{} - module.filterPackagingSpec = module.filterInstallablePackagingSpec + module.filesystemBuilder = module initFilesystemModule(module, module) return module } @@ -285,13 +293,32 @@ func (f *filesystem) partitionName() string { return proptools.StringDefault(f.properties.Partition_name, f.Name()) } -func (f *filesystem) filterInstallablePackagingSpec(ps android.PackagingSpec) bool { +func (f *filesystem) FilterPackagingSpec(ps android.PackagingSpec) bool { // Filesystem module respects the installation semantic. A PackagingSpec from a module with // IsSkipInstall() is skipped. + if ps.SkipInstall() { + return false + } if proptools.Bool(f.properties.Is_auto_generated) { // TODO (spandandas): Remove this. - return !ps.SkipInstall() && (ps.Partition() == f.PartitionType()) + pt := f.PartitionType() + return ps.Partition() == pt || strings.HasPrefix(ps.Partition(), pt+"/") + } + return true +} + +func (f *filesystem) ModifyPackagingSpec(ps *android.PackagingSpec) { + // Sometimes, android.modulePartition() returns a path with >1 path components. + // This makes the partition field of packagingSpecs have multiple components, like + // "system/product". Right now, the filesystem module doesn't look at the partition field + // when deciding what path to install the file under, only the RelPathInPackage field, so + // we move the later path components from partition to relPathInPackage. This should probably + // be revisited in the future. + prefix := f.PartitionType() + "/" + if strings.HasPrefix(ps.Partition(), prefix) { + subPartition := strings.TrimPrefix(ps.Partition(), prefix) + ps.SetPartition(f.PartitionType()) + ps.SetRelPathInPackage(filepath.Join(subPartition, ps.RelPathInPackage())) } - return !ps.SkipInstall() } var pctx = android.NewPackageContext("android/soong/filesystem") @@ -313,19 +340,20 @@ func (f *filesystem) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.InstallFile(f.installDir, f.installFileName(), f.output) ctx.SetOutputFiles([]android.Path{f.output}, "") - f.fileListFile = android.PathForModuleOut(ctx, "fileList").OutputPath - android.WriteFileRule(ctx, f.fileListFile, f.installedFilesList()) + fileListFile := android.PathForModuleOut(ctx, "fileList") + android.WriteFileRule(ctx, fileListFile, f.installedFilesList()) android.SetProvider(ctx, FilesystemProvider, FilesystemInfo{ - FileListFile: f.fileListFile, + FileListFile: fileListFile, }) + f.fileListFile = fileListFile if proptools.Bool(f.properties.Unchecked_module) { ctx.UncheckedModule() } } -func (f *filesystem) appendToEntry(ctx android.ModuleContext, installedFile android.OutputPath) { +func (f *filesystem) appendToEntry(ctx android.ModuleContext, installedFile android.Path) { partitionBaseDir := android.PathForModuleOut(ctx, "root", f.partitionName()).String() + "/" relPath, inTargetPartition := strings.CutPrefix(installedFile.String(), partitionBaseDir) @@ -387,25 +415,6 @@ func (f *filesystem) buildNonDepsFiles(ctx android.ModuleContext, builder *andro builder.Command().Text("ln -sf").Text(proptools.ShellEscape(target)).Text(dst.String()) f.appendToEntry(ctx, dst) } - - // create extra files if there's any - if f.buildExtraFiles != nil { - rootForExtraFiles := android.PathForModuleGen(ctx, "root-extra").OutputPath - extraFiles := f.buildExtraFiles(ctx, rootForExtraFiles) - for _, extraFile := range extraFiles { - rel, err := filepath.Rel(rootForExtraFiles.String(), extraFile.String()) - if err != nil || strings.HasPrefix(rel, "..") { - ctx.ModuleErrorf("can't make %q relative to %q", extraFile, rootForExtraFiles) - } - f.appendToEntry(ctx, rootDir.Join(ctx, rel)) - } - if len(extraFiles) > 0 { - builder.Command().BuiltTool("merge_directories"). - Implicits(extraFiles.Paths()). - Text(rootDir.String()). - Text(rootForExtraFiles.String()) - } - } } func (f *filesystem) copyPackagingSpecs(ctx android.ModuleContext, builder *android.RuleBuilder, specs map[string]android.PackagingSpec, rootDir, rebasedDir android.WritablePath) []string { @@ -435,7 +444,7 @@ func (f *filesystem) copyFilesToProductOut(ctx android.ModuleContext, builder *a builder.Command().Textf("cp -prf %s/* %s", rebasedDir, installPath) } -func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) android.OutputPath { +func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) android.Path { rootDir := android.PathForModuleOut(ctx, "root").OutputPath rebasedDir := rootDir if f.properties.Base_dir != nil { @@ -452,7 +461,7 @@ func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) androi f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir) f.buildEventLogtagsFile(ctx, builder, rebasedDir) f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir) - f.buildLinkerConfigFile(ctx, builder, rebasedDir) + f.filesystemBuilder.BuildLinkerConfigFile(ctx, builder, rebasedDir) f.copyFilesToProductOut(ctx, builder, rebasedDir) // run host_init_verifier @@ -464,7 +473,7 @@ func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) androi FlagWithArg("--out_system=", rootDir.String()+"/system") propFile, toolDeps := f.buildPropFile(ctx) - output := android.PathForModuleOut(ctx, f.installFileName()).OutputPath + output := android.PathForModuleOut(ctx, f.installFileName()) builder.Command().BuiltTool("build_image"). Text(rootDir.String()). // input directory Input(propFile). @@ -478,14 +487,14 @@ func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) androi return output } -func (f *filesystem) buildFileContexts(ctx android.ModuleContext) android.OutputPath { +func (f *filesystem) buildFileContexts(ctx android.ModuleContext) android.Path { builder := android.NewRuleBuilder(pctx, ctx) fcBin := android.PathForModuleOut(ctx, "file_contexts.bin") builder.Command().BuiltTool("sefcontext_compile"). FlagWithOutput("-o ", fcBin). Input(android.PathForModuleSrc(ctx, proptools.String(f.properties.File_contexts))) builder.Build("build_filesystem_file_contexts", fmt.Sprintf("Creating filesystem file contexts for %s", f.BaseModuleName())) - return fcBin.OutputPath + return fcBin } // Calculates avb_salt from entry list (sorted) for deterministic output. @@ -493,7 +502,7 @@ func (f *filesystem) salt() string { return sha1sum(f.entries) } -func (f *filesystem) buildPropFile(ctx android.ModuleContext) (propFile android.OutputPath, toolDeps android.Paths) { +func (f *filesystem) buildPropFile(ctx android.ModuleContext) (android.Path, android.Paths) { var deps android.Paths var propFileString strings.Builder addStr := func(name string, value string) { @@ -589,7 +598,7 @@ func (f *filesystem) buildPropFile(ctx android.ModuleContext) (propFile android. } f.checkFsTypePropertyError(ctx, fst, fsTypeStr(fst)) - propFile = android.PathForModuleOut(ctx, "prop").OutputPath + propFile := android.PathForModuleOut(ctx, "prop") android.WriteFileRuleVerbatim(ctx, propFile, propFileString.String()) return propFile, deps } @@ -614,7 +623,7 @@ func (f *filesystem) checkFsTypePropertyError(ctx android.ModuleContext, t fsTyp } } -func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) android.OutputPath { +func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) android.Path { if proptools.Bool(f.properties.Use_avb) { ctx.PropertyErrorf("use_avb", "signing compresed cpio image using avbtool is not supported."+ "Consider adding this to bootimg module and signing the entire boot image.") @@ -643,10 +652,10 @@ func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir) f.buildEventLogtagsFile(ctx, builder, rebasedDir) f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir) - f.buildLinkerConfigFile(ctx, builder, rebasedDir) + f.filesystemBuilder.BuildLinkerConfigFile(ctx, builder, rebasedDir) f.copyFilesToProductOut(ctx, builder, rebasedDir) - output := android.PathForModuleOut(ctx, f.installFileName()).OutputPath + output := android.PathForModuleOut(ctx, f.installFileName()) cmd := builder.Command(). BuiltTool("mkbootfs"). Text(rootDir.String()) // input directory @@ -679,6 +688,7 @@ var validPartitions = []string{ "vendor_dlkm", "odm_dlkm", "system_dlkm", + "ramdisk", } func (f *filesystem) addMakeBuiltFiles(ctx android.ModuleContext, builder *android.RuleBuilder, rootDir android.Path) { @@ -735,14 +745,14 @@ func (f *filesystem) buildEventLogtagsFile(ctx android.ModuleContext, builder *a f.appendToEntry(ctx, eventLogtagsPath) } -func (f *filesystem) buildLinkerConfigFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) { - if !proptools.Bool(f.properties.Linkerconfig.Gen_linker_config) { +func (f *filesystem) BuildLinkerConfigFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) { + if !proptools.Bool(f.properties.Linker_config.Gen_linker_config) { return } provideModules, _ := f.getLibsForLinkerConfig(ctx) output := rebasedDir.Join(ctx, "etc", "linker.config.pb") - linkerconfig.BuildLinkerConfig(ctx, builder, android.PathsForModuleSrc(ctx, f.properties.Linkerconfig.Linker_config_srcs), provideModules, nil, output) + linkerconfig.BuildLinkerConfig(ctx, builder, android.PathsForModuleSrc(ctx, f.properties.Linker_config.Linker_config_srcs), provideModules, nil, output) f.appendToEntry(ctx, output) } @@ -802,7 +812,7 @@ func (f *filesystem) SignedOutputPath() android.Path { // Note that "apex" module installs its contents to "apex"(fake partition) as well // for symbol lookup by imitating "activated" paths. func (f *filesystem) gatherFilteredPackagingSpecs(ctx android.ModuleContext) map[string]android.PackagingSpec { - specs := f.PackagingBase.GatherPackagingSpecsWithFilter(ctx, f.filterPackagingSpec) + specs := f.PackagingBase.GatherPackagingSpecsWithFilterAndModifier(ctx, f.filesystemBuilder.FilterPackagingSpec, f.filesystemBuilder.ModifyPackagingSpec) return specs } @@ -828,13 +838,7 @@ type filesystemDefaults struct { android.ModuleBase android.DefaultsModuleBase - properties filesystemDefaultsProperties -} - -type filesystemDefaultsProperties struct { - // Identifies which partition this is for //visibility:any_system_image (and others) visibility - // checks, and will be used in the future for API surface checks. - Partition_type *string + properties FilesystemProperties } // android_filesystem_defaults is a default module for android_filesystem and android_system_image diff --git a/filesystem/filesystem_test.go b/filesystem/filesystem_test.go index 801a17545..f325d96ef 100644 --- a/filesystem/filesystem_test.go +++ b/filesystem/filesystem_test.go @@ -156,11 +156,15 @@ func TestFileSystemFillsLinkerConfigWithStubLibs(t *testing.T) { result := fixture.RunTestWithBp(t, ` android_system_image { name: "myfilesystem", + base_dir: "system", deps: [ "libfoo", "libbar", ], - linker_config_src: "linker.config.json", + linker_config: { + gen_linker_config: true, + linker_config_srcs: ["linker.config.json"], + }, } cc_library { @@ -176,7 +180,7 @@ func TestFileSystemFillsLinkerConfigWithStubLibs(t *testing.T) { `) module := result.ModuleForTests("myfilesystem", "android_common") - output := module.Output("system/etc/linker.config.pb") + output := module.Output("out/soong/.intermediates/myfilesystem/android_common/root/system/etc/linker.config.pb") android.AssertStringDoesContain(t, "linker.config.pb should have libfoo", output.RuleParams.Command, "libfoo.so") @@ -223,7 +227,10 @@ func TestFileSystemGathersItemsOnlyInSystemPartition(t *testing.T) { deps: ["foo"], }, }, - linker_config_src: "linker.config.json", + linker_config: { + gen_linker_config: true, + linker_config_srcs: ["linker.config.json"], + }, } component { name: "foo", @@ -318,7 +325,10 @@ func TestFileSystemWithCoverageVariants(t *testing.T) { deps: [ "libfoo", ], - linker_config_src: "linker.config.json", + linker_config: { + gen_linker_config: true, + linker_config_srcs: ["linker.config.json"], + }, } cc_library { @@ -699,9 +709,9 @@ func TestInstallLinkerConfigFile(t *testing.T) { android_filesystem { name: "myfilesystem", deps: ["libfoo_has_no_stubs", "libfoo_has_stubs"], - linkerconfig: { - gen_linker_config: true, - linker_config_srcs: ["linker.config.json"], + linker_config: { + gen_linker_config: true, + linker_config_srcs: ["linker.config.json"], }, partition_type: "vendor", } diff --git a/filesystem/fsverity_metadata.go b/filesystem/fsverity_metadata.go index 199c84516..3119f2f28 100644 --- a/filesystem/fsverity_metadata.go +++ b/filesystem/fsverity_metadata.go @@ -33,7 +33,7 @@ type fsverityProperties struct { Libs []string `android:"path"` } -func (f *filesystem) writeManifestGeneratorListFile(ctx android.ModuleContext, outputPath android.OutputPath, matchedSpecs []android.PackagingSpec, rebasedDir android.OutputPath) { +func (f *filesystem) writeManifestGeneratorListFile(ctx android.ModuleContext, outputPath android.WritablePath, matchedSpecs []android.PackagingSpec, rebasedDir android.OutputPath) { var buf strings.Builder for _, spec := range matchedSpecs { buf.WriteString(rebasedDir.Join(ctx, spec.RelPathInPackage()).String()) @@ -113,12 +113,12 @@ func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, build f.appendToEntry(ctx, manifestPbPath) manifestGeneratorListPath := android.PathForModuleOut(ctx, "fsverity_manifest.list") - f.writeManifestGeneratorListFile(ctx, manifestGeneratorListPath.OutputPath, matchedSpecs, rebasedDir) + f.writeManifestGeneratorListFile(ctx, manifestGeneratorListPath, matchedSpecs, rebasedDir) sb.WriteRune('@') sb.WriteString(manifestGeneratorListPath.String()) sb.WriteRune('\n') cmd.Implicit(manifestGeneratorListPath) - f.appendToEntry(ctx, manifestGeneratorListPath.OutputPath) + f.appendToEntry(ctx, manifestGeneratorListPath) // STEP 2-2: generate BuildManifest.apk (unsigned) aapt2Path := ctx.Config().HostToolPath(ctx, "aapt2") diff --git a/filesystem/logical_partition.go b/filesystem/logical_partition.go index 988a57b08..d0888a9c8 100644 --- a/filesystem/logical_partition.go +++ b/filesystem/logical_partition.go @@ -32,7 +32,7 @@ type logicalPartition struct { properties logicalPartitionProperties - output android.OutputPath + output android.Path installDir android.InstallPath } @@ -95,11 +95,14 @@ func (l *logicalPartition) GenerateAndroidBuildActions(ctx android.ModuleContext builder := android.NewRuleBuilder(pctx, ctx) // Sparse the filesystem images and calculate their sizes - sparseImages := make(map[string]android.OutputPath) - sparseImageSizes := make(map[string]android.OutputPath) + sparseImages := make(map[string]android.Path) + sparseImageSizes := make(map[string]android.Path) sparsePartitions := func(partitions []partitionProperties) { for _, part := range partitions { + if part.Filesystem == nil { + continue + } sparseImg, sizeTxt := sparseFilesystem(ctx, part, builder) pName := proptools.String(part.Name) sparseImages[pName] = sparseImg @@ -185,31 +188,29 @@ func (l *logicalPartition) GenerateAndroidBuildActions(ctx android.ModuleContext addPartitionsToGroup(group.Partitions, gName) } - l.output = android.PathForModuleOut(ctx, l.installFileName()).OutputPath - cmd.FlagWithOutput("--output=", l.output) + output := android.PathForModuleOut(ctx, l.installFileName()) + cmd.FlagWithOutput("--output=", output) builder.Build("build_logical_partition", fmt.Sprintf("Creating %s", l.BaseModuleName())) l.installDir = android.PathForModuleInstall(ctx, "etc") - ctx.InstallFile(l.installDir, l.installFileName(), l.output) + ctx.InstallFile(l.installDir, l.installFileName(), output) - ctx.SetOutputFiles([]android.Path{l.output}, "") + ctx.SetOutputFiles([]android.Path{output}, "") + l.output = output } // Add a rule that converts the filesystem for the given partition to the given rule builder. The // path to the sparse file and the text file having the size of the partition are returned. -func sparseFilesystem(ctx android.ModuleContext, p partitionProperties, builder *android.RuleBuilder) (sparseImg android.OutputPath, sizeTxt android.OutputPath) { - if p.Filesystem == nil { - return - } - img := android.PathForModuleSrc(ctx, proptools.String(p.Filesystem)) +func sparseFilesystem(ctx android.ModuleContext, p partitionProperties, builder *android.RuleBuilder) (android.Path, android.Path) { + img := android.PathForModuleSrc(ctx, *p.Filesystem) name := proptools.String(p.Name) - sparseImg = android.PathForModuleOut(ctx, name+".img").OutputPath + sparseImg := android.PathForModuleOut(ctx, name+".img") builder.Temporary(sparseImg) builder.Command().BuiltTool("img2simg").Input(img).Output(sparseImg) - sizeTxt = android.PathForModuleOut(ctx, name+"-size.txt").OutputPath + sizeTxt := android.PathForModuleOut(ctx, name+"-size.txt") builder.Temporary(sizeTxt) builder.Command().BuiltTool("sparse_img").Flag("--get_partition_size").Input(sparseImg). Text("| ").Text("tr").FlagWithArg("-d ", "'\n'"). diff --git a/filesystem/raw_binary.go b/filesystem/raw_binary.go index ad36c2935..707fba06f 100644 --- a/filesystem/raw_binary.go +++ b/filesystem/raw_binary.go @@ -42,7 +42,7 @@ type rawBinary struct { properties rawBinaryProperties - output android.OutputPath + output android.Path installDir android.InstallPath } @@ -71,7 +71,7 @@ func (r *rawBinary) installFileName() string { func (r *rawBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) { inputFile := android.PathForModuleSrc(ctx, proptools.String(r.properties.Src)) - outputFile := android.PathForModuleOut(ctx, r.installFileName()).OutputPath + outputFile := android.PathForModuleOut(ctx, r.installFileName()) ctx.Build(pctx, android.BuildParams{ Rule: toRawBinary, @@ -83,11 +83,11 @@ func (r *rawBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) { }, }) - r.output = outputFile r.installDir = android.PathForModuleInstall(ctx, "etc") - ctx.InstallFile(r.installDir, r.installFileName(), r.output) + ctx.InstallFile(r.installDir, r.installFileName(), outputFile) - ctx.SetOutputFiles([]android.Path{r.output}, "") + ctx.SetOutputFiles([]android.Path{outputFile}, "") + r.output = outputFile } var _ android.AndroidMkEntriesProvider = (*rawBinary)(nil) diff --git a/filesystem/system_image.go b/filesystem/system_image.go index 898987dbc..d03eab45b 100644 --- a/filesystem/system_image.go +++ b/filesystem/system_image.go @@ -17,27 +17,24 @@ package filesystem import ( "android/soong/android" "android/soong/linkerconfig" + + "strings" + + "github.com/google/blueprint/proptools" ) type systemImage struct { filesystem - - properties systemImageProperties } -type systemImageProperties struct { - // Path to the input linker config json file. - Linker_config_src *string `android:"path"` -} +var _ filesystemBuilder = (*systemImage)(nil) // android_system_image is a specialization of android_filesystem for the 'system' partition. // Currently, the only difference is the inclusion of linker.config.pb file which specifies // the provided and the required libraries to and from APEXes. func SystemImageFactory() android.Module { module := &systemImage{} - module.AddProperties(&module.properties) - module.filesystem.buildExtraFiles = module.buildExtraFiles - module.filesystem.filterPackagingSpec = module.filterPackagingSpec + module.filesystemBuilder = module initFilesystemModule(module, &module.filesystem) return module } @@ -46,30 +43,23 @@ func (s systemImage) FsProps() FilesystemProperties { return s.filesystem.properties } -func (s *systemImage) buildExtraFiles(ctx android.ModuleContext, root android.OutputPath) android.OutputPaths { - if s.filesystem.properties.Partition_type != nil { - ctx.PropertyErrorf("partition_type", "partition_type must be unset on an android_system_image module. It is assumed to be 'system'.") +func (s *systemImage) BuildLinkerConfigFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) { + if !proptools.Bool(s.filesystem.properties.Linker_config.Gen_linker_config) { + return } - lc := s.buildLinkerConfigFile(ctx, root) - // Add more files if needed - return []android.OutputPath{lc} -} - -func (s *systemImage) buildLinkerConfigFile(ctx android.ModuleContext, root android.OutputPath) android.OutputPath { - input := android.PathForModuleSrc(ctx, android.String(s.properties.Linker_config_src)) - output := root.Join(ctx, "system", "etc", "linker.config.pb") provideModules, requireModules := s.getLibsForLinkerConfig(ctx) - builder := android.NewRuleBuilder(pctx, ctx) - linkerconfig.BuildLinkerConfig(ctx, builder, android.Paths{input}, provideModules, requireModules, output) - builder.Build("conv_linker_config", "Generate linker config protobuf "+output.String()) - return output + output := rebasedDir.Join(ctx, "etc", "linker.config.pb") + linkerconfig.BuildLinkerConfig(ctx, builder, android.PathsForModuleSrc(ctx, s.filesystem.properties.Linker_config.Linker_config_srcs), provideModules, requireModules, output) + + s.appendToEntry(ctx, output) } // Filter the result of GatherPackagingSpecs to discard items targeting outside "system" / "root" // partition. Note that "apex" module installs its contents to "apex"(fake partition) as well // for symbol lookup by imitating "activated" paths. -func (s *systemImage) filterPackagingSpec(ps android.PackagingSpec) bool { +func (s *systemImage) FilterPackagingSpec(ps android.PackagingSpec) bool { return !ps.SkipInstall() && - (ps.Partition() == "system" || ps.Partition() == "root") + (ps.Partition() == "system" || ps.Partition() == "root" || + strings.HasPrefix(ps.Partition(), "system/")) } diff --git a/filesystem/vbmeta.go b/filesystem/vbmeta.go index 6a3fc1f18..6a4785933 100644 --- a/filesystem/vbmeta.go +++ b/filesystem/vbmeta.go @@ -26,14 +26,25 @@ import ( func init() { android.RegisterModuleType("vbmeta", VbmetaFactory) + pctx.HostBinToolVariable("avbtool", "avbtool") } +var ( + extractPublicKeyRule = pctx.AndroidStaticRule("avb_extract_public_key", + blueprint.RuleParams{ + Command: `${avbtool} extract_public_key --key $in --output $out`, + CommandDeps: []string{ + "${avbtool}", + }, + }) +) + type vbmeta struct { android.ModuleBase properties VbmetaProperties - output android.OutputPath + output android.Path installDir android.InstallPath } @@ -60,8 +71,15 @@ type VbmetaProperties struct { // have to be signed (use_avb: true). Partitions proptools.Configurable[[]string] - // List of chained partitions that this vbmeta deletages the verification. - Chained_partitions []ChainedPartitionProperties + // Metadata about the chained partitions that this vbmeta delegates the verification. + // This is an alternative to chained_partitions, using chained_partitions instead is simpler + // in most cases. However, this property allows building this vbmeta partition without + // its chained partitions existing in this build. + Chained_partition_metadata []ChainedPartitionProperties + + // List of chained partitions that this vbmeta delegates the verification. They are the + // names of other vbmeta modules. + Chained_partitions []string // List of key-value pair of avb properties Avb_properties []avbProperty @@ -93,6 +111,20 @@ type ChainedPartitionProperties struct { Private_key *string `android:"path"` } +type vbmetaPartitionInfo struct { + // Name of the partition + Name string + + // Rollback index location, non-negative int + RollbackIndexLocation int + + // The path to the public key of the private key used to sign this partition. Derived from + // the private key. + PublicKey android.Path +} + +var vbmetaPartitionProvider = blueprint.NewProvider[vbmetaPartitionInfo]() + // vbmeta is the partition image that has the verification information for other partitions. func VbmetaFactory() android.Module { module := &vbmeta{} @@ -103,13 +135,18 @@ func VbmetaFactory() android.Module { type vbmetaDep struct { blueprint.BaseDependencyTag - kind string } -var vbmetaPartitionDep = vbmetaDep{kind: "partition"} +type chainedPartitionDep struct { + blueprint.BaseDependencyTag +} + +var vbmetaPartitionDep = vbmetaDep{} +var vbmetaChainedPartitionDep = chainedPartitionDep{} func (v *vbmeta) DepsMutator(ctx android.BottomUpMutatorContext) { ctx.AddDependency(ctx.Module(), vbmetaPartitionDep, v.properties.Partitions.GetOrDefault(ctx, nil)...) + ctx.AddDependency(ctx.Module(), vbmetaChainedPartitionDep, v.properties.Chained_partitions...) } func (v *vbmeta) installFileName() string { @@ -124,10 +161,6 @@ func (v *vbmeta) partitionName() string { const vbmetaMaxSize = 64 * 1024 func (v *vbmeta) GenerateAndroidBuildActions(ctx android.ModuleContext) { - extractedPublicKeys := v.extractPublicKeys(ctx) - - v.output = android.PathForModuleOut(ctx, v.installFileName()).OutputPath - builder := android.NewRuleBuilder(pctx, ctx) cmd := builder.Command().BuiltTool("avbtool").Text("make_vbmeta_image") @@ -175,43 +208,99 @@ func (v *vbmeta) GenerateAndroidBuildActions(ctx android.ModuleContext) { cmd.FlagWithInput("--include_descriptors_from_image ", signedImage) } - for i, cp := range v.properties.Chained_partitions { - name := proptools.String(cp.Name) + seenRils := make(map[int]bool) + for _, cp := range ctx.GetDirectDepsWithTag(vbmetaChainedPartitionDep) { + info, ok := android.OtherModuleProvider(ctx, cp, vbmetaPartitionProvider) + if !ok { + ctx.PropertyErrorf("chained_partitions", "Expected all modules in chained_partitions to provide vbmetaPartitionProvider, but %s did not", cp.Name()) + continue + } + if info.Name == "" { + ctx.PropertyErrorf("chained_partitions", "name must be specified") + continue + } + + ril := info.RollbackIndexLocation + if ril < 0 { + ctx.PropertyErrorf("chained_partitions", "rollback index location must be 0, 1, 2, ...") + continue + } else if seenRils[ril] { + ctx.PropertyErrorf("chained_partitions", "Multiple chained partitions with the same rollback index location %d", ril) + continue + } + seenRils[ril] = true + + publicKey := info.PublicKey + cmd.FlagWithArg("--chain_partition ", fmt.Sprintf("%s:%d:%s", info.Name, ril, publicKey.String())) + cmd.Implicit(publicKey) + } + for _, cpm := range v.properties.Chained_partition_metadata { + name := proptools.String(cpm.Name) if name == "" { ctx.PropertyErrorf("chained_partitions", "name must be specified") continue } - ril := proptools.IntDefault(cp.Rollback_index_location, i+1) + ril := proptools.IntDefault(cpm.Rollback_index_location, -1) if ril < 0 { - ctx.PropertyErrorf("chained_partitions", "must be 0, 1, 2, ...") + ctx.PropertyErrorf("chained_partition_metadata", "rollback index location must be 0, 1, 2, ...") + continue + } else if seenRils[ril] { + ctx.PropertyErrorf("chained_partition_metadata", "Multiple chained partitions with the same rollback index location %d", ril) continue } + seenRils[ril] = true var publicKey android.Path - if cp.Public_key != nil { - publicKey = android.PathForModuleSrc(ctx, proptools.String(cp.Public_key)) + if cpm.Public_key != nil { + publicKey = android.PathForModuleSrc(ctx, *cpm.Public_key) + } else if cpm.Private_key != nil { + privateKey := android.PathForModuleSrc(ctx, *cpm.Private_key) + extractedPublicKey := android.PathForModuleOut(ctx, "chained_metadata", name+".avbpubkey") + ctx.Build(pctx, android.BuildParams{ + Rule: extractPublicKeyRule, + Input: privateKey, + Output: extractedPublicKey, + }) + publicKey = extractedPublicKey } else { - publicKey = extractedPublicKeys[name] + ctx.PropertyErrorf("public_key", "Either public_key or private_key must be specified") + continue } + cmd.FlagWithArg("--chain_partition ", fmt.Sprintf("%s:%d:%s", name, ril, publicKey.String())) cmd.Implicit(publicKey) } - cmd.FlagWithOutput("--output ", v.output) + output := android.PathForModuleOut(ctx, v.installFileName()) + cmd.FlagWithOutput("--output ", output) // libavb expects to be able to read the maximum vbmeta size, so we must provide a partition // which matches this or the read will fail. builder.Command().Text("truncate"). FlagWithArg("-s ", strconv.Itoa(vbmetaMaxSize)). - Output(v.output) + Output(output) builder.Build("vbmeta", fmt.Sprintf("vbmeta %s", ctx.ModuleName())) v.installDir = android.PathForModuleInstall(ctx, "etc") - ctx.InstallFile(v.installDir, v.installFileName(), v.output) - - ctx.SetOutputFiles([]android.Path{v.output}, "") + ctx.InstallFile(v.installDir, v.installFileName(), output) + + extractedPublicKey := android.PathForModuleOut(ctx, v.partitionName()+".avbpubkey") + ctx.Build(pctx, android.BuildParams{ + Rule: extractPublicKeyRule, + Input: key, + Output: extractedPublicKey, + }) + + android.SetProvider(ctx, vbmetaPartitionProvider, vbmetaPartitionInfo{ + Name: v.partitionName(), + RollbackIndexLocation: ril, + PublicKey: extractedPublicKey, + }) + + ctx.SetOutputFiles([]android.Path{output}, "") + v.output = output } // Returns the embedded shell command that prints the rollback index @@ -224,43 +313,6 @@ func (v *vbmeta) rollbackIndexCommand(ctx android.ModuleContext) string { } } -// Extract public keys from chained_partitions.private_key. The keys are indexed with the partition -// name. -func (v *vbmeta) extractPublicKeys(ctx android.ModuleContext) map[string]android.OutputPath { - result := make(map[string]android.OutputPath) - - builder := android.NewRuleBuilder(pctx, ctx) - for _, cp := range v.properties.Chained_partitions { - if cp.Private_key == nil { - continue - } - - name := proptools.String(cp.Name) - if name == "" { - ctx.PropertyErrorf("chained_partitions", "name must be specified") - continue - } - - if _, ok := result[name]; ok { - ctx.PropertyErrorf("chained_partitions", "name %q is duplicated", name) - continue - } - - privateKeyFile := android.PathForModuleSrc(ctx, proptools.String(cp.Private_key)) - publicKeyFile := android.PathForModuleOut(ctx, name+".avbpubkey").OutputPath - - builder.Command(). - BuiltTool("avbtool"). - Text("extract_public_key"). - FlagWithInput("--key ", privateKeyFile). - FlagWithOutput("--output ", publicKeyFile) - - result[name] = publicKeyFile - } - builder.Build("vbmeta_extract_public_key", fmt.Sprintf("Extract public keys for %s", ctx.ModuleName())) - return result -} - var _ android.AndroidMkProviderInfoProducer = (*vbmeta)(nil) func (v *vbmeta) PrepareAndroidMKProviderInfo(config android.Config) *android.AndroidMkProviderInfo { diff --git a/fsgen/filesystem_creator.go b/fsgen/filesystem_creator.go index bcf236160..c74579aba 100644 --- a/fsgen/filesystem_creator.go +++ b/fsgen/filesystem_creator.go @@ -62,7 +62,9 @@ func filesystemCreatorFactory() android.Module { module.AddProperties(&module.properties) android.AddLoadHook(module, func(ctx android.LoadHookContext) { generatedPrebuiltEtcModuleNames := createPrebuiltEtcModules(ctx) - createFsGenState(ctx, generatedPrebuiltEtcModuleNames) + avbpubkeyGenerated := createAvbpubkeyModule(ctx) + createFsGenState(ctx, generatedPrebuiltEtcModuleNames, avbpubkeyGenerated) + module.createAvbKeyFilegroups(ctx) module.createInternalModules(ctx) }) @@ -211,6 +213,14 @@ func partitionSpecificFsProps(fsProps *filesystem.FilesystemProperties, partitio } } +var ( + dlkmPartitions = []string{ + "system_dlkm", + "vendor_dlkm", + "odm_dlkm", + } +) + // Creates a soong module to build the given partition. Returns false if we can't support building // it. func (f *filesystemCreator) createPartition(ctx android.LoadHookContext, partitionType string) bool { @@ -222,23 +232,17 @@ func (f *filesystemCreator) createPartition(ctx android.LoadHookContext, partiti } if partitionType == "vendor" || partitionType == "product" { - fsProps.Linkerconfig.Gen_linker_config = proptools.BoolPtr(true) - fsProps.Linkerconfig.Linker_config_srcs = f.createLinkerConfigSourceFilegroups(ctx, partitionType) + fsProps.Linker_config.Gen_linker_config = proptools.BoolPtr(true) + fsProps.Linker_config.Linker_config_srcs = f.createLinkerConfigSourceFilegroups(ctx, partitionType) } - if partitionType == "system_dlkm" { - kernelModules := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.SystemKernelModules - f.createPrebuiltKernelModules(ctx, partitionType, kernelModules) + if android.InList(partitionType, dlkmPartitions) { + f.createPrebuiltKernelModules(ctx, partitionType) } var module android.Module if partitionType == "system" { module = ctx.CreateModule(filesystem.SystemImageFactory, baseProps, fsProps) - } else if partitionType == "system_dlkm" { - // Do not set partition_type. build/soong/android/paths#modulePartition currently does not support dlkm - // partitions. Since `android_filesystem` uses a partition based filter, setting the partition here - // would result in missing in entries. - module = ctx.CreateModule(filesystem.FilesystemFactory, baseProps, fsProps) } else { // Explicitly set the partition. fsProps.Partition_type = proptools.StringPtr(partitionType) @@ -251,21 +255,104 @@ func (f *filesystemCreator) createPartition(ctx android.LoadHookContext, partiti return true } +// Creates filegroups for the files specified in BOARD_(partition_)AVB_KEY_PATH +func (f *filesystemCreator) createAvbKeyFilegroups(ctx android.LoadHookContext) { + partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + var files []string + + if len(partitionVars.BoardAvbKeyPath) > 0 { + files = append(files, partitionVars.BoardAvbKeyPath) + } + for _, partition := range android.SortedKeys(partitionVars.PartitionQualifiedVariables) { + specificPartitionVars := partitionVars.PartitionQualifiedVariables[partition] + if len(specificPartitionVars.BoardAvbKeyPath) > 0 { + files = append(files, specificPartitionVars.BoardAvbKeyPath) + } + } + + fsGenState := ctx.Config().Get(fsGenStateOnceKey).(*FsGenState) + for _, file := range files { + if _, ok := fsGenState.avbKeyFilegroups[file]; ok { + continue + } + if file == "external/avb/test/data/testkey_rsa4096.pem" { + // There already exists a checked-in filegroup for this commonly-used key, just use that + fsGenState.avbKeyFilegroups[file] = "avb_testkey_rsa4096" + continue + } + dir := filepath.Dir(file) + base := filepath.Base(file) + name := fmt.Sprintf("avb_key_%x", strings.ReplaceAll(file, "/", "_")) + ctx.CreateModuleInDirectory( + android.FileGroupFactory, + dir, + &struct { + Name *string + Srcs []string + Visibility []string + }{ + Name: proptools.StringPtr(name), + Srcs: []string{base}, + Visibility: []string{"//visibility:public"}, + }, + ) + fsGenState.avbKeyFilegroups[file] = name + } +} + // createPrebuiltKernelModules creates `prebuilt_kernel_modules`. These modules will be added to deps of the // autogenerated *_dlkm filsystem modules. Each _dlkm partition should have a single prebuilt_kernel_modules dependency. // This ensures that the depmod artifacts (modules.* installed in /lib/modules/) are generated with a complete view. - -// The input `kernelModules` is a space separated list of .ko files in the workspace. -func (f *filesystemCreator) createPrebuiltKernelModules(ctx android.LoadHookContext, partitionType string, kernelModules []string) { +func (f *filesystemCreator) createPrebuiltKernelModules(ctx android.LoadHookContext, partitionType string) { fsGenState := ctx.Config().Get(fsGenStateOnceKey).(*FsGenState) name := generatedModuleName(ctx.Config(), fmt.Sprintf("%s-kernel-modules", partitionType)) props := &struct { - Name *string - Srcs []string + Name *string + Srcs []string + System_deps []string + System_dlkm_specific *bool + Vendor_dlkm_specific *bool + Odm_dlkm_specific *bool + Load_by_default *bool + Blocklist_file *string }{ Name: proptools.StringPtr(name), - Srcs: kernelModules, } + switch partitionType { + case "system_dlkm": + props.Srcs = ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.SystemKernelModules + props.System_dlkm_specific = proptools.BoolPtr(true) + if len(ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.SystemKernelLoadModules) == 0 { + // Create empty modules.load file for system + // https://source.corp.google.com/h/googleplex-android/platform/build/+/ef55daac9954896161b26db4f3ef1781b5a5694c:core/Makefile;l=695-700;drc=549fe2a5162548bd8b47867d35f907eb22332023;bpv=1;bpt=0 + props.Load_by_default = proptools.BoolPtr(false) + } + if blocklistFile := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.SystemKernelBlocklistFile; blocklistFile != "" { + props.Blocklist_file = proptools.StringPtr(blocklistFile) + } + case "vendor_dlkm": + props.Srcs = ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.VendorKernelModules + if len(ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.SystemKernelModules) > 0 { + props.System_deps = []string{":" + generatedModuleName(ctx.Config(), "system_dlkm-kernel-modules") + "{.modules}"} + } + props.Vendor_dlkm_specific = proptools.BoolPtr(true) + if blocklistFile := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.VendorKernelBlocklistFile; blocklistFile != "" { + props.Blocklist_file = proptools.StringPtr(blocklistFile) + } + case "odm_dlkm": + props.Srcs = ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.OdmKernelModules + props.Odm_dlkm_specific = proptools.BoolPtr(true) + if blocklistFile := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.OdmKernelBlocklistFile; blocklistFile != "" { + props.Blocklist_file = proptools.StringPtr(blocklistFile) + } + default: + ctx.ModuleErrorf("DLKM is not supported for %s\n", partitionType) + } + + if len(props.Srcs) == 0 { + return // do not generate `prebuilt_kernel_modules` if there are no sources + } + kernelModule := ctx.CreateModuleInDirectory( kernel.PrebuiltKernelModulesFactory, ".", // create in root directory for now @@ -378,16 +465,43 @@ func generateBaseProps(namePtr *string) *filesystemBaseProperty { } func generateFsProps(ctx android.EarlyModuleContext, partitionType string) (*filesystem.FilesystemProperties, bool) { + fsGenState := ctx.Config().Get(fsGenStateOnceKey).(*FsGenState) fsProps := &filesystem.FilesystemProperties{} partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse - specificPartitionVars := partitionVars.PartitionQualifiedVariables[partitionType] - - // BOARD_SYSTEMIMAGE_FILE_SYSTEM_TYPE - fsType := specificPartitionVars.BoardFileSystemType - if fsType == "" { - fsType = "ext4" //default + var boardAvbEnable bool + var boardAvbKeyPath string + var boardAvbAlgorithm string + var boardAvbRollbackIndex string + var fsType string + if strings.Contains(partitionType, "ramdisk") { + fsType = "compressed_cpio" + } else { + specificPartitionVars := partitionVars.PartitionQualifiedVariables[partitionType] + fsType = specificPartitionVars.BoardFileSystemType + boardAvbEnable = partitionVars.BoardAvbEnable + boardAvbKeyPath = specificPartitionVars.BoardAvbKeyPath + boardAvbAlgorithm = specificPartitionVars.BoardAvbAlgorithm + boardAvbRollbackIndex = specificPartitionVars.BoardAvbRollbackIndex + if boardAvbEnable { + if boardAvbKeyPath == "" { + boardAvbKeyPath = partitionVars.BoardAvbKeyPath + } + if boardAvbAlgorithm == "" { + boardAvbAlgorithm = partitionVars.BoardAvbAlgorithm + } + if boardAvbRollbackIndex == "" { + boardAvbRollbackIndex = partitionVars.BoardAvbRollbackIndex + } + } + if fsType == "" { + fsType = "ext4" //default + } + } + if boardAvbKeyPath != "" { + boardAvbKeyPath = ":" + fsGenState.avbKeyFilegroups[boardAvbKeyPath] } + fsProps.Type = proptools.StringPtr(fsType) if filesystem.GetFsTypeFromString(ctx, *fsProps.Type).IsUnknown() { // Currently the android_filesystem module type only supports a handful of FS types like ext4, erofs @@ -399,13 +513,13 @@ func generateFsProps(ctx android.EarlyModuleContext, partitionType string) (*fil fsProps.Unchecked_module = proptools.BoolPtr(true) // BOARD_AVB_ENABLE - fsProps.Use_avb = proptools.BoolPtr(partitionVars.BoardAvbEnable) + fsProps.Use_avb = proptools.BoolPtr(boardAvbEnable) // BOARD_AVB_KEY_PATH - fsProps.Avb_private_key = proptools.StringPtr(specificPartitionVars.BoardAvbKeyPath) + fsProps.Avb_private_key = proptools.StringPtr(boardAvbKeyPath) // BOARD_AVB_ALGORITHM - fsProps.Avb_algorithm = proptools.StringPtr(specificPartitionVars.BoardAvbAlgorithm) + fsProps.Avb_algorithm = proptools.StringPtr(boardAvbAlgorithm) // BOARD_AVB_SYSTEM_ROLLBACK_INDEX - if rollbackIndex, err := strconv.ParseInt(specificPartitionVars.BoardAvbRollbackIndex, 10, 64); err == nil { + if rollbackIndex, err := strconv.ParseInt(boardAvbRollbackIndex, 10, 64); err == nil { fsProps.Rollback_index = proptools.Int64Ptr(rollbackIndex) } @@ -544,8 +658,10 @@ func generateBpContent(ctx android.EarlyModuleContext, partitionType string) str } baseProps := generateBaseProps(proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), partitionType))) - deps := ctx.Config().Get(fsGenStateOnceKey).(*FsGenState).fsDeps[partitionType] - depProps := generateDepStruct(*deps) + fsGenState := ctx.Config().Get(fsGenStateOnceKey).(*FsGenState) + deps := fsGenState.fsDeps[partitionType] + highPriorityDeps := fsGenState.generatedPrebuiltEtcModuleNames + depProps := generateDepStruct(*deps, highPriorityDeps) result, err := proptools.RepackProperties([]interface{}{baseProps, fsProps, depProps}) if err != nil { diff --git a/fsgen/filesystem_creator_test.go b/fsgen/filesystem_creator_test.go index 199eaad11..fe4a40325 100644 --- a/fsgen/filesystem_creator_test.go +++ b/fsgen/filesystem_creator_test.go @@ -16,6 +16,7 @@ package fsgen import ( "android/soong/android" + "android/soong/etc" "android/soong/filesystem" "android/soong/java" "testing" @@ -46,6 +47,12 @@ func TestFileSystemCreatorSystemImageProps(t *testing.T) { }), android.FixtureMergeMockFs(android.MockFS{ "external/avb/test/data/testkey_rsa4096.pem": nil, + "external/avb/test/Android.bp": []byte(` + filegroup { + name: "avb_testkey_rsa4096", + srcs: ["data/testkey_rsa4096.pem"], + } + `), "build/soong/fsgen/Android.bp": []byte(` soong_filesystem_creator { name: "foo", @@ -65,8 +72,8 @@ func TestFileSystemCreatorSystemImageProps(t *testing.T) { ) android.AssertStringEquals( t, - "Property expected to match the product variable 'BOARD_AVB_KEY_PATH'", - "external/avb/test/data/testkey_rsa4096.pem", + "Property the avb_private_key property to be set to the existing filegroup", + ":avb_testkey_rsa4096", proptools.String(fooSystem.FsProps().Avb_private_key), ) android.AssertStringEquals( @@ -253,3 +260,129 @@ java_library { _, libFooInDeps := (*resolvedSystemDeps)["libfoo"] android.AssertBoolEquals(t, "libfoo should not appear in deps because it has been overridden by libbaz. The latter is a required dep of libbar, which is listed in PRODUCT_PACKAGES", false, libFooInDeps) } + +func TestPrebuiltEtcModuleGen(t *testing.T) { + result := android.GroupFixturePreparers( + android.PrepareForIntegrationTestWithAndroid, + android.PrepareForTestWithAndroidBuildComponents, + android.PrepareForTestWithAllowMissingDependencies, + filesystem.PrepareForTestWithFilesystemBuildComponents, + prepareForTestWithFsgenBuildComponents, + android.FixtureModifyConfig(func(config android.Config) { + config.TestProductVariables.PartitionVarsForSoongMigrationOnlyDoNotUse.ProductCopyFiles = []string{ + "frameworks/base/config/preloaded-classes:system/etc/preloaded-classes", + "frameworks/base/data/keyboards/Vendor_0079_Product_0011.kl:system/usr/keylayout/subdir/Vendor_0079_Product_0011.kl", + "frameworks/base/data/keyboards/Vendor_0079_Product_18d4.kl:system/usr/keylayout/subdir/Vendor_0079_Product_18d4.kl", + "some/non/existing/file.txt:system/etc/file.txt", + "device/sample/etc/apns-full-conf.xml:product/etc/apns-conf.xml:google", + "device/sample/etc/apns-full-conf.xml:product/etc/apns-conf-2.xml", + } + config.TestProductVariables.PartitionVarsForSoongMigrationOnlyDoNotUse.PartitionQualifiedVariables = + map[string]android.PartitionQualifiedVariablesType{ + "system": { + BoardFileSystemType: "ext4", + }, + } + }), + android.FixtureMergeMockFs(android.MockFS{ + "external/avb/test/data/testkey_rsa4096.pem": nil, + "build/soong/fsgen/Android.bp": []byte(` + soong_filesystem_creator { + name: "foo", + } + `), + "frameworks/base/config/preloaded-classes": nil, + "frameworks/base/data/keyboards/Vendor_0079_Product_0011.kl": nil, + "frameworks/base/data/keyboards/Vendor_0079_Product_18d4.kl": nil, + "device/sample/etc/apns-full-conf.xml": nil, + }), + ).RunTest(t) + + checkModuleProp := func(m android.Module, matcher func(actual interface{}) bool) bool { + for _, prop := range m.GetProperties() { + + if matcher(prop) { + return true + } + } + return false + } + + // check generated prebuilt_* module type install path and install partition + generatedModule := result.ModuleForTests("system-frameworks_base_config-etc-0", "android_arm64_armv8-a").Module() + etcModule, _ := generatedModule.(*etc.PrebuiltEtc) + android.AssertStringEquals( + t, + "module expected to have etc install path", + "etc", + etcModule.BaseDir(), + ) + android.AssertBoolEquals( + t, + "module expected to be installed in system partition", + true, + !generatedModule.InstallInProduct() && + !generatedModule.InstallInVendor() && + !generatedModule.InstallInSystemExt(), + ) + + // check generated prebuilt_* module specifies correct relative_install_path property + generatedModule = result.ModuleForTests("system-frameworks_base_data_keyboards-usr_keylayout_subdir-0", "android_arm64_armv8-a").Module() + etcModule, _ = generatedModule.(*etc.PrebuiltEtc) + android.AssertStringEquals( + t, + "module expected to set correct relative_install_path properties", + "subdir", + etcModule.SubDir(), + ) + + // check that prebuilt_* module is not generated for non existing source file + android.AssertPanicMessageContains( + t, + "prebuilt_* module not generated for non existing source file", + "failed to find module \"system-some_non_existing-etc-0\"", + func() { result.ModuleForTests("system-some_non_existing-etc-0", "android_arm64_armv8-a") }, + ) + + // check that duplicate src file can exist in PRODUCT_COPY_FILES and generates separate modules + generatedModule0 := result.ModuleForTests("product-device_sample_etc-etc-0", "android_arm64_armv8-a").Module() + generatedModule1 := result.ModuleForTests("product-device_sample_etc-etc-1", "android_arm64_armv8-a").Module() + + // check that generated prebuilt_* module sets correct srcs and dsts property + eval := generatedModule0.ConfigurableEvaluator(android.PanickingConfigAndErrorContext(result.TestContext)) + android.AssertBoolEquals( + t, + "module expected to set correct srcs and dsts properties", + true, + checkModuleProp(generatedModule0, func(actual interface{}) bool { + if p, ok := actual.(*etc.PrebuiltEtcProperties); ok { + srcs := p.Srcs.GetOrDefault(eval, nil) + dsts := p.Dsts.GetOrDefault(eval, nil) + return len(srcs) == 1 && + srcs[0] == "apns-full-conf.xml" && + len(dsts) == 1 && + dsts[0] == "apns-conf.xml" + } + return false + }), + ) + + // check that generated prebuilt_* module sets correct srcs and dsts property + eval = generatedModule1.ConfigurableEvaluator(android.PanickingConfigAndErrorContext(result.TestContext)) + android.AssertBoolEquals( + t, + "module expected to set correct srcs and dsts properties", + true, + checkModuleProp(generatedModule1, func(actual interface{}) bool { + if p, ok := actual.(*etc.PrebuiltEtcProperties); ok { + srcs := p.Srcs.GetOrDefault(eval, nil) + dsts := p.Dsts.GetOrDefault(eval, nil) + return len(srcs) == 1 && + srcs[0] == "apns-full-conf.xml" && + len(dsts) == 1 && + dsts[0] == "apns-conf-2.xml" + } + return false + }), + ) +} diff --git a/fsgen/fsgen_mutators.go b/fsgen/fsgen_mutators.go index e530a5b0a..7b3026404 100644 --- a/fsgen/fsgen_mutators.go +++ b/fsgen/fsgen_mutators.go @@ -17,6 +17,7 @@ package fsgen import ( "fmt" "slices" + "strings" "sync" "android/soong/android" @@ -65,6 +66,10 @@ type FsGenState struct { fsDepsMutex sync.Mutex // Map of _all_ soong module names to their corresponding installation properties moduleToInstallationProps map[string]installationProperties + // List of prebuilt_* modules that are autogenerated. + generatedPrebuiltEtcModuleNames []string + // Mapping from a path to an avb key to the name of a filegroup module that contains it + avbKeyFilegroups map[string]string } type installationProperties struct { @@ -80,7 +85,7 @@ func defaultDepCandidateProps(config android.Config) *depCandidateProps { } func generatedPartitions(ctx android.LoadHookContext) []string { - generatedPartitions := []string{"system"} + generatedPartitions := []string{"system", "ramdisk"} if ctx.DeviceConfig().SystemExtPath() == "system_ext" { generatedPartitions = append(generatedPartitions, "system_ext") } @@ -99,16 +104,23 @@ func generatedPartitions(ctx android.LoadHookContext) []string { if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingSystemDlkmImage { generatedPartitions = append(generatedPartitions, "system_dlkm") } + if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingVendorDlkmImage { + generatedPartitions = append(generatedPartitions, "vendor_dlkm") + } + if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingOdmDlkmImage { + generatedPartitions = append(generatedPartitions, "odm_dlkm") + } + return generatedPartitions } -func createFsGenState(ctx android.LoadHookContext, generatedPrebuiltEtcModuleNames []string) *FsGenState { +func createFsGenState(ctx android.LoadHookContext, generatedPrebuiltEtcModuleNames []string, avbpubkeyGenerated bool) *FsGenState { return ctx.Config().Once(fsGenStateOnceKey, func() interface{} { partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse candidates := android.FirstUniqueStrings(android.Concat(partitionVars.ProductPackages, partitionVars.ProductPackagesDebug)) candidates = android.Concat(candidates, generatedPrebuiltEtcModuleNames) - return &FsGenState{ + fsGenState := FsGenState{ depCandidates: candidates, fsDeps: map[string]*multilibDeps{ // These additional deps are added according to the cuttlefish system image bp. @@ -146,12 +158,37 @@ func createFsGenState(ctx android.LoadHookContext, generatedPrebuiltEtcModuleNam "com.android.vndk.v33": defaultDepCandidateProps(ctx.Config()), "com.android.vndk.v34": defaultDepCandidateProps(ctx.Config()), }, - "system_dlkm": {}, - "userdata": {}, + "userdata": {}, + "system_dlkm": { + // these are phony required deps of the phony fs_config_dirs_nonsystem + "fs_config_dirs_system_dlkm": defaultDepCandidateProps(ctx.Config()), + "fs_config_files_system_dlkm": defaultDepCandidateProps(ctx.Config()), + // build props are automatically added to `ALL_DEFAULT_INSTALLED_MODULES` + "system_dlkm-build.prop": defaultDepCandidateProps(ctx.Config()), + }, + "vendor_dlkm": { + "fs_config_dirs_vendor_dlkm": defaultDepCandidateProps(ctx.Config()), + "fs_config_files_vendor_dlkm": defaultDepCandidateProps(ctx.Config()), + "vendor_dlkm-build.prop": defaultDepCandidateProps(ctx.Config()), + }, + "odm_dlkm": { + "fs_config_dirs_odm_dlkm": defaultDepCandidateProps(ctx.Config()), + "fs_config_files_odm_dlkm": defaultDepCandidateProps(ctx.Config()), + "odm_dlkm-build.prop": defaultDepCandidateProps(ctx.Config()), + }, + "ramdisk": {}, }, - fsDepsMutex: sync.Mutex{}, - moduleToInstallationProps: map[string]installationProperties{}, + fsDepsMutex: sync.Mutex{}, + moduleToInstallationProps: map[string]installationProperties{}, + generatedPrebuiltEtcModuleNames: generatedPrebuiltEtcModuleNames, + avbKeyFilegroups: map[string]string{}, + } + + if avbpubkeyGenerated { + (*fsGenState.fsDeps["product"])["system_other_avbpubkey"] = defaultDepCandidateProps(ctx.Config()) } + + return &fsGenState }).(*FsGenState) } @@ -182,29 +219,31 @@ func appendDepIfAppropriate(mctx android.BottomUpMutatorContext, deps *multilibD } func collectDepsMutator(mctx android.BottomUpMutatorContext) { + m := mctx.Module() + if m.Target().Os.Class != android.Device { + return + } fsGenState := mctx.Config().Get(fsGenStateOnceKey).(*FsGenState) - m := mctx.Module() - if m.Target().Os.Class == android.Device && slices.Contains(fsGenState.depCandidates, mctx.ModuleName()) { + fsGenState.fsDepsMutex.Lock() + defer fsGenState.fsDepsMutex.Unlock() + + if slices.Contains(fsGenState.depCandidates, mctx.ModuleName()) { installPartition := m.PartitionTag(mctx.DeviceConfig()) - fsGenState.fsDepsMutex.Lock() // Only add the module as dependency when: // - its enabled // - its namespace is included in PRODUCT_SOONG_NAMESPACES if m.Enabled(mctx) && m.ExportedToMake() { appendDepIfAppropriate(mctx, fsGenState.fsDeps[installPartition], installPartition) } - fsGenState.fsDepsMutex.Unlock() } // store the map of module to (required,overrides) even if the module is not in PRODUCT_PACKAGES. // the module might be installed transitively. - if m.Target().Os.Class == android.Device && m.Enabled(mctx) && m.ExportedToMake() { - fsGenState.fsDepsMutex.Lock() + if m.Enabled(mctx) && m.ExportedToMake() { fsGenState.moduleToInstallationProps[m.Name()] = installationProperties{ Required: m.RequiredModuleNames(mctx), Overrides: m.Overrides(), } - fsGenState.fsDepsMutex.Unlock() } } @@ -251,7 +290,7 @@ func setDepsMutator(mctx android.BottomUpMutatorContext) { soongGeneratedPartitionMap := getAllSoongGeneratedPartitionNames(mctx.Config(), fsGenState.soongGeneratedPartitions) m := mctx.Module() if partition, ok := soongGeneratedPartitionMap[m.Name()]; ok { - depsStruct := generateDepStruct(*fsDeps[partition]) + depsStruct := generateDepStruct(*fsDeps[partition], fsGenState.generatedPrebuiltEtcModuleNames) if err := proptools.AppendMatchingProperties(m.GetProperties(), depsStruct, nil); err != nil { mctx.ModuleErrorf(err.Error()) } @@ -301,12 +340,21 @@ func removeOverriddenDeps(mctx android.BottomUpMutatorContext) { var HighPriorityDeps = []string{} -func generateDepStruct(deps map[string]*depCandidateProps) *packagingPropsStruct { +func isHighPriorityDep(depName string) bool { + for _, highPriorityDeps := range HighPriorityDeps { + if strings.HasPrefix(depName, highPriorityDeps) { + return true + } + } + return false +} + +func generateDepStruct(deps map[string]*depCandidateProps, highPriorityDeps []string) *packagingPropsStruct { depsStruct := packagingPropsStruct{} for depName, depProps := range deps { bitness := getBitness(depProps.Arch) fullyQualifiedDepName := fullyQualifiedModuleName(depName, depProps.Namespace) - if android.InList(depName, HighPriorityDeps) { + if android.InList(depName, highPriorityDeps) { depsStruct.High_priority_deps = append(depsStruct.High_priority_deps, fullyQualifiedDepName) } else if android.InList("32", bitness) && android.InList("64", bitness) { // If both 32 and 64 bit variants are enabled for this module @@ -341,6 +389,7 @@ func generateDepStruct(deps map[string]*depCandidateProps) *packagingPropsStruct depsStruct.Multilib.Prefer32.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Prefer32.Deps) depsStruct.Multilib.Both.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Both.Deps) depsStruct.Multilib.Common.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Common.Deps) + depsStruct.High_priority_deps = android.SortedUniqueStrings(depsStruct.High_priority_deps) return &depsStruct } diff --git a/fsgen/prebuilt_etc_modules_gen.go b/fsgen/prebuilt_etc_modules_gen.go index 362ac31b8..efbc462e1 100644 --- a/fsgen/prebuilt_etc_modules_gen.go +++ b/fsgen/prebuilt_etc_modules_gen.go @@ -85,15 +85,30 @@ func appendIfCorrectInstallPartition(partitionToInstallPathList []partitionToIns } } -func uniqueExistingProductCopyFileMap(ctx android.LoadHookContext) map[string]string { +// Create a map of source files to the list of destination files from PRODUCT_COPY_FILES entries. +// Note that the value of the map is a list of string, given that a single source file can be +// copied to multiple files. +// This function also checks the existence of the source files, and validates that there is no +// multiple source files copying to the same dest file. +func uniqueExistingProductCopyFileMap(ctx android.LoadHookContext) map[string][]string { seen := make(map[string]bool) - filtered := make(map[string]string) + filtered := make(map[string][]string) + + for _, copyFilePair := range ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.ProductCopyFiles { + srcDestList := strings.Split(copyFilePair, ":") + if len(srcDestList) < 2 { + ctx.ModuleErrorf("PRODUCT_COPY_FILES must follow the format \"src:dest\", got: %s", copyFilePair) + } + src, dest := srcDestList[0], srcDestList[1] + + // Some downstream branches use absolute path as entries in PRODUCT_COPY_FILES. + // Convert them to relative path from top and check if they do not escape the tree root. + relSrc := android.ToRelativeSourcePath(ctx, src) - for src, dest := range ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.ProductCopyFiles { if _, ok := seen[dest]; !ok { - if optionalPath := android.ExistentPathForSource(ctx, src); optionalPath.Valid() { + if optionalPath := android.ExistentPathForSource(ctx, relSrc); optionalPath.Valid() { seen[dest] = true - filtered[src] = dest + filtered[relSrc] = append(filtered[relSrc], dest) } } } @@ -121,12 +136,14 @@ func processProductCopyFiles(ctx android.LoadHookContext) map[string]*prebuiltSr groupedSources := map[string]*prebuiltSrcGroupByInstallPartition{} for _, src := range android.SortedKeys(productCopyFileMap) { - dest := productCopyFileMap[src] + destFiles := productCopyFileMap[src] srcFileDir := filepath.Dir(src) if _, ok := groupedSources[srcFileDir]; !ok { groupedSources[srcFileDir] = newPrebuiltSrcGroupByInstallPartition() } - appendIfCorrectInstallPartition(partitionToInstallPathList, dest, filepath.Base(src), groupedSources[srcFileDir]) + for _, dest := range destFiles { + appendIfCorrectInstallPartition(partitionToInstallPathList, dest, filepath.Base(src), groupedSources[srcFileDir]) + } } return groupedSources @@ -182,6 +199,7 @@ var ( "res": etc.PrebuiltResFactory, "rfs": etc.PrebuiltRfsFactory, "tts": etc.PrebuiltVoicepackFactory, + "tvconfig": etc.PrebuiltTvConfigFactory, "tvservice": etc.PrebuiltTvServiceFactory, "usr/share": etc.PrebuiltUserShareFactory, "usr/hyphen-data": etc.PrebuiltUserHyphenDataFactory, @@ -195,12 +213,9 @@ var ( } ) -func createPrebuiltEtcModule(ctx android.LoadHookContext, partition, srcDir, destDir string, destFiles []srcBaseFileInstallBaseFileTuple) string { - moduleProps := &prebuiltModuleProperties{} - propsList := []interface{}{moduleProps} - +func generatedPrebuiltEtcModuleName(partition, srcDir, destDir string, count int) string { // generated module name follows the pattern: - // <install partition>-<src file path>-<relative install path from partition root>-<install file extension> + // <install partition>-<src file path>-<relative install path from partition root>-<number> // Note that all path separators are replaced with "_" in the name moduleName := partition if !android.InList(srcDir, []string{"", "."}) { @@ -209,33 +224,27 @@ func createPrebuiltEtcModule(ctx android.LoadHookContext, partition, srcDir, des if !android.InList(destDir, []string{"", "."}) { moduleName += fmt.Sprintf("-%s", strings.ReplaceAll(destDir, string(filepath.Separator), "_")) } - if len(destFiles) > 0 { - if ext := filepath.Ext(destFiles[0].srcBaseFile); ext != "" { - moduleName += fmt.Sprintf("-%s", strings.TrimPrefix(ext, ".")) - } - } - moduleProps.Name = proptools.StringPtr(moduleName) + moduleName += fmt.Sprintf("-%d", count) - allCopyFileNamesUnchanged := true - var srcBaseFiles, installBaseFiles []string + return moduleName +} + +func groupDestFilesBySrc(destFiles []srcBaseFileInstallBaseFileTuple) (ret map[string][]srcBaseFileInstallBaseFileTuple, maxLen int) { + ret = map[string][]srcBaseFileInstallBaseFileTuple{} + maxLen = 0 for _, tuple := range destFiles { - if tuple.srcBaseFile != tuple.installBaseFile { - allCopyFileNamesUnchanged = false + if _, ok := ret[tuple.srcBaseFile]; !ok { + ret[tuple.srcBaseFile] = []srcBaseFileInstallBaseFileTuple{} } - srcBaseFiles = append(srcBaseFiles, tuple.srcBaseFile) - installBaseFiles = append(installBaseFiles, tuple.installBaseFile) + ret[tuple.srcBaseFile] = append(ret[tuple.srcBaseFile], tuple) + maxLen = max(maxLen, len(ret[tuple.srcBaseFile])) } + return ret, maxLen +} - // Find out the most appropriate module type to generate - var etcInstallPathKey string - for _, etcInstallPath := range android.SortedKeys(etcInstallPathToFactoryList) { - // Do not break when found but iterate until the end to find a module with more - // specific install path - if strings.HasPrefix(destDir, etcInstallPath) { - etcInstallPathKey = etcInstallPath - } - } - destDir, _ = filepath.Rel(etcInstallPathKey, destDir) +func prebuiltEtcModuleProps(moduleName, partition string) prebuiltModuleProperties { + moduleProps := prebuiltModuleProperties{} + moduleProps.Name = proptools.StringPtr(moduleName) // Set partition specific properties switch partition { @@ -247,39 +256,82 @@ func createPrebuiltEtcModule(ctx android.LoadHookContext, partition, srcDir, des moduleProps.Soc_specific = proptools.BoolPtr(true) } - // Set appropriate srcs, dsts, and releative_install_path based on - // the source and install file names - if allCopyFileNamesUnchanged { - moduleProps.Srcs = srcBaseFiles - - // Specify relative_install_path if it is not installed in the root directory of the - // partition - if !android.InList(destDir, []string{"", "."}) { - propsList = append(propsList, &prebuiltSubdirProperties{ - Relative_install_path: proptools.StringPtr(destDir), - }) - } - } else { - moduleProps.Srcs = srcBaseFiles - dsts := []string{} - for _, installBaseFile := range installBaseFiles { - dsts = append(dsts, filepath.Join(destDir, installBaseFile)) - } - moduleProps.Dsts = dsts - } - moduleProps.No_full_install = proptools.BoolPtr(true) moduleProps.NamespaceExportedToMake = true moduleProps.Visibility = []string{"//visibility:public"} - ctx.CreateModuleInDirectory(etcInstallPathToFactoryList[etcInstallPathKey], srcDir, propsList...) + return moduleProps +} - return moduleName +func createPrebuiltEtcModulesInDirectory(ctx android.LoadHookContext, partition, srcDir, destDir string, destFiles []srcBaseFileInstallBaseFileTuple) (moduleNames []string) { + groupedDestFiles, maxLen := groupDestFilesBySrc(destFiles) + + // Find out the most appropriate module type to generate + var etcInstallPathKey string + for _, etcInstallPath := range android.SortedKeys(etcInstallPathToFactoryList) { + // Do not break when found but iterate until the end to find a module with more + // specific install path + if strings.HasPrefix(destDir, etcInstallPath) { + etcInstallPathKey = etcInstallPath + } + } + relDestDirFromInstallDirBase, _ := filepath.Rel(etcInstallPathKey, destDir) + + for fileIndex := range maxLen { + srcTuple := []srcBaseFileInstallBaseFileTuple{} + for _, srcFile := range android.SortedKeys(groupedDestFiles) { + groupedDestFile := groupedDestFiles[srcFile] + if len(groupedDestFile) > fileIndex { + srcTuple = append(srcTuple, groupedDestFile[fileIndex]) + } + } + + moduleName := generatedPrebuiltEtcModuleName(partition, srcDir, destDir, fileIndex) + moduleProps := prebuiltEtcModuleProps(moduleName, partition) + modulePropsPtr := &moduleProps + propsList := []interface{}{modulePropsPtr} + + allCopyFileNamesUnchanged := true + var srcBaseFiles, installBaseFiles []string + for _, tuple := range srcTuple { + if tuple.srcBaseFile != tuple.installBaseFile { + allCopyFileNamesUnchanged = false + } + srcBaseFiles = append(srcBaseFiles, tuple.srcBaseFile) + installBaseFiles = append(installBaseFiles, tuple.installBaseFile) + } + + // Set appropriate srcs, dsts, and releative_install_path based on + // the source and install file names + if allCopyFileNamesUnchanged { + modulePropsPtr.Srcs = srcBaseFiles + + // Specify relative_install_path if it is not installed in the root directory of the + // partition + if !android.InList(relDestDirFromInstallDirBase, []string{"", "."}) { + propsList = append(propsList, &prebuiltSubdirProperties{ + Relative_install_path: proptools.StringPtr(relDestDirFromInstallDirBase), + }) + } + } else { + modulePropsPtr.Srcs = srcBaseFiles + dsts := []string{} + for _, installBaseFile := range installBaseFiles { + dsts = append(dsts, filepath.Join(relDestDirFromInstallDirBase, installBaseFile)) + } + modulePropsPtr.Dsts = dsts + } + + ctx.CreateModuleInDirectory(etcInstallPathToFactoryList[etcInstallPathKey], srcDir, propsList...) + moduleNames = append(moduleNames, moduleName) + } + + return moduleNames } func createPrebuiltEtcModulesForPartition(ctx android.LoadHookContext, partition, srcDir string, destDirFilesMap map[string][]srcBaseFileInstallBaseFileTuple) (ret []string) { for _, destDir := range android.SortedKeys(destDirFilesMap) { - ret = append(ret, createPrebuiltEtcModule(ctx, partition, srcDir, destDir, destDirFilesMap[destDir])) + ret = append(ret, createPrebuiltEtcModulesInDirectory(ctx, partition, srcDir, destDir, destDirFilesMap[destDir])...) } return ret } @@ -298,3 +350,28 @@ func createPrebuiltEtcModules(ctx android.LoadHookContext) (ret []string) { return ret } + +func createAvbpubkeyModule(ctx android.LoadHookContext) bool { + avbKeyPath := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BoardAvbKeyPath + if avbKeyPath == "" { + return false + } + ctx.CreateModuleInDirectory( + etc.AvbpubkeyModuleFactory, + ".", + &struct { + Name *string + Product_specific *bool + Private_key *string + No_full_install *bool + Visibility []string + }{ + Name: proptools.StringPtr("system_other_avbpubkey"), + Product_specific: proptools.BoolPtr(true), + Private_key: proptools.StringPtr(avbKeyPath), + No_full_install: proptools.BoolPtr(true), + Visibility: []string{"//visibility:public"}, + }, + ) + return true +} diff --git a/fsgen/vbmeta_partitions.go b/fsgen/vbmeta_partitions.go index 280e4052d..b7fff687d 100644 --- a/fsgen/vbmeta_partitions.go +++ b/fsgen/vbmeta_partitions.go @@ -51,9 +51,10 @@ func createVbmetaPartitions(ctx android.LoadHookContext, generatedPartitionTypes var result []vbmetaModuleInfo - var chainedPartitions []filesystem.ChainedPartitionProperties + var chainedPartitions []string var partitionTypesHandledByChainedPartitions []string - for chainedName, props := range partitionVars.ChainedVbmetaPartitions { + for _, chainedName := range android.SortedKeys(partitionVars.ChainedVbmetaPartitions) { + props := partitionVars.ChainedVbmetaPartitions[chainedName] chainedName = "vbmeta_" + chainedName if len(props.Partitions) == 0 { continue @@ -117,11 +118,7 @@ func createVbmetaPartitions(ctx android.LoadHookContext, generatedPartitionTypes }, ).HideFromMake() - chainedPartitions = append(chainedPartitions, filesystem.ChainedPartitionProperties{ - Name: &chainedName, - Rollback_index_location: &ril, - Private_key: &props.Key, - }) + chainedPartitions = append(chainedPartitions, name) result = append(result, vbmetaModuleInfo{ moduleName: name, @@ -156,6 +153,10 @@ func createVbmetaPartitions(ctx android.LoadHookContext, generatedPartitionTypes // Already handled by a chained vbmeta partition continue } + if partitionType == "ramdisk" { + // ramdisk is never signed with avb information + continue + } partitionModules = append(partitionModules, generatedModuleNameForPartition(ctx.Config(), partitionType)) } diff --git a/genrule/genrule.go b/genrule/genrule.go index 1282bfbae..9d2dbc71a 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -115,7 +115,7 @@ func (t hostToolDependencyTag) AllowDisabledModuleDependency(target android.Modu func (t hostToolDependencyTag) AllowDisabledModuleDependencyProxy( ctx android.OtherModuleProviderContext, target android.ModuleProxy) bool { return android.OtherModuleProviderOrDefault( - ctx, target, android.CommonPropertiesProviderKey).ReplacedByPrebuilt + ctx, target, android.CommonModuleInfoKey).ReplacedByPrebuilt } var _ android.AllowDisabledModuleDependency = (*hostToolDependencyTag)(nil) @@ -353,7 +353,7 @@ func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) { if h, ok := android.OtherModuleProvider(ctx, module, android.HostToolProviderKey); ok { // A HostToolProvider provides the path to a tool, which will be copied // into the sandbox. - if !android.OtherModuleProviderOrDefault(ctx, module, android.CommonPropertiesProviderKey).Enabled { + if !android.OtherModuleProviderOrDefault(ctx, module, android.CommonModuleInfoKey).Enabled { if ctx.Config().AllowMissingDependencies() { ctx.AddMissingDependencies([]string{tool}) } else { diff --git a/java/app.go b/java/app.go index 94c9e5b82..8bb73cb38 100644 --- a/java/app.go +++ b/java/app.go @@ -848,7 +848,7 @@ func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) android.P packageName := packageNameProp.Get() fileName := "privapp_allowlist_" + packageName + ".xml" - outPath := android.PathForModuleOut(ctx, fileName).OutputPath + outPath := android.PathForModuleOut(ctx, fileName) ctx.Build(pctx, android.BuildParams{ Rule: modifyAllowlist, Input: android.PathForModuleSrc(ctx, *a.appProperties.Privapp_allowlist), @@ -857,7 +857,7 @@ func (a *AndroidApp) createPrivappAllowlist(ctx android.ModuleContext) android.P "packageName": packageName, }, }) - return &outPath + return outPath } func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { @@ -1465,8 +1465,9 @@ func (a *AndroidTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { a.data = append(a.data, android.PathsForModuleSrc(ctx, a.testProperties.Device_common_data)...) a.data = append(a.data, android.PathsForModuleSrc(ctx, a.testProperties.Device_first_data)...) a.data = append(a.data, android.PathsForModuleSrc(ctx, a.testProperties.Device_first_prefer32_data)...) + android.SetProvider(ctx, tradefed.BaseTestProviderKey, tradefed.BaseTestProviderData{ - InstalledFiles: a.data, + TestcaseRelDataFiles: testcaseRel(a.data), OutputFile: a.OutputFile(), TestConfig: a.testConfig, HostRequiredModuleNames: a.HostRequiredModuleNames(), @@ -1474,6 +1475,8 @@ func (a *AndroidTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { IsHost: false, LocalCertificate: a.certificate.AndroidMkString(), IsUnitTest: Bool(a.testProperties.Test_options.Unit_test), + MkInclude: "$(BUILD_SYSTEM)/soong_app_prebuilt.mk", + MkAppClass: "APPS", }) android.SetProvider(ctx, android.TestOnlyProviderKey, android.TestModuleInformation{ TestOnly: true, @@ -1482,6 +1485,14 @@ func (a *AndroidTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { } +func testcaseRel(paths android.Paths) []string { + relPaths := []string{} + for _, p := range paths { + relPaths = append(relPaths, p.Rel()) + } + return relPaths +} + func (a *AndroidTest) FixTestConfig(ctx android.ModuleContext, testConfig android.Path) android.Path { if testConfig == nil { return nil diff --git a/java/app_import.go b/java/app_import.go index 6b88f1c43..f044c6848 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -88,7 +88,7 @@ type AndroidAppImport struct { hideApexVariantFromMake bool - provenanceMetaDataFile android.OutputPath + provenanceMetaDataFile android.Path } type AndroidAppImportProperties struct { @@ -259,7 +259,7 @@ func (a *AndroidAppImport) DepsMutator(ctx android.BottomUpMutatorContext) { } func (a *AndroidAppImport) uncompressEmbeddedJniLibs( - ctx android.ModuleContext, inputPath android.Path, outputPath android.OutputPath) { + ctx android.ModuleContext, inputPath android.Path, outputPath android.WritablePath) { // Test apps don't need their JNI libraries stored uncompressed. As a matter of fact, messing // with them may invalidate pre-existing signature data. if ctx.InstallInTestcases() && (Bool(a.properties.Presigned) || Bool(a.properties.Preprocessed)) { @@ -345,7 +345,7 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext // Uncompress JNI libraries in the apk jnisUncompressed := android.PathForModuleOut(ctx, "jnis-uncompressed", ctx.ModuleName()+".apk") - a.uncompressEmbeddedJniLibs(ctx, srcApk, jnisUncompressed.OutputPath) + a.uncompressEmbeddedJniLibs(ctx, srcApk, jnisUncompressed) var pathFragments []string relInstallPath := String(a.properties.Relative_install_path) @@ -493,7 +493,7 @@ func (a *AndroidAppImport) Certificate() Certificate { return a.certificate } -func (a *AndroidAppImport) ProvenanceMetaDataFile() android.OutputPath { +func (a *AndroidAppImport) ProvenanceMetaDataFile() android.Path { return a.provenanceMetaDataFile } diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index 7c0f54431..c778f0404 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -111,7 +111,7 @@ type BootclasspathFragmentCoverageAffectedProperties struct { // property. // // The order of this list matters as it is the order that is used in the bootclasspath. - Contents []string + Contents proptools.Configurable[[]string] `android:"arch_variant"` // The properties for specifying the API stubs provided by this fragment. BootclasspathAPIProperties @@ -295,8 +295,8 @@ func testBootclasspathFragmentFactory() android.Module { return m } -func (m *BootclasspathFragmentModule) bootclasspathFragmentPropertyCheck(ctx android.EarlyModuleContext) { - contents := m.properties.Contents +func (m *BootclasspathFragmentModule) bootclasspathFragmentPropertyCheck(ctx android.ModuleContext) { + contents := m.properties.Contents.GetOrDefault(ctx, nil) if len(contents) == 0 { ctx.PropertyErrorf("contents", "required property is missing") return @@ -434,7 +434,7 @@ func (b *BootclasspathFragmentModule) ComponentDepsMutator(ctx android.BottomUpM module := ctx.Module() _, isSourceModule := module.(*BootclasspathFragmentModule) - for _, name := range b.properties.Contents { + for _, name := range b.properties.Contents.GetOrDefault(ctx, nil) { // A bootclasspath_fragment must depend only on other source modules, while the // prebuilt_bootclasspath_fragment must only depend on other prebuilt modules. // @@ -588,7 +588,7 @@ func (b *BootclasspathFragmentModule) configuredJars(ctx android.ModuleContext) return global.ArtApexJars } - possibleUpdatableModules := gatherPossibleApexModuleNamesAndStems(ctx, b.properties.Contents, bootclasspathFragmentContentDepTag) + possibleUpdatableModules := gatherPossibleApexModuleNamesAndStems(ctx, b.properties.Contents.GetOrDefault(ctx, nil), bootclasspathFragmentContentDepTag) jars, unknown := global.ApexBootJars.Filter(possibleUpdatableModules) // TODO(satayev): for apex_test we want to include all contents unconditionally to classpaths @@ -600,7 +600,7 @@ func (b *BootclasspathFragmentModule) configuredJars(ctx android.ModuleContext) } else if android.InList("test_framework-apexd", possibleUpdatableModules) { jars = jars.Append("com.android.apex.test_package", "test_framework-apexd") } else if global.ApexBootJars.Len() != 0 { - unknown = android.RemoveListFromList(unknown, b.properties.Coverage.Contents) + unknown = android.RemoveListFromList(unknown, b.properties.Coverage.Contents.GetOrDefault(ctx, nil)) _, unknown = android.RemoveFromList("core-icu4j", unknown) // This module only exists in car products. // So ignore it even if it is not in PRODUCT_APEX_BOOT_JARS. @@ -847,7 +847,7 @@ func (b *BootclasspathFragmentModule) getProfilePath() android.Path { // Collect information for opening IDE project files in java/jdeps.go. func (b *BootclasspathFragmentModule) IDEInfo(ctx android.BaseModuleContext, dpInfo *android.IdeInfo) { - dpInfo.Deps = append(dpInfo.Deps, b.properties.Contents...) + dpInfo.Deps = append(dpInfo.Deps, b.properties.Contents.GetOrDefault(ctx, nil)...) } type bootclasspathFragmentMemberType struct { @@ -923,7 +923,7 @@ func (b *bootclasspathFragmentSdkMemberProperties) PopulateFromVariant(ctx andro module := variant.(*BootclasspathFragmentModule) b.Image_name = module.properties.Image_name - b.Contents = module.properties.Contents + b.Contents = module.properties.Contents.GetOrDefault(ctx.SdkModuleContext(), nil) // Get the hidden API information from the module. mctx := ctx.SdkModuleContext() diff --git a/java/bootclasspath_fragment_test.go b/java/bootclasspath_fragment_test.go index 60f1a50e1..3aa1258a3 100644 --- a/java/bootclasspath_fragment_test.go +++ b/java/bootclasspath_fragment_test.go @@ -191,7 +191,8 @@ func TestBootclasspathFragment_Coverage(t *testing.T) { checkContents := func(t *testing.T, result *android.TestResult, expected ...string) { module := result.Module("myfragment", "android_common").(*BootclasspathFragmentModule) - android.AssertArrayString(t, "contents property", expected, module.properties.Contents) + eval := module.ConfigurableEvaluator(android.PanickingConfigAndErrorContext(result.TestContext)) + android.AssertArrayString(t, "contents property", expected, module.properties.Contents.GetOrDefault(eval, nil)) } preparer := android.GroupFixturePreparers( diff --git a/java/classpath_fragment.go b/java/classpath_fragment.go index 18a5dae6c..fdccd3a84 100644 --- a/java/classpath_fragment.go +++ b/java/classpath_fragment.go @@ -18,9 +18,10 @@ package java import ( "fmt" + "strings" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" - "strings" "android/soong/android" ) @@ -103,7 +104,7 @@ type classpathJar struct { func gatherPossibleApexModuleNamesAndStems(ctx android.ModuleContext, contents []string, tag blueprint.DependencyTag) []string { set := map[string]struct{}{} for _, name := range contents { - dep, _ := ctx.GetDirectDepWithTag(name, tag).(android.Module) + dep := ctx.GetDirectDepWithTag(name, tag) set[ModuleStemForDeapexing(dep)] = struct{}{} if m, ok := dep.(ModuleWithStem); ok { set[m.Stem()] = struct{}{} diff --git a/java/dex.go b/java/dex.go index 516a91709..983377ead 100644 --- a/java/dex.go +++ b/java/dex.go @@ -430,17 +430,18 @@ type compileDexParams struct { // Adds --art-profile to r8/d8 command. // r8/d8 will output a generated profile file to match the optimized dex code. func (d *dexer) addArtProfile(ctx android.ModuleContext, dexParams *compileDexParams) (flags []string, deps android.Paths, artProfileOutputPath *android.OutputPath) { - if dexParams.artProfileInput != nil { - artProfileInputPath := android.PathForModuleSrc(ctx, *dexParams.artProfileInput) - artProfileOutputPathValue := android.PathForModuleOut(ctx, "profile.prof.txt").OutputPath - artProfileOutputPath = &artProfileOutputPathValue - flags = []string{ - "--art-profile", - artProfileInputPath.String(), - artProfileOutputPath.String(), - } - deps = append(deps, artProfileInputPath) + if dexParams.artProfileInput == nil { + return nil, nil, nil + } + artProfileInputPath := android.PathForModuleSrc(ctx, *dexParams.artProfileInput) + artProfileOutputPathValue := android.PathForModuleOut(ctx, "profile.prof.txt").OutputPath + artProfileOutputPath = &artProfileOutputPathValue + flags = []string{ + "--art-profile", + artProfileInputPath.String(), + artProfileOutputPath.String(), } + deps = append(deps, artProfileInputPath) return flags, deps, artProfileOutputPath } diff --git a/java/droiddoc.go b/java/droiddoc.go index 2980d91de..82713920d 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -427,9 +427,9 @@ func (j *Javadoc) collectDeps(ctx android.ModuleContext) deps { // Find the corresponding aconfig_declarations module name for such case. for _, src := range j.properties.Srcs { if moduleName, tag := android.SrcIsModuleWithTag(src); moduleName != "" { - otherModule := android.GetModuleFromPathDep(ctx, moduleName, tag) + otherModule := android.GetModuleProxyFromPathDep(ctx, moduleName, tag) if otherModule != nil { - if dep, ok := android.OtherModuleProvider(ctx, otherModule, android.CodegenInfoProvider); ok { + if dep, ok := android.OtherModuleProvider(ctx, *otherModule, android.CodegenInfoProvider); ok { deps.aconfigProtoFiles = append(deps.aconfigProtoFiles, dep.IntermediateCacheOutputPaths...) } } diff --git a/java/droidstubs.go b/java/droidstubs.go index 6bcdf85a2..cf3e21925 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -997,12 +997,13 @@ func (d *Droidstubs) everythingOptionalCmd(ctx android.ModuleContext, cmd *andro msg := `$'` + // Enclose with $' ... ' `************************************************************\n` + `Your API changes are triggering API Lint warnings or errors.\n` + - `To make these errors go away, fix the code according to the\n` + - `error and/or warning messages above.\n` + `\n` + - `If it is not possible to do so, there are workarounds:\n` + + `To make the failures go away:\n` + `\n` + - `1. You can suppress the errors with @SuppressLint("<id>")\n` + + `1. REQUIRED: Read the messages carefully and address them by` + + ` fixing the API if appropriate.\n` + + `2. If the failure is a false positive, you can suppress it with:\n` + + ` @SuppressLint("<id>")\n` + ` where the <id> is given in brackets in the error message above.\n` if baselineFile.Valid() { @@ -1010,8 +1011,8 @@ func (d *Droidstubs) everythingOptionalCmd(ctx android.ModuleContext, cmd *andro cmd.FlagWithOutput("--update-baseline:api-lint ", updatedBaselineOutput) msg += fmt.Sprintf(``+ - `2. You can update the baseline by executing the following\n`+ - ` command:\n`+ + `3. FOR LSC ONLY: You can update the baseline by executing\n` + + ` the following command:\n`+ ` (cd $ANDROID_BUILD_TOP && cp \\\n`+ ` "%s" \\\n`+ ` "%s")\n`+ @@ -1019,7 +1020,7 @@ func (d *Droidstubs) everythingOptionalCmd(ctx android.ModuleContext, cmd *andro ` repository, you will need approval.\n`, updatedBaselineOutput, baselineFile.Path()) } else { msg += fmt.Sprintf(``+ - `2. You can add a baseline file of existing lint failures\n`+ + `3. FOR LSC ONLY: You can add a baseline file of existing lint failures\n`+ ` to the build rule of %s.\n`, d.Name()) } // Note the message ends with a ' (single quote), to close the $' ... ' . diff --git a/java/java.go b/java/java.go index 1d572faef..64ef782f5 100644 --- a/java/java.go +++ b/java/java.go @@ -226,9 +226,9 @@ var ( // Rule for generating device binary default wrapper deviceBinaryWrapper = pctx.StaticRule("deviceBinaryWrapper", blueprint.RuleParams{ - Command: `echo -e '#!/system/bin/sh\n` + + Command: `printf '#!/system/bin/sh\n` + `export CLASSPATH=/system/framework/$jar_name\n` + - `exec app_process /$partition/bin $main_class "$$@"'> ${out}`, + `exec app_process /$partition/bin $main_class "$$@"\n'> ${out}`, Description: "Generating device binary wrapper ${jar_name}", }, "jar_name", "partition", "main_class") ) @@ -1557,14 +1557,16 @@ func (j *TestHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.Test.generateAndroidBuildActionsWithConfig(ctx, configs) android.SetProvider(ctx, tradefed.BaseTestProviderKey, tradefed.BaseTestProviderData{ - InstalledFiles: j.data, - OutputFile: j.outputFile, - TestConfig: j.testConfig, - RequiredModuleNames: j.RequiredModuleNames(ctx), - TestSuites: j.testProperties.Test_suites, - IsHost: true, - LocalSdkVersion: j.sdkVersion.String(), - IsUnitTest: Bool(j.testProperties.Test_options.Unit_test), + TestcaseRelDataFiles: testcaseRel(j.data), + OutputFile: j.outputFile, + TestConfig: j.testConfig, + RequiredModuleNames: j.RequiredModuleNames(ctx), + TestSuites: j.testProperties.Test_suites, + IsHost: true, + LocalSdkVersion: j.sdkVersion.String(), + IsUnitTest: Bool(j.testProperties.Test_options.Unit_test), + MkInclude: "$(BUILD_SYSTEM)/soong_java_prebuilt.mk", + MkAppClass: "JAVA_LIBRARIES", }) } diff --git a/java/platform_bootclasspath.go b/java/platform_bootclasspath.go index 8b0ca97a1..d09a02e50 100644 --- a/java/platform_bootclasspath.go +++ b/java/platform_bootclasspath.go @@ -184,7 +184,7 @@ func (b *platformBootclasspathModule) GenerateAndroidBuildActions(ctx android.Mo } jarArgs := resourcePathsToJarArgs(transitiveSrcFiles) jarArgs = append(jarArgs, "-srcjar") // Move srcfiles to the right package - srcjar := android.PathForModuleOut(ctx, ctx.ModuleName()+"-transitive.srcjar").OutputPath + srcjar := android.PathForModuleOut(ctx, ctx.ModuleName()+"-transitive.srcjar") TransformResourcesToJar(ctx, srcjar, jarArgs, transitiveSrcFiles) // Gather all the fragments dependencies. diff --git a/java/ravenwood.go b/java/ravenwood.go index 4c9fdc212..4c43a9ffb 100644 --- a/java/ravenwood.go +++ b/java/ravenwood.go @@ -14,6 +14,8 @@ package java import ( + "strconv" + "android/soong/android" "android/soong/tradefed" @@ -36,6 +38,14 @@ var ravenwoodRuntimeTag = dependencyTag{name: "ravenwoodruntime"} var ravenwoodTestResourceApkTag = dependencyTag{name: "ravenwoodtestresapk"} var ravenwoodTestInstResourceApkTag = dependencyTag{name: "ravenwoodtest-inst-res-apk"} +var genManifestProperties = pctx.AndroidStaticRule("genManifestProperties", + blueprint.RuleParams{ + Command: "echo targetSdkVersionInt=$targetSdkVersionInt > $out && " + + "echo targetSdkVersionRaw=$targetSdkVersionRaw >> $out && " + + "echo packageName=$packageName >> $out && " + + "echo instPackageName=$instPackageName >> $out", + }, "targetSdkVersionInt", "targetSdkVersionRaw", "packageName", "instPackageName") + const ravenwoodUtilsName = "ravenwood-utils" const ravenwoodRuntimeName = "ravenwood-runtime" @@ -68,6 +78,17 @@ type ravenwoodTestProperties struct { // the ravenwood test can access it. This APK will be loaded as resources of the test // instrumentation app itself. Inst_resource_apk *string + + // Specify the package name of the test target apk. + // This will be set to the target Context's package name. + // (i.e. Instrumentation.getTargetContext().getPackageName()) + // If this is omitted, Package_name will be used. + Package_name *string + + // Specify the package name of this test module. + // This will be set to the test Context's package name. + //(i.e. Instrumentation.getContext().getPackageName()) + Inst_package_name *string } type ravenwoodTest struct { @@ -216,6 +237,27 @@ func (r *ravenwoodTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { copyResApk(ravenwoodTestResourceApkTag, "ravenwood-res.apk") copyResApk(ravenwoodTestInstResourceApkTag, "ravenwood-inst-res.apk") + // Generate manifest properties + propertiesOutputPath := android.PathForModuleGen(ctx, "ravenwood.properties") + + targetSdkVersion := proptools.StringDefault(r.deviceProperties.Target_sdk_version, "") + targetSdkVersionInt := r.TargetSdkVersion(ctx).FinalOrFutureInt() // FinalOrFutureInt may be 10000. + packageName := proptools.StringDefault(r.ravenwoodTestProperties.Package_name, "") + instPackageName := proptools.StringDefault(r.ravenwoodTestProperties.Inst_package_name, "") + ctx.Build(pctx, android.BuildParams{ + Rule: genManifestProperties, + Description: "genManifestProperties", + Output: propertiesOutputPath, + Args: map[string]string{ + "targetSdkVersionInt": strconv.Itoa(targetSdkVersionInt), + "targetSdkVersionRaw": targetSdkVersion, + "packageName": packageName, + "instPackageName": instPackageName, + }, + }) + installProps := ctx.InstallFile(installPath, "ravenwood.properties", propertiesOutputPath) + installDeps = append(installDeps, installProps) + // Install our JAR with all dependencies ctx.InstallFile(installPath, ctx.ModuleName()+".jar", r.outputFile, installDeps...) } diff --git a/java/ravenwood_test.go b/java/ravenwood_test.go index 5d62ede74..6394a9ac9 100644 --- a/java/ravenwood_test.go +++ b/java/ravenwood_test.go @@ -177,6 +177,12 @@ func TestRavenwoodTest(t *testing.T) { resource_apk: "app2", inst_resource_apk: "app3", sdk_version: "test_current", + target_sdk_version: "34", + package_name: "a.b.c", + inst_package_name: "x.y.z", + } + android_ravenwood_test { + name: "ravenwood-test-empty", } `) @@ -199,12 +205,16 @@ func TestRavenwoodTest(t *testing.T) { // Verify that we've emitted test artifacts in expected location outputJar := module.Output(installPathPrefix + "/ravenwood-test/ravenwood-test.jar") module.Output(installPathPrefix + "/ravenwood-test/ravenwood-test.config") + module.Output(installPathPrefix + "/ravenwood-test/ravenwood.properties") module.Output(installPathPrefix + "/ravenwood-test/lib64/jni-lib1.so") module.Output(installPathPrefix + "/ravenwood-test/lib64/libblue.so") module.Output(installPathPrefix + "/ravenwood-test/lib64/libpink.so") module.Output(installPathPrefix + "/ravenwood-test/ravenwood-res-apks/ravenwood-res.apk") module.Output(installPathPrefix + "/ravenwood-test/ravenwood-res-apks/ravenwood-inst-res.apk") + module = ctx.ModuleForTests("ravenwood-test-empty", "android_common") + module.Output(installPathPrefix + "/ravenwood-test-empty/ravenwood.properties") + // ravenwood-runtime*.so are included in the runtime, so it shouldn't be emitted. for _, o := range module.AllOutputs() { android.AssertStringDoesNotContain(t, "runtime libs shouldn't be included", o, "/ravenwood-test/lib64/ravenwood-runtime") diff --git a/java/systemserver_classpath_fragment.go b/java/systemserver_classpath_fragment.go index aad106007..608a61628 100644 --- a/java/systemserver_classpath_fragment.go +++ b/java/systemserver_classpath_fragment.go @@ -19,6 +19,7 @@ import ( "android/soong/dexpreopt" "github.com/google/blueprint" + "github.com/google/blueprint/proptools" ) func init() { @@ -98,12 +99,12 @@ type systemServerClasspathFragmentProperties struct { // List of system_server classpath jars, could be either java_library, or java_sdk_library. // // The order of this list matters as it is the order that is used in the SYSTEMSERVERCLASSPATH. - Contents []string + Contents proptools.Configurable[[]string] `android:"arch_variant"` // List of jars that system_server loads dynamically using separate classloaders. // // The order does not matter. - Standalone_contents []string + Standalone_contents proptools.Configurable[[]string] `android:"arch_variant"` } func systemServerClasspathFactory() android.Module { @@ -116,7 +117,7 @@ func systemServerClasspathFactory() android.Module { } func (s *SystemServerClasspathModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { - if len(s.properties.Contents) == 0 && len(s.properties.Standalone_contents) == 0 { + if len(s.properties.Contents.GetOrDefault(ctx, nil)) == 0 && len(s.properties.Standalone_contents.GetOrDefault(ctx, nil)) == 0 { ctx.PropertyErrorf("contents", "Either contents or standalone_contents needs to be non-empty") } @@ -152,7 +153,7 @@ func (s *SystemServerClasspathModule) setPartitionInfoOfLibraries(ctx android.Mo func (s *SystemServerClasspathModule) configuredJars(ctx android.ModuleContext) android.ConfiguredJarList { global := dexpreopt.GetGlobalConfig(ctx) - possibleUpdatableModules := gatherPossibleApexModuleNamesAndStems(ctx, s.properties.Contents, systemServerClasspathFragmentContentDepTag) + possibleUpdatableModules := gatherPossibleApexModuleNamesAndStems(ctx, s.properties.Contents.GetOrDefault(ctx, nil), systemServerClasspathFragmentContentDepTag) jars, unknown := global.ApexSystemServerJars.Filter(possibleUpdatableModules) // TODO(satayev): remove geotz ssc_fragment, since geotz is not part of SSCP anymore. _, unknown = android.RemoveFromList("geotz", unknown) @@ -184,7 +185,7 @@ func (s *SystemServerClasspathModule) configuredJars(ctx android.ModuleContext) func (s *SystemServerClasspathModule) standaloneConfiguredJars(ctx android.ModuleContext) android.ConfiguredJarList { global := dexpreopt.GetGlobalConfig(ctx) - possibleUpdatableModules := gatherPossibleApexModuleNamesAndStems(ctx, s.properties.Standalone_contents, systemServerClasspathFragmentContentDepTag) + possibleUpdatableModules := gatherPossibleApexModuleNamesAndStems(ctx, s.properties.Standalone_contents.GetOrDefault(ctx, nil), systemServerClasspathFragmentContentDepTag) jars, _ := global.ApexStandaloneSystemServerJars.Filter(possibleUpdatableModules) // TODO(jiakaiz): add a check to ensure that the contents are declared in make. @@ -245,8 +246,8 @@ func (s *SystemServerClasspathModule) ComponentDepsMutator(ctx android.BottomUpM module := ctx.Module() _, isSourceModule := module.(*SystemServerClasspathModule) var deps []string - deps = append(deps, s.properties.Contents...) - deps = append(deps, s.properties.Standalone_contents...) + deps = append(deps, s.properties.Contents.GetOrDefault(ctx, nil)...) + deps = append(deps, s.properties.Standalone_contents.GetOrDefault(ctx, nil)...) for _, name := range deps { // A systemserverclasspath_fragment must depend only on other source modules, while the @@ -260,8 +261,8 @@ func (s *SystemServerClasspathModule) ComponentDepsMutator(ctx android.BottomUpM // Collect information for opening IDE project files in java/jdeps.go. func (s *SystemServerClasspathModule) IDEInfo(ctx android.BaseModuleContext, dpInfo *android.IdeInfo) { - dpInfo.Deps = append(dpInfo.Deps, s.properties.Contents...) - dpInfo.Deps = append(dpInfo.Deps, s.properties.Standalone_contents...) + dpInfo.Deps = append(dpInfo.Deps, s.properties.Contents.GetOrDefault(ctx, nil)...) + dpInfo.Deps = append(dpInfo.Deps, s.properties.Standalone_contents.GetOrDefault(ctx, nil)...) } type systemServerClasspathFragmentMemberType struct { @@ -302,8 +303,8 @@ type systemServerClasspathFragmentSdkMemberProperties struct { func (s *systemServerClasspathFragmentSdkMemberProperties) PopulateFromVariant(ctx android.SdkMemberContext, variant android.Module) { module := variant.(*SystemServerClasspathModule) - s.Contents = module.properties.Contents - s.Standalone_contents = module.properties.Standalone_contents + s.Contents = module.properties.Contents.GetOrDefault(ctx.SdkModuleContext(), nil) + s.Standalone_contents = module.properties.Standalone_contents.GetOrDefault(ctx.SdkModuleContext(), nil) } func (s *systemServerClasspathFragmentSdkMemberProperties) AddToPropertySet(ctx android.SdkMemberContext, propertySet android.BpPropertySet) { diff --git a/kernel/prebuilt_kernel_modules.go b/kernel/prebuilt_kernel_modules.go index 4c0a9112b..13d648235 100644 --- a/kernel/prebuilt_kernel_modules.go +++ b/kernel/prebuilt_kernel_modules.go @@ -47,6 +47,17 @@ type prebuiltKernelModulesProperties struct { // List or filegroup of prebuilt kernel module files. Should have .ko suffix. Srcs []string `android:"path,arch_variant"` + // List of system_dlkm kernel modules that the local kernel modules depend on. + // The deps will be assembled into intermediates directory for running depmod + // but will not be added to the current module's installed files. + System_deps []string `android:"path,arch_variant"` + + // If false, then srcs will not be included in modules.load. + // This feature is used by system_dlkm + Load_by_default *bool + + Blocklist_file *string `android:"path"` + // Kernel version that these modules are for. Kernel modules are installed to // /lib/modules/<kernel_version> directory in the corresponding partition. Default is "". Kernel_version *string @@ -81,9 +92,11 @@ func (pkm *prebuiltKernelModules) GenerateAndroidBuildActions(ctx android.Module if !pkm.installable() { pkm.SkipInstall() } + modules := android.PathsForModuleSrc(ctx, pkm.properties.Srcs) + systemModules := android.PathsForModuleSrc(ctx, pkm.properties.System_deps) - depmodOut := runDepmod(ctx, modules) + depmodOut := pkm.runDepmod(ctx, modules, systemModules) strippedModules := stripDebugSymbols(ctx, modules) installDir := android.PathForModuleInstall(ctx, "lib", "modules") @@ -98,6 +111,23 @@ func (pkm *prebuiltKernelModules) GenerateAndroidBuildActions(ctx android.Module ctx.InstallFile(installDir, "modules.dep", depmodOut.modulesDep) ctx.InstallFile(installDir, "modules.softdep", depmodOut.modulesSoftdep) ctx.InstallFile(installDir, "modules.alias", depmodOut.modulesAlias) + pkm.installBlocklistFile(ctx, installDir) + + ctx.SetOutputFiles(modules, ".modules") +} + +func (pkm *prebuiltKernelModules) installBlocklistFile(ctx android.ModuleContext, installDir android.InstallPath) { + if pkm.properties.Blocklist_file == nil { + return + } + blocklistOut := android.PathForModuleOut(ctx, "modules.blocklist") + + ctx.Build(pctx, android.BuildParams{ + Rule: processBlocklistFile, + Input: android.PathForModuleSrc(ctx, proptools.String(pkm.properties.Blocklist_file)), + Output: blocklistOut, + }) + ctx.InstallFile(installDir, "modules.blocklist", blocklistOut) } var ( @@ -137,35 +167,93 @@ type depmodOutputs struct { modulesAlias android.OutputPath } -func runDepmod(ctx android.ModuleContext, modules android.Paths) depmodOutputs { +var ( + // system/lib/modules/foo.ko: system/lib/modules/bar.ko + // will be converted to + // /system/lib/modules/foo.ko: /system/lib/modules/bar.ko + addLeadingSlashToPaths = pctx.AndroidStaticRule("add_leading_slash", + blueprint.RuleParams{ + Command: `sed -e 's|\([^: ]*lib/modules/[^: ]*\)|/\1|g' $in > $out`, + }, + ) + // Remove empty lines. Raise an exception if line is _not_ formatted as `blocklist $name.ko` + processBlocklistFile = pctx.AndroidStaticRule("process_blocklist_file", + blueprint.RuleParams{ + Command: `rm -rf $out && awk <$in > $out` + + ` '/^#/ { print; next }` + + ` NF == 0 { next }` + + ` NF != 2 || $$1 != "blocklist"` + + ` { print "Invalid blocklist line " FNR ": " $$0 >"/dev/stderr";` + + ` exit_status = 1; next }` + + ` { $$1 = $$1; print }` + + ` END { exit exit_status }'`, + }, + ) +) + +// This is the path in soong intermediates where the .ko files will be copied. +// The layout should match the layout on device so that depmod can create meaningful modules.* files. +func modulesDirForAndroidDlkm(ctx android.ModuleContext, modulesDir android.OutputPath, system bool) android.OutputPath { + if ctx.InstallInSystemDlkm() || system { + // The first component can be either system or system_dlkm + // system works because /system/lib/modules is a symlink to /system_dlkm/lib/modules. + // system was chosen to match the contents of the kati built modules.dep + return modulesDir.Join(ctx, "system", "lib", "modules") + } else if ctx.InstallInVendorDlkm() { + return modulesDir.Join(ctx, "vendor", "lib", "modules") + } else if ctx.InstallInOdmDlkm() { + return modulesDir.Join(ctx, "odm", "lib", "modules") + } else { + // not an android dlkm module. + return modulesDir + } +} + +func (pkm *prebuiltKernelModules) runDepmod(ctx android.ModuleContext, modules android.Paths, systemModules android.Paths) depmodOutputs { baseDir := android.PathForModuleOut(ctx, "depmod").OutputPath fakeVer := "0.0" // depmod demands this anyway modulesDir := baseDir.Join(ctx, "lib", "modules", fakeVer) + modulesCpDir := modulesDirForAndroidDlkm(ctx, modulesDir, false) builder := android.NewRuleBuilder(pctx, ctx) // Copy the module files to a temporary dir - builder.Command().Text("rm").Flag("-rf").Text(modulesDir.String()) - builder.Command().Text("mkdir").Flag("-p").Text(modulesDir.String()) + builder.Command().Text("rm").Flag("-rf").Text(modulesCpDir.String()) + builder.Command().Text("mkdir").Flag("-p").Text(modulesCpDir.String()) for _, m := range modules { - builder.Command().Text("cp").Input(m).Text(modulesDir.String()) + builder.Command().Text("cp").Input(m).Text(modulesCpDir.String()) + } + + modulesDirForSystemDlkm := modulesDirForAndroidDlkm(ctx, modulesDir, true) + if len(systemModules) > 0 { + builder.Command().Text("mkdir").Flag("-p").Text(modulesDirForSystemDlkm.String()) + } + for _, m := range systemModules { + builder.Command().Text("cp").Input(m).Text(modulesDirForSystemDlkm.String()) } // Enumerate modules to load modulesLoad := modulesDir.Join(ctx, "modules.load") - var basenames []string - for _, m := range modules { - basenames = append(basenames, filepath.Base(m.String())) + // If Load_by_default is set to false explicitly, create an empty modules.load + if pkm.properties.Load_by_default != nil && !*pkm.properties.Load_by_default { + builder.Command().Text("rm").Flag("-rf").Text(modulesLoad.String()) + builder.Command().Text("touch").Output(modulesLoad) + } else { + var basenames []string + for _, m := range modules { + basenames = append(basenames, filepath.Base(m.String())) + } + builder.Command(). + Text("echo").Flag("\"" + strings.Join(basenames, " ") + "\""). + Text("|").Text("tr").Flag("\" \"").Flag("\"\\n\""). + Text(">").Output(modulesLoad) } - builder.Command(). - Text("echo").Flag("\"" + strings.Join(basenames, " ") + "\""). - Text("|").Text("tr").Flag("\" \"").Flag("\"\\n\""). - Text(">").Output(modulesLoad) // Run depmod to build modules.dep/softdep/alias files modulesDep := modulesDir.Join(ctx, "modules.dep") modulesSoftdep := modulesDir.Join(ctx, "modules.softdep") modulesAlias := modulesDir.Join(ctx, "modules.alias") + builder.Command().Text("mkdir").Flag("-p").Text(modulesDir.String()) builder.Command(). BuiltTool("depmod"). FlagWithArg("-b ", baseDir.String()). @@ -176,5 +264,16 @@ func runDepmod(ctx android.ModuleContext, modules android.Paths) depmodOutputs { builder.Build("depmod", fmt.Sprintf("depmod %s", ctx.ModuleName())) - return depmodOutputs{modulesLoad, modulesDep, modulesSoftdep, modulesAlias} + finalModulesDep := modulesDep + // Add a leading slash to paths in modules.dep of android dlkm + if ctx.InstallInSystemDlkm() || ctx.InstallInVendorDlkm() || ctx.InstallInOdmDlkm() { + finalModulesDep := modulesDep.ReplaceExtension(ctx, "intermediates") + ctx.Build(pctx, android.BuildParams{ + Rule: addLeadingSlashToPaths, + Input: modulesDep, + Output: finalModulesDep, + }) + } + + return depmodOutputs{modulesLoad, finalModulesDep, modulesSoftdep, modulesAlias} } diff --git a/provenance/provenance_singleton.go b/provenance/provenance_singleton.go index 679632c10..c372db2b3 100644 --- a/provenance/provenance_singleton.go +++ b/provenance/provenance_singleton.go @@ -46,7 +46,7 @@ var ( ) type ProvenanceMetadata interface { - ProvenanceMetaDataFile() android.OutputPath + ProvenanceMetaDataFile() android.Path } func init() { @@ -74,7 +74,7 @@ func (p *provenanceInfoSingleton) GenerateBuildActions(context android.Singleton return false } if p, ok := module.(ProvenanceMetadata); ok { - return p.ProvenanceMetaDataFile().String() != "" + return p.ProvenanceMetaDataFile() != nil } return false } @@ -101,7 +101,7 @@ func (p *provenanceInfoSingleton) GenerateBuildActions(context android.Singleton context.Phony("droidcore", android.PathForPhony(context, "provenance_metadata")) } -func GenerateArtifactProvenanceMetaData(ctx android.ModuleContext, artifactPath android.Path, installedFile android.InstallPath) android.OutputPath { +func GenerateArtifactProvenanceMetaData(ctx android.ModuleContext, artifactPath android.Path, installedFile android.InstallPath) android.Path { onDevicePathOfInstalledFile := android.InstallPathToOnDevicePath(ctx, installedFile) artifactMetaDataFile := android.PathForIntermediates(ctx, "provenance_metadata", ctx.ModuleDir(), ctx.ModuleName(), "provenance_metadata.textproto") ctx.Build(pctx, android.BuildParams{ diff --git a/rust/rust.go b/rust/rust.go index ed38ad7a7..48f946e1c 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -1001,6 +1001,10 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { ctx.Phony("rust", ctx.RustModule().OutputFile().Path()) } + android.SetProvider(ctx, cc.LinkableInfoKey, cc.LinkableInfo{ + StaticExecutable: mod.StaticExecutable(), + }) + mod.setOutputFiles(ctx) buildComplianceMetadataInfo(ctx, mod, deps) diff --git a/scripts/gen_build_prop.py b/scripts/gen_build_prop.py index 0b7780e96..5f52d6f9e 100644 --- a/scripts/gen_build_prop.py +++ b/scripts/gen_build_prop.py @@ -608,7 +608,7 @@ def main(): build_product_prop(args) case "vendor": build_vendor_prop(args) - case "system_dlkm" | "vendor_dlkm" | "odm_dlkm": + case "system_dlkm" | "vendor_dlkm" | "odm_dlkm" | "bootimage": build_prop(args, gen_build_info=False, gen_common_build_props=True, variables=[]) case _: sys.exit(f"not supported partition {args.partition}") diff --git a/scripts/ninja_determinism_test.py b/scripts/ninja_determinism_test.py new file mode 100755 index 000000000..e207b9613 --- /dev/null +++ b/scripts/ninja_determinism_test.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 + +import asyncio +import argparse +import dataclasses +import hashlib +import os +import re +import socket +import subprocess +import sys +import zipfile + +from typing import List + +def get_top() -> str: + path = '.' + while not os.path.isfile(os.path.join(path, 'build/soong/soong_ui.bash')): + if os.path.abspath(path) == '/': + sys.exit('Could not find android source tree root.') + path = os.path.join(path, '..') + return os.path.abspath(path) + + +_PRODUCT_REGEX = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)(?:(?:-([a-zA-Z_][a-zA-Z0-9_]*))?-(user|userdebug|eng))?') + + +@dataclasses.dataclass(frozen=True) +class Product: + """Represents a TARGET_PRODUCT and TARGET_BUILD_VARIANT.""" + product: str + release: str + variant: str + + def __post_init__(self): + if not _PRODUCT_REGEX.match(str(self)): + raise ValueError(f'Invalid product name: {self}') + + def __str__(self): + return self.product + '-' + self.release + '-' + self.variant + + +async def run_make_nothing(product: Product, out_dir: str) -> bool: + """Runs a build and returns if it succeeded or not.""" + with open(os.path.join(out_dir, 'build.log'), 'wb') as f: + result = await asyncio.create_subprocess_exec( + 'prebuilts/build-tools/linux-x86/bin/nsjail', + '-q', + '--cwd', + os.getcwd(), + '-e', + '-B', + '/', + '-B', + f'{os.path.abspath(out_dir)}:{os.path.join(os.getcwd(), "out")}', + '--time_limit', + '0', + '--skip_setsid', + '--keep_caps', + '--disable_clone_newcgroup', + '--disable_clone_newnet', + '--rlimit_as', + 'soft', + '--rlimit_core', + 'soft', + '--rlimit_cpu', + 'soft', + '--rlimit_fsize', + 'soft', + '--rlimit_nofile', + 'soft', + '--proc_rw', + '--hostname', + socket.gethostname(), + '--', + 'build/soong/soong_ui.bash', + '--make-mode', + f'TARGET_PRODUCT={product.product}', + f'TARGET_RELEASE={product.release}', + f'TARGET_BUILD_VARIANT={product.variant}', + '--skip-ninja', + 'nothing', stdout=f, stderr=subprocess.STDOUT) + return await result.wait() == 0 + +SUBNINJA_OR_INCLUDE_REGEX = re.compile(rb'\n(?:include|subninja) ') + +def find_subninjas_and_includes(contents) -> List[str]: + results = [] + def get_path_from_directive(i): + j = contents.find(b'\n', i) + if j < 0: + path_bytes = contents[i:] + else: + path_bytes = contents[i:j] + path_bytes = path_bytes.removesuffix(b'\r') + path = path_bytes.decode() + if '$' in path: + sys.exit('includes/subninjas with variables are unsupported: '+path) + return path + + if contents.startswith(b"include "): + results.append(get_path_from_directive(len(b"include "))) + elif contents.startswith(b"subninja "): + results.append(get_path_from_directive(len(b"subninja "))) + + for match in SUBNINJA_OR_INCLUDE_REGEX.finditer(contents): + results.append(get_path_from_directive(match.end())) + + return results + + +def transitively_included_ninja_files(out_dir: str, ninja_file: str, seen): + with open(ninja_file, 'rb') as f: + contents = f.read() + + results = [ninja_file] + seen[ninja_file] = True + sub_files = find_subninjas_and_includes(contents) + for sub_file in sub_files: + sub_file = os.path.join(out_dir, sub_file.removeprefix('out/')) + if sub_file not in seen: + results.extend(transitively_included_ninja_files(out_dir, sub_file, seen)) + + return results + + +def hash_ninja_file(out_dir: str, ninja_file: str, hasher): + with open(ninja_file, 'rb') as f: + contents = f.read() + + sub_files = find_subninjas_and_includes(contents) + + hasher.update(contents) + + for sub_file in sub_files: + hash_ninja_file(out_dir, os.path.join(out_dir, sub_file.removeprefix('out/')), hasher) + + +def hash_files(files: List[str]) -> str: + hasher = hashlib.md5() + for file in files: + with open(file, 'rb') as f: + hasher.update(f.read()) + return hasher.hexdigest() + + +def dist_ninja_files(out_dir: str, zip_name: str, ninja_files: List[str]): + dist_dir = os.getenv('DIST_DIR', os.path.join(os.getenv('OUT_DIR', 'out'), 'dist')) + os.makedirs(dist_dir, exist_ok=True) + + with open(os.path.join(dist_dir, zip_name), 'wb') as f: + with zipfile.ZipFile(f, mode='w') as zf: + for ninja_file in ninja_files: + zf.write(ninja_file, arcname=os.path.basename(out_dir)+'/out/' + os.path.relpath(ninja_file, out_dir)) + + +async def main(): + parser = argparse.ArgumentParser() + args = parser.parse_args() + + os.chdir(get_top()) + subprocess.check_call(['touch', 'build/soong/Android.bp']) + + product = Product( + 'aosp_cf_x86_64_phone', + 'trunk_staging', + 'userdebug', + ) + os.environ['TARGET_PRODUCT'] = 'aosp_cf_x86_64_phone' + os.environ['TARGET_RELEASE'] = 'trunk_staging' + os.environ['TARGET_BUILD_VARIANT'] = 'userdebug' + + out_dir1 = os.path.join(os.getenv('OUT_DIR', 'out'), 'determinism_test_out1') + out_dir2 = os.path.join(os.getenv('OUT_DIR', 'out'), 'determinism_test_out2') + + os.makedirs(out_dir1, exist_ok=True) + os.makedirs(out_dir2, exist_ok=True) + + success1, success2 = await asyncio.gather( + run_make_nothing(product, out_dir1), + run_make_nothing(product, out_dir2)) + + if not success1: + with open(os.path.join(out_dir1, 'build.log'), 'r') as f: + print(f.read(), file=sys.stderr) + sys.exit('build failed') + if not success2: + with open(os.path.join(out_dir2, 'build.log'), 'r') as f: + print(f.read(), file=sys.stderr) + sys.exit('build failed') + + ninja_files1 = transitively_included_ninja_files(out_dir1, os.path.join(out_dir1, f'combined-{product.product}.ninja'), {}) + ninja_files2 = transitively_included_ninja_files(out_dir2, os.path.join(out_dir2, f'combined-{product.product}.ninja'), {}) + + dist_ninja_files(out_dir1, 'determinism_test_files_1.zip', ninja_files1) + dist_ninja_files(out_dir2, 'determinism_test_files_2.zip', ninja_files2) + + hash1 = hash_files(ninja_files1) + hash2 = hash_files(ninja_files2) + + if hash1 != hash2: + sys.exit("ninja files were not deterministic! See disted determinism_test_files_1/2.zip") + + print("Success, ninja files were deterministic") + + +if __name__ == "__main__": + asyncio.run(main()) + + diff --git a/sh/sh_binary.go b/sh/sh_binary.go index 853f3d368..7f5a4261f 100644 --- a/sh/sh_binary.go +++ b/sh/sh_binary.go @@ -15,6 +15,7 @@ package sh import ( + "fmt" "path/filepath" "strings" @@ -164,6 +165,9 @@ type TestProperties struct { // Test options. Test_options android.CommonTestOptions + + // a list of extra test configuration files that should be installed with the module. + Extra_test_configs []string `android:"path,arch_variant"` } type ShBinary struct { @@ -186,8 +190,9 @@ type ShTest struct { installDir android.InstallPath - data []android.DataPath - testConfig android.Path + data []android.DataPath + testConfig android.Path + extraTestConfigs android.Paths dataModules map[string]android.Path } @@ -199,7 +204,7 @@ func (s *ShBinary) HostToolPath() android.OptionalPath { func (s *ShBinary) DepsMutator(ctx android.BottomUpMutatorContext) { } -func (s *ShBinary) OutputFile() android.OutputPath { +func (s *ShBinary) OutputFile() android.Path { return s.outputFilePath } @@ -471,6 +476,7 @@ func (s *ShTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { HostTemplate: "${ShellTestConfigTemplate}", }) + s.extraTestConfigs = android.PathsForModuleSrc(ctx, s.testProperties.Extra_test_configs) s.dataModules = make(map[string]android.Path) ctx.VisitDirectDeps(func(dep android.Module) { depTag := ctx.OtherModuleDependencyTag(dep) @@ -510,6 +516,27 @@ func (s *ShTest) GenerateAndroidBuildActions(ctx android.ModuleContext) { installedData := ctx.InstallTestData(s.installDir, s.data) s.installedFile = ctx.InstallExecutable(s.installDir, s.outputFilePath.Base(), s.outputFilePath, installedData...) + + mkEntries := s.AndroidMkEntries()[0] + android.SetProvider(ctx, tradefed.BaseTestProviderKey, tradefed.BaseTestProviderData{ + TestcaseRelDataFiles: addArch(ctx.Arch().ArchType.String(), installedData.Paths()), + OutputFile: s.outputFilePath, + TestConfig: s.testConfig, + TestSuites: s.testProperties.Test_suites, + IsHost: false, + IsUnitTest: Bool(s.testProperties.Test_options.Unit_test), + MkInclude: mkEntries.Include, + MkAppClass: mkEntries.Class, + InstallDir: s.installDir, + }) +} + +func addArch(archType string, paths android.Paths) []string { + archRelPaths := []string{} + for _, p := range paths { + archRelPaths = append(archRelPaths, fmt.Sprintf("%s/%s", archType, p.Rel())) + } + return archRelPaths } func (s *ShTest) InstallInData() bool { @@ -533,6 +560,9 @@ func (s *ShTest) AndroidMkEntries() []android.AndroidMkEntries { entries.AddStrings("LOCAL_TEST_DATA_BINS", s.testProperties.Data_bins...) } entries.SetBoolIfTrue("LOCAL_COMPATIBILITY_PER_TESTCASE_DIRECTORY", Bool(s.testProperties.Per_testcase_directory)) + if len(s.extraTestConfigs) > 0 { + entries.AddStrings("LOCAL_EXTRA_FULL_TEST_CONFIGS", s.extraTestConfigs.Strings()...) + } s.testProperties.Test_options.SetAndroidMkEntries(entries) }, diff --git a/sh/sh_binary_test.go b/sh/sh_binary_test.go index 5a5043946..28f997d8e 100644 --- a/sh/sh_binary_test.go +++ b/sh/sh_binary_test.go @@ -176,6 +176,22 @@ func TestShTestHost(t *testing.T) { android.AssertBoolEquals(t, "LOCAL_IS_UNIT_TEST", true, actualData) } +func TestShTestExtraTestConfig(t *testing.T) { + result, _ := testShBinary(t, ` + sh_test { + name: "foo", + src: "test.sh", + filename: "test.sh", + extra_test_configs: ["config1.xml", "config2.xml"], + } + `) + + mod := result.ModuleForTests("foo", "android_arm64_armv8-a").Module().(*ShTest) + entries := android.AndroidMkEntriesForTest(t, result, mod)[0] + actualData := entries.EntryMap["LOCAL_EXTRA_FULL_TEST_CONFIGS"] + android.AssertStringPathsRelativeToTopEquals(t, "extra_configs", result.Config(), []string{"config1.xml", "config2.xml"}, actualData) +} + func TestShTestHost_dataDeviceModules(t *testing.T) { ctx, config := testShBinary(t, ` sh_test_host { diff --git a/tradefed/providers.go b/tradefed/providers.go index 0abac1279..0ae841dc5 100644 --- a/tradefed/providers.go +++ b/tradefed/providers.go @@ -9,8 +9,8 @@ import ( // Data that test_module_config[_host] modules types will need from // their dependencies to write out build rules and AndroidMkEntries. type BaseTestProviderData struct { - // data files and apps for android_test - InstalledFiles android.Paths + // data files and apps installed for tests, relative to testcases dir. + TestcaseRelDataFiles []string // apk for android_test OutputFile android.Path // Either handwritten or generated TF xml. @@ -28,6 +28,12 @@ type BaseTestProviderData struct { LocalCertificate string // Indicates if the base module was a unit test. IsUnitTest bool + // The .mk file is used AndroidMkEntries for base (soong_java_prebuilt, etc.) + MkInclude string + // The AppClass to use for the AndroidMkEntries for the base. + MkAppClass string + // value for LOCAL_MODULE_PATH. The directory where the module is installed. + InstallDir android.InstallPath } var BaseTestProviderKey = blueprint.NewProvider[BaseTestProviderData]() diff --git a/tradefed_modules/Android.bp b/tradefed_modules/Android.bp index 67d91b27c..a765a0532 100644 --- a/tradefed_modules/Android.bp +++ b/tradefed_modules/Android.bp @@ -9,6 +9,7 @@ bootstrap_go_package { "blueprint", "soong-android", "soong-java", + "soong-sh", "soong-tradefed", ], srcs: [ diff --git a/tradefed_modules/test_module_config.go b/tradefed_modules/test_module_config.go index 5c13d64e8..988352cd2 100644 --- a/tradefed_modules/test_module_config.go +++ b/tradefed_modules/test_module_config.go @@ -196,7 +196,7 @@ func TestModuleConfigFactory() android.Module { module := &testModuleConfigModule{} module.AddProperties(&module.tradefedProperties) - android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibCommon) + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) android.InitDefaultableModule(module) return module @@ -216,13 +216,28 @@ func TestModuleConfigHostFactory() android.Module { // Implements android.AndroidMkEntriesProvider var _ android.AndroidMkEntriesProvider = (*testModuleConfigModule)(nil) +func (m *testModuleConfigModule) nativeExtraEntries(entries *android.AndroidMkEntries) { + // TODO(ron) provider for suffix and STEM? + entries.SetString("LOCAL_MODULE_SUFFIX", "") + // Should the stem and path use the base name or our module name? + entries.SetString("LOCAL_MODULE_STEM", m.provider.OutputFile.Rel()) + entries.SetPath("LOCAL_MODULE_PATH", m.provider.InstallDir) +} + +func (m *testModuleConfigModule) javaExtraEntries(entries *android.AndroidMkEntries) { + // The app_prebuilt_internal.mk files try create a copy of the OutputFile as an .apk. + // Normally, this copies the "package.apk" from the intermediate directory here. + // To prevent the copy of the large apk and to prevent confusion with the real .apk we + // link to, we set the STEM here to a bogus name and we set OutputFile to a small file (our manifest). + // We do this so we don't have to add more conditionals to base_rules.mk + // soong_java_prebult has the same issue for .jars so use this in both module types. + entries.SetString("LOCAL_MODULE_STEM", fmt.Sprintf("UNUSED-%s", *m.Base)) + entries.SetString("LOCAL_MODULE_TAGS", "tests") +} + func (m *testModuleConfigModule) AndroidMkEntries() []android.AndroidMkEntries { - appClass := "APPS" - include := "$(BUILD_SYSTEM)/soong_app_prebuilt.mk" - if m.isHost { - appClass = "JAVA_LIBRARIES" - include = "$(BUILD_SYSTEM)/soong_java_prebuilt.mk" - } + appClass := m.provider.MkAppClass + include := m.provider.MkInclude return []android.AndroidMkEntries{{ Class: appClass, OutputFile: android.OptionalPathForPath(m.manifest), @@ -231,7 +246,6 @@ func (m *testModuleConfigModule) AndroidMkEntries() []android.AndroidMkEntries { ExtraEntries: []android.AndroidMkExtraEntriesFunc{ func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { entries.SetPath("LOCAL_FULL_TEST_CONFIG", m.testConfig) - entries.SetString("LOCAL_MODULE_TAGS", "tests") entries.SetString("LOCAL_TEST_MODULE_CONFIG_BASE", *m.Base) if m.provider.LocalSdkVersion != "" { entries.SetString("LOCAL_SDK_VERSION", m.provider.LocalSdkVersion) @@ -244,13 +258,11 @@ func (m *testModuleConfigModule) AndroidMkEntries() []android.AndroidMkEntries { entries.AddCompatibilityTestSuites(m.tradefedProperties.Test_suites...) entries.AddStrings("LOCAL_HOST_REQUIRED_MODULES", m.provider.HostRequiredModuleNames...) - // The app_prebuilt_internal.mk files try create a copy of the OutputFile as an .apk. - // Normally, this copies the "package.apk" from the intermediate directory here. - // To prevent the copy of the large apk and to prevent confusion with the real .apk we - // link to, we set the STEM here to a bogus name and we set OutputFile to a small file (our manifest). - // We do this so we don't have to add more conditionals to base_rules.mk - // soong_java_prebult has the same issue for .jars so use this in both module types. - entries.SetString("LOCAL_MODULE_STEM", fmt.Sprintf("UNUSED-%s", *m.Base)) + if m.provider.MkAppClass == "NATIVE_TESTS" { + m.nativeExtraEntries(entries) + } else { + m.javaExtraEntries(entries) + } // In normal java/app modules, the module writes LOCAL_COMPATIBILITY_SUPPORT_FILES // and then base_rules.mk ends up copying each of those dependencies from .intermediates to the install directory. @@ -357,16 +369,19 @@ func (m *testModuleConfigModule) generateManifestAndConfig(ctx android.ModuleCon // FrameworksServicesTests // └── x86_64 // └── FrameworksServicesTests.apk - symlinkName := fmt.Sprintf("%s/%s", ctx.DeviceConfig().DeviceArch(), baseApk.Base()) - // Only android_test, not java_host_test puts the output in the DeviceArch dir. - if m.provider.IsHost || ctx.DeviceConfig().DeviceArch() == "" { - // testcases/CtsDevicePolicyManagerTestCases - // ├── CtsDevicePolicyManagerTestCases.jar - symlinkName = baseApk.Base() + if m.provider.MkAppClass != "NATIVE_TESTS" { + symlinkName := fmt.Sprintf("%s/%s", ctx.DeviceConfig().DeviceArch(), baseApk.Base()) + // Only android_test, not java_host_test puts the output in the DeviceArch dir. + if m.provider.IsHost || ctx.DeviceConfig().DeviceArch() == "" { + // testcases/CtsDevicePolicyManagerTestCases + // ├── CtsDevicePolicyManagerTestCases.jar + symlinkName = baseApk.Base() + } + + target := installedBaseRelativeToHere(symlinkName, *m.tradefedProperties.Base) + installedApk := ctx.InstallAbsoluteSymlink(installDir, symlinkName, target) + m.supportFiles = append(m.supportFiles, installedApk) } - target := installedBaseRelativeToHere(symlinkName, *m.tradefedProperties.Base) - installedApk := ctx.InstallAbsoluteSymlink(installDir, symlinkName, target) - m.supportFiles = append(m.supportFiles, installedApk) // 3) Symlink for all data deps // And like this for data files and required modules @@ -374,8 +389,7 @@ func (m *testModuleConfigModule) generateManifestAndConfig(ctx android.ModuleCon // ├── data // │ └── broken_shortcut.xml // ├── JobTestApp.apk - for _, f := range m.provider.InstalledFiles { - symlinkName := f.Rel() + for _, symlinkName := range m.provider.TestcaseRelDataFiles { target := installedBaseRelativeToHere(symlinkName, *m.tradefedProperties.Base) installedPath := ctx.InstallAbsoluteSymlink(installDir, symlinkName, target) m.supportFiles = append(m.supportFiles, installedPath) @@ -386,7 +400,7 @@ func (m *testModuleConfigModule) generateManifestAndConfig(ctx android.ModuleCon // 5) We provide so we can be listed in test_suites. android.SetProvider(ctx, tradefed.BaseTestProviderKey, tradefed.BaseTestProviderData{ - InstalledFiles: m.supportFiles.Paths(), + TestcaseRelDataFiles: testcaseRel(m.supportFiles.Paths()), OutputFile: baseApk, TestConfig: m.testConfig, HostRequiredModuleNames: m.provider.HostRequiredModuleNames, @@ -400,6 +414,14 @@ func (m *testModuleConfigModule) generateManifestAndConfig(ctx android.ModuleCon var _ android.AndroidMkEntriesProvider = (*testModuleConfigHostModule)(nil) +func testcaseRel(paths android.Paths) []string { + relPaths := []string{} + for _, p := range paths { + relPaths = append(relPaths, p.Rel()) + } + return relPaths +} + // Given a relative path to a file in the current directory or a subdirectory, // return a relative path under our sibling directory named `base`. // There should be one "../" for each subdir we descend plus one to backup to "base". diff --git a/tradefed_modules/test_module_config_test.go b/tradefed_modules/test_module_config_test.go index cf6c7d17f..efd4a0425 100644 --- a/tradefed_modules/test_module_config_test.go +++ b/tradefed_modules/test_module_config_test.go @@ -16,6 +16,7 @@ package tradefed_modules import ( "android/soong/android" "android/soong/java" + "android/soong/sh" "fmt" "strconv" "strings" @@ -54,6 +55,8 @@ const bp = ` ` +const variant = "android_arm64_armv8-a" + // Ensure we create files needed and set the AndroidMkEntries needed func TestModuleConfigAndroidTest(t *testing.T) { @@ -62,7 +65,7 @@ func TestModuleConfigAndroidTest(t *testing.T) { android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), ).RunTestWithBp(t, bp) - derived := ctx.ModuleForTests("derived_test", "android_common") + derived := ctx.ModuleForTests("derived_test", variant) // Assert there are rules to create these files. derived.Output("test_module_config.manifest") derived.Output("test_config_fixer/derived_test.config") @@ -88,7 +91,7 @@ func TestModuleConfigAndroidTest(t *testing.T) { // And some new derived entries are there. android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE_TAGS"], []string{"tests"}) - android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "derived_test/android_common/test_config_fixer/derived_test.config") + android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], fmt.Sprintf("derived_test/%s/test_config_fixer/derived_test.config", variant)) // Check the footer lines. Our support files should depend on base's support files. convertedActual := make([]string, 5) @@ -105,6 +108,80 @@ func TestModuleConfigAndroidTest(t *testing.T) { }) } +func TestModuleConfigShTest(t *testing.T) { + ctx := android.GroupFixturePreparers( + sh.PrepareForTestWithShBuildComponents, + android.PrepareForTestWithAndroidBuildComponents, + android.FixtureMergeMockFs(android.MockFS{ + "test.sh": nil, + "testdata/data1": nil, + "testdata/sub/data2": nil, + }), + android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), + ).RunTestWithBp(t, ` + sh_test { + name: "shell_test", + src: "test.sh", + filename: "test.sh", + test_suites: ["general-tests"], + data: [ + "testdata/data1", + "testdata/sub/data2", + ], + } + test_module_config { + name: "conch", + base: "shell_test", + test_suites: ["general-tests"], + options: [{name: "SomeName", value: "OptionValue"}], + } + `) + derived := ctx.ModuleForTests("conch", variant) // + conch := derived.Module().(*testModuleConfigModule) + android.AssertArrayString(t, "TestcaseRelDataFiles", []string{"arm64/testdata/data1", "arm64/testdata/sub/data2"}, conch.provider.TestcaseRelDataFiles) + android.AssertStringEquals(t, "Rel OutputFile", "test.sh", conch.provider.OutputFile.Rel()) + + // Assert there are rules to create these files. + derived.Output("test_module_config.manifest") + derived.Output("test_config_fixer/conch.config") + + // Ensure some basic rules exist. + entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0] + + // Ensure some entries from base are there, specifically support files for data and helper apps. + // Do not use LOCAL_COMPATIBILITY_SUPPORT_FILES, but instead use LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES + android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config, + []string{"out/soong/target/product/test_device/testcases/conch/arm64/testdata/data1", + "out/soong/target/product/test_device/testcases/conch/arm64/testdata/sub/data2"}, + entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"]) + android.AssertArrayString(t, "", entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"], []string{}) + + android.AssertStringEquals(t, "app class", "NATIVE_TESTS", entries.Class) + android.AssertArrayString(t, "required modules", []string{"shell_test"}, entries.EntryMap["LOCAL_REQUIRED_MODULES"]) + android.AssertArrayString(t, "host required modules", []string{}, entries.EntryMap["LOCAL_HOST_REQUIRED_MODULES"]) + android.AssertArrayString(t, "cert", []string{}, entries.EntryMap["LOCAL_CERTIFICATE"]) + + // And some new derived entries are there. + android.AssertArrayString(t, "tags", []string{}, entries.EntryMap["LOCAL_MODULE_TAGS"]) + + android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], + fmt.Sprintf("conch/%s/test_config_fixer/conch.config", variant)) + + // Check the footer lines. Our support files should depend on base's support files. + convertedActual := make([]string, 4) + for i, e := range entries.FooterLinesForTests() { + // AssertStringPathsRelativeToTop doesn't replace both instances + convertedActual[i] = strings.Replace(e, ctx.Config.SoongOutDir(), "", 2) + } + android.AssertArrayString(t, fmt.Sprintf("%s", ctx.Config.SoongOutDir()), convertedActual, []string{ + "include $(BUILD_SYSTEM)/soong_cc_rust_prebuilt.mk", + "/target/product/test_device/testcases/conch/arm64/testdata/data1: /target/product/test_device/testcases/shell_test/arm64/testdata/data1", + "/target/product/test_device/testcases/conch/arm64/testdata/sub/data2: /target/product/test_device/testcases/shell_test/arm64/testdata/sub/data2", + "", + }) + +} + // Make sure we call test-config-fixer with the right args. func TestModuleConfigOptions(t *testing.T) { @@ -114,7 +191,7 @@ func TestModuleConfigOptions(t *testing.T) { ).RunTestWithBp(t, bp) // Check that we generate a rule to make a new AndroidTest.xml/Module.config file. - derived := ctx.ModuleForTests("derived_test", "android_common") + derived := ctx.ModuleForTests("derived_test", variant) rule_cmd := derived.Rule("fix_test_config").RuleParams.Command android.AssertStringDoesContain(t, "Bad FixConfig rule inputs", rule_cmd, `--test-runner-options='[{"Name":"exclude-filter","Key":"","Value":"android.test.example.devcodelab.DevCodelabTest#testHelloFail"},{"Name":"include-annotation","Key":"","Value":"android.platform.test.annotations.LargeTest"}]'`) @@ -211,8 +288,7 @@ func TestModuleConfigNoFiltersOrAnnotationsShouldFail(t *testing.T) { ).ExtendWithErrorHandler( android.FixtureExpectsAtLeastOneErrorMatchingPattern("Test options must be given")). RunTestWithBp(t, badBp) - - ctx.ModuleForTests("derived_test", "android_common") + ctx.ModuleForTests("derived_test", variant) } func TestModuleConfigMultipleDerivedTestsWriteDistinctMakeEntries(t *testing.T) { @@ -250,7 +326,7 @@ func TestModuleConfigMultipleDerivedTestsWriteDistinctMakeEntries(t *testing.T) ).RunTestWithBp(t, multiBp) { - derived := ctx.ModuleForTests("derived_test", "android_common") + derived := ctx.ModuleForTests("derived_test", variant) entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0] // All these should be the same in both derived tests android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config, @@ -260,13 +336,13 @@ func TestModuleConfigMultipleDerivedTestsWriteDistinctMakeEntries(t *testing.T) entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"]) // Except this one, which points to the updated tradefed xml file. - android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "derived_test/android_common/test_config_fixer/derived_test.config") + android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], fmt.Sprintf("derived_test/%s/test_config_fixer/derived_test.config", variant)) // And this one, the module name. android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"derived_test"}) } { - derived := ctx.ModuleForTests("another_derived_test", "android_common") + derived := ctx.ModuleForTests("another_derived_test", variant) entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0] // All these should be the same in both derived tests android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config, @@ -275,7 +351,8 @@ func TestModuleConfigMultipleDerivedTestsWriteDistinctMakeEntries(t *testing.T) "out/soong/target/product/test_device/testcases/another_derived_test/data/testfile"}, entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"]) // Except this one, which points to the updated tradefed xml file. - android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "another_derived_test/android_common/test_config_fixer/another_derived_test.config") + android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], + fmt.Sprintf("another_derived_test/%s/test_config_fixer/another_derived_test.config", variant)) // And this one, the module name. android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"another_derived_test"}) } diff --git a/ui/build/androidmk_denylist.go b/ui/build/androidmk_denylist.go index c54d55faa..6f57cb1c4 100644 --- a/ui/build/androidmk_denylist.go +++ b/ui/build/androidmk_denylist.go @@ -29,6 +29,9 @@ var androidmk_denylist []string = []string{ "device/google_car/", "device/sample/", "frameworks/", + "hardware/libhardware/", + "hardware/libhardware_legacy/", + "hardware/ril/", // Do not block other directories in kernel/, see b/319658303. "kernel/configs/", "kernel/prebuilts/", @@ -37,8 +40,10 @@ var androidmk_denylist []string = []string{ "libnativehelper/", "packages/", "pdk/", + "platform_testing/", "prebuilts/", "sdk/", + "system/", "test/", "trusty/", // Add back toolchain/ once defensive Android.mk files are removed diff --git a/ui/build/soong.go b/ui/build/soong.go index e6d01dd1d..0963f76b7 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -433,13 +433,13 @@ func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string) } } -func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error { +func updateSymlinks(ctx Context, dir, prevCWD, cwd string, updateSemaphore chan struct{}) error { defer symlinkWg.Done() visit := func(path string, d fs.DirEntry, err error) error { if d.IsDir() && path != dir { symlinkWg.Add(1) - go updateSymlinks(ctx, path, prevCWD, cwd) + go updateSymlinks(ctx, path, prevCWD, cwd, updateSemaphore) return filepath.SkipDir } f, err := d.Info() @@ -470,12 +470,27 @@ func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error { return nil } + <-updateSemaphore + defer func() { updateSemaphore <- struct{}{} }() if err := filepath.WalkDir(dir, visit); err != nil { return err } return nil } +// b/376466642: If the concurrency of updateSymlinks is unbounded, Go's runtime spawns a +// theoretically unbounded number of threads to handle blocking syscalls. This causes the runtime to +// panic due to hitting thread limits in rare cases. Limiting to GOMAXPROCS concurrent symlink +// updates should make this a non-issue. +func newUpdateSemaphore() chan struct{} { + numPermits := runtime.GOMAXPROCS(0) + c := make(chan struct{}, numPermits) + for i := 0; i < numPermits; i++ { + c <- struct{}{} + } + return c +} + func fixOutDirSymlinks(ctx Context, config Config, outDir string) error { cwd, err := os.Getwd() if err != nil { @@ -508,7 +523,7 @@ func fixOutDirSymlinks(ctx Context, config Config, outDir string) error { } symlinkWg.Add(1) - if err := updateSymlinks(ctx, outDir, prevCWD, cwd); err != nil { + if err := updateSymlinks(ctx, outDir, prevCWD, cwd, newUpdateSemaphore()); err != nil { return err } symlinkWg.Wait() diff --git a/ui/metrics/Android.bp b/ui/metrics/Android.bp index 77871fc09..591e3cca6 100644 --- a/ui/metrics/Android.bp +++ b/ui/metrics/Android.bp @@ -26,6 +26,7 @@ bootstrap_go_package { "soong-ui-metrics_proto", "soong-ui-mk_metrics_proto", "soong-shared", + "soong-ui-metrics_combined_proto", ], srcs: [ "hostinfo.go", @@ -63,6 +64,19 @@ bootstrap_go_package { } bootstrap_go_package { + name: "soong-ui-metrics_combined_proto", + pkgPath: "android/soong/ui/metrics/combined_metrics_proto", + deps: [ + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-find_input_delta-proto", + ], + srcs: [ + "metrics_proto/metrics.pb.go", + ], +} + +bootstrap_go_package { name: "soong-ui-metrics_upload_proto", pkgPath: "android/soong/ui/metrics/upload_proto", deps: [ diff --git a/ui/metrics/metrics_proto/Android.bp b/ui/metrics/metrics_proto/Android.bp new file mode 100644 index 000000000..aae5266d0 --- /dev/null +++ b/ui/metrics/metrics_proto/Android.bp @@ -0,0 +1,16 @@ +python_library_host { + name: "soong-metrics-proto-py", + srcs: [ + "metrics.proto", + ], + visibility: [ + "//build/make/ci", + ], + libs: [ + "libprotobuf-python", + ], + proto: { + include_dirs: ["external/protobuf/src"], + canonical_path_from_root: false, + }, +} diff --git a/ui/metrics/metrics_proto/combined_metrics.pb.go b/ui/metrics/metrics_proto/combined_metrics.pb.go new file mode 100644 index 000000000..f49d64d54 --- /dev/null +++ b/ui/metrics/metrics_proto/combined_metrics.pb.go @@ -0,0 +1,239 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.21.12 +// source: combined_metrics.proto + +package metrics_proto + +import ( + find_input_delta_proto "android/soong/cmd/find_input_delta/find_input_delta_proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// These field numbers are also found in the inner message declarations. +// We verify that the values are the same, and that every enum value is checked +// in combined_metrics_test.go. +// Do not change this enum without also updating: +// - the submessage's .proto file +// - combined_metrics_test.go +type FieldNumbers int32 + +const ( + FieldNumbers_FIELD_NUMBERS_UNSPECIFIED FieldNumbers = 0 + FieldNumbers_FIELD_NUMBERS_FILE_LIST FieldNumbers = 1 +) + +// Enum value maps for FieldNumbers. +var ( + FieldNumbers_name = map[int32]string{ + 0: "FIELD_NUMBERS_UNSPECIFIED", + 1: "FIELD_NUMBERS_FILE_LIST", + } + FieldNumbers_value = map[string]int32{ + "FIELD_NUMBERS_UNSPECIFIED": 0, + "FIELD_NUMBERS_FILE_LIST": 1, + } +) + +func (x FieldNumbers) Enum() *FieldNumbers { + p := new(FieldNumbers) + *p = x + return p +} + +func (x FieldNumbers) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FieldNumbers) Descriptor() protoreflect.EnumDescriptor { + return file_combined_metrics_proto_enumTypes[0].Descriptor() +} + +func (FieldNumbers) Type() protoreflect.EnumType { + return &file_combined_metrics_proto_enumTypes[0] +} + +func (x FieldNumbers) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *FieldNumbers) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = FieldNumbers(num) + return nil +} + +// Deprecated: Use FieldNumbers.Descriptor instead. +func (FieldNumbers) EnumDescriptor() ([]byte, []int) { + return file_combined_metrics_proto_rawDescGZIP(), []int{0} +} + +type SoongCombinedMetrics struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // cmd/find_input_delta/find_input_delta_proto.FileList + FileList *find_input_delta_proto.FileList `protobuf:"bytes,1,opt,name=file_list,json=fileList" json:"file_list,omitempty"` +} + +func (x *SoongCombinedMetrics) Reset() { + *x = SoongCombinedMetrics{} + if protoimpl.UnsafeEnabled { + mi := &file_combined_metrics_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SoongCombinedMetrics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SoongCombinedMetrics) ProtoMessage() {} + +func (x *SoongCombinedMetrics) ProtoReflect() protoreflect.Message { + mi := &file_combined_metrics_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SoongCombinedMetrics.ProtoReflect.Descriptor instead. +func (*SoongCombinedMetrics) Descriptor() ([]byte, []int) { + return file_combined_metrics_proto_rawDescGZIP(), []int{0} +} + +func (x *SoongCombinedMetrics) GetFileList() *find_input_delta_proto.FileList { + if x != nil { + return x.FileList + } + return nil +} + +var File_combined_metrics_proto protoreflect.FileDescriptor + +var file_combined_metrics_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x13, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, 0x3b, 0x63, + 0x6d, 0x64, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, + 0x6c, 0x74, 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, + 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x5d, 0x0a, 0x14, 0x53, 0x6f, + 0x6f, 0x6e, 0x67, 0x43, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x12, 0x45, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, + 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x2a, 0x4a, 0x0a, 0x0c, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x49, 0x45, + 0x4c, 0x44, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, + 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x46, 0x49, 0x45, 0x4c, + 0x44, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x53, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4c, + 0x49, 0x53, 0x54, 0x10, 0x01, 0x42, 0x28, 0x5a, 0x26, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, + 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, +} + +var ( + file_combined_metrics_proto_rawDescOnce sync.Once + file_combined_metrics_proto_rawDescData = file_combined_metrics_proto_rawDesc +) + +func file_combined_metrics_proto_rawDescGZIP() []byte { + file_combined_metrics_proto_rawDescOnce.Do(func() { + file_combined_metrics_proto_rawDescData = protoimpl.X.CompressGZIP(file_combined_metrics_proto_rawDescData) + }) + return file_combined_metrics_proto_rawDescData +} + +var file_combined_metrics_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_combined_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_combined_metrics_proto_goTypes = []interface{}{ + (FieldNumbers)(0), // 0: soong_build_metrics.FieldNumbers + (*SoongCombinedMetrics)(nil), // 1: soong_build_metrics.SoongCombinedMetrics + (*find_input_delta_proto.FileList)(nil), // 2: android.find_input_delta_proto.FileList +} +var file_combined_metrics_proto_depIdxs = []int32{ + 2, // 0: soong_build_metrics.SoongCombinedMetrics.file_list:type_name -> android.find_input_delta_proto.FileList + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_combined_metrics_proto_init() } +func file_combined_metrics_proto_init() { + if File_combined_metrics_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_combined_metrics_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SoongCombinedMetrics); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_combined_metrics_proto_rawDesc, + NumEnums: 1, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_combined_metrics_proto_goTypes, + DependencyIndexes: file_combined_metrics_proto_depIdxs, + EnumInfos: file_combined_metrics_proto_enumTypes, + MessageInfos: file_combined_metrics_proto_msgTypes, + }.Build() + File_combined_metrics_proto = out.File + file_combined_metrics_proto_rawDesc = nil + file_combined_metrics_proto_goTypes = nil + file_combined_metrics_proto_depIdxs = nil +} diff --git a/ui/metrics/metrics_proto/combined_metrics.proto b/ui/metrics/metrics_proto/combined_metrics.proto new file mode 100644 index 000000000..3cd9a5313 --- /dev/null +++ b/ui/metrics/metrics_proto/combined_metrics.proto @@ -0,0 +1,36 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package soong_build_metrics; +option go_package = "android/soong/ui/metrics/metrics_proto"; + +import "cmd/find_input_delta/find_input_delta_proto/file_list.proto"; + +// These field numbers are also found in the inner message declarations. +// We verify that the values are the same, and that every enum value is checked +// in combined_metrics_test.go. +// Do not change this enum without also updating: +// - the submessage's .proto file +// - combined_metrics_test.go +enum FieldNumbers { + FIELD_NUMBERS_UNSPECIFIED = 0; + FIELD_NUMBERS_FILE_LIST = 1; +} + +message SoongCombinedMetrics { + // cmd/find_input_delta/find_input_delta_proto.FileList + optional android.find_input_delta_proto.FileList file_list = 1; +} diff --git a/ui/metrics/metrics_proto/combined_metrics_test.go b/ui/metrics/metrics_proto/combined_metrics_test.go new file mode 100644 index 000000000..eedb12a34 --- /dev/null +++ b/ui/metrics/metrics_proto/combined_metrics_test.go @@ -0,0 +1,33 @@ +package metrics_proto + +import ( + "testing" + + find_input_delta_proto "android/soong/cmd/find_input_delta/find_input_delta_proto" +) + +func TestCombinedMetricsMessageNums(t *testing.T) { + testCases := []struct { + Name string + FieldNumbers map[string]int32 + }{ + { + Name: "find_input_delta_proto", + FieldNumbers: find_input_delta_proto.FieldNumbers_value, + }, + } + verifiedMap := make(map[string]bool) + for _, tc := range testCases { + for k, v := range tc.FieldNumbers { + if FieldNumbers_value[k] != v { + t.Errorf("%s: Expected FieldNumbers.%s == %v, found %v", tc.Name, k, FieldNumbers_value[k], v) + } + verifiedMap[k] = true + } + } + for k, v := range FieldNumbers_value { + if !verifiedMap[k] { + t.Errorf("No test case verifies FieldNumbers.%s=%v", k, v) + } + } +} diff --git a/ui/metrics/metrics_proto/regen.sh b/ui/metrics/metrics_proto/regen.sh index 8eb2d747b..5e5f9b83f 100755 --- a/ui/metrics/metrics_proto/regen.sh +++ b/ui/metrics/metrics_proto/regen.sh @@ -12,6 +12,6 @@ if ! hash aprotoc &>/dev/null; then die "could not find aprotoc. ${error_msg}" fi -if ! aprotoc --go_out=paths=source_relative:. metrics.proto; then +if ! aprotoc --go_out=paths=source_relative:. -I .:../../.. metrics.proto combined_metrics.proto; then die "build failed. ${error_msg}" fi |