diff options
Diffstat (limited to 'filesystem/filesystem.go')
| -rw-r--r-- | filesystem/filesystem.go | 304 |
1 files changed, 190 insertions, 114 deletions
diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index 6dfbfd1be..68cbee95f 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -20,6 +20,7 @@ import ( "io" "path/filepath" "slices" + "sort" "strconv" "strings" @@ -173,12 +174,6 @@ type FilesystemProperties struct { // Mount point for this image. Default is "/" Mount_point *string - // If set to the name of a partition ("system", "vendor", etc), this filesystem module - // will also include the contents of the make-built staging directories. If any soong - // modules would be installed to the same location as a make module, they will overwrite - // the make version. - Include_make_built_files string - // When set, builds etc/event-log-tags file by merging logtags from all dependencies. // Default is false Build_logtags *bool @@ -215,6 +210,22 @@ type FilesystemProperties struct { // Additional dependencies used for building android products Android_filesystem_deps AndroidFilesystemDeps + + // Name of the output. Default is $(module_name).img + Stem *string + + // The size of the partition on the device. It will be a build error if this built partition + // image exceeds this size. + Partition_size *int64 + + // Whether to format f2fs and ext4 in a way that supports casefolding + Support_casefolding *bool + + // Whether to format f2fs and ext4 in a way that supports project quotas + Support_project_quota *bool + + // Whether to enable per-file compression in f2fs + Enable_compression *bool } type AndroidFilesystemDeps struct { @@ -347,16 +358,46 @@ func (fs fsType) IsUnknown() bool { type FilesystemInfo struct { // The built filesystem image Output android.Path + // An additional hermetic filesystem image. + // e.g. this will contain inodes with pinned timestamps. + // This will be copied to target_files.zip + OutputHermetic android.Path // A text file containing the list of paths installed on the partition. FileListFile android.Path // The root staging directory used to build the output filesystem. If consuming this, make sure // to add a dependency on the Output file, as you cannot add dependencies on directories // in ninja. RootDir android.Path + // The rebased staging directory used to build the output filesystem. If consuming this, make + // sure to add a dependency on the Output file, as you cannot add dependencies on directories + // in ninja. In many cases this is the same as RootDir, only in the system partition is it + // different. There, it points to the "system" sub-directory of RootDir. + RebasedDir android.Path + // A text file with block data of the .img file + // This is an implicit output of `build_image` + MapFile android.Path + // Name of the module that produced this FilesystemInfo origionally. (though it may be + // re-exported by super images or boot images) + ModuleName string + // The property file generated by this module and passed to build_image. + // It's exported here so that system_other can reuse system's property file. + BuildImagePropFile android.Path + // Paths to all the tools referenced inside of the build image property file. + BuildImagePropFileDeps android.Paths + // Packaging specs to be installed on the system_other image, for the initial boot's dexpreopt. + SpecsForSystemOther map[string]android.PackagingSpec } var FilesystemProvider = blueprint.NewProvider[FilesystemInfo]() +type FilesystemDefaultsInfo struct { + // Identifies which partition this is for //visibility:any_system_image (and others) visibility + // checks, and will be used in the future for API surface checks. + PartitionType string +} + +var FilesystemDefaultsInfoProvider = blueprint.NewProvider[FilesystemDefaultsInfo]() + func GetFsTypeFromString(ctx android.EarlyModuleContext, typeStr string) fsType { switch typeStr { case "ext4": @@ -384,7 +425,7 @@ func (f *filesystem) fsType(ctx android.ModuleContext) fsType { } func (f *filesystem) installFileName() string { - return f.BaseModuleName() + ".img" + return proptools.StringDefault(f.properties.Stem, f.BaseModuleName()+".img") } func (f *filesystem) partitionName() string { @@ -431,14 +472,36 @@ func (f *filesystem) GenerateAndroidBuildActions(ctx android.ModuleContext) { ctx.PropertyErrorf("include_files_of", "include_files_of is only supported for cpio and compressed cpio filesystem types.") } - var rootDir android.OutputPath + rootDir := android.PathForModuleOut(ctx, f.rootDirString()).OutputPath + rebasedDir := rootDir + if f.properties.Base_dir != nil { + rebasedDir = rootDir.Join(ctx, *f.properties.Base_dir) + } + builder := android.NewRuleBuilder(pctx, ctx) + + // Wipe the root dir to get rid of leftover files from prior builds + builder.Command().Textf("rm -rf %s && mkdir -p %s", rootDir, rootDir) + specs := f.gatherFilteredPackagingSpecs(ctx) + f.entries = f.copyPackagingSpecs(ctx, builder, specs, rootDir, rebasedDir) + + f.buildNonDepsFiles(ctx, builder, rootDir) + f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir) + f.buildEventLogtagsFile(ctx, builder, rebasedDir) + f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir) + f.filesystemBuilder.BuildLinkerConfigFile(ctx, builder, rebasedDir) + + var mapFile android.Path + var outputHermetic android.Path + var buildImagePropFile android.Path + var buildImagePropFileDeps android.Paths switch f.fsType(ctx) { case ext4Type, erofsType, f2fsType: - f.output, rootDir = f.buildImageUsingBuildImage(ctx) + f.output, outputHermetic, buildImagePropFile, buildImagePropFileDeps = f.buildImageUsingBuildImage(ctx, builder, rootDir, rebasedDir) + mapFile = f.getMapFile(ctx) case compressedCpioType: - f.output, rootDir = f.buildCpioImage(ctx, true) + f.output = f.buildCpioImage(ctx, builder, rootDir, true) case cpioType: - f.output, rootDir = f.buildCpioImage(ctx, false) + f.output = f.buildCpioImage(ctx, builder, rootDir, false) default: return } @@ -454,11 +517,21 @@ func (f *filesystem) GenerateAndroidBuildActions(ctx android.ModuleContext) { fileListFile := android.PathForModuleOut(ctx, "fileList") android.WriteFileRule(ctx, fileListFile, f.installedFilesList()) - android.SetProvider(ctx, FilesystemProvider, FilesystemInfo{ - Output: f.output, - FileListFile: fileListFile, - RootDir: rootDir, - }) + fsInfo := FilesystemInfo{ + Output: f.output, + OutputHermetic: outputHermetic, + FileListFile: fileListFile, + RootDir: rootDir, + RebasedDir: rebasedDir, + MapFile: mapFile, + ModuleName: ctx.ModuleName(), + BuildImagePropFile: buildImagePropFile, + BuildImagePropFileDeps: buildImagePropFileDeps, + SpecsForSystemOther: f.systemOtherFiles(ctx), + } + + android.SetProvider(ctx, FilesystemProvider, fsInfo) + f.fileListFile = fileListFile if proptools.Bool(f.properties.Unchecked_module) { @@ -466,6 +539,11 @@ func (f *filesystem) GenerateAndroidBuildActions(ctx android.ModuleContext) { } } +func (f *filesystem) getMapFile(ctx android.ModuleContext) android.WritablePath { + // create the filepath by replacing the extension of the corresponding img file + return android.PathForModuleOut(ctx, f.installFileName()).ReplaceExtension(ctx, "map") +} + func (f *filesystem) validateVintfFragments(ctx android.ModuleContext) { visitedModule := map[string]bool{} packagingSpecs := f.gatherFilteredPackagingSpecs(ctx) @@ -524,12 +602,12 @@ func validatePartitionType(ctx android.ModuleContext, p partition) { ctx.PropertyErrorf("partition_type", "partition_type must be one of %s, found: %s", validPartitions, p.PartitionType()) } - ctx.VisitDirectDepsWithTag(android.DefaultsDepTag, func(m android.Module) { - if fdm, ok := m.(*filesystemDefaults); ok { - if p.PartitionType() != fdm.PartitionType() { + ctx.VisitDirectDepsProxyWithTag(android.DefaultsDepTag, func(m android.ModuleProxy) { + if fdm, ok := android.OtherModuleProvider(ctx, m, FilesystemDefaultsInfoProvider); ok { + if p.PartitionType() != fdm.PartitionType { ctx.PropertyErrorf("partition_type", "%s doesn't match with the partition type %s of the filesystem default module %s", - p.PartitionType(), fdm.PartitionType(), m.Name()) + p.PartitionType(), fdm.PartitionType, m.Name()) } } }) @@ -592,38 +670,16 @@ func (f *filesystem) copyPackagingSpecs(ctx android.ModuleContext, builder *andr return f.CopySpecsToDirs(ctx, builder, dirsToSpecs) } -func (f *filesystem) copyFilesToProductOut(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) { - if f.Name() != ctx.Config().SoongDefinedSystemImage() { - return - } - installPath := android.PathForModuleInPartitionInstall(ctx, f.partitionName()) - builder.Command().Textf("cp -prf %s/* %s", rebasedDir, installPath) -} - func (f *filesystem) rootDirString() string { return f.partitionName() } -func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) (android.Path, android.OutputPath) { - rootDir := android.PathForModuleOut(ctx, f.rootDirString()).OutputPath - rebasedDir := rootDir - if f.properties.Base_dir != nil { - rebasedDir = rootDir.Join(ctx, *f.properties.Base_dir) - } - builder := android.NewRuleBuilder(pctx, ctx) - // Wipe the root dir to get rid of leftover files from prior builds - builder.Command().Textf("rm -rf %s && mkdir -p %s", rootDir, rootDir) - specs := f.gatherFilteredPackagingSpecs(ctx) - f.entries = f.copyPackagingSpecs(ctx, builder, specs, rootDir, rebasedDir) - - f.buildNonDepsFiles(ctx, builder, rootDir) - f.addMakeBuiltFiles(ctx, builder, rootDir) - f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir) - f.buildEventLogtagsFile(ctx, builder, rebasedDir) - f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir) - f.filesystemBuilder.BuildLinkerConfigFile(ctx, builder, rebasedDir) - f.copyFilesToProductOut(ctx, builder, rebasedDir) - +func (f *filesystem) buildImageUsingBuildImage( + ctx android.ModuleContext, + builder *android.RuleBuilder, + rootDir android.OutputPath, + rebasedDir android.OutputPath, +) (android.Path, android.Path, android.Path, android.Paths) { // run host_init_verifier // Ideally we should have a concept of pluggable linters that verify the generated image. // While such concept is not implement this will do. @@ -641,6 +697,7 @@ func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) (andro pathToolDirs := []string{filepath.Dir(fec.String())} output := android.PathForModuleOut(ctx, f.installFileName()) + builder.Command().Text("touch").Output(f.getMapFile(ctx)) builder.Command(). Textf("PATH=%s:$PATH", strings.Join(pathToolDirs, ":")). BuiltTool("build_image"). @@ -651,10 +708,31 @@ func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) (andro Output(output). Text(rootDir.String()) // directory where to find fs_config_files|dirs + // Add an additional cmd to create a hermetic img file. This will contain pinned timestamps e.g. + propFilePinnedTimestamp := android.PathForModuleOut(ctx, "for_target_files", "prop") + builder.Command().Textf("cat").Input(propFile).Flag(">").Output(propFilePinnedTimestamp). + Textf(" && echo use_fixed_timestamp=true >> %s", propFilePinnedTimestamp). + Textf(" && echo block_list=%s >> %s", f.getMapFile(ctx).String(), propFilePinnedTimestamp) // mapfile will be an implicit output + + outputHermetic := android.PathForModuleOut(ctx, "for_target_files", f.installFileName()) + builder.Command(). + Textf("PATH=%s:$PATH", strings.Join(pathToolDirs, ":")). + BuiltTool("build_image"). + Text(rootDir.String()). // input directory + Flag(propFilePinnedTimestamp.String()). + Implicits(toolDeps). + Implicit(fec). + Output(outputHermetic). + Text(rootDir.String()) // directory where to find fs_config_files|dirs + + if f.properties.Partition_size != nil { + assertMaxImageSize(builder, output, *f.properties.Partition_size, false) + } + // rootDir is not deleted. Might be useful for quick inspection. builder.Build("build_filesystem_image", fmt.Sprintf("Creating filesystem %s", f.BaseModuleName())) - return output, rootDir + return output, outputHermetic, propFile, toolDeps } func (f *filesystem) buildFileContexts(ctx android.ModuleContext) android.Path { @@ -669,12 +747,9 @@ func (f *filesystem) buildFileContexts(ctx android.ModuleContext) android.Path { func (f *filesystem) buildPropFile(ctx android.ModuleContext) (android.Path, android.Paths) { var deps android.Paths - var propFileString strings.Builder + var lines []string addStr := func(name string, value string) { - propFileString.WriteString(name) - propFileString.WriteRune('=') - propFileString.WriteString(value) - propFileString.WriteRune('\n') + lines = append(lines, fmt.Sprintf("%s=%s", name, value)) } addPath := func(name string, path android.Path) { addStr(name, path.String()) @@ -760,12 +835,11 @@ func (f *filesystem) buildPropFile(ctx android.ModuleContext) (android.Path, and addStr("hash_seed", uuid) } - // TODO(b/381120092): This should only be added if none of the size-related properties are set, - // but currently soong built partitions don't have size properties. Make code: - // https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/Makefile;l=2262;drc=39cd33701c9278db0e7e481a090605f428d5b12d - // Make uses system_disable_sparse but disable_sparse has the same effect, and we shouldn't need - // to qualify it because each partition gets its own property file built. - addStr("disable_sparse", "true") + // Disable sparse only when partition size is not defined. disable_sparse has the same + // effect as <partition name>_disable_sparse. + if f.properties.Partition_size == nil { + addStr("disable_sparse", "true") + } fst := f.fsType(ctx) switch fst { @@ -788,8 +862,26 @@ func (f *filesystem) buildPropFile(ctx android.ModuleContext) (android.Path, and } f.checkFsTypePropertyError(ctx, fst, fsTypeStr(fst)) + if f.properties.Partition_size != nil { + addStr("partition_size", strconv.FormatInt(*f.properties.Partition_size, 10)) + } + + if proptools.BoolDefault(f.properties.Support_casefolding, false) { + addStr("needs_casefold", "1") + } + + if proptools.BoolDefault(f.properties.Support_project_quota, false) { + addStr("needs_projid", "1") + } + + if proptools.BoolDefault(f.properties.Enable_compression, false) { + addStr("needs_compress", "1") + } + + sort.Strings(lines) + propFilePreProcessing := android.PathForModuleOut(ctx, "prop_pre_processing") - android.WriteFileRuleVerbatim(ctx, propFilePreProcessing, propFileString.String()) + android.WriteFileRule(ctx, propFilePreProcessing, strings.Join(lines, "\n")) propFile := android.PathForModuleOut(ctx, "prop") ctx.Build(pctx, android.BuildParams{ Rule: textFileProcessorRule, @@ -832,7 +924,12 @@ func includeFilesRootDir(ctx android.ModuleContext) (rootDirs android.Paths, par return rootDirs, partitions } -func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) (android.Path, android.OutputPath) { +func (f *filesystem) buildCpioImage( + ctx android.ModuleContext, + builder *android.RuleBuilder, + rootDir android.OutputPath, + compressed bool, +) android.Path { if proptools.Bool(f.properties.Use_avb) { ctx.PropertyErrorf("use_avb", "signing compresed cpio image using avbtool is not supported."+ "Consider adding this to bootimg module and signing the entire boot image.") @@ -842,28 +939,6 @@ func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) ctx.PropertyErrorf("file_contexts", "file_contexts is not supported for compressed cpio image.") } - if f.properties.Include_make_built_files != "" { - ctx.PropertyErrorf("include_make_built_files", "include_make_built_files is not supported for compressed cpio image.") - } - - rootDir := android.PathForModuleOut(ctx, f.rootDirString()).OutputPath - rebasedDir := rootDir - if f.properties.Base_dir != nil { - rebasedDir = rootDir.Join(ctx, *f.properties.Base_dir) - } - builder := android.NewRuleBuilder(pctx, ctx) - // Wipe the root dir to get rid of leftover files from prior builds - builder.Command().Textf("rm -rf %s && mkdir -p %s", rootDir, rootDir) - specs := f.gatherFilteredPackagingSpecs(ctx) - f.entries = f.copyPackagingSpecs(ctx, builder, specs, rootDir, rebasedDir) - - f.buildNonDepsFiles(ctx, builder, rootDir) - f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir) - f.buildEventLogtagsFile(ctx, builder, rebasedDir) - f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir) - f.filesystemBuilder.BuildLinkerConfigFile(ctx, builder, rebasedDir) - f.copyFilesToProductOut(ctx, builder, rebasedDir) - rootDirs, partitions := includeFilesRootDir(ctx) output := android.PathForModuleOut(ctx, f.installFileName()) @@ -893,7 +968,7 @@ func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) // rootDir is not deleted. Might be useful for quick inspection. builder.Build("build_cpio_image", fmt.Sprintf("Creating filesystem %s", f.BaseModuleName())) - return output, rootDir + return output } var validPartitions = []string{ @@ -913,27 +988,6 @@ var validPartitions = []string{ "recovery", } -func (f *filesystem) addMakeBuiltFiles(ctx android.ModuleContext, builder *android.RuleBuilder, rootDir android.Path) { - partition := f.properties.Include_make_built_files - if partition == "" { - return - } - if !slices.Contains(validPartitions, partition) { - ctx.PropertyErrorf("include_make_built_files", "Expected one of %#v, found %q", validPartitions, partition) - return - } - stampFile := fmt.Sprintf("target/product/%s/obj/PACKAGING/%s_intermediates/staging_dir.stamp", ctx.Config().DeviceName(), partition) - fileListFile := fmt.Sprintf("target/product/%s/obj/PACKAGING/%s_intermediates/file_list.txt", ctx.Config().DeviceName(), partition) - stagingDir := fmt.Sprintf("target/product/%s/%s", ctx.Config().DeviceName(), partition) - - builder.Command().BuiltTool("merge_directories"). - Implicit(android.PathForArbitraryOutput(ctx, stampFile)). - Text("--ignore-duplicates"). - FlagWithInput("--file-list", android.PathForArbitraryOutput(ctx, fileListFile)). - Text(rootDir.String()). - Text(android.PathForArbitraryOutput(ctx, stagingDir).String()) -} - func (f *filesystem) buildEventLogtagsFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) { if !proptools.Bool(f.properties.Build_logtags) { return @@ -1020,8 +1074,21 @@ func (f *filesystem) SignedOutputPath() android.Path { // Note that "apex" module installs its contents to "apex"(fake partition) as well // for symbol lookup by imitating "activated" paths. func (f *filesystem) gatherFilteredPackagingSpecs(ctx android.ModuleContext) map[string]android.PackagingSpec { - specs := f.PackagingBase.GatherPackagingSpecsWithFilterAndModifier(ctx, f.filesystemBuilder.FilterPackagingSpec, f.filesystemBuilder.ModifyPackagingSpec) - return specs + return f.PackagingBase.GatherPackagingSpecsWithFilterAndModifier(ctx, f.filesystemBuilder.FilterPackagingSpec, f.filesystemBuilder.ModifyPackagingSpec) +} + +// Dexpreopt files are installed to system_other. Collect the packaingSpecs for the dexpreopt files +// from this partition to export to the system_other partition later. +func (f *filesystem) systemOtherFiles(ctx android.ModuleContext) map[string]android.PackagingSpec { + filter := func(spec android.PackagingSpec) bool { + // For some reason system_other packaging specs don't set the partition field. + return strings.HasPrefix(spec.RelPathInPackage(), "system_other/") + } + modifier := func(spec *android.PackagingSpec) { + spec.SetRelPathInPackage(strings.TrimPrefix(spec.RelPathInPackage(), "system_other/")) + spec.SetPartition("system_other") + } + return f.PackagingBase.GatherPackagingSpecsWithFilterAndModifier(ctx, filter, modifier) } func sha1sum(values []string) string { @@ -1066,6 +1133,9 @@ var _ partition = (*filesystemDefaults)(nil) func (f *filesystemDefaults) GenerateAndroidBuildActions(ctx android.ModuleContext) { validatePartitionType(ctx, f) + android.SetProvider(ctx, FilesystemDefaultsInfoProvider, FilesystemDefaultsInfo{ + PartitionType: f.PartitionType(), + }) } // getLibsForLinkerConfig returns @@ -1075,13 +1145,16 @@ func (f *filesystemDefaults) GenerateAndroidBuildActions(ctx android.ModuleConte // `linkerconfig.BuildLinkerConfig` will convert these two to a linker.config.pb for the filesystem // (1) will be added to --provideLibs if they are C libraries with a stable interface (has stubs) // (2) will be added to --requireLibs if they are C libraries with a stable interface (has stubs) -func (f *filesystem) getLibsForLinkerConfig(ctx android.ModuleContext) ([]android.Module, []android.Module) { +func (f *filesystem) getLibsForLinkerConfig(ctx android.ModuleContext) ([]android.ModuleProxy, []android.ModuleProxy) { // we need "Module"s for packaging items - modulesInPackageByModule := make(map[android.Module]bool) + modulesInPackageByModule := make(map[android.ModuleProxy]bool) modulesInPackageByName := make(map[string]bool) deps := f.gatherFilteredPackagingSpecs(ctx) - ctx.WalkDeps(func(child, parent android.Module) bool { + ctx.WalkDepsProxy(func(child, parent android.ModuleProxy) bool { + if !android.OtherModuleProviderOrDefault(ctx, child, android.CommonModuleInfoKey).Enabled { + return false + } for _, ps := range android.OtherModuleProviderOrDefault( ctx, child, android.InstallFilesProvider).PackagingSpecs { if _, ok := deps[ps.RelPathInPackage()]; ok && ps.Partition() == f.PartitionType() { @@ -1093,13 +1166,16 @@ func (f *filesystem) getLibsForLinkerConfig(ctx android.ModuleContext) ([]androi return true }) - provideModules := make([]android.Module, 0, len(modulesInPackageByModule)) + provideModules := make([]android.ModuleProxy, 0, len(modulesInPackageByModule)) for mod := range modulesInPackageByModule { provideModules = append(provideModules, mod) } - var requireModules []android.Module - ctx.WalkDeps(func(child, parent android.Module) bool { + var requireModules []android.ModuleProxy + ctx.WalkDepsProxy(func(child, parent android.ModuleProxy) bool { + if !android.OtherModuleProviderOrDefault(ctx, child, android.CommonModuleInfoKey).Enabled { + return false + } _, parentInPackage := modulesInPackageByModule[parent] _, childInPackageName := modulesInPackageByName[child.Name()] |