diff options
87 files changed, 2378 insertions, 891 deletions
diff --git a/Android.bp b/Android.bp index 1219c626a..434ee9f96 100644 --- a/Android.bp +++ b/Android.bp @@ -172,6 +172,9 @@ build_prop { name: "system-build.prop", stem: "build.prop", product_config: ":product_config", + footer_files: [ + ":applied_backported_fixes", + ], // Currently, only microdroid, Ravenwood, and cf system image can refer to system-build.prop visibility: [ "//build/make/target/product/generic", diff --git a/aconfig/codegen/java_aconfig_library.go b/aconfig/codegen/java_aconfig_library.go index ebca4134c..9f399bf76 100644 --- a/aconfig/codegen/java_aconfig_library.go +++ b/aconfig/codegen/java_aconfig_library.go @@ -72,7 +72,7 @@ func (callbacks *JavaAconfigDeclarationsLibraryCallbacks) DepsMutator(module *ja module.AddSharedLibrary("aconfig-annotations-lib") // TODO(b/303773055): Remove the annotation after access issue is resolved. module.AddSharedLibrary("unsupportedappusage") - module.AddSharedLibrary("aconfig_storage_reader_java") + module.AddSharedLibrary("aconfig_storage_stub") } } diff --git a/android/Android.bp b/android/Android.bp index a9a3564ab..dfea8f999 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -111,6 +111,7 @@ bootstrap_go_package { "testing.go", "util.go", "variable.go", + "vendor_api_levels.go", "vintf_fragment.go", "vintf_data.go", "visibility.go", diff --git a/android/apex.go b/android/apex.go index 9277ff31b..db9391204 100644 --- a/android/apex.go +++ b/android/apex.go @@ -16,7 +16,6 @@ package android import ( "fmt" - "reflect" "slices" "sort" "strconv" @@ -62,19 +61,6 @@ type ApexInfo struct { // that are merged together. InApexVariants []string - // List of APEX Soong module names that this module is part of. Note that the list includes - // different variations of the same APEX. For example, if module `foo` is included in the - // apex `com.android.foo`, and also if there is an override_apex module - // `com.mycompany.android.foo` overriding `com.android.foo`, then this list contains both - // `com.android.foo` and `com.mycompany.android.foo`. If the APEX Soong module is a - // prebuilt, the name here doesn't have the `prebuilt_` prefix. - InApexModules []string - - // Pointers to the ApexContents struct each of which is for apexBundle modules that this - // module is part of. The ApexContents gives information about which modules the apexBundle - // has and whether a module became part of the apexBundle via a direct dependency or not. - ApexContents []*ApexContents - // True if this is for a prebuilt_apex. // // If true then this will customize the apex processing to make it suitable for handling @@ -105,7 +91,6 @@ func (i ApexInfo) AddJSONData(d *map[string]interface{}) { (*d)["Apex"] = map[string]interface{}{ "ApexVariationName": i.ApexVariationName, "MinSdkVersion": i.MinSdkVersion, - "InApexModules": i.InApexModules, "InApexVariants": i.InApexVariants, "ForPrebuiltApex": i.ForPrebuiltApex, } @@ -140,15 +125,6 @@ func (i ApexInfo) InApexVariant(apexVariant string) bool { return false } -func (i ApexInfo) InApexModule(apexModuleName string) bool { - for _, a := range i.InApexModules { - if a == apexModuleName { - return true - } - } - return false -} - // To satisfy the comparable interface func (i ApexInfo) Equal(other any) bool { otherApexInfo, ok := other.(ApexInfo) @@ -156,13 +132,11 @@ func (i ApexInfo) Equal(other any) bool { i.MinSdkVersion == otherApexInfo.MinSdkVersion && i.Updatable == otherApexInfo.Updatable && i.UsePlatformApis == otherApexInfo.UsePlatformApis && - reflect.DeepEqual(i.InApexVariants, otherApexInfo.InApexVariants) && - reflect.DeepEqual(i.InApexModules, otherApexInfo.InApexModules) + slices.Equal(i.InApexVariants, otherApexInfo.InApexVariants) } // ApexBundleInfo contains information about the dependencies of an apex type ApexBundleInfo struct { - Contents *ApexContents } var ApexBundleInfoProvider = blueprint.NewMutatorProvider[ApexBundleInfo]("apex_info") @@ -220,15 +194,8 @@ type ApexModule interface { // this after apex.apexMutator is run. InAnyApex() bool - // Returns true if this module is directly in any APEX. Call this AFTER apex.apexMutator is - // run. - DirectlyInAnyApex() bool - - // NotInPlatform tells whether or not this module is included in an APEX and therefore - // shouldn't be exposed to the platform (i.e. outside of the APEX) directly. A module is - // considered to be included in an APEX either when there actually is an APEX that - // explicitly has the module as its dependency or the module is not available to the - // platform, which indicates that the module belongs to at least one or more other APEXes. + // NotInPlatform returns true if the module is not available to the platform due to + // apex_available being set and not containing "//apex_available:platform". NotInPlatform() bool // Tests if this module could have APEX variants. Even when a module type implements @@ -285,20 +252,6 @@ type ApexProperties struct { // Default is ["//apex_available:platform"]. Apex_available []string - // See ApexModule.InAnyApex() - InAnyApex bool `blueprint:"mutated"` - - // See ApexModule.DirectlyInAnyApex() - DirectlyInAnyApex bool `blueprint:"mutated"` - - // AnyVariantDirectlyInAnyApex is true in the primary variant of a module if _any_ variant - // of the module is directly in any apex. This includes host, arch, asan, etc. variants. It - // is unused in any variant that is not the primary variant. Ideally this wouldn't be used, - // as it incorrectly mixes arch variants if only one arch is in an apex, but a few places - // depend on it, for example when an ASAN variant is created before the apexMutator. Call - // this after apex.apexMutator is run. - AnyVariantDirectlyInAnyApex bool `blueprint:"mutated"` - // See ApexModule.NotAvailableForPlatform() NotAvailableForPlatform bool `blueprint:"mutated"` @@ -335,16 +288,6 @@ type AlwaysRequireApexVariantTag interface { AlwaysRequireApexVariant() bool } -// Marker interface that identifies dependencies that should inherit the DirectlyInAnyApex state -// from the parent to the child. For example, stubs libraries are marked as DirectlyInAnyApex if -// their implementation is in an apex. -type CopyDirectlyInAnyApexTag interface { - blueprint.DependencyTag - - // Method that differentiates this interface from others. - CopyDirectlyInAnyApex() -} - // Interface that identifies dependencies to skip Apex dependency check type SkipApexAllowedDependenciesCheck interface { // Returns true to skip the Apex dependency check, which limits the allowed dependency in build. @@ -395,40 +338,27 @@ func (m *ApexModuleBase) ApexAvailable() []string { func (m *ApexModuleBase) BuildForApex(apex ApexInfo) { m.apexInfosLock.Lock() defer m.apexInfosLock.Unlock() - for i, v := range m.apexInfos { - if v.ApexVariationName == apex.ApexVariationName { - if len(apex.InApexModules) != 1 { - panic(fmt.Errorf("Newly created apexInfo must be for a single APEX")) - } - // Even when the ApexVariantNames are the same, the given ApexInfo might - // actually be for different APEX. This can happen when an APEX is - // overridden via override_apex. For example, there can be two apexes - // `com.android.foo` (from the `apex` module type) and - // `com.mycompany.android.foo` (from the `override_apex` module type), both - // of which has the same ApexVariantName `com.android.foo`. Add the apex - // name to the list so that it's not lost. - if !InList(apex.InApexModules[0], v.InApexModules) { - m.apexInfos[i].InApexModules = append(m.apexInfos[i].InApexModules, apex.InApexModules[0]) - } - return - } + if slices.ContainsFunc(m.apexInfos, func(existing ApexInfo) bool { + return existing.ApexVariationName == apex.ApexVariationName + }) { + return } m.apexInfos = append(m.apexInfos, apex) } // Implements ApexModule func (m *ApexModuleBase) InAnyApex() bool { - return m.ApexProperties.InAnyApex -} - -// Implements ApexModule -func (m *ApexModuleBase) DirectlyInAnyApex() bool { - return m.ApexProperties.DirectlyInAnyApex + for _, apex_name := range m.ApexProperties.Apex_available { + if apex_name != AvailableToPlatform { + return true + } + } + return false } // Implements ApexModule func (m *ApexModuleBase) NotInPlatform() bool { - return m.ApexProperties.AnyVariantDirectlyInAnyApex || !m.AvailableFor(AvailableToPlatform) + return !m.AvailableFor(AvailableToPlatform) } // Implements ApexModule @@ -585,8 +515,6 @@ func mergeApexVariations(apexInfos []ApexInfo) (merged []ApexInfo, aliases [][2] if index, exists := seen[mergedName]; exists { // Variants having the same mergedName are deduped merged[index].InApexVariants = append(merged[index].InApexVariants, variantName) - merged[index].InApexModules = append(merged[index].InApexModules, apexInfo.InApexModules...) - merged[index].ApexContents = append(merged[index].ApexContents, apexInfo.ApexContents...) merged[index].Updatable = merged[index].Updatable || apexInfo.Updatable // Platform APIs is allowed for this module only when all APEXes containing // the module are with `use_platform_apis: true`. @@ -596,8 +524,6 @@ func mergeApexVariations(apexInfos []ApexInfo) (merged []ApexInfo, aliases [][2] seen[mergedName] = len(merged) apexInfo.ApexVariationName = mergedName apexInfo.InApexVariants = CopyOf(apexInfo.InApexVariants) - apexInfo.InApexModules = CopyOf(apexInfo.InApexModules) - apexInfo.ApexContents = append([]*ApexContents(nil), apexInfo.ApexContents...) apexInfo.TestApexes = CopyOf(apexInfo.TestApexes) merged = append(merged, apexInfo) } @@ -685,15 +611,6 @@ func MutateApexTransition(ctx BaseModuleContext, variation string) { apexInfos, _ = mergeApexVariations(apexInfos) } - var inApex ApexMembership - for _, a := range apexInfos { - for _, apexContents := range a.ApexContents { - inApex = inApex.merge(apexContents.contents[ctx.ModuleName()]) - } - } - base.ApexProperties.InAnyApex = true - base.ApexProperties.DirectlyInAnyApex = inApex == directlyInApex - if platformVariation && !ctx.Host() && !module.AvailableFor(AvailableToPlatform) && module.NotAvailableForPlatform() { // Do not install the module for platform, but still allow it to output // uninstallable AndroidMk entries in certain cases when they have side @@ -783,93 +700,6 @@ func UpdateUniqueApexVariationsForDeps(mctx BottomUpMutatorContext, am ApexModul }) } -// UpdateDirectlyInAnyApex uses the final module to store if any variant of this module is directly -// in any APEX, and then copies the final value to all the modules. It also copies the -// DirectlyInAnyApex value to any transitive dependencies with a CopyDirectlyInAnyApexTag -// dependency tag. -func UpdateDirectlyInAnyApex(mctx BottomUpMutatorContext, am ApexModule) { - base := am.apexModuleBase() - // Copy DirectlyInAnyApex and InAnyApex from any transitive dependencies with a - // CopyDirectlyInAnyApexTag dependency tag. - mctx.WalkDeps(func(child, parent Module) bool { - if _, ok := mctx.OtherModuleDependencyTag(child).(CopyDirectlyInAnyApexTag); ok { - depBase := child.(ApexModule).apexModuleBase() - depBase.apexPropertiesLock.Lock() - defer depBase.apexPropertiesLock.Unlock() - depBase.ApexProperties.DirectlyInAnyApex = base.ApexProperties.DirectlyInAnyApex - depBase.ApexProperties.InAnyApex = base.ApexProperties.InAnyApex - return true - } - return false - }) - - if base.ApexProperties.DirectlyInAnyApex { - // Variants of a module are always visited sequentially in order, so it is safe to - // write to another variant of this module. For a BottomUpMutator the - // PrimaryModule() is visited first and FinalModule() is visited last. - mctx.FinalModule().(ApexModule).apexModuleBase().ApexProperties.AnyVariantDirectlyInAnyApex = true - } - - // If this is the FinalModule (last visited module) copy - // AnyVariantDirectlyInAnyApex to all the other variants - if mctx.IsFinalModule(am) { - mctx.VisitAllModuleVariants(func(variant Module) { - variant.(ApexModule).apexModuleBase().ApexProperties.AnyVariantDirectlyInAnyApex = - base.ApexProperties.AnyVariantDirectlyInAnyApex - }) - } -} - -// ApexMembership tells how a module became part of an APEX. -type ApexMembership int - -const ( - notInApex ApexMembership = 0 - indirectlyInApex = iota - directlyInApex -) - -// ApexContents gives an information about member modules of an apexBundle. Each apexBundle has an -// apexContents, and modules in that apex have a provider containing the apexContents of each -// apexBundle they are part of. -type ApexContents struct { - // map from a module name to its membership in this apexBundle - contents map[string]ApexMembership -} - -// NewApexContents creates and initializes an ApexContents that is suitable -// for use with an apex module. -// - contents is a map from a module name to information about its membership within -// the apex. -func NewApexContents(contents map[string]ApexMembership) *ApexContents { - return &ApexContents{ - contents: contents, - } -} - -// Updates an existing membership by adding a new direct (or indirect) membership -func (i ApexMembership) Add(direct bool) ApexMembership { - if direct || i == directlyInApex { - return directlyInApex - } - return indirectlyInApex -} - -// Merges two membership into one. Merging is needed because a module can be a part of an apexBundle -// in many different paths. For example, it could be dependend on by the apexBundle directly, but at -// the same time, there might be an indirect dependency to the module. In that case, the more -// specific dependency (the direct one) is chosen. -func (i ApexMembership) merge(other ApexMembership) ApexMembership { - if other == directlyInApex || i == directlyInApex { - return directlyInApex - } - - if other == indirectlyInApex || i == indirectlyInApex { - return indirectlyInApex - } - return notInApex -} - //////////////////////////////////////////////////////////////////////////////////////////////////// //Below are routines for extra safety checks. // diff --git a/android/apex_test.go b/android/apex_test.go index 347bf7d98..78597b234 100644 --- a/android/apex_test.go +++ b/android/apex_test.go @@ -37,7 +37,6 @@ func Test_mergeApexVariations(t *testing.T) { ApexVariationName: "foo", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"foo"}, - InApexModules: []string{"foo"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, @@ -46,7 +45,6 @@ func Test_mergeApexVariations(t *testing.T) { ApexVariationName: "apex10000", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"foo"}, - InApexModules: []string{"foo"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, @@ -61,14 +59,12 @@ func Test_mergeApexVariations(t *testing.T) { ApexVariationName: "foo", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"foo"}, - InApexModules: []string{"foo"}, ForPrebuiltApex: NotForPrebuiltApex, }, { ApexVariationName: "bar", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"bar"}, - InApexModules: []string{"bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, @@ -77,7 +73,6 @@ func Test_mergeApexVariations(t *testing.T) { ApexVariationName: "apex10000", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"foo", "bar"}, - InApexModules: []string{"foo", "bar"}, }}, wantAliases: [][2]string{ {"foo", "apex10000"}, @@ -91,14 +86,12 @@ func Test_mergeApexVariations(t *testing.T) { ApexVariationName: "foo", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"foo"}, - InApexModules: []string{"foo"}, ForPrebuiltApex: NotForPrebuiltApex, }, { ApexVariationName: "bar", MinSdkVersion: uncheckedFinalApiLevel(30), InApexVariants: []string{"bar"}, - InApexModules: []string{"bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, @@ -107,14 +100,12 @@ func Test_mergeApexVariations(t *testing.T) { ApexVariationName: "apex10000", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"foo"}, - InApexModules: []string{"foo"}, ForPrebuiltApex: NotForPrebuiltApex, }, { ApexVariationName: "apex30", MinSdkVersion: uncheckedFinalApiLevel(30), InApexVariants: []string{"bar"}, - InApexModules: []string{"bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, @@ -130,7 +121,6 @@ func Test_mergeApexVariations(t *testing.T) { ApexVariationName: "foo", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"foo"}, - InApexModules: []string{"foo"}, ForPrebuiltApex: NotForPrebuiltApex, }, { @@ -138,7 +128,6 @@ func Test_mergeApexVariations(t *testing.T) { MinSdkVersion: FutureApiLevel, Updatable: true, InApexVariants: []string{"bar"}, - InApexModules: []string{"bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, @@ -148,7 +137,6 @@ func Test_mergeApexVariations(t *testing.T) { MinSdkVersion: FutureApiLevel, Updatable: true, InApexVariants: []string{"foo", "bar"}, - InApexModules: []string{"foo", "bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, @@ -164,7 +152,6 @@ func Test_mergeApexVariations(t *testing.T) { ApexVariationName: "foo", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"foo"}, - InApexModules: []string{"foo"}, ForPrebuiltApex: NotForPrebuiltApex, }, { @@ -172,7 +159,6 @@ func Test_mergeApexVariations(t *testing.T) { MinSdkVersion: FutureApiLevel, Updatable: true, InApexVariants: []string{"bar"}, - InApexModules: []string{"bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, // This one should not be merged in with the others because it is for @@ -182,7 +168,6 @@ func Test_mergeApexVariations(t *testing.T) { MinSdkVersion: FutureApiLevel, Updatable: true, InApexVariants: []string{"baz"}, - InApexModules: []string{"baz"}, ForPrebuiltApex: ForPrebuiltApex, }, }, @@ -192,7 +177,6 @@ func Test_mergeApexVariations(t *testing.T) { MinSdkVersion: FutureApiLevel, Updatable: true, InApexVariants: []string{"foo", "bar"}, - InApexModules: []string{"foo", "bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, { @@ -200,7 +184,6 @@ func Test_mergeApexVariations(t *testing.T) { MinSdkVersion: FutureApiLevel, Updatable: true, InApexVariants: []string{"baz"}, - InApexModules: []string{"baz"}, ForPrebuiltApex: ForPrebuiltApex, }, }, @@ -216,7 +199,6 @@ func Test_mergeApexVariations(t *testing.T) { ApexVariationName: "foo", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"foo"}, - InApexModules: []string{"foo"}, ForPrebuiltApex: NotForPrebuiltApex, }, { @@ -224,7 +206,6 @@ func Test_mergeApexVariations(t *testing.T) { MinSdkVersion: FutureApiLevel, UsePlatformApis: true, InApexVariants: []string{"bar"}, - InApexModules: []string{"bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, @@ -233,7 +214,6 @@ func Test_mergeApexVariations(t *testing.T) { ApexVariationName: "apex10000", MinSdkVersion: FutureApiLevel, InApexVariants: []string{"foo", "bar"}, - InApexModules: []string{"foo", "bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, @@ -250,7 +230,6 @@ func Test_mergeApexVariations(t *testing.T) { MinSdkVersion: FutureApiLevel, UsePlatformApis: true, InApexVariants: []string{"foo"}, - InApexModules: []string{"foo"}, ForPrebuiltApex: NotForPrebuiltApex, }, { @@ -258,7 +237,6 @@ func Test_mergeApexVariations(t *testing.T) { MinSdkVersion: FutureApiLevel, UsePlatformApis: true, InApexVariants: []string{"bar"}, - InApexModules: []string{"bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, @@ -268,7 +246,6 @@ func Test_mergeApexVariations(t *testing.T) { MinSdkVersion: FutureApiLevel, UsePlatformApis: true, InApexVariants: []string{"foo", "bar"}, - InApexModules: []string{"foo", "bar"}, ForPrebuiltApex: NotForPrebuiltApex, }, }, diff --git a/android/build_prop.go b/android/build_prop.go index cda56f1e0..2f71bc03f 100644 --- a/android/build_prop.go +++ b/android/build_prop.go @@ -15,6 +15,8 @@ package android import ( + "fmt" + "github.com/google/blueprint/proptools" ) @@ -173,7 +175,16 @@ func (p *buildPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { postProcessCmd.Text(outputFilePath.String()) postProcessCmd.Flags(p.properties.Block_list) - rule.Command().Text("echo").Text(proptools.NinjaAndShellEscape("# end of file")).FlagWithArg(">> ", outputFilePath.String()) + for _, footer := range p.properties.Footer_files { + path := PathForModuleSrc(ctx, footer) + rule.appendText(outputFilePath, "####################################") + rule.appendTextf(outputFilePath, "# Adding footer from %v", footer) + rule.appendTextf(outputFilePath, "# with path %v", path) + rule.appendText(outputFilePath, "####################################") + rule.Command().Text("cat").FlagWithInput("", path).FlagWithArg(">> ", outputFilePath.String()) + } + + rule.appendText(outputFilePath, "# end of file") rule.Build(ctx.ModuleName(), "generating build.prop") @@ -184,6 +195,14 @@ func (p *buildPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { p.outputFilePath = outputFilePath } +func (r *RuleBuilder) appendText(path ModuleOutPath, text string) { + r.Command().Text("echo").Text(proptools.NinjaAndShellEscape(text)).FlagWithArg(">> ", path.String()) +} + +func (r *RuleBuilder) appendTextf(path ModuleOutPath, format string, a ...any) { + r.appendText(path, fmt.Sprintf(format, a...)) +} + func (p *buildPropModule) AndroidMkEntries() []AndroidMkEntries { return []AndroidMkEntries{{ Class: "ETC", diff --git a/android/config.go b/android/config.go index 27d3b87ba..dff3ea5bd 100644 --- a/android/config.go +++ b/android/config.go @@ -285,6 +285,10 @@ func (c Config) ReleaseCreateAconfigStorageFile() bool { return c.config.productVariables.GetBuildFlagBool("RELEASE_CREATE_ACONFIG_STORAGE_FILE") } +func (c Config) ReleaseUseSystemFeatureBuildFlags() bool { + return c.config.productVariables.GetBuildFlagBool("RELEASE_USE_SYSTEM_FEATURE_BUILD_FLAGS") +} + // A DeviceConfig object represents the configuration for a particular device // being built. For now there will only be one of these, but in the future there // may be multiple devices being built. @@ -1834,6 +1838,10 @@ func (c *config) ApexCompressionEnabled() bool { return Bool(c.productVariables.CompressedApex) && !c.UnbundledBuildApps() } +func (c *config) DefaultApexPayloadType() string { + return StringDefault(c.productVariables.DefaultApexPayloadType, "ext4") +} + func (c *config) UseSoongSystemImage() bool { return Bool(c.productVariables.UseSoongSystemImage) } diff --git a/android/container.go b/android/container.go index 2a3777b30..27b17ed99 100644 --- a/android/container.go +++ b/android/container.go @@ -93,7 +93,7 @@ var globallyAllowlistedDependencies = []string{ // TODO(b/363016634): Remove from the allowlist when the module is converted // to java_sdk_library and the java_aconfig_library modules depend on the stub. - "aconfig_storage_reader_java", + "aconfig_storage_stub", // framework-res provides core resources essential for building apps and system UI. // This module is implicitly added as a dependency for java modules even when the @@ -382,7 +382,7 @@ func (c *ContainersInfo) BelongingContainers() []*container { func (c *ContainersInfo) ApexNames() (ret []string) { for _, apex := range c.belongingApexes { - ret = append(ret, apex.InApexModules...) + ret = append(ret, apex.InApexVariants...) } slices.Sort(ret) return ret diff --git a/android/module.go b/android/module.go index ce995ad77..3bf4f0c9e 100644 --- a/android/module.go +++ b/android/module.go @@ -2450,6 +2450,8 @@ func (e configurationEvalutor) EvaluateConfiguration(condition proptools.Configu return proptools.ConfigurableValueString(v) case "bool": return proptools.ConfigurableValueBool(v == "true") + case "string_list": + return proptools.ConfigurableValueStringList(strings.Split(v, " ")) default: panic("unhandled soong config variable type: " + ty) } diff --git a/android/module_context.go b/android/module_context.go index 20149074e..ae7b54f66 100644 --- a/android/module_context.go +++ b/android/module_context.go @@ -831,6 +831,11 @@ func (m *moduleContext) ModuleInfoJSON() *ModuleInfoJSON { } func (m *moduleContext) SetOutputFiles(outputFiles Paths, tag string) { + for _, outputFile := range outputFiles { + if outputFile == nil { + panic("outputfiles cannot be nil") + } + } if tag == "" { if len(m.outputFiles.DefaultOutputFiles) > 0 { m.ModuleErrorf("Module %s default OutputFiles cannot be overwritten", m.ModuleName()) diff --git a/android/neverallow.go b/android/neverallow.go index 326150be5..22155043b 100644 --- a/android/neverallow.go +++ b/android/neverallow.go @@ -55,7 +55,7 @@ func init() { AddNeverAllowRules(createJavaDeviceForHostRules()...) AddNeverAllowRules(createCcSdkVariantRules()...) AddNeverAllowRules(createUncompressDexRules()...) - AddNeverAllowRules(createInitFirstStageRules()...) + AddNeverAllowRules(createInstallInRootAllowingRules()...) AddNeverAllowRules(createProhibitFrameworkAccessRules()...) AddNeverAllowRules(createCcStubsRule()) AddNeverAllowRules(createProhibitHeaderOnlyRule()) @@ -235,15 +235,16 @@ func createUncompressDexRules() []Rule { } } -func createInitFirstStageRules() []Rule { +func createInstallInRootAllowingRules() []Rule { return []Rule{ NeverAllow(). Without("name", "init_first_stage_defaults"). Without("name", "init_first_stage"). Without("name", "init_first_stage.microdroid"). + Without("name", "librecovery_ui_ext"). With("install_in_root", "true"). NotModuleType("prebuilt_root"). - Because("install_in_root is only for init_first_stage."), + Because("install_in_root is only for init_first_stage or librecovery_ui_ext."), } } diff --git a/android/packaging.go b/android/packaging.go index e71d983eb..dcd8844f0 100644 --- a/android/packaging.go +++ b/android/packaging.go @@ -410,9 +410,9 @@ func (PackagingItemAlwaysDepTag) IsPackagingItem() bool { return true } -// highPriorityDepTag provides default implementation of HighPriorityPackagingItem interface. type highPriorityDepTag struct { - blueprint.DependencyTag + blueprint.BaseDependencyTag + PackagingItemAlwaysDepTag } // See PackageModule.AddDeps @@ -433,7 +433,7 @@ func (p *PackagingBase) AddDeps(ctx BottomUpMutatorContext, depTag blueprint.Dep } depTagToUse := depTag if highPriority { - depTagToUse = highPriorityDepTag{depTag} + depTagToUse = highPriorityDepTag{} } ctx.AddFarVariationDependencies(targetVariation, depTagToUse, dep) diff --git a/android/prebuilt_test.go b/android/prebuilt_test.go index 5e4af0ba5..b90ef3b1c 100644 --- a/android/prebuilt_test.go +++ b/android/prebuilt_test.go @@ -508,11 +508,10 @@ func (p *prebuiltModule) Name() string { } func (p *prebuiltModule) GenerateAndroidBuildActions(ctx ModuleContext) { - var src Path if len(p.properties.Srcs) >= 1 { - src = p.prebuilt.SingleSourcePath(ctx) + src := p.prebuilt.SingleSourcePath(ctx) + ctx.SetOutputFiles(Paths{src}, "") } - ctx.SetOutputFiles(Paths{src}, "") } func (p *prebuiltModule) Prebuilt() *Prebuilt { diff --git a/android/rule_builder.go b/android/rule_builder.go index 403c18418..83f8b9992 100644 --- a/android/rule_builder.go +++ b/android/rule_builder.go @@ -488,21 +488,15 @@ func (r *RuleBuilder) depFileMergerCmd(depFiles WritablePaths) *RuleBuilderComma Inputs(depFiles.Paths()) } -// BuildWithNinjaVars adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for -// Outputs. This function will not escape Ninja variables, so it may be used to write sandbox manifests using Ninja variables. -func (r *RuleBuilder) BuildWithUnescapedNinjaVars(name string, desc string) { - r.build(name, desc, false) -} - // Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for // Outputs. func (r *RuleBuilder) Build(name string, desc string) { - r.build(name, desc, true) + r.build(name, desc) } var sandboxEnvOnceKey = NewOnceKey("sandbox_environment_variables") -func (r *RuleBuilder) build(name string, desc string, ninjaEscapeCommandString bool) { +func (r *RuleBuilder) build(name string, desc string) { name = ninjaNameEscape(name) if len(r.missingDeps) > 0 { @@ -611,6 +605,7 @@ func (r *RuleBuilder) build(name string, desc string, ninjaEscapeCommandString b nsjailCmd.WriteString(" -m none:/tmp:tmpfs:size=1073741824") // 1GB, should be enough nsjailCmd.WriteString(" -D nsjail_build_sandbox") nsjailCmd.WriteString(" --disable_rlimits") + nsjailCmd.WriteString(" --skip_setsid") // ABFS relies on process-groups to track file operations nsjailCmd.WriteString(" -q") nsjailCmd.WriteString(" -- ") nsjailCmd.WriteString("/bin/bash -c ") @@ -764,30 +759,7 @@ func (r *RuleBuilder) build(name string, desc string, ninjaEscapeCommandString b if err != nil { ReportPathErrorf(r.ctx, "sbox manifest failed to marshal: %q", err) } - if ninjaEscapeCommandString { - WriteFileRule(r.ctx, r.sboxManifestPath, string(pbText)) - } else { - // We need to have a rule to write files that is - // defined on the RuleBuilder's pctx in order to - // write Ninja variables in the string. - // The WriteFileRule function above rule can only write - // raw strings because it is defined on the android - // package's pctx, and it can't access variables defined - // in another context. - r.ctx.Build(r.pctx, BuildParams{ - Rule: r.ctx.Rule(r.pctx, "unescapedWriteFile", blueprint.RuleParams{ - Command: `rm -rf ${out} && cat ${out}.rsp > ${out}`, - Rspfile: "${out}.rsp", - RspfileContent: "${content}", - Description: "write file", - }, "content"), - Output: r.sboxManifestPath, - Description: "write sbox manifest " + r.sboxManifestPath.Base(), - Args: map[string]string{ - "content": string(pbText), - }, - }) - } + WriteFileRule(r.ctx, r.sboxManifestPath, string(pbText)) // Generate a new string to use as the command line of the sbox rule. This uses // a RuleBuilderCommand as a convenience method of building the command line, then @@ -881,9 +853,7 @@ func (r *RuleBuilder) build(name string, desc string, ninjaEscapeCommandString b pool = localPool } - if ninjaEscapeCommandString { - commandString = proptools.NinjaEscape(commandString) - } + commandString = proptools.NinjaEscape(commandString) args_vars := make([]string, len(r.args)) i := 0 diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go index 6a8a964a1..e1a1e08c4 100644 --- a/android/rule_builder_test.go +++ b/android/rule_builder_test.go @@ -475,10 +475,9 @@ type testRuleBuilderModule struct { Srcs []string Flags []string - Restat bool - Sbox bool - Sbox_inputs bool - Unescape_ninja_vars bool + Restat bool + Sbox bool + Sbox_inputs bool } } @@ -498,7 +497,7 @@ func (t *testRuleBuilderModule) GenerateAndroidBuildActions(ctx ModuleContext) { testRuleBuilder_Build(ctx, in, implicit, orderOnly, validation, t.properties.Flags, out, outDep, outDir, - manifestPath, t.properties.Restat, t.properties.Sbox, t.properties.Sbox_inputs, t.properties.Unescape_ninja_vars, + manifestPath, t.properties.Restat, t.properties.Sbox, t.properties.Sbox_inputs, rspFile, rspFileContents, rspFile2, rspFileContents2) } @@ -523,14 +522,14 @@ func (t *testRuleBuilderSingleton) GenerateBuildActions(ctx SingletonContext) { manifestPath := PathForOutput(ctx, "singleton/sbox.textproto") testRuleBuilder_Build(ctx, in, implicit, orderOnly, validation, nil, out, outDep, outDir, - manifestPath, true, false, false, false, + manifestPath, true, false, false, rspFile, rspFileContents, rspFile2, rspFileContents2) } func testRuleBuilder_Build(ctx BuilderContext, in Paths, implicit, orderOnly, validation Path, flags []string, out, outDep, outDir, manifestPath WritablePath, - restat, sbox, sboxInputs, unescapeNinjaVars bool, + restat, sbox, sboxInputs bool, rspFile WritablePath, rspFileContents Paths, rspFile2 WritablePath, rspFileContents2 Paths) { rule := NewRuleBuilder(pctx_ruleBuilderTest, ctx) @@ -558,11 +557,7 @@ func testRuleBuilder_Build(ctx BuilderContext, in Paths, implicit, orderOnly, va rule.Restat() } - if unescapeNinjaVars { - rule.BuildWithUnescapedNinjaVars("rule", "desc") - } else { - rule.Build("rule", "desc") - } + rule.Build("rule", "desc") } var prepareForRuleBuilderTest = FixtureRegisterWithContext(func(ctx RegistrationContext) { @@ -777,48 +772,3 @@ func TestRuleBuilderHashInputs(t *testing.T) { }) } } - -func TestRuleBuilderWithNinjaVarEscaping(t *testing.T) { - bp := ` - rule_builder_test { - name: "foo_sbox_escaped", - flags: ["${cmdFlags}"], - sbox: true, - sbox_inputs: true, - } - rule_builder_test { - name: "foo_sbox_unescaped", - flags: ["${cmdFlags}"], - sbox: true, - sbox_inputs: true, - unescape_ninja_vars: true, - } - ` - result := GroupFixturePreparers( - prepareForRuleBuilderTest, - FixtureWithRootAndroidBp(bp), - ).RunTest(t) - - escapedNinjaMod := result.ModuleForTests("foo_sbox_escaped", "").Output("sbox.textproto") - AssertStringEquals(t, "expected rule", "android/soong/android.rawFileCopy", escapedNinjaMod.Rule.String()) - AssertStringDoesContain( - t, - "", - ContentFromFileRuleForTests(t, result.TestContext, escapedNinjaMod), - "${cmdFlags}", - ) - - unescapedNinjaMod := result.ModuleForTests("foo_sbox_unescaped", "").Rule("unescapedWriteFile") - AssertStringDoesContain( - t, - "", - unescapedNinjaMod.BuildParams.Args["content"], - "${cmdFlags}", - ) - AssertStringDoesNotContain( - t, - "", - unescapedNinjaMod.BuildParams.Args["content"], - "$${cmdFlags}", - ) -} diff --git a/android/sbom.go b/android/sbom.go index 2a5499ed8..f2b9c0ff1 100644 --- a/android/sbom.go +++ b/android/sbom.go @@ -15,9 +15,7 @@ package android import ( - "io" "path/filepath" - "strings" "github.com/google/blueprint" ) @@ -55,21 +53,7 @@ func (this *sbomSingleton) GenerateBuildActions(ctx SingletonContext) { if !ctx.Config().HasDeviceProduct() { return } - // Get all METADATA files and add them as implicit input - metadataFileListFile := PathForArbitraryOutput(ctx, ".module_paths", "METADATA.list") - f, err := ctx.Config().fs.Open(metadataFileListFile.String()) - if err != nil { - panic(err) - } - b, err := io.ReadAll(f) - if err != nil { - panic(err) - } - allMetadataFiles := strings.Split(string(b), "\n") - implicits := []Path{metadataFileListFile} - for _, path := range allMetadataFiles { - implicits = append(implicits, PathForSource(ctx, path)) - } + implicits := []Path{} prodVars := ctx.Config().productVariables buildFingerprintFile := PathForArbitraryOutput(ctx, "target", "product", String(prodVars.DeviceName), "build_fingerprint.txt") implicits = append(implicits, buildFingerprintFile) diff --git a/android/selects_test.go b/android/selects_test.go index 90d7091e0..1397ed8b7 100644 --- a/android/selects_test.go +++ b/android/selects_test.go @@ -1031,6 +1031,54 @@ my_module_type { my_string_list: &[]string{"d2", "e2", "f2", "a1", "b1", "c1"}, }, }, + { + name: "string list variables", + bp: ` +my_module_type { + name: "foo", + my_string_list: ["a"] + select(soong_config_variable("my_namespace", "my_var"), { + any @ my_var: my_var, + default: [], + }), +} +`, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_var": "b c", + }, + }, + vendorVarTypes: map[string]map[string]string{ + "my_namespace": { + "my_var": "string_list", + }, + }, + provider: selectsTestProvider{ + my_string_list: &[]string{"a", "b", "c"}, + }, + }, + { + name: "string list variables don't match string matchers", + bp: ` +my_module_type { + name: "foo", + my_string_list: ["a"] + select(soong_config_variable("my_namespace", "my_var"), { + "foo": ["b"], + default: [], + }), +} +`, + vendorVars: map[string]map[string]string{ + "my_namespace": { + "my_var": "b c", + }, + }, + vendorVarTypes: map[string]map[string]string{ + "my_namespace": { + "my_var": "string_list", + }, + }, + expectedError: `Expected all branches of a select on condition soong_config_variable\("my_namespace", "my_var"\) to have type string_list, found string`, + }, } for _, tc := range testCases { diff --git a/android/test_config.go b/android/test_config.go index f2510387f..3609e6b78 100644 --- a/android/test_config.go +++ b/android/test_config.go @@ -45,6 +45,7 @@ func TestConfig(buildDir string, env map[string]string, bp string, fs map[string Platform_version_active_codenames: []string{"S", "Tiramisu"}, DeviceSystemSdkVersions: []string{"29", "30", "S"}, Platform_systemsdk_versions: []string{"29", "30", "S", "Tiramisu"}, + VendorApiLevel: stringPtr("202404"), AAPTConfig: []string{"normal", "large", "xlarge", "hdpi", "xhdpi", "xxhdpi"}, AAPTPreferredConfig: stringPtr("xhdpi"), AAPTCharacteristics: stringPtr("nosdcard"), diff --git a/android/variable.go b/android/variable.go index 2d43c6da4..36ddc1c81 100644 --- a/android/variable.go +++ b/android/variable.go @@ -408,9 +408,10 @@ type ProductVariables struct { Ndk_abis *bool `json:",omitempty"` - ForceApexSymlinkOptimization *bool `json:",omitempty"` - CompressedApex *bool `json:",omitempty"` - Aml_abis *bool `json:",omitempty"` + ForceApexSymlinkOptimization *bool `json:",omitempty"` + CompressedApex *bool `json:",omitempty"` + DefaultApexPayloadType *string `json:",omitempty"` + Aml_abis *bool `json:",omitempty"` DexpreoptGlobalConfig *string `json:",omitempty"` @@ -609,15 +610,25 @@ type PartitionVariables struct { ProductUseDynamicPartitionSize bool `json:",omitempty"` CopyImagesForTargetFilesZip bool `json:",omitempty"` + VendorSecurityPatch string `json:",omitempty"` + // Boot image stuff - ProductBuildBootImage bool `json:",omitempty"` - ProductBuildInitBootImage bool `json:",omitempty"` - BoardUsesRecoveryAsBoot bool `json:",omitempty"` - BoardPrebuiltBootimage string `json:",omitempty"` - BoardPrebuiltInitBootimage string `json:",omitempty"` - BoardBootimagePartitionSize string `json:",omitempty"` - BoardInitBootimagePartitionSize string `json:",omitempty"` - BoardBootHeaderVersion string `json:",omitempty"` + BuildingRamdiskImage bool `json:",omitempty"` + ProductBuildBootImage bool `json:",omitempty"` + ProductBuildVendorBootImage string `json:",omitempty"` + ProductBuildInitBootImage bool `json:",omitempty"` + BoardUsesRecoveryAsBoot bool `json:",omitempty"` + BoardPrebuiltBootimage string `json:",omitempty"` + BoardPrebuiltInitBootimage string `json:",omitempty"` + BoardBootimagePartitionSize string `json:",omitempty"` + BoardInitBootimagePartitionSize string `json:",omitempty"` + BoardBootHeaderVersion string `json:",omitempty"` + TargetKernelPath string `json:",omitempty"` + BoardUsesGenericKernelImage bool `json:",omitempty"` + BootSecurityPatch string `json:",omitempty"` + InitBootSecurityPatch string `json:",omitempty"` + BoardIncludeDtbInBootimg bool `json:",omitempty"` + InternalKernelCmdline []string `json:",omitempty"` // Avb (android verified boot) stuff BoardAvbEnable bool `json:",omitempty"` diff --git a/android/vendor_api_levels.go b/android/vendor_api_levels.go new file mode 100644 index 000000000..4d364fde6 --- /dev/null +++ b/android/vendor_api_levels.go @@ -0,0 +1,49 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "strconv" +) + +func getSdkVersionOfVendorApiLevel(apiLevel int) (int, bool) { + ok := true + sdkVersion := -1 + switch apiLevel { + case 202404: + sdkVersion = 35 + case 202504: + sdkVersion = 36 + default: + ok = false + } + return sdkVersion, ok +} + +func GetSdkVersionForVendorApiLevel(vendorApiLevel string) (ApiLevel, error) { + vendorApiLevelInt, err := strconv.Atoi(vendorApiLevel) + if err != nil { + return NoneApiLevel, fmt.Errorf("The vendor API level %q must be able to be parsed as an integer", vendorApiLevel) + } + if vendorApiLevelInt < 35 { + return uncheckedFinalApiLevel(vendorApiLevelInt), nil + } + + if sdkInt, ok := getSdkVersionOfVendorApiLevel(vendorApiLevelInt); ok { + return uncheckedFinalApiLevel(sdkInt), nil + } + return NoneApiLevel, fmt.Errorf("Unknown vendor API level %q. Requires updating the map in vendor_api_level.go?", vendorApiLevel) +} diff --git a/apex/aconfig_test.go b/apex/aconfig_test.go index 76227a9ab..0eb8ef479 100644 --- a/apex/aconfig_test.go +++ b/apex/aconfig_test.go @@ -60,6 +60,7 @@ func TestValidationAcrossContainersExportedPass(t *testing.T) { apex_available: [ "myapex", ], + compile_dex: true, } aconfig_declarations { name: "my_aconfig_declarations_foo", @@ -339,6 +340,7 @@ func TestValidationAcrossContainersNotExportedFail(t *testing.T) { apex_available: [ "myapex", ], + compile_dex: true, } aconfig_declarations { name: "my_aconfig_declarations_foo", @@ -761,6 +763,7 @@ func TestValidationNotPropagateAcrossShared(t *testing.T) { apex_available: [ "myapex", ], + compile_dex: true, } java_library { name: "my_java_library_foo", diff --git a/apex/apex.go b/apex/apex.go index dc24df3d1..0e40d7c0e 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -67,7 +67,6 @@ func RegisterPostDepsMutators(ctx android.RegisterMutatorsContext) { // it should create a platform variant. ctx.BottomUp("mark_platform_availability", markPlatformAvailability) ctx.Transition("apex", &apexTransitionMutator{}) - ctx.BottomUp("apex_directly_in_any", apexDirectlyInAnyMutator).MutatesDependencies() } type apexBundleProperties struct { @@ -435,6 +434,7 @@ type apexBundle struct { archProperties apexArchBundleProperties overridableProperties overridableProperties vndkProperties apexVndkProperties // only for apex_vndk modules + testProperties apexTestProperties // only for apex_test modules /////////////////////////////////////////////////////////////////////////////////////////// // Inputs @@ -993,25 +993,7 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { return true } - // Records whether a certain module is included in this apexBundle via direct dependency or - // inndirect dependency. - contents := make(map[string]android.ApexMembership) - mctx.WalkDeps(func(child, parent android.Module) bool { - if !continueApexDepsWalk(child, parent) { - return false - } - // If the parent is apexBundle, this child is directly depended. - _, directDep := parent.(*apexBundle) - depName := mctx.OtherModuleName(child) - contents[depName] = contents[depName].Add(directDep) - return true - }) - - // The membership information is saved for later access - apexContents := android.NewApexContents(contents) - android.SetProvider(mctx, android.ApexBundleInfoProvider, android.ApexBundleInfo{ - Contents: apexContents, - }) + android.SetProvider(mctx, android.ApexBundleInfoProvider, android.ApexBundleInfo{}) minSdkVersion := a.minSdkVersion(mctx) // When min_sdk_version is not set, the apex is built against FutureApiLevel. @@ -1039,8 +1021,6 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { Updatable: a.Updatable(), UsePlatformApis: a.UsePlatformApis(), InApexVariants: []string{apexVariationName}, - InApexModules: []string{a.Name()}, // could be com.mycompany.android.foo - ApexContents: []*android.ApexContents{apexContents}, TestApexes: testApexes, BaseApexName: mctx.ModuleName(), ApexAvailableName: proptools.String(a.properties.Apex_available_name), @@ -1242,14 +1222,6 @@ func apexModuleTypeRequiresVariant(module ApexInfoMutator) bool { return true } -// See android.UpdateDirectlyInAnyApex -// TODO(jiyong): move this to android/apex.go? -func apexDirectlyInAnyMutator(mctx android.BottomUpMutatorContext) { - if am, ok := mctx.Module().(android.ApexModule); ok { - android.UpdateDirectlyInAnyApex(mctx, am) - } -} - const ( // File extensions of an APEX for different packaging methods imageApexSuffix = ".apex" @@ -1325,6 +1297,23 @@ func (a *apexBundle) UsePlatformApis() bool { return proptools.BoolDefault(a.properties.Platform_apis, false) } +type apexValidationType int + +const ( + hostApexVerifier apexValidationType = iota + apexSepolicyTests +) + +func (a *apexBundle) skipValidation(validationType apexValidationType) bool { + switch validationType { + case hostApexVerifier: + return proptools.Bool(a.testProperties.Skip_validations.Host_apex_verifier) + case apexSepolicyTests: + return proptools.Bool(a.testProperties.Skip_validations.Apex_sepolicy_tests) + } + panic("Unknown validation type") +} + // getCertString returns the name of the cert that should be used to sign this APEX. This is // basically from the "certificate" property, but could be overridden by the device config. func (a *apexBundle) getCertString(ctx android.BaseModuleContext) string { @@ -1752,7 +1741,8 @@ func (a *apexBundle) setSystemLibLink(ctx android.ModuleContext) { } func (a *apexBundle) setPayloadFsType(ctx android.ModuleContext) { - switch proptools.StringDefault(a.properties.Payload_fs_type, ext4FsType) { + defaultFsType := ctx.Config().DefaultApexPayloadType() + switch proptools.StringDefault(a.properties.Payload_fs_type, defaultFsType) { case ext4FsType: a.payloadFsType = ext4 case f2fsFsType: @@ -2082,7 +2072,7 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, // // Skip the dependency in unbundled builds where the device image is not // being built. - if ch.IsStubsImplementationRequired() && !am.DirectlyInAnyApex() && !ctx.Config().UnbundledBuild() { + if ch.IsStubsImplementationRequired() && !am.NotInPlatform() && !ctx.Config().UnbundledBuild() { // we need a module name for Make name := ch.ImplementationModuleNameForMake(ctx) + ch.Properties.SubName if !android.InList(name, a.makeModulesToInstall) { @@ -2179,8 +2169,6 @@ func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, ctx.PropertyErrorf("systemserverclasspath_fragments", "systemserverclasspath_fragment content %q of type %q is not supported", depName, ctx.OtherModuleType(child)) } - } else if _, ok := depTag.(android.CopyDirectlyInAnyApexTag); ok { - // nothing } else if depTag == android.DarwinUniversalVariantTag { // nothing } else if depTag == android.RequiredDepTag { @@ -2457,10 +2445,14 @@ func newApexBundle() *apexBundle { return module } -func ApexBundleFactory(testApex bool) android.Module { - bundle := newApexBundle() - bundle.testApex = testApex - return bundle +type apexTestProperties struct { + // Boolean flags for validation checks. Test APEXes can turn on/off individual checks. + Skip_validations struct { + // Skips `Apex_sepolicy_tests` check if true + Apex_sepolicy_tests *bool + // Skips `Host_apex_verifier` check if true + Host_apex_verifier *bool + } } // apex_test is an APEX for testing. The difference from the ordinary apex module type is that @@ -2468,6 +2460,7 @@ func ApexBundleFactory(testApex bool) android.Module { func TestApexBundleFactory() android.Module { bundle := newApexBundle() bundle.testApex = true + bundle.AddProperties(&bundle.testProperties) return bundle } diff --git a/apex/apex_test.go b/apex/apex_test.go index d0494d67d..6e9295911 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -495,6 +495,7 @@ func TestBasicApex(t *testing.T) { "//apex_available:platform", "myapex", ], + compile_dex: true, } dex_import { @@ -664,6 +665,7 @@ func TestDefaults(t *testing.T) { sdk_version: "none", system_modules: "none", apex_available: [ "myapex" ], + compile_dex: true, } android_app { @@ -2035,6 +2037,7 @@ func TestApexMinSdkVersion_SupportsCodeNames_JavaLibs(t *testing.T) { apex_available: [ "myapex" ], sdk_version: "current", min_sdk_version: "S", // should be okay + compile_dex: true, } `, android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { @@ -2584,6 +2587,7 @@ func TestApexMinSdkVersion_ErrorIfIncompatibleVersion(t *testing.T) { "myapex", ], min_sdk_version: "30", + compile_dex: true, } `) @@ -2611,6 +2615,7 @@ func TestApexMinSdkVersion_ErrorIfIncompatibleVersion(t *testing.T) { // Compile against core API surface sdk_version: "core_current", min_sdk_version: "30", + compile_dex: true, } `) @@ -2658,6 +2663,7 @@ func TestApexMinSdkVersion_Okay(t *testing.T) { ], apex_available: ["myapex"], min_sdk_version: "29", + compile_dex: true, } java_library { @@ -2737,6 +2743,7 @@ func TestJavaStableSdkVersion(t *testing.T) { srcs: ["foo/bar/MyClass.java"], sdk_version: "test_current", apex_available: ["myapex"], + compile_dex: true, } `, }, @@ -2761,6 +2768,7 @@ func TestJavaStableSdkVersion(t *testing.T) { sdk_version: "current", apex_available: ["myapex"], min_sdk_version: "29", + compile_dex: true, } `, }, @@ -2784,6 +2792,7 @@ func TestJavaStableSdkVersion(t *testing.T) { srcs: ["foo/bar/MyClass.java"], sdk_version: "test_current", apex_available: ["myapex"], + compile_dex: true, } `, }, @@ -2807,6 +2816,7 @@ func TestJavaStableSdkVersion(t *testing.T) { srcs: ["foo/bar/MyClass.java"], sdk_version: "core_platform", apex_available: ["myapex"], + compile_dex: true, } `, preparer: java.FixtureUseLegacyCorePlatformApi("myjar-uses-legacy"), @@ -2835,6 +2845,7 @@ func TestJavaStableSdkVersion(t *testing.T) { sdk_version: "current", apex_available: ["myapex"], static_libs: ["transitive-jar"], + compile_dex: true, } java_library { name: "transitive-jar", @@ -5913,6 +5924,7 @@ func TestErrorsIfDepsAreNotEnabled(t *testing.T) { system_modules: "none", enabled: false, apex_available: ["myapex"], + compile_dex: true, } `) } @@ -7089,6 +7101,51 @@ func TestApexAvailable_PrefixMatch(t *testing.T) { `) } +func TestApexValidation_TestApexCanSkipInitRcCheck(t *testing.T) { + t.Parallel() + ctx := testApex(t, ` + apex_test { + name: "myapex", + key: "myapex.key", + skip_validations: { + host_apex_verifier: true, + }, + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + + validations := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("signapk").Validations.Strings() + if android.SuffixInList(validations, "host_apex_verifier.timestamp") { + t.Error("should not run host_apex_verifier") + } +} + +func TestApexValidation_TestApexCheckInitRc(t *testing.T) { + t.Parallel() + ctx := testApex(t, ` + apex_test { + name: "myapex", + key: "myapex.key", + updatable: false, + } + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + `) + + validations := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("signapk").Validations.Strings() + if !android.SuffixInList(validations, "host_apex_verifier.timestamp") { + t.Error("should run host_apex_verifier") + } +} + func TestOverrideApex(t *testing.T) { t.Parallel() ctx := testApex(t, ` @@ -7525,6 +7582,7 @@ func TestJavaSDKLibrary_WithinApex(t *testing.T) { apex_available: ["myapex"], sdk_version: "none", system_modules: "none", + compile_dex: true, } java_library { @@ -7534,6 +7592,7 @@ func TestJavaSDKLibrary_WithinApex(t *testing.T) { apex_available: ["myapex"], sdk_version: "none", system_modules: "none", + compile_dex: true, } prebuilt_apis { @@ -7643,6 +7702,7 @@ func TestJavaSDKLibrary_ImportPreferred(t *testing.T) { apex_available: ["myapex"], sdk_version: "none", system_modules: "none", + compile_dex: true, } `), "source/a.java": nil, @@ -7664,6 +7724,7 @@ func TestJavaSDKLibrary_ImportPreferred(t *testing.T) { public: { enabled: true, }, + compile_dex: true, } `), "prebuilt/a.jar": nil, @@ -7680,6 +7741,7 @@ func TestJavaSDKLibrary_ImportPreferred(t *testing.T) { public: { jars: ["a.jar"], }, + compile_dex: true, } `), }), withFiles(filesForSdkLibrary), @@ -7758,6 +7820,7 @@ func TestCompatConfig(t *testing.T) { sdk_version: "none", system_modules: "none", apex_available: [ "myapex" ], + compile_dex: true, } // Make sure that a preferred prebuilt does not affect the apex contents. @@ -7997,6 +8060,7 @@ func TestSymlinksFromApexToSystem(t *testing.T) { "//apex_available:platform", ], min_sdk_version: "33", + compile_dex: true, } java_library { @@ -8605,6 +8669,7 @@ func TestApexPermittedPackagesRules(t *testing.T) { apex_available: ["myapex"], sdk_version: "none", system_modules: "none", + compile_dex: true, } java_library { name: "nonbcp_lib2", @@ -8613,6 +8678,7 @@ func TestApexPermittedPackagesRules(t *testing.T) { permitted_packages: ["a.b"], sdk_version: "none", system_modules: "none", + compile_dex: true, } apex { name: "myapex", @@ -8638,6 +8704,7 @@ func TestApexPermittedPackagesRules(t *testing.T) { permitted_packages: ["foo.bar"], sdk_version: "none", system_modules: "none", + compile_dex: true, } java_library { name: "bcp_lib2", @@ -8646,6 +8713,7 @@ func TestApexPermittedPackagesRules(t *testing.T) { permitted_packages: ["foo.bar", "bar.baz"], sdk_version: "none", system_modules: "none", + compile_dex: true, } apex { name: "myapex", @@ -8676,6 +8744,7 @@ func TestApexPermittedPackagesRules(t *testing.T) { sdk_version: "none", min_sdk_version: "29", system_modules: "none", + compile_dex: true, } java_library { name: "bcp_lib_unrestricted", @@ -8685,6 +8754,7 @@ func TestApexPermittedPackagesRules(t *testing.T) { sdk_version: "none", min_sdk_version: "29", system_modules: "none", + compile_dex: true, } apex { name: "myapex", @@ -9834,6 +9904,7 @@ func TestApexStrictUpdtabilityLint(t *testing.T) { }, sdk_version: "current", min_sdk_version: "29", + compile_dex: true, } ` fs := android.MockFS{ @@ -10268,6 +10339,7 @@ func TestAconfigFilesJavaDeps(t *testing.T) { apex_available: [ "myapex", ], + compile_dex: true, } java_library { @@ -10279,6 +10351,7 @@ func TestAconfigFilesJavaDeps(t *testing.T) { apex_available: [ "myapex", ], + compile_dex: true, } aconfig_declarations { @@ -10365,6 +10438,7 @@ func TestAconfigFilesJavaAndCcDeps(t *testing.T) { apex_available: [ "myapex", ], + compile_dex: true, } cc_library { @@ -10691,6 +10765,7 @@ func TestAconfigFilesOnlyMatchCurrentApex(t *testing.T) { apex_available: [ "myapex", ], + compile_dex: true, } java_library { @@ -10702,6 +10777,7 @@ func TestAconfigFilesOnlyMatchCurrentApex(t *testing.T) { apex_available: [ "myapex", ], + compile_dex: true, } aconfig_declarations { @@ -10776,6 +10852,7 @@ func TestAconfigFilesRemoveDuplicates(t *testing.T) { apex_available: [ "myapex", ], + compile_dex: true, } java_library { @@ -10787,6 +10864,7 @@ func TestAconfigFilesRemoveDuplicates(t *testing.T) { apex_available: [ "myapex", ], + compile_dex: true, } aconfig_declarations { @@ -11469,6 +11547,7 @@ func TestApexMinSdkVersionOverride(t *testing.T) { apex_available: ["com.android.apex30"], min_sdk_version: "30", sdk_version: "current", + compile_dex: true, } override_apex { @@ -11760,6 +11839,7 @@ func TestSdkLibraryTransitiveClassLoaderContext(t *testing.T) { "com.android.foo30", ], sdk_version: "core_current", + compile_dex: true, } java_library { diff --git a/apex/builder.go b/apex/builder.go index d0acc8d6c..e5ae10622 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -924,14 +924,14 @@ func (a *apexBundle) buildApex(ctx android.ModuleContext) { var validations android.Paths validations = append(validations, runApexLinkerconfigValidation(ctx, unsignedOutputFile, imageDir)) // TODO(b/279688635) deapexer supports [ext4] - if !a.testApex && suffix == imageApexSuffix && ext4 == a.payloadFsType { + if !a.skipValidation(apexSepolicyTests) && suffix == imageApexSuffix && ext4 == a.payloadFsType { validations = append(validations, runApexSepolicyTests(ctx, unsignedOutputFile)) } if !a.testApex && len(a.properties.Unwanted_transitive_deps) > 0 { validations = append(validations, runApexElfCheckerUnwanted(ctx, unsignedOutputFile, a.properties.Unwanted_transitive_deps)) } - if !a.testApex && android.InList(a.payloadFsType, []fsType{ext4, erofs}) { + if !a.skipValidation(hostApexVerifier) && android.InList(a.payloadFsType, []fsType{ext4, erofs}) { validations = append(validations, runApexHostVerifier(ctx, a, unsignedOutputFile)) } ctx.Build(pctx, android.BuildParams{ @@ -1220,7 +1220,7 @@ func runApexLinkerconfigValidation(ctx android.ModuleContext, apexFile android.P // $ deapexer list -Z {apex_file} > {file_contexts} // $ apex_sepolicy_tests -f {file_contexts} func runApexSepolicyTests(ctx android.ModuleContext, apexFile android.Path) android.Path { - timestamp := android.PathForModuleOut(ctx, "sepolicy_tests.timestamp") + timestamp := android.PathForModuleOut(ctx, "apex_sepolicy_tests.timestamp") ctx.Build(pctx, android.BuildParams{ Rule: apexSepolicyTestsRule, Input: apexFile, diff --git a/apex/container_test.go b/apex/container_test.go index d1dfb9cab..395793f61 100644 --- a/apex/container_test.go +++ b/apex/container_test.go @@ -15,10 +15,11 @@ package apex import ( - "android/soong/android" - "android/soong/java" "fmt" "testing" + + "android/soong/android" + "android/soong/java" ) var checkContainerMatch = func(t *testing.T, name string, container string, expected bool, actual bool) { @@ -329,6 +330,7 @@ func TestUpdatableAndNonUpdatableApexesIdenticalMinSdkVersion(t *testing.T) { ], min_sdk_version: "30", sdk_version: "current", + compile_dex: true, } `) diff --git a/apex/prebuilt.go b/apex/prebuilt.go index acf3b91fe..f93eada8b 100644 --- a/apex/prebuilt.go +++ b/apex/prebuilt.go @@ -306,10 +306,6 @@ func (p *prebuiltCommon) DepIsInSameApex(ctx android.BaseModuleContext, dep andr // extra copying of files. Contrast that with source apex modules that has to build each variant // from source. func (p *prebuiltCommon) apexInfoMutator(mctx android.TopDownMutatorContext) { - - // Collect direct dependencies into contents. - contents := make(map[string]android.ApexMembership) - // Collect the list of dependencies. var dependencies []android.ApexModule mctx.WalkDeps(func(child, parent android.Module) bool { @@ -347,29 +343,19 @@ func (p *prebuiltCommon) apexInfoMutator(mctx android.TopDownMutatorContext) { // behavior whether there is a corresponding source module present or not. depName = android.RemoveOptionalPrebuiltPrefix(depName) - // Remember if this module was added as a direct dependency. - direct := parent == mctx.Module() - contents[depName] = contents[depName].Add(direct) - // Add the module to the list of dependencies that need to have an APEX variant. dependencies = append(dependencies, child.(android.ApexModule)) return true }) - // Create contents for the prebuilt_apex and store it away for later use. - apexContents := android.NewApexContents(contents) - android.SetProvider(mctx, android.ApexBundleInfoProvider, android.ApexBundleInfo{ - Contents: apexContents, - }) + android.SetProvider(mctx, android.ApexBundleInfoProvider, android.ApexBundleInfo{}) // Create an ApexInfo for the prebuilt_apex. apexVariationName := p.ApexVariationName() apexInfo := android.ApexInfo{ ApexVariationName: apexVariationName, InApexVariants: []string{apexVariationName}, - InApexModules: []string{p.BaseModuleName()}, // BaseModuleName() to avoid the prebuilt_ prefix. - ApexContents: []*android.ApexContents{apexContents}, ForPrebuiltApex: true, } diff --git a/bpf/libbpf/libbpf_prog.go b/bpf/libbpf/libbpf_prog.go index 0ca7af1f7..3b26d4654 100644 --- a/bpf/libbpf/libbpf_prog.go +++ b/bpf/libbpf/libbpf_prog.go @@ -158,7 +158,8 @@ func (libbpf *libbpfProg) GenerateAndroidBuildActions(ctx android.ModuleContext) "-Wall", "-Werror", "-Wextra", - + // Flag to assist with the transition to libbpf + "-DENABLE_LIBBPF", "-isystem bionic/libc/include", "-isystem bionic/libc/kernel/uapi", // The architecture doesn't matter here, but asm/types.h is included by linux/types.h. @@ -205,7 +206,7 @@ func (libbpf *libbpfProg) GenerateAndroidBuildActions(ctx android.ModuleContext) if strings.ContainsRune(src.Base(), '_') { ctx.ModuleErrorf("invalid character '_' in source name") } - obj := android.ObjPathWithExt(ctx, "unstripped", src, "o") + obj := android.ObjPathWithExt(ctx, "unstripped", src, "bpf") ctx.Build(pctx, android.BuildParams{ Rule: libbpfProgCcRule, @@ -218,7 +219,7 @@ func (libbpf *libbpfProg) GenerateAndroidBuildActions(ctx android.ModuleContext) }, }) - objStripped := android.ObjPathWithExt(ctx, "", src, "o") + objStripped := android.ObjPathWithExt(ctx, "", src, "bpf") ctx.Build(pctx, android.BuildParams{ Rule: libbpfProgStripRule, Input: obj, @@ -230,7 +231,7 @@ func (libbpf *libbpfProg) GenerateAndroidBuildActions(ctx android.ModuleContext) libbpf.objs = append(libbpf.objs, objStripped.WithoutRel()) } - installDir := android.PathForModuleInstall(ctx, "etc", "bpf/libbpf") + installDir := android.PathForModuleInstall(ctx, "etc", "bpf") if len(libbpf.properties.Relative_install_path) > 0 { installDir = installDir.Join(ctx, libbpf.properties.Relative_install_path) } @@ -251,7 +252,7 @@ func (libbpf *libbpfProg) AndroidMk() android.AndroidMkData { fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir) fmt.Fprintln(w) var localModulePath string - localModulePath = "LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/bpf/libbpf" + localModulePath = "LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/bpf" if len(libbpf.properties.Relative_install_path) > 0 { localModulePath += "/" + libbpf.properties.Relative_install_path } diff --git a/bpf/libbpf/libbpf_prog_test.go b/bpf/libbpf/libbpf_prog_test.go index 7f3653df9..2b3b37841 100644 --- a/bpf/libbpf/libbpf_prog_test.go +++ b/bpf/libbpf/libbpf_prog_test.go @@ -41,7 +41,7 @@ var prepareForLibbpfProgTest = android.GroupFixturePreparers( func TestLibbpfProgDataDependency(t *testing.T) { bp := ` libbpf_prog { - name: "bpf.o", + name: "bpf.bpf", srcs: ["bpf.c"], } @@ -49,7 +49,7 @@ func TestLibbpfProgDataDependency(t *testing.T) { name: "vts_test_binary_bpf_module", compile_multilib: "first", srcs: ["BpfTest.cpp"], - data: [":bpf.o"], + data: [":bpf.bpf"], gtest: false, } ` @@ -60,7 +60,7 @@ func TestLibbpfProgDataDependency(t *testing.T) { func TestLibbpfProgSourceName(t *testing.T) { bp := ` libbpf_prog { - name: "bpf_invalid_name.o", + name: "bpf_invalid_name.bpf", srcs: ["bpf_invalid_name.c"], } ` diff --git a/cc/api_level.go b/cc/api_level.go index 69a0d3ae4..3dac571a9 100644 --- a/cc/api_level.go +++ b/cc/api_level.go @@ -41,12 +41,25 @@ func MinApiForArch(ctx android.EarlyModuleContext, } } +// Native API levels cannot be less than the MinApiLevelForArch. This function +// sets the lower bound of the API level with the MinApiLevelForArch. +func nativeClampedApiLevel(ctx android.BaseModuleContext, + apiLevel android.ApiLevel) android.ApiLevel { + + min := MinApiForArch(ctx, ctx.Arch().ArchType) + + if apiLevel.LessThan(min) { + return min + } + + return apiLevel +} + func nativeApiLevelFromUser(ctx android.BaseModuleContext, raw string) (android.ApiLevel, error) { - min := MinApiForArch(ctx, ctx.Arch().ArchType) if raw == "minimum" { - return min, nil + return MinApiForArch(ctx, ctx.Arch().ArchType), nil } value, err := android.ApiLevelFromUser(ctx, raw) @@ -54,15 +67,12 @@ func nativeApiLevelFromUser(ctx android.BaseModuleContext, return android.NoneApiLevel, err } - if value.LessThan(min) { - return min, nil - } - - return value, nil + return nativeClampedApiLevel(ctx, value), nil } func nativeApiLevelOrPanic(ctx android.BaseModuleContext, raw string) android.ApiLevel { + value, err := nativeApiLevelFromUser(ctx, raw) if err != nil { panic(err.Error()) diff --git a/cc/binary.go b/cc/binary.go index 2ac9a45bc..4b77bea64 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -505,7 +505,7 @@ func (binary *binaryDecorator) install(ctx ModuleContext, file android.Path) { // The original path becomes a symlink to the corresponding file in the // runtime APEX. translatedArch := ctx.Target().NativeBridge == android.NativeBridgeEnabled - if InstallToBootstrap(ctx.baseModuleName(), ctx.Config()) && !ctx.Host() && ctx.directlyInAnyApex() && + if InstallToBootstrap(ctx.baseModuleName(), ctx.Config()) && !ctx.Host() && !ctx.isSdkVariant() && !translatedArch && ctx.apexVariationName() == "" && !ctx.inRamdisk() && !ctx.inRecovery() && !ctx.inVendorRamdisk() { diff --git a/cc/builder.go b/cc/builder.go index 2948ca316..b98bef9be 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -945,7 +945,7 @@ func transformObjToDynamicBinary(ctx android.ModuleContext, func transformDumpToLinkedDump(ctx android.ModuleContext, sAbiDumps android.Paths, soFile android.Path, baseName string, exportedIncludeDirs []string, symbolFile android.OptionalPath, excludedSymbolVersions, excludedSymbolTags, includedSymbolTags []string, - api string, isLlndk bool) android.Path { + api string) android.Path { outputFile := android.PathForModuleOut(ctx, baseName+".lsdump") @@ -966,9 +966,6 @@ func transformDumpToLinkedDump(ctx android.ModuleContext, sAbiDumps android.Path for _, tag := range includedSymbolTags { symbolFilterStr += " --include-symbol-tag " + tag } - if isLlndk { - symbolFilterStr += " --symbol-tag-policy MatchTagOnly" - } apiLevelsJson := android.GetApiLevelsJson(ctx) implicits = append(implicits, apiLevelsJson) symbolFilterStr += " --api-map " + apiLevelsJson.String() @@ -341,15 +341,14 @@ type BaseProperties struct { // If true, always create an sdk variant and don't create a platform variant. Sdk_variant_only *bool - AndroidMkSharedLibs []string `blueprint:"mutated"` - AndroidMkStaticLibs []string `blueprint:"mutated"` - AndroidMkRlibs []string `blueprint:"mutated"` - AndroidMkRuntimeLibs []string `blueprint:"mutated"` - AndroidMkWholeStaticLibs []string `blueprint:"mutated"` - AndroidMkHeaderLibs []string `blueprint:"mutated"` - HideFromMake bool `blueprint:"mutated"` - PreventInstall bool `blueprint:"mutated"` - ApexesProvidingSharedLibs []string `blueprint:"mutated"` + AndroidMkSharedLibs []string `blueprint:"mutated"` + AndroidMkStaticLibs []string `blueprint:"mutated"` + AndroidMkRlibs []string `blueprint:"mutated"` + AndroidMkRuntimeLibs []string `blueprint:"mutated"` + AndroidMkWholeStaticLibs []string `blueprint:"mutated"` + AndroidMkHeaderLibs []string `blueprint:"mutated"` + HideFromMake bool `blueprint:"mutated"` + PreventInstall bool `blueprint:"mutated"` // Set by DepsMutator. AndroidMkSystemSharedLibs []string `blueprint:"mutated"` @@ -548,7 +547,6 @@ type ModuleContextIntf interface { apexSdkVersion() android.ApiLevel bootstrap() bool nativeCoverage() bool - directlyInAnyApex() bool isPreventInstall() bool isCfiAssemblySupportEnabled() bool getSharedFlags() *SharedFlags @@ -1692,10 +1690,6 @@ func (ctx *moduleContextImpl) nativeCoverage() bool { return ctx.mod.nativeCoverage() } -func (ctx *moduleContextImpl) directlyInAnyApex() bool { - return ctx.mod.DirectlyInAnyApex() -} - func (ctx *moduleContextImpl) isPreventInstall() bool { return ctx.mod.Properties.PreventInstall } @@ -1885,7 +1879,7 @@ var ( // Returns true if a stub library could be installed in multiple apexes func (c *Module) stubLibraryMultipleApexViolation(ctx android.ModuleContext) bool { // If this is not an apex variant, no check necessary - if !c.InAnyApex() { + if info, ok := android.ModuleProvider(ctx, android.ApexInfoProvider); !ok || info.IsForPlatform() { return false } // If this is not a stub library, no check necessary @@ -3290,18 +3284,6 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { c.Properties.AndroidMkHeaderLibs = append( c.Properties.AndroidMkHeaderLibs, makeLibName) case libDepTag.shared(): - if lib := moduleLibraryInterface(dep); lib != nil { - if lib.buildStubs() && dep.(android.ApexModule).InAnyApex() { - // Add the dependency to the APEX(es) providing the library so that - // m <module> can trigger building the APEXes as well. - depApexInfo, _ := android.OtherModuleProvider(ctx, dep, android.ApexInfoProvider) - for _, an := range depApexInfo.InApexVariants { - c.Properties.ApexesProvidingSharedLibs = append( - c.Properties.ApexesProvidingSharedLibs, an) - } - } - } - // Note: the order of libs in this list is not important because // they merely serve as Make dependencies and do not affect this lib itself. c.Properties.AndroidMkSharedLibs = append( diff --git a/cc/cc_test.go b/cc/cc_test.go index 144b90b94..98af7b655 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -40,9 +40,6 @@ func TestMain(m *testing.M) { var prepareForCcTest = android.GroupFixturePreparers( PrepareForIntegrationTestWithCc, - android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { - variables.VendorApiLevel = StringPtr("202404") - }), ) var apexVariationName = "apex28" @@ -1008,7 +1005,7 @@ func TestLlndkLibrary(t *testing.T) { android.AssertArrayString(t, "variants for llndk stubs", expected, actual) params := result.ModuleForTests("libllndk", "android_vendor_arm_armv7-a-neon_shared").Description("generate stub") - android.AssertSame(t, "use Vendor API level for default stubs", "999999", params.Args["apiLevel"]) + android.AssertSame(t, "use Vendor API level for default stubs", "35", params.Args["apiLevel"]) checkExportedIncludeDirs := func(module, variant string, expectedSystemDirs []string, expectedDirs ...string) { t.Helper() diff --git a/cc/config/global.go b/cc/config/global.go index 36690d6be..27aac959a 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -290,14 +290,13 @@ var ( "-Wno-error=deprecated", // in external/googletest/googletest // Disabling until the warning is fixed in libc++abi header files b/366180429 "-Wno-deprecated-dynamic-exception-spec", - // New warnings to be fixed after clang-r475365 - "-Wno-error=enum-constexpr-conversion", // http://b/243964282 // New warnings to be fixed after clang-r522817 "-Wno-error=invalid-offsetof", "-Wno-error=thread-safety-reference-return", // Allow using VLA CXX extension. "-Wno-vla-cxx-extension", + "-Wno-cast-function-type-mismatch", } noOverride64GlobalCflags = []string{} @@ -386,7 +385,7 @@ var ( // prebuilts/clang default settings. ClangDefaultBase = "prebuilts/clang/host" - ClangDefaultVersion = "clang-r530567" + ClangDefaultVersion = "clang-r536225" ClangDefaultShortVersion = "19" // Directories with warnings from Android.bp files. diff --git a/cc/image.go b/cc/image.go index ee4048390..9766af3df 100644 --- a/cc/image.go +++ b/cc/image.go @@ -179,9 +179,6 @@ type ImageMutatableModule interface { // SnapshotVersion returns the snapshot version for this module. SnapshotVersion(mctx android.ImageInterfaceContext) string - // SdkVersion returns the SDK version for this module. - SdkVersion() string - // ExtraVariants returns the list of extra variants this module requires. ExtraVariants() []string @@ -370,7 +367,7 @@ func MutateImage(mctx android.ImageInterfaceContext, m ImageMutatableModule) { if m.HasProductVariant() { productVariantNeeded = true } - } else if vendorSpecific && m.SdkVersion() == "" { + } else if vendorSpecific { // This will be available in /vendor (or /odm) only vendorVariantNeeded = true } else { @@ -380,7 +377,7 @@ func MutateImage(mctx android.ImageInterfaceContext, m ImageMutatableModule) { coreVariantNeeded = true } - if coreVariantNeeded && productSpecific && m.SdkVersion() == "" { + if coreVariantNeeded && productSpecific { // The module has "product_specific: true" that does not create core variant. coreVariantNeeded = false productVariantNeeded = true diff --git a/cc/library.go b/cc/library.go index 4ce506e90..ea8794644 100644 --- a/cc/library.go +++ b/cc/library.go @@ -566,10 +566,16 @@ func (library *libraryDecorator) getHeaderAbiCheckerProperties(m *Module) header func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects { if ctx.IsLlndk() { - futureVendorApiLevel := android.ApiLevelOrPanic(ctx, "999999") + // Get the matching SDK version for the vendor API level. + version, err := android.GetSdkVersionForVendorApiLevel(ctx.Config().VendorApiLevel()) + if err != nil { + panic(err) + } + + // This is the vendor variant of an LLNDK library, build the LLNDK stubs. nativeAbiResult := parseNativeAbiDefinition(ctx, String(library.Properties.Llndk.Symbol_file), - futureVendorApiLevel, "--llndk") + nativeClampedApiLevel(ctx, version), "--llndk") objs := compileStubLibrary(ctx, flags, nativeAbiResult.stubSrc) if !Bool(library.Properties.Llndk.Unversioned) { library.versionScriptPath = android.OptionalPathForPath( @@ -656,7 +662,7 @@ func (library *libraryDecorator) compileModuleLibApiStubs(ctx ModuleContext, fla // However, having this distinction helps guard accidental // promotion or demotion of API and also helps the API review process b/191371676 var flag string - if ctx.Module().(android.ApexModule).NotInPlatform() { + if ctx.notInPlatform() { flag = "--apex" } else { flag = "--systemapi" @@ -740,6 +746,7 @@ type versionedInterface interface { hasLLNDKStubs() bool hasLLNDKHeaders() bool hasVendorPublicLibrary() bool + isLLNDKMovedToApex() bool } var _ libraryInterface = (*libraryDecorator)(nil) @@ -1284,15 +1291,14 @@ func (library *libraryDecorator) llndkIncludeDirsForAbiCheck(ctx ModuleContext, func (library *libraryDecorator) linkLlndkSAbiDumpFiles(ctx ModuleContext, deps PathDeps, sAbiDumpFiles android.Paths, soFile android.Path, libFileName string, excludeSymbolVersions, excludeSymbolTags []string, - vendorApiLevel string) android.Path { - // NDK symbols in version 34 are LLNDK symbols. Those in version 35 are not. + sdkVersionForVendorApiLevel string) android.Path { return transformDumpToLinkedDump(ctx, sAbiDumpFiles, soFile, libFileName+".llndk", library.llndkIncludeDirsForAbiCheck(ctx, deps), android.OptionalPathForModuleSrc(ctx, library.Properties.Llndk.Symbol_file), append([]string{"*_PLATFORM", "*_PRIVATE"}, excludeSymbolVersions...), append([]string{"platform-only"}, excludeSymbolTags...), - []string{"llndk=" + vendorApiLevel}, "34", true /* isLlndk */) + []string{"llndk"}, sdkVersionForVendorApiLevel) } func (library *libraryDecorator) linkApexSAbiDumpFiles(ctx ModuleContext, @@ -1305,7 +1311,7 @@ func (library *libraryDecorator) linkApexSAbiDumpFiles(ctx ModuleContext, android.OptionalPathForModuleSrc(ctx, library.Properties.Stubs.Symbol_file), append([]string{"*_PLATFORM", "*_PRIVATE"}, excludeSymbolVersions...), append([]string{"platform-only"}, excludeSymbolTags...), - []string{"apex", "systemapi"}, sdkVersion, false /* isLlndk */) + []string{"apex", "systemapi"}, sdkVersion) } func getRefAbiDumpFile(ctx android.ModuleInstallPathContext, @@ -1443,7 +1449,7 @@ func (library *libraryDecorator) linkSAbiDumpFiles(ctx ModuleContext, deps PathD android.OptionalPathForModuleSrc(ctx, library.symbolFileForAbiCheck(ctx)), headerAbiChecker.Exclude_symbol_versions, headerAbiChecker.Exclude_symbol_tags, - []string{} /* includeSymbolTags */, currSdkVersion, false /* isLlndk */) + []string{} /* includeSymbolTags */, currSdkVersion) var llndkDump, apexVariantDump android.Path tags := classifySourceAbiDump(ctx.Module().(*Module)) @@ -1451,12 +1457,17 @@ func (library *libraryDecorator) linkSAbiDumpFiles(ctx ModuleContext, deps PathD for _, tag := range tags { if tag == llndkLsdumpTag && currVendorVersion != "" { if llndkDump == nil { + sdkVersion, err := android.GetSdkVersionForVendorApiLevel(currVendorVersion) + if err != nil { + ctx.ModuleErrorf("Cannot create %s llndk dump: %s", fileName, err) + return + } // TODO(b/323447559): Evaluate if replacing sAbiDumpFiles with implDump is faster llndkDump = library.linkLlndkSAbiDumpFiles(ctx, deps, objs.sAbiDumpFiles, soFile, fileName, headerAbiChecker.Exclude_symbol_versions, headerAbiChecker.Exclude_symbol_tags, - currVendorVersion) + nativeClampedApiLevel(ctx, sdkVersion).String()) } addLsdumpPath(ctx.Config(), string(tag)+":"+llndkDump.String()) } else if tag == apexLsdumpTag { @@ -1750,21 +1761,17 @@ func (library *libraryDecorator) installSymlinkToRuntimeApex(ctx ModuleContext, func (library *libraryDecorator) install(ctx ModuleContext, file android.Path) { if library.shared() { - if library.hasStubsVariants() && !ctx.Host() && ctx.directlyInAnyApex() { + translatedArch := ctx.Target().NativeBridge == android.NativeBridgeEnabled + if library.hasStubsVariants() && !ctx.Host() && !ctx.isSdkVariant() && + InstallToBootstrap(ctx.baseModuleName(), ctx.Config()) && !library.buildStubs() && + !translatedArch && !ctx.inRamdisk() && !ctx.inVendorRamdisk() && !ctx.inRecovery() { // Bionic libraries (e.g. libc.so) is installed to the bootstrap subdirectory. // The original path becomes a symlink to the corresponding file in the // runtime APEX. - translatedArch := ctx.Target().NativeBridge == android.NativeBridgeEnabled - if InstallToBootstrap(ctx.baseModuleName(), ctx.Config()) && !library.buildStubs() && - !translatedArch && !ctx.inRamdisk() && !ctx.inVendorRamdisk() && !ctx.inRecovery() { - if ctx.Device() { - library.installSymlinkToRuntimeApex(ctx, file) - } - library.baseInstaller.subDir = "bootstrap" + if ctx.Device() { + library.installSymlinkToRuntimeApex(ctx, file) } - } else if ctx.directlyInAnyApex() && ctx.IsLlndk() && !isBionic(ctx.baseModuleName()) { - // Skip installing LLNDK (non-bionic) libraries moved to APEX. - ctx.Module().HideFromMake() + library.baseInstaller.subDir = "bootstrap" } library.baseInstaller.install(ctx, file) @@ -1848,6 +1855,11 @@ func (library *libraryDecorator) hasLLNDKHeaders() bool { return Bool(library.Properties.Llndk.Llndk_headers) } +// isLLNDKMovedToApex returns true if this cc_library module sets the llndk.moved_to_apex property. +func (library *libraryDecorator) isLLNDKMovedToApex() bool { + return Bool(library.Properties.Llndk.Moved_to_apex) +} + // hasVendorPublicLibrary returns true if this cc_library module has a variant that will build // vendor public library stubs. func (library *libraryDecorator) hasVendorPublicLibrary() bool { diff --git a/cc/linker.go b/cc/linker.go index f9d58ea20..b96d13983 100644 --- a/cc/linker.go +++ b/cc/linker.go @@ -235,13 +235,16 @@ type BaseLinkerProperties struct { // Generate compact dynamic relocation table, default true. Pack_relocations *bool `android:"arch_variant"` - // local file name to pass to the linker as --version-script + // local file name to pass to the linker as --version-script. Not supported on darwin, and will fail to build + // if provided to the darwin variant of a module. Version_script *string `android:"path,arch_variant"` - // local file name to pass to the linker as --dynamic-list + // local file name to pass to the linker as --dynamic-list. Not supported on darwin, and will fail to build + // if provided to the darwin variant of a module. Dynamic_list *string `android:"path,arch_variant"` - // local files to pass to the linker as --script + // local files to pass to the linker as --script. Not supported on darwin or windows, and will fail to build + // if provided to the darwin or windows variant of a module. Linker_scripts []string `android:"path,arch_variant"` // list of static libs that should not be used to build this module @@ -560,7 +563,7 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { if versionScript.Valid() { if ctx.Darwin() { - ctx.PropertyErrorf("version_script", "Not supported on Darwin") + ctx.AddMissingDependencies([]string{"version_script_not_supported_on_darwin"}) } else { flags.Local.LdFlags = append(flags.Local.LdFlags, config.VersionScriptFlagPrefix+versionScript.String()) @@ -578,7 +581,7 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { dynamicList := android.OptionalPathForModuleSrc(ctx, linker.Properties.Dynamic_list) if dynamicList.Valid() { if ctx.Darwin() { - ctx.PropertyErrorf("dynamic_list", "Not supported on Darwin") + ctx.AddMissingDependencies([]string{"dynamic_list_not_supported_on_darwin"}) } else { flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--dynamic-list,"+dynamicList.String()) @@ -587,13 +590,17 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { } linkerScriptPaths := android.PathsForModuleSrc(ctx, linker.Properties.Linker_scripts) - if len(linkerScriptPaths) > 0 && (ctx.Darwin() || ctx.Windows()) { - ctx.PropertyErrorf("linker_scripts", "Only supported for ELF files") - } else { - for _, linkerScriptPath := range linkerScriptPaths { - flags.Local.LdFlags = append(flags.Local.LdFlags, - "-Wl,--script,"+linkerScriptPath.String()) - flags.LdFlagsDeps = append(flags.LdFlagsDeps, linkerScriptPath) + if len(linkerScriptPaths) > 0 { + if ctx.Darwin() { + ctx.AddMissingDependencies([]string{"linker_scripts_not_supported_on_darwin"}) + } else if ctx.Windows() { + ctx.PropertyErrorf("linker_scripts", "Only supported for ELF files") + } else { + for _, linkerScriptPath := range linkerScriptPaths { + flags.Local.LdFlags = append(flags.Local.LdFlags, + "-Wl,--script,"+linkerScriptPath.String()) + flags.LdFlagsDeps = append(flags.LdFlagsDeps, linkerScriptPath) + } } } } diff --git a/cc/llndk_library.go b/cc/llndk_library.go index c7950f9ff..162dd5429 100644 --- a/cc/llndk_library.go +++ b/cc/llndk_library.go @@ -57,17 +57,18 @@ type llndkLibraryProperties struct { // if true, make this module available to provide headers to other modules that set // llndk.symbol_file. Llndk_headers *bool + + // moved_to_apex marks this module has having been distributed through an apex module. + Moved_to_apex *bool } func makeLlndkVars(ctx android.MakeVarsContext) { - // Make uses LLNDK_MOVED_TO_APEX_LIBRARIES to avoid installing libraries on /system if - // they been moved to an apex. + // Make uses LLNDK_MOVED_TO_APEX_LIBRARIES to generate the linker config. movedToApexLlndkLibraries := make(map[string]bool) ctx.VisitAllModules(func(module android.Module) { if library := moduleLibraryInterface(module); library != nil && library.hasLLNDKStubs() { - // Skip bionic libs, they are handled in different manner - name := library.implementationModuleName(module.(*Module).BaseModuleName()) - if module.(android.ApexModule).DirectlyInAnyApex() && !isBionic(name) { + if library.isLLNDKMovedToApex() { + name := library.implementationModuleName(module.(*Module).BaseModuleName()) movedToApexLlndkLibraries[name] = true } } diff --git a/cc/ndkstubgen/test_ndkstubgen.py b/cc/ndkstubgen/test_ndkstubgen.py index 22f31d9f1..6c24b4f19 100755 --- a/cc/ndkstubgen/test_ndkstubgen.py +++ b/cc/ndkstubgen/test_ndkstubgen.py @@ -473,16 +473,17 @@ class IntegrationTest(unittest.TestCase): VERSION_35 { # introduced=35 global: wiggle; - waggle; - waggle; # llndk=202404 - bubble; # llndk=202404 - duddle; - duddle; # llndk=202504 + waggle; # llndk } VERSION_34; + VERSION_36 { # introduced=36 + global: + abc; + xyz; # llndk + } VERSION_35; """)) f = copy(self.filter) f.llndk = True - f.api = 202404 + f.api = 35 parser = symbolfile.SymbolFileParser(input_file, {}, f) versions = parser.parse() @@ -497,8 +498,8 @@ class IntegrationTest(unittest.TestCase): expected_src = textwrap.dedent("""\ void foo() {} void bar() {} + void wiggle() {} void waggle() {} - void bubble() {} """) self.assertEqual(expected_src, src_file.getvalue()) @@ -510,8 +511,8 @@ class IntegrationTest(unittest.TestCase): }; VERSION_35 { global: + wiggle; waggle; - bubble; } VERSION_34; """) self.assertEqual(expected_version, version_file.getvalue()) @@ -521,15 +522,15 @@ class IntegrationTest(unittest.TestCase): LIBANDROID { global: foo; # introduced=34 - bar; # introduced=35 - bar; # llndk=202404 - baz; # introduced=35 + bar; # introduced=35 llndk + baz; # introduced=V + qux; # introduced=36 }; """)) f = copy(self.filter) f.llndk = True - f.api = 202404 - parser = symbolfile.SymbolFileParser(input_file, {}, f) + f.api = 35 + parser = symbolfile.SymbolFileParser(input_file, {'V': 35}, f) versions = parser.parse() src_file = io.StringIO() @@ -543,6 +544,7 @@ class IntegrationTest(unittest.TestCase): expected_src = textwrap.dedent("""\ void foo() {} void bar() {} + void baz() {} """) self.assertEqual(expected_src, src_file.getvalue()) @@ -551,6 +553,7 @@ class IntegrationTest(unittest.TestCase): global: foo; bar; + baz; }; """) self.assertEqual(expected_version, version_file.getvalue()) diff --git a/cc/sanitize.go b/cc/sanitize.go index f0d734376..d8d8c7aef 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -1504,9 +1504,6 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { if Bool(sanProps.Memtag_globals) { sanitizers = append(sanitizers, "memtag-globals") - // TODO(mitchp): For now, enable memtag-heap with memtag-globals because the linker - // isn't new enough (https://reviews.llvm.org/differential/changeset/?ref=4243566). - sanitizers = append(sanitizers, "memtag-heap") } if Bool(sanProps.Fuzzer) { diff --git a/cc/symbolfile/__init__.py b/cc/symbolfile/__init__.py index 4553616ac..f2bd18690 100644 --- a/cc/symbolfile/__init__.py +++ b/cc/symbolfile/__init__.py @@ -103,24 +103,13 @@ class Tags: @property def has_llndk_tags(self) -> bool: """Returns True if any LL-NDK tags are set.""" - for tag in self.tags: - if tag == 'llndk' or tag.startswith('llndk='): - return True - return False + return 'llndk' in self.tags @property def has_platform_only_tags(self) -> bool: """Returns True if any platform-only tags are set.""" return 'platform-only' in self.tags - def copy_introduced_from(self, tags: Tags) -> None: - """Copies introduced= or introduced-*= tags.""" - for tag in tags: - if tag.startswith('introduced=') or tag.startswith('introduced-'): - name, _ = split_tag(tag) - if not any(self_tag.startswith(name + '=') for self_tag in self.tags): - self.tags += (tag,) - @dataclass class Symbol: @@ -158,8 +147,6 @@ def is_api_level_tag(tag: Tag) -> bool: """Returns true if this tag has an API level that may need decoding.""" if tag.startswith('llndk-deprecated='): return True - if tag.startswith('llndk='): - return True if tag.startswith('introduced='): return True if tag.startswith('introduced-'): @@ -245,21 +232,19 @@ class Filter: self.systemapi = systemapi self.ndk = ndk + def _symbol_in_arch_api(self, tags: Tags) -> bool: + if not symbol_in_arch(tags, self.arch): + return True + if not symbol_in_api(tags, self.arch, self.api): + return True + return False + def _should_omit_tags(self, tags: Tags) -> bool: """Returns True if the tagged object should be omitted. This defines the rules shared between version tagging and symbol tagging. """ - # LLNDK mode/tags follow the similar filtering except that API level checking - # is based llndk= instead of introduced=. - if self.llndk: - if tags.has_mode_tags and not tags.has_llndk_tags: - return True - if not symbol_in_arch(tags, self.arch): - return True - if not symbol_in_llndk_api(tags, self.arch, self.api): - return True - return False + # The apex and llndk tags will only exclude APIs from other modes. If in # APEX or LLNDK mode and neither tag is provided, we fall back to the # default behavior because all NDK symbols are implicitly available to # APEX and LLNDK. @@ -268,12 +253,10 @@ class Filter: return False if self.systemapi and tags.has_systemapi_tags: return False + if self.llndk and tags.has_llndk_tags: + return self._symbol_in_arch_api(tags) return True - if not symbol_in_arch(tags, self.arch): - return True - if not symbol_in_api(tags, self.arch, self.api): - return True - return False + return self._symbol_in_arch_api(tags) def should_omit_version(self, version: Version) -> bool: """Returns True if the version section should be omitted. @@ -286,10 +269,6 @@ class Filter: return True if version.tags.has_platform_only_tags: return True - # Include all versions when targeting LLNDK because LLNDK symbols are self-versioned. - # Empty version block will be handled separately. - if self.llndk: - return False return self._should_omit_tags(version.tags) def should_omit_symbol(self, symbol: Symbol) -> bool: @@ -302,6 +281,7 @@ class Filter: return self._should_omit_tags(symbol.tags) + def symbol_in_arch(tags: Tags, arch: Arch) -> bool: """Returns true if the symbol is present for the given architecture.""" has_arch_tags = False @@ -316,14 +296,6 @@ def symbol_in_arch(tags: Tags, arch: Arch) -> bool: # for the tagged architectures. return not has_arch_tags -def symbol_in_llndk_api(tags: Iterable[Tag], arch: Arch, api: int) -> bool: - """Returns true if the symbol is present for the given LLNDK API level.""" - # Check llndk= first. - for tag in tags: - if tag.startswith('llndk='): - return api >= int(get_tag_value(tag)) - # If not, we keep old behavior: NDK symbols in <= 34 are LLNDK symbols. - return symbol_in_api(tags, arch, 34) def symbol_in_api(tags: Iterable[Tag], arch: Arch, api: int) -> bool: """Returns true if the symbol is present for the given API level.""" @@ -400,7 +372,6 @@ class SymbolFileParser: f'Unexpected contents at top level: {self.current_line}') self.check_no_duplicate_symbols(versions) - self.check_llndk_introduced(versions) return versions def check_no_duplicate_symbols(self, versions: Iterable[Version]) -> None: @@ -429,31 +400,6 @@ class SymbolFileParser: raise MultiplyDefinedSymbolError( sorted(list(multiply_defined_symbols))) - def check_llndk_introduced(self, versions: Iterable[Version]) -> None: - """Raises errors when llndk= is missing for new llndk symbols.""" - if not self.filter.llndk: - return - - def assert_llndk_with_version(tags: Tags, name: str) -> None: - has_llndk_introduced = False - for tag in tags: - if tag.startswith('llndk='): - has_llndk_introduced = True - break - if not has_llndk_introduced: - raise ParseError(f'{name}: missing version. `llndk=yyyymm`') - - arch = self.filter.arch - for version in versions: - # llndk symbols >= introduced=35 should be tagged - # explicitly with llndk=yyyymm. - for symbol in version.symbols: - if not symbol.tags.has_llndk_tags: - continue - if symbol_in_api(symbol.tags, arch, 34): - continue - assert_llndk_with_version(symbol.tags, symbol.name) - def parse_version(self) -> Version: """Parses a single version section and returns a Version object.""" assert self.current_line is not None @@ -487,9 +433,7 @@ class SymbolFileParser: else: raise ParseError('Unknown visiblity label: ' + visibility) elif global_scope and not cpp_symbols: - symbol = self.parse_symbol() - symbol.tags.copy_introduced_from(tags) - symbols.append(symbol) + symbols.append(self.parse_symbol()) else: # We're in a hidden scope or in 'extern "C++"' block. Ignore # everything. diff --git a/cc/symbolfile/test_symbolfile.py b/cc/symbolfile/test_symbolfile.py index 8b412b98a..14bb737ee 100644 --- a/cc/symbolfile/test_symbolfile.py +++ b/cc/symbolfile/test_symbolfile.py @@ -344,45 +344,6 @@ class OmitSymbolTest(unittest.TestCase): self.assertInclude(f_llndk, s_none) self.assertInclude(f_llndk, s_llndk) - def test_omit_llndk_versioned(self) -> None: - f_ndk = self.filter - f_ndk.api = 35 - - f_llndk = copy(f_ndk) - f_llndk.llndk = True - f_llndk.api = 202404 - - s = Symbol('foo', Tags()) - s_llndk = Symbol('foo', Tags.from_strs(['llndk'])) - s_llndk_202404 = Symbol('foo', Tags.from_strs(['llndk=202404'])) - s_34 = Symbol('foo', Tags.from_strs(['introduced=34'])) - s_34_llndk = Symbol('foo', Tags.from_strs(['introduced=34', 'llndk'])) - s_35 = Symbol('foo', Tags.from_strs(['introduced=35'])) - s_35_llndk_202404 = Symbol('foo', Tags.from_strs(['introduced=35', 'llndk=202404'])) - s_35_llndk_202504 = Symbol('foo', Tags.from_strs(['introduced=35', 'llndk=202504'])) - - # When targeting NDK, omit LLNDK tags - self.assertInclude(f_ndk, s) - self.assertOmit(f_ndk, s_llndk) - self.assertOmit(f_ndk, s_llndk_202404) - self.assertInclude(f_ndk, s_34) - self.assertOmit(f_ndk, s_34_llndk) - self.assertInclude(f_ndk, s_35) - self.assertOmit(f_ndk, s_35_llndk_202404) - self.assertOmit(f_ndk, s_35_llndk_202504) - - # When targeting LLNDK, old symbols without any mode tags are included as LLNDK - self.assertInclude(f_llndk, s) - # When targeting LLNDK, old symbols with #llndk are included as LLNDK - self.assertInclude(f_llndk, s_llndk) - self.assertInclude(f_llndk, s_llndk_202404) - self.assertInclude(f_llndk, s_34) - self.assertInclude(f_llndk, s_34_llndk) - # When targeting LLNDK, new symbols(>=35) should be tagged with llndk-introduced=. - self.assertOmit(f_llndk, s_35) - self.assertInclude(f_llndk, s_35_llndk_202404) - self.assertOmit(f_llndk, s_35_llndk_202504) - def test_omit_apex(self) -> None: f_none = self.filter f_apex = copy(f_none) @@ -494,8 +455,8 @@ class SymbolFileParseTest(unittest.TestCase): # should_omit_tags() can differently based on introduced API level when treating # LLNDK-available symbols. expected_symbols = [ - Symbol('baz', Tags.from_strs(['introduced=35'])), - Symbol('qux', Tags.from_strs(['apex', 'llndk', 'introduced=35'])), + Symbol('baz', Tags()), + Symbol('qux', Tags.from_strs(['apex', 'llndk'])), ] self.assertEqual(expected_symbols, version.symbols) @@ -643,19 +604,6 @@ class SymbolFileParseTest(unittest.TestCase): ] self.assertEqual(expected_symbols, version.symbols) - def test_parse_llndk_version_is_missing(self) -> None: - input_file = io.StringIO(textwrap.dedent("""\ - VERSION_1 { # introduced=35 - foo; - bar; # llndk - }; - """)) - f = copy(self.filter) - f.llndk = True - parser = symbolfile.SymbolFileParser(input_file, {}, f) - with self.assertRaises(symbolfile.ParseError): - parser.parse() - def main() -> None: suite = unittest.TestLoader().loadTestsFromName(__name__) diff --git a/cmd/find_input_delta/find_input_delta/main.go b/cmd/find_input_delta/find_input_delta/main.go index 6b657ea57..a8645843c 100644 --- a/cmd/find_input_delta/find_input_delta/main.go +++ b/cmd/find_input_delta/find_input_delta/main.go @@ -85,4 +85,11 @@ func main() { if err = file_list.Format(os.Stdout, template); err != nil { panic(err) } + + metrics_file := os.Getenv("SOONG_METRICS_AGGREGATION_FILE") + if metrics_file != "" { + if err = file_list.SendMetrics(metrics_file); err != nil { + panic(err) + } + } } diff --git a/cmd/find_input_delta/find_input_delta_lib/Android.bp b/cmd/find_input_delta/find_input_delta_lib/Android.bp index 95bdba81d..ef9c65b1d 100644 --- a/cmd/find_input_delta/find_input_delta_lib/Android.bp +++ b/cmd/find_input_delta/find_input_delta_lib/Android.bp @@ -25,6 +25,7 @@ bootstrap_go_package { "golang-protobuf-runtime-protoimpl", "soong-cmd-find_input_delta-proto", "soong-cmd-find_input_delta-proto_internal", + "android-archive-zip", "blueprint-pathtools", ], srcs: [ diff --git a/cmd/find_input_delta/find_input_delta_lib/file_list.go b/cmd/find_input_delta/find_input_delta_lib/file_list.go index 23337adc3..01242a0e7 100644 --- a/cmd/find_input_delta/find_input_delta_lib/file_list.go +++ b/cmd/find_input_delta/find_input_delta_lib/file_list.go @@ -15,10 +15,15 @@ package find_input_delta_lib import ( + "fmt" "io" + "os" + "path/filepath" + "slices" "text/template" fid_exp "android/soong/cmd/find_input_delta/find_input_delta_proto" + "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/proto" ) @@ -47,28 +52,148 @@ type FileList struct { // The modified files Changes []FileList + + // Map of file_extension:counts + ExtCountMap map[string]*FileCounts + + // Total number of added/changed/deleted files. + TotalDelta uint32 +} + +// The maximum number of files that will be recorded by name. +var MaxFilesRecorded uint32 = 50 + +type FileCounts struct { + Additions uint32 + Deletions uint32 + Changes uint32 +} + +func FileListFactory(name string) *FileList { + return &FileList{ + Name: name, + ExtCountMap: make(map[string]*FileCounts), + } +} + +func (fl *FileList) addFile(name string) { + fl.Additions = append(fl.Additions, name) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Additions += 1 +} + +func (fl *FileList) deleteFile(name string) { + fl.Deletions = append(fl.Deletions, name) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Deletions += 1 +} + +func (fl *FileList) changeFile(name string, ch *FileList) { + fl.Changes = append(fl.Changes, *ch) + fl.TotalDelta += 1 + ext := filepath.Ext(name) + if _, ok := fl.ExtCountMap[ext]; !ok { + fl.ExtCountMap[ext] = &FileCounts{} + } + fl.ExtCountMap[ext].Changes += 1 +} + +func (fl FileList) ToProto() (*fid_exp.FileList, error) { + var count uint32 + return fl.toProto(&count) } -func (fl FileList) Marshal() (*fid_exp.FileList, error) { +func (fl FileList) toProto(count *uint32) (*fid_exp.FileList, error) { ret := &fid_exp.FileList{ Name: proto.String(fl.Name), } - if len(fl.Additions) > 0 { - ret.Additions = fl.Additions + for _, a := range fl.Additions { + if *count >= MaxFilesRecorded { + break + } + ret.Additions = append(ret.Additions, a) + *count += 1 } for _, ch := range fl.Changes { - change, err := ch.Marshal() - if err != nil { - return nil, err + if *count >= MaxFilesRecorded { + break + } else { + // Pre-increment to limit what the call adds. + *count += 1 + change, err := ch.toProto(count) + if err != nil { + return nil, err + } + ret.Changes = append(ret.Changes, change) + } + } + for _, d := range fl.Deletions { + if *count >= MaxFilesRecorded { + break } - ret.Changes = append(ret.Changes, change) + ret.Deletions = append(ret.Deletions, d) } - if len(fl.Deletions) > 0 { - ret.Deletions = fl.Deletions + ret.TotalDelta = proto.Uint32(*count) + exts := []string{} + for k := range fl.ExtCountMap { + exts = append(exts, k) + } + slices.Sort(exts) + for _, k := range exts { + v := fl.ExtCountMap[k] + ret.Counts = append(ret.Counts, &fid_exp.FileCount{ + Extension: proto.String(k), + Additions: proto.Uint32(v.Additions), + Deletions: proto.Uint32(v.Deletions), + Modifications: proto.Uint32(v.Changes), + }) } return ret, nil } +func (fl FileList) SendMetrics(path string) error { + if path == "" { + return fmt.Errorf("No path given") + } + message, err := fl.ToProto() + if err != nil { + return err + } + + // Marshal the message wrapped in SoongCombinedMetrics. + data := protowire.AppendVarint( + []byte{}, + protowire.EncodeTag( + protowire.Number(fid_exp.FieldNumbers_FIELD_NUMBERS_FILE_LIST), + protowire.BytesType)) + size := uint64(proto.Size(message)) + data = protowire.AppendVarint(data, size) + data, err = proto.MarshalOptions{UseCachedSize: true}.MarshalAppend(data, message) + if err != nil { + return err + } + + out, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if err := out.Close(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to close %s: %v\n", path, err) + } + }() + _, err = out.Write(data) + return err +} + func (fl FileList) Format(wr io.Writer, format string) error { tmpl, err := template.New("filelist").Parse(format) if err != nil { diff --git a/cmd/find_input_delta/find_input_delta_lib/fs.go b/cmd/find_input_delta/find_input_delta_lib/fs.go index 4a83ed7ff..09a8aa6cb 100644 --- a/cmd/find_input_delta/find_input_delta_lib/fs.go +++ b/cmd/find_input_delta/find_input_delta_lib/fs.go @@ -15,7 +15,6 @@ package find_input_delta_lib import ( - "io" "io/fs" "os" ) @@ -30,14 +29,6 @@ type fileSystem interface { ReadFile(path string) ([]byte, error) } -type file interface { - io.Closer - io.Reader - io.ReaderAt - io.Seeker - Stat() (os.FileInfo, error) -} - // osFS implements fileSystem using the local disk. type osFS struct{} diff --git a/cmd/find_input_delta/find_input_delta_lib/internal_state.go b/cmd/find_input_delta/find_input_delta_lib/internal_state.go index b2ff8c704..2b8c39527 100644 --- a/cmd/find_input_delta/find_input_delta_lib/internal_state.go +++ b/cmd/find_input_delta/find_input_delta_lib/internal_state.go @@ -18,9 +18,11 @@ import ( "errors" "fmt" "io/fs" + "regexp" "slices" fid_proto "android/soong/cmd/find_input_delta/find_input_delta_proto_internal" + "android/soong/third_party/zip" "github.com/google/blueprint/pathtools" "google.golang.org/protobuf/proto" ) @@ -57,6 +59,7 @@ func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (* // If we ever have an easy hash, assign it here. } if inspect_contents { + // NOTE: When we find it useful, we can parallelize the file inspection for speed. contents, err := InspectFileContents(input) if err != nil { return ret, err @@ -70,14 +73,36 @@ func CreateState(inputs []string, inspect_contents bool, fsys StatReadFileFS) (* return ret, nil } +// We ignore any suffix digit caused by sharding. +var InspectExtsZipRegexp = regexp.MustCompile("\\.(jar|apex|apk)[0-9]*$") + // Inspect the file and extract the state of the elements in the archive. // If this is not an archive of some sort, nil is returned. func InspectFileContents(name string) ([]*fid_proto.PartialCompileInput, error) { - // TODO: Actually inspect the contents. - fmt.Printf("inspecting contents for %s\n", name) + if InspectExtsZipRegexp.Match([]byte(name)) { + return inspectZipFileContents(name) + } return nil, nil } +func inspectZipFileContents(name string) ([]*fid_proto.PartialCompileInput, error) { + rc, err := zip.OpenReader(name) + if err != nil { + return nil, err + } + ret := []*fid_proto.PartialCompileInput{} + for _, v := range rc.File { + pci := &fid_proto.PartialCompileInput{ + Name: proto.String(v.Name), + MtimeNsec: proto.Int64(v.ModTime().UnixNano()), + Hash: proto.String(fmt.Sprintf("%08x", v.CRC32)), + } + ret = append(ret, pci) + // We do not support nested inspection. + } + return ret, nil +} + func WriteState(s *fid_proto.PartialCompileInputs, path string) error { data, err := proto.Marshal(s) if err != nil { @@ -91,9 +116,7 @@ func CompareInternalState(prior, other *fid_proto.PartialCompileInputs, target s } func CompareInputFiles(prior, other []*fid_proto.PartialCompileInput, name string) *FileList { - fl := &FileList{ - Name: name, - } + fl := FileListFactory(name) PriorMap := make(map[string]*fid_proto.PartialCompileInput, len(prior)) // We know that the lists are properly sorted, so we can simply compare them. for _, v := range prior { @@ -105,17 +128,17 @@ func CompareInputFiles(prior, other []*fid_proto.PartialCompileInput, name strin otherMap[name] = v if _, ok := PriorMap[name]; !ok { // Added file - fl.Additions = append(fl.Additions, name) + fl.addFile(name) } else if !proto.Equal(PriorMap[name], v) { // Changed file - fl.Changes = append(fl.Changes, *CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name)) + fl.changeFile(name, CompareInputFiles(PriorMap[name].GetContents(), v.GetContents(), name)) } } for _, v := range prior { name := v.GetName() if _, ok := otherMap[name]; !ok { // Deleted file - fl.Deletions = append(fl.Deletions, name) + fl.deleteFile(name) } } return fl diff --git a/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go b/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go index 20b8efaa8..c168d5a6b 100644 --- a/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go +++ b/cmd/find_input_delta/find_input_delta_lib/internal_state_test.go @@ -199,7 +199,7 @@ func TestCompareInternalState(t *testing.T) { }, }, { - Name: "one of each", + Name: "one each add modify delete", Target: "foo", Prior: &fid_proto.PartialCompileInputs{ InputFiles: []*fid_proto.PartialCompileInput{ @@ -222,11 +222,62 @@ func TestCompareInternalState(t *testing.T) { Deletions: []string{"file2"}, }, }, + { + Name: "interior one each add modify delete", + Target: "bar", + Prior: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file1", 405, "", []*fid_proto.PartialCompileInput{ + protoFile("innerC", 400, "crc32:11111111", nil), + protoFile("innerD", 400, "crc32:44444444", nil), + }), + }, + }, + New: &fid_proto.PartialCompileInputs{ + InputFiles: []*fid_proto.PartialCompileInput{ + protoFile("file1", 505, "", []*fid_proto.PartialCompileInput{ + protoFile("innerA", 400, "crc32:55555555", nil), + protoFile("innerC", 500, "crc32:66666666", nil), + }), + }, + }, + Expected: &FileList{ + Name: "bar", + Changes: []FileList{FileList{ + Name: "file1", + Additions: []string{"innerA"}, + Changes: []FileList{FileList{Name: "innerC"}}, + Deletions: []string{"innerD"}, + }}, + }, + }, } for _, tc := range testCases { actual := CompareInternalState(tc.Prior, tc.New, tc.Target) if !tc.Expected.Equal(actual) { - t.Errorf("%s: expected %q, actual %q", tc.Name, tc.Expected, actual) + t.Errorf("%s: expected %v, actual %v", tc.Name, tc.Expected, actual) + } + } +} + +func TestCompareInspectExtsZipRegexp(t *testing.T) { + testCases := []struct { + Name string + Expected bool + }{ + {Name: ".jar", Expected: true}, + {Name: ".jar5", Expected: true}, + {Name: ".apex", Expected: true}, + {Name: ".apex9", Expected: true}, + {Name: ".apexx", Expected: false}, + {Name: ".apk", Expected: true}, + {Name: ".apk3", Expected: true}, + {Name: ".go", Expected: false}, + } + for _, tc := range testCases { + actual := InspectExtsZipRegexp.Match([]byte(tc.Name)) + if tc.Expected != actual { + t.Errorf("%s: expected %v, actual %v", tc.Name, tc.Expected, actual) } } } diff --git a/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go b/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go index 648ef224a..745de2db2 100644 --- a/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go +++ b/cmd/find_input_delta/find_input_delta_proto/file_list.pb.go @@ -19,7 +19,7 @@ // protoc v3.21.12 // source: file_list.proto -package proto +package find_input_delta_proto import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -35,6 +35,62 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type FieldNumbers int32 + +const ( + FieldNumbers_FIELD_NUMBERS_UNSPECIFIED FieldNumbers = 0 + FieldNumbers_FIELD_NUMBERS_FILE_LIST FieldNumbers = 1 +) + +// Enum value maps for FieldNumbers. +var ( + FieldNumbers_name = map[int32]string{ + 0: "FIELD_NUMBERS_UNSPECIFIED", + 1: "FIELD_NUMBERS_FILE_LIST", + } + FieldNumbers_value = map[string]int32{ + "FIELD_NUMBERS_UNSPECIFIED": 0, + "FIELD_NUMBERS_FILE_LIST": 1, + } +) + +func (x FieldNumbers) Enum() *FieldNumbers { + p := new(FieldNumbers) + *p = x + return p +} + +func (x FieldNumbers) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FieldNumbers) Descriptor() protoreflect.EnumDescriptor { + return file_file_list_proto_enumTypes[0].Descriptor() +} + +func (FieldNumbers) Type() protoreflect.EnumType { + return &file_file_list_proto_enumTypes[0] +} + +func (x FieldNumbers) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *FieldNumbers) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = FieldNumbers(num) + return nil +} + +// Deprecated: Use FieldNumbers.Descriptor instead. +func (FieldNumbers) EnumDescriptor() ([]byte, []int) { + return file_file_list_proto_rawDescGZIP(), []int{0} +} + type FileList struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -46,10 +102,14 @@ type FileList struct { Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` // The added files. Additions []string `protobuf:"bytes,2,rep,name=additions" json:"additions,omitempty"` - // The deleted files. - Deletions []string `protobuf:"bytes,3,rep,name=deletions" json:"deletions,omitempty"` // The changed files. - Changes []*FileList `protobuf:"bytes,4,rep,name=changes" json:"changes,omitempty"` + Changes []*FileList `protobuf:"bytes,3,rep,name=changes" json:"changes,omitempty"` + // The deleted files. + Deletions []string `protobuf:"bytes,4,rep,name=deletions" json:"deletions,omitempty"` + // Count of files added/changed/deleted. + TotalDelta *uint32 `protobuf:"varint,5,opt,name=total_delta,json=totalDelta" json:"total_delta,omitempty"` + // Counts by extension. + Counts []*FileCount `protobuf:"bytes,6,rep,name=counts" json:"counts,omitempty"` } func (x *FileList) Reset() { @@ -98,6 +158,13 @@ func (x *FileList) GetAdditions() []string { return nil } +func (x *FileList) GetChanges() []*FileList { + if x != nil { + return x.Changes + } + return nil +} + func (x *FileList) GetDeletions() []string { if x != nil { return x.Deletions @@ -105,32 +172,135 @@ func (x *FileList) GetDeletions() []string { return nil } -func (x *FileList) GetChanges() []*FileList { +func (x *FileList) GetTotalDelta() uint32 { + if x != nil && x.TotalDelta != nil { + return *x.TotalDelta + } + return 0 +} + +func (x *FileList) GetCounts() []*FileCount { if x != nil { - return x.Changes + return x.Counts } return nil } +type FileCount struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The file extension + Extension *string `protobuf:"bytes,1,opt,name=extension" json:"extension,omitempty"` + // Number of added files with this extension. + Additions *uint32 `protobuf:"varint,2,opt,name=additions" json:"additions,omitempty"` + // Number of modified files with this extension. + Modifications *uint32 `protobuf:"varint,3,opt,name=modifications" json:"modifications,omitempty"` + // Number of deleted files with this extension. + Deletions *uint32 `protobuf:"varint,4,opt,name=deletions" json:"deletions,omitempty"` +} + +func (x *FileCount) Reset() { + *x = FileCount{} + if protoimpl.UnsafeEnabled { + mi := &file_file_list_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileCount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileCount) ProtoMessage() {} + +func (x *FileCount) ProtoReflect() protoreflect.Message { + mi := &file_file_list_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileCount.ProtoReflect.Descriptor instead. +func (*FileCount) Descriptor() ([]byte, []int) { + return file_file_list_proto_rawDescGZIP(), []int{1} +} + +func (x *FileCount) GetExtension() string { + if x != nil && x.Extension != nil { + return *x.Extension + } + return "" +} + +func (x *FileCount) GetAdditions() uint32 { + if x != nil && x.Additions != nil { + return *x.Additions + } + return 0 +} + +func (x *FileCount) GetModifications() uint32 { + if x != nil && x.Modifications != nil { + return *x.Modifications + } + return 0 +} + +func (x *FileCount) GetDeletions() uint32 { + if x != nil && x.Deletions != nil { + return *x.Deletions + } + return 0 +} + var File_file_list_proto protoreflect.FileDescriptor var file_file_list_proto_rawDesc = []byte{ 0x0a, 0x0f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0x9e, 0x01, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x12, + 0x6f, 0x22, 0x82, 0x02, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x42, - 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, - 0x65, 0x73, 0x42, 0x26, 0x5a, 0x24, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, - 0x6f, 0x6e, 0x67, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, - 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x42, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, + 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x6e, 0x67, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x64, 0x65, 0x6c, 0x74, + 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x65, + 0x6c, 0x74, 0x61, 0x12, 0x41, 0x0a, 0x06, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, + 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x06, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x65, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x2a, 0x4a, 0x0a, 0x0c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4e, 0x75, 0x6d, + 0x62, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x4e, 0x55, + 0x4d, 0x42, 0x45, 0x52, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x5f, 0x4e, 0x55, 0x4d, + 0x42, 0x45, 0x52, 0x53, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x01, + 0x42, 0x3b, 0x5a, 0x39, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, + 0x67, 0x2f, 0x63, 0x6d, 0x64, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -145,17 +315,21 @@ func file_file_list_proto_rawDescGZIP() []byte { return file_file_list_proto_rawDescData } -var file_file_list_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_file_list_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_file_list_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_file_list_proto_goTypes = []interface{}{ - (*FileList)(nil), // 0: android.find_input_delta_proto.FileList + (FieldNumbers)(0), // 0: android.find_input_delta_proto.FieldNumbers + (*FileList)(nil), // 1: android.find_input_delta_proto.FileList + (*FileCount)(nil), // 2: android.find_input_delta_proto.FileCount } var file_file_list_proto_depIdxs = []int32{ - 0, // 0: android.find_input_delta_proto.FileList.changes:type_name -> android.find_input_delta_proto.FileList - 1, // [1:1] is the sub-list for method output_type - 1, // [1:1] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 1, // 0: android.find_input_delta_proto.FileList.changes:type_name -> android.find_input_delta_proto.FileList + 2, // 1: android.find_input_delta_proto.FileList.counts:type_name -> android.find_input_delta_proto.FileCount + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_file_list_proto_init() } @@ -176,19 +350,32 @@ func file_file_list_proto_init() { return nil } } + file_file_list_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileCount); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_file_list_proto_rawDesc, - NumEnums: 0, - NumMessages: 1, + NumEnums: 1, + NumMessages: 2, NumExtensions: 0, NumServices: 0, }, GoTypes: file_file_list_proto_goTypes, DependencyIndexes: file_file_list_proto_depIdxs, + EnumInfos: file_file_list_proto_enumTypes, MessageInfos: file_file_list_proto_msgTypes, }.Build() File_file_list_proto = out.File diff --git a/cmd/find_input_delta/find_input_delta_proto/file_list.proto b/cmd/find_input_delta/find_input_delta_proto/file_list.proto index d7faca9c8..7180358c8 100644 --- a/cmd/find_input_delta/find_input_delta_proto/file_list.proto +++ b/cmd/find_input_delta/find_input_delta_proto/file_list.proto @@ -15,7 +15,12 @@ syntax = "proto2"; package android.find_input_delta_proto; -option go_package = "android/soong/find_input_delta/proto"; +option go_package = "android/soong/cmd/find_input_delta/find_input_delta_proto"; + +enum FieldNumbers { + FIELD_NUMBERS_UNSPECIFIED = 0; + FIELD_NUMBERS_FILE_LIST = 1; +} message FileList { // The name of the file. @@ -26,9 +31,29 @@ message FileList { // The added files. repeated string additions = 2; + // The changed files. + repeated FileList changes = 3; + // The deleted files. - repeated string deletions = 3; + repeated string deletions = 4; - // The changed files. - repeated FileList changes = 4; + // Count of files added/changed/deleted. + optional uint32 total_delta = 5; + + // Counts by extension. + repeated FileCount counts = 6; +} + +message FileCount { + // The file extension + optional string extension = 1; + + // Number of added files with this extension. + optional uint32 additions = 2; + + // Number of modified files with this extension. + optional uint32 modifications = 3; + + // Number of deleted files with this extension. + optional uint32 deletions = 4; } diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go index 2229a324d..c5b048bb8 100644 --- a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go +++ b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.pb.go @@ -19,7 +19,7 @@ // protoc v3.21.12 // source: internal_state.proto -package proto +package find_input_delta_proto_internal import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -94,7 +94,7 @@ type PartialCompileInput struct { Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` // The timestamp of the file in (Unix) nanoseconds. MtimeNsec *int64 `protobuf:"varint,2,opt,name=mtime_nsec,json=mtimeNsec" json:"mtime_nsec,omitempty"` - // The hash of the file, in the form ‘{HASHNAME}:{VALUE}’ + // The hash of the file. For crc32 hashes, this will be 8 hex digits. Hash *string `protobuf:"bytes,3,opt,name=hash" json:"hash,omitempty"` // Contents of the file, if the file was inspected (such as jar files, etc). Contents []*PartialCompileInput `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty"` @@ -164,29 +164,33 @@ var File_internal_state_proto protoreflect.FileDescriptor var file_internal_state_proto_rawDesc = []byte{ 0x0a, 0x14, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x27, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6c, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, - 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x54, - 0x0a, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, - 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, - 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x46, - 0x69, 0x6c, 0x65, 0x73, 0x22, 0xad, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, - 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, 0x63, 0x12, - 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, - 0x61, 0x73, 0x68, 0x12, 0x4f, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, - 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, - 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, - 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x73, 0x42, 0x26, 0x5a, 0x24, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, - 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, - 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x22, + 0x75, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, + 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x5d, 0x0a, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x61, + 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, + 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x0a, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x22, 0xb6, 0x01, 0x0a, 0x13, 0x50, 0x61, 0x72, 0x74, 0x69, + 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, + 0x63, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x58, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, + 0x64, 0x2e, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, + 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, + 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x42, + 0x40, 0x5a, 0x3e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, + 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, + 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, + 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, } var ( @@ -203,12 +207,12 @@ func file_internal_state_proto_rawDescGZIP() []byte { var file_internal_state_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_internal_state_proto_goTypes = []interface{}{ - (*PartialCompileInputs)(nil), // 0: android.find_input_delta_proto.PartialCompileInputs - (*PartialCompileInput)(nil), // 1: android.find_input_delta_proto.PartialCompileInput + (*PartialCompileInputs)(nil), // 0: android.find_input_delta_proto_internal.PartialCompileInputs + (*PartialCompileInput)(nil), // 1: android.find_input_delta_proto_internal.PartialCompileInput } var file_internal_state_proto_depIdxs = []int32{ - 1, // 0: android.find_input_delta_proto.PartialCompileInputs.input_files:type_name -> android.find_input_delta_proto.PartialCompileInput - 1, // 1: android.find_input_delta_proto.PartialCompileInput.contents:type_name -> android.find_input_delta_proto.PartialCompileInput + 1, // 0: android.find_input_delta_proto_internal.PartialCompileInputs.input_files:type_name -> android.find_input_delta_proto_internal.PartialCompileInput + 1, // 1: android.find_input_delta_proto_internal.PartialCompileInput.contents:type_name -> android.find_input_delta_proto_internal.PartialCompileInput 2, // [2:2] is the sub-list for method output_type 2, // [2:2] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name diff --git a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto index 113fc6434..54c90cc72 100644 --- a/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto +++ b/cmd/find_input_delta/find_input_delta_proto_internal/internal_state.proto @@ -14,8 +14,8 @@ // limitations under the License. syntax = "proto2"; -package android.find_input_delta_proto; -option go_package = "android/soong/find_input_delta/proto"; +package android.find_input_delta_proto_internal; +option go_package = "android/soong/find_input_delta/find_input_delta_proto_internal"; // The state of all inputs. message PartialCompileInputs { @@ -31,7 +31,7 @@ message PartialCompileInput { // The timestamp of the file in (Unix) nanoseconds. optional int64 mtime_nsec = 2; - // The hash of the file, in the form ‘{HASHNAME}:{VALUE}’ + // The hash of the file. For crc32 hashes, this will be 8 hex digits. optional string hash = 3; // Contents of the file, if the file was inspected (such as jar files, etc). diff --git a/filesystem/bootimg.go b/filesystem/bootimg.go index c9bd61788..0ffec2654 100644 --- a/filesystem/bootimg.go +++ b/filesystem/bootimg.go @@ -26,19 +26,21 @@ import ( ) func init() { - android.RegisterModuleType("bootimg", bootimgFactory) + android.RegisterModuleType("bootimg", BootimgFactory) } type bootimg struct { android.ModuleBase - properties bootimgProperties + properties BootimgProperties output android.Path installDir android.InstallPath + + bootImageType bootImageType } -type bootimgProperties struct { +type BootimgProperties struct { // Set the name of the output. Defaults to <module_name>.img. Stem *string @@ -56,9 +58,13 @@ type bootimgProperties struct { // https://source.android.com/devices/bootloader/boot-image-header Header_version *string - // Determines if this image is for the vendor_boot partition. Default is false. Refer to - // https://source.android.com/devices/bootloader/partitions/vendor-boot-partitions - Vendor_boot *bool + // Determines the specific type of boot image this module is building. Can be boot, + // vendor_boot or init_boot. Defaults to boot. + // Refer to https://source.android.com/devices/bootloader/partitions/vendor-boot-partitions + // for vendor_boot. + // Refer to https://source.android.com/docs/core/architecture/partitions/generic-boot for + // init_boot. + Boot_image_type *string // Optional kernel commandline arguments Cmdline []string `android:"arch_variant"` @@ -67,9 +73,19 @@ type bootimgProperties struct { // and `header_version` is greater than or equal to 4. Bootconfig *string `android:"arch_variant,path"` + // The size of the partition on the device. It will be a build error if this built partition + // image exceeds this size. + Partition_size *int64 + // When set to true, sign the image with avbtool. Default is false. Use_avb *bool + // This can either be "default", or "make_legacy". "make_legacy" will sign the boot image + // like how build/make/core/Makefile does, to get bit-for-bit backwards compatibility. But + // we may want to reconsider if it's necessary to have two modes in the future. The default + // is "default" + Avb_mode *string + // Name of the partition stored in vbmeta desc. Defaults to the name of this module. Partition_name *string @@ -79,10 +95,65 @@ type bootimgProperties struct { // Hash and signing algorithm for avbtool. Default is SHA256_RSA4096. Avb_algorithm *string + + // The index used to prevent rollback of the image on device. + Avb_rollback_index *int64 + + // The security patch passed to as the com.android.build.<type>.security_patch avb property. + // Replacement for the make variables BOOT_SECURITY_PATCH / INIT_BOOT_SECURITY_PATCH. + Security_patch *string +} + +type bootImageType int + +const ( + unsupported bootImageType = iota + boot + vendorBoot + initBoot +) + +func toBootImageType(ctx android.ModuleContext, bootImageType string) bootImageType { + switch bootImageType { + case "boot": + return boot + case "vendor_boot": + return vendorBoot + case "init_boot": + return initBoot + default: + ctx.ModuleErrorf("Unknown boot_image_type %s. Must be one of \"boot\", \"vendor_boot\", or \"init_boot\"", bootImageType) + } + return unsupported +} + +func (b bootImageType) String() string { + switch b { + case boot: + return "boot" + case vendorBoot: + return "vendor_boot" + case initBoot: + return "init_boot" + default: + panic("unknown boot image type") + } +} + +func (b bootImageType) isBoot() bool { + return b == boot +} + +func (b bootImageType) isVendorBoot() bool { + return b == vendorBoot +} + +func (b bootImageType) isInitBoot() bool { + return b == initBoot } // bootimg is the image for the boot partition. It consists of header, kernel, ramdisk, and dtb. -func bootimgFactory() android.Module { +func BootimgFactory() android.Module { module := &bootimg{} module.AddProperties(&module.properties) android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) @@ -112,12 +183,40 @@ func (b *bootimg) partitionName() string { } func (b *bootimg) GenerateAndroidBuildActions(ctx android.ModuleContext) { - vendor := proptools.Bool(b.properties.Vendor_boot) - unsignedOutput := b.buildBootImage(ctx, vendor) + b.bootImageType = toBootImageType(ctx, proptools.StringDefault(b.properties.Boot_image_type, "boot")) + if b.bootImageType == unsupported { + return + } + + kernelProp := proptools.String(b.properties.Kernel_prebuilt) + if b.bootImageType.isVendorBoot() && kernelProp != "" { + ctx.PropertyErrorf("kernel_prebuilt", "vendor_boot partition can't have kernel") + return + } + if b.bootImageType.isBoot() && kernelProp == "" { + ctx.PropertyErrorf("kernel_prebuilt", "boot partition must have kernel") + return + } + var kernel android.Path + if kernelProp != "" { + kernel = android.PathForModuleSrc(ctx, kernelProp) + } + + unsignedOutput := b.buildBootImage(ctx, kernel) output := unsignedOutput if proptools.Bool(b.properties.Use_avb) { - output = b.signImage(ctx, unsignedOutput) + // This bootimg module supports 2 modes of avb signing. It is not clear to this author + // why there are differences, but one of them is to match the behavior of make-built boot + // images. + switch proptools.StringDefault(b.properties.Avb_mode, "default") { + case "default": + output = b.signImage(ctx, unsignedOutput) + case "make_legacy": + output = b.addAvbFooter(ctx, unsignedOutput, kernel) + default: + ctx.PropertyErrorf("avb_mode", `Unknown value for avb_mode, expected "default" or "make_legacy", got: %q`, *b.properties.Avb_mode) + } } b.installDir = android.PathForModuleInstall(ctx, "etc") @@ -127,23 +226,20 @@ func (b *bootimg) GenerateAndroidBuildActions(ctx android.ModuleContext) { b.output = output } -func (b *bootimg) buildBootImage(ctx android.ModuleContext, vendor bool) android.Path { +func (b *bootimg) buildBootImage(ctx android.ModuleContext, kernel android.Path) android.Path { output := android.PathForModuleOut(ctx, "unsigned", b.installFileName()) builder := android.NewRuleBuilder(pctx, ctx) cmd := builder.Command().BuiltTool("mkbootimg") - kernel := proptools.String(b.properties.Kernel_prebuilt) - if vendor && kernel != "" { - ctx.PropertyErrorf("kernel_prebuilt", "vendor_boot partition can't have kernel") - return output - } - if !vendor && kernel == "" { - ctx.PropertyErrorf("kernel_prebuilt", "boot partition must have kernel") - return output + if kernel != nil { + cmd.FlagWithInput("--kernel ", kernel) } - if kernel != "" { - cmd.FlagWithInput("--kernel ", android.PathForModuleSrc(ctx, kernel)) + + // These arguments are passed for boot.img and init_boot.img generation + if b.bootImageType.isBoot() || b.bootImageType.isInitBoot() { + cmd.FlagWithArg("--os_version ", ctx.Config().PlatformVersionLastStable()) + cmd.FlagWithArg("--os_patch_level ", ctx.Config().PlatformSecurityPatch()) } dtbName := proptools.String(b.properties.Dtb_prebuilt) @@ -155,7 +251,7 @@ func (b *bootimg) buildBootImage(ctx android.ModuleContext, vendor bool) android cmdline := strings.Join(b.properties.Cmdline, " ") if cmdline != "" { flag := "--cmdline " - if vendor { + if b.bootImageType.isVendorBoot() { flag = "--vendor_cmdline " } cmd.FlagWithArg(flag, proptools.ShellEscapeIncludingSpaces(cmdline)) @@ -182,7 +278,7 @@ func (b *bootimg) buildBootImage(ctx android.ModuleContext, vendor bool) android ramdisk := ctx.GetDirectDepWithTag(ramdiskName, bootimgRamdiskDep) if filesystem, ok := ramdisk.(*filesystem); ok { flag := "--ramdisk " - if vendor { + if b.bootImageType.isVendorBoot() { flag = "--vendor_ramdisk " } cmd.FlagWithInput(flag, filesystem.OutputPath()) @@ -194,7 +290,7 @@ func (b *bootimg) buildBootImage(ctx android.ModuleContext, vendor bool) android bootconfig := proptools.String(b.properties.Bootconfig) if bootconfig != "" { - if !vendor { + if !b.bootImageType.isVendorBoot() { ctx.PropertyErrorf("bootconfig", "requires vendor_boot: true") return output } @@ -205,16 +301,81 @@ func (b *bootimg) buildBootImage(ctx android.ModuleContext, vendor bool) android cmd.FlagWithInput("--vendor_bootconfig ", android.PathForModuleSrc(ctx, bootconfig)) } + // Output flag for boot.img and init_boot.img flag := "--output " - if vendor { + if b.bootImageType.isVendorBoot() { flag = "--vendor_boot " } cmd.FlagWithOutput(flag, output) + if b.properties.Partition_size != nil { + assertMaxImageSize(builder, output, *b.properties.Partition_size, proptools.Bool(b.properties.Use_avb)) + } + builder.Build("build_bootimg", fmt.Sprintf("Creating %s", b.BaseModuleName())) return output } +func (b *bootimg) addAvbFooter(ctx android.ModuleContext, unsignedImage android.Path, kernel android.Path) android.Path { + output := android.PathForModuleOut(ctx, b.installFileName()) + builder := android.NewRuleBuilder(pctx, ctx) + builder.Command().Text("cp").Input(unsignedImage).Output(output) + cmd := builder.Command().BuiltTool("avbtool"). + Text("add_hash_footer"). + FlagWithInput("--image ", output) + + if b.properties.Partition_size != nil { + cmd.FlagWithArg("--partition_size ", strconv.FormatInt(*b.properties.Partition_size, 10)) + } else { + cmd.Flag("--dynamic_partition_size") + } + + // If you don't provide a salt, avbtool will use random bytes for the salt. + // This is bad for determinism (cached builds and diff tests are affected), so instead, + // we try to provide a salt. The requirements for a salt are not very clear, one aspect of it + // is that if it's unpredictable, attackers trying to change the contents of a partition need + // to find a new hash collision every release, because the salt changed. + if kernel != nil { + cmd.Textf(`--salt $(sha256sum "%s" | cut -d " " -f 1)`, kernel.String()) + cmd.Implicit(kernel) + } else { + cmd.Textf(`--salt $(sha256sum "%s" "%s" | cut -d " " -f 1 | tr -d '\n')`, ctx.Config().BuildNumberFile(ctx), ctx.Config().Getenv("BUILD_DATETIME_FILE")) + cmd.OrderOnly(ctx.Config().BuildNumberFile(ctx)) + } + + cmd.FlagWithArg("--partition_name ", b.bootImageType.String()) + + if b.properties.Avb_algorithm != nil { + cmd.FlagWithArg("--algorithm ", proptools.NinjaAndShellEscape(*b.properties.Avb_algorithm)) + } + + if b.properties.Avb_private_key != nil { + key := android.PathForModuleSrc(ctx, proptools.String(b.properties.Avb_private_key)) + cmd.FlagWithInput("--key ", key) + } + + if !b.bootImageType.isVendorBoot() { + cmd.FlagWithArg("--prop ", proptools.NinjaAndShellEscape(fmt.Sprintf( + "com.android.build.%s.os_version:%s", b.bootImageType.String(), ctx.Config().PlatformVersionLastStable()))) + } + + fingerprintFile := ctx.Config().BuildFingerprintFile(ctx) + cmd.FlagWithArg("--prop ", fmt.Sprintf("com.android.build.%s.fingerprint:$(cat %s)", b.bootImageType.String(), fingerprintFile.String())) + cmd.OrderOnly(fingerprintFile) + + if b.properties.Security_patch != nil { + cmd.FlagWithArg("--prop ", proptools.NinjaAndShellEscape(fmt.Sprintf( + "com.android.build.%s.security_patch:%s", b.bootImageType.String(), *b.properties.Security_patch))) + } + + if b.properties.Avb_rollback_index != nil { + cmd.FlagWithArg("--rollback_index ", strconv.FormatInt(*b.properties.Avb_rollback_index, 10)) + } + + builder.Build("add_avb_footer", fmt.Sprintf("Adding avb footer to %s", b.BaseModuleName())) + return output +} + func (b *bootimg) signImage(ctx android.ModuleContext, unsignedImage android.Path) android.Path { propFile, toolDeps := b.buildPropFile(ctx) diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go index c34677060..dadacae3d 100644 --- a/filesystem/filesystem.go +++ b/filesystem/filesystem.go @@ -162,6 +162,10 @@ type FilesystemProperties struct { // Determines if the module is auto-generated from Soong or not. If the module is // auto-generated, its deps are exempted from visibility enforcement. Is_auto_generated *bool + + // Path to the dev nodes description file. This is only needed for building the ramdisk + // partition and should not be explicitly specified. + Dev_nodes_description_file *string `android:"path" blueprint:"mutated"` } // Additional properties required to generate erofs FS partitions. @@ -210,6 +214,10 @@ func initFilesystemModule(module android.DefaultableModule, filesystemModule *fi filesystemModule.PackagingBase.AllowHighPriorityDeps = true android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) android.InitDefaultableModule(module) + + android.AddLoadHook(module, func(ctx android.LoadHookContext) { + filesystemModule.setDevNodesDescriptionProp() + }) } type depTag struct { @@ -229,6 +237,16 @@ func (t depTagWithVisibilityEnforcementBypass) ExcludeFromVisibilityEnforcement( var dependencyTagWithVisibilityEnforcementBypass = depTagWithVisibilityEnforcementBypass{} +// ramdiskDevNodesDescription is the name of the filegroup module that provides the file that +// contains the description of dev nodes added to the CPIO archive for the ramdisk partition. +const ramdiskDevNodesDescription = "ramdisk_node_list" + +func (f *filesystem) setDevNodesDescriptionProp() { + if proptools.String(f.properties.Partition_name) == "ramdisk" { + f.properties.Dev_nodes_description_file = proptools.StringPtr(":" + ramdiskDevNodesDescription) + } +} + func (f *filesystem) DepsMutator(ctx android.BottomUpMutatorContext) { if proptools.Bool(f.properties.Is_auto_generated) { f.AddDeps(ctx, dependencyTagWithVisibilityEnforcementBypass) @@ -659,6 +677,9 @@ func (f *filesystem) buildCpioImage(ctx android.ModuleContext, compressed bool) cmd := builder.Command(). BuiltTool("mkbootfs"). Text(rootDir.String()) // input directory + if nodeList := f.properties.Dev_nodes_description_file; nodeList != nil { + cmd.FlagWithInput("-n ", android.PathForModuleSrc(ctx, proptools.String(nodeList))) + } if compressed { cmd.Text("|"). BuiltTool("lz4"). @@ -689,6 +710,7 @@ var validPartitions = []string{ "odm_dlkm", "system_dlkm", "ramdisk", + "vendor_ramdisk", } func (f *filesystem) addMakeBuiltFiles(ctx android.ModuleContext, builder *android.RuleBuilder, rootDir android.Path) { @@ -904,3 +926,26 @@ func (f *filesystem) getLibsForLinkerConfig(ctx android.ModuleContext) ([]androi return provideModules, requireModules } + +// Checks that the given file doesn't exceed the given size, and will also print a warning +// if it's nearing the maximum size. Equivalent to assert-max-image-size in make: +// https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/definitions.mk;l=3455;drc=993c4de29a02a6accd60ceaaee153307e1a18d10 +func assertMaxImageSize(builder *android.RuleBuilder, image android.Path, maxSize int64, addAvbLater bool) { + if addAvbLater { + // The value 69632 is derived from MAX_VBMETA_SIZE + MAX_FOOTER_SIZE in avbtool. + // Logic copied from make: + // https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/Makefile;l=228;drc=a6a0007ef24e16c0b79f439beac4a118416717e6 + maxSize -= 69632 + } + cmd := builder.Command() + cmd.Textf(`file="%s"; maxsize="%d";`+ + `total=$(stat -c "%%s" "$file" | tr -d '\n');`+ + `if [ "$total" -gt "$maxsize" ]; then `+ + ` echo "error: $file too large ($total > $maxsize)";`+ + ` false;`+ + `elif [ "$total" -gt $((maxsize - 32768)) ]; then `+ + ` echo "WARNING: $file approaching size limit ($total now; limit $maxsize)";`+ + `fi`, + image, maxSize) + cmd.Implicit(image) +} diff --git a/filesystem/filesystem_test.go b/filesystem/filesystem_test.go index f325d96ef..746e4de74 100644 --- a/filesystem/filesystem_test.go +++ b/filesystem/filesystem_test.go @@ -666,7 +666,9 @@ func TestUseSharedVariationOfNativeLib(t *testing.T) { partition := result.ModuleForTests("myfilesystem", "android_common") fileList := android.ContentFromFileRuleForTests(t, result.TestContext, partition.Output("fileList")) - android.AssertDeepEquals(t, "cc_library listed in deps", "lib64/libc++.so\nlib64/libc.so\nlib64/libdl.so\nlib64/libfoo.so\nlib64/libm.so\n", fileList) + android.AssertDeepEquals(t, "cc_library listed in deps", + "lib64/bootstrap/libc.so\nlib64/bootstrap/libdl.so\nlib64/bootstrap/libm.so\nlib64/libc++.so\nlib64/libc.so\nlib64/libdl.so\nlib64/libfoo.so\nlib64/libm.so\n", + fileList) } // binfoo1 overrides binbar. transitive deps of binbar should not be installed. @@ -701,7 +703,9 @@ cc_library { partition := result.ModuleForTests("myfilesystem", "android_common") fileList := android.ContentFromFileRuleForTests(t, result.TestContext, partition.Output("fileList")) - android.AssertDeepEquals(t, "Shared library dep of overridden binary should not be installed", fileList, "bin/binfoo1\nlib64/libc++.so\nlib64/libc.so\nlib64/libdl.so\nlib64/libfoo2.so\nlib64/libm.so\n") + android.AssertDeepEquals(t, "Shared library dep of overridden binary should not be installed", + "bin/binfoo1\nlib64/bootstrap/libc.so\nlib64/bootstrap/libdl.so\nlib64/bootstrap/libm.so\nlib64/libc++.so\nlib64/libc.so\nlib64/libdl.so\nlib64/libfoo2.so\nlib64/libm.so\n", + fileList) } func TestInstallLinkerConfigFile(t *testing.T) { @@ -754,3 +758,28 @@ func TestOverrideModulesInDeps(t *testing.T) { fileList := android.ContentFromFileRuleForTests(t, result.TestContext, partition.Output("fileList")) android.AssertStringEquals(t, "filesystem with override app", "app/myoverrideapp/myoverrideapp.apk\n", fileList) } + +func TestRamdiskPartitionSetsDevNodes(t *testing.T) { + result := android.GroupFixturePreparers( + fixture, + android.FixtureMergeMockFs(android.MockFS{ + "ramdisk_node_list": nil, + }), + ).RunTestWithBp(t, ` + android_filesystem { + name: "ramdisk_filesystem", + partition_name: "ramdisk", + } + filegroup { + name: "ramdisk_node_list", + srcs: ["ramdisk_node_list"], + } + `) + + android.AssertBoolEquals( + t, + "Generated ramdisk image expected to depend on \"ramdisk_node_list\" module", + true, + java.CheckModuleHasDependency(t, result.TestContext, "ramdisk_filesystem", "android_common", "ramdisk_node_list"), + ) +} diff --git a/fsgen/Android.bp b/fsgen/Android.bp index 8cd7518cb..a0225811e 100644 --- a/fsgen/Android.bp +++ b/fsgen/Android.bp @@ -13,6 +13,7 @@ bootstrap_go_package { "soong-kernel", ], srcs: [ + "boot_imgs.go", "filesystem_creator.go", "fsgen_mutators.go", "prebuilt_etc_modules_gen.go", diff --git a/fsgen/boot_imgs.go b/fsgen/boot_imgs.go new file mode 100644 index 000000000..799dbc9e3 --- /dev/null +++ b/fsgen/boot_imgs.go @@ -0,0 +1,285 @@ +package fsgen + +import ( + "android/soong/android" + "android/soong/filesystem" + "fmt" + "path/filepath" + "strconv" + "strings" + + "github.com/google/blueprint/proptools" +) + +func createBootImage(ctx android.LoadHookContext, dtbImg dtbImg) bool { + partitionVariables := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + + if partitionVariables.TargetKernelPath == "" { + // There are potentially code paths that don't set TARGET_KERNEL_PATH + return false + } + + kernelDir := filepath.Dir(partitionVariables.TargetKernelPath) + kernelBase := filepath.Base(partitionVariables.TargetKernelPath) + kernelFilegroupName := generatedModuleName(ctx.Config(), "kernel") + + ctx.CreateModuleInDirectory( + android.FileGroupFactory, + kernelDir, + &struct { + Name *string + Srcs []string + Visibility []string + }{ + Name: proptools.StringPtr(kernelFilegroupName), + Srcs: []string{kernelBase}, + Visibility: []string{"//visibility:public"}, + }, + ) + + var partitionSize *int64 + if partitionVariables.BoardBootimagePartitionSize != "" { + // Base of zero will allow base 10 or base 16 if starting with 0x + parsed, err := strconv.ParseInt(partitionVariables.BoardBootimagePartitionSize, 0, 64) + if err != nil { + panic(fmt.Sprintf("BOARD_BOOTIMAGE_PARTITION_SIZE must be an int, got %s", partitionVariables.BoardBootimagePartitionSize)) + } + partitionSize = &parsed + } + + var securityPatch *string + if partitionVariables.BootSecurityPatch != "" { + securityPatch = &partitionVariables.BootSecurityPatch + } + + avbInfo := getAvbInfo(ctx.Config(), "boot") + + bootImageName := generatedModuleNameForPartition(ctx.Config(), "boot") + + var dtbPrebuilt *string + if dtbImg.include && dtbImg.imgType == "boot" { + dtbPrebuilt = proptools.StringPtr(":" + dtbImg.name) + } + + var cmdline []string + if !buildingVendorBootImage(partitionVariables) { + cmdline = partitionVariables.InternalKernelCmdline + } + + ctx.CreateModule( + filesystem.BootimgFactory, + &filesystem.BootimgProperties{ + Kernel_prebuilt: proptools.StringPtr(":" + kernelFilegroupName), + Header_version: proptools.StringPtr(partitionVariables.BoardBootHeaderVersion), + Partition_size: partitionSize, + Use_avb: avbInfo.avbEnable, + Avb_mode: avbInfo.avbMode, + Avb_private_key: avbInfo.avbkeyFilegroup, + Avb_rollback_index: avbInfo.avbRollbackIndex, + Avb_algorithm: avbInfo.avbAlgorithm, + Security_patch: securityPatch, + Dtb_prebuilt: dtbPrebuilt, + Cmdline: cmdline, + }, + &struct { + Name *string + }{ + Name: proptools.StringPtr(bootImageName), + }, + ) + return true +} + +func createVendorBootImage(ctx android.LoadHookContext, dtbImg dtbImg) bool { + partitionVariables := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + + bootImageName := generatedModuleNameForPartition(ctx.Config(), "vendor_boot") + + avbInfo := getAvbInfo(ctx.Config(), "vendor_boot") + + var dtbPrebuilt *string + if dtbImg.include && dtbImg.imgType == "vendor_boot" { + dtbPrebuilt = proptools.StringPtr(":" + dtbImg.name) + } + + cmdline := partitionVariables.InternalKernelCmdline + + ctx.CreateModule( + filesystem.BootimgFactory, + &filesystem.BootimgProperties{ + Boot_image_type: proptools.StringPtr("vendor_boot"), + Ramdisk_module: proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), "vendor_ramdisk")), + Header_version: proptools.StringPtr(partitionVariables.BoardBootHeaderVersion), + Use_avb: avbInfo.avbEnable, + Avb_mode: avbInfo.avbMode, + Avb_private_key: avbInfo.avbkeyFilegroup, + Avb_rollback_index: avbInfo.avbRollbackIndex, + Avb_algorithm: avbInfo.avbAlgorithm, + Dtb_prebuilt: dtbPrebuilt, + Cmdline: cmdline, + }, + &struct { + Name *string + }{ + Name: proptools.StringPtr(bootImageName), + }, + ) + return true +} + +func createInitBootImage(ctx android.LoadHookContext) bool { + partitionVariables := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + + bootImageName := generatedModuleNameForPartition(ctx.Config(), "init_boot") + + var securityPatch *string + if partitionVariables.InitBootSecurityPatch != "" { + securityPatch = &partitionVariables.InitBootSecurityPatch + } else if partitionVariables.BootSecurityPatch != "" { + securityPatch = &partitionVariables.BootSecurityPatch + } + + var partitionSize *int64 + if partitionVariables.BoardInitBootimagePartitionSize != "" { + // Base of zero will allow base 10 or base 16 if starting with 0x + parsed, err := strconv.ParseInt(partitionVariables.BoardInitBootimagePartitionSize, 0, 64) + if err != nil { + panic(fmt.Sprintf("BOARD_INIT_BOOT_IMAGE_PARTITION_SIZE must be an int, got %s", partitionVariables.BoardInitBootimagePartitionSize)) + } + partitionSize = &parsed + } + + avbInfo := getAvbInfo(ctx.Config(), "init_boot") + + ctx.CreateModule( + filesystem.BootimgFactory, + &filesystem.BootimgProperties{ + Boot_image_type: proptools.StringPtr("init_boot"), + Ramdisk_module: proptools.StringPtr(generatedModuleNameForPartition(ctx.Config(), "ramdisk")), + Header_version: proptools.StringPtr(partitionVariables.BoardBootHeaderVersion), + Security_patch: securityPatch, + Partition_size: partitionSize, + Use_avb: avbInfo.avbEnable, + Avb_mode: avbInfo.avbMode, + Avb_private_key: avbInfo.avbkeyFilegroup, + Avb_rollback_index: avbInfo.avbRollbackIndex, + Avb_algorithm: avbInfo.avbAlgorithm, + }, + &struct { + Name *string + }{ + Name: proptools.StringPtr(bootImageName), + }, + ) + return true +} + +// Returns the equivalent of the BUILDING_BOOT_IMAGE variable in make. Derived from this logic: +// https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/board_config.mk;l=458;drc=5b55f926830963c02ab1d2d91e46442f04ba3af0 +func buildingBootImage(partitionVars android.PartitionVariables) bool { + if partitionVars.BoardUsesRecoveryAsBoot { + return false + } + + if partitionVars.ProductBuildBootImage { + return true + } + + if len(partitionVars.BoardPrebuiltBootimage) > 0 { + return false + } + + if len(partitionVars.BoardBootimagePartitionSize) > 0 { + return true + } + + // TODO: return true if BOARD_KERNEL_BINARIES is set and has a *_BOOTIMAGE_PARTITION_SIZE + // variable. However, I don't think BOARD_KERNEL_BINARIES is ever set in practice. + + return false +} + +// Returns the equivalent of the BUILDING_VENDOR_BOOT_IMAGE variable in make. Derived from this logic: +// https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/board_config.mk;l=518;drc=5b55f926830963c02ab1d2d91e46442f04ba3af0 +func buildingVendorBootImage(partitionVars android.PartitionVariables) bool { + if v, exists := boardBootHeaderVersion(partitionVars); exists && v >= 3 { + x := partitionVars.ProductBuildVendorBootImage + if x == "" || x == "true" { + return true + } + } + + return false +} + +// Derived from: https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/board_config.mk;l=480;drc=5b55f926830963c02ab1d2d91e46442f04ba3af0 +func buildingInitBootImage(partitionVars android.PartitionVariables) bool { + if !partitionVars.ProductBuildInitBootImage { + if partitionVars.BoardUsesRecoveryAsBoot || len(partitionVars.BoardPrebuiltInitBootimage) > 0 { + return false + } else if len(partitionVars.BoardInitBootimagePartitionSize) > 0 { + return true + } + } else { + if partitionVars.BoardUsesRecoveryAsBoot { + panic("PRODUCT_BUILD_INIT_BOOT_IMAGE is true, but so is BOARD_USES_RECOVERY_AS_BOOT. Use only one option.") + } + return true + } + return false +} + +func boardBootHeaderVersion(partitionVars android.PartitionVariables) (int, bool) { + if len(partitionVars.BoardBootHeaderVersion) == 0 { + return 0, false + } + v, err := strconv.ParseInt(partitionVars.BoardBootHeaderVersion, 10, 32) + if err != nil { + panic(fmt.Sprintf("BOARD_BOOT_HEADER_VERSION must be an int, got: %q", partitionVars.BoardBootHeaderVersion)) + } + return int(v), true +} + +type dtbImg struct { + // whether to include the dtb image in boot image + include bool + + // name of the generated dtb image filegroup name + name string + + // type of the boot image that the dtb image argument should be specified + imgType string +} + +func createDtbImgFilegroup(ctx android.LoadHookContext) dtbImg { + partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + if !partitionVars.BoardIncludeDtbInBootimg { + return dtbImg{include: false} + } + for _, copyFilePair := range partitionVars.ProductCopyFiles { + srcDestList := strings.Split(copyFilePair, ":") + if len(srcDestList) < 2 { + ctx.ModuleErrorf("PRODUCT_COPY_FILES must follow the format \"src:dest\", got: %s", copyFilePair) + } + if srcDestList[1] == "dtb.img" { + moduleName := generatedModuleName(ctx.Config(), "dtb_img_filegroup") + ctx.CreateModuleInDirectory( + android.FileGroupFactory, + filepath.Dir(srcDestList[0]), + &struct { + Name *string + Srcs []string + }{ + Name: proptools.StringPtr(moduleName), + Srcs: []string{filepath.Base(srcDestList[1])}, + }, + ) + imgType := "vendor_boot" + if !buildingVendorBootImage(partitionVars) { + imgType = "boot" + } + return dtbImg{include: true, name: moduleName, imgType: imgType} + } + } + return dtbImg{include: false} +} diff --git a/fsgen/filesystem_creator.go b/fsgen/filesystem_creator.go index d46f67904..e8b0a4fb7 100644 --- a/fsgen/filesystem_creator.go +++ b/fsgen/filesystem_creator.go @@ -47,6 +47,10 @@ type filesystemCreatorProps struct { Vbmeta_module_names []string `blueprint:"mutated"` Vbmeta_partition_names []string `blueprint:"mutated"` + + Boot_image string `blueprint:"mutated" android:"path_device_first"` + Vendor_boot_image string `blueprint:"mutated" android:"path_device_first"` + Init_boot_image string `blueprint:"mutated" android:"path_device_first"` } type filesystemCreator struct { @@ -64,12 +68,49 @@ func filesystemCreatorFactory() android.Module { generatedPrebuiltEtcModuleNames := createPrebuiltEtcModules(ctx) avbpubkeyGenerated := createAvbpubkeyModule(ctx) createFsGenState(ctx, generatedPrebuiltEtcModuleNames, avbpubkeyGenerated) + module.createAvbKeyFilegroups(ctx) module.createInternalModules(ctx) }) return module } +func generatedPartitions(ctx android.LoadHookContext) []string { + partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + generatedPartitions := []string{"system"} + if ctx.DeviceConfig().SystemExtPath() == "system_ext" { + generatedPartitions = append(generatedPartitions, "system_ext") + } + if ctx.DeviceConfig().BuildingVendorImage() && ctx.DeviceConfig().VendorPath() == "vendor" { + generatedPartitions = append(generatedPartitions, "vendor") + } + if ctx.DeviceConfig().BuildingProductImage() && ctx.DeviceConfig().ProductPath() == "product" { + generatedPartitions = append(generatedPartitions, "product") + } + if ctx.DeviceConfig().BuildingOdmImage() && ctx.DeviceConfig().OdmPath() == "odm" { + generatedPartitions = append(generatedPartitions, "odm") + } + if ctx.DeviceConfig().BuildingUserdataImage() && ctx.DeviceConfig().UserdataPath() == "data" { + generatedPartitions = append(generatedPartitions, "userdata") + } + if partitionVars.BuildingSystemDlkmImage { + generatedPartitions = append(generatedPartitions, "system_dlkm") + } + if partitionVars.BuildingVendorDlkmImage { + generatedPartitions = append(generatedPartitions, "vendor_dlkm") + } + if partitionVars.BuildingOdmDlkmImage { + generatedPartitions = append(generatedPartitions, "odm_dlkm") + } + if partitionVars.BuildingRamdiskImage { + generatedPartitions = append(generatedPartitions, "ramdisk") + } + if buildingVendorBootImage(partitionVars) { + generatedPartitions = append(generatedPartitions, "vendor_ramdisk") + } + return generatedPartitions +} + func (f *filesystemCreator) createInternalModules(ctx android.LoadHookContext) { soongGeneratedPartitions := generatedPartitions(ctx) finalSoongGeneratedPartitions := make([]string, 0, len(soongGeneratedPartitions)) @@ -82,6 +123,31 @@ func (f *filesystemCreator) createInternalModules(ctx android.LoadHookContext) { } } + partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + dtbImg := createDtbImgFilegroup(ctx) + + if buildingBootImage(partitionVars) { + if createBootImage(ctx, dtbImg) { + f.properties.Boot_image = ":" + generatedModuleNameForPartition(ctx.Config(), "boot") + } else { + f.properties.Unsupported_partition_types = append(f.properties.Unsupported_partition_types, "boot") + } + } + if buildingVendorBootImage(partitionVars) { + if createVendorBootImage(ctx, dtbImg) { + f.properties.Vendor_boot_image = ":" + generatedModuleNameForPartition(ctx.Config(), "vendor_boot") + } else { + f.properties.Unsupported_partition_types = append(f.properties.Unsupported_partition_types, "vendor_boot") + } + } + if buildingInitBootImage(partitionVars) { + if createInitBootImage(ctx) { + f.properties.Init_boot_image = ":" + generatedModuleNameForPartition(ctx.Config(), "init_boot") + } else { + f.properties.Unsupported_partition_types = append(f.properties.Unsupported_partition_types, "init_boot") + } + } + for _, x := range createVbmetaPartitions(ctx, finalSoongGeneratedPartitions) { f.properties.Vbmeta_module_names = append(f.properties.Vbmeta_module_names, x.moduleName) f.properties.Vbmeta_partition_names = append(f.properties.Vbmeta_partition_names, x.partitionName) @@ -139,13 +205,13 @@ func (f *filesystemCreator) createDeviceModule( ctx.CreateModule(filesystem.AndroidDeviceFactory, baseProps, partitionProps) } -func partitionSpecificFsProps(fsProps *filesystem.FilesystemProperties, partitionType string) { +func partitionSpecificFsProps(fsProps *filesystem.FilesystemProperties, partitionVars android.PartitionVariables, partitionType string) { switch partitionType { case "system": fsProps.Build_logtags = proptools.BoolPtr(true) // https://source.corp.google.com/h/googleplex-android/platform/build//639d79f5012a6542ab1f733b0697db45761ab0f3:core/packaging/flags.mk;l=21;drc=5ba8a8b77507f93aa48cc61c5ba3f31a4d0cbf37;bpv=1;bpt=0 fsProps.Gen_aconfig_flags_pb = proptools.BoolPtr(true) - // Identical to that of the generic_system_image + // Identical to that of the aosp_shared_system_image fsProps.Fsverity.Inputs = []string{ "etc/boot-image.prof", "etc/dirty-image-objects", @@ -175,7 +241,6 @@ func partitionSpecificFsProps(fsProps *filesystem.FilesystemProperties, partitio Name: proptools.StringPtr("system/lib/modules"), }, } - fsProps.Base_dir = proptools.StringPtr("system") case "system_ext": fsProps.Fsverity.Inputs = []string{ "framework/*", @@ -197,7 +262,6 @@ func partitionSpecificFsProps(fsProps *filesystem.FilesystemProperties, partitio Name: proptools.StringPtr("vendor/lib/modules"), }, } - fsProps.Base_dir = proptools.StringPtr("vendor") case "odm": fsProps.Symlinks = []filesystem.SymlinkDefinition{ filesystem.SymlinkDefinition{ @@ -205,10 +269,30 @@ func partitionSpecificFsProps(fsProps *filesystem.FilesystemProperties, partitio Name: proptools.StringPtr("odm/lib/modules"), }, } - fsProps.Base_dir = proptools.StringPtr("odm") case "userdata": fsProps.Base_dir = proptools.StringPtr("data") - + case "ramdisk": + // Following the logic in https://cs.android.com/android/platform/superproject/main/+/c3c5063df32748a8806ce5da5dd0db158eab9ad9:build/make/core/Makefile;l=1307 + fsProps.Dirs = android.NewSimpleConfigurable([]string{ + "debug_ramdisk", + "dev", + "metadata", + "mnt", + "proc", + "second_stage_resources", + "sys", + }) + if partitionVars.BoardUsesGenericKernelImage { + fsProps.Dirs.AppendSimpleValue([]string{ + "first_stage_ramdisk/debug_ramdisk", + "first_stage_ramdisk/dev", + "first_stage_ramdisk/metadata", + "first_stage_ramdisk/mnt", + "first_stage_ramdisk/proc", + "first_stage_ramdisk/second_stage_resources", + "first_stage_ramdisk/sys", + }) + } } } @@ -230,9 +314,11 @@ func (f *filesystemCreator) createPartition(ctx android.LoadHookContext, partiti return false } - if partitionType == "vendor" || partitionType == "product" { + if partitionType == "vendor" || partitionType == "product" || partitionType == "system" { fsProps.Linker_config.Gen_linker_config = proptools.BoolPtr(true) - fsProps.Linker_config.Linker_config_srcs = f.createLinkerConfigSourceFilegroups(ctx, partitionType) + if partitionType != "system" { + fsProps.Linker_config.Linker_config_srcs = f.createLinkerConfigSourceFilegroups(ctx, partitionType) + } } if android.InList(partitionType, dlkmPartitions) { @@ -254,6 +340,51 @@ func (f *filesystemCreator) createPartition(ctx android.LoadHookContext, partiti return true } +// Creates filegroups for the files specified in BOARD_(partition_)AVB_KEY_PATH +func (f *filesystemCreator) createAvbKeyFilegroups(ctx android.LoadHookContext) { + partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + var files []string + + if len(partitionVars.BoardAvbKeyPath) > 0 { + files = append(files, partitionVars.BoardAvbKeyPath) + } + for _, partition := range android.SortedKeys(partitionVars.PartitionQualifiedVariables) { + specificPartitionVars := partitionVars.PartitionQualifiedVariables[partition] + if len(specificPartitionVars.BoardAvbKeyPath) > 0 { + files = append(files, specificPartitionVars.BoardAvbKeyPath) + } + } + + fsGenState := ctx.Config().Get(fsGenStateOnceKey).(*FsGenState) + for _, file := range files { + if _, ok := fsGenState.avbKeyFilegroups[file]; ok { + continue + } + if file == "external/avb/test/data/testkey_rsa4096.pem" { + // There already exists a checked-in filegroup for this commonly-used key, just use that + fsGenState.avbKeyFilegroups[file] = "avb_testkey_rsa4096" + continue + } + dir := filepath.Dir(file) + base := filepath.Base(file) + name := fmt.Sprintf("avb_key_%x", strings.ReplaceAll(file, "/", "_")) + ctx.CreateModuleInDirectory( + android.FileGroupFactory, + dir, + &struct { + Name *string + Srcs []string + Visibility []string + }{ + Name: proptools.StringPtr(name), + Srcs: []string{base}, + Visibility: []string{"//visibility:public"}, + }, + ) + fsGenState.avbKeyFilegroups[file] = name + } +} + // createPrebuiltKernelModules creates `prebuilt_kernel_modules`. These modules will be added to deps of the // autogenerated *_dlkm filsystem modules. Each _dlkm partition should have a single prebuilt_kernel_modules dependency. // This ensures that the depmod artifacts (modules.* installed in /lib/modules/) are generated with a complete view. @@ -274,7 +405,7 @@ func (f *filesystemCreator) createPrebuiltKernelModules(ctx android.LoadHookCont } switch partitionType { case "system_dlkm": - props.Srcs = ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.SystemKernelModules + props.Srcs = android.ExistentPathsForSources(ctx, ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.SystemKernelModules).Strings() props.System_dlkm_specific = proptools.BoolPtr(true) if len(ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.SystemKernelLoadModules) == 0 { // Create empty modules.load file for system @@ -285,7 +416,7 @@ func (f *filesystemCreator) createPrebuiltKernelModules(ctx android.LoadHookCont props.Blocklist_file = proptools.StringPtr(blocklistFile) } case "vendor_dlkm": - props.Srcs = ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.VendorKernelModules + props.Srcs = android.ExistentPathsForSources(ctx, ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.VendorKernelModules).Strings() if len(ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.SystemKernelModules) > 0 { props.System_deps = []string{":" + generatedModuleName(ctx.Config(), "system_dlkm-kernel-modules") + "{.modules}"} } @@ -294,7 +425,7 @@ func (f *filesystemCreator) createPrebuiltKernelModules(ctx android.LoadHookCont props.Blocklist_file = proptools.StringPtr(blocklistFile) } case "odm_dlkm": - props.Srcs = ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.OdmKernelModules + props.Srcs = android.ExistentPathsForSources(ctx, ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.OdmKernelModules).Strings() props.Odm_dlkm_specific = proptools.BoolPtr(true) if blocklistFile := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.OdmKernelBlocklistFile; blocklistFile != "" { props.Blocklist_file = proptools.StringPtr(blocklistFile) @@ -422,18 +553,17 @@ func generateFsProps(ctx android.EarlyModuleContext, partitionType string) (*fil fsProps := &filesystem.FilesystemProperties{} partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse - var specificPartitionVars android.PartitionQualifiedVariablesType - var boardAvbEnable bool + var avbInfo avbInfo var fsType string if strings.Contains(partitionType, "ramdisk") { fsType = "compressed_cpio" } else { - specificPartitionVars = partitionVars.PartitionQualifiedVariables[partitionType] - boardAvbEnable = partitionVars.BoardAvbEnable + specificPartitionVars := partitionVars.PartitionQualifiedVariables[partitionType] fsType = specificPartitionVars.BoardFileSystemType - } - if fsType == "" { - fsType = "ext4" //default + avbInfo = getAvbInfo(ctx.Config(), partitionType) + if fsType == "" { + fsType = "ext4" //default + } } fsProps.Type = proptools.StringPtr(fsType) @@ -447,23 +577,23 @@ func generateFsProps(ctx android.EarlyModuleContext, partitionType string) (*fil fsProps.Unchecked_module = proptools.BoolPtr(true) // BOARD_AVB_ENABLE - fsProps.Use_avb = proptools.BoolPtr(boardAvbEnable) + fsProps.Use_avb = avbInfo.avbEnable // BOARD_AVB_KEY_PATH - fsProps.Avb_private_key = proptools.StringPtr(specificPartitionVars.BoardAvbKeyPath) + fsProps.Avb_private_key = avbInfo.avbkeyFilegroup // BOARD_AVB_ALGORITHM - fsProps.Avb_algorithm = proptools.StringPtr(specificPartitionVars.BoardAvbAlgorithm) + fsProps.Avb_algorithm = avbInfo.avbAlgorithm // BOARD_AVB_SYSTEM_ROLLBACK_INDEX - if rollbackIndex, err := strconv.ParseInt(specificPartitionVars.BoardAvbRollbackIndex, 10, 64); err == nil { - fsProps.Rollback_index = proptools.Int64Ptr(rollbackIndex) - } + fsProps.Rollback_index = avbInfo.avbRollbackIndex fsProps.Partition_name = proptools.StringPtr(partitionType) - fsProps.Base_dir = proptools.StringPtr(partitionType) + if !strings.Contains(partitionType, "ramdisk") { + fsProps.Base_dir = proptools.StringPtr(partitionType) + } fsProps.Is_auto_generated = proptools.BoolPtr(true) - partitionSpecificFsProps(fsProps, partitionType) + partitionSpecificFsProps(fsProps, partitionVars, partitionType) // system_image properties that are not set: // - filesystemProperties.Avb_hash_algorithm @@ -480,7 +610,56 @@ func generateFsProps(ctx android.EarlyModuleContext, partitionType string) (*fil return fsProps, true } -func (f *filesystemCreator) createDiffTest(ctx android.ModuleContext, partitionType string) android.Path { +type avbInfo struct { + avbEnable *bool + avbKeyPath *string + avbkeyFilegroup *string + avbAlgorithm *string + avbRollbackIndex *int64 + avbMode *string +} + +func getAvbInfo(config android.Config, partitionType string) avbInfo { + partitionVars := config.ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse + specificPartitionVars := partitionVars.PartitionQualifiedVariables[partitionType] + var result avbInfo + boardAvbEnable := partitionVars.BoardAvbEnable + if boardAvbEnable { + result.avbEnable = proptools.BoolPtr(true) + if specificPartitionVars.BoardAvbKeyPath != "" { + result.avbKeyPath = proptools.StringPtr(specificPartitionVars.BoardAvbKeyPath) + } else if partitionVars.BoardAvbKeyPath != "" { + result.avbKeyPath = proptools.StringPtr(partitionVars.BoardAvbKeyPath) + } + if specificPartitionVars.BoardAvbAlgorithm != "" { + result.avbAlgorithm = proptools.StringPtr(specificPartitionVars.BoardAvbAlgorithm) + } else if partitionVars.BoardAvbAlgorithm != "" { + result.avbAlgorithm = proptools.StringPtr(partitionVars.BoardAvbAlgorithm) + } + if specificPartitionVars.BoardAvbRollbackIndex != "" { + parsed, err := strconv.ParseInt(specificPartitionVars.BoardAvbRollbackIndex, 10, 64) + if err != nil { + panic(fmt.Sprintf("Rollback index must be an int, got %s", specificPartitionVars.BoardAvbRollbackIndex)) + } + result.avbRollbackIndex = &parsed + } else if partitionVars.BoardAvbRollbackIndex != "" { + parsed, err := strconv.ParseInt(partitionVars.BoardAvbRollbackIndex, 10, 64) + if err != nil { + panic(fmt.Sprintf("Rollback index must be an int, got %s", partitionVars.BoardAvbRollbackIndex)) + } + result.avbRollbackIndex = &parsed + } + result.avbMode = proptools.StringPtr("make_legacy") + } + if result.avbKeyPath != nil { + fsGenState := config.Get(fsGenStateOnceKey).(*FsGenState) + filegroup := fsGenState.avbKeyFilegroups[*result.avbKeyPath] + result.avbkeyFilegroup = proptools.StringPtr(":" + filegroup) + } + return result +} + +func (f *filesystemCreator) createFileListDiffTest(ctx android.ModuleContext, partitionType string) android.Path { partitionModuleName := generatedModuleNameForPartition(ctx.Config(), partitionType) systemImage := ctx.GetDirectDepWithTag(partitionModuleName, generatedFilesystemDepTag) filesystemInfo, ok := android.OtherModuleProvider(ctx, systemImage, filesystem.FilesystemProvider) @@ -525,13 +704,17 @@ func createVbmetaDiff(ctx android.ModuleContext, vbmetaModuleName string, vbmeta makeVbmetaFile := android.PathForArbitraryOutput(ctx, fmt.Sprintf("target/product/%s/%s.img", ctx.Config().DeviceName(), vbmetaPartitionName)) diffTestResultFile := android.PathForModuleOut(ctx, fmt.Sprintf("diff_test_%s.txt", vbmetaModuleName)) + createDiffTest(ctx, diffTestResultFile, soongVbMetaFile, makeVbmetaFile) + return diffTestResultFile +} + +func createDiffTest(ctx android.ModuleContext, diffTestResultFile android.WritablePath, file1 android.Path, file2 android.Path) { builder := android.NewRuleBuilder(pctx, ctx) builder.Command().Text("diff"). - Input(soongVbMetaFile). - Input(makeVbmetaFile) + Input(file1). + Input(file2) builder.Command().Text("touch").Output(diffTestResultFile) - builder.Build(vbmetaModuleName+" diff test", vbmetaModuleName+" diff test") - return diffTestResultFile + builder.Build("diff test "+diffTestResultFile.String(), "diff test") } type systemImageDepTagType struct { @@ -568,7 +751,7 @@ func (f *filesystemCreator) GenerateAndroidBuildActions(ctx android.ModuleContex var diffTestFiles []android.Path for _, partitionType := range f.properties.Generated_partition_types { - diffTestFile := f.createDiffTest(ctx, partitionType) + diffTestFile := f.createFileListDiffTest(ctx, partitionType) diffTestFiles = append(diffTestFiles, diffTestFile) ctx.Phony(fmt.Sprintf("soong_generated_%s_filesystem_test", partitionType), diffTestFile) } @@ -582,6 +765,30 @@ func (f *filesystemCreator) GenerateAndroidBuildActions(ctx android.ModuleContex diffTestFiles = append(diffTestFiles, diffTestFile) ctx.Phony(fmt.Sprintf("soong_generated_%s_filesystem_test", f.properties.Vbmeta_partition_names[i]), diffTestFile) } + if f.properties.Boot_image != "" { + diffTestFile := android.PathForModuleOut(ctx, "boot_diff_test.txt") + soongBootImg := android.PathForModuleSrc(ctx, f.properties.Boot_image) + makeBootImage := android.PathForArbitraryOutput(ctx, fmt.Sprintf("target/product/%s/boot.img", ctx.Config().DeviceName())) + createDiffTest(ctx, diffTestFile, soongBootImg, makeBootImage) + diffTestFiles = append(diffTestFiles, diffTestFile) + ctx.Phony("soong_generated_boot_filesystem_test", diffTestFile) + } + if f.properties.Vendor_boot_image != "" { + diffTestFile := android.PathForModuleOut(ctx, "vendor_boot_diff_test.txt") + soongBootImg := android.PathForModuleSrc(ctx, f.properties.Vendor_boot_image) + makeBootImage := android.PathForArbitraryOutput(ctx, fmt.Sprintf("target/product/%s/vendor_boot.img", ctx.Config().DeviceName())) + createDiffTest(ctx, diffTestFile, soongBootImg, makeBootImage) + diffTestFiles = append(diffTestFiles, diffTestFile) + ctx.Phony("soong_generated_vendor_boot_filesystem_test", diffTestFile) + } + if f.properties.Init_boot_image != "" { + diffTestFile := android.PathForModuleOut(ctx, "init_boot_diff_test.txt") + soongBootImg := android.PathForModuleSrc(ctx, f.properties.Init_boot_image) + makeBootImage := android.PathForArbitraryOutput(ctx, fmt.Sprintf("target/product/%s/init_boot.img", ctx.Config().DeviceName())) + createDiffTest(ctx, diffTestFile, soongBootImg, makeBootImage) + diffTestFiles = append(diffTestFiles, diffTestFile) + ctx.Phony("soong_generated_init_boot_filesystem_test", diffTestFile) + } ctx.Phony("soong_generated_filesystem_tests", diffTestFiles...) } diff --git a/fsgen/filesystem_creator_test.go b/fsgen/filesystem_creator_test.go index 6111a4117..fe4a40325 100644 --- a/fsgen/filesystem_creator_test.go +++ b/fsgen/filesystem_creator_test.go @@ -47,6 +47,12 @@ func TestFileSystemCreatorSystemImageProps(t *testing.T) { }), android.FixtureMergeMockFs(android.MockFS{ "external/avb/test/data/testkey_rsa4096.pem": nil, + "external/avb/test/Android.bp": []byte(` + filegroup { + name: "avb_testkey_rsa4096", + srcs: ["data/testkey_rsa4096.pem"], + } + `), "build/soong/fsgen/Android.bp": []byte(` soong_filesystem_creator { name: "foo", @@ -66,8 +72,8 @@ func TestFileSystemCreatorSystemImageProps(t *testing.T) { ) android.AssertStringEquals( t, - "Property expected to match the product variable 'BOARD_AVB_KEY_PATH'", - "external/avb/test/data/testkey_rsa4096.pem", + "Property the avb_private_key property to be set to the existing filegroup", + ":avb_testkey_rsa4096", proptools.String(fooSystem.FsProps().Avb_private_key), ) android.AssertStringEquals( diff --git a/fsgen/fsgen_mutators.go b/fsgen/fsgen_mutators.go index e0e103a98..9472a50ae 100644 --- a/fsgen/fsgen_mutators.go +++ b/fsgen/fsgen_mutators.go @@ -68,6 +68,8 @@ type FsGenState struct { moduleToInstallationProps map[string]installationProperties // List of prebuilt_* modules that are autogenerated. generatedPrebuiltEtcModuleNames []string + // Mapping from a path to an avb key to the name of a filegroup module that contains it + avbKeyFilegroups map[string]string } type installationProperties struct { @@ -82,36 +84,6 @@ func defaultDepCandidateProps(config android.Config) *depCandidateProps { } } -func generatedPartitions(ctx android.LoadHookContext) []string { - generatedPartitions := []string{"system", "ramdisk"} - if ctx.DeviceConfig().SystemExtPath() == "system_ext" { - generatedPartitions = append(generatedPartitions, "system_ext") - } - if ctx.DeviceConfig().BuildingVendorImage() && ctx.DeviceConfig().VendorPath() == "vendor" { - generatedPartitions = append(generatedPartitions, "vendor") - } - if ctx.DeviceConfig().BuildingProductImage() && ctx.DeviceConfig().ProductPath() == "product" { - generatedPartitions = append(generatedPartitions, "product") - } - if ctx.DeviceConfig().BuildingOdmImage() && ctx.DeviceConfig().OdmPath() == "odm" { - generatedPartitions = append(generatedPartitions, "odm") - } - if ctx.DeviceConfig().BuildingUserdataImage() && ctx.DeviceConfig().UserdataPath() == "data" { - generatedPartitions = append(generatedPartitions, "userdata") - } - if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingSystemDlkmImage { - generatedPartitions = append(generatedPartitions, "system_dlkm") - } - if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingVendorDlkmImage { - generatedPartitions = append(generatedPartitions, "vendor_dlkm") - } - if ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse.BuildingOdmDlkmImage { - generatedPartitions = append(generatedPartitions, "odm_dlkm") - } - - return generatedPartitions -} - func createFsGenState(ctx android.LoadHookContext, generatedPrebuiltEtcModuleNames []string, avbpubkeyGenerated bool) *FsGenState { return ctx.Config().Once(fsGenStateOnceKey, func() interface{} { partitionVars := ctx.Config().ProductVariables().PartitionVarsForSoongMigrationOnlyDoNotUse @@ -126,7 +98,6 @@ func createFsGenState(ctx android.LoadHookContext, generatedPrebuiltEtcModuleNam "com.android.apex.cts.shim.v1_prebuilt": defaultDepCandidateProps(ctx.Config()), "dex_bootjars": defaultDepCandidateProps(ctx.Config()), "framework_compatibility_matrix.device.xml": defaultDepCandidateProps(ctx.Config()), - "init.environ.rc-soong": defaultDepCandidateProps(ctx.Config()), "libcompiler_rt": defaultDepCandidateProps(ctx.Config()), "libdmabufheap": defaultDepCandidateProps(ctx.Config()), "libgsi": defaultDepCandidateProps(ctx.Config()), @@ -174,11 +145,13 @@ func createFsGenState(ctx android.LoadHookContext, generatedPrebuiltEtcModuleNam "fs_config_files_odm_dlkm": defaultDepCandidateProps(ctx.Config()), "odm_dlkm-build.prop": defaultDepCandidateProps(ctx.Config()), }, - "ramdisk": {}, + "ramdisk": {}, + "vendor_ramdisk": {}, }, fsDepsMutex: sync.Mutex{}, moduleToInstallationProps: map[string]installationProperties{}, generatedPrebuiltEtcModuleNames: generatedPrebuiltEtcModuleNames, + avbKeyFilegroups: map[string]string{}, } if avbpubkeyGenerated { @@ -386,6 +359,7 @@ func generateDepStruct(deps map[string]*depCandidateProps, highPriorityDeps []st depsStruct.Multilib.Prefer32.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Prefer32.Deps) depsStruct.Multilib.Both.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Both.Deps) depsStruct.Multilib.Common.Deps = android.SortedUniqueStrings(depsStruct.Multilib.Common.Deps) + depsStruct.High_priority_deps = android.SortedUniqueStrings(depsStruct.High_priority_deps) return &depsStruct } diff --git a/fsgen/vbmeta_partitions.go b/fsgen/vbmeta_partitions.go index b7fff687d..11c57590c 100644 --- a/fsgen/vbmeta_partitions.go +++ b/fsgen/vbmeta_partitions.go @@ -19,6 +19,7 @@ import ( "android/soong/filesystem" "slices" "strconv" + "strings" "github.com/google/blueprint/proptools" ) @@ -153,8 +154,10 @@ func createVbmetaPartitions(ctx android.LoadHookContext, generatedPartitionTypes // Already handled by a chained vbmeta partition continue } - if partitionType == "ramdisk" { + if strings.Contains(partitionType, "ramdisk") || strings.Contains(partitionType, "boot") { // ramdisk is never signed with avb information + // boot partitions just have the avb footer, and don't have a corresponding vbmeta + // partition. continue } partitionModules = append(partitionModules, generatedModuleNameForPartition(ctx.Config(), partitionType)) diff --git a/java/app.go b/java/app.go index 8bb73cb38..7f80160a3 100644 --- a/java/app.go +++ b/java/app.go @@ -164,7 +164,7 @@ type appProperties struct { type overridableAppProperties struct { // The name of a certificate in the default certificate directory, blank to use the default product certificate, // or an android_app_certificate module name in the form ":module". - Certificate *string + Certificate proptools.Configurable[string] `android:"replace_instead_of_append"` // Name of the signing certificate lineage file or filegroup module. Lineage *string `android:"path"` @@ -1252,7 +1252,7 @@ func (a *AndroidApp) getCertString(ctx android.BaseModuleContext) string { if overridden { return ":" + certificate } - return String(a.overridableAppProperties.Certificate) + return a.overridableAppProperties.Certificate.GetOrDefault(ctx, "") } func (a *AndroidApp) DepIsInSameApex(ctx android.BaseModuleContext, dep android.Module) bool { diff --git a/java/app_import.go b/java/app_import.go index f044c6848..8951c7d9c 100644 --- a/java/app_import.go +++ b/java/app_import.go @@ -97,7 +97,7 @@ type AndroidAppImportProperties struct { // The name of a certificate in the default certificate directory or an android_app_certificate // module name in the form ":module". Should be empty if presigned or default_dev_cert is set. - Certificate *string + Certificate proptools.Configurable[string] `android:"replace_instead_of_append"` // Names of extra android_app_certificate modules to sign the apk with in the form ":module". Additional_certificates []string @@ -240,7 +240,7 @@ func disablePrebuiltsWithoutApkMutator(ctx android.BottomUpMutatorContext) { } func (a *AndroidAppImport) DepsMutator(ctx android.BottomUpMutatorContext) { - cert := android.SrcIsModule(String(a.properties.Certificate)) + cert := android.SrcIsModule(a.properties.Certificate.GetOrDefault(ctx, "")) if cert != "" { ctx.AddDependency(ctx.Module(), certificateTag, cert) } @@ -323,7 +323,7 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext } numCertPropsSet := 0 - if String(a.properties.Certificate) != "" { + if a.properties.Certificate.GetOrDefault(ctx, "") != "" { numCertPropsSet++ } if Bool(a.properties.Presigned) { @@ -406,7 +406,7 @@ func (a *AndroidAppImport) generateAndroidBuildActions(ctx android.ModuleContext // If the certificate property is empty at this point, default_dev_cert must be set to true. // Which makes processMainCert's behavior for the empty cert string WAI. _, _, certificates := collectAppDeps(ctx, a, false, false) - a.certificate, certificates = processMainCert(a.ModuleBase, String(a.properties.Certificate), certificates, ctx) + a.certificate, certificates = processMainCert(a.ModuleBase, a.properties.Certificate.GetOrDefault(ctx, ""), certificates, ctx) signed := android.PathForModuleOut(ctx, "signed", apkFilename) var lineageFile android.Path if lineage := String(a.properties.Lineage); lineage != "" { diff --git a/java/base.go b/java/base.go index 8dad2d9ad..3bf2e23d8 100644 --- a/java/base.go +++ b/java/base.go @@ -714,10 +714,10 @@ func (j *Module) provideHiddenAPIPropertyInfo(ctx android.ModuleContext) { // helper method for java modules to set OutputFilesProvider func setOutputFiles(ctx android.ModuleContext, m Module) { - ctx.SetOutputFiles(append(android.Paths{m.outputFile}, m.extraOutputFiles...), "") - ctx.SetOutputFiles(android.Paths{m.outputFile}, android.DefaultDistTag) - ctx.SetOutputFiles(android.Paths{m.implementationAndResourcesJar}, ".jar") - ctx.SetOutputFiles(android.Paths{m.headerJarFile}, ".hjar") + ctx.SetOutputFiles(append(android.PathsIfNonNil(m.outputFile), m.extraOutputFiles...), "") + ctx.SetOutputFiles(android.PathsIfNonNil(m.outputFile), android.DefaultDistTag) + ctx.SetOutputFiles(android.PathsIfNonNil(m.implementationAndResourcesJar), ".jar") + ctx.SetOutputFiles(android.PathsIfNonNil(m.headerJarFile), ".hjar") if m.dexer.proguardDictionary.Valid() { ctx.SetOutputFiles(android.Paths{m.dexer.proguardDictionary.Path()}, ".proguard_map") } @@ -766,7 +766,8 @@ func (j *Module) shouldInstrumentInApex(ctx android.BaseModuleContext) bool { apexInfo, _ := android.ModuleProvider(ctx, android.ApexInfoProvider) isJacocoAgent := ctx.ModuleName() == "jacocoagent" - if j.DirectlyInAnyApex() && !isJacocoAgent && !apexInfo.IsForPlatform() { + compileDex := Bool(j.dexProperties.Compile_dex) || Bool(j.properties.Installable) + if compileDex && !isJacocoAgent && !apexInfo.IsForPlatform() { if !inList(ctx.ModuleName(), config.InstrumentFrameworkModules) { return true } else if ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_FRAMEWORK") { @@ -1735,7 +1736,12 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspath completeStaticLibsImplementationJarsToCombine := completeStaticLibsImplementationJars - if j.shouldInstrument(ctx) { + apexInfo, _ := android.ModuleProvider(ctx, android.ApexInfoProvider) + + // Enable dex compilation for the APEX variants, unless it is disabled explicitly + compileDex := Bool(j.dexProperties.Compile_dex) || Bool(j.properties.Installable) + + if j.shouldInstrument(ctx) && (!ctx.Device() || compileDex) { instrumentedOutputFile := j.instrument(ctx, flags, outputFile, jarName, specs) completeStaticLibsImplementationJarsToCombine = depset.New(depset.PREORDER, android.Paths{instrumentedOutputFile}, nil) outputFile = instrumentedOutputFile @@ -1764,19 +1770,7 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspath j.implementationAndResourcesJar = outputFile - // Enable dex compilation for the APEX variants, unless it is disabled explicitly - compileDex := j.dexProperties.Compile_dex - apexInfo, _ := android.ModuleProvider(ctx, android.ApexInfoProvider) - if j.DirectlyInAnyApex() && !apexInfo.IsForPlatform() { - if compileDex == nil { - compileDex = proptools.BoolPtr(true) - } - if j.deviceProperties.Hostdex == nil { - j.deviceProperties.Hostdex = proptools.BoolPtr(true) - } - } - - if ctx.Device() && (Bool(j.properties.Installable) || Bool(compileDex)) { + if ctx.Device() && compileDex { if j.hasCode(ctx) { if j.shouldInstrumentStatic(ctx) { j.dexer.extraProguardFlagsFiles = append(j.dexer.extraProguardFlagsFiles, @@ -2328,7 +2322,10 @@ func (m *Module) getSdkLinkType(ctx android.BaseModuleContext, name string) (ret "stable.core.platform.api.stubs", "stub-annotations", "private-stub-annotations-jar", "core-lambda-stubs", - "core-generated-annotation-stubs": + "core-generated-annotation-stubs", + // jacocoagent only uses core APIs, but has to specify a non-core sdk_version so it can use + // a prebuilt SDK to avoid circular dependencies when it statically included in the bootclasspath. + "jacocoagent": return javaCore, true case android.SdkPublic.DefaultJavaLibraryName(): return javaSdk, true diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index c778f0404..375a1aaf1 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -83,10 +83,6 @@ func (b bootclasspathFragmentContentDependencyTag) ExportMember() bool { return true } -// Contents of bootclasspath fragments in an apex are considered to be directly in the apex, as if -// they were listed in java_libs. -func (b bootclasspathFragmentContentDependencyTag) CopyDirectlyInAnyApex() {} - // Contents of bootclasspath fragments require files from prebuilt apex files. func (b bootclasspathFragmentContentDependencyTag) RequiresFilesFromPrebuiltApex() {} @@ -96,7 +92,6 @@ var bootclasspathFragmentContentDepTag = bootclasspathFragmentContentDependencyT var _ android.ExcludeFromVisibilityEnforcementTag = bootclasspathFragmentContentDepTag var _ android.ReplaceSourceWithPrebuilt = bootclasspathFragmentContentDepTag var _ android.SdkMemberDependencyTag = bootclasspathFragmentContentDepTag -var _ android.CopyDirectlyInAnyApexTag = bootclasspathFragmentContentDepTag var _ android.RequiresFilesFromPrebuiltApexTag = bootclasspathFragmentContentDepTag func IsBootclasspathFragmentContentDepTag(tag blueprint.DependencyTag) bool { diff --git a/java/classpath_fragment.go b/java/classpath_fragment.go index fdccd3a84..88a8fb80a 100644 --- a/java/classpath_fragment.go +++ b/java/classpath_fragment.go @@ -105,6 +105,10 @@ func gatherPossibleApexModuleNamesAndStems(ctx android.ModuleContext, contents [ set := map[string]struct{}{} for _, name := range contents { dep := ctx.GetDirectDepWithTag(name, tag) + if dep == nil && ctx.Config().AllowMissingDependencies() { + // Ignore apex boot jars from dexpreopt if it does not exist, and missing deps are allowed. + continue + } set[ModuleStemForDeapexing(dep)] = struct{}{} if m, ok := dep.(ModuleWithStem); ok { set[m.Stem()] = struct{}{} diff --git a/java/dex.go b/java/dex.go index dea71f538..983377ead 100644 --- a/java/dex.go +++ b/java/dex.go @@ -295,7 +295,7 @@ func (d *dexer) d8Flags(ctx android.ModuleContext, dexParams *compileDexParams) return d8Flags, d8Deps, artProfileOutput } -func (d *dexer) r8Flags(ctx android.ModuleContext, dexParams *compileDexParams) (r8Flags []string, r8Deps android.Paths, artProfileOutput *android.OutputPath) { +func (d *dexer) r8Flags(ctx android.ModuleContext, dexParams *compileDexParams, debugMode bool) (r8Flags []string, r8Deps android.Paths, artProfileOutput *android.OutputPath) { flags := dexParams.flags opt := d.dexProperties.Optimize @@ -363,7 +363,9 @@ func (d *dexer) r8Flags(ctx android.ModuleContext, dexParams *compileDexParams) r8Flags = append(r8Flags, "--force-proguard-compatibility") } - if Bool(opt.Optimize) || Bool(opt.Obfuscate) { + // Avoid unnecessary stack frame noise by only injecting source map ids for non-debug + // optimized or obfuscated targets. + if (Bool(opt.Optimize) || Bool(opt.Obfuscate)) && !debugMode { // TODO(b/213833843): Allow configuration of the prefix via a build variable. var sourceFilePrefix = "go/retraceme " var sourceFileTemplate = "\"" + sourceFilePrefix + "%MAP_ID\"" @@ -483,7 +485,8 @@ func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParam proguardUsageZip, proguardConfiguration, } - r8Flags, r8Deps, r8ArtProfileOutputPath := d.r8Flags(ctx, dexParams) + debugMode := android.InList("--debug", commonFlags) + r8Flags, r8Deps, r8ArtProfileOutputPath := d.r8Flags(ctx, dexParams, debugMode) rule := r8 args := map[string]string{ "r8Flags": strings.Join(append(commonFlags, r8Flags...), " "), diff --git a/java/droiddoc.go b/java/droiddoc.go index 82713920d..2dda72b0e 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -578,7 +578,6 @@ func (j *Javadoc) GenerateAndroidBuildActions(ctx android.ModuleContext) { rule.Build("javadoc", "javadoc") - ctx.SetOutputFiles(android.Paths{j.stubsSrcJar}, "") ctx.SetOutputFiles(android.Paths{j.docZip}, ".docs.zip") } diff --git a/java/droidstubs.go b/java/droidstubs.go index cf3e21925..e955949af 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -769,15 +769,6 @@ func addMetalavaConfigFilesToCmd(cmd *android.RuleBuilderCommand, configFiles an // property is defined, apply transformations and only revert the flagged apis that are not // enabled via release configurations and are not specified in aconfig_declarations func generateRevertAnnotationArgs(ctx android.ModuleContext, cmd *android.RuleBuilderCommand, stubsType StubsType, aconfigFlagsPaths android.Paths) { - - if len(aconfigFlagsPaths) == 0 { - cmd.Flag("--revert-annotation android.annotation.FlaggedApi") - return - } - - releasedFlaggedApisFile := android.PathForModuleOut(ctx, fmt.Sprintf("released-flagged-apis-%s.txt", stubsType.String())) - revertAnnotationsFile := android.PathForModuleOut(ctx, fmt.Sprintf("revert-annotations-%s.txt", stubsType.String())) - var filterArgs string switch stubsType { // No flagged apis specific flags need to be passed to metalava when generating @@ -799,6 +790,15 @@ func generateRevertAnnotationArgs(ctx android.ModuleContext, cmd *android.RuleBu } } + if len(aconfigFlagsPaths) == 0 { + // This argument should not be added for "everything" stubs + cmd.Flag("--revert-annotation android.annotation.FlaggedApi") + return + } + + releasedFlaggedApisFile := android.PathForModuleOut(ctx, fmt.Sprintf("released-flagged-apis-%s.txt", stubsType.String())) + revertAnnotationsFile := android.PathForModuleOut(ctx, fmt.Sprintf("revert-annotations-%s.txt", stubsType.String())) + ctx.Build(pctx, android.BuildParams{ Rule: gatherReleasedFlaggedApisRule, Inputs: aconfigFlagsPaths, @@ -1011,7 +1011,7 @@ func (d *Droidstubs) everythingOptionalCmd(ctx android.ModuleContext, cmd *andro cmd.FlagWithOutput("--update-baseline:api-lint ", updatedBaselineOutput) msg += fmt.Sprintf(``+ - `3. FOR LSC ONLY: You can update the baseline by executing\n` + + `3. FOR LSC ONLY: You can update the baseline by executing\n`+ ` the following command:\n`+ ` (cd $ANDROID_BUILD_TOP && cp \\\n`+ ` "%s" \\\n`+ @@ -1374,7 +1374,7 @@ func (d *Droidstubs) setOutputFiles(ctx android.ModuleContext) { for _, stubType := range android.SortedKeys(stubsTypeToPrefix) { tagWithPrefix := stubsTypeToPrefix[stubType] + tag outputFile, err := tagToOutputFileFunc[tag](stubType) - if err == nil { + if err == nil && outputFile != nil { ctx.SetOutputFiles(android.Paths{outputFile}, tagWithPrefix) } } diff --git a/java/java.go b/java/java.go index 64ef782f5..260d33610 100644 --- a/java/java.go +++ b/java/java.go @@ -2311,14 +2311,17 @@ func (al *ApiLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { case libTag: if provider, ok := android.OtherModuleProvider(ctx, dep, JavaInfoProvider); ok { classPaths = append(classPaths, provider.HeaderJars...) + al.aconfigProtoFiles = append(al.aconfigProtoFiles, provider.AconfigIntermediateCacheOutputPaths...) } case bootClasspathTag: if provider, ok := android.OtherModuleProvider(ctx, dep, JavaInfoProvider); ok { bootclassPaths = append(bootclassPaths, provider.HeaderJars...) + al.aconfigProtoFiles = append(al.aconfigProtoFiles, provider.AconfigIntermediateCacheOutputPaths...) } case staticLibTag: if provider, ok := android.OtherModuleProvider(ctx, dep, JavaInfoProvider); ok { staticLibs = append(staticLibs, provider.HeaderJars...) + al.aconfigProtoFiles = append(al.aconfigProtoFiles, provider.AconfigIntermediateCacheOutputPaths...) } case systemModulesTag: if sm, ok := android.OtherModuleProvider(ctx, dep, SystemModulesProvider); ok { diff --git a/java/java_test.go b/java/java_test.go index 54eb3e14e..d415679bd 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -3050,6 +3050,7 @@ func TestCoverage(t *testing.T) { java_library { name: "android.car", srcs: ["android.car.java"], + installable: true, } `) diff --git a/java/rro.go b/java/rro.go index 8bb9be2eb..f225e1f4e 100644 --- a/java/rro.go +++ b/java/rro.go @@ -50,7 +50,7 @@ type RuntimeResourceOverlay struct { type RuntimeResourceOverlayProperties struct { // the name of a certificate in the default certificate directory or an android_app_certificate // module name in the form ":module". - Certificate *string + Certificate proptools.Configurable[string] `android:"replace_instead_of_append"` // Name of the signing certificate lineage file. Lineage *string @@ -119,7 +119,7 @@ func (r *RuntimeResourceOverlay) DepsMutator(ctx android.BottomUpMutatorContext) r.aapt.deps(ctx, sdkDep) } - cert := android.SrcIsModule(String(r.properties.Certificate)) + cert := android.SrcIsModule(r.properties.Certificate.GetOrDefault(ctx, "")) if cert != "" { ctx.AddDependency(ctx.Module(), certificateTag, cert) } @@ -166,7 +166,7 @@ func (r *RuntimeResourceOverlay) GenerateAndroidBuildActions(ctx android.ModuleC // Sign the built package _, _, certificates := collectAppDeps(ctx, r, false, false) - r.certificate, certificates = processMainCert(r.ModuleBase, String(r.properties.Certificate), certificates, ctx) + r.certificate, certificates = processMainCert(r.ModuleBase, r.properties.Certificate.GetOrDefault(ctx, ""), certificates, ctx) signed := android.PathForModuleOut(ctx, "signed", r.Name()+".apk") var lineageFile android.Path if lineage := String(r.properties.Lineage); lineage != "" { diff --git a/java/sdk_library.go b/java/sdk_library.go index f6dfcdd19..78917768b 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -1314,12 +1314,6 @@ var implLibraryTag = sdkLibraryComponentTag{name: "impl-library"} var _ android.InstallNeededDependencyTag = sdkLibraryComponentTag{} -// To satisfy the CopyDirectlyInAnyApexTag interface. Implementation library of the sdk library -// in an apex is considered to be directly in the apex, as if it was listed in java_libs. -func (t sdkLibraryComponentTag) CopyDirectlyInAnyApex() {} - -var _ android.CopyDirectlyInAnyApexTag = implLibraryTag - func (t sdkLibraryComponentTag) InstallDepNeeded() bool { return t.name == "xml-permissions-file" || t.name == "impl-library" } diff --git a/java/systemserver_classpath_fragment.go b/java/systemserver_classpath_fragment.go index 608a61628..3176ad94c 100644 --- a/java/systemserver_classpath_fragment.go +++ b/java/systemserver_classpath_fragment.go @@ -218,16 +218,11 @@ func (b systemServerClasspathFragmentContentDependencyTag) ExportMember() bool { return true } -// Contents of system server fragments in an apex are considered to be directly in the apex, as if -// they were listed in java_libs. -func (systemServerClasspathFragmentContentDependencyTag) CopyDirectlyInAnyApex() {} - // Contents of system server fragments require files from prebuilt apex files. func (systemServerClasspathFragmentContentDependencyTag) RequiresFilesFromPrebuiltApex() {} var _ android.ReplaceSourceWithPrebuilt = systemServerClasspathFragmentContentDepTag var _ android.SdkMemberDependencyTag = systemServerClasspathFragmentContentDepTag -var _ android.CopyDirectlyInAnyApexTag = systemServerClasspathFragmentContentDepTag var _ android.RequiresFilesFromPrebuiltApexTag = systemServerClasspathFragmentContentDepTag // The tag used for the dependency between the systemserverclasspath_fragment module and its contents. diff --git a/java/testing.go b/java/testing.go index 988514dea..0ea4e6408 100644 --- a/java/testing.go +++ b/java/testing.go @@ -190,6 +190,7 @@ var PrepareForTestWithJacocoInstrumentation = android.GroupFixturePreparers( "//apex_available:anyapex", "//apex_available:platform", ], + compile_dex: true, } `)), ) @@ -427,7 +428,7 @@ func gatherRequiredDepsForTest() string { "stub-annotations", "aconfig-annotations-lib", - "aconfig_storage_reader_java", + "aconfig_storage_stub", "unsupportedappusage", } diff --git a/rust/bindgen.go b/rust/bindgen.go index d59057994..898e7923b 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -300,6 +300,11 @@ func (b *bindgenDecorator) GenerateSource(ctx ModuleContext, deps PathDeps) andr // it cannot recognize. Turn off unknown warning flags warning. cflags = append(cflags, "-Wno-unknown-warning-option") + // Suppress warnings while testing a new compiler. + if ctx.Config().IsEnvTrue("LLVM_NEXT") { + cflags = append(cflags, "-Wno-everything") + } + outputFile := android.PathForModuleOut(ctx, b.BaseSourceProvider.getStem(ctx)+".rs") var cmd, cmdDesc string diff --git a/systemfeatures/Android.bp b/systemfeatures/Android.bp new file mode 100644 index 000000000..a65a6b644 --- /dev/null +++ b/systemfeatures/Android.bp @@ -0,0 +1,18 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-systemfeatures", + pkgPath: "android/soong/systemfeatures", + deps: [ + "blueprint", + "blueprint-proptools", + "soong", + "soong-android", + "soong-java", + ], + srcs: ["system_features.go"], + testSrcs: ["system_features_test.go"], + pluginFor: ["soong_build"], +} diff --git a/systemfeatures/OWNERS b/systemfeatures/OWNERS new file mode 100644 index 000000000..3e4480647 --- /dev/null +++ b/systemfeatures/OWNERS @@ -0,0 +1,2 @@ +jdduke@google.com +shayba@google.com diff --git a/systemfeatures/system_features.go b/systemfeatures/system_features.go new file mode 100644 index 000000000..0c1a56676 --- /dev/null +++ b/systemfeatures/system_features.go @@ -0,0 +1,102 @@ +// Copyright 2024 Google Inc. All rights reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package systemfeatures + +import ( + "fmt" + "sort" + "strings" + + "android/soong/android" + "android/soong/genrule" +) + +var ( + pctx = android.NewPackageContext("android/soong/systemfeatures") +) + +func init() { + registerSystemFeaturesComponents(android.InitRegistrationContext) +} + +func registerSystemFeaturesComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("java_system_features_srcs", JavaSystemFeaturesSrcsFactory) +} + +type javaSystemFeaturesSrcs struct { + android.ModuleBase + properties struct { + // The fully qualified class name for the generated code, e.g., com.android.Foo + Full_class_name string + } + outputFiles android.WritablePaths +} + +var _ genrule.SourceFileGenerator = (*javaSystemFeaturesSrcs)(nil) +var _ android.SourceFileProducer = (*javaSystemFeaturesSrcs)(nil) + +func (m *javaSystemFeaturesSrcs) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Create a file name appropriate for the given fully qualified (w/ package) class name. + classNameParts := strings.Split(m.properties.Full_class_name, ".") + outputDir := android.PathForModuleGen(ctx) + outputFileName := classNameParts[len(classNameParts)-1] + ".java" + outputFile := android.PathForModuleGen(ctx, outputFileName).OutputPath + + // Collect all RELEASE_SYSTEM_FEATURE_$K:$V build flags into a list of "$K:$V" pairs. + var features []string + for k, v := range ctx.Config().ProductVariables().BuildFlags { + if strings.HasPrefix(k, "RELEASE_SYSTEM_FEATURE_") { + shortFeatureName := strings.TrimPrefix(k, "RELEASE_SYSTEM_FEATURE_") + features = append(features, fmt.Sprintf("%s:%s", shortFeatureName, v)) + } + } + // Ensure sorted outputs for consistency of flag ordering in ninja outputs. + sort.Strings(features) + + rule := android.NewRuleBuilder(pctx, ctx) + rule.Command().Text("rm -rf").Text(outputDir.String()) + rule.Command().Text("mkdir -p").Text(outputDir.String()) + rule.Command(). + BuiltTool("systemfeatures-gen-tool"). + Flag(m.properties.Full_class_name). + FlagForEachArg("--feature=", features). + FlagWithArg("--readonly=", fmt.Sprint(ctx.Config().ReleaseUseSystemFeatureBuildFlags())). + FlagWithOutput(" > ", outputFile) + rule.Build(ctx.ModuleName(), "Generating systemfeatures srcs filegroup") + + m.outputFiles = append(m.outputFiles, outputFile) +} + +func (m *javaSystemFeaturesSrcs) Srcs() android.Paths { + return m.outputFiles.Paths() +} + +func (m *javaSystemFeaturesSrcs) GeneratedSourceFiles() android.Paths { + return m.outputFiles.Paths() +} + +func (m *javaSystemFeaturesSrcs) GeneratedDeps() android.Paths { + return m.outputFiles.Paths() +} + +func (m *javaSystemFeaturesSrcs) GeneratedHeaderDirs() android.Paths { + return nil +} + +func JavaSystemFeaturesSrcsFactory() android.Module { + module := &javaSystemFeaturesSrcs{} + module.AddProperties(&module.properties) + android.InitAndroidModule(module) + return module +} diff --git a/systemfeatures/system_features_test.go b/systemfeatures/system_features_test.go new file mode 100644 index 000000000..558bb958e --- /dev/null +++ b/systemfeatures/system_features_test.go @@ -0,0 +1,51 @@ +// Copyright 2024 Google Inc. All rights reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package systemfeatures + +import ( + "android/soong/android" + + "testing" +) + +func TestJavaSystemFeaturesSrcs(t *testing.T) { + bp := ` +java_system_features_srcs { + name: "system-features-srcs", + full_class_name: "com.android.test.RoSystemFeatures", +} +` + + res := android.GroupFixturePreparers( + android.FixtureRegisterWithContext(registerSystemFeaturesComponents), + android.PrepareForTestWithBuildFlag("RELEASE_USE_SYSTEM_FEATURE_BUILD_FLAGS", "true"), + android.PrepareForTestWithBuildFlag("RELEASE_SYSTEM_FEATURE_AUTOMOTIVE", "0"), + android.PrepareForTestWithBuildFlag("RELEASE_SYSTEM_FEATURE_TELEVISION", "UNAVAILABLE"), + android.PrepareForTestWithBuildFlag("RELEASE_SYSTEM_FEATURE_WATCH", ""), + android.PrepareForTestWithBuildFlag("RELEASE_NOT_SYSTEM_FEATURE_FOO", "BAR"), + ).RunTestWithBp(t, bp) + + module := res.ModuleForTests("system-features-srcs", "") + cmd := module.Rule("system-features-srcs").RuleParams.Command + android.AssertStringDoesContain(t, "Expected fully class name", cmd, " com.android.test.RoSystemFeatures ") + android.AssertStringDoesContain(t, "Expected readonly flag", cmd, "--readonly=true") + android.AssertStringDoesContain(t, "Expected AUTOMOTIVE feature flag", cmd, "--feature=AUTOMOTIVE:0 ") + android.AssertStringDoesContain(t, "Expected TELEVISION feature flag", cmd, "--feature=TELEVISION:UNAVAILABLE ") + android.AssertStringDoesContain(t, "Expected WATCH feature flag", cmd, "--feature=WATCH: ") + android.AssertStringDoesNotContain(t, "Unexpected FOO arg from non-system feature flag", cmd, "FOO") + + systemFeaturesModule := module.Module().(*javaSystemFeaturesSrcs) + expectedOutputPath := "out/soong/.intermediates/system-features-srcs/gen/RoSystemFeatures.java" + android.AssertPathsRelativeToTopEquals(t, "Expected output file", []string{expectedOutputPath}, systemFeaturesModule.Srcs()) +} diff --git a/ui/build/soong.go b/ui/build/soong.go index e6d01dd1d..0963f76b7 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -433,13 +433,13 @@ func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string) } } -func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error { +func updateSymlinks(ctx Context, dir, prevCWD, cwd string, updateSemaphore chan struct{}) error { defer symlinkWg.Done() visit := func(path string, d fs.DirEntry, err error) error { if d.IsDir() && path != dir { symlinkWg.Add(1) - go updateSymlinks(ctx, path, prevCWD, cwd) + go updateSymlinks(ctx, path, prevCWD, cwd, updateSemaphore) return filepath.SkipDir } f, err := d.Info() @@ -470,12 +470,27 @@ func updateSymlinks(ctx Context, dir, prevCWD, cwd string) error { return nil } + <-updateSemaphore + defer func() { updateSemaphore <- struct{}{} }() if err := filepath.WalkDir(dir, visit); err != nil { return err } return nil } +// b/376466642: If the concurrency of updateSymlinks is unbounded, Go's runtime spawns a +// theoretically unbounded number of threads to handle blocking syscalls. This causes the runtime to +// panic due to hitting thread limits in rare cases. Limiting to GOMAXPROCS concurrent symlink +// updates should make this a non-issue. +func newUpdateSemaphore() chan struct{} { + numPermits := runtime.GOMAXPROCS(0) + c := make(chan struct{}, numPermits) + for i := 0; i < numPermits; i++ { + c <- struct{}{} + } + return c +} + func fixOutDirSymlinks(ctx Context, config Config, outDir string) error { cwd, err := os.Getwd() if err != nil { @@ -508,7 +523,7 @@ func fixOutDirSymlinks(ctx Context, config Config, outDir string) error { } symlinkWg.Add(1) - if err := updateSymlinks(ctx, outDir, prevCWD, cwd); err != nil { + if err := updateSymlinks(ctx, outDir, prevCWD, cwd, newUpdateSemaphore()); err != nil { return err } symlinkWg.Wait() diff --git a/ui/metrics/Android.bp b/ui/metrics/Android.bp index 77871fc09..591e3cca6 100644 --- a/ui/metrics/Android.bp +++ b/ui/metrics/Android.bp @@ -26,6 +26,7 @@ bootstrap_go_package { "soong-ui-metrics_proto", "soong-ui-mk_metrics_proto", "soong-shared", + "soong-ui-metrics_combined_proto", ], srcs: [ "hostinfo.go", @@ -63,6 +64,19 @@ bootstrap_go_package { } bootstrap_go_package { + name: "soong-ui-metrics_combined_proto", + pkgPath: "android/soong/ui/metrics/combined_metrics_proto", + deps: [ + "golang-protobuf-reflect-protoreflect", + "golang-protobuf-runtime-protoimpl", + "soong-cmd-find_input_delta-proto", + ], + srcs: [ + "metrics_proto/metrics.pb.go", + ], +} + +bootstrap_go_package { name: "soong-ui-metrics_upload_proto", pkgPath: "android/soong/ui/metrics/upload_proto", deps: [ diff --git a/ui/metrics/metrics_proto/combined_metrics.pb.go b/ui/metrics/metrics_proto/combined_metrics.pb.go new file mode 100644 index 000000000..f49d64d54 --- /dev/null +++ b/ui/metrics/metrics_proto/combined_metrics.pb.go @@ -0,0 +1,239 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.21.12 +// source: combined_metrics.proto + +package metrics_proto + +import ( + find_input_delta_proto "android/soong/cmd/find_input_delta/find_input_delta_proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// These field numbers are also found in the inner message declarations. +// We verify that the values are the same, and that every enum value is checked +// in combined_metrics_test.go. +// Do not change this enum without also updating: +// - the submessage's .proto file +// - combined_metrics_test.go +type FieldNumbers int32 + +const ( + FieldNumbers_FIELD_NUMBERS_UNSPECIFIED FieldNumbers = 0 + FieldNumbers_FIELD_NUMBERS_FILE_LIST FieldNumbers = 1 +) + +// Enum value maps for FieldNumbers. +var ( + FieldNumbers_name = map[int32]string{ + 0: "FIELD_NUMBERS_UNSPECIFIED", + 1: "FIELD_NUMBERS_FILE_LIST", + } + FieldNumbers_value = map[string]int32{ + "FIELD_NUMBERS_UNSPECIFIED": 0, + "FIELD_NUMBERS_FILE_LIST": 1, + } +) + +func (x FieldNumbers) Enum() *FieldNumbers { + p := new(FieldNumbers) + *p = x + return p +} + +func (x FieldNumbers) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FieldNumbers) Descriptor() protoreflect.EnumDescriptor { + return file_combined_metrics_proto_enumTypes[0].Descriptor() +} + +func (FieldNumbers) Type() protoreflect.EnumType { + return &file_combined_metrics_proto_enumTypes[0] +} + +func (x FieldNumbers) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *FieldNumbers) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = FieldNumbers(num) + return nil +} + +// Deprecated: Use FieldNumbers.Descriptor instead. +func (FieldNumbers) EnumDescriptor() ([]byte, []int) { + return file_combined_metrics_proto_rawDescGZIP(), []int{0} +} + +type SoongCombinedMetrics struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // cmd/find_input_delta/find_input_delta_proto.FileList + FileList *find_input_delta_proto.FileList `protobuf:"bytes,1,opt,name=file_list,json=fileList" json:"file_list,omitempty"` +} + +func (x *SoongCombinedMetrics) Reset() { + *x = SoongCombinedMetrics{} + if protoimpl.UnsafeEnabled { + mi := &file_combined_metrics_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SoongCombinedMetrics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SoongCombinedMetrics) ProtoMessage() {} + +func (x *SoongCombinedMetrics) ProtoReflect() protoreflect.Message { + mi := &file_combined_metrics_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SoongCombinedMetrics.ProtoReflect.Descriptor instead. +func (*SoongCombinedMetrics) Descriptor() ([]byte, []int) { + return file_combined_metrics_proto_rawDescGZIP(), []int{0} +} + +func (x *SoongCombinedMetrics) GetFileList() *find_input_delta_proto.FileList { + if x != nil { + return x.FileList + } + return nil +} + +var File_combined_metrics_proto protoreflect.FileDescriptor + +var file_combined_metrics_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x13, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, 0x3b, 0x63, + 0x6d, 0x64, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, + 0x6c, 0x74, 0x61, 0x2f, 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, + 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x5d, 0x0a, 0x14, 0x53, 0x6f, + 0x6f, 0x6e, 0x67, 0x43, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x12, 0x45, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2e, + 0x66, 0x69, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, + 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, + 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x2a, 0x4a, 0x0a, 0x0c, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x49, 0x45, + 0x4c, 0x44, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, + 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x46, 0x49, 0x45, 0x4c, + 0x44, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x53, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4c, + 0x49, 0x53, 0x54, 0x10, 0x01, 0x42, 0x28, 0x5a, 0x26, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, + 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, +} + +var ( + file_combined_metrics_proto_rawDescOnce sync.Once + file_combined_metrics_proto_rawDescData = file_combined_metrics_proto_rawDesc +) + +func file_combined_metrics_proto_rawDescGZIP() []byte { + file_combined_metrics_proto_rawDescOnce.Do(func() { + file_combined_metrics_proto_rawDescData = protoimpl.X.CompressGZIP(file_combined_metrics_proto_rawDescData) + }) + return file_combined_metrics_proto_rawDescData +} + +var file_combined_metrics_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_combined_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_combined_metrics_proto_goTypes = []interface{}{ + (FieldNumbers)(0), // 0: soong_build_metrics.FieldNumbers + (*SoongCombinedMetrics)(nil), // 1: soong_build_metrics.SoongCombinedMetrics + (*find_input_delta_proto.FileList)(nil), // 2: android.find_input_delta_proto.FileList +} +var file_combined_metrics_proto_depIdxs = []int32{ + 2, // 0: soong_build_metrics.SoongCombinedMetrics.file_list:type_name -> android.find_input_delta_proto.FileList + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_combined_metrics_proto_init() } +func file_combined_metrics_proto_init() { + if File_combined_metrics_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_combined_metrics_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SoongCombinedMetrics); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_combined_metrics_proto_rawDesc, + NumEnums: 1, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_combined_metrics_proto_goTypes, + DependencyIndexes: file_combined_metrics_proto_depIdxs, + EnumInfos: file_combined_metrics_proto_enumTypes, + MessageInfos: file_combined_metrics_proto_msgTypes, + }.Build() + File_combined_metrics_proto = out.File + file_combined_metrics_proto_rawDesc = nil + file_combined_metrics_proto_goTypes = nil + file_combined_metrics_proto_depIdxs = nil +} diff --git a/ui/metrics/metrics_proto/combined_metrics.proto b/ui/metrics/metrics_proto/combined_metrics.proto new file mode 100644 index 000000000..3cd9a5313 --- /dev/null +++ b/ui/metrics/metrics_proto/combined_metrics.proto @@ -0,0 +1,36 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package soong_build_metrics; +option go_package = "android/soong/ui/metrics/metrics_proto"; + +import "cmd/find_input_delta/find_input_delta_proto/file_list.proto"; + +// These field numbers are also found in the inner message declarations. +// We verify that the values are the same, and that every enum value is checked +// in combined_metrics_test.go. +// Do not change this enum without also updating: +// - the submessage's .proto file +// - combined_metrics_test.go +enum FieldNumbers { + FIELD_NUMBERS_UNSPECIFIED = 0; + FIELD_NUMBERS_FILE_LIST = 1; +} + +message SoongCombinedMetrics { + // cmd/find_input_delta/find_input_delta_proto.FileList + optional android.find_input_delta_proto.FileList file_list = 1; +} diff --git a/ui/metrics/metrics_proto/combined_metrics_test.go b/ui/metrics/metrics_proto/combined_metrics_test.go new file mode 100644 index 000000000..eedb12a34 --- /dev/null +++ b/ui/metrics/metrics_proto/combined_metrics_test.go @@ -0,0 +1,33 @@ +package metrics_proto + +import ( + "testing" + + find_input_delta_proto "android/soong/cmd/find_input_delta/find_input_delta_proto" +) + +func TestCombinedMetricsMessageNums(t *testing.T) { + testCases := []struct { + Name string + FieldNumbers map[string]int32 + }{ + { + Name: "find_input_delta_proto", + FieldNumbers: find_input_delta_proto.FieldNumbers_value, + }, + } + verifiedMap := make(map[string]bool) + for _, tc := range testCases { + for k, v := range tc.FieldNumbers { + if FieldNumbers_value[k] != v { + t.Errorf("%s: Expected FieldNumbers.%s == %v, found %v", tc.Name, k, FieldNumbers_value[k], v) + } + verifiedMap[k] = true + } + } + for k, v := range FieldNumbers_value { + if !verifiedMap[k] { + t.Errorf("No test case verifies FieldNumbers.%s=%v", k, v) + } + } +} diff --git a/ui/metrics/metrics_proto/regen.sh b/ui/metrics/metrics_proto/regen.sh index 8eb2d747b..5e5f9b83f 100755 --- a/ui/metrics/metrics_proto/regen.sh +++ b/ui/metrics/metrics_proto/regen.sh @@ -12,6 +12,6 @@ if ! hash aprotoc &>/dev/null; then die "could not find aprotoc. ${error_msg}" fi -if ! aprotoc --go_out=paths=source_relative:. metrics.proto; then +if ! aprotoc --go_out=paths=source_relative:. -I .:../../.. metrics.proto combined_metrics.proto; then die "build failed. ${error_msg}" fi diff --git a/ui/status/log.go b/ui/status/log.go index 14df346de..7bfd39641 100644 --- a/ui/status/log.go +++ b/ui/status/log.go @@ -22,6 +22,8 @@ import ( "io/ioutil" "os" "strings" + "sync" + "time" "google.golang.org/protobuf/proto" @@ -31,7 +33,10 @@ import ( ) type verboseLog struct { - w io.WriteCloser + w *gzip.Writer + lock *sync.Mutex + data chan []string + stop chan bool } func NewVerboseLog(log logger.Logger, filename string) StatusOutput { @@ -47,9 +52,42 @@ func NewVerboseLog(log logger.Logger, filename string) StatusOutput { w := gzip.NewWriter(f) - return &verboseLog{ - w: w, + l := &verboseLog{ + w: w, + lock: &sync.Mutex{}, + data: make(chan []string), + stop: make(chan bool), } + l.startWriter() + return l +} + +func (v *verboseLog) startWriter() { + go func() { + tick := time.Tick(time.Second) + for { + select { + case <-v.stop: + close(v.data) + v.w.Close() + return + case <-tick: + v.w.Flush() + case dataList := <-v.data: + for _, data := range dataList { + fmt.Fprint(v.w, data) + } + } + } + }() +} + +func (v *verboseLog) stopWriter() { + v.stop <- true +} + +func (v *verboseLog) queueWrite(s ...string) { + v.data <- s } func (v *verboseLog) StartAction(action *Action, counts Counts) {} @@ -60,27 +98,27 @@ func (v *verboseLog) FinishAction(result ActionResult, counts Counts) { cmd = result.Description } - fmt.Fprintf(v.w, "[%d/%d] %s\n", counts.FinishedActions, counts.TotalActions, cmd) + v.queueWrite(fmt.Sprintf("[%d/%d] ", counts.FinishedActions, counts.TotalActions), cmd, "\n") if result.Error != nil { - fmt.Fprintf(v.w, "FAILED: %s\n", strings.Join(result.Outputs, " ")) + v.queueWrite("FAILED: ", strings.Join(result.Outputs, " "), "\n") } if result.Output != "" { - fmt.Fprintln(v.w, result.Output) + v.queueWrite(result.Output, "\n") } } func (v *verboseLog) Flush() { - v.w.Close() + v.stopWriter() } func (v *verboseLog) Message(level MsgLevel, message string) { - fmt.Fprintf(v.w, "%s%s\n", level.Prefix(), message) + v.queueWrite(level.Prefix(), message, "\n") } func (v *verboseLog) Write(p []byte) (int, error) { - fmt.Fprint(v.w, string(p)) + v.queueWrite(string(p)) return len(p), nil } |